diff --git a/.agents/mcps b/.agents/mcps index a34c9b720..0fcca530b 160000 --- a/.agents/mcps +++ b/.agents/mcps @@ -1 +1 @@ -Subproject commit a34c9b7203df6b4ff78e9691da9e060f72228830 +Subproject commit 0fcca530b2c3f012aaa5e9589755e3bb0dd1206f diff --git a/.agents/skills b/.agents/skills index bc1b7ef14..565442ef7 160000 --- a/.agents/skills +++ b/.agents/skills @@ -1 +1 @@ -Subproject commit bc1b7ef1446636b0e55cd0bef4c00bfee20d1160 +Subproject commit 565442ef74e2a0e0ba2802e6f120f07b293c8520 diff --git a/.github/scripts/ci_repo_lock.sh b/.github/scripts/ci_repo_lock.sh index 5b47fc888..9f265534d 100755 --- a/.github/scripts/ci_repo_lock.sh +++ b/.github/scripts/ci_repo_lock.sh @@ -25,26 +25,31 @@ repo_reset() { cd "${repo}" # Initialize the compiler in stack and its LLVM dependency explicitly. - git fetch --force --prune origin "${want_sha}" + git fetch --no-recurse-submodules --force --prune origin "${want_sha}" git checkout --detach --force "${want_sha}" git reset --hard "${want_sha}" git clean -ffd rm -rf arch/out - rm -rf stack/compiler/thirdparty/buddy-mlir/llvm/build - rm -rf stack/compiler/thirdparty/buddy-mlir/build + # Generated outputs (CMake caches, RTL, workloads) are ignored by git clean + # and keep stale source paths; every run starts from a fresh output tree. + rm -rf examples/designs/*/output + rm -rf stack/thirdparty/buddy-mlir/llvm/build + rm -rf stack/thirdparty/buddy-mlir/build git submodule sync git submodule update --init --force # submodules in submodules we used need update by hands - git -C arch/thirdparty/rocket-chip submodule sync -- dependencies/cde - git -C arch/thirdparty/rocket-chip submodule update --init --force -- dependencies/cde + git -C examples/prefab/cores/rocket/thirdparty/rocket-chip submodule sync -- dependencies/cde + git -C examples/prefab/cores/rocket/thirdparty/rocket-chip submodule update --init --force -- dependencies/cde git -C stack submodule sync - git -C stack submodule update --init --force -- compiler/thirdparty/buddy-mlir - git -C stack/compiler/thirdparty/buddy-mlir submodule sync - git -C stack/compiler/thirdparty/buddy-mlir submodule update --init --force -- llvm + git -C stack submodule update --init --force -- os/linux thirdparty/buddy-mlir + git -C stack submodule update --init --depth 1 --single-branch --recommend-shallow -- \ + os/thirdparty/linux os/thirdparty/busybox os/thirdparty/opensbi + git -C stack/thirdparty/buddy-mlir submodule sync + git -C stack/thirdparty/buddy-mlir submodule update --init --force -- llvm git submodule foreach 'git clean -ffd' have_sha="$(git rev-parse HEAD)" diff --git a/.github/scripts/setproxy.sh b/.github/scripts/setproxy.sh index f9d76cbae..f6e6077db 100644 --- a/.github/scripts/setproxy.sh +++ b/.github/scripts/setproxy.sh @@ -1 +1,7 @@ -export ALL_PROXY=http://127.0.0.1:7890 +export http_proxy=http://127.0.0.1:7890 +export https_proxy="$http_proxy" +export HTTP_PROXY="$http_proxy" +export HTTPS_PROXY="$https_proxy" +export ALL_PROXY="$http_proxy" +export no_proxy=localhost,127.0.0.1,::1 +export NO_PROXY="$no_proxy" diff --git a/.github/workflows/README.zh.md b/.github/workflows/README.zh.md index 314dee3e5..a857fe524 100644 --- a/.github/workflows/README.zh.md +++ b/.github/workflows/README.zh.md @@ -3,7 +3,7 @@ push/PR 到 `main` 会自动触发 `check.yaml`: 1. **pre-commit**:拉代码, 并校验格式 -2. **chip-check**:通过后,matrix 并行跑各 chip(共享同一个 `~/Code/buckyball` 环境,基于 `ci_repo_lock.sh` 协调) +2. **design-check**:通过后,matrix 并行跑各 design(共享同一个 `~/Code/buckyball` 环境,基于 `ci_repo_lock.sh` 协调) `regression.yml` 是 reviewer 的可选测试。注意:Environment `regression` 必须在仓库 Settings → Environments 里打开 Required reviewers,否则会直接跑。 也可以在 Actions → regression → Run workflow 中选择任意分支手动启动,测试触发时该分支的最新 commit。 diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml index 23eb7b9a7..bf5b551fc 100644 --- a/.github/workflows/check.yaml +++ b/.github/workflows/check.yaml @@ -46,7 +46,7 @@ jobs: cd "${root}" # fetch first to ensure if we modify ci_repo_lock.sh, we still use the old version to checkout. - git fetch --force --prune origin "${CHECKOUT_SHA}" + git fetch --no-recurse-submodules --force --prune origin "${CHECKOUT_SHA}" git show "${CHECKOUT_SHA}:.github/scripts/ci_repo_lock.sh" | bash -s -- \ prepare "${CHECKOUT_SHA}" "${root}" "${CHECKOUT_REF}" @@ -59,6 +59,7 @@ jobs: - name: Check pre-commit shell: zsh {0} run: | + set -euo pipefail root=~/Code/buckyball source "${root}/.github/scripts/setproxy.sh" cd "${root}" @@ -72,27 +73,32 @@ jobs: cd "${root}" nix develop -c bbdev config --install - chip-check: - name: ${{ matrix.chip }} + design-check: + name: ${{ matrix.design }} needs: pre-commit runs-on: check strategy: fail-fast: false matrix: include: - - chip: toy + - design: toy + config: ToyVerilatorConfig run_bemu_tests: true run_verilator_tests: true - - chip: pebble + - design: pebble + config: PebbleVerilatorConfig run_bemu_tests: true run_verilator_tests: true - - chip: goban + - design: goban + config: GobanVerilatorConfig run_bemu_tests: true run_verilator_tests: true - - chip: poly + - design: poly + config: PolyVerilatorConfig run_bemu_tests: true run_verilator_tests: true - - chip: multi-rocket + - design: multi-rocket + config: MultiRocketVerilatorConfig run_bemu_tests: true run_verilator_tests: true env: @@ -100,7 +106,7 @@ jobs: steps: - name: Print information run: | - echo "chip=${{ matrix.chip }}" + echo "design=${{ matrix.design }}" echo "event=${{ github.event_name }}" echo "sha=${CHECKOUT_SHA}" echo "ref=${CHECKOUT_REF}" @@ -112,7 +118,7 @@ jobs: root="${BB_ROOT/#\~/$HOME}" source "${root}/.github/scripts/setproxy.sh" cd "${root}" - git fetch --force --prune origin "${CHECKOUT_SHA}" + git fetch --no-recurse-submodules --force --prune origin "${CHECKOUT_SHA}" git show "${CHECKOUT_SHA}:.github/scripts/ci_repo_lock.sh" | bash -s -- \ enter "${CHECKOUT_SHA}" "${root}" "${CHECKOUT_REF}" @@ -130,9 +136,9 @@ jobs: set -euo pipefail root="${BB_ROOT/#\~/$HOME}" cd "${root}" - nix develop -c bbdev compiler --build '--chip ${{ matrix.chip }}' - nix develop -c bbdev workload --clean '--chip ${{ matrix.chip }}' - nix develop -c bbdev workload --build '--chip ${{ matrix.chip }}' + nix develop -c bbdev stack --build '--design ${{ matrix.design }} --device ${{ matrix.config }}' + nix develop -c bbdev workload --clean '--design ${{ matrix.design }} --device ${{ matrix.config }}' + nix develop -c bbdev workload --build '--design ${{ matrix.design }} --device ${{ matrix.config }}' - name: bemu tests if: matrix.run_bemu_tests @@ -141,9 +147,11 @@ jobs: set -euo pipefail root="${BB_ROOT/#\~/$HOME}" cd "${root}" - nix develop -c bbdev bebop-bemu --batch '--chip ${{ matrix.chip }} --test bare-tests --jobs 16 --clean-before' - nix develop -c bbdev kernel --build '--chip ${{ matrix.chip }}' - nix develop -c bbdev bebop-bemu --batch '--chip ${{ matrix.chip }} --test linux-tests --jobs 16' + # Other design jobs share this checkout; drop only this config's old batch runs. + rm -rf "${root}"/log/*-${{ matrix.design }}-bebop-bemu-device-${{ matrix.config }}-*(N) + nix develop -c bbdev bebop-bemu --batch "--design ${{ matrix.design }} --device ${{ matrix.config }} --test bare-tests --jobs 16" + nix develop -c bbdev kernel --build '--design ${{ matrix.design }} --device ${{ matrix.config }}' + nix develop -c bbdev bebop-bemu --batch "--design ${{ matrix.design }} --device ${{ matrix.config }} --test linux-tests --jobs 16" - name: verilator tests if: matrix.run_verilator_tests @@ -152,44 +160,33 @@ jobs: set -euo pipefail root="${BB_ROOT/#\~/$HOME}" cd "${root}" - nix develop -c bbdev bebop-verilator --clean '--chip ${{ matrix.chip }}' - nix develop -c bbdev bebop-verilator --verilog '--chip ${{ matrix.chip }} --diff' - nix develop -c bbdev bebop-verilator --build '--jobs 100 --chip ${{ matrix.chip }} --diff' - nix develop -c bbdev bebop-verilator --batch '--chip ${{ matrix.chip }} --test bare-tests --diff --jobs 16 --clean-before' - - - name: Detect bebop JUnit files - if: always() && (matrix.run_bemu_tests || matrix.run_verilator_tests) - id: junit - shell: zsh {0} - run: | - root="${BB_ROOT/#\~/$HOME}" - files=("${root}/bebop/target/${{ matrix.chip }}/nextest/junit-bbdev-"*.xml(N)) - if (( ${#files} == 0 )); then - echo "found=false" >> "$GITHUB_OUTPUT" - else - echo "found=true" >> "$GITHUB_OUTPUT" - fi + rm -rf "${root}"/log/*-${{ matrix.design }}-bebop-verilator-device-${{ matrix.config }}-*(N) + nix develop -c bbdev bebop-verilator --clean '--design ${{ matrix.design }} --device ${{ matrix.config }}' + nix develop -c bbdev bebop-verilator --verilog '--design ${{ matrix.design }} --device ${{ matrix.config }} --diff' + nix develop -c bbdev bebop-verilator --build '--jobs 100 --design ${{ matrix.design }} --device ${{ matrix.config }} --diff' + nix develop -c bbdev bebop-verilator --batch "--design ${{ matrix.design }} --device ${{ matrix.config }} --test bare-tests --diff --jobs 16" - name: Publish bebop test report - if: always() && (matrix.run_bemu_tests || matrix.run_verilator_tests) && steps.junit.outputs.found == 'true' + if: always() && (matrix.run_bemu_tests || matrix.run_verilator_tests) uses: mikepenz/action-junit-report@v4 with: - report_paths: "~/Code/buckyball/bebop/target/${{ matrix.chip }}/nextest/junit-bbdev-*.xml" - check_name: "bebop nextest (${{ matrix.chip }})" + report_paths: "${{ env.BB_ROOT }}/log/*-${{ matrix.design }}-bebop-*-device-${{ matrix.config }}-*/junit.xml" + check_name: "bebop regression (${{ matrix.design }})" detailed_summary: true include_passed: true - name: Upload bebop JUnit artifacts - if: always() && (matrix.run_bemu_tests || matrix.run_verilator_tests) && steps.junit.outputs.found == 'true' + if: always() && (matrix.run_bemu_tests || matrix.run_verilator_tests) uses: actions/upload-artifact@v4 + # The runner host has no DNS of its own; uploads go through the proxy. env: - ALL_PROXY: "" - HTTP_PROXY: "" - HTTPS_PROXY: "" - NO_PROXY: github.com,api.github.com,uploads.github.com,objects.githubusercontent.com,.blob.core.windows.net + HTTP_PROXY: http://127.0.0.1:7890 + HTTPS_PROXY: http://127.0.0.1:7890 + NO_PROXY: localhost,127.0.0.1,::1 with: - name: bebop-nextest-junit-${{ matrix.chip }} - path: "~/Code/buckyball/bebop/target/${{ matrix.chip }}/nextest/junit-bbdev-*.xml" + name: bebop-junit-${{ matrix.design }} + path: "${{ env.BB_ROOT }}/log/*-${{ matrix.design }}-bebop-*-device-${{ matrix.config }}-*/junit.xml" + if-no-files-found: error - name: Exit repo lock if: always() diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 08d94738d..a4d1aea9e 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -45,7 +45,7 @@ jobs: root="${BB_ROOT/#\~/$HOME}" source "${root}/.github/scripts/setproxy.sh" cd "${root}" - git fetch --force --prune origin "${CHECKOUT_SHA}" + git fetch --no-recurse-submodules --force --prune origin "${CHECKOUT_SHA}" git show "${CHECKOUT_SHA}:.github/scripts/ci_repo_lock.sh" | bash -s -- \ prepare "${CHECKOUT_SHA}" "${root}" "${CHECKOUT_REF}" nix build @@ -63,26 +63,31 @@ jobs: nix develop -c bbdev config --install eval-area-power: - name: eval-area-power-${{ matrix.chip }} + name: eval-area-power-${{ matrix.design }} needs: gate runs-on: check strategy: fail-fast: false matrix: include: - - chip: toy + - design: toy + config: ToyVerilatorConfig run_area: true run_power: false - - chip: pebble + - design: pebble + config: PebbleVerilatorConfig run_area: true run_power: false - - chip: goban + - design: goban + config: GobanVerilatorConfig run_area: true run_power: false - - chip: poly + - design: poly + config: PolyTapeoutConfig run_area: true run_power: false - - chip: multi-rocket + - design: multi-rocket + config: MultiRocketVerilatorConfig run_area: true run_power: false env: @@ -97,7 +102,7 @@ jobs: root="${BB_ROOT/#\~/$HOME}" source "${root}/.github/scripts/setproxy.sh" cd "${root}" - git fetch --force --prune origin "${CHECKOUT_SHA}" + git fetch --no-recurse-submodules --force --prune origin "${CHECKOUT_SHA}" git show "${CHECKOUT_SHA}:.github/scripts/ci_repo_lock.sh" | bash -s -- \ enter "${CHECKOUT_SHA}" "${root}" "${CHECKOUT_REF}" @@ -111,7 +116,7 @@ jobs: set +u source ~/.zshrc set -u - nix develop -c bbdev dc --area '--chip ${{ matrix.chip }}' + nix develop -c bbdev dc --area '--design ${{ matrix.design }} --device ${{ matrix.config }}' - name: DC power if: matrix.run_power @@ -123,7 +128,7 @@ jobs: set +u source ~/.zshrc set -u - nix develop -c bbdev dc --power '--chip ${{ matrix.chip }}' + nix develop -c bbdev dc --power '--design ${{ matrix.design }} --device ${{ matrix.config }}' - name: Exit repo lock if: always() && (matrix.run_area || matrix.run_power) @@ -135,22 +140,27 @@ jobs: exit "${CHECKOUT_SHA}" "${root}" "${CHECKOUT_REF}" verification: - name: verification-${{ matrix.chip }} + name: verification-${{ matrix.design }} needs: gate runs-on: check strategy: fail-fast: false matrix: include: - - chip: toy + - design: toy + config: ToyVerilatorConfig run_uvm: true - - chip: pebble + - design: pebble + config: PebbleVerilatorConfig run_uvm: true - - chip: goban + - design: goban + config: GobanVerilatorConfig run_uvm: true - - chip: poly + - design: poly + config: PolyVerilatorConfig run_uvm: true - - chip: multi-rocket + - design: multi-rocket + config: MultiRocketVerilatorConfig run_uvm: true env: BB_ROOT: ~/Code/buckyball @@ -164,11 +174,11 @@ jobs: root="${BB_ROOT/#\~/$HOME}" source "${root}/.github/scripts/setproxy.sh" cd "${root}" - git fetch --force --prune origin "${CHECKOUT_SHA}" + git fetch --no-recurse-submodules --force --prune origin "${CHECKOUT_SHA}" git show "${CHECKOUT_SHA}:.github/scripts/ci_repo_lock.sh" | bash -s -- \ enter "${CHECKOUT_SHA}" "${root}" "${CHECKOUT_REF}" - - name: Chip UVM + - name: Design UVM if: matrix.run_uvm shell: zsh {0} run: | @@ -178,8 +188,8 @@ jobs: set +u source ~/.zshrc set -u - nix develop -c bbdev bebop-verilator --verilog '--chip ${{ matrix.chip }}' - nix develop -c bbdev uvm --run '--chip ${{ matrix.chip }}' + nix develop -c bbdev bebop-verilator --verilog '--design ${{ matrix.design }} --device ${{ matrix.config }}' + nix develop -c bbdev uvm --run '--design ${{ matrix.design }} --device ${{ matrix.config }}' - name: Exit repo lock if: always() && matrix.run_uvm @@ -191,7 +201,7 @@ jobs: exit "${CHECKOUT_SHA}" "${root}" "${CHECKOUT_REF}" fpga-bitstream: - name: fpga-bitstream-${{ matrix.chip }} + name: fpga-bitstream-${{ matrix.design }} needs: gate runs-on: check timeout-minutes: 720 @@ -199,15 +209,20 @@ jobs: fail-fast: false matrix: include: - - chip: toy + - design: toy + config: ToyVerilatorConfig build_bitstream: true - - chip: pebble + - design: pebble + config: PebbleVerilatorConfig build_bitstream: true - - chip: goban + - design: goban + config: GobanVerilatorConfig build_bitstream: true - - chip: poly + - design: poly + config: PolyP2EConfig build_bitstream: true - - chip: multi-rocket + - design: multi-rocket + config: MultiRocketVerilatorConfig build_bitstream: true env: BB_ROOT: ~/Code/buckyball @@ -220,7 +235,7 @@ jobs: root="${BB_ROOT/#\~/$HOME}" source "${root}/.github/scripts/setproxy.sh" cd "${root}" - git fetch --force --prune origin "${CHECKOUT_SHA}" + git fetch --no-recurse-submodules --force --prune origin "${CHECKOUT_SHA}" git show "${CHECKOUT_SHA}:.github/scripts/ci_repo_lock.sh" | bash -s -- \ enter "${CHECKOUT_SHA}" "${root}" "${CHECKOUT_REF}" @@ -231,14 +246,14 @@ jobs: set -euo pipefail root="${BB_ROOT/#\~/$HOME}" cd "${root}" - for build_dir in "${root}"/bebop/build/"${{ matrix.chip }}"-*(N); do + for build_dir in "${root}"/bebop/build/"${{ matrix.design }}"-*(N); do rm -rf "${build_dir}" done - nix develop -c bbdev bebop-p2e --verilog '--chip ${{ matrix.chip }} --diff' - nix develop -c bbdev bebop-p2e --buildbitstream '--chip ${{ matrix.chip }} --diff' - bits=("${root}"/bebop/build/"${{ matrix.chip }}"-*/fpgaCompDir/bitstream.bit(N)) + nix develop -c bbdev bebop-p2e --verilog '--design ${{ matrix.design }} --device ${{ matrix.config }} --diff' + nix develop -c bbdev bebop-p2e --buildbitstream '--design ${{ matrix.design }} --device ${{ matrix.config }} --diff' + bits=("${root}"/bebop/build/"${{ matrix.design }}"-*/fpgaCompDir/bitstream.bit(N)) if (( ${#bits} != 1 )); then - echo "ERROR: expected exactly 1 bitstream.bit for ${{ matrix.chip }}, got ${#bits}" >&2 + echo "ERROR: expected exactly 1 bitstream.bit for ${{ matrix.design }}, got ${#bits}" >&2 printf '%s\n' "${bits[@]}" >&2 exit 1 fi @@ -254,26 +269,31 @@ jobs: exit "${CHECKOUT_SHA}" "${root}" "${CHECKOUT_REF}" fpga-check: - name: fpga-check-${{ matrix.chip }} + name: fpga-check-${{ matrix.design }} needs: fpga-bitstream runs-on: check strategy: fail-fast: false matrix: include: - - chip: toy + - design: toy + config: ToyVerilatorConfig run_fpga_bare: true run_fpga_linux: true - - chip: pebble + - design: pebble + config: PebbleVerilatorConfig run_fpga_bare: true run_fpga_linux: true - - chip: goban + - design: goban + config: GobanVerilatorConfig run_fpga_bare: true run_fpga_linux: false - - chip: poly + - design: poly + config: PolyP2EConfig run_fpga_bare: true run_fpga_linux: false - - chip: multi-rocket + - design: multi-rocket + config: MultiRocketVerilatorConfig run_fpga_bare: true run_fpga_linux: false env: @@ -287,7 +307,7 @@ jobs: root="${BB_ROOT/#\~/$HOME}" source "${root}/.github/scripts/setproxy.sh" cd "${root}" - git fetch --force --prune origin "${CHECKOUT_SHA}" + git fetch --no-recurse-submodules --force --prune origin "${CHECKOUT_SHA}" git show "${CHECKOUT_SHA}:.github/scripts/ci_repo_lock.sh" | bash -s -- \ enter "${CHECKOUT_SHA}" "${root}" "${CHECKOUT_REF}" @@ -298,23 +318,23 @@ jobs: set -euo pipefail root="${BB_ROOT/#\~/$HOME}" cd "${root}" - bits=("${root}"/bebop/build/"${{ matrix.chip }}"-*/fpgaCompDir/bitstream.bit(N)) + bits=("${root}"/bebop/build/"${{ matrix.design }}"-*/fpgaCompDir/bitstream.bit(N)) if (( ${#bits} != 1 )); then - echo "ERROR: expected exactly 1 bitstream.bit for ${{ matrix.chip }}, got ${#bits}" >&2 + echo "ERROR: expected exactly 1 bitstream.bit for ${{ matrix.design }}, got ${#bits}" >&2 printf '%s\n' "${bits[@]}" >&2 exit 1 fi if [[ "${{ matrix.run_fpga_bare }}" == "true" ]]; then - nix develop -c bbdev compiler --build "--chip ${{ matrix.chip }}" - nix develop -c bbdev workload --build "--chip ${{ matrix.chip }}" - nix develop -c bbdev workload --tohex "--chip ${{ matrix.chip }}" - nix develop -c bbdev bebop-p2e --batch "--chip ${{ matrix.chip }} --bitstream ${bits[1]} --test bare-tests --diff" + nix develop -c bbdev stack --build "--design ${{ matrix.design }} --device ${{ matrix.config }}" + nix develop -c bbdev workload --build "--design ${{ matrix.design }} --device ${{ matrix.config }}" + nix develop -c bbdev workload --tohex "--design ${{ matrix.design }} --device ${{ matrix.config }}" + nix develop -c bbdev bebop-p2e --batch "--design ${{ matrix.design }} --device ${{ matrix.config }} --bitstream ${bits[1]} --test bare-tests --diff" fi if [[ "${{ matrix.run_fpga_linux }}" == "true" ]]; then - nix develop -c bbdev kernel --build "--chip ${{ matrix.chip }}" - nix develop -c bbdev bebop-p2e --batch "--chip ${{ matrix.chip }} --bitstream ${bits[1]} --test linux-tests --diff" + nix develop -c bbdev kernel --build "--design ${{ matrix.design }} --device ${{ matrix.config }}" + nix develop -c bbdev bebop-p2e --batch "--design ${{ matrix.design }} --device ${{ matrix.config }} --bitstream ${bits[1]} --test linux-tests --diff" fi - name: Exit repo lock @@ -327,20 +347,24 @@ jobs: exit "${CHECKOUT_SHA}" "${root}" "${CHECKOUT_REF}" eval-performance: - name: eval-performance-${{ matrix.chip }}-${{ matrix.model }} + name: eval-performance-${{ matrix.design }}-${{ matrix.model }} needs: fpga-bitstream runs-on: check strategy: fail-fast: false matrix: include: - - chip: pebble + - design: pebble + config: PebbleVerilatorConfig model: lenet - - chip: pebble + - design: pebble + config: PebbleVerilatorConfig model: mobilenet - # - chip: pebble + # - design: pebble + # config: PebbleVerilatorConfig # model: resnet - # - chip: pebble + # - design: pebble + # config: PebbleVerilatorConfig # model: yolo env: BB_ROOT: ~/Code/buckyball @@ -352,7 +376,7 @@ jobs: root="${BB_ROOT/#\~/$HOME}" source "${root}/.github/scripts/setproxy.sh" cd "${root}" - git fetch --force --prune origin "${CHECKOUT_SHA}" + git fetch --no-recurse-submodules --force --prune origin "${CHECKOUT_SHA}" git show "${CHECKOUT_SHA}:.github/scripts/ci_repo_lock.sh" | bash -s -- \ enter "${CHECKOUT_SHA}" "${root}" "${CHECKOUT_REF}" @@ -366,9 +390,9 @@ jobs: echo "ERROR: eval-performance requires a real model, got model=none" >&2 exit 1 fi - nix develop -c bbdev compiler --build "--chip ${{ matrix.chip }}" - nix develop -c bbdev workload --build "--chip ${{ matrix.chip }} --model ${{ matrix.model }}" - nix develop -c bbdev kernel --build "--chip ${{ matrix.chip }} --model ${{ matrix.model }}" + nix develop -c bbdev stack --build "--design ${{ matrix.design }} --device ${{ matrix.config }}" + nix develop -c bbdev workload --build "--design ${{ matrix.design }} --device ${{ matrix.config }} --model ${{ matrix.model }}" + nix develop -c bbdev kernel --build "--design ${{ matrix.design }} --device ${{ matrix.config }} --model ${{ matrix.model }}" - name: Model performance shell: zsh {0} @@ -376,13 +400,13 @@ jobs: set -euo pipefail root="${BB_ROOT/#\~/$HOME}" cd "${root}" - bits=("${root}"/bebop/build/"${{ matrix.chip }}"-*/fpgaCompDir/bitstream.bit(N)) + bits=("${root}"/bebop/build/"${{ matrix.design }}"-*/fpgaCompDir/bitstream.bit(N)) if (( ${#bits} != 1 )); then - echo "ERROR: expected exactly 1 bitstream.bit for ${{ matrix.chip }}, got ${#bits}" >&2 + echo "ERROR: expected exactly 1 bitstream.bit for ${{ matrix.design }}, got ${#bits}" >&2 printf '%s\n' "${bits[@]}" >&2 exit 1 fi - nix develop -c bbdev bebop-p2e --runworkload "--chip ${{ matrix.chip }} --bitstream ${bits[1]} --model ${{ matrix.model }}" + nix develop -c bbdev bebop-p2e --runworkload "--design ${{ matrix.design }} --device ${{ matrix.config }} --bitstream ${bits[1]} --model ${{ matrix.model }}" - name: Exit repo lock if: always() diff --git a/.gitignore b/.gitignore index 96d7f8f59..751857d60 100644 --- a/.gitignore +++ b/.gitignore @@ -66,3 +66,9 @@ skills-lock.json .mcp.json .tmp *.egg-info/ + +# VCS run artifacts +ucli.key +cm.log +tr_db.log +waste/ diff --git a/.gitmodules b/.gitmodules index 0c0497896..ecc807c1c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,34 +1,15 @@ -[submodule "bbdev"] - path = bbdev - url = https://github.com/DangoSys/bbdev.git [submodule "thirdparty/waveform-mcp"] path = thirdparty/waveform-mcp url = https://github.com/jiegec/waveform-mcp.git [submodule "bebop"] path = bebop url = https://github.com/DangoSys/bebop -[submodule "bb-tests/workloads/lib/kernel"] - path = bb-tests/workloads/lib/kernel - url = https://github.com/DangoSys/bbkernel.git -[submodule "bb-tests/thirdparty/busybox"] - path = bb-tests/thirdparty/busybox - url = https://github.com/mirror/busybox.git -[submodule "bb-tests/thirdparty/linux"] - path = bb-tests/thirdparty/linux - url = https://github.com/firesim/linux.git -[submodule "bb-tests/thirdparty/opensbi"] - path = bb-tests/thirdparty/opensbi - url = https://github.com/riscv-software-src/opensbi.git - branch = master [submodule "docs"] path = docs url = https://github.com/DangoSys/document.git [submodule "verify"] path = verify url = https://github.com/DangoSys/verify -[submodule "arch/thirdparty/rocket-chip"] - path = arch/thirdparty/rocket-chip - url = https://github.com/SEU-ACAL/rocket-chip.git [submodule "arch/thirdparty/berkeley-hardfloat"] path = arch/thirdparty/berkeley-hardfloat url = https://github.com/SEU-ACAL/berkeley-hardfloat.git @@ -41,9 +22,6 @@ [submodule ".agents/mcps"] path = .agents/mcps url = https://github.com/DangoSys/mcps.git -[submodule "arch/src/main/scala/framework/system/core/seed"] - path = arch/src/main/scala/framework/system/core/seed - url = https://github.com/DangoSys/seed.git [submodule "stack"] path = stack url = https://github.com/buddy-compiler/buddy-examples @@ -54,3 +32,37 @@ [submodule "thirdparty/soc-framework"] path = thirdparty/soc-framework url = https://github.com/DangoSys/soc-framework.git +[submodule "arch/thirdparty/redmule"] + path = arch/thirdparty/redmule + url = https://github.com/pulp-platform/redmule.git +[submodule "arch/thirdparty/cvfpu"] + path = arch/thirdparty/cvfpu + url = https://github.com/pulp-platform/cvfpu.git +[submodule "arch/thirdparty/common_cells"] + path = arch/thirdparty/common_cells + url = https://github.com/pulp-platform/common_cells.git +[submodule "arch/thirdparty/hci"] + path = arch/thirdparty/hci + url = https://github.com/pulp-platform/hci.git +[submodule "arch/thirdparty/hwpe-stream"] + path = arch/thirdparty/hwpe-stream + url = https://github.com/pulp-platform/hwpe-stream.git +[submodule "arch/thirdparty/hwpe-ctrl"] + path = arch/thirdparty/hwpe-ctrl + url = https://github.com/pulp-platform/hwpe-ctrl.git +[submodule "arch/thirdparty/tech_cells_generic"] + path = arch/thirdparty/tech_cells_generic + url = https://github.com/pulp-platform/tech_cells_generic.git +[submodule "examples/prefab/core/rocket/thirdparty/rocket-chip"] + path = examples/prefab/cores/rocket/thirdparty/rocket-chip + url = https://github.com/SEU-ACAL/rocket-chip.git + +[submodule "examples/prefab/core/xiangshan/thirdparty/xiangshan"] + path = examples/prefab/cores/xiangshan/thirdparty/xiangshan + url = https://github.com/OpenXiangShan/XiangShan.git +[submodule "examples/prefab/cores/boom/thirdparty/riscv-boom"] + path = examples/prefab/cores/boom/thirdparty/riscv-boom + url = https://github.com/riscv-boom/riscv-boom.git +[submodule "examples/prefab/cores/boom/thirdparty/rocket-chip"] + path = examples/prefab/cores/boom/thirdparty/rocket-chip + url = https://github.com/chipsalliance/rocket-chip.git diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8dc263385..49cc840dc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,7 +26,7 @@ repos: hooks: - id: flake8 language: system - exclude: (^|/)bbdev/api/steps/config/scripts/chip_pb2\.py$ + exclude: (^|/)bbdev/api/steps/config/scripts/[^/]+_pb2\.py$ args: - --max-line-length=120 # Adjust as per your style guide - --ignore=F821,F403,F405,F401,W503,E203,E402,E401,W605,E712,E711,F841 diff --git a/README.md b/README.md index 931972e49..b5655f62b 100644 --- a/README.md +++ b/README.md @@ -49,16 +49,17 @@ nix develop **3. Verify Installation** -Run Verilator simulation test to verify installation: +Run Verilator simulation test to verify installation. `--device` names a device config +registered in the design's `configs/sims/default.toml` (`[targets.device.configs]`): ```bash -bbdev verilator --run '--jobs 16 --chip toy --binary toy-toy-ctest-relu_i32_16x16_test-baremetal --batch' +bbdev verilator --run '--jobs 16 --design toy --device ToyVerilatorConfig --binary toy-toy-ctest-relu_i32_16x16_test-baremetal --batch' ``` **4. Try faster simulation using bebop** ```bash -bbdev bebop-verilator --run '--chip toy --binary toy-toy-ctest-relu_i32_16x16_test-baremetal --itrace --mtrace --pmctrace --ctrace' +bbdev bebop-verilator --run '--design toy --device ToyVerilatorConfig --binary toy-toy-ctest-relu_i32_16x16_test-baremetal --itrace --mtrace --pmctrace --ctrace' ``` ## Tutorial diff --git a/arch/.gitignore b/arch/.gitignore index 4dfc44f7a..e0edfb216 100644 --- a/arch/.gitignore +++ b/arch/.gitignore @@ -5,6 +5,7 @@ project/project/ .metals/ .bloop/ out/ +output/ build/ *.vcd @@ -16,5 +17,5 @@ build/ log/ *.sv -!src/main/scala/framework/**/src/main/resources/*.sv +!src/main/scala/framework/**/src/main/resources/**/*.sv gemmini_params.h diff --git a/arch/build.sc b/arch/build.sc index 7fc755c9d..22746c12c 100644 --- a/arch/build.sc +++ b/arch/build.sc @@ -1,458 +1,12 @@ import mill._ -import mill.define.Sources -import mill.modules.Util -import mill.scalalib.TestModule.ScalaTest -import scalalib._ -import mill.javalib.JavaModule -import mill.bsp._ - -object protoJava extends JavaModule { - def protoDir = T { - os.pwd / os.up / "bbdev" / "api" / "steps" / "config" / "scripts" / "proto" - } - - def protoSources = T.sources { - Seq(PathRef(protoDir() / "chip.proto")) - } - - def generatedSources = T { - val dir = protoDir() - val proto = protoSources().head.path - os.proc("protoc", s"-I$dir", s"--java_out=${T.dest}", proto).call() - Seq(PathRef(T.dest)) - } - - override def zincIncrementalCompilation = T { false } - - override def ivyDeps = Agg(ivy"com.google.protobuf:protobuf-java:4.35.1") -} - -trait FrameworkModule extends SbtModule { - def moduleRoot: os.Path - - override def millSourcePath = moduleRoot - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") -} - -val frameworkRoot = os.pwd / "src" / "main" / "scala" / "framework" - -object queue extends FrameworkModule { - override def moduleRoot = frameworkRoot / "mem-core" / "queue" -} - -object axis extends FrameworkModule { - override def moduleRoot = frameworkRoot / "mem-core" / "axis" - override def scalacOptions = super.scalacOptions() ++ Seq("-Ymacro-annotations") -} - -object axi4 extends FrameworkModule { - override def moduleDeps = Seq(queue) - override def moduleRoot = frameworkRoot / "mem-core" / "axi4" - override def mainClass = Some("memcore.bus.axi4.Emit") -} - -object ddr extends FrameworkModule { - override def moduleRoot = frameworkRoot / "mem-core" / "ddr" - override def moduleDeps = Seq(chi, axi4) - override def mainClass = Some("memcore.memory.ddr.Emit") -} - -object chi extends FrameworkModule { - override def moduleDeps = Seq(queue) - override def moduleRoot = frameworkRoot / "mem-core" / "chi" - override def scalacOptions = super.scalacOptions() ++ Seq("-Ymacro-annotations") -} - -object cpu_mem extends FrameworkModule { - override def moduleRoot = frameworkRoot / "mem-core" / "cpu_mem" - override def moduleDeps = Seq(chi, mmu) -} - -object mmu extends FrameworkModule { - override def moduleRoot = frameworkRoot / "mem-core" / "mmu" - override def moduleDeps = Seq(chi) -} - -object fetch extends FrameworkModule { - override def moduleRoot = frameworkRoot / "mem-core" / "fetch" -} - -object preflight extends FrameworkModule { - override def moduleRoot = frameworkRoot / "mem-core" / "preflight" - override def moduleDeps = Seq(chi, mmu) - override def mainClass = Some("memcore.memory.preflight.Emit") -} - -object interlock extends FrameworkModule { - override def moduleRoot = frameworkRoot / "mem-core" / "interlock" - override def mainClass = Some("memcore.memory.interlock.Emit") -} - -object bank extends FrameworkModule { - override def moduleRoot = frameworkRoot / "mem-core" / "bank" - override def moduleDeps = Seq(axis) - override def scalacOptions = super.scalacOptions() ++ Seq("-Ymacro-annotations") -} - -object cache extends FrameworkModule { - override def moduleDeps = Seq(queue) - override def moduleRoot = frameworkRoot / "mem-core" / "cache" - override def sources = T.sources { super.sources() ++ Seq(PathRef(moduleRoot / "configs")) } - override def ivyDeps = super.ivyDeps() ++ Agg(ivy"tech.sparse::toml-scala:0.2.2") - override def scalacOptions = super.scalacOptions() ++ Seq("-Ymacro-annotations") -} - -object mesh_shm extends FrameworkModule { - override def scalacOptions = super.scalacOptions() ++ Seq("-Ymacro-annotations") - override def moduleRoot = frameworkRoot / "mem-core" / "mesh_shm" - override def moduleDeps = Seq(bank) -} - -object coherence extends FrameworkModule { - override def moduleRoot = frameworkRoot / "mem-core" / "coherence" - override def moduleDeps = Seq(chi, cache) - override def sources = T.sources { super.sources() ++ Seq(PathRef(moduleRoot / "configs")) } - override def scalacOptions = super.scalacOptions() ++ Seq("-Ymacro-annotations") -} - -object blink extends FrameworkModule { - override def moduleRoot = frameworkRoot / "balldomain" / "blink" - override def moduleDeps = Seq(rocket_bb, ant, coherence, cpu_mem, ddr, interlock) - override def ivyDeps = super.ivyDeps() ++ Agg(ivy"com.lihaoyi::upickle:3.3.1") - override def scalacOptions = super.scalacOptions() ++ Seq("-Ymacro-annotations") - override def sources = T.sources { - Seq( - PathRef(frameworkRoot / "top" / "GlobalConfig.scala"), - PathRef(frameworkRoot / "top" / "configs" / "DesignConfig.scala"), - PathRef(frameworkRoot / "top" / "configs" / "SimParam.scala"), - PathRef(frameworkRoot / "memdomain" / "configs" / "MemDomainParam.scala"), - PathRef(frameworkRoot / "frontend" / "configs" / "FrontendParam.scala"), - PathRef(frameworkRoot / "rvv" / "src" / "main" / "scala" / "configs" / "RvvParam.scala"), - PathRef(frameworkRoot / "balldomain" / "configs" / "BallDomainParam.scala"), - PathRef(frameworkRoot / "system" / "tile" / "configs" / "TileParam.scala"), - PathRef(frameworkRoot / "balldomain" / "blink" / "blink.scala"), - PathRef(frameworkRoot / "balldomain" / "blink" / "bank.scala"), - PathRef(frameworkRoot / "balldomain" / "blink" / "status.scala"), - PathRef(frameworkRoot / "balldomain" / "blink" / "baseball.scala"), - PathRef(frameworkRoot / "balldomain" / "blink" / "SubRobRow.scala"), - PathRef(frameworkRoot / "balldomain" / "blink" / "mmio" / "MmioRead.scala"), - PathRef(frameworkRoot / "balldomain" / "blink" / "mmio" / "MmioWrite.scala"), - PathRef(frameworkRoot / "memdomain" / "backend" / "banks" / "SramIO.scala"), - PathRef(frameworkRoot / "memdomain" / "backend" / "mmio" / "MmioIO.scala"), - PathRef(frameworkRoot / "balldomain" / "rs" / "interfaces.scala"), - PathRef(frameworkRoot / "balldomain" / "decoder" / "BallDecodeCmd.scala"), - PathRef(frameworkRoot / "frontend" / "decoder" / "PostGDCmd.scala"), - PathRef(frameworkRoot / "frontend" / "scoreboard" / "BankAccessInfo.scala") - ) - } -} - -object arith extends FrameworkModule { - override def moduleRoot = frameworkRoot / "arith" -} - -object spm extends FrameworkModule { - override def moduleRoot = frameworkRoot / "mem-core" / "spm" - override def moduleDeps = Seq(bank) -} - -object tss extends FrameworkModule { - override def moduleRoot = frameworkRoot / "mem-core" / "tss" - override def moduleDeps = Seq(spm) - override def mainClass = Some("memcore.memory.tss.Emit") -} - -object tls extends FrameworkModule { - override def moduleRoot = frameworkRoot / "mem-core" / "tls" - override def moduleDeps = Seq(spm) - override def mainClass = Some("memcore.memory.tls.Emit") -} - -object ant extends FrameworkModule { - override def moduleRoot = frameworkRoot / "system" / "core" / "ant" - override def moduleDeps = Seq(tls, tss, arith) - override def mainClass = Some("framework.ant.Emit") -} - -object rvv extends FrameworkModule { - override def moduleRoot = frameworkRoot / "rvv" - override def moduleDeps = Seq(hardfloat, blink, arith) - override def sources = T.sources { - os.walk(moduleRoot / "src" / "main" / "scala") - .filter(path => path.ext == "scala" && path.last != "RvvParam.scala") - .map(PathRef(_)) - } - override def scalacOptions = super.scalacOptions() ++ Seq("-Ymacro-annotations") -} - -object seed extends FrameworkModule { - override def moduleRoot = frameworkRoot / "system" / "core" / "seed" -} - -object rocket_bb extends FrameworkModule { - override def moduleRoot = frameworkRoot / "system" / "core" / "rocket" - override def moduleDeps = Seq(rocketchip) - override def ivyDeps = super.ivyDeps() ++ Agg(ivy"com.lihaoyi::upickle:3.3.1") - override def sources = T.sources { - os.walk(moduleRoot) - .filter(path => path.ext == "scala" && !Set("CoreParameters.scala", "CpuInterfaces.scala", "TileInterrupts.scala", "TileParameters.scala", "RocketCoreParameters.scala", "Utility.scala", "VectorInterfaces.scala", "RocketCpuParam.scala").contains(path.last)) - .map(PathRef(_)) - } -} - -object root_chip extends FrameworkModule { - override def moduleRoot = frameworkRoot / "root" / "root-chip" - override def moduleDeps = Seq(axis, chi, bank) -} - -object root_core extends FrameworkModule { - override def mainClass = Some("hier.core.rocket.Emit") - override def moduleRoot = frameworkRoot / "root" / "root-core" - override def moduleDeps = Seq(rocket_bb, fetch, cpu_mem, chi, coherence, interlock, preflight) - override def sources = T.sources { - super.sources() ++ Seq( - PathRef(os.pwd / os.up / "examples" / "cores" / "rocket" / "arch" / "src" / "main" / "scala" / "cpu"), - PathRef(frameworkRoot / "system" / "core" / "clink" / "base.scala") - ) - } -} - -object root_tile extends FrameworkModule { - override def mainClass = Some("hier.tile.memory.Emit") - override def moduleRoot = frameworkRoot / "root" / "root-tile" - override def moduleDeps = Seq(chi, coherence, cpu_mem, mmu, root_core, ddr) -} - -object buckyball extends SbtModule { m => - override def millSourcePath = os.pwd - override def scalaVersion = "2.13.16" - - override def scalacOptions = Seq( - "-language:reflectiveCalls", - "-deprecation", - "-feature", - "-Xcheckinit", - "-Ymacro-annotations" - ) - - override def moduleDeps = Seq( - rocketchip, - gemmini, - protoJava, - axis, - fetch, - chi, - bank, - mesh_shm, - cache, - coherence, - ddr, - rvv, - seed, - ant, - rocket_bb, - root_chip, - root_core, - root_tile - ) - - override def sources = T.sources { - val examples = os.pwd / os.up / "examples" - def archSrcs(kind: String) = - os.list(examples / kind) - .filter(os.isDir) - .map(_ / "arch" / "src" / "main" / "scala") - .filter(os.exists) - .flatMap(root => os.walk(root).filter(path => path.ext == "scala")) - .map(PathRef(_)) - - def configSrcs(kind: String) = - os.list(examples / kind) - .filter(os.isDir) - .map(_ / "configs") - .filter(os.exists) - .map(PathRef(_)) - - val sharedBlink = blink.sources().map(_.path).toSet - val localSources = os.walk(os.pwd / "src" / "main" / "scala") - .filter(path => path.ext == "scala") - .filterNot(sharedBlink.contains) - .filterNot(path => path.toString.contains("/framework/root/")) - .filterNot(path => path.toString.contains("/verification/")) - .filterNot(path => path.toString.contains("/framework/mem-core/")) - .filterNot(path => path.toString.contains("/framework/rvv/")) - .filterNot(path => path.toString.contains("/framework/system/core/seed/")) - .filterNot(path => path.toString.contains("/framework/system/core/ant/")) - .filterNot(path => path.toString.contains("/framework/arith/")) - .filterNot(path => path.toString.contains("/framework/system/core/rocket/")) - .filterNot(path => path == frameworkRoot / "system" / "core" / "clink" / "base.scala") - .map(PathRef(_)) - val coreSources = archSrcs("cores") - .filterNot(_.path.toString.contains("/cores/rocket/arch/src/main/scala/cpu/")) - localSources ++ archSrcs("balls") ++ coreSources ++ archSrcs("chips") ++ - configSrcs("balls") ++ configSrcs("chips") ++ configSrcs("cores") - } - - override def ivyDeps = Agg( - ivy"org.chipsalliance::chisel:6.7.0", - ivy"org.apache.commons:commons-lang3:3.12.0", - ivy"org.apache.commons:commons-text:1.9", - ivy"org.yaml:snakeyaml:2.0", - ivy"com.lihaoyi::sourcecode:0.3.0", - ivy"com.lihaoyi::upickle:3.3.1", - ivy"tech.sparse::toml-scala:0.2.2", - ivy"com.google.protobuf:protobuf-java:4.35.1" - ) - - override def scalacPluginIvyDeps = Agg( - ivy"org.chipsalliance:::chisel-plugin:6.7.0" - ) - - object test extends ScalaModule with TestModule.ScalaTest { - override def scalaVersion = T("2.13.16") - override def moduleDeps = Seq(m) - - override def ivyDeps = Agg( - ivy"org.scalatest::scalatest::3.2.19" - ) - - } - -} - -// Define cde module - must be compiled first -object cde extends SbtModule { - override def millSourcePath = - os.pwd / "thirdparty" / "rocket-chip" / "dependencies" / "cde" - override def scalaVersion = "2.13.16" - - override def sources = T.sources { - Seq(PathRef(millSourcePath / "cde" / "src")) - } - - override def ivyDeps = Agg( - ivy"org.chipsalliance::chisel:6.7.0" - ) - - override def scalacPluginIvyDeps = Agg( - ivy"org.chipsalliance:::chisel-plugin:6.7.0" - ) - -} - -// Define hardfloat module -object hardfloat extends SbtModule { - override def millSourcePath = - os.pwd / "thirdparty" / "berkeley-hardfloat" - override def scalaVersion = "2.13.16" - - // Override sources to match build.sbt behavior - override def sources = T.sources { - super.sources() ++ Seq( - PathRef(millSourcePath / "hardfloat" / "src" / "main" / "scala") - ) - } - - override def ivyDeps = Agg( - ivy"org.chipsalliance::chisel:6.7.0" - ) - - override def scalacPluginIvyDeps = Agg( - ivy"org.chipsalliance:::chisel-plugin:6.7.0" - ) - -} - -// Define midas_target_utils module -object midas_target_utils extends SbtModule { - override def millSourcePath = - os.pwd / os.up / "thirdparty" / "firesim" / "sim" / "midas" / "targetutils" - override def scalaVersion = "2.13.16" - - override def ivyDeps = Agg( - ivy"org.chipsalliance::chisel:6.7.0" - ) - - override def scalacPluginIvyDeps = Agg( - ivy"org.chipsalliance:::chisel-plugin:6.7.0" - ) - -} - -// Define rocket-chip module with proper dependencies -object rocketchip extends SbtModule { - override def millSourcePath = - os.pwd / "thirdparty" / "rocket-chip" - override def scalaVersion = "2.13.16" - - override def sources = T.sources { - val src = millSourcePath / "src" / "main" / "scala" - val files = Seq( - "rocket/ALU.scala", "rocket/AMOALU.scala", "rocket/BTB.scala", "rocket/Breakpoint.scala", - "rocket/CSR.scala", "rocket/Consts.scala", "rocket/CustomInstructions.scala", "rocket/DebugROB.scala", - "rocket/Decode.scala", "rocket/Events.scala", "rocket/IBuf.scala", "rocket/IDecode.scala", - "rocket/Instructions.scala", "rocket/Instructions32.scala", "rocket/Multiplier.scala", "rocket/PMP.scala", - "rocket/RVC.scala", "rocket/package.scala", "tile/FPU.scala", "tile/CustomCSRs.scala", - "util/Misc.scala", "util/ClockGate.scala", "util/CoreMonitor.scala", "util/Property.scala", - "util/Counters.scala", "util/ECC.scala", "util/Replacement.scala", "util/ShiftReg.scala", - "util/ShiftQueue.scala", "util/HellaQueue.scala", "util/LCG.scala", "util/PlusArg.scala", - "util/MuxLiteral.scala", "util/Arbiters.scala", "util/AsyncResetReg.scala", "util/Timer.scala", - "util/GeneratorUtils.scala", "unittest/UnitTest.scala" - ) - val adapter = frameworkRoot / "system" / "core" / "rocket" - files.map(f => PathRef(src / os.RelPath(f))) ++ Seq( - PathRef(adapter / "CoreParameters.scala"), PathRef(adapter / "CpuInterfaces.scala"), - PathRef(adapter / "TileInterrupts.scala"), PathRef(adapter / "TileParameters.scala"), - PathRef(adapter / "RocketCoreParameters.scala"), PathRef(adapter / "Utility.scala"), - PathRef(adapter / "VectorInterfaces.scala"), PathRef(adapter / "configs" / "RocketCpuParam.scala") - ) - } - - override def moduleDeps = Seq(cde, hardfloat, midas_target_utils) - - override def ivyDeps = Agg( - ivy"org.chipsalliance::chisel:6.7.0", - ivy"com.lihaoyi::upickle:3.3.1", - ivy"com.lihaoyi::mainargs:0.5.0", - ivy"org.json4s::json4s-jackson:4.0.5", - ivy"org.scala-graph::graph-core:1.13.5" - ) - - override def scalacPluginIvyDeps = Agg( - ivy"org.chipsalliance:::chisel-plugin:6.7.0" - ) - -} - - -// Define gemmini module -object gemmini extends SbtModule { - override def millSourcePath = - os.pwd / "thirdparty" / "gemmini" - override def scalaVersion = "2.13.16" - - override def sources = T.sources { - Seq("Arithmetic", "Dataflow", "MeshWithDelays", "Mesh", "Tile", "PE", "Util", "Transposer", "TagQueue", "Pipeline", "Shifter", "SyncMem") - .map(name => PathRef(millSourcePath / "src" / "main" / "scala" / "gemmini" / s"$name.scala")) - } - override def moduleDeps = Seq(hardfloat) - - override def ivyDeps = Agg( - ivy"org.chipsalliance::chisel:6.7.0" - ) - - override def scalacPluginIvyDeps = Agg( - ivy"org.chipsalliance:::chisel-plugin:6.7.0" - ) - -} - - -object memdomain_ack extends FrameworkModule { - override def moduleRoot = frameworkRoot / "memdomain" / "verification" - override def moduleDeps = Seq(buckyball, ddr, preflight) - override def mainClass = Some("framework.memdomain.verification.Emit") +import $file.thirdparty.{build => external} +import $file.src.main.scala.framework.{build => framework_build} +import $file.src.main.scala.sims.{build => simulation} +import $file.^.examples.{build => example_build} + +object architecture extends Module { + def thirdparty = external + def framework = framework_build + def examples = example_build + def sims = simulation } diff --git a/arch/settings.sc b/arch/settings.sc new file mode 100644 index 000000000..1688c733a --- /dev/null +++ b/arch/settings.sc @@ -0,0 +1,34 @@ +import mill._ +import mill.scalalib._ + +val archRoot: os.Path = millSourcePath + +trait FrameworkModule extends SbtModule { + def moduleRoot: os.Path + override def millSourcePath = moduleRoot + override def forkWorkingDir = T(moduleRoot) + override def scalaVersion = "2.13.16" + override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0", ivy"com.lihaoyi::upickle:3.3.1") + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") +} + +trait IpModule extends FrameworkModule { + def framework: ScalaModule + override def moduleDeps = Seq(framework) + override def sources = T.sources(Seq(PathRef(moduleRoot / "src" / "main" / "scala" / "Emit.scala"))) +} + +object lib extends FrameworkModule { + override def moduleRoot = archRoot / "src" / "main" / "scala" / "framework" / "lib" + + override def sources = T.sources { + Seq( + PathRef(moduleRoot / "queue"), + PathRef(moduleRoot / "arbiter"), + PathRef(moduleRoot / "arith"), + PathRef(moduleRoot / "dpi") + ) + } + +} diff --git a/arch/src/csrc/include/ioe/mmio.h b/arch/src/csrc/include/ioe/mmio.h index 311ae2823..35b252273 100644 --- a/arch/src/csrc/include/ioe/mmio.h +++ b/arch/src/csrc/include/ioe/mmio.h @@ -5,7 +5,7 @@ // These functions are called from the RTL via DPI-C when software writes to // the SCU registers (0x6000_0000 for sim_exit, 0x6002_0000 for UART). // -// The SCU (System Control Unit) is instantiated per-tile inside BBTile and +// The SCU (System Control Unit) is instantiated per-core inside BBCore and // intercepts MMIO writes before they reach the system bus. // // DPI-C functions (implemented in mmio.cc): diff --git a/arch/src/csrc/include/monitor/trace.h b/arch/src/csrc/include/monitor/trace.h index dab0b5f1c..af6db349f 100644 --- a/arch/src/csrc/include/monitor/trace.h +++ b/arch/src/csrc/include/monitor/trace.h @@ -14,25 +14,35 @@ void dpi_bdb_set_clk(unsigned long long c); // DPI-C function for instruction trace (itrace) // Called from GlobalROB when instructions are allocated/issued/completed void dpi_itrace(unsigned int is_issue, // 2 = alloc, 1 = issue, 0 = complete - unsigned int hart_id_lo, unsigned int hart_id_hi, + unsigned int thread_id_lo, unsigned int thread_id_hi, unsigned int rob_id, unsigned int inst_id_lo, unsigned int inst_id_hi, unsigned int domain_id, unsigned int funct, unsigned int pc_lo, unsigned int pc_hi, unsigned int rs1_lo, unsigned int rs1_hi, unsigned int rs2_lo, unsigned int rs2_hi); +// Passive Ant retirement observation from the thread trace collector. +void dpi_ant_trace(unsigned int thread_lo, unsigned int thread_hi, + unsigned int pc_lo, unsigned int pc_hi, + unsigned int instruction, unsigned int rd, + unsigned int data_lo, unsigned int data_hi); + // DPI-C function for memory trace (mtrace) // Called from MemBackend when read/write requests are made -void dpi_mtrace(unsigned char is_write, // 1 = write, 0 = read - unsigned char is_shared, unsigned int channel, - unsigned long long hart_id, unsigned int vbank_id, - unsigned int pbank_id, unsigned int group_id, unsigned int addr, - unsigned long long data_lo, unsigned long long data_hi); +void dpi_mtrace(unsigned int is_write, unsigned int is_shared, + unsigned int channel, unsigned int thread_id_lo, + unsigned int thread_id_hi, unsigned int rob_id, + unsigned int inst_id_lo, unsigned int inst_id_hi, + unsigned int vbank_id, unsigned int pbank_id, + unsigned int group_id, unsigned int addr, + unsigned int write_mask, unsigned int data_lo_lo, + unsigned int data_lo_hi, unsigned int data_hi_lo, + unsigned int data_hi_hi); // DPI-C function for Ball PMC trace (pmctrace) // Called from BallCyclePMC when a Ball completes a task void dpi_pmctrace(unsigned int ball_id, unsigned int rob_id, - unsigned long long elapsed); + unsigned int elapsed_lo, unsigned int elapsed_hi); // DPI-C function for memory PMC trace (pmctrace) // Called from MemCyclePMC when a load/store completes @@ -41,9 +51,18 @@ void dpi_mem_pmctrace(unsigned char is_store, // 1 = store, 0 = load // DPI-C function for cycle counter trace (ctrace) // Called from TraceBall when counter commands are executed -void dpi_ctrace(unsigned char subcmd, // 0=START, 1=STOP, 2=READ - unsigned int ctr_id, unsigned long long tag, - unsigned long long elapsed, unsigned long long cycle); +void dpi_ctrace(unsigned int subcmd, // 0=START, 1=STOP, 2=READ + unsigned int ctr_id, unsigned int tag_lo, unsigned int tag_hi, + unsigned int elapsed_lo, unsigned int elapsed_hi, + unsigned int cycle_lo, unsigned int cycle_hi); + +// DPI-C function for bank hash trace (banktrace) +// Called when a Ball instruction writes a bank +void dpi_btrace(unsigned int inst_id_lo, unsigned int inst_id_hi, + unsigned int thread_id_lo, unsigned int thread_id_hi, + unsigned int owner_thread_id_lo, + unsigned int owner_thread_id_hi, unsigned int w0_vbank, + unsigned int w0_hash); #ifdef __cplusplus } diff --git a/arch/src/csrc/src/monitor/ioe/BBSimDRAM.cc b/arch/src/csrc/src/monitor/ioe/BBSimDRAM.cc index 1a450339d..f75182cee 100644 --- a/arch/src/csrc/src/monitor/ioe/BBSimDRAM.cc +++ b/arch/src/csrc/src/monitor/ioe/BBSimDRAM.cc @@ -97,10 +97,12 @@ static void load_elf_to_mem(const char *path, uint8_t *data, uint64_t mem_base, fflush(stdout); } -extern "C" void * -bbsim_memory_init(int chip_id, long long int mem_size, long long int word_size, - long long int line_size, long long int id_bits, - long long int clock_hz, long long int mem_base) { +extern "C" void *bbsim_memory_init(int device_id, long long int mem_size, + long long int word_size, + long long int line_size, + long long int id_bits, + long long int clock_hz, + long long int mem_base) { mm_t *mm; s_vpi_vlog_info info; @@ -125,11 +127,11 @@ bbsim_memory_init(int chip_id, long long int mem_size, long long int word_size, abort(); } - while (chip_id >= (int)mem_data.size()) + while (device_id >= (int)mem_data.size()) mem_data.push_back(std::map()); - if (mem_data[chip_id].find(mem_base) != mem_data[chip_id].end()) { - assert(mem_data[chip_id][mem_base].size == (size_t)mem_size); + if (mem_data[device_id].find(mem_base) != mem_data[device_id].end()) { + assert(mem_data[device_id][mem_base].size == (size_t)mem_size); } else { uint8_t *data = (uint8_t *)mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); @@ -143,13 +145,13 @@ bbsim_memory_init(int chip_id, long long int mem_size, long long int word_size, load_elf_to_mem(elf_file.c_str(), data, (uint64_t)mem_base, (uint64_t)mem_size); - mem_data[chip_id][mem_base] = {data, (size_t)mem_size}; + mem_data[device_id][mem_base] = {data, (size_t)mem_size}; } // External AXI memory is always modeled by DRAMSim3. Internal SRAMs are // modeled separately by the generated Verilog memory models. mm = (mm_t *)(new mm_dramsim3_t( - mem_base, mem_size, word_size, line_size, mem_data[chip_id][mem_base], + mem_base, mem_size, word_size, line_size, mem_data[device_id][mem_base], memory_ini, local_ini_dir, dramsim_outdir, 1 << id_bits, clock_hz)); channels.push_back(mm); diff --git a/arch/src/csrc/src/monitor/ioe/mm_dramsim3.cc b/arch/src/csrc/src/monitor/ioe/mm_dramsim3.cc index 06dccedd0..ce31e1f84 100644 --- a/arch/src/csrc/src/monitor/ioe/mm_dramsim3.cc +++ b/arch/src/csrc/src/monitor/ioe/mm_dramsim3.cc @@ -99,12 +99,9 @@ void mm_dramsim3_t::tick(bool reset, bool w_last, bool r_ready, bool b_ready) { - ar_ready_cache = !reset && mem->WillAcceptTransaction(ar_addr, false); - aw_ready_cache = - !reset && !store_inflight && mem->WillAcceptTransaction(aw_addr, true); - - bool ar_fire = ar_valid && ar_ready_cache; - bool aw_fire = aw_valid && aw_ready_cache; + // Consume only the READY values presented to RTL before this clock edge. + bool ar_fire = !reset && ar_valid && ar_ready(); + bool aw_fire = !reset && aw_valid && aw_ready(); bool w_fire = !reset && w_valid && w_ready(); bool r_fire = !reset && r_valid() && r_ready; bool b_fire = !reset && b_valid() && b_ready; @@ -154,6 +151,10 @@ void mm_dramsim3_t::tick(bool reset, mem->ClockTick(); cycle++; + ar_ready_cache = !reset && mem->WillAcceptTransaction(ar_addr, false); + aw_ready_cache = + !reset && !store_inflight && mem->WillAcceptTransaction(aw_addr, true); + if (reset) { while (!bresp.empty()) bresp.pop(); diff --git a/arch/src/csrc/src/monitor/trace/ctrace.cc b/arch/src/csrc/src/monitor/trace/ctrace.cc index 039411c8d..f0b2be024 100644 --- a/arch/src/csrc/src/monitor/trace/ctrace.cc +++ b/arch/src/csrc/src/monitor/trace/ctrace.cc @@ -9,31 +9,6 @@ extern const char *log_path; static FILE *ctrace_fp = NULL; -// Backdoor write state -static uint32_t bd_write_row_cursor = 0; -static uint64_t bd_write_data_lo = 0; -static uint64_t bd_write_data_hi = 0; - -// Backdoor read state -static uint32_t bd_read_row_cursor = 0; - -// Backdoor test data generator -// Generates a simple incremental pattern for each row. -// 128-bit = 16 bytes, each byte = (row * 16 + byte_offset) & 0xFF -static void generate_test_data(uint32_t row, uint64_t *data_lo, - uint64_t *data_hi) { - uint8_t data[16]; - for (int i = 0; i < 16; i++) { - data[i] = (uint8_t)((row * 16 + i) & 0xFF); - } - *data_lo = 0; - *data_hi = 0; - for (int i = 0; i < 8; i++) { - *data_lo |= ((uint64_t)data[i]) << (i * 8); - *data_hi |= ((uint64_t)data[i + 8]) << (i * 8); - } -} - static void init_ctrace() { if (ctrace_fp == NULL && log_path != NULL) { ctrace_fp = fopen(log_path, "a"); @@ -43,25 +18,22 @@ static void init_ctrace() { } } -static void u128_hex(char *buf, size_t n, unsigned long long hi, - unsigned long long lo) { - int ret = snprintf(buf, n, "0x%016llX%016llX", hi, lo); - if (ret < 0 || (size_t)ret >= n) { - panic("snprintf failed in ctrace u128_hex"); - } -} - // ================================================================ // Cycle counter DPI-C // ================================================================ -extern "C" void dpi_ctrace(unsigned char subcmd, unsigned int ctr_id, - unsigned long long tag, unsigned long long elapsed, - unsigned long long cycle) { +extern "C" void dpi_ctrace(unsigned int subcmd, unsigned int ctr_id, + unsigned int tag_lo, unsigned int tag_hi, + unsigned int elapsed_lo, unsigned int elapsed_hi, + unsigned int cycle_lo, unsigned int cycle_hi) { if (!bdb_trace_on(BDB_TR_CTRACE)) { return; } init_ctrace(); + unsigned long long tag = ((unsigned long long)tag_hi << 32) | tag_lo; + unsigned long long elapsed = + ((unsigned long long)elapsed_hi << 32) | elapsed_lo; + unsigned long long cycle = ((unsigned long long)cycle_hi << 32) | cycle_lo; if (ctrace_fp) { switch (subcmd) { @@ -91,74 +63,26 @@ extern "C" void dpi_ctrace(unsigned char subcmd, unsigned int ctr_id, } } -// ================================================================ -// Backdoor DPI-C functions -// RTL calls these each iteration. C++ auto-increments row. -// bank_id comes from RTL (instruction encoding), not from C++. -// ================================================================ - -// RTL calls this to get the next row to read. -// Returns row in [31:0], upper bits = 0. -extern "C" unsigned long long dpi_backdoor_get_read_addr(void) { - uint32_t row = bd_read_row_cursor; - bd_read_row_cursor++; - return (uint64_t)row; -} - -// RTL calls this to get the next row to write. -// Also pre-generates test data for this row. -// Returns row in [31:0], upper bits = 0. -extern "C" unsigned long long dpi_backdoor_get_write_addr(void) { - uint32_t row = bd_write_row_cursor; - generate_test_data(row, &bd_write_data_lo, &bd_write_data_hi); - bd_write_row_cursor++; - return (uint64_t)row; -} - -// RTL calls this to get write data (128-bit as lo/hi). -// Data was pre-generated by the preceding get_write_addr call. -extern "C" void dpi_backdoor_get_write_data(unsigned long long *data_lo, - unsigned long long *data_hi) { - *data_lo = bd_write_data_lo; - *data_hi = bd_write_data_hi; -} - -// RTL calls this after reading SRAM — logs the data to NDJSON trace file -extern "C" void dpi_backdoor_put_read_data(unsigned int bank_id, - unsigned int row, - unsigned long long data_lo, - unsigned long long data_hi) { - if (!bdb_trace_on(BDB_TR_BANKTRACE)) { - return; - } - init_ctrace(); - if (ctrace_fp) { - char data_hex[35]; - u128_hex(data_hex, sizeof(data_hex), data_hi, data_lo); - fprintf(ctrace_fp, - "{\"type\":\"banktrace\",\"clk\":%llu,\"event\":\"backdoor_read\"," - "\"bank_id\":%u,\"row\":%u,\"data\":\"%s\"}\n", - (unsigned long long)bdb_rtl_clk, bank_id, row, data_hex); - fflush(ctrace_fp); - } -} - -// RTL calls this after writing SRAM — logs the write to NDJSON trace file -extern "C" void dpi_backdoor_put_write_done(unsigned int bank_id, - unsigned int row, - unsigned long long data_lo, - unsigned long long data_hi) { +// RTL calls this when a Ball instruction writes a bank: logs the bank hash +extern "C" void dpi_btrace(unsigned int inst_id_lo, unsigned int inst_id_hi, + unsigned int thread_id_lo, unsigned int thread_id_hi, + unsigned int owner_thread_id_lo, + unsigned int owner_thread_id_hi, + unsigned int w0_vbank, unsigned int w0_hash) { if (!bdb_trace_on(BDB_TR_BANKTRACE)) { return; } init_ctrace(); if (ctrace_fp) { - char data_hex[35]; - u128_hex(data_hex, sizeof(data_hex), data_hi, data_lo); fprintf(ctrace_fp, - "{\"type\":\"banktrace\",\"clk\":%llu,\"event\":\"backdoor_write\"," - "\"bank_id\":%u,\"row\":%u,\"data\":\"%s\"}\n", - (unsigned long long)bdb_rtl_clk, bank_id, row, data_hex); + "{\"type\":\"banktrace\",\"clk\":%llu,\"event\":\"btrace\"," + "\"inst_id\":%llu,\"thread_id\":%llu,\"owner_thread_id\":%llu," + "\"vbank\":%u,\"hash\":\"0x%08X\"}\n", + (unsigned long long)bdb_rtl_clk, + ((unsigned long long)inst_id_hi << 32) | inst_id_lo, + ((unsigned long long)thread_id_hi << 32) | thread_id_lo, + ((unsigned long long)owner_thread_id_hi << 32) | owner_thread_id_lo, + w0_vbank, w0_hash); fflush(ctrace_fp); } } diff --git a/arch/src/csrc/src/monitor/trace/itrace.cc b/arch/src/csrc/src/monitor/trace/itrace.cc index 34f78a3b3..23d7fbaff 100644 --- a/arch/src/csrc/src/monitor/trace/itrace.cc +++ b/arch/src/csrc/src/monitor/trace/itrace.cc @@ -3,6 +3,7 @@ #include "utils/debug.h" #include #include +#include // Global log file pointer (shared with monitor.cc) extern const char *log_path; @@ -29,7 +30,7 @@ static void u64_hex(char *buf, size_t n, unsigned long long v) { // Called when an instruction is allocated/issued/completed in GlobalROB extern "C" void dpi_itrace(unsigned int is_issue, // 2 = alloc, 1 = issue, 0 = complete - unsigned int hart_id_lo, unsigned int hart_id_hi, + unsigned int thread_id_lo, unsigned int thread_id_hi, unsigned int rob_id, unsigned int inst_id_lo, unsigned int inst_id_hi, unsigned int domain_id, unsigned int funct, unsigned int pc_lo, unsigned int pc_hi, unsigned int rs1_lo, @@ -38,10 +39,11 @@ dpi_itrace(unsigned int is_issue, // 2 = alloc, 1 = issue, 0 = complete return; } init_itrace(); + const char *scope = svGetNameFromScope(svGetScope()); if (itrace_fp) { - unsigned long long hart_id = - ((unsigned long long)hart_id_hi << 32) | hart_id_lo; + unsigned long long thread_id = + ((unsigned long long)thread_id_hi << 32) | thread_id_lo; unsigned long long inst_id = ((unsigned long long)inst_id_hi << 32) | inst_id_lo; unsigned long long pc = ((unsigned long long)pc_hi << 32) | pc_lo; @@ -55,31 +57,58 @@ dpi_itrace(unsigned int is_issue, // 2 = alloc, 1 = issue, 0 = complete u64_hex(rs2_hex, sizeof(rs2_hex), rs2); if (is_issue == 2) { fprintf(itrace_fp, - "{\"type\":\"itrace\",\"clk\":%llu,\"event\":\"alloc\",\"hart_" + "{\"type\":\"itrace\",\"clk\":%llu,\"scope\":\"%s\",\"event\":" + "\"alloc\",\"thread_" "id\":%llu," "\"rob_id\":%u,\"inst_id\":%llu,\"domain_id\":%u,\"funct\":\"0x%" "02x\"," "\"pc\":\"%s\",\"rs1\":\"%s\",\"rs2\":\"%s\"}\n", - (unsigned long long)bdb_rtl_clk, hart_id, rob_id, inst_id, - domain_id, funct, pc_hex, rs1_hex, rs2_hex); + (unsigned long long)bdb_rtl_clk, scope, thread_id, rob_id, + inst_id, domain_id, funct, pc_hex, rs1_hex, rs2_hex); } else if (is_issue == 1) { fprintf(itrace_fp, - "{\"type\":\"itrace\",\"clk\":%llu,\"event\":\"issue\",\"hart_" + "{\"type\":\"itrace\",\"clk\":%llu,\"scope\":\"%s\",\"event\":" + "\"issue\",\"thread_" "id\":%llu," "\"rob_id\":%u,\"inst_id\":%llu,\"domain_id\":%u,\"funct\":\"0x%" "02x\"," "\"pc\":\"%s\",\"rs1\":\"%s\",\"rs2\":\"%s\"}\n", - (unsigned long long)bdb_rtl_clk, hart_id, rob_id, inst_id, - domain_id, funct, pc_hex, rs1_hex, rs2_hex); + (unsigned long long)bdb_rtl_clk, scope, thread_id, rob_id, + inst_id, domain_id, funct, pc_hex, rs1_hex, rs2_hex); } else { fprintf( itrace_fp, - "{\"type\":\"itrace\",\"clk\":%llu,\"event\":\"complete\"," - "\"hart_id\":%llu,\"rob_id\":%u,\"inst_id\":%llu,\"domain_id\":%u," + "{\"type\":\"itrace\",\"clk\":%llu,\"scope\":\"%s\",\"event\":" + "\"complete\"," + "\"thread_id\":%llu,\"rob_id\":%u,\"inst_id\":%llu,\"domain_id\":%u," "\"funct\":\"0x%02x\",\"pc\":\"%s\"}\n", - (unsigned long long)bdb_rtl_clk, hart_id, rob_id, inst_id, domain_id, - funct, pc_hex); + (unsigned long long)bdb_rtl_clk, scope, thread_id, rob_id, inst_id, + domain_id, funct, pc_hex); } fflush(itrace_fp); } } + +extern "C" void dpi_ant_trace(unsigned int thread_lo, unsigned int thread_hi, + unsigned int pc_lo, unsigned int pc_hi, + unsigned int instruction, unsigned int rd, + unsigned int data_lo, unsigned int data_hi) { + if (!bdb_trace_on(BDB_TR_ITRACE)) + return; + init_itrace(); + const char *scope = svGetNameFromScope(svGetScope()); + if (!itrace_fp) + return; + const auto thread = + (static_cast(thread_hi) << 32) | thread_lo; + const auto pc = (static_cast(pc_hi) << 32) | pc_lo; + const auto data = (static_cast(data_hi) << 32) | data_lo; + fprintf(itrace_fp, + "{\"type\":\"ant_trace\",\"clk\":%llu,\"scope\":\"%s\",\"thread_id\":" + "%llu," + "\"pc\":\"0x%016llx\",\"instruction\":\"0x%08x\"," + "\"rd\":%u,\"data\":\"0x%016llx\"}\n", + static_cast(bdb_rtl_clk), scope, thread, pc, + instruction, rd, data); + fflush(itrace_fp); +} diff --git a/arch/src/csrc/src/monitor/trace/mtrace.cc b/arch/src/csrc/src/monitor/trace/mtrace.cc index 319d3bb5d..63c6bc0b5 100644 --- a/arch/src/csrc/src/monitor/trace/mtrace.cc +++ b/arch/src/csrc/src/monitor/trace/mtrace.cc @@ -3,6 +3,7 @@ #include "utils/debug.h" #include #include +#include // Global log file pointer (shared with monitor.cc) extern const char *log_path; @@ -28,37 +29,53 @@ static void u128_hex(char *buf, size_t n, unsigned long long hi, // DPI-C function for memory trace (mtrace) // Called when MemBackend performs read/write operations -extern "C" void dpi_mtrace(unsigned char is_write, // 1 = write, 0 = read - unsigned char is_shared, unsigned int channel, - unsigned long long hart_id, unsigned int vbank_id, - unsigned int pbank_id, unsigned int group_id, - unsigned int addr, unsigned long long data_lo, - unsigned long long data_hi) { +extern "C" void dpi_mtrace(unsigned int is_write, unsigned int is_shared, + unsigned int channel, unsigned int thread_id_lo, + unsigned int thread_id_hi, unsigned int rob_id, + unsigned int inst_id_lo, unsigned int inst_id_hi, + unsigned int vbank_id, unsigned int pbank_id, + unsigned int group_id, unsigned int addr, + unsigned int write_mask, unsigned int data_lo_lo, + unsigned int data_lo_hi, unsigned int data_hi_lo, + unsigned int data_hi_hi) { if (!bdb_trace_on(BDB_TR_MTRACE)) { return; } init_mtrace(); + const char *scope = svGetNameFromScope(svGetScope()); if (mtrace_fp) { + const unsigned long long thread_id = + ((unsigned long long)thread_id_hi << 32) | thread_id_lo; + const unsigned long long inst_id = + ((unsigned long long)inst_id_hi << 32) | inst_id_lo; + const unsigned long long data_lo = + ((unsigned long long)data_lo_hi << 32) | data_lo_lo; + const unsigned long long data_hi = + ((unsigned long long)data_hi_hi << 32) | data_hi_lo; char data_hex[35]; if (is_write) { u128_hex(data_hex, sizeof(data_hex), data_hi, data_lo); - fprintf( - mtrace_fp, - "{\"type\":\"mtrace\",\"clk\":%llu,\"event\":\"write\"," - "\"channel\":%u," - "\"hart_id\":%llu,\"is_shared\":%u,\"vbank_id\":%u,\"pbank_id\":%u," - "\"group_id\":%u,\"addr\":\"0x%08x\",\"data\":\"%s\"}\n", - (unsigned long long)bdb_rtl_clk, channel, hart_id, is_shared, - vbank_id, pbank_id, group_id, addr, data_hex); + fprintf(mtrace_fp, + "{\"type\":\"mtrace\",\"clk\":%llu,\"scope\":\"%s\",\"event\":" + "\"write\"," + "\"channel\":%u," + "\"thread_id\":%llu,\"rob_id\":%u,\"inst_id\":%llu,\"is_shared\":" + "%u,\"vbank_id\":%u,\"pbank_id\":%u," + "\"group_id\":%u,\"addr\":\"0x%08x\",\"write_mask\":%u,\"data\":" + "\"%s\"}\n", + (unsigned long long)bdb_rtl_clk, scope, channel, thread_id, + rob_id, inst_id, is_shared, vbank_id, pbank_id, group_id, addr, + write_mask, data_hex); } else { - fprintf( - mtrace_fp, - "{\"type\":\"mtrace\",\"clk\":%llu,\"event\":\"read\",\"channel\":%u," - "\"hart_id\":%llu,\"is_shared\":%u,\"vbank_id\":%u,\"pbank_id\":%u," - "\"group_id\":%u,\"addr\":\"0x%08x\"}\n", - (unsigned long long)bdb_rtl_clk, channel, hart_id, is_shared, - vbank_id, pbank_id, group_id, addr); + fprintf(mtrace_fp, + "{\"type\":\"mtrace\",\"clk\":%llu,\"scope\":\"%s\",\"event\":" + "\"read\",\"channel\":%u," + "\"thread_id\":%llu,\"rob_id\":%u,\"inst_id\":%llu,\"is_shared\":" + "%u,\"vbank_id\":%u,\"pbank_id\":%u," + "\"group_id\":%u,\"addr\":\"0x%08x\"}\n", + (unsigned long long)bdb_rtl_clk, scope, channel, thread_id, + rob_id, inst_id, is_shared, vbank_id, pbank_id, group_id, addr); } fflush(mtrace_fp); } diff --git a/arch/src/csrc/src/monitor/trace/pmctrace.cc b/arch/src/csrc/src/monitor/trace/pmctrace.cc index 62fee5011..a3c973d20 100644 --- a/arch/src/csrc/src/monitor/trace/pmctrace.cc +++ b/arch/src/csrc/src/monitor/trace/pmctrace.cc @@ -21,10 +21,12 @@ static void init_pmctrace() { // DPI-C function for Ball PMC trace // Called when a Ball completes a task, reports elapsed cycles extern "C" void dpi_pmctrace(unsigned int ball_id, unsigned int rob_id, - unsigned long long elapsed) { + unsigned int elapsed_lo, unsigned int elapsed_hi) { if (!bdb_trace_on(BDB_TR_PMCTRACE)) { return; } + const unsigned long long elapsed = + ((unsigned long long)elapsed_hi << 32) | elapsed_lo; init_pmctrace(); if (pmctrace_fp) { diff --git a/arch/src/main/resources/bootrom/bootrom.S b/arch/src/main/resources/bootrom/bootrom.S index 453601beb..ddbb5cd8a 100644 --- a/arch/src/main/resources/bootrom/bootrom.S +++ b/arch/src/main/resources/bootrom/bootrom.S @@ -1,6 +1,6 @@ # Buckyball BootROM: validate mhartid and jump to the DRAM entry in M-mode. # The SoC fills the two dwords at _config (ROM offsets 0x8 and 0x10) when it -# builds the ROM, so this image is the same for every chip. OpenSBI takes the +# builds the ROM, so this image is the same for every design. OpenSBI takes the # DTB embedded in its fw_payload. .section .text diff --git a/arch/src/main/scala/configs/GlobalConfig.scala b/arch/src/main/scala/configs/GlobalConfig.scala new file mode 100644 index 000000000..b0ec9081c --- /dev/null +++ b/arch/src/main/scala/configs/GlobalConfig.scala @@ -0,0 +1,28 @@ +package configs + +import buckyball.config.Design +import framework.core.configs.CoreParam +import scala.jdk.CollectionConverters._ + +import chisel3.experimental.SerializableModuleParameter +import framework.device.configs.DeviceParam +import sims.configs.SimParam + +case class GlobalConfig( + device: DeviceParam, + sim: SimParam) + extends SerializableModuleParameter + +object GlobalConfig { + + def fromProto(design: Design, cores: Seq[CoreParam]): GlobalConfig = { + require(design.hasDevice && design.hasSims, "Design device and simulation configuration are required") + GlobalConfig(DeviceParam.fromProto(design.getDevice, cores), SimParam.fromProto(design.getSims)) + } + + def fromProto(design: Design): GlobalConfig = { + require(design.getThreadsCount == 0, "Thread implementations must supply their core parameters") + fromProto(design, design.getDevice.getCoresList.asScala.map(p => CoreParam.fromProto(p, Seq.empty)).toSeq) + } + +} diff --git a/arch/src/main/scala/framework/acc/balldomain/build.sc b/arch/src/main/scala/framework/acc/balldomain/build.sc new file mode 100644 index 000000000..87ff3a23b --- /dev/null +++ b/arch/src/main/scala/framework/acc/balldomain/build.sc @@ -0,0 +1,33 @@ +import mill._ +import $file.src.main.scala.ball.{build => ball_build} +import $file.src.main.scala.bbus.{build => bbus_build} +import $file.^.^.^.^.^.^.settings + + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/BallDomain.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/mmio_bank/MmioBank.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/sub_rob/SubROB.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/sub_rob/Expansion.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/sub_rob/configs/Params.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/BallDomainParam.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/decoder/DISA.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/decoder/Decoder.scala")) + ) ++ + ball_build.sources() ++ + bbus_build.sources() +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) ++ + ball_build.emitSources() ++ + bbus_build.emitSources() +) diff --git a/arch/src/main/scala/framework/acc/balldomain/src/main/scala/BallDomain.scala b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/BallDomain.scala new file mode 100644 index 000000000..fd945f6e5 --- /dev/null +++ b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/BallDomain.scala @@ -0,0 +1,198 @@ +package framework.acc.balldomain + +import framework.lib.arbiter.{Arbiter} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.core.memory.access.preflight.configs.PmpParams +import framework.acc.balldomain.ball.blink.BallResponse +import framework.acc.balldomain.bbus.{BBus, BallFactory, BallRequest} +import framework.acc.balldomain.decoder.Decoder +import framework.acc.balldomain.sub_rob.{SubROB, SubRobRow} +import framework.acc.balldomain.mmio_bank.{MetadataWrite, MmioBank} +import framework.acc.frontend.scheduler.{Complete, Issue, MemoryIssue} +import framework.acc.memdomain.{BankLayout, BankLayoutRequest, BankPort, Response => MemoryResponse} + +/** GP kernel Ball calls; the GP domain resolves its launch groups before issuing. */ +class KernelBallPort(f: FrontendParam, m: MemDomainParam) extends Bundle { + val request = Decoupled(new BallRequest(f, m)) + val response = Flipped(Decoupled(new BallResponse(f))) +} + +/** BallDomain owns child identities; MemDomain receives ordinary tagged memory operations. */ +class DomainMemory(f: FrontendParam, m: MemDomainParam, p: PmpParams) extends Bundle { + val request = Decoupled(new MemoryIssue(f, m, p)) + val response = Flipped(Decoupled(new MemoryResponse(f))) + val metadata = new MetadataWrite(m) +} + +@instantiable +class BallDomain( + fp: FrontendParam, + mp: MemDomainParam, + gp: GpDomainParam, + bp: BallDomainParam, + pp: PmpParams, + factories: Map[String, BallFactory]) + extends Module { + + @public + val io = IO(new Bundle { + val issue = Flipped(Decoupled(new MemoryIssue(fp, mp, pp))) + val complete = Decoupled(new MemoryResponse(fp)) + val memory = new DomainMemory(fp, mp, pp) + val banks = Vec(mp.bankRequestPorts, new BankPort(fp, mp)) + val layoutRequest = Decoupled(new BankLayoutRequest(fp)) + val layoutResponse = Flipped(Decoupled(new BankLayout(mp))) + val kernel = Option.when(gp.enable)(Flipped(new KernelBallPort(fp, mp))) + val traceStarted = Output(Vec(bp.balls.size, Valid(UInt(log2Ceil(fp.rob_entries).W)))) + val traceCompleted = Output(Vec(bp.balls.size, Valid(UInt(log2Ceil(fp.rob_entries).W)))) + }) + + val bbus: Instance[BBus] = Instantiate(new BBus(fp, mp, gp, bp, factories)) + val expander = bbus.expansion + if (expander.nonEmpty) require( + mp.mmioEnable && !fp.bank_alias_enable, + "Expanded instructions own architectural scratch banks until their parent completes" + ) + val decoder: Instance[Decoder] = Instantiate(new Decoder(fp, mp, bp)) + bbus.io.cmdReq <> decoder.io.command + + // Frontend and SubROB work enter as an Issue; the Ball sees only its BLink command. + val operation = Wire(new Issue(fp, mp)) + val access = operation.renamedBankAccess + val request = decoder.io.issue.bits + request.target := 0.U + request.instId := operation.instId + request.enabled(0) := access.rd_bank_0_valid + request.enabled(1) := access.rd_bank_1_valid + request.enabled(2) := access.wr_bank_valid + request.cmd.rob_id := operation.rob_id + request.cmd.funct := operation.cmd.cmd.raw_inst(31, 25) + request.cmd.rs2 := operation.cmd.cmd.rs2Data + request.cmd.iter := operation.cmd.iter + request.cmd.op1_bank := access.rd_bank_0_id + request.cmd.op2_bank := access.rd_bank_1_id + request.cmd.wr_bank := access.wr_bank_id + request.cmd.op1_group := 0.U + request.cmd.op2_group := 0.U + request.cmd.wr_group := 0.U + request.cmd.op1_col := operation.cmd.op1_col + request.cmd.op2_col := operation.cmd.op2_col + request.cmd.wr_col := operation.cmd.wr_col + request.cmd.wr_read_bank := access.rd_wr_id + request.cmd.wr_arch_bank := operation.cmd.architecturalDestination + io.banks <> bbus.io.banks + io.layoutRequest <> bbus.io.layoutRequest + bbus.io.layoutResponse <> io.layoutResponse + io.traceStarted := bbus.io.traceStarted + io.traceCompleted := bbus.io.traceCompleted + if (gp.enable) { + val kernelDecoder: Instance[Decoder] = Instantiate(new Decoder(fp, mp, bp)) + kernelDecoder.io.issue <> io.kernel.get.request + bbus.io.kernelReq.get <> kernelDecoder.io.command + io.kernel.get.response <> bbus.io.kernelResp.get + } + if (mp.mmioEnable) { + val reads = bp.balls.map(_.mmioReadBW).sum + val writes = bp.balls.map(_.mmioWriteBW).sum + val mmio: Instance[MmioBank] = Instantiate(new MmioBank(mp, reads + expander.size, writes + expander.size)) + mmio.io.upload <> io.memory.metadata + for (i <- 0 until reads) mmio.io.reads(i) <> bbus.io.mmioRead(i) + for (i <- 0 until writes) mmio.io.writes(i) <> bbus.io.mmioWrite(i) + expander.foreach { e => mmio.io.reads(reads) <> e.read; mmio.io.writes(writes) <> e.write } + } else { + require(bp.balls.forall(b => b.mmioReadBW == 0 && b.mmioWriteBW == 0)) + io.memory.metadata.request.ready := false.B + io.memory.metadata.response.valid := false.B + io.memory.metadata.response.bits := false.B + when(io.memory.metadata.request.valid)(assert(false.B, "MMIO metadata bank is disabled")) + } + + io.complete.bits := 0.U.asTypeOf(io.complete.bits) + io.memory.request.valid := false.B + io.memory.request.bits := 0.U.asTypeOf(io.memory.request.bits) + io.memory.response.ready := false.B + expander match { + case None => + decoder.io.issue.valid := io.issue.valid + operation := io.issue.bits.operation + io.issue.ready := decoder.io.issue.ready + io.complete.valid := bbus.io.cmdResp.valid + io.complete.bits.rob_id := bbus.io.cmdResp.bits.rob_id + bbus.io.cmdResp.ready := io.complete.ready + case Some(e) => + val sub: Instance[SubROB] = Instantiate(new SubROB(fp, mp, bp)) + val active = RegInit(false.B) + val parent = Reg(chiselTypeOf(io.issue.bits)) + val ballPending = RegInit(false.B) + val memoryPending = RegInit(false.B) + val ballTag = Reg(chiselTypeOf(sub.io.issueSubId(0))) + val memoryTag = Reg(chiselTypeOf(sub.io.issueSubId(1))) + val faulted = RegInit(false.B) + val faultReported = RegInit(false.B) + val fault = Reg(chiselTypeOf(io.complete.bits.fault)) + e.command.bits.rob_id := io.issue.bits.operation.rob_id + e.command.bits.funct := io.issue.bits.operation.cmd.cmd.raw_inst(31, 25) + e.command.bits.rs2 := io.issue.bits.operation.cmd.cmd.rs2Data + io.issue.ready := !active && !faulted && Mux(e.accepts, e.command.ready, decoder.io.issue.ready) + e.command.valid := io.issue.valid && e.accepts && !active && !faulted + when(io.issue.fire && e.starts) { active := true.B; parent := io.issue.bits } + sub.io.write <> e.rows + sub.io.write.valid := e.rows.valid && active && !faulted + e.rows.ready := sub.io.write.ready && active && !faulted + val child = Wire(new Issue(fp, mp)) + child := parent.operation + child.cmd := sub.io.issue(0).bits + child.cmd.cmd.pc := parent.operation.cmd.cmd.pc + child.renamedBankAccess := sub.io.issue(0).bits.bankAccess + decoder.io.issue.valid := !faulted && Mux( + active, + sub.io.issue(0).valid && !ballPending, + io.issue.valid && !e.accepts + ) + operation := Mux(active, child, io.issue.bits.operation) + sub.io.issue(0).ready := active && !faulted && !ballPending && decoder.io.issue.ready + when(sub.io.issue(0).fire) { ballPending := true.B; ballTag := sub.io.issueSubId(0) } + io.memory.request.valid := active && !faulted && sub.io.issue(1).valid && !memoryPending + io.memory.request.bits := parent + io.memory.request.bits.operation.cmd := sub.io.issue(1).bits + io.memory.request.bits.operation.cmd.cmd.pc := parent.operation.cmd.cmd.pc + io.memory.request.bits.operation.renamedBankAccess := sub.io.issue(1).bits.bankAccess + sub.io.issue(1).ready := active && !faulted && !memoryPending && io.memory.request.ready + when(io.memory.request.fire) { memoryPending := true.B; memoryTag := sub.io.issueSubId(1) } + val completed = Instantiate(new Arbiter(chiselTypeOf(ballTag), 2)) + completed.io.in(0).valid := active && ballPending && bbus.io.cmdResp.valid + completed.io.in(0).bits := ballTag + completed.io.in(1).valid := active && memoryPending && io.memory.response.valid + completed.io.in(1).bits := memoryTag + sub.io.subComplete <> completed.io.out + io.memory.response.ready := memoryPending && completed.io.in(1).ready + when(completed.io.in(0).fire)(ballPending := false.B) + when(io.memory.response.fire) { + memoryPending := false.B + when(io.memory.response.bits.fault.error =/= 0.U) { + faulted := true.B; fault := io.memory.response.bits.fault + } + } + val responses = Instantiate(new Arbiter(new Complete(fp), 3)) + responses.io.in(0).valid := !active && bbus.io.cmdResp.valid + responses.io.in(0).bits.rob_id := bbus.io.cmdResp.bits.rob_id + responses.io.in(1) <> e.complete + responses.io.in(2).valid := sub.io.masterComplete.valid && !faulted + responses.io.in(2).bits.rob_id := sub.io.masterComplete.bits + sub.io.masterComplete.ready := responses.io.in(2).ready && !faulted + when(sub.io.masterComplete.fire)(active := false.B) + bbus.io.cmdResp.ready := Mux(active, ballPending && completed.io.in(0).ready, responses.io.in(0).ready) + io.complete.valid := Mux(faulted, !faultReported, responses.io.out.valid) + io.complete.bits.rob_id := Mux(faulted, parent.operation.rob_id, responses.io.out.bits.rob_id) + io.complete.bits.fault := Mux(faulted, fault, 0.U.asTypeOf(fault)) + responses.io.out.ready := io.complete.ready && !faulted + when(io.complete.fire && faulted)(faultReported := true.B) + } +} diff --git a/arch/src/main/scala/framework/acc/balldomain/src/main/scala/Emit.scala b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/Emit.scala new file mode 100644 index 000000000..c7fdc1969 --- /dev/null +++ b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/Emit.scala @@ -0,0 +1,39 @@ +package framework.acc.balldomain + +import framework.acc.balldomain.bbus.BallFactory +import framework.acc.balldomain.ball.blink.BlinkIO + +import chisel3.Module +import _root_.configs.GlobalConfig +import framework.acc.balldomain.ball.blink.HasBlink + +object Emit { + + def emit( + b: GlobalConfig, + factories: Map[String, BallFactory], + hartId: Int, + threadId: Int, + directory: String, + firtoolOpts: Array[String] + ): Unit = { + val core = + b.device.cores.find(_.hartId == hartId).getOrElse(throw new IllegalArgumentException(s"Unknown hart ID: $hartId")) + val thread = + core.threads.find(_.id == threadId).getOrElse(throw new IllegalArgumentException(s"Unknown thread ID: $threadId")) + + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new BallDomain( + thread.acc.get.frontend, + thread.acc.get.memDomain, + thread.acc.get.gpDomain, + thread.acc.get.ballDomain, + core.privileged.permission, + factories + ), + firtoolOpts = firtoolOpts, + args = Array("--target-dir", directory, "--split-verilog") + ) + } + +} diff --git a/arch/src/main/scala/framework/acc/balldomain/src/main/scala/ball/blink/baseball.scala b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/ball/blink/baseball.scala new file mode 100644 index 000000000..341fce7b7 --- /dev/null +++ b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/ball/blink/baseball.scala @@ -0,0 +1,27 @@ +//===- baseball.scala ------ Ball Interface Contract ----------------------===// +// +// Copyright 2026 The Buckyball Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// +package framework.acc.balldomain.ball.blink + +import chisel3.experimental.hierarchy.{instantiable, public} + +// All Balls expose the same instance boundary. +@instantiable +trait HasBlink { + @public + val blink: BlinkIO +} diff --git a/arch/src/main/scala/framework/acc/balldomain/src/main/scala/ball/blink/blink.scala b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/ball/blink/blink.scala new file mode 100644 index 000000000..6567f678f --- /dev/null +++ b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/ball/blink/blink.scala @@ -0,0 +1,76 @@ +//===- blink.scala --------- Ball Link Interface --------------------------===// +// +// Copyright 2026 The Buckyball Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// +package framework.acc.balldomain.ball.blink + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import framework.acc.memdomain.{BankRead, BankWrite} +import framework.acc.balldomain.mmio_bank.{MmioRead, MmioWriteReq} + +/** + * One Ball operation. Banks are already renamed; each operand covers `col` groups starting at `group`. + */ +class BallCommand(fp: FrontendParam, mp: MemDomainParam) extends Bundle { + val rob_id = UInt(log2Up(fp.rob_entries).W) + val funct = UInt(7.W) + val rs2 = UInt(fp.commandDataBits.W) + val iter = UInt(fp.iter_len.W) + val op1_bank = UInt(fp.bank_id_len.W) + val op2_bank = UInt(fp.bank_id_len.W) + val wr_bank = UInt(fp.bank_id_len.W) + val op1_group = UInt(mp.groupIdWidth.W) + val op2_group = UInt(mp.groupIdWidth.W) + val wr_group = UInt(mp.groupIdWidth.W) + val op1_col = UInt(mp.groupCountWidth.W) + val op2_col = UInt(mp.groupCountWidth.W) + val wr_col = UInt(mp.groupCountWidth.W) + // Renamed bank holding the previous destination value for accumulation. + val wr_read_bank = UInt(fp.bank_id_len.W) + // Destination bank before renaming; identifies chained writes to one architectural bank. + val wr_arch_bank = UInt(fp.bank_id_len.W) +} + +class BallResponse(fp: FrontendParam) extends Bundle { + val rob_id = UInt(log2Up(fp.rob_entries).W) +} + +class BallStatus extends Bundle { + val idle = Output(Bool()) + val running = Output(Bool()) +} + +class BlinkIO( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Bundle { + val ball = bp.balls.find(_.id == id).get + + val mmioRead = Vec(ball.mmioReadBW, Flipped(new MmioRead(mp))) + val mmioWrite = Vec(ball.mmioWriteBW, Decoupled(new MmioWriteReq(mp))) + val status = new BallStatus + val cmdReq = Flipped(Decoupled(new BallCommand(fp, mp))) + val cmdResp = Decoupled(new BallResponse(fp)) + val bankRead = Vec(ball.inBW, Flipped(new BankRead(fp, mp))) + val bankWrite = Vec(ball.outBW, Flipped(new BankWrite(fp, mp))) +} diff --git a/arch/src/main/scala/framework/acc/balldomain/src/main/scala/ball/blink/build.sc b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/ball/blink/build.sc new file mode 100644 index 000000000..1b997830d --- /dev/null +++ b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/ball/blink/build.sc @@ -0,0 +1,12 @@ +import mill._ + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("baseball.scala")), + PathRef(millSourcePath / os.RelPath("blink.scala")) + ) +) + +def emitSources = T.sources( + Seq.empty[PathRef] +) diff --git a/arch/src/main/scala/framework/acc/balldomain/src/main/scala/ball/build.sc b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/ball/build.sc new file mode 100644 index 000000000..d208a889f --- /dev/null +++ b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/ball/build.sc @@ -0,0 +1,10 @@ +import mill._ +import $file.blink.{build => blink_build} + +def sources = T.sources( + blink_build.sources() +) + +def emitSources = T.sources( + blink_build.emitSources() +) diff --git a/arch/src/main/scala/framework/acc/balldomain/src/main/scala/bbus/bbus.scala b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/bbus/bbus.scala new file mode 100644 index 000000000..8d27bd9fc --- /dev/null +++ b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/bbus/bbus.scala @@ -0,0 +1,192 @@ +package framework.acc.balldomain.bbus + +import framework.lib.queue.Queue +import framework.lib.arbiter.{Arbiter, RRArbiter} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.balldomain.sub_rob.Expansion +import framework.acc.balldomain.ball.blink.BlinkIO +import framework.acc.frontend.configs.FrontendParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.memdomain.{BankLayout, BankLayoutRequest, BankPort} +import framework.acc.balldomain.bbus.channels.Channels +import framework.acc.balldomain.ball.blink.{BallCommand, BallResponse} +import framework.acc.balldomain.decoder.DISA.BALL_INIT_BITPAT +import framework.acc.balldomain.mmio_bank.{MmioRead, MmioWriteReq} + +/** A Ball command routed to `target`; `enabled` marks which of op1, op2 and wr name a bank. */ +class BallRequest(fp: FrontendParam, mp: MemDomainParam) extends Bundle { + val target = UInt(5.W) + val instId = UInt(64.W) // Observation identity; never controls scheduling. + val enabled = Vec(3, Bool()) + val cmd = new BallCommand(fp, mp) +} + +/** Interconnect for the Ball instances listed in the BallDomain configuration. */ +@instantiable +class BBus( + fp: FrontendParam, + mp: MemDomainParam, + gp: GpDomainParam, + bp: BallDomainParam, + factories: Map[String, BallFactory]) + extends Module { + val numBalls = bp.balls.size + + @public + val io = IO(new Bundle { + val mmioRead = Vec(bp.balls.map(_.mmioReadBW).sum, Flipped(new MmioRead(mp))) + val mmioWrite = Vec(bp.balls.map(_.mmioWriteBW).sum, Decoupled(new MmioWriteReq(mp))) + val traceStarted = Output(Vec(numBalls, Valid(UInt(log2Ceil(fp.rob_entries).W)))) + val traceCompleted = Output(Vec(numBalls, Valid(UInt(log2Ceil(fp.rob_entries).W)))) + // Frontend commands still need their group counts from the bank layout. + val cmdReq = Flipped(Decoupled(new BallRequest(fp, mp))) + val cmdResp = Decoupled(new BallResponse(fp)) + val layoutRequest = Decoupled(new BankLayoutRequest(fp)) + val layoutResponse = Flipped(Decoupled(new BankLayout(mp))) + val banks = Vec(mp.bankRequestPorts, new BankPort(fp, mp)) + // GP kernel commands arrive complete and complete back to the kernel. + val kernelReq = Option.when(gp.enable)(Flipped(Decoupled(new BallRequest(fp, mp)))) + val kernelResp = Option.when(gp.enable)(Decoupled(new BallResponse(fp))) + }) + + val channels: Instance[Channels] = Instantiate(new Channels(fp, mp, bp)) + val instructionIds = Reg(Vec(numBalls, UInt(64.W))) + channels.io.instructionIds := instructionIds + io.banks <> channels.io.mem_req + + // Apply BALL_INIT on the cycle after the command handshake. Driving reset + // directly from cmd.fire would feed the Ball's reset-dependent ready signal + // back into that same handshake. + val ballBootReset = RegInit(VecInit(Seq.fill(numBalls)(false.B))) + + val instances = bp.balls.zipWithIndex.map { case (ball, index) => + withReset(reset.asBool || ballBootReset(index)) { + factories(ball.component).instantiate(fp, mp, bp, ball.id) + } + } + + val balls = instances.map(_.blink) + val expanded = instances.flatMap(_.expansion) + require(expanded.size <= 1, "One instruction expander is supported per BallDomain") + require(expanded.nonEmpty == fp.sub_rob_enable) + + @public + val expansion: Option[Expansion] = expanded.headOption.map { source => + val port = IO(new Expansion(fp, mp, bp)) + port <> source + port + } + + balls.flatMap(_.mmioRead).zip(io.mmioRead).foreach { case (source, port) => port <> source } + balls.flatMap(_.mmioWrite).zip(io.mmioWrite).foreach { case (source, port) => port <> source } + + val pending = Instantiate(new Queue(new BallRequest(fp, mp), 4)) + io.layoutRequest.valid := io.cmdReq.valid && pending.io.enq.ready + pending.io.enq.valid := io.cmdReq.valid && io.layoutRequest.ready + pending.io.enq.bits := io.cmdReq.bits + io.cmdReq.ready := pending.io.enq.ready && io.layoutRequest.ready + io.layoutRequest.bits.banks(0) := io.cmdReq.bits.cmd.op1_bank + io.layoutRequest.bits.banks(1) := io.cmdReq.bits.cmd.op2_bank + io.layoutRequest.bits.banks(2) := io.cmdReq.bits.cmd.wr_bank + io.layoutRequest.bits.enabled := io.cmdReq.bits.enabled + val external = Wire(Decoupled(new BallRequest(fp, mp))) + external.valid := pending.io.deq.valid && io.layoutResponse.valid + external.bits := pending.io.deq.bits + external.bits.cmd.op1_col := io.layoutResponse.bits.groups(0) + external.bits.cmd.op2_col := io.layoutResponse.bits.groups(1) + external.bits.cmd.wr_col := io.layoutResponse.bits.groups(2) + pending.io.deq.ready := external.ready && io.layoutResponse.valid + io.layoutResponse.ready := external.ready && pending.io.deq.valid + + val command = Wire(Decoupled(new BallRequest(fp, mp))) + val fromKernel = WireDefault(false.B) + val kernelOwner = RegInit(VecInit(Seq.fill(numBalls)(false.B))) + val kernelPending = WireDefault(false.B) + if (gp.enable) { + val arbiter = Instantiate(new RRArbiter(new BallRequest(fp, mp), 2)) + arbiter.io.in(0) <> external + arbiter.io.in(1) <> io.kernelReq.get + command <> arbiter.io.out + fromKernel := arbiter.io.chosen === 1.U + assert(PopCount(kernelOwner) <= 1.U, "GP domain issues only one outstanding Ball operation") + } else { + command <> external + } + + val isBallInit = command.bits.cmd.funct === BALL_INIT_BITPAT + val initPending = RegInit(VecInit(Seq.fill(numBalls)(false.B))) + val initResp = Reg(Vec(numBalls, new BallResponse(fp))) + val targetMatches = VecInit(bp.balls.map(b => command.bits.target === b.id.U)) + val responses = Instantiate(new Arbiter(new BallResponse(fp), numBalls)) + val kernelDone = Wire(Vec(numBalls, Valid(new BallResponse(fp)))) + + for (i <- 0 until numBalls) { + channels.io.active(i) := balls(i).status.running + + val targetMatch = targetMatches(i) + balls(i).cmdReq.valid := command.valid && !isBallInit && !initPending(i) && targetMatch + balls(i).cmdReq.bits := command.bits.cmd + + val done = Mux(initPending(i), true.B, balls(i).cmdResp.valid) + val resp = Mux(initPending(i), initResp(i), balls(i).cmdResp.bits) + responses.io.in(i).valid := !kernelOwner(i) && done + responses.io.in(i).bits := resp + kernelDone(i).valid := kernelOwner(i) && done + kernelDone(i).bits := resp + + val accepted = Mux(kernelOwner(i), io.kernelResp.map(_.ready).getOrElse(false.B), responses.io.in(i).ready) + balls(i).cmdResp.ready := !initPending(i) && accepted + when(done && accepted) { + initPending(i) := false.B + kernelOwner(i) := false.B + } + + when(command.fire && targetMatch) { + instructionIds(i) := command.bits.instId + kernelOwner(i) := fromKernel + } + ballBootReset(i) := command.fire && isBallInit && targetMatch + when(command.fire && isBallInit && targetMatch) { + initPending(i) := true.B + initResp(i).rob_id := command.bits.cmd.rob_id + } + + io.traceStarted(i).valid := command.fire && targetMatch + io.traceStarted(i).bits := command.bits.cmd.rob_id + io.traceCompleted(i).valid := done && accepted + io.traceCompleted(i).bits := resp.rob_id + } + + io.kernelResp.foreach { port => + port.valid := kernelDone.map(_.valid).reduce(_ || _) + port.bits := Mux1H(kernelDone.map(_.valid), kernelDone.map(_.bits)) + } + + command.ready := + VecInit((0 until numBalls).map(i => balls(i).cmdReq.ready && !initPending(i) && targetMatches(i))).asUInt.orR + + io.cmdResp <> responses.io.out + + // Ball bank ports occupy consecutive Channel slots in configuration order. + channels.io.bankRead.zip(balls.flatMap(_.bankRead)).foreach { case (channel, port) => channel <> port } + channels.io.bankWrite.zip(balls.flatMap(_.bankWrite)).foreach { case (channel, port) => channel <> port } +} + +case class BallPorts(blink: BlinkIO, expansion: Option[Expansion]) + +trait BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts + +} diff --git a/arch/src/main/scala/framework/acc/balldomain/src/main/scala/bbus/build.sc b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/bbus/build.sc new file mode 100644 index 000000000..f3aff3ac2 --- /dev/null +++ b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/bbus/build.sc @@ -0,0 +1,39 @@ +import mill._ + +// The component library owns the generated table; framework only knows the contract. +def registry(sources: Seq[PathRef], componentRoot: os.Path, destination: os.Path): Seq[PathRef] = { + val declarations = sources.flatMap { source => + val text = os.read(source.path) + val names = "(?m)^object (\\w+) extends BallFactory".r.findAllMatchIn(text).map(_.group(1)).toSeq + if (names.isEmpty) Seq.empty + else { + val pkg = "(?m)^package ([\\w.]+)".r.findFirstMatchIn(text).get.group(1) + val component = source.path.relativeTo(componentRoot).segments.head + names.map(name => component -> s"$pkg.$name") + } + }.sortBy(_._1) + require(declarations.map(_._1).distinct.size == declarations.size, "Duplicate component factory") + val entries = declarations.map { case (component, factory) => s""""$component" -> $factory""" } + val output = destination / "BallComponents.scala" + os.write.over( + output, + s"""package framework.acc.balldomain.bbus.generated +import framework.acc.balldomain.bbus.BallFactory +object BallComponents { + val factories: Map[String, BallFactory] = Map(${entries.mkString(",\n")}) +} +""" + ) + Seq(PathRef(output)) +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("bbus.scala")), + PathRef(millSourcePath / os.RelPath("channels/Channels.scala")) + ) +) + +def emitSources = T.sources( + Seq.empty[PathRef] +) diff --git a/arch/src/main/scala/framework/acc/balldomain/src/main/scala/bbus/channels/Channels.scala b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/bbus/channels/Channels.scala new file mode 100644 index 000000000..1a0654093 --- /dev/null +++ b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/bbus/channels/Channels.scala @@ -0,0 +1,202 @@ +package framework.acc.balldomain.bbus.channels + +import chisel3.experimental.hierarchy.Instantiate + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import framework.lib.queue.Queue +import chisel3._ +import chisel3.util._ +import framework.acc.memdomain.{BankRead, BankWrite} +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.acc.memdomain.BankPort + +/** + * Channel allocation and release + * Connects memory clients to the layout table + * + * Configured Ball read and write ports retain their channels until completion. + * The port ordering is determined by the Ball connection configuration. + * Requests retain their channel until the owning Ball completes and responses drain. + */ +@instantiable +class Channels(fp: FrontendParam, mp: MemDomainParam, bp: BallDomainParam) extends Module { + // Each configured Ball owns its declared read and write ports. + val ownerCount = bp.balls.size + val totalRead = bp.balls.map(_.inBW).sum + val totalWrite = bp.balls.map(_.outBW).sum + + @public + val io = IO(new Bundle { + // Ball read and write ports in configuration order. + val bankRead = Vec(totalRead, new BankRead(fp, mp)) + val bankWrite = Vec(totalWrite, new BankWrite(fp, mp)) + val active = Input(Vec(bp.balls.size, Bool())) + + val instructionIds = Input(Vec(ownerCount, UInt(64.W))) + + // Output to the layout table + val mem_req = Vec(mp.bankRequestPorts, new BankPort(fp, mp)) + }) + + val readPortOwner = bp.balls.zipWithIndex.flatMap { case (ball, owner) => Seq.fill(ball.inBW)(owner) } + val writePortOwner = bp.balls.zipWithIndex.flatMap { case (ball, owner) => Seq.fill(ball.outBW)(owner) } + + val readInstructionIds = Wire(Vec(totalRead, UInt(64.W))) + val readClients = Wire(Vec(totalRead, new BankRead(fp, mp))) + for (i <- 0 until totalRead) { + val source = io.bankRead(i) + val request = Wire(Decoupled(new Bundle { + val bank = chiselTypeOf(source.bank_id) + val rob = chiselTypeOf(source.rob_id) + val group = chiselTypeOf(source.group_id) + val address = chiselTypeOf(source.io.req.bits) + val instId = UInt(64.W) + })) + request.valid := source.io.req.valid + request.bits.bank := source.bank_id + request.bits.rob := source.rob_id + request.bits.group := source.group_id + request.bits.address := source.io.req.bits + request.bits.instId := io.instructionIds(readPortOwner(i)) + source.io.req.ready := request.ready + val queue = Instantiate(new Queue(chiselTypeOf(request.bits), 8, false, false)) + queue.io.enq <> request + readInstructionIds(i) := queue.io.deq.bits.instId + val target = readClients(i) + target.io.req.valid := queue.io.deq.valid + target.io.req.bits := queue.io.deq.bits.address + target.bank_id := queue.io.deq.bits.bank + target.rob_id := queue.io.deq.bits.rob + target.group_id := queue.io.deq.bits.group + queue.io.deq.ready := target.io.req.ready + source.io.resp <> target.io.resp + } + + // ----------------------------------------------------------------------------- + // Mapping table for configured Ball request ports + // ----------------------------------------------------------------------------- + class MappingTableEntry extends Bundle { + val valid = Bool() + val isRead = Bool() + val id = UInt(log2Ceil(math.max(totalRead, totalWrite)).W) + val owner = UInt(math.max(1, log2Ceil(ownerCount)).W) + } + + val mappingTable = RegInit(VecInit( + Seq.fill(mp.bankRequestPorts)(0.U.asTypeOf(new MappingTableEntry)) + )) + + // A route stays assigned until its Ball releases it, so maintain the reverse ownership at allocation time. This keeps + // readiness and pending detection from repeatedly scanning every backend + // request port for the same port. + val portSlots = math.max(totalRead, totalWrite) + val readPortAllocated = RegInit(VecInit(Seq.fill(portSlots)(false.B))) + val writePortAllocated = RegInit(VecInit(Seq.fill(portSlots)(false.B))) + val channelPending = + RegInit(VecInit(Seq.fill(mp.bankRequestPorts)(false.B))) + + // Allocate or release one channel per cycle, retaining its owner through response drain. + for (i <- 0 until totalRead) { + readClients(i).io.req.ready := false.B + readClients(i).io.resp.valid := false.B + readClients(i).io.resp.bits := DontCare + + } + + for (i <- 0 until totalWrite) { + io.bankWrite(i).io.req.ready := false.B + io.bankWrite(i).io.resp.valid := false.B + io.bankWrite(i).io.resp.bits := DontCare + } + + val pendingReads = VecInit((0 until totalRead).map(i => io.active(readPortOwner(i)) && !readPortAllocated(i))) + val pendingWrites = VecInit((0 until totalWrite).map(i => io.active(writePortOwner(i)) && !writePortAllocated(i))) + + val pending = VecInit(pendingReads ++ pendingWrites) + val freePorts = mappingTable.map(entry => !entry.valid) + + // Connect mapped entries to layout request ports + for (i <- 0 until mp.bankRequestPorts) { + io.mem_req(i).read.req.valid := false.B + io.mem_req(i).read.req.bits := DontCare + io.mem_req(i).read.resp.ready := false.B + io.mem_req(i).write.req.valid := false.B + io.mem_req(i).write.req.bits := DontCare + io.mem_req(i).write.resp.ready := false.B + io.mem_req(i).bank_id := 0.U + io.mem_req(i).group_id := 0.U + io.mem_req(i).is_shared := false.B + io.mem_req(i).thread_id := fp.ownerId.U(64.W) + io.mem_req(i).rob_id := 0.U + io.mem_req(i).inst_id := 0.U + + val isRead = mappingTable(i).isRead + val rid = mappingTable(i).id + val wid = mappingTable(i).id + val ballRead = readClients(rid).io + val ballWrite = io.bankWrite(wid).io + val rbank_id = readClients(rid).bank_id + val wbank_id = io.bankWrite(wid).bank_id + val rgroup_id = readClients(rid).group_id + val wgroup_id = io.bankWrite(wid).group_id + val rrob_id = readClients(rid).rob_id + val wrob_id = io.bankWrite(wid).rob_id + + when(mappingTable(i).valid) { + when(isRead) { + io.mem_req(i).read <> ballRead + io.mem_req(i).bank_id := rbank_id + io.mem_req(i).group_id := rgroup_id + io.mem_req(i).rob_id := rrob_id + io.mem_req(i).inst_id := readInstructionIds(rid) + }.otherwise { + io.mem_req(i).write <> ballWrite + io.mem_req(i).bank_id := wbank_id + io.mem_req(i).group_id := wgroup_id + io.mem_req(i).rob_id := wrob_id + io.mem_req(i).inst_id := io.instructionIds(mappingTable(i).owner) + } + } + } + + for (i <- 0 until mp.bankRequestPorts) { + when(io.mem_req(i).read.req.fire || io.mem_req(i).write.req.fire)(channelPending(i) := true.B) + when(io.mem_req(i).read.resp.fire || io.mem_req(i).write.resp.fire)(channelPending(i) := false.B) + } + + val release = VecInit((0 until mp.bankRequestPorts).map { i => + val ownerFinished = !io.active(mappingTable(i).owner) + val drained = !channelPending(i) && !(io.mem_req(i).read.resp.valid || io.mem_req(i).write.resp.valid || + io.mem_req(i).read.req.valid || io.mem_req(i).write.req.valid) + mappingTable(i).valid && ownerFinished && drained + }) + + when(release.asUInt.orR) { + val channel = PriorityEncoder(release) + when(mappingTable(channel).isRead) { + readPortAllocated(mappingTable(channel).id) := false.B + }.otherwise { + writePortAllocated(mappingTable(channel).id) := false.B + } + mappingTable(channel) := 0.U.asTypeOf(new MappingTableEntry) + }.elsewhen(pending.asUInt.orR && freePorts.reduce(_ || _)) { + val channel = PriorityEncoder(freePorts) + val port = PriorityEncoder(pending) + mappingTable(channel).valid := true.B + mappingTable(channel).isRead := port < totalRead.U + mappingTable(channel).id := Mux(port < totalRead.U, port, port - totalRead.U) + mappingTable(channel).owner := Mux( + port < totalRead.U, + MuxLookup(port, 0.U)(readPortOwner.zipWithIndex.map { case (ball, i) => i.U -> ball.U }.toSeq), + MuxLookup(port - totalRead.U, 0.U)(writePortOwner.zipWithIndex.map { case (ball, i) => i.U -> ball.U }.toSeq) + ) + when(port < totalRead.U) { + readPortAllocated(port) := true.B + }.otherwise { + writePortAllocated(port - totalRead.U) := true.B + } + } +} diff --git a/arch/src/main/scala/framework/balldomain/bbus/pmc/ballcyclepmc.spec.en.md b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/bbus/pmc/ballcyclepmc.spec.en.md similarity index 100% rename from arch/src/main/scala/framework/balldomain/bbus/pmc/ballcyclepmc.spec.en.md rename to arch/src/main/scala/framework/acc/balldomain/src/main/scala/bbus/pmc/ballcyclepmc.spec.en.md diff --git a/arch/src/main/scala/framework/balldomain/bbus/pmc/ballcyclepmc.spec.zh.md b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/bbus/pmc/ballcyclepmc.spec.zh.md similarity index 100% rename from arch/src/main/scala/framework/balldomain/bbus/pmc/ballcyclepmc.spec.zh.md rename to arch/src/main/scala/framework/acc/balldomain/src/main/scala/bbus/pmc/ballcyclepmc.spec.zh.md diff --git a/arch/src/main/scala/framework/acc/balldomain/src/main/scala/configs/BallDomainParam.scala b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/configs/BallDomainParam.scala new file mode 100644 index 000000000..6f1717c9e --- /dev/null +++ b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/configs/BallDomainParam.scala @@ -0,0 +1,67 @@ +package framework.acc.balldomain.configs + +import buckyball.config.{BallDomain => BallDomainProto, BallInstance => BallInstanceProto} +import scala.jdk.CollectionConverters._ +import com.google.protobuf.{Any => BallParameters} + +import upickle.default.{macroRW, readwriter, ReadWriter} +import BallRouteParam.parametersRW + +case class InstructionParam(mnemonic: String, funct7: Int) { + require(funct7 >= 0 && funct7 < 128, "funct7 must fit the 7-bit instruction field") +} + +object InstructionParam { implicit val rw: ReadWriter[InstructionParam] = macroRW } + +case class BallRouteParam( + id: Int, + name: String, + inBW: Int, + outBW: Int, + mmioReadBW: Int, + mmioWriteBW: Int, + instructions: Seq[InstructionParam], + component: String, + parameters: BallParameters, + rtlClass: String) { + require(inBW > 0 && outBW > 0, "BBus widths must be positive") + require(id >= 0 && id < 32, "Ball ID must fit the 5-bit target field") + require( + instructions.map(_.mnemonic).distinct.size == instructions.size, + "Duplicate instruction mnemonic in Ball instance" + ) +} + +object BallRouteParam { + + def fromProto(source: BallInstanceProto): BallRouteParam = BallRouteParam( + source.getId, + source.getName, + source.getInBw, + source.getOutBw, + source.getMmioReadBw, + source.getMmioWriteBw, + source.getInstructionsList.asScala.map(i => InstructionParam(i.getMnemonic, i.getFunct7)).toSeq, + source.getComponent, + source.getParameters, + source.getRtlClass + ) + + implicit val parametersRW: ReadWriter[BallParameters] = + readwriter[Array[Byte]].bimap[BallParameters](_.toByteArray, bytes => BallParameters.parseFrom(bytes)) + implicit val rw: ReadWriter[BallRouteParam] = macroRW +} + +case class BallDomainParam(balls: Seq[BallRouteParam]) { + require(balls.map(_.id).distinct.size == balls.size, "Duplicate Ball ID") + require(balls.map(_.name).distinct.size == balls.size, "Duplicate Ball instance name") + + val instructions = balls.flatMap(_.instructions) + require(instructions.map(_.funct7).distinct.size == instructions.size, "Duplicate instruction encoding") +} + +object BallDomainParam { + def fromProto(p: BallDomainProto): BallDomainParam = + BallDomainParam(p.getBallsList.asScala.map(BallRouteParam.fromProto).toSeq) + implicit val rw: ReadWriter[BallDomainParam] = macroRW +} diff --git a/arch/src/main/scala/framework/acc/balldomain/src/main/scala/decoder/DISA.scala b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/decoder/DISA.scala new file mode 100644 index 000000000..e9a32479a --- /dev/null +++ b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/decoder/DISA.scala @@ -0,0 +1,7 @@ +package framework.acc.balldomain.decoder + +import chisel3.util.BitPat + +object DISA { + val BALL_INIT_BITPAT = BitPat("b0000101") +} diff --git a/arch/src/main/scala/framework/acc/balldomain/src/main/scala/decoder/Decoder.scala b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/decoder/Decoder.scala new file mode 100644 index 000000000..48385b480 --- /dev/null +++ b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/decoder/Decoder.scala @@ -0,0 +1,47 @@ +package framework.acc.balldomain.decoder + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.acc.balldomain.bbus.BallRequest + +/** Selects the Ball that owns each funct7; BALL_INIT names its target in rs2[4:0]. */ +@instantiable +class Decoder(fp: FrontendParam, mp: MemDomainParam, bp: BallDomainParam) extends Module { + + @public + val io = IO(new Bundle { + val issue = Flipped(Decoupled(new BallRequest(fp, mp))) + val command = Decoupled(new BallRequest(fp, mp)) + }) + + val funct = io.issue.bits.cmd.funct + val init = funct === DISA.BALL_INIT_BITPAT + val target = WireDefault(0.U(5.W)) + val registered = WireDefault(false.B) + for { + ball <- bp.balls + entry <- ball.instructions + } { + when(funct === entry.funct7.U) { + target := ball.id.U + registered := true.B + } + } + when(init) { + target := io.issue.bits.cmd.rs2(4, 0) + registered := VecInit(bp.balls.map(b => target === b.id.U)).asUInt.orR + } + io.command.valid := io.issue.valid && registered + io.command.bits := io.issue.bits + io.command.bits.target := target + io.issue.ready := io.command.ready && registered + when(io.issue.valid) { + assert(registered, "Ball command is not registered") + when(init)(assert(io.issue.bits.cmd.rs2(63, 5) === 0.U, "BALL_INIT reserves rs2[63:5]")) + } +} diff --git a/arch/src/main/scala/framework/acc/balldomain/src/main/scala/mmio_bank/MmioBank.scala b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/mmio_bank/MmioBank.scala new file mode 100644 index 000000000..9dae1ef21 --- /dev/null +++ b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/mmio_bank/MmioBank.scala @@ -0,0 +1,131 @@ +package framework.acc.balldomain.mmio_bank + +import framework.lib.arbiter.{Arbiter, RRArbiter} +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.acc.memdomain.configs.MemDomainParam + +class MmioReadReq(p: MemDomainParam) extends Bundle { val addr = UInt(log2Ceil(p.mmioTotalBytes).W) } +class MmioReadResp(p: MemDomainParam) extends Bundle { val data = UInt(p.mmioReadWidth.W) } + +class MmioRead(p: MemDomainParam) extends Bundle { + val req = Flipped(Decoupled(new MmioReadReq(p))) + val resp = Decoupled(new MmioReadResp(p)) +} + +class MmioWriteReq(p: MemDomainParam) extends Bundle { + val addr = UInt(log2Ceil(p.mmioTotalBytes).W) + val data = UInt(p.mmioReadWidth.W) +} + +/** Metadata receiver: one addressed DMA beat, acknowledged after every byte is stored. */ +class MetadataWrite(p: MemDomainParam) extends Bundle { + + val request = Flipped(Decoupled(new Bundle { + val addr = UInt(log2Ceil(p.mmioTotalBytes).W) + val data = UInt(p.bankWidth.W) + val mask = UInt((p.bankWidth / 8).W) + })) + + val response = Decoupled(Bool()) +} + +/** Banked byte metadata SRAM; arbitration and response ownership remain local. */ +@instantiable +class MmioBank(p: MemDomainParam, readers: Int, writers: Int) extends Module { + require(p.mmioEnable && p.mmioBankWidth == 8 && p.mmioReadWidth == 8) + val bytes = p.bankWidth / 8 + val bankBits = math.max(1, log2Ceil(p.mmioBankNum)) + val rowBits = math.max(1, log2Ceil(p.mmioBankEntries)) + + @public + val io = IO(new Bundle { + val upload = new MetadataWrite(p) + val reads = Vec(readers, new MmioRead(p)) + val writes = Vec(writers, Flipped(Decoupled(new MmioWriteReq(p)))) + }) + + val idle :: storing :: respond :: Nil = Enum(3) + val state = RegInit(idle) + val beat = Reg(chiselTypeOf(io.upload.request.bits)) + val cursor = RegInit(0.U(math.max(1, log2Ceil(bytes)).W)) + io.upload.request.ready := state === idle + io.upload.response.valid := state === respond + io.upload.response.bits := true.B + when(io.upload.request.fire) { + assert(io.upload.request.bits.addr +& bytes.U <= p.mmioTotalBytes.U) + beat := io.upload.request.bits + cursor := 0.U + state := storing + } + when(state === storing) { + when(cursor +& p.mmioBankNum.U >= bytes.U)(state := respond) + .otherwise(cursor := cursor + p.mmioBankNum.U) + } + when(io.upload.response.fire)(state := idle) + + val readerBusy = RegInit(VecInit(Seq.fill(readers)(false.B))) + val readReady = Wire(Vec(readers, Vec(p.mmioBankNum, Bool()))) + val readValid = Wire(Vec(readers, Vec(p.mmioBankNum, Bool()))) + val readData = Wire(Vec(readers, Vec(p.mmioBankNum, UInt(8.W)))) + val writeReady = Wire(Vec(writers, Vec(p.mmioBankNum, Bool()))) + for (bank <- 0 until p.mmioBankNum) { + val memory = SyncReadMem(p.mmioBankEntries, UInt(8.W)) + val writes = Instantiate(new Arbiter(new MmioWriteReq(p), writers + 1)) + val offsets = (0 until p.mmioBankNum).map { offset => + val index = cursor +& offset.U + index < bytes.U && (beat.addr + index) % p.mmioBankNum.U === bank.U + } + val index = cursor +& PriorityEncoder(VecInit(offsets)) + writes.io.in(0).valid := state === storing && VecInit(offsets).asUInt.orR + writes.io.in(0).bits.addr := beat.addr + index + writes.io.in(0).bits.data := Mux(beat.mask(index(log2Ceil(bytes) - 1, 0)), (beat.data >> (index << 3))(7, 0), 0.U) + for (client <- 0 until writers) { + val selected = io.writes(client).bits.addr % p.mmioBankNum.U === bank.U + writes.io.in(client + 1).valid := io.writes(client).valid && selected + writes.io.in(client + 1).bits := io.writes(client).bits + writeReady(client)(bank) := selected && writes.io.in(client + 1).ready + } + writes.io.out.ready := true.B + val pending = RegInit(false.B) + val held = RegInit(false.B) + val owner = Reg(UInt(math.max(1, log2Ceil(readers)).W)) + val value = Reg(UInt(8.W)) + val reads = Instantiate(new RRArbiter(new MmioReadReq(p), math.max(1, readers))) + if (readers == 0) { + reads.io.in(0).valid := false.B + reads.io.in(0).bits := 0.U.asTypeOf(reads.io.in(0).bits) + } + for (client <- 0 until readers) { + val selected = io.reads(client).req.bits.addr % p.mmioBankNum.U === bank.U + reads.io.in(client).valid := io.reads(client).req.valid && selected && !readerBusy(client) + reads.io.in(client).bits := io.reads(client).req.bits + readReady(client)(bank) := selected && !readerBusy(client) && reads.io.in(client).ready + readValid(client)(bank) := held && owner === client.U + readData(client)(bank) := value + } + reads.io.out.ready := !writes.io.out.valid && !pending && !held + val raw = memory.readWrite( + (Mux(writes.io.out.valid, writes.io.out.bits.addr, reads.io.out.bits.addr) / p.mmioBankNum.U)(rowBits - 1, 0), + writes.io.out.bits.data, + writes.io.out.fire || reads.io.out.fire, + writes.io.out.fire + ) + pending := reads.io.out.fire + when(reads.io.out.fire)(owner := reads.io.chosen) + when(pending) { value := raw; held := true.B } + if (readers > 0) + when(held && (if (readers == 1) io.reads(0).resp.ready else io.reads(owner).resp.ready))(held := false.B) + } + for (client <- 0 until readers) { + io.reads(client).req.ready := readReady(client).asUInt.orR + io.reads(client).resp.valid := readValid(client).asUInt.orR + io.reads(client).resp.bits.data := Mux1H(readValid(client), readData(client)) + when(io.reads(client).req.fire)(readerBusy(client) := true.B) + when(io.reads(client).resp.fire)(readerBusy(client) := false.B) + } + for (client <- 0 until writers) io.writes(client).ready := writeReady(client).asUInt.orR +} diff --git a/arch/src/main/scala/framework/acc/balldomain/src/main/scala/sub_rob/Expansion.scala b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/sub_rob/Expansion.scala new file mode 100644 index 000000000..d21390d4c --- /dev/null +++ b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/sub_rob/Expansion.scala @@ -0,0 +1,34 @@ +package framework.acc.balldomain.sub_rob + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.acc.frontend.scheduler.Complete +import framework.acc.balldomain.mmio_bank.{MmioRead, MmioWriteReq} + +/** CISC operands presented to a Ball-owned expander; address-space state stays in BallDomain. */ +class ExpansionCommand(f: FrontendParam) extends Bundle { + val rob_id = UInt(log2Up(f.rob_entries).W) + val funct = UInt(7.W) + val rs2 = UInt(f.commandDataBits.W) +} + +/** A Ball-owned expander supplies rows to the shared SubROB. */ +class Expansion(f: FrontendParam, m: MemDomainParam, b: BallDomainParam) extends Bundle { + val command = Flipped(Decoupled(new ExpansionCommand(f))) + val accepts = Output(Bool()) + val starts = Output(Bool()) + val rows = Decoupled(new SubRobRow(f, m, b)) + val complete = Decoupled(new Complete(f)) + val read = Flipped(new MmioRead(m)) + val write = Decoupled(new MmioWriteReq(m)) +} + +@instantiable +trait HasExpansion { + @public + val expansion: Expansion +} diff --git a/arch/src/main/scala/framework/acc/balldomain/src/main/scala/sub_rob/SubROB.scala b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/sub_rob/SubROB.scala new file mode 100644 index 000000000..3e2435176 --- /dev/null +++ b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/sub_rob/SubROB.scala @@ -0,0 +1,277 @@ +package framework.acc.balldomain.sub_rob + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.acc.frontend.decoder.{DomainId, PostGDCmd} +import framework.acc.balldomain.sub_rob.configs.Params + +@instantiable +class SubROB(val frontendParam: FrontendParam, val memParam: MemDomainParam, val ballParam: BallDomainParam) + extends Module { + val commandDataBits = frontendParam.commandDataBits + + val p = Params( + depth = frontendParam.sub_rob_depth, + masterEntries = frontendParam.rob_entries, + balls = ballParam.balls.size + ) + + val subRobDepth = p.depth + val subIdBits = log2Up(subRobDepth * 4) + val robIdBits = log2Up(p.masterEntries) + val ballIdBits = log2Up(p.balls) + val rowIdBits = log2Up(subRobDepth) + + @public + val io = IO(new Bundle { + // Ball → SubROB: write a row of sub-instructions + val write = Flipped(Decoupled(new SubRobRow(frontendParam, memParam, ballParam))) + // SubROB → GlobalRS: issue independent Ball and memory slots + val issue = Vec(2, Decoupled(new PostGDCmd(frontendParam, memParam))) + val issueSubId = Output(Vec(2, UInt(subIdBits.W))) + val issueMasterRobId = Output(UInt(robIdBits.W)) + // GlobalRS → SubROB: a sub-instruction completed (sub_rob_id) + val subComplete = Flipped(Decoupled(UInt(subIdBits.W))) + // SubROB → main ROB: all sub-instructions done + val masterComplete = Decoupled(UInt(robIdBits.W)) + // Status + val occupied = Output(Bool()) + val lockedBallId = Output(UInt(ballIdBits.W)) + }) + + // ------------------------------------------------------------------------- + // Storage + // ------------------------------------------------------------------------- + val sram = SyncReadMem(subRobDepth, new SubRobRow(frontendParam, memParam, ballParam)) + + val writePtr = RegInit(0.U(rowIdBits.W)) + val readPtr = RegInit(0.U(rowIdBits.W)) + val rowCount = RegInit(0.U((log2Up(subRobDepth) + 1).W)) + val lockedBallId = RegInit(0.U(ballIdBits.W)) + val masterRobId = RegInit(0.U(robIdBits.W)) + + def nextPtr(p: UInt): UInt = Mux(p === (subRobDepth - 1).U, 0.U, p + 1.U) + + val occupied = RegInit(false.B) + val lastQueued = RegInit(false.B) + val isFull = rowCount === subRobDepth.U + val ballMatch = + !occupied || (io.write.bits.ball_id === lockedBallId && io.write.bits.master_rob_id === masterRobId) + + // ------------------------------------------------------------------------- + // FSM + // ------------------------------------------------------------------------- + // States: + // sIdle - nothing to do (rowCount == 0) + // sReadReq - issue SRAM read request for readPtr row + // sReadWait - wait 1 cycle for sramRaw to be latched into sramData + // sReadResp - sramData valid; dispatch one slot per domain per cycle + // sWaitSlots - all slots issued; wait for all subCompletes + // sWaitMaster - all rows done; fire masterComplete then return to sIdle + val sIdle :: sReadReq :: sReadWait :: sReadResp :: sWaitSlots :: sWaitMaster :: Nil = + Enum(6) + val state = RegInit(sIdle) + + // Write path: accept if not full AND ball_id matches (or not occupied yet) + io.write.ready := !isFull && !lastQueued && ballMatch && state =/= sReadReq && state =/= sWaitMaster + + when(io.write.fire) { + occupied := true.B + when(io.write.bits.last)(lastQueued := true.B) + for (slot <- io.write.bits.slots) { + when(slot.valid) { + assert( + slot.cmd.domain_id === DomainId.BALL || slot.cmd.domain_id === DomainId.MEM, + "SubROB supports Ball and memory slots" + ) + } + } + when(occupied) { + assert( + io.write.bits.master_rob_id === masterRobId, + "SubROB row master_rob_id mismatch" + ) + } + writePtr := nextPtr(writePtr) + when(!occupied) { + lockedBallId := io.write.bits.ball_id + masterRobId := io.write.bits.master_rob_id + } + } + + // SyncReadMem: read issued in sReadReq (cycle N), data valid on cycle N+1. + // We latch it into sramData with RegEnable so it stays stable during sReadResp + // and sWaitSlots, even when the read port is not enabled. + val sramReadEn = state === sReadReq + + val sramRaw = sram.readWrite( + Mux(io.write.fire, writePtr, readPtr), + io.write.bits, + sramReadEn || io.write.fire, + io.write.fire + ) + + val dataFresh = + RegNext(sramReadEn, false.B) // pulse on first cycle of sReadResp + val sramData = RegEnable(sramRaw, dataFresh) + val readPtrReg = RegEnable(readPtr, sramReadEn) + + // Per-row tracking registers (reset when moving to next row) + // slotIssued: have we already sent this slot to a domain? + // slotDone: have we received subComplete for this slot? + val slotIssued = RegInit(VecInit(Seq.fill(4)(false.B))) + val slotDone = RegInit(VecInit(Seq.fill(4)(false.B))) + + // Combinational: which slots still need to be issued? + val slotNeedsIssue = VecInit( + (0 until 4).map(i => sramData.slots(i).valid && !slotIssued(i)) + ) + + val selected = Wire(Vec(2, UInt(2.W))) + for (port <- 0 until 2) { + val domain = if (port == 0) DomainId.BALL else DomainId.MEM + val candidates = + VecInit( + (0 until 4).map(slot => slotNeedsIssue(slot) && sramData.slots(slot).cmd.domain_id === domain) + ) + selected(port) := PriorityEncoder(candidates.asUInt) + io.issue(port).valid := state === sReadResp && candidates.asUInt.orR + io.issue(port).bits := sramData.slots(selected(port)).cmd + io.issueSubId(port) := readPtrReg * 4.U + selected(port) + } + + // Combinational: are all valid slots done? + val allSlotsDone = VecInit( + (0 until 4).map(i => !sramData.slots(i).valid || slotDone(i)) + ).asUInt.andR + + io.issueMasterRobId := masterRobId + + // subComplete: always ready, mark the corresponding slot done + io.subComplete.ready := true.B + when(io.subComplete.fire) { + val subId = io.subComplete.bits + val subRow = subId / 4.U + val slotIdx = subId(1, 0) + assert( + state === sReadResp || state === sWaitSlots || state === sWaitMaster, + "SubROB subComplete arrived in invalid state" + ) + assert(subRow === readPtrReg, "SubROB subComplete points to non-active row") + slotDone(slotIdx) := true.B + } + + // masterComplete + io.masterComplete.valid := state === sWaitMaster + io.masterComplete.bits := masterRobId + + // Status outputs + io.occupied := occupied + io.lockedBallId := lockedBallId + + // ------------------------------------------------------------------------- + // FSM transitions + // ------------------------------------------------------------------------- + // Helper: advance past current row (called when row is complete) + val rowAdvance = WireDefault(false.B) + + def advanceRow(): Unit = { + readPtr := nextPtr(readPtr) + rowAdvance := true.B + slotIssued.foreach(_ := false.B) + slotDone.foreach(_ := false.B) + } + + switch(state) { + is(sIdle) { + when(rowCount =/= 0.U)(state := sReadReq) + } + is(sReadReq) { + // SRAM read issued, data available on sramRaw next cycle + state := sReadWait + } + is(sReadWait) { + // sramRaw is valid; RegEnable latches it into sramData at this clock edge. + // sramData will be stable starting next cycle (sReadResp). + state := sReadResp + } + is(sReadResp) { + // Fire issued slot + for (port <- 0 until 2) { + when(io.issue(port).fire)(slotIssued(selected(port)) := true.B) + } + // Check if all slots have been issued + // After marking current slot issued, recalculate + val allIssued = VecInit( + (0 until 4).map(i => + !sramData.slots(i).valid || slotIssued(i) || + VecInit( + (0 until 2).map(port => io.issue(port).fire && selected(port) === i.U) + ).asUInt.orR + ) + ).asUInt.andR + + when(allIssued) { + // Check fast path: all already done (e.g., empty row or immediate completion) + val allDoneNow = VecInit( + (0 until 4).map(i => !sramData.slots(i).valid || slotDone(i)) + ).asUInt.andR + when(allDoneNow) { + // Fast path: skip sWaitSlots + advanceRow() + state := Mux( + sramData.last, + sWaitMaster, + Mux(rowCount === 1.U && !io.write.fire, sIdle, sReadReq) + ) + }.otherwise { + state := sWaitSlots + } + } + // else: stay in sReadResp to dispatch remaining slots + } + is(sWaitSlots) { + when(allSlotsDone) { + advanceRow() + state := Mux( + sramData.last, + sWaitMaster, + Mux(rowCount === 1.U && !io.write.fire, sIdle, sReadReq) + ) + } + } + is(sWaitMaster) { + when(io.masterComplete.fire) { + occupied := false.B + lastQueued := false.B + lockedBallId := 0.U + masterRobId := 0.U + state := sIdle + } + } + } + when(io.write.fire =/= rowAdvance) { + rowCount := Mux(io.write.fire, rowCount + 1.U, rowCount - 1.U) + } + +} + +class SubRobSlot(val frontendParam: FrontendParam, val memParam: MemDomainParam) extends Bundle { + val commandDataBits = frontendParam.commandDataBits + val valid = Bool() + val cmd = new PostGDCmd(frontendParam, memParam) +} + +class SubRobRow(val frontendParam: FrontendParam, val memParam: MemDomainParam, val ballParam: BallDomainParam) + extends Bundle { + val commandDataBits = frontendParam.commandDataBits + val slots = Vec(4, new SubRobSlot(frontendParam, memParam)) + val ball_id = UInt(log2Up(ballParam.balls.size).W) + val master_rob_id = UInt(log2Up(frontendParam.rob_entries).W) + val last = Bool() +} diff --git a/arch/src/main/scala/framework/acc/balldomain/src/main/scala/sub_rob/configs/Params.scala b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/sub_rob/configs/Params.scala new file mode 100644 index 000000000..55893c34e --- /dev/null +++ b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/sub_rob/configs/Params.scala @@ -0,0 +1,7 @@ +package framework.acc.balldomain.sub_rob.configs + +case class Params(depth: Int, masterEntries: Int, balls: Int) { + require(depth > 1) + require(masterEntries > 1) + require(balls > 0) +} diff --git a/arch/src/main/scala/framework/acc/build.sc b/arch/src/main/scala/framework/acc/build.sc new file mode 100644 index 000000000..962b2c0a2 --- /dev/null +++ b/arch/src/main/scala/framework/acc/build.sc @@ -0,0 +1,25 @@ +import mill._ +import $file.debug.{build => debug_build} +import $file.balldomain.{build => balldomain_build} +import $file.frontend.{build => frontend_build} +import $file.gpdomain.{build => gpdomain_build} +import $file.memdomain.{build => memdomain_build} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("configs/AccParam.scala")) + ) ++ + debug_build.sources() ++ + balldomain_build.sources() ++ + frontend_build.sources() ++ + gpdomain_build.sources() ++ + memdomain_build.sources() +) + +def emitSources = T.sources( + debug_build.emitSources() ++ + balldomain_build.emitSources() ++ + frontend_build.emitSources() ++ + gpdomain_build.emitSources() ++ + memdomain_build.emitSources() +) diff --git a/arch/src/main/scala/framework/acc/configs/AccParam.scala b/arch/src/main/scala/framework/acc/configs/AccParam.scala new file mode 100644 index 000000000..f48b52430 --- /dev/null +++ b/arch/src/main/scala/framework/acc/configs/AccParam.scala @@ -0,0 +1,63 @@ +package framework.acc.configs + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.memdomain.configs.MemDomainParam +import buckyball.config.{CorePlacement, ThreadInstance} +import framework.acc.memdomain.tomem.configs.{Params => TransportParam} +import upickle.default.{macroRW, ReadWriter} + +case class AccParam( + frontend: FrontendParam, + memDomain: MemDomainParam, + gpDomain: GpDomainParam, + ballDomain: BallDomainParam, + transport: TransportParam) { + require( + BigInt(memDomain.virtualBankCount) <= (BigInt(1) << frontend.bank_id_len), + "virtualBankCount must fit the frontend bank ID width" + ) + require( + frontend.max_private_vbank < memDomain.virtualBankCount, + "Private bank upper bound must be below virtualBankCount" + ) + if (memDomain.sharedEnable) { + require( + frontend.shared_vbank_base > frontend.max_private_vbank, + "Shared bank base must be above the private bank range" + ) + require( + frontend.shared_vbank_base <= memDomain.virtualBankCount, + "Shared bank base must not exceed virtualBankCount" + ) + } +} + +object AccParam { + + def fromProto(core: CorePlacement): AccParam = { + require(core.hasAccelerator, "Core has no local accelerator") + val acc = core.getAccelerator + AccParam( + FrontendParam.fromProto(acc.getFrontend, core.getParam.getXLen, core.getHartId), + MemDomainParam.fromProto(acc.getMem, core.getSharedMem, core.getVirtualBankCount, 1), + GpDomainParam.fromProto(acc.getGpdomain), + BallDomainParam.fromProto(acc.getBalldomain), + TransportParam.fromProto(acc.getTransport) + ) + } + + def fromProto(thread: ThreadInstance, core: CorePlacement): AccParam = { + require(thread.hasTransport) + AccParam( + FrontendParam.fromProto(thread.getFrontend, core.getParam.getXLen, thread.getLocalId), + MemDomainParam.fromProto(thread.getMem, core.getSharedMem, core.getVirtualBankCount, core.getThreadIndicesCount), + GpDomainParam.fromProto(thread.getGpdomain), + BallDomainParam.fromProto(thread.getBalldomain), + TransportParam.fromProto(thread.getTransport) + ) + } + + implicit val rw: ReadWriter[AccParam] = macroRW +} diff --git a/arch/src/main/scala/framework/acc/debug/BankCheckDPI.scala b/arch/src/main/scala/framework/acc/debug/BankCheckDPI.scala new file mode 100644 index 000000000..47c68f526 --- /dev/null +++ b/arch/src/main/scala/framework/acc/debug/BankCheckDPI.scala @@ -0,0 +1,77 @@ +package framework.acc.debug + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public} +import chisel3.util._ +import framework.lib.dpi.DpiGuard + +@instantiable +class BankCheckDPI extends BlackBox with HasBlackBoxInline { + + @public + val io = IO(new Bundle { + val clock = Input(Clock()) + val reset = Input(Bool()) + val instId = Input(UInt(64.W)) + val threadId = Input(UInt(64.W)) + val ownerThreadId = Input(UInt(64.W)) + val w0Vbank = Input(UInt(32.W)) + val w0Hash = Input(UInt(32.W)) + val fire = Input(Bool()) + val produced = Input(UInt(64.W)) + val idle = Input(Bool()) + }) + + setInline( + "BankCheckDPI.v", + """ + |module BankCheckDPI( + | input clock, + | input reset, + | input [63:0] instId, + | input [63:0] threadId, + | input [63:0] ownerThreadId, + | input [31:0] w0Vbank, + | input [31:0] w0Hash, + | input fire, + | input [63:0] produced, + | input idle + |); + |""".stripMargin + DpiGuard.wrapBTrace( + """ + | export "DPI-C" function btrace_snapshot; + | function void btrace_snapshot( + | output int unsigned thread_lo, + | output int unsigned thread_hi, + | output int unsigned produced_lo, + | output int unsigned produced_hi, + | output int unsigned is_idle + | ); + | thread_lo = threadId[31:0]; + | thread_hi = threadId[63:32]; + | produced_lo = produced[31:0]; + | produced_hi = produced[63:32]; + | is_idle = {31'b0, idle}; + | endfunction + | import "DPI-C" context function void dpi_btrace( + | input int unsigned inst_id_lo, + | input int unsigned inst_id_hi, + | input int unsigned thread_id_lo, + | input int unsigned thread_id_hi, + | input int unsigned owner_thread_id_lo, + | input int unsigned owner_thread_id_hi, + | input int unsigned w0_vbank, + | input int unsigned w0_hash + | ); + | always @(posedge clock) begin + | if (!reset && fire) begin + | dpi_btrace(instId[31:0], instId[63:32], threadId[31:0], threadId[63:32], ownerThreadId[31:0], ownerThreadId[63:32], w0Vbank, w0Hash); + | end + | end + |""".stripMargin + ) + + """ + |endmodule + |""".stripMargin + ) +} diff --git a/arch/src/main/scala/framework/acc/debug/Debug.scala b/arch/src/main/scala/framework/acc/debug/Debug.scala new file mode 100644 index 000000000..bf415f62d --- /dev/null +++ b/arch/src/main/scala/framework/acc/debug/Debug.scala @@ -0,0 +1,179 @@ +package framework.acc.debug + +import sims.configs.SimParam +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.memdomain.PhysicalBankHash +import framework.thread.ant.Retire +import framework.acc.debug.trace.{AcceleratorEvents, InstructionEvent, Trace} + +class DebugCommit(frontendParam: FrontendParam) extends Bundle { + val robId = UInt(log2Ceil(frontendParam.rob_entries).W) + val instId = UInt(64.W) + val threadId = UInt(64.W) + val bank = UInt(frontendParam.bank_id_len.W) +} + +/** Producer-facing debug handshake. Unlike trace, completion may wait for checked. */ +class DebugPort(frontendParam: FrontendParam) extends Bundle { + + val check = Decoupled(new Bundle { + val robId = UInt(log2Ceil(frontendParam.rob_entries).W) + val bank = UInt(frontendParam.bank_id_len.W) + }) + + val checked = Flipped(Decoupled(UInt(log2Ceil(frontendParam.rob_entries).W))) + val commit = Output(Valid(new DebugCommit(frontendParam))) + val idle = Output(Bool()) +} + +/** Frontend debug boundary: observations never share the differential-check handshake. */ +class FrontendDebugPort(frontendParam: FrontendParam, simp: SimParam) extends Bundle { + val diffTest = simp.diffTest + val trace = Output(Vec(4, new InstructionEvent)) + val control = Option.when(diffTest)(new DebugPort(frontendParam)) +} + +/** Producer-facing thread debug connection, assembled from the owning child IP ports. */ +class AcceleratorDebugPort( + frontendParam: FrontendParam, + memParam: MemDomainParam, + gpParam: GpDomainParam, + ballParam: BallDomainParam, + simp: SimParam) + extends Bundle { + val diffTest = simp.diffTest + + val frontend = new FrontendDebugPort(frontendParam, simp) + val trace = Output(new AcceleratorEvents(frontendParam, memParam, gpParam, ballParam)) + + val banks = Option.when(diffTest)( + Output(Vec( + memParam.bankNum + (if (memParam.sharedEnable) memParam.sharedBankNum else 0), + new PhysicalBankHash(frontendParam) + )) + ) + +} + +/** Differential bank checks may delay completion and serialize retirement. */ +@instantiable +class BankCheck( + frontendParam: FrontendParam, + memParam: MemDomainParam, + simp: SimParam, + traceId: Int, + sharedOwnerId: Int) + extends Module { + val diffTest = simp.diffTest + val ownerId = frontendParam.ownerId + require(diffTest) + + val bankCount = memParam.bankNum + (if (memParam.sharedEnable) memParam.sharedBankNum else 0) + + @public + val io = IO(new Bundle { + val pipeline = Flipped(new DebugPort(frontendParam)) + val banks = Input(Vec(bankCount, new PhysicalBankHash(frontendParam))) + }) + + val collecting = RegInit(false.B) + val ready = RegInit(false.B) + val robId = Reg(chiselTypeOf(io.pipeline.check.bits.robId)) + val bank = Reg(chiselTypeOf(io.pipeline.check.bits.bank)) + val group = Reg(UInt(32.W)) + val currentHash = Reg(UInt(32.W)) + val hashes = Reg(Vec(frontendParam.rob_entries, UInt(32.W))) + io.pipeline.check.ready := !collecting && !ready + when(io.pipeline.check.fire) { + robId := io.pipeline.check.bits.robId + bank := io.pipeline.check.bits.bank + group := 0.U + currentHash := 0.U + collecting := true.B + } + + val matches = VecInit(io.banks.map(entry => + entry.valid && entry.threadId === ownerId.U && entry.vbankId === bank && entry.groupId === group + )) + + val nextMatches = VecInit(io.banks.map(entry => + entry.valid && entry.threadId === ownerId.U && entry.vbankId === bank && entry.groupId === group + 1.U + )) + + val physical = Mux1H(matches, io.banks) + def rotateLeft(value: UInt, amount: Int): UInt = Cat(value(31 - amount, 0), value(31, 32 - amount)) + val mixed = physical.statusHash ^ rotateLeft(physical.groupId, 7) ^ "h9e3779b9".U + val nextHash = rotateLeft(currentHash, 5) ^ mixed ^ rotateLeft(mixed, 13) + when(collecting) { + assert(PopCount(matches) === 1.U, "Debug requires one physical bank per virtual bank group") + when(nextMatches.asUInt.orR) { + currentHash := nextHash + group := group + 1.U + }.otherwise { + hashes(robId) := nextHash + collecting := false.B + ready := true.B + } + } + io.pipeline.checked.valid := ready + io.pipeline.checked.bits := robId + when(io.pipeline.checked.fire)(ready := false.B) + + val produced = RegInit(0.U(64.W)) + when(io.pipeline.commit.valid)(produced := produced + 1.U) + val sink: Instance[BankCheckDPI] = Instantiate(new BankCheckDPI) + sink.io.clock := clock + sink.io.reset := reset.asBool + sink.io.instId := io.pipeline.commit.bits.instId + sink.io.threadId := traceId.U + sink.io.ownerThreadId := Mux( + memParam.sharedEnable.B && io.pipeline.commit.bits.bank >= frontendParam.shared_vbank_base.U, + sharedOwnerId.U, + traceId.U + ) + sink.io.w0Vbank := io.pipeline.commit.bits.bank + sink.io.w0Hash := hashes(io.pipeline.commit.bits.robId) + sink.io.fire := io.pipeline.commit.valid + sink.io.produced := produced + sink.io.idle := io.pipeline.idle +} + +/** Thread-level observability: passive trace plus explicitly enabled timing-affecting checks. */ +@instantiable +class Debug( + frontendParam: FrontendParam, + memParam: MemDomainParam, + gpParam: GpDomainParam, + ballParam: BallDomainParam, + simp: SimParam, + traceId: Int, + sharedOwnerId: Int) + extends Module { + val diffTest = simp.diffTest + val ownerId = frontendParam.ownerId + + @public + val io = IO(new Bundle { + val ant = Input(Valid(new Retire)) + val acc = Flipped(new AcceleratorDebugPort(frontendParam, memParam, gpParam, ballParam, simp)) + + }) + + val trace: Instance[Trace] = Instantiate(new Trace(frontendParam, memParam, gpParam, ballParam, traceId)) + trace.io.ant := io.ant + trace.io.acc := io.acc.trace + trace.io.instructions := io.acc.frontend.trace + io.acc.frontend.control.foreach { port => + val check: Instance[BankCheck] = + Instantiate(new BankCheck(frontendParam, memParam, simp, traceId, sharedOwnerId)) + check.io.pipeline <> port + check.io.banks := io.acc.banks.get + } +} diff --git a/arch/src/main/scala/framework/acc/debug/build.sc b/arch/src/main/scala/framework/acc/debug/build.sc new file mode 100644 index 000000000..c35b4bc69 --- /dev/null +++ b/arch/src/main/scala/framework/acc/debug/build.sc @@ -0,0 +1,17 @@ +import mill._ + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("BankCheckDPI.scala")), + PathRef(millSourcePath / os.RelPath("Debug.scala")), + PathRef(millSourcePath / os.RelPath("trace/AntTraceDPI.scala")), + PathRef(millSourcePath / os.RelPath("trace/ITraceDPI.scala")), + PathRef(millSourcePath / os.RelPath("trace/MTraceDPI.scala")), + PathRef(millSourcePath / os.RelPath("trace/PMCTraceDPI.scala")), + PathRef(millSourcePath / os.RelPath("trace/Trace.scala")) + ) +) + +def emitSources = T.sources( + Seq.empty[PathRef] +) diff --git a/arch/src/main/scala/framework/acc/debug/trace/AntTraceDPI.scala b/arch/src/main/scala/framework/acc/debug/trace/AntTraceDPI.scala new file mode 100644 index 000000000..999fe189d --- /dev/null +++ b/arch/src/main/scala/framework/acc/debug/trace/AntTraceDPI.scala @@ -0,0 +1,47 @@ +package framework.acc.debug.trace + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.lib.dpi.DpiGuard + +@instantiable +class AntTraceDPI extends BlackBox with HasBlackBoxInline { + + @public + val io = IO(new Bundle { + val clock = Input(Clock()) + val reset = Input(Bool()) + val thread_id = Input(UInt(64.W)) + val pc = Input(UInt(64.W)) + val instruction = Input(UInt(32.W)) + val rd = Input(UInt(5.W)) + val data = Input(UInt(64.W)) + val enable = Input(Bool()) + }) + + setInline( + "AntTraceDPI.v", + """ + |module AntTraceDPI( + | input clock, input reset, input enable, + | input [63:0] thread_id, input [63:0] pc, + | input [31:0] instruction, input [4:0] rd, input [63:0] data + |); + |""".stripMargin + DpiGuard.wrapITrace( + """ + | import "DPI-C" context function void dpi_ant_trace( + | input int unsigned thread_lo, input int unsigned thread_hi, + | input int unsigned pc_lo, input int unsigned pc_hi, + | input int unsigned instruction, input int unsigned rd, + | input int unsigned data_lo, input int unsigned data_hi + | ); + | always @(posedge clock) begin + | if (!reset && enable) + | dpi_ant_trace(thread_id[31:0], thread_id[63:32], pc[31:0], pc[63:32], + | instruction, {27'b0, rd}, data[31:0], data[63:32]); + | end + |""".stripMargin + ) + "endmodule\n" + ) +} diff --git a/arch/src/main/scala/framework/frontend/globalrs/ITraceDPI.scala b/arch/src/main/scala/framework/acc/debug/trace/ITraceDPI.scala similarity index 86% rename from arch/src/main/scala/framework/frontend/globalrs/ITraceDPI.scala rename to arch/src/main/scala/framework/acc/debug/trace/ITraceDPI.scala index 08126e6c6..6e553ec43 100644 --- a/arch/src/main/scala/framework/frontend/globalrs/ITraceDPI.scala +++ b/arch/src/main/scala/framework/acc/debug/trace/ITraceDPI.scala @@ -1,16 +1,19 @@ -package framework.frontend.globalrs +package framework.acc.debug.trace import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public} import chisel3.util._ -import framework.dpi.DpiGuard +import framework.lib.dpi.DpiGuard +@instantiable class ITraceDPI extends BlackBox with HasBlackBoxInline { + @public val io = IO(new Bundle { val clock = Input(Clock()) val reset = Input(Bool()) val is_issue = Input(UInt(8.W)) - val hart_id = Input(UInt(64.W)) + val thread_id = Input(UInt(64.W)) val rob_id = Input(UInt(32.W)) val inst_id = Input(UInt(64.W)) val domain_id = Input(UInt(32.W)) @@ -28,7 +31,7 @@ class ITraceDPI extends BlackBox with HasBlackBoxInline { | input clock, | input reset, | input [7:0] is_issue, - | input [63:0] hart_id, + | input [63:0] thread_id, | input [31:0] rob_id, | input [63:0] inst_id, | input [31:0] domain_id, @@ -41,8 +44,8 @@ class ITraceDPI extends BlackBox with HasBlackBoxInline { |""".stripMargin + DpiGuard.wrapITrace(""" | import "DPI-C" context function void dpi_itrace( | input int unsigned is_issue, - | input int unsigned hart_id_lo, - | input int unsigned hart_id_hi, + | input int unsigned thread_id_lo, + | input int unsigned thread_id_hi, | input int unsigned rob_id, | input int unsigned inst_id_lo, | input int unsigned inst_id_hi, @@ -56,7 +59,7 @@ class ITraceDPI extends BlackBox with HasBlackBoxInline { | input int unsigned rs2_data_hi | ); | reg [7:0] is_issue_reg; - | reg [63:0] hart_id_reg; + | reg [63:0] thread_id_reg; | reg [31:0] rob_id_reg; | reg [63:0] inst_id_reg; | reg [31:0] domain_id_reg; @@ -71,13 +74,13 @@ class ITraceDPI extends BlackBox with HasBlackBoxInline { | valid_reg <= 1'b0; | end else begin | if (valid_reg) begin - | dpi_itrace(is_issue_reg, hart_id_reg[31:0], hart_id_reg[63:32], rob_id_reg, inst_id_reg[31:0], inst_id_reg[63:32], domain_id_reg, funct_reg, pc_reg[31:0], pc_reg[63:32], rs1_data_reg[31:0], rs1_data_reg[63:32], rs2_data_reg[31:0], rs2_data_reg[63:32]); + | dpi_itrace({24'b0, is_issue_reg}, thread_id_reg[31:0], thread_id_reg[63:32], rob_id_reg, inst_id_reg[31:0], inst_id_reg[63:32], domain_id_reg, funct_reg, pc_reg[31:0], pc_reg[63:32], rs1_data_reg[31:0], rs1_data_reg[63:32], rs2_data_reg[31:0], rs2_data_reg[63:32]); | end | | valid_reg <= enable; | if (enable) begin | is_issue_reg <= is_issue; - | hart_id_reg <= hart_id; + | thread_id_reg <= thread_id; | rob_id_reg <= rob_id; | inst_id_reg <= inst_id; | domain_id_reg <= domain_id; diff --git a/arch/src/main/scala/framework/acc/debug/trace/MTraceDPI.scala b/arch/src/main/scala/framework/acc/debug/trace/MTraceDPI.scala new file mode 100644 index 000000000..a47177ad8 --- /dev/null +++ b/arch/src/main/scala/framework/acc/debug/trace/MTraceDPI.scala @@ -0,0 +1,118 @@ +package framework.acc.debug.trace + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.lib.dpi.DpiGuard + +@instantiable +class MTraceDPI extends BlackBox with HasBlackBoxInline { + + @public + val io = IO(new Bundle { + val clock = Input(Clock()) + val reset = Input(Bool()) + val is_write = Input(UInt(8.W)) + val is_shared = Input(UInt(8.W)) + val channel = Input(UInt(32.W)) + val thread_id = Input(UInt(64.W)) + val rob_id = Input(UInt(32.W)) + val inst_id = Input(UInt(64.W)) + val vbank_id = Input(UInt(32.W)) + val pbank_id = Input(UInt(32.W)) + val group_id = Input(UInt(32.W)) + val addr = Input(UInt(32.W)) + val write_mask = Input(UInt(32.W)) + val data_lo = Input(UInt(64.W)) + val data_hi = Input(UInt(64.W)) + val enable = Input(Bool()) + }) + + setInline( + "MTraceDPI.v", + """ + |module MTraceDPI( + | input clock, + | input reset, + | input [7:0] is_write, + | input [7:0] is_shared, + | input [31:0] channel, + | input [63:0] thread_id, + | input [31:0] rob_id, + | input [63:0] inst_id, + | input [31:0] vbank_id, + | input [31:0] pbank_id, + | input [31:0] group_id, + | input [31:0] addr, + | input [31:0] write_mask, + | input [63:0] data_lo, + | input [63:0] data_hi, + | input enable + |); + |""".stripMargin + DpiGuard.wrapMTrace(""" + | import "DPI-C" context function void dpi_mtrace( + | input int unsigned is_write, + | input int unsigned is_shared, + | input int unsigned channel, + | input int unsigned thread_id_lo, + | input int unsigned thread_id_hi, + | input int unsigned rob_id, + | input int unsigned inst_id_lo, + | input int unsigned inst_id_hi, + | input int unsigned vbank_id, + | input int unsigned pbank_id, + | input int unsigned group_id, + | input int unsigned addr, + | input int unsigned write_mask, + | input int unsigned data_lo_lo, + | input int unsigned data_lo_hi, + | input int unsigned data_hi_lo, + | input int unsigned data_hi_hi + | ); + | reg [7:0] is_write_reg; + | reg [7:0] is_shared_reg; + | reg [31:0] channel_reg; + | reg [63:0] thread_id_reg; + | reg [31:0] rob_id_reg; + | reg [63:0] inst_id_reg; + | reg [31:0] vbank_id_reg; + | reg [31:0] pbank_id_reg; + | reg [31:0] group_id_reg; + | reg [31:0] addr_reg; + | reg [31:0] write_mask_reg; + | reg [63:0] data_lo_reg; + | reg [63:0] data_hi_reg; + | reg valid_reg; + | + | always @(posedge clock) begin + | if (reset) begin + | valid_reg <= 1'b0; + | end else begin + | if (valid_reg) begin + | dpi_mtrace({24'b0, is_write_reg}, {24'b0, is_shared_reg}, channel_reg, thread_id_reg[31:0], thread_id_reg[63:32], rob_id_reg, inst_id_reg[31:0], inst_id_reg[63:32], vbank_id_reg, pbank_id_reg, group_id_reg, addr_reg, write_mask_reg, data_lo_reg[31:0], data_lo_reg[63:32], data_hi_reg[31:0], data_hi_reg[63:32]); + | end + | + | valid_reg <= enable; + | if (enable) begin + | is_write_reg <= is_write; + | is_shared_reg <= is_shared; + | channel_reg <= channel; + | thread_id_reg <= thread_id; + | rob_id_reg <= rob_id; + | inst_id_reg <= inst_id; + | vbank_id_reg <= vbank_id; + | pbank_id_reg <= pbank_id; + | group_id_reg <= group_id; + | addr_reg <= addr; + | write_mask_reg <= write_mask; + | data_lo_reg <= data_lo; + | data_hi_reg <= data_hi; + | end + | end + | end + |""".stripMargin) + + """ + |endmodule + """.stripMargin + ) +} diff --git a/arch/src/main/scala/framework/balldomain/bbus/pmc/PMCTraceDPI.scala b/arch/src/main/scala/framework/acc/debug/trace/PMCTraceDPI.scala similarity index 91% rename from arch/src/main/scala/framework/balldomain/bbus/pmc/PMCTraceDPI.scala rename to arch/src/main/scala/framework/acc/debug/trace/PMCTraceDPI.scala index 411dc25b0..43047b2e5 100644 --- a/arch/src/main/scala/framework/balldomain/bbus/pmc/PMCTraceDPI.scala +++ b/arch/src/main/scala/framework/acc/debug/trace/PMCTraceDPI.scala @@ -1,11 +1,15 @@ -package framework.balldomain.bbus.pmc +package framework.acc.debug.trace + +import chisel3.experimental.hierarchy.{instantiable, public} import chisel3._ import chisel3.util._ -import framework.dpi.DpiGuard +import framework.lib.dpi.DpiGuard +@instantiable class PMCTraceDPI extends BlackBox with HasBlackBoxInline { + @public val io = IO(new Bundle { val clock = Input(Clock()) val reset = Input(Bool()) diff --git a/arch/src/main/scala/framework/acc/debug/trace/Trace.scala b/arch/src/main/scala/framework/acc/debug/trace/Trace.scala new file mode 100644 index 000000000..d96d3ea6b --- /dev/null +++ b/arch/src/main/scala/framework/acc/debug/trace/Trace.scala @@ -0,0 +1,131 @@ +package framework.acc.debug.trace + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.thread.ant.Retire + +class InstructionEvent extends Bundle { + val is_issue = UInt(8.W) + val thread_id = UInt(64.W) + val rob_id = UInt(32.W) + val inst_id = UInt(64.W) + val domain_id = UInt(32.W) + val funct = UInt(32.W) + val pc = UInt(64.W) + val rs1_data = UInt(64.W) + val rs2_data = UInt(64.W) + val enable = Bool() +} + +class MemoryEvent extends Bundle { + val is_write = UInt(8.W) + val is_shared = UInt(8.W) + val channel = UInt(32.W) + val thread_id = UInt(64.W) + val rob_id = UInt(32.W) + val inst_id = UInt(64.W) + val vbank_id = UInt(32.W) + val pbank_id = UInt(32.W) + val group_id = UInt(32.W) + val addr = UInt(32.W) + val write_mask = UInt(32.W) + val data_lo = UInt(64.W) + val data_hi = UInt(64.W) + val enable = Bool() +} + +class AcceleratorEvents( + frontendParam: FrontendParam, + memParam: MemDomainParam, + gpParam: GpDomainParam, + ballParam: BallDomainParam) + extends Bundle { + + val memoryEvents = memParam.bankRequestPorts + + (if (gpParam.enable) gpParam.memory.memoryPorts else 0) + 2 + memParam.bankNum + val memory = Vec(memoryEvents, new MemoryEvent) + val ballStarted = Vec(ballParam.balls.size, Valid(UInt(log2Ceil(frontendParam.rob_entries).W))) + val ballCompleted = Vec(ballParam.balls.size, Valid(UInt(log2Ceil(frontendParam.rob_entries).W))) +} + +/** Owns trace sinks and performance counters for one hardware thread. */ +@instantiable +class Trace( + frontendParam: FrontendParam, + memParam: MemDomainParam, + gpParam: GpDomainParam, + ballParam: BallDomainParam, + threadId: Int) + extends Module { + + @public + val io = IO(new Bundle { + val instructions = Input(Vec(4, new InstructionEvent)) + val ant = Input(Valid(new Retire)) + val acc = Input(new AcceleratorEvents(frontendParam, memParam, gpParam, ballParam)) + }) + + val ant: Instance[AntTraceDPI] = Instantiate(new AntTraceDPI) + ant.io.clock := clock + ant.io.reset := reset.asBool + ant.io.thread_id := threadId.U(64.W) + ant.io.pc := io.ant.bits.pc + ant.io.instruction := io.ant.bits.instruction + ant.io.rd := io.ant.bits.rd + ant.io.data := io.ant.bits.data + ant.io.enable := io.ant.valid + + for (event <- io.instructions) { + val sink: Instance[ITraceDPI] = Instantiate(new ITraceDPI) + sink.io.clock := clock + sink.io.reset := reset.asBool + sink.io.is_issue := event.is_issue + sink.io.thread_id := threadId.U(64.W) + sink.io.rob_id := event.rob_id + sink.io.inst_id := event.inst_id + sink.io.domain_id := event.domain_id + sink.io.funct := event.funct + sink.io.pc := event.pc + sink.io.rs1_data := event.rs1_data + sink.io.rs2_data := event.rs2_data + sink.io.enable := event.enable + } + for (event <- io.acc.memory) { + val sink: Instance[MTraceDPI] = Instantiate(new MTraceDPI) + sink.io.clock := clock + sink.io.reset := reset.asBool + sink.io.is_write := event.is_write + sink.io.is_shared := event.is_shared + sink.io.channel := event.channel + sink.io.thread_id := threadId.U(64.W) + sink.io.rob_id := event.rob_id + sink.io.inst_id := event.inst_id + sink.io.vbank_id := event.vbank_id + sink.io.pbank_id := event.pbank_id + sink.io.group_id := event.group_id + sink.io.addr := event.addr + sink.io.write_mask := event.write_mask + sink.io.data_lo := event.data_lo + sink.io.data_hi := event.data_hi + sink.io.enable := event.enable + } + val cycle = RegInit(0.U(64.W)) + cycle := cycle + 1.U + val starts = Reg(Vec(io.acc.ballStarted.size, UInt(64.W))) + for (index <- io.acc.ballStarted.indices) { + when(io.acc.ballStarted(index).valid)(starts(index) := cycle) + val sink = Instantiate(new PMCTraceDPI) + sink.io.clock := clock + sink.io.reset := reset.asBool + sink.io.ball_id := index.U + sink.io.rob_id := io.acc.ballCompleted(index).bits + sink.io.elapsed := cycle - starts(index) + sink.io.enable := io.acc.ballCompleted(index).valid + } +} diff --git a/arch/src/main/scala/framework/acc/frontend/build.sc b/arch/src/main/scala/framework/acc/frontend/build.sc new file mode 100644 index 000000000..ccb8cf074 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/build.sc @@ -0,0 +1,32 @@ +import mill._ +import $file.src.main.scala.boot.{build => boot_build} +import $file.src.main.scala.scheduler.{build => scheduler_build} +import $file.^.^.^.^.^.^.settings + + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.acc.frontend.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Frontend.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Ingress.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/RoCC.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/FrontendParam.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/decoder/DISA.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/decoder/GlobalDecoder.scala")) + ) ++ + boot_build.sources() ++ + scheduler_build.sources() +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) ++ + boot_build.emitSources() ++ + scheduler_build.emitSources() +) diff --git a/arch/src/main/scala/framework/acc/frontend/configs/default.toml b/arch/src/main/scala/framework/acc/frontend/configs/default.toml new file mode 100644 index 000000000..5e290e70a --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/configs/default.toml @@ -0,0 +1,9 @@ +bankAliasEnable = true +robEntries = 8 +rsOutOfOrderResponse = true +bankIdLen = 10 +maxPrivateVbank = 23 +sharedVbankBase = 32 +iterLen = 34 +subRobEnable = false +subRobDepth = 64 diff --git a/arch/src/main/scala/framework/acc/frontend/src/csrc/decoder/Cargo.lock b/arch/src/main/scala/framework/acc/frontend/src/csrc/decoder/Cargo.lock new file mode 100644 index 000000000..7ea1f5ec0 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/csrc/decoder/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "decoder-reference" +version = "0.1.0" diff --git a/arch/src/main/scala/framework/acc/frontend/src/csrc/decoder/Cargo.toml b/arch/src/main/scala/framework/acc/frontend/src/csrc/decoder/Cargo.toml new file mode 100644 index 000000000..201fb3abf --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/csrc/decoder/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "decoder-reference" +version = "0.1.0" +edition = "2021" +[lib] +crate-type = ["cdylib", "staticlib"] diff --git a/arch/src/main/scala/framework/acc/frontend/src/csrc/decoder/src/lib.rs b/arch/src/main/scala/framework/acc/frontend/src/csrc/decoder/src/lib.rs new file mode 100644 index 000000000..bace2123e --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/csrc/decoder/src/lib.rs @@ -0,0 +1,23 @@ +#[no_mangle] +pub extern "C" fn decoder_ref_domain( + funct: u32, + opcode: u32, + rs1: u64, + rvv: u32, + banks: u32, + bits: u32, +) -> u32 { + let bank = rs1 & ((1u64 << bits) - 1); + assert!(funct != 0 && ![0x57, 7, 0x27].contains(&opcode)); + if funct == 1 { + 0 + } else if [44, 79].contains(&funct) + || (rvv != 0 && funct == 32 && bank >= banks as u64 && bank < (banks + 2) as u64) + { + 2 + } else if [13, 16, 32, 33, 34, 35].contains(&funct) { + 1 + } else { + 3 + } +} diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/memory_system.cm_hier b/arch/src/main/scala/framework/acc/frontend/src/main/resources/decoder/decoder.cm_hier similarity index 100% rename from arch/src/main/scala/framework/memdomain/verification/src/main/resources/memory_system.cm_hier rename to arch/src/main/scala/framework/acc/frontend/src/main/resources/decoder/decoder.cm_hier diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/resources/decoder/decoder.f b/arch/src/main/scala/framework/acc/frontend/src/main/resources/decoder/decoder.f new file mode 100644 index 000000000..bf67c3e35 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/resources/decoder/decoder.f @@ -0,0 +1,8 @@ ++incdir+@RESOURCES@ ++incdir+@RTL@ ++incdir+@VERIFY@/uvm/src/core ++define+BUCKYBALL_DISABLE_TRACE_DPI +@VERIFY@/uvm/src/core/scenario_if.sv +@VERIFY@/uvm/src/core/scenario_package.sv +-F @RTL@/GlobalDecoder/filelist.f +@RESOURCES@/decoder_tb.sv diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/resources/decoder/decoder_tb.sv b/arch/src/main/scala/framework/acc/frontend/src/main/resources/decoder/decoder_tb.sv new file mode 100644 index 000000000..71e67f290 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/resources/decoder/decoder_tb.sv @@ -0,0 +1,75 @@ +module decoder_tb; + import uvm_pkg::*; + import scenario_pkg::*; + `include "uvm_macros.svh" + logic clock = 0, reset = 1; + always #5 clock = ~clock; + scenario_if ctl (clock); + `include "decoder_signals.svh" +GlobalDecoder dut ( + `include "decoder_ports.svh" + ); + `include "decoder_clocking.svh" + int checks = 0; + function automatic void check(bit good, string message); + verification_pkg::evidence::checks++; + checks++; + if (!good) `uvm_fatal("DECODER", message) + endfunction + `include "decoder_config.svh" + import "DPI-C" function int unsigned decoder_ref_domain( + int unsigned funct, + int unsigned opcode, + longint unsigned rs1, + int unsigned rvv, + int unsigned banks, + int unsigned bits + ); + task automatic decode(int funct, int expected_bid, longint unsigned rs1, longint unsigned rs2); + io_id_i_valid = 1; + io_id_i_bits_cmd_funct = funct; + io_id_i_bits_cmd_opcode = 'h7b; + io_id_i_bits_cmd_rs1Data = rs1; + io_id_i_bits_cmd_rs2Data = rs2; + io_id_o_ready = 0; + repeat (2) begin + @(sample); + check(sample.io_id_o_valid && !sample.io_id_i_ready, "decode backpressure mismatch"); + check(sample.io_id_o_bits_domain_id == decoder_ref_domain( + funct, 'h7b, rs1, `DECODER_RVV, `DECODER_VBANKS, `DECODER_BANK_BITS), + "domain decode mismatch"); + check( + sample.io_id_o_bits_ball_bid == expected_bid && sample.io_id_o_bits_cmd_rs1Data == rs1 && sample.io_id_o_bits_cmd_rs2Data == rs2, + "instruction payload mismatch"); + @(negedge clock); + end + io_id_o_ready = 1; + @(sample); + check(sample.io_id_i_ready, "decoder did not accept ready output"); + @(negedge clock); + io_id_i_valid = 0; + endtask + + initial begin + uvm_config_db#(virtual scenario_if)::set(null, "*", "vif", ctl); + uvm_config_db#(time)::set(null, "uvm_test_top", "timeout", 2ms); + run_test("protocol_test"); + end + initial begin + `include "decoder_init.svh" + wait (ctl.start); + repeat (3) @(sample); + @(negedge clock); + reset = 0; + decode(1, 0, 0, 0); + decode(32, 0, 0, 1056); + decode(33, 0, 2 << 20, 16); + decode(34, 0, (64'h1 << 63) | (2 << 20), 16); + decode(35, 0, 0, 16); + decode(16, 0, 0, 16); + decode(13, 0, 0, 0); + `include "decoder_cases.svh" +`uvm_info("DECODER", $sformatf("%0d checked outputs", checks), UVM_LOW) + ctl.done = 1; + end +endmodule diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/Emit.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/Emit.scala new file mode 100644 index 000000000..c3e49b0e9 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/Emit.scala @@ -0,0 +1,33 @@ +package framework.acc.frontend + +import sims.configs.SimParam +import _root_.configs.GlobalConfig + +object Emit { + + def emit( + b: GlobalConfig, + hartId: Int, + threadId: Int, + directory: String, + firtoolOpts: Array[String] + ): Unit = { + val core = + b.device.cores.find(_.hartId == hartId).getOrElse(throw new IllegalArgumentException(s"Unknown hart ID: $hartId")) + val thread = + core.threads.find(_.id == threadId).getOrElse(throw new IllegalArgumentException(s"Unknown thread ID: $threadId")) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Frontend( + thread.acc.get.frontend, + thread.acc.get.memDomain, + thread.acc.get.gpDomain, + thread.acc.get.ballDomain, + core.privileged.permission, + b.sim.copy(diffTest = false) + ), + firtoolOpts = firtoolOpts, + args = Array("--target-dir", directory, "--split-verilog") + ) + } + +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/Frontend.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/Frontend.scala new file mode 100644 index 000000000..beac08af1 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/Frontend.scala @@ -0,0 +1,97 @@ +package framework.acc.frontend + +import sims.configs.SimParam +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.core.memory.access.preflight.configs.PmpParams + +import framework.acc.debug.FrontendDebugPort + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.frontend.boot.BootRom +import framework.acc.frontend.decoder.GlobalDecoder +import framework.acc.memdomain.bank_router.BankMappingIO +import framework.acc.frontend.scheduler.{Complete, Issue, MemoryIssue, Scheduler} +import framework.acc.memdomain.{Response => MemoryResponse} + +class Command(dataBits: Int) extends Bundle { + + val raw_inst = UInt(32.W) + val pc = UInt(dataBits.W) + val rs1Data = UInt(dataBits.W) + val rs2Data = UInt(dataBits.W) +} + +/** Frontend composition; boundary ports connect directly to their owning child IP. */ +@instantiable +class Frontend( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val gpParam: GpDomainParam, + val ballParam: BallDomainParam, + val permissionParam: PmpParams, + val simp: SimParam) + extends Module { + val diffTest = simp.diffTest + val commandDataBits = frontendParam.commandDataBits + val ownerId = frontendParam.ownerId + + @public + val io = IO(new Bundle { + + val rocc = new RoCCIO(commandDataBits, permissionParam) + + // Request payloads leave Frontend; each ready signal returns from its domain. + val ballRequest = Decoupled(new MemoryIssue(frontendParam, memParam, permissionParam)) + // Completion payloads enter Frontend; Frontend drives their ready signals. + val ballCompletion = Flipped(Decoupled(new MemoryResponse(frontendParam))) + + // Bank-version update request to MemDomain; its ready input gates the update. + val mappingUpdate = new BankMappingIO(frontendParam) + val memoryRequest = Decoupled(new MemoryIssue(frontendParam, memParam, permissionParam)) + val memoryCompletion = Flipped(Decoupled(new MemoryResponse(frontendParam))) + + val gpRequest = Option.when(gpParam.enable)(Decoupled(new Issue(frontendParam, memParam))) + val gpCompletion = Option.when(gpParam.enable)(Flipped(Decoupled(new Complete(frontendParam)))) + + val barrier = new Bundle { + val arrive = Output(Bool()) + val release = Input(Bool()) + } + + val debug = new FrontendDebugPort(frontendParam, simp) + }) + + val ingress: Instance[Ingress] = Instantiate(new Ingress(permissionParam, frontendParam)) + val boot: Instance[BootRom] = Instantiate(new BootRom(ballParam.balls.map(_.id), permissionParam, frontendParam)) + val gDecoder: Instance[GlobalDecoder] = + Instantiate(new GlobalDecoder(frontendParam, memParam, gpParam, permissionParam)) + val scheduler: Instance[Scheduler] = + Instantiate(new Scheduler(frontendParam, memParam, gpParam, permissionParam, simp)) + + io.rocc <> ingress.io.rocc + boot.io.command <> ingress.io.command + ingress.io.fault := scheduler.io.fault + ingress.io.idle := boot.io.idle + gDecoder.io.command <> boot.io.cmd + scheduler.io.admission.command <> gDecoder.io.decoded + boot.io.schedulerIdle := scheduler.io.admission.idle + + io.ballRequest <> scheduler.io.ballRequest + scheduler.io.ballCompletion <> io.ballCompletion + io.memoryRequest <> scheduler.io.memoryRequest + scheduler.io.memoryCompletion <> io.memoryCompletion + io.gpRequest.zip(scheduler.io.gpRequest).foreach { case (port, source) => port <> source } + io.gpCompletion.zip(scheduler.io.gpCompletion).foreach { case (port, sink) => sink <> port } + io.debug.trace := scheduler.io.traceInstructions + io.debug.control.zip(scheduler.io.debug).foreach { case (port, source) => port <> source } + io.mappingUpdate <> scheduler.io.mappingUpdate + + io.barrier.arrive := scheduler.io.barrier.arrive + scheduler.io.barrier.release := io.barrier.release + +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/Ingress.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/Ingress.scala new file mode 100644 index 000000000..b9335d7d1 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/Ingress.scala @@ -0,0 +1,56 @@ +package framework.acc.frontend + +import framework.core.memory.access.preflight.configs.PmpParams +import framework.acc.frontend.configs.FrontendParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.acc.memdomain.MemoryContext + +class CommandPacket(permissionParam: PmpParams, fp: FrontendParam) extends Bundle { + val commandDataBits = fp.commandDataBits + val instruction = new Command(commandDataBits) + val context = new MemoryContext(permissionParam) + val runtime = Bool() +} + +/** Holds an accepted RoCC command until the frontend takes it. */ +@instantiable +class Ingress(permissionParam: PmpParams, fp: FrontendParam) extends Module { + val commandDataBits = fp.commandDataBits + + @public + val io = IO(new Bundle { + val rocc = new RoCCIO(commandDataBits, permissionParam) + val command = Decoupled(new CommandPacket(permissionParam, fp)) + val idle = Input(Bool()) + val fault = Input(Bool()) + }) + + val pending = RegInit(false.B) + val packet = Reg(new CommandPacket(permissionParam, fp)) + io.rocc.cmd.ready := !pending && !io.fault && !reset.asBool + when(io.rocc.cmd.fire) { + assert(io.rocc.cmd.bits.instruction.raw_inst(6, 0) === "h7b".U, "RoCC accepts accelerator commands only") + assert( + !io.rocc.cmd.bits.instruction.raw_inst(14), + "Accelerator commands report completion without register responses" + ) + packet.instruction := io.rocc.cmd.bits.instruction + packet.context.satp := io.rocc.cmd.bits.satp + packet.context.privilege := io.rocc.cmd.bits.effectivePrivilege + packet.context.sum := io.rocc.cmd.bits.sum + packet.context.mxr := io.rocc.cmd.bits.mxr + packet.context.pmp := io.rocc.cmd.bits.pmp + packet.runtime := true.B + pending := true.B + } + io.command.valid := pending && !io.fault + io.command.bits := packet + when(io.command.fire)(pending := false.B) + io.rocc.busy := pending || !io.idle || io.fault + io.rocc.interrupt := io.fault + io.rocc.resp.valid := false.B + io.rocc.resp.bits := 0.U.asTypeOf(io.rocc.resp.bits) +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/RoCC.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/RoCC.scala new file mode 100644 index 000000000..ad3a73d17 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/RoCC.scala @@ -0,0 +1,30 @@ +package framework.acc.frontend + +import chisel3._ +import chisel3.util._ +import framework.core.memory.access.preflight.PmpEntry +import framework.core.memory.access.preflight.configs.PmpParams + +class Response(dataBits: Int) extends Bundle { + + val rd = UInt(5.W) + val data = UInt(dataBits.W) +} + +/** RoCC instruction operands and the frozen submitting address-space context. */ +class RoCCCommand(dataBits: Int, permission: PmpParams) extends Bundle { + val instruction = new Command(dataBits) + val satp = UInt(64.W) + val effectivePrivilege = UInt(2.W) + val sum = Bool() + val mxr = Bool() + val pmp = Vec(permission.entries, new PmpEntry(permission)) +} + +/** Accelerator-facing RoCC direction; the host uses Flipped. */ +class RoCCIO(dataBits: Int, permission: PmpParams) extends Bundle { + val cmd = Flipped(Decoupled(new RoCCCommand(dataBits, permission))) + val resp = Decoupled(new Response(dataBits)) + val busy = Output(Bool()) + val interrupt = Output(Bool()) +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/build.sc b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/build.sc new file mode 100644 index 000000000..827b8bd4d --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/build.sc @@ -0,0 +1,20 @@ +import mill._ +import $file.^.^.^.^.^.^.^.^.^.^.settings +val ipRoot = millSourcePath + +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.acc.frontend.boot.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Boot.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/configs/uvm.toml b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/configs/uvm.toml new file mode 100644 index 000000000..e57661e5d --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/configs/uvm.toml @@ -0,0 +1,21 @@ +[parameters] +ballIds = [0, 3, 31] + +[parameters.permission] +addressBits = 44 +entries = 16 + +[parameters.frontend] +ownerId = 0 +commandDataBits = 64 +rob_entries = 4 +bank_alias_enable = true +rs_out_of_order_response = true +bank_id_len = 4 +max_private_vbank = 3 +shared_vbank_base = 16 +iter_len = 16 +sub_rob_enable = false +sub_rob_depth = 4 + +[defines] diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/agent/driver.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/agent/driver.svh new file mode 100644 index 000000000..87656e013 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/agent/driver.svh @@ -0,0 +1,25 @@ +class driver extends uvm_driver #(cycle_item); + `uvm_component_utils(driver) + virtual boot_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual boot_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Boot interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item req; + forever begin + seq_item_port.get_next_item(req); + @(negedge vif.clock); + vif.reset = req.reset; + vif.inputs = req.inputs; + @(posedge vif.clock); + req.outputs = vif.outputs; + #1; + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/agent/item.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/agent/item.svh new file mode 100644 index 000000000..8b1798268 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/agent/item.svh @@ -0,0 +1,18 @@ +class cycle_item extends uvm_sequence_item; + `uvm_object_utils(cycle_item) + logic [`BOOT_INPUT_BITS-1:0] inputs; + logic [`BOOT_OUTPUT_BITS-1:0] outputs; + bit reset; + function new(string name = "cycle_item"); + super.new(name); + endfunction +endclass +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(payload, UVM_ALL_ON) + `uvm_object_utils_end + logic [`BOOT_PACKET_BITS-1:0] payload; + function new(string name = "result_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/agent/monitor.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/agent/monitor.svh new file mode 100644 index 000000000..1959ba4bb --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/agent/monitor.svh @@ -0,0 +1,25 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual boot_if vif; + uvm_analysis_port #(cycle_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual boot_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Boot monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item t; + forever begin + @(posedge vif.clock); + t = cycle_item::type_id::create("observed"); + t.inputs = vif.inputs; + t.outputs = vif.outputs; + t.reset = vif.reset; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/env/env.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/env/env.svh new file mode 100644 index 000000000..f1fc73e22 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/env/env.svh @@ -0,0 +1,31 @@ +class env extends uvm_env; + `uvm_component_utils(env) + driver drv; + monitor mon; + model reference; + uvm_sequencer #(cycle_item) seqr; + result_board #(result_item, 0) scoreboard; + uvm_analysis_imp #(bit, env) reset_export; + function new(string name, uvm_component parent); + super.new(name, parent); + reset_export = new("reset_export", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + drv = driver::type_id::create("driver", this); + mon = monitor::type_id::create("monitor", this); + reference = model::type_id::create("model", this); + seqr = new("seqr", this); + scoreboard = result_board#(result_item, 0)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + drv.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(reference.analysis_export); + reference.expected_ap.connect(scoreboard.expected_export); + reference.actual_ap.connect(scoreboard.actual_export); + reference.reset_ap.connect(reset_export); + endfunction + function void write(bit value); + scoreboard.cancel_pending(); + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/env/model.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/env/model.svh new file mode 100644 index 000000000..8ac568d0f --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/env/model.svh @@ -0,0 +1,100 @@ +class model extends uvm_subscriber #(cycle_item); + `uvm_component_utils(model) + int ball_ids[`BOOT_BALL_COUNT] = `BOOT_BALL_IDS; + bit start_epoch = 1, waiting = 0, runtime_phase = 0; + int seen = 0; + uvm_analysis_port #(result_item) expected_ap, actual_ap; + uvm_analysis_port #(bit) reset_ap; + covergroup behavior with function sample ( + bit runtime_command, int target, bit input_stall, bit output_stall, bit scheduler_idle + ); + option.per_instance = 1; + command_kind: coverpoint runtime_command; + initialization_target: coverpoint target iff (!runtime_command) { + bins selected[] = `BOOT_ID_BINS; + } + source_backpressure: coverpoint input_stall; + sink_backpressure: coverpoint output_stall; + scheduler_drained: coverpoint scheduler_idle; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected_ap = new("expected_ap", this); + actual_ap = new("actual_ap", this); + reset_ap = new("reset_ap", this); + behavior = new(); + endfunction + function void write(cycle_item t); + result_item expected, actual; + bit input_fire, output_fire, scheduler_idle; + if (t.reset) begin + reset_ap.write(1); + start_epoch = 1; + waiting = 0; + runtime_phase = 0; + seen = 0; + return; + end + if (start_epoch) begin + foreach (ball_ids[i]) begin + expected = result_item::type_id::create("initialization"); + expected.payload = '0; + expected.payload[`BOOT_OUT_CMD_BITS_INSTRUCTION_RAW_INST_OFFSET-`BOOT_OUT_PACKET_OFFSET+:32] = 32'h0a00007b; + expected.payload[`BOOT_OUT_CMD_BITS_INSTRUCTION_RS2DATA_OFFSET-`BOOT_OUT_PACKET_OFFSET+:`BOOT_OUT_CMD_BITS_INSTRUCTION_RS2DATA_WIDTH] = ball_ids[i]; + expected_ap.write(expected); + end + start_epoch = 0; + end + input_fire = `BI(t.inputs, COMMAND_VALID) && `BO(t.outputs, COMMAND_READY); + output_fire = `BO(t.outputs, CMD_VALID) && `BI(t.inputs, CMD_READY); + scheduler_idle = `BI(t.inputs, SCHEDULERIDLE); + if (seen < `BOOT_BALL_COUNT) + evidence::check(!`BO(t.outputs, COMMAND_READY), + "Boot accepted runtime input before initialization ended"); + if (waiting) begin + if (seen < `BOOT_BALL_COUNT || !scheduler_idle) + evidence::check(!`BO(t.outputs, CMD_VALID), + "Boot issued a command before prior work drained"); + if (scheduler_idle) waiting = 0; + end + if (seen == `BOOT_BALL_COUNT && !waiting && ( + `BO(t.outputs, COMMAND_READY) + || + `BO(t.outputs, IDLE) + )) + runtime_phase = 1; + if (runtime_phase) begin + evidence::check(`BO(t.outputs, COMMAND_READY) === `BI(t.inputs, CMD_READY), + "Boot runtime ready did not pass through"); + evidence::check(`BO(t.outputs, CMD_VALID) === `BI(t.inputs, COMMAND_VALID), + "Boot runtime valid did not pass through"); + end + if (input_fire) begin + evidence::check(seen == `BOOT_BALL_COUNT && !waiting, + "Boot forwarded a runtime command before initialization drained"); + expected = result_item::type_id::create("runtime"); + expected.payload = t.inputs[`BOOT_IN_PACKET_OFFSET+:`BOOT_PACKET_BITS]; + expected_ap.write(expected); + end + if (output_fire) begin + actual = result_item::type_id::create("actual"); + actual.payload = t.outputs[`BOOT_OUT_PACKET_OFFSET+:`BOOT_PACKET_BITS]; + if (seen < `BOOT_BALL_COUNT) begin + behavior.sample(0, ball_ids[seen], `BI(t.inputs, COMMAND_VALID) && ! + `BO(t.outputs, COMMAND_READY), 0, scheduler_idle); + seen++; + waiting = 1; + end else begin + evidence::check(input_fire, "Boot produced an unsolicited runtime output"); + behavior.sample(1, 0, 0, 0, scheduler_idle); + end + actual_ap.write(actual); + end + if (!output_fire) + behavior.sample(1, 0, `BI(t.inputs, COMMAND_VALID) && !`BO(t.outputs, COMMAND_READY), + `BO(t.outputs, CMD_VALID) && !`BI(t.inputs, CMD_READY), scheduler_idle); + if (`BO(t.outputs, IDLE)) + evidence::check(seen == `BOOT_BALL_COUNT && scheduler_idle && !`BI(t.inputs, COMMAND_VALID) + && !`BO(t.outputs, CMD_VALID), "Boot declared idle before all work drained"); + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/filelist.f b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/filelist.f new file mode 100644 index 000000000..c357eb6a5 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/BootRom/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/interface.sv b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/interface.sv new file mode 100644 index 000000000..71dc6d8a5 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/interface.sv @@ -0,0 +1,14 @@ +`include "parameters.svh" +interface boot_if ( + input logic clock +); + logic reset = 1; + logic [`BOOT_INPUT_BITS-1:0] inputs; + logic [`BOOT_OUTPUT_BITS-1:0] outputs; + assert property (@(posedge clock) disable iff (reset) + outputs[`BOOT_OUT_CMD_VALID_OFFSET] && !inputs[`BOOT_IN_CMD_READY_OFFSET] |=> + outputs[`BOOT_OUT_CMD_VALID_OFFSET] && $stable( + outputs[`BOOT_OUT_PACKET_OFFSET+:`BOOT_PACKET_BITS] + )) + else $fatal(1, "Boot output changed under backpressure"); +endinterface diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/package.sv b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/package.sv new file mode 100644 index 000000000..ed6ff959c --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/package.sv @@ -0,0 +1,15 @@ +`include "parameters.svh" +`define BI(P, N) P[`BOOT_IN_``N``_OFFSET+:`BOOT_IN_``N``_WIDTH] +`define BO(P, N) P[`BOOT_OUT_``N``_OFFSET+:`BOOT_OUT_``N``_WIDTH] +package boot_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..c7441adc5 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/seq/sequence.svh @@ -0,0 +1,69 @@ +class scenario_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(scenario_sequence) + logic [`BOOT_INPUT_BITS-1:0] pins = '0; + logic [`BOOT_OUTPUT_BITS-1:0] observed; + bit reset = 1; + int pending_work = 0, cycle = 0; + function new(string name = "scenario_sequence"); + super.new(name); + endfunction + task tick(int cycles = 1); + cycle_item req; + repeat (cycles) begin + `BI(pins, SCHEDULERIDLE) = pending_work == 0; + req = cycle_item::type_id::create("cycle"); + start_item(req); + req.inputs = pins; + req.reset = reset; + finish_item(req); + observed = req.outputs; + if (pending_work > 0) pending_work--; + if (!reset && `BO(observed, CMD_VALID) && `BI(pins, CMD_READY)) pending_work = 3; + cycle++; + end + endtask + task restart(); + pins = '0; + reset = 1; + pending_work = 0; + tick(2); + reset = 0; + tick(); + endtask + task send(logic [`BOOT_PACKET_BITS-1:0] payload); + pins[`BOOT_IN_PACKET_OFFSET+:`BOOT_PACKET_BITS] = payload; + `BI(pins, COMMAND_VALID) = 1; + do begin + `BI(pins, CMD_READY) = cycle % 3 == 0; + tick(); + end while (! + `BO(observed, COMMAND_READY) + ); + `BI(pins, COMMAND_VALID) = 0; + endtask + task drain(); + `BI(pins, CMD_READY) = 1; + do tick(); while (!`BO(observed, IDLE)); + endtask + task body(); + logic [`BOOT_PACKET_BITS-1:0] packet; + restart(); + // A runtime producer may hold VALID while the ROM is still initializing balls. + packet = '1; + packet[`BOOT_IN_COMMAND_BITS_INSTRUCTION_RAW_INST_OFFSET-`BOOT_IN_PACKET_OFFSET+:32] = 32'h4a00007b; + send(packet); + drain(); + for (int bit_index = 0; bit_index < `BOOT_PACKET_BITS; bit_index++) begin + packet = (`BOOT_PACKET_BITS'(1) << bit_index) ^ ((bit_index % 2) ? '1 : '0); + packet[`BOOT_IN_COMMAND_BITS_INSTRUCTION_RAW_INST_OFFSET-`BOOT_IN_PACKET_OFFSET+:32] = (32'(bit_index%128)<<25) | 'h7b; + send(packet); + end + drain(); + // Restart after only one initialization command; the remaining expected commands are cancelled. + restart(); + `BI(pins, CMD_READY) = 1; + do tick(); while (!`BO(observed, CMD_VALID)); + restart(); + drain(); + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/suite.toml b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/suite.toml new file mode 100644 index 000000000..54539e5fb --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/suite.toml @@ -0,0 +1,20 @@ +[model] +manifest = "" +library = "" + +[[tests]] +name = "boot" +top = "boot_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 300000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +functional_groups = ["boot_pkg::model::behavior"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/tb.sv b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/tb.sv new file mode 100644 index 000000000..3cd0e2dfb --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/tb.sv @@ -0,0 +1,16 @@ +module boot_tb; + import uvm_pkg::*; + import boot_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + boot_if bus (clock); + BootRom dut ( + .clock(clock), + .reset(bus.reset), + `include "ports.svh" + ); + initial begin + uvm_config_db#(virtual boot_if)::set(null, "uvm_test_top*", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/tests/test.svh new file mode 100644 index 000000000..54d137a6f --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/tests/test.svh @@ -0,0 +1,15 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.start(test_env.seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/scala/Boot.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/scala/Boot.scala new file mode 100644 index 000000000..0d6e27a31 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/scala/Boot.scala @@ -0,0 +1,93 @@ +package framework.acc.frontend.boot + +import framework.acc.frontend.configs.FrontendParam +import framework.core.memory.access.preflight.configs.PmpParams + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public} +import chisel3.util._ +import framework.acc.balldomain.decoder.DISA.BALL_INIT_BITPAT +import framework.acc.frontend.CommandPacket + +@instantiable +class BootRom(val ballIds: Seq[Int], val permissionParam: PmpParams, val fp: FrontendParam) extends Module { + val commandDataBits = fp.commandDataBits + + val commandBits = commandDataBits + + @public + val io = IO(new Bundle { + + val command = Flipped(Decoupled(new CommandPacket(permissionParam, fp))) + val cmd = Decoupled(new CommandPacket(permissionParam, fp)) + val schedulerIdle = Input(Bool()) + val idle = Output(Bool()) + }) + + val ballInitRecords = ballIds.map { id => + require(id >= 0 && id < 32, s"BALL_INIT target ID out of range: $id") + BuckyballCommand(BALL_INIT_BITPAT.value.toInt, rs1 = 0, rs2 = BigInt(id)) + } + + val bootRecords = ballInitRecords + + val bootFuncts = VecInit(bootRecords.map(r => r.funct.U(7.W))) + val bootRs1Data = VecInit(bootRecords.map(r => r.rs1.U(commandBits.W))) + val bootRs2Data = VecInit(bootRecords.map(r => r.rs2.U(commandBits.W))) + val bootPcWidth = math.max(1, log2Ceil(bootRecords.length + 1)) + + val active = RegInit(true.B) + val drain = RegInit(false.B) + val waitIdle = RegInit(false.B) + val pc = RegInit(0.U(bootPcWidth.W)) + + val atEnd = pc === bootRecords.length.U(bootPcWidth.W) + + val current = Wire(new CommandPacket(permissionParam, fp)) + current := 0.U.asTypeOf(new CommandPacket(permissionParam, fp)) + current.instruction.raw_inst := Cat( + Mux(atEnd, 0.U(7.W), bootFuncts(pc)), + 0.U(18.W), + BuckyballCommand.Custom3Opcode.U(7.W) + ) + current.instruction.rs1Data := Mux(atEnd, 0.U, bootRs1Data(pc)) + current.instruction.rs2Data := Mux(atEnd, 0.U, bootRs2Data(pc)) + + val injectValid = active && !drain && !waitIdle && !atEnd + + when(active && !drain && atEnd) { + drain := true.B + } + when(waitIdle && io.schedulerIdle) { + waitIdle := false.B + } + when(drain && io.schedulerIdle) { + active := false.B + } + when(active && io.cmd.fire) { + pc := pc + 1.U + waitIdle := true.B + } + + // External commands enter only after all reset-time initialization has retired. + io.cmd.valid := Mux(active, injectValid, io.command.valid) + io.cmd.bits := Mux(active, current, io.command.bits) + io.command.ready := !active && io.cmd.ready + io.idle := !active && io.schedulerIdle && !io.command.valid + + when(io.command.fire) { + assert(!io.command.bits.instruction.raw_inst(14), "Frontend commands must not request a register response") + } +} + +/** + * Elaboration-time representation of one Buckyball custom instruction. + * Domain ISA builders own the funct7 and operand encodings; boot only streams + * these already-formed commands. + */ +case class BuckyballCommand(funct: Int, rs1: BigInt = 0, rs2: BigInt = 0) + +object BuckyballCommand { + val Custom3Opcode = 0x7b + val Custom3Funct3 = 3 +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/scala/Emit.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/scala/Emit.scala new file mode 100644 index 000000000..4cd4a4b88 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/scala/Emit.scala @@ -0,0 +1,60 @@ +package framework.acc.frontend.boot + +import chisel3._ +import java.nio.file.{Files, Paths} +import framework.acc.frontend.configs.FrontendParam +import framework.core.memory.access.preflight.configs.PmpParams + +object Emit extends App { + val Array("--config", config, "--output-dir", output) = args + val values = ujson.read(Files.readString(Paths.get(config))) + val frontend = upickle.default.read[FrontendParam](values("frontend")) + val permission = upickle.default.read[PmpParams](values("permission")) + val ballIds = values("ballIds").arr.toSeq.map(_.num.toInt) + Files.createDirectories(Paths.get(output)) + val ports = scala.collection.mutable.ArrayBuffer[String]() + val definitions = scala.collection.mutable.ArrayBuffer[String]() + val offsets = scala.collection.mutable.Map[String, Int]() + var inputOffset = 0 + var outputOffset = 0 + var packetBits = 0 + + def fields(data: Data, path: String): Seq[(String, Int)] = data match { + case record: Record => record.elements.toSeq.flatMap { case (name, field) => + fields(field, if (path.isEmpty) name else s"${path}_$name") + } + case vector: Vec[_] => vector.toSeq.zipWithIndex.flatMap { case (field, index) => fields(field, s"${path}_$index") } + case leaf => Seq(path -> leaf.getWidth) + } + + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + { + val dut = new BootRom(ballIds, permission, frontend) + packetBits = dut.io.command.bits.getWidth + for ((name, width) <- fields(dut.io, "")) { + val input = + name == "cmd_ready" || name == "schedulerIdle" || name == "command_valid" || name.startsWith("command_bits_") + val direction = if (input) "IN" else "OUT" + val signal = if (input) "inputs" else "outputs" + val offset = if (input) inputOffset else outputOffset + definitions += s"`define BOOT_${direction}_${name.toUpperCase}_OFFSET $offset" + definitions += s"`define BOOT_${direction}_${name.toUpperCase}_WIDTH $width" + offsets(name) = offset + ports += s".io_$name(bus.$signal[$offset +: $width])" + if (input) inputOffset += width else outputOffset += width + } + dut + }, + args = Array("--target-dir", s"$output/BootRom", "--split-verilog") + ) + definitions += s"`define BOOT_INPUT_BITS $inputOffset" + definitions += s"`define BOOT_OUTPUT_BITS $outputOffset" + definitions += s"`define BOOT_PACKET_BITS $packetBits" + definitions += s"`define BOOT_IN_PACKET_OFFSET ${offsets.filter(_._1.startsWith("command_bits_")).values.min}" + definitions += s"`define BOOT_OUT_PACKET_OFFSET ${offsets.filter(_._1.startsWith("cmd_bits_")).values.min}" + definitions += s"`define BOOT_BALL_COUNT ${ballIds.size}" + definitions += s"`define BOOT_BALL_IDS ${ballIds.mkString("'{", ",", "}")}" + definitions += s"`define BOOT_ID_BINS ${ballIds.mkString("{", ",", "}")}" + Files.writeString(Paths.get(output, "parameters.svh"), definitions.mkString("\n") + "\n") + Files.writeString(Paths.get(output, "ports.svh"), ports.mkString(",\n") + "\n") +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/configs/FrontendParam.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/configs/FrontendParam.scala new file mode 100644 index 000000000..8590840ea --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/configs/FrontendParam.scala @@ -0,0 +1,45 @@ +package framework.acc.frontend.configs + +import buckyball.config.FrontendConfig + +import upickle.default._ + +/** + * Frontend parameters - includes all frontend configuration. + */ +case class FrontendParam( + ownerId: Int, + commandDataBits: Int, + rob_entries: Int, + bank_alias_enable: Boolean, + rs_out_of_order_response: Boolean, + bank_id_len: Int, + max_private_vbank: Int, + /** ISA bank-id namespace base for shared banks: raw id >= this means shared. */ + shared_vbank_base: Int, + iter_len: Int, + sub_rob_enable: Boolean, + sub_rob_depth: Int) + +object FrontendParam { + + def fromProto(p: FrontendConfig, commandDataBits: Int, ownerId: Int): FrontendParam = { + require(p.hasBankAliasEnable, "Frontend bank alias policy is required") + FrontendParam( + ownerId, + commandDataBits, + p.getRobEntries, + p.getBankAliasEnable, + p.getRsOutOfOrderResponse, + p.getBankIdLen, + p.getMaxPrivateVbank, + p.getSharedVbankBase, + p.getIterLen, + p.getSubRobEnable, + p.getSubRobDepth + ) + } + + implicit val rw: ReadWriter[FrontendParam] = macroRW + +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/decoder/DISA.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/decoder/DISA.scala new file mode 100644 index 000000000..a0258e9f3 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/decoder/DISA.scala @@ -0,0 +1,16 @@ +package framework.acc.frontend.decoder + +import chisel3._ +import chisel3.util._ + +object DISA { + val BARRIER_BITPAT = BitPat("b0000001") // 1 (0x01) — enable=000, opcode=1 +} + +// Domain ID constants +object DomainId { + val FRONTEND = 0.U(4.W) // Frontend (barrier), does not enter ROB queue + val GP = 2.U(4.W) // Private kernel execution and image control + val MEM = 1.U(4.W) // Memory domain + val BALL = 3.U(4.W) // Ball domain +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/decoder/GlobalDecoder.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/decoder/GlobalDecoder.scala new file mode 100644 index 000000000..7caffcb92 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/decoder/GlobalDecoder.scala @@ -0,0 +1,178 @@ +package framework.acc.frontend.decoder + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.core.memory.access.preflight.configs.PmpParams + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.acc.frontend.scheduler.scoreboard.BankAccessInfo + +import framework.acc.frontend.{Command, CommandPacket} +import framework.acc.memdomain.MemoryContext +import framework.acc.frontend.decoder.DISA._ +import framework.acc.gpdomain.decoder.DISA._ +import framework.acc.memdomain.DISA._ + +class PostGDCmd(frontendParam: FrontendParam, memParam: MemDomainParam) extends Bundle { + val commandDataBits = frontendParam.commandDataBits + + val runtime = Bool() + val domain_id = UInt(4.W) + val iter = UInt(frontendParam.iter_len.W) + val cmd = new Command(commandDataBits) + val bankAccess = new BankAccessInfo(frontendParam.bank_id_len) + val architecturalDestination = UInt(frontendParam.bank_id_len.W) + val op1_col = UInt(memParam.groupCountWidth.W) + val op2_col = UInt(memParam.groupCountWidth.W) + val wr_col = UInt(memParam.groupCountWidth.W) + val wr_read_bank = UInt(frontendParam.bank_id_len.W) + val isBarrier = Bool() +} + +/** Only the frontend pipeline carries address-space context for later memory dispatch. */ +class DecodedInstruction(frontendParam: FrontendParam, memParam: MemDomainParam, permissionParam: PmpParams) + extends Bundle { + val commandDataBits = frontendParam.commandDataBits + val operation = new PostGDCmd(frontendParam, memParam) + val context = new MemoryContext(permissionParam) +} + +@instantiable +class GlobalDecoder( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val gpParam: GpDomainParam, + val permissionParam: PmpParams) + extends Module { + val commandDataBits = frontendParam.commandDataBits + + val bankIdLen = frontendParam.bank_id_len + + @public + val io = IO(new Bundle { + + val command = Flipped(Decoupled(new CommandPacket(permissionParam, frontendParam))) + + val decoded = Decoupled(new DecodedInstruction(frontendParam, memParam, permissionParam)) + }) + + // Decoding follows the consumer handshake. + io.command.ready := io.decoded.ready + + val func7 = io.command.bits.instruction.raw_inst(31, 25) + val opcode = io.command.bits.instruction.raw_inst(6, 0) + val rs1 = io.command.bits.instruction.rs1Data + + // Route memory and GP commands to their owning domains. + val is_mem_inst = (func7 === MVIN_MMIO_BITPAT) || (func7 === MVIN_BITPAT) || + (func7 === MVIN_2D_BITPAT) || + (func7 === MVOUT_BITPAT) || + (func7 === MSET_BITPAT) || + (func7 === MVOVER_BITPAT) || + (func7 === MVIN_KERNEL_BITPAT) + + val is_barrier_inst = func7 === BARRIER_BITPAT + + val is_kernel_inst = func7 === RUN_KERNEL_BITPAT + val is_bare_rvv = opcode === "h57".U || opcode === "h07".U || opcode === "h27".U + val is_ball_inst = !is_mem_inst && !is_barrier_inst && !is_bare_rvv && !is_kernel_inst + + when(io.command.fire) { + assert(func7 =/= 0.U, "GlobalDecoder: funct7 zero is not a Buckyball instruction") + assert(!is_bare_rvv, "GlobalDecoder: bare RVV instructions are not Buckyball commands") + when(is_kernel_inst) { + assert(gpParam.enable.B, "GlobalDecoder: kernel command requires gpDomain.enable=true") + } + } + + // Encode domain ID + val domain_id = MuxCase( + DomainId.BALL, + Seq( + is_barrier_inst -> DomainId.FRONTEND, + is_kernel_inst -> DomainId.GP, + is_mem_inst -> DomainId.MEM, + is_ball_inst -> DomainId.BALL + ) + ) + + // ------------------------------------------------------------------------- + // Bank access info extraction — enable flags from funct7[6:4] + // + // Unified rs1 layout (defined in isa.h): + // rs1[9:0] = bank_0 (rd_bank_0 or MSET config bank) + // rs1[19:10] = bank_1 (rd_bank_1, dual-operand only) + // rs1[29:20] = bank_2 (wr_bank for Ball instructions) + // rs1[62:30] = iter; rs1[63] declares an in-place update. + // + // funct7[6:4] enable encoding: + // 000 = no bank access + // 001 = 1 read (bank0) + // 010 = 1 write (bank2) + // 011 = 1 read + 1 write (bank0 read, bank2 write) + // 100 = 2 read + 1 write (bank0+bank1 read, bank2 write) + // 101,110,111 = no bank access (extended opcode space) + // ------------------------------------------------------------------------- + val bankAccess = Wire(new BankAccessInfo(frontendParam.bank_id_len)) + val enableBits = func7(6, 4) + + // Decode enable from funct7[6:4] + val msetTransfer = func7 === MSET_BITPAT && io.command.bits.instruction.rs2Data(12) + val hasRd0 = enableBits === 1.U || enableBits === 3.U || enableBits === 4.U || msetTransfer + val hasRd1 = enableBits === 4.U || msetTransfer + val hasWr = (enableBits === 2.U || enableBits === 3.U || enableBits === 4.U) && func7 =/= MVIN_MMIO_BITPAT + + bankAccess.rd_bank_0_valid := hasRd0 + bankAccess.rd_bank_0_id := rs1(bankIdLen - 1, 0) + bankAccess.rd_bank_1_valid := hasRd1 + bankAccess.rd_bank_1_id := Mux(msetTransfer, rs1(bankIdLen + 19, 20), rs1(bankIdLen + 9, 10)) + bankAccess.wr_bank_valid := hasWr + + val inplaceUpdate = hasWr && func7 =/= MSET_BITPAT && + (rs1( + 63 + ) || is_kernel_inst || func7 === MVIN_2D_BITPAT || (func7 === MVIN_BITPAT && io.command.bits.instruction.rs2Data(63))) + + val readOldWrite = hasWr && func7 =/= MSET_BITPAT && (rs1(29) || inplaceUpdate) + val writeBank = rs1(28, 20).pad(bankIdLen) + bankAccess.wr_bank_id := Mux( + func7 === MSET_BITPAT, + Mux(msetTransfer, rs1(bankIdLen + 19, 20), rs1(bankIdLen - 1, 0)), + writeBank + ) + bankAccess.rd_wr_valid := readOldWrite + bankAccess.rd_wr_id := writeBank + bankAccess.wr_update := inplaceUpdate + + def legalBank(raw: UInt): Bool = + raw <= frontendParam.max_private_vbank.U || + (gpParam.enable.B && raw >= memParam.virtualBankCount.U && raw < (memParam.virtualBankCount + 2).U) || + (memParam.sharedEnable.B && raw >= frontendParam.shared_vbank_base.U && + raw < memParam.virtualBankCount.U) + + val usesArchitecturalBank = is_ball_inst || func7 === MSET_BITPAT || func7 === MVIN_BITPAT || + func7 === MVIN_2D_BITPAT || func7 === MVOUT_BITPAT + when(io.command.fire && usesArchitecturalBank) { + when(hasRd0)(assert(legalBank(bankAccess.rd_bank_0_id), "GlobalDecoder: bank0 is outside configured ranges")) + when(hasRd1)(assert(legalBank(bankAccess.rd_bank_1_id), "GlobalDecoder: bank1 is outside configured ranges")) + when(hasWr)(assert(legalBank(bankAccess.wr_bank_id), "GlobalDecoder: write bank is outside configured ranges")) + } + + // Output control + io.decoded.bits.context := io.command.bits.context + io.decoded.bits.operation.runtime := io.command.bits.runtime + io.decoded.valid := io.command.valid + io.decoded.bits.operation.domain_id := domain_id + io.decoded.bits.operation.iter := rs1(62, 30) + io.decoded.bits.operation.cmd := io.command.bits.instruction + io.decoded.bits.operation.bankAccess := bankAccess + io.decoded.bits.operation.architecturalDestination := bankAccess.wr_bank_id + io.decoded.bits.operation.op1_col := 0.U + io.decoded.bits.operation.op2_col := 0.U + io.decoded.bits.operation.wr_col := 0.U + io.decoded.bits.operation.wr_read_bank := 0.U + io.decoded.bits.operation.isBarrier := is_barrier_inst +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/build.sc b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/build.sc new file mode 100644 index 000000000..a98718ea5 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/build.sc @@ -0,0 +1,29 @@ +import mill._ +import $file.src.main.scala.bat.{build => bat_build} +import $file.src.main.scala.rob.global.{build => rob_global_build} +import $file.src.main.scala.scoreboard.{build => scoreboard_build} +import $file.^.^.^.^.^.^.^.^.^.^.settings +val ipRoot = millSourcePath + +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.acc.frontend.scheduler.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Scheduler.scala")) + ) ++ + bat_build.sources() ++ + rob_global_build.sources() ++ + scoreboard_build.sources() +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) ++ + bat_build.emitSources() ++ + rob_global_build.emitSources() ++ + scoreboard_build.emitSources() +) diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/csrc/rename_retire.cpp b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/csrc/rename_retire.cpp new file mode 100644 index 000000000..b02854893 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/csrc/rename_retire.cpp @@ -0,0 +1,140 @@ +#include "VScheduler.h" +#include "verilated.h" +#include +#include +#include +#include + +struct Issued { + unsigned read, write; +}; +struct Reclaimed { + unsigned previous, current; +}; + +int main(int argc, char **argv) { + Verilated::commandArgs(argc, argv); + Verilated::randReset(0); + VScheduler dut; + std::map issued; + std::vector reclaimed; + unsigned cycle = 0; + auto check = [](bool condition, const char *message) { + if (!condition) + throw std::runtime_error(message); + }; + auto tick = [&]() { + dut.clock = 0; + dut.eval(); + if (dut.io_ball_issue_valid && dut.io_ball_issue_ready) { + const unsigned slot = dut.io_ball_issue_bits_rob_id; + issued[slot] = {dut.io_ball_issue_bits_renamedBankAccess_rd_bank_0_id, + dut.io_ball_issue_bits_renamedBankAccess_wr_bank_id}; + std::printf("cycle=%u issue slot=%u read=%u write=%u\n", cycle, slot, + issued[slot].read, issued[slot].write); + } + if (dut.io_mapping_valid && dut.io_mapping_ready && + dut.io_mapping_bits_retire) { + reclaimed.push_back( + {dut.io_mapping_bits_previous, dut.io_mapping_bits_vbank}); + std::printf("cycle=%u reclaim previous=%u current=%u\n", cycle, + dut.io_mapping_bits_previous, dut.io_mapping_bits_vbank); + } + dut.clock = 1; + dut.eval(); + dut.clock = 0; + dut.eval(); + ++cycle; + }; + auto wait = [&](auto condition) { + for (unsigned n = 0; n < 100; ++n) { + dut.eval(); + if (condition()) + return; + tick(); + } + throw std::runtime_error("Handshake timed out"); + }; + dut.reset = 1; + for (unsigned i = 0; i < 4; ++i) + tick(); + dut.reset = 0; + dut.io_ball_issue_ready = 1; + dut.io_memory_issue_ready = 1; + dut.io_mapping_ready = 1; + dut.io_thread_id = 1; + auto send = [&](bool read, bool write) { + dut.io_decode_cmd_i_bits_domain_id = 3; + dut.io_decode_cmd_i_bits_cmd_funct = 0x63; + dut.io_decode_cmd_i_bits_cmd_pc = 4 + cycle * 4; + dut.io_decode_cmd_i_bits_bankAccess_rd_bank_0_valid = read; + dut.io_decode_cmd_i_bits_bankAccess_rd_bank_0_id = 1; + dut.io_decode_cmd_i_bits_bankAccess_wr_bank_valid = write; + dut.io_decode_cmd_i_bits_bankAccess_wr_bank_id = 1; + dut.io_decode_cmd_i_valid = 1; + wait([&] { return dut.io_decode_cmd_i_ready; }); + const unsigned slot = dut.io_allocation_bits_rob_id; + tick(); + dut.io_decode_cmd_i_valid = 0; + return slot; + }; + auto complete = [&](unsigned slot) { + dut.io_ball_complete_bits_rob_id = slot; + dut.io_ball_complete_valid = 1; + wait([&] { return dut.io_ball_complete_ready; }); + tick(); + dut.io_ball_complete_valid = 0; + }; + try { + const auto reader = send(true, false); + const auto writer1 = send(false, true); + const auto writer2 = send(false, true); + wait([&] { + return issued.count(reader) && issued.count(writer1) && + issued.count(writer2); + }); + check(issued[reader].read == 1, "Old reader lost its binding"); + const auto first = issued[writer1].write, second = issued[writer2].write; + check(first != 1 && second != first, + "Consecutive writes must use distinct aliases"); + complete(writer2); + complete(writer1); + for (unsigned i = 0; i < 12; ++i) + tick(); + check(reclaimed.empty(), + "Out-of-order completion freed an alias before retirement"); + dut.io_mapping_ready = 0; + complete(reader); + wait([&] { return dut.io_mapping_valid && dut.io_mapping_bits_retire; }); + for (unsigned i = 0; i < 10; ++i) { + check(dut.io_mapping_bits_previous == 1 && + dut.io_mapping_bits_vbank == first, + "Reclaim payload changed under backpressure"); + check(!(dut.io_retired & ((1u << writer1) | (1u << writer2))), + "ROB retired a writer before reclaim was accepted"); + tick(); + } + dut.io_mapping_ready = 1; + wait([&] { return reclaimed.size() == 2; }); + check(reclaimed[0].previous == 1 && reclaimed[0].current == first, + "First stale binding mismatch"); + check(reclaimed[1].previous == first && reclaimed[1].current == second, + "Second stale binding mismatch"); + const auto writer3 = send(true, true); + wait([&] { return issued.count(writer3); }); + check(issued[writer3].read == second, + "Read-modify-write did not read the previous binding"); + check(issued[writer3].write == first, + "Retired alias was not returned to the free list"); + complete(writer3); + wait([&] { return reclaimed.size() == 3 && dut.io_idle; }); + check(reclaimed[2].previous == second && reclaimed[2].current == first, + "Reused alias retirement mismatch"); + std::puts("PASS: out-of-order completion, reclaim backpressure, repeated " + "rename and alias reuse"); + } catch (const std::exception &error) { + std::fprintf(stderr, "FAIL at cycle %u: %s\n", cycle, error.what()); + return 1; + } + dut.final(); +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/Emit.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/Emit.scala new file mode 100644 index 000000000..aa1eab2aa --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/Emit.scala @@ -0,0 +1,35 @@ +package framework.acc.frontend.scheduler + +import sims.configs.SimParam +import _root_.configs.GlobalConfig +import java.nio.file.{Files, Paths} +import upickle.default.read + +object Emit { + + def emit( + b: GlobalConfig, + hartId: Int, + threadId: Int, + directory: String, + firtoolOpts: Array[String] + ): Unit = { + val core = + b.device.cores.find(_.hartId == hartId).getOrElse(throw new IllegalArgumentException(s"Unknown hart ID: $hartId")) + val thread = + core.threads.find(_.id == threadId).getOrElse(throw new IllegalArgumentException(s"Unknown thread ID: $threadId")) + + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Scheduler( + thread.acc.get.frontend, + thread.acc.get.memDomain, + thread.acc.get.gpDomain, + core.privileged.permission, + b.sim + ), + firtoolOpts = firtoolOpts, + args = Array("--target-dir", directory, "--split-verilog") + ) + } + +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/Scheduler.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/Scheduler.scala new file mode 100644 index 000000000..964a70066 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/Scheduler.scala @@ -0,0 +1,239 @@ +package framework.acc.frontend.scheduler + +import sims.configs.SimParam +import framework.lib.arbiter.{Arbiter} + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.core.memory.access.preflight.configs.PmpParams + +import framework.lib.queue.Queue + +import chisel3._ +import chisel3.util._ +import chisel3.experimental._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.frontend.scheduler.rob.global.GlobalROB +import framework.acc.frontend.decoder.{DecodedInstruction, DomainId, PostGDCmd} +import framework.acc.frontend.scheduler.bat.BankAliasTable +import framework.acc.frontend.scheduler.scoreboard.{BankAccessInfo, BankScoreboard} +import framework.acc.debug.trace.InstructionEvent +import framework.acc.debug.DebugPort +import framework.acc.memdomain.bank_router.BankMappingIO +import framework.acc.memdomain.{Response => MemoryResponse, MemoryContext} +import framework.acc.memdomain.toddr.DmaError + +/** Work dispatched to a domain, independent of the ROB's storage representation. */ +class Issue(frontendParam: FrontendParam, memParam: MemDomainParam) extends Bundle { + val commandDataBits = frontendParam.commandDataBits + val p = frontendParam + val cmd = new PostGDCmd(frontendParam, memParam) + val renamedBankAccess = new BankAccessInfo(frontendParam.bank_id_len) + val rob_id = UInt(log2Up(p.rob_entries).W) + val instId = UInt(64.W) // Observation identity; never controls scheduling. + val target = UInt(5.W) +} + +/** Memory dispatch alone carries the submitting address-space state. */ +class MemoryIssue(frontendParam: FrontendParam, memParam: MemDomainParam, permissionParam: PmpParams) extends Bundle { + val commandDataBits = frontendParam.commandDataBits + val operation = new Issue(frontendParam, memParam) + val context = new MemoryContext(permissionParam) +} + +/** Completion of an admitted operation; legality is an issue-time contract. */ +class Complete(frontendParam: FrontendParam) extends Bundle { + val rob_id = UInt(log2Up(frontendParam.rob_entries).W) +} + +@instantiable +class Scheduler( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val gpParam: GpDomainParam, + val permissionParam: PmpParams, + val simp: SimParam) + extends Module { + val diffTest = simp.diffTest + val commandDataBits = frontendParam.commandDataBits + val ownerId = frontendParam.ownerId + + @public + val io = IO(new Bundle { + + val admission = new Bundle { + val command = Flipped(Decoupled(new DecodedInstruction(frontendParam, memParam, permissionParam))) + val idle = Output(Bool()) + } + + val ballRequest = Decoupled(new MemoryIssue(frontendParam, memParam, permissionParam)) + val ballCompletion = Flipped(Decoupled(new MemoryResponse(frontendParam))) + val memoryRequest = Decoupled(new MemoryIssue(frontendParam, memParam, permissionParam)) + val memoryCompletion = Flipped(Decoupled(new MemoryResponse(frontendParam))) + val gpRequest = Option.when(gpParam.enable)(Decoupled(new Issue(frontendParam, memParam))) + val gpCompletion = + Option.when(gpParam.enable)(Flipped(Decoupled(new Complete(frontendParam)))) + val mappingUpdate = new BankMappingIO(frontendParam) + + val barrier = new Bundle { + val arrive = Output(Bool()) + val release = Input(Bool()) + } + + val fault = Output(Bool()) + val traceInstructions = Output(Vec(4, new InstructionEvent)) + val debug = Option.when(diffTest)(new DebugPort(frontendParam)) + }) + + val rob: Instance[GlobalROB] = + Instantiate(new GlobalROB(frontendParam, memParam, gpParam, permissionParam, simp.diffTest)) + + val resourceCount = memParam.virtualBankCount + (if (gpParam.enable) 2 else 0) + + val bat: Instance[BankAliasTable] = Instantiate( + new BankAliasTable(framework.acc.frontend.scheduler.bat.configs.Params( + frontendParam.bank_id_len, + resourceCount, + frontendParam.rob_entries + )) + ) + + val scoreboard: Instance[BankScoreboard] = Instantiate( + new BankScoreboard(framework.acc.frontend.scheduler.scoreboard.configs.Params( + frontendParam.bank_id_len, + 2 * resourceCount + frontendParam.rob_entries, + frontendParam.rob_entries + )) + ) + + scoreboard.io.alloc := rob.io.scoreboard.alloc + scoreboard.io.issue := rob.io.scoreboard.issue + scoreboard.io.complete := rob.io.scoreboard.complete + scoreboard.io.queryVec := rob.io.scoreboard.query + rob.io.scoreboard.hazard := scoreboard.io.hazardVec + bat.io.alloc := rob.io.bat.alloc + bat.io.reclaim := rob.io.bat.reclaim + rob.io.bat.stale := bat.io.stale + rob.io.bat.ready := bat.io.ready + rob.io.bat.renamed := bat.io.alloc_renamed + io.mappingUpdate <> rob.io.mapping + + io.traceInstructions := rob.io.traceInstructions + io.debug.zip(rob.io.debug).foreach { case (port, source) => + port <> source + port.idle := io.admission.idle && !io.admission.command.valid && !port.commit.valid + } + + val failed = RegInit(false.B) + val memoryFailed = io.memoryCompletion.bits.fault.error =/= DmaError.None.U + when( + (io.memoryCompletion.fire && memoryFailed) || (io.ballCompletion.fire && io.ballCompletion.bits.fault.error =/= 0.U) + )(failed := true.B) + io.fault := failed + + val isBarrierCmd = io.admission.command.valid && io.admission.command.bits.operation.isBarrier + val barrierWaitROB = RegInit(false.B) + val barrierWaitRelease = RegInit(false.B) + when(isBarrierCmd && !barrierWaitROB && !barrierWaitRelease) { + barrierWaitROB := true.B + } + when(barrierWaitROB && rob.io.empty) { + barrierWaitROB := false.B + barrierWaitRelease := true.B + } + when(barrierWaitRelease && io.barrier.release) { + barrierWaitRelease := false.B + } + io.barrier.arrive := barrierWaitRelease + + val isFrontendCmd = io.admission.command.bits.operation.isBarrier + // Extended control operations and metadata loads own the local scratch state until retirement. + val funct = io.admission.command.bits.operation.cmd.raw_inst(31, 25) + val exclusive = funct >= 80.U || funct === 35.U + val exclusiveActive = RegInit(false.B) + when(exclusiveActive && rob.io.empty)(exclusiveActive := false.B) + when(rob.io.alloc.fire && exclusive)(exclusiveActive := true.B) + val anyStall = barrierWaitROB || barrierWaitRelease || failed || exclusiveActive || (exclusive && !rob.io.empty) + rob.io.alloc.valid := io.admission.command.valid && !isFrontendCmd && !anyStall + rob.io.alloc.bits := io.admission.command.bits + io.admission.command.ready := Mux( + isFrontendCmd, + !anyStall, + rob.io.alloc.ready && !anyStall + ) + + val is_ball_domain = rob.io.issue.bits.cmd.domain_id === DomainId.BALL + val is_mem_domain = rob.io.issue.bits.cmd.domain_id === DomainId.MEM + val is_kernel_domain = rob.io.issue.bits.cmd.domain_id === DomainId.GP + + val mainIssueEntry = Wire(new Issue(frontendParam, memParam)) + mainIssueEntry.instId := rob.io.issue.bits.instId + mainIssueEntry.target := 0.U + mainIssueEntry.cmd := rob.io.issue.bits.cmd + mainIssueEntry.renamedBankAccess := rob.io.issue.bits.renamedBankAccess + mainIssueEntry.rob_id := rob.io.issue.bits.rob_id + mainIssueEntry.cmd.wr_read_bank := rob.io.issue.bits.renamedBankAccess.rd_wr_id + val rawOperands = rob.io.issue.bits.cmd.cmd.rs1Data + val access = rob.io.issue.bits.renamedBankAccess + when( + rob.io.issue.bits.cmd.cmd.raw_inst(31, 25) =/= framework.acc.memdomain.DISA.MSET_BITPAT + ) { + mainIssueEntry.cmd.cmd.rs1Data := Cat( + rawOperands(63, 30), + Mux(access.wr_bank_valid, access.wr_bank_id, rawOperands(29, 20)), + Mux(access.rd_bank_1_valid, access.rd_bank_1_id, rawOperands(19, 10)), + Mux(access.rd_bank_0_valid, access.rd_bank_0_id, rawOperands(9, 0)) + ) + } + + if (gpParam.enable) { + io.gpRequest.get.valid := rob.io.issue.valid && is_kernel_domain && !failed + io.gpRequest.get.bits := mainIssueEntry + } + val kernelReady = if (gpParam.enable) io.gpRequest.get.ready else false.B + + val completeArb = Instantiate(new Arbiter(new Complete(frontendParam), if (gpParam.enable) 3 else 2)) + + val completeQueue = Instantiate(new Queue( + UInt(log2Up(frontendParam.rob_entries).W), + frontendParam.rob_entries, + false, + false + )) + + rob.io.complete <> completeQueue.io.deq + completeArb.io.in(0).valid := io.ballCompletion.valid && io.ballCompletion.bits.fault.error === 0.U + completeArb.io.in(0).bits.rob_id := io.ballCompletion.bits.rob_id + io.ballCompletion.ready := completeArb.io.in(0).ready || io.ballCompletion.bits.fault.error =/= 0.U + completeArb.io.in(1).valid := io.memoryCompletion.valid && !memoryFailed + completeArb.io.in(1).bits.rob_id := io.memoryCompletion.bits.rob_id + io.memoryCompletion.ready := memoryFailed || completeArb.io.in(1).ready + + if (gpParam.enable) { completeArb.io.in(2) <> io.gpCompletion.get } + + val completeBits = completeArb.io.out.bits + + io.ballRequest.valid := rob.io.issue.valid && is_ball_domain && !failed + io.ballRequest.bits.operation := mainIssueEntry + io.ballRequest.bits.context := rob.io.issue.bits.context + io.memoryRequest.valid := rob.io.issue.valid && is_mem_domain && !failed + io.memoryRequest.bits.operation := mainIssueEntry + io.memoryRequest.bits.context := rob.io.issue.bits.context + rob.io.issue.ready := !failed && ((is_ball_domain && io.ballRequest.ready) || + (is_mem_domain && io.memoryRequest.ready) || (is_kernel_domain && kernelReady)) + + completeQueue.io.enq.valid := completeArb.io.out.valid + completeQueue.io.enq.bits := completeBits.rob_id + completeArb.io.out.ready := completeQueue.io.enq.ready + io.admission.idle := rob.io.empty && !barrierWaitROB && !barrierWaitRelease && !failed + + // Reset-time initialization is not a runtime command. Monitor runtime blocking locally. + val runtimeStarted = RegInit(false.B) + when(io.admission.command.fire && io.admission.command.bits.operation.runtime)(runtimeStarted := true.B) + val blocked = rob.io.full || barrierWaitROB || barrierWaitRelease + val blockedCycles = RegInit(0.U(32.W)) + blockedCycles := Mux(runtimeStarted && blocked, blockedCycles + 1.U, 0.U) + assert(blockedCycles < 10000000.U, "Accelerator: busy for too long!") + +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/build.sc b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/build.sc new file mode 100644 index 000000000..05a0fdf48 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/build.sc @@ -0,0 +1,21 @@ +import mill._ +import $file.^.^.^.^.^.^.^.^.^.^.^.^.^.^.settings +val ipRoot = millSourcePath + +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.acc.frontend.scheduler.bat.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/BankAliasTable.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/configs/uvm.toml b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/configs/uvm.toml new file mode 100644 index 000000000..968f67903 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/configs/uvm.toml @@ -0,0 +1,6 @@ +[parameters] +bankIdBits = 4 +virtualBanks = 4 +robEntries = 4 + +[defines] diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/agent/driver.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/agent/driver.svh new file mode 100644 index 000000000..d549e9643 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/agent/driver.svh @@ -0,0 +1,25 @@ +class driver extends uvm_driver #(cycle_item); + `uvm_component_utils(driver) + virtual bank_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual bank_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Bank scheduling interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item req; + forever begin + seq_item_port.get_next_item(req); + @(negedge vif.clock); + vif.reset = req.reset; + vif.inputs = req.inputs; + @(posedge vif.clock); + req.outputs = vif.outputs; + #1; + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/agent/item.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/agent/item.svh new file mode 100644 index 000000000..111706948 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/agent/item.svh @@ -0,0 +1,18 @@ +class cycle_item extends uvm_sequence_item; + `uvm_object_utils(cycle_item) + logic [`BAT_INPUT_BITS-1:0] inputs; + logic [`BAT_OUTPUT_BITS-1:0] outputs; + bit reset; + function new(string name = "cycle_item"); + super.new(name); + endfunction +endclass +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(payload, UVM_ALL_ON) + `uvm_object_utils_end + logic [`BAT_OUTPUT_BITS-1:0] payload; + function new(string name = "result_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/agent/monitor.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/agent/monitor.svh new file mode 100644 index 000000000..dc93b2fb6 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/agent/monitor.svh @@ -0,0 +1,25 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual bank_if vif; + uvm_analysis_port #(cycle_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual bank_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Bank scheduling monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item t; + forever begin + @(posedge vif.clock); + t = cycle_item::type_id::create("observed"); + t.inputs = vif.inputs; + t.outputs = vif.outputs; + t.reset = vif.reset; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/env/env.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/env/env.svh new file mode 100644 index 000000000..ed95874fb --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/env/env.svh @@ -0,0 +1,25 @@ +class env extends uvm_env; + `uvm_component_utils(env) + driver drv; + monitor mon; + model reference; + uvm_sequencer #(cycle_item) seqr; + result_board #(result_item, 0) scoreboard; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + drv = driver::type_id::create("driver", this); + mon = monitor::type_id::create("monitor", this); + reference = model::type_id::create("model", this); + seqr = new("seqr", this); + scoreboard = result_board#(result_item, 0)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + drv.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(reference.analysis_export); + reference.expected_ap.connect(scoreboard.expected_export); + reference.actual_ap.connect(scoreboard.actual_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/env/model.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/env/model.svh new file mode 100644 index 000000000..24b93f3f0 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/env/model.svh @@ -0,0 +1,99 @@ +class model extends uvm_subscriber #(cycle_item); + `uvm_component_utils(model) + localparam int V = `BAT_VIRTUALBANKS, A = 2 * V + `BAT_ROBENTRIES; + int current[V]; + bit [A-1:0] free_aliases; + uvm_analysis_port #(result_item) expected_ap, actual_ap; + covergroup behavior with function sample ( + bit rename, bit reset_read, bit update, bit ready, int stale + ); + option.per_instance = 1; + renaming: coverpoint rename; + reset_mapping: coverpoint reset_read; + update_alias: coverpoint update; + free_capacity: coverpoint ready; + stale_aliases: coverpoint stale {bins combinations[] = {[0 : 3]};} + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected_ap = new("expected_ap", this); + actual_ap = new("actual_ap", this); + behavior = new(); + endfunction + function void write(cycle_item t); + result_item expected, actual; + int fresh, read0, read1, read_wr, wr, renamed_write, stale0, stale1; + bit rename, reset_read, update, resets_write, stale0_valid, stale1_valid; + if (t.reset) begin + foreach (current[b]) current[b] = b; + free_aliases = '1; + for (int b = 0; b < V; b++) free_aliases[b] = 0; + return; + end + expected = result_item::type_id::create("expected"); + actual = result_item::type_id::create("actual"); + expected.payload = '0; + actual.payload = t.outputs; + fresh = 0; + for (int slot = 0; slot < A; slot++) + if (free_aliases[slot]) begin + fresh = slot; + break; + end + read0 = `BI(t.inputs, ALLOC_RAW_RD_BANK_0_ID); + read1 = `BI(t.inputs, ALLOC_RAW_RD_BANK_1_ID); + read_wr = `BI(t.inputs, ALLOC_RAW_RD_WR_ID); + wr = `BI(t.inputs, ALLOC_RAW_WR_BANK_ID); + rename = `BI(t.inputs, ALLOC_RENAME); + reset_read = `BI(t.inputs, ALLOC_RESET_READ0); + update = `BI(t.inputs, ALLOC_RAW_WR_UPDATE); + renamed_write = rename ? fresh : update ? current[wr] : wr; + // The fresh-alias payload has no meaning when allocation is blocked. + if (rename && free_aliases == 0) begin + actual.payload[`BAT_OUT_ALLOC_RENAMED_WR_BANK_ID_OFFSET +: `BAT_OUT_ALLOC_RENAMED_WR_BANK_ID_WIDTH] = 0; + end + resets_write = + `BI(t.inputs, ALLOC_RAW_WR_BANK_VALID) + && !rename && !update && current[wr] != wr; + stale0_valid = rename || (!reset_read && resets_write) || (reset_read && current[read0] != read0); + stale1_valid = reset_read && resets_write && read0 != wr; + stale0 = reset_read ? current[read0] : current[wr]; + stale1 = current[wr]; + expected.payload[`BAT_OUT_READY_OFFSET+:`BAT_OUT_READY_WIDTH] = !rename || free_aliases != 0; + expected.payload[`BAT_OUT_ALLOC_RENAMED_RD_BANK_0_VALID_OFFSET +: `BAT_OUT_ALLOC_RENAMED_RD_BANK_0_VALID_WIDTH] = `BI( + t.inputs, ALLOC_RAW_RD_BANK_0_VALID); + expected.payload[`BAT_OUT_ALLOC_RENAMED_RD_BANK_1_VALID_OFFSET +: `BAT_OUT_ALLOC_RENAMED_RD_BANK_1_VALID_WIDTH] = `BI( + t.inputs, ALLOC_RAW_RD_BANK_1_VALID); + expected.payload[`BAT_OUT_ALLOC_RENAMED_RD_WR_VALID_OFFSET +: `BAT_OUT_ALLOC_RENAMED_RD_WR_VALID_WIDTH] = `BI( + t.inputs, ALLOC_RAW_RD_WR_VALID); + expected.payload[`BAT_OUT_ALLOC_RENAMED_WR_BANK_VALID_OFFSET +: `BAT_OUT_ALLOC_RENAMED_WR_BANK_VALID_WIDTH] = `BI( + t.inputs, ALLOC_RAW_WR_BANK_VALID); + expected.payload[`BAT_OUT_ALLOC_RENAMED_WR_UPDATE_OFFSET +: `BAT_OUT_ALLOC_RENAMED_WR_UPDATE_WIDTH] = `BI( + t.inputs, ALLOC_RAW_WR_UPDATE); + expected.payload[`BAT_OUT_ALLOC_RENAMED_RD_BANK_0_ID_OFFSET +: `BAT_OUT_ALLOC_RENAMED_RD_BANK_0_ID_WIDTH] = + `BI(t.inputs, ALLOC_RAW_RD_BANK_0_VALID) + ? current[read0] : 0; + expected.payload[`BAT_OUT_ALLOC_RENAMED_RD_BANK_1_ID_OFFSET +: `BAT_OUT_ALLOC_RENAMED_RD_BANK_1_ID_WIDTH] = + `BI(t.inputs, ALLOC_RAW_RD_BANK_1_VALID) + ? current[read1] : 0; + expected.payload[`BAT_OUT_ALLOC_RENAMED_RD_WR_ID_OFFSET +: `BAT_OUT_ALLOC_RENAMED_RD_WR_ID_WIDTH] = + `BI(t.inputs, ALLOC_RAW_RD_WR_VALID) + ? current[read_wr] : 0; + expected.payload[`BAT_OUT_ALLOC_RENAMED_WR_BANK_ID_OFFSET +: `BAT_OUT_ALLOC_RENAMED_WR_BANK_ID_WIDTH] = renamed_write; + expected.payload[`BAT_OUT_STALE_0_VALID_OFFSET+:`BAT_OUT_STALE_0_VALID_WIDTH] = stale0_valid; + expected.payload[`BAT_OUT_STALE_0_BITS_OFFSET+:`BAT_OUT_STALE_0_BITS_WIDTH] = stale0; + expected.payload[`BAT_OUT_STALE_1_VALID_OFFSET+:`BAT_OUT_STALE_1_VALID_WIDTH] = stale1_valid; + expected.payload[`BAT_OUT_STALE_1_BITS_OFFSET+:`BAT_OUT_STALE_1_BITS_WIDTH] = stale1; + behavior.sample(rename, reset_read, update, !rename || free_aliases != 0, + stale0_valid | (int'(stale1_valid) << 1)); + expected_ap.write(expected); + actual_ap.write(actual); + if (`BI(t.inputs, RECLAIM_VALID) && `BI(t.inputs, RECLAIM_BITS) >= V) + free_aliases[`BI(t.inputs, RECLAIM_BITS)] = 1; + if (`BI(t.inputs, ALLOC_VALID)) begin + if (rename) free_aliases[fresh] = 0; + if (`BI(t.inputs, ALLOC_RAW_WR_BANK_VALID)) current[wr] = renamed_write; + if (reset_read) current[read0] = read0; + end + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/filelist.f b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/filelist.f new file mode 100644 index 000000000..3eea691bb --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/BankAliasTable/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/interface.sv b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/interface.sv new file mode 100644 index 000000000..14c732776 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/interface.sv @@ -0,0 +1,8 @@ +`include "parameters.svh" +interface bank_if ( + input logic clock +); + logic reset = 1; + logic [`BAT_INPUT_BITS-1:0] inputs; + logic [`BAT_OUTPUT_BITS-1:0] outputs; +endinterface diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/package.sv b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/package.sv new file mode 100644 index 000000000..c173de906 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/package.sv @@ -0,0 +1,15 @@ +`include "parameters.svh" +`define BI(P, N) P[`BAT_IN_``N``_OFFSET +: `BAT_IN_``N``_WIDTH] +`define BO(P, N) P[`BAT_OUT_``N``_OFFSET +: `BAT_OUT_``N``_WIDTH] +package bat_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..42e4dec66 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/seq/sequence.svh @@ -0,0 +1,152 @@ +class scenario_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(scenario_sequence) + localparam int V = `BAT_VIRTUALBANKS, A = 2 * V + `BAT_ROBENTRIES; + logic [`BAT_INPUT_BITS-1:0] pins = '0; + logic [`BAT_OUTPUT_BITS-1:0] observed; + bit reset = 1; + int stale[$]; + function new(string name = "scenario_sequence"); + super.new(name); + endfunction + task tick(int cycles = 1); + cycle_item req; + repeat (cycles) begin + req = cycle_item::type_id::create("cycle"); + start_item(req); + req.inputs = pins; + req.reset = reset; + finish_item(req); + observed = req.outputs; + if (!reset && `BI(pins, ALLOC_VALID)) begin + if (`BO(observed, STALE_0_VALID)) stale.push_back(`BO(observed, STALE_0_BITS)); + if (`BO(observed, STALE_1_VALID)) stale.push_back(`BO(observed, STALE_1_BITS)); + end + end + endtask + task restart(); + pins = '0; + reset = 1; + stale.delete(); + tick(2); + reset = 0; + tick(); + endtask + task allocate(int bank, bit rename, bit reset_read, bit update, int read0, read1, read_wr, + int mask = 7); + `BI(pins, ALLOC_VALID) = 1; + `BI(pins, ALLOC_RENAME) = rename; + `BI(pins, ALLOC_RESET_READ0) = reset_read; + `BI(pins, ALLOC_RAW_WR_UPDATE) = update; + `BI(pins, ALLOC_RAW_WR_BANK_VALID) = 1; + `BI(pins, ALLOC_RAW_WR_BANK_ID) = bank; + `BI(pins, ALLOC_RAW_RD_BANK_0_VALID) = mask[0]; + `BI(pins, ALLOC_RAW_RD_BANK_0_ID) = read0; + `BI(pins, ALLOC_RAW_RD_BANK_1_VALID) = mask[1]; + `BI(pins, ALLOC_RAW_RD_BANK_1_ID) = read1; + `BI(pins, ALLOC_RAW_RD_WR_VALID) = mask[2]; + `BI(pins, ALLOC_RAW_RD_WR_ID) = read_wr; + tick(); + `BI(pins, ALLOC_VALID) = 0; + endtask + task reclaim_all(); + `BI(pins, ALLOC_VALID) = 0; + `BI(pins, ALLOC_RENAME) = 0; + `BI(pins, ALLOC_RESET_READ0) = 0; + while (stale.size() != 0) begin + `BI(pins, RECLAIM_VALID) = 1; + `BI(pins, RECLAIM_BITS) = stale.pop_front(); + tick(); + end + `BI(pins, RECLAIM_VALID) = 0; + endtask + task body(); + restart(); + for (int n = 0; n < A - V; n++) allocate(n % V, 1, 0, 0, n % V, (n + 1) % V, (n + 2) % V); + `BI(pins, ALLOC_RENAME) = 1; + tick(); + evidence::check(!`BO(observed, READY), "BAT free list did not exhaust"); + reclaim_all(); + for (int n = 0; n < 4 * V; n++) begin + allocate(n % V, 1, 0, 0, (n + 1) % V, (n + 2) % V, n % V); + reclaim_all(); + end + restart(); + for (int bank = 0; bank < V; bank++) allocate(bank, 1, 0, 0, bank, bank, bank); + reclaim_all(); + for (int mask = 0; mask < 8; mask++) + for (int read0 = 0; read0 < V; read0++) + for (int read1 = 0; read1 < V; read1++) + for (int read_wr = 0; read_wr < V; read_wr++) + allocate(read0, 0, 0, 1, read0, read1, read_wr, mask); + restart(); + allocate(0, 1, 0, 0, 0, 0, 0); + allocate(0, 0, 1, 0, 1 % V, 0, 0); + reclaim_all(); + restart(); + allocate(0, 1, 0, 0, 0, 0, 0); + allocate(1 % V, 1, 0, 0, 1 % V, 0, 0); + allocate(0, 0, 1, 0, 1 % V, 0, 0); + reclaim_all(); + restart(); + allocate(0, 1, 0, 0, 0, 0, 0); + allocate(0, 0, 1, 1, 0, 0, 0); + reclaim_all(); + // A retired alias becomes available on the next cycle, alongside an independent allocation. + restart(); + for (int n = 0; n < A - V - 1; n++) allocate(n % V, 1, 0, 0, n % V, n % V, n % V); + while (stale.size() && stale[0] < V) begin + `BI(pins, RECLAIM_VALID) = 1; + `BI(pins, RECLAIM_BITS) = stale.pop_front(); + tick(); + end + `BI(pins, RECLAIM_VALID) = 1; + `BI(pins, RECLAIM_BITS) = stale.pop_front(); + allocate(0, 1, 0, 0, 0, 0, 0); + `BI(pins, RECLAIM_VALID) = 0; + reclaim_all(); + restart(); + endtask +endclass +class invalid_sequence extends scenario_sequence; + `uvm_object_utils(invalid_sequence) + function new(string name = "invalid_sequence"); + super.new(name); + endfunction + task body(); + string scenario; + if (!$value$plusargs("CASE=%s", scenario)) + `uvm_fatal("CASE", "BAT assertion scenario is required") + restart(); + if (scenario == "full") begin + for (int n = 0; n <= A - V; n++) allocate(0, 1, 0, 0, 0, 0, 0); + end else if (scenario == "reclaim") begin + `BI(pins, RECLAIM_VALID) = 1; + `BI(pins, RECLAIM_BITS) = A; + tick(); + end else begin + `BI(pins, ALLOC_VALID) = 1; + case (scenario) + "read0": begin + `BI(pins, ALLOC_RAW_RD_BANK_0_VALID) = 1; + `BI(pins, ALLOC_RAW_RD_BANK_0_ID) = V; + end + "read1": begin + `BI(pins, ALLOC_RAW_RD_BANK_1_VALID) = 1; + `BI(pins, ALLOC_RAW_RD_BANK_1_ID) = V; + end + "update": begin + `BI(pins, ALLOC_RAW_RD_WR_VALID) = 1; + `BI(pins, ALLOC_RAW_RD_WR_ID) = V; + end + "write": begin + `BI(pins, ALLOC_RAW_WR_BANK_VALID) = 1; + `BI(pins, ALLOC_RAW_WR_BANK_ID) = V; + end + default: `uvm_fatal("CASE", "Unknown BAT assertion scenario") + endcase + tick(); + end + tick(3); + `uvm_fatal("EXPECTED_ASSERTION", "Invalid BAT input did not trigger its assertion") + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/suite.toml b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/suite.toml new file mode 100644 index 000000000..9e6f5dcd1 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/suite.toml @@ -0,0 +1,122 @@ +[model] +manifest = "" +library = "" + +[[tests]] +name = "bat" +top = "bat_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 300000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +functional_groups = ["bat_pkg::model::behavior"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +name = "bad_full" +top = "bat_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 300000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +functional_groups = [] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=full"] +assertion = "BAT allocation has no free virtual bank" + +[[tests]] +name = "bad_reclaim" +top = "bat_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 300000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +functional_groups = [] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=reclaim"] +assertion = "BAT reclaim alias is outside the allocated namespace" + +[[tests]] +name = "bad_read0" +top = "bat_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 300000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +functional_groups = [] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=read0"] +assertion = "BAT read bank is outside the architectural namespace" + +[[tests]] +name = "bad_read1" +top = "bat_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 300000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +functional_groups = [] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=read1"] +assertion = "BAT second read bank is outside the architectural namespace" + +[[tests]] +name = "bad_update" +top = "bat_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 300000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +functional_groups = [] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=update"] +assertion = "BAT update source is outside the architectural namespace" + +[[tests]] +name = "bad_write" +top = "bat_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 300000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +functional_groups = [] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=write"] +assertion = "BAT write bank is outside the architectural namespace" diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/tb.sv b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/tb.sv new file mode 100644 index 000000000..13242a5d5 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/tb.sv @@ -0,0 +1,16 @@ +module bat_tb; + import uvm_pkg::*; + import bat_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + bank_if bus (clock); + BankAliasTable dut ( + .clock(clock), + .reset(bus.reset), + `include "ports.svh" + ); + initial begin + uvm_config_db#(virtual bank_if)::set(null, "uvm_test_top*", "vif", bus); + run_test(); + end +endmodule diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/tests/test.svh new file mode 100644 index 000000000..0230681bb --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/tests/test.svh @@ -0,0 +1,35 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.start(test_env.seqr); + endtask +endclass +class invalid_test extends base_test; + `uvm_component_utils(invalid_test) + driver drv; + uvm_sequencer #(cycle_item) seqr; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + drv = driver::type_id::create("driver", this); + seqr = new("seqr", this); + endfunction + function void connect_phase(uvm_phase phase); + drv.seq_item_port.connect(seqr.seq_item_export); + endfunction + task execute(); + invalid_sequence seq = invalid_sequence::type_id::create("scenario"); + seq.start(seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/scala/BankAliasTable.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/scala/BankAliasTable.scala new file mode 100644 index 000000000..3e2f9d741 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/scala/BankAliasTable.scala @@ -0,0 +1,106 @@ +package framework.acc.frontend.scheduler.bat + +import framework.acc.frontend.scheduler.scoreboard.BankAccessInfo +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.acc.frontend.scheduler.bat.configs.Params + +/** Current mappings and a free list; overwritten aliases return only at retirement. */ +@instantiable +class BankAliasTable(val p: Params) extends Module { + + val bankIdLen = p.bankIdBits + val vbankNum = p.virtualBanks + val robEntries = p.robEntries + val aliasCount = 2 * vbankNum + robEntries + + @public + val io = IO(new Bundle { + + val alloc = Input(new Bundle { + val valid = Bool() + val rename = Bool() + val reset_read0 = Bool() + val raw = new BankAccessInfo(p.bankIdBits) + }) + + val ready = Output(Bool()) + val alloc_renamed = Output(new BankAccessInfo(p.bankIdBits)) + val reclaim = Flipped(Valid(UInt(bankIdLen.W))) + val stale = Output(Vec(2, Valid(UInt(bankIdLen.W)))) + }) + + val current = RegInit(VecInit((0 until vbankNum).map(_.U(bankIdLen.W)))) + + val initialFree = ((BigInt(1) << aliasCount) - 1) ^ ((BigInt(1) << vbankNum) - 1) + val free = RegInit(initialFree.U(aliasCount.W)) + val available = free + + val fresh = PriorityEncoder(available) + val raw = io.alloc.raw + + // Architectural IDs are checked before allocation; the register array uses only its index bits. + def currentAlias(bank: UInt): UInt = + if (vbankNum == 1) current(0) else current(bank(log2Ceil(vbankNum) - 1, 0)) + + io.ready := !io.alloc.rename || available.orR + io.alloc_renamed := raw + io.alloc_renamed.rd_bank_0_id := Mux( + raw.rd_bank_0_valid, + currentAlias(raw.rd_bank_0_id), + 0.U + ) + io.alloc_renamed.rd_bank_1_id := Mux( + raw.rd_bank_1_valid, + currentAlias(raw.rd_bank_1_id), + 0.U + ) + io.alloc_renamed.rd_wr_id := Mux(raw.rd_wr_valid, currentAlias(raw.rd_wr_id), 0.U) + io.alloc_renamed.wr_bank_id := Mux( + io.alloc.rename, + fresh, + Mux(raw.wr_update, currentAlias(raw.wr_bank_id), raw.wr_bank_id) + ) + val resetBank = raw.rd_bank_0_id + val resetsWrite = raw.wr_bank_valid && !io.alloc.rename && !raw.wr_update && + currentAlias(raw.wr_bank_id) =/= raw.wr_bank_id + io.stale(0).valid := io.alloc.rename || (!io.alloc.reset_read0 && resetsWrite) || + (io.alloc.reset_read0 && currentAlias(resetBank) =/= resetBank) + io.stale(0).bits := Mux(io.alloc.reset_read0, currentAlias(resetBank), currentAlias(raw.wr_bank_id)) + io.stale(1).valid := io.alloc.reset_read0 && resetsWrite && resetBank =/= raw.wr_bank_id + io.stale(1).bits := currentAlias(raw.wr_bank_id) + + when(io.reclaim.valid)(assert(io.reclaim.bits < aliasCount.U, "BAT reclaim alias is outside the allocated namespace")) + + val returned = + Mux(io.reclaim.valid && io.reclaim.bits >= vbankNum.U, UIntToOH(io.reclaim.bits, aliasCount), 0.U(aliasCount.W)) + val allocated = Mux(io.alloc.valid && io.alloc.rename, UIntToOH(fresh, aliasCount), 0.U(aliasCount.W)) + free := (free | returned) & ~allocated + + when(io.alloc.valid) { + assert(io.ready, "BAT allocation has no free virtual bank") + when(raw.rd_bank_0_valid || io.alloc.reset_read0)(assert( + raw.rd_bank_0_id < vbankNum.U, + "BAT read bank is outside the architectural namespace" + )) + when(raw.rd_bank_1_valid)(assert( + raw.rd_bank_1_id < vbankNum.U, + "BAT second read bank is outside the architectural namespace" + )) + when(raw.rd_wr_valid)(assert(raw.rd_wr_id < vbankNum.U, "BAT update source is outside the architectural namespace")) + when(raw.wr_bank_valid || io.alloc.rename)(assert( + raw.wr_bank_id < vbankNum.U, + "BAT write bank is outside the architectural namespace" + )) + val access = io.alloc_renamed + when(raw.wr_bank_valid) { + if (vbankNum == 1) current(0) := access.wr_bank_id + else current(raw.wr_bank_id(log2Ceil(vbankNum) - 1, 0)) := access.wr_bank_id + } + when(io.alloc.reset_read0) { + if (vbankNum == 1) current(0) := raw.rd_bank_0_id + else current(raw.rd_bank_0_id(log2Ceil(vbankNum) - 1, 0)) := raw.rd_bank_0_id + } + } +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/scala/Emit.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/scala/Emit.scala new file mode 100644 index 000000000..d69019735 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/scala/Emit.scala @@ -0,0 +1,54 @@ +package framework.acc.frontend.scheduler.bat + +import chisel3._ +import java.nio.file.{Files, Paths} +import framework.acc.frontend.scheduler.bat.configs.Params + +object Emit extends App { + val Array("--config", config, "--output-dir", output) = args + val p = upickle.default.read[Params](Files.readString(Paths.get(config))) + Files.createDirectories(Paths.get(output)) + + def fields(data: Data, path: String): Seq[(String, Int)] = data match { + case record: Record => record.elements.toSeq.flatMap { case (name, field) => + fields(field, if (path.isEmpty) name else s"${path}_$name") + } + case vector: Vec[_] => vector.toSeq.zipWithIndex.flatMap { case (field, index) => fields(field, s"${path}_$index") } + case leaf => Seq(path -> leaf.getWidth) + } + + val inputRoots = Set("alloc", "reclaim") + val ports = scala.collection.mutable.ArrayBuffer[String]() + val definitions = scala.collection.mutable.ArrayBuffer[String]() + val offsets = scala.collection.mutable.Map[String, Int]() + var inputOffset = 0 + var outputOffset = 0 + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + { + val dut = new BankAliasTable(p) + for { + (root, bundle) <- dut.io.elements.toSeq + (name, width) <- fields(bundle, root) + } { + val input = inputRoots.contains(root) + val direction = if (input) "IN" else "OUT" + val signal = if (input) "inputs" else "outputs" + val offset = if (input) inputOffset else outputOffset + definitions += s"`define BAT_${direction}_${name.toUpperCase}_OFFSET $offset" + definitions += s"`define BAT_${direction}_${name.toUpperCase}_WIDTH $width" + ports += s".io_$name(bus.$signal[$offset +: $width])" + offsets(name) = offset + if (input) inputOffset += width else outputOffset += width + } + dut + }, + args = Array("--target-dir", s"$output/BankAliasTable", "--split-verilog") + ) + definitions += s"`define BAT_INPUT_BITS $inputOffset" + definitions += s"`define BAT_OUTPUT_BITS $outputOffset" + definitions += s"`define BAT_BANKIDBITS ${p.bankIdBits}" + definitions += s"`define BAT_VIRTUALBANKS ${p.virtualBanks}" + definitions += s"`define BAT_ROBENTRIES ${p.robEntries}" + Files.writeString(Paths.get(output, "parameters.svh"), definitions.mkString("\n") + "\n") + Files.writeString(Paths.get(output, "ports.svh"), ports.mkString(",\n") + "\n") +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..43a00fd63 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/scala/configs/Params.scala @@ -0,0 +1,12 @@ +package framework.acc.frontend.scheduler.bat.configs + +import upickle.default.{macroRW, ReadWriter} + +case class Params(bankIdBits: Int, virtualBanks: Int, robEntries: Int) { + require(bankIdBits > 0) + require(virtualBanks > 0) + require(robEntries > 1) + require(2 * virtualBanks + robEntries <= (1 << bankIdBits)) +} + +object Params { implicit val rw: ReadWriter[Params] = macroRW } diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/build.sc b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/build.sc new file mode 100644 index 000000000..0846b1c10 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/build.sc @@ -0,0 +1,23 @@ +import mill._ +import $file.^.^.^.^.^.^.^.^.^.^.^.^.^.^.^.settings +val ipRoot = millSourcePath + +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.acc.frontend.scheduler.rob.global.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/DependencyResolver.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/GlobalROB.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Rob.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/RobParams.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/configs/uvm.toml b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/configs/uvm.toml new file mode 100644 index 000000000..82c562bec --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/configs/uvm.toml @@ -0,0 +1,57 @@ +[parameters] +diffTest = false + +[parameters.frontend] +ownerId = 7 +commandDataBits = 64 +rob_entries = 4 +bank_alias_enable = true +rs_out_of_order_response = true +bank_id_len = 4 +max_private_vbank = 3 +shared_vbank_base = 16 +iter_len = 16 +sub_rob_enable = false +sub_rob_depth = 4 + +[parameters.memory] +sharedPorts = 0 +bankNum = 2 +bankLayoutEnable = true +bankWidth = 128 +bankEntries = 8 +bankMaskLen = 16 +virtualBankCount = 4 +sharedEnable = false +sharedEntries = 0 +sharedBankEntries = 0 +sharedBankNum = 0 +sharedInputChannels = 0 +sharedDefaultGroupCount = 0 +bankRequestPorts = 1 +dma_buswidth = 128 +memAddrLen = 44 +mmioEnable = false +mmioBankNum = 0 +mmioBankEntries = 0 +mmioBankWidth = 0 +mmioReadWidth = 0 + +[parameters.permission] +addressBits = 44 +entries = 16 + +[parameters.gp] +enable = false +laneNumber = 1 +vLen = 128 +eLen = 64 +iBufWords = 16 +memoryPorts = 1 + +[parameters.rob] +entries = 4 +dataBits = 64 +retireWidth = 4 + +[defines] diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/agent/driver.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/agent/driver.svh new file mode 100644 index 000000000..456ddc43c --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/agent/driver.svh @@ -0,0 +1,27 @@ +class driver extends uvm_driver #(command_item); + `uvm_component_utils(driver) + virtual global_if vif; + command_item pending; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual global_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Global ROB input interface is required") + endfunction + task run_phase(uvm_phase phase); + forever begin + @(negedge vif.clock); + if (pending == null) seq_item_port.try_next_item(pending); + vif.inputs[`GROB_IN_ALLOC_OFFSET+:`GROB_IN_ALLOC_BITS] = pending!=null ? pending.payload : '0; + `GI(vif.inputs, ALLOC_VALID) = pending != null; + @(posedge vif.clock); + if (!vif.reset && pending != null && `GO(vif.outputs, ALLOC_READY)) begin + #1; + seq_item_port.item_done(); + pending = null; + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/agent/item.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/agent/item.svh new file mode 100644 index 000000000..a6787bc57 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/agent/item.svh @@ -0,0 +1,52 @@ +typedef logic [`GROB_IN_ALLOC_BITS-1:0] command_t; +typedef logic [`GROB_IN_OPERATION_BITS-1:0] operation_t; +typedef logic [`GROB_IN_RENAMED_BITS-1:0] bank_access_t; +typedef logic [`GROB_ISSUE_BITS-1:0] issue_t; +class command_item extends uvm_sequence_item; + `uvm_object_utils(command_item) + command_t payload; + function new(string name = "command_item"); + super.new(name); + endfunction +endclass +class command_sequence extends uvm_sequence #(command_item); + `uvm_object_utils(command_sequence) + command_item item; + function new(string name = "command_sequence"); + super.new(name); + endfunction + task body(); + start_item(item); + finish_item(item); + endtask +endclass +class sample_item extends uvm_sequence_item; + `uvm_object_utils(sample_item) + logic [`GROB_INPUT_BITS-1:0] inputs; + logic [`GROB_OUTPUT_BITS-1:0] outputs; + bit reset; + function new(string name = "sample_item"); + super.new(name); + endfunction +endclass +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(payload, UVM_ALL_ON) + `uvm_object_utils_end + issue_t payload; + function new(string name = "result_item"); + super.new(name); + endfunction +endclass +class transaction; + operation_t operation; + issue_t expected; + bank_access_t renamed; + int slot, serial, raw_write, new_alias, stale_raw, funct; + longint unsigned pc, inst_id; + bit [1:0] stale_valid; + int stale[2]; + bit selected = 0, delivered = 0, completed = 0, allocated = 0, config_op = 0, movement = 0; + bit [`GROB_VBANKS-1:0] reads, writes; + int due; +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/agent/monitor.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/agent/monitor.svh new file mode 100644 index 000000000..06034f023 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/agent/monitor.svh @@ -0,0 +1,25 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual global_if vif; + uvm_analysis_port #(sample_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual global_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Global ROB monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + sample_item t; + forever begin + @(posedge vif.clock); + t = sample_item::type_id::create("observed"); + t.inputs = vif.inputs; + t.outputs = vif.outputs; + t.reset = vif.reset; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/agent/peer.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/agent/peer.svh new file mode 100644 index 000000000..d90bb1085 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/agent/peer.svh @@ -0,0 +1,61 @@ +class peer_driver extends uvm_component; + `uvm_component_utils(peer_driver) + virtual global_if vif; + model reference; + int query_offsets[`GROB_ENTRIES][9] = `GROB_QUERY_OFFSETS; + int hazard_offsets[`GROB_ENTRIES] = `GROB_HAZARD_OFFSETS; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual global_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Global ROB peer interface is required") + endfunction + task run_phase(uvm_phase phase); + bank_access_t renamed; + bit ready, rename, reset_read; + bit [1:0] stale_valid; + int stale0, stale1, chosen; + forever begin + @(negedge vif.clock); + #1; + reference.predict_bat(vif.inputs, renamed, ready, rename, reset_read, stale_valid, stale0, + stale1); + vif.inputs[`GROB_IN_BAT_READY_OFFSET+:`GROB_IN_BAT_READY_WIDTH] = ready; + vif.inputs[`GROB_IN_RENAMED_OFFSET+:`GROB_IN_RENAMED_BITS] = renamed; + vif.inputs[`GROB_IN_BAT_STALE_0_VALID_OFFSET+:`GROB_IN_BAT_STALE_0_VALID_WIDTH]=stale_valid[0]; + vif.inputs[`GROB_IN_BAT_STALE_0_BITS_OFFSET+:`GROB_IN_BAT_STALE_0_BITS_WIDTH] = stale0; + vif.inputs[`GROB_IN_BAT_STALE_1_VALID_OFFSET+:`GROB_IN_BAT_STALE_1_VALID_WIDTH]=stale_valid[1]; + vif.inputs[`GROB_IN_BAT_STALE_1_BITS_OFFSET+:`GROB_IN_BAT_STALE_1_BITS_WIDTH] = stale1; + vif.inputs[`GROB_IN_ISSUE_READY_OFFSET+:`GROB_IN_ISSUE_READY_WIDTH]=!vif.reset&&!reference.hold_issue&&reference.cycle%5!=0; + vif.inputs[`GROB_IN_MAPPING_READY_OFFSET+:`GROB_IN_MAPPING_READY_WIDTH]=!vif.reset&&!reference.hold_mapping&&reference.cycle%4!=0; + for (int lane = 0; lane < `GROB_ENTRIES; lane++) begin + bit hazard = 0; + for (int port = 0; port < 3; port++) begin + if (vif.outputs[query_offsets[lane][port*2]]) + hazard |= reference.busy[vif.outputs[query_offsets[lane][port*2+1]+:`GROB_BANK_BITS]]; + end + vif.inputs[hazard_offsets[lane]] = hazard; + end + if (vif.reset) begin + vif.inputs[`GROB_IN_COMPLETE_VALID_OFFSET+:`GROB_IN_COMPLETE_VALID_WIDTH] = 0; + vif.inputs[`GROB_IN_COMPLETE_BITS_OFFSET+:`GROB_IN_COMPLETE_BITS_WIDTH] = 0; + end else begin + if (reference.active_completion == null && !reference.hold_complete) begin + chosen = -1; + foreach (reference.completion_queue[i]) begin + if(reference.completion_queue[i].due<=reference.cycle&&(chosen<0||reference.reverse_complete)) + chosen = i; + end + if (chosen >= 0) begin + reference.active_completion = reference.completion_queue[chosen]; + reference.completion_queue.delete(chosen); + end + end + vif.inputs[`GROB_IN_COMPLETE_VALID_OFFSET+:`GROB_IN_COMPLETE_VALID_WIDTH]=reference.active_completion!=null; + vif.inputs[`GROB_IN_COMPLETE_BITS_OFFSET+:`GROB_IN_COMPLETE_BITS_WIDTH]=reference.active_completion!=null?reference.active_completion.slot:0; + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/env/env.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/env/env.svh new file mode 100644 index 000000000..387df80ca --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/env/env.svh @@ -0,0 +1,36 @@ +class env extends uvm_env; + `uvm_component_utils(env) + driver source; + monitor mon; + peer_driver peer; + model reference; + reset_controller reset; + uvm_sequencer #(command_item) seqr; + result_board #(result_item, 0) scoreboard; + uvm_analysis_imp #(bit, env) reset_export; + function new(string name, uvm_component parent); + super.new(name, parent); + reset_export = new("reset_export", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + source = driver::type_id::create("source", this); + mon = monitor::type_id::create("monitor", this); + peer = peer_driver::type_id::create("peer", this); + reference = model::type_id::create("model", this); + reset = reset_controller::type_id::create("reset", this); + seqr = new("seqr", this); + scoreboard = result_board#(result_item, 0)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + source.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(reference.analysis_export); + peer.reference = reference; + reference.expected_ap.connect(scoreboard.expected_export); + reference.actual_ap.connect(scoreboard.actual_export); + reference.reset_ap.connect(reset_export); + endfunction + function void write(bit value); + scoreboard.cancel_pending(); + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/env/model.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/env/model.svh new file mode 100644 index 000000000..588a8be0d --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/env/model.svh @@ -0,0 +1,475 @@ +class model extends uvm_subscriber #(sample_item); + `uvm_component_utils(model) + localparam int N = `GROB_ENTRIES, V = `GROB_VBANKS, A = `GROB_ALIASES; + transaction slots[N], order[$], completion_queue[$], active_completion; + int current[V], tail = 0, cycle = 0, serial = 0; + longint unsigned next_inst = 1; + bit [A-1:0] free_aliases, busy; + bit hold_issue = 0, hold_mapping = 0, hold_complete = 0, reverse_complete = 0; + int allocated = 0, selected = 0, delivered = 0, completed = 0, retired = 0, reordered = 0; + sample_item latest; + int cancelled_live = 0; + int trace_offsets[4][10] = `GROB_TRACE_OFFSETS; + int query_offsets[N][9] = `GROB_QUERY_OFFSETS; + int ba_offsets[9] = `GROB_BA_OFFSETS; + uvm_analysis_port #(result_item) expected_ap, actual_ap; + uvm_analysis_port #(bit) reset_ap; + covergroup behavior with function sample ( + int count, bit stalled, bit mapping_stall, bit reordered, bit config_op, bit movement + ); + option.per_instance = 1; + occupancy: coverpoint count {bins empty = {0}; bins partial = {[1 : N - 1]}; bins full = {N};} + issue_backpressure: coverpoint stalled; + mapping_backpressure: coverpoint mapping_stall; + out_of_order_completion: coverpoint reordered; + configuration: coverpoint config_op; + movement_command: coverpoint movement; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected_ap = new("expected_ap", this); + actual_ap = new("actual_ap", this); + reset_ap = new("reset_ap", this); + behavior = new(); + endfunction + function void predict_bat(logic [`GROB_INPUT_BITS-1:0] pins, output bank_access_t renamed, + output bit ready, rename, reset_read, output bit [1:0] stale_valid, + output int stale0, stale1); + int fresh, wr, r0, r1, rw, funct; + bit update, resets_write; + renamed = '0; + fresh = 0; + for (int index = 0; index < A; index++) + if (free_aliases[index]) begin + fresh = index; + break; + end + wr = `GI(pins, ALLOC_BITS_OPERATION_BANKACCESS_WR_BANK_ID); + r0 = `GI(pins, ALLOC_BITS_OPERATION_BANKACCESS_RD_BANK_0_ID); + r1 = `GI(pins, ALLOC_BITS_OPERATION_BANKACCESS_RD_BANK_1_ID); + rw = `GI(pins, ALLOC_BITS_OPERATION_BANKACCESS_RD_WR_ID); + funct = `GI(pins, ALLOC_BITS_OPERATION_CMD_RAW_INST) >> 25; + update = `GI(pins, ALLOC_BITS_OPERATION_BANKACCESS_WR_UPDATE); + rename = `GROB_RENAMING && + `GI(pins, ALLOC_BITS_OPERATION_BANKACCESS_WR_BANK_VALID) + && wr < V && funct != 32 && !update; + reset_read = funct == 32 && (`GI(pins, ALLOC_BITS_OPERATION_CMD_RS2DATA) & 64'h1000) != 0; + ready = !rename || free_aliases != 0; + `BA(renamed, RD_BANK_0_VALID) = `GI(pins, ALLOC_BITS_OPERATION_BANKACCESS_RD_BANK_0_VALID); + `BA(renamed, RD_BANK_1_VALID) = `GI(pins, ALLOC_BITS_OPERATION_BANKACCESS_RD_BANK_1_VALID); + `BA(renamed, RD_WR_VALID) = `GI(pins, ALLOC_BITS_OPERATION_BANKACCESS_RD_WR_VALID); + `BA(renamed, WR_BANK_VALID) = `GI(pins, ALLOC_BITS_OPERATION_BANKACCESS_WR_BANK_VALID); + `BA(renamed, WR_UPDATE) = update; + `BA(renamed, RD_BANK_0_ID) = `BA(renamed, RD_BANK_0_VALID) ? current[r0] : 0; + `BA(renamed, RD_BANK_1_ID) = `BA(renamed, RD_BANK_1_VALID) ? current[r1] : 0; + `BA(renamed, RD_WR_ID) = `BA(renamed, RD_WR_VALID) ? current[rw] : 0; + `BA(renamed, WR_BANK_ID) = rename ? fresh : update ? current[wr] : wr; + resets_write = `BA(renamed, WR_BANK_VALID) && !rename && !update && current[wr] != wr; + stale_valid[0] = rename || (!reset_read && resets_write) || (reset_read && current[r0] != r0); + stale_valid[1] = reset_read && resets_write && r0 != wr; + stale0 = reset_read ? current[r0] : current[wr]; + stale1 = current[wr]; + endfunction + function bit hazard(bank_access_t access); + return ( + `BA(access, RD_BANK_0_VALID) + && busy[ + `BA(access, RD_BANK_0_ID) + ]) || ( + `BA(access, RD_BANK_1_VALID) + && busy[ + `BA(access, RD_BANK_1_ID) + ]) || ( + `BA(access, RD_WR_VALID) + && busy[ + `BA(access, RD_WR_ID) + ]); + endfunction + function bit eligible(transaction candidate); + bit conflict, overlap; + if (hazard(candidate.renamed)) return 0; + foreach (order[i]) begin + transaction older = order[i]; + if (older == candidate) break; + overlap=((candidate.reads&older.writes)|(candidate.writes&(older.reads|older.writes)))!=0; + conflict = (!`GROB_RENAMING || !`GROB_LAYOUT) ? overlap : 0; + conflict |= ( + `OP(candidate.operation, BANKACCESS_WR_UPDATE) + && overlap) || ( + `OP(older.operation, BANKACCESS_WR_UPDATE) + && (candidate.reads & older.writes) != 0); + if (!older.completed && conflict) return 0; + if((candidate.config_op&&older.config_op)||((candidate.config_op||older.config_op)&& + ((candidate.reads|candidate.writes)&(older.reads|older.writes))!=0)) + return 0; + if (candidate.movement && !older.completed) return 0; + end + return 1; + endfunction + function void verify_trace(logic [`GROB_OUTPUT_BITS-1:0] outputs, int index, transaction r, + int kind); + evidence::check( + outputs[trace_offsets[index][0]+:8]==kind&& + outputs[trace_offsets[index][1]+:64]==`GROB_OWNER&&outputs[trace_offsets[index][2]+:32]==r.slot&& + outputs[trace_offsets[index][3]+:64]==r.inst_id&&outputs[trace_offsets[index][4]+:32]== + `OP(r.operation, DOMAIN_ID) + && outputs[trace_offsets[index][5]+:32] == r.funct && + outputs[trace_offsets[index][6]+:64] == r.pc && outputs[trace_offsets[index][7]+:64] == + `OP(r.operation, CMD_RS1DATA) && outputs[trace_offsets[index][8]+:64] == + `OP(r.operation, CMD_RS2DATA), + "Global ROB public instruction observation changed its fields"); + endfunction + function void write(sample_item t); + transaction candidate, map_retire, r, ignored, oldest_incomplete; + result_item expected, actual; + bank_access_t renamed, query_access; + bit bat_ready, rename, reset_read, input_fire, select_fire, issue_fire, complete_fire, map_fire; + bit [1:0] stale_valid; + bit movement_live, has_retirement, needs_mapping, reads_previous, completion_reordered; + int stale0, stale1, head, map_index, retire_count, cid; + latest = t; + if (t.reset) begin + reset_ap.write(1); + cancelled_live += order.size(); + order.delete(); + completion_queue.delete(); + active_completion = null; + tail = 0; + next_inst = 1; + busy = 0; + free_aliases = '1; + foreach (slots[i]) slots[i] = null; + foreach (current[i]) begin + current[i] = i; + free_aliases[i] = 0; + end + return; + end + cycle++; + completion_reordered = 0; + predict_bat(t.inputs, renamed, bat_ready, rename, reset_read, stale_valid, stale0, stale1); + head = order.size() ? order[0].slot : tail; + input_fire = `GI(t.inputs, ALLOC_VALID) && `GO(t.outputs, ALLOC_READY); + select_fire = `GO(t.outputs, SCOREBOARD_ISSUE_VALID); + issue_fire = `GO(t.outputs, ISSUE_VALID) && `GI(t.inputs, ISSUE_READY); + complete_fire = `GI(t.inputs, COMPLETE_VALID) && `GO(t.outputs, COMPLETE_READY); + map_fire = `GO(t.outputs, MAPPING_VALID) && `GI(t.inputs, MAPPING_READY); + candidate = null; + map_retire = null; + map_index = 0; + retire_count = 0; + movement_live = 0; + foreach (order[i]) begin + if (candidate == null && !order[i].selected && !order[i].completed) candidate = order[i]; + if (order[i].movement && !order[i].completed) movement_live = 1; + end + // Retirement is a completed FIFO prefix. A final mapping acknowledgement releases its entry. + foreach (order[i]) begin + if (!order[i].completed) break; + if (order[i].stale_valid != 0) begin + map_retire = order[i]; + map_index = order[i].stale_valid[0] ? 0 : 1; + break; + end + end + if (map_fire && `GO(t.outputs, MAPPING_BITS_RETIRE)) begin + evidence::check(map_retire != null, "Global ROB retired an unsolicited mapping"); + evidence::check(`GO(t.outputs, MAPPING_BITS_BANK) + == (map_index == 0 ? map_retire.stale_raw : map_retire.raw_write) && + `GO(t.outputs, MAPPING_BITS_VBANK) + == (map_index == 0 ? map_retire.new_alias : map_retire.raw_write) && + `GO(t.outputs, MAPPING_BITS_PREVIOUS) == map_retire.stale[map_index] && + `GO(t.outputs, MAPPING_BITS_THREADID) == `GROB_OWNER && ! + `GO(t.outputs, MAPPING_BITS_ALLOCATE) && !`GO(t.outputs, MAPPING_BITS_REUSE) + && !`GO(t.outputs, MAPPING_BITS_WRITE), + "Global ROB mapping retirement metadata mismatch"); + end + foreach (order[i]) begin + if (!order[i].completed) break; + if (order[i].stale_valid != 0 && !(order[i] == map_retire && map_fire && + `GO(t.outputs, MAPPING_BITS_RETIRE) + && $countones( + order[i].stale_valid + ) == 1)) + break; + retire_count++; + end + has_retirement = retire_count != 0; + evidence::check(`GO(t.outputs, EMPTY) == (order.size() == 0) && `GO(t.outputs, FULL) + == (order.size() == N), "Global ROB occupancy flags mismatch"); + evidence::check(`GO(t.outputs, ALLOC_READY) + == (order.size() < N && !has_retirement && bat_ready && !movement_live), + "Global ROB admission rule mismatch"); + evidence::check(`GO(t.outputs, BAT_ALLOC_VALID) == input_fire, + "Global ROB BAT allocation handshake mismatch"); + evidence::check(`GO(t.outputs, TRACEINSTRUCTIONS_0_ENABLE) == input_fire && + `GO(t.outputs, TRACEINSTRUCTIONS_2_ENABLE) == complete_fire, + "Global ROB public observation handshakes mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_ALLOC_VALID) == (input_fire && rename), + "Global ROB scoreboard allocation handshake mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_COMPLETE_VALID) == complete_fire, + "Global ROB completion handshake mismatch"); + evidence::check(`GO(t.outputs, TRACEINSTRUCTIONS_1_ENABLE) == select_fire, + "Global ROB issue observation mismatch"); + for (int age = 0; age < N; age++) begin + int slot = (head + age) % N; + if (slots[slot] != null) + for (int field = 0; field < 9; field++) begin + if (field % 2 == 0) + evidence::check( + t.outputs[query_offsets[age][field]] === slots[slot].renamed[ba_offsets[field]], + "Global ROB query flag lost its entry"); + else + evidence::check( + t.outputs[query_offsets[age][field]+:`GROB_BANK_BITS]===slots[slot].renamed[ba_offsets[field]+:`GROB_BANK_BITS], + "Global ROB query ID lost its alias"); + end + end + if (map_fire && !`GO(t.outputs, MAPPING_BITS_RETIRE)) begin + evidence::check(candidate != null && eligible(candidate) && select_fire, + "Global ROB mapping allocation bypassed a dependency"); + needs_mapping = candidate.allocated || ( + `BA(candidate.renamed, WR_BANK_VALID) + && ! + `BA(candidate.renamed, WR_UPDATE) + && candidate.raw_write < V && !candidate.config_op); + reads_previous = ( + `BA(candidate.renamed, RD_BANK_0_VALID) + && + `BA(candidate.renamed, RD_BANK_0_ID) + == candidate.stale[0]) || ( + `BA(candidate.renamed, RD_BANK_1_VALID) + && + `BA(candidate.renamed, RD_BANK_1_ID) + == candidate.stale[0]) || ( + `BA(candidate.renamed, RD_WR_VALID) + && + `BA(candidate.renamed, RD_WR_ID) + == candidate.stale[0]); + evidence::check(needs_mapping && `GO(t.outputs, MAPPING_BITS_BANK) == candidate.raw_write && + `GO(t.outputs, MAPPING_BITS_VBANK) == `BA(candidate.renamed, WR_BANK_ID) && + `GO(t.outputs, MAPPING_BITS_PREVIOUS) == candidate.stale[0] && + `GO(t.outputs, MAPPING_BITS_THREADID) == `GROB_OWNER && + `GO(t.outputs, MAPPING_BITS_ALLOCATE) == candidate.allocated && + `GO(t.outputs, MAPPING_BITS_REUSE) + == (candidate.allocated && candidate.slot == head && !reads_previous) && + `GO(t.outputs, MAPPING_BITS_WRITE) == (`BA(candidate.renamed, WR_BANK_VALID) + && !`BA(candidate.renamed, WR_UPDATE) + && candidate.raw_write < V && !candidate.config_op), + "Global ROB mapping issue metadata mismatch"); + end + if (select_fire) begin + evidence::check(candidate != null && eligible(candidate), + "Global ROB selected a blocked or younger instruction"); + needs_mapping = candidate.allocated || ( + `BA(candidate.renamed, WR_BANK_VALID) + && ! + `BA(candidate.renamed, WR_UPDATE) + && candidate.raw_write < V && !candidate.config_op); + if (needs_mapping) + evidence::check(map_fire && !`GO(t.outputs, MAPPING_BITS_RETIRE), + "Global ROB selected work before mapping acceptance"); + + evidence::check(`GO(t.outputs, TRACEINSTRUCTIONS_1_ROB_ID) == candidate.slot && + `GO(t.outputs, TRACEINSTRUCTIONS_1_PC) == candidate.pc && + `GO(t.outputs, TRACEINSTRUCTIONS_1_INST_ID) == candidate.inst_id && + `GO(t.outputs, TRACEINSTRUCTIONS_1_THREAD_ID) == `GROB_OWNER, + "Global ROB issue identity mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_ISSUE_BITS_RD_BANK_0_VALID) === + `BA(candidate.renamed, RD_BANK_0_VALID), + "Global ROB scoreboard issue field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_ISSUE_BITS_RD_BANK_0_ID) === + `BA(candidate.renamed, RD_BANK_0_ID), + "Global ROB scoreboard issue field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_ISSUE_BITS_RD_BANK_1_VALID) === + `BA(candidate.renamed, RD_BANK_1_VALID), + "Global ROB scoreboard issue field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_ISSUE_BITS_RD_BANK_1_ID) === + `BA(candidate.renamed, RD_BANK_1_ID), + "Global ROB scoreboard issue field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_ISSUE_BITS_RD_WR_VALID) === + `BA(candidate.renamed, RD_WR_VALID), + "Global ROB scoreboard issue field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_ISSUE_BITS_RD_WR_ID) === + `BA(candidate.renamed, RD_WR_ID), + "Global ROB scoreboard issue field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_ISSUE_BITS_WR_BANK_VALID) === + `BA(candidate.renamed, WR_BANK_VALID), + "Global ROB scoreboard issue field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_ISSUE_BITS_WR_BANK_ID) === + `BA(candidate.renamed, WR_BANK_ID), + "Global ROB scoreboard issue field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_ISSUE_BITS_WR_UPDATE) === + `BA(candidate.renamed, WR_UPDATE), + "Global ROB scoreboard issue field mismatch"); + verify_trace(t.outputs, 1, candidate, 1); + candidate.selected = 1; + selected++; + end + if (issue_fire) begin + int slot = `GO(t.outputs, ISSUE_BITS_ROB_ID); + r = slots[slot]; + evidence::check(r != null && r.selected && !r.delivered, + "Global ROB delivered an unselected or duplicate instruction"); + actual = result_item::type_id::create("issued"); + actual.payload = t.outputs[`GROB_ISSUE_OFFSET+:`GROB_ISSUE_BITS]; + actual_ap.write(actual); + r.delivered = 1; + r.due = cycle + (r.serial % 4 == 0 ? 17 : 3); + completion_queue.push_back(r); + delivered++; + end + if (complete_fire) begin + cid = `GI(t.inputs, COMPLETE_BITS); + r = slots[cid]; + evidence::check(r != null && r.delivered && !r.completed && active_completion == r, + "Global ROB accepted an invalid completion"); + oldest_incomplete = null; + foreach (order[i]) + if (oldest_incomplete == null && !order[i].completed) oldest_incomplete = order[i]; + if (r != oldest_incomplete) begin + reordered++; + completion_reordered = 1; + end + evidence::check(`GO(t.outputs, TRACEINSTRUCTIONS_2_ROB_ID) == r.slot && + `GO(t.outputs, TRACEINSTRUCTIONS_2_PC) == r.pc && + `GO(t.outputs, TRACEINSTRUCTIONS_2_INST_ID) == r.inst_id, + "Global ROB completion identity mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_COMPLETE_BITS_RD_BANK_0_VALID) === + `BA(r.renamed, RD_BANK_0_VALID), + "Global ROB scoreboard complete field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_COMPLETE_BITS_RD_BANK_0_ID) === + `BA(r.renamed, RD_BANK_0_ID), + "Global ROB scoreboard complete field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_COMPLETE_BITS_RD_BANK_1_VALID) === + `BA(r.renamed, RD_BANK_1_VALID), + "Global ROB scoreboard complete field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_COMPLETE_BITS_RD_BANK_1_ID) === + `BA(r.renamed, RD_BANK_1_ID), + "Global ROB scoreboard complete field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_COMPLETE_BITS_RD_WR_VALID) === + `BA(r.renamed, RD_WR_VALID), "Global ROB scoreboard complete field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_COMPLETE_BITS_RD_WR_ID) === + `BA(r.renamed, RD_WR_ID), "Global ROB scoreboard complete field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_COMPLETE_BITS_WR_BANK_VALID) === + `BA(r.renamed, WR_BANK_VALID), + "Global ROB scoreboard complete field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_COMPLETE_BITS_WR_BANK_ID) === + `BA(r.renamed, WR_BANK_ID), "Global ROB scoreboard complete field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_COMPLETE_BITS_WR_UPDATE) === + `BA(r.renamed, WR_UPDATE), "Global ROB scoreboard complete field mismatch"); + verify_trace(t.outputs, 2, r, 0); + r.completed = 1; + completed++; + active_completion = null; + end + if (input_fire) begin + evidence::check(t.inputs[`GROB_IN_RENAMED_OFFSET+:`GROB_IN_RENAMED_BITS] === renamed, + "Global ROB peer alias changed before allocation"); + evidence::check(`GO(t.outputs, BAT_ALLOC_RENAME) == rename && + `GO(t.outputs, BAT_ALLOC_RESET_READ0) == reset_read, + "Global ROB requested the wrong alias operation"); + r = new; + r.operation = t.inputs[`GROB_IN_OPERATION_OFFSET+:`GROB_IN_OPERATION_BITS]; + r.renamed = renamed; + r.slot = tail; + r.serial = serial++; + r.pc = `OP(r.operation, CMD_PC); + r.inst_id = `OP(r.operation, RUNTIME) ? next_inst++ : 0; + r.funct = `OP(r.operation, CMD_RAW_INST) >> 25; + r.config_op = r.funct == 32; + r.movement = r.funct == 13; + r.raw_write = `OP(r.operation, BANKACCESS_WR_BANK_ID); + r.new_alias = reset_read ? + `OP(r.operation, BANKACCESS_RD_BANK_0_ID) + : + `BA(renamed, WR_BANK_ID); + r.stale_raw = reset_read ? `OP(r.operation, BANKACCESS_RD_BANK_0_ID) : r.raw_write; + r.allocated = rename; + r.stale_valid = stale_valid; + r.stale[0] = stale0; + r.stale[1] = stale1; + r.reads = 0; + r.writes = 0; + if (`OP(r.operation, BANKACCESS_RD_BANK_0_VALID)) + r.reads[`OP(r.operation, BANKACCESS_RD_BANK_0_ID)] = 1; + if (`OP(r.operation, BANKACCESS_RD_BANK_1_VALID)) + r.reads[`OP(r.operation, BANKACCESS_RD_BANK_1_ID)] = 1; + if (`OP(r.operation, BANKACCESS_RD_WR_VALID)) + r.reads[`OP(r.operation, BANKACCESS_RD_WR_ID)] = 1; + if (`OP(r.operation, BANKACCESS_WR_BANK_VALID)) r.writes[r.raw_write] = 1; + r.expected = '0; + r.expected[`GROB_OUT_COMMAND_OFFSET-`GROB_ISSUE_OFFSET+:`GROB_OUT_COMMAND_BITS] = r.operation; + r.expected[`GROB_OUT_CONTEXT_OFFSET-`GROB_ISSUE_OFFSET+:`GROB_OUT_CONTEXT_BITS]=t.inputs[`GROB_IN_CONTEXT_OFFSET+:`GROB_IN_CONTEXT_BITS]; + r.expected[`GROB_OUT_RENAMED_OFFSET-`GROB_ISSUE_OFFSET+:`GROB_OUT_RENAMED_BITS] = renamed; + `ISS(r.expected, ROB_ID) = tail; + `ISS(r.expected, INSTID) = r.inst_id; + expected = result_item::type_id::create("allocated"); + expected.payload = r.expected; + expected_ap.write(expected); + verify_trace(t.outputs, 0, r, 2); + slots[tail] = r; + order.push_back(r); + tail = (tail + 1) % N; + allocated++; + if (rename) begin + evidence::check(`GO(t.outputs, SCOREBOARD_ALLOC_BITS_RD_BANK_0_VALID) === + `BA(renamed, RD_BANK_0_VALID), + "Global ROB scoreboard alloc field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_ALLOC_BITS_RD_BANK_0_ID) === + `BA(renamed, RD_BANK_0_ID), "Global ROB scoreboard alloc field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_ALLOC_BITS_RD_BANK_1_VALID) === + `BA(renamed, RD_BANK_1_VALID), + "Global ROB scoreboard alloc field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_ALLOC_BITS_RD_BANK_1_ID) === + `BA(renamed, RD_BANK_1_ID), "Global ROB scoreboard alloc field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_ALLOC_BITS_RD_WR_VALID) === + `BA(renamed, RD_WR_VALID), "Global ROB scoreboard alloc field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_ALLOC_BITS_RD_WR_ID) === `BA(renamed, RD_WR_ID), + "Global ROB scoreboard alloc field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_ALLOC_BITS_WR_BANK_VALID) === + `BA(renamed, WR_BANK_VALID), "Global ROB scoreboard alloc field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_ALLOC_BITS_WR_BANK_ID) === + `BA(renamed, WR_BANK_ID), "Global ROB scoreboard alloc field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_ALLOC_BITS_WR_UPDATE) === `BA(renamed, WR_UPDATE), + "Global ROB scoreboard alloc field mismatch"); + end + if (rename) free_aliases[`BA(renamed, WR_BANK_ID)] = 0; + if (`BA(renamed, WR_BANK_VALID)) current[r.raw_write] = `BA(renamed, WR_BANK_ID); + if (reset_read) + current[ + `OP(r.operation, BANKACCESS_RD_BANK_0_ID) + ] = + `OP(r.operation, BANKACCESS_RD_BANK_0_ID); + end + if (select_fire && `BA(candidate.renamed, WR_BANK_VALID)) + busy[`BA(candidate.renamed, WR_BANK_ID)] = 1; + if (input_fire && rename) busy[`BA(renamed, WR_BANK_ID)] = 1; + if (complete_fire) begin + r = slots[cid]; + if (`BA(r.renamed, WR_BANK_VALID)) busy[`BA(r.renamed, WR_BANK_ID)] = 0; + end + if (map_fire && `GO(t.outputs, MAPPING_BITS_RETIRE)) begin + map_retire.stale_valid[map_index] = 0; + if (map_retire.stale[map_index] >= V) free_aliases[map_retire.stale[map_index]] = 1; + end + behavior.sample(order.size(), `GO(t.outputs, ISSUE_VALID) && !`GI(t.inputs, ISSUE_READY), + `GO(t.outputs, MAPPING_VALID) && !`GI(t.inputs, MAPPING_READY), + completion_reordered, input_fire && ( + `GI(t.inputs, ALLOC_BITS_OPERATION_CMD_RAW_INST) >> 25) == 32, input_fire && ( + `GI(t.inputs, ALLOC_BITS_OPERATION_CMD_RAW_INST) >> 25) == 13); + repeat (retire_count) begin + ignored = order.pop_front(); + slots[ignored.slot] = null; + retired++; + end + endfunction + function void check_phase(uvm_phase phase); + super.check_phase(phase); + evidence::check(order.size() == 0 && completion_queue.size() == 0 && active_completion == null, + "Global ROB left live instructions"); + evidence::check(retired + cancelled_live == allocated, + "Global ROB allocation/retirement accounting mismatch"); + evidence::check(reordered != 0, "Global ROB did not exercise out-of-order completion"); + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/filelist.f b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/filelist.f new file mode 100644 index 000000000..ce6c74713 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@/GlobalROB ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/GlobalROB/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/interface.sv b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/interface.sv new file mode 100644 index 000000000..303a1ff62 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/interface.sv @@ -0,0 +1,14 @@ +`include "parameters.svh" +interface global_if ( + input logic clock +); + logic reset; + logic [`GROB_INPUT_BITS-1:0] inputs; + logic [`GROB_OUTPUT_BITS-1:0] outputs; + assert property (@(posedge clock) disable iff (reset) + outputs[`GROB_OUT_ISSUE_VALID_OFFSET] && !inputs[`GROB_IN_ISSUE_READY_OFFSET] |=> + outputs[`GROB_OUT_ISSUE_VALID_OFFSET] && $stable( + outputs[`GROB_ISSUE_OFFSET+:`GROB_ISSUE_BITS] + )) + else $fatal(1, "Global ROB issue payload changed under backpressure"); +endinterface diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/package.sv b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/package.sv new file mode 100644 index 000000000..6c5d3663e --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/package.sv @@ -0,0 +1,24 @@ +`include "parameters.svh" +`define GI(P, N) P[`GROB_IN_``N``_OFFSET+:`GROB_IN_``N``_WIDTH] +`define GO(P, N) P[`GROB_OUT_``N``_OFFSET+:`GROB_OUT_``N``_WIDTH] +`define OP(P, + N) P[`GROB_IN_ALLOC_BITS_OPERATION_``N``_OFFSET-`GROB_IN_OPERATION_OFFSET+:`GROB_IN_ALLOC_BITS_OPERATION_``N``_WIDTH] +`define CP( + P, N) P[`GROB_IN_ALLOC_BITS_``N``_OFFSET-`GROB_IN_ALLOC_OFFSET+:`GROB_IN_ALLOC_BITS_``N``_WIDTH] +`define BA(P, + N) P[`GROB_IN_BAT_RENAMED_``N``_OFFSET-`GROB_IN_RENAMED_OFFSET+:`GROB_IN_BAT_RENAMED_``N``_WIDTH] +`define ISS( + P, N) P[`GROB_OUT_ISSUE_BITS_``N``_OFFSET-`GROB_ISSUE_OFFSET+:`GROB_OUT_ISSUE_BITS_``N``_WIDTH] +package global_rob_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + `include "agent/item.svh" + `include "env/model.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/peer.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/seq/sequence.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/seq/sequence.svh new file mode 100644 index 000000000..ff829cb8a --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/seq/sequence.svh @@ -0,0 +1,123 @@ +class scenario_sequence extends uvm_sequence; + `uvm_object_utils(scenario_sequence) + env test_env; + virtual global_if vif; + function new(string name = "scenario_sequence"); + super.new(name); + endfunction + task send(int serial, int kind, int write_bank, int read0, int read1); + command_sequence seq = command_sequence::type_id::create("command"); + command_t packet = '0; + longint unsigned rs1, rs2; + int funct; + bit wr, rd0, rd1, update, config_op, movement; + operation_t operation; + wr = kind == 1 || kind == 3 || kind == 4 || kind == 5 || kind == 6 || kind == 7; + rd0 = kind == 2 || kind == 3 || kind == 4 || kind == 5 || kind == 7; + rd1 = kind == 4 || kind == 5 || kind == 7; + update = kind == 5; + config_op = kind == 6 || kind == 7; + movement = kind == 8; + if (kind == 6) read0 = write_bank; + if (kind == 7) read1 = write_bank; + case (kind) + 0: funct = 5; + 1: funct = 'h24; + 2: funct = 'h14; + 3: funct = 'h34; + 4, 5: funct = 'h44; + 6, 7: funct = 32; + 8: funct = 13; + default: `uvm_fatal("KIND", "Unknown global ROB scenario command") + endcase + rs1=(64'(serial%17+1)<<30)|(64'(write_bank)<<20)|(64'(read1)<<10)|read0|(update?64'h8000000000000000:0); + rs2 = config_op ? (kind == 7 ? 64'h1002 : 64'h2) : (64'h123456789abcdef0 ^ serial); + for (int bit_index = 0; bit_index < `GROB_IN_CONTEXT_BITS; bit_index++) + packet[`GROB_IN_CONTEXT_OFFSET-`GROB_IN_ALLOC_OFFSET+bit_index]=(serial>>(bit_index%16))^((bit_index+serial)%3==0); + operation = '0; + `OP(operation, RUNTIME) = serial % 5 != 0; + `OP(operation, DOMAIN_ID) = config_op || movement ? 1 : 3; + `OP(operation, ITER) = serial % 17 + 1; + `OP(operation, CMD_RAW_INST) = (funct << 25) | 'h7b; + `OP(operation, CMD_PC) = 64'hdead000010000000 + 64'(serial) * 4; + `OP(operation, CMD_RS1DATA) = rs1; + `OP(operation, CMD_RS2DATA) = rs2; + `OP(operation, BANKACCESS_WR_BANK_VALID) = wr; + `OP(operation, BANKACCESS_WR_BANK_ID) = write_bank; + `OP(operation, BANKACCESS_RD_BANK_0_VALID) = rd0; + `OP(operation, BANKACCESS_RD_BANK_0_ID) = read0; + `OP(operation, BANKACCESS_RD_BANK_1_VALID) = rd1; + `OP(operation, BANKACCESS_RD_BANK_1_ID) = read1; + `OP(operation, BANKACCESS_RD_WR_VALID) = update; + `OP(operation, BANKACCESS_RD_WR_ID) = write_bank; + `OP(operation, BANKACCESS_WR_UPDATE) = update; + `OP(operation, ARCHITECTURALDESTINATION) = write_bank; + packet[`GROB_IN_OPERATION_OFFSET-`GROB_IN_ALLOC_OFFSET+:`GROB_IN_OPERATION_BITS] = operation; + seq.item = command_item::type_id::create("item"); + seq.item.payload = packet; + seq.start(test_env.seqr); + endtask + task drain(); + do + @(negedge vif.clock); + while (test_env.reference.order.size() != 0 || ! + `GO(test_env.reference.latest.outputs, EMPTY) + ); + repeat (3) @(negedge vif.clock); + endtask + task body(); + int before_delivered; + test_env.reference.hold_complete = 1; + test_env.reference.hold_issue = 1; + test_env.reference.reverse_complete = 1; + test_env.reset.pulse(4); + for (int i = 0; i < `GROB_ENTRIES; i++) send(i + 1, 1, i % `GROB_VBANKS, 0, 0); + repeat (5) @(negedge vif.clock); + evidence::check(`GO(test_env.reference.latest.outputs, FULL), + "Global ROB did not reach full occupancy"); + fork + send(`GROB_ENTRIES + 1, 1, 0, 0, 0); + begin + repeat (5) @(negedge vif.clock); + evidence::check(!`GO(test_env.reference.latest.outputs, ALLOC_READY), + "Global ROB admitted work while full"); + test_env.reference.hold_issue = 0; + test_env.reference.hold_complete = 0; + end + join + drain(); + before_delivered = test_env.reference.delivered; + test_env.reference.hold_mapping = 1; + send(6, 1, 0, 0, 0); + send(7, 2, 0, 1 % `GROB_VBANKS, 0); + repeat (10) @(negedge vif.clock); + evidence::check(test_env.reference.delivered == before_delivered && + `GO(test_env.reference.latest.outputs, MAPPING_VALID), + "Global ROB bypassed a blocked oldest mapping request"); + test_env.reference.hold_mapping = 0; + drain(); + for (int n = 10; n < 510; n++) + send(n, n % 9, (n / 3) % `GROB_VBANKS, n % `GROB_VBANKS, (n + 1) % `GROB_VBANKS); + drain(); + // A partially selected issue pipeline and all its queued work cancel on reset. + test_env.reference.hold_issue = 1; + test_env.reference.hold_complete = 1; + for (int n = 0; n < `GROB_ENTRIES; n++) send(600 + n, 1, n % `GROB_VBANKS, 0, 0); + repeat (6) @(negedge vif.clock); + test_env.reset.pulse(4); + test_env.reference.hold_issue = 0; + test_env.reference.hold_complete = 0; + for (int n = 0; n < 32; n++) + send(700 + n, n % 6, n % `GROB_VBANKS, (n + 1) % `GROB_VBANKS, (n + 2) % `GROB_VBANKS); + drain(); + evidence::check(evidence::cancelled == `GROB_ENTRIES, + "Global ROB reset cancellation count mismatch"); + `uvm_info( + "GLOBAL_ROB", + $sformatf( + "allocated=%0d selected=%0d delivered=%0d completed=%0d retired=%0d reset_cancelled=%0d reordered=%0d", + test_env.reference.allocated, test_env.reference.selected, + test_env.reference.delivered, test_env.reference.completed, test_env.reference.retired, + test_env.reference.cancelled_live, test_env.reference.reordered), UVM_LOW) + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/tb.sv b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/tb.sv new file mode 100644 index 000000000..119039c69 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/tb.sv @@ -0,0 +1,19 @@ +module global_rob_tb; + import uvm_pkg::*; + import global_rob_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + clock_if timing (clock); + global_if bus (clock); + assign bus.reset = timing.reset; + GlobalROB dut ( + .clock(clock), + .reset(bus.reset), + `include "ports.svh" + ); + initial begin + uvm_config_db#(virtual global_if)::set(null, "uvm_test_top*", "vif", bus); + uvm_config_db#(virtual clock_if)::set(null, "uvm_test_top.env.reset", "timing", timing); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/tests/test.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/tests/test.svh new file mode 100644 index 000000000..8acdd1560 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/tests/test.svh @@ -0,0 +1,22 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + virtual global_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + if (!uvm_config_db#(virtual global_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Global ROB timing interface is required") + if (`GROB_DIFF || `GROB_GP) + `uvm_fatal("PROFILE", "Global ROB scenarios require normal execution with private banks") + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.test_env = test_env; + seq.vif = vif; + seq.start(null); + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/agent/driver.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/agent/driver.svh new file mode 100644 index 000000000..308166c78 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/agent/driver.svh @@ -0,0 +1,25 @@ +class driver extends uvm_driver #(cycle_item); + `uvm_component_utils(driver) + virtual rob_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual rob_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "ROB storage interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item req; + forever begin + seq_item_port.get_next_item(req); + @(negedge vif.clock); + vif.reset = req.reset; + vif.inputs = req.inputs; + @(posedge vif.clock); + req.outputs = vif.outputs; + #1; + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/agent/item.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/agent/item.svh new file mode 100644 index 000000000..6ecbd9a3f --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/agent/item.svh @@ -0,0 +1,18 @@ +class cycle_item extends uvm_sequence_item; + `uvm_object_utils(cycle_item) + logic [`ROB_INPUT_BITS-1:0] inputs; + logic [`ROB_OUTPUT_BITS-1:0] outputs; + bit reset; + function new(string name = "cycle_item"); + super.new(name); + endfunction +endclass +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(payload, UVM_ALL_ON) + `uvm_object_utils_end + logic [`ROB_OUTPUT_BITS-1:0] payload; + function new(string name = "result_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/agent/monitor.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/agent/monitor.svh new file mode 100644 index 000000000..81d33e587 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/agent/monitor.svh @@ -0,0 +1,25 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual rob_if vif; + uvm_analysis_port #(cycle_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual rob_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "ROB storage monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item t; + forever begin + @(posedge vif.clock); + t = cycle_item::type_id::create("observed"); + t.inputs = vif.inputs; + t.outputs = vif.outputs; + t.reset = vif.reset; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/env/env.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/env/env.svh new file mode 100644 index 000000000..ed95874fb --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/env/env.svh @@ -0,0 +1,25 @@ +class env extends uvm_env; + `uvm_component_utils(env) + driver drv; + monitor mon; + model reference; + uvm_sequencer #(cycle_item) seqr; + result_board #(result_item, 0) scoreboard; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + drv = driver::type_id::create("driver", this); + mon = monitor::type_id::create("monitor", this); + reference = model::type_id::create("model", this); + seqr = new("seqr", this); + scoreboard = result_board#(result_item, 0)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + drv.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(reference.analysis_export); + reference.expected_ap.connect(scoreboard.expected_export); + reference.actual_ap.connect(scoreboard.actual_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/env/model.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/env/model.svh new file mode 100644 index 000000000..49284a2ed --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/env/model.svh @@ -0,0 +1,117 @@ +class model extends uvm_subscriber #(cycle_item); + `uvm_component_utils(model) + localparam int N = `ROB_ENTRIES; + logic [`ROB_DATA_BITS-1:0] data[N]; + bit valid[N], issued[N], done[N], barrier[N]; + bit [N-1:0] dependencies[N]; + int order[$], tail = 0; + int state_offsets[N][5] = `ROB_STATE_OFFSETS; + int ready_offsets[N] = `ROB_READY_OFFSETS; + uvm_analysis_port #(result_item) expected_ap, actual_ap; + covergroup behavior with function sample ( + int count, int retired, bit blocked, bit simultaneous, bit commit_barrier + ); + option.per_instance = 1; + occupancy: coverpoint count {bins empty = {0}; bins partial = {[1 : N - 1]}; bins full = {N};} + retirement: coverpoint retired { + bins none = {0}; bins single = {1}; bins multiple = {[2 : `ROB_RETIRE_WIDTH]}; + } + source_backpressure: coverpoint blocked; + issue_complete: coverpoint simultaneous; + sticky_dependency: coverpoint commit_barrier; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected_ap = new("expected_ap", this); + actual_ap = new("actual_ap", this); + behavior = new(); + endfunction + function void write(cycle_item t); + result_item expected, actual; + bit [N-1:0] candidates, retiring, old_barriers; + int retire_count, in_flight, head, ignored; + bit admit; + if (t.reset) begin + foreach (data[i]) begin + data[i] = 0; + valid[i] = 0; + issued[i] = 0; + done[i] = 0; + barrier[i] = 0; + dependencies[i] = 0; + end + order.delete(); + tail = 0; + return; + end + candidates = 0; + retiring = 0; + retire_count = 0; + in_flight = 0; + old_barriers = 0; + foreach (barrier[i]) old_barriers[i] = barrier[i]; + for (int age = 0; age < `ROB_RETIRE_WIDTH && age < order.size(); age++) begin + if (!done[order[age]]) break; + candidates[order[age]] = 1; + end + for (int age = 0; age < `ROB_RETIRE_WIDTH && age < order.size(); age++) begin + if (!done[order[age]] || !t.inputs[ready_offsets[order[age]]]) break; + retiring[order[age]] = 1; + retire_count++; + end + head = order.size() ? order[0] : tail; + admit = order.size() < N && retiring == 0; + foreach (valid[i]) if (valid[i] && issued[i] && !done[i]) in_flight++; + expected = result_item::type_id::create("expected"); + actual = result_item::type_id::create("actual"); + expected.payload = '0; + actual.payload = t.outputs; + expected.payload[`ROB_OUT_ALLOC_READY_OFFSET+:`ROB_OUT_ALLOC_READY_WIDTH] = admit; + expected.payload[`ROB_OUT_RETIRECANDIDATES_OFFSET+:`ROB_OUT_RETIRECANDIDATES_WIDTH] = candidates; + expected.payload[`ROB_OUT_RETIRED_OFFSET+:`ROB_OUT_RETIRED_WIDTH] = retiring; + expected.payload[`ROB_OUT_HEAD_OFFSET+:`ROB_OUT_HEAD_WIDTH] = head; + expected.payload[`ROB_OUT_TAIL_OFFSET+:`ROB_OUT_TAIL_WIDTH] = tail; + expected.payload[`ROB_OUT_ISSUEDCOUNT_OFFSET+:`ROB_OUT_ISSUEDCOUNT_WIDTH] = in_flight; + expected.payload[`ROB_OUT_EMPTY_OFFSET+:`ROB_OUT_EMPTY_WIDTH] = order.size() == 0; + expected.payload[`ROB_OUT_FULL_OFFSET+:`ROB_OUT_FULL_WIDTH] = order.size() == N; + foreach (data[i]) begin + expected.payload[state_offsets[i][0]+:`ROB_DATA_BITS] = data[i]; + expected.payload[state_offsets[i][1]] = valid[i]; + expected.payload[state_offsets[i][2]] = issued[i]; + expected.payload[state_offsets[i][3]] = done[i]; + expected.payload[state_offsets[i][4]+:N] = dependencies[i]; + end + behavior.sample(order.size(), retire_count, `RI(t.inputs, ALLOC_VALID) && !admit, + `RI(t.inputs, ISSUE_VALID) && `RI(t.inputs, COMPLETE_VALID), + `RI(t.inputs, ALLOC_BITS_COMMITBARRIER)); + expected_ap.write(expected); + actual_ap.write(actual); + if (`RI(t.inputs, ALLOC_VALID) && admit) begin + data[tail] = `RI(t.inputs, ALLOC_BITS_DATA); + valid[tail] = 1; + issued[tail] = 0; + done[tail] = 0; + barrier[tail] = `RI(t.inputs, ALLOC_BITS_COMMITBARRIER); + dependencies[tail] = `RI(t.inputs, ALLOC_BITS_DEPENDENCIES); + order.push_back(tail); + tail = (tail + 1) % N; + end + if (`RI(t.inputs, ISSUE_VALID)) issued[`RI(t.inputs, ISSUE_BITS)] = 1; + if (`RI(t.inputs, COMPLETE_VALID)) done[`RI(t.inputs, COMPLETE_BITS)] = 1; + foreach (dependencies[i]) begin + dependencies[i] &= ~retiring; + if ( + `RI(t.inputs, COMPLETE_VALID) + && !barrier[i] && !old_barriers[ + `RI(t.inputs, COMPLETE_BITS) + ]) + dependencies[i][`RI(t.inputs, COMPLETE_BITS)] = 0; + end + repeat (retire_count) begin + ignored = order.pop_front(); + valid[ignored] = 0; + issued[ignored] = 0; + done[ignored] = 0; + end + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/filelist.f b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/filelist.f new file mode 100644 index 000000000..a90a3eb69 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@/Rob ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/Rob/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/interface.sv b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/interface.sv new file mode 100644 index 000000000..b95dcc654 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/interface.sv @@ -0,0 +1,8 @@ +`include "parameters.svh" +interface rob_if ( + input logic clock +); + logic reset = 1; + logic [`ROB_INPUT_BITS-1:0] inputs; + logic [`ROB_OUTPUT_BITS-1:0] outputs; +endinterface diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/package.sv b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/package.sv new file mode 100644 index 000000000..68bacc400 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/package.sv @@ -0,0 +1,15 @@ +`include "parameters.svh" +`define RI(P, N) P[`ROB_IN_``N``_OFFSET+:`ROB_IN_``N``_WIDTH] +`define RO(P, N) P[`ROB_OUT_``N``_OFFSET+:`ROB_OUT_``N``_WIDTH] +package rob_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/seq/sequence.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/seq/sequence.svh new file mode 100644 index 000000000..23e9d498c --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/seq/sequence.svh @@ -0,0 +1,154 @@ +class scenario_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(scenario_sequence) + localparam int N = `ROB_ENTRIES; + logic [`ROB_INPUT_BITS-1:0] pins = '0; + logic [`ROB_OUTPUT_BITS-1:0] observed; + bit reset = 1; + bit [N-1:0] live = 0, issued = 0, done = 0; + int ready_offsets[N] = `ROB_READY_OFFSETS; + function new(string name = "scenario_sequence"); + super.new(name); + endfunction + task tick(int cycles = 1); + cycle_item req; + int slot; + repeat (cycles) begin + req = cycle_item::type_id::create("cycle"); + start_item(req); + req.inputs = pins; + req.reset = reset; + finish_item(req); + observed = req.outputs; + if (reset) begin + live = 0; + issued = 0; + done = 0; + end else begin + if (`RI(pins, ALLOC_VALID) && `RO(observed, ALLOC_READY)) begin + slot = `RO(observed, TAIL); + live[slot] = 1; + issued[slot] = 0; + done[slot] = 0; + `RI(pins, ALLOC_VALID) = 0; + end + if (`RI(pins, ISSUE_VALID)) issued[`RI(pins, ISSUE_BITS)] = 1; + if (`RI(pins, COMPLETE_VALID)) done[`RI(pins, COMPLETE_BITS)] = 1; + live &= ~`RO(observed, RETIRED); + issued &= ~`RO(observed, RETIRED); + done &= ~`RO(observed, RETIRED); + end + end + endtask + task restart(); + pins = '0; + reset = 1; + tick(2); + reset = 0; + tick(); + endtask + task body(); + int selected; + restart(); + for (int n = 0; n < N; n++) begin + `RI(pins, ALLOC_VALID) = 1; + `RI(pins, ALLOC_BITS_DATA) = 64'hdeadbeef00000000 | n; + `RI(pins, ALLOC_BITS_DEPENDENCIES) = live; + `RI(pins, ALLOC_BITS_COMMITBARRIER) = n % 2; + do tick(); while (`RI(pins, ALLOC_VALID)); + end + tick(3); + evidence::check(`RO(observed, FULL), "ROB did not fill"); + for (int slot = 0; slot < N; slot++) begin + `RI(pins, ISSUE_VALID) = 1; + `RI(pins, ISSUE_BITS) = slot; + tick(); + end + `RI(pins, ISSUE_VALID) = 0; + for (int slot = N - 1; slot >= 0; slot--) begin + `RI(pins, COMPLETE_VALID) = 1; + `RI(pins, COMPLETE_BITS) = slot; + tick(); + end + `RI(pins, COMPLETE_VALID) = 0; + tick(2); + foreach (ready_offsets[i]) pins[ready_offsets[i]] = 1; + tick(2); + evidence::check(`RO(observed, EMPTY), "ROB did not retire its completed batch"); + for (int cycle = 0; cycle < 1000; cycle++) begin + if (!`RI(pins, ALLOC_VALID) && cycle % 3 != 0) begin + `RI(pins, ALLOC_VALID) = 1; + `RI(pins, ALLOC_BITS_DATA) = (64'h9e3779b97f4a7c15 * cycle) ^ 64'hcafe00001234beef; + `RI(pins, ALLOC_BITS_DEPENDENCIES) = live; + `RI(pins, ALLOC_BITS_COMMITBARRIER) = cycle % 4 == 0; + end + `RI(pins, ISSUE_VALID) = 0; + `RI(pins, COMPLETE_VALID) = 0; + for (int i = 0; i < N; i++) begin + selected = (cycle + i) % N; + if (live[selected] && !issued[selected] && !done[selected]) begin + `RI(pins, ISSUE_VALID) = 1; + `RI(pins, ISSUE_BITS) = selected; + break; + end + end + for (int i = 0; i < N; i++) begin + selected = (N - 1 - i + cycle) % N; + if (live[selected] && issued[selected] && !done[selected]) begin + `RI(pins, COMPLETE_VALID) = 1; + `RI(pins, COMPLETE_BITS) = selected; + break; + end + end + foreach (ready_offsets[i]) pins[ready_offsets[i]] = (cycle + i) % 5 != 0; + tick(); + end + `RI(pins, ALLOC_VALID) = 0; + `RI(pins, ISSUE_VALID) = 0; + foreach (ready_offsets[i]) pins[ready_offsets[i]] = 1; + while (live != 0) begin + `RI(pins, COMPLETE_VALID) = 0; + for (int i = 0; i < N; i++) + if (live[i] && !done[i]) begin + `RI(pins, COMPLETE_VALID) = 1; + `RI(pins, COMPLETE_BITS) = i; + break; + end + tick(); + end + `RI(pins, COMPLETE_VALID) = 0; + // A one-cycle unit may issue and complete the same live entry in one clock. + `RI(pins, ALLOC_VALID) = 1; + `RI(pins, ALLOC_BITS_DATA) = '1; + do tick(); while (`RI(pins, ALLOC_VALID)); + selected = `RO(observed, TAIL); + `RI(pins, ISSUE_VALID) = 1; + `RI(pins, ISSUE_BITS) = selected; + `RI(pins, COMPLETE_VALID) = 1; + `RI(pins, COMPLETE_BITS) = selected; + tick(); + `RI(pins, ISSUE_VALID) = 0; + `RI(pins, COMPLETE_VALID) = 0; + tick(3); + endtask +endclass +class invalid_sequence extends scenario_sequence; + `uvm_object_utils(invalid_sequence) + function new(string name = "invalid_sequence"); + super.new(name); + endfunction + task body(); + string scenario; + if (!$value$plusargs("CASE=%s", scenario)) + `uvm_fatal("CASE", "ROB assertion scenario is required") + restart(); + if (scenario == "issue") begin + `RI(pins, ISSUE_VALID) = 1; + `RI(pins, ISSUE_BITS) = 0; + end else begin + `RI(pins, COMPLETE_VALID) = 1; + `RI(pins, COMPLETE_BITS) = 0; + end + tick(3); + `uvm_fatal("EXPECTED_ASSERTION", "Invalid ROB command did not trigger its assertion") + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/tb.sv b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/tb.sv new file mode 100644 index 000000000..cba4070f9 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/tb.sv @@ -0,0 +1,16 @@ +module rob_tb; + import uvm_pkg::*; + import rob_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + rob_if bus (clock); + Rob dut ( + .clock(clock), + .reset(bus.reset), + `include "ports.svh" + ); + initial begin + uvm_config_db#(virtual rob_if)::set(null, "uvm_test_top*", "vif", bus); + run_test(); + end +endmodule diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/tests/test.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/tests/test.svh new file mode 100644 index 000000000..0230681bb --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/tests/test.svh @@ -0,0 +1,35 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.start(test_env.seqr); + endtask +endclass +class invalid_test extends base_test; + `uvm_component_utils(invalid_test) + driver drv; + uvm_sequencer #(cycle_item) seqr; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + drv = driver::type_id::create("driver", this); + seqr = new("seqr", this); + endfunction + function void connect_phase(uvm_phase phase); + drv.seq_item_port.connect(seqr.seq_item_export); + endfunction + task execute(); + invalid_sequence seq = invalid_sequence::type_id::create("scenario"); + seq.start(seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/suite.toml b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/suite.toml new file mode 100644 index 000000000..3f1dbbb1d --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/suite.toml @@ -0,0 +1,71 @@ +[model] +manifest = "" +library = "" + +[[tests]] +name = "rob" +top = "rob_tb" +test = "protocol_test" +filelist = "rob/filelist.f" +timeout_ns = 300000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +functional_groups = ["rob_pkg::model::behavior"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +name = "rob_bad_issue" +top = "rob_tb" +test = "invalid_test" +filelist = "rob/filelist.f" +timeout_ns = 300000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +functional_groups = [] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=issue"] +assertion = "ROB issue must name a live unissued entry" + +[[tests]] +name = "rob_bad_complete" +top = "rob_tb" +test = "invalid_test" +filelist = "rob/filelist.f" +timeout_ns = 300000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +functional_groups = [] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=complete"] +assertion = "ROB completion must name a live incomplete entry" + +[[tests]] +name = "global_rob" +top = "global_rob_tb" +test = "protocol_test" +filelist = "global_rob/filelist.f" +timeout_ns = 1000000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +functional_groups = ["global_rob_pkg::model::behavior"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/DependencyResolver.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/DependencyResolver.scala new file mode 100644 index 000000000..c7e7eede0 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/DependencyResolver.scala @@ -0,0 +1,124 @@ +package framework.acc.frontend.scheduler.rob.global + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.core.memory.access.preflight.configs.PmpParams + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public} +import chisel3.util._ +import framework.acc.frontend.decoder.PostGDCmd +import framework.acc.memdomain.DISA.MSET_BITPAT +import framework.acc.memdomain.MvoverISA + +@instantiable +class DependencyResolver( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val gpParam: GpDomainParam, + val permissionParam: PmpParams) + extends Module { + val commandDataBits = frontendParam.commandDataBits + val robDepth = frontendParam.rob_entries + + val resourceCount = + memParam.virtualBankCount + (if (gpParam.enable) 2 + else 0) + + @public + val io = IO(new Bundle { + val alloc = Input(new PostGDCmd(frontendParam, memParam)) + val entries = Input(Vec(robDepth, new GlobalRobEntry(frontendParam, memParam, permissionParam))) + val valid = Input(Vec(robDepth, Bool())) + val complete = Input(Vec(robDepth, Bool())) + val dependencies = Output(UInt(robDepth.W)) + val movementLive = Output(Bool()) + }) + + val entryReads = Wire(Vec(robDepth, UInt(resourceCount.W))) + val entryWrites = Wire(Vec(robDepth, UInt(resourceCount.W))) + val entryIsConfig = Wire(Vec(robDepth, Bool())) + + for (slot <- 0 until robDepth) { + val access = io.entries(slot).cmd.bankAccess + val read0 = Mux( + access.rd_bank_0_valid && access.rd_bank_0_id < resourceCount.U, + UIntToOH(access.rd_bank_0_id, resourceCount), + 0.U(resourceCount.W) + ) + val read1 = Mux( + access.rd_bank_1_valid && access.rd_bank_1_id < resourceCount.U, + UIntToOH(access.rd_bank_1_id, resourceCount), + 0.U(resourceCount.W) + ) + val readWrite = Mux( + access.rd_wr_valid && access.rd_wr_id < resourceCount.U, + UIntToOH(access.rd_wr_id, resourceCount), + 0.U(resourceCount.W) + ) + entryReads(slot) := read0 | read1 | readWrite + entryWrites(slot) := Mux( + access.wr_bank_valid && access.wr_bank_id < resourceCount.U, + UIntToOH(access.wr_bank_id, resourceCount), + 0.U(resourceCount.W) + ) + entryIsConfig(slot) := io.entries(slot).cmd.cmd.raw_inst(31, 25) === MSET_BITPAT + } + + val allocAccess = io.alloc.bankAccess + + val allocRead0 = Mux( + allocAccess.rd_bank_0_valid && allocAccess.rd_bank_0_id < resourceCount.U, + UIntToOH(allocAccess.rd_bank_0_id, resourceCount), + 0.U(resourceCount.W) + ) + + val allocRead1 = Mux( + allocAccess.rd_bank_1_valid && allocAccess.rd_bank_1_id < resourceCount.U, + UIntToOH(allocAccess.rd_bank_1_id, resourceCount), + 0.U(resourceCount.W) + ) + + val allocReadWrite = Mux( + allocAccess.rd_wr_valid && allocAccess.rd_wr_id < resourceCount.U, + UIntToOH(allocAccess.rd_wr_id, resourceCount), + 0.U(resourceCount.W) + ) + + val allocReads = allocRead0 | allocRead1 | allocReadWrite + + val allocWrites = Mux( + allocAccess.wr_bank_valid && allocAccess.wr_bank_id < resourceCount.U, + UIntToOH(allocAccess.wr_bank_id, resourceCount), + 0.U(resourceCount.W) + ) + + val allocIsConfig = io.alloc.cmd.raw_inst(31, 25) === MSET_BITPAT + + val unrenamed = + if (frontendParam.bank_alias_enable && memParam.bankLayoutEnable) + ~((BigInt(1) << memParam.virtualBankCount) - 1).U(resourceCount.W) + else Fill(resourceCount, 1.U(1.W)) + + val dependency = Wire(Vec(robDepth, Bool())) + for (older <- 0 until robDepth) { + val olderAccess = io.entries(older).cmd.bankAccess + val olderUse = entryReads(older) | entryWrites(older) + val bankConflict = + (((allocReads & entryWrites(older)) | (allocWrites & olderUse)) & unrenamed).orR || + (allocAccess.wr_update && ((allocReads & entryWrites(older)) | (allocWrites & olderUse)).orR) || + (olderAccess.wr_update && (allocReads & entryWrites(older)).orR) + val configConflict = (allocIsConfig && entryIsConfig(older)) || + ((allocIsConfig || entryIsConfig(older)) && ((allocReads | allocWrites) & olderUse).orR) + dependency(older) := + (io.valid(older) && !io.complete(older) && bankConflict) || + (io.valid(older) && configConflict) || + (io.valid(older) && !io.complete(older) && io.alloc.cmd.raw_inst(31, 25) === MvoverISA.Funct.U) + } + + io.dependencies := dependency.asUInt + io.movementLive := VecInit((0 until robDepth).map(slot => + io.valid(slot) && !io.complete(slot) && io.entries(slot).cmd.cmd.raw_inst(31, 25) === MvoverISA.Funct.U + )).asUInt.orR +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/Emit.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/Emit.scala new file mode 100644 index 000000000..b20fafac6 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/Emit.scala @@ -0,0 +1,152 @@ +package framework.acc.frontend.scheduler.rob.global + +import chisel3._ +import java.nio.file.{Files, Paths} +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.core.memory.access.preflight.configs.PmpParams +import framework.acc.frontend.scheduler.rob.global.configs.RobParams + +object Emit extends App { + val Array("--config", config, "--output-dir", output) = args + val values = ujson.read(Files.readString(Paths.get(config))) + val frontend = upickle.default.read[FrontendParam](values("frontend")) + val memory = upickle.default.read[MemDomainParam](values("memory")) + val gp = upickle.default.read[GpDomainParam](values("gp")) + val permission = upickle.default.read[PmpParams](values("permission")) + val storage = upickle.default.read[RobParams](values("rob")) + val diffTest = values("diffTest").bool + Files.createDirectories(Paths.get(output)) + + def fields(data: Data, path: String): Seq[(String, Int)] = data match { + case record: Record => record.elements.toSeq.flatMap { case (name, field) => + fields(field, if (path.isEmpty) name else s"${path}_$name") + } + case vector: Vec[_] => vector.toSeq.zipWithIndex.flatMap { case (field, index) => fields(field, s"${path}_$index") } + case leaf => Seq(path -> leaf.getWidth) + } + + for (name <- Seq("Rob", "GlobalROB")) { + val prefix = if (name == "Rob") "ROB" else "GROB" + val ports = scala.collection.mutable.ArrayBuffer[String]() + val definitions = scala.collection.mutable.ArrayBuffer[String]() + val offsets = scala.collection.mutable.Map[String, Int]() + val widths = scala.collection.mutable.Map[String, Int]() + var inputOffset = 0 + var outputOffset = 0 + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + { + val (module, io) = + if (name == "Rob") { + val dut = new Rob(storage); (dut, dut.io) + } else { + val dut = new GlobalROB(frontend, memory, gp, permission, diffTest); (dut, dut.io) + } + for { + (root, data) <- io.elements.toSeq + (field, width) <- fields(data, root) + } { + val input = + if (name == "Rob") Set("alloc", "issue", "complete", "retireReady").contains(root) && field != "alloc_ready" + else field == "alloc_valid" || field.startsWith( + "alloc_bits_" + ) || field == "issue_ready" || field == "mapping_ready" || + field == "complete_valid" || field == "complete_bits" || field == "bat_ready" || field.startsWith( + "bat_renamed_" + ) || + field.startsWith("bat_stale_") || field.startsWith("scoreboard_hazard_") + val direction = if (input) "IN" else "OUT" + val signal = if (input) "inputs" else "outputs" + val offset = if (input) inputOffset else outputOffset + definitions += s"`define ${prefix}_${direction}_${field.toUpperCase}_OFFSET $offset" + definitions += s"`define ${prefix}_${direction}_${field.toUpperCase}_WIDTH $width" + ports += s".io_$field(bus.$signal[$offset +: $width])" + offsets(field) = offset; widths(field) = width + if (input) inputOffset += width else outputOffset += width + } + module + }, + args = Array("--target-dir", s"$output/$name", "--split-verilog") + ) + definitions += s"`define ${prefix}_INPUT_BITS $inputOffset" + definitions += s"`define ${prefix}_OUTPUT_BITS $outputOffset" + if (name == "Rob") { + definitions += s"`define ROB_ENTRIES ${storage.entries}" + definitions += s"`define ROB_DATA_BITS ${storage.dataBits}" + definitions += s"`define ROB_RETIRE_WIDTH ${storage.retireWidth}" + val state = (0 until storage.entries).map(i => + Seq("data", "valid", "issued", "completed", "dependencies").map(n => offsets(s"${n}_$i")).mkString( + "'{", + ",", + "}" + ) + ).mkString("'{", ",", "}") + val ready = (0 until storage.entries).map(i => offsets(s"retireReady_$i")).mkString("'{", ",", "}") + definitions += s"`define ROB_STATE_OFFSETS $state" + definitions += s"`define ROB_READY_OFFSETS $ready" + } else { + definitions += s"`define GROB_ENTRIES ${frontend.rob_entries}" + definitions += s"`define GROB_VBANKS ${memory.virtualBankCount}" + definitions += s"`define GROB_BANK_BITS ${frontend.bank_id_len}" + definitions += s"`define GROB_OWNER ${frontend.ownerId}" + definitions += s"`define GROB_ALIASES ${2 * (memory.virtualBankCount + (if (gp.enable) 2 else 0)) + frontend.rob_entries}" + definitions += s"`define GROB_RENAMING ${if (frontend.bank_alias_enable) 1 else 0}" + definitions += s"`define GROB_LAYOUT ${if (memory.bankLayoutEnable) 1 else 0}" + definitions += s"`define GROB_GP ${if (gp.enable) 1 else 0}" + definitions += s"`define GROB_DIFF ${if (diffTest) 1 else 0}" + for ( + (key, fieldPrefix) <- Seq( + "IN_OPERATION" -> "alloc_bits_operation_", + "OUT_COMMAND" -> "issue_bits_cmd_", + "IN_CONTEXT" -> "alloc_bits_context_", + "OUT_CONTEXT" -> "issue_bits_context_", + "IN_RENAMED" -> "bat_renamed_", + "OUT_RENAMED" -> "issue_bits_renamedBankAccess_" + ) + ) { + val fs = offsets.keys.filter(_.startsWith(fieldPrefix)).toSeq + definitions += s"`define GROB_${key}_OFFSET ${fs.map(offsets).min}" + definitions += s"`define GROB_${key}_BITS ${fs.map(widths).sum}" + } + val issueFields = offsets.keys.filter(_.startsWith("issue_bits_")).toSeq + definitions += s"`define GROB_ISSUE_OFFSET ${issueFields.map(offsets).min}" + definitions += s"`define GROB_ISSUE_BITS ${issueFields.map(widths).sum}" + val queryFields = Seq( + "rd_bank_0_valid", + "rd_bank_0_id", + "rd_bank_1_valid", + "rd_bank_1_id", + "rd_wr_valid", + "rd_wr_id", + "wr_bank_valid", + "wr_bank_id", + "wr_update" + ) + val queries = (0 until frontend.rob_entries).map(i => + queryFields.map(n => offsets(s"scoreboard_query_${i}_$n")).mkString("'{", ",", "}") + ).mkString("'{", ",", "}") + definitions += s"`define GROB_QUERY_OFFSETS $queries" + val baBase = offsets.keys.filter(_.startsWith("bat_renamed_")).map(offsets).min + val baOffsets = queryFields.map(n => offsets(s"bat_renamed_$n") - baBase).mkString("'{", ",", "}") + definitions += s"`define GROB_BA_OFFSETS $baOffsets" + val traceFields = + Seq("is_issue", "thread_id", "rob_id", "inst_id", "domain_id", "funct", "pc", "rs1_data", "rs2_data", "enable") + val traces = (0 until 4).map(i => + traceFields.map(n => offsets(s"traceInstructions_${i}_$n")).mkString("'{", ",", "}") + ).mkString("'{", ",", "}") + definitions += s"`define GROB_TRACE_OFFSETS $traces" + + for ((key, fieldPrefix) <- Seq("IN_ALLOC" -> "alloc_bits_", "MAPPING" -> "mapping_bits_")) { + val fs = offsets.keys.filter(_.startsWith(fieldPrefix)).toSeq + definitions += s"`define GROB_${key}_OFFSET ${fs.map(offsets).min}" + definitions += s"`define GROB_${key}_BITS ${fs.map(widths).sum}" + } + + val hazards = (0 until frontend.rob_entries).map(i => offsets(s"scoreboard_hazard_$i")).mkString("'{", ",", "}") + definitions += s"`define GROB_HAZARD_OFFSETS $hazards" + } + Files.writeString(Paths.get(output, name, "parameters.svh"), definitions.mkString("\n") + "\n") + Files.writeString(Paths.get(output, name, "ports.svh"), ports.mkString(",\n") + "\n") + } +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/GlobalROB.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/GlobalROB.scala new file mode 100644 index 000000000..beb10f5eb --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/GlobalROB.scala @@ -0,0 +1,478 @@ +package framework.acc.frontend.scheduler.rob.global + +import framework.lib.arbiter.{Arbiter} + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.core.memory.access.preflight.configs.PmpParams + +import chisel3._ +import chisel3.util._ +import chisel3.experimental._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.frontend.decoder.{DecodedInstruction, DomainId, PostGDCmd} +import framework.acc.frontend.scheduler.scoreboard.BankAccessInfo +import framework.acc.memdomain.DISA.MSET_BITPAT +import framework.acc.memdomain.DISA.MVIN_KERNEL_BITPAT +import framework.acc.gpdomain.decoder.DISA.RUN_KERNEL_BITPAT +import framework.acc.memdomain.bank_router.{BankMapping, BankMappingIO} +import framework.acc.memdomain.{MemoryContext, PhysicalBankHash} +import framework.acc.debug.trace.InstructionEvent +import framework.acc.debug.DebugPort + +class GlobalRobEntry(val frontendParam: FrontendParam, val memParam: MemDomainParam, val permissionParam: PmpParams) + extends Bundle { + val commandDataBits = frontendParam.commandDataBits + val context = new MemoryContext(permissionParam) + val cmd = new PostGDCmd(frontendParam, memParam) + val renamedBankAccess = new BankAccessInfo(frontendParam.bank_id_len) + val rob_id = UInt(log2Up(frontendParam.rob_entries).W) + val instId = UInt(64.W) +} + +@instantiable +class GlobalROB( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val gpParam: GpDomainParam, + val permissionParam: PmpParams, + val diffTest: Boolean) + extends Module { + val commandDataBits = frontendParam.commandDataBits + val ownerId = frontendParam.ownerId + + require( + !memParam.sharedEnable || (frontendParam.bank_alias_enable && memParam.bankLayoutEnable), + "BAT/BLT ablation modes require private-bank workloads" + ) + + val robDepth = frontendParam.rob_entries + val idWidth = log2Up(robDepth) + + val resourceCount = + memParam.virtualBankCount + (if (gpParam.enable) 2 + else 0) + + val scoreBankNum = 2 * resourceCount + robDepth + require( + scoreBankNum <= (1 << frontendParam.bank_id_len), + "Bank resource IDs and ROB aliases exceed bank_id_len" + ) + val sharedHashCount = + if (memParam.sharedEnable) memParam.sharedBankNum else 0 + val physicalBankCount = memParam.bankNum + sharedHashCount + + require( + frontendParam.max_private_vbank < memParam.virtualBankCount, + s"max_private_vbank(${frontendParam.max_private_vbank}) must be < virtualBankCount(${memParam.virtualBankCount})" + ) + + @public + val io = IO(new Bundle { + val traceInstructions = Output(Vec(4, new InstructionEvent)) + val alloc = + Flipped(new DecoupledIO(new DecodedInstruction(frontendParam, memParam, permissionParam))) + val issue = new DecoupledIO(new GlobalRobEntry(frontendParam, memParam, permissionParam)) + val complete = Flipped(new DecoupledIO(UInt(idWidth.W))) + val mapping = new BankMappingIO(frontendParam) + + val bat = new Bundle { + + val alloc = Output(new Bundle { + val valid = Bool() + val rename = Bool() + val reset_read0 = Bool() + val raw = new BankAccessInfo(frontendParam.bank_id_len) + }) + + val reclaim = Output(Valid(UInt(frontendParam.bank_id_len.W))) + val stale = Input(Vec(2, Valid(UInt(frontendParam.bank_id_len.W)))) + val ready = Input(Bool()) + val renamed = Input(new BankAccessInfo(frontendParam.bank_id_len)) + } + + val scoreboard = new Bundle { + val alloc = Valid(new BankAccessInfo(frontendParam.bank_id_len)) + val issue = Valid(new BankAccessInfo(frontendParam.bank_id_len)) + val complete = Valid(new BankAccessInfo(frontendParam.bank_id_len)) + + val query = Output(Vec( + robDepth, + new BankAccessInfo(frontendParam.bank_id_len) + )) + + val hazard = Input(Vec(robDepth, Bool())) + } + + val empty = Output(Bool()) + val full = Output(Bool()) + + val debug = Option.when(diffTest)(new DebugPort(frontendParam)) + + }) + + val mappingIssue = Wire(Decoupled(new BankMapping(frontendParam))) + val mappingRetire = Wire(Decoupled(new BankMapping(frontendParam))) + val mappings = Instantiate(new Arbiter(new BankMapping(frontendParam), 2)) + mappings.io.in(0) <> mappingRetire + mappings.io.in(1) <> mappingIssue + io.mapping.valid := mappings.io.out.valid + io.mapping.bits := mappings.io.out.bits + mappings.io.out.ready := io.mapping.ready + + // --------------------------------------------------------------------------- + // Passive instruction observations consumed by the thread trace collector. + // --------------------------------------------------------------------------- + val itraceAlloc = io.traceInstructions(0) + val itraceIssue = io.traceInstructions(1) + val itraceComp = io.traceInstructions(2) + + for (t <- Seq(itraceAlloc, itraceIssue, itraceComp)) { + t.is_issue := 0.U + t.thread_id := ownerId.U(64.W) + t.rob_id := 0.U + t.inst_id := 0.U + t.domain_id := 0.U + t.funct := 0.U + t.pc := 0.U + t.rs1_data := 0.U + t.rs2_data := 0.U + t.enable := false.B + } + + // --------------------------------------------------------------------------- + // Storage + // --------------------------------------------------------------------------- + val queue: Instance[Rob] = Instantiate(new Rob(configs.RobParams( + robDepth, + (new GlobalRobEntry(frontendParam, memParam, permissionParam)).getWidth, + if (diffTest) 1 else robDepth + ))) + + val robEntries = + VecInit(queue.io.data.map(_.asTypeOf(new GlobalRobEntry(frontendParam, memParam, permissionParam)))) + val robValid = queue.io.valid + val robIssued = queue.io.issued + val robComplete = queue.io.completed + val allocatedBank = RegInit(VecInit(Seq.fill(robDepth)(false.B))) + val nextInstId = RegInit(1.U(64.W)) + + val headPtr = queue.io.head + val tailPtr = queue.io.tail + val issuedCount = queue.io.issuedCount + + val dependencies: Instance[DependencyResolver] = + Instantiate(new DependencyResolver(frontendParam, memParam, gpParam, permissionParam)) + dependencies.io.alloc := io.alloc.bits.operation + dependencies.io.entries := robEntries + dependencies.io.valid := robValid + dependencies.io.complete := robComplete + + // Dependency state is maintained incrementally instead of rebuilding all + // older-entry comparisons every cycle. A bit in slot k names the ROB slot + // that currently blocks k. Ordinary RAW dependencies disappear at complete; + // dependencies involving a mapping/config instruction disappear at commit. + // Keeping one union mask is sufficient because a config dependency is never + // allowed to clear earlier than commit, even when it also looks like a RAW. + val dependencyMask = queue.io.dependencies + + val isEmpty = queue.io.empty + val isFull = queue.io.full + val fullRequestBlocked = isFull && io.alloc.valid + val previousFullRequestBlocked = RegNext(fullRequestBlocked, false.B) + val itraceRobFull = io.traceInstructions(3) + itraceRobFull.is_issue := Mux(fullRequestBlocked, 3.U, 4.U) + itraceRobFull.thread_id := ownerId.U(64.W) + itraceRobFull.rob_id := tailPtr + itraceRobFull.inst_id := 0.U + itraceRobFull.domain_id := 0.U + itraceRobFull.funct := 0.U + itraceRobFull.pc := io.alloc.bits.operation.cmd.pc + itraceRobFull.rs1_data := 0.U + itraceRobFull.rs2_data := 0.U + itraceRobFull.enable := fullRequestBlocked =/= previousFullRequestBlocked + + // --------------------------------------------------------------------------- + // Allocate: enqueue decoded instruction into ROB + // rob_id == tailPtr at allocation time (no separate counter needed) + // --------------------------------------------------------------------------- + val commitMask = queue.io.retired.asBools + val staleValid = RegInit(VecInit(Seq.fill(robDepth)(0.U(2.W)))) + val staleBank = Reg(Vec(robDepth, Vec(2, UInt(frontendParam.bank_id_len.W)))) + val staleRaw = Reg(Vec(robDepth, UInt(frontendParam.bank_id_len.W))) + val newBank = Reg(Vec(robDepth, UInt(frontendParam.bank_id_len.W))) + + val retireSlots = VecInit((0 until robDepth).map { offset => + val sum = headPtr +& offset.U + Mux(sum >= robDepth.U, sum - robDepth.U, sum)(idWidth - 1, 0) + }) + + val pendingReclaim = VecInit(retireSlots.map(slot => queue.io.retireCandidates(slot) && staleValid(slot).orR)) + val reclaimSlot = retireSlots(PriorityEncoder(pendingReclaim)) + val reclaimIndex = PriorityEncoder(staleValid(reclaimSlot)) + val targetRaw = robEntries(reclaimSlot).cmd.bankAccess.wr_bank_id + mappingRetire.valid := pendingReclaim.asUInt.orR + mappingRetire.bits.bank := Mux(reclaimIndex === 0.U, staleRaw(reclaimSlot), targetRaw) + mappingRetire.bits.vbank := Mux(reclaimIndex === 0.U, newBank(reclaimSlot), targetRaw) + mappingRetire.bits.previous := staleBank(reclaimSlot)(reclaimIndex) + mappingRetire.bits.threadId := ownerId.U(64.W) + mappingRetire.bits.allocate := false.B + mappingRetire.bits.reuse := false.B + mappingRetire.bits.retire := true.B + mappingRetire.bits.write := false.B + when(mappingRetire.fire) { + staleValid(reclaimSlot) := staleValid(reclaimSlot) & ~UIntToOH(reclaimIndex, 2) + } + for (slot <- 0 until robDepth) { + queue.io.retireReady(slot) := !staleValid(slot).orR || + (slot.U === reclaimSlot && mappingRetire.ready && PopCount(staleValid(slot)) === 1.U) + when(commitMask(slot))(staleValid(slot) := 0.U) + } + io.bat.reclaim.valid := mappingRetire.fire + io.bat.reclaim.bits := mappingRetire.bits.previous + + val headNeedsDebug = + robEntries(headPtr).instId =/= 0.U && robEntries(headPtr).cmd.cmd.raw_inst(31, 25) =/= MSET_BITPAT && + robEntries(headPtr).cmd.bankAccess.wr_bank_valid && robEntries( + headPtr + ).cmd.bankAccess.wr_bank_id < memParam.virtualBankCount.U + + val hasCommit = queue.io.retired.orR + + val admit = io.bat.ready && !dependencies.io.movementLive + io.alloc.ready := queue.io.alloc.ready && admit + queue.io.alloc.valid := io.alloc.valid && admit + io.bat.alloc.valid := io.alloc.fire + io.bat.alloc.raw := io.alloc.bits.operation.bankAccess + io.bat.alloc.rename := frontendParam.bank_alias_enable.B && io.alloc.bits.operation.bankAccess.wr_bank_valid && + io.alloc.bits.operation.bankAccess.wr_bank_id < memParam.virtualBankCount.U && + io.alloc.bits.operation.cmd.raw_inst(31, 25) =/= MSET_BITPAT && !io.alloc.bits.operation.bankAccess.wr_update + io.bat.alloc.reset_read0 := io.alloc.bits.operation.cmd.raw_inst( + 31, + 25 + ) === MSET_BITPAT && io.alloc.bits.operation.cmd + .rs2Data(12) + + // Mark write alias as busy in scoreboard at alloc time (not issue time). + io.scoreboard.alloc.valid := io.alloc.fire && io.bat.alloc.rename + io.scoreboard.alloc.bits := io.bat.renamed + + val allocIsConfig = io.alloc.bits.operation.cmd.raw_inst(31, 25) === MSET_BITPAT + val allocUsesInstId = io.alloc.bits.operation.runtime + val allocation = Wire(new GlobalRobEntry(frontendParam, memParam, permissionParam)) + allocation.context := io.alloc.bits.context + allocation.cmd := io.alloc.bits.operation + allocation.renamedBankAccess := io.bat.renamed + allocation.rob_id := tailPtr + allocation.instId := Mux(allocUsesInstId, nextInstId, 0.U) + queue.io.alloc.bits.data := allocation.asUInt + queue.io.alloc.bits.dependencies := dependencies.io.dependencies + queue.io.alloc.bits.commitBarrier := allocIsConfig + + when(io.alloc.fire) { + itraceAlloc.is_issue := 2.U + itraceAlloc.rob_id := tailPtr + itraceAlloc.inst_id := allocation.instId + itraceAlloc.domain_id := io.alloc.bits.operation.domain_id + itraceAlloc.funct := io.alloc.bits.operation.cmd.raw_inst(31, 25) + itraceAlloc.pc := io.alloc.bits.operation.cmd.pc + itraceAlloc.rs1_data := io.alloc.bits.operation.cmd.rs1Data + itraceAlloc.rs2_data := io.alloc.bits.operation.cmd.rs2Data + itraceAlloc.enable := true.B + + allocatedBank(tailPtr) := io.bat.alloc.rename + staleValid(tailPtr) := VecInit(io.bat.stale.map(_.valid)).asUInt + staleBank(tailPtr) := VecInit(io.bat.stale.map(_.bits)) + staleRaw(tailPtr) := Mux( + io.bat.alloc.reset_read0, + io.alloc.bits.operation.bankAccess.rd_bank_0_id, + io.alloc.bits.operation.bankAccess.wr_bank_id + ) + newBank(tailPtr) := Mux( + io.bat.alloc.reset_read0, + io.alloc.bits.operation.bankAccess.rd_bank_0_id, + io.bat.renamed.wr_bank_id + ) + + when(allocUsesInstId) { + nextInstId := nextInstId + 1.U + } + } + + // --------------------------------------------------------------------------- + // Complete: mark entry as completed, release scoreboard resources + // --------------------------------------------------------------------------- + if (diffTest) { + val debug = io.debug.get + val incomingAccess = robEntries(io.complete.bits).cmd.bankAccess + val needsCheck = robEntries(io.complete.bits).instId =/= 0.U && + !(robEntries(io.complete.bits).cmd.domain_id === DomainId.MEM && + robEntries(io.complete.bits).cmd.cmd.raw_inst(31, 25) === MSET_BITPAT) && + incomingAccess.wr_bank_valid && incomingAccess.wr_bank_id < memParam.virtualBankCount.U + debug.check.valid := io.complete.valid && needsCheck + debug.check.bits.robId := io.complete.bits + debug.check.bits.bank := (if (memParam.bankLayoutEnable) + robEntries(io.complete.bits).renamedBankAccess.wr_bank_id + else incomingAccess.wr_bank_id) + io.complete.ready := !needsCheck || (debug.checked.valid && debug.checked.bits === io.complete.bits) + debug.checked.ready := io.complete.fire && needsCheck + } else { + io.complete.ready := true.B + } + + io.scoreboard.complete.valid := false.B + io.scoreboard.complete.bits := 0.U.asTypeOf(io.scoreboard.complete.bits) + + val issueFired = WireDefault(false.B) + queue.io.complete.valid := io.complete.fire + queue.io.complete.bits := io.complete.bits + + when(io.complete.fire) { + val cid = io.complete.bits + io.scoreboard.complete.valid := true.B + io.scoreboard.complete.bits := robEntries(cid).renamedBankAccess + + itraceComp.is_issue := 0.U + itraceComp.rob_id := cid + itraceComp.inst_id := robEntries(cid).instId + itraceComp.domain_id := robEntries(cid).cmd.domain_id + itraceComp.funct := robEntries(cid).cmd.cmd.raw_inst(31, 25) + itraceComp.pc := robEntries(cid).cmd.cmd.pc + itraceComp.rs1_data := robEntries(cid).cmd.cmd.rs1Data + itraceComp.rs2_data := robEntries(cid).cmd.cmd.rs2Data + itraceComp.enable := true.B + } + + // --------------------------------------------------------------------------- + // Issue: select the oldest unissued entry; a blocked entry stalls younger entries. + // --------------------------------------------------------------------------- + val scanValid = Wire(Vec(robDepth, Bool())) + val scanReady = Wire(Vec(robDepth, Bool())) + // Check each candidate against older ROB entries using mask intersections. + // The masks are precomputed once per slot, so this retains the original + // age-sensitive RAW/WAR/WAW semantics without replicating three equality + // comparators for every bank access pair. + for (i <- 0 until robDepth) { + val age = headPtr +& i.U + val ptr = Mux(age >= robDepth.U, age - robDepth.U, age)(idWidth - 1, 0) + scanValid(i) := robValid(ptr) && !robIssued(ptr) && !robComplete(ptr) + io.scoreboard.query(i) := robEntries(ptr).renamedBankAccess + scanReady(i) := scanValid(i) && !io.scoreboard.hazard(i) && !dependencyMask( + ptr + ).orR + } + + val firstReady = PriorityEncoder(scanValid.asUInt) + val hasReady = scanValid.asUInt.orR && scanReady(firstReady) + val issueAge = headPtr +& firstReady + val actualIssuePtr = Mux(issueAge >= robDepth.U, issueAge - robDepth.U, issueAge)(idWidth - 1, 0) + + val canIssue = hasReady + + val issueStageValid = RegInit(false.B) + val issueStageEntry = Reg(new GlobalRobEntry(frontendParam, memParam, permissionParam)) + val issueStageFire = issueStageValid && io.issue.ready + val issueStageReady = !issueStageValid || issueStageFire + + // Keep the wide ROB entry and bank-column mux one register boundary away + // from the age/scoreboard scan. The pointer stage can accept one candidate + // per cycle; the payload stage drains at one per cycle, so steady-state II + // remains one while the extra latency is hidden by in-flight commands. + val issuePtrValid = RegInit(false.B) + val issuePtr = Reg(UInt(idWidth.W)) + // The BLT is still installing the mapping this instruction issued; hold it. + val issuePtrFire = issuePtrValid && issueStageReady && !io.mapping.busy + val issuePtrReady = !issuePtrValid || issuePtrFire + mappingIssue.bits.bank := robEntries(actualIssuePtr).cmd.bankAccess.wr_bank_id + mappingIssue.bits.vbank := robEntries(actualIssuePtr).renamedBankAccess.wr_bank_id + val issueAccess = robEntries(actualIssuePtr).renamedBankAccess + val previousAlias = staleBank(actualIssuePtr)(0) + + val readsPrevious = (issueAccess.rd_bank_0_valid && issueAccess.rd_bank_0_id === previousAlias) || + (issueAccess.rd_bank_1_valid && issueAccess.rd_bank_1_id === previousAlias) || + (issueAccess.rd_wr_valid && issueAccess.rd_wr_id === previousAlias) + + // At the ROB head all older users have retired; younger users see the new alias. + mappingIssue.bits.previous := previousAlias + mappingIssue.bits.reuse := allocatedBank(actualIssuePtr) && actualIssuePtr === headPtr && !readsPrevious + mappingIssue.bits.allocate := allocatedBank(actualIssuePtr) + mappingIssue.bits.retire := false.B + mappingIssue.bits.write := robEntries(actualIssuePtr).cmd.bankAccess.wr_bank_valid && + !robEntries(actualIssuePtr).cmd.bankAccess.wr_update && + robEntries( + actualIssuePtr + ).cmd.bankAccess.wr_bank_id < memParam.virtualBankCount.U && + robEntries(actualIssuePtr).cmd.cmd.raw_inst(31, 25) =/= MSET_BITPAT + mappingIssue.bits.threadId := ownerId.U(64.W) + val needsMapping = mappingIssue.bits.allocate || mappingIssue.bits.write + mappingIssue.valid := canIssue && issuePtrReady && needsMapping + val issueLoad = canIssue && issuePtrReady && (!needsMapping || mappingIssue.ready) + val issuePayload = Wire(new GlobalRobEntry(frontendParam, memParam, permissionParam)) + issuePayload := robEntries(issuePtr) + + io.issue.valid := issueStageValid + io.issue.bits := issueStageEntry + + when(issuePtrReady) { + issuePtrValid := issueLoad + when(issueLoad) { + issuePtr := actualIssuePtr + } + } + when(issueStageReady) { + issueStageValid := issuePtrFire + when(issuePtrFire) { + issueStageEntry := issuePayload + } + } + + io.scoreboard.issue.valid := false.B + io.scoreboard.issue.bits := 0.U.asTypeOf(io.scoreboard.issue.bits) + + when(issueLoad) { + issueFired := true.B + io.scoreboard.issue.valid := true.B + io.scoreboard.issue.bits := robEntries(actualIssuePtr).renamedBankAccess + + itraceIssue.is_issue := 1.U + itraceIssue.rob_id := robEntries(actualIssuePtr).rob_id + itraceIssue.inst_id := robEntries(actualIssuePtr).instId + itraceIssue.domain_id := robEntries(actualIssuePtr).cmd.domain_id + itraceIssue.funct := robEntries(actualIssuePtr).cmd.cmd.raw_inst(31, 25) + itraceIssue.pc := robEntries(actualIssuePtr).cmd.cmd.pc + itraceIssue.rs1_data := robEntries(actualIssuePtr).cmd.cmd.rs1Data + itraceIssue.rs2_data := robEntries(actualIssuePtr).cmd.cmd.rs2Data + itraceIssue.enable := true.B + } + + queue.io.issue.valid := issueFired + queue.io.issue.bits := actualIssuePtr + + io.debug.foreach { debug => + debug.commit.valid := hasCommit && headNeedsDebug + debug.commit.bits.robId := headPtr + debug.commit.bits.instId := robEntries(headPtr).instId + debug.commit.bits.threadId := ownerId.U(64.W) + debug.commit.bits.bank := robEntries(headPtr).cmd.bankAccess.wr_bank_id + debug.idle := io.empty && !io.alloc.valid && !debug.commit.valid + } + + val mappingCommits = VecInit( + (0 until robDepth).map(i => + commitMask(i) && robEntries(i).cmd.domain_id === DomainId.MEM && + robEntries(i).cmd.cmd.raw_inst(31, 25) === MSET_BITPAT + ) + ) + + assert( + PopCount(mappingCommits) <= 1.U, + "Mapping dependency permits one configuration commit" + ) + // --------------------------------------------------------------------------- + // Status outputs + // --------------------------------------------------------------------------- + io.empty := isEmpty + io.full := isFull + +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/Rob.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/Rob.scala new file mode 100644 index 000000000..d0768fcdb --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/Rob.scala @@ -0,0 +1,123 @@ +package framework.acc.frontend.scheduler.rob.global + +import chisel3._ +import framework.acc.frontend.scheduler.rob.global.configs.RobParams +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} + +@instantiable +class Rob(val p: RobParams) extends Module { + + @public + val io = IO(new Bundle { + + val alloc = Flipped(Decoupled(new Bundle { + val data = UInt(p.dataBits.W) + val dependencies = UInt(p.entries.W) + // Dependencies involving this entry clear only at retirement. + val commitBarrier = Bool() + })) + + val issue = Flipped(Valid(UInt(p.idBits.W))) + val complete = Flipped(Valid(UInt(p.idBits.W))) + val data = Output(Vec(p.entries, UInt(p.dataBits.W))) + val valid = Output(Vec(p.entries, Bool())) + val issued = Output(Vec(p.entries, Bool())) + val completed = Output(Vec(p.entries, Bool())) + val dependencies = Output(Vec(p.entries, UInt(p.entries.W))) + val retireReady = Input(Vec(p.entries, Bool())) + val retireCandidates = Output(UInt(p.entries.W)) + val retired = Output(UInt(p.entries.W)) + val head = Output(UInt(p.idBits.W)) + val tail = Output(UInt(p.idBits.W)) + val issuedCount = Output(UInt(log2Ceil(p.entries + 1).W)) + val empty = Output(Bool()) + val full = Output(Bool()) + }) + + val data = RegInit(VecInit(Seq.fill(p.entries)(0.U(p.dataBits.W)))) + val valid = RegInit(VecInit(Seq.fill(p.entries)(false.B))) + val issued = RegInit(VecInit(Seq.fill(p.entries)(false.B))) + val completed = RegInit(VecInit(Seq.fill(p.entries)(false.B))) + val dependencies = RegInit(VecInit(Seq.fill(p.entries)(0.U(p.entries.W)))) + val barriers = RegInit(VecInit(Seq.fill(p.entries)(false.B))) + val head = RegInit(0.U(p.idBits.W)) + val tail = RegInit(0.U(p.idBits.W)) + val issuedCount = RegInit(0.U(log2Ceil(p.entries + 1).W)) + def wrap(value: UInt): UInt = Mux(value >= p.entries.U, value - p.entries.U, value)(p.idBits - 1, 0) + + val candidate = Wire(Vec(p.retireWidth, Bool())) + val eligible = Wire(Vec(p.retireWidth + 1, Bool())) + val retire = Wire(Vec(p.retireWidth, Bool())) + val keep = Wire(Vec(p.retireWidth + 1, Bool())) + eligible(0) := true.B + keep(0) := true.B + for (offset <- 0 until p.retireWidth) { + val index = wrap(head +& offset.U) + candidate(offset) := eligible(offset) && valid(index) && completed(index) + eligible(offset + 1) := candidate(offset) + retire(offset) := keep(offset) && valid(index) && completed(index) && io.retireReady(index) + keep(offset + 1) := retire(offset) + } + io.retireCandidates := VecInit((0 until p.entries).map { slot => + (0 until p.retireWidth).map(offset => candidate(offset) && wrap(head +& offset.U) === slot.U).reduce(_ || _) + }).asUInt + + val retired = VecInit((0 until p.entries).map { slot => + (0 until p.retireWidth).map(offset => retire(offset) && wrap(head +& offset.U) === slot.U).reduce(_ || _) + }).asUInt + + val completeMask = Mux(io.complete.valid, UIntToOH(io.complete.bits, p.entries), 0.U(p.entries.W)) + io.empty := head === tail && !valid(head) + io.full := head === tail && valid(head) + // Preserve the integrated ROB's commit-cycle admission rule. + io.alloc.ready := !io.full && !retired.orR + + when(io.alloc.fire) { + data(tail) := io.alloc.bits.data + valid(tail) := true.B + issued(tail) := false.B + completed(tail) := false.B + barriers(tail) := io.alloc.bits.commitBarrier + tail := wrap(tail +& 1.U) + } + when(io.issue.valid) { + assert( + io.issue.bits < p.entries.U && valid(io.issue.bits) && !issued(io.issue.bits) && !completed(io.issue.bits), + "ROB issue must name a live unissued entry" + ) + issued(io.issue.bits) := true.B + } + when(io.complete.valid) { + assert( + io.complete.bits < p.entries.U && valid(io.complete.bits) && !completed(io.complete.bits), + "ROB completion must name a live incomplete entry" + ) + completed(io.complete.bits) := true.B + } + val completingIssued = io.complete.valid && + (issued(io.complete.bits) || (io.issue.valid && io.issue.bits === io.complete.bits)) + issuedCount := issuedCount + io.issue.valid.asUInt - completingIssued.asUInt + for (slot <- 0 until p.entries) { + when(retired(slot)) { + valid(slot) := false.B + issued(slot) := false.B + completed(slot) := false.B + } + val allocating = io.alloc.fire && tail === slot.U + val barrier = Mux(allocating, io.alloc.bits.commitBarrier, barriers(slot)) + val sticky = barriers.asUInt | Fill(p.entries, barrier) + val clear = retired | (completeMask & ~sticky) + dependencies(slot) := Mux(allocating, io.alloc.bits.dependencies, dependencies(slot)) & ~clear + } + head := wrap(head +& PopCount(retire)) + io.data := data + io.valid := valid + io.issued := issued + io.completed := completed + io.dependencies := dependencies + io.retired := retired + io.head := head + io.tail := tail + io.issuedCount := issuedCount +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/configs/RobParams.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/configs/RobParams.scala new file mode 100644 index 000000000..efefb7e22 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/configs/RobParams.scala @@ -0,0 +1,11 @@ +package framework.acc.frontend.scheduler.rob.global.configs + +import chisel3.util.log2Ceil +import upickle.default.{macroRW, ReadWriter} + +case class RobParams(entries: Int, dataBits: Int, retireWidth: Int) { + require(entries > 1 && dataBits > 0 && retireWidth > 0 && retireWidth <= entries) + val idBits = log2Ceil(entries) +} + +object RobParams { implicit val rw: ReadWriter[RobParams] = macroRW } diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/build.sc b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/build.sc new file mode 100644 index 000000000..0c9885b37 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/build.sc @@ -0,0 +1,22 @@ +import mill._ +import $file.^.^.^.^.^.^.^.^.^.^.^.^.^.^.settings +val ipRoot = millSourcePath + +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.acc.frontend.scheduler.scoreboard.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/BankAccessInfo.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/BankScoreboard.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/configs/uvm.toml b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/configs/uvm.toml new file mode 100644 index 000000000..9e69ca533 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/configs/uvm.toml @@ -0,0 +1,6 @@ +[parameters] +bankIdBits = 4 +banks = 12 +entries = 4 + +[defines] diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/agent/driver.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/agent/driver.svh new file mode 100644 index 000000000..d549e9643 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/agent/driver.svh @@ -0,0 +1,25 @@ +class driver extends uvm_driver #(cycle_item); + `uvm_component_utils(driver) + virtual bank_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual bank_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Bank scheduling interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item req; + forever begin + seq_item_port.get_next_item(req); + @(negedge vif.clock); + vif.reset = req.reset; + vif.inputs = req.inputs; + @(posedge vif.clock); + req.outputs = vif.outputs; + #1; + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/agent/item.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/agent/item.svh new file mode 100644 index 000000000..0649002f2 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/agent/item.svh @@ -0,0 +1,18 @@ +class cycle_item extends uvm_sequence_item; + `uvm_object_utils(cycle_item) + logic [`SB_INPUT_BITS-1:0] inputs; + logic [`SB_OUTPUT_BITS-1:0] outputs; + bit reset; + function new(string name = "cycle_item"); + super.new(name); + endfunction +endclass +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(payload, UVM_ALL_ON) + `uvm_object_utils_end + logic [`SB_OUTPUT_BITS-1:0] payload; + function new(string name = "result_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/agent/monitor.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/agent/monitor.svh new file mode 100644 index 000000000..dc93b2fb6 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/agent/monitor.svh @@ -0,0 +1,25 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual bank_if vif; + uvm_analysis_port #(cycle_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual bank_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Bank scheduling monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item t; + forever begin + @(posedge vif.clock); + t = cycle_item::type_id::create("observed"); + t.inputs = vif.inputs; + t.outputs = vif.outputs; + t.reset = vif.reset; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/env/env.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/env/env.svh new file mode 100644 index 000000000..ed95874fb --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/env/env.svh @@ -0,0 +1,25 @@ +class env extends uvm_env; + `uvm_component_utils(env) + driver drv; + monitor mon; + model reference; + uvm_sequencer #(cycle_item) seqr; + result_board #(result_item, 0) scoreboard; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + drv = driver::type_id::create("driver", this); + mon = monitor::type_id::create("monitor", this); + reference = model::type_id::create("model", this); + seqr = new("seqr", this); + scoreboard = result_board#(result_item, 0)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + drv.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(reference.analysis_export); + reference.expected_ap.connect(scoreboard.expected_export); + reference.actual_ap.connect(scoreboard.actual_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/env/model.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/env/model.svh new file mode 100644 index 000000000..55253e2b4 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/env/model.svh @@ -0,0 +1,57 @@ +class model extends uvm_subscriber #(cycle_item); + `uvm_component_utils(model) + bit busy[`SB_BANKS]; + int query_offsets[`SB_ENTRIES][6] = `SB_QUERY_OFFSETS; + uvm_analysis_port #(result_item) expected_ap, actual_ap; + covergroup behavior with function sample ( + int bank, bit allocated, bit issued, bit hazard, int read_mask + ); + option.per_instance = 1; + allocated_bank: coverpoint bank iff (allocated) {bins aliases[] = {[0 : `SB_BANKS - 1]};} + issued_bank: coverpoint bank iff (issued) {bins aliases[] = {[0 : `SB_BANKS - 1]};} + read_ports: coverpoint read_mask {bins combinations[] = {[0 : 7]};} + blocked: coverpoint hazard; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected_ap = new("expected_ap", this); + actual_ap = new("actual_ap", this); + behavior = new(); + endfunction + function void write(cycle_item t); + result_item expected, actual; + if (t.reset) begin + foreach (busy[b]) busy[b] = 0; + return; + end + expected = result_item::type_id::create("expected"); + actual = result_item::type_id::create("actual"); + expected.payload = '0; + actual.payload = t.outputs; + for (int lane = 0; lane < `SB_ENTRIES; lane++) begin + bit hazard = 0; + int read_mask = 0; + for (int port = 0; port < 3; port++) begin + bit enabled = t.inputs[query_offsets[lane][port*2]]; + int bank = t.inputs[query_offsets[lane][port*2+1]+:`SB_BANKIDBITS]; + if (enabled) begin + hazard |= busy[bank]; + read_mask |= 1 << port; + end + end + expected.payload[lane] = hazard; + behavior.sample(`BI(t.inputs, ALLOC_BITS_WR_BANK_ID), `BI(t.inputs, ALLOC_VALID) && + `BI(t.inputs, ALLOC_BITS_WR_BANK_VALID), 0, hazard, read_mask); + behavior.sample(`BI(t.inputs, ISSUE_BITS_WR_BANK_ID), 0, `BI(t.inputs, ISSUE_VALID) && + `BI(t.inputs, ISSUE_BITS_WR_BANK_VALID), hazard, read_mask); + end + expected_ap.write(expected); + actual_ap.write(actual); + if (`BI(t.inputs, ALLOC_VALID) && `BI(t.inputs, ALLOC_BITS_WR_BANK_VALID)) + busy[`BI(t.inputs, ALLOC_BITS_WR_BANK_ID)] = 1; + if (`BI(t.inputs, ISSUE_VALID) && `BI(t.inputs, ISSUE_BITS_WR_BANK_VALID)) + busy[`BI(t.inputs, ISSUE_BITS_WR_BANK_ID)] = 1; + if (`BI(t.inputs, COMPLETE_VALID) && `BI(t.inputs, COMPLETE_BITS_WR_BANK_VALID)) + busy[`BI(t.inputs, COMPLETE_BITS_WR_BANK_ID)] = 0; + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/filelist.f b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/filelist.f new file mode 100644 index 000000000..4553acc1c --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/BankScoreboard/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/interface.sv b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/interface.sv new file mode 100644 index 000000000..90a5a959a --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/interface.sv @@ -0,0 +1,8 @@ +`include "parameters.svh" +interface bank_if ( + input logic clock +); + logic reset = 1; + logic [`SB_INPUT_BITS-1:0] inputs; + logic [`SB_OUTPUT_BITS-1:0] outputs; +endinterface diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/package.sv b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/package.sv new file mode 100644 index 000000000..dd191ab1a --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/package.sv @@ -0,0 +1,15 @@ +`include "parameters.svh" +`define BI(P, N) P[`SB_IN_``N``_OFFSET +: `SB_IN_``N``_WIDTH] +`define BO(P, N) P[`SB_OUT_``N``_OFFSET +: `SB_OUT_``N``_WIDTH] +package scoreboard_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..9126bcf84 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/seq/sequence.svh @@ -0,0 +1,95 @@ +class scenario_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(scenario_sequence) + logic [`SB_INPUT_BITS-1:0] pins = '0; + bit reset = 1; + int query_offsets[`SB_ENTRIES][6] = `SB_QUERY_OFFSETS; + function new(string name = "scenario_sequence"); + super.new(name); + endfunction + task tick(int cycles = 1); + cycle_item req; + repeat (cycles) begin + req = cycle_item::type_id::create("cycle"); + start_item(req); + req.inputs = pins; + req.reset = reset; + finish_item(req); + end + endtask + task queries(int mask, int bank, int alternate); + for (int lane = 0; lane < `SB_ENTRIES; lane++) + for (int port = 0; port < 3; port++) begin + pins[query_offsets[lane][port*2]] = (mask >> port) & 1; + pins[query_offsets[lane][port*2+1]+:`SB_BANKIDBITS] = (lane+port)%2 == 0 ? bank : alternate; + end + endtask + task body(); + tick(2); + reset = 0; + for (int bank = 0; bank < `SB_BANKS; bank++) begin + `BI(pins, ALLOC_VALID) = 1; + `BI(pins, ALLOC_BITS_WR_BANK_VALID) = 1; + `BI(pins, ALLOC_BITS_WR_BANK_ID) = bank; + tick(); + `BI(pins, ALLOC_VALID) = 0; + for (int mask = 0; mask < 8; mask++) + for (int alternate = 0; alternate < `SB_BANKS; alternate++) begin + queries(mask, bank, alternate); + tick(); + end + `BI(pins, COMPLETE_VALID) = 1; + `BI(pins, COMPLETE_BITS_WR_BANK_VALID) = 1; + `BI(pins, COMPLETE_BITS_WR_BANK_ID) = bank; + tick(); + `BI(pins, COMPLETE_VALID) = 0; + tick(); + `BI(pins, ISSUE_VALID) = 1; + `BI(pins, ISSUE_BITS_WR_BANK_VALID) = 1; + `BI(pins, ISSUE_BITS_WR_BANK_ID) = bank; + tick(); + `BI(pins, ISSUE_VALID) = 0; + queries(7, bank, (bank + 1) % `SB_BANKS); + tick(); + // Retirement wins when the same alias is presented by issue and complete together. + `BI(pins, ISSUE_VALID) = 1; + `BI(pins, COMPLETE_VALID) = 1; + tick(); + `BI(pins, ISSUE_VALID) = 0; + `BI(pins, COMPLETE_VALID) = 0; + tick(); + end + // Read/read sharing never creates a write dependency. + `BI(pins, ISSUE_BITS_WR_BANK_VALID) = 0; + `BI(pins, COMPLETE_BITS_WR_BANK_VALID) = 0; + for (int bank = 0; bank < `SB_BANKS; bank++) begin + `BI(pins, ISSUE_VALID) = 1; + `BI(pins, ISSUE_BITS_RD_BANK_0_VALID) = 1; + `BI(pins, ISSUE_BITS_RD_BANK_0_ID) = bank; + `BI(pins, ISSUE_BITS_RD_BANK_1_VALID) = 1; + `BI(pins, ISSUE_BITS_RD_BANK_1_ID) = bank; + `BI(pins, ISSUE_BITS_RD_WR_VALID) = 1; + `BI(pins, ISSUE_BITS_RD_WR_ID) = bank; + queries(7, bank, bank); + tick(`SB_ENTRIES); + `BI(pins, ISSUE_VALID) = 0; + `BI(pins, COMPLETE_VALID) = 1; + `BI(pins, COMPLETE_BITS_RD_BANK_0_VALID) = 1; + `BI(pins, COMPLETE_BITS_RD_BANK_0_ID) = bank; + `BI(pins, COMPLETE_BITS_RD_BANK_1_VALID) = 1; + `BI(pins, COMPLETE_BITS_RD_BANK_1_ID) = bank; + `BI(pins, COMPLETE_BITS_RD_WR_VALID) = 1; + `BI(pins, COMPLETE_BITS_RD_WR_ID) = bank; + tick(`SB_ENTRIES); + `BI(pins, COMPLETE_VALID) = 0; + end + `BI(pins, ALLOC_VALID) = 1; + `BI(pins, ALLOC_BITS_WR_BANK_VALID) = 1; + tick(); + reset = 1; + tick(2); + pins = '0; + reset = 0; + queries(7, 0, `SB_BANKS - 1); + tick(2); + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/suite.toml b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/suite.toml new file mode 100644 index 000000000..e022b231e --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/suite.toml @@ -0,0 +1,20 @@ +[model] +manifest = "" +library = "" + +[[tests]] +name = "scoreboard" +top = "scoreboard_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 300000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +functional_groups = ["scoreboard_pkg::model::behavior"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/tb.sv b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/tb.sv new file mode 100644 index 000000000..a766519fb --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/tb.sv @@ -0,0 +1,16 @@ +module scoreboard_tb; + import uvm_pkg::*; + import scoreboard_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + bank_if bus (clock); + BankScoreboard dut ( + .clock(clock), + .reset(bus.reset), + `include "ports.svh" + ); + initial begin + uvm_config_db#(virtual bank_if)::set(null, "uvm_test_top*", "vif", bus); + run_test(); + end +endmodule diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/tests/test.svh new file mode 100644 index 000000000..54d137a6f --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/tests/test.svh @@ -0,0 +1,15 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.start(test_env.seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/frontend/scoreboard/BankAccessInfo.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/scala/BankAccessInfo.scala similarity index 78% rename from arch/src/main/scala/framework/frontend/scoreboard/BankAccessInfo.scala rename to arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/scala/BankAccessInfo.scala index 8b3b2f2c1..f089ef508 100644 --- a/arch/src/main/scala/framework/frontend/scoreboard/BankAccessInfo.scala +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/scala/BankAccessInfo.scala @@ -1,4 +1,4 @@ -package framework.frontend.scoreboard +package framework.acc.frontend.scheduler.scoreboard import chisel3._ @@ -7,12 +7,16 @@ import chisel3._ // It's instruction-agnostic: the scoreboard only sees read/write bank_id. //====----------------------------------------------------------====// class BankAccessInfo(val bankIdLen: Int) extends Bundle { + val rd_bank_0_valid = Bool() val rd_bank_0_id = UInt(bankIdLen.W) val rd_bank_1_valid = Bool() val rd_bank_1_id = UInt(bankIdLen.W) + val rd_wr_valid = Bool() + val rd_wr_id = UInt(bankIdLen.W) val wr_bank_valid = Bool() val wr_bank_id = UInt(bankIdLen.W) + val wr_update = Bool() } object BankAccessInfo { @@ -23,8 +27,11 @@ object BankAccessInfo { w.rd_bank_0_id := 0.U w.rd_bank_1_valid := false.B w.rd_bank_1_id := 0.U + w.rd_wr_valid := false.B + w.rd_wr_id := 0.U w.wr_bank_valid := false.B w.wr_bank_id := 0.U + w.wr_update := false.B w } diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/scala/BankScoreboard.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/scala/BankScoreboard.scala new file mode 100644 index 000000000..567350e8b --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/scala/BankScoreboard.scala @@ -0,0 +1,113 @@ +package framework.acc.frontend.scheduler.scoreboard + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.acc.frontend.scheduler.scoreboard.configs.Params + +//====----------------------------------------------------------====// +// Bank Scoreboard: tracks in-flight read/write operations per bank. +// +// Hazard rules: +// - Read bank X → requires bankWrBusy(X) == false (RAW) +// - Write bank X → requires bankRdCount(X) == 0 (WAR) +// AND bankWrBusy(X) == false (WAW) +// +// bankRdCount: multi-bit counter (multiple concurrent readers allowed, RR is OK) +// bankWrBusy: 1-bit flag (WAW rule guarantees at most 1 writer in-flight) +// +// Lifecycle: +// - alloc: sets bankWrBusy for the renamed write alias. This must happen +// at alloc time (not issue) because the BAT updates v2a at alloc, +// and subsequent reads of the same vbank will be renamed to the +// new alias immediately. If we only set bankWrBusy at issue, the +// window between alloc and issue lets dependent reads slip through +// the hazard check and issue out-of-order. +// - issue: increments bankRdCount for the renamed read aliases. +// - complete: decrements bankRdCount and clears bankWrBusy. +// +// Issue and complete may fire in the same cycle. Updates are computed +// combinationally so both increments and decrements take effect. +//====----------------------------------------------------------====// + +@instantiable +class BankScoreboard(val p: Params) extends Module { + + val bankIdLen = p.bankIdBits + val bankNum = p.banks + val robEntries = p.entries + + val cntWidth = log2Ceil(3 * robEntries + 1) + + @public + val io = IO(new Bundle { + val alloc = Flipped(Valid(new BankAccessInfo(p.bankIdBits))) + val issue = Flipped(Valid(new BankAccessInfo(p.bankIdBits))) + val complete = Flipped(Valid(new BankAccessInfo(p.bankIdBits))) + + val queryVec = Input(Vec( + robEntries, + new BankAccessInfo(p.bankIdBits) + )) + + val hazardVec = Output(Vec(robEntries, Bool())) + }) + + val bankRdCount = RegInit(VecInit(Seq.fill(bankNum)(0.U(cntWidth.W)))) + val bankWrBusy = RegInit(VecInit(Seq.fill(bankNum)(false.B))) + def bankIndex(id: UInt): UInt = id(log2Ceil(bankNum) - 1, 0) + + // --- Hazard detection (reads current register state) --- + def hazardOf(q: BankAccessInfo): Bool = { + val rd0_hazard = q.rd_bank_0_valid && bankWrBusy(bankIndex(q.rd_bank_0_id)) + val rd1_hazard = q.rd_bank_1_valid && bankWrBusy(bankIndex(q.rd_bank_1_id)) + // Write hazard is NOT checked here. BAT eliminates WAW/WAR by giving each + // write a unique alias that no other instruction uses. Checking wr_hazard + // would falsely block the writer itself, since io.alloc marks the writer's + // own alias as busy in the same cycle the entry enters the ROB. + val rdWrHazard = q.rd_wr_valid && bankWrBusy(bankIndex(q.rd_wr_id)) + rd0_hazard || rd1_hazard || rdWrHazard + } + + for (i <- 0 until robEntries) { + io.hazardVec(i) := hazardOf(io.queryVec(i)) + } + + // --- Compute per-bank deltas to handle simultaneous io.issue + io.complete --- + for (bank <- 0 until bankNum) { + val bankU = bank.U(bankIdLen.W) + + // Read counter: +1 per io.issue read, -1 per io.complete read + val issRd0 = + io.issue.valid && io.issue.bits.rd_bank_0_valid && (io.issue.bits.rd_bank_0_id === bankU) + val issRd1 = + io.issue.valid && io.issue.bits.rd_bank_1_valid && (io.issue.bits.rd_bank_1_id === bankU) + val cmpRd0 = + io.complete.valid && io.complete.bits.rd_bank_0_valid && (io.complete.bits.rd_bank_0_id === bankU) + val cmpRd1 = + io.complete.valid && io.complete.bits.rd_bank_1_valid && (io.complete.bits.rd_bank_1_id === bankU) + val issRdWr = + io.issue.valid && io.issue.bits.rd_wr_valid && io.issue.bits.rd_wr_id === bankU + val cmpRdWr = + io.complete.valid && io.complete.bits.rd_wr_valid && io.complete.bits.rd_wr_id === bankU + + val rdInc = issRd0.asUInt +& issRd1.asUInt +& issRdWr.asUInt + val rdDec = cmpRd0.asUInt +& cmpRd1.asUInt +& cmpRdWr.asUInt + bankRdCount(bank) := bankRdCount(bank) + rdInc - rdDec + + // Write flag: io.alloc sets (alias is reserved as soon as it is renamed), + // io.complete clears. If both happen to same bank, io.complete takes priority. + val allocWr = + io.alloc.valid && io.alloc.bits.wr_bank_valid && (io.alloc.bits.wr_bank_id === bankU) + val issueWr = + io.issue.valid && io.issue.bits.wr_bank_valid && (io.issue.bits.wr_bank_id === bankU) + val cmpWr = + io.complete.valid && io.complete.bits.wr_bank_valid && (io.complete.bits.wr_bank_id === bankU) + + when((allocWr || issueWr) && !cmpWr) { + bankWrBusy(bank) := true.B + }.elsewhen(cmpWr) { + bankWrBusy(bank) := false.B + } + } +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/scala/Emit.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/scala/Emit.scala new file mode 100644 index 000000000..2dfd516b2 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/scala/Emit.scala @@ -0,0 +1,61 @@ +package framework.acc.frontend.scheduler.scoreboard + +import chisel3._ +import java.nio.file.{Files, Paths} +import framework.acc.frontend.scheduler.scoreboard.configs.Params + +object Emit extends App { + val Array("--config", config, "--output-dir", output) = args + val p = upickle.default.read[Params](Files.readString(Paths.get(config))) + Files.createDirectories(Paths.get(output)) + + def fields(data: Data, path: String): Seq[(String, Int)] = data match { + case record: Record => record.elements.toSeq.flatMap { case (name, field) => + fields(field, if (path.isEmpty) name else s"${path}_$name") + } + case vector: Vec[_] => vector.toSeq.zipWithIndex.flatMap { case (field, index) => fields(field, s"${path}_$index") } + case leaf => Seq(path -> leaf.getWidth) + } + + val inputRoots = Set("alloc", "issue", "complete", "queryVec") + val ports = scala.collection.mutable.ArrayBuffer[String]() + val definitions = scala.collection.mutable.ArrayBuffer[String]() + val offsets = scala.collection.mutable.Map[String, Int]() + var inputOffset = 0 + var outputOffset = 0 + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + { + val dut = new BankScoreboard(p) + for { + (root, bundle) <- dut.io.elements.toSeq + (name, width) <- fields(bundle, root) + } { + val input = inputRoots.contains(root) + val direction = if (input) "IN" else "OUT" + val signal = if (input) "inputs" else "outputs" + val offset = if (input) inputOffset else outputOffset + definitions += s"`define SB_${direction}_${name.toUpperCase}_OFFSET $offset" + definitions += s"`define SB_${direction}_${name.toUpperCase}_WIDTH $width" + ports += s".io_$name(bus.$signal[$offset +: $width])" + offsets(name) = offset + if (input) inputOffset += width else outputOffset += width + } + dut + }, + args = Array("--target-dir", s"$output/BankScoreboard", "--split-verilog") + ) + definitions += s"`define SB_INPUT_BITS $inputOffset" + definitions += s"`define SB_OUTPUT_BITS $outputOffset" + definitions += s"`define SB_BANKIDBITS ${p.bankIdBits}" + definitions += s"`define SB_BANKS ${p.banks}" + definitions += s"`define SB_ENTRIES ${p.entries}" + + val queryOffsets = (0 until p.entries).map { index => + Seq("rd_bank_0_valid", "rd_bank_0_id", "rd_bank_1_valid", "rd_bank_1_id", "rd_wr_valid", "rd_wr_id") + .map(name => offsets(s"queryVec_${index}_$name")).mkString("'{", ",", "}") + }.mkString("'{", ",", "}") + + definitions += s"`define SB_QUERY_OFFSETS $queryOffsets" + Files.writeString(Paths.get(output, "parameters.svh"), definitions.mkString("\n") + "\n") + Files.writeString(Paths.get(output, "ports.svh"), ports.mkString(",\n") + "\n") +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..e1115b533 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/scala/configs/Params.scala @@ -0,0 +1,10 @@ +package framework.acc.frontend.scheduler.scoreboard.configs + +import upickle.default.{macroRW, ReadWriter} + +case class Params(bankIdBits: Int, banks: Int, entries: Int) { + require(bankIdBits > 0 && banks > 1 && banks <= (BigInt(1) << bankIdBits)) + require(entries > 1) +} + +object Params { implicit val rw: ReadWriter[Params] = macroRW } diff --git a/arch/src/main/scala/framework/rvv/.gitignore b/arch/src/main/scala/framework/acc/gpdomain/.gitignore similarity index 100% rename from arch/src/main/scala/framework/rvv/.gitignore rename to arch/src/main/scala/framework/acc/gpdomain/.gitignore diff --git a/arch/src/main/scala/framework/mem-core/LICENSE b/arch/src/main/scala/framework/acc/gpdomain/LICENSE similarity index 100% rename from arch/src/main/scala/framework/mem-core/LICENSE rename to arch/src/main/scala/framework/acc/gpdomain/LICENSE diff --git a/arch/src/main/scala/framework/acc/gpdomain/README.md b/arch/src/main/scala/framework/acc/gpdomain/README.md new file mode 100644 index 000000000..17b8d47b0 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/README.md @@ -0,0 +1 @@ +# gpdomain diff --git a/arch/src/main/scala/framework/acc/gpdomain/build.sc b/arch/src/main/scala/framework/acc/gpdomain/build.sc new file mode 100644 index 000000000..88250a654 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/build.sc @@ -0,0 +1,33 @@ +import mill._ +import $file.src.main.scala.execution.{build => execution_build} +import $file.src.main.scala.loader.{build => loader_build} +import $file.src.main.scala.memory.{build => memory_build} +import $file.^.^.^.^.^.^.settings + +val ipRoot: os.Path = millSourcePath + +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.acc.gpdomain.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/GpDomain.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/GpDomainParam.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/decoder/DISA.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/decoder/Decoder.scala")) + ) ++ + execution_build.sources() ++ + loader_build.sources() ++ + memory_build.sources() +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) ++ + execution_build.emitSources() ++ + loader_build.emitSources() ++ + memory_build.emitSources() +) diff --git a/examples/cores/pebble/configs/rvv/default.toml b/arch/src/main/scala/framework/acc/gpdomain/configs/default.toml similarity index 100% rename from examples/cores/pebble/configs/rvv/default.toml rename to arch/src/main/scala/framework/acc/gpdomain/configs/default.toml diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/csrc/Cargo.lock b/arch/src/main/scala/framework/acc/gpdomain/src/csrc/Cargo.lock new file mode 100644 index 000000000..f7cc692c4 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/csrc/Cargo.lock @@ -0,0 +1,128 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rustc_apfloat" +version = "0.2.3+llvm-462a31f5a5ab" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "486c2179b4796f65bfe2ee33679acf0927ac83ecf583ad6c91c3b4570911b9ad" +dependencies = [ + "bitflags", + "smallvec", +] + +[[package]] +name = "rvv-reference-model" +version = "0.1.0" +dependencies = [ + "rustc_apfloat", + "serde_json", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "smallvec" +version = "1.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e" + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/csrc/Cargo.toml b/arch/src/main/scala/framework/acc/gpdomain/src/csrc/Cargo.toml new file mode 100644 index 000000000..dd0f78908 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/csrc/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "rvv-reference-model" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["cdylib", "staticlib"] + +[dependencies] +rustc_apfloat = "0.2.3" + +[build-dependencies] +serde_json = "1" + +[workspace] diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/csrc/build.rs b/arch/src/main/scala/framework/acc/gpdomain/src/csrc/build.rs new file mode 100644 index 000000000..1d7d1e4ce --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/csrc/build.rs @@ -0,0 +1,64 @@ +use std::{env, fs, path::PathBuf, process::Command}; +fn main() { + let root = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()) + .ancestors() + .nth(12) + .unwrap() + .to_path_buf(); + let ip = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()).parent().unwrap().parent().unwrap().to_path_buf(); + let parameters = ip.join("src/main/resources/gpdomain.json"); + println!("cargo:rerun-if-changed={}", parameters.display()); + let config: serde_json::Value = serde_json::from_str(&fs::read_to_string(parameters).unwrap()).unwrap(); + let value = |key: &str| config[key].as_u64().unwrap() as usize; + let constants = format!("const VLEN: usize = {};\nconst ELEN: usize = {};\nconst IBUF_BYTES: usize = {};\n", + value("vLen"), value("eLen"), value("iBufWords") * 4); + fs::write(PathBuf::from(env::var_os("OUT_DIR").unwrap()).join("params.rs"), constants).unwrap(); + let rtl = ip.join("build/KernelEngine"); + fs::create_dir_all(&rtl).unwrap(); + fs::write(rtl.join("rvv_config.svh"), format!("`ifndef RVV_MEMORY_PORTS\n`define RVV_MEMORY_PORTS {}\n`endif\n", value("memoryPorts"))).unwrap(); + let mut connections = String::new(); + for port in 0..value("memoryPorts") { + for (direction, fields) in [ + ("Read", vec![("bank_id", "read_bank"), ("group_id", "read_group"), ("io_req_valid", "read_valid"), ("io_req_ready", "read_ready"), ("io_req_bits_addr", "read_row"), ("io_resp_valid", "read_response_valid"), ("io_resp_ready", "read_response_ready"), ("io_resp_bits_data", "read_data")]), + ("Write", vec![("bank_id", "write_bank"), ("group_id", "write_group"), ("io_req_valid", "write_valid"), ("io_req_ready", "write_ready"), ("io_req_bits_addr", "write_row"), ("io_resp_valid", "write_response_valid"), ("io_resp_ready", "write_response_ready"), ("io_req_bits_data", "write_data"), ("io_resp_bits_ok", "write_ok")]), + ] { + for (field, signal) in fields { + connections += &format!(".io_bank{direction}_{port}_{field}(vif.{signal}[{port}]),\n"); + } + } + for bit in 0..16 { + connections += &format!(".io_bankWrite_{port}_io_req_bits_mask_{bit}(vif.write_mask[{port}][{bit}]),\n"); + } + } + connections.truncate(connections.trim_end().len() - 1); + fs::write(rtl.join("rvv_ports.svh"), connections).unwrap(); + let source = root.join("stack/kernels/rvv"); + for name in [ + "CMakeLists.txt", + "kernel.ld", + "kernels.h", + "silu.cpp", + "snake.cpp", + "quant.cpp", + "math/exp.cpp", + "math/sin.cpp", + "math/cos.cpp", + "math/erf.cpp", + "rope.cpp", + "trig.cpp", + "math/math.h", + "math/constants.h", + ] { + println!("cargo:rerun-if-changed={}", source.join(name).display()); + } + let output = PathBuf::from(env::var_os("OUT_DIR").unwrap()).join("kernels"); + assert!(Command::new("cmake") + .args(["-G", "Ninja", "-S"]) + .arg(&source) + .arg("-B") + .arg(&output) + .status() + .unwrap() + .success()); + assert!(Command::new("cmake").arg("--build").arg(&output).status().unwrap().success()); +} diff --git a/arch/src/main/scala/framework/rvv/src/csrc/src/ffi.rs b/arch/src/main/scala/framework/acc/gpdomain/src/csrc/src/ffi.rs similarity index 99% rename from arch/src/main/scala/framework/rvv/src/csrc/src/ffi.rs rename to arch/src/main/scala/framework/acc/gpdomain/src/csrc/src/ffi.rs index 946c82f5f..37f556787 100644 --- a/arch/src/main/scala/framework/rvv/src/csrc/src/ffi.rs +++ b/arch/src/main/scala/framework/acc/gpdomain/src/csrc/src/ffi.rs @@ -1,5 +1,5 @@ use crate::model::{Model, BANKS, BANK_BYTES}; -use rvv::Memory; +use crate::rvv::Memory; use std::{ path::Path, sync::{Mutex, OnceLock}, diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/csrc/src/lib.rs b/arch/src/main/scala/framework/acc/gpdomain/src/csrc/src/lib.rs new file mode 100644 index 000000000..aaba4be55 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/csrc/src/lib.rs @@ -0,0 +1,4 @@ +#[path = "../../../../../../../../../../bebop/src/nodes/bemu/src/thread/kernels/mod.rs"] +mod rvv; +mod ffi; +mod model; diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/csrc/src/model.rs b/arch/src/main/scala/framework/acc/gpdomain/src/csrc/src/model.rs new file mode 100644 index 000000000..699f809a5 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/csrc/src/model.rs @@ -0,0 +1,903 @@ +include!(concat!(env!("OUT_DIR"), "/params.rs")); + +use crate::rvv::{Engine, Fault, Memory, MemoryError}; +use std::path::Path; + +pub const BANK_BYTES: usize = 4096; +pub const BANKS: usize = 6; +#[derive(Clone)] +pub struct Banks(pub Vec); +impl Banks { + fn range(&self, address: u64, bytes: usize) -> Result, MemoryError> { + let bank = (address >> 32) as usize; + let offset = (address & 0xffff_ffff) as usize; + if bank >= BANKS || offset + bytes > BANK_BYTES { + return Err(MemoryError); + } + Ok(bank * BANK_BYTES + offset..bank * BANK_BYTES + offset + bytes) + } + fn put(&mut self, bank: usize, offset: usize, data: &[u8]) { + self.0[bank * BANK_BYTES + offset..bank * BANK_BYTES + offset + data.len()] + .copy_from_slice(data); + } +} +impl Memory for Banks { + fn ball_command(&mut self, _: u32, _: u64, _: u64) -> Result { + Err(MemoryError) + } + fn read(&mut self, address: u64, bytes: usize) -> Result { + let mut value = [0u8; 8]; + value[..bytes].copy_from_slice(&self.0[self.range(address, bytes)?]); + Ok(u64::from_le_bytes(value)) + } + fn write(&mut self, address: u64, bytes: usize, value: u64) -> Result<(), MemoryError> { + let range = self.range(address, bytes)?; + self.0[range].copy_from_slice(&value.to_le_bytes()[..bytes]); + Ok(()) + } +} +pub struct Case { + pub name: String, + pub program: Vec, + pub constants: Vec, + pub entry: u32, + pub args: [u64; 8], + pub initial: Banks, + pub expected_cause: u32, + pub fixed_expected: Option, +} +pub struct Model { + pub cases: Vec, + pub current: usize, + pub actual: Banks, + pub expected: Banks, + pub engine: Engine, + pub fault: Option, +} +fn i(op: u32, rd: u32, f: u32, rs: u32, imm: i32) -> u32 { + ((imm as u32 & 4095) << 20) | rs << 15 | f << 12 | rd << 7 | op +} +fn s(op: u32, f: u32, rs1: u32, rs2: u32, imm: u32) -> u32 { + (imm >> 5) << 25 | rs2 << 20 | rs1 << 15 | f << 12 | (imm & 31) << 7 | op +} +fn r(op: u32, rd: u32, f: u32, rs1: u32, rs2: u32, f7: u32) -> u32 { + f7 << 25 | rs2 << 20 | rs1 << 15 | f << 12 | rd << 7 | op +} +fn initial() -> Banks { + let mut b = Banks(vec![0; BANKS * BANK_BYTES]); + for offset in 0..BANK_BYTES { + b.0[BANK_BYTES + offset] = (offset as u8).wrapping_mul(73).wrapping_add(19); + } + for n in 0..256 { + let x = (n as f32 - 123.0) / 31.0; + b.put(2, n * 4, &x.to_le_bytes()); + b.put(3, n * 4, &(0.25 + n as f32 / 128.0).to_le_bytes()); + } + b +} +fn micro(name: &str, words: &[u32], args: [u64; 8]) -> Case { + Case { + name: name.into(), + program: words.iter().flat_map(|w| w.to_le_bytes()).collect(), + constants: vec![0; 512], + entry: 0, + args, + initial: initial(), + expected_cause: 0, + fixed_expected: None, + } +} +fn elf(path: &Path, name: &str, args: [u64; 8]) -> Case { + let bytes = std::fs::read(path).unwrap(); + assert_eq!(&bytes[..7], b"\x7fELF\x02\x01\x01"); + let u64_at = |p| u64::from_le_bytes(bytes[p..p + 8].try_into().unwrap()); + let u32_at = |p| u32::from_le_bytes(bytes[p..p + 4].try_into().unwrap()); + let u16_at = |p| u16::from_le_bytes(bytes[p..p + 2].try_into().unwrap()); + assert_eq!(u16_at(18), 243); + let mut c = micro(name, &[], args); + c.entry = u32::try_from(u64_at(24)).expect("kernel entry exceeds local address width"); + assert_eq!(u16_at(58), 64, "invalid ELF64 section header size"); + let sections = usize::try_from(u64_at(40)).unwrap(); + for n in 0..u16_at(60) as usize { + let p = sections.checked_add(n.checked_mul(64).unwrap()).unwrap(); + let flags = u64_at(p + 8); + let address = u32::try_from(u64_at(p + 16)).expect("kernel section exceeds local address width"); + let size = usize::try_from(u64_at(p + 32)).unwrap(); + if flags & 2 == 0 || size == 0 { + continue; + } + let offset = usize::try_from(u64_at(p + 24)).unwrap(); + if flags & 4 != 0 { + assert_eq!(address, 0); + c.program = bytes[offset..offset.checked_add(size).unwrap()].to_vec(); + } else { + assert_ne!( + u32_at(p + 4), + 8, + "allocated NOBITS requires explicit initialization" + ); + assert_eq!(address, 0x40000000); + assert_eq!(flags & 1, 0, "kernel globals must be read-only"); + c.constants.resize(size.max(512), 0); + c.constants[..size].copy_from_slice(&bytes[offset..offset.checked_add(size).unwrap()]); + } + } + assert!(!c.program.is_empty() && c.program.len() <= 4096 && c.program.len() % 4 == 0); + c +} +// Independent integer arithmetic gold: the BEMU Engine does not implement these fixed-point opcodes. +fn fixed_gold(case: &Case, bits: usize, op: u32, mode: u32) -> Banks { + let mut gold = case.initial.clone(); + let bytes = bits / 8; + let mask = (1i128 << bits) - 1; + let minimum = -(1i128 << (bits - 1)); + let maximum = (1i128 << (bits - 1)) - 1; + let read = |offset: usize, size: usize| { + let mut value = [0; 8]; + value[..size].copy_from_slice(&case.initial.0[2 * BANK_BYTES + offset..2 * BANK_BYTES + offset + size]); + u64::from_le_bytes(value) as i128 + }; + let signed = |value: i128, width: usize| if value & (1i128 << (width - 1)) != 0 { value - (1i128 << width) } else { value }; + let mut saturated = false; + for lane in 0..5 { + let a = read(lane * bytes, bytes); + let b = read(128 + lane * bytes, bytes); + let mut result = a; + if mode != 3 || lane >= 2 { + let (value, shift) = match op { + 8 => (a + b, 1), 9 => (signed(a, bits) + signed(b, bits), 1), + 10 => (a - b, 1), 11 => (signed(a, bits) - signed(b, bits), 1), + 39 => (signed(a, bits) * signed(b, bits), bits - 1), + 42 => (a, b as usize & (bits - 1)), + 43 => (signed(a, bits), b as usize & (bits - 1)), + 46 => (read(lane * bytes * 2, bytes * 2), b as usize & (bits * 2 - 1)), + 47 => (signed(read(lane * bytes * 2, bytes * 2), bits * 2), b as usize & (bits * 2 - 1)), + _ => unreachable!(), + }; + let truncated = value >> shift; + let half = shift != 0 && value & (1i128 << (shift - 1)) != 0; + let low = shift != 0 && value & ((1i128 << (shift - 1)) - 1) != 0; + let increment = match mode { + 0 => half, 1 => half && (low || truncated & 1 != 0), 2 => false, + 3 => truncated & 1 == 0 && value & ((1i128 << shift) - 1) != 0, + _ => unreachable!(), + }; + result = truncated + i128::from(increment); + if op == 39 || op == 47 { + saturated |= result < minimum || result > maximum; + result = result.clamp(minimum, maximum); + } else if op == 46 { + saturated |= result > mask; + result = result.min(mask); + } + } + gold.put(1, lane * bytes, &((result & mask) as u64).to_le_bytes()[..bytes]); + } + gold.put(1, 128, &u32::from(saturated).to_le_bytes()); + gold +} + +// Additional coverage for balanced integer reductions and bounded rounding carry. +fn integer_depth_cases(args: [u64; 8]) -> Vec { + let mut cases = vec![]; + for sew in 0..3 { + let bits = 8usize << sew; + let mask = u64::MAX >> (64 - bits); + let values = [1u64 << (bits - 1), mask >> 1, 0, 1, mask]; + let width = [0, 5, 6, 7][sew as usize]; + for reduction in [true, false] { + let operations: Vec<(u32, u32)> = if reduction { + (0..8).map(|op| (op, 2)).collect() + } else { + let mut ops = vec![(8, 2), (9, 2), (10, 2), (11, 2), (39, 0), (42, 0), (43, 0)]; + if sew < 2 { ops.extend([(46, 0), (47, 0)]); } + ops + }; + for (op, kind) in operations { + for mode in 0..(if reduction { 5 } else { 4 }) { + let mut a = args; + a[2] = 5; + let lmul = if 5 * bits > VLEN { 1 } else { 0 }; + let mut words = vec![ + i(0x57, 5, 7, 12, ((sew << 3) | lmul) as i32), + (1 << 25) | (11 << 15) | (width << 12) | (8 << 7) | 7, + i(0x13, 6, 0, 11, 128), + (1 << 25) | (6 << 15) | (width << 12) | (16 << 7) | 7, + (1 << 25) | (11 << 15) | (width << 12) | (24 << 7) | 7, + ]; + let mut vm = 1; + if reduction { + if mode == 1 { + words.push((0x19 << 26) | (1 << 25) | (8 << 20) | (3 << 12) | 0x57); + vm = 0; + } else if mode == 2 { + words.push((0x1b << 26) | (1 << 25) | (2 << 12) | 0x57); + vm = 0; + } else if mode == 3 { + words.push(i(0x57, 5, 7, 0, 0xc00 | (sew << 3) as i32)); + } else if mode == 4 { + words.push(i(0x73, 0, 5, 1, 8)); + } + } else { + words.push(i(0x73, 0, 5, mode, 0xa)); + words.push(i(0x73, 0, 5, 0, 9)); + if op == 46 || op == 47 { + words.push((1 << 25) | (11 << 15) | ([5, 6, 7][sew as usize] << 12) | (8 << 7) | 7); + } + } + if !reduction && mode == 3 { words.push(i(0x73, 0, 5, 2, 8)); } + words.push((op << 26) | (vm << 25) | (8 << 20) | (16 << 15) | (kind << 12) | (24 << 7) | 0x57); + words.push(i(0x57, 5, 7, 12, ((sew << 3) | lmul) as i32)); + words.push((1 << 25) | (10 << 15) | (width << 12) | (24 << 7) | 0x27); + if !reduction { + words.push(i(0x73, 6, 2, 0, 9)); + words.push(s(0x23, 2, 10, 6, 128)); + } + words.push(0x8067); + let mut case = micro(&format!("depth_{}_e{bits}_op{op}_mode{mode}", if reduction {"reduce"} else {"round"}), &words, a); + for (index, value) in values.into_iter().enumerate() { + let bytes = bits / 8; + let rhs = if reduction || op == 39 || kind == 2 { value } else { [0, 1, bits as u64 - 1, bits as u64 / 2, 3][index] }; + case.initial.put(2, index * bytes, &value.to_le_bytes()[..bytes]); + case.initial.put(2, 128 + index * bytes, &rhs.to_le_bytes()[..bytes]); + } + if !reduction { case.fixed_expected = Some(fixed_gold(&case, bits, op, mode)); } + if reduction && mode == 4 { case.expected_cause = 2; } + cases.push(case); + } + } + } + } + cases +} + +impl Model { + pub fn new(kernel_dir: &Path) -> Self { + let mut cases = vec![]; + let mut args = [0; 8]; + args[0] = 1 << 32; + args[1] = 2 << 32; + for kernel in ["silu", "swiglu", "snake", "quant"] { + let lengths: &[u32] = if kernel == "quant" { + &[32, 64] + } else { + &[1, 31, 33, 65] + }; + for &n in lengths { + let mut a = args; + match kernel { + "swiglu" => { + a[2] = 3 << 32; + a[3] = u64::from(n); + } + "snake" => { + a[2] = 2; + a[3] = u64::from(n); + a[4] = 48; + a[5] = 56; + } + "quant" => { + a[1] = args[0] + 1024; + a[2] = args[1]; + a[3] = u64::from(n); + a[4] = args[0] + 1536; + } + _ => a[2] = u64::from(n), + } + if kernel == "silu" { a[3] = args[0] + 1024; } + if kernel == "swiglu" { a[4] = args[0] + 1024; } + let mut c = elf( + &kernel_dir.join(format!("{kernel}.elf")), + &format!("{kernel}_{n}"), + a, + ); + if kernel == "snake" { + c.initial.put( + 0, + 48, + &[ + 0.0f32.to_le_bytes(), + 0.5f32.to_le_bytes(), + 0.25f32.to_le_bytes(), + (-0.25f32).to_le_bytes(), + ] + .concat(), + ); + } + cases.push(c); + } + } + // Unaligned loads/stores and every RV32M operation, including divide by zero and overflow. + for &(a, b) in &[ + (17u32, 3u32), + (0x80000000, 0xffffffff), + (0xffffffef, 0), + (0xffffffff, 7), + ] { + let mut aargs = args; + aargs[2] = u64::from(a); + aargs[3] = u64::from(b); + let mut words = vec![]; + for f in 0..8 { + words.push(r(0x33, 5, f, 12, 13, 1)); + words.push(s(0x23, 2, 10, 5, f * 4 + 1)); + } + for f in [0, 1, 2, 4, 5] { + words.push(i(3, 6, f, 10, 1)); + words.push(s(0x23, 2, 10, 6, 48 + f * 4)); + } + words.push(0x8067); + cases.push(micro(&format!("scalar_m_{a:x}_{b:x}"), &words, aargs)); + } + for funct in [0, 1, 4, 5, 6, 7] { + for &(left, right) in &[(0xffffffff, 1), (7, 7)] { + let mut a = args; + a[2] = left; + a[3] = right; + let words = [ + i(0x13, 5, 0, 0, 0), + (13 << 20) | (12 << 15) | (funct << 12) | (4 << 8) | 0x63, + i(0x13, 5, 0, 0, 1), + s(0x23, 2, 10, 5, 0), + 0x8067, + ]; + cases.push(micro( + &format!("branch_{funct}_{left:x}_{right:x}"), + &words, + a, + )); + } + } + for sew in [2, 3] { + for (vl, start) in [(0, 0), (4, 0), (4, 1), (4, 4)] { + let width = if sew == 2 { 6 } else { 7 }; + let words = [ + i(0x57, 5, 7, 4, 0xc00 | (sew << 3)), + (1 << 25) | (11 << 15) | (width << 12) | (9 << 7) | 7, + i(0x57, 5, 7, 12, (sew << 3) | 3), + i(0x73, 0, 5, start, 8), + r(0x57, 9, 6, 13, 0, 33), + i(0x57, 5, 7, 4, 0xc00 | (sew << 3)), + (1 << 25) | (10 << 15) | (width << 12) | (9 << 7) | 0x27, + i(0x73, 7, 2, 0, 1), + s(0x23, 2, 10, 7, 64), + 0x8067, + ]; + let mut a = args; + a[2] = vl; + a[3] = 0x80000001; + let mut c = micro(&format!("scalar_insert_e{}_vl{vl}_start{start}", 8 << sew), &words, a); + if sew == 3 && ELEN == 32 { c.expected_cause = 2; } + cases.push(c); + } + } + for scalar in [0x3f800000u64, 0x7f800123] { + for (vl, start) in [(0, 0), (4, 0), (4, 1), (4, 4)] { + let words = [ + i(0x57, 5, 7, 4, 0xc00 | (2 << 3)), + (1 << 25) | (11 << 15) | (6 << 12) | (9 << 7) | 7, + i(7, 3, 2, 13, 0), + i(0x57, 5, 7, 12, (2 << 3) | 3), + i(0x73, 0, 5, start, 8), + r(0x57, 9, 5, 3, 0, 33), + i(0x57, 5, 7, 4, 0xc00 | (2 << 3)), + (1 << 25) | (10 << 15) | (6 << 12) | (9 << 7) | 0x27, + i(0x73, 7, 2, 0, 1), + s(0x23, 2, 10, 7, 64), + 0x8067, + ]; + let mut a = args; + a[2] = vl; + a[3] = 3 << 32; + let mut c = micro(&format!("floating_insert_{scalar:x}_vl{vl}_start{start}"), &words, a); + c.initial.put(3, 0, &scalar.to_le_bytes()); + cases.push(c); + } + } + let mut a = args; + a[2] = 17; + cases.push(micro( + "vset_avl_register_rules", + &[ + i(0x57, 5, 7, 12, 16), + s(0x23, 2, 10, 5, 0), + i(0x57, 5, 7, 0, 16), + s(0x23, 2, 10, 5, 4), + i(0x57, 0, 7, 0, 16), + i(0x73, 5, 2, 0, 0xc20), + s(0x23, 2, 10, 5, 8), + i(0x57, 5, 7, 3, 0xc10u32 as i32), + s(0x23, 2, 10, 5, 12), + 0x8067, + ], + a, + )); + // SEW and integer/fractional LMUL, VL=0, mask/tail preservation, CSR visibility. + for sew in 0..4 { + for lmul in [0, 1, 2, 3, 5, 6, 7] { + for avl in [0, 1, 17, 63] { + let vt = (sew << 3) | lmul; + let mut a = args; + a[2] = avl; + let width = [0, 5, 6, 7][sew as usize]; + let words = [ + i(0x57, 5, 7, 12, vt), + (1 << 25) | (11 << 15) | (width << 12) | (8 << 7) | 7, + r(0x57, 16, 0, 8, 8, 0) | (1 << 25), + (1 << 25) | (10 << 15) | (width << 12) | (16 << 7) | 0x27, + i(0x73, 6, 2, 0, 0xc20), + s(0x23, 2, 10, 6, 1536), + i(0x73, 6, 2, 0, 0xc21), + s(0x23, 2, 10, 6, 1540), + 0x8067, + ]; + let mut c = micro( + &format!("vector_sew{}_lmul{lmul}_avl{avl}", 8 << sew), + &words, + a, + ); + if (8 << sew) > ELEN as i32 || (lmul >= 5 && (8 << sew) > (ELEN as i32 >> (8 - lmul))) { + c.expected_cause = 2; + } + cases.push(c); + } + } + } + for sew in 0..4 { + let mut a = args; + a[2] = 17; + let width = [0, 5, 6, 7][sew as usize]; + let words = [ + i(0x57, 5, 7, 12, sew << 3), + (1 << 25) | (11 << 15) | (width << 12) | (8 << 7) | 7, + (0x18 << 26) | (1 << 25) | (8 << 20) | (3 << 12) | 0x57, + (8 << 20) | (8 << 15) | (16 << 7) | 0x57, + (1 << 25) | (10 << 15) | (width << 12) | (16 << 7) | 0x27, + 0x8067, + ]; + let mut c = micro(&format!("masked_sew{}", 8 << sew), &words, a); + if sew == 3 && ELEN == 32 { c.expected_cause = 2; } + cases.push(c); + } + for sew in [2, 3] { + let mut a = args; + a[2] = 17; + let width = if sew == 2 { 6 } else { 7 }; + let mut c = micro( + &format!("vector_floating_{}", 8 << sew), + &[ + i(0x57, 5, 7, 12, sew << 3), + (1 << 25) | (11 << 15) | (width << 12) | (8 << 7) | 7, + (1 << 25) | (8 << 20) | (8 << 15) | (1 << 12) | (16 << 7) | 0x57, + (0x2c << 26) | (1 << 25) | (8 << 20) | (8 << 15) | (1 << 12) | (16 << 7) | 0x57, + (1 << 25) | (10 << 15) | (width << 12) | (16 << 7) | 0x27, + (0x12 << 26) + | (1 << 25) + | (16 << 20) + | (1 << 15) + | (1 << 12) + | (24 << 7) + | 0x57, + i(0x13, 7, 0, 10, 1024), + (1 << 25) | (7 << 15) | (width << 12) | (24 << 7) | 0x27, + i(0x73, 6, 2, 0, 1), + s(0x23, 2, 10, 6, 1536), + 0x8067, + ], + a, + ); + if sew == 3 { + if ELEN == 32 { c.expected_cause = 2; } + for n in 0..17 { + c.initial + .put(2, n * 8, &(n as f64 / 4.0 - 2.0).to_le_bytes()); + } + } + cases.push(c); + } + // Scalar IEEE arithmetic/FMA/conversion flags are stored through CSR reads. + for fmt in 0..2 { + let load_f = if fmt == 0 { 2 } else { 3 }; + let step = if fmt == 0 { 4 } else { 8 }; + let mut c = micro( + &format!("floating_{}", 32 << fmt), + &[ + i(7, 1, load_f, 11, 0), + i(7, 2, load_f, 11, step), + r(0x53, 3, 0, 1, 2, fmt), + (2 << 27) | (fmt << 25) | (2 << 20) | (1 << 15) | (4 << 7) | 0x43, + s(0x27, load_f, 10, 3, 0), + s(0x27, load_f, 10, 4, 8), + r(0x53, 5, 0, 1, 0, 0x60 | fmt), + s(0x23, 2, 10, 5, 16), + i(0x73, 6, 2, 0, 1), + s(0x23, 2, 10, 6, 20), + 0x8067, + ], + args, + ); + if fmt == 0 { + c.initial.put( + 2, + 0, + &[1.5f32.to_le_bytes(), (-2.25f32).to_le_bytes()].concat(), + ); + } else { + if ELEN == 32 { c.expected_cause = 2; } + c.initial.put( + 2, + 0, + &[1.5f64.to_le_bytes(), (-2.25f64).to_le_bytes()].concat(), + ); + } + cases.push(c); + } + // Raw IEEE encodings preserve signaling NaNs and signed zero in the inputs. + for fmt in 0..2 { + let fixtures: &[(&str, [u64; 3])] = if fmt == 0 { + &[ + ("zeros", [0, 0x80000000, 0x3f800000]), + ("division_by_zero", [0x3f800000, 0, 0xbf800000]), + ("negative_zero", [0x80000000, 0x40000000, 0]), + ("qnan", [0x7fc12345, 0x3f800000, 0xbf800000]), + ("snan", [0x7f812345, 0x3f800000, 0]), + ("infinities", [0x7f800000, 0, 0xff800000]), + ("subnormal", [1, 0x3f000000, 0x00800000]), + ("overflow", [0x7f7fffff, 0x40000000, 0xff7fffff]), + ("conversion_ties", [0x3fc00000, 0xc0200000, 0x3f800000]), + ("division_inexact", [0x3f800000, 0x40400000, 0x80000000]), + ("fused_cancellation", [0x3f800001, 0x3f800001, 0xbf800000]), + ] + } else { + &[ + ("zeros", [0, 0x8000000000000000, 0x3ff0000000000000]), + ( + "division_by_zero", + [0x3ff0000000000000, 0, 0xbff0000000000000], + ), + ("negative_zero", [0x8000000000000000, 0x4000000000000000, 0]), + ( + "qnan", + [0x7ff8123456789abc, 0x3ff0000000000000, 0xbff0000000000000], + ), + ("snan", [0x7ff0123456789abc, 0x3ff0000000000000, 0]), + ("infinities", [0x7ff0000000000000, 0, 0xfff0000000000000]), + ("subnormal", [1, 0x3fe0000000000000, 0x0010000000000000]), + ( + "overflow", + [0x7fefffffffffffff, 0x4000000000000000, 0xffefffffffffffff], + ), + ( + "conversion_ties", + [0x3ff8000000000000, 0xc004000000000000, 0x3ff0000000000000], + ), + ( + "division_inexact", + [0x3ff0000000000000, 0x4008000000000000, 0x8000000000000000], + ), + ( + "fused_cancellation", + [0x3ff0000000000001, 0x3ff0000000000001, 0xbff0000000000000], + ), + ("narrowing_tie", [0x3ff0000010000000, 0x3ff0000000000000, 0]), + ] + }; + let width = if fmt == 0 { 2 } else { 3 }; + let bytes = 1usize << width; + for frm in 0..5 { + for &(name, values) in fixtures { + if frm != 0 + && !matches!( + name, + "conversion_ties" | "fused_cancellation" | "narrowing_tie" + ) + { + continue; + } + let mut a = args; + a[2] = 0x01000001; + a[3] = 0xffffffff; + let mut words = vec![i(0x73, 0, 5, frm, 2)]; + for f in 1..=3 { + words.push(i(7, f, width, 11, (f as usize - 1) as i32 * bytes as i32)); + } + // Clear and sample fflags for every operation; result and flags share a 32-byte record. + let mut operations = vec![]; + for op in [0, 4, 8, 12] { + operations.push((r(0x53, 4, 7, 1, 2, op | fmt), s(0x27, width, 10, 4, 0))); + } + for op in [0x43, 0x47, 0x4b, 0x4f] { + operations.push(( + (3 << 27) + | (fmt << 25) + | (2 << 20) + | (1 << 15) + | (7 << 12) + | (4 << 7) + | op, + s(0x27, width, 10, 4, 0), + )); + } + for source in [1, 2] { + for unsigned in 0..2 { + operations.push(( + r(0x53, 5, 7, source, unsigned, 0x60 | fmt), + s(0x23, 2, 10, 5, 0), + )); + } + } + for unsigned in 0..2 { + operations.push(( + r(0x53, 4, 7, 12 + unsigned, unsigned, 0x68 | fmt), + s(0x27, width, 10, 4, 0), + )); + } + let target = 1 - fmt; + if fmt == 1 { operations.push(( + r(0x53, 4, 7, 1, fmt, 0x20 | target), + s(0x27, if target == 0 { 2 } else { 3 }, 10, 4, 0), + )); } + for (n, (operation, store)) in operations.into_iter().enumerate() { + let offset = n as u32 * 32; + words.extend([ + i(0x73, 0, 5, 0, 1), + operation, + store | ((offset >> 5) << 25), + i(0x73, 6, 2, 0, 1), + s(0x23, 2, 10, 6, offset + 16), + ]); + } + words.extend([i(0x73, 6, 2, 0, 2), s(0x23, 2, 10, 6, 1536), 0x8067]); + let mut c = micro( + &format!("scalar_ieee_{}_frm{frm}_{name}", 32 << fmt), + &words, + a, + ); + if fmt == 1 && ELEN == 32 { c.expected_cause = 2; } + for (n, bits) in values.into_iter().enumerate() { + c.initial.put(2, n * bytes, &bits.to_le_bytes()[..bytes]); + } + cases.push(c); + } + // The vector case runs the same exceptional inputs lane by lane. + let mut a = args; + a[2] = fixtures.len() as u64; + a[3] = 3 << 32; + a[4] = 5 << 32; + let vwidth: u32 = if fmt == 0 { 6 } else { 7 }; + let load = + |vd: u32, rs: u32| (1 << 25) | (rs << 15) | (vwidth << 12) | (vd << 7) | 7; + let store = |vd: u32| (1 << 25) | (7 << 15) | (vwidth << 12) | (vd << 7) | 0x27; + let mut words = vec![ + i(0x73, 0, 5, frm, 2), + i(0x57, 5, 7, 12, (fmt as i32 + 2) << 3), + load(8, 11), + load(12, 13), + ]; + for (n, operation) in [0, 2, 0x24, 0x20, 0x2c].into_iter().enumerate() { + let offset = n as u32 * 128; + words.extend([ + i(0x73, 0, 5, 0, 1), + load(20, 14), + (operation << 26) + | (1 << 25) + | (12 << 20) + | (8 << 15) + | (1 << 12) + | (20 << 7) + | 0x57, + i(0x13, 7, 0, 10, offset as i32), + store(20), + i(0x73, 6, 2, 0, 1), + s(0x23, 2, 10, 6, 1024 + n as u32 * 4), + ]); + } + for unsigned in 0..2 { + words.extend([ + i(0x73, 0, 5, 0, 1), + (0x12 << 26) + | (1 << 25) + | (8 << 20) + | ((1 - unsigned) << 15) + | (1 << 12) + | (24 << 7) + | 0x57, + i(0x13, 7, 0, 10, 640 + unsigned as i32 * 128), + store(24), + i(0x73, 6, 2, 0, 1), + s(0x23, 2, 10, 6, 1044 + unsigned * 4), + ]); + } + words.push(0x8067); + let mut c = micro(&format!("vector_ieee_{}_frm{frm}", 32 << fmt), &words, a); + if fmt == 1 && ELEN == 32 { c.expected_cause = 2; } + for (n, (_, values)) in fixtures.iter().enumerate() { + for (bank, bits) in [(2, values[0]), (3, values[1]), (5, values[2])] { + c.initial.put(bank, n * bytes, &bits.to_le_bytes()[..bytes]); + } + } + cases.push(c); + } + } + cases.push(micro( + "bank_line_crossing_word", + &[i(3, 5, 2, 11, 15), s(0x23, 2, 10, 5, 15), 0x8067], + args, + )); + let mut double_load = micro( + "bank_line_crossing_double", + &[i(7, 5, 3, 11, 11), s(0x27, 3, 10, 5, 11), 0x8067], + args, + ); + if ELEN == 32 { double_load.expected_cause = 2; } + cases.push(double_load); + let mut a = args; + a[2] = 17; + let mut c = micro( + "illegal_group_alignment", + &[ + i(0x57, 5, 7, 12, 17), + (1 << 25) | (11 << 15) | (6 << 12) | (9 << 7) | 7, + ], + a, + ); + c.expected_cause = 2; + cases.push(c); + cases.push(micro("illegal_opcode", &[0xffffffff], args)); + cases.push(micro("load_fault", &[i(3, 5, 2, 11, 0)], { + let mut a = args; + a[1] = 6 << 32; + a + })); + cases.push(micro("store_fault", &[s(0x23, 2, 10, 11, 0)], { + let mut a = args; + a[0] = (1 << 32) + 4094; + a + })); + for (n, cause) in [ + (cases.len() - 3, 2), + (cases.len() - 2, 5), + (cases.len() - 1, 7), + ] { + cases[n].expected_cause = cause; + } + cases.extend(integer_depth_cases(args)); + let mut local = micro( + "local_stack_and_constants", + &[ + i(0x13, 2, 0, 2, -16), + s(0x23, 2, 2, 11, 0), + s(0x23, 2, 2, 12, 4), + i(3, 5, 2, 2, 0), + i(3, 6, 2, 2, 4), + s(0x23, 2, 10, 5, 0), + s(0x23, 2, 10, 6, 4), + 0x400003b7, + i(3, 5, 2, 7, 0), + s(0x23, 2, 10, 5, 8), + i(0x13, 2, 0, 2, 16), + 0x8067, + ], + args, + ); + local.constants[..4].copy_from_slice(&0x72cafe31u32.to_le_bytes()); + cases.push(local); + let mut readonly = micro( + "store_readonly_constants", + &[0x400002b7, s(0x23, 2, 5, 11, 0), 0x8067], + args, + ); + readonly.expected_cause = 7; + cases.push(readonly); + for kernel in ["sin", "cos"] { + let values: Vec = (0..65).map(|n| (n as f32 - 32.0) * 0.75).chain([ + 120.0, -120.0, 10000.0, -10000.0, 1e10, -1e10, 1e30, -1e30, + f32::MAX, -f32::MAX, f32::from_bits(1), -f32::from_bits(1), + 0.0, -0.0, f32::INFINITY, f32::NEG_INFINITY, f32::NAN, + ]).collect(); + let mut a = args; + a[2] = values.len() as u64; + a[3] = a[0] + 1024; + let mut c = elf(&kernel_dir.join(format!("{kernel}.elf")), &format!("{kernel}_fp32_range"), a); + for (n, value) in values.iter().enumerate() { c.initial.put(2, n * 4, &value.to_le_bytes()); } + cases.push(c); + } + for kernel in ["gelu", "tanh"] { + let mut a = args; + a[2] = 37; + a[3] = a[0] + 1024; + cases.push(elf(&kernel_dir.join(format!("{kernel}.elf")), &format!("{kernel}_fp32"), a)); + } + let b = initial(); + Self { + cases, + current: 0, + actual: b.clone(), + expected: b, + engine: Engine::new(VLEN, ELEN, IBUF_BYTES), + fault: None, + } + } + pub fn select(&mut self, n: usize) { + self.current = n; + let c = &self.cases[n]; + self.actual = c.initial.clone(); + let descriptor = [ + u64::from(c.entry), + c.program.len() as u64, + 0x40002000, + u64::from(c.args[0]), + u64::from(c.args[1]), + u64::from(c.args[2]), + u64::from(c.args[3]), + u64::from(c.args[4]), + u64::from(c.args[5]), + u64::from(c.args[6]), + u64::from(c.args[7]), + 0, + ]; + for (index, value) in descriptor.iter().enumerate() { + self.actual.put(1, 2048 + index * 8, &value.to_le_bytes()); + } + self.expected = match &c.fixed_expected { + Some(gold) => { + let mut expected = gold.clone(); + let descriptor = BANK_BYTES + 2048..BANK_BYTES + 2144; + expected.0[descriptor.clone()].copy_from_slice(&self.actual.0[descriptor]); + expected + } + None => self.actual.clone(), + }; + self.engine.load_constants(n % 2, &c.constants); + self.engine.load_program(n % 2, &c.program); + self.fault = if c.fixed_expected.is_some() { None } else { self + .engine + .run( + n % 2, + c.entry, + c.program.len() as u32, + c.args.map(u64::from), + 0x40002000, + &mut self.expected, + ) + .err() }; + assert_eq!( + self.fault.map(|f| f.cause).unwrap_or(0), + c.expected_cause, + "unexpected reference outcome for {}: {:?}", + c.name, self.fault + ); + let read_float = |banks: &Banks, address: u64| { + f32::from_le_bytes(banks.0[banks.range(address, 4).unwrap()].try_into().unwrap()) + }; + let operation = c.name.split('_').next().unwrap(); + if matches!(operation, "silu" | "swiglu" | "snake" | "sin" | "cos") { + let length = c.args[if matches!(operation, "swiglu" | "snake") { 3 } else { 2 }] as usize; + let channels = if operation == "snake" { c.args[2] as usize } else { 1 }; + for channel in 0..channels { + for index in 0..length { + let offset = ((channel * length + index) * 4) as u64; + let x = read_float(&c.initial, c.args[1] + offset); + let expected = match operation { + "silu" => x / (1.0 + (-x).exp()), + "swiglu" => x / (1.0 + (-x).exp()) * read_float(&c.initial, c.args[2] + offset), + "snake" => { + let alpha = read_float(&c.initial, c.args[4] + channel as u64 * 4).exp(); + let beta = read_float(&c.initial, c.args[5] + channel as u64 * 4).exp() + 1e-9; + let sine = (x * alpha).sin(); + x + sine * sine / beta + } + "sin" => x.sin(), + "cos" => x.cos(), + _ => unreachable!(), + }; + let actual = read_float(&self.expected, c.args[0] + offset); + assert!(if expected.is_nan() { actual.is_nan() } else { + actual == expected || (actual - expected).abs() <= 2e-6 * (1.0 + expected.abs()) + }, "{} element {index}: {actual} != libm {expected}", c.name); + } + } + } + eprintln!("RVV case {n}: {}", c.name); + } +} diff --git a/arch/src/main/scala/framework/mem-core/axis/src/csrc/README.md b/arch/src/main/scala/framework/acc/gpdomain/src/main/resources/README.md similarity index 100% rename from arch/src/main/scala/framework/mem-core/axis/src/csrc/README.md rename to arch/src/main/scala/framework/acc/gpdomain/src/main/resources/README.md diff --git a/arch/src/main/scala/framework/rvv/src/main/resources/rvv.json b/arch/src/main/scala/framework/acc/gpdomain/src/main/resources/gpdomain.json similarity index 100% rename from arch/src/main/scala/framework/rvv/src/main/resources/rvv.json rename to arch/src/main/scala/framework/acc/gpdomain/src/main/resources/gpdomain.json diff --git a/arch/src/main/scala/framework/rvv/src/main/resources/interface.sv b/arch/src/main/scala/framework/acc/gpdomain/src/main/resources/interface.sv similarity index 100% rename from arch/src/main/scala/framework/rvv/src/main/resources/interface.sv rename to arch/src/main/scala/framework/acc/gpdomain/src/main/resources/interface.sv diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/resources/package.sv b/arch/src/main/scala/framework/acc/gpdomain/src/main/resources/package.sv new file mode 100644 index 000000000..12694ca7e --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/resources/package.sv @@ -0,0 +1,97 @@ +package rvv_pkg; + timeunit 1ns; timeprecision 1ps; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + import "DPI-C" function void rvv_model_init(); + import "DPI-C" function int unsigned rvv_case_count(); + import "DPI-C" function void rvv_case_select(int unsigned n); + import "DPI-C" function longint unsigned rvv_case_meta(int unsigned field); + import "DPI-C" function int unsigned rvv_image_bytes(); + import "DPI-C" function int unsigned rvv_program_word(int unsigned n); + import "DPI-C" function int unsigned rvv_image_word(int unsigned n); + import "DPI-C" function void rvv_memory_masked_write( + longint unsigned address, + longint unsigned data, + int unsigned mask + ); + import "DPI-C" function longint unsigned rvv_expected_status(int unsigned field); + import "DPI-C" function longint unsigned rvv_memory_read( + longint unsigned address, + int unsigned size + ); + import "DPI-C" function int unsigned rvv_memory_error( + longint unsigned address, + int unsigned size + ); + import "DPI-C" function void rvv_memory_write( + longint unsigned address, + int unsigned size, + longint unsigned data, + int unsigned mask + ); + import "DPI-C" function int unsigned rvv_compare_memory(); + import "DPI-C" function int unsigned rvv_compare_image(); + + class launch_item extends uvm_sequence_item; + bit execute; + bit protocol_fault; + bit [31:0] protocol_cause, protocol_pc, protocol_instruction; + bit [63:0] protocol_tval; + bit [3:0] rob; + `uvm_object_utils(launch_item) + function new(string name = "launch_item"); + super.new(name); + endfunction + endclass + class completion_item extends uvm_sequence_item; + bit fault; + bit [15:0] write_bank; + bit [3:0] rob; + bit [31:0] pc, instruction, cause; + bit [63:0] tval; + `uvm_object_utils_begin(completion_item) + `uvm_field_int(fault, UVM_DEFAULT) + `uvm_field_int(rob, UVM_DEFAULT) + `uvm_field_int(write_bank, UVM_DEFAULT) + `uvm_field_int(pc, UVM_DEFAULT) + `uvm_field_int(instruction, UVM_DEFAULT | UVM_NOCOMPARE) + `uvm_field_int(cause, UVM_DEFAULT) + `uvm_field_int(tval, UVM_DEFAULT) + `uvm_object_utils_end + function new(string name = "completion_item"); + super.new(name); + endfunction + function bit do_compare(uvm_object rhs, uvm_comparer comparer); + completion_item other; + if (!$cast(other, rhs)) return 0; + return super.do_compare(rhs, comparer) && (!fault || instruction == other.instruction); + endfunction + endclass + class kernel_model extends reference_model #(launch_item, completion_item); + `uvm_component_utils(kernel_model) + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void write(launch_item t); + completion_item expected = completion_item::type_id::create("expected"); + expected.rob = t.rob; + expected.fault = t.execute ? rvv_expected_status(0) : 0; + expected.pc = t.execute ? rvv_expected_status(1) : 0; + expected.instruction = t.execute ? rvv_expected_status(2) : 0; + expected.cause = t.execute ? rvv_expected_status(3) : 0; + expected.tval = t.execute ? rvv_expected_status(4) : 0; + if (t.protocol_fault) begin + expected.fault = 1; + expected.pc = t.protocol_pc; + expected.instruction = t.protocol_instruction; + expected.cause = t.protocol_cause; + expected.tval = t.protocol_tval; + end + expected.write_bank = t.execute && !expected.fault ? 1 : 0; + expected_ap.write(expected); + endfunction + endclass + typedef environment#(launch_item, completion_item, kernel_model) kernel_env; + `include "test.svh" +endpackage diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/resources/suite.toml b/arch/src/main/scala/framework/acc/gpdomain/src/main/resources/suite.toml new file mode 100644 index 000000000..10972b204 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/resources/suite.toml @@ -0,0 +1,20 @@ +[model] +manifest = "src/csrc/Cargo.toml" +library = "librvv_reference_model" + +[[tests]] +functional_groups = [] +name = "topkernel" +top = "topkernel_tb" +test = "protocol_test" +filelist = "topkernel.f" +timeout_ns = 40000000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/resources/test.svh b/arch/src/main/scala/framework/acc/gpdomain/src/main/resources/test.svh new file mode 100644 index 000000000..dc916616c --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/resources/test.svh @@ -0,0 +1,265 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + virtual rvv_if vif; + kernel_env test_env; + int unsigned completed = 0; + bit protocol_fault = 0; + bit reject_write = 0; + bit [31:0] protocol_cause, protocol_pc, protocol_instruction; + bit [63:0] protocol_tval; + int unsigned requests[`RVV_MEMORY_PORTS]; + function new(string name, uvm_component parent); + super.new(name, parent); + + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual rvv_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "RVV interface missing") + test_env = kernel_env::type_id::create("env", this); + endfunction + task memory_port(int p); + bit read_pending = 0, write_pending = 0; + int read_delay, write_delay; + bit [31:0] seed = 32'h629731a5 ^ (p * 32'h1234567); + longint unsigned address; + forever begin + @(negedge vif.clock); + seed = {seed[30:0], seed[31] ^ seed[21] ^ seed[1] ^ seed[0]}; + if (vif.reset) begin + read_pending = 0; + write_pending = 0; + vif.read_ready[p] = 0; + vif.write_ready[p] = 0; + vif.read_response_valid[p] = 0; + vif.write_response_valid[p] = 0; + end else begin + vif.read_ready[p] = !read_pending && seed[0]; + vif.write_ready[p] = !write_pending && seed[1]; + if (read_pending && read_delay != 0) read_delay--; + if (write_pending && write_delay != 0) write_delay--; + vif.read_response_valid[p] = read_pending && read_delay == 0; + vif.write_response_valid[p] = write_pending && write_delay == 0; + end + @(posedge vif.clock); + if (!vif.reset) begin + if (vif.read_valid[p] && vif.read_ready[p]) begin + requests[p]++; + read_pending = 1; + read_delay = 1 + seed[4:2]; + address = (longint'(vif.read_bank[p] + vif.read_group[p]) << 32) | (int'(vif.read_row[p]) << 4); + vif.read_data[p] = {rvv_memory_read(address + 8, 3), rvv_memory_read(address, 3)}; + end + if (vif.write_valid[p] && vif.write_ready[p]) begin + requests[p]++; + write_pending = 1; + write_delay = 1 + seed[7:5]; + address = (longint'(vif.write_bank[p] + vif.write_group[p]) << 32) | (int'(vif.write_row[p]) << 4); + if (!reject_write) begin + rvv_memory_masked_write(address, vif.write_data[p][63:0], vif.write_mask[p][7:0]); + rvv_memory_masked_write(address + 8, vif.write_data[p][127:64], + vif.write_mask[p][15:8]); + end + vif.write_ok[p] = !reject_write; + end + if (vif.read_response_valid[p] && vif.read_response_ready[p]) read_pending = 0; + if (vif.write_response_valid[p] && vif.write_response_ready[p]) write_pending = 0; + end + end + endtask + task monitor(); + forever begin + @(posedge vif.clock); + if (!vif.reset) begin + if (vif.command_valid && vif.command_ready) begin + launch_item item = launch_item::type_id::create("launch"); + item.execute = vif.command_funct7 == 79; + item.protocol_fault = protocol_fault; + item.protocol_cause = protocol_cause; + item.protocol_tval = protocol_tval; + item.protocol_pc = protocol_pc; + item.protocol_instruction = protocol_instruction; + item.rob = vif.command_rob; + test_env.input_export.write(item); + end + if (vif.done_valid && vif.done_ready) begin + completion_item item = completion_item::type_id::create("completion"); + item.fault = vif.done_fault; + item.pc = vif.done_pc; + item.instruction = vif.done_instruction; + item.cause = vif.done_cause; + item.tval = vif.done_tval; + item.rob = vif.done_rob; + item.write_bank = vif.done_write_bank; + test_env.output_export.write(item); + end + end + end + endtask + task command(int funct7, longint unsigned rs1, longint unsigned rs2); + @(negedge vif.clock); + vif.command_funct7 = funct7; + vif.command_rs1 = rs1; + vif.command_rs2 = rs2; + vif.command_rob = completed % 16; + vif.command_valid = 1; + do @(posedge vif.clock); while (!vif.command_ready); + @(negedge vif.clock); + vif.command_valid = 0; + endtask + task retire(); + bit [250:0] held; + while (!vif.done_valid) @(negedge vif.clock); + held = { + vif.done_rob, + vif.done_write_bank, + vif.done_fault, + vif.done_pc, + vif.done_instruction, + vif.done_cycles, + vif.done_cause, + vif.done_tval, + vif.done_fflags, + vif.done_vxsat + }; + repeat (5) begin + @(negedge vif.clock); + if(!vif.done_valid || held !== {vif.done_rob,vif.done_write_bank,vif.done_fault,vif.done_pc, + vif.done_instruction,vif.done_cycles,vif.done_cause,vif.done_tval,vif.done_fflags,vif.done_vxsat}) + `uvm_fatal("COMPLETION", "completion changed under backpressure") + if (vif.command_ready || !vif.busy) + `uvm_fatal("COMPLETION", "completion allowed command before retirement") + end + vif.done_ready = 1; + @(negedge vif.clock); + vif.done_ready = 0; + completed++; + test_env.scoreboard.wait_checked(completed); + endtask + task upload(); + command(44, (longint'(rvv_image_bytes()) << 30) | ((6 + rvv_case_meta(0)) << 20), 64'h10000000); + for (int word_index = 0; word_index < rvv_image_bytes() / 4; word_index++) begin + @(negedge vif.clock); + vif.image_valid = 1; + vif.image_data = rvv_image_word(word_index); + do @(posedge vif.clock); while (!vif.image_ready); + end + @(negedge vif.clock); + vif.image_valid = 0; + retire(); + if (!rvv_compare_image()) `uvm_fatal("IMAGE", "image load modified compute banks") + endtask + task execute(); + vif.reset = 1; + vif.image_valid = 0; + vif.command_valid = 0; + vif.done_ready = 0; + foreach (vif.read_ready[p]) begin + vif.read_ready[p] = 0; + vif.write_ready[p] = 0; + vif.read_response_valid[p] = 0; + vif.write_response_valid[p] = 0; + vif.read_data[p] = 0; + vif.write_ok[p] = 0; + end + rvv_model_init(); + fork + begin + for (int p = 0; p < `RVV_MEMORY_PORTS; p++) begin + automatic int port = p; + fork + memory_port(port); + join_none + end + wait fork; + end + monitor(); + join_none + repeat (4) @(negedge vif.clock); + vif.reset = 0; + for (int case_index = 0; case_index < rvv_case_count(); case_index++) begin + rvv_case_select(case_index); + upload(); + command(79, (64'd1 << 20) | ((6 + rvv_case_meta(0)) << 10), 2048); + while (!vif.done_valid) @(negedge vif.clock); + if (!rvv_compare_memory()) + `uvm_fatal("REFERENCE", $sformatf("case %0d bank contents differ", case_index)) + retire(); + `uvm_info("RVV", $sformatf("case %0d completed and compared", case_index), UVM_LOW) + end + foreach (requests[p]) + if (requests[p] == 0) `uvm_fatal("PORTS", $sformatf("bank port %0d was never exercised", p)) + // A malformed header reports a command fault before any code/data is written. + protocol_fault = 1; + protocol_cause = 2; + protocol_tval = 32'hdeadbeef; + command(44, (longint'(rvv_image_bytes()) << 30) | (64'd6 << 20), 0); + for (int index = 0; index < 6; index++) begin + @(negedge vif.clock); + vif.image_valid = 1; + vif.image_data = index == 0 ? 32'hdeadbeef : rvv_image_word(index); + do @(posedge vif.clock); while (!vif.image_ready); + end + @(negedge vif.clock); + vif.image_valid = 0; + retire(); + protocol_tval = 25; + command(44, (64'd25 << 30) | (64'd6 << 20), 0); + retire(); + protocol_cause = 1; + protocol_tval = (64'd1 << 20) | (64'd6 << 10); + command(79, protocol_tval, 2048); + retire(); + protocol_cause = 2; + protocol_tval = 127; + command(127, 0, 0); + retire(); + // Writable globals and BSS are forbidden in a kernel image. + for (int invalid_field = 3; invalid_field <= 5; invalid_field += 2) begin + protocol_cause = 2; + protocol_tval = invalid_field == 3 ? 32'h50000 : 4; + command(44, (longint'(rvv_image_bytes()) << 30) | (64'd6 << 20), 0); + for (int index = 0; index < 6; index++) begin + @(negedge vif.clock); + vif.image_valid = 1; + vif.image_data = index == invalid_field ? + (invalid_field == 3 ? 32'h50000 : 4) : rvv_image_word(index); + do @(posedge vif.clock); while (!vif.image_ready); + end + @(negedge vif.clock); + vif.image_valid = 0; + retire(); + end + // A failed compute-bank write acknowledgement is an execution fault. + protocol_fault = 0; + rvv_case_select(14); + upload(); + protocol_fault = 1; + protocol_cause = 7; + protocol_tval = (64'd1 << 32) + 1; + protocol_pc = 4; + protocol_instruction = rvv_program_word(1); + reject_write = 1; + command(79, (64'd1 << 20) | ((6 + rvv_case_meta(0)) << 10), 2048); + retire(); + reject_write = 0; + protocol_pc = 0; + protocol_instruction = 0; + protocol_fault = 0; + // Reset cancels a command while it waits for a descriptor bank response. + rvv_case_select(rvv_case_count() - 2); + upload(); + command(79, (64'd1 << 20) | ((6 + rvv_case_meta(0)) << 10), 2048); + do @(posedge vif.clock); while (!(vif.read_valid[0] && vif.read_ready[0])); + @(negedge vif.clock); + vif.reset = 1; + test_env.scoreboard.cancel_pending(); + repeat (3) @(negedge vif.clock); + vif.reset = 0; + repeat (5) begin + @(negedge vif.clock); + if (vif.done_valid || vif.busy) `uvm_fatal("RESET", "reset did not cancel pending kernel") + end + `uvm_info("RVV", $sformatf("checked %0d Blink commands", completed), UVM_LOW) + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/resources/topkernel.f b/arch/src/main/scala/framework/acc/gpdomain/src/main/resources/topkernel.f new file mode 100644 index 000000000..bb71b8eb7 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/resources/topkernel.f @@ -0,0 +1,8 @@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv ++incdir+@RESOURCES@ ++incdir+@RTL@/KernelEngine +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/KernelEngine/filelist.f +@RESOURCES@/topkernel_tb.sv diff --git a/arch/src/main/scala/framework/rvv/src/main/resources/topkernel_tb.sv b/arch/src/main/scala/framework/acc/gpdomain/src/main/resources/topkernel_tb.sv similarity index 98% rename from arch/src/main/scala/framework/rvv/src/main/resources/topkernel_tb.sv rename to arch/src/main/scala/framework/acc/gpdomain/src/main/resources/topkernel_tb.sv index a81179167..b7b912972 100644 --- a/arch/src/main/scala/framework/rvv/src/main/resources/topkernel_tb.sv +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/resources/topkernel_tb.sv @@ -48,7 +48,7 @@ module topkernel_tb; KernelEngine dut ( .clock(clock), .reset(vif.reset), - .io_channelReady(1'b1), + .io_bankAccessReady(1'b1), .io_cmdReq_valid(vif.command_valid), .io_cmdReq_ready(vif.command_ready), .io_cmdReq_bits_cmd_funct7(vif.command_funct7), diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/Emit.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/Emit.scala new file mode 100644 index 000000000..841ae5377 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/Emit.scala @@ -0,0 +1,26 @@ +package framework.acc.gpdomain + +import _root_.configs.GlobalConfig + +object Emit { + + def emit( + b: GlobalConfig, + hartId: Int, + threadId: Int, + directory: String, + firtoolOpts: Array[String] + ): Unit = { + val core = + b.device.cores.find(_.hartId == hartId).getOrElse(throw new IllegalArgumentException(s"Unknown hart ID: $hartId")) + val thread = + core.threads.find(_.id == threadId).getOrElse(throw new IllegalArgumentException(s"Unknown thread ID: $threadId")) + + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new GpDomain(thread.acc.get.frontend, thread.acc.get.memDomain, thread.acc.get.gpDomain), + firtoolOpts = firtoolOpts, + args = Array("--target-dir", directory, "--split-verilog") + ) + } + +} diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/GpDomain.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/GpDomain.scala new file mode 100644 index 000000000..c4bfaa995 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/GpDomain.scala @@ -0,0 +1,76 @@ +package framework.acc.gpdomain + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.frontend.scheduler.{Complete, Issue} +import framework.acc.memdomain.{BankLayout, BankLayoutRequest, BankPort} +import framework.acc.memdomain.toddr.DmaStatus +import framework.acc.gpdomain.execution.{Execution, KernelCompletion} +import framework.acc.gpdomain.loader.Loader +import framework.acc.gpdomain.memory.Memory +import framework.acc.balldomain.KernelBallPort +import framework.acc.gpdomain.decoder.Decoder +import framework.acc.gpdomain.loader.ImageLoad + +class ImagePort extends Bundle { + val start = Decoupled(new ImageLoad) + val data = Decoupled(UInt(32.W)) + val terminal = Decoupled(new DmaStatus) + val complete = Flipped(Decoupled(new KernelCompletion)) + val release = Valid(Bool()) +} + +@instantiable +class GpDomain(val frontendParam: FrontendParam, val memParam: MemDomainParam, val gpParam: GpDomainParam) + extends Module { + val commandDataBits = frontendParam.commandDataBits + val ownerId = frontendParam.ownerId + require(gpParam.enable) + + @public + val io = IO(new Bundle { + val layoutRequest = Decoupled(new BankLayoutRequest(frontendParam)) + val layoutResponse = Flipped(Decoupled(new BankLayout(memParam))) + val command = Flipped(Decoupled(new Issue(frontendParam, memParam))) + val complete = Decoupled(new Complete(frontendParam)) + val image = Flipped(new ImagePort) + val banks = Vec(gpParam.memory.memoryPorts, new BankPort(frontendParam, memParam)) + val ball = new KernelBallPort(frontendParam, memParam) + }) + + val loader: Instance[Loader] = Instantiate(new Loader(frontendParam, memParam, gpParam)) + val execution: Instance[Execution] = Instantiate(new Execution(frontendParam, memParam, gpParam)) + val memory: Instance[Memory] = Instantiate(new Memory(frontendParam, memParam, gpParam)) + val decoder: Instance[Decoder] = Instantiate(new Decoder(frontendParam, memParam)) + + decoder.io.issue <> io.command + io.layoutRequest <> decoder.io.layoutRequest + decoder.io.layoutResponse <> io.layoutResponse + loader.io.cmdReq <> decoder.io.request + + execution.io.program <> loader.io.program + execution.io.launch <> loader.io.launch + loader.io.done <> execution.io.done + loader.io.memory <> memory.io.loader + + for (port <- 0 until gpParam.memory.memoryPorts) { + memory.io.request(port) <> execution.io.memoryRequest(port) + execution.io.memoryResponse(port) <> memory.io.response(port) + io.banks(port) <> memory.io.banks(port) + } + + io.ball <> execution.io.ball + + loader.io.uploadStart <> io.image.start + loader.io.image <> io.image.data + loader.io.imageTerminal <> io.image.terminal + io.image.complete <> loader.io.uploadComplete + loader.io.release := io.image.release + + io.complete <> loader.io.cmdResp +} diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/configs/GpDomainParam.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/configs/GpDomainParam.scala new file mode 100644 index 000000000..c6dd993f4 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/configs/GpDomainParam.scala @@ -0,0 +1,31 @@ +package framework.acc.gpdomain.configs + +import buckyball.config.GpDomainConfig + +import upickle.default._ +import framework.acc.gpdomain.execution.configs.{Params => ExecutionParams} +import framework.acc.gpdomain.loader.configs.{Params => LoaderParams} +import framework.acc.gpdomain.memory.configs.{Params => MemoryParams} + +case class GpDomainParam( + enable: Boolean, + laneNumber: Int, + vLen: Int, + eLen: Int, + iBufWords: Int, + memoryPorts: Int) { + val execution = ExecutionParams(laneNumber, vLen, eLen, iBufWords, memoryPorts) + val memory = MemoryParams(memoryPorts, eLen) + val loader = LoaderParams(iBufWords, memory.constBytes) +} + +object GpDomainParam { + + def fromProto(p: GpDomainConfig): GpDomainParam = { + require(p.hasEnable, "GP enable policy is required") + GpDomainParam(p.getEnable, p.getLaneNumber, p.getVLen, p.getELen, p.getIBufWords, p.getMemoryPorts) + } + + implicit val rw: ReadWriter[GpDomainParam] = macroRW[GpDomainParam] + +} diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/decoder/DISA.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/decoder/DISA.scala new file mode 100644 index 000000000..dd061f459 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/decoder/DISA.scala @@ -0,0 +1,7 @@ +package framework.acc.gpdomain.decoder + +import chisel3.util.BitPat + +object DISA { + val RUN_KERNEL_BITPAT = BitPat("b1001111") +} diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/decoder/Decoder.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/decoder/Decoder.scala new file mode 100644 index 000000000..799761e59 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/decoder/Decoder.scala @@ -0,0 +1,48 @@ +package framework.acc.gpdomain.decoder + +import framework.lib.queue.Queue +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.acc.frontend.scheduler.Issue +import framework.acc.memdomain.{BankLayout, BankLayoutRequest} + +@instantiable +class Decoder(val frontendParam: FrontendParam, val memParam: MemDomainParam) extends Module { + val commandDataBits = frontendParam.commandDataBits + + @public + val io = IO(new Bundle { + val issue = Flipped(Decoupled(new Issue(frontendParam, memParam))) + val layoutRequest = Decoupled(new BankLayoutRequest(frontendParam)) + val layoutResponse = Flipped(Decoupled(new BankLayout(memParam))) + val request = Decoupled(new Issue(frontendParam, memParam)) + }) + + val run = io.issue.bits.cmd.cmd.raw_inst(31, 25) === DISA.RUN_KERNEL_BITPAT + + val pending = Instantiate(new Queue(new Issue(frontendParam, memParam), 4)) + io.layoutRequest.valid := io.issue.valid && run && pending.io.enq.ready + pending.io.enq.valid := io.issue.valid && run && io.layoutRequest.ready + pending.io.enq.bits := io.issue.bits + io.issue.ready := run && pending.io.enq.ready && io.layoutRequest.ready + io.layoutRequest.bits.banks(0) := io.issue.bits.renamedBankAccess.rd_bank_0_id + io.layoutRequest.bits.banks(1) := 0.U // Operand 1 names the program buffer, not a data bank. + io.layoutRequest.bits.banks(2) := io.issue.bits.renamedBankAccess.wr_bank_id + io.layoutRequest.bits.enabled(0) := true.B + io.layoutRequest.bits.enabled(1) := false.B + io.layoutRequest.bits.enabled(2) := true.B + io.request.valid := pending.io.deq.valid && io.layoutResponse.valid + io.request.bits := pending.io.deq.bits + io.request.bits.cmd.op1_col := io.layoutResponse.bits.groups(0) + io.request.bits.cmd.op2_col := 0.U + io.request.bits.cmd.wr_col := io.layoutResponse.bits.groups(2) + pending.io.deq.ready := io.request.ready && io.layoutResponse.valid + io.layoutResponse.ready := io.request.ready && pending.io.deq.valid + when(io.issue.valid)(assert(run, "GPDomain accepts only RUN_KERNEL")) +} diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/build.sc b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/build.sc new file mode 100644 index 000000000..a2cbc2e3e --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/build.sc @@ -0,0 +1,31 @@ +import mill._ +import $file.^.^.^.^.^.^.^.^.^.^.settings + +val ipRoot: os.Path = millSourcePath + +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.acc.gpdomain.execution.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Execution.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/IBuf.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/arithmetic/FALU.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/arithmetic/IALU.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/scalar/ScalarExecution.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/scalar/ScalarFloating.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/scalar/ScalarRF.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/vector/Operands.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/vector/VRF.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/vector/VectorThread.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/Emit.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/Emit.scala new file mode 100644 index 000000000..7418ed1ae --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/Emit.scala @@ -0,0 +1,26 @@ +package framework.acc.gpdomain.execution + +import _root_.configs.GlobalConfig + +object Emit { + + def emit( + b: GlobalConfig, + hartId: Int, + threadId: Int, + directory: String, + firtoolOpts: Array[String] + ): Unit = { + val core = + b.device.cores.find(_.hartId == hartId).getOrElse(throw new IllegalArgumentException(s"Unknown hart ID: $hartId")) + val thread = + core.threads.find(_.id == threadId).getOrElse(throw new IllegalArgumentException(s"Unknown thread ID: $threadId")) + + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Execution(thread.acc.get.frontend, thread.acc.get.memDomain, thread.acc.get.gpDomain), + firtoolOpts = firtoolOpts, + args = Array("--target-dir", directory, "--split-verilog") + ) + } + +} diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/Execution.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/Execution.scala new file mode 100644 index 000000000..bf9456913 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/Execution.scala @@ -0,0 +1,441 @@ +package framework.acc.gpdomain.execution + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.memdomain.configs.MemDomainParam + +import framework.acc.gpdomain.loader.KernelContext +import framework.acc.balldomain.KernelBallPort +import framework.acc.gpdomain.execution.scalar.{ScalarExecution, ScalarFloating, ScalarRF} +import framework.acc.gpdomain.execution.vector.VectorThread +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} + +class ProgramWrite extends Bundle { + val buffer = Bool() + val first = Bool() + val address = UInt(32.W) + val data = UInt(32.W) +} + +class KernelLaunch(frontendParam: FrontendParam, memParam: MemDomainParam) extends Bundle { + val context = new KernelContext(frontendParam, memParam) + val iBuffer = Bool() + val entry = UInt(32.W) + val end = UInt(32.W) + val stack = UInt(64.W) + val args = Vec(8, UInt(64.W)) +} + +class KernelCompletion extends Bundle { + val fault = Bool() + val pc = UInt(32.W) + val instruction = UInt(32.W) + val cycles = UInt(64.W) + val cause = UInt(32.W) + val tval = UInt(64.W) + val fflags = UInt(5.W) + val vxsat = Bool() +} + +class VectorMemoryRequest extends Bundle { + val address = UInt(64.W) + val write = Bool() + val data = UInt(64.W) + val mask = UInt(8.W) + val size = UInt(2.W) +} + +class VectorMemoryResponse extends Bundle { + val data = UInt(64.W) + val error = Bool() +} + +@instantiable +class Execution(val frontendParam: FrontendParam, val memParam: MemDomainParam, val gpParam: GpDomainParam) + extends Module { + val p = gpParam.execution + + @public + val io = IO(new Bundle { + val program = Flipped(Decoupled(new ProgramWrite)) + val launch = Flipped(Decoupled(new KernelLaunch(frontendParam, memParam))) + val done = Decoupled(new KernelCompletion) + val ball = new KernelBallPort(frontendParam, memParam) + val memoryRequest = Vec(p.memoryPorts, Decoupled(new VectorMemoryRequest)) + val memoryResponse = Vec(p.memoryPorts, Flipped(Decoupled(new VectorMemoryResponse))) + }) + + val iBuf: Seq[Instance[IBuf]] = Seq.fill(2)(Instantiate(new IBuf(gpParam))) + val registers: Instance[ScalarRF] = Instantiate(new ScalarRF(frontendParam, memParam)) + val scalar: Instance[ScalarExecution] = Instantiate(new ScalarExecution()) + val floating: Instance[ScalarFloating] = Instantiate(new ScalarFloating(gpParam)) + val vector: Instance[VectorThread] = Instantiate(new VectorThread(gpParam)) + + val idle :: fetch :: fetchWait :: execute :: vectorWait :: scalarWait :: floatWait :: memorySend :: memoryWait :: ballSend :: ballWait :: completed :: Nil = + Enum(12) + + val state = RegInit(idle) + val activeBuffer = RegInit(false.B) + val end = Reg(UInt(32.W)) + val pc = Reg(UInt(32.W)) + val instruction = RegInit(0.U(32.W)) + val cycles = RegInit(0.U(64.W)) + val fault = RegInit(false.B) + val cause = RegInit(0.U(32.W)) + val tval = RegInit(0.U(64.W)) + val fflags = RegInit(0.U(5.W)) + val frm = RegInit(0.U(3.W)) + val vxrm = RegInit(0.U(2.W)) + val vxsat = RegInit(false.B) + val scalarTarget = Reg(UInt(5.W)) + val scalarNextPc = Reg(UInt(32.W)) + + val scalarDivide = (instruction(6, 0) === "h33".U || instruction(6, 0) === "h3b".U) && instruction( + 31, + 25 + ) === 1.U && instruction(14, 12) >= 4.U + + val memory = Reg(new VectorMemoryRequest) + val memoryFloating = Reg(Bool()) + val memoryUnsigned = Reg(Bool()) + + def finish(isFault: Bool, code: UInt, value: UInt): Unit = { + state := completed + fault := isFault + cause := code + tval := value + } + + val context = Reg(new KernelContext(frontendParam, memParam)) + + // A kernel names one launch group per local bank: the read groups first, then the write groups. + val ballRs1 = registers.io.xReadData1 + val ballFunct = instruction(31, 25) + val ballEnable = ballFunct(6, 4) + val ballLocal = VecInit(Seq(0, 10, 20).map(lsb => ballRs1(frontendParam.bank_id_len + lsb - 1, lsb))) + + val ballUsed = VecInit( + ballEnable === 1.U || ballEnable === 3.U || ballEnable === 4.U, + ballEnable === 4.U, + ballEnable === 2.U || ballEnable === 3.U || ballEnable === 4.U + ) + + val ballFromRead = ballLocal.map(_ < context.readGroups) + val ballGroups = context.readGroups +& context.writeGroups + val ballRequest = io.ball.request.bits + ballRequest.target := 0.U + ballRequest.instId := context.instId + ballRequest.enabled := ballUsed + ballRequest.cmd.rob_id := context.rob_id + ballRequest.cmd.funct := ballFunct + ballRequest.cmd.rs2 := registers.io.xReadData2 + ballRequest.cmd.iter := ballRs1(62, 30) + ballRequest.cmd.op1_bank := Mux(ballFromRead(0), context.readBank, context.writeBank) + ballRequest.cmd.op2_bank := Mux(ballFromRead(1), context.readBank, context.writeBank) + ballRequest.cmd.wr_bank := context.writeBank + ballRequest.cmd.op1_group := Mux(ballFromRead(0), ballLocal(0), ballLocal(0) - context.readGroups) + ballRequest.cmd.op2_group := Mux(ballFromRead(1), ballLocal(1), ballLocal(1) - context.readGroups) + ballRequest.cmd.wr_group := ballLocal(2) - context.readGroups + ballRequest.cmd.op1_col := 1.U + ballRequest.cmd.op2_col := 1.U + ballRequest.cmd.wr_col := 1.U + ballRequest.cmd.wr_read_bank := 0.U + ballRequest.cmd.wr_arch_bank := ballLocal(2) + io.ball.request.valid := state === ballSend + io.ball.response.ready := state === ballWait + when(io.ball.request.fire)(state := ballWait) + when(io.ball.request.valid) { + for (i <- 0 until 2) { + assert(!ballUsed(i) || ballLocal(i) < ballGroups, "GP Ball read bank outside launch groups") + } + assert( + !ballUsed(2) || !ballFromRead(2) && ballLocal(2) < ballGroups, + "GP Ball write outside write groups" + ) + } + + val running = state =/= idle && state =/= completed + io.launch.ready := state === idle + io.done.valid := state === completed + io.done.bits.fault := fault + io.done.bits.pc := pc + io.done.bits.instruction := instruction + io.done.bits.cycles := cycles + io.done.bits.cause := cause + io.done.bits.tval := tval + io.done.bits.fflags := fflags + io.done.bits.vxsat := vxsat + + for (i <- 0 until 2) { + iBuf(i).io.upload.valid := io.program.valid && io.program.bits.buffer === i.U + iBuf(i).io.upload.bits := io.program.bits + iBuf(i).io.uploadEnabled := !io.launch.valid && (!running || activeBuffer =/= i.U) + iBuf(i).io.fetchEnabled := state === fetch && activeBuffer === i.U && pc < end + iBuf(i).io.fetchAddress := pc + } + io.program.ready := Mux(io.program.bits.buffer, iBuf(1).io.upload.ready, iBuf(0).io.upload.ready) + val imageWords = Mux(io.launch.bits.iBuffer, iBuf(1).io.wordCount, iBuf(0).io.wordCount) + val fetchLoaded = Mux(activeBuffer, iBuf(1).io.loaded, iBuf(0).io.loaded) + val fetchedInstruction = Mux(activeBuffer, iBuf(1).io.instruction, iBuf(0).io.instruction) + + registers.io.initialize.valid := io.launch.fire + registers.io.initialize.bits := io.launch.bits + registers.io.xReadAddress1 := instruction(19, 15) + registers.io.xReadAddress2 := instruction(24, 20) + registers.io.fReadAddress1 := instruction(19, 15) + registers.io.fReadAddress2 := instruction(24, 20) + registers.io.fReadAddress3 := instruction(31, 27) + registers.io.xWrite.valid := false.B + registers.io.xWrite.bits.address := instruction(11, 7) + registers.io.xWrite.bits.data := 0.U + registers.io.fWrite.valid := false.B + registers.io.fWrite.bits.address := instruction(11, 7) + registers.io.fWrite.bits.data := 0.U + scalar.io.divRequest.valid := false.B + scalar.io.divRequest.bits.a := registers.io.xReadData1 + scalar.io.divRequest.bits.b := registers.io.xReadData2 + scalar.io.divRequest.bits.sew := Mux(instruction(6, 0) === "h3b".U, 2.U, 3.U) + scalar.io.divRequest.bits.signed := !instruction(12) + scalar.io.divRequest.bits.remainder := instruction(13) + scalar.io.divResponse.ready := state === scalarWait && !io.launch.fire + scalar.io.clear := io.launch.fire + scalar.io.instruction := instruction + scalar.io.pc := pc + scalar.io.source1 := registers.io.xReadData1 + scalar.io.source2 := registers.io.xReadData2 + floating.io.instruction := instruction + floating.io.source1 := registers.io.fReadData1 + floating.io.source2 := registers.io.fReadData2 + floating.io.source3 := registers.io.fReadData3 + floating.io.xSource := registers.io.xReadData1 + floating.io.frm := frm + floating.io.start := false.B + vector.io.initialize := false.B + vector.io.roundingMode := frm + vector.io.vxrm := vxrm + vector.io.vstartWrite.valid := false.B + vector.io.vstartWrite.bits := 0.U + vector.io.issue.valid := false.B + vector.io.issue.bits.instruction := instruction + vector.io.issue.bits.scalar1 := registers.io.xReadData1 + vector.io.issue.bits.scalar2 := registers.io.xReadData2 + vector.io.issue.bits.floating := registers.io.fReadData1 + vector.io.result.ready := state === vectorWait + + for (i <- 0 until p.memoryPorts) { + io.memoryRequest(i) <> vector.io.memoryRequest(i) + vector.io.memoryResponse(i) <> io.memoryResponse(i) + } + when(io.ball.response.fire) { + registers.io.xWrite.valid := instruction(11, 7) =/= 0.U + registers.io.xWrite.bits.data := 0.U + pc := pc + 4.U + state := fetch + } + when(state === scalarWait && scalar.io.divResponse.fire) { + registers.io.xWrite.valid := true.B + registers.io.xWrite.bits.address := scalarTarget + registers.io.xWrite.bits.data := Mux( + instruction(6, 0) === "h3b".U, + Cat(Fill(32, scalar.io.divResponse.bits(31)), scalar.io.divResponse.bits(31, 0)), + scalar.io.divResponse.bits + ) + pc := scalarNextPc + state := fetch + } + when(state === memorySend || state === memoryWait) { + io.memoryRequest(0).valid := state === memorySend + io.memoryRequest(0).bits := memory + vector.io.memoryRequest(0).ready := false.B + vector.io.memoryResponse(0).valid := false.B + io.memoryResponse(0).ready := state === memoryWait + } + + val opcode = instruction(6, 0) + val funct3 = instruction(14, 12) + val vectorInstruction = opcode === "h57".U || + (opcode === "h07".U || opcode === "h27".U) && (funct3 === 0.U || funct3 >= 5.U) + val floatingInstruction = opcode === "h53".U || opcode === "h43".U || opcode === "h47".U || + opcode === "h4b".U || opcode === "h4f".U + val load = opcode === "h03".U || opcode === "h07".U + val store = opcode === "h23".U || opcode === "h27".U + val csrAddress = instruction(31, 20) + val csrKnown = WireDefault(true.B) + val csrValue = WireDefault(0.U(64.W)) + switch(csrAddress) { + is("h001".U)(csrValue := fflags) + is("h002".U)(csrValue := frm) + is("h003".U)(csrValue := Cat(frm, fflags)) + is("h008".U)(csrValue := vector.io.vstart) + is("h009".U)(csrValue := vxsat) + is("h00a".U)(csrValue := vxrm) + is("h00f".U)(csrValue := Cat(vxrm, vxsat)) + is("hc20".U)(csrValue := vector.io.vl) + is("hc21".U)(csrValue := vector.io.vtype) + is("hc22".U)(csrValue := (p.vLen / 8).U) + } + csrKnown := Seq(0x001, 0x002, 0x003, 0x008, 0x009, 0x00a, 0x00f, 0xc20, 0xc21, 0xc22) + .map(a => csrAddress === a.U).reduce(_ || _) + val csrSource = Mux(funct3(2), instruction(19, 15), registers.io.xReadData1) + val csrWrite = funct3(1, 0) === 1.U || instruction(19, 15) =/= 0.U + + val csrNew = MuxLookup(funct3(1, 0), csrValue)(Seq( + 1.U -> csrSource, + 2.U -> (csrValue | csrSource), + 3.U -> (csrValue & ~csrSource) + )) + + when(running)(cycles := cycles + 1.U) + when(io.done.fire)(state := idle) + when(io.launch.fire) { + context := io.launch.bits.context + activeBuffer := io.launch.bits.iBuffer + pc := io.launch.bits.entry + end := io.launch.bits.end + instruction := 0.U + cycles := 0.U + fflags := 0.U + frm := 0.U + fault := false.B + cause := 0.U + tval := 0.U + state := fetch + when(io.launch.bits.entry(1, 0).orR || io.launch.bits.end(1, 0).orR) { + finish(true.B, 0.U, io.launch.bits.entry) + }.elsewhen(io.launch.bits.entry >= io.launch.bits.end || io.launch.bits.end > imageWords * 4.U) { + finish(true.B, 1.U, io.launch.bits.entry) + } + } + when(state === fetch) { + when(pc === end)(finish(false.B, 0.U, 0.U)) + .elsewhen(pc(1, 0).orR)(finish(true.B, 0.U, pc)) + .elsewhen(pc >= end || !fetchLoaded)(finish(true.B, 1.U, pc)) + .otherwise(state := fetchWait) + } + when(state === fetchWait) { instruction := fetchedInstruction; state := execute } + when(state === execute) { + when(opcode === "h7b".U && funct3 === 3.U) { + state := ballSend + }.elsewhen(vectorInstruction) { + vector.io.issue.valid := true.B + when(vector.io.issue.fire)(state := vectorWait) + }.elsewhen(floatingInstruction) { + when(!floating.io.legal)(finish(true.B, 2.U, instruction)) + .otherwise { + floating.io.start := floating.io.ready + when(floating.io.ready)(state := floatWait) + } + }.elsewhen(load || store) { + val fp = opcode === "h07".U || opcode === "h27".U + val size = Mux(fp, Mux(funct3 === 3.U, 3.U, 2.U), funct3(1, 0)) + val legal = Mux( + fp, + funct3 === 2.U || (p.eLen == 64).B && funct3 === 3.U, + Mux(store, funct3 <= 3.U, funct3 <= 6.U) + ) + val immediate = Mux( + store, + Cat(Fill(52, instruction(31)), instruction(31, 25), instruction(11, 7)), + Cat(Fill(52, instruction(31)), instruction(31, 20)) + ) + val address = registers.io.xReadData1 + immediate + memory.address := address + memory.write := store + memory.data := Mux(fp, registers.io.fReadData2, registers.io.xReadData2) + memory.size := size + memory.mask := MuxLookup(size, 1.U(8.W))(Seq(0.U -> 1.U, 1.U -> 3.U, 2.U -> 15.U, 3.U -> 255.U)) + memoryFloating := fp + memoryUnsigned := funct3(2) + when(!legal)(finish(true.B, 2.U, instruction)) + .otherwise(state := memorySend) + }.elsewhen(opcode === "h73".U) { + when(funct3 === 0.U) { + when(instruction === "h00100073".U)(finish(true.B, 3.U, pc)) + .elsewhen(instruction === "h00000073".U)(finish(true.B, 11.U, 0.U)) + .otherwise(finish(true.B, 2.U, instruction)) + }.elsewhen(funct3 === 4.U || !csrKnown || csrWrite && csrAddress(11, 10) === 3.U) { + finish(true.B, 2.U, instruction) + }.otherwise { + registers.io.xWrite.valid := true.B + registers.io.xWrite.bits.data := csrValue + when(csrWrite) { + switch(csrAddress) { + is("h001".U)(fflags := csrNew(4, 0)) + is("h002".U)(frm := csrNew(2, 0)) + is("h003".U) { frm := csrNew(7, 5); fflags := csrNew(4, 0) } + is("h008".U) { vector.io.vstartWrite.valid := true.B; vector.io.vstartWrite.bits := csrNew } + is("h009".U)(vxsat := csrNew(0)) + is("h00a".U)(vxrm := csrNew(1, 0)) + is("h00f".U) { vxrm := csrNew(2, 1); vxsat := csrNew(0) } + } + } + pc := pc + 4.U + state := fetch + } + }.elsewhen(!scalar.io.legal)(finish(true.B, 2.U, instruction)) + .elsewhen(scalar.io.nextPc(63, 32).orR)(finish(true.B, 1.U, scalar.io.nextPc)) + .elsewhen(scalar.io.nextPc(1, 0).orR)(finish(true.B, 0.U, scalar.io.nextPc)) + .elsewhen(scalarDivide) { + scalar.io.divRequest.valid := true.B + when(scalar.io.divRequest.fire) { + scalarTarget := scalar.io.destination + scalarNextPc := scalar.io.nextPc + state := scalarWait + } + }.otherwise { + registers.io.xWrite.valid := scalar.io.write + registers.io.xWrite.bits.data := scalar.io.result + pc := scalar.io.nextPc + state := fetch + } + } + when(state === memorySend && io.memoryRequest(0).fire)(state := memoryWait) + when(state === memoryWait && io.memoryResponse(0).fire) { + when(io.memoryResponse(0).bits.error)(finish(true.B, Mux(memory.write, 7.U, 5.U), memory.address)) + .otherwise { + when(!memory.write) { + val data = io.memoryResponse(0).bits.data + when(memoryFloating) { + registers.io.fWrite.valid := true.B + registers.io.fWrite.bits.data := Mux(memory.size === 2.U, Cat("hffffffff".U(32.W), data(31, 0)), data) + }.otherwise { + registers.io.xWrite.valid := true.B + registers.io.xWrite.bits.data := MuxLookup(memory.size, data)(Seq( + 0.U -> Cat(Fill(56, !memoryUnsigned && data(7)), data(7, 0)), + 1.U -> Cat(Fill(48, !memoryUnsigned && data(15)), data(15, 0)), + 2.U -> Cat(Fill(32, !memoryUnsigned && data(31)), data(31, 0)) + )) + } + } + pc := pc + 4.U + state := fetch + } + } + when((state === floatWait || state === execute && floatingInstruction) && floating.io.valid) { + when(!floating.io.legal)(finish(true.B, 2.U, instruction)) + .otherwise { + registers.io.fWrite.valid := floating.io.floatWrite + registers.io.fWrite.bits.data := floating.io.result + registers.io.xWrite.valid := floating.io.integerWrite + registers.io.xWrite.bits.data := floating.io.result + fflags := fflags | floating.io.flags + pc := pc + 4.U + state := fetch + } + } + when(vector.io.result.fire) { + when(vector.io.result.bits.fault)(finish(true.B, vector.io.result.bits.cause, vector.io.result.bits.tval)) + .otherwise { + registers.io.xWrite.valid := vector.io.result.bits.scalarWrite + registers.io.xWrite.bits.data := vector.io.result.bits.scalarData + registers.io.fWrite.valid := vector.io.result.bits.floatWrite + registers.io.fWrite.bits.data := vector.io.result.bits.floatData + fflags := fflags | vector.io.result.bits.flags + vxsat := vxsat || vector.io.result.bits.saturated + pc := pc + 4.U + state := fetch + } + } +} diff --git a/arch/src/main/scala/framework/rvv/src/main/scala/frontend/IBuf.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/IBuf.scala similarity index 88% rename from arch/src/main/scala/framework/rvv/src/main/scala/frontend/IBuf.scala rename to arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/IBuf.scala index 0bd92d18c..2c6750233 100644 --- a/arch/src/main/scala/framework/rvv/src/main/scala/frontend/IBuf.scala +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/IBuf.scala @@ -1,13 +1,14 @@ -package framework.rvv +package framework.acc.gpdomain.execution + +import framework.acc.gpdomain.configs.GpDomainParam import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public} -import framework.top.GlobalConfig @instantiable -class IBuf(val b: GlobalConfig) extends Module { - val p = b.rvv +class IBuf(val gpParam: GpDomainParam) extends Module { + val p = gpParam.execution @public val io = IO(new Bundle { diff --git a/arch/src/main/scala/framework/rvv/src/main/scala/execution/FALU.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/arithmetic/FALU.scala similarity index 98% rename from arch/src/main/scala/framework/rvv/src/main/scala/execution/FALU.scala rename to arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/arithmetic/FALU.scala index 5a964cc48..2a9a8a3c0 100644 --- a/arch/src/main/scala/framework/rvv/src/main/scala/execution/FALU.scala +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/arithmetic/FALU.scala @@ -1,4 +1,6 @@ -package framework.rvv +package framework.acc.gpdomain.execution.arithmetic + +import framework.acc.gpdomain.configs.GpDomainParam import chisel3._ import chisel3.util._ @@ -6,7 +8,9 @@ import chisel3.experimental.hierarchy.{instantiable, public} import hardfloat._ @instantiable -class FALU(val maxWidth: Int) extends Module { +class FALU(gpParam: GpDomainParam) extends Module { + val maxWidth = gpParam.eLen + require(maxWidth == 32 || maxWidth == 64) @public diff --git a/arch/src/main/scala/framework/rvv/src/main/scala/execution/IALU.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/arithmetic/IALU.scala similarity index 81% rename from arch/src/main/scala/framework/rvv/src/main/scala/execution/IALU.scala rename to arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/arithmetic/IALU.scala index dc053c515..96d875784 100644 --- a/arch/src/main/scala/framework/rvv/src/main/scala/execution/IALU.scala +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/arithmetic/IALU.scala @@ -1,6 +1,8 @@ -package framework.rvv +package framework.acc.gpdomain.execution.arithmetic -import framework.arith.{DivRem, DivRemRequest} +import framework.acc.gpdomain.configs.GpDomainParam + +import framework.lib.arith.{DivRem, DivRemRequest} import chisel3._ import chisel3.util._ @@ -8,7 +10,9 @@ import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantia /** Element arithmetic shared by the e8/e16/e32/e64 integer vector instructions. */ @instantiable -class IALU(val maxWidth: Int) extends Module { +class IALU(gpParam: GpDomainParam) extends Module { + val maxWidth = gpParam.eLen + require(maxWidth == 32 || maxWidth == 64) val maxSew = log2Ceil(maxWidth / 8) val productWidth = 2 * maxWidth @@ -32,14 +36,15 @@ class IALU(val maxWidth: Int) extends Module { val legal = Output(Bool()) }) - val divider: Instance[DivRem] = Instantiate(new DivRem(maxWidth)) + val divider: Instance[DivRem] = + Instantiate(new DivRem(maxWidth)) divider.io.request <> io.divRequest io.divResponse <> divider.io.response divider.io.clear := io.clear val mask = MuxLookup(io.sew, Fill(maxWidth, 1.U(1.W)))(Seq(0.U -> "hff".U, 1.U -> "hffff".U, 2.U -> "hffffffff".U)) val a = io.a & mask - val b = io.b & mask + val rhs = io.b & mask val c = io.c & mask val sa = MuxLookup(io.sew, a)(Seq( @@ -48,19 +53,19 @@ class IALU(val maxWidth: Int) extends Module { 2.U -> a(31, 0).asSInt.pad(maxWidth).asUInt )).asSInt - val sb = MuxLookup(io.sew, b)(Seq( - 0.U -> b(7, 0).asSInt.pad(maxWidth).asUInt, - 1.U -> b(15, 0).asSInt.pad(maxWidth).asUInt, - 2.U -> b(31, 0).asSInt.pad(maxWidth).asUInt + val sb = MuxLookup(io.sew, rhs)(Seq( + 0.U -> rhs(7, 0).asSInt.pad(maxWidth).asUInt, + 1.U -> rhs(15, 0).asSInt.pad(maxWidth).asUInt, + 2.U -> rhs(31, 0).asSInt.pad(maxWidth).asUInt )).asSInt val width = 8.U(7.W) << io.sew - val shift = b(5, 0) & (width - 1.U) + val shift = rhs(5, 0) & (width - 1.U) val signedProduct = sa * sb - val unsignedProduct = a * b - val mixedProduct = sa * Cat(0.U(1.W), b).asSInt + val unsignedProduct = a * rhs + val mixedProduct = sa * Cat(0.U(1.W), rhs).asSInt val reverseMixedProduct = Cat(0.U(1.W), a).asSInt * sb - val unsignedSum = a +& b + val unsignedSum = a +& rhs val signedSum = sa +& sb val signedDifference = sa -& sb val maximum = Cat(0.U(1.W), mask(maxWidth - 1, 1)).asSInt @@ -68,7 +73,7 @@ class IALU(val maxWidth: Int) extends Module { val signedOverflow = signedSum > maximum || signedSum < minimum val differenceOverflow = signedDifference > maximum || signedDifference < minimum val carrySum = unsignedSum + io.carry - val borrow = a < (b +& io.carry) + val borrow = a < (rhs +& io.carry) val wideMask = MuxLookup(io.sew, Fill(maxWidth, 1.U(1.W)))(Seq(0.U -> "hffff".U, 1.U -> "hffffffff".U)) val wideA = io.a & wideMask @@ -104,7 +109,7 @@ class IALU(val maxWidth: Int) extends Module { switch(io.funct6) { is(8.U)(roundInput := unsignedSum.zext) is(9.U)(roundInput := signedSum) - is(10.U)(roundInput := a.zext - b.zext) + is(10.U)(roundInput := a.zext - rhs.zext) is(11.U)(roundInput := signedDifference) } }.otherwise { @@ -150,13 +155,13 @@ class IALU(val maxWidth: Int) extends Module { is(37.U)(io.result := unsignedProduct & mask) is(38.U)(io.result := (mixedProduct.asUInt >> width) & mask) is(39.U)(io.result := (signedProduct.asUInt >> width) & mask) - is(41.U)(io.result := (b * c + a) & mask) - is(43.U)(io.result := (a - b * c) & mask) + is(41.U)(io.result := (rhs * c + a) & mask) + is(43.U)(io.result := (a - rhs * c) & mask) is(45.U)(io.result := (unsignedProduct + c) & mask) is(47.U)(io.result := (c - unsignedProduct) & mask) - is(48.U, 52.U)(io.result := (wideFirst +& b) & wideMask) + is(48.U, 52.U)(io.result := (wideFirst +& rhs) & wideMask) is(49.U, 53.U)(io.result := (signedWideFirst +& sb).asUInt & wideMask) - is(50.U, 54.U)(io.result := (wideFirst - b) & wideMask) + is(50.U, 54.U)(io.result := (wideFirst - rhs) & wideMask) is(51.U, 55.U)(io.result := (signedWideFirst - sb).asUInt & wideMask) is(56.U)(io.result := unsignedProduct & wideMask) is(58.U)(io.result := mixedProduct.asUInt & wideMask) @@ -173,35 +178,35 @@ class IALU(val maxWidth: Int) extends Module { (io.sew < maxSew.U && io.funct6 >= 48.U && io.funct6 =/= 57.U)) }.otherwise { switch(io.funct6) { - is(0.U)(io.result := (a + b) & mask) - is(2.U)(io.result := (a - b) & mask) - is(3.U)(io.result := (b - a) & mask) - is(4.U)(io.result := Mux(a < b, a, b)) - is(5.U)(io.result := Mux(sa < sb, a, b)) - is(6.U)(io.result := Mux(a > b, a, b)) - is(7.U)(io.result := Mux(sa > sb, a, b)) - is(9.U)(io.result := a & b) - is(10.U)(io.result := a | b) - is(11.U)(io.result := a ^ b) + is(0.U)(io.result := (a + rhs) & mask) + is(2.U)(io.result := (a - rhs) & mask) + is(3.U)(io.result := (rhs - a) & mask) + is(4.U)(io.result := Mux(a < rhs, a, rhs)) + is(5.U)(io.result := Mux(sa < sb, a, rhs)) + is(6.U)(io.result := Mux(a > rhs, a, rhs)) + is(7.U)(io.result := Mux(sa > sb, a, rhs)) + is(9.U)(io.result := a & rhs) + is(10.U)(io.result := a | rhs) + is(11.U)(io.result := a ^ rhs) is(16.U)(io.result := carrySum & mask) is(17.U)(io.result := (carrySum >> width)(0)) - is(18.U)(io.result := (a - b - io.carry) & mask) + is(18.U)(io.result := (a - rhs - io.carry) & mask) is(19.U)(io.result := borrow) - is(23.U)(io.result := Mux(io.carry, b, a)) - is(24.U)(io.result := a === b) - is(25.U)(io.result := a =/= b) - is(26.U)(io.result := a < b) + is(23.U)(io.result := Mux(io.carry, rhs, a)) + is(24.U)(io.result := a === rhs) + is(25.U)(io.result := a =/= rhs) + is(26.U)(io.result := a < rhs) is(27.U)(io.result := sa < sb) - is(28.U)(io.result := a <= b) + is(28.U)(io.result := a <= rhs) is(29.U)(io.result := sa <= sb) - is(30.U)(io.result := a > b) + is(30.U)(io.result := a > rhs) is(31.U)(io.result := sa > sb) is(32.U) { io.result := Mux(unsignedSum > mask, mask, unsignedSum); io.saturated := unsignedSum > mask } is(33.U) { io.result := Mux(signedOverflow, Mux(sa < 0.S, minimum, maximum), signedSum).asUInt & mask io.saturated := signedOverflow } - is(34.U) { io.result := Mux(a < b, 0.U, a - b); io.saturated := a < b } + is(34.U) { io.result := Mux(a < rhs, 0.U, a - rhs); io.saturated := a < rhs } is(35.U) { io.result := Mux(differenceOverflow, Mux(sa < 0.S, minimum, maximum), signedDifference).asUInt & mask io.saturated := differenceOverflow diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..9634e9caa --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/configs/Params.scala @@ -0,0 +1,26 @@ +package framework.acc.gpdomain.execution.configs + +import chisel3.util.{isPow2, log2Ceil} + +case class Params( + laneNumber: Int, + vLen: Int, + eLen: Int, + iBufWords: Int, + memoryPorts: Int) { + require(laneNumber > 0) + require(vLen >= 128 && vLen >= eLen && isPow2(vLen)) + require(eLen == 32 || eLen == 64) + require(iBufWords > 0 && isPow2(iBufWords)) + require(memoryPorts > 0) + require(laneNumber <= memoryPorts) + + val wordBits: Int = eLen + val wordOffsetBits: Int = log2Ceil(wordBits) + val maxSew: Int = log2Ceil(eLen / 8) + val wordsPerRegister: Int = vLen / wordBits + val registerBanks: Int = math.min(laneNumber, wordsPerRegister) + val elementsPerRegister: Int = vLen / eLen + + require(isPow2(registerBanks), "RVV register bank count must be a power of two") +} diff --git a/arch/src/main/scala/framework/rvv/src/main/scala/execution/ScalarExecution.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/scalar/ScalarExecution.scala similarity index 97% rename from arch/src/main/scala/framework/rvv/src/main/scala/execution/ScalarExecution.scala rename to arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/scalar/ScalarExecution.scala index 1c5292e3c..337b8e6cd 100644 --- a/arch/src/main/scala/framework/rvv/src/main/scala/execution/ScalarExecution.scala +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/scalar/ScalarExecution.scala @@ -1,13 +1,13 @@ -package framework.rvv +package framework.acc.gpdomain.execution.scalar -import framework.arith.{DivRem, DivRemRequest} +import framework.lib.arith.{DivRem, DivRemRequest} import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} @instantiable -class ScalarExecution extends Module { +class ScalarExecution() extends Module { @public val io = IO(new Bundle { diff --git a/arch/src/main/scala/framework/rvv/src/main/scala/execution/ScalarFloating.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/scalar/ScalarFloating.scala similarity index 93% rename from arch/src/main/scala/framework/rvv/src/main/scala/execution/ScalarFloating.scala rename to arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/scalar/ScalarFloating.scala index 72bb0023d..ab978be64 100644 --- a/arch/src/main/scala/framework/rvv/src/main/scala/execution/ScalarFloating.scala +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/scalar/ScalarFloating.scala @@ -1,11 +1,15 @@ -package framework.rvv +package framework.acc.gpdomain.execution.scalar +import framework.acc.gpdomain.configs.GpDomainParam + +import framework.acc.gpdomain.execution.arithmetic.FALU import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} @instantiable -class ScalarFloating(val maxWidth: Int) extends Module { +class ScalarFloating(gpParam: GpDomainParam) extends Module { + val maxWidth = gpParam.eLen @public val io = IO(new Bundle { @@ -35,7 +39,8 @@ class ScalarFloating(val maxWidth: Int) extends Module { val usesRounding = fmaInstruction || Seq(0, 4, 8, 12, 0x2c, 0x20, 0x60, 0x68).map(n => operation === n.U).reduce(_ || _) val rounding = Mux(usesRounding, Mux(funct === 7.U, io.frm, funct), 0.U) - val falu: Instance[FALU] = Instantiate(new FALU(maxWidth)) + val falu: Instance[FALU] = + Instantiate(new FALU(gpParam)) def unbox(source: UInt): UInt = Mux(double, source, Mux(source(63, 32).andR, source(31, 0), "h7fc00000".U)) falu.io.a := unbox(io.source1) falu.io.b := unbox(io.source2) diff --git a/arch/src/main/scala/framework/rvv/src/main/scala/register/ScalarRF.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/scalar/ScalarRF.scala similarity index 80% rename from arch/src/main/scala/framework/rvv/src/main/scala/register/ScalarRF.scala rename to arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/scalar/ScalarRF.scala index c237121fc..3baa5e393 100644 --- a/arch/src/main/scala/framework/rvv/src/main/scala/register/ScalarRF.scala +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/scalar/ScalarRF.scala @@ -1,15 +1,19 @@ -package framework.rvv +package framework.acc.gpdomain.execution.scalar +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import framework.acc.gpdomain.execution.KernelLaunch import chisel3._ import chisel3.util.Valid import chisel3.experimental.hierarchy.{instantiable, public} @instantiable -class ScalarRF extends Module { +class ScalarRF(frontendParam: FrontendParam, memParam: MemDomainParam) extends Module { @public val io = IO(new Bundle { - val initialize = Flipped(Valid(new KernelLaunch)) + val initialize = Flipped(Valid(new KernelLaunch(frontendParam, memParam))) val xReadAddress1 = Input(UInt(5.W)) val xReadAddress2 = Input(UInt(5.W)) diff --git a/arch/src/main/scala/framework/rvv/src/main/scala/register/Operands.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/vector/Operands.scala similarity index 96% rename from arch/src/main/scala/framework/rvv/src/main/scala/register/Operands.scala rename to arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/vector/Operands.scala index 84db6ecfb..9b7a1b807 100644 --- a/arch/src/main/scala/framework/rvv/src/main/scala/register/Operands.scala +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/vector/Operands.scala @@ -1,13 +1,14 @@ -package framework.rvv +package framework.acc.gpdomain.execution.vector + +import framework.acc.gpdomain.configs.GpDomainParam import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import framework.top.GlobalConfig @instantiable -class Operands(val b: GlobalConfig) extends Module { - val p = b.rvv +class Operands(val gpParam: GpDomainParam) extends Module { + val p = gpParam.execution val wordsPerRegister = p.wordsPerRegister val bankCount = p.registerBanks val bankBits = log2Ceil(bankCount) @@ -16,7 +17,8 @@ class Operands(val b: GlobalConfig) extends Module { val snapshotWords = 8 * wordsPerRegister val snapshotBits = log2Ceil(snapshotWords) - @public val io = IO(new Bundle { + @public + val io = IO(new Bundle { val initialize = Input(Bool()) val busy = Output(Bool()) @@ -45,7 +47,7 @@ class Operands(val b: GlobalConfig) extends Module { val snapshotDone = Decoupled(Bool()) }) - val vrf = Instantiate(new VRF(b)) + val vrf = Instantiate(new VRF(gpParam)) vrf.io.initialize := io.initialize val idle :: reading :: readDone :: writing :: writeDone :: copying :: copyDone :: Nil = Enum(7) val state = RegInit(idle) diff --git a/arch/src/main/scala/framework/rvv/src/main/scala/register/VRF.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/vector/VRF.scala similarity index 94% rename from arch/src/main/scala/framework/rvv/src/main/scala/register/VRF.scala rename to arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/vector/VRF.scala index da415034e..e708c3e0a 100644 --- a/arch/src/main/scala/framework/rvv/src/main/scala/register/VRF.scala +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/vector/VRF.scala @@ -1,13 +1,14 @@ -package framework.rvv +package framework.acc.gpdomain.execution.vector + +import framework.acc.gpdomain.configs.GpDomainParam import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public} -import framework.top.GlobalConfig @instantiable -class VRF(val b: GlobalConfig) extends Module { - val p = b.rvv +class VRF(val gpParam: GpDomainParam) extends Module { + val p = gpParam.execution val wordBits = p.wordBits val wordsPerRegister = p.vLen / wordBits val bankCount = p.registerBanks diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/vector/VectorThread.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/vector/VectorThread.scala new file mode 100644 index 000000000..8a01f93c5 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/vector/VectorThread.scala @@ -0,0 +1,941 @@ +package framework.acc.gpdomain.execution.vector + +import framework.acc.gpdomain.configs.GpDomainParam + +import framework.acc.gpdomain.execution.{VectorMemoryRequest, VectorMemoryResponse} +import framework.acc.gpdomain.execution.arithmetic.{FALU, IALU} +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} + +class VectorIssue extends Bundle { + val instruction = UInt(32.W) + val scalar1 = UInt(64.W) + val scalar2 = UInt(64.W) + val floating = UInt(64.W) +} + +class VectorResult extends Bundle { + val fault = Bool() + val cause = UInt(32.W) + val tval = UInt(64.W) + val scalarWrite = Bool() + val scalarData = UInt(64.W) + val floatWrite = Bool() + val floatData = UInt(64.W) + val flags = UInt(5.W) + val saturated = Bool() +} + +@instantiable +class VectorThread(val gpParam: GpDomainParam) extends Module { + val p = gpParam.execution + + @public + val io = IO(new Bundle { + val initialize = Input(Bool()) + val roundingMode = Input(UInt(3.W)) + val vxrm = Input(UInt(2.W)) + val vstartWrite = Flipped(Valid(UInt(64.W))) + val vl = Output(UInt(32.W)) + val vtype = Output(UInt(64.W)) + val vstart = Output(UInt(64.W)) + val issue = Flipped(Decoupled(new VectorIssue)) + val result = Decoupled(new VectorResult) + val memoryRequest = Vec(p.memoryPorts, Decoupled(new VectorMemoryRequest)) + val memoryResponse = + Vec(p.memoryPorts, Flipped(Decoupled(new VectorMemoryResponse))) + }) + + val idle :: snapshot :: capture :: execute :: commit :: Nil = Enum(5) + val state = RegInit(idle) + val resultValid = RegInit(false.B) + val result = RegInit(0.U.asTypeOf(new VectorResult)) + val instruction = Reg(UInt(32.W)) + val scalar1 = Reg(UInt(64.W)) + val scalar2 = Reg(UInt(64.W)) + val floatScalar = Reg(UInt(64.W)) + val vl = RegInit(0.U(32.W)) + val vtype = RegInit("h8000000000000000".U(64.W)) + val vstart = RegInit(0.U(64.W)) + val base = Reg(UInt(32.W)) + val readPortCount = math.max(p.laneNumber, p.memoryPorts) + val wordsPerRegister = p.wordsPerRegister + val maskWords = Reg(Vec(readPortCount, UInt(p.wordBits.W))) + val operandWords = Reg(Vec(readPortCount, UInt(p.wordBits.W))) + val sourceWords = Reg(Vec(readPortCount, UInt(p.wordBits.W))) + val destinationWords = Reg(Vec(readPortCount, UInt(p.wordBits.W))) + val readPhase = RegInit(0.U(3.W)) + val readSent = RegInit(false.B) + val snapshotSent = RegInit(false.B) + val firstBatch = RegInit(true.B) + val writeSent = RegInit(false.B) + val resumeState = Reg(UInt(3.W)) + val finishRequested = WireDefault(false.B) + val accumulator = Reg(UInt(p.wordBits.W)) + val count = Reg(UInt(32.W)) + val offered = RegInit(VecInit(Seq.fill(p.memoryPorts)(false.B))) + val issued = RegInit(VecInit(Seq.fill(p.memoryPorts)(false.B))) + val received = RegInit(VecInit(Seq.fill(p.memoryPorts)(false.B))) + val memoryData = Reg(Vec(p.memoryPorts, UInt(p.wordBits.W))) + val memoryFailed = RegInit(VecInit(Seq.fill(p.memoryPorts)(false.B))) + val divBatchValid = RegInit(false.B) + val divActive = RegInit(VecInit(Seq.fill(p.laneNumber)(false.B))) + val divStarted = RegInit(VecInit(Seq.fill(p.laneNumber)(false.B))) + val divDone = RegInit(VecInit(Seq.fill(p.laneNumber)(false.B))) + val divData = Reg(Vec(p.laneNumber, UInt(p.wordBits.W))) + val fpStarted = RegInit(VecInit(Seq.fill(p.laneNumber)(false.B))) + val fpDone = RegInit(VecInit(Seq.fill(p.laneNumber)(false.B))) + val fpData = Reg(Vec(p.laneNumber, UInt(p.wordBits.W))) + val fpFlags = Reg(Vec(p.laneNumber, UInt(5.W))) + val fpLegal = Reg(Vec(p.laneNumber, Bool())) + + val opcode = instruction(6, 0) + val rd = instruction(11, 7) + val kind = instruction(14, 12) + val rs1 = instruction(19, 15) + val rs2 = instruction(24, 20) + val vm = instruction(25) + val op = instruction(31, 26) + val sew = vtype(4, 3) + val lmul = Cat(vtype(2), vtype(2, 0)).asSInt + val configure = opcode === "h57".U && kind === 7.U + val load = opcode === "h07".U + val store = opcode === "h27".U + val memory = load || store + val fp = opcode === "h57".U && (kind === 1.U || kind === 5.U) + val integer = + opcode === "h57".U && (kind === 0.U || kind === 2.U || kind === 3.U || kind === 4.U || kind === 6.U) + val divide = integer && (kind === 2.U || kind === 6.U) && op >= 32.U && op <= 35.U + val multiply = kind === 2.U || kind === 6.U + val vectorOperand = kind === 0.U || kind === 1.U || kind === 2.U + val maskLogical = integer && kind === 2.U && op >= 24.U && op <= 31.U + + val compare = + (integer && !multiply && (op >= 24.U && op <= 31.U || op === 17.U || op === 19.U)) || + (fp && Seq(24, 25, 27, 28, 29, 31).map(n => op === n.U).reduce(_ || _)) + + val carryOperation = integer && !multiply && op >= 16.U && op <= 19.U + val carryMask = carryOperation && op(0) + val merge = op === 23.U && (fp || integer && !multiply) + val scalarMoveOut = + op === 16.U && rs1 === 0.U && vm && (kind === 2.U || kind === 1.U) + val scalarMoveIn = + op === 16.U && rs2 === 0.U && vm && (kind === 6.U || kind === 5.U) + val population = integer && kind === 2.U && op === 16.U && rs1 === 16.U + val first = integer && kind === 2.U && op === 16.U && rs1 === 17.U + val identity = + integer && kind === 2.U && op === 20.U && rs1 === 17.U && rs2 === 0.U + val intReduction = integer && kind === 2.U && op <= 7.U + val fpReduction = + fp && kind === 1.U && (op === 1.U || op === 3.U || op === 5.U || op === 7.U) + val reduction = intReduction || fpReduction + val wholeMove = integer && kind === 3.U && op === 39.U + val memMode = instruction(27, 26) + + val memWidth = MuxLookup(kind, 0.U(2.W))( + Seq(0.U -> 0.U, 5.U -> 1.U, 6.U -> 2.U, 7.U -> 3.U) + ) + + val wholeMemory = memory && memMode === 0.U && rs2 === 8.U + val indexed = memMode(0) + val memSew = Mux(indexed && !wholeMemory, sew, memWidth) + val intWide = integer && multiply && op >= 48.U + val intNarrow = integer && !multiply && op >= 44.U && op <= 47.U + val floatWide = fp && op === 18.U && rs1 >= 8.U && rs1 <= 15.U + val floatNarrow = fp && op === 18.U && rs1 >= 16.U && rs1 <= 23.U + val extension = integer && multiply && op === 18.U + + val sourceSew = Mux( + memory, + Mux(indexed, memWidth, memSew), + Mux( + intNarrow || floatNarrow || intWide && op >= 52.U && op <= 55.U, + sew +& 1.U, + Mux( + extension, + sew - Mux(rs1(2, 1) === 1.U, 3.U, Mux(rs1(2, 1) === 2.U, 2.U, 1.U)), + sew + ) + ) + ) + + val destinationSew = + Mux(memory, memSew, Mux(intWide || floatWide, sew +& 1.U, sew)) + + val limit = Mux( + wholeMemory, + ((instruction(31, 29) +& 1.U) * p.vLen.U) >> (memWidth +& 3.U), + Mux(wholeMove, ((rs1 +& 1.U) * p.vLen.U) >> (sew +& 3.U), vl) + ) + + def element(word: UInt, index: UInt, width: UInt): UInt = { + val shift = (index << (width.pad(3) + 3.U))(p.wordOffsetBits - 1, 0) + val mask = MuxLookup(width, Fill(p.wordBits, 1.U(1.W)))( + Seq(0.U -> 255.U, 1.U -> 65535.U, 2.U -> "hffffffff".U) + ) + ((word >> shift) & mask)(p.eLen - 1, 0) + } + + def maskElement(word: UInt, index: UInt): Bool = (word >> index(p.wordOffsetBits - 1, 0))(0) + + def wordAddress( + register: UInt, + index: UInt, + width: UInt, + maskBit: Bool = false.B + ): UInt = { + val shift = Mux(maskBit, index, index << (width.pad(3) + 3.U)) + val address = Mux(maskBit, register, register + (shift >> log2Ceil(p.vLen)))(4, 0) + val within = shift.pad(log2Ceil(p.vLen))(log2Ceil(p.vLen) - 1, 0) + address * wordsPerRegister.U + (within >> p.wordOffsetBits) + } + + def groupLegal(register: UInt, width: UInt): Bool = { + val exponent = lmul.pad(6) + width.zext - sew.zext + val registers = 1.U(6.W) << Mux(exponent > 0.S, exponent.asUInt, 0.U) + exponent >= -3.S && exponent <= 3.S && (register & (registers - 1.U)) === 0.U && register +& registers <= 32.U + } + + def overlapLegal( + destination: UInt, + destinationWidth: UInt, + operand: UInt, + operandWidth: UInt + ): Bool = { + val destinationExponent = lmul.pad(6) + destinationWidth.zext - sew.zext + val operandExponent = lmul.pad(6) + operandWidth.zext - sew.zext + val destinationCount = 1 + .U(6.W) << Mux(destinationExponent > 0.S, destinationExponent.asUInt, 0.U) + val operandCount = + 1.U(6.W) << Mux(operandExponent > 0.S, operandExponent.asUInt, 0.U) + val overlap = + destination < operand +& operandCount && operand < destination +& destinationCount + !overlap || destinationWidth === operandWidth || + (destinationWidth < operandWidth && destination === operand) || + (destinationWidth > operandWidth && operandExponent >= 0.S && destination +& destinationCount === operand +& operandCount) + } + + def maximum(width: UInt, multiplier: UInt): UInt = { + val elements = p.vLen.U(32.W) >> (width.pad(3) + 3.U) + Mux(multiplier(2), elements >> (8.U - multiplier), elements << multiplier) + } + + val vlmax = maximum(sew, vtype(2, 0)) + val registers: Instance[Operands] = Instantiate(new Operands(gpParam)) + registers.io.initialize := io.initialize + val writes = Wire(chiselTypeOf(registers.io.write.bits)) + val pendingWrites = Reg(chiselTypeOf(registers.io.write.bits)) + for (lane <- 0 until p.laneNumber) { + writes(lane) := 0.U.asTypeOf(writes(lane)) + } + registers.io.write.valid := state === commit && !writeSent && !io.initialize + registers.io.write.bits := pendingWrites + registers.io.writeDone.ready := state === commit && writeSent && !io.initialize + when(registers.io.write.fire)(writeSent := true.B) + when(registers.io.writeDone.fire) { + writeSent := false.B + state := resumeState + } + + def writeElement( + lane: Int, + index: UInt, + width: UInt, + data: UInt, + maskBit: Bool = false.B + ): Unit = { + val bitOffset = Mux(maskBit, index, index << (width.pad(3) + 3.U)) + val within = bitOffset.pad(log2Ceil(p.vLen))(log2Ceil(p.vLen) - 1, 0) + val mask = Mux( + maskBit, + 1.U(p.wordBits.W), + MuxLookup(width, Fill(p.wordBits, 1.U(1.W)))( + Seq(0.U -> 255.U, 1.U -> 65535.U, 2.U -> "hffffffff".U) + ) + ) + writes(lane).valid := true.B + writes(lane).address := wordAddress(rd, index, width, maskBit) + writes(lane).mask := (mask << within(p.wordOffsetBits - 1, 0))(p.wordBits - 1, 0) + writes(lane).data := (data << within(p.wordOffsetBits - 1, 0))(p.wordBits - 1, 0) + } + + def finish( + fault: Bool = false.B, + cause: UInt = 0.U, + value: UInt = 0.U + ): Unit = { + finishRequested := true.B + state := idle + resultValid := true.B + result.fault := fault + result.cause := cause + result.tval := value + when(!fault)(vstart := 0.U) + } + + def nextBatch(step: UInt): Unit = { + when(base + step >= limit)(finish()).otherwise { + base := base + step + state := capture + readPhase := 0.U + } + offered.foreach(_ := false.B) + issued.foreach(_ := false.B) + received.foreach(_ := false.B) + memoryFailed.foreach(_ := false.B) + divBatchValid := false.B + divStarted.foreach(_ := false.B) + divDone.foreach(_ := false.B) + fpStarted.foreach(_ := false.B) + fpDone.foreach(_ := false.B) + } + + io.vl := vl + io.vtype := vtype + io.vstart := vstart + io.issue.ready := state === idle && !resultValid && !registers.io.busy && !io.initialize + io.result.valid := resultValid && state === idle && !registers.io.busy + io.result.bits := result + when(io.result.fire)(resultValid := false.B) + when(io.vstartWrite.valid)(vstart := io.vstartWrite.bits) + when(io.issue.fire) { + instruction := io.issue.bits.instruction + scalar1 := io.issue.bits.scalar1 + scalar2 := io.issue.bits.scalar2 + floatScalar := io.issue.bits.floating + state := snapshot + readPhase := 0.U + readSent := false.B + snapshotSent := false.B + firstBatch := true.B + writeSent := false.B + result := 0.U.asTypeOf(new VectorResult) + base := vstart + count := 0.U + offered.foreach(_ := false.B) + issued.foreach(_ := false.B) + received.foreach(_ := false.B) + memoryFailed.foreach(_ := false.B) + divBatchValid := false.B + divStarted.foreach(_ := false.B) + divDone.foreach(_ := false.B) + fpStarted.foreach(_ := false.B) + fpDone.foreach(_ := false.B) + } + + val instructionLegal = Wire(Bool()) + val alu: Seq[Instance[IALU]] = Seq.fill(p.laneNumber)(Instantiate(new IALU(gpParam))) + val falu: Seq[Instance[FALU]] = Seq.fill(p.laneNumber)(Instantiate(new FALU(gpParam))) + val laneA = Wire(Vec(p.laneNumber, UInt(p.wordBits.W))) + val laneB = Wire(Vec(p.laneNumber, UInt(p.wordBits.W))) + val laneSelected = Wire(Vec(p.laneNumber, Bool())) + val laneValue = Wire(Vec(p.laneNumber, UInt(p.wordBits.W))) + val laneFlags = Wire(Vec(p.laneNumber, UInt(5.W))) + val laneLegal = Wire(Vec(p.laneNumber, Bool())) + val fpComplete = Wire(Vec(p.laneNumber, Bool())) + val gather = + integer && !multiply && (op === 12.U || op === 14.U && kind === 0.U) + val slide = + integer && !multiply && (op === 14.U || op === 15.U) && kind =/= 0.U + val slideOne = integer && multiply && (op === 14.U || op === 15.U) + + val operandReads = (0 until readPortCount).map { port => + val index = base + port.U + val scalar = Mux(kind === 3.U, Cat(Fill(59, rs1(4)), rs1), scalar1) + val floatingScalar = Mux(sew === 2.U && !floatScalar(63, 32).andR, "h7fc00000".U, floatScalar) + val b = + Mux( + vectorOperand, + element(operandWords(port), index, Mux(gather && op === 14.U, 1.U, sew)), + Mux(fp, floatingScalar, scalar) + ) + val slideOffset = Mux(slideOne, 1.U, Mux(kind === 3.U, rs1, scalar1)) + val sourceIndex = Mux(gather && !slide, b, Mux(op === 14.U, index - slideOffset, index + slideOffset)) + val permuteRead = gather || slide || slideOne + // element already keeps only the low group bit offset; upper index bits cannot affect its read. + val a = element( + sourceWords(port), + Mux(scalarMoveOut, 0.U, Mux(permuteRead, sourceIndex(31, 0), index)), + Mux(permuteRead, sew, sourceSew) + ) + val c = element(destinationWords(port), index, Mux(fp, sew, destinationSew)) + (a, b, c, sourceIndex) + } + + val permuteRead = gather || slide || slideOne + registers.io.snapshot.valid := state === snapshot && permuteRead && !snapshotSent && !io.initialize + registers.io.snapshot.bits := rs2 + registers.io.snapshotDone.ready := state === snapshot && snapshotSent && !io.initialize + when(registers.io.snapshot.fire)(snapshotSent := true.B) + when(state === snapshot && !permuteRead || registers.io.snapshotDone.fire) { + state := Mux(configure, execute, capture) + readPhase := 0.U + readSent := false.B + } + + registers.io.read.valid := state === capture && !readSent && !io.initialize + registers.io.readResult.ready := state === capture && readSent && !io.initialize + for (port <- 0 until readPortCount) { + val index = base + port.U + val maskSource = maskLogical || population || first + val sourceIndex = Mux(scalarMoveOut, 0.U, Mux(permuteRead, operandReads(port)._4(31, 0), index)) + val sourceWidth = Mux(scalarMoveOut || permuteRead, sew, sourceSew) + val shift = sourceIndex << (sourceWidth.pad(3) + 3.U) + registers.io.read.bits(port).address := MuxLookup(readPhase, wordAddress(rs1, 0.U, sew))( + Seq( + 0.U -> wordAddress(0.U, index, sew, true.B), + 1.U -> wordAddress(rs1, index, Mux(gather && op === 14.U, 1.U, sew), maskLogical), + 2.U -> Mux(permuteRead, shift >> p.wordOffsetBits, wordAddress(rs2, sourceIndex, sourceWidth, maskSource)), + 3.U -> wordAddress(rd, index, Mux(fp, sew, destinationSew)) + ) + ) + registers.io.read.bits(port).snapshot := readPhase === 2.U && permuteRead + } + when(registers.io.read.fire)(readSent := true.B) + when(registers.io.readResult.fire) { + readSent := false.B + switch(readPhase) { + is(0.U)(maskWords := registers.io.readResult.bits) + is(1.U)(operandWords := registers.io.readResult.bits) + is(2.U)(sourceWords := registers.io.readResult.bits) + is(3.U)(destinationWords := registers.io.readResult.bits) + is(4.U) { + accumulator := registers.io.readResult.bits(0) & MuxLookup(sew, Fill(p.eLen, 1.U(1.W)))( + Seq(0.U -> 255.U, 1.U -> 65535.U, 2.U -> "hffffffff".U) + ) + firstBatch := false.B + } + } + when(readPhase === 4.U || readPhase === 3.U && !firstBatch) { + state := execute + }.otherwise(readPhase := readPhase + 1.U) + } + + for (lane <- 0 until p.laneNumber) { + val index = base + lane.U + val selected = maskElement(maskWords(lane), index) + val scalar = Mux( + kind === 3.U, + Cat(Fill(59, rs1(4)), rs1), + scalar1 + ) + val floatingScalar = + Mux(sew === 2.U && !floatScalar(63, 32).andR, "h7fc00000".U, floatScalar) + val b = operandReads(lane)._2 + val slideOffset = Mux(slideOne, 1.U, Mux(kind === 3.U, rs1, scalar1)) + val up = op === 14.U + val sourceIndex = operandReads(lane)._4 + val a = operandReads(lane)._1 + laneA(lane) := a + laneB(lane) := b + laneSelected( + lane + ) := index < limit && (wholeMove || vm || selected || merge || carryOperation) && + (!slide || !up || index >= slideOffset) + alu(lane).io.divRequest.valid := state === execute && instructionLegal && divide && + divBatchValid && divActive(lane) && !divStarted(lane) && !io.initialize + alu(lane).io.divRequest.bits.a := a + alu(lane).io.divRequest.bits.b := b + alu(lane).io.divRequest.bits.sew := sew + alu(lane).io.divRequest.bits.signed := op(0) + alu(lane).io.divRequest.bits.remainder := op(1) + alu(lane).io.divResponse.ready := state === execute && divide && divBatchValid && + divStarted(lane) && !divDone(lane) && !io.initialize + alu(lane).io.clear := io.initialize + when(alu(lane).io.divRequest.fire)(divStarted(lane) := true.B) + when(alu(lane).io.divResponse.fire) { + divData(lane) := alu(lane).io.divResponse.bits + divDone(lane) := true.B + } + alu(lane).io.a := a + alu(lane).io.b := b + alu(lane).io.c := operandReads(lane)._3 + alu(lane).io.carry := (selected && (!carryMask || !vm)) || merge && vm + alu(lane).io.vxrm := io.vxrm + alu(lane).io.selector := rs1 + alu(lane).io.sew := sew + alu(lane).io.funct6 := op + alu(lane).io.multiplyClass := multiply + falu(lane).io.a := a + falu(lane).io.b := Mux(fpReduction, accumulator, b) + falu(lane).io.c := operandReads(lane)._3 + falu(lane).io.sew := sew + falu(lane).io.op := Mux( + fpReduction, + Mux(op === 5.U, 4.U, Mux(op === 7.U, 6.U, 0.U)), + op + ) + falu(lane).io.subop := rs1 + falu(lane).io.roundingMode := io.roundingMode + val fpRequired = + laneSelected(lane) && !merge && (!fpReduction || (lane == 0).B) + falu( + lane + ).io.start := state === execute && instructionLegal && fp && fpRequired && !fpStarted( + lane + ) && falu( + lane + ).io.ready + fpComplete(lane) := !fpRequired || fpDone(lane) || falu(lane).io.valid + laneFlags(lane) := Mux( + merge, + 0.U, + Mux(fpDone(lane), fpFlags(lane), falu(lane).io.flags) + ) + laneLegal(lane) := merge || Mux( + fpDone(lane), + fpLegal(lane), + falu(lane).io.legal + ) + val floatingValue = Mux(fpDone(lane), fpData(lane), falu(lane).io.result) + val gathered = Mux(sourceIndex < vlmax, a, 0.U) + val slid = Mux( + slideOne && Mux(up, index === 0.U, index === vl - 1.U), + scalar, + Mux(!up && sourceIndex >= vlmax, 0.U, a) + ) + val maskA = maskElement(sourceWords(lane), index) + val maskB = maskElement(operandWords(lane), index) + val maskResult = MuxLookup(op, false.B)( + Seq( + 24.U -> (maskA && !maskB), + 25.U -> (maskA && maskB), + 26.U -> (maskA || maskB), + 27.U -> (maskA ^ maskB), + 28.U -> (maskA || !maskB), + 29.U -> !(maskA && maskB), + 30.U -> !(maskA || maskB), + 31.U -> !(maskA ^ maskB) + ) + ) + laneValue(lane) := Mux( + fp, + Mux(merge, Mux(vm || selected, b, a), floatingValue), + Mux( + wholeMove, + a, + Mux( + maskLogical, + maskResult, + Mux( + identity, + index, + Mux( + slide || slideOne, + slid, + Mux(gather, gathered, alu(lane).io.result) + ) + ) + ) + ) + ) + when(falu(lane).io.start)(fpStarted(lane) := true.B) + when(state === execute && falu(lane).io.valid) { + fpDone(lane) := true.B + fpData(lane) := falu(lane).io.result + fpFlags(lane) := falu(lane).io.flags + fpLegal(lane) := falu(lane).io.legal + } + } + + val destinationGroupLegal = + compare || maskLogical || population || first || reduction || scalarMoveOut || scalarMoveIn || + groupLegal(rd, destinationSew) + + val sourceGroupLegal = + maskLogical || population || first || scalarMoveIn || scalarMoveOut || groupLegal( + rs2, + sourceSew + ) + + val operandGroupLegal = + !vectorOperand || op === 18.U || reduction || scalarMoveOut || maskLogical || population || first || + groupLegal(rs1, sew) + + val aluLegal = alu + .map(_.io.legal) + .reduce(_ && _) || maskLogical || identity || gather || slide || slideOne || + scalarMoveOut || scalarMoveIn || population || first || intReduction || wholeMove + + val fpOperationLegal = Seq(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 16, 18, 19, 23, 24, 25, 27, 28, 29, 31, 32, 33, 36, 39, + 40, 41, 42, 43, 44, 45, 46, 47) + .map(n => op === n.U) + .reduce(_ || _) + + val ordinaryLegal = + !vtype( + 63 + ) && sourceSew <= p.maxSew.U && destinationSew <= p.maxSew.U && destinationGroupLegal && sourceGroupLegal && operandGroupLegal && + (!(intWide || floatWide || intNarrow || floatNarrow || extension) || + overlapLegal(rd, destinationSew, rs2, sourceSew) && + (!vectorOperand || op === 18.U || overlapLegal( + rd, + destinationSew, + rs1, + sew + ))) && + (vm || rd =/= 0.U || compare || maskLogical || population || first || reduction) && + (!merge || !vm || rs2 === 0.U) && (!maskLogical || vm) && + (!carryOperation || carryMask || !vm) && + (!(reduction || population || first) || vstart === 0.U) && + (!fp || sew >= 2.U && fpOperationLegal && io.roundingMode <= 4.U && + (scalarMoveOut || scalarMoveIn || merge || falu(0).io.legal) && + (op =/= 16.U || scalarMoveOut || scalarMoveIn) && + (op =/= 18.U && op =/= 19.U || kind === 1.U) && (!merge || kind === 5.U)) && + (!integer || aluLegal) + + val wholeMoveLegal = vm && vstart === 0.U && Seq(0, 1, 3, 7) + .map(n => rs1 === n.U) + .reduce(_ || _) && + (rd & rs1) === 0.U && (rs2 & rs1) === 0.U && rd +& rs1 < 32.U && rs2 +& rs1 < 32.U + + val memoryLegal = memWidth <= p.maxSew.U && memSew <= p.maxSew.U && Seq( + 0, + 5, + 6, + 7 + ).map(n => kind === n.U).reduce(_ || _) && !instruction(28) && + Mux( + wholeMemory, + vm && Seq(0, 1, 3, 7) + .map(n => instruction(31, 29) === n.U) + .reduce(_ || _) && + (rd & instruction(31, 29)) === 0.U && rd +& instruction( + 31, + 29 + ) < 32.U && (!store || memWidth === 0.U), + !vtype(63) && instruction( + 31, + 29 + ) === 0.U && (memMode =/= 0.U || rs2 === 0.U) && + groupLegal(rd, memSew) && (!indexed || groupLegal( + rs2, + memWidth + )) && (vm || store || rd =/= 0.U) + ) + + val legal = Mux( + memory, + memoryLegal, + Mux(wholeMove, wholeMoveLegal, (integer || fp) && ordinaryLegal) + ) + + instructionLegal := legal + + val required = Wire(Vec(p.memoryPorts, Bool())) + val addresses = Wire(Vec(p.memoryPorts, UInt(64.W))) + val canceled = Wire(Vec(p.memoryPorts, Bool())) + for (port <- 0 until p.memoryPorts) { + val index = base + port.U + required( + port + ) := (port < p.laneNumber).B && index < limit && (wholeMemory || vm || maskElement( + maskWords(port), + index + )) + addresses(port) := scalar1 + Mux( + indexed && !wholeMemory, + operandReads(port)._1, + Mux(memMode === 2.U, index * scalar2, index << memSew) + ) + val earlierFailure = (0 until port) + .map { prior => + required(prior) && (memoryFailed(prior) || io + .memoryResponse(prior) + .fire && io.memoryResponse(prior).bits.error) + } + .foldLeft(false.B)(_ || _) + val earlierPending = (0 until port) + .map { prior => + val distance = addresses(port) - addresses(prior) + val reverseDistance = addresses(prior) - addresses(port) + val collision = + store && (distance < (1.U << memSew) || reverseDistance < (1.U << memSew)) + required(prior) && (memMode === 3.U || collision) && !received(prior) + } + .foldLeft(false.B)(_ || _) + canceled(port) := !issued(port) && !offered(port) && earlierFailure + io.memoryRequest(port) + .valid := state === execute && memory && legal && required( + port + ) && !issued( + port + ) && !earlierPending && !canceled(port) + io.memoryRequest(port).bits.address := addresses(port) + io.memoryRequest(port).bits.write := store + io.memoryRequest(port).bits.data := operandReads(port)._3 + io.memoryRequest(port).bits.mask := MuxLookup(memSew, 255.U(8.W))( + Seq(0.U -> 1.U, 1.U -> 3.U, 2.U -> 15.U) + ) + io.memoryRequest(port).bits.size := memSew + io.memoryResponse(port).ready := state === execute && memory && issued( + port + ) && !received(port) + when(io.memoryRequest(port).valid)(offered(port) := true.B) + when(io.memoryRequest(port).fire)(issued(port) := true.B) + when(io.memoryResponse(port).fire) { + received(port) := true.B + memoryData(port) := io.memoryResponse(port).bits.data + memoryFailed(port) := io.memoryResponse(port).bits.error + } + } + + val memoryComplete = (0 until p.memoryPorts) + .map(i => !required(i) || canceled(i) || received(i) || io.memoryResponse(i).fire) + .reduce(_ && _) + + val failures = VecInit( + (0 until p.memoryPorts).map(i => + required(i) && (memoryFailed(i) || io + .memoryResponse(i) + .fire && io.memoryResponse(i).bits.error) + ) + ) + + val failureIndex = PriorityEncoder(failures.asUInt) + + when(state === execute) { + when(configure) { + val immediate = instruction(31, 30) === 3.U + val registerType = instruction(31, 25) === 64.U + val requested = Mux( + registerType, + scalar2, + Mux(immediate, instruction(29, 20), instruction(30, 20)) + ) + val requestedSew = requested(5, 3) + val requestedLmul = requested(2, 0) + val typeLegal = + requested(63, 8) === 0.U && requestedSew <= p.maxSew.U && requestedLmul =/= 4.U && + (!requestedLmul( + 2 + ) || (8.U << requestedSew) <= (p.eLen.U >> (8.U - requestedLmul))) + val newMaximum = maximum(requestedSew, requestedLmul) + val keepLength = !immediate && rs1 === 0.U && rd === 0.U + when( + !(immediate || registerType || !instruction( + 31 + )) || typeLegal && keepLength && newMaximum =/= vlmax + ) { + finish(true.B, 2.U, instruction) + }.otherwise { + val avl = Mux( + immediate, + rs1, + Mux(rs1 =/= 0.U, scalar1, Mux(rd =/= 0.U, newMaximum, vl)) + ) + val length = Mux(typeLegal, Mux(avl < newMaximum, avl, newMaximum), 0.U) + vl := length + vtype := Mux(typeLegal, requested, "h8000000000000000".U) + result.scalarWrite := rd =/= 0.U + result.scalarData := length + finish() + } + }.elsewhen(!legal) { + finish(true.B, 2.U, instruction) + }.elsewhen(scalarMoveOut) { + val value = element(sourceWords(0), 0.U, sew) + when(fp) { + result.floatWrite := true.B + result.floatData := Mux( + sew === 2.U, + Cat("hffffffff".U(32.W), value(31, 0)), + value + ) + }.otherwise { + result.scalarWrite := rd =/= 0.U + result.scalarData := MuxLookup(sew, value)( + Seq( + 0.U -> Cat(Fill(56, value(7)), value(7, 0)), + 1.U -> Cat(Fill(48, value(15)), value(15, 0)), + 2.U -> Cat(Fill(32, value(31)), value(31, 0)) + ) + ) + } + finish() + }.elsewhen(scalarMoveIn) { + when(vl =/= 0.U && vstart < vl) { + val scalar = Mux( + sew === 2.U && !floatScalar(63, 32).andR, + "h7fc00000".U, + floatScalar + ) + writeElement( + 0, + 0.U, + sew, + Mux(fp, scalar, scalar1) + ) + } + finish() + }.elsewhen(base >= limit) { + when(population || first) { + result.scalarWrite := rd =/= 0.U + result.scalarData := Mux(first, "hffffffffffffffff".U, count) + } + finish() + }.elsewhen(memory) { + when(memoryComplete) { + for (lane <- 0 until p.laneNumber) { + when( + load && required( + lane + ) && (!failures.asUInt.orR || lane.U < failureIndex) + ) { + writeElement( + lane, + base + lane.U, + memSew, + Mux( + io.memoryResponse(lane).fire, + io.memoryResponse(lane).bits.data, + memoryData(lane) + ) + ) + } + } + when(failures.asUInt.orR) { + vstart := base + failureIndex + finish(true.B, Mux(load, 5.U, 7.U), addresses(failureIndex)) + }.otherwise(nextBatch(p.laneNumber.U)) + } + }.elsewhen(population || first) { + val selected = VecInit((0 until p.laneNumber).map { lane => + val index = base + lane.U + index < vl && maskElement(sourceWords(lane), index) && + (vm || maskElement(maskWords(lane), index)) + }) + val total = count + PopCount(selected) + count := total + when(first && selected.asUInt.orR || base + p.laneNumber.U >= vl) { + result.scalarWrite := rd =/= 0.U + result.scalarData := Mux( + first, + Mux( + selected.asUInt.orR, + base + PriorityEncoder(selected.asUInt), + "hffffffff".U + ), + total + ) + finish() + }.otherwise { + base := base + p.laneNumber.U + readPhase := 0.U + state := capture + } + }.elsewhen(intReduction) { + // All integer reductions are associative at the destination width. Keep + // the original leaf order and ignore masked lanes instead of serializing them. + def combine(left: (Bool, UInt), right: (Bool, UInt)): (Bool, UInt) = { + val (leftValid, a) = left + val (rightValid, b) = right + val width = 8.U(7.W) << sew + val sa = (a << (p.eLen.U - width))(p.eLen - 1, 0).asSInt + val sb = (b << (p.eLen.U - width))(p.eLen - 1, 0).asSInt + val value = MuxLookup(op, a + b)(Seq( + 1.U -> (a & b), + 2.U -> (a | b), + 3.U -> (a ^ b), + 4.U -> Mux(b < a, b, a), + 5.U -> Mux(sb < sa, b, a), + 6.U -> Mux(b > a, b, a), + 7.U -> Mux(sb > sa, b, a) + )) + (leftValid || rightValid, Mux(!leftValid, b, Mux(!rightValid, a, value))) + } + def tree(values: Seq[(Bool, UInt)]): (Bool, UInt) = { + if (values.size == 1) values.head + else { + val (left, right) = values.splitAt(values.size / 2) + combine(tree(left), tree(right)) + } + } + val reduced = tree(Seq(true.B -> accumulator) ++ + (0 until p.laneNumber).map(lane => laneSelected(lane) -> laneA(lane)))._2 + accumulator := reduced + when(base + p.laneNumber.U >= vl)(writeElement(0, 0.U, sew, reduced)) + nextBatch(p.laneNumber.U) + }.elsewhen(fpReduction) { + when(fpComplete(0)) { + when(laneSelected(0) && !laneLegal(0))(finish(true.B, 2.U, instruction)) + .otherwise { + val value = Mux(laneSelected(0), laneValue(0), accumulator) + accumulator := value + when(laneSelected(0))(result.flags := result.flags | laneFlags(0)) + when(base + 1.U >= vl)(writeElement(0, 0.U, sew, value)) + nextBatch(1.U) + } + } + }.elsewhen(divide) { + when(!io.initialize) { + when(!divBatchValid) { + divActive := laneSelected + divBatchValid := true.B + }.elsewhen((0 until p.laneNumber).map(i => !divActive(i) || divDone(i)).reduce(_ && _)) { + for (lane <- 0 until p.laneNumber) { + when(divActive(lane)) { + writeElement(lane, base + lane.U, destinationSew, divData(lane)) + } + } + nextBatch(p.laneNumber.U) + } + } + }.elsewhen(!fp || fpComplete.reduce(_ && _)) { + val floatingLegal = (0 until p.laneNumber) + .map(i => !laneSelected(i) || laneLegal(i)) + .reduce(_ && _) + when(fp && !floatingLegal)(finish(true.B, 2.U, instruction)).otherwise { + for (lane <- 0 until p.laneNumber) { + when(laneSelected(lane)) { + writeElement( + lane, + base + lane.U, + destinationSew, + laneValue(lane), + compare || maskLogical + ) + } + } + when(fp) { + result.flags := result.flags | (0 until p.laneNumber) + .map(i => Mux(laneSelected(i), laneFlags(i), 0.U)) + .reduce(_ | _) + } + when(integer && !wholeMove) { + result.saturated := result.saturated || (0 until p.laneNumber) + .map(i => laneSelected(i) && alu(i).io.saturated) + .reduce(_ || _) + } + nextBatch(p.laneNumber.U) + } + } + } + when(state === execute && writes.map(_.valid).reduce(_ || _) && !io.initialize) { + pendingWrites := writes + resumeState := Mux(finishRequested, idle, capture) + state := commit + writeSent := false.B + } + when(io.initialize) { + state := idle + readSent := false.B + snapshotSent := false.B + writeSent := false.B + firstBatch := true.B + readPhase := 0.U + resultValid := false.B + vl := 0.U + vtype := "h8000000000000000".U + vstart := 0.U + divBatchValid := false.B + divStarted.foreach(_ := false.B) + divDone.foreach(_ := false.B) + fpStarted.foreach(_ := false.B) + fpDone.foreach(_ := false.B) + offered.foreach(_ := false.B) + issued.foreach(_ := false.B) + received.foreach(_ := false.B) + } +} diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/loader/build.sc b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/loader/build.sc new file mode 100644 index 000000000..6e754ceb7 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/loader/build.sc @@ -0,0 +1,23 @@ +import mill._ +import $file.^.^.^.^.^.^.^.^.^.^.settings + +val ipRoot: os.Path = millSourcePath + +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.acc.gpdomain.loader.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/ImageLoader.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Loader.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/loader/src/main/scala/Emit.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/loader/src/main/scala/Emit.scala new file mode 100644 index 000000000..ceebc616d --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/loader/src/main/scala/Emit.scala @@ -0,0 +1,26 @@ +package framework.acc.gpdomain.loader + +import _root_.configs.GlobalConfig + +object Emit { + + def emit( + b: GlobalConfig, + hartId: Int, + threadId: Int, + directory: String, + firtoolOpts: Array[String] + ): Unit = { + val core = + b.device.cores.find(_.hartId == hartId).getOrElse(throw new IllegalArgumentException(s"Unknown hart ID: $hartId")) + val thread = + core.threads.find(_.id == threadId).getOrElse(throw new IllegalArgumentException(s"Unknown thread ID: $threadId")) + + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Loader(thread.acc.get.frontend, thread.acc.get.memDomain, thread.acc.get.gpDomain), + firtoolOpts = firtoolOpts, + args = Array("--target-dir", directory, "--split-verilog") + ) + } + +} diff --git a/arch/src/main/scala/framework/rvv/src/main/scala/frontend/ImageLoader.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/loader/src/main/scala/ImageLoader.scala similarity index 88% rename from arch/src/main/scala/framework/rvv/src/main/scala/frontend/ImageLoader.scala rename to arch/src/main/scala/framework/acc/gpdomain/src/main/scala/loader/src/main/scala/ImageLoader.scala index af49ca7f8..18ceb6bbc 100644 --- a/arch/src/main/scala/framework/rvv/src/main/scala/frontend/ImageLoader.scala +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/loader/src/main/scala/ImageLoader.scala @@ -1,13 +1,29 @@ -package framework.rvv +package framework.acc.gpdomain.loader +import framework.acc.gpdomain.configs.GpDomainParam + +import framework.acc.gpdomain.execution.{ProgramWrite, VectorMemoryRequest, VectorMemoryResponse} import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public} -import framework.top.GlobalConfig + +class ImageLoad extends Bundle { + val buffer = Bool() + val bytes = UInt(32.W) +} + +class ImageResult extends Bundle { + val fault = Bool() + val cause = UInt(32.W) + val tval = UInt(32.W) + val entry = UInt(32.W) + val textBytes = UInt(32.W) + val constBytes = UInt(32.W) +} @instantiable -class ImageLoader(val b: GlobalConfig) extends Module { - val p = b.rvv +class ImageLoader(val gpParam: GpDomainParam) extends Module { + val p = gpParam.loader @public val io = IO(new Bundle { diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/loader/src/main/scala/Loader.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/loader/src/main/scala/Loader.scala new file mode 100644 index 000000000..b7e768e55 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/loader/src/main/scala/Loader.scala @@ -0,0 +1,237 @@ +package framework.acc.gpdomain.loader + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.memdomain.configs.MemDomainParam + +import framework.acc.frontend.scheduler.{Complete, Issue} +import framework.acc.memdomain.toddr.DmaStatus +import framework.acc.gpdomain.execution.{ + KernelCompletion, + KernelLaunch, + ProgramWrite, + VectorMemoryRequest, + VectorMemoryResponse +} +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} + +class KernelContext(frontendParam: FrontendParam, memParam: MemDomainParam) extends Bundle { + val instId = UInt(64.W) + val rob_id = UInt(log2Ceil(frontendParam.rob_entries).W) + val readBank = UInt(frontendParam.bank_id_len.W) + val writeBank = UInt(frontendParam.bank_id_len.W) + val readGroups = UInt(memParam.groupCountWidth.W) + val writeGroups = UInt(memParam.groupCountWidth.W) +} + +class LoaderMemoryPort(frontendParam: FrontendParam, memParam: MemDomainParam) extends Bundle { + val request = Decoupled(new VectorMemoryRequest) + val response = Flipped(Decoupled(new VectorMemoryResponse)) + val control = Output(Bool()) + val buffer = Output(Bool()) + val loading = Output(Bool()) + val constBytes = Output(UInt(32.W)) + val context = Output(new KernelContext(frontendParam, memParam)) +} + +@instantiable +class Loader(val frontendParam: FrontendParam, val memParam: MemDomainParam, val gpParam: GpDomainParam) + extends Module { + val commandDataBits = frontendParam.commandDataBits + require(gpParam.enable, "GP domain must be explicitly enabled") + + @public + val io = IO(new Bundle { + val running = Output(Bool()) + val cmdReq = Flipped(Decoupled(new Issue(frontendParam, memParam))) + val cmdResp = Decoupled(new Complete(frontendParam)) + val uploadStart = Flipped(Decoupled(new ImageLoad)) + val uploadComplete = Decoupled(new KernelCompletion) + val release = Flipped(Valid(Bool())) + val image = Flipped(Decoupled(UInt(32.W))) + val imageTerminal = Flipped(Decoupled(new DmaStatus)) + val program = Decoupled(new ProgramWrite) + val launch = Decoupled(new KernelLaunch(frontendParam, memParam)) + val done = Flipped(Decoupled(new KernelCompletion)) + val memory = new LoaderMemoryPort(frontendParam, memParam) + }) + + val request = io.cmdReq + val uploading = RegInit(false.B) + + val parser: Instance[ImageLoader] = Instantiate(new ImageLoader(gpParam)) + val idle :: loading :: descriptorSend :: descriptorWait :: launch :: executing :: completed :: Nil = Enum(7) + + val state = RegInit(idle) + val command = Reg(new Issue(frontendParam, memParam)) + val validImage = RegInit(VecInit(Seq.fill(2)(false.B))) + val imageEntry = Reg(Vec(2, UInt(32.W))) + val imageBytes = Reg(Vec(2, UInt(32.W))) + val imageConstants = Reg(Vec(2, UInt(32.W))) + val buffer = Reg(Bool()) + val descriptor = Reg(Vec(12, UInt(64.W))) + val word = Reg(UInt(4.W)) + val terminalSeen = RegInit(false.B) + val terminalFailed = RegInit(false.B) + val terminalCancelled = RegInit(false.B) + val result = RegInit(0.U.asTypeOf(new KernelCompletion)) + + request.ready := state === idle && !io.uploadStart.valid + io.uploadStart.ready := state === idle + val completedImage = state === completed && (!uploading || result.fault || terminalSeen) + io.cmdResp.valid := completedImage && !uploading + io.uploadComplete.valid := completedImage && uploading + io.uploadComplete.bits := result + io.cmdResp.bits.rob_id := command.rob_id + when(io.cmdResp.valid)(assert(!result.fault, "GP task violated its execution contract")) + io.running := state =/= idle + io.memory.context.instId := command.instId + io.memory.context.rob_id := command.rob_id + io.memory.context.readBank := command.cmd.cmd.rs1Data(9, 0) + io.memory.context.writeBank := command.cmd.cmd.rs1Data(29, 20) + io.memory.context.readGroups := command.cmd.op1_col + io.memory.context.writeGroups := command.cmd.wr_col + io.memory.control := state === loading || state === descriptorSend || state === descriptorWait + io.memory.buffer := buffer + io.memory.loading := state === loading + io.memory.constBytes := imageConstants(buffer) + + parser.io.start.valid := io.uploadStart.fire + parser.io.start.bits := io.uploadStart.bits + parser.io.image <> io.image + io.imageTerminal.ready := state =/= idle && uploading && !terminalSeen + val terminalError = io.imageTerminal.fire && io.imageTerminal.bits.error =/= 0.U + parser.io.abort := terminalError + when(io.imageTerminal.fire) { + terminalSeen := true.B + terminalFailed := io.imageTerminal.bits.error =/= 0.U + terminalCancelled := io.imageTerminal.bits.error === 10.U + when(terminalError) { + validImage(buffer) := false.B + when(!result.fault) { + result.fault := true.B + result.cause := 5.U + result.tval := io.imageTerminal.bits.address + } + } + } + parser.io.done.ready := state === loading + io.program <> parser.io.program + io.launch.valid := state === launch + io.launch.bits.context := io.memory.context + io.launch.bits.iBuffer := buffer + io.launch.bits.entry := descriptor(0)(31, 0) + io.launch.bits.end := descriptor(1)(31, 0) + io.launch.bits.stack := descriptor(2) + for (i <- 0 until 8) (io.launch.bits.args(i) := descriptor(i + 3)) + io.done.ready := state === executing + + io.memory.request.valid := state === loading && parser.io.memoryRequest.valid + io.memory.request.bits := parser.io.memoryRequest.bits + parser.io.memoryRequest.ready := state === loading && io.memory.request.ready + parser.io.memoryResponse.valid := state === loading && io.memory.response.valid + parser.io.memoryResponse.bits := io.memory.response.bits + io.memory.response.ready := state === loading && parser.io.memoryResponse.ready + when(state === descriptorSend) { + io.memory.request.valid := true.B + io.memory.request.bits.address := (command.cmd.op1_col.pad(64) << 32) + command.cmd.cmd.rs2Data + word * 8.U + io.memory.request.bits.write := false.B + io.memory.request.bits.data := 0.U + io.memory.request.bits.mask := 255.U + io.memory.request.bits.size := 3.U + } + when(state === descriptorWait)(io.memory.response.ready := true.B) + + when(io.release.valid)(validImage(io.release.bits) := false.B) + when(io.uploadStart.fire) { + uploading := true.B + command := 0.U.asTypeOf(command) + buffer := io.uploadStart.bits.buffer + validImage(io.uploadStart.bits.buffer) := false.B + result := 0.U.asTypeOf(result) + terminalSeen := false.B + terminalFailed := false.B + terminalCancelled := false.B + state := loading + } + when(request.fire) { + assert(request.bits.cmd.cmd.raw_inst(31, 25) === 79.U, "Loader launch requires RUN_KERNEL") + uploading := false.B + command := request.bits + result := 0.U.asTypeOf(result) + terminalSeen := false.B + terminalFailed := false.B + terminalCancelled := false.B + val program = request.bits.cmd.cmd.rs1Data(19, 10) + val programValid = + program >= memParam.virtualBankCount.U && program < (memParam.virtualBankCount + 2).U + val slot = program - memParam.virtualBankCount.U + buffer := slot(0) + word := 0.U + state := descriptorSend + when( + !programValid || !validImage(slot(0)) || request.bits.cmd.op1_col === 0.U || + request.bits.cmd.wr_col === 0.U || request.bits.cmd.cmd.rs1Data(63, 30).orR || + request.bits.cmd.cmd.rs1Data(9, 0) === request.bits.cmd.cmd.rs1Data(29, 20) || + request.bits.cmd.cmd.rs2Data(63, 32).orR || + (request.bits.cmd.cmd.rs2Data( + 31, + 0 + ) +& 96.U) > (memParam.bankEntries * (memParam.bankWidth / 8)).U + ) { + result.fault := true.B + result.cause := 1.U + result.tval := request.bits.cmd.cmd.rs1Data + state := completed + } + } + when(parser.io.done.fire) { + val nativeFault = parser.io.done.bits.fault && parser.io.done.bits.cause =/= 5.U + when((!terminalFailed && !terminalError) || + (nativeFault && (terminalCancelled || (io.imageTerminal.fire && io.imageTerminal.bits.error === 10.U)))) { + result.fault := parser.io.done.bits.fault + result.cause := parser.io.done.bits.cause + result.tval := parser.io.done.bits.tval + } + imageEntry(buffer) := parser.io.done.bits.entry + imageBytes(buffer) := parser.io.done.bits.textBytes + imageConstants(buffer) := parser.io.done.bits.constBytes + state := completed + } + when(state === descriptorSend && io.memory.request.fire)(state := descriptorWait) + when(state === descriptorWait && io.memory.response.fire) { + when(io.memory.response.bits.error) { + result.fault := true.B + result.cause := 5.U + result.tval := (command.cmd.op1_col.pad(64) << 32) + command.cmd.cmd.rs2Data + word * 8.U + state := completed + }.otherwise { + descriptor(word) := io.memory.response.bits.data + word := word + 1.U + state := Mux(word === 11.U, launch, descriptorSend) + } + } + when(state === launch) { + when(descriptor(0) =/= imageEntry(buffer) || descriptor(1) =/= imageBytes(buffer) || descriptor( + 11 + ) =/= 0.U || descriptor(2) =/= "h40002000".U) { + io.launch.valid := false.B + result.fault := true.B + result.cause := 1.U + result.tval := descriptor(0) + state := completed + }.elsewhen(io.launch.fire)(state := executing) + } + when(io.done.fire) { + result := io.done.bits + state := completed + } + when(io.cmdResp.fire || io.uploadComplete.fire) { + when(uploading && !result.fault && terminalSeen && !terminalFailed) { + validImage(buffer) := true.B + } + state := idle + } +} diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/loader/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/loader/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..d9c5c321b --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/loader/src/main/scala/configs/Params.scala @@ -0,0 +1,8 @@ +package framework.acc.gpdomain.loader.configs + +case class Params( + iBufWords: Int, + constBytes: Int) { + require(iBufWords > 0) + require(constBytes > 0) +} diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/build.sc b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/build.sc new file mode 100644 index 000000000..f4df32f36 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/build.sc @@ -0,0 +1,24 @@ +import mill._ +import $file.^.^.^.^.^.^.^.^.^.^.settings + +val ipRoot: os.Path = millSourcePath + +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.acc.gpdomain.memory.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/BankPort.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/KernelMemory.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Memory.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/src/main/scala/BankPort.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/src/main/scala/BankPort.scala new file mode 100644 index 000000000..0fe1f588a --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/src/main/scala/BankPort.scala @@ -0,0 +1,122 @@ +package framework.acc.gpdomain.memory + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import framework.acc.gpdomain._ +import framework.acc.gpdomain.execution.{VectorMemoryRequest, VectorMemoryResponse} +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.acc.memdomain.{BankPort => BankAccessPort} +import framework.acc.gpdomain.loader.KernelContext + +class BankRequest(frontendParam: FrontendParam, memParam: MemDomainParam) extends Bundle { + val operation = new VectorMemoryRequest + val context = new KernelContext(frontendParam, memParam) +} + +@instantiable +class BankPort(val frontendParam: FrontendParam, val memParam: MemDomainParam) extends Module { + val ownerId = frontendParam.ownerId + val lineBytes = memParam.bankWidth / 8 + val offsetBits = log2Ceil(lineBytes) + require(memParam.bankWidth >= 64 && isPow2( + memParam.bankWidth + )) + require(lineBytes <= 65536) + require(memParam.bankMaskLen == lineBytes) + + @public + val io = IO(new Bundle { + val request = Flipped(Decoupled(new BankRequest(frontendParam, memParam))) + val response = Decoupled(new VectorMemoryResponse) + val bank = new BankAccessPort(frontendParam, memParam) + }) + + val idle :: readRequest :: readResponse :: writeRequest :: writeResponse :: complete :: Nil = Enum(6) + val state = RegInit(idle) + val request = Reg(new BankRequest(frontendParam, memParam)) + val consumed = RegInit(0.U(4.W)) + val result = RegInit(0.U(64.W)) + val error = RegInit(false.B) + + val byteCount = (1.U(4.W) << request.operation.size)(3, 0) + val byteAddress = request.operation.address(31, 0) +& consumed + val lineOffset = byteAddress(offsetBits - 1, 0) + val remaining = byteCount - consumed + val lineRemaining = lineBytes.U - lineOffset + val chunk = Mux(remaining <= lineRemaining, remaining, lineRemaining)(3, 0) + val nextConsumed = consumed + chunk + val lineAddress = byteAddress >> offsetBits + val readShifted = io.bank.read.resp.bits.data >> (lineOffset << 3) + val chunkMask = ((1.U(65.W) << (chunk << 3)) - 1.U)(63, 0) + val readChunk = readShifted.pad(64)(63, 0) & chunkMask + + io.request.ready := state === idle + io.response.valid := state === complete + io.response.bits.data := result + io.response.bits.error := error + val localBank = request.operation.address(63, 32) + val fromRead = localBank < request.context.readGroups + val parentBank = Mux(fromRead, request.context.readBank, request.context.writeBank) + val parentGroup = Mux(fromRead, localBank, localBank - request.context.readGroups) + io.bank.bank_id := parentBank + io.bank.group_id := parentGroup + io.bank.rob_id := request.context.rob_id + io.bank.inst_id := request.context.instId + io.bank.thread_id := ownerId.U + io.bank.is_shared := false.B // BankRouter resolves private/shared ownership from the alias. + io.bank.read.req.valid := state === readRequest + io.bank.read.req.bits.addr := lineAddress + io.bank.read.resp.ready := state === readResponse + io.bank.write.req.valid := state === writeRequest + io.bank.write.req.bits.addr := lineAddress + io.bank.write.req.bits.data := (request.operation.data >> (consumed << 3)).pad( + memParam.bankWidth + ) << (lineOffset << 3) + io.bank.write.resp.ready := state === writeResponse + for (i <- 0 until lineBytes) { + val sourceByte = consumed + i.U - lineOffset + io.bank.write.req.bits.mask(i) := i.U >= lineOffset && i.U < lineOffset +& chunk && request.operation.mask( + sourceByte(2, 0) + ) + } + + when(io.request.fire) { + request := io.request.bits + consumed := 0.U + result := 0.U + val end = io.request.bits.operation.address(31, 0) +& (1.U(33.W) << io.request.bits.operation.size) + val invalid = io.request.bits.operation.address( + 63, + 32 + ) >= (io.request.bits.context.readGroups +& io.request.bits.context.writeGroups) || + (io.request.bits.operation.write && io.request.bits.operation.address( + 63, + 32 + ) < io.request.bits.context.readGroups) || + end > (BigInt(memParam.bankEntries) * lineBytes).U + error := invalid + state := Mux(invalid, complete, Mux(io.request.bits.operation.write, writeRequest, readRequest)) + } + when(io.bank.read.req.fire) { + state := readResponse + } + when(io.bank.read.resp.fire) { + result := result | (readChunk << (consumed << 3)) + consumed := nextConsumed + state := Mux(nextConsumed === byteCount, complete, readRequest) + } + when(io.bank.write.req.fire) { + state := writeResponse + } + when(io.bank.write.resp.fire) { + consumed := nextConsumed + error := !io.bank.write.resp.bits.ok + state := Mux(!io.bank.write.resp.bits.ok || nextConsumed === byteCount, complete, writeRequest) + } + when(io.response.fire) { + state := idle + } +} diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/src/main/scala/Emit.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/src/main/scala/Emit.scala new file mode 100644 index 000000000..a7aabb852 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/src/main/scala/Emit.scala @@ -0,0 +1,26 @@ +package framework.acc.gpdomain.memory + +import _root_.configs.GlobalConfig + +object Emit { + + def emit( + b: GlobalConfig, + hartId: Int, + threadId: Int, + directory: String, + firtoolOpts: Array[String] + ): Unit = { + val core = + b.device.cores.find(_.hartId == hartId).getOrElse(throw new IllegalArgumentException(s"Unknown hart ID: $hartId")) + val thread = + core.threads.find(_.id == threadId).getOrElse(throw new IllegalArgumentException(s"Unknown thread ID: $threadId")) + + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Memory(thread.acc.get.frontend, thread.acc.get.memDomain, thread.acc.get.gpDomain), + firtoolOpts = firtoolOpts, + args = Array("--target-dir", directory, "--split-verilog") + ) + } + +} diff --git a/arch/src/main/scala/framework/rvv/src/main/scala/memory/KernelMemory.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/src/main/scala/KernelMemory.scala similarity index 92% rename from arch/src/main/scala/framework/rvv/src/main/scala/memory/KernelMemory.scala rename to arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/src/main/scala/KernelMemory.scala index 3ca93e5d4..575a5b816 100644 --- a/arch/src/main/scala/framework/rvv/src/main/scala/memory/KernelMemory.scala +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/src/main/scala/KernelMemory.scala @@ -1,13 +1,19 @@ -package framework.rvv +package framework.acc.gpdomain.memory +import framework.lib.arbiter.{RRArbiter} +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import framework.acc.gpdomain.configs.GpDomainParam + +import framework.acc.gpdomain._ +import framework.acc.gpdomain.execution.{VectorMemoryRequest, VectorMemoryResponse} import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public} -import framework.top.GlobalConfig @instantiable -class KernelMemory(val b: GlobalConfig) extends Module { - val p = b.rvv +class KernelMemory(val gpParam: GpDomainParam) extends Module { + val p = gpParam.memory @public val io = IO(new Bundle { @@ -38,7 +44,7 @@ class KernelMemory(val b: GlobalConfig) extends Module { VecInit((0 until p.memoryPorts).map(port => io.request(port).valid && !pending(port) && io.request(port).bits.write)) val hasWrite = writes.asUInt.orR val writePort = PriorityEncoder(writes) - val arbiter = Module(new RRArbiter(new VectorMemoryRequest, p.memoryPorts)) + val arbiter = Instantiate(new RRArbiter(new VectorMemoryRequest, p.memoryPorts)) arbiter.io.out.ready := state === idle for (port <- 0 until p.memoryPorts) { val eligible = !pending(port) && (!hasWrite || writePort === port.U) diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/src/main/scala/Memory.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/src/main/scala/Memory.scala new file mode 100644 index 000000000..1bb3635d4 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/src/main/scala/Memory.scala @@ -0,0 +1,82 @@ +package framework.acc.gpdomain.memory + +import framework.lib.queue.Queue + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.memdomain.{BankPort => BankAccessPort} +import framework.acc.gpdomain.execution.{VectorMemoryRequest, VectorMemoryResponse} +import framework.acc.gpdomain.loader.LoaderMemoryPort + +class ResponseOwner extends Bundle { + val local = Bool() + val loader = Bool() +} + +@instantiable +class Memory(val frontendParam: FrontendParam, val memParam: MemDomainParam, val gpParam: GpDomainParam) + extends Module { + val ownerId = frontendParam.ownerId + val p = gpParam.memory + + @public + val io = IO(new Bundle { + val loader = Flipped(new LoaderMemoryPort(frontendParam, memParam)) + val request = Vec(p.memoryPorts, Flipped(Decoupled(new VectorMemoryRequest))) + val response = Vec(p.memoryPorts, Decoupled(new VectorMemoryResponse)) + val banks = Vec(p.memoryPorts, new BankAccessPort(frontendParam, memParam)) + }) + + val local: Instance[KernelMemory] = Instantiate(new KernelMemory(gpParam)) + val ports: Seq[Instance[BankPort]] = + Seq.fill(p.memoryPorts)(Instantiate(new BankPort(frontendParam, memParam))) + + local.io.buffer := io.loader.buffer + local.io.loading := io.loader.loading + local.io.constBytes := io.loader.constBytes + + for (i <- 0 until p.memoryPorts) { + io.banks(i) <> ports(i).io.bank + val owners = Instantiate(new Queue( + new ResponseOwner, + 2 + )) + val fromLoader = if (i == 0) io.loader.control else false.B + val request = if (i == 0) Mux(fromLoader, io.loader.request.bits, io.request(i).bits) else io.request(i).bits + val offered = + if (i == 0) Mux(fromLoader, io.loader.request.valid, io.request(i).valid) + else !io.loader.control && io.request(i).valid + val internal = request.address >= "h40000000".U && request.address < "h40002000".U + val destinationReady = Mux(internal, local.io.request(i).ready, ports(i).io.request.ready) + + local.io.request(i).valid := offered && internal && owners.io.enq.ready + local.io.request(i).bits := request + ports(i).io.request.valid := offered && !internal && owners.io.enq.ready + ports(i).io.request.bits.operation := request + ports(i).io.request.bits.context := io.loader.context + owners.io.enq.valid := offered && destinationReady + owners.io.enq.bits.local := internal + owners.io.enq.bits.loader := fromLoader + io.request(i).ready := !io.loader.control && owners.io.enq.ready && destinationReady + if (i == 0) io.loader.request.ready := io.loader.control && owners.io.enq.ready && destinationReady + + // Hold response ownership until acceptance, even when the request source changes. + val responseValid = Mux(owners.io.deq.bits.local, local.io.response(i).valid, ports(i).io.response.valid) + val responseBits = Mux(owners.io.deq.bits.local, local.io.response(i).bits, ports(i).io.response.bits) + val responseReady = Mux(owners.io.deq.bits.loader, io.loader.response.ready, io.response(i).ready) + io.response(i).valid := owners.io.deq.valid && !owners.io.deq.bits.loader && responseValid + io.response(i).bits := responseBits + if (i == 0) { + io.loader.response.valid := owners.io.deq.valid && owners.io.deq.bits.loader && responseValid + io.loader.response.bits := responseBits + } + local.io.response(i).ready := owners.io.deq.valid && owners.io.deq.bits.local && responseReady + ports(i).io.response.ready := owners.io.deq.valid && !owners.io.deq.bits.local && responseReady + owners.io.deq.ready := responseValid && responseReady + } +} diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..76f3a3bce --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/src/main/scala/configs/Params.scala @@ -0,0 +1,11 @@ +package framework.acc.gpdomain.memory.configs + +case class Params( + memoryPorts: Int, + wordBits: Int) { + require(memoryPorts > 0) + require(wordBits == 32 || wordBits == 64) + + val constBytes: Int = 4096 + val stackBytes: Int = 4096 +} diff --git a/arch/src/main/scala/framework/acc/memdomain/build.sc b/arch/src/main/scala/framework/acc/memdomain/build.sc new file mode 100644 index 000000000..4803374bc --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/build.sc @@ -0,0 +1,36 @@ +import mill._ +import $file.src.main.scala.bank_router.{build => bank_router_build} +import $file.src.main.scala.prm.{build => prm_build} +import $file.src.main.scala.tomem.{build => tomem_build} +import $file.^.^.^.^.^.^.settings + + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.acc.memdomain.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/MemDomain.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/MemoryControl.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/MemDomainParam.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/ctrl/Ctrl.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/ctrl/configer/Configer.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/ctrl/decoder/DISA.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/ctrl/decoder/Decoder.scala")) + ) ++ + bank_router_build.sources() ++ + prm_build.sources() ++ + tomem_build.sources() +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) ++ + bank_router_build.emitSources() ++ + prm_build.emitSources() ++ + tomem_build.emitSources() +) diff --git a/arch/src/main/scala/framework/acc/memdomain/configs/default.toml b/arch/src/main/scala/framework/acc/memdomain/configs/default.toml new file mode 100644 index 000000000..f98b00337 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/configs/default.toml @@ -0,0 +1,19 @@ +addrLen = 39 + +[bank] +layoutEnable = true +num = 24 +width = 128 +entries = 512 +maskLen = 16 +requestPorts = 13 + +[dma] +busWidth = 128 + +[mmio] +enable = false +bankNum = 5 +bankEntries = 1024 +bankWidth = 8 +readWidth = 8 diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/Emit.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/Emit.scala new file mode 100644 index 000000000..fc189a982 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/Emit.scala @@ -0,0 +1,39 @@ +package framework.acc.memdomain + +import sims.configs.SimParam +import framework.core.tbus.configs.Layout + +import _root_.configs.GlobalConfig +import java.nio.file.{Files, Paths} +import upickle.default.read + +object Emit { + + def emit( + b: GlobalConfig, + hartId: Int, + threadId: Int, + directory: String, + firtoolOpts: Array[String] + ): Unit = { + val core = + b.device.cores.find(_.hartId == hartId).getOrElse(throw new IllegalArgumentException(s"Unknown hart ID: $hartId")) + val thread = + core.threads.find(_.id == threadId).getOrElse(throw new IllegalArgumentException(s"Unknown thread ID: $threadId")) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new MemDomain( + thread.acc.get.frontend, + thread.acc.get.memDomain, + thread.acc.get.gpDomain, + thread.acc.get.transport, + core.privileged.permission, + core.memory.dcache.tracking, + core.memory.memoryAccess.walker, + b.sim.copy(diffTest = false) + ), + firtoolOpts = firtoolOpts, + args = Array("--target-dir", directory, "--split-verilog") + ) + } + +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/MemDomain.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/MemDomain.scala new file mode 100644 index 000000000..16253eca3 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/MemDomain.scala @@ -0,0 +1,232 @@ +package framework.acc.memdomain + +import sims.configs.SimParam +import framework.lib.arbiter.{RRArbiter} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.core.memory.access.preflight.configs.PmpParams +import framework.ip.memory.interlock.{Params => TrackingParam} +import framework.acc.memdomain.tomem.configs.{Params => TransportParam} +import framework.core.memory.access.mmu.configs.{Params => WalkerParam} +import framework.acc.debug.trace.MemoryEvent +import framework.ip.memory.bank.{SramReadIO, SramReadReq, SramReadResp, SramWriteIO, SramWriteReq, SramWriteResp} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.frontend.scheduler.MemoryIssue +import framework.acc.balldomain.DomainMemory +import framework.acc.memdomain.tomem.ToMem +import framework.core.memory.MemoryPort +import framework.core.memory.access.preflight.Preparation +import framework.ip.bus.axi4 +import framework.acc.memdomain.bank_router.{BankMappingIO, BankRouter} +import framework.acc.memdomain.prm.PRM +import framework.acc.memdomain.{MvoverISA, MvoverPort} +import framework.ip.memory.mesh_shm.MeshLocalBankPort +import framework.acc.gpdomain.ImagePort +import framework.thread.tlink.BankAccess + +class PhysicalBankHash(val frontendParam: FrontendParam) extends Bundle { + val valid = Bool() + val threadId = UInt(64.W) + val vbankId = UInt(frontendParam.bank_id_len.W) + val pbankId = UInt(32.W) + val groupId = UInt(32.W) + val statusHash = UInt(32.W) +} + +class BankRead(val frontendParam: FrontendParam, val memParam: MemDomainParam) extends Bundle { + val bank_id = Input(UInt(frontendParam.bank_id_len.W)) + val rob_id = Input(UInt(log2Up(frontendParam.rob_entries).W)) + val group_id = Input(UInt(memParam.groupIdWidth.W)) + + val io = new SramReadIO(memParam) +} + +class BankWrite(val frontendParam: FrontendParam, val memParam: MemDomainParam) extends Bundle { + val bank_id = Input(UInt(frontendParam.bank_id_len.W)) + val rob_id = Input(UInt(log2Up(frontendParam.rob_entries).W)) + val group_id = Input(UInt(memParam.groupIdWidth.W)) + + val io = new SramWriteIO(memParam) +} + +class BankPort(frontendParam: FrontendParam, memParam: MemDomainParam) extends Bundle { + val write = Flipped(new SramWriteIO(memParam)) + val read = Flipped(new SramReadIO(memParam)) + val bank_id = Output(UInt(frontendParam.bank_id_len.W)) + val group_id = Output(UInt(memParam.groupIdWidth.W)) + val is_shared = Output(Bool()) + val thread_id = Output(UInt(64.W)) + val rob_id = Output(UInt(log2Up(frontendParam.rob_entries).W)) + val inst_id = Output(UInt(64.W)) +} + +class LayoutQuery(frontendParam: FrontendParam, memParam: MemDomainParam) extends Bundle { + val bank = Output(UInt(frontendParam.bank_id_len.W)) + val groups = Input(UInt(memParam.groupCountWidth.W)) +} + +class BankLayoutRequest(frontendParam: FrontendParam) extends Bundle { + val banks = Vec(3, UInt(frontendParam.bank_id_len.W)) + val enabled = Vec(3, Bool()) +} + +class BankLayout(memParam: MemDomainParam) extends Bundle { + val groups = Vec(3, UInt(memParam.groupCountWidth.W)) +} + +@instantiable +class MemDomain( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val gpParam: GpDomainParam, + val transportParam: TransportParam, + val permissionParam: PmpParams, + val trackingParam: TrackingParam, + val walkerParam: WalkerParam, + val simp: SimParam) + extends Module { + val sharedPorts = memParam.sharedPorts + val diffTest = simp.diffTest + val commandDataBits = frontendParam.commandDataBits + val ownerId = frontendParam.ownerId + + val kernelPorts = + if (gpParam.enable) + gpParam.memory.memoryPorts + else 0 + + val sharedHashCount = if (memParam.sharedEnable) memParam.sharedBankNum else 0 + + @public + val io = IO(new Bundle { + val traceMemory = Output(Vec(memParam.bankRequestPorts + kernelPorts + 2 + memParam.bankNum, new MemoryEvent)) +// Command Channel + val issue = Flipped(Decoupled(new MemoryIssue(frontendParam, memParam, permissionParam))) + val complete = Decoupled(new Response(frontendParam)) + val mapping = Flipped(new BankMappingIO(frontendParam)) +// Inside Channel + val banks = Vec(memParam.bankRequestPorts, Flipped(new BankPort(frontendParam, memParam))) + val ballLayoutRequest = Flipped(Decoupled(new BankLayoutRequest(frontendParam))) + val ballLayoutResponse = Decoupled(new BankLayout(memParam)) + val gpLayoutRequest = Option.when(gpParam.enable)(Flipped(Decoupled(new BankLayoutRequest(frontendParam)))) + val gpLayoutResponse = Option.when(gpParam.enable)(Decoupled(new BankLayout(memParam))) + val gpBanks = Option.when(gpParam.enable)(Vec(kernelPorts, Flipped(new BankPort(frontendParam, memParam)))) + val image = Option.when(gpParam.enable)(new ImagePort) +// Outside Channel + val ball = Flipped(new DomainMemory(frontendParam, memParam, permissionParam)) + val memory = new MemoryPort(trackingParam, walkerParam) + val ddr = new axi4.Port(transportParam.axi) + + val shared = new Bundle { + val bank = new BankAccess(frontendParam, memParam) + val move = new MvoverPort + } + + val bank_hashes = + if (diffTest) { + Some(Output(Vec(memParam.bankNum + sharedHashCount, new PhysicalBankHash(frontendParam)))) + } else { + None + } + + val shared_bank_hashes = + if (diffTest && memParam.sharedEnable) { + Some(Input(Vec(sharedHashCount, new PhysicalBankHash(frontendParam)))) + } else { + None + } + + }) + + val control: Instance[MemoryControl] = + Instantiate(new MemoryControl(gpParam, transportParam, permissionParam, trackingParam, walkerParam)) + + val preparation: Instance[Preparation] = Instantiate(new Preparation( + framework.core.memory.access.preflight.configs.Config( + permissionParam, + transportParam.mapping, + transportParam.regions + ), + walkerParam + )) + + val ctrl: Instance[Ctrl] = Instantiate(new Ctrl(frontendParam, memParam, permissionParam)) + val toMem: Instance[ToMem] = + Instantiate(new ToMem(frontendParam, memParam, gpParam, transportParam, permissionParam)) + val bankRouter: Instance[BankRouter] = + Instantiate(new BankRouter(frontendParam, memParam, gpParam, simp.diffTest)) + val prm: Instance[PRM] = Instantiate(new PRM(memParam, simp.diffTest)) + + io.traceMemory := bankRouter.io.traceMemory + prm.io.banks <> bankRouter.io.physical + bankRouter.io.mapping <> io.mapping + io.shared.bank.mapping <> bankRouter.io.shared.mapping + bankRouter.io.hashes.foreach(_ := prm.io.hashes.get) + io.bank_hashes.foreach { hashes => + for (i <- 0 until memParam.bankNum) + hashes(i) := bankRouter.io.bankHashes.get(i) + for (i <- 0 until sharedHashCount) + hashes(memParam.bankNum + i) := io.shared_bank_hashes.get(i) + } + + // Query the router layout. + bankRouter.io.queryVbank := toMem.io.queryVbank + bankRouter.io.queryShared := toMem.io.queryShared + toMem.io.queryGroups := bankRouter.io.queryGroups + ctrl.io.clearBusy := bankRouter.io.clearBusy + +//===----------------------------------------------------------------------===// +// Control and transport connections +//===----------------------------------------------------------------------===// + val commands = Instantiate(new RRArbiter( + new MemoryIssue(frontendParam, memParam, permissionParam), + 2 + )) + + commands.io.in(0) <> io.issue + commands.io.in(1) <> io.ball.request + ctrl.io.issue <> commands.io.out + val childOwner = RegInit(VecInit(Seq.fill(frontendParam.rob_entries)(false.B))) + when(ctrl.io.issue.fire)(childOwner(ctrl.io.issue.bits.operation.rob_id) := commands.io.chosen === 1.U) + val child = childOwner(ctrl.io.complete.bits.rob_id) + io.complete.valid := ctrl.io.complete.valid && !child + io.complete.bits := ctrl.io.complete.bits + io.ball.response.valid := ctrl.io.complete.valid && child + io.ball.response.bits := ctrl.io.complete.bits + ctrl.io.complete.ready := Mux(child, io.ball.response.ready, io.complete.ready) + io.ball.metadata <> toMem.io.metadata + io.shared.move <> ctrl.io.mvover + ctrl.io.request <> toMem.io.request + toMem.io.response <> ctrl.io.response + + control.io.memory <> toMem.io.memory.control + control.io.preparation <> preparation.io + io.memory <> control.io.service + io.ddr <> toMem.io.memory.axi + + bankRouter.io.requests <> io.banks + bankRouter.io.ballLayoutRequest <> io.ballLayoutRequest + io.ballLayoutResponse <> bankRouter.io.ballLayoutResponse + io.gpLayoutRequest.zip(bankRouter.io.gpLayoutRequest).foreach { case (source, sink) => sink <> source } + io.gpLayoutResponse.zip(bankRouter.io.gpLayoutResponse).foreach { case (sink, source) => sink <> source } + + io.image.zip(toMem.io.image).foreach { case (sink, source) => sink <> source } + io.gpBanks.foreach { ports => + for (port <- 0 until kernelPorts) bankRouter.io.directRequest(port) <> ports(port) + } + + bankRouter.io.directRequest(kernelPorts) <> toMem.io.bankRead + bankRouter.io.directRequest(kernelPorts + 1) <> toMem.io.bankWrite + bankRouter.io.shared.local <> io.shared.bank.local + bankRouter.io.config <> ctrl.io.config + + // Shared path passthrough + io.shared.bank.requests <> bankRouter.io.shared.requests + io.shared.bank.config <> bankRouter.io.shared.config +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/MemoryControl.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/MemoryControl.scala new file mode 100644 index 000000000..fa90ae0bb --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/MemoryControl.scala @@ -0,0 +1,242 @@ +package framework.acc.memdomain + +import framework.acc.gpdomain.configs.GpDomainParam +import framework.core.memory.access.preflight.configs.PmpParams +import framework.ip.memory.interlock.{Params => TrackingParam} +import framework.acc.memdomain.tomem.configs.{Params => TransportParam} +import framework.core.memory.access.mmu.configs.{Params => WalkerParam} + +import framework.core.memory.access.preflight.configs.PmpParams + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.core.memory.MemoryPort +import framework.core.memory.access.preflight.{Command, Error, PmpEntry, PreparationPort} +import framework.acc.memdomain.toddr.{DmaError, DmaStatus} +import framework.acc.memdomain.tomem.ControlPort + +/** Address-space state frozen with the instruction and consumed only by memory requests. */ +class MemoryContext(permission: PmpParams) extends Bundle { + val satp = UInt(64.W) + val privilege = UInt(2.W) + val sum = Bool() + val mxr = Bool() + val pmp = + Vec(permission.entries, new PmpEntry(permission)) +} + +/** Tracks the three DMA producers, independent of frontend tags and ROB retirement. */ +@instantiable +class MemoryControl( + gpParam: GpDomainParam, + transportParam: TransportParam, + permissionParam: PmpParams, + trackingParam: TrackingParam, + walkerParam: WalkerParam) + extends Module { + val transport = transportParam + val producers = if (gpParam.enable) 3 else 2 + require(transport.tracking.entries >= producers && transport.prepared.idBits >= log2Ceil(producers)) + + @public + val io = IO(new Bundle { + val service = new MemoryPort(trackingParam, walkerParam) + val preparation = new PreparationPort(transportParam.mapping, permissionParam, walkerParam) + val memory = Flipped(new ControlPort(transportParam, permissionParam)) + }) + + val active = RegInit(VecInit(Seq.fill(3)(false.B))) + val dispatchPending = RegInit(VecInit(Seq.fill(3)(false.B))) + val preparePending = RegInit(VecInit(Seq.fill(3)(false.B))) + val mapReady = RegInit(VecInit(Seq.fill(3)(false.B))) + val granted = RegInit(VecInit(Seq.fill(3)(false.B))) + val doneIssued = RegInit(VecInit(Seq.fill(3)(false.B))) + val coherent = RegInit(VecInit(Seq.fill(3)(false.B))) + val released = RegInit(VecInit(Seq.fill(3)(false.B))) + val completionPending = RegInit(VecInit(Seq.fill(3)(false.B))) + val completionStatus = Reg(Vec(3, new DmaStatus)) + val contexts = Reg(Vec(3, new MemoryContext(permissionParam))) + val shapes = Reg(Vec(3, new Command(transportParam.mapping))) + val halted = RegInit(false.B) + val firstFault = RegInit(0.U.asTypeOf(new DmaStatus)) + + def stop(error: UInt, address: UInt): Unit = { + when(!halted) { + halted := true.B + firstFault.error := error + firstFault.address := address + } + } + + def dmaError(error: UInt): UInt = MuxLookup(error, DmaError.Context.U)(Seq( + Error.Ok.U -> DmaError.None.U, + Error.Shape.U -> DmaError.Shape.U, + Error.Overflow.U -> DmaError.Shape.U, + Error.PageFault.U -> DmaError.PageFault.U, + Error.AccessFault.U -> DmaError.AccessFault.U, + Error.Capacity.U -> DmaError.Capacity.U, + Error.Context.U -> DmaError.Context.U + )) + + // Producer 0 is the bank reader, 1 the writer, and 2 the image reader. + for (i <- 0 until 3) { + val access = if (i == 1) io.memory.write.bits.access else io.memory.read.bits.access + val offered = + if (i >= producers) false.B + else if (i == 1) io.memory.write.valid + else io.memory.read.valid && io.memory.read.bits.producer === i.U + when(offered && !active(i) && !halted) { + active(i) := true.B + dispatchPending(i) := true.B + preparePending(i) := false.B + mapReady(i) := false.B + granted(i) := false.B + doneIssued(i) := false.B + coherent(i) := false.B + released(i) := false.B + contexts(i) := access.context + shapes(i).id := i.U + shapes(i).baseVA := access.baseVA + shapes(i).rows := access.rows(31, 0) + shapes(i).columns := access.columns(31, 0) + shapes(i).spanBytes := access.spanBytes(31, 0) + shapes(i).columnStride := access.columnStride(31, 0) + shapes(i).rowStride := access.rowStride(31, 0) + shapes(i).write := (i == 1).B + shapes(i).mode := access.context.satp(63, 60) + shapes(i).rootPpn := access.context.satp(43, 0) + shapes(i).privilege := access.context.privilege + shapes(i).sum := access.context.sum + shapes(i).mxr := access.context.mxr + when(access.rows(63, 32).orR || access.columns(63, 32).orR || access.spanBytes(63, 32).orR || + access.columnStride(63, 32).orR || access.rowStride(63, 32).orR) { + stop(DmaError.Shape.U, access.baseVA) + } + } + io.memory.decisions(i).parentId := i.U + io.memory.decisions(i).valid := active(i) && ((mapReady(i) && granted(i)) || halted) + io.memory.decisions(i).fault := firstFault + val finished = io.memory.finished(i) + when(finished.valid && finished.bits.error =/= DmaError.None.U) { + stop(finished.bits.error, finished.bits.address) + } + // A rejected image may finish locally without ever starting DMA. + val localFailure = !active(i) && finished.bits.error =/= DmaError.None.U + when(!completionPending(i) && finished.valid && (localFailure || (active(i) && (released(i) || halted)))) { + completionPending(i) := true.B + completionStatus(i) := Mux(halted, firstFault, finished.bits) + } + io.memory.complete(i).valid := completionPending(i) + io.memory.complete(i).bits := completionStatus(i) + when(io.memory.complete(i).fire) { + completionPending(i) := false.B + active(i) := false.B + } + } + + val dispatchOffer = RegInit(false.B) + val dispatchSlot = Reg(UInt(2.W)) + when(!dispatchOffer && dispatchPending.asUInt.orR && !halted) { + dispatchOffer := true.B + dispatchSlot := PriorityEncoder(dispatchPending) + } + io.memory.coherence.dispatch.valid := dispatchOffer && !halted + io.memory.coherence.dispatch.bits.id := dispatchSlot + when(io.memory.coherence.dispatch.fire) { + dispatchOffer := false.B + dispatchPending(dispatchSlot) := false.B + preparePending(dispatchSlot) := true.B + } + + val prepareOffer = RegInit(false.B) + val prepareSlot = Reg(UInt(2.W)) + when(!prepareOffer && preparePending.asUInt.orR && !halted) { + prepareOffer := true.B + prepareSlot := PriorityEncoder(preparePending) + } + io.preparation.command.valid := prepareOffer && !halted + io.preparation.command.bits := shapes(prepareSlot) + when(io.preparation.command.fire) { + prepareOffer := false.B + preparePending(prepareSlot) := false.B + } + val contextId = io.preparation.contextTag + io.preparation.contextValid := contextId < 3.U && active(contextId) + io.preparation.contextId := contextId + io.preparation.contextPmp := contexts(contextId).pmp + + val range = io.preparation.ranges + val info = io.memory.coherence.accessInfo + info.valid := range.valid && range.bits.error === Error.Ok.U && !halted + info.bits.id := range.bits.id + info.bits.hasMemory := true.B + info.bits.base := range.bits.pa + info.bits.bytes := range.bits.bytes + info.bits.write := range.bits.write + info.bits.last := range.bits.last + range.ready := range.bits.error =/= Error.Ok.U || (info.ready && !halted) + when(range.fire && range.bits.error =/= Error.Ok.U) { + stop(dmaError(range.bits.error), range.bits.va) + } + io.preparation.ready.ready := true.B + when(io.preparation.ready.fire) { + val result = io.preparation.ready.bits + assert(result.id < 3.U && active(result.id), "Prepared result has no DMA owner") + mapReady(result.id) := true.B + when(result.error =/= Error.Ok.U)(stop(dmaError(result.error), result.va)) + } + io.memory.coherence.grant.ready := true.B + when(io.memory.coherence.grant.fire) { + val id = io.memory.coherence.grant.bits.tag + assert(id < 3.U && active(id), "Coherence grant has no DMA owner") + granted(id) := true.B + } + io.preparation.queries := io.memory.queries + io.memory.mappings := io.preparation.results + + // Finishing the bank stream precedes post-DMA cache maintenance and map release. + val doneCandidates = + VecInit((0 until 3).map(i => active(i) && io.memory.finished(i).valid && mapReady(i) && granted(i) && !doneIssued(i))) + val doneOffer = RegInit(false.B) + val doneSlot = Reg(UInt(2.W)) + when(!doneOffer && doneCandidates.asUInt.orR && !halted) { + doneOffer := true.B + doneSlot := PriorityEncoder(doneCandidates) + } + io.memory.coherence.done.valid := doneOffer && !halted + io.memory.coherence.done.bits.tag := doneSlot + io.memory.coherence.done.bits.ok := true.B + when(io.memory.coherence.done.fire) { + doneOffer := false.B + doneIssued(doneSlot) := true.B + } + io.memory.coherence.complete.ready := true.B + when(io.memory.coherence.complete.fire) { + val id = io.memory.coherence.complete.bits.tag + assert(id < 3.U && active(id), "Coherence completion has no DMA owner") + coherent(id) := true.B + } + val releaseCandidates = VecInit((0 until 3).map(i => active(i) && coherent(i) && !released(i))) + val releaseOffer = RegInit(false.B) + val releaseSlot = Reg(UInt(2.W)) + when(!releaseOffer && releaseCandidates.asUInt.orR && !halted) { + releaseOffer := true.B + releaseSlot := PriorityEncoder(releaseCandidates) + } + io.preparation.release.valid := releaseOffer && !halted + io.preparation.release.bits.id := releaseSlot + when(io.preparation.release.fire) { + releaseOffer := false.B + released(releaseSlot) := true.B + } + + io.memory.coherence.cpuQuery := io.service.cpuQuery + io.service.cpuAllow := io.memory.coherence.cpuAllow && !halted + io.service.maintenance <> io.memory.coherence.maintenance + io.memory.coherence.maintained <> io.service.maintained + io.service.pteRequest <> io.preparation.pteRequest + io.preparation.pteResponse <> io.service.pteResponse + io.memory.coherence.cancel.valid := false.B + io.memory.coherence.cancel.bits := 0.U.asTypeOf(io.memory.coherence.cancel.bits) +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/build.sc b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/build.sc new file mode 100644 index 000000000..4998fa422 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/build.sc @@ -0,0 +1,23 @@ +import mill._ +import $file.^.^.^.^.^.^.^.^.^.^.settings + +val ipRoot: os.Path = millSourcePath + +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.acc.memdomain.bank_router.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/BankRouter.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/blt/AccPipe.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/blt/BLT.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/configs/uvm.toml b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/configs/uvm.toml new file mode 100644 index 000000000..df9b1fde4 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/configs/uvm.toml @@ -0,0 +1,48 @@ +[parameters] +diffTest = false + +[parameters.frontend] +ownerId = 7 +commandDataBits = 64 +rob_entries = 4 +bank_alias_enable = true +rs_out_of_order_response = true +bank_id_len = 4 +max_private_vbank = 3 +shared_vbank_base = 16 +iter_len = 16 +sub_rob_enable = false +sub_rob_depth = 4 + +[parameters.memory] +sharedPorts = 0 +bankNum = 4 +bankLayoutEnable = true +bankWidth = 128 +bankEntries = 8 +bankMaskLen = 16 +virtualBankCount = 4 +sharedEnable = false +sharedEntries = 0 +sharedBankEntries = 0 +sharedBankNum = 0 +sharedInputChannels = 0 +sharedDefaultGroupCount = 0 +bankRequestPorts = 2 +dma_buswidth = 128 +memAddrLen = 44 +mmioEnable = false +mmioBankNum = 0 +mmioBankEntries = 0 +mmioBankWidth = 0 +mmioReadWidth = 0 + +[parameters.gp] +enable = false +laneNumber = 1 +vLen = 128 +eLen = 64 +iBufWords = 16 +memoryPorts = 1 + +[defines] diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/agent/driver.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/agent/driver.svh new file mode 100644 index 000000000..6f9eeb3d9 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/agent/driver.svh @@ -0,0 +1,25 @@ +class driver extends uvm_driver #(cycle_item); + `uvm_component_utils(driver) + virtual router_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual router_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "TSS interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item req; + forever begin + seq_item_port.get_next_item(req); + @(negedge vif.clock); + vif.reset = req.reset; + vif.source = req.inputs; + @(posedge vif.clock); + req.outputs = vif.outputs; + #1; + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/agent/item.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/agent/item.svh new file mode 100644 index 000000000..0cbe365ce --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/agent/item.svh @@ -0,0 +1,22 @@ +class cycle_item extends uvm_sequence_item; + `uvm_object_utils(cycle_item) + logic [`BR_INPUT_BITS-1:0] inputs; + logic [`BR_OUTPUT_BITS-1:0] outputs; + bit reset; + function new(string name = "cycle_item"); + super.new(name); + endfunction +endclass +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(data, UVM_ALL_ON) + `uvm_field_int(writing, UVM_ALL_ON) + `uvm_field_int(port, UVM_ALL_ON) + `uvm_object_utils_end + logic [`BR_DATA_BITS-1:0] data; + bit writing; + int port; + function new(string name = "result_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/agent/monitor.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/agent/monitor.svh new file mode 100644 index 000000000..d42bc55c3 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/agent/monitor.svh @@ -0,0 +1,25 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual router_if vif; + uvm_analysis_port #(cycle_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual router_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "TSS monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item t; + forever begin + @(posedge vif.clock); + t = cycle_item::type_id::create("observed"); + t.inputs = vif.inputs; + t.outputs = vif.outputs; + t.reset = vif.reset; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/agent/peer.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/agent/peer.svh new file mode 100644 index 000000000..a4bafede6 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/agent/peer.svh @@ -0,0 +1,56 @@ +class peer extends uvm_component; + `uvm_component_utils(peer) + virtual router_if vif; + const int banks[`BR_BANKS][17] = `BR_BANK_OFFSETS; + logic [`BR_DATA_BITS-1:0] memory[`BR_BANKS][`BR_ROWS]; + bit initialized[`BR_BANKS][`BR_ROWS]; + int cycle; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual router_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Router interface is required") + endfunction + task run_phase(uvm_phase phase); + forever begin + logic [`BR_INPUT_BITS-1:0] next_backend; + @(posedge vif.clock); + next_backend = '0; + cycle++; + if (!vif.reset) + for (int b = 0; b < `BR_BANKS; b++) begin + if (vif.outputs[banks[b][14]]) + for (int row = 0; row < `BR_ROWS; row++) begin + memory[b][row] = '0; + initialized[b][row] = 1; + end + if (vif.outputs[banks[b][0]] && vif.inputs[banks[b][1]]) begin + int row = int'(vif.outputs[banks[b][2]+:16]); + if (row >= `BR_ROWS || !initialized[b][row]) + `uvm_fatal("MEMORY", "Uninitialized physical bank read") + next_backend[banks[b][3]] = 1; + next_backend[banks[b][5]+:`BR_DATA_BITS] = memory[b][row]; + end + if (vif.outputs[banks[b][6]] && vif.inputs[banks[b][7]]) begin + int row = int'(vif.outputs[banks[b][8]+:16]); + if (row >= `BR_ROWS) `uvm_fatal("MEMORY", "Physical bank address overflow") + for (int byte_index = 0; byte_index < `BR_MASK_BITS; byte_index++) + if (vif.outputs[banks[b][10]+byte_index]) + memory[b][row][byte_index*8+:8] = vif.outputs[banks[b][9]+byte_index*8+:8]; + // Every row is explicitly initialized by a full write before partial writes. + initialized[b][row] = 1; + next_backend[banks[b][11]] = 1; + next_backend[banks[b][13]] = 1; + end + end + @(negedge vif.clock); + for (int b = 0; b < `BR_BANKS; b++) begin + next_backend[banks[b][1]] = cycle % 4 != b % 4; + next_backend[banks[b][7]] = cycle % 5 != b % 5; + end + vif.backend = next_backend; + end + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/env/env.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/env/env.svh new file mode 100644 index 000000000..ea4857b7d --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/env/env.svh @@ -0,0 +1,33 @@ +class env extends uvm_env; + `uvm_component_utils(env) + uvm_analysis_imp #(uvm_sequence_item, env) reset_events; + driver source; + monitor mon; + peer backend; + model reference; + uvm_sequencer #(cycle_item) seqr; + result_board #(result_item, 1) scoreboard; + function new(string name, uvm_component parent); + super.new(name, parent); + reset_events = new("reset_events", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + source = driver::type_id::create("source", this); + mon = monitor::type_id::create("monitor", this); + backend = peer::type_id::create("backend", this); + reference = model::type_id::create("model", this); + seqr = new("seqr", this); + scoreboard = result_board#(result_item, 1)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + source.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(reference.analysis_export); + reference.expected.connect(scoreboard.expected_export); + reference.actual.connect(scoreboard.actual_export); + reference.cancelled.connect(reset_events); + endfunction + function void write(uvm_sequence_item t); + scoreboard.cancel_pending(); + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/env/model.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/env/model.svh new file mode 100644 index 000000000..0fbb446ca --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/env/model.svh @@ -0,0 +1,225 @@ +class model extends uvm_subscriber #(cycle_item); + `uvm_component_utils(model) + uvm_analysis_port #(result_item) expected, actual; + uvm_analysis_port #(uvm_sequence_item) cancelled; + const int clients[`BR_CLIENTS][17] = `BR_CLIENT_OFFSETS; + const int banks[`BR_BANKS][17] = `BR_BANK_OFFSETS; + bit live[`BR_BANKS]; + int architecture[`BR_BANKS], alias_id[`BR_BANKS], group_id[`BR_BANKS]; + int columns[`BR_ALIASES], layouts[`BR_ALIASES]; + logic [`BR_DATA_BITS-1:0] memory[`BR_BANKS][`BR_ROWS]; + bit initialized[`BR_BANKS][`BR_ROWS]; + bit pending[`BR_CLIENTS], writing[`BR_CLIENTS]; + bit accepted_now[`BR_CLIENTS]; + int selected_now[`BR_CLIENTS], row_now[`BR_CLIENTS]; + logic [`BR_DATA_BITS-1:0] data_now[`BR_CLIENTS]; + logic [`BR_MASK_BITS-1:0] mask_now[`BR_CLIENTS]; + int delayed_query[2]; + bit query_enabled[2]; + covergroup behavior with function sample ( + int port, bit write, int groups, bit stall, bit remapped, bit retired, bit transfer + ); + option.per_instance = 1; + ports: coverpoint port {bins all[] = {[0 : `BR_CLIENTS - 1]};} + direction: coverpoint write; + layout: coverpoint groups {bins single = {1}; bins multi = {[2 : `BR_BANKS]};} + backpressure: coverpoint stall; + renaming: coverpoint remapped; + retirement: coverpoint retired; + transfers: coverpoint transfer; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected = new("expected", this); + actual = new("actual", this); + cancelled = new("cancelled", this); + behavior = new(); + endfunction + function void verify(bit ok, string message); + evidence::checks++; + if (!ok) `uvm_fatal("ROUTER", message) + endfunction + function int route(int tag, int group_index); + int found = -1; + foreach (live[b]) + if (live[b] && alias_id[b] == tag && group_id[b] == group_index) begin + verify(found == -1, "Ambiguous logical bank binding"); + found = b; + end + verify(found >= 0, "Request has no allocated bank binding"); + return found; + endfunction + function void write(cycle_item t); + bit bound[`BR_BANKS]; + if (t.reset) begin + foreach (live[b]) live[b] = 0; + foreach (columns[a]) begin + columns[a] = 0; + layouts[a] = 0; + end + foreach (pending[c]) pending[c] = 0; + query_enabled[0] = 0; + query_enabled[1] = 0; + cancelled.write(null); + return; + end + if (query_enabled[1]) + verify(t.outputs[`BR_OUT_QUERYGROUPS_OFFSET+:`BR_OUT_QUERYGROUPS_WIDTH] == delayed_query[1], + "Group count query mismatch"); + delayed_query[1] = columns[delayed_query[0]]; + query_enabled[1] = query_enabled[0]; + delayed_query[0] = int'(t.inputs[`BR_IN_QUERYVBANK_OFFSET+:`BR_IN_QUERYVBANK_WIDTH]); + query_enabled[0] = 1; + foreach (pending[c]) begin + result_item result; + int tag, group_index, row, physical; + bit wr, rd; + accepted_now[c] = 0; + rd = t.inputs[clients[c][0]] && t.outputs[clients[c][1]]; + wr = t.inputs[clients[c][6]] && t.outputs[clients[c][7]]; + verify(!(rd && wr), "Read and write accepted simultaneously on one port"); + if (rd || wr) begin + verify(!pending[c], "Second request before the port response retired"); + tag = int'(t.inputs[clients[c][14]+:`BR_IN_REQUESTS_0_BANK_ID_WIDTH]); + group_index = int'(t.inputs[clients[c][15]+:`BR_IN_REQUESTS_0_GROUP_ID_WIDTH]); + physical = route(tag, group_index); + row = int'(t.inputs[clients[c][wr?8 : 2]+:16]); + verify(row < `BR_ROWS, "Logical bank row overflow"); + accepted_now[c] = 1; + selected_now[c] = physical; + row_now[c] = row; + data_now[c] = t.inputs[clients[c][9]+:`BR_DATA_BITS]; + mask_now[c] = t.inputs[clients[c][10]+:`BR_MASK_BITS]; + result = result_item::type_id::create("expected"); + result.port = c; + result.writing = wr; + result.set_transaction_id(c); + if (wr) begin + for (int b = 0; b < `BR_MASK_BITS; b++) + if (mask_now[c][b]) memory[physical][row][b*8+:8] = data_now[c][b*8+:8]; + initialized[physical][row] = 1; + result.data = 1; + end else begin + verify(initialized[physical][row], "Read of an uninitialized logical bank row"); + result.data = memory[physical][row]; + end + expected.write(result); + pending[c] = 1; + writing[c] = wr; + behavior.sample(c, wr, columns[tag], 0, tag >= 4, 0, 0); + end + if(t.outputs[clients[c][3]]&&t.inputs[clients[c][4]]||t.outputs[clients[c][11]]&&t.inputs[clients[c][12]])begin + verify(pending[c], "Unsolicited client response"); + result = result_item::type_id::create("actual"); + result.port = c; + result.writing = writing[c]; + result.set_transaction_id(c); + result.data=writing[c]?`BR_DATA_BITS'(t.outputs[clients[c][13]]):t.outputs[clients[c][5]+:`BR_DATA_BITS]; + actual.write(result); + pending[c] = 0; + end + if(t.outputs[clients[c][3]]&&!t.inputs[clients[c][4]]||t.outputs[clients[c][11]]&&!t.inputs[clients[c][12]]) + behavior.sample(c, writing[c], 1, 1, 0, 0, 0); + end + for (int b = 0; b < `BR_BANKS; b++) begin + bit rd, wr; + int owner; + rd = t.outputs[banks[b][0]] && t.inputs[banks[b][1]]; + wr = t.outputs[banks[b][6]] && t.inputs[banks[b][7]]; + if (rd || wr) begin + owner = -1; + foreach (accepted_now[c]) + if (accepted_now[c] && selected_now[c] == b) begin + verify(owner == -1, "Multiple clients accepted by one bank"); + owner = c; + end + verify(owner >= 0, "Physical request without an accepted client"); + verify(writing[owner] == wr, "Physical write/read direction mismatch"); + verify(t.outputs[banks[b][wr?8 : 2]+:16] == row_now[owner], + "Physical row changed during routing"); + if (wr) + verify( + t.outputs[banks[b][9]+:`BR_DATA_BITS]===data_now[owner]&&t.outputs[banks[b][10]+:`BR_MASK_BITS]===mask_now[owner], + "Write data or byte mask changed during routing"); + accepted_now[owner] = 0; + end + end + foreach (accepted_now[c]) + verify(!accepted_now[c], "Client accepted without a physical bank request"); + if (t.inputs[`BR_IN_CONFIG_VALID_OFFSET] && t.outputs[`BR_OUT_CONFIG_READY_OFFSET]) begin + int tag, group_index, chosen; + bit alloc, transfer; + tag = int'(t.inputs[`BR_IN_CONFIG_BITS_VBANK_ID_OFFSET+:`BR_IN_CONFIG_BITS_VBANK_ID_WIDTH]); + group_index=int'(t.inputs[`BR_IN_CONFIG_BITS_GROUP_ID_OFFSET+:`BR_IN_CONFIG_BITS_GROUP_ID_WIDTH]); + alloc = t.inputs[`BR_IN_CONFIG_BITS_ALLOC_OFFSET]; + transfer = t.inputs[`BR_IN_CONFIG_BITS_TRANSFER_OFFSET]; + if (transfer) begin + int source=int'(t.inputs[`BR_IN_CONFIG_BITS_SOURCE_BANK_ID_OFFSET+:`BR_IN_CONFIG_BITS_SOURCE_BANK_ID_WIDTH]); + foreach (live[b]) + if (live[b] && architecture[b] == source) begin + alias_id[b] = tag; + architecture[b] = tag; + group_id[b] += columns[tag]; + end + columns[tag] += columns[source]; + columns[source] = 0; + layouts[tag] += layouts[source]; + layouts[source] = 0; + behavior.sample(0, 0, columns[tag], 0, 0, 0, 1); + end else if (alloc) begin + if (group_index == 0) foreach (live[b]) if (live[b] && architecture[b] == tag) live[b] = 0; + chosen = -1; + foreach (live[b]) if (!live[b] && chosen == -1) chosen = b; + verify(chosen >= 0, "Allocation exceeded physical capacity"); + live[chosen] = 1; + architecture[chosen] = tag; + alias_id[chosen] = tag; + group_id[chosen] = group_index; + bound[chosen] = 1; + columns[tag] = t.inputs[`BR_IN_CONFIG_BITS_IS_MULTI_OFFSET] ? group_index + 1 : 1; + layouts[tag] = columns[tag]; + if (t.inputs[`BR_IN_CONFIG_BITS_CLEAR_OFFSET]) + for (int row = 0; row < `BR_ROWS; row++) begin + memory[chosen][row] = '0; + initialized[chosen][row] = 1; + end + end else begin + foreach (live[b]) if (live[b] && architecture[b] == tag) live[b] = 0; + columns[tag] = 0; + layouts[tag] = 0; + end + end + if (t.inputs[`BR_IN_MAPPING_VALID_OFFSET] && t.outputs[`BR_OUT_MAPPING_READY_OFFSET]) begin + int bank_id, tag, previous, count; + bank_id = int'(t.inputs[`BR_IN_MAPPING_BITS_BANK_OFFSET+:`BR_IN_MAPPING_BITS_BANK_WIDTH]); + tag = int'(t.inputs[`BR_IN_MAPPING_BITS_VBANK_OFFSET+:`BR_IN_MAPPING_BITS_VBANK_WIDTH]); + previous=int'(t.inputs[`BR_IN_MAPPING_BITS_PREVIOUS_OFFSET+:`BR_IN_MAPPING_BITS_PREVIOUS_WIDTH]); + count = layouts[bank_id]; + if (t.inputs[`BR_IN_MAPPING_BITS_ALLOCATE_OFFSET]) begin + if (t.inputs[`BR_IN_MAPPING_BITS_REUSE_OFFSET]) + foreach (live[b]) if (live[b] && alias_id[b] == previous) live[b] = 0; + for (int group_index = 0; group_index < count; group_index++) begin + int chosen = -1; + foreach (live[b]) if (!live[b] && chosen == -1) chosen = b; + verify(chosen >= 0, "Alias allocation exceeded physical capacity"); + live[chosen] = 1; + architecture[chosen] = bank_id; + alias_id[chosen] = tag; + group_id[chosen] = group_index; + bound[chosen] = 1; + end + columns[tag] = count; + behavior.sample(0, 0, count, 0, 1, 0, 0); + end + if (t.inputs[`BR_IN_MAPPING_BITS_RETIRE_OFFSET]) begin + foreach (live[b]) if (live[b] && alias_id[b] == previous) live[b] = 0; + columns[previous] = 0; + behavior.sample(0, 0, count, 0, tag >= 4, 1, 0); + end + if(t.inputs[`BR_IN_MAPPING_BITS_WRITE_OFFSET]&&!t.inputs[`BR_IN_MAPPING_BITS_ALLOCATE_OFFSET]) + foreach (live[b]) if (live[b] && alias_id[b] == tag) bound[b] = 1; + end + foreach (bound[b]) + verify(t.outputs[banks[b][16]] == bound[b], "Physical binding pulse mismatch"); + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/filelist.f b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/filelist.f new file mode 100644 index 000000000..fecc7d73e --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/BankRouter/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/interface.sv b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/interface.sv new file mode 100644 index 000000000..38389c2fa --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/interface.sv @@ -0,0 +1,9 @@ +`include "parameters.svh" +interface router_if ( + input logic clock +); + logic reset = 1; + logic [`BR_INPUT_BITS-1:0] source = '0, backend = '0; + wire [ `BR_INPUT_BITS-1:0] inputs = source | backend; + wire [`BR_OUTPUT_BITS-1:0] outputs; +endinterface diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/package.sv b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/package.sv new file mode 100644 index 000000000..301cc988f --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/package.sv @@ -0,0 +1,14 @@ +`include "parameters.svh" +package router_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/peer.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..0d0db7514 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/seq/sequence.svh @@ -0,0 +1,108 @@ +class scenario_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(scenario_sequence) + const int clients[`BR_CLIENTS][17] = `BR_CLIENT_OFFSETS; + logic [`BR_INPUT_BITS-1:0] pins; + logic [`BR_OUTPUT_BITS-1:0] observed; + function new(string name = "scenario"); + super.new(name); + endfunction + task tick(bit reset = 0); + cycle_item t = cycle_item::type_id::create("cycle"); + start_item(t); + t.inputs = pins; + t.reset = reset; + finish_item(t); + observed = t.outputs; + endtask + task layout(int tag, int groups, bit clear = 0, bit transfer = 0, int source = 0); + for (int group_index = 0; group_index < (groups == 0 ? 1 : groups); group_index++) begin + pins[`BR_IN_CONFIG_VALID_OFFSET] = 1; + pins[`BR_IN_CONFIG_BITS_VBANK_ID_OFFSET+:`BR_IN_CONFIG_BITS_VBANK_ID_WIDTH] = tag; + pins[`BR_IN_CONFIG_BITS_GROUP_ID_OFFSET+:`BR_IN_CONFIG_BITS_GROUP_ID_WIDTH] = group_index; + pins[`BR_IN_CONFIG_BITS_IS_MULTI_OFFSET] = groups > 1; + pins[`BR_IN_CONFIG_BITS_ALLOC_OFFSET] = groups > 0; + pins[`BR_IN_CONFIG_BITS_CLEAR_OFFSET] = clear; + pins[`BR_IN_CONFIG_BITS_THREAD_ID_OFFSET+:64] = 7; + pins[`BR_IN_CONFIG_BITS_TRANSFER_OFFSET] = transfer; + pins[`BR_IN_CONFIG_BITS_SOURCE_BANK_ID_OFFSET+:`BR_IN_CONFIG_BITS_SOURCE_BANK_ID_WIDTH]=source; + do tick(); while (!observed[`BR_OUT_CONFIG_READY_OFFSET]); + pins[`BR_IN_CONFIG_VALID_OFFSET] = 0; + if (transfer) break; + end + repeat (3) tick(); + endtask + task mapping(int bank_id, int tag, int previous, bit allocate, bit retire, bit reuse = 0); + pins[`BR_IN_MAPPING_VALID_OFFSET] = 1; + pins[`BR_IN_MAPPING_BITS_BANK_OFFSET+:`BR_IN_MAPPING_BITS_BANK_WIDTH] = bank_id; + pins[`BR_IN_MAPPING_BITS_VBANK_OFFSET+:`BR_IN_MAPPING_BITS_VBANK_WIDTH] = tag; + pins[`BR_IN_MAPPING_BITS_PREVIOUS_OFFSET+:`BR_IN_MAPPING_BITS_PREVIOUS_WIDTH] = previous; + pins[`BR_IN_MAPPING_BITS_ALLOCATE_OFFSET] = allocate; + pins[`BR_IN_MAPPING_BITS_RETIRE_OFFSET] = retire; + pins[`BR_IN_MAPPING_BITS_REUSE_OFFSET] = reuse; + pins[`BR_IN_MAPPING_BITS_THREADID_OFFSET+:64] = 7; + do tick(); while (!observed[`BR_OUT_MAPPING_READY_OFFSET]); + pins[`BR_IN_MAPPING_VALID_OFFSET] = 0; + repeat (3) tick(); + endtask + task transfer(int port, int tag, int group_index, int row, bit writing, + logic [`BR_DATA_BITS-1:0] data = '0, logic [`BR_MASK_BITS-1:0] mask = '1, + int stall = 0); + pins[clients[port][14]+:`BR_IN_REQUESTS_0_BANK_ID_WIDTH] = tag; + pins[clients[port][15]+:`BR_IN_REQUESTS_0_GROUP_ID_WIDTH] = group_index; + pins[clients[port][16]+:64] = 7; + pins[clients[port][writing?6 : 0]] = 1; + pins[clients[port][writing?8 : 2]+:16] = row; + pins[clients[port][9]+:`BR_DATA_BITS] = data; + pins[clients[port][10]+:`BR_MASK_BITS] = mask; + pins[clients[port][writing?12 : 4]] = 0; + do tick(); while (!observed[clients[port][writing?7 : 1]]); + pins[clients[port][writing?6 : 0]] = 0; + // Change the live route payload after acceptance; the old response keeps its owner. + pins[clients[port][14]+:`BR_IN_REQUESTS_0_BANK_ID_WIDTH] = (tag + 1) % `BR_ALIASES; + do tick(); while (!observed[clients[port][writing?11 : 3]]); + repeat (stall) tick(); + pins[clients[port][writing?12 : 4]] = 1; + tick(); + endtask + task body(); + pins = '0; + repeat (3) tick(1); + tick(); + layout(0, 1, 1); + layout(1, 1, 1); + for (int c = 0; c < `BR_CLIENTS; c++) + for (int row = 0; row < `BR_ROWS; row++) begin + transfer(c, c % 2, 0, row, 1, + `BR_DATA_BITS'(64'h123456789abcdef0) ^ (128'hdeadbeef << 64) ^ row ^ (c << 8), '1, + c % 4); + transfer(c, c % 2, 0, row, 0, '0, '1, c % 4); + end + for (int bit_index = 0; bit_index < `BR_MASK_BITS; bit_index++) begin + transfer(bit_index % `BR_CLIENTS, 0, 0, 0, 1, '1, `BR_MASK_BITS'(1 << bit_index)); + transfer((bit_index + 1) % `BR_CLIENTS, 0, 0, 0, 0); + end + layout(0, 0); + layout(1, 0); + layout(2, 2, 1); + for (int c = 0; c < `BR_CLIENTS; c++) + for (int g = 0; g < 2; g++) transfer(c, 2, g, c, 0, '0, '1, 5); + mapping(2, 6, 2, 1, 0); + for (int g = 0; g < 2; g++) + for (int row = 0; row < `BR_ROWS; row++) + transfer(g, 6, g, row, 1, `BR_DATA_BITS'(row + g * 64), '1); + mapping(2, 6, 2, 0, 1); + for (int g = 0; g < 2; g++) transfer(3, 6, g, 0, 0); + mapping(2, 10, 6, 1, 0, 1); + for (int g = 0; g < 2; g++) begin + transfer(g, 10, g, 0, 1, '1); + transfer(g, 10, g, 0, 0); + end + mapping(2, 10, 6, 0, 1); + layout(2, 0); + layout(0, 1, 1); + layout(1, 1, 1); + layout(0, 1, 0, 1, 1); + for (int g = 0; g < 2; g++) transfer(3, 0, g, 0, 0); + repeat (6) tick(); + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/suite.toml b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/suite.toml new file mode 100644 index 000000000..021179d07 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/suite.toml @@ -0,0 +1,19 @@ +[model] +manifest="" +library="" +[[tests]] +name="bank_router" +top="bank_router_tb" +test="protocol_test" +filelist="filelist.f" +timeout_ns=200000 +wall_timeout_s=120 +kind="functional" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=["router_pkg::model::behavior"] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=[] +assertion="" diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/tb.sv b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/tb.sv new file mode 100644 index 000000000..6c5e4142c --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/tb.sv @@ -0,0 +1,15 @@ +module bank_router_tb; + import uvm_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + router_if bus (clock); + BankRouter dut ( + .clock(clock), + .reset(bus.reset), + `include "ports.svh" + ); + initial begin + uvm_config_db#(virtual router_if)::set(null, "uvm_test_top.env.*", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/tests/test.svh new file mode 100644 index 000000000..54d137a6f --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/tests/test.svh @@ -0,0 +1,15 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.start(test_env.seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/scala/BankRouter.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/scala/BankRouter.scala new file mode 100644 index 000000000..97660fa2e --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/scala/BankRouter.scala @@ -0,0 +1,353 @@ +package framework.acc.memdomain.bank_router + +import framework.lib.queue.Queue + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.memdomain.configs.MemDomainParam + +import framework.acc.debug.trace.MemoryEvent + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.memdomain._ +import framework.acc.memdomain.bank_router.blt.BankLayoutTable +import framework.acc.memdomain.prm.PhysicalBankPort +import framework.thread.tlink.BankAccess + +/** Allocate a renamed binding, or commit a new binding and retire its previous one. */ +class BankMapping(frontendParam: FrontendParam) extends Bundle { + val bank = UInt(frontendParam.bank_id_len.W) + val vbank = UInt(frontendParam.bank_id_len.W) + val previous = UInt(frontendParam.bank_id_len.W) + val threadId = UInt(64.W) + val allocate = Bool() + val reuse = Bool() + val retire = Bool() + val write = Bool() +} + +// The BLT installs an accepted mapping over several cycles; busy stays high +// until it is installed, and the ROB holds the issuing instruction meanwhile. +class BankMappingIO(frontendParam: FrontendParam) extends DecoupledIO(new BankMapping(frontendParam)) { + val busy = Input(Bool()) +} + +@instantiable +class BankRouter( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val gpParam: GpDomainParam, + val diffTest: Boolean) + extends Module { + val sharedPorts = memParam.sharedPorts + val ownerId = frontendParam.ownerId + + val gpRequests = + if (gpParam.enable) + gpParam.memory.memoryPorts + else 0 + + @public + val io = IO(new Bundle { + + val traceMemory = Output(Vec( + memParam.bankRequestPorts + gpRequests + 2 + memParam.bankNum, + new MemoryEvent + )) + + val requests = Vec(memParam.bankRequestPorts, Flipped(new BankPort(frontendParam, memParam))) + val directRequest = Vec(gpRequests + 2, Flipped(new BankPort(frontendParam, memParam))) + val config = Flipped(Decoupled(new LayoutConfig(frontendParam, memParam))) + val mapping = Flipped(new BankMappingIO(frontendParam)) + val queryVbank = Input(UInt(frontendParam.bank_id_len.W)) + val queryShared = Input(Bool()) + val queryGroups = Output(UInt(memParam.groupCountWidth.W)) + val ballLayoutRequest = Flipped(Decoupled(new BankLayoutRequest(frontendParam))) + val ballLayoutResponse = Decoupled(new BankLayout(memParam)) + val gpLayoutRequest = + Option.when(gpParam.enable)(Flipped(Decoupled(new BankLayoutRequest(frontendParam)))) + val gpLayoutResponse = + Option.when(gpParam.enable)(Decoupled(new BankLayout(memParam))) + val clearBusy = Output(Bool()) + val physical = Vec(memParam.bankNum, Flipped(new PhysicalBankPort(memParam))) + + val hashes = + if (diffTest) Some(Input(Vec(memParam.bankNum, UInt(32.W)))) + else None + + val bankHashes = + if (diffTest) + Some(Output(Vec(memParam.bankNum, new PhysicalBankHash(frontendParam)))) + else None + + val shared = new BankAccess(frontendParam, memParam) + }) + + val blt: Instance[BankLayoutTable] = Instantiate(new BankLayoutTable(frontendParam, memParam, gpParam, diffTest)) + blt.io.physical <> io.physical + blt.io.hashes.foreach(_ := io.hashes.get) + io.bankHashes.foreach(_ := blt.io.bank_hashes.get) + io.traceMemory := blt.io.traceMemory + io.clearBusy := blt.io.clearBusy + blt.io.query_vbank_id := io.queryVbank + + val aliasCount = + 2 * (memParam.virtualBankCount + (if (gpParam.enable) 2 + else + 0)) + frontendParam.rob_entries + + // Shared layout shape belongs to this thread; physical allocation stays in SHM. + val sharedColumns = RegInit(VecInit(Seq.fill(memParam.virtualBankCount)(0.U( + memParam.groupCountWidth.W + )))) + + val sharedGroups = + RegInit(VecInit(Seq.fill(aliasCount)(0.U(memParam.groupCountWidth.W)))) + when(io.shared.config.fire) { + val config = io.shared.config.bits + when(config.transfer) { + val groups = sharedColumns(config.source_bank_id) +& sharedColumns(config.vbank_id) + sharedColumns(config.source_bank_id) := 0.U + sharedGroups(config.source_bank_id) := 0.U + sharedColumns(config.vbank_id) := groups + sharedGroups(config.vbank_id) := groups + }.otherwise { + val groups = Mux(config.alloc, config.group_id +& 1.U, 0.U) + sharedColumns(config.vbank_id) := groups + sharedGroups(config.vbank_id) := groups + } + } + when(io.shared.mapping.fire) { + val change = io.shared.mapping.bits + when(change.allocate)(sharedGroups(change.vbank) := sharedColumns(change.bank)) + when(change.retire)(sharedGroups(change.previous) := 0.U) + } + val sharedQuery = RegNext(io.queryVbank, 0.U) + val sharedResult = RegNext(sharedGroups(sharedQuery), 0.U) + val queryShared = RegNext(RegNext(io.queryShared, false.B), false.B) + io.queryGroups := Mux(queryShared, sharedResult, blt.io.query_group_count) + + val owners = RegInit(VecInit((0 until aliasCount).map(i => + (i % (memParam.virtualBankCount + (if (gpParam.enable) 2 + else 0))).U( + frontendParam.bank_id_len.W + ) + ))) + + val committed = RegInit(VecInit((0 until memParam.virtualBankCount).map(_.U( + frontendParam.bank_id_len.W + )))) + + // Each client has independent lookup lanes and credits; responses hold under backpressure. + val layoutRequests = Seq(io.ballLayoutRequest) ++ io.gpLayoutRequest.toSeq + val layoutResponses = Seq(io.ballLayoutResponse) ++ io.gpLayoutResponse.toSeq + for (((request, response), client) <- layoutRequests.zip(layoutResponses).zipWithIndex) { + val replies = Instantiate(new Queue(new BankLayout(memParam), 4)) + val outstanding = RegInit(0.U(3.W)) + request.ready := outstanding < 4.U + when(request.fire =/= response.fire) { + outstanding := Mux(request.fire, outstanding + 1.U, outstanding - 1.U) + } + replies.io.enq.valid := RegNext(RegNext(request.fire, false.B), false.B) + for (operand <- 0 until 3) { + val query = blt.io.layout(3 * client + operand) + val bank = request.bits.banks(operand) + query.bank := bank + val delayedBank = RegNext(bank, 0.U) + val sharedCount = RegNext(sharedGroups(delayedBank), 0.U) + val owner = owners(bank) + val shared = memParam.sharedEnable.B && + owner >= frontendParam.shared_vbank_base.U && + owner < memParam.virtualBankCount.U + val returnedShared = RegNext(RegNext(shared, false.B), false.B) + val enabled = RegNext(RegNext(request.bits.enabled(operand), false.B), false.B) + replies.io.enq.bits.groups(operand) := Mux(enabled, Mux(returnedShared, sharedCount, query.groups), 0.U) + when(request.fire && request.bits.enabled(operand)) { + assert(bank < aliasCount.U, "Layout query bank exceeds the configured namespace") + } + } + assert(!replies.io.enq.valid || replies.io.enq.ready, "Layout response credit was lost") + response <> replies.io.deq + } + + val localBusy = WireDefault(false.B) + + val sharedMapping = + memParam.sharedEnable.B && io.mapping.bits.bank >= frontendParam.shared_vbank_base.U && + io.mapping.bits.bank < memParam.virtualBankCount.U && + (io.mapping.bits.allocate || io.mapping.bits.retire || io.mapping.bits.write) + + val retireAllowed = !io.mapping.bits.retire || !localBusy + blt.io.mapping.valid := io.mapping.valid && !sharedMapping && retireAllowed + blt.io.mapping.bits := io.mapping.bits + io.shared.mapping.valid := io.mapping.valid && sharedMapping + io.shared.mapping.bits := io.mapping.bits + io.mapping.ready := Mux(sharedMapping, io.shared.mapping.ready, blt.io.mapping.ready && retireAllowed) + io.mapping.busy := blt.io.mapping.busy + when(io.mapping.fire && (io.mapping.bits.allocate || io.mapping.bits.write)) { + owners(io.mapping.bits.vbank) := io.mapping.bits.bank + } + when(io.mapping.fire && io.mapping.bits.retire) { + committed(io.mapping.bits.bank) := io.mapping.bits.vbank + } + + blt.io.config.valid := io.config.valid && !io.config.bits.is_shared + blt.io.config.bits := io.config.bits + io.shared.config.valid := io.config.valid && io.config.bits.is_shared + io.shared.config.bits := io.config.bits + io.config.ready := Mux(io.config.bits.is_shared, io.shared.config.ready, blt.io.config.ready) + + val logical = Wire(Vec(memParam.bankRequestPorts, new BankPort(frontendParam, memParam))) + for (i <- logical.indices) { + logical(i) <> io.requests(i) + } + + if (memParam.sharedEnable && memParam.bankWidth == 128) { + // Reserve channel 0 for Mesh-initiated private-Bank access only when its + // ordinary request/response pair has drained. This preserves the normal + // AccPipe route and prevents one-cycle SRAM responses from being lost. + val normal = io.requests(0) + val routed = logical(0) + val local = io.shared.local + val normalReadPending = RegInit(false.B) + val normalWritePending = RegInit(false.B) + val localPending = RegInit(false.B) + localBusy := localPending + val localIsWrite = RegInit(false.B) + val localTag = Reg(chiselTypeOf(local.request.bits.tag)) + val normalActive = normalReadPending || normalWritePending || + normal.read.req.valid || normal.write.req.valid + val localRequest = local.request.valid && !localPending && !normalActive + val localRead = localRequest && !local.request.bits.write + val localWrite = localRequest && local.request.bits.write + + routed.bank_id := Mux(localRequest, committed(local.request.bits.bank), normal.bank_id) + routed.group_id := Mux(localRequest, 0.U, normal.group_id) + routed.is_shared := Mux(localRequest, false.B, normal.is_shared) + routed.thread_id := Mux(localRequest, ownerId.U(64.W), normal.thread_id) + routed.rob_id := Mux(localRequest, 0.U, normal.rob_id) + routed.inst_id := Mux(localRequest, 0.U, normal.inst_id) + routed.read.req.valid := localRead || (normal.read.req.valid && !localPending) + routed.read.req.bits.addr := Mux(localRead, local.request.bits.addr, normal.read.req.bits.addr) + routed.write.req.valid := localWrite || (normal.write.req.valid && !localPending) + routed.write.req.bits.addr := Mux(localWrite, local.request.bits.addr, normal.write.req.bits.addr) + routed.write.req.bits.data := Mux(localWrite, local.request.bits.data, normal.write.req.bits.data) + routed.write.req.bits.mask := Mux(localWrite, VecInit(local.request.bits.mask.asBools), normal.write.req.bits.mask) + normal.read.req.ready := !localPending && routed.read.req.ready + normal.write.req.ready := !localPending && routed.write.req.ready + local.request.ready := !normalActive && !localPending && + Mux(local.request.bits.write, routed.write.req.ready, routed.read.req.ready) + + routed.read.resp.ready := Mux(localPending && !localIsWrite, local.response.ready, normal.read.resp.ready) + routed.write.resp.ready := Mux(localPending && localIsWrite, local.response.ready, normal.write.resp.ready) + normal.read.resp.valid := routed.read.resp.valid && !localPending + normal.read.resp.bits := routed.read.resp.bits + normal.write.resp.valid := routed.write.resp.valid && !localPending + normal.write.resp.bits := routed.write.resp.bits + local.response.valid := localPending && Mux(localIsWrite, routed.write.resp.valid, routed.read.resp.valid) + local.response.bits.data := routed.read.resp.bits.data + local.response.bits.tag := localTag + local.response.bits.error := false.B + + when(normal.read.req.fire)(normalReadPending := true.B) + when(normal.read.resp.fire)(normalReadPending := false.B) + when(normal.write.req.fire)(normalWritePending := true.B) + when(normal.write.resp.fire)(normalWritePending := false.B) + when(local.request.fire) { + assert(local.request.bits.bank <= frontendParam.max_private_vbank.U) + assert(local.request.bits.addr < memParam.bankEntries.U) + assert(!local.request.bits.write || local.request.bits.mask.andR) + localPending := true.B + localIsWrite := local.request.bits.write + localTag := local.request.bits.tag + } + when(local.response.fire)(localPending := false.B) + } else { + io.shared.local.request.ready := false.B + io.shared.local.response.valid := false.B + io.shared.local.response.bits := 0.U.asTypeOf(io.shared.local.response.bits) + } + + val routedRequests = logical.toSeq ++ io.directRequest.toSeq + require( + !memParam.sharedEnable || io.shared.requests.size >= routedRequests.size, + "Shared-bank channels must cover Ball, GP and DMA request ports" + ) + val privateRequests = blt.io.mem_req.toSeq ++ blt.io.kernel_req.toSeq + for (i <- routedRequests.indices) { + val request = routedRequests(i) + val memory = privateRequests(i) + val owner = owners(request.bank_id) + val shared = + memParam.sharedEnable.B && owner >= frontendParam.shared_vbank_base.U && + owner < memParam.virtualBankCount.U + memory <> request + memory.is_shared := false.B + if (i < io.shared.requests.size) { + val remote = io.shared.requests(i) + // Private and shared responses may have different latency. Keep one route per accepted transfer. + val readOwners = Instantiate(new Queue(Bool(), 8)) + val writeOwners = Instantiate(new Queue(Bool(), 8)) + remote.bank_id := request.bank_id + remote.group_id := request.group_id + remote.thread_id := request.thread_id + remote.rob_id := request.rob_id + remote.inst_id := request.inst_id + remote.is_shared := true.B + + val readReady = Mux(shared, remote.read.req.ready, memory.read.req.ready) + memory.read.req.valid := request.read.req.valid && !shared && readOwners.io.enq.ready + remote.read.req.valid := request.read.req.valid && shared && readOwners.io.enq.ready + remote.read.req.bits := request.read.req.bits + request.read.req.ready := readReady && readOwners.io.enq.ready + readOwners.io.enq.valid := request.read.req.valid && readReady + readOwners.io.enq.bits := shared + val readShared = readOwners.io.deq.bits + val readValid = Mux(readShared, remote.read.resp.valid, memory.read.resp.valid) + request.read.resp.valid := readOwners.io.deq.valid && readValid + request.read.resp.bits := Mux(readShared, remote.read.resp.bits, memory.read.resp.bits) + memory.read.resp.ready := readOwners.io.deq.valid && !readShared && request.read.resp.ready + remote.read.resp.ready := readOwners.io.deq.valid && readShared && request.read.resp.ready + readOwners.io.deq.ready := readValid && request.read.resp.ready + + val writeReady = Mux(shared, remote.write.req.ready, memory.write.req.ready) + memory.write.req.valid := request.write.req.valid && !shared && writeOwners.io.enq.ready + remote.write.req.valid := request.write.req.valid && shared && writeOwners.io.enq.ready + remote.write.req.bits := request.write.req.bits + request.write.req.ready := writeReady && writeOwners.io.enq.ready + writeOwners.io.enq.valid := request.write.req.valid && writeReady + writeOwners.io.enq.bits := shared + val writeShared = writeOwners.io.deq.bits + val writeValid = Mux(writeShared, remote.write.resp.valid, memory.write.resp.valid) + request.write.resp.valid := writeOwners.io.deq.valid && writeValid + request.write.resp.bits := Mux(writeShared, remote.write.resp.bits, memory.write.resp.bits) + memory.write.resp.ready := writeOwners.io.deq.valid && !writeShared && request.write.resp.ready + remote.write.resp.ready := writeOwners.io.deq.valid && writeShared && request.write.resp.ready + writeOwners.io.deq.ready := writeValid && request.write.resp.ready + } else { + memory.read.req.valid := request.read.req.valid && !shared + memory.write.req.valid := request.write.req.valid && !shared + request.read.req.ready := memory.read.req.ready && !shared + request.write.req.ready := memory.write.req.ready && !shared + when(request.read.req.valid || request.write.req.valid)(assert(!shared, "Shared request has no core port")) + } + } + + for (i <- routedRequests.size until io.shared.requests.size) { + val remote = io.shared.requests(i) + remote.bank_id := 0.U + remote.group_id := 0.U + remote.thread_id := 0.U + remote.rob_id := 0.U + remote.inst_id := 0.U + remote.is_shared := true.B + remote.read.req.valid := false.B + remote.read.req.bits := 0.U.asTypeOf(remote.read.req.bits) + remote.read.resp.ready := false.B + remote.write.req.valid := false.B + remote.write.req.bits := 0.U.asTypeOf(remote.write.req.bits) + remote.write.resp.ready := false.B + } +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/scala/Emit.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/scala/Emit.scala new file mode 100644 index 000000000..b46540f0e --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/scala/Emit.scala @@ -0,0 +1,104 @@ +package framework.acc.memdomain.bank_router + +import chisel3._ +import chisel3.reflect.DataMirror +import java.nio.file.{Files, Paths} +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.acc.gpdomain.configs.GpDomainParam + +object Emit extends App { + val Array("--config", config, "--output-dir", directory) = args + val values = ujson.read(Files.readString(Paths.get(config))) + val frontend = upickle.default.read[FrontendParam](values("frontend")) + val memory = upickle.default.read[MemDomainParam](values("memory")) + val gp = upickle.default.read[GpDomainParam](values("gp")) + + val defs = scala.collection.mutable.ArrayBuffer[String]( + s"`define BR_BANKS ${memory.bankNum}", + s"`define BR_ROWS ${memory.bankEntries}", + s"`define BR_CLIENTS ${memory.bankRequestPorts + 2}", + s"`define BR_ALIASES ${2 * memory.virtualBankCount + frontend.rob_entries}", + s"`define BR_DATA_BITS ${memory.bankWidth}", + s"`define BR_MASK_BITS ${memory.bankMaskLen}" + ) + + val ports = scala.collection.mutable.ArrayBuffer[String]() + val offsets = scala.collection.mutable.Map[String, Int]() + var inOffset = 0; var outOffset = 0 + + def leaves(data: Data, path: String): Seq[(String, Data)] = data match { + case record: Record => record.elements.toSeq.flatMap { case (name, field) => + leaves(field, if (path.isEmpty) name else s"${path}_$name") + } + case vector: Vec[_] => vector.toSeq.zipWithIndex.flatMap { case (field, index) => leaves(field, s"${path}_$index") } + case field => if (field.getWidth > 0) Seq(path -> field) else Seq.empty + } + + Files.createDirectories(Paths.get(directory)) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + { + val dut = new BankRouter(frontend, memory, gp, values("diffTest").bool) + for ((name, field) <- leaves(dut.io, "")) { + val input = DataMirror.directionOf(field) == ActualDirection.Input + val direction = if (input) "IN" else "OUT"; val signal = if (input) "inputs" else "outputs"; + val offset = if (input) inOffset else outOffset; val width = field.getWidth + defs += s"`define BR_${direction}_${name.toUpperCase}_OFFSET $offset"; + defs += s"`define BR_${direction}_${name.toUpperCase}_WIDTH $width"; offsets(name) = offset + ports += s".io_$name(bus.$signal[$offset +: $width])" + if (input) inOffset += width else outOffset += width + }; dut + }, + args = Array("--target-dir", s"$directory/BankRouter", "--split-verilog") + ) + defs += s"`define BR_INPUT_BITS $inOffset"; defs += s"`define BR_OUTPUT_BITS $outOffset" + val clientNames = + (0 until memory.bankRequestPorts).map(i => s"requests_$i") ++ (0 until 2).map(i => s"directRequest_$i") + + val clientFields = Seq( + "read_req_valid", + "read_req_ready", + "read_req_bits_addr", + "read_resp_valid", + "read_resp_ready", + "read_resp_bits_data", + "write_req_valid", + "write_req_ready", + "write_req_bits_addr", + "write_req_bits_data", + "write_req_bits_mask_0", + "write_resp_valid", + "write_resp_ready", + "write_resp_bits_ok", + "bank_id", + "group_id", + "thread_id" + ) + + val bankFields = Seq( + "sramRead_req_valid", + "sramRead_req_ready", + "sramRead_req_bits_addr", + "sramRead_resp_valid", + "sramRead_resp_ready", + "sramRead_resp_bits_data", + "sramWrite_req_valid", + "sramWrite_req_ready", + "sramWrite_req_bits_addr", + "sramWrite_req_bits_data", + "sramWrite_req_bits_mask_0", + "sramWrite_resp_valid", + "sramWrite_resp_ready", + "sramWrite_resp_bits_ok", + "clear", + "clearing", + "bind" + ) + + def table(names: Seq[String], fields: Seq[String]) = + names.map(n => fields.map(f => offsets(s"${n}_$f")).mkString("'{", ",", "}")).mkString("'{", ",", "}") + defs += s"`define BR_CLIENT_OFFSETS ${table(clientNames, clientFields)}" + defs += s"`define BR_BANK_OFFSETS ${table((0 until memory.bankNum).map(i => s"physical_$i"), bankFields)}" + Files.writeString(Paths.get(directory, "parameters.svh"), defs.mkString("\n") + "\n") + Files.writeString(Paths.get(directory, "ports.svh"), ports.mkString(",\n") + "\n") +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/scala/blt/AccPipe.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/scala/blt/AccPipe.scala new file mode 100644 index 000000000..9919e91c1 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/scala/blt/AccPipe.scala @@ -0,0 +1,102 @@ +package framework.acc.memdomain.bank_router.blt + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import framework.ip.memory.bank.{SramReadIO, SramReadReq, SramReadResp, SramWriteIO, SramWriteReq, SramWriteResp} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} + +@instantiable +class AccPipe(val frontendParam: FrontendParam, val memParam: MemDomainParam) extends Module { + + @public + val io = IO(new Bundle { + val sramRead = Flipped(new SramReadIO(memParam)) + val sramWrite = Flipped(new SramWriteIO(memParam)) + + val mem_req = new Bundle { + val read = new SramReadIO(memParam) + val write = new SramWriteIO(memParam) + val bank_id = Input(UInt(frontendParam.bank_id_len.W)) + val group_id = Input(UInt(memParam.groupIdWidth.W)) + val thread_id = Input(UInt(64.W)) + } + + val busy = Output(Bool()) + val group_id = Output(UInt(memParam.groupIdWidth.W)) + val bank_id = Output(UInt(frontendParam.bank_id_len.W)) + val thread_id = Output(UInt(64.W)) + }) + + // Each group has its own physical bank, so no address shifting is needed. + + //group_id output + val group_id_reg = RegInit(0.U(memParam.groupIdWidth.W)) + io.group_id := group_id_reg + + //Bank_id output + val bank_id_reg = RegInit(0.U(frontendParam.bank_id_len.W)) + io.bank_id := bank_id_reg + val thread_id_reg = RegInit(0.U(64.W)) + io.thread_id := thread_id_reg + + val rd_hold = RegInit(false.B) + val rd_data_reg = RegInit(0.U(memParam.bankWidth.W)) + val wr_hold = RegInit(false.B) + val wr_ok_reg = RegInit(false.B) + val busy_reg = RegInit(false.B) + + val canStart = !busy_reg && !rd_hold && !wr_hold + val hasWriteReq = io.mem_req.write.req.valid + val wrReq = canStart && hasWriteReq + val rdReq = canStart && !hasWriteReq && io.mem_req.read.req.valid + + io.sramRead.req.valid := rdReq + io.sramRead.req.bits.addr := io.mem_req.read.req.bits.addr + io.sramRead.resp.ready := !rd_hold + + io.sramWrite.req.valid := wrReq + io.sramWrite.req.bits.addr := io.mem_req.write.req.bits.addr + io.sramWrite.req.bits.data := io.mem_req.write.req.bits.data + io.sramWrite.req.bits.mask := io.mem_req.write.req.bits.mask + io.sramWrite.resp.ready := !wr_hold + + io.mem_req.read.req.ready := canStart && !hasWriteReq && io.sramRead.req.ready + io.mem_req.read.resp.valid := rd_hold + io.mem_req.read.resp.bits.data := rd_data_reg + + io.mem_req.write.req.ready := canStart && io.sramWrite.req.ready + io.mem_req.write.resp.valid := wr_hold + io.mem_req.write.resp.bits.ok := wr_ok_reg + + when(rd_hold && io.mem_req.read.resp.ready) { + rd_hold := false.B + }.elsewhen(io.sramRead.resp.fire) { + rd_hold := true.B + rd_data_reg := io.sramRead.resp.bits.data + } + + when(wr_hold && io.mem_req.write.resp.ready) { + wr_hold := false.B + }.elsewhen(io.sramWrite.resp.fire) { + wr_hold := true.B + wr_ok_reg := io.sramWrite.resp.bits.ok + } + + when(io.mem_req.read.req.fire || io.mem_req.write.req.fire) { + group_id_reg := io.mem_req.group_id + bank_id_reg := io.mem_req.bank_id + thread_id_reg := io.mem_req.thread_id + } + when(io.mem_req.read.req.fire || io.mem_req.write.req.fire) { + busy_reg := true.B + } + when(io.mem_req.read.resp.fire || io.mem_req.write.resp.fire) { + busy_reg := false.B + } + + io.busy := busy_reg +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/scala/blt/BLT.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/scala/blt/BLT.scala new file mode 100644 index 000000000..897e86e0f --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/scala/blt/BLT.scala @@ -0,0 +1,634 @@ +package framework.acc.memdomain.bank_router.blt + +import framework.lib.arbiter.{RRArbiter} + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.memdomain.{LayoutConfig, LayoutQuery} +import framework.acc.memdomain.BankPort +import framework.acc.debug.trace.MemoryEvent +import framework.acc.memdomain.bank_router.{BankMapping, BankMappingIO} +import framework.acc.memdomain.prm.PhysicalBankPort +import framework.acc.memdomain.PhysicalBankHash + +class BankRequest(memp: MemDomainParam) extends Bundle { + val write = Bool() + val addr = UInt(16.W) + val data = UInt(memp.bankWidth.W) + val mask = Vec(memp.bankMaskLen, Bool()) +} + +@instantiable +class BankLayoutTable( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val gpParam: GpDomainParam, + val diffTest: Boolean) + extends Module { + + val kernelRequests = + if (gpParam.enable) + gpParam.memory.memoryPorts + else 0 + + val requestCount = memParam.bankRequestPorts + kernelRequests + 2 + + @public + val io = IO(new Bundle { + val traceMemory = + Output(Vec(requestCount + memParam.bankNum, new MemoryEvent)) + val mem_req = Vec(memParam.bankRequestPorts, Flipped(new BankPort(frontendParam, memParam))) + val kernel_req = Vec(kernelRequests + 2, Flipped(new BankPort(frontendParam, memParam))) + val config = Flipped(Decoupled(new LayoutConfig(frontendParam, memParam))) + val mapping = Flipped(new BankMappingIO(frontendParam)) + + // Query interface for frontend to get group count + val query_vbank_id = Input(UInt(frontendParam.bank_id_len.W)) + val query_group_count = Output(UInt(memParam.groupCountWidth.W)) + val layout = + Vec(3 * (if (gpParam.enable) 2 else 1), Flipped(new LayoutQuery(frontendParam, memParam))) + val clearBusy = Output(Bool()) + val physical = Vec(memParam.bankNum, Flipped(new PhysicalBankPort(memParam))) + + val hashes = + if (diffTest) Some(Input(Vec(memParam.bankNum, UInt(32.W)))) + else None + + val bank_hashes = + if (diffTest) + Some(Output(Vec(memParam.bankNum, new PhysicalBankHash(frontendParam)))) + else None + + }) + + val requests = io.mem_req.toSeq ++ io.kernel_req.toSeq + + val banks = io.physical.toSeq + val accPipes: Seq[Instance[AccPipe]] = Seq.fill(requestCount)(Instantiate(new AccPipe(frontendParam, memParam))) + + // Per-channel memory trace DPI-C modules to avoid losing simultaneous events + val mtraces = io.traceMemory.take(requestCount) + for (mt <- mtraces) { + mt.is_write := 0.U + mt.is_shared := 0.U + mt.channel := 0.U + mt.thread_id := 0.U + mt.rob_id := 0.U + mt.inst_id := 0.U + mt.vbank_id := 0.U + mt.pbank_id := 0.U + mt.group_id := 0.U + mt.addr := 0.U + mt.write_mask := 0.U + mt.data_lo := 0.U + mt.data_hi := 0.U + mt.enable := false.B + } + + // ----------------------------------------------------------------------------- + // Mapping table + // ----------------------------------------------------------------------------- + class MappingTableEntry extends Bundle { + val valid = Bool() + val thread_id = UInt(64.W) + val vbank_id = UInt(frontendParam.bank_id_len.W) + val architectural_id = UInt(frontendParam.bank_id_len.W) + val is_multi = Bool() + val group_id = UInt(memParam.groupIdWidth.W) + } + + val mappingTable = RegInit( + VecInit(Seq.fill(memParam.bankNum)(0.U.asTypeOf(new MappingTableEntry))) + ) + + // The frontend contract reserves [0, max_private_vbank] for private + // banks. Keep the direct route table at that architectural size. + val privateVbankCount = + 2 * (memParam.virtualBankCount + (if (gpParam.enable) 2 + else + 0)) + frontendParam.rob_entries + + require( + privateVbankCount > 0 && privateVbankCount <= (1 << frontendParam.bank_id_len), + "Private alias IDs exceed bank_id_len" + ) + val groupIndexWidth = log2Up(memParam.bankNum) + val pbankIndexWidth = log2Up(memParam.bankNum) + val privateVbankIdWidth = log2Up(privateVbankCount) + + // A non-multi vbank has exactly one route, so keep that common case as a + // small indexed table. Multi-bank mappings retain their per-group entries + // in mappingTable and use the fallback scan below. This avoids the much + // larger 32x24 valid+pbank register matrix while still removing the full + // mapping scan from the overwhelmingly common single-bank path. + val singleRouteValid = RegInit(VecInit(Seq.fill(privateVbankCount)(false.B))) + + val singleRoutePbank = RegInit( + VecInit(Seq.fill(privateVbankCount)(0.U(pbankIndexWidth.W))) + ) + + // The private namespace is bounded by the frontend contract, while + // multi-bank groups remain represented individually in mappingTable. + val groupCountByVbank = RegInit( + VecInit(Seq.fill(privateVbankCount)(0.U(memParam.groupCountWidth.W))) + ) + + val owners = RegInit(VecInit((0 until privateVbankCount).map(i => + (i % (memParam.virtualBankCount + (if (gpParam.enable) 2 + else 0))).U( + frontendParam.bank_id_len.W + ) + ))) + + val committed = RegInit(VecInit((0 until memParam.virtualBankCount).map(_.U( + frontendParam.bank_id_len.W + )))) + + // Record binding and lifetime changes, including aliases with no active accesses. + for ((entry, physical) <- mappingTable.zipWithIndex) { + val live = entry.valid && true.B + val current = entry.valid && committed(entry.architectural_id) === entry.vbank_id + val previous = RegNext(entry, 0.U.asTypeOf(entry)) + val previousLive = RegNext(live, false.B) + val previousCurrent = RegNext(current, false.B) + val trace = io.traceMemory(requestCount + physical) + trace.is_write := Mux(entry.valid, 2.U, 3.U) + trace.is_shared := 0.U + trace.channel := 0.U + trace.thread_id := entry.thread_id + trace.rob_id := 0.U + trace.inst_id := 0.U + trace.vbank_id := entry.vbank_id + trace.pbank_id := physical.U + trace.group_id := entry.group_id + trace.addr := entry.architectural_id + trace.write_mask := 0.U + trace.data_lo := live.asUInt + trace.data_hi := current.asUInt + trace.enable := (entry.valid || previous.valid) && + (entry.asUInt =/= previous.asUInt || live =/= previousLive || current =/= previousCurrent) + } + + val architecturalColumns = RegInit(VecInit(Seq.fill( + memParam.virtualBankCount + )(0.U(memParam.groupCountWidth.W)))) + + def addEntry( + thread_id: UInt, + vbank_id: UInt, + pbank_id: UInt, + is_multi: Bool, + group_id: UInt + ): Unit = { + val entry = mappingTable(pbank_id) + entry.valid := true.B + entry.thread_id := thread_id + entry.vbank_id := vbank_id + entry.architectural_id := vbank_id + entry.is_multi := is_multi + entry.group_id := group_id + } + + def deleteEntry(vbank_id: UInt): Unit = { + for (i <- 0 until memParam.bankNum) { + when(mappingTable(i).valid && mappingTable(i).architectural_id === vbank_id) { + mappingTable(i).valid := false.B + } + } + } + + // ----------------------------------------------------------------------------- + // Default Value + // ----------------------------------------------------------------------------- + + for (i <- 0 until requestCount) { + accPipes(i).io.mem_req.write <> requests(i).write + accPipes(i).io.mem_req.read <> requests(i).read + accPipes(i).io.mem_req.bank_id := requests(i).bank_id + accPipes(i).io.mem_req.group_id := requests(i).group_id + accPipes(i).io.mem_req.thread_id := requests(i).thread_id + + // Bank-side defaults (only driven when a bank is actually connected) + accPipes(i).io.sramRead.req.ready := false.B + accPipes(i).io.sramRead.resp.valid := false.B + accPipes(i).io.sramRead.resp.bits := DontCare + + accPipes(i).io.sramWrite.req.ready := false.B + accPipes(i).io.sramWrite.resp.valid := false.B + accPipes(i).io.sramWrite.resp.bits := DontCare + } + + banks.zipWithIndex.foreach { + case (bank, _) => + bank.sramRead.req.valid := false.B + bank.sramRead.req.bits := DontCare + bank.sramRead.resp.ready := true.B + + bank.sramWrite.req.valid := false.B + bank.sramWrite.req.bits := DontCare + bank.sramWrite.resp.ready := true.B + bank.clear := false.B + bank.bind := false.B + } + io.clearBusy := VecInit(banks.map(_.clearing)).asUInt.orR + + // High while a mapping is being installed (see Bank Alloc/Release). + val mappingBusy = RegInit(false.B) + + val realloc = io.config.bits.group_id === 0.U + + val freePbankMask = + VecInit(mappingTable.map(entry => + !entry.valid || + (realloc && entry.architectural_id === io.config.bits.vbank_id) + )) + + val hasFreePbank = freePbankMask.asUInt.orR + + val configPbank = + if (memParam.bankLayoutEnable) PriorityEncoder(freePbankMask) + else (io.config.bits.vbank_id +& io.config.bits.group_id)(pbankIndexWidth - 1, 0) + + val configBankAvailable = + if (memParam.bankLayoutEnable) hasFreePbank + else freePbankMask(configPbank) + + if (!memParam.bankLayoutEnable) { + when(io.config.valid) { + assert(!io.config.bits.transfer, "Fixed bank layout requires compiler-lowered transfers") + when(io.config.bits.alloc) { + assert( + io.config.bits.vbank_id +& io.config.bits.group_id < memParam.bankNum.U, + "Fixed bank layout allocation exceeds physical bank count" + ) + } + } + when(io.mapping.valid && io.mapping.bits.allocate) { + assert( + io.mapping.bits.bank +& architecturalColumns( + io.mapping.bits.bank + ) <= memParam.bankNum.U, + "Fixed bank alias exceeds physical bank count" + ) + } + } + io.config.ready := !mappingBusy && (!io.config.bits.alloc || configBankAvailable) + when(io.config.valid && !mappingBusy && io.config.bits.alloc && !configBankAvailable) { + assert(false.B, "BLT allocation failed: no free physical bank\n") + } + + // ----------------------------------------------------------------------------- + // Bank Alloc/Release + // ----------------------------------------------------------------------------- + + // A mapping is accepted into registers and installed over the following + // cycles, one group per cycle. busy stays high until it is installed; the + // ROB holds the issuing instruction until then, so no consumer ever sees a + // partially installed mapping. + val change = Reg(new BankMapping(frontendParam)) + val nextGroup = RegInit(0.U(log2Up(memParam.bankNum + 1).W)) + io.mapping.busy := mappingBusy + + // An allocation is accepted only when enough banks exist: the unbound ones + // plus, for an in-place rename, the groups of the alias it replaces. + val incoming = io.mapping.bits + val reusable = Mux(incoming.reuse, groupCountByVbank(incoming.previous(privateVbankIdWidth - 1, 0)), 0.U) + val enough = PopCount(mappingTable.map(!_.valid)) +& reusable >= architecturalColumns(incoming.bank) + io.mapping.ready := !mappingBusy && !io.config.valid && (!incoming.allocate || + (if (memParam.bankLayoutEnable) enough else true.B)) + when(io.mapping.fire) { + change := incoming + nextGroup := 0.U + mappingBusy := true.B + } + + val columns = architecturalColumns(change.bank) + val freeBanks = + VecInit(mappingTable.map(entry => !entry.valid || (change.reuse && entry.vbank_id === change.previous))).asUInt + val first = mappingBusy && nextGroup === 0.U + when(mappingBusy) { + nextGroup := nextGroup + 1.U + when(!change.allocate || nextGroup + 1.U >= columns)(mappingBusy := false.B) + } + when(first && change.allocate) { + assert( + columns =/= 0.U && columns <= memParam.bankNum.U, + "Renamed bank requires an allocated layout" + ) + owners(change.vbank) := change.bank + } + when(first && change.retire) { + committed(change.bank) := change.vbank + singleRouteValid(change.previous) := false.B + groupCountByVbank(change.previous) := 0.U + for (entry <- mappingTable) { + when(entry.valid && entry.vbank_id === change.previous)(entry.valid := false.B) + } + } + for (bank <- 0 until memParam.bankNum) { + when(first && change.write && !change.allocate && + mappingTable(bank).valid && mappingTable(bank).vbank_id === change.vbank) { + io.physical(bank).bind := true.B + } + } + + when(mappingBusy && change.allocate) { + val physical = + if (memParam.bankLayoutEnable) PriorityEncoder(freeBanks) + else (change.bank + nextGroup)(pbankIndexWidth - 1, 0) + if (memParam.bankLayoutEnable) { + assert(freeBanks.orR, "BLT mapping found no free physical bank") + addEntry(change.threadId, change.vbank, physical, columns > 1.U, nextGroup) + mappingTable(physical).architectural_id := change.bank + } else { + assert( + mappingTable(physical).valid && mappingTable(physical).architectural_id === change.bank, + "Fixed bank alias requires its allocated physical slot" + ) + mappingTable(physical).vbank_id := change.vbank + } + for (bank <- 0 until memParam.bankNum) { + when(physical === bank.U)(io.physical(bank).bind := true.B) + } + when(nextGroup === 0.U) { + singleRoutePbank(change.vbank) := physical + singleRouteValid(change.vbank) := columns === 1.U + groupCountByVbank(change.vbank) := columns + } + } + + when(io.config.fire) { + val vbank = io.config.bits.vbank_id + val vbankIdx = vbank(privateVbankIdWidth - 1, 0) + when(io.config.bits.transfer) { + val source = io.config.bits.source_bank_id + val sourceIdx = source(privateVbankIdWidth - 1, 0) + val sourceCount = architecturalColumns(source) + val targetCount = architecturalColumns(vbank) + val sourceRoutes = VecInit(mappingTable.map(entry => entry.valid && entry.architectural_id === source)) + val totalCount = targetCount +& sourceCount + assert(source =/= vbank && sourceCount =/= 0.U, "Private bank transfer requires a distinct allocated source") + assert( + totalCount <= memParam.bankNum.U, + "Private bank transfer exceeds physical bank count" + ) + for (entry <- mappingTable) { + when(entry.valid && entry.architectural_id === source) { + entry.vbank_id := vbank + entry.architectural_id := vbank + entry.group_id := targetCount + entry.group_id + entry.is_multi := totalCount > 1.U + } + when(entry.valid && entry.architectural_id === vbank) { + entry.vbank_id := vbank + entry.is_multi := totalCount > 1.U + } + } + singleRouteValid(sourceIdx) := false.B + singleRouteValid(vbankIdx) := totalCount === 1.U + when(totalCount === 1.U)(singleRoutePbank(vbankIdx) := PriorityEncoder(sourceRoutes.asUInt)) + groupCountByVbank(sourceIdx) := 0.U + groupCountByVbank(vbankIdx) := totalCount + architecturalColumns(source) := 0.U + architecturalColumns(vbank) := totalCount + }.elsewhen(io.config.bits.alloc) { + val freePbank = configPbank + + for (i <- 0 until memParam.bankNum) { + io.physical(i).bind := freePbank === i.U + } + + // The bank hash already treats a newly bound bank as zero, so the clear writes need no hash update. + for (i <- 0 until memParam.bankNum) { + when(io.config.bits.clear && freePbank === i.U)(banks(i).clear := true.B) + } + // Match bemu mset: realloc of the same vbank frees prior physical banks first. + // Configer emits one fire per group; only group 0 drops the old mapping. + when(io.config.bits.group_id === 0.U) { + deleteEntry(io.config.bits.vbank_id) + singleRouteValid(vbankIdx) := false.B + } + addEntry( + io.config.bits.thread_id, + io.config.bits.vbank_id, + freePbank, + io.config.bits.is_multi, + io.config.bits.group_id + ) + when(!io.config.bits.is_multi) { + singleRoutePbank(vbankIdx) := freePbank + singleRouteValid(vbankIdx) := true.B + } + groupCountByVbank(vbankIdx) := Mux(io.config.bits.is_multi, io.config.bits.group_id +& 1.U, 1.U) + architecturalColumns(vbank) := Mux(io.config.bits.is_multi, io.config.bits.group_id +& 1.U, 1.U) + }.otherwise { + deleteEntry(io.config.bits.vbank_id) + singleRouteValid(vbankIdx) := false.B + groupCountByVbank(vbankIdx) := 0.U + architecturalColumns(vbank) := 0.U + } + } + + // ----------------------------------------------------------------------------- + // Query interface: return group count for a given vbank_id + // ----------------------------------------------------------------------------- + val queryVbankId = RegNext(io.query_vbank_id, 0.U) + val queryVbank = queryVbankId(privateVbankIdWidth - 1, 0) + io.query_group_count := RegNext(groupCountByVbank(queryVbank), 0.U) + for (port <- io.layout) { + val bank = RegNext(port.bank(privateVbankIdWidth - 1, 0), 0.U) + port.groups := RegNext(groupCountByVbank(bank), 0.U) + } + + // ----------------------------------------------------------------------------- + // Connect AccPipe and Banks + // ----------------------------------------------------------------------------- + def emitTrace( + ch: Int, + isWrite: UInt, + pbankId: UInt, + addr: UInt, + writeMask: UInt, + dataLo: UInt, + dataHi: UInt, + en: Bool + ): Unit = { + mtraces(ch).is_write := isWrite + mtraces(ch).is_shared := requests(ch).is_shared.asUInt + mtraces(ch).channel := ch.U + mtraces(ch).thread_id := requests(ch).thread_id + mtraces(ch).rob_id := requests(ch).rob_id + mtraces(ch).inst_id := requests(ch).inst_id + mtraces(ch).vbank_id := requests(ch).bank_id + mtraces(ch).pbank_id := pbankId + mtraces(ch).group_id := requests(ch).group_id + mtraces(ch).addr := addr + mtraces(ch).write_mask := writeMask + mtraces(ch).data_lo := dataLo + mtraces(ch).data_hi := dataHi + mtraces(ch).enable := en + } + + // The route is held for the complete AccPipe transaction. Keep the route + // vectors in one place so the bank-side command mux and response demux share + // exactly the same ownership predicate. + val activeRouteOHs = Seq.fill(requestCount)(Wire(UInt(memParam.bankNum.W))) + + for (i <- 0 until requestCount) { + val routePbankReg = RegInit(0.U(pbankIndexWidth.W)) + val routeValidReg = RegInit(false.B) + val routePending = RegInit(false.B) + val requestVbank = requests(i).bank_id + val requestVbankIdx = requestVbank(privateVbankIdWidth - 1, 0) + val requestGroup = requests(i).group_id(groupIndexWidth - 1, 0) + val singleRoute = singleRoutePbank(requestVbankIdx) + val singleValid = singleRouteValid(requestVbankIdx) + val multiRouteMatch = VecInit(mappingTable.map(entry => + entry.valid && entry.is_multi && + entry.vbank_id === requestVbank && entry.group_id === requestGroup + )) + val multiRoute = PriorityEncoder(multiRouteMatch) + val multiValid = multiRouteMatch.asUInt.orR + val requestRoute = + if (memParam.bankLayoutEnable) Mux(singleValid, singleRoute, multiRoute) + else (owners(requestVbankIdx) + requestGroup)(pbankIndexWidth - 1, 0) + val requestRouteValid = + if (memParam.bankLayoutEnable) singleValid || multiValid + else mappingTable(requestRoute).valid && + mappingTable(requestRoute).architectural_id === owners(requestVbankIdx) && + mappingTable(requestRoute).group_id === requestGroup + // Route lookup is deliberately one cycle ahead of the SRAM request. The + // route is captured while the AccPipe is idle, then held through the + // entire request/response lifetime. This removes the live mapping-table + // lookup from every wide bank-side mux select while preserving one request + // per channel per cycle once the pipe is running. + val requestSeen = requests(i).read.req.valid || requests(i).write.req.valid + val activeRoute = routePbankReg + val requestActive = routePending || accPipes(i).io.busy + val activeRouteValid = routeValidReg && requestActive + val activeRouteOH = UIntToOH(activeRoute, memParam.bankNum) & + Fill(memParam.bankNum, activeRouteValid) + + activeRouteOHs(i) := activeRouteOH + + when(!routePending && !accPipes(i).io.busy && requestSeen && requestRouteValid) { + routePbankReg := requestRoute + routeValidReg := true.B + routePending := true.B + } + when(requests(i).read.req.fire || requests(i).write.req.fire) { + routePending := false.B + } + + // Requests are attributed once the selected SRAM accepts them. + when(accPipes(i).io.sramRead.req.fire) { + emitTrace(i, 0.U, activeRoute, accPipes(i).io.sramRead.req.bits.addr, 0.U, 0.U, 0.U, true.B) + } + + // Arrival trace: the write has crossed arbitration and is accepted by the + // selected SPM SRAM port. Keep attribution metadata from the same AccPipe + // transaction and data from the bank-side request. + when(accPipes(i).io.sramWrite.req.fire) { + emitTrace( + i, + 1.U, + activeRoute, + accPipes(i).io.sramWrite.req.bits.addr, + accPipes(i).io.sramWrite.req.bits.mask.asUInt, + accPipes(i).io.sramWrite.req.bits.data(63, 0), + accPipes(i).io.sramWrite.req.bits.data(127, 64), + true.B + ) + } + + } + + // Fixed GP request slots share the physical-bank arbitration with ordinary + // channels. Capture the accepted request's owner for the one-cycle SRAM + // response; a stalled requester must never receive another owner's response. + val responseRoutes = + Seq.fill(requestCount)(WireDefault(0.U(memParam.bankNum.W))) + val responseRouteBits = + Seq.fill(requestCount)(Wire(Vec(memParam.bankNum, Bool()))) + + for (j <- 0 until memParam.bankNum) { + val arb = Instantiate(new RRArbiter( + new BankRequest(memParam), + requestCount + )) + + for (i <- 0 until requestCount) { + val pipe = accPipes(i) + arb.io.in(i).valid := activeRouteOHs(i)(j) && + (pipe.io.sramRead.req.valid || pipe.io.sramWrite.req.valid) + arb.io.in(i).bits.write := pipe.io.sramWrite.req.valid + arb.io.in(i).bits.addr := Mux( + pipe.io.sramWrite.req.valid, + pipe.io.sramWrite.req.bits.addr, + pipe.io.sramRead.req.bits.addr + ) + arb.io.in(i).bits.data := pipe.io.sramWrite.req.bits.data + arb.io.in(i).bits.mask := pipe.io.sramWrite.req.bits.mask + when(activeRouteOHs(i)(j)) { + pipe.io.sramRead.req.ready := arb.io.in(i).ready && !arb.io.in(i).bits.write + pipe.io.sramWrite.req.ready := arb.io.in(i).ready && arb.io.in(i).bits.write + } + responseRouteBits(i)(j) := RegNext(arb.io.in(i).fire, false.B) + } + + val bank = banks(j) + bank.sramRead.req.valid := arb.io.out.valid && !arb.io.out.bits.write + bank.sramRead.req.bits.addr := arb.io.out.bits.addr + bank.sramWrite.req.valid := arb.io.out.valid && arb.io.out.bits.write + bank.sramWrite.req.bits.addr := arb.io.out.bits.addr + bank.sramWrite.req.bits.data := arb.io.out.bits.data + bank.sramWrite.req.bits.mask := arb.io.out.bits.mask + arb.io.out.ready := Mux(arb.io.out.bits.write, bank.sramWrite.req.ready, bank.sramRead.req.ready) + + bank.sramRead.resp.ready := VecInit((0 until requestCount).map { i => + responseRouteBits(i)(j) && accPipes(i).io.sramRead.resp.ready + }).asUInt.orR + bank.sramWrite.resp.ready := VecInit((0 until requestCount).map { i => + responseRouteBits(i)(j) && accPipes(i).io.sramWrite.resp.ready + }).asUInt.orR + when(bank.sramRead.resp.valid)(assert(bank.sramRead.resp.ready)) + when(bank.sramWrite.resp.valid)(assert(bank.sramWrite.resp.ready)) + } + + for (i <- 0 until requestCount) { + responseRoutes(i) := responseRouteBits(i).asUInt + val readRespSel = VecInit((0 until memParam.bankNum).map { j => + responseRoutes(i)(j) && banks(j).sramRead.resp.valid + }).asUInt + val writeRespSel = VecInit((0 until memParam.bankNum).map { j => + responseRoutes(i)(j) && banks(j).sramWrite.resp.valid + }).asUInt + accPipes(i).io.sramRead.resp.valid := readRespSel.orR + accPipes(i).io.sramRead.resp.bits.data := (0 until memParam.bankNum).map { + j => + Fill(memParam.bankWidth, responseRoutes(i)(j)) & banks( + j + ).sramRead.resp.bits.data + }.reduce(_ | _) + accPipes(i).io.sramWrite.resp.valid := writeRespSel.orR + accPipes(i).io.sramWrite.resp.bits.ok := writeRespSel.orR + } + + io.bank_hashes.foreach { states => + for (i <- 0 until memParam.bankNum) { + states(i).valid := mappingTable(i).valid + states(i).threadId := mappingTable(i).thread_id + states(i).vbankId := (if (memParam.bankLayoutEnable) + mappingTable(i).vbank_id + else mappingTable(i).architectural_id) + states(i).pbankId := i.U + states(i).groupId := mappingTable(i).group_id + states(i).statusHash := io.hashes.get(i) + } + } +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/configs/MemDomainParam.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/configs/MemDomainParam.scala new file mode 100644 index 000000000..993f5883c --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/configs/MemDomainParam.scala @@ -0,0 +1,91 @@ +package framework.acc.memdomain.configs + +import buckyball.config.{MemDomainConfig, SharedMemConfig} + +import upickle.default._ + +/** + * MemDomain Parameter + */ +case class MemDomainParam( + sharedPorts: Int, + bankNum: Int, + bankLayoutEnable: Boolean, + bankWidth: Int, + bankEntries: Int, + bankMaskLen: Int, + virtualBankCount: Int, + sharedEnable: Boolean, + sharedEntries: Int, + sharedBankEntries: Int, + sharedBankNum: Int, + sharedInputChannels: Int, + sharedDefaultGroupCount: Int, + bankRequestPorts: Int, + dma_buswidth: Int, + memAddrLen: Int, + mmioEnable: Boolean, + mmioBankNum: Int, + mmioBankEntries: Int, + mmioBankWidth: Int, + mmioReadWidth: Int) { + + if (sharedEnable) { + require(sharedBankEntries >= 2 && sharedBankEntries <= 65536) + require(sharedEntries == sharedBankNum * sharedBankEntries, "Shared pool size must equal banks times rows per bank") + } + + // MMIO derived values + val mmioBankBytes: Int = mmioBankEntries * (mmioBankWidth / 8) + val mmioTotalBytes: Int = mmioBankNum * mmioBankBytes + val vbankIdWidth: Int = chisel3.util.log2Up(virtualBankCount) + val maxGroupCount: Int = math.max(bankNum, if (sharedEnable) sharedBankNum else bankNum) + val groupCountWidth: Int = chisel3.util.log2Up(maxGroupCount + 1) + val groupIdWidth: Int = chisel3.util.log2Up(maxGroupCount) +} + +object MemDomainParam { + + def fromProto( + p: MemDomainConfig, + shared: SharedMemConfig, + virtualBanks: Int, + participants: Int + ): MemDomainParam = { + require(p.hasBank && p.hasDma && p.hasMmio, "Memory domain configuration is incomplete") + if (shared.getEnable) require(participants > 0 && shared.getInputChannels % participants == 0) + val bank = p.getBank + require(bank.hasLayoutEnable, "Bank layout policy is required") + if (shared.getEnable) { + require(shared.getBankEntries > 0 && shared.getEntries % shared.getBankEntries == 0) + require(shared.getBankWidth == bank.getWidth, "Private and shared bank interfaces must agree") + } + val mmio = p.getMmio + MemDomainParam( + if (shared.getEnable) shared.getInputChannels / participants else 0, + bank.getNum, + bank.getLayoutEnable, + bank.getWidth, + bank.getEntries, + bank.getMaskLen, + virtualBanks, + shared.getEnable, + shared.getEntries, + shared.getBankEntries, + if (shared.getEnable) shared.getEntries / shared.getBankEntries else 0, + shared.getInputChannels, + shared.getDefaultGroupCount, + bank.getRequestPorts, + p.getDma.getBusWidth, + p.getAddrLen, + mmio.getEnable, + mmio.getBankNum, + mmio.getBankEntries, + mmio.getBankWidth, + mmio.getReadWidth + ) + } + + implicit val rw: ReadWriter[MemDomainParam] = macroRW + +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/ctrl/Ctrl.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/ctrl/Ctrl.scala new file mode 100644 index 000000000..7b6e9775f --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/ctrl/Ctrl.scala @@ -0,0 +1,102 @@ +package framework.acc.memdomain + +import framework.lib.arbiter.{Arbiter} + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.core.memory.access.preflight.configs.PmpParams + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.frontend.scheduler.MemoryIssue +import framework.acc.memdomain.toddr.DmaStatus + +class Request(val frontendParam: FrontendParam, val memParam: MemDomainParam, val permissionParam: PmpParams) + extends Bundle { + val context = new MemoryContext(permissionParam) + val instId = UInt(64.W) + val cmd = new DecodedCommand(frontendParam, memParam) + val rob_id = UInt(log2Up(frontendParam.rob_entries).W) +} + +class Response(val frontendParam: FrontendParam) extends Bundle { + val fault = new DmaStatus + val rob_id = UInt(log2Up(frontendParam.rob_entries).W) +} + +@instantiable +class Ctrl(val frontendParam: FrontendParam, val memParam: MemDomainParam, val permissionParam: PmpParams) + extends Module { + val commandDataBits = frontendParam.commandDataBits + val ownerId = frontendParam.ownerId + + @public + val io = IO(new Bundle { + val issue = Flipped(Decoupled(new MemoryIssue(frontendParam, memParam, permissionParam))) + val complete = Decoupled(new Response(frontendParam)) + val request = Decoupled(new Request(frontendParam, memParam, permissionParam)) + val response = Flipped(Decoupled(new Response(frontendParam))) + val config = Decoupled(new LayoutConfig(frontendParam, memParam)) + val clearBusy = Input(Bool()) + val mvover = new MvoverPort + }) + + val decoder: Instance[Decoder] = Instantiate(new Decoder(frontendParam, memParam)) + val configer: Instance[Configer] = Instantiate(new Configer(frontendParam, memParam, permissionParam)) + + val operation = io.issue.bits.operation + val isMvover = operation.cmd.cmd.raw_inst(31, 25) === MvoverISA.Funct.U + val mvoverPending = RegInit(false.B) + val mvoverRobId = Reg(chiselTypeOf(operation.rob_id)) + + io.mvover.command.valid := io.issue.valid && isMvover && !mvoverPending + io.mvover.command.bits.sourceThread := operation.cmd.cmd.rs1Data(7, 0) + io.mvover.command.bits.targetThread := operation.cmd.cmd.rs1Data(15, 8) + io.mvover.command.bits.sourceBank := operation.cmd.cmd.rs1Data(25, 16) + io.mvover.command.bits.targetBank := operation.cmd.cmd.rs1Data(35, 26) + io.mvover.command.bits.sourceAddr := operation.cmd.cmd.rs2Data(15, 0) + io.mvover.command.bits.targetAddr := operation.cmd.cmd.rs2Data(31, 16) + io.mvover.command.bits.rows := operation.cmd.cmd.rs2Data(47, 32) +& 1.U + when(io.mvover.command.fire) { + mvoverPending := true.B + mvoverRobId := operation.rob_id + } + + decoder.io.request.valid := io.issue.valid && !isMvover && !mvoverPending + decoder.io.request.bits := operation.cmd + io.issue.ready := !mvoverPending && Mux(isMvover, io.mvover.command.ready, decoder.io.request.ready) + + val decoded = decoder.io.command + val control = + decoded.bits.is_config && decoded.bits.bank_id < memParam.virtualBankCount.U + configer.io.request.valid := decoded.valid && control + configer.io.request.bits.cmd := decoded.bits + configer.io.request.bits.rob_id := operation.rob_id + configer.io.request.bits.context := io.issue.bits.context + configer.io.request.bits.instId := operation.instId + io.request.valid := decoded.valid && !control + io.request.bits.cmd := decoded.bits + io.request.bits.rob_id := operation.rob_id + io.request.bits.context := io.issue.bits.context + io.request.bits.instId := operation.instId + decoded.ready := Mux(control, configer.io.request.ready, io.request.ready) + + configer.io.clearBusy := io.clearBusy + io.config <> configer.io.config + + val completion = Instantiate(new Arbiter(new Response(frontendParam), 2)) + completion.io.in(0) <> io.response + completion.io.in(1) <> configer.io.response + + val moved = mvoverPending && io.mvover.completion.valid + io.complete.valid := moved || completion.io.out.valid + io.complete.bits.fault := Mux(moved, 0.U.asTypeOf(new DmaStatus), completion.io.out.bits.fault) + io.complete.bits.rob_id := Mux(moved, mvoverRobId, completion.io.out.bits.rob_id) + completion.io.out.ready := io.complete.ready && !moved + io.mvover.completion.ready := io.complete.ready && mvoverPending + when(io.mvover.completion.fire) { + assert(!io.mvover.completion.bits, "Bank movement violated its admitted layout") + mvoverPending := false.B + } +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/ctrl/configer/Configer.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/ctrl/configer/Configer.scala new file mode 100644 index 000000000..29db4d710 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/ctrl/configer/Configer.scala @@ -0,0 +1,138 @@ +package framework.acc.memdomain + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.core.memory.access.preflight.configs.PmpParams + +import chisel3._ +import chisel3.util._ + +class LayoutConfig(val frontendParam: FrontendParam, val memParam: MemDomainParam) extends Bundle { + val vbank_id = Output(UInt(frontendParam.bank_id_len.W)) + val is_shared = Output(Bool()) + val is_multi = Output(Bool()) + val alloc = Output(Bool()) + val transfer = Output(Bool()) + val source_bank_id = Output(UInt(frontendParam.bank_id_len.W)) + val clear = Output(Bool()) + val group_id = Output(UInt(memParam.groupIdWidth.W)) + val thread_id = Output(UInt(64.W)) +} + +import framework.acc.memdomain.{Request, Response} +import chisel3.experimental.hierarchy.{instantiable, public} + +@instantiable +class Configer(val frontendParam: FrontendParam, val memParam: MemDomainParam, val permissionParam: PmpParams) + extends Module { + val ownerId = frontendParam.ownerId + + val rob_id_width = log2Up(frontendParam.rob_entries) + + @public + val io = IO(new Bundle { + val request = Flipped(Decoupled(new Request(frontendParam, memParam, permissionParam))) + val response = Decoupled(new Response(frontendParam)) + val config = Decoupled(new LayoutConfig(frontendParam, memParam)) + // A physical bank is still clearing; MSET.clear completes only after it drains. + val clearBusy = Input(Bool()) + }) + + val idle :: config :: clearWait :: resp :: Nil = Enum(4) + + val state = RegInit(idle) + val transfer_reg = RegInit(false.B) + val source_bank_id_reg = RegInit(0.U(frontendParam.bank_id_len.W)) + val alloc_reg = RegInit(false.B) + val is_shared_reg = RegInit(false.B) + val col_reg = RegInit(0.U(memParam.groupCountWidth.W)) + val clear_reg = RegInit(false.B) + val vbank_id_reg = RegInit(0.U(frontendParam.bank_id_len.W)) + val rob_id_reg = RegInit(0.U(rob_id_width.W)) + val counter = RegInit(0.U(memParam.groupCountWidth.W)) + + io.config.bits.is_multi := false.B + io.config.bits.is_shared := false.B + io.config.bits.transfer := false.B + io.config.bits.source_bank_id := 0.U + io.config.bits.alloc := false.B + io.config.bits.clear := false.B + io.config.bits.vbank_id := 0.U(frontendParam.bank_id_len.W) + io.config.bits.group_id := 0.U + io.config.bits.thread_id := ownerId.U(64.W) + io.config.valid := false.B + io.response.valid := false.B + io.response.bits := 0.U.asTypeOf(io.response.bits) + io.response.bits.fault := 0.U.asTypeOf(io.response.bits.fault) + io.response.bits.rob_id := 0.U(rob_id_width.W) + + io.request.ready := state === idle + + when(state === idle) { + when(io.request.valid) { + when(io.request.fire) { + val rawCol = io.request.bits.cmd.special(9, 5) + val alloc = io.request.bits.cmd.special(10) + val fullCol = + if (memParam.sharedEnable) { + Mux( + io.request.bits.cmd.is_shared, + memParam.sharedBankNum.U(col_reg.getWidth.W), + memParam.bankNum.U(col_reg.getWidth.W) + ) + } else { + memParam.bankNum.U(col_reg.getWidth.W) + } + + state := config + col_reg := Mux( + io.request.bits.cmd.transfer, + 1.U, + Mux(alloc && rawCol === 0.U, fullCol, Mux(rawCol > 1.U, rawCol, 1.U)) + ) + transfer_reg := io.request.bits.cmd.transfer + source_bank_id_reg := io.request.bits.cmd.source_bank_id + alloc_reg := alloc + clear_reg := io.request.bits.cmd.clear + is_shared_reg := io.request.bits.cmd.is_shared + vbank_id_reg := io.request.bits.cmd.bank_id + rob_id_reg := io.request.bits.rob_id + assert( + !(io.request.bits.cmd.clear && io.request.bits.cmd.is_shared), + "MSET clear is currently supported for private banks only" + ) + } + } + + }.elsewhen(state === config) { + io.config.bits.is_multi := col_reg > 1.U + io.config.bits.is_shared := is_shared_reg + io.config.bits.transfer := transfer_reg + io.config.bits.source_bank_id := source_bank_id_reg + io.config.bits.alloc := alloc_reg + io.config.bits.clear := clear_reg && alloc_reg + io.config.bits.vbank_id := vbank_id_reg + io.config.bits.group_id := counter(memParam.groupIdWidth - 1, 0) + io.config.valid := true.B + + when(io.config.fire) { + when(counter === col_reg - 1.U) { + counter := 0.U + state := Mux(clear_reg && alloc_reg, clearWait, resp) + }.otherwise { + counter := counter + 1.U + } + } + }.elsewhen(state === clearWait) { + // Each bound bank started clearing when its allocation fired. + when(!io.clearBusy)(state := resp) + }.elsewhen(state === resp) { + io.response.valid := true.B + io.response.bits.rob_id := rob_id_reg + + when(io.response.fire) { + state := idle + counter := 0.U + } + } +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/ctrl/decoder/DISA.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/ctrl/decoder/DISA.scala new file mode 100644 index 000000000..a7c19ce26 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/ctrl/decoder/DISA.scala @@ -0,0 +1,17 @@ +package framework.acc.memdomain + +import chisel3._ +import chisel3.util._ + +object DISA { + val MVOVER_BITPAT = BitPat("b0001101") + val MVIN_MMIO_BITPAT = BitPat("b0100011") + val MSET_FUNCT = 0x20 + // enable=010 (1 write), opcode group + val MSET_BITPAT = BitPat("b0100000") // 32 (0x20) — enable=010, opcode=0 + val MVIN_BITPAT = BitPat("b0100001") // 33 (0x21) — enable=010, opcode=1 + val MVIN_2D_BITPAT = BitPat("b0100010") // 34 (0x22) — 2D DMA into bank rows + val MVIN_KERNEL_BITPAT = BitPat("b0101100") + // enable=001 (1 read), opcode group + val MVOUT_BITPAT = BitPat("b0010000") // 16 (0x10) — enable=001, opcode=0 +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/ctrl/decoder/Decoder.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/ctrl/decoder/Decoder.scala new file mode 100644 index 000000000..f0dce0afe --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/ctrl/decoder/Decoder.scala @@ -0,0 +1,172 @@ +package framework.acc.memdomain + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import framework.acc.frontend.decoder.{DomainId, PostGDCmd} + +class DecodedCommand(frontendParam: FrontendParam, memParam: MemDomainParam) extends Bundle { + val is_shared = Bool() + val is_load = Bool() + val is_store = Bool() + val is_config = Bool() + val clear = Bool() + val transfer = Bool() + val source_bank_id = UInt(frontendParam.bank_id_len.W) + val is_mvin_2d = Bool() + val is_image = Bool() + val is_mmio = Bool() + val mem_addr = UInt(memParam.memAddrLen.W) + val iter = UInt(frontendParam.iter_len.W) + val bank_id = UInt(frontendParam.bank_id_len.W) + val special = UInt(64.W) +} + +import framework.acc.memdomain.DISA._ +import chisel3.experimental.hierarchy.{instantiable, public} + +// LS decode fields (iter removed from decode table — always from rs1[63:48]) +object LSDecodeFields extends Enumeration { + type Field = Value + val LD_EN, ST_EN, MEMADDR, BANK_ID, SPECIAL, VALID = Value +} + +// Default constants for Mem decoder +object DecodeConstants { + val Y = true.B + val N = false.B + val DADDR = 0.U(15.W) + val DSPECIAL = 0.U(64.W) +} + +@instantiable +class Decoder(val frontendParam: FrontendParam, val memParam: MemDomainParam) extends Module { + val commandDataBits = frontendParam.commandDataBits + import DecodeConstants._ + + val sharedVbankBase = frontendParam.shared_vbank_base + require(sharedVbankBase > 0, s"sharedVbankBase($sharedVbankBase) must be > 0") + require( + sharedVbankBase < (1 << frontendParam.bank_id_len), + s"sharedVbankBase($sharedVbankBase) must fit in bankIdLen(${frontendParam.bank_id_len})" + ) + require( + sharedVbankBase > frontendParam.max_private_vbank, + s"sharedVbankBase($sharedVbankBase) must be > maxPrivateVbank(${frontendParam.max_private_vbank})" + ) + + @public + val io = IO(new Bundle { + val request = Flipped(Decoupled(new PostGDCmd(frontendParam, memParam))) + val command = Decoupled(new DecodedCommand(frontendParam, memParam)) + }) + + val memAddrLen = memParam.memAddrLen + val bankIdLen = frontendParam.bank_id_len + val iterLen = frontendParam.iter_len + + // Only process Mem instructions + io.request.ready := io.command.ready + + val func7 = io.request.bits.cmd.raw_inst(31, 25) + val rs1 = io.request.bits.cmd.rs1Data + val rs2 = io.request.bits.cmd.rs2Data + + // Unified encoding: + // rs1[9:0] = read/config bank (BANK0) + // rs1[29:20] = write bank (BANK2) + // rs1[62:30] = iter; rs1[63] declares an in-place update. + // funct7[6:4] = enable (bank access flags, decoded by GlobalDecoder) + // rs2[38:0] = mem_addr (for MVIN/MVOUT, 39-bit) + // + import LSDecodeFields._ + // format: off + val ls_default_decode = List(N, N, DADDR, DADDR, DSPECIAL, N) + val ls_decode_list = ListLookup(func7, ls_default_decode, Array( + MSET_BITPAT -> List(N, N, 0.U(memAddrLen.W), rs1(bankIdLen - 1, 0), rs2, Y), + MVIN_MMIO_BITPAT -> List(Y, N, rs2(memAddrLen - 1, 0), 0.U, rs2, Y), + MVIN_BITPAT -> List(Y, N, rs2(memAddrLen - 1, 0), rs1(bankIdLen + 19, 20), rs2, Y), + MVIN_2D_BITPAT -> List(Y, N, Cat(rs2(35, 0), 0.U(3.W)), rs1(bankIdLen + 19, 20), rs2, Y), + MVIN_KERNEL_BITPAT -> List(Y, N, rs2(memAddrLen - 1, 0), rs1(bankIdLen + 19, 20), rs2, Y), + MVOUT_BITPAT -> List(N, Y, rs2(memAddrLen - 1, 0), rs1(bankIdLen - 1, 0), rs2, Y) + ) + ) + // format: on + + assert( + !(io.request.fire && !ls_decode_list(LSDecodeFields.VALID.id).asBool), + s"Decoder: Invalid command opcode, func7 = 0x%x\n", + func7 + ) + +// ----------------------------------------------------------------------------- +// Output assignment +// ----------------------------------------------------------------------------- + io.command.valid := io.request.valid && (io.request.bits.domain_id === DomainId.MEM) + + val transfer = func7 === MSET_BITPAT && rs2(12) + val raw_bank_id = Mux(func7 === MVIN_BITPAT || func7 === MVIN_2D_BITPAT || transfer, rs1(29, 20), rs1(9, 0)) + + val architectural_bank = Mux( + func7 === MVIN_BITPAT || func7 === MVIN_2D_BITPAT || transfer, + io.request.bits.bankAccess.wr_bank_id, + io.request.bits.bankAccess.rd_bank_0_id + ) + + val config_bank = Mux(func7 === MSET_BITPAT && !transfer, io.request.bits.bankAccess.wr_bank_id, architectural_bank) + val shared_bank_id = + memParam.sharedEnable.B && config_bank >= sharedVbankBase.U + // format: off + // BB_BANK0 is encoded in rs1[9:0]. Keep the configured architectural ID; + // private and shared banks occupy disjoint ranges in that namespace. + io.command.bits.is_shared := io.command.valid && shared_bank_id + io.command.bits.is_load := Mux(io.command.valid, ls_decode_list(LSDecodeFields.LD_EN.id).asBool, false.B) + io.command.bits.is_store := Mux(io.command.valid, ls_decode_list(LSDecodeFields.ST_EN.id).asBool, false.B) + io.command.bits.is_config := Mux(io.command.valid, func7 === MSET_BITPAT, false.B) + io.command.bits.transfer := io.command.valid && transfer + io.command.bits.source_bank_id := rs1(bankIdLen - 1, 0) + io.command.bits.clear := io.command.valid && (func7 === MSET_BITPAT) && rs2(11) + io.command.bits.is_mvin_2d := Mux(io.command.valid, func7 === MVIN_2D_BITPAT, false.B) + io.command.bits.is_mmio := func7 === MVIN_MMIO_BITPAT + io.command.bits.is_image := func7 === MVIN_KERNEL_BITPAT + io.command.bits.mem_addr := Mux(io.command.valid, ls_decode_list(LSDecodeFields.MEMADDR.id).asUInt, 0.U(memParam.memAddrLen.W)) + io.command.bits.iter := Mux(io.command.valid, rs1(62, 30), 0.U(iterLen.W)) + + when(io.request.fire && func7 === MSET_BITPAT) { + when(transfer) { + assert(rs1(63, 30) === 0.U && rs1(19, 10) === 0.U, "MSET transfer reserves rs1 outside BANK0/BANK2") + assert(rs2 === (1.U << 12), "MSET transfer reserves other rs2 bits") + assert(rs1(9, 0) =/= rs1(29, 20), "MSET transfer requires distinct source and target") + assert((rs1(9, 0) >= sharedVbankBase.U) === (rs1(29, 20) >= sharedVbankBase.U), "MSET transfer requires one bank namespace") + }.otherwise { + assert(rs1(63, 10) === 0.U, "MSET reserves rs1[63:10]") + assert(rs2(63, 12) === 0.U, "MSET reserves rs2[63:12]") + } + assert(!(rs2(11) && !rs2(10)), "MSET clear requires alloc=1") + } + when(io.request.fire && (func7 === MVIN_BITPAT || func7 === MVOUT_BITPAT)) { + assert(rs2(63) || rs2(62, 58) === 0.U, "Whole-bank DMA reserves selected group bits") + } + when(io.request.fire && func7 === MVIN_2D_BITPAT) { + val height = rs1(62, 30) + val pixelBytes = rs2(42, 36) << 3 + val width = rs2(61, 59) +& 1.U + val validBytes = Mux(rs2(62), 8.U(5.W), 16.U(5.W)) + assert(height =/= 0.U, "MVIN_2D height must be non-zero") + assert(rs2(42, 36) =/= 0.U, "MVIN_2D pixel bytes must be non-zero") + assert(rs2(52, 43) =/= 0.U, "MVIN_2D source width must be non-zero") + assert(validBytes <= pixelBytes, "MVIN_2D valid bytes exceed pixel bytes") + assert((rs2(58, 53) +& (height * width)) <= Mux(shared_bank_id, memParam.sharedBankEntries.U, memParam.bankEntries.U), "MVIN_2D destination exceeds bank") + assert(!rs2(63), "MVIN_2D reserves rs2[63]") + } + when(io.request.fire && (func7 === MVIN_BITPAT || func7 === MVIN_2D_BITPAT)) { + assert(rs1(19, 0) === 0.U, "MVIN write-bank encoding reserves rs1[19:0]") + } + + val ls_bank_id = ls_decode_list(LSDecodeFields.BANK_ID.id).asUInt + io.command.bits.bank_id := Mux(io.command.valid, Mux(transfer, raw_bank_id, ls_bank_id), 0.U(frontendParam.bank_id_len.W)) + io.command.bits.special := Mux(io.command.valid, ls_decode_list(LSDecodeFields.SPECIAL.id).asUInt, 0.U(64.W)) + // format: on +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/build.sc b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/build.sc new file mode 100644 index 000000000..d1c452eda --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/build.sc @@ -0,0 +1,23 @@ +import mill._ +import $file.^.^.^.^.^.^.^.^.^.^.settings + +val ipRoot = millSourcePath + +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.acc.memdomain.prm.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Bank.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/MvoverIO.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/PRM.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/configs/uvm.toml b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/configs/uvm.toml new file mode 100644 index 000000000..0b7df89d6 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/configs/uvm.toml @@ -0,0 +1,27 @@ +[parameters] +diffTest = false + +[parameters.memory] +sharedPorts = 0 +bankNum = 2 +bankLayoutEnable = false +bankWidth = 128 +bankEntries = 8 +bankMaskLen = 16 +virtualBankCount = 4 +sharedEnable = false +sharedEntries = 0 +sharedBankEntries = 0 +sharedBankNum = 0 +sharedInputChannels = 0 +sharedDefaultGroupCount = 0 +bankRequestPorts = 1 +dma_buswidth = 128 +memAddrLen = 44 +mmioEnable = false +mmioBankNum = 0 +mmioBankEntries = 0 +mmioBankWidth = 0 +mmioReadWidth = 0 + +[defines] diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/agent/driver.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/agent/driver.svh new file mode 100644 index 000000000..e345f7d57 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/agent/driver.svh @@ -0,0 +1,36 @@ +class driver extends uvm_driver #(cycle_item); + `uvm_component_utils(driver) + virtual bus_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual bus_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "PRM driver requires an interface") + endfunction + task run_phase(uvm_phase phase); + cycle_item req; + forever begin + seq_item_port.get_next_item(req); + @(negedge vif.clock); + vif.reset = req.reset; + foreach (vif.ports[b]) begin + vif.ports[b].read_valid = req.pins[b].read_valid; + vif.ports[b].read_addr = req.pins[b].read_addr; + vif.ports[b].read_result_ready = req.pins[b].read_result_ready; + vif.ports[b].write_valid = req.pins[b].write_valid; + vif.ports[b].write_addr = req.pins[b].write_addr; + vif.ports[b].write_data = req.pins[b].write_data; + vif.ports[b].write_mask = req.pins[b].write_mask; + vif.ports[b].write_result_ready = req.pins[b].write_result_ready; + vif.ports[b].clear = req.pins[b].clear; + vif.ports[b].bind_generation = req.pins[b].bind_generation; + end + @(posedge vif.clock); + req.sampled = vif.ports; + #1; + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/agent/item.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/agent/item.svh new file mode 100644 index 000000000..69fae1d35 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/agent/item.svh @@ -0,0 +1,23 @@ +class cycle_item extends uvm_sequence_item; + `uvm_object_utils(cycle_item) + bank_signals pins[BANKS], sampled[BANKS]; + bit reset; + function new(string name = "cycle_item"); + super.new(name); + endfunction +endclass +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(data, UVM_ALL_ON) + `uvm_field_int(ok, UVM_ALL_ON) + `uvm_field_int(bank, UVM_ALL_ON) + `uvm_field_int(write, UVM_ALL_ON) + `uvm_object_utils_end + logic [WIDTH-1:0] data; + logic ok; + int bank; + bit write; + function new(string name = "result_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/agent/monitor.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/agent/monitor.svh new file mode 100644 index 000000000..644e6ad42 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/agent/monitor.svh @@ -0,0 +1,24 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual bus_if vif; + uvm_analysis_port #(cycle_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual bus_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "PRM monitor requires an interface") + endfunction + task run_phase(uvm_phase phase); + cycle_item t; + forever begin + @(posedge vif.clock); + t = cycle_item::type_id::create("observed"); + t.sampled = vif.ports; + t.reset = vif.reset; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/env/env.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/env/env.svh new file mode 100644 index 000000000..f6b4be38d --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/env/env.svh @@ -0,0 +1,31 @@ +class env extends uvm_env; + `uvm_component_utils(env) + driver drv; + monitor mon; + memory_model model; + uvm_sequencer #(cycle_item) seqr; + result_board #(result_item, 1) scoreboard; + uvm_analysis_imp #(bit, env) reset_export; + function new(string name, uvm_component parent); + super.new(name, parent); + reset_export = new("reset_export", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + drv = driver::type_id::create("driver", this); + mon = monitor::type_id::create("monitor", this); + model = memory_model::type_id::create("model", this); + seqr = new("seqr", this); + scoreboard = result_board#(result_item, 1)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + drv.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(model.analysis_export); + model.expected_ap.connect(scoreboard.expected_export); + model.actual_ap.connect(scoreboard.actual_export); + model.reset_ap.connect(reset_export); + endfunction + function void write(bit value); + scoreboard.cancel_pending(); + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/env/model.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/env/model.svh new file mode 100644 index 000000000..e53468236 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/env/model.svh @@ -0,0 +1,108 @@ +class memory_model extends uvm_subscriber #(cycle_item); + `uvm_component_utils(memory_model) + logic [WIDTH-1:0] words[BANKS][ROWS]; + bit [MASKS-1:0] initialized[BANKS][ROWS]; + int expected_serial[BANKS][2], actual_serial[BANKS][2]; + uvm_analysis_port #(result_item) expected_ap, actual_ap; + uvm_analysis_port #(bit) reset_ap; + covergroup accesses with function sample (int bank, bit write, int row, logic [MASKS-1:0] mask); + option.per_instance = 1; + bank_index: coverpoint bank {bins banks[] = {[0 : BANKS - 1]};} + direction: coverpoint write; + address: coverpoint row { + bins first = {0}; bins interior = {[1 : ROWS - 2]}; bins last = {ROWS - 1}; + } + write_mask: coverpoint mask iff (write) { + bins none = {'0}; bins full = {'1}; bins partial = default; + } + bank_direction: cross bank_index, direction; + endgroup + covergroup protocol with function sample ( + bit clear, bit clearing, bit read_stall, bit write_stall, bit collision + ); + option.per_instance = 1; + clear_request: coverpoint clear; + clearing_active: coverpoint clearing; + read_backpressure: coverpoint read_stall; + write_backpressure: coverpoint write_stall; + read_write_collision: coverpoint collision; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected_ap = new("expected_ap", this); + actual_ap = new("actual_ap", this); + reset_ap = new("reset_ap", this); + accesses = new(); + protocol = new(); + foreach (expected_serial[b, d]) begin + expected_serial[b][d] = 0; + actual_serial[b][d] = 0; + end + endfunction + function void write(cycle_item t); + result_item expected, actual; + if (t.reset) begin + reset_ap.write(1); + foreach (actual_serial[b, d]) actual_serial[b][d] = expected_serial[b][d]; + return; + end + foreach (t.sampled[b]) begin + bank_signals s = t.sampled[b]; + protocol.sample(s.clear, s.clearing, s.read_result_valid && !s.read_result_ready, + s.write_result_valid && !s.write_result_ready, s.read_valid && s.write_valid); + if (s.clear || s.clearing) + evidence::check(!s.read_ready && !s.write_ready, "PRM accepted an access during clear"); + if (s.clear) begin + for (int row = 0; row < ROWS; row++) begin + words[b][row] = '0; + initialized[b][row] = '1; + end + end + if (s.write_valid && s.write_ready) begin + evidence::check(!s.read_ready, "PRM read bypassed a same-cycle write"); + for (int m = 0; m < MASKS; m++) + if (s.write_mask[m]) begin + words[b][s.write_addr][m*(WIDTH/MASKS)+:(WIDTH/MASKS)] = s.write_data[m*(WIDTH/MASKS)+:(WIDTH/MASKS)]; + initialized[b][s.write_addr][m] = 1; + end + expected = result_item::type_id::create("write_ack"); + expected.bank = b; + expected.write = 1; + expected.data = 0; + expected.ok = 1; + expected.set_transaction_id((expected_serial[b][1]++ * BANKS + b) * 2 + 1); + expected_ap.write(expected); + accesses.sample(b, 1, s.write_addr, s.write_mask); + end + if (s.read_valid && s.read_ready) begin + evidence::check(initialized[b][s.read_addr] == '1, "PRM read of uninitialized storage"); + expected = result_item::type_id::create("read"); + expected.bank = b; + expected.write = 0; + expected.data = words[b][s.read_addr]; + expected.ok = 1; + expected.set_transaction_id((expected_serial[b][0]++ * BANKS + b) * 2); + expected_ap.write(expected); + accesses.sample(b, 0, s.read_addr, '0); + end + if (s.write_result_valid && s.write_result_ready) begin + actual = result_item::type_id::create("write_result"); + actual.bank = b; + actual.write = 1; + actual.data = 0; + actual.ok = s.write_ok; + actual.set_transaction_id((actual_serial[b][1]++ * BANKS + b) * 2 + 1); + actual_ap.write(actual); + end + if (s.read_result_valid && s.read_result_ready) begin + actual = result_item::type_id::create("read_result"); + actual.bank = b; + actual.write = 0; + actual.data = s.read_data; + actual.ok = 1; + actual.set_transaction_id((actual_serial[b][0]++ * BANKS + b) * 2); + actual_ap.write(actual); + end + end + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/filelist.f b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/filelist.f new file mode 100644 index 000000000..8c3e77d47 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/filelist.f @@ -0,0 +1,9 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/types.svh +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/PRM/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/interface.sv b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/interface.sv new file mode 100644 index 000000000..f78feb6a3 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/interface.sv @@ -0,0 +1,21 @@ +interface bus_if ( + input logic clock +); + import prm_types::*; + logic reset = 1; + bank_signals ports[BANKS]; + for (genvar b = 0; b < BANKS; b++) begin + assert property (@(posedge clock) disable iff (reset) + ports[b].read_result_valid && !ports[b].read_result_ready |=> + ports[b].read_result_valid && $stable( + ports[b].read_data + )) + else $fatal(1, "PRM read response changed under backpressure"); + assert property (@(posedge clock) disable iff (reset) + ports[b].write_result_valid && !ports[b].write_result_ready |=> + ports[b].write_result_valid && $stable( + ports[b].write_ok + )) + else $fatal(1, "PRM write response changed under backpressure"); + end +endinterface diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/package.sv b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/package.sv new file mode 100644 index 000000000..8d916d3ca --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/package.sv @@ -0,0 +1,13 @@ +package prm_pkg; + import uvm_pkg::*; + import verification_pkg::*; + import prm_types::*; + `include "uvm_macros.svh" + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..c2f9f7a81 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/seq/sequence.svh @@ -0,0 +1,140 @@ +class access_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(access_sequence) + env test_env; + bank_signals pins[BANKS], observed[BANKS]; + bit reset = 1; + int checked = 0; + function new(string name = "access_sequence"); + super.new(name); + endfunction + function logic [WIDTH-1:0] pattern(int bank, int seed); + logic [WIDTH-1:0] value; + for (int bit_index = 0; bit_index < WIDTH; bit_index++) + value[bit_index] = ((seed * 32'h9e3779b9) >> (bit_index % 32)) ^ ((bit_index + bank) % 3 == 0); + return value; + endfunction + task tick(int cycles = 1); + cycle_item req; + repeat (cycles) begin + req = cycle_item::type_id::create("cycle"); + start_item(req); + req.reset = reset; + req.pins = pins; + finish_item(req); + observed = req.sampled; + end + endtask + task drain(int count); + checked += count; + while (test_env.scoreboard.checked != checked) tick(); + endtask + task access (bit write, int row, logic [MASKS-1:0] mask = '1, int seed = 1, int stall = 0); + bit pending[BANKS]; + foreach (pins[b]) begin + pending[b] = 1; + pins[b].read_valid = !write; + pins[b].read_addr = row; + pins[b].write_valid = write; + pins[b].write_addr = row; + pins[b].write_data = pattern(b, seed); + pins[b].write_mask = mask; + pins[b].read_result_ready = stall == 0; + pins[b].write_result_ready = stall == 0; + end + do begin + tick(); + foreach (pins[b]) + if (pending[b] && (write ? observed[b].write_ready : observed[b].read_ready)) begin + pending[b] = 0; + pins[b].read_valid = 0; + pins[b].write_valid = 0; + end + end while (pending.or()); + tick(stall); + foreach (pins[b]) begin + pins[b].read_result_ready = 1; + pins[b].write_result_ready = 1; + end + drain(BANKS); + endtask + task reset_pending(bit write); + foreach (pins[b]) begin + pins[b].read_valid = !write; + pins[b].read_addr = ROWS - 1; + pins[b].write_valid = write; + pins[b].write_addr = ROWS - 1; + pins[b].write_data = pattern(b, 77); + pins[b].write_mask = '1; + pins[b].read_result_ready = 0; + pins[b].write_result_ready = 0; + end + tick(); + foreach (pins[b]) begin + evidence::check(write ? observed[b].write_ready : observed[b].read_ready, + "PRM cancellation request was not accepted"); + pins[b].read_valid = 0; + pins[b].write_valid = 0; + end + tick(3); + reset = 1; + tick(2); + reset = 0; + foreach (pins[b]) begin + pins[b].read_result_ready = 1; + pins[b].write_result_ready = 1; + end + tick(); + access (0, ROWS - 1); + endtask + task body(); + foreach (pins[b]) pins[b] = '0; + tick(4); + reset = 0; + foreach (pins[b]) begin + pins[b].clear = 1; + pins[b].read_result_ready = 1; + pins[b].write_result_ready = 1; + end + tick(); + foreach (pins[b]) pins[b].clear = 0; + tick(ROWS + 2); + for (int row = 0; row < ROWS; row++) access (0, row); + for (int row = 0; row < ROWS; row++) access (1, row, '1, row + 1); + for (int row = 0; row < ROWS; row++) access (0, row, '0, 0, 3); + for (int m = 0; m < MASKS; m++) begin + access (1, m % ROWS, MASKS'(1) << m, m + 100, 3); + access (0, m % ROWS); + end + access (1, ROWS - 1, '0, 0); + access (0, ROWS - 1); + // Concurrent read/write requests use the write port first, then read back its update. + foreach (pins[b]) begin + pins[b].read_valid = 1; + pins[b].read_addr = 0; + pins[b].write_valid = 1; + pins[b].write_addr = 0; + pins[b].write_data = pattern(b, 999); + pins[b].write_mask = '1; + end + tick(); + foreach (pins[b]) begin + evidence::check(observed[b].write_ready && !observed[b].read_ready, + "PRM did not prioritize a write"); + pins[b].write_valid = 0; + end + tick(); + foreach (pins[b]) begin + evidence::check(observed[b].read_ready, "PRM read did not resume"); + pins[b].read_valid = 0; + end + drain(2 * BANKS); + reset_pending(0); + reset_pending(1); + foreach (pins[b]) pins[b].clear = 1; + tick(); + foreach (pins[b]) pins[b].clear = 0; + tick(ROWS + 2); + for (int row = 0; row < ROWS; row++) access (0, row); + evidence::check(evidence::cancelled == 2 * BANKS, "PRM reset cancellation count mismatch"); + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/suite.toml b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/suite.toml new file mode 100644 index 000000000..3837e4d46 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/suite.toml @@ -0,0 +1,20 @@ +[model] +manifest = "" +library = "" + +[[tests]] +name = "prm" +top = "prm_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 100000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +functional_groups = ["prm_pkg::memory_model::accesses", "prm_pkg::memory_model::protocol"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/tb.sv b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/tb.sv new file mode 100644 index 000000000..5efd2ddc9 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/tb.sv @@ -0,0 +1,19 @@ +module prm_tb; + import uvm_pkg::*; + import prm_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + bus_if bus (clock); + PRM dut ( + .clock(clock), + .reset(bus.reset), + `include "ports.svh" + ); + if (!`PRM_DIFF_TEST) begin + for (genvar b = 0; b < `PRM_BANKS; b++) assign bus.ports[b].hash = 0; + end + initial begin + uvm_config_db#(virtual bus_if)::set(null, "uvm_test_top.env.*", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/tests/test.svh new file mode 100644 index 000000000..70eb6d956 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/tests/test.svh @@ -0,0 +1,16 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + access_sequence seq = access_sequence::type_id::create("scenario"); + seq.test_env = test_env; + seq.start(test_env.seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/types.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/types.svh new file mode 100644 index 000000000..e95a5e53a --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/types.svh @@ -0,0 +1,13 @@ +`include "parameters.svh" +package prm_types; + localparam int BANKS = `PRM_BANKS, ROWS = `PRM_ROWS, WIDTH = `PRM_WIDTH, MASKS = `PRM_MASKS; + typedef struct packed { + logic read_valid, read_ready, read_result_valid, read_result_ready; + logic write_valid, write_ready, write_result_valid, write_result_ready, write_ok; + logic [15:0] read_addr, write_addr; + logic [WIDTH-1:0] read_data, write_data; + logic [MASKS-1:0] write_mask; + logic clear, clearing, bind_generation; + logic [31:0] hash; + } bank_signals; +endpackage diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/scala/Bank.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/scala/Bank.scala new file mode 100644 index 000000000..64772e43d --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/scala/Bank.scala @@ -0,0 +1,101 @@ +package framework.acc.memdomain.prm + +import framework.ip.memory.bank.{SramReadIO, SramReadReq, SramReadResp, SramWriteIO, SramWriteReq, SramWriteResp} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.acc.memdomain.configs.MemDomainParam + +/** + * Bank: Pure SRAM bank + * Simple read/write memory without any accumulation logic + * Each bank is a single-port SRAM. `clear` zeroes every row locally, one row per cycle; + * the ports refuse requests while `clearing`, so banks clear in parallel. + */ +@instantiable +class Bank(val b: MemDomainParam) extends Module { + require( + b.bankEntries >= 2 && b.bankEntries <= 65536 + ) + val indexBits = log2Ceil(b.bankEntries) + val mask_len = b.bankMaskLen + val mask_elem = UInt((b.bankWidth / mask_len).W) + + @public + val io = IO(new Bundle { + val sramRead = new SramReadIO(b) + val sramWrite = new SramWriteIO(b) + val clear = Input(Bool()) + val clearing = Output(Bool()) + }) + + val mem = SyncReadMem(b.bankEntries, Vec(mask_len, mask_elem)) + + // ----------------------------------------------------------------------------- + // Local clear + // ----------------------------------------------------------------------------- + val clearing = RegInit(false.B) + val clearRow = RegInit(0.U(log2Ceil(b.bankEntries).W)) + io.clearing := clearing + when(io.clear) { + clearing := true.B + clearRow := 0.U + }.elsewhen(clearing) { + clearRow := clearRow + 1.U + when(clearRow === (b.bankEntries - 1).U)(clearing := false.B) + } + + val readPending = RegNext(io.sramRead.req.fire, false.B) + val readHeld = RegInit(false.B) + val readData = Reg(UInt(b.bankWidth.W)) + val writeValid = RegInit(false.B) + + io.sramRead.resp.valid := readPending || readHeld + io.sramWrite.resp.valid := writeValid + io.sramWrite.resp.bits.ok := true.B + io.sramWrite.req.ready := !io.clear && !clearing && (!writeValid || io.sramWrite.resp.ready) + io.sramRead.req.ready := !io.clear && !clearing && !io.sramWrite.req.fire && + (!io.sramRead.resp.valid || io.sramRead.resp.ready) + + val ren = io.sramRead.req.fire + val wen = clearing || io.sramWrite.req.fire + + when(io.sramRead.req.fire) { + assert( + io.sramRead.req.bits.addr < b.bankEntries.U, + "SRAM read exceeds bank depth" + ) + } + when(io.sramWrite.req.fire) { + assert( + io.sramWrite.req.bits.addr < b.bankEntries.U, + "SRAM write exceeds bank depth" + ) + } + + val rdata = mem.readWrite( + Mux( + clearing, + clearRow, + Mux(wen, io.sramWrite.req.bits.addr(indexBits - 1, 0), io.sramRead.req.bits.addr(indexBits - 1, 0)) + ), + Mux(clearing, 0.U.asTypeOf(Vec(mask_len, mask_elem)), io.sramWrite.req.bits.data.asTypeOf(Vec(mask_len, mask_elem))), + Mux(clearing, VecInit(Seq.fill(mask_len)(true.B)), io.sramWrite.req.bits.mask), + ren || wen, + wen + ) + + io.sramRead.resp.bits.data := Mux(readHeld, readData, rdata.asUInt) + when(readPending && !io.sramRead.resp.ready) { + readHeld := true.B + readData := rdata.asUInt + }.elsewhen(io.sramRead.resp.fire) { + readHeld := false.B + } + when(io.sramWrite.req.fire) { + writeValid := true.B + }.elsewhen(io.sramWrite.resp.fire) { + writeValid := false.B + } +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/scala/Emit.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/scala/Emit.scala new file mode 100644 index 000000000..0dce01275 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/scala/Emit.scala @@ -0,0 +1,55 @@ +package framework.acc.memdomain.prm + +import java.nio.file.{Files, Paths} +import framework.acc.memdomain.configs.MemDomainParam + +object Emit extends App { + val Array("--config", config, "--output-dir", output) = args + val values = ujson.read(Files.readString(Paths.get(config))) + val p = upickle.default.read[MemDomainParam](values("memory")) + val diffTest = values("diffTest").bool + Files.createDirectories(Paths.get(output)) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new PRM(p, diffTest), + args = Array("--target-dir", s"$output/PRM", "--split-verilog") + ) + Files.writeString( + Paths.get(output, "parameters.svh"), + s"""`ifndef PRM_PARAMETERS +`define PRM_PARAMETERS +`define PRM_BANKS ${p.bankNum} +`define PRM_ROWS ${p.bankEntries} +`define PRM_WIDTH ${p.bankWidth} +`define PRM_MASKS ${p.bankMaskLen} +`define PRM_DIFF_TEST ${if (diffTest) 1 else 0} +`endif +""" + ) + + val ports = (0 until p.bankNum).flatMap { bank => + val prefix = s"io_banks_$bank" + val fields = Seq( + "sramRead_req_valid" -> "read_valid", + "sramRead_req_ready" -> "read_ready", + "sramRead_req_bits_addr" -> "read_addr", + "sramRead_resp_valid" -> "read_result_valid", + "sramRead_resp_ready" -> "read_result_ready", + "sramRead_resp_bits_data" -> "read_data", + "sramWrite_req_valid" -> "write_valid", + "sramWrite_req_ready" -> "write_ready", + "sramWrite_req_bits_addr" -> "write_addr", + "sramWrite_req_bits_data" -> "write_data", + "sramWrite_resp_valid" -> "write_result_valid", + "sramWrite_resp_ready" -> "write_result_ready", + "sramWrite_resp_bits_ok" -> "write_ok", + "clear" -> "clear", + "clearing" -> "clearing", + "bind" -> "bind_generation" + ).map { case (port, signal) => s".${prefix}_$port(bus.ports[$bank].$signal)" } + val masks = + (0 until p.bankMaskLen).map(mask => s".${prefix}_sramWrite_req_bits_mask_$mask(bus.ports[$bank].write_mask[$mask])") + fields ++ masks ++ (if (diffTest) Seq(s".io_hashes_$bank(bus.ports[$bank].hash)") else Seq.empty) + } + + Files.writeString(Paths.get(output, "ports.svh"), ports.mkString(",\n")) +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/scala/MvoverIO.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/scala/MvoverIO.scala new file mode 100644 index 000000000..136e968f6 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/scala/MvoverIO.scala @@ -0,0 +1,27 @@ +package framework.acc.memdomain + +import chisel3._ +import chisel3.util._ + +object MvoverISA { + val Funct = 0x0d + val ThreadBits = 8 + val BankBits = 10 + val AddressBits = 16 +} + +/** Contiguous rows between logical bank endpoints attached to one fabric. */ +class MvoverCommand extends Bundle { + val sourceThread = UInt(MvoverISA.ThreadBits.W) + val sourceBank = UInt(MvoverISA.BankBits.W) + val sourceAddr = UInt(MvoverISA.AddressBits.W) + val targetThread = UInt(MvoverISA.ThreadBits.W) + val targetBank = UInt(MvoverISA.BankBits.W) + val targetAddr = UInt(MvoverISA.AddressBits.W) + val rows = UInt((MvoverISA.AddressBits + 1).W) +} + +class MvoverPort extends Bundle { + val command = Decoupled(new MvoverCommand) + val completion = Flipped(Decoupled(Bool())) +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/scala/PRM.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/scala/PRM.scala new file mode 100644 index 000000000..d949f187d --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/scala/PRM.scala @@ -0,0 +1,58 @@ +package framework.acc.memdomain.prm + +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.memdomain.prm.Bank +import framework.ip.memory.bank.{SramReadIO, SramWriteIO} +import framework.lib.hash.BankHashMonitor + +class PhysicalBankPort(memParam: MemDomainParam) extends Bundle { + val sramRead = new SramReadIO(memParam) + val sramWrite = new SramWriteIO(memParam) + val clear = Input(Bool()) + val clearing = Output(Bool()) + val bind = Input(Bool()) +} + +@instantiable +class PRM(val memParam: MemDomainParam, val diffTest: Boolean) extends Module { + + @public + val io = IO(new Bundle { + val banks = Vec(memParam.bankNum, new PhysicalBankPort(memParam)) + + val hashes = + if (diffTest) Some(Output(Vec(memParam.bankNum, UInt(32.W)))) + else None + + }) + + val banks: Seq[Instance[Bank]] = + Seq.fill(memParam.bankNum)(Instantiate( + new Bank(memParam) + )) + + for (i <- banks.indices) { + val bank = banks(i) + val port = io.banks(i) + bank.io.sramRead <> port.sramRead + bank.io.sramWrite <> port.sramWrite + bank.io.clear := port.clear + port.clearing := bank.io.clearing + if (diffTest) { + val monitor: Instance[BankHashMonitor] = + Instantiate(new BankHashMonitor(memParam)) + monitor.io.bind := port.bind + monitor.io.write.valid := port.sramWrite.req.valid && bank.io.sramWrite.req.ready + monitor.io.write.bits.addr := port.sramWrite.req.bits.addr + monitor.io.write.bits.mask := port.sramWrite.req.bits.mask + monitor.io.write.bits.data := port.sramWrite.req.bits.data + bank.io.sramWrite.req.valid := port.sramWrite.req.valid && monitor.io.write.ready + port.sramWrite.req.ready := bank.io.sramWrite.req.ready && monitor.io.write.ready + io.hashes.get(i) := monitor.io.statusHash + } + } +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/build.sc b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/build.sc new file mode 100644 index 000000000..c9cb32733 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/build.sc @@ -0,0 +1,28 @@ +import mill._ +import $file.src.main.scala.toddr.{build => toddr_build} +import $file.src.main.scala.tol1.{build => tol1_build} +import $file.^.^.^.^.^.^.^.^.^.^.settings + +val ipRoot = millSourcePath + +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.acc.memdomain.tomem.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/ToMem.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) ++ + toddr_build.sources() ++ + tol1_build.sources() +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) ++ + toddr_build.emitSources() ++ + tol1_build.emitSources() +) diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/configs/default.toml b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/configs/default.toml new file mode 100644 index 000000000..c6c2b8674 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/configs/default.toml @@ -0,0 +1,81 @@ + +[tracking] +entries = 24 +addressBits = 44 +idBits = 8 +lineBytes = 64 +maxRanges = 8 + + +[prepared] +contexts = 24 +maxRanges = 8 +idBits = 8 +beatBytes = 16 + +[prepared.bus] +nodeIdBits = 7 +addressBits = 44 +dataBits = 256 + + + +[axi] +addressBits = 44 +dataBits = 128 +idBits = 4 + + +[[regions]] +base = 0x80000000 +bytes = 0x20000000 +cacheable = true +executable = true +readable = true +writable = true +atomic = true +normal = true + + +[[regions]] +base = 0x10000 +bytes = 0x10000 +cacheable = true +executable = true +readable = true +writable = false +atomic = false +normal = true + + +[[regions]] +base = 0x2000000 +bytes = 0x10000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + + +[[regions]] +base = 0xc000000 +bytes = 0x4000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + + +[[regions]] +base = 0x60000000 +bytes = 0x10000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/configs/uvm.toml b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/configs/uvm.toml new file mode 100644 index 000000000..1fd3322af --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/configs/uvm.toml @@ -0,0 +1,82 @@ +[parameters] + +[parameters.frontend] +ownerId = 7 +commandDataBits = 64 +rob_entries = 4 +bank_alias_enable = true +rs_out_of_order_response = true +bank_id_len = 4 +max_private_vbank = 3 +shared_vbank_base = 16 +iter_len = 16 +sub_rob_enable = false +sub_rob_depth = 4 + +[parameters.memory] +sharedPorts = 0 +bankNum = 2 +bankLayoutEnable = true +bankWidth = 128 +bankEntries = 8 +bankMaskLen = 16 +virtualBankCount = 4 +sharedEnable = false +sharedEntries = 0 +sharedBankEntries = 0 +sharedBankNum = 0 +sharedInputChannels = 0 +sharedDefaultGroupCount = 0 +bankRequestPorts = 1 +dma_buswidth = 128 +memAddrLen = 44 +mmioEnable = true +mmioBankNum = 1 +mmioBankEntries = 16 +mmioBankWidth = 128 +mmioReadWidth = 32 + +[parameters.permission] +addressBits = 44 +entries = 16 + +[parameters.gp] +enable = true +laneNumber = 1 +vLen = 128 +eLen = 64 +iBufWords = 16 +memoryPorts = 1 + + + +[parameters.transport.tracking] +entries=4 +addressBits=44 +idBits=8 +lineBytes=64 +maxRanges=8 +[parameters.transport.prepared] +contexts=4 +maxRanges=8 +idBits=8 +beatBytes=16 +[parameters.transport.prepared.bus] +nodeIdBits=7 +addressBits=44 +dataBits=256 +[parameters.transport.axi] +addressBits=44 +dataBits=128 +idBits=4 +[[parameters.transport.regions]] +base="0" +bytes="17592186044416" +normal=true +cacheable=true +atomic=true +readable=true +writable=true +executable=true + +[defines] diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/agent/driver.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/agent/driver.svh new file mode 100644 index 000000000..5a833dd30 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/agent/driver.svh @@ -0,0 +1,25 @@ +class driver extends uvm_driver #(cycle_item); + `uvm_component_utils(driver) + virtual tomem_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual tomem_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "TSS interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item req; + forever begin + seq_item_port.get_next_item(req); + @(negedge vif.clock); + vif.reset = req.reset; + vif.source = req.inputs; + @(posedge vif.clock); + req.outputs = vif.outputs; + #1; + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/agent/fixture.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/agent/fixture.svh new file mode 100644 index 000000000..534a8094d --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/agent/fixture.svh @@ -0,0 +1,19 @@ +class fixture extends uvm_object; + `uvm_object_utils(fixture) + int groups[`TM_BANKS]; + bit read_error, write_error, bank_error, image_fault; + int admission_error, completion_error; + longint unsigned physical_offset = 'h100000; + function new(string name = "fixture"); + super.new(name); + foreach (groups[b]) groups[b] = 2; + endfunction + function byte unsigned initial_byte(longint unsigned address); + return 8'(address ^ (address >> 8) ^ (address >> 16)); + endfunction + function bit [127:0] initial_bank(int bank_id, int group_index, int row); + bit [127:0] value; + for (int b = 0; b < 16; b++) value[b*8+:8] = 8'(bank_id * 37 + group_index * 19 + row * 7 + b); + return value; + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/agent/item.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/agent/item.svh new file mode 100644 index 000000000..e30bf3bb5 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/agent/item.svh @@ -0,0 +1,18 @@ +class cycle_item extends uvm_sequence_item; + `uvm_object_utils(cycle_item) + logic [`TM_INPUT_BITS-1:0] inputs; + logic [`TM_OUTPUT_BITS-1:0] outputs; + bit reset; + function new(string name = "cycle_item"); + super.new(name); + endfunction +endclass +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(value, UVM_ALL_ON) + `uvm_object_utils_end + bit [511:0] value; + function new(string name = "result"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/agent/monitor.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/agent/monitor.svh new file mode 100644 index 000000000..9a85039ff --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/agent/monitor.svh @@ -0,0 +1,25 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual tomem_if vif; + uvm_analysis_port #(cycle_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual tomem_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "TSS monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item t; + forever begin + @(posedge vif.clock); + t = cycle_item::type_id::create("observed"); + t.inputs = vif.inputs; + t.outputs = vif.outputs; + t.reset = vif.reset; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/agent/peer.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/agent/peer.svh new file mode 100644 index 000000000..7251f9293 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/agent/peer.svh @@ -0,0 +1,211 @@ +class peer extends uvm_component; + `uvm_component_utils(peer) + virtual tomem_if vif; + fixture image; + bit [127:0] bank_memory[`TM_BANKS][`TM_GROUPS][`TM_ROWS]; + byte unsigned memory[longint unsigned]; + int cycle, read_beats, write_beats, rid, bid, bank_due, bank_ack_due, metadata_due, image_due; + longint unsigned read_address, write_address; + bit b_pending, bank_pending, bank_ack, metadata_pending, image_pending; + bit [127:0] bank_data; + bit complete_pending[3], complete_armed[3]; + int complete_due[3], complete_error[3]; + longint unsigned complete_address[3]; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual tomem_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "ToMem interface is required") + endfunction + function void start_of_simulation_phase(uvm_phase phase); + foreach (bank_memory[b, g, row]) bank_memory[b][g][row] = image.initial_bank(b, g, row); + endfunction + function bit [127:0] word_at(longint unsigned address); + bit [127:0] value; + for (int b = 0; b < 16; b++) + value[b*8+:8] = memory.exists(address + b) ? memory[address+b] : + image.initial_byte(address + b); + return value; + endfunction + task run_phase(uvm_phase phase); + forever begin + @(posedge vif.clock); + cycle++; + if (vif.reset) begin + read_beats = 0; + write_beats = 0; + b_pending = 0; + bank_pending = 0; + bank_ack = 0; + metadata_pending = 0; + image_pending = 0; + foreach (complete_pending[p]) begin + complete_pending[p] = 0; + complete_armed[p] = 1; + end + end else begin + if (`TO(MEMORY_AXI_R_READY) && `TI(MEMORY_AXI_R_VALID)) begin + read_beats--; + read_address += 16; + end + if (`TO(MEMORY_AXI_B_READY) && `TI(MEMORY_AXI_B_VALID)) b_pending = 0; + if (`TO(MEMORY_AXI_AR_VALID) && `TI(MEMORY_AXI_AR_READY)) begin + read_address = `TO(MEMORY_AXI_AR_BITS_ADDR); + read_beats = `TO(MEMORY_AXI_AR_BITS_LEN) + 1; + rid = `TO(MEMORY_AXI_AR_BITS_ID); + end + if (`TO(MEMORY_AXI_AW_VALID) && `TI(MEMORY_AXI_AW_READY)) begin + write_address = `TO(MEMORY_AXI_AW_BITS_ADDR); + write_beats = `TO(MEMORY_AXI_AW_BITS_LEN) + 1; + bid = `TO(MEMORY_AXI_AW_BITS_ID); + end + if (`TO(MEMORY_AXI_W_VALID) && `TI(MEMORY_AXI_W_READY)) begin + for (int b = 0; b < 16; b++) + if (vif.outputs[`TM_OUT_MEMORY_AXI_W_BITS_STRB_OFFSET+b]) + memory[write_address+b] = vif.outputs[`TM_OUT_MEMORY_AXI_W_BITS_DATA_OFFSET+b*8+:8]; + write_address += 16; + write_beats--; + if (write_beats == 0) b_pending = 1; + end + if (`TO(BANKREAD_READ_RESP_READY) && `TI(BANKREAD_READ_RESP_VALID)) bank_pending = 0; + if (`TO(BANKREAD_READ_REQ_VALID) && `TI(BANKREAD_READ_REQ_READY)) begin + bank_data = bank_memory[ + `TO(BANKREAD_BANK_ID) + ][ + `TO(BANKREAD_GROUP_ID) + ][ + `TO(BANKREAD_READ_REQ_BITS_ADDR) + ]; + bank_pending = 1; + bank_due = cycle + 2; + end + if (`TO(BANKWRITE_WRITE_RESP_READY) && `TI(BANKWRITE_WRITE_RESP_VALID)) bank_ack = 0; + if (`TO(BANKWRITE_WRITE_REQ_VALID) && `TI(BANKWRITE_WRITE_REQ_READY)) begin + if (!image.bank_error) + bank_memory[ + `TO(BANKWRITE_BANK_ID) + ][ + `TO(BANKWRITE_GROUP_ID) + ][ + `TO(BANKWRITE_WRITE_REQ_BITS_ADDR) + ] = + `TO(BANKWRITE_WRITE_REQ_BITS_DATA); + bank_ack = 1; + bank_ack_due = cycle + 3; + end + if (`TO(METADATA_RESPONSE_READY) && `TI(METADATA_RESPONSE_VALID)) metadata_pending = 0; + if (`TO(METADATA_REQUEST_VALID) && `TI(METADATA_REQUEST_READY)) begin + metadata_pending = 1; + metadata_due = cycle + 2; + end + if (`TO(IMAGE_COMPLETE_READY) && `TI(IMAGE_COMPLETE_VALID)) image_pending = 0; + if (`TO(IMAGE_TERMINAL_VALID) && `TI(IMAGE_TERMINAL_READY)) begin + image_pending = 1; + image_due = cycle + 3; + end + if (`TO(MEMORY_CONTROL_COMPLETE_0_READY) && `TI(MEMORY_CONTROL_COMPLETE_0_VALID)) + complete_pending[0] = 0; + if (!`TO(MEMORY_CONTROL_FINISHED_0_VALID)) complete_armed[0] = 1; + if (`TO(MEMORY_CONTROL_FINISHED_0_VALID) && complete_armed[0] && !complete_pending[0]) begin + complete_armed[0] = 0; + complete_pending[0] = 1; + complete_due[0] = cycle + 6; + complete_error[0] = image.completion_error != 0 ? image.completion_error : `TO( + MEMORY_CONTROL_FINISHED_0_BITS_ERROR); + complete_address[0] = image.completion_error != 0 ? 64'hbeef : `TO( + MEMORY_CONTROL_FINISHED_0_BITS_ADDRESS); + end + if (`TO(MEMORY_CONTROL_COMPLETE_1_READY) && `TI(MEMORY_CONTROL_COMPLETE_1_VALID)) + complete_pending[1] = 0; + if (!`TO(MEMORY_CONTROL_FINISHED_1_VALID)) complete_armed[1] = 1; + if (`TO(MEMORY_CONTROL_FINISHED_1_VALID) && complete_armed[1] && !complete_pending[1]) begin + complete_armed[1] = 0; + complete_pending[1] = 1; + complete_due[1] = cycle + 6; + complete_error[1] = image.completion_error != 0 ? image.completion_error : `TO( + MEMORY_CONTROL_FINISHED_1_BITS_ERROR); + complete_address[1] = image.completion_error != 0 ? 64'hbeef : `TO( + MEMORY_CONTROL_FINISHED_1_BITS_ADDRESS); + end + if (`TO(MEMORY_CONTROL_COMPLETE_2_READY) && `TI(MEMORY_CONTROL_COMPLETE_2_VALID)) + complete_pending[2] = 0; + if (!`TO(MEMORY_CONTROL_FINISHED_2_VALID)) complete_armed[2] = 1; + if (`TO(MEMORY_CONTROL_FINISHED_2_VALID) && complete_armed[2] && !complete_pending[2]) begin + complete_armed[2] = 0; + complete_pending[2] = 1; + complete_due[2] = cycle + 6; + complete_error[2] = image.completion_error != 0 ? image.completion_error : `TO( + MEMORY_CONTROL_FINISHED_2_BITS_ERROR); + complete_address[2] = image.completion_error != 0 ? 64'hbeef : `TO( + MEMORY_CONTROL_FINISHED_2_BITS_ADDRESS); + end + end + @(negedge vif.clock); + vif.backend = '0; + vif.backend[`TM_IN_QUERYGROUPS_OFFSET+:`TM_IN_QUERYGROUPS_WIDTH] = image.groups[ + `TO(QUERYVBANK) + ]; + vif.backend[`TM_IN_MEMORY_AXI_AR_READY_OFFSET+:`TM_IN_MEMORY_AXI_AR_READY_WIDTH] = read_beats == 0 && cycle % 4 != 0; + vif.backend[`TM_IN_MEMORY_AXI_R_VALID_OFFSET+:`TM_IN_MEMORY_AXI_R_VALID_WIDTH] = read_beats > 0; + vif.backend[`TM_IN_MEMORY_AXI_R_BITS_DATA_OFFSET+:`TM_IN_MEMORY_AXI_R_BITS_DATA_WIDTH] = word_at( + read_address); + vif.backend[`TM_IN_MEMORY_AXI_R_BITS_ID_OFFSET+:`TM_IN_MEMORY_AXI_R_BITS_ID_WIDTH] = rid; + vif.backend[`TM_IN_MEMORY_AXI_R_BITS_RESP_OFFSET+:`TM_IN_MEMORY_AXI_R_BITS_RESP_WIDTH] = image.read_error ? 2 : 0; + vif.backend[`TM_IN_MEMORY_AXI_R_BITS_LAST_OFFSET+:`TM_IN_MEMORY_AXI_R_BITS_LAST_WIDTH] = read_beats == 1; + vif.backend[`TM_IN_MEMORY_AXI_AW_READY_OFFSET+:`TM_IN_MEMORY_AXI_AW_READY_WIDTH] = write_beats == 0 && !b_pending && cycle % 5 != 0; + vif.backend[`TM_IN_MEMORY_AXI_W_READY_OFFSET+:`TM_IN_MEMORY_AXI_W_READY_WIDTH] = write_beats > 0 && cycle % 3 != 0; + vif.backend[`TM_IN_MEMORY_AXI_B_VALID_OFFSET+:`TM_IN_MEMORY_AXI_B_VALID_WIDTH] = b_pending && cycle % 4 != 0; + vif.backend[`TM_IN_MEMORY_AXI_B_BITS_ID_OFFSET+:`TM_IN_MEMORY_AXI_B_BITS_ID_WIDTH] = bid; + vif.backend[`TM_IN_MEMORY_AXI_B_BITS_RESP_OFFSET+:`TM_IN_MEMORY_AXI_B_BITS_RESP_WIDTH] = image.write_error ? 2 : 0; + vif.backend[`TM_IN_BANKREAD_READ_REQ_READY_OFFSET+:`TM_IN_BANKREAD_READ_REQ_READY_WIDTH] = !bank_pending && cycle % 3 != 0; + vif.backend[`TM_IN_BANKREAD_READ_RESP_VALID_OFFSET+:`TM_IN_BANKREAD_READ_RESP_VALID_WIDTH] = bank_pending && cycle >= bank_due; + vif.backend[`TM_IN_BANKREAD_READ_RESP_BITS_DATA_OFFSET+:`TM_IN_BANKREAD_READ_RESP_BITS_DATA_WIDTH] = bank_data; + vif.backend[`TM_IN_BANKWRITE_WRITE_REQ_READY_OFFSET+:`TM_IN_BANKWRITE_WRITE_REQ_READY_WIDTH] = !bank_ack && cycle % 4 != 0; + vif.backend[`TM_IN_BANKWRITE_WRITE_RESP_VALID_OFFSET+:`TM_IN_BANKWRITE_WRITE_RESP_VALID_WIDTH] = bank_ack && cycle >= bank_ack_due; + vif.backend[`TM_IN_BANKWRITE_WRITE_RESP_BITS_OK_OFFSET+:`TM_IN_BANKWRITE_WRITE_RESP_BITS_OK_WIDTH] = !image.bank_error; + vif.backend[`TM_IN_METADATA_REQUEST_READY_OFFSET+:`TM_IN_METADATA_REQUEST_READY_WIDTH] = !metadata_pending && cycle % 3 != 0; + vif.backend[`TM_IN_METADATA_RESPONSE_VALID_OFFSET+:`TM_IN_METADATA_RESPONSE_VALID_WIDTH] = metadata_pending && cycle >= metadata_due; + vif.backend[`TM_IN_METADATA_RESPONSE_BITS_OFFSET+:`TM_IN_METADATA_RESPONSE_BITS_WIDTH] = !image.bank_error; + vif.backend[`TM_IN_IMAGE_START_READY_OFFSET+:`TM_IN_IMAGE_START_READY_WIDTH] = cycle % 4 != 0; + vif.backend[`TM_IN_IMAGE_DATA_READY_OFFSET+:`TM_IN_IMAGE_DATA_READY_WIDTH] = cycle % 5 != 0; + vif.backend[`TM_IN_IMAGE_TERMINAL_READY_OFFSET+:`TM_IN_IMAGE_TERMINAL_READY_WIDTH] = cycle % 3 != 0; + vif.backend[`TM_IN_IMAGE_COMPLETE_VALID_OFFSET+:`TM_IN_IMAGE_COMPLETE_VALID_WIDTH] = image_pending && cycle >= image_due; + vif.backend[`TM_IN_IMAGE_COMPLETE_BITS_FAULT_OFFSET+:`TM_IN_IMAGE_COMPLETE_BITS_FAULT_WIDTH] = image.image_fault; + vif.backend[`TM_IN_IMAGE_COMPLETE_BITS_CAUSE_OFFSET+:`TM_IN_IMAGE_COMPLETE_BITS_CAUSE_WIDTH] = 5; + vif.backend[`TM_IN_IMAGE_COMPLETE_BITS_TVAL_OFFSET+:`TM_IN_IMAGE_COMPLETE_BITS_TVAL_WIDTH] = 'hf00d; + vif.backend[`TM_IN_MEMORY_CONTROL_DECISIONS_0_VALID_OFFSET+:`TM_IN_MEMORY_CONTROL_DECISIONS_0_VALID_WIDTH] = 1; + vif.backend[`TM_IN_MEMORY_CONTROL_DECISIONS_0_PARENTID_OFFSET+:`TM_IN_MEMORY_CONTROL_DECISIONS_0_PARENTID_WIDTH] = 0; + vif.backend[`TM_IN_MEMORY_CONTROL_DECISIONS_0_FAULT_ERROR_OFFSET+:`TM_IN_MEMORY_CONTROL_DECISIONS_0_FAULT_ERROR_WIDTH] = image.admission_error; + vif.backend[`TM_IN_MEMORY_CONTROL_DECISIONS_0_FAULT_ADDRESS_OFFSET+:`TM_IN_MEMORY_CONTROL_DECISIONS_0_FAULT_ADDRESS_WIDTH] = image.admission_error == 0 ? 0 : 64'hdead; + vif.backend[`TM_IN_MEMORY_CONTROL_COMPLETE_0_VALID_OFFSET+:`TM_IN_MEMORY_CONTROL_COMPLETE_0_VALID_WIDTH] = complete_pending[0] && cycle >= complete_due[0]; + vif.backend[`TM_IN_MEMORY_CONTROL_COMPLETE_0_BITS_ERROR_OFFSET+:`TM_IN_MEMORY_CONTROL_COMPLETE_0_BITS_ERROR_WIDTH] = complete_error[0]; + vif.backend[`TM_IN_MEMORY_CONTROL_COMPLETE_0_BITS_ADDRESS_OFFSET+:`TM_IN_MEMORY_CONTROL_COMPLETE_0_BITS_ADDRESS_WIDTH] = complete_address[0]; + vif.backend[`TM_IN_MEMORY_CONTROL_DECISIONS_1_VALID_OFFSET+:`TM_IN_MEMORY_CONTROL_DECISIONS_1_VALID_WIDTH] = 1; + vif.backend[`TM_IN_MEMORY_CONTROL_DECISIONS_1_PARENTID_OFFSET+:`TM_IN_MEMORY_CONTROL_DECISIONS_1_PARENTID_WIDTH] = 1; + vif.backend[`TM_IN_MEMORY_CONTROL_DECISIONS_1_FAULT_ERROR_OFFSET+:`TM_IN_MEMORY_CONTROL_DECISIONS_1_FAULT_ERROR_WIDTH] = image.admission_error; + vif.backend[`TM_IN_MEMORY_CONTROL_DECISIONS_1_FAULT_ADDRESS_OFFSET+:`TM_IN_MEMORY_CONTROL_DECISIONS_1_FAULT_ADDRESS_WIDTH] = image.admission_error == 0 ? 0 : 64'hdead; + vif.backend[`TM_IN_MEMORY_CONTROL_COMPLETE_1_VALID_OFFSET+:`TM_IN_MEMORY_CONTROL_COMPLETE_1_VALID_WIDTH] = complete_pending[1] && cycle >= complete_due[1]; + vif.backend[`TM_IN_MEMORY_CONTROL_COMPLETE_1_BITS_ERROR_OFFSET+:`TM_IN_MEMORY_CONTROL_COMPLETE_1_BITS_ERROR_WIDTH] = complete_error[1]; + vif.backend[`TM_IN_MEMORY_CONTROL_COMPLETE_1_BITS_ADDRESS_OFFSET+:`TM_IN_MEMORY_CONTROL_COMPLETE_1_BITS_ADDRESS_WIDTH] = complete_address[1]; + vif.backend[`TM_IN_MEMORY_CONTROL_DECISIONS_2_VALID_OFFSET+:`TM_IN_MEMORY_CONTROL_DECISIONS_2_VALID_WIDTH] = 1; + vif.backend[`TM_IN_MEMORY_CONTROL_DECISIONS_2_PARENTID_OFFSET+:`TM_IN_MEMORY_CONTROL_DECISIONS_2_PARENTID_WIDTH] = 2; + vif.backend[`TM_IN_MEMORY_CONTROL_DECISIONS_2_FAULT_ERROR_OFFSET+:`TM_IN_MEMORY_CONTROL_DECISIONS_2_FAULT_ERROR_WIDTH] = image.admission_error; + vif.backend[`TM_IN_MEMORY_CONTROL_DECISIONS_2_FAULT_ADDRESS_OFFSET+:`TM_IN_MEMORY_CONTROL_DECISIONS_2_FAULT_ADDRESS_WIDTH] = image.admission_error == 0 ? 0 : 64'hdead; + vif.backend[`TM_IN_MEMORY_CONTROL_COMPLETE_2_VALID_OFFSET+:`TM_IN_MEMORY_CONTROL_COMPLETE_2_VALID_WIDTH] = complete_pending[2] && cycle >= complete_due[2]; + vif.backend[`TM_IN_MEMORY_CONTROL_COMPLETE_2_BITS_ERROR_OFFSET+:`TM_IN_MEMORY_CONTROL_COMPLETE_2_BITS_ERROR_WIDTH] = complete_error[2]; + vif.backend[`TM_IN_MEMORY_CONTROL_COMPLETE_2_BITS_ADDRESS_OFFSET+:`TM_IN_MEMORY_CONTROL_COMPLETE_2_BITS_ADDRESS_WIDTH] = complete_address[2]; + vif.backend[`TM_IN_MEMORY_CONTROL_MAPPINGS_0_HIT_OFFSET+:`TM_IN_MEMORY_CONTROL_MAPPINGS_0_HIT_WIDTH] = 1; + vif.backend[`TM_IN_MEMORY_CONTROL_MAPPINGS_0_PA_OFFSET+:`TM_IN_MEMORY_CONTROL_MAPPINGS_0_PA_WIDTH] = + `TO(MEMORY_CONTROL_QUERIES_0_VA) + + image.physical_offset; + vif.backend[`TM_IN_MEMORY_CONTROL_MAPPINGS_0_ERROR_OFFSET+:`TM_IN_MEMORY_CONTROL_MAPPINGS_0_ERROR_WIDTH] = 0; + vif.backend[`TM_IN_MEMORY_CONTROL_MAPPINGS_1_HIT_OFFSET+:`TM_IN_MEMORY_CONTROL_MAPPINGS_1_HIT_WIDTH] = 1; + vif.backend[`TM_IN_MEMORY_CONTROL_MAPPINGS_1_PA_OFFSET+:`TM_IN_MEMORY_CONTROL_MAPPINGS_1_PA_WIDTH] = + `TO(MEMORY_CONTROL_QUERIES_1_VA) + + image.physical_offset; + vif.backend[`TM_IN_MEMORY_CONTROL_MAPPINGS_1_ERROR_OFFSET+:`TM_IN_MEMORY_CONTROL_MAPPINGS_1_ERROR_WIDTH] = 0; + end + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/env/env.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/env/env.svh new file mode 100644 index 000000000..eeda0417e --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/env/env.svh @@ -0,0 +1,36 @@ +class env extends uvm_env; + `uvm_component_utils(env) + uvm_analysis_imp #(uvm_sequence_item, env) reset_events; + driver source; + monitor mon; + peer backend; + model reference; + uvm_sequencer #(cycle_item) seqr; + result_board #(result_item, 1) scoreboard; + function new(string name, uvm_component parent); + super.new(name, parent); + reset_events = new("reset_events", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + source = driver::type_id::create("source", this); + mon = monitor::type_id::create("monitor", this); + backend = peer::type_id::create("backend", this); + reference = model::type_id::create("model", this); + seqr = new("seqr", this); + scoreboard = result_board#(result_item, 1)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + fixture image = fixture::type_id::create("image"); + reference.image = image; + backend.image = image; + source.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(reference.analysis_export); + reference.expected.connect(scoreboard.expected_export); + reference.actual.connect(scoreboard.actual_export); + reference.cancelled.connect(reset_events); + endfunction + function void write(uvm_sequence_item t); + scoreboard.cancel_pending(); + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/env/model.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/env/model.svh new file mode 100644 index 000000000..90e175cbe --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/env/model.svh @@ -0,0 +1,390 @@ +class model extends uvm_subscriber #(cycle_item); + `uvm_component_utils(model) + fixture image; + uvm_analysis_port #(result_item) expected, actual; + uvm_analysis_port #(uvm_sequence_item) cancelled; + typedef struct { + int tag, bank_id, rows, groups, group_base, start_row, width, pixel, source_width; + longint unsigned base, special; + bit writing, is_2d, is_image, is_mmio, is_config; + int words, index, fault; + longint unsigned fault_address; + } operation; + operation load_op, store_op; + bit load_active, store_active, pending[int]; + int + load_serial, + store_serial, + read_beats, + write_beats, + image_words, + bank_acks, + bank_reads, + metadata_acks; + bit read_addresses[longint unsigned]; + longint unsigned write_address; + byte unsigned memory[longint unsigned], store_bytes[longint unsigned]; + bit [127:0] banks[`TM_BANKS][`TM_GROUPS][`TM_ROWS]; + result_item load_results[$]; + covergroup behavior with function sample ( + int kind, int fault, bit overlap, bit stall, bit reset_busy + ); + option.per_instance = 1; + operations: coverpoint kind {bins all[] = {[0 : 5]};} + faults: coverpoint fault { + bins success = {0}; + bins access_fault = {2}; + bins bank_fault = {5}; + bins shape = {7}; + bins completion = {10}; + } + concurrency: coverpoint overlap; + backpressure: coverpoint stall; + reset_cancellation: coverpoint reset_busy; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected = new("expected", this); + actual = new("actual", this); + cancelled = new("cancelled", this); + behavior = new(); + endfunction + function void start_of_simulation_phase(uvm_phase phase); + foreach (banks[b, g, row]) banks[b][g][row] = image.initial_bank(b, g, row); + endfunction + function void verify(bit ok, string message); + evidence::checks++; + if (!ok) `uvm_fatal("TOMEM", message) + endfunction + function byte unsigned byte_at(longint unsigned address); + return memory.exists(address) ? memory[address] : image.initial_byte(address); + endfunction + function bit [127:0] word_at(longint unsigned address); + bit [127:0] value; + for (int b = 0; b < 16; b++) value[b*8+:8] = byte_at(address + b); + return value; + endfunction + function operation decode(cycle_item t); + operation op; + op.tag = `MI(t, REQUEST_BITS_ROB_ID); + op.bank_id = `MI(t, REQUEST_BITS_CMD_BANK_ID); + op.rows = `MI(t, REQUEST_BITS_CMD_ITER); + op.base = `MI(t, REQUEST_BITS_CMD_MEM_ADDR); + op.special = `MI(t, REQUEST_BITS_CMD_SPECIAL); + op.writing = `MI(t, REQUEST_BITS_CMD_IS_STORE); + op.is_2d = `MI(t, REQUEST_BITS_CMD_IS_MVIN_2D); + op.is_image = `MI(t, REQUEST_BITS_CMD_IS_IMAGE); + op.is_mmio = `MI(t, REQUEST_BITS_CMD_IS_MMIO); + op.is_config = `MI(t, REQUEST_BITS_CMD_IS_CONFIG); + op.width = ((op.special >> 59) & 7) + 1; + op.pixel = ((op.special >> 36) & 127) * 8; + op.source_width = (op.special >> 43) & 1023; + op.group_base=op.special[63]&&!op.is_2d&&!op.is_image&&!op.is_mmio?int'((op.special>>58)&31):0; + op.groups=op.is_image||op.is_mmio||op.is_config?1:(op.special[63]&&!op.is_2d?1:image.groups[op.bank_id]); + op.start_row = op.is_2d ? int'((op.special >> 53) & 63) : 0; + op.words = op.is_image ? (op.rows + 15) / 16 : op.rows * (op.is_2d ? op.width : op.groups); + op.index = 0; + op.fault = 0; + op.fault_address = 0; + if (!op.is_config) begin + if (image.admission_error != 0) begin + op.fault = image.admission_error; + op.fault_address = 'hdead; + end else if (op.rows == 0) begin + op.fault = 7; + op.fault_address = op.base; + end else if (op.writing ? image.write_error : image.read_error) begin + op.fault = 2; + op.fault_address = ((op.base & ~64'd15) + image.physical_offset); + end else if (!op.writing && image.bank_error) begin + op.fault = 5; + op.fault_address = op.base; + end else if (op.is_image && image.image_fault) begin + op.fault = 5; + op.fault_address = 'hf00d; + end + end + return op; + endfunction + function longint unsigned load_address(operation op, int index); + if (op.is_image || op.is_mmio) return op.base + index * 16 + image.physical_offset; + if (op.is_2d) + return op.base+(index/op.width)*op.source_width*op.pixel+(index%op.width)*op.pixel+image.physical_offset; + return op.base+(index/op.groups)*op.groups*((op.special>>39)&19'h7ffff)*16+(index%op.groups)*16+image.physical_offset; + endfunction + function void predict(cycle_item t); + operation op = decode(t); + result_item result; + verify(!pending.exists(op.tag), "Duplicate live ROB tag"); + pending[op.tag] = 1; + if (op.writing) begin + verify(!store_active, "Overlapping stores"); + store_op = op; + store_active = 1; + store_serial++; + if (op.fault == 0 || image.write_error) + for (int row = 0; row < op.rows; row++) + for (int group_index = 0; group_index < op.groups; group_index++) begin + bit [127:0] data = banks[op.bank_id][op.group_base+group_index][row]; + longint unsigned address=op.base+row*op.groups*((op.special>>39)&19'h7ffff)*16+group_index*16+image.physical_offset; + for (int b = 0; b < 16; b++) store_bytes[address+b] = data[b*8+:8]; + end + end else begin + verify(!load_active, "Overlapping loads"); + load_op = op; + load_active = 1; + load_serial++; + image_words = 0; + read_addresses.delete(); + if (!op.is_config && image.admission_error == 0 && op.rows > 0) + for (int i = 0; i < op.words; i++) begin + longint unsigned address = load_address(op, i); + for (int b = 0; b < 16; b++) read_addresses[(address+b)&~64'd15] = 1; + end + if(!op.is_config&&(op.fault==0||image.bank_error||(op.is_image&&image.image_fault)))begin + int words = image.bank_error ? 1 : op.words; + if (op.is_image) begin + for (int i = 0; i < (op.rows + 3) / 4; i++) begin + result = result_item::type_id::create("image_expected"); + result.value = 32'({ + byte_at(op.base + image.physical_offset + i * 4 + 3), + byte_at(op.base + image.physical_offset + i * 4 + 2), + byte_at(op.base + image.physical_offset + i * 4 + 1), + byte_at(op.base + image.physical_offset + i * 4) + }); + result.set_transaction_id((2 << 20) | (load_serial << 10) | i); + expected.write(result); + end + end else + for (int i = 0; i < words; i++) begin + bit [127:0] data = word_at(load_address(op, i)); + int group_index = op.group_base + (op.is_2d ? 0 : i % op.groups); + int row = op.start_row + (op.is_2d ? i : i / op.groups); + if (op.is_2d && op.special[62]) data[127:64] = 0; + result = result_item::type_id::create("load_expected"); + if (op.is_mmio) + result.value = { + 64'(((op.special >> 39) & 17'h1ffff) + i * 16), + data, + 16'((17'd1 << ((op.special >> 56) & 255)) - 1) + }; + else result.value = {32'(op.bank_id), 32'(group_index), 32'(row), data, 16'hffff}; + result.set_transaction_id((1 << 20) | (load_serial << 10) | i); + expected.write(result); + load_results.push_back(result); + end + end + end + result = result_item::type_id::create("completion_expected"); + result.value = { + 32'(op.tag), + 32'(image.completion_error != 0 ? image.completion_error : op.fault), + image.completion_error != 0 ? 64'hbeef : op.fault_address + }; + result.set_transaction_id(op.tag); + expected.write(result); + behavior.sample( + op.is_config ? 5 : op.is_image ? 3 : op.is_mmio ? 4 : op.is_2d ? 2 : op.writing ? 1 : 0, + image.completion_error != 0 ? image.completion_error : op.fault, + load_active && store_active, 0, 0); + endfunction + function void write(cycle_item t); + if (t.reset) begin + behavior.sample(0, 0, 0, 0, load_active || store_active); + cancelled.write(null); + load_active = 0; + store_active = 0; + pending.delete(); + store_bytes.delete(); + load_results.delete(); + read_beats = 0; + write_beats = 0; + bank_acks = 0; + bank_reads = 0; + metadata_acks = 0; + read_addresses.delete(); + return; + end + if (`MI(t, REQUEST_VALID) && `MO(t, REQUEST_READY)) predict(t); + if (`MO(t, MEMORY_AXI_AR_VALID) && `MI(t, MEMORY_AXI_AR_READY)) begin + verify(load_active, "AXI read without a load owner"); + verify(`MO(t, MEMORY_AXI_AR_BITS_SIZE) == 4 && `MO(t, MEMORY_AXI_AR_BITS_BURST) == 1 && + `MO(t, MEMORY_AXI_AR_BITS_ID) == 0, "AXI read attributes mismatch"); + verify(read_beats == 0, "Overlapping AXI read bursts"); + read_beats = `MO(t, MEMORY_AXI_AR_BITS_LEN) + 1; + verify((`MO(t, MEMORY_AXI_AR_BITS_ADDR) & 4095) + read_beats * 16 <= 4096, + "AXI read crossed a page"); + verify((`MO(t, MEMORY_AXI_AR_BITS_ADDR) & 15) == 0, "Unaligned AXI read"); + for (int i = 0; i < read_beats; i++) + verify(read_addresses.exists(`MO(t, MEMORY_AXI_AR_BITS_ADDR) + i * 16), + "AXI read escaped the accepted command footprint"); + end + if (`MI(t, MEMORY_AXI_R_VALID) && `MO(t, MEMORY_AXI_R_READY)) begin + verify(read_beats > 0, "AXI response without a request"); + verify(`MI(t, MEMORY_AXI_R_BITS_LAST) == (read_beats == 1), "RLAST mismatch"); + read_beats--; + end + if (`MO(t, MEMORY_AXI_AW_VALID) && `MI(t, MEMORY_AXI_AW_READY)) begin + verify(store_active && write_beats == 0, "AXI write without a store owner"); + write_address = `MO(t, MEMORY_AXI_AW_BITS_ADDR); + write_beats = `MO(t, MEMORY_AXI_AW_BITS_LEN) + 1; + verify(`MO(t, MEMORY_AXI_AW_BITS_SIZE) == 4 && `MO(t, MEMORY_AXI_AW_BITS_BURST) == 1 && + `MO(t, MEMORY_AXI_AW_BITS_ID) == 1, "AXI write attributes mismatch"); + verify((write_address & 4095) + write_beats * 16 <= 4096, "AXI write crossed a page"); + end + if (`MO(t, MEMORY_AXI_W_VALID) && `MI(t, MEMORY_AXI_W_READY)) begin + verify(write_beats > 0, "Write data without AW"); + verify(`MO(t, MEMORY_AXI_W_BITS_LAST) == (write_beats == 1), "WLAST mismatch"); + for (int b = 0; b < 16; b++) begin + bit expected_strobe = store_bytes.exists(write_address + b); + verify(t.outputs[`TM_OUT_MEMORY_AXI_W_BITS_STRB_OFFSET+b] == expected_strobe, + "Store byte strobe mismatch"); + if (expected_strobe) begin + verify( + t.outputs[`TM_OUT_MEMORY_AXI_W_BITS_DATA_OFFSET+b*8+:8]===store_bytes[write_address+b], + "Store data differs from logical bank contents"); + memory[write_address+b] = store_bytes[write_address+b]; + store_bytes.delete(write_address + b); + end + end + write_address += 16; + write_beats--; + end + if (`MO(t, BANKREAD_READ_REQ_VALID) && `MI(t, BANKREAD_READ_REQ_READY)) begin + int i = store_op.index; + verify(store_active, "Bank read without a store owner"); + verify(`MO(t, BANKREAD_BANK_ID) == store_op.bank_id && `MO(t, BANKREAD_GROUP_ID) + == store_op.group_base + i % store_op.groups && `MO(t, BANKREAD_READ_REQ_BITS_ADDR) + == i / store_op.groups, "Store bank address/group mismatch"); + store_op.index++; + bank_reads++; + end + if (`MO(t, BANKWRITE_WRITE_REQ_VALID) && `MI(t, BANKWRITE_WRITE_REQ_READY)) begin + result_item result = result_item::type_id::create("load_actual"); + result_item plan; + verify(load_active && load_results.size() != 0, "Unexpected bank write"); + plan = load_results.pop_front(); + result.value = { + 32'(`MO(t, BANKWRITE_BANK_ID)), + 32'(`MO(t, BANKWRITE_GROUP_ID)), + 32'(`MO(t, BANKWRITE_WRITE_REQ_BITS_ADDR)), + 128'(`MO(t, BANKWRITE_WRITE_REQ_BITS_DATA)), + 16'hffff + }; + for (int b = 0; b < 16; b++) + verify(t.outputs[`TM_OUT_BANKWRITE_WRITE_REQ_BITS_MASK_0_OFFSET+b], + "Load did not replace the whole bank row"); + result.set_transaction_id(plan.get_transaction_id()); + actual.write(result); + if (!image.bank_error) + banks[ + `MO(t, BANKWRITE_BANK_ID) + ][ + `MO(t, BANKWRITE_GROUP_ID) + ][ + `MO(t, BANKWRITE_WRITE_REQ_BITS_ADDR) + ] = + `MO(t, BANKWRITE_WRITE_REQ_BITS_DATA); + load_op.index++; + bank_acks++; + end + if (`MO(t, METADATA_REQUEST_VALID) && `MI(t, METADATA_REQUEST_READY)) begin + result_item result = result_item::type_id::create("metadata_actual"); + result_item plan; + verify(load_active && load_op.is_mmio && load_results.size() != 0, + "Unexpected metadata write"); + plan = load_results.pop_front(); + result.value = { + 64'(`MO(t, METADATA_REQUEST_BITS_ADDR)), + 128'(`MO(t, METADATA_REQUEST_BITS_DATA)), + 16'(`MO(t, METADATA_REQUEST_BITS_MASK)) + }; + result.set_transaction_id(plan.get_transaction_id()); + actual.write(result); + load_op.index++; + metadata_acks++; + end + if (`MI(t, BANKREAD_READ_RESP_VALID) && `MO(t, BANKREAD_READ_RESP_READY)) begin + verify(bank_reads > 0, "Unexpected bank read response"); + bank_reads--; + end + if (`MI(t, BANKWRITE_WRITE_RESP_VALID) && `MO(t, BANKWRITE_WRITE_RESP_READY)) begin + verify(bank_acks > 0, "Unexpected bank write acknowledgment"); + bank_acks--; + end + if (`MI(t, METADATA_RESPONSE_VALID) && `MO(t, METADATA_RESPONSE_READY)) begin + verify(metadata_acks > 0, "Unexpected metadata acknowledgment"); + metadata_acks--; + end + if (`MO(t, IMAGE_START_VALID) && `MI(t, IMAGE_START_READY)) + verify(load_active && load_op.is_image && `MO(t, IMAGE_START_BITS_BYTES) == load_op.rows && + `MO(t, IMAGE_START_BITS_BUFFER) == load_op.bank_id - 4, + "Image start descriptor mismatch"); + if (`MO(t, IMAGE_TERMINAL_VALID) && `MI(t, IMAGE_TERMINAL_READY)) begin + int terminal_error = image.image_fault && load_op.fault == 5 ? 0 : load_op.fault; + verify(`MO(t, IMAGE_TERMINAL_BITS_ERROR) == terminal_error && + `MO(t, IMAGE_TERMINAL_BITS_ADDRESS) + == (terminal_error == 0 ? 64'd0 : load_op.fault_address), + "Image terminal status mismatch"); + end + if (`MO(t, IMAGE_DATA_VALID) && `MI(t, IMAGE_DATA_READY)) begin + result_item result = result_item::type_id::create("image_actual"); + verify(load_active && load_op.is_image, "Image data without an owner"); + result.value = `MO(t, IMAGE_DATA_BITS); + result.set_transaction_id((2 << 20) | (load_serial << 10) | image_words); + actual.write(result); + image_words++; + end + if (`MO(t, MEMORY_CONTROL_FINISHED_0_VALID)) begin + operation op = load_op; + verify(load_active, "Completion without an owner"); + verify(`MO(t, MEMORY_CONTROL_FINISHED_0_BITS_ERROR) == op.fault && + `MO(t, MEMORY_CONTROL_FINISHED_0_BITS_ADDRESS) == op.fault_address, + "Finished status differs from DMA/bank result"); + verify(load_results.size() == 0 && read_beats == 0 && bank_acks == 0 && metadata_acks == 0, + "Finished before data drained"); + end + if (`MO(t, MEMORY_CONTROL_FINISHED_1_VALID)) begin + operation op = store_op; + verify(store_active, "Completion without an owner"); + verify(`MO(t, MEMORY_CONTROL_FINISHED_1_BITS_ERROR) == op.fault && + `MO(t, MEMORY_CONTROL_FINISHED_1_BITS_ADDRESS) == op.fault_address, + "Finished status differs from DMA/bank result"); + verify(store_bytes.num() == 0 && write_beats == 0 && bank_reads == 0, + "Finished before data drained"); + end + if (`MO(t, MEMORY_CONTROL_FINISHED_2_VALID)) begin + operation op = load_op; + verify(load_active, "Completion without an owner"); + verify(`MO(t, MEMORY_CONTROL_FINISHED_2_BITS_ERROR) == op.fault && + `MO(t, MEMORY_CONTROL_FINISHED_2_BITS_ADDRESS) == op.fault_address, + "Finished status differs from DMA/bank result"); + verify(load_results.size() == 0 && read_beats == 0 && bank_acks == 0 && metadata_acks == 0, + "Finished before data drained"); + end + if (`MO(t, RESPONSE_VALID) && `MI(t, RESPONSE_READY)) begin + result_item result = result_item::type_id::create("completion_actual"); + int tag = `MO(t, RESPONSE_BITS_ROB_ID); + verify(pending.exists(tag), "Unsolicited command completion"); + result.value = { + 32'(tag), 32'(`MO(t, RESPONSE_BITS_FAULT_ERROR)), 64'(`MO(t, RESPONSE_BITS_FAULT_ADDRESS)) + }; + result.set_transaction_id(tag); + actual.write(result); + pending.delete(tag); + if (store_active && store_op.tag == tag) store_active = 0; + else begin + verify(load_active && load_op.tag == tag, "Completion owner mismatch"); + load_active = 0; + end + end + if (`MO(t, RESPONSE_VALID) && !`MI(t, RESPONSE_READY)) + behavior.sample(0, 0, load_active && store_active, 1, 0); + endfunction + function void check_phase(uvm_phase phase); + super.check_phase(phase); + verify( + pending.num()==0&&!load_active&&!store_active&&load_results.size()==0&&store_bytes.num()==0, + "ToMem did not drain"); + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/filelist.f b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/filelist.f new file mode 100644 index 000000000..b8cb9ffa1 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/ToMem/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/interface.sv b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/interface.sv new file mode 100644 index 000000000..b5639322d --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/interface.sv @@ -0,0 +1,9 @@ +`include "parameters.svh" +interface tomem_if ( + input logic clock +); + logic reset = 1; + logic [`TM_INPUT_BITS-1:0] source = '0, backend = '0; + wire [ `TM_INPUT_BITS-1:0] inputs = source | backend; + wire [`TM_OUTPUT_BITS-1:0] outputs; +endinterface diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/package.sv b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/package.sv new file mode 100644 index 000000000..a6d32602f --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/package.sv @@ -0,0 +1,21 @@ +`include "parameters.svh" +`define TI(F) vif.inputs[`TM_IN_``F``_OFFSET+:`TM_IN_``F``_WIDTH] +`define TO(F) vif.outputs[`TM_OUT_``F``_OFFSET+:`TM_OUT_``F``_WIDTH] +`define TB(F) vif.backend[`TM_IN_``F``_OFFSET+:`TM_IN_``F``_WIDTH] +`define TS(F) pins[`TM_IN_``F``_OFFSET+:`TM_IN_``F``_WIDTH] +`define MI(T, F) T.inputs[`TM_IN_``F``_OFFSET+:`TM_IN_``F``_WIDTH] +`define MO(T, F) T.outputs[`TM_OUT_``F``_OFFSET+:`TM_OUT_``F``_WIDTH] +package tomem_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + `include "agent/item.svh" + `include "agent/fixture.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/peer.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..ff6980172 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/seq/sequence.svh @@ -0,0 +1,118 @@ +class scenario_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(scenario_sequence) + model reference; + logic [`TM_INPUT_BITS-1:0] pins; + logic [`TM_OUTPUT_BITS-1:0] observed; + function new(string name = "scenario"); + super.new(name); + endfunction + task tick(bit reset = 0); + cycle_item t = cycle_item::type_id::create("cycle"); + start_item(t); + t.inputs = pins; + t.reset = reset; + finish_item(t); + observed = t.outputs; + endtask + task send(int tag, int kind, longint unsigned address, int rows, int bank_id, + longint unsigned special = (64'd1 << 39)); + pins[`TM_IN_REQUEST_BITS_ROB_ID_OFFSET+:`TM_IN_REQUEST_BITS_ROB_ID_WIDTH] = tag; + pins[`TM_IN_REQUEST_BITS_INSTID_OFFSET+:`TM_IN_REQUEST_BITS_INSTID_WIDTH]=64'h123456789abcdef0+tag; + pins[`TM_IN_REQUEST_BITS_CONTEXT_PRIVILEGE_OFFSET+:`TM_IN_REQUEST_BITS_CONTEXT_PRIVILEGE_WIDTH]=3; + pins[`TM_IN_REQUEST_BITS_CONTEXT_SATP_OFFSET+:`TM_IN_REQUEST_BITS_CONTEXT_SATP_WIDTH]=64'h8000000000000010; + pins[`TM_IN_REQUEST_BITS_CMD_IS_STORE_OFFSET+:`TM_IN_REQUEST_BITS_CMD_IS_STORE_WIDTH]=kind==1; + pins[`TM_IN_REQUEST_BITS_CMD_IS_LOAD_OFFSET+:`TM_IN_REQUEST_BITS_CMD_IS_LOAD_WIDTH] = kind != 1; + pins[`TM_IN_REQUEST_BITS_CMD_IS_MVIN_2D_OFFSET+:`TM_IN_REQUEST_BITS_CMD_IS_MVIN_2D_WIDTH]=kind==2; + pins[`TM_IN_REQUEST_BITS_CMD_IS_IMAGE_OFFSET+:`TM_IN_REQUEST_BITS_CMD_IS_IMAGE_WIDTH]=kind==3; + pins[`TM_IN_REQUEST_BITS_CMD_IS_MMIO_OFFSET+:`TM_IN_REQUEST_BITS_CMD_IS_MMIO_WIDTH] = kind == 4; + pins[`TM_IN_REQUEST_BITS_CMD_IS_CONFIG_OFFSET+:`TM_IN_REQUEST_BITS_CMD_IS_CONFIG_WIDTH]=kind==5; + pins[`TM_IN_REQUEST_BITS_CMD_MEM_ADDR_OFFSET+:`TM_IN_REQUEST_BITS_CMD_MEM_ADDR_WIDTH] = address; + pins[`TM_IN_REQUEST_BITS_CMD_ITER_OFFSET+:`TM_IN_REQUEST_BITS_CMD_ITER_WIDTH] = rows; + pins[`TM_IN_REQUEST_BITS_CMD_BANK_ID_OFFSET+:`TM_IN_REQUEST_BITS_CMD_BANK_ID_WIDTH] = bank_id; + pins[`TM_IN_REQUEST_BITS_CMD_SPECIAL_OFFSET+:`TM_IN_REQUEST_BITS_CMD_SPECIAL_WIDTH] = special; + pins[`TM_IN_REQUEST_VALID_OFFSET+:`TM_IN_REQUEST_VALID_WIDTH] = 1; + do tick(); while (!observed[`TM_OUT_REQUEST_READY_OFFSET]); + pins[`TM_IN_REQUEST_VALID_OFFSET+:`TM_IN_REQUEST_VALID_WIDTH] = 0; + pins[`TM_IN_REQUEST_BITS_CMD_MEM_ADDR_OFFSET+:`TM_IN_REQUEST_BITS_CMD_MEM_ADDR_WIDTH] = '1; + endtask + task completed(int tag); + do tick(); while (reference.pending.exists(tag)); + repeat (3) tick(); + endtask + task body(); + pins = '0; + pins[`TM_IN_RESPONSE_READY_OFFSET+:`TM_IN_RESPONSE_READY_WIDTH] = 1; + repeat (3) tick(1); + tick(); + for (int offset = 0; offset < 16; offset++) begin + send(0, 0, 'h1000 + offset, 2, 0); + completed(0); + send(1, 1, 'h9000 + offset, 2, 1); + completed(1); + send(0, 0, 'h9000 + offset, 2, 0); + completed(0); + end + send(0, 0, 'h1ff3, 3, 0, 64'd3 << 39); + completed(0); + send(1, 1, 'h8ff5, 3, 1, 64'd3 << 39); + completed(1); + send(0, 0, 'h3000, 3, 0, (64'd1 << 63) | (64'd1 << 58) | (64'd1 << 39)); + completed(0); + send(1, 1, 'ha000, 3, 1, (64'd1 << 63) | (64'd1 << 58) | (64'd1 << 39)); + completed(1); + send(0, 2, 'h4003, 2, 0, + (64'd1 << 62) | (64'd2 << 59) | (64'd2 << 53) | (64'd7 << 43) | (64'd1 << 36)); + completed(0); + for (int length = 4; length <= 68; length += 32) begin + send(2, 3, 'h5000, length, 4); + completed(2); + end + send(3, 4, 'h6000, 3, 0, (64'd7 << 56) | (64'd32 << 39)); + completed(3); + send(2, 5, 0, 0, 4); + completed(2); + // Independent read and write producers retain their own ROB tags and data. + send(0, 0, 'h7000, 4, 0); + send(1, 1, 'hb000, 4, 1); + completed(0); + completed(1); + pins[`TM_IN_RESPONSE_READY_OFFSET+:`TM_IN_RESPONSE_READY_WIDTH] = 0; + send(0, 0, 'h7000, 1, 0); + repeat (80) tick(); + pins[`TM_IN_RESPONSE_READY_OFFSET+:`TM_IN_RESPONSE_READY_WIDTH] = 1; + completed(0); + reference.image.read_error = 1; + send(0, 0, 'h8000, 2, 0); + completed(0); + reference.image.read_error = 0; + reference.image.write_error = 1; + send(1, 1, 'hc000, 2, 1); + completed(1); + reference.image.write_error = 0; + reference.image.bank_error = 1; + send(0, 0, 'h8000, 2, 0); + completed(0); + reference.image.bank_error = 0; + reference.image.image_fault = 1; + send(2, 3, 'h5000, 12, 5); + completed(2); + reference.image.image_fault = 0; + reference.image.admission_error = 7; + send(0, 0, 'h8000, 2, 0); + completed(0); + reference.image.admission_error = 0; + reference.image.completion_error = 10; + send(0, 0, 'h8000, 1, 0); + completed(0); + reference.image.completion_error = 0; + send(0, 0, 'h8000, 0, 0); + completed(0); + send(0, 0, 'h8000, 4, 0); + repeat (8) tick(); + repeat (3) tick(1); + tick(); + send(0, 0, 'h8000, 1, 0); + completed(0); + repeat (8) tick(); + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/suite.toml b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/suite.toml new file mode 100644 index 000000000..b3228fee5 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/suite.toml @@ -0,0 +1,19 @@ +[model] +manifest="" +library="" +[[tests]] +name="tomem" +top="tomem_tb" +test="protocol_test" +filelist="filelist.f" +timeout_ns=1000000 +wall_timeout_s=120 +kind="functional" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=["tomem_pkg::model::behavior"] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=[] +assertion="" diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/tb.sv b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/tb.sv new file mode 100644 index 000000000..6e6fd8e9f --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/tb.sv @@ -0,0 +1,15 @@ +module tomem_tb; + import uvm_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + tomem_if bus (clock); + ToMem dut ( + .clock(clock), + .reset(bus.reset), + `include "ports.svh" + ); + initial begin + uvm_config_db#(virtual tomem_if)::set(null, "uvm_test_top.env.*", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/tests/test.svh new file mode 100644 index 000000000..b11c898c9 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/tests/test.svh @@ -0,0 +1,16 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.reference = test_env.reference; + seq.start(test_env.seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/Emit.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/Emit.scala new file mode 100644 index 000000000..435d8501f --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/Emit.scala @@ -0,0 +1,57 @@ +package framework.acc.memdomain.tomem + +import chisel3._ +import chisel3.reflect.DataMirror +import java.nio.file.{Files, Paths} +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.core.memory.access.preflight.configs.PmpParams + +object Emit extends App { + val Array("--config", config, "--output-dir", directory) = args + val values = ujson.read(Files.readString(Paths.get(config))) + val frontend = upickle.default.read[FrontendParam](values("frontend")) + val memory = upickle.default.read[MemDomainParam](values("memory")) + val gp = upickle.default.read[GpDomainParam](values("gp")) + val permission = upickle.default.read[PmpParams](values("permission")) + val transport = upickle.default.read[configs.Params](values("transport")) + + val definitions = scala.collection.mutable.ArrayBuffer[String]( + s"`define TM_BANKS ${memory.virtualBankCount + 2}", + s"`define TM_ROWS ${memory.bankEntries}", + s"`define TM_GROUPS ${memory.bankNum}", + s"`define TM_DATA_BITS ${memory.bankWidth}" + ) + + val binding = scala.collection.mutable.ArrayBuffer[String]() + var inputOffset = 0; var outputOffset = 0 + Files.createDirectories(Paths.get(directory)) + + def leaves(data: Data, path: String): Seq[(String, Data)] = data match { + case record: Record => record.elements.toSeq.flatMap { case (name, field) => + leaves(field, if (path.isEmpty) name else s"${path}_$name") + } + case vector: Vec[_] => vector.toSeq.zipWithIndex.flatMap { case (field, index) => leaves(field, s"${path}_$index") } + case field => if (field.getWidth > 0) Seq(path -> field) else Seq.empty + } + + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + { + val dut = new ToMem(frontend, memory, gp, transport, permission) + for ((name, field) <- leaves(dut.io, "")) { + val input = DataMirror.directionOf(field) == ActualDirection.Input + val direction = if (input) "IN" else "OUT"; val signal = if (input) "inputs" else "outputs"; + val offset = if (input) inputOffset else outputOffset; val width = field.getWidth + definitions += s"`define TM_${direction}_${name.toUpperCase}_OFFSET $offset"; + definitions += s"`define TM_${direction}_${name.toUpperCase}_WIDTH $width" + binding += s".io_$name(bus.$signal[$offset +: $width])"; + if (input) inputOffset += width else outputOffset += width + }; dut + }, + args = Array("--target-dir", s"$directory/ToMem", "--split-verilog") + ) + definitions += s"`define TM_INPUT_BITS $inputOffset"; definitions += s"`define TM_OUTPUT_BITS $outputOffset" + Files.writeString(Paths.get(directory, "parameters.svh"), definitions.mkString("\n") + "\n") + Files.writeString(Paths.get(directory, "ports.svh"), binding.mkString(",\n") + "\n") +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/ToMem.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/ToMem.scala new file mode 100644 index 000000000..864c9e8cf --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/ToMem.scala @@ -0,0 +1,434 @@ +package framework.acc.memdomain.tomem + +import framework.lib.arbiter.{Arbiter} + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.core.memory.access.preflight.configs.PmpParams +import framework.acc.memdomain.tomem.configs.{Params => TransportParam} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.gpdomain.ImagePort +import framework.acc.balldomain.mmio_bank.MetadataWrite +import framework.acc.memdomain.{BankPort, Request, Response} +import framework.core.memory.access.preflight.{MapQuery, MapResult} +import framework.ip.bus.axi4 +import framework.acc.memdomain.toddr._ +import framework.lib.queue.Queue + +class ControlPort(transportParam: TransportParam, permissionParam: PmpParams) extends Bundle { + + val read = Output(Valid(new DmaReadCommand( + transportParam, + permissionParam + ))) + + val write = Output(Valid(new WriteTransfer( + transportParam, + permissionParam + ))) + + val finished = Output(Vec(3, Valid(new DmaStatus))) + val complete = Flipped(Vec(3, Decoupled(new DmaStatus))) + + val decisions = Input(Vec( + 3, + new DmaDecision(transportParam) + )) + + val queries = Output(Vec( + 2, + new MapQuery(transportParam.mapping) + )) + + val mappings = Input(Vec( + 2, + new MapResult(transportParam.mapping) + )) + + val coherence = + new framework.acc.memdomain.tol1.Port(transportParam.interlock) + +} + +class Port(transportParam: TransportParam, permissionParam: PmpParams) extends Bundle { + val control = new ControlPort(transportParam, permissionParam) + + val axi = new axi4.Port(transportParam.axi) + +} + +@instantiable +class ToMem( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val gpParam: GpDomainParam, + val transportParam: TransportParam, + val permissionParam: PmpParams) + extends Module { + val ownerId = frontendParam.ownerId + require( + memParam.bankWidth == 128 && memParam.dma_buswidth == 128 + ) + val rowBytes = memParam.bankWidth / 8 + + @public + val io = IO(new Bundle { + val request = Flipped(Decoupled(new Request(frontendParam, memParam, permissionParam))) + val response = Decoupled(new Response(frontendParam)) + val bankRead = new BankPort(frontendParam, memParam) + val bankWrite = new BankPort(frontendParam, memParam) + val memory = new framework.acc.memdomain.tomem.Port(transportParam, permissionParam) + val metadata = Flipped(new MetadataWrite(memParam)) + val image = if (gpParam.enable) Some(new ImagePort) else None + val queryVbank = Output(UInt(frontendParam.bank_id_len.W)) + val queryShared = Output(Bool()) + val queryGroups = Input(UInt(memParam.groupCountWidth.W)) + }) + + io.bankRead.thread_id := ownerId.U + io.bankWrite.thread_id := ownerId.U + io.bankRead.write.req.valid := false.B + io.bankRead.write.req.bits := 0.U.asTypeOf(io.bankRead.write.req.bits) + io.bankRead.write.resp.ready := true.B + io.bankWrite.read.req.valid := false.B + io.bankWrite.read.req.bits := 0.U.asTypeOf(io.bankWrite.read.req.bits) + io.bankWrite.read.resp.ready := true.B + + val toDDR: Instance[ToDDR] = + Instantiate(new ToDDR(transportParam, permissionParam)) + + val toL1: Instance[framework.acc.memdomain.tol1.ToL1] = + Instantiate( + new framework.acc.memdomain.tol1.ToL1(transportParam.interlock) + ) + + val dma = + Wire(new DmaPort(transportParam, permissionParam)) + toDDR.io.dma <> dma + toDDR.io.decisions := io.memory.control.decisions + io.memory.control.queries := toDDR.io.queries + toDDR.io.mappings := io.memory.control.mappings + io.memory.axi <> toDDR.io.axi + io.memory.control.coherence <> toL1.io + io.memory.control.read.bits := dma.read.bits + io.memory.control.write.valid := dma.write.valid + io.memory.control.write.bits := dma.write.bits + + val idle :: query :: queryWait :: prepare :: Nil = Enum(4) + val setup = RegInit(idle) + val request = Reg(new Request(frontendParam, memParam, permissionParam)) + val readIdle :: readOffer :: readReceive :: readBank :: readBankWait :: readTerminal :: imageWait :: readComplete :: Nil = + Enum(8) + val readState = RegInit(readIdle) + val writeIdle :: writeOffer :: writeStream :: writeWait :: writeDrain :: writeComplete :: Nil = Enum(6) + val writeState = RegInit(writeIdle) + val read = Reg(new Request(frontendParam, memParam, permissionParam)) + val write = Reg(new Request(frontendParam, memParam, permissionParam)) + val readGroups = Reg(UInt(memParam.groupCountWidth.W)) + val writeGroups = Reg(UInt(memParam.groupCountWidth.W)) + val readGroupBase = Reg(UInt(memParam.groupIdWidth.W)) + val writeGroupBase = Reg(UInt(memParam.groupIdWidth.W)) + val readGroup = Reg(UInt(memParam.groupIdWidth.W)) + val writeGroup = Reg(UInt(memParam.groupIdWidth.W)) + val readRow = Reg(UInt(16.W)) + val writeRow = Reg(UInt(16.W)) + val readWords = Reg(UInt(32.W)) + val readData = Reg(UInt(128.W)) + val readLast = Reg(Bool()) + val imageWord = Reg(UInt(2.W)) + val imageBytes = Reg(UInt(32.W)) + val imageStarted = RegInit(false.B) + val writeFault = Reg(new DmaStatus) + val readFault = Reg(new DmaStatus) + val writeCursor = Reg(UInt(64.W)) + val writeFinal = Reg(Bool()) + val readsFinished = RegInit(false.B) + + val wantsWrite = io.request.bits.cmd.is_store + io.request.ready := setup === idle && Mux(wantsWrite, writeState === writeIdle, readState === readIdle) + when(io.request.fire) { request := io.request.bits; setup := query } + io.queryVbank := request.cmd.bank_id + io.queryShared := request.cmd.is_shared + when(setup === query)(setup := queryWait) + when(setup === queryWait)(setup := prepare) + + val selected = request.cmd.special(63) && !request.cmd.is_mvin_2d && !request.cmd.is_image && !request.cmd.is_mmio + val groups = + Mux(request.cmd.is_image || request.cmd.is_config || request.cmd.is_mmio, 1.U, Mux(selected, 1.U, io.queryGroups)) + val base = Mux(selected, request.cmd.special(62, 58), 0.U) + when(setup === prepare) { + setup := idle + when(request.cmd.is_image || request.cmd.is_config) { + assert( + gpParam.enable.B && request.cmd.bank_id >= memParam.virtualBankCount.U && + request.cmd.bank_id < (memParam.virtualBankCount + 2).U, + "Program bank is outside the image slots" + ) + } + when(request.cmd.is_store) { + writeFault := 0.U.asTypeOf(new DmaStatus) + write := request + writeGroups := groups + writeGroupBase := base + writeGroup := 0.U + writeRow := 0.U + writeCursor := request.cmd.mem_addr + + readsFinished := false.B + writeFinal := false.B + writeState := writeOffer + }.otherwise { + read := request + readGroups := groups + readGroupBase := base + readGroup := 0.U + readRow := Mux(request.cmd.is_mvin_2d, request.cmd.special(58, 53), 0.U) + readWords := 0.U + imageWord := 0.U + imageBytes := request.cmd.iter + imageStarted := false.B + readFault := 0.U.asTypeOf(readFault) + readState := Mux(request.cmd.is_config, readComplete, readOffer) + } + when(!request.cmd.is_image && !request.cmd.is_config && !request.cmd.is_mmio) { + assert(groups =/= 0.U, "Memory stream requires an allocated bank") + when(selected)(assert(base < io.queryGroups, "Selected group exceeds allocation")) + } + } + + val imageRead = read.cmd.is_image + + val imageRejected = + if (gpParam.enable) + io.image.get.complete.valid && io.image.get.complete.bits.fault + else false.B + + val width = read.cmd.special(61, 59) +& 1.U + val beats = + Mux(imageRead, (imageBytes.pad(33) + 15.U) >> 4, read.cmd.iter * Mux(read.cmd.is_mvin_2d, width, readGroups)) + val stride = Mux(imageRead || read.cmd.is_mvin_2d || read.cmd.is_mmio, 1.U, read.cmd.special(57, 39)) + + val readOfferReady = + if (gpParam.enable) !imageRead || imageStarted && !imageRejected + else true.B + + dma.read.valid := readState === readOffer && readOfferReady + dma.read.bits.producer := Mux(imageRead, 2.U, 0.U) + dma.read.bits.transfer.vaddr := read.cmd.mem_addr + dma.read.bits.transfer.len := Mux(imageRead, imageBytes, beats << 4) + dma.read.bits.transfer.groups := readGroups + dma.read.bits.transfer.stride := stride + dma.read.bits.transfer.is_2d := read.cmd.is_mvin_2d + dma.read.bits.transfer.pixel_bytes := read.cmd.special(42, 36) << 3 + dma.read.bits.transfer.source_width := read.cmd.special(52, 43) + dma.read.bits.transfer.core_width := width + when(dma.read.fire)(readState := readReceive) + dma.readResult.ready := readState === readReceive + when(dma.readResult.fire) { + readData := dma.readResult.bits.data + readLast := dma.readResult.bits.last + imageWord := 0.U + when(dma.readResult.bits.fault.error =/= DmaError.None.U || readFault.error =/= DmaError.None.U || imageRejected) { + when(dma.readResult.bits.fault.error =/= DmaError.None.U)(readFault := dma.readResult.bits.fault) + when(imageRejected && readFault.error === DmaError.None.U)(readFault.error := DmaError.Cancelled.U) + when(dma.readResult.bits.last)(readState := Mux(imageRead, readTerminal, readComplete)) + }.otherwise(readState := readBank) + } + + io.bankWrite.bank_id := read.cmd.bank_id + io.bankWrite.rob_id := read.rob_id + io.bankWrite.group_id := readGroupBase + readGroup + io.bankWrite.write.req.valid := readState === readBank && !imageRead && !read.cmd.is_mmio + io.bankWrite.write.req.bits.addr := readRow + // MVIN_2D replaces the entire row, including zero-padding unused bytes. + io.bankWrite.write.req.bits.data := + Mux(read.cmd.is_mvin_2d && read.cmd.special(62), Cat(0.U(64.W), readData(63, 0)), readData) + for (i <- 0 until 16) io.bankWrite.write.req.bits.mask(i) := true.B + io.bankWrite.write.resp.ready := readState === readBankWait && !read.cmd.is_mmio + io.metadata.request.valid := readState === readBank && read.cmd.is_mmio + io.metadata.request.bits.addr := read.cmd.special(55, 39) + (readRow << 4) + io.metadata.request.bits.data := readData + io.metadata.request.bits.mask := ((1.U(17.W) << read.cmd.special(63, 56)) - 1.U)(15, 0) + io.metadata.response.ready := readState === readBankWait && read.cmd.is_mmio + when(io.metadata.request.fire) { + assert(memParam.mmioEnable.B && read.cmd.special(63, 56) >= 1.U && read.cmd.special(63, 56) <= 16.U) + assert(read.cmd.special(55, 39) +& (read.cmd.iter << 4) <= memParam.mmioTotalBytes.U) + } + when(io.bankWrite.write.req.fire || io.metadata.request.fire)(readState := readBankWait) + when(io.bankWrite.write.resp.fire || io.metadata.response.fire) { + when(Mux(read.cmd.is_mmio, !io.metadata.response.bits, !io.bankWrite.write.resp.bits.ok)) { + readFault.error := DmaError.Bank.U; readFault.address := read.cmd.mem_addr + } + when(readGroup +& 1.U === readGroups || read.cmd.is_mvin_2d) { + readGroup := 0.U + readRow := readRow + 1.U + }.otherwise(readGroup := readGroup + 1.U) + readState := Mux(readLast, readComplete, readReceive) + } + + io.image.foreach { image => + image.start.valid := readState === readOffer && imageRead && !imageStarted + image.start.bits.buffer := (read.cmd.bank_id - memParam.virtualBankCount.U)( + 0 + ) + image.start.bits.bytes := imageBytes + when(image.start.fire)(imageStarted := true.B) + image.data.valid := readState === readBank && imageRead && !imageRejected + image.data.bits := (readData >> (imageWord << 5))(31, 0) + when(image.data.fire) { + readWords := readWords + 1.U + imageWord := imageWord + 1.U + when((readWords +& 1.U) * 4.U >= imageBytes || imageWord === 3.U) { + readState := Mux(readLast, readTerminal, readReceive) + } + } + when(imageRead && imageStarted && imageRejected) { + when(readState === readOffer) { + readFault.error := DmaError.Cancelled.U + readState := readTerminal + } + when(readState === readBank) { + readFault.error := DmaError.Cancelled.U + readState := Mux(readLast, readTerminal, readReceive) + } + } + image.terminal.valid := readState === readTerminal + image.terminal.bits := readFault + when(image.terminal.fire)(readState := imageWait) + image.complete.ready := readState === imageWait + when(image.complete.fire) { + when(image.complete.bits.fault && (readFault.error === DmaError.None.U || readFault.error === DmaError.Cancelled.U)) { + readFault.error := Mux( + image.complete.bits.cause === 5.U || image.complete.bits.cause === 7.U, + DmaError.Bank.U, + DmaError.Shape.U + ) + readFault.address := image.complete.bits.tval + } + readState := readComplete + } + image.release.valid := setup === prepare && request.cmd.is_config + image.release.bits := (request.cmd.bank_id - memParam.virtualBankCount.U)( + 0 + ) + } + + val tags = Instantiate(new Queue(Bool(), 2, false, true)) + + val data = Instantiate(new Queue( + new WriteData(transportParam), + 2, + true, + false + )) + + // Accepted SRAM reads reserve a slot until their data leaves the staging queue. + val reserved = tags.io.count +& data.io.count + + val totalBytes = write.cmd.iter * writeGroups * rowBytes.U + val writeStride = write.cmd.special(57, 39) + val descriptorBytes = Mux(writeStride === 1.U, totalBytes, writeGroups * rowBytes.U) + dma.write.valid := writeState === writeOffer + dma.write.bits.vaddr := writeCursor + dma.write.bits.len := descriptorBytes + when(dma.write.fire) { + writeFinal := writeStride === 1.U || writeRow +& 1.U === write.cmd.iter + writeState := writeStream + } + val lastGroup = writeGroup +& 1.U === writeGroups + val lastRow = writeRow +& 1.U === write.cmd.iter + val last = lastGroup && (writeStride =/= 1.U || lastRow) + io.bankRead.inst_id := write.instId + io.bankWrite.inst_id := read.instId + io.bankRead.bank_id := write.cmd.bank_id + io.bankRead.rob_id := write.rob_id + io.bankRead.group_id := writeGroupBase + writeGroup + io.bankRead.read.req.valid := writeState === writeStream && !readsFinished && reserved < 2.U && tags.io.enq.ready + io.bankRead.read.req.bits.addr := writeRow + tags.io.enq.valid := io.bankRead.read.req.fire + tags.io.enq.bits := last + io.bankRead.read.resp.ready := data.io.enq.ready && tags.io.deq.valid + data.io.enq.valid := io.bankRead.read.resp.valid && tags.io.deq.valid + data.io.enq.bits.data := io.bankRead.read.resp.bits.data + data.io.enq.bits.last := tags.io.deq.bits + tags.io.deq.ready := io.bankRead.read.resp.fire + when(io.bankRead.read.req.fire) { + when(lastGroup) { writeGroup := 0.U; writeRow := writeRow + 1.U } + .otherwise(writeGroup := writeGroup + 1.U) + when(last)(readsFinished := true.B) + } + dma.writeData.valid := writeState === writeStream && data.io.deq.valid + dma.writeData.bits := data.io.deq.bits + data.io.deq.ready := writeState === writeStream && dma.writeData.ready || writeState === writeDrain + when(dma.writeData.fire && dma.writeData.bits.last)(writeState := writeWait) + dma.writeResult.ready := writeState === writeWait + when(dma.writeResult.fire) { + when(dma.writeResult.bits.fault.error =/= DmaError.None.U) { + writeFault := dma.writeResult.bits.fault + + writeState := writeDrain + }.elsewhen(writeFinal)(writeState := writeComplete) + .otherwise { + writeCursor := writeCursor + writeGroups * writeStride * rowBytes.U + readsFinished := false.B + writeState := writeOffer + } + } + when(writeState === writeDrain && reserved === 0.U)(writeState := writeComplete) + + val readProducer = Mux(imageRead, 2.U, 0.U) + for (i <- 0 until 3) { + io.memory.control.finished(i).valid := (if (i == 1) writeState === writeComplete + else + readState === readComplete && !read.cmd.is_config && readProducer === i.U) + io.memory.control.finished(i).bits := (if (i == 1) writeFault else readFault) + io.memory.control.complete(i).ready := false.B + } + val completion = Instantiate(new Arbiter(new Response(frontendParam), 2)) + completion.io.in(0).valid := readState === readComplete && + (read.cmd.is_config || io.memory.control.complete(readProducer).valid) + completion.io.in(0).bits.rob_id := read.rob_id + completion.io.in(0).bits.fault := Mux(read.cmd.is_config, readFault, io.memory.control.complete(readProducer).bits) + completion.io.in(1).valid := writeState === writeComplete && io.memory.control.complete(1).valid + completion.io.in(1).bits.rob_id := write.rob_id + completion.io.in(1).bits.fault := io.memory.control.complete(1).bits + io.memory.control.complete( + 0 + ).ready := readState === readComplete && !imageRead && !read.cmd.is_config && completion.io.in(0).ready + io.memory.control.complete( + 2 + ).ready := readState === readComplete && imageRead && !read.cmd.is_config && completion.io.in(0).ready + io.memory.control.complete(1).ready := writeState === writeComplete && completion.io.in(1).ready + io.response <> completion.io.out + when(completion.io.in(0).fire)(readState := readIdle) + when(completion.io.in(1).fire)(writeState := writeIdle) + io.bankRead.is_shared := write.cmd.is_shared + io.bankWrite.is_shared := read.cmd.is_shared + + // Publish the read shape before image-loader backpressure can delay DMA. + // The setup stage accepts streams in order; coherence reservation follows that order. + io.memory.control.read.valid := readState === readOffer + val readAccess = dma.read.bits.access + readAccess.context := read.context + readAccess.baseVA := read.cmd.mem_addr + readAccess.rows := Mux(imageRead, 1.U, read.cmd.iter) + readAccess.columns := Mux(read.cmd.is_mvin_2d, width, 1.U) + readAccess.spanBytes := Mux(imageRead, (imageBytes.pad(64) + 15.U) & ~15.U(64.W), readGroups * rowBytes.U) + readAccess.rowStride := Mux( + read.cmd.is_mvin_2d, + read.cmd.special(52, 43) * (read.cmd.special(42, 36) << 3), + readGroups * stride * rowBytes.U + ) + readAccess.columnStride := Mux(read.cmd.is_mvin_2d, read.cmd.special(42, 36) << 3, 0.U) + dma.write.bits.access.context := write.context + dma.write.bits.access.baseVA := write.cmd.mem_addr + dma.write.bits.access.rows := write.cmd.iter + dma.write.bits.access.columnStride := 0.U + dma.write.bits.access.columns := 1.U + dma.write.bits.access.spanBytes := writeGroups * rowBytes.U + dma.write.bits.access.rowStride := writeGroups * writeStride * rowBytes.U +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..5f389790c --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/configs/Params.scala @@ -0,0 +1,42 @@ +package framework.acc.memdomain.tomem.configs + +import buckyball.config.TransportHardware +import scala.jdk.CollectionConverters._ + +import chisel3.util.log2Ceil +import framework.core.memory.access.PhysicalRegion +import framework.core.memory.access.preflight +import framework.ip.bus.{axi4, chi} +import framework.ip.memory.interlock +import upickle.default.{macroRW, ReadWriter} + +case class Params( + tracking: framework.ip.memory.interlock.Params, + prepared: preflight.Params, + axi: axi4.Params, + regions: Seq[PhysicalRegion]) { + val contexts = 1 << log2Ceil(tracking.entries) + val interlock = tracking.copy(entries = contexts) + val mapping = prepared.copy(contexts = contexts) + require(tracking.idBits >= log2Ceil(contexts)) + require(prepared.idBits == tracking.idBits && prepared.maxRanges == tracking.maxRanges) + require(tracking.addressBits == prepared.bus.addressBits && prepared.bus.addressBits == axi.addressBits) + require(axi.dataBits == 128 && prepared.beatBytes == axi.bytes) +} + +object Params { + + def fromProto(p: TransportHardware): Params = Params( + interlock.Params.fromProto(p.getTracking), + preflight.Params.fromProto(p.getPrepared), + axi4.Params.fromProto(p.getAxi), + p.getRegionsList.asScala.map(PhysicalRegion.fromProto).toSeq + ) + + implicit val chiRW: ReadWriter[chi.Params] = macroRW + implicit val axiRW: ReadWriter[axi4.Params] = macroRW + implicit val trackingRW: ReadWriter[interlock.Params] = macroRW + implicit val preparedRW: ReadWriter[preflight.Params] = macroRW + implicit val regionRW: ReadWriter[PhysicalRegion] = macroRW + implicit val rw: ReadWriter[Params] = macroRW +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/build.sc b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/build.sc new file mode 100644 index 000000000..8456bf352 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/build.sc @@ -0,0 +1,25 @@ +import mill._ +import $file.^.^.^.^.^.^.^.^.^.^.^.^.^.^.settings + +val ipRoot: os.Path = millSourcePath + +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.acc.memdomain.toddr.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/DmaPort.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/DmaStatus.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/ReadDma.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/ToDDR.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/WriteDma.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/configs/uvm.toml b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/configs/uvm.toml new file mode 100644 index 000000000..82d7fe90d --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/configs/uvm.toml @@ -0,0 +1,32 @@ +[parameters.permission] +addressBits=44 +entries=16 +[parameters.transport.tracking] +entries=4 +addressBits=44 +idBits=8 +lineBytes=64 +maxRanges=8 +[parameters.transport.prepared] +contexts=4 +maxRanges=8 +idBits=8 +beatBytes=16 +[parameters.transport.prepared.bus] +nodeIdBits=7 +addressBits=44 +dataBits=256 +[parameters.transport.axi] +addressBits=44 +dataBits=128 +idBits=4 +[[parameters.transport.regions]] +base="0" +bytes="17592186044416" +normal=true +cacheable=true +atomic=true +readable=true +writable=true +executable=true +[defines] diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/csrc/Cargo.lock b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/csrc/Cargo.lock new file mode 100644 index 000000000..627bfe6b8 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/csrc/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "dma-reference" +version = "0.1.0" diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/csrc/Cargo.toml b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/csrc/Cargo.toml new file mode 100644 index 000000000..2b1c44751 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/csrc/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "dma-reference" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["cdylib", "staticlib"] diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/csrc/src/lib.rs b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/csrc/src/lib.rs new file mode 100644 index 000000000..fbb816fbf --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/csrc/src/lib.rs @@ -0,0 +1,4 @@ +#[no_mangle] +pub extern "C" fn dma_ref_byte(address: u64) -> u32 { + ((address ^ (address >> 8)) & 255) as u32 +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/env/env.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/env/env.svh new file mode 100644 index 000000000..3227267e4 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/env/env.svh @@ -0,0 +1,44 @@ +class env extends uvm_env; + `uvm_component_utils(env) + virtual dma_if vif; + covergroup behavior with function sample ( + longint unsigned base, + int length, + int error, + bit bus_error, + bit two_dimensional, + bit cancellation + ); + option.per_instance = 1; + alignment: coverpoint (base % 16) {bins aligned = {0}; bins unaligned = {[1 : 15]};} + lengths: coverpoint length { + bins invalid = {0}; bins short_transfer = {[1 : 64]}; bins long_transfer = {[65 : $]}; + } + faults: coverpoint error {bins success = {0}; bins bus = {2}; bins shape = {7};} + backend_errors: coverpoint bus_error; + dimensionality: coverpoint two_dimensional; + reset_cancellation: coverpoint cancellation; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + behavior = new(); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual dma_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "DMA interface is required") + endfunction + int cases = 0, beats = 0, bursts = 0, checks = 0; + function automatic bit [127:0] data_at(longint unsigned address); + bit [127:0] value; + for (int i = 0; i < 16; ++i) value[i*8+:8] = 8'(dma_ref_byte(address + i)); + return value; + endfunction + task automatic verify_contract(bit good, string reason); + verification_pkg::evidence::checks++; + checks++; + if (!good) `uvm_fatal("READ_DMA", reason) + endtask + + `include "seq/scenario.svh" +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/filelist.f b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/filelist.f new file mode 100644 index 000000000..75a203454 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/ReadDma/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/interface.sv b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/interface.sv new file mode 100644 index 000000000..dd4db7cf6 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/interface.sv @@ -0,0 +1,10 @@ +interface dma_if ( + input logic clock +); + logic reset = 1; + `include "read_dma_signals.svh" + `include "read_dma_clocking.svh" + task initialize(); + `include "read_dma_init.svh" + endtask +endinterface diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/package.sv b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/package.sv new file mode 100644 index 000000000..57b04a4cf --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/package.sv @@ -0,0 +1,8 @@ +package dma_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + import "DPI-C" function int unsigned dma_ref_byte(longint unsigned address); + `include "env/env.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/seq/scenario.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/seq/scenario.svh new file mode 100644 index 000000000..052ac0b1c --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/seq/scenario.svh @@ -0,0 +1,162 @@ +task automatic run_case(longint unsigned base, int length, int groups, int stride, bit mode2d = 0, + int pixel = 16, int width = 1, int source = 1, int expected_error = 0, + bit bus_error = 0, int cancel_after = 0); + int count, seen, pending, issued, tick, fault_seen; + longint unsigned read_address, expected_address, query_address; + int query_bytes; + bit [127:0] held_data; + bit held, held_last; + bit [ 3:0] held_error; + bit [63:0] held_address; + count = (length + 15) / 16; + seen = 0; + pending = 0; + issued = 0; + fault_seen = 0; + held = 0; + cases++; + behavior.sample(base, length, expected_error, bus_error, mode2d, cancel_after != 0); + @(negedge vif.clock); + vif.io_req_bits_vaddr = base; + vif.io_req_bits_len = length; + vif.io_req_bits_groups = groups; + vif.io_req_bits_stride = stride; + vif.io_req_bits_is_2d = mode2d; + vif.io_req_bits_pixel_bytes = pixel; + vif.io_req_bits_core_width = width; + vif.io_req_bits_source_width = source; + vif.io_req_valid = 1; + do @(vif.sample); while (!vif.sample.io_req_ready); + @(negedge vif.clock); + vif.io_req_valid = 0; + for (tick = 0; tick < 20000; ++tick) begin + vif.io_mapping_hit = 1; + vif.io_mapping_error = 0; + vif.io_mapping_pa = vif.io_query_va; + vif.io_axi_ar_ready = pending == 0 && tick % 3 != 0; + vif.io_axi_r_valid = pending != 0; + vif.io_axi_r_bits_data = data_at(read_address); + vif.io_axi_r_bits_last = pending == 1; + vif.io_axi_r_bits_resp = bus_error && issued == 1 ? 2 : 0; + vif.io_resp_ready = tick % 5 == 0; + @(vif.sample); + if (held) + verify_contract( + vif.sample.io_resp_valid && vif.sample.io_resp_bits_data == held_data && + vif.sample.io_resp_bits_last == held_last && vif.sample.io_resp_bits_fault_error == held_error && + vif.sample.io_resp_bits_fault_address == held_address, + "response changed under backpressure"); + held = vif.sample.io_resp_valid && !vif.sample.io_resp_ready; + held_data = vif.sample.io_resp_bits_data; + held_last = vif.sample.io_resp_bits_last; + held_error = vif.sample.io_resp_bits_fault_error; + held_address = vif.sample.io_resp_bits_fault_address; + if (vif.sample.io_query_valid) begin + verify_contract( + !vif.sample.io_query_write && vif.sample.io_query_va[3:0] == 0 && + vif.sample.io_query_bytes != 0 && vif.sample.io_query_bytes % 16 == 0, + "invalid prepared footprint"); + query_address = vif.sample.io_mapping_pa; + query_bytes = vif.sample.io_query_bytes; + end + if (vif.sample.io_axi_ar_valid && vif.sample.io_axi_ar_ready) begin + verify_contract( + vif.sample.io_axi_ar_bits_addr == query_address && + (int'(vif.sample.io_axi_ar_bits_len)+1)*16 == query_bytes, + "burst differs from prepared footprint"); + verify_contract(vif.sample.io_axi_ar_bits_size == 4 && vif.sample.io_axi_ar_bits_burst == 1, + "AXI request must be 128-bit INCR"); + verify_contract( + vif.sample.io_axi_ar_bits_addr[3:0] == 0 && + int'(vif.sample.io_axi_ar_bits_addr[11:0]) + (int'(vif.sample.io_axi_ar_bits_len)+1)*16 <= 4096, + "burst alignment/page boundary"); + pending = int'(vif.sample.io_axi_ar_bits_len) + 1; + read_address = vif.sample.io_axi_ar_bits_addr; + bursts++; + end + if (vif.sample.io_axi_r_valid && vif.sample.io_axi_r_ready) begin + pending--; + issued++; + read_address += 16; + end + if (vif.sample.io_resp_valid && vif.sample.io_resp_ready) begin + if (vif.sample.io_resp_bits_fault_error != 0) begin + verify_contract( + vif.sample.io_resp_bits_fault_error == expected_error && vif.sample.io_resp_bits_last, + "wrong terminal fault"); + verify_contract(pending == 0, "fault completed before accepted burst drained"); + fault_seen++; + end else begin + verify_contract(expected_error == 0 || bus_error, "shape failure dispatched data"); + expected_address = mode2d ? base + (seen/width)*source*pixel + (seen%width)*pixel : + base + (seen/groups)*groups*stride*16 + (seen%groups)*16; + verify_contract(vif.sample.io_resp_bits_data == data_at(expected_address), + "data/stride/unaligned assembly mismatch"); + verify_contract(vif.sample.io_resp_bits_addrcounter == seen, "beat index mismatch"); + if (!bus_error) + verify_contract(vif.sample.io_resp_bits_last == (seen + 1 == count), "last mismatch"); + seen++; + beats++; + end + if (vif.sample.io_resp_bits_last) begin + verify_contract(fault_seen == (expected_error != 0), "missing/unexpected terminal error"); + if (!expected_error) verify_contract(seen == count, "incomplete response"); + @(negedge vif.clock); + vif.io_resp_ready = 0; + vif.io_axi_r_valid = 0; + vif.io_axi_ar_ready = 0; + return; + end + end + @(negedge vif.clock); + if (cancel_after != 0 && tick == cancel_after) begin + verify_contract(issued == 0, "vif.reset fixture must precede AXI issue"); + vif.reset = 1; + vif.io_axi_r_valid = 0; + vif.io_axi_ar_ready = 0; + repeat (3) @(vif.sample); + @(negedge vif.clock); + vif.reset = 0; + repeat (4) begin + @(vif.sample); + verify_contract( + !vif.sample.io_resp_valid && !vif.sample.io_axi_ar_valid && !vif.sample.io_busy, + "vif.reset leaked old work"); + end + return; + end + end + `uvm_fatal("READ_DMA", "request timed out") +endtask + +task execute(); + vif.initialize(); + repeat (4) @(vif.sample); + @(negedge vif.clock); + vif.reset = 0; + vif.io_decisionValid = 1; + run_case('h1000, 16, 1, 1); + run_case('h1ff0, 16 * 257, 1, 1); + run_case('h3003, 53, 3, 5); + run_case('h4ff9, 17, 1, 1); + run_case('h6000, 16 * 19, 3, 7); + run_case('h8000, 16 * 64, 63, 524287); + run_case('h8000, 16 * 63, 63, 524287); + run_case('h9000, 16 * 20, 1, 1, 1, 7, 3, 7); + run_case('haff0, 16 * 20, 1, 1, 1, 16, 3, 7); + run_case('hb000, 16 * 17, 1, 1, 1, 1023, 15, 1023); + run_case(64'hfffffffffffffff0, 16, 1, 1, 0, 16, 1, 1, 0); + run_case(64'hfffffffffffffff1, 16, 1, 1, 0, 16, 1, 1, 7); + run_case(64'hfffffff000000000, 16 * 65536, 63, 524287, 0, 16, 1, 1, 7); + run_case(64'hfffffffffff00000, 16 * 65536, 1, 1, 1, 1023, 15, 1023, 7); + run_case('hc000, 0, 1, 1, 0, 16, 1, 1, 7); + run_case('hc000, 16, 0, 1, 0, 16, 1, 1, 7); + run_case('hc000, 16, 1, 1, 1, 0, 3, 3, 7); + run_case('hd000, 64, 1, 1, 0, 16, 1, 1, 2, 1); + run_case('he000, 256, 3, 1, 0, 16, 1, 1, 0, 0, 5); + run_case('hf000, 32, 1, 1); + `uvm_info("READ_DMA", $sformatf("%0d cases %0d checks %0d beats %0d bursts", cases, checks, + beats, bursts), UVM_LOW) + + +endtask diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/tb.sv b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/tb.sv new file mode 100644 index 000000000..e79321835 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/tb.sv @@ -0,0 +1,13 @@ +module read_dma_tb; + import uvm_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + dma_if bus (clock); + ReadDma dut ( + `include "read_dma_binding.svh" + ); + initial begin + uvm_config_db#(virtual dma_if)::set(null, "uvm_test_top.env", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/tests/test.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/tests/test.svh new file mode 100644 index 000000000..398043c00 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/tests/test.svh @@ -0,0 +1,14 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + test_env.execute(); + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/suite.toml b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/suite.toml new file mode 100644 index 000000000..8ada1f1c7 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/suite.toml @@ -0,0 +1,54 @@ +[model] +manifest="src/csrc/Cargo.toml" +library="libdma_reference" + +[[tests]] +functional_groups = ["dma_pkg::env::behavior"] +name = "dma" +top = "dma_tb" +test = "protocol_test" +filelist = "toddr/filelist.f" +timeout_ns = 2000000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +functional_groups = ["dma_pkg::env::behavior"] +name = "read_dma" +top = "read_dma_tb" +test = "protocol_test" +filelist = "read_dma/filelist.f" +timeout_ns = 2000000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +functional_groups = ["dma_pkg::env::behavior"] +name = "write_dma" +top = "write_dma_tb" +test = "protocol_test" +filelist = "write_dma/filelist.f" +timeout_ns = 2000000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/env/env.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/env/env.svh new file mode 100644 index 000000000..b605189a6 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/env/env.svh @@ -0,0 +1,42 @@ +class env extends uvm_env; + `uvm_component_utils(env) + virtual dma_if vif; + covergroup behavior with function sample ( + longint unsigned base, + int length, + int error, + bit bus_error, + bit two_dimensional, + bit cancellation + ); + option.per_instance = 1; + alignment: coverpoint (base % 16) {bins aligned = {0}; bins unaligned = {[1 : 15]};} + lengths: coverpoint length { + bins invalid = {0}; bins short_transfer = {[1 : 64]}; bins long_transfer = {[65 : $]}; + } + faults: coverpoint error {bins success = {0}; bins bus = {2}; bins shape = {7};} + backend_errors: coverpoint bus_error; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + behavior = new(); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual dma_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "DMA interface is required") + endfunction + int checks = 0, cases = 0, bursts = 0; + function automatic void verify_contract(bit good, string reason); + verification_pkg::evidence::checks++; + checks++; + if (!good) `uvm_fatal("WRITE_DMA", reason) + endfunction + function automatic bit [127:0] data_at(longint unsigned address); + bit [127:0] value; + for (int i = 0; i < 16; i++) value[8*i+:8] = 8'(dma_ref_byte(address + i)); + return value; + endfunction + + `include "seq/scenario.svh" +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/filelist.f b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/filelist.f new file mode 100644 index 000000000..b38d44b6f --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/ToDDR/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/interface.sv b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/interface.sv new file mode 100644 index 000000000..a111b8902 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/interface.sv @@ -0,0 +1,10 @@ +interface dma_if ( + input logic clock +); + logic reset = 1; + `include "toddr_signals.svh" + `include "toddr_clocking.svh" + task initialize(); + `include "toddr_init.svh" + endtask +endinterface diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/package.sv b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/package.sv new file mode 100644 index 000000000..57b04a4cf --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/package.sv @@ -0,0 +1,8 @@ +package dma_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + import "DPI-C" function int unsigned dma_ref_byte(longint unsigned address); + `include "env/env.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/seq/scenario.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/seq/scenario.svh new file mode 100644 index 000000000..7677e756f --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/seq/scenario.svh @@ -0,0 +1,186 @@ +task automatic run_case(longint unsigned base, int length, int expected_error = 0, + bit bus_error = 0); + int sent, remaining, accepted_bytes, tick; + longint unsigned write_address; + bit pending_b, held; + bit [127:0] held_data; + bit [15:0] held_mask; + bit held_last; + bit held_response, held_done; + bit [3:0] held_fault; + sent = 0; + remaining = 0; + accepted_bytes = 0; + pending_b = 0; + held = 0; + held_response = 0; + cases++; + behavior.sample(base, length, expected_error, bus_error, 0, 0); + vif.io_dma_write_bits_vaddr = base; + vif.io_dma_write_bits_len = length; + vif.io_dma_write_valid = 1; + do @(vif.sample); while (!vif.sample.io_dma_write_ready); + @(negedge vif.clock); + vif.io_dma_write_valid = 0; + for (tick = 0; tick < 30000; tick++) begin + vif.io_mappings_1_hit = 1; + vif.io_mappings_1_error = 0; + vif.io_mappings_1_pa = vif.io_queries_1_va; + vif.io_dma_writeData_valid = sent < length / 16 && tick % 3 != 0; + vif.io_dma_writeData_bits_data = data_at(base + sent * 16); + vif.io_dma_writeData_bits_last = sent + 1 == length / 16; + vif.io_axi_aw_ready = remaining == 0 && !pending_b && tick % 4 != 0; + vif.io_axi_w_ready = remaining > 0 && tick % 5 != 0; + vif.io_axi_b_valid = pending_b && tick % 3 == 0; + vif.io_axi_b_bits_id = 1; + vif.io_axi_b_bits_resp = bus_error ? 2 : 0; + vif.io_dma_writeResult_ready = tick % 7 == 0; + @(vif.sample); + if (held) + verify_contract( + vif.sample.io_axi_w_valid && vif.sample.io_axi_w_bits_data == held_data && vif.sample.io_axi_w_bits_strb == held_mask && vif.sample.io_axi_w_bits_last == held_last, + "W changed under backpressure"); + held = vif.sample.io_axi_w_valid && !vif.sample.io_axi_w_ready; + held_data = vif.sample.io_axi_w_bits_data; + held_mask = vif.sample.io_axi_w_bits_strb; + held_last = vif.sample.io_axi_w_bits_last; + if (held_response) + verify_contract( + vif.sample.io_dma_writeResult_valid && vif.sample.io_dma_writeResult_bits_done == held_done && vif.sample.io_dma_writeResult_bits_fault_error == held_fault, + "response changed under backpressure"); + held_response = vif.sample.io_dma_writeResult_valid && !vif.sample.io_dma_writeResult_ready; + held_done = vif.sample.io_dma_writeResult_bits_done; + held_fault = vif.sample.io_dma_writeResult_bits_fault_error; + if (vif.sample.io_dma_writeData_valid && vif.sample.io_dma_writeData_ready) sent++; + if (vif.sample.io_axi_aw_valid && vif.sample.io_axi_aw_ready) begin + remaining = int'(vif.sample.io_axi_aw_bits_len) + 1; + write_address = vif.sample.io_axi_aw_bits_addr; + verify_contract( + vif.sample.io_axi_aw_bits_id == 1 && vif.sample.io_axi_aw_bits_size == 4 && vif.sample.io_axi_aw_bits_burst == 1, + "AW format mismatch"); + verify_contract(write_address % 16 == 0 && (write_address % 4096) + remaining * 16 <= 4096, + "AW crosses a page"); + bursts++; + end + if (vif.sample.io_axi_w_valid && vif.sample.io_axi_w_ready) begin + verify_contract(remaining > 0 && vif.sample.io_axi_w_bits_last == (remaining == 1), + "WLAST mismatch"); + for (int i = 0; i < 16; i++) begin + verify_contract( + vif.sample.io_axi_w_bits_strb[i] == (write_address+i >= base && write_address+i < base+length), + "byte mask changed the transfer footprint"); + if (vif.sample.io_axi_w_bits_strb[i]) begin + verify_contract(vif.sample.io_axi_w_bits_data[8*i+:8] == 8'(dma_ref_byte(write_address + i + )), "unaligned write data mismatch"); + accepted_bytes++; + end + end + remaining--; + write_address += 16; + if (remaining == 0) pending_b = 1; + end + if (vif.sample.io_axi_b_valid && vif.sample.io_axi_b_ready) pending_b = 0; + if (vif.sample.io_dma_writeResult_valid && vif.sample.io_dma_writeResult_ready) begin + verify_contract( + vif.sample.io_dma_writeResult_bits_fault_error == expected_error && vif.sample.io_dma_writeResult_bits_done == (expected_error == 0), + "completion status mismatch"); + verify_contract(remaining == 0 && !pending_b, + "completed before accepted AXI traffic drained"); + if (expected_error == 0) + verify_contract(sent == length / 16 && accepted_bytes == length, + "incomplete write transfer"); + @(negedge vif.clock); + vif.io_dma_writeData_valid = 0; + vif.io_axi_aw_ready = 0; + vif.io_axi_w_ready = 0; + vif.io_axi_b_valid = 0; + vif.io_dma_writeResult_ready = 0; + return; + end + @(negedge vif.clock); + end + `uvm_fatal("WRITE_DMA", "request timed out") +endtask +int overlap_cycles = 0; +task automatic run_read(int producer = 2); + int pending, seen; + longint unsigned address; + pending = 0; + seen = 0; + vif.io_decisions_0_valid = 1; + vif.io_decisions_2_valid = 1; + vif.io_dma_read_bits_producer = producer; + vif.io_dma_read_bits_transfer_vaddr = 'hc000; + vif.io_dma_read_bits_transfer_len = 1024; + vif.io_dma_read_bits_transfer_groups = 1; + vif.io_dma_read_bits_transfer_stride = 1; + vif.io_dma_read_valid = 1; + do @(vif.sample); while (!vif.sample.io_dma_read_ready); + @(negedge vif.clock); + vif.io_dma_read_valid = 0; + for (int tick = 0; tick < 10000; tick++) begin + vif.io_mappings_0_hit = 1; + vif.io_mappings_0_error = 0; + vif.io_mappings_0_pa = vif.io_queries_0_va; + vif.io_axi_ar_ready = pending == 0; + vif.io_axi_r_valid = pending != 0; + vif.io_axi_r_bits_data = data_at(address); + vif.io_axi_r_bits_last = pending == 1; + vif.io_dma_readResult_ready = tick % 3 != 0; + @(vif.sample); + if (vif.sample.io_dma_readBusy && vif.sample.io_dma_writeBusy) overlap_cycles++; + if (vif.sample.io_axi_ar_valid && vif.sample.io_axi_ar_ready) begin + pending = int'(vif.sample.io_axi_ar_bits_len) + 1; + address = vif.sample.io_axi_ar_bits_addr; + end + if (vif.sample.io_axi_r_valid && vif.sample.io_axi_r_ready) begin + pending--; + address += 16; + end + if (vif.sample.io_dma_readResult_valid && vif.sample.io_dma_readResult_ready) begin + verify_contract(vif.sample.io_dma_readResult_bits_data == data_at('hc000 + seen * 16 + ) && vif.sample.io_dma_readResult_bits_fault_error == 0, + "concurrent read data mismatch"); + verify_contract( + vif.sample.io_dma_readResult_bits_addrcounter == seen && vif.sample.io_dma_readResult_bits_last == (seen == 63), + "concurrent read ordering mismatch"); + seen++; + if (seen == 64) begin + @(negedge vif.clock); + vif.io_axi_r_valid = 0; + vif.io_axi_ar_ready = 0; + vif.io_dma_readResult_ready = 0; + return; + end + end + @(negedge vif.clock); + end + `uvm_fatal("DMA", "concurrent read timed out") +endtask + +task execute(); + vif.initialize(); + repeat (3) @(vif.sample); + @(negedge vif.clock); + vif.reset = 0; + vif.io_decisions_1_valid = 1; + fork + run_read(); + begin + run_case('h1000, 16); + run_case('h1ff0, 16 * 257); + for (int offset = 1; offset < 16; offset++) run_case('h4ff0 + offset, 16 * 33); + run_case('h9000, 0, 7); + run_case('h9000, 17, 7); + run_case(64'hfffffffffffffff1, 16, 7); + run_case('ha003, 16 * 8, 2, 1); + run_case('hb000, 32); + end + join + run_read(0); + verify_contract(overlap_cycles > 0, "read and write engines did not overlap"); + `uvm_info("WRITE_DMA", $sformatf("%0d cases %0d checks %0d bursts", cases, checks, bursts), + UVM_LOW) + + +endtask diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/tb.sv b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/tb.sv new file mode 100644 index 000000000..5a414bf4e --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/tb.sv @@ -0,0 +1,13 @@ +module dma_tb; + import uvm_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + dma_if bus (clock); + ToDDR dut ( + `include "toddr_binding.svh" + ); + initial begin + uvm_config_db#(virtual dma_if)::set(null, "uvm_test_top.env", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/tests/test.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/tests/test.svh new file mode 100644 index 000000000..398043c00 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/tests/test.svh @@ -0,0 +1,14 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + test_env.execute(); + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/env/env.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/env/env.svh new file mode 100644 index 000000000..b605189a6 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/env/env.svh @@ -0,0 +1,42 @@ +class env extends uvm_env; + `uvm_component_utils(env) + virtual dma_if vif; + covergroup behavior with function sample ( + longint unsigned base, + int length, + int error, + bit bus_error, + bit two_dimensional, + bit cancellation + ); + option.per_instance = 1; + alignment: coverpoint (base % 16) {bins aligned = {0}; bins unaligned = {[1 : 15]};} + lengths: coverpoint length { + bins invalid = {0}; bins short_transfer = {[1 : 64]}; bins long_transfer = {[65 : $]}; + } + faults: coverpoint error {bins success = {0}; bins bus = {2}; bins shape = {7};} + backend_errors: coverpoint bus_error; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + behavior = new(); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual dma_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "DMA interface is required") + endfunction + int checks = 0, cases = 0, bursts = 0; + function automatic void verify_contract(bit good, string reason); + verification_pkg::evidence::checks++; + checks++; + if (!good) `uvm_fatal("WRITE_DMA", reason) + endfunction + function automatic bit [127:0] data_at(longint unsigned address); + bit [127:0] value; + for (int i = 0; i < 16; i++) value[8*i+:8] = 8'(dma_ref_byte(address + i)); + return value; + endfunction + + `include "seq/scenario.svh" +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/filelist.f b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/filelist.f new file mode 100644 index 000000000..60f89a58f --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/WriteDma/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/interface.sv b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/interface.sv new file mode 100644 index 000000000..86ef990fc --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/interface.sv @@ -0,0 +1,10 @@ +interface dma_if ( + input logic clock +); + logic reset = 1; + `include "write_dma_signals.svh" + `include "write_dma_clocking.svh" + task initialize(); + `include "write_dma_init.svh" + endtask +endinterface diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/package.sv b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/package.sv new file mode 100644 index 000000000..57b04a4cf --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/package.sv @@ -0,0 +1,8 @@ +package dma_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + import "DPI-C" function int unsigned dma_ref_byte(longint unsigned address); + `include "env/env.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/seq/scenario.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/seq/scenario.svh new file mode 100644 index 000000000..97e1e61fc --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/seq/scenario.svh @@ -0,0 +1,123 @@ +task automatic run_case(longint unsigned base, int length, int expected_error = 0, + bit bus_error = 0); + int sent, remaining, accepted_bytes, tick; + longint unsigned write_address; + bit pending_b, held; + bit [127:0] held_data; + bit [15:0] held_mask; + bit held_last; + bit held_response, held_done; + bit [3:0] held_fault; + sent = 0; + remaining = 0; + accepted_bytes = 0; + pending_b = 0; + held = 0; + held_response = 0; + cases++; + behavior.sample(base, length, expected_error, bus_error, 0, 0); + vif.io_req_bits_vaddr = base; + vif.io_req_bits_len = length; + vif.io_req_valid = 1; + do @(vif.sample); while (!vif.sample.io_req_ready); + @(negedge vif.clock); + vif.io_req_valid = 0; + for (tick = 0; tick < 30000; tick++) begin + vif.io_mapping_hit = 1; + vif.io_mapping_error = 0; + vif.io_mapping_pa = vif.io_query_va; + vif.io_data_valid = sent < length / 16 && tick % 3 != 0; + vif.io_data_bits_data = data_at(base + sent * 16); + vif.io_data_bits_last = sent + 1 == length / 16; + vif.io_axi_aw_ready = remaining == 0 && !pending_b && tick % 4 != 0; + vif.io_axi_w_ready = remaining > 0 && tick % 5 != 0; + vif.io_axi_b_valid = pending_b && tick % 3 == 0; + vif.io_axi_b_bits_id = 1; + vif.io_axi_b_bits_resp = bus_error ? 2 : 0; + vif.io_resp_ready = tick % 7 == 0; + @(vif.sample); + if (held) + verify_contract( + vif.sample.io_axi_w_valid && vif.sample.io_axi_w_bits_data == held_data && vif.sample.io_axi_w_bits_strb == held_mask && vif.sample.io_axi_w_bits_last == held_last, + "W changed under backpressure"); + held = vif.sample.io_axi_w_valid && !vif.sample.io_axi_w_ready; + held_data = vif.sample.io_axi_w_bits_data; + held_mask = vif.sample.io_axi_w_bits_strb; + held_last = vif.sample.io_axi_w_bits_last; + if (held_response) + verify_contract( + vif.sample.io_resp_valid && vif.sample.io_resp_bits_done == held_done && vif.sample.io_resp_bits_fault_error == held_fault, + "response changed under backpressure"); + held_response = vif.sample.io_resp_valid && !vif.sample.io_resp_ready; + held_done = vif.sample.io_resp_bits_done; + held_fault = vif.sample.io_resp_bits_fault_error; + if (vif.sample.io_data_valid && vif.sample.io_data_ready) sent++; + if (vif.sample.io_axi_aw_valid && vif.sample.io_axi_aw_ready) begin + remaining = int'(vif.sample.io_axi_aw_bits_len) + 1; + write_address = vif.sample.io_axi_aw_bits_addr; + verify_contract( + vif.sample.io_axi_aw_bits_id == 1 && vif.sample.io_axi_aw_bits_size == 4 && vif.sample.io_axi_aw_bits_burst == 1, + "AW format mismatch"); + verify_contract(write_address % 16 == 0 && (write_address % 4096) + remaining * 16 <= 4096, + "AW crosses a page"); + bursts++; + end + if (vif.sample.io_axi_w_valid && vif.sample.io_axi_w_ready) begin + verify_contract(remaining > 0 && vif.sample.io_axi_w_bits_last == (remaining == 1), + "WLAST mismatch"); + for (int i = 0; i < 16; i++) begin + verify_contract( + vif.sample.io_axi_w_bits_strb[i] == (write_address+i >= base && write_address+i < base+length), + "byte mask changed the transfer footprint"); + if (vif.sample.io_axi_w_bits_strb[i]) begin + verify_contract(vif.sample.io_axi_w_bits_data[8*i+:8] == 8'(dma_ref_byte(write_address + i + )), "unaligned write data mismatch"); + accepted_bytes++; + end + end + remaining--; + write_address += 16; + if (remaining == 0) pending_b = 1; + end + if (vif.sample.io_axi_b_valid && vif.sample.io_axi_b_ready) pending_b = 0; + if (vif.sample.io_resp_valid && vif.sample.io_resp_ready) begin + verify_contract( + vif.sample.io_resp_bits_fault_error == expected_error && vif.sample.io_resp_bits_done == (expected_error == 0), + "completion status mismatch"); + verify_contract(remaining == 0 && !pending_b, + "completed before accepted AXI traffic drained"); + if (expected_error == 0) + verify_contract(sent == length / 16 && accepted_bytes == length, + "incomplete write transfer"); + @(negedge vif.clock); + vif.io_data_valid = 0; + vif.io_axi_aw_ready = 0; + vif.io_axi_w_ready = 0; + vif.io_axi_b_valid = 0; + vif.io_resp_ready = 0; + return; + end + @(negedge vif.clock); + end + `uvm_fatal("WRITE_DMA", "request timed out") +endtask + +task execute(); + vif.initialize(); + repeat (3) @(vif.sample); + @(negedge vif.clock); + vif.reset = 0; + vif.io_decisionValid = 1; + run_case('h1000, 16); + run_case('h1ff0, 16 * 257); + for (int offset = 1; offset < 16; offset++) run_case('h4ff0 + offset, 16 * 33); + run_case('h9000, 0, 7); + run_case('h9000, 17, 7); + run_case(64'hfffffffffffffff1, 16, 7); + run_case('ha003, 16 * 8, 2, 1); + run_case('hb000, 32); + `uvm_info("WRITE_DMA", $sformatf("%0d cases %0d checks %0d bursts", cases, checks, bursts), + UVM_LOW) + + +endtask diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/tb.sv b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/tb.sv new file mode 100644 index 000000000..296921171 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/tb.sv @@ -0,0 +1,13 @@ +module write_dma_tb; + import uvm_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + dma_if bus (clock); + WriteDma dut ( + `include "write_dma_binding.svh" + ); + initial begin + uvm_config_db#(virtual dma_if)::set(null, "uvm_test_top.env", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/tests/test.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/tests/test.svh new file mode 100644 index 000000000..398043c00 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/tests/test.svh @@ -0,0 +1,14 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + test_env.execute(); + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/DmaPort.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/DmaPort.scala new file mode 100644 index 000000000..df0229ee3 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/DmaPort.scala @@ -0,0 +1,85 @@ +package framework.acc.memdomain.toddr + +import framework.core.memory.access.preflight.configs.PmpParams + +import framework.acc.memdomain.tomem.configs.{Params => ModuleParam} + +import chisel3._ +import framework.acc.memdomain.MemoryContext +import chisel3.util._ +import framework.core.memory.access.preflight.Params + +class ReadTransfer extends Bundle { + val vaddr = UInt(64.W) + val len = UInt(32.W) + val stride = UInt(19.W) + val groups = UInt(6.W) + val is_2d = Bool() + val pixel_bytes = UInt(10.W) + val source_width = UInt(10.W) + val core_width = UInt(4.W) +} + +class ReadResult(b: ModuleParam) extends Bundle { + val dataWidth = b.axi.dataBits + + val data = UInt(dataWidth.W) + val last = Bool() + val addrcounter = UInt(16.W) + val fault = new DmaStatus +} + +class WriteTransfer(b: ModuleParam, permission: PmpParams) extends Bundle { + val vaddr = UInt(64.W) + val len = UInt(32.W) + val access = new AccessRange(b, permission) +} + +class WriteData(b: ModuleParam) extends Bundle { + val dataWidth = b.axi.dataBits + + val data = UInt(dataWidth.W) + val last = Bool() +} + +class WriteResult extends Bundle { + val done = Bool() + val fault = new DmaStatus +} + +class DmaDecision(b: ModuleParam) extends Bundle { + val p = b.mapping + + val valid = Bool() + val parentId = UInt(p.idBits.W) + val fault = new DmaStatus +} + +class AccessRange(b: ModuleParam, permission: PmpParams) extends Bundle { + val context = new MemoryContext(permission) + val baseVA = UInt(64.W) + val rows = UInt(64.W) + val columns = UInt(64.W) + val spanBytes = UInt(64.W) + val columnStride = UInt(64.W) + val rowStride = UInt(64.W) +} + +class DmaReadCommand(b: ModuleParam, permission: PmpParams) extends Bundle { + val transfer = new ReadTransfer + val producer = UInt(2.W) + val access = new AccessRange(b, permission) +} + +/** Accelerator-side logical transfers. Translation and external-memory transport belong to the system. */ +class DmaPort(b: ModuleParam, permission: PmpParams) extends Bundle { + val dataBits = b.axi.dataBits + + val read = Decoupled(new DmaReadCommand(b, permission)) + val readResult = Flipped(Decoupled(new ReadResult(b))) + val write = Decoupled(new WriteTransfer(b, permission)) + val writeData = Decoupled(new WriteData(b)) + val writeResult = Flipped(Decoupled(new WriteResult)) + val readBusy = Input(Bool()) + val writeBusy = Input(Bool()) +} diff --git a/arch/src/main/scala/framework/memdomain/frontend/mem/dma/DmaStatus.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/DmaStatus.scala similarity index 89% rename from arch/src/main/scala/framework/memdomain/frontend/mem/dma/DmaStatus.scala rename to arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/DmaStatus.scala index d0e84e73d..128740835 100644 --- a/arch/src/main/scala/framework/memdomain/frontend/mem/dma/DmaStatus.scala +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/DmaStatus.scala @@ -1,4 +1,4 @@ -package framework.memdomain.frontend.mem.dma +package framework.acc.memdomain.toddr import chisel3._ diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/Emit.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/Emit.scala new file mode 100644 index 000000000..88967af02 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/Emit.scala @@ -0,0 +1,66 @@ +package framework.acc.memdomain.toddr + +import chisel3._ +import chisel3.reflect.DataMirror +import java.nio.file.{Files, Paths} +import framework.acc.memdomain.tomem.configs.{Params => TransportParam} +import framework.core.memory.access.preflight.configs.PmpParams + +object Emit extends App { + val Array("--config", config, "--output-dir", directory) = args + val values = ujson.read(Files.readString(Paths.get(config))) + val transport = upickle.default.read[TransportParam](values("transport")) + val permission = upickle.default.read[PmpParams](values("permission")) + Files.createDirectories(Paths.get(directory)) + + def leaves(data: Data, prefix: String): Seq[(String, Data)] = data match { + case record: Record => record.elements.toSeq.flatMap { case (name, field) => leaves(field, s"${prefix}_$name") } + case vector: Vec[_] => vector.zipWithIndex.flatMap { case (field, index) => + leaves(field, s"${prefix}_$index") + }.toSeq + case field => if (field.getWidth > 0) Seq(prefix -> field) else Seq.empty + } + + for ((name, top) <- Seq("toddr" -> "ToDDR", "read_dma" -> "ReadDma", "write_dma" -> "WriteDma")) { + var interface: Data = null + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + name match { + case "toddr" => + val m = + new ToDDR(transport, permission); + interface = m.io; m + case "read_dma" => val m = new ReadDma(transport); interface = m.io; m + case "write_dma" => + val m = + new WriteDma(transport, permission); + interface = m.io; m + }, + args = Array("--target-dir", s"$directory/$top", "--split-verilog") + ) + val fields = leaves(interface, "io") + Files.writeString( + Paths.get(s"$directory/${name}_signals.svh"), + fields.map { case (n, d) => s"logic [${d.getWidth - 1}:0] $n;" }.mkString("\n") + "\n" + ) + Files.writeString( + Paths.get(s"$directory/${name}_ports.svh"), + (Seq(".clock(clock)", ".reset(reset)") ++ fields.map { case (n, _) => s".$n($n)" }).mkString(",\n") + "\n" + ) + Files.writeString( + Paths.get(s"$directory/${name}_binding.svh"), + (Seq(".clock(clock)", ".reset(bus.reset)") ++ fields.map { case (n, _) => s".$n(bus.$n)" }).mkString(",\n") + "\n" + ) + Files.writeString( + Paths.get(s"$directory/${name}_init.svh"), + fields.collect { case (n, d) if DataMirror.directionOf(d) == ActualDirection.Input => s"$n = '0;" }.mkString( + "\n" + ) + "\n" + ) + Files.writeString( + Paths.get(s"$directory/${name}_clocking.svh"), + "clocking sample @(posedge clock);\ndefault input #1step;\ninput reset;\n" + fields.map { case (n, _) => + s"input $n;" + }.mkString("\n") + "\nendclocking\n" + ) + } +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/ReadDma.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/ReadDma.scala new file mode 100644 index 000000000..2ec930674 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/ReadDma.scala @@ -0,0 +1,236 @@ +package framework.acc.memdomain.toddr + +import framework.acc.memdomain.tomem.configs.{Params => ModuleParam} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.core.memory.access.preflight.{Error, MapQuery, MapResult} +import framework.ip.bus.axi4 + +@instantiable +class ReadDma(val b: ModuleParam) extends Module { + val p = b + + val prepared = p.mapping + val axiParams = p.axi + + require(axiParams.dataBits == 128) + require(prepared.beatBytes == 16 && prepared.bus.addressBits == axiParams.addressBits) + + @public + val io = IO(new Bundle { + val req = Flipped(Decoupled(new ReadTransfer)) + val resp = + Decoupled(new ReadResult(b)) + val parentId = Input(UInt(prepared.idBits.W)) + val decisionValid = Input(Bool()) + val decisionFault = Input(new DmaStatus) + val busy = Output(Bool()) + + val query = Output(new MapQuery(b.mapping)) + + val mapping = Input(new MapResult(b.mapping)) + + val axi = new axi4.Port(axiParams) + + }) + + val (idle :: shape :: divide :: products :: offset :: check :: plan :: query :: lookup :: answer :: validate :: + offer :: receive :: respond :: drain :: failed :: Nil) = Enum(16) + val state = RegInit(idle) + val command = Reg(new ReadTransfer) + val parent = Reg(UInt(prepared.idBits.W)) + val total = Reg(UInt(17.W)) + val progress = RegInit(0.U(17.W)) + val group = RegInit(0.U(6.W)) + val column = RegInit(0.U(4.W)) + val address = Reg(UInt(64.W)) + val queryVA = Reg(UInt(64.W)) + val physical = Reg(UInt(axiParams.addressBits.W)) + val candidate = Reg(UInt(9.W)) + val remaining = Reg(UInt(9.W)) + val second = RegInit(false.B) + val firstPart = Reg(UInt(128.W)) + val result = Reg(UInt(128.W)) + val fault = RegInit(0.U.asTypeOf(new DmaStatus)) + val mapped = Reg(new MapResult(b.mapping)) + + io.req.ready := state === idle && io.decisionValid + io.busy := state =/= idle + io.resp.valid := state === respond || state === failed + io.resp.bits.data := Mux(state === failed, 0.U, result) + io.resp.bits.last := state === failed || progress +& 1.U === total + io.resp.bits.addrcounter := progress(15, 0) + io.resp.bits.fault := Mux(state === failed, fault, 0.U.asTypeOf(new DmaStatus)) + io.query := 0.U.asTypeOf(new MapQuery(b.mapping)) + // The prepared map answers a query held for three cycles. + io.query.valid := state === query || state === lookup || state === answer + io.query.id := parent + io.query.va := queryVA + io.query.bytes := candidate << 4 + io.query.write := false.B + io.axi.ar.valid := state === offer + io.axi.ar.bits := 0.U.asTypeOf(new axi4.Address(axiParams)) + io.axi.ar.bits.addr := physical + io.axi.ar.bits.len := candidate - 1.U + io.axi.ar.bits.size := 4.U + io.axi.ar.bits.burst := 1.U + io.axi.r.ready := state === receive || state === drain + io.axi.aw.valid := false.B + io.axi.aw.bits := 0.U.asTypeOf(new axi4.Address(axiParams)) + io.axi.w.valid := false.B + io.axi.w.bits := 0.U.asTypeOf(new axi4.WriteData(axiParams)) + io.axi.b.ready := false.B + + def fail(error: UInt, at: UInt): Unit = { + fault.error := error + fault.address := at + state := failed + } + + val count = (io.req.bits.len.pad(33) + 15.U) >> 4 + when(io.req.fire) { + command := io.req.bits; parent := io.parentId + address := io.req.bits.vaddr; progress := 0.U; group := 0.U; column := 0.U; second := false.B + fault := 0.U.asTypeOf(new DmaStatus) + val bad = io.req.bits.len === 0.U || count > 65536.U || + io.req.bits.groups === 0.U || io.req.bits.stride === 0.U || + (io.req.bits.is_2d && (io.req.bits.pixel_bytes === 0.U || io.req.bits.core_width === 0.U || + io.req.bits.source_width < io.req.bits.core_width)) + when(io.decisionFault.error =/= DmaError.None.U) { + fail(io.decisionFault.error, io.decisionFault.address) + }.elsewhen(bad)(fail(DmaError.Shape.U, io.req.bits.vaddr)).otherwise { + total := count(16, 0); state := shape + } + } + + // At most 65536 beats: divide the 16-bit last index without a combinational divider. + val quotient = Reg(UInt(16.W)) + val remainder = Reg(UInt(6.W)) + val divisor = Reg(UInt(6.W)) + val divideLeft = Reg(UInt(5.W)) + val rowStride = Reg(UInt(29.W)) + val columnStride = Reg(UInt(10.W)) + val rowOffset = Reg(UInt(45.W)) + val columnOffset = Reg(UInt(16.W)) + val lastOffset = Reg(UInt(46.W)) + val shapeDivisor = Mux(command.is_2d, command.core_width.pad(6), command.groups) + when(state === shape) { + quotient := (total - 1.U)(15, 0) + remainder := 0.U + divisor := shapeDivisor + divideLeft := 16.U + rowStride := Mux(command.is_2d, command.source_width * command.pixel_bytes, (command.groups * command.stride) << 4) + columnStride := Mux(command.is_2d, command.pixel_bytes, 16.U) + state := Mux(shapeDivisor === 1.U, products, divide) + } + val shiftedRemainder = Cat(remainder, quotient(15)) + val subtractDivisor = shiftedRemainder >= divisor + when(state === divide) { + quotient := Cat(quotient(14, 0), subtractDivisor) + remainder := Mux(subtractDivisor, shiftedRemainder - divisor, shiftedRemainder)(5, 0) + divideLeft := divideLeft - 1.U + when(divideLeft === 1.U)(state := products) + } + when(state === products) { + rowOffset := quotient * rowStride + columnOffset := remainder * columnStride + state := offset + } + when(state === offset) { + lastOffset := rowOffset +& columnOffset +& 15.U + state := check + } + val lastByte = command.vaddr +& lastOffset + when(state === check) { + when(lastByte(64))(fail(DmaError.Shape.U, command.vaddr)).otherwise(state := plan) + } + + val contiguous = Mux( + command.is_2d, + Mux( + command.pixel_bytes =/= 16.U, + 1.U, + Mux(command.source_width === command.core_width, total - progress, command.core_width - column) + ), + Mux(command.stride === 1.U, total - progress, command.groups - group) + ) + + val pageBeats = (4096.U(13.W) - address(11, 0)) >> 4 + val commandBeats = Mux(contiguous < (total - progress), contiguous, total - progress) + val limited = Mux(commandBeats < 256.U, commandBeats, 256.U) + when(state === plan) { + queryVA := address & ~15.U(64.W) + candidate := Mux(address(3, 0) =/= 0.U, 1.U, Mux(limited < pageBeats, limited, pageBeats)) + state := query + } + when(state === query)(state := lookup) + when(state === lookup)(state := answer) + when(state === answer) { mapped := io.mapping; state := validate } + when(state === validate) { + when(!mapped.hit || mapped.error =/= Error.Ok.U) { + val error = MuxLookup(mapped.error, DmaError.AccessFault.U)(Seq( + Error.Shape.U -> DmaError.Shape.U, + Error.Overflow.U -> DmaError.Shape.U, + Error.PageFault.U -> DmaError.PageFault.U, + Error.Capacity.U -> DmaError.Capacity.U, + Error.Context.U -> DmaError.Context.U + )) + fail(error, queryVA) + }.elsewhen(mapped.pa(3, 0) =/= 0.U || (mapped.pa(11, 0) +& (candidate << 4)) > 4096.U) { + fail(DmaError.AccessFault.U, queryVA) + }.otherwise { physical := mapped.pa; state := offer } + } + when(io.axi.ar.fire) { remaining := candidate; state := receive } + when(io.axi.r.fire) { + when(state === drain) { + when(io.axi.r.bits.last)(state := failed) + }.otherwise { + val badId = io.axi.r.bits.id =/= 0.U + val badLast = io.axi.r.bits.last =/= (remaining === 1.U) + val badResponse = io.axi.r.bits.resp =/= 0.U + assert(!badLast, "ReadDma AXI RLAST does not match ARLEN") + when(badId || badLast || badResponse) { + fault.error := Mux(badId || badLast || io.axi.r.bits.resp === 1.U, DmaError.Protocol.U, DmaError.AccessFault.U) + fault.address := physical + state := Mux(badLast || io.axi.r.bits.last, failed, drain) + }.otherwise { + remaining := remaining - 1.U + physical := physical + 16.U + when(address(3, 0) =/= 0.U && !second) { + firstPart := (io.axi.r.bits.data >> (address(3, 0) * 8.U))(127, 0) + second := true.B; queryVA := queryVA + 16.U; candidate := 1.U; state := query + }.otherwise { + result := Mux( + second, + firstPart | (io.axi.r.bits.data << ((16.U - address(3, 0)) * 8.U))(127, 0), + io.axi.r.bits.data + ) + state := respond + } + } + } + } + when(io.resp.fire) { + when(state === failed || progress +& 1.U === total)(state := idle).otherwise { + val next = Mux( + command.is_2d, + Mux( + column +& 1.U === command.core_width, + address + (command.source_width - command.core_width +& 1.U) * command.pixel_bytes, + address + command.pixel_bytes + ), + Mux( + group +& 1.U === command.groups, + address + 16.U + command.groups * (command.stride - 1.U) * 16.U, + address + 16.U + ) + ) + when(command.is_2d)(column := Mux(column +& 1.U === command.core_width, 0.U, column + 1.U)) + .otherwise(group := Mux(group +& 1.U === command.groups, 0.U, group + 1.U)) + address := next; progress := progress + 1.U; second := false.B + state := Mux(remaining =/= 0.U, receive, plan) + } + } +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/ToDDR.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/ToDDR.scala new file mode 100644 index 000000000..4a4df2a9b --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/ToDDR.scala @@ -0,0 +1,85 @@ +package framework.acc.memdomain.toddr + +import framework.core.memory.access.preflight.configs.PmpParams + +import framework.acc.memdomain.tomem.configs.{Params => ModuleParam} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.core.memory.access.preflight.{MapQuery, MapResult} +import framework.ip.bus.axi4 + +@instantiable +class ToDDR(val b: ModuleParam, permission: PmpParams) extends Module { + val p = b + + val prepared = p.mapping + val axiParams = p.axi + + @public + val io = IO(new Bundle { + val dma = + Flipped(new DmaPort(b, permission)) + + // One decision for each physical producer: loader, storer, kernel image. + val decisions = Input(Vec( + 3, + new DmaDecision(b) + )) + + val queries = Output(Vec( + 2, + new MapQuery(b.mapping) + )) + + val mappings = Input(Vec( + 2, + new MapResult(b.mapping) + )) + + val axi = new axi4.Port(axiParams) + + }) + + val reader: Instance[ReadDma] = Instantiate(new ReadDma(b)) + val writer: Instance[WriteDma] = Instantiate(new WriteDma(b, permission)) + val kernel = io.dma.read.bits.producer === 2.U + val readDecision = Mux(kernel, io.decisions(2), io.decisions(0)) + when(io.dma.read.valid) { + assert(io.dma.read.bits.producer === 0.U || kernel, "DMA read has an unknown producer") + } + reader.io.req.valid := io.dma.read.valid + reader.io.req.bits := io.dma.read.bits.transfer + io.dma.read.ready := reader.io.req.ready + io.dma.readResult <> reader.io.resp + reader.io.parentId := readDecision.parentId + reader.io.decisionValid := readDecision.valid + reader.io.decisionFault := readDecision.fault + writer.io.req <> io.dma.write + writer.io.data <> io.dma.writeData + io.dma.writeResult <> writer.io.resp + writer.io.parentId := io.decisions(1).parentId + writer.io.decisionValid := io.decisions(1).valid + writer.io.decisionFault := io.decisions(1).fault + io.dma.readBusy := reader.io.busy + io.dma.writeBusy := writer.io.busy + io.queries(0) := reader.io.query + io.queries(1) := writer.io.query + reader.io.mapping := io.mappings(0) + writer.io.mapping := io.mappings(1) + + // AXI read and write channels retain independent command owners. + io.axi.ar <> reader.io.axi.ar + reader.io.axi.r <> io.axi.r + io.axi.aw <> writer.io.axi.aw + io.axi.w <> writer.io.axi.w + writer.io.axi.b <> io.axi.b + reader.io.axi.aw.ready := false.B + reader.io.axi.w.ready := false.B + reader.io.axi.b.valid := false.B + reader.io.axi.b.bits := 0.U.asTypeOf(new axi4.WriteResponse(axiParams)) + writer.io.axi.ar.ready := false.B + writer.io.axi.r.valid := false.B + writer.io.axi.r.bits := 0.U.asTypeOf(new axi4.ReadData(axiParams)) +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/WriteDma.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/WriteDma.scala new file mode 100644 index 000000000..ca9cd3017 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/WriteDma.scala @@ -0,0 +1,221 @@ +package framework.acc.memdomain.toddr + +import framework.core.memory.access.preflight.configs.PmpParams + +import framework.acc.memdomain.tomem.configs.{Params => ModuleParam} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.core.memory.access.preflight.{Error, MapQuery, MapResult} +import framework.ip.bus.axi4 + +@instantiable +class WriteDma(val b: ModuleParam, permission: PmpParams) extends Module { + val p = b + + val prepared = p.mapping + val axiParams = p.axi + + require(axiParams.dataBits == 128) + require(prepared.beatBytes == 16 && prepared.bus.addressBits == axiParams.addressBits) + + @public + val io = IO(new Bundle { + val req = Flipped(Decoupled(new WriteTransfer(b, permission))) + val data = + Flipped(Decoupled(new WriteData(b))) + val resp = Decoupled(new WriteResult) + val parentId = Input(UInt(prepared.idBits.W)) + val decisionValid = Input(Bool()) + val decisionFault = Input(new DmaStatus) + val busy = Output(Bool()) + + val query = Output(new MapQuery(b.mapping)) + + val mapping = Input(new MapResult(b.mapping)) + + val axi = new axi4.Port(axiParams) + + }) + + val idle :: plan :: query :: lookup :: answer :: validate :: stream :: waitB :: drain :: respond :: Nil = Enum(10) + val state = RegInit(idle) + val command = Reg(new WriteTransfer(b, permission)) + val parent = Reg(UInt(prepared.idBits.W)) + val endVA = Reg(UInt(64.W)) + val cursorVA = Reg(UInt(64.W)) + val inputTotal = Reg(UInt(29.W)) + val consumed = RegInit(0.U(29.W)) + val sourceEnded = RegInit(false.B) + val remaining = Reg(UInt(29.W)) + val firstBeat = RegInit(true.B) + val beats = Reg(UInt(9.W)) + val beatIndex = RegInit(0.U(9.W)) + val awSent = RegInit(false.B) + val wDone = RegInit(false.B) + val queryVA = Reg(UInt(64.W)) + val queryBytes = Reg(UInt(13.W)) + val headOffset = Reg(UInt(4.W)) + val physical = Reg(UInt(axiParams.addressBits.W)) + val carry = RegInit(0.U(128.W)) + val outputValid = RegInit(false.B) + val outputData = Reg(UInt(128.W)) + val outputMask = Reg(UInt(16.W)) + val outputLast = Reg(Bool()) + val fault = RegInit(0.U.asTypeOf(new DmaStatus)) + val mapped = Reg(new MapResult(b.mapping)) + val offset = command.vaddr(3, 0) + val expectedLast = consumed +& 1.U === inputTotal + val outputRoom = !outputValid || (io.axi.w.ready && !outputLast) + val outputIndex = beatIndex + io.axi.w.fire.asUInt + val outputHead = firstBeat && !io.axi.w.fire + + io.req.ready := state === idle && io.decisionValid + io.busy := state =/= idle + io.data.ready := (state === stream && !wDone && outputRoom && fault.error === DmaError.None.U && consumed < inputTotal) || + (state === drain && !sourceEnded && consumed < inputTotal) + io.resp.valid := state === respond + io.resp.bits.done := fault.error === DmaError.None.U + io.resp.bits.fault := fault + io.query := 0.U.asTypeOf(new MapQuery(b.mapping)) + // The prepared map answers a query held for three cycles. + io.query.valid := state === query || state === lookup || state === answer + io.query.id := parent + io.query.va := queryVA + io.query.bytes := queryBytes + io.query.write := true.B + io.axi.aw.valid := state === stream && !awSent + io.axi.aw.bits := 0.U.asTypeOf(new axi4.Address(axiParams)) + io.axi.aw.bits.id := 1.U + io.axi.aw.bits.addr := physical + io.axi.aw.bits.len := beats - 1.U + io.axi.aw.bits.size := 4.U + io.axi.aw.bits.burst := 1.U + io.axi.w.valid := state === stream && outputValid + io.axi.w.bits.data := outputData + io.axi.w.bits.strb := outputMask + io.axi.w.bits.last := outputLast + io.axi.b.ready := state === waitB + io.axi.ar.valid := false.B + io.axi.ar.bits := 0.U.asTypeOf(new axi4.Address(axiParams)) + io.axi.r.ready := false.B + + def terminate(error: UInt, at: UInt): Unit = { + fault.error := error + fault.address := at + state := Mux(sourceEnded || consumed === inputTotal, respond, drain) + } + + when(io.req.fire) { + command := io.req.bits; parent := io.parentId + consumed := 0.U; sourceEnded := false.B; firstBeat := true.B + carry := 0.U; outputValid := false.B; fault := 0.U.asTypeOf(new DmaStatus) + val finalByte = io.req.bits.vaddr +& (io.req.bits.len - 1.U) + val malformed = io.req.bits.len === 0.U || io.req.bits.len(3, 0) =/= 0.U || finalByte(64) + inputTotal := io.req.bits.len >> 4 + remaining := (io.req.bits.len >> 4) +& (io.req.bits.vaddr(3, 0) =/= 0.U).asUInt + cursorVA := io.req.bits.vaddr & ~15.U(64.W) + endVA := finalByte(63, 0) + when(malformed) { + fault.error := DmaError.Shape.U; fault.address := io.req.bits.vaddr; state := respond + }.elsewhen(io.decisionFault.error =/= DmaError.None.U) { + fault := io.decisionFault; state := drain + }.otherwise(state := plan) + } + + when(state === plan) { + val pageBeats = (4096.U(13.W) - cursorVA(11, 0)) >> 4 + val capped = Mux(remaining < 256.U, remaining, 256.U) + val extent = Mux(capped < pageBeats, capped, pageBeats) + val head = Mux(firstBeat, offset, 0.U) + val start = cursorVA + head + val covered = (extent << 4) - head + val commandBytes = endVA - start + 1.U + beats := extent; beatIndex := 0.U; awSent := false.B; wDone := false.B + headOffset := head; queryVA := start + queryBytes := Mux(commandBytes < covered, commandBytes, covered) + state := query + } + when(state === query)(state := lookup) + when(state === lookup)(state := answer) + when(state === answer) { mapped := io.mapping; state := validate } + when(state === validate) { + val base = mapped.pa.pad(axiParams.addressBits + 1) - headOffset + val lastPA = base +& ((beats << 4) - 1.U) + val invalid = !mapped.hit || mapped.error =/= Error.Ok.U || + base(axiParams.addressBits) || base(3, 0) =/= 0.U || + lastPA(lastPA.getWidth - 1, axiParams.addressBits).orR || + (base >> 12) =/= (lastPA >> 12) + when(invalid) { + val error = MuxLookup(mapped.error, DmaError.AccessFault.U)(Seq( + Error.Shape.U -> DmaError.Shape.U, + Error.Overflow.U -> DmaError.Shape.U, + Error.PageFault.U -> DmaError.PageFault.U, + Error.Capacity.U -> DmaError.Capacity.U, + Error.Context.U -> DmaError.Context.U + )) + terminate(error, queryVA) + }.otherwise { physical := base(axiParams.addressBits - 1, 0); state := stream } + } + when(io.axi.aw.fire)(awSent := true.B) + + // An accepted old W can be replaced immediately; refills take priority over clearing valid. + when(io.axi.w.fire)(outputValid := false.B) + when(io.data.fire) { + val properLast = io.data.bits.last === expectedLast + assert(properLast, "WriteDma input LAST does not match command length") + consumed := consumed + 1.U + when(io.data.bits.last || expectedLast)(sourceEnded := true.B) + when(state === stream) { + outputData := Mux( + properLast, + Mux(offset === 0.U, io.data.bits.data, carry | (io.data.bits.data << (offset * 8.U))(127, 0)), + 0.U + ) + outputMask := Mux(properLast, Mux(outputHead, ("hffff".U(16.W) << offset)(15, 0), "hffff".U), 0.U) + outputLast := outputIndex +& 1.U === beats + outputValid := true.B + carry := (io.data.bits.data >> ((16.U - offset) * 8.U))(127, 0) + when(!properLast && fault.error === DmaError.None.U) { + fault.error := DmaError.Protocol.U + fault.address := command.vaddr + (consumed << 4) + } + } + } + when(state === stream && !wDone && outputRoom) { + when(fault.error =/= DmaError.None.U) { + // Complete an already accepted AW even if the producer ended early. + outputData := 0.U; outputMask := 0.U + outputLast := outputIndex +& 1.U === beats; outputValid := true.B + }.elsewhen(consumed === inputTotal) { + outputData := carry + outputMask := ((1.U(17.W) << offset) - 1.U)(15, 0) + outputLast := outputIndex +& 1.U === beats; outputValid := true.B + } + } + when(io.axi.w.fire) { + firstBeat := false.B + cursorVA := cursorVA + 16.U; remaining := remaining - 1.U + beatIndex := beatIndex + 1.U + when(outputLast)(wDone := true.B) + } + when(state === stream && (awSent || io.axi.aw.fire) && + (wDone || (io.axi.w.fire && outputLast)))(state := waitB) + when(io.axi.b.fire) { + val bad = io.axi.b.bits.id =/= 1.U || io.axi.b.bits.resp =/= 0.U + when(bad && fault.error === DmaError.None.U) { + fault.error := Mux( + io.axi.b.bits.id =/= 1.U || io.axi.b.bits.resp === 1.U, + DmaError.Protocol.U, + DmaError.AccessFault.U + ) + fault.address := physical + } + when(bad || fault.error =/= DmaError.None.U) { + state := Mux(sourceEnded || consumed === inputTotal, respond, drain) + }.elsewhen(remaining === 0.U)(state := respond).otherwise(state := plan) + } + when(state === drain && (sourceEnded || consumed === inputTotal))(state := respond) + when(io.resp.fire)(state := idle) +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/build.sc b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/build.sc new file mode 100644 index 000000000..18880442c --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/build.sc @@ -0,0 +1,20 @@ +import mill._ +import $file.^.^.^.^.^.^.^.^.^.^.^.^.^.^.settings +val ipRoot = millSourcePath + +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.acc.memdomain.tol1.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/ToL1.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/configs/uvm.toml b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/configs/uvm.toml new file mode 100644 index 000000000..9896ccd3d --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/configs/uvm.toml @@ -0,0 +1,8 @@ +[parameters] +entries = 4 +addressBits = 44 +idBits = 8 +lineBytes = 64 +maxRanges = 8 + +[defines] diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/src/main/resources/filelist.f b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/src/main/resources/filelist.f new file mode 100644 index 000000000..4d52f5094 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/src/main/resources/filelist.f @@ -0,0 +1,9 @@ ++incdir+@RTL@ ++incdir+@VERIFY@/uvm/src/protocol/interlock ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/protocol/interlock/types.svh +@VERIFY@/uvm/src/protocol/interlock/interface.sv +@VERIFY@/uvm/src/protocol/interlock/package.sv +-F @RTL@/ToL1/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/src/main/resources/suite.toml b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/src/main/resources/suite.toml new file mode 100644 index 000000000..cdf810147 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/src/main/resources/suite.toml @@ -0,0 +1,343 @@ +[model] +manifest = "../../../../../../../../../../../../../../../verify/uvm/src/protocol/interlock/model/Cargo.toml" +library = "libinterlock_reference" + +[[tests]] +functional_groups = ["interlock_pkg::ownership_model::activity", "interlock_pkg::ownership_model::maintenance_kind", "interlock_pkg::ownership_model::probes"] +name = "tol1" +top = "tol1_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +functional_groups = [] +name = "tol1_bad_cancel_info" +top = "tol1_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=cancel_info"] +assertion = "Interlock cancel conflicts with the same command" + +[[tests]] +functional_groups = [] +name = "tol1_bad_cancel_sealed" +top = "tol1_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=cancel_sealed"] +assertion = "Interlock cancel requires an unsealed reservation" + +[[tests]] +functional_groups = [] +name = "tol1_bad_cancel_unknown" +top = "tol1_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=cancel_unknown"] +assertion = "Interlock cancel ID has no reservation" + +[[tests]] +functional_groups = [] +name = "tol1_bad_capacity" +top = "tol1_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=capacity"] +assertion = "Interlock range list must seal within capacity" + +[[tests]] +functional_groups = [] +name = "tol1_bad_cpu_alignment" +top = "tol1_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=cpu_alignment"] +assertion = "Interlock CPU access must be naturally aligned" + +[[tests]] +functional_groups = [] +name = "tol1_bad_cpu_size" +top = "tol1_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=cpu_size"] +assertion = "Interlock CPU size must be 1, 2, 4 or 8 bytes" + +[[tests]] +functional_groups = [] +name = "tol1_bad_direction" +top = "tol1_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=direction"] +assertion = "Interlock ranges must have one access direction" + +[[tests]] +functional_groups = [] +name = "tol1_bad_dma_failed" +top = "tol1_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=dma_failed"] +assertion = "Interlock DMA failed" + +[[tests]] +functional_groups = [] +name = "tol1_bad_dma_ungranted" +top = "tol1_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=dma_ungranted"] +assertion = "Interlock DMA done requires a granted access" + +[[tests]] +functional_groups = [] +name = "tol1_bad_dma_unknown" +top = "tol1_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=dma_unknown"] +assertion = "Interlock DMA done tag has no reservation" + +[[tests]] +functional_groups = [] +name = "tol1_bad_duplicate" +top = "tol1_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=duplicate"] +assertion = "Interlock dispatch ID is already live" + +[[tests]] +functional_groups = [] +name = "tol1_bad_empty_range" +top = "tol1_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=empty_range"] +assertion = "Interlock memory range is empty" + +[[tests]] +functional_groups = [] +name = "tol1_bad_info_duplicate" +top = "tol1_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=info_duplicate"] +assertion = "Interlock access info is already declared" + +[[tests]] +functional_groups = [] +name = "tol1_bad_info_unknown" +top = "tol1_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=info_unknown"] +assertion = "Interlock access info ID has no reservation" + +[[tests]] +functional_groups = [] +name = "tol1_bad_maint_failed" +top = "tol1_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=maint_failed"] +assertion = "Interlock maintenance failed" + +[[tests]] +functional_groups = [] +name = "tol1_bad_maint_tag" +top = "tol1_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=maint_tag"] +assertion = "Interlock maintenance response tag mismatch" + +[[tests]] +functional_groups = [] +name = "tol1_bad_maint_unknown" +top = "tol1_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=maint_unknown"] +assertion = "Interlock maintenance response has no request" + +[[tests]] +functional_groups = [] +name = "tol1_bad_no_memory_seal" +top = "tol1_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=no_memory_seal"] +assertion = "Interlock no-memory declaration must seal an empty command" + +[[tests]] +functional_groups = [] +name = "tol1_bad_overflow_range" +top = "tol1_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=overflow_range"] +assertion = "Interlock memory range exceeds physical width" diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/src/main/resources/tb.sv b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/src/main/resources/tb.sv new file mode 100644 index 000000000..261fc3269 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/src/main/resources/tb.sv @@ -0,0 +1,16 @@ +module tol1_tb; + import uvm_pkg::*; + import interlock_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + interlock_if bus (clock); + ToL1 dut ( + .clock(clock), + .reset(bus.reset), + `include "ports.svh" + ); + initial begin + uvm_config_db#(virtual interlock_if)::set(null, "uvm_test_top.*", "vif", bus); + run_test(); + end +endmodule diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/src/main/scala/Emit.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/src/main/scala/Emit.scala new file mode 100644 index 000000000..8fc571848 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/src/main/scala/Emit.scala @@ -0,0 +1,72 @@ +package framework.acc.memdomain.tol1 + +import framework.ip.memory.interlock._ + +import chisel3._ +import java.nio.file.{Files, Paths} + +object Emit extends App { + val Array("--config", config, "--output-dir", output) = args + val values = ujson.read(Files.readString(Paths.get(config))) + + val p = Params( + values("entries").num.toInt, + values("addressBits").num.toInt, + values("idBits").num.toInt, + values("lineBytes").num.toInt, + values("maxRanges").num.toInt + ) + + Files.createDirectories(Paths.get(output)) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new ToL1(p), + args = Array("--target-dir", s"$output/ToL1", "--split-verilog") + ) + + val definitions = scala.collection.mutable.ArrayBuffer[String]( + s"`define INTERLOCK_ENTRIES ${p.entries}", + s"`define INTERLOCK_ADDRESS_BITS ${p.addressBits}", + s"`define INTERLOCK_ID_BITS ${p.idBits}", + s"`define INTERLOCK_LINE_BYTES ${p.lineBytes}", + s"`define INTERLOCK_MAX_RANGES ${p.maxRanges}" + ) + + val ports = scala.collection.mutable.ArrayBuffer[String]() + + def channel( + kind: String, + prefix: String, + signal: String, + data: Bundle, + decoupled: Boolean = true + ): Unit = { + var offset = 0 + for ((name, field) <- data.elements.toSeq) { + val width = field.getWidth + definitions += s"`define INTERLOCK_${kind}_${name.toUpperCase}_OFFSET $offset" + definitions += s"`define INTERLOCK_${kind}_${name.toUpperCase}_WIDTH $width" + ports += s".$prefix$name(bus.signals.${signal}_bits[$offset +: $width])" + offset += width + } + definitions += s"`define INTERLOCK_${kind}_WIDTH $offset" + if (decoupled) { + val channelPrefix = prefix.stripSuffix("bits_") + ports += s".${channelPrefix}valid(bus.signals.${signal}_valid)" + ports += s".${channelPrefix}ready(bus.signals.${signal}_ready)" + } + } + + channel("DISPATCH", "io_dispatch_bits_", "dispatch", new Dispatch(p)) + channel("CANCEL", "io_cancel_bits_", "cancel", new Dispatch(p)) + channel("ACCESS_INFO", "io_accessInfo_bits_", "access_info", new AccessInfo(p)) + channel("MAINTENANCE", "io_maintenance_bits_", "maintenance", new Maintenance(p)) + channel("MAINTAINED", "io_maintained_bits_", "maintained", new Acknowledgement(p)) + channel("GRANT", "io_grant_bits_", "grant", new Tag(p)) + channel("DONE", "io_done_bits_", "done", new Acknowledgement(p)) + channel("COMPLETE", "io_complete_bits_", "complete", new Tag(p)) + channel("CPU_QUERY", "io_cpuQuery_", "cpu_query", new CpuQuery(p), decoupled = false) + ports += ".io_cpuAllow(bus.signals.cpu_allow)" + ports += ".io_cpuProbeAllow(bus.signals.cpu_probe_allow)" + Files.write(Paths.get(output, "parameters.svh"), (definitions.mkString("\n") + "\n").getBytes) + Files.write(Paths.get(output, "ports.svh"), ports.mkString(",\n").getBytes) +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/src/main/scala/ToL1.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/src/main/scala/ToL1.scala new file mode 100644 index 000000000..436d4687d --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/src/main/scala/ToL1.scala @@ -0,0 +1,52 @@ +package framework.acc.memdomain.tol1 + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.ip.memory.interlock.{ + AccessInfo, + Acknowledgement, + CpuQuery, + Dispatch, + Interlock, + Maintenance, + Params, + Tag +} + +class Port(val p: Params) extends Bundle { + + val dispatch = Flipped(Decoupled(new Dispatch(p))) + val cancel = Flipped(Decoupled(new Dispatch(p))) + val accessInfo = Flipped(Decoupled(new AccessInfo(p))) + val cpuQuery = Input(new CpuQuery(p)) + val cpuAllow = Output(Bool()) + val cpuProbeAllow = Output(Bool()) + val maintenance = Decoupled(new Maintenance(p)) + val maintained = + Flipped(Decoupled(new Acknowledgement(p))) + val grant = Decoupled(new Tag(p)) + val done = Flipped(Decoupled(new Acknowledgement(p))) + val complete = Decoupled(new Tag(p)) +} + +@instantiable +class ToL1(val p: Params) extends Module { + + @public + val io = IO(new Port(p)) + + val interlock: Instance[Interlock] = Instantiate(new Interlock(p)) + + interlock.io.dispatch <> io.dispatch + interlock.io.cancel <> io.cancel + interlock.io.accessInfo <> io.accessInfo + interlock.io.cpuQuery := io.cpuQuery + io.cpuAllow := interlock.io.cpuAllow + io.cpuProbeAllow := interlock.io.cpuProbeAllow + io.maintenance <> interlock.io.maintenance + interlock.io.maintained <> io.maintained + io.grant <> interlock.io.grant + interlock.io.done <> io.done + io.complete <> interlock.io.complete +} diff --git a/arch/src/main/scala/framework/balldomain/BallDomain.scala b/arch/src/main/scala/framework/balldomain/BallDomain.scala deleted file mode 100644 index 88f936054..000000000 --- a/arch/src/main/scala/framework/balldomain/BallDomain.scala +++ /dev/null @@ -1,125 +0,0 @@ -package framework.balldomain - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.top.GlobalConfig -import framework.balldomain.bbus.BBus -import framework.balldomain.decoder.BallDomainDecoder -import framework.balldomain.rs.BallReservationStation -import framework.balldomain.blink.{BankRead, BankWrite, SubRobRow} -import framework.balldomain.blink.mmio.{MmioRead, MmioWrite} -import framework.frontend.globalrs.{GlobalSchedComplete, GlobalSchedIssue} -import framework.rvv.RvvBallCommand -import framework.frontend.decoder.{DomainId, PostGDCmd} - -/** - * Ball domain. - * - * BBus reflectively constructs ball instances from `b.ballDomain.ballIdMappings`; - * `BallDomainDecoder` reads `b.ballDomain.ballISA` to route funct7 → bid. - * Both are pure data-driven, so no example needs its own subclass. - */ -@instantiable -class BallDomain(val b: GlobalConfig) extends Module { - val totalBallRead = b.ballDomain.ballIdMappings.map(_.inBW).sum - val totalBallWrite = b.ballDomain.ballIdMappings.map(_.outBW).sum - val totalMmioRead = b.ballDomain.ballIdMappings.map(_.mmioReadBW).sum - val totalMmioWrite = b.ballDomain.ballIdMappings.map(_.mmioWriteBW).sum - - @public - val global_issue_i = IO(Flipped(Decoupled(new GlobalSchedIssue(b)))) - @public - val global_complete_o = IO(Decoupled(new GlobalSchedComplete(b))) - @public - val ballChannelActive = IO(Output(Vec(b.ballDomain.ballNum, Bool()))) - @public - val ballChannelReady = IO(Input(Vec(b.ballDomain.ballNum, Bool()))) - @public - val bankRead = IO(Vec(totalBallRead, Flipped(new BankRead(b)))) - @public - val bankWrite = IO(Vec(totalBallWrite, Flipped(new BankWrite(b)))) - @public - val mmioRead = IO(Vec(totalMmioRead, Flipped(new MmioRead(b)))) - @public - val mmioWrite = IO(Vec(totalMmioWrite, Flipped(new MmioWrite(b)))) - @public - val subRobReq = IO(Vec(b.ballDomain.ballNum, Decoupled(new SubRobRow(b)))) - @public val kernelBanks = if (b.rvv.enable) Some(IO(Input(new framework.rvv.BankLayout(b)))) else None - @public val internalCommand = if (b.rvv.enable) Some(IO(Flipped(Decoupled(new RvvBallCommand)))) else None - @public val internalComplete = if (b.rvv.enable) Some(IO(Decoupled(UInt(64.W)))) else None - @public val ownerRobId = if (b.rvv.enable) Some(IO(Input(UInt(log2Up(b.frontend.rob_entries).W)))) else None - - val bbus: Instance[BBus] = Instantiate(new BBus(b)) - val ballDecoder: Instance[BallDomainDecoder] = Instantiate(new BallDomainDecoder(b)) - val ballRs: Instance[BallReservationStation] = Instantiate(new BallReservationStation(b)) - - if (b.rvv.enable) { - bbus.kernelBanks.get := kernelBanks.get - val decoder = Instantiate(new BallDomainDecoder(b)) - decoder.cmd_i.valid := internalCommand.get.valid - decoder.cmd_i.bits := 0.U.asTypeOf(new PostGDCmd(b)) - decoder.cmd_i.bits.domain_id := DomainId.BALL - decoder.cmd_i.bits.op1_col := 1.U - decoder.cmd_i.bits.op2_col := 1.U - decoder.cmd_i.bits.wr_col := 1.U - decoder.cmd_i.bits.cmd.funct := internalCommand.get.bits.funct7 - decoder.cmd_i.bits.cmd.rs1Data := internalCommand.get.bits.rs1 - decoder.cmd_i.bits.cmd.rs2Data := internalCommand.get.bits.rs2 - internalCommand.get.ready := decoder.cmd_i.ready - bbus.internalCommand.get.valid := decoder.ball_decode_cmd_o.valid - bbus.internalCommand.get.bits.cmd := decoder.ball_decode_cmd_o.bits - bbus.internalCommand.get.bits.rob_id := ownerRobId.get - bbus.internalCommand.get.bits.is_sub := false.B - bbus.internalCommand.get.bits.sub_rob_id := 0.U - decoder.ball_decode_cmd_o.ready := bbus.internalCommand.get.ready - internalComplete.get <> bbus.internalComplete.get - } - -//===-------------------------------------------------------------------===// -// Global RS -> Decoder -//===-------------------------------------------------------------------===// - ballDecoder.cmd_i.valid := global_issue_i.valid - ballDecoder.cmd_i.bits := global_issue_i.bits.cmd - global_issue_i.ready := ballDecoder.cmd_i.ready - -//===-------------------------------------------------------------------===// -// Decoder -> Local BallRS -//===-------------------------------------------------------------------===// - ballRs.ball_decode_cmd_i.valid := ballDecoder.ball_decode_cmd_o.valid - ballRs.ball_decode_cmd_i.bits.cmd := ballDecoder.ball_decode_cmd_o.bits - ballRs.ball_decode_cmd_i.bits.rob_id := global_issue_i.bits.rob_id - ballRs.ball_decode_cmd_i.bits.is_sub := global_issue_i.bits.is_sub - ballRs.ball_decode_cmd_i.bits.sub_rob_id := global_issue_i.bits.sub_rob_id - ballDecoder.ball_decode_cmd_o.ready := ballRs.ball_decode_cmd_i.ready - -//===-------------------------------------------------------------------===// -// Local BallRS -> BBus -//===-------------------------------------------------------------------===// - bbus.cmdReq <> ballRs.issue_o.balls - ballRs.commit_i.balls <> bbus.cmdResp - -//===-------------------------------------------------------------------===// -// BBus -> Mem Domain -//===-------------------------------------------------------------------===// - bbus.bankRead <> bankRead - bbus.bankWrite <> bankWrite - ballChannelActive := bbus.ballChannelActive - bbus.ballChannelReady := ballChannelReady - bbus.mmioRead <> mmioRead - bbus.mmioWrite <> mmioWrite - - for (i <- 0 until b.ballDomain.ballNum) { - subRobReq(i) <> bbus.subRobReq(i) - } - -//===-------------------------------------------------------------------===// -// Local RS completion -> Global RS -//===-------------------------------------------------------------------===// - global_complete_o.valid := ballRs.complete_o.valid - global_complete_o.bits.rob_id := ballRs.complete_o.bits.rob_id - global_complete_o.bits.is_sub := ballRs.complete_o.bits.is_sub - global_complete_o.bits.sub_rob_id := ballRs.complete_o.bits.sub_rob_id - global_complete_o.bits.fault := 0.U.asTypeOf(global_complete_o.bits.fault) - ballRs.complete_o.ready := global_complete_o.ready -} diff --git a/arch/src/main/scala/framework/balldomain/bbus/bbus.scala b/arch/src/main/scala/framework/balldomain/bbus/bbus.scala deleted file mode 100644 index 15a5106f1..000000000 --- a/arch/src/main/scala/framework/balldomain/bbus/bbus.scala +++ /dev/null @@ -1,255 +0,0 @@ -package framework.balldomain.bbus - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.top.GlobalConfig -import framework.balldomain.rs.{BallRsComplete, BallRsIssue} -import framework.balldomain.blink.HasBlink -import framework.balldomain.bbus.pmc.BallCyclePMC -import framework.balldomain.bbus.cmdrouter.CmdRouter -import framework.balldomain.isa.BallISA -import framework.balldomain.blink.{BankRead, BankWrite, SubRobRow} -import framework.balldomain.blink.mmio.{MmioRead, MmioWrite} -import java.lang.reflect.InvocationTargetException - -/** - * BBus - Ball bus, manages connections and arbitration of multiple Ball devices. - * - * Ball generators are produced reflectively from `b.ballDomain.ballIdMappings`: - * each mapping carries a `ballClass` FQCN whose constructor `(GlobalConfig)` is - * invoked. Framework does not maintain or interpret the list of balls; the - * config layer is the single source of truth. - */ -@instantiable -class BBus(val b: GlobalConfig) extends Module { - val numBalls = b.ballDomain.ballNum - val totalBallRead = b.ballDomain.ballIdMappings.map(_.inBW).sum - val totalBallWrite = b.ballDomain.ballIdMappings.map(_.outBW).sum - val totalMmioRead = b.ballDomain.ballIdMappings.map(_.mmioReadBW).sum - val totalMmioWrite = b.ballDomain.ballIdMappings.map(_.mmioWriteBW).sum - - // Rs - bbus - balls - @public - val cmdReq = IO(Vec(numBalls, Flipped(Decoupled(new BallRsIssue(b))))) - @public - val cmdResp = IO(Vec(numBalls, Decoupled(new BallRsComplete(b)))) - @public - val ballChannelActive = IO(Output(Vec(numBalls, Bool()))) - @public - val ballChannelReady = IO(Input(Vec(numBalls, Bool()))) - // balls - bbus - @public - val bankRead = IO(Vec(totalBallRead, Flipped(new BankRead(b)))) - @public - val bankWrite = IO(Vec(totalBallWrite, Flipped(new BankWrite(b)))) - @public - val mmioRead = IO(Vec(totalMmioRead, Flipped(new MmioRead(b)))) - @public - val mmioWrite = IO(Vec(totalMmioWrite, Flipped(new MmioWrite(b)))) - // balls - bbus - SubROB - @public - val subRobReq = IO(Vec(numBalls, Decoupled(new SubRobRow(b)))) - @public val kernelBanks = if (b.rvv.enable) Some(IO(Input(new framework.rvv.BankLayout(b)))) else None - @public val internalCommand = if (b.rvv.enable) Some(IO(Flipped(Decoupled(new BallRsIssue(b))))) else None - @public val internalComplete = if (b.rvv.enable) Some(IO(Decoupled(UInt(64.W)))) else None - - require(b.ballDomain.ballIdMappings.length == numBalls, "ballNum must match ballIdMappings length") - - // Apply BALL_INIT on the cycle after the command handshake. Driving reset - // directly from cmd.fire would feed the Ball's reset-dependent ready signal - // back into that same handshake. - val ballBootReset = RegInit(VecInit(Seq.fill(numBalls)(false.B))) - - val balls = b.ballDomain.ballIdMappings.zipWithIndex.map { case (mapping, index) => - withReset(reset.asBool || ballBootReset(index)) { - Module { - try { - val cls = Class.forName(mapping.ballClass) - val ctor = cls.getConstructor(classOf[GlobalConfig]) - ctor.newInstance(b).asInstanceOf[HasBlink with Module] - } catch { - case e: InvocationTargetException => - val cause = Option(e.getCause).getOrElse(e) - throw new RuntimeException( - s"Failed to instantiate ball ${mapping.ballName} (${mapping.ballClass}): ${cause.getMessage}", - cause - ) - case e: Throwable => - throw new RuntimeException( - s"Failed to instantiate ball ${mapping.ballName} (${mapping.ballClass}): ${e.getMessage}", - e - ) - } - } - } - } - - val cmdRouter: Instance[CmdRouter] = Instantiate(new CmdRouter(b)) - val pmc: Instance[BallCyclePMC] = Instantiate(new BallCyclePMC(b)) - -// ----------------------------------------------------------------------------- -// cmd router -// ----------------------------------------------------------------------------- - - val idle_ball = VecInit(balls.map(_.blink.cmdReq.ready)) - - cmdRouter.io.cmdReq_i <> cmdReq - cmdRouter.io.ballIdle := idle_ball - - val command = Wire(Decoupled(new BallRsIssue(b))) - val internal = WireDefault(false.B) - val internalOwner = RegInit(VecInit(Seq.fill(numBalls)(false.B))) - if (b.rvv.enable) { - val arbiter = Module(new RRArbiter(new BallRsIssue(b), 2)) - arbiter.io.in(0) <> cmdRouter.io.cmdReq_o - arbiter.io.in(1) <> internalCommand.get - command <> arbiter.io.out - internal := arbiter.io.chosen === 1.U - internalComplete.get.valid := VecInit( - (0 until numBalls).map(i => internalOwner(i) && balls(i).blink.cmdResp.valid) - ).asUInt.orR - internalComplete.get.bits := 0.U - assert(PopCount(internalOwner) <= 1.U, "RVV issues only one outstanding Ball operation") - } else { - command <> cmdRouter.io.cmdReq_o - } - - val isBallInit = command.bits.cmd.funct7 === BallISA.InitFunct.U - val initPending = RegInit(VecInit(Seq.fill(numBalls)(false.B))) - val initResp = Reg(Vec(numBalls, new BallRsComplete(b))) - val targetMatches = VecInit(b.ballDomain.ballIdMappings.map(m => command.bits.cmd.bid === m.ballId.U)) - - for (i <- 0 until numBalls) { - val mapping = b.ballDomain.ballIdMappings(i) - ballChannelActive(i) := (if (mapping.inBW == 0 && mapping.outBW == 0) false.B else balls(i).blink.status.running) - balls(i).blink.channelReady := ballChannelReady(i) - - val targetMatch = targetMatches(i) - balls(i).blink.cmdReq.valid := command.valid && !isBallInit && !initPending(i) && targetMatch - balls(i).blink.cmdReq.bits := command.bits - - cmdRouter.io.cmdResp_i(i).valid := !internalOwner(i) && Mux(initPending(i), true.B, balls(i).blink.cmdResp.valid) - cmdRouter.io.cmdResp_i(i).bits := Mux(initPending(i), initResp(i), balls(i).blink.cmdResp.bits) - balls(i).blink.cmdResp.ready := !initPending(i) && Mux( - internalOwner(i), - (if (b.rvv.enable) internalComplete.get.ready else false.B), - cmdRouter.io.cmdResp_i(i).ready - ) - when(command.fire && targetMatch)(internalOwner(i) := internal) - when(balls(i).blink.cmdResp.fire || initPending(i) && cmdRouter.io.cmdResp_i(i).fire) { - internalOwner(i) := false.B - } - if (b.rvv.enable) { - when(internalOwner(i) && initPending(i)) { - internalComplete.get.valid := true.B - when(internalComplete.get.fire) { initPending(i) := false.B; internalOwner(i) := false.B } - } - } - - ballBootReset(i) := command.fire && isBallInit && targetMatch - when(command.fire && isBallInit && targetMatch) { - initPending(i) := true.B - initResp(i).rob_id := command.bits.rob_id - initResp(i).is_sub := command.bits.is_sub - initResp(i).sub_rob_id := command.bits.sub_rob_id - } - when(initPending(i) && cmdRouter.io.cmdResp_i(i).fire) { - initPending(i) := false.B - } - } - - val targetReady = - VecInit((0 until numBalls).map(i => balls(i).blink.cmdReq.ready && !initPending(i) && targetMatches(i))).asUInt.orR - command.ready := targetReady - - cmdResp <> cmdRouter.io.cmdResp_o - -// ----------------------------------------------------------------------------- -// PMC - Performance Monitor Counter -// ----------------------------------------------------------------------------- - for (i <- 0 until numBalls) { - pmc.io.cmdReq_i(i).valid := cmdRouter.io.cmdReq_i(i).fire || command.fire && internal && targetMatches(i) - pmc.io.cmdReq_i(i).bits := Mux(internal, command.bits, cmdRouter.io.cmdReq_i(i).bits) - pmc.io.cmdResp_o(i).valid := cmdRouter.io.cmdResp_o(i).valid || - internalOwner(i) && (balls(i).blink.cmdResp.fire || initPending(i) && (if (b.rvv.enable) internalComplete.get.fire - else false.B)) - pmc.io.cmdResp_o(i).bits := Mux( - internalOwner(i), - Mux(initPending(i), initResp(i), balls(i).blink.cmdResp.bits), - cmdRouter.io.cmdResp_o(i).bits - ) - } - -// Connect balls' bankRead and bankWrite to memrouter - var readChannelIdx = 0 - var writeChannelIdx = 0 - - for (((mapping, ball), index) <- b.ballDomain.ballIdMappings.zip(balls).zipWithIndex) { - val inBW = mapping.inBW - val outBW = mapping.outBW - - for (i <- 0 until inBW) { - bankRead(readChannelIdx) <> ball.blink.bankRead(i) - if (b.rvv.enable) { - val local = ball.blink.bankRead(i).bank_id - val layout = kernelBanks.get - when(internalOwner(index)) { - bankRead(readChannelIdx).bank_id := Mux(local < layout.readGroups, layout.readBank, layout.writeBank) - bankRead(readChannelIdx).group_id := Mux(local < layout.readGroups, local, local - layout.readGroups) - when(ball.blink.bankRead(i).io.req.valid) { - assert(local < (layout.readGroups +& layout.writeGroups), "RVV Ball read bank outside launch groups") - } - } - } - readChannelIdx = readChannelIdx + 1 - } - - for (i <- 0 until outBW) { - bankWrite(writeChannelIdx) <> ball.blink.bankWrite(i) - if (b.rvv.enable) { - val local = ball.blink.bankWrite(i).bank_id - val layout = kernelBanks.get - when(internalOwner(index)) { - bankWrite(writeChannelIdx).bank_id := layout.writeBank - bankWrite(writeChannelIdx).group_id := local - layout.readGroups - when(ball.blink.bankWrite(i).io.req.valid) { - assert( - local >= layout.readGroups && local < (layout.readGroups +& layout.writeGroups), - "RVV Ball write outside write groups" - ) - } - } - } - writeChannelIdx = writeChannelIdx + 1 - } - } - - var mmioWriteChannelIdx = 0 - for ((mapping, ball) <- b.ballDomain.ballIdMappings.zip(balls)) { - val mmioWriteBW = mapping.mmioWriteBW - for (i <- 0 until mmioWriteBW) { - mmioWrite(mmioWriteChannelIdx) <> ball.blink.mmioWrite(i) - mmioWriteChannelIdx = mmioWriteChannelIdx + 1 - } - } - - // Internal RVV operations complete directly; they cannot expand into SubROB. - for (i <- 0 until numBalls) { - subRobReq(i).valid := balls(i).blink.subRobReq.valid && !internalOwner(i) - subRobReq(i).bits := balls(i).blink.subRobReq.bits - balls(i).blink.subRobReq.ready := subRobReq(i).ready && !internalOwner(i) - assert(!(internalOwner(i) && balls(i).blink.subRobReq.valid), "RVV Ball operation cannot issue SubROB work") - } - - // Connect configurable MMIO metadata channels. - var mmioReadChannelIdx = 0 - for ((mapping, ball) <- b.ballDomain.ballIdMappings.zip(balls)) { - val mmioReadBW = mapping.mmioReadBW - for (i <- 0 until mmioReadBW) { - mmioRead(mmioReadChannelIdx) <> ball.blink.mmioRead(i) - mmioReadChannelIdx = mmioReadChannelIdx + 1 - } - } - -} diff --git a/arch/src/main/scala/framework/balldomain/bbus/cmdrouter/CmdRouter.scala b/arch/src/main/scala/framework/balldomain/bbus/cmdrouter/CmdRouter.scala deleted file mode 100644 index 97d804ea7..000000000 --- a/arch/src/main/scala/framework/balldomain/bbus/cmdrouter/CmdRouter.scala +++ /dev/null @@ -1,60 +0,0 @@ -//===- CmdRouter.scala ------ Command Router of Ball Domain ---------------===// -// -// Copyright 2026 The Buckyball Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//===----------------------------------------------------------------------===// -// -// Request channel (req): N inputs (cmdReq_i) -> 1 output (cmdReq_o) -// -// Response channel (resp): N inputs (cmdResp_i) -> N outputs (cmdResp_o), with -// direct point-to-point connections -// -//===----------------------------------------------------------------------===// -package framework.balldomain.bbus.cmdrouter - -import chisel3._ -import chisel3.util._ -import framework.top.GlobalConfig -import framework.balldomain.rs.{BallRsComplete, BallRsIssue} -import chisel3.experimental.hierarchy.{instantiable, public} - -@instantiable -class CmdRouter(val b: GlobalConfig) extends Module { - val numBalls = b.ballDomain.ballNum - - @public - val io = IO(new Bundle { - val cmdReq_i = Vec(numBalls, Flipped(Decoupled(new BallRsIssue(b)))) - val cmdResp_i = Vec(numBalls, Flipped(Decoupled(new BallRsComplete(b)))) - val cmdReq_o = Decoupled(new BallRsIssue(b)) - val cmdResp_o = Vec(numBalls, Decoupled(new BallRsComplete(b))) - val ballIdle = Input(Vec(numBalls, Bool())) - }) - - val arbiter = Module(new RRArbiter(new BallRsIssue(b), numBalls)) - val ballIdleR = RegNext(io.ballIdle, VecInit(Seq.fill(numBalls)(false.B))) - - for (i <- 0 until numBalls) { - arbiter.io.in(i).valid := io.cmdReq_i(i).valid && ballIdleR(i) - arbiter.io.in(i).bits := io.cmdReq_i(i).bits - io.cmdReq_i(i).ready := arbiter.io.in(i).ready && ballIdleR(i) - } - - io.cmdReq_o <> arbiter.io.out - - for (i <- 0 until numBalls) { - io.cmdResp_o(i) <> io.cmdResp_i(i) - } -} diff --git a/arch/src/main/scala/framework/balldomain/bbus/cmdrouter/cmdrouter.spec.en.md b/arch/src/main/scala/framework/balldomain/bbus/cmdrouter/cmdrouter.spec.en.md deleted file mode 100644 index 537b47c36..000000000 --- a/arch/src/main/scala/framework/balldomain/bbus/cmdrouter/cmdrouter.spec.en.md +++ /dev/null @@ -1,33 +0,0 @@ -# CmdRouter - -## TL;DR -- 【Module Function】Arbitrates `numBalls` request lanes from `cmdReq_i` into one `cmdReq_o`, and forwards each `cmdResp_i` lane to the matching `cmdResp_o` lane. -- 【Module Placement】This module is `CmdRouter` on the `bbus` in `balldomain`, responsible for command routing. -- 【Module Inputs/Outputs】 - - Command dispatch: `cmdReq_i` from frontend to bbus; after round-robin arbitration, single `cmdReq_o` to one ball. - - Command completion response: `cmdResp_i` from each ball; forwarded lane-by-lane through `cmdResp_o` to frontend. - - BBus drives `ballIdle` to report whether each ball is idle. -- 【Key Points】 - - Request arbitration uses registered `ballIdleR` only, so `ballIdle` affects admission with a one-cycle delay. - -## Interface -| Direction | Signal | Type | Meaning | -| --- | --- | --- | --- | -| Input | `cmdReq_i` | `Vec(numBalls, Flipped(Decoupled(BallRsIssue)))` | Per-Ball request ingress channels using `valid/ready` handshake. | -| Input | `cmdResp_i` | `Vec(numBalls, Flipped(Decoupled(BallRsComplete)))` | Per-Ball response ingress channels, forwarded lane by lane. | -| Input | `ballIdle` | `Vec(numBalls, Bool)` | Per-Ball idle indicators, registered before request gating. | -| Output | `cmdReq_o` | `Decoupled(BallRsIssue)` | Single arbitrated request egress channel. | -| Output | `cmdResp_o` | `Vec(numBalls, Decoupled(BallRsComplete))` | Response egress channels directly mapped to `cmdResp_i` by index. | - -## Core Behavior -The module instantiates `RRArbiter(BallRsIssue, numBalls)` for round-robin request selection and computes `ballIdleR = RegNext(io.ballIdle, false)` as the lane-enable condition. For any index `i`, a request is presented to arbitration only when both `cmdReq_i(i).valid` and `ballIdleR(i)` are true. The same `ballIdleR(i)` condition gates `cmdReq_i(i).ready`, so upstream always sees `ready = false` when that lane is not idle in the registered view. The arbiter output is directly connected to `cmdReq_o` with `<>`, so `valid/bits/ready` semantics follow the standard `RRArbiter` plus Decoupled connection behavior. The response path performs no selection or transformation: each lane uses `cmdResp_o(i) <> cmdResp_i(i)` and keeps independent handshakes. - -## Verification Checklist -- With all `ballIdle` lanes held at `0` and all `cmdReq_i.valid` held at `1`, `cmdReq_o.valid` must remain `0`; any observed `1` is fail. -- Raise one `ballIdle(i)` from `0` to `1` at cycle `t` while keeping `cmdReq_i(i).valid=1`; that lane may be admitted no earlier than cycle `t+1`; admission at `t` is fail. -- With `ballIdle(i)=0` and `cmdReq_i(i).valid=1`, `cmdReq_i(i).ready` must stay `0`; any cycle with `ready=1` is fail. -- With two or more continuously valid lanes and their `ballIdleR` all `1`, `cmdReq_o.bits` source should rotate under round-robin instead of sticking forever to one lane; persistent starvation is fail. -- Inject a response transaction on any lane `i` while driving `cmdResp_o(i).ready=1`; `cmdResp_o(i).valid` and `bits` must match input in the same cycle; mismatch is fail. -- Drive different response streams on lanes `i` and `j (i!=j)` concurrently; outputs must remain lane-isolated with no cross-lane mixing; any mixing is fail. -- When downstream forces `cmdReq_o.ready=0`, request handshakes must stop and no fake completion can occur; any observed `fire` is fail. -- Toggle `ballIdle` across adjacent cycles and verify gating always follows one-cycle-late `RegNext` behavior; same-cycle pass-through is fail. diff --git a/arch/src/main/scala/framework/balldomain/bbus/cmdrouter/cmdrouter.spec.zh.md b/arch/src/main/scala/framework/balldomain/bbus/cmdrouter/cmdrouter.spec.zh.md deleted file mode 100644 index 7431fdbe9..000000000 --- a/arch/src/main/scala/framework/balldomain/bbus/cmdrouter/cmdrouter.spec.zh.md +++ /dev/null @@ -1,33 +0,0 @@ -# CmdRouter - -## TL;DR -- 【模块功能】将 `numBalls` 路 `cmdReq_i` 按轮询仲裁汇聚到单一路 `cmdReq_o`,并把每路 `cmdResp_i` 一一透传到对应的 `cmdResp_o`。 -- 【模块定位】该模块是 `balldomain`的`bbus`上的 `CmdRouter` 负责指令路由`。 -- 【模块输入/输出】 - - 指令分发:从前端到bbus输入 `cmdReq_i`,选择后输出单路 `cmdReq_o` 给某个ball。 - - 指令(完成后)响应:从每个ball输入`cmdResp_o` 的响应信号,选择后输出单路给前端。 - - bbus通过 `ballIdle` 获取每个ball是否空闲 -- 【关键点】 - - 请求仲裁只看寄存后的 `ballIdleR` - -## Interface -| 方向 | 信号名 | 类型 | 含义 | -| --- | --- | --- | --- | -| 输入 | `cmdReq_i` | `Vec(numBalls, Flipped(Decoupled(BallRsIssue)))` | 来自各 Ball 的请求输入通道,按 `valid/ready` 握手。 | -| 输入 | `cmdResp_i` | `Vec(numBalls, Flipped(Decoupled(BallRsComplete)))` | 来自各 Ball 的响应输入通道,逐路透传到输出。 | -| 输入 | `ballIdle` | `Vec(numBalls, Bool)` | 各 Ball 的空闲状态输入,先经 `RegNext` 后参与请求门控。 | -| 输出 | `cmdReq_o` | `Decoupled(BallRsIssue)` | 仲裁后的单一路请求输出通道。 | -| 输出 | `cmdResp_o` | `Vec(numBalls, Decoupled(BallRsComplete))` | 与 `cmdResp_i` 同索引直连的响应输出通道。 | - -## Core Behavior -模块内部实例化 `RRArbiter(BallRsIssue, numBalls)` 对请求进行轮询仲裁,并使用 `ballIdleR = RegNext(io.ballIdle, false)` 作为每路请求的使能条件。对任意索引 `i`,只有当 `cmdReq_i(i).valid` 与 `ballIdleR(i)` 同时为真时,该路才会向仲裁器宣告有效请求;同一条件也用于反向门控 `cmdReq_i(i).ready`,因此当 `ballIdleR(i)` 为假时,上游看到的 `ready` 必为假。仲裁器输出通过 `<>` 直接连接到 `cmdReq_o`,其 `valid/bits/ready` 握手语义由 `RRArbiter` 与该直连共同决定。响应路径不做选择与变换,每个索引 `i` 上 `cmdResp_o(i) <> cmdResp_i(i)`,保持逐路独立握手。 - -## Verification Checklist -- 在 `ballIdle` 全为 `0` 且各路 `cmdReq_i.valid` 为 `1` 的场景,期望 `cmdReq_o.valid` 始终为 `0`;若出现 `1` 判 fail。 -- 在第 `t` 周期将某一路 `ballIdle(i)` 从 `0` 拉到 `1`,并保持 `cmdReq_i(i).valid=1`,期望最早在 `t+1` 周期该路才可能被仲裁;若 `t` 周期已被放行判 fail。 -- 在某一路 `ballIdle(i)=0` 且 `cmdReq_i(i).valid=1` 时,检查 `cmdReq_i(i).ready` 必为 `0`;任一周期为 `1` 判 fail。 -- 在两路及以上持续有效且对应 `ballIdleR` 均为 `1` 的场景,观察 `cmdReq_o.bits` 来源应随轮询发生切换,不应永久固定在同一路;若长期饥饿某一路判 fail。 -- 对任意索引 `i` 注入 `cmdResp_i(i)` 事务并驱动 `cmdResp_o(i).ready=1`,期望同拍 `cmdResp_o(i).valid` 与 `bits` 等于输入;不一致判 fail。 -- 对任意索引 `i/j(i!=j)` 同时施加不同响应流量,期望两路 `cmdResp_o` 互不串扰;若出现跨索引数据混入判 fail。 -- 在下游将 `cmdReq_o.ready` 拉低时,期望仲裁输出握手暂停且不会伪造完成;若无 `ready` 仍出现 `fire` 判 fail。 -- 将 `ballIdle` 在相邻周期内反复翻转,验证门控行为始终按 `RegNext` 延后一拍;若出现当拍直通效果判 fail。 diff --git a/arch/src/main/scala/framework/balldomain/bbus/pmc/BallCyclePMC.scala b/arch/src/main/scala/framework/balldomain/bbus/pmc/BallCyclePMC.scala deleted file mode 100644 index 00f04b7dd..000000000 --- a/arch/src/main/scala/framework/balldomain/bbus/pmc/BallCyclePMC.scala +++ /dev/null @@ -1,83 +0,0 @@ -//===- BallCyclePMC.scala ------ Ball Performance Counter -----------------===// -// -// Copyright 2026 The Buckyball Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//===----------------------------------------------------------------------===// -// -// This DPI-C module is used to collect the performance counter data from each -// ball mounted on the bbus. Each ball has its own PMCModule. -// PMCModule just triggers at two points: -// -// 1. When the ball issues a command. -// 2. When the ball completes a command. -// -//===----------------------------------------------------------------------===// - -package framework.balldomain.bbus.pmc - -import chisel3._ -import chisel3.util._ -import framework.top.GlobalConfig -import framework.balldomain.rs.BallRsIssue -import framework.balldomain.rs.BallRsComplete -import chisel3.experimental.hierarchy.{instantiable, public} - -@instantiable -class BallCyclePMC(val b: GlobalConfig) extends Module { - val numBalls = b.ballDomain.ballNum - - @public - val io = IO(new Bundle { - val cmdReq_i = Input(Vec(numBalls, Valid(new BallRsIssue(b)))) - val cmdResp_o = Input(Vec(numBalls, Valid(new BallRsComplete(b)))) - }) - -// A global cycle counter after reset. - val cycleCounter = RegInit(0.U(64.W)) - val startTime = RegInit(VecInit(Seq.fill(b.frontend.rob_entries)(0.U(64.W)))) - val pmcTraces = Seq.fill(numBalls)(Module(new PMCTraceDPI)) - - cycleCounter := cycleCounter + 1.U - -//===---------- Initialize the PMC traces ----------===// - for (pt <- pmcTraces) { - pt.io.clock := clock - pt.io.reset := reset.asBool - pt.io.ball_id := 0.U - pt.io.rob_id := 0.U - pt.io.elapsed := 0.U - pt.io.enable := false.B - } - -//===---------- Collect the performance counter data ----------===// -// Trigger at the start of the command. - for (i <- 0 until numBalls) { - when(io.cmdReq_i(i).valid) { - startTime(io.cmdReq_i(i).bits.rob_id) := cycleCounter - } - } - -// Trigger at the end of the command. - for (i <- 0 until numBalls) { - when(io.cmdResp_o(i).valid) { - val robId = io.cmdResp_o(i).bits.rob_id - val elapsed = cycleCounter - startTime(robId) - pmcTraces(i).io.ball_id := i.U - pmcTraces(i).io.rob_id := robId - pmcTraces(i).io.elapsed := elapsed - pmcTraces(i).io.enable := true.B - } - } -} diff --git a/arch/src/main/scala/framework/balldomain/blink/SubRobRow.scala b/arch/src/main/scala/framework/balldomain/blink/SubRobRow.scala deleted file mode 100644 index aae0fef5a..000000000 --- a/arch/src/main/scala/framework/balldomain/blink/SubRobRow.scala +++ /dev/null @@ -1,54 +0,0 @@ -package framework.balldomain.blink - -import chisel3._ -import chisel3.util._ -import framework.top.GlobalConfig -import framework.frontend.decoder.PostGDCmd -import framework.frontend.scoreboard.BankAccessInfo - -class SubRobSlot(val b: GlobalConfig) extends Bundle { - val valid = Bool() - val cmd = new PostGDCmd(b) -} - -class SubRobRow(val b: GlobalConfig) extends Bundle { - val slots = Vec(4, new SubRobSlot(b)) - val ball_id = UInt(log2Up(b.ballDomain.ballNum).W) - val master_rob_id = UInt(log2Up(b.frontend.rob_entries).W) -} - -/** Tie-off for subRobReq when a Ball does not use SubROB. */ -object SubRobRow { - - def tieOff(b: GlobalConfig): SubRobRow = { - val w = Wire(new SubRobRow(b)) - w.ball_id := 0.U - w.master_rob_id := 0.U - val bankIdLen = b.frontend.bank_id_len - for (i <- 0 until 4) { - w.slots(i).valid := false.B - w.slots(i).cmd.domain_id := 0.U - w.slots(i).cmd.ball_bid := 0.U - w.slots(i).cmd.cmd.raw_inst := 0.U - w.slots(i).cmd.cmd.pc := 0.U - w.slots(i).cmd.cmd.funct := 0.U - w.slots(i).cmd.cmd.funct3 := 0.U - w.slots(i).cmd.cmd.rs2 := 0.U - w.slots(i).cmd.cmd.rs1 := 0.U - w.slots(i).cmd.cmd.xd := false.B - w.slots(i).cmd.cmd.xs1 := false.B - w.slots(i).cmd.cmd.xs2 := false.B - w.slots(i).cmd.cmd.rd := 0.U - w.slots(i).cmd.cmd.opcode := 0.U - w.slots(i).cmd.cmd.rs1Data := 0.U - w.slots(i).cmd.cmd.rs2Data := 0.U - w.slots(i).cmd.bankAccess := BankAccessInfo.none(bankIdLen) - w.slots(i).cmd.op1_col := 0.U - w.slots(i).cmd.op2_col := 0.U - w.slots(i).cmd.wr_col := 0.U - w.slots(i).cmd.isBarrier := false.B - } - w - } - -} diff --git a/arch/src/main/scala/framework/balldomain/blink/bank.scala b/arch/src/main/scala/framework/balldomain/blink/bank.scala deleted file mode 100644 index 1c811e9fc..000000000 --- a/arch/src/main/scala/framework/balldomain/blink/bank.scala +++ /dev/null @@ -1,34 +0,0 @@ -package framework.balldomain.blink - -import chisel3._ -import chisel3.util._ -import framework.memdomain.backend.banks.{SramReadIO, SramWriteIO} -import framework.top.GlobalConfig - -trait HasBankId { - val b: GlobalConfig - val bank_id = Input(UInt(b.memDomain.vbankIdWidth.W)) -} - -trait HasRobId { - val b: GlobalConfig - val rob_id = Input(UInt(log2Up(b.frontend.rob_entries).W)) -} - -trait HasBallId { - val b: GlobalConfig - val ball_id = Input(UInt(log2Up(b.ballDomain.ballNum).W)) -} - -trait HasAccGroupId { - val b: GlobalConfig - val group_id = Input(UInt(b.memDomain.groupIdWidth.W)) -} - -class BankRead(val b: GlobalConfig) extends Bundle with HasBankId with HasRobId with HasBallId with HasAccGroupId { - val io = new SramReadIO(b) -} - -class BankWrite(val b: GlobalConfig) extends Bundle with HasBankId with HasRobId with HasBallId with HasAccGroupId { - val io = new SramWriteIO(b) -} diff --git a/arch/src/main/scala/framework/balldomain/blink/baseball.scala b/arch/src/main/scala/framework/balldomain/blink/baseball.scala deleted file mode 100644 index b00fc630f..000000000 --- a/arch/src/main/scala/framework/balldomain/blink/baseball.scala +++ /dev/null @@ -1,14 +0,0 @@ -package framework.balldomain.blink - -import chisel3._ -import chisel3.util._ - -// all balls must have a blink interface -trait HasBlink { - def blink: BlinkIO -} - -// can be customized by user to add additional status signals -trait HasBallStatus { - def status: BallStatus -} diff --git a/arch/src/main/scala/framework/balldomain/blink/blink.scala b/arch/src/main/scala/framework/balldomain/blink/blink.scala deleted file mode 100644 index 4cda0481c..000000000 --- a/arch/src/main/scala/framework/balldomain/blink/blink.scala +++ /dev/null @@ -1,38 +0,0 @@ -package framework.balldomain.blink - -import chisel3._ -import chisel3.util._ -import framework.top.GlobalConfig -import framework.balldomain.rs.{BallRsComplete, BallRsIssue} -import framework.balldomain.blink.mmio.{MmioRead, MmioWrite} - -class BlinkPortIO[I <: Data, R <: Data]( - b: GlobalConfig, - inBW: Int, - outBW: Int, - issue: I, - complete: R) - extends Bundle - with HasBallStatus { - val status = new BallStatus() - val channelReady = Input(Bool()) - val cmdReq = Flipped(Decoupled(issue)) - val cmdResp = Decoupled(complete) - val bankRead = Vec(inBW, Flipped(new BankRead(b))) - val bankWrite = Vec(outBW, Flipped(new BankWrite(b))) -} - -class BlinkIO( - b: GlobalConfig, - inBW: Int, - outBW: Int, - mmioReadBW: Int = 0, - mmioWriteBW: Int = 0) - extends BlinkPortIO(b, inBW, outBW, new BallRsIssue(b), new BallRsComplete(b)) { - val subRobReq = Decoupled(new SubRobRow(b)) - - // MMIO metadata channels. Like bankRead/bankWrite, their number is a Ball - // configuration property rather than a framework-wide fixed interface. - val mmioRead = Vec(mmioReadBW, Flipped(new MmioRead(b))) - val mmioWrite = Vec(mmioWriteBW, Flipped(new MmioWrite(b))) -} diff --git a/arch/src/main/scala/framework/balldomain/blink/mmio/MmioRead.scala b/arch/src/main/scala/framework/balldomain/blink/mmio/MmioRead.scala deleted file mode 100644 index 7b3ed17ff..000000000 --- a/arch/src/main/scala/framework/balldomain/blink/mmio/MmioRead.scala +++ /dev/null @@ -1,50 +0,0 @@ -package framework.balldomain.blink.mmio - -import chisel3._ -import chisel3.util._ -import framework.top.GlobalConfig -import framework.balldomain.blink.{HasBallId, HasRobId} -import framework.memdomain.backend.mmio.{MmioReadReq, MmioReadResp} - -/** - * MmioRead: Ball-facing MMIO read port (part of BlinkIO). - * - * Defined from the CONSUMER (MmioRouter) perspective: - * - req is Flipped(Decoupled) so the consumer drives ready and reads valid/bits - * - resp is Decoupled so the consumer drives valid/bits and reads ready - * - ball_id, rob_id are Input (driven by the Ball when consumed) - * - * BlinkIO wraps this in Flipped(...), so from the Ball's perspective: - * - req becomes Decoupled (Ball drives valid/bits, reads ready) - * - resp becomes Flipped(Decoupled) (Ball reads valid/bits, drives ready) - * - ball_id, rob_id become Output (Ball drives them) - */ -class MmioRead(val b: GlobalConfig) extends Bundle with HasBallId with HasRobId { - val req = Flipped(Decoupled(new MmioReadReq(b))) - val resp = Decoupled(new MmioReadResp(b)) -} - -object MmioRead { - - /** - * Tie off an MmioRead port from the Ball's perspective (BlinkIO.mmioRead). - * Ball's perspective after Flipped: - * - req: Decoupled (Ball drives) - tie valid=0, bits=0 - * - resp: Flipped(Decoupled) (Ball reads) - tie ready=0 - * - ball_id, rob_id: Output (Ball drives) - tie to 0 - */ - def tieOff(port: MmioRead): Unit = { - port.req.valid := false.B - port.req.bits := 0.U.asTypeOf(port.req.bits) - port.resp.ready := false.B - port.ball_id := 0.U - port.rob_id := 0.U - } - - def tieOff(ports: Vec[MmioRead]): Unit = { - for (port <- ports) { - tieOff(port) - } - } - -} diff --git a/arch/src/main/scala/framework/balldomain/blink/mmio/MmioWrite.scala b/arch/src/main/scala/framework/balldomain/blink/mmio/MmioWrite.scala deleted file mode 100644 index 5e42c4d44..000000000 --- a/arch/src/main/scala/framework/balldomain/blink/mmio/MmioWrite.scala +++ /dev/null @@ -1,34 +0,0 @@ -package framework.balldomain.blink.mmio - -import chisel3._ -import chisel3.util.Decoupled -import framework.balldomain.blink.{HasBallId, HasRobId} -import framework.memdomain.backend.mmio.MmioWriteReq -import framework.top.GlobalConfig - -/** - * Ball-facing byte write port for the unified MMIO address space. - * - * BlinkIO wraps this in Flipped(...), so a Ball produces req and observes - * ready. A multi-byte value uses one configured line per byte. - */ -class MmioWrite(val b: GlobalConfig) extends Bundle with HasBallId with HasRobId { - val req = Flipped(Decoupled(new MmioWriteReq(b))) -} - -object MmioWrite { - - def tieOff(port: MmioWrite): Unit = { - port.req.valid := false.B - port.req.bits := 0.U.asTypeOf(port.req.bits) - port.ball_id := 0.U - port.rob_id := 0.U - } - - def tieOff(ports: Vec[MmioWrite]): Unit = { - for (port <- ports) { - tieOff(port) - } - } - -} diff --git a/arch/src/main/scala/framework/balldomain/blink/status.scala b/arch/src/main/scala/framework/balldomain/blink/status.scala deleted file mode 100644 index 843444a31..000000000 --- a/arch/src/main/scala/framework/balldomain/blink/status.scala +++ /dev/null @@ -1,9 +0,0 @@ -package framework.balldomain.blink - -import chisel3._ -import chisel3.util._ - -class BallStatus extends Bundle { - val idle = Output(Bool()) - val running = Output(Bool()) -} diff --git a/arch/src/main/scala/framework/balldomain/configs/BallDomainParam.scala b/arch/src/main/scala/framework/balldomain/configs/BallDomainParam.scala deleted file mode 100644 index cadf020f4..000000000 --- a/arch/src/main/scala/framework/balldomain/configs/BallDomainParam.scala +++ /dev/null @@ -1,66 +0,0 @@ -package framework.balldomain.configs - -import upickle.default._ - -case class BallIdMapping( - ballId: Int, - ballName: String, - ballClass: String, - config: Option[String] = None, - inBW: Int, - outBW: Int, - mmioReadBW: Int = 0, - mmioWriteBW: Int = 0, - ballParams: Map[String, String] = Map.empty) - -case class BallISAEntry( - mnemonic: String, - funct7: Int, - bid: Int) - -case class BallDomainParam( - ballNum: Int, - ballIdMappings: Seq[BallIdMapping], - ballISA: Seq[BallISAEntry]) { - - require(ballNum == ballIdMappings.length, "ballNum must match ballIdMappings length") - require( - ballIdMappings.map(_.ballId).distinct.length == ballIdMappings.length, - "ballIdMappings contains duplicate ballId" - ) - require( - ballISA.map(_.mnemonic).distinct.length == ballISA.length, - "ballISA contains duplicate mnemonic" - ) - require( - ballISA.map(_.funct7).distinct.length == ballISA.length, - "ballISA contains duplicate funct7" - ) - require( - ballISA.forall(entry => entry.funct7 >= 0 && entry.funct7 < 128), - "ballISA funct7 must fit the 7-bit instruction field" - ) - require( - ballISA.forall(entry => ballIdMappings.exists(_.ballId == entry.bid)), - "ballISA bid must reference a configured ballId" - ) - - def mapping(ballName: String): BallIdMapping = - ballIdMappings.find(_.ballName == ballName) match { - case Some(m) => m - case None => throw new RuntimeException(s"No ballIdMapping for ballName=$ballName") - } - -} - -object BallDomainParam { - implicit val ballIdMappingRW: ReadWriter[BallIdMapping] = macroRW - implicit val ballISAEntryRW: ReadWriter[BallISAEntry] = macroRW - implicit val rw: ReadWriter[BallDomainParam] = macroRW - - /** - * Empty default. Each example's GlobalConfig assembler is responsible for - * supplying its own `BallDomainParam` (typically via a layered JSON loader). - */ - def apply(): BallDomainParam = BallDomainParam(0, Seq.empty, Seq.empty) -} diff --git a/arch/src/main/scala/framework/balldomain/configs/BallParamLoader.scala b/arch/src/main/scala/framework/balldomain/configs/BallParamLoader.scala deleted file mode 100644 index 64283bda9..000000000 --- a/arch/src/main/scala/framework/balldomain/configs/BallParamLoader.scala +++ /dev/null @@ -1,66 +0,0 @@ -package framework.balldomain.configs - -import framework.top.GlobalConfig -import toml.{Toml, Value} -import java.nio.file.Paths - -object BallParamLoader { - - def load(path: String): Map[String, Value] = { - val content = scala.io.Source.fromFile(path, "UTF-8").mkString - Toml.parse(content) match { - case Right(Value.Tbl(root)) => root - case Right(_) => - throw new RuntimeException(s"TOML root must be a table in $path") - case Left((addr, msg)) => - throw new RuntimeException(s"TOML parse error at $addr: $msg in $path") - } - } - - def ball(root: Map[String, Value]): Map[String, Value] = - root.get("ball") match { - case Some(Value.Tbl(t)) => t - case None => throw new RuntimeException("Missing TOML section [ball]") - case _ => throw new RuntimeException("Expected TOML section [ball]") - } - - private def configPath(mapping: BallIdMapping): String = { - val config = mapping.config.getOrElse( - throw new RuntimeException(s"Ball ${mapping.ballName} has no config") - ) - if (!Paths.get(config).toFile.isFile) { - throw new RuntimeException(s"Ball ${mapping.ballName} config not found: $config") - } - config - } - - def ballTable(b: GlobalConfig, ballName: String): Map[String, Value] = { - val mapping = b.ballDomain.ballIdMappings.find(_.ballName == ballName) match { - case Some(m) => m - case None => throw new RuntimeException(s"No ballIdMapping for ballName=$ballName") - } - ball(load(configPath(mapping))) - } - - def int(table: Map[String, Value], key: String): Int = - table.get(key) match { - case Some(Value.Num(n)) => n.toInt - case None => throw new RuntimeException(s"Missing integer at key '$key'") - case _ => throw new RuntimeException(s"Expected integer at key '$key'") - } - - def str(table: Map[String, Value], key: String): String = - table.get(key) match { - case Some(Value.Str(s)) => s - case None => throw new RuntimeException(s"Missing string at key '$key'") - case _ => throw new RuntimeException(s"Expected string at key '$key'") - } - - def bool(table: Map[String, Value], key: String): Boolean = - table.get(key) match { - case Some(Value.Bool(b)) => b - case None => throw new RuntimeException(s"Missing boolean at key '$key'") - case _ => throw new RuntimeException(s"Expected boolean at key '$key'") - } - -} diff --git a/arch/src/main/scala/framework/balldomain/decoder/BallDecodeCmd.scala b/arch/src/main/scala/framework/balldomain/decoder/BallDecodeCmd.scala deleted file mode 100644 index e34e6f580..000000000 --- a/arch/src/main/scala/framework/balldomain/decoder/BallDecodeCmd.scala +++ /dev/null @@ -1,35 +0,0 @@ -package framework.balldomain.decoder - -import chisel3._ -import chisel3.util._ - -/** - * Wire-level command bundle exchanged between user-defined Ball decoders - * and framework infrastructure (BallReservationStation, BBus). The - * concrete decode logic (funct7 → BID/operands) lives in user code. - */ -class BallDecodeCmd(numBanks: Int, iterLen: Int, groupCountWidth: Int) extends Bundle { - val bid = UInt(5.W) - val funct7 = UInt(7.W) - val iter = UInt(iterLen.W) - - val op1_en = Bool() - val op2_en = Bool() - val wr_spad_en = Bool() - val op1_from_spad = Bool() - val op2_from_spad = Bool() - val special = UInt(64.W) - - val op1_bank = UInt(log2Up(numBanks).W) - val op2_bank = UInt(log2Up(numBanks).W) - val wr_bank = UInt(log2Up(numBanks).W) - val op1_col = UInt(groupCountWidth.W) - val op2_col = UInt(groupCountWidth.W) - val wr_col = UInt(groupCountWidth.W) - - // MMIO metadata bank: which main bank's MMIO region to access - val meta_bank = UInt(log2Up(numBanks).W) - - val rs1 = UInt(64.W) - val rs2 = UInt(64.W) -} diff --git a/arch/src/main/scala/framework/balldomain/decoder/BallDomainDecoder.scala b/arch/src/main/scala/framework/balldomain/decoder/BallDomainDecoder.scala deleted file mode 100644 index ca6db292d..000000000 --- a/arch/src/main/scala/framework/balldomain/decoder/BallDomainDecoder.scala +++ /dev/null @@ -1,107 +0,0 @@ -package framework.balldomain.decoder - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.balldomain.isa.BallISA -import framework.frontend.decoder.{DomainId, PostGDCmd} -import framework.top.GlobalConfig - -/** - * Ball-domain instruction decoder. - * - * - funct7 encoding contract → enable bits, operand/write banks, iter - * count, pass-through of rs1/rs2/funct7 (pure framework concern). - * - funct7 → bid routing, generated from `b.ballDomain.ballISA` - * (a `(mnemonic, funct7, bid)` table the user provides via - * TOML config); no per-example Scala decoder is needed. - */ -@instantiable -class BallDomainDecoder(val b: GlobalConfig) extends Module { - val bankIdLen = b.frontend.bank_id_len - val iterLen = b.frontend.iter_len - - @public - val cmd_i = IO(Flipped(Decoupled(new PostGDCmd(b)))) - - @public - val ball_decode_cmd_o = - IO(Decoupled(new BallDecodeCmd(b.memDomain.virtualBankCount, iterLen, b.memDomain.groupCountWidth))) - - cmd_i.ready := ball_decode_cmd_o.ready - - val func7 = cmd_i.bits.cmd.funct - val rs1 = cmd_i.bits.cmd.rs1Data - val rs2 = cmd_i.bits.cmd.rs2Data - - val op1_bank_raw = rs1(bankIdLen - 1, 0) - val op2_bank_raw = rs1(bankIdLen + 9, 10) - val wr_bank_raw = rs1(bankIdLen + 19, 20) - val iter_raw = rs1(63, 30) - - // funct7[6:4] enable encoding: - // 000 no bank 001 1rd 010 1wr 011 1rd+1wr - // 100 2rd+1wr 101/110/111 no bank (extended opcode space) - val enableBits = func7(6, 4) - val hasRd0 = enableBits === 1.U || enableBits === 3.U || enableBits === 4.U - val hasRd1 = enableBits === 4.U - val hasWr = enableBits === 2.U || enableBits === 3.U || enableBits === 4.U - - // funct7 -> bid lookup, generated from b.ballDomain.ballISA - require(!b.ballDomain.ballISA.exists(_.funct7 == BallISA.InitFunct), "BALL_INIT funct7 is framework-reserved") - - val bidWire = WireDefault(0.U(5.W)) - val isaValidWire = WireDefault(false.B) - b.ballDomain.ballISA.foreach { e => - when(func7 === e.funct7.U(7.W)) { - bidWire := e.bid.U - isaValidWire := true.B - } - } - - val isBallInit = func7 === BallISA.InitFunct.U - val decodedBid = Mux(isBallInit, rs2(4, 0), bidWire) - - ball_decode_cmd_o.valid := cmd_i.valid && (cmd_i.bits.domain_id === DomainId.BALL) - ball_decode_cmd_o.bits.bid := Mux(ball_decode_cmd_o.valid, decodedBid, 0.U) - ball_decode_cmd_o.bits.funct7 := Mux(ball_decode_cmd_o.valid, func7, 0.U) - ball_decode_cmd_o.bits.iter := Mux(ball_decode_cmd_o.valid, iter_raw, 0.U(iterLen.W)) - ball_decode_cmd_o.bits.special := Mux(ball_decode_cmd_o.valid, rs2, 0.U(64.W)) - ball_decode_cmd_o.bits.op1_bank := Mux(ball_decode_cmd_o.valid && hasRd0, op1_bank_raw, 0.U) - ball_decode_cmd_o.bits.op2_bank := Mux(ball_decode_cmd_o.valid && hasRd1, op2_bank_raw, 0.U) - ball_decode_cmd_o.bits.wr_bank := Mux(ball_decode_cmd_o.valid && hasWr, wr_bank_raw, 0.U) - ball_decode_cmd_o.bits.op1_col := Mux(ball_decode_cmd_o.valid && hasRd0, cmd_i.bits.op1_col, 0.U) - ball_decode_cmd_o.bits.op2_col := Mux(ball_decode_cmd_o.valid && hasRd1, cmd_i.bits.op2_col, 0.U) - ball_decode_cmd_o.bits.wr_col := Mux(ball_decode_cmd_o.valid && hasWr, cmd_i.bits.wr_col, 0.U) - ball_decode_cmd_o.bits.op1_en := ball_decode_cmd_o.valid && hasRd0 - ball_decode_cmd_o.bits.op2_en := ball_decode_cmd_o.valid && hasRd1 - ball_decode_cmd_o.bits.wr_spad_en := ball_decode_cmd_o.valid && hasWr - ball_decode_cmd_o.bits.op1_from_spad := ball_decode_cmd_o.valid && hasRd0 - ball_decode_cmd_o.bits.op2_from_spad := ball_decode_cmd_o.valid && hasRd1 - - // MMIO meta_bank: default to wr_bank (typical case: Ball's output bank - // is bound to a per-output-block MMIO scale region). - // Balls that need different binding can override via their decode logic. - ball_decode_cmd_o.bits.meta_bank := Mux(ball_decode_cmd_o.valid && hasWr, wr_bank_raw, 0.U) - - ball_decode_cmd_o.bits.rs1 := rs1 - ball_decode_cmd_o.bits.rs2 := rs2 - - assert( - !(cmd_i.fire && (cmd_i.bits.domain_id === DomainId.BALL) && !isBallInit && !isaValidWire), - "BallDomainDecoder: funct7 has no matching entry in ballISA" - ) - when(cmd_i.fire && isBallInit) { - assert(rs1 === 0.U, "BALL_INIT requires rs1=0") - assert(rs2(63, 5) === 0.U, "BALL_INIT reserves rs2[63:5]") - assert( - VecInit(b.ballDomain.ballIdMappings.map(m => rs2(4, 0) === m.ballId.U)).asUInt.orR, - "BALL_INIT target ID is not configured" - ) - } - assert( - !(ball_decode_cmd_o.valid && ball_decode_cmd_o.bits.op1_en && ball_decode_cmd_o.bits.op2_en && - ball_decode_cmd_o.bits.op1_bank === ball_decode_cmd_o.bits.op2_bank), - "BallDomainDecoder: Ball instruction OpA and OpB cannot access the same bank" - ) -} diff --git a/arch/src/main/scala/framework/balldomain/isa/BallISA.scala b/arch/src/main/scala/framework/balldomain/isa/BallISA.scala deleted file mode 100644 index ad7dbfac1..000000000 --- a/arch/src/main/scala/framework/balldomain/isa/BallISA.scala +++ /dev/null @@ -1,14 +0,0 @@ -package framework.balldomain.isa - -import framework.system.core.rocket.BuckyballCommand - -/** Framework-reserved BallDomain instructions. */ -object BallISA { - val InitFunct = 0x05 - - def init(ballId: Int): BuckyballCommand = { - require(ballId >= 0 && ballId < 32, s"BALL_INIT target ID out of range: $ballId") - BuckyballCommand(InitFunct, rs1 = 0, rs2 = BigInt(ballId)) - } - -} diff --git a/arch/src/main/scala/framework/balldomain/rs/interfaces.scala b/arch/src/main/scala/framework/balldomain/rs/interfaces.scala deleted file mode 100644 index a19dbe71b..000000000 --- a/arch/src/main/scala/framework/balldomain/rs/interfaces.scala +++ /dev/null @@ -1,22 +0,0 @@ -package framework.balldomain.rs - -import chisel3._ -import chisel3.util._ -import framework.top.GlobalConfig -import framework.balldomain.decoder.BallDecodeCmd - -// Ball domain issue interface - includes global rob_id -class BallRsIssue(b: GlobalConfig) extends Bundle { - val cmd = new BallDecodeCmd(b.memDomain.virtualBankCount, b.frontend.iter_len, b.memDomain.groupCountWidth) - // Global ROB ID - val rob_id = UInt(log2Up(b.frontend.rob_entries).W) - val is_sub = Bool() - val sub_rob_id = UInt(log2Up(b.frontend.sub_rob_depth * 4).W) -} - -// Ball domain completion interface -class BallRsComplete(b: GlobalConfig) extends Bundle { - val rob_id = UInt(log2Up(b.frontend.rob_entries).W) - val is_sub = Bool() - val sub_rob_id = UInt(log2Up(b.frontend.sub_rob_depth * 4).W) -} diff --git a/arch/src/main/scala/framework/balldomain/rs/reservationStation.scala b/arch/src/main/scala/framework/balldomain/rs/reservationStation.scala deleted file mode 100644 index 0fac1316e..000000000 --- a/arch/src/main/scala/framework/balldomain/rs/reservationStation.scala +++ /dev/null @@ -1,121 +0,0 @@ -package framework.balldomain.rs - -import memcore.memory.queue.Queue - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.top.GlobalConfig -import framework.balldomain.decoder.BallDecodeCmd - -// Generic Ball domain issue interface - supports dynamic number of Ball devices -class BallIssueInterface(b: GlobalConfig) extends Bundle { - val balls = Vec(b.ballDomain.ballNum, Decoupled(new BallRsIssue(b))) -} - -// Generic Ball domain completion interface - supports dynamic number of Ball devices -class BallCommitInterface(b: GlobalConfig) extends Bundle { - val balls = Vec(b.ballDomain.ballNum, Flipped(Decoupled(new BallRsComplete(b)))) -} - -// Local Ball reservation station - simple FIFO scheduler -@instantiable -class BallReservationStation(val b: GlobalConfig) extends Module { - - @public - val ball_decode_cmd_i = IO(Flipped(new DecoupledIO(new Bundle { - val cmd = new BallDecodeCmd(b.memDomain.virtualBankCount, b.frontend.iter_len, b.memDomain.groupCountWidth) - // Global ROB ID - val rob_id = UInt(log2Up(b.frontend.rob_entries).W) - val is_sub = Bool() - val sub_rob_id = UInt(log2Up(b.frontend.sub_rob_depth * 4).W) - }))) - - // Rs -> BallController (multi-channel issue) - @public - val issue_o = IO(new BallIssueInterface(b)) - - @public - val commit_i = IO(new BallCommitInterface(b)) - - // Output completion signal (with global rob_id, single channel) - @public - val complete_o = IO(Decoupled(new BallRsComplete(b))) - - // Simple FIFO queue, only for buffering - val fifo = Module(new Queue( - new Bundle { - val cmd = new BallDecodeCmd(b.memDomain.virtualBankCount, b.frontend.iter_len, b.memDomain.groupCountWidth) - val rob_id = UInt(log2Up(b.frontend.rob_entries).W) - val is_sub = Bool() - val sub_rob_id = UInt(log2Up(b.frontend.sub_rob_depth * 4).W) - }, - entries = 4 - )) // Small buffer is sufficient - -// ----------------------------------------------------------------------------- -// Inbound - FIFO enqueue -// ----------------------------------------------------------------------------- - fifo.io.enq <> ball_decode_cmd_i - -// ----------------------------------------------------------------------------- -// Outbound - instruction issue (dispatch to corresponding Ball device based on bid) -// ----------------------------------------------------------------------------- - val headEntry = fifo.io.deq.bits - - // Build ballId to index mapping from config - // Config order should match the order in busRegister.scala - // Each index in issue_o.balls corresponds to a ball registered in BBus - val numConfiguredBalls = b.ballDomain.ballIdMappings.length - - // Set issue signals for each Ball device - // Use configured ball id mappings: index i in issue_o.balls corresponds to ballIdMappings(i).ballId - for (i <- 0 until b.ballDomain.ballNum) { - if (i < numConfiguredBalls) { - val configuredBallId = b.ballDomain.ballIdMappings(i).ballId.U - issue_o.balls(i).valid := fifo.io.deq.valid && headEntry.cmd.bid === configuredBallId - issue_o.balls(i).bits.cmd := headEntry.cmd - issue_o.balls(i).bits.rob_id := headEntry.rob_id - issue_o.balls(i).bits.is_sub := headEntry.is_sub - issue_o.balls(i).bits.sub_rob_id := headEntry.sub_rob_id - } else { - // Unused slots - no valid signal - issue_o.balls(i).valid := false.B - issue_o.balls(i).bits.cmd := DontCare - issue_o.balls(i).bits.rob_id := DontCare - issue_o.balls(i).bits.is_sub := DontCare - issue_o.balls(i).bits.sub_rob_id := DontCare - } - } - - // FIFO deq.ready - can only dequeue when target Ball device is ready - // Find which index corresponds to the requested ballId - fifo.io.deq.ready := VecInit( - (0 until b.ballDomain.ballNum).map { idx => - if (idx < numConfiguredBalls) { - val configuredBallId = b.ballDomain.ballIdMappings(idx).ballId.U - (headEntry.cmd.bid === configuredBallId) && issue_o.balls(idx).ready - } else { - false.B - } - } - ).asUInt.orR - -// ----------------------------------------------------------------------------- -// Completion signal processing - directly forward to global RS -// ----------------------------------------------------------------------------- - val completeArb = Module(new Arbiter(new BallRsComplete(b), b.ballDomain.ballNum)) - - // Connect completion signals from all Ball devices to arbiter - for (i <- 0 until b.ballDomain.ballNum) { - completeArb.io.in(i).valid := commit_i.balls(i).valid - completeArb.io.in(i).bits := commit_i.balls(i).bits - commit_i.balls(i).ready := completeArb.io.in(i).ready - } - - // Forward completion signal (with global rob_id) - complete_o.valid := completeArb.io.out.valid - complete_o.bits := completeArb.io.out.bits - completeArb.io.out.ready := complete_o.ready - -} diff --git a/arch/src/main/scala/framework/build.sc b/arch/src/main/scala/framework/build.sc new file mode 100644 index 000000000..12de97e80 --- /dev/null +++ b/arch/src/main/scala/framework/build.sc @@ -0,0 +1,236 @@ +import mill._ +import mill.scalalib._ +import $file.device.{build => device_build} +import $file.core.{build => core_build} +import $file.thread.{build => thread_build} +import $file.acc.{build => acc_build} +import $file.ip.{build => ip_build} +import $file.lib.{build => lib_build} +import $file.acc.balldomain.{build => acc_balldomain_build} +import $file.acc.frontend.{build => acc_frontend_build} +import $file.acc.frontend.src.main.scala.boot.{build => acc_frontend_boot_build} +import $file.acc.frontend.src.main.scala.scheduler.{build => acc_frontend_scheduler_build} +import $file.acc.frontend.src.main.scala.scheduler.src.main.scala.bat.{build => acc_frontend_scheduler_bat_build} +import $file.acc.frontend.src.main.scala.scheduler.src.main.scala.rob.global.{build => acc_frontend_scheduler_rob_global_build} +import $file.acc.frontend.src.main.scala.scheduler.src.main.scala.scoreboard.{build => acc_frontend_scheduler_scoreboard_build} +import $file.acc.gpdomain.{build => acc_gpdomain_build} +import $file.acc.gpdomain.src.main.scala.execution.{build => acc_gpdomain_execution_build} +import $file.acc.gpdomain.src.main.scala.loader.{build => acc_gpdomain_loader_build} +import $file.acc.gpdomain.src.main.scala.memory.{build => acc_gpdomain_memory_build} +import $file.acc.memdomain.{build => acc_memdomain_build} +import $file.acc.memdomain.src.main.scala.bank_router.{build => acc_memdomain_bank_router_build} +import $file.acc.memdomain.src.main.scala.prm.{build => acc_memdomain_prm_build} +import $file.acc.memdomain.src.main.scala.tomem.{build => acc_memdomain_tomem_build} +import $file.acc.memdomain.src.main.scala.tomem.src.main.scala.toddr.{build => acc_memdomain_tomem_toddr_build} +import $file.acc.memdomain.src.main.scala.tomem.src.main.scala.tol1.{build => acc_memdomain_tomem_tol1_build} +import $file.core.controller.admission.{build => core_controller_admission_build} +import $file.core.execution.fetch.{build => core_execution_fetch_build} +import $file.core.execution.fpu.{build => core_execution_fpu_build} +import $file.core.execution.instruction.{build => core_execution_instruction_build} +import $file.core.execution.integer.{build => core_execution_integer_build} +import $file.core.execution.muldiv.{build => core_execution_muldiv_build} +import $file.core.execution.privileged.{build => core_execution_privileged_build} +import $file.core.execution.register_file.{build => core_execution_register_file_build} +import $file.core.memory.access.{build => core_memory_access_build} +import $file.core.memory.access.src.main.scala.cache.{build => core_memory_access_cache_build} +import $file.core.memory.access.src.main.scala.mmu.{build => core_memory_access_mmu_build} +import $file.core.memory.access.src.main.scala.preflight.{build => core_memory_access_preflight_build} +import $file.core.memory.home.{build => core_memory_home_build} +import $file.core.tbus.{build => core_tbus_build} +import $file.core.tbus.src.main.scala.router.{build => core_tbus_router_build} +import $file.core.tss.{build => core_tss_build} +import $file.device.ddr.{build => device_ddr_build} +import $file.ip.axi4.{build => ip_axi4_build} +import $file.ip.axis.{build => ip_axis_build} +import $file.ip.bank.{build => ip_bank_build} +import $file.ip.cache.{build => ip_cache_build} +import $file.ip.chi.{build => ip_chi_build} +import $file.ip.coherence.{build => ip_coherence_build} +import $file.ip.interlock.{build => ip_interlock_build} +import $file.ip.mesh_shm.{build => ip_mesh_shm_build} +import $file.ip.spm.{build => ip_spm_build} +import $file.thread.ant.{build => thread_ant_build} +import $file.^.^.^.^.thirdparty.{build => thirdparty_build} + +val frameworkRoot: os.Path = millSourcePath +val resourceRoot: os.Path = millSourcePath / os.up / os.up / "resources" + +object library extends SbtModule { + override def resources = T.sources(Seq(PathRef(resourceRoot))) + override def scalaVersion = "2.13.16" + override def moduleDeps = Seq(thirdparty_build.protoJava, thirdparty_build.hardfloat) + override def ivyDeps = + Agg(ivy"org.chipsalliance::chisel:6.7.0", ivy"com.lihaoyi::upickle:3.3.1") + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") + + override def sources = T.sources( + device_build.sources() ++ core_build.sources() ++ thread_build.sources() ++ + acc_build.sources() ++ ip_build.sources() ++ lib_build.sources() ++ + Seq(PathRef(frameworkRoot / os.up / "configs"), PathRef(frameworkRoot / os.up / "sims" / "configs")) + ) + +} + +object emitters extends SbtModule { + override def scalaVersion = "2.13.16" + override def moduleDeps = Seq(library) + override def ivyDeps = library.ivyDeps() + override def scalacPluginIvyDeps = library.scalacPluginIvyDeps() + override def scalacOptions = library.scalacOptions() + override def sources = T.sources( + device_build.emitSources() ++ core_build.emitSources() ++ thread_build.emitSources() ++ + acc_build.emitSources() ++ ip_build.emitSources() ++ lib_build.emitSources() + ) +} + +object acc extends Module { + object balldomain extends acc_balldomain_build.Emitter { + override def framework = library + } + object frontend extends acc_frontend_build.Emitter { + override def framework = library + object boot extends acc_frontend_boot_build.Emitter { + override def framework = library + } + object scheduler extends acc_frontend_scheduler_build.Emitter { + override def framework = library + object bat extends acc_frontend_scheduler_bat_build.Emitter { + override def framework = library + } + object rob extends Module { + object global extends acc_frontend_scheduler_rob_global_build.Emitter { + override def framework = library + } + } + object scoreboard extends acc_frontend_scheduler_scoreboard_build.Emitter { + override def framework = library + } + } + } + object gpdomain extends acc_gpdomain_build.Emitter { + override def framework = library + object execution extends acc_gpdomain_execution_build.Emitter { + override def framework = library + } + object loader extends acc_gpdomain_loader_build.Emitter { + override def framework = library + } + object memory extends acc_gpdomain_memory_build.Emitter { + override def framework = library + } + } + object memdomain extends acc_memdomain_build.Emitter { + override def framework = library + object bank_router extends acc_memdomain_bank_router_build.Emitter { + override def framework = library + } + object prm extends acc_memdomain_prm_build.Emitter { + override def framework = library + } + object tomem extends acc_memdomain_tomem_build.Emitter { + override def framework = library + object toddr extends acc_memdomain_tomem_toddr_build.Emitter { + override def framework = library + } + object tol1 extends acc_memdomain_tomem_tol1_build.Emitter { + override def framework = library + } + } + } +} +object core extends Module { + object controller extends Module { + object admission extends core_controller_admission_build.Emitter { + override def framework = library + } + } + object execution extends Module { + object fetch extends core_execution_fetch_build.Emitter { + override def framework = library + } + object fpu extends core_execution_fpu_build.Emitter { + override def framework = library + } + object instruction extends core_execution_instruction_build.Emitter { + override def framework = library + } + object integer extends core_execution_integer_build.Emitter { + override def framework = library + } + object muldiv extends core_execution_muldiv_build.Emitter { + override def framework = library + } + object privileged extends core_execution_privileged_build.Emitter { + override def framework = library + } + object register_file extends core_execution_register_file_build.Emitter { + override def framework = library + } + } + object memory extends Module { + object access extends core_memory_access_build.Emitter { + override def framework = library + object cache extends core_memory_access_cache_build.Emitter { + override def framework = library + } + object mmu extends core_memory_access_mmu_build.Emitter { + override def framework = library + } + object preflight extends core_memory_access_preflight_build.Emitter { + override def framework = library + } + } + object home extends core_memory_home_build.Emitter { + override def framework = library + } + } + object tbus extends core_tbus_build.Emitter { + override def framework = library + object router extends core_tbus_router_build.Emitter { + override def framework = library + } + } + object tss extends core_tss_build.Emitter { + override def framework = library + } +} +object device extends Module { + object ddr extends device_ddr_build.Emitter { + override def framework = library + } +} +object ip extends Module { + object axi4 extends ip_axi4_build.Emitter { + override def framework = library + } + object axis extends ip_axis_build.Emitter { + override def framework = library + } + object bank extends ip_bank_build.Emitter { + override def framework = library + } + object cache extends ip_cache_build.Emitter { + override def framework = library + } + object chi extends ip_chi_build.Emitter { + override def framework = library + } + object coherence extends ip_coherence_build.Emitter { + override def framework = library + } + object interlock extends ip_interlock_build.Emitter { + override def framework = library + } + object mesh_shm extends ip_mesh_shm_build.Emitter { + override def framework = library + } + object spm extends ip_spm_build.Emitter { + override def framework = library + } +} +object thread extends Module { + object ant extends thread_ant_build.Emitter { + override def framework = library + } +} diff --git a/arch/src/main/scala/framework/core/build.sc b/arch/src/main/scala/framework/core/build.sc new file mode 100644 index 000000000..0685199c5 --- /dev/null +++ b/arch/src/main/scala/framework/core/build.sc @@ -0,0 +1,31 @@ +import mill._ +import $file.clink.{build => clink_build} +import $file.controller.{build => controller_build} +import $file.execution.{build => execution_build} +import $file.memory.{build => memory_build} +import $file.shm.{build => shm_build} +import $file.tbus.{build => tbus_build} +import $file.tss.{build => tss_build} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("configs/CoreParam.scala")) + ) ++ + clink_build.sources() ++ + controller_build.sources() ++ + execution_build.sources() ++ + memory_build.sources() ++ + shm_build.sources() ++ + tbus_build.sources() ++ + tss_build.sources() +) + +def emitSources = T.sources( + clink_build.emitSources() ++ + controller_build.emitSources() ++ + execution_build.emitSources() ++ + memory_build.emitSources() ++ + shm_build.emitSources() ++ + tbus_build.emitSources() ++ + tss_build.emitSources() +) diff --git a/arch/src/main/scala/framework/core/clink/basecore.scala b/arch/src/main/scala/framework/core/clink/basecore.scala new file mode 100644 index 000000000..fb2f9b82c --- /dev/null +++ b/arch/src/main/scala/framework/core/clink/basecore.scala @@ -0,0 +1,5 @@ +package framework.core.clink + +trait HasCLink { + def clink: CLinkIO +} diff --git a/arch/src/main/scala/framework/core/clink/build.sc b/arch/src/main/scala/framework/core/clink/build.sc new file mode 100644 index 000000000..956ae69c1 --- /dev/null +++ b/arch/src/main/scala/framework/core/clink/build.sc @@ -0,0 +1,13 @@ +import mill._ + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("basecore.scala")), + PathRef(millSourcePath / os.RelPath("clink.scala")), + PathRef(millSourcePath / os.RelPath("configs/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq.empty[PathRef] +) diff --git a/arch/src/main/scala/framework/core/clink/clink.scala b/arch/src/main/scala/framework/core/clink/clink.scala new file mode 100644 index 000000000..6fea650ff --- /dev/null +++ b/arch/src/main/scala/framework/core/clink/clink.scala @@ -0,0 +1,68 @@ +package framework.core.clink + +import framework.core.clink.configs.{Params => CLinkParam} +import framework.ip.bus.chi +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.acc.memdomain.{MvoverPort, PhysicalBankHash} +import framework.thread.tlink.BankAccess + +import chisel3._ +import chisel3.util._ +import framework.ip.bus.axi4 +import framework.ip.bus.chi.snf.{LineRequest, LineResponse} + +class SharedPort(axi: axi4.Params) extends Bundle { + val tx = new axi4.Port(axi) + val rx = Flipped(new axi4.Port(axi)) +} + +/** An explicitly attached shared-bank fabric; it does not imply hardware threads. */ +class CoreBanks(geometry: Option[(FrontendParam, MemDomainParam, Int)], diff: Boolean) extends Bundle { + val access = geometry.map { case (frontend, memory, ports) => new BankAccess(frontend, memory) } + val move = new MvoverPort + + val barrier = Option.when(geometry.nonEmpty)(new Bundle { + val arrive = Output(Bool()) + val release = Input(Bool()) + }) + + val hashes = geometry.filter(_ => diff).map { case (frontend, memory, _) => + Input(Vec(memory.sharedBankNum, new PhysicalBankHash(frontend))) + } + +} + +/** Core memory protocols and hardware interrupt notifications behind CLink. */ +class CLinkIO( + val p: CLinkParam, + bankInterface: Option[CoreBanks] = None) + extends Bundle { + + val banks = bankInterface.map(_.cloneType) + + val interrupts = Input(Vec(p.harts, new CoreInterrupts)) + + val memory = new Bundle { + + val coherent = Vec(p.requesters, new chi.RequesterPort(p.line)) + + val backing = Option.when(p.backing)(new Bundle { + val request = Decoupled(new LineRequest(p.line)) + val response = Flipped(Decoupled(new LineResponse(p.line))) + }) + + val uncached = new axi4.Port(p.uncached) + + val ddr = new axi4.Port(p.ddr) + } + + val c2c = p.c2c.map(new SharedPort(_)) +} + +class CoreInterrupts extends Bundle { + val timer = Bool() + val software = Bool() + val external = Bool() + val supervisorExternal = Bool() +} diff --git a/arch/src/main/scala/framework/core/clink/configs/Params.scala b/arch/src/main/scala/framework/core/clink/configs/Params.scala new file mode 100644 index 000000000..aeed8283d --- /dev/null +++ b/arch/src/main/scala/framework/core/clink/configs/Params.scala @@ -0,0 +1,23 @@ +package framework.core.clink.configs + +import framework.ip.bus.{axi4, chi} +import upickle.default.{macroRW, ReadWriter} + +case class Params( + harts: Int, + backing: Boolean, + requesterIds: Seq[Int], + line: chi.Params, + uncached: axi4.Params, + ddr: axi4.Params, + c2c: Option[axi4.Params]) { + def requesters: Int = requesterIds.size + require( + requesterIds.distinct.size == requesterIds.size && requesterIds.forall(_ > 0), + "Invalid CHI requester node IDs" + ) + def shared: Boolean = c2c.nonEmpty + require(!(backing && requesters > 0), "A core exports either backing requests or a coherent requester") +} + +object Params { implicit val rw: ReadWriter[Params] = macroRW } diff --git a/arch/src/main/scala/framework/core/configs/CoreParam.scala b/arch/src/main/scala/framework/core/configs/CoreParam.scala new file mode 100644 index 000000000..ca364d8c1 --- /dev/null +++ b/arch/src/main/scala/framework/core/configs/CoreParam.scala @@ -0,0 +1,211 @@ +package framework.core.configs + +import buckyball.config.{CorePlacement, Design} +import framework.acc.configs.AccParam +import framework.core.tbus.router.configs.{Params => RouterParam} +import scala.jdk.CollectionConverters._ +import framework.ip.memory.spm +import framework.core.memory.access.preflight.configs.PmpParams + +import framework.core.execution.fetch.configs.{Params => FetchParams} +import framework.core.execution.instruction.configs.{Params => InstructionParams} +import framework.core.execution.integer.configs.{Params => IntegerParams} +import framework.core.execution.muldiv.configs.{Params => MulDivParams} +import framework.core.execution.fpu.configs.{Params => FpuParams} +import framework.core.execution.privileged.configs.{Params => PrivilegedParams} +import framework.core.execution.register_file.configs.{Params => RegisterFileParams} +import framework.core.controller.admission.configs.{Params => AdmissionParams} +import framework.thread.configs.ThreadParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.core.clink.configs.{Params => CLinkParams} +import framework.core.tbus.configs.{Params => TBusParams} +import framework.core.tss.configs.{Params => TaskStorageParams} +import framework.device.cbus.C2CParams +import framework.ip.bus.chi.rnf.RnfParams +import framework.ip.bus.axi4.{InterconnectParam, Params => AxiParam} +import framework.core.execution.configs.ExecutionParam +import framework.core.memory.configs.MemoryParam +import framework.core.controller.configs.{CommandsParam, ControllerParam, TasksParam} + +case class CoreParam( + hartId: Int, + resetVector: BigInt, + mtimeAddress: BigInt, + clink: CLinkParams, + tbus: Option[TBusParams], + tss: Option[TaskStorageParams], + c2c: C2CParams, + transferLocalAccess: Boolean, + transferLeaseAccess: Boolean, + dmaCache: Option[RnfParams], + interconnect: InterconnectParam, + threads: Seq[ThreadParam], + acc: Option[AccParam], + memory: MemoryParam, + fetch: FetchParams, + instruction: InstructionParams, + integer: IntegerParams, + muldiv: MulDivParams, + fpu: FpuParams, + privileged: PrivilegedParams, + registerFile: RegisterFileParams, + admission: AdmissionParams) { + + // TBus uses the bank/tag widths, not a thread's private scheduling policy. + def sharedFrontend: FrontendParam = threads.map(_.acc.get.frontend).reduce { (left, right) => + left.copy( + bank_id_len = math.max(left.bank_id_len, right.bank_id_len), + rob_entries = math.max(left.rob_entries, right.rob_entries) + ) + } + + // Physical shared SRAM geometry must agree. Private capacity and virtual namespaces may differ. + def sharedMemory: MemDomainParam = threads.map(_.acc.get.memDomain).reduce { (left, right) => + require( + ( + left.bankWidth, + left.bankMaskLen, + left.sharedEnable, + left.sharedBankNum, + left.sharedBankEntries, + left.sharedInputChannels + ) == + (right.bankWidth, right.bankMaskLen, right.sharedEnable, right.sharedBankNum, right.sharedBankEntries, right.sharedInputChannels), + "Threads attached to one TBus must agree on the shared SRAM interface" + ) + left.copy( + bankNum = math.max(left.bankNum, right.bankNum), + bankEntries = left.sharedBankEntries, + virtualBankCount = math.max(left.virtualBankCount, right.virtualBankCount) + ) + } + + def controller: ControllerParam = { + val taskParams = Option.when(threads.nonEmpty)( + TasksParam(threads.map(_.ant.get), threads.map(_.signature), tss.get, privileged.permission) + ) + ControllerParam( + CommandsParam( + memory.dcache.tracking, + integer.xLen, + privileged.permission, + admission.moves, + threads.nonEmpty, + clink.shared + ), + taskParams + ) + } + + def execution(directAccelerator: Boolean): ExecutionParam = ExecutionParam( + hartId, + resetVector, + threads.nonEmpty || clink.shared || directAccelerator, + directAccelerator, + admission.moves, + memory.dcache.tracking, + mtimeAddress, + memory.memoryAccess, + instruction, + registerFile, + integer, + muldiv, + fpu, + privileged + ) + +} + +object CoreParam { + + def fromProto( + design: Design, + core: CorePlacement, + externalHome: Boolean + ): CoreParam = { + val threads = core.getThreadIndicesList.asScala.map { index => + val thread = design.getThreads(index.intValue) + require(thread.getIndex == index.intValue) + ThreadParam.fromProto(thread, core) + }.toSeq + val corep = fromProto(core, threads) + corep.copy(clink = + corep.clink.copy( + backing = !externalHome, + requesterIds = + if (externalHome) + Seq(corep.memory.icache.nodeId, corep.memory.dcache.cache.nodeId) ++ corep.dmaCache.map(_.nodeId) + else Seq.empty + ) + ) + } + + def fromProto(p: CorePlacement, threads: Seq[ThreadParam]): CoreParam = { + require( + p.hasMemory && p.hasParam && p.hasSharedMem && (threads.isEmpty || p.hasTss), + "Core configuration is incomplete" + ) + require(p.getThreadIndicesCount == threads.size, "Core thread configuration is incomplete") + val accp = Option.when(p.hasAccelerator)(AccParam.fromProto(p)) + val memp = MemoryParam.fromProto(p.getMemory, threads.size) + val cp = p.getParam + val permission = PmpParams(cp.getPaddrBits, cp.getNPmps) + val shared = p.getSharedMem + val tss = p.getTss + val ids = threads.map(_.id) + val directShared = threads.isEmpty && accp.exists(_.memDomain.sharedEnable) + val participants = if (directShared) Seq(0) else ids + val crossbar = AxiParam.fromProto(p.getInterconnect.getAxi) + val c2c = p.getC2C + val c2cAxi = AxiParam.fromProto(c2c.getAxi) + require(ids == ids.indices, "TBus thread IDs must be ordered and contiguous") + CoreParam( + hartId = p.getHartId, + resetVector = BigInt(java.lang.Long.toUnsignedString(p.getResetVector)), + mtimeAddress = BigInt(java.lang.Long.toUnsignedString(p.getMtimeAddress)), + clink = CLinkParams( + 1, + true, + Seq.empty, + memp.memoryAccess.memory.chi, + memp.uncached.axi, + accp.map(_.transport.axi).getOrElse(crossbar), + Option.when(c2c.getEnabled)(c2cAxi) + ), + tbus = Option.when(p.hasTbus)(TBusParams( + RouterParam(if (c2c.getEnabled) 3 + (if (directShared) 1 else 0) else 0), + participants, + participants.size, + p.getTbus.getBankNodesList.asScala.map(_.intValue).toSeq, + !directShared && p.getController.getMoves + )), + tss = Option.when(threads.nonEmpty)(TaskStorageParams( + spm.Params(BigInt(java.lang.Long.toUnsignedString(tss.getBase)), tss.getBytes, tss.getDataBits), + ids.size + )), + c2c = C2CParams( + BigInt(shared.getEntries) * (shared.getBankWidth / 8), + shared.getBankEntries * (shared.getBankWidth / 8), + c2c.getPeerCores, + c2cAxi + ), + transferLocalAccess = directShared && c2c.getEnabled, + transferLeaseAccess = c2c.getEnabled, + dmaCache = Option.when(p.hasDmaCache)(RnfParams.fromProto(p.getDmaCache, memp.home.memory.chi)), + interconnect = InterconnectParam(crossbar, ids.size, Some(p.getInterconnect.getSlots)), + threads = threads, + acc = accp, + memory = memp, + fetch = FetchParams(p.getFetch.getInstructionBytes), + instruction = InstructionParams(cp.getXLen), + integer = IntegerParams(cp.getXLen), + muldiv = MulDivParams(cp.getXLen), + fpu = FpuParams(cp.getXLen, p.getFpu.getFLen), + privileged = PrivilegedParams(permission), + registerFile = RegisterFileParams(cp.getXLen), + admission = AdmissionParams(permission, p.getController.getMoves) + ) + } + +} diff --git a/arch/src/main/scala/framework/core/controller/Controller.scala b/arch/src/main/scala/framework/core/controller/Controller.scala new file mode 100644 index 000000000..d135653a1 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/Controller.scala @@ -0,0 +1,54 @@ +package framework.core.controller + +import framework.thread.ant.Task +import framework.ip.memory.spm +import chisel3._ +import chisel3.util.MixedVec +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.core.controller.configs.ControllerParam +import framework.core.controller.admission.{CommandAdmission, Commands, TaskEndpoint, TaskRouter} +import framework.core.controller.task.Service +import framework.acc.memdomain.MvoverPort +import framework.core.tss.TssControlPort + +/** Core command admission, task management and thread launch control. */ +@instantiable +class Controller(b: ControllerParam) extends Module { + val commands = b.commands + + @public + val io = IO(new Bundle { + + val execution = Flipped(new Commands(commands.tracking, commands.dataBits, commands.permission)) + val threads = b.tasks.map(p => MixedVec(p.locals.map(local => Flipped(new Task(local, p.permission))))) + val move = new MvoverPort + val remote = Option.when(commands.remote)(new TaskEndpoint) + val tss = b.tasks.map(p => new TssControlPort(p.tss)) + val memoryAllowed = Output(Bool()) + val drained = Output(Bool()) + }) + + val admission: Instance[CommandAdmission] = Instantiate(new CommandAdmission(commands)) + val router: Instance[TaskRouter] = Instantiate(new TaskRouter(commands)) + admission.io.execution <> io.execution + admission.io.move <> io.move + router.io.command <> admission.io.taskRequest + admission.io.taskResponse <> router.io.response + io.remote.zip(router.io.remote).foreach { case (port, endpoint) => port <> endpoint } + io.memoryAllowed := admission.io.memoryAllowed + io.drained := admission.io.idle && !router.io.busy + + if (b.tasks.nonEmpty) { + val service: Instance[Service] = Instantiate(new Service(b.tasks.get)) + service.io.request <> router.io.local.request + router.io.local.reply <> service.io.reply + router.io.local.busy := !service.io.idle + io.tss.get <> service.io.tss + service.io.tasks <> io.threads.get + } else { + router.io.local.request.ready := false.B + router.io.local.reply.valid := false.B + router.io.local.reply.bits := 0.U + router.io.local.busy := false.B + } +} diff --git a/arch/src/main/scala/framework/core/controller/admission/build.sc b/arch/src/main/scala/framework/core/controller/admission/build.sc new file mode 100644 index 000000000..43fc6a4c7 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/build.sc @@ -0,0 +1,23 @@ +import mill._ +import $file.^.^.^.^.^.^.^.settings + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.core.controller.admission.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/CommandAdmission.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/DirectAdmission.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/TaskRouter.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/core/controller/admission/configs/uvm.toml b/arch/src/main/scala/framework/core/controller/admission/configs/uvm.toml new file mode 100644 index 000000000..5669ed6cb --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/configs/uvm.toml @@ -0,0 +1,15 @@ +[parameters] +dataBits=64 +moves=true +taskEnabled=true +remote=true +[parameters.permission] +addressBits=44 +entries=4 +[parameters.tracking] +entries=4 +addressBits=44 +idBits=8 +lineBytes=64 +maxRanges=8 +[defines] diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/env/env.svh b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/env/env.svh new file mode 100644 index 000000000..1f8748f8e --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/env/env.svh @@ -0,0 +1,25 @@ +class env extends uvm_env; + `uvm_component_utils(env) + driver source; + monitor mon; + model reference; + uvm_sequencer #(cycle_item) seqr; + result_board #(result_item, 0) scoreboard; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + source = driver::type_id::create("source", this); + mon = monitor::type_id::create("monitor", this); + reference = model::type_id::create("model", this); + seqr = new("seqr", this); + scoreboard = result_board#(result_item, 0)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + source.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(reference.analysis_export); + reference.expected.connect(scoreboard.expected_export); + reference.actual.connect(scoreboard.actual_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/env/model.svh b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/env/model.svh new file mode 100644 index 000000000..16c3c29b9 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/env/model.svh @@ -0,0 +1,148 @@ +class model extends uvm_subscriber #(cycle_item); + `uvm_component_utils(model) + uvm_analysis_port #(result_item) expected, actual; + const int copies[`AD_PAYLOAD_FIELDS][4] = `AD_PAYLOAD_MAP; + bit pending, moving, has_result, work_done, response_done, move_error, remote_owner; + int saved_tag, destination, tags[$]; + covergroup behavior with function sample (int route, bit stalled, bit cancelled); + option.per_instance = 1; + routes: coverpoint route {bins all[] = {[0 : 1]};} + backpressure: coverpoint stalled; + reset_cancellation: coverpoint cancelled; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected = new("expected", this); + actual = new("actual", this); + behavior = new(); + endfunction + function void write(cycle_item t); + logic [`AD_OUTPUT_BITS-1:0] want, mask; + result_item predicted, received; + + if (t.reset) begin + behavior.sample(0, 0, pending || tags.size() != 0); + pending = 0; + tags.delete(); + return; + end + want = '0; + mask = '0; + begin + bit offered, task_command, move_command, write_result; + logic [31:0] instruction; + instruction = `AI(t, EXECUTION_COMMAND_BITS_INSTRUCTION_RAW_INST); + offered = `AI(t, EXECUTION_COMMAND_VALID); + task_command = instruction[6:0] == 'h2b; + move_command = instruction[6:0] == 'h7b && instruction[31:25] == 'h0d; + want[`AD_OUT_EXECUTION_COMMAND_READY_OFFSET+:`AD_OUT_EXECUTION_COMMAND_READY_WIDTH]=!pending&&(task_command? + `AI(t, TASKREQUEST_READY) + : (move_command && + `AI(t, MOVE_COMMAND_READY) + )); + mask[`AD_OUT_EXECUTION_COMMAND_READY_OFFSET+:`AD_OUT_EXECUTION_COMMAND_READY_WIDTH] = '1; + + want[`AD_OUT_TASKREQUEST_VALID_OFFSET+:`AD_OUT_TASKREQUEST_VALID_WIDTH]=!pending&&offered&&task_command; + mask[`AD_OUT_TASKREQUEST_VALID_OFFSET+:`AD_OUT_TASKREQUEST_VALID_WIDTH] = '1; + + want[`AD_OUT_MOVE_COMMAND_VALID_OFFSET+:`AD_OUT_MOVE_COMMAND_VALID_WIDTH]=!pending&&offered&&move_command; + mask[`AD_OUT_MOVE_COMMAND_VALID_OFFSET+:`AD_OUT_MOVE_COMMAND_VALID_WIDTH] = '1; + + want[`AD_OUT_MOVE_COMMAND_BITS_ROWS_OFFSET+:`AD_OUT_MOVE_COMMAND_BITS_ROWS_WIDTH]={1'b0,t.inputs[`AD_IN_EXECUTION_COMMAND_BITS_INSTRUCTION_RS2DATA_OFFSET+32+:16]}+17'd1; + mask[`AD_OUT_MOVE_COMMAND_BITS_ROWS_OFFSET+:`AD_OUT_MOVE_COMMAND_BITS_ROWS_WIDTH] = '1; + + want[`AD_OUT_MOVE_COMPLETION_READY_OFFSET+:`AD_OUT_MOVE_COMPLETION_READY_WIDTH]=pending&&moving&&!work_done; + mask[`AD_OUT_MOVE_COMPLETION_READY_OFFSET+:`AD_OUT_MOVE_COMPLETION_READY_WIDTH] = '1; + + want[`AD_OUT_TASKRESPONSE_READY_OFFSET+:`AD_OUT_TASKRESPONSE_READY_WIDTH]=pending&&!moving&&!work_done&&(!has_result|| + `AI(t, EXECUTION_RESPONSE_READY) + ); + mask[`AD_OUT_TASKRESPONSE_READY_OFFSET+:`AD_OUT_TASKRESPONSE_READY_WIDTH] = '1; + + want[`AD_OUT_EXECUTION_RESPONSE_VALID_OFFSET+:`AD_OUT_EXECUTION_RESPONSE_VALID_WIDTH]=pending&&has_result&&!response_done&&(moving?work_done: + `AI(t, TASKRESPONSE_VALID) + ); + mask[`AD_OUT_EXECUTION_RESPONSE_VALID_OFFSET+:`AD_OUT_EXECUTION_RESPONSE_VALID_WIDTH] = '1; + + if (want[`AD_OUT_EXECUTION_RESPONSE_VALID_OFFSET]) begin + want[`AD_OUT_EXECUTION_RESPONSE_BITS_RD_OFFSET+:`AD_OUT_EXECUTION_RESPONSE_BITS_RD_WIDTH]=moving?destination:`AI( + t, TASKRESPONSE_BITS_RD); + mask[`AD_OUT_EXECUTION_RESPONSE_BITS_RD_OFFSET+:`AD_OUT_EXECUTION_RESPONSE_BITS_RD_WIDTH]='1; + + want[`AD_OUT_EXECUTION_RESPONSE_BITS_DATA_OFFSET+:`AD_OUT_EXECUTION_RESPONSE_BITS_DATA_WIDTH]=moving?64'(move_error):`AI( + t, TASKRESPONSE_BITS_DATA); + mask[`AD_OUT_EXECUTION_RESPONSE_BITS_DATA_OFFSET+:`AD_OUT_EXECUTION_RESPONSE_BITS_DATA_WIDTH]='1; + + end + want[`AD_OUT_EXECUTION_COMPLETE_VALID_OFFSET+:`AD_OUT_EXECUTION_COMPLETE_VALID_WIDTH]=pending&&work_done&&(!has_result||response_done); + mask[`AD_OUT_EXECUTION_COMPLETE_VALID_OFFSET+:`AD_OUT_EXECUTION_COMPLETE_VALID_WIDTH] = '1; + + if (want[`AD_OUT_EXECUTION_COMPLETE_VALID_OFFSET]) begin + want[`AD_OUT_EXECUTION_COMPLETE_BITS_TAG_OFFSET+:`AD_OUT_EXECUTION_COMPLETE_BITS_TAG_WIDTH]=saved_tag; + mask[`AD_OUT_EXECUTION_COMPLETE_BITS_TAG_OFFSET+:`AD_OUT_EXECUTION_COMPLETE_BITS_TAG_WIDTH]='1; + end + want[`AD_OUT_MEMORYALLOWED_OFFSET+:`AD_OUT_MEMORYALLOWED_WIDTH]=!(offered&&move_command||pending&&(moving||work_done)); + mask[`AD_OUT_MEMORYALLOWED_OFFSET+:`AD_OUT_MEMORYALLOWED_WIDTH] = '1; + + want[`AD_OUT_IDLE_OFFSET+:`AD_OUT_IDLE_WIDTH] = !pending; + mask[`AD_OUT_IDLE_OFFSET+:`AD_OUT_IDLE_WIDTH] = '1; + + + end + foreach (copies[field]) + for (int b = 0; b < copies[field][2]; b++) + if (want[copies[field][3]]) begin + want[copies[field][1]+b] = t.inputs[copies[field][0]+b]; + mask[copies[field][1]+b] = 1; + end + predicted = result_item::type_id::create("expected"); + received = result_item::type_id::create("actual"); + predicted.value = want & mask; + received.value = t.outputs & mask; + expected.write(predicted); + actual.write(received); + begin + logic [31:0] instruction; + bit task_command, move_command; + instruction = `AI(t, EXECUTION_COMMAND_BITS_INSTRUCTION_RAW_INST); + task_command = instruction[6:0] == 'h2b; + move_command = instruction[6:0] == 'h7b && instruction[31:25] == 'h0d; + if (`AI(t, EXECUTION_COMMAND_VALID) && `AO(t, EXECUTION_COMMAND_READY)) begin + pending = 1; + moving = move_command; + has_result = instruction[14]; + destination = instruction[11:7]; + saved_tag = `AI(t, EXECUTION_COMMAND_BITS_TAG); + work_done = 0; + response_done = 0; + behavior.sample(move_command, 0, 0); + end + if (`AI(t, TASKRESPONSE_VALID) && `AO(t, TASKRESPONSE_READY)) work_done = 1; + if (`AI(t, MOVE_COMPLETION_VALID) && `AO(t, MOVE_COMPLETION_READY)) begin + work_done = 1; + move_error = `AI(t, MOVE_COMPLETION_BITS); + end + if (`AO(t, EXECUTION_RESPONSE_VALID) && `AI(t, EXECUTION_RESPONSE_READY)) response_done = 1; + if (`AO(t, EXECUTION_COMPLETE_VALID) && `AI(t, EXECUTION_COMPLETE_READY)) pending = 0; + if ( + `AI(t, EXECUTION_COMMAND_VALID) + && ! + `AO(t, EXECUTION_COMMAND_READY) + || + `AO(t, EXECUTION_COMPLETE_VALID) + && ! + `AI(t, EXECUTION_COMPLETE_READY) + || + `AO(t, EXECUTION_RESPONSE_VALID) + && ! + `AI(t, EXECUTION_RESPONSE_READY) + ) + behavior.sample(moving, 1, 0); + + end + endfunction + function void check_phase(uvm_phase phase); + super.check_phase(phase); + if (pending || tags.size() != 0) `uvm_fatal("PENDING", "Admission ownership did not drain") + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/filelist.f b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/filelist.f new file mode 100644 index 000000000..9b229b6fb --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@/CommandAdmission ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/../common/interface.sv +@RESOURCES@/package.sv +-F @RTL@/CommandAdmission/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/package.sv b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/package.sv new file mode 100644 index 000000000..3793ede46 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/package.sv @@ -0,0 +1,17 @@ +`include "parameters.svh" +`define AI(T, F) T.inputs[`AD_IN_``F``_OFFSET+:`AD_IN_``F``_WIDTH] +`define AO(T, F) T.outputs[`AD_OUT_``F``_OFFSET+:`AD_OUT_``F``_WIDTH] +`define EXPECT(F, + V) want[`AD_OUT_``F``_OFFSET+:`AD_OUT_``F``_WIDTH]=V;mask[`AD_OUT_``F``_OFFSET+:`AD_OUT_``F``_WIDTH]='1; +package admission_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + `include "../common/agent/item.svh" + `include "../common/agent/driver.svh" + `include "../common/agent/monitor.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/seq/sequence.svh b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/seq/sequence.svh new file mode 100644 index 000000000..87ff49a49 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/seq/sequence.svh @@ -0,0 +1,88 @@ +class scenario_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(scenario_sequence) + logic [ `AD_INPUT_BITS-1:0] pins; + logic [`AD_OUTPUT_BITS-1:0] observed; + function new(string name = "scenario"); + super.new(name); + endfunction + task tick(bit reset = 0); + cycle_item t = cycle_item::type_id::create("cycle"); + start_item(t); + t.inputs = pins; + t.reset = reset; + finish_item(t); + observed = t.outputs; + endtask + task command(int tag, logic [31:0] instruction, logic [63:0] rs1, logic [63:0] rs2); + for (int b = 0; b < `AD_COMMAND_WIDTH; b++) + pins[`AD_COMMAND_OFFSET+b] = ((b + tag) % 3 == 0) ^ ((b + tag) % 7 == 0); + pins[`AD_IN_EXECUTION_COMMAND_BITS_INSTRUCTION_RAW_INST_OFFSET+:32]=instruction|((tag%32)<<7); + pins[`AD_IN_EXECUTION_COMMAND_BITS_INSTRUCTION_RS1DATA_OFFSET+:64] = rs1; + pins[`AD_IN_EXECUTION_COMMAND_BITS_INSTRUCTION_RS2DATA_OFFSET+:64] = rs2; + pins[`AD_IN_EXECUTION_COMMAND_BITS_TAG_OFFSET+:`AD_IN_EXECUTION_COMMAND_BITS_TAG_WIDTH] = tag; + pins[`AD_IN_EXECUTION_COMMAND_VALID_OFFSET] = 1; + endtask + task accepted(); + do tick(); while (!observed[`AD_OUT_EXECUTION_COMMAND_READY_OFFSET]); + endtask + task body(); + string negative; + pins = '0; + repeat (3) tick(1); + tick(); + if ($value$plusargs("CASE=%s", negative)) begin + command(0, 0, 0, 0); + repeat (8) tick(); + `uvm_fatal("MISSING_ASSERTION", "Expected admission assertion did not fire") + end + for (int i = 0; i < 48; i++) begin + bit moving = i % 2; + bit result = i % 3 != 0; + command(i, moving ? (32'h1a00007b | (result << 14)) : (32'h2b | (result << 14)), + i * 64'hfedcba987654321, ~(i * 64'h123456789abcdef)); + pins[`AD_IN_TASKREQUEST_READY_OFFSET] = 0; + pins[`AD_IN_MOVE_COMMAND_READY_OFFSET] = 0; + repeat (3) tick(); + pins[`AD_IN_TASKREQUEST_READY_OFFSET] = 1; + pins[`AD_IN_MOVE_COMMAND_READY_OFFSET] = 1; + accepted(); + pins[`AD_IN_EXECUTION_COMMAND_VALID_OFFSET] = 0; + pins[`AD_IN_EXECUTION_RESPONSE_READY_OFFSET] = 0; + pins[`AD_IN_EXECUTION_COMPLETE_READY_OFFSET] = 0; + repeat (2) tick(); + if (moving) begin + pins[`AD_IN_MOVE_COMPLETION_BITS_OFFSET] = i % 4 >= 2; + pins[`AD_IN_MOVE_COMPLETION_VALID_OFFSET] = 1; + do tick(); while (!observed[`AD_OUT_MOVE_COMPLETION_READY_OFFSET]); + pins[`AD_IN_MOVE_COMPLETION_VALID_OFFSET] = 0; + if (result) begin + repeat (3) tick(); + pins[`AD_IN_EXECUTION_RESPONSE_READY_OFFSET] = 1; + tick(); + end + end else begin + pins[`AD_IN_TASKRESPONSE_BITS_RD_OFFSET+:5] = 5'(31 - i); + pins[`AD_IN_TASKRESPONSE_BITS_DATA_OFFSET+:64] = 64'hcafe000000000000 + i; + pins[`AD_IN_TASKRESPONSE_VALID_OFFSET] = 1; + if (result) begin + repeat (3) tick(); + pins[`AD_IN_EXECUTION_RESPONSE_READY_OFFSET] = 1; + end + do tick(); while (!observed[`AD_OUT_TASKRESPONSE_READY_OFFSET]); + pins[`AD_IN_TASKRESPONSE_VALID_OFFSET] = 0; + end + repeat (3) tick(); + pins[`AD_IN_EXECUTION_COMPLETE_READY_OFFSET] = 1; + do tick(); while (!observed[`AD_OUT_EXECUTION_COMPLETE_VALID_OFFSET]); + tick(); + end + command(250, 32'h402b, '1, '1); + accepted(); + pins[`AD_IN_EXECUTION_COMMAND_VALID_OFFSET] = 0; + repeat (3) tick(); + repeat (2) tick(1); + pins = '0; + tick(); + repeat (4) tick(); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/tb.sv b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/tb.sv new file mode 100644 index 000000000..dbe564378 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/tb.sv @@ -0,0 +1,15 @@ +module admission_tb; + import uvm_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + admission_if bus (clock); + CommandAdmission dut ( + .clock(clock), + .reset(bus.reset), + `include "ports.svh" + ); + initial begin + uvm_config_db#(virtual admission_if)::set(null, "uvm_test_top.env.*", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/tests/test.svh b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/tests/test.svh new file mode 100644 index 000000000..54d137a6f --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/tests/test.svh @@ -0,0 +1,15 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.start(test_env.seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/common/agent/driver.svh b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/common/agent/driver.svh new file mode 100644 index 000000000..ade23d264 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/common/agent/driver.svh @@ -0,0 +1,25 @@ +class driver extends uvm_driver #(cycle_item); + `uvm_component_utils(driver) + virtual admission_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual admission_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Admission interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item req; + forever begin + seq_item_port.get_next_item(req); + @(negedge vif.clock); + vif.reset = req.reset; + vif.inputs = req.inputs; + @(posedge vif.clock); + req.outputs = vif.outputs; + #1; + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/common/agent/item.svh b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/common/agent/item.svh new file mode 100644 index 000000000..f287d532a --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/common/agent/item.svh @@ -0,0 +1,18 @@ +class cycle_item extends uvm_sequence_item; + `uvm_object_utils(cycle_item) + logic [`AD_INPUT_BITS-1:0] inputs; + logic [`AD_OUTPUT_BITS-1:0] outputs; + bit reset; + function new(string name = "cycle_item"); + super.new(name); + endfunction +endclass +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(value, UVM_ALL_ON) + `uvm_object_utils_end + logic [`AD_OUTPUT_BITS-1:0] value; + function new(string name = "result"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/common/agent/monitor.svh b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/common/agent/monitor.svh new file mode 100644 index 000000000..57dedf7d8 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/common/agent/monitor.svh @@ -0,0 +1,25 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual admission_if vif; + uvm_analysis_port #(cycle_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual admission_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Admission monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item t; + forever begin + @(posedge vif.clock); + t = cycle_item::type_id::create("observed"); + t.inputs = vif.inputs; + t.outputs = vif.outputs; + t.reset = vif.reset; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/common/interface.sv b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/common/interface.sv new file mode 100644 index 000000000..4933b0aab --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/common/interface.sv @@ -0,0 +1,8 @@ +`include "parameters.svh" +interface admission_if ( + input logic clock +); + logic reset = 1; + logic [`AD_INPUT_BITS-1:0] inputs = '0; + wire [`AD_OUTPUT_BITS-1:0] outputs; +endinterface diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/env/env.svh b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/env/env.svh new file mode 100644 index 000000000..1f8748f8e --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/env/env.svh @@ -0,0 +1,25 @@ +class env extends uvm_env; + `uvm_component_utils(env) + driver source; + monitor mon; + model reference; + uvm_sequencer #(cycle_item) seqr; + result_board #(result_item, 0) scoreboard; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + source = driver::type_id::create("source", this); + mon = monitor::type_id::create("monitor", this); + reference = model::type_id::create("model", this); + seqr = new("seqr", this); + scoreboard = result_board#(result_item, 0)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + source.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(reference.analysis_export); + reference.expected.connect(scoreboard.expected_export); + reference.actual.connect(scoreboard.actual_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/env/model.svh b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/env/model.svh new file mode 100644 index 000000000..de6209c06 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/env/model.svh @@ -0,0 +1,143 @@ +class model extends uvm_subscriber #(cycle_item); + `uvm_component_utils(model) + uvm_analysis_port #(result_item) expected, actual; + const int copies[`AD_PAYLOAD_FIELDS][4] = `AD_PAYLOAD_MAP; + bit pending, moving, has_result, work_done, response_done, move_error, remote_owner; + int saved_tag, destination, tags[$]; + covergroup behavior with function sample (int route, bit stalled, bit cancelled); + option.per_instance = 1; + routes: coverpoint route {bins all[] = {[0 : 1]};} + backpressure: coverpoint stalled; + reset_cancellation: coverpoint cancelled; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected = new("expected", this); + actual = new("actual", this); + behavior = new(); + endfunction + function void write(cycle_item t); + logic [`AD_OUTPUT_BITS-1:0] want, mask; + result_item predicted, received; + + if (t.reset) begin + behavior.sample(0, 0, pending || tags.size() != 0); + pending = 0; + tags.delete(); + return; + end + want = '0; + mask = '0; + begin + bit management, available; + logic [31:0] instruction; + instruction = `AI(t, EXECUTION_COMMAND_BITS_INSTRUCTION_RAW_INST); + management = instruction[6:0] == 'h2b; + available = !pending && ! + `AI(t, REMOTE_BUSY) + && tags.size() < `AD_ENTRIES && (!management || (! + `AI(t, ACCELERATOR_BUSY) + && tags.size() == 0)); + want[`AD_OUT_EXECUTION_COMMAND_READY_OFFSET+:`AD_OUT_EXECUTION_COMMAND_READY_WIDTH]=available&&(management? + `AI(t, REMOTE_REQUEST_READY) + : + `AI(t, ACCELERATOR_CMD_READY) + ); + mask[`AD_OUT_EXECUTION_COMMAND_READY_OFFSET+:`AD_OUT_EXECUTION_COMMAND_READY_WIDTH] = '1; + + want[`AD_OUT_ACCELERATOR_CMD_VALID_OFFSET+:`AD_OUT_ACCELERATOR_CMD_VALID_WIDTH] = + `AI(t, EXECUTION_COMMAND_VALID) + && available && !management; + mask[`AD_OUT_ACCELERATOR_CMD_VALID_OFFSET+:`AD_OUT_ACCELERATOR_CMD_VALID_WIDTH] = '1; + + want[`AD_OUT_REMOTE_REQUEST_VALID_OFFSET+:`AD_OUT_REMOTE_REQUEST_VALID_WIDTH] = + `AI(t, EXECUTION_COMMAND_VALID) + && available && management; + mask[`AD_OUT_REMOTE_REQUEST_VALID_OFFSET+:`AD_OUT_REMOTE_REQUEST_VALID_WIDTH] = '1; + + want[`AD_OUT_ACCELERATOR_RESP_READY_OFFSET+:`AD_OUT_ACCELERATOR_RESP_READY_WIDTH] = 0; + mask[`AD_OUT_ACCELERATOR_RESP_READY_OFFSET+:`AD_OUT_ACCELERATOR_RESP_READY_WIDTH] = '1; + + want[`AD_OUT_EXECUTION_RESPONSE_VALID_OFFSET+:`AD_OUT_EXECUTION_RESPONSE_VALID_WIDTH]=pending&&!work_done&&has_result&&`AI( + t, REMOTE_REPLY_VALID); + mask[`AD_OUT_EXECUTION_RESPONSE_VALID_OFFSET+:`AD_OUT_EXECUTION_RESPONSE_VALID_WIDTH] = '1; + + if (want[`AD_OUT_EXECUTION_RESPONSE_VALID_OFFSET]) begin + want[`AD_OUT_EXECUTION_RESPONSE_BITS_RD_OFFSET+:`AD_OUT_EXECUTION_RESPONSE_BITS_RD_WIDTH]=destination; + mask[`AD_OUT_EXECUTION_RESPONSE_BITS_RD_OFFSET+:`AD_OUT_EXECUTION_RESPONSE_BITS_RD_WIDTH]='1; + want[`AD_OUT_EXECUTION_RESPONSE_BITS_DATA_OFFSET+:`AD_OUT_EXECUTION_RESPONSE_BITS_DATA_WIDTH]=`AI( + t, REMOTE_REPLY_BITS); + mask[`AD_OUT_EXECUTION_RESPONSE_BITS_DATA_OFFSET+:`AD_OUT_EXECUTION_RESPONSE_BITS_DATA_WIDTH]='1; + end + want[`AD_OUT_REMOTE_REPLY_READY_OFFSET+:`AD_OUT_REMOTE_REPLY_READY_WIDTH]=pending&&!work_done&&(!has_result|| + `AI(t, EXECUTION_RESPONSE_READY) + ); + mask[`AD_OUT_REMOTE_REPLY_READY_OFFSET+:`AD_OUT_REMOTE_REPLY_READY_WIDTH] = '1; + + want[`AD_OUT_EXECUTION_COMPLETE_VALID_OFFSET+:`AD_OUT_EXECUTION_COMPLETE_VALID_WIDTH]=tags.size()!=0&&(pending?work_done:! + `AI(t, ACCELERATOR_BUSY) + ); + mask[`AD_OUT_EXECUTION_COMPLETE_VALID_OFFSET+:`AD_OUT_EXECUTION_COMPLETE_VALID_WIDTH] = '1; + + if (want[`AD_OUT_EXECUTION_COMPLETE_VALID_OFFSET]) begin + want[`AD_OUT_EXECUTION_COMPLETE_BITS_TAG_OFFSET+:`AD_OUT_EXECUTION_COMPLETE_BITS_TAG_WIDTH]=tags[0]; + mask[`AD_OUT_EXECUTION_COMPLETE_BITS_TAG_OFFSET+:`AD_OUT_EXECUTION_COMPLETE_BITS_TAG_WIDTH]='1; + end + want[`AD_OUT_DRAINED_OFFSET+:`AD_OUT_DRAINED_WIDTH] = !pending && tags.size() == 0 && ! + `AI(t, ACCELERATOR_BUSY) + && ! + `AI(t, REMOTE_BUSY); + mask[`AD_OUT_DRAINED_OFFSET+:`AD_OUT_DRAINED_WIDTH] = '1; + + + end + foreach (copies[field]) + for (int b = 0; b < copies[field][2]; b++) + if (want[copies[field][3]]) begin + want[copies[field][1]+b] = t.inputs[copies[field][0]+b]; + mask[copies[field][1]+b] = 1; + end + predicted = result_item::type_id::create("expected"); + received = result_item::type_id::create("actual"); + predicted.value = want & mask; + received.value = t.outputs & mask; + expected.write(predicted); + actual.write(received); + begin + logic [31:0] instruction; + bit management; + instruction = `AI(t, EXECUTION_COMMAND_BITS_INSTRUCTION_RAW_INST); + management = instruction[6:0] == 'h2b; + if (`AO(t, EXECUTION_COMPLETE_VALID) && `AI(t, EXECUTION_COMPLETE_READY)) begin + int tag = tags.pop_front(); + if (pending) pending = 0; + end + if (`AI(t, EXECUTION_COMMAND_VALID) && `AO(t, EXECUTION_COMMAND_READY)) begin + tags.push_back(`AI(t, EXECUTION_COMMAND_BITS_TAG)); + if (management) begin + pending = 1; + work_done = 0; + has_result = instruction[14]; + destination = instruction[11:7]; + end + behavior.sample(management, 0, 0); + end + if (`AI(t, REMOTE_REPLY_VALID) && `AO(t, REMOTE_REPLY_READY)) work_done = 1; + if ( + `AI(t, EXECUTION_COMMAND_VALID) + && ! + `AO(t, EXECUTION_COMMAND_READY) + || + `AO(t, EXECUTION_COMPLETE_VALID) + && ! + `AI(t, EXECUTION_COMPLETE_READY) + ) + behavior.sample(management, 1, 0); + + end + endfunction + function void check_phase(uvm_phase phase); + super.check_phase(phase); + if (pending || tags.size() != 0) `uvm_fatal("PENDING", "Admission ownership did not drain") + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/filelist.f b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/filelist.f new file mode 100644 index 000000000..a5bedca99 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@/DirectAdmission ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/../common/interface.sv +@RESOURCES@/package.sv +-F @RTL@/DirectAdmission/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/package.sv b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/package.sv new file mode 100644 index 000000000..3793ede46 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/package.sv @@ -0,0 +1,17 @@ +`include "parameters.svh" +`define AI(T, F) T.inputs[`AD_IN_``F``_OFFSET+:`AD_IN_``F``_WIDTH] +`define AO(T, F) T.outputs[`AD_OUT_``F``_OFFSET+:`AD_OUT_``F``_WIDTH] +`define EXPECT(F, + V) want[`AD_OUT_``F``_OFFSET+:`AD_OUT_``F``_WIDTH]=V;mask[`AD_OUT_``F``_OFFSET+:`AD_OUT_``F``_WIDTH]='1; +package admission_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + `include "../common/agent/item.svh" + `include "../common/agent/driver.svh" + `include "../common/agent/monitor.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/seq/sequence.svh b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/seq/sequence.svh new file mode 100644 index 000000000..5f04c9a41 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/seq/sequence.svh @@ -0,0 +1,92 @@ +class scenario_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(scenario_sequence) + logic [ `AD_INPUT_BITS-1:0] pins; + logic [`AD_OUTPUT_BITS-1:0] observed; + function new(string name = "scenario"); + super.new(name); + endfunction + task tick(bit reset = 0); + cycle_item t = cycle_item::type_id::create("cycle"); + start_item(t); + t.inputs = pins; + t.reset = reset; + finish_item(t); + observed = t.outputs; + endtask + task command(int tag, logic [31:0] instruction, logic [63:0] rs1, logic [63:0] rs2); + for (int b = 0; b < `AD_COMMAND_WIDTH; b++) + pins[`AD_COMMAND_OFFSET+b] = ((b + tag) % 3 == 0) ^ ((b + tag) % 7 == 0); + pins[`AD_IN_EXECUTION_COMMAND_BITS_INSTRUCTION_RAW_INST_OFFSET+:32]=instruction|((tag%32)<<7); + pins[`AD_IN_EXECUTION_COMMAND_BITS_INSTRUCTION_RS1DATA_OFFSET+:64] = rs1; + pins[`AD_IN_EXECUTION_COMMAND_BITS_INSTRUCTION_RS2DATA_OFFSET+:64] = rs2; + pins[`AD_IN_EXECUTION_COMMAND_BITS_TAG_OFFSET+:`AD_IN_EXECUTION_COMMAND_BITS_TAG_WIDTH] = tag; + pins[`AD_IN_EXECUTION_COMMAND_VALID_OFFSET] = 1; + endtask + task accepted(); + do tick(); while (!observed[`AD_OUT_EXECUTION_COMMAND_READY_OFFSET]); + endtask + task body(); + string negative; + pins = '0; + repeat (3) tick(1); + tick(); + if ($value$plusargs("CASE=%s", negative)) begin + pins[`AD_IN_ACCELERATOR_CMD_READY_OFFSET] = 1; + if (negative == "reply") pins[`AD_IN_ACCELERATOR_RESP_VALID_OFFSET] = 1; + else if (negative == "interrupt") pins[`AD_IN_ACCELERATOR_INTERRUPT_OFFSET] = 1; + else command(0, 32'h407b, 0, 0); + repeat (8) tick(); + `uvm_fatal("MISSING_ASSERTION", "Expected admission assertion did not fire") + end + pins[`AD_IN_ACCELERATOR_CMD_READY_OFFSET] = 1; + pins[`AD_IN_EXECUTION_COMPLETE_READY_OFFSET] = 0; + pins[`AD_IN_ACCELERATOR_BUSY_OFFSET] = 1; + for (int i = 0; i < `AD_ENTRIES; i++) begin + command(i, 32'h7b, i * 64'h123456789abcdef, ~64'(i)); + accepted(); + pins[`AD_IN_EXECUTION_COMMAND_VALID_OFFSET] = 0; + end + command(100, 32'h7b, '1, '1); + repeat (5) tick(); + pins[`AD_IN_EXECUTION_COMMAND_VALID_OFFSET] = 0; + pins[`AD_IN_ACCELERATOR_BUSY_OFFSET] = 0; + pins[`AD_IN_EXECUTION_COMPLETE_READY_OFFSET] = 1; + do tick(); while (!observed[`AD_OUT_DRAINED_OFFSET]); + for (int i = 0; i < 32; i++) begin + bit result = i % 2; + pins[`AD_IN_REMOTE_BUSY_OFFSET] = 1; + command(16 + i, (32'((13 + i % 20) << 25) | 32'h2b | (result << 14)), i * 64'h123456789abcdef, + ~64'(i)); + repeat (3) tick(); + pins[`AD_IN_REMOTE_BUSY_OFFSET] = 0; + pins[`AD_IN_REMOTE_REQUEST_READY_OFFSET] = 0; + repeat (3) tick(); + pins[`AD_IN_REMOTE_REQUEST_READY_OFFSET] = 1; + accepted(); + pins[`AD_IN_EXECUTION_COMMAND_VALID_OFFSET] = 0; + pins[`AD_IN_EXECUTION_RESPONSE_READY_OFFSET] = 0; + pins[`AD_IN_EXECUTION_COMPLETE_READY_OFFSET] = 0; + pins[`AD_IN_REMOTE_REPLY_BITS_OFFSET+:64] = 64'h1234000056780000 + i; + pins[`AD_IN_REMOTE_REPLY_VALID_OFFSET] = 1; + if (result) begin + repeat (3) tick(); + pins[`AD_IN_EXECUTION_RESPONSE_READY_OFFSET] = 1; + end + do tick(); while (!observed[`AD_OUT_REMOTE_REPLY_READY_OFFSET]); + pins[`AD_IN_REMOTE_REPLY_VALID_OFFSET] = 0; + repeat (3) tick(); + pins[`AD_IN_EXECUTION_COMPLETE_READY_OFFSET] = 1; + do tick(); while (!observed[`AD_OUT_EXECUTION_COMPLETE_VALID_OFFSET]); + tick(); + end + pins[`AD_IN_EXECUTION_COMPLETE_READY_OFFSET] = 0; + command(200, 32'h7b, '1, '1); + accepted(); + pins[`AD_IN_EXECUTION_COMMAND_VALID_OFFSET] = 0; + repeat (3) tick(); + repeat (2) tick(1); + pins = '0; + tick(); + repeat (4) tick(); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/tb.sv b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/tb.sv new file mode 100644 index 000000000..ecf85aee1 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/tb.sv @@ -0,0 +1,15 @@ +module admission_tb; + import uvm_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + admission_if bus (clock); + DirectAdmission dut ( + .clock(clock), + .reset(bus.reset), + `include "ports.svh" + ); + initial begin + uvm_config_db#(virtual admission_if)::set(null, "uvm_test_top.env.*", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/tests/test.svh b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/tests/test.svh new file mode 100644 index 000000000..54d137a6f --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/tests/test.svh @@ -0,0 +1,15 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.start(test_env.seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/env/env.svh b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/env/env.svh new file mode 100644 index 000000000..1f8748f8e --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/env/env.svh @@ -0,0 +1,25 @@ +class env extends uvm_env; + `uvm_component_utils(env) + driver source; + monitor mon; + model reference; + uvm_sequencer #(cycle_item) seqr; + result_board #(result_item, 0) scoreboard; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + source = driver::type_id::create("source", this); + mon = monitor::type_id::create("monitor", this); + reference = model::type_id::create("model", this); + seqr = new("seqr", this); + scoreboard = result_board#(result_item, 0)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + source.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(reference.analysis_export); + reference.expected.connect(scoreboard.expected_export); + reference.actual.connect(scoreboard.actual_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/env/model.svh b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/env/model.svh new file mode 100644 index 000000000..69aee0888 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/env/model.svh @@ -0,0 +1,86 @@ +class model extends uvm_subscriber #(cycle_item); + `uvm_component_utils(model) + uvm_analysis_port #(result_item) expected, actual; + const int copies[`AD_PAYLOAD_FIELDS][4] = `AD_PAYLOAD_MAP; + bit pending, moving, has_result, work_done, response_done, move_error, remote_owner; + int saved_tag, destination, tags[$]; + covergroup behavior with function sample (int route, bit stalled, bit cancelled); + option.per_instance = 1; + routes: coverpoint route {bins all[] = {[0 : 0]};} + backpressure: coverpoint stalled; + reset_cancellation: coverpoint cancelled; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected = new("expected", this); + actual = new("actual", this); + behavior = new(); + endfunction + function void write(cycle_item t); + logic [`AD_OUTPUT_BITS-1:0] want, mask; + result_item predicted, received; + + if (t.reset) begin + behavior.sample(0, 0, pending || tags.size() != 0); + pending = 0; + tags.delete(); + return; + end + want = '0; + mask = '0; + begin + want[`AD_OUT_MOVE_COMMAND_VALID_OFFSET+:`AD_OUT_MOVE_COMMAND_VALID_WIDTH] = + `AI(t, ROCC_CMD_VALID) + && !pending; + mask[`AD_OUT_MOVE_COMMAND_VALID_OFFSET+:`AD_OUT_MOVE_COMMAND_VALID_WIDTH] = '1; + + want[`AD_OUT_ROCC_CMD_READY_OFFSET+:`AD_OUT_ROCC_CMD_READY_WIDTH] = + `AI(t, MOVE_COMMAND_READY) + && !pending; + mask[`AD_OUT_ROCC_CMD_READY_OFFSET+:`AD_OUT_ROCC_CMD_READY_WIDTH] = '1; + + want[`AD_OUT_MOVE_COMMAND_BITS_ROWS_OFFSET+:`AD_OUT_MOVE_COMMAND_BITS_ROWS_WIDTH]= + {1'b0, t.inputs[`AD_IN_ROCC_CMD_BITS_INSTRUCTION_RS2DATA_OFFSET+32+:16]} + 17'd1; + mask[`AD_OUT_MOVE_COMMAND_BITS_ROWS_OFFSET+:`AD_OUT_MOVE_COMMAND_BITS_ROWS_WIDTH] = '1; + + want[`AD_OUT_MOVE_COMPLETION_READY_OFFSET+:`AD_OUT_MOVE_COMPLETION_READY_WIDTH] = pending; + mask[`AD_OUT_MOVE_COMPLETION_READY_OFFSET+:`AD_OUT_MOVE_COMPLETION_READY_WIDTH] = '1; + + want[`AD_OUT_ROCC_BUSY_OFFSET+:`AD_OUT_ROCC_BUSY_WIDTH] = pending; + mask[`AD_OUT_ROCC_BUSY_OFFSET+:`AD_OUT_ROCC_BUSY_WIDTH] = '1; + + want[`AD_OUT_ROCC_INTERRUPT_OFFSET+:`AD_OUT_ROCC_INTERRUPT_WIDTH] = 0; + mask[`AD_OUT_ROCC_INTERRUPT_OFFSET+:`AD_OUT_ROCC_INTERRUPT_WIDTH] = '1; + + want[`AD_OUT_ROCC_RESP_VALID_OFFSET+:`AD_OUT_ROCC_RESP_VALID_WIDTH] = 0; + mask[`AD_OUT_ROCC_RESP_VALID_OFFSET+:`AD_OUT_ROCC_RESP_VALID_WIDTH] = '1; + + + end + foreach (copies[field]) + for (int b = 0; b < copies[field][2]; b++) + if (want[copies[field][3]]) begin + want[copies[field][1]+b] = t.inputs[copies[field][0]+b]; + mask[copies[field][1]+b] = 1; + end + predicted = result_item::type_id::create("expected"); + received = result_item::type_id::create("actual"); + predicted.value = want & mask; + received.value = t.outputs & mask; + expected.write(predicted); + actual.write(received); + begin + if (`AI(t, ROCC_CMD_VALID) && `AO(t, ROCC_CMD_READY)) begin + pending = 1; + behavior.sample(0, 0, 0); + end + if (`AI(t, MOVE_COMPLETION_VALID) && `AO(t, MOVE_COMPLETION_READY)) pending = 0; + if (`AI(t, ROCC_CMD_VALID) && !`AO(t, ROCC_CMD_READY)) behavior.sample(0, 1, 0); + + end + endfunction + function void check_phase(uvm_phase phase); + super.check_phase(phase); + if (pending || tags.size() != 0) `uvm_fatal("PENDING", "Admission ownership did not drain") + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/filelist.f b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/filelist.f new file mode 100644 index 000000000..deff3ae10 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@/MoveAdmission ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/../common/interface.sv +@RESOURCES@/package.sv +-F @RTL@/MoveAdmission/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/package.sv b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/package.sv new file mode 100644 index 000000000..3793ede46 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/package.sv @@ -0,0 +1,17 @@ +`include "parameters.svh" +`define AI(T, F) T.inputs[`AD_IN_``F``_OFFSET+:`AD_IN_``F``_WIDTH] +`define AO(T, F) T.outputs[`AD_OUT_``F``_OFFSET+:`AD_OUT_``F``_WIDTH] +`define EXPECT(F, + V) want[`AD_OUT_``F``_OFFSET+:`AD_OUT_``F``_WIDTH]=V;mask[`AD_OUT_``F``_OFFSET+:`AD_OUT_``F``_WIDTH]='1; +package admission_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + `include "../common/agent/item.svh" + `include "../common/agent/driver.svh" + `include "../common/agent/monitor.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/seq/sequence.svh b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/seq/sequence.svh new file mode 100644 index 000000000..e5976bf84 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/seq/sequence.svh @@ -0,0 +1,61 @@ +class scenario_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(scenario_sequence) + logic [ `AD_INPUT_BITS-1:0] pins; + logic [`AD_OUTPUT_BITS-1:0] observed; + function new(string name = "scenario"); + super.new(name); + endfunction + task tick(bit reset = 0); + cycle_item t = cycle_item::type_id::create("cycle"); + start_item(t); + t.inputs = pins; + t.reset = reset; + finish_item(t); + observed = t.outputs; + endtask + task command(int tag, logic [31:0] instruction, logic [63:0] rs1, logic [63:0] rs2); + for (int b = 0; b < `AD_COMMAND_WIDTH; b++) + pins[`AD_COMMAND_OFFSET+b] = ((b + tag) % 3 == 0) ^ ((b + tag) % 7 == 0); + pins[`AD_IN_ROCC_CMD_BITS_INSTRUCTION_RAW_INST_OFFSET+:32] = instruction | ((tag % 32) << 7); + pins[`AD_IN_ROCC_CMD_BITS_INSTRUCTION_RS1DATA_OFFSET+:64] = rs1; + pins[`AD_IN_ROCC_CMD_BITS_INSTRUCTION_RS2DATA_OFFSET+:64] = rs2; + pins[`AD_IN_ROCC_CMD_VALID_OFFSET] = 1; + endtask + task accepted(); + do tick(); while (!observed[`AD_OUT_ROCC_CMD_READY_OFFSET]); + endtask + task body(); + string negative; + pins = '0; + repeat (3) tick(1); + tick(); + if ($value$plusargs("CASE=%s", negative)) begin + command(0, 32'h1a00707b, 0, 0); + repeat (8) tick(); + `uvm_fatal("MISSING_ASSERTION", "Expected admission assertion did not fire") + end + for (int i = 0; i < 48; i++) begin + command(i, 32'h1a00307b, i * 64'hfedcba987654321, + i == 47 ? 64'h0000ffff12345678 : ~(i * 64'h123456789abcdef)); + pins[`AD_IN_MOVE_COMMAND_READY_OFFSET] = 0; + repeat (3) tick(); + pins[`AD_IN_MOVE_COMMAND_READY_OFFSET] = 1; + accepted(); + pins[`AD_IN_ROCC_CMD_VALID_OFFSET] = 0; + repeat (5) tick(); + pins[`AD_IN_MOVE_COMPLETION_BITS_OFFSET] = i % 2; + pins[`AD_IN_MOVE_COMPLETION_VALID_OFFSET] = 1; + do tick(); while (!observed[`AD_OUT_MOVE_COMPLETION_READY_OFFSET]); + pins[`AD_IN_MOVE_COMPLETION_VALID_OFFSET] = 0; + tick(); + end + command(0, 32'h1a00307b, '1, '1); + accepted(); + pins[`AD_IN_ROCC_CMD_VALID_OFFSET] = 0; + repeat (3) tick(); + repeat (2) tick(1); + pins = '0; + tick(); + repeat (4) tick(); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/tb.sv b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/tb.sv new file mode 100644 index 000000000..a5ff595c4 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/tb.sv @@ -0,0 +1,15 @@ +module admission_tb; + import uvm_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + admission_if bus (clock); + MoveAdmission dut ( + .clock(clock), + .reset(bus.reset), + `include "ports.svh" + ); + initial begin + uvm_config_db#(virtual admission_if)::set(null, "uvm_test_top.env.*", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/tests/test.svh b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/tests/test.svh new file mode 100644 index 000000000..54d137a6f --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/tests/test.svh @@ -0,0 +1,15 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.start(test_env.seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/suite.toml b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/suite.toml new file mode 100644 index 000000000..6225286ac --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/suite.toml @@ -0,0 +1,156 @@ +[model] +manifest="" +library="" + +[[tests]] +name="command" +top="admission_tb" +test="protocol_test" +filelist="command/filelist.f" +timeout_ns=500000 +wall_timeout_s=120 +kind="functional" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=["admission_pkg::model::behavior"] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=[] +assertion="" + +[[tests]] +name="move" +top="admission_tb" +test="protocol_test" +filelist="move/filelist.f" +timeout_ns=500000 +wall_timeout_s=120 +kind="functional" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=["admission_pkg::model::behavior"] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=[] +assertion="" + +[[tests]] +name="direct" +top="admission_tb" +test="protocol_test" +filelist="direct/filelist.f" +timeout_ns=500000 +wall_timeout_s=120 +kind="functional" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=["admission_pkg::model::behavior"] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=[] +assertion="" + +[[tests]] +name="task" +top="admission_tb" +test="protocol_test" +filelist="task/filelist.f" +timeout_ns=500000 +wall_timeout_s=120 +kind="functional" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=["admission_pkg::model::behavior"] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=[] +assertion="" + +[[tests]] +name="command_bad_unsupported" +top="admission_tb" +test="protocol_test" +filelist="command/filelist.f" +timeout_ns=500000 +wall_timeout_s=120 +kind="assertion" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=[] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=["+CASE=unsupported"] +assertion="CPU-only controller received an unsupported compute instruction" + +[[tests]] +name="move_bad_posted" +top="admission_tb" +test="protocol_test" +filelist="move/filelist.f" +timeout_ns=500000 +wall_timeout_s=120 +kind="assertion" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=[] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=["+CASE=posted"] +assertion="Move admission only accepts posted bank moves" + +[[tests]] +name="direct_bad_unsupported" +top="admission_tb" +test="protocol_test" +filelist="direct/filelist.f" +timeout_ns=500000 +wall_timeout_s=120 +kind="assertion" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=[] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=["+CASE=unsupported"] +assertion="Direct core received an unsupported command" + +[[tests]] +name="direct_bad_reply" +top="admission_tb" +test="protocol_test" +filelist="direct/filelist.f" +timeout_ns=500000 +wall_timeout_s=120 +kind="assertion" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=[] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=["+CASE=reply"] +assertion="Direct accelerator returned an unexpected register result" + +[[tests]] +name="direct_bad_interrupt" +top="admission_tb" +test="protocol_test" +filelist="direct/filelist.f" +timeout_ns=500000 +wall_timeout_s=120 +kind="assertion" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=[] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=["+CASE=interrupt"] +assertion="Direct accelerator fault" diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/env/env.svh b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/env/env.svh new file mode 100644 index 000000000..1f8748f8e --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/env/env.svh @@ -0,0 +1,25 @@ +class env extends uvm_env; + `uvm_component_utils(env) + driver source; + monitor mon; + model reference; + uvm_sequencer #(cycle_item) seqr; + result_board #(result_item, 0) scoreboard; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + source = driver::type_id::create("source", this); + mon = monitor::type_id::create("monitor", this); + reference = model::type_id::create("model", this); + seqr = new("seqr", this); + scoreboard = result_board#(result_item, 0)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + source.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(reference.analysis_export); + reference.expected.connect(scoreboard.expected_export); + reference.actual.connect(scoreboard.actual_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/env/model.svh b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/env/model.svh new file mode 100644 index 000000000..440cf8030 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/env/model.svh @@ -0,0 +1,126 @@ +class model extends uvm_subscriber #(cycle_item); + `uvm_component_utils(model) + uvm_analysis_port #(result_item) expected, actual; + const int copies[`AD_PAYLOAD_FIELDS][4] = `AD_PAYLOAD_MAP; + bit pending, moving, has_result, work_done, response_done, move_error, remote_owner; + int saved_tag, destination, tags[$]; + covergroup behavior with function sample (int route, bit stalled, bit cancelled); + option.per_instance = 1; + routes: coverpoint route {bins all[] = {[0 : 1]};} + backpressure: coverpoint stalled; + reset_cancellation: coverpoint cancelled; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected = new("expected", this); + actual = new("actual", this); + behavior = new(); + endfunction + function void write(cycle_item t); + logic [`AD_OUTPUT_BITS-1:0] want, mask; + result_item predicted, received; + + if (t.reset) begin + behavior.sample(0, 0, pending || tags.size() != 0); + pending = 0; + tags.delete(); + return; + end + want = '0; + mask = '0; + begin + bit remote_command; + logic [31:0] instruction; + instruction = `AI(t, COMMAND_BITS_INSTRUCTION_RAW_INST); + remote_command = instruction[31:25] >= 13; + want[`AD_OUT_COMMAND_READY_OFFSET+:`AD_OUT_COMMAND_READY_WIDTH]=!pending&&(remote_command? + `AI(t, REMOTE_REQUEST_READY) + : + `AI(t, LOCAL_REQUEST_READY) + ); + mask[`AD_OUT_COMMAND_READY_OFFSET+:`AD_OUT_COMMAND_READY_WIDTH] = '1; + + want[`AD_OUT_LOCAL_REQUEST_VALID_OFFSET+:`AD_OUT_LOCAL_REQUEST_VALID_WIDTH] = + `AI(t, COMMAND_VALID) + && !pending && !remote_command; + mask[`AD_OUT_LOCAL_REQUEST_VALID_OFFSET+:`AD_OUT_LOCAL_REQUEST_VALID_WIDTH] = '1; + + want[`AD_OUT_REMOTE_REQUEST_VALID_OFFSET+:`AD_OUT_REMOTE_REQUEST_VALID_WIDTH] = + `AI(t, COMMAND_VALID) + && !pending && remote_command; + mask[`AD_OUT_REMOTE_REQUEST_VALID_OFFSET+:`AD_OUT_REMOTE_REQUEST_VALID_WIDTH] = '1; + + want[`AD_OUT_RESPONSE_VALID_OFFSET+:`AD_OUT_RESPONSE_VALID_WIDTH] = pending && (remote_owner ? + `AI(t, REMOTE_REPLY_VALID) + : + `AI(t, LOCAL_REPLY_VALID) + ); + mask[`AD_OUT_RESPONSE_VALID_OFFSET+:`AD_OUT_RESPONSE_VALID_WIDTH] = '1; + + if (want[`AD_OUT_RESPONSE_VALID_OFFSET]) begin + want[`AD_OUT_RESPONSE_BITS_RD_OFFSET+:`AD_OUT_RESPONSE_BITS_RD_WIDTH] = destination; + mask[`AD_OUT_RESPONSE_BITS_RD_OFFSET+:`AD_OUT_RESPONSE_BITS_RD_WIDTH] = '1; + want[`AD_OUT_RESPONSE_BITS_DATA_OFFSET+:`AD_OUT_RESPONSE_BITS_DATA_WIDTH] = remote_owner ? + `AI(t, REMOTE_REPLY_BITS) + : + `AI(t, LOCAL_REPLY_BITS); + mask[`AD_OUT_RESPONSE_BITS_DATA_OFFSET+:`AD_OUT_RESPONSE_BITS_DATA_WIDTH] = '1; + end + want[`AD_OUT_LOCAL_REPLY_READY_OFFSET+:`AD_OUT_LOCAL_REPLY_READY_WIDTH]=pending&&!remote_owner&&`AI( + t, RESPONSE_READY); + mask[`AD_OUT_LOCAL_REPLY_READY_OFFSET+:`AD_OUT_LOCAL_REPLY_READY_WIDTH] = '1; + + want[`AD_OUT_REMOTE_REPLY_READY_OFFSET+:`AD_OUT_REMOTE_REPLY_READY_WIDTH]=pending&&remote_owner&&`AI( + t, RESPONSE_READY); + mask[`AD_OUT_REMOTE_REPLY_READY_OFFSET+:`AD_OUT_REMOTE_REPLY_READY_WIDTH] = '1; + + want[`AD_OUT_BUSY_OFFSET+:`AD_OUT_BUSY_WIDTH] = pending || + `AI(t, LOCAL_BUSY) + || + `AI(t, REMOTE_BUSY); + mask[`AD_OUT_BUSY_OFFSET+:`AD_OUT_BUSY_WIDTH] = '1; + + + end + foreach (copies[field]) + for (int b = 0; b < copies[field][2]; b++) + if (want[copies[field][3]]) begin + want[copies[field][1]+b] = t.inputs[copies[field][0]+b]; + mask[copies[field][1]+b] = 1; + end + predicted = result_item::type_id::create("expected"); + received = result_item::type_id::create("actual"); + predicted.value = want & mask; + received.value = t.outputs & mask; + expected.write(predicted); + actual.write(received); + begin + logic [31:0] instruction; + bit remote_command; + instruction = `AI(t, COMMAND_BITS_INSTRUCTION_RAW_INST); + remote_command = instruction[31:25] >= 13; + if (`AI(t, COMMAND_VALID) && `AO(t, COMMAND_READY)) begin + pending = 1; + remote_owner = remote_command; + destination = instruction[11:7]; + behavior.sample(remote_command, 0, 0); + end + if (`AO(t, RESPONSE_VALID) && `AI(t, RESPONSE_READY)) pending = 0; + if ( + `AI(t, COMMAND_VALID) + && ! + `AO(t, COMMAND_READY) + || + `AO(t, RESPONSE_VALID) + && ! + `AI(t, RESPONSE_READY) + ) + behavior.sample(remote_owner, 1, 0); + + end + endfunction + function void check_phase(uvm_phase phase); + super.check_phase(phase); + if (pending || tags.size() != 0) `uvm_fatal("PENDING", "Admission ownership did not drain") + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/filelist.f b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/filelist.f new file mode 100644 index 000000000..181f872a3 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@/TaskRouter ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/../common/interface.sv +@RESOURCES@/package.sv +-F @RTL@/TaskRouter/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/package.sv b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/package.sv new file mode 100644 index 000000000..3793ede46 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/package.sv @@ -0,0 +1,17 @@ +`include "parameters.svh" +`define AI(T, F) T.inputs[`AD_IN_``F``_OFFSET+:`AD_IN_``F``_WIDTH] +`define AO(T, F) T.outputs[`AD_OUT_``F``_OFFSET+:`AD_OUT_``F``_WIDTH] +`define EXPECT(F, + V) want[`AD_OUT_``F``_OFFSET+:`AD_OUT_``F``_WIDTH]=V;mask[`AD_OUT_``F``_OFFSET+:`AD_OUT_``F``_WIDTH]='1; +package admission_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + `include "../common/agent/item.svh" + `include "../common/agent/driver.svh" + `include "../common/agent/monitor.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/seq/sequence.svh b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/seq/sequence.svh new file mode 100644 index 000000000..5079a481e --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/seq/sequence.svh @@ -0,0 +1,68 @@ +class scenario_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(scenario_sequence) + logic [ `AD_INPUT_BITS-1:0] pins; + logic [`AD_OUTPUT_BITS-1:0] observed; + function new(string name = "scenario"); + super.new(name); + endfunction + task tick(bit reset = 0); + cycle_item t = cycle_item::type_id::create("cycle"); + start_item(t); + t.inputs = pins; + t.reset = reset; + finish_item(t); + observed = t.outputs; + endtask + task command(int tag, logic [31:0] instruction, logic [63:0] rs1, logic [63:0] rs2); + for (int b = 0; b < `AD_COMMAND_WIDTH; b++) + pins[`AD_COMMAND_OFFSET+b] = ((b + tag) % 3 == 0) ^ ((b + tag) % 7 == 0); + pins[`AD_IN_COMMAND_BITS_INSTRUCTION_RAW_INST_OFFSET+:32] = instruction | ((tag % 32) << 7); + pins[`AD_IN_COMMAND_BITS_INSTRUCTION_RS1DATA_OFFSET+:64] = rs1; + pins[`AD_IN_COMMAND_BITS_INSTRUCTION_RS2DATA_OFFSET+:64] = rs2; + pins[`AD_IN_COMMAND_BITS_TAG_OFFSET+:`AD_IN_COMMAND_BITS_TAG_WIDTH] = tag; + pins[`AD_IN_COMMAND_VALID_OFFSET] = 1; + endtask + task accepted(); + do tick(); while (!observed[`AD_OUT_COMMAND_READY_OFFSET]); + endtask + task body(); + string negative; + pins = '0; + repeat (3) tick(1); + tick(); + for (int i = 0; i < 48; i++) begin + bit remote = i % 2; + command(i, (32'((remote ? 13 + i % 20 : i % 13) << 25) | 32'h2b), i * 64'h123456789abcdef, + ~64'(i)); + pins[`AD_IN_LOCAL_REQUEST_READY_OFFSET] = remote; + pins[`AD_IN_REMOTE_REQUEST_READY_OFFSET] = !remote; + repeat (3) tick(); + pins[`AD_IN_LOCAL_REQUEST_READY_OFFSET] = 1; + pins[`AD_IN_REMOTE_REQUEST_READY_OFFSET] = 1; + accepted(); + pins[`AD_IN_COMMAND_VALID_OFFSET] = 0; + pins[`AD_IN_RESPONSE_READY_OFFSET] = 0; + pins[`AD_IN_LOCAL_REPLY_BITS_OFFSET+:64] = 64'h9876543200000000 + i; + pins[`AD_IN_REMOTE_REPLY_BITS_OFFSET+:64] = 64'h1234567800000000 + i; + pins[`AD_IN_LOCAL_REPLY_VALID_OFFSET] = remote; + pins[`AD_IN_REMOTE_REPLY_VALID_OFFSET] = !remote; + repeat (3) tick(); + pins[`AD_IN_LOCAL_REPLY_VALID_OFFSET] = 1; + pins[`AD_IN_REMOTE_REPLY_VALID_OFFSET] = 1; + repeat (3) tick(); + pins[`AD_IN_RESPONSE_READY_OFFSET] = 1; + do tick(); while (!observed[`AD_OUT_RESPONSE_VALID_OFFSET]); + pins[`AD_IN_LOCAL_REPLY_VALID_OFFSET] = 0; + pins[`AD_IN_REMOTE_REPLY_VALID_OFFSET] = 0; + tick(); + end + command(0, 32'h2b, '1, '1); + accepted(); + pins[`AD_IN_COMMAND_VALID_OFFSET] = 0; + repeat (3) tick(); + repeat (2) tick(1); + pins = '0; + tick(); + repeat (4) tick(); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/tb.sv b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/tb.sv new file mode 100644 index 000000000..49c6210ad --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/tb.sv @@ -0,0 +1,15 @@ +module admission_tb; + import uvm_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + admission_if bus (clock); + TaskRouter dut ( + .clock(clock), + .reset(bus.reset), + `include "ports.svh" + ); + initial begin + uvm_config_db#(virtual admission_if)::set(null, "uvm_test_top.env.*", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/tests/test.svh b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/tests/test.svh new file mode 100644 index 000000000..54d137a6f --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/tests/test.svh @@ -0,0 +1,15 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.start(test_env.seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/scala/CommandAdmission.scala b/arch/src/main/scala/framework/core/controller/admission/src/main/scala/CommandAdmission.scala new file mode 100644 index 000000000..8cde3f83b --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/scala/CommandAdmission.scala @@ -0,0 +1,151 @@ +package framework.core.controller.admission + +import framework.acc.frontend.{Response, RoCCIO} +import framework.acc.frontend.Command +import framework.core.memory.access.preflight.PmpEntry +import framework.ip.memory.interlock.{Tag, Params => TrackingParam} +import framework.core.memory.access.preflight.configs.PmpParams +import framework.core.controller.configs.CommandsParam +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.acc.memdomain.{MvoverISA, MvoverPort} + +class CommandSnapshot(tracking: TrackingParam, dataBits: Int, permission: PmpParams) extends Bundle { + + val tag = UInt(tracking.idBits.W) + val instruction = new Command(dataBits) + val satp = UInt(64.W) + val effectivePrivilege = UInt(2.W) + val sum = Bool() + val mxr = Bool() + val pmp = Vec(permission.entries, new PmpEntry(permission)) +} + +/** Execution-side task and bank-move admission, with completion ordering. */ +class Commands(tracking: TrackingParam, dataBits: Int, permission: PmpParams) extends Bundle { + val command = Decoupled(new CommandSnapshot(tracking, dataBits, permission)) + val response = Flipped(Decoupled(new Response(dataBits))) + val complete = Flipped(Decoupled(new Tag(tracking))) +} + +/** CPU-only controller command path: task RPC and bank moves have explicit completion owners. */ +@instantiable +class CommandAdmission(b: CommandsParam) extends Module { + val tracking = b.tracking + val moves = b.moves + + @public + val io = IO(new Bundle { + val execution = + Flipped(new Commands(tracking, b.dataBits, b.permission)) + + val taskRequest = Decoupled(new CommandSnapshot( + tracking, + b.dataBits, + b.permission + )) + + val taskResponse = Flipped(Decoupled(new Response(b.dataBits))) + val move = new MvoverPort + val memoryAllowed = Output(Bool()) + val idle = Output(Bool()) + }) + + val idle :: task :: moving :: moveResponse :: releasing :: Nil = Enum(5) + val state = RegInit(idle) + val tag = Reg(UInt(tracking.idBits.W)) + val rd = Reg(UInt(5.W)) + val failedMove = Reg(Bool()) + val instruction = io.execution.command.bits.instruction + val isTask = instruction.raw_inst(6, 0) === "h2b".U + val isMove = moves.B && instruction.raw_inst(6, 0) === "h7b".U && instruction.raw_inst(31, 25) === MvoverISA.Funct.U + + val writeResult = Reg(Bool()) + io.taskRequest.valid := state === idle && io.execution.command.valid && isTask && !reset.asBool + io.taskRequest.bits := io.execution.command.bits + + io.execution.command.ready := state === idle && !reset.asBool && + Mux(isTask, io.taskRequest.ready, isMove && io.move.command.ready) + when(io.execution.command.valid && state === idle && !reset.asBool) { + assert( + b.taskEnabled.B && (isTask || isMove), + "CPU-only controller received an unsupported compute instruction" + ) + } + io.move.command.valid := state === idle && io.execution.command.valid && isMove && !reset.asBool + io.move.command.bits.sourceThread := instruction.rs1Data(7, 0) + io.move.command.bits.targetThread := instruction.rs1Data(15, 8) + io.move.command.bits.sourceBank := instruction.rs1Data(25, 16) + io.move.command.bits.targetBank := instruction.rs1Data(35, 26) + io.move.command.bits.sourceAddr := instruction.rs2Data(15, 0) + io.move.command.bits.targetAddr := instruction.rs2Data(31, 16) + io.move.command.bits.rows := instruction.rs2Data(47, 32) +& 1.U + + when(io.execution.command.fire) { + writeResult := instruction.raw_inst(14) + tag := io.execution.command.bits.tag + rd := instruction.raw_inst(11, 7) + state := Mux(isTask, task, moving) + } + io.move.completion.ready := state === moving && !reset.asBool + when(io.move.completion.fire) { + failedMove := io.move.completion.bits + state := Mux(writeResult, moveResponse, releasing) + } + + io.execution.response.valid := ((state === task && io.taskResponse.valid && writeResult) || state === moveResponse) && !reset.asBool + io.execution.response.bits := io.taskResponse.bits + when(state === moveResponse) { + io.execution.response.bits.rd := rd + io.execution.response.bits.data := failedMove.asUInt + } + io.taskResponse.ready := state === task && (!writeResult || io.execution.response.ready) && !reset.asBool + when(io.taskResponse.fire || (state === moveResponse && io.execution.response.fire))(state := releasing) + io.execution.complete.valid := state === releasing && !reset.asBool + io.execution.complete.bits.tag := tag + when(io.execution.complete.fire)(state := idle) + // A move may have retired at CPU dispatch while its work is still pending here. + // Do not let a following CPU store publish a completion flag before the bank move ends. + val pendingMove = io.execution.command.valid && isMove + io.memoryAllowed := !(pendingMove || state === moving || state === moveResponse || state === releasing) + io.idle := state === idle +} + +/** One posted move command remains busy until the shared fabric reports completion. */ +@instantiable +class MoveAdmission(dataBits: Int, permission: PmpParams) extends Module { + + @public + val io = IO(new Bundle { + val rocc = new RoCCIO(dataBits, permission) + val move = new MvoverPort + }) + + val moving = RegInit(false.B) + val command = io.rocc.cmd + val instruction = command.bits.instruction + io.move.command.valid := command.valid && !moving + io.move.command.bits.sourceThread := instruction.rs1Data(7, 0) + io.move.command.bits.targetThread := instruction.rs1Data(15, 8) + io.move.command.bits.sourceBank := instruction.rs1Data(25, 16) + io.move.command.bits.targetBank := instruction.rs1Data(35, 26) + io.move.command.bits.sourceAddr := instruction.rs2Data(15, 0) + io.move.command.bits.targetAddr := instruction.rs2Data(31, 16) + io.move.command.bits.rows := instruction.rs2Data(47, 32) +& 1.U + command.ready := io.move.command.ready && !moving + when(command.valid) { + assert( + instruction.raw_inst(6, 0) === "h7b".U && + instruction.raw_inst(31, 25) === MvoverISA.Funct.U && instruction.raw_inst(14, 12) === 3.U, + "Move admission only accepts posted bank moves" + ) + } + when(io.move.command.fire)(moving := true.B) + io.move.completion.ready := moving + when(io.move.completion.fire)(moving := false.B) + io.rocc.busy := moving + io.rocc.interrupt := false.B + io.rocc.resp.valid := false.B + io.rocc.resp.bits := 0.U.asTypeOf(io.rocc.resp.bits) +} diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/scala/DirectAdmission.scala b/arch/src/main/scala/framework/core/controller/admission/src/main/scala/DirectAdmission.scala new file mode 100644 index 000000000..c9d207aa5 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/scala/DirectAdmission.scala @@ -0,0 +1,81 @@ +package framework.core.controller.admission + +import framework.lib.queue.Queue +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.core.controller.configs.CommandsParam +import framework.acc.frontend.RoCCIO + +/** Tracks posted accelerator commands and orders cross-core management after their drain. */ +@instantiable +class DirectAdmission(p: CommandsParam) extends Module { + require(p.remote) + + @public + val io = IO(new Bundle { + val execution = Flipped(new Commands(p.tracking, p.dataBits, p.permission)) + val accelerator = Flipped(new RoCCIO(p.dataBits, p.permission)) + val remote = new TaskEndpoint + val drained = Output(Bool()) + }) + + val idle :: managing :: releasing :: Nil = Enum(3) + val state = RegInit(idle) + val tags = Instantiate(new Queue(UInt(p.tracking.idBits.W), p.tracking.entries)) + val destination = Reg(UInt(5.W)) + val writeResult = Reg(Bool()) + val command = io.execution.command.bits + val instruction = command.instruction + val management = instruction.raw_inst(6, 0) === "h2b".U + val available = state === idle && !io.remote.busy && !reset.asBool && tags.io.enq.ready && + (!management || (!io.accelerator.busy && !tags.io.deq.valid)) + + io.accelerator.cmd.valid := io.execution.command.valid && available && !management + io.accelerator.cmd.bits.instruction := instruction + io.accelerator.cmd.bits.satp := command.satp + io.accelerator.cmd.bits.effectivePrivilege := command.effectivePrivilege + io.accelerator.cmd.bits.sum := command.sum + io.accelerator.cmd.bits.mxr := command.mxr + io.accelerator.cmd.bits.pmp := command.pmp + io.accelerator.resp.ready := false.B + + io.remote.request.valid := io.execution.command.valid && available && management + io.remote.request.bits.operation := instruction.raw_inst(31, 25) + io.remote.request.bits.context := instruction.rs1Data(63, 32) + io.remote.request.bits.field := instruction.rs1Data(31, 0) + io.remote.request.bits.data := instruction.rs2Data + io.execution.command.ready := available && Mux(management, io.remote.request.ready, io.accelerator.cmd.ready) + when(io.execution.command.fire) { + assert( + Mux( + management, + instruction.raw_inst(31, 25) >= 13.U, + instruction.raw_inst(6, 0) === "h7b".U && !instruction.raw_inst(14) + ), + "Direct core received an unsupported command" + ) + destination := instruction.raw_inst(11, 7) + writeResult := instruction.raw_inst(14) + when(management)(state := managing) + } + tags.io.enq.valid := io.execution.command.fire + tags.io.enq.bits := command.tag + io.execution.response.valid := state === managing && writeResult && io.remote.reply.valid + io.execution.response.bits.rd := destination + io.execution.response.bits.data := io.remote.reply.bits + io.remote.reply.ready := state === managing && (!writeResult || io.execution.response.ready) + when(io.remote.reply.fire)(state := releasing) + val completed = state === releasing || (state === idle && !io.accelerator.busy) + io.execution.complete.valid := tags.io.deq.valid && completed + io.execution.complete.bits.tag := tags.io.deq.bits + tags.io.deq.ready := io.execution.complete.ready && completed + when(io.execution.complete.fire && state === releasing)(state := idle) + io.drained := state === idle && !tags.io.deq.valid && !io.accelerator.busy && !io.remote.busy + when(!reset.asBool) { + assert(!io.accelerator.resp.valid, "Direct accelerator returned an unexpected register result") + assert(!io.accelerator.interrupt, "Direct accelerator fault") + } +} diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/scala/Emit.scala b/arch/src/main/scala/framework/core/controller/admission/src/main/scala/Emit.scala new file mode 100644 index 000000000..207e94758 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/scala/Emit.scala @@ -0,0 +1,134 @@ +package framework.core.controller.admission + +import chisel3._ +import chisel3.reflect.DataMirror +import java.nio.file.{Files, Paths} +import framework.core.controller.configs.CommandsParam +import framework.core.memory.access.preflight.configs.PmpParams +import framework.ip.memory.interlock.{Params => TrackingParam} + +object Emit extends App { + val Array("--config", config, "--output-dir", directory) = args + val values = ujson.read(Files.readString(Paths.get(config))) + + implicit val trackingRW: upickle.default.ReadWriter[TrackingParam] = upickle.default.macroRW + + val p = CommandsParam( + upickle.default.read[TrackingParam](values("tracking")), + values("dataBits").num.toInt, + upickle.default.read[PmpParams](values("permission")), + values("moves").bool, + values("taskEnabled").bool, + values("remote").bool + ) + + Files.createDirectories(Paths.get(directory)) + + def leaves(data: Data, path: String): Seq[(String, Data)] = data match { + case record: Record => record.elements.toSeq.flatMap { case (name, field) => + leaves(field, if (path.isEmpty) name else s"${path}_$name") + } + case vector: Vec[_] => vector.toSeq.zipWithIndex.flatMap { case (field, index) => leaves(field, s"${path}_$index") } + case field => if (field.getWidth > 0) Seq(path -> field) else Seq.empty + } + + for ((top, kind) <- Seq("CommandAdmission", "MoveAdmission", "DirectAdmission", "TaskRouter").zipWithIndex) { + val definitions = scala.collection.mutable.ArrayBuffer[String]( + s"`define AD_KIND $kind", + s"`define AD_ENTRIES ${p.tracking.entries}", + s"`define AD_DATA_BITS ${p.dataBits}" + ) + val binding = scala.collection.mutable.ArrayBuffer[String]() + val offsets = scala.collection.mutable.Map[String, Int]() + val widths = scala.collection.mutable.Map[String, Int]() + var inputOffset = 0; var outputOffset = 0 + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + { + val (dut, io) = kind match { + case 0 => val m = new CommandAdmission(p); (m, m.io) + case 1 => val m = new MoveAdmission(p.dataBits, p.permission); (m, m.io) + case 2 => val m = new DirectAdmission(p); (m, m.io) + case 3 => val m = new TaskRouter(p); (m, m.io) + } + for ((name, field) <- leaves(io, "")) { + val input = DataMirror.directionOf(field) == ActualDirection.Input + val direction = if (input) "IN" else "OUT"; val signal = if (input) "inputs" else "outputs"; + val offset = if (input) inputOffset else outputOffset; val width = field.getWidth + definitions += s"`define AD_${direction}_${name.toUpperCase}_OFFSET $offset"; + definitions += s"`define AD_${direction}_${name.toUpperCase}_WIDTH $width" + offsets(name) = offset; widths(name) = width + binding += s".io_$name(bus.$signal[$offset +: $width])"; + if (input) inputOffset += width else outputOffset += width + }; dut + }, + args = Array("--target-dir", s"$directory/$top", "--split-verilog") + ) + val command = kind match { case 1 => "rocc_cmd"; case 3 => "command"; case _ => "execution_command" } + val source = s"${command}_bits_" + val instruction = s"${source}instruction_" + val maps = scala.collection.mutable.ArrayBuffer[(Int, Int, Int, Int)]() + def copyPayload(target: String): Unit = for (name <- offsets.keys.toSeq.sorted if name.startsWith(target)) { + val from = source + name.stripPrefix(target); + maps += ((offsets(from), offsets(name), widths(name), offsets(name.take(name.indexOf("_bits")) + "_valid"))) + } + def control(target: String): Unit = { + for ( + (name, from, lsb) <- + Seq(("operation", "raw_inst", 25), ("context", "rs1Data", 32), ("field", "rs1Data", 0), ("data", "rs2Data", 0)) + ) { + val to = s"${target}_$name"; + maps += ( + ( + offsets(instruction + from) + lsb, + offsets(to), + widths(to), + offsets(to.take(to.indexOf("_bits")) + "_valid") + ) + ) + } + } + kind match { + case 0 => copyPayload("taskRequest_bits_") + case 1 => () + case 2 => copyPayload("accelerator_cmd_bits_"); control("remote_request_bits") + case 3 => + control("local_request_bits_control"); control("remote_request_bits") + for (name <- offsets.keys.toSeq.sorted if name.startsWith("local_request_bits_context_")) { + val from = source + name.stripPrefix("local_request_bits_context_"); + maps += ((offsets(from), offsets(name), widths(name), offsets(name.take(name.indexOf("_bits")) + "_valid"))) + } + } + if (kind == 0 || kind == 1) { + for ( + (name, from, lsb) <- Seq( + ("sourceThread", "rs1Data", 0), + ("targetThread", "rs1Data", 8), + ("sourceBank", "rs1Data", 16), + ("targetBank", "rs1Data", 26), + ("sourceAddr", "rs2Data", 0), + ("targetAddr", "rs2Data", 16) + ) + ) { + val to = s"move_command_bits_$name"; + maps += ( + ( + offsets(instruction + from) + lsb, + offsets(to), + widths(to), + offsets(to.take(to.indexOf("_bits")) + "_valid") + ) + ) + } + } + definitions += s"`define AD_PAYLOAD_MAP ${maps.map { case (from, to, width, gate) => + s"'{$from,$to,$width,$gate}" + }.mkString("'{", ",", "}")}" + definitions += s"`define AD_PAYLOAD_FIELDS ${maps.size}" + val commandFields = offsets.keys.filter(_.startsWith(source)).toSeq + definitions += s"`define AD_COMMAND_OFFSET ${commandFields.map(offsets).min}" + definitions += s"`define AD_COMMAND_WIDTH ${commandFields.map(widths).sum}" + definitions += s"`define AD_INPUT_BITS $inputOffset"; definitions += s"`define AD_OUTPUT_BITS $outputOffset" + Files.writeString(Paths.get(directory, top, "parameters.svh"), definitions.mkString("\n") + "\n") + Files.writeString(Paths.get(directory, top, "ports.svh"), binding.mkString(",\n") + "\n") + } +} diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/scala/TaskRouter.scala b/arch/src/main/scala/framework/core/controller/admission/src/main/scala/TaskRouter.scala new file mode 100644 index 000000000..699f99857 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/scala/TaskRouter.scala @@ -0,0 +1,85 @@ +package framework.core.controller.admission + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.core.controller.configs.CommandsParam +import framework.device.cbus.ControlRequest +import framework.core.controller.task.TaskRequest +import framework.acc.frontend.Response + +class TaskEndpoint extends Bundle { + val request = Decoupled(new ControlRequest) + val reply = Flipped(Decoupled(UInt(64.W))) + val busy = Input(Bool()) +} + +/** Retains the reply destination and owner of each accepted management command. */ +@instantiable +class TaskRouter(b: CommandsParam) extends Module { + val remoteEnabled = b.remote + + @public + val io = IO(new Bundle { + + val command = Flipped(Decoupled(new CommandSnapshot( + b.tracking, + b.dataBits, + b.permission + ))) + + val response = Decoupled(new Response(b.dataBits)) + val busy = Output(Bool()) + + val local = new Bundle { + val request = Decoupled(new TaskRequest(b.permission)) + val reply = Flipped(Decoupled(UInt(64.W))) + val busy = Input(Bool()) + } + + val remote = Option.when(remoteEnabled)(new TaskEndpoint) + }) + + val pending = RegInit(false.B) + val remoteOwner = Reg(Bool()) + val destination = Reg(UInt(5.W)) + val command = io.command.bits + val instruction = command.instruction + val remote = instruction.raw_inst(31, 25) >= 13.U + val request = Wire(new ControlRequest) + request.operation := instruction.raw_inst(31, 25) + request.context := instruction.rs1Data(63, 32) + request.field := instruction.rs1Data(31, 0) + request.data := instruction.rs2Data + io.local.request.bits.control := request + io.local.request.bits.context.satp := command.satp + io.local.request.bits.context.effectivePrivilege := command.effectivePrivilege + io.local.request.bits.context.sum := command.sum + io.local.request.bits.context.mxr := command.mxr + io.local.request.bits.context.pmp := command.pmp + io.local.request.valid := io.command.valid && !pending && !remote + io.local.reply.ready := pending && !remoteOwner && io.response.ready + io.response.bits.rd := destination + if (remoteEnabled) { + val port = io.remote.get + port.request.bits := request + port.request.valid := io.command.valid && !pending && remote + port.reply.ready := pending && remoteOwner && io.response.ready + io.command.ready := !pending && Mux(remote, port.request.ready, io.local.request.ready) + io.response.valid := pending && Mux(remoteOwner, port.reply.valid, io.local.reply.valid) + io.response.bits.data := Mux(remoteOwner, port.reply.bits, io.local.reply.bits) + io.busy := pending || io.local.busy || port.busy + } else { + io.command.ready := !pending && !remote && io.local.request.ready + io.response.valid := pending && io.local.reply.valid + io.response.bits.data := io.local.reply.bits + io.busy := pending || io.local.busy + when(io.command.valid)(assert(!remote, "No cross-core task endpoint")) + } + when(io.command.fire) { + pending := true.B + remoteOwner := remote + destination := instruction.raw_inst(11, 7) + } + when(io.response.fire)(pending := false.B) +} diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/core/controller/admission/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..3ae1d8536 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/scala/configs/Params.scala @@ -0,0 +1,7 @@ +package framework.core.controller.admission.configs + +import framework.core.memory.access.preflight.configs.PmpParams +import upickle.default.{macroRW, ReadWriter} + +case class Params(permission: PmpParams, moves: Boolean) +object Params { implicit val rw: ReadWriter[Params] = macroRW } diff --git a/arch/src/main/scala/framework/core/controller/build.sc b/arch/src/main/scala/framework/core/controller/build.sc new file mode 100644 index 000000000..6ae568944 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/build.sc @@ -0,0 +1,16 @@ +import mill._ +import $file.admission.{build => admission_build} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("Controller.scala")), + PathRef(millSourcePath / os.RelPath("configs/ControllerParam.scala")), + PathRef(millSourcePath / os.RelPath("task/Control.scala")), + PathRef(millSourcePath / os.RelPath("task/Service.scala")) + ) ++ + admission_build.sources() +) + +def emitSources = T.sources( + admission_build.emitSources() +) diff --git a/arch/src/main/scala/framework/core/controller/configs/ControllerParam.scala b/arch/src/main/scala/framework/core/controller/configs/ControllerParam.scala new file mode 100644 index 000000000..0c60f424b --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/configs/ControllerParam.scala @@ -0,0 +1,26 @@ +package framework.core.controller.configs + +import framework.thread.ant.configs.AntParam +import framework.core.tss.configs.{Params => TssParam} +import framework.core.memory.access.preflight.configs.PmpParams +import framework.ip.memory.interlock.{Params => TrackingParam} + +case class CommandsParam( + tracking: TrackingParam, + dataBits: Int, + permission: PmpParams, + moves: Boolean, + taskEnabled: Boolean, + remote: Boolean) + +case class TasksParam( + locals: Seq[AntParam], + signatures: Seq[BigInt], + tss: TssParam, + permission: PmpParams) { + val count = signatures.size + require(count > 0 && locals.size == count && tss.ports == count) + require(locals.forall(p => p.shared == tss.memory && p.taskBits <= 64)) +} + +case class ControllerParam(commands: CommandsParam, tasks: Option[TasksParam]) diff --git a/arch/src/main/scala/framework/core/controller/configs/default.toml b/arch/src/main/scala/framework/core/controller/configs/default.toml new file mode 100644 index 000000000..37a096595 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/configs/default.toml @@ -0,0 +1 @@ +moves = false diff --git a/arch/src/main/scala/framework/core/controller/task/Control.scala b/arch/src/main/scala/framework/core/controller/task/Control.scala new file mode 100644 index 000000000..963f4def4 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/task/Control.scala @@ -0,0 +1,165 @@ +package framework.core.controller.task + +import framework.core.controller.configs.TasksParam +import framework.core.memory.access.preflight.configs.PmpParams + +import framework.thread.ant.{Launch, TaskContext} +import framework.thread.ant.Completion +import framework.device.cbus.ControlRequest +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} + +object ControlOp { + val Query = 0 + val Stage = 1 + val Start = 2 + val Acknowledge = 3 + val Cancel = 4 + val Acquire = 5 + val Release = 6 +} + +/** A management request carries the submitter's context through every accepting endpoint. */ +class TaskRequest(permission: PmpParams) extends Bundle { + val control = new ControlRequest + val context = new TaskContext(permission) +} + +/** + * Task registers for one software owner. Invalid descriptors and resource reuse are + * programming errors. Execution contract violations assert at their source. + */ +@instantiable +class Control(val b: TasksParam) extends Module { + val contexts = b.count + val signatures = VecInit(b.signatures.map(_.U(64.W))) + require(contexts > 0) + + @public + val io = IO(new Bundle { + val request = Flipped(Decoupled(new TaskRequest(b.permission))) + val response = Decoupled(UInt(64.W)) + val storageBusy = Input(Bool()) + + val start = MixedVec(b.locals.map(p => Decoupled(new Launch(p, b.permission)))) + val result = MixedVec(b.locals.map(p => Flipped(Decoupled(new Completion(p))))) + + val cancel = Output(Vec(contexts, Bool())) + val inUse = Output(Bool()) + val available = Output(Vec(contexts, Bool())) + }) + + val group = RegInit(false.B) + val owned = RegInit(VecInit(Seq.fill(contexts)(false.B))) + val done = RegInit(VecInit(Seq.fill(contexts)(false.B))) + val staged = Reg(Vec(contexts, Vec(6, UInt(64.W)))) + val fields = RegInit(VecInit(Seq.fill(contexts)(0.U(6.W)))) + + val results = Reg(Vec( + contexts, + new Bundle { + val task = UInt(64.W) + val value = UInt(64.W) + val cancelled = Bool() + } + )) + + val pending = RegInit(false.B) + val answer = Reg(UInt(64.W)) + val req = io.request.bits.control + val index = req.context(math.max(1, log2Ceil(contexts)) - 1, 0) + val inRange = req.context < contexts.U + val free = !owned(index) && !done(index) + val canStart = inRange && free && group && + fields(index) === 63.U && staged(index)(5) === signatures(index) + val launching = req.operation === ControlOp.Start.U && canStart + io.request.ready := !pending && !reset.asBool && (!launching || VecInit(io.start.map(_.ready))(index)) + io.response.valid := pending && !reset.asBool + io.response.bits := answer + io.inUse := group + when(io.response.fire)(pending := false.B) + for (i <- 0 until contexts) { + io.available(i) := !owned(i) && !done(i) + io.start(i).valid := io.request.valid && !pending && !reset.asBool && launching && req.context === i.U + io.start(i).bits.descriptor.task := staged(i)(0) + io.start(i).bits.descriptor.entry := staged(i)(1) + io.start(i).bits.descriptor.codeEnd := staged(i)(2) + io.start(i).bits.descriptor.argument := staged(i)(3) + io.start(i).bits.descriptor.stack := staged(i)(4) + io.start(i).bits.context := io.request.bits.context + // Cancel accepts immediately, independently of the start-ready path. + io.cancel(i) := io.request.valid && !pending && !reset.asBool && + req.operation === ControlOp.Cancel.U && req.context === i.U && owned(i) + io.result(i).ready := owned(i) && !done(i) && !reset.asBool + when(io.result(i).valid && !reset.asBool) { + assert(owned(i) && !done(i), "Ant result arrived without a live task") + assert(io.result(i).bits.task === staged(i)(0), "Ant completion task does not match its submission") + } + when(io.result(i).fire) { + results(i).task := io.result(i).bits.task + results(i).value := io.result(i).bits.value + results(i).cancelled := io.result(i).bits.cancelled + owned(i) := false.B + done(i) := true.B + } + } + when(io.request.fire) { + pending := true.B + answer := 0.U + assert(req.operation <= ControlOp.Release.U, "Invalid Ant control operation") + when(req.operation <= ControlOp.Cancel.U && !(req.operation === ControlOp.Query.U && req.field === 7.U)) { + assert(inRange, "Invalid Ant context index") + } + switch(req.operation) { + is(ControlOp.Query.U) { + assert(req.field < 10.U, "Invalid Ant query field") + when(req.field >= 8.U)(assert(done(index), "Ant result read before completion")) + answer := MuxLookup(req.field, 0.U)(Seq( + 0.U -> signatures(index), + 1.U -> Cat(done(index) && results(index).cancelled, done(index), owned(index), true.B), + 2.U -> VecInit(b.locals.map(p => p.codeBytes.U(64.W)))(index), + 3.U -> VecInit(b.locals.map(p => p.data.base.U(64.W)))(index), + 4.U -> VecInit(b.locals.map(p => p.data.bytes.U(64.W)))(index), + 5.U -> b.tss.memory.base.U, + 6.U -> b.tss.memory.bytes.U, + 7.U -> contexts.U, + 8.U -> results(index).task, + 9.U -> results(index).value + )) + } + is(ControlOp.Stage.U) { + assert(free, "Ant descriptor overwritten before resource release") + assert(req.field < 6.U, "Invalid Ant descriptor field") + when(req.field === 0.U)(assert( + VecInit(b.locals.map(p => (req.data >> p.taskBits) === 0.U))(index), + "Ant task ID overflow" + )) + staged(index)(req.field(2, 0)) := req.data + fields(index) := fields(index) | UIntToOH(req.field, 6) + } + is(ControlOp.Start.U) { + assert(free, "Ant context reused before completion acknowledgement") + assert(group, "Ant submitted without TSS group ownership") + assert(fields(index) === 63.U, "Incomplete Ant descriptor") + assert(staged(index)(5) === signatures(index), "Ant program fingerprint mismatch") + when(canStart) { owned(index) := true.B; fields(index) := 0.U } + } + is(ControlOp.Acknowledge.U) { + assert(done(index), "Ant completion acknowledged before result") + done(index) := false.B + } + is(ControlOp.Acquire.U) { + assert(!group && !io.storageBusy, "Ant group acquired before loader completion or previous release") + group := true.B + } + is(ControlOp.Release.U) { + assert( + group && !owned.asUInt.orR && !done.asUInt.orR && !io.storageBusy, + "Ant group released before tasks and storage drained" + ) + group := false.B + } + } + } +} diff --git a/arch/src/main/scala/framework/core/controller/task/Service.scala b/arch/src/main/scala/framework/core/controller/task/Service.scala new file mode 100644 index 000000000..303a4850b --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/task/Service.scala @@ -0,0 +1,118 @@ +package framework.core.controller.task + +import chisel3.experimental.hierarchy.Instance + +import framework.core.controller.configs.TasksParam +import framework.core.memory.access.preflight.configs.PmpParams + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import framework.core.tss.TssControlPort +import framework.thread.ant.Task +import framework.ip.memory.spm + +/** + * Core execution service. The caller supplies the uncached management transport and + * task-bound NPU/DMA owner; neither CPU cache nor address translation lives here. + */ +@instantiable +class Service(val b: TasksParam) extends Module { + val contexts = b.count + + @public + val io = IO(new Bundle { + val request = Flipped(Decoupled(new TaskRequest(b.permission))) + val reply = Decoupled(UInt(64.W)) + + val tasks = MixedVec(b.locals.map(p => Flipped(new Task(p, b.permission)))) + + val idle = Output(Bool()) + val tss = new TssControlPort(b.tss) + }) + + val control: Instance[Control] = Instantiate(new Control(b)) + + require(b.tss.ports == contexts) + + val transferBusy = Wire(Bool()) + val running = VecInit(io.tasks.map(_.running)) + val storageBusy = io.tasks.map(_.storageBusy).reduce(_ || _) || + io.tss.busy.asUInt.orR || io.tss.hostBusy + control.io.storageBusy := storageBusy + io.tss.inUse := control.io.inUse || running.asUInt.orR || transferBusy + io.tss.cancel := control.io.cancel + for (i <- 0 until contexts) { + io.tasks(i).inUse := control.io.inUse + io.tasks(i).start.valid := control.io.start(i).valid && !io.tss.hostBusy && !io.tss.busy(i) + io.tasks(i).start.bits := control.io.start(i).bits + control.io.start(i).ready := io.tasks(i).start.ready && !io.tss.hostBusy && !io.tss.busy(i) + // A cancelled Ant may finish locally before its accepted TSS access drains. + control.io.result(i).valid := io.tasks(i).result.valid && !io.tss.busy(i) + control.io.result(i).bits := io.tasks(i).result.bits + io.tasks(i).result.ready := control.io.result(i).ready && !io.tss.busy(i) + io.tasks(i).cancel := control.io.cancel(i) + } + val idle :: controlReply :: transferring :: sharedResponse :: loadReply :: Nil = Enum(5) + val state = RegInit(idle) + val request = io.request.bits.control + val isTransfer = request.operation >= 7.U && request.operation <= 10.U + val isShared = request.operation === 11.U || request.operation === 12.U + val isLoad = isTransfer || isShared + val localIndex = request.context(math.max(1, log2Ceil(contexts)) - 1, 0) + val owner = Reg(UInt(math.max(1, log2Ceil(contexts)).W)) + val value = Reg(UInt(64.W)) + transferBusy := state === transferring + val transferAllowed = request.context < contexts.U && control.io.available(localIndex) && + !running(localIndex) && !VecInit(io.tasks.map(_.storageBusy))(localIndex) && !io.tss.busy(localIndex) + for (i <- 0 until contexts) { + val task = io.tasks(i) + task.transfer.valid := state === idle && io.request.valid && isTransfer && transferAllowed && request.context === i.U + task.transfer.bits.code := request.operation <= 8.U + task.transfer.bits.toLocal := !request.operation(0) + task.transfer.bits.localOffset := request.field + task.transfer.bits.sharedOffset := request.data(31, 0) + task.transfer.bits.bytes := request.data(63, 32) + task.transferred.ready := state === transferring && owner === i.U + when(task.transferred.fire) { + assert(task.transferred.bits, "Ant TSS transfer failed") + value := 0.U + state := loadReply + } + } + val sharedValid = request.context === 0.U && request.field(2, 0) === 0.U && + (request.field +& 8.U) <= b.tss.memory.bytes.U + val port = io.tss.host + port.request.valid := state === idle && io.request.valid && isShared && sharedValid && !reset.asBool + port.request.bits := 0.U.asTypeOf(port.request.bits) + port.request.bits.address := b.tss.memory.base.U + request.field + port.request.bits.size := 3.U + port.request.bits.write := request.operation === 12.U + port.request.bits.data := request.data + port.request.bits.mask := Mux(request.operation === 12.U, 255.U, 0.U) + port.response.ready := state === sharedResponse + when(port.response.fire) { + assert(!port.response.bits.error, "TSS management access failed") + value := port.response.bits.data(63, 0) + state := loadReply + } + when(state === idle && io.request.valid && isLoad && !reset.asBool) { + assert(Mux(isTransfer, transferAllowed, sharedValid), "Invalid TSS transfer or storage ownership") + } + control.io.request.valid := state === idle && io.request.valid && !isLoad && !reset.asBool + control.io.request.bits := io.request.bits + io.request.ready := state === idle && !reset.asBool && Mux( + isTransfer, + transferAllowed && VecInit(io.tasks.map(_.transfer.ready))(localIndex), + Mux(isShared, sharedValid && port.request.ready, control.io.request.ready) + ) + when(io.request.fire) { + owner := localIndex + state := Mux(isTransfer, transferring, Mux(isShared, sharedResponse, controlReply)) + } + io.reply.valid := state === loadReply || (state === controlReply && control.io.response.valid) + io.reply.bits := Mux(state === loadReply, value, control.io.response.bits) + control.io.response.ready := state === controlReply && io.reply.ready + when(io.reply.fire)(state := idle) + io.idle := state === idle && !running.asUInt.orR && !storageBusy +} diff --git a/arch/src/main/scala/framework/core/execution/Execution.scala b/arch/src/main/scala/framework/core/execution/Execution.scala new file mode 100644 index 000000000..62c5e6b83 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/Execution.scala @@ -0,0 +1,60 @@ +package framework.core.execution + +import chisel3.util._ +import framework.acc.frontend.RoCCIO +import framework.core.controller.admission.Commands +import framework.core.clink.CoreInterrupts +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.core.execution.configs.ExecutionParam +import framework.core.execution.pipeline.{Retirement, Sequencer} +import framework.core.execution.instruction.Instruction +import framework.core.execution.register_file.RegisterFile +import framework.core.execution.integer.Integer +import framework.core.execution.muldiv.MulDiv +import framework.core.execution.fpu.FPU +import framework.core.execution.privileged.Privileged +import framework.core.memory.access.{UncachedRequest, UncachedResponse, VirtualPort} + +/** Composes instruction execution; memory requests are served by the core memory hierarchy. */ +@instantiable +class Execution(b: ExecutionParam, localRocc: Boolean) extends Module { + + @public + val io = IO(new Bundle { + val interrupts = Input(new CoreInterrupts) + val fetch = Flipped(new VirtualPort(b.memoryAccess)) + val data = Flipped(new VirtualPort(b.memoryAccess)) + val timeRequest = Decoupled(new UncachedRequest(b.memoryAccess.memory)) + val timeResponse = Flipped(Decoupled(new UncachedResponse(b.memoryAccess.memory))) + val controller = Option.when(!localRocc)(new Commands(b.tracking, b.integer.xLen, b.privileged.permission)) + val rocc = Option.when(localRocc)(Flipped(new RoCCIO(b.integer.xLen, b.privileged.permission))) + val workDrained = Input(Bool()) + val retirement = Valid(new Retirement) + }) + + val sequencer: Instance[Sequencer] = Instantiate(new Sequencer(b, localRocc)) + val instruction: Instance[Instruction] = Instantiate(new Instruction(b.instruction)) + val registers: Instance[RegisterFile] = Instantiate(new RegisterFile(b.registerFile)) + val integer: Instance[Integer] = Instantiate(new Integer(b.integer)) + val multiply: Instance[MulDiv] = Instantiate(new MulDiv(b.muldiv)) + val floating: Instance[FPU] = Instantiate(new FPU(b.fpu)) + val privileged: Instance[Privileged] = Instantiate(new Privileged(b.privileged)) + + sequencer.io.instruction <> instruction.io + sequencer.io.registers <> registers.io + sequencer.io.integer <> integer.io + sequencer.io.multiply <> multiply.io + sequencer.io.floating <> floating.io + sequencer.io.privileged <> privileged.io + sequencer.io.interrupts := io.interrupts + sequencer.io.controller.zip(io.controller).foreach { case (child, parent) => child <> parent } + sequencer.io.rocc.zip(io.rocc).foreach { case (child, parent) => child <> parent } + io.retirement := sequencer.io.retirement + sequencer.io.workDrained := io.workDrained + + io.fetch <> sequencer.io.fetch + io.data <> sequencer.io.memory + io.timeRequest <> sequencer.io.timeRequest + sequencer.io.timeResponse <> io.timeResponse +} diff --git a/arch/src/main/scala/framework/core/execution/build.sc b/arch/src/main/scala/framework/core/execution/build.sc new file mode 100644 index 000000000..48dd6158f --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/build.sc @@ -0,0 +1,33 @@ +import mill._ +import $file.fetch.{build => fetch_build} +import $file.fpu.{build => fpu_build} +import $file.instruction.{build => instruction_build} +import $file.integer.{build => integer_build} +import $file.muldiv.{build => muldiv_build} +import $file.privileged.{build => privileged_build} +import $file.register_file.{build => register_file_build} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("Execution.scala")), + PathRef(millSourcePath / os.RelPath("configs/ExecutionParam.scala")), + PathRef(millSourcePath / os.RelPath("pipeline/Sequencer.scala")) + ) ++ + fetch_build.sources() ++ + fpu_build.sources() ++ + instruction_build.sources() ++ + integer_build.sources() ++ + muldiv_build.sources() ++ + privileged_build.sources() ++ + register_file_build.sources() +) + +def emitSources = T.sources( + fetch_build.emitSources() ++ + fpu_build.emitSources() ++ + instruction_build.emitSources() ++ + integer_build.emitSources() ++ + muldiv_build.emitSources() ++ + privileged_build.emitSources() ++ + register_file_build.emitSources() +) diff --git a/arch/src/main/scala/framework/core/execution/configs/ExecutionParam.scala b/arch/src/main/scala/framework/core/execution/configs/ExecutionParam.scala new file mode 100644 index 000000000..14eca1a71 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/configs/ExecutionParam.scala @@ -0,0 +1,26 @@ +package framework.core.execution.configs + +import framework.core.execution.instruction.configs.{Params => InstructionParam} +import framework.core.execution.register_file.configs.{Params => RegisterFileParam} +import framework.core.execution.integer.configs.{Params => IntegerParam} +import framework.core.execution.muldiv.configs.{Params => MulDivParam} +import framework.core.execution.fpu.configs.{Params => FpuParam} +import framework.core.execution.privileged.configs.{Params => PrivilegedParam} +import framework.core.memory.access.configs.{Params => AccessParam} +import framework.ip.memory.interlock.{Params => TrackingParam} + +case class ExecutionParam( + hartId: Int, + resetVector: BigInt, + commandsEnabled: Boolean, + directAccelerator: Boolean, + moves: Boolean, + tracking: TrackingParam, + mtimeAddress: BigInt, + memoryAccess: AccessParam, + instruction: InstructionParam, + registerFile: RegisterFileParam, + integer: IntegerParam, + muldiv: MulDivParam, + fpu: FpuParam, + privileged: PrivilegedParam) diff --git a/arch/src/main/scala/framework/core/execution/fetch/build.sc b/arch/src/main/scala/framework/core/execution/fetch/build.sc new file mode 100644 index 000000000..05531cd7e --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fetch/build.sc @@ -0,0 +1,22 @@ +import mill._ +import $file.^.^.^.^.^.^.^.settings + + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.core.execution.fetch.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Fetch.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/core/execution/fetch/configs/uvm.toml b/arch/src/main/scala/framework/core/execution/fetch/configs/uvm.toml new file mode 100644 index 000000000..50aab1014 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fetch/configs/uvm.toml @@ -0,0 +1,4 @@ +[parameters] +instructionBytes = 2 + +[defines] diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/csrc/Cargo.lock b/arch/src/main/scala/framework/core/execution/fetch/src/csrc/Cargo.lock similarity index 100% rename from arch/src/main/scala/framework/mem-core/fetch/src/csrc/Cargo.lock rename to arch/src/main/scala/framework/core/execution/fetch/src/csrc/Cargo.lock diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/csrc/Cargo.toml b/arch/src/main/scala/framework/core/execution/fetch/src/csrc/Cargo.toml similarity index 100% rename from arch/src/main/scala/framework/mem-core/fetch/src/csrc/Cargo.toml rename to arch/src/main/scala/framework/core/execution/fetch/src/csrc/Cargo.toml diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/csrc/src/instructions.rs b/arch/src/main/scala/framework/core/execution/fetch/src/csrc/src/instructions.rs similarity index 100% rename from arch/src/main/scala/framework/mem-core/fetch/src/csrc/src/instructions.rs rename to arch/src/main/scala/framework/core/execution/fetch/src/csrc/src/instructions.rs diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/csrc/src/lib.rs b/arch/src/main/scala/framework/core/execution/fetch/src/csrc/src/lib.rs similarity index 100% rename from arch/src/main/scala/framework/mem-core/fetch/src/csrc/src/lib.rs rename to arch/src/main/scala/framework/core/execution/fetch/src/csrc/src/lib.rs diff --git a/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/agent/driver.svh b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/agent/driver.svh new file mode 100644 index 000000000..1efe4bd9f --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/agent/driver.svh @@ -0,0 +1,62 @@ +class driver extends uvm_driver #(control_item); + `uvm_component_utils(driver) + virtual fetch_if vif; + program_image image; + control_item pending; + longint unsigned reset_vector = 0, satp = 0; + int privilege = 3; + bit sum = 0, mxr = 0; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual fetch_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Fetch control interface is required") + endfunction + task run_phase(uvm_phase phase); + forever begin + @(negedge vif.clock); + if (pending == null) seq_item_port.try_next_item(pending); + vif.inputs[`FETCH_IN_REDIRECT_VALID_OFFSET+:`FETCH_IN_REDIRECT_VALID_WIDTH] = 0; + vif.inputs[`FETCH_IN_FLUSH_OFFSET+:`FETCH_IN_FLUSH_WIDTH] = 0; + vif.inputs[`FETCH_IN_INVALIDATE_OFFSET+:`FETCH_IN_INVALIDATE_WIDTH] = 0; + if (pending != null) begin + case (pending.kind) + REDIRECT: begin + vif.inputs[`FETCH_IN_REDIRECT_VALID_OFFSET+:`FETCH_IN_REDIRECT_VALID_WIDTH] = 1; + vif.inputs[`FETCH_IN_REDIRECT_BITS_OFFSET+:`FETCH_IN_REDIRECT_BITS_WIDTH]=pending.target; + end + FENCE: begin + vif.inputs[`FETCH_IN_REDIRECT_VALID_OFFSET+:`FETCH_IN_REDIRECT_VALID_WIDTH] = 1; + vif.inputs[`FETCH_IN_REDIRECT_BITS_OFFSET+:`FETCH_IN_REDIRECT_BITS_WIDTH]=pending.target; + vif.inputs[`FETCH_IN_FLUSH_OFFSET+:`FETCH_IN_FLUSH_WIDTH] = 1; + image.version = pending.version; + end + CONTEXT: begin + vif.inputs[`FETCH_IN_REDIRECT_VALID_OFFSET+:`FETCH_IN_REDIRECT_VALID_WIDTH] = 1; + vif.inputs[`FETCH_IN_REDIRECT_BITS_OFFSET+:`FETCH_IN_REDIRECT_BITS_WIDTH]=pending.target; + privilege = pending.privilege; + satp = pending.satp; + sum = pending.sum; + mxr = pending.mxr; + end + INVALIDATE: vif.inputs[`FETCH_IN_INVALIDATE_OFFSET+:`FETCH_IN_INVALIDATE_WIDTH] = 1; + RESET_VECTOR: reset_vector = pending.target; + BAD_FLUSH: vif.inputs[`FETCH_IN_FLUSH_OFFSET+:`FETCH_IN_FLUSH_WIDTH] = 1; + endcase + end + vif.inputs[`FETCH_IN_RESETVECTOR_OFFSET+:`FETCH_IN_RESETVECTOR_WIDTH] = reset_vector; + vif.inputs[`FETCH_IN_CONTEXT_PRIVILEGE_OFFSET+:`FETCH_IN_CONTEXT_PRIVILEGE_WIDTH] = privilege; + vif.inputs[`FETCH_IN_CONTEXT_SATP_OFFSET+:`FETCH_IN_CONTEXT_SATP_WIDTH] = satp; + vif.inputs[`FETCH_IN_CONTEXT_SUM_OFFSET+:`FETCH_IN_CONTEXT_SUM_WIDTH] = sum; + vif.inputs[`FETCH_IN_CONTEXT_MXR_OFFSET+:`FETCH_IN_CONTEXT_MXR_WIDTH] = mxr; + @(posedge vif.clock); + if (pending != null) begin + #1; + seq_item_port.item_done(); + pending = null; + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/agent/item.svh b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/agent/item.svh new file mode 100644 index 000000000..c5eca9906 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/agent/item.svh @@ -0,0 +1,57 @@ +typedef enum int { + REDIRECT, + FENCE, + CONTEXT, + INVALIDATE, + RESET_VECTOR, + BAD_FLUSH +} control_kind; +class program_image; + int version = 0; +endclass +class control_item extends uvm_sequence_item; + `uvm_object_utils(control_item) + control_kind kind; + longint unsigned target, satp; + int privilege, version; + bit sum, mxr; + function new(string name = "control_item"); + super.new(name); + endfunction +endclass +class control_sequence extends uvm_sequence #(control_item); + `uvm_object_utils(control_sequence) + control_item item; + function new(string name = "control_sequence"); + super.new(name); + endfunction + task body(); + start_item(item); + finish_item(item); + endtask +endclass +class sample_item extends uvm_sequence_item; + `uvm_object_utils(sample_item) + logic [`FETCH_INPUT_BITS-1:0] inputs; + logic [`FETCH_OUTPUT_BITS-1:0] outputs; + bit reset; + function new(string name = "sample_item"); + super.new(name); + endfunction +endclass +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(payload, UVM_ALL_ON) + `uvm_object_utils_end + logic [`FETCH_PACKET_BITS-1:0] payload; + function new(string name = "result_item"); + super.new(name); + endfunction +endclass +class word_transaction; + longint unsigned pc, address; + int epoch, version, due; + logic [63:0] data; + logic [511:0] line; + bit executable, pf, af; +endclass diff --git a/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/agent/monitor.svh b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/agent/monitor.svh new file mode 100644 index 000000000..3a5d4a910 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/agent/monitor.svh @@ -0,0 +1,25 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual fetch_if vif; + uvm_analysis_port #(sample_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual fetch_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Fetch monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + sample_item t; + forever begin + @(posedge vif.clock); + t = sample_item::type_id::create("observed"); + t.inputs = vif.inputs; + t.outputs = vif.outputs; + t.reset = vif.reset; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/agent/peer.svh b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/agent/peer.svh new file mode 100644 index 000000000..f94d41949 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/agent/peer.svh @@ -0,0 +1,95 @@ +class peer_driver extends uvm_component; + `uvm_component_utils(peer_driver) + virtual fetch_if vif; + model reference; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual fetch_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Fetch backend interface is required") + endfunction + task run_phase(uvm_phase phase); + forever begin + @(negedge vif.clock); + #1; + vif.inputs[`FETCH_IN_PACKET_READY_OFFSET+:`FETCH_IN_PACKET_READY_WIDTH]=!vif.reset&&reference.packet_ready&&reference.cycle%5>=2; + vif.inputs[`FETCH_IN_REQUEST_READY_OFFSET+:`FETCH_IN_REQUEST_READY_WIDTH]=!vif.reset&&!reference.hold_request&&reference.cycle%4!=0; + vif.inputs[`FETCH_IN_RESPONSE_VALID_OFFSET+:`FETCH_IN_RESPONSE_VALID_WIDTH]=!vif.reset&&reference.word!=null&&!reference.hold_word&&reference.cycle>=reference.word.due; + if (reference.word != null) begin + vif.inputs[`FETCH_IN_RESPONSE_BITS_DATA_OFFSET+:`FETCH_IN_RESPONSE_BITS_DATA_WIDTH]=reference.word.data; + vif.inputs[`FETCH_IN_RESPONSE_BITS_LINE_OFFSET+:`FETCH_IN_RESPONSE_BITS_LINE_WIDTH]=reference.word.line; + vif.inputs[`FETCH_IN_RESPONSE_BITS_LINEEXECUTABLE_OFFSET+:`FETCH_IN_RESPONSE_BITS_LINEEXECUTABLE_WIDTH]=reference.word.executable; + vif.inputs[`FETCH_IN_RESPONSE_BITS_PAGEFAULT_OFFSET+:`FETCH_IN_RESPONSE_BITS_PAGEFAULT_WIDTH]=reference.word.pf; + vif.inputs[`FETCH_IN_RESPONSE_BITS_ACCESSFAULT_OFFSET+:`FETCH_IN_RESPONSE_BITS_ACCESSFAULT_WIDTH]=reference.word.af; + end + vif.inputs[`FETCH_IN_MAINTENANCE_READY_OFFSET+:`FETCH_IN_MAINTENANCE_READY_WIDTH]=!vif.reset&&!reference.hold_maintenance; + vif.inputs[`FETCH_IN_MAINTAINED_VALID_OFFSET+:`FETCH_IN_MAINTAINED_VALID_WIDTH]=!vif.reset&&reference.maintenance_active&&!reference.hold_ack&&reference.cycle>=reference.ack_due; + vif.inputs[`FETCH_IN_MAINTAINED_BITS_OFFSET+:`FETCH_IN_MAINTAINED_BITS_WIDTH] = 1; + end + endtask +endclass +class invalid_peer extends uvm_component; + `uvm_component_utils(invalid_peer) + virtual fetch_if vif; + program_image image; + bit pending_word = 0, pending_ack = 0; + longint unsigned address; + string scenario; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual fetch_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Fetch assertion backend is required") + if (!$value$plusargs("CASE=%s", scenario)) + `uvm_fatal("CASE", "Fetch assertion scenario is required") + endfunction + task run_phase(uvm_phase phase); + logic [511:0] line; + forever begin + @(posedge vif.clock); + if (vif.reset) begin + pending_word = 0; + pending_ack = 0; + end else begin + if ( + `FO(vif.outputs, REQUEST_VALID) + && vif.inputs[`FETCH_IN_REQUEST_READY_OFFSET+:`FETCH_IN_REQUEST_READY_WIDTH]) begin + pending_word = 1; + address = `FO(vif.outputs, REQUEST_BITS_ADDR); + end + if (vif.inputs[`FETCH_IN_RESPONSE_VALID_OFFSET+:`FETCH_IN_RESPONSE_VALID_WIDTH] && + `FO(vif.outputs, RESPONSE_READY) + ) + pending_word = 0; + if ( + `FO(vif.outputs, MAINTENANCE_VALID) + && vif.inputs[`FETCH_IN_MAINTENANCE_READY_OFFSET+:`FETCH_IN_MAINTENANCE_READY_WIDTH]) + pending_ack = 1; + if (vif.inputs[`FETCH_IN_MAINTAINED_VALID_OFFSET+:`FETCH_IN_MAINTAINED_VALID_WIDTH] && + `FO(vif.outputs, MAINTAINED_READY) + ) + pending_ack = 0; + end + @(negedge vif.clock); + #1; + vif.inputs[`FETCH_IN_PACKET_READY_OFFSET+:`FETCH_IN_PACKET_READY_WIDTH] = 0; + vif.inputs[`FETCH_IN_REQUEST_READY_OFFSET+:`FETCH_IN_REQUEST_READY_WIDTH] = !vif.reset; + vif.inputs[`FETCH_IN_RESPONSE_VALID_OFFSET+:`FETCH_IN_RESPONSE_VALID_WIDTH]=pending_word&&!vif.reset; + vif.inputs[`FETCH_IN_RESPONSE_BITS_DATA_OFFSET+:`FETCH_IN_RESPONSE_BITS_DATA_WIDTH]=fetch_ref_word( + address, image.version); + for (int word = 0; word < 8; word++) + line[word*64+:64] = fetch_ref_word((address & ~64'h3f) + word * 8, image.version); + vif.inputs[`FETCH_IN_RESPONSE_BITS_LINE_OFFSET+:`FETCH_IN_RESPONSE_BITS_LINE_WIDTH] = line; + vif.inputs[`FETCH_IN_RESPONSE_BITS_LINEEXECUTABLE_OFFSET+:`FETCH_IN_RESPONSE_BITS_LINEEXECUTABLE_WIDTH]=0; + vif.inputs[`FETCH_IN_RESPONSE_BITS_PAGEFAULT_OFFSET+:`FETCH_IN_RESPONSE_BITS_PAGEFAULT_WIDTH]=0; + vif.inputs[`FETCH_IN_RESPONSE_BITS_ACCESSFAULT_OFFSET+:`FETCH_IN_RESPONSE_BITS_ACCESSFAULT_WIDTH]=0; + vif.inputs[`FETCH_IN_MAINTENANCE_READY_OFFSET+:`FETCH_IN_MAINTENANCE_READY_WIDTH]=!vif.reset; + vif.inputs[`FETCH_IN_MAINTAINED_VALID_OFFSET+:`FETCH_IN_MAINTAINED_VALID_WIDTH]=pending_ack&&!vif.reset; + vif.inputs[`FETCH_IN_MAINTAINED_BITS_OFFSET+:`FETCH_IN_MAINTAINED_BITS_WIDTH]=scenario!="ack"; + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/env/env.svh b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/env/env.svh new file mode 100644 index 000000000..86c211c1b --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/env/env.svh @@ -0,0 +1,40 @@ +class env extends uvm_env; + `uvm_component_utils(env) + driver control; + monitor mon; + peer_driver peer; + model reference; + program_image image; + reset_controller reset; + uvm_sequencer #(control_item) seqr; + result_board #(result_item, 0) scoreboard; + uvm_analysis_imp #(bit, env) reset_export; + function new(string name, uvm_component parent); + super.new(name, parent); + reset_export = new("reset_export", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + image = new; + control = driver::type_id::create("control", this); + mon = monitor::type_id::create("monitor", this); + peer = peer_driver::type_id::create("peer", this); + reference = model::type_id::create("model", this); + reset = reset_controller::type_id::create("reset", this); + seqr = new("seqr", this); + scoreboard = result_board#(result_item, 0)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + control.seq_item_port.connect(seqr.seq_item_export); + control.image = image; + reference.image = image; + mon.ap.connect(reference.analysis_export); + peer.reference = reference; + reference.expected_ap.connect(scoreboard.expected_export); + reference.actual_ap.connect(scoreboard.actual_export); + reference.reset_ap.connect(reset_export); + endfunction + function void write(bit value); + scoreboard.cancel_pending(); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/env/model.svh b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/env/model.svh new file mode 100644 index 000000000..0cd1259a8 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/env/model.svh @@ -0,0 +1,224 @@ +class model extends uvm_subscriber #(sample_item); + `uvm_component_utils(model) + program_image image; + sample_item latest; + logic [`FETCH_INPUT_BITS-1:0] previous_inputs = '0; + longint unsigned cursor = 0, last_pc = 0; + bit last_pf = 0, last_af = 0, last_valid = 0, offered_packet = 0; + int epoch = 0, cycle = 0, checked = 0, reads = 0, replies = 0, reset_words = 0, drained_words = 0; + int maintenance_requests = 0, maintenance_acks = 0, reset_maintenance = 0, ack_due = 0; + bit needs_maintenance = 0, maintenance_active = 0; + word_transaction offer, word; + bit + packet_ready = 0, + hold_request = 0, + hold_word = 0, + hold_maintenance = 0, + hold_ack = 0, + line_executable = 1; + bit pf_enabled = 0, af_enabled = 0; + longint unsigned pf_address = 0, af_address = 0; + uvm_analysis_port #(result_item) expected_ap, actual_ap; + uvm_analysis_port #(bit) reset_ap; + covergroup behavior with function sample ( + int privilege, + bit packet_stall, + bit word_stall, + bit redirect, + bit fence, + bit invalidate, + int mask, + bit pf, + bit af, + bit executable, + bit packet_valid, + bit response_valid + ); + option.per_instance = 1; + context_privilege: coverpoint privilege { + bins user_mode = {0}; bins supervisor = {1}; bins machine_mode = {3}; + } + cpu_backpressure: coverpoint packet_stall; + word_backpressure: coverpoint word_stall; + redirect_event: coverpoint redirect; + fence_event: coverpoint fence; + invalidate_event: coverpoint invalidate; + page_fault: coverpoint pf iff (packet_valid); + access_fault: coverpoint af iff (packet_valid); + executable_line: coverpoint executable iff (response_valid); + packet_mask: coverpoint mask iff (packet_valid) {bins legal[] = `FETCH_MASK_VALUES;} + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected_ap = new("expected_ap", this); + actual_ap = new("actual_ap", this); + reset_ap = new("reset_ap", this); + behavior = new(); + endfunction + function void write(sample_item t); + result_item expected, actual; + bit cancel, pf, af, request_fire, response_fire, maintenance_fire, ack_fire, packet_fire; + logic [`FETCH_PACKET_BITS-1:0] actual_packet; + longint unsigned expected_npc; + latest = t; + if (t.reset) begin + reset_ap.write(1); + if (word != null) reset_words++; + if (maintenance_active) reset_maintenance++; + offer = null; + word = null; + maintenance_active = 0; + needs_maintenance = 0; + offered_packet = 0; + last_valid = 0; + cursor = `FI(t.inputs, RESETVECTOR); + epoch++; + previous_inputs = t.inputs; + return; + end + cycle++; + expected_npc = `FI(t.inputs, REDIRECT_VALID) ? `FI(t.inputs, REDIRECT_BITS) : cursor; + evidence::check(`FO(t.outputs, NPC) === expected_npc, + "Fetch next-PC output disagrees with accepted CPU packets and redirects"); + // The word descriptor is formed before its physical handshake and remains stable across redirects. + if (`FO(t.outputs, REQUEST_VALID) && offer == null) begin + offer = new; + offer.pc = cursor; + offer.address = cursor & ~64'h7; + offer.epoch = epoch; + evidence::check( + `FO(t.outputs, REQUEST_BITS_ADDR) === offer.address && + `FO(t.outputs, REQUEST_BITS_CONTEXT_PRIVILEGE) === `FI(previous_inputs, CONTEXT_PRIVILEGE) + && `FO(t.outputs, REQUEST_BITS_CONTEXT_SATP) === `FI(previous_inputs, CONTEXT_SATP) && + `FO(t.outputs, REQUEST_BITS_CONTEXT_SUM) === `FI(previous_inputs, CONTEXT_SUM) && + `FO(t.outputs, REQUEST_BITS_CONTEXT_MXR) === `FI(previous_inputs, CONTEXT_MXR) && + `FO(t.outputs, REQUEST_BITS_EXECUTE) === 1'b1, + "Fetch word request lost its PC or captured address-space context"); + end + cancel = `FI(t.inputs, REDIRECT_VALID) || `FI(t.inputs, FLUSH); + if (cancel) begin + reset_ap.write(1); + offered_packet = 0; + last_valid = 0; + epoch++; + if (`FI(t.inputs, REDIRECT_VALID)) cursor = `FI(t.inputs, REDIRECT_BITS); + evidence::check(!`FO(t.outputs, PACKET_VALID), "Fetch exposed a cancelled CPU packet"); + end + if (`FI(t.inputs, FLUSH)) needs_maintenance = 1; + request_fire = `FO(t.outputs, REQUEST_VALID) && `FI(t.inputs, REQUEST_READY); + response_fire = `FI(t.inputs, RESPONSE_VALID) && `FO(t.outputs, RESPONSE_READY); + maintenance_fire = `FO(t.outputs, MAINTENANCE_VALID) && `FI(t.inputs, MAINTENANCE_READY); + ack_fire = `FI(t.inputs, MAINTAINED_VALID) && `FO(t.outputs, MAINTAINED_READY); + packet_fire = `FO(t.outputs, PACKET_VALID) && `FI(t.inputs, PACKET_READY); + if (offer != null && needs_maintenance) + evidence::check(offer.epoch != epoch, "Fetch issued a new word before maintenance completed"); + if (needs_maintenance || maintenance_active) + evidence::check(!`FO(t.outputs, PACKET_VALID), + "Fetch delivered instructions before maintenance completed"); + if (request_fire) begin + evidence::check(offer != null && word == null && !maintenance_active, + "Fetch overlapped backend word transactions"); + word = offer; + offer = null; + word.version = image.version; + word.due = cycle + 2 + reads % 3; + word.pf = pf_enabled && word.address == pf_address; + word.af = af_enabled && word.address == af_address; + word.executable = line_executable && !word.pf && !word.af; + word.data = word.pf || word.af ? 0 : fetch_ref_word(word.address, word.version); + word.line = '0; + if (!word.pf && !word.af) + for (int index = 0; index < 8; index++) + word.line[index*64+:64] = + fetch_ref_word((word.address & ~64'h3f) + index * 8, word.version); + reads++; + end + if (response_fire) begin + if (word == null) `uvm_fatal("WORD_RESPONSE", "Fetch accepted an unsolicited word") + evidence::check(`FI(t.inputs, RESPONSE_BITS_DATA) === word.data && + `FI(t.inputs, RESPONSE_BITS_LINE) === word.line && + `FI(t.inputs, RESPONSE_BITS_PAGEFAULT) === word.pf && + `FI(t.inputs, RESPONSE_BITS_ACCESSFAULT) === word.af && + `FI(t.inputs, RESPONSE_BITS_LINEEXECUTABLE) === word.executable, + "Fetch backend response disagrees with its programmed image"); + if (word.epoch == epoch) begin + last_pc = word.pc; + last_pf = word.pf; + last_af = word.af; + last_valid = 1; + end else drained_words++; + word = null; + replies++; + end + if (maintenance_fire) begin + evidence::check(word == null && offer == null && !maintenance_active && needs_maintenance && + `FO(t.outputs, MAINTENANCE_BITS), + "Fetch maintenance did not drain its outstanding word"); + maintenance_active = 1; + maintenance_requests++; + ack_due = cycle + 4; + end + if (ack_fire) begin + evidence::check(maintenance_active && `FI(t.inputs, MAINTAINED_BITS), + "Fetch consumed an invalid maintenance acknowledgement"); + maintenance_active = 0; + needs_maintenance = 0; + maintenance_acks++; + end + if (`FI(t.inputs, FLUSH)) needs_maintenance = 1; + if (!cancel && `FO(t.outputs, PACKET_VALID)) begin + pf = last_valid && last_pc == cursor ? last_pf : 0; + af = last_valid && last_pc == cursor ? last_af : 0; + expected = result_item::type_id::create("expected"); + expected.payload = '0; + expected.payload[`FETCH_OUT_PACKET_BITS_PC_OFFSET-`FETCH_PACKET_OFFSET+:`FETCH_OUT_PACKET_BITS_PC_WIDTH]=cursor; + expected.payload[`FETCH_OUT_PACKET_BITS_DATA_OFFSET-`FETCH_PACKET_OFFSET+:`FETCH_OUT_PACKET_BITS_DATA_WIDTH]=pf||af?0:fetch_ref_group( + cursor, image.version); + expected.payload[`FETCH_OUT_PACKET_BITS_MASK_OFFSET-`FETCH_PACKET_OFFSET+:`FETCH_OUT_PACKET_BITS_MASK_WIDTH]=`FETCH_INSTRUCTION_BYTES==2?(cursor[1]?2:3):1; + expected.payload[`FETCH_OUT_PACKET_BITS_PAGEFAULT_OFFSET-`FETCH_PACKET_OFFSET+:`FETCH_OUT_PACKET_BITS_PAGEFAULT_WIDTH]=pf; + expected.payload[`FETCH_OUT_PACKET_BITS_ACCESSFAULT_OFFSET-`FETCH_PACKET_OFFSET+:`FETCH_OUT_PACKET_BITS_ACCESSFAULT_WIDTH]=af; + actual_packet = t.outputs[`FETCH_PACKET_OFFSET+:`FETCH_PACKET_BITS]; + if (`FETCH_INSTRUCTION_BYTES == 2 && cursor[1]) begin + expected.payload[`FETCH_OUT_PACKET_BITS_DATA_OFFSET-`FETCH_PACKET_OFFSET+:16] = 0; + actual_packet[`FETCH_OUT_PACKET_BITS_DATA_OFFSET-`FETCH_PACKET_OFFSET+:16] = 0; + end + if (!offered_packet) begin + evidence::check(actual_packet === expected.payload, $sformatf( + "Fetch offered wrong packet pc=%h expected=%h got=%h", + cursor, + expected.payload, + actual_packet + )); + expected_ap.write(expected); + offered_packet = 1; + end + if (packet_fire) begin + actual = result_item::type_id::create("actual"); + actual.payload = actual_packet; + actual_ap.write(actual); + offered_packet = 0; + last_valid = 0; + cursor = (cursor & ~64'h3) + 4; + checked++; + end + end + behavior.sample( + `FI(t.inputs, CONTEXT_PRIVILEGE), `FO(t.outputs, PACKET_VALID) && ! + `FI(t.inputs, PACKET_READY), `FO(t.outputs, REQUEST_VALID) && !`FI(t.inputs, REQUEST_READY), + `FI(t.inputs, REDIRECT_VALID), `FI(t.inputs, FLUSH), `FI(t.inputs, INVALIDATE), + `FO(t.outputs, PACKET_BITS_MASK), `FO(t.outputs, PACKET_BITS_PAGEFAULT), + `FO(t.outputs, PACKET_BITS_ACCESSFAULT), `FI(t.inputs, RESPONSE_BITS_LINEEXECUTABLE), + `FO(t.outputs, PACKET_VALID), response_fire); + previous_inputs = t.inputs; + endfunction + function void check_phase(uvm_phase phase); + super.check_phase(phase); + evidence::check(word == null && offer == null && !maintenance_active && !needs_maintenance, + "Fetch backend did not drain or reset"); + evidence::check( + reads==replies+reset_words&&maintenance_requests==maintenance_acks+reset_maintenance, + "Fetch backend transaction accounting mismatch"); + evidence::check(drained_words != 0 && maintenance_acks != 0, + "Fetch did not exercise cancelled-word drain and maintenance acknowledgement"); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/filelist.f b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/filelist.f new file mode 100644 index 000000000..f68ec10bc --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/Fetch/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/interface.sv b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/interface.sv new file mode 100644 index 000000000..87dddda2d --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/interface.sv @@ -0,0 +1,20 @@ +`include "parameters.svh" +interface fetch_if ( + input logic clock +); + logic reset; + logic [`FETCH_INPUT_BITS-1:0] inputs = '0; + logic [`FETCH_OUTPUT_BITS-1:0] outputs; + assert property (@(posedge clock) disable iff (reset) + outputs[`FETCH_OUT_REQUEST_VALID_OFFSET] && !inputs[`FETCH_IN_REQUEST_READY_OFFSET] |=> + outputs[`FETCH_OUT_REQUEST_VALID_OFFSET] && $stable( + outputs[`FETCH_REQUEST_OFFSET+:`FETCH_REQUEST_BITS] + )) + else $fatal(1, "Fetch word request changed under backpressure"); + assert property (@(posedge clock) disable iff (reset || inputs[`FETCH_IN_REDIRECT_VALID_OFFSET] || inputs[`FETCH_IN_FLUSH_OFFSET]) + outputs[`FETCH_OUT_PACKET_VALID_OFFSET] && !inputs[`FETCH_IN_PACKET_READY_OFFSET] |=> + outputs[`FETCH_OUT_PACKET_VALID_OFFSET] && $stable( + outputs[`FETCH_PACKET_OFFSET+:`FETCH_PACKET_BITS] + )) + else $fatal(1, "Fetch packet changed under backpressure"); +endinterface diff --git a/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/package.sv b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/package.sv new file mode 100644 index 000000000..692ec5f77 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/package.sv @@ -0,0 +1,26 @@ +`include "parameters.svh" +`define FI(P, N) P[`FETCH_IN_``N``_OFFSET+:`FETCH_IN_``N``_WIDTH] +`define FO(P, N) P[`FETCH_OUT_``N``_OFFSET+:`FETCH_OUT_``N``_WIDTH] +`define PK(P, + N) P[`FETCH_OUT_PACKET_BITS_``N``_OFFSET-`FETCH_PACKET_OFFSET+:`FETCH_OUT_PACKET_BITS_``N``_WIDTH] +package fetch_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + import "DPI-C" function longint unsigned fetch_ref_word( + input longint unsigned address, + input int unsigned version + ); + import "DPI-C" function int unsigned fetch_ref_group( + input longint unsigned address, + input int unsigned version + ); + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "env/model.svh" + `include "agent/peer.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..9213cad3a --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/seq/sequence.svh @@ -0,0 +1,235 @@ +class scenario_sequence extends uvm_sequence; + `uvm_object_utils(scenario_sequence) + env test_env; + virtual fetch_if vif; + function new(string name = "scenario_sequence"); + super.new(name); + endfunction + task operate(control_kind kind, longint unsigned target = 0, int version = 0, int privilege = 3, + longint unsigned satp = 0, bit sum = 0, bit mxr = 0); + control_sequence seq = control_sequence::type_id::create("control"); + seq.item = control_item::type_id::create("item"); + seq.item.kind = kind; + seq.item.target = target; + seq.item.version = version; + seq.item.privilege = privilege; + seq.item.satp = satp; + seq.item.sum = sum; + seq.item.mxr = mxr; + seq.start(test_env.seqr); + endtask + task packet_at(longint unsigned target); + do @(negedge vif.clock); while (!`FO(test_env.reference.latest.outputs, PACKET_VALID)); + evidence::check(`FO(test_env.reference.latest.outputs, PACKET_BITS_PC) === target, + "Fetch packet appeared at the wrong redirected PC"); + endtask + task consume(int count); + int target = test_env.reference.checked + count; + test_env.reference.packet_ready = 1; + while (test_env.reference.checked < target) @(negedge vif.clock); + test_env.reference.packet_ready = 0; + @(negedge vif.clock); + endtask + task body(); + int baseline, target; + longint unsigned high; + operate(RESET_VECTOR, 'h1000); + test_env.reset.pulse(4); + packet_at('h1000); + consume(64); + // Fill both line buffers, then alternate held packets without a backend read. + operate(FENCE, 'h1000); + packet_at('h1000); + operate(REDIRECT, 'h1040); + packet_at('h1040); + baseline = test_env.reference.reads; + for (int n = 0; n < 8; n++) begin + operate(REDIRECT, n % 2 ? 'h1040 : 'h1000); + packet_at(n % 2 ? 'h1040 : 'h1000); + evidence::check(test_env.reference.reads == baseline, + "Fetch did not retain two executable lines"); + end + operate(REDIRECT, 'h1080); + packet_at('h1080); + baseline = test_env.reference.reads; + operate(REDIRECT, 'h1040); + packet_at('h1040); + evidence::check(test_env.reference.reads == baseline, + "Fetch replaced its most recently used line"); + operate(REDIRECT, 'h1000); + packet_at('h1000); + evidence::check(test_env.reference.reads == baseline + 1, + "Fetch did not replace its least recently used line"); + // Every individual context field participates in the line-buffer key. + for (int field = 0; field < 4; field++) begin + operate(CONTEXT, 'h1000, 0, 3, 0, 0, 0); + operate(FENCE, 'h1000); + packet_at('h1000); + baseline = test_env.reference.reads; + operate(CONTEXT, 'h1000, 0, field == 0 ? 1 : 3, field == 1 ? 64'h8000000000001234 : 0, + field == 2, field == 3); + packet_at('h1000); + evidence::check(test_env.reference.reads == baseline + 1, + "Fetch ignored an address-space context field"); + operate(CONTEXT, 'h1000, 0, 3, 0, 0, 0); + packet_at('h1000); + evidence::check(test_env.reference.reads == baseline + 1, + "Fetch lost the retained prior context"); + end + baseline = test_env.reference.reads; + operate(INVALIDATE); + operate(REDIRECT, 'h1000); + packet_at('h1000); + evidence::check(test_env.reference.reads == baseline + 1, + "Fetch did not invalidate its buffered line"); + // Redirects drain stalled offers and accepted words instead of cancelling the backend port. + test_env.reference.hold_request = 1; + operate(REDIRECT, 'h2000); + wait (`FO(test_env.reference.latest.outputs, REQUEST_VALID)); + operate(REDIRECT, 'h3000); + repeat (4) @(negedge vif.clock); + test_env.reference.hold_request = 0; + packet_at('h3000); + test_env.reference.hold_word = 1; + operate(REDIRECT, 'h4000); + wait (test_env.reference.word != null); + operate(REDIRECT, 'h5000); + repeat (4) @(negedge vif.clock); + test_env.reference.hold_word = 0; + packet_at('h5000); + // An image replacement and fence occur atomically; the older response must not refill either buffer. + test_env.reference.hold_word = 1; + operate(REDIRECT, 'h6000); + wait (test_env.reference.word != null); + test_env.reference.hold_maintenance = 1; + test_env.reference.hold_ack = 1; + operate(FENCE, 'h7000, 1); + repeat (4) @(negedge vif.clock); + evidence::check(!`FO(test_env.reference.latest.outputs, MAINTENANCE_VALID), + "Fetch maintenance overtook the outstanding word"); + test_env.reference.hold_word = 0; + wait (`FO(test_env.reference.latest.outputs, MAINTENANCE_VALID)); + repeat (4) @(negedge vif.clock); + test_env.reference.hold_maintenance = 0; + wait (test_env.reference.maintenance_active); + repeat (4) @(negedge vif.clock); + test_env.reference.hold_ack = 0; + packet_at('h7000); + operate(REDIRECT, 'h6000); + packet_at('h6000); + consume(32); + // Fault packets carry no instruction data and are never buffered. + test_env.reference.line_executable = 0; + test_env.reference.pf_enabled = 1; + test_env.reference.af_enabled = 1; + test_env.reference.pf_address = 'ha000; + test_env.reference.af_address = 'ha008; + operate(FENCE, 'ha000, 1); + packet_at('ha000); + consume(6); + test_env.reference.pf_enabled = 0; + test_env.reference.af_enabled = 0; + if (`FETCH_INSTRUCTION_BYTES == 2) begin + operate(FENCE, 'hb002, 1); + packet_at('hb002); + consume(12); + end + // Exercise every virtual-address bit without interpreting instructions in this IP. + test_env.reference.line_executable = 1; + for (int bit_index = 2; bit_index < 64; bit_index++) begin + high = 64'(1) << bit_index; + operate(FENCE, high, bit_index + 2); + packet_at(high); + consume(1); + end + // SATP mode, PPN and ASID bits are independently part of the buffered context. + for (int bit_index = 0; bit_index < 60; bit_index++) begin + operate(CONTEXT, 'h1000, 0, 1, 64'h8000000000000000); + packet_at('h1000); + baseline = test_env.reference.reads; + operate(CONTEXT, 'h1000, 0, 1, 64'h8000000000000000 | (64'(1) << bit_index)); + packet_at('h1000); + evidence::check(test_env.reference.reads == baseline + 1, "Fetch ignored a SATP context bit"); + operate(CONTEXT, 'h1000, 0, 1, 64'h8000000000000000); + packet_at('h1000); + evidence::check(test_env.reference.reads == baseline + 1, + "Fetch did not retain its baseline SATP context"); + end + operate(CONTEXT, 'h1000, 0, 0, 0); + packet_at('h1000); + consume(2); + // Reset at offer, receive and held CPU-packet boundaries. + for (int phase_index = 0; phase_index < 3; phase_index++) begin + test_env.reference.hold_request = phase_index == 0; + test_env.reference.hold_word = phase_index == 1; + operate(REDIRECT, 'hc000 + phase_index * 'h1000); + if (phase_index == 0) wait (`FO(test_env.reference.latest.outputs, REQUEST_VALID)); + else if (phase_index == 1) wait (test_env.reference.word != null); + else packet_at('he000); + test_env.reset.pulse(3); + test_env.reference.hold_request = 0; + test_env.reference.hold_word = 0; + packet_at('h1000); + consume(2); + end + test_env.reference.hold_ack = 1; + operate(FENCE, 'hf000, test_env.image.version); + wait (test_env.reference.maintenance_active); + test_env.reset.pulse(3); + test_env.reference.hold_ack = 0; + packet_at('h1000); + consume(2); + test_env.reference.hold_maintenance = 1; + operate(FENCE, 'hf000, test_env.image.version); + wait (`FO(test_env.reference.latest.outputs, MAINTENANCE_VALID)); + test_env.reset.pulse(3); + test_env.reference.hold_maintenance = 0; + packet_at('h1000); + consume(2); + test_env.reference.packet_ready = 0; + test_env.reset.pulse(3); + `uvm_info("FETCH", + $sformatf( + "packets=%0d backend_reads=%0d replies=%0d redirected_drains=%0d maintenance=%0d", + test_env.reference.checked, test_env.reference.reads, test_env.reference.replies, + test_env.reference.drained_words, test_env.reference.maintenance_acks), UVM_LOW) + endtask +endclass +class invalid_sequence extends uvm_sequence; + `uvm_object_utils(invalid_sequence) + reset_controller reset; + uvm_sequencer #(control_item) seqr; + function new(string name = "invalid_sequence"); + super.new(name); + endfunction + task command(control_kind kind, longint unsigned target = 0, int privilege = 3); + control_sequence seq = control_sequence::type_id::create("command"); + seq.item = control_item::type_id::create("item"); + seq.item.kind = kind; + seq.item.target = target; + seq.item.privilege = privilege; + seq.item.version = 0; + seq.item.satp = 0; + seq.item.sum = 0; + seq.item.mxr = 0; + seq.start(seqr); + endtask + task body(); + string scenario; + if (!$value$plusargs("CASE=%s", scenario)) + `uvm_fatal("CASE", "Fetch assertion scenario is required") + if (scenario == "reset") command(RESET_VECTOR, 1); + reset.pulse(4); + case (scenario) + "reset": begin + end + "redirect": command(REDIRECT, 1); + "flush": command(BAD_FLUSH); + "privilege": command(CONTEXT, 'h1000, 2); + "ack": command(FENCE, 0); + default: `uvm_fatal("CASE", "Unknown fetch assertion scenario") + endcase + #1000ns; + `uvm_fatal("EXPECTED_ASSERTION", "Invalid fetch input did not trigger its assertion") + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/suite.toml b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/suite.toml new file mode 100644 index 000000000..87ca37d58 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/suite.toml @@ -0,0 +1,105 @@ +[model] +manifest = "src/csrc/Cargo.toml" +library = "libfetch_reference_model" + +[[tests]] +functional_groups = ["fetch_pkg::model::behavior"] +name = "fetch" +top = "fetch_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 200000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +functional_groups = [] +name = "fetch_bad_ack" +top = "fetch_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 200000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=ack"] +assertion = "Fetch maintenance response must acknowledge completion" + +[[tests]] +functional_groups = [] +name = "fetch_bad_flush" +top = "fetch_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 200000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=flush"] +assertion = "Fetch flush requires a restart redirect" + +[[tests]] +functional_groups = [] +name = "fetch_bad_privilege" +top = "fetch_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 200000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=privilege"] +assertion = "Fetch privilege must be U, S or M" + +[[tests]] +functional_groups = [] +name = "fetch_bad_redirect" +top = "fetch_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 200000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=redirect"] +assertion = "Fetch redirect must be instruction aligned" + +[[tests]] +functional_groups = [] +name = "fetch_bad_reset" +top = "fetch_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 200000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=reset"] +assertion = "Fetch reset vector must be instruction aligned" diff --git a/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/tb.sv b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/tb.sv new file mode 100644 index 000000000..432235910 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/tb.sv @@ -0,0 +1,19 @@ +module fetch_tb; + import uvm_pkg::*; + import fetch_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + clock_if timing (clock); + fetch_if bus (clock); + assign bus.reset = timing.reset; + Fetch dut ( + .clock(clock), + .reset(bus.reset), + `include "ports.svh" + ); + initial begin + uvm_config_db#(virtual fetch_if)::set(null, "uvm_test_top*", "vif", bus); + uvm_config_db#(virtual clock_if)::set(null, "uvm_test_top*", "timing", timing); + run_test(); + end +endmodule diff --git a/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/tests/test.svh new file mode 100644 index 000000000..bfc358fee --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/tests/test.svh @@ -0,0 +1,50 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + virtual fetch_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + if (!uvm_config_db#(virtual fetch_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Fetch timing interface is required") + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.test_env = test_env; + seq.vif = vif; + seq.start(null); + endtask +endclass +class invalid_test extends base_test; + `uvm_component_utils(invalid_test) + driver control; + invalid_peer peer; + program_image image; + reset_controller reset; + uvm_sequencer #(control_item) seqr; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + image = new; + control = driver::type_id::create("control", this); + peer = invalid_peer::type_id::create("peer", this); + reset = reset_controller::type_id::create("reset", this); + seqr = new("seqr", this); + endfunction + function void connect_phase(uvm_phase phase); + control.seq_item_port.connect(seqr.seq_item_export); + control.image = image; + peer.image = image; + endfunction + task execute(); + invalid_sequence seq = invalid_sequence::type_id::create("violation"); + seq.reset = reset; + seq.seqr = seqr; + seq.start(null); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/fetch/src/main/scala/Emit.scala b/arch/src/main/scala/framework/core/execution/fetch/src/main/scala/Emit.scala new file mode 100644 index 000000000..da795ae24 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fetch/src/main/scala/Emit.scala @@ -0,0 +1,69 @@ +package framework.core.execution.fetch + +import chisel3._ +import java.nio.file.{Files, Paths} +import framework.core.execution.fetch.configs.Params + +object Emit extends App { + val Array("--config", config, "--output-dir", output) = args + val p = upickle.default.read[Params](Files.readString(Paths.get(config))) + Files.createDirectories(Paths.get(output)) + val ports = scala.collection.mutable.ArrayBuffer[String]() + val definitions = scala.collection.mutable.ArrayBuffer[String]() + val offsets = scala.collection.mutable.Map[String, Int]() + val widths = scala.collection.mutable.Map[String, Int]() + var inputOffset = 0 + var outputOffset = 0 + + def fields(data: Data, path: String): Seq[(String, Int)] = data match { + case record: Record => record.elements.toSeq.flatMap { case (name, field) => + fields(field, if (path.isEmpty) name else s"${path}_$name") + } + case vector: Vec[_] => vector.toSeq.zipWithIndex.flatMap { case (field, index) => fields(field, s"${path}_$index") } + case leaf => Seq(path -> leaf.getWidth) + } + + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + { + val dut = new Fetch(p) + for ((name, width) <- fields(dut.io, "")) { + val input = Set( + "resetVector", + "flush", + "invalidate", + "redirect_valid", + "redirect_bits", + "packet_ready", + "request_ready", + "response_valid", + "maintenance_ready", + "maintained_valid", + "maintained_bits" + ).contains(name) || name.startsWith("context_") || name.startsWith("response_bits_") + val direction = if (input) "IN" else "OUT" + val signal = if (input) "inputs" else "outputs" + val offset = if (input) inputOffset else outputOffset + definitions += s"`define FETCH_${direction}_${name.toUpperCase}_OFFSET $offset" + definitions += s"`define FETCH_${direction}_${name.toUpperCase}_WIDTH $width" + ports += s".io_$name(bus.$signal[$offset +: $width])" + offsets(name) = offset; widths(name) = width + if (input) inputOffset += width else outputOffset += width + } + dut + }, + args = Array("--target-dir", s"$output/Fetch", "--split-verilog") + ) + definitions += s"`define FETCH_INPUT_BITS $inputOffset" + definitions += s"`define FETCH_OUTPUT_BITS $outputOffset" + definitions += s"`define FETCH_INSTRUCTION_BYTES ${p.instructionBytes}" + definitions += s"`define FETCH_MASK_VALUES ${if (p.instructionBytes == 2) "{2,3}" else "{1}"}" + for ( + (name, prefix) <- Seq("REQUEST" -> "request_bits_", "RESPONSE" -> "response_bits_", "PACKET" -> "packet_bits_") + ) { + val names = offsets.keys.filter(_.startsWith(prefix)).toSeq + definitions += s"`define FETCH_${name}_OFFSET ${names.map(offsets).min}" + definitions += s"`define FETCH_${name}_BITS ${names.map(widths).sum}" + } + Files.writeString(Paths.get(output, "parameters.svh"), definitions.mkString("\n") + "\n") + Files.writeString(Paths.get(output, "ports.svh"), ports.mkString(",\n") + "\n") +} diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/main/scala/Fetch.scala b/arch/src/main/scala/framework/core/execution/fetch/src/main/scala/Fetch.scala similarity index 93% rename from arch/src/main/scala/framework/mem-core/fetch/src/main/scala/Fetch.scala rename to arch/src/main/scala/framework/core/execution/fetch/src/main/scala/Fetch.scala index bcabb3243..d8b85f370 100644 --- a/arch/src/main/scala/framework/mem-core/fetch/src/main/scala/Fetch.scala +++ b/arch/src/main/scala/framework/core/execution/fetch/src/main/scala/Fetch.scala @@ -1,13 +1,10 @@ -package memcore.memory.fetch +package framework.core.execution.fetch import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public} -case class Params(instructionBytes: Int = 2) { - require(instructionBytes == 2 || instructionBytes == 4) - val lanes: Int = 4 / instructionBytes -} +import framework.core.execution.fetch.configs.{Params => ModuleParam} class Context extends Bundle { val privilege = UInt(2.W) @@ -31,7 +28,9 @@ class Response extends Bundle { val accessFault = Bool() } -class Packet(p: Params) extends Bundle { +class Packet(b: ModuleParam) extends Bundle { + val p = b + val pc = UInt(64.W) val data = UInt(32.W) val mask = UInt(p.lanes.W) @@ -45,8 +44,10 @@ class Packet(p: Params) extends Bundle { * a request, which replaces the less recently used line. Lines are keyed by virtual line and fetch * context, and are dropped by fence.i (`flush`) and sfence.vma (`invalidate`). */ + @instantiable -class Fetch(p: Params) extends Module { +class Fetch(b: ModuleParam) extends Module { + val p = b @public val io = IO(new Bundle { @@ -55,7 +56,7 @@ class Fetch(p: Params) extends Module { val redirect = Flipped(Valid(UInt(64.W))) val flush = Input(Bool()) val invalidate = Input(Bool()) - val packet = Decoupled(new Packet(p)) + val packet = Decoupled(new Packet(b)) val request = Decoupled(new Request) val response = Flipped(Decoupled(new Response)) val maintenance = Decoupled(Bool()) @@ -68,7 +69,7 @@ class Fetch(p: Params) extends Module { val pc = Reg(UInt(64.W)) val readPc = Reg(UInt(64.W)) val command = Reg(new Request) - val answer = Reg(new Packet(p)) + val answer = Reg(new Packet(b)) val discard = RegInit(false.B) val pendingMaintenance = RegInit(false.B) val cancelled = io.redirect.valid || io.flush @@ -89,7 +90,7 @@ class Fetch(p: Params) extends Module { def bufferedLine(addr: UInt): UInt = Mux(hits(addr)(1), bufferData(1), bufferData(0)) def packetAt(addr: UInt, data: UInt): Packet = { - val packet = Wire(new Packet(p)) + val packet = Wire(new Packet(b)) packet.pc := addr packet.data := (data >> (addr(5, 2) << 5))(31, 0) packet.mask := (if (p.instructionBytes == 2) (3.U(2.W) << addr(1))(1, 0) else 1.U) @@ -143,7 +144,7 @@ class Fetch(p: Params) extends Module { state := idle }.otherwise { answer := packetAt(readPc, io.response.bits.line) - answer.data := Mux(fault, 0.U, (io.response.bits.line >> (readPc(5, 2) << 5))(31, 0)) + answer.data := Mux(fault, 0.U, (io.response.bits.data >> (readPc(2) << 5))(31, 0)) answer.pageFault := io.response.bits.pageFault answer.accessFault := io.response.bits.accessFault state := deliver diff --git a/arch/src/main/scala/framework/core/execution/fetch/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/core/execution/fetch/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..2c00df7b1 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fetch/src/main/scala/configs/Params.scala @@ -0,0 +1,11 @@ +package framework.core.execution.fetch.configs + +import chisel3.util._ +import upickle.default.{macroRW, ReadWriter} + +case class Params(instructionBytes: Int) { + require(instructionBytes == 2 || instructionBytes == 4) + val lanes: Int = 4 / instructionBytes +} + +object Params { implicit val rw: ReadWriter[Params] = macroRW } diff --git a/arch/src/main/scala/framework/core/execution/fetch/src/main/scala/configs/default.toml b/arch/src/main/scala/framework/core/execution/fetch/src/main/scala/configs/default.toml new file mode 100644 index 000000000..37359c823 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fetch/src/main/scala/configs/default.toml @@ -0,0 +1 @@ +instructionBytes = 2 diff --git a/arch/src/main/scala/framework/core/execution/fpu/build.sc b/arch/src/main/scala/framework/core/execution/fpu/build.sc new file mode 100644 index 000000000..4bb2d1ef2 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/build.sc @@ -0,0 +1,21 @@ +import mill._ +import $file.^.^.^.^.^.^.^.settings + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.core.execution.fpu.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/FPU.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/core/execution/fpu/configs/uvm.toml b/arch/src/main/scala/framework/core/execution/fpu/configs/uvm.toml new file mode 100644 index 000000000..422e3e9f3 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/configs/uvm.toml @@ -0,0 +1,5 @@ +[parameters] +xLen = 64 +fLen = 64 + +[defines] diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/csrc/Cargo.lock b/arch/src/main/scala/framework/core/execution/fpu/src/csrc/Cargo.lock new file mode 100644 index 000000000..90aa159e8 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/csrc/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "fpu-reference" +version = "0.1.0" diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/csrc/Cargo.toml b/arch/src/main/scala/framework/core/execution/fpu/src/csrc/Cargo.toml new file mode 100644 index 000000000..20557768f --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/csrc/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "fpu-reference" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["cdylib"] diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/csrc/src/lib.rs b/arch/src/main/scala/framework/core/execution/fpu/src/csrc/src/lib.rs new file mode 100644 index 000000000..b78e3f544 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/csrc/src/lib.rs @@ -0,0 +1,58 @@ +mod vectors; +type Input = (u32, u64, u64, u64, u64, u32); +type Output = (u64, u32, bool, bool); + +#[no_mangle] +pub extern "C" fn fpu_case_count() -> u32 { + vectors::CASES.len() as u32 +} + +#[no_mangle] +pub extern "C" fn fpu_cancel_case() -> u32 { + vectors::CASES + .iter() + .position(|(input, _)| input.0 & 0x7f == 0x53 && input.0 >> 27 == 3) + .unwrap() as u32 +} + +#[no_mangle] +pub unsafe extern "C" fn fpu_case( + index: u32, + instruction: *mut u32, + integer: *mut u64, + a: *mut u64, + rhs: *mut u64, + c: *mut u64, + rounding: *mut u32, +) { + let input = vectors::CASES[index as usize].0; + *instruction = input.0; + *integer = input.1; + *a = input.2; + *rhs = input.3; + *c = input.4; + *rounding = input.5; +} + +#[no_mangle] +pub unsafe extern "C" fn fpu_expected( + instruction: u32, + integer: u64, + a: u64, + rhs: u64, + c: u64, + rounding: u32, + data: *mut u64, + flags: *mut u32, + is_integer: *mut u8, + illegal: *mut u8, +) { + let observed = (instruction, integer, a, rhs, c, rounding); + let Some((_, expected)) = vectors::CASES.iter().find(|(input, _)| *input == observed) else { + panic!("No reference for observed FPU request: {observed:?}"); + }; + *data = expected.0; + *flags = expected.1; + *is_integer = u8::from(expected.2); + *illegal = u8::from(expected.3); +} diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/csrc/src/vectors.rs b/arch/src/main/scala/framework/core/execution/fpu/src/csrc/src/vectors.rs new file mode 100644 index 000000000..73dc800be --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/csrc/src/vectors.rs @@ -0,0 +1,3485 @@ +use super::{Input, Output}; + +// instruction, integer, a, rhs, c, rounding -> data, flags, integer, illegal +pub const CASES: &[(Input, Output)] = &[ + // S exact arithmetic + ( + ( + 0x00000053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff40000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff40400000, 0, false, false), + ), + // S exact arithmetic + ( + ( + 0x08000053, + 0x0000000000000000, + 0xffffffff40400000, + 0xffffffff40000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff3f800000, 0, false, false), + ), + // S exact arithmetic + ( + ( + 0x10000053, + 0x0000000000000000, + 0xffffffff3fc00000, + 0xffffffff40000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff40400000, 0, false, false), + ), + // S exact arithmetic + ( + ( + 0x18000053, + 0x0000000000000000, + 0xffffffff40c00000, + 0xffffffff40400000, + 0xffffffff00000000, + 0, + ), + (0xffffffff40000000, 0, false, false), + ), + // S sqrt 4 + ( + ( + 0x58000053, + 0x0000000000000000, + 0xffffffff40800000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff40000000, 0, false, false), + ), + // S sqrt negative + ( + ( + 0x58000053, + 0x0000000000000000, + 0xffffffffbf800000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff7fc00000, 16, false, false), + ), + // S divide by zero + ( + ( + 0x18000053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff7f800000, 8, false, false), + ), + // S zero divided by zero + ( + ( + 0x18000053, + 0x0000000000000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff7fc00000, 16, false, false), + ), + // S infinity divided by infinity + ( + ( + 0x18000053, + 0x0000000000000000, + 0xffffffff7f800000, + 0xffffffff7f800000, + 0xffffffff00000000, + 0, + ), + (0xffffffff7fc00000, 16, false, false), + ), + // S opposite infinities + ( + ( + 0x00000053, + 0x0000000000000000, + 0xffffffff7f800000, + 0xffffffffff800000, + 0xffffffff00000000, + 0, + ), + (0xffffffff7fc00000, 16, false, false), + ), + // S quiet NaN arithmetic + ( + ( + 0x00000053, + 0x0000000000000000, + 0xffffffff7fc00011, + 0xffffffff3f800000, + 0xffffffff00000000, + 0, + ), + (0xffffffff7fc00000, 0, false, false), + ), + // S signaling NaN arithmetic + ( + ( + 0x00000053, + 0x0000000000000000, + 0xffffffff7f800001, + 0xffffffff3f800000, + 0xffffffff00000000, + 0, + ), + (0xffffffff7fc00000, 16, false, false), + ), + // S fused sign variants + ( + ( + 0x00000043, + 0x0000000000000000, + 0xffffffff40000000, + 0xffffffff40400000, + 0xffffffff40800000, + 0, + ), + (0xffffffff41200000, 0, false, false), + ), + // S fused sign variants + ( + ( + 0x00000047, + 0x0000000000000000, + 0xffffffff40000000, + 0xffffffff40400000, + 0xffffffff40800000, + 0, + ), + (0xffffffff40000000, 0, false, false), + ), + // S fused sign variants + ( + ( + 0x0000004b, + 0x0000000000000000, + 0xffffffff40000000, + 0xffffffff40400000, + 0xffffffff40800000, + 0, + ), + (0xffffffffc0000000, 0, false, false), + ), + // S fused sign variants + ( + ( + 0x0000004f, + 0x0000000000000000, + 0xffffffff40000000, + 0xffffffff40400000, + 0xffffffff40800000, + 0, + ), + (0xffffffffc1200000, 0, false, false), + ), + // S fused cancellation retains product precision + ( + ( + 0x00000043, + 0x0000000000000000, + 0xffffffff3f800001, + 0xffffffff3f7ffffe, + 0xffffffffbf800000, + 0, + ), + (0xffffffffa8800000, 0, false, false), + ), + // S positive half-ulp rounding + ( + ( + 0x00000053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff33800000, + 0xffffffff00000000, + 0, + ), + (0xffffffff3f800000, 1, false, false), + ), + // S negative half-ulp rounding + ( + ( + 0x00000053, + 0x0000000000000000, + 0xffffffffbf800000, + 0xffffffffb3800000, + 0xffffffff00000000, + 0, + ), + (0xffffffffbf800000, 1, false, false), + ), + // S positive half-ulp rounding + ( + ( + 0x00007053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff33800000, + 0xffffffff00000000, + 0, + ), + (0xffffffff3f800000, 1, false, false), + ), + // S negative half-ulp rounding + ( + ( + 0x00007053, + 0x0000000000000000, + 0xffffffffbf800000, + 0xffffffffb3800000, + 0xffffffff00000000, + 0, + ), + (0xffffffffbf800000, 1, false, false), + ), + // S positive overflow + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffff7f7fffff, + 0xffffffff40000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff7f800000, 5, false, false), + ), + // S negative overflow + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffffff7fffff, + 0xffffffff40000000, + 0xffffffff00000000, + 0, + ), + (0xffffffffff800000, 5, false, false), + ), + // S positive tiny tie + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffff00000001, + 0xffffffff3f000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff00000000, 3, false, false), + ), + // S negative tiny tie + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffff80000001, + 0xffffffff3f000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff80000000, 3, false, false), + ), + // S cancellation zero sign + ( + ( + 0x00007053, + 0x0000000000000000, + 0xffffffff00000000, + 0xffffffff80000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff00000000, 0, false, false), + ), + // S float to signed integer tie + ( + ( + 0xc0207053, + 0x0000000000000000, + 0xffffffff3fc00000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000002, 1, true, false), + ), + // S negative integer tie + ( + ( + 0xc0207053, + 0x0000000000000000, + 0xffffffffbfc00000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xfffffffffffffffe, 1, true, false), + ), + // S positive half-ulp rounding + ( + ( + 0x00001053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff33800000, + 0xffffffff00000000, + 0, + ), + (0xffffffff3f800000, 1, false, false), + ), + // S negative half-ulp rounding + ( + ( + 0x00001053, + 0x0000000000000000, + 0xffffffffbf800000, + 0xffffffffb3800000, + 0xffffffff00000000, + 0, + ), + (0xffffffffbf800000, 1, false, false), + ), + // S positive half-ulp rounding + ( + ( + 0x00007053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff33800000, + 0xffffffff00000000, + 1, + ), + (0xffffffff3f800000, 1, false, false), + ), + // S negative half-ulp rounding + ( + ( + 0x00007053, + 0x0000000000000000, + 0xffffffffbf800000, + 0xffffffffb3800000, + 0xffffffff00000000, + 1, + ), + (0xffffffffbf800000, 1, false, false), + ), + // S positive overflow + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffff7f7fffff, + 0xffffffff40000000, + 0xffffffff00000000, + 1, + ), + (0xffffffff7f7fffff, 5, false, false), + ), + // S negative overflow + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffffff7fffff, + 0xffffffff40000000, + 0xffffffff00000000, + 1, + ), + (0xffffffffff7fffff, 5, false, false), + ), + // S positive tiny tie + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffff00000001, + 0xffffffff3f000000, + 0xffffffff00000000, + 1, + ), + (0xffffffff00000000, 3, false, false), + ), + // S negative tiny tie + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffff80000001, + 0xffffffff3f000000, + 0xffffffff00000000, + 1, + ), + (0xffffffff80000000, 3, false, false), + ), + // S cancellation zero sign + ( + ( + 0x00007053, + 0x0000000000000000, + 0xffffffff00000000, + 0xffffffff80000000, + 0xffffffff00000000, + 1, + ), + (0xffffffff00000000, 0, false, false), + ), + // S float to signed integer tie + ( + ( + 0xc0207053, + 0x0000000000000000, + 0xffffffff3fc00000, + 0xffffffff00000000, + 0xffffffff00000000, + 1, + ), + (0x0000000000000001, 1, true, false), + ), + // S negative integer tie + ( + ( + 0xc0207053, + 0x0000000000000000, + 0xffffffffbfc00000, + 0xffffffff00000000, + 0xffffffff00000000, + 1, + ), + (0xffffffffffffffff, 1, true, false), + ), + // S positive half-ulp rounding + ( + ( + 0x00002053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff33800000, + 0xffffffff00000000, + 0, + ), + (0xffffffff3f800000, 1, false, false), + ), + // S negative half-ulp rounding + ( + ( + 0x00002053, + 0x0000000000000000, + 0xffffffffbf800000, + 0xffffffffb3800000, + 0xffffffff00000000, + 0, + ), + (0xffffffffbf800001, 1, false, false), + ), + // S positive half-ulp rounding + ( + ( + 0x00007053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff33800000, + 0xffffffff00000000, + 2, + ), + (0xffffffff3f800000, 1, false, false), + ), + // S negative half-ulp rounding + ( + ( + 0x00007053, + 0x0000000000000000, + 0xffffffffbf800000, + 0xffffffffb3800000, + 0xffffffff00000000, + 2, + ), + (0xffffffffbf800001, 1, false, false), + ), + // S positive overflow + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffff7f7fffff, + 0xffffffff40000000, + 0xffffffff00000000, + 2, + ), + (0xffffffff7f7fffff, 5, false, false), + ), + // S negative overflow + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffffff7fffff, + 0xffffffff40000000, + 0xffffffff00000000, + 2, + ), + (0xffffffffff800000, 5, false, false), + ), + // S positive tiny tie + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffff00000001, + 0xffffffff3f000000, + 0xffffffff00000000, + 2, + ), + (0xffffffff00000000, 3, false, false), + ), + // S negative tiny tie + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffff80000001, + 0xffffffff3f000000, + 0xffffffff00000000, + 2, + ), + (0xffffffff80000001, 3, false, false), + ), + // S cancellation zero sign + ( + ( + 0x00007053, + 0x0000000000000000, + 0xffffffff00000000, + 0xffffffff80000000, + 0xffffffff00000000, + 2, + ), + (0xffffffff80000000, 0, false, false), + ), + // S float to signed integer tie + ( + ( + 0xc0207053, + 0x0000000000000000, + 0xffffffff3fc00000, + 0xffffffff00000000, + 0xffffffff00000000, + 2, + ), + (0x0000000000000001, 1, true, false), + ), + // S negative integer tie + ( + ( + 0xc0207053, + 0x0000000000000000, + 0xffffffffbfc00000, + 0xffffffff00000000, + 0xffffffff00000000, + 2, + ), + (0xfffffffffffffffe, 1, true, false), + ), + // S positive half-ulp rounding + ( + ( + 0x00003053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff33800000, + 0xffffffff00000000, + 0, + ), + (0xffffffff3f800001, 1, false, false), + ), + // S negative half-ulp rounding + ( + ( + 0x00003053, + 0x0000000000000000, + 0xffffffffbf800000, + 0xffffffffb3800000, + 0xffffffff00000000, + 0, + ), + (0xffffffffbf800000, 1, false, false), + ), + // S positive half-ulp rounding + ( + ( + 0x00007053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff33800000, + 0xffffffff00000000, + 3, + ), + (0xffffffff3f800001, 1, false, false), + ), + // S negative half-ulp rounding + ( + ( + 0x00007053, + 0x0000000000000000, + 0xffffffffbf800000, + 0xffffffffb3800000, + 0xffffffff00000000, + 3, + ), + (0xffffffffbf800000, 1, false, false), + ), + // S positive overflow + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffff7f7fffff, + 0xffffffff40000000, + 0xffffffff00000000, + 3, + ), + (0xffffffff7f800000, 5, false, false), + ), + // S negative overflow + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffffff7fffff, + 0xffffffff40000000, + 0xffffffff00000000, + 3, + ), + (0xffffffffff7fffff, 5, false, false), + ), + // S positive tiny tie + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffff00000001, + 0xffffffff3f000000, + 0xffffffff00000000, + 3, + ), + (0xffffffff00000001, 3, false, false), + ), + // S negative tiny tie + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffff80000001, + 0xffffffff3f000000, + 0xffffffff00000000, + 3, + ), + (0xffffffff80000000, 3, false, false), + ), + // S cancellation zero sign + ( + ( + 0x00007053, + 0x0000000000000000, + 0xffffffff00000000, + 0xffffffff80000000, + 0xffffffff00000000, + 3, + ), + (0xffffffff00000000, 0, false, false), + ), + // S float to signed integer tie + ( + ( + 0xc0207053, + 0x0000000000000000, + 0xffffffff3fc00000, + 0xffffffff00000000, + 0xffffffff00000000, + 3, + ), + (0x0000000000000002, 1, true, false), + ), + // S negative integer tie + ( + ( + 0xc0207053, + 0x0000000000000000, + 0xffffffffbfc00000, + 0xffffffff00000000, + 0xffffffff00000000, + 3, + ), + (0xffffffffffffffff, 1, true, false), + ), + // S positive half-ulp rounding + ( + ( + 0x00004053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff33800000, + 0xffffffff00000000, + 0, + ), + (0xffffffff3f800001, 1, false, false), + ), + // S negative half-ulp rounding + ( + ( + 0x00004053, + 0x0000000000000000, + 0xffffffffbf800000, + 0xffffffffb3800000, + 0xffffffff00000000, + 0, + ), + (0xffffffffbf800001, 1, false, false), + ), + // S positive half-ulp rounding + ( + ( + 0x00007053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff33800000, + 0xffffffff00000000, + 4, + ), + (0xffffffff3f800001, 1, false, false), + ), + // S negative half-ulp rounding + ( + ( + 0x00007053, + 0x0000000000000000, + 0xffffffffbf800000, + 0xffffffffb3800000, + 0xffffffff00000000, + 4, + ), + (0xffffffffbf800001, 1, false, false), + ), + // S positive overflow + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffff7f7fffff, + 0xffffffff40000000, + 0xffffffff00000000, + 4, + ), + (0xffffffff7f800000, 5, false, false), + ), + // S negative overflow + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffffff7fffff, + 0xffffffff40000000, + 0xffffffff00000000, + 4, + ), + (0xffffffffff800000, 5, false, false), + ), + // S positive tiny tie + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffff00000001, + 0xffffffff3f000000, + 0xffffffff00000000, + 4, + ), + (0xffffffff00000001, 3, false, false), + ), + // S negative tiny tie + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffff80000001, + 0xffffffff3f000000, + 0xffffffff00000000, + 4, + ), + (0xffffffff80000001, 3, false, false), + ), + // S cancellation zero sign + ( + ( + 0x00007053, + 0x0000000000000000, + 0xffffffff00000000, + 0xffffffff80000000, + 0xffffffff00000000, + 4, + ), + (0xffffffff00000000, 0, false, false), + ), + // S float to signed integer tie + ( + ( + 0xc0207053, + 0x0000000000000000, + 0xffffffff3fc00000, + 0xffffffff00000000, + 0xffffffff00000000, + 4, + ), + (0x0000000000000002, 1, true, false), + ), + // S negative integer tie + ( + ( + 0xc0207053, + 0x0000000000000000, + 0xffffffffbfc00000, + 0xffffffff00000000, + 0xffffffff00000000, + 4, + ), + (0xfffffffffffffffe, 1, true, false), + ), + // S reserved instruction rounding + ( + ( + 0x00005053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff40000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000000, 0, false, true), + ), + // S reserved instruction rounding + ( + ( + 0x00006053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff40000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000000, 0, false, true), + ), + // S reserved dynamic rounding + ( + ( + 0x00007053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff40000000, + 0xffffffff00000000, + 5, + ), + (0x0000000000000000, 0, false, true), + ), + // S reserved dynamic rounding + ( + ( + 0x00007053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff40000000, + 0xffffffff00000000, + 6, + ), + (0x0000000000000000, 0, false, true), + ), + // S reserved dynamic rounding + ( + ( + 0x00007053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff40000000, + 0xffffffff00000000, + 7, + ), + (0x0000000000000000, 0, false, true), + ), + // S sign injection + ( + ( + 0x20000053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffffc0000000, + 0xffffffff00000000, + 0, + ), + (0xffffffffbf800000, 0, false, false), + ), + // S sign injection + ( + ( + 0x20001053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffffc0000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff3f800000, 0, false, false), + ), + // S sign injection + ( + ( + 0x20002053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffffc0000000, + 0xffffffff00000000, + 0, + ), + (0xffffffffbf800000, 0, false, false), + ), + // S sign injection preserves payload + ( + ( + 0x20000053, + 0x0000000000000000, + 0xffffffff7fc00011, + 0xffffffffc0000000, + 0xffffffff00000000, + 0, + ), + (0xffffffffffc00011, 0, false, false), + ), + // S min/max signed zero + ( + ( + 0x28000053, + 0x0000000000000000, + 0xffffffff80000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff80000000, 0, false, false), + ), + // S min/max finite + ( + ( + 0x28000053, + 0x0000000000000000, + 0xffffffffc0000000, + 0xffffffff3f800000, + 0xffffffff00000000, + 0, + ), + (0xffffffffc0000000, 0, false, false), + ), + // S min/max quiet NaN + ( + ( + 0x28000053, + 0x0000000000000000, + 0xffffffff7fc00000, + 0xffffffff3f800000, + 0xffffffff00000000, + 0, + ), + (0xffffffff3f800000, 0, false, false), + ), + // S min/max signaling NaN + ( + ( + 0x28000053, + 0x0000000000000000, + 0xffffffff7f800001, + 0xffffffff3f800000, + 0xffffffff00000000, + 0, + ), + (0xffffffff3f800000, 16, false, false), + ), + // S min/max two NaNs + ( + ( + 0x28000053, + 0x0000000000000000, + 0xffffffff7fc00003, + 0xffffffff7fc00007, + 0xffffffff00000000, + 0, + ), + (0xffffffff7fc00000, 0, false, false), + ), + // S min/max signed zero + ( + ( + 0x28001053, + 0x0000000000000000, + 0xffffffff80000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff00000000, 0, false, false), + ), + // S min/max finite + ( + ( + 0x28001053, + 0x0000000000000000, + 0xffffffffc0000000, + 0xffffffff3f800000, + 0xffffffff00000000, + 0, + ), + (0xffffffff3f800000, 0, false, false), + ), + // S min/max quiet NaN + ( + ( + 0x28001053, + 0x0000000000000000, + 0xffffffff7fc00000, + 0xffffffff3f800000, + 0xffffffff00000000, + 0, + ), + (0xffffffff3f800000, 0, false, false), + ), + // S min/max signaling NaN + ( + ( + 0x28001053, + 0x0000000000000000, + 0xffffffff7f800001, + 0xffffffff3f800000, + 0xffffffff00000000, + 0, + ), + (0xffffffff3f800000, 16, false, false), + ), + // S min/max two NaNs + ( + ( + 0x28001053, + 0x0000000000000000, + 0xffffffff7fc00003, + 0xffffffff7fc00007, + 0xffffffff00000000, + 0, + ), + (0xffffffff7fc00000, 0, false, false), + ), + // S comparison ordered + ( + ( + 0xa0000053, + 0x0000000000000000, + 0xffffffffbf800000, + 0xffffffff3f800000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000001, 0, true, false), + ), + // S comparison signed zero + ( + ( + 0xa0000053, + 0x0000000000000000, + 0xffffffff80000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000001, 0, true, false), + ), + // S comparison quiet NaN + ( + ( + 0xa0000053, + 0x0000000000000000, + 0xffffffff7fc00000, + 0xffffffff3f800000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000000, 16, true, false), + ), + // S comparison signaling NaN + ( + ( + 0xa0000053, + 0x0000000000000000, + 0xffffffff7f800001, + 0xffffffff3f800000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000000, 16, true, false), + ), + // S comparison ordered + ( + ( + 0xa0001053, + 0x0000000000000000, + 0xffffffffbf800000, + 0xffffffff3f800000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000001, 0, true, false), + ), + // S comparison signed zero + ( + ( + 0xa0001053, + 0x0000000000000000, + 0xffffffff80000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000000, 0, true, false), + ), + // S comparison quiet NaN + ( + ( + 0xa0001053, + 0x0000000000000000, + 0xffffffff7fc00000, + 0xffffffff3f800000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000000, 16, true, false), + ), + // S comparison signaling NaN + ( + ( + 0xa0001053, + 0x0000000000000000, + 0xffffffff7f800001, + 0xffffffff3f800000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000000, 16, true, false), + ), + // S comparison ordered + ( + ( + 0xa0002053, + 0x0000000000000000, + 0xffffffffbf800000, + 0xffffffff3f800000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000000, 0, true, false), + ), + // S comparison signed zero + ( + ( + 0xa0002053, + 0x0000000000000000, + 0xffffffff80000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000001, 0, true, false), + ), + // S comparison quiet NaN + ( + ( + 0xa0002053, + 0x0000000000000000, + 0xffffffff7fc00000, + 0xffffffff3f800000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000000, 0, true, false), + ), + // S comparison signaling NaN + ( + ( + 0xa0002053, + 0x0000000000000000, + 0xffffffff7f800001, + 0xffffffff3f800000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000000, 16, true, false), + ), + // S classification 0 + ( + ( + 0xe0001053, + 0x0000000000000000, + 0xffffffffff800000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000001, 0, true, false), + ), + // S classification 1 + ( + ( + 0xe0001053, + 0x0000000000000000, + 0xffffffffbf800000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000002, 0, true, false), + ), + // S classification 2 + ( + ( + 0xe0001053, + 0x0000000000000000, + 0xffffffff80000001, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000004, 0, true, false), + ), + // S classification 3 + ( + ( + 0xe0001053, + 0x0000000000000000, + 0xffffffff80000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000008, 0, true, false), + ), + // S classification 4 + ( + ( + 0xe0001053, + 0x0000000000000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000010, 0, true, false), + ), + // S classification 5 + ( + ( + 0xe0001053, + 0x0000000000000000, + 0xffffffff00000001, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000020, 0, true, false), + ), + // S classification 6 + ( + ( + 0xe0001053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000040, 0, true, false), + ), + // S classification 7 + ( + ( + 0xe0001053, + 0x0000000000000000, + 0xffffffff7f800000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000080, 0, true, false), + ), + // S classification 8 + ( + ( + 0xe0001053, + 0x0000000000000000, + 0xffffffff7f800001, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000100, 0, true, false), + ), + // S classification 9 + ( + ( + 0xe0001053, + 0x0000000000000000, + 0xffffffff7fc00000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000200, 0, true, false), + ), + // S move float bits to integer + ( + ( + 0xe0000053, + 0x0000000000000000, + 0x0000000080000001, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff80000001, 0, true, false), + ), + // S move integer bits to float + ( + ( + 0xf0000053, + 0x0000000080000001, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff80000001, 0, false, false), + ), + // S integer conversion type 0 + ( + ( + 0xd0000053, + 0x0000000000000003, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff40400000, 0, false, false), + ), + // S float conversion type 0 + ( + ( + 0xc0000053, + 0x0000000000000000, + 0xffffffff40400000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000003, 0, true, false), + ), + // S invalid positive conversion + ( + ( + 0xc0000053, + 0x0000000000000000, + 0xffffffff7f800000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x000000007fffffff, 16, true, false), + ), + // S invalid positive conversion + ( + ( + 0xc0000053, + 0x0000000000000000, + 0xffffffff7fc00000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x000000007fffffff, 16, true, false), + ), + // S invalid negative conversion + ( + ( + 0xc0000053, + 0x0000000000000000, + 0xffffffffff800000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff80000000, 16, true, false), + ), + // S integer conversion boundary + ( + ( + 0xd0000053, + 0xffffffffffffffff, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffffbf800000, 0, false, false), + ), + // S integer conversion type 1 + ( + ( + 0xd0100053, + 0x0000000000000003, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff40400000, 0, false, false), + ), + // S float conversion type 1 + ( + ( + 0xc0100053, + 0x0000000000000000, + 0xffffffff40400000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000003, 0, true, false), + ), + // S invalid positive conversion + ( + ( + 0xc0100053, + 0x0000000000000000, + 0xffffffff7f800000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffffffffffff, 16, true, false), + ), + // S invalid positive conversion + ( + ( + 0xc0100053, + 0x0000000000000000, + 0xffffffff7fc00000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffffffffffff, 16, true, false), + ), + // S invalid negative conversion + ( + ( + 0xc0100053, + 0x0000000000000000, + 0xffffffffff800000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000000, 16, true, false), + ), + // S integer conversion boundary + ( + ( + 0xd0100053, + 0xffffffffffffffff, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff4f800000, 1, false, false), + ), + // S integer conversion type 2 + ( + ( + 0xd0200053, + 0x0000000000000003, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff40400000, 0, false, false), + ), + // S float conversion type 2 + ( + ( + 0xc0200053, + 0x0000000000000000, + 0xffffffff40400000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000003, 0, true, false), + ), + // S invalid positive conversion + ( + ( + 0xc0200053, + 0x0000000000000000, + 0xffffffff7f800000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x7fffffffffffffff, 16, true, false), + ), + // S invalid positive conversion + ( + ( + 0xc0200053, + 0x0000000000000000, + 0xffffffff7fc00000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x7fffffffffffffff, 16, true, false), + ), + // S invalid negative conversion + ( + ( + 0xc0200053, + 0x0000000000000000, + 0xffffffffff800000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x8000000000000000, 16, true, false), + ), + // S integer conversion boundary + ( + ( + 0xd0200053, + 0xffffffffffffffff, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffffbf800000, 0, false, false), + ), + // S integer conversion type 3 + ( + ( + 0xd0300053, + 0x0000000000000003, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff40400000, 0, false, false), + ), + // S float conversion type 3 + ( + ( + 0xc0300053, + 0x0000000000000000, + 0xffffffff40400000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000003, 0, true, false), + ), + // S invalid positive conversion + ( + ( + 0xc0300053, + 0x0000000000000000, + 0xffffffff7f800000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffffffffffff, 16, true, false), + ), + // S invalid positive conversion + ( + ( + 0xc0300053, + 0x0000000000000000, + 0xffffffff7fc00000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffffffffffff, 16, true, false), + ), + // S invalid negative conversion + ( + ( + 0xc0300053, + 0x0000000000000000, + 0xffffffffff800000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000000, 16, true, false), + ), + // S integer conversion boundary + ( + ( + 0xd0300053, + 0xffffffffffffffff, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff5f800000, 1, false, false), + ), + // S cross-format exact + ( + ( + 0x40100053, + 0x0000000000000000, + 0x3ff8000000000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff3fc00000, 0, false, false), + ), + // S unboxed single is quiet NaN + ( + ( + 0x00000053, + 0x0000000000000000, + 0x000000003f800000, + 0xffffffff3f800000, + 0xffffffff00000000, + 0, + ), + (0xffffffff7fc00000, 0, false, false), + ), + // S unboxed single classification + ( + ( + 0xe0001053, + 0x0000000000000000, + 0x000000003f800000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000200, 0, true, false), + ), + // S unsupported format + ( + ( + 0x04000053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff40000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000000, 0, false, true), + ), + // S unsupported format + ( + ( + 0x06000053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff40000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000000, 0, false, true), + ), + // S unsupported operation + ( + ( + 0xf8000053, + 0x0000000000000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000000, 0, false, true), + ), + // S unsupported opcode + ( + ( + 0x00000000, + 0x0000000000000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000000, 0, false, true), + ), + // D exact arithmetic + ( + ( + 0x02000053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x4000000000000000, + 0x0000000000000000, + 0, + ), + (0x4008000000000000, 0, false, false), + ), + // D exact arithmetic + ( + ( + 0x0a000053, + 0x0000000000000000, + 0x4008000000000000, + 0x4000000000000000, + 0x0000000000000000, + 0, + ), + (0x3ff0000000000000, 0, false, false), + ), + // D exact arithmetic + ( + ( + 0x12000053, + 0x0000000000000000, + 0x3ff8000000000000, + 0x4000000000000000, + 0x0000000000000000, + 0, + ), + (0x4008000000000000, 0, false, false), + ), + // D exact arithmetic + ( + ( + 0x1a000053, + 0x0000000000000000, + 0x4018000000000000, + 0x4008000000000000, + 0x0000000000000000, + 0, + ), + (0x4000000000000000, 0, false, false), + ), + // D sqrt 4 + ( + ( + 0x5a000053, + 0x0000000000000000, + 0x4010000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x4000000000000000, 0, false, false), + ), + // D sqrt negative + ( + ( + 0x5a000053, + 0x0000000000000000, + 0xbff0000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x7ff8000000000000, 16, false, false), + ), + // D divide by zero + ( + ( + 0x1a000053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x7ff0000000000000, 8, false, false), + ), + // D zero divided by zero + ( + ( + 0x1a000053, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x7ff8000000000000, 16, false, false), + ), + // D infinity divided by infinity + ( + ( + 0x1a000053, + 0x0000000000000000, + 0x7ff0000000000000, + 0x7ff0000000000000, + 0x0000000000000000, + 0, + ), + (0x7ff8000000000000, 16, false, false), + ), + // D opposite infinities + ( + ( + 0x02000053, + 0x0000000000000000, + 0x7ff0000000000000, + 0xfff0000000000000, + 0x0000000000000000, + 0, + ), + (0x7ff8000000000000, 16, false, false), + ), + // D quiet NaN arithmetic + ( + ( + 0x02000053, + 0x0000000000000000, + 0x7ff8000000000011, + 0x3ff0000000000000, + 0x0000000000000000, + 0, + ), + (0x7ff8000000000000, 0, false, false), + ), + // D signaling NaN arithmetic + ( + ( + 0x02000053, + 0x0000000000000000, + 0x7ff0000000000001, + 0x3ff0000000000000, + 0x0000000000000000, + 0, + ), + (0x7ff8000000000000, 16, false, false), + ), + // D fused sign variants + ( + ( + 0x02000043, + 0x0000000000000000, + 0x4000000000000000, + 0x4008000000000000, + 0x4010000000000000, + 0, + ), + (0x4024000000000000, 0, false, false), + ), + // D fused sign variants + ( + ( + 0x02000047, + 0x0000000000000000, + 0x4000000000000000, + 0x4008000000000000, + 0x4010000000000000, + 0, + ), + (0x4000000000000000, 0, false, false), + ), + // D fused sign variants + ( + ( + 0x0200004b, + 0x0000000000000000, + 0x4000000000000000, + 0x4008000000000000, + 0x4010000000000000, + 0, + ), + (0xc000000000000000, 0, false, false), + ), + // D fused sign variants + ( + ( + 0x0200004f, + 0x0000000000000000, + 0x4000000000000000, + 0x4008000000000000, + 0x4010000000000000, + 0, + ), + (0xc024000000000000, 0, false, false), + ), + // D fused cancellation retains product precision + ( + ( + 0x02000043, + 0x0000000000000000, + 0x3ff0000000000001, + 0x3feffffffffffffe, + 0xbff0000000000000, + 0, + ), + (0xb970000000000000, 0, false, false), + ), + // D positive half-ulp rounding + ( + ( + 0x02000053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x3ca0000000000000, + 0x0000000000000000, + 0, + ), + (0x3ff0000000000000, 1, false, false), + ), + // D negative half-ulp rounding + ( + ( + 0x02000053, + 0x0000000000000000, + 0xbff0000000000000, + 0xbca0000000000000, + 0x0000000000000000, + 0, + ), + (0xbff0000000000000, 1, false, false), + ), + // D positive half-ulp rounding + ( + ( + 0x02007053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x3ca0000000000000, + 0x0000000000000000, + 0, + ), + (0x3ff0000000000000, 1, false, false), + ), + // D negative half-ulp rounding + ( + ( + 0x02007053, + 0x0000000000000000, + 0xbff0000000000000, + 0xbca0000000000000, + 0x0000000000000000, + 0, + ), + (0xbff0000000000000, 1, false, false), + ), + // D positive overflow + ( + ( + 0x12007053, + 0x0000000000000000, + 0x7fefffffffffffff, + 0x4000000000000000, + 0x0000000000000000, + 0, + ), + (0x7ff0000000000000, 5, false, false), + ), + // D negative overflow + ( + ( + 0x12007053, + 0x0000000000000000, + 0xffefffffffffffff, + 0x4000000000000000, + 0x0000000000000000, + 0, + ), + (0xfff0000000000000, 5, false, false), + ), + // D positive tiny tie + ( + ( + 0x12007053, + 0x0000000000000000, + 0x0000000000000001, + 0x3fe0000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000000, 3, false, false), + ), + // D negative tiny tie + ( + ( + 0x12007053, + 0x0000000000000000, + 0x8000000000000001, + 0x3fe0000000000000, + 0x0000000000000000, + 0, + ), + (0x8000000000000000, 3, false, false), + ), + // D cancellation zero sign + ( + ( + 0x02007053, + 0x0000000000000000, + 0x0000000000000000, + 0x8000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000000, 0, false, false), + ), + // D float to signed integer tie + ( + ( + 0xc2207053, + 0x0000000000000000, + 0x3ff8000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000002, 1, true, false), + ), + // D negative integer tie + ( + ( + 0xc2207053, + 0x0000000000000000, + 0xbff8000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0xfffffffffffffffe, 1, true, false), + ), + // D positive half-ulp rounding + ( + ( + 0x02001053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x3ca0000000000000, + 0x0000000000000000, + 0, + ), + (0x3ff0000000000000, 1, false, false), + ), + // D negative half-ulp rounding + ( + ( + 0x02001053, + 0x0000000000000000, + 0xbff0000000000000, + 0xbca0000000000000, + 0x0000000000000000, + 0, + ), + (0xbff0000000000000, 1, false, false), + ), + // D positive half-ulp rounding + ( + ( + 0x02007053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x3ca0000000000000, + 0x0000000000000000, + 1, + ), + (0x3ff0000000000000, 1, false, false), + ), + // D negative half-ulp rounding + ( + ( + 0x02007053, + 0x0000000000000000, + 0xbff0000000000000, + 0xbca0000000000000, + 0x0000000000000000, + 1, + ), + (0xbff0000000000000, 1, false, false), + ), + // D positive overflow + ( + ( + 0x12007053, + 0x0000000000000000, + 0x7fefffffffffffff, + 0x4000000000000000, + 0x0000000000000000, + 1, + ), + (0x7fefffffffffffff, 5, false, false), + ), + // D negative overflow + ( + ( + 0x12007053, + 0x0000000000000000, + 0xffefffffffffffff, + 0x4000000000000000, + 0x0000000000000000, + 1, + ), + (0xffefffffffffffff, 5, false, false), + ), + // D positive tiny tie + ( + ( + 0x12007053, + 0x0000000000000000, + 0x0000000000000001, + 0x3fe0000000000000, + 0x0000000000000000, + 1, + ), + (0x0000000000000000, 3, false, false), + ), + // D negative tiny tie + ( + ( + 0x12007053, + 0x0000000000000000, + 0x8000000000000001, + 0x3fe0000000000000, + 0x0000000000000000, + 1, + ), + (0x8000000000000000, 3, false, false), + ), + // D cancellation zero sign + ( + ( + 0x02007053, + 0x0000000000000000, + 0x0000000000000000, + 0x8000000000000000, + 0x0000000000000000, + 1, + ), + (0x0000000000000000, 0, false, false), + ), + // D float to signed integer tie + ( + ( + 0xc2207053, + 0x0000000000000000, + 0x3ff8000000000000, + 0x0000000000000000, + 0x0000000000000000, + 1, + ), + (0x0000000000000001, 1, true, false), + ), + // D negative integer tie + ( + ( + 0xc2207053, + 0x0000000000000000, + 0xbff8000000000000, + 0x0000000000000000, + 0x0000000000000000, + 1, + ), + (0xffffffffffffffff, 1, true, false), + ), + // D positive half-ulp rounding + ( + ( + 0x02002053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x3ca0000000000000, + 0x0000000000000000, + 0, + ), + (0x3ff0000000000000, 1, false, false), + ), + // D negative half-ulp rounding + ( + ( + 0x02002053, + 0x0000000000000000, + 0xbff0000000000000, + 0xbca0000000000000, + 0x0000000000000000, + 0, + ), + (0xbff0000000000001, 1, false, false), + ), + // D positive half-ulp rounding + ( + ( + 0x02007053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x3ca0000000000000, + 0x0000000000000000, + 2, + ), + (0x3ff0000000000000, 1, false, false), + ), + // D negative half-ulp rounding + ( + ( + 0x02007053, + 0x0000000000000000, + 0xbff0000000000000, + 0xbca0000000000000, + 0x0000000000000000, + 2, + ), + (0xbff0000000000001, 1, false, false), + ), + // D positive overflow + ( + ( + 0x12007053, + 0x0000000000000000, + 0x7fefffffffffffff, + 0x4000000000000000, + 0x0000000000000000, + 2, + ), + (0x7fefffffffffffff, 5, false, false), + ), + // D negative overflow + ( + ( + 0x12007053, + 0x0000000000000000, + 0xffefffffffffffff, + 0x4000000000000000, + 0x0000000000000000, + 2, + ), + (0xfff0000000000000, 5, false, false), + ), + // D positive tiny tie + ( + ( + 0x12007053, + 0x0000000000000000, + 0x0000000000000001, + 0x3fe0000000000000, + 0x0000000000000000, + 2, + ), + (0x0000000000000000, 3, false, false), + ), + // D negative tiny tie + ( + ( + 0x12007053, + 0x0000000000000000, + 0x8000000000000001, + 0x3fe0000000000000, + 0x0000000000000000, + 2, + ), + (0x8000000000000001, 3, false, false), + ), + // D cancellation zero sign + ( + ( + 0x02007053, + 0x0000000000000000, + 0x0000000000000000, + 0x8000000000000000, + 0x0000000000000000, + 2, + ), + (0x8000000000000000, 0, false, false), + ), + // D float to signed integer tie + ( + ( + 0xc2207053, + 0x0000000000000000, + 0x3ff8000000000000, + 0x0000000000000000, + 0x0000000000000000, + 2, + ), + (0x0000000000000001, 1, true, false), + ), + // D negative integer tie + ( + ( + 0xc2207053, + 0x0000000000000000, + 0xbff8000000000000, + 0x0000000000000000, + 0x0000000000000000, + 2, + ), + (0xfffffffffffffffe, 1, true, false), + ), + // D positive half-ulp rounding + ( + ( + 0x02003053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x3ca0000000000000, + 0x0000000000000000, + 0, + ), + (0x3ff0000000000001, 1, false, false), + ), + // D negative half-ulp rounding + ( + ( + 0x02003053, + 0x0000000000000000, + 0xbff0000000000000, + 0xbca0000000000000, + 0x0000000000000000, + 0, + ), + (0xbff0000000000000, 1, false, false), + ), + // D positive half-ulp rounding + ( + ( + 0x02007053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x3ca0000000000000, + 0x0000000000000000, + 3, + ), + (0x3ff0000000000001, 1, false, false), + ), + // D negative half-ulp rounding + ( + ( + 0x02007053, + 0x0000000000000000, + 0xbff0000000000000, + 0xbca0000000000000, + 0x0000000000000000, + 3, + ), + (0xbff0000000000000, 1, false, false), + ), + // D positive overflow + ( + ( + 0x12007053, + 0x0000000000000000, + 0x7fefffffffffffff, + 0x4000000000000000, + 0x0000000000000000, + 3, + ), + (0x7ff0000000000000, 5, false, false), + ), + // D negative overflow + ( + ( + 0x12007053, + 0x0000000000000000, + 0xffefffffffffffff, + 0x4000000000000000, + 0x0000000000000000, + 3, + ), + (0xffefffffffffffff, 5, false, false), + ), + // D positive tiny tie + ( + ( + 0x12007053, + 0x0000000000000000, + 0x0000000000000001, + 0x3fe0000000000000, + 0x0000000000000000, + 3, + ), + (0x0000000000000001, 3, false, false), + ), + // D negative tiny tie + ( + ( + 0x12007053, + 0x0000000000000000, + 0x8000000000000001, + 0x3fe0000000000000, + 0x0000000000000000, + 3, + ), + (0x8000000000000000, 3, false, false), + ), + // D cancellation zero sign + ( + ( + 0x02007053, + 0x0000000000000000, + 0x0000000000000000, + 0x8000000000000000, + 0x0000000000000000, + 3, + ), + (0x0000000000000000, 0, false, false), + ), + // D float to signed integer tie + ( + ( + 0xc2207053, + 0x0000000000000000, + 0x3ff8000000000000, + 0x0000000000000000, + 0x0000000000000000, + 3, + ), + (0x0000000000000002, 1, true, false), + ), + // D negative integer tie + ( + ( + 0xc2207053, + 0x0000000000000000, + 0xbff8000000000000, + 0x0000000000000000, + 0x0000000000000000, + 3, + ), + (0xffffffffffffffff, 1, true, false), + ), + // D positive half-ulp rounding + ( + ( + 0x02004053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x3ca0000000000000, + 0x0000000000000000, + 0, + ), + (0x3ff0000000000001, 1, false, false), + ), + // D negative half-ulp rounding + ( + ( + 0x02004053, + 0x0000000000000000, + 0xbff0000000000000, + 0xbca0000000000000, + 0x0000000000000000, + 0, + ), + (0xbff0000000000001, 1, false, false), + ), + // D positive half-ulp rounding + ( + ( + 0x02007053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x3ca0000000000000, + 0x0000000000000000, + 4, + ), + (0x3ff0000000000001, 1, false, false), + ), + // D negative half-ulp rounding + ( + ( + 0x02007053, + 0x0000000000000000, + 0xbff0000000000000, + 0xbca0000000000000, + 0x0000000000000000, + 4, + ), + (0xbff0000000000001, 1, false, false), + ), + // D positive overflow + ( + ( + 0x12007053, + 0x0000000000000000, + 0x7fefffffffffffff, + 0x4000000000000000, + 0x0000000000000000, + 4, + ), + (0x7ff0000000000000, 5, false, false), + ), + // D negative overflow + ( + ( + 0x12007053, + 0x0000000000000000, + 0xffefffffffffffff, + 0x4000000000000000, + 0x0000000000000000, + 4, + ), + (0xfff0000000000000, 5, false, false), + ), + // D positive tiny tie + ( + ( + 0x12007053, + 0x0000000000000000, + 0x0000000000000001, + 0x3fe0000000000000, + 0x0000000000000000, + 4, + ), + (0x0000000000000001, 3, false, false), + ), + // D negative tiny tie + ( + ( + 0x12007053, + 0x0000000000000000, + 0x8000000000000001, + 0x3fe0000000000000, + 0x0000000000000000, + 4, + ), + (0x8000000000000001, 3, false, false), + ), + // D cancellation zero sign + ( + ( + 0x02007053, + 0x0000000000000000, + 0x0000000000000000, + 0x8000000000000000, + 0x0000000000000000, + 4, + ), + (0x0000000000000000, 0, false, false), + ), + // D float to signed integer tie + ( + ( + 0xc2207053, + 0x0000000000000000, + 0x3ff8000000000000, + 0x0000000000000000, + 0x0000000000000000, + 4, + ), + (0x0000000000000002, 1, true, false), + ), + // D negative integer tie + ( + ( + 0xc2207053, + 0x0000000000000000, + 0xbff8000000000000, + 0x0000000000000000, + 0x0000000000000000, + 4, + ), + (0xfffffffffffffffe, 1, true, false), + ), + // D reserved instruction rounding + ( + ( + 0x02005053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x4000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000000, 0, false, true), + ), + // D reserved instruction rounding + ( + ( + 0x02006053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x4000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000000, 0, false, true), + ), + // D reserved dynamic rounding + ( + ( + 0x02007053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x4000000000000000, + 0x0000000000000000, + 5, + ), + (0x0000000000000000, 0, false, true), + ), + // D reserved dynamic rounding + ( + ( + 0x02007053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x4000000000000000, + 0x0000000000000000, + 6, + ), + (0x0000000000000000, 0, false, true), + ), + // D reserved dynamic rounding + ( + ( + 0x02007053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x4000000000000000, + 0x0000000000000000, + 7, + ), + (0x0000000000000000, 0, false, true), + ), + // D sign injection + ( + ( + 0x22000053, + 0x0000000000000000, + 0x3ff0000000000000, + 0xc000000000000000, + 0x0000000000000000, + 0, + ), + (0xbff0000000000000, 0, false, false), + ), + // D sign injection + ( + ( + 0x22001053, + 0x0000000000000000, + 0x3ff0000000000000, + 0xc000000000000000, + 0x0000000000000000, + 0, + ), + (0x3ff0000000000000, 0, false, false), + ), + // D sign injection + ( + ( + 0x22002053, + 0x0000000000000000, + 0x3ff0000000000000, + 0xc000000000000000, + 0x0000000000000000, + 0, + ), + (0xbff0000000000000, 0, false, false), + ), + // D sign injection preserves payload + ( + ( + 0x22000053, + 0x0000000000000000, + 0x7ff8000000000011, + 0xc000000000000000, + 0x0000000000000000, + 0, + ), + (0xfff8000000000011, 0, false, false), + ), + // D min/max signed zero + ( + ( + 0x2a000053, + 0x0000000000000000, + 0x8000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x8000000000000000, 0, false, false), + ), + // D min/max finite + ( + ( + 0x2a000053, + 0x0000000000000000, + 0xc000000000000000, + 0x3ff0000000000000, + 0x0000000000000000, + 0, + ), + (0xc000000000000000, 0, false, false), + ), + // D min/max quiet NaN + ( + ( + 0x2a000053, + 0x0000000000000000, + 0x7ff8000000000000, + 0x3ff0000000000000, + 0x0000000000000000, + 0, + ), + (0x3ff0000000000000, 0, false, false), + ), + // D min/max signaling NaN + ( + ( + 0x2a000053, + 0x0000000000000000, + 0x7ff0000000000001, + 0x3ff0000000000000, + 0x0000000000000000, + 0, + ), + (0x3ff0000000000000, 16, false, false), + ), + // D min/max two NaNs + ( + ( + 0x2a000053, + 0x0000000000000000, + 0x7ff8000000000003, + 0x7ff8000000000007, + 0x0000000000000000, + 0, + ), + (0x7ff8000000000000, 0, false, false), + ), + // D min/max signed zero + ( + ( + 0x2a001053, + 0x0000000000000000, + 0x8000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000000, 0, false, false), + ), + // D min/max finite + ( + ( + 0x2a001053, + 0x0000000000000000, + 0xc000000000000000, + 0x3ff0000000000000, + 0x0000000000000000, + 0, + ), + (0x3ff0000000000000, 0, false, false), + ), + // D min/max quiet NaN + ( + ( + 0x2a001053, + 0x0000000000000000, + 0x7ff8000000000000, + 0x3ff0000000000000, + 0x0000000000000000, + 0, + ), + (0x3ff0000000000000, 0, false, false), + ), + // D min/max signaling NaN + ( + ( + 0x2a001053, + 0x0000000000000000, + 0x7ff0000000000001, + 0x3ff0000000000000, + 0x0000000000000000, + 0, + ), + (0x3ff0000000000000, 16, false, false), + ), + // D min/max two NaNs + ( + ( + 0x2a001053, + 0x0000000000000000, + 0x7ff8000000000003, + 0x7ff8000000000007, + 0x0000000000000000, + 0, + ), + (0x7ff8000000000000, 0, false, false), + ), + // D comparison ordered + ( + ( + 0xa2000053, + 0x0000000000000000, + 0xbff0000000000000, + 0x3ff0000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000001, 0, true, false), + ), + // D comparison signed zero + ( + ( + 0xa2000053, + 0x0000000000000000, + 0x8000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000001, 0, true, false), + ), + // D comparison quiet NaN + ( + ( + 0xa2000053, + 0x0000000000000000, + 0x7ff8000000000000, + 0x3ff0000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000000, 16, true, false), + ), + // D comparison signaling NaN + ( + ( + 0xa2000053, + 0x0000000000000000, + 0x7ff0000000000001, + 0x3ff0000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000000, 16, true, false), + ), + // D comparison ordered + ( + ( + 0xa2001053, + 0x0000000000000000, + 0xbff0000000000000, + 0x3ff0000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000001, 0, true, false), + ), + // D comparison signed zero + ( + ( + 0xa2001053, + 0x0000000000000000, + 0x8000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000000, 0, true, false), + ), + // D comparison quiet NaN + ( + ( + 0xa2001053, + 0x0000000000000000, + 0x7ff8000000000000, + 0x3ff0000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000000, 16, true, false), + ), + // D comparison signaling NaN + ( + ( + 0xa2001053, + 0x0000000000000000, + 0x7ff0000000000001, + 0x3ff0000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000000, 16, true, false), + ), + // D comparison ordered + ( + ( + 0xa2002053, + 0x0000000000000000, + 0xbff0000000000000, + 0x3ff0000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000000, 0, true, false), + ), + // D comparison signed zero + ( + ( + 0xa2002053, + 0x0000000000000000, + 0x8000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000001, 0, true, false), + ), + // D comparison quiet NaN + ( + ( + 0xa2002053, + 0x0000000000000000, + 0x7ff8000000000000, + 0x3ff0000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000000, 0, true, false), + ), + // D comparison signaling NaN + ( + ( + 0xa2002053, + 0x0000000000000000, + 0x7ff0000000000001, + 0x3ff0000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000000, 16, true, false), + ), + // D classification 0 + ( + ( + 0xe2001053, + 0x0000000000000000, + 0xfff0000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000001, 0, true, false), + ), + // D classification 1 + ( + ( + 0xe2001053, + 0x0000000000000000, + 0xbff0000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000002, 0, true, false), + ), + // D classification 2 + ( + ( + 0xe2001053, + 0x0000000000000000, + 0x8000000000000001, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000004, 0, true, false), + ), + // D classification 3 + ( + ( + 0xe2001053, + 0x0000000000000000, + 0x8000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000008, 0, true, false), + ), + // D classification 4 + ( + ( + 0xe2001053, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000010, 0, true, false), + ), + // D classification 5 + ( + ( + 0xe2001053, + 0x0000000000000000, + 0x0000000000000001, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000020, 0, true, false), + ), + // D classification 6 + ( + ( + 0xe2001053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000040, 0, true, false), + ), + // D classification 7 + ( + ( + 0xe2001053, + 0x0000000000000000, + 0x7ff0000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000080, 0, true, false), + ), + // D classification 8 + ( + ( + 0xe2001053, + 0x0000000000000000, + 0x7ff0000000000001, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000100, 0, true, false), + ), + // D classification 9 + ( + ( + 0xe2001053, + 0x0000000000000000, + 0x7ff8000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000200, 0, true, false), + ), + // D move float bits to integer + ( + ( + 0xe2000053, + 0x0000000000000000, + 0x0123456789abcdef, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0123456789abcdef, 0, true, false), + ), + // D move integer bits to float + ( + ( + 0xf2000053, + 0x0123456789abcdef, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0123456789abcdef, 0, false, false), + ), + // D integer conversion type 0 + ( + ( + 0xd2000053, + 0x0000000000000003, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x4008000000000000, 0, false, false), + ), + // D float conversion type 0 + ( + ( + 0xc2000053, + 0x0000000000000000, + 0x4008000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000003, 0, true, false), + ), + // D invalid positive conversion + ( + ( + 0xc2000053, + 0x0000000000000000, + 0x7ff0000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x000000007fffffff, 16, true, false), + ), + // D invalid positive conversion + ( + ( + 0xc2000053, + 0x0000000000000000, + 0x7ff8000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x000000007fffffff, 16, true, false), + ), + // D invalid negative conversion + ( + ( + 0xc2000053, + 0x0000000000000000, + 0xfff0000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0xffffffff80000000, 16, true, false), + ), + // D integer conversion boundary + ( + ( + 0xd2000053, + 0xffffffffffffffff, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0xbff0000000000000, 0, false, false), + ), + // D integer conversion type 1 + ( + ( + 0xd2100053, + 0x0000000000000003, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x4008000000000000, 0, false, false), + ), + // D float conversion type 1 + ( + ( + 0xc2100053, + 0x0000000000000000, + 0x4008000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000003, 0, true, false), + ), + // D invalid positive conversion + ( + ( + 0xc2100053, + 0x0000000000000000, + 0x7ff0000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0xffffffffffffffff, 16, true, false), + ), + // D invalid positive conversion + ( + ( + 0xc2100053, + 0x0000000000000000, + 0x7ff8000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0xffffffffffffffff, 16, true, false), + ), + // D invalid negative conversion + ( + ( + 0xc2100053, + 0x0000000000000000, + 0xfff0000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000000, 16, true, false), + ), + // D integer conversion boundary + ( + ( + 0xd2100053, + 0xffffffffffffffff, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x41efffffffe00000, 0, false, false), + ), + // D integer conversion type 2 + ( + ( + 0xd2200053, + 0x0000000000000003, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x4008000000000000, 0, false, false), + ), + // D float conversion type 2 + ( + ( + 0xc2200053, + 0x0000000000000000, + 0x4008000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000003, 0, true, false), + ), + // D invalid positive conversion + ( + ( + 0xc2200053, + 0x0000000000000000, + 0x7ff0000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x7fffffffffffffff, 16, true, false), + ), + // D invalid positive conversion + ( + ( + 0xc2200053, + 0x0000000000000000, + 0x7ff8000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x7fffffffffffffff, 16, true, false), + ), + // D invalid negative conversion + ( + ( + 0xc2200053, + 0x0000000000000000, + 0xfff0000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x8000000000000000, 16, true, false), + ), + // D integer conversion boundary + ( + ( + 0xd2200053, + 0xffffffffffffffff, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0xbff0000000000000, 0, false, false), + ), + // D integer conversion type 3 + ( + ( + 0xd2300053, + 0x0000000000000003, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x4008000000000000, 0, false, false), + ), + // D float conversion type 3 + ( + ( + 0xc2300053, + 0x0000000000000000, + 0x4008000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000003, 0, true, false), + ), + // D invalid positive conversion + ( + ( + 0xc2300053, + 0x0000000000000000, + 0x7ff0000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0xffffffffffffffff, 16, true, false), + ), + // D invalid positive conversion + ( + ( + 0xc2300053, + 0x0000000000000000, + 0x7ff8000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0xffffffffffffffff, 16, true, false), + ), + // D invalid negative conversion + ( + ( + 0xc2300053, + 0x0000000000000000, + 0xfff0000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000000, 16, true, false), + ), + // D integer conversion boundary + ( + ( + 0xd2300053, + 0xffffffffffffffff, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x43f0000000000000, 1, false, false), + ), + // D cross-format exact + ( + ( + 0x42000053, + 0x0000000000000000, + 0xffffffff3fc00000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x3ff8000000000000, 0, false, false), + ), + // D unsupported format + ( + ( + 0x04000053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x4000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000000, 0, false, true), + ), + // D unsupported format + ( + ( + 0x06000053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x4000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000000, 0, false, true), + ), + // D unsupported operation + ( + ( + 0xfa000053, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000000, 0, false, true), + ), + // D unsupported opcode + ( + ( + 0x02000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000000, 0, false, true), + ), +]; diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/agent/agent.svh b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/agent/agent.svh new file mode 100644 index 000000000..6d2a96092 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/agent/agent.svh @@ -0,0 +1,19 @@ +class agent extends uvm_agent; + `uvm_component_utils(agent) + uvm_sequencer #(request_item) seqr; + driver drv; + monitor mon; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + seqr = uvm_sequencer#(request_item)::type_id::create("seqr", this); + drv = driver::type_id::create("driver", this); + mon = monitor::type_id::create("monitor", this); + endfunction + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + drv.seq_item_port.connect(seqr.seq_item_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/agent/driver.svh b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/agent/driver.svh new file mode 100644 index 000000000..a28f247d1 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/agent/driver.svh @@ -0,0 +1,55 @@ +class driver extends uvm_driver #(request_item); + `uvm_component_utils(driver) + virtual port_if vif; + reset_controller resetter; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual port_if)::get( + this, "", "vif", vif + ) || !uvm_config_db#(reset_controller)::get( + this, "", "resetter", resetter + )) + `uvm_fatal("INTERFACE", "FPU driver requires ports and reset controller") + endfunction + task run_phase(uvm_phase phase); + request_item req; + vif.request_valid = 0; + vif.response_ready = 0; + vif.instruction = 0; + vif.integer_value = 0; + vif.a = 0; + vif.rhs = 0; + vif.c = 0; + vif.rounding = 0; + forever begin + seq_item_port.get_next_item(req); + do @(negedge vif.clock); while (vif.reset); + vif.instruction = req.instruction; + vif.integer_value = req.integer_value; + vif.a = req.a; + vif.rhs = req.rhs; + vif.c = req.c; + vif.rounding = req.rounding; + vif.stall = req.stall; + vif.cancel = req.cancel; + vif.request_valid = 1; + do @(posedge vif.clock); while (!vif.request_ready); + @(negedge vif.clock); + vif.request_valid = 0; + if (req.cancel) begin + resetter.pulse(2); + end else begin + while (!vif.response_valid) @(negedge vif.clock); + repeat (req.stall) @(negedge vif.clock); + vif.response_ready = 1; + @(posedge vif.clock); + @(negedge vif.clock); + vif.response_ready = 0; + end + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/agent/item.svh b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/agent/item.svh new file mode 100644 index 000000000..17846b089 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/agent/item.svh @@ -0,0 +1,39 @@ +class request_item extends uvm_sequence_item; + logic [31:0] instruction; + logic [63:0] integer_value; + logic [63:0] a; + logic [63:0] rhs; + logic [63:0] c; + logic [2:0] rounding; + int unsigned stall; + logic cancel; + `uvm_object_utils_begin(request_item) + `uvm_field_int(instruction, UVM_ALL_ON) + `uvm_field_int(integer_value, UVM_ALL_ON) + `uvm_field_int(a, UVM_ALL_ON) + `uvm_field_int(rhs, UVM_ALL_ON) + `uvm_field_int(c, UVM_ALL_ON) + `uvm_field_int(rounding, UVM_ALL_ON) + `uvm_field_int(stall, UVM_ALL_ON) + `uvm_field_int(cancel, UVM_ALL_ON) + `uvm_object_utils_end + function new(string name = "request_item"); + super.new(name); + endfunction +endclass + +class response_item extends uvm_sequence_item; + logic [63:0] value; + logic [4:0] flags; + logic integer_result; + logic illegal; + `uvm_object_utils_begin(response_item) + `uvm_field_int(value, UVM_ALL_ON) + `uvm_field_int(flags, UVM_ALL_ON) + `uvm_field_int(integer_result, UVM_ALL_ON) + `uvm_field_int(illegal, UVM_ALL_ON) + `uvm_object_utils_end + function new(string name = "response_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/agent/monitor.svh b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/agent/monitor.svh new file mode 100644 index 000000000..640970c5a --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/agent/monitor.svh @@ -0,0 +1,57 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual port_if vif; + uvm_analysis_port #(request_item) requests; + uvm_analysis_port #(response_item) responses; + uvm_analysis_port #(bit) reset_ap; + function new(string name, uvm_component parent); + super.new(name, parent); + requests = new("requests", this); + responses = new("responses", this); + reset_ap = new("reset_ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual port_if)::get(this, "", "vif", vif)) + `uvm_fatal("INTERFACE", "FPU monitor requires ports") + endfunction + task run_phase(uvm_phase phase); + bit pending = 0; + request_item req; + response_item rsp; + forever begin + @(posedge vif.clock); + if (vif.reset) begin + pending = 0; + reset_ap.write(1); + end else begin + if (vif.request_valid && vif.request_ready) begin + if (pending) `uvm_fatal("OUTSTANDING", "FPU accepted a second request") + req = request_item::type_id::create("request"); + req.instruction = vif.instruction; + req.integer_value = vif.integer_value; + req.a = vif.a; + req.rhs = vif.rhs; + req.c = vif.c; + req.rounding = vif.rounding; + req.stall = vif.stall; + req.cancel = vif.cancel; + requests.write(req); + pending = 1; + end else if (pending && vif.request_ready) + `uvm_fatal("BUSY", "FPU reopened its request port before completion") + if (vif.response_valid && !pending) + `uvm_fatal("EXTRA_RESULT", "FPU produced an unsolicited result") + if (vif.response_valid && vif.response_ready) begin + rsp = response_item::type_id::create("response"); + rsp.value = vif.illegal ? 0 : vif.value; + rsp.flags = vif.flags; + rsp.integer_result = vif.illegal ? 0 : vif.integer_result; + rsp.illegal = vif.illegal; + responses.write(rsp); + pending = 0; + end + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/env/coverage.svh b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/env/coverage.svh new file mode 100644 index 000000000..8d32b1064 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/env/coverage.svh @@ -0,0 +1,47 @@ +`uvm_analysis_imp_decl(_request) +`uvm_analysis_imp_decl(_response) +class coverage extends uvm_component; + `uvm_component_utils(coverage) + uvm_analysis_imp_request #(request_item, coverage) requests; + uvm_analysis_imp_response #(response_item, coverage) responses; + request_item active; + covergroup behavior with function sample (request_item req, response_item rsp); + option.per_instance = 1; + opcode: coverpoint req.instruction[6:0] { + bins arithmetic = {7'h53}; bins fused[] = {7'h43, 7'h47, 7'h4b, 7'h4f}; bins invalid = {0}; + } + operation: coverpoint req.instruction[31:27] iff (req.instruction[6:0] == 7'h53) { + bins supported[] = {0, 1, 2, 3, 4, 5, 8, 11, 20, 24, 26, 28, 30}; bins invalid = {31}; + } + format: coverpoint req.instruction[26:25] {bins formats[] = {0, 1}; bins invalid = {[2 : 3]};} + rounding: coverpoint (req.instruction[14:12] == 7 ? req.rounding : req.instruction[14:12]) + iff (req.instruction[6:0] inside {7'h43,7'h47,7'h4b,7'h4f} || + (req.instruction[6:0] == 7'h53 && req.instruction[31:27] inside {0,1,2,3,8,11,24,26})) { + bins modes[] = {[0 : 4]}; bins invalid = {[5 : 7]}; + } + integer_type: coverpoint req.instruction[21:20] iff (req.instruction[31:27] inside {24, 26}) { + bins types[] = {[0 : 3]}; + } + flags: coverpoint rsp.flags {bins exceptions[] = {0, 1, 3, 5, 8, 16};} + result_kind: coverpoint rsp.integer_result; + illegal: coverpoint rsp.illegal; + classification: coverpoint rsp.value[9:0] + iff (req.instruction[6:0] == 7'h53 && req.instruction[31:27] == 28 && + req.instruction[14:12] == 1 && !rsp.illegal) { + bins classes[] = {1, 2, 4, 8, 16, 32, 64, 128, 256, 512}; + } + backpressure: coverpoint req.stall {bins stalls[] = {0, 3};} + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + requests = new("requests", this); + responses = new("responses", this); + behavior = new(); + endfunction + function void write_request(request_item req); + active = req; + endfunction + function void write_response(response_item rsp); + behavior.sample(active, rsp); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/env/env.svh b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/env/env.svh new file mode 100644 index 000000000..17f7af36e --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/env/env.svh @@ -0,0 +1,27 @@ +`uvm_analysis_imp_decl(_reset) +class env extends environment #(request_item, response_item, model); + `uvm_component_utils(env) + agent bus; + coverage cov; + uvm_analysis_imp_reset #(bit, env) reset_export; + function new(string name, uvm_component parent); + super.new(name, parent); + reset_export = new("reset_export", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + cov = coverage::type_id::create("coverage", this); + bus = agent::type_id::create("agent", this); + endfunction + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + bus.mon.requests.connect(input_export); + bus.mon.requests.connect(cov.requests); + bus.mon.reset_ap.connect(reset_export); + bus.mon.responses.connect(output_export); + bus.mon.responses.connect(cov.responses); + endfunction + function void write_reset(bit asserted); + scoreboard.cancel_pending(); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/env/model.svh b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/env/model.svh new file mode 100644 index 000000000..eb00ef84d --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/env/model.svh @@ -0,0 +1,20 @@ +class model extends reference_model #(request_item, response_item); + `uvm_component_utils(model) + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void write(request_item req); + response_item rsp = response_item::type_id::create("expected"); + longint unsigned value; + int unsigned flags; + byte unsigned integer_result, illegal; + fpu_expected(req.instruction, req.integer_value, req.a, req.rhs, req.c, req.rounding, value, + flags, integer_result, illegal); + rsp.value = illegal ? 0 : value; + rsp.flags = flags[4:0]; + rsp.integer_result = illegal ? 0 : integer_result[0]; + rsp.illegal = illegal[0]; + expected_ap.write(rsp); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/filelist.f b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/filelist.f new file mode 100644 index 000000000..ec2877c75 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/filelist.f @@ -0,0 +1,7 @@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv ++incdir+@RESOURCES@ +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/FPU/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/interface.sv b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/interface.sv new file mode 100644 index 000000000..57e3fd0e7 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/interface.sv @@ -0,0 +1,29 @@ +interface port_if ( + input logic clock, + input logic reset +); + logic request_valid, request_ready, response_valid, response_ready; + logic [31:0] instruction; + logic [63:0] integer_value; + logic [63:0] a; + logic [63:0] rhs; + logic [63:0] c; + logic [2:0] rounding; + int unsigned stall; + logic cancel; + logic [63:0] value; + logic [4:0] flags; + logic integer_result; + logic illegal; + assert property (@(posedge clock) disable iff (reset) + request_valid && !request_ready |=> request_valid && + $stable( + {instruction, integer_value, a, rhs, c, rounding} + )) + else $fatal(1, "FPU request changed under backpressure"); + assert property (@(posedge clock) disable iff (reset) + response_valid && !response_ready |=> response_valid && $stable( + {value, flags, integer_result, illegal} + )) + else $fatal(1, "FPU result changed under backpressure"); +endinterface diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/package.sv b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/package.sv new file mode 100644 index 000000000..34827bdbe --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/package.sv @@ -0,0 +1,38 @@ +package fpu_pkg; + timeunit 1ns; timeprecision 1ps; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + import "DPI-C" function int unsigned fpu_case_count(); + import "DPI-C" function int unsigned fpu_cancel_case(); + import "DPI-C" function void fpu_case( + input int unsigned index, + output int unsigned instruction, + output longint unsigned integer_value, + a, + rhs, + c, + output int unsigned rounding + ); + import "DPI-C" function void fpu_expected( + input int unsigned instruction, + input longint unsigned integer_value, + a, + rhs, + c, + input int unsigned rounding, + output longint unsigned value, + output int unsigned flags, + output byte unsigned integer_result, + illegal + ); + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/agent.svh" + `include "env/model.svh" + `include "env/coverage.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..a4470fe36 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/seq/sequence.svh @@ -0,0 +1,27 @@ +class stimulus extends uvm_sequence #(request_item); + `uvm_object_utils(stimulus) + int unsigned sent = 0; + function new(string name = "stimulus"); + super.new(name); + endfunction + task body(); + request_item req = request_item::type_id::create("request"); + int unsigned rounding; + start_item(req); + fpu_case(fpu_cancel_case(), req.instruction, req.integer_value, req.a, req.rhs, req.c, + rounding); + req.rounding = rounding[2:0]; + req.stall = 0; + req.cancel = 1; + finish_item(req); + for (int i = 0; i < fpu_case_count(); i++) begin + start_item(req); + fpu_case(i, req.instruction, req.integer_value, req.a, req.rhs, req.c, rounding); + req.rounding = rounding[2:0]; + req.stall = (i % 2) * 3; + req.cancel = 0; + finish_item(req); + sent++; + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/suite.toml b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/suite.toml new file mode 100644 index 000000000..4b9957736 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/suite.toml @@ -0,0 +1,20 @@ +[model] +manifest = "src/csrc/Cargo.toml" +library = "libfpu_reference" + +[[tests]] +name = "fpu" +top = "fpu_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 2000000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +functional_groups = ["fpu_pkg::coverage::behavior"] +sink_ready = 1 +sink_stall = 0 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/tb.sv b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/tb.sv new file mode 100644 index 000000000..42c3d96b9 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/tb.sv @@ -0,0 +1,34 @@ +module fpu_tb; + import uvm_pkg::*; + import fpu_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + clock_if timing (clock); + port_if bus ( + clock, + timing.reset + ); + FPU dut ( + .clock(clock), + .reset(bus.reset), + .io_request_valid(bus.request_valid), + .io_request_ready(bus.request_ready), + .io_request_bits_instruction(bus.instruction), + .io_request_bits_integer(bus.integer_value), + .io_request_bits_a(bus.a), + .io_request_bits_rhs(bus.rhs), + .io_request_bits_c(bus.c), + .io_request_bits_rounding(bus.rounding), + .io_response_valid(bus.response_valid), + .io_response_ready(bus.response_ready), + .io_response_bits_data(bus.value), + .io_response_bits_flags(bus.flags), + .io_response_bits_integer(bus.integer_result), + .io_response_bits_illegal(bus.illegal) + ); + initial begin + uvm_config_db#(virtual clock_if)::set(null, "uvm_test_top*", "timing", timing); + uvm_config_db#(virtual port_if)::set(null, "uvm_test_top*", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/tests/test.svh new file mode 100644 index 000000000..ef256d9e1 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/tests/test.svh @@ -0,0 +1,27 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env bench; + reset_controller resetter; + virtual port_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + resetter = reset_controller::type_id::create("reset", this); + uvm_config_db#(reset_controller)::set(this, "env.agent.driver", "resetter", resetter); + bench = env::type_id::create("env", this); + if (!uvm_config_db#(virtual port_if)::get(this, "", "vif", vif)) + `uvm_fatal("INTERFACE", "Test interface is required") + endfunction + task execute(); + stimulus sequence_ = stimulus::type_id::create("sequence"); + resetter.pulse(4); + sequence_.start(bench.bus.seqr); + bench.scoreboard.wait_checked(sequence_.sent); + @(negedge vif.clock); + evidence::check(evidence::cancelled == 1, + "Reset did not cancel the outstanding floating divide"); + evidence::check(bench.scoreboard.checked == sequence_.sent, "Comparison count mismatch"); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/main/scala/Emit.scala b/arch/src/main/scala/framework/core/execution/fpu/src/main/scala/Emit.scala new file mode 100644 index 000000000..6518b056d --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/main/scala/Emit.scala @@ -0,0 +1,15 @@ +package framework.core.execution.fpu + +import framework.core.execution.fpu.configs.Params +import java.nio.file.{Files, Paths} +import upickle.default.read + +object Emit extends App { + require(args.length >= 2 && args(0) == "--config") + val param = read[Params](Files.readString(Paths.get(args(1)))) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new FPU(param), + firtoolOpts = args.drop(2), + args = Array("--target-dir", "build/FPU", "--split-verilog") + ) +} diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/main/scala/FPU.scala b/arch/src/main/scala/framework/core/execution/fpu/src/main/scala/FPU.scala new file mode 100644 index 000000000..8d05c6bb0 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/main/scala/FPU.scala @@ -0,0 +1,250 @@ +package framework.core.execution.fpu + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.lib.arith.Fma +import framework.core.execution.fpu.configs.{Params => ModuleParam} + +class Request(b: ModuleParam) extends Bundle { + val p = b + + val instruction = UInt(32.W) + val integer = UInt(p.xLen.W) + val a = UInt(p.fLen.W) + val rhs = UInt(p.fLen.W) + val c = UInt(p.fLen.W) + val rounding = UInt(3.W) +} + +class Response(b: ModuleParam) extends Bundle { + val p = b + + val data = UInt(p.xLen.W) + val flags = UInt(5.W) + val integer = Bool() + val illegal = Bool() +} + +/** IEEE binary32/binary64 arithmetic, conversions and register transfers. */ +class FPUIO(b: ModuleParam) extends Bundle { + val p = b + + val request = Flipped(Decoupled(new Request(b))) + val response = Decoupled(new Response(b)) + +} + +@instantiable +class FPU(b: ModuleParam) extends Module { + val p = b + + @public + val io = IO(new FPUIO(b)) + + val idle :: arithmetic :: divide :: respond :: Nil = Enum(4) + val state = RegInit(idle) + val selectedDouble = Reg(Bool()) + val answer = Reg(new Response(b)) + val request = io.request.bits + val instruction = request.instruction + val opcode = instruction(6, 0) + val operation = instruction(31, 27) + val funct3 = instruction(14, 12) + val rs2 = instruction(24, 20) + val fused = opcode === "h43".U || opcode === "h47".U || opcode === "h4b".U || opcode === "h4f".U + val double = instruction(25) + val rounding = Mux(funct3 === 7.U, request.rounding, funct3) + val rounded = fused || operation <= 3.U || operation === 8.U || operation === 11.U || + operation === 24.U || operation === 26.U + val formatLegal = instruction(26) === 0.U + val legal = WireDefault(false.B) + val toInteger = !fused && (operation === 20.U || operation === 24.U || operation === 28.U) + val isArithmetic = fused || operation <= 2.U + val isDivide = operation === 3.U || operation === 11.U + when(fused)(legal := true.B) + when(opcode === "h53".U) { + switch(operation) { + is(0.U, 1.U, 2.U, 3.U)(legal := true.B) + is(4.U)(legal := funct3 <= 2.U) + is(5.U)(legal := funct3 <= 1.U) + is(8.U)(legal := rs2 === Mux(double, 0.U, 1.U)) + is(11.U)(legal := rs2 === 0.U) + is(20.U)(legal := funct3 <= 2.U) + is(24.U, 26.U)(legal := rs2 <= 3.U) + is(28.U)(legal := rs2 === 0.U && funct3 <= 1.U) + is(30.U)(legal := rs2 === 0.U && funct3 === 0.U) + } + } + val illegal = !legal || !formatLegal || (rounded && rounding > 4.U) + io.request.ready := state === idle + io.response.valid := state === respond + io.response.bits := answer + when(io.response.fire)(state := idle) + + val simpleData = Wire(Vec(2, UInt(64.W))) + val simpleFlags = Wire(Vec(2, UInt(5.W))) + val arithmeticData = Wire(Vec(2, UInt(64.W))) + val arithmeticFlags = Wire(Vec(2, UInt(5.W))) + val arithmeticValid = Wire(Vec(2, Bool())) + val divideData = Wire(Vec(2, UInt(64.W))) + val divideFlags = Wire(Vec(2, UInt(5.W))) + val divideValid = Wire(Vec(2, Bool())) + val divideReady = Wire(Vec(2, Bool())) + + for (((exp, sig), index) <- Seq((8, 24), (11, 53)).zipWithIndex) { + val width = exp + sig + val single = index == 0 + val canonical = if (single) BigInt("7fc00000", 16) else BigInt("7ff8000000000000", 16) + val rawA = if (single) Mux(request.a(63, 32).andR, request.a(31, 0), canonical.U(32.W)) else request.a + val rawB = if (single) Mux(request.rhs(63, 32).andR, request.rhs(31, 0), canonical.U(32.W)) else request.rhs + val rawC = if (single) Mux(request.c(63, 32).andR, request.c(31, 0), canonical.U(32.W)) else request.c + val a = hardfloat.recFNFromFN(exp, sig, rawA) + val b = hardfloat.recFNFromFN(exp, sig, rawB) + val c = hardfloat.recFNFromFN(exp, sig, rawC) + val unpackedA = hardfloat.rawFloatFromRecFN(exp, sig, a) + val unpackedB = hardfloat.rawFloatFromRecFN(exp, sig, b) + val signNaN = hardfloat.isSigNaNRawFloat(unpackedA) || hardfloat.isSigNaNRawFloat(unpackedB) + val start = io.request.fire && !illegal && double === (index == 1).B + val fma: Instance[Fma] = Instantiate(new Fma(3, exp, sig)) + fma.io.validin := start && isArithmetic + fma.io.roundingMode := rounding + fma.io.detectTininess := 1.U + fma.io.op := Mux(fused, opcode(3, 2), Mux(operation === 1.U, 1.U, 0.U)) + fma.io.a := a + fma.io.b := Mux( + fused || operation === 2.U, + b, + hardfloat.recFNFromFN(exp, sig, (BigInt((1 << (exp - 1)) - 1) << (sig - 1)).U(width.W)) + ) + fma.io.c := Mux( + fused, + c, + Mux( + operation === 2.U, + hardfloat.recFNFromFN(exp, sig, Cat(rawA(width - 1) ^ rawB(width - 1), 0.U((width - 1).W))), + b + ) + ) + val fmaNan = hardfloat.rawFloatFromRecFN(exp, sig, fma.io.out).isNaN + val fmaRaw = Mux(fmaNan, canonical.U(width.W), hardfloat.fNFromRecFN(exp, sig, fma.io.out)) + arithmeticData(index) := (if (single) Cat(~0.U(32.W), fmaRaw) else fmaRaw) + arithmeticFlags(index) := fma.io.exceptionFlags + arithmeticValid(index) := fma.io.validout + + val div = Module(new hardfloat.DivSqrtRecFN_small(exp, sig, 0)) + div.io.inValid := start && isDivide + div.io.sqrtOp := operation === 11.U + div.io.a := a + div.io.b := b + div.io.roundingMode := rounding + div.io.detectTininess := 1.U + val divRaw = Mux( + hardfloat.rawFloatFromRecFN(exp, sig, div.io.out).isNaN, + canonical.U(width.W), + hardfloat.fNFromRecFN(exp, sig, div.io.out) + ) + divideData(index) := (if (single) Cat(~0.U(32.W), divRaw) else divRaw) + divideFlags(index) := div.io.exceptionFlags + divideValid(index) := div.io.outValid_div || div.io.outValid_sqrt + divideReady(index) := div.io.inReady + + val compare = Module(new hardfloat.CompareRecFN(exp, sig)) + compare.io.a := a + compare.io.b := b + compare.io.signaling := funct3 =/= 2.U + val bothZero = unpackedA.isZero && unpackedB.isZero + val chooseA = + Mux(funct3(0), compare.io.gt || (bothZero && !rawA(width - 1)), compare.io.lt || (bothZero && rawA(width - 1))) + val minimumMaximum = Mux( + unpackedA.isNaN && unpackedB.isNaN, + canonical.U(width.W), + Mux(unpackedA.isNaN, rawB, Mux(unpackedB.isNaN, rawA, Mux(chooseA, rawA, rawB))) + ) + val injectedSign = MuxLookup(funct3, rawB(width - 1))(Seq( + 1.U -> !rawB(width - 1), + 2.U -> (rawA(width - 1) ^ rawB(width - 1)) + )) + val injected = Cat(injectedSign, rawA(width - 2, 0)) + + val intInput = Mux(rs2(1), request.integer, Cat(Fill(32, !rs2(0) && request.integer(31)), request.integer(31, 0))) + val fromInt = Module(new hardfloat.INToRecFN(64, exp, sig)) + fromInt.io.in := intInput + fromInt.io.signedIn := !rs2(0) + fromInt.io.roundingMode := rounding + fromInt.io.detectTininess := 1.U + val fromIntRaw = hardfloat.fNFromRecFN(exp, sig, fromInt.io.out) + val convertedIntegers = Wire(Vec(2, UInt(64.W))) + val convertedFlags = Wire(Vec(2, UInt(5.W))) + for ((intWidth, intIndex) <- Seq(32, 64).zipWithIndex) { + val toInt = Module(new hardfloat.RecFNToIN(exp, sig, intWidth)) + toInt.io.in := a + toInt.io.signedOut := !rs2(0) + toInt.io.roundingMode := rounding + convertedIntegers(intIndex) := (if (intWidth == 32) Cat(Fill(32, toInt.io.out(31)), toInt.io.out) + else toInt.io.out) + convertedFlags(intIndex) := Cat(toInt.io.intExceptionFlags(2, 1).orR, 0.U(3.W), toInt.io.intExceptionFlags(0)) + } + val (sourceExp, sourceSig) = if (single) (11, 53) else (8, 24) + val source = + if (single) request.a + else + Mux(request.a(63, 32).andR, request.a(31, 0), "h7fc00000".U(32.W)) + val convert = Module(new hardfloat.RecFNToRecFN(sourceExp, sourceSig, exp, sig)) + convert.io.in := hardfloat.recFNFromFN(sourceExp, sourceSig, source) + convert.io.roundingMode := rounding + convert.io.detectTininess := 1.U + val convertedFloat = Mux( + hardfloat.rawFloatFromRecFN(exp, sig, convert.io.out).isNaN, + canonical.U(width.W), + hardfloat.fNFromRecFN(exp, sig, convert.io.out) + ) + val floatResult = MuxLookup(operation, 0.U(width.W))(Seq( + 4.U -> injected, + 5.U -> minimumMaximum, + 8.U -> convertedFloat, + 26.U -> fromIntRaw, + 30.U -> request.integer(width - 1, 0) + )) + val integerResult = MuxLookup(operation, 0.U(64.W))(Seq( + 20.U -> Cat( + 0.U(63.W), + Mux(funct3 === 0.U, compare.io.lt || compare.io.eq, Mux(funct3 === 1.U, compare.io.lt, compare.io.eq)) + ), + 24.U -> convertedIntegers(rs2(1)), + 28.U -> Mux( + funct3(0), + hardfloat.classifyRecFN(exp, sig, a).pad(64), + if (single) Cat(Fill(32, request.a(31)), request.a(31, 0)) else request.a + ) + )) + simpleData(index) := Mux(toInteger, integerResult, if (single) Cat(~0.U(32.W), floatResult) else floatResult) + simpleFlags(index) := MuxLookup(operation, 0.U(5.W))(Seq( + 5.U -> Cat(signNaN, 0.U(4.W)), + 8.U -> convert.io.exceptionFlags, + 20.U -> compare.io.exceptionFlags, + 24.U -> convertedFlags(rs2(1)), + 26.U -> fromInt.io.exceptionFlags + )) + } + + when(io.request.fire) { + selectedDouble := double + answer.data := simpleData(double) + answer.flags := Mux(illegal, 0.U, simpleFlags(double)) + answer.integer := toInteger + answer.illegal := illegal + state := Mux(illegal, respond, Mux(isArithmetic, arithmetic, Mux(isDivide, divide, respond))) + when(!illegal && isDivide)(assert(divideReady(double), "Floating divider must be idle on accepted command")) + } + when(state === arithmetic && arithmeticValid(selectedDouble)) { + answer.data := arithmeticData(selectedDouble) + answer.flags := arithmeticFlags(selectedDouble) + state := respond + } + when(state === divide && divideValid(selectedDouble)) { + answer.data := divideData(selectedDouble) + answer.flags := divideFlags(selectedDouble) + state := respond + } +} diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/core/execution/fpu/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..e3fb405e9 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/main/scala/configs/Params.scala @@ -0,0 +1,6 @@ +package framework.core.execution.fpu.configs + +import upickle.default.{macroRW, ReadWriter} + +case class Params(xLen: Int, fLen: Int) { require(xLen == 64 && fLen == 64) } +object Params { implicit val rw: ReadWriter[Params] = macroRW } diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/main/scala/configs/default.toml b/arch/src/main/scala/framework/core/execution/fpu/src/main/scala/configs/default.toml new file mode 100644 index 000000000..180fa9af7 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/main/scala/configs/default.toml @@ -0,0 +1 @@ +fLen = 64 diff --git a/arch/src/main/scala/framework/core/execution/instruction/LICENSE.Berkeley b/arch/src/main/scala/framework/core/execution/instruction/LICENSE.Berkeley new file mode 100644 index 000000000..7cff15e40 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/LICENSE.Berkeley @@ -0,0 +1,24 @@ +Copyright (c) 2012-2014, The Regents of the University of California +(Regents). All Rights Reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. Neither the name of the Regents nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, +SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING +OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS +BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED +HEREUNDER IS PROVIDED "AS IS". REGENTS HAS NO OBLIGATION TO PROVIDE +MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. diff --git a/arch/src/main/scala/framework/core/execution/instruction/LICENSE.SiFive b/arch/src/main/scala/framework/core/execution/instruction/LICENSE.SiFive new file mode 100644 index 000000000..7e7093375 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/LICENSE.SiFive @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2016-2017 SiFive, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/arch/src/main/scala/framework/core/execution/instruction/build.sc b/arch/src/main/scala/framework/core/execution/instruction/build.sc new file mode 100644 index 000000000..2d6650403 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/build.sc @@ -0,0 +1,21 @@ +import mill._ +import $file.^.^.^.^.^.^.^.settings + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.core.execution.instruction.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Instruction.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/core/execution/instruction/configs/uvm.toml b/arch/src/main/scala/framework/core/execution/instruction/configs/uvm.toml new file mode 100644 index 000000000..3da02cd69 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/configs/uvm.toml @@ -0,0 +1,4 @@ +[parameters] +xLen = 64 + +[defines] diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/csrc/Cargo.lock b/arch/src/main/scala/framework/core/execution/instruction/src/csrc/Cargo.lock new file mode 100644 index 000000000..6935dd56e --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/csrc/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "instruction-reference" +version = "0.1.0" diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/csrc/Cargo.toml b/arch/src/main/scala/framework/core/execution/instruction/src/csrc/Cargo.toml new file mode 100644 index 000000000..f94657703 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/csrc/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "instruction-reference" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["cdylib"] diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/csrc/src/core/mod.rs b/arch/src/main/scala/framework/core/execution/instruction/src/csrc/src/core/mod.rs new file mode 100644 index 000000000..fb27202ca --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/csrc/src/core/mod.rs @@ -0,0 +1 @@ +pub mod riscv; diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/csrc/src/core/riscv/mod.rs b/arch/src/main/scala/framework/core/execution/instruction/src/csrc/src/core/riscv/mod.rs new file mode 100644 index 000000000..97ffb8d9c --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/csrc/src/core/riscv/mod.rs @@ -0,0 +1,6 @@ +#[path = "../../../../../../../../../../../../../bebop/src/nodes/bemu/src/core/riscv/trap.rs"] +mod trap; +#[path = "../../../../../../../../../../../../../bebop/src/nodes/bemu/src/core/riscv/compressed.rs"] +mod compressed; +pub use trap::Trap; +pub use compressed::decode as expand_compressed; diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/csrc/src/lib.rs b/arch/src/main/scala/framework/core/execution/instruction/src/csrc/src/lib.rs new file mode 100644 index 000000000..be67a8efd --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/csrc/src/lib.rs @@ -0,0 +1,152 @@ +mod core; + +fn signed(value: u32, width: u32) -> u64 { + ((value << (32 - width)) as i32 >> (32 - width)) as i64 as u64 +} + +#[no_mangle] +pub unsafe extern "C" fn instruction_expected( + bits: u32, + expanded: *mut u32, + illegal: *mut u8, + immediate: *mut u64, + atomic: *mut u32, + function: *mut u32, + word: *mut u8, + immediate_mode: *mut u8, +) { + let insn = if bits & 3 != 3 { + match core::riscv::expand_compressed(bits as u16) { + Ok(value) => value, + Err(_) => { + *expanded = 0; + *illegal = 1; + *immediate = 0; + *atomic = 0; + *function = 0; + *word = 0; + *immediate_mode = 0; + return; + } + } + } else { + bits + }; + let opcode = insn & 127; + let funct3 = (insn >> 12) & 7; + let funct7 = insn >> 25; + let rd = (insn >> 7) & 31; + let rs2 = (insn >> 20) & 31; + let is_word = matches!(opcode, 0x1b | 0x3b); + let atomic_code = match funct7 >> 2 { + 0 => 2, + 1 => 1, + 2 => 10, + 3 => 11, + 4 => 3, + 8 => 5, + 12 => 4, + 16 => 6, + 20 => 7, + 24 => 8, + 28 => 9, + _ => 0, + }; + let legal = match opcode { + 0x37 | 0x17 | 0x6f => true, + 0x67 => funct3 == 0, + 0x63 => matches!(funct3, 0 | 1 | 4 | 5 | 6 | 7), + 0x13 | 0x1b => match funct3 { + 0 => true, + 1 => { + if is_word { + funct7 == 0 + } else { + funct7 >> 1 == 0 + } + } + 5 => { + if is_word { + matches!(funct7, 0 | 32) + } else { + matches!(funct7 >> 1, 0 | 16) + } + } + _ => !is_word, + }, + 0x33 | 0x3b => { + if funct7 == 1 { + !is_word || funct3 == 0 || funct3 >= 4 + } else { + (funct7 == 0 || (funct7 == 32 && matches!(funct3, 0 | 5))) + && (!is_word || matches!(funct3, 0 | 1 | 5)) + } + } + 0x03 => funct3 <= 6, + 0x23 => funct3 <= 3, + 0x07 | 0x27 => matches!(funct3, 2 | 3), + 0x2f => matches!(funct3, 2 | 3) && atomic_code != 0 && (funct7 >> 2 != 2 || rs2 == 0), + 0x43 | 0x47 | 0x4b | 0x4f | 0x53 => true, + 0x0f => matches!(funct3, 0 | 1), + 0x73 => { + if funct3 == 0 { + matches!( + insn, + 0x00000073 | 0x00100073 | 0x30200073 | 0x10200073 | 0x10500073 + ) || (funct7 == 9 && rd == 0) + } else { + funct3 != 4 + } + } + 0x2b => funct3 == 7, + 0x7b => true, + _ => false, + }; + *expanded = insn; + *illegal = u8::from(!legal); + *immediate = match opcode { + 0x23 | 0x27 => signed(((insn >> 25) << 5) | ((insn >> 7) & 31), 12), + 0x63 => signed( + ((insn >> 31) << 12) + | (((insn >> 7) & 1) << 11) + | (((insn >> 25) & 63) << 5) + | (((insn >> 8) & 15) << 1), + 13, + ), + 0x6f => signed( + ((insn >> 31) << 20) + | (((insn >> 12) & 255) << 12) + | (((insn >> 20) & 1) << 11) + | (((insn >> 21) & 1023) << 1), + 21, + ), + 0x17 | 0x37 => signed(insn & 0xfffff000, 32), + _ => signed(insn >> 20, 12), + }; + *atomic = atomic_code; + *function = match funct3 { + 0 => { + if matches!(opcode, 0x33 | 0x3b) && funct7 & 32 != 0 { + 10 + } else { + 0 + } + } + 1 => 1, + 2 => 12, + 3 => 14, + 4 => 4, + 5 => { + if funct7 & 32 != 0 { + 11 + } else { + 5 + } + } + 6 => 6, + 7 => 7, + _ => unreachable!(), + }; + *word = u8::from(is_word); + *immediate_mode = u8::from(matches!(opcode, 0x13 | 0x1b)); +} diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/agent/agent.svh b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/agent/agent.svh new file mode 100644 index 000000000..6d2a96092 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/agent/agent.svh @@ -0,0 +1,19 @@ +class agent extends uvm_agent; + `uvm_component_utils(agent) + uvm_sequencer #(request_item) seqr; + driver drv; + monitor mon; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + seqr = uvm_sequencer#(request_item)::type_id::create("seqr", this); + drv = driver::type_id::create("driver", this); + mon = monitor::type_id::create("monitor", this); + endfunction + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + drv.seq_item_port.connect(seqr.seq_item_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/agent/driver.svh b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/agent/driver.svh new file mode 100644 index 000000000..343374875 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/agent/driver.svh @@ -0,0 +1,27 @@ +class driver extends uvm_driver #(request_item); + `uvm_component_utils(driver) + virtual port_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual port_if)::get(this, "", "vif", vif)) + `uvm_fatal("INTERFACE", "Driver interface is required") + endfunction + task run_phase(uvm_phase phase); + request_item req; + vif.sample = 0; + vif.bits = 0; + forever begin + seq_item_port.get_next_item(req); + do @(negedge vif.clock); while (vif.reset); + vif.bits = req.bits; + vif.sample = 1; + @(posedge vif.clock); + seq_item_port.item_done(); + @(negedge vif.clock); + vif.sample = 0; + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/agent/item.svh b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/agent/item.svh new file mode 100644 index 000000000..e0f3bdade --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/agent/item.svh @@ -0,0 +1,47 @@ +class request_item extends uvm_sequence_item; + logic [31:0] bits; + `uvm_object_utils_begin(request_item) + `uvm_field_int(bits, UVM_ALL_ON) + `uvm_object_utils_end + function new(string name = "request_item"); + super.new(name); + endfunction +endclass + +class response_item extends uvm_sequence_item; + logic [31:0] expanded; + logic compressed; + logic illegal; + logic [6:0] opcode; + logic [2:0] funct3; + logic [6:0] funct7; + logic [4:0] rs1; + logic [4:0] rs2; + logic [4:0] rd; + logic [11:0] csr_address; + logic [63:0] immediate; + logic [3:0] atomic; + logic [4:0] function_code; + logic word; + logic immediate_mode; + `uvm_object_utils_begin(response_item) + `uvm_field_int(expanded, UVM_ALL_ON) + `uvm_field_int(compressed, UVM_ALL_ON) + `uvm_field_int(illegal, UVM_ALL_ON) + `uvm_field_int(opcode, UVM_ALL_ON) + `uvm_field_int(funct3, UVM_ALL_ON) + `uvm_field_int(funct7, UVM_ALL_ON) + `uvm_field_int(rs1, UVM_ALL_ON) + `uvm_field_int(rs2, UVM_ALL_ON) + `uvm_field_int(rd, UVM_ALL_ON) + `uvm_field_int(csr_address, UVM_ALL_ON) + `uvm_field_int(immediate, UVM_ALL_ON) + `uvm_field_int(atomic, UVM_ALL_ON) + `uvm_field_int(function_code, UVM_ALL_ON) + `uvm_field_int(word, UVM_ALL_ON) + `uvm_field_int(immediate_mode, UVM_ALL_ON) + `uvm_object_utils_end + function new(string name = "response_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/agent/monitor.svh b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/agent/monitor.svh new file mode 100644 index 000000000..0bb1954a4 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/agent/monitor.svh @@ -0,0 +1,45 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual port_if vif; + uvm_analysis_port #(request_item) requests; + uvm_analysis_port #(response_item) responses; + function new(string name, uvm_component parent); + super.new(name, parent); + requests = new("requests", this); + responses = new("responses", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual port_if)::get(this, "", "vif", vif)) + `uvm_fatal("INTERFACE", "Monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + request_item req; + response_item rsp; + forever begin + @(posedge vif.clock); + if (!vif.reset && vif.sample) begin + req = request_item::type_id::create("request"); + rsp = response_item::type_id::create("response"); + req.bits = vif.bits; + rsp.expanded = vif.illegal ? 0 : vif.expanded; + rsp.compressed = vif.compressed; + rsp.illegal = vif.illegal; + rsp.opcode = vif.illegal ? 0 : vif.opcode; + rsp.funct3 = vif.illegal ? 0 : vif.funct3; + rsp.funct7 = vif.illegal ? 0 : vif.funct7; + rsp.rs1 = vif.illegal ? 0 : vif.rs1; + rsp.rs2 = vif.illegal ? 0 : vif.rs2; + rsp.rd = vif.illegal ? 0 : vif.rd; + rsp.csr_address = vif.illegal ? 0 : vif.csr_address; + rsp.immediate = vif.illegal ? 0 : vif.immediate; + rsp.atomic = vif.illegal ? 0 : vif.atomic; + rsp.function_code = vif.illegal ? 0 : vif.function_code; + rsp.word = vif.illegal ? 0 : vif.word; + rsp.immediate_mode = vif.illegal ? 0 : vif.immediate_mode; + requests.write(req); + responses.write(rsp); + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/env/coverage.svh b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/env/coverage.svh new file mode 100644 index 000000000..c71181aa1 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/env/coverage.svh @@ -0,0 +1,25 @@ +class coverage extends uvm_subscriber #(request_item); + `uvm_component_utils(coverage) + covergroup behavior with function sample (request_item t); + option.per_instance = 1; + quadrant: coverpoint t.bits[1:0] {bins compressed[] = {[0 : 2]};} + compressed_opcode: coverpoint t.bits[15:13] iff (t.bits[1:0] != 3) { + bins functions[] = {[0 : 7]}; + } + compressed_encodings: cross quadrant, compressed_opcode; + opcode: coverpoint t.bits[6:0] iff (t.bits[1:0] == 3) { + bins opcodes[]={7'h03,7'h07,7'h0f,7'h13,7'h17,7'h1b,7'h23,7'h27,7'h2b,7'h2f,7'h33,7'h37,7'h3b,7'h43,7'h47,7'h4b,7'h4f,7'h53,7'h63,7'h67,7'h6f,7'h73,7'h7b}; + } + rd: coverpoint t.bits[11:7] {bins registers[] = {[0 : 31]};} + rs1: coverpoint t.bits[19:15] iff (t.bits[1:0] == 3) {bins registers[] = {[0 : 31]};} + rs2: coverpoint t.bits[24:20] iff (t.bits[1:0] == 3) {bins registers[] = {[0 : 31]};} + immediate_sign: coverpoint t.bits[31] iff (t.bits[1:0] == 3); + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + behavior = new(); + endfunction + function void write(request_item t); + behavior.sample(t); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/env/env.svh b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/env/env.svh new file mode 100644 index 000000000..5a8082f7a --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/env/env.svh @@ -0,0 +1,19 @@ +class env extends environment #(request_item, response_item, model); + `uvm_component_utils(env) + agent bus; + coverage cov; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + cov = coverage::type_id::create("coverage", this); + bus = agent::type_id::create("agent", this); + endfunction + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + bus.mon.requests.connect(input_export); + bus.mon.requests.connect(cov.analysis_export); + bus.mon.responses.connect(output_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/env/model.svh b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/env/model.svh new file mode 100644 index 000000000..c3369607b --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/env/model.svh @@ -0,0 +1,32 @@ +class model extends reference_model #(request_item, response_item); + `uvm_component_utils(model) + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void write(request_item req); + response_item rsp = response_item::type_id::create("expected"); + int unsigned expanded, atomic, function_code; + longint unsigned immediate; + byte unsigned illegal, word, immediate_mode; + instruction_expected(req.bits, expanded, illegal, immediate, atomic, function_code, word, + immediate_mode); + rsp.set_name($sformatf("expected_%08h", req.bits)); + rsp.compressed = req.bits[1:0] != 3; + rsp.illegal = illegal[0]; + rsp.expanded = illegal ? 0 : expanded; + rsp.opcode = illegal ? 0 : expanded[6:0]; + rsp.funct3 = illegal ? 0 : expanded[14:12]; + rsp.funct7 = illegal ? 0 : expanded[31:25]; + rsp.rs1 = illegal ? 0 : expanded[19:15]; + rsp.rs2 = illegal ? 0 : expanded[24:20]; + rsp.rd = illegal ? 0 : expanded[11:7]; + rsp.csr_address = illegal ? 0 : expanded[31:20]; + rsp.immediate = illegal ? 0 : immediate; + rsp.atomic = illegal ? 0 : atomic[3:0]; + rsp.function_code = illegal ? 0 : function_code[4:0]; + rsp.word = illegal ? 0 : word[0]; + rsp.immediate_mode = illegal ? 0 : immediate_mode[0]; + expected_ap.write(rsp); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/filelist.f b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/filelist.f new file mode 100644 index 000000000..fd2376e6c --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/filelist.f @@ -0,0 +1,7 @@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv ++incdir+@RESOURCES@ +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/Instruction/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/interface.sv b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/interface.sv new file mode 100644 index 000000000..342d5d325 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/interface.sv @@ -0,0 +1,22 @@ +interface port_if ( + input logic clock, + input logic reset +); + logic sample; + logic [31:0] bits; + logic [31:0] expanded; + logic compressed; + logic illegal; + logic [6:0] opcode; + logic [2:0] funct3; + logic [6:0] funct7; + logic [4:0] rs1; + logic [4:0] rs2; + logic [4:0] rd; + logic [11:0] csr_address; + logic [63:0] immediate; + logic [3:0] atomic; + logic [4:0] function_code; + logic word; + logic immediate_mode; +endinterface diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/package.sv b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/package.sv new file mode 100644 index 000000000..88d268d71 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/package.sv @@ -0,0 +1,25 @@ +package instruction_pkg; + timeunit 1ns; timeprecision 1ps; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + import "DPI-C" function void instruction_expected( + input int unsigned bits, + output int unsigned expanded, + output byte unsigned illegal, + output longint unsigned immediate, + output int unsigned atomic, + function_code, + output byte unsigned word, + immediate_mode + ); + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/agent.svh" + `include "env/model.svh" + `include "env/coverage.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..aeda933df --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/seq/sequence.svh @@ -0,0 +1,40 @@ +class stimulus extends uvm_sequence #(request_item); + `uvm_object_utils(stimulus) + int unsigned sent = 0; + function new(string name = "stimulus"); + super.new(name); + endfunction + task body(); + request_item req = request_item::type_id::create("request"); + int funct7s[15] = '{0, 1, 4, 8, 9, 12, 16, 32, 33, 48, 64, 80, 96, 112, 127}; + int system_words[6] = '{'h00000073, 'h00100073, 'h30200073, 'h10200073, 'h10500073, 'h12000073}; + for (int bits = 0; bits < 65536; bits++) begin + if ((bits & 3) == 3) continue; + start_item(req); + req.bits = bits; + finish_item(req); + sent++; + if ((bits & 'hfc) == 0) begin + start_item(req); + req.bits = 'hffff0000 | bits; + finish_item(req); + sent++; + end + end + for (int opcode = 3; opcode < 128; opcode += 4) + for (int funct3 = 0; funct3 < 8; funct3++) + foreach (funct7s[index]) + for (int rd = 0; rd < 32; rd++) begin + start_item(req); + req.bits=(funct7s[index]<<25)|(((rd+13)&31)<<20)|(((rd+7)&31)<<15)|(funct3<<12)|(rd<<7)|opcode; + finish_item(req); + sent++; + end + foreach (system_words[index]) begin + start_item(req); + req.bits = system_words[index]; + finish_item(req); + sent++; + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/suite.toml b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/suite.toml new file mode 100644 index 000000000..c72329195 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/suite.toml @@ -0,0 +1,20 @@ +[model] +manifest = "src/csrc/Cargo.toml" +library = "libinstruction_reference" + +[[tests]] +name = "instruction" +top = "instruction_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 10000000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["cond", "tgl"] +functional_groups = ["instruction_pkg::coverage::behavior"] +sink_ready = 1 +sink_stall = 0 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/tb.sv b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/tb.sv new file mode 100644 index 000000000..26e28ebc1 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/tb.sv @@ -0,0 +1,36 @@ +module instruction_tb; + import uvm_pkg::*; + import instruction_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + clock_if timing (clock); + port_if bus ( + clock, + timing.reset + ); + Instruction dut ( + .clock(clock), + .reset(bus.reset), + .io_bits(bus.bits), + .io_expanded(bus.expanded), + .io_compressed(bus.compressed), + .io_illegal(bus.illegal), + .io_opcode(bus.opcode), + .io_funct3(bus.funct3), + .io_funct7(bus.funct7), + .io_rs1(bus.rs1), + .io_rs2(bus.rs2), + .io_rd(bus.rd), + .io_csrAddress(bus.csr_address), + .io_immediate(bus.immediate), + .io_atomicCode(bus.atomic), + .io_integerFunction(bus.function_code), + .io_integerWord(bus.word), + .io_integerImmediate(bus.immediate_mode) + ); + initial begin + uvm_config_db#(virtual clock_if)::set(null, "uvm_test_top*", "timing", timing); + uvm_config_db#(virtual port_if)::set(null, "uvm_test_top*", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/tests/test.svh new file mode 100644 index 000000000..f29ec05f4 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/tests/test.svh @@ -0,0 +1,24 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env bench; + reset_controller resetter; + virtual port_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + resetter = reset_controller::type_id::create("reset", this); + bench = env::type_id::create("env", this); + if (!uvm_config_db#(virtual port_if)::get(this, "", "vif", vif)) + `uvm_fatal("INTERFACE", "Test interface is required") + endfunction + task execute(); + stimulus sequence_ = stimulus::type_id::create("sequence"); + resetter.pulse(4); + sequence_.start(bench.bus.seqr); + bench.scoreboard.wait_checked(sequence_.sent); + @(negedge vif.clock); + evidence::check(bench.scoreboard.checked == sequence_.sent, "Comparison count mismatch"); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/main/scala/Emit.scala b/arch/src/main/scala/framework/core/execution/instruction/src/main/scala/Emit.scala new file mode 100644 index 000000000..967f33d54 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/main/scala/Emit.scala @@ -0,0 +1,15 @@ +package framework.core.execution.instruction + +import framework.core.execution.instruction.configs.Params +import java.nio.file.{Files, Paths} +import upickle.default.read + +object Emit extends App { + require(args.length >= 2 && args(0) == "--config") + val param = read[Params](Files.readString(Paths.get(args(1)))) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Instruction(param), + firtoolOpts = args.drop(2), + args = Array("--target-dir", "build/Instruction", "--split-verilog") + ) +} diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/main/scala/Instruction.scala b/arch/src/main/scala/framework/core/execution/instruction/src/main/scala/Instruction.scala new file mode 100644 index 000000000..943a0bc2a --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/main/scala/Instruction.scala @@ -0,0 +1,315 @@ +package framework.core.execution.instruction + +import chisel3._ +import chisel3.util._ +import framework.ip.bus.chi.rnf.CacheAtomic +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.core.execution.instruction.configs.{Params => ModuleParam} + +/** Expands RV64C instructions and decodes their fields and immediate operand. */ +class InstructionIO(b: ModuleParam) extends Bundle { + val p = b + + val bits = Input(UInt(32.W)) + val expanded = Output(UInt(32.W)) + val compressed = Output(Bool()) + val illegal = Output(Bool()) + val opcode = Output(UInt(7.W)) + val funct3 = Output(UInt(3.W)) + val funct7 = Output(UInt(7.W)) + val rs1 = Output(UInt(5.W)) + val rs2 = Output(UInt(5.W)) + val rd = Output(UInt(5.W)) + val csrAddress = Output(UInt(12.W)) + val immediate = Output(UInt(p.xLen.W)) + val atomicCode = Output(UInt(4.W)) + val integerFunction = Output(UInt(5.W)) + val integerWord = Output(Bool()) + val integerImmediate = Output(Bool()) + +} + +@instantiable +class Instruction(b: ModuleParam) extends Module { + val p = b + + @public + val io = IO(new InstructionIO(b)) + + val compressedIllegal = WireDefault(false.B) + val c = io.bits(15, 0) + val rd = c(11, 7) + val rs2 = c(6, 2) + val compactRd = Cat(1.U(2.W), c(4, 2)) + val compactRs1 = Cat(1.U(2.W), c(9, 7)) + val signedImmediate = Cat(Fill(7, c(12)), c(6, 2)) + val loadWordOffset = Cat(0.U(5.W), c(5), c(12, 10), c(6), 0.U(2.W)) + val loadDoubleOffset = Cat(0.U(4.W), c(6, 5), c(12, 10), 0.U(3.W)) + val jumpOffset = Cat(Fill(10, c(12)), c(12), c(8), c(10, 9), c(6), c(7), c(2), c(11), c(5, 3), 0.U(1.W)) + val branchOffset = Cat(Fill(5, c(12)), c(6, 5), c(2), c(11, 10), c(4, 3), 0.U(1.W)) + val addSpOffset = Cat(Fill(3, c(12)), c(4, 3), c(5), c(2), c(6), 0.U(4.W)) + val addSpUnsigned = Cat(0.U(2.W), c(10, 7), c(12, 11), c(5), c(6), 0.U(2.W)) + val stackWordLoad = Cat(0.U(4.W), c(3, 2), c(12), c(6, 4), 0.U(2.W)) + val stackDoubleLoad = Cat(0.U(3.W), c(4, 2), c(12), c(6, 5), 0.U(3.W)) + val stackWordStore = Cat(0.U(4.W), c(8, 7), c(12, 9), 0.U(2.W)) + val stackDoubleStore = Cat(0.U(3.W), c(9, 7), c(12, 10), 0.U(3.W)) + + io.compressed := c(1, 0) =/= 3.U + io.expanded := io.bits + compressedIllegal := false.B + when(io.compressed) { + io.expanded := "h00000013".U + compressedIllegal := true.B + switch(c(1, 0)) { + is(0.U) { + switch(c(15, 13)) { + is(0.U) { // C.ADDI4SPN + io.expanded := Cat(addSpUnsigned, 2.U(5.W), 0.U(3.W), compactRd, "h13".U(7.W)) + compressedIllegal := !addSpUnsigned.orR + } + is(1.U, 2.U, 3.U) { // C.FLD, C.LW, C.LD + val word = c(15, 13) === 2.U + io.expanded := Cat( + Mux(word, loadWordOffset, loadDoubleOffset), + compactRs1, + Mux(word, 2.U(3.W), 3.U(3.W)), + compactRd, + Mux(c(15, 13) === 1.U, "h07".U(7.W), "h03".U(7.W)) + ) + compressedIllegal := false.B + } + is(5.U, 6.U, 7.U) { // C.FSD, C.SW, C.SD + val word = c(15, 13) === 6.U + val offset = Mux(word, loadWordOffset, loadDoubleOffset) + io.expanded := Cat( + offset(11, 5), + compactRd, + compactRs1, + Mux(word, 2.U(3.W), 3.U(3.W)), + offset(4, 0), + Mux(c(15, 13) === 5.U, "h27".U(7.W), "h23".U(7.W)) + ) + compressedIllegal := false.B + } + } + } + is(1.U) { + switch(c(15, 13)) { + is(0.U, 1.U) { // C.ADDI/NOP and C.ADDIW + io.expanded := Cat(signedImmediate, rd, 0.U(3.W), rd, Mux(c(13), "h1b".U(7.W), "h13".U(7.W))) + compressedIllegal := c(13) && rd === 0.U + } + is(2.U) { // C.LI, including hints + io.expanded := Cat(signedImmediate, 0.U(5.W), 0.U(3.W), rd, "h13".U(7.W)) + compressedIllegal := false.B + } + is(3.U) { // C.ADDI16SP / C.LUI + io.expanded := Mux( + rd === 2.U, + Cat(addSpOffset, 2.U(5.W), 0.U(3.W), 2.U(5.W), "h13".U(7.W)), + Cat(Fill(15, c(12)), c(6, 2), rd, "h37".U(7.W)) + ) + compressedIllegal := !Cat(c(12), c(6, 2)).orR + } + is(4.U) { + compressedIllegal := false.B + switch(c(11, 10)) { + is(0.U, 1.U) { // C.SRLI / C.SRAI + io.expanded := Cat( + 0.U(1.W), + c(10), + 0.U(4.W), + c(12), + c(6, 2), + compactRs1, + 5.U(3.W), + compactRs1, + "h13".U(7.W) + ) + } + is(2.U) { // C.ANDI + io.expanded := Cat(signedImmediate, compactRs1, 7.U(3.W), compactRs1, "h13".U(7.W)) + } + is(3.U) { + val sub = c(6, 5) === 0.U + val funct = MuxLookup(c(6, 5), 0.U(3.W))(Seq(0.U -> 0.U, 1.U -> 4.U, 2.U -> 6.U, 3.U -> 7.U)) + io.expanded := Cat( + Mux(sub, 32.U(7.W), 0.U(7.W)), + compactRd, + compactRs1, + Mux(c(12), 0.U(3.W), funct), + compactRs1, + Mux(c(12), "h3b".U(7.W), "h33".U(7.W)) + ) + compressedIllegal := c(12) && c(6, 5) >= 2.U + } + } + } + is(5.U) { // C.J + io.expanded := Cat( + jumpOffset(20), + jumpOffset(10, 1), + jumpOffset(11), + jumpOffset(19, 12), + 0.U(5.W), + "h6f".U(7.W) + ) + compressedIllegal := false.B + } + is(6.U, 7.U) { // C.BEQZ / C.BNEZ + io.expanded := Cat( + branchOffset(12), + branchOffset(10, 5), + 0.U(5.W), + compactRs1, + Cat(0.U(2.W), c(13)), + branchOffset(4, 1), + branchOffset(11), + "h63".U(7.W) + ) + compressedIllegal := false.B + } + } + } + is(2.U) { + switch(c(15, 13)) { + is(0.U) { // C.SLLI, including hints + io.expanded := Cat(0.U(6.W), c(12), c(6, 2), rd, 1.U(3.W), rd, "h13".U(7.W)) + compressedIllegal := false.B + } + is(1.U, 2.U, 3.U) { // C.FLDSP / C.LWSP / C.LDSP + val word = c(15, 13) === 2.U + io.expanded := Cat( + Mux(word, stackWordLoad, stackDoubleLoad), + 2.U(5.W), + Mux(word, 2.U(3.W), 3.U(3.W)), + rd, + Mux(c(15, 13) === 1.U, "h07".U(7.W), "h03".U(7.W)) + ) + compressedIllegal := c(15, 13) =/= 1.U && rd === 0.U + } + is(4.U) { + compressedIllegal := false.B + when(rs2 =/= 0.U) { // C.MV / C.ADD, including hints + io.expanded := Cat(0.U(7.W), rs2, Mux(c(12), rd, 0.U(5.W)), 0.U(3.W), rd, "h33".U(7.W)) + }.elsewhen(c(12) && rd === 0.U) { + io.expanded := "h00100073".U // C.EBREAK + }.otherwise { // C.JR / C.JALR + io.expanded := Cat(0.U(12.W), rd, 0.U(3.W), Mux(c(12), 1.U(5.W), 0.U(5.W)), "h67".U(7.W)) + compressedIllegal := rd === 0.U + } + } + is(5.U, 6.U, 7.U) { // C.FSDSP / C.SWSP / C.SDSP + val word = c(15, 13) === 6.U + val offset = Mux(word, stackWordStore, stackDoubleStore) + io.expanded := Cat( + offset(11, 5), + rs2, + 2.U(5.W), + Mux(word, 2.U(3.W), 3.U(3.W)), + offset(4, 0), + Mux(c(15, 13) === 5.U, "h27".U(7.W), "h23".U(7.W)) + ) + compressedIllegal := false.B + } + } + } + } + } + + val insn = io.expanded + io.opcode := insn(6, 0) + io.funct3 := insn(14, 12) + io.funct7 := insn(31, 25) + io.rs1 := insn(19, 15) + io.rs2 := insn(24, 20) + io.rd := insn(11, 7) + io.csrAddress := insn(31, 20) + + val immediateI = Cat(Fill(p.xLen - 12, insn(31)), insn(31, 20)) + val immediateS = Cat(Fill(p.xLen - 12, insn(31)), insn(31, 25), insn(11, 7)) + val immediateB = Cat(Fill(p.xLen - 12, insn(31)), insn(7), insn(30, 25), insn(11, 8), 0.U(1.W)) + val immediateJ = Cat(Fill(p.xLen - 20, insn(31)), insn(19, 12), insn(20), insn(30, 21), 0.U(1.W)) + val immediateU = Cat(Fill(p.xLen - 32, insn(31)), insn(31, 12), 0.U(12.W)) + io.immediate := MuxLookup(io.opcode, immediateI)(Seq( + "h23".U -> immediateS, + "h27".U -> immediateS, + "h63".U -> immediateB, + "h6f".U -> immediateJ, + "h17".U -> immediateU, + "h37".U -> immediateU + )) + + io.atomicCode := MuxLookup(io.funct7(6, 2), CacheAtomic.None.U(4.W))(Seq( + 0.U -> CacheAtomic.Add.U, + 1.U -> CacheAtomic.Swap.U, + 2.U -> CacheAtomic.LR.U, + 3.U -> CacheAtomic.SC.U, + 4.U -> CacheAtomic.Xor.U, + 8.U -> CacheAtomic.Or.U, + 12.U -> CacheAtomic.And.U, + 16.U -> CacheAtomic.Min.U, + 20.U -> CacheAtomic.Max.U, + 24.U -> CacheAtomic.MinU.U, + 28.U -> CacheAtomic.MaxU.U + )) + io.integerWord := io.opcode === "h1b".U || io.opcode === "h3b".U + io.integerImmediate := io.opcode === "h13".U || io.opcode === "h1b".U + val subtract = (io.opcode === "h33".U || io.opcode === "h3b".U) && io.funct7(5) && io.funct3 === 0.U + io.integerFunction := MuxLookup(io.funct3, 0.U(5.W))(Seq( + 0.U -> Mux(subtract, 10.U, 0.U), + 1.U -> 1.U, + 2.U -> 12.U, + 3.U -> 14.U, + 4.U -> 4.U, + 5.U -> Mux(io.funct7(5), 11.U, 5.U), + 6.U -> 6.U, + 7.U -> 7.U + )) + + val shift = io.funct3 === 1.U || io.funct3 === 5.U + + val immediateLegal = Mux(io.integerWord, io.funct3 === 0.U || shift, true.B) && + (!shift || Mux(io.integerWord, io.funct7 === 0.U, io.funct7(6, 1) === 0.U) || + (io.funct3 === 5.U && Mux(io.integerWord, io.funct7 === 32.U, io.funct7(6, 1) === 16.U))) + + val registerLegal = Mux( + io.funct7 === 1.U, + !io.integerWord || io.funct3 === 0.U || io.funct3 >= 4.U, + (io.funct7 === 0.U || (io.funct7 === 32.U && (io.funct3 === 0.U || io.funct3 === 5.U))) && + (!io.integerWord || io.funct3 === 0.U || io.funct3 === 1.U || io.funct3 === 5.U) + ) + + val systemLegal = Seq("h00000073", "h00100073", "h30200073", "h10200073", "h10500073") + .map(code => insn === code.U(32.W)).reduce(_ || _) || (io.funct7 === "h09".U && io.rd === 0.U) + + val encodingLegal = MuxLookup(io.opcode, false.B)(Seq( + "h37".U -> true.B, + "h17".U -> true.B, + "h6f".U -> true.B, + "h67".U -> (io.funct3 === 0.U), + "h63".U -> (io.funct3 =/= 2.U && io.funct3 =/= 3.U), + "h13".U -> immediateLegal, + "h1b".U -> immediateLegal, + "h33".U -> registerLegal, + "h3b".U -> registerLegal, + "h03".U -> (io.funct3 =/= 7.U), + "h23".U -> (io.funct3 <= 3.U), + "h07".U -> (io.funct3 === 2.U || io.funct3 === 3.U), + "h27".U -> (io.funct3 === 2.U || io.funct3 === 3.U), + "h2f".U -> ((io.funct3 === 2.U || io.funct3 === 3.U) && io.atomicCode =/= CacheAtomic.None.U && + (io.funct7(6, 2) =/= 2.U || io.rs2 === 0.U)), + "h43".U -> true.B, + "h47".U -> true.B, + "h4b".U -> true.B, + "h4f".U -> true.B, + "h53".U -> true.B, + "h0f".U -> (io.funct3 === 0.U || io.funct3 === 1.U), + "h73".U -> Mux(io.funct3 =/= 0.U, io.funct3 =/= 4.U, systemLegal), + "h2b".U -> (io.funct3 === 7.U), + "h7b".U -> true.B + )) + + io.illegal := compressedIllegal || !encodingLegal + +} diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/core/execution/instruction/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..ceca7b03d --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/main/scala/configs/Params.scala @@ -0,0 +1,6 @@ +package framework.core.execution.instruction.configs + +import upickle.default.{macroRW, ReadWriter} + +case class Params(xLen: Int) { require(xLen == 64) } +object Params { implicit val rw: ReadWriter[Params] = macroRW } diff --git a/arch/src/main/scala/framework/core/execution/integer/LICENSE.Berkeley b/arch/src/main/scala/framework/core/execution/integer/LICENSE.Berkeley new file mode 100644 index 000000000..7cff15e40 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/LICENSE.Berkeley @@ -0,0 +1,24 @@ +Copyright (c) 2012-2014, The Regents of the University of California +(Regents). All Rights Reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. Neither the name of the Regents nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, +SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING +OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS +BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED +HEREUNDER IS PROVIDED "AS IS". REGENTS HAS NO OBLIGATION TO PROVIDE +MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. diff --git a/arch/src/main/scala/framework/core/execution/integer/LICENSE.SiFive b/arch/src/main/scala/framework/core/execution/integer/LICENSE.SiFive new file mode 100644 index 000000000..7e7093375 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/LICENSE.SiFive @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2016-2017 SiFive, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/arch/src/main/scala/framework/core/execution/integer/build.sc b/arch/src/main/scala/framework/core/execution/integer/build.sc new file mode 100644 index 000000000..4647dd8d9 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/build.sc @@ -0,0 +1,21 @@ +import mill._ +import $file.^.^.^.^.^.^.^.settings + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.core.execution.integer.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Integer.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/core/execution/integer/configs/uvm.toml b/arch/src/main/scala/framework/core/execution/integer/configs/uvm.toml new file mode 100644 index 000000000..3da02cd69 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/configs/uvm.toml @@ -0,0 +1,4 @@ +[parameters] +xLen = 64 + +[defines] diff --git a/arch/src/main/scala/framework/core/execution/integer/src/main/resources/agent/agent.svh b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/agent/agent.svh new file mode 100644 index 000000000..6d2a96092 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/agent/agent.svh @@ -0,0 +1,19 @@ +class agent extends uvm_agent; + `uvm_component_utils(agent) + uvm_sequencer #(request_item) seqr; + driver drv; + monitor mon; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + seqr = uvm_sequencer#(request_item)::type_id::create("seqr", this); + drv = driver::type_id::create("driver", this); + mon = monitor::type_id::create("monitor", this); + endfunction + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + drv.seq_item_port.connect(seqr.seq_item_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/integer/src/main/resources/agent/driver.svh b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/agent/driver.svh new file mode 100644 index 000000000..7bf3af2be --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/agent/driver.svh @@ -0,0 +1,33 @@ +class driver extends uvm_driver #(request_item); + `uvm_component_utils(driver) + virtual port_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual port_if)::get(this, "", "vif", vif)) + `uvm_fatal("INTERFACE", "Driver interface is required") + endfunction + task run_phase(uvm_phase phase); + request_item req; + vif.sample = 0; + vif.a = 0; + vif.rhs = 0; + vif.fn = 0; + vif.dw = 0; + forever begin + seq_item_port.get_next_item(req); + do @(negedge vif.clock); while (vif.reset); + vif.a = req.a; + vif.rhs = req.rhs; + vif.fn = req.fn; + vif.dw = req.dw; + vif.sample = 1; + @(posedge vif.clock); + seq_item_port.item_done(); + @(negedge vif.clock); + vif.sample = 0; + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/integer/src/main/resources/agent/item.svh b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/agent/item.svh new file mode 100644 index 000000000..4fdc50670 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/agent/item.svh @@ -0,0 +1,29 @@ +class request_item extends uvm_sequence_item; + logic [63:0] a; + logic [63:0] rhs; + logic [4:0] fn; + logic dw; + `uvm_object_utils_begin(request_item) + `uvm_field_int(a, UVM_ALL_ON) + `uvm_field_int(rhs, UVM_ALL_ON) + `uvm_field_int(fn, UVM_ALL_ON) + `uvm_field_int(dw, UVM_ALL_ON) + `uvm_object_utils_end + function new(string name = "request_item"); + super.new(name); + endfunction +endclass + +class response_item extends uvm_sequence_item; + logic [63:0] value; + logic [63:0] adder; + logic comparison; + `uvm_object_utils_begin(response_item) + `uvm_field_int(value, UVM_ALL_ON) + `uvm_field_int(adder, UVM_ALL_ON) + `uvm_field_int(comparison, UVM_ALL_ON) + `uvm_object_utils_end + function new(string name = "response_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/integer/src/main/resources/agent/monitor.svh b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/agent/monitor.svh new file mode 100644 index 000000000..89371d505 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/agent/monitor.svh @@ -0,0 +1,36 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual port_if vif; + uvm_analysis_port #(request_item) requests; + uvm_analysis_port #(response_item) responses; + function new(string name, uvm_component parent); + super.new(name, parent); + requests = new("requests", this); + responses = new("responses", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual port_if)::get(this, "", "vif", vif)) + `uvm_fatal("INTERFACE", "Monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + request_item req; + response_item rsp; + forever begin + @(posedge vif.clock); + if (!vif.reset && vif.sample) begin + req = request_item::type_id::create("request"); + rsp = response_item::type_id::create("response"); + req.a = vif.a; + req.rhs = vif.rhs; + req.fn = vif.fn; + req.dw = vif.dw; + rsp.value = vif.value; + rsp.adder = vif.adder; + rsp.comparison = vif.comparison; + requests.write(req); + responses.write(rsp); + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/integer/src/main/resources/env/coverage.svh b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/env/coverage.svh new file mode 100644 index 000000000..f6f8178f8 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/env/coverage.svh @@ -0,0 +1,30 @@ +class coverage extends uvm_subscriber #(request_item); + `uvm_component_utils(coverage) + covergroup behavior with function sample (request_item t); + option.per_instance = 1; + operation: coverpoint t.fn { + bins arithmetic[] = {0, 10}; + bins shift[] = {1, 5, 11, 17, 18, 19}; + bins logic_ops[] = {4, 6, 7, 8, 9, 24, 25, 26}; + bins compare_ops[] = {12, 13, 14, 15, 28, 29, 30, 31}; + bins unary = {16}; + } + width: coverpoint t.dw; + unary: coverpoint t.rhs[11:0] iff (t.fn == 16) { + bins selectors[] = {0, 1, 2, 'h287, 'h6b8, 'h080, 'h604, 'h605}; + } + shift_amount: coverpoint (t.rhs & (t.dw ? 63 : 31)) iff (t.fn inside {1, 5, 11, 17, 18, 19}) { + bins boundaries[] = {0, 1, 31, 32, 63}; + } + sign_a: coverpoint t.a[63]; + sign_rhs: coverpoint t.rhs[63]; + operand_signs: cross sign_a, sign_rhs; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + behavior = new(); + endfunction + function void write(request_item t); + behavior.sample(t); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/integer/src/main/resources/env/env.svh b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/env/env.svh new file mode 100644 index 000000000..5a8082f7a --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/env/env.svh @@ -0,0 +1,19 @@ +class env extends environment #(request_item, response_item, model); + `uvm_component_utils(env) + agent bus; + coverage cov; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + cov = coverage::type_id::create("coverage", this); + bus = agent::type_id::create("agent", this); + endfunction + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + bus.mon.requests.connect(input_export); + bus.mon.requests.connect(cov.analysis_export); + bus.mon.responses.connect(output_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/integer/src/main/resources/env/model.svh b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/env/model.svh new file mode 100644 index 000000000..92e9996ae --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/env/model.svh @@ -0,0 +1,65 @@ +class model extends reference_model #(request_item, response_item); + `uvm_component_utils(model) + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void write(request_item req); + response_item rsp = response_item::type_id::create("expected"); + logic [63:0] value; + int width = req.dw ? 64 : 32; + int shift = req.rhs & (width - 1); + logic [63:0] operand = req.dw ? req.a : {32'b0, req.a[31:0]}; + int count; + rsp.adder = req.fn[3] ? req.a - req.rhs : req.a + req.rhs; + rsp.comparison = req.fn[0] ^ (!req.fn[3] ? req.a == req.rhs : + req.fn[1] ? req.a < req.rhs : $signed(req.a) < $signed(req.rhs)); + case (req.fn) + 0: value = req.a + req.rhs; + 10: value = req.a - req.rhs; + 1: value = operand << shift; + 5: value = operand >> shift; + 11: value = req.dw ? $signed(req.a) >>> shift : $signed(req.a[31:0]) >>> shift; + 4: value = req.a ^ req.rhs; + 6: value = req.a | req.rhs; + 7: value = req.a & req.rhs; + 8: value = req.rhs != 0 ? req.a : 0; + 9: value = req.rhs == 0 ? req.a : 0; + 12, 13: value = $signed(req.a) < $signed(req.rhs); + 14, 15: value = req.a < req.rhs; + 17: value = (operand << shift) | (operand >> (width - shift)); + 18: value = (operand >> shift) | (operand << (width - shift)); + 19: value = (operand >> shift) & 1; + 24: value = req.a & ~req.rhs; + 25: value = req.a | ~req.rhs; + 26: value = req.a ^ ~req.rhs; + 28: value = $signed(req.a) > $signed(req.rhs) ? req.a : req.rhs; + 29: value = $signed(req.a) < $signed(req.rhs) ? req.a : req.rhs; + 30: value = req.a > req.rhs ? req.a : req.rhs; + 31: value = req.a < req.rhs ? req.a : req.rhs; + 16: begin + value = 0; + case (req.rhs[11:0]) + 12'h287: for (int i = 0; i < 8; i++) value[i*8+:8] = |req.a[i*8+:8] ? 8'hff : 0; + 12'h6b8: for (int i = 0; i < 8; i++) value[i*8+:8] = req.a[(7-i)*8+:8]; + 12'h080: value = {48'b0, req.a[15:0]}; + 12'h604: value = {{56{req.a[7]}}, req.a[7:0]}; + 12'h605: value = {{48{req.a[15]}}, req.a[15:0]}; + 0, 1, 2: begin + count = 0; + if (req.rhs == 2) begin + for (int i = 0; i < width; i++) count += operand[i]; + end else begin + while (count < width && !operand[req.rhs==0?width-1-count : count]) count++; + end + value = count; + end + default: `uvm_fatal("OPERATION", "Unknown unary instruction") + endcase + end + default: `uvm_fatal("OPERATION", "Unknown integer operation") + endcase + rsp.value = req.dw ? value : {{32{value[31]}}, value[31:0]}; + expected_ap.write(rsp); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/integer/src/main/resources/filelist.f b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/filelist.f new file mode 100644 index 000000000..1f4d31b52 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/filelist.f @@ -0,0 +1,7 @@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv ++incdir+@RESOURCES@ +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/Integer/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/execution/integer/src/main/resources/interface.sv b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/interface.sv new file mode 100644 index 000000000..10b689346 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/interface.sv @@ -0,0 +1,13 @@ +interface port_if ( + input logic clock, + input logic reset +); + logic sample; + logic [63:0] a; + logic [63:0] rhs; + logic [4:0] fn; + logic dw; + logic [63:0] value; + logic [63:0] adder; + logic comparison; +endinterface diff --git a/arch/src/main/scala/framework/core/execution/integer/src/main/resources/package.sv b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/package.sv new file mode 100644 index 000000000..e21dddad9 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/package.sv @@ -0,0 +1,15 @@ +package integer_pkg; + timeunit 1ns; timeprecision 1ps; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/agent.svh" + `include "env/model.svh" + `include "env/coverage.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/execution/integer/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..b400a82e9 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/seq/sequence.svh @@ -0,0 +1,66 @@ +class stimulus extends uvm_sequence #(request_item); + `uvm_object_utils(stimulus) + int unsigned sent = 0; + reset_controller resetter; + function new(string name = "stimulus"); + super.new(name); + endfunction + task body(); + request_item req = request_item::type_id::create("request"); + logic [63:0] operands[8] = '{0, 1, 31, 32, 63, '1, 64'h8000000000000000, 64'h55555555aaaaaaaa}; + int operations[24] = '{ + 0, + 10, + 1, + 5, + 11, + 4, + 6, + 7, + 8, + 9, + 12, + 13, + 14, + 15, + 17, + 18, + 19, + 24, + 25, + 26, + 28, + 29, + 30, + 31 + }; + int unary[8] = '{0, 1, 2, 'h287, 'h6b8, 'h080, 'h604, 'h605}; + for (int word = 0; word < 2; word++) begin + foreach (operations[op]) begin + if (word && !(operations[op] inside {0, 10, 1, 5, 11, 17, 18})) continue; + foreach (operands[a]) + foreach (operands[b]) begin + start_item(req); + req.a = operands[a]; + req.rhs = operands[b]; + req.fn = operations[op]; + req.dw = !word; + finish_item(req); + sent++; + end + end + foreach (unary[op]) begin + if (word && op >= 3) continue; + foreach (operands[a]) begin + start_item(req); + req.a = operands[a]; + req.rhs = unary[op]; + req.fn = 16; + req.dw = !word; + finish_item(req); + sent++; + end + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/integer/src/main/resources/suite.toml b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/suite.toml new file mode 100644 index 000000000..1fd36e0df --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/suite.toml @@ -0,0 +1,20 @@ +[model] +manifest = "" +library = "" + +[[tests]] +functional_groups = ["integer_pkg::coverage::behavior"] +name = "integer" +top = "integer_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 1000000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["cond", "tgl"] +sink_ready = 1 +sink_stall = 0 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/core/execution/integer/src/main/resources/tb.sv b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/tb.sv new file mode 100644 index 000000000..f6df99158 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/tb.sv @@ -0,0 +1,27 @@ +module integer_tb; + import uvm_pkg::*; + import integer_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + clock_if timing (clock); + port_if bus ( + clock, + timing.reset + ); + Integer dut ( + .clock(clock), + .reset(bus.reset), + .io_in1(bus.a), + .io_in2(bus.rhs), + .io_fn(bus.fn), + .io_dw(bus.dw), + .io_out(bus.value), + .io_adder_out(bus.adder), + .io_cmp_out(bus.comparison) + ); + initial begin + uvm_config_db#(virtual clock_if)::set(null, "uvm_test_top*", "timing", timing); + uvm_config_db#(virtual port_if)::set(null, "uvm_test_top*", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/core/execution/integer/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/tests/test.svh new file mode 100644 index 000000000..d5f2d6ed9 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/tests/test.svh @@ -0,0 +1,25 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env bench; + reset_controller resetter; + virtual port_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + resetter = reset_controller::type_id::create("reset", this); + bench = env::type_id::create("env", this); + if (!uvm_config_db#(virtual port_if)::get(this, "", "vif", vif)) + `uvm_fatal("INTERFACE", "Test interface is required") + endfunction + task execute(); + stimulus sequence_ = stimulus::type_id::create("sequence"); + resetter.pulse(4); + sequence_.resetter = resetter; + sequence_.start(bench.bus.seqr); + bench.scoreboard.wait_checked(sequence_.sent); + @(negedge vif.clock); + evidence::check(bench.scoreboard.checked == sequence_.sent, "Comparison count mismatch"); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/integer/src/main/scala/Emit.scala b/arch/src/main/scala/framework/core/execution/integer/src/main/scala/Emit.scala new file mode 100644 index 000000000..c77d7b22b --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/src/main/scala/Emit.scala @@ -0,0 +1,15 @@ +package framework.core.execution.integer + +import framework.core.execution.integer.configs.Params +import java.nio.file.{Files, Paths} +import upickle.default.read + +object Emit extends App { + require(args.length >= 2 && args(0) == "--config") + val param = read[Params](Files.readString(Paths.get(args(1)))) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Integer(param), + firtoolOpts = args.drop(2), + args = Array("--target-dir", "build/Integer", "--split-verilog") + ) +} diff --git a/arch/src/main/scala/framework/core/execution/integer/src/main/scala/Integer.scala b/arch/src/main/scala/framework/core/execution/integer/src/main/scala/Integer.scala new file mode 100644 index 000000000..2d02fda4b --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/src/main/scala/Integer.scala @@ -0,0 +1,93 @@ +// See LICENSE.Berkeley and LICENSE.SiFive for license details. +package framework.core.execution.integer + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.core.execution.integer.configs.{Params => ModuleParam} + +class IntegerIO(b: ModuleParam) extends Bundle { + val p = b + + val in1 = Input(UInt(p.xLen.W)) + val in2 = Input(UInt(p.xLen.W)) + val fn = Input(UInt(5.W)) + val dw = Input(Bool()) + val out = Output(UInt(p.xLen.W)) + val adder_out = Output(UInt(p.xLen.W)) + val cmp_out = Output(Bool()) + +} + +@instantiable +class Integer(b: ModuleParam) extends Module { + val p = b + + @public + val io = IO(new IntegerIO(b)) + + val a = io.in1 + val rhs = io.in2 + val fn = io.fn + val subtract = fn(3) + val inverted = Mux(subtract, ~rhs, rhs) + val difference = a ^ inverted + val conjunction = a & inverted + val sum = a + inverted + subtract + val less = Mux(a(63) === rhs(63), sum(63), Mux(fn(1), rhs(63), a(63))) + io.adder_out := sum + io.cmp_out := fn(0) ^ Mux(!subtract, difference === 0.U, less) + val shift = Cat(rhs(5) && io.dw, rhs(4, 0)) + val high = Mux(io.dw, a(63, 32), Fill(32, subtract && a(31))) + val input = Cat(high, a(31, 0)) + val reverse = !(fn === 5.U || fn === 11.U || fn === 18.U || fn === 19.U) + val shiftedInput = Mux(reverse, Reverse(input), input) + val right = (Cat(subtract && shiftedInput(63), shiftedInput).asSInt >> shift)(63, 0) + val left = Reverse(right) + val shifts = Mux(fn === 5.U || fn === 11.U || fn === 19.U, right, 0.U) | Mux(fn === 1.U, left, 0.U) + val logic = Mux(fn === 4.U || fn === 6.U || fn === 25.U || fn === 26.U, difference, 0.U) | + Mux(fn === 6.U || fn === 7.U || fn === 25.U || fn === 24.U, conjunction, 0.U) + val comparison = (fn >= 12.U && fn <= 15.U) && less + val conditional = Mux((fn === 8.U && rhs.orR) || (fn === 9.U && !rhs.orR), a, 0.U) + val shiftLogic = comparison.asUInt | logic | (shifts & Mux(fn === 19.U, 1.U, ~0.U(64.W))) | conditional + + val countInput = MuxLookup(Cat(!io.dw, !rhs(0)), 0.U)(Seq( + 0.U -> a, + 1.U -> Reverse(a), + 2.U -> Cat(1.U(1.W), a(31, 0)), + 3.U -> Cat(1.U(1.W), Reverse(a(31, 0))) + )) + + val populationInput = Mux(rhs(1), Mux(io.dw, a, a(31, 0)), PriorityEncoderOH(Cat(1.U(1.W), countInput)) - 1.U)(63, 0) + val count = PopCount(populationInput) + val bytes = a.asTypeOf(Vec(8, UInt(8.W))) + + val unary = MuxLookup(rhs(11, 0), count)(Seq( + "h287".U -> VecInit(bytes.map(byte => Fill(8, byte.orR))).asUInt, + "h6b8".U -> VecInit(bytes.reverse).asUInt, + "h080".U -> a(15, 0), + "h604".U -> Cat(Fill(56, a(7)), a(7, 0)), + "h605".U -> Cat(Fill(48, a(15)), a(15, 0)) + )) + + val rotationShift = Mux(io.dw, 64.U, 32.U) - shift + val rotationInput = Mux(fn(0), input, Reverse(input)) + val rotateRight = (rotationInput >> rotationShift)(63, 0) + val rotateLeft = Reverse(rotateRight) + val rotation = Mux(fn(0), rotateRight, rotateLeft) | Mux(fn(0), left, right) + val minimumMaximum = Mux(io.cmp_out, rhs, a) + + val result = MuxLookup(fn, shiftLogic)(Seq( + 0.U -> sum, + 10.U -> sum, + 16.U -> unary, + 28.U -> minimumMaximum, + 29.U -> minimumMaximum, + 30.U -> minimumMaximum, + 31.U -> minimumMaximum, + 17.U -> rotation, + 18.U -> rotation + )) + + io.out := Mux(io.dw, result, Cat(Fill(32, result(31)), result(31, 0))) +} diff --git a/arch/src/main/scala/framework/core/execution/integer/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/core/execution/integer/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..aabdb1464 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/src/main/scala/configs/Params.scala @@ -0,0 +1,6 @@ +package framework.core.execution.integer.configs + +import upickle.default.{macroRW, ReadWriter} + +case class Params(xLen: Int) { require(xLen == 64) } +object Params { implicit val rw: ReadWriter[Params] = macroRW } diff --git a/arch/src/main/scala/framework/core/execution/muldiv/build.sc b/arch/src/main/scala/framework/core/execution/muldiv/build.sc new file mode 100644 index 000000000..754f56a30 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/muldiv/build.sc @@ -0,0 +1,21 @@ +import mill._ +import $file.^.^.^.^.^.^.^.settings + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.core.execution.muldiv.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/MulDiv.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/core/execution/muldiv/configs/uvm.toml b/arch/src/main/scala/framework/core/execution/muldiv/configs/uvm.toml new file mode 100644 index 000000000..3da02cd69 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/muldiv/configs/uvm.toml @@ -0,0 +1,4 @@ +[parameters] +xLen = 64 + +[defines] diff --git a/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/agent/agent.svh b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/agent/agent.svh new file mode 100644 index 000000000..6d2a96092 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/agent/agent.svh @@ -0,0 +1,19 @@ +class agent extends uvm_agent; + `uvm_component_utils(agent) + uvm_sequencer #(request_item) seqr; + driver drv; + monitor mon; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + seqr = uvm_sequencer#(request_item)::type_id::create("seqr", this); + drv = driver::type_id::create("driver", this); + mon = monitor::type_id::create("monitor", this); + endfunction + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + drv.seq_item_port.connect(seqr.seq_item_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/agent/driver.svh b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/agent/driver.svh new file mode 100644 index 000000000..885c3e2b9 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/agent/driver.svh @@ -0,0 +1,51 @@ +class driver extends uvm_driver #(request_item); + `uvm_component_utils(driver) + virtual port_if vif; + reset_controller resetter; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual port_if)::get( + this, "", "vif", vif + ) || !uvm_config_db#(reset_controller)::get( + this, "", "resetter", resetter + )) + `uvm_fatal("INTERFACE", "MulDiv driver requires ports and reset controller") + endfunction + task run_phase(uvm_phase phase); + request_item req; + vif.request_valid = 0; + vif.response_ready = 0; + vif.a = 0; + vif.rhs = 0; + vif.operation = 0; + vif.word = 0; + forever begin + seq_item_port.get_next_item(req); + do @(negedge vif.clock); while (vif.reset); + vif.a = req.a; + vif.rhs = req.rhs; + vif.operation = req.operation; + vif.word = req.word; + vif.stall = req.stall; + vif.cancel = req.cancel; + vif.request_valid = 1; + do @(posedge vif.clock); while (!vif.request_ready); + @(negedge vif.clock); + vif.request_valid = 0; + if (req.cancel) begin + resetter.pulse(2); + end else begin + while (!vif.response_valid) @(negedge vif.clock); + repeat (req.stall) @(negedge vif.clock); + vif.response_ready = 1; + @(posedge vif.clock); + @(negedge vif.clock); + vif.response_ready = 0; + end + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/agent/item.svh b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/agent/item.svh new file mode 100644 index 000000000..4184086de --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/agent/item.svh @@ -0,0 +1,29 @@ +class request_item extends uvm_sequence_item; + logic [63:0] a; + logic [63:0] rhs; + logic [2:0] operation; + logic word; + int unsigned stall; + logic cancel; + `uvm_object_utils_begin(request_item) + `uvm_field_int(a, UVM_ALL_ON) + `uvm_field_int(rhs, UVM_ALL_ON) + `uvm_field_int(operation, UVM_ALL_ON) + `uvm_field_int(word, UVM_ALL_ON) + `uvm_field_int(stall, UVM_ALL_ON) + `uvm_field_int(cancel, UVM_ALL_ON) + `uvm_object_utils_end + function new(string name = "request_item"); + super.new(name); + endfunction +endclass + +class response_item extends uvm_sequence_item; + logic [63:0] value; + `uvm_object_utils_begin(response_item) + `uvm_field_int(value, UVM_ALL_ON) + `uvm_object_utils_end + function new(string name = "response_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/agent/monitor.svh b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/agent/monitor.svh new file mode 100644 index 000000000..1febc4201 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/agent/monitor.svh @@ -0,0 +1,52 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual port_if vif; + uvm_analysis_port #(request_item) requests; + uvm_analysis_port #(response_item) responses; + uvm_analysis_port #(bit) reset_ap; + function new(string name, uvm_component parent); + super.new(name, parent); + requests = new("requests", this); + responses = new("responses", this); + reset_ap = new("reset_ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual port_if)::get(this, "", "vif", vif)) + `uvm_fatal("INTERFACE", "MulDiv monitor requires ports") + endfunction + task run_phase(uvm_phase phase); + bit pending = 0; + request_item req; + response_item rsp; + forever begin + @(posedge vif.clock); + if (vif.reset) begin + pending = 0; + reset_ap.write(1); + end else begin + if (vif.request_valid && vif.request_ready) begin + if (pending) `uvm_fatal("OUTSTANDING", "MulDiv accepted a second request") + req = request_item::type_id::create("request"); + req.a = vif.a; + req.rhs = vif.rhs; + req.operation = vif.operation; + req.word = vif.word; + req.stall = vif.stall; + req.cancel = vif.cancel; + requests.write(req); + pending = 1; + end else if (pending && vif.request_ready) + `uvm_fatal("BUSY", "MulDiv reopened its request port before completion") + if (vif.response_valid && !pending) + `uvm_fatal("EXTRA_RESULT", "MulDiv produced an unsolicited result") + if (vif.response_valid && vif.response_ready) begin + rsp = response_item::type_id::create("response"); + rsp.value = vif.value; + responses.write(rsp); + pending = 0; + end + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/env/coverage.svh b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/env/coverage.svh new file mode 100644 index 000000000..cbc2cb93d --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/env/coverage.svh @@ -0,0 +1,34 @@ +class coverage extends uvm_subscriber #(request_item); + `uvm_component_utils(coverage) + covergroup behavior with function sample (request_item t); + option.per_instance = 1; + operation: coverpoint t.operation {bins operations[] = {[0 : 7]};} + word: coverpoint t.word; + operation_width: cross operation, word{ + ignore_bins invalid_word = binsof (operation) intersect {[1 : 3]} && binsof (word) intersect { + 1 + }; + } + zero_divisor: coverpoint (t.rhs == 0) iff (t.operation >= 4); + signed_overflow: coverpoint (t.a == 64'h8000000000000000 && t.rhs == '1) + iff (!t.word && t.operation inside {4,6}) { + bins seen = {1}; + } + word_overflow: coverpoint (t.a[31:0] == 32'h80000000 && t.rhs[31:0] == '1) + iff (t.word && t.operation inside {4,6}) { + bins seen = {1}; + } + sign_a: coverpoint t.a[63]; + sign_rhs: coverpoint t.rhs[63]; + signs: cross sign_a, sign_rhs; + backpressure: coverpoint t.stall {bins stalls[] = {0, 3};} + cancelled: coverpoint t.cancel; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + behavior = new(); + endfunction + function void write(request_item t); + behavior.sample(t); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/env/env.svh b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/env/env.svh new file mode 100644 index 000000000..8bd17acae --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/env/env.svh @@ -0,0 +1,26 @@ +`uvm_analysis_imp_decl(_reset) +class env extends environment #(request_item, response_item, model); + `uvm_component_utils(env) + agent bus; + coverage cov; + uvm_analysis_imp_reset #(bit, env) reset_export; + function new(string name, uvm_component parent); + super.new(name, parent); + reset_export = new("reset_export", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + cov = coverage::type_id::create("coverage", this); + bus = agent::type_id::create("agent", this); + endfunction + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + bus.mon.requests.connect(input_export); + bus.mon.requests.connect(cov.analysis_export); + bus.mon.reset_ap.connect(reset_export); + bus.mon.responses.connect(output_export); + endfunction + function void write_reset(bit asserted); + scoreboard.cancel_pending(); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/env/model.svh b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/env/model.svh new file mode 100644 index 000000000..08bc0404c --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/env/model.svh @@ -0,0 +1,35 @@ +class model extends reference_model #(request_item, response_item); + `uvm_component_utils(model) + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void write(request_item req); + response_item rsp = response_item::type_id::create("expected"); + bit signed_a = req.operation < 4 ? req.operation inside {1, 2} : !req.operation[0]; + bit signed_b = req.operation < 4 ? req.operation == 1 : !req.operation[0]; + logic [63:0] a = req.word ? (signed_a ? {{32{req.a[31]}},req.a[31:0]} : {32'b0,req.a[31:0]}) : req.a; + logic [63:0] b = req.word ? (signed_b ? {{32{req.rhs[31]}},req.rhs[31:0]} : {32'b0,req.rhs[31:0]}) : req.rhs; + logic signed [128:0] wide_a = signed_a ? $signed({{65{a[63]}}, a}) : $signed({65'b0, a}); + logic signed [128:0] wide_b = signed_b ? $signed({{65{b[63]}}, b}) : $signed({65'b0, b}); + logic signed [257:0] product = wide_a * wide_b; + logic [63:0] answer; + if (req.operation < 4) begin + answer = req.operation == 0 ? product[63:0] : product[127:64]; + end else if (b == 0) begin + answer = req.operation[1] ? a : '1; + end else if (!req.operation[0] && a == 64'h8000000000000000 && b == '1) begin + answer = req.operation[1] ? 0 : a; + end else begin + case (req.operation) + 4: answer = $signed(a) / $signed(b); + 5: answer = a / b; + 6: answer = $signed(a) % $signed(b); + 7: answer = a % b; + default: `uvm_fatal("OPERATION", "Invalid RV64M operation") + endcase + end + rsp.value = req.word ? {{32{answer[31]}}, answer[31:0]} : answer; + expected_ap.write(rsp); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/filelist.f b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/filelist.f new file mode 100644 index 000000000..cab59cc74 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/filelist.f @@ -0,0 +1,7 @@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv ++incdir+@RESOURCES@ +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/MulDiv/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/interface.sv b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/interface.sv new file mode 100644 index 000000000..d78bd4d68 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/interface.sv @@ -0,0 +1,23 @@ +interface port_if ( + input logic clock, + input logic reset +); + logic request_valid, request_ready, response_valid, response_ready; + logic [63:0] a; + logic [63:0] rhs; + logic [2:0] operation; + logic word; + int unsigned stall; + logic cancel; + logic [63:0] value; + assert property (@(posedge clock) disable iff (reset) + request_valid && !request_ready |=> request_valid && $stable( + {a, rhs, operation, word} + )) + else $fatal(1, "MulDiv request changed under backpressure"); + assert property (@(posedge clock) disable iff (reset) + response_valid && !response_ready |=> response_valid && $stable( + value + )) + else $fatal(1, "MulDiv result changed under backpressure"); +endinterface diff --git a/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/package.sv b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/package.sv new file mode 100644 index 000000000..da90543ec --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/package.sv @@ -0,0 +1,15 @@ +package muldiv_pkg; + timeunit 1ns; timeprecision 1ps; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/agent.svh" + `include "env/model.svh" + `include "env/coverage.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..8b5acb2c7 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/seq/sequence.svh @@ -0,0 +1,46 @@ +class stimulus extends uvm_sequence #(request_item); + `uvm_object_utils(stimulus) + int unsigned sent = 0; + function new(string name = "stimulus"); + super.new(name); + endfunction + task body(); + request_item req = request_item::type_id::create("request"); + logic [63:0] operands[10] = '{ + 0, + 1, + 3, + '1, + 64'h8000000000000000, + 64'h7fffffffffffffff, + 64'h80000000, + 64'h7fffffff, + 64'hffffffff, + 64'h123456789abcdef0 + }; + start_item(req); + req.a = 123456789; + req.rhs = 7; + req.operation = 4; + req.word = 0; + req.stall = 0; + req.cancel = 1; + finish_item(req); + for (int word = 0; word < 2; word++) + for (int operation = 0; operation < 8; operation++) begin + if (word && operation inside {1, 2, 3}) continue; + foreach (operands[a]) + foreach (operands[b]) begin + start_item(req); + req.a = operands[a]; + req.rhs = operands[b]; + req.operation = operation; + req.word = word; + req.stall = ((a + b) % 2) * 3; + req.cancel = 0; + finish_item(req); + sent++; + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/suite.toml b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/suite.toml new file mode 100644 index 000000000..6d7c8d91d --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/suite.toml @@ -0,0 +1,20 @@ +[model] +manifest = "" +library = "" + +[[tests]] +name = "muldiv" +top = "muldiv_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 2000000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +functional_groups = ["muldiv_pkg::coverage::behavior"] +sink_ready = 1 +sink_stall = 0 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/tb.sv b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/tb.sv new file mode 100644 index 000000000..9c8f8f43e --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/tb.sv @@ -0,0 +1,29 @@ +module muldiv_tb; + import uvm_pkg::*; + import muldiv_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + clock_if timing (clock); + port_if bus ( + clock, + timing.reset + ); + MulDiv dut ( + .clock(clock), + .reset(bus.reset), + .io_request_valid(bus.request_valid), + .io_request_ready(bus.request_ready), + .io_request_bits_a(bus.a), + .io_request_bits_rhs(bus.rhs), + .io_request_bits_operation(bus.operation), + .io_request_bits_word(bus.word), + .io_response_valid(bus.response_valid), + .io_response_ready(bus.response_ready), + .io_response_bits(bus.value) + ); + initial begin + uvm_config_db#(virtual clock_if)::set(null, "uvm_test_top*", "timing", timing); + uvm_config_db#(virtual port_if)::set(null, "uvm_test_top*", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/tests/test.svh new file mode 100644 index 000000000..7013953b6 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/tests/test.svh @@ -0,0 +1,26 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env bench; + reset_controller resetter; + virtual port_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + resetter = reset_controller::type_id::create("reset", this); + uvm_config_db#(reset_controller)::set(this, "env.agent.driver", "resetter", resetter); + bench = env::type_id::create("env", this); + if (!uvm_config_db#(virtual port_if)::get(this, "", "vif", vif)) + `uvm_fatal("INTERFACE", "Test interface is required") + endfunction + task execute(); + stimulus sequence_ = stimulus::type_id::create("sequence"); + resetter.pulse(4); + sequence_.start(bench.bus.seqr); + bench.scoreboard.wait_checked(sequence_.sent); + @(negedge vif.clock); + evidence::check(evidence::cancelled == 1, "Reset did not cancel the outstanding divide"); + evidence::check(bench.scoreboard.checked == sequence_.sent, "Comparison count mismatch"); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/muldiv/src/main/scala/Emit.scala b/arch/src/main/scala/framework/core/execution/muldiv/src/main/scala/Emit.scala new file mode 100644 index 000000000..d6833741d --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/muldiv/src/main/scala/Emit.scala @@ -0,0 +1,15 @@ +package framework.core.execution.muldiv + +import framework.core.execution.muldiv.configs.Params +import java.nio.file.{Files, Paths} +import upickle.default.read + +object Emit extends App { + require(args.length >= 2 && args(0) == "--config") + val param = read[Params](Files.readString(Paths.get(args(1)))) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new MulDiv(param), + firtoolOpts = args.drop(2), + args = Array("--target-dir", "build/MulDiv", "--split-verilog") + ) +} diff --git a/arch/src/main/scala/framework/core/execution/muldiv/src/main/scala/MulDiv.scala b/arch/src/main/scala/framework/core/execution/muldiv/src/main/scala/MulDiv.scala new file mode 100644 index 000000000..ce072d196 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/muldiv/src/main/scala/MulDiv.scala @@ -0,0 +1,99 @@ +package framework.core.execution.muldiv + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.core.execution.muldiv.configs.{Params => ModuleParam} + +class Request(b: ModuleParam) extends Bundle { + val p = b + + val a = UInt(p.xLen.W) + val rhs = UInt(p.xLen.W) + val operation = UInt(3.W) + val word = Bool() +} + +/** RV64M multiplier and restoring divider; one operation owns the circuit until response. */ +class MulDivIO(b: ModuleParam) extends Bundle { + val p = b + + val request = Flipped(Decoupled(new Request(b))) + val response = Decoupled(UInt(p.xLen.W)) + +} + +@instantiable +class MulDiv(b: ModuleParam) extends Module { + val p = b + + @public + val io = IO(new MulDivIO(b)) + + val idle :: divide :: respond :: Nil = Enum(3) + val state = RegInit(idle) + val quotient = Reg(UInt(64.W)) + val remainder = Reg(UInt(65.W)) + val divisor = Reg(UInt(65.W)) + val count = Reg(UInt(7.W)) + val negateQuotient = Reg(Bool()) + val negateRemainder = Reg(Bool()) + val returnRemainder = Reg(Bool()) + val word = Reg(Bool()) + val result = Reg(UInt(64.W)) + + io.request.ready := state === idle + io.response.valid := state === respond + io.response.bits := Mux(word, Cat(Fill(32, result(31)), result(31, 0)), result) + when(io.response.fire)(state := idle) + + val r = io.request.bits + val signedDivision = !r.operation(0) + val signedA = Mux(r.operation(2), signedDivision, r.operation === 1.U || r.operation === 2.U) + val signedB = Mux(r.operation(2), signedDivision, r.operation === 1.U) + val a = Mux(r.word, Cat(Fill(32, signedA && r.a(31)), r.a(31, 0)), r.a) + val rhs = Mux(r.word, Cat(Fill(32, signedB && r.rhs(31)), r.rhs(31, 0)), r.rhs) + val negativeA = signedA && a(63) + val negativeB = signedB && rhs(63) + val magnitudeA = Mux(negativeA, -a, a) + val magnitudeB = Mux(negativeB, -rhs, rhs) + val product = (Cat(negativeA, a).asSInt * Cat(negativeB, rhs).asSInt).asUInt + + when(io.request.fire) { + word := r.word + when(!r.operation(2)) { + result := Mux(r.operation === 0.U, product(63, 0), product(127, 64)) + state := respond + }.otherwise { + quotient := magnitudeA + remainder := 0.U + divisor := magnitudeB + count := 64.U + negateQuotient := negativeA ^ negativeB + negateRemainder := negativeA + returnRemainder := r.operation(1) + when(rhs === 0.U) { + result := Mux(r.operation(1), a, ~0.U(64.W)) + state := respond + }.otherwise(state := divide) + } + } + + val shifted = Cat(remainder(63, 0), quotient(63)) + val subtract = shifted >= divisor + val nextRemainder = Mux(subtract, shifted - divisor, shifted) + val nextQuotient = Cat(quotient(62, 0), subtract) + when(state === divide) { + remainder := nextRemainder + quotient := nextQuotient + count := count - 1.U + when(count === 1.U) { + result := Mux( + returnRemainder, + Mux(negateRemainder, -nextRemainder(63, 0), nextRemainder(63, 0)), + Mux(negateQuotient, -nextQuotient, nextQuotient) + ) + state := respond + } + } +} diff --git a/arch/src/main/scala/framework/core/execution/muldiv/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/core/execution/muldiv/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..4e2a371ac --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/muldiv/src/main/scala/configs/Params.scala @@ -0,0 +1,6 @@ +package framework.core.execution.muldiv.configs + +import upickle.default.{macroRW, ReadWriter} + +case class Params(xLen: Int) { require(xLen == 64) } +object Params { implicit val rw: ReadWriter[Params] = macroRW } diff --git a/arch/src/main/scala/framework/core/execution/pipeline/Sequencer.scala b/arch/src/main/scala/framework/core/execution/pipeline/Sequencer.scala new file mode 100644 index 000000000..9f8b41223 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/pipeline/Sequencer.scala @@ -0,0 +1,456 @@ +package framework.core.execution.pipeline + +import chisel3._ +import chisel3.util._ +import framework.acc.frontend.RoCCIO +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.core.execution.configs.ExecutionParam +import framework.core.execution.instruction.InstructionIO +import framework.core.execution.register_file.RegisterFileIO +import framework.core.execution.integer.IntegerIO +import framework.core.execution.muldiv.MulDivIO +import framework.core.execution.fpu.FPUIO +import framework.core.execution.privileged.{PrivilegedIO, Trap} +import framework.core.memory.access.{UncachedRequest, UncachedResponse, VirtualPort, VirtualRequest} +import framework.core.controller.admission.Commands +import framework.acc.memdomain.MvoverISA +import framework.core.clink.CoreInterrupts +import framework.ip.bus.chi.rnf.CacheAtomic + +class Retirement extends Bundle { + val pc = UInt(64.W) + val instruction = UInt(32.W) + val destination = UInt(5.W) + val data = UInt(64.W) + val integer = Bool() + val floating = Bool() +} + +/** Architectural sequencing and retirement; functional units are connected by Execution. */ +@instantiable +class Sequencer(b: ExecutionParam, localRocc: Boolean) extends Module { + val core = b + val resetVector = core.resetVector + val hartId = core.hartId + val commandsEnabled = core.commandsEnabled + val tracking = core.tracking + + @public + val io = IO(new Bundle { + val interrupts = Input(new CoreInterrupts) + val controller = Option.when(!localRocc)(new Commands(b.tracking, b.integer.xLen, b.privileged.permission)) + val rocc = Option.when(localRocc)(Flipped(new RoCCIO(b.integer.xLen, b.privileged.permission))) + val workDrained = Input(Bool()) + val retirement = Valid(new Retirement) + val fetch = Flipped(new VirtualPort(b.memoryAccess)) + val memory = Flipped(new VirtualPort(b.memoryAccess)) + val timeRequest = Decoupled(new UncachedRequest(b.memoryAccess.memory)) + val timeResponse = Flipped(Decoupled(new UncachedResponse(b.memoryAccess.memory))) + val instruction = Flipped(new InstructionIO(b.instruction)) + val registers = Flipped(new RegisterFileIO(b.registerFile)) + val integer = Flipped(new IntegerIO(b.integer)) + val multiply = Flipped(new MulDivIO(b.muldiv)) + val floating = Flipped(new FPUIO(b.fpu)) + val privileged = Flipped(new PrivilegedIO(b.privileged)) + }) + + val retirement = io.retirement + val starting :: fetchLow :: waitLow :: fetchHigh :: waitHigh :: operands :: execute :: sendMemory :: waitMemory :: sendTime :: waitTime :: waitMultiply :: waitFloating :: reserveCommand :: sendCommand :: waitCommand :: fence :: sleeping :: trapped :: Nil = + Enum(19) + val state = RegInit(starting) + val pc = Reg(UInt(64.W)) + val encoded = Reg(UInt(32.W)) + val half = Reg(UInt(16.W)) + val compressed = Reg(Bool()) + + val memoryRequest = Reg(new VirtualRequest(b.memoryAccess)) + + val trap = Reg(new Trap) + val fpRegisters = Reg(Vec(32, UInt(64.W))) + val commandResponse = RegInit(false.B) + + io.instruction.bits := encoded + val insn = io.instruction.expanded + val opcode = io.instruction.opcode + val funct3 = io.instruction.funct3 + val funct7 = io.instruction.funct7 + val rs1 = io.instruction.rs1 + val rs2 = io.instruction.rs2 + val rd = io.instruction.rd + io.registers.readAddress(0) := rs1 + io.registers.readAddress(1) := rs2 + val a = Reg(UInt(64.W)) + val operandB = Reg(UInt(64.W)) + when(state === operands) { + a := io.registers.readData(0) + operandB := io.registers.readData(1) + state := execute + } + val immediate = io.instruction.immediate + val sequentialPc = pc + Mux(compressed, 2.U, 4.U) + val writeInteger = WireDefault(false.B) + val writeFloating = WireDefault(false.B) + val writeData = WireDefault(0.U(64.W)) + val commit = WireDefault(false.B) + val nextPc = WireDefault(sequentialPc) + val exception = WireDefault(false.B) + val cause = WireDefault(2.U(64.W)) + val value = WireDefault(Mux(compressed, encoded(15, 0).pad(64), encoded.pad(64))) + val legal = WireDefault(false.B) + val startMemory = WireDefault(false.B) + val store = opcode === "h23".U || opcode === "h27".U + val fpMemory = opcode === "h07".U || opcode === "h27".U + val atomic = opcode === "h2f".U + val atomicOperation = funct7(6, 2) + val writesMemory = store || (atomic && atomicOperation =/= 2.U) + val effectiveAddress = a + Mux(atomic, 0.U, immediate) + val memorySize = Mux(atomic, funct3, funct3(1, 0)) + val memoryData = Mux(fpMemory, fpRegisters(rs2), operandB) + + val atomicCode = io.instruction.atomicCode + + io.privileged.hartId := hartId.U(64.W) + io.privileged.interrupts := io.interrupts + io.privileged.retired := commit + io.privileged.floating.valid := writeFloating || (state === waitFloating && io.floating.response.fire && !io.floating.response.bits.illegal) + io.privileged.floating.bits := Mux(state === waitFloating, io.floating.response.bits.flags, 0.U) + io.privileged.trap.valid := state === trapped + io.privileged.trap.bits := trap + io.privileged.returning.valid := false.B + io.privileged.returning.bits := 0.U + io.privileged.csr.address := io.instruction.csrAddress + io.privileged.csr.operation := funct3(1, 0) + io.privileged.csr.operand := Mux(funct3(2), rs1.pad(64), a) + io.privileged.csr.write := funct3(1, 0) === 1.U || rs1 =/= 0.U + io.privileged.csr.commit := false.B + + io.registers.write.valid := writeInteger && commit + io.registers.write.bits.address := rd + io.registers.write.bits.data := writeData + when(writeFloating && commit)(fpRegisters(rd) := writeData) + retirement.valid := commit + retirement.bits.pc := pc + retirement.bits.instruction := encoded + retirement.bits.destination := rd + retirement.bits.data := writeData + retirement.bits.integer := writeInteger && rd =/= 0.U + retirement.bits.floating := writeFloating + + io.integer.in1 := a + io.integer.in2 := Mux(io.instruction.integerImmediate, immediate, operandB) + io.integer.dw := !io.instruction.integerWord + io.integer.fn := io.instruction.integerFunction + io.multiply.request.valid := false.B + io.multiply.request.bits.a := a + io.multiply.request.bits.rhs := operandB + io.multiply.request.bits.operation := funct3 + io.multiply.request.bits.word := opcode === "h3b".U + io.multiply.response.ready := state === waitMultiply + io.floating.request.valid := false.B + io.floating.request.bits.instruction := insn + io.floating.request.bits.integer := a + io.floating.request.bits.a := fpRegisters(rs1) + io.floating.request.bits.rhs := fpRegisters(rs2) + io.floating.request.bits.c := fpRegisters(funct7(6, 2)) + io.floating.request.bits.rounding := io.privileged.rounding + io.floating.response.ready := state === waitFloating + + val flushTranslation = WireDefault(false.B) + io.fetch.flushTranslation := flushTranslation + io.memory.flushTranslation := flushTranslation + io.fetch.request.valid := (state === fetchLow && !io.privileged.interrupt.valid) || state === fetchHigh + io.fetch.request.bits := 0.U.asTypeOf(io.fetch.request.bits) + io.fetch.request.bits.vaddr := Mux(state === fetchHigh, pc + 2.U, pc) + io.fetch.request.bits.size := 1.U + io.fetch.request.bits.execute := true.B + io.fetch.request.bits.privilege := io.privileged.privilege + io.fetch.request.bits.sum := io.privileged.sum + io.fetch.request.bits.mxr := io.privileged.mxr + io.fetch.request.bits.satpMode := io.privileged.satp(63, 60) + io.fetch.request.bits.rootPpn := io.privileged.satp(43, 0) + io.fetch.response.ready := state === waitLow || state === waitHigh + io.memory.request.valid := state === sendMemory + io.memory.request.bits := memoryRequest + io.memory.response.ready := state === waitMemory + + when(state === starting) { pc := resetVector.U(64.W); state := fetchLow } + when(state === fetchLow && io.privileged.interrupt.valid) { + trap.pc := pc; trap.cause := io.privileged.interrupt.bits; trap.value := 0.U; state := trapped + } + when(io.fetch.request.fire)(state := Mux(state === fetchHigh, waitHigh, waitLow)) + when(io.fetch.response.fire) { + val response = io.fetch.response.bits + when(response.accessFault || response.pageFault || response.misaligned) { + trap.pc := pc + trap.cause := Mux(response.misaligned, 0.U, Mux(response.pageFault, 12.U, 1.U)) + trap.value := Mux(state === waitHigh, pc + 2.U, pc) + state := trapped + }.elsewhen(state === waitHigh) { + encoded := Cat(response.data(15, 0), half) + compressed := false.B + state := operands + }.elsewhen(response.data(1, 0) =/= 3.U) { + encoded := response.data(15, 0) + compressed := true.B + state := operands + }.otherwise { + half := response.data(15, 0) + state := fetchHigh + } + } + when(state === trapped) { + printf(p"TRAP pc=0x${Hexadecimal(trap.pc)} cause=0x${Hexadecimal(trap.cause)} value=0x${Hexadecimal(trap.value)}\n") + pc := io.privileged.redirect.bits; state := fetchLow + } + + when(state === execute) { + switch(opcode) { + is("h37".U) { legal := true.B; writeData := immediate; writeInteger := true.B; commit := true.B } + is("h17".U) { legal := true.B; writeData := pc + immediate; writeInteger := true.B; commit := true.B } + is("h6f".U) { + legal := true.B; nextPc := pc + immediate; writeData := sequentialPc; writeInteger := true.B; commit := true.B + } + is("h67".U) { + legal := true.B; nextPc := (a + immediate) & ~1.U(64.W) + writeData := sequentialPc; writeInteger := true.B; commit := true.B + } + is("h63".U) { + legal := true.B + val taken = MuxLookup(funct3, false.B)(Seq( + 0.U -> (a === operandB), + 1.U -> (a =/= operandB), + 4.U -> (a.asSInt < operandB.asSInt), + 5.U -> (a.asSInt >= operandB.asSInt), + 6.U -> (a < operandB), + 7.U -> (a >= operandB) + )) + nextPc := Mux(taken, pc + immediate, sequentialPc); commit := true.B + } + is("h13".U, "h1b".U) { + legal := true.B + writeData := io.integer.out; writeInteger := true.B; commit := true.B + } + is("h33".U, "h3b".U) { + when(funct7 === 1.U) { + legal := true.B + io.multiply.request.valid := legal && !io.instruction.illegal + when(io.multiply.request.fire)(state := waitMultiply) + }.otherwise { + legal := true.B + writeData := io.integer.out; writeInteger := true.B; commit := true.B + } + } + is("h03".U, "h23".U, "h07".U, "h27".U) { + legal := !fpMemory || io.privileged.floatingEnabled + startMemory := true.B + } + is("h2f".U) { + legal := true.B + startMemory := true.B + } + is("h43".U, "h47".U, "h4b".U, "h4f".U, "h53".U) { + legal := io.privileged.floatingEnabled + io.floating.request.valid := legal && !io.instruction.illegal + when(io.floating.request.fire)(state := waitFloating) + } + is("h0f".U) { + legal := true.B + state := fence + } + is("h73".U) { + when(funct3 =/= 0.U) { + legal := !io.privileged.csr.illegal + when(io.privileged.csr.memoryRead && legal && !io.instruction.illegal) { + state := sendTime + }.otherwise { + io.privileged.csr.commit := legal && !io.instruction.illegal + writeData := io.privileged.csr.data; writeInteger := true.B; commit := true.B + } + }.otherwise { + switch(insn) { + is("h00000073".U) { + legal := true.B; exception := true.B; cause := 8.U + io.privileged.privilege; value := 0.U + } + is("h00100073".U) { legal := true.B; exception := true.B; cause := 3.U; value := pc } + is("h30200073".U) { + legal := io.privileged.privilege === 3.U + io.privileged.returning.valid := legal; io.privileged.returning.bits := 3.U + nextPc := io.privileged.redirect.bits; commit := true.B + } + is("h10200073".U) { + legal := io.privileged.privilege >= 1.U && (io.privileged.privilege === 3.U || !io.privileged.tsr) + io.privileged.returning.valid := legal; io.privileged.returning.bits := 1.U + nextPc := io.privileged.redirect.bits; commit := true.B + } + is("h10500073".U) { + legal := io.privileged.privilege >= 1.U && (io.privileged.privilege === 3.U || !io.privileged.tw) + commit := true.B; state := sleeping + } + } + when(funct7 === "h09".U && rd === 0.U) { + legal := io.privileged.privilege >= 1.U && (io.privileged.privilege === 3.U || !io.privileged.tvm) + flushTranslation := legal; commit := true.B + } + } + } + is("h2b".U) { + legal := commandsEnabled.B && !localRocc.B + when(legal)(state := (if (localRocc) sendCommand else reserveCommand)) + } + is("h7b".U) { + legal := commandsEnabled.B && (core.directAccelerator.B || (core.moves.B && funct7 === MvoverISA.Funct.U)) + when(legal)(state := (if (localRocc) sendCommand else reserveCommand)) + } + } + when(startMemory && legal && !io.instruction.illegal) { + memoryRequest := 0.U.asTypeOf(memoryRequest) + memoryRequest.vaddr := effectiveAddress + memoryRequest.size := memorySize + memoryRequest.write := store + memoryRequest.signed := !funct3(2) && !fpMemory + memoryRequest.data := memoryData + memoryRequest.atomic := Mux(atomic, atomicCode, CacheAtomic.None.U) + memoryRequest.privilege := io.privileged.dataPrivilege + memoryRequest.sum := io.privileged.sum + memoryRequest.mxr := io.privileged.mxr + memoryRequest.satpMode := io.privileged.satp(63, 60) + memoryRequest.rootPpn := io.privileged.satp(43, 0) + state := sendMemory + } + when(!legal || io.instruction.illegal) { + exception := true.B; cause := 2.U + } + } + + when(io.memory.request.fire)(state := waitMemory) + when(io.memory.response.fire) { + val response = io.memory.response.bits + when(response.misaligned || response.pageFault || response.accessFault) { + exception := true.B + cause := Mux( + response.misaligned, + Mux(writesMemory, 6.U, 4.U), + Mux(response.pageFault, Mux(writesMemory, 15.U, 13.U), Mux(writesMemory, 7.U, 5.U)) + ) + value := memoryRequest.vaddr + }.otherwise { + commit := true.B + writeInteger := !store && !fpMemory + writeFloating := !store && fpMemory + writeData := Mux(fpMemory && funct3 === 2.U, Cat(~0.U(32.W), response.data(31, 0)), response.data) + } + } + when(io.multiply.response.fire) { writeInteger := true.B; writeData := io.multiply.response.bits; commit := true.B } + when(io.floating.response.fire) { + when(io.floating.response.bits.illegal)(exception := true.B) + .otherwise { + writeInteger := io.floating.response.bits.integer; writeFloating := !io.floating.response.bits.integer + writeData := io.floating.response.bits.data; commit := true.B + } + } + when(state === fence && io.workDrained)(commit := true.B) + when(state === sleeping && io.privileged.wake)(state := fetchLow) + when(commit) { + pc := nextPc + when(!(state === execute && insn === "h10500073".U))(state := fetchLow) + } + when(exception) { + commit := false.B + io.privileged.csr.commit := false.B + trap.pc := pc; trap.cause := cause; trap.value := value; state := trapped + } + + if (localRocc) { + val rocc = io.rocc.get + rocc.cmd.valid := state === sendCommand + rocc.cmd.bits.instruction.raw_inst := insn + rocc.cmd.bits.instruction.pc := pc + rocc.cmd.bits.instruction.rs1Data := a + rocc.cmd.bits.instruction.rs2Data := operandB + rocc.cmd.bits.satp := io.privileged.satp + rocc.cmd.bits.effectivePrivilege := io.privileged.dataPrivilege + rocc.cmd.bits.sum := io.privileged.sum + rocc.cmd.bits.mxr := io.privileged.mxr + rocc.cmd.bits.pmp := io.privileged.pmp + when(rocc.cmd.fire) { + commandResponse := funct3(2) + when(funct3(2))(state := waitCommand).otherwise(commit := true.B) + } + rocc.resp.ready := state === waitCommand + when(rocc.resp.fire) { + assert( + commandResponse && rocc.resp.bits.rd === rd, + "NPU response destination does not match the accepted command" + ) + writeInteger := true.B; writeData := rocc.resp.bits.data; commit := true.B + } + } else { + val commandsLive = RegInit(VecInit(Seq.fill(tracking.entries)(false.B))) + val commandTag = Reg(UInt(tracking.idBits.W)) + val host = io.controller.get + val availableTag = PriorityEncoder(~commandsLive.asUInt) + when(state === reserveCommand && !commandsLive.asUInt.andR && !reset.asBool) { + commandTag := availableTag + commandsLive(availableTag) := true.B + state := sendCommand + } + host.command.valid := state === sendCommand + host.command.bits.tag := commandTag + host.command.bits.instruction.raw_inst := insn + host.command.bits.instruction.pc := pc + host.command.bits.instruction.rs1Data := a + host.command.bits.instruction.rs2Data := operandB + host.command.bits.satp := io.privileged.satp + host.command.bits.effectivePrivilege := io.privileged.dataPrivilege + host.command.bits.sum := io.privileged.sum + host.command.bits.mxr := io.privileged.mxr + host.command.bits.pmp := io.privileged.pmp + when(host.command.fire) { + commandResponse := funct3(2) + when(funct3(2))(state := waitCommand).otherwise(commit := true.B) + } + host.response.ready := state === waitCommand + when(host.response.fire) { + assert( + commandResponse && host.response.bits.rd === rd, + "NPU response destination does not match the accepted command" + ) + writeInteger := true.B; writeData := host.response.bits.data; commit := true.B + } + host.complete.ready := true.B + when(host.complete.fire)(commandsLive(host.complete.bits.tag) := false.B) + } + // Privileged answers an authorization one cycle after it is presented; the reply then retains + // the decision through backpressure. + for ((virtual, index) <- Seq(io.fetch, io.memory).zipWithIndex) { + val deciding = RegInit(false.B) + val pending = RegInit(false.B) + val allowed = Reg(Bool()) + io.privileged.authorization(index) := virtual.authorizationRequest.bits + virtual.authorizationRequest.ready := !deciding && !pending + virtual.authorizationResponse.valid := pending + virtual.authorizationResponse.bits := allowed + when(virtual.authorizationRequest.fire)(deciding := true.B) + when(deciding) { deciding := false.B; pending := true.B; allowed := io.privileged.permitted(index) } + when(virtual.authorizationResponse.fire)(pending := false.B) + } + + val timeRequest = io.timeRequest + timeRequest.valid := state === sendTime + timeRequest.bits.addr := b.mtimeAddress.U + timeRequest.bits.tag := 0.U + timeRequest.bits.size := 3.U + timeRequest.bits.write := false.B + timeRequest.bits.data := 0.U + when(timeRequest.fire)(state := waitTime) + val timeResponse = io.timeResponse + timeResponse.ready := state === waitTime + when(timeResponse.fire) { + when(timeResponse.bits.error) { + exception := true.B; cause := 2.U; value := encoded + }.otherwise { + writeInteger := true.B; writeData := timeResponse.bits.data; commit := true.B + } + } + +} diff --git a/arch/src/main/scala/framework/core/execution/privileged/build.sc b/arch/src/main/scala/framework/core/execution/privileged/build.sc new file mode 100644 index 000000000..56a117440 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/build.sc @@ -0,0 +1,21 @@ +import mill._ +import $file.^.^.^.^.^.^.^.settings + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.core.execution.privileged.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Privileged.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/core/execution/privileged/configs/uvm.toml b/arch/src/main/scala/framework/core/execution/privileged/configs/uvm.toml new file mode 100644 index 000000000..b6f3b4b96 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/configs/uvm.toml @@ -0,0 +1,5 @@ +[parameters.permission] +addressBits = 44 +entries = 16 + +[defines] diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/csrc/Cargo.lock b/arch/src/main/scala/framework/core/execution/privileged/src/csrc/Cargo.lock new file mode 100644 index 000000000..c84e4c329 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/csrc/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "privileged-reference" +version = "0.1.0" diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/csrc/Cargo.toml b/arch/src/main/scala/framework/core/execution/privileged/src/csrc/Cargo.toml new file mode 100644 index 000000000..86130c2a7 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/csrc/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "privileged-reference" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["cdylib"] diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/csrc/src/core/mod.rs b/arch/src/main/scala/framework/core/execution/privileged/src/csrc/src/core/mod.rs new file mode 100644 index 000000000..fb27202ca --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/csrc/src/core/mod.rs @@ -0,0 +1 @@ +pub mod riscv; diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/csrc/src/core/riscv/mod.rs b/arch/src/main/scala/framework/core/execution/privileged/src/csrc/src/core/riscv/mod.rs new file mode 100644 index 000000000..ef3108d2b --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/csrc/src/core/riscv/mod.rs @@ -0,0 +1,14 @@ +#[path = "../../../../../../../../../../../../../bebop/src/nodes/bemu/src/core/riscv/bus.rs"] +pub mod bus; +#[path = "../../../../../../../../../../../../../bebop/src/nodes/bemu/src/core/riscv/csr.rs"] +pub mod csr; +#[path = "../../../../../../../../../../../../../bebop/src/nodes/bemu/src/core/riscv/mmu.rs"] +pub mod mmu; +#[path = "../../../../../../../../../../../../../bebop/src/nodes/bemu/src/core/riscv/pmp.rs"] +pub mod pmp; +#[path = "../../../../../../../../../../../../../bebop/src/nodes/bemu/src/core/riscv/privilege.rs"] +pub mod privilege; +#[path = "../../../../../../../../../../../../../bebop/src/nodes/bemu/src/core/riscv/trap.rs"] +pub mod trap; +pub use privilege::Privilege; +pub use trap::{Access, Trap}; diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/csrc/src/lib.rs b/arch/src/main/scala/framework/core/execution/privileged/src/csrc/src/lib.rs new file mode 100644 index 000000000..3e013d40b --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/csrc/src/lib.rs @@ -0,0 +1,326 @@ +mod core; +use core::riscv::{ + csr::{Csrs, FS, MIE, SIE}, + Privilege, +}; +use std::ffi::c_void; + +#[derive(Default, Clone, Copy)] +struct Inputs { + address: u16, + operand: u64, + operation: u32, + write: bool, + commit: bool, + trap: bool, + pc: u64, + cause: u64, + value: u64, + returning: bool, + return_mode: u32, + retired: bool, + floating: bool, + flags: u8, +} +struct Model { + csrs: Csrs, + privilege: Privilege, + satp: u64, + address_bits: u32, + inputs: Inputs, + values: [u64; 18], + read_value: u64, + illegal: bool, +} +fn mode(value: u32) -> Privilege { + match value { + 0 => Privilege::User, + 1 => Privilege::Supervisor, + 3 => Privilege::Machine, + _ => panic!("Reserved privilege encoding"), + } +} +unsafe fn model<'a>(p: *mut c_void) -> &'a mut Model { + &mut *p.cast::() +} +#[no_mangle] +pub extern "C" fn privileged_create(address_bits: u32) -> *mut c_void { + assert!((32..=56).contains(&address_bits)); + Box::into_raw(Box::new(Model { + csrs: Csrs::default(), + privilege: Privilege::Machine, + satp: 0, + address_bits, + inputs: Inputs::default(), + values: [0; 18], + read_value: 0, + illegal: false, + })) + .cast() +} +#[no_mangle] +pub unsafe extern "C" fn privileged_destroy(p: *mut c_void) { + drop(Box::from_raw(p.cast::())); +} +#[no_mangle] +pub unsafe extern "C" fn privileged_reset(p: *mut c_void) { + let m = model(p); + m.csrs = Csrs::default(); + m.privilege = Privilege::Machine; + m.satp = 0; +} +#[no_mangle] +pub unsafe extern "C" fn privileged_evaluate( + p: *mut c_void, + address: u32, + operand: u64, + operation: u32, + write: u8, + commit: u8, + trap: u8, + pc: u64, + cause: u64, + value: u64, + returning: u8, + return_mode: u32, + hart: u64, + interrupts: u64, + retired: u8, + floating: u8, + flags: u32, +) { + let m = model(p); + m.inputs = Inputs { + address: address as u16, + operand, + operation, + write: write != 0, + commit: commit != 0, + trap: trap != 0, + pc, + cause, + value, + returning: returning != 0, + return_mode, + retired: retired != 0, + floating: floating != 0, + flags: flags as u8, + }; + let read = match address { + // The hardware profile exposes RV64 mstatush as zero and omits environment CSRs. + 0x310 if m.privilege == Privilege::Machine => Ok(0), + 0x10a | 0x30a => Err(core::riscv::csr::CsrError), + _ => m + .csrs + .read(address as u16, m.privilege, hart, 0, interrupts), + }; + m.illegal = read.is_err() || (write != 0 && address >> 10 == 3); + m.read_value = read.unwrap_or(0); + if address == 0x180 && !m.illegal { + m.read_value = m.satp; + } + let pending = m.csrs.software_pending | interrupts; + let enabled = pending & m.csrs.ie; + let machine = m.privilege != Privilege::Machine || m.csrs.status & MIE != 0; + let supervisor = m.privilege == Privilege::User + || (m.privilege == Privilege::Supervisor && m.csrs.status & SIE != 0); + let eligible = enabled + & ((if machine { !m.csrs.ideleg } else { 0 }) + | (if supervisor { m.csrs.ideleg } else { 0 })); + let interrupt = [11, 3, 7, 9, 1, 5] + .into_iter() + .find(|bit| eligible & (1 << bit) != 0); + let delegated = m.privilege != Privilege::Machine + && (if cause >> 63 != 0 { + m.csrs.ideleg + } else { + m.csrs.edeleg + }) & (1 << (cause & 63)) + != 0; + let vector = m.csrs.tvec[usize::from(!delegated)]; + let redirect = if trap != 0 { + (vector & !3).wrapping_add(if cause >> 63 != 0 && vector & 1 != 0 { + (cause & 63) * 4 + } else { + 0 + }) + } else if return_mode == 3 { + m.csrs.epc[1] + } else { + m.csrs.epc[0] + }; + let status = m.csrs.status; + m.values = [ + if m.illegal { 0 } else { m.read_value }, + u64::from(m.illegal), + u64::from(address == 0xc01), + u64::from(trap != 0 || returning != 0), + if trap != 0 || returning != 0 { + redirect + } else { + 0 + }, + u64::from(interrupt.is_some()), + interrupt.map(|bit| 1u64 << 63 | bit).unwrap_or(0), + u64::from(enabled != 0), + m.privilege as u64, + if status & (1 << 17) != 0 { + (status >> 11) & 3 + } else { + m.privilege as u64 + }, + m.satp, + (status >> 18) & 1, + (status >> 19) & 1, + u64::from(status & FS != 0), + u64::from(m.csrs.fcsr >> 5), + (status >> 20) & 1, + (status >> 21) & 1, + (status >> 22) & 1, + ]; +} +#[no_mangle] +pub unsafe extern "C" fn privileged_value(p: *mut c_void, index: u32) -> u64 { + model(p).values[index as usize] +} +#[no_mangle] +pub unsafe extern "C" fn privileged_pmp(p: *mut c_void, index: u32, field: u32) -> u64 { + let m = model(p); + let address = m.csrs.pmp.address(index as usize); + match field { + 0 => u64::from(m.csrs.pmp.config(index as usize)), + 1 => address, + 2 => (((address ^ address.wrapping_add(1)) << 2) | 3) & ((1 << m.address_bits) - 1), + _ => panic!("Unknown PMP field"), + } +} +#[no_mangle] +pub unsafe extern "C" fn privileged_permitted( + p: *mut c_void, + address: u64, + size: u32, + read: u8, + write: u8, + execute: u8, + privilege: u32, +) -> u8 { + let m = model(p); + let end = u128::from(address) + (1u128 << size); + for index in 0..16 { + let cfg = m.csrs.pmp.config(index); + let encoded = u128::from(m.csrs.pmp.address(index)); + let (low, high) = match (cfg >> 3) & 3 { + 0 => continue, + 1 => ( + if index == 0 { + 0 + } else { + u128::from(m.csrs.pmp.address(index - 1)) * 4 + }, + encoded * 4, + ), + 2 => (encoded * 4, encoded * 4 + 4), + 3 => { + let bytes = (1u128 << (m.csrs.pmp.address(index).trailing_ones() + 3)) + .min(1u128 << m.address_bits); + let low = encoded * 4 & !(bytes - 1); + (low, low + bytes) + } + _ => unreachable!(), + }; + if low >= high || u128::from(address) >= high || end <= low { + continue; + } + let access = (read == 0 || cfg & 1 != 0) + && (write == 0 || cfg & 2 != 0) + && (execute == 0 || cfg & 4 != 0); + return u8::from( + u128::from(address) >= low + && end <= high + && ((privilege == 3 && cfg & 0x80 == 0) || access), + ); + } + u8::from(privilege == 3) +} +#[no_mangle] +pub unsafe extern "C" fn privileged_advance(p: *mut c_void) { + let m = model(p); + let i = m.inputs; + let old_status = m.csrs.status; + let old_privilege = m.privilege; + if m.csrs.inhibit & 1 == 0 { + m.csrs.cycle = m.csrs.cycle.wrapping_add(1); + } + if i.retired && m.csrs.inhibit & 4 == 0 { + m.csrs.instret = m.csrs.instret.wrapping_add(1); + } + if i.floating { + m.csrs.fcsr |= i.flags & 31; + m.csrs.status |= FS; + } + if i.commit && i.write && !m.illegal { + let value = match i.operation { + 2 => m.read_value | i.operand, + 3 => m.read_value & !i.operand, + _ => i.operand, + }; + match i.address { + 0x180 => { + if value >> 60 == 0 || value >> 60 == 8 { + m.satp = value; + m.csrs.satp.write(value); + } + } + 0x310 => (), + address => { + let value = if (0x3b0..=0x3bf).contains(&address) { + value & ((1u64 << (m.address_bits - 2)) - 1) + } else { + value + }; + m.csrs + .write(address, value, old_privilege) + .expect("Legal CSR write rejected by the reference"); + } + } + } + if i.trap { + let delegated = old_privilege != Privilege::Machine + && (if i.cause >> 63 != 0 { + m.csrs.ideleg + } else { + m.csrs.edeleg + }) & (1 << (i.cause & 63)) + != 0; + let slot = usize::from(!delegated); + m.csrs.epc[slot] = i.pc & !1; + m.csrs.cause[slot] = i.cause; + m.csrs.tval[slot] = i.value; + if delegated { + m.csrs.status = old_status & !0x122 + | ((old_status >> 1) & 1) << 5 + | u64::from(old_privilege == Privilege::Supervisor) << 8; + m.privilege = Privilege::Supervisor; + } else { + m.csrs.status = + old_status & !0x1888 | ((old_status >> 3) & 1) << 7 | (old_privilege as u64) << 11; + m.privilege = Privilege::Machine; + } + } else if i.returning { + if i.return_mode == 3 { + m.privilege = mode(((old_status >> 11) & 3) as u32); + m.csrs.status = old_status & !0x1888 | ((old_status >> 7) & 1) << 3 | 0x80; + if m.privilege != Privilege::Machine { + m.csrs.status &= !(1 << 17); + } + } else { + assert_eq!(i.return_mode, 1); + m.privilege = if old_status & 0x100 != 0 { + Privilege::Supervisor + } else { + Privilege::User + }; + m.csrs.status = old_status & !0x20122 | ((old_status >> 5) & 1) << 1 | 0x20; + } + } +} diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/agent/driver.svh b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/agent/driver.svh new file mode 100644 index 000000000..cb38e4c51 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/agent/driver.svh @@ -0,0 +1,25 @@ +class driver extends uvm_driver #(cycle_item); + `uvm_component_utils(driver) + virtual privileged_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual privileged_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Privileged interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item req; + forever begin + seq_item_port.get_next_item(req); + @(negedge vif.clock); + vif.reset = req.reset; + vif.inputs = req.inputs; + @(posedge vif.clock); + req.outputs = vif.outputs; + #1; + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/agent/item.svh b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/agent/item.svh new file mode 100644 index 000000000..b50179a31 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/agent/item.svh @@ -0,0 +1,18 @@ +class cycle_item extends uvm_sequence_item; + `uvm_object_utils(cycle_item) + logic [`PRIV_INPUT_BITS-1:0] inputs; + logic [`PRIV_OUTPUT_BITS-1:0] outputs; + bit reset; + function new(string name = "cycle_item"); + super.new(name); + endfunction +endclass +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(payload, UVM_ALL_ON) + `uvm_object_utils_end + logic [`PRIV_OUTPUT_BITS-1:0] payload; + function new(string name = "result_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/agent/monitor.svh b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/agent/monitor.svh new file mode 100644 index 000000000..e4e167708 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/agent/monitor.svh @@ -0,0 +1,25 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual privileged_if vif; + uvm_analysis_port #(cycle_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual privileged_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Privileged monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item t; + forever begin + @(posedge vif.clock); + t = cycle_item::type_id::create("observed"); + t.inputs = vif.inputs; + t.outputs = vif.outputs; + t.reset = vif.reset; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/env/env.svh b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/env/env.svh new file mode 100644 index 000000000..416baa90b --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/env/env.svh @@ -0,0 +1,25 @@ +class env extends uvm_env; + `uvm_component_utils(env) + driver drv; + monitor mon; + reference_model reference; + uvm_sequencer #(cycle_item) seqr; + result_board #(result_item, 0) scoreboard; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + drv = driver::type_id::create("driver", this); + mon = monitor::type_id::create("monitor", this); + reference = reference_model::type_id::create("model", this); + seqr = new("seqr", this); + scoreboard = result_board#(result_item, 0)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + drv.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(reference.analysis_export); + reference.expected_ap.connect(scoreboard.expected_export); + reference.actual_ap.connect(scoreboard.actual_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/env/model.svh b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/env/model.svh new file mode 100644 index 000000000..e24bf5331 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/env/model.svh @@ -0,0 +1,150 @@ +class reference_model extends uvm_subscriber #(cycle_item); + `uvm_component_utils(reference_model) + chandle reference; + int pmp_offsets[16][8] = `PRIV_PMP_OFFSETS; + uvm_analysis_port #(result_item) expected_ap, actual_ap; + // permitted answers the authorization of the previous cycle. + bit [1:0] permitted; + bit primed; + covergroup behavior with function sample ( + int address, int operation, bit write, int privilege, bit trap, bit returning, bit interrupt + ); + option.per_instance = 1; + csr_operation: coverpoint operation {bins kinds[] = {[0 : 3]};} + access_direction: coverpoint write; + current_privilege: coverpoint privilege { + bins user_mode = {0}; bins supervisor = {1}; bins machine_mode = {3}; + } + trap_event: coverpoint trap; + return_event: coverpoint returning; + interrupt_event: coverpoint interrupt; + pmp_address: coverpoint (address - 'h3b0) iff (address >= 'h3b0 && address <= 'h3bf && write) { + bins entries[] = {[0 : 15]}; + } + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected_ap = new("expected_ap", this); + actual_ap = new("actual_ap", this); + behavior = new(); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + reference = privileged_create(`PRIV_ADDRESS_BITS); + endfunction + function void write(cycle_item t); + result_item expected, actual; + longint unsigned interrupts, cfg; + if (t.reset) begin + privileged_reset(reference); + primed = 0; + return; + end + expected = result_item::type_id::create("expected"); + actual = result_item::type_id::create("actual"); + expected.payload = '0; + actual.payload = t.outputs; + interrupts = (longint'( + `PIN(t.inputs, INTERRUPTS_SOFTWARE) + ) << 3) | (longint'( + `PIN(t.inputs, INTERRUPTS_TIMER) + ) << 7) | (longint'( + `PIN(t.inputs, INTERRUPTS_EXTERNAL) + ) << 11) | (longint'( + `PIN(t.inputs, INTERRUPTS_SUPERVISOREXTERNAL) + ) << 9); + privileged_evaluate( + reference, `PIN(t.inputs, CSR_ADDRESS), `PIN(t.inputs, CSR_OPERAND), + `PIN(t.inputs, CSR_OPERATION), `PIN(t.inputs, CSR_WRITE), `PIN(t.inputs, CSR_COMMIT), + `PIN(t.inputs, TRAP_VALID), `PIN(t.inputs, TRAP_BITS_PC), `PIN(t.inputs, TRAP_BITS_CAUSE), + `PIN(t.inputs, TRAP_BITS_VALUE), `PIN(t.inputs, RETURNING_VALID), + `PIN(t.inputs, RETURNING_BITS), `PIN(t.inputs, HARTID), interrupts, `PIN(t.inputs, RETIRED), + `PIN(t.inputs, FLOATING_VALID), `PIN(t.inputs, FLOATING_BITS)); + expected.payload[`PRIV_OUT_CSR_DATA_OFFSET+:`PRIV_OUT_CSR_DATA_WIDTH] = + privileged_value(reference, 0); + expected.payload[`PRIV_OUT_CSR_ILLEGAL_OFFSET+:`PRIV_OUT_CSR_ILLEGAL_WIDTH] = + privileged_value(reference, 1); + expected.payload[`PRIV_OUT_CSR_MEMORYREAD_OFFSET +: `PRIV_OUT_CSR_MEMORYREAD_WIDTH] = privileged_value( + reference, 2); + expected.payload[`PRIV_OUT_REDIRECT_VALID_OFFSET +: `PRIV_OUT_REDIRECT_VALID_WIDTH] = privileged_value( + reference, 3); + expected.payload[`PRIV_OUT_REDIRECT_BITS_OFFSET +: `PRIV_OUT_REDIRECT_BITS_WIDTH] = privileged_value( + reference, 4); + expected.payload[`PRIV_OUT_INTERRUPT_VALID_OFFSET +: `PRIV_OUT_INTERRUPT_VALID_WIDTH] = privileged_value( + reference, 5); + expected.payload[`PRIV_OUT_INTERRUPT_BITS_OFFSET +: `PRIV_OUT_INTERRUPT_BITS_WIDTH] = privileged_value( + reference, 6); + expected.payload[`PRIV_OUT_WAKE_OFFSET+:`PRIV_OUT_WAKE_WIDTH] = privileged_value(reference, 7); + expected.payload[`PRIV_OUT_PRIVILEGE_OFFSET+:`PRIV_OUT_PRIVILEGE_WIDTH] = + privileged_value(reference, 8); + expected.payload[`PRIV_OUT_DATAPRIVILEGE_OFFSET +: `PRIV_OUT_DATAPRIVILEGE_WIDTH] = privileged_value( + reference, 9); + expected.payload[`PRIV_OUT_SATP_OFFSET+:`PRIV_OUT_SATP_WIDTH] = privileged_value(reference, 10); + expected.payload[`PRIV_OUT_SUM_OFFSET+:`PRIV_OUT_SUM_WIDTH] = privileged_value(reference, 11); + expected.payload[`PRIV_OUT_MXR_OFFSET+:`PRIV_OUT_MXR_WIDTH] = privileged_value(reference, 12); + expected.payload[`PRIV_OUT_FLOATINGENABLED_OFFSET +: `PRIV_OUT_FLOATINGENABLED_WIDTH] = privileged_value( + reference, 13); + expected.payload[`PRIV_OUT_ROUNDING_OFFSET+:`PRIV_OUT_ROUNDING_WIDTH] = + privileged_value(reference, 14); + expected.payload[`PRIV_OUT_TVM_OFFSET+:`PRIV_OUT_TVM_WIDTH] = privileged_value(reference, 15); + expected.payload[`PRIV_OUT_TW_OFFSET+:`PRIV_OUT_TW_WIDTH] = privileged_value(reference, 16); + expected.payload[`PRIV_OUT_TSR_OFFSET+:`PRIV_OUT_TSR_WIDTH] = privileged_value(reference, 17); + expected.payload[`PRIV_OUT_PERMITTED_0_OFFSET+:`PRIV_OUT_PERMITTED_0_WIDTH] = + primed ? permitted[0] : actual.payload[`PRIV_OUT_PERMITTED_0_OFFSET+:`PRIV_OUT_PERMITTED_0_WIDTH]; + expected.payload[`PRIV_OUT_PERMITTED_1_OFFSET+:`PRIV_OUT_PERMITTED_1_WIDTH] = + primed ? permitted[1] : actual.payload[`PRIV_OUT_PERMITTED_1_OFFSET+:`PRIV_OUT_PERMITTED_1_WIDTH]; + permitted[0] = privileged_permitted( + reference, + `PIN(t.inputs, AUTHORIZATION_0_PADDR), + `PIN(t.inputs, AUTHORIZATION_0_SIZE), + `PIN(t.inputs, AUTHORIZATION_0_READ), + `PIN(t.inputs, AUTHORIZATION_0_WRITE), + `PIN(t.inputs, AUTHORIZATION_0_EXECUTE), + `PIN(t.inputs, AUTHORIZATION_0_PRIVILEGE) + ); + permitted[1] = privileged_permitted( + reference, + `PIN(t.inputs, AUTHORIZATION_1_PADDR), + `PIN(t.inputs, AUTHORIZATION_1_SIZE), + `PIN(t.inputs, AUTHORIZATION_1_READ), + `PIN(t.inputs, AUTHORIZATION_1_WRITE), + `PIN(t.inputs, AUTHORIZATION_1_EXECUTE), + `PIN(t.inputs, AUTHORIZATION_1_PRIVILEGE) + ); + primed = 1; + for (int index = 0; index < 16; index++) begin + cfg = privileged_pmp(reference, index, 0); + expected.payload[pmp_offsets[index][0]] = cfg[7]; + expected.payload[pmp_offsets[index][1]+:2] = cfg[6:5]; + expected.payload[pmp_offsets[index][2]+:2] = cfg[4:3]; + expected.payload[pmp_offsets[index][3]] = cfg[2]; + expected.payload[pmp_offsets[index][4]] = cfg[1]; + expected.payload[pmp_offsets[index][5]] = cfg[0]; + expected.payload[pmp_offsets[index][6]+:(`PRIV_ADDRESS_BITS-2)] = + privileged_pmp(reference, index, 1); + expected.payload[pmp_offsets[index][7]+:`PRIV_ADDRESS_BITS] = + privileged_pmp(reference, index, 2); + end + if (!actual.payload[`PRIV_OUT_REDIRECT_VALID_OFFSET+:`PRIV_OUT_REDIRECT_VALID_WIDTH]) begin + actual.payload[`PRIV_OUT_REDIRECT_BITS_OFFSET+:`PRIV_OUT_REDIRECT_BITS_WIDTH] = 0; + end + if (!actual.payload[`PRIV_OUT_INTERRUPT_VALID_OFFSET+:`PRIV_OUT_INTERRUPT_VALID_WIDTH]) begin + actual.payload[`PRIV_OUT_INTERRUPT_BITS_OFFSET+:`PRIV_OUT_INTERRUPT_BITS_WIDTH] = 0; + end + if (actual.payload[`PRIV_OUT_CSR_ILLEGAL_OFFSET+:`PRIV_OUT_CSR_ILLEGAL_WIDTH]) begin + actual.payload[`PRIV_OUT_CSR_DATA_OFFSET+:`PRIV_OUT_CSR_DATA_WIDTH] = 0; + end + behavior.sample( + `PIN(t.inputs, CSR_ADDRESS), `PIN(t.inputs, CSR_OPERATION), `PIN(t.inputs, CSR_WRITE), + actual.payload[`PRIV_OUT_PRIVILEGE_OFFSET+:`PRIV_OUT_PRIVILEGE_WIDTH], + `PIN(t.inputs, TRAP_VALID), `PIN(t.inputs, RETURNING_VALID), + actual.payload[`PRIV_OUT_INTERRUPT_VALID_OFFSET+:`PRIV_OUT_INTERRUPT_VALID_WIDTH]); + expected_ap.write(expected); + actual_ap.write(actual); + privileged_advance(reference); + endfunction + function void final_phase(uvm_phase phase); + privileged_destroy(reference); + super.final_phase(phase); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/filelist.f b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/filelist.f new file mode 100644 index 000000000..deed612e5 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/Privileged/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/interface.sv b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/interface.sv new file mode 100644 index 000000000..b0c238888 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/interface.sv @@ -0,0 +1,8 @@ +`include "parameters.svh" +interface privileged_if ( + input logic clock +); + logic reset = 1; + logic [`PRIV_INPUT_BITS-1:0] inputs; + logic [`PRIV_OUTPUT_BITS-1:0] outputs; +endinterface diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/package.sv b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/package.sv new file mode 100644 index 000000000..a1159b0d0 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/package.sv @@ -0,0 +1,56 @@ +`include "parameters.svh" +`define PIN(P, N) P[`PRIV_IN_``N``_OFFSET +: `PRIV_IN_``N``_WIDTH] +`define POUT(P, N) P[`PRIV_OUT_``N``_OFFSET +: `PRIV_OUT_``N``_WIDTH] +package privileged_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + import "DPI-C" function chandle privileged_create(input int unsigned address_bits); + import "DPI-C" function void privileged_destroy(input chandle model); + import "DPI-C" function void privileged_reset(input chandle model); + import "DPI-C" function void privileged_evaluate( + input chandle model, + input int unsigned address, + input longint unsigned operand, + input int unsigned operation, + input byte unsigned wr, + commit, + trap, + input longint unsigned pc, + cause, + value, + input byte unsigned returning, + input int unsigned return_mode, + input longint unsigned hart, + interrupts, + input byte unsigned retired, + floating, + input int unsigned flags + ); + import "DPI-C" function longint unsigned privileged_value( + input chandle model, + input int unsigned index + ); + import "DPI-C" function longint unsigned privileged_pmp( + input chandle model, + input int unsigned index, + field + ); + import "DPI-C" function byte unsigned privileged_permitted( + input chandle model, + input longint unsigned address, + input int unsigned size, + input byte unsigned rd, + wr, + execute, + input int unsigned privilege + ); + import "DPI-C" function void privileged_advance(input chandle model); + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..fe8c9dfba --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/seq/sequence.svh @@ -0,0 +1,211 @@ +class scenario_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(scenario_sequence) + logic [`PRIV_INPUT_BITS-1:0] pins = '0; + bit reset = 1; + function new(string name = "scenario_sequence"); + super.new(name); + endfunction + task tick(int cycles = 1); + cycle_item req; + repeat (cycles) begin + req = cycle_item::type_id::create("cycle"); + start_item(req); + req.inputs = pins; + req.reset = reset; + finish_item(req); + end + endtask + task restart(); + pins = '0; + reset = 1; + tick(2); + reset = 0; + `PIN(pins, HARTID) = 1; + tick(); + endtask + task csr(int address, bit write = 0, longint unsigned value = 0, int operation = 1, + bit commit = 1); + `PIN(pins, CSR_ADDRESS) = address; + `PIN(pins, CSR_WRITE) = write; + `PIN(pins, CSR_OPERAND) = value; + `PIN(pins, CSR_OPERATION) = operation; + `PIN(pins, CSR_COMMIT) = commit; + tick(); + `PIN(pins, CSR_WRITE) = 0; + `PIN(pins, CSR_COMMIT) = 0; + endtask + task trap_event(longint unsigned pc, cause, value); + `PIN(pins, TRAP_VALID) = 1; + `PIN(pins, TRAP_BITS_PC) = pc; + `PIN(pins, TRAP_BITS_CAUSE) = cause; + `PIN(pins, TRAP_BITS_VALUE) = value; + tick(); + `PIN(pins, TRAP_VALID) = 0; + endtask + task return_event(int mode); + `PIN(pins, RETURNING_VALID) = 1; + `PIN(pins, RETURNING_BITS) = mode; + tick(); + `PIN(pins, RETURNING_VALID) = 0; + endtask + task authorize(longint unsigned address, int size, int kind, int privilege); + `PIN(pins, AUTHORIZATION_0_PADDR) = address; + `PIN(pins, AUTHORIZATION_1_PADDR) = address; + `PIN(pins, AUTHORIZATION_0_SIZE) = size; + `PIN(pins, AUTHORIZATION_1_SIZE) = size; + `PIN(pins, AUTHORIZATION_0_READ) = kind == 0; + `PIN(pins, AUTHORIZATION_1_READ) = kind == 0; + `PIN(pins, AUTHORIZATION_0_WRITE) = kind == 1; + `PIN(pins, AUTHORIZATION_1_WRITE) = kind == 1; + `PIN(pins, AUTHORIZATION_0_EXECUTE) = kind == 2; + `PIN(pins, AUTHORIZATION_1_EXECUTE) = kind == 2; + `PIN(pins, AUTHORIZATION_0_PRIVILEGE) = privilege; + `PIN(pins, AUTHORIZATION_1_PRIVILEGE) = 3; + `PIN(pins, AUTHORIZATION_0_ISPTE) = size % 2; + `PIN(pins, AUTHORIZATION_1_ISPTE) = size % 2; + tick(); + endtask + task body(); + int registers[22] = '{ + 'h105, + 'h140, + 'h141, + 'h142, + 'h143, + 'h180, + 'h300, + 'h302, + 'h303, + 'h304, + 'h305, + 'h306, + 'h310, + 'h320, + 'h340, + 'h341, + 'h342, + 'h343, + 'h344, + 'hb00, + 'hb02, + 'h106 + }; + int offsets[8] = '{-1, 0, 1, 3, 4, 63, 64, 128}; + restart(); + for (int address = 0; address < 4096; address++) csr(address); + foreach (registers[index]) begin + csr(registers[index], 1, 0); + csr(registers[index], 1, '1); + csr(registers[index], 1, 64'h0123456789abcdef, 1, 0); + for (int bit_index = 0; bit_index < 64; bit_index++) begin + csr(registers[index], 1, 64'(1) << bit_index, 2); + csr(registers[index]); + csr(registers[index], 1, 64'(1) << bit_index, 3); + end + end + restart(); + csr('h300, 1, 'h2000); + for (int rounding = 0; rounding < 8; rounding++) begin + csr(2, 1, rounding); + for (int flags = 0; flags < 32; flags++) begin + csr(1, 1, 0); + `PIN(pins, FLOATING_VALID) = 1; + `PIN(pins, FLOATING_BITS) = flags; + tick(); + `PIN(pins, FLOATING_VALID) = 0; + csr(3); + end + end + // Every implemented PMP entry, addressing mode, access permission and lock state. + for (int index = 0; index < 16; index++) + for (int mode = 0; mode < 4; mode++) + for (int permissions = 0; permissions < 8; permissions++) + for (int locked = 0; locked < 2; locked++) begin + longint unsigned base = 'h1000 + index * 'h100; + longint unsigned encoded = (base >> 2) | (mode == 3 ? 7 : 0); + int pmp_config = (locked << 7) | (mode << 3) | permissions; + restart(); + if (mode == 1) begin + if (index > 0) csr('h3b0 + index - 1, 1, base >> 2); + encoded = (base + 64) >> 2; + end + csr('h3b0 + index, 1, encoded); + csr('h3a0 + (index / 8) * 2, 1, 64'(pmp_config) << ((index % 8) * 8)); + for (int sample = 0; sample < 8; sample ++) + for (int kind = 0; kind < 3; kind++) + authorize(base + offsets[sample], sample, kind, sample % 2); + csr('h3b0 + index, 1, '1); + if (index > 0) csr('h3b0 + index - 1, 1, '1); + csr('h3a0 + (index / 8) * 2, 1, 0); + end + // An empty or reversed TOR range must not claim an overlapping access. + restart(); + csr('h3b0, 1, 'h410); + csr('h3b1, 1, 'h400); + csr('h3a0, 1, 64'h8f00); + authorize('hfe0, 7, 0, 0); + // CSR privilege checks cover the entire address space in S and U modes. + for (int target = 0; target < 2; target++) begin + restart(); + csr('h300, 1, target << 11); + csr('h341, 1, 'h1000); + return_event(3); + for (int address = 0; address < 4096; address++) csr(address); + trap_event('h12345, 64'h800000000000000b, 'hdeadbeef); + csr('h341); + csr('h342); + csr('h343); + end + for (int target = 0; target < 2; target++) + for (int vector_mode = 0; vector_mode < 2; vector_mode++) begin + restart(); + csr('h305, 1, 'h1000 | vector_mode); + csr('h105, 1, 'h2000 | vector_mode); + csr('h302, 1, 'hb3ff); + csr('h303, 1, 'h222); + csr('h304, 1, 'haaa); + csr('h300, 1, (target << 11) | 'h20); + csr('h341, 1, 'h1000); + return_event(3); + trap_event('h12345, 8, 'hbad0); + csr('h141); + csr('h142); + csr('h143); + return_event(1); + trap_event('h34567, 64'h800000000000000b, 'hbad1); + csr('h341); + return_event(3); + end + // Interrupt masking, delegation and priority across all external pending combinations. + for (int target = 0; target < 4; target++) begin + if (target == 2) continue; + for (int enable = 0; enable < 4; enable++) begin + restart(); + csr('h303, 1, 'h222); + csr('h304, 1, 'haaa); + csr('h344, 1, 'h222); + csr('h300, 1, (target << 11) | ((enable & 1) << 3) | ((enable >> 1) << 1)); + if (target != 3) return_event(3); + for (int pending = 0; pending < 16; pending++) begin + `PIN(pins, INTERRUPTS_TIMER) = pending[0]; + `PIN(pins, INTERRUPTS_SOFTWARE) = pending[1]; + `PIN(pins, INTERRUPTS_EXTERNAL) = pending[2]; + `PIN(pins, INTERRUPTS_SUPERVISOREXTERNAL) = pending[3]; + tick(); + end + end + end + restart(); + for (int inhibit = 0; inhibit < 8; inhibit++) begin + csr('h320, 1, inhibit); + for (int n = 0; n < 16; n++) begin + `PIN(pins, RETIRED) = n % 2; + csr('hb00); + csr('hb02); + end + end + `PIN(pins, RETIRED) = 0; + csr('h320, 1, 0); + tick(2); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/suite.toml b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/suite.toml new file mode 100644 index 000000000..c2c21da36 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/suite.toml @@ -0,0 +1,20 @@ +[model] +manifest = "src/csrc/Cargo.toml" +library = "libprivileged_reference" + +[[tests]] +name = "privileged" +top = "privileged_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 2000000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +functional_groups = ["privileged_pkg::reference_model::behavior"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/tb.sv b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/tb.sv new file mode 100644 index 000000000..e122383fa --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/tb.sv @@ -0,0 +1,16 @@ +module privileged_tb; + import uvm_pkg::*; + import privileged_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + privileged_if bus (clock); + Privileged dut ( + .clock(clock), + .reset(bus.reset), + `include "ports.svh" + ); + initial begin + uvm_config_db#(virtual privileged_if)::set(null, "uvm_test_top.env.*", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/tests/test.svh new file mode 100644 index 000000000..54d137a6f --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/tests/test.svh @@ -0,0 +1,15 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.start(test_env.seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/main/scala/Emit.scala b/arch/src/main/scala/framework/core/execution/privileged/src/main/scala/Emit.scala new file mode 100644 index 000000000..89b705a1f --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/main/scala/Emit.scala @@ -0,0 +1,57 @@ +package framework.core.execution.privileged + +import chisel3._ +import java.nio.file.{Files, Paths} +import framework.core.execution.privileged.configs.Params + +object Emit extends App { + val Array("--config", config, "--output-dir", output) = args + val p = upickle.default.read[Params](Files.readString(Paths.get(config))) + Files.createDirectories(Paths.get(output)) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Privileged(p), + args = Array("--target-dir", s"$output/Privileged", "--split-verilog") + ) + val inputs = Set("hartId", "interrupts", "trap", "returning", "retired", "floating", "authorization") + + def fields(data: Data, path: String): Seq[(String, Int, Boolean)] = data match { + case record: Record => record.elements.toSeq.flatMap { case (name, field) => + fields(field, if (path.isEmpty) name else s"${path}_$name") + } + case vector: Vec[_] => vector.toSeq.zipWithIndex.flatMap { case (field, index) => fields(field, s"${path}_$index") } + case leaf => Seq(( + path, + leaf.getWidth, + inputs.contains(path.takeWhile(_ != '_')) || + (path.startsWith("csr_") && !Set("csr_data", "csr_illegal", "csr_memoryRead").contains(path)) + )) + } + + var inputOffset = 0 + var outputOffset = 0 + val definitions = scala.collection.mutable.ArrayBuffer[String]() + val ports = scala.collection.mutable.ArrayBuffer[String]() + val offsets = scala.collection.mutable.Map[String, Int]() + for ((name, width, input) <- fields(new PrivilegedIO(p), "")) { + val direction = if (input) "IN" else "OUT" + val signal = if (input) "inputs" else "outputs" + val offset = if (input) inputOffset else outputOffset + definitions += s"`define PRIV_${direction}_${name.toUpperCase}_OFFSET $offset" + definitions += s"`define PRIV_${direction}_${name.toUpperCase}_WIDTH $width" + ports += s".io_$name(bus.$signal[$offset +: $width])" + if (input) inputOffset += width else outputOffset += width + offsets(name) = offset + } + + val pmpOffsets = (0 until p.permission.entries).map { index => + val names = Seq("cfg_l", "cfg_res", "cfg_a", "cfg_x", "cfg_w", "cfg_r", "addr", "mask") + names.map(name => offsets(s"pmp_${index}_$name")).mkString("'{", ",", "}") + }.mkString("'{", ",", "}") + + definitions += s"`define PRIV_INPUT_BITS $inputOffset" + definitions += s"`define PRIV_OUTPUT_BITS $outputOffset" + definitions += s"`define PRIV_ADDRESS_BITS ${p.permission.addressBits}" + definitions += s"`define PRIV_PMP_OFFSETS $pmpOffsets" + Files.writeString(Paths.get(output, "parameters.svh"), definitions.mkString("\n") + "\n") + Files.writeString(Paths.get(output, "ports.svh"), ports.mkString(",\n") + "\n") +} diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/main/scala/Privileged.scala b/arch/src/main/scala/framework/core/execution/privileged/src/main/scala/Privileged.scala new file mode 100644 index 000000000..c3a185161 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/main/scala/Privileged.scala @@ -0,0 +1,328 @@ +package framework.core.execution.privileged + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.core.clink.CoreInterrupts +import framework.core.memory.access.PhysicalAuthorization +import framework.core.memory.access.preflight.PmpEntry +import framework.core.memory.access.preflight.configs.PmpParams +import framework.core.execution.privileged.configs.{Params => ModuleParam} +import framework.core.execution.privileged.configs.Params + +class Trap extends Bundle { + val pc = UInt(64.W) + val cause = UInt(64.W) + val value = UInt(64.W) +} + +/** M/S/U architectural state, precise traps and physical memory permissions. */ +class PrivilegedIO(b: ModuleParam) extends Bundle { + val p = b + implicit val permission: PmpParams = p.permission + + val hartId = Input(UInt(64.W)) + val interrupts = Input(new CoreInterrupts) + + val csr = new Bundle { + val address = Input(UInt(12.W)) + val operand = Input(UInt(64.W)) + val operation = Input(UInt(2.W)) + val write = Input(Bool()) + val commit = Input(Bool()) + val data = Output(UInt(64.W)) + val illegal = Output(Bool()) + val memoryRead = Output(Bool()) + } + + val trap = Flipped(Valid(new Trap)) + val returning = Flipped(Valid(UInt(2.W))) + val redirect = Valid(UInt(64.W)) + val interrupt = Valid(UInt(64.W)) + val wake = Output(Bool()) + val retired = Input(Bool()) + val floating = Flipped(Valid(UInt(5.W))) + val privilege = Output(UInt(2.W)) + val dataPrivilege = Output(UInt(2.W)) + val satp = Output(UInt(64.W)) + val sum = Output(Bool()) + val mxr = Output(Bool()) + val floatingEnabled = Output(Bool()) + val rounding = Output(UInt(3.W)) + val tvm = Output(Bool()) + val tw = Output(Bool()) + val tsr = Output(Bool()) + val authorization = Input(Vec(2, new PhysicalAuthorization)) + val permitted = Output(Vec(2, Bool())) + + val pmp = Output(Vec( + permission.entries, + new PmpEntry(b.permission) + )) + +} + +@instantiable +class Privileged(b: ModuleParam) extends Module { + val p = b + implicit val permission: PmpParams = p.permission + require(permission.entries == 16) + + @public + val io = IO(new PrivilegedIO(b)) + + val privilege = RegInit(3.U(2.W)) + val status = RegInit(0.U(64.W)) + val supervisorMask = BigInt("80000003000de122", 16) + val statusMask = BigInt("00000000007e79aa", 16) + val statusRead = status | (BigInt(2).U(64.W) << 32) | (BigInt(2).U(64.W) << 34) | + Mux(status(14, 13) === 3.U, BigInt("8000000000000000", 16).U, 0.U) + val mie = RegInit(0.U(64.W)) + val softwarePending = RegInit(0.U(64.W)) + val medeleg = RegInit(0.U(64.W)) + val mideleg = RegInit(0.U(64.W)) + val mtvec = RegInit(0.U(64.W)) + val stvec = RegInit(0.U(64.W)) + val mscratch = RegInit(0.U(64.W)) + val sscratch = RegInit(0.U(64.W)) + val mepc = RegInit(0.U(64.W)) + val sepc = RegInit(0.U(64.W)) + val mcause = RegInit(0.U(64.W)) + val scause = RegInit(0.U(64.W)) + val mtval = RegInit(0.U(64.W)) + val stval = RegInit(0.U(64.W)) + val satp = RegInit(0.U(64.W)) + val mcounteren = RegInit(0.U(3.W)) + val scounteren = RegInit(0.U(3.W)) + val inhibit = RegInit(0.U(3.W)) + val cycles = RegInit(0.U(64.W)) + val instructions = RegInit(0.U(64.W)) + val fflags = RegInit(0.U(5.W)) + val frm = RegInit(0.U(3.W)) + val pmpConfig = RegInit(VecInit(Seq.fill(16)(0.U(8.W)))) + val pmpAddress = RegInit(VecInit(Seq.fill(16)(0.U((permission.addressBits - 2).W)))) + + when(!inhibit(0))(cycles := cycles + 1.U) + when(io.retired && !inhibit(2))(instructions := instructions + 1.U) + when(io.floating.valid) { + fflags := fflags | io.floating.bits + status := status | (3.U(64.W) << 13) + } + + val pending = softwarePending | + (io.interrupts.software.asUInt << 3) | (io.interrupts.timer.asUInt << 7) | + (io.interrupts.external.asUInt << 11) | (io.interrupts.supervisorExternal.asUInt << 9) + + val enabled = pending & mie + val machineEnabled = privilege < 3.U || status(3) + val supervisorEnabled = privilege === 0.U || (privilege === 1.U && status(1)) + val eligible = enabled & (Mux(machineEnabled, ~mideleg, 0.U(64.W)) | + Mux(supervisorEnabled, mideleg, 0.U(64.W))) + val interruptPriority = Seq(11, 3, 7, 9, 1, 5) + io.interrupt.valid := eligible.orR + io.interrupt.bits := BigInt("8000000000000000", 16).U | + PriorityMux(interruptPriority.map(i => eligible(i) -> i.U(64.W)) :+ (true.B -> 0.U(64.W))) + io.wake := enabled.orR + io.privilege := privilege + io.dataPrivilege := Mux(status(17), status(12, 11), privilege) + io.satp := satp + io.sum := status(18) + io.mxr := status(19) + io.floatingEnabled := status(14, 13).orR + io.rounding := frm + io.tvm := status(20) + io.tw := status(21) + io.tsr := status(22) + + val csr = io.csr + val floatingCsr = csr.address >= 1.U && csr.address <= 3.U + val counterCsr = csr.address >= "hc00".U && csr.address <= "hc02".U + val counterBit = csr.address(1, 0) + val counterAllowed = privilege === 3.U || (mcounteren(counterBit) && + (privilege === 1.U || scounteren(counterBit))) + val known = WireDefault(false.B) + csr.data := 0.U + + val read = Seq( + 0x001 -> fflags.pad(64), + 0x002 -> frm.pad(64), + 0x003 -> Cat(frm, fflags).pad(64), + 0x100 -> (statusRead & supervisorMask.U(64.W)), + 0x104 -> (mie & mideleg), + 0x105 -> stvec, + 0x106 -> scounteren.pad(64), + 0x140 -> sscratch, + 0x141 -> sepc, + 0x142 -> scause, + 0x143 -> stval, + 0x144 -> (pending & mideleg), + 0x180 -> satp, + 0x300 -> statusRead, + 0x301 -> BigInt("800000000014112d", 16).U(64.W), + 0x302 -> medeleg, + 0x303 -> mideleg, + 0x304 -> mie, + 0x305 -> mtvec, + 0x306 -> mcounteren.pad(64), + 0x310 -> 0.U(64.W), + 0x320 -> inhibit.pad(64), + 0x340 -> mscratch, + 0x341 -> mepc, + 0x342 -> mcause, + 0x343 -> mtval, + 0x344 -> pending, + 0x3a0 -> pmpConfig.take(8).reverse.reduce(Cat(_, _)), + 0x3a2 -> pmpConfig.drop(8).reverse.reduce(Cat(_, _)), + 0xb00 -> cycles, + 0xb02 -> instructions, + 0xc00 -> cycles, + 0xc02 -> instructions, + 0xf11 -> 0.U(64.W), + 0xf12 -> 0.U(64.W), + 0xf13 -> 0.U(64.W), + 0xf14 -> io.hartId, + 0xf15 -> 0.U(64.W) + ) ++ (0 until 16).map(i => (0x3b0 + i) -> pmpAddress(i).pad(64)) + + for ((address, data) <- read) { + when(csr.address === address.U) { known := true.B; csr.data := data } + } + csr.memoryRead := csr.address === "hc01".U + when(csr.memoryRead)(known := true.B) + csr.illegal := !known || privilege < csr.address(9, 8) || + (csr.write && csr.address(11, 10) === 3.U) || + (floatingCsr && !io.floatingEnabled) || (counterCsr && !counterAllowed) || + (csr.address === "h180".U && privilege === 1.U && status(20)) + + val writeData = MuxLookup(csr.operation, csr.operand)(Seq( + 2.U -> (csr.data | csr.operand), + 3.U -> (csr.data & ~csr.operand) + )) + + when(csr.commit && csr.write && !csr.illegal) { + switch(csr.address) { + is("h001".U) { fflags := writeData(4, 0); status := status | (3.U(64.W) << 13) } + is("h002".U) { frm := writeData(2, 0); status := status | (3.U(64.W) << 13) } + is("h003".U) { fflags := writeData(4, 0); frm := writeData(7, 5); status := status | (3.U(64.W) << 13) } + is("h100".U) { + status := (status & ~supervisorMask.U(64.W)) | (writeData & supervisorMask.U(64.W) & statusMask.U(64.W)) + } + is("h104".U)(mie := (mie & ~mideleg) | (writeData & mideleg)) + is("h105".U)(stvec := Cat(writeData(63, 2), 0.U(1.W), writeData(1, 0) === 1.U)) + is("h106".U)(scounteren := writeData(2, 0)) + is("h140".U)(sscratch := writeData) + is("h141".U)(sepc := writeData & ~1.U(64.W)) + is("h142".U)(scause := writeData) + is("h143".U)(stval := writeData) + is("h144".U)(softwarePending := (softwarePending & ~2.U(64.W)) | (writeData & mideleg & 2.U(64.W))) + is("h180".U)(when(writeData(63, 60) === 0.U || writeData(63, 60) === 8.U)(satp := writeData)) + is("h300".U) { + status := (writeData & statusMask.U(64.W) & ~(3.U(64.W) << 11)) | + (Mux(writeData(12, 11) === 2.U, 0.U(2.W), writeData(12, 11)).pad(64) << 11) + } + is("h302".U)(medeleg := writeData & "hb3ff".U) + is("h303".U)(mideleg := writeData & "h222".U) + is("h304".U)(mie := writeData & "haaa".U) + is("h305".U)(mtvec := Cat(writeData(63, 2), 0.U(1.W), writeData(1, 0) === 1.U)) + is("h306".U)(mcounteren := writeData(2, 0)) + is("h320".U)(inhibit := writeData(2, 0) & 5.U) + is("h340".U)(mscratch := writeData) + is("h341".U)(mepc := writeData & ~1.U(64.W)) + is("h342".U)(mcause := writeData) + is("h343".U)(mtval := writeData) + is("h344".U)(softwarePending := writeData & "h222".U) + is("hb00".U)(cycles := writeData) + is("hb02".U)(instructions := writeData) + } + for (i <- 0 until 16) { + when(csr.address === (0x3a0 + (i / 8) * 2).U && !pmpConfig(i)(7)) { + val cfg = writeData(8 * (i % 8) + 7, 8 * (i % 8)) + pmpConfig(i) := cfg & "h9d".U | Mux(cfg(0) && cfg(1), 2.U, 0.U) + } + val lockedByTor = if (i == 15) false.B else pmpConfig(i + 1)(7) && pmpConfig(i + 1)(4, 3) === 1.U + when(csr.address === (0x3b0 + i).U && !pmpConfig(i)(7) && !lockedByTor) { + pmpAddress(i) := writeData(permission.addressBits - 3, 0) + } + } + } + + val supervisorTrap = + privilege =/= 3.U && Mux(io.trap.bits.cause(63), mideleg(io.trap.bits.cause(5, 0)), medeleg(io.trap.bits.cause(5, 0))) + val trapVector = Mux(supervisorTrap, stvec, mtvec) + io.redirect.valid := io.trap.valid || io.returning.valid + io.redirect.bits := Mux( + io.trap.valid, + (trapVector & ~3.U(64.W)) + Mux(io.trap.bits.cause(63) && trapVector(0), io.trap.bits.cause(5, 0) << 2, 0.U), + Mux(io.returning.bits === 3.U, mepc, sepc) + ) + when(io.trap.valid) { + when(supervisorTrap) { + sepc := io.trap.bits.pc & ~1.U(64.W) + scause := io.trap.bits.cause + stval := io.trap.bits.value + status := (status & ~"h122".U(64.W)) | (status(1).asUInt << 5) | ((privilege === 1.U).asUInt << 8) + privilege := 1.U + }.otherwise { + mepc := io.trap.bits.pc & ~1.U(64.W) + mcause := io.trap.bits.cause + mtval := io.trap.bits.value + status := (status & ~"h1888".U(64.W)) | (status(3).asUInt << 7) | (privilege.pad(64) << 11) + privilege := 3.U + } + }.elsewhen(io.returning.valid) { + when(io.returning.bits === 3.U) { + privilege := status(12, 11) + status := (status & ~"h1888".U(64.W) & Mux(status(12, 11) === 3.U, ~0.U(64.W), ~"h20000".U(64.W))) | + (status(7).asUInt << 3) | "h80".U + }.otherwise { + privilege := Mux(status(8), 1.U, 0.U) + status := (status & ~"h20122".U(64.W)) | (status(5).asUInt << 1) | "h20".U + } + } + + for (i <- 0 until 16) { + io.pmp(i).cfg := pmpConfig(i).asTypeOf(io.pmp(i).cfg) + io.pmp(i).addr := pmpAddress(i) + io.pmp(i).mask := ((pmpAddress(i) ^ (pmpAddress(i) + 1.U)) << 2) | 3.U + } + // PMP bounds and the authorization are registered: permitted answers the authorization of the + // previous cycle against the PMP state of that same cycle. Bounds are inclusive so the + // decision path only compares. + val bounds = Reg(Vec(16, new PmpBound)) + val requests = RegNext(io.authorization) + // Last byte of each authorization: paddr + 2^size - 1. + val lasts = RegNext(VecInit(io.authorization.map(a => a.paddr +& ((1.U(8.W) << a.size) - 1.U)))) + for (i <- 0 until 16) { + val encoded = (pmpAddress(i) << 2).pad(64) + val mask = io.pmp(i).mask.pad(64) + val mode = pmpConfig(i)(4, 3) + val below = if (i == 0) 0.U(64.W) else (pmpAddress(i - 1) << 2).pad(64) + bounds(i).low := Mux(mode === 1.U, below, Mux(mode === 3.U, encoded & ~mask, encoded)) + bounds(i).last := MuxLookup(mode, encoded | 3.U)(Seq( + 1.U -> ((pmpAddress(i) - 1.U) << 2 | 3.U).pad(64), + 3.U -> (encoded | mask) + )) + bounds(i).enabled := mode =/= 0.U && (mode =/= 1.U || below < encoded) + bounds(i).config := pmpConfig(i) + } + for (port <- 0 until 2) { + val request = requests(port) + val last = lasts(port) + val matches = bounds.map { bound => + val hit = bound.enabled && request.paddr <= bound.last && last >= bound.low + val allowed = (request.privilege === 3.U && !bound.config(7)) || + ((!request.read || bound.config(0)) && (!request.write || bound.config(1)) && + (!request.execute || bound.config(2))) + hit -> (allowed && request.paddr >= bound.low && last <= bound.last) + } + io.permitted(port) := PriorityMux(matches :+ (true.B -> (request.privilege === 3.U))) + } +} + +/** One PMP entry as an inclusive byte range [low, last]. */ +class PmpBound extends Bundle { + val low = UInt(64.W) + val last = UInt(64.W) + val enabled = Bool() + val config = UInt(8.W) +} diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/core/execution/privileged/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..e90a22e1d --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/main/scala/configs/Params.scala @@ -0,0 +1,7 @@ +package framework.core.execution.privileged.configs + +import framework.core.memory.access.preflight.configs.PmpParams +import upickle.default.{macroRW, ReadWriter} + +case class Params(permission: PmpParams) +object Params { implicit val rw: ReadWriter[Params] = macroRW } diff --git a/arch/src/main/scala/framework/core/execution/register_file/build.sc b/arch/src/main/scala/framework/core/execution/register_file/build.sc new file mode 100644 index 000000000..e47500e02 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/build.sc @@ -0,0 +1,21 @@ +import mill._ +import $file.^.^.^.^.^.^.^.settings + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.core.execution.register_file.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/RegisterFile.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/core/execution/register_file/configs/uvm.toml b/arch/src/main/scala/framework/core/execution/register_file/configs/uvm.toml new file mode 100644 index 000000000..3da02cd69 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/configs/uvm.toml @@ -0,0 +1,4 @@ +[parameters] +xLen = 64 + +[defines] diff --git a/arch/src/main/scala/framework/core/execution/register_file/src/csrc/dual_read.cpp b/arch/src/main/scala/framework/core/execution/register_file/src/csrc/dual_read.cpp new file mode 100644 index 000000000..0a33b028d --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/src/csrc/dual_read.cpp @@ -0,0 +1,45 @@ +#include "VRegisterFile.h" +#include "verilated.h" +#include +#include +#include +#include +int main() { + VerilatedContext context; + VRegisterFile dut(&context); + std::array reference{}; + for (unsigned i = 1; i < 32; ++i) { + dut.io_write_valid = 1; + dut.io_write_bits_address = i; + dut.io_write_bits_data = reference[i] = 0x1234000000000000ULL + i; + dut.clock = 0; + dut.eval(); + dut.clock = 1; + dut.eval(); + } + std::mt19937_64 random(0x6447); + for (unsigned i = 0; i < 60000; ++i) { + unsigned a = random() % 32, b = random() % 32, w = random() % 32; + bool valid = random() % 2; + uint64_t data = random(); + dut.io_readAddress_0 = a; + dut.io_readAddress_1 = b; + dut.io_write_valid = valid; + dut.io_write_bits_address = w; + dut.io_write_bits_data = data; + dut.clock = 0; + dut.eval(); + assert(dut.io_readData_0 == (a == 0 ? 0 + : valid && a == w ? data + : reference[a])); + assert(dut.io_readData_1 == (b == 0 ? 0 + : valid && b == w ? data + : reference[b])); + dut.clock = 1; + dut.eval(); + if (valid && w) + reference[w] = data; + } + puts("PASS: 60000 dual-read register-file checks, write bypass and x0 " + "suppression"); +} diff --git a/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/agent/agent.svh b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/agent/agent.svh new file mode 100644 index 000000000..6d2a96092 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/agent/agent.svh @@ -0,0 +1,19 @@ +class agent extends uvm_agent; + `uvm_component_utils(agent) + uvm_sequencer #(request_item) seqr; + driver drv; + monitor mon; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + seqr = uvm_sequencer#(request_item)::type_id::create("seqr", this); + drv = driver::type_id::create("driver", this); + mon = monitor::type_id::create("monitor", this); + endfunction + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + drv.seq_item_port.connect(seqr.seq_item_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/agent/driver.svh b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/agent/driver.svh new file mode 100644 index 000000000..38a8b5c0d --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/agent/driver.svh @@ -0,0 +1,35 @@ +class driver extends uvm_driver #(request_item); + `uvm_component_utils(driver) + virtual port_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual port_if)::get(this, "", "vif", vif)) + `uvm_fatal("INTERFACE", "Driver interface is required") + endfunction + task run_phase(uvm_phase phase); + request_item req; + vif.sample = 0; + vif.read0 = 0; + vif.read1 = 0; + vif.write = 0; + vif.address = 0; + vif.data = 0; + forever begin + seq_item_port.get_next_item(req); + do @(negedge vif.clock); while (vif.reset); + vif.read0 = req.read0; + vif.read1 = req.read1; + vif.write = req.write; + vif.address = req.address; + vif.data = req.data; + vif.sample = 1; + @(posedge vif.clock); + seq_item_port.item_done(); + @(negedge vif.clock); + vif.sample = 0; + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/agent/item.svh b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/agent/item.svh new file mode 100644 index 000000000..5cfddf213 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/agent/item.svh @@ -0,0 +1,29 @@ +class request_item extends uvm_sequence_item; + logic [4:0] read0; + logic [4:0] read1; + logic write; + logic [4:0] address; + logic [63:0] data; + `uvm_object_utils_begin(request_item) + `uvm_field_int(read0, UVM_ALL_ON) + `uvm_field_int(read1, UVM_ALL_ON) + `uvm_field_int(write, UVM_ALL_ON) + `uvm_field_int(address, UVM_ALL_ON) + `uvm_field_int(data, UVM_ALL_ON) + `uvm_object_utils_end + function new(string name = "request_item"); + super.new(name); + endfunction +endclass + +class response_item extends uvm_sequence_item; + logic [63:0] data0; + logic [63:0] data1; + `uvm_object_utils_begin(response_item) + `uvm_field_int(data0, UVM_ALL_ON) + `uvm_field_int(data1, UVM_ALL_ON) + `uvm_object_utils_end + function new(string name = "response_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/agent/monitor.svh b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/agent/monitor.svh new file mode 100644 index 000000000..a741412f7 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/agent/monitor.svh @@ -0,0 +1,36 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual port_if vif; + uvm_analysis_port #(request_item) requests; + uvm_analysis_port #(response_item) responses; + function new(string name, uvm_component parent); + super.new(name, parent); + requests = new("requests", this); + responses = new("responses", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual port_if)::get(this, "", "vif", vif)) + `uvm_fatal("INTERFACE", "Monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + request_item req; + response_item rsp; + forever begin + @(posedge vif.clock); + if (!vif.reset && vif.sample) begin + req = request_item::type_id::create("request"); + rsp = response_item::type_id::create("response"); + req.read0 = vif.read0; + req.read1 = vif.read1; + req.write = vif.write; + req.address = vif.address; + req.data = vif.data; + rsp.data0 = vif.data0; + rsp.data1 = vif.data1; + requests.write(req); + responses.write(rsp); + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/env/coverage.svh b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/env/coverage.svh new file mode 100644 index 000000000..d88990b0c --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/env/coverage.svh @@ -0,0 +1,21 @@ +class coverage extends uvm_subscriber #(request_item); + `uvm_component_utils(coverage) + covergroup behavior with function sample (request_item t); + option.per_instance = 1; + read0: coverpoint t.read0 {bins addresses[] = {[0 : 31]};} + read1: coverpoint t.read1 {bins addresses[] = {[0 : 31]};} + read_pairs: cross read0, read1; + write: coverpoint t.write; + destination: coverpoint t.address iff (t.write) {bins addresses[] = {[0 : 31]};} + bypass0: coverpoint (t.write && t.address != 0 && t.address == t.read0); + bypass1: coverpoint (t.write && t.address != 0 && t.address == t.read1); + write_zero: coverpoint (t.write && t.address == 0) {bins seen = {1};} + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + behavior = new(); + endfunction + function void write(request_item t); + behavior.sample(t); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/env/env.svh b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/env/env.svh new file mode 100644 index 000000000..5a8082f7a --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/env/env.svh @@ -0,0 +1,19 @@ +class env extends environment #(request_item, response_item, model); + `uvm_component_utils(env) + agent bus; + coverage cov; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + cov = coverage::type_id::create("coverage", this); + bus = agent::type_id::create("agent", this); + endfunction + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + bus.mon.requests.connect(input_export); + bus.mon.requests.connect(cov.analysis_export); + bus.mon.responses.connect(output_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/env/model.svh b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/env/model.svh new file mode 100644 index 000000000..385609f57 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/env/model.svh @@ -0,0 +1,20 @@ +class model extends reference_model #(request_item, response_item); + `uvm_component_utils(model) + logic [63:0] registers[32]; + bit initialized[32]; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void write(request_item req); + response_item rsp = response_item::type_id::create("expected"); + if (req.write && req.address != 0) begin + registers[req.address] = req.data; + initialized[req.address] = 1; + end + if ((req.read0 != 0 && !initialized[req.read0]) || (req.read1 != 0 && !initialized[req.read1])) + `uvm_fatal("UNINITIALIZED", "Reading a register before its first write") + rsp.data0 = req.read0 == 0 ? 0 : registers[req.read0]; + rsp.data1 = req.read1 == 0 ? 0 : registers[req.read1]; + expected_ap.write(rsp); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/filelist.f b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/filelist.f new file mode 100644 index 000000000..ca0929461 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/filelist.f @@ -0,0 +1,7 @@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv ++incdir+@RESOURCES@ +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/RegisterFile/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/interface.sv b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/interface.sv new file mode 100644 index 000000000..8ef3bb674 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/interface.sv @@ -0,0 +1,13 @@ +interface port_if ( + input logic clock, + input logic reset +); + logic sample; + logic [4:0] read0; + logic [4:0] read1; + logic write; + logic [4:0] address; + logic [63:0] data; + logic [63:0] data0; + logic [63:0] data1; +endinterface diff --git a/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/package.sv b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/package.sv new file mode 100644 index 000000000..e8dc35920 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/package.sv @@ -0,0 +1,15 @@ +package register_file_pkg; + timeunit 1ns; timeprecision 1ps; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/agent.svh" + `include "env/model.svh" + `include "env/coverage.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..6fca06d4a --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/seq/sequence.svh @@ -0,0 +1,47 @@ +class stimulus extends uvm_sequence #(request_item); + `uvm_object_utils(stimulus) + int unsigned sent = 0; + reset_controller resetter; + function new(string name = "stimulus"); + super.new(name); + endfunction + task body(); + request_item req = request_item::type_id::create("request"); + for (int i = 1; i < 32; i++) begin + start_item(req); + req.read0 = 0; + req.read1 = 0; + req.write = 1; + req.address = i; + req.data = 64'h0102040810204081 * i; + finish_item(req); + sent++; + end + resetter.pulse(2); + for (int i = 0; i < 32; i++) + for (int j = 0; j < 32; j++) begin + start_item(req); + req.read0 = i; + req.read1 = j; + req.write = 0; + req.address = 0; + req.data = 0; + finish_item(req); + sent++; + end + for (int i = 0; i < 32; i++) begin + start_item(req); + req.read0 = i; + req.read1 = i; + req.write = 1; + req.address = i; + req.data = '1; + finish_item(req); + sent++; + start_item(req); + req.write = 0; + finish_item(req); + sent++; + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/suite.toml b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/suite.toml new file mode 100644 index 000000000..514f36aec --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/suite.toml @@ -0,0 +1,20 @@ +[model] +manifest = "" +library = "" + +[[tests]] +functional_groups = ["register_file_pkg::coverage::behavior"] +name = "register_file" +top = "register_file_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 1000000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 0 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/tb.sv b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/tb.sv new file mode 100644 index 000000000..c9e5e3497 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/tb.sv @@ -0,0 +1,27 @@ +module register_file_tb; + import uvm_pkg::*; + import register_file_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + clock_if timing (clock); + port_if bus ( + clock, + timing.reset + ); + RegisterFile dut ( + .clock(clock), + .reset(bus.reset), + .io_readAddress_0(bus.read0), + .io_readAddress_1(bus.read1), + .io_readData_0(bus.data0), + .io_readData_1(bus.data1), + .io_write_valid(bus.sample && bus.write), + .io_write_bits_address(bus.address), + .io_write_bits_data(bus.data) + ); + initial begin + uvm_config_db#(virtual clock_if)::set(null, "uvm_test_top*", "timing", timing); + uvm_config_db#(virtual port_if)::set(null, "uvm_test_top*", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/tests/test.svh new file mode 100644 index 000000000..d5f2d6ed9 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/tests/test.svh @@ -0,0 +1,25 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env bench; + reset_controller resetter; + virtual port_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + resetter = reset_controller::type_id::create("reset", this); + bench = env::type_id::create("env", this); + if (!uvm_config_db#(virtual port_if)::get(this, "", "vif", vif)) + `uvm_fatal("INTERFACE", "Test interface is required") + endfunction + task execute(); + stimulus sequence_ = stimulus::type_id::create("sequence"); + resetter.pulse(4); + sequence_.resetter = resetter; + sequence_.start(bench.bus.seqr); + bench.scoreboard.wait_checked(sequence_.sent); + @(negedge vif.clock); + evidence::check(bench.scoreboard.checked == sequence_.sent, "Comparison count mismatch"); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/register_file/src/main/scala/Emit.scala b/arch/src/main/scala/framework/core/execution/register_file/src/main/scala/Emit.scala new file mode 100644 index 000000000..915b42ec6 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/src/main/scala/Emit.scala @@ -0,0 +1,15 @@ +package framework.core.execution.register_file + +import framework.core.execution.register_file.configs.Params +import java.nio.file.{Files, Paths} +import upickle.default.read + +object Emit extends App { + require(args.length >= 2 && args(0) == "--config") + val param = read[Params](Files.readString(Paths.get(args(1)))) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new RegisterFile(param), + firtoolOpts = args.drop(2), + args = Array("--target-dir", "build/RegisterFile", "--split-verilog") + ) +} diff --git a/arch/src/main/scala/framework/core/execution/register_file/src/main/scala/RegisterFile.scala b/arch/src/main/scala/framework/core/execution/register_file/src/main/scala/RegisterFile.scala new file mode 100644 index 000000000..045511a99 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/src/main/scala/RegisterFile.scala @@ -0,0 +1,40 @@ +package framework.core.execution.register_file + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.core.execution.register_file.configs.{Params => ModuleParam} + +class RegisterFileIO(b: ModuleParam) extends Bundle { + val p = b + + val readAddress = Input(Vec(2, UInt(5.W))) + val readData = Output(Vec(2, UInt(p.xLen.W))) + + val write = Flipped(Valid(new Bundle { + val address = UInt(5.W) + val data = UInt(p.xLen.W) + })) + +} + +@instantiable +class RegisterFile(b: ModuleParam) extends Module { + val p = b + + @public + val io = IO(new RegisterFileIO(b)) + + val registers = Reg(Vec(32, UInt(p.xLen.W))) + when(io.write.valid && io.write.bits.address =/= 0.U) { + registers(io.write.bits.address) := io.write.bits.data + } + for (port <- 0 until 2) { + val address = io.readAddress(port) + io.readData(port) := Mux( + address === 0.U, + 0.U, + Mux(io.write.valid && io.write.bits.address === address, io.write.bits.data, registers(address)) + ) + } +} diff --git a/arch/src/main/scala/framework/core/execution/register_file/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/core/execution/register_file/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..7c3717f46 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/src/main/scala/configs/Params.scala @@ -0,0 +1,6 @@ +package framework.core.execution.register_file.configs + +import upickle.default.{macroRW, ReadWriter} + +case class Params(xLen: Int) { require(xLen == 64) } +object Params { implicit val rw: ReadWriter[Params] = macroRW } diff --git a/arch/src/main/scala/framework/core/memory/Memory.scala b/arch/src/main/scala/framework/core/memory/Memory.scala new file mode 100644 index 000000000..a387b680f --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/Memory.scala @@ -0,0 +1,65 @@ +package framework.core.memory + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.core.memory.configs.MemoryParam +import framework.core.memory.access.{UncachedRequest, UncachedResponse, VirtualPort} +import framework.ip.bus.axi4 +import framework.ip.bus.chi.RequesterPort +import framework.ip.bus.chi.snf.{LineRequest, LineResponse} + +/** Core memory hierarchy and shared cache services for attached threads. */ +@instantiable +class Memory(b: MemoryParam, externalHome: Boolean) extends Module { + val core = b + + @public + val io = IO(new Bundle { + val fetch = new VirtualPort(b.memoryAccess) + val data = new VirtualPort(b.memoryAccess) + val timeRequest = Flipped(Decoupled(new UncachedRequest(b.memoryAccess.memory))) + val timeResponse = Decoupled(new UncachedResponse(b.memoryAccess.memory)) + + val threads = Option.when(core.threadPorts > 0)(Vec( + core.threadPorts, + Flipped(new MemoryPort(b.dcache.tracking, b.memoryAccess.walker)) + )) + + val commandAllowed = Input(Bool()) + val uncached = new axi4.Port(b.uncached.axi) + + val coherent = Vec(if (externalHome) 2 else 0, new RequesterPort(core.memoryAccess.memory.chi)) + + val backing = Option.when(!externalHome)(new Bundle { + val request = Decoupled(new LineRequest(core.home.memory.chi)) + val response = Flipped(Decoupled(new LineResponse(core.home.memory.chi))) + }) + + }) + + val system: Instance[MemorySystem] = Instantiate(new MemorySystem(b, externalHome)) + system.io.fetch <> io.fetch + system.io.memory <> io.data + system.io.timeRequest <> io.timeRequest + io.timeResponse <> system.io.timeResponse + system.io.commandAllowed := io.commandAllowed + io.uncached <> system.io.uncached + io.backing.zip(system.io.backing).foreach { case (port, backing) => port <> backing } + io.coherent <> system.io.coherent + + if (core.threadPorts > 0) { + val threads: Instance[ThreadMemory] = + Instantiate(new ThreadMemory(b.threadPorts, b.dcache.tracking, b.memoryAccess.walker)) + threads.io.clients <> io.threads.get + system.io.services <> threads.io.system + } else { + system.io.services.cpuAllow := true.B + system.io.services.maintenance.valid := false.B + system.io.services.maintenance.bits := 0.U.asTypeOf(system.io.services.maintenance.bits) + system.io.services.pteRequest.valid := false.B + system.io.services.pteRequest.bits := 0.U.asTypeOf(system.io.services.pteRequest.bits) + system.io.services.maintained.ready := true.B + system.io.services.pteResponse.ready := true.B + } +} diff --git a/arch/src/main/scala/framework/core/memory/MemorySystem.scala b/arch/src/main/scala/framework/core/memory/MemorySystem.scala new file mode 100644 index 000000000..eaba50a62 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/MemorySystem.scala @@ -0,0 +1,177 @@ +package framework.core.memory + +import framework.lib.arbiter.{Arbiter} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.core.memory.configs.MemoryParam +import framework.core.memory.uncached.Uncached +import framework.core.memory.access.{UncachedRequest, UncachedResponse, VirtualAccess, VirtualPort} +import framework.core.memory.access.cache.Cache +import framework.core.memory.home.Home +import framework.ip.bus.axi4 +import framework.ip.bus.chi.RequesterPort +import framework.ip.bus.chi.snf.{LineRequest, LineResponse} +import framework.ip.bus.chi.rnf.{BankedChiCache, CacheAccess, CacheAtomic} +import framework.lib.queue.Queue + +/** Translation, cache services and ordered cached/uncached reply routing. */ +@instantiable +class MemorySystem(b: MemoryParam, externalHome: Boolean) extends Module { + require(b.uncached.ports == 1, "MemorySystem arbitrates requests into one uncached adapter input") + val core = b + val access = core.memoryAccess.memory + require(b.icache.chi == access.chi, "Instruction cache and memory access CHI must match") + require(b.dcache.cache.chi == access.chi, "Data cache and memory access CHI must match") + if (!externalHome) require(b.home.memory.chi == access.chi, "Local Home and cache fabric CHI must match") + + @public + val io = IO(new Bundle { + val fetch = new VirtualPort(b.memoryAccess) + val memory = new VirtualPort(b.memoryAccess) + val timeRequest = Flipped(Decoupled(new UncachedRequest(b.memoryAccess.memory))) + val timeResponse = Decoupled(new UncachedResponse(b.memoryAccess.memory)) + val services = Flipped(new MemoryPort(b.dcache.tracking, b.memoryAccess.walker)) + val commandAllowed = Input(Bool()) + val uncached = new axi4.Port(b.uncached.axi) + + val coherent = Vec(if (externalHome) 2 else 0, new RequesterPort(access.chi)) + + val backing = Option.when(!externalHome)(new Bundle { + val request = Decoupled(new LineRequest(core.home.memory.chi)) + val response = Flipped(Decoupled(new LineResponse(core.home.memory.chi))) + }) + + }) + + val fetch: Instance[VirtualAccess] = + Instantiate(new VirtualAccess(b.memoryAccess)) + + val memory: Instance[VirtualAccess] = + Instantiate(new VirtualAccess(b.memoryAccess)) + + val icache: Instance[BankedChiCache] = Instantiate(new BankedChiCache(core.icache)) + + val dcache: Instance[Cache] = + Instantiate(new Cache(b.dcache)) + + fetch.io.active := true.B + fetch.io.lookup.config := 0.U.asTypeOf(fetch.io.lookup.config) + fetch.io.lookup.req := 0.U.asTypeOf(fetch.io.lookup.req) + fetch.io.flushTranslation := io.fetch.flushTranslation + fetch.io.request <> io.fetch.request + io.fetch.response <> fetch.io.response + io.fetch.authorizationRequest <> fetch.io.authorizationRequest + fetch.io.authorizationResponse <> io.fetch.authorizationResponse + memory.io.active := true.B + memory.io.lookup.config := 0.U.asTypeOf(memory.io.lookup.config) + memory.io.lookup.req := 0.U.asTypeOf(memory.io.lookup.req) + memory.io.flushTranslation := io.memory.flushTranslation + memory.io.request <> io.memory.request + memory.io.request.valid := io.memory.request.valid && io.commandAllowed + io.memory.request.ready := memory.io.request.ready && io.commandAllowed + io.memory.response <> memory.io.response + io.memory.authorizationRequest <> memory.io.authorizationRequest + memory.io.authorizationResponse <> io.memory.authorizationResponse + val services = io.services + val memoryAllowed = services.cpuAllow + services.cpuQuery := 0.U.asTypeOf(services.cpuQuery) + services.cpuQuery.valid := memory.io.cacheRequest.valid || fetch.io.cacheRequest.valid + services.cpuQuery.paddr := Mux( + memory.io.cacheRequest.valid, + memory.io.cacheRequest.bits.addr, + fetch.io.cacheRequest.bits.addr + ) + services.cpuQuery.sizeLog2 := Mux( + memory.io.cacheRequest.valid && + memory.io.cacheRequest.bits.atomic =/= CacheAtomic.None.U && memory.io.cacheRequest.bits.atomicWord, + 2.U, + 3.U + ) + services.cpuQuery.write := memory.io.cacheRequest.valid && (memory.io.cacheRequest.bits.write || + (memory.io.cacheRequest.bits.atomic =/= CacheAtomic.None.U && memory.io.cacheRequest.bits.atomic =/= CacheAtomic.LR.U)) + + icache.io.access.valid := fetch.io.cacheRequest.valid && memoryAllowed + icache.io.access.bits := fetch.io.cacheRequest.bits + fetch.io.cacheRequest.ready := icache.io.access.ready && memoryAllowed + fetch.io.cacheResponse <> icache.io.result + + val cacheRequests = Instantiate(new Arbiter( + new CacheAccess(access.chi), + 2 + )) + + cacheRequests.io.in(0).valid := memory.io.cacheRequest.valid && memoryAllowed + cacheRequests.io.in(0).bits := memory.io.cacheRequest.bits + memory.io.cacheRequest.ready := cacheRequests.io.in(0).ready && memoryAllowed + cacheRequests.io.in(1).valid := services.pteRequest.valid + cacheRequests.io.in(1).bits := 0.U.asTypeOf(cacheRequests.io.in(1).bits) + cacheRequests.io.in(1).bits.addr := services.pteRequest.bits.addr + services.pteRequest.ready := cacheRequests.io.in(1).ready + + val cacheOwners = Instantiate(new Queue(Bool(), 2, false, false)) + + dcache.io.access.valid := cacheRequests.io.out.valid && cacheOwners.io.enq.ready + dcache.io.access.bits := cacheRequests.io.out.bits + cacheRequests.io.out.ready := dcache.io.access.ready && cacheOwners.io.enq.ready + cacheOwners.io.enq.valid := dcache.io.access.fire + cacheOwners.io.enq.bits := cacheRequests.io.chosen === 1.U + memory.io.cacheResponse.valid := dcache.io.result.valid && cacheOwners.io.deq.valid && !cacheOwners.io.deq.bits + memory.io.cacheResponse.bits := dcache.io.result.bits + services.pteResponse.valid := dcache.io.result.valid && cacheOwners.io.deq.valid && cacheOwners.io.deq.bits + services.pteResponse.bits.data := dcache.io.result.bits.data + services.pteResponse.bits.error := dcache.io.result.bits.error + dcache.io.result.ready := cacheOwners.io.deq.valid && Mux( + cacheOwners.io.deq.bits, + services.pteResponse.ready, + memory.io.cacheResponse.ready + ) + cacheOwners.io.deq.ready := dcache.io.result.fire + dcache.io.maintenance <> services.maintenance + services.maintained <> dcache.io.maintained + dcache.io.olderRequestsDrained := cacheOwners.io.count === 0.U + + val uncachedRequests = Instantiate(new Arbiter( + new UncachedRequest(b.memoryAccess.memory), + 3 + )) + + uncachedRequests.io.in(0) <> fetch.io.uncachedRequest + uncachedRequests.io.in(1) <> memory.io.uncachedRequest + + val uncached: Instance[Uncached] = Instantiate(new Uncached(b.memoryAccess.memory, b.uncached)) + + uncached.io.request(0) <> uncachedRequests.io.out + io.uncached <> uncached.io.axi + uncachedRequests.io.in(2) <> io.timeRequest + val uncachedOwner = Reg(UInt(2.W)) + when(uncachedRequests.io.out.fire)(uncachedOwner := uncachedRequests.io.chosen) + fetch.io.uncachedResponse.valid := uncached.io.response(0).valid && uncachedOwner === 0.U + memory.io.uncachedResponse.valid := uncached.io.response(0).valid && uncachedOwner === 1.U + fetch.io.uncachedResponse.bits := uncached.io.response(0).bits + memory.io.uncachedResponse.bits := uncached.io.response(0).bits + + val timeResponse = io.timeResponse + + timeResponse.valid := uncached.io.response(0).valid && uncachedOwner === 2.U + timeResponse.bits := uncached.io.response(0).bits + uncached.io.response(0).ready := VecInit(Seq( + fetch.io.uncachedResponse.ready, + memory.io.uncachedResponse.ready, + timeResponse.ready + ))(uncachedOwner) + if (externalHome) { + io.coherent(0) <> icache.io.chi + io.coherent(1) <> dcache.io.chi + } else { + val home: Instance[Home] = Instantiate(new Home(b.home)) + home.io.active := true.B + home.io.blockRequesterRsp := false.B + home.io.requesters(0) <> icache.io.chi + home.io.requesters(1) <> dcache.io.chi + home.io.memoryReq <> io.backing.get.request + home.io.memoryResp <> io.backing.get.response + } + +} diff --git a/arch/src/main/scala/framework/core/memory/ThreadMemory.scala b/arch/src/main/scala/framework/core/memory/ThreadMemory.scala new file mode 100644 index 000000000..d8112bef5 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/ThreadMemory.scala @@ -0,0 +1,86 @@ +package framework.core.memory + +import framework.core.memory.access.mmu.{PteRequest, PteResponse} +import framework.ip.memory.interlock.{Acknowledgement, CpuQuery, Maintenance} +import chisel3._ +import framework.ip.memory.interlock.{Params => TrackingParam} +import framework.core.memory.access.mmu.configs.{Params => WalkerParam} +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} + +/** Thread-side cache services; requests retain their original admission tag. */ +class MemoryPort(tracking: TrackingParam, walker: WalkerParam) extends Bundle { + + val cpuQuery = Input(new CpuQuery(tracking)) + val cpuAllow = Output(Bool()) + val maintenance = Decoupled(new Maintenance(tracking)) + val maintained = Flipped(Decoupled(new Acknowledgement(tracking))) + val pteRequest = Decoupled(new PteRequest(walker)) + val pteResponse = Flipped(Decoupled(new PteResponse)) +} + +/** Independent fair, single-owner cache-maintenance and page-table service arbiters. */ +@instantiable +class ThreadMemory(ports: Int, tracking: TrackingParam, walker: WalkerParam) extends Module { + require(ports > 0) + + @public + val io = IO(new Bundle { + val clients = Vec(ports, Flipped(new MemoryPort(tracking, walker))) + val system = new MemoryPort(tracking, walker) + }) + + io.system.cpuAllow := io.clients.map(_.cpuAllow).reduce(_ && _) + io.clients.foreach(_.cpuQuery := io.system.cpuQuery) + + def arbitrate[T <: Data, R <: Data]( + requests: Seq[DecoupledIO[T]], + responses: Seq[DecoupledIO[R]], + request: DecoupledIO[T], + response: DecoupledIO[R] + ): Unit = { + val ownerBits = math.max(1, log2Ceil(ports)) + val next = RegInit(0.U(ownerBits.W)) + val owner = Reg(UInt(ownerBits.W)) + val occupied = RegInit(false.B) + val sent = RegInit(false.B) + val packet = Reg(chiselTypeOf(request.bits)) + val candidates = VecInit(requests.map(_.valid)).asUInt + val later = VecInit(requests.zipWithIndex.map { case (r, i) => r.valid && i.U >= next }).asUInt + val selected = PriorityEncoder(Mux(later.orR, later, candidates)) + for (i <- 0 until ports) { + requests(i).ready := !occupied && selected === i.U && !reset.asBool + responses(i).valid := occupied && sent && owner === i.U && response.valid + responses(i).bits := response.bits + when(requests(i).fire) { + packet := requests(i).bits + owner := i.U + occupied := true.B + sent := false.B + next := (if (i + 1 == ports) 0 else i + 1).U + } + } + request.valid := occupied && !sent && !reset.asBool + request.bits := packet + when(request.fire)(sent := true.B) + response.ready := occupied && sent && VecInit(responses.map(_.ready))(owner) && !reset.asBool + when(response.valid)(assert(occupied && sent, "Ant cache service returned a response without a sent owner")) + when(response.fire) { + occupied := false.B + sent := false.B + } + } + + arbitrate( + io.clients.map(_.maintenance), + io.clients.map(_.maintained), + io.system.maintenance, + io.system.maintained + ) + arbitrate( + io.clients.map(_.pteRequest), + io.clients.map(_.pteResponse), + io.system.pteRequest, + io.system.pteResponse + ) +} diff --git a/arch/src/main/scala/framework/core/memory/access/build.sc b/arch/src/main/scala/framework/core/memory/access/build.sc new file mode 100644 index 000000000..d50a3fe29 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/build.sc @@ -0,0 +1,32 @@ +import mill._ +import $file.src.main.scala.cache.{build => cache_build} +import $file.src.main.scala.mmu.{build => mmu_build} +import $file.src.main.scala.preflight.{build => preflight_build} +import $file.^.^.^.^.^.^.^.settings + + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.core.memory.access.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/PhysicalAccess.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/VirtualAccess.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) ++ + cache_build.sources() ++ + mmu_build.sources() ++ + preflight_build.sources() +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) ++ + cache_build.emitSources() ++ + mmu_build.emitSources() ++ + preflight_build.emitSources() +) diff --git a/arch/src/main/scala/framework/core/memory/access/configs/uvm.toml b/arch/src/main/scala/framework/core/memory/access/configs/uvm.toml new file mode 100644 index 000000000..1df39ea51 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/configs/uvm.toml @@ -0,0 +1,37 @@ +[parameters] +translationEntries=4 +lineBits=512 +[parameters.memory] +tagBits=6 +[parameters.memory.chi] +nodeIdBits=7 +addressBits=44 +dataBits=256 +[[parameters.regions]] +base="4096" +bytes="1048576" +cacheable=true +executable=true +readable=true +writable=true +atomic=true +normal=true +[[parameters.regions]] +base="2097152" +bytes="4096" +cacheable=false +executable=false +readable=true +writable=true +atomic=false +normal=false +[[parameters.regions]] +base="3145728" +bytes="4096" +cacheable=true +executable=false +readable=true +writable=false +atomic=false +normal=true +[defines] diff --git a/arch/src/main/scala/framework/core/memory/access/src/csrc/Cargo.lock b/arch/src/main/scala/framework/core/memory/access/src/csrc/Cargo.lock new file mode 100644 index 000000000..50eddeffd --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/csrc/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "mem-access-reference-model" +version = "0.1.0" diff --git a/arch/src/main/scala/framework/core/memory/access/src/csrc/Cargo.toml b/arch/src/main/scala/framework/core/memory/access/src/csrc/Cargo.toml new file mode 100644 index 000000000..9cd2edfde --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/csrc/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "mem-access-reference-model" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["cdylib"] diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/csrc/src/access.rs b/arch/src/main/scala/framework/core/memory/access/src/csrc/src/access.rs similarity index 95% rename from arch/src/main/scala/framework/mem-core/cpu_mem/src/csrc/src/access.rs rename to arch/src/main/scala/framework/core/memory/access/src/csrc/src/access.rs index 553bfe964..bd840030c 100644 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/csrc/src/access.rs +++ b/arch/src/main/scala/framework/core/memory/access/src/csrc/src/access.rs @@ -1,5 +1,5 @@ #[no_mangle] -pub unsafe extern "C" fn cpu_mem_ref_prepare( +pub unsafe extern "C" fn mem_access_ref_prepare( addr: u64, size: u32, write: u32, data: u64, atomic: u32, cacheable: u32, normal: u32, address_bits: u32, target: *mut u32, misaligned: *mut u32, access_fault: *mut u32, bus_addr: *mut u64, bus_data: *mut u64, bus_mask: *mut u32, atomic_word: *mut u32, @@ -30,7 +30,7 @@ pub unsafe extern "C" fn cpu_mem_ref_prepare( } #[no_mangle] -pub extern "C" fn cpu_mem_ref_result( +pub extern "C" fn mem_access_ref_result( addr: u64, size: u32, write: u32, signed: u32, atomic: u32, cacheable: u32, raw: u64, error: u32, ) -> u64 { diff --git a/arch/src/main/scala/framework/core/memory/access/src/csrc/src/lib.rs b/arch/src/main/scala/framework/core/memory/access/src/csrc/src/lib.rs new file mode 100644 index 000000000..d91305c8d --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/csrc/src/lib.rs @@ -0,0 +1,5 @@ +mod access; +pub use access::*; + +#[path = "../../main/scala/mmu/src/csrc/src/translation.rs"] +mod translation; diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/env/env.svh b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/env/env.svh new file mode 100644 index 000000000..6654b5ae5 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/env/env.svh @@ -0,0 +1,382 @@ +class env extends uvm_env; + `uvm_component_utils(env) + virtual cpu_control_if control; + virtual stream_if #(`ACCESS_REQ_WIDTH) request; + virtual stream_if #(`ACCESS_RESP_WIDTH) response; + virtual stream_if #(`ACCESS_CACHE_WIDTH) cache_request; + virtual stream_if #(`ACCESS_CRESULT_WIDTH) cache_response; + virtual stream_if #(`ACCESS_UNCACHED_WIDTH) uncached_request; + virtual stream_if #(`ACCESS_URESULT_WIDTH) uncached_response; + result_board #(result_item, 0) scoreboard; + int checked = 0, cache_checked = 0, uncached_checked = 0, faults_checked = 0, cancelled = 0; + covergroup behavior with function sample ( + int size, int atomic_op, int target, bit writing, bit fault, bit error + ); + option.per_instance = 1; + sizes: coverpoint size {bins all[] = {[0 : 3]};} + atomics: coverpoint atomic_op {bins all[] = {[0 : 11]};} + targets: coverpoint target {bins all[] = {[0 : 2]};} + direction: coverpoint writing; + faults: coverpoint fault; + errors: coverpoint error; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + behavior = new(); + + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + scoreboard = result_board#(result_item, 0)::type_id::create("scoreboard", this); + if (!uvm_config_db#(virtual cpu_control_if)::get( + this, "", "control", control + ) || !uvm_config_db#(virtual stream_if #(`ACCESS_REQ_WIDTH))::get( + this, "", "request", request + ) || !uvm_config_db#(virtual stream_if #(`ACCESS_RESP_WIDTH))::get( + this, "", "response", response + ) || !uvm_config_db#(virtual stream_if #(`ACCESS_CACHE_WIDTH))::get( + this, "", "cache_request", cache_request + ) || !uvm_config_db#(virtual stream_if #(`ACCESS_CRESULT_WIDTH))::get( + this, "", "cache_response", cache_response + ) || !uvm_config_db#(virtual stream_if #(`ACCESS_UNCACHED_WIDTH))::get( + this, "", "uncached_request", uncached_request + ) || !uvm_config_db#(virtual stream_if #(`ACCESS_URESULT_WIDTH))::get( + this, "", "uncached_response", uncached_response + )) + `uvm_fatal("VIF", "Memory access interfaces missing") + endfunction + task transact(longint unsigned addr, int size, bit write, bit signed_load, longint unsigned data, + int atomic_op, bit cacheable, longint unsigned raw = 64'h80ff7f0181fe8000, + bit error = 0, bit normal_ram = 0, bit device = 0); + req_t packet = '0; + result_item predicted = result_item::type_id::create("predicted"); + result_item received = result_item::type_id::create("received"); + int unsigned target, misaligned, access_fault, bus_mask, atomic_word; + longint unsigned bus_addr, bus_data, expected_data; + bit [`ACCESS_RESP_WIDTH-1:0] expected_response; + int tag = checked & ((1 << `CPU_TAG_BITS) - 1); + mem_access_ref_prepare(addr, size, write, data, atomic_op, cacheable, + !device && (cacheable || normal_ram), `CPU_PHYSICAL_BITS, target, + misaligned, access_fault, bus_addr, bus_data, bus_mask, atomic_word); + expected_data = target == 0 ? 0 : + mem_access_ref_result(addr, size, write, signed_load, atomic_op, cacheable, raw, error); + expected_response = '0; + if (target == 1 && !error) + expected_response[`CPUF(RESP, LINE)] = `CPU_LINE_BITS'(raw) ^ (`CPU_LINE_BITS'(1) << 511); + expected_response[`CPUF(RESP, TAG)] = tag; + expected_response[`CPUF(RESP, DATA)] = expected_data; + expected_response[`CPUF(RESP, MISALIGNED)] = misaligned; + expected_response[`CPUF(RESP, ACCESSFAULT)] = access_fault || (target != 0 && error); + packet[`CPUF(REQ, ADDR)] = addr; + packet[`CPUF(REQ, TAG)] = tag; + packet[`CPUF(REQ, SIZE)] = size; + packet[`CPUF(REQ, WRITE)] = write; + packet[`CPUF(REQ, SIGNED)] = signed_load; + packet[`CPUF(REQ, DATA)] = data; + packet[`CPUF(REQ, ATOMIC)] = atomic_op; + packet[`CPUF(REQ, CACHEABLE)] = cacheable; + packet[`CPUF(REQ, NORMAL)] = !device && (cacheable || normal_ram); + @(control.cb); + response.ready <= 0; + cache_request.ready <= 0; + uncached_request.ready <= 0; + request.bits <= packet; + request.valid <= 1; + do @(request.sample); while (!request.sample.ready); + request.valid <= 0; + predicted.packet = expected_response; + scoreboard.expected_export.write(predicted); + if (target == 1) begin + do @(cache_request.sample); while (!cache_request.sample.valid); + if (cache_request.sample.bits[ + `CPUF(CACHE, ADDR) + ] !== bus_addr || cache_request.sample.bits[ + `CPUF(CACHE, DATA) + ] !== bus_data || cache_request.sample.bits[ + `CPUF(CACHE, MASK) + ] !== bus_mask || cache_request.sample.bits[ + `CPUF(CACHE, WRITE) + ] !== write || cache_request.sample.bits[ + `CPUF(CACHE, ATOMIC) + ] !== atomic_op || cache_request.sample.bits[ + `CPUF(CACHE, ATOMICWORD) + ] !== atomic_word || uncached_request.valid) + `uvm_fatal("CACHE", $sformatf( + "Cache translation mismatch addr=%h size=%0d atomic=%0d got=%h", + addr, + size, + atomic_op, + cache_request.sample.bits + )) + repeat (3) @(control.cb); + cache_request.ready <= 1; + @(cache_request.sample); + cache_request.ready <= 0; + repeat (2) @(control.cb); + cache_response.bits <= '0; + cache_response.bits[ + `CPUF(CRESULT, LINE) + ] <= `CPU_LINE_BITS'(raw) ^ (`CPU_LINE_BITS'(1) << 511); + cache_response.bits[`CPUF(CRESULT, DATA)] <= raw; + cache_response.bits[`CPUF(CRESULT, ERROR)] <= error; + cache_response.valid <= 1; + do @(cache_response.sample); while (!cache_response.sample.ready); + cache_response.valid <= 0; + cache_checked++; + end else if (target == 2) begin + cache_response.bits[`CPUF(CRESULT, DATA)] <= ~raw; + do @(uncached_request.sample); while (!uncached_request.sample.valid); + if (uncached_request.sample.bits[ + `CPUF(UNCACHED, ADDR) + ] !== addr || uncached_request.sample.bits[ + `CPUF(UNCACHED, SIZE) + ] !== size || uncached_request.sample.bits[ + `CPUF(UNCACHED, WRITE) + ] !== write || uncached_request.sample.bits[ + `CPUF(UNCACHED, DATA) + ] !== data || uncached_request.sample.bits[ + `CPUF(UNCACHED, TAG) + ] !== tag || cache_request.valid) + `uvm_fatal("MMIO", "Uncached access expanded or changed its address/size/data/tag") + repeat (3) @(control.cb); + uncached_request.ready <= 1; + @(uncached_request.sample); + uncached_request.ready <= 0; + repeat (2) @(control.cb); + uncached_response.bits <= '0; +`ifdef CPU_BAD_TAG + uncached_response.bits[`CPUF(URESULT, TAG)] <= tag ^ 1; +`else + uncached_response.bits[`CPUF(URESULT, TAG)] <= tag; +`endif + uncached_response.bits[`CPUF(URESULT, DATA)] <= raw; + uncached_response.bits[`CPUF(URESULT, ERROR)] <= error; + uncached_response.valid <= 1; + do @(uncached_response.sample); while (!uncached_response.sample.ready); + uncached_response.valid <= 0; + uncached_checked++; + end else begin + repeat (3) begin + @(control.cb); + if (cache_request.valid || uncached_request.valid) + `uvm_fatal("FAULT", "Faulting request reached an external memory port") + end + faults_checked++; + end + do @(response.sample); while (!response.sample.valid); + if (response.sample.bits !== expected_response) + `uvm_fatal("RESULT", $sformatf( + "CPU result mismatch addr=%h size=%0d atomic=%0d got=%h expected=%h", + addr, + size, + atomic_op, + response.sample.bits, + expected_response + )) + repeat (3) begin + @(control.cb); + if (request.ready || response.bits !== expected_response || !response.valid || + cache_request.valid || uncached_request.valid) + `uvm_fatal("SINGLE", "Outstanding CPU response changed or accepted another operation") + end + response.ready <= 1; + @(response.sample); + response.ready <= 0; + received.packet = response.sample.bits; + scoreboard.actual_export.write(received); + behavior.sample(size, atomic_op, target, write, misaligned || access_fault, error); + checked++; + endtask + // Reset jointly cancels the CPU operation and its memory response producer; stores are not rolled back. + task cancel_at(int phase_index, bit cacheable); + req_t packet = '0; + result_item predicted = result_item::type_id::create("predicted"); + result_item received = result_item::type_id::create("received"); + packet[`CPUF(REQ, ADDR)] = 64'h5000; + packet[`CPUF(REQ, SIZE)] = 3; + packet[`CPUF(REQ, TAG)] = 63; + packet[`CPUF(REQ, CACHEABLE)] = cacheable; + packet[`CPUF(REQ, NORMAL)] = cacheable; + @(control.cb); + request.bits <= packet; + request.valid <= 1; + cache_request.ready <= 0; + uncached_request.ready <= 0; + response.ready <= 0; + do @(request.sample); while (!request.sample.ready); + request.valid <= 0; + predicted.packet = '0; + scoreboard.expected_export.write(predicted); + if (cacheable) begin + do @(cache_request.sample); while (!cache_request.sample.valid); + if (phase_index != 0) begin + cache_request.ready <= 1; + @(cache_request.sample); + cache_request.ready <= 0; + if (phase_index == 2) begin + cache_response.bits <= '0; + cache_response.valid <= 1; + do @(cache_response.sample); while (!cache_response.sample.ready); + cache_response.valid <= 0; + do @(response.sample); while (!response.sample.valid); + end + end + end else begin + do @(uncached_request.sample); while (!uncached_request.sample.valid); + if (phase_index != 0) begin + uncached_request.ready <= 1; + @(uncached_request.sample); + uncached_request.ready <= 0; + if (phase_index == 2) begin + uncached_response.bits <= '0; + uncached_response.bits[`CPUF(URESULT, TAG)] <= 63; + uncached_response.valid <= 1; + do @(uncached_response.sample); while (!uncached_response.sample.ready); + uncached_response.valid <= 0; + do @(response.sample); while (!response.sample.valid); + end + end + end + control.cb.reset <= 1; + scoreboard.cancel_pending(); + cache_response.valid <= 0; + // Payload and VALID are ignored during joint reset, including an unrelated tag. + uncached_response.bits <= '0; + uncached_response.valid <= !cacheable && phase_index == 1; + repeat (2) @(control.cb); + uncached_response.valid <= 0; + control.cb.reset <= 0; + repeat (2) @(control.cb); + if (response.valid || cache_request.valid || uncached_request.valid || !request.ready) + `uvm_fatal("RESET", "Reset retained a cancelled transaction") + cancelled++; + endtask + task execute(); + control.reset = 1; + request.valid = 1; + request.bits = '1; + response.ready = 0; + cache_request.ready = 0; + uncached_request.ready = 0; + cache_response.valid = 0; + cache_response.bits = '0; + uncached_response.valid = 0; + uncached_response.bits = '0; + repeat (4) @(control.cb); + request.valid <= 0; + control.cb.reset <= 0; +`ifdef ACCESS_BAD_CASE + begin + req_t bad = '0; + bad[`CPUF(REQ, SIZE)] = `ACCESS_BAD_CASE == 1 ? 4 : (`ACCESS_BAD_CASE == 4 ? 1 : 3); + bad[`CPUF(REQ, ATOMIC)] = `ACCESS_BAD_CASE == 2 ? 12 : (`ACCESS_BAD_CASE == 3 ? 15 : 1); + bad[`CPUF(REQ, WRITE)] = `ACCESS_BAD_CASE == 5; + bad[`CPUF(REQ, CACHEABLE)] = 1; + @(control.cb); + request.bits <= bad; + request.valid <= 1; + repeat (10) @(control.cb); + `uvm_fatal("NO_ASSERT", "Invalid Memory access contract was accepted") + end +`elsif CPU_BAD_TAG + transact(64'h60020000, 0, 0, 0, 0, 0, 0); + `uvm_fatal("NO_ASSERT", "Incorrect uncached response tag was accepted") +`else + for (int phase_index = 0; phase_index < 3; phase_index++) + for (int cacheable = 0; cacheable < 2; cacheable++) cancel_at(phase_index, cacheable); + for (int cacheable = 0; cacheable < 2; cacheable++) + for (int size = 0; size < 4; size++) + for (int offset = 0; offset < 8; offset++) + for (int sign_bit = 0; sign_bit < 2; sign_bit++) begin + transact(64'h60020000 + offset, size, 0, sign_bit, 0, 0, cacheable); + transact(64'h60020000 + offset, size, 1, 0, 64'hfedcba9876543210, 0, cacheable); + end + // Every aligned upper/lower Word atomic lane, including sign-expanded results and SC status. + for (int op = 1; op <= 11; op++) + for (int size = 2; size < 4; size++) + for (int offset = 0; offset < 8; offset += 4) begin + longint unsigned raw = op == 11 ? 1 : (size == 2 ? 64'hffffffff80000081 : 64'h8123456789abcdef); + transact(64'h1000 + offset, size, 0, 0, 64'h8877665544332211, op, 1, raw); + transact(64'h1000 + offset, size, 0, 1, 64'h1122334455667788, op, 1, raw, 1); + transact(64'h1000 + offset, size, 0, 0, 1, op, 0, raw); + end + for (int size = 2; size < 4; size++) + for (int offset = 1; offset < 8; offset++) + if (offset % (1 << size) != 0) transact(64'h1000 + offset, size, 0, 0, 1, 1, 1); + for (int size = 0; size < 4; size++) begin + transact(64'h2000, size, 0, 1, 0, 0, 1, '1); + transact(64'h2000, size, 0, 0, 0, 0, 1, 0); + transact(64'h2000, size, 0, 0, 0, 0, 1, '1, 1); + transact(64'h60020004, size, 0, 1, 0, 0, 0, '1, 1); + transact(64'h60020004, size, 1, 0, '1, 0, 0, 0, 1); + transact(64'h60020000, size, 0, 1, 0, 0, 0, '1); + transact(64'h60020000, size, 0, 0, 0, 0, 0, '1); + transact(64'h60020000, size, 1, 0, '1, 0, 0, 0, 1); + end + for (int bit_index = 3; bit_index < 64; bit_index++) begin + transact(64'h1 << bit_index, 3, 0, 0, 0, 0, 1); + transact(64'h1 << bit_index, 3, 1, 0, '1, 0, 0); + end + transact(64'h1004, 2, 0, 1, 0, 0, 1, 0); + transact(64'h60020004, 2, 0, 1, 0, 0, 0, 0); + transact(64'h1000, 3, 1, 0, 0, 0, 1, 0, 1); + transact(64'h1000, 2, 0, 0, 1, 11, 1, 0); + transact(64'h1004, 2, 0, 0, 1, 11, 1, 0); + transact(64'h1000, 3, 0, 0, 1, 11, 1, 0); + transact(64'h1000, 3, 1, 0, 0, 0, 1); + transact(64'h1000, 3, 1, 0, '1, 0, 1); + transact((64'h1 << `CPU_PHYSICAL_BITS) - 8, 3, 0, 0, 0, 0, 1); + transact((64'h1 << `CPU_PHYSICAL_BITS) - 1, 0, 1, 0, '1, 0, 1); + // Noncacheable normal memory, including every atomic, faults without issuing traffic. + for (int size = 0; size < 4; size++) begin + transact(64'h3000, size, 0, 0, 0, 0, 0, 0, 0, 1); + transact(64'h3000, size, 1, 0, 1, 0, 0, 0, 0, 1); + transact(64'h3001, size, 0, 0, 0, 0, 0, 0, 0, 1); + transact(64'h60020000, size, 0, 0, 0, 0, 1, 0, 0, 0, 1); + transact(64'h60020000, size, 1, 0, 1, 0, 1, 0, 0, 0, 1); + transact(64'h60020001, size, 0, 0, 0, 0, 1, 0, 0, 0, 1); + end + for (int op = 1; op <= 11; op++) + for (int size = 2; size <= 3; size++) begin + transact(64'h3000 + (size == 2 ? 4 : 0), size, 0, 0, 64'h76543210, op, 0, + op == 11 ? 0 : 64'hffffffff81234567, 0, 1); + transact(64'h3000, size, 0, 0, 1, op, 0, 64'hdeadbeef12345678, 1, 1); + end + transact(64'h3004, 2, 0, 0, 1, 11, 0, 1, 0, 1); + // Reset ignores even VALID=1 garbage payload; no transaction may escape reset. + control.cb.reset <= 1; + request.valid <= 1; + request.bits <= '1; + repeat (2) @(control.cb); + request.valid <= 0; + request.bits <= '0; + control.cb.reset <= 0; + repeat (2) @(control.cb); + if (response.valid || cache_request.valid || uncached_request.valid || !request.ready) + `uvm_fatal("RESET_PAYLOAD", "Reset payload escaped as a transaction") + // VALID=0 payload is unconstrained; it must not create an operation. + response.ready <= 1; + cache_request.ready <= 1; + uncached_request.ready <= 1; + request.bits <= '1; + cache_response.bits <= '1; + uncached_response.bits <= '1; + repeat (2) @(control.cb); + request.bits <= '0; + cache_response.bits <= '0; + + uncached_response.bits <= '0; + repeat (2) @(control.cb); + if (response.valid || cache_request.valid || uncached_request.valid || !request.ready) + `uvm_fatal("IDLE", "Idle payload generated an operation") + `uvm_info("MEM_ACCESS", $sformatf( + "Checked %0d CPU results, %0d cache requests, %0d exact uncached requests, %0d suppressed faults, %0d joint-reset cancellations", + checked, + cache_checked, + uncached_checked, + faults_checked, + cancelled + ), UVM_LOW) +`endif + endtask +endclass diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/interface.sv b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/interface.sv similarity index 100% rename from arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/interface.sv rename to arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/interface.sv diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access.f b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access.f new file mode 100644 index 000000000..a8de5d1cb --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access.f @@ -0,0 +1,10 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core + +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/core/stream_if.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/PhysicalAccess/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_atomic.f b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_atomic.f new file mode 100644 index 000000000..2d2363053 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_atomic.f @@ -0,0 +1,11 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core ++define+MEM_ACCESS_TOP=mem_access_bad_atomic_tb ++define+ACCESS_BAD_CASE=3 +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/core/stream_if.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/PhysicalAccess/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_atomic_write.f b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_atomic_write.f new file mode 100644 index 000000000..03c2ec8a5 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_atomic_write.f @@ -0,0 +1,11 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core ++define+MEM_ACCESS_TOP=mem_access_bad_atomic_write_tb ++define+ACCESS_BAD_CASE=5 +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/core/stream_if.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/PhysicalAccess/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_fence.f b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_fence.f new file mode 100644 index 000000000..5a8b53b8e --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_fence.f @@ -0,0 +1,11 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core ++define+MEM_ACCESS_TOP=mem_access_bad_fence_tb ++define+ACCESS_BAD_CASE=2 +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/core/stream_if.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/PhysicalAccess/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_operand.f b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_operand.f new file mode 100644 index 000000000..4f160b666 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_operand.f @@ -0,0 +1,11 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core ++define+MEM_ACCESS_TOP=mem_access_bad_operand_tb ++define+ACCESS_BAD_CASE=4 +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/core/stream_if.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/PhysicalAccess/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_size.f b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_size.f new file mode 100644 index 000000000..45365a4d6 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_size.f @@ -0,0 +1,11 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core ++define+MEM_ACCESS_TOP=mem_access_bad_size_tb ++define+ACCESS_BAD_CASE=1 +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/core/stream_if.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/PhysicalAccess/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_tag.f b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_tag.f new file mode 100644 index 000000000..4514b69b2 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_tag.f @@ -0,0 +1,11 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core ++define+MEM_ACCESS_TOP=mem_access_bad_tag_tb ++define+CPU_BAD_TAG +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/core/stream_if.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/PhysicalAccess/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/package.sv b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/package.sv new file mode 100644 index 000000000..ad7579ba0 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/package.sv @@ -0,0 +1,46 @@ +`include "physical_parameters.svh" +package mem_access_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + `define CPUF(kind, field) `CPU_``kind``_``field``_OFFSET +: `CPU_``kind``_``field``_WIDTH + typedef bit [`ACCESS_REQ_WIDTH-1:0] req_t; + import "DPI-C" function void mem_access_ref_prepare( + input longint unsigned addr, + input int unsigned size, + write, + input longint unsigned data, + input int unsigned atomic_op, + cacheable, + normal, + address_bits, + output int unsigned target, + misaligned, + access_fault, + output longint unsigned bus_addr, + bus_data, + output int unsigned bus_mask, + atomic_word + ); + import "DPI-C" function longint unsigned mem_access_ref_result( + input longint unsigned addr, + input int unsigned size, + write, + signed_load, + atomic_op, + cacheable, + input longint unsigned raw, + input int unsigned error + ); + class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(packet, UVM_ALL_ON) + `uvm_object_utils_end + logic [`ACCESS_RESP_WIDTH-1:0] packet; + function new(string name = "result"); + super.new(name); + endfunction + endclass + `include "env/env.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/tb.sv b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/tb.sv new file mode 100644 index 000000000..90e705daf --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/tb.sv @@ -0,0 +1,89 @@ +`include "physical_parameters.svh" +`ifndef MEM_ACCESS_TOP +`define MEM_ACCESS_TOP mem_access_tb +`endif +module `MEM_ACCESS_TOP; + import uvm_pkg::*; + import mem_access_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + cpu_control_if control (clock); + stream_if #(`ACCESS_REQ_WIDTH) request ( + clock, + control.reset + ); + stream_if #(`ACCESS_RESP_WIDTH) response ( + clock, + control.reset + ); + stream_if #(`ACCESS_CACHE_WIDTH) cache_request ( + clock, + control.reset + ); + stream_if #(`ACCESS_CRESULT_WIDTH) cache_response ( + clock, + control.reset + ); + stream_if #(`ACCESS_UNCACHED_WIDTH) uncached_request ( + clock, + control.reset + ); + stream_if #(`ACCESS_URESULT_WIDTH) uncached_response ( + clock, + control.reset + ); + int unsigned in_flight = 0; + bit issued = 0, returned = 0, uncached = 0; + always @(posedge clock) begin + if (control.reset) begin + in_flight = 0; + issued = 0; + returned = 0; + end else begin + if (in_flight != 0 && request.ready) + $fatal(1, "CPU port became ready before consuming its response"); + if (request.valid && request.ready) begin + if (in_flight != 0) $fatal(1, "Memory access accepted overlapping requests"); + in_flight = 1; + issued = 0; + returned = 0; + end + if ((cache_request.valid && cache_request.ready) || (uncached_request.valid && uncached_request.ready)) begin + if (in_flight != 1 || issued) $fatal(1, "Unexpected or duplicate backend request"); + issued = 1; + uncached = uncached_request.valid && uncached_request.ready; + end + if ((cache_response.valid && cache_response.ready) || (uncached_response.valid && uncached_response.ready)) begin + if (in_flight != 1 || !issued || returned || + uncached != (uncached_response.valid && uncached_response.ready)) + $fatal(1, "Unexpected or duplicate backend response"); + returned = 1; + end + if (response.valid && response.ready) begin + if (in_flight != 1 || (issued && !returned)) $fatal(1, "Unexpected CPU result"); + in_flight = 0; + end + end + end + PhysicalAccess dut ( + .clock(clock), + .reset(control.reset), + `include "physical_ports.svh" + ); + initial begin + uvm_config_db#(virtual cpu_control_if)::set(null, "uvm_test_top*", "control", control); + uvm_config_db#(virtual stream_if #(`ACCESS_REQ_WIDTH))::set(null, "uvm_test_top*", "request", + request); + uvm_config_db#(virtual stream_if #(`ACCESS_RESP_WIDTH))::set(null, "uvm_test_top*", "response", + response); + uvm_config_db#(virtual stream_if #(`ACCESS_CACHE_WIDTH))::set(null, "uvm_test_top*", + "cache_request", cache_request); + uvm_config_db#(virtual stream_if #(`ACCESS_CRESULT_WIDTH))::set( + null, "uvm_test_top*", "cache_response", cache_response); + uvm_config_db#(virtual stream_if #(`ACCESS_UNCACHED_WIDTH))::set( + null, "uvm_test_top*", "uncached_request", uncached_request); + uvm_config_db#(virtual stream_if #(`ACCESS_URESULT_WIDTH))::set( + null, "uvm_test_top*", "uncached_response", uncached_response); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/tests/test.svh b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/tests/test.svh new file mode 100644 index 000000000..398043c00 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/tests/test.svh @@ -0,0 +1,14 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + test_env.execute(); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/suite.toml b/arch/src/main/scala/framework/core/memory/access/src/main/resources/suite.toml new file mode 100644 index 000000000..2807cc85a --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/suite.toml @@ -0,0 +1,139 @@ +[model] +manifest="src/csrc/Cargo.toml" +library="libmem_access_reference_model" + +[[tests]] +name="virtual_access" +top="virtual_access_tb" +test="protocol_test" +filelist="virtual/filelist.f" +timeout_ns=500000 +wall_timeout_s=120 +kind="functional" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=["virtual_pkg::model::behavior"] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=[] +assertion="" + +[[tests]] +functional_groups = ["mem_access_pkg::env::behavior"] +name = "mem_access" +top = "mem_access_tb" +test = "protocol_test" +filelist = "physical/mem_access.f" +timeout_ns = 200000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +functional_groups = [] +name = "mem_access_bad_atomic" +top = "mem_access_bad_atomic_tb" +test = "protocol_test" +filelist = "physical/mem_access_bad_atomic.f" +timeout_ns = 200000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "Memory access port does not accept Fence or unknown atomics" + +[[tests]] +functional_groups = [] +name = "mem_access_bad_atomic_write" +top = "mem_access_bad_atomic_write_tb" +test = "protocol_test" +filelist = "physical/mem_access_bad_atomic_write.f" +timeout_ns = 200000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "Memory atomic requires a 32 or 64 bit operand" + +[[tests]] +functional_groups = [] +name = "mem_access_bad_fence" +top = "mem_access_bad_fence_tb" +test = "protocol_test" +filelist = "physical/mem_access_bad_fence.f" +timeout_ns = 200000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "Memory access port does not accept Fence or unknown atomics" + +[[tests]] +functional_groups = [] +name = "mem_access_bad_operand" +top = "mem_access_bad_operand_tb" +test = "protocol_test" +filelist = "physical/mem_access_bad_operand.f" +timeout_ns = 200000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "Memory atomic requires a 32 or 64 bit operand" + +[[tests]] +functional_groups = [] +name = "mem_access_bad_size" +top = "mem_access_bad_size_tb" +test = "protocol_test" +filelist = "physical/mem_access_bad_size.f" +timeout_ns = 200000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "Memory access size must be 1, 2, 4 or 8 bytes" + +[[tests]] +functional_groups = [] +name = "mem_access_bad_tag" +top = "mem_access_bad_tag_tb" +test = "protocol_test" +filelist = "physical/mem_access_bad_tag.f" +timeout_ns = 200000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "Uncached response tag mismatch" diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/agent/driver.svh b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/agent/driver.svh new file mode 100644 index 000000000..e0c289e6b --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/agent/driver.svh @@ -0,0 +1,25 @@ +class driver extends uvm_driver #(cycle_item); + `uvm_component_utils(driver) + virtual virtual_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual virtual_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "TSS interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item req; + forever begin + seq_item_port.get_next_item(req); + @(negedge vif.clock); + vif.reset = req.reset; + vif.source = req.inputs; + @(posedge vif.clock); + req.outputs = vif.outputs; + #1; + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/agent/fixture.svh b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/agent/fixture.svh new file mode 100644 index 000000000..c38fe090e --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/agent/fixture.svh @@ -0,0 +1,19 @@ +class fixture extends uvm_object; + `uvm_object_utils(fixture) + logic [63:0] ptes[longint unsigned]; + bit errors[longint unsigned]; + bit deny_pte, deny_data, backend_error; + longint unsigned authorized_pte; + function new(string name = "fixture"); + super.new(name); + endfunction + function logic [63:0] word(longint unsigned address); + return 64'h80ff7f0181fe8000 ^ (address * 64'h0101010101010101); + endfunction + function logic [`VA_LINE_BITS-1:0] line(longint unsigned address); + logic [`VA_LINE_BITS-1:0] value; + for (int i = 0; i < `VA_LINE_BITS / 64; i++) + value[i*64+:64] = word((address & ~64'd63) + i * 8); + return value; + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/agent/item.svh b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/agent/item.svh new file mode 100644 index 000000000..1179c85f6 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/agent/item.svh @@ -0,0 +1,26 @@ +class cycle_item extends uvm_sequence_item; + `uvm_object_utils(cycle_item) + logic [`VA_INPUT_BITS-1:0] inputs; + logic [`VA_OUTPUT_BITS-1:0] outputs; + bit reset; + function new(string name = "cycle_item"); + super.new(name); + endfunction +endclass +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(tag, UVM_ALL_ON) + `uvm_field_int(data, UVM_ALL_ON) + `uvm_field_int(line, UVM_ALL_ON) + `uvm_field_int(misaligned, UVM_ALL_ON) + `uvm_field_int(page_fault, UVM_ALL_ON) + `uvm_field_int(access_fault, UVM_ALL_ON) + `uvm_object_utils_end + int tag; + bit [63:0] data; + bit [`VA_LINE_BITS-1:0] line; + bit misaligned, page_fault, access_fault; + function new(string name = "result_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/agent/monitor.svh b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/agent/monitor.svh new file mode 100644 index 000000000..46c8093ac --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/agent/monitor.svh @@ -0,0 +1,25 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual virtual_if vif; + uvm_analysis_port #(cycle_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual virtual_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "TSS monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item t; + forever begin + @(posedge vif.clock); + t = cycle_item::type_id::create("observed"); + t.inputs = vif.inputs; + t.outputs = vif.outputs; + t.reset = vif.reset; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/agent/peer.svh b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/agent/peer.svh new file mode 100644 index 000000000..bf81965cf --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/agent/peer.svh @@ -0,0 +1,68 @@ +class peer extends uvm_component; + `uvm_component_utils(peer) + virtual virtual_if vif; + fixture image; + bit auth_pending, cache_pending, uncached_pending; + bit auth_allow, cache_error; + logic [63:0] cache_data, uncached_data; + logic [`VA_LINE_BITS-1:0] cache_line; + int tag, cycle, auth_due, cache_due, uncached_due; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual virtual_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Virtual access interface is required") + endfunction + task run_phase(uvm_phase phase); + forever begin + @(posedge vif.clock); + cycle++; + if (vif.reset) begin + auth_pending = 0; + cache_pending = 0; + uncached_pending = 0; + end else begin + if (`VO(AUTHORIZATIONRESPONSE_READY) && `VI(AUTHORIZATIONRESPONSE_VALID)) auth_pending = 0; + if (`VO(CACHERESPONSE_READY) && `VI(CACHERESPONSE_VALID)) cache_pending = 0; + if (`VO(UNCACHEDRESPONSE_READY) && `VI(UNCACHEDRESPONSE_VALID)) uncached_pending = 0; + if (`VO(AUTHORIZATIONREQUEST_VALID) && `VI(AUTHORIZATIONREQUEST_READY)) begin + bit pte = `VO(AUTHORIZATIONREQUEST_BITS_ISPTE); + auth_pending = 1; + auth_due = cycle + 3; + auth_allow = pte ? !image.deny_pte : !image.deny_data; + end + if (`VO(CACHEREQUEST_VALID) && `VI(CACHEREQUEST_READY)) begin + longint unsigned address = `VO(CACHEREQUEST_BITS_ADDR); + cache_pending = 1; + cache_due = cycle + 4; + cache_data = image.ptes.exists(address) ? image.ptes[address] : image.word(address); + cache_error = image.ptes.exists(address) ? image.errors[address] : image.backend_error; + cache_line = image.line(address); + end + if (`VO(UNCACHEDREQUEST_VALID) && `VI(UNCACHEDREQUEST_READY)) begin + uncached_pending = 1; + uncached_due = cycle + 4; + uncached_data = image.word(`VO(UNCACHEDREQUEST_BITS_ADDR)); + tag = `VO(UNCACHEDREQUEST_BITS_TAG); + end + end + @(negedge vif.clock); + vif.backend = '0; + vif.backend[`VA_IN_AUTHORIZATIONREQUEST_READY_OFFSET+:`VA_IN_AUTHORIZATIONREQUEST_READY_WIDTH]=cycle%4!=0; + vif.backend[`VA_IN_AUTHORIZATIONRESPONSE_VALID_OFFSET+:`VA_IN_AUTHORIZATIONRESPONSE_VALID_WIDTH]=auth_pending&&cycle>=auth_due; + vif.backend[`VA_IN_AUTHORIZATIONRESPONSE_BITS_OFFSET+:`VA_IN_AUTHORIZATIONRESPONSE_BITS_WIDTH]=auth_allow; + vif.backend[`VA_IN_CACHEREQUEST_READY_OFFSET+:`VA_IN_CACHEREQUEST_READY_WIDTH]=cycle%5!=0; + vif.backend[`VA_IN_UNCACHEDREQUEST_READY_OFFSET+:`VA_IN_UNCACHEDREQUEST_READY_WIDTH]=cycle%3!=0; + vif.backend[`VA_IN_CACHERESPONSE_VALID_OFFSET+:`VA_IN_CACHERESPONSE_VALID_WIDTH]=cache_pending&&cycle>=cache_due; + vif.backend[`VA_IN_CACHERESPONSE_BITS_DATA_OFFSET+:`VA_IN_CACHERESPONSE_BITS_DATA_WIDTH]=cache_data; + vif.backend[`VA_IN_CACHERESPONSE_BITS_ERROR_OFFSET+:`VA_IN_CACHERESPONSE_BITS_ERROR_WIDTH]=cache_error; + vif.backend[`VA_IN_CACHERESPONSE_BITS_LINE_OFFSET+:`VA_IN_CACHERESPONSE_BITS_LINE_WIDTH]=cache_line; + vif.backend[`VA_IN_UNCACHEDRESPONSE_VALID_OFFSET+:`VA_IN_UNCACHEDRESPONSE_VALID_WIDTH]=uncached_pending&&cycle>=uncached_due; + vif.backend[`VA_IN_UNCACHEDRESPONSE_BITS_DATA_OFFSET+:`VA_IN_UNCACHEDRESPONSE_BITS_DATA_WIDTH]=uncached_data; + vif.backend[`VA_IN_UNCACHEDRESPONSE_BITS_TAG_OFFSET+:`VA_IN_UNCACHEDRESPONSE_BITS_TAG_WIDTH]=tag; + vif.backend[`VA_IN_UNCACHEDRESPONSE_BITS_ERROR_OFFSET+:`VA_IN_UNCACHEDRESPONSE_BITS_ERROR_WIDTH]=image.backend_error; + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/env/env.svh b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/env/env.svh new file mode 100644 index 000000000..26a25d996 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/env/env.svh @@ -0,0 +1,36 @@ +class env extends uvm_env; + `uvm_component_utils(env) + uvm_analysis_imp #(uvm_sequence_item, env) reset_events; + driver source; + monitor mon; + peer backend; + model reference; + uvm_sequencer #(cycle_item) seqr; + result_board #(result_item, 0) scoreboard; + function new(string name, uvm_component parent); + super.new(name, parent); + reset_events = new("reset_events", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + source = driver::type_id::create("source", this); + mon = monitor::type_id::create("monitor", this); + backend = peer::type_id::create("backend", this); + reference = model::type_id::create("model", this); + seqr = new("seqr", this); + scoreboard = result_board#(result_item, 0)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + fixture image = fixture::type_id::create("image"); + reference.image = image; + backend.image = image; + source.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(reference.analysis_export); + reference.expected.connect(scoreboard.expected_export); + reference.actual.connect(scoreboard.actual_export); + reference.cancelled.connect(reset_events); + endfunction + function void write(uvm_sequence_item t); + scoreboard.cancel_pending(); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/env/model.svh b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/env/model.svh new file mode 100644 index 000000000..8d9c3283d --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/env/model.svh @@ -0,0 +1,219 @@ +class model extends uvm_subscriber #(cycle_item); + `uvm_component_utils(model) + fixture image; + chandle walker; + uvm_analysis_port #(result_item) expected, actual; + uvm_analysis_port #(uvm_sequence_item) cancelled; + typedef struct { + longint unsigned pa; + int size; + bit reading, writing, executing, pte; + int privilege; + } authorization; + typedef struct { + longint unsigned address, data; + int mask, atomic_op, word; + bit writing, pte, uncached; + int tag, size; + } transfer; + authorization authorizations[$]; + transfer transfers[$]; + bit pending; + covergroup behavior with function sample ( + int size, + int privilege, + int mode, + int target, + bit write, + int atomic_op, + bit misaligned, + bit pf, + bit af, + bit denied, + bit error + ); + option.per_instance = 1; + sizes: coverpoint size {bins all[] = {[0 : 3]};} + privileges: coverpoint privilege {bins all[] = {0, 1, 3};} + modes: coverpoint mode {bins all[] = {0, 8};} + targets: coverpoint target {bins none = {0}; bins cached = {1}; bins uncached = {2};} + direction: coverpoint write; + atomics: coverpoint atomic_op { + bins ordinary = {0}; bins lr = {10}; bins sc = {11}; bins other = {[1 : 9]}; + } + alignment: coverpoint misaligned; + page_faults: coverpoint pf; + access_faults: coverpoint af; + permission: coverpoint denied; + backend_errors: coverpoint error; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected = new("expected", this); + actual = new("actual", this); + cancelled = new("cancelled", this); + behavior = new(); + walker = mmu_ref_create(`VA_ADDRESS_BITS, 0); + endfunction + function void verify(bit ok, string message); + evidence::checks++; + if (!ok) `uvm_fatal("VIRTUAL_ACCESS", message) + endfunction + function void set_pte(longint unsigned address, logic [63:0] value, bit error = 0); + image.ptes[address] = value; + image.errors[address] = error; + mmu_ref_program(walker, address, value, error); + endfunction + function void predict(cycle_item t); + longint unsigned va, pa, root, data, bus_address, bus_data, raw, pte_address; + int size, tag, privilege, mode, atomic_op, target, misaligned, af, pf, level, mask, atomic_word; + bit writing, executing, signed_load, read_data, write_data, region_ok, cacheable, normal; + result_item result = result_item::type_id::create("expected"); + va = `MI(t, REQUEST_BITS_VADDR); + size = `MI(t, REQUEST_BITS_SIZE); + tag = `MI(t, REQUEST_BITS_TAG); + privilege = `MI(t, REQUEST_BITS_PRIVILEGE); + mode = `MI(t, REQUEST_BITS_SATPMODE); + root = `MI(t, REQUEST_BITS_ROOTPPN); + data = `MI(t, REQUEST_BITS_DATA); + writing = `MI(t, REQUEST_BITS_WRITE); + executing = `MI(t, REQUEST_BITS_EXECUTE); + signed_load = `MI(t, REQUEST_BITS_SIGNED); + atomic_op = `MI(t, REQUEST_BITS_ATOMIC); + result.tag = tag; + misaligned = (va & ((64'd1 << size) - 1)) != 0; + result.misaligned = misaligned; + target = 0; + pf = 0; + af = 0; + verify(!pending && authorizations.size() == 0 && transfers.size() == 0, + "Overlapping virtual requests"); + if (!misaligned) begin + // The fixture denies a PTE through the authorization boundary, before cache access. + foreach (image.ptes[address]) + mmu_ref_program(walker, address, image.ptes[address], + image.errors[address] || image.deny_pte); + verify(mmu_ref_translate( + walker, + va, + mode, + root, + privilege, + writing || (atomic_op != 0 && atomic_op != 10), + executing, + `MI(t, REQUEST_BITS_SUM), + `MI(t, REQUEST_BITS_MXR), + pa, + pf, + af, + level + ) == 0, "Missing page table fixture"); + while (mmu_ref_peek( + walker, pte_address + ) != 0) begin + authorizations.push_back('{pte_address, 3, 1, 0, 0, 1, 1}); + if (!image.deny_pte) transfers.push_back('{pte_address, 0, 0, 0, 0, 0, 1, 0, 0, 3}); + verify(mmu_ref_consume(walker, pte_address) != 0, "PTE trace consumption failed"); + end + result.page_fault = pf; + result.access_fault = af; + if (!pf && !af) begin + longint unsigned last = pa + ((64'd1 << size) - 1); + read_data = !executing && !writing && atomic_op != 11; + write_data = writing || (atomic_op != 0 && atomic_op != 10); + cacheable = pa >= 4096 && last < 1052672 || pa >= 3145728 && last < 3149824; + normal = cacheable; + region_ok=(pa>=4096&&last<1052672)|| +(pa>=2097152&&last<2101248&&!executing&&atomic_op==0)|| +(pa>=3145728&&last<3149824&&!executing&&!write_data&&atomic_op==0); + authorizations.push_back('{pa, size, read_data, write_data, executing, 0, privilege}); + if (region_ok && !image.deny_data) begin + mem_access_ref_prepare(pa, size, writing, data, atomic_op, cacheable, normal, + `VA_ADDRESS_BITS, target, misaligned, af, bus_address, bus_data, + mask, atomic_word); + verify(target != 0 && !af && !misaligned, "Invalid physical reference plan"); + transfers.push_back('{bus_address, bus_data, mask, atomic_op, atomic_word, writing, 0, + target == 2, tag, size}); + raw = image.word(bus_address); + result.data = mem_access_ref_result(pa, size, writing, signed_load, atomic_op, cacheable, + raw, image.backend_error); + result.access_fault = image.backend_error; + if (target == 1 && !image.backend_error) result.line = image.line(bus_address); + end else result.access_fault = 1; + end + end + expected.write(result); + pending = 1; + behavior.sample(size, privilege, mode, target, writing, atomic_op, misaligned, pf, + result.access_fault, image.deny_pte || image.deny_data, image.backend_error); + endfunction + function void write(cycle_item t); + if (t.reset) begin + cancelled.write(null); + pending = 0; + authorizations.delete(); + transfers.delete(); + mmu_ref_reset(walker); + return; + end + if (`MI(t, REQUEST_VALID) && `MO(t, REQUEST_READY)) predict(t); + if (`MO(t, AUTHORIZATIONREQUEST_VALID)) begin + authorization a; + verify(authorizations.size() != 0, "Unexpected authorization request"); + a = authorizations[0]; + verify(`MO(t, AUTHORIZATIONREQUEST_BITS_PADDR) == a.pa && + `MO(t, AUTHORIZATIONREQUEST_BITS_SIZE) == a.size && + `MO(t, AUTHORIZATIONREQUEST_BITS_READ) == a.reading && + `MO(t, AUTHORIZATIONREQUEST_BITS_WRITE) == a.writing && + `MO(t, AUTHORIZATIONREQUEST_BITS_EXECUTE) == a.executing && + `MO(t, AUTHORIZATIONREQUEST_BITS_ISPTE) == a.pte && + `MO(t, AUTHORIZATIONREQUEST_BITS_PRIVILEGE) == a.privilege, + "Authorization footprint mismatch"); + if (`MI(t, AUTHORIZATIONREQUEST_READY)) void'(authorizations.pop_front()); + end + if (`MO(t, CACHEREQUEST_VALID) || `MO(t, UNCACHEDREQUEST_VALID)) begin + transfer a; + bit uncached; + verify(transfers.size() != 0, "Speculative backend request without authorization"); + a = transfers[0]; + uncached = `MO(t, UNCACHEDREQUEST_VALID); + verify(uncached == a.uncached, "Cache/MMIO target mismatch"); + if (uncached) begin + verify(`MO(t, UNCACHEDREQUEST_BITS_ADDR) == a.address && `MO(t, UNCACHEDREQUEST_BITS_SIZE) + == a.size && `MO(t, UNCACHEDREQUEST_BITS_TAG) == a.tag && + `MO(t, UNCACHEDREQUEST_BITS_WRITE) == a.writing && `MO(t, UNCACHEDREQUEST_BITS_DATA) + == a.data, "Uncached request mismatch"); + if (`MI(t, UNCACHEDREQUEST_READY)) void'(transfers.pop_front()); + end else begin + verify(`MO(t, CACHEREQUEST_BITS_ADDR) == a.address && `MO(t, CACHEREQUEST_BITS_WRITE) + == a.writing && `MO(t, CACHEREQUEST_BITS_DATA) == a.data && + `MO(t, CACHEREQUEST_BITS_MASK) == a.mask && `MO(t, CACHEREQUEST_BITS_ATOMIC) + == a.atomic_op && `MO(t, CACHEREQUEST_BITS_ATOMICWORD) == a.word, + "Cache/PTE request mismatch"); + if (`MI(t, CACHEREQUEST_READY)) void'(transfers.pop_front()); + end + end + if (`MO(t, RESPONSE_VALID) && `MI(t, RESPONSE_READY)) begin + result_item result = result_item::type_id::create("actual"); + verify(pending && authorizations.size() == 0 && transfers.size() == 0, + "Early result before the planned accesses drained"); + result.tag = `MO(t, RESPONSE_BITS_TAG); + result.data = `MO(t, RESPONSE_BITS_DATA); + result.line = `MO(t, RESPONSE_BITS_LINE); + result.misaligned = `MO(t, RESPONSE_BITS_MISALIGNED); + result.page_fault = `MO(t, RESPONSE_BITS_PAGEFAULT); + result.access_fault = `MO(t, RESPONSE_BITS_ACCESSFAULT); + actual.write(result); + pending = 0; + end + endfunction + function void check_phase(uvm_phase phase); + super.check_phase(phase); + verify(!pending && authorizations.size() == 0 && transfers.size() == 0, + "Virtual access did not drain"); + endfunction + function void final_phase(uvm_phase phase); + mmu_ref_destroy(walker); + super.final_phase(phase); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/filelist.f b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/filelist.f new file mode 100644 index 000000000..d2b648311 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/VirtualAccess/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/interface.sv b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/interface.sv new file mode 100644 index 000000000..4cb146951 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/interface.sv @@ -0,0 +1,9 @@ +`include "virtual_parameters.svh" +interface virtual_if ( + input logic clock +); + logic reset = 1; + logic [`VA_INPUT_BITS-1:0] source = '0, backend = '0; + wire [ `VA_INPUT_BITS-1:0] inputs = source | backend; + wire [`VA_OUTPUT_BITS-1:0] outputs; +endinterface diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/package.sv b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/package.sv new file mode 100644 index 000000000..cf65ee932 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/package.sv @@ -0,0 +1,106 @@ +`include "virtual_parameters.svh" +`define VI(F) vif.inputs[`VA_IN_``F``_OFFSET+:`VA_IN_``F``_WIDTH] +`define VO(F) vif.outputs[`VA_OUT_``F``_OFFSET+:`VA_OUT_``F``_WIDTH] +`define VB(F) vif.backend[`VA_IN_``F``_OFFSET+:`VA_IN_``F``_WIDTH] +`define VS(F) pins[`VA_IN_``F``_OFFSET+:`VA_IN_``F``_WIDTH] +`define MI(T, F) T.inputs[`VA_IN_``F``_OFFSET+:`VA_IN_``F``_WIDTH] +`define MO(T, F) T.outputs[`VA_OUT_``F``_OFFSET+:`VA_OUT_``F``_WIDTH] +package virtual_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + import "DPI-C" function chandle mmu_ref_create( + input int unsigned address_bits, + entries + ); + import "DPI-C" function void mmu_ref_destroy(input chandle model); + import "DPI-C" function void mmu_ref_clear(input chandle model); + import "DPI-C" function void mmu_ref_program( + input chandle model, + input longint unsigned address, + value, + input int unsigned error + ); + import "DPI-C" function int unsigned mmu_ref_read( + input chandle model, + input longint unsigned address, + output longint unsigned value, + output int unsigned error + ); + import "DPI-C" function int unsigned mmu_ref_pending(input chandle model); + import "DPI-C" function int unsigned mmu_ref_peek( + input chandle model, + output longint unsigned address + ); + import "DPI-C" function int unsigned mmu_ref_consume( + input chandle model, + input longint unsigned address + ); + import "DPI-C" function int unsigned mmu_ref_translate( + input chandle model, + input longint unsigned address, + input int unsigned mode, + input longint unsigned root, + input int unsigned privilege, + wr, + execute, + sum, + mxr, + output longint unsigned paddr, + output int unsigned pf, + af, + level + ); + import "DPI-C" function int unsigned mmu_ref_lookup( + input chandle model, + input longint unsigned address, + input int unsigned mode, + input longint unsigned root, + input int unsigned privilege, + wr, + execute, + sum, + mxr, + output longint unsigned paddr + ); + import "DPI-C" function int unsigned mmu_ref_cached(input chandle model); + import "DPI-C" function void mmu_ref_result(input chandle model); + import "DPI-C" function void mmu_ref_reset(input chandle model); + import "DPI-C" function void mmu_ref_flush(input chandle model); + import "DPI-C" function void mem_access_ref_prepare( + input longint unsigned addr, + input int unsigned size, + write, + input longint unsigned data, + input int unsigned atomic_op, + cacheable, + normal, + address_bits, + output int unsigned target, + misaligned, + access_fault, + output longint unsigned bus_addr, + bus_data, + output int unsigned bus_mask, + atomic_word + ); + import "DPI-C" function longint unsigned mem_access_ref_result( + input longint unsigned addr, + input int unsigned size, + write, + signed_load, + atomic_op, + cacheable, + input longint unsigned raw, + input int unsigned error + ); + `include "agent/item.svh" + `include "agent/fixture.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/peer.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/seq/sequence.svh b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/seq/sequence.svh new file mode 100644 index 000000000..c639ccce3 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/seq/sequence.svh @@ -0,0 +1,110 @@ +class scenario_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(scenario_sequence) + model reference; + logic [`VA_INPUT_BITS-1:0] pins; + logic [`VA_OUTPUT_BITS-1:0] observed; + int tag; + function new(string name = "scenario"); + super.new(name); + endfunction + task tick(bit reset = 0); + cycle_item t = cycle_item::type_id::create("cycle"); + start_item(t); + t.inputs = pins; + t.reset = reset; + finish_item(t); + observed = t.outputs; + endtask + task transact(longint unsigned address, int size, bit writing = 0, bit signed_load = 0, + int atomic_op = 0, bit executing = 0, int privilege = 3, int mode = 0, bit sum = 0, + bit mxr = 0, int cancel_after = 0); + pins[`VA_IN_REQUEST_BITS_VADDR_OFFSET+:`VA_IN_REQUEST_BITS_VADDR_WIDTH] = address; + pins[`VA_IN_REQUEST_BITS_SIZE_OFFSET+:`VA_IN_REQUEST_BITS_SIZE_WIDTH] = size; + pins[`VA_IN_REQUEST_BITS_TAG_OFFSET+:`VA_IN_REQUEST_BITS_TAG_WIDTH] = tag++; + pins[`VA_IN_REQUEST_BITS_WRITE_OFFSET+:`VA_IN_REQUEST_BITS_WRITE_WIDTH] = writing; + pins[`VA_IN_REQUEST_BITS_SIGNED_OFFSET+:`VA_IN_REQUEST_BITS_SIGNED_WIDTH] = signed_load; + pins[`VA_IN_REQUEST_BITS_ATOMIC_OFFSET+:`VA_IN_REQUEST_BITS_ATOMIC_WIDTH] = atomic_op; + pins[`VA_IN_REQUEST_BITS_EXECUTE_OFFSET+:`VA_IN_REQUEST_BITS_EXECUTE_WIDTH] = executing; + pins[`VA_IN_REQUEST_BITS_DATA_OFFSET+:`VA_IN_REQUEST_BITS_DATA_WIDTH] = 64'hf1e2d3c4b5a69788; + pins[`VA_IN_REQUEST_BITS_PRIVILEGE_OFFSET+:`VA_IN_REQUEST_BITS_PRIVILEGE_WIDTH] = privilege; + pins[`VA_IN_REQUEST_BITS_SATPMODE_OFFSET+:`VA_IN_REQUEST_BITS_SATPMODE_WIDTH] = mode; + pins[`VA_IN_REQUEST_BITS_ROOTPPN_OFFSET+:`VA_IN_REQUEST_BITS_ROOTPPN_WIDTH] = 16; + pins[`VA_IN_REQUEST_BITS_SUM_OFFSET+:`VA_IN_REQUEST_BITS_SUM_WIDTH] = sum; + pins[`VA_IN_REQUEST_BITS_MXR_OFFSET+:`VA_IN_REQUEST_BITS_MXR_WIDTH] = mxr; + pins[`VA_IN_FLUSHTRANSLATION_OFFSET+:`VA_IN_FLUSHTRANSLATION_WIDTH] = 1; + tick(); + pins[`VA_IN_FLUSHTRANSLATION_OFFSET+:`VA_IN_FLUSHTRANSLATION_WIDTH] = 0; + pins[`VA_IN_REQUEST_VALID_OFFSET+:`VA_IN_REQUEST_VALID_WIDTH] = 1; + pins[`VA_IN_RESPONSE_READY_OFFSET+:`VA_IN_RESPONSE_READY_WIDTH] = 0; + do tick(); while (!observed[`VA_OUT_REQUEST_READY_OFFSET]); + pins[`VA_IN_REQUEST_VALID_OFFSET+:`VA_IN_REQUEST_VALID_WIDTH] = 0; + if (cancel_after > 0) begin + repeat (cancel_after) tick(); + repeat (2) tick(1); + tick(); + return; + end + do tick(); while (!observed[`VA_OUT_RESPONSE_VALID_OFFSET]); + repeat (4) tick(); + pins[`VA_IN_RESPONSE_READY_OFFSET+:`VA_IN_RESPONSE_READY_WIDTH] = 1; + tick(); + tick(); + endtask + task body(); + pins = '0; + pins[`VA_IN_ACTIVE_OFFSET+:`VA_IN_ACTIVE_WIDTH] = 1; + repeat (3) tick(1); + tick(); + for (int size = 0; size < 4; size++) + for (int offset = 0; offset < 16; offset++) + for (int direction = 0; direction < 2; direction++) begin + transact('h2000 + offset, size, direction, 0); + transact('h200000 + offset, size, direction, 1); + end + for (int atomic_op = 1; atomic_op <= 11; atomic_op++) begin + transact('h2000, 2, 0, 0, atomic_op); + transact('h2008, 3, 0, 0, atomic_op); + end + transact('h2000, 1, 0, 0, 0, 1); + transact('h200000, 1, 0, 0, 0, 1); + transact('h300000, 3); + transact('h300000, 3, 1); + transact('h500000, 3); + for (int kind = 0; kind < 2; kind++) begin + reference.image.deny_data = kind == 0; + reference.image.backend_error = kind == 1; + transact('h2000, 3); + transact('h200000, 3); + reference.image.deny_data = 0; + reference.image.backend_error = 0; + end + reference.set_pte('h10000, ('h11 << 10) | 1); + reference.set_pte('h11000, ('h12 << 10) | 1); + reference.set_pte('h12008, ('h2 << 10) | 'hcf); + for (int privilege = 0; privilege < 4; privilege++) + if (privilege != 2) begin + transact('h1000, 3, 0, 0, 0, 0, privilege, 8); + transact('h1000, 3, 1, 0, 0, 0, privilege, 8); + end + transact('h1002, 1, 0, 0, 0, 1, 1, 8); + transact(64'h4000000000, 3, 0, 0, 0, 0, 1, 8); + reference.set_pte('h12008, ('h2 << 10) | 'hdf); + transact('h1000, 3, 0, 0, 0, 0, 0, 8); + transact('h1000, 3, 0, 0, 0, 0, 1, 8, 1); + reference.set_pte('h12008, ('h2 << 10) | 'hc9); + transact('h1000, 3, 0, 0, 0, 0, 1, 8, 0, 1); + transact('h1000, 3, 0, 0, 0, 0, 1, 8); + reference.set_pte('h12008, 0); + transact('h1000, 3, 0, 0, 0, 0, 1, 8); + reference.set_pte('h12008, ('h2 << 10) | 'hcf, 1); + transact('h1000, 3, 0, 0, 0, 0, 1, 8); + reference.set_pte('h12008, ('h2 << 10) | 'hcf); + reference.image.deny_pte = 1; + transact('h1000, 3, 0, 0, 0, 0, 1, 8); + reference.image.deny_pte = 0; + transact('h2000, 3, 0, 0, 0, 0, 3, 0, 0, 0, 2); + transact('h2000, 3, 0, 0, 0, 0, 3, 0, 0, 0, 7); + transact('h2000, 3); + repeat (8) tick(); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/tb.sv b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/tb.sv new file mode 100644 index 000000000..cab5cca4f --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/tb.sv @@ -0,0 +1,15 @@ +module virtual_access_tb; + import uvm_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + virtual_if bus (clock); + VirtualAccess dut ( + .clock(clock), + .reset(bus.reset), + `include "virtual_ports.svh" + ); + initial begin + uvm_config_db#(virtual virtual_if)::set(null, "uvm_test_top.env.*", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/tests/test.svh b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/tests/test.svh new file mode 100644 index 000000000..b11c898c9 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/tests/test.svh @@ -0,0 +1,16 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.reference = test_env.reference; + seq.start(test_env.seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/Emit.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/Emit.scala new file mode 100644 index 000000000..06b563d60 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/Emit.scala @@ -0,0 +1,88 @@ +package framework.core.memory.access + +import chisel3._ +import chisel3.reflect.DataMirror +import java.nio.file.{Files, Paths} +import framework.core.memory.access.configs.Params +import framework.ip.bus.chi.rnf.{CacheAccess, CacheResult} + +object Emit extends App { + val Array("--config", config, "--output-dir", directory) = args + val p = upickle.default.read[Params](Files.readString(Paths.get(config))) + Files.createDirectories(Paths.get(directory)) + + val defs = scala.collection.mutable.ArrayBuffer[String]( + s"`define CPU_PHYSICAL_BITS ${p.memory.chi.addressBits}", + s"`define CPU_TAG_BITS ${p.memory.tagBits}", + s"`define CPU_LINE_BITS ${p.lineBits}" + ) + + val ports = scala.collection.mutable.ArrayBuffer[String]() + + def channel( + kind: String, + prefix: String, + signal: String, + bundle: Bundle + ): Unit = { + var offset = 0 + for ((name, field) <- bundle.elements.toSeq) { + val width = field.getWidth + if (width > 0) { + defs += s"`define CPU_${kind}_${name.toUpperCase}_OFFSET $offset"; + defs += s"`define CPU_${kind}_${name.toUpperCase}_WIDTH $width"; + ports += s".${prefix}_bits_$name($signal.bits[$offset +: $width])"; offset += width + } + } + defs += s"`define ACCESS_${kind}_WIDTH $offset" + ports += s".${prefix}_valid($signal.valid)"; ports += s".${prefix}_ready($signal.ready)" + } + + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new PhysicalAccess(p), + args = Array("--target-dir", s"$directory/PhysicalAccess", "--split-verilog") + ) + channel("REQ", "io_request", "request", new AccessRequest(p)) + channel("RESP", "io_response", "response", new AccessResponse(p)) + channel("CACHE", "io_cacheRequest", "cache_request", new CacheAccess(p.memory.chi)) + channel("CRESULT", "io_cacheResponse", "cache_response", new CacheResult(p.lineBits)) + channel("UNCACHED", "io_uncachedRequest", "uncached_request", new UncachedRequest(p.memory)) + channel("URESULT", "io_uncachedResponse", "uncached_response", new UncachedResponse(p.memory)) + Files.writeString(Paths.get(directory, "physical_parameters.svh"), defs.mkString("\n") + "\n") + Files.writeString(Paths.get(directory, "physical_ports.svh"), ports.mkString(",\n") + "\n") + + val definitions = scala.collection.mutable.ArrayBuffer[String]( + s"`define VA_LINE_BITS ${p.lineBits}", + s"`define VA_ADDRESS_BITS ${p.memory.chi.addressBits}" + ) + + val binding = scala.collection.mutable.ArrayBuffer[String]() + var inputOffset = 0; var outputOffset = 0 + + def fields(data: Data, path: String): Seq[(String, Data)] = data match { + case record: Record => record.elements.toSeq.flatMap { case (name, field) => + fields(field, if (path.isEmpty) name else s"${path}_$name") + } + case vector: Vec[_] => vector.toSeq.zipWithIndex.flatMap { case (field, index) => fields(field, s"${path}_$index") } + case leaf => if (leaf.getWidth > 0) Seq(path -> leaf) else Seq.empty + } + + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + { + val dut = new VirtualAccess(p) + for ((name, field) <- fields(dut.io, "")) { + val input = DataMirror.directionOf(field) == ActualDirection.Input + val direction = if (input) "IN" else "OUT"; val signal = if (input) "inputs" else "outputs"; + val offset = if (input) inputOffset else outputOffset; val width = field.getWidth + definitions += s"`define VA_${direction}_${name.toUpperCase}_OFFSET $offset"; + definitions += s"`define VA_${direction}_${name.toUpperCase}_WIDTH $width" + binding += s".io_$name(bus.$signal[$offset +: $width])" + if (input) inputOffset += width else outputOffset += width + }; dut + }, + args = Array("--target-dir", s"$directory/VirtualAccess", "--split-verilog") + ) + definitions += s"`define VA_INPUT_BITS $inputOffset"; definitions += s"`define VA_OUTPUT_BITS $outputOffset" + Files.writeString(Paths.get(directory, "virtual_parameters.svh"), definitions.mkString("\n") + "\n") + Files.writeString(Paths.get(directory, "virtual_ports.svh"), binding.mkString(",\n") + "\n") +} diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/PhysicalAccess.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/PhysicalAccess.scala new file mode 100644 index 000000000..abb86362d --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/PhysicalAccess.scala @@ -0,0 +1,151 @@ +package framework.core.memory.access + +import framework.core.memory.access.configs.{Params => ModuleParam} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.ip.bus.chi.Params +import framework.ip.bus.chi.rnf.{CacheAccess, CacheAtomic, CacheResult} + +case class AccessParams(chi: Params = Params(), tagBits: Int = 6) { + require(chi.addressBits < 64 && tagBits > 0) +} + +class AccessRequest(b: ModuleParam) extends Bundle { + val p = b.memory + + val addr = UInt(64.W) + val tag = UInt(p.tagBits.W) + val size = UInt(3.W) + val write = Bool() + val signed = Bool() + val data = UInt(64.W) + val atomic = UInt(4.W) + val cacheable = Bool() + val normal = Bool() +} + +/** `line` is the whole cached line for full-line clients; it is absent by default. */ +class AccessResponse(b: ModuleParam) extends Bundle { + val p = b.memory + val lineBits = b.lineBits + + val tag = UInt(p.tagBits.W) + val data = UInt(64.W) + val misaligned = Bool() + val accessFault = Bool() + val line = UInt(lineBits.W) +} + +class UncachedRequest(p: AccessParams) extends Bundle { + + val addr = UInt(64.W) + val tag = UInt(p.tagBits.W) + val size = UInt(3.W) + val write = Bool() + val data = UInt(64.W) +} + +class UncachedResponse(p: AccessParams) extends Bundle { + + val tag = UInt(p.tagBits.W) + val data = UInt(64.W) + val error = Bool() +} + +@instantiable +class PhysicalAccess(b: ModuleParam) extends Module { + val p = b.memory + val lineBits = b.lineBits + + @public + val io = IO(new Bundle { + val request = Flipped(Decoupled(new AccessRequest(b))) + val response = Decoupled(new AccessResponse(b)) + + val cacheRequest = Decoupled(new CacheAccess(p.chi)) + + val cacheResponse = Flipped(Decoupled(new CacheResult(lineBits))) + val uncachedRequest = Decoupled(new UncachedRequest(b.memory)) + val uncachedResponse = Flipped(Decoupled(new UncachedResponse(b.memory))) + }) + + val idle :: issueCache :: waitCache :: issueUncached :: waitUncached :: respond :: Nil = Enum(6) + val state = RegInit(idle) + val command = Reg(new AccessRequest(b)) + val answer = Reg(new AccessResponse(b)) + + val mask = MuxLookup(command.size, 255.U(8.W))(Seq( + 0.U -> 1.U(8.W), + 1.U -> 3.U(8.W), + 2.U -> 15.U(8.W) + )) + + io.request.ready := state === idle + when(io.request.fire) { + val r = io.request.bits + assert(r.size <= 3.U, "Memory access size must be 1, 2, 4 or 8 bytes") + assert(r.atomic <= CacheAtomic.SC.U, "Memory access port does not accept Fence or unknown atomics") + assert( + r.atomic === CacheAtomic.None.U || (!r.write && r.size >= 2.U), + "Memory atomic requires a 32 or 64 bit operand" + ) + val misaligned = (r.addr & ((1.U(64.W) << r.size) - 1.U)).orR + val accessFault = r.addr(63, p.chi.addressBits).orR || (r.normal && !r.cacheable) || + (!r.normal && (r.cacheable || r.atomic =/= CacheAtomic.None.U)) + command := r + answer.tag := r.tag + answer.data := 0.U + if (lineBits > 0) answer.line := 0.U + answer.misaligned := misaligned + answer.accessFault := !misaligned && accessFault + state := Mux(misaligned || accessFault, respond, Mux(r.cacheable, issueCache, issueUncached)) + } + + val atomic = command.atomic =/= CacheAtomic.None.U + io.cacheRequest.valid := state === issueCache + io.cacheRequest.bits.addr := Mux(atomic, command.addr, command.addr & ~7.U(64.W))(p.chi.addressBits - 1, 0) + io.cacheRequest.bits.write := command.write + io.cacheRequest.bits.data := Mux(atomic, command.data, command.data << (command.addr(2, 0) << 3)) + io.cacheRequest.bits.mask := Mux(atomic, 255.U, Mux(command.write, mask << command.addr(2, 0), 0.U)) + io.cacheRequest.bits.atomic := command.atomic + io.cacheRequest.bits.atomicWord := atomic && command.size === 2.U + when(io.cacheRequest.fire)(state := waitCache) + io.cacheResponse.ready := state === waitCache + io.uncachedRequest.valid := state === issueUncached + io.uncachedRequest.bits.addr := command.addr + io.uncachedRequest.bits.tag := command.tag + io.uncachedRequest.bits.size := command.size + io.uncachedRequest.bits.write := command.write + io.uncachedRequest.bits.data := command.data + when(io.uncachedRequest.fire)(state := waitUncached) + io.uncachedResponse.ready := state === waitUncached + + val raw = + Mux(state === waitUncached, io.uncachedResponse.bits.data, io.cacheResponse.bits.data >> (command.addr(2, 0) << 3)) + + val load = MuxLookup(command.size, raw)(Seq( + 0.U -> Cat(Fill(56, command.signed && raw(7)), raw(7, 0)), + 1.U -> Cat(Fill(48, command.signed && raw(15)), raw(15, 0)), + 2.U -> Cat(Fill(32, command.signed && raw(31)), raw(31, 0)) + )) + + when(io.cacheResponse.fire || io.uncachedResponse.fire) { + val error = Mux(state === waitUncached, io.uncachedResponse.bits.error, io.cacheResponse.bits.error) + when(io.uncachedResponse.fire) { + assert(io.uncachedResponse.bits.tag === command.tag, "Uncached response tag mismatch") + } + answer.data := Mux( + error || command.write, + 0.U, + Mux(atomic, io.cacheResponse.bits.data, load) + ) + answer.accessFault := error + if (lineBits > 0) answer.line := Mux(state === waitCache && !error, io.cacheResponse.bits.line, 0.U) + state := respond + } + io.response.valid := state === respond + io.response.bits := answer + when(io.response.fire)(state := idle) +} diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/VirtualAccess.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/VirtualAccess.scala new file mode 100644 index 000000000..daed79033 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/VirtualAccess.scala @@ -0,0 +1,349 @@ +package framework.core.memory.access + +import buckyball.config.RegionHardware + +import framework.core.memory.access.configs.{Params => ModuleParam} +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.ip.bus.chi.rnf.{CacheAccess, CacheAtomic, CacheResult} +import framework.core.memory.access.mmu.Walker + +class VirtualPort(b: ModuleParam) extends Bundle { + val flushTranslation = Input(Bool()) + val request = Flipped(Decoupled(new VirtualRequest(b))) + val response = Decoupled(new VirtualResponse(b)) + val authorizationRequest = Decoupled(new PhysicalAuthorization) + val authorizationResponse = Flipped(Decoupled(Bool())) +} + +case class PhysicalRegion( + base: BigInt, + bytes: BigInt, + cacheable: Boolean, + executable: Boolean, + readable: Boolean, + writable: Boolean, + atomic: Boolean, + normal: Boolean) + +object PhysicalRegion { + + def fromProto(p: RegionHardware): PhysicalRegion = PhysicalRegion( + BigInt(java.lang.Long.toUnsignedString(p.getBase)), + BigInt(java.lang.Long.toUnsignedString(p.getBytes)), + p.getCacheable, + p.getExecutable, + p.getReadable, + p.getWritable, + p.getAtomic, + p.getNormal + ) + +} + +class PhysicalAuthorization extends Bundle { + val paddr = UInt(64.W) + val size = UInt(3.W) + val read = Bool() + val write = Bool() + val execute = Bool() + val isPte = Bool() + val privilege = UInt(2.W) +} + +class VirtualRequest(b: ModuleParam) extends Bundle { + val p = b.memory + + val vaddr = UInt(64.W) + val tag = UInt(p.tagBits.W) + val size = UInt(3.W) + val write = Bool() + val execute = Bool() + val signed = Bool() + val data = UInt(64.W) + val atomic = UInt(4.W) + val privilege = UInt(2.W) + val sum = Bool() + val mxr = Bool() + val satpMode = UInt(4.W) + val rootPpn = UInt(44.W) +} + +class VirtualResponse(b: ModuleParam) extends Bundle { + val p = b.memory + val lineBits = b.lineBits + + val tag = UInt(p.tagBits.W) + val data = UInt(64.W) + val misaligned = Bool() + val pageFault = Bool() + val accessFault = Bool() + // Whole cached line of the translated word, for full-line clients only. + val line = UInt(lineBits.W) +} + +/** Single-outstanding virtual data/instruction-word frontend; cache ownership is external. */ +@instantiable +class VirtualAccess(b: ModuleParam) extends Module { + val cp = b.memory + val regions = b.regions + val lineBits = b.lineBits + val translationEntries = b.translationEntries + + require(regions.nonEmpty) + for ((region, i) <- regions.zipWithIndex) { + require(region.base >= 0 && region.bytes > 0 && region.base + region.bytes <= (BigInt(1) << cp.chi.addressBits)) + require(!region.cacheable || (region.base % 64 == 0 && region.bytes % 64 == 0)) + require( + region.normal || (!region.cacheable && !region.executable && !region.atomic), + "Device regions cannot be cached, executable or atomic" + ) + for (other <- regions.drop(i + 1)) { + require(region.base + region.bytes <= other.base || other.base + other.bytes <= region.base) + } + } + + @public + val io = IO(new Bundle { + val active = Input(Bool()) + // SFENCE.VMA: drop every cached translation. + val flushTranslation = Input(Bool()) + + // Combinational translation through the TLB only, for a caller's fast path. + val lookup = new Bundle { + val config = Input(new framework.core.memory.access.mmu.Config) + val req = Input(new framework.core.memory.access.mmu.Request) + val hit = Output(Bool()) + val paddr = Output(UInt(cp.chi.addressBits.W)) + } + + val request = Flipped(Decoupled(new VirtualRequest(b))) + val response = Decoupled(new VirtualResponse(b)) + val authorizationRequest = Decoupled(new PhysicalAuthorization) + val authorizationResponse = Flipped(Decoupled(Bool())) + + val cacheRequest = Decoupled(new CacheAccess(cp.chi)) + + val cacheResponse = Flipped(Decoupled(new CacheResult(lineBits))) + val uncachedRequest = Decoupled(new UncachedRequest(b.memory)) + val uncachedResponse = Flipped(Decoupled(new UncachedResponse(b.memory))) + + val observedTranslation = + Output(Valid(new framework.core.memory.access.mmu.Response(b.walker))) + + val observedPhysical = Output(Valid(new AccessRequest(b))) + + val observedCache = Output(Valid(new Bundle { + val pte = Bool() + + val access = new CacheAccess(cp.chi) + + })) + + }) + + val walker: Instance[Walker] = Instantiate(new Walker(b.walker)) + + val physical: Instance[PhysicalAccess] = Instantiate(new PhysicalAccess(b)) + + val idle :: startWalk :: waitWalk :: authorize :: waitAuthorization :: startAccess :: waitAccess :: respond :: Nil = + Enum(8) + val state = RegInit(idle) + val command = Reg(new VirtualRequest(b)) + val answer = Reg(new VirtualResponse(b)) + val paddr = Reg(UInt(64.W)) + val cacheable = Reg(Bool()) + val normal = Reg(Bool()) + val pteInFlight = RegInit(false.B) + val pteError = RegInit(false.B) + val pteAuthorized = RegInit(false.B) + val pteAuthorizationPending = RegInit(false.B) + val regionAllowed = Reg(Bool()) + def matches(first: UInt, last: UInt): Seq[Bool] = + regions.map(region => first >= region.base.U(65.W) && last < (region.base + region.bytes).U(65.W)) + io.request.ready := state === idle && io.active + when(io.request.fire) { + val r = io.request.bits + assert(r.size <= 3.U, "Virtual LSU size must be 1, 2, 4 or 8 bytes") + assert(r.atomic <= CacheAtomic.SC.U, "Virtual LSU does not accept Fence or unknown atomics") + assert( + r.atomic === CacheAtomic.None.U || (!r.write && r.size >= 2.U), + "Virtual LSU atomic requires a 32 or 64 bit operand" + ) + assert(r.satpMode === 0.U || r.satpMode === 8.U, "Virtual LSU requires Bare or Sv39 mode") + assert( + r.privilege === 0.U || r.privilege === 1.U || r.privilege === 3.U, + "Virtual LSU effective privilege must be U, S or M" + ) + assert( + !r.execute || (!r.write && r.atomic === CacheAtomic.None.U && r.size >= 1.U && !r.signed), + "Virtual instruction fetch requires unsigned 16, 32 or 64 bit read without atomic" + ) + command := r + answer := 0.U.asTypeOf(answer) + answer.tag := r.tag + val misaligned = (r.vaddr & ((1.U(64.W) << r.size) - 1.U)).orR + answer.misaligned := misaligned + state := Mux(misaligned, respond, startWalk) + } + + walker.io.flush := io.flushTranslation + walker.io.lookup.config := io.lookup.config + walker.io.lookup.req := io.lookup.req + io.lookup.hit := walker.io.lookup.hit + io.lookup.paddr := walker.io.lookup.paddr + walker.io.config.mode := command.satpMode + walker.io.config.rootPpn := command.rootPpn + walker.io.req.valid := state === startWalk + walker.io.req.bits.vaddr := command.vaddr + walker.io.req.bits.write := command.write || + (command.atomic =/= CacheAtomic.None.U && command.atomic =/= CacheAtomic.LR.U) + walker.io.req.bits.execute := command.execute + walker.io.req.bits.privilege := command.privilege + walker.io.req.bits.sum := command.sum + walker.io.req.bits.mxr := command.mxr + when(walker.io.req.fire)(state := waitWalk) + walker.io.resp.ready := state === waitWalk + + val lastByte = walker.io.resp.bits.paddr.pad(65) + MuxLookup(command.size, 0.U(4.W))( + Seq(1.U -> 1.U(4.W), 2.U -> 3.U(4.W), 3.U -> 7.U(4.W)) + ) + + val dataRegions = matches(walker.io.resp.bits.paddr.pad(65), lastByte) + val isAtomic = command.atomic =/= CacheAtomic.None.U + val readsData = !command.execute && !command.write && command.atomic =/= CacheAtomic.SC.U + val writesData = command.write || (isAtomic && command.atomic =/= CacheAtomic.LR.U) + + val permittedRegion = dataRegions.zip(regions).map { case (hit, region) => + hit && (!region.normal || region.cacheable).B && (!command.execute || (region.normal && region.executable).B) && + (!readsData || region.readable.B) && (!writesData || region.writable.B) && + (!isAtomic || region.atomic.B) + }.reduce(_ || _) + + when(walker.io.resp.fire) { + when(walker.io.resp.bits.pageFault || walker.io.resp.bits.accessFault) { + answer.pageFault := walker.io.resp.bits.pageFault + answer.accessFault := walker.io.resp.bits.accessFault + state := respond + }.otherwise { + paddr := walker.io.resp.bits.paddr + cacheable := dataRegions.zip(regions).map { case (hit, region) => hit && region.cacheable.B }.reduce(_ || _) + normal := dataRegions.zip(regions).map { case (hit, region) => hit && region.normal.B }.reduce(_ || _) + regionAllowed := permittedRegion + state := authorize + } + } + + physical.io.request.valid := state === startAccess + physical.io.request.bits.addr := paddr + physical.io.request.bits.tag := command.tag + physical.io.request.bits.size := command.size + physical.io.request.bits.write := command.write + physical.io.request.bits.signed := command.signed + physical.io.request.bits.data := command.data + physical.io.request.bits.atomic := command.atomic + physical.io.request.bits.cacheable := cacheable + physical.io.request.bits.normal := normal + when(physical.io.request.fire)(state := waitAccess) + physical.io.response.ready := state === waitAccess + when(physical.io.response.fire) { + assert(physical.io.response.bits.tag === command.tag, "Virtual LSU physical response tag mismatch") + answer.data := physical.io.response.bits.data + answer.misaligned := physical.io.response.bits.misaligned + answer.accessFault := physical.io.response.bits.accessFault + if (lineBits > 0) answer.line := physical.io.response.bits.line + state := respond + } + + val walking = state === startWalk || state === waitWalk + val accessing = state === startAccess || state === waitAccess + val pteFirst = walker.io.access.bits.addr.pad(65) + val pteRegions = matches(pteFirst, pteFirst + 7.U) + + val pteAllowed = pteRegions.zip(regions).map { case (hit, region) => + hit && (region.normal && region.readable && region.cacheable).B + }.reduce(_ || _) + + // The Thread owns a per-command PMP snapshot. Each permission decision is a + // single-outstanding handshake; neither a stalled query nor its response + // allows speculative cache or MMIO traffic. PTE accesses use S privilege. + io.authorizationRequest.valid := state === authorize || + (walking && walker.io.access.valid && !pteAuthorized && !pteAuthorizationPending && !pteError) + io.authorizationRequest.bits.paddr := Mux(walking, walker.io.access.bits.addr.pad(64), paddr) + io.authorizationRequest.bits.size := Mux(walking, 3.U, command.size) + io.authorizationRequest.bits.read := walking || readsData + io.authorizationRequest.bits.write := !walking && writesData + io.authorizationRequest.bits.execute := !walking && command.execute + io.authorizationRequest.bits.isPte := walking + io.authorizationRequest.bits.privilege := Mux(walking, 1.U, command.privilege) + when(io.authorizationRequest.fire) { + when(walking) { + pteAuthorizationPending := true.B + } + .otherwise(state := waitAuthorization) + } + io.authorizationResponse.ready := pteAuthorizationPending || state === waitAuthorization + when(io.authorizationResponse.fire) { + when(pteAuthorizationPending) { + pteAuthorizationPending := false.B + pteAuthorized := true.B + pteError := !io.authorizationResponse.bits || !pteAllowed + }.otherwise { + when(io.authorizationResponse.bits && regionAllowed)(state := startAccess) + .otherwise { answer.accessFault := true.B; state := respond } + } + } + io.cacheRequest.valid := Mux( + walking, + walker.io.access.valid && pteAuthorized && !pteError, + accessing && physical.io.cacheRequest.valid + ) + io.cacheRequest.bits := physical.io.cacheRequest.bits + when(walking) { + io.cacheRequest.bits := 0.U.asTypeOf(io.cacheRequest.bits) + io.cacheRequest.bits.addr := walker.io.access.bits.addr + } + io.uncachedRequest.valid := accessing && physical.io.uncachedRequest.valid + io.uncachedRequest.bits := physical.io.uncachedRequest.bits + walker.io.access.ready := walking && pteAuthorized && + (pteError || io.cacheRequest.ready) + when(walker.io.access.fire) { + pteAuthorized := false.B + pteInFlight := !pteError + } + walker.io.result.valid := walking && ((!pteAuthorized && pteError) || (pteInFlight && io.cacheResponse.valid)) + walker.io.result.bits := 0.U.asTypeOf(walker.io.result.bits) + walker.io.result.bits.data := Mux(pteError || io.cacheResponse.bits.error, 0.U, io.cacheResponse.bits.data) + walker.io.result.bits.error := pteError || io.cacheResponse.bits.error + when(walker.io.result.fire) { pteError := false.B; pteInFlight := false.B } + physical.io.cacheRequest.ready := accessing && io.cacheRequest.ready + physical.io.cacheResponse.valid := accessing && io.cacheResponse.valid + physical.io.cacheResponse.bits := io.cacheResponse.bits + physical.io.uncachedRequest.ready := accessing && io.uncachedRequest.ready + physical.io.uncachedResponse.valid := accessing && io.uncachedResponse.valid + physical.io.uncachedResponse.bits := io.uncachedResponse.bits + io.cacheResponse.ready := Mux( + walking, + pteInFlight && walker.io.result.ready, + accessing && physical.io.cacheResponse.ready + ) + io.uncachedResponse.ready := accessing && physical.io.uncachedResponse.ready + when(io.cacheResponse.valid) { + assert((walking && pteInFlight) || accessing, "Virtual LSU cache result has no owner") + } + when(io.uncachedResponse.valid) { + assert(accessing, "Virtual LSU uncached result has no owner") + } + + io.response.valid := state === respond + io.response.bits := answer + when(io.response.fire)(state := idle) + io.observedTranslation.valid := walker.io.resp.fire + io.observedTranslation.bits := walker.io.resp.bits + io.observedPhysical.valid := physical.io.request.fire + io.observedPhysical.bits := physical.io.request.bits + io.observedCache.valid := io.cacheRequest.fire + io.observedCache.bits.pte := walking + io.observedCache.bits.access := io.cacheRequest.bits +} diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/build.sc b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/build.sc new file mode 100644 index 000000000..d4bc46a7a --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/build.sc @@ -0,0 +1,22 @@ +import mill._ +import $file.^.^.^.^.^.^.^.^.^.^.^.settings + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.core.memory.access.cache.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Cache.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Maintenance.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/configs/uvm.toml b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/configs/uvm.toml new file mode 100644 index 000000000..87cf84120 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/configs/uvm.toml @@ -0,0 +1,19 @@ +[parameters.cache] +nodeId=1 +cacheLines=8 +homeId=64 +homeCount=1 +banks=2 +lineResult=false +probe=false +[parameters.cache.chi] +nodeIdBits=7 +addressBits=44 +dataBits=256 +[parameters.tracking] +entries=4 +addressBits=44 +idBits=8 +lineBytes=64 +maxRanges=8 +[defines] diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/agent/cmo_if.sv b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/agent/cmo_if.sv new file mode 100644 index 000000000..b0d144c6d --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/agent/cmo_if.sv @@ -0,0 +1,19 @@ +`include "parameters.svh" +interface cmo_if ( + input logic clock, + input logic reset +); + logic valid = 0, ready, response_valid, response_ready = 1, ok; + logic [`CMO_ID_BITS-1:0] tag = 0, response_tag; + logic [`CHI_ADDRESS_BITS-1:0] first_line = 0, last_line = 0; + logic [1:0] operation = 0; + logic older_drained = 1; + assert property (@(posedge clock) disable iff (reset) valid && !ready |=> valid && $stable( + {tag, first_line, last_line, operation} + )) + else $fatal(1, "Maintenance request changed under backpressure"); + assert property(@(posedge clock) disable iff(reset)response_valid&&!response_ready|=>response_valid&&$stable( + {response_tag, ok} + )) + else $fatal(1, "Maintenance response changed under backpressure"); +endinterface diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/env/model.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/env/model.svh new file mode 100644 index 000000000..4e50ae22f --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/env/model.svh @@ -0,0 +1,116 @@ +class cmo_result extends uvm_sequence_item; + `uvm_object_utils_begin(cmo_result) + `uvm_field_int(tag, UVM_ALL_ON) + `uvm_field_int(ok, UVM_ALL_ON) + `uvm_object_utils_end + int tag; + bit ok; + function new(string name = "result"); + super.new(name); + endfunction +endclass +class cache_model extends protocol_model; + `uvm_component_utils(cache_model) + virtual cmo_if cmo; + result_board #(cmo_result, 0) acknowledgments; + bit maintenance_pending; + longint unsigned next_line, last_line; + int operation, maintenance_tag, issued, returned, fail_index = -1; + covergroup maintenance with function sample (int op, bit ok, bit stalled, bit overlap); + option.per_instance = 1; + operations: coverpoint op {bins all[] = {[0 : 2]};} + errors: coverpoint ok; + backpressure: coverpoint stalled; + concurrency: coverpoint overlap; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + maintenance = new(); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual cmo_if)::get(this, "", "cmo", cmo)) + `uvm_fatal("VIF", "Cache maintenance interface is required") + acknowledgments = result_board#(cmo_result, 0)::type_id::create("maintenance_results", this); + endfunction + function void write(sample_item t); + sample_item demand; + rsp_t response, received_rsp; + req_t request; + demand = sample_item::type_id::create("demand"); + demand.sampled = t.sampled; + if (t.sampled.reset) begin + acknowledgments.cancel_pending(); + maintenance_pending = 0; + issued = 0; + returned = 0; + end else begin + if (cmo.valid && cmo.ready) begin + cmo_result result = cmo_result::type_id::create("expected"); + if (maintenance_pending) `uvm_fatal("CMO", "Overlapping maintenance admissions") + maintenance_pending = 1; + next_line = cmo.first_line; + last_line = cmo.last_line; + operation = cmo.operation; + maintenance_tag = cmo.tag; + issued = 0; + returned = 0; + result.tag = maintenance_tag; + result.ok = fail_index < 0 || fail_index > int'((last_line - next_line) / 64); + acknowledgments.write_expected(result); + evidence::check(cmo.older_drained && t.sampled.outstanding == 0 && !t.sampled.access_ready, + "Maintenance admitted before drain or alongside a CPU request"); + end + if (t.sampled.req_valid && t.sampled.req_ready) begin + request = req_t'(t.sampled.req); + if (request.txn == `CHI_BANKS) begin + int opcode=operation==0?`CMO_CLEAN:operation==1?`CMO_CLEAN_INVALID:`CMO_INVALID; + evidence::check( + maintenance_pending&&request.addr==next_line&&request.opcode==opcode&&request.src==1&&request.tgt==64&&request.size==6&&request.snp_attr&&request.mem_attr==12&&request.allow_retry, + "Incorrect maintenance request footprint or attributes"); + response = '0; + response.src = 64; + response.tgt = 1; + response.txn = `CHI_BANKS; + response.opcode = `CMO_COMP; + response.error = issued == fail_index ? 2 : 0; + if (scenario == "bad_maintenance_identity") response.src = 65; + if (scenario == "bad_maintenance_opcode") response.opcode = 1; + rx_rsp_queue.push_back(response); + issued++; + demand.sampled.req_valid = 0; + end + end + received_rsp = rsp_t'(t.sampled.rx_rsp); + if (t.sampled.rx_rsp_valid && received_rsp.txn == `CHI_BANKS) begin + response = rsp_t'(t.sampled.rx_rsp); + if (t.sampled.rx_rsp_ready) begin + rsp_active = 0; + returned++; + if (response.error == 0) next_line += 64; + end + demand.sampled.rx_rsp_valid = 0; + demand.sampled.rx_rsp_ready = 0; + end + if (cmo.response_valid && cmo.response_ready) begin + cmo_result result = cmo_result::type_id::create("actual"); + evidence::check( + maintenance_pending && issued == returned && (next_line > last_line || fail_index >= 0), + "Maintenance acknowledged before all completions"); + result.tag = cmo.response_tag; + result.ok = cmo.ok; + acknowledgments.write_actual(result); + maintenance_pending = 0; + maintenance.sample(operation, cmo.ok, 0, t.sampled.outstanding > 0); + end + if (cmo.valid && !cmo.ready || cmo.response_valid && !cmo.response_ready) + maintenance.sample(cmo.operation, 1, 1, t.sampled.outstanding > 0); + if (maintenance_pending && t.sampled.access_valid) maintenance.sample(operation, 1, 0, 1); + end + super.write(demand); + endfunction + function void check_phase(uvm_phase phase); + super.check_phase(phase); + if (maintenance_pending) `uvm_fatal("CMO", "Maintenance did not drain") + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/filelist.f b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/filelist.f new file mode 100644 index 000000000..1591e158c --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/filelist.f @@ -0,0 +1,12 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core ++incdir+@VERIFY@/uvm/src/protocol/chi ++incdir+@VERIFY@/uvm/src/protocol/chi/rnf +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/protocol/chi/rnf/types.svh +@VERIFY@/uvm/src/protocol/chi/rnf/interface.sv +@RESOURCES@/agent/cmo_if.sv +@RESOURCES@/package.sv +-F @RTL@/Cache/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/package.sv b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/package.sv new file mode 100644 index 000000000..c47e402be --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/package.sv @@ -0,0 +1,6 @@ +package core_cache_pkg; + `include "rnf/body.svh" + `include "env/model.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..3039f1e0f --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/seq/sequence.svh @@ -0,0 +1,72 @@ +class cache_sequence extends scenario_sequence; + `uvm_object_utils(cache_sequence) + virtual cmo_if cmo; + cache_model checked_state; + function new(string name = "scenario"); + super.new(name); + endfunction + task operation(int op, int lines, int tag, int failure = -1); + longint unsigned checked_before = checked_state.acknowledgments.checked; + `uvm_info("CACHE_CMO", + $sformatf("op=%0d lines=%0d tag=%0d error=%0d", op, lines, tag, failure), UVM_HIGH) + checked_state.fail_index = failure; + @(negedge vif.clock); + cmo.valid = 1; + cmo.first_line = 'h40000; + cmo.last_line = 'h40000 + (lines - 1) * 64; + cmo.operation = op; + cmo.tag = tag; + cmo.response_ready = 0; + do @(posedge vif.clock); while (!cmo.ready); + @(negedge vif.clock); + cmo.valid = 0; + wait (cmo.response_valid); + repeat (4) @(negedge vif.clock); + cmo.response_ready = 1; + wait (checked_state.acknowledgments.checked == checked_before + 1); + @(negedge vif.clock); + endtask + task body(); + `uvm_info("CACHE_STAGE", "Starting demand regression", UVM_HIGH) + super.body(); + `uvm_info("CACHE_STAGE", "Starting maintenance regression", UVM_HIGH) + for (int op = 0; op < 3; op++) begin + cmo.older_drained = 0; + fork + operation(op, 1 + op, 20 + op); + begin + repeat (8) @(negedge vif.clock); + evidence::check(!cmo.ready, "Maintenance ignored an older undrained request"); + cmo.older_drained = 1; + end + join + operation(op, 3, 40 + op, op); + end + // An independent demand request progresses while a serialized CMO is outstanding. + state.hold_req = 1; + fork + operation(0, 4, 60); + begin + repeat (6) @(negedge vif.clock); + issue(16320); + state.hold_req = 0; + settle(); + end + join + state.hold_req = 1; + @(negedge vif.clock); + cmo.valid = 1; + cmo.first_line = 'h50000; + cmo.last_line = 'h50100; + cmo.operation = 2; + cmo.tag = 200; + do @(posedge vif.clock); while (!cmo.ready); + @(negedge vif.clock); + cmo.valid = 0; + repeat (3) @(negedge vif.clock); + test_env.reset.pulse(3); + state.hold_req = 0; + repeat (4) @(negedge vif.clock); + operation(2, 2, 201); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/suite.toml b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/suite.toml new file mode 100644 index 000000000..23b2331f7 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/suite.toml @@ -0,0 +1,105 @@ +[model] +manifest="../../../../../../../../../../../../verify/uvm/src/protocol/chi/rnf/model/Cargo.toml" +library="librnf_reference_model" + +[[tests]] +name="normal" +top="core_cache_tb" +test="cache_test" +filelist="filelist.f" +timeout_ns=1000000 +wall_timeout_s=120 +kind="functional" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=["core_cache_pkg::protocol_model::behavior","core_cache_pkg::cache_model::maintenance"] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=["+CASE=normal"] +assertion="" + +[[tests]] +name="bad_maintenance_alignment" +top="core_cache_tb" +test="cache_test" +filelist="filelist.f" +timeout_ns=1000000 +wall_timeout_s=120 +kind="assertion" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=[] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=["+CASE=bad_maintenance_alignment"] +assertion="Maintenance endpoints must be cache-line aligned" + +[[tests]] +name="bad_maintenance_range" +top="core_cache_tb" +test="cache_test" +filelist="filelist.f" +timeout_ns=1000000 +wall_timeout_s=120 +kind="assertion" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=[] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=["+CASE=bad_maintenance_range"] +assertion="Maintenance range is reversed" + +[[tests]] +name="bad_maintenance_operation" +top="core_cache_tb" +test="cache_test" +filelist="filelist.f" +timeout_ns=1000000 +wall_timeout_s=120 +kind="assertion" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=[] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=["+CASE=bad_maintenance_operation"] +assertion="Unknown maintenance operation" + +[[tests]] +name="bad_maintenance_identity" +top="core_cache_tb" +test="cache_test" +filelist="filelist.f" +timeout_ns=1000000 +wall_timeout_s=120 +kind="assertion" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=[] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=["+CASE=bad_maintenance_identity"] +assertion="Maintenance completion identity mismatch" + +[[tests]] +name="bad_maintenance_opcode" +top="core_cache_tb" +test="cache_test" +filelist="filelist.f" +timeout_ns=1000000 +wall_timeout_s=120 +kind="assertion" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=[] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=["+CASE=bad_maintenance_opcode"] +assertion="Maintenance requires a completion response" diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/tb.sv b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/tb.sv new file mode 100644 index 000000000..7a81925a3 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/tb.sv @@ -0,0 +1,169 @@ +module core_cache_tb; + import uvm_pkg::*; + import core_cache_pkg::*; + logic clock = 0; + clock_if timing (clock); + always #5 clock = ~clock; + chi_rnf_if vif (clock); + cmo_if cmo ( + clock, + vif.reset + ); + Cache dut ( + .clock(clock), + .reset(vif.reset), + .io_chi_req_bits_qos(vif.req[0+:4]), + .io_chi_req_bits_tgtId(vif.req[4+:7]), + .io_chi_req_bits_srcId(vif.req[11+:7]), + .io_chi_req_bits_txnId(vif.req[18+:12]), + .io_chi_req_bits_returnNid(vif.req[30+:7]), + .io_chi_req_bits_stashNidValidEndian(vif.req[37+:1]), + .io_chi_req_bits_returnTxnId(vif.req[38+:12]), + .io_chi_req_bits_opcode(vif.req[50+:7]), + .io_chi_req_bits_multiReq(vif.req[57+:1]), + .io_chi_req_bits_size(vif.req[58+:6]), + .io_chi_req_bits_addr(vif.req[64+:44]), + .io_chi_req_bits_pas(vif.req[108+:3]), + .io_chi_req_bits_likelyShared(vif.req[111+:1]), + .io_chi_req_bits_allowRetry(vif.req[112+:1]), + .io_chi_req_bits_order(vif.req[113+:2]), + .io_chi_req_bits_pCrdType(vif.req[115+:4]), + .io_chi_req_bits_memAttr(vif.req[119+:4]), + .io_chi_req_bits_snpAttr(vif.req[123+:1]), + .io_chi_req_bits_lpid(vif.req[124+:8]), + .io_chi_req_bits_exclSnoopMe(vif.req[132+:1]), + .io_chi_req_bits_expCompAck(vif.req[133+:1]), + .io_chi_req_bits_tagOp(vif.req[134+:2]), + .io_chi_req_bits_traceTag(vif.req[136+:1]), + .io_chi_req_valid(vif.req_valid), + .io_chi_req_ready(vif.req_ready), + .io_chi_txRsp_bits_qos(vif.rsp[0+:4]), + .io_chi_txRsp_bits_tgtId(vif.rsp[4+:7]), + .io_chi_txRsp_bits_srcId(vif.rsp[11+:7]), + .io_chi_txRsp_bits_txnId(vif.rsp[18+:12]), + .io_chi_txRsp_bits_opcode(vif.rsp[30+:5]), + .io_chi_txRsp_bits_respErr(vif.rsp[35+:2]), + .io_chi_txRsp_bits_resp(vif.rsp[37+:3]), + .io_chi_txRsp_bits_fwdState(vif.rsp[40+:3]), + .io_chi_txRsp_bits_cBusy(vif.rsp[43+:3]), + .io_chi_txRsp_bits_dbid(vif.rsp[46+:12]), + .io_chi_txRsp_bits_pCrdType(vif.rsp[58+:4]), + .io_chi_txRsp_bits_tagOp(vif.rsp[62+:2]), + .io_chi_txRsp_bits_traceTag(vif.rsp[64+:1]), + .io_chi_txRsp_bits_cacheLineId(vif.rsp[65+:6]), + .io_chi_txRsp_valid(vif.rsp_valid), + .io_chi_txRsp_ready(vif.rsp_ready), + .io_chi_txDat_bits_qos(vif.dat[0+:4]), + .io_chi_txDat_bits_tgtId(vif.dat[4+:7]), + .io_chi_txDat_bits_srcId(vif.dat[11+:7]), + .io_chi_txDat_bits_txnId(vif.dat[18+:12]), + .io_chi_txDat_bits_homeNid(vif.dat[30+:7]), + .io_chi_txDat_bits_opcode(vif.dat[37+:4]), + .io_chi_txDat_bits_respErr(vif.dat[41+:2]), + .io_chi_txDat_bits_resp(vif.dat[43+:3]), + .io_chi_txDat_bits_dataSource(vif.dat[46+:8]), + .io_chi_txDat_bits_dataPull(vif.dat[54+:1]), + .io_chi_txDat_bits_cBusy(vif.dat[55+:3]), + .io_chi_txDat_bits_dbid(vif.dat[58+:16]), + .io_chi_txDat_bits_ccid(vif.dat[74+:2]), + .io_chi_txDat_bits_dataId(vif.dat[76+:2]), + .io_chi_txDat_bits_cacheLineId(vif.dat[78+:6]), + .io_chi_txDat_bits_tagOp(vif.dat[84+:2]), + .io_chi_txDat_bits_tag(vif.dat[86+:8]), + .io_chi_txDat_bits_tagUpdate(vif.dat[94+:2]), + .io_chi_txDat_bits_traceTag(vif.dat[96+:1]), + .io_chi_txDat_bits_copyAtHome(vif.dat[97+:1]), + .io_chi_txDat_bits_numDat(vif.dat[98+:2]), + .io_chi_txDat_bits_replicate(vif.dat[100+:1]), + .io_chi_txDat_bits_be(vif.dat[101+:32]), + .io_chi_txDat_bits_data(vif.dat[133+:256]), + .io_chi_txDat_valid(vif.dat_valid), + .io_chi_txDat_ready(vif.dat_ready), + .io_chi_rxRsp_bits_qos(vif.rx_rsp[0+:4]), + .io_chi_rxRsp_bits_tgtId(vif.rx_rsp[4+:7]), + .io_chi_rxRsp_bits_srcId(vif.rx_rsp[11+:7]), + .io_chi_rxRsp_bits_txnId(vif.rx_rsp[18+:12]), + .io_chi_rxRsp_bits_opcode(vif.rx_rsp[30+:5]), + .io_chi_rxRsp_bits_respErr(vif.rx_rsp[35+:2]), + .io_chi_rxRsp_bits_resp(vif.rx_rsp[37+:3]), + .io_chi_rxRsp_bits_fwdState(vif.rx_rsp[40+:3]), + .io_chi_rxRsp_bits_cBusy(vif.rx_rsp[43+:3]), + .io_chi_rxRsp_bits_dbid(vif.rx_rsp[46+:12]), + .io_chi_rxRsp_bits_pCrdType(vif.rx_rsp[58+:4]), + .io_chi_rxRsp_bits_tagOp(vif.rx_rsp[62+:2]), + .io_chi_rxRsp_bits_traceTag(vif.rx_rsp[64+:1]), + .io_chi_rxRsp_bits_cacheLineId(vif.rx_rsp[65+:6]), + .io_chi_rxRsp_valid(vif.rx_rsp_valid), + .io_chi_rxRsp_ready(vif.rx_rsp_ready), + .io_chi_rxDat_bits_qos(vif.rx_dat[0+:4]), + .io_chi_rxDat_bits_tgtId(vif.rx_dat[4+:7]), + .io_chi_rxDat_bits_srcId(vif.rx_dat[11+:7]), + .io_chi_rxDat_bits_txnId(vif.rx_dat[18+:12]), + .io_chi_rxDat_bits_homeNid(vif.rx_dat[30+:7]), + .io_chi_rxDat_bits_opcode(vif.rx_dat[37+:4]), + .io_chi_rxDat_bits_respErr(vif.rx_dat[41+:2]), + .io_chi_rxDat_bits_resp(vif.rx_dat[43+:3]), + .io_chi_rxDat_bits_dataSource(vif.rx_dat[46+:8]), + .io_chi_rxDat_bits_dataPull(vif.rx_dat[54+:1]), + .io_chi_rxDat_bits_cBusy(vif.rx_dat[55+:3]), + .io_chi_rxDat_bits_dbid(vif.rx_dat[58+:16]), + .io_chi_rxDat_bits_ccid(vif.rx_dat[74+:2]), + .io_chi_rxDat_bits_dataId(vif.rx_dat[76+:2]), + .io_chi_rxDat_bits_cacheLineId(vif.rx_dat[78+:6]), + .io_chi_rxDat_bits_tagOp(vif.rx_dat[84+:2]), + .io_chi_rxDat_bits_tag(vif.rx_dat[86+:8]), + .io_chi_rxDat_bits_tagUpdate(vif.rx_dat[94+:2]), + .io_chi_rxDat_bits_traceTag(vif.rx_dat[96+:1]), + .io_chi_rxDat_bits_copyAtHome(vif.rx_dat[97+:1]), + .io_chi_rxDat_bits_numDat(vif.rx_dat[98+:2]), + .io_chi_rxDat_bits_replicate(vif.rx_dat[100+:1]), + .io_chi_rxDat_bits_be(vif.rx_dat[101+:32]), + .io_chi_rxDat_bits_data(vif.rx_dat[133+:256]), + .io_chi_rxDat_valid(vif.rx_dat_valid), + .io_chi_rxDat_ready(vif.rx_dat_ready), + .io_chi_snp_bits_qos(vif.snp[0+:4]), + .io_chi_snp_bits_srcId(vif.snp[4+:7]), + .io_chi_snp_bits_txnId(vif.snp[11+:12]), + .io_chi_snp_bits_fwdNid(vif.snp[23+:7]), + .io_chi_snp_bits_fwdTxnId(vif.snp[30+:12]), + .io_chi_snp_bits_opcode(vif.snp[42+:5]), + .io_chi_snp_bits_addr(vif.snp[47+:41]), + .io_chi_snp_bits_pas(vif.snp[88+:3]), + .io_chi_snp_bits_doNotGoToSd(vif.snp[91+:1]), + .io_chi_snp_bits_retToSrc(vif.snp[92+:1]), + .io_chi_snp_bits_traceTag(vif.snp[93+:1]), + .io_chi_snp_valid(vif.snp_valid), + .io_chi_snp_ready(vif.snp_ready), + .io_access_valid(vif.access_valid), + .io_access_ready(vif.access_ready), + .io_access_bits_write(vif.access_write), + .io_access_bits_addr(vif.access_addr), + .io_access_bits_data(vif.access_data), + .io_access_bits_mask(vif.access_mask), + .io_access_bits_atomic(vif.access_atomic), + .io_access_bits_atomicWord(vif.access_atomic_word), + .io_result_valid(vif.result_valid), + .io_result_ready(vif.result_ready), + .io_result_bits_error(vif.result_error), + .io_result_bits_data(vif.result_data), + .io_outstanding(vif.outstanding), + .io_maintenance_valid(cmo.valid), + .io_maintenance_ready(cmo.ready), + .io_maintenance_bits_tag(cmo.tag), + .io_maintenance_bits_firstLine(cmo.first_line), + .io_maintenance_bits_lastLine(cmo.last_line), + .io_maintenance_bits_op(cmo.operation), + .io_maintained_valid(cmo.response_valid), + .io_maintained_ready(cmo.response_ready), + .io_maintained_bits_tag(cmo.response_tag), + .io_maintained_bits_ok(cmo.ok), + .io_olderRequestsDrained(cmo.older_drained) + ); + assign vif.reset = timing.reset; + initial begin + uvm_config_db#(virtual chi_rnf_if)::set(null, "uvm_test_top*", "vif", vif); + uvm_config_db#(virtual clock_if)::set(null, "uvm_test_top.env.reset", "timing", timing); + uvm_config_db#(virtual cmo_if)::set(null, "uvm_test_top*", "cmo", cmo); + run_test("cache_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/tests/test.svh new file mode 100644 index 000000000..4768cb593 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/tests/test.svh @@ -0,0 +1,41 @@ +class cache_test extends protocol_test; + `uvm_component_utils(cache_test) + virtual cmo_if cmo; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + protocol_model::type_id::set_type_override(cache_model::get_type()); + super.build_phase(phase); + if (!uvm_config_db#(virtual cmo_if)::get(this, "", "cmo", cmo)) + `uvm_fatal("VIF", "Cache maintenance interface is required") + endfunction + task execute(); + cache_sequence seq = cache_sequence::type_id::create("scenario"); + seq.test_env = test_env; + seq.state = test_env.state; + seq.vif = vif; + seq.scenario = test_env.state.scenario; + seq.cmo = cmo; + if (!$cast(seq.checked_state, test_env.state)) + `uvm_fatal("TYPE", "Maintenance model is required") + if (seq.scenario == "normal") seq.start(null); + else begin + repeat (3) @(negedge vif.clock); + test_env.reset.pulse(4); + cmo.first_line = 'h40000; + cmo.last_line = 'h40000; + cmo.operation = 0; + cmo.tag = 1; + if (seq.scenario == "bad_maintenance_alignment") cmo.first_line = 'h40001; + if (seq.scenario == "bad_maintenance_range") cmo.last_line = 'h3ffc0; + if (seq.scenario == "bad_maintenance_operation") cmo.operation = 3; + cmo.valid = 1; + do @(posedge vif.clock); while (!cmo.ready); + @(negedge vif.clock); + cmo.valid = 0; + repeat (40) @(negedge vif.clock); + `uvm_fatal("MISSING_ASSERTION", "Expected maintenance assertion did not fire") + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/scala/Cache.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/scala/Cache.scala new file mode 100644 index 000000000..408375823 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/scala/Cache.scala @@ -0,0 +1,84 @@ +package framework.core.memory.access.cache + +import framework.lib.arbiter.{RRArbiter} + +import framework.core.memory.access.cache.configs.{Params => ModuleParam} + +import framework.lib.queue.Queue + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.ip.bus.chi.{RequestFlit, RequesterPort} +import framework.ip.bus.chi.rnf.{BankedChiCache, CacheAccess, CacheProbe, CacheResult, RnfParams} +import framework.ip.memory.interlock.{Acknowledgement, Maintenance => Range, Params} + +/** A thread's own L1 and range maintenance client share one explicit CHI requester. */ +@instantiable +class Cache(b: ModuleParam) extends Module { + val config = b.cache + val tracking = b.tracking + + val c = config.chi + + @public + val io = IO(new Bundle { + + val access = Flipped(Decoupled(new CacheAccess(c))) + + val result = + Decoupled(new CacheResult(0)) + + val probe = Option.when(config.probe)(new CacheProbe(c)) + + val maintenance = Flipped(Decoupled(new Range(b.tracking))) + val maintained = Decoupled(new Acknowledgement(b.tracking)) + // Includes previously accepted virtual accesses, page walks and fetches outside this L1. + val olderRequestsDrained = Input(Bool()) + + val chi = new RequesterPort(c) + + val outstanding = Output(UInt(32.W)) + }) + + val cache: Instance[BankedChiCache] = Instantiate(new BankedChiCache(b.cache)) + val maintenance: Instance[Maintenance] = Instantiate(new Maintenance(b)) + val drained = io.olderRequestsDrained && cache.io.outstanding === 0.U + maintenance.io.request.valid := io.maintenance.valid && drained + maintenance.io.request.bits := io.maintenance.bits + io.maintenance.ready := maintenance.io.request.ready && drained + // Existing owners may keep progressing until drained. At admission, a new CPU + // request cannot race the CMO; subsequent conflicts belong to the external interlock. + val admitMaintenance = io.maintenance.valid && io.maintenance.ready + cache.io.access.valid := io.access.valid && !admitMaintenance + cache.io.access.bits := io.access.bits + io.access.ready := cache.io.access.ready && !admitMaintenance + io.result <> cache.io.result + io.probe.foreach(_ <> cache.io.probe.get) + maintenance.io.drained := true.B + io.maintained <> maintenance.io.response + io.outstanding := cache.io.outstanding + + val requests = Instantiate(new RRArbiter( + new RequestFlit(c), + 2 + )) + + requests.io.in(0) <> cache.io.chi.req + requests.io.in(1) <> maintenance.io.req + + val requestQueue = Instantiate(new Queue(new RequestFlit(c), 2, false, false)) + requestQueue.io.enq <> requests.io.out + io.chi.req <> requestQueue.io.deq + io.chi.txRsp <> cache.io.chi.txRsp + io.chi.txDat <> cache.io.chi.txDat + cache.io.chi.rxDat <> io.chi.rxDat + cache.io.chi.snp <> io.chi.snp + + val maintenanceResponse = io.chi.rxRsp.bits.txnId === config.banks.U + maintenance.io.rsp.valid := io.chi.rxRsp.valid && maintenanceResponse + maintenance.io.rsp.bits := io.chi.rxRsp.bits + cache.io.chi.rxRsp.valid := io.chi.rxRsp.valid && !maintenanceResponse + cache.io.chi.rxRsp.bits := io.chi.rxRsp.bits + io.chi.rxRsp.ready := Mux(maintenanceResponse, maintenance.io.rsp.ready, cache.io.chi.rxRsp.ready) +} diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/scala/Emit.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/scala/Emit.scala new file mode 100644 index 000000000..1d2d2e8e9 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/scala/Emit.scala @@ -0,0 +1,34 @@ +package framework.core.memory.access.cache + +import java.nio.file.{Files, Paths} +import framework.ip.bus.chi.{Opcode, RequestFlit, ResponseFlit, SnoopFlit} + +object Emit extends App { + val Array("--config", config, "--output-dir", directory) = args + val p = upickle.default.read[configs.Params](Files.readString(Paths.get(config))) + val c = p.cache.chi + Files.createDirectories(Paths.get(directory)) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Cache(p), + args = Array("--target-dir", s"$directory/Cache", "--split-verilog") + ) + Files.writeString( + Paths.get(directory, "parameters.svh"), + s"""`define CHI_NODE_BITS ${c.nodeIdBits} +`define CHI_ADDRESS_BITS ${c.addressBits} +`define CHI_DATA_BITS ${c.dataBits} +`define CHI_REQ_BITS ${(new RequestFlit(c)).flitWidth} +`define CHI_RSP_BITS ${(new ResponseFlit(c)).flitWidth} +`define CHI_SNP_BITS ${(new SnoopFlit(c)).flitWidth} +`define CHI_NODE_ID ${p.cache.nodeId} +`define CHI_HOME_ID ${p.cache.homeId} +`define CHI_BANKS ${p.cache.banks} +`define CHI_CACHE_LINES ${p.cache.cacheLines} +`define CMO_ID_BITS ${p.tracking.idBits} +`define CMO_CLEAN ${Opcode.CleanShared} +`define CMO_CLEAN_INVALID ${Opcode.CleanInvalid} +`define CMO_INVALID ${Opcode.MakeInvalid} +`define CMO_COMP ${Opcode.Comp} +""" + ) +} diff --git a/examples/cores/rocket/arch/src/main/scala/cpu/Maintenance.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/scala/Maintenance.scala similarity index 81% rename from examples/cores/rocket/arch/src/main/scala/cpu/Maintenance.scala rename to arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/scala/Maintenance.scala index bd2e3af40..dd5f23b84 100644 --- a/examples/cores/rocket/arch/src/main/scala/cpu/Maintenance.scala +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/scala/Maintenance.scala @@ -1,15 +1,20 @@ -package hier.core.rocket +package framework.core.memory.access.cache + +import framework.core.memory.access.cache.configs.{Params => ModuleParam} import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public} -import memcore.bus.chi.{HomeMapping, Opcode, RequestFlit, ResponseFlit} -import memcore.bus.chi.rnf.RnfParams -import memcore.memory.interlock.{Acknowledgement, Maintenance => Range, MaintenanceOp, Params} +import framework.ip.bus.chi.{HomeMapping, Opcode, RequestFlit, ResponseFlit} +import framework.ip.bus.chi.rnf.RnfParams +import framework.ip.memory.interlock.{Acknowledgement, Maintenance => Range, MaintenanceOp, Params} -/** Serial range CMO client sharing its core's CHI node, with a distinct transaction ID. */ +/** Serial range CMO client sharing its thread's CHI node, with a distinct transaction ID. */ @instantiable -class Maintenance(config: RnfParams, tracking: Params) extends Module { +class Maintenance(b: ModuleParam) extends Module { + val config = b.cache + val tracking = b.tracking + val c = config.chi val mapping = HomeMapping(config.homeCount, config.homeId) val transaction = config.banks @@ -18,18 +23,21 @@ class Maintenance(config: RnfParams, tracking: Params) extends Module { @public val io = IO(new Bundle { - val request = Flipped(Decoupled(new Range(tracking))) - val response = Decoupled(new Acknowledgement(tracking)) + val request = Flipped(Decoupled(new Range(b.tracking))) + val response = Decoupled(new Acknowledgement(b.tracking)) // The caller drains older accepted CPU accesses and fills before asserting this. val drained = Input(Bool()) - val req = Decoupled(new RequestFlit(c)) - val rsp = Flipped(Decoupled(new ResponseFlit(c))) - val idle = Output(Bool()) + + val req = Decoupled(new RequestFlit(c)) + + val rsp = Flipped(Decoupled(new ResponseFlit(c))) + + val idle = Output(Bool()) }) val idle :: drain :: send :: waitResponse :: respond :: Nil = Enum(5) val state = RegInit(idle) - val command = Reg(new Range(tracking)) + val command = Reg(new Range(b.tracking)) val address = Reg(UInt(c.addressBits.W)) val succeeded = RegInit(false.B) io.idle := state === idle diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..277b2ba56 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/scala/configs/Params.scala @@ -0,0 +1,15 @@ +package framework.core.memory.access.cache.configs + +import framework.ip.bus.chi.rnf.RnfParams +import framework.ip.memory.interlock.{Params => TrackingParams} +import upickle.default.{macroRW, ReadWriter} + +case class Params(cache: RnfParams, tracking: TrackingParams) + +object Params { + implicit val chiRW: ReadWriter[framework.ip.bus.chi.Params] = macroRW + implicit val cacheRW: ReadWriter[framework.ip.memory.cache.configs.CacheParams] = macroRW + implicit val rnfRW: ReadWriter[RnfParams] = macroRW + implicit val trackingRW: ReadWriter[TrackingParams] = macroRW + implicit val rw: ReadWriter[Params] = macroRW +} diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..8aa5f85e8 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/configs/Params.scala @@ -0,0 +1,21 @@ +package framework.core.memory.access.configs + +import framework.core.memory.access.mmu.configs.{Params => WalkerParam} + +import framework.core.memory.access.{AccessParams, PhysicalRegion} +import upickle.default.{macroRW, ReadWriter} + +case class Params( + memory: AccessParams, + regions: Seq[PhysicalRegion], + translationEntries: Int, + lineBits: Int) { + val walker = WalkerParam(memory.chi, translationEntries) +} + +object Params { + implicit val chiRW: ReadWriter[framework.ip.bus.chi.Params] = macroRW + implicit val regionRW: ReadWriter[PhysicalRegion] = macroRW + implicit val memoryRW: ReadWriter[AccessParams] = macroRW + implicit val rw: ReadWriter[Params] = macroRW +} diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/build.sc b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/build.sc new file mode 100644 index 000000000..3c130a3fe --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/build.sc @@ -0,0 +1,22 @@ +import mill._ +import $file.^.^.^.^.^.^.^.^.^.^.^.settings + + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.core.memory.access.mmu.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Walker.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/configs/uvm.toml b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/configs/uvm.toml new file mode 100644 index 000000000..b961665cb --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/configs/uvm.toml @@ -0,0 +1,9 @@ +[parameters] +entries = 4 + +[parameters.bus] +nodeIdBits = 7 +addressBits = 44 +dataBits = 256 + +[defines] diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/csrc/Cargo.lock b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/csrc/Cargo.lock similarity index 100% rename from arch/src/main/scala/framework/mem-core/mmu/src/csrc/Cargo.lock rename to arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/csrc/Cargo.lock diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/csrc/Cargo.toml b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/csrc/Cargo.toml similarity index 100% rename from arch/src/main/scala/framework/mem-core/mmu/src/csrc/Cargo.toml rename to arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/csrc/Cargo.toml diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/csrc/src/lib.rs b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/csrc/src/lib.rs similarity index 100% rename from arch/src/main/scala/framework/mem-core/mmu/src/csrc/src/lib.rs rename to arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/csrc/src/lib.rs diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/csrc/src/translation.rs b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/csrc/src/translation.rs new file mode 100644 index 000000000..9ba36d3f1 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/csrc/src/translation.rs @@ -0,0 +1,352 @@ +use std::collections::{HashMap, VecDeque}; +use std::ffi::c_void; +#[derive(Clone, Copy)] +struct Pte { + value: u64, + error: bool, +} +#[derive(Clone, Copy)] +struct Leaf { + mode: u32, + root: u64, + vpn: u64, + level: u32, + pte: u64, +} +struct Model { + address_bits: u32, + memory: HashMap, + expected_reads: VecDeque, + tlb: Vec>, + victim: usize, + epoch: u64, + candidate: Option<(Leaf, u64)>, + cached: bool, +} +#[derive(Default)] +struct Translation { + address: u64, + page_fault: bool, + access_fault: bool, + level: u32, +} +impl Translation { + fn fault(page: bool, level: u32) -> Self { + Self { + page_fault: page, + access_fault: !page, + level, + ..Self::default() + } + } +} +struct Request { + address: u64, + mode: u32, + root: u64, + privilege: u32, + write: bool, + execute: bool, + sum: bool, + mxr: bool, +} +fn canonical(address: u64) -> bool { + address <= 0x0000_003f_ffff_ffff || address >= 0xffff_ffc0_0000_0000 +} +impl Model { + fn leaf(&self, r: &Request, level: u32, value: u64) -> Translation { + let ppn = (value >> 10) & ((1u64 << 44) - 1); + let user = value & 16 != 0; + let privilege = if r.privilege == 0 { + user + } else { + !user || (r.sum && !r.execute) + }; + let operation = if r.execute { + value & 8 != 0 + } else if r.write { + value & 4 != 0 + } else { + value & 2 != 0 || (r.mxr && value & 8 != 0) + }; + if !privilege + || !operation + || value & 64 == 0 + || (r.write && value & 128 == 0) + || ppn & ((1u64 << (9 * level)) - 1) != 0 + { + return Translation::fault(true, level); + } + let bytes = 4096u64 << (9 * level); + let address = ppn * 4096 + (r.address & (bytes - 1)); + if address >= (1u64 << self.address_bits) { + Translation::fault(false, level) + } else { + Translation { + address, + level, + ..Translation::default() + } + } + } + fn cached(&self, r: &Request) -> Option { + if r.privilege == 3 || r.mode == 0 || !canonical(r.address) { + return None; + } + self.tlb + .iter() + .flatten() + .find(|entry| { + entry.mode == r.mode + && entry.root == r.root + && entry.vpn == r.address >> (12 + 9 * entry.level) + }) + .map(|entry| self.leaf(r, entry.level, entry.pte)) + } + fn walk(&self, r: &Request, reads: &mut Vec) -> Result { + let limit = 1u64 << self.address_bits; + if r.privilege == 3 || r.mode == 0 { + return Ok(if r.address >= limit { + Translation::fault(false, 0) + } else { + Translation { + address: r.address, + ..Translation::default() + } + }); + } + if !canonical(r.address) { + return Ok(Translation::fault(true, 2)); + } + let mut ppn = r.root; + for level in (0..=2u32).rev() { + let address = ppn * 4096 + ((r.address >> (12 + 9 * level)) & 511) * 8; + if address >= limit { + return Ok(Translation::fault(false, level)); + } + reads.push(address); + let pte = *self.memory.get(&address).ok_or(address)?; + if pte.error { + return Ok(Translation::fault(false, level)); + } + let value = pte.value; + if value & 1 == 0 || value & 6 == 4 || value >> 54 != 0 { + return Ok(Translation::fault(true, level)); + } + ppn = (value >> 10) & ((1u64 << 44) - 1); + if value & 10 != 0 { + return Ok(self.leaf(r, level, value)); + } + if level == 0 || value & 0xd0 != 0 { + return Ok(Translation::fault(true, level)); + } + } + unreachable!() + } +} +#[no_mangle] +pub extern "C" fn mmu_ref_create(address_bits: u32, entries: u32) -> *mut c_void { + assert!((44..=52).contains(&address_bits)); + Box::into_raw(Box::new(Model { + address_bits, + memory: HashMap::new(), + expected_reads: VecDeque::new(), + tlb: vec![None; entries as usize], + victim: 0, + epoch: 0, + candidate: None, + cached: false, + })) + .cast() +} +#[no_mangle] +pub unsafe extern "C" fn mmu_ref_destroy(p: *mut c_void) { + drop(Box::from_raw(p.cast::())); +} +#[no_mangle] +pub unsafe extern "C" fn mmu_ref_clear(p: *mut c_void) { + let m = &mut *p.cast::(); + assert!(m.expected_reads.is_empty()); + m.memory.clear(); +} +#[no_mangle] +pub unsafe extern "C" fn mmu_ref_program(p: *mut c_void, address: u64, value: u64, error: u32) { + assert_eq!(address % 8, 0); + (*p.cast::()).memory.insert( + address, + Pte { + value, + error: error != 0, + }, + ); +} +#[no_mangle] +pub unsafe extern "C" fn mmu_ref_read( + p: *mut c_void, + address: u64, + value: *mut u64, + error: *mut u32, +) -> u32 { + if let Some(pte) = (*p.cast::()).memory.get(&address) { + *value = pte.value; + *error = u32::from(pte.error); + 1 + } else { + 0 + } +} +#[no_mangle] +pub unsafe extern "C" fn mmu_ref_pending(p: *mut c_void) -> u32 { + (*p.cast::()).expected_reads.len() as u32 +} +#[no_mangle] +pub unsafe extern "C" fn mmu_ref_peek(p: *mut c_void, address: *mut u64) -> u32 { + if let Some(next) = (*p.cast::()).expected_reads.front() { + *address = *next; + 1 + } else { + 0 + } +} +#[no_mangle] +pub unsafe extern "C" fn mmu_ref_consume(p: *mut c_void, address: u64) -> u32 { + let queue = &mut (*p.cast::()).expected_reads; + if queue.front() != Some(&address) { + return 0; + } + queue.pop_front(); + 1 +} +#[no_mangle] +pub unsafe extern "C" fn mmu_ref_translate( + p: *mut c_void, + address: u64, + mode: u32, + root: u64, + privilege: u32, + write: u32, + execute: u32, + sum: u32, + mxr: u32, + paddr: *mut u64, + page_fault: *mut u32, + access_fault: *mut u32, + level: *mut u32, +) -> u32 { + assert!(mode == 0 || mode == 8); + assert!(matches!(privilege, 0 | 1 | 3)); + assert!(write == 0 || execute == 0); + assert!(root < 1u64 << 44); + let m = &mut *p.cast::(); + let r = Request { + address, + mode, + root, + privilege, + write: write != 0, + execute: execute != 0, + sum: sum != 0, + mxr: mxr != 0, + }; + let mut reads = Vec::new(); + m.candidate = None; + let cached = m.cached(&r); + m.cached = cached.is_some(); + let result = match cached { + Some(answer) => Ok(answer), + None => m.walk(&r, &mut reads), + }; + match result { + Ok(answer) => { + if !answer.page_fault && !answer.access_fault && !reads.is_empty() { + let pte = m.memory[reads.last().unwrap()].value; + m.candidate = Some(( + Leaf { + mode, + root, + vpn: address >> (12 + 9 * answer.level), + level: answer.level, + pte, + }, + m.epoch, + )); + } + m.expected_reads.extend(reads); + *paddr = answer.address; + *page_fault = u32::from(answer.page_fault); + *access_fault = u32::from(answer.access_fault); + *level = answer.level; + 0 + } + Err(address) => { + *paddr = address; + 1 + } + } +} +#[no_mangle] +pub unsafe extern "C" fn mmu_ref_cached(p: *mut c_void) -> u32 { + u32::from((*p.cast::()).cached) +} +#[no_mangle] +pub unsafe extern "C" fn mmu_ref_result(p: *mut c_void) { + let m = &mut *p.cast::(); + if m.expected_reads.is_empty() { + if let Some((leaf, epoch)) = m.candidate.take() { + if epoch == m.epoch && !m.tlb.is_empty() { + m.tlb[m.victim] = Some(leaf); + m.victim = (m.victim + 1) % m.tlb.len(); + } + } + } +} +#[no_mangle] +pub unsafe extern "C" fn mmu_ref_flush(p: *mut c_void) { + let m = &mut *p.cast::(); + m.tlb.fill(None); + m.epoch += 1; +} +#[no_mangle] +pub unsafe extern "C" fn mmu_ref_reset(p: *mut c_void) { + let m = &mut *p.cast::(); + m.tlb.fill(None); + m.epoch += 1; + m.victim = 0; + m.candidate = None; + m.expected_reads.clear(); +} +#[no_mangle] +pub unsafe extern "C" fn mmu_ref_lookup( + p: *mut c_void, + address: u64, + mode: u32, + root: u64, + privilege: u32, + write: u32, + execute: u32, + sum: u32, + mxr: u32, + paddr: *mut u64, +) -> u32 { + let m = &*p.cast::(); + let r = Request { + address, + mode, + root, + privilege, + write: write != 0, + execute: execute != 0, + sum: sum != 0, + mxr: mxr != 0, + }; + if privilege == 3 || mode == 0 { + *paddr = address; + return u32::from(address < 1u64 << m.address_bits); + } + if let Some(answer) = m.cached(&r) { + *paddr = answer.address; + u32::from(!answer.page_fault && !answer.access_fault) + } else { + 0 + } +} diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/agent/driver.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/agent/driver.svh new file mode 100644 index 000000000..6fc15df50 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/agent/driver.svh @@ -0,0 +1,48 @@ +class driver extends uvm_driver #(request_item); + `uvm_component_utils(driver) + virtual mmu_if vif; + request_item pending; + int mode = 0; + longint unsigned root = 0; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual mmu_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Walker input interface is required") + endfunction + task run_phase(uvm_phase phase); + forever begin + @(negedge vif.clock); + if (pending == null) seq_item_port.try_next_item(pending); + vif.inputs[`MMU_IN_FLUSH_OFFSET+:`MMU_IN_FLUSH_WIDTH] = 0; + vif.inputs[`MMU_IN_REQ_VALID_OFFSET+:`MMU_IN_REQ_VALID_WIDTH] = 0; + if (pending != null) begin + if (pending.kind == 0) begin + mode = pending.mode; + root = pending.root; + vif.inputs[`MMU_REQ_OFFSET+:`MMU_REQ_BITS] = pending.payload; + vif.inputs[`MMU_IN_REQ_VALID_OFFSET+:`MMU_IN_REQ_VALID_WIDTH] = 1; + end else if (pending.kind == 1) vif.inputs[`MMU_IN_FLUSH_OFFSET+:`MMU_IN_FLUSH_WIDTH] = 1; + else begin + vif.inputs[`MMU_LOOKUP_OFFSET+:`MMU_LOOKUP_BITS] = pending.payload; + vif.inputs[`MMU_IN_LOOKUP_CONFIG_MODE_OFFSET+:`MMU_IN_LOOKUP_CONFIG_MODE_WIDTH]=pending.mode; + vif.inputs[`MMU_IN_LOOKUP_CONFIG_ROOTPPN_OFFSET+:`MMU_IN_LOOKUP_CONFIG_ROOTPPN_WIDTH]=pending.root; + end + end + vif.inputs[`MMU_IN_CONFIG_MODE_OFFSET+:`MMU_IN_CONFIG_MODE_WIDTH] = mode; + vif.inputs[`MMU_IN_CONFIG_ROOTPPN_OFFSET+:`MMU_IN_CONFIG_ROOTPPN_WIDTH] = root; + @(posedge vif.clock); + if (!vif.reset && pending != null && (pending.kind != 0 || `MO(vif.outputs, REQ_READY))) begin + if (pending.kind == 0 && pending.change_inputs) begin + mode = pending.next_mode; + root = pending.next_root; + end + #1; + seq_item_port.item_done(); + pending = null; + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/agent/item.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/agent/item.svh new file mode 100644 index 000000000..9c3f6c649 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/agent/item.svh @@ -0,0 +1,43 @@ +typedef logic [`MMU_REQ_BITS-1:0] request_t; +class request_item extends uvm_sequence_item; + `uvm_object_utils(request_item) + request_t payload; + int kind = 0; + int mode; + longint unsigned root; + bit change_inputs = 0; + int next_mode; + longint unsigned next_root; + function new(string name = "request_item"); + super.new(name); + endfunction +endclass +class request_sequence extends uvm_sequence #(request_item); + `uvm_object_utils(request_sequence) + request_item item; + function new(string name = "request_sequence"); + super.new(name); + endfunction + task body(); + start_item(item); + finish_item(item); + endtask +endclass +class sample_item extends uvm_sequence_item; + `uvm_object_utils(sample_item) + logic [`MMU_INPUT_BITS-1:0] inputs; + logic [`MMU_OUTPUT_BITS-1:0] outputs; + bit reset; + function new(string name = "sample_item"); + super.new(name); + endfunction +endclass +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(payload, UVM_ALL_ON) + `uvm_object_utils_end + logic [`MMU_RESP_BITS-1:0] payload; + function new(string name = "result_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/agent/monitor.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/agent/monitor.svh new file mode 100644 index 000000000..bc3f0cc43 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/agent/monitor.svh @@ -0,0 +1,25 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual mmu_if vif; + uvm_analysis_port #(sample_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual mmu_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Walker monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + sample_item t; + forever begin + @(posedge vif.clock); + t = sample_item::type_id::create("observed"); + t.inputs = vif.inputs; + t.outputs = vif.outputs; + t.reset = vif.reset; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/agent/peer.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/agent/peer.svh new file mode 100644 index 000000000..2330591b8 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/agent/peer.svh @@ -0,0 +1,24 @@ +class peer_driver extends uvm_component; + `uvm_component_utils(peer_driver) + virtual mmu_if vif; + model reference; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual mmu_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Walker PTE interface is required") + endfunction + task run_phase(uvm_phase phase); + forever begin + @(negedge vif.clock); + #1; + vif.inputs[`MMU_IN_ACCESS_READY_OFFSET+:`MMU_IN_ACCESS_READY_WIDTH]=!vif.reset&&!reference.hold_access&&reference.cycle%5>=2; + vif.inputs[`MMU_IN_RESP_READY_OFFSET+:`MMU_IN_RESP_READY_WIDTH]=!vif.reset&&!reference.hold_response&&reference.cycle%7>=3; + vif.inputs[`MMU_IN_RESULT_VALID_OFFSET+:`MMU_IN_RESULT_VALID_WIDTH]=!vif.reset&&reference.pending_result&&!reference.hold_result&&reference.cycle>=reference.due; + vif.inputs[`MMU_IN_RESULT_BITS_DATA_OFFSET+:`MMU_IN_RESULT_BITS_DATA_WIDTH]=reference.result_data; + vif.inputs[`MMU_IN_RESULT_BITS_ERROR_OFFSET+:`MMU_IN_RESULT_BITS_ERROR_WIDTH]=reference.result_error; + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/env/env.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/env/env.svh new file mode 100644 index 000000000..ae371c302 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/env/env.svh @@ -0,0 +1,36 @@ +class env extends uvm_env; + `uvm_component_utils(env) + driver source; + monitor mon; + peer_driver peer; + model reference; + reset_controller reset; + uvm_sequencer #(request_item) seqr; + result_board #(result_item, 0) scoreboard; + uvm_analysis_imp #(bit, env) reset_export; + function new(string name, uvm_component parent); + super.new(name, parent); + reset_export = new("reset_export", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + source = driver::type_id::create("source", this); + mon = monitor::type_id::create("monitor", this); + peer = peer_driver::type_id::create("peer", this); + reference = model::type_id::create("model", this); + reset = reset_controller::type_id::create("reset", this); + seqr = new("seqr", this); + scoreboard = result_board#(result_item, 0)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + source.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(reference.analysis_export); + peer.reference = reference; + reference.expected_ap.connect(scoreboard.expected_export); + reference.actual_ap.connect(scoreboard.actual_export); + reference.reset_ap.connect(reset_export); + endfunction + function void write(bit value); + scoreboard.cancel_pending(); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/env/model.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/env/model.svh new file mode 100644 index 000000000..05a77fbcc --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/env/model.svh @@ -0,0 +1,158 @@ +class model extends uvm_subscriber #(sample_item); + `uvm_component_utils(model) + chandle reference; + sample_item latest; + bit active = 0, pending_result = 0, hold_access = 0, hold_response = 0, hold_result = 0; + longint unsigned result_data; + bit result_error; + int due = 0, cycle = 0, accepted = 0, returned = 0, reads = 0, cancelled = 0; + string current_case; + uvm_analysis_port #(result_item) expected_ap, actual_ap; + uvm_analysis_port #(bit) reset_ap; + covergroup behavior with function sample ( + int privilege, bit write, bit execute, bit hit, int level, bit pf, bit af + ); + option.per_instance = 1; + privilege_mode: coverpoint privilege { + bins user_mode = {0}; bins supervisor = {1}; bins machine_mode = {3}; + } + write_access: coverpoint write; + execute_access: coverpoint execute; + cached_translation: coverpoint hit; + page_size: coverpoint level iff (!pf && !af) {bins sizes[] = {[0 : 2]};} + page_fault: coverpoint pf; + access_fault: coverpoint af; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected_ap = new("expected_ap", this); + actual_ap = new("actual_ap", this); + reset_ap = new("reset_ap", this); + behavior = new(); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + reference = mmu_ref_create(`MMU_ADDRESS_BITS, `MMU_TLB_ENTRIES); + endfunction + function void write(sample_item t); + result_item expected, actual; + longint unsigned paddr, next_address, value, probe_paddr, address; + int unsigned pf, af, level, error, probe_hit; + bit req_fire, resp_fire, pte_result; + latest = t; + if (t.reset) begin + reset_ap.write(1); + if (active) cancelled++; + active = 0; + pending_result = 0; + mmu_ref_reset(reference); + return; + end + cycle++; + probe_hit = mmu_ref_lookup( + reference, + t.inputs[`MMU_IN_LOOKUP_REQ_VADDR_OFFSET+:`MMU_IN_LOOKUP_REQ_VADDR_WIDTH], + t.inputs[`MMU_IN_LOOKUP_CONFIG_MODE_OFFSET+:`MMU_IN_LOOKUP_CONFIG_MODE_WIDTH], + t.inputs[`MMU_IN_LOOKUP_CONFIG_ROOTPPN_OFFSET+:`MMU_IN_LOOKUP_CONFIG_ROOTPPN_WIDTH], + t.inputs[`MMU_IN_LOOKUP_REQ_PRIVILEGE_OFFSET+:`MMU_IN_LOOKUP_REQ_PRIVILEGE_WIDTH], + t.inputs[`MMU_IN_LOOKUP_REQ_WRITE_OFFSET+:`MMU_IN_LOOKUP_REQ_WRITE_WIDTH], + t.inputs[`MMU_IN_LOOKUP_REQ_EXECUTE_OFFSET+:`MMU_IN_LOOKUP_REQ_EXECUTE_WIDTH], + t.inputs[`MMU_IN_LOOKUP_REQ_SUM_OFFSET+:`MMU_IN_LOOKUP_REQ_SUM_WIDTH], + t.inputs[`MMU_IN_LOOKUP_REQ_MXR_OFFSET+:`MMU_IN_LOOKUP_REQ_MXR_WIDTH], + probe_paddr + ); + evidence::check(`MO(t.outputs, LOOKUP_HIT) === 1'(probe_hit), $sformatf( + "%s lookup hit mismatch va=%h mode=%0d root=%h expected=%0d got=%0d", + current_case, + t.inputs[`MMU_IN_LOOKUP_REQ_VADDR_OFFSET+:`MMU_IN_LOOKUP_REQ_VADDR_WIDTH], + t.inputs[`MMU_IN_LOOKUP_CONFIG_MODE_OFFSET+:`MMU_IN_LOOKUP_CONFIG_MODE_WIDTH], + t.inputs[`MMU_IN_LOOKUP_CONFIG_ROOTPPN_OFFSET+:`MMU_IN_LOOKUP_CONFIG_ROOTPPN_WIDTH], + probe_hit, + `MO(t.outputs, LOOKUP_HIT) + )); + if (probe_hit) + evidence::check(`MO(t.outputs, LOOKUP_PADDR) === `MMU_ADDRESS_BITS'(probe_paddr), + "Walker lookup returned the wrong physical address"); + req_fire = t.inputs[`MMU_IN_REQ_VALID_OFFSET+:`MMU_IN_REQ_VALID_WIDTH] && `MO(t.outputs, + REQ_READY); + resp_fire = + `MO(t.outputs, RESP_VALID) + && t.inputs[`MMU_IN_RESP_READY_OFFSET+:`MMU_IN_RESP_READY_WIDTH]; + pte_result = t.inputs[`MMU_IN_RESULT_VALID_OFFSET+:`MMU_IN_RESULT_VALID_WIDTH] && `MO( + t.outputs, RESULT_READY); + if (resp_fire) begin + evidence::check(active, "Walker returned an unsolicited translation"); + actual = result_item::type_id::create("actual"); + actual.payload = t.outputs[`MMU_RESP_OFFSET+:`MMU_RESP_BITS]; + actual_ap.write(actual); + active = 0; + returned++; + end + if (req_fire) begin + evidence::check(!active, "Walker overlapped address translations"); + if (mmu_ref_translate( + reference, + t.inputs[`MMU_IN_REQ_BITS_VADDR_OFFSET+:`MMU_IN_REQ_BITS_VADDR_WIDTH], + t.inputs[`MMU_IN_CONFIG_MODE_OFFSET+:`MMU_IN_CONFIG_MODE_WIDTH], + t.inputs[`MMU_IN_CONFIG_ROOTPPN_OFFSET+:`MMU_IN_CONFIG_ROOTPPN_WIDTH], + t.inputs[`MMU_IN_REQ_BITS_PRIVILEGE_OFFSET+:`MMU_IN_REQ_BITS_PRIVILEGE_WIDTH], + t.inputs[`MMU_IN_REQ_BITS_WRITE_OFFSET+:`MMU_IN_REQ_BITS_WRITE_WIDTH], + t.inputs[`MMU_IN_REQ_BITS_EXECUTE_OFFSET+:`MMU_IN_REQ_BITS_EXECUTE_WIDTH], + t.inputs[`MMU_IN_REQ_BITS_SUM_OFFSET+:`MMU_IN_REQ_BITS_SUM_WIDTH], + t.inputs[`MMU_IN_REQ_BITS_MXR_OFFSET+:`MMU_IN_REQ_BITS_MXR_WIDTH], + paddr, + pf, + af, + level + ) != 0) + `uvm_fatal("PTE_FIXTURE", $sformatf( + "%s requires an unprogrammed PTE at %h", current_case, paddr)) + expected = result_item::type_id::create("expected"); + expected.payload = '0; + expected.payload[`MMU_OUT_RESP_BITS_PADDR_OFFSET-`MMU_RESP_OFFSET+:`MMU_OUT_RESP_BITS_PADDR_WIDTH]=paddr; + expected.payload[`MMU_OUT_RESP_BITS_PAGEFAULT_OFFSET-`MMU_RESP_OFFSET] = pf; + expected.payload[`MMU_OUT_RESP_BITS_ACCESSFAULT_OFFSET-`MMU_RESP_OFFSET] = af; + expected.payload[`MMU_OUT_RESP_BITS_LEVEL_OFFSET-`MMU_RESP_OFFSET+:2] = level; + expected_ap.write(expected); + active = 1; + accepted++; + behavior.sample(t.inputs[`MMU_IN_REQ_BITS_PRIVILEGE_OFFSET+:`MMU_IN_REQ_BITS_PRIVILEGE_WIDTH], + t.inputs[`MMU_IN_REQ_BITS_WRITE_OFFSET+:`MMU_IN_REQ_BITS_WRITE_WIDTH], + t.inputs[`MMU_IN_REQ_BITS_EXECUTE_OFFSET+:`MMU_IN_REQ_BITS_EXECUTE_WIDTH], + mmu_ref_cached(reference), level, pf, af); + end + if (pte_result) begin + evidence::check(pending_result, "Walker accepted an unsolicited PTE result"); + mmu_ref_result(reference); + pending_result = 0; + end + if (`MO(t.outputs, ACCESS_VALID)) begin + address = `MO(t.outputs, ACCESS_BITS_ADDR); + evidence::check(mmu_ref_peek(reference, next_address + ) && address === next_address && address[2:0] == 0, $sformatf( + "%s unexpected PTE access %h", current_case, address)); + if (t.inputs[`MMU_IN_ACCESS_READY_OFFSET+:`MMU_IN_ACCESS_READY_WIDTH]) begin + evidence::check(!pending_result, "Walker overlapped PTE accesses"); + if (!mmu_ref_read(reference, address, value, error)) + `uvm_fatal("PTE_FIXTURE", "Walker read an unprogrammed PTE") + evidence::check(mmu_ref_consume(reference, address), "Walker skipped an expected PTE read"); + result_data = value; + result_error = error; + reads++; + due = cycle + 2 + reads % 4; + pending_result = 1; + end + end + if (t.inputs[`MMU_IN_FLUSH_OFFSET+:`MMU_IN_FLUSH_WIDTH]) mmu_ref_flush(reference); + endfunction + function void check_phase(uvm_phase phase); + super.check_phase(phase); + evidence::check(!active && !pending_result && mmu_ref_pending(reference + ) == 0 && accepted == returned + cancelled, + "Walker transactions did not drain"); + endfunction + function void final_phase(uvm_phase phase); + mmu_ref_destroy(reference); + super.final_phase(phase); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/filelist.f b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/filelist.f new file mode 100644 index 000000000..b9e867ad7 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/Walker/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/interface.sv b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/interface.sv new file mode 100644 index 000000000..3bfcf2600 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/interface.sv @@ -0,0 +1,20 @@ +`include "parameters.svh" +interface mmu_if ( + input logic clock +); + logic reset; + logic [`MMU_INPUT_BITS-1:0] inputs = '0; + logic [`MMU_OUTPUT_BITS-1:0] outputs; + assert property (@(posedge clock) disable iff (reset) + outputs[`MMU_OUT_ACCESS_VALID_OFFSET] && !inputs[`MMU_IN_ACCESS_READY_OFFSET] |=> + outputs[`MMU_OUT_ACCESS_VALID_OFFSET] && $stable( + outputs[`MMU_OUT_ACCESS_BITS_ADDR_OFFSET+:`MMU_OUT_ACCESS_BITS_ADDR_WIDTH] + )) + else $fatal(1, "Walker PTE address changed under backpressure"); + assert property (@(posedge clock) disable iff (reset) + outputs[`MMU_OUT_RESP_VALID_OFFSET] && !inputs[`MMU_IN_RESP_READY_OFFSET] |=> + outputs[`MMU_OUT_RESP_VALID_OFFSET] && $stable( + outputs[`MMU_RESP_OFFSET+:`MMU_RESP_BITS] + )) + else $fatal(1, "Walker translation changed under backpressure"); +endinterface diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/package.sv b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/package.sv new file mode 100644 index 000000000..33cded8b5 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/package.sv @@ -0,0 +1,76 @@ +`include "parameters.svh" +`define MI(P, N) P[`MMU_IN_``N``_OFFSET+:`MMU_IN_``N``_WIDTH] +`define MO(P, N) P[`MMU_OUT_``N``_OFFSET+:`MMU_OUT_``N``_WIDTH] +`define RQ(N) `MMU_IN_REQ_BITS_``N``_OFFSET-`MMU_REQ_OFFSET+:`MMU_IN_REQ_BITS_``N``_WIDTH +package mmu_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + typedef bit [63:0] u64; + import "DPI-C" function chandle mmu_ref_create( + input int unsigned address_bits, + entries + ); + import "DPI-C" function void mmu_ref_destroy(input chandle model); + import "DPI-C" function void mmu_ref_clear(input chandle model); + import "DPI-C" function void mmu_ref_program( + input chandle model, + input longint unsigned address, + value, + input int unsigned error + ); + import "DPI-C" function int unsigned mmu_ref_read( + input chandle model, + input longint unsigned address, + output longint unsigned value, + output int unsigned error + ); + import "DPI-C" function int unsigned mmu_ref_pending(input chandle model); + import "DPI-C" function int unsigned mmu_ref_peek( + input chandle model, + output longint unsigned address + ); + import "DPI-C" function int unsigned mmu_ref_consume( + input chandle model, + input longint unsigned address + ); + import "DPI-C" function int unsigned mmu_ref_translate( + input chandle model, + input longint unsigned address, + input int unsigned mode, + input longint unsigned root, + input int unsigned privilege, + wr, + execute, + sum, + mxr, + output longint unsigned paddr, + output int unsigned pf, + af, + level + ); + import "DPI-C" function int unsigned mmu_ref_lookup( + input chandle model, + input longint unsigned address, + input int unsigned mode, + input longint unsigned root, + input int unsigned privilege, + wr, + execute, + sum, + mxr, + output longint unsigned paddr + ); + import "DPI-C" function int unsigned mmu_ref_cached(input chandle model); + import "DPI-C" function void mmu_ref_result(input chandle model); + import "DPI-C" function void mmu_ref_reset(input chandle model); + import "DPI-C" function void mmu_ref_flush(input chandle model); + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "env/model.svh" + `include "agent/peer.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..e8c91cc61 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/seq/sequence.svh @@ -0,0 +1,331 @@ +class scenario_sequence extends uvm_sequence; + `uvm_object_utils(scenario_sequence) + env test_env; + virtual mmu_if vif; + localparam u64 MAX_PA = (64'h1 << `MMU_ADDRESS_BITS) - 1; + localparam u64 HIGH_PPN = 64'h1 << (`MMU_ADDRESS_BITS - 12); + function new(string name = "scenario_sequence"); + super.new(name); + endfunction + task operation(int kind, request_t payload = '0, int mode = 8, u64 root = 16, bit change = 0, + int next_mode = 0, u64 next_root = 0); + request_sequence seq = request_sequence::type_id::create("operation"); + seq.item = request_item::type_id::create("item"); + seq.item.kind = kind; + seq.item.payload = payload; + seq.item.mode = mode; + seq.item.root = root; + seq.item.change_inputs = change; + seq.item.next_mode = next_mode; + seq.item.next_root = next_root; + seq.start(test_env.seqr); + endtask + task flush(); + operation(1); + endtask + task probe(request_t r, int mode = 8, u64 root = 16); + operation(2, r, mode, root); + endtask + task send(request_t r, int mode = 8, u64 root = 16, bit change_inputs = 0, int next_mode = 0, + u64 next_root = 0); + operation(0, r, mode, root, change_inputs, next_mode, next_root); + endtask + task run_case(string name, request_t r, int mode = 8, u64 root = 16, bit change_inputs = 0); + int target = test_env.scoreboard.checked + 1; + test_env.reference.current_case = name; + send(r, mode, root, change_inputs); + test_env.scoreboard.wait_checked(target); + @(negedge vif.clock); + evidence::check(mmu_ref_pending(test_env.reference.reference + ) == 0 && !test_env.reference.pending_result, + "Walker skipped expected PTE responses"); + endtask + function request_t request(u64 va, int operation = 0, int privilege = 1, bit sum = 0, + bit mxr = 0); + request_t r = '0; + r[`RQ(VADDR)] = va; + r[`RQ(WRITE)] = operation == 1; + r[`RQ(EXECUTE)] = operation == 2; + r[`RQ(PRIVILEGE)] = privilege; + r[`RQ(SUM)] = sum; + r[`RQ(MXR)] = mxr; + return r; + endfunction + function void program_pte(u64 address, u64 value, bit error = 0); + mmu_ref_program(test_env.reference.reference, address, value, error); + endfunction + function u64 pte(u64 address); + longint unsigned value; + int unsigned error; + if (!mmu_ref_read(test_env.reference.reference, address, value, error)) + `uvm_fatal("FIXTURE", "Attempt to modify an unprogrammed PTE") + return value; + endfunction + function u64 pte_address(u64 va, int level); + return (64'h12 - level) * 4096 + ((va >> (12 + 9 * level)) & 511) * 8; + endfunction + task prepare(u64 va, int leaf_level, u64 flags = 64'hcf, u64 mapped = 64'h80000000); + flush(); + mmu_ref_clear(test_env.reference.reference); + for (int level = 2; level >= leaf_level; level--) begin + if (level == leaf_level) program_pte(pte_address(va, level), ((mapped >> 12) << 10) | flags); + else program_pte(pte_address(va, level), ((64'h13 - level) << 10) | 1); + end + endtask + task table(u64 address, u64 mapped, u64 flags = 64'hcf, u64 root = 16); + for (int level = 2; level >= 0; level--) begin + u64 word_address = (root + 2 - level) * 4096 + ((address >> (12 + 9 * level)) & 511) * 8; + program_pte(word_address, + level == 0 ? ((mapped >> 12) << 10) | flags : ((root + 3 - level) << 10) | 1); + end + endtask + task body(); + u64 va = 64'h123456789, flags, mapped; + int permissions[5] = '{1, 3, 4, 5, 7}; + int target, before_reads; + test_env.reset.pulse(4); + prepare(va, 0, 64'h43, 64'h81234000); + target = test_env.scoreboard.checked + 2; + test_env.reference.current_case = "back_to_back_cached_permission_recheck"; + test_env.reference.hold_access = 1; + test_env.reference.hold_response = 1; + send(request(va)); + fork + send(request(va, 2)); + begin + wait (`MO(test_env.reference.latest.outputs, ACCESS_VALID)); + repeat (8) @(negedge vif.clock); + test_env.reference.hold_access = 0; + wait (`MO(test_env.reference.latest.outputs, RESP_VALID)); + repeat (8) @(negedge vif.clock); + test_env.reference.hold_response = 0; + end + join + test_env.scoreboard.wait_checked(target); + // Three page sizes and both canonical sign regions, including page edges. + for (int level = 0; level < 3; level++) begin + for (int position = 0; position < 3; position++) begin + u64 size = 64'h1000 << (9 * level); + u64 address = (va & ~(size - 1)) + (position == 0 ? 0 : position == 1 ? 123 : size - 1); + prepare(address, level); + run_case($sformatf("page_size_%0d_edge_%0d", level, position), request(address)); + address |= 64'hffffffc000000000; + prepare(address, level); + run_case($sformatf("negative_canonical_%0d_%0d", level, position), request(address)); + end + end + // Permissions: all valid RWX forms, U/S, U-page, SUM, MXR, load/store/fetch. + foreach (permissions[i]) + for (int privilege = 0; privilege < 2; privilege++) + for (int user_page = 0; user_page < 2; user_page++) + for (int sum = 0; sum < 2; sum++) + for (int mxr = 0; mxr < 2; mxr++) + for (int operation = 0; operation < 3; operation++) begin + flags = 64'hc1 | (u64'(permissions[i]) << 1) | (u64'(user_page) << 4); + prepare(va, 0, flags, 64'h81234000); + run_case($sformatf( + "perm_%0d_prv%0d_u%0d_sum%0d_mxr%0d_op%0d", + permissions[i], + privilege, + user_page, + sum, + mxr, + operation + ), request(va, operation, privilege, sum, mxr)); + end + for (int level = 0; level < 3; level++) + for (int ad = 0; ad < 4; ad++) + for (int operation = 0; operation < 3; operation++) begin + flags = 64'hf | (u64'(ad) << 6); + prepare(va, level, flags); + run_case($sformatf("svade_level%0d_ad%0d_op%0d", level, ad, operation), request( + va, operation)); + end + for (int level = 0; level < 3; level++) begin + for (int reserved_bit = 54; reserved_bit < 64; reserved_bit++) begin + prepare(va, level); + program_pte(pte_address(va, level), pte(pte_address(va, level)) | (64'h1 << reserved_bit)); + run_case($sformatf("reserved_leaf_l%0d_bit%0d", level, reserved_bit), request(va)); + end + for (int encoding = 0; encoding < 3; encoding++) begin + prepare(va, level, encoding == 0 ? 64'hce : encoding == 1 ? 64'hc5 : 64'hcd); + run_case($sformatf("invalid_pte_l%0d_encoding%0d", level, encoding), request(va)); + end + prepare(va, level); + program_pte(pte_address(va, level), '1, 1); + run_case($sformatf("backend_error_l%0d", level), request(va)); + prepare(va, level, 64'hcf, HIGH_PPN * 4096); + run_case($sformatf("physical_leaf_overflow_l%0d", level), request(va)); + prepare(va, level, 64'hf, HIGH_PPN * 4096); + run_case($sformatf("pte_fault_precedes_final_pa_overflow_l%0d", level), request(va)); + end + for (int level = 1; level <= 2; level++) begin + for (int bit_index = 0; bit_index < 9 * level; bit_index++) begin + prepare(va, level); + program_pte(pte_address(va, level), pte(pte_address(va, level) + ) | (64'h1 << (10 + bit_index))); + run_case($sformatf("superpage_alignment_l%0d_bit%0d", level, bit_index), request(va)); + end + for (int bad = 0; bad < 3; bad++) begin + prepare(va, 0); + program_pte(pte_address(va, level), pte(pte_address(va, level) + ) | (64'h1 << (bad == 0 ? 4 : bad == 1 ? 6 : 7))); + run_case($sformatf("nonleaf_reserved_l%0d_%0d", level, bad), request(va)); + end + for (int bit_index = 54; bit_index < 64; bit_index++) begin + prepare(va, 0); + program_pte(pte_address(va, level), pte(pte_address(va, level)) | (64'h1 << bit_index)); + run_case($sformatf("nonleaf_upper_reserved_l%0d_%0d", level, bit_index), request(va)); + end + prepare(va, 0); + program_pte(pte_address(va, level), (HIGH_PPN << 10) | 1); + run_case($sformatf("next_table_address_overflow_l%0d", level), request(va)); + end + prepare(va, 0); + program_pte(pte_address(va, 0), (64'h13 << 10) | 1); + run_case("nonleaf_at_bottom", request(va)); + prepare(va, 0, 64'h3ef); // G and both RSW bits are permitted and ignored for translation. + for (int level = 1; level <= 2; level++) + program_pte(pte_address(va, level), pte(pte_address(va, level)) | 64'h320); + run_case("software_and_global_bits", request(va)); + prepare(va, 0, 64'h59); + run_case("latched_root_sum_mxr_privilege", request(va, 0, 1, 1, 1), 8, 16, 1); + mmu_ref_clear(test_env.reference.reference); + run_case("root_physical_overflow", request(va), 8, HIGH_PPN); + run_case("positive_noncanonical", request(64'h0000004000000000)); + run_case("upper_noncanonical", request(64'hffffffbfffffffff)); + for (int operation = 0; operation < 3; operation++) begin + for (int privilege = 0; privilege < 2; privilege++) begin + run_case("bare_noncanonical_physical", request(64'h8000000000, operation, privilege), 0, 0); + run_case("bare_last_physical_byte", request(MAX_PA, operation, privilege), 0, 0); + run_case("bare_physical_overflow", request(MAX_PA + 1, operation, privilege), 0, 0); + end + run_case("machine_ignores_sv39", request(64'h8000000000, operation, 3), 8, HIGH_PPN); + run_case("machine_physical_overflow", request(MAX_PA + 1, operation, 3), 8, 16); + end + // Single-address translations on opposite sides of a base-page boundary. + prepare(64'hfff, 0, 64'hcf, MAX_PA & ~64'hfff); + run_case("last_byte_before_physical_limit", request(64'hfff)); + prepare(64'h1000, 0, 64'hcf, MAX_PA + 1); + run_case("next_page_physical_overflow", request(64'h1000)); + // The last PTE word of the highest implemented physical table page is legal. + mmu_ref_clear(test_env.reference.reference); + program_pte(MAX_PA - 7, 64'hcf); + run_case("last_physical_pte_word", request(64'hffffffffc0000000), 8, MAX_PA >> 12); + // Warm translations re-check permissions without re-reading page tables. + prepare(va, 0, 64'h4f, 64'h81234000); + run_case("cached_read", request(va)); + before_reads = test_env.reference.reads; + run_case("cached_store_dirty_fault", request(va, 1)); + run_case("cached_user_fault", request(va, 0, 0)); + evidence::check(test_env.reference.reads == before_reads, + "Walker did not reuse its cached leaf"); + probe(request(va)); + probe(request(va, 0, 0)); + // Updating memory without SFENCE does not change an already cached leaf. + program_pte(pte_address(va, 0), ((64'h84567000 >> 12) << 10) | 64'hcf); + run_case("cached_old_pte", request(va)); + flush(); + run_case("after_sfence_new_pte", request(va)); + // Every configured entry and round-robin eviction, checked through the public lookup port. + flush(); + mmu_ref_clear(test_env.reference.reference); + for (int index = 0; index <= `MMU_TLB_ENTRIES; index++) + table('h400000 + index * 4096, 'h90000000 + index * 4096); + for (int index = 0; index < `MMU_TLB_ENTRIES; index++) + run_case("fill_tlb", request('h400000 + index * 4096)); + for (int index = 0; index < `MMU_TLB_ENTRIES; index++) probe(request('h400000 + index * 4096)); + run_case("evict_oldest_tlb_entry", request('h400000 + `MMU_TLB_ENTRIES * 4096)); + for (int index = 0; index <= `MMU_TLB_ENTRIES; index++) probe(request('h400000 + index * 4096)); + // A changed live config cannot relabel a walk that was accepted under another root. + prepare(va, 0, 64'hcf, 64'h81234000); + target = test_env.scoreboard.checked + 1; + test_env.reference.current_case = "latched_tlb_fill_root"; + send(request(va), 8, 16, 1, 8, 32); + test_env.scoreboard.wait_checked(target); + probe(request(va), 8, 16); + probe(request(va), 8, 32); + table(va, 64'h84567000, 64'hcf, 32); + run_case("different_root_same_vpn", request(va), 8, 32); + probe(request(va), 8, 16); + probe(request(va), 8, 32); + // SFENCE during an outstanding PTE response must prevent a later stale TLB refill. + prepare(va, 0, 64'hcf, 64'h81234000); + target = test_env.scoreboard.checked + 1; + test_env.reference.current_case = "flush_during_walk"; + test_env.reference.hold_result = 1; + send(request(va)); + wait (test_env.reference.pending_result); + flush(); + test_env.reference.hold_result = 0; + test_env.scoreboard.wait_checked(target); + probe(request(va)); + // SFENCE wins when the final leaf response and flush share a clock edge. + prepare(va, 0, 64'hcf, 64'h81234000); + target = test_env.scoreboard.checked + 1; + test_env.reference.current_case = "flush_on_fill"; + send(request(va)); + wait (test_env.reference.pending_result && mmu_ref_pending(test_env.reference.reference) == 0); + test_env.reference.hold_result = 1; + repeat (5) @(negedge vif.clock); + fork + flush(); + begin + wait (`MI(vif.inputs, FLUSH)); + test_env.reference.hold_result = 0; + end + join + test_env.scoreboard.wait_checked(target); + probe(request(va)); + // Reset cancellation at the offer, memory wait and held translation response boundaries. + for (int phase_index = 0; phase_index < 3; phase_index++) begin + prepare(va, 0, 64'hcf, 64'h81234000); + test_env.reference.current_case = "reset_cancel_translation"; + test_env.reference.hold_access = phase_index == 0; + test_env.reference.hold_result = phase_index == 1; + test_env.reference.hold_response = phase_index == 2; + send(request(va)); + if (phase_index == 0) wait (`MO(test_env.reference.latest.outputs, ACCESS_VALID)); + else if (phase_index == 1) wait (test_env.reference.pending_result); + else wait (`MO(test_env.reference.latest.outputs, RESP_VALID)); + test_env.reset.pulse(3); + test_env.reference.hold_access = 0; + test_env.reference.hold_result = 0; + test_env.reference.hold_response = 0; + run_case("after_reset_rewalk", request(va)); + end + evidence::check(evidence::cancelled == 3, "Walker reset cancellation count mismatch"); + `uvm_info("WALKER", $sformatf("accepted=%0d returned=%0d PTEreads=%0d reset_cancelled=%0d", + test_env.reference.accepted, test_env.reference.returned, + test_env.reference.reads, test_env.reference.cancelled), UVM_LOW) + endtask +endclass +class invalid_sequence extends uvm_sequence #(request_item); + `uvm_object_utils(invalid_sequence) + function new(string name = "invalid_sequence"); + super.new(name); + endfunction + task body(); + request_item req = request_item::type_id::create("invalid"); + string scenario; + if (!$value$plusargs("CASE=%s", scenario)) + `uvm_fatal("CASE", "Walker assertion scenario is required") + start_item(req); + req.payload = '0; + req.mode = 8; + req.root = 16; + req.payload[`RQ(PRIVILEGE)] = 1; + case (scenario) + "mode": req.mode = 9; + "privilege": req.payload[`RQ(PRIVILEGE)] = 2; + "access": begin + req.payload[`RQ(WRITE)] = 1; + req.payload[`RQ(EXECUTE)] = 1; + end + default: `uvm_fatal("CASE", "Unknown walker assertion scenario") + endcase + finish_item(req); + #100ns; + `uvm_fatal("EXPECTED_ASSERTION", "Invalid walker input did not trigger its assertion") + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/suite.toml b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/suite.toml new file mode 100644 index 000000000..21bc2e14f --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/suite.toml @@ -0,0 +1,71 @@ +[model] +manifest = "src/csrc/Cargo.toml" +library = "libmmu_reference_model" + +[[tests]] +functional_groups = ["mmu_pkg::model::behavior"] +name = "walker" +top = "walker_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 1000000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +functional_groups = [] +name = "walker_bad_access" +top = "walker_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 1000000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=access"] +assertion = "Walker access cannot both write and execute" + +[[tests]] +functional_groups = [] +name = "walker_bad_mode" +top = "walker_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 1000000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=mode"] +assertion = "Walker requires Bare or Sv39 mode" + +[[tests]] +functional_groups = [] +name = "walker_bad_privilege" +top = "walker_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 1000000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=privilege"] +assertion = "Walker effective privilege must be U, S or M" diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/tb.sv b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/tb.sv new file mode 100644 index 000000000..398baf810 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/tb.sv @@ -0,0 +1,19 @@ +module walker_tb; + import uvm_pkg::*; + import mmu_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + clock_if timing (clock); + mmu_if bus (clock); + assign bus.reset = timing.reset; + Walker dut ( + .clock(clock), + .reset(bus.reset), + `include "ports.svh" + ); + initial begin + uvm_config_db#(virtual mmu_if)::set(null, "uvm_test_top*", "vif", bus); + uvm_config_db#(virtual clock_if)::set(null, "uvm_test_top*", "timing", timing); + run_test(); + end +endmodule diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/tests/test.svh new file mode 100644 index 000000000..49cb108dd --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/tests/test.svh @@ -0,0 +1,43 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + virtual mmu_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + if (!uvm_config_db#(virtual mmu_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Walker timing interface is required") + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.test_env = test_env; + seq.vif = vif; + seq.start(null); + endtask +endclass +class invalid_test extends base_test; + `uvm_component_utils(invalid_test) + driver source; + reset_controller reset; + uvm_sequencer #(request_item) seqr; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + source = driver::type_id::create("source", this); + reset = reset_controller::type_id::create("reset", this); + seqr = new("seqr", this); + endfunction + function void connect_phase(uvm_phase phase); + source.seq_item_port.connect(seqr.seq_item_export); + endfunction + task execute(); + invalid_sequence seq = invalid_sequence::type_id::create("scenario"); + reset.pulse(4); + seq.start(seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/scala/Emit.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/scala/Emit.scala new file mode 100644 index 000000000..7e1ccda87 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/scala/Emit.scala @@ -0,0 +1,57 @@ +package framework.core.memory.access.mmu + +import chisel3._ +import java.nio.file.{Files, Paths} +import framework.core.memory.access.mmu.configs.Params + +object Emit extends App { + val Array("--config", config, "--output-dir", output) = args + val p = upickle.default.read[Params](Files.readString(Paths.get(config))) + Files.createDirectories(Paths.get(output)) + val ports = scala.collection.mutable.ArrayBuffer[String]() + val definitions = scala.collection.mutable.ArrayBuffer[String]() + val offsets = scala.collection.mutable.Map[String, Int]() + val widths = scala.collection.mutable.Map[String, Int]() + var inputOffset = 0 + var outputOffset = 0 + + def fields(data: Data, path: String): Seq[(String, Int)] = data match { + case record: Record => record.elements.toSeq.flatMap { case (name, field) => + fields(field, if (path.isEmpty) name else s"${path}_$name") + } + case vector: Vec[_] => vector.toSeq.zipWithIndex.flatMap { case (field, index) => fields(field, s"${path}_$index") } + case leaf => Seq(path -> leaf.getWidth) + } + + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + { + val dut = new Walker(p) + for ((name, width) <- fields(dut.io, "")) { + val input = name.startsWith("config_") || name == "flush" || name.startsWith("lookup_config_") || + name.startsWith("lookup_req_") || name == "req_valid" || name.startsWith("req_bits_") || + name == "resp_ready" || name == "access_ready" || name == "result_valid" || name.startsWith("result_bits_") + val direction = if (input) "IN" else "OUT" + val signal = if (input) "inputs" else "outputs" + val offset = if (input) inputOffset else outputOffset + definitions += s"`define MMU_${direction}_${name.toUpperCase}_OFFSET $offset" + definitions += s"`define MMU_${direction}_${name.toUpperCase}_WIDTH $width" + ports += s".io_$name(bus.$signal[$offset +: $width])" + offsets(name) = offset; widths(name) = width + if (input) inputOffset += width else outputOffset += width + } + dut + }, + args = Array("--target-dir", s"$output/Walker", "--split-verilog") + ) + definitions += s"`define MMU_INPUT_BITS $inputOffset" + definitions += s"`define MMU_OUTPUT_BITS $outputOffset" + definitions += s"`define MMU_TLB_ENTRIES ${p.entries}" + definitions += s"`define MMU_ADDRESS_BITS ${p.bus.addressBits}" + for ((name, prefix) <- Seq("REQ" -> "req_bits_", "RESP" -> "resp_bits_", "LOOKUP" -> "lookup_req_")) { + val names = offsets.keys.filter(_.startsWith(prefix)).toSeq + definitions += s"`define MMU_${name}_OFFSET ${names.map(offsets).min}" + definitions += s"`define MMU_${name}_BITS ${names.map(widths).sum}" + } + Files.writeString(Paths.get(output, "parameters.svh"), definitions.mkString("\n") + "\n") + Files.writeString(Paths.get(output, "ports.svh"), ports.mkString(",\n") + "\n") +} diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/main/scala/Walker.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/scala/Walker.scala similarity index 82% rename from arch/src/main/scala/framework/mem-core/mmu/src/main/scala/Walker.scala rename to arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/scala/Walker.scala index 2a0765a76..af52bbcd8 100644 --- a/arch/src/main/scala/framework/mem-core/mmu/src/main/scala/Walker.scala +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/scala/Walker.scala @@ -1,10 +1,20 @@ -package memcore.memory.mmu +package framework.core.memory.access.mmu + +import framework.core.memory.access.mmu.configs.{Params => ModuleParam} import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public} -import memcore.bus.chi.Params -import memcore.bus.chi.rnf.{CacheAccess, CacheAtomic, CacheResult} + +/** One naturally aligned, read-only 64-bit page-table entry access. */ +class PteRequest(b: ModuleParam) extends Bundle { + val addr = UInt(b.bus.addressBits.W) +} + +class PteResponse extends Bundle { + val data = UInt(64.W) + val error = Bool() +} /** One address translation; the caller resolves effective privilege (including MPRV). */ class Request extends Bundle { @@ -16,7 +26,9 @@ class Request extends Bundle { val mxr = Bool() } -class Response(p: Params) extends Bundle { +class Response(b: ModuleParam) extends Bundle { + val p = b.bus + val paddr = UInt(p.addressBits.W) val pageFault = Bool() val accessFault = Bool() @@ -38,7 +50,9 @@ class Config extends Bundle { * walk, so a caller can take a fast path on a hit. */ @instantiable -class Walker(p: Params = Params(), entries: Int = 0) extends Module { +class Walker(b: ModuleParam) extends Module { + val p = b.bus + val entries = b.entries @public val io = IO(new Bundle { @@ -53,23 +67,25 @@ class Walker(p: Params = Params(), entries: Int = 0) extends Module { } val req = Flipped(Decoupled(new Request)) - val resp = Decoupled(new Response(p)) - val access = Decoupled(new CacheAccess(p)) - val result = Flipped(Decoupled(new CacheResult)) + val resp = Decoupled(new Response(b)) + val access = Decoupled(new PteRequest(b)) + val result = Flipped(Decoupled(new PteResponse)) }) val idle :: issue :: waitResult :: respond :: Nil = Enum(4) val state = RegInit(idle) val request = Reg(new Request) + val requestConfig = Reg(new Config) + val staleFill = RegInit(false.B) val ppn = Reg(UInt(44.W)) val level = RegInit(0.U(2.W)) - val response = Reg(new Response(p)) + val response = Reg(new Response(b)) - private def vpn(address: UInt, walkLevel: UInt): UInt = + def vpn(address: UInt, walkLevel: UInt): UInt = MuxLookup(walkLevel, address(20, 12))(Seq(2.U -> address(38, 30), 1.U -> address(29, 21))) /** Leaf address, permission/A/D fault and physical-width overflow for one request. */ - private def leafOf(pte: UInt, walkLevel: UInt, r: Request): (UInt, Bool, Bool) = { + def leafOf(pte: UInt, walkLevel: UInt, r: Request): (UInt, Bool, Bool) = { val ptePpn = pte(53, 10) val supervisor = r.privilege === 1.U val userPermission = Mux(supervisor, !pte(4) || (r.sum && !r.execute), pte(4)) @@ -99,7 +115,7 @@ class Walker(p: Params = Params(), entries: Int = 0) extends Module { val tlb = Reg(Vec(entries max 1, new Entry)) val tlbVictim = RegInit(0.U(log2Ceil(entries max 2).W)) - private def tlbMatches(vaddr: UInt, config: Config): Seq[Bool] = tlb.zip(tlbValid).map { case (e, valid) => + def tlbMatches(vaddr: UInt, config: Config): Seq[Bool] = tlb.zip(tlbValid).map { case (e, valid) => val vpn = vaddr(38, 12) (entries > 0).B && valid && e.mode === config.mode && e.root === config.rootPpn && MuxLookup(e.level, vpn === e.vpn)(Seq( @@ -109,7 +125,7 @@ class Walker(p: Params = Params(), entries: Int = 0) extends Module { } /** (hit, address, page fault, access fault, level) for a request served without a walk. */ - private def translate(r: Request, config: Config): (Bool, UInt, Bool, Bool, UInt) = { + def translate(r: Request, config: Config): (Bool, UInt, Bool, Bool, UInt) = { val identity = r.privilege === 3.U || config.mode === 0.U val canonical = r.vaddr(63, 39) === Fill(25, r.vaddr(38)) val matches = tlbMatches(r.vaddr, config) @@ -128,7 +144,7 @@ class Walker(p: Params = Params(), entries: Int = 0) extends Module { io.lookup.paddr := paddr } - private def finish( + def finish( address: UInt, pageFault: Bool, accessFault: Bool, @@ -150,7 +166,9 @@ class Walker(p: Params = Params(), entries: Int = 0) extends Module { "Walker effective privilege must be U, S or M" ) assert(!(r.write && r.execute), "Walker access cannot both write and execute") - request := r + request := r + requestConfig := io.config + staleFill := false.B val (hit, paddr, pageFault, accessFault, walkLevel) = translate(r, io.config) when(hit) { finish(paddr, pageFault, accessFault, walkLevel) @@ -165,13 +183,8 @@ class Walker(p: Params = Params(), entries: Int = 0) extends Module { // A PTE index occupies exactly the low 12 bits of its 4-KiB table page. val pteAddress = Cat(ppn, vpn(request.vaddr, level), 0.U(3.W)) val pteAddressOverflow = pteAddress(55, p.addressBits).orR - io.access.valid := state === issue && !pteAddressOverflow - io.access.bits.addr := pteAddress(p.addressBits - 1, 0) - io.access.bits.write := false.B - io.access.bits.data := 0.U - io.access.bits.mask := 0.U - io.access.bits.atomic := CacheAtomic.None.U - io.access.bits.atomicWord := false.B + io.access.valid := state === issue && !pteAddressOverflow + io.access.bits.addr := pteAddress(p.addressBits - 1, 0) when(state === issue && pteAddressOverflow) { finish(0.U, false.B, true.B, level) }.elsewhen(io.access.fire) { @@ -191,10 +204,10 @@ class Walker(p: Params = Params(), entries: Int = 0) extends Module { finish(0.U, true.B, false.B, level) }.elsewhen(leaf) { finish(leafAddress(p.addressBits - 1, 0), leafFault, !leafFault && leafOverflow, level) - if (entries > 0) when(!leafFault && !leafOverflow) { + if (entries > 0) when(!leafFault && !leafOverflow && !staleFill && !io.flush) { tlbValid(tlbVictim) := true.B - tlb(tlbVictim).mode := io.config.mode - tlb(tlbVictim).root := io.config.rootPpn + tlb(tlbVictim).mode := requestConfig.mode + tlb(tlbVictim).root := requestConfig.rootPpn tlb(tlbVictim).vpn := request.vaddr(38, 12) tlb(tlbVictim).level := level tlb(tlbVictim).pte := pte @@ -210,7 +223,10 @@ class Walker(p: Params = Params(), entries: Int = 0) extends Module { } // SFENCE.VMA wins over a same-cycle fill. - when(io.flush)(tlbValid.foreach(_ := false.B)) + when(io.flush) { + tlbValid.foreach(_ := false.B) + staleFill := true.B + } io.resp.valid := state === respond io.resp.bits := response diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..842d96885 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/scala/configs/Params.scala @@ -0,0 +1,11 @@ +package framework.core.memory.access.mmu.configs + +import framework.ip.bus.chi +import upickle.default.{macroRW, ReadWriter} + +case class Params(bus: chi.Params, entries: Int) + +object Params { + implicit val chiRW: ReadWriter[chi.Params] = macroRW + implicit val rw: ReadWriter[Params] = macroRW +} diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/build.sc b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/build.sc new file mode 100644 index 000000000..2f0176451 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/build.sc @@ -0,0 +1,28 @@ +import mill._ +import $file.^.^.^.^.^.^.^.^.^.^.^.settings + + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.core.memory.access.preflight.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Permission.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/PermissionCheck.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Preflight.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Preparation.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/PreparedMap.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Segmenter.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Config.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/configs/uvm.toml b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/configs/uvm.toml new file mode 100644 index 000000000..ad95fb0a7 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/configs/uvm.toml @@ -0,0 +1,22 @@ +[parameters] +contexts=4 +maxRanges=8 +idBits=8 +beatBytes=16 +[parameters.bus] +nodeIdBits=7 +addressBits=44 +dataBits=256 +[parameters.permission] +addressBits=44 +entries=4 +[[parameters.regions]] +base="0" +bytes="17592186044416" +cacheable=true +executable=true +readable=true +writable=true +atomic=true +normal=true +[defines] diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/csrc/Cargo.lock b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/csrc/Cargo.lock similarity index 100% rename from arch/src/main/scala/framework/mem-core/preflight/src/csrc/Cargo.lock rename to arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/csrc/Cargo.lock diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/csrc/Cargo.toml b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/csrc/Cargo.toml similarity index 100% rename from arch/src/main/scala/framework/mem-core/preflight/src/csrc/Cargo.toml rename to arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/csrc/Cargo.toml diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/csrc/src/lib.rs b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/csrc/src/lib.rs similarity index 100% rename from arch/src/main/scala/framework/mem-core/preflight/src/csrc/src/lib.rs rename to arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/csrc/src/lib.rs diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/csrc/src/preparation.rs b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/csrc/src/preparation.rs similarity index 99% rename from arch/src/main/scala/framework/mem-core/preflight/src/csrc/src/preparation.rs rename to arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/csrc/src/preparation.rs index e7be0416c..8e7afff13 100644 --- a/arch/src/main/scala/framework/mem-core/preflight/src/csrc/src/preparation.rs +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/csrc/src/preparation.rs @@ -12,7 +12,7 @@ struct Model { } #[no_mangle] pub extern "C" fn preflight_ref_create(address_bits:u32, beat:u32)->*mut c_void { - Box::into_raw(Box::new(Model{mmu:mmu::mmu_ref_create(address_bits),address_bits,beat:beat as u64, + Box::into_raw(Box::new(Model{mmu:mmu::mmu_ref_create(address_bits, 0),address_bits,beat:beat as u64, plans:HashMap::new(),deny_pte:None,deny_range:None})).cast() } #[no_mangle] diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/csrc/src/prepared_map.rs b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/csrc/src/prepared_map.rs similarity index 100% rename from arch/src/main/scala/framework/mem-core/preflight/src/csrc/src/prepared_map.rs rename to arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/csrc/src/prepared_map.rs diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/agent/service.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/agent/service.svh new file mode 100644 index 000000000..ca4024b56 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/agent/service.svh @@ -0,0 +1,113 @@ +task service(); + forever begin + @(ctl.sample); + if (!ctl.sample.reset) begin + if (authorization.sample.valid) + behavior.sample(0, authorization.sample.bits[`PF(AUTH, WRITE)], 0, + authorization.sample.bits[`PF(AUTH, ISPTE)], !authorization.sample.ready); + if (prepared.sample.valid) + behavior.sample(prepared.sample.bits[`PF(OUT, ERROR)], prepared.sample.bits[`PF(OUT, WRITE) + ], prepared.sample.bits[`PF(OUT, LAST)], 0, !prepared.sample.ready); + cycle++; + if (cmd.sample.valid && !cmd.sample.ready) command_stalls++; + if (prepared.sample.valid && !prepared.sample.ready) output_stalls++; + if (authorization.sample.valid && !authorization.sample.ready) auth_stalls++; + if (cmd.sample.valid && cmd.sample.ready) predict(cmd.sample.bits); + if (permission.sample.valid && permission.sample.ready) begin + verify_contract(auth_pending, "Unsolicited permission consumed"); + if (auth_pte && auth_allow) begin + pte_grant = 1; + grant_id = auth_id; + end + auth_pending = 0; + end + if (authorization.sample.valid && authorization.sample.ready) begin + int unsigned allow; + int id = authorization.sample.bits[`PF(AUTH, ID)]; + verify_contract(!$isunknown(authorization.sample.bits) && !auth_pending, + "Overlapping/unknown authorization"); + verify_contract(preflight_ref_authorize( + model, + id, + authorization.sample.bits[ + `PF(AUTH, PA) + ], + authorization.sample.bits[ + `PF(AUTH, BYTES) + ], + authorization.sample.bits[ + `PF(AUTH, WRITE) + ], + authorization.sample.bits[ + `PF(AUTH, ISPTE) + ], + authorization.sample.bits[ + `PF(AUTH, PRIVILEGE) + ], + allow + ) == 1, "Authorization lost snapshot, range or PTE S privilege"); + auth_pending = 1; + auth_id = id; + auth_allow = allow; + auth_pte = authorization.sample.bits[`PF(AUTH, ISPTE)]; + grant_pa = authorization.sample.bits[`PF(AUTH, PA)]; + auth_due = cycle + 3; + permit_packet = '0; + permit_packet[`PF(PERMIT, ID)] = id; + permit_packet[`PF(PERMIT, ALLOW)] = allow; + auth_count++; + end + if (pte_resp.sample.valid && pte_resp.sample.ready) pte_pending = 0; + if (pte_req.sample.valid && pte_req.sample.ready) begin + u64 value; + int unsigned error; + verify_contract(pte_grant && !pte_pending && !$isunknown(pte_req.sample.bits), + "PTE accessed before authorization"); + verify_contract(pte_req.sample.bits[`PF(PTE, ADDR)] == grant_pa && grant_pa[2:0] == 0, + "PTE request changed address or is not 8-byte aligned"); + verify_contract(preflight_ref_pte(model, grant_id, grant_pa, value, error) == 1, + "PTE address chain differs from architectural walker"); + pte_packet = '0; + pte_packet[`PF(PTERESP, DATA)] = value; + pte_packet[`PF(PTERESP, ERROR)] = error; + pte_id = grant_id; + pte_pending = 1; + pte_due = cycle + 2 + (pte_count % 4); + pte_grant = 0; + pte_count++; + end + if (prepared.sample.valid) begin + int id = prepared.sample.bits[`PF(OUT, ID)]; + verify_contract(live.exists(id) && !$isunknown(prepared.sample.bits), + "Unsolicited/unknown prepared plan"); + verify_contract( + preflight_ref_pending(model, id + ) == 0 && !(auth_pending && auth_id == id) && !(pte_pending && pte_id == id), + "Partial plan exposed before all translations/permissions completed"); + if (prepared.sample.ready) begin + result_item actual = new; + actual.packet = prepared.sample.bits; + actual.set_transaction_id((id << 4) | output_index[id]); + scoreboard.actual_export.write(actual); + output_index[id]++; + segments++; + if (prepared.sample.bits[`PF(OUT, LAST)]) begin + failures[int'(prepared.sample.bits[`PF(OUT, ERROR)])]++; + retired++; + live.delete(id); + output_index.delete(id); + preflight_ref_retire(model, id); + end + end + end + end + @(negedge ctl.clock); + authorization.ready = !ctl.reset && !auth_pending && cycle % 5 >= 2; + permission.valid = !ctl.reset && auth_pending && !hold_permission && cycle >= auth_due; + permission.bits = permit_packet; + pte_req.ready = !ctl.reset && !pte_pending && cycle % 4 != 0; + pte_resp.valid = !ctl.reset && pte_pending && cycle >= pte_due; + pte_resp.bits = pte_packet; + prepared.ready = !ctl.reset && !hold_output && cycle % 7 >= 3; + end +endtask diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/env/env.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/env/env.svh new file mode 100644 index 000000000..120889ac5 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/env/env.svh @@ -0,0 +1,170 @@ +class env extends uvm_env; + `uvm_component_utils(env) + virtual preflight_control_if ctl; + virtual stream_if #(`PF_CMD_WIDTH) cmd; + virtual stream_if #(`PF_OUT_WIDTH) prepared; + virtual stream_if #(`PF_AUTH_WIDTH) authorization; + virtual stream_if #(`PF_PERMIT_WIDTH) permission; + virtual stream_if #(`PF_PTE_WIDTH) pte_req; + virtual stream_if #(`PF_PTERESP_WIDTH) pte_resp; + result_board #(result_item, 1) scoreboard; + chandle model; + bit live[int]; + int output_index[int]; + int cycle = 0, submitted = 0, retired = 0, segments = 0, failures[7] = '{default: 0}, peak = 0; + int + auth_count = 0, + pte_count = 0, + command_stalls = 0, + output_stalls = 0, + auth_stalls = 0, + cancelled = 0; + bit hold_output = 0, hold_permission = 0, auth_pending = 0, pte_pending = 0, pte_grant = 0; + int auth_id, pte_id, grant_id, auth_due, pte_due; + bit auth_allow, auth_pte; + u64 grant_pa; + logic [`PF_PERMIT_WIDTH-1:0] permit_packet; + logic [`PF_PTERESP_WIDTH-1:0] pte_packet; + u64 next_table = 'h10000; + covergroup behavior with function sample (int error, bit writing, bit last, bit pte, bit stall); + option.per_instance = 1; + outcomes: coverpoint error {bins all[] = {[0 : 6]};} + direction: coverpoint writing; + records: coverpoint last; + authorization_kind: coverpoint pte; + backpressure: coverpoint stall; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + behavior = new(); + + endfunction + function void verify_contract(bit ok, string message); + verification_pkg::evidence::checks++; + if (!ok) `uvm_fatal("PREFLIGHT", message) + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + model = preflight_ref_create(`PF_ADDRESS_BITS, `PF_BEAT_BYTES); + scoreboard = result_board#(result_item, 1)::type_id::create("scoreboard", this); + verify_contract(uvm_config_db#(virtual preflight_control_if)::get(this, "", "ctl", ctl), + "Missing control"); + verify_contract(uvm_config_db#(virtual stream_if #(`PF_CMD_WIDTH))::get(this, "", "cmd", cmd), + "Missing command"); + verify_contract(uvm_config_db#(virtual stream_if #(`PF_OUT_WIDTH))::get( + this, "", "prepared", prepared), "Missing output"); + verify_contract(uvm_config_db#(virtual stream_if #(`PF_AUTH_WIDTH))::get( + this, "", "authorization", authorization), "Missing authorization"); + verify_contract(uvm_config_db#(virtual stream_if #(`PF_PERMIT_WIDTH))::get( + this, "", "permission", permission), "Missing permission"); + verify_contract(uvm_config_db#(virtual stream_if #(`PF_PTE_WIDTH))::get( + this, "", "pte_req", pte_req), "Missing PTE request"); + verify_contract(uvm_config_db#(virtual stream_if #(`PF_PTERESP_WIDTH))::get( + this, "", "pte_resp", pte_resp), "Missing PTE response"); + endfunction + function command_t request(int id, u64 base, int rows, int span, int row_stride, int cols = 1, + int col_stride = 0, bit write = 0, int mode = 0, u64 root = 0, + int privilege = 1, bit sum = 0, bit mxr = 0); + command_t q = '0; + q[`PF(CMD, ID)] = id; + q[`PF(CMD, BASEVA)] = base; + q[`PF(CMD, ROWS)] = rows; + q[`PF(CMD, COLUMNS)] = cols; + q[`PF(CMD, SPANBYTES)] = span; + q[`PF(CMD, ROWSTRIDE)] = row_stride; + q[`PF(CMD, COLUMNSTRIDE)] = col_stride; + q[`PF(CMD, WRITE)] = write; + q[`PF(CMD, MODE)] = mode; + q[`PF(CMD, ROOTPPN)] = root; + q[`PF(CMD, PRIVILEGE)] = privilege; + q[`PF(CMD, SUM)] = sum; + q[`PF(CMD, MXR)] = mxr; + return q; + endfunction + function u64 mapping(u64 va, int pages, u64 flags = 'hc7, int backend_error_page = -1); + u64 root = next_table; + next_table += 'h3000; + preflight_ref_program(model, root + ((va >> 30) & 511) * 8, ((root + 4096) >> 12) << 10 | 1, 0); + preflight_ref_program(model, root + 4096 + ((va >> 21) & 511) * 8, + ((root + 8192) >> 12) << 10 | 1, 0); + for (int i = 0; i < pages; i++) begin + u64 a = (va & ~64'hfff) + i * 4096; + // Deliberately noncontiguous 4KiB physical pages; no identity-map shortcut. + preflight_ref_program(model, root + 8192 + ((a >> 12) & 511) * 8, + ((64'h800000 + i * 8192) >> 12) << 10 | flags, i == backend_error_page); + end + return root >> 12; + endfunction + function void predict(command_t q); + int id = q[`PF(CMD, ID)]; + int count; + verify_contract(!live.exists(id), "Duplicate admitted command ID"); + verify_contract(preflight_ref_submit( + model, + id, + q[ + `PF(CMD, BASEVA) + ], + q[ + `PF(CMD, ROWS) + ], + q[ + `PF(CMD, COLUMNS) + ], + q[ + `PF(CMD, SPANBYTES) + ], + q[ + `PF(CMD, COLUMNSTRIDE) + ], + q[ + `PF(CMD, ROWSTRIDE) + ], + q[ + `PF(CMD, WRITE) + ], + q[ + `PF(CMD, MODE) + ], + q[ + `PF(CMD, ROOTPPN) + ], + q[ + `PF(CMD, PRIVILEGE) + ], + q[ + `PF(CMD, SUM) + ], + q[ + `PF(CMD, MXR) + ] + ) == 0, "Uninitialized PTE in reference fixture"); + live[id] = 1; + output_index[id] = 0; + submitted++; + if (live.num() > peak) peak = live.num(); + count = preflight_ref_count(model, id); + for (int i = 0; i < count; i++) begin + result_item expected = new; + u64 va, pa; + int unsigned bytes, write, last, error; + preflight_ref_expected(model, id, i, va, pa, bytes, write, last, error); + expected.packet = '0; + expected.packet[`PF(OUT, ID)] = id; + expected.packet[`PF(OUT, VA)] = va; + expected.packet[`PF(OUT, PA)] = pa; + expected.packet[`PF(OUT, BYTES)] = bytes; + expected.packet[`PF(OUT, WRITE)] = write; + expected.packet[`PF(OUT, LAST)] = last; + expected.packet[`PF(OUT, ERROR)] = error; + expected.set_transaction_id((id << 4) | i); + scoreboard.expected_export.write(expected); + end + endfunction + function void final_phase(uvm_phase phase); + preflight_ref_destroy(model); + super.final_phase(phase); + endfunction + `include "agent/service.svh" + `include "seq/scenario.svh" +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/filelist.f b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/filelist.f new file mode 100644 index 000000000..481c37ef5 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/filelist.f @@ -0,0 +1,9 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/core/stream_if.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/Preflight/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/control_if.sv b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/interface.sv similarity index 100% rename from arch/src/main/scala/framework/mem-core/preflight/src/main/resources/control_if.sv rename to arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/interface.sv diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/package.sv b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/package.sv new file mode 100644 index 000000000..f42b8d59c --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/package.sv @@ -0,0 +1,96 @@ +`include "parameters.svh" +`define PF(K, F) `PF_``K``_``F``_OFFSET +: `PF_``K``_``F``_WIDTH +package preflight_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + typedef longint unsigned u64; + typedef bit [`PF_CMD_WIDTH-1:0] command_t; + import "DPI-C" function chandle preflight_ref_create( + input int unsigned address_bits, + beat + ); + import "DPI-C" function void preflight_ref_destroy(input chandle model); + import "DPI-C" function void preflight_ref_program( + input chandle model, + input longint unsigned address, + value, + input int unsigned error + ); + import "DPI-C" function void preflight_ref_policy( + input chandle model, + input int unsigned pte_valid, + input longint unsigned pte, + input int unsigned range_valid, + input longint unsigned first, + last + ); + import "DPI-C" function int unsigned preflight_ref_submit( + input chandle model, + input int unsigned id, + input longint unsigned base, + input int unsigned rows, + cols, + span, + col_stride, + row_stride, + write, + mode, + input longint unsigned root, + input int unsigned privilege, + sum, + mxr + ); + import "DPI-C" function int unsigned preflight_ref_count( + input chandle model, + input int unsigned id + ); + import "DPI-C" function void preflight_ref_expected( + input chandle model, + input int unsigned id, + index, + output longint unsigned va, + pa, + output int unsigned bytes, + write, + last, + error + ); + import "DPI-C" function int unsigned preflight_ref_authorize( + input chandle model, + input int unsigned id, + input longint unsigned pa, + input int unsigned bytes, + write, + pte, + privilege, + output int unsigned allow + ); + import "DPI-C" function int unsigned preflight_ref_pte( + input chandle model, + input int unsigned id, + input longint unsigned address, + output longint unsigned value, + output int unsigned error + ); + import "DPI-C" function int unsigned preflight_ref_pending( + input chandle model, + input int unsigned id + ); + import "DPI-C" function void preflight_ref_retire( + input chandle model, + input int unsigned id + ); + import "DPI-C" function void preflight_ref_reset(input chandle model); + class result_item extends uvm_sequence_item; + bit [`PF_OUT_WIDTH-1:0] packet; + `uvm_object_utils_begin(result_item) + `uvm_field_int(packet, UVM_DEFAULT) + `uvm_object_utils_end + function new(string name = "result_item"); + super.new(name); + endfunction + endclass + `include "env/env.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/seq/scenario.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/seq/scenario.svh new file mode 100644 index 000000000..966fe31d4 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/seq/scenario.svh @@ -0,0 +1,174 @@ +task submit(command_t q); + @(negedge ctl.clock); + cmd.bits = q; + cmd.valid = 1; + do @(cmd.sample); while (!cmd.sample.ready); + @(negedge ctl.clock); + cmd.valid = 0; + cmd.bits = ~q; +endtask +task drain(); + wait (live.num() == 0); + @(ctl.sample); + verify_contract(!auth_pending && !pte_pending && !pte_grant, + "Preparation endpoints did not drain"); +endtask +task execute(); + u64 root, root2; + int prior, gap_stride; + command_t q; + ctl.reset = 1; + cmd.valid = 0; + cmd.bits = '0; + prepared.ready = 0; + authorization.ready = 0; + permission.valid = 0; + permission.bits = '0; + pte_req.ready = 0; + pte_resp.valid = 0; + pte_resp.bits = '0; + repeat (5) @(negedge ctl.clock); + ctl.reset = 0; + fork + service(); + join_none + // Current BERT maximum: 1024 dense 16B rows normalize to five pages. + root = mapping('h400000f0, 9); + root2 = mapping('h50000000, 9); + hold_output = 1; + submit(request(1, 'h400000f0, 1024, 16, 16, 1, 0, 0, 8, root)); + submit(request(2, 'h50000000, 2048, 16, 16, 1, 0, 1, 8, root2)); // exactly eight pages + submit(request(3, 'h60000ff8, 1, 16, 16)); // alignment-expanded beat crosses page + submit(request(4, 'h70000000, 1, 16, 16)); + wait (prepared.sample.valid); + repeat (8) @(ctl.sample); + fork + submit(request(5, 'h70002000, 1, 16, 16)); + begin + repeat (12) @(ctl.sample); + hold_output = 0; + end + join + drain(); + submit(request(6, 'h50000000, 2304, 16, 16, 1, 0, 0, 8, root2)); + drain(); // ninth page: one error, zero successful output + submit(request(7, 'h60000000, 2, 16, 8192, 2, 4096)); + drain(); // sparse 2D, four extents + submit(request(8, 'h60000000, 1, 16, 16)); + drain(); // progress immediately after capacity error + root = mapping('h40010000, 1, 'h43); + submit(request(9, 'h40010000, 1, 16, 16, 1, 0, 1, 8, root)); + drain(); + root = mapping('h40020000, 1, 'h7); + submit(request(10, 'h40020000, 1, 16, 16, 1, 0, 0, 8, root)); + drain(); + root = mapping('h40030000, 1, 'hc7, 0); + submit(request(11, 'h40030000, 1, 16, 16, 1, 0, 0, 8, root)); + drain(); + root = mapping('h40040000, 1); + preflight_ref_policy(model, 1, (root << 12) + ((64'h40040000 >> 30) & 511) * 8, 0, 0, 0); + submit(request(12, 'h40040000, 1, 16, 16, 1, 0, 0, 8, root)); + drain(); + preflight_ref_policy(model, 0, 0, 1, 'h800000, 'h80000f); + submit(request(13, 'h40040000, 1, 16, 16, 1, 0, 0, 8, root)); + drain(); + preflight_ref_policy(model, 0, 0, 1, 'h71000000, 'h71000003); + // Denied bytes precede the user span: the aligned transport range is still authorized. + submit(request(14, 'h71000004, 1, 4, 4)); + drain(); + preflight_ref_policy(model, 0, 0, 0, 0, 0); + submit(request(15, (64'h1 << `PF_ADDRESS_BITS) - 16, 1, 32, 32)); + drain(); + submit(request(16, 64'hfffffffffffffff8, 1, 16, 16)); + drain(); + submit(request(17, 'h80000000, 0, 16, 16)); + drain(); + q = request(18, 'h80000000, 1, 16, 16); + q[`PF(CMD, MODE)] = 9; + submit(q); + drain(); + root = mapping('h40050000, 1, 'h59); + submit(request(19, 'h40050000, 1, 16, 16, 1, 0, 0, 8, root, 1, 1, 1)); + drain(); + // External snapshot service captures policy at command acceptance, not from later live settings. + hold_permission = 1; + submit(request(20, 'h72000000, 1, 16, 16)); + preflight_ref_policy(model, 0, 0, 1, 'h72000000, 'h7200000f); + wait (auth_pending); + repeat (8) @(ctl.sample); + hold_permission = 0; + drain(); + preflight_ref_policy(model, 0, 0, 0, 0, 0); + submit(request(21, 'h73000000, 1, 16, 256, 16, 16)); + drain(); // columns retains all 32 input bits + q = request(22, 64'hfffffffffffff000, -1, -1, -1, -1, -1); + submit(q); + drain(); + // Write authorization covers actual bytes; read authorization covers transport padding. + prior = failures[4]; + preflight_ref_policy(model, 0, 0, 1, 'h74000000, 'h74000003); + submit(request(23, 'h74000004, 1, 4, 4, 1, 0, 1)); + drain(); + verify_contract(failures[4] == prior, "write was denied for untouched prefix padding"); + submit(request(24, 'h74000004, 1, 4, 4)); + drain(); + verify_contract(failures[4] == prior + 1, "read failed to authorize actual prefix overread"); + prior = failures[4]; + preflight_ref_policy(model, 0, 0, 1, 'h74000108, 'h7400013f); + submit(request(25, 'h74000104, 1, 4, 4, 1, 0, 1)); + drain(); + verify_contract(failures[4] == prior, "write was denied for untouched tail padding"); + submit(request(26, 'h74000104, 1, 4, 4)); + drain(); + verify_contract(failures[4] == prior + 1, "read failed to authorize actual tail overread"); + prior = failures[4]; + preflight_ref_policy(model, 0, 0, 1, 'h74002ff0, 'h74002ffb); + submit(request(27, 'h74002ffc, 1, 8, 8, 1, 0, 1)); + drain(); + verify_contract(failures[4] == prior, "cross-page write included untouched bytes"); + submit(request(28, 'h74002ffc, 1, 8, 8)); + drain(); + verify_contract(failures[4] == prior + 1, "cross-page read omitted padded transport bytes"); + preflight_ref_policy(model, 0, 0, 0, 0, 0); + // MobileNet's gather has nine aligned segments on two noncontiguous PA pages. + // Every same-page gap must be explicitly authorized before the plan retains a union. + root = mapping('h74003e00, 2); + prior = failures[5]; + submit(request(29, 'h74003e00, 1, 16, 4032, 8, 72, 0, 8, root)); + drain(); + verify_contract(failures[5] == prior, "Authorized MobileNet gather exceeded range capacity"); + // Splitting preserves the exact accessed bytes and keeps each list within eight records. + prior = failures[5]; + submit(request(30, 'h74003e00, 1, 16, 4032, 4, 72, 0, 8, root)); + drain(); + submit(request(31, 'h74003f20, 1, 16, 4032, 4, 72, 0, 8, root)); + drain(); + verify_contract(failures[5] == prior, "Split MobileNet gather exceeded range capacity"); + // A denied hole must keep both individually legal spans, rather than poison the command. + gap_stride = `PF_BEAT_BYTES == 16 ? 72 : 2 * `PF_BEAT_BYTES + 8; + preflight_ref_policy(model, 0, 0, 1, 'h800e00 + `PF_BEAT_BYTES, + 'h800e00 + (gap_stride / `PF_BEAT_BYTES) * `PF_BEAT_BYTES - 1); + prior = failures[4]; + submit(request(32, 'h74003e00, 1, 16, 4032, 2, gap_stride, 0, 8, root)); + drain(); + verify_contract(failures[4] == prior, "Denied union rejected individually authorized pixels"); + // Denial of an actual transport byte must still yield one terminal fault, no partial plan. + preflight_ref_policy(model, 0, 0, 1, 'h800e00 + gap_stride, 'h800e00 + gap_stride + 7); + submit(request(33, 'h74003e00, 1, 16, 4032, 2, gap_stride, 0, 8, root)); + drain(); + verify_contract(failures[4] == prior + 1, "Actual MobileNet pixel bytes escaped authorization"); + preflight_ref_policy(model, 0, 0, 0, 0, 0); + verify_contract( + peak==4&&submitted==retired&&command_stalls>0&&output_stalls>0&&auth_stalls>0, + "Missing concurrency/backpressure/drain coverage"); + for (int error = 1; error < 7; error++) + verify_contract(failures[error] > 0, "Missing typed error scenario"); + verify_contract(scoreboard.checked == segments, "Not every output descriptor was compared"); + `uvm_info( + "PREFLIGHT_PASS", + $sformatf( + "commands=%0d descriptors=%0d peak=%0d PTE=%0d auth=%0d errors shape/overflow/PF/AF/capacity/context=%0d/%0d/%0d/%0d/%0d/%0d stalls cmd/out/auth=%0d/%0d/%0d; all plans/permissions drained", + retired, segments, peak, pte_count, auth_count, failures[1], failures[2], failures[3], + failures[4], failures[5], failures[6], command_stalls, output_stalls, auth_stalls), + UVM_LOW) +endtask diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/tb.sv b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/tb.sv new file mode 100644 index 000000000..ba54c8e3b --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/tb.sv @@ -0,0 +1,52 @@ +`include "parameters.svh" +module preflight_tb; + import uvm_pkg::*; + import preflight_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + preflight_control_if ctl (clock); + stream_if #(`PF_CMD_WIDTH) cmd ( + clock, + ctl.reset + ); + stream_if #(`PF_OUT_WIDTH) prepared ( + clock, + ctl.reset + ); + stream_if #(`PF_AUTH_WIDTH) authorization ( + clock, + ctl.reset + ); + stream_if #(`PF_PERMIT_WIDTH) permission ( + clock, + ctl.reset + ); + stream_if #(`PF_PTE_WIDTH) pte_req ( + clock, + ctl.reset + ); + stream_if #(`PF_PTERESP_WIDTH) pte_resp ( + clock, + ctl.reset + ); + Preflight dut ( + .clock(clock), + .reset(ctl.reset), + `include "preflight_ports.svh" + ); + initial begin + uvm_config_db#(virtual preflight_control_if)::set(null, "uvm_test_top.env", "ctl", ctl); + uvm_config_db#(virtual stream_if #(`PF_CMD_WIDTH))::set(null, "uvm_test_top.env", "cmd", cmd); + uvm_config_db#(virtual stream_if #(`PF_OUT_WIDTH))::set(null, "uvm_test_top.env", "prepared", + prepared); + uvm_config_db#(virtual stream_if #(`PF_AUTH_WIDTH))::set(null, "uvm_test_top.env", + "authorization", authorization); + uvm_config_db#(virtual stream_if #(`PF_PERMIT_WIDTH))::set(null, "uvm_test_top.env", + "permission", permission); + uvm_config_db#(virtual stream_if #(`PF_PTE_WIDTH))::set(null, "uvm_test_top.env", "pte_req", + pte_req); + uvm_config_db#(virtual stream_if #(`PF_PTERESP_WIDTH))::set(null, "uvm_test_top.env", + "pte_resp", pte_resp); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/tests/test.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/tests/test.svh new file mode 100644 index 000000000..398043c00 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/tests/test.svh @@ -0,0 +1,14 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + test_env.execute(); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/agent/driver.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/agent/driver.svh new file mode 100644 index 000000000..d57501589 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/agent/driver.svh @@ -0,0 +1,25 @@ +class driver extends uvm_driver #(cycle_item); + `uvm_component_utils(driver) + virtual preparation_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual preparation_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "TSS interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item req; + forever begin + seq_item_port.get_next_item(req); + @(negedge vif.clock); + vif.reset = req.reset; + vif.source = req.inputs; + @(posedge vif.clock); + req.outputs = vif.outputs; + #1; + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/agent/fixture.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/agent/fixture.svh new file mode 100644 index 000000000..80199dac4 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/agent/fixture.svh @@ -0,0 +1,8 @@ +class fixture extends uvm_object; + `uvm_object_utils(fixture) + int policy[1<<`PREP_ID_BITS]; + bit pte_error; + function new(string name = "fixture"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/agent/item.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/agent/item.svh new file mode 100644 index 000000000..75524d2ad --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/agent/item.svh @@ -0,0 +1,18 @@ +class cycle_item extends uvm_sequence_item; + `uvm_object_utils(cycle_item) + logic [`PREP_INPUT_BITS-1:0] inputs; + logic [`PREP_OUTPUT_BITS-1:0] outputs; + bit reset; + function new(string name = "cycle_item"); + super.new(name); + endfunction +endclass +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(value, UVM_ALL_ON) + `uvm_object_utils_end + bit [511:0] value; + function new(string name = "result"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/agent/monitor.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/agent/monitor.svh new file mode 100644 index 000000000..8a27911bb --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/agent/monitor.svh @@ -0,0 +1,25 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual preparation_if vif; + uvm_analysis_port #(cycle_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual preparation_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "TSS monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item t; + forever begin + @(posedge vif.clock); + t = cycle_item::type_id::create("observed"); + t.inputs = vif.inputs; + t.outputs = vif.outputs; + t.reset = vif.reset; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/agent/peer.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/agent/peer.svh new file mode 100644 index 000000000..3408512a7 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/agent/peer.svh @@ -0,0 +1,81 @@ +class peer extends uvm_component; + `uvm_component_utils(peer) + virtual preparation_if vif; + fixture image; + bit pending; + int cycle, due; + bit error; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual preparation_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Preparation interface is required") + endfunction + task run_phase(uvm_phase phase); + forever begin + @(posedge vif.clock); + cycle++; + if (vif.reset) pending = 0; + else begin + if (`PO(PTERESPONSE_READY) && `PI(PTERESPONSE_VALID)) pending = 0; + if (`PO(PTEREQUEST_VALID) && `PI(PTEREQUEST_READY)) begin + pending = 1; + due = cycle + 3; + error = image.pte_error; + end + end + @(negedge vif.clock); + vif.backend = '0; + begin + int id = `PO(CONTEXTTAG); + int policy = image.policy[id]; + vif.backend[`PREP_IN_CONTEXTVALID_OFFSET+:`PREP_IN_CONTEXTVALID_WIDTH] = policy != 4; + vif.backend[`PREP_IN_CONTEXTID_OFFSET+:`PREP_IN_CONTEXTID_WIDTH] = id; + vif.backend[`PREP_IN_CONTEXTPMP_0_ADDR_OFFSET+:`PREP_IN_CONTEXTPMP_0_ADDR_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_0_MASK_OFFSET+:`PREP_IN_CONTEXTPMP_0_MASK_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_0_CFG_A_OFFSET+:`PREP_IN_CONTEXTPMP_0_CFG_A_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_0_CFG_L_OFFSET+:`PREP_IN_CONTEXTPMP_0_CFG_L_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_0_CFG_R_OFFSET+:`PREP_IN_CONTEXTPMP_0_CFG_R_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_0_CFG_W_OFFSET+:`PREP_IN_CONTEXTPMP_0_CFG_W_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_1_ADDR_OFFSET+:`PREP_IN_CONTEXTPMP_1_ADDR_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_1_MASK_OFFSET+:`PREP_IN_CONTEXTPMP_1_MASK_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_1_CFG_A_OFFSET+:`PREP_IN_CONTEXTPMP_1_CFG_A_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_1_CFG_L_OFFSET+:`PREP_IN_CONTEXTPMP_1_CFG_L_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_1_CFG_R_OFFSET+:`PREP_IN_CONTEXTPMP_1_CFG_R_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_1_CFG_W_OFFSET+:`PREP_IN_CONTEXTPMP_1_CFG_W_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_2_ADDR_OFFSET+:`PREP_IN_CONTEXTPMP_2_ADDR_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_2_MASK_OFFSET+:`PREP_IN_CONTEXTPMP_2_MASK_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_2_CFG_A_OFFSET+:`PREP_IN_CONTEXTPMP_2_CFG_A_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_2_CFG_L_OFFSET+:`PREP_IN_CONTEXTPMP_2_CFG_L_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_2_CFG_R_OFFSET+:`PREP_IN_CONTEXTPMP_2_CFG_R_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_2_CFG_W_OFFSET+:`PREP_IN_CONTEXTPMP_2_CFG_W_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_3_ADDR_OFFSET+:`PREP_IN_CONTEXTPMP_3_ADDR_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_3_MASK_OFFSET+:`PREP_IN_CONTEXTPMP_3_MASK_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_3_CFG_A_OFFSET+:`PREP_IN_CONTEXTPMP_3_CFG_A_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_3_CFG_L_OFFSET+:`PREP_IN_CONTEXTPMP_3_CFG_L_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_3_CFG_R_OFFSET+:`PREP_IN_CONTEXTPMP_3_CFG_R_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_3_CFG_W_OFFSET+:`PREP_IN_CONTEXTPMP_3_CFG_W_WIDTH] = 0; + if (policy == 1 || policy == 2 || policy == 6) begin + vif.backend[`PREP_IN_CONTEXTPMP_0_CFG_A_OFFSET+:`PREP_IN_CONTEXTPMP_0_CFG_A_WIDTH] = 3; + vif.backend[`PREP_IN_CONTEXTPMP_0_CFG_L_OFFSET+:`PREP_IN_CONTEXTPMP_0_CFG_L_WIDTH] = 1; + vif.backend[`PREP_IN_CONTEXTPMP_0_CFG_R_OFFSET+:`PREP_IN_CONTEXTPMP_0_CFG_R_WIDTH]=policy!=1; + vif.backend[`PREP_IN_CONTEXTPMP_0_CFG_W_OFFSET+:`PREP_IN_CONTEXTPMP_0_CFG_W_WIDTH]=policy==6; + vif.backend[`PREP_IN_CONTEXTPMP_0_ADDR_OFFSET+:`PREP_IN_CONTEXTPMP_0_ADDR_WIDTH] = '1; + vif.backend[`PREP_IN_CONTEXTPMP_0_MASK_OFFSET+:`PREP_IN_CONTEXTPMP_0_MASK_WIDTH] = '1; + end + if (policy == 3) begin + vif.backend[`PREP_IN_CONTEXTPMP_0_ADDR_OFFSET+:`PREP_IN_CONTEXTPMP_0_ADDR_WIDTH]='h2008>>2; + vif.backend[`PREP_IN_CONTEXTPMP_1_ADDR_OFFSET+:`PREP_IN_CONTEXTPMP_1_ADDR_WIDTH]='h2004>>2; + vif.backend[`PREP_IN_CONTEXTPMP_1_CFG_A_OFFSET+:`PREP_IN_CONTEXTPMP_1_CFG_A_WIDTH] = 1; + vif.backend[`PREP_IN_CONTEXTPMP_1_CFG_L_OFFSET+:`PREP_IN_CONTEXTPMP_1_CFG_L_WIDTH] = 1; + end + end + vif.backend[`PREP_IN_PTEREQUEST_READY_OFFSET+:`PREP_IN_PTEREQUEST_READY_WIDTH]=cycle%3!=0; + vif.backend[`PREP_IN_PTERESPONSE_VALID_OFFSET+:`PREP_IN_PTERESPONSE_VALID_WIDTH]=pending&&cycle>=due; + vif.backend[`PREP_IN_PTERESPONSE_BITS_DATA_OFFSET+:`PREP_IN_PTERESPONSE_BITS_DATA_WIDTH]=64'hcf; + vif.backend[`PREP_IN_PTERESPONSE_BITS_ERROR_OFFSET+:`PREP_IN_PTERESPONSE_BITS_ERROR_WIDTH]=error; + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/env/env.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/env/env.svh new file mode 100644 index 000000000..eeda0417e --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/env/env.svh @@ -0,0 +1,36 @@ +class env extends uvm_env; + `uvm_component_utils(env) + uvm_analysis_imp #(uvm_sequence_item, env) reset_events; + driver source; + monitor mon; + peer backend; + model reference; + uvm_sequencer #(cycle_item) seqr; + result_board #(result_item, 1) scoreboard; + function new(string name, uvm_component parent); + super.new(name, parent); + reset_events = new("reset_events", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + source = driver::type_id::create("source", this); + mon = monitor::type_id::create("monitor", this); + backend = peer::type_id::create("backend", this); + reference = model::type_id::create("model", this); + seqr = new("seqr", this); + scoreboard = result_board#(result_item, 1)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + fixture image = fixture::type_id::create("image"); + reference.image = image; + backend.image = image; + source.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(reference.analysis_export); + reference.expected.connect(scoreboard.expected_export); + reference.actual.connect(scoreboard.actual_export); + reference.cancelled.connect(reset_events); + endfunction + function void write(uvm_sequence_item t); + scoreboard.cancel_pending(); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/env/model.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/env/model.svh new file mode 100644 index 000000000..563aab945 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/env/model.svh @@ -0,0 +1,223 @@ +class model extends uvm_subscriber #(cycle_item); + `uvm_component_utils(model) + fixture image; + chandle reference; + uvm_analysis_port #(result_item) expected, actual; + uvm_analysis_port #(uvm_sequence_item) cancelled; + int index[int], errors[int], counts[int]; + longint unsigned addresses[int][int], physical[int][int]; + int lengths[int][int]; + bit direction[int], notified[int], live[int]; + longint query_key; + // A lookup answers in the third cycle of a held query; recent queries and expectations per port. + logic [255:0] recent[2][$]; + covergroup behavior with function sample ( + int error, bit writing, int mode, int policy, bit hit, bit released + ); + option.per_instance = 1; + outcomes: coverpoint error { + bins success = {0}; + bins shape = {1}; + bins page_fault = {3}; + bins access_fault = {4}; + bins capacity = {5}; + } + directions: coverpoint writing; + modes: coverpoint mode {bins all[] = {0, 8};} + pmp: coverpoint policy {bins all[] = {0, 1, 2, 3, 4, 6};} + queries: coverpoint hit; + release_events: coverpoint released; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected = new("expected", this); + actual = new("actual", this); + cancelled = new("cancelled", this); + reference = preflight_ref_create(`PREP_ADDRESS_BITS, 16); + behavior = new(); + endfunction + function void verify(bit ok, string message); + evidence::checks++; + if (!ok) `uvm_fatal("PREPARATION", message) + endfunction + function void predict(cycle_item t); + int id, policy, count; + bit deny; + int mode; + longint unsigned base; + id = `AI(t, COMMAND_BITS_ID); + policy = image.policy[id]; + mode = `AI(t, COMMAND_BITS_MODE); + base = `AI(t, COMMAND_BITS_BASEVA); + deny = policy == 1 || policy == 4 || (policy == 2 && `AI(t, COMMAND_BITS_WRITE)); + preflight_ref_program(reference, 'h10000, 'hcf, image.pte_error); + preflight_ref_policy(reference, deny && mode == 8, 'h10000, deny, 0, + (64'd1 << `PREP_ADDRESS_BITS) - 1); + verify(preflight_ref_submit( + reference, + id, + base, + `AI(t, COMMAND_BITS_ROWS), + `AI(t, COMMAND_BITS_COLUMNS), + `AI(t, COMMAND_BITS_SPANBYTES), + `AI(t, COMMAND_BITS_COLUMNSTRIDE), + `AI(t, COMMAND_BITS_ROWSTRIDE), + `AI(t, COMMAND_BITS_WRITE), + mode, + `AI(t, COMMAND_BITS_ROOTPPN), + `AI(t, COMMAND_BITS_PRIVILEGE), + `AI(t, COMMAND_BITS_SUM), + `AI(t, COMMAND_BITS_MXR) + ) == 0, "Missing preparation fixture"); + verify(!live.exists(id), "Duplicate live command ID"); + live[id] = 1; + direction[id] = `AI(t, COMMAND_BITS_WRITE); + index[id] = 0; + count = preflight_ref_count(reference, id); + counts[id] = count; + for (int i = 0; i < count; i++) begin + result_item result = result_item::type_id::create("range_expected"); + longint unsigned va, pa; + int bytes, writing, last, error; + preflight_ref_expected(reference, id, i, va, pa, bytes, writing, last, error); + result.value = {32'(id), va, pa, 32'(bytes), 32'(writing), 32'(last), 32'(error)}; + result.set_transaction_id((id << 4) | i); + expected.write(result); + addresses[id][i] = va; + physical[id][i] = pa; + lengths[id][i] = bytes; + if (last) begin + result = result_item::type_id::create("ready_expected"); + result.value = {32'(id), 32'(error), error == 0 ? 64'd0 : va}; + result.set_transaction_id((1 << 20) | id); + expected.write(result); + errors[id] = error; + behavior.sample(error, writing, mode, policy, 0, 0); + end + end + endfunction + function void query(cycle_item t, int port, int valid, int id, longint unsigned va, int bytes, + bit writing, bit hit, longint unsigned pa, int error); + bit expected_hit, shape; + longint unsigned expected_pa; + int expected_error; + logic [64:0] finish_address = {1'b0, va} + bytes; + bit releasing = `AI(t, RELEASE_VALID) && `AO(t, RELEASE_READY) && `AI(t, RELEASE_BITS_ID) == id; + shape = bytes > 0 && !finish_address[64] && int'(va & 4095) + bytes <= 4096; + expected_hit = 0; + expected_pa = 0; + if (valid && shape && live.exists( + id + ) && notified.exists( + id + ) && errors[id] == 0 && !releasing && direction[id] == writing) + for (int i = 0; i < counts[id]; i++) + if (va >= addresses[id][i] && finish_address <= addresses[id][i] + lengths[id][i]) begin + expected_hit = 1; + expected_pa = physical[id][i] + va - addresses[id][i]; + end + expected_error = !valid || expected_hit ? 0 : (!shape ? 1 : 4); + recent[port].push_back({ + 32'(valid), + 32'(id), + va, + 32'(bytes), + 1'(writing), + 1'(expected_hit), + expected_pa, + 3'(expected_error) + }); + if (recent[port].size() > 3) void'(recent[port].pop_front()); + if (recent[port].size() < 3 || recent[port][0] !== recent[port][1] || + recent[port][1] !== recent[port][2]) + return; + verify(hit == expected_hit && pa == expected_pa && error == expected_error, $sformatf( + "Query port%0d id%0d VA%h expected hit%0d PA%h error%0d got hit%0d PA%h error%0d", + port, + id, + va, + expected_hit, + expected_pa, + expected_error, + hit, + pa, + error + )); + if (valid) behavior.sample(expected_error, writing, 0, 0, hit, releasing); + endfunction + function void write(cycle_item t); + if (t.reset) begin + cancelled.write(null); + preflight_ref_reset(reference); + recent[0].delete(); + recent[1].delete(); + index.delete(); + errors.delete(); + counts.delete(); + addresses.delete(); + physical.delete(); + lengths.delete(); + direction.delete(); + notified.delete(); + live.delete(); + return; + end + query(t, 0, `AI(t, QUERIES_0_VALID), `AI(t, QUERIES_0_ID), `AI(t, QUERIES_0_VA), + `AI(t, QUERIES_0_BYTES), `AI(t, QUERIES_0_WRITE), `AO(t, RESULTS_0_HIT), + `AO(t, RESULTS_0_PA), `AO(t, RESULTS_0_ERROR)); + query(t, 1, `AI(t, QUERIES_1_VALID), `AI(t, QUERIES_1_ID), `AI(t, QUERIES_1_VA), + `AI(t, QUERIES_1_BYTES), `AI(t, QUERIES_1_WRITE), `AO(t, RESULTS_1_HIT), + `AO(t, RESULTS_1_PA), `AO(t, RESULTS_1_ERROR)); + if (`AI(t, COMMAND_VALID) && `AO(t, COMMAND_READY)) predict(t); + if (`AO(t, PTEREQUEST_VALID) && `AI(t, PTEREQUEST_READY)) begin + longint unsigned value; + int error, id; + id = `AO(t, CONTEXTTAG); + verify(preflight_ref_pte(reference, id, `AO(t, PTEREQUEST_BITS_ADDR), value, error) == 1, + "PTE request differs from preparation plan"); + end + if (`AO(t, RANGES_VALID) && `AI(t, RANGES_READY)) begin + int id = `AO(t, RANGES_BITS_ID); + result_item result = result_item::type_id::create("range_actual"); + verify(live.exists(id), "Prepared range has no live command"); + result.value = { + 32'(id), + 64'(`AO(t, RANGES_BITS_VA)), + 64'(`AO(t, RANGES_BITS_PA)), + 32'(`AO(t, RANGES_BITS_BYTES)), + 32'(`AO(t, RANGES_BITS_WRITE)), + 32'(`AO(t, RANGES_BITS_LAST)), + 32'(`AO(t, RANGES_BITS_ERROR)) + }; + result.set_transaction_id((id << 4) | index[id]); + actual.write(result); + index[id]++; + end + if (`AO(t, READY_VALID) && `AI(t, READY_READY)) begin + int id = `AO(t, READY_BITS_ID); + result_item result = result_item::type_id::create("ready_actual"); + verify(live.exists(id) && index[id] == counts[id], + "Notification before all ranges were retained"); + result.value = {32'(id), 32'(`AO(t, READY_BITS_ERROR)), 64'(`AO(t, READY_BITS_VA))}; + result.set_transaction_id((1 << 20) | id); + actual.write(result); + notified[id] = 1; + end + if (`AI(t, RELEASE_VALID) && `AO(t, RELEASE_READY)) begin + int id = `AI(t, RELEASE_BITS_ID); + verify(notified.exists(id), "Released a mapping before notification"); + preflight_ref_retire(reference, id); + live.delete(id); + notified.delete(id); + behavior.sample(errors[id], direction[id], 0, 0, 0, 1); + end + endfunction + function void check_phase(uvm_phase phase); + super.check_phase(phase); + verify(live.num() == 0, "Mappings did not release"); + endfunction + function void final_phase(uvm_phase phase); + preflight_ref_destroy(reference); + super.final_phase(phase); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/filelist.f b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/filelist.f new file mode 100644 index 000000000..3cda6a201 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/Preparation/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/interface.sv b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/interface.sv new file mode 100644 index 000000000..92bb989a4 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/interface.sv @@ -0,0 +1,9 @@ +`include "preparation_parameters.svh" +interface preparation_if ( + input logic clock +); + logic reset = 1; + logic [`PREP_INPUT_BITS-1:0] source = '0, backend = '0; + wire [ `PREP_INPUT_BITS-1:0] inputs = source | backend; + wire [`PREP_OUTPUT_BITS-1:0] outputs; +endinterface diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/package.sv b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/package.sv new file mode 100644 index 000000000..348bcb50b --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/package.sv @@ -0,0 +1,97 @@ +`include "preparation_parameters.svh" +`define PI(F) vif.inputs[`PREP_IN_``F``_OFFSET+:`PREP_IN_``F``_WIDTH] +`define PO(F) vif.outputs[`PREP_OUT_``F``_OFFSET+:`PREP_OUT_``F``_WIDTH] +`define PB(F) vif.backend[`PREP_IN_``F``_OFFSET+:`PREP_IN_``F``_WIDTH] +`define PS(F) pins[`PREP_IN_``F``_OFFSET+:`PREP_IN_``F``_WIDTH] +`define AI(T, F) T.inputs[`PREP_IN_``F``_OFFSET+:`PREP_IN_``F``_WIDTH] +`define AO(T, F) T.outputs[`PREP_OUT_``F``_OFFSET+:`PREP_OUT_``F``_WIDTH] +package preparation_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + import "DPI-C" function chandle preflight_ref_create( + input int unsigned address_bits, + beat + ); + import "DPI-C" function void preflight_ref_destroy(input chandle model); + import "DPI-C" function void preflight_ref_program( + input chandle model, + input longint unsigned address, + value, + input int unsigned error + ); + import "DPI-C" function void preflight_ref_policy( + input chandle model, + input int unsigned pte_valid, + input longint unsigned pte, + input int unsigned range_valid, + input longint unsigned first, + last + ); + import "DPI-C" function int unsigned preflight_ref_submit( + input chandle model, + input int unsigned id, + input longint unsigned base, + input int unsigned rows, + cols, + span, + col_stride, + row_stride, + write, + mode, + input longint unsigned root, + input int unsigned privilege, + sum, + mxr + ); + import "DPI-C" function int unsigned preflight_ref_count( + input chandle model, + input int unsigned id + ); + import "DPI-C" function void preflight_ref_expected( + input chandle model, + input int unsigned id, + index, + output longint unsigned va, + pa, + output int unsigned bytes, + write, + last, + error + ); + import "DPI-C" function int unsigned preflight_ref_authorize( + input chandle model, + input int unsigned id, + input longint unsigned pa, + input int unsigned bytes, + write, + pte, + privilege, + output int unsigned allow + ); + import "DPI-C" function int unsigned preflight_ref_pte( + input chandle model, + input int unsigned id, + input longint unsigned address, + output longint unsigned value, + output int unsigned error + ); + import "DPI-C" function int unsigned preflight_ref_pending( + input chandle model, + input int unsigned id + ); + import "DPI-C" function void preflight_ref_retire( + input chandle model, + input int unsigned id + ); + import "DPI-C" function void preflight_ref_reset(input chandle model); + `include "agent/item.svh" + `include "agent/fixture.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/peer.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/seq/sequence.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/seq/sequence.svh new file mode 100644 index 000000000..18dd3141d --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/seq/sequence.svh @@ -0,0 +1,113 @@ +class scenario_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(scenario_sequence) + model reference; + logic [`PREP_INPUT_BITS-1:0] pins; + logic [`PREP_OUTPUT_BITS-1:0] observed; + function new(string name = "scenario"); + super.new(name); + endfunction + task tick(bit reset = 0); + cycle_item t = cycle_item::type_id::create("cycle"); + start_item(t); + t.inputs = pins; + t.reset = reset; + finish_item(t); + observed = t.outputs; + endtask + task submit(int id, longint unsigned base, int rows, int columns, int span, int column_stride, + int row_stride, bit writing = 0, int policy = 0, int mode = 0, int privilege = 3); + reference.image.policy[id] = policy; + pins[`PREP_IN_COMMAND_BITS_ID_OFFSET+:`PREP_IN_COMMAND_BITS_ID_WIDTH] = id; + pins[`PREP_IN_COMMAND_BITS_BASEVA_OFFSET+:`PREP_IN_COMMAND_BITS_BASEVA_WIDTH] = base; + pins[`PREP_IN_COMMAND_BITS_ROWS_OFFSET+:`PREP_IN_COMMAND_BITS_ROWS_WIDTH] = rows; + pins[`PREP_IN_COMMAND_BITS_COLUMNS_OFFSET+:`PREP_IN_COMMAND_BITS_COLUMNS_WIDTH] = columns; + pins[`PREP_IN_COMMAND_BITS_SPANBYTES_OFFSET+:`PREP_IN_COMMAND_BITS_SPANBYTES_WIDTH] = span; + pins[`PREP_IN_COMMAND_BITS_COLUMNSTRIDE_OFFSET+:`PREP_IN_COMMAND_BITS_COLUMNSTRIDE_WIDTH]=column_stride; + pins[`PREP_IN_COMMAND_BITS_ROWSTRIDE_OFFSET+:`PREP_IN_COMMAND_BITS_ROWSTRIDE_WIDTH]=row_stride; + pins[`PREP_IN_COMMAND_BITS_WRITE_OFFSET+:`PREP_IN_COMMAND_BITS_WRITE_WIDTH] = writing; + pins[`PREP_IN_COMMAND_BITS_MODE_OFFSET+:`PREP_IN_COMMAND_BITS_MODE_WIDTH] = mode; + pins[`PREP_IN_COMMAND_BITS_PRIVILEGE_OFFSET+:`PREP_IN_COMMAND_BITS_PRIVILEGE_WIDTH] = privilege; + pins[`PREP_IN_COMMAND_BITS_ROOTPPN_OFFSET+:`PREP_IN_COMMAND_BITS_ROOTPPN_WIDTH] = 16; + pins[`PREP_IN_COMMAND_VALID_OFFSET+:`PREP_IN_COMMAND_VALID_WIDTH] = 1; + do tick(); while (!observed[`PREP_OUT_COMMAND_READY_OFFSET]); + pins[`PREP_IN_COMMAND_VALID_OFFSET+:`PREP_IN_COMMAND_VALID_WIDTH] = 0; + endtask + task notified(int id); + do tick(); while (!reference.notified.exists(id)); + endtask + task release_tag(int id); + pins[`PREP_IN_RELEASE_BITS_ID_OFFSET+:`PREP_IN_RELEASE_BITS_ID_WIDTH] = id; + pins[`PREP_IN_RELEASE_VALID_OFFSET+:`PREP_IN_RELEASE_VALID_WIDTH] = 1; + do tick(); while (!observed[`PREP_OUT_RELEASE_READY_OFFSET]); + pins[`PREP_IN_RELEASE_VALID_OFFSET+:`PREP_IN_RELEASE_VALID_WIDTH] = 0; + tick(); + endtask + task probe(int id, longint unsigned address, int bytes, bit writing = 0); + pins[`PREP_IN_QUERIES_0_VALID_OFFSET+:`PREP_IN_QUERIES_0_VALID_WIDTH] = 1; + pins[`PREP_IN_QUERIES_0_ID_OFFSET+:`PREP_IN_QUERIES_0_ID_WIDTH] = id; + pins[`PREP_IN_QUERIES_0_VA_OFFSET+:`PREP_IN_QUERIES_0_VA_WIDTH] = address; + pins[`PREP_IN_QUERIES_0_BYTES_OFFSET+:`PREP_IN_QUERIES_0_BYTES_WIDTH] = bytes; + pins[`PREP_IN_QUERIES_0_WRITE_OFFSET+:`PREP_IN_QUERIES_0_WRITE_WIDTH] = writing ^ 0; + pins[`PREP_IN_QUERIES_1_VALID_OFFSET+:`PREP_IN_QUERIES_1_VALID_WIDTH] = 1; + pins[`PREP_IN_QUERIES_1_ID_OFFSET+:`PREP_IN_QUERIES_1_ID_WIDTH] = id; + pins[`PREP_IN_QUERIES_1_VA_OFFSET+:`PREP_IN_QUERIES_1_VA_WIDTH] = address; + pins[`PREP_IN_QUERIES_1_BYTES_OFFSET+:`PREP_IN_QUERIES_1_BYTES_WIDTH] = bytes; + pins[`PREP_IN_QUERIES_1_WRITE_OFFSET+:`PREP_IN_QUERIES_1_WRITE_WIDTH] = writing ^ 1; + repeat (3) tick(); + endtask + task check_one(int id, longint unsigned base, int rows, int columns, int span, int column_stride, + int row_stride, bit writing = 0, int policy = 0, int mode = 0, int privilege = 3); + submit(id, base, rows, columns, span, column_stride, row_stride, writing, policy, mode, + privilege); + notified(id); + probe(id, base, 1, writing); + probe(id, base, 0, writing); + probe(id, base + 8192, 16, writing); + probe(id, base, 16, writing); + release_tag(id); + probe(id, base, 1, writing); + endtask + task body(); + pins = '0; + pins[`PREP_IN_RANGES_READY_OFFSET+:`PREP_IN_RANGES_READY_WIDTH] = 1; + pins[`PREP_IN_READY_READY_OFFSET+:`PREP_IN_READY_READY_WIDTH] = 1; + repeat (3) tick(1); + tick(); + check_one(0, 'h2000, 1, 1, 16, 16, 16); + check_one(1, 'h2ff7, 2, 1, 16, 16, 32); + check_one(2, 'h2000, 2, 3, 8, 24, 128, 1); + check_one(3, 'h4000, 9, 1, 16, 16, 4096, 1); + check_one(4, 'h4000, 0, 1, 16, 16, 16); + check_one(5, 'h2000, 1, 1, 16, 16, 16, 1, 1); + check_one(6, 'h2000, 1, 1, 16, 16, 16, 0, 2); + check_one(7, 'h2000, 1, 1, 16, 16, 16, 1, 2); + check_one(8, 'h2000, 1, 1, 16, 16, 16, 1, 3); + check_one(9, 'h2000, 1, 1, 16, 16, 16, 0, 4); + check_one(10, 'h5000, 2, 1, 16, 16, 32, 0, 6, 8, 1); + reference.image.pte_error = 1; + check_one(11, 'h5000, 1, 1, 16, 16, 16, 0, 6, 8, 1); + reference.image.pte_error = 0; + // An invalid root entry produces a page fault independently of permission failures. + check_one(12, 64'h4000000000, 1, 1, 16, 16, 16, 0, 6, 8, 1); + pins[`PREP_IN_READY_READY_OFFSET+:`PREP_IN_READY_READY_WIDTH] = 0; + pins[`PREP_IN_RANGES_READY_OFFSET+:`PREP_IN_RANGES_READY_WIDTH] = 0; + for (int id = 20; id < 24; id++) submit(id, 'h6000 + (id - 20) * 4096, 1, 1, 16, 16, 16); + repeat (8) tick(); + pins[`PREP_IN_RANGES_READY_OFFSET+:`PREP_IN_RANGES_READY_WIDTH] = 1; + repeat (40) tick(); + pins[`PREP_IN_READY_READY_OFFSET+:`PREP_IN_READY_READY_WIDTH] = 1; + for (int id = 20; id < 24; id++) begin + notified(id); + probe(id, 'h6000 + (id - 20) * 4096, 16); + end + for (int id = 20; id < 24; id++) release_tag(id); + pins[`PREP_IN_RANGES_READY_OFFSET+:`PREP_IN_RANGES_READY_WIDTH] = 0; + submit(30, 'h8000, 1, 1, 16, 16, 16); + repeat (10) tick(); + repeat (2) tick(1); + pins[`PREP_IN_RANGES_READY_OFFSET+:`PREP_IN_RANGES_READY_WIDTH] = 1; + tick(); + check_one(30, 'h8000, 1, 1, 16, 16, 16); + repeat (8) tick(); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/tb.sv b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/tb.sv new file mode 100644 index 000000000..64475ed3b --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/tb.sv @@ -0,0 +1,15 @@ +module preparation_tb; + import uvm_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + preparation_if bus (clock); + Preparation dut ( + .clock(clock), + .reset(bus.reset), + `include "preparation_ports.svh" + ); + initial begin + uvm_config_db#(virtual preparation_if)::set(null, "uvm_test_top.env.*", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/tests/test.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/tests/test.svh new file mode 100644 index 000000000..b11c898c9 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/tests/test.svh @@ -0,0 +1,16 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.reference = test_env.reference; + seq.start(test_env.seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/agent/service.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/agent/service.svh new file mode 100644 index 000000000..30309a5b2 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/agent/service.svh @@ -0,0 +1,49 @@ +task monitor(); + forever begin + @(ctl.sample); + if (ctl.sample.reset) begin + pmap_ref_reset(model); + recent[0].delete(); + recent[1].delete(); + notified.delete(); + status.delete(); + end else begin + query_check(0, ctl.sample.query_0, ctl.sample.result_0); + query_check(1, ctl.sample.query_1, ctl.sample.result_1); + if (ctl.sample.query_0[`PF(QUERY, VALID)] && ctl.sample.query_1[`PF(QUERY, VALID)]) dual++; + if (mapped.sample.valid) begin + int error, id; + u64 va; + id = mapped.sample.bits[`PF(MAP_READY, ID)]; + verify_contract(pmap_ref_ready(model, id, mapped.sample.ready, error, va) == 1, + "unexpected notification"); + verify_contract(mapped.sample.bits[`PF(MAP_READY, ERROR) + ] === 3'(error) && mapped.sample.bits[`PF(MAP_READY, VA)] === va, + "notification mismatch"); + behavior.sample(0, 0, 1, error); + if (mapped.sample.ready) begin + notified[id] = 1; + status[id] = error; + notifications++; + end else stalls++; + end + if (retire.sample.valid && retire.sample.ready) begin + int id; + id = retire.sample.bits[`PF(RELEASE, ID)]; + pmap_ref_release(model, id); + notified.delete(id); + retired++; + end + if (reserve.sample.valid && reserve.sample.ready) begin + pmap_ref_reserve(model, reserve.sample.bits[`PF(RESERVE, ID)]); + allocations++; + end + if (prepared.sample.valid && prepared.sample.ready) + pmap_ref_prepared(model, prepared.sample.bits[`PF(OUT, ID)], prepared.sample.bits[ + `PF(OUT, VA)], prepared.sample.bits[`PF(OUT, PA)], prepared.sample.bits[ + `PF(OUT, BYTES)], prepared.sample.bits[`PF(OUT, WRITE)], + prepared.sample.bits[`PF(OUT, LAST)], prepared.sample.bits[`PF(OUT, ERROR) + ]); + end + end +endtask diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/env/env.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/env/env.svh new file mode 100644 index 000000000..e5c713cc9 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/env/env.svh @@ -0,0 +1,101 @@ +class env extends uvm_env; + `uvm_component_utils(env) + virtual prepared_map_control_if ctl; + virtual stream_if #(`PF_RESERVE_WIDTH) reserve; + virtual stream_if #(`PF_OUT_WIDTH) prepared; + virtual stream_if #(`PF_MAP_READY_WIDTH) mapped; + virtual stream_if #(`PF_RELEASE_WIDTH) retire; + chandle model; + // A lookup answers in the third cycle of a held query; recent queries and expectations per port. + logic [`PF_QUERY_WIDTH+`PF_RESULT_WIDTH-1:0] recent[2][$]; + bit notified[int]; + int status[int]; + int + queries = 0, + hits = 0, + misses = 0, + allocations = 0, + notifications = 0, + retired = 0, + stalls = 0, + dual = 0, + cancelled = 0; + covergroup behavior with function sample (bit hit, bit writing, bit valid, int error); + option.per_instance = 1; + hits: coverpoint hit; + direction: coverpoint writing; + enabled: coverpoint valid; + outcomes: coverpoint error {bins all[] = {[0 : 6]};} + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + behavior = new(); + + endfunction + function void verify_contract(bit ok, string message); + verification_pkg::evidence::checks++; + if (!ok) `uvm_fatal("PREPARED_MAP", message) + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + verify_contract(uvm_config_db#(virtual prepared_map_control_if)::get(this, "", "ctl", ctl), + "ctl missing"); + verify_contract(uvm_config_db#(virtual stream_if #(`PF_RESERVE_WIDTH))::get( + this, "", "reserve", reserve), "reserve missing"); + verify_contract(uvm_config_db#(virtual stream_if #(`PF_OUT_WIDTH))::get( + this, "", "prepared", prepared), "prepared missing"); + verify_contract(uvm_config_db#(virtual stream_if #(`PF_MAP_READY_WIDTH))::get( + this, "", "mapped", mapped), "mapped missing"); + verify_contract(uvm_config_db#(virtual stream_if #(`PF_RELEASE_WIDTH))::get( + this, "", "retire", retire), "retire missing"); + model = pmap_ref_create(`PF_ADDRESS_BITS); + endfunction + function void query_check(int port, logic [`PF_QUERY_WIDTH-1:0] q, + logic [`PF_RESULT_WIDTH-1:0] actual); + int hit, error; + u64 pa; + logic [`PF_RESULT_WIDTH-1:0] expected = '0; + pmap_ref_query(model, q[`PF(QUERY, VALID)], q[`PF(QUERY, ID)], q[`PF(QUERY, VA)], q[ + `PF(QUERY, BYTES)], q[`PF(QUERY, WRITE)], + retire.sample.valid && retire.sample.ready, retire.sample.bits[`PF(RELEASE, ID)], + hit, pa, error); + expected[`PF(RESULT, HIT)] = 1'(hit); + expected[`PF(RESULT, PA)] = `PF_ADDRESS_BITS'(pa); + expected[`PF(RESULT, ERROR)] = 3'(error); + recent[port].push_back({q, expected}); + if (recent[port].size() > 3) void'(recent[port].pop_front()); + if (recent[port].size() < 3 || recent[port][0] !== recent[port][1] || + recent[port][1] !== recent[port][2]) + return; + verify_contract(actual[`PF(RESULT, HIT)] === 1'(hit) && actual[`PF(RESULT, PA) + ] === `PF_ADDRESS_BITS'(pa) && actual[`PF(RESULT, ERROR)] === 3'(error), + $sformatf( + "query id%0d VA%h bytes%0d expected hit%0d PA%h error%0d got %h", + q[ + `PF(QUERY, ID) + ], + q[ + `PF(QUERY, VA) + ], + q[ + `PF(QUERY, BYTES) + ], + hit, + pa, + error, + actual + )); + behavior.sample(hit, q[`PF(QUERY, WRITE)], q[`PF(QUERY, VALID)], error); + if (q[`PF(QUERY, VALID)]) begin + queries++; + if (hit) hits++; + else misses++; + end + endfunction + function void final_phase(uvm_phase phase); + pmap_ref_destroy(model); + super.final_phase(phase); + endfunction + `include "agent/service.svh" + `include "seq/scenario.svh" +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/filelist.f b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/filelist.f new file mode 100644 index 000000000..ee3c1a197 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/filelist.f @@ -0,0 +1,9 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/core/stream_if.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/PreparedMap/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/interface.sv b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/interface.sv new file mode 100644 index 000000000..7b84c0369 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/interface.sv @@ -0,0 +1,12 @@ +`include "parameters.svh" +interface prepared_map_control_if ( + input logic clock +); + logic reset, hold_ready; + logic [`PF_QUERY_WIDTH-1:0] query_0, query_1; + logic [`PF_RESULT_WIDTH-1:0] result_0, result_1; + clocking sample @(posedge clock); + default input #1step; + input reset, query_0, query_1, result_0, result_1; + endclocking +endinterface diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/package.sv b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/package.sv new file mode 100644 index 000000000..114541285 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/package.sv @@ -0,0 +1,52 @@ +`include "parameters.svh" +`define PF(K, F) `PF_``K``_``F``_OFFSET +: `PF_``K``_``F``_WIDTH +package prepared_map_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + typedef longint unsigned u64; + import "DPI-C" function chandle pmap_ref_create(input int unsigned address_bits); + import "DPI-C" function void pmap_ref_destroy(input chandle model); + import "DPI-C" function void pmap_ref_reset(input chandle model); + import "DPI-C" function void pmap_ref_reserve( + input chandle model, + input int unsigned id + ); + import "DPI-C" function void pmap_ref_prepared( + input chandle model, + input int unsigned id, + input u64 va, + pa, + input int unsigned bytes, + write, + last, + error + ); + import "DPI-C" function int unsigned pmap_ref_ready( + input chandle model, + input int unsigned id, + fire, + output int unsigned error, + output u64 va + ); + import "DPI-C" function void pmap_ref_query( + input chandle model, + input int unsigned valid, + id, + input u64 va, + input int unsigned bytes, + write, + retiring, + retiring_id, + output int unsigned hit, + output u64 pa, + output int unsigned error + ); + import "DPI-C" function void pmap_ref_release( + input chandle model, + input int unsigned id + ); + import "DPI-C" function int unsigned pmap_ref_pending(input chandle model); + `include "env/env.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/seq/scenario.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/seq/scenario.svh new file mode 100644 index 000000000..1a07f81fd --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/seq/scenario.svh @@ -0,0 +1,168 @@ +task reserve_tag(int id); + @(negedge ctl.clock); + reserve.bits = '0; + reserve.bits[`PF(RESERVE, ID)] = id; + reserve.valid = 1; + do @(ctl.sample); while (!reserve.sample.ready); + @(negedge ctl.clock); + reserve.valid = 0; +endtask +task record(int id, u64 va, u64 pa, int bytes, bit write, bit last, int error = 0); + @(negedge ctl.clock); + prepared.bits = '0; + prepared.bits[`PF(OUT, ID)] = id; + prepared.bits[`PF(OUT, VA)] = va; + prepared.bits[`PF(OUT, PA)] = pa; + prepared.bits[`PF(OUT, BYTES)] = bytes; + prepared.bits[`PF(OUT, WRITE)] = write; + prepared.bits[`PF(OUT, LAST)] = last; + prepared.bits[`PF(OUT, ERROR)] = error; + prepared.valid = 1; + do @(ctl.sample); while (!prepared.sample.ready); + @(negedge ctl.clock); + prepared.valid = 0; +endtask +function logic [`PF_QUERY_WIDTH-1:0] query(int id, u64 va, int bytes, bit write); + logic [`PF_QUERY_WIDTH-1:0] q = '0; + q[`PF(QUERY, VALID)] = 1; + q[`PF(QUERY, ID)] = id; + q[`PF(QUERY, VA)] = va; + q[`PF(QUERY, BYTES)] = bytes; + q[`PF(QUERY, WRITE)] = write; + return q; +endfunction +task probes(logic [`PF_QUERY_WIDTH-1:0] a, logic [`PF_QUERY_WIDTH-1:0] b, int clocks = 3); + @(negedge ctl.clock); + ctl.query_0 = a; + ctl.query_1 = b; + repeat (clocks) @(ctl.sample); + @(negedge ctl.clock); + ctl.query_0 = '0; + ctl.query_1 = '0; +endtask +task wait_ready(int id, int error = 0); + while (!notified.exists(id)) @(negedge ctl.clock); + verify_contract(status[id] == error, "wrong terminal status"); +endtask +task retire_tag(int id, bit probe_release = 0); + @(negedge ctl.clock); + retire.bits = '0; + retire.bits[`PF(RELEASE, ID)] = id; + retire.valid = 1; + if (probe_release) begin + ctl.query_0 = query(id, 'h1010, 8, 0); + ctl.query_1 = query(2, 'h4004, 8, 1); + end + do @(ctl.sample); while (!retire.sample.ready); + @(negedge ctl.clock); + retire.valid = 0; + ctl.query_0 = '0; + ctl.query_1 = '0; +endtask +task execute(); + ctl.reset = 1; + ctl.hold_ready = 0; + ctl.query_0 = '0; + ctl.query_1 = '0; + reserve.valid = 0; + reserve.bits = '0; + prepared.valid = 0; + prepared.bits = '0; + retire.valid = 0; + retire.bits = '0; + fork + monitor(); + join_none + repeat (3) @(negedge ctl.clock); + ctl.reset = 0; + reserve_tag(1); + record(1, 'h1000, 'h8000, 4096, 0, 0); + probes(query(1, 'h1010, 16, 0), query(1, 'h1010, 16, 1)); + @(negedge ctl.clock); + ctl.hold_ready = 1; + record(1, 'h2000, 'hc000, 4096, 0, 1); + probes(query(1, 'h1010, 16, 0), query(1, 'h2010, 16, 0), 8); + @(negedge ctl.clock); + ctl.hold_ready = 0; + wait_ready(1); + probes(query(1, 'h1ffe, 2, 0), query(1, 'h2ffe, 2, 0)); + probes(query(1, 'h1fff, 2, 0), query(1, 'h3000, 4, 0)); + probes(query(1, 'h1000, 16, 1), query(1, 64'hffffffffffffffff, 2, 0)); + probes(query(1, 'h1000, 0, 0), query(99, 'h1000, 8, 0)); + reserve_tag(2); + record(2, 'h4000, 'h9000, 64, 1, 0); + record(2, 64'hffffffffffffffe0, (64'd1 << `PF_ADDRESS_BITS) - 32, 32, 1, 1); + wait_ready(2); + probes(query(1, 'h1f00, 16, 0), query(2, 'h4004, 16, 1)); + probes(query(2, 64'hfffffffffffffff8, 8, 1), query(2, 64'hfffffffffffffff8, 9, 1)); + retire_tag(1, 1); + probes(query(1, 'h1010, 16, 0), query(2, 'h4004, 16, 1)); + reserve_tag(1); + record(1, 'h6000, 'ha000, 128, 0, 1); + wait_ready(1); + probes(query(1, 'h1010, 16, 0), query(1, 'h6040, 64, 0)); + reserve_tag(3); + record(3, 'h7000, 'hb000, 64, 0, 0); + record(3, 'h8000, 0, 0, 0, 1, 3); + wait_ready(3, 3); + probes(query(3, 'h7004, 8, 0), query(3, 'h8000, 8, 0)); + reserve_tag(4); + fork + reserve_tag(5); + begin + repeat (6) begin + @(negedge ctl.clock); + verify_contract(!reserve.ready, "full table accepted fifth reserve"); + end + record(4, 'h9000, 0, 0, 0, 1, 4); + wait_ready(4, 4); + retire_tag(4); + end + join + for (int i = 0; i < 8; i++) begin + record(5, 'h10000 + i * 4096, 'h30000 + (7 - i) * 4096, 32, 0, i == 7); + end + wait_ready(5); + probes(query(5, 'h17018, 8, 0), query(5, 'h10008, 8, 0)); + retire_tag(1); + retire_tag(2); + retire_tag(3); + retire_tag(5); + for (int code = 1; code <= 6; code++) begin + reserve_tag(20 + code); + record(20 + code, 'h9000 + code * 16, 0, 0, code % 2, 1, code); + wait_ready(20 + code, code); + retire_tag(20 + code); + end + // Retain an explicitly authorized same-page union and a discontiguous next PA page. + reserve_tag(10); + record(10, 'h74003e00, 'h800e00, 512, 0, 0); + record(10, 'h74004000, 'h802000, 16, 0, 1); + wait_ready(10); + probes(query(10, 'h74003e10, 16, 0), query(10, 'h74004000, 16, 0)); + probes(query(10, 'h74003df0, 16, 0), query(10, 'h74004010, 8, 0)); + probes(query(10, 'h74003ff8, 16, 0), query(10, 'h74003e48, 16, 1)); + retire_tag(10); + reserve_tag(9); + record(9, 'h9000, 'h19000, 64, 0, 1); + // Reset cancels a terminal notification still under backpressure. + @(negedge ctl.clock); + ctl.hold_ready = 1; + repeat (2) @(negedge ctl.clock); + cancelled += pmap_ref_pending(model); + ctl.reset = 1; + repeat (3) @(negedge ctl.clock); + ctl.reset = 0; + ctl.hold_ready = 0; + probes(query(9, 'h9000, 8, 0), query(1, 'h6000, 8, 0)); + repeat (3) @(negedge ctl.clock); + verify_contract(pmap_ref_pending(model) == 0, "mapping remained after retirement/reset"); + verify_contract(hits > 0 && misses > 0 && dual > 0 && stalls >= 4 && cancelled == 1, + "required checks did not execute"); + `uvm_info( + "PREPARED_MAP", + $sformatf( + "Checked %0d queries, %0d hits, %0d misses; reserve=%0d ready=%0d retire=%0d stalls=%0d dual=%0d reset_cancel=%0d", + queries, hits, misses, allocations, notifications, retired, stalls, dual, cancelled), + UVM_LOW) +endtask diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/tb.sv b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/tb.sv new file mode 100644 index 000000000..d0725b78c --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/tb.sv @@ -0,0 +1,42 @@ +`include "parameters.svh" +module prepared_map_tb; + import uvm_pkg::*; + import prepared_map_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + prepared_map_control_if ctl (clock); + stream_if #(`PF_RESERVE_WIDTH) reserve ( + clock, + ctl.reset + ); + stream_if #(`PF_OUT_WIDTH) prepared ( + clock, + ctl.reset + ); + stream_if #(`PF_MAP_READY_WIDTH) mapped ( + clock, + ctl.reset + ); + stream_if #(`PF_RELEASE_WIDTH) retire ( + clock, + ctl.reset + ); + assign mapped.ready = !ctl.reset && !ctl.hold_ready; + PreparedMap dut ( + .clock(clock), + .reset(ctl.reset), + `include "prepared_map_ports.svh" + ); + initial begin + uvm_config_db#(virtual prepared_map_control_if)::set(null, "uvm_test_top.env", "ctl", ctl); + uvm_config_db#(virtual stream_if #(`PF_RESERVE_WIDTH))::set(null, "uvm_test_top.env", "reserve", + reserve); + uvm_config_db#(virtual stream_if #(`PF_OUT_WIDTH))::set(null, "uvm_test_top.env", "prepared", + prepared); + uvm_config_db#(virtual stream_if #(`PF_MAP_READY_WIDTH))::set(null, "uvm_test_top.env", + "mapped", mapped); + uvm_config_db#(virtual stream_if #(`PF_RELEASE_WIDTH))::set(null, "uvm_test_top.env", "retire", + retire); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/tests/test.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/tests/test.svh new file mode 100644 index 000000000..398043c00 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/tests/test.svh @@ -0,0 +1,14 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + test_env.execute(); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/suite.toml b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/suite.toml new file mode 100644 index 000000000..ec50d41b1 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/suite.toml @@ -0,0 +1,54 @@ +[model] +manifest="src/csrc/Cargo.toml" +library="libpreflight_reference" + +[[tests]] +functional_groups = ["preflight_pkg::env::behavior"] +name = "preflight" +top = "preflight_tb" +test = "protocol_test" +filelist = "preflight/filelist.f" +timeout_ns = 1000000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +functional_groups = ["prepared_map_pkg::env::behavior"] +name = "prepared_map" +top = "prepared_map_tb" +test = "protocol_test" +filelist = "prepared_map/filelist.f" +timeout_ns = 1000000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +name="preparation" +top="preparation_tb" +test="protocol_test" +filelist="preparation/filelist.f" +timeout_ns=500000 +wall_timeout_s=120 +kind="functional" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=["preparation_pkg::model::behavior"] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=[] +assertion="" diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/Emit.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/Emit.scala new file mode 100644 index 000000000..0ad00a3da --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/Emit.scala @@ -0,0 +1,126 @@ +package framework.core.memory.access.preflight + +import chisel3._ +import java.nio.file.{Files, Paths} +import framework.ip.bus.chi +import framework.core.memory.access.mmu.{PteRequest, PteResponse} + +object Emit extends App { + val Array("--config", config, "--output-dir", directory) = args + val values = ujson.read(Files.readString(Paths.get(config))) + val bus = upickle.default.read[chi.Params](values("bus"))(upickle.default.macroRW[chi.Params]) + + val p = Params( + bus, + values("contexts").num.toInt, + values("maxRanges").num.toInt, + values("idBits").num.toInt, + values("beatBytes").num.toInt + ) + + Files.createDirectories(Paths.get(directory)) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Preflight(p), + args = Array("--target-dir", s"$directory/Preflight", "--split-verilog") + ) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new PreparedMap(p), + args = Array("--target-dir", s"$directory/PreparedMap", "--split-verilog") + ) + + val defs = scala.collection.mutable.ArrayBuffer[String]( + s"`define PF_ADDRESS_BITS ${bus.addressBits}", + s"`define PF_BEAT_BYTES ${p.beatBytes}", + s"`define PF_CONTEXTS ${p.contexts}" + ) + + val ports = scala.collection.mutable.ArrayBuffer[String]() + val mapPorts = scala.collection.mutable.ArrayBuffer[String]() + + def channel( + kind: String, + prefix: String, + signal: String, + bundle: Bundle, + target: scala.collection.mutable.ArrayBuffer[String], + stream: Boolean = true, + define: Boolean = true + ): Unit = { + var offset = 0 + for ((name, field) <- bundle.elements.toSeq) { + val width = field.getWidth + if (define) { + defs += s"`define PF_${kind}_${name.toUpperCase}_OFFSET $offset"; + defs += s"`define PF_${kind}_${name.toUpperCase}_WIDTH $width" + } + val bits = if (stream) "_bits" else "" + val payload = if (stream) s"$signal.bits" else signal + target += s".$prefix${bits}_$name($payload[$offset +: $width])"; offset += width + } + if (define) defs += s"`define PF_${kind}_WIDTH $offset" + if (stream) { target += s".${prefix}_valid($signal.valid)"; target += s".${prefix}_ready($signal.ready)" } + } + + channel("CMD", "io_command", "cmd", new Command(p), ports) + channel("OUT", "io_prepared", "prepared", new PreparedSegment(p), ports) + channel("AUTH", "io_authorization", "authorization", new Authorization(p), ports) + channel("PERMIT", "io_permission", "permission", new Permission(p), ports) + channel("PTE", "io_pteRequest", "pte_req", new PteRequest(p.walker), ports) + channel("PTERESP", "io_pteResponse", "pte_resp", new PteResponse, ports) + channel("RESERVE", "io_reserve", "reserve", new MapTag(p), mapPorts) + channel("OUT", "io_prepared", "prepared", new PreparedSegment(p), mapPorts, define = false) + channel("MAP_READY", "io_ready", "mapped", new MapReady(p), mapPorts) + channel("RELEASE", "io_release", "retire", new MapTag(p), mapPorts) + for (i <- 0 until 2) { + channel("QUERY", s"io_queries_$i", s"ctl.query_$i", new MapQuery(p), mapPorts, stream = false, define = i == 0) + channel("RESULT", s"io_results_$i", s"ctl.result_$i", new MapResult(p), mapPorts, stream = false, define = i == 0) + } + Files.writeString(Paths.get(directory, "parameters.svh"), defs.mkString("\n") + "\n") + Files.writeString(Paths.get(directory, "preflight_ports.svh"), ports.mkString(",\n") + "\n") + Files.writeString(Paths.get(directory, "prepared_map_ports.svh"), mapPorts.mkString(",\n") + "\n") + val permission = upickle.default.read[configs.PmpParams](values("permission")) + + val regions = values("regions").arr.toSeq.map(v => + upickle.default.read[framework.core.memory.access.PhysicalRegion](v)(configs.Config.regionRW) + ) + + val spec = configs.Config(permission, p, regions) + + val definitions = scala.collection.mutable.ArrayBuffer[String]( + s"`define PREP_CONTEXTS ${p.contexts}", + s"`define PREP_ID_BITS ${p.idBits}", + s"`define PREP_ADDRESS_BITS ${bus.addressBits}", + s"`define PREP_PMP_ENTRIES ${permission.entries}" + ) + + val binding = scala.collection.mutable.ArrayBuffer[String]() + var inputOffset = 0; var outputOffset = 0 + + def leaves(data: Data, path: String): Seq[(String, Data)] = data match { + case record: Record => record.elements.toSeq.flatMap { case (name, field) => + leaves(field, if (path.isEmpty) name else s"${path}_$name") + } + case vector: Vec[_] => vector.toSeq.zipWithIndex.flatMap { case (field, index) => leaves(field, s"${path}_$index") } + case field => Seq(path -> field) + } + + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + { + val dut = new Preparation(spec, p.walker) + for ((name, field) <- leaves(dut.io, "")) { + val input = chisel3.reflect.DataMirror.directionOf(field) == ActualDirection.Input + val direction = if (input) "IN" else "OUT"; val signal = if (input) "inputs" else "outputs"; + val offset = if (input) inputOffset else outputOffset; val width = field.getWidth + definitions += s"`define PREP_${direction}_${name.toUpperCase}_OFFSET $offset"; + definitions += s"`define PREP_${direction}_${name.toUpperCase}_WIDTH $width" + binding += s".io_$name(bus.$signal[$offset +: $width])"; + if (input) inputOffset += width else outputOffset += width + }; dut + }, + args = Array("--target-dir", s"$directory/Preparation", "--split-verilog") + ) + definitions += s"`define PREP_INPUT_BITS $inputOffset"; definitions += s"`define PREP_OUTPUT_BITS $outputOffset" + Files.writeString(Paths.get(directory, "preparation_parameters.svh"), definitions.mkString("\n") + "\n") + Files.writeString(Paths.get(directory, "preparation_ports.svh"), binding.mkString(",\n") + "\n") + +} diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/Permission.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/Permission.scala new file mode 100644 index 000000000..35017ac11 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/Permission.scala @@ -0,0 +1,23 @@ +package framework.core.memory.access.preflight + +import framework.core.memory.access.preflight.configs.PmpParams + +import chisel3._ + +/** Frozen RISC-V physical-memory protection state at the architectural boundary. */ +class PmpConfig extends Bundle { + val l = Bool() + val res = UInt(2.W) + val a = UInt(2.W) + val x = Bool() + val w = Bool() + val r = Bool() +} + +class PmpEntry(b: PmpParams) extends Bundle { + implicit val permission = b + + val cfg = new PmpConfig + val addr = UInt((permission.addressBits - 2).W) + val mask = UInt(permission.addressBits.W) +} diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/PermissionCheck.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/PermissionCheck.scala new file mode 100644 index 000000000..f167ae673 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/PermissionCheck.scala @@ -0,0 +1,108 @@ +package framework.core.memory.access.preflight + +import framework.core.memory.access.preflight.configs.PmpParams + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.core.memory.access.preflight.configs.PmpParams +import framework.core.memory.access.preflight.PmpEntry +import framework.core.memory.access.PhysicalRegion +import framework.core.memory.access.preflight.{Authorization, Permission, Params => PreparationParams} + +/** Checks the complete transport range against the submitting command's frozen PMP image. */ +@instantiable +class PermissionCheck(val specification: configs.Config) extends Module { + val config = specification.parameters + val regions = specification.regions + val permissionParam = specification.permission + implicit val permission = permissionParam + + val physicalBits = math.min(config.bus.addressBits, permission.addressBits) + require(regions.nonEmpty) + regions.foreach(r => require(r.base >= 0 && r.bytes > 0 && r.base + r.bytes <= (BigInt(1) << physicalBits))) + + @public + val io = IO(new Bundle { + val request = Flipped(Decoupled(new Authorization(config))) + // Selected from the captured admission context using request.bits.id, never the live CPU CSRs. + val contextValid = Input(Bool()) + val contextId = Input(UInt(config.idBits.W)) + val pmp = Input(Vec(permission.entries, new PmpEntry(permissionParam))) + val response = Decoupled(new Permission(config)) + }) + + val idle :: checking :: reporting :: Nil = Enum(3) + val state = RegInit(idle) + val request = Reg(new Authorization(config)) + val capturedPmp = Reg(Vec(permission.entries, new PmpEntry(permissionParam))) + val address = Reg(UInt(config.bus.addressBits.W)) + val remaining = Reg(UInt(13.W)) + val allowed = Reg(Bool()) + io.request.ready := state === idle && !reset.asBool + io.response.valid := state === reporting && !reset.asBool + io.response.bits.id := request.id + io.response.bits.allow := allowed + + when(io.request.fire) { + val r = io.request.bits + val end = r.pa +& r.bytes + val shape = io.contextValid && io.contextId === r.id && r.bytes =/= 0.U && r.bytes <= 4096.U && + end <= (BigInt(1) << physicalBits).U && + (r.privilege === 0.U || r.privilege === 1.U || r.privilege === 3.U) && + (!r.isPte || (!r.write && r.bytes === 8.U && r.privilege === 1.U)) + val regionAllowed = regions.map { region => + val accessAllowed = Mux(r.write, region.writable.B, region.readable.B) + region.normal.B && r.pa >= region.base.U && end <= (region.base + region.bytes).U && accessAllowed + }.reduce(_ || _) + request := r + capturedPmp := io.pmp + address := r.pa + remaining := r.bytes + allowed := shape && regionAllowed + state := Mux(shape && regionAllowed, checking, reporting) + } + + // A page segment may cover different PMP entries. Check each actual transport + // transaction (at most one 64-byte line), never a widened page-sized access. + val size = (1 to 6).foldLeft(0.U(3.W)) { (selected, logBytes) => + Mux(remaining >= (1 << logBytes).U && address(logBytes - 1, 0) === 0.U, logBytes.U, selected) + } + + val bytes = (1.U(13.W) << size)(12, 0) + val accessEnd = address +& bytes + val machine = request.privilege === 3.U + + val permitted = + (0 until permission.entries).reverse.foldLeft(if (permission.entries == 0) true.B else machine) { (previous, i) => + val entry = capturedPmp(i) + val encoded = (entry.addr << 2).pad(permission.addressBits) + val low = Mux( + entry.cfg.a === 1.U, + if (i == 0) 0.U(permission.addressBits.W) else (capturedPmp(i - 1).addr << 2), + Mux(entry.cfg.a === 3.U, encoded & ~entry.mask, encoded) + ) + val high = Mux( + entry.cfg.a === 1.U, + encoded.pad(permission.addressBits + 1), + Mux(entry.cfg.a === 3.U, low +& entry.mask + 1.U, encoded +& 4.U) + ) + val hit = entry.cfg.a =/= 0.U && low < high && address < high && accessEnd > low + val contained = address >= low && accessEnd <= high + val accessAllowed = Mux(request.write, entry.cfg.w, entry.cfg.r) || (machine && !entry.cfg.l) + Mux(hit, contained && accessAllowed, previous) + } + + when(state === checking) { + when(!permitted) { + allowed := false.B + state := reporting + }.elsewhen(remaining === bytes) { + state := reporting + }.otherwise { + address := address + bytes + remaining := remaining - bytes + } + } + when(io.response.fire)(state := idle) +} diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/Preflight.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/Preflight.scala new file mode 100644 index 000000000..87b60d5fc --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/Preflight.scala @@ -0,0 +1,337 @@ +package framework.core.memory.access.preflight + +import framework.core.memory.access.preflight.{Params => ModuleParam} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.core.memory.access.mmu.{PteRequest, PteResponse, Walker} + +/** Preparation only. A consumer retains the entire successful mapping until DMA retires. */ +@instantiable +class Preflight(b: ModuleParam) extends Module { + val p = b + + val contextBits = log2Ceil(p.contexts) + + @public + val io = IO(new Bundle { + val command = Flipped(Decoupled(new Command(b))) + val prepared = Decoupled(new PreparedSegment(b)) + val authorization = Decoupled(new Authorization(b)) + val permission = Flipped(Decoupled(new Permission(b))) + + val pteRequest = Decoupled(new PteRequest(b.walker)) + + val pteResponse = Flipped(Decoupled(new PteResponse)) + }) + + val free :: queued :: working :: ready :: Nil = Enum(4) + val phase = RegInit(VecInit(Seq.fill(p.contexts)(free))) + val commands = Reg(Vec(p.contexts, new Command(b))) + val counts = RegInit(VecInit(Seq.fill(p.contexts)(0.U(4.W)))) + val errors = Reg(Vec(p.contexts, UInt(3.W))) + val faultVA = Reg(Vec(p.contexts, UInt(64.W))) + val vas = Reg(Vec(p.contexts, Vec(p.maxRanges, UInt(64.W)))) + val pas = Reg(Vec(p.contexts, Vec(p.maxRanges, UInt(p.bus.addressBits.W)))) + val sizes = Reg(Vec(p.contexts, Vec(p.maxRanges, UInt(13.W)))) + val available = VecInit(phase.map(_ === free)) + val duplicate = + (0 until p.contexts).map(i => phase(i) =/= free && commands(i).id === io.command.bits.id).reduce(_ || _) + io.command.ready := available.asUInt.orR && !duplicate && !reset.asBool + when(io.command.fire) { + val slot = PriorityEncoder(available.asUInt) + commands(slot) := io.command.bits; counts(slot) := 0.U; errors(slot) := Error.Ok.U; phase(slot) := queued + } + + def select(candidates: Vec[Bool], cursor: UInt): UInt = { + MuxCase( + 0.U(contextBits.W), + (0 until p.contexts).map { off => + val sum = cursor +& off.U + val index = Mux(sum >= p.contexts.U, sum - p.contexts.U, sum)(contextBits - 1, 0) + candidates(index) -> index + } + ) + } + + val (idle :: checkShape :: multiply :: measure :: span :: sendSpan :: segment :: walk :: waitWalk :: authorize :: + waitPermission :: save :: retire :: Nil) = Enum(13) + val state = RegInit(idle) + val worker = Reg(UInt(contextBits.W)) + val cursor = RegInit(0.U(contextBits.W)) + val row = Reg(UInt(32.W)); val column = Reg(UInt(32.W)) + val dense = Reg(Bool()); val wholeBytes = Reg(UInt(32.W)) + val spanAddress = Reg(UInt(64.W)); val spanBytes = Reg(UInt(32.W)) + val spanLast = Reg(Bool()) + val currentVA = Reg(UInt(64.W)); val currentPA = Reg(UInt(p.bus.addressBits.W)) + val currentBytes = Reg(UInt(13.W)); val segmentLast = Reg(Bool()) + val unionAuthorization = RegInit(false.B) + // The worker's command, copied when it is selected so shape arithmetic starts at registers. + val cmd = Reg(new Command(b)) + // VA of the current row and of the current span: baseVA + row * rowStride (+ column * columnStride). + val rowStart = Reg(UInt(67.W)); val spanStart = Reg(UInt(67.W)) + // Shape products, built eight multiplier bits per cycle: columns * spanBytes, rows * rowStride, + // columns * columnStride and rows * (columns * spanBytes)(31, 0). + val products = Reg(Vec(4, UInt(64.W))) + val product = Reg(UInt(2.W)); val digit = Reg(UInt(2.W)) + val outcome = Reg(UInt(3.W)); val outcomeVA = Reg(UInt(64.W)) + val previous = (counts(worker) - 1.U)(2, 0) + val previousVA = vas(worker)(previous) + val previousPA = pas(worker)(previous) + val previousEnd = previousVA +& sizes(worker)(previous) + val currentEnd = currentVA +& currentBytes + val cancelSegmenter = WireDefault(false.B) + + val splitter: Instance[Segmenter] = Instantiate(new Segmenter) + + splitter.io.cancel := cancelSegmenter + + val walker: Instance[Walker] = Instantiate(new Walker(p.walker)) + + // The outcome is recorded one cycle after it is decided. + def finish(code: UInt, address: UInt): Unit = { + outcome := code; outcomeVA := address; state := retire + } + + when(state === retire) { + errors(worker) := outcome; faultVA(worker) := outcomeVA; phase(worker) := ready; state := idle + when(outcome =/= Error.Ok.U) { counts(worker) := 0.U; cancelSegmenter := true.B } + } + + val waiting = VecInit(phase.map(_ === queued)) + when(state === idle && waiting.asUInt.orR) { + val slot = select(waiting, cursor) + worker := slot; cursor := slot + 1.U; phase(slot) := working; cmd := commands(slot); state := checkShape + } + when(state === checkShape) { + when(cmd.rows === 0.U || cmd.columns === 0.U || cmd.spanBytes === 0.U || + (cmd.rows > 1.U && cmd.rowStride === 0.U) || (cmd.columns > 1.U && cmd.columnStride === 0.U)) { + finish(Error.Shape.U, cmd.baseVA) + }.elsewhen(cmd.mode =/= 0.U && cmd.mode =/= 8.U || cmd.privilege === 2.U) { + finish(Error.Context.U, cmd.baseVA) + }.otherwise { + product := 0.U; digit := 0.U; state := multiply + } + } + // Least significant multiplier byte first. The upper half of products(k) carries the running + // sum, which stays below 2^32; each step shifts one finished product byte into the lower half. + val multiplicand = VecInit(cmd.columns, cmd.rows, cmd.columns, cmd.rows)(product) + val multiplier = VecInit(cmd.spanBytes, cmd.rowStride, cmd.columnStride, products(0)(31, 0))(product) + val multiplierByte = VecInit((0 until 4).map(i => multiplier(8 * i + 7, 8 * i)))(digit) + when(state === multiply) { + val step = Mux(digit === 0.U, 0.U, products(product)(63, 32)) +& multiplicand * multiplierByte + products(product) := Cat(step(39, 0), products(product)(31, 8)) + digit := digit + 1.U + when(digit === 3.U) { + product := product + 1.U + when(product === 3.U)(state := measure) + } + } + when(state === measure) { + val rowExtent = products(0) + val total = products(3) + val isDense = (cmd.columns === 1.U || cmd.columnStride === cmd.spanBytes) && + (cmd.rows === 1.U || cmd.rowStride === rowExtent) + // baseVA + (rows - 1) * rowStride + (columns - 1) * columnStride + spanBytes - 1 + val end = cmd.baseVA.pad(67) + products(1).pad(67) - cmd.rowStride.pad(67) + products(2).pad(67) - + cmd.columnStride.pad(67) + cmd.spanBytes.pad(67) - 1.U + // rows >= 1, so rows * rowExtent exceeds 32 bits whenever rowExtent does. + when(end(66, 64).orR || (isDense && (rowExtent(63, 32).orR || total(63, 32).orR))) { + finish(Error.Overflow.U, cmd.baseVA) + }.otherwise { + dense := isDense; wholeBytes := total(31, 0); row := 0.U; column := 0.U + rowStart := cmd.baseVA; spanStart := cmd.baseVA; state := span + } + } + when(state === span) { + val bytes = Mux(dense, wholeBytes, cmd.spanBytes) + val beatShift = log2Ceil(p.beatBytes) + // Reads widen to whole beats: (start % beat + bytes + beat - 1) rounded down to a beat. + val widened = (spanStart(beatShift - 1, 0).pad(34) + bytes.pad(34) + (p.beatBytes - 1).U) >> beatShift << beatShift + val length = Mux(cmd.write, bytes.pad(34), widened) + val first = Mux(cmd.write, spanStart(63, 0), spanStart(63, beatShift) << beatShift) + when(length(33, 32).orR)(finish(Error.Overflow.U, spanStart(63, 0))) + .otherwise { + spanAddress := first; spanBytes := length(31, 0) + spanLast := dense || (row === cmd.rows - 1.U && column === cmd.columns - 1.U) + state := sendSpan + } + } + splitter.io.descriptor.valid := state === sendSpan + splitter.io.descriptor.bits.addr := spanAddress + splitter.io.descriptor.bits.bytes := spanBytes + splitter.io.descriptor.bits.rowBytes := spanBytes + splitter.io.descriptor.bits.rowStride := spanBytes + when(splitter.io.descriptor.fire)(state := segment) + splitter.io.segment.ready := state === segment + when(splitter.io.segment.fire) { + currentVA := splitter.io.segment.bits.addr; currentBytes := splitter.io.segment.bits.bytes(12, 0) + segmentLast := splitter.io.segment.bits.last; state := walk + } + walker.io.flush := false.B + walker.io.lookup.config := 0.U.asTypeOf(walker.io.lookup.config) + walker.io.lookup.req := 0.U.asTypeOf(walker.io.lookup.req) + walker.io.config.mode := cmd.mode; walker.io.config.rootPpn := cmd.rootPpn + walker.io.req.valid := state === walk + walker.io.req.bits.vaddr := currentVA; walker.io.req.bits.write := cmd.write; walker.io.req.bits.execute := false.B + walker.io.req.bits.privilege := cmd.privilege; walker.io.req.bits.sum := cmd.sum; walker.io.req.bits.mxr := cmd.mxr + when(walker.io.req.fire)(state := waitWalk) + walker.io.resp.ready := state === waitWalk + when(walker.io.resp.fire) { + val end = walker.io.resp.bits.paddr.pad(p.bus.addressBits + 2) + currentBytes.pad(p.bus.addressBits + 2) - 1.U + when(walker.io.resp.bits.pageFault)(finish(Error.PageFault.U, currentVA)) + .elsewhen(walker.io.resp.bits.accessFault || end(p.bus.addressBits + 1, p.bus.addressBits).orR) { + finish(Error.AccessFault.U, currentVA) + }.otherwise { + currentPA := walker.io.resp.bits.paddr + unionAuthorization := counts(worker) =/= 0.U && previousVA(63, 12) === currentVA(63, 12) && + currentVA > previousEnd && previousPA +& (currentVA - previousVA) === walker.io.resp.bits.paddr + state := authorize + } + } + val pteIdle :: pteAsk :: pteWait :: pteIssue :: pteReturn :: pteReject :: pteRejectReturn :: Nil = Enum(7) + val pteState = RegInit(pteIdle) + when(pteState === pteIdle && walker.io.access.valid)(pteState := pteAsk) + io.authorization.valid := (pteState === pteAsk || state === authorize) && !reset.asBool + io.authorization.bits.id := cmd.id + io.authorization.bits.isPte := pteState === pteAsk + io.authorization.bits.pa := Mux( + pteState === pteAsk, + walker.io.access.bits.addr, + Mux(unionAuthorization, previousPA, currentPA) + ) + io.authorization.bits.bytes := Mux( + pteState === pteAsk, + 8.U, + Mux(unionAuthorization, currentEnd - previousVA, currentBytes) + ) + io.authorization.bits.write := pteState =/= pteAsk && cmd.write + io.authorization.bits.privilege := Mux(pteState === pteAsk, 1.U, cmd.privilege) + when(io.authorization.fire) { + when(pteState === pteAsk)(pteState := pteWait).otherwise(state := waitPermission) + } + io.permission.ready := (pteState === pteWait || state === waitPermission) && !reset.asBool + when(io.permission.fire) { + assert(io.permission.bits.id === cmd.id, "Preflight permission response tag mismatch") + when(pteState === pteWait)(pteState := Mux(io.permission.bits.allow, pteIssue, pteReject)) + .otherwise { + when(io.permission.bits.allow)(state := save) + .elsewhen(unionAuthorization) { + unionAuthorization := false.B + state := authorize + }.otherwise(finish(Error.AccessFault.U, currentVA)) + } + } + io.pteRequest.valid := pteState === pteIssue && walker.io.access.valid && !reset.asBool + io.pteRequest.bits := walker.io.access.bits + walker.io.access.ready := pteState === pteReject || pteState === pteIssue && io.pteRequest.ready + when(walker.io.access.fire)(pteState := Mux(pteState === pteReject, pteRejectReturn, pteReturn)) + walker.io.result.valid := pteState === pteRejectReturn || pteState === pteReturn && io.pteResponse.valid + walker.io.result.bits := 0.U.asTypeOf(walker.io.result.bits) + walker.io.result.bits.data := Mux(pteState === pteRejectReturn, 0.U, io.pteResponse.bits.data) + walker.io.result.bits.error := pteState === pteRejectReturn || io.pteResponse.bits.error + io.pteResponse.ready := pteState === pteReturn && walker.io.result.ready && !reset.asBool + when(walker.io.result.fire)(pteState := pteIdle) + // Touching spans already cover their union. Gapped spans merge only after that whole + // same-page physical union was explicitly authorized; a denied union keeps exact spans. + val merge = counts(worker) =/= 0.U && previousVA(63, 12) === currentVA(63, 12) && + currentVA >= previousVA && (currentVA <= previousEnd || unionAuthorization) + val saveCapacity = merge || counts(worker) =/= p.maxRanges.U + when(state === save && !saveCapacity)(finish(Error.Capacity.U, currentVA)) + when(state === save && saveCapacity) { + when(merge) { + assert( + pas(worker)(previous) +& (currentVA - previousVA) === currentPA, + "Preflight same-page ranges disagree on PA" + ) + when(currentEnd > previousEnd)(sizes(worker)(previous) := currentEnd - previousVA) + }.otherwise { + val index = counts(worker)(2, 0) + vas(worker)(index) := currentVA; pas(worker)(index) := currentPA; sizes(worker)(index) := currentBytes + counts(worker) := counts(worker) + 1.U + } + when(segmentLast && spanLast)(finish(Error.Ok.U, currentVA)) + .elsewhen(segmentLast) { + when(column === cmd.columns - 1.U) { + column := 0.U; row := row + 1.U + rowStart := rowStart + cmd.rowStride; spanStart := rowStart + cmd.rowStride + }.otherwise { column := column + 1.U; spanStart := spanStart + cmd.columnStride } + state := span + }.otherwise(state := segment) + } + val offer = RegInit(false.B); val outputSlot = Reg(UInt(contextBits.W)); val outputIndex = Reg(UInt(3.W)) + val outputCursor = RegInit(0.U(contextBits.W)) + val prepared = VecInit(phase.map(_ === ready)) + when(!offer && prepared.asUInt.orR) { + offer := true.B; outputSlot := select(prepared, outputCursor); outputIndex := 0.U + } + io.prepared.valid := offer && !reset.asBool + val failed = errors(outputSlot) =/= Error.Ok.U + io.prepared.bits.id := commands(outputSlot).id + io.prepared.bits.va := Mux(failed, faultVA(outputSlot), vas(outputSlot)(outputIndex)) + io.prepared.bits.pa := Mux(failed, 0.U, pas(outputSlot)(outputIndex)) + io.prepared.bits.bytes := Mux(failed, 0.U, sizes(outputSlot)(outputIndex)) + io.prepared.bits.write := commands(outputSlot).write + io.prepared.bits.error := errors(outputSlot) + io.prepared.bits.last := failed || outputIndex === counts(outputSlot) - 1.U + when(io.prepared.fire) { + when(io.prepared.bits.last) { phase(outputSlot) := free; offer := false.B; outputCursor := outputSlot + 1.U } + .otherwise(outputIndex := outputIndex + 1.U) + } +} + +/** Shape describes transport bytes, including any read beyond the user's useful payload. */ +class Command(b: ModuleParam) extends Bundle { + val p = b + + val id = UInt(p.idBits.W) + val baseVA = UInt(64.W) + val rows = UInt(32.W) + val columns = UInt(32.W) + val spanBytes = UInt(32.W) + val columnStride = UInt(32.W) + val rowStride = UInt(32.W) + val write = Bool() + val mode = UInt(4.W) + val rootPpn = UInt(44.W) + val privilege = UInt(2.W) + val sum = Bool() + val mxr = Bool() +} + +object Error { + val Ok = 0; val Shape = 1; val Overflow = 2; val PageFault = 3 + val AccessFault = 4; val Capacity = 5; val Context = 6 +} + +/** Failure returns one terminal zero-PA record. The caller retains successful maps and IDs until DMA retirement. */ +class PreparedSegment(b: ModuleParam) extends Bundle { + val p = b + + val id = UInt(p.idBits.W) + val va = UInt(64.W) + val pa = UInt(p.bus.addressBits.W) + val bytes = UInt(13.W) + val write = Bool() + val last = Bool() + val error = UInt(3.W) +} + +class Authorization(b: ModuleParam) extends Bundle { + val p = b + + val id = UInt(p.idBits.W) + val pa = UInt(p.bus.addressBits.W) + val bytes = UInt(13.W) + val write = Bool() + val isPte = Bool() + val privilege = UInt(2.W) +} + +class Permission(b: ModuleParam) extends Bundle { + val p = b + + val id = UInt(p.idBits.W) + val allow = Bool() +} diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/Preparation.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/Preparation.scala new file mode 100644 index 000000000..25fbf4c16 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/Preparation.scala @@ -0,0 +1,88 @@ +package framework.core.memory.access.preflight + +import framework.core.memory.access.preflight.configs.PmpParams +import framework.core.memory.access.mmu.configs.{Params => WalkerParam} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.core.memory.access.preflight.configs.PmpParams +import framework.core.memory.access.preflight.PmpEntry +import framework.core.memory.access.mmu.{PteRequest, PteResponse} +import framework.core.memory.access.PhysicalRegion +import framework.core.memory.access.preflight.{ + Command, + PreparedSegment, + Preflight, + PreparedMap, + MapQuery, + MapResult, + MapReady, + MapTag, + Params => PreparationParams +} + +/** Client boundary for an externally selected DMA address-preparation implementation. */ +class PreparationPort(config: PreparationParams, permissionParam: PmpParams, walkerParam: WalkerParam) extends Bundle { + implicit val permission: PmpParams = permissionParam + + val command = Decoupled(new Command(config)) + val contextTag = Input(UInt(config.idBits.W)) + val contextValid = Output(Bool()) + val contextId = Output(UInt(config.idBits.W)) + val contextPmp = Output(Vec(permission.entries, new PmpEntry(permissionParam))) + val ranges = Flipped(Decoupled(new PreparedSegment(config))) + val ready = Flipped(Decoupled(new MapReady(config))) + val release = Decoupled(new MapTag(config)) + val queries = Output(Vec(2, new MapQuery(config))) + val results = Input(Vec(2, new MapResult(config))) + val pteRequest = Flipped(Decoupled(new PteRequest(walkerParam))) + val pteResponse = Decoupled(new PteResponse) +} + +/** Uses the admission snapshot and retains translations until DMA and cache completion. */ +@instantiable +class Preparation(val specification: configs.Config, walkerParam: WalkerParam) extends Module { + val config = specification.parameters + val permissionParam = specification.permission + implicit val permission: PmpParams = permissionParam + + @public + val io = IO(Flipped(new PreparationPort(config, permissionParam, walkerParam))) + + val preflight: Instance[Preflight] = + Instantiate(new Preflight(config)) + val maps: Instance[PreparedMap] = + Instantiate(new PreparedMap(config)) + + val check: Instance[PermissionCheck] = + Instantiate(new PermissionCheck(specification)) + + preflight.io.command.valid := io.command.valid && maps.io.reserve.ready && !reset.asBool + preflight.io.command.bits := io.command.bits + maps.io.reserve.valid := io.command.valid && preflight.io.command.ready && !reset.asBool + maps.io.reserve.bits.id := io.command.bits.id + io.command.ready := preflight.io.command.ready && maps.io.reserve.ready && !reset.asBool + assert(preflight.io.command.fire === io.command.fire, "Preparation command reservation is not atomic") + assert(maps.io.reserve.fire === io.command.fire, "Preparation map reservation is not atomic") + + val authorization = preflight.io.authorization + io.contextTag := authorization.bits.id + check.io.request <> authorization + check.io.contextValid := io.contextValid + check.io.contextId := io.contextId + check.io.pmp := io.contextPmp + preflight.io.permission <> check.io.response + io.pteRequest <> preflight.io.pteRequest + preflight.io.pteResponse <> io.pteResponse + + maps.io.prepared.valid := preflight.io.prepared.valid && io.ranges.ready + maps.io.prepared.bits := preflight.io.prepared.bits + io.ranges.valid := preflight.io.prepared.valid && maps.io.prepared.ready + io.ranges.bits := preflight.io.prepared.bits + preflight.io.prepared.ready := maps.io.prepared.ready && io.ranges.ready + io.ready <> maps.io.ready + maps.io.queries := io.queries + io.results := maps.io.results + maps.io.release <> io.release +} diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/PreparedMap.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/PreparedMap.scala new file mode 100644 index 000000000..6dad42c53 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/PreparedMap.scala @@ -0,0 +1,185 @@ +package framework.core.memory.access.preflight + +import framework.core.memory.access.preflight.{Params => ModuleParam} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} + +class MapTag(b: ModuleParam) extends Bundle { + val p = b + val id = UInt(p.idBits.W) +} + +class MapReady(b: ModuleParam) extends Bundle { + val p = b + + val id = UInt(p.idBits.W) + val error = UInt(3.W) + val va = UInt(64.W) +} + +class MapQuery(b: ModuleParam) extends Bundle { + val p = b + + val valid = Bool() + val id = UInt(p.idBits.W) + val va = UInt(64.W) + val bytes = UInt(13.W) + val write = Bool() +} + +class MapResult(b: ModuleParam) extends Bundle { + val p = b + + val hit = Bool() + val pa = UInt(p.bus.addressBits.W) + val error = UInt(3.W) +} + +@instantiable +class PreparedMap(b: ModuleParam) extends Module { + val p = b + + @public + val io = IO(new Bundle { + val reserve = Flipped(Decoupled(new MapTag(b))) + val prepared = Flipped(Decoupled(new PreparedSegment(b))) + val ready = Decoupled(new MapReady(b)) + val release = Flipped(Decoupled(new MapTag(b))) + val queries = Input(Vec(2, new MapQuery(b))) + val results = Output(Vec(2, new MapResult(b))) + }) + + val slotBits = log2Ceil(p.contexts) + val live = RegInit(VecInit(Seq.fill(p.contexts)(false.B))) + val sealedList = RegInit(VecInit(Seq.fill(p.contexts)(false.B))) + val notified = RegInit(VecInit(Seq.fill(p.contexts)(false.B))) + val tags = Reg(Vec(p.contexts, UInt(p.idBits.W))) + val counts = RegInit(VecInit(Seq.fill(p.contexts)(0.U(log2Ceil(p.maxRanges + 1).W)))) + val errors = RegInit(VecInit(Seq.fill(p.contexts)(Error.Ok.U(3.W)))) + val faultVA = Reg(Vec(p.contexts, UInt(64.W))) + val direction = Reg(Vec(p.contexts, Bool())) + val vas = Reg(Vec(p.contexts, Vec(p.maxRanges, UInt(64.W)))) + val pas = Reg(Vec(p.contexts, Vec(p.maxRanges, UInt(p.bus.addressBits.W)))) + val sizes = Reg(Vec(p.contexts, Vec(p.maxRanges, UInt(13.W)))) + + def matches(id: UInt): Vec[Bool] = VecInit((0 until p.contexts).map(i => live(i) && tags(i) === id)) + val free = VecInit(live.map(!_)) + val reserveMatch = matches(io.reserve.bits.id) + io.reserve.ready := free.asUInt.orR && !reserveMatch.asUInt.orR && !reset.asBool + when(io.reserve.fire) { + val slot = PriorityEncoder(free) + live(slot) := true.B; tags(slot) := io.reserve.bits.id + sealedList(slot) := false.B; notified(slot) := false.B + counts(slot) := 0.U; errors(slot) := Error.Ok.U; faultVA(slot) := 0.U + } + + val preparedMatch = matches(io.prepared.bits.id) + val preparedSlot = PriorityEncoder(preparedMatch) + io.prepared.ready := preparedMatch.asUInt.orR && !sealedList(preparedSlot) && !reset.asBool + val reservingPrepared = io.reserve.fire && io.reserve.bits.id === io.prepared.bits.id + when(io.prepared.valid && !reset.asBool && !reservingPrepared) { + assert(preparedMatch.asUInt.orR, "PreparedMap record has no reservation") + assert(!sealedList(preparedSlot), "PreparedMap record follows the terminal record") + } + when(io.prepared.fire) { + val record = io.prepared.bits + val lastVA = record.va +& (record.bytes - 1.U) + val lastPA = record.pa +& (record.bytes - 1.U) + val shapeOK = record.bytes =/= 0.U && record.bytes <= 4096.U && + (record.va(11, 0) +& record.bytes) <= 4096.U && + (record.pa(11, 0) +& record.bytes) <= 4096.U && + !lastVA(64) && !lastPA(p.bus.addressBits) + val capacityOK = counts(preparedSlot) < p.maxRanges.U && + (counts(preparedSlot) =/= (p.maxRanges - 1).U || record.last) + val directionOK = counts(preparedSlot) === 0.U || direction(preparedSlot) === record.write + when(record.error =/= Error.Ok.U) { + assert(record.last, "PreparedMap failure must be terminal") + errors(preparedSlot) := record.error; faultVA(preparedSlot) := record.va + sealedList(preparedSlot) := true.B; counts(preparedSlot) := 0.U + }.otherwise { + assert(shapeOK, "PreparedMap successful record must fit one VA and PA page") + assert(capacityOK, "PreparedMap ranges must seal within capacity") + assert(directionOK, "PreparedMap ranges must have one direction") + when(!shapeOK || !capacityOK || !directionOK) { + errors(preparedSlot) := Mux(!capacityOK, Error.Capacity.U, Error.Shape.U) + faultVA(preparedSlot) := record.va; sealedList(preparedSlot) := true.B + counts(preparedSlot) := 0.U + }.otherwise { + val index = counts(preparedSlot)(log2Ceil(p.maxRanges) - 1, 0) + vas(preparedSlot)(index) := record.va; pas(preparedSlot)(index) := record.pa + sizes(preparedSlot)(index) := record.bytes; direction(preparedSlot) := record.write + counts(preparedSlot) := counts(preparedSlot) + 1.U + when(record.last)(sealedList(preparedSlot) := true.B) + } + } + } + + val offer = RegInit(false.B) + val offeredSlot = Reg(UInt(slotBits.W)) + val awaiting = VecInit((0 until p.contexts).map(i => live(i) && sealedList(i) && !notified(i))) + when(!offer && awaiting.asUInt.orR) { + offer := true.B; offeredSlot := PriorityEncoder(awaiting) + } + io.ready.valid := offer && !reset.asBool + io.ready.bits.id := tags(offeredSlot) + io.ready.bits.error := errors(offeredSlot) + io.ready.bits.va := faultVA(offeredSlot) + when(io.ready.fire) { notified(offeredSlot) := true.B; offer := false.B } + + val releaseMatch = matches(io.release.bits.id) + val releaseSlot = PriorityEncoder(releaseMatch) + io.release.ready := releaseMatch.asUInt.orR && notified(releaseSlot) && !reset.asBool + when(io.release.fire) { + live(releaseSlot) := false.B; sealedList(releaseSlot) := false.B + notified(releaseSlot) := false.B; counts(releaseSlot) := 0.U + errors(releaseSlot) := Error.Ok.U + } + + // A lookup takes three cycles and the query must be held throughout: the owning context is + // registered, then the first covering range, and the third cycle answers. The owning context + // must not change while it is queried. + for (port <- 0 until 2) { + val query = io.queries(port) + val owner = matches(query.id) + val owned = RegNext(owner.asUInt.orR) + val slot = RegNext(PriorityEncoder(owner)) + val last = query.va +& (query.bytes - 1.U) + def covers(context: UInt, index: Int): Bool = + index.U < counts(context) && query.va >= vas(context)(index) && + last < (vas(context)(index) +& sizes(context)(index)) && direction(context) === query.write + val hits = VecInit((0 until p.maxRanges).map(covers(slot, _))) + val covered = RegNext(hits.asUInt.orR) + val index = RegNext(PriorityEncoder(hits)) + val retiring = io.release.fire && io.release.bits.id === query.id + val shapeOK = query.bytes =/= 0.U && !last(64) && + (query.va(11, 0) +& query.bytes) <= 4096.U + val authorized = owned && sealedList(slot) && notified(slot) && + errors(slot) === Error.Ok.U && !retiring && !reset.asBool + val translated = pas(slot)(index) +& (query.va - vas(slot)(index)) + val hit = query.valid && shapeOK && authorized && covered + io.results(port).hit := hit + io.results(port).pa := Mux(hit, translated(p.bus.addressBits - 1, 0), 0.U) + io.results(port).error := Mux(!query.valid || hit, Error.Ok.U, Mux(!shapeOK, Error.Shape.U, Error.AccessFault.U)) + + // Map invariants, checked on the query of this cycle against its current owner. + val present = PriorityEncoder(owner) + val presentHits = VecInit((0 until p.maxRanges).map(covers(present, _))) + val presentAllowed = owner.asUInt.orR && sealedList(present) && notified(present) && + errors(present) === Error.Ok.U && !retiring && !reset.asBool + val presentPA = pas(present)(PriorityEncoder(presentHits)) +& + (query.va - vas(present)(PriorityEncoder(presentHits))) + when(query.valid && shapeOK && presentAllowed && presentHits.asUInt.orR) { + assert(!presentPA(presentPA.getWidth - 1, p.bus.addressBits).orR, "PreparedMap translated address overflows PA") + for (other <- 0 until p.maxRanges) { + when(presentHits(other)) { + assert( + (pas(present)(other) +& (query.va - vas(present)(other))) === presentPA, + "PreparedMap overlapping records disagree on PA" + ) + } + } + } + } +} diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/main/scala/Segmenter.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/Segmenter.scala similarity index 95% rename from arch/src/main/scala/framework/mem-core/preflight/src/main/scala/Segmenter.scala rename to arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/Segmenter.scala index 68d3b1e3a..eb5423dc5 100644 --- a/arch/src/main/scala/framework/mem-core/preflight/src/main/scala/Segmenter.scala +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/Segmenter.scala @@ -1,4 +1,4 @@ -package memcore.memory.preflight +package framework.core.memory.access.preflight import chisel3._ import chisel3.util._ @@ -27,7 +27,10 @@ class Segment extends Bundle { * row. No output segment crosses a row, a page, or maxSegmentBytes. */ @instantiable -class Segmenter(pageBytes: Int = 4096, maxSegmentBytes: Int = 16384) extends Module { +class Segmenter extends Module { + val pageBytes = 4096 + val maxSegmentBytes = pageBytes + require(isPow2(pageBytes) && pageBytes >= 8) require(isPow2(maxSegmentBytes) && maxSegmentBytes >= 8) diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/configs/Config.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/configs/Config.scala new file mode 100644 index 000000000..424be2d59 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/configs/Config.scala @@ -0,0 +1,18 @@ +package framework.core.memory.access.preflight.configs + +import framework.core.memory.access.PhysicalRegion +import framework.core.memory.access.preflight.Params +import framework.ip.bus.chi +import upickle.default.{macroRW, ReadWriter} + +case class PmpParams(addressBits: Int, entries: Int) +object PmpParams { implicit val rw: ReadWriter[PmpParams] = macroRW } + +case class Config(permission: PmpParams, parameters: Params, regions: Seq[PhysicalRegion]) + +object Config { + implicit val busRW: ReadWriter[chi.Params] = macroRW + implicit val parametersRW: ReadWriter[Params] = macroRW + implicit val regionRW: ReadWriter[PhysicalRegion] = macroRW + implicit val rw: ReadWriter[Config] = macroRW +} diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..0cecb126c --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/configs/Params.scala @@ -0,0 +1,33 @@ +package framework.core.memory.access.preflight + +import buckyball.config.PreparationHardware + +import framework.core.memory.access.mmu.configs.{Params => WalkerParam} + +import framework.ip.bus.chi + +case class Params( + bus: chi.Params = chi.Params(), + contexts: Int = 4, + maxRanges: Int = 8, + idBits: Int = 8, + beatBytes: Int = 16) { + val walker = WalkerParam(bus, 0) + + require(contexts >= 2) + require(maxRanges == 8) + require(idBits > 0 && bus.addressBits >= 12 && bus.addressBits < 64) + require(beatBytes >= 8 && beatBytes <= 64 && (beatBytes & (beatBytes - 1)) == 0) +} + +object Params { + + def fromProto(p: PreparationHardware): Params = Params( + chi.Params.fromProto(p.getBus), + p.getContexts, + p.getMaxRanges, + p.getIdBits, + p.getBeatBytes + ) + +} diff --git a/arch/src/main/scala/framework/core/memory/build.sc b/arch/src/main/scala/framework/core/memory/build.sc new file mode 100644 index 000000000..0bd309691 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/build.sc @@ -0,0 +1,21 @@ +import mill._ +import $file.access.{build => access_build} +import $file.home.{build => home_build} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("Memory.scala")), + PathRef(millSourcePath / os.RelPath("MemorySystem.scala")), + PathRef(millSourcePath / os.RelPath("ThreadMemory.scala")), + PathRef(millSourcePath / os.RelPath("configs/MemoryParam.scala")), + PathRef(millSourcePath / os.RelPath("uncached/Uncached.scala")), + PathRef(millSourcePath / os.RelPath("uncached/configs/UncachedParam.scala")) + ) ++ + access_build.sources() ++ + home_build.sources() +) + +def emitSources = T.sources( + access_build.emitSources() ++ + home_build.emitSources() +) diff --git a/arch/src/main/scala/framework/core/memory/configs/MemoryParam.scala b/arch/src/main/scala/framework/core/memory/configs/MemoryParam.scala new file mode 100644 index 000000000..90a1738b0 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/configs/MemoryParam.scala @@ -0,0 +1,44 @@ +package framework.core.memory.configs + +import framework.core.memory.access.configs.{Params => AccessParam} +import framework.core.memory.access.cache.configs.{Params => CacheParam} +import framework.core.memory.home.configs.{Params => HomeParam} +import framework.ip.bus.chi.rnf.RnfParams +import framework.core.memory.uncached.configs.UncachedParam +import buckyball.config.MemoryConfig +import scala.jdk.CollectionConverters._ +import framework.core.memory.access.{AccessParams, PhysicalRegion} +import framework.ip.memory.interlock.{Params => TrackingParam} +import framework.ip.memory.coherence.configs.CoherenceParams + +case class MemoryParam( + memoryAccess: AccessParam, + icache: RnfParams, + dcache: CacheParam, + home: HomeParam, + threadPorts: Int, + uncached: UncachedParam) + +object MemoryParam { + + def fromProto(p: MemoryConfig, threadPorts: Int): MemoryParam = { + require(p.isInitialized, "Memory IP configuration is incomplete") + val a = p.getAccess + val homep = HomeParam(CoherenceParams.fromProto(p.getHome)) + val requester = homep.memory.chi + MemoryParam( + AccessParam( + AccessParams(requester, a.getTagBits), + a.getRegionsList.asScala.map(PhysicalRegion.fromProto).toSeq, + a.getTranslationEntries, + a.getResultLineBits + ), + RnfParams.fromProto(p.getIcache, requester), + CacheParam(RnfParams.fromProto(p.getDcache.getCache, requester), TrackingParam.fromProto(p.getDcache.getTracking)), + homep, + threadPorts, + UncachedParam.fromProto(p.getUncached) + ) + } + +} diff --git a/arch/src/main/scala/framework/core/memory/home/build.sc b/arch/src/main/scala/framework/core/memory/home/build.sc new file mode 100644 index 000000000..8c1539ece --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/home/build.sc @@ -0,0 +1,21 @@ +import mill._ +import $file.^.^.^.^.^.^.^.settings + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.core.memory.home.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Home.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/core/memory/home/configs/uvm.toml b/arch/src/main/scala/framework/core/memory/home/configs/uvm.toml new file mode 100644 index 000000000..a0b6176d7 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/home/configs/uvm.toml @@ -0,0 +1,14 @@ +[parameters] +nodeIdBits = 7 +addressBits = 44 +dataBits = 256 +agents = 2 +mshrEntries = 4 +homeId = 64 +cacheSets = 4 +cacheWays = 2 +cacheIdBits = 8 +metadataBits = 4 +responseDepth = 2 + +[defines] diff --git a/arch/src/main/scala/framework/core/memory/home/src/main/resources/agent/transport.svh b/arch/src/main/scala/framework/core/memory/home/src/main/resources/agent/transport.svh new file mode 100644 index 000000000..1f2d2af3d --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/home/src/main/resources/agent/transport.svh @@ -0,0 +1,57 @@ +class transport_item extends uvm_sequence_item; + `uvm_object_utils_begin(transport_item) + `uvm_field_int(value, UVM_ALL_ON) + `uvm_object_utils_end + logic [`COH_DAT_WIDTH-1:0] value; + function new(string name = "flit"); + super.new(name); + endfunction +endclass +class transport_monitor extends uvm_component; + `uvm_component_utils(transport_monitor) + virtual home_transport_if vif; + result_board #(transport_item, 0) boards[6][`COH_AGENTS]; + covergroup behavior with function sample (int channel, int node); + option.per_instance = 1; + channels: coverpoint channel {bins all[] = {[0 : 5]};} + nodes: coverpoint node {bins all[] = {[1 : `COH_AGENTS]};} + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + behavior = new(); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual home_transport_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Home transport interface is required") + foreach (boards[c, n]) + boards[c][n] = result_board#(transport_item, 0)::type_id::create( + $sformatf("channel%0d_node%0d", c, n + 1), this); + endfunction + task run_phase(uvm_phase phase); + forever begin + @(posedge vif.clock); + foreach (boards[c, n]) if (vif.reset) boards[c][n].cancel_pending(); + if (!vif.reset) + for (int c = 0; c < 6; c++) begin + if (vif.expected_valid[c]) begin + transport_item t = transport_item::type_id::create("expected"); + int node = vif.expected_node[c]; + if (node < 1 || node > `COH_AGENTS) + `uvm_fatal("NODE", "Unexpected source or target node at Home") + t.value = vif.expected_bits[c]; + boards[c][node-1].write_expected(t); + behavior.sample(c, node); + end + if (vif.actual_valid[c]) begin + transport_item t = transport_item::type_id::create("actual"); + int node = vif.actual_node[c]; + if (node < 1 || node > `COH_AGENTS) + `uvm_fatal("NODE", "Unexpected delivered node at Home") + t.value = vif.actual_bits[c]; + boards[c][node-1].write_actual(t); + end + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/home/src/main/resources/agent/transport_if.sv b/arch/src/main/scala/framework/core/memory/home/src/main/resources/agent/transport_if.sv new file mode 100644 index 000000000..e0291b49b --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/home/src/main/resources/agent/transport_if.sv @@ -0,0 +1,9 @@ +`include "coherence_config.svh" +interface home_transport_if ( + input logic clock, + input logic reset +); + logic [5:0] expected_valid, actual_valid; + logic [`COH_DAT_WIDTH-1:0] expected_bits[6], actual_bits[6]; + int expected_node[6], actual_node[6]; +endinterface diff --git a/arch/src/main/scala/framework/core/memory/home/src/main/resources/filelist.f b/arch/src/main/scala/framework/core/memory/home/src/main/resources/filelist.f new file mode 100644 index 000000000..822f9033f --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/home/src/main/resources/filelist.f @@ -0,0 +1,11 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core ++incdir+@VERIFY@/uvm/src/protocol/chi +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/core/stream_if.sv +@RESOURCES@/interface.sv +@RESOURCES@/agent/transport_if.sv +@RESOURCES@/package.sv +-F @RTL@/Home/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/mem-core/coherence/src/main/resources/interface.sv b/arch/src/main/scala/framework/core/memory/home/src/main/resources/interface.sv similarity index 100% rename from arch/src/main/scala/framework/mem-core/coherence/src/main/resources/interface.sv rename to arch/src/main/scala/framework/core/memory/home/src/main/resources/interface.sv diff --git a/arch/src/main/scala/framework/core/memory/home/src/main/resources/package.sv b/arch/src/main/scala/framework/core/memory/home/src/main/resources/package.sv new file mode 100644 index 000000000..05db26d39 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/home/src/main/resources/package.sv @@ -0,0 +1,8 @@ +`include "coherence_config.svh" +`define CF(K, F) `COH_``K``_``F``_OFFSET+:`COH_``K``_``F``_WIDTH +package home_pkg; + `include "coherence/body.svh" + `include "agent/transport.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/memory/home/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/core/memory/home/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..e3ddc8ea2 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/home/src/main/resources/seq/sequence.svh @@ -0,0 +1,16 @@ +class home_sequence extends access_sequence; + `uvm_object_utils(home_sequence) + function new(string name = "scenario"); + super.new(name); + endfunction + virtual task drive_idle(req_t payload, int cycles = 1, bit blocked = 0); + // The requester boundary may accept into its credit FIFO while the Home is busy. + super.drive_idle(payload, cycles, 0); + endtask + virtual task send(int node, int opcode, longint unsigned address, bit error = 0); + int key = (node << 12) | (serial & 4095); + // Requester-owned transaction IDs are reusable after the final response arrives. + while (test_env.model.requests.exists(key)) @(control.cb); + super.send(node, opcode, address, error); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/home/src/main/resources/suite.toml b/arch/src/main/scala/framework/core/memory/home/src/main/resources/suite.toml new file mode 100644 index 000000000..a98cd21a0 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/home/src/main/resources/suite.toml @@ -0,0 +1,19 @@ +[model] +manifest="../../../../../../../../verify/uvm/src/protocol/chi/coherence/model/Cargo.toml" +library="libcoherence_reference_model" +[[tests]] +name="home" +top="home_tb" +test="home_test" +filelist="filelist.f" +timeout_ns=1000000 +wall_timeout_s=120 +kind="functional" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=["home_pkg::system_model::operations","home_pkg::system_model::occupancy","home_pkg::transport_monitor::behavior"] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=[] +assertion="" diff --git a/arch/src/main/scala/framework/core/memory/home/src/main/resources/tb.sv b/arch/src/main/scala/framework/core/memory/home/src/main/resources/tb.sv new file mode 100644 index 000000000..f9d871339 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/home/src/main/resources/tb.sv @@ -0,0 +1,228 @@ +`include "coherence_config.svh" +module home_tb; + import uvm_pkg::*; + import home_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + coherence_control_if control (clock); + stream_if #(`COH_REQ_WIDTH) source_if ( + clock, + control.reset + ); + stream_if #(`COH_RSP_WIDTH) rx_rsp ( + clock, + control.reset + ); + stream_if #(`COH_DAT_WIDTH) rx_dat ( + clock, + control.reset + ); + stream_if #(`COH_TXRSP_WIDTH) tx_rsp ( + clock, + control.reset + ); + stream_if #(`COH_TXDAT_WIDTH) sink_if ( + clock, + control.reset + ); + stream_if #(`COH_SNP_CHANNEL_WIDTH) tx_snp ( + clock, + control.reset + ); + stream_if #(`COH_MEMREQ_WIDTH) mem_req ( + clock, + control.reset + ); + stream_if #(`COH_MEMRESP_WIDTH) mem_resp ( + clock, + control.reset + ); + stream_if #(`COH_REQ_WIDTH) observedReq ( + clock, + control.reset + ); + stream_if #(`COH_RSP_WIDTH) observedRxRsp ( + clock, + control.reset + ); + stream_if #(`COH_DAT_WIDTH) observedRxDat ( + clock, + control.reset + ); + logic [`COH_AGENTS-1:0] req_valid, req_ready; + logic [`COH_REQ_WIDTH-1:0] req_bits[`COH_AGENTS]; + logic [`COH_AGENTS-1:0] txRsp_valid, txRsp_ready; + logic [`COH_RSP_WIDTH-1:0] txRsp_bits[`COH_AGENTS]; + logic [`COH_AGENTS-1:0] txDat_valid, txDat_ready; + logic [`COH_DAT_WIDTH-1:0] txDat_bits[`COH_AGENTS]; + logic [`COH_AGENTS-1:0] rxRsp_valid, rxRsp_ready; + logic [`COH_TXRSP_WIDTH-1:0] rxRsp_bits[`COH_AGENTS]; + logic [`COH_AGENTS-1:0] rxDat_valid, rxDat_ready; + logic [`COH_TXDAT_WIDTH-1:0] rxDat_bits[`COH_AGENTS]; + logic [`COH_AGENTS-1:0] snp_valid, snp_ready; + logic [`COH_SNP_WIDTH-1:0] snp_bits[`COH_AGENTS]; + wire observedReq_valid; + wire [`COH_REQ_WIDTH-1:0] observedReq_bits; + wire observedRxRsp_valid; + wire [`COH_RSP_WIDTH-1:0] observedRxRsp_bits; + wire observedRxDat_valid; + wire [`COH_DAT_WIDTH-1:0] observedRxDat_bits; + wire observedRsp_valid; + wire [`COH_TXRSP_WIDTH-1:0] observedRsp_bits; + wire observedDat_valid; + wire [`COH_TXDAT_WIDTH-1:0] observedDat_bits; + wire observedSnp_valid; + wire [`COH_SNP_CHANNEL_WIDTH-1:0] observedSnp_bits; + assign observedReq.valid = observedReq_valid; + assign observedReq.bits = observedReq_bits; + assign observedReq.ready = 1; + assign observedRxRsp.valid = observedRxRsp_valid; + assign observedRxRsp.bits = observedRxRsp_bits; + assign observedRxRsp.ready = 1; + assign observedRxDat.valid = observedRxDat_valid; + assign observedRxDat.bits = observedRxDat_bits; + assign observedRxDat.ready = 1; + Home dut ( + .clock(clock), + .reset(control.reset), + .io_active(!control.reset), + .io_blockRequesterRsp(1'b0), + .io_outstanding(control.outstanding), + `include "home_ports.svh" + ); + for (genvar i = 0; i < `COH_AGENTS; i++) begin + assign req_valid[i] = source_if.valid && source_if.bits[`CF(REQ, SRCID)] == i + 1; + assign req_bits[i] = source_if.bits; + assign txRsp_valid[i] = rx_rsp.valid && rx_rsp.bits[`CF(RSP, SRCID)] == i + 1; + assign txRsp_bits[i] = rx_rsp.bits; + assign txDat_valid[i] = rx_dat.valid && rx_dat.bits[`CF(DAT, SRCID)] == i + 1; + assign txDat_bits[i] = rx_dat.bits; + end + always_comb begin + source_if.ready = 0; + rx_rsp.ready = 0; + rx_dat.ready = 0; + for (int i = 0; i < `COH_AGENTS; i++) begin + source_if.ready |= source_if.bits[`CF(REQ, SRCID)] == i + 1 && req_ready[i]; + rx_rsp.ready |= rx_rsp.bits[`CF(RSP, SRCID)] == i + 1 && txRsp_ready[i]; + rx_dat.ready |= rx_dat.bits[`CF(DAT, SRCID)] == i + 1 && txDat_ready[i]; + end + end + int selected_rxRsp = -1; + always @(posedge clock) if (control.reset || (tx_rsp.valid && tx_rsp.ready)) selected_rxRsp <= -1; + always @(negedge clock) + if (!control.reset && selected_rxRsp < 0) + for (int i = 0; i < `COH_AGENTS; i++) + if (selected_rxRsp < 0 && rxRsp_valid[i]) selected_rxRsp = i; + always_comb begin + tx_rsp.valid = 0; + tx_rsp.bits = '0; + rxRsp_ready = '0; + if (selected_rxRsp >= 0) begin + tx_rsp.valid = rxRsp_valid[selected_rxRsp]; + tx_rsp.bits = rxRsp_bits[selected_rxRsp]; + rxRsp_ready[selected_rxRsp] = tx_rsp.ready; + end + end + int selected_rxDat = -1; + always @(posedge clock) + if (control.reset || (sink_if.valid && sink_if.ready)) + selected_rxDat <= -1; + always @(negedge clock) + if (!control.reset && selected_rxDat < 0) + for (int i = 0; i < `COH_AGENTS; i++) + if (selected_rxDat < 0 && rxDat_valid[i]) selected_rxDat = i; + always_comb begin + sink_if.valid = 0; + sink_if.bits = '0; + rxDat_ready = '0; + if (selected_rxDat >= 0) begin + sink_if.valid = rxDat_valid[selected_rxDat]; + sink_if.bits = rxDat_bits[selected_rxDat]; + rxDat_ready[selected_rxDat] = sink_if.ready; + end + end + int selected_snp = -1; + always @(posedge clock) if (control.reset || (tx_snp.valid && tx_snp.ready)) selected_snp <= -1; + always @(negedge clock) + if (!control.reset && selected_snp < 0) + for (int i = 0; i < `COH_AGENTS; i++) if (selected_snp < 0 && snp_valid[i]) selected_snp = i; + always_comb begin + tx_snp.valid = 0; + tx_snp.bits = '0; + snp_ready = '0; + if (selected_snp >= 0) begin + tx_snp.valid = snp_valid[selected_snp]; + tx_snp.bits = {`COH_SNP_TARGET_WIDTH'(selected_snp + 1), snp_bits[selected_snp]}; + snp_ready[selected_snp] = tx_snp.ready; + end + end + home_transport_if transport ( + clock, + control.reset + ); + assign transport.expected_valid[0] = source_if.valid && source_if.ready; + assign transport.expected_bits[0] = `COH_DAT_WIDTH'(source_if.bits); + assign transport.expected_node[0] = source_if.bits[`CF(REQ, SRCID)]; + assign transport.actual_valid[0] = observedReq.valid; + assign transport.actual_bits[0] = `COH_DAT_WIDTH'(observedReq.bits); + assign transport.actual_node[0] = observedReq.bits[`CF(REQ, SRCID)]; + assign transport.expected_valid[1] = rx_rsp.valid && rx_rsp.ready; + assign transport.expected_bits[1] = `COH_DAT_WIDTH'(rx_rsp.bits); + assign transport.expected_node[1] = rx_rsp.bits[`CF(RSP, SRCID)]; + assign transport.actual_valid[1] = observedRxRsp.valid; + assign transport.actual_bits[1] = `COH_DAT_WIDTH'(observedRxRsp.bits); + assign transport.actual_node[1] = observedRxRsp.bits[`CF(RSP, SRCID)]; + assign transport.expected_valid[2] = rx_dat.valid && rx_dat.ready; + assign transport.expected_bits[2] = `COH_DAT_WIDTH'(rx_dat.bits); + assign transport.expected_node[2] = rx_dat.bits[`CF(DAT, SRCID)]; + assign transport.actual_valid[2] = observedRxDat.valid; + assign transport.actual_bits[2] = `COH_DAT_WIDTH'(observedRxDat.bits); + assign transport.actual_node[2] = observedRxDat.bits[`CF(DAT, SRCID)]; + assign transport.expected_valid[3] = observedRsp_valid; + assign transport.expected_bits[3] = `COH_DAT_WIDTH'(observedRsp_bits); + assign transport.expected_node[3] = observedRsp_bits[`CF(TXRSP, TGTID)]; + assign transport.actual_valid[3] = tx_rsp.valid && tx_rsp.ready; + assign transport.actual_bits[3] = `COH_DAT_WIDTH'(tx_rsp.bits); + assign transport.actual_node[3] = tx_rsp.bits[`CF(TXRSP, TGTID)]; + assign transport.expected_valid[4] = observedDat_valid; + assign transport.expected_bits[4] = `COH_DAT_WIDTH'(observedDat_bits); + assign transport.expected_node[4] = observedDat_bits[`CF(TXDAT, TGTID)]; + assign transport.actual_valid[4] = sink_if.valid && sink_if.ready; + assign transport.actual_bits[4] = `COH_DAT_WIDTH'(sink_if.bits); + assign transport.actual_node[4] = sink_if.bits[`CF(TXDAT, TGTID)]; + assign transport.expected_valid[5] = observedSnp_valid; + assign transport.expected_bits[5] = `COH_DAT_WIDTH'(observedSnp_bits); + assign transport.expected_node[5] = observedSnp_bits[`CF(SNP, TARGET)]; + assign transport.actual_valid[5] = tx_snp.valid && tx_snp.ready; + assign transport.actual_bits[5] = `COH_DAT_WIDTH'(tx_snp.bits); + assign transport.actual_node[5] = tx_snp.bits[`CF(SNP, TARGET)]; + initial begin + uvm_config_db#(virtual coherence_control_if)::set(null, "uvm_test_top*", "control", control); + uvm_config_db#(virtual stream_if #(`COH_REQ_WIDTH))::set(null, "uvm_test_top*", "req", + source_if); + uvm_config_db#(virtual stream_if #(`COH_RSP_WIDTH))::set(null, "uvm_test_top*", "rx_rsp", + rx_rsp); + uvm_config_db#(virtual stream_if #(`COH_DAT_WIDTH))::set(null, "uvm_test_top*", "rx_dat", + rx_dat); + uvm_config_db#(virtual stream_if #(`COH_TXRSP_WIDTH))::set(null, "uvm_test_top*", "tx_rsp", + tx_rsp); + uvm_config_db#(virtual stream_if #(`COH_TXDAT_WIDTH))::set(null, "uvm_test_top*", "tx_dat", + sink_if); + uvm_config_db#(virtual stream_if #(`COH_SNP_CHANNEL_WIDTH))::set(null, "uvm_test_top*", "snp", + tx_snp); + uvm_config_db#(virtual stream_if #(`COH_MEMREQ_WIDTH))::set(null, "uvm_test_top*", "mem_req", + mem_req); + uvm_config_db#(virtual stream_if #(`COH_MEMRESP_WIDTH))::set(null, "uvm_test_top*", "mem_resp", + mem_resp); + uvm_config_db#(virtual stream_if #(`COH_REQ_WIDTH))::set(null, "uvm_test_top.env.bus.mon", + "req", observedReq); + uvm_config_db#(virtual stream_if #(`COH_RSP_WIDTH))::set(null, "uvm_test_top.env.bus.mon", + "rx_rsp", observedRxRsp); + uvm_config_db#(virtual stream_if #(`COH_DAT_WIDTH))::set(null, "uvm_test_top.env.bus.mon", + "rx_dat", observedRxDat); + uvm_config_db#(virtual home_transport_if)::set(null, "uvm_test_top.transport", "vif", + transport); + run_test("home_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/core/memory/home/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/core/memory/home/src/main/resources/tests/test.svh new file mode 100644 index 000000000..181e07297 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/home/src/main/resources/tests/test.svh @@ -0,0 +1,17 @@ +class home_test extends protocol_test; + `uvm_component_utils(home_test) + transport_monitor transport; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + transport = transport_monitor::type_id::create("transport", this); + endfunction + task execute(); + home_sequence seq = home_sequence::type_id::create("scenario"); + seq.test_env = test_env; + seq.control = control; + seq.start(test_env.bus.seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/home/src/main/scala/Emit.scala b/arch/src/main/scala/framework/core/memory/home/src/main/scala/Emit.scala new file mode 100644 index 000000000..3a7900e4d --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/home/src/main/scala/Emit.scala @@ -0,0 +1,169 @@ +package framework.core.memory.home + +import java.nio.file.{Files, Paths} +import framework.ip.bus.chi._ +import framework.ip.memory.coherence.configs.CoherenceParams + +object Emit extends App { + val Array("--config", config, "--output-dir", directory) = args + val values = ujson.read(Files.readString(Paths.get(config))) + val output = Paths.get(directory) + Files.createDirectories(output) + val chi = Params(values("nodeIdBits").num.toInt, values("addressBits").num.toInt, values("dataBits").num.toInt) + + val cache = framework.ip.memory.cache.configs.CacheParams( + chi.addressBits, + 64, + values("cacheSets").num.toInt, + values("cacheWays").num.toInt, + values("cacheIdBits").num.toInt, + values("metadataBits").num.toInt, + values("responseDepth").num.toInt + ) + + val p = + CoherenceParams(chi, cache, values("agents").num.toInt, values("mshrEntries").num.toInt, values("homeId").num.toInt) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Home(configs.Params(p)), + args = Array("--target-dir", output.resolve("Home").toString, "--split-verilog") + ) + val definitions = scala.collection.mutable.ArrayBuffer[String]() + val ports = scala.collection.mutable.ArrayBuffer[String]() + + def fields(flit: Flit): Seq[(String, Int)] = flit.fieldsLSB.map { field => + flit.elements.find(_._2 eq field).get._1 -> field.getWidth + } + + def channel( + kind: String, + prefix: String, + instance: String, + fs: Seq[(String, Int)] + ): Unit = { + var offset = 0 + for ((name, width) <- fs if width > 0) { + definitions += s"`define COH_${kind}_${name.toUpperCase}_OFFSET $offset" + definitions += s"`define COH_${kind}_${name.toUpperCase}_WIDTH $width" + ports += s".$prefix$name($instance.bits[$offset +: $width])" + offset += width + } + definitions += s"`define COH_${kind}_WIDTH $offset" + } + + channel("REQ", "io_req_bits_", "source_if", fields(new RequestFlit(p.chi))) + channel("RSP", "io_rxRsp_bits_", "rx_rsp", fields(new ResponseFlit(p.chi))) + channel("DAT", "io_rxDat_bits_", "rx_dat", fields(new DataFlit(p.chi))) + channel("TXRSP", "io_rsp_bits_", "tx_rsp", fields(new ResponseFlit(p.chi))) + channel("TXDAT", "io_dat_bits_", "sink_if", fields(new DataFlit(p.chi))) + channel("SNP", "io_snp_bits_flit_", "tx_snp", fields(new SnoopFlit(p.chi))) + val snpWidth = (new SnoopFlit(p.chi)).flitWidth + ports += s".io_snp_bits_targetNode(tx_snp.bits[$snpWidth +: ${p.chi.nodeIdBits}])" + definitions += s"`define COH_SNP_TARGET_OFFSET $snpWidth" + definitions += s"`define COH_SNP_TARGET_WIDTH ${p.chi.nodeIdBits}" + definitions += s"`define COH_SNP_CHANNEL_WIDTH ${snpWidth + p.chi.nodeIdBits}" + channel( + "MEMREQ", + "io_memoryReq_bits_", + "mem_req", + Seq("id" -> p.chi.txnIdBits, "addr" -> p.chi.addressBits, "write" -> 1, "data" -> 512, "mask" -> 64) + ) + channel("MEMRESP", "io_memoryResp_bits_", "mem_resp", Seq("id" -> p.chi.txnIdBits, "data" -> 512, "error" -> 1)) + for ( + (port, instance) <- Seq( + "req" -> "source_if", + "rxRsp" -> "rx_rsp", + "rxDat" -> "rx_dat", + "rsp" -> "tx_rsp", + "dat" -> "sink_if", + "snp" -> "tx_snp", + "memoryReq" -> "mem_req", + "memoryResp" -> "mem_resp" + ) + ) { + ports += s".io_${port}_valid($instance.valid)" + ports += s".io_${port}_ready($instance.ready)" + } + for ( + (name, value) <- Seq( + "AGENTS" -> p.agents, + "MSHRS" -> p.mshrEntries, + "HOME" -> p.homeId, + "SETS" -> p.cache.sets, + "WAYS" -> p.cache.ways, + "BEATS" -> p.chi.beatsPerLine, + "DATA_BITS" -> p.chi.dataBits, + "OUTSTANDING_BITS" -> chisel3.util.log2Ceil(p.mshrEntries + 1), + "READ_SHARED" -> Opcode.ReadShared, + "READ_NSD" -> Opcode.ReadNotSharedDirty, + "READ_UNIQUE" -> Opcode.ReadUnique, + "MAKE_UNIQUE" -> Opcode.MakeUnique, + "EVICT" -> Opcode.Evict, + "WRITEBACK" -> Opcode.WriteBackFull, + "CLEAN_SHARED" -> Opcode.CleanShared, + "CLEAN_INVALID" -> Opcode.CleanInvalid, + "MAKE_INVALID" -> Opcode.MakeInvalid, + "COMP" -> Opcode.Comp, + "COMP_DBID" -> Opcode.CompDBIDResp, + "COMP_ACK" -> Opcode.CompAck, + "COMP_DATA" -> Opcode.CompData, + "SNP_RESP" -> Opcode.SnpResp, + "SNP_DATA" -> Opcode.SnpRespData, + "COPYBACK_DATA" -> Opcode.CopyBackWriteData, + "SNP_UNIQUE" -> Opcode.SnpUnique, + "SNP_INVALID" -> Opcode.SnpCleanInvalid, + "SNP_CLEAN_SHARED" -> Opcode.SnpCleanShared, + "SNP_MAKE_INVALID" -> Opcode.SnpMakeInvalid, + "SNP_SHARED" -> Opcode.SnpNotSharedDirty + ) + ) { + definitions += s"`define COH_$name $value" + } + Files.writeString( + output.resolve("coherence_config.svh"), + "`ifndef COHERENCE_CONFIG_SVH\n`define COHERENCE_CONFIG_SVH\n" + definitions.mkString("\n") + "\n`endif\n" + ) + + val homePorts = scala.collection.mutable.ArrayBuffer[String]() + + def bindFlit(prefix: String, signal: String, flit: Flit): Unit = { + var offset = 0 + for ((name, width) <- fields(flit) if width > 0) { + homePorts += s".${prefix}_$name($signal[$offset +: $width])"; offset += width + } + } + + for (i <- 0 until p.agents) { + for ( + (channel, flit) <- Seq( + "req" -> new RequestFlit(p.chi), + "txRsp" -> new ResponseFlit(p.chi), + "txDat" -> new DataFlit(p.chi), + "rxRsp" -> new ResponseFlit(p.chi), + "rxDat" -> new DataFlit(p.chi), + "snp" -> new SnoopFlit(p.chi) + ) + ) { + homePorts += s".io_requesters_${i}_${channel}_valid(${channel}_valid[$i])" + homePorts += s".io_requesters_${i}_${channel}_ready(${channel}_ready[$i])" + bindFlit(s"io_requesters_${i}_${channel}_bits", s"${channel}_bits[$i]", flit) + } + } + for ( + (name, flit) <- Seq( + "Req" -> new RequestFlit(p.chi), + "RxRsp" -> new ResponseFlit(p.chi), + "RxDat" -> new DataFlit(p.chi), + "Rsp" -> new ResponseFlit(p.chi), + "Dat" -> new DataFlit(p.chi), + "Snp" -> new SnoopFlit(p.chi) + ) + ) { + val signal = s"observed$name" + homePorts += s".io_${signal}_valid(${signal}_valid)" + bindFlit(s"io_${signal}_bits" + (if (name == "Snp") "_flit" else ""), s"${signal}_bits", flit) + if (name == "Snp") + homePorts += s".io_observedSnp_bits_targetNode(observedSnp_bits[$snpWidth +: ${p.chi.nodeIdBits}])" + } + homePorts ++= ports.filter(p => p.startsWith(".io_memory")) + Files.writeString(output.resolve("home_ports.svh"), homePorts.mkString(",\n") + "\n") +} diff --git a/arch/src/main/scala/framework/core/memory/home/src/main/scala/Home.scala b/arch/src/main/scala/framework/core/memory/home/src/main/scala/Home.scala new file mode 100644 index 000000000..b03ed63db --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/home/src/main/scala/Home.scala @@ -0,0 +1,72 @@ +package framework.core.memory.home + +import framework.core.memory.home.configs.{Params => ModuleParam} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.ip.bus.chi._ +import framework.ip.bus.chi.snf.{LineRequest, LineResponse} +import framework.ip.memory.coherence.Coherence +import framework.ip.memory.coherence.configs.CoherenceParams + +/** One Core-owned shared L2/Home and its explicit requester fabric. */ +@instantiable +class Home(b: ModuleParam) extends Module { + val p = b.memory + + val c = p.chi + + @public + val io = IO(new Bundle { + val active = Input(Bool()) + val blockRequesterRsp = Input(Bool()) + + val requesters = Vec( + p.agents, + Flipped(new RequesterPort(c)) + ) + + val memoryReq = Decoupled(new LineRequest(c)) + + val memoryResp = Flipped(Decoupled(new LineResponse(c))) + + val outstanding = Output(UInt(log2Ceil(p.mshrEntries + 1).W)) + + val observedReq = Output(Valid(new RequestFlit(c))) + + val observedRsp = Output(Valid(new ResponseFlit(c))) + + val observedDat = Output(Valid(new DataFlit(c))) + + val observedSnp = Output(Valid(new DirectedSnoop(c))) + + val observedRxRsp = Output(Valid(new ResponseFlit(c))) + + val observedRxDat = Output(Valid(new DataFlit(c))) + + }) + + val home: Instance[Coherence] = Instantiate(new Coherence(b.memory)) + + val fabric: Instance[Fabric] = Instantiate(new Fabric(c, p.agents, p.homeId)) + + fabric.io.active := io.active + fabric.io.blockRequesterRsp := io.blockRequesterRsp + fabric.io.requesters <> io.requesters + home.io.req <> fabric.io.req + home.io.rxRsp <> fabric.io.rxRsp + home.io.rxDat <> fabric.io.rxDat + fabric.io.rsp <> home.io.rsp + fabric.io.dat <> home.io.dat + fabric.io.snp <> home.io.snp + io.memoryReq <> home.io.memoryReq + home.io.memoryResp <> io.memoryResp + io.outstanding := home.io.outstanding + io.observedReq.valid := home.io.req.fire; io.observedReq.bits := home.io.req.bits + io.observedRsp.valid := home.io.rsp.fire; io.observedRsp.bits := home.io.rsp.bits + io.observedDat.valid := home.io.dat.fire; io.observedDat.bits := home.io.dat.bits + io.observedSnp.valid := home.io.snp.fire; io.observedSnp.bits := home.io.snp.bits + io.observedRxRsp.valid := home.io.rxRsp.fire; io.observedRxRsp.bits := home.io.rxRsp.bits + io.observedRxDat.valid := home.io.rxDat.fire; io.observedRxDat.bits := home.io.rxDat.bits +} diff --git a/arch/src/main/scala/framework/core/memory/home/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/core/memory/home/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..13767bbbf --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/home/src/main/scala/configs/Params.scala @@ -0,0 +1,13 @@ +package framework.core.memory.home.configs + +import framework.ip.memory.coherence.configs.CoherenceParams +import upickle.default.{macroRW, ReadWriter} + +case class Params(memory: CoherenceParams) + +object Params { + implicit val chiRW: ReadWriter[framework.ip.bus.chi.Params] = macroRW + implicit val cacheRW: ReadWriter[framework.ip.memory.cache.configs.CacheParams] = macroRW + implicit val coherenceRW: ReadWriter[CoherenceParams] = macroRW + implicit val rw: ReadWriter[Params] = macroRW +} diff --git a/arch/src/main/scala/framework/core/memory/uncached/Uncached.scala b/arch/src/main/scala/framework/core/memory/uncached/Uncached.scala new file mode 100644 index 000000000..baff80aa0 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/uncached/Uncached.scala @@ -0,0 +1,77 @@ +package framework.core.memory.uncached + +import framework.core.memory.uncached.configs.UncachedParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.ip.bus.axi4 +import framework.core.memory.access.{AccessParams, UncachedRequest, UncachedResponse} + +/** Uncached CPU accesses share one AXI master; responses retain the CPU port and tag. */ +@instantiable +class Uncached(requestp: AccessParams, uncachedp: UncachedParam) extends Module { + val ports = uncachedp.ports + val axip = uncachedp.axi + + @public + val io = IO(new Bundle { + val request = Vec(ports, Flipped(Decoupled(new UncachedRequest(requestp)))) + val response = Vec(ports, Decoupled(new UncachedResponse(requestp))) + val axi = new axi4.Port(axip) + }) + + val indexBits = math.max(1, log2Ceil(ports)) + val idle :: issue :: waitResponse :: respond :: Nil = Enum(4) + val state = RegInit(idle) + val cursor = RegInit(0.U(indexBits.W)) + val owner = Reg(UInt(indexBits.W)) + val command = Reg(new UncachedRequest(requestp)) + val answer = Reg(new UncachedResponse(requestp)) + val addressSent = RegInit(false.B) + val dataSent = RegInit(false.B) + val ordered = (0 until ports).map(i => (cursor +& i.U) % ports.U) + val selected = if (ports == 1) 0.U else PriorityMux(ordered.map(i => io.request(i).valid), ordered) + val present = io.request.map(_.valid).reduce(_ || _) + for (i <- 0 until ports) { + io.request(i).ready := state === idle && present && selected === i.U + io.response(i).valid := state === respond && owner === i.U + io.response(i).bits := answer + when(io.response(i).fire)(state := idle) + } + when(state === idle && present) { + owner := selected + command := (if (ports == 1) io.request.head.bits else io.request(selected).bits) + addressSent := false.B + dataSent := false.B + cursor := Mux(selected === (ports - 1).U, 0.U, selected + 1.U) + state := issue + } + for (address <- Seq(io.axi.aw, io.axi.ar)) { + address.bits := 0.U.asTypeOf(address.bits) + address.bits.addr := command.addr + address.bits.size := command.size + address.bits.burst := 1.U + } + io.axi.aw.valid := state === issue && command.write && !addressSent + io.axi.ar.valid := state === issue && !command.write + val shift = command.addr(log2Ceil(axip.bytes) - 1, 0) + io.axi.w.valid := state === issue && command.write && !dataSent + io.axi.w.bits.data := command.data << (shift << 3) + io.axi.w.bits.strb := (((1.U((axip.bytes + 1).W) << (1.U << command.size)) - 1.U) << shift) + io.axi.w.bits.last := true.B + when(io.axi.aw.fire)(addressSent := true.B) + when(io.axi.w.fire)(dataSent := true.B) + when(io.axi.ar.fire || (command.write && (addressSent || io.axi.aw.fire) && (dataSent || io.axi.w.fire))) { + when(state === issue)(state := waitResponse) + } + io.axi.r.ready := state === waitResponse && !command.write + io.axi.b.ready := state === waitResponse && command.write + when(io.axi.r.fire || io.axi.b.fire) { + when(io.axi.r.fire)(assert(io.axi.r.bits.last)) + answer.tag := command.tag + answer.data := Mux(command.write, 0.U, io.axi.r.bits.data >> (shift << 3)) + answer.error := Mux(command.write, io.axi.b.bits.resp =/= 0.U, io.axi.r.bits.resp =/= 0.U) + state := respond + } +} diff --git a/arch/src/main/scala/framework/core/memory/uncached/configs/UncachedParam.scala b/arch/src/main/scala/framework/core/memory/uncached/configs/UncachedParam.scala new file mode 100644 index 000000000..870d4eea7 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/uncached/configs/UncachedParam.scala @@ -0,0 +1,15 @@ +package framework.core.memory.uncached.configs + +import buckyball.config.UncachedConfig +import framework.ip.bus.axi4 +import upickle.default.{macroRW, ReadWriter} + +case class UncachedParam(ports: Int, axi: axi4.Params) { + require(ports > 0) +} + +object UncachedParam { + def fromProto(p: UncachedConfig): UncachedParam = + UncachedParam(p.getPorts, axi4.Params.fromProto(p.getAxi)) + implicit val rw: ReadWriter[UncachedParam] = macroRW +} diff --git a/arch/src/main/scala/framework/core/memory/uncached/configs/default.toml b/arch/src/main/scala/framework/core/memory/uncached/configs/default.toml new file mode 100644 index 000000000..90c728ed1 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/uncached/configs/default.toml @@ -0,0 +1,2 @@ +ports = 1 +axi = "default" diff --git a/arch/src/main/scala/framework/core/shm/SHM.scala b/arch/src/main/scala/framework/core/shm/SHM.scala new file mode 100644 index 000000000..33e6e6d5d --- /dev/null +++ b/arch/src/main/scala/framework/core/shm/SHM.scala @@ -0,0 +1,19 @@ +package framework.core.shm + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.core.tbus.configs.{Params => TBusParam} + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.ip.memory.mesh_shm.{MeshSharedMem, MeshSharedMemIO} + +@instantiable +class SHM(frontendParam: FrontendParam, memParam: MemDomainParam, tbusParam: TBusParam) extends Module { + val p = tbusParam.router.mesh(frontendParam, memParam, tbusParam) + @public + val io = IO(new MeshSharedMemIO(p)) + + val memory: Instance[MeshSharedMem] = Instantiate(new MeshSharedMem(p)) + io <> memory.io +} diff --git a/arch/src/main/scala/framework/core/shm/build.sc b/arch/src/main/scala/framework/core/shm/build.sc new file mode 100644 index 000000000..6ac2f75a6 --- /dev/null +++ b/arch/src/main/scala/framework/core/shm/build.sc @@ -0,0 +1,11 @@ +import mill._ + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("SHM.scala")) + ) +) + +def emitSources = T.sources( + Seq.empty[PathRef] +) diff --git a/arch/src/main/scala/framework/core/tbus/build.sc b/arch/src/main/scala/framework/core/tbus/build.sc new file mode 100644 index 000000000..0a875595e --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/build.sc @@ -0,0 +1,53 @@ +import mill._ +import $file.src.main.scala.router.{build => router_build} +import $file.^.^.^.^.^.^.settings + +val ipRoot: os.Path = millSourcePath + +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.core.tbus.Emit") +} + +// The component library owns the generated table; framework only knows the contract. +def registry(sources: Seq[PathRef], componentRoot: os.Path, destination: os.Path): Seq[PathRef] = { + val declarations = sources.flatMap { source => + val text = os.read(source.path) + val names = "(?m)^object (\\w+) extends ThreadFactory".r.findAllMatchIn(text).map(_.group(1)).toSeq + if (names.isEmpty) Seq.empty + else { + val pkg = "(?m)^package ([\\w.]+)".r.findFirstMatchIn(text).get.group(1) + val component = source.path.relativeTo(componentRoot).segments.head + names.map(name => component -> s"$pkg.$name") + } + }.sortBy(_._1) + require(declarations.map(_._1).distinct.size == declarations.size, "Duplicate component factory") + val entries = declarations.map { case (component, factory) => s""""$component" -> $factory""" } + val output = destination / "ThreadComponents.scala" + os.write.over( + output, + s"""package framework.core.tbus.generated +import framework.core.tbus.ThreadFactory +object ThreadComponents { + val factories: Map[String, ThreadFactory] = Map(${entries.mkString(",\n")}) +} +""" + ) + Seq(PathRef(output)) +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/SharedNetwork.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/TBus.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) ++ + router_build.sources() +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) ++ + router_build.emitSources() +) diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/Emit.scala b/arch/src/main/scala/framework/core/tbus/src/main/scala/Emit.scala new file mode 100644 index 000000000..51a2ed754 --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/Emit.scala @@ -0,0 +1,36 @@ +package framework.core.tbus + +import sims.configs.SimParam +import _root_.configs.GlobalConfig +import java.nio.file.{Files, Paths} +import upickle.default.read + +object Emit { + + def emit( + b: GlobalConfig, + hartId: Int, + threadId: Int, + directory: String, + firtoolOpts: Array[String] + ): Unit = { + val core = + b.device.cores.find(_.hartId == hartId).getOrElse(throw new IllegalArgumentException(s"Unknown hart ID: $hartId")) + val thread = + core.threads.find(_.id == threadId).getOrElse(throw new IllegalArgumentException(s"Unknown thread ID: $threadId")) + + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new SharedNetwork( + thread.acc.get.frontend, + thread.acc.get.memDomain, + core.tbus.get, + core.threads.map(_.id), + core.threads.map(_.id), + b.sim + ), + firtoolOpts = firtoolOpts, + args = Array("--target-dir", directory, "--split-verilog") + ) + } + +} diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/SharedNetwork.scala b/arch/src/main/scala/framework/core/tbus/src/main/scala/SharedNetwork.scala new file mode 100644 index 000000000..4603165d2 --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/SharedNetwork.scala @@ -0,0 +1,129 @@ +package framework.core.tbus + +import sims.configs.SimParam +import framework.lib.arbiter.{Arbiter} + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.core.tbus.configs.{Params => TBusParam} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.core.tbus.configs.Layout +import framework.thread.tlink.{ExternalLink, SharedLink} +import framework.core.tbus.router.Router +import framework.acc.memdomain.{LayoutConfig, MvoverCommand, PhysicalBankHash} +import framework.ip.memory.mesh_shm.MeshSharedMemIO + +/** Shared-bank interconnect: endpoint arbitration, mapping and synchronization. */ +@instantiable +class SharedNetwork( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val tbusParam: TBusParam, + val endpointIds: Seq[Int], + val traceIds: Seq[Int], + val simp: SimParam) + extends Module { + val diffTest = simp.diffTest + val p = tbusParam + val participants = p.participants + require(participants.nonEmpty && participants.distinct.size == participants.size) + require(participants.forall(slot => slot >= 0 && slot < p.slots)) + val ports = Layout.portsPerThread(memParam, tbusParam) + + @public + val io = IO(new Bundle { + val shm = Flipped(new MeshSharedMemIO(p.router.mesh(frontendParam, memParam, tbusParam))) + val threads = Vec(participants.size, Flipped(new SharedLink(frontendParam, memParam))) + val external = Option.when(p.router.physicalPorts > 0 || p.externalMove)(new ExternalLink(memParam, tbusParam)) + val bankHashes = Option.when(diffTest)(Output(Vec(Layout.totalBank(memParam), new PhysicalBankHash(frontendParam)))) + }) + + val router: Instance[Router] = + Instantiate(new Router(frontendParam, memParam, tbusParam, endpointIds, traceIds, simp.diffTest)) + val barrier: Instance[Barrier] = Instantiate(new Barrier(tbusParam)) + + router.io.memory <> io.shm + io.bankHashes.foreach(_ := router.io.bank_hashes.get) + + val endpointSlots = VecInit(participants.map(_.U(8.W))) + val endpointOwners = VecInit(participants.map(slot => endpointIds(slot).U(64.W))) + io.external.foreach { port => + router.io.physical <> port.memory + port.lease.foreach { lease => + router.io.lease.get <> lease + router.io.leaseOwnerThreadId.get := endpointOwners(lease.request.bits.endpoint) + when(lease.request.valid && !reset.asBool) { + assert(lease.request.bits.endpoint < participants.size.U, "Shared lease endpoint is outside TBus") + } + } + } + + for ((slot, endpoint) <- participants.zipWithIndex) { + val link = io.threads(endpoint) + for (port <- 0 until ports) { + router.io.mem_req(endpoint * ports + port) <> link.bank.requests(port) + } + router.io.localBanks(slot) <> link.bank.local + router.io.mapping(slot) <> link.bank.mapping + barrier.io.arrive(endpoint) := link.barrierArrive + link.barrierRelease := barrier.io.release(endpoint) + } + + for (slot <- 0 until p.slots if !participants.contains(slot)) { + val local = router.io.localBanks(slot) + local.request.ready := false.B + local.response.valid := false.B + local.response.bits := 0.U.asTypeOf(local.response.bits) + router.io.mapping(slot).valid := false.B + router.io.mapping(slot).bits := 0.U.asTypeOf(router.io.mapping(slot).bits) + } + + val moves = io.external.toSeq.flatMap(_.move.toSeq) ++ io.threads.map(_.move) + val moveArb = Instantiate(new Arbiter(new MvoverCommand, moves.size)) + val moveOwner = Reg(UInt(math.max(1, log2Ceil(moves.size)).W)) + + for ((port, endpoint) <- moves.zipWithIndex) { + moveArb.io.in(endpoint) <> port.command + port.completion.valid := router.io.mvover.completion.valid && moveOwner === endpoint.U + port.completion.bits := router.io.mvover.completion.bits + } + router.io.mvover.command <> moveArb.io.out + router.io.mvover.command.bits.sourceThread := endpointSlots(moveArb.io.out.bits.sourceThread) + router.io.mvover.command.bits.targetThread := endpointSlots(moveArb.io.out.bits.targetThread) + router.io.mvover.completion.ready := VecInit(moves.map(_.completion.ready))(moveOwner) + when(moveArb.io.out.fire) { + assert( + moveArb.io.out.bits.sourceThread < participants.size.U && + moveArb.io.out.bits.targetThread < participants.size.U, + "Bank movement endpoint is outside TBus" + ) + moveOwner := moveArb.io.chosen + } + + val layouts = Instantiate(new Arbiter(new LayoutConfig(frontendParam, memParam), participants.size)) + for (endpoint <- participants.indices) { layouts.io.in(endpoint) <> io.threads(endpoint).bank.config } + router.io.config <> layouts.io.out +} + +/** Core-local barrier; synchronization releases all participating endpoints together. */ +@instantiable +class Barrier(val tbusParam: TBusParam) extends Module { + val count = tbusParam.participants.size + + @public + val io = IO(new Bundle { + val arrive = Input(Vec(count, Bool())) + val release = Output(Vec(count, Bool())) + }) + + val arrived = RegInit(VecInit(Seq.fill(count)(false.B))) + val release = arrived.asUInt.andR + for (endpoint <- 0 until count) { + when(io.arrive(endpoint))(arrived(endpoint) := true.B) + io.release(endpoint) := release + } + when(release)(arrived.foreach(_ := false.B)) +} diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/TBus.scala b/arch/src/main/scala/framework/core/tbus/src/main/scala/TBus.scala new file mode 100644 index 000000000..1a2517406 --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/TBus.scala @@ -0,0 +1,148 @@ +package framework.core.tbus + +import sims.configs.SimParam +import framework.core.tbus.configs.Layout + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.core.configs.CoreParam +import framework.thread.tlink.TLinkIO +import framework.core.memory.access.preflight.configs.PmpParams +import framework.ip.memory.interlock.{Params => TrackingParam} +import framework.core.memory.access.mmu.configs.{Params => WalkerParam} +import framework.thread.configs.ThreadParam +import framework.ip.memory.mesh_shm.MeshSharedMemIO +import framework.acc.memdomain.MvoverPort +import framework.core.controller.admission.TaskEndpoint +import framework.thread.ant.Task +import framework.core.memory.MemoryPort +import framework.ip.memory.spm +import framework.core.tbus.SharedNetwork +import framework.device.cbus.{Control => TransferControl, C2CTransfer, SharedAxiEndpoint} +import framework.core.clink.SharedPort +import framework.ip.bus.axi4 +import framework.ip.bus.axi4.Interconnect + +@instantiable +class TBus( + corep: CoreParam, + simp: SimParam, + traceIds: Seq[Int], + generators: Seq[() => TLinkIO]) + extends Module { + val diffTest = simp.diffTest + val core = corep + require(traceIds.size == core.threads.size) + + @public + val io = IO(new Bundle { + + val control = Option.when(core.threads.nonEmpty)(MixedVec(core.threads.map(thread => + new Task(thread.ant.get, core.privileged.permission) + ))) + + val memory = Option.when(core.threads.nonEmpty)(Vec( + core.threads.size, + new MemoryPort(core.memory.dcache.tracking, core.memory.memoryAccess.walker) + )) + + val remote = Option.when(core.clink.shared)(Flipped(new TaskEndpoint)) + val move = Flipped(new MvoverPort) + val ddr = new axi4.Port(core.interconnect.bus) + val c2c = Option.when(core.clink.shared)(new SharedPort(core.c2c.axi)) + + val shm = Option.when(core.threads.nonEmpty)(Flipped(new MeshSharedMemIO(core.tbus.get.router.mesh( + core.sharedFrontend, + core.sharedMemory, + core.tbus.get + )))) + + val tss = Option.when(core.threads.nonEmpty)(Vec(core.threads.size, new spm.Port(core.tss.get.memory))) + }) + + require(generators.size == core.threads.size) + val endpoints = generators.map(_()) + + require(core.tss.get.ports == endpoints.size) + require(core.interconnect.masters == endpoints.size) + require(core.tbus.get.participants == core.threads.map(_.id) && core.tbus.get.slots == endpoints.size) + + val banks: Instance[SharedNetwork] = Instantiate(new SharedNetwork( + core.sharedFrontend, + core.sharedMemory, + core.tbus.get, + core.threads.map(_.id), + traceIds, + simp + )) + + val ddr: Instance[Interconnect] = Instantiate(new Interconnect(core.interconnect)) + io.ddr <> ddr.io.out + io.shm.get <> banks.io.shm + + if (core.clink.shared) { + require(core.tbus.get.router.physicalPorts == 3 && !core.transferLocalAccess && core.transferLeaseAccess) + val transferControl: Instance[TransferControl] = Instantiate(new TransferControl( + core.sharedMemory, + core.c2c, + core.transferLeaseAccess, + core.transferLocalAccess + )) + val transfer: Instance[C2CTransfer] = Instantiate(new C2CTransfer(core.c2c)) + val endpoint: Instance[SharedAxiEndpoint] = Instantiate(new SharedAxiEndpoint(core.c2c)) + transferControl.io.coreId := core.hartId.U + transfer.io.command <> transferControl.io.command + transferControl.io.completion <> transfer.io.completion + banks.io.external.get.memory(0) <> transfer.io.source + banks.io.external.get.memory(1) <> endpoint.io.read + banks.io.external.get.memory(2) <> endpoint.io.write + banks.io.external.get.lease.get <> transferControl.io.lease.get + io.c2c.get.tx <> transfer.io.mem + endpoint.io.mem <> io.c2c.get.rx + transferControl.io.request <> io.remote.get.request + io.remote.get.reply <> transferControl.io.reply + io.remote.get.busy := !transferControl.io.idle || endpoint.io.busy + } else { + require(core.tbus.get.router.physicalPorts == 0) + + } + require(core.tbus.get.externalMove) + banks.io.external.get.move.get <> io.move + + for ((link, index) <- endpoints.zipWithIndex) { + io.control.get(index) <> link.task + io.tss.get(index) <> link.tss + io.memory.get(index) <> link.memory + ddr.io.in(index) <> link.ddr + banks.io.threads(index) <> link.shm + link.sharedHashes.foreach(_ := banks.io.bankHashes.get) + } +} + +object TBus { + + def generators( + corep: CoreParam, + simp: SimParam, + traceIds: Seq[Int], + factories: Map[String, ThreadFactory] + ): Seq[() => TLinkIO] = { + require(corep.threads.size == traceIds.size) + corep.threads.zip(traceIds).map { case (threadp, traceId) => + () => factories(threadp.component).instantiate(corep, threadp, simp, traceId) + } + } + +} + +trait ThreadFactory { + + def instantiate( + corep: CoreParam, + threadp: ThreadParam, + simp: SimParam, + traceId: Int + ): TLinkIO + +} diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/core/tbus/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..994858c98 --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/configs/Params.scala @@ -0,0 +1,41 @@ +package framework.core.tbus.configs + +import framework.acc.memdomain.configs.MemDomainParam +import framework.core.tbus.configs.{Params => TBusParam} + +import upickle.default.{macroRW, ReadWriter} + +case class Params( + router: framework.core.tbus.router.configs.Params, + participants: Seq[Int], + slots: Int, + bankNodes: Seq[Int], + externalMove: Boolean) + +object Params { implicit val rw: ReadWriter[Params] = macroRW } + +object Layout { + + def totalBank(memParam: MemDomainParam): Int = { + require(memParam.sharedEnable) + require(memParam.sharedBankNum > 0) + memParam.sharedBankNum + } + + def portsPerThread(memParam: MemDomainParam, tbusParam: TBusParam): Int = { + if (!memParam.sharedEnable) return 0 + val threads = tbusParam.participants.size + require(threads > 0) + require(memParam.sharedInputChannels > 0) + require(memParam.sharedInputChannels % threads == 0) + val channels = memParam.sharedInputChannels / threads + require(channels > 0) + channels + } + + def totalChannel(memParam: MemDomainParam): Int = + if (memParam.sharedEnable) + memParam.sharedInputChannels + else 0 + +} diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/build.sc b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/build.sc new file mode 100644 index 000000000..11436c15a --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/build.sc @@ -0,0 +1,21 @@ +import mill._ +import $file.^.^.^.^.^.^.^.^.^.^.settings + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.core.tbus.router.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Router.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/configs/uvm.toml b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/configs/uvm.toml new file mode 100644 index 000000000..d0bfc6b0c --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/configs/uvm.toml @@ -0,0 +1,57 @@ +[parameters] +diffTest = false +endpointIds=[10,20] +traceIds=[100,101] + +[parameters.frontend] +ownerId = 7 +commandDataBits = 64 +rob_entries = 4 +bank_alias_enable = true +rs_out_of_order_response = true +bank_id_len = 4 +max_private_vbank = 3 +shared_vbank_base = 16 +iter_len = 16 +sub_rob_enable = false +sub_rob_depth = 4 + +[parameters.memory] +sharedPorts = 2 +bankNum = 4 +bankLayoutEnable = true +bankWidth = 128 +bankEntries = 8 +bankMaskLen = 16 +virtualBankCount = 4 +sharedEnable = true +sharedEntries = 32 +sharedBankEntries = 8 +sharedBankNum = 4 +sharedInputChannels = 4 +sharedDefaultGroupCount = 1 +bankRequestPorts = 2 +dma_buswidth = 128 +memAddrLen = 44 +mmioEnable = false +mmioBankNum = 0 +mmioBankEntries = 0 +mmioBankWidth = 0 +mmioReadWidth = 0 + +[parameters.gp] +enable = false +laneNumber = 1 +vLen = 128 +eLen = 64 +iBufWords = 16 +memoryPorts = 1 + +[parameters.tbus] +participants=[0,1] +slots=2 +bankNodes=[0,3] +externalMove=true +[parameters.tbus.router] +physicalPorts=1 +[defines] diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/agent/driver.svh b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/agent/driver.svh new file mode 100644 index 000000000..e24058a6c --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/agent/driver.svh @@ -0,0 +1,25 @@ +class driver extends uvm_driver #(cycle_item); + `uvm_component_utils(driver) + virtual tbus_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual tbus_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "TSS interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item req; + forever begin + seq_item_port.get_next_item(req); + @(negedge vif.clock); + vif.reset = req.reset; + vif.source = req.inputs; + @(posedge vif.clock); + req.outputs = vif.outputs; + #1; + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/agent/fixture.svh b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/agent/fixture.svh new file mode 100644 index 000000000..e54717dad --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/agent/fixture.svh @@ -0,0 +1,8 @@ +class fixture extends uvm_object; + `uvm_object_utils(fixture) + bit error[`TR_CLIENTS+`TR_EXTERNAL]; + bit move_error; + function new(string name = "fixture"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/agent/item.svh b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/agent/item.svh new file mode 100644 index 000000000..5b652e1be --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/agent/item.svh @@ -0,0 +1,23 @@ +class cycle_item extends uvm_sequence_item; + `uvm_object_utils(cycle_item) + logic [`TR_INPUT_BITS-1:0] inputs; + logic [`TR_OUTPUT_BITS-1:0] outputs; + bit reset; + function new(string name = "cycle_item"); + super.new(name); + endfunction +endclass +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(data, UVM_ALL_ON) + `uvm_field_int(error, UVM_ALL_ON) + `uvm_field_int(kind, UVM_ALL_ON) + `uvm_field_int(port, UVM_ALL_ON) + `uvm_object_utils_end + logic [127:0] data; + bit error; + int kind, port; + function new(string name = "result"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/agent/monitor.svh b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/agent/monitor.svh new file mode 100644 index 000000000..17e208527 --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/agent/monitor.svh @@ -0,0 +1,25 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual tbus_if vif; + uvm_analysis_port #(cycle_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual tbus_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "TSS monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item t; + forever begin + @(posedge vif.clock); + t = cycle_item::type_id::create("observed"); + t.inputs = vif.inputs; + t.outputs = vif.outputs; + t.reset = vif.reset; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/agent/peer.svh b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/agent/peer.svh new file mode 100644 index 000000000..bf82043ea --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/agent/peer.svh @@ -0,0 +1,70 @@ +class peer extends uvm_component; + `uvm_component_utils(peer) + virtual tbus_if vif; + fixture image; + const int offsets[`TR_CLIENTS+`TR_EXTERNAL][15] = `TR_MESH_OFFSETS; + bit [127:0] memory[`TR_BANKS][`TR_ROWS]; + bit [15:0] initialized[`TR_BANKS][`TR_ROWS]; + bit + pending[`TR_CLIENTS+`TR_EXTERNAL], + writing[`TR_CLIENTS+`TR_EXTERNAL], + error[`TR_CLIENTS+`TR_EXTERNAL]; + bit [127:0] response[`TR_CLIENTS+`TR_EXTERNAL]; + int tag[`TR_CLIENTS+`TR_EXTERNAL], due[`TR_CLIENTS+`TR_EXTERNAL], cycle; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual tbus_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "TBus interface is required") + endfunction + task run_phase(uvm_phase phase); + forever begin + @(posedge vif.clock); + cycle++; + for (int c = 0; c < `TR_CLIENTS + `TR_EXTERNAL; c++) begin + if (vif.reset) pending[c] = 0; + else begin + if (pending[c] && vif.inputs[offsets[c][8]] && vif.outputs[offsets[c][9]]) pending[c] = 0; + if (vif.outputs[offsets[c][0]] && vif.inputs[offsets[c][1]]) begin + int bank_id = int'(vif.outputs[offsets[c][2]+:$clog2(`TR_BANKS)]); + int row = int'(vif.outputs[offsets[c][3]+1+:16]); + bit write_request = vif.outputs[offsets[c][3]]; + bit [15:0] mask = vif.outputs[offsets[c][5]+:16]; + if (pending[c] || bank_id >= `TR_BANKS || row >= `TR_ROWS) + `uvm_fatal("BACKEND", "Invalid or overlapping Mesh request") + pending[c] = 1; + writing[c] = write_request; + error[c] = image.error[c]; + tag[c] = int'(vif.outputs[offsets[c][6]+:`TR_TAG_BITS]); + due[c] = cycle + 2 + c % 4; + response[c] = 0; + if (write_request && !error[c]) begin + for (int b = 0; b < 16; b++) + if (mask[b]) begin + memory[bank_id][row][b*8+:8] = vif.outputs[offsets[c][4]+b*8+:8]; + initialized[bank_id][row][b] = 1; + end + end else if (!write_request) begin + if (initialized[bank_id][row] != '1) + `uvm_fatal("MEMORY", "Uninitialized Mesh bank read") + response[c] = memory[bank_id][row]; + end + end + end + end + @(negedge vif.clock); + vif.backend = '0; + for (int c = 0; c < `TR_CLIENTS + `TR_EXTERNAL; c++) begin + vif.backend[offsets[c][1]] = !pending[c] && cycle % 5 != c % 5; + vif.backend[offsets[c][8]] = pending[c] && cycle >= due[c]; + vif.backend[offsets[c][10]+:128] = response[c]; + vif.backend[offsets[c][11]+:16] = '1; + vif.backend[offsets[c][12]+:`TR_TAG_BITS] = tag[c]; + vif.backend[offsets[c][13]+:2] = {writing[c], error[c]}; + vif.backend[offsets[c][14]] = 1; + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/env/env.svh b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/env/env.svh new file mode 100644 index 000000000..eeda0417e --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/env/env.svh @@ -0,0 +1,36 @@ +class env extends uvm_env; + `uvm_component_utils(env) + uvm_analysis_imp #(uvm_sequence_item, env) reset_events; + driver source; + monitor mon; + peer backend; + model reference; + uvm_sequencer #(cycle_item) seqr; + result_board #(result_item, 1) scoreboard; + function new(string name, uvm_component parent); + super.new(name, parent); + reset_events = new("reset_events", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + source = driver::type_id::create("source", this); + mon = monitor::type_id::create("monitor", this); + backend = peer::type_id::create("backend", this); + reference = model::type_id::create("model", this); + seqr = new("seqr", this); + scoreboard = result_board#(result_item, 1)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + fixture image = fixture::type_id::create("image"); + reference.image = image; + backend.image = image; + source.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(reference.analysis_export); + reference.expected.connect(scoreboard.expected_export); + reference.actual.connect(scoreboard.actual_export); + reference.cancelled.connect(reset_events); + endfunction + function void write(uvm_sequence_item t); + scoreboard.cancel_pending(); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/env/model.svh b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/env/model.svh new file mode 100644 index 000000000..96650b9d3 --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/env/model.svh @@ -0,0 +1,379 @@ +class model extends uvm_subscriber #(cycle_item); + `uvm_component_utils(model) + localparam int BANKS = `TR_BANKS; + fixture image; + uvm_analysis_port #(result_item) expected, actual; + uvm_analysis_port #(uvm_sequence_item) cancelled; + const int clients[`TR_CLIENTS][17] = `TR_CLIENT_OFFSETS; + const int mesh[`TR_CLIENTS+`TR_EXTERNAL][15] = `TR_MESH_OFFSETS; + const int mappings[`TR_THREADS][10] = `TR_MAPPING_OFFSETS; + const int owners[`TR_THREADS] = `TR_OWNERS; + const int copies[`TR_LOCAL_FIELDS][4] = `TR_LOCAL_COPIES; + bit live[BANKS]; + int owner[BANKS], architecture[BANKS], alias_id[BANKS], group_id[BANKS], leases[BANKS]; + int layouts[`TR_THREADS][`TR_ALIASES], committed[`TR_THREADS][`TR_ALIASES]; + logic [127:0] memory[BANKS][`TR_ROWS]; + bit [15:0] initialized[BANKS][`TR_ROWS]; + bit pending[`TR_CLIENTS+`TR_EXTERNAL], writing[`TR_CLIENTS+`TR_EXTERNAL]; + int + selected[`TR_CLIENTS+`TR_EXTERNAL], + rows[`TR_CLIENTS+`TR_EXTERNAL], + tags[`TR_CLIENTS+`TR_EXTERNAL]; + bit accepted_now[`TR_CLIENTS+`TR_EXTERNAL]; + logic [127:0] data_now[`TR_CLIENTS+`TR_EXTERNAL]; + bit [15:0] masks_now[`TR_CLIENTS+`TR_EXTERNAL]; + bit lease_pending, move_pending; + covergroup behavior with function sample ( + int port, bit writing, int owner, int event_kind, bit error, bit blocked + ); + option.per_instance = 1; + ports: coverpoint port {bins all[] = {[0 : `TR_CLIENTS + `TR_EXTERNAL - 1]};} + direction: coverpoint writing; + tenants: coverpoint owner {bins all[] = {[0 : `TR_THREADS - 1]};} + events: coverpoint event_kind { + bins access = {0}; + bins configuration = {1}; + bins renamed = {2}; + bins retirement = {3}; + bins export_lease = {4}; + bins release_lease = {5}; + bins move = {6}; + bins transfer = {7}; + } + errors: coverpoint error; + backpressure: coverpoint blocked; + endgroup + covergroup epochs with function sample (bit pending); + option.per_instance = 1; + cancellation: coverpoint pending; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected = new("expected", this); + actual = new("actual", this); + cancelled = new("cancelled", this); + behavior = new(); + epochs = new(); + endfunction + function void verify(bit ok, string reason); + evidence::checks++; + if (!ok) `uvm_fatal("TBUS_ROUTER", reason) + endfunction + function int tenant(longint unsigned id); + foreach (owners[t]) if (owners[t] == id) return t; + `uvm_fatal("OWNER", "Unknown configured owner") + return -1; + endfunction + function int route(int thread, int tag, int group_index); + int found = -1; + foreach (live[b]) + if (live[b] && owner[b] == thread && alias_id[b] == tag && group_id[b] == group_index) begin + verify(found == -1, "Ambiguous shared binding"); + found = b; + end + verify(found >= 0, "Missing shared binding"); + return found; + endfunction + function void request(int port, int bank_id, int row, bit wr, logic [127:0] data, + bit [15:0] mask); + result_item result = result_item::type_id::create("expected"); + verify(!pending[port] && row < `TR_ROWS, "Overlapping or out-of-range shared access"); + result.port = port; + result.kind = port < `TR_CLIENTS ? 0 : 1; + result.error = image.error[port]; + result.set_transaction_id(port); + selected[port] = bank_id; + rows[port] = row; + data_now[port] = data; + masks_now[port] = mask; + writing[port] = wr; + accepted_now[port] = 1; + pending[port] = 1; + if (wr) begin + if (!image.error[port]) + for (int b = 0; b < 16; b++) + if (mask[b]) begin + memory[bank_id][row][b*8+:8] = data[b*8+:8]; + initialized[bank_id][row][b] = 1; + end + result.data = 0; + end else begin + verify(initialized[bank_id][row] == '1, "Read of uninitialized shared memory"); + result.data = memory[bank_id][row]; + end + expected.write(result); + behavior.sample(port, wr, owner[bank_id], 0, image.error[port], 0); + endfunction + function void write(cycle_item t); + if (t.reset) begin + bit unfinished = lease_pending || move_pending; + foreach (pending[c]) unfinished |= pending[c]; + epochs.sample(unfinished); + cancelled.write(null); + foreach (live[b]) begin + live[b] = 0; + leases[b] = 0; + end + foreach (layouts[o, a]) begin + layouts[o][a] = 0; + committed[o][a] = a; + end + foreach (pending[c]) begin + pending[c] = 0; + tags[c] = 0; + end + lease_pending = 0; + move_pending = 0; + return; + end + foreach (accepted_now[c]) accepted_now[c] = 0; + for (int c = 0; c < `TR_CLIENTS; c++) begin + bit rd = t.inputs[clients[c][0]] && t.outputs[clients[c][1]]; + bit wr = t.inputs[clients[c][6]] && t.outputs[clients[c][7]]; + verify(!(rd && wr), "Client accepted both read and write"); + if (rd || wr) begin + int thread = tenant(t.inputs[clients[c][16]+:64]); + int alias_value = int'(t.inputs[clients[c][14]+:`TR_IN_MEM_REQ_0_BANK_ID_WIDTH]); + int group_index = int'(t.inputs[clients[c][15]+:`TR_IN_MEM_REQ_0_GROUP_ID_WIDTH]); + request(c, route(thread, alias_value, group_index), + int'(t.inputs[clients[c][wr?8 : 2]+:16]), wr, t.inputs[clients[c][9]+:128], + t.inputs[clients[c][10]+:16]); + end + if(t.outputs[clients[c][3]]&&t.inputs[clients[c][4]]||t.outputs[clients[c][11]]&&t.inputs[clients[c][12]])begin + result_item result = result_item::type_id::create("actual"); + verify(pending[c], "Client response has no owner"); + result.port = c; + result.kind = 0; + result.data = writing[c] ? 128'd0 : t.outputs[clients[c][5]+:128]; + result.error = writing[c] ? !t.outputs[clients[c][13]] : 0; + result.set_transaction_id(c); + actual.write(result); + pending[c] = 0; + end + if(t.outputs[clients[c][3]]&&!t.inputs[clients[c][4]]||t.outputs[clients[c][11]]&&!t.inputs[clients[c][12]]) + behavior.sample(c, writing[c], 0, 0, 0, 1); + end + if (`RI(t, PHYSICAL_0_REQUEST_VALID) && `RO(t, PHYSICAL_0_REQUEST_READY)) begin + longint unsigned address = `RI(t, PHYSICAL_0_REQUEST_BITS_ADDRESS); + int bank_id = int'(address / (`TR_ROWS * 16)); + int row = int'(address % (16 * `TR_ROWS)) / 16; + verify(address % 16 == 0 && bank_id < BANKS && live[bank_id], + "Invalid physical shared access"); + request(`TR_CLIENTS, bank_id, row, `RI(t, PHYSICAL_0_REQUEST_BITS_WRITE), + `RI(t, PHYSICAL_0_REQUEST_BITS_DATA), `RI(t, PHYSICAL_0_REQUEST_BITS_MASK)); + end + if (`RO(t, PHYSICAL_0_RESPONSE_VALID) && `RI(t, PHYSICAL_0_RESPONSE_READY)) begin + result_item result = result_item::type_id::create("physical_actual"); + verify(pending[`TR_CLIENTS], "Physical response has no owner"); + result.port = `TR_CLIENTS; + result.kind = 1; + result.data = `RO(t, PHYSICAL_0_RESPONSE_BITS_DATA); + result.error = `RO(t, PHYSICAL_0_RESPONSE_BITS_ERROR); + result.set_transaction_id(`TR_CLIENTS); + actual.write(result); + pending[`TR_CLIENTS] = 0; + end + for (int c = 0; c < `TR_CLIENTS + `TR_EXTERNAL; c++) begin + if (t.outputs[mesh[c][0]] && t.inputs[mesh[c][1]]) begin + verify(accepted_now[c], "Mesh request has no accepted client"); + verify(t.outputs[mesh[c][2]+:$clog2(BANKS + )] == selected[c] && t.outputs[mesh[c][3]+:17] == ((rows[c] << 1) | writing[c]), + "Shared bank, row or direction routing mismatch"); + verify(t.outputs[mesh[c][7]], "Shared row request did not terminate its packet"); + if (c < `TR_CLIENTS) begin + verify(t.outputs[mesh[c][6]+:`TR_TAG_BITS] == tags[c], + "Shared channel tag did not advance"); + tags[c] = (tags[c] + 1) % (1 << `TR_TAG_BITS); + end else verify(t.outputs[mesh[c][6]+:`TR_TAG_BITS] == 0, "Physical port tag must be zero"); + if (writing[c]) + verify( + t.outputs[mesh[c][4]+:128]===data_now[c]&&t.outputs[mesh[c][5]+:16]==masks_now[c], + "Shared write data or mask changed"); + accepted_now[c] = 0; + end + verify(!accepted_now[c], "Client accepted without forwarding its Mesh request"); + end + for (int thread = 0; thread < `TR_THREADS; thread++) begin + if (t.inputs[mappings[thread][0]]) begin + int previous = int'(t.inputs[mappings[thread][4]+:`TR_IN_MAPPING_0_BITS_PREVIOUS_WIDTH]); + bit held; + held = 0; + foreach (live[b]) + if (live[b] && owner[b] == thread && alias_id[b] == previous && leases[b] != 0) held = 1; + if (held && (t.inputs[mappings[thread][7]] || t.inputs[mappings[thread][8]])) begin + verify(!t.outputs[mappings[thread][1]], "Mapping reclaimed a live export"); + behavior.sample(0, 0, thread, 3, 0, 1); + end + end + if (t.inputs[mappings[thread][0]] && t.outputs[mappings[thread][1]]) begin + int bank_id = int'(t.inputs[mappings[thread][2]+:`TR_IN_MAPPING_0_BITS_BANK_WIDTH]); + int tag = int'(t.inputs[mappings[thread][3]+:`TR_IN_MAPPING_0_BITS_VBANK_WIDTH]); + int previous = int'(t.inputs[mappings[thread][4]+:`TR_IN_MAPPING_0_BITS_PREVIOUS_WIDTH]); + verify(t.inputs[mappings[thread][5]+:64] == owners[thread], + "Mapping port changed its owner"); + if (t.inputs[mappings[thread][6]]) begin + bit free_bank[BANKS]; + foreach (live[b]) + free_bank[b]=!live[b]||(t.inputs[mappings[thread][7]]&&owner[b]==thread&&alias_id[b]==previous); + for (int group_index = 0; group_index < layouts[thread][bank_id]; group_index++) begin + int chosen = -1; + foreach (free_bank[b]) if (free_bank[b] && chosen < 0) chosen = b; + verify(chosen >= 0, "Alias allocation exceeded capacity"); + free_bank[chosen] = 0; + verify(leases[chosen] == 0, "Replaced a leased shared bank"); + live[chosen] = 1; + owner[chosen] = thread; + architecture[chosen] = bank_id; + alias_id[chosen] = tag; + group_id[chosen] = group_index; + end + behavior.sample(0, 0, thread, 2, 0, 0); + end + if (t.inputs[mappings[thread][8]]) begin + committed[thread][bank_id] = tag; + foreach (live[b]) + if (live[b] && owner[b] == thread && alias_id[b] == previous) live[b] = 0; + behavior.sample(0, 0, thread, 3, 0, 0); + end + end + end + if (`RI(t, CONFIG_VALID) && `RO(t, CONFIG_READY)) begin + int thread = tenant(`RI(t, CONFIG_BITS_THREAD_ID)); + int bank_id = `RI(t, CONFIG_BITS_VBANK_ID); + int group_index = `RI(t, CONFIG_BITS_GROUP_ID); + if (`RI(t, CONFIG_BITS_TRANSFER)) begin + int source = `RI(t, CONFIG_BITS_SOURCE_BANK_ID); + int prior = layouts[thread][bank_id]; + foreach (live[b]) + if (live[b] && owner[b] == thread && architecture[b] == source) begin + verify(leases[b] == 0, "Transferred a leased bank"); + alias_id[b] = bank_id; + architecture[b] = bank_id; + group_id[b] += prior; + end + layouts[thread][bank_id] += layouts[thread][source]; + layouts[thread][source] = 0; + behavior.sample(0, 0, thread, 7, 0, 0); + end else begin + if (group_index == 0 || !`RI(t, CONFIG_BITS_ALLOC)) + foreach (live[b]) + if (live[b] && owner[b] == thread && architecture[b] == bank_id) begin + verify(leases[b] == 0, "Changed a leased bank mapping"); + live[b] = 0; + end + if (`RI(t, CONFIG_BITS_ALLOC)) begin + int chosen = -1; + foreach (live[b]) if (!live[b] && chosen < 0) chosen = b; + verify(chosen >= 0, "Configuration exceeded shared capacity"); + live[chosen] = 1; + owner[chosen] = thread; + architecture[chosen] = bank_id; + alias_id[chosen] = bank_id; + group_id[chosen] = group_index; + layouts[thread][bank_id] = group_index + 1; + end else layouts[thread][bank_id] = 0; + behavior.sample(0, 0, thread, 1, 0, 0); + end + end + if (`RI(t, LEASE_REQUEST_VALID) && `RO(t, LEASE_REQUEST_READY)) begin + int thread = tenant(`RI(t, LEASEOWNERTHREADID)); + int bank_id = `RI(t, LEASE_REQUEST_BITS_VBANK); + int group_index = `RI(t, LEASE_REQUEST_BITS_GROUP); + int chosen = -1; + bit release_lease = `RI(t, LEASE_REQUEST_BITS_RELEASE); + result_item result = result_item::type_id::create("lease_expected"); + verify(!lease_pending, "Overlapping lease requests"); + foreach (live[b]) + if(live[b]&&owner[b]==thread&&architecture[b]==bank_id&&group_id[b]==group_index&&(release_lease?leases[b]!=0:alias_id[b]==committed[thread][bank_id]))begin + verify(chosen < 0, "Ambiguous lease binding"); + chosen = b; + end + verify(chosen >= 0, "Lease has no current shared binding"); + if (release_lease) begin + leases[chosen]--; + result.data = 0; + end else begin + leases[chosen]++; + result.data = chosen * `TR_ROWS * 16; + end + result.port = -1; + result.kind = 2; + result.set_transaction_id(100); + expected.write(result); + lease_pending = 1; + behavior.sample(0, 0, thread, release_lease ? 5 : 4, 0, 0); + end + if (`RO(t, LEASE_RESPONSE_VALID) && `RI(t, LEASE_RESPONSE_READY)) begin + result_item result = result_item::type_id::create("lease_actual"); + verify(lease_pending, "Lease response without request"); + result.port = -1; + result.kind = 2; + result.data = `RO(t, LEASE_RESPONSE_BITS); + result.set_transaction_id(100); + actual.write(result); + lease_pending = 0; + end + if (`RI(t, MVOVER_COMMAND_VALID) && `RO(t, MVOVER_COMMAND_READY)) begin + result_item result = result_item::type_id::create("move_expected"); + verify(!move_pending, "Overlapping move requests"); + verify(`RO(t, MEMORY_TRANSFERCOMMAND_VALID) && `RO(t, MEMORY_TRANSFERCOMMAND_BITS_TAG) == 0, + "Move was not forwarded with its reserved tag"); + verify(`RI(t, MVOVER_COMMAND_BITS_SOURCETHREAD) == + `RO(t, MEMORY_TRANSFERCOMMAND_BITS_SOURCETHREAD), + "Move command field SOURCETHREAD changed"); + verify(`RI(t, MVOVER_COMMAND_BITS_TARGETTHREAD) == + `RO(t, MEMORY_TRANSFERCOMMAND_BITS_TARGETTHREAD), + "Move command field TARGETTHREAD changed"); + verify(`RI(t, MVOVER_COMMAND_BITS_SOURCEBANK) == + `RO(t, MEMORY_TRANSFERCOMMAND_BITS_SOURCEBANK), + "Move command field SOURCEBANK changed"); + verify(`RI(t, MVOVER_COMMAND_BITS_TARGETBANK) == + `RO(t, MEMORY_TRANSFERCOMMAND_BITS_TARGETBANK), + "Move command field TARGETBANK changed"); + verify(`RI(t, MVOVER_COMMAND_BITS_SOURCEADDR) == + `RO(t, MEMORY_TRANSFERCOMMAND_BITS_SOURCEADDR), + "Move command field SOURCEADDR changed"); + verify(`RI(t, MVOVER_COMMAND_BITS_TARGETADDR) == + `RO(t, MEMORY_TRANSFERCOMMAND_BITS_TARGETADDR), + "Move command field TARGETADDR changed"); + verify(`RI(t, MVOVER_COMMAND_BITS_ROWS) == `RO(t, MEMORY_TRANSFERCOMMAND_BITS_ROWS), + "Move command field ROWS changed"); + + result.port = -2; + result.kind = 3; + result.data = 0; + result.error = image.move_error; + result.set_transaction_id(101); + expected.write(result); + move_pending = 1; + behavior.sample(0, 0, 0, 6, image.move_error, 0); + end + verify(`RO(t, MVOVER_COMMAND_READY) == `RI(t, MEMORY_TRANSFERCOMMAND_READY), + "Move readiness changed"); + verify(`RO(t, MVOVER_COMPLETION_VALID) == `RI(t, MEMORY_TRANSFERCOMPLETION_VALID) && + `RO(t, MEMORY_TRANSFERCOMPLETION_READY) == `RI(t, MVOVER_COMPLETION_READY), + "Move completion handshake changed"); + if (`RO(t, MVOVER_COMPLETION_VALID) && `RI(t, MVOVER_COMPLETION_READY)) begin + result_item result = result_item::type_id::create("move_actual"); + verify(move_pending, "Unsolicited move completion"); + result.port = -2; + result.kind = 3; + result.data = 0; + result.error = `RO(t, MVOVER_COMPLETION_BITS); + result.set_transaction_id(101); + actual.write(result); + move_pending = 0; + end + foreach (copies[f]) + if (copies[f][3] < 0 || t.inputs[copies[f][3]]) + for (int b = 0; b < copies[f][2]; b++) + verify(t.outputs[copies[f][1]+b] === t.inputs[copies[f][0]+b], + "Thread-local bank passthrough changed"); + endfunction + function void check_phase(uvm_phase phase); + super.check_phase(phase); + foreach (pending[c]) verify(!pending[c], "Shared access did not drain"); + verify(!move_pending && !lease_pending, "Control transaction did not drain"); + foreach (leases[b]) verify(leases[b] == 0, "Outstanding shared export"); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/filelist.f b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/filelist.f new file mode 100644 index 000000000..4c29c10a6 --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/filelist.f @@ -0,0 +1,9 @@ ++define+BUCKYBALL_DISABLE_MTRACE_DPI ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/Router/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/interface.sv b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/interface.sv new file mode 100644 index 000000000..656dc02b5 --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/interface.sv @@ -0,0 +1,9 @@ +`include "parameters.svh" +interface tbus_if ( + input logic clock +); + logic reset = 1; + logic [`TR_INPUT_BITS-1:0] source = '0, backend = '0; + wire [ `TR_INPUT_BITS-1:0] inputs = source | backend; + wire [`TR_OUTPUT_BITS-1:0] outputs; +endinterface diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/package.sv b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/package.sv new file mode 100644 index 000000000..98fb5ea9e --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/package.sv @@ -0,0 +1,18 @@ +`include "parameters.svh" +`define RI(T, F) T.inputs[`TR_IN_``F``_OFFSET+:`TR_IN_``F``_WIDTH] +`define RO(T, F) T.outputs[`TR_OUT_``F``_OFFSET+:`TR_OUT_``F``_WIDTH] +package tbus_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + `include "agent/item.svh" + `include "agent/fixture.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/peer.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" + `include "tests/invalid.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..414a104be --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/seq/sequence.svh @@ -0,0 +1,281 @@ +class scenario_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(scenario_sequence) + model reference; + const int clients[`TR_CLIENTS][17] = `TR_CLIENT_OFFSETS; + const int mappings[`TR_THREADS][10] = `TR_MAPPING_OFFSETS; + const int owners[`TR_THREADS] = `TR_OWNERS; + const int copies[`TR_LOCAL_FIELDS][4] = `TR_LOCAL_COPIES; + logic [`TR_INPUT_BITS-1:0] pins; + logic [`TR_OUTPUT_BITS-1:0] observed; + int mapping_fired[`TR_THREADS]; + function new(string name = "scenario"); + super.new(name); + endfunction + task tick(bit reset = 0); + cycle_item t = cycle_item::type_id::create("cycle"); + start_item(t); + t.inputs = pins; + t.reset = reset; + finish_item(t); + observed = t.outputs; + for (int thread = 0; thread < `TR_THREADS; thread++) + if (pins[mappings[thread][0]] && observed[mappings[thread][1]]) begin + pins[mappings[thread][0]] = 0; + mapping_fired[thread]++; + end + endtask + task layout(int thread, int bank_id, int groups, bit transfer = 0, int source = 0); + for (int group_index = 0; group_index < (groups == 0 ? 1 : groups); group_index++) begin + pins[`TR_IN_CONFIG_VALID_OFFSET+:`TR_IN_CONFIG_VALID_WIDTH] = 1; + pins[`TR_IN_CONFIG_BITS_THREAD_ID_OFFSET+:`TR_IN_CONFIG_BITS_THREAD_ID_WIDTH]=owners[thread]; + pins[`TR_IN_CONFIG_BITS_VBANK_ID_OFFSET+:`TR_IN_CONFIG_BITS_VBANK_ID_WIDTH] = bank_id; + pins[`TR_IN_CONFIG_BITS_GROUP_ID_OFFSET+:`TR_IN_CONFIG_BITS_GROUP_ID_WIDTH] = group_index; + pins[`TR_IN_CONFIG_BITS_ALLOC_OFFSET+:`TR_IN_CONFIG_BITS_ALLOC_WIDTH] = groups > 0; + pins[`TR_IN_CONFIG_BITS_IS_MULTI_OFFSET+:`TR_IN_CONFIG_BITS_IS_MULTI_WIDTH] = groups > 1; + pins[`TR_IN_CONFIG_BITS_TRANSFER_OFFSET+:`TR_IN_CONFIG_BITS_TRANSFER_WIDTH] = transfer; + pins[`TR_IN_CONFIG_BITS_SOURCE_BANK_ID_OFFSET+:`TR_IN_CONFIG_BITS_SOURCE_BANK_ID_WIDTH]=source; + pins[`TR_IN_CONFIG_BITS_IS_SHARED_OFFSET+:`TR_IN_CONFIG_BITS_IS_SHARED_WIDTH] = 1; + do tick(); while (!observed[`TR_OUT_CONFIG_READY_OFFSET]); + pins[`TR_IN_CONFIG_VALID_OFFSET+:`TR_IN_CONFIG_VALID_WIDTH] = 0; + if (transfer) break; + end + repeat (2) tick(); + endtask + task map_offer(int thread, int bank_id, int tag, int previous, bit allocate, bit retire, + bit reuse = 0); + pins[mappings[thread][0]] = 1; + pins[mappings[thread][2]+:`TR_IN_MAPPING_0_BITS_BANK_WIDTH] = bank_id; + pins[mappings[thread][3]+:`TR_IN_MAPPING_0_BITS_VBANK_WIDTH] = tag; + pins[mappings[thread][4]+:`TR_IN_MAPPING_0_BITS_PREVIOUS_WIDTH] = previous; + pins[mappings[thread][5]+:64] = owners[thread]; + pins[mappings[thread][6]] = allocate; + pins[mappings[thread][7]] = reuse; + pins[mappings[thread][8]] = retire; + pins[mappings[thread][9]] = allocate; + endtask + task mapping(int thread, int bank_id, int tag, int previous, bit allocate, bit retire, + bit reuse = 0); + map_offer(thread, bank_id, tag, previous, allocate, retire, reuse); + do tick(); while (pins[mappings[thread][0]]); + repeat (2) tick(); + endtask + task access (int port, int bank_id, int group_index, int row, bit write_request, + logic [127:0] data = 0, bit [15:0] mask = '1, int stall = 2); + pins[clients[port][14]+:`TR_IN_MEM_REQ_0_BANK_ID_WIDTH] = bank_id; + pins[clients[port][15]+:`TR_IN_MEM_REQ_0_GROUP_ID_WIDTH] = group_index; + pins[clients[port][16]+:64] = owners[port/(`TR_CLIENTS/`TR_THREADS)]; + pins[clients[port][write_request?6 : 0]] = 1; + pins[clients[port][write_request?8 : 2]+:16] = row; + pins[clients[port][9]+:128] = data; + pins[clients[port][10]+:16] = mask; + pins[clients[port][write_request?12 : 4]] = 0; + do tick(); while (!observed[clients[port][write_request?7 : 1]]); + pins[clients[port][write_request?6 : 0]] = 0; + // Ownership follows the accepted command while live inputs move to a different bank. + pins[clients[port][14]+:`TR_IN_MEM_REQ_0_BANK_ID_WIDTH] = (bank_id + 1) % `TR_ALIASES; + do tick(); while (!observed[clients[port][write_request?11 : 3]]); + repeat (stall) tick(); + pins[clients[port][write_request?12 : 4]] = 1; + tick(); + endtask + task lease(int thread, int bank_id, int group_index, bit release_lease, + output longint unsigned address); + pins[`TR_IN_LEASEOWNERTHREADID_OFFSET+:`TR_IN_LEASEOWNERTHREADID_WIDTH] = owners[thread]; + pins[`TR_IN_LEASE_REQUEST_BITS_ENDPOINT_OFFSET+:`TR_IN_LEASE_REQUEST_BITS_ENDPOINT_WIDTH]=thread; + pins[`TR_IN_LEASE_REQUEST_BITS_VBANK_OFFSET+:`TR_IN_LEASE_REQUEST_BITS_VBANK_WIDTH] = bank_id; + pins[`TR_IN_LEASE_REQUEST_BITS_GROUP_OFFSET+:`TR_IN_LEASE_REQUEST_BITS_GROUP_WIDTH]=group_index; + pins[`TR_IN_LEASE_REQUEST_BITS_RELEASE_OFFSET+:`TR_IN_LEASE_REQUEST_BITS_RELEASE_WIDTH]=release_lease; + pins[`TR_IN_LEASE_REQUEST_VALID_OFFSET+:`TR_IN_LEASE_REQUEST_VALID_WIDTH] = 1; + pins[`TR_IN_LEASE_RESPONSE_READY_OFFSET+:`TR_IN_LEASE_RESPONSE_READY_WIDTH] = 0; + do tick(); while (!observed[`TR_OUT_LEASE_REQUEST_READY_OFFSET]); + pins[`TR_IN_LEASE_REQUEST_VALID_OFFSET+:`TR_IN_LEASE_REQUEST_VALID_WIDTH] = 0; + do tick(); while (!observed[`TR_OUT_LEASE_RESPONSE_VALID_OFFSET]); + address = observed[`TR_OUT_LEASE_RESPONSE_BITS_OFFSET+:64]; + repeat (3) tick(); + pins[`TR_IN_LEASE_RESPONSE_READY_OFFSET+:`TR_IN_LEASE_RESPONSE_READY_WIDTH] = 1; + tick(); + endtask + task physical(longint unsigned address, bit writing, logic [127:0] data = 0, + bit [15:0] mask = '1); + pins[`TR_IN_PHYSICAL_0_REQUEST_BITS_ADDRESS_OFFSET+:`TR_IN_PHYSICAL_0_REQUEST_BITS_ADDRESS_WIDTH]=address; + pins[`TR_IN_PHYSICAL_0_REQUEST_BITS_WRITE_OFFSET+:`TR_IN_PHYSICAL_0_REQUEST_BITS_WRITE_WIDTH]=writing; + pins[`TR_IN_PHYSICAL_0_REQUEST_BITS_DATA_OFFSET+:`TR_IN_PHYSICAL_0_REQUEST_BITS_DATA_WIDTH]=data; + pins[`TR_IN_PHYSICAL_0_REQUEST_BITS_MASK_OFFSET+:`TR_IN_PHYSICAL_0_REQUEST_BITS_MASK_WIDTH]=mask; + pins[`TR_IN_PHYSICAL_0_REQUEST_VALID_OFFSET+:`TR_IN_PHYSICAL_0_REQUEST_VALID_WIDTH] = 1; + pins[`TR_IN_PHYSICAL_0_RESPONSE_READY_OFFSET+:`TR_IN_PHYSICAL_0_RESPONSE_READY_WIDTH] = 0; + do tick(); while (!observed[`TR_OUT_PHYSICAL_0_REQUEST_READY_OFFSET]); + pins[`TR_IN_PHYSICAL_0_REQUEST_VALID_OFFSET+:`TR_IN_PHYSICAL_0_REQUEST_VALID_WIDTH] = 0; + do tick(); while (!observed[`TR_OUT_PHYSICAL_0_RESPONSE_VALID_OFFSET]); + repeat (4) tick(); + pins[`TR_IN_PHYSICAL_0_RESPONSE_READY_OFFSET+:`TR_IN_PHYSICAL_0_RESPONSE_READY_WIDTH] = 1; + tick(); + endtask + task move(int serial); + reference.image.move_error = serial % 2; + pins[`TR_IN_MVOVER_COMMAND_BITS_SOURCETHREAD_OFFSET+:`TR_IN_MVOVER_COMMAND_BITS_SOURCETHREAD_WIDTH]=serial; + pins[`TR_IN_MVOVER_COMMAND_BITS_TARGETTHREAD_OFFSET+:`TR_IN_MVOVER_COMMAND_BITS_TARGETTHREAD_WIDTH]=255-serial; + pins[`TR_IN_MVOVER_COMMAND_BITS_SOURCEBANK_OFFSET+:`TR_IN_MVOVER_COMMAND_BITS_SOURCEBANK_WIDTH]=serial*7; + pins[`TR_IN_MVOVER_COMMAND_BITS_TARGETBANK_OFFSET+:`TR_IN_MVOVER_COMMAND_BITS_TARGETBANK_WIDTH]=1023-serial; + pins[`TR_IN_MVOVER_COMMAND_BITS_SOURCEADDR_OFFSET+:`TR_IN_MVOVER_COMMAND_BITS_SOURCEADDR_WIDTH]=serial*131; + pins[`TR_IN_MVOVER_COMMAND_BITS_TARGETADDR_OFFSET+:`TR_IN_MVOVER_COMMAND_BITS_TARGETADDR_WIDTH]=65535-serial; + pins[`TR_IN_MVOVER_COMMAND_BITS_ROWS_OFFSET+:`TR_IN_MVOVER_COMMAND_BITS_ROWS_WIDTH]=serial==15?65536:serial+1; + pins[`TR_IN_MVOVER_COMMAND_VALID_OFFSET+:`TR_IN_MVOVER_COMMAND_VALID_WIDTH] = 1; + pins[`TR_IN_MEMORY_TRANSFERCOMMAND_READY_OFFSET+:`TR_IN_MEMORY_TRANSFERCOMMAND_READY_WIDTH] = 0; + repeat (3) tick(); + pins[`TR_IN_MEMORY_TRANSFERCOMMAND_READY_OFFSET+:`TR_IN_MEMORY_TRANSFERCOMMAND_READY_WIDTH] = 1; + tick(); + pins[`TR_IN_MVOVER_COMMAND_VALID_OFFSET+:`TR_IN_MVOVER_COMMAND_VALID_WIDTH] = 0; + pins[`TR_IN_MEMORY_TRANSFERCOMMAND_READY_OFFSET+:`TR_IN_MEMORY_TRANSFERCOMMAND_READY_WIDTH] = 0; + repeat (2) tick(); + pins[`TR_IN_MEMORY_TRANSFERCOMPLETION_VALID_OFFSET+:`TR_IN_MEMORY_TRANSFERCOMPLETION_VALID_WIDTH]=1; + pins[`TR_IN_MEMORY_TRANSFERCOMPLETION_BITS_TAG_OFFSET+:`TR_IN_MEMORY_TRANSFERCOMPLETION_BITS_TAG_WIDTH]='1; + pins[`TR_IN_MEMORY_TRANSFERCOMPLETION_BITS_ERROR_OFFSET+:`TR_IN_MEMORY_TRANSFERCOMPLETION_BITS_ERROR_WIDTH]=reference.image.move_error; + pins[`TR_IN_MVOVER_COMPLETION_READY_OFFSET+:`TR_IN_MVOVER_COMPLETION_READY_WIDTH] = 0; + repeat (4) tick(); + pins[`TR_IN_MVOVER_COMPLETION_READY_OFFSET+:`TR_IN_MVOVER_COMPLETION_READY_WIDTH] = 1; + tick(); + pins[`TR_IN_MEMORY_TRANSFERCOMPLETION_VALID_OFFSET+:`TR_IN_MEMORY_TRANSFERCOMPLETION_VALID_WIDTH]=0; + tick(); + endtask + task body(); + longint unsigned address, discard; + pins = '0; + repeat (3) tick(1); + tick(); + layout(0, 0, 1); + layout(1, 0, 1); + for (int c = 0; c < `TR_CLIENTS; c++) + for (int row = 0; row < `TR_ROWS; row++) begin + access (c, 0, 0, row, 1, + 128'h123456789abcdef0fedcba9876543210 ^ (128'(c * 37 + row) << 32)); + access (c, 0, 0, row, 0); + end + for (int bit_index = 0; bit_index < 16; bit_index++) begin + access (bit_index % `TR_CLIENTS, 0, 0, 0, 1, '1, 16'(1 << bit_index)); + access ((bit_index + 1) % `TR_CLIENTS, 0, 0, 0, 0); + end + access (0, 0, 0, 0, 1, 0, 0); + access (0, 0, 0, 0, 0); + reference.image.error[1] = 1; + access (1, 0, 0, 0, 1, '1); + reference.image.error[1] = 0; + access (1, 0, 0, 0, 0); + for (int thread = 0; thread < 2; thread++) begin + lease(thread, 0, 0, 0, address); + physical(address, 0); + physical(address + 16, 1, 128'hcafef00d0123456789abcdefdeadbeef); + physical(address + 16, 0); + reference.image.error[`TR_CLIENTS] = 1; + physical(address, 0); + reference.image.error[`TR_CLIENTS] = 0; + lease(thread, 0, 0, 1, discard); + end + // Four independently stalled clients retain their response ownership. + begin + bit waiting[`TR_CLIENTS], any_waiting, all_returned; + for (int c = 0; c < `TR_CLIENTS; c++) begin + pins[clients[c][14]+:`TR_IN_MEM_REQ_0_BANK_ID_WIDTH] = 0; + pins[clients[c][15]+:`TR_IN_MEM_REQ_0_GROUP_ID_WIDTH] = 0; + pins[clients[c][16]+:64] = owners[c/2]; + pins[clients[c][2]+:16] = c; + pins[clients[c][0]] = 1; + pins[clients[c][4]] = 0; + waiting[c] = 1; + end + do begin + tick(); + any_waiting = 0; + foreach (waiting[c]) begin + if (waiting[c] && observed[clients[c][1]]) begin + waiting[c] = 0; + pins[clients[c][0]] = 0; + end + any_waiting |= waiting[c]; + end + end while (any_waiting); + do begin + tick(); + all_returned = 1; + for (int c = 0; c < `TR_CLIENTS; c++) all_returned &= observed[clients[c][3]]; + end while (!all_returned); + repeat (5) tick(); + for (int c = 0; c < `TR_CLIENTS; c++) pins[clients[c][4]] = 1; + tick(); + end + layout(0, 0, 0); + layout(1, 0, 0); + layout(0, 0, 1); + layout(0, 1, 1); + access (0, 0, 0, 0, 1, 128'h11112222333344445555666677778888); + access (1, 1, 0, 0, 1, 128'h9999aaaabbbbccccddddeeeeffff0000); + layout(0, 0, 1, 1, 1); + access (0, 0, 0, 0, 0); + access (1, 0, 1, 0, 0); + mapping(0, 0, 6, 0, 1, 0); + for (int group_index = 0; group_index < 2; group_index++) + for (int row = 0; row < `TR_ROWS; row++) + access (group_index, 6, group_index, row, 1, 128'(row + 100 * group_index)); + lease(0, 0, 0, 0, address); + map_offer(0, 0, 6, 0, 0, 1); + repeat (8) tick(); + evidence::check(pins[mappings[0][0]], "Lease did not block retirement"); + lease(0, 0, 0, 1, discard); + do tick(); while (pins[mappings[0][0]]); + lease(0, 0, 0, 0, address); + physical(address, 0); + physical(address + 16, 0); + lease(0, 0, 0, 1, discard); + mapping(0, 0, 10, 6, 1, 0, 1); + mapping(0, 0, 10, 6, 0, 1); + access (0, 10, 0, 0, 1, '1); + access (0, 10, 0, 0, 0); + for (int i = 0; i < 16; i++) move(i); + // Request and response payloads cross the Thread-local endpoint unchanged. + for (int pattern = 0; pattern < 32; pattern++) begin + foreach (copies[f]) + for (int b = 0; b < copies[f][2]; b++) + pins[copies[f][0]+b] = ((b + pattern) % 3 == 0) ^ ((b + pattern) % 7 == 0); + tick(); + end + foreach (copies[f]) for (int b = 0; b < copies[f][2]; b++) pins[copies[f][0]+b] = 0; + + // Joint reset cancels held logical, physical, lease and move transactions. + pins[clients[0][14]+:`TR_IN_MEM_REQ_0_BANK_ID_WIDTH] = 10; + pins[clients[0][15]+:`TR_IN_MEM_REQ_0_GROUP_ID_WIDTH] = 0; + pins[clients[0][16]+:64] = owners[0]; + pins[clients[0][2]+:16] = 0; + pins[clients[0][0]] = 1; + pins[clients[0][4]] = 0; + do tick(); while (!observed[clients[0][1]]); + pins[clients[0][0]] = 0; + pins[`TR_IN_LEASEOWNERTHREADID_OFFSET+:64] = owners[0]; + pins[`TR_IN_LEASE_REQUEST_BITS_ENDPOINT_OFFSET+:`TR_IN_LEASE_REQUEST_BITS_ENDPOINT_WIDTH] = 0; + pins[`TR_IN_LEASE_REQUEST_BITS_VBANK_OFFSET+:`TR_IN_LEASE_REQUEST_BITS_VBANK_WIDTH] = 0; + pins[`TR_IN_LEASE_REQUEST_BITS_GROUP_OFFSET+:`TR_IN_LEASE_REQUEST_BITS_GROUP_WIDTH] = 0; + pins[`TR_IN_LEASE_REQUEST_BITS_RELEASE_OFFSET] = 0; + pins[`TR_IN_LEASE_REQUEST_VALID_OFFSET] = 1; + pins[`TR_IN_LEASE_RESPONSE_READY_OFFSET] = 0; + do tick(); while (!observed[`TR_OUT_LEASE_REQUEST_READY_OFFSET]); + pins[`TR_IN_LEASE_REQUEST_VALID_OFFSET] = 0; + do tick(); while (!observed[`TR_OUT_LEASE_RESPONSE_VALID_OFFSET]); + address = observed[`TR_OUT_LEASE_RESPONSE_BITS_OFFSET+:64]; + pins[`TR_IN_PHYSICAL_0_REQUEST_BITS_ADDRESS_OFFSET+:64] = address; + pins[`TR_IN_PHYSICAL_0_REQUEST_BITS_WRITE_OFFSET] = 0; + pins[`TR_IN_PHYSICAL_0_REQUEST_VALID_OFFSET] = 1; + pins[`TR_IN_PHYSICAL_0_RESPONSE_READY_OFFSET] = 0; + do tick(); while (!observed[`TR_OUT_PHYSICAL_0_REQUEST_READY_OFFSET]); + pins[`TR_IN_PHYSICAL_0_REQUEST_VALID_OFFSET] = 0; + pins[`TR_IN_MVOVER_COMMAND_VALID_OFFSET] = 1; + pins[`TR_IN_MEMORY_TRANSFERCOMMAND_READY_OFFSET] = 1; + tick(); + pins[`TR_IN_MVOVER_COMMAND_VALID_OFFSET] = 0; + pins[`TR_IN_MEMORY_TRANSFERCOMMAND_READY_OFFSET] = 0; + repeat (8) tick(); + pins = '0; + repeat (3) tick(1); + tick(); + layout(0, 0, 1); + access (0, 0, 0, 0, 0); + repeat (8) tick(); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/suite.toml b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/suite.toml new file mode 100644 index 000000000..0df07d94a --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/suite.toml @@ -0,0 +1,88 @@ +[model] +manifest="" +library="" + +[[tests]] +name="normal" +top="tbus_router_tb" +test="protocol_test" +filelist="filelist.f" +timeout_ns=500000 +wall_timeout_s=120 +kind="functional" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=["tbus_pkg::model::behavior","tbus_pkg::model::epochs"] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=[] +assertion="" + +[[tests]] +name="physical" +top="tbus_router_tb" +test="invalid_test" +filelist="filelist.f" +timeout_ns=500000 +wall_timeout_s=120 +kind="assertion" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=[] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=["+CASE=physical"] +assertion="C2C physical access requires an allocated shared bank and aligned address" + +[[tests]] +name="lease" +top="tbus_router_tb" +test="invalid_test" +filelist="filelist.f" +timeout_ns=500000 +wall_timeout_s=120 +kind="assertion" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=[] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=["+CASE=lease"] +assertion="Shared lease requires one allocated owner/bank/group" + +[[tests]] +name="transfer" +top="tbus_router_tb" +test="invalid_test" +filelist="filelist.f" +timeout_ns=500000 +wall_timeout_s=120 +kind="assertion" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=[] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=["+CASE=transfer"] +assertion="Shared bank transfer requires a distinct allocated source" + +[[tests]] +name="alias" +top="tbus_router_tb" +test="invalid_test" +filelist="filelist.f" +timeout_ns=500000 +wall_timeout_s=120 +kind="assertion" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=[] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=["+CASE=alias"] +assertion="Renamed shared bank requires an allocated layout" diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/tb.sv b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/tb.sv new file mode 100644 index 000000000..fe7c9d8ee --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/tb.sv @@ -0,0 +1,15 @@ +module tbus_router_tb; + import uvm_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + tbus_if bus (clock); + Router dut ( + .clock(clock), + .reset(bus.reset), + `include "ports.svh" + ); + initial begin + uvm_config_db#(virtual tbus_if)::set(null, "uvm_test_top*", "vif", bus); + run_test(); + end +endmodule diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/tests/invalid.svh b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/tests/invalid.svh new file mode 100644 index 000000000..b5fb026be --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/tests/invalid.svh @@ -0,0 +1,63 @@ +class invalid_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(invalid_sequence) + logic [`TR_INPUT_BITS-1:0] pins; + function new(string name = "invalid"); + super.new(name); + endfunction + task tick(bit reset = 0); + cycle_item t = cycle_item::type_id::create("cycle"); + start_item(t); + t.inputs = pins; + t.reset = reset; + finish_item(t); + endtask + task body(); + string scenario; + if (!$value$plusargs("CASE=%s", scenario)) `uvm_fatal("CASE", "Invalid test requires a case") + pins = '0; + repeat (3) tick(1); + tick(); + if (scenario == "physical") begin + pins[`TR_IN_PHYSICAL_0_REQUEST_VALID_OFFSET] = 1; + pins[`TR_IN_PHYSICAL_0_REQUEST_BITS_ADDRESS_OFFSET+:64] = 1; + end + if (scenario == "lease") begin + pins[`TR_IN_LEASE_REQUEST_VALID_OFFSET] = 1; + pins[`TR_IN_LEASEOWNERTHREADID_OFFSET+:64] = 10; + end + if (scenario == "transfer") begin + pins[`TR_IN_CONFIG_VALID_OFFSET] = 1; + pins[`TR_IN_CONFIG_BITS_TRANSFER_OFFSET] = 1; + pins[`TR_IN_CONFIG_BITS_THREAD_ID_OFFSET+:64] = 10; + pins[`TR_IN_CONFIG_BITS_SOURCE_BANK_ID_OFFSET+:`TR_IN_CONFIG_BITS_SOURCE_BANK_ID_WIDTH] = 1; + end + if (scenario == "alias") begin + pins[`TR_IN_MAPPING_0_VALID_OFFSET] = 1; + pins[`TR_IN_MAPPING_0_BITS_ALLOCATE_OFFSET] = 1; + pins[`TR_IN_MAPPING_0_BITS_THREADID_OFFSET+:64] = 10; + pins[`TR_IN_MAPPING_0_BITS_VBANK_OFFSET+:`TR_IN_MAPPING_0_BITS_VBANK_WIDTH] = 6; + end + repeat (10) tick(); + `uvm_fatal("MISSING_ASSERTION", "Expected Router assertion did not fire") + endtask +endclass +class invalid_test extends base_test; + `uvm_component_utils(invalid_test) + driver source; + uvm_sequencer #(cycle_item) seqr; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + source = driver::type_id::create("source", this); + seqr = new("seqr", this); + endfunction + function void connect_phase(uvm_phase phase); + source.seq_item_port.connect(seqr.seq_item_export); + endfunction + task execute(); + invalid_sequence seq = invalid_sequence::type_id::create("invalid"); + seq.start(seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/tests/test.svh new file mode 100644 index 000000000..b11c898c9 --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/tests/test.svh @@ -0,0 +1,16 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.reference = test_env.reference; + seq.start(test_env.seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/scala/Emit.scala b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/scala/Emit.scala new file mode 100644 index 000000000..5f012fc26 --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/scala/Emit.scala @@ -0,0 +1,142 @@ +package framework.core.tbus.router + +import chisel3._ +import chisel3.reflect.DataMirror +import java.nio.file.{Files, Paths} +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.acc.gpdomain.configs.GpDomainParam + +object Emit extends App { + val Array("--config", config, "--output-dir", directory) = args + val values = ujson.read(Files.readString(Paths.get(config))) + val frontend = upickle.default.read[FrontendParam](values("frontend")) + val memory = upickle.default.read[MemDomainParam](values("memory")) + val bus = upickle.default.read[framework.core.tbus.configs.Params](values("tbus")) + val endpointIds = values("endpointIds").arr.toSeq.map(_.num.toInt) + val traceIds = values("traceIds").arr.toSeq.map(_.num.toInt) + + val defs = scala.collection.mutable.ArrayBuffer[String]( + s"`define TR_BANKS ${memory.sharedBankNum}", + s"`define TR_ROWS ${memory.sharedBankEntries}", + s"`define TR_CLIENTS ${memory.sharedInputChannels}", + s"`define TR_THREADS ${bus.slots}", + s"`define TR_EXTERNAL ${bus.router.physicalPorts}", + s"`define TR_DATA_BITS ${memory.bankWidth}", + s"`define TR_MASK_BITS ${memory.bankMaskLen}", + s"`define TR_TAG_BITS ${math.max(1, chisel3.util.log2Ceil(frontend.rob_entries))}", + s"`define TR_ALIASES ${2 * memory.virtualBankCount + frontend.rob_entries}", + s"`define TR_OWNERS ${endpointIds.mkString("'{", ",", "}")}" + ) + + val ports = scala.collection.mutable.ArrayBuffer[String]() + val leavesWidth = scala.collection.mutable.Map[String, Int]() + val offsets = scala.collection.mutable.Map[String, Int]() + var inOffset = 0; var outOffset = 0 + + def leaves(data: Data, path: String): Seq[(String, Data)] = data match { + case record: Record => record.elements.toSeq.flatMap { case (name, field) => + leaves(field, if (path.isEmpty) name else s"${path}_$name") + } + case vector: Vec[_] => vector.toSeq.zipWithIndex.flatMap { case (field, index) => leaves(field, s"${path}_$index") } + case field => if (field.getWidth > 0) Seq(path -> field) else Seq.empty + } + + Files.createDirectories(Paths.get(directory)) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + { + val dut = new Router(frontend, memory, bus, endpointIds, traceIds, values("diffTest").bool) + for ((name, field) <- leaves(dut.io, "")) { + val input = DataMirror.directionOf(field) == ActualDirection.Input + val direction = if (input) "IN" else "OUT"; val signal = if (input) "inputs" else "outputs"; + val offset = if (input) inOffset else outOffset; val width = field.getWidth + defs += s"`define TR_${direction}_${name.toUpperCase}_OFFSET $offset"; + defs += s"`define TR_${direction}_${name.toUpperCase}_WIDTH $width"; offsets(name) = offset; + leavesWidth(name) = width + ports += s".io_$name(bus.$signal[$offset +: $width])" + if (input) inOffset += width else outOffset += width + }; dut + }, + args = Array("--target-dir", s"$directory/Router", "--split-verilog") + ) + defs += s"`define TR_INPUT_BITS $inOffset"; defs += s"`define TR_OUTPUT_BITS $outOffset" + val clientNames = (0 until memory.sharedInputChannels).map(i => s"mem_req_$i") + + val clientFields = Seq( + "read_req_valid", + "read_req_ready", + "read_req_bits_addr", + "read_resp_valid", + "read_resp_ready", + "read_resp_bits_data", + "write_req_valid", + "write_req_ready", + "write_req_bits_addr", + "write_req_bits_data", + "write_req_bits_mask_0", + "write_resp_valid", + "write_resp_ready", + "write_resp_bits_ok", + "bank_id", + "group_id", + "thread_id" + ) + + val meshNames = (0 until memory.sharedInputChannels).map(i => + s"memory_channels_$i" + ) ++ (0 until bus.router.physicalPorts).map(i => s"memory_external_$i") + + val meshFields = Seq( + "request_valid", + "request_ready", + "request_bits_tdest", + "request_bits_tuser", + "request_bits_tdata", + "request_bits_tkeep", + "request_bits_tid", + "request_bits_tlast", + "response_valid", + "response_ready", + "response_bits_tdata", + "response_bits_tkeep", + "response_bits_tid", + "response_bits_tuser", + "response_bits_tlast" + ) + + val mappingNames = (0 until bus.slots).map(i => s"mapping_$i") + + val mappingFields = Seq( + "valid", + "ready", + "bits_bank", + "bits_vbank", + "bits_previous", + "bits_threadId", + "bits_allocate", + "bits_reuse", + "bits_retire", + "bits_write" + ) + + def table(names: Seq[String], fields: Seq[String]) = + names.map(n => fields.map(f => offsets(s"${n}_$f")).mkString("'{", ",", "}")).mkString("'{", ",", "}") + defs += s"`define TR_CLIENT_OFFSETS ${table(clientNames, clientFields)}" + defs += s"`define TR_MESH_OFFSETS ${table(meshNames, meshFields)}" + defs += s"`define TR_MAPPING_OFFSETS ${table(mappingNames, mappingFields)}" + val copies = scala.collection.mutable.ArrayBuffer[String]() + for ((name, offset) <- offsets.toSeq.sortBy(_._1) if name.startsWith("memory_localBanks_")) { + val other = "localBanks_" + name.stripPrefix("memory_localBanks_") + val memoryInput = name.contains("_request_") && !name.endsWith("_ready") || name.contains("_response_ready") + val from = if (memoryInput) name else other; val to = if (memoryInput) other else name + val gate = + if (name.endsWith("_ready") || name.endsWith("_valid")) -1 + else offsets(from.take(from.indexOf("_bits")) + "_valid") + val width = leavesWidth(name) + copies += s"'{${offsets(from)},${offsets(to)},$width,$gate}" + } + defs += s"`define TR_LOCAL_COPIES ${copies.mkString("'{", ",", "}")}" + defs += s"`define TR_LOCAL_FIELDS ${copies.size}" + Files.writeString(Paths.get(directory, "parameters.svh"), defs.mkString("\n") + "\n") + Files.writeString(Paths.get(directory, "ports.svh"), ports.mkString(",\n") + "\n") +} diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/scala/Router.scala b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/scala/Router.scala new file mode 100644 index 000000000..de26dc0c3 --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/scala/Router.scala @@ -0,0 +1,645 @@ +package framework.core.tbus.router + +import framework.lib.arbiter.{Arbiter} + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.core.tbus.configs.{Params => TBusParam} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.memdomain.BankPort +import framework.acc.debug.trace.MTraceDPI +import framework.lib.hash.BankHashMonitor +import framework.acc.memdomain.bank_router.blt.AccPipe +import framework.acc.memdomain.PhysicalBankHash +import framework.acc.memdomain.LayoutConfig +import framework.core.tbus.configs.Layout +import framework.acc.memdomain.bank_router.BankMapping +import framework.ip.memory.mesh_shm.{MeshLocalBankPort, MeshSharedMemIO} +import framework.acc.memdomain.{MvoverISA, MvoverPort} + +@instantiable +class Router( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val tbusParam: TBusParam, + val endpointIds: Seq[Int], + val traceIds: Seq[Int], + val diffTest: Boolean) + extends Module { + val externalPhysicalPorts = tbusParam.router.physicalPorts + require(externalPhysicalPorts >= 0) + val nThreads = tbusParam.slots + val totalBanks = Layout.totalBank(memParam) + val totalChannel = Layout.totalChannel(memParam) + + val sharedMemory = memParam.copy( + bankEntries = memParam.sharedBankEntries + ) + + val tagBits = math.max(1, log2Ceil(frontendParam.rob_entries)) + + val threadIds = VecInit(endpointIds.map(_.U(64.W))) + require(traceIds.size == endpointIds.size) + def traceId(owner: UInt): UInt = Mux1H(endpointIds.map(id => owner === id.U), traceIds.map(_.U(64.W))) + + val memoryParams = tbusParam.router.mesh(frontendParam, memParam, tbusParam) + + @public + val io = IO(new Bundle { + + val memory = Flipped(new MeshSharedMemIO(memoryParams)) + + val mem_req = Vec(totalChannel, Flipped(new BankPort(frontendParam, memParam))) + val lease = Option.when(externalPhysicalPorts > 0)(Flipped(new LeasePort)) + val leaseOwnerThreadId = Option.when(externalPhysicalPorts > 0)(Input(UInt(64.W))) + + val physical = Vec( + externalPhysicalPorts, + Flipped(new PhysicalPort(memParam.bankWidth)) + ) + + val mvover = Flipped(new MvoverPort) + + val localBanks = + Vec( + nThreads, + new MeshLocalBankPort( + MvoverISA.AddressBits, + MvoverISA.BankBits, + memParam.bankWidth, + tagBits + ) + ) + + val config = Flipped(Decoupled(new LayoutConfig(frontendParam, memParam))) + val mapping = Vec(nThreads, Flipped(Decoupled(new BankMapping(frontendParam)))) + + val bank_hashes = if (diffTest) Some(Output(Vec(totalBanks, new PhysicalBankHash(frontendParam)))) else None + }) + + require(io.memory.transferCommand.bits.sourceAddr.getWidth <= 16) + require(io.memory.transferCommand.bits.sourceThread.getWidth <= 8) + io.memory.transferCommand.valid := io.mvover.command.valid + io.memory.transferCommand.bits.sourceThread := io.mvover.command.bits.sourceThread + io.memory.transferCommand.bits.targetThread := io.mvover.command.bits.targetThread + io.memory.transferCommand.bits.sourceBank := io.mvover.command.bits.sourceBank + io.memory.transferCommand.bits.targetBank := io.mvover.command.bits.targetBank + io.memory.transferCommand.bits.sourceAddr := io.mvover.command.bits.sourceAddr + io.memory.transferCommand.bits.targetAddr := io.mvover.command.bits.targetAddr + io.memory.transferCommand.bits.rows := io.mvover.command.bits.rows + io.memory.transferCommand.bits.tag := 0.U + io.mvover.command.ready := io.memory.transferCommand.ready + io.mvover.completion.valid := io.memory.transferCompletion.valid + io.mvover.completion.bits := io.memory.transferCompletion.bits.error + io.memory.transferCompletion.ready := io.mvover.completion.ready + for ((local, index) <- io.memory.localBanks.zipWithIndex) { + io.localBanks(index).request.valid := local.request.valid + io.localBanks(index).request.bits := local.request.bits + local.request.ready := io.localBanks(index).request.ready + local.response.valid := io.localBanks(index).response.valid + local.response.bits := io.localBanks(index).response.bits + io.localBanks(index).response.ready := local.response.ready + } + + val accPipes: Seq[Instance[AccPipe]] = Seq.fill(totalChannel)(Instantiate(new AccPipe(frontendParam, memParam))) + + val hashMonitors: Option[Seq[Instance[BankHashMonitor]]] = + if (diffTest) { + Some(Seq.fill(totalBanks)(Instantiate(new BankHashMonitor(sharedMemory)))) + } else { + None + } + + // Per-channel memory trace DPI-C modules to avoid losing simultaneous events + val mtraces: Seq[Instance[MTraceDPI]] = Seq.fill(totalChannel + externalPhysicalPorts)(Instantiate(new MTraceDPI)) + for (mt <- mtraces) { + mt.io.clock := clock + mt.io.reset := reset.asBool + mt.io.is_write := 0.U + mt.io.is_shared := 0.U + mt.io.channel := 0.U + mt.io.thread_id := 0.U + mt.io.rob_id := 0.U + mt.io.inst_id := 0.U + mt.io.vbank_id := 0.U + mt.io.pbank_id := 0.U + mt.io.group_id := 0.U + mt.io.addr := 0.U + mt.io.write_mask := 0.U + mt.io.data_lo := 0.U + mt.io.data_hi := 0.U + mt.io.enable := false.B + } + + // ----------------------------------------------------------------------------- + // Mapping table + // ----------------------------------------------------------------------------- + class MappingTableEntry extends Bundle { + val valid = Bool() + val thread_id = UInt(64.W) + val vbank_id = UInt(frontendParam.bank_id_len.W) + val architectural_id = UInt(frontendParam.bank_id_len.W) + val is_multi = Bool() + val group_id = UInt(memParam.groupIdWidth.W) + } + + val mappingTable = RegInit(VecInit(Seq.fill(totalBanks)(0.U.asTypeOf(new MappingTableEntry)))) + + val architecturalColumns = RegInit(VecInit(Seq.fill(nThreads)(VecInit( + Seq.fill(memParam.virtualBankCount)(0.U( + memParam.groupCountWidth.W + )) + )))) + + val committed = RegInit(VecInit(Seq.fill(nThreads)(VecInit( + (0 until memParam.virtualBankCount).map(_.U( + frontendParam.bank_id_len.W + )) + )))) + + val leases = Option.when(externalPhysicalPorts > 0)(RegInit(VecInit(Seq.fill(totalBanks)(0.U(32.W))))) + + // Record binding and lifetime changes, including aliases with no active accesses. + for ((entry, physical) <- mappingTable.zipWithIndex) { + val ownerThread = PriorityEncoder(VecInit(threadIds.map(_ === entry.thread_id)).asUInt) + val live = entry.valid + val current = entry.valid && committed(ownerThread)(entry.architectural_id) === entry.vbank_id + val previous = RegNext(entry, 0.U.asTypeOf(entry)) + val previousLive = RegNext(live, false.B) + val previousCurrent = RegNext(current, false.B) + val trace = Instantiate(new MTraceDPI) + trace.io.clock := clock + trace.io.reset := reset.asBool + trace.io.is_write := Mux(entry.valid, 2.U, 3.U) + trace.io.is_shared := 1.U + trace.io.channel := 0.U + trace.io.thread_id := traceId(entry.thread_id) + trace.io.rob_id := 0.U + trace.io.inst_id := 0.U + trace.io.vbank_id := entry.vbank_id + trace.io.pbank_id := physical.U + trace.io.group_id := entry.group_id + trace.io.addr := entry.architectural_id + trace.io.write_mask := 0.U + trace.io.data_lo := live.asUInt + trace.io.data_hi := current.asUInt + trace.io.enable := (entry.valid || previous.valid) && + (entry.asUInt =/= previous.asUInt || live =/= previousLive || current =/= previousCurrent) + } + + io.lease.foreach { port => + val pending = RegInit(false.B) + val result = Reg(UInt(64.W)) + val ownerThread = PriorityEncoder(VecInit(threadIds.map(_ === io.leaseOwnerThreadId.get)).asUInt) + val currentAlias = committed(ownerThread)(port.request.bits.vbank) + val matches = VecInit(mappingTable.zipWithIndex.map { case (entry, physical) => + entry.valid && + entry.thread_id === io.leaseOwnerThreadId.get && entry.architectural_id === port.request.bits.vbank && + entry.group_id === port.request.bits.group && + Mux(port.request.bits.release, leases.get(physical) =/= 0.U, entry.vbank_id === currentAlias) + }) + val index = PriorityEncoder(matches) + port.request.ready := !pending && !io.config.valid && !reset.asBool + port.response.valid := pending && !reset.asBool + port.response.bits := result + when(port.request.fire) { + assert(PopCount(matches) === 1.U, "Shared lease requires one allocated owner/bank/group") + val count = leases.get(index) + when(port.request.bits.release) { + assert(count =/= 0.U, "Shared lease release has no export") + count := count - 1.U + result := 0.U + }.otherwise { + assert(count =/= "hffffffff".U, "Shared lease count overflow") + count := count + 1.U + result := index * (memParam.sharedBankEntries * (memParam.bankWidth / 8)).U( + 64.W + ) + } + pending := true.B + } + when(port.response.fire)(pending := false.B) + } + + val rowBits = log2Ceil(memParam.sharedBankEntries) + val bytesPerRow = memParam.bankWidth / 8 + val byteBits = log2Ceil(bytesPerRow) + for (i <- 0 until externalPhysicalPorts) { + val physical = io.physical(i) + val channel = io.memory.external(i) + val bank = physical.request.bits.address >> (rowBits + byteBits) + val legal = bank < totalBanks.U && physical.request.bits.address(byteBits - 1, 0) === 0.U + val bankIndex = bank(math.max(1, log2Ceil(totalBanks)) - 1, 0) + val allocated = VecInit(mappingTable.map(_.valid))(bankIndex) + when(physical.request.valid && !reset.asBool) { + assert(legal && allocated, "C2C physical access requires an allocated shared bank and aligned address") + } + channel.request.valid := physical.request.valid && legal && allocated + channel.request.bits := 0.U.asTypeOf(channel.request.bits) + channel.request.bits.bank := bankIndex + channel.request.bits.tuser := Cat( + physical.request.bits.address(rowBits + byteBits - 1, byteBits).pad(16), + physical.request.bits.write + ) + channel.request.bits.data := physical.request.bits.data + channel.request.bits.mask := physical.request.bits.mask + channel.request.bits.tag := 0.U + channel.request.bits.tlast := true.B + physical.request.ready := channel.request.ready && legal && allocated + physical.response.valid := channel.response.valid + physical.response.bits.data := channel.response.bits.data + physical.response.bits.error := channel.response.bits.error + channel.response.ready := physical.response.ready + } + + for (i <- 0 until externalPhysicalPorts) { + val physical = io.physical(i) + val held = Reg(new PhysicalRequest(memParam.bankWidth)) + val entry = Reg(new MappingTableEntry) + val pending = RegInit(false.B) + val bankBits = log2Ceil( + memParam.sharedBankEntries * (memParam.bankWidth / 8) + ) + when(physical.request.fire) { + assert(!pending, "Physical shared port accepts one outstanding request") + held := physical.request.bits + entry := mappingTable((physical.request.bits.address >> bankBits)(math.max(1, log2Ceil(totalBanks)) - 1, 0)) + pending := true.B + } + when(physical.response.valid && !reset.asBool)(assert(pending)) + when(physical.response.fire)(pending := false.B) + val trace = mtraces(totalChannel + i) + val value = Mux(held.write, held.data, physical.response.bits.data) + trace.io.is_write := held.write.asUInt + trace.io.is_shared := 1.U + trace.io.channel := (totalChannel + i).U + trace.io.thread_id := traceId(entry.thread_id) + trace.io.vbank_id := entry.vbank_id + trace.io.pbank_id := held.address >> bankBits + trace.io.group_id := entry.group_id + trace.io.addr := (held.address >> 4) & (memParam.sharedBankEntries - 1).U + trace.io.write_mask := Mux(held.write, held.mask, 0.U) + trace.io.data_lo := value(63, 0) + trace.io.data_hi := value(127, 64) + trace.io.enable := physical.response.fire && !physical.response.bits.error + } + + def addEntry( + thread_id: UInt, + vbank_id: UInt, + pbank_id: UInt, + is_multi: Bool, + group_id: UInt + ): Unit = { + val duplicate = mappingTable.map(entry => + entry.valid && + !(group_id === 0.U && entry.thread_id === thread_id && entry.vbank_id === vbank_id) && + (entry.thread_id === thread_id) && + (entry.vbank_id === vbank_id) && + (entry.group_id === group_id) + ).reduce(_ || _) + when(duplicate) { + assert(false.B, "Router duplicate allocation: thread=%d vbank=%d group=%d\n", thread_id, vbank_id, group_id) + } + + val entry = mappingTable(pbank_id) + leases.foreach(counts => assert(counts(pbank_id) === 0.U, "Shared bank allocation replaces live exports")) + entry.valid := true.B + entry.thread_id := thread_id + entry.vbank_id := vbank_id + entry.architectural_id := vbank_id + entry.is_multi := is_multi + entry.group_id := group_id + } + + def deleteEntry(thread_id: UInt, vbank_id: UInt): Unit = { + val thread = PriorityEncoder(VecInit(threadIds.map(_ === thread_id)).asUInt) + val found = architecturalColumns(thread)(vbank_id) =/= 0.U + when(!found) { + assert(false.B, "Router release missing allocation: thread=%d vbank=%d\n", thread_id, vbank_id) + } + clearVbank(thread_id, vbank_id) + } + + def clearVbank(thread_id: UInt, vbank_id: UInt): Unit = { + for (i <- 0 until totalBanks) { + when(mappingTable(i).valid && mappingTable(i).architectural_id === vbank_id && mappingTable( + i + ).thread_id === thread_id) { + leases.foreach(counts => assert(counts(i) === 0.U, "Shared bank mapping changed with live exports")) + mappingTable(i).valid := false.B + } + } + } + + // ----------------------------------------------------------------------------- + // Default Value + // ----------------------------------------------------------------------------- + + for (i <- 0 until totalChannel) { + accPipes(i).io.mem_req.write <> io.mem_req(i).write + accPipes(i).io.mem_req.read <> io.mem_req(i).read + accPipes(i).io.mem_req.bank_id := io.mem_req(i).bank_id + accPipes(i).io.mem_req.group_id := io.mem_req(i).group_id + accPipes(i).io.mem_req.thread_id := io.mem_req(i).thread_id + + // Bank-side defaults (only driven when a bank is actually connected) + accPipes(i).io.sramRead.req.ready := false.B + accPipes(i).io.sramRead.resp.valid := false.B + accPipes(i).io.sramRead.resp.bits := DontCare + + accPipes(i).io.sramWrite.req.ready := false.B + accPipes(i).io.sramWrite.resp.valid := false.B + accPipes(i).io.sramWrite.resp.bits := DontCare + } + + val realloc = io.config.bits.group_id === 0.U + + val freeBanks = VecInit(mappingTable.map(entry => !entry.valid)).asUInt + + val freePbankMask = VecInit(mappingTable.zipWithIndex.map { case (entry, physical) => + freeBanks(physical) || + (realloc && entry.thread_id === io.config.bits.thread_id && entry.architectural_id === io.config.bits.vbank_id) + }) + + val hasFreePbank = freePbankMask.asUInt.orR + io.config.ready := !io.config.bits.alloc || hasFreePbank + when(io.config.valid && io.config.bits.alloc && !hasFreePbank) { + assert(false.B, "Router allocation failed: no free physical shared bank\n") + } + + // ----------------------------------------------------------------------------- + // Bank Alloc/Release + // ----------------------------------------------------------------------------- + + hashMonitors.foreach(_.foreach(_.io.bind := false.B)) + + val mappings = Instantiate(new Arbiter(new BankMapping(frontendParam), nThreads)) + for (thread <- 0 until nThreads) { mappings.io.in(thread) <> io.mapping(thread) } + val change = mappings.io.out.bits + val owner = mappings.io.chosen + val columns = architecturalColumns(owner)(change.bank) + + val leaseBusy = VecInit(mappingTable.zipWithIndex.map { case (entry, physical) => + val held = if (externalPhysicalPorts > 0) leases.get(physical) =/= 0.U else false.B + entry.valid && entry.thread_id === change.threadId && entry.vbank_id === change.previous && held + }).asUInt.orR + + val reusableBanks = VecInit(mappingTable.map(entry => + !entry.valid || (change.reuse && entry.thread_id === change.threadId && entry.vbank_id === change.previous) + )).asUInt + + mappings.io.out.ready := !io.config.valid && (!change.allocate || PopCount(reusableBanks) >= columns) && + (!(change.retire || change.reuse) || !leaseBusy) + when(mappings.io.out.valid && change.allocate) { + assert(columns =/= 0.U && columns <= totalBanks.U, "Renamed shared bank requires an allocated layout") + } + when(mappings.io.out.fire && change.retire) { + committed(owner)(change.bank) := change.vbank + for (entry <- mappingTable) { + when(entry.valid && entry.thread_id === change.threadId && entry.vbank_id === change.previous) { + entry.valid := false.B + } + } + } + val remaining = Wire(Vec(totalBanks + 1, UInt(totalBanks.W))) + remaining(0) := reusableBanks + for (group <- 0 until totalBanks) { + val physical = PriorityEncoder(remaining(group)) + remaining(group + 1) := remaining(group) & ~UIntToOH(physical, totalBanks) + when(mappings.io.out.fire && change.allocate && group.U < columns) { + assert(change.allocate && columns =/= 0.U, "Shared alias requires an allocated layout") + addEntry(change.threadId, change.vbank, physical, columns > 1.U, group.U) + mappingTable(physical).architectural_id := change.bank + hashMonitors.foreach { hashes => + for (bank <- 0 until totalBanks) { when(physical === bank.U)(hashes(bank).io.bind := true.B) } + } + } + } + + when(io.config.fire) { + val thread = PriorityEncoder(VecInit(threadIds.map(_ === io.config.bits.thread_id)).asUInt) + when(io.config.bits.transfer) { + val source = io.config.bits.source_bank_id + val target = io.config.bits.vbank_id + val owner = io.config.bits.thread_id + val sourceCount = architecturalColumns(thread)(source) + val targetCount = architecturalColumns(thread)(target) + val totalCount = sourceCount +& targetCount + architecturalColumns(thread)(source) := 0.U + architecturalColumns(thread)(target) := totalCount + assert(source =/= target && sourceCount =/= 0.U, "Shared bank transfer requires a distinct allocated source") + assert(totalCount <= totalBanks.U, "Shared bank transfer exceeds physical bank count") + for ((entry, physical) <- mappingTable.zipWithIndex) { + when(entry.valid && entry.thread_id === owner && entry.architectural_id === source) { + leases.foreach(counts => assert(counts(physical) === 0.U, "Shared bank transfer source has live exports")) + entry.vbank_id := target + entry.architectural_id := target + entry.group_id := targetCount + entry.group_id + entry.is_multi := totalCount > 1.U + } + when(entry.valid && entry.thread_id === owner && entry.architectural_id === target) { + entry.vbank_id := target + entry.is_multi := totalCount > 1.U + } + } + }.elsewhen(io.config.bits.alloc) { + architecturalColumns(thread)(io.config.bits.vbank_id) := io.config.bits.group_id +& 1.U + when(io.config.bits.group_id === 0.U) { + clearVbank(io.config.bits.thread_id, io.config.bits.vbank_id) + } + val pbankId = PriorityEncoder(freePbankMask) + hashMonitors.foreach { hashes => + for (i <- 0 until totalBanks) { + hashes(i).io.bind := pbankId === i.U + } + } + printf( + p"[Router][ALLOC] thread=${io.config.bits.thread_id} vbank=0x${Hexadecimal(io.config.bits.vbank_id)} " + + p"group=${io.config.bits.group_id} pbank=$pbankId is_multi=${io.config.bits.is_multi}\n" + ) + addEntry( + io.config.bits.thread_id, + io.config.bits.vbank_id, + pbankId, + io.config.bits.is_multi, + io.config.bits.group_id + ) + }.otherwise { + architecturalColumns(thread)(io.config.bits.vbank_id) := 0.U + printf( + p"[Router][RELEASE] thread=${io.config.bits.thread_id} vbank=0x${Hexadecimal(io.config.bits.vbank_id)}\n" + ) + deleteEntry(io.config.bits.thread_id, io.config.bits.vbank_id) + } + } + + // ----------------------------------------------------------------------------- + // Connect AccPipe and Banks + // ----------------------------------------------------------------------------- + def emitTrace( + ch: Int, + isWrite: UInt, + pbankId: UInt, + addr: UInt, + writeMask: UInt, + dataLo: UInt, + dataHi: UInt, + en: Bool + ): Unit = { + mtraces(ch).io.is_write := isWrite + mtraces(ch).io.is_shared := io.mem_req(ch).is_shared.asUInt + mtraces(ch).io.channel := ch.U + mtraces(ch).io.thread_id := traceId(io.mem_req(ch).thread_id) + mtraces(ch).io.rob_id := io.mem_req(ch).rob_id + mtraces(ch).io.inst_id := io.mem_req(ch).inst_id + mtraces(ch).io.vbank_id := io.mem_req(ch).bank_id + mtraces(ch).io.pbank_id := pbankId + mtraces(ch).io.group_id := io.mem_req(ch).group_id + mtraces(ch).io.addr := addr + mtraces(ch).io.write_mask := writeMask + mtraces(ch).io.data_lo := dataLo + mtraces(ch).io.data_hi := dataHi + mtraces(ch).io.enable := en + } + + for (i <- 0 until totalChannel) { + val activeHart = Mux(accPipes(i).io.busy, accPipes(i).io.thread_id, io.mem_req(i).thread_id) + val activeBank = Mux(accPipes(i).io.busy, accPipes(i).io.bank_id, io.mem_req(i).bank_id) + val activeGroup = Mux(accPipes(i).io.busy, accPipes(i).io.group_id, io.mem_req(i).group_id) + val req_valid = io.mem_req(i).read.req.valid || io.mem_req(i).write.req.valid || accPipes(i).io.busy + + val tracePbankId = Wire(UInt(32.W)) + tracePbankId := 0.U + for (j <- 0 until totalBanks) { + val trace_hit_bank = mappingTable(j).valid && + (mappingTable(j).thread_id === activeHart) && + (mappingTable(j).vbank_id === activeBank) && + (!mappingTable(j).is_multi || + (mappingTable(j).is_multi && (mappingTable(j).group_id === activeGroup))) + when(trace_hit_bank) { + tracePbankId := j.U + } + } + + // Memory trace: read request + when(io.mem_req(i).read.req.fire) { + emitTrace(i, 0.U, tracePbankId, io.mem_req(i).read.req.bits.addr, 0.U, 0.U, 0.U, true.B) + } + + // Arrival trace: observe acceptance at the selected shared SPM SRAM port. + when(accPipes(i).io.sramWrite.req.fire) { + emitTrace( + i, + 1.U, + tracePbankId, + accPipes(i).io.sramWrite.req.bits.addr, + accPipes(i).io.sramWrite.req.bits.mask.asUInt, + accPipes(i).io.sramWrite.req.bits.data(63, 0), + accPipes(i).io.sramWrite.req.bits.data(127, 64), + true.B + ) + } + + val channel = io.memory.channels(i) + val mapped = mappingTable.map(entry => + entry.valid && + entry.thread_id === activeHart && + entry.vbank_id === activeBank && + (!entry.is_multi || entry.group_id === activeGroup) + ).reduce(_ || _) + val writeReq = accPipes(i).io.sramWrite.req.valid + val nextTag = RegInit(0.U(tagBits.W)) + + channel.request.valid := mapped && (writeReq || accPipes(i).io.sramRead.req.valid) + channel.request.bits.bank := tracePbankId + channel.request.bits.tuser := Cat( + Mux(writeReq, accPipes(i).io.sramWrite.req.bits.addr, accPipes(i).io.sramRead.req.bits.addr), + writeReq + ) + channel.request.bits.tlast := true.B + channel.request.bits.data := accPipes(i).io.sramWrite.req.bits.data + channel.request.bits.mask := accPipes(i).io.sramWrite.req.bits.mask.asUInt + channel.request.bits.tag := nextTag + accPipes(i).io.sramRead.req.ready := mapped && !writeReq && channel.request.ready + accPipes(i).io.sramWrite.req.ready := mapped && channel.request.ready + accPipes(i).io.sramRead.resp.valid := channel.response.valid && !channel.response.bits.write + accPipes(i).io.sramRead.resp.bits.data := channel.response.bits.data + accPipes(i).io.sramWrite.resp.valid := channel.response.valid && channel.response.bits.write + accPipes(i).io.sramWrite.resp.bits.ok := !channel.response.bits.error + channel.response.ready := Mux( + channel.response.bits.write, + accPipes(i).io.sramWrite.resp.ready, + accPipes(i).io.sramRead.resp.ready + ) + when(channel.request.fire) { + nextTag := nextTag + 1.U + } + + } + + io.memory.bankWrites.foreach(_.ready := true.B) + + hashMonitors.foreach { hashes => + for (j <- 0 until totalBanks) { + val monitor = hashes(j) + + val write = io.memory.bankWrites(j) + write.ready := monitor.io.write.ready + monitor.io.write.valid := write.valid + monitor.io.write.bits.addr := write.bits.addr + monitor.io.write.bits.mask := write.bits.mask.asBools + monitor.io.write.bits.data := write.bits.data + + } + } + + io.bank_hashes.foreach { states => + for (i <- 0 until totalBanks) { + states(i).valid := mappingTable(i).valid + states(i).threadId := mappingTable(i).thread_id + states(i).vbankId := mappingTable(i).vbank_id + states(i).pbankId := i.U + states(i).groupId := mappingTable(i).group_id + states(i).statusHash := hashMonitors.get(i).io.statusHash + } + } +} + +/** Core shared SRAM byte space; not a Thread virtual-bank or private-bank address. */ +class PhysicalRequest(dataBits: Int) extends Bundle { + + val address = UInt(64.W) + val write = Bool() + val data = UInt(dataBits.W) + val mask = UInt((dataBits / 8).W) +} + +class PhysicalResponse(dataBits: Int) extends Bundle { + + val data = UInt(dataBits.W) + val error = Bool() +} + +/** One outstanding transaction per port; a write response acknowledges actual SRAM completion. */ +class PhysicalPort(dataBits: Int) extends Bundle { + + val request = Decoupled(new PhysicalRequest(dataBits)) + val response = Flipped(Decoupled(new PhysicalResponse(dataBits))) +} + +class LeaseRequest extends Bundle { + val endpoint = UInt(32.W) + val vbank = UInt(16.W) + val group = UInt(16.W) + val release = Bool() +} + +/** One outstanding mapping lease operation; export returns its physical byte base. */ +class LeasePort extends Bundle { + val request = Decoupled(new LeaseRequest) + val response = Flipped(Decoupled(UInt(64.W))) +} diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..897b127dd --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/scala/configs/Params.scala @@ -0,0 +1,45 @@ +package framework.core.tbus.router.configs + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.core.tbus.configs.{Params => TBusParam} + +import chisel3.util.log2Ceil +import framework.core.tbus.configs.Layout +import framework.acc.memdomain.MvoverISA +import framework.ip.memory.mesh_shm.{MeshSharedMemParams, MeshThreadAttachment} +import upickle.default.{macroRW, ReadWriter} + +case class Params(physicalPorts: Int) { + + def mesh(frontendParam: FrontendParam, memParam: MemDomainParam, tbusParam: TBusParam): MeshSharedMemParams = { + require( + memParam.bankWidth == 128 && memParam.bankMaskLen == 16 + ) + val totalBanks = Layout.totalBank(memParam) + val nThreads = tbusParam.slots + val tagBits = math.max(1, log2Ceil(frontendParam.rob_entries)) + val externalPhysicalPorts = physicalPorts + val physicalBanks = totalBanks + val rows = math.ceil(math.sqrt(physicalBanks.toDouble)).toInt + val columns = (physicalBanks + rows - 1) / rows + val perThread = Layout.portsPerThread(memParam, tbusParam) + MeshSharedMemParams( + rows = rows, + cols = columns, + entriesPerBank = memParam.sharedBankEntries, + dataBits = memParam.bankWidth, + tagBits = tagBits, + localBankBits = MvoverISA.BankBits, + threads = (0 until nThreads).map { thread => + val ports = if (tbusParam.participants.contains(thread)) perThread else 0 + MeshThreadAttachment(tbusParam.bankNodes(thread), ports) + }, + externalChannels = externalPhysicalPorts, + visibleBanks = totalBanks + ) + } + +} + +object Params { implicit val rw: ReadWriter[Params] = macroRW } diff --git a/arch/src/main/scala/framework/core/tss/build.sc b/arch/src/main/scala/framework/core/tss/build.sc new file mode 100644 index 000000000..4a781e221 --- /dev/null +++ b/arch/src/main/scala/framework/core/tss/build.sc @@ -0,0 +1,22 @@ +import mill._ +import $file.^.^.^.^.^.^.settings + + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.core.tss.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Store.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/core/tss/configs/uvm.toml b/arch/src/main/scala/framework/core/tss/configs/uvm.toml new file mode 100644 index 000000000..38cb7675d --- /dev/null +++ b/arch/src/main/scala/framework/core/tss/configs/uvm.toml @@ -0,0 +1,9 @@ +[parameters] +ports = 2 + +[parameters.memory] +base = "4096" +bytes = 256 +dataBits = 128 + +[defines] diff --git a/arch/src/main/scala/framework/core/tss/src/main/resources/agent/driver.svh b/arch/src/main/scala/framework/core/tss/src/main/resources/agent/driver.svh new file mode 100644 index 000000000..be52642be --- /dev/null +++ b/arch/src/main/scala/framework/core/tss/src/main/resources/agent/driver.svh @@ -0,0 +1,25 @@ +class driver extends uvm_driver #(cycle_item); + `uvm_component_utils(driver) + virtual tss_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual tss_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "TSS interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item req; + forever begin + seq_item_port.get_next_item(req); + @(negedge vif.clock); + vif.reset = req.reset; + vif.inputs = req.inputs; + @(posedge vif.clock); + req.outputs = vif.outputs; + #1; + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/tss/src/main/resources/agent/item.svh b/arch/src/main/scala/framework/core/tss/src/main/resources/agent/item.svh new file mode 100644 index 000000000..78ff84d64 --- /dev/null +++ b/arch/src/main/scala/framework/core/tss/src/main/resources/agent/item.svh @@ -0,0 +1,23 @@ +class cycle_item extends uvm_sequence_item; + `uvm_object_utils(cycle_item) + logic [`TSS_INPUT_BITS-1:0] inputs; + logic [`TSS_OUTPUT_BITS-1:0] outputs; + bit reset; + function new(string name = "cycle_item"); + super.new(name); + endfunction +endclass +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(value, UVM_ALL_ON) + `uvm_field_int(error, UVM_ALL_ON) + `uvm_field_int(port, UVM_ALL_ON) + `uvm_object_utils_end + logic [`TSS_DATA_BITS-1:0] value; + bit error; + int port; + int message_kind; + function new(string name = "result_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/tss/src/main/resources/agent/monitor.svh b/arch/src/main/scala/framework/core/tss/src/main/resources/agent/monitor.svh new file mode 100644 index 000000000..57bfe2544 --- /dev/null +++ b/arch/src/main/scala/framework/core/tss/src/main/resources/agent/monitor.svh @@ -0,0 +1,25 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual tss_if vif; + uvm_analysis_port #(cycle_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual tss_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "TSS monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item t; + forever begin + @(posedge vif.clock); + t = cycle_item::type_id::create("observed"); + t.inputs = vif.inputs; + t.outputs = vif.outputs; + t.reset = vif.reset; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/tss/src/main/resources/env/env.svh b/arch/src/main/scala/framework/core/tss/src/main/resources/env/env.svh new file mode 100644 index 000000000..26a84020c --- /dev/null +++ b/arch/src/main/scala/framework/core/tss/src/main/resources/env/env.svh @@ -0,0 +1,34 @@ +class env extends uvm_env; + `uvm_component_utils(env) + driver source; + monitor mon; + model reference; + uvm_sequencer #(cycle_item) seqr; + result_board #(result_item, 0) boards[`TSS_PORTS]; + uvm_analysis_imp #(result_item, env) results; + function new(string name, uvm_component parent); + super.new(name, parent); + results = new("results", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + source = driver::type_id::create("source", this); + mon = monitor::type_id::create("monitor", this); + reference = model::type_id::create("model", this); + seqr = new("seqr", this); + foreach (boards[p]) + boards[p] = result_board#(result_item, 0)::type_id::create($sformatf("port%0d", p), this); + endfunction + function void connect_phase(uvm_phase phase); + source.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(reference.analysis_export); + reference.results.connect(results); + endfunction + function void write(result_item t); + case (t.message_kind) + 0: boards[t.port].write_expected(t); + 1: boards[t.port].write_actual(t); + 2: boards[t.port].cancel_pending(); + endcase + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/tss/src/main/resources/env/model.svh b/arch/src/main/scala/framework/core/tss/src/main/resources/env/model.svh new file mode 100644 index 000000000..e30ea3b2a --- /dev/null +++ b/arch/src/main/scala/framework/core/tss/src/main/resources/env/model.svh @@ -0,0 +1,155 @@ +class model extends uvm_subscriber #(cycle_item); + `uvm_component_utils(model) + uvm_analysis_port #(result_item) results; + byte unsigned memory[`TSS_BYTES]; + bit initialized[`TSS_BYTES]; + bit pending[`TSS_PORTS], draining[`TSS_PORTS]; + int overtakes[`TSS_PORTS]; + const int offsets[`TSS_PORTS][13] = `TSS_PORT_OFFSETS; + longint unsigned accepted, completed, cancelled; + covergroup behavior with function sample ( + int port, + bit writing, + int size, + int masking, + bit error, + bit cancelling, + bit in_use, + bit holding, + bit contending + ); + option.per_instance = 1; + ports: coverpoint port {bins all[] = {[0 : `TSS_PORTS - 1]};} + direction: coverpoint writing; + sizes: coverpoint size { + bins all[] = {[0 : $clog2(`TSS_MASK_BITS)]}; + ignore_bins invalid = {[$clog2(`TSS_MASK_BITS) + 1 : (1 << `TSS_SIZE_BITS) - 1]}; + } + masks: coverpoint masking {bins all[] = {[0 : 2]};} + errors: coverpoint error; + cancels: coverpoint cancelling; + usage: coverpoint in_use; + stalls: coverpoint holding; + contention: coverpoint contending; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + results = new("results", this); + behavior = new(); + endfunction + function void event_result(int port, int kind, logic [`TSS_DATA_BITS-1:0] value = '0, + bit error = 0); + result_item t = result_item::type_id::create("result"); + t.port = port; + t.message_kind = kind; + t.value = value; + t.error = error; + results.write(t); + endfunction + function void write(cycle_item t); + int fires, offers; + if (t.reset) begin + foreach (pending[p]) begin + if (pending[p]) cancelled++; + event_result(p, 2); + pending[p] = 0; + draining[p] = 0; + overtakes[p] = 0; + end + return; + end + offers = 0; + fires = 0; + for (int p = 0; p < `TSS_PORTS; p++) begin + if (t.inputs[offsets[p][0]]) offers++; + if (t.inputs[offsets[p][0]] && t.outputs[offsets[p][6]]) fires++; + end + if (fires > 1) `uvm_fatal("ARBITRATION", "Multiple memory requests accepted in one cycle") + for (int p = 0; p < `TSS_PORTS; p++) begin + bit cancel_port, valid, ready, rvalid, rready, busy, in_use, eligible; + cancel_port = p < `TSS_CLIENTS && t.inputs[offsets[p][12]]; + in_use = t.inputs[`TSS_IN_INUSE_OFFSET]; + valid = t.inputs[offsets[p][0]]; + ready = t.outputs[offsets[p][6]]; + rvalid = t.outputs[offsets[p][7]]; + rready = t.inputs[offsets[p][10]]; + busy = t.outputs[offsets[p][11]]; + verification_pkg::evidence::checks += 6; + if (pending[p] && !busy) `uvm_fatal("BUSY", "An outstanding request lost its busy indication") + if (!pending[p] && !draining[p] && busy) `uvm_fatal("BUSY", "An idle port is marked busy") + if (draining[p] && !busy) draining[p] = 0; + if ((cancel_port || (p < `TSS_CLIENTS && !in_use)) && ready) + `uvm_fatal("GATING", "Disabled client accepted a request") + if (draining[p] && (ready || rvalid)) + `uvm_fatal("CANCEL", "A cancelled request became visible") + if (cancel_port && pending[p]) begin + event_result(p, 2); + pending[p] = 0; + draining[p] = 1; + cancelled++; + behavior.sample(p, 0, 0, 0, 0, 1, in_use, 0, offers > 1); + end + if (rvalid && !pending[p]) `uvm_fatal("RESPONSE", "Response without an outstanding request") + if (rvalid && rready) begin + event_result(p, 1, t.outputs[offsets[p][8]+:`TSS_DATA_BITS], t.outputs[offsets[p][9]]); + pending[p] = 0; + completed++; + end + eligible = valid && !busy && !cancel_port && (p == `TSS_CLIENTS || in_use); + if (!eligible || ready) overtakes[p] = 0; + else if (fires != 0) begin + overtakes[p]++; + if (overtakes[p] >= `TSS_PORTS) + `uvm_fatal("FAIRNESS", + "Continuously eligible client was overtaken by a full arbitration round") + end + if (valid && ready) begin + logic [63:0] address; + logic [64:0] finish_address, limit_address; + logic [`TSS_DATA_BITS-1:0] value, data; + logic [`TSS_MASK_BITS-1:0] mask, allowed; + int size, count, start; + bit writing, legal; + if (pending[p] || draining[p]) + `uvm_fatal("OUTSTANDING", "Port accepted a second outstanding request") + address = t.inputs[offsets[p][1]+:64]; + size = t.inputs[offsets[p][2]+:`TSS_SIZE_BITS]; + count = 1 << size; + writing = t.inputs[offsets[p][3]]; + data = t.inputs[offsets[p][4]+:`TSS_DATA_BITS]; + mask = t.inputs[offsets[p][5]+:`TSS_MASK_BITS]; + allowed = `TSS_MASK_BITS'((64'd1 << count) - 1); + finish_address = {1'b0, address} + count; + limit_address = {1'b0, `TSS_BASE} + `TSS_BYTES; + legal=count<=`TSS_MASK_BITS && (address&(count-1))==0 && address>=`TSS_BASE && finish_address<=limit_address && !(writing&&((mask&~allowed)!=0)); + value = '0; + if (legal) begin + start = int'(address - `TSS_BASE); + for (int b = 0; b < count; b++) begin + if (writing) begin + if (mask[b]) begin + memory[start+b] = data[b*8+:8]; + initialized[start+b] = 1; + end + end else begin + if (!initialized[start+b]) `uvm_fatal("REFERENCE", "Read of an uninitialized byte") + value[b*8+:8] = memory[start+b]; + end + end + end + event_result(p, 0, value, !legal); + pending[p] = 1; + accepted++; + behavior.sample(p, writing, size, mask == 0 ? 0 : (mask == allowed ? 1 : 2), !legal, 0, + in_use, rvalid && !rready, offers > 1); + end else if (rvalid && !rready) behavior.sample(p, 0, 0, 0, 0, 0, in_use, 1, offers > 1); + end + endfunction + function void check_phase(uvm_phase phase); + super.check_phase(phase); + if (accepted != completed + cancelled) + `uvm_fatal("CONSERVATION", "Accepted requests did not retire or cancel") + foreach (pending[p]) + if (pending[p] || draining[p]) `uvm_fatal("PENDING", "Port has not drained") + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/tss/src/main/resources/filelist.f b/arch/src/main/scala/framework/core/tss/src/main/resources/filelist.f new file mode 100644 index 000000000..ffef9daea --- /dev/null +++ b/arch/src/main/scala/framework/core/tss/src/main/resources/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/Store/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/tss/src/main/resources/interface.sv b/arch/src/main/scala/framework/core/tss/src/main/resources/interface.sv new file mode 100644 index 000000000..70c5fbe60 --- /dev/null +++ b/arch/src/main/scala/framework/core/tss/src/main/resources/interface.sv @@ -0,0 +1,17 @@ +`include "parameters.svh" +interface tss_if ( + input logic clock +); + localparam int PORT_OFFSETS[`TSS_PORTS][13] = `TSS_PORT_OFFSETS; + logic reset = 1; + logic [`TSS_INPUT_BITS-1:0] inputs = '0; + logic [`TSS_OUTPUT_BITS-1:0] outputs; + for (genvar p = 0; p < `TSS_PORTS; p++) begin + assert property (@(posedge clock) disable iff(reset || (p<`TSS_CLIENTS && inputs[PORT_OFFSETS[p][12]])) + outputs[PORT_OFFSETS[p][7]] && !inputs[PORT_OFFSETS[p][10]] |=> outputs[PORT_OFFSETS[p][7]] && + $stable( + {outputs[PORT_OFFSETS[p][8]+:`TSS_DATA_BITS], outputs[PORT_OFFSETS[p][9]]} + )) + else $fatal(1, "TSS port response changed under backpressure"); + end +endinterface diff --git a/arch/src/main/scala/framework/core/tss/src/main/resources/package.sv b/arch/src/main/scala/framework/core/tss/src/main/resources/package.sv new file mode 100644 index 000000000..a3c5a5347 --- /dev/null +++ b/arch/src/main/scala/framework/core/tss/src/main/resources/package.sv @@ -0,0 +1,13 @@ +`include "parameters.svh" +package tss_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/tss/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/core/tss/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..10ea4f77a --- /dev/null +++ b/arch/src/main/scala/framework/core/tss/src/main/resources/seq/sequence.svh @@ -0,0 +1,164 @@ +class scenario_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(scenario_sequence) + const int offsets[`TSS_PORTS][13] = `TSS_PORT_OFFSETS; + logic [`TSS_INPUT_BITS-1:0] pins; + logic [`TSS_OUTPUT_BITS-1:0] observed; + int serial; + function new(string name = "scenario_sequence"); + super.new(name); + endfunction + task tick(bit reset = 0); + cycle_item t = cycle_item::type_id::create("cycle"); + start_item(t); + t.inputs = pins; + t.reset = reset; + finish_item(t); + observed = t.outputs; + endtask + task request(int p, logic [63:0] address, int size, bit writing, logic [`TSS_DATA_BITS-1:0] data, + logic [`TSS_MASK_BITS-1:0] mask); + pins[offsets[p][0]] = 1; + pins[offsets[p][1]+:64] = address; + pins[offsets[p][2]+:`TSS_SIZE_BITS] = size; + pins[offsets[p][3]] = writing; + pins[offsets[p][4]+:`TSS_DATA_BITS] = data; + pins[offsets[p][5]+:`TSS_MASK_BITS] = mask; + endtask + task accept(int p); + do tick(); while (!observed[offsets[p][6]]); + pins[offsets[p][0]] = 0; + endtask + task access (int p, logic [63:0] address, int size, bit writing = 0, + logic [`TSS_DATA_BITS-1:0] data = '0, logic [`TSS_MASK_BITS-1:0] mask = '0); + request(p, address, size, writing, data, mask); + accept(p); + do tick(); while (!observed[offsets[p][7]]); + endtask + task together(bit writing, bit hold_responses); + bit waiting[`TSS_PORTS]; + bit any_waiting; + for (int p = 0; p < `TSS_PORTS; p++) begin + request(p, `TSS_BASE + p * `TSS_MASK_BITS, $clog2(`TSS_MASK_BITS), writing, + `TSS_DATA_BITS'(serial * 1234567 + p), '1); + pins[offsets[p][10]] = !hold_responses; + waiting[p] = 1; + end + serial++; + do begin + tick(); + any_waiting = 0; + for (int p = 0; p < `TSS_PORTS; p++) begin + if (waiting[p] && observed[offsets[p][6]]) begin + waiting[p] = 0; + pins[offsets[p][0]] = 0; + end + any_waiting |= waiting[p]; + end + end while (any_waiting); + if (hold_responses) begin + bit all_valid; + do begin + tick(); + all_valid = 1; + for (int p = 0; p < `TSS_PORTS; p++) all_valid &= observed[offsets[p][7]]; + end while (!all_valid); + end else repeat (8) tick(); + endtask + task body(); + pins = '0; + for (int p = 0; p < `TSS_PORTS; p++) pins[offsets[p][10]] = 1; + repeat (2) tick(1); + tick(); + for (int p = 0; p < `TSS_CLIENTS; p++) + request(p, `TSS_BASE + p * `TSS_MASK_BITS, $clog2(`TSS_MASK_BITS), 0, '0, '0); + for (int row = 0; row < `TSS_BYTES / `TSS_MASK_BITS; row++) begin + logic [`TSS_DATA_BITS-1:0] pattern; + for (int b = 0; b < `TSS_MASK_BITS; b++) pattern[b*8+:8] = 8'(row * 17 + b); + access (`TSS_CLIENTS, `TSS_BASE + row * `TSS_MASK_BITS, $clog2(`TSS_MASK_BITS), 1, pattern, + '1); + end + pins[`TSS_IN_INUSE_OFFSET] = 1; + begin + bit waiting[`TSS_CLIENTS]; + bit any_waiting; + foreach (waiting[p]) waiting[p] = 1; + do begin + tick(); + any_waiting = 0; + foreach (waiting[p]) begin + if (waiting[p] && observed[offsets[p][6]]) begin + waiting[p] = 0; + pins[offsets[p][0]] = 0; + end + any_waiting |= waiting[p]; + end + end while (any_waiting); + repeat (8) tick(); + end + for (int size = 0; size <= $clog2(`TSS_MASK_BITS); size++) + for (int address = 0; address < `TSS_BYTES; address += 1 << size) + access ((address + size) % `TSS_PORTS, `TSS_BASE + address, size); + for (int b = 0; b < `TSS_MASK_BITS; b++) begin + access (b % `TSS_PORTS, `TSS_BASE, $clog2(`TSS_MASK_BITS), 1, ~`TSS_DATA_BITS'(b), + `TSS_MASK_BITS'(1 << b)); + access ((b + 1) % `TSS_PORTS, `TSS_BASE, $clog2(`TSS_MASK_BITS)); + end + access (0, `TSS_BASE, $clog2(`TSS_MASK_BITS), 1, '1, '0); + access (0, `TSS_BASE, $clog2(`TSS_MASK_BITS)); + repeat (32) together(serial % 2, 0); + pins[offsets[0][10]] = 0; + request(0, `TSS_BASE, 0, 0, '0, '0); + accept(0); + do tick(); while (!observed[offsets[0][7]]); + for (int n = 0; n < 10; n++) access (1 + n % (`TSS_PORTS - 1), `TSS_BASE + 64 + n, 0); + pins[offsets[0][10]] = 1; + tick(); + together(1, 1); + pins[offsets[0][12]] = 1; + tick(); + pins[offsets[0][12]] = 0; + for (int p = 1; p < `TSS_PORTS; p++) pins[offsets[p][10]] = 1; + repeat (8) tick(); + pins[offsets[0][10]] = 1; + access (0, `TSS_BASE, $clog2(`TSS_MASK_BITS)); + for (int p = 0; p < `TSS_CLIENTS; p++) begin + pins[offsets[p][10]] = 0; + request(p, `TSS_BASE + 32 + p * `TSS_MASK_BITS, $clog2(`TSS_MASK_BITS), 1, + `TSS_DATA_BITS'(64'hcafef00d + p), '1); + accept(p); + pins[offsets[p][12]] = 1; + tick(); + pins[offsets[p][12]] = 0; + do tick(); while (observed[offsets[p][11]]); + pins[offsets[p][10]] = 1; + access (p, `TSS_BASE + 32 + p * `TSS_MASK_BITS, $clog2(`TSS_MASK_BITS)); + end + together(1, 1); + repeat (2) tick(1); + pins = '0; + pins[`TSS_IN_INUSE_OFFSET] = 1; + for (int p = 0; p < `TSS_PORTS; p++) pins[offsets[p][10]] = 1; + tick(); + for (int p = 0; p < `TSS_PORTS; p++) + access (p, `TSS_BASE + p * `TSS_MASK_BITS, $clog2(`TSS_MASK_BITS)); + pins[offsets[0][10]] = 0; + request(0, `TSS_BASE, 0, 0, '0, '0); + accept(0); + pins[`TSS_IN_INUSE_OFFSET] = 0; + pins[offsets[0][10]] = 1; + repeat (6) tick(); + request(0, `TSS_BASE, 0, 0, '0, '0); + access (`TSS_CLIENTS, `TSS_BASE + 128, 0); + pins[offsets[0][0]] = 0; + pins[`TSS_IN_INUSE_OFFSET] = 1; + for (int p = 0; p < `TSS_PORTS; p++) begin + access (p, `TSS_BASE - 1, 0); + access (p, `TSS_BASE + `TSS_BYTES, 0); + access (p, 64'hffffffffffffffff, 0); + access (p, `TSS_BASE + 1, 1); + access (p, `TSS_BASE, $clog2(`TSS_MASK_BITS) + 1); + access (p, `TSS_BASE, 0, 1, '1, `TSS_MASK_BITS'(2)); + end + repeat (12) tick(); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/tss/src/main/resources/suite.toml b/arch/src/main/scala/framework/core/tss/src/main/resources/suite.toml new file mode 100644 index 000000000..6b77c4d9a --- /dev/null +++ b/arch/src/main/scala/framework/core/tss/src/main/resources/suite.toml @@ -0,0 +1,20 @@ +[model] +manifest = "" +library = "" + +[[tests]] +name = "tss" +top = "tss_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 200000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line","cond","tgl"] +functional_groups = ["tss_pkg::model::behavior"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/core/tss/src/main/resources/tb.sv b/arch/src/main/scala/framework/core/tss/src/main/resources/tb.sv new file mode 100644 index 000000000..ac79ca1a7 --- /dev/null +++ b/arch/src/main/scala/framework/core/tss/src/main/resources/tb.sv @@ -0,0 +1,16 @@ +module tss_tb; + import uvm_pkg::*; + import tss_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + tss_if bus (clock); + Store dut ( + .clock(clock), + .reset(bus.reset), + `include "ports.svh" + ); + initial begin + uvm_config_db#(virtual tss_if)::set(null, "uvm_test_top*", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/core/tss/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/core/tss/src/main/resources/tests/test.svh new file mode 100644 index 000000000..54d137a6f --- /dev/null +++ b/arch/src/main/scala/framework/core/tss/src/main/resources/tests/test.svh @@ -0,0 +1,15 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.start(test_env.seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/tss/src/main/scala/Emit.scala b/arch/src/main/scala/framework/core/tss/src/main/scala/Emit.scala new file mode 100644 index 000000000..c492cad35 --- /dev/null +++ b/arch/src/main/scala/framework/core/tss/src/main/scala/Emit.scala @@ -0,0 +1,88 @@ +package framework.core.tss + +import chisel3._ +import java.nio.file.{Files, Paths} +import framework.ip.memory.spm +import framework.core.tss.configs.Params + +object Emit extends App { + val Array("--config", config, "--output-dir", output) = args + val values = ujson.read(Files.readString(Paths.get(config))) + val mem = values("memory") + + val p = Params( + spm.Params(BigInt(mem("base").str), mem("bytes").num.toInt, mem("dataBits").num.toInt), + values("ports").num.toInt + ) + + Files.createDirectories(Paths.get(output)) + val ports = scala.collection.mutable.ArrayBuffer[String]() + val definitions = scala.collection.mutable.ArrayBuffer[String]() + val offsets = scala.collection.mutable.Map[String, Int]() + var inputOffset = 0 + var outputOffset = 0 + + def fields(data: Data, path: String): Seq[(String, Int)] = data match { + case record: Record => record.elements.toSeq.flatMap { case (name, field) => + fields(field, if (path.isEmpty) name else s"${path}_$name") + } + case vector: Vec[_] => vector.toSeq.zipWithIndex.flatMap { case (field, index) => fields(field, s"${path}_$index") } + case leaf => Seq(path -> leaf.getWidth) + } + + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + { + val dut = new Store(p) + for ((name, width) <- fields(dut.io, "")) { + val input = name == "inUse" || name.startsWith("cancel_") || name.endsWith("request_valid") || name.contains( + "request_bits_" + ) || name.endsWith("response_ready") + val direction = if (input) "IN" else "OUT" + val signal = if (input) "inputs" else "outputs" + val offset = if (input) inputOffset else outputOffset + definitions += s"`define TSS_${direction}_${name.toUpperCase}_OFFSET $offset" + definitions += s"`define TSS_${direction}_${name.toUpperCase}_WIDTH $width" + ports += s".io_$name(bus.$signal[$offset +: $width])" + offsets(name) = offset + if (input) inputOffset += width else outputOffset += width + } + dut + }, + args = Array("--target-dir", s"$output/Store", "--split-verilog") + ) + definitions += s"`define TSS_INPUT_BITS $inputOffset" + definitions += s"`define TSS_OUTPUT_BITS $outputOffset" + definitions += s"`define TSS_CLIENTS ${p.ports}" + definitions += s"`define TSS_PORTS ${p.ports + 1}" + definitions += s"`define TSS_DATA_BITS ${p.memory.dataBits}" + definitions += s"`define TSS_MASK_BITS ${p.memory.beatBytes}" + definitions += s"`define TSS_SIZE_BITS ${p.memory.sizeBits}" + definitions += s"`define TSS_BYTES ${p.memory.bytes}" + definitions += s"`define TSS_BASE 64'h${p.memory.base.toString(16)}" + + val portOffsets = (0 to p.ports).map { index => + val name = if (index == p.ports) "host" else s"clients_$index" + val fields = Seq( + "request_valid", + "request_bits_address", + "request_bits_size", + "request_bits_write", + "request_bits_data", + "request_bits_mask", + "request_ready", + "response_valid", + "response_bits_data", + "response_bits_error", + "response_ready" + ).map(field => offsets(s"${name}_$field")) + (fields ++ Seq( + if (index == p.ports) offsets("hostBusy") else offsets(s"busy_$index"), + if (index == p.ports) 0 + else offsets(s"cancel_$index") + )).mkString("'{", ",", "}") + }.mkString("'{", ",", "}") + + definitions += s"`define TSS_PORT_OFFSETS $portOffsets" + Files.writeString(Paths.get(output, "parameters.svh"), definitions.mkString("\n") + "\n") + Files.writeString(Paths.get(output, "ports.svh"), ports.mkString(",\n") + "\n") +} diff --git a/arch/src/main/scala/framework/core/tss/src/main/scala/Store.scala b/arch/src/main/scala/framework/core/tss/src/main/scala/Store.scala new file mode 100644 index 000000000..e88e4be99 --- /dev/null +++ b/arch/src/main/scala/framework/core/tss/src/main/scala/Store.scala @@ -0,0 +1,94 @@ +package framework.core.tss + +import framework.core.tss.configs.{Params => ModuleParam} +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import framework.ip.memory.spm.{Memory, Params => MemoryParams, Port, Response} +import framework.core.tss.configs.Params + +class TssControlPort(b: ModuleParam) extends Bundle { + val p = b + val host = new Port(p.memory) + val inUse = Output(Bool()) + val cancel = Output(Vec(p.ports, Bool())) + val busy = Input(Vec(p.ports, Bool())) + val hostBusy = Input(Bool()) +} + +class TssPort(b: ModuleParam) extends TssControlPort(b) { + val clients = Vec(p.ports, new Port(p.memory)) +} + +/** + * One shared SRAM, fair arbitration, one outstanding request and one response slot per port. + * SRAM service is serialized; a held response does not block other clients. inUse covers the + * lifetime of the cooperating group or a private-store transfer. The host may stage disjoint + * ranges while threads run; software owns those ranges. Cancel drains only its owning port. + */ +@instantiable +class Store(b: ModuleParam) extends Module { + val p = b + + val count = p.ports + 1 + val host = p.ports + val indexBits = log2Ceil(count) + + @public + val io = IO(Flipped(new TssPort(b))) + + val memory = Instantiate(new Memory(p.memory)) + val ports = io.clients.toSeq :+ io.host + val occupied = RegInit(VecInit(Seq.fill(count)(false.B))) + val available = RegInit(VecInit(Seq.fill(count)(false.B))) + val cancelled = RegInit(VecInit(Seq.fill(count)(false.B))) + val answers = Reg(Vec(count, new Response(p.memory))) + val active = RegInit(false.B) + val owner = Reg(UInt(indexBits.W)) + val next = RegInit(0.U(indexBits.W)) + val cancel = VecInit(io.cancel.toSeq :+ false.B) + + val eligible = VecInit(ports.zipWithIndex.map { case (port, i) => + val allowed = + if (i == host) true.B + else io.inUse + port.request.valid && !occupied(i) && !cancel(i) && allowed && !reset.asBool + }) + + val after = VecInit((0 until count).map(i => eligible(i) && i.U >= next)).asUInt + val chosen = Mux(after.orR, PriorityEncoder(after), PriorityEncoder(eligible.asUInt)) + memory.io.readOnly := false.B + memory.io.port.request.valid := !active && eligible.asUInt.orR + memory.io.port.request.bits := VecInit(ports.map(_.request.bits))(chosen) + memory.io.port.response.ready := active + when(memory.io.port.request.fire) { + active := true.B + owner := chosen + occupied(chosen) := true.B + cancelled(chosen) := false.B + next := Mux(chosen === (count - 1).U, 0.U, chosen + 1.U) + } + for ((port, i) <- ports.zipWithIndex) { + port.request.ready := !active && eligible(i) && chosen === i.U && memory.io.port.request.ready + port.response.valid := available(i) && !cancelled(i) && !cancel(i) && !reset.asBool + port.response.bits := answers(i) + when(cancel(i) && occupied(i))(cancelled(i) := true.B) + when(port.response.fire || (cancel(i) && available(i))) { + available(i) := false.B + occupied(i) := false.B + cancelled(i) := false.B + } + } + when(memory.io.port.response.fire) { + active := false.B + when(cancelled(owner) || cancel(owner)) { + occupied(owner) := false.B + cancelled(owner) := false.B + }.otherwise { + answers(owner) := memory.io.port.response.bits + available(owner) := true.B + } + } + io.busy := VecInit(occupied.take(p.ports)) + io.hostBusy := occupied(host) +} diff --git a/arch/src/main/scala/framework/core/tss/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/core/tss/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..f19b5823c --- /dev/null +++ b/arch/src/main/scala/framework/core/tss/src/main/scala/configs/Params.scala @@ -0,0 +1,13 @@ +package framework.core.tss.configs + +import framework.ip.memory.spm.{Params => MemoryParams} +import upickle.default.{macroRW, ReadWriter} + +case class Params(memory: MemoryParams, ports: Int) { + require(ports > 0) +} + +object Params { + implicit val memoryRW: ReadWriter[MemoryParams] = macroRW + implicit val rw: ReadWriter[Params] = macroRW +} diff --git a/arch/src/main/scala/framework/device/build.sc b/arch/src/main/scala/framework/device/build.sc new file mode 100644 index 000000000..2ae5a0909 --- /dev/null +++ b/arch/src/main/scala/framework/device/build.sc @@ -0,0 +1,22 @@ +import mill._ +import $file.cbus.{build => cbus_build} +import $file.ddr.{build => ddr_build} +import $file.dlink.{build => dlink_build} +import $file.peripherals.{build => peripherals_build} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("configs/DeviceParam.scala")) + ) ++ + cbus_build.sources() ++ + ddr_build.sources() ++ + dlink_build.sources() ++ + peripherals_build.sources() +) + +def emitSources = T.sources( + cbus_build.emitSources() ++ + ddr_build.emitSources() ++ + dlink_build.emitSources() ++ + peripherals_build.emitSources() +) diff --git a/arch/src/main/scala/framework/device/cbus/C2C.scala b/arch/src/main/scala/framework/device/cbus/C2C.scala new file mode 100644 index 000000000..15bf97ef3 --- /dev/null +++ b/arch/src/main/scala/framework/device/cbus/C2C.scala @@ -0,0 +1,134 @@ +package framework.device.cbus + +import framework.device.cbus.C2CParams + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import framework.core.tbus.router.PhysicalPort +import framework.ip.bus.axi4 +import framework.lib.queue.Queue + +case class C2CParams( + sharedBytes: BigInt, + bankBytes: Int, + cores: Int, + axi: axi4.Params) { + require(sharedBytes > 0 && bankBytes >= 16 && isPow2(bankBytes) && sharedBytes % bankBytes == 0) + require(cores > 0 && axi.dataBits == 128) + val sharedAddressBits: Int = math.max(1, log2Ceil(sharedBytes)) + val coreBits: Int = math.max(1, log2Ceil(cores)) + require(sharedAddressBits + coreBits <= axi.addressBits && sharedAddressBits <= 32) +} + +class C2CCommand(b: C2CParams) extends Bundle { + val p = b + + val sourceByteAddress = UInt(64.W) + val targetCore = UInt(p.coreBits.W) + val targetByteAddress = UInt(64.W) + val bytes = UInt(32.W) +} + +/** + * Push shared SRAM bytes to another Core. Starts are 16-byte aligned; the final beat may be partial. + * One command is active. Bank mappings remain owned by the caller through completion. + * Reset is coordinated with the fabric and SRAM ports, not an independent transaction cancellation. + */ +@instantiable +class C2CTransfer(b: C2CParams) extends Module { + val p = b + + @public + val io = IO(new Bundle { + val command = Flipped(Decoupled(new C2CCommand(b))) + val completion = Decoupled(Bool()) // false = no error, after the final destination write response + val source = + new PhysicalPort(b.axi.dataBits) + val mem = new axi4.Port(p.axi) + }) + + val idle :: plan :: stream :: reply :: complete :: Nil = Enum(5) + val state = RegInit(idle) + val source = Reg(UInt(p.sharedAddressBits.W)) + val target = Reg(UInt(p.sharedAddressBits.W)) + val core = Reg(UInt(p.coreBits.W)) + val remaining = Reg(UInt((p.sharedAddressBits + 1).W)) + val burstBytes = Reg(UInt(13.W)) + val beats = Reg(UInt(9.W)) + val issued = RegInit(0.U(9.W)) + val sent = RegInit(0.U(9.W)) + val pending = RegInit(false.B) + val awSent = RegInit(false.B) + val data = Instantiate(new Queue(UInt(128.W), 2, false, false)) + val live = !reset.asBool + io.command.ready := state === idle && live + io.completion.valid := state === complete && live + io.completion.bits := false.B + when(io.completion.fire)(state := idle) + when(io.command.fire) { + val cmd = io.command.bits + assert(cmd.bytes =/= 0.U && cmd.sourceByteAddress(3, 0) === 0.U && cmd.targetByteAddress(3, 0) === 0.U) + assert(cmd.targetCore < p.cores.U, "C2C target must be a Core system ID") + assert( + cmd.sourceByteAddress +& cmd.bytes <= p.sharedBytes.U && cmd.targetByteAddress +& cmd.bytes <= p.sharedBytes.U, + "C2C command exceeds Core shared SRAM" + ) + source := cmd.sourceByteAddress; target := cmd.targetByteAddress + core := cmd.targetCore; remaining := cmd.bytes; state := plan + } + def minimum(a: UInt, b: UInt): UInt = Mux(a < b, a, b) + val sourcePage = 4096.U(13.W) - source.pad(12)(11, 0) + val targetPage = 4096.U(13.W) - target.pad(12)(11, 0) + val sourceBank = p.bankBytes.U - (source & (p.bankBytes - 1).U) + val targetBank = p.bankBytes.U - (target & (p.bankBytes - 1).U) + val nextBytes = Seq(remaining, sourcePage, targetPage, sourceBank, targetBank, 4096.U).reduce(minimum) + when(state === plan) { + burstBytes := nextBytes; beats := (nextBytes +& 15.U) >> 4 + issued := 0.U; sent := 0.U; pending := false.B; awSent := false.B; state := stream + assert(data.io.count === 0.U && nextBytes =/= 0.U) + } + io.mem.aw.valid := state === stream && !awSent && live + io.mem.aw.bits := 0.U.asTypeOf(new axi4.Address(p.axi)) + io.mem.aw.bits.addr := Cat(core, target(p.sharedAddressBits - 1, 0)) + io.mem.aw.bits.len := beats - 1.U + io.mem.aw.bits.size := 4.U + io.mem.aw.bits.burst := 1.U + when(io.mem.aw.fire)(awSent := true.B) + io.source.request.valid := state === stream && issued < beats && !pending && data.io.count < 2.U && live + io.source.request.bits.address := source + (issued << 4) + io.source.request.bits.write := false.B + io.source.request.bits.data := 0.U + io.source.request.bits.mask := 0.U + when(io.source.request.fire) { pending := true.B; issued := issued + 1.U } + data.io.enq.valid := state === stream && pending && io.source.response.valid && !io.source.response.bits.error && live + data.io.enq.bits := io.source.response.bits.data + io.source.response.ready := state === stream && pending && data.io.enq.ready && live + when(io.source.response.valid && live) { + assert(pending && !io.source.response.bits.error, "C2C shared source read failed") + } + when(io.source.response.fire)(pending := false.B) + val last = sent +& 1.U === beats + val tail = burstBytes(3, 0) + io.mem.w.valid := state === stream && awSent && data.io.deq.valid && live + io.mem.w.bits.data := data.io.deq.bits + io.mem.w.bits.strb := Mux(last && tail =/= 0.U, (1.U(17.W) << tail) - 1.U, "hffff".U) + io.mem.w.bits.last := last + data.io.deq.ready := state === stream && awSent && io.mem.w.ready && live + when(io.mem.w.fire) { sent := sent + 1.U; when(last)(state := reply) } + io.mem.b.ready := state === reply && live + when(io.mem.b.fire) { + assert(io.mem.b.bits.id === 0.U && io.mem.b.bits.resp === 0.U, "C2C destination write failed") + assert(!pending && data.io.count === 0.U && issued === beats && sent === beats) + when(io.mem.b.bits.id === 0.U && io.mem.b.bits.resp === 0.U) { + when(remaining === burstBytes)(state := complete).otherwise { + source := source + burstBytes; target := target + burstBytes + remaining := remaining - burstBytes; state := plan + } + } + } + io.mem.ar.valid := false.B + io.mem.ar.bits := 0.U.asTypeOf(new axi4.Address(p.axi)) + io.mem.r.ready := false.B + when(live)(assert(!io.mem.r.valid, "C2C push engine received an unsolicited AXI read response")) +} diff --git a/arch/src/main/scala/framework/device/cbus/CBus.scala b/arch/src/main/scala/framework/device/cbus/CBus.scala new file mode 100644 index 000000000..d2326c7c2 --- /dev/null +++ b/arch/src/main/scala/framework/device/cbus/CBus.scala @@ -0,0 +1,148 @@ +package framework.device.cbus + +import sims.configs.SimParam +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import framework.device.configs.DeviceParam +import framework.core.configs.CoreParam +import buckyball.config.{CorePlacement, Design} +import scala.jdk.CollectionConverters._ +import framework.core.clink.{CLinkIO, CoreInterrupts} +import framework.device.ddr.Bridge +import framework.ip.bus.axi4 +import framework.ip.bus.chi.RequesterPort +import framework.ip.bus.chi.snf.LineSramIO +import framework.device.dlink.BaseDevice + +/** Routes core memory requests, shared-storage transfers and hardware interrupts. */ +@instantiable +class CBus(b: DeviceParam, generators: Seq[() => CLinkIO]) extends BaseDevice(b.ddr.axi) { + require(generators.size == b.cores.size) + val corePorts = generators.map(_()) + val ports = corePorts + require( + corePorts.zip(b.cores).forall { case (port, corep) => port.p == corep.clink }, + "Core implementation must expose its configured CLink" + ) + require( + ports.forall(_.p.ddr == b.ddr.axi), + "CBus DDR connections require matching AXI protocols or an explicit adapter" + ) + require( + ports.forall(_.p.uncached == b.ddr.axi), + "CBus uncached connections require matching AXI protocols or an explicit adapter" + ) + require( + ports.filter(port => port.p.backing || port.p.requesters > 0).forall(_.p.line == b.ddr.line), + "CBus line connections require matching CHI protocols or an explicit adapter" + ) + val backingPorts = corePorts.flatMap(_.memory.backing.toSeq) + val coherentNodes = corePorts.flatMap(port => port.p.requesterIds.zip(port.memory.coherent)) + require( + coherentNodes.map(_._1).sorted == (1 to coherentNodes.size), + "CHI Home requires one connection for every requester node ID" + ) + val coherentPorts = coherentNodes.sortBy(_._1).map(_._2) + val bankPorts = corePorts.flatMap(_.banks.toSeq) + @public + val banks = IO(MixedVec(ports.flatMap(_.banks.toSeq).map(_.cloneType))) + bankPorts.zip(banks).foreach { case (core, fabric) => fabric <> core } + val hartCounts = corePorts.map(_.interrupts.size) + val hartBase = hartCounts.scanLeft(0)(_ + _) + + @public + val backing = IO(Vec(backingPorts.size, Flipped(new LineSramIO(b.ddr.line)))) + @public + val coherent = IO(Vec(coherentPorts.size, new RequesterPort(b.ddr.line))) + @public + val memory = IO(Vec(backingPorts.size, new LineSramIO(b.ddr.line))) + @public + val uncached = IO(new axi4.Port(b.ddr.axi)) + @public + val external = IO(Flipped(new axi4.Port(b.ddr.axi))) + @public + val interrupts = IO(Input(Vec(hartCounts.sum, new CoreInterrupts))) + + for ((port, index) <- backingPorts.zipWithIndex) { + backing(index).req <> port.request + port.response <> backing(index).resp + } + for ((port, index) <- coherentPorts.zipWithIndex) coherent(index) <> port + for { + (core, index) <- corePorts.zipWithIndex + hart <- core.interrupts.indices + } { + core.interrupts(hart) := interrupts(hartBase(index) + hart) + } + + val linked = corePorts.indices.filter(i => corePorts(i).c2c.isDefined) + if (linked.nonEmpty) { + val fabric = Instantiate(new Fabric(b)) + for ((core, index) <- linked.zipWithIndex) { + fabric.io.source(index) <> corePorts(core).c2c.get.tx + corePorts(core).c2c.get.rx <> fabric.io.target(index) + } + } + + val bridgeMasters = if (backingPorts.nonEmpty) 1 else 0 + + val memoryBus = + Instantiate(new axi4.Interconnect(axi4.InterconnectParam(b.ddr.axi, corePorts.size + bridgeMasters + 1, None))) + + if (backingPorts.nonEmpty) { + require(b.ddr.clients == backingPorts.size) + val bridge = Instantiate(new Bridge(b.ddr)) + memoryBus.io.in(0) <> bridge.io.axi + for (index <- backingPorts.indices) { + bridge.io.request(index) <> memory(index).req + memory(index).resp <> bridge.io.response(index) + } + } + for ((core, index) <- corePorts.zipWithIndex) memoryBus.io.in(bridgeMasters + index) <> core.memory.ddr + memoryBus.io.in(corePorts.size + bridgeMasters) <> external + dlink.axi <> memoryBus.io.out + + val cpuMemory = Instantiate(new axi4.Interconnect(axi4.InterconnectParam(b.ddr.axi, corePorts.size, None))) + + for ((core, index) <- corePorts.zipWithIndex) cpuMemory.io.in(index) <> core.memory.uncached + uncached <> cpuMemory.io.out +} + +trait CoreFactory { + def component: String + def parameters(design: Design, source: CorePlacement): CoreParam + + def instantiate( + source: CorePlacement, + corep: CoreParam, + simp: SimParam, + traceIds: Seq[Int] + ): CLinkIO + +} + +object CBus { + + def parameters(design: Design, components: Map[String, CoreFactory]): Seq[CoreParam] = + design.getDevice.getCoresList.asScala.map { source => + components(source.getComponent).parameters(design, source) + }.toSeq + + def generators( + design: Design, + devicep: DeviceParam, + simp: SimParam, + components: Map[String, CoreFactory] + ): Seq[() => CLinkIO] = { + val sources = design.getDevice.getCoresList.asScala.toSeq + require(sources.size == devicep.cores.size) + sources.zip(devicep.cores).map { case (source, corep) => + require(source.getHartId == corep.hartId, "Core configuration order does not match design.pb") + val factory = components(source.getComponent) + val traceIds = source.getThreadIndicesList.asScala.map(_.intValue + sources.size).toSeq + () => factory.instantiate(source, corep, simp, traceIds) + } + } + +} diff --git a/arch/src/main/scala/framework/device/cbus/Control.scala b/arch/src/main/scala/framework/device/cbus/Control.scala new file mode 100644 index 000000000..95d8717d3 --- /dev/null +++ b/arch/src/main/scala/framework/device/cbus/Control.scala @@ -0,0 +1,145 @@ +package framework.device.cbus + +import framework.device.cbus.C2CParams +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.core.tbus.router.{LeasePort, PhysicalPort} + +/** The controller stages one transfer; the start instruction returns after destination commit. */ +class ControlRequest extends Bundle { + val operation = UInt(7.W) + val context = UInt(32.W) + val field = UInt(32.W) + val data = UInt(64.W) +} + +@instantiable +class Control( + memParam: MemDomainParam, + transferParam: C2CParams, + leaseAccess: Boolean, + localAccess: Boolean) + extends Module { + val p = transferParam + + @public + val io = IO(new Bundle { + val request = Flipped(Decoupled(new ControlRequest)) + val reply = Decoupled(UInt(64.W)) + val coreId = Input(UInt(p.coreBits.W)) + val command = Decoupled(new C2CCommand(transferParam)) + val completion = Flipped(Decoupled(Bool())) + val idle = Output(Bool()) + val local = Option.when(localAccess)(new PhysicalPort(memParam.bankWidth)) + val lease = Option.when(leaseAccess)(new LeasePort) + }) + + val idle :: running :: reading :: leasing :: responding :: Nil = Enum(5) + val state = RegInit(idle) + val fields = RegInit(0.U(4.W)) + val staged = Reg(Vec(4, UInt(64.W))) + val answer = Reg(UInt(64.W)) + val request = io.request.bits + val start = request.operation === 14.U + val access = request.operation === 16.U || request.operation === 17.U + val lease = request.operation === 18.U || request.operation === 19.U + val upperHalf = Reg(Bool()) + val localWrite = Reg(Bool()) + val localReady = io.local.map(_.request.ready).getOrElse(false.B) + val leaseReady = io.lease.map(_.request.ready).getOrElse(false.B) + io.lease.foreach { port => + port.request.valid := state === idle && io.request.valid && lease && !reset.asBool + port.request.bits.endpoint := request.context + port.request.bits.vbank := request.field(31, 16) + port.request.bits.group := request.field(15, 0) + port.request.bits.release := request.operation === 19.U + port.response.ready := state === leasing && !reset.asBool + when(port.response.fire) { + answer := port.response.bits + state := responding + } + } + io.local.foreach { port => + val address = Cat(request.context, request.field) + port.request.valid := state === idle && io.request.valid && access && !reset.asBool + port.request.bits.address := address & ~15.U(64.W) + port.request.bits.write := request.operation === 17.U + port.request.bits.data := Cat(request.data, request.data) + port.request.bits.mask := Mux(address(3), "hff00".U, "h00ff".U) + port.response.ready := state === reading && !reset.asBool + when(port.response.fire) { + assert(!port.response.bits.error, "Main shared-storage access failed") + answer := Mux(localWrite, 0.U, Mux(upperHalf, port.response.bits.data(127, 64), port.response.bits.data(63, 0))) + state := responding + } + } + io.command.valid := state === idle && io.request.valid && start && !reset.asBool + io.command.bits.sourceByteAddress := staged(0) + io.command.bits.targetCore := staged(1) + io.command.bits.targetByteAddress := staged(2) + io.command.bits.bytes := staged(3) + io.request.ready := state === idle && !reset.asBool && + Mux(lease, leaseReady, Mux(access, localReady, !start || io.command.ready)) + io.reply.valid := state === responding && !reset.asBool + io.reply.bits := answer + io.completion.ready := state === running && !reset.asBool + io.idle := state === idle + when(io.request.valid && state === idle && !reset.asBool) { + assert( + (request.operation >= 13.U && request.operation <= 15.U) || + (localAccess.B && access) || (leaseAccess.B && lease), + "Invalid CLink management operation" + ) + } + when(io.request.fire) { + when(!access && !lease)(assert(request.context === 0.U, "CLink management has one core-owned transfer context")) + answer := 0.U + state := responding + when(lease) { + assert(request.data === 0.U, "CLink shared lease reserves rs2") + state := leasing + } + when(access) { + val address = Cat(request.context, request.field) + assert(address(2, 0) === 0.U && (address +& 8.U) <= p.sharedBytes.U, "Invalid main shared-storage access") + upperHalf := address(3) + localWrite := request.operation === 17.U + state := reading + } + switch(request.operation) { + is(13.U) { + assert(request.field < 4.U, "Invalid CLink transfer descriptor field") + staged(request.field(1, 0)) := request.data + fields := fields | UIntToOH(request.field, 4) + } + is(14.U) { + assert(request.field === 0.U, "CLink start reserves the field selector") + assert(fields.andR, "Incomplete CLink transfer descriptor") + assert( + staged(1) < p.cores.U && staged(1) =/= io.coreId, + "CLink transfer must target another core" + ) + assert(staged(3) > 0.U && (staged(3) >> 32) === 0.U, "Invalid CLink transfer size") + fields := 0.U + state := running + } + is(15.U) { + assert(request.field < 4.U, "Invalid CLink geometry query") + answer := MuxLookup(request.field, 0.U)(Seq( + 0.U -> p.sharedBytes.U(64.W), + 1.U -> p.bankBytes.U(64.W), + 2.U -> io.coreId.pad(64), + 3.U -> p.cores.U(64.W) + )) + } + } + } + when(io.completion.fire) { + assert(!io.completion.bits, "CLink transfer failed") + state := responding + } + when(io.reply.fire)(state := idle) +} diff --git a/arch/src/main/scala/framework/device/cbus/Fabric.scala b/arch/src/main/scala/framework/device/cbus/Fabric.scala new file mode 100644 index 000000000..a0c9306bb --- /dev/null +++ b/arch/src/main/scala/framework/device/cbus/Fabric.scala @@ -0,0 +1,92 @@ +package framework.device.cbus + +import framework.device.configs.DeviceParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import framework.ip.bus.axi4 + +/** Core-addressed shared-storage fabric, separate from the DDR and CPU coherence paths. */ +@instantiable +class Fabric(b: DeviceParam) extends Module { + val cores = b.cores.filter(_.clink.shared) + val p = cores.head.clink.c2c.get + require(cores.forall(_.clink.c2c.contains(p)), "C2C endpoints must use the fabric AXI protocol") + val hartIds = cores.map(_.hartId) + val sharedBytes = cores.head.c2c.sharedBytes + require(cores.forall(_.c2c.sharedBytes == sharedBytes)) + + require(hartIds.nonEmpty && hartIds.distinct.size == hartIds.size && hartIds.forall(_ >= 0)) + require(p.dataBits == 128 && sharedBytes >= p.bytes) + val localBits = log2Ceil(sharedBytes) + val indexBits = math.max(1, log2Ceil(hartIds.size)) + require(p.addressBits >= localBits + log2Ceil(hartIds.max + 1)) + + @public + val io = IO(new Bundle { + val source = Vec(hartIds.size, Flipped(new axi4.Port(p))) + val target = Vec(hartIds.size, new axi4.Port(p)) + }) + + val targets = + hartIds.indices.map(_ => Instantiate(new axi4.Interconnect(axi4.InterconnectParam(p, hartIds.size, None)))) + + for (i <- hartIds.indices) { io.target(i) <> targets(i).io.out } + + for (source <- hartIds.indices) { + val port = io.source(source) + val reading = RegInit(false.B) + val writing = RegInit(false.B) + val readTarget = Reg(UInt(indexBits.W)) + val writeTarget = Reg(UInt(indexBits.W)) + val readMatches = VecInit(hartIds.map(id => (port.ar.bits.addr >> localBits) === id.U)) + val writeMatches = VecInit(hartIds.map(id => (port.aw.bits.addr >> localBits) === id.U)) + val readSelection = PriorityEncoder(readMatches) + val writeSelection = PriorityEncoder(writeMatches) + + def check(address: axi4.Address, matched: Bool): Unit = { + val offset = address.addr(localBits - 1, 0) + val bytes = (address.len +& 1.U) << log2Ceil(p.bytes) + assert(matched, "CLink address does not select a shared-storage core") + assert( + address.size === log2Ceil(p.bytes).U && address.burst === 1.U && !address.lock, + "CLink requires full-width INCR bursts" + ) + assert( + offset(log2Ceil(p.bytes) - 1, 0) === 0.U && (offset +& bytes) <= sharedBytes.U, + "CLink burst exceeds shared storage or is unaligned" + ) + assert((offset.pad(12)(11, 0) +& bytes) <= 4096.U, "CLink burst crosses a 4KiB boundary") + } + when(port.ar.valid && !reading)(check(port.ar.bits, readMatches.asUInt.orR)) + when(port.aw.valid && !writing)(check(port.aw.bits, writeMatches.asUInt.orR)) + when(port.ar.fire) { reading := true.B; readTarget := readSelection } + when(port.r.fire && port.r.bits.last)(reading := false.B) + when(port.aw.fire) { writing := true.B; writeTarget := writeSelection } + when(port.b.fire)(writing := false.B) + + port.ar.ready := !reading && readMatches.asUInt.orR && VecInit(targets.map(_.io.in(source).ar.ready))(readSelection) + port.aw.ready := !writing && writeMatches.asUInt.orR && VecInit(targets.map(_.io.in(source).aw.ready))( + writeSelection + ) + port.w.ready := writing && VecInit(targets.map(_.io.in(source).w.ready))(writeTarget) + port.r.valid := reading && VecInit(targets.map(_.io.in(source).r.valid))(readTarget) + port.r.bits := VecInit(targets.map(_.io.in(source).r.bits))(readTarget) + port.b.valid := writing && VecInit(targets.map(_.io.in(source).b.valid))(writeTarget) + port.b.bits := VecInit(targets.map(_.io.in(source).b.bits))(writeTarget) + for (target <- hartIds.indices) { + val route = targets(target).io.in(source) + route.ar.valid := port.ar.valid && !reading && readMatches(target) + route.ar.bits := port.ar.bits + route.ar.bits.addr := port.ar.bits.addr(localBits - 1, 0) + route.aw.valid := port.aw.valid && !writing && writeMatches(target) + route.aw.bits := port.aw.bits + route.aw.bits.addr := port.aw.bits.addr(localBits - 1, 0) + route.w.valid := port.w.valid && writing && writeTarget === target.U + route.w.bits := port.w.bits + route.r.ready := port.r.ready && reading && readTarget === target.U + route.b.ready := port.b.ready && writing && writeTarget === target.U + } + } +} diff --git a/arch/src/main/scala/framework/system/tile/tlink/SharedAxiEndpoint.scala b/arch/src/main/scala/framework/device/cbus/SharedAxiEndpoint.scala similarity index 83% rename from arch/src/main/scala/framework/system/tile/tlink/SharedAxiEndpoint.scala rename to arch/src/main/scala/framework/device/cbus/SharedAxiEndpoint.scala index e0b24ea05..da03bdd2e 100644 --- a/arch/src/main/scala/framework/system/tile/tlink/SharedAxiEndpoint.scala +++ b/arch/src/main/scala/framework/device/cbus/SharedAxiEndpoint.scala @@ -1,24 +1,30 @@ -package framework.system.tile.tlink +package framework.device.cbus + +import framework.device.cbus.C2CParams import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public} -import framework.memdomain.backend.shared.SharedPhysicalPort -import memcore.bus.axi4 +import framework.core.tbus.router.PhysicalPort +import framework.ip.bus.axi4 /** - * AXI access to the existing Tile shared SRAM. No storage is instantiated here. + * AXI access to the existing Core shared SRAM. No storage is instantiated here. * Read and write bursts each have one outstanding transaction and independent ports. * W acceptance is not completion: B follows the actual final shared-bank write acknowledgement. */ @instantiable -class SharedAxiEndpoint(p: T2TParams) extends Module { +class SharedAxiEndpoint(b: C2CParams) extends Module { + val p = b - @public val io = IO(new Bundle { + @public + val io = IO(new Bundle { val busy = Output(Bool()) val mem = Flipped(new axi4.Port(p.axi)) - val read = new SharedPhysicalPort(128) - val write = new SharedPhysicalPort(128) + val read = + new PhysicalPort(b.axi.dataBits) + val write = + new PhysicalPort(b.axi.dataBits) }) val rIdle :: rIssue :: rWait :: rReply :: Nil = Enum(4) @@ -41,11 +47,11 @@ class SharedAxiEndpoint(p: T2TParams) extends Module { val bytes = (address.len +& 1.U) << 4 assert( address.size === 4.U && address.burst === 1.U && !address.lock && address.addr(3, 0) === 0.U, - "T2T shared SRAM requires unlocked aligned 128-bit INCR bursts" + "C2C shared SRAM requires unlocked aligned 128-bit INCR bursts" ) - assert(address.addr +& bytes <= p.sharedBytes.U, "T2T AXI address exceeds Tile shared SRAM") - assert(address.addr.pad(12)(11, 0) +& bytes <= 4096.U, "T2T AXI burst crosses 4KiB") - assert((address.addr & (p.bankBytes - 1).U) +& bytes <= p.bankBytes.U, "T2T AXI burst crosses a shared bank") + assert(address.addr +& bytes <= p.sharedBytes.U, "C2C AXI address exceeds Core shared SRAM") + assert(address.addr.pad(12)(11, 0) +& bytes <= 4096.U, "C2C AXI burst crosses 4KiB") + assert((address.addr & (p.bankBytes - 1).U) +& bytes <= p.bankBytes.U, "C2C AXI burst crosses a shared bank") } io.mem.ar.ready := readState === rIdle && live @@ -62,7 +68,7 @@ class SharedAxiEndpoint(p: T2TParams) extends Module { when(io.read.request.fire)(readState := rWait) io.read.response.ready := readState === rWait && live when(io.read.response.fire) { - assert(!io.read.response.bits.error, "T2T shared SRAM read failed") + assert(!io.read.response.bits.error, "C2C shared SRAM read failed") when(!io.read.response.bits.error) { readData := io.read.response.bits.data; readState := rReply } } io.mem.r.valid := readState === rReply && live @@ -84,7 +90,7 @@ class SharedAxiEndpoint(p: T2TParams) extends Module { } io.mem.w.ready := writeState === wData && live when(io.mem.w.fire) { - assert(io.mem.w.bits.last === (writeRemaining === 1.U), "T2T AXI WLAST does not match AWLEN") + assert(io.mem.w.bits.last === (writeRemaining === 1.U), "C2C AXI WLAST does not match AWLEN") writeData := io.mem.w.bits.data; writeMask := io.mem.w.bits.strb; writeState := wIssue } io.write.request.valid := writeState === wIssue && live @@ -95,7 +101,7 @@ class SharedAxiEndpoint(p: T2TParams) extends Module { when(io.write.request.fire)(writeState := wWait) io.write.response.ready := writeState === wWait && live when(io.write.response.fire) { - assert(!io.write.response.bits.error, "T2T shared SRAM write failed") + assert(!io.write.response.bits.error, "C2C shared SRAM write failed") when(!io.write.response.bits.error) { when(writeRemaining === 1.U)(writeState := wReply).otherwise { writeAddress := writeAddress + 16.U; writeRemaining := writeRemaining - 1.U; writeState := wData diff --git a/arch/src/main/scala/framework/device/cbus/build.sc b/arch/src/main/scala/framework/device/cbus/build.sc new file mode 100644 index 000000000..d30e2cfd7 --- /dev/null +++ b/arch/src/main/scala/framework/device/cbus/build.sc @@ -0,0 +1,46 @@ +import mill._ + +// The registry is compiled with its consumers; framework never imports component implementations. +def registry(sources: Seq[PathRef], destination: os.Path): Seq[PathRef] = { + val files = sources.flatMap { source => + if (os.isDir(source.path)) + os.walk(source.path).filter(p => p.ext == "scala" && !p.segments.exists(Set("out", "generated", "target", "build"))) + else Seq(source.path) + } + val factories = files.flatMap { source => + val text = os.read(source) + val names = "(?m)^object (\\w+) extends CoreFactory".r.findAllMatchIn(text).map(_.group(1)).toSeq + if (names.isEmpty) Seq.empty + else { + val pkg = "(?m)^package ([\\w.]+)".r.findFirstMatchIn(text).get.group(1) + names.map(name => s"$pkg.$name") + } + }.distinct.sorted + val output = destination / "CoreComponents.scala" + os.write.over( + output, + s"""package framework.device.cbus.generated +import framework.device.cbus.CoreFactory +object CoreComponents { + val components: Seq[CoreFactory] = Seq(${factories.mkString(",\n")}) + require(components.map(_.component).distinct.size == components.size, "Duplicate Core component registration") + val factories: Map[String, CoreFactory] = components.map(f => f.component -> f).toMap +} +""" + ) + Seq(PathRef(output)) +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("C2C.scala")), + PathRef(millSourcePath / os.RelPath("CBus.scala")), + PathRef(millSourcePath / os.RelPath("Control.scala")), + PathRef(millSourcePath / os.RelPath("Fabric.scala")), + PathRef(millSourcePath / os.RelPath("SharedAxiEndpoint.scala")) + ) +) + +def emitSources = T.sources( + Seq.empty[PathRef] +) diff --git a/arch/src/main/scala/framework/device/configs/DeviceParam.scala b/arch/src/main/scala/framework/device/configs/DeviceParam.scala new file mode 100644 index 000000000..470942a13 --- /dev/null +++ b/arch/src/main/scala/framework/device/configs/DeviceParam.scala @@ -0,0 +1,45 @@ +package framework.device.configs + +import buckyball.config.DeviceConfig + +import framework.core.configs.CoreParam +import framework.device.ddr.{Params => DdrParams} +import framework.device.peripherals.bootrom.configs.BootRomParams +import framework.device.peripherals.clint.configs.ClintParams +import framework.device.peripherals.plic.configs.PlicParams +import framework.device.peripherals.scu.configs.SCUParams + +case class DeviceParam( + cores: Seq[CoreParam], + ddr: DdrParams, + bootrom: BootRomParams, + clint: ClintParams, + plic: PlicParams, + scu: SCUParams) { + val hartIds = cores.map(_.hartId) + require(hartIds.distinct.size == hartIds.size, "Duplicate core hart ID") + require( + clint.hartIds == hartIds && plic.hartIds == hartIds, + "Peripheral interrupt ordering must match the configured core instances" + ) +} + +object DeviceParam { + + def fromProto(p: DeviceConfig, cores: Seq[CoreParam]): DeviceParam = { + require( + p.hasDdr && p.hasBootrom && p.hasClint && p.hasPlic && p.hasScu && p.isInitialized, + "Device configuration is incomplete" + ) + require(p.getCoresCount == cores.size, "Device core configuration is incomplete") + DeviceParam( + cores, + DdrParams.fromProto(p.getDdr), + BootRomParams.fromProto(p.getBootrom), + ClintParams.fromProto(p.getClint), + PlicParams.fromProto(p.getPlic), + SCUParams.fromProto(p.getScu) + ) + } + +} diff --git a/arch/src/main/scala/framework/device/ddr/build.sc b/arch/src/main/scala/framework/device/ddr/build.sc new file mode 100644 index 000000000..5e5f532e5 --- /dev/null +++ b/arch/src/main/scala/framework/device/ddr/build.sc @@ -0,0 +1,22 @@ +import mill._ +import $file.^.^.^.^.^.^.settings + + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.device.ddr.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Bridge.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/device/ddr/configs/default.toml b/arch/src/main/scala/framework/device/ddr/configs/default.toml new file mode 100644 index 000000000..c68d70845 --- /dev/null +++ b/arch/src/main/scala/framework/device/ddr/configs/default.toml @@ -0,0 +1,13 @@ +clients = 2 +slotsPerClient = 4 +dataBits = 128 +idBits = 4 +idBase = 0 +base = 0x80000000 +bytes = 0x20000000 +model = 0 + +[line] +nodeIdBits = 7 +addressBits = 44 +dataBits = 256 diff --git a/arch/src/main/scala/framework/device/ddr/configs/uvm.toml b/arch/src/main/scala/framework/device/ddr/configs/uvm.toml new file mode 100644 index 000000000..b6bdc76f5 --- /dev/null +++ b/arch/src/main/scala/framework/device/ddr/configs/uvm.toml @@ -0,0 +1,14 @@ +[parameters] +clients=2 +slotsPerClient=4 +dataBits=128 +idBits=4 +idBase=0 +base="2147483648" +bytes="536870912" +model="0" +[parameters.line] +nodeIdBits=7 +addressBits=44 +dataBits=256 +[defines] diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/csrc/Cargo.lock b/arch/src/main/scala/framework/device/ddr/src/csrc/Cargo.lock similarity index 100% rename from arch/src/main/scala/framework/mem-core/ddr/src/csrc/Cargo.lock rename to arch/src/main/scala/framework/device/ddr/src/csrc/Cargo.lock diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/csrc/Cargo.toml b/arch/src/main/scala/framework/device/ddr/src/csrc/Cargo.toml similarity index 100% rename from arch/src/main/scala/framework/mem-core/ddr/src/csrc/Cargo.toml rename to arch/src/main/scala/framework/device/ddr/src/csrc/Cargo.toml diff --git a/arch/src/main/scala/framework/mem-core/bank/src/csrc/src/lib.rs b/arch/src/main/scala/framework/device/ddr/src/csrc/src/lib.rs similarity index 100% rename from arch/src/main/scala/framework/mem-core/bank/src/csrc/src/lib.rs rename to arch/src/main/scala/framework/device/ddr/src/csrc/src/lib.rs diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/csrc/src/memory.rs b/arch/src/main/scala/framework/device/ddr/src/csrc/src/memory.rs similarity index 100% rename from arch/src/main/scala/framework/mem-core/ddr/src/csrc/src/memory.rs rename to arch/src/main/scala/framework/device/ddr/src/csrc/src/memory.rs diff --git a/arch/src/main/scala/framework/device/ddr/src/main/resources/agent/item.svh b/arch/src/main/scala/framework/device/ddr/src/main/resources/agent/item.svh new file mode 100644 index 000000000..5430e1893 --- /dev/null +++ b/arch/src/main/scala/framework/device/ddr/src/main/resources/agent/item.svh @@ -0,0 +1,17 @@ +class result_item extends uvm_sequence_item; + bit [511:0] data; + bit error; + int tag, client; + `uvm_object_utils_begin(result_item) + + `uvm_field_int(data, UVM_DEFAULT) + `uvm_field_int(error, UVM_DEFAULT) + + `uvm_field_int(tag, UVM_DEFAULT) + `uvm_field_int(client, UVM_DEFAULT) + + `uvm_object_utils_end + function new(string name = "result_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/device/ddr/src/main/resources/agent/service.svh b/arch/src/main/scala/framework/device/ddr/src/main/resources/agent/service.svh new file mode 100644 index 000000000..869a823f6 --- /dev/null +++ b/arch/src/main/scala/framework/device/ddr/src/main/resources/agent/service.svh @@ -0,0 +1,166 @@ +task service(); + forever begin + @(ctl.sample); + if (!ctl.sample.reset) begin + cycle++; + for (int c = 0; c < 2; c++) begin + if (req[c].sample.valid && !req[c].sample.ready) request_stalls++; + if (resp[c].sample.valid && !resp[c].sample.ready) response_stalls++; + if (req[c].sample.valid && req[c].sample.ready) begin + result_item expected = new; + req_t q = req[c].sample.bits; + int key = (c << 12) | int'(q[`DF(REQ, ID)]); + bit [511:0] line; + verify_contract(!commands.exists(key), "Accepted duplicate live client tag"); + verify_contract(ddr_ref_read(model, q[`DF(REQ, ADDR)], line), + "Uninitialized fixture line"); + commands[key] = '{q, c, 0}; + expected.set_transaction_id(key); + expected.client = c; + expected.tag = q[`DF(REQ, ID)]; + expected.error = q[`DF(REQ, WRITE)] ? write_error.exists(q[`DF(REQ, ADDR)]) : + read_error.exists(q[`DF(REQ, ADDR)]); + expected.data = (q[`DF(REQ, WRITE)] || expected.error) ? '0 : line; + scoreboard.expected_export.write(expected); + accepted++; + behavior.sample(c, q[`DF(REQ, WRITE)], q[`DF(REQ, MASK)] == 0 ? 0 : (q[`DF(REQ, MASK) + ] == 64'hffffffffffffffff ? 1 : 2), expected.error, commands.num(), 0); + if (commands.num() > peak) peak = commands.num(); + end + end + if (aw.sample.valid && !aw.sample.ready) aw_stalls++; + if (w.sample.valid && !w.sample.ready) w_stalls++; + if (aw.sample.valid && aw.sample.ready) begin + int key = observe_address(aw.sample.bits, 1); + int id = aw.sample.bits[`DF(A, ID)]; + aw_ids.push_back(id); + aw_keys.push_back(key); + aw_serials.push_back(aw_count); + aw_count++; + end + if (w.sample.valid && w.sample.ready) begin + verify_contract(!$isunknown(w.sample.bits), "Unknown W payload"); + verify_contract(w.sample.bits[`DF(W, LAST)] == (assembling_beat == `DDR_BEATS - 1), + "Wrong WLAST"); + assembling.data[assembling_beat*`DDR_DATA_BITS+:`DDR_DATA_BITS] = w.sample.bits[ + `DF(W, DATA) + ]; + assembling.mask[assembling_beat*(`DDR_BEAT_BYTES)+:(`DDR_BEAT_BYTES)] = w.sample.bits[ + `DF(W, STRB) + ]; + if (assembling_beat == `DDR_BEATS - 1) begin + w_lines.push_back(assembling); + assembling = '0; + assembling_beat = 0; + w_bursts++; + end else assembling_beat++; + end + if (aw_ids.size() && w_lines.size()) begin + int id = aw_ids.pop_front(); + int key = aw_keys.pop_front(); + int serial = aw_serials.pop_front(); + wline line = w_lines.pop_front(); + req_t q = commands[key].packet; + verify_contract(!writing_ids.exists(id) && !reading_ids.exists(id), + "AXI ID reused while active"); + verify_contract(line.data === q[`DF(REQ, DATA)] && line.mask === q[`DF(REQ, MASK)], + "AW/W order, payload or byte mask mismatch"); + verify_contract(ddr_ref_write(model, q[`DF(REQ, ADDR)], line.data, line.mask), + "Write outside initialized DDR"); + writing_ids[id] = '{key, cycle + 7, serial}; + end + if (ar.sample.valid && ar.sample.ready) begin + int key = observe_address(ar.sample.bits, 0); + int id = ar.sample.bits[`DF(A, ID)]; + bit [511:0] line; + verify_contract(!reading_ids.exists(id) && !writing_ids.exists(id), + "AXI ID reused while active"); + verify_contract(ddr_ref_read(model, ar.sample.bits[`DF(A, ADDR)], line), + "Read outside initialized DDR"); + reading_ids[id] = '{key, 0, ar_count, line}; + ar_count++; + end + if (r.sample.valid && r.sample.ready) begin + int key = reading_ids[r_id].key; + r_beats++; + if (last_r_id >= 0 && last_r_id != r_id && reading_ids.exists( + last_r_id + ) && reading_ids[last_r_id].serial == last_r_serial) + interleaves++; + last_r_id = r_id; + last_r_serial = reading_ids[r_id].serial; + if (reading_ids[r_id].beat == `DDR_BEATS - 1) begin + if (reading_ids[r_id].serial < max_r_serial) r_reorders++; + else max_r_serial = reading_ids[r_id].serial; + commands[key].done = 1; + reading_ids.delete(r_id); + end else reading_ids[r_id].beat++; + r_active = 0; + r_cursor = (r_id + 1) % 16; + end + if (b.sample.valid && b.sample.ready) begin + if (writing_ids[b_id].serial < max_b_serial) b_reorders++; + else max_b_serial = writing_ids[b_id].serial; + commands[writing_ids[b_id].key].done = 1; + writing_ids.delete(b_id); + b_active = 0; + end + for (int c = 0; c < 2; c++) + if (resp[c].sample.valid && resp[c].sample.ready) begin + result_item actual = new; + int key = (c << 12) | int'(resp[c].sample.bits[`DF(RESP, ID)]); + verify_contract(!$isunknown(resp[c].sample.bits) && commands.exists(key + ) && commands[key].done, "Early/unknown line response before final R or B"); + actual.set_transaction_id(key); + actual.client = c; + actual.tag = resp[c].sample.bits[`DF(RESP, ID)]; + actual.data = resp[c].sample.bits[`DF(RESP, DATA)]; + actual.error = resp[c].sample.bits[`DF(RESP, ERROR)]; + scoreboard.actual_export.write(actual); + if (actual.error) error_results++; + returned[c]++; + commands.delete(key); + end + end + @(negedge ctl.clock); + aw.ready = !ctl.reset && !hold_aw && cycle % 5 != 0; + w.ready = !ctl.reset && !hold_w && cycle % 7 >= 2; + ar.ready = !ctl.reset && !hold_ar && cycle % 4 != 0; + for (int c = 0; c < 2; c++) resp[c].ready = !ctl.reset && !hold_resp[c] && cycle % 5 != c; + if (!r_active && !hold_r) + for (int off = 0; off < 16; off++) begin + int id = (r_cursor + off) % 16; + if (!r_active && reading_ids.exists(id)) begin + int key = reading_ids[id].key; + u64 address = commands[key].packet[`DF(REQ, ADDR)]; + int beat = reading_ids[id].beat; + r_active = 1; + r_id = id; + r_packet = '0; + r_packet[`DF(R, ID)] = id; + r_packet[`DF(R, DATA)] = reading_ids[id].line[beat*`DDR_DATA_BITS+:`DDR_DATA_BITS]; + r_packet[`DF(R, LAST)] = beat == `DDR_BEATS - 1; + if (read_error.exists(address) && beat == read_error[address]) + r_packet[`DF(R, RESP)] = beat % 2 ? 3 : 2; + end + end + if (!b_active && !hold_b) + for (int id = 15; id >= 0; id--) + if (!b_active && writing_ids.exists(id) && writing_ids[id].due <= cycle) begin + b_active = 1; + b_id = id; + b_packet = '0; + b_packet[`DF(B, ID)] = id; + if (write_error.exists(commands[writing_ids[id].key].packet[`DF(REQ, ADDR)])) + b_packet[ + `DF(B, RESP) + ] = write_error[commands[writing_ids[id].key].packet[ + `DF(REQ, ADDR) + ]]; + end + r.valid = !ctl.reset && r_active; + r.bits = r_packet; + b.valid = !ctl.reset && b_active; + b.bits = b_packet; + end +endtask diff --git a/arch/src/main/scala/framework/device/ddr/src/main/resources/env/env.svh b/arch/src/main/scala/framework/device/ddr/src/main/resources/env/env.svh new file mode 100644 index 000000000..38de213b8 --- /dev/null +++ b/arch/src/main/scala/framework/device/ddr/src/main/resources/env/env.svh @@ -0,0 +1,147 @@ +class env extends uvm_env; + `uvm_component_utils(env) + virtual ddr_control_if ctl; + virtual stream_if #(`DDR_REQ_WIDTH) req[2]; + virtual stream_if #(`DDR_RESP_WIDTH) resp[2]; + virtual stream_if #(`DDR_A_WIDTH) aw, ar; + virtual stream_if #(`DDR_W_WIDTH) w; + virtual stream_if #(`DDR_B_WIDTH) b; + virtual stream_if #(`DDR_R_WIDTH) r; + result_board #(result_item, 1) scoreboard; + chandle model; + typedef struct { + req_t packet; + int client; + bit done; + } command; + typedef struct { + int key, beat, serial; + bit [511:0] line; + } reading; + typedef struct {int key, due, serial;} writing; + typedef struct packed { + bit [511:0] data; + bit [63:0] mask; + } wline; + command commands[int]; + reading reading_ids[int]; + writing writing_ids[int]; + int aw_ids[$], aw_keys[$], aw_serials[$]; + wline w_lines[$]; + wline assembling; + int assembling_beat = 0; + int read_error[u64], write_error[u64]; + bit hold_aw = 0, hold_w = 0, hold_ar = 0, hold_b = 0, hold_r = 0, hold_resp[2] = '{0, 0}; + bit r_active = 0, b_active = 0; + int r_id = 0, b_id = 0, last_r_id = -1, last_r_serial = -1, r_cursor = 0, r_beats = 0; + logic [`DDR_R_WIDTH-1:0] r_packet; + logic [`DDR_B_WIDTH-1:0] b_packet; + int + cycle = 0, + accepted = 0, + returned[2] = '{0, 0}, + cancelled = 0, + peak = 0, + aw_count = 0, + w_bursts = 0, + ar_count = 0; + int + request_stalls = 0, + response_stalls = 0, + aw_stalls = 0, + w_stalls = 0, + interleaves = 0, + error_results = 0; + int invalid_case = -1; + int b_reorders = 0, r_reorders = 0, max_b_serial = -1, max_r_serial = -1; + covergroup behavior with function sample ( + int client, bit writing, int mask_kind, bit error, int occupancy, bit reset_busy + ); + option.per_instance = 1; + clients: coverpoint client {bins all[] = {0, 1};} + direction: coverpoint writing; + masks: coverpoint mask_kind {bins all[] = {[0 : 2]};} + errors: coverpoint error; + occupancy_level: coverpoint occupancy { + bins one = {1}; bins full = {`DDR_SLOTS}; ignore_bins other = {[2 : `DDR_SLOTS - 1]}; + } + reset_cancellation: coverpoint reset_busy; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + behavior = new(); + + endfunction + function void verify_contract(bit ok, string message); + verification_pkg::evidence::checks++; + if (!ok) `uvm_fatal("DDR_CONTRACT", message) + endfunction + function bit [511:0] pattern(int seed); + bit [511:0] data; + for (int i = 0; i < 8; i++) + data[i*64+:64]=64'h0123456789abcdef^(64'h0101010101010101*(seed+i))^(64'h1<<((seed+i*7)%64)); + return data; + endfunction + function void initialize(u64 address, int seed); + ddr_ref_program(model, address, pattern(seed)); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + model = ddr_ref_create(); + scoreboard = result_board#(result_item, 1)::type_id::create("scoreboard", this); + verify_contract(uvm_config_db#(virtual ddr_control_if)::get(this, "", "ctl", ctl), + "Missing control"); + for (int c = 0; c < 2; c++) begin + verify_contract(uvm_config_db#(virtual stream_if #(`DDR_REQ_WIDTH))::get( + this, "", $sformatf("req%0d", c), req[c]), "Missing request"); + verify_contract(uvm_config_db#(virtual stream_if #(`DDR_RESP_WIDTH))::get( + this, "", $sformatf("resp%0d", c), resp[c]), "Missing response"); + end + verify_contract(uvm_config_db#(virtual stream_if #(`DDR_A_WIDTH))::get(this, "", "aw", aw), + "Missing AW"); + verify_contract(uvm_config_db#(virtual stream_if #(`DDR_A_WIDTH))::get(this, "", "ar", ar), + "Missing AR"); + verify_contract(uvm_config_db#(virtual stream_if #(`DDR_W_WIDTH))::get(this, "", "w", w), + "Missing W"); + verify_contract(uvm_config_db#(virtual stream_if #(`DDR_B_WIDTH))::get(this, "", "b", b), + "Missing B"); + verify_contract(uvm_config_db#(virtual stream_if #(`DDR_R_WIDTH))::get(this, "", "r", r), + "Missing R"); + void'(uvm_config_db#(int)::get(this, "", "invalid_case", invalid_case)); + endfunction + function int address_owner(u64 address, bit write); + int found = -1; + foreach (commands[key]) + if (commands[key].packet[ + `DF(REQ, ADDR) + ] == address && commands[key].packet[ + `DF(REQ, WRITE) + ] == write) begin + verify_contract(found == -1, "Ambiguous fixture address"); + found = key; + end + verify_contract(found != -1, "AXI address without accepted line command"); + return found; + endfunction + function int observe_address(logic [`DDR_A_WIDTH-1:0] a, bit write); + int key, id; + verify_contract(!$isunknown(a), "Unknown AXI address payload"); + key = address_owner(a[`DF(A, ADDR)], write); + id = a[`DF(A, ID)]; + verify_contract(id >= `DDR_ID_BASE && id < `DDR_ID_BASE + `DDR_SLOTS, + "AXI ID outside configured namespace"); + verify_contract((id - `DDR_ID_BASE) / `DDR_SLOTS_PER_CLIENT == commands[key].client, + "AXI ID crossed client namespace"); + verify_contract(a[`DF(A, LEN)] == `DDR_BEATS - 1 && a[`DF(A, SIZE)] == $clog2((`DDR_BEAT_BYTES) + ) && a[`DF(A, BURST)] == 1, "Wrong AXI burst geometry"); + verify_contract(a[`DF(A, LOCK)] == 0 && a[`DF(A, CACHE)] == 0 && a[`DF(A, PROT)] == 0 && a[ + `DF(A, QOS)] == 0 && a[`DF(A, REGION)] == 0, "Unexpected AXI attributes"); + return key; + endfunction + function void final_phase(uvm_phase phase); + ddr_ref_destroy(model); + super.final_phase(phase); + endfunction + `include "agent/service.svh" + `include "seq/scenario.svh" +endclass diff --git a/arch/src/main/scala/framework/device/ddr/src/main/resources/filelist.f b/arch/src/main/scala/framework/device/ddr/src/main/resources/filelist.f new file mode 100644 index 000000000..88f237394 --- /dev/null +++ b/arch/src/main/scala/framework/device/ddr/src/main/resources/filelist.f @@ -0,0 +1,9 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/core/stream_if.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/Bridge/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/control_if.sv b/arch/src/main/scala/framework/device/ddr/src/main/resources/interface.sv similarity index 100% rename from arch/src/main/scala/framework/mem-core/ddr/src/main/resources/control_if.sv rename to arch/src/main/scala/framework/device/ddr/src/main/resources/interface.sv diff --git a/arch/src/main/scala/framework/device/ddr/src/main/resources/package.sv b/arch/src/main/scala/framework/device/ddr/src/main/resources/package.sv new file mode 100644 index 000000000..8ab853857 --- /dev/null +++ b/arch/src/main/scala/framework/device/ddr/src/main/resources/package.sv @@ -0,0 +1,30 @@ +`include "parameters.svh" +`define DF(K, F) `DDR_``K``_``F``_OFFSET +: `DDR_``K``_``F``_WIDTH +package ddr_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + import "DPI-C" function chandle ddr_ref_create(); + import "DPI-C" function void ddr_ref_destroy(input chandle model); + import "DPI-C" function void ddr_ref_program( + input chandle model, + input longint unsigned address, + input bit [511:0] data + ); + import "DPI-C" function int unsigned ddr_ref_read( + input chandle model, + input longint unsigned address, + output bit [511:0] data + ); + import "DPI-C" function int unsigned ddr_ref_write( + input chandle model, + input longint unsigned address, + input bit [511:0] data, + input longint unsigned mask + ); + typedef longint unsigned u64; + typedef bit [`DDR_REQ_WIDTH-1:0] req_t; + `include "agent/item.svh" + `include "env/env.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/device/ddr/src/main/resources/seq/scenario.svh b/arch/src/main/scala/framework/device/ddr/src/main/resources/seq/scenario.svh new file mode 100644 index 000000000..9f5d9ef24 --- /dev/null +++ b/arch/src/main/scala/framework/device/ddr/src/main/resources/seq/scenario.svh @@ -0,0 +1,258 @@ +task submit(int client, int id, u64 address, bit write, bit [63:0] mask = '1); + req_t q = '0; + q[`DF(REQ, ID)] = id; + q[`DF(REQ, ADDR)] = address; + q[`DF(REQ, WRITE)] = write; + q[`DF(REQ, DATA)] = pattern(id + client * 17 + int'(address >> 6)); + q[`DF(REQ, MASK)] = mask; + @(negedge ctl.clock); + req[client].bits = q; + req[client].valid = 1; + do @(req[client].sample); while (!req[client].sample.ready); + @(negedge ctl.clock); + req[client].valid = 0; +endtask +task batch(u64 base, bit write); + for (int c = 0; c < 2; c++) + for (int i = 0; i < `DDR_SLOTS_PER_CLIENT; i++) + initialize(base + c * 4096 + i * 64, i + c * 7 + int'(base >> 6)); + fork + begin + for (int i = 0; i < `DDR_SLOTS_PER_CLIENT; i++) + submit(0, 12'h800 + i, base + i * 64, write, + i == 0 ? 0 : i == 1 ? '1 : i == 2 ? 64'h5555555555555555 : 64'haaaaaaaaaaaaaaaa); + end + begin + for (int i = 0; i < `DDR_SLOTS_PER_CLIENT; i++) + submit(1, 12'h800 + i, base + 4096 + i * 64, write); + end + join +endtask +task drain(); + wait (commands.num() == 0); + @(ctl.sample); + verify_contract( + !reading_ids.num()&&!writing_ids.num()&&!aw_ids.size()&&!w_lines.size()&&!assembling_beat&&!r_active&&!b_active, + "Backend did not drain"); +endtask +task joint_reset(); + @(negedge ctl.clock); + ctl.reset = 1; + req[0].valid = 0; + req[1].valid = 0; + repeat (3) @(ctl.sample); + behavior.sample(0, 0, 0, 0, commands.num(), commands.num() != 0); + cancelled += commands.num(); + commands.delete(); + reading_ids.delete(); + writing_ids.delete(); + aw_ids.delete(); + aw_keys.delete(); + aw_serials.delete(); + w_lines.delete(); + scoreboard.cancel_pending(); + assembling = '0; + assembling_beat = 0; + r_active = 0; + b_active = 0; + @(negedge ctl.clock); + ctl.reset = 0; +endtask +function int finished_commands(); + int count = 0; + foreach (commands[key]) if (commands[key].done) count++; + return count; +endfunction +task invalid_scenario(); + int id, beats; + bit aw_seen; + if (invalid_case == 0) begin + submit(0, 0, 'h10001, 0); + end else if (invalid_case == 1) begin + @(negedge ctl.clock); + r.bits = '0; + r.bits[`DF(R, ID)] = 0; + r.valid = 1; + end else if (invalid_case == 2 || invalid_case == 6) begin + aw.ready = 1; + w.ready = invalid_case == 6; + submit(0, 0, 'h10000, 1); + beats = 0; + aw_seen = 0; + do begin + @(ctl.sample); + if (aw.sample.valid && aw.sample.ready) begin + id = aw.sample.bits[`DF(A, ID)]; + aw_seen = 1; + end + if (w.sample.valid && w.sample.ready) beats++; + end while (!aw_seen || (invalid_case == 6 && beats != `DDR_BEATS)); + @(negedge ctl.clock); + b.bits = '0; + b.bits[`DF(B, ID)] = id; + b.bits[`DF(B, RESP)] = invalid_case == 6 ? 1 : 0; + b.valid = 1; + end else begin + ar.ready = 1; + submit(0, 0, 'h10000, 0); + do @(ctl.sample); while (!ar.sample.valid || !ar.sample.ready); + id = ar.sample.bits[`DF(A, ID)]; + for (int beat = 0; beat < (invalid_case == 4 ? `DDR_BEATS : 1); beat++) begin + @(negedge ctl.clock); + r.bits = '0; + r.bits[`DF(R, ID)] = id; + r.bits[`DF(R, LAST)] = invalid_case == 3; + r.bits[`DF(R, RESP)] = invalid_case == 5 ? 1 : 0; + r.valid = 1; + do @(r.sample); while (!r.sample.ready); + end + end + repeat (8) @(ctl.sample); + `uvm_fatal("MISSING_ASSERTION", "Invalid protocol did not trigger expected DUT assertion") +endtask +task execute(); + int before_aw, before_w, before_ar, before_r; + int initial_returned; + ctl.reset = 1; + req[0].valid = 0; + req[1].valid = 0; + req[0].bits = '0; + req[1].bits = '0; + resp[0].ready = 0; + resp[1].ready = 0; + aw.ready = 0; + ar.ready = 0; + w.ready = 0; + b.valid = 0; + b.bits = '0; + r.valid = 0; + r.bits = '0; + repeat (5) @(negedge ctl.clock); + ctl.reset = 0; + if (invalid_case >= 0) begin + invalid_scenario(); + return; + end + fork + service(); + join_none + // W must progress even when AW has never been accepted; all eight slots stay live until B. + hold_aw = 1; + hold_b = 1; + hold_resp[0] = 1; + batch('h10000, 1); + wait (w_bursts == `DDR_SLOTS); + verify_contract(aw_count == 0 && returned[0] == 0 && returned[1] == 0, + "W-before-AW or B-delayed behavior failed"); + hold_aw = 0; + wait (aw_count == `DDR_SLOTS); + initialize('h18000, 99); + fork + submit(0, 12'h800, 'h18000, 1); + begin + hold_b = 0; + wait (returned[1] == `DDR_SLOTS_PER_CLIENT); + verify_contract(returned[0] == 0, "Client0 backpressure leaked"); + hold_resp[0] = 0; + end + join + drain(); + for (int c = 0; c < 2; c++) + for (int i = 0; i < `DDR_SLOTS_PER_CLIENT; i++) + submit(c, 12'h900 + i, 'h10000 + c * 4096 + i * 64, 0); + drain(); + // Conversely allow addresses to run ahead of every W beat. + before_aw = aw_count; + before_w = w_bursts; + hold_w = 1; + batch('h20000, 1); + wait (aw_count == before_aw + `DDR_SLOTS); + verify_contract(w_bursts == before_w, "AW-before-W stalled ordering failed"); + hold_w = 0; + drain(); + // Concurrent reads and writes: hold B until all independent reads return. + hold_b = 1; + initial_returned = returned[0] + returned[1]; + for (int c = 0; c < 2; c++) + for (int i = 0; i < `DDR_SLOTS_PER_CLIENT; i++) initialize('h30000 + c * 4096 + i * 64, 30 + i); + fork + begin + for (int i = 0; i < `DDR_SLOTS_PER_CLIENT; i++) + submit(0, 12'ha00 + i, 'h30000 + i * 64, i[0]); + end + begin + for (int i = 0; i < `DDR_SLOTS_PER_CLIENT; i++) + submit(1, 12'ha00 + i, 'h31000 + i * 64, i[0]); + end + join + wait (returned[0] + returned[1] == initial_returned + 2 * ((`DDR_SLOTS_PER_CLIENT + 1) / 2)); + hold_b = 0; + drain(); + for (int i = 0; i < 4; i++) begin + initialize('h40000 + i * 64, 40 + i); + read_error['h40000+i*64] = i % `DDR_BEATS; + submit(i % 2, 12'hb00 + i, 'h40000 + i * 64, 0); + end + initialize('h41000, 44); + write_error['h41000] = 2; + submit(1, 12'hb10, 'h41000, 1); + initialize('h41040, 45); + write_error['h41040] = 3; + submit(0, 12'hb10, 'h41040, 1); + drain(); + before_ar = ar_count; + hold_r = 1; + batch('h48000, 0); + wait (ar_count == before_ar + `DDR_SLOTS); + r_cursor = `DDR_ID_BASE; + hold_r = 0; + drain(); + // Each byte strobe is independently checked, then read back through the reference. + for (int i = 0; i < 64; i++) begin + initialize('h50000 + i * 64, 50 + i); + submit(i % 2, 12'hc00 + i, 'h50000 + i * 64, 1, 64'h1 << i); + drain(); + submit(i % 2, 12'hd00 + i, 'h50000 + i * 64, 0); + drain(); + end + // Coordinated reset cancels queued AW-only, W-only and stalled line completions. + before_aw = aw_count; + hold_w = 1; + batch('h60000, 1); + wait (aw_count == before_aw + `DDR_SLOTS); + joint_reset(); + hold_w = 0; + before_w = w_bursts; + hold_aw = 1; + batch('h70000, 1); + wait (w_bursts == before_w + `DDR_SLOTS); + joint_reset(); + hold_aw = 0; + before_ar = ar_count; + before_r = r_beats; + hold_r = 1; + batch('h78000, 0); + wait (ar_count == before_ar + `DDR_SLOTS); + hold_r = 0; + wait (r_beats >= before_r + 1); + joint_reset(); + hold_resp[0] = 1; + hold_resp[1] = 1; + batch('h80000, 0); + do @(ctl.sample); while (finished_commands() != `DDR_SLOTS); + joint_reset(); + hold_resp[0] = 0; + hold_resp[1] = 0; + batch('h90000, 0); + drain(); + verify_contract( + peak==`DDR_SLOTS&&(`DDR_BEATS==1?interleaves==0:interleaves>0)&&b_reorders>0&&r_reorders>0&&request_stalls>0&&response_stalls>0&&aw_stalls>0&&w_stalls>0&&error_results==6&&cancelled==4*`DDR_SLOTS, + "Missing concurrency, stalls, errors or reset coverage"); + verify_contract(accepted == returned[0] + returned[1] + cancelled, "Lost line response"); + `uvm_info( + "DDR_PASS", + $sformatf( + "accepted=%0d returned=%0d/%0d cancelled=%0d peak=%0d errors=%0d readInterleave=%0d R/Breorder=%0d/%0d stalls req/resp/AW/W=%0d/%0d/%0d/%0d; full64B payload/mask/tag and all queues drained", + accepted, returned[0], returned[1], cancelled, peak, error_results, interleaves, + r_reorders, b_reorders, request_stalls, response_stalls, aw_stalls, w_stalls), UVM_LOW) +endtask diff --git a/arch/src/main/scala/framework/device/ddr/src/main/resources/suite.toml b/arch/src/main/scala/framework/device/ddr/src/main/resources/suite.toml new file mode 100644 index 000000000..1af7a2050 --- /dev/null +++ b/arch/src/main/scala/framework/device/ddr/src/main/resources/suite.toml @@ -0,0 +1,139 @@ +[model] +manifest="src/csrc/Cargo.toml" +library="libddr_reference" + +[[tests]] +functional_groups = ["ddr_pkg::env::behavior"] +name = "bridge" +top = "bridge_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 1000000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +functional_groups = [] +name = "bad_alignment" +top = "bridge_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 1000000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+invalid_case=0"] +assertion = "DDR line request must be 64 byte aligned" + +[[tests]] +functional_groups = [] +name = "bad_early_b" +top = "bridge_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 1000000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+invalid_case=2"] +assertion = "DDR received B before AW and complete W or for inactive ID" + +[[tests]] +functional_groups = [] +name = "bad_early_rlast" +top = "bridge_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 1000000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+invalid_case=3"] +assertion = "DDR AXI RLAST does not match line beat count" + +[[tests]] +functional_groups = [] +name = "bad_late_rlast" +top = "bridge_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 1000000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+invalid_case=4"] +assertion = "DDR AXI RLAST does not match line beat count" + +[[tests]] +functional_groups = [] +name = "bad_read_exokay" +top = "bridge_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 1000000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+invalid_case=5"] +assertion = "DDR nonexclusive read received EXOKAY" + +[[tests]] +functional_groups = [] +name = "bad_rid" +top = "bridge_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 1000000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+invalid_case=1"] +assertion = "DDR received R for an inactive AXI ID" + +[[tests]] +functional_groups = [] +name = "bad_write_exokay" +top = "bridge_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 1000000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+invalid_case=6"] +assertion = "DDR nonexclusive write received EXOKAY" diff --git a/arch/src/main/scala/framework/device/ddr/src/main/resources/tb.sv b/arch/src/main/scala/framework/device/ddr/src/main/resources/tb.sv new file mode 100644 index 000000000..d026fb891 --- /dev/null +++ b/arch/src/main/scala/framework/device/ddr/src/main/resources/tb.sv @@ -0,0 +1,75 @@ +`include "parameters.svh" +`ifndef DDR_TB +`define DDR_TB bridge_tb +`endif +module `DDR_TB; + import uvm_pkg::*; + import ddr_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + ddr_control_if ctl (clock); + stream_if #(`DDR_REQ_WIDTH) + req0 ( + clock, + ctl.reset + ), + req1 ( + clock, + ctl.reset + ); + stream_if #(`DDR_RESP_WIDTH) + resp0 ( + clock, + ctl.reset + ), + resp1 ( + clock, + ctl.reset + ); + stream_if #(`DDR_A_WIDTH) + aw ( + clock, + ctl.reset + ), + ar ( + clock, + ctl.reset + ); + stream_if #(`DDR_W_WIDTH) w ( + clock, + ctl.reset + ); + stream_if #(`DDR_B_WIDTH) b ( + clock, + ctl.reset + ); + stream_if #(`DDR_R_WIDTH) r ( + clock, + ctl.reset + ); + Bridge dut ( + .clock(clock), + .reset(ctl.reset), + `include "ports.svh" + ); + initial begin + int invalid_case; + uvm_config_db#(virtual ddr_control_if)::set(null, "uvm_test_top.env", "ctl", ctl); + uvm_config_db#(virtual stream_if #(`DDR_REQ_WIDTH))::set(null, "uvm_test_top.env", "req0", + req0); + uvm_config_db#(virtual stream_if #(`DDR_REQ_WIDTH))::set(null, "uvm_test_top.env", "req1", + req1); + uvm_config_db#(virtual stream_if #(`DDR_RESP_WIDTH))::set(null, "uvm_test_top.env", "resp0", + resp0); + uvm_config_db#(virtual stream_if #(`DDR_RESP_WIDTH))::set(null, "uvm_test_top.env", "resp1", + resp1); + uvm_config_db#(virtual stream_if #(`DDR_A_WIDTH))::set(null, "uvm_test_top.env", "aw", aw); + uvm_config_db#(virtual stream_if #(`DDR_A_WIDTH))::set(null, "uvm_test_top.env", "ar", ar); + uvm_config_db#(virtual stream_if #(`DDR_W_WIDTH))::set(null, "uvm_test_top.env", "w", w); + uvm_config_db#(virtual stream_if #(`DDR_B_WIDTH))::set(null, "uvm_test_top.env", "b", b); + uvm_config_db#(virtual stream_if #(`DDR_R_WIDTH))::set(null, "uvm_test_top.env", "r", r); + if ($value$plusargs("invalid_case=%d", invalid_case)) + uvm_config_db#(int)::set(null, "uvm_test_top.env", "invalid_case", invalid_case); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/device/ddr/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/device/ddr/src/main/resources/tests/test.svh new file mode 100644 index 000000000..398043c00 --- /dev/null +++ b/arch/src/main/scala/framework/device/ddr/src/main/resources/tests/test.svh @@ -0,0 +1,14 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + test_env.execute(); + endtask +endclass diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/scala/Bridge.scala b/arch/src/main/scala/framework/device/ddr/src/main/scala/Bridge.scala similarity index 89% rename from arch/src/main/scala/framework/mem-core/ddr/src/main/scala/Bridge.scala rename to arch/src/main/scala/framework/device/ddr/src/main/scala/Bridge.scala index eca9f127b..4fa2b031c 100644 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/scala/Bridge.scala +++ b/arch/src/main/scala/framework/device/ddr/src/main/scala/Bridge.scala @@ -1,22 +1,38 @@ -package memcore.memory.ddr +package framework.device.ddr -import memcore.memory.queue.Queue +import framework.lib.arbiter.{RRArbiter} +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import framework.device.ddr.{Params => ModuleParam} + +import framework.lib.queue.Queue import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public} -import memcore.bus.axi4 -import memcore.bus.chi.snf.{LineRequest, LineResponse} +import framework.ip.bus.axi4 +import framework.ip.bus.chi.snf.{LineRequest, LineResponse} /** Joint-reset transport: AXI slave and bridge must discard old transactions together. */ @instantiable -class Bridge(p: Params) extends Module { +class Bridge(b: ModuleParam) extends Module { + val p = b @public val io = IO(new Bundle { - val request = Vec(p.clients, Flipped(Decoupled(new LineRequest(p.line)))) - val response = Vec(p.clients, Decoupled(new LineResponse(p.line))) - val axi = new axi4.Port(p.axi) + + val request = Vec( + p.clients, + Flipped(Decoupled(new LineRequest(p.line))) + ) + + val response = Vec( + p.clients, + Decoupled(new LineResponse(p.line)) + ) + + val axi = new axi4.Port(p.axi) + val outstanding = Output(UInt(log2Ceil(p.slots + 1).W)) }) @@ -35,18 +51,12 @@ class Bridge(p: Params) extends Module { val complete = RegInit(VecInit(Seq.fill(p.slots)(false.B))) val errors = RegInit(VecInit(Seq.fill(p.slots)(false.B))) val readBeat = RegInit(VecInit(Seq.fill(p.slots)(0.U(beatBits.W)))) - val awQueue = Module(new Queue(UInt(indexBits.W), p.slots)) - val wQueue = Module(new Queue(UInt(indexBits.W), p.slots)) - val arb = Module(new RRArbiter(UInt(indexBits.W), p.clients) { + val awQueue = Instantiate(new Queue(UInt(indexBits.W), p.slots, false, false)) - override lazy val lastGrant = { - val last = RegInit(0.U(math.max(1, log2Ceil(p.clients)).W)) - when(io.out.fire)(last := io.chosen) - last - } + val wQueue = Instantiate(new Queue(UInt(indexBits.W), p.slots, false, false)) - }) + val arb = Instantiate(new RRArbiter(UInt(indexBits.W), p.clients)) for (c <- 0 until p.clients) { val indices = c * p.slotsPerClient until (c + 1) * p.slotsPerClient diff --git a/arch/src/main/scala/framework/device/ddr/src/main/scala/Emit.scala b/arch/src/main/scala/framework/device/ddr/src/main/scala/Emit.scala new file mode 100644 index 000000000..928845fcd --- /dev/null +++ b/arch/src/main/scala/framework/device/ddr/src/main/scala/Emit.scala @@ -0,0 +1,116 @@ +package framework.device.ddr + +import chisel3._ +import java.nio.file.{Files, Paths} +import framework.ip.bus.axi4 +import framework.ip.bus.chi +import framework.ip.bus.chi.snf.{LineRequest, LineResponse} + +object Emit extends App { + val Array("--config", config, "--output-dir", directory) = args + val values = ujson.read(Files.readString(Paths.get(config))) + + val p = Params( + upickle.default.read[chi.Params](values("line")), + values("clients").num.toInt, + values("slotsPerClient").num.toInt, + values("dataBits").num.toInt, + values("idBits").num.toInt, + values("idBase").num.toInt, + BigInt(values("base").str), + BigInt(values("bytes").str), + BigInt(values("model").str) + ) + + Files.createDirectories(Paths.get(directory)) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Bridge(p), + args = Array("--target-dir", s"$directory/Bridge", "--split-verilog") + ) + + val defs = scala.collection.mutable.ArrayBuffer[String]( + s"`define DDR_DATA_BITS ${p.dataBits}", + s"`define DDR_BEAT_BYTES ${p.axi.bytes}", + s"`define DDR_BEATS ${p.beats}", + s"`define DDR_SLOTS ${p.slots}", + s"`define DDR_SLOTS_PER_CLIENT ${p.slotsPerClient}", + s"`define DDR_ID_BASE ${p.idBase}" + ) + + val ports = scala.collection.mutable.ArrayBuffer[String]() + + def channel( + kind: String, + prefix: String, + signal: String, + bundle: Bundle, + define: Boolean = true + ): Unit = { + var offset = 0 + for ((name, field) <- bundle.elements.toSeq) { + val width = field.getWidth + if (define) { + defs += s"`define DDR_${kind}_${name.toUpperCase}_OFFSET $offset" + defs += s"`define DDR_${kind}_${name.toUpperCase}_WIDTH $width" + } + ports += s".${prefix}_bits_$name($signal.bits[$offset +: $width])" + offset += width + } + if (define) defs += s"`define DDR_${kind}_WIDTH $offset" + ports += s".${prefix}_valid($signal.valid)" + ports += s".${prefix}_ready($signal.ready)" + } + + for (c <- 0 until p.clients) { + channel( + "REQ", + s"io_request_$c", + s"req$c", + new LineRequest(p.line), + c == 0 + ) + channel( + "RESP", + s"io_response_$c", + s"resp$c", + new LineResponse(p.line), + c == 0 + ) + } + channel( + "A", + "io_axi_aw", + "aw", + new axi4.Address(p.axi) + ) + channel( + "A", + "io_axi_ar", + "ar", + new axi4.Address(p.axi), + false + ) + channel( + "W", + "io_axi_w", + "w", + new axi4.WriteData(p.axi) + ) + channel( + "B", + "io_axi_b", + "b", + new axi4.WriteResponse(p.axi) + ) + channel( + "R", + "io_axi_r", + "r", + new axi4.ReadData(p.axi) + ) + Files.writeString( + Paths.get(s"$directory/parameters.svh"), + "`ifndef DDR_CONFIG_SVH\n`define DDR_CONFIG_SVH\n" + defs.mkString("\n") + "\n`endif\n" + ) + Files.writeString(Paths.get(s"$directory/ports.svh"), ports.mkString(",\n") + "\n") +} diff --git a/arch/src/main/scala/framework/device/ddr/src/main/scala/Params.scala b/arch/src/main/scala/framework/device/ddr/src/main/scala/Params.scala new file mode 100644 index 000000000..e061af72e --- /dev/null +++ b/arch/src/main/scala/framework/device/ddr/src/main/scala/Params.scala @@ -0,0 +1,47 @@ +package framework.device.ddr + +import buckyball.config.DdrHardware +import scala.jdk.CollectionConverters._ + +import framework.ip.bus.chi +import framework.ip.bus.axi4 +import upickle.default.{macroRW, ReadWriter} + +case class Params( + line: chi.Params = chi.Params(), + clients: Int = 2, + slotsPerClient: Int = 4, + dataBits: Int = 128, + idBits: Int = 4, + idBase: Int = 0, + base: BigInt = BigInt("80000000", 16), + bytes: BigInt = BigInt("20000000", 16), + model: BigInt = 0) { + require(line.addressBits >= 6) + require(clients > 0 && slotsPerClient > 0) + require(Set(64, 128, 256, 512).contains(dataBits)) + val slots = clients * slotsPerClient + require(slots > 1 && idBase >= 0 && BigInt(idBase) + slots <= (BigInt(1) << idBits)) + require(model <= bytes) + val axi = axi4.Params(line.addressBits, dataBits, idBits) + val beats = 512 / dataBits +} + +object Params { + + def fromProto(p: DdrHardware): Params = + Params( + chi.Params.fromProto(p.getLine), + p.getClients, + p.getSlotsPerClient, + p.getDataBits, + p.getIdBits, + p.getIdBase, + BigInt(java.lang.Long.toUnsignedString(p.getBase)), + BigInt(java.lang.Long.toUnsignedString(p.getBytes)), + BigInt(java.lang.Long.toUnsignedString(p.getModel)) + ) + + implicit val chiRW: ReadWriter[chi.Params] = macroRW + implicit val rw: ReadWriter[Params] = macroRW +} diff --git a/arch/src/main/scala/framework/device/dlink/basedevice.scala b/arch/src/main/scala/framework/device/dlink/basedevice.scala new file mode 100644 index 000000000..f072c5eaf --- /dev/null +++ b/arch/src/main/scala/framework/device/dlink/basedevice.scala @@ -0,0 +1,16 @@ +package framework.device.dlink + +import framework.ip.bus.axi4.{Params => ModuleParam} + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public} + +trait HasDLink { def dlink: DLinkIO } + +@instantiable +abstract class BaseDevice(b: ModuleParam) extends Module with HasDLink { + val linkParams = b + + @public + val dlink = IO(new DLinkIO(b)) +} diff --git a/arch/src/main/scala/framework/device/dlink/build.sc b/arch/src/main/scala/framework/device/dlink/build.sc new file mode 100644 index 000000000..600bf9b00 --- /dev/null +++ b/arch/src/main/scala/framework/device/dlink/build.sc @@ -0,0 +1,12 @@ +import mill._ + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("basedevice.scala")), + PathRef(millSourcePath / os.RelPath("dlink.scala")) + ) +) + +def emitSources = T.sources( + Seq.empty[PathRef] +) diff --git a/arch/src/main/scala/framework/device/dlink/dlink.scala b/arch/src/main/scala/framework/device/dlink/dlink.scala new file mode 100644 index 000000000..8bb1af6bc --- /dev/null +++ b/arch/src/main/scala/framework/device/dlink/dlink.scala @@ -0,0 +1,13 @@ +package framework.device.dlink + +import framework.ip.bus.axi4.{Params => ModuleParam} + +import chisel3._ +import framework.ip.bus.axi4 + +/** Unified memory access; address decoding belongs to the device. */ +class DLinkIO(b: ModuleParam) extends Bundle { + val p = b + + val axi = new framework.ip.bus.axi4.Port(b) +} diff --git a/arch/src/main/scala/framework/device/peripherals/Controller.scala b/arch/src/main/scala/framework/device/peripherals/Controller.scala new file mode 100644 index 000000000..74b7faba8 --- /dev/null +++ b/arch/src/main/scala/framework/device/peripherals/Controller.scala @@ -0,0 +1,176 @@ +package framework.device.peripherals + +import chisel3.experimental.hierarchy.Instance + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import framework.device.peripherals.clint.Clint +import framework.device.peripherals.plic.Plic +import framework.device.peripherals.scu.SystemControl +import framework.ip.bus.axi4 +import framework.device.configs.DeviceParam +import framework.device.peripherals.plic.configs.PlicParams +import framework.core.clink.CoreInterrupts +import framework.core.memory.access.{AccessParams, UncachedRequest, UncachedResponse} + +/** One register access presented for a single cycle; data is right-aligned at `addr`. */ +class DeviceAccess extends Bundle { + val addr = UInt(64.W) + val write = Bool() + val size = UInt(3.W) + val data = UInt(64.W) +} + +/** Register-block interface: `read` and `error` answer the access presented in the same cycle. */ +class DevicePort extends Bundle { + val access = Flipped(Valid(new DeviceAccess)) + val read = Output(UInt(64.W)) + val error = Output(Bool()) +} + +/** Device interrupt events and the resulting per-hart interrupt notifications. */ +class HardwareInterrupt(p: PlicParams) extends Bundle { + val sourceCount = p.sources + val hartCount = p.hartIds.size + val sources = Input(UInt(sourceCount.W)) + val harts = Output(Vec(hartCount, new CoreInterrupts)) +} + +/** AXI4 register endpoint: single transfers up to 64 bits, with full transfer strobes or a zero-strobe no-op. */ +@instantiable +class Controller(device: DeviceParam, ap: axi4.Params, cp: AccessParams) extends Module { + val hartIds = device.plic.hartIds + + @public + val io = IO(new Bundle { + val axi = Flipped(new axi4.Port(ap)) + val hardwareInterrupt = new HardwareInterrupt(device.plic) + }) + + require(ap.dataBits >= 64 && cp.tagBits >= ap.idBits) + val request = Wire(Decoupled(new UncachedRequest(cp))) + val response = Wire(Decoupled(new UncachedResponse(cp))) + val aw = Reg(new axi4.Address(ap)) + val ar = Reg(new axi4.Address(ap)) + val wd = Reg(new axi4.WriteData(ap)) + val awHeld = RegInit(false.B) + val arHeld = RegInit(false.B) + val wHeld = RegInit(false.B) + io.axi.aw.ready := !awHeld + io.axi.ar.ready := !arHeld + io.axi.w.ready := !wHeld + when(io.axi.aw.fire) { + assert(io.axi.aw.bits.len === 0.U && !io.axi.aw.bits.lock, "Peripheral AXI accepts single nonexclusive transfers") + aw := io.axi.aw.bits; awHeld := true.B + } + when(io.axi.ar.fire) { + assert(io.axi.ar.bits.len === 0.U && !io.axi.ar.bits.lock, "Peripheral AXI accepts single nonexclusive transfers") + ar := io.axi.ar.bits; arHeld := true.B + } + when(io.axi.w.fire) { + assert(io.axi.w.bits.last, "Peripheral AXI write must contain one beat") + wd := io.axi.w.bits; wHeld := true.B + } + + val bPending = RegInit(false.B) + val rPending = RegInit(false.B) + val bAnswer = Reg(new axi4.WriteResponse(ap)) + val rAnswer = Reg(new axi4.ReadData(ap)) + io.axi.b.valid := bPending + io.axi.b.bits := bAnswer + io.axi.r.valid := rPending + io.axi.r.bits := rAnswer + when(io.axi.b.fire)(bPending := false.B) + when(io.axi.r.fire)(rPending := false.B) + + val active = RegInit(false.B) + val preferWrite = RegInit(false.B) + val canWrite = awHeld && wHeld && !bPending + val canRead = arHeld && !rPending + val writing = canWrite && (!canRead || preferWrite) + val address = Mux(writing, aw, ar) + val lane = address.addr(log2Ceil(ap.bytes) - 1, 0) + val expectedMask = (((1.U((ap.bytes + 1).W) << (1.U << address.size)) - 1.U) << lane)(ap.bytes - 1, 0) + val noWrite = writing && wd.strb === 0.U + val invalidAccess = address.size > 3.U || (address.addr & ((1.U(ap.addressBits.W) << address.size) - 1.U)).orR || + (writing && !noWrite && wd.strb =/= expectedMask) + request.valid := !active && (canWrite || canRead) + request.bits.addr := address.addr + request.bits.tag := address.id + request.bits.size := address.size + request.bits.write := writing + request.bits.data := (wd.data >> (lane << 3))(63, 0) + val responseWrite = Reg(Bool()) + val responseId = Reg(UInt(ap.idBits.W)) + val responseLane = Reg(UInt(log2Ceil(ap.bytes).W)) + when(request.fire) { + active := true.B + preferWrite := !writing + responseWrite := writing + responseId := address.id + responseLane := lane + when(writing) { awHeld := false.B; wHeld := false.B }.otherwise(arHeld := false.B) + } + response.ready := active && Mux(responseWrite, !bPending, !rPending) + when(response.fire) { + active := false.B + when(responseWrite) { + bPending := true.B + bAnswer.id := responseId + bAnswer.resp := Mux(response.bits.error, 2.U, 0.U) + }.otherwise { + rPending := true.B + rAnswer.id := responseId + rAnswer.data := response.bits.data << (responseLane << 3) + rAnswer.resp := Mux(response.bits.error, 2.U, 0.U) + rAnswer.last := true.B + } + } + + val clint: Instance[Clint] = Instantiate(new Clint(device.clint)) + val plic: Instance[Plic] = Instantiate(new Plic(device.plic)) + val scu: Instance[SystemControl] = Instantiate(new SystemControl(device.scu, cp)) + plic.io.sources := io.hardwareInterrupt.sources + for (i <- hartIds.indices) { + io.hardwareInterrupt.harts(i).timer := clint.io.mtip(i) + io.hardwareInterrupt.harts(i).software := clint.io.msip(i) + io.hardwareInterrupt.harts(i).external := plic.io.meip(i) + io.hardwareInterrupt.harts(i).supervisorExternal := plic.io.seip(i) + } + + val addr = request.bits.addr + val inClint = addr >= device.clint.base.U && addr < (device.clint.base + device.clint.bytes).U + val inPlic = addr >= device.plic.base.U && addr < (device.plic.base + device.plic.bytes).U + val inScu = addr >= device.scu.baseAddress.U && addr < (device.scu.baseAddress + device.scu.totalSizeBytes).U + val pending = RegInit(false.B) + val selectedScu = Reg(Bool()) + val answer = Reg(new UncachedResponse(cp)) + + request.ready := !pending && Mux(inScu && !invalidAccess && !noWrite, scu.io.request.ready, true.B) + scu.io.request.valid := request.valid && !pending && inScu && !invalidAccess && !noWrite + scu.io.request.bits := request.bits + for (port <- Seq(clint.io.port, plic.io.port)) { + port.access.bits.addr := addr + port.access.bits.write := request.bits.write + port.access.bits.size := request.bits.size + port.access.bits.data := request.bits.data + } + clint.io.port.access.valid := request.fire && inClint && !invalidAccess && !noWrite + plic.io.port.access.valid := request.fire && inPlic && !invalidAccess && !noWrite + when(request.fire) { + pending := true.B + selectedScu := inScu && !invalidAccess && !noWrite + answer.tag := request.bits.tag + answer.data := Mux(request.bits.write, 0.U, Mux(inClint, clint.io.port.read, plic.io.port.read)) + answer.error := invalidAccess || (!noWrite && Mux( + inClint, + clint.io.port.error, + Mux(inPlic, plic.io.port.error, !inScu) + )) + } + response.valid := pending && Mux(selectedScu, scu.io.response.valid, true.B) + response.bits := Mux(selectedScu, scu.io.response.bits, answer) + scu.io.response.ready := pending && selectedScu && response.ready + when(response.fire)(pending := false.B) +} diff --git a/arch/src/main/scala/framework/device/peripherals/bootrom/BootRom.scala b/arch/src/main/scala/framework/device/peripherals/bootrom/BootRom.scala new file mode 100644 index 000000000..d999749c9 --- /dev/null +++ b/arch/src/main/scala/framework/device/peripherals/bootrom/BootRom.scala @@ -0,0 +1,79 @@ +package framework.device.peripherals.bootrom + +import chisel3._ +import framework.device.peripherals.bootrom.configs.BootRomParams +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import chisel3.util._ +import framework.ip.bus.chi.{Params => ChiParams} +import framework.ip.bus.chi.snf.{LineRequest, LineResponse} + +/** + * The BootROM image (`bootrom/bootrom.rv64.img`, built from `bootrom.S`) with its two config dwords + * filled: the hart count at offset 0x8 and the DRAM entry at offset 0x10. + */ +object BootRom { + + def apply( + base: BigInt, + bytes: BigInt, + hartCount: Int, + entry: BigInt + ): BootRomParams = { + val stream = getClass.getResourceAsStream("/bootrom/bootrom.rv64.img") + require(stream != null, "missing BootROM resource bootrom/bootrom.rv64.img") + val image = + try stream.readAllBytes().toSeq + finally stream.close() + require(image.size >= 24 && image.slice(8, 24).forall(_ == 0), "BootROM image lacks its zeroed config dwords") + def dword(value: BigInt): Seq[Byte] = (0 until 8).map(i => ((value >> (8 * i)) & 0xff).toByte) + BootRomParams(base, bytes, image.take(8) ++ dword(hartCount) ++ dword(entry) ++ image.drop(24)) + } + +} + +/** + * Serves read-only BootROM cache lines. Address routing belongs to the enclosing design. The + * ROM is a cacheable, executable, read-only region, so fetches and loads reach it as line reads; + * a write to a ROM line can only come from a faulty requester and answers with an error. + */ +@instantiable +class BootRomLines(p: BootRomParams, chi: ChiParams) extends Module { + + @public + val io = IO(new Bundle { + val request = Flipped(Decoupled(new LineRequest(chi))) + val response = Decoupled(new LineResponse(chi)) + }) + + val lines = p.image.padTo(((p.image.size + 63) / 64) * 64, 0.toByte).grouped(64).map { line => + line.zipWithIndex.map { case (byte, i) => BigInt(byte & 0xff) << (8 * i) }.sum + }.toSeq + + val rom = VecInit(lines.map(_.U(512.W))) + + val offset = io.request.bits.addr - p.base.U + val inRom = io.request.bits.addr >= p.base.U && io.request.bits.addr < (p.base + p.bytes).U + val line = offset >> 6 + val pending = RegInit(false.B) + val id = Reg(UInt(chi.txnIdBits.W)) + val data = Reg(UInt(512.W)) + val error = Reg(Bool()) + + io.request.ready := !pending + when(io.request.fire) { + pending := true.B + id := io.request.bits.id + error := !inRom || io.request.bits.write || io.request.bits.addr(5, 0).orR + data := Mux( + inRom && !io.request.bits.write && line < lines.size.U, + rom(line(log2Ceil(lines.size.max(2)) - 1, 0)), + 0.U + ) + } + + io.response.valid := pending + io.response.bits.id := id + io.response.bits.data := Mux(error, 0.U, data) + io.response.bits.error := error + when(io.response.fire)(pending := false.B) +} diff --git a/arch/src/main/scala/framework/device/peripherals/bootrom/configs/Parameters.scala b/arch/src/main/scala/framework/device/peripherals/bootrom/configs/Parameters.scala new file mode 100644 index 000000000..dd222a127 --- /dev/null +++ b/arch/src/main/scala/framework/device/peripherals/bootrom/configs/Parameters.scala @@ -0,0 +1,21 @@ +package framework.device.peripherals.bootrom.configs + +import buckyball.config.BootRomHardware +import scala.jdk.CollectionConverters._ + +import upickle.default.{macroRW, ReadWriter} + +case class BootRomParams(base: BigInt, bytes: BigInt, image: Seq[Byte]) { + require(base % 64 == 0 && bytes % 64 == 0 && image.size <= bytes, "BootROM must be whole cache lines") +} + +object BootRomParams { + + def fromProto(p: BootRomHardware): BootRomParams = BootRomParams( + BigInt(java.lang.Long.toUnsignedString(p.getBase)), + BigInt(java.lang.Long.toUnsignedString(p.getBytes)), + p.getImageList.asScala.map(_.toByte).toSeq + ) + + implicit val rw: ReadWriter[BootRomParams] = macroRW +} diff --git a/arch/src/main/scala/framework/device/peripherals/bootrom/configs/default.toml b/arch/src/main/scala/framework/device/peripherals/bootrom/configs/default.toml new file mode 100644 index 000000000..fa2e61cc6 --- /dev/null +++ b/arch/src/main/scala/framework/device/peripherals/bootrom/configs/default.toml @@ -0,0 +1,4 @@ +base = 0x10000 +bytes = 0x10000 +image = "arch/src/main/resources/bootrom/bootrom.rv64.img" +entry = 0x80000000 diff --git a/arch/src/main/scala/framework/device/peripherals/build.sc b/arch/src/main/scala/framework/device/peripherals/build.sc new file mode 100644 index 000000000..7292c6014 --- /dev/null +++ b/arch/src/main/scala/framework/device/peripherals/build.sc @@ -0,0 +1,20 @@ +import mill._ + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("Controller.scala")), + PathRef(millSourcePath / os.RelPath("bootrom/BootRom.scala")), + PathRef(millSourcePath / os.RelPath("bootrom/configs/Parameters.scala")), + PathRef(millSourcePath / os.RelPath("clint/Clint.scala")), + PathRef(millSourcePath / os.RelPath("clint/configs/Parameters.scala")), + PathRef(millSourcePath / os.RelPath("plic/Plic.scala")), + PathRef(millSourcePath / os.RelPath("plic/configs/Parameters.scala")), + PathRef(millSourcePath / os.RelPath("scu/SCU.scala")), + PathRef(millSourcePath / os.RelPath("scu/SystemControl.scala")), + PathRef(millSourcePath / os.RelPath("scu/configs/Parameters.scala")) + ) +) + +def emitSources = T.sources( + Seq.empty[PathRef] +) diff --git a/arch/src/main/scala/framework/device/peripherals/clint/Clint.scala b/arch/src/main/scala/framework/device/peripherals/clint/Clint.scala new file mode 100644 index 000000000..016b2c174 --- /dev/null +++ b/arch/src/main/scala/framework/device/peripherals/clint/Clint.scala @@ -0,0 +1,69 @@ +package framework.device.peripherals.clint + +import chisel3._ +import framework.device.peripherals.clint.configs.ClintParams +import framework.device.peripherals.DevicePort +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import chisel3.util._ + +/** `mtime` advances once every `tickCycles` clock cycles; the device tree timebase must match. */ +/** Standard CLINT layout: msip at 4*hart, mtimecmp at 0x4000 + 8*hart, mtime at 0xbff8. */ +@instantiable +class Clint(p: ClintParams) extends Module { + val hartIds = p.hartIds + require(hartIds.distinct.size == hartIds.size && hartIds.max < 4095) + val n = hartIds.size + + @public + val io = IO(new Bundle { + val port = new DevicePort + val msip = Output(Vec(n, Bool())) + val mtip = Output(Vec(n, Bool())) + }) + + val msip = RegInit(VecInit(Seq.fill(n)(false.B))) + val mtimecmp = RegInit(VecInit(Seq.fill(n)(~0.U(64.W)))) + val mtime = RegInit(0.U(64.W)) + val ticks = RegInit(0.U(log2Ceil(p.tickCycles + 1).W)) + + val access = io.port.access.bits + val offset = access.addr - p.base.U + val write = io.port.access.valid && access.write + val word = access.size === 2.U + val double = access.size === 3.U + + // Each register answers a 32-bit access to either half, and a 64-bit access to its base. + val hits = Wire(Vec(2 * n + 1, Bool())) + val reads = Wire(Vec(2 * n + 1, UInt(64.W))) + + def register( + slot: Int, + at: BigInt, + state: UInt, + update: UInt => Unit, + wide: Boolean + ): Unit = { + val value = WireDefault(UInt(64.W), state) + val low = word && offset === at.U + val high = wide.B && word && offset === (at + 4).U + val full = wide.B && double && offset === at.U + hits(slot) := low || high || full + reads(slot) := Mux(high, value(63, 32), Mux(low, value(31, 0), value)) + when(write && low)(update(if (wide) Cat(value(63, 32), access.data(31, 0)) else access.data(31, 0))) + when(write && high)(update(Cat(access.data(31, 0), value(31, 0)))) + when(write && full)(update(access.data)) + } + + ticks := Mux(ticks === (p.tickCycles - 1).U, 0.U, ticks + 1.U) + when(ticks === (p.tickCycles - 1).U)(mtime := mtime + 1.U) + for ((hart, i) <- hartIds.zipWithIndex) { + register(i, 4 * hart, msip(i).asUInt, v => msip(i) := v(0), wide = false) + register(n + i, 0x4000 + 8 * BigInt(hart), mtimecmp(i), v => mtimecmp(i) := v, wide = true) + io.msip(i) := msip(i) + io.mtip(i) := mtime >= mtimecmp(i) + } + register(2 * n, 0xbff8, mtime, v => mtime := v, wide = true) + + io.port.read := Mux1H(hits, reads) + io.port.error := !hits.asUInt.orR +} diff --git a/arch/src/main/scala/framework/device/peripherals/clint/configs/Parameters.scala b/arch/src/main/scala/framework/device/peripherals/clint/configs/Parameters.scala new file mode 100644 index 000000000..eeb10f6fc --- /dev/null +++ b/arch/src/main/scala/framework/device/peripherals/clint/configs/Parameters.scala @@ -0,0 +1,26 @@ +package framework.device.peripherals.clint.configs + +import buckyball.config.ClintHardware +import scala.jdk.CollectionConverters._ + +import upickle.default.{macroRW, ReadWriter} + +case class ClintParams( + hartIds: Seq[Int], + base: BigInt = BigInt("2000000", 16), + bytes: BigInt = BigInt("10000", 16), + tickCycles: Int = 1000) { + require(tickCycles >= 1) +} + +object ClintParams { + + def fromProto(p: ClintHardware): ClintParams = ClintParams( + p.getHartIdsList.asScala.map(_.intValue).toSeq, + BigInt(java.lang.Long.toUnsignedString(p.getBase)), + BigInt(java.lang.Long.toUnsignedString(p.getBytes)), + p.getTickCycles + ) + + implicit val rw: ReadWriter[ClintParams] = macroRW +} diff --git a/arch/src/main/scala/framework/device/peripherals/clint/configs/default.toml b/arch/src/main/scala/framework/device/peripherals/clint/configs/default.toml new file mode 100644 index 000000000..417a2788c --- /dev/null +++ b/arch/src/main/scala/framework/device/peripherals/clint/configs/default.toml @@ -0,0 +1,4 @@ +base = 0x2000000 +bytes = 0x10000 +tickCycles = 1000 +hartIds = [0] diff --git a/arch/src/main/scala/framework/system/device/Plic.scala b/arch/src/main/scala/framework/device/peripherals/plic/Plic.scala similarity index 89% rename from arch/src/main/scala/framework/system/device/Plic.scala rename to arch/src/main/scala/framework/device/peripherals/plic/Plic.scala index a36517d70..04daa7222 100644 --- a/arch/src/main/scala/framework/system/device/Plic.scala +++ b/arch/src/main/scala/framework/device/peripherals/plic/Plic.scala @@ -1,28 +1,24 @@ -package framework.system.device +package framework.device.peripherals.plic import chisel3._ +import framework.device.peripherals.plic.configs.PlicParams +import framework.device.peripherals.DevicePort import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} import chisel3.util._ -case class PlicParams( - base: BigInt = BigInt("c000000", 16), - bytes: BigInt = BigInt("4000000", 16), - sources: Int = 1, - priorityBits: Int = 3) { - require(sources >= 1 && sources <= 31 && priorityBits >= 1 && priorityBits <= 32) -} - /** * RISC-V PLIC with level-triggered gateways and 32-bit registers. Hart h owns context 2h (M) and * 2h+1 (S). A claim read clears the source's pending bit and holds it in service until completed. */ @instantiable -class Plic(p: PlicParams, hartIds: Seq[Int]) extends Module { +class Plic(p: PlicParams) extends Module { + val hartIds = p.hartIds require(hartIds.distinct.size == hartIds.size) val n = hartIds.size val contexts = 2 * (hartIds.max + 1) - @public val io = IO(new Bundle { + @public + val io = IO(new Bundle { val port = new DevicePort val sources = Input(UInt(p.sources.W)) val meip = Output(Vec(n, Bool())) diff --git a/arch/src/main/scala/framework/device/peripherals/plic/configs/Parameters.scala b/arch/src/main/scala/framework/device/peripherals/plic/configs/Parameters.scala new file mode 100644 index 000000000..be11ef0d7 --- /dev/null +++ b/arch/src/main/scala/framework/device/peripherals/plic/configs/Parameters.scala @@ -0,0 +1,28 @@ +package framework.device.peripherals.plic.configs + +import buckyball.config.PlicHardware +import scala.jdk.CollectionConverters._ + +import upickle.default.{macroRW, ReadWriter} + +case class PlicParams( + hartIds: Seq[Int], + base: BigInt = BigInt("c000000", 16), + bytes: BigInt = BigInt("4000000", 16), + sources: Int = 1, + priorityBits: Int = 3) { + require(sources >= 1 && sources <= 31 && priorityBits >= 1 && priorityBits <= 32) +} + +object PlicParams { + + def fromProto(p: PlicHardware): PlicParams = PlicParams( + p.getHartIdsList.asScala.map(_.intValue).toSeq, + BigInt(java.lang.Long.toUnsignedString(p.getBase)), + BigInt(java.lang.Long.toUnsignedString(p.getBytes)), + p.getSources, + p.getPriorityBits + ) + + implicit val rw: ReadWriter[PlicParams] = macroRW +} diff --git a/arch/src/main/scala/framework/device/peripherals/plic/configs/default.toml b/arch/src/main/scala/framework/device/peripherals/plic/configs/default.toml new file mode 100644 index 000000000..dea72f94e --- /dev/null +++ b/arch/src/main/scala/framework/device/peripherals/plic/configs/default.toml @@ -0,0 +1,5 @@ +base = 0xc000000 +bytes = 0x4000000 +sources = 1 +priorityBits = 3 +hartIds = [0] diff --git a/arch/src/main/scala/framework/device/peripherals/scu/SCU.scala b/arch/src/main/scala/framework/device/peripherals/scu/SCU.scala new file mode 100644 index 000000000..fa858becc --- /dev/null +++ b/arch/src/main/scala/framework/device/peripherals/scu/SCU.scala @@ -0,0 +1,116 @@ +package framework.device.peripherals.scu + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public} +import chisel3.util.HasBlackBoxInline + +/** + * Single DPI-C bridge module for all harts. The hart_id is supplied as an + * input signal rather than being baked into the module name, so only one + * BlackBox is generated and DPI-C imports are not duplicated. + * + * Has separate hart_id inputs for uart and exit operations because different + * harts may write uart and exit simultaneously. + */ +@instantiable +class SCUWriteDPI extends BlackBox with HasBlackBoxInline { + override def desiredName = "SCUWriteDPI" + + @public + val io = IO(new Bundle { + val clock = Input(Clock()) + val reset = Input(Bool()) + val uart_hart_id = Input(UInt(32.W)) + val uart_valid = Input(Bool()) + val uart_data = Input(UInt(8.W)) + val exit_hart_id = Input(UInt(32.W)) + val exit_valid = Input(Bool()) + val exit_code = Input(UInt(32.W)) + }) + + setInline( + "SCUWriteDPI.v", + s""" + |import "DPI-C" context function void scu_uart_write(input int unsigned hart_id, input int unsigned ch); + |import "DPI-C" context function void scu_sim_exit(input int unsigned hart_id, input int unsigned code); + | + |module SCUWriteDPI( + | input clock, + | input reset, + | input [31:0] uart_hart_id, + | input uart_valid, + | input [7:0] uart_data, + | input [31:0] exit_hart_id, + | input exit_valid, + | input [31:0] exit_code + |); + | always @(posedge clock) begin + | if (!reset) begin + | if (uart_valid) begin + | scu_uart_write(uart_hart_id, {24'h0, uart_data}); + | end + | + | if (exit_valid) begin + | scu_sim_exit(exit_hart_id, exit_code); + | end + | end + | end + |endmodule + """.stripMargin + ) +} + +@instantiable +class SCUReadDPI extends BlackBox with HasBlackBoxInline { + override def desiredName = "SCUReadDPI" + + @public + val io = IO(new Bundle { + val clock = Input(Clock()) + val reset = Input(Bool()) + val hart_id = Input(UInt(32.W)) + val enable = Input(Bool()) + val pop = Input(Bool()) + val rx_valid = Output(Bool()) + val rx_data = Output(UInt(8.W)) + }) + + setInline( + "SCUReadDPI.v", + s""" + |import "DPI-C" context function void scu_uart_rx_sample( + | input int unsigned hart_id, + | input int unsigned pop, + | output int unsigned valid, + | output int unsigned data + |); + | + |module SCUReadDPI( + | input clock, + | input reset, + | input [31:0] hart_id, + | input enable, + | input pop, + | output reg rx_valid, + | output reg [7:0] rx_data + |); + | integer valid; + | integer data; + | always @(posedge clock) begin + | if (reset) begin + | rx_valid <= 1'b0; + | rx_data <= 8'h0; + | end else if (enable) begin + | scu_uart_rx_sample(hart_id, {31'h0, pop}, valid, data); + | if (pop) begin + | rx_valid <= 1'b0; + | end else begin + | rx_valid <= (valid != 0); + | rx_data <= data[7:0]; + | end + | end + | end + |endmodule + """.stripMargin + ) +} diff --git a/arch/src/main/scala/framework/device/peripherals/scu/SystemControl.scala b/arch/src/main/scala/framework/device/peripherals/scu/SystemControl.scala new file mode 100644 index 000000000..7a04954c5 --- /dev/null +++ b/arch/src/main/scala/framework/device/peripherals/scu/SystemControl.scala @@ -0,0 +1,81 @@ +package framework.device.peripherals.scu + +import chisel3._ +import framework.device.peripherals.scu.configs.SCUParams +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import chisel3.util._ +import framework.core.memory.access.{AccessParams, UncachedRequest, UncachedResponse} + +/** + * SCU on the System device ports, with the register layout documented on SCUParams. The hart sub-region is + * selected by address, so software keeps using `base + hartId * stride` regardless of port. + * Data is right-aligned at `addr`, as on every uncached CPU port. + */ +@instantiable +class SystemControl(params: SCUParams, p: AccessParams) extends Module { + val strideBits = log2Ceil(params.strideBytes) + + @public + val io = IO(new Bundle { + val request = Flipped(Decoupled(new UncachedRequest(p))) + val response = Decoupled(new UncachedResponse(p)) + }) + + val request = io.request + val response = io.response + val write: Instance[SCUWriteDPI] = Instantiate(new SCUWriteDPI) + val read: Instance[SCUReadDPI] = Instantiate(new SCUReadDPI) + val ready = RegInit(VecInit(Seq.fill(params.maxHarts)(0.U(8.W)))) + + val pending = RegInit(false.B) + val tag = Reg(UInt(p.tagBits.W)) + val error = Reg(Bool()) + val data = Reg(UInt(64.W)) + val status = Reg(Bool()) + + val offset = request.bits.addr - params.baseAddress.U + val hart = offset(31, strideBits) + val readyIndex = if (params.maxHarts == 1) 0.U(0.W) else hart(log2Ceil(params.maxHarts) - 1, 0) + val register = offset(strideBits - 1, 0) + val inside = request.bits.addr >= params.baseAddress.U && + request.bits.addr < (params.baseAddress + params.totalSizeBytes).U && hart < params.maxHarts.U + val fire = request.fire + val wr = request.bits.write + val exit = inside && register === 0x00000.U && wr + val uartTx = inside && register === 0x20000.U && wr + val uartRx = inside && register === 0x20004.U && !wr + val uartSt = inside && register === 0x20005.U && !wr + val readyReg = inside && register === 0x20006.U + + write.io.clock := clock + write.io.reset := reset.asBool + write.io.exit_hart_id := hart + write.io.exit_valid := fire && exit + write.io.exit_code := request.bits.data(31, 0) + write.io.uart_hart_id := hart + write.io.uart_valid := fire && uartTx + write.io.uart_data := request.bits.data(7, 0) + + // The DPI samples on this edge; status reads answer with the fresh sample next cycle. + read.io.clock := clock + read.io.reset := reset.asBool + read.io.hart_id := hart + read.io.enable := fire && (uartRx || uartSt) + read.io.pop := fire && uartRx + + request.ready := !pending + when(fire) { + pending := true.B + tag := request.bits.tag + error := !(exit || uartTx || uartRx || uartSt || readyReg) + status := uartSt + data := Mux(uartRx, read.io.rx_data, Mux(readyReg && !wr, ready(readyIndex), 0.U)) + when(readyReg && wr)(ready(readyIndex) := request.bits.data(7, 0)) + } + + response.valid := pending + response.bits.tag := tag + response.bits.error := error + response.bits.data := Mux(status, "h60".U | read.io.rx_valid.asUInt, data) + when(response.fire)(pending := false.B) +} diff --git a/arch/src/main/scala/framework/device/peripherals/scu/configs/Parameters.scala b/arch/src/main/scala/framework/device/peripherals/scu/configs/Parameters.scala new file mode 100644 index 000000000..634564a53 --- /dev/null +++ b/arch/src/main/scala/framework/device/peripherals/scu/configs/Parameters.scala @@ -0,0 +1,64 @@ +package framework.device.peripherals.scu.configs + +import buckyball.config.ScuHardware +import scala.jdk.CollectionConverters._ + +import upickle.default.{macroRW, ReadWriter} + +/** + * SCU is a global multi-hart System Control Unit (see SystemControl). It + * provides a per-hart sub-region for UART output and simulation exit. Each hart accesses its own sub-region via + * `baseAddress + hartId * strideBytes`. + * + * Address layout (default values): + * baseAddress = 0x60000000 + * strideBytes = 0x40000 (256 KiB per hart) + * totalSizeBytes = 0x10000000 (256 MiB total, supports up to 1024 harts) + * maxHarts = 64 + * + * Within each hart's sub-region: + * +0x00000: simExit register (write triggers $finish with exit code) + * +0x20000: uartTx register (write outputs a byte via DPI-C) + * +0x20004: uartRx register (read pops a byte from host input) + * +0x20005: status register (read-only, bit0 rx-valid, bit5/6 tx-ready) + * +0x20006: ready register (read/write software-visible byte) + * + * @param baseAddress physical base address (must be aligned to totalSizeBytes) + * @param strideBytes bytes per hart (must be power of two) + * @param totalSizeBytes total address window (must be power of two, + * >= maxHarts * strideBytes) + * @param maxHarts number of hart sub-regions. Accesses targeting + * hartId >= maxHarts return an access error. + */ +case class SCUParams( + baseAddress: BigInt = BigInt("60000000", 16), + strideBytes: BigInt = BigInt("40000", 16), + totalSizeBytes: BigInt = BigInt("10000000", 16), + maxHarts: Int = 64) { + require( + strideBytes > 0 && (strideBytes & (strideBytes - 1)) == 0, + s"SCU strideBytes ($strideBytes) must be a positive power of two" + ) + require( + totalSizeBytes > 0 && (totalSizeBytes & (totalSizeBytes - 1)) == 0, + s"SCU totalSizeBytes ($totalSizeBytes) must be a positive power of two" + ) + require(maxHarts > 0, s"SCU maxHarts ($maxHarts) must be positive") + require( + BigInt(maxHarts) * strideBytes <= totalSizeBytes, + s"SCU maxHarts ($maxHarts) * strideBytes ($strideBytes) = " + + s"${BigInt(maxHarts) * strideBytes} exceeds totalSizeBytes ($totalSizeBytes)" + ) +} + +object SCUParams { + + def fromProto(p: ScuHardware): SCUParams = SCUParams( + BigInt(java.lang.Long.toUnsignedString(p.getBaseAddress)), + BigInt(java.lang.Long.toUnsignedString(p.getStrideBytes)), + BigInt(java.lang.Long.toUnsignedString(p.getTotalSizeBytes)), + p.getMaxHarts + ) + + implicit val rw: ReadWriter[SCUParams] = macroRW +} diff --git a/arch/src/main/scala/framework/device/peripherals/scu/configs/default.toml b/arch/src/main/scala/framework/device/peripherals/scu/configs/default.toml new file mode 100644 index 000000000..9c3640f38 --- /dev/null +++ b/arch/src/main/scala/framework/device/peripherals/scu/configs/default.toml @@ -0,0 +1,4 @@ +baseAddress = 0x60000000 +strideBytes = 0x40000 +totalSizeBytes = 0x10000000 +maxHarts = 64 diff --git a/arch/src/main/scala/framework/frontend/Frontend.scala b/arch/src/main/scala/framework/frontend/Frontend.scala deleted file mode 100644 index 004ca1513..000000000 --- a/arch/src/main/scala/framework/frontend/Frontend.scala +++ /dev/null @@ -1,119 +0,0 @@ -package framework.frontend - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.frontend.boot.BootRom -import framework.frontend.decoder.GlobalDecoder -import framework.frontend.globalrs.{GlobalSchedComplete, GlobalSchedIssue, GlobalScheduler, RobAllocation} -import framework.top.GlobalConfig -import framework.system.core.rocket.{RoCCCommandBB, RoCCResponseBB} -import framework.balldomain.blink.SubRobRow -import framework.memdomain.backend.banks.btrace.PhysicalBankHash -import framework.memdomain.backend.shared.SharedMemLayout - -/** - * Frontend Module - * Encapsulates GlobalDecoder and global scheduler - */ -@instantiable -class Frontend(val b: GlobalConfig) extends Module { - - val sharedHashCount = if (b.memDomain.sharedEnable) SharedMemLayout.totalBank(b) else 0 - - @public - val io = IO(new Bundle { - val hartid = Input(UInt(b.tile.xLen.W)) - val sharedBankOwnerHartId = Input(UInt(b.tile.xLen.W)) - - // RoCC command input - val cmd = Flipped(Decoupled(new Bundle { - val cmd = new RoCCCommandBB(b.tile.xLen) - })) - - // Issue to domains - val ball_issue_o = Decoupled(new GlobalSchedIssue(b)) - val mem_issue_o = Decoupled(new GlobalSchedIssue(b)) - // Complete from domains - val ball_complete_i = Flipped(Decoupled(new GlobalSchedComplete(b))) - val mem_complete_i = Flipped(Decoupled(new GlobalSchedComplete(b))) - val kernel_issue_o = if (b.rvv.enable) Some(Decoupled(new GlobalSchedIssue(b))) else None - val kernel_complete_i = if (b.rvv.enable) Some(Flipped(Decoupled(new GlobalSchedComplete(b)))) else None - - // Ball -> SubROB request passthrough - val ball_subrob_req_i = Flipped(Vec(b.ballDomain.ballNum, Decoupled(new SubRobRow(b)))) - val inst_ids = Output(Vec(b.frontend.rob_entries, UInt(64.W))) - val allocation = Valid(new RobAllocation(b)) - val retired = Output(UInt(b.frontend.rob_entries.W)) - - val bank_hashes = - if (b.sim.diffTest) { - Some(Input(Vec(b.memDomain.bankNum + sharedHashCount, new PhysicalBankHash(b)))) - } else { - None - } - - // RoCC response - val resp = Decoupled(new RoCCResponseBB(b.tile.xLen)) - val busy = Output(Bool()) - val idle = Output(Bool()) - // Propagates the Global ROB retirement pulse to the host bridge. - - // Barrier interface — passthrough to GlobalRS - val barrier_arrive = Output(Bool()) - val barrier_release = Input(Bool()) - }) - - val gDecoder: Instance[GlobalDecoder] = Instantiate(new GlobalDecoder(b)) - val scheduler: Instance[GlobalScheduler] = Instantiate(new GlobalScheduler(b)) - - val boot: Instance[BootRom] = Instantiate(new BootRom(b)) - - boot.io.schedulerIdle := scheduler.io.idle - boot.io.cmd.ready := boot.io.active && gDecoder.io.id_i.ready - - gDecoder.io.id_i.valid := Mux(boot.io.active, boot.io.cmd.valid, io.cmd.valid) - gDecoder.io.id_i.bits.cmd := Mux(boot.io.active, boot.io.cmd.bits.cmd, io.cmd.bits.cmd) - // RoCC can't accept new instructions when boot is active - io.cmd.ready := !boot.io.active && gDecoder.io.id_i.ready - - scheduler.io.decode_cmd_i <> gDecoder.io.id_o - scheduler.io.hart_id := io.hartid - scheduler.io.sharedBankOwnerHartId := io.sharedBankOwnerHartId - scheduler.io.bank_hashes.foreach(_ := io.bank_hashes.get) - - io.ball_issue_o <> scheduler.io.ball_issue_o - io.mem_issue_o <> scheduler.io.mem_issue_o - if (b.rvv.enable) { - io.kernel_issue_o.get <> scheduler.io.kernel_issue_o.get - scheduler.io.kernel_complete_i.get <> io.kernel_complete_i.get - } - io.inst_ids := scheduler.io.inst_ids - io.allocation := scheduler.io.allocation - io.retired := scheduler.io.retired - io.idle := !boot.io.active && scheduler.io.idle && !gDecoder.io.id_o.valid - - scheduler.io.ball_complete_i <> io.ball_complete_i - scheduler.io.mem_complete_i <> io.mem_complete_i - - // Wire SubROB request from BallDomain through to scheduler - for (i <- 0 until b.ballDomain.ballNum) { - scheduler.io.ball_subrob_req_i(i) <> io.ball_subrob_req_i(i) - } - - io.resp <> scheduler.io.scheduler_rocc_o.resp - - // io.busy = 1 means NPU will block CPU - // This is the only time when NPU will block CPU: RoB can't accept new - // instructions (ROB full or barrier) - // - // Why we add boot.io.active here is because when boot is active, RoB is - // typically full with mset instructions. This situation is not need to - // block CPU. - io.busy := !boot.io.active && scheduler.io.scheduler_rocc_o.busy - - // Barrier passthrough - io.barrier_arrive := scheduler.io.barrier_arrive - scheduler.io.barrier_release := io.barrier_release - -} diff --git a/arch/src/main/scala/framework/frontend/boot/Boot.scala b/arch/src/main/scala/framework/frontend/boot/Boot.scala deleted file mode 100644 index 00bb950ab..000000000 --- a/arch/src/main/scala/framework/frontend/boot/Boot.scala +++ /dev/null @@ -1,72 +0,0 @@ -package framework.frontend.boot - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public} -import chisel3.util._ -import framework.balldomain.isa.BallISA -import framework.memdomain.boot.MemBoot -import framework.system.core.rocket.{BuckyballCommand, RoCCCommandBB} -import framework.top.GlobalConfig - -@instantiable -class BootRom(val b: GlobalConfig) extends Module { - - @public - val io = IO(new Bundle { - - val cmd = Decoupled(new Bundle { - val cmd = new RoCCCommandBB(b.tile.xLen) - }) - - val schedulerIdle = Input(Bool()) - val active = Output(Bool()) - }) - - val ballInitRecords = b.ballDomain.ballIdMappings.map { mapping => - BallISA.init(mapping.ballId) - } - - val bootRecords = - MemBoot.initializationCommands(b) ++ ballInitRecords ++ MemBoot.releaseCommands(b) ++ - Seq.empty[BuckyballCommand] - - val bootFuncts = VecInit(bootRecords.map(r => r.funct.U(7.W))) - val bootRs1Data = VecInit(bootRecords.map(r => r.rs1.U(b.tile.xLen.W))) - val bootRs2Data = VecInit(bootRecords.map(r => r.rs2.U(b.tile.xLen.W))) - val bootPcWidth = math.max(1, log2Ceil(bootRecords.length + 1)) - - val active = RegInit(true.B) - val drain = RegInit(false.B) - val waitIdle = RegInit(false.B) - val pc = RegInit(0.U(bootPcWidth.W)) - - val atEnd = pc === bootRecords.length.U(bootPcWidth.W) - - val current = Wire(new RoCCCommandBB(b.tile.xLen)) - current := 0.U.asTypeOf(new RoCCCommandBB(b.tile.xLen)) - current.funct := Mux(atEnd, 0.U, bootFuncts(pc)) - current.funct3 := BuckyballCommand.Custom3Funct3.U - current.opcode := BuckyballCommand.Custom3Opcode.U - current.rs1Data := Mux(atEnd, 0.U, bootRs1Data(pc)) - current.rs2Data := Mux(atEnd, 0.U, bootRs2Data(pc)) - - val injectValid = active && !drain && !waitIdle && !atEnd - - when(active && !drain && atEnd) { - drain := true.B - } - when(waitIdle && io.schedulerIdle) { - waitIdle := false.B - } - when(drain && io.schedulerIdle) { - active := false.B - } - when(io.cmd.fire) { - pc := pc + 1.U - waitIdle := true.B - } - - io.cmd.valid := injectValid - io.cmd.bits.cmd := current - io.active := active -} diff --git a/arch/src/main/scala/framework/frontend/configs/FrontendParam.scala b/arch/src/main/scala/framework/frontend/configs/FrontendParam.scala deleted file mode 100644 index dedf00289..000000000 --- a/arch/src/main/scala/framework/frontend/configs/FrontendParam.scala +++ /dev/null @@ -1,33 +0,0 @@ -package framework.frontend.configs - -import upickle.default._ - -/** - * Frontend parameters - includes all frontend configuration. - */ -case class FrontendParam( - rob_entries: Int, - rs_out_of_order_response: Boolean, - bank_id_len: Int, - vbank_id_upper_bound: Int, - /** ISA bank-id namespace base for shared banks: raw id >= this means shared. */ - shared_bank_id_base: Int, - iter_len: Int, - sub_rob_enable: Boolean, - sub_rob_depth: Int) - -object FrontendParam { - implicit val rw: ReadWriter[FrontendParam] = macroRW - - def apply(): FrontendParam = FrontendParam( - rob_entries = 0, - rs_out_of_order_response = false, - bank_id_len = 0, - vbank_id_upper_bound = 0, - shared_bank_id_base = 0, - iter_len = 0, - sub_rob_enable = false, - sub_rob_depth = 0 - ) - -} diff --git a/arch/src/main/scala/framework/frontend/decoder/GISA.scala b/arch/src/main/scala/framework/frontend/decoder/GISA.scala deleted file mode 100644 index cd66dbd12..000000000 --- a/arch/src/main/scala/framework/frontend/decoder/GISA.scala +++ /dev/null @@ -1,19 +0,0 @@ -package framework.frontend.decoder - -import chisel3._ -import chisel3.util._ - -object GISA { - // enable=000, opcode group for no-bank-access instructions - val MVIN_KERNEL_BITPAT = BitPat("b0101100") // 44: writes program bank - val RUN_KERNEL_BITPAT = BitPat("b1001111") // 79: read data/program, write data - val BARRIER_BITPAT = BitPat("b0000001") // 1 (0x01) — enable=000, opcode=1 -} - -// Domain ID constants -object DomainId { - val FRONTEND = 0.U(4.W) // Frontend (barrier), does not enter ROB queue - val RVV = 2.U(4.W) // Private kernel execution and image control - val MEM = 1.U(4.W) // Memory domain - val BALL = 3.U(4.W) // Ball domain -} diff --git a/arch/src/main/scala/framework/frontend/decoder/GobalDecoder.scala b/arch/src/main/scala/framework/frontend/decoder/GobalDecoder.scala deleted file mode 100644 index 4d5ada3ca..000000000 --- a/arch/src/main/scala/framework/frontend/decoder/GobalDecoder.scala +++ /dev/null @@ -1,139 +0,0 @@ -package framework.frontend.decoder - -import chisel3._ -import chisel3.util._ -import chisel3.stage._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.top.GlobalConfig -import framework.frontend.scoreboard.BankAccessInfo -import freechips.rocketchip.tile._ - -import framework.frontend.decoder.GISA._ -import framework.memdomain.frontend.cmd.decoder.DISA._ - -import framework.system.core.rocket.RoCCCommandBB - -@instantiable -class GlobalDecoder(val b: GlobalConfig) extends Module { - - val bankIdLen = b.frontend.bank_id_len - - @public - val io = IO(new Bundle { - - val id_i = Flipped(Decoupled(new Bundle { - val cmd = new RoCCCommandBB(b.tile.xLen) - })) - - val id_o = Decoupled(new PostGDCmd(b)) - }) - - // If reservation station is blocked, id_i is also blocked - io.id_i.ready := io.id_o.ready - - val func7 = io.id_i.bits.cmd.funct - val funct3 = io.id_i.bits.cmd.funct3 - val opcode = io.id_i.bits.cmd.opcode - val rs1 = io.id_i.bits.cmd.rs1Data - - // Private kernel commands are issued by the RVV controller. - val is_mem_inst = (func7 === MVIN_BITPAT) || - (func7 === MVIN_2D_BITPAT) || - (func7 === MVOUT_BITPAT) || - (func7 === MSET_BITPAT) || - (func7 === MVIN_MMIO_BITPAT) || - (func7 === MVOVER_BITPAT) - - val is_barrier_inst = func7 === BARRIER_BITPAT - - val programBank = rs1(bankIdLen - 1, 0) >= b.memDomain.virtualBankCount.U && - rs1(bankIdLen - 1, 0) < (b.memDomain.virtualBankCount + 2).U - val isProgramRelease = b.rvv.enable.B && func7 === MSET_BITPAT && programBank - val is_kernel_inst = func7 === MVIN_KERNEL_BITPAT || func7 === RUN_KERNEL_BITPAT || isProgramRelease - val is_bare_rvv = opcode === "h57".U || opcode === "h07".U || opcode === "h27".U - val is_ball_inst = !is_mem_inst && !is_barrier_inst && !is_bare_rvv && !is_kernel_inst - - when(io.id_i.fire) { - assert(func7 =/= 0.U, "GlobalDecoder: funct7 zero is not a Buckyball instruction") - assert(!is_bare_rvv, "GlobalDecoder: bare RVV instructions are not Buckyball commands") - when(is_kernel_inst) { - assert(b.rvv.enable.B, "GlobalDecoder: kernel command requires rvv.enable=true") - } - } - - // Encode domain ID - val domain_id = MuxCase( - DomainId.BALL, - Seq( - is_barrier_inst -> DomainId.FRONTEND, - is_kernel_inst -> DomainId.RVV, - is_mem_inst -> DomainId.MEM, - is_ball_inst -> DomainId.BALL - ) - ) - - // ------------------------------------------------------------------------- - // Bank access info extraction — enable flags from funct7[6:4] - // - // Unified rs1 layout (defined in isa.h): - // rs1[9:0] = bank_0 (rd_bank_0 or MSET config bank) - // rs1[19:10] = bank_1 (rd_bank_1, dual-operand only) - // rs1[29:20] = bank_2 (wr_bank for Ball instructions) - // rs1[63:30] = iter (34-bit) - // - // funct7[6:4] enable encoding: - // 000 = no bank access - // 001 = 1 read (bank0) - // 010 = 1 write (bank2) - // 011 = 1 read + 1 write (bank0 read, bank2 write) - // 100 = 2 read + 1 write (bank0+bank1 read, bank2 write) - // 101,110,111 = no bank access (extended opcode space) - // ------------------------------------------------------------------------- - val bankAccess = Wire(new BankAccessInfo(bankIdLen)) - val enableBits = func7(6, 4) - - // Decode enable from funct7[6:4] - val msetTransfer = func7 === MSET_BITPAT && io.id_i.bits.cmd.rs2Data(12) - val hasRd0 = enableBits === 1.U || enableBits === 3.U || enableBits === 4.U || msetTransfer - val hasRd1 = enableBits === 4.U - val hasWr = (enableBits === 2.U || enableBits === 3.U || enableBits === 4.U) && func7 =/= MVIN_MMIO_BITPAT - - val ballBid = WireDefault(0.U(5.W)) - b.ballDomain.ballISA.foreach { entry => - when(func7 === entry.funct7.U) { - ballBid := entry.bid.U - } - } - - bankAccess.rd_bank_0_valid := hasRd0 - bankAccess.rd_bank_0_id := rs1(bankIdLen - 1, 0) - bankAccess.rd_bank_1_valid := hasRd1 - bankAccess.rd_bank_1_id := rs1(bankIdLen + 9, 10) - bankAccess.wr_bank_valid := hasWr - bankAccess.wr_bank_id := Mux(func7 === MSET_BITPAT && !msetTransfer, rs1(bankIdLen - 1, 0), rs1(bankIdLen + 19, 20)) - - private def legalBank(raw: UInt): Bool = - raw <= b.frontend.vbank_id_upper_bound.U || - (b.rvv.enable.B && raw >= b.memDomain.virtualBankCount.U && raw < (b.memDomain.virtualBankCount + 2).U) || - (b.memDomain.sharedEnable.B && raw >= b.frontend.shared_bank_id_base.U && - raw < b.memDomain.virtualBankCount.U) - - val usesArchitecturalBank = is_ball_inst || func7 === MSET_BITPAT || func7 === MVIN_BITPAT || - func7 === MVIN_2D_BITPAT || func7 === MVOUT_BITPAT - when(io.id_i.fire && usesArchitecturalBank) { - when(hasRd0)(assert(legalBank(bankAccess.rd_bank_0_id), "GlobalDecoder: bank0 is outside configured ranges")) - when(hasRd1)(assert(legalBank(bankAccess.rd_bank_1_id), "GlobalDecoder: bank1 is outside configured ranges")) - when(hasWr)(assert(legalBank(bankAccess.wr_bank_id), "GlobalDecoder: write bank is outside configured ranges")) - } - - // Output control - io.id_o.valid := io.id_i.valid - io.id_o.bits.domain_id := domain_id - io.id_o.bits.ball_bid := ballBid - io.id_o.bits.cmd := io.id_i.bits.cmd - io.id_o.bits.bankAccess := bankAccess - io.id_o.bits.op1_col := 0.U - io.id_o.bits.op2_col := 0.U - io.id_o.bits.wr_col := 0.U - io.id_o.bits.isBarrier := is_barrier_inst -} diff --git a/arch/src/main/scala/framework/frontend/decoder/PostGDCmd.scala b/arch/src/main/scala/framework/frontend/decoder/PostGDCmd.scala deleted file mode 100644 index ae5d68448..000000000 --- a/arch/src/main/scala/framework/frontend/decoder/PostGDCmd.scala +++ /dev/null @@ -1,17 +0,0 @@ -package framework.frontend.decoder - -import chisel3._ -import framework.top.GlobalConfig -import framework.frontend.scoreboard.BankAccessInfo -import framework.system.core.rocket.RoCCCommandBB - -class PostGDCmd(val b: GlobalConfig) extends Bundle { - val domain_id = UInt(4.W) - val ball_bid = UInt(5.W) - val cmd = new RoCCCommandBB(b.tile.xLen) - val bankAccess = new BankAccessInfo(b.frontend.bank_id_len) - val op1_col = UInt(b.memDomain.groupCountWidth.W) - val op2_col = UInt(b.memDomain.groupCountWidth.W) - val wr_col = UInt(b.memDomain.groupCountWidth.W) - val isBarrier = Bool() -} diff --git a/arch/src/main/scala/framework/frontend/globalrs/GlobalROB.scala b/arch/src/main/scala/framework/frontend/globalrs/GlobalROB.scala deleted file mode 100644 index 2df0a27fa..000000000 --- a/arch/src/main/scala/framework/frontend/globalrs/GlobalROB.scala +++ /dev/null @@ -1,592 +0,0 @@ -package framework.frontend.globalrs - -import chisel3._ -import chisel3.util._ -import chisel3.experimental._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.top.GlobalConfig -import framework.frontend.decoder.{DomainId, PostGDCmd} -import framework.frontend.scoreboard.{BankAccessInfo, BankAliasTable, BankScoreboard} -import framework.memdomain.frontend.cmd.decoder.DISA.{MSET_BITPAT, MVIN_MMIO_BITPAT} -import framework.frontend.decoder.GISA.{MVIN_KERNEL_BITPAT, RUN_KERNEL_BITPAT} -import framework.memdomain.backend.shared.SharedMemLayout -import framework.memdomain.backend.banks.btrace.{BTraceRecord, PhysicalBankHash} - -@instantiable -class GlobalROB(val b: GlobalConfig) extends Module { - - val robDepth = b.frontend.rob_entries - val idWidth = log2Up(robDepth) - val resourceCount = b.memDomain.virtualBankCount + (if (b.rvv.enable) 2 else 0) - val scoreBankNum = resourceCount + robDepth - require(scoreBankNum <= (1 << b.frontend.bank_id_len), "Bank resource IDs and ROB aliases exceed bank_id_len") - val sharedHashCount = if (b.memDomain.sharedEnable) SharedMemLayout.totalBank(b) else 0 - val physicalBankCount = b.memDomain.bankNum + sharedHashCount - - require( - b.frontend.vbank_id_upper_bound < b.memDomain.virtualBankCount, - s"vbank_id_upper_bound(${b.frontend.vbank_id_upper_bound}) must be < virtualBankCount(${b.memDomain.virtualBankCount})" - ) - - @public - val io = IO(new Bundle { - val hart_id = Input(UInt(b.tile.xLen.W)) - val alloc = Flipped(new DecoupledIO(new PostGDCmd(b))) - val issue = new DecoupledIO(new GlobalRobEntry(b)) - val complete = Flipped(new DecoupledIO(UInt(idWidth.W))) - val allocation = Valid(new RobAllocation(b)) - val retired = Output(UInt(robDepth.W)) - - val empty = Output(Bool()) - val full = Output(Bool()) - val head_ptr = Output(UInt(idWidth.W)) - val issued_count = Output(UInt(log2Up(robDepth + 1).W)) - val entry_valid = Output(Vec(robDepth, Bool())) - val entry_complete = Output(Vec(robDepth, Bool())) - - val subRobActive = Input(Bool()) - val inst_ids = Output(Vec(robDepth, UInt(64.W))) - - val trace = if (b.sim.diffTest) Some(Output(Valid(new BTraceRecord))) else None - val traceCount = if (b.sim.diffTest) Some(Output(UInt(64.W))) else None - - val bank_hashes = - if (b.sim.diffTest) { - Some(Input(Vec(physicalBankCount, new PhysicalBankHash(b)))) - } else { - None - } - - }) - - // --------------------------------------------------------------------------- - // BAT + Bank Scoreboard - // --------------------------------------------------------------------------- - val bat: Instance[BankAliasTable] = Instantiate( - new BankAliasTable( - bankIdLen = b.frontend.bank_id_len, - vbankUpper = resourceCount - 1, - robEntries = robDepth - ) - ) - - val scoreboard: Instance[BankScoreboard] = - Instantiate(new BankScoreboard(b.frontend.bank_id_len, scoreBankNum, robDepth)) - - // --------------------------------------------------------------------------- - // Instruction trace (DPI-C, defined in ITraceDPI.scala) - // --------------------------------------------------------------------------- - val itraceAlloc = Module(new ITraceDPI) - val itraceIssue = Module(new ITraceDPI) - val itraceComp = Module(new ITraceDPI) - - for (t <- Seq(itraceAlloc, itraceIssue, itraceComp)) { - t.io.clock := clock - t.io.reset := reset.asBool - t.io.is_issue := 0.U - t.io.hart_id := io.hart_id - t.io.rob_id := 0.U - t.io.inst_id := 0.U - t.io.domain_id := 0.U - t.io.funct := 0.U - t.io.pc := 0.U - t.io.rs1_data := 0.U - t.io.rs2_data := 0.U - t.io.enable := false.B - } - - // --------------------------------------------------------------------------- - // Storage - // --------------------------------------------------------------------------- - val robEntries = RegInit(VecInit(Seq.fill(robDepth)(0.U.asTypeOf(new GlobalRobEntry(b))))) - val robValid = RegInit(VecInit(Seq.fill(robDepth)(false.B))) - val robIssued = RegInit(VecInit(Seq.fill(robDepth)(false.B))) - val robComplete = RegInit(VecInit(Seq.fill(robDepth)(false.B))) - val instIds = RegInit(VecInit(Seq.fill(robDepth)(0.U(64.W)))) - val nextInstId = RegInit(1.U(64.W)) - - val statusHashes = - if (b.sim.diffTest) { - Some(RegInit(VecInit(Seq.fill(robDepth)(0.U(32.W))))) - } else { - None - } - - io.inst_ids := instIds - - val headPtr = RegInit(0.U(idWidth.W)) - val tailPtr = RegInit(0.U(idWidth.W)) - val issuedCount = RegInit(0.U(log2Up(robDepth + 1).W)) - val bankColWidth = b.memDomain.groupCountWidth - val bankCols = RegInit(VecInit(Seq.fill(b.memDomain.virtualBankCount)(0.U(bankColWidth.W)))) - // In-flight ownership is tracked in the architectural vbank namespace. - val vbankMaskWidth = resourceCount - val vbankBusy = RegInit(0.U(vbankMaskWidth.W)) - - // Dependency state is maintained incrementally instead of rebuilding all - // older-entry comparisons every cycle. A bit in slot k names the ROB slot - // that currently blocks k. Ordinary RAW dependencies disappear at complete; - // dependencies involving a mapping/config instruction disappear at commit. - // Keeping one union mask is sufficient because a config dependency is never - // allowed to clear earlier than commit, even when it also looks like a RAW. - val dependencyMask = RegInit(VecInit(Seq.fill(robDepth)(0.U(robDepth.W)))) - - val isEmpty = headPtr === tailPtr && !robValid(headPtr) - val isFull = headPtr === tailPtr && robValid(headPtr) - - def nextPtr(p: UInt): UInt = Mux(p === (robDepth - 1).U, 0.U, p + 1.U) - def wrapPtr(v: UInt): UInt = Mux(v >= robDepth.U, v - robDepth.U, v) - def robIdx(v: UInt): UInt = v(idWidth - 1, 0) - - def isMappingConfig(entry: GlobalRobEntry): Bool = - entry.cmd.cmd.funct === MSET_BITPAT - - def accessUsesBank(access: BankAccessInfo, bank: UInt): Bool = - (access.rd_bank_0_valid && access.rd_bank_0_id === bank) || - (access.rd_bank_1_valid && access.rd_bank_1_id === bank) || - (access.wr_bank_valid && access.wr_bank_id === bank) - - // Build one architectural-vbank mask per ROB slot once, then hazard checks - // become mask intersections instead of repeating ID comparisons. - val hazardBankCount = resourceCount - - def rawReadMask(access: BankAccessInfo): UInt = { - val rd0 = Mux( - access.rd_bank_0_valid && access.rd_bank_0_id < hazardBankCount.U, - UIntToOH(access.rd_bank_0_id, hazardBankCount), - 0.U(hazardBankCount.W) - ) - val rd1 = Mux( - access.rd_bank_1_valid && access.rd_bank_1_id < hazardBankCount.U, - UIntToOH(access.rd_bank_1_id, hazardBankCount), - 0.U(hazardBankCount.W) - ) - rd0 | rd1 - } - - def rawWriteMask(access: BankAccessInfo): UInt = Mux( - access.wr_bank_valid && access.wr_bank_id < hazardBankCount.U, - UIntToOH(access.wr_bank_id, hazardBankCount), - 0.U(hazardBankCount.W) - ) - - def rawUseMask(access: BankAccessInfo): UInt = - rawReadMask(access) | rawWriteMask(access) - - // --------------------------------------------------------------------------- - // Allocate: enqueue decoded instruction into ROB - // rob_id == tailPtr at allocation time (no separate counter needed) - // --------------------------------------------------------------------------- - val commitMask = Wire(Vec(robDepth, Bool())) - for (i <- 0 until robDepth) { - commitMask(i) := false.B - } - bat.io.free.valid := commitMask.asUInt.orR - bat.io.free.mask := commitMask - - val commitScan = Wire(Vec(robDepth, Bool())) - val commitKeep = Wire(Vec(robDepth + 1, Bool())) - - val headNeedsTrace = instIds(headPtr) =/= 0.U && !isMappingConfig(robEntries(headPtr)) && - robEntries(headPtr).cmd.bankAccess.wr_bank_valid && robEntries( - headPtr - ).cmd.bankAccess.wr_bank_id < b.memDomain.virtualBankCount.U - - commitKeep(0) := true.B - for (i <- 0 until robDepth) { - val ptr = robIdx(wrapPtr(headPtr + i.U)) - commitScan(i) := commitKeep(i) && robValid(ptr) && robComplete(ptr) - commitKeep(i + 1) := (if (b.sim.diffTest) false.B else commitScan(i)) - } - val hasCommit = commitScan.asUInt.orR - val tailAlias = Wire(UInt(b.frontend.bank_id_len.W)) - tailAlias := - resourceCount.U(b.frontend.bank_id_len.W) + tailPtr - - val tailAliasLive = WireDefault(false.B) - for (i <- 0 until robDepth) { - when(robValid(i) && !robComplete(i) && accessUsesBank(robEntries(i).renamedBankAccess, tailAlias)) { - tailAliasLive := true.B - } - } - - io.alloc.ready := !isFull && !hasCommit && !tailAliasLive - io.allocation.valid := io.alloc.fire - io.allocation.bits.rob_id := tailPtr - io.retired := commitMask.asUInt - bat.io.alloc.valid := io.alloc.fire - bat.io.alloc.rob_id := tailPtr - bat.io.alloc.raw := io.alloc.bits.bankAccess - - // Mark write alias as busy in scoreboard at alloc time (not issue time). - scoreboard.alloc.valid := io.alloc.fire && io.alloc.bits.bankAccess.wr_bank_valid - scoreboard.alloc.bits := bat.io.alloc_renamed - - // Bank masks are shared by allocation and issue. This avoids rebuilding the - // UIntToOH decoders once for every older-entry comparison at allocation. - val entryRawReads = Wire(Vec(robDepth, UInt(hazardBankCount.W))) - val entryRawWrites = Wire(Vec(robDepth, UInt(hazardBankCount.W))) - val entryIsConfig = Wire(Vec(robDepth, Bool())) - for (slot <- 0 until robDepth) { - entryRawReads(slot) := rawReadMask(robEntries(slot).cmd.bankAccess) - entryRawWrites(slot) := rawWriteMask(robEntries(slot).cmd.bankAccess) - entryIsConfig(slot) := isMappingConfig(robEntries(slot)) - } - - // A newly allocated entry is always younger than all currently valid ROB - // entries (tailPtr points at the first free slot). Capture its older-entry - // dependencies once here; issue no longer repeats pairwise comparisons. - val allocReadMask = rawReadMask(io.alloc.bits.bankAccess) - val allocWriteMask = rawWriteMask(io.alloc.bits.bankAccess) - val allocIsConfig = io.alloc.bits.cmd.funct === MSET_BITPAT - val allocIsBall = io.alloc.bits.domain_id === DomainId.BALL - // MMIO is a separate shared storage resource, independent of tensor banks. - val mmioReaders = b.ballDomain.ballIdMappings.filter(_.mmioReadBW > 0) - val mmioWriters = b.ballDomain.ballIdMappings.filter(_.mmioWriteBW > 0) - val allocMmioRead = mmioReaders.map(m => allocIsBall && io.alloc.bits.ball_bid === m.ballId.U) - .foldLeft(false.B)(_ || _) - val allocMmioWrite = io.alloc.bits.cmd.funct === MVIN_MMIO_BITPAT || - mmioWriters.map(m => allocIsBall && io.alloc.bits.ball_bid === m.ballId.U).foldLeft(false.B)(_ || _) - val allocDependencyBits = Wire(Vec(robDepth, Bool())) - for (older <- 0 until robDepth) { - val olderUseMask = entryRawReads(older) | entryRawWrites(older) - val rawConflict = ((allocReadMask & entryRawWrites(older)) | - (allocWriteMask & olderUseMask)).orR - val configConflict = (allocIsConfig && entryIsConfig(older)) || - ((allocIsConfig || entryIsConfig(older)) && ((allocReadMask | allocWriteMask) & olderUseMask).orR) - val sameBall = allocIsBall && - robEntries(older).cmd.domain_id === DomainId.BALL && - robEntries(older).cmd.ball_bid === io.alloc.bits.ball_bid - val olderIsBall = robEntries(older).cmd.domain_id === DomainId.BALL - val olderMmioRead = mmioReaders - .map(m => olderIsBall && robEntries(older).cmd.ball_bid === m.ballId.U).foldLeft(false.B)(_ || _) - val olderMmioWrite = robEntries(older).cmd.cmd.funct === MVIN_MMIO_BITPAT || - mmioWriters.map(m => olderIsBall && robEntries(older).cmd.ball_bid === m.ballId.U).foldLeft(false.B)(_ || _) - val mmioConflict = (allocMmioRead && olderMmioWrite) || - (allocMmioWrite && (olderMmioRead || olderMmioWrite)) - // Config conflicts include entries that have completed but are waiting to - // commit. RAW conflicts and same-Ball commands release at completion. - allocDependencyBits(older) := - (robValid(older) && !robComplete(older) && (rawConflict || sameBall || mmioConflict)) || - (robValid(older) && configConflict) - } - val allocDependencies = allocDependencyBits.asUInt - val allocUsesInstId = io.alloc.bits.cmd.pc =/= 0.U - - when(io.alloc.fire) { - itraceAlloc.io.is_issue := 2.U - itraceAlloc.io.rob_id := tailPtr - itraceAlloc.io.inst_id := Mux(allocUsesInstId, nextInstId, 0.U) - itraceAlloc.io.domain_id := io.alloc.bits.domain_id - itraceAlloc.io.funct := io.alloc.bits.cmd.funct - itraceAlloc.io.pc := io.alloc.bits.cmd.pc - itraceAlloc.io.rs1_data := io.alloc.bits.cmd.rs1Data - itraceAlloc.io.rs2_data := io.alloc.bits.cmd.rs2Data - itraceAlloc.io.enable := true.B - - robEntries(tailPtr).cmd := io.alloc.bits - robEntries(tailPtr).renamedBankAccess := bat.io.alloc_renamed - robEntries(tailPtr).rob_id := tailPtr - instIds(tailPtr) := Mux(allocUsesInstId, nextInstId, 0.U) - robValid(tailPtr) := true.B - robIssued(tailPtr) := false.B - robComplete(tailPtr) := false.B - statusHashes.foreach { hashes => - hashes(tailPtr) := 0.U - } - tailPtr := nextPtr(tailPtr) - when(allocUsesInstId) { - nextInstId := nextInstId + 1.U - } - } - - // --------------------------------------------------------------------------- - // Complete: mark entry as completed, release scoreboard resources - // --------------------------------------------------------------------------- - if (b.sim.diffTest) { - val collecting = RegInit(false.B) - val collectCid = Reg(UInt(idWidth.W)) - val collectGroup = Reg(UInt(32.W)) - val currentHash = Reg(UInt(32.W)) - val hashReady = RegInit(false.B) - val hashCid = Reg(UInt(idWidth.W)) - - val incomingAccess = robEntries(io.complete.bits).cmd.bankAccess - val incomingNeedsHash = instIds(io.complete.bits) =/= 0.U && !isMappingConfig(robEntries(io.complete.bits)) && - incomingAccess.wr_bank_valid && incomingAccess.wr_bank_id < b.memDomain.virtualBankCount.U - io.complete.ready := !incomingNeedsHash || (hashReady && hashCid === io.complete.bits) - - when(!collecting && !hashReady && io.complete.valid && incomingNeedsHash) { - collecting := true.B - collectCid := io.complete.bits - collectGroup := 0.U - currentHash := 0.U - } - - val vbank = robEntries(collectCid).cmd.bankAccess.wr_bank_id - val matches = VecInit(io.bank_hashes.get.map { state => - state.valid && state.hartId === io.hart_id && state.vbankId === vbank && state.groupId === collectGroup - }) - val nextMatches = VecInit(io.bank_hashes.get.map { state => - state.valid && state.hartId === io.hart_id && state.vbankId === vbank && state.groupId === collectGroup + 1.U - }) - val physical = Mux1H(matches, io.bank_hashes.get) - def rotateLeft(value: UInt, amount: Int): UInt = Cat(value(31 - amount, 0), value(31, 32 - amount)) - - val mixed = physical.statusHash ^ rotateLeft(physical.groupId, 7) ^ "h9e3779b9".U - val nextHash = rotateLeft(currentHash, 5) ^ mixed ^ rotateLeft(mixed, 13) - - when(collecting) { - assert(PopCount(matches) === 1.U, "BTrace requires exactly one physical bank per vbank group") - when(nextMatches.asUInt.orR) { - currentHash := nextHash - collectGroup := collectGroup + 1.U - }.otherwise { - statusHashes.get(collectCid) := nextHash - collecting := false.B - hashReady := true.B - hashCid := collectCid - } - } - - when(io.complete.fire) { - hashReady := false.B - } - } else { - io.complete.ready := true.B - } - - scoreboard.complete.valid := false.B - scoreboard.complete.bits := 0.U.asTypeOf(scoreboard.complete.bits) - - val issueFired = WireDefault(false.B) - val completeIssuedEntry = io.complete.fire && robIssued(io.complete.bits) - - when(io.complete.fire) { - val cid = io.complete.bits - robComplete(cid) := true.B - scoreboard.complete.valid := true.B - scoreboard.complete.bits := robEntries(cid).renamedBankAccess - - itraceComp.io.is_issue := 0.U - itraceComp.io.rob_id := cid - itraceComp.io.inst_id := instIds(cid) - itraceComp.io.domain_id := robEntries(cid).cmd.domain_id - itraceComp.io.funct := robEntries(cid).cmd.cmd.funct - itraceComp.io.pc := robEntries(cid).cmd.cmd.pc - itraceComp.io.rs1_data := robEntries(cid).cmd.cmd.rs1Data - itraceComp.io.rs2_data := robEntries(cid).cmd.cmd.rs2Data - itraceComp.io.enable := true.B - } - - val completedBitMask = Mux( - io.complete.fire, - UIntToOH(io.complete.bits, robDepth), - 0.U(robDepth.W) - ) - - // --------------------------------------------------------------------------- - // Issue: scan from head for first issuable entry (valid && !issued && !complete) - // --------------------------------------------------------------------------- - val scanValid = Wire(Vec(robDepth, Bool())) - val scanReady = Wire(Vec(robDepth, Bool())) - val vbankMasks = Wire(Vec(robDepth, UInt(vbankMaskWidth.W))) - for (slot <- 0 until robDepth) { - // Reuse the raw hazard mask for active-command ownership. - vbankMasks(slot) := entryRawReads(slot) | entryRawWrites(slot) - } - - // Check each candidate against older ROB entries using mask intersections. - // The masks are precomputed once per slot, so this retains the original - // age-sensitive RAW/WAR/WAW semantics without replicating three equality - // comparators for every bank access pair. - for (i <- 0 until robDepth) { - val ptr = robIdx(wrapPtr(headPtr + i.U)) - scanValid(i) := robValid(ptr) && !robIssued(ptr) && !robComplete(ptr) - scoreboard.queryVec(i) := robEntries(ptr).renamedBankAccess - scanReady(i) := scanValid(i) && !scoreboard.hazardVec(i) && - !dependencyMask(ptr).orR && - !(vbankMasks(ptr) & vbankBusy).orR - } - - val hasReady = scanReady.asUInt.orR - val firstReady = PriorityEncoder(scanReady.asUInt) - val actualIssuePtr = robIdx(wrapPtr(headPtr + firstReady)) - - scoreboard.query := robEntries(actualIssuePtr).renamedBankAccess - val canIssue = hasReady - - val issueStageValid = RegInit(false.B) - val issueStageEntry = Reg(new GlobalRobEntry(b)) - val issueStageFire = issueStageValid && !io.subRobActive && io.issue.ready - val issueStageReady = !issueStageValid || issueStageFire - - // Keep the wide ROB entry and bank-column mux one register boundary away - // from the age/scoreboard scan. The pointer stage can accept one candidate - // per cycle; the payload stage drains at one per cycle, so steady-state II - // remains one while the extra latency is hidden by in-flight commands. - val issuePtrValid = RegInit(false.B) - val issuePtr = Reg(UInt(idWidth.W)) - val issuePtrFire = issuePtrValid && issueStageReady - val issuePtrReady = !issuePtrValid || issuePtrFire - val issueLoad = canIssue && !io.subRobActive && issuePtrReady - val issuePayload = Wire(new GlobalRobEntry(b)) - issuePayload := robEntries(issuePtr) - issuePayload.cmd.op1_col := Mux( - issuePayload.cmd.bankAccess.rd_bank_0_valid && issuePayload.cmd.bankAccess.rd_bank_0_id < b.memDomain.virtualBankCount.U, - bankCols(issuePayload.cmd.bankAccess.rd_bank_0_id(b.memDomain.vbankIdWidth - 1, 0)), - 0.U - ) - issuePayload.cmd.op2_col := Mux( - issuePayload.cmd.bankAccess.rd_bank_1_valid && issuePayload.cmd.bankAccess.rd_bank_1_id < b.memDomain.virtualBankCount.U, - bankCols(issuePayload.cmd.bankAccess.rd_bank_1_id(b.memDomain.vbankIdWidth - 1, 0)), - 0.U - ) - issuePayload.cmd.wr_col := Mux( - issuePayload.cmd.bankAccess.wr_bank_valid && issuePayload.cmd.bankAccess.wr_bank_id < b.memDomain.virtualBankCount.U, - bankCols(issuePayload.cmd.bankAccess.wr_bank_id(b.memDomain.vbankIdWidth - 1, 0)), - 0.U - ) - - io.issue.valid := issueStageValid && !io.subRobActive - io.issue.bits := issueStageEntry - - when(issuePtrReady) { - issuePtrValid := issueLoad - when(issueLoad) { - issuePtr := actualIssuePtr - } - } - when(issueStageReady) { - issueStageValid := issuePtrValid - when(issuePtrFire) { - issueStageEntry := issuePayload - } - } - - scoreboard.issue.valid := false.B - scoreboard.issue.bits := 0.U.asTypeOf(scoreboard.issue.bits) - - when(issueLoad) { - robIssued(actualIssuePtr) := true.B - issueFired := true.B - scoreboard.issue.valid := true.B - scoreboard.issue.bits := robEntries(actualIssuePtr).renamedBankAccess - - itraceIssue.io.is_issue := 1.U - itraceIssue.io.rob_id := robEntries(actualIssuePtr).rob_id - itraceIssue.io.inst_id := instIds(actualIssuePtr) - itraceIssue.io.domain_id := robEntries(actualIssuePtr).cmd.domain_id - itraceIssue.io.funct := robEntries(actualIssuePtr).cmd.cmd.funct - itraceIssue.io.pc := robEntries(actualIssuePtr).cmd.cmd.pc - itraceIssue.io.rs1_data := robEntries(actualIssuePtr).cmd.cmd.rs1Data - itraceIssue.io.rs2_data := robEntries(actualIssuePtr).cmd.cmd.rs2Data - itraceIssue.io.enable := true.B - } - - // Preserve the original update ordering when issue and complete coincide: - // a completion clears ownership after the newly issued transaction marks - // its banks busy. - val issuedVbankMask = Mux(issueLoad, vbankMasks(actualIssuePtr), 0.U) - val completedVbankMask = Mux(io.complete.fire, vbankMasks(io.complete.bits), 0.U) - when(issueLoad || io.complete.fire) { - vbankBusy := - (vbankBusy & ~completedVbankMask) | issuedVbankMask - } - - issuedCount := issuedCount + issueFired.asUInt - completeIssuedEntry.asUInt - - // --------------------------------------------------------------------------- - // Commit: clear completed entries. - // Explicitly skip entries being allocated or completed this cycle. - // --------------------------------------------------------------------------- - for (i <- 0 until robDepth) { - val hits = (0 until robDepth).map { off => - val ptr = robIdx(wrapPtr(headPtr + off.U)) - commitScan(off) && ptr === i.U - } - commitMask(i) := hits.reduce(_ || _) - when(commitMask(i)) { - robValid(i) := false.B - robIssued(i) := false.B - robComplete(i) := false.B - } - } - - if (b.sim.diffTest) { - val trace = io.trace.get - trace.valid := hasCommit && headNeedsTrace - trace.bits.instId := instIds(headPtr) - trace.bits.hartId := io.hart_id - trace.bits.w0Vbank := robEntries(headPtr).cmd.bankAccess.wr_bank_id - trace.bits.w0Hash := statusHashes.get(headPtr) - - val produced = RegInit(0.U(64.W)) - when(trace.valid)(produced := produced + 1.U) - io.traceCount.get := produced - } - - // Maintain dependency masks at the same architectural events that gate the - // issue scan. A dependency involving a config entry (on either side) clears - // only at commit; all other dependencies clear at complete. This is the - // union-mask form of the former separate RAW/config masks. - val configEntryMask = entryIsConfig.asUInt - for (slot <- 0 until robDepth) { - val isAllocatedSlot = io.alloc.fire && tailPtr === slot.U - val slotIsConfig = Mux(isAllocatedSlot, allocIsConfig, entryIsConfig(slot)) - // Each dependency bit names the older entry that blocks this slot. Keep a - // config-related edge through completion when either endpoint is config. - val configRelated = configEntryMask | Fill(robDepth, slotIsConfig) - val clearMask = commitMask.asUInt | (completedBitMask & ~configRelated) - when(isAllocatedSlot) { - dependencyMask(slot) := allocDependencies & ~clearMask - }.otherwise { - dependencyMask(slot) := dependencyMask(slot) & ~clearMask - } - } - - val mappingCommits = VecInit((0 until robDepth).map(i => - commitMask(i) && robEntries(i).cmd.domain_id === DomainId.MEM && - robEntries(i).cmd.cmd.funct === MSET_BITPAT - )) - - assert(PopCount(mappingCommits) <= 1.U, "Mapping dependency permits one configuration commit") - val mapping = robEntries(PriorityEncoder(mappingCommits.asUInt)).cmd - val destination = mapping.bankAccess.wr_bank_id - val source = mapping.bankAccess.rd_bank_0_id - val transferColumns = bankCols(destination(b.memDomain.vbankIdWidth - 1, 0)) + - bankCols(source(b.memDomain.vbankIdWidth - 1, 0)) - - val fullColumns = - if (b.memDomain.sharedEnable) { - Mux(destination >= b.frontend.shared_bank_id_base.U, SharedMemLayout.totalBank(b).U, b.memDomain.bankNum.U) - } else b.memDomain.bankNum.U - - val allocatedColumns = Mux(mapping.cmd.rs2Data(9, 5) === 0.U, fullColumns, mapping.cmd.rs2Data(9, 5)) - for (bank <- 0 until b.memDomain.virtualBankCount) { - when(mappingCommits.asUInt.orR) { - when(destination === bank.U) { - bankCols(bank) := Mux( - mapping.cmd.rs2Data(12), - transferColumns, - Mux(mapping.cmd.rs2Data(10), allocatedColumns, 0.U) - ) - } - when(mapping.cmd.rs2Data(12) && source === bank.U)(bankCols(bank) := 0.U) - } - } - - val commitCount = PopCount(commitScan) - headPtr := wrapPtr(headPtr + commitCount) - - // --------------------------------------------------------------------------- - // Status outputs - // --------------------------------------------------------------------------- - io.empty := isEmpty - io.full := isFull - io.head_ptr := headPtr - io.issued_count := issuedCount - io.entry_valid := robValid - io.entry_complete := robComplete - -} diff --git a/arch/src/main/scala/framework/frontend/globalrs/GlobalScheduler.scala b/arch/src/main/scala/framework/frontend/globalrs/GlobalScheduler.scala deleted file mode 100644 index c09e27c1c..000000000 --- a/arch/src/main/scala/framework/frontend/globalrs/GlobalScheduler.scala +++ /dev/null @@ -1,258 +0,0 @@ -package framework.frontend.globalrs - -import memcore.memory.queue.Queue - -import chisel3._ -import chisel3.util._ -import chisel3.experimental._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.top.GlobalConfig -import framework.frontend.decoder.{DomainId, PostGDCmd} -import framework.frontend.decoder.GISA._ -import framework.frontend.scoreboard.BankAccessInfo -import framework.system.core.rocket.RoCCResponseBB -import framework.balldomain.blink.SubRobRow -import framework.memdomain.backend.banks.btrace.{BTraceDPI, PhysicalBankHash} -import framework.memdomain.backend.shared.SharedMemLayout -import framework.memdomain.frontend.mem.dma.DmaStatus - -class GlobalRobEntry(val b: GlobalConfig) extends Bundle { - val cmd = new PostGDCmd(b) - val renamedBankAccess = new BankAccessInfo(b.frontend.bank_id_len) - val rob_id = UInt(log2Up(b.frontend.rob_entries).W) -} - -class GlobalSchedIssue(b: GlobalConfig) extends GlobalRobEntry(b) { - val is_sub = Bool() - val sub_rob_id = UInt(log2Up(b.frontend.sub_rob_depth * 4).W) -} - -class GlobalSchedComplete(b: GlobalConfig) extends Bundle { - val rob_id = UInt(log2Up(b.frontend.rob_entries).W) - val is_sub = Bool() - val sub_rob_id = UInt(log2Up(b.frontend.sub_rob_depth * 4).W) - val fault = new DmaStatus -} - -class RobAllocation(b: GlobalConfig) extends Bundle { - val rob_id = UInt(log2Up(b.frontend.rob_entries).W) -} - -@instantiable -class GlobalScheduler(val b: GlobalConfig) extends Module { - - val sharedHashCount = if (b.memDomain.sharedEnable) SharedMemLayout.totalBank(b) else 0 - - @public - val io = IO(new Bundle { - val hart_id = Input(UInt(b.tile.xLen.W)) - val sharedBankOwnerHartId = Input(UInt(b.tile.xLen.W)) - val decode_cmd_i = Flipped(new DecoupledIO(new PostGDCmd(b))) - val ball_issue_o = Decoupled(new GlobalSchedIssue(b)) - val mem_issue_o = Decoupled(new GlobalSchedIssue(b)) - val ball_complete_i = Flipped(Decoupled(new GlobalSchedComplete(b))) - val mem_complete_i = Flipped(Decoupled(new GlobalSchedComplete(b))) - val kernel_issue_o = if (b.rvv.enable) Some(Decoupled(new GlobalSchedIssue(b))) else None - val kernel_complete_i = if (b.rvv.enable) Some(Flipped(Decoupled(new GlobalSchedComplete(b)))) else None - val ball_subrob_req_i = Flipped(Vec(b.ballDomain.ballNum, Decoupled(new SubRobRow(b)))) - val inst_ids = Output(Vec(b.frontend.rob_entries, UInt(64.W))) - val allocation = Valid(new RobAllocation(b)) - val retired = Output(UInt(b.frontend.rob_entries.W)) - - val bank_hashes = - if (b.sim.diffTest) { - Some(Input(Vec(b.memDomain.bankNum + sharedHashCount, new PhysicalBankHash(b)))) - } else { - None - } - - val scheduler_rocc_o = new Bundle { - val resp = new DecoupledIO(new RoCCResponseBB(b.tile.xLen)) - val busy = Output(Bool()) - } - - val idle = Output(Bool()) - - val barrier_arrive = Output(Bool()) - val barrier_release = Input(Bool()) - }) - - val rob: Instance[GlobalROB] = Instantiate(new GlobalROB(b)) - rob.io.hart_id := io.hart_id - rob.io.bank_hashes.foreach(_ := io.bank_hashes.get) - io.inst_ids := rob.io.inst_ids - io.allocation := rob.io.allocation - io.retired := rob.io.retired - - if (b.sim.diffTest) { - val btrace = Module(new BTraceDPI) - val trace = rob.io.trace.get - // VVAC stalls logical clocks when its nonblocking channel is full. - btrace.io.clock := clock - btrace.io.reset := reset.asBool - btrace.io.instId := trace.bits.instId - btrace.io.hartId := io.hart_id - val sharedBank = trace.bits.w0Vbank > b.frontend.vbank_id_upper_bound.U && - trace.bits.w0Vbank >= b.frontend.shared_bank_id_base.U && - trace.bits.w0Vbank < b.memDomain.virtualBankCount.U - btrace.io.ownerHartId := Mux(sharedBank, io.sharedBankOwnerHartId, io.hart_id) - btrace.io.w0Vbank := trace.bits.w0Vbank - btrace.io.w0Hash := trace.bits.w0Hash - btrace.io.fire := trace.valid - btrace.io.produced := rob.io.traceCount.get - btrace.io.idle := io.idle && !io.decode_cmd_i.valid && !trace.valid - } - - val isBarrierCmd = io.decode_cmd_i.valid && io.decode_cmd_i.bits.isBarrier - val barrierWaitROB = RegInit(false.B) - val barrierWaitRelease = RegInit(false.B) - when(isBarrierCmd && !barrierWaitROB && !barrierWaitRelease) { - barrierWaitROB := true.B - } - when(barrierWaitROB && rob.io.empty) { - barrierWaitROB := false.B - barrierWaitRelease := true.B - } - when(barrierWaitRelease && io.barrier_release) { - barrierWaitRelease := false.B - } - io.barrier_arrive := barrierWaitRelease - - val isFrontendCmd = io.decode_cmd_i.bits.isBarrier - val anyStall = barrierWaitROB || barrierWaitRelease - rob.io.alloc.valid := io.decode_cmd_i.valid && !isFrontendCmd && !anyStall - rob.io.alloc.bits := io.decode_cmd_i.bits - io.decode_cmd_i.ready := Mux( - isFrontendCmd, - !anyStall, - rob.io.alloc.ready && !anyStall - ) - - val is_ball_domain = rob.io.issue.bits.cmd.domain_id === DomainId.BALL - val is_mem_domain = rob.io.issue.bits.cmd.domain_id === DomainId.MEM - val is_kernel_domain = rob.io.issue.bits.cmd.domain_id === DomainId.RVV - - val mainIssueEntry = Wire(new GlobalSchedIssue(b)) - mainIssueEntry.cmd := rob.io.issue.bits.cmd - mainIssueEntry.renamedBankAccess := rob.io.issue.bits.renamedBankAccess - mainIssueEntry.rob_id := rob.io.issue.bits.rob_id - mainIssueEntry.is_sub := false.B - mainIssueEntry.sub_rob_id := 0.U - - if (b.rvv.enable) { - io.kernel_issue_o.get.valid := rob.io.issue.valid && is_kernel_domain - io.kernel_issue_o.get.bits := mainIssueEntry - } - val kernelReady = if (b.rvv.enable) io.kernel_issue_o.get.ready else false.B - - val completeArb = Module(new Arbiter(new GlobalSchedComplete(b), if (b.rvv.enable) 3 else 2)) - val completeQueue = Module(new Queue(UInt(log2Up(b.frontend.rob_entries).W), b.frontend.rob_entries)) - rob.io.complete <> completeQueue.io.deq - completeArb.io.in(0).valid := io.ball_complete_i.valid - completeArb.io.in(0).bits := io.ball_complete_i.bits - io.ball_complete_i.ready := completeArb.io.in(0).ready - completeArb.io.in(1).valid := io.mem_complete_i.valid - completeArb.io.in(1).bits := io.mem_complete_i.bits - io.mem_complete_i.ready := completeArb.io.in(1).ready - - if (b.rvv.enable) { completeArb.io.in(2) <> io.kernel_complete_i.get } - - val completeBits = completeArb.io.out.bits - - if (b.frontend.sub_rob_enable) { - val subRob: Instance[SubROB] = Instantiate(new SubROB(b)) - - val subRobWriteArb = Module(new Arbiter(new SubRobRow(b), b.ballDomain.ballNum)) - for (i <- 0 until b.ballDomain.ballNum) { - subRobWriteArb.io.in(i) <> io.ball_subrob_req_i(i) - } - subRob.io.write <> subRobWriteArb.io.out - - val subRobIssueValid = subRob.io.issue.valid - val subRobCmd = subRob.io.issue.bits - - val subRobIssueEntry = Wire(new GlobalSchedIssue(b)) - subRobIssueEntry.cmd := subRobCmd - subRobIssueEntry.renamedBankAccess := 0.U.asTypeOf(subRobIssueEntry.renamedBankAccess) - subRobIssueEntry.rob_id := subRob.io.issueMasterRobId - subRobIssueEntry.is_sub := true.B - subRobIssueEntry.sub_rob_id := subRob.io.issueSubId - - val subRobIssBall = subRobCmd.domain_id === DomainId.BALL - val subRobIssMem = subRobCmd.domain_id === DomainId.MEM - when(subRobIssueValid) { - assert( - (subRobCmd.cmd.funct =/= MVIN_KERNEL_BITPAT) && (subRobCmd.cmd.funct =/= RUN_KERNEL_BITPAT), - "RVV kernel commands cannot originate from SubROB" - ) - } - - io.ball_issue_o.valid := Mux( - subRobIssueValid && subRobIssBall, - true.B, - rob.io.issue.valid && is_ball_domain && !subRobIssueValid - ) - io.ball_issue_o.bits := Mux(subRobIssueValid && subRobIssBall, subRobIssueEntry, mainIssueEntry) - - io.mem_issue_o.valid := Mux( - subRobIssueValid && subRobIssMem, - true.B, - rob.io.issue.valid && is_mem_domain && !subRobIssueValid - ) - io.mem_issue_o.bits := Mux(subRobIssueValid && subRobIssMem, subRobIssueEntry, mainIssueEntry) - - subRob.io.issue.ready := - (subRobIssBall && io.ball_issue_o.ready) || - (subRobIssMem && io.mem_issue_o.ready) - - if (b.rvv.enable) { io.kernel_issue_o.get.valid := rob.io.issue.valid && is_kernel_domain && !subRobIssueValid } - rob.io.issue.ready := !subRobIssueValid && ( - (is_ball_domain && io.ball_issue_o.ready) || - (is_mem_domain && io.mem_issue_o.ready) || (is_kernel_domain && kernelReady) - ) - rob.io.subRobActive := subRobIssueValid - - subRob.io.subComplete.valid := completeArb.io.out.valid && completeBits.is_sub - subRob.io.subComplete.bits := completeBits.sub_rob_id - - val normalComplete = completeArb.io.out.valid && !completeBits.is_sub - val masterComplete = subRob.io.masterComplete.valid - val completeRobId = Mux(masterComplete, subRob.io.masterComplete.bits, completeBits.rob_id) - - completeQueue.io.enq.valid := masterComplete || normalComplete - completeQueue.io.enq.bits := completeRobId - subRob.io.masterComplete.ready := masterComplete && completeQueue.io.enq.ready - completeArb.io.out.ready := Mux( - completeBits.is_sub, - subRob.io.subComplete.ready, - !masterComplete && completeQueue.io.enq.ready - ) - io.idle := rob.io.empty && !barrierWaitROB && !barrierWaitRelease && !subRob.io.occupied - } else { - for (i <- 0 until b.ballDomain.ballNum) { - io.ball_subrob_req_i(i).ready := false.B - } - - io.ball_issue_o.valid := rob.io.issue.valid && is_ball_domain - io.ball_issue_o.bits := mainIssueEntry - io.mem_issue_o.valid := rob.io.issue.valid && is_mem_domain - io.mem_issue_o.bits := mainIssueEntry - rob.io.issue.ready := (is_ball_domain && io.ball_issue_o.ready) || - (is_mem_domain && io.mem_issue_o.ready) || (is_kernel_domain && kernelReady) - rob.io.subRobActive := false.B - - when(completeArb.io.out.valid) { - assert(!completeBits.is_sub, "SubROB completion observed when frontend.sub_rob_enable=false") - } - completeQueue.io.enq.valid := completeArb.io.out.valid - completeQueue.io.enq.bits := completeBits.rob_id - completeArb.io.out.ready := completeQueue.io.enq.ready - io.idle := rob.io.empty && !barrierWaitROB && !barrierWaitRelease - } - - io.scheduler_rocc_o.resp.valid := false.B - io.scheduler_rocc_o.resp.bits.rd := 0.U - io.scheduler_rocc_o.resp.bits.data := 0.U - io.scheduler_rocc_o.busy := rob.io.full || barrierWaitROB || barrierWaitRelease - -} diff --git a/arch/src/main/scala/framework/frontend/globalrs/SubROB.scala b/arch/src/main/scala/framework/frontend/globalrs/SubROB.scala deleted file mode 100644 index ce5091d5a..000000000 --- a/arch/src/main/scala/framework/frontend/globalrs/SubROB.scala +++ /dev/null @@ -1,206 +0,0 @@ -package framework.frontend.globalrs - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.top.GlobalConfig -import framework.balldomain.blink.SubRobRow -import framework.frontend.decoder.PostGDCmd - -@instantiable -class SubROB(val b: GlobalConfig) extends Module { - val subRobDepth = b.frontend.sub_rob_depth - val subIdBits = log2Up(subRobDepth * 4) - val robIdBits = log2Up(b.frontend.rob_entries) - val ballIdBits = log2Up(b.ballDomain.ballNum) - val rowIdBits = log2Up(subRobDepth) - - @public - val io = IO(new Bundle { - // Ball → SubROB: write a row of sub-instructions - val write = Flipped(Decoupled(new SubRobRow(b))) - // SubROB → GlobalRS: issue one slot at a time (PostGDCmd, domain_id determines target) - val issue = Decoupled(new PostGDCmd(b)) - val issueSubId = Output(UInt(subIdBits.W)) - val issueMasterRobId = Output(UInt(robIdBits.W)) - // GlobalRS → SubROB: a sub-instruction completed (sub_rob_id) - val subComplete = Flipped(Decoupled(UInt(subIdBits.W))) - // SubROB → main ROB: all sub-instructions done - val masterComplete = Decoupled(UInt(robIdBits.W)) - // Status - val occupied = Output(Bool()) - val lockedBallId = Output(UInt(ballIdBits.W)) - }) - - // ------------------------------------------------------------------------- - // Storage - // ------------------------------------------------------------------------- - val sram = SyncReadMem(subRobDepth, new SubRobRow(b)) - - val writePtr = RegInit(0.U(rowIdBits.W)) - val readPtr = RegInit(0.U(rowIdBits.W)) - val rowCount = RegInit(0.U((log2Up(subRobDepth) + 1).W)) - val lockedBallId = RegInit(0.U(ballIdBits.W)) - val masterRobId = RegInit(0.U(robIdBits.W)) - - def nextPtr(p: UInt): UInt = Mux(p === (subRobDepth - 1).U, 0.U, p + 1.U) - - val occupied = rowCount > 0.U - val isFull = rowCount === subRobDepth.U - val ballMatch = !occupied || (io.write.bits.ball_id === lockedBallId) - - // ------------------------------------------------------------------------- - // FSM - // ------------------------------------------------------------------------- - // States: - // sIdle - nothing to do (rowCount == 0) - // sReadReq - issue SRAM read request for readPtr row - // sReadWait - wait 1 cycle for sramRaw to be latched into sramData - // sReadResp - sramData valid; dispatch valid slots one per cycle - // sWaitSlots - all slots issued; wait for all subCompletes - // sWaitMaster - all rows done; fire masterComplete then return to sIdle - val sIdle :: sReadReq :: sReadWait :: sReadResp :: sWaitSlots :: sWaitMaster :: Nil = Enum(6) - val state = RegInit(sIdle) - - // Write path: accept if not full AND ball_id matches (or not occupied yet) - io.write.ready := !isFull && ballMatch && state =/= sReadReq && state =/= sWaitMaster - - when(io.write.fire) { - when(occupied) { - assert(io.write.bits.master_rob_id === masterRobId, "SubROB row master_rob_id mismatch") - } - writePtr := nextPtr(writePtr) - when(!occupied) { - lockedBallId := io.write.bits.ball_id - masterRobId := io.write.bits.master_rob_id - } - } - - // SyncReadMem: read issued in sReadReq (cycle N), data valid on cycle N+1. - // We latch it into sramData with RegEnable so it stays stable during sReadResp - // and sWaitSlots, even when the read port is not enabled. - val sramReadEn = state === sReadReq - - val sramRaw = sram.readWrite( - Mux(io.write.fire, writePtr, readPtr), - io.write.bits, - sramReadEn || io.write.fire, - io.write.fire - ) - - val dataFresh = RegNext(sramReadEn, false.B) // pulse on first cycle of sReadResp - val sramData = RegEnable(sramRaw, dataFresh) - val readPtrReg = RegEnable(readPtr, sramReadEn) - - // Per-row tracking registers (reset when moving to next row) - // slotIssued: have we already sent this slot to a domain? - // slotDone: have we received subComplete for this slot? - val slotIssued = RegInit(VecInit(Seq.fill(4)(false.B))) - val slotDone = RegInit(VecInit(Seq.fill(4)(false.B))) - - // Combinational: which slots still need to be issued? - val slotNeedsIssue = VecInit((0 until 4).map(i => sramData.slots(i).valid && !slotIssued(i))) - val hasSlotToIssue = slotNeedsIssue.asUInt.orR - val firstSlotIdx = PriorityEncoder(slotNeedsIssue.asUInt) - - // Combinational: are all valid slots done? - val allSlotsDone = VecInit((0 until 4).map(i => !sramData.slots(i).valid || slotDone(i))).asUInt.andR - - // Issue output (valid only in sReadResp when there's a slot to issue) - io.issue.valid := (state === sReadResp) && hasSlotToIssue - io.issue.bits := sramData.slots(firstSlotIdx).cmd - io.issueSubId := readPtrReg * 4.U + firstSlotIdx - io.issueMasterRobId := masterRobId - - // subComplete: always ready, mark the corresponding slot done - io.subComplete.ready := true.B - when(io.subComplete.fire) { - val subId = io.subComplete.bits - val subRow = subId / 4.U - val slotIdx = subId(1, 0) - assert( - state === sReadResp || state === sWaitSlots || state === sWaitMaster, - "SubROB subComplete arrived in invalid state" - ) - assert(subRow === readPtrReg, "SubROB subComplete points to non-active row") - slotDone(slotIdx) := true.B - } - - // masterComplete - io.masterComplete.valid := state === sWaitMaster - io.masterComplete.bits := masterRobId - - // Status outputs - io.occupied := occupied - io.lockedBallId := lockedBallId - - // ------------------------------------------------------------------------- - // FSM transitions - // ------------------------------------------------------------------------- - // Helper: advance past current row (called when row is complete) - val rowAdvance = WireDefault(false.B) - - def advanceRow(): Unit = { - readPtr := nextPtr(readPtr) - rowAdvance := true.B - slotIssued.foreach(_ := false.B) - slotDone.foreach(_ := false.B) - } - - switch(state) { - is(sIdle) { - when(occupied)(state := sReadReq) - } - is(sReadReq) { - // SRAM read issued, data available on sramRaw next cycle - state := sReadWait - } - is(sReadWait) { - // sramRaw is valid; RegEnable latches it into sramData at this clock edge. - // sramData will be stable starting next cycle (sReadResp). - state := sReadResp - } - is(sReadResp) { - // Fire issued slot - when(io.issue.fire) { - slotIssued(firstSlotIdx) := true.B - } - // Check if all slots have been issued - // After marking current slot issued, recalculate - val allIssued = VecInit((0 until 4).map(i => - !sramData.slots(i).valid || slotIssued(i) || - (io.issue.fire && firstSlotIdx === i.U) - )).asUInt.andR - - when(allIssued) { - // Check fast path: all already done (e.g., empty row or immediate completion) - val allDoneNow = VecInit((0 until 4).map(i => !sramData.slots(i).valid || slotDone(i))).asUInt.andR - when(allDoneNow) { - // Fast path: skip sWaitSlots - advanceRow() - state := Mux(rowCount === 1.U && !io.write.fire, sWaitMaster, sReadReq) - }.otherwise { - state := sWaitSlots - } - } - // else: stay in sReadResp to dispatch remaining slots - } - is(sWaitSlots) { - when(allSlotsDone) { - advanceRow() - state := Mux(rowCount === 1.U && !io.write.fire, sWaitMaster, sReadReq) - } - } - is(sWaitMaster) { - when(io.masterComplete.fire) { - lockedBallId := 0.U - masterRobId := 0.U - state := sIdle - } - } - } - when(io.write.fire =/= rowAdvance) { - rowCount := Mux(io.write.fire, rowCount + 1.U, rowCount - 1.U) - } - -} diff --git a/arch/src/main/scala/framework/frontend/scoreboard/BankAliasTable.scala b/arch/src/main/scala/framework/frontend/scoreboard/BankAliasTable.scala deleted file mode 100644 index 545b06deb..000000000 --- a/arch/src/main/scala/framework/frontend/scoreboard/BankAliasTable.scala +++ /dev/null @@ -1,126 +0,0 @@ -package framework.frontend.scoreboard - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} - -/** - * BAT (Bank Alias Table) - * - * Rename virtual bank IDs into the high-ID alias namespace for scoreboard. - * Lifetime policy: per ROB entry. - * - * ID partition: - * [0, vbankUpper] : virtual/architected bank IDs - * [vbankUpper + 1, maxBankId] : renamed alias IDs (one alias per ROB entry) - */ -@instantiable -class BankAliasTable(val bankIdLen: Int, val vbankUpper: Int, val robEntries: Int) extends Module { - val aliasIdLen = bankIdLen - val robIdLen = log2Up(robEntries) - val vbankNum = vbankUpper + 1 - val vbankIdLen = log2Up(vbankNum) - val maxBankId = (1 << bankIdLen) - 1 - val aliasBase = vbankNum - - @public - val io = IO(new Bundle { - - val alloc = Input(new Bundle { - val valid = Bool() - val rob_id = UInt(robIdLen.W) - val raw = new BankAccessInfo(aliasIdLen) - }) - - val alloc_renamed = Output(new BankAccessInfo(aliasIdLen)) - - val free = Input(new Bundle { - val valid = Bool() - val mask = Vec(robEntries, Bool()) - }) - - }) - - // Current architectural alias for each virtual bank. - val v2a = RegInit(VecInit((0 until vbankNum).map(_.U(aliasIdLen.W)))) - - // Extra aliases [aliasBase, aliasBase + robEntries - 1] are one-per-ROB. - val aliasInUse = RegInit(VecInit(Seq.fill(robEntries)(false.B))) - - // Per-entry metadata for commit-time free. - val entHasWrite = RegInit(VecInit(Seq.fill(robEntries)(false.B))) - val entNewAlias = RegInit(VecInit(Seq.fill(robEntries)(0.U(aliasIdLen.W)))) - val entWrVbank = RegInit(VecInit(Seq.fill(robEntries)(0.U(vbankIdLen.W)))) - - require(vbankUpper >= 0, s"BAT vbankUpper must be non-negative, got $vbankUpper") - require(vbankUpper < maxBankId, s"BAT vbankUpper must be < maxBankId($maxBankId), got $vbankUpper") - require( - aliasBase + robEntries - 1 <= maxBankId, - s"BAT alias range exceeds bank_id_len space: base=$aliasBase entries=$robEntries max=$maxBankId" - ) - - private def extraAlias(robId: UInt): UInt = aliasBase.U(aliasIdLen.W) + robId - private def toVbankIdx(v: UInt): UInt = v(vbankIdLen - 1, 0) - - private def mapVbank(v: UInt): UInt = { - val idx = toVbankIdx(v) - val out = WireDefault(0.U(aliasIdLen.W)) - for (i <- 0 until vbankNum) { - when(idx === i.U(vbankIdLen.W)) { - out := v2a(i) - } - } - out - } - - val q = io.alloc.raw - io.alloc_renamed.rd_bank_0_valid := q.rd_bank_0_valid - io.alloc_renamed.rd_bank_1_valid := q.rd_bank_1_valid - io.alloc_renamed.wr_bank_valid := q.wr_bank_valid - io.alloc_renamed.rd_bank_0_id := Mux(q.rd_bank_0_valid, mapVbank(q.rd_bank_0_id), 0.U) - io.alloc_renamed.rd_bank_1_id := Mux(q.rd_bank_1_valid, mapVbank(q.rd_bank_1_id), 0.U) - io.alloc_renamed.wr_bank_id := Mux(q.wr_bank_valid, extraAlias(io.alloc.rob_id), 0.U) - - when(io.alloc.valid) { - val rid = io.alloc.rob_id - when(q.rd_bank_0_valid) { - // assert(q.rd_bank_0_id <= vbankUpper.U, "BAT rd_bank_0_id exceeds virtual bank upper bound") - } - when(q.rd_bank_1_valid) { - // assert(q.rd_bank_1_id <= vbankUpper.U, "BAT rd_bank_1_id exceeds virtual bank upper bound") - } - when(q.wr_bank_valid) { - // assert(q.wr_bank_id <= vbankUpper.U, "BAT wr_bank_id exceeds virtual bank upper bound") - } - - entHasWrite(rid) := q.wr_bank_valid - entNewAlias(rid) := Mux(q.wr_bank_valid, extraAlias(rid), 0.U) - entWrVbank(rid) := toVbankIdx(q.wr_bank_id) - - when(q.wr_bank_valid) { - assert(!aliasInUse(rid), "BAT alias reused before free") - aliasInUse(rid) := true.B - v2a(toVbankIdx(q.wr_bank_id)) := extraAlias(rid) - } - } - - when(io.free.valid) { - for (i <- 0 until robEntries) { - when(io.free.mask(i)) { - when(entHasWrite(i)) { - assert(aliasInUse(i), "BAT free on non-allocated alias") - aliasInUse(i) := false.B - - // Commit is in order. If this is still the current alias, no younger - // write exists and every older write has already committed. - when(v2a(entWrVbank(i)) === entNewAlias(i)) { - v2a(entWrVbank(i)) := entWrVbank(i) - } - } - entHasWrite(i) := false.B - entNewAlias(i) := 0.U - entWrVbank(i) := 0.U - } - } - } -} diff --git a/arch/src/main/scala/framework/frontend/scoreboard/BankScoreboard.scala b/arch/src/main/scala/framework/frontend/scoreboard/BankScoreboard.scala deleted file mode 100644 index a793d3699..000000000 --- a/arch/src/main/scala/framework/frontend/scoreboard/BankScoreboard.scala +++ /dev/null @@ -1,97 +0,0 @@ -package framework.frontend.scoreboard - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} - -//====----------------------------------------------------------====// -// Bank Scoreboard: tracks in-flight read/write operations per bank. -// -// Hazard rules: -// - Read bank X → requires bankWrBusy(X) == false (RAW) -// - Write bank X → requires bankRdCount(X) == 0 (WAR) -// AND bankWrBusy(X) == false (WAW) -// -// bankRdCount: multi-bit counter (multiple concurrent readers allowed, RR is OK) -// bankWrBusy: 1-bit flag (WAW rule guarantees at most 1 writer in-flight) -// -// Lifecycle: -// - alloc: sets bankWrBusy for the renamed write alias. This must happen -// at alloc time (not issue) because the BAT updates v2a at alloc, -// and subsequent reads of the same vbank will be renamed to the -// new alias immediately. If we only set bankWrBusy at issue, the -// window between alloc and issue lets dependent reads slip through -// the hazard check and issue out-of-order. -// - issue: increments bankRdCount for the renamed read aliases. -// - complete: decrements bankRdCount and clears bankWrBusy. -// -// Issue and complete may fire in the same cycle. Updates are computed -// combinationally so both increments and decrements take effect. -//====----------------------------------------------------------====// - -@instantiable -class BankScoreboard(val bankIdLen: Int, val bankNum: Int, val robEntries: Int) extends Module { - - val cntWidth = log2Ceil(robEntries + 1) - - @public - val alloc = IO(Flipped(Valid(new BankAccessInfo(bankIdLen)))) - @public - val issue = IO(Flipped(Valid(new BankAccessInfo(bankIdLen)))) - @public - val complete = IO(Flipped(Valid(new BankAccessInfo(bankIdLen)))) - @public - val query = IO(Input(new BankAccessInfo(bankIdLen))) - @public - val hasHazard = IO(Output(Bool())) - @public - val queryVec = IO(Input(Vec(robEntries, new BankAccessInfo(bankIdLen)))) - @public - val hazardVec = IO(Output(Vec(robEntries, Bool()))) - - val bankRdCount = RegInit(VecInit(Seq.fill(bankNum)(0.U(cntWidth.W)))) - val bankWrBusy = RegInit(VecInit(Seq.fill(bankNum)(false.B))) - private def bankIndex(id: UInt): UInt = id(log2Ceil(bankNum) - 1, 0) - - // --- Hazard detection (reads current register state) --- - private def hazardOf(q: BankAccessInfo): Bool = { - val rd0_hazard = q.rd_bank_0_valid && bankWrBusy(bankIndex(q.rd_bank_0_id)) - val rd1_hazard = q.rd_bank_1_valid && bankWrBusy(bankIndex(q.rd_bank_1_id)) - // Write hazard is NOT checked here. BAT eliminates WAW/WAR by giving each - // write a unique alias that no other instruction uses. Checking wr_hazard - // would falsely block the writer itself, since alloc marks the writer's - // own alias as busy in the same cycle the entry enters the ROB. - rd0_hazard || rd1_hazard - } - - hasHazard := hazardOf(query) - for (i <- 0 until robEntries) { - hazardVec(i) := hazardOf(queryVec(i)) - } - - // --- Compute per-bank deltas to handle simultaneous issue + complete --- - for (bank <- 0 until bankNum) { - val bankU = bank.U(bankIdLen.W) - - // Read counter: +1 per issue read, -1 per complete read - val issRd0 = issue.valid && issue.bits.rd_bank_0_valid && (issue.bits.rd_bank_0_id === bankU) - val issRd1 = issue.valid && issue.bits.rd_bank_1_valid && (issue.bits.rd_bank_1_id === bankU) - val cmpRd0 = complete.valid && complete.bits.rd_bank_0_valid && (complete.bits.rd_bank_0_id === bankU) - val cmpRd1 = complete.valid && complete.bits.rd_bank_1_valid && (complete.bits.rd_bank_1_id === bankU) - - val rdInc = issRd0.asUInt +& issRd1.asUInt - val rdDec = cmpRd0.asUInt +& cmpRd1.asUInt - bankRdCount(bank) := bankRdCount(bank) + rdInc - rdDec - - // Write flag: alloc sets (alias is reserved as soon as it is renamed), - // complete clears. If both happen to same bank, complete takes priority. - val allocWr = alloc.valid && alloc.bits.wr_bank_valid && (alloc.bits.wr_bank_id === bankU) - val cmpWr = complete.valid && complete.bits.wr_bank_valid && (complete.bits.wr_bank_id === bankU) - - when(allocWr && !cmpWr) { - bankWrBusy(bank) := true.B - }.elsewhen(cmpWr) { - bankWrBusy(bank) := false.B - } - } -} diff --git a/arch/src/main/scala/framework/mem-core/.gitignore b/arch/src/main/scala/framework/ip/.gitignore similarity index 100% rename from arch/src/main/scala/framework/mem-core/.gitignore rename to arch/src/main/scala/framework/ip/.gitignore diff --git a/arch/src/main/scala/framework/root/LICENSE b/arch/src/main/scala/framework/ip/LICENSE similarity index 100% rename from arch/src/main/scala/framework/root/LICENSE rename to arch/src/main/scala/framework/ip/LICENSE diff --git a/arch/src/main/scala/framework/mem-core/README.md b/arch/src/main/scala/framework/ip/README.md similarity index 100% rename from arch/src/main/scala/framework/mem-core/README.md rename to arch/src/main/scala/framework/ip/README.md diff --git a/arch/src/main/scala/framework/ip/axi4/build.sc b/arch/src/main/scala/framework/ip/axi4/build.sc new file mode 100644 index 000000000..11322f523 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/build.sc @@ -0,0 +1,24 @@ +import mill._ +import $file.^.^.^.^.^.^.settings + + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.ip.bus.axi4.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Decoder.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Interconnect.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Interface.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/ip/axi4/configs/default.toml b/arch/src/main/scala/framework/ip/axi4/configs/default.toml new file mode 100644 index 000000000..f2c3f04c6 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/configs/default.toml @@ -0,0 +1,3 @@ +addressBits = 44 +dataBits = 128 +idBits = 4 diff --git a/arch/src/main/scala/framework/ip/axi4/configs/uvm.toml b/arch/src/main/scala/framework/ip/axi4/configs/uvm.toml new file mode 100644 index 000000000..2ec323bbd --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/configs/uvm.toml @@ -0,0 +1,17 @@ +[parameters] +addressBits = 32 +dataBits = 64 +idBits = 3 +masters = 3 +slots = 2 + +[[parameters.regions]] +base="4096" +bytes="256" +[[parameters.regions]] +base="8192" +bytes="512" +[[parameters.regions]] +base="12288" +bytes="1024" +[defines] diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/agent/driver.svh b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/agent/driver.svh new file mode 100644 index 000000000..21b5cf438 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/agent/driver.svh @@ -0,0 +1,41 @@ +class driver extends uvm_driver #(cycle_item); + `uvm_component_utils(driver) + virtual decoder_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual decoder_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Decoder interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item req; + forever begin + seq_item_port.get_next_item(req); + @(negedge vif.clock); + vif.reset = req.reset; + vif.ports[0].aw_valid = req.ports[0].aw_valid; + vif.ports[0].ar_valid = req.ports[0].ar_valid; + vif.ports[0].w_valid = req.ports[0].w_valid; + vif.ports[0].r_ready = req.ports[0].r_ready; + vif.ports[0].b_ready = req.ports[0].b_ready; + vif.ports[0].aw = req.ports[0].aw; + vif.ports[0].ar = req.ports[0].ar; + vif.ports[0].w = req.ports[0].w; + for (int p = 1; p < 3; p++) begin + vif.ports[p].aw_ready = req.ports[p].aw_ready; + vif.ports[p].ar_ready = req.ports[p].ar_ready; + vif.ports[p].w_ready = req.ports[p].w_ready; + vif.ports[p].r_valid = req.ports[p].r_valid; + vif.ports[p].b_valid = req.ports[p].b_valid; + vif.ports[p].r = req.ports[p].r; + vif.ports[p].b = req.ports[p].b; + end + @(posedge vif.clock); + req.ports = vif.ports; + #1; + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/agent/item.svh b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/agent/item.svh new file mode 100644 index 000000000..cf44232c4 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/agent/item.svh @@ -0,0 +1,17 @@ +class cycle_item extends uvm_sequence_item; + `uvm_object_utils(cycle_item) + port_signals ports[3]; + bit reset; + function new(string name = "cycle"); + super.new(name); + endfunction +endclass +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(value, UVM_ALL_ON) + `uvm_object_utils_end + logic [3*$bits(port_signals)-1:0] value; + function new(string name = "result"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/agent/monitor.svh b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/agent/monitor.svh new file mode 100644 index 000000000..48f06fcb0 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/agent/monitor.svh @@ -0,0 +1,24 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual decoder_if vif; + uvm_analysis_port #(cycle_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual decoder_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Decoder interface is required") + endfunction + task run_phase(uvm_phase phase); + forever begin + cycle_item t; + @(posedge vif.clock); + t = cycle_item::type_id::create("observed"); + t.ports = vif.ports; + t.reset = vif.reset; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/env/env.svh b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/env/env.svh new file mode 100644 index 000000000..1f8748f8e --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/env/env.svh @@ -0,0 +1,25 @@ +class env extends uvm_env; + `uvm_component_utils(env) + driver source; + monitor mon; + model reference; + uvm_sequencer #(cycle_item) seqr; + result_board #(result_item, 0) scoreboard; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + source = driver::type_id::create("source", this); + mon = monitor::type_id::create("monitor", this); + reference = model::type_id::create("model", this); + seqr = new("seqr", this); + scoreboard = result_board#(result_item, 0)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + source.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(reference.analysis_export); + reference.expected.connect(scoreboard.expected_export); + reference.actual.connect(scoreboard.actual_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/env/model.svh b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/env/model.svh new file mode 100644 index 000000000..9fb588c29 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/env/model.svh @@ -0,0 +1,112 @@ +class model extends uvm_subscriber #(cycle_item); + `uvm_component_utils(model) + uvm_analysis_port #(result_item) expected, actual; + const logic [`AXI_ADDRESS_BITS-1:0] bases[`DECODER_REGIONS] = `DECODER_BASES; + const logic [`AXI_ADDRESS_BITS-1:0] sizes[`DECODER_REGIONS] = `DECODER_SIZES; + bit read_busy, write_busy, data_pending; + int read_target, write_target; + covergroup behavior with function sample ( + int read_target, int write_target, bit concurrent, bit held, bit leading_data, bit reset_busy + ); + option.per_instance = 1; + read_routes: coverpoint read_target {bins all[] = {1, 2};} + write_routes: coverpoint write_target {bins all[] = {1, 2};} + overlap: coverpoint concurrent; + backpressure: coverpoint held; + data_before_address: coverpoint leading_data; + reset_cancellation: coverpoint reset_busy; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected = new("expected", this); + actual = new("actual", this); + behavior = new(); + endfunction + function int branch(logic [`AXI_ADDRESS_BITS-1:0] address); + foreach (bases[i]) if (address >= bases[i] && address - bases[i] < sizes[i]) return 1; + return 2; + endfunction + function logic [3*$bits(port_signals)-1:0] outputs(port_signals ports[3]); + port_signals result[3] = '{default: '0}; + result[0].aw_ready = ports[0].aw_ready; + result[0].ar_ready = ports[0].ar_ready; + result[0].w_ready = ports[0].w_ready; + result[0].r_valid = ports[0].r_valid; + result[0].b_valid = ports[0].b_valid; + if (ports[0].r_valid) result[0].r = ports[0].r; + if (ports[0].b_valid) result[0].b = ports[0].b; + for (int p = 1; p < 3; p++) begin + result[p].aw_valid = ports[p].aw_valid; + result[p].ar_valid = ports[p].ar_valid; + result[p].w_valid = ports[p].w_valid; + result[p].r_ready = ports[p].r_ready; + result[p].b_ready = ports[p].b_ready; + if (ports[p].aw_valid) result[p].aw = ports[p].aw; + if (ports[p].ar_valid) result[p].ar = ports[p].ar; + if (ports[p].w_valid) result[p].w = ports[p].w; + end + return {result[2], result[1], result[0]}; + endfunction + function void write(cycle_item t); + port_signals want[3] = '{default: '0}; + int read_branch, write_branch; + result_item predicted, received; + if (t.reset) begin + behavior.sample(1, 1, 0, 0, 0, read_busy || write_busy); + read_busy = 0; + write_busy = 0; + data_pending = 0; + return; + end + predicted = result_item::type_id::create("expected"); + received = result_item::type_id::create("actual"); + read_branch = branch(t.ports[0].ar.addr); + write_branch = branch(t.ports[0].aw.addr); + want[read_branch].ar_valid = t.ports[0].ar_valid && !read_busy; + want[read_branch].ar = t.ports[0].ar; + want[0].ar_ready = !read_busy && t.ports[read_branch].ar_ready; + want[write_branch].aw_valid = t.ports[0].aw_valid && !write_busy; + want[write_branch].aw = t.ports[0].aw; + want[0].aw_ready = !write_busy && t.ports[write_branch].aw_ready; + if (read_busy) begin + want[0].r_valid = t.ports[read_target].r_valid; + want[0].r = t.ports[read_target].r; + want[read_target].r_ready = t.ports[0].r_ready; + end + if (write_busy) begin + want[0].b_valid = t.ports[write_target].b_valid; + want[0].b = t.ports[write_target].b; + want[write_target].b_ready = t.ports[0].b_ready; + if (data_pending) begin + want[write_target].w_valid = t.ports[0].w_valid; + want[write_target].w = t.ports[0].w; + want[0].w_ready = t.ports[write_target].w_ready; + end + end + predicted.value = outputs(want); + received.value = outputs(t.ports); + expected.write(predicted); + actual.write(received); + behavior.sample( + read_busy ? read_target : read_branch, write_busy ? write_target : write_branch, + read_busy && write_busy, + (t.ports[0].r_valid && !t.ports[0].r_ready) || (t.ports[0].w_valid && !t.ports[0].w_ready), + t.ports[0].w_valid && !write_busy, 0); + if (t.ports[0].ar_valid && t.ports[0].ar_ready) begin + read_busy = 1; + read_target = read_branch; + end + if (t.ports[0].r_valid && t.ports[0].r_ready && t.ports[0].r.last) read_busy = 0; + if (t.ports[0].aw_valid && t.ports[0].aw_ready) begin + write_busy = 1; + data_pending = 1; + write_target = write_branch; + end + if (t.ports[0].w_valid && t.ports[0].w_ready && t.ports[0].w.last) data_pending = 0; + if (t.ports[0].b_valid && t.ports[0].b_ready) write_busy = 0; + endfunction + function void check_phase(uvm_phase phase); + super.check_phase(phase); + if (read_busy || write_busy) `uvm_fatal("PENDING", "Decoder transaction did not drain") + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/filelist.f b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/filelist.f new file mode 100644 index 000000000..b71763cde --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/filelist.f @@ -0,0 +1,9 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/../interconnect/types.svh +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/Decoder/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/interface.sv b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/interface.sv new file mode 100644 index 000000000..14a1c9077 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/interface.sv @@ -0,0 +1,7 @@ +interface decoder_if ( + input logic clock +); + import axi_types::*; + logic reset = 1; + port_signals ports[3]; +endinterface diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/package.sv b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/package.sv new file mode 100644 index 000000000..ba6f3829b --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/package.sv @@ -0,0 +1,14 @@ +`include "decoder_regions.svh" +package decoder_pkg; + import uvm_pkg::*; + import verification_pkg::*; + import axi_types::*; + `include "uvm_macros.svh" + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/seq/sequence.svh b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/seq/sequence.svh new file mode 100644 index 000000000..3f4e481e8 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/seq/sequence.svh @@ -0,0 +1,117 @@ +class scenario_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(scenario_sequence) + port_signals pins[3] = '{default: '0}, observed[3]; + const logic [`AXI_ADDRESS_BITS-1:0] bases[`DECODER_REGIONS] = `DECODER_BASES; + const logic [`AXI_ADDRESS_BITS-1:0] sizes[`DECODER_REGIONS] = `DECODER_SIZES; + int serial; + function new(string name = "scenario"); + super.new(name); + endfunction + task tick(bit reset = 0); + cycle_item t = cycle_item::type_id::create("cycle"); + start_item(t); + t.ports = pins; + t.reset = reset; + finish_item(t); + observed = t.ports; + endtask + task burst(logic [`AXI_ADDRESS_BITS-1:0] read_address, + logic [`AXI_ADDRESS_BITS-1:0] write_address, int beats, bit cancel = 0); + bit read_sent, write_sent; + int rtarget, wtarget; + pins = '{default: '0}; + pins[0].ar = '1; + pins[0].ar.addr = read_address; + pins[0].ar.len = beats - 1; + pins[0].ar.id = I'(serial); + pins[0].ar.burst = 1; + pins[0].aw = '0; + pins[0].aw.addr = write_address; + pins[0].aw.len = beats - 1; + pins[0].aw.id = I'(~serial); + pins[0].aw.burst = 1; + pins[0].w = '1; + pins[0].w.last = beats == 1; + pins[0].w_valid = 1; + repeat (3) tick(); + pins[0].ar_valid = 1; + pins[0].aw_valid = 1; + for (int cycle = 0; !read_sent || !write_sent; cycle++) begin + pins[1].ar_ready = cycle % 3 != 0; + pins[2].ar_ready = cycle % 3 == 0; + pins[1].aw_ready = cycle % 4 == 0; + pins[2].aw_ready = cycle % 4 != 0; + tick(); + if (!read_sent && observed[0].ar_ready) begin + read_sent = 1; + pins[0].ar_valid = 0; + rtarget = observed[1].ar_valid ? 1 : 2; + end + if (!write_sent && observed[0].aw_ready) begin + write_sent = 1; + pins[0].aw_valid = 0; + wtarget = observed[1].aw_valid ? 1 : 2; + end + end + // Offered addresses keep changing after acceptance; response owners stay captured. + pins[0].ar.addr = ~read_address; + pins[0].aw.addr = ~write_address; + if (cancel) begin + repeat (2) tick(1); + pins = '{default: '0}; + tick(); + return; + end + for (int beat = 0; beat < beats; beat++) begin + pins[rtarget].r_valid = 1; + pins[rtarget].r.id = pins[0].ar.id; + pins[rtarget].r.data = D'(64'h123456789abcdef0) ^ beat ^ serial; + pins[rtarget].r.resp = 2'(serial); + pins[rtarget].r.last = beat == beats - 1; + pins[0].w.data = ~pins[rtarget].r.data; + pins[0].w.strb = (beat % 2) ? '1 : 'h55; + pins[0].w.last = beat == beats - 1; + pins[0].w_valid = 1; + // Idle-branch traffic must not leak onto the selected response path. + pins[3-rtarget].r_valid = 1; + pins[3-rtarget].r = '1; + pins[0].r_ready = 0; + pins[wtarget].w_ready = 0; + repeat (3) tick(); + pins[0].r_ready = 1; + pins[wtarget].w_ready = 1; + tick(); + pins[rtarget].r_valid = 0; + pins[3-rtarget].r_valid = 0; + pins[0].w_valid = 0; + end + pins[wtarget].b_valid = 1; + pins[wtarget].b.id = pins[0].aw.id; + pins[wtarget].b.resp = 2'(~serial); + pins[3-wtarget].b_valid = 1; + pins[3-wtarget].b = '1; + pins[0].b_ready = 0; + repeat (3) tick(); + pins[0].b_ready = 1; + tick(); + pins = '{default: '0}; + tick(); + serial++; + endtask + task body(); + repeat (3) tick(1); + tick(); + foreach (bases[i]) begin + burst(bases[i] - 1, bases[i], 1); + burst(bases[i], bases[i] + sizes[i] - 1, 2); + burst(bases[i] + sizes[i] - 1, bases[i] + sizes[i], 3); + burst(bases[i] + sizes[i], bases[i] - 1, 4); + end + for (int i = 0; i < 32; i++) + burst(i % 2 ? bases[i%`DECODER_REGIONS] : 32'h80000000 + i * 64, + i % 3 ? 32'h90000000 + i * 64 : bases[i%`DECODER_REGIONS], 1 + i % 8); + burst(bases[0], 32'h80000000, 4, 1); + burst(32'h80000000, bases[0], 4, 1); + burst(bases[1], bases[2], 4); + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/tb.sv b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/tb.sv new file mode 100644 index 000000000..72d32c46d --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/tb.sv @@ -0,0 +1,15 @@ +module decoder_tb; + import uvm_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + decoder_if bus (clock); + Decoder dut ( + .clock(clock), + .reset(bus.reset), + `include "decoder_ports.svh" + ); + initial begin + uvm_config_db#(virtual decoder_if)::set(null, "uvm_test_top.env.*", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/tests/test.svh b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/tests/test.svh new file mode 100644 index 000000000..54d137a6f --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/tests/test.svh @@ -0,0 +1,15 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.start(test_env.seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/agent/driver.svh b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/agent/driver.svh new file mode 100644 index 000000000..8ec1f6e3e --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/agent/driver.svh @@ -0,0 +1,156 @@ +class driver extends uvm_driver #(batch); + `uvm_component_utils(driver) + virtual bus_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual bus_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "AXI driver requires an interface") + endfunction + task run_phase(uvm_phase phase); + batch req; + address posted_reads[$], posted_writes[$]; + int read_beat[$]; + int w_sent[M]; + int w_completed, r_completed, b_completed, cycles; + int read_selected, write_selected; + int ar_sent[M], aw_sent[M]; + bit write_responded[$]; + address a; + for (int m = 0; m < M; m++) begin + vif.ports[m].ar_valid = 0; + vif.ports[m].aw_valid = 0; + vif.ports[m].w_valid = 0; + vif.ports[m].r_ready = 0; + vif.ports[m].b_ready = 0; + vif.ports[m].ar = '0; + vif.ports[m].aw = '0; + vif.ports[m].w = '0; + end + vif.ports[M].ar_ready = 0; + vif.ports[M].aw_ready = 0; + vif.ports[M].w_ready = 0; + vif.ports[M].r_valid = 0; + vif.ports[M].b_valid = 0; + vif.ports[M].r = '0; + vif.ports[M].b = '0; + forever begin + seq_item_port.get_next_item(req); + posted_reads.delete(); + posted_writes.delete(); + read_beat.delete(); + write_responded.delete(); + w_completed = 0; + r_completed = 0; + b_completed = 0; + cycles = 0; + read_selected = -1; + write_selected = -1; + foreach (w_sent[m]) begin + w_sent[m] = 0; + ar_sent[m] = 0; + aw_sent[m] = 0; + end + while (r_completed < 2 * M || b_completed < 2 * M) begin + do @(negedge vif.clock); while (vif.reset); + for (int m = 0; m < M; m++) begin + a = '0; + a.id = req.serial; + a.addr = 'h1000 + req.serial * 'h100000 + m * 'h10000 + ar_sent[m] * 'h1000; + a.len = req.beats - 1; + a.size = $clog2(D / 8); + a.burst = (req.beats == 256 || req.beats == 1) ? 1 : req.serial % 3; + a.cache = req.serial; + a.prot = req.serial; + a.qos = req.serial ^ m; + a.region = ~req.serial; + vif.ports[m].ar = a; + a.addr = 'h1800 + req.serial * 'h100000 + m * 'h10000 + aw_sent[m] * 'h1000; + vif.ports[m].aw = a; + vif.ports[m].ar_valid = ar_sent[m] < 2; + vif.ports[m].aw_valid = aw_sent[m] < 2; + vif.ports[m].w_valid = w_sent[m] < 2 * req.beats; + a.addr = 'h1800 + req.serial * 'h100000 + m * 'h10000 + (w_sent[m] / req.beats) * 'h1000; + vif.ports[m].w.data = read_pattern(a, w_sent[m] % req.beats) ^ 'h55aa; + vif.ports[m].w.strb = (req.serial % 2) ? (w_sent[m] ^ m) : '1; + vif.ports[m].w.last = w_sent[m] % req.beats == req.beats - 1; + vif.ports[m].r_ready = !req.stalls || (cycles + m) % 5 == 0; + vif.ports[m].b_ready = !req.stalls || (cycles + m) % 4 == 0; + end + vif.ports[M].ar_ready = !req.stalls || cycles % 4 == 0; + vif.ports[M].aw_ready = !req.stalls || cycles % 9 == 0; + vif.ports[M].w_ready = !req.stalls || cycles % 3 == 0; + if (read_selected < 0 && cycles > 12) begin + for (int n = posted_reads.size() - 1; n >= 0; n--) begin + if (read_beat[n] <= posted_reads[n].len) begin + read_selected = n; + break; + end + end + end + vif.ports[M].r_valid = read_selected >= 0; + if (read_selected >= 0) begin + a = posted_reads[read_selected]; + vif.ports[M].r.id = a.id; + vif.ports[M].r.data = read_pattern(a, read_beat[read_selected]); + vif.ports[M].r.resp = (req.serial % 3 == 0) ? 0 : (req.serial % 3) + 1; + vif.ports[M].r.last = read_beat[read_selected] == a.len; + end + if (write_selected < 0 && cycles > 15) begin + for (int n = posted_writes.size() - 1; n >= 0; n--) begin + if (n < w_completed && !write_responded[n]) begin + write_selected = n; + break; + end + end + end + vif.ports[M].b_valid = write_selected >= 0; + if (write_selected >= 0) begin + vif.ports[M].b.id = posted_writes[write_selected].id; + vif.ports[M].b.resp = (req.serial % 3 == 0) ? 0 : (req.serial % 3) + 1; + end + @(posedge vif.clock); + for (int m = 0; m < M; m++) begin + if (vif.ports[m].aw_valid && vif.ports[m].aw_ready) aw_sent[m]++; + if (vif.ports[m].ar_valid && vif.ports[m].ar_ready) ar_sent[m]++; + if (vif.ports[m].w_valid && vif.ports[m].w_ready) w_sent[m]++; + end + if (vif.ports[M].ar_valid && vif.ports[M].ar_ready) begin + posted_reads.push_back(vif.ports[M].ar); + read_beat.push_back(0); + end + if (vif.ports[M].aw_valid && vif.ports[M].aw_ready) begin + posted_writes.push_back(vif.ports[M].aw); + write_responded.push_back(0); + end + if (vif.ports[M].w_valid && vif.ports[M].w_ready) begin + if (vif.ports[M].w.last) begin + w_completed++; + end + end + if (vif.ports[M].r_valid && vif.ports[M].r_ready) begin + read_beat[read_selected]++; + if (vif.ports[M].r.last) r_completed++; + read_selected = -1; + end + if (vif.ports[M].b_valid && vif.ports[M].b_ready) begin + write_responded[write_selected] = 1; + b_completed++; + write_selected = -1; + end + cycles++; + end + @(negedge vif.clock); + for (int m = 0; m < M; m++) begin + vif.ports[m].ar_valid = 0; + vif.ports[m].aw_valid = 0; + vif.ports[m].w_valid = 0; + end + vif.ports[M].r_valid = 0; + vif.ports[M].b_valid = 0; + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/agent/item.svh b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/agent/item.svh new file mode 100644 index 000000000..b44e5bf61 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/agent/item.svh @@ -0,0 +1,33 @@ +class batch extends uvm_sequence_item; + `uvm_object_utils(batch) + int unsigned serial; + int unsigned beats; + bit stalls; + function new(string name = "batch"); + super.new(name); + endfunction +endclass + +class snapshot extends uvm_sequence_item; + `uvm_object_utils(snapshot) + port_signals ports[M+1]; + int unsigned outstanding; + bit reset; + function new(string name = "sample"); + super.new(name); + endfunction +endclass + +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(payload, UVM_ALL_ON) + `uvm_field_int(channel, UVM_ALL_ON) + `uvm_field_int(owner, UVM_ALL_ON) + `uvm_object_utils_end + logic [D+128:0] payload; + int channel; + int owner; + function new(string name = "result_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/agent/monitor.svh b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/agent/monitor.svh new file mode 100644 index 000000000..fe3b2ec5d --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/agent/monitor.svh @@ -0,0 +1,25 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual bus_if vif; + uvm_analysis_port #(snapshot) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual bus_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "AXI monitor requires an interface") + endfunction + task run_phase(uvm_phase phase); + snapshot observed; + forever begin + @(posedge vif.clock); + observed = snapshot::type_id::create("observed"); + observed.ports = vif.ports; + observed.reset = vif.reset; + observed.outstanding = vif.outstanding; + ap.write(observed); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/env/env.svh b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/env/env.svh new file mode 100644 index 000000000..1c167dfec --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/env/env.svh @@ -0,0 +1,27 @@ +class env extends uvm_env; + `uvm_component_utils(env) + driver drv; + monitor mon; + uvm_sequencer #(batch) seqr; + routing_model model; + result_board #(result_item, 0) scoreboard; + reset_controller reset; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + drv = driver::type_id::create("driver", this); + mon = monitor::type_id::create("monitor", this); + seqr = new("seqr", this); + model = routing_model::type_id::create("model", this); + scoreboard = result_board#(result_item, 0)::type_id::create("scoreboard", this); + reset = reset_controller::type_id::create("reset", this); + endfunction + function void connect_phase(uvm_phase phase); + drv.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(model.analysis_export); + model.expected_ap.connect(scoreboard.expected_export); + model.actual_ap.connect(scoreboard.actual_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/env/model.svh b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/env/model.svh new file mode 100644 index 000000000..f30a78105 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/env/model.svh @@ -0,0 +1,186 @@ +class routing_model extends uvm_subscriber #(snapshot); + `uvm_component_utils(routing_model) + typedef struct { + address addr; + int master; + int beats; + } transaction; + transaction reads[$], writes[$], owners[$]; + transaction read_slots[int], write_slots[int]; + transaction read_order[longint][$], write_order[longint][$]; + int live_reads = 0, live_writes = 0; + uvm_analysis_port #(result_item) expected_ap, actual_ap; + covergroup behavior with function sample (int length, bit read, int owner, int response_code); + option.per_instance = 1; + burst_length: coverpoint length { + bins one = {1}; bins short_burst = {[2 : 16]}; bins maximum = {256}; + } + direction: coverpoint read; + master: coverpoint owner {bins each_master[] = {[0 : M - 1]};} + response: coverpoint response_code {bins codes[] = {0, 2, 3};} + endgroup + covergroup load with function sample (int active, int live, bit stalled); + option.per_instance = 1; + concurrency: coverpoint active {bins single = {1}; bins competing = {[2 : M]};} + occupancy: coverpoint live { + bins full = {2 * `AXI_SLOTS}; bins active = {[1 : 2 * `AXI_SLOTS - 1]}; + } + backpressure: coverpoint stalled; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected_ap = new("expected_ap", this); + actual_ap = new("actual_ap", this); + behavior = new(); + load = new(); + endfunction + function void compare_value(logic [D+128:0] expected, logic [D+128:0] actual, int channel, + int owner); + result_item e = result_item::type_id::create("expected"); + result_item a = result_item::type_id::create("actual"); + e.payload = expected; + a.payload = actual; + e.channel = channel; + a.channel = channel; + e.owner = owner; + a.owner = owner; + expected_ap.write(e); + actual_ap.write(a); + endfunction + function void write(snapshot s); + transaction t; + address original; + int active, routed, slot; + longint key; + transaction retired; + read_data response_data; + response response_status; + if (s.reset) begin + evidence::check(live_reads == 0 && live_writes == 0, "Reset with unretired AXI transactions"); + return; + end + evidence::check(s.outstanding == live_reads + live_writes, "AXI outstanding count mismatch"); + active = 0; + for (int m = 0; m < M; m++) begin + if (s.ports[m].ar_valid || s.ports[m].aw_valid) active++; + if (s.ports[m].ar_valid && s.ports[m].ar_ready) begin + t.addr = s.ports[m].ar; + t.master = m; + t.beats = 0; + key = (longint'(m) << I) | t.addr.id; + if (!read_order.exists(key)) read_order[key] = {}; + read_order[key].push_back(t); + reads.push_back(t); + live_reads++; + end + if (s.ports[m].aw_valid && s.ports[m].aw_ready) begin + t.addr = s.ports[m].aw; + t.master = m; + t.beats = 0; + key = (longint'(m) << I) | t.addr.id; + if (!write_order.exists(key)) write_order[key] = {}; + write_order[key].push_back(t); + writes.push_back(t); + owners.push_back(t); + live_writes++; + end + end + load.sample(active, s.outstanding, + (s.ports[M].ar_valid && !s.ports[M].ar_ready) || + (s.ports[M].aw_valid && !s.ports[M].aw_ready) || + (s.ports[M].r_valid && !s.ports[M].r_ready) || + (s.ports[M].b_valid && !s.ports[M].b_ready)); + if (s.ports[M].ar_valid && s.ports[M].ar_ready) begin + if (reads.size() == 0) `uvm_fatal("AR", "Unrequested downstream read") + t = reads.pop_front(); + original = s.ports[M].ar; + slot = original.id; + original.id = t.addr.id; + evidence::check(slot < `AXI_SLOTS && !read_slots.exists(slot), + "Read slot reused before retirement"); + compare_value(t.addr, original, 0, t.master); + read_slots[slot] = t; + end + if (s.ports[M].aw_valid && s.ports[M].aw_ready) begin + if (writes.size() == 0) `uvm_fatal("AW", "Unrequested downstream write") + t = writes.pop_front(); + original = s.ports[M].aw; + slot = original.id; + original.id = t.addr.id; + evidence::check(slot < `AXI_SLOTS && !write_slots.exists(slot), + "Write slot reused before retirement"); + compare_value(t.addr, original, 1, t.master); + write_slots[slot] = t; + end + routed = 0; + for (int m = 0; m < M; m++) + if (s.ports[m].w_valid && s.ports[m].w_ready) begin + if (owners.size() == 0) `uvm_fatal("W", "Write data without an accepted address") + evidence::check(owners[0].master == m && s.ports[M].w_valid && s.ports[M].w_ready, + "Write data routed to the wrong owner"); + compare_value(s.ports[m].w, s.ports[M].w, 2, m); + evidence::check(s.ports[m].w.last == (owners[0].beats == owners[0].addr.len), + "Write burst length mismatch"); + owners[0].beats++; + if (s.ports[m].w.last) t = owners.pop_front(); + routed++; + end + evidence::check(routed == int'(s.ports[M].w_valid && s.ports[M].w_ready), + "Lost or duplicated W beat"); + routed = 0; + for (int m = 0; m < M; m++) + if (s.ports[m].r_valid && s.ports[m].r_ready) begin + slot = s.ports[M].r.id; + if (!read_slots.exists(slot)) `uvm_fatal("R", "Response for an unissued read") + t = read_slots[slot]; + key = (longint'(t.master) << I) | t.addr.id; + evidence::check(read_order[key][0].addr == t.addr, + "Read responses reordered within an original ID"); + evidence::check(t.master == m && s.ports[M].r_valid && s.ports[M].r_ready, + "Read response routed to the wrong master"); + response_data = s.ports[M].r; + response_data.id = t.addr.id; + compare_value(response_data, s.ports[m].r, 3, m); + evidence::check(response_data.last == (t.beats == t.addr.len), "Read burst length mismatch"); + behavior.sample(t.addr.len + 1, 1, m, response_data.resp); + read_slots[slot].beats++; + if (response_data.last) begin + retired = read_order[key].pop_front(); + if (read_order[key].size() == 0) read_order.delete(key); + read_slots.delete(slot); + live_reads--; + end + routed++; + end + evidence::check(routed == int'(s.ports[M].r_valid && s.ports[M].r_ready), + "Lost or duplicated R beat"); + routed = 0; + for (int m = 0; m < M; m++) + if (s.ports[m].b_valid && s.ports[m].b_ready) begin + slot = s.ports[M].b.id; + if (!write_slots.exists(slot)) `uvm_fatal("B", "Response for an unissued write") + t = write_slots[slot]; + key = (longint'(t.master) << I) | t.addr.id; + evidence::check(write_order[key][0].addr == t.addr, + "Write responses reordered within an original ID"); + evidence::check(t.master == m && s.ports[M].b_valid && s.ports[M].b_ready, + "Write response routed to the wrong master"); + response_status = s.ports[M].b; + response_status.id = t.addr.id; + compare_value(response_status, s.ports[m].b, 4, m); + behavior.sample(t.addr.len + 1, 0, m, response_status.resp); + retired = write_order[key].pop_front(); + if (write_order[key].size() == 0) write_order.delete(key); + write_slots.delete(slot); + live_writes--; + routed++; + end + evidence::check(routed == int'(s.ports[M].b_valid && s.ports[M].b_ready), + "Lost or duplicated B response"); + endfunction + function void check_phase(uvm_phase phase); + super.check_phase(phase); + evidence::check(live_reads == 0 && live_writes == 0 && owners.size() == 0, + "AXI transactions remain outstanding"); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/filelist.f b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/filelist.f new file mode 100644 index 000000000..92d911b88 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/filelist.f @@ -0,0 +1,9 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/types.svh +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/Interconnect/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/interface.sv b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/interface.sv new file mode 100644 index 000000000..de9d57756 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/interface.sv @@ -0,0 +1,35 @@ +interface bus_if ( + input logic clock, + input logic reset +); + import axi_types::*; + port_signals ports[M+1]; + logic [$clog2(2*`AXI_SLOTS+1)-1:0] outstanding; + for (genvar p = 0; p <= M; p++) begin + assert property (@(posedge clock) disable iff (reset) + ports[p].aw_valid && !ports[p].aw_ready |=> ports[p].aw_valid && $stable( + ports[p].aw + )) + else $fatal(1, "AXI AW changed while stalled"); + assert property (@(posedge clock) disable iff (reset) + ports[p].ar_valid && !ports[p].ar_ready |=> ports[p].ar_valid && $stable( + ports[p].ar + )) + else $fatal(1, "AXI AR changed while stalled"); + assert property (@(posedge clock) disable iff (reset) + ports[p].w_valid && !ports[p].w_ready |=> ports[p].w_valid && $stable( + ports[p].w + )) + else $fatal(1, "AXI W changed while stalled"); + assert property (@(posedge clock) disable iff (reset) + ports[p].r_valid && !ports[p].r_ready |=> ports[p].r_valid && $stable( + ports[p].r + )) + else $fatal(1, "AXI R changed while stalled"); + assert property (@(posedge clock) disable iff (reset) + ports[p].b_valid && !ports[p].b_ready |=> ports[p].b_valid && $stable( + ports[p].b + )) + else $fatal(1, "AXI B changed while stalled"); + end +endinterface diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/package.sv b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/package.sv new file mode 100644 index 000000000..fb40ddb20 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/package.sv @@ -0,0 +1,13 @@ +package interconnect_pkg; + import uvm_pkg::*; + import verification_pkg::*; + import axi_types::*; + `include "uvm_macros.svh" + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/seq/sequence.svh b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/seq/sequence.svh new file mode 100644 index 000000000..f0e0c7076 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/seq/sequence.svh @@ -0,0 +1,22 @@ +class traffic extends uvm_sequence #(batch); + `uvm_object_utils(traffic) + function new(string name = "traffic"); + super.new(name); + endfunction + task body(); + batch req; + for (int n = 0; n < 16; n++) begin + req = batch::type_id::create("batch"); + start_item(req); + req.serial = n; + case (n % 4) + 0: req.beats = 1; + 1: req.beats = 2; + 2: req.beats = 16; + 3: req.beats = 256; + endcase + req.stalls = n >= 8; + finish_item(req); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/tb.sv b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/tb.sv new file mode 100644 index 000000000..a1a0bbb1a --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/tb.sv @@ -0,0 +1,22 @@ +module interconnect_tb; + import uvm_pkg::*; + import interconnect_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + clock_if timing (clock); + bus_if bus ( + clock, + timing.reset + ); + Interconnect dut ( + .clock(clock), + .reset(timing.reset), + .io_outstanding(bus.outstanding), + `include "ports.svh" + ); + initial begin + uvm_config_db#(virtual bus_if)::set(null, "uvm_test_top.env.*", "vif", bus); + uvm_config_db#(virtual clock_if)::set(null, "uvm_test_top.env.reset", "timing", timing); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/tests/test.svh b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/tests/test.svh new file mode 100644 index 000000000..b3eb6a6e9 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/tests/test.svh @@ -0,0 +1,17 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + traffic seq = traffic::type_id::create("traffic"); + test_env.reset.pulse(4); + seq.start(test_env.seqr); + test_env.reset.pulse(2); + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/types.svh b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/types.svh new file mode 100644 index 000000000..19f28a2ce --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/types.svh @@ -0,0 +1,46 @@ +`include "parameters.svh" +package axi_types; + localparam int M = `AXI_MASTERS; + localparam int D = `AXI_DATA_BITS; + localparam int I = `AXI_ID_BITS; + typedef struct packed { + logic [I-1:0] id; + logic [`AXI_ADDRESS_BITS-1:0] addr; + logic [7:0] len; + logic [2:0] size; + logic [1:0] burst; + logic lock; + logic [3:0] cache; + logic [2:0] prot; + logic [3:0] qos; + logic [3:0] region; + } address; + typedef struct packed { + logic [D-1:0] data; + logic [D/8-1:0] strb; + logic last; + } write_data; + typedef struct packed { + logic [I-1:0] id; + logic [D-1:0] data; + logic [1:0] resp; + logic last; + } read_data; + typedef struct packed { + logic [I-1:0] id; + logic [1:0] resp; + } response; + typedef struct packed { + logic aw_valid, aw_ready, ar_valid, ar_ready, w_valid, w_ready; + logic r_valid, r_ready, b_valid, b_ready; + address aw, ar; + write_data w; + read_data r; + response b; + } port_signals; + function automatic logic [D-1:0] read_pattern(address addr, int beat); + logic [D-1:0] value; + for (int b = 0; b < D / 8; b++) value[b*8+:8] = (addr.addr >> 3) ^ (beat * 17) ^ b; + return value; + endfunction +endpackage diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/suite.toml b/arch/src/main/scala/framework/ip/axi4/src/main/resources/suite.toml new file mode 100644 index 000000000..862416c63 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/suite.toml @@ -0,0 +1,37 @@ +[model] +manifest = "" +library = "" + +[[tests]] +name = "interconnect" +top = "interconnect_tb" +test = "protocol_test" +filelist = "interconnect/filelist.f" +timeout_ns = 2000000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +functional_groups = ["interconnect_pkg::routing_model::behavior", "interconnect_pkg::routing_model::load"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +name="decoder" +top="decoder_tb" +test="protocol_test" +filelist="decoder/filelist.f" +timeout_ns=200000 +wall_timeout_s=120 +kind="functional" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=["decoder_pkg::model::behavior"] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=[] +assertion="" diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/scala/Decoder.scala b/arch/src/main/scala/framework/ip/axi4/src/main/scala/Decoder.scala new file mode 100644 index 000000000..14caed4af --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/scala/Decoder.scala @@ -0,0 +1,64 @@ +package framework.ip.bus.axi4 + +import framework.ip.bus.axi4.{Params => AxiParam} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} + +/** Address-selected AXI branch. One read burst and one write burst may be live independently. */ +@instantiable +class Decoder(val regions: Seq[(BigInt, BigInt)], axiParam: AxiParam) extends Module { + val p = axiParam + + @public + val io = IO(new Bundle { + val in = Flipped(new Port(axiParam)) + val matched = new Port(axiParam) + val other = new Port(axiParam) + }) + + val readBusy = RegInit(false.B) + val writeDataPending = RegInit(false.B) + val writeBusy = RegInit(false.B) + val readMatched = Reg(Bool()) + val writeMatched = Reg(Bool()) + + val arMatched = regions.map { case (base, bytes) => + io.in.ar.bits.addr >= base.U && io.in.ar.bits.addr < (base + bytes).U + }.reduce(_ || _) + + val awMatched = regions.map { case (base, bytes) => + io.in.aw.bits.addr >= base.U && io.in.aw.bits.addr < (base + bytes).U + }.reduce(_ || _) + + io.matched.ar.valid := io.in.ar.valid && !readBusy && arMatched + io.other.ar.valid := io.in.ar.valid && !readBusy && !arMatched + io.matched.ar.bits := io.in.ar.bits + io.other.ar.bits := io.in.ar.bits + io.in.ar.ready := !readBusy && Mux(arMatched, io.matched.ar.ready, io.other.ar.ready) + when(io.in.ar.fire) { readBusy := true.B; readMatched := arMatched } + io.in.r.valid := readBusy && Mux(readMatched, io.matched.r.valid, io.other.r.valid) + io.in.r.bits := Mux(readMatched, io.matched.r.bits, io.other.r.bits) + io.matched.r.ready := readBusy && readMatched && io.in.r.ready + io.other.r.ready := readBusy && !readMatched && io.in.r.ready + when(io.in.r.fire && io.in.r.bits.last)(readBusy := false.B) + + io.matched.aw.valid := io.in.aw.valid && !writeBusy && awMatched + io.other.aw.valid := io.in.aw.valid && !writeBusy && !awMatched + io.matched.aw.bits := io.in.aw.bits + io.other.aw.bits := io.in.aw.bits + io.in.aw.ready := !writeBusy && Mux(awMatched, io.matched.aw.ready, io.other.aw.ready) + when(io.in.aw.fire) { writeBusy := true.B; writeDataPending := true.B; writeMatched := awMatched } + io.matched.w.valid := io.in.w.valid && writeBusy && writeDataPending && writeMatched + io.other.w.valid := io.in.w.valid && writeBusy && writeDataPending && !writeMatched + io.matched.w.bits := io.in.w.bits + io.other.w.bits := io.in.w.bits + io.in.w.ready := writeBusy && writeDataPending && Mux(writeMatched, io.matched.w.ready, io.other.w.ready) + when(io.in.w.fire && io.in.w.bits.last)(writeDataPending := false.B) + io.in.b.valid := writeBusy && Mux(writeMatched, io.matched.b.valid, io.other.b.valid) + io.in.b.bits := Mux(writeMatched, io.matched.b.bits, io.other.b.bits) + io.matched.b.ready := writeBusy && writeMatched && io.in.b.ready + io.other.b.ready := writeBusy && !writeMatched && io.in.b.ready + when(io.in.b.fire)(writeBusy := false.B) +} diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/scala/Emit.scala b/arch/src/main/scala/framework/ip/axi4/src/main/scala/Emit.scala new file mode 100644 index 000000000..0bd41da82 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/scala/Emit.scala @@ -0,0 +1,72 @@ +package framework.ip.bus.axi4 + +import java.nio.file.{Files, Paths} + +object Emit extends App { + val Array("--config", config, "--output-dir", output) = args + val values = ujson.read(Files.readString(Paths.get(config))) + val p = Params(values("addressBits").num.toInt, values("dataBits").num.toInt, values("idBits").num.toInt) + val masters = values("masters").num.toInt + val slots = values("slots").num.toInt + Files.createDirectories(Paths.get(output)) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Interconnect(InterconnectParam(p, masters, Some(slots))), + args = Array("--target-dir", s"$output/Interconnect", "--split-verilog") + ) + Files.writeString( + Paths.get(output, "parameters.svh"), + s"""`ifndef AXI_PARAMETERS +`define AXI_PARAMETERS +`define AXI_ADDRESS_BITS ${p.addressBits} +`define AXI_DATA_BITS ${p.dataBits} +`define AXI_ID_BITS ${p.idBits} +`define AXI_MASTERS $masters +`define AXI_SLOTS $slots +`endif +""" + ) + + val fields = Seq( + "aw" -> Seq("id", "addr", "len", "size", "burst", "lock", "cache", "prot", "qos", "region"), + "ar" -> Seq("id", "addr", "len", "size", "burst", "lock", "cache", "prot", "qos", "region"), + "w" -> Seq("data", "strb", "last"), + "r" -> Seq("id", "data", "resp", "last"), + "b" -> Seq("id", "resp") + ) + + val ports = (0 to masters).flatMap { index => + val prefix = if (index == masters) "io_out" else s"io_in_$index" + fields.flatMap { case (channel, members) => + Seq( + s".${prefix}_${channel}_valid(bus.ports[$index].${channel}_valid)", + s".${prefix}_${channel}_ready(bus.ports[$index].${channel}_ready)" + ) ++ + members.map(field => s".${prefix}_${channel}_bits_$field(bus.ports[$index].$channel.$field)") + } + } + + Files.writeString(Paths.get(output, "ports.svh"), ports.mkString(",\n")) + val regions = values("regions").arr.toSeq.map(v => (BigInt(v("base").str), BigInt(v("bytes").str))) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Decoder(regions, p), + args = Array("--target-dir", s"$output/Decoder", "--split-verilog") + ) + + val decoderPorts = Seq("in", "matched", "other").zipWithIndex.flatMap { case (prefix, index) => + fields.flatMap { case (channel, members) => + Seq( + s".io_${prefix}_${channel}_valid(bus.ports[$index].${channel}_valid)", + s".io_${prefix}_${channel}_ready(bus.ports[$index].${channel}_ready)" + ) ++ members.map(field => s".io_${prefix}_${channel}_bits_$field(bus.ports[$index].$channel.$field)") + } + } + + Files.writeString(Paths.get(output, "decoder_ports.svh"), decoderPorts.mkString(",\n")) + Files.writeString( + Paths.get(output, "decoder_regions.svh"), + s"`define DECODER_REGIONS ${regions.size}\n" + + s"`define DECODER_BASES ${regions.map { case (base, _) => s"${p.addressBits}'h${base.toString(16)}" }.mkString("'{", ",", "}")}\n" + + s"`define DECODER_SIZES ${regions.map { case (_, size) => s"${p.addressBits}'h${size.toString(16)}" }.mkString("'{", ",", "}")}\n" + ) + +} diff --git a/arch/src/main/scala/framework/mem-core/axi4/src/main/scala/Interconnect.scala b/arch/src/main/scala/framework/ip/axi4/src/main/scala/Interconnect.scala similarity index 94% rename from arch/src/main/scala/framework/mem-core/axi4/src/main/scala/Interconnect.scala rename to arch/src/main/scala/framework/ip/axi4/src/main/scala/Interconnect.scala index f16874d9f..adb987dd4 100644 --- a/arch/src/main/scala/framework/mem-core/axi4/src/main/scala/Interconnect.scala +++ b/arch/src/main/scala/framework/ip/axi4/src/main/scala/Interconnect.scala @@ -1,6 +1,8 @@ -package memcore.bus.axi4 +package framework.ip.bus.axi4 -import memcore.memory.queue.Queue +import framework.ip.bus.axi4.InterconnectParam + +import framework.lib.queue.Queue import chisel3._ import chisel3.util._ @@ -18,7 +20,8 @@ class AddressSelector(p: Params, masters: Int) extends Module { require(masters > 0) val masterBits = math.max(1, log2Ceil(masters)) - @public val io = IO(new Bundle { + @public + val io = IO(new Bundle { val in = Vec(masters, Flipped(Decoupled(new Address(p)))) val enabled = Input(Vec(masters, Bool())) val slot = Input(UInt(p.idBits.W)) @@ -59,9 +62,15 @@ class WriteOwner(p: Params, masters: Int) extends Bundle { val slot = UInt(p.idBits.W) } +case class InterconnectParam(bus: Params, masters: Int, slots: Option[Int]) + /** Normal AXI4 transfers, dynamically mapped onto a bounded DDR ID space. */ @instantiable -class Interconnect(p: Params, masters: Int, slots: Option[Int] = None) extends Module { +class Interconnect(b: InterconnectParam) extends Module { + val config = b + val p = config.bus + val masters = config.masters + val slots = config.slots require(masters > 0) val capacity = slots.getOrElse { @@ -73,7 +82,8 @@ class Interconnect(p: Params, masters: Int, slots: Option[Int] = None) extends M require(capacity > 1 && BigInt(capacity) <= (BigInt(1) << p.idBits)) val masterBits = math.max(1, log2Ceil(masters)) - @public val io = IO(new Bundle { + @public + val io = IO(new Bundle { val in = Vec(masters, Flipped(new Port(p))) val out = new Port(p) val outstanding = Output(UInt(log2Ceil(2 * capacity + 1).W)) @@ -95,7 +105,7 @@ class Interconnect(p: Params, masters: Int, slots: Option[Int] = None) extends M val reads: Instance[AddressSelector] = Instantiate(new AddressSelector(p, masters)) val writes: Instance[AddressSelector] = Instantiate(new AddressSelector(p, masters)) - val owners = Module(new Queue(new WriteOwner(p, masters), capacity)) + val owners = Instantiate(new Queue(new WriteOwner(p, masters), capacity)) val readFree = VecInit(readLive.map(live => !live)) val writeFree = VecInit(writeLive.map(live => !live)) reads.io.slot := PriorityEncoder(readFree) diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/scala/Interface.scala b/arch/src/main/scala/framework/ip/axi4/src/main/scala/Interface.scala new file mode 100644 index 000000000..b39ba1fe7 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/scala/Interface.scala @@ -0,0 +1,44 @@ +package framework.ip.bus.axi4 + +import chisel3._ +import chisel3.util._ + +class Address(p: Params) extends Bundle { + val id = UInt(p.idBits.W) + val addr = UInt(p.addressBits.W) + val len = UInt(8.W) + val size = UInt(3.W) + val burst = UInt(2.W) + val lock = Bool() + val cache = UInt(4.W) + val prot = UInt(3.W) + val qos = UInt(4.W) + val region = UInt(4.W) +} + +class WriteData(p: Params) extends Bundle { + val data = UInt(p.dataBits.W) + val strb = UInt(p.bytes.W) + val last = Bool() +} + +class ReadData(p: Params) extends Bundle { + val id = UInt(p.idBits.W) + val data = UInt(p.dataBits.W) + val resp = UInt(2.W) + val last = Bool() +} + +class WriteResponse(p: Params) extends Bundle { + val id = UInt(p.idBits.W) + val resp = UInt(2.W) +} + +/** AXI4 memory-mapped master. Optional user sidebands are absent. */ +class Port(p: Params) extends Bundle { + val aw = Decoupled(new Address(p)) + val w = Decoupled(new WriteData(p)) + val b = Flipped(Decoupled(new WriteResponse(p))) + val ar = Decoupled(new Address(p)) + val r = Flipped(Decoupled(new ReadData(p))) +} diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/scala/Params.scala b/arch/src/main/scala/framework/ip/axi4/src/main/scala/Params.scala new file mode 100644 index 000000000..3d3a32ede --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/scala/Params.scala @@ -0,0 +1,17 @@ +package framework.ip.bus.axi4 + +import upickle.default.{macroRW, ReadWriter} + +import buckyball.config.AxiHardware + +case class Params(addressBits: Int = 44, dataBits: Int = 128, idBits: Int = 4) { + require(addressBits > 0 && idBits > 0) + require(dataBits >= 8 && dataBits <= 1024 && (dataBits & (dataBits - 1)) == 0) + val bytes = dataBits / 8 +} + +object Params { + implicit val rw: ReadWriter[Params] = macroRW + + def fromProto(p: AxiHardware): Params = Params(p.getAddressBits, p.getDataBits, p.getIdBits) +} diff --git a/arch/src/main/scala/framework/ip/axis/build.sc b/arch/src/main/scala/framework/ip/axis/build.sc new file mode 100644 index 000000000..33bf0bcf7 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/build.sc @@ -0,0 +1,24 @@ +import mill._ +import $file.^.^.^.^.^.^.settings + + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.ip.bus.axi.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Interface.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Master.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/PacketArbiter.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Slave.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/ip/axis/configs/uvm.toml b/arch/src/main/scala/framework/ip/axis/configs/uvm.toml new file mode 100644 index 000000000..6d370b490 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/configs/uvm.toml @@ -0,0 +1,6 @@ +[parameters] +dataBits = 32 +inputs = 2 + +[defines] +AXIS_DATA_BITS = "dataBits" diff --git a/arch/src/main/scala/framework/mem-core/axis/src/csrc/Cargo.lock b/arch/src/main/scala/framework/ip/axis/src/csrc/Cargo.lock similarity index 100% rename from arch/src/main/scala/framework/mem-core/axis/src/csrc/Cargo.lock rename to arch/src/main/scala/framework/ip/axis/src/csrc/Cargo.lock diff --git a/arch/src/main/scala/framework/mem-core/axis/src/csrc/Cargo.toml b/arch/src/main/scala/framework/ip/axis/src/csrc/Cargo.toml similarity index 100% rename from arch/src/main/scala/framework/mem-core/axis/src/csrc/Cargo.toml rename to arch/src/main/scala/framework/ip/axis/src/csrc/Cargo.toml diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/README.md b/arch/src/main/scala/framework/ip/axis/src/csrc/README.md similarity index 100% rename from arch/src/main/scala/framework/mem-core/axis/src/main/resources/README.md rename to arch/src/main/scala/framework/ip/axis/src/csrc/README.md diff --git a/arch/src/main/scala/framework/ip/axis/src/csrc/src/arbiter.rs b/arch/src/main/scala/framework/ip/axis/src/csrc/src/arbiter.rs new file mode 100644 index 000000000..853d9cacc --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/csrc/src/arbiter.rs @@ -0,0 +1,23 @@ +#[derive(Default)] +pub(crate) struct ArbiterModel { + selected: Option, +} + +impl ArbiterModel { + pub(crate) fn step(&mut self, valid: u32, ready: bool, last: u32) -> Option { + let selected = self + .selected + .or_else(|| (valid != 0).then(|| valid.trailing_zeros())); + let selected = selected?; + if valid & (1 << selected) == 0 { + return None; + } + + self.selected = if ready && last & (1 << selected) != 0 { + None + } else { + Some(selected) + }; + Some(selected) + } +} diff --git a/arch/src/main/scala/framework/mem-core/axis/src/csrc/src/ffi.rs b/arch/src/main/scala/framework/ip/axis/src/csrc/src/ffi.rs similarity index 100% rename from arch/src/main/scala/framework/mem-core/axis/src/csrc/src/ffi.rs rename to arch/src/main/scala/framework/ip/axis/src/csrc/src/ffi.rs diff --git a/arch/src/main/scala/framework/mem-core/axis/src/csrc/src/lib.rs b/arch/src/main/scala/framework/ip/axis/src/csrc/src/lib.rs similarity index 100% rename from arch/src/main/scala/framework/mem-core/axis/src/csrc/src/lib.rs rename to arch/src/main/scala/framework/ip/axis/src/csrc/src/lib.rs diff --git a/arch/src/main/scala/framework/ip/axis/src/csrc/src/model.rs b/arch/src/main/scala/framework/ip/axis/src/csrc/src/model.rs new file mode 100644 index 000000000..3b9bd7f47 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/csrc/src/model.rs @@ -0,0 +1,23 @@ +use std::collections::VecDeque; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Beat { + pub(crate) data: u32, + pub(crate) keep: u8, + pub(crate) last: bool, +} + +#[derive(Default)] +pub(crate) struct AxiSModel { + beats: VecDeque, +} + +impl AxiSModel { + pub(crate) fn push(&mut self, beat: Beat) { + self.beats.push_back(beat); + } + + pub(crate) fn pop(&mut self) -> Option { + self.beats.pop_front() + } +} diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/agent/agent.svh b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/agent/agent.svh new file mode 100644 index 000000000..bb76bdeab --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/agent/agent.svh @@ -0,0 +1,22 @@ +class agent extends uvm_agent; + `uvm_component_utils(agent) + uvm_sequencer #(cycle_item) seqr; + driver drv; + monitor mon; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + seqr = uvm_sequencer#(cycle_item)::type_id::create("seqr", this); + drv = driver::type_id::create("driver", this); + mon = monitor::type_id::create("monitor", this); + endfunction + + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + drv.seq_item_port.connect(seqr.seq_item_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/agent/driver.svh b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/agent/driver.svh new file mode 100644 index 000000000..9322d5f01 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/agent/driver.svh @@ -0,0 +1,37 @@ +class driver extends uvm_driver #(cycle_item); + `uvm_component_utils(driver) + virtual arbiter_if vif; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual arbiter_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Arbiter driver requires vif") + endfunction + + task run_phase(uvm_phase phase); + cycle_item req; + vif.reset = 1; + vif.in_valid = 0; + vif.in_last = 0; + vif.in_data = '{0, 0}; + vif.in_keep = '{0, 0}; + vif.out_ready = 0; + forever begin + seq_item_port.get_next_item(req); + // Drive on falling edges; monitor and DUT sample on rising edges. + @(negedge vif.clock); + vif.reset = req.reset; + vif.in_valid = req.valid; + vif.in_data = req.data; + vif.in_keep = req.keep; + vif.in_last = req.last; + vif.out_ready = req.ready; + @(posedge vif.clock); + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/agent/item.svh b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/agent/item.svh new file mode 100644 index 000000000..d73806502 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/agent/item.svh @@ -0,0 +1,40 @@ +class cycle_item extends uvm_sequence_item; + bit reset; + logic [1:0] valid; + logic [`AXIS_DATA_BITS-1:0] data[2]; + logic [`AXIS_DATA_BITS/8-1:0] keep[2]; + logic [1:0] last; + logic ready; + logic [1:0] accepted; + logic output_valid; + logic output_last; + `uvm_object_utils_begin(cycle_item) + `uvm_field_int(reset, UVM_ALL_ON) + `uvm_field_int(valid, UVM_ALL_ON) + `uvm_field_sarray_int(data, UVM_ALL_ON) + `uvm_field_sarray_int(keep, UVM_ALL_ON) + `uvm_field_int(last, UVM_ALL_ON) + `uvm_field_int(ready, UVM_ALL_ON) + `uvm_object_utils_end + function new(string name = "cycle_item"); + super.new(name); + endfunction +endclass + +class result_item extends uvm_sequence_item; + logic valid; + logic [1:0] ready; + logic [`AXIS_DATA_BITS-1:0] data; + logic [`AXIS_DATA_BITS/8-1:0] keep; + logic last; + `uvm_object_utils_begin(result_item) + `uvm_field_int(valid, UVM_ALL_ON) + `uvm_field_int(ready, UVM_ALL_ON) + `uvm_field_int(data, UVM_ALL_ON) + `uvm_field_int(keep, UVM_ALL_ON) + `uvm_field_int(last, UVM_ALL_ON) + `uvm_object_utils_end + function new(string name = "result_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/agent/monitor.svh b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/agent/monitor.svh new file mode 100644 index 000000000..5f5341fda --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/agent/monitor.svh @@ -0,0 +1,50 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual arbiter_if vif; + uvm_analysis_port #(cycle_item) input_ap; + uvm_analysis_port #(result_item) output_ap; + int unsigned accepted = 0; + int unsigned transferred = 0; + + function new(string name, uvm_component parent); + super.new(name, parent); + input_ap = new("input_ap", this); + output_ap = new("output_ap", this); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual arbiter_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Arbiter monitor requires vif") + endfunction + + task run_phase(uvm_phase phase); + cycle_item observed; + result_item actual; + forever begin + @(posedge vif.clock); + observed = cycle_item::type_id::create("observed"); + observed.reset = vif.reset; + observed.valid = vif.in_valid; + observed.data = vif.in_data; + observed.keep = vif.in_keep; + observed.last = vif.in_last; + observed.ready = vif.out_ready; + observed.accepted = vif.in_valid & vif.in_ready; + observed.output_valid = vif.out_valid; + observed.output_last = vif.out_last; + input_ap.write(observed); + if (!vif.reset) begin + accepted += $countones(observed.accepted); + if (vif.out_valid && vif.out_ready) transferred++; + actual = result_item::type_id::create("actual"); + actual.valid = vif.out_valid; + actual.ready = vif.in_ready; + actual.data = vif.out_valid ? vif.out_data : 0; + actual.keep = vif.out_valid ? vif.out_keep : 0; + actual.last = vif.out_valid ? vif.out_last : 0; + output_ap.write(actual); + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/env/coverage.svh b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/env/coverage.svh new file mode 100644 index 000000000..94dd8bb3f --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/env/coverage.svh @@ -0,0 +1,44 @@ +class coverage extends uvm_subscriber #(cycle_item); + `uvm_component_utils(coverage) + bit port1_stalled; + bit packet_open; + bit reset_pending; + + covergroup behavior with function sample ( + cycle_item t, bit late_competitor, bit locked_bubble, bit reset_held + ); + option.per_instance = 1; + contenders: coverpoint t.valid iff (!t.reset) {bins masks[] = {[0 : 3]};} + winner: coverpoint t.accepted iff (!t.reset && t.ready) { + bins idle = {0}; bins port0 = {1}; bins port1 = {2}; + } + output_state: coverpoint {t.output_valid, t.ready} iff (!t.reset) {bins states[] = {[0 : 3]};} + late_arrival: coverpoint late_competitor {bins seen = {1};} + owner_gap: coverpoint locked_bubble {bins seen = {1};} + reset_cancellation: coverpoint reset_held {bins seen = {1};} + last_stall: coverpoint (t.output_valid && t.output_last && !t.ready) iff (!t.reset) { + bins seen = {1}; + } + keep0: coverpoint t.keep[0] iff (!t.reset && t.accepted[0]) {bins masks[] = {[0 : 15]};} + endgroup + + function new(string name, uvm_component parent); + super.new(name, parent); + behavior = new(); + endfunction + + function void write(cycle_item t); + behavior.sample(t, port1_stalled && t.valid == 3 && !t.reset, + packet_open && t.valid != 0 && !t.output_valid && !t.reset, + t.reset && reset_pending); + if (t.reset) begin + port1_stalled = 0; + packet_open = 0; + reset_pending = 0; + end else begin + port1_stalled = t.valid == 2 && t.output_valid && !t.ready; + if (t.output_valid) packet_open = !(t.ready && t.output_last); + reset_pending = packet_open; + end + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/env/env.svh b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/env/env.svh new file mode 100644 index 000000000..0d2fc4aff --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/env/env.svh @@ -0,0 +1,22 @@ +class env extends environment #(cycle_item, result_item, ref_model); + `uvm_component_utils(env) + agent bus; + coverage cov; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + bus = agent::type_id::create("agent", this); + cov = coverage::type_id::create("coverage", this); + endfunction + + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + bus.mon.input_ap.connect(input_export); + bus.mon.input_ap.connect(cov.analysis_export); + bus.mon.output_ap.connect(output_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/env/ref_model.svh b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/env/ref_model.svh new file mode 100644 index 000000000..ab629ae3e --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/env/ref_model.svh @@ -0,0 +1,36 @@ +class ref_model extends reference_model #(cycle_item, result_item); + `uvm_component_utils(ref_model) + chandle model; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + model = axis_arbiter_create(); + endfunction + + function void write(cycle_item t); + int unsigned selected; + result_item expected; + if (t.reset) begin + axis_arbiter_destroy(model); + model = axis_arbiter_create(); + return; + end + selected = axis_arbiter_step(model, t.valid, t.ready, t.last); + expected = result_item::type_id::create("expected"); + expected.valid = selected != 32'hffffffff; + expected.ready = expected.valid && t.ready ? 2'b01 << selected : 0; + expected.data = expected.valid ? t.data[selected] : 0; + expected.keep = expected.valid ? t.keep[selected] : 0; + expected.last = expected.valid ? t.last[selected] : 0; + expected_ap.write(expected); + endfunction + + function void final_phase(uvm_phase phase); + axis_arbiter_destroy(model); + super.final_phase(phase); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/filelist.f b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/filelist.f new file mode 100644 index 000000000..8af0617ea --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/filelist.f @@ -0,0 +1,7 @@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv ++incdir+@RESOURCES@ +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/PacketArbiter/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/interface.sv b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/interface.sv new file mode 100644 index 000000000..8b6bb7f0a --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/interface.sv @@ -0,0 +1,36 @@ +interface arbiter_if ( + input logic clock +); + logic reset; + logic [ 1:0] in_valid; + logic [ 1:0] in_ready; + logic [ `AXIS_DATA_BITS-1:0] in_data [2]; + logic [`AXIS_DATA_BITS/8-1:0] in_keep [2]; + logic [ 1:0] in_last; + logic out_valid; + logic out_ready; + logic [ `AXIS_DATA_BITS-1:0] out_data; + logic [`AXIS_DATA_BITS/8-1:0] out_keep; + logic out_last; + + assert property (@(posedge clock) disable iff (reset) $onehot0(in_ready)) + else $fatal(1, "Arbiter accepted more than one input"); + assert property (@(posedge clock) disable iff (reset) + out_valid && !out_ready |=> out_valid && $stable( + {out_data, out_keep, out_last} + )) + else $fatal(1, "Arbiter output changed under backpressure"); + + for (genvar i = 0; i < 2; i++) begin + assert property (@(posedge clock) disable iff (reset) + in_valid[i] && !in_ready[i] |=> in_valid[i] && + $stable( + {in_data[i], in_keep[i], in_last[i]} + )) + else $fatal(1, "Source changed an unaccepted input beat"); + end + + cover property (@(posedge clock) disable iff (reset) &in_valid); + cover property (@(posedge clock) disable iff (reset) in_valid[0] && in_ready[0]); + cover property (@(posedge clock) disable iff (reset) in_valid[1] && in_ready[1]); +endinterface diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/package.sv b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/package.sv new file mode 100644 index 000000000..13592d413 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/package.sv @@ -0,0 +1,24 @@ +package arbiter_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + + import "DPI-C" function chandle axis_arbiter_create(); + import "DPI-C" function void axis_arbiter_destroy(input chandle model); + import "DPI-C" function int unsigned axis_arbiter_step( + input chandle model, + input int unsigned valid, + input byte unsigned ready, + input int unsigned last + ); + + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/agent.svh" + `include "env/ref_model.svh" + `include "env/coverage.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/seq/sequence.svh b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/seq/sequence.svh new file mode 100644 index 000000000..52d168957 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/seq/sequence.svh @@ -0,0 +1,139 @@ +class protocol_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(protocol_sequence) + cycle_item signals; + int unsigned cycles = 0; + int unsigned transfers = 0; + + function new(string name = "protocol_sequence"); + super.new(name); + endfunction + + task tick(int unsigned count, int unsigned beats = 0); + cycle_item req; + repeat (count) begin + req = cycle_item::type_id::create("cycle"); + start_item(req); + req.copy(signals); + finish_item(req); + if (!signals.reset) cycles++; + transfers += beats; + end + endtask + + task body(); + signals = cycle_item::type_id::create("signals"); + signals.reset = 1; + signals.valid = 0; + signals.data = '{0, 32'h55555555}; + signals.keep = '{4'h1, 4'hf}; + signals.last = 0; + signals.ready = 0; + tick(4); + signals.reset = 0; + tick(2); + + `uvm_info("SCENARIO", "Simultaneous packets use fixed priority and preserve packet ownership", + UVM_LOW) + signals.valid = 3; + signals.ready = 0; + tick(2); + signals.ready = 1; + tick(1, 1); + signals.data[0] = '1; + signals.keep[0] = 15; + signals.last = 1; + tick(1, 1); + signals.valid = 2; + tick(1, 1); + signals.data[1] = '1; + signals.keep[1] = 10; + signals.last = 2; + tick(1, 1); + signals.valid = 0; + tick(1); + + `uvm_info("SCENARIO", "A late higher-priority input cannot replace a stalled first beat", + UVM_LOW) + signals.valid = 2; + signals.data[1] = 32'hbbbbbbbb; + signals.keep[1] = 5; + signals.last = 2; + signals.ready = 0; + tick(1); + signals.valid = 3; + signals.data[0] = 32'haaaaaaaa; + signals.keep[0] = 10; + signals.last = 3; + tick(2); + signals.ready = 1; + tick(1, 1); + signals.valid = 1; + tick(1, 1); + signals.valid = 0; + tick(1); + + `uvm_info("SCENARIO", "A locked source may pause; stalled TLAST retains ownership", UVM_LOW) + signals.valid = 2; + signals.data[1] = 32'h12345678; + signals.last = 0; + tick(1, 1); + signals.valid = 1; + signals.last = 1; + tick(2); + signals.valid = 3; + signals.data[1] = 32'h87654321; + signals.last = 3; + signals.ready = 0; + tick(2); + signals.ready = 1; + tick(1, 1); + signals.valid = 1; + tick(1, 1); + signals.valid = 0; + tick(1); + + `uvm_info("SCENARIO", "Single-beat packets preserve every TKEEP mask", UVM_LOW) + signals.valid = 1; + signals.last = 1; + for (int mask = 0; mask < (1 << (`AXIS_DATA_BITS / 8)); mask++) begin + signals.keep[0] = mask; + signals.data[0] = 32'h9e3779b9 * (mask + 1); + tick(1, 1); + end + signals.valid = 0; + tick(1); + + `uvm_info("SCENARIO", "Reset releases packet ownership", UVM_LOW) + signals.valid = 2; + signals.last = 0; + tick(1, 1); + signals.valid = 1; + signals.last = 1; + tick(1); + signals.reset = 1; + signals.valid = 0; + tick(2); + signals.reset = 0; + signals.valid = 1; + tick(1, 1); + signals.valid = 0; + tick(1); + + `uvm_info("SCENARIO", "Reset cancels a stalled first beat", UVM_LOW) + signals.data[1] = 0; + signals.valid = 2; + signals.last = 2; + signals.ready = 0; + tick(1); + signals.reset = 1; + signals.valid = 0; + tick(2); + signals.reset = 0; + signals.valid = 1; + signals.last = 1; + signals.ready = 1; + tick(1, 1); + signals.valid = 0; + tick(2); + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/tb.sv b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/tb.sv new file mode 100644 index 000000000..ea40c226e --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/tb.sv @@ -0,0 +1,36 @@ +module arbiter_tb; + import uvm_pkg::*; + import arbiter_pkg::*; + + logic clock = 1'b0; + wire reset; + always #5 clock = ~clock; + + arbiter_if source_if (clock); + assign reset = source_if.reset; + + PacketArbiter dut ( + .clock(clock), + .reset(reset), + .io_in_0_ready(source_if.in_ready[0]), + .io_in_0_valid(source_if.in_valid[0]), + .io_in_0_bits_tdata(source_if.in_data[0]), + .io_in_0_bits_tkeep(source_if.in_keep[0]), + .io_in_0_bits_tlast(source_if.in_last[0]), + .io_in_1_ready(source_if.in_ready[1]), + .io_in_1_valid(source_if.in_valid[1]), + .io_in_1_bits_tdata(source_if.in_data[1]), + .io_in_1_bits_tkeep(source_if.in_keep[1]), + .io_in_1_bits_tlast(source_if.in_last[1]), + .io_out_ready(source_if.out_ready), + .io_out_valid(source_if.out_valid), + .io_out_bits_tdata(source_if.out_data), + .io_out_bits_tkeep(source_if.out_keep), + .io_out_bits_tlast(source_if.out_last) + ); + + initial begin + uvm_config_db#(virtual arbiter_if)::set(null, "*", "vif", source_if); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/tests/test.svh b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/tests/test.svh new file mode 100644 index 000000000..df1c61638 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/tests/test.svh @@ -0,0 +1,33 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + virtual arbiter_if vif; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual arbiter_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Arbiter test requires vif") + test_env = env::type_id::create("env", this); + endfunction + + task execute(); + protocol_sequence seq = protocol_sequence::type_id::create("seq"); + seq.start(test_env.bus.seqr); + test_env.scoreboard.wait_checked(seq.cycles); + // The monitor finishes the last rising-edge sample before accounting is read. + @(negedge vif.clock); + if (test_env.scoreboard.checked != seq.cycles || + test_env.bus.mon.accepted != seq.transfers || + test_env.bus.mon.transferred != seq.transfers || + test_env.scoreboard.pending() != 0 || + test_env.scoreboard.pending() != 0) + `uvm_fatal("ARBITER_COMPLETE", "Arbiter transaction accounting mismatch") + `uvm_info("ARBITER_CHECKED", $sformatf("cycles=%0d accepted=%0d transferred=%0d pending=0", + test_env.scoreboard.checked, test_env.bus.mon.accepted, + test_env.bus.mon.transferred), UVM_LOW) + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/common/coverage.svh b/arch/src/main/scala/framework/ip/axis/src/main/resources/common/coverage.svh new file mode 100644 index 000000000..e475a6d8f --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/common/coverage.svh @@ -0,0 +1,19 @@ +class coverage extends uvm_subscriber #(item); + `uvm_component_utils(coverage) + covergroup behavior with function sample (logic [`AXIS_DATA_BITS/8-1:0] keep, bit last); + option.per_instance = 1; + masks: coverpoint keep { + bins none = {0}; + bins full = {(1 << (`AXIS_DATA_BITS / 8)) - 1}; + bins partial = {[1 : (1 << (`AXIS_DATA_BITS / 8)) - 2]}; + } + packet_boundary: coverpoint last; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + behavior = new(); + endfunction + function void write(item t); + behavior.sample(t.keep, t.last); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/master/env/env.svh b/arch/src/main/scala/framework/ip/axis/src/main/resources/master/env/env.svh new file mode 100644 index 000000000..301af8cce --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/master/env/env.svh @@ -0,0 +1,44 @@ +class env extends environment #(item, item, ref_model); + `uvm_component_utils(env) + + coverage coverage_monitor; + source_agent source; + sink sink_driver; + monitor sink_monitor; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + virtual axis_if #(`AXIS_DATA_BITS) source_vif; + virtual axis_if #(`AXIS_DATA_BITS) sink_vif; + + super.build_phase(phase); + if (!uvm_config_db#(virtual axis_if #(`AXIS_DATA_BITS))::get( + this, "", "source_vif", source_vif + )) begin + `uvm_fatal("AXIS_VIF", "environment requires source_vif") + end + if (!uvm_config_db#(virtual axis_if #(`AXIS_DATA_BITS))::get( + this, "", "sink_vif", sink_vif + )) begin + `uvm_fatal("AXIS_VIF", "environment requires sink_vif") + end + uvm_config_db#(virtual axis_if #(`AXIS_DATA_BITS))::set(this, "source.*", "vif", source_vif); + uvm_config_db#(virtual axis_if #(`AXIS_DATA_BITS))::set(this, "sink_driver", "vif", sink_vif); + uvm_config_db#(virtual axis_if #(`AXIS_DATA_BITS))::set(this, "sink_monitor", "vif", sink_vif); + + coverage_monitor = coverage::type_id::create("coverage", this); + source = source_agent::type_id::create("source", this); + sink_driver = sink::type_id::create("sink_driver", this); + sink_monitor = monitor::type_id::create("sink_monitor", this); + endfunction + + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + source.mon.ap.connect(input_export); + sink_monitor.ap.connect(output_export); + sink_monitor.ap.connect(coverage_monitor.analysis_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/master/env/ref_model.svh b/arch/src/main/scala/framework/ip/axis/src/main/resources/master/env/ref_model.svh new file mode 100644 index 000000000..6606e1a50 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/master/env/ref_model.svh @@ -0,0 +1,39 @@ +class ref_model extends reference_model #(item, item); + `uvm_component_utils(ref_model) + + chandle model; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + model = axis_ref_create(); + if (model == null) begin + `uvm_fatal("AXIS_MODEL", "failed to create Rust reference model") + end + endfunction + + function void write(item t); + item expected; + int unsigned data; + byte unsigned keep; + byte unsigned last; + + axis_ref_push(model, t.data, t.keep, t.last); + if (!axis_ref_pop(model, data, keep, last)) begin + `uvm_fatal("AXIS_MODEL", "reference model produced no output") + end + expected = item::type_id::create("expected"); + expected.data = data; + expected.keep = keep[3:0]; + expected.last = last[0]; + expected_ap.write(expected); + endfunction + + function void final_phase(uvm_phase phase); + axis_ref_destroy(model); + super.final_phase(phase); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/master/filelist.f b/arch/src/main/scala/framework/ip/axis/src/main/resources/master/filelist.f new file mode 100644 index 000000000..1af63e578 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/master/filelist.f @@ -0,0 +1,9 @@ ++incdir+@VERIFY@/uvm/src/core ++incdir+@VERIFY@/uvm/src/protocol/axis ++incdir+@RESOURCES@ +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/protocol/axis/interface.sv +@VERIFY@/uvm/src/protocol/axis/package.sv +@RESOURCES@/package.sv +-F @RTL@/Master/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/master/package.sv b/arch/src/main/scala/framework/ip/axis/src/main/resources/master/package.sv new file mode 100644 index 000000000..6684fd6e3 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/master/package.sv @@ -0,0 +1,31 @@ +package axis_pkg; + import uvm_pkg::*; + import verification_pkg::*; + import axis_vip_pkg::*; + typedef axis_item#(`AXIS_DATA_BITS) item; + typedef axis_source_agent#(`AXIS_DATA_BITS) source_agent; + typedef axis_monitor#(`AXIS_DATA_BITS) monitor; + typedef axis_sink#(`AXIS_DATA_BITS) sink; + `include "uvm_macros.svh" + + import "DPI-C" function chandle axis_ref_create(); + import "DPI-C" function void axis_ref_destroy(input chandle model); + import "DPI-C" function void axis_ref_push( + input chandle model, + input int unsigned data, + input byte unsigned keep, + input byte unsigned last + ); + import "DPI-C" function byte unsigned axis_ref_pop( + input chandle model, + output int unsigned data, + output byte unsigned keep, + output byte unsigned last + ); + + `include "env/ref_model.svh" + `include "../common/coverage.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/master/seq/sequence.svh b/arch/src/main/scala/framework/ip/axis/src/main/resources/master/seq/sequence.svh new file mode 100644 index 000000000..2f9270f30 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/master/seq/sequence.svh @@ -0,0 +1,19 @@ +class protocol_sequence extends uvm_sequence #(item); + `uvm_object_utils(protocol_sequence) + + function new(string name = "protocol_sequence"); + super.new(name); + endfunction + + task body(); + item req; + for (int i = 0; i < 32; i++) begin + req = item::type_id::create($sformatf("item_%0d", i)); + start_item(req); + req.data = $urandom(); + req.keep = i % (1 << (`AXIS_DATA_BITS / 8)); + req.last = (i % 4) == 3; + finish_item(req); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/master/tb.sv b/arch/src/main/scala/framework/ip/axis/src/main/resources/master/tb.sv new file mode 100644 index 000000000..1c06a459c --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/master/tb.sv @@ -0,0 +1,43 @@ +module master_tb; + import uvm_pkg::*; + import axis_pkg::*; + + logic clock = 1'b0; + logic reset = 1'b0; + always #5 clock = ~clock; + + axis_if #(`AXIS_DATA_BITS) source_if (clock); + axis_if #(`AXIS_DATA_BITS) sink_if (clock); + + assign source_if.reset = reset; + assign sink_if.reset = reset; + + Master dut ( + .clock(clock), + .reset(reset), + .io_in_ready(source_if.tready), + .io_in_valid(source_if.tvalid), + .io_in_bits_tdata(source_if.tdata), + .io_in_bits_tkeep(source_if.tkeep), + .io_in_bits_tlast(source_if.tlast), + .io_axis_tvalid(sink_if.tvalid), + .io_axis_tready(sink_if.tready), + .io_axis_tdata(sink_if.tdata), + .io_axis_tkeep(sink_if.tkeep), + .io_axis_tlast(sink_if.tlast) + ); + + initial begin + reset = 1'b1; + repeat (4) @(negedge clock); + reset = 1'b0; + end + + initial begin + uvm_config_db#(virtual axis_if #(`AXIS_DATA_BITS))::set(null, "uvm_test_top.env", "source_vif", + source_if); + uvm_config_db#(virtual axis_if #(`AXIS_DATA_BITS))::set(null, "uvm_test_top.env", "sink_vif", + sink_if); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/master/tests/test.svh b/arch/src/main/scala/framework/ip/axis/src/main/resources/master/tests/test.svh new file mode 100644 index 000000000..644a639ca --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/master/tests/test.svh @@ -0,0 +1,23 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + + env test_env; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + + task execute(); + protocol_sequence seq; + seq = protocol_sequence::type_id::create("seq"); + seq.start(test_env.source.seqr); + test_env.scoreboard.wait_checked(32); + `uvm_info("AXIS_CHECKED", $sformatf("Checked %0d transfers", test_env.scoreboard.checked), + UVM_LOW) + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/env/env.svh b/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/env/env.svh new file mode 100644 index 000000000..301af8cce --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/env/env.svh @@ -0,0 +1,44 @@ +class env extends environment #(item, item, ref_model); + `uvm_component_utils(env) + + coverage coverage_monitor; + source_agent source; + sink sink_driver; + monitor sink_monitor; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + virtual axis_if #(`AXIS_DATA_BITS) source_vif; + virtual axis_if #(`AXIS_DATA_BITS) sink_vif; + + super.build_phase(phase); + if (!uvm_config_db#(virtual axis_if #(`AXIS_DATA_BITS))::get( + this, "", "source_vif", source_vif + )) begin + `uvm_fatal("AXIS_VIF", "environment requires source_vif") + end + if (!uvm_config_db#(virtual axis_if #(`AXIS_DATA_BITS))::get( + this, "", "sink_vif", sink_vif + )) begin + `uvm_fatal("AXIS_VIF", "environment requires sink_vif") + end + uvm_config_db#(virtual axis_if #(`AXIS_DATA_BITS))::set(this, "source.*", "vif", source_vif); + uvm_config_db#(virtual axis_if #(`AXIS_DATA_BITS))::set(this, "sink_driver", "vif", sink_vif); + uvm_config_db#(virtual axis_if #(`AXIS_DATA_BITS))::set(this, "sink_monitor", "vif", sink_vif); + + coverage_monitor = coverage::type_id::create("coverage", this); + source = source_agent::type_id::create("source", this); + sink_driver = sink::type_id::create("sink_driver", this); + sink_monitor = monitor::type_id::create("sink_monitor", this); + endfunction + + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + source.mon.ap.connect(input_export); + sink_monitor.ap.connect(output_export); + sink_monitor.ap.connect(coverage_monitor.analysis_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/env/ref_model.svh b/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/env/ref_model.svh new file mode 100644 index 000000000..6606e1a50 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/env/ref_model.svh @@ -0,0 +1,39 @@ +class ref_model extends reference_model #(item, item); + `uvm_component_utils(ref_model) + + chandle model; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + model = axis_ref_create(); + if (model == null) begin + `uvm_fatal("AXIS_MODEL", "failed to create Rust reference model") + end + endfunction + + function void write(item t); + item expected; + int unsigned data; + byte unsigned keep; + byte unsigned last; + + axis_ref_push(model, t.data, t.keep, t.last); + if (!axis_ref_pop(model, data, keep, last)) begin + `uvm_fatal("AXIS_MODEL", "reference model produced no output") + end + expected = item::type_id::create("expected"); + expected.data = data; + expected.keep = keep[3:0]; + expected.last = last[0]; + expected_ap.write(expected); + endfunction + + function void final_phase(uvm_phase phase); + axis_ref_destroy(model); + super.final_phase(phase); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/filelist.f b/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/filelist.f new file mode 100644 index 000000000..8bde69336 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/filelist.f @@ -0,0 +1,9 @@ ++incdir+@VERIFY@/uvm/src/core ++incdir+@VERIFY@/uvm/src/protocol/axis ++incdir+@RESOURCES@ +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/protocol/axis/interface.sv +@VERIFY@/uvm/src/protocol/axis/package.sv +@RESOURCES@/package.sv +-F @RTL@/Slave/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/package.sv b/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/package.sv new file mode 100644 index 000000000..6684fd6e3 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/package.sv @@ -0,0 +1,31 @@ +package axis_pkg; + import uvm_pkg::*; + import verification_pkg::*; + import axis_vip_pkg::*; + typedef axis_item#(`AXIS_DATA_BITS) item; + typedef axis_source_agent#(`AXIS_DATA_BITS) source_agent; + typedef axis_monitor#(`AXIS_DATA_BITS) monitor; + typedef axis_sink#(`AXIS_DATA_BITS) sink; + `include "uvm_macros.svh" + + import "DPI-C" function chandle axis_ref_create(); + import "DPI-C" function void axis_ref_destroy(input chandle model); + import "DPI-C" function void axis_ref_push( + input chandle model, + input int unsigned data, + input byte unsigned keep, + input byte unsigned last + ); + import "DPI-C" function byte unsigned axis_ref_pop( + input chandle model, + output int unsigned data, + output byte unsigned keep, + output byte unsigned last + ); + + `include "env/ref_model.svh" + `include "../common/coverage.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/seq/sequence.svh b/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/seq/sequence.svh new file mode 100644 index 000000000..2f9270f30 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/seq/sequence.svh @@ -0,0 +1,19 @@ +class protocol_sequence extends uvm_sequence #(item); + `uvm_object_utils(protocol_sequence) + + function new(string name = "protocol_sequence"); + super.new(name); + endfunction + + task body(); + item req; + for (int i = 0; i < 32; i++) begin + req = item::type_id::create($sformatf("item_%0d", i)); + start_item(req); + req.data = $urandom(); + req.keep = i % (1 << (`AXIS_DATA_BITS / 8)); + req.last = (i % 4) == 3; + finish_item(req); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/tb.sv b/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/tb.sv new file mode 100644 index 000000000..ac2cf0f9e --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/tb.sv @@ -0,0 +1,43 @@ +module slave_tb; + import uvm_pkg::*; + import axis_pkg::*; + + logic clock = 1'b0; + logic reset = 1'b0; + always #5 clock = ~clock; + + axis_if #(`AXIS_DATA_BITS) source_if (clock); + axis_if #(`AXIS_DATA_BITS) sink_if (clock); + + assign source_if.reset = reset; + assign sink_if.reset = reset; + + Slave dut ( + .clock(clock), + .reset(reset), + .io_axis_tvalid(source_if.tvalid), + .io_axis_tready(source_if.tready), + .io_axis_tdata(source_if.tdata), + .io_axis_tkeep(source_if.tkeep), + .io_axis_tlast(source_if.tlast), + .io_out_ready(sink_if.tready), + .io_out_valid(sink_if.tvalid), + .io_out_bits_tdata(sink_if.tdata), + .io_out_bits_tkeep(sink_if.tkeep), + .io_out_bits_tlast(sink_if.tlast) + ); + + initial begin + reset = 1'b1; + repeat (4) @(negedge clock); + reset = 1'b0; + end + + initial begin + uvm_config_db#(virtual axis_if #(`AXIS_DATA_BITS))::set(null, "uvm_test_top.env", "source_vif", + source_if); + uvm_config_db#(virtual axis_if #(`AXIS_DATA_BITS))::set(null, "uvm_test_top.env", "sink_vif", + sink_if); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/tests/test.svh b/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/tests/test.svh new file mode 100644 index 000000000..644a639ca --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/tests/test.svh @@ -0,0 +1,23 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + + env test_env; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + + task execute(); + protocol_sequence seq; + seq = protocol_sequence::type_id::create("seq"); + seq.start(test_env.source.seqr); + test_env.scoreboard.wait_checked(32); + `uvm_info("AXIS_CHECKED", $sformatf("Checked %0d transfers", test_env.scoreboard.checked), + UVM_LOW) + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/suite.toml b/arch/src/main/scala/framework/ip/axis/src/main/resources/suite.toml new file mode 100644 index 000000000..82ea43ede --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/suite.toml @@ -0,0 +1,54 @@ +[model] +manifest = "src/csrc/Cargo.toml" +library = "libaxis_reference_model" + +[[tests]] +name = "arbiter" +functional_groups = ["arbiter_pkg::coverage::behavior"] +top = "arbiter_tb" +test = "protocol_test" +filelist = "arbiter/filelist.f" +timeout_ns = 100000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +name = "master" +functional_groups = ["axis_pkg::coverage::behavior"] +top = "master_tb" +test = "protocol_test" +filelist = "master/filelist.f" +timeout_ns = 100000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +name = "slave" +functional_groups = ["axis_pkg::coverage::behavior"] +top = "slave_tb" +test = "protocol_test" +filelist = "slave/filelist.f" +timeout_ns = 100000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/ip/axis/src/main/scala/Emit.scala b/arch/src/main/scala/framework/ip/axis/src/main/scala/Emit.scala new file mode 100644 index 000000000..eef624205 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/scala/Emit.scala @@ -0,0 +1,22 @@ +package framework.ip.bus.axi + +object Emit extends App { + val Array("--config", config, "--output-dir", output) = args + val parameters = ujson.read(os.read(os.Path(config))) + val dataBits = parameters("dataBits").num.toInt + val inputs = parameters("inputs").num.toInt + require(inputs == 2, "The current UVM arbiter interface requires two inputs") + require(dataBits == 16 || dataBits == 32, "The current UVM reference supports 16 or 32 bits") + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Master(dataBits = dataBits), + args = Array("--target-dir", s"$output/Master", "--split-verilog") + ) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Slave(dataBits = dataBits), + args = Array("--target-dir", s"$output/Slave", "--split-verilog") + ) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new PacketArbiter(inputs = inputs, dataBits = dataBits), + args = Array("--target-dir", s"$output/PacketArbiter", "--split-verilog") + ) +} diff --git a/arch/src/main/scala/framework/ip/axis/src/main/scala/Interface.scala b/arch/src/main/scala/framework/ip/axis/src/main/scala/Interface.scala new file mode 100644 index 000000000..c2412be0e --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/scala/Interface.scala @@ -0,0 +1,51 @@ +package framework.ip.bus.axi + +import chisel3._ + +/** Physical AXI4-Stream signals, directed from the master's perspective. */ +class Port( + dataBits: Int, + idBits: Int = 0, + destBits: Int = 0, + userBits: Int = 0) + extends Bundle { + require(dataBits > 0 && dataBits % 8 == 0) + val tvalid = Output(Bool()) + val tready = Input(Bool()) + val tdata = Output(UInt(dataBits.W)) + val tkeep = Output(UInt((dataBits / 8).W)) + val tlast = Output(Bool()) + val tid = Output(UInt(idBits.W)) + val tdest = Output(UInt(destBits.W)) + val tuser = Output(UInt(userBits.W)) +} + +/** + * AXI4-Stream payload signals. + * + * A `Decoupled[Beat]` is the full AXI-S channel: `valid` and `ready` + * are TVALID and TREADY. This bundle exposes the remaining standard signal + * names directly in generated RTL. + */ +class Beat( + dataBits: Int, + idBits: Int = 0, + destBits: Int = 0, + userBits: Int = 0) + extends Bundle { + require(dataBits > 0 && dataBits % 8 == 0) + val tdata = UInt(dataBits.W) + val tkeep = UInt((dataBits / 8).W) + val tlast = Bool() + val tid = UInt(idBits.W) + val tdest = UInt(destBits.W) + val tuser = UInt(userBits.W) + + // Internal aliases preserve concise datapath code while retaining AXI-S names. + def data: UInt = tdata + def keep: UInt = tkeep + def last: Bool = tlast + def id: UInt = tid + def dest: UInt = tdest + def user: UInt = tuser +} diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/scala/Master.scala b/arch/src/main/scala/framework/ip/axis/src/main/scala/Master.scala similarity index 96% rename from arch/src/main/scala/framework/mem-core/axis/src/main/scala/Master.scala rename to arch/src/main/scala/framework/ip/axis/src/main/scala/Master.scala index 529fe4b55..523f054d4 100644 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/scala/Master.scala +++ b/arch/src/main/scala/framework/ip/axis/src/main/scala/Master.scala @@ -1,4 +1,4 @@ -package memcore.bus.axi +package framework.ip.bus.axi import chisel3._ import chisel3.util._ diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/scala/PacketArbiter.scala b/arch/src/main/scala/framework/ip/axis/src/main/scala/PacketArbiter.scala similarity index 92% rename from arch/src/main/scala/framework/mem-core/axis/src/main/scala/PacketArbiter.scala rename to arch/src/main/scala/framework/ip/axis/src/main/scala/PacketArbiter.scala index 7db666c76..c51ead948 100644 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/scala/PacketArbiter.scala +++ b/arch/src/main/scala/framework/ip/axis/src/main/scala/PacketArbiter.scala @@ -1,4 +1,4 @@ -package memcore.bus.axi +package framework.ip.bus.axi import chisel3._ import chisel3.util._ @@ -42,8 +42,8 @@ class PacketArbiter( io.in(i).ready := io.out.ready && io.out.valid && active === i.U } - when(io.out.fire) { - locked := !io.out.bits.last + when(io.out.valid) { + locked := !io.out.ready || !io.out.bits.last selected := active } } diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/scala/Slave.scala b/arch/src/main/scala/framework/ip/axis/src/main/scala/Slave.scala similarity index 96% rename from arch/src/main/scala/framework/mem-core/axis/src/main/scala/Slave.scala rename to arch/src/main/scala/framework/ip/axis/src/main/scala/Slave.scala index 32cad3250..1cc331b5f 100644 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/scala/Slave.scala +++ b/arch/src/main/scala/framework/ip/axis/src/main/scala/Slave.scala @@ -1,4 +1,4 @@ -package memcore.bus.axi +package framework.ip.bus.axi import chisel3._ import chisel3.util._ diff --git a/arch/src/main/scala/framework/ip/bank/build.sc b/arch/src/main/scala/framework/ip/bank/build.sc new file mode 100644 index 000000000..2278ff7f9 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/build.sc @@ -0,0 +1,23 @@ +import mill._ +import $file.^.^.^.^.^.^.settings + + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.ip.memory.bank.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Bank.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/BankSet.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/SramIO.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/ip/bank/configs/uvm.toml b/arch/src/main/scala/framework/ip/bank/configs/uvm.toml new file mode 100644 index 000000000..eab9824ac --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/configs/uvm.toml @@ -0,0 +1,7 @@ +[parameters] +dataBits = 32 +banks = 4 +entriesPerBank = 16 +tagBits = 8 + +[defines] diff --git a/arch/src/main/scala/framework/mem-core/bank/src/csrc/Cargo.lock b/arch/src/main/scala/framework/ip/bank/src/csrc/Cargo.lock similarity index 100% rename from arch/src/main/scala/framework/mem-core/bank/src/csrc/Cargo.lock rename to arch/src/main/scala/framework/ip/bank/src/csrc/Cargo.lock diff --git a/arch/src/main/scala/framework/mem-core/bank/src/csrc/Cargo.toml b/arch/src/main/scala/framework/ip/bank/src/csrc/Cargo.toml similarity index 100% rename from arch/src/main/scala/framework/mem-core/bank/src/csrc/Cargo.toml rename to arch/src/main/scala/framework/ip/bank/src/csrc/Cargo.toml diff --git a/arch/src/main/scala/framework/mem-core/bank/src/csrc/README.md b/arch/src/main/scala/framework/ip/bank/src/csrc/README.md similarity index 100% rename from arch/src/main/scala/framework/mem-core/bank/src/csrc/README.md rename to arch/src/main/scala/framework/ip/bank/src/csrc/README.md diff --git a/arch/src/main/scala/framework/mem-core/cache/src/csrc/src/lib.rs b/arch/src/main/scala/framework/ip/bank/src/csrc/src/lib.rs similarity index 100% rename from arch/src/main/scala/framework/mem-core/cache/src/csrc/src/lib.rs rename to arch/src/main/scala/framework/ip/bank/src/csrc/src/lib.rs diff --git a/arch/src/main/scala/framework/ip/bank/src/csrc/src/memory.rs b/arch/src/main/scala/framework/ip/bank/src/csrc/src/memory.rs new file mode 100644 index 000000000..f20568ab4 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/csrc/src/memory.rs @@ -0,0 +1,50 @@ +struct Bank { + words: Vec, + initialized: Vec, +} + +impl Bank { + fn access(&mut self, addr: usize, write: bool, data: u32, mask: u8) -> u32 { + if write { + for byte in 0..4 { + if mask & (1 << byte) != 0 { + let bits = 0xff << (byte * 8); + self.words[addr] = (self.words[addr] & !bits) | (data & bits); + } + } + self.initialized[addr] |= mask; + 0 + } else { + assert_eq!( + self.initialized[addr], 0xf, + "read of uninitialized bank word" + ); + self.words[addr] + } + } +} + +#[no_mangle] +pub extern "C" fn bank_ref_create(entries: u32) -> *mut std::ffi::c_void { + Box::into_raw(Box::new(Bank { + words: vec![0; entries as usize], + initialized: vec![0; entries as usize], + })) + .cast() +} + +#[no_mangle] +pub unsafe extern "C" fn bank_ref_destroy(model: *mut std::ffi::c_void) { + drop(Box::from_raw(model.cast::())); +} + +#[no_mangle] +pub unsafe extern "C" fn bank_ref_access( + model: *mut std::ffi::c_void, + addr: u32, + write: u8, + data: u32, + mask: u8, +) -> u32 { + (*model.cast::()).access(addr as usize, write != 0, data, mask) +} diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/agent/agent.svh b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/agent/agent.svh new file mode 100644 index 000000000..3520a6583 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/agent/agent.svh @@ -0,0 +1,22 @@ +class agent extends uvm_agent; + `uvm_component_utils(agent) + uvm_sequencer #(request_item) seqr; + driver drv; + monitor mon; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + seqr = uvm_sequencer#(request_item)::type_id::create("seqr", this); + drv = driver::type_id::create("driver", this); + mon = monitor::type_id::create("monitor", this); + endfunction + + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + drv.seq_item_port.connect(seqr.seq_item_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/agent/driver.svh b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/agent/driver.svh new file mode 100644 index 000000000..c8a88163a --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/agent/driver.svh @@ -0,0 +1,65 @@ +class driver extends uvm_driver #(request_item); + `uvm_component_utils(driver) + virtual port_if request; + virtual port_if response; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual port_if)::get( + this, "", "request", request + ) || !uvm_config_db#(virtual port_if)::get( + this, "", "response", response + )) + `uvm_fatal("VIF", "Bank driver interfaces missing") + endfunction + + task run_phase(uvm_phase phase); + request_item req; + request.valid = 0; + request.write = 0; + request.addr = 0; + request.data = 0; + request.mask = 0; + request.tag = 0; + request.reset = 1; + response.ready = 0; + forever begin + seq_item_port.get_next_item(req); + if (req.reset_command) begin + @(negedge request.clock); + request.reset = 1; + repeat (4) @(negedge request.clock); + request.reset = 0; + end else begin + @(negedge request.clock); + response.ready = 0; + request.valid = 1; + request.write = req.write; + request.addr = req.addr; + request.data = req.data; + request.mask = req.mask; + request.tag = req.tag; + do @(posedge request.clock); while (!request.ready); + @(negedge request.clock); + request.valid = 0; + if (req.cancel_response) begin + request.reset = 1; + repeat (2) @(negedge request.clock); + request.reset = 0; + repeat (3) @(negedge request.clock); + end else begin + while (!response.valid) @(negedge request.clock); + repeat (req.stall_cycles + 1) @(negedge request.clock); + response.ready = 1; + @(negedge request.clock); + // Keep the sink ready while idle; the next transfer selects its stall. + end + end + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/agent/item.svh b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/agent/item.svh new file mode 100644 index 000000000..c4abdb058 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/agent/item.svh @@ -0,0 +1,37 @@ +class request_item extends uvm_sequence_item; + bit reset_command; + bit cancel_response; + int unsigned stall_cycles; + logic [`BANK_ROW_BITS-1:0] addr; + logic write; + logic [31:0] data; + logic [3:0] mask; + logic [`BANK_TAG_BITS-1:0] tag; + `uvm_object_utils_begin(request_item) + `uvm_field_int(reset_command, UVM_ALL_ON) + `uvm_field_int(cancel_response, UVM_ALL_ON) + `uvm_field_int(stall_cycles, UVM_ALL_ON) + `uvm_field_int(addr, UVM_ALL_ON) + `uvm_field_int(write, UVM_ALL_ON) + `uvm_field_int(data, UVM_ALL_ON) + `uvm_field_int(mask, UVM_ALL_ON) + `uvm_field_int(tag, UVM_ALL_ON) + `uvm_object_utils_end + function new(string name = "request_item"); + super.new(name); + endfunction +endclass + +class response_item extends uvm_sequence_item; + logic [31:0] data; + logic [`BANK_TAG_BITS-1:0] tag; + logic error; + `uvm_object_utils_begin(response_item) + `uvm_field_int(data, UVM_ALL_ON) + `uvm_field_int(tag, UVM_ALL_ON) + `uvm_field_int(error, UVM_ALL_ON) + `uvm_object_utils_end + function new(string name = "response_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/agent/monitor.svh b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/agent/monitor.svh new file mode 100644 index 000000000..d65c67821 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/agent/monitor.svh @@ -0,0 +1,74 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual port_if request; + virtual port_if response; + uvm_analysis_port #(request_item) request_ap; + uvm_analysis_port #(response_item) response_ap; + uvm_analysis_port #(bit) reset_ap; + int unsigned accepted = 0; + int unsigned retired = 0; + int unsigned cancelled = 0; + + function new(string name, uvm_component parent); + super.new(name, parent); + request_ap = new("request_ap", this); + response_ap = new("response_ap", this); + reset_ap = new("reset_ap", this); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual port_if)::get( + this, "", "request", request + ) || !uvm_config_db#(virtual port_if)::get( + this, "", "response", response + )) + `uvm_fatal("VIF", "Bank monitor interfaces missing") + endfunction + + task run_phase(uvm_phase phase); + request_item req; + response_item rsp; + bit pending = 0; + forever begin + @(posedge request.clock); + if (request.reset) begin + if (pending) cancelled++; + pending = 0; + reset_ap.write(1); + end else begin + if ($isunknown({request.valid, request.ready, response.valid, response.ready})) + `uvm_fatal("BANK_UNKNOWN", "unknown Bank handshake") + if (request.valid && request.ready) begin + if (pending) `uvm_fatal("BANK_PENDING", "accepted a second outstanding request") + if ($isunknown({request.addr, request.write, request.data, request.mask, request.tag})) + `uvm_fatal("BANK_UNKNOWN", "unknown accepted request") + req = request_item::type_id::create("request"); + req.addr = request.addr; + req.write = request.write; + req.data = request.data; + req.mask = request.mask; + req.tag = request.tag; + pending = 1; + accepted++; + request_ap.write(req); + end + if (response.valid && !pending) + `uvm_fatal("BANK_EXTRA", "response without an outstanding request") + if (pending && !(request.valid && request.ready) && request.ready !== 1'b0) + `uvm_fatal("BANK_PENDING", "request ready with an outstanding response") + if (response.valid && response.ready) begin + if ($isunknown({response.data, response.tag, response.error})) + `uvm_fatal("BANK_UNKNOWN", "unknown accepted response") + rsp = response_item::type_id::create("response"); + rsp.data = response.data; + rsp.tag = response.tag; + rsp.error = response.error; + response_ap.write(rsp); + pending = 0; + retired++; + end + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/env/coverage.svh b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/env/coverage.svh new file mode 100644 index 000000000..2c19ddeaf --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/env/coverage.svh @@ -0,0 +1,20 @@ +class coverage extends uvm_subscriber #(request_item); + `uvm_component_utils(coverage) + + covergroup accesses with function sample (request_item req); + option.per_instance = 1; + direction: coverpoint req.write; + address: coverpoint req.addr {bins rows[] = {[0 : `BANK_ROWS - 1]};} + write_mask: coverpoint req.mask iff (req.write) {bins masks[] = {[0 : 15]};} + direction_address: cross direction, address; + endgroup + + function new(string name, uvm_component parent); + super.new(name, parent); + accesses = new(); + endfunction + + function void write(request_item req); + accesses.sample(req); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/env/env.svh b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/env/env.svh new file mode 100644 index 000000000..551c59c41 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/env/env.svh @@ -0,0 +1,32 @@ +`uvm_analysis_imp_decl(_reset) + +class env extends environment #(request_item, response_item, ref_model); + `uvm_component_utils(env) + agent bus; + coverage cov; + uvm_analysis_imp_reset #(bit, env) reset_export; + + function new(string name, uvm_component parent); + super.new(name, parent); + reset_export = new("reset_export", this); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + bus = agent::type_id::create("agent", this); + cov = coverage::type_id::create("coverage", this); + endfunction + + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + bus.mon.request_ap.connect(input_export); + bus.mon.request_ap.connect(cov.analysis_export); + bus.mon.response_ap.connect(output_export); + bus.mon.reset_ap.connect(reset_export); + endfunction + + function void write_reset(bit asserted); + // Reset cancels responses; accepted writes remain in the reference memory. + scoreboard.cancel_pending(); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/env/ref_model.svh b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/env/ref_model.svh new file mode 100644 index 000000000..2a173fd5d --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/env/ref_model.svh @@ -0,0 +1,26 @@ +class ref_model extends reference_model #(request_item, response_item); + `uvm_component_utils(ref_model) + chandle model; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + model = bank_ref_create(`BANK_ROWS); + endfunction + + function void write(request_item t); + response_item expected = response_item::type_id::create("expected"); + expected.data = bank_ref_access(model, t.addr, t.write, t.data, t.mask); + expected.tag = t.tag; + expected.error = 0; + expected_ap.write(expected); + endfunction + + function void final_phase(uvm_phase phase); + bank_ref_destroy(model); + super.final_phase(phase); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/filelist.f b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/filelist.f new file mode 100644 index 000000000..c6d59af04 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv ++incdir+@RESOURCES@ +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/Bank/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/interface.sv b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/interface.sv new file mode 100644 index 000000000..963a9bc06 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/interface.sv @@ -0,0 +1,20 @@ +`include "parameters.svh" +interface port_if ( + input logic clock +); + logic reset; + logic valid; + logic ready; + logic [`BANK_ROW_BITS-1:0] addr; + logic write; + logic [31:0] data; + logic [3:0] mask; + logic [`BANK_TAG_BITS-1:0] tag; + logic error; + + assert property (@(posedge clock) disable iff (reset) valid && !ready |=> valid && $stable( + {addr, write, data, mask, tag, error} + )) + else $fatal(1, "Bank payload changed under backpressure"); + cover property (@(posedge clock) disable iff (reset) valid && !ready ##1 valid && ready); +endinterface diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/package.sv b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/package.sv new file mode 100644 index 000000000..e06e390b1 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/package.sv @@ -0,0 +1,26 @@ +`include "parameters.svh" +package bank_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + + import "DPI-C" function chandle bank_ref_create(input int unsigned entries); + import "DPI-C" function void bank_ref_destroy(input chandle model); + import "DPI-C" function int unsigned bank_ref_access( + input chandle model, + input int unsigned addr, + input byte unsigned write, + input int unsigned data, + input byte unsigned mask + ); + + `include "agent/item.svh" + `include "env/ref_model.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/agent.svh" + `include "env/coverage.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/seq/sequence.svh b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/seq/sequence.svh new file mode 100644 index 000000000..ae11e60a0 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/seq/sequence.svh @@ -0,0 +1,43 @@ +class access_sequence extends uvm_sequence #(request_item); + `uvm_object_utils(access_sequence) + int unsigned expected_responses = 0; + int unsigned expected_cancellations = 0; + + function new(string name = "access_sequence"); + super.new(name); + endfunction + + task access (bit wr, bit [`BANK_ROW_BITS-1:0] addr, bit [31:0] data, bit [3:0] mask, + bit [`BANK_TAG_BITS-1:0] tag, int unsigned stall, bit cancel_response = 0); + request_item req = request_item::type_id::create("access"); + start_item(req); + req.write = wr; + req.addr = addr; + req.data = data; + req.mask = mask; + req.tag = tag; + req.stall_cycles = stall; + req.cancel_response = cancel_response; + finish_item(req); + if (cancel_response) expected_cancellations++; + else expected_responses++; + endtask + + task body(); + request_item req = request_item::type_id::create("reset"); + start_item(req); + req.reset_command = 1; + finish_item(req); + for (int a = 0; a < `BANK_ROWS; a++) access (1, a, 32'h9e3779b9 * (a + 1), 15, a, 0); + for (int a = 0; a < `BANK_ROWS; a++) access (0, a, 0, 0, 255 - a, 2); + for (int m = 0; m < 16; m++) begin + access (1, m % `BANK_ROWS, ~(32'h9e3779b9 * (m + 1)), m, m, 5); + access (0, m % `BANK_ROWS, 0, 0, m, 0); + end + access (1, `BANK_ROWS - 1, 'hface1234, 15, 255, 0, 1); + access (0, `BANK_ROWS - 1, 0, 0, 42, 3); + access (0, `BANK_ROWS - 1, 0, 15, 255, 0, 1); + access (0, `BANK_ROWS - 1, 0, 0, 42, 3); + for (int a = 0; a < `BANK_ROWS; a++) access (0, a, 0, 0, a, 0); + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/tb.sv b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/tb.sv new file mode 100644 index 000000000..fd8330982 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/tb.sv @@ -0,0 +1,36 @@ +module bank_tb; + import uvm_pkg::*; + import bank_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + port_if source_if (clock); + port_if sink_if (clock); + Bank dut ( + .clock(clock), + .reset(source_if.reset), + .io_request_valid(source_if.valid), + .io_request_ready(source_if.ready), + .io_request_bits_addr(source_if.addr), + .io_request_bits_write(source_if.write), + .io_request_bits_data(source_if.data), + .io_request_bits_mask(source_if.mask), + .io_request_bits_tag(source_if.tag), + .io_response_valid(sink_if.valid), + .io_response_ready(sink_if.ready), + .io_response_bits_data(sink_if.data), + .io_response_bits_tag(sink_if.tag), + .io_response_bits_error(sink_if.error) + ); + assign sink_if.reset = source_if.reset; + assign source_if.error = 0; + assign sink_if.addr = 0; + assign sink_if.write = 0; + assign sink_if.mask = 0; + assert property (@(posedge clock) source_if.reset |=> !sink_if.valid && source_if.ready) + else $fatal(1, "Bank reset did not discard the pending response"); + initial begin + uvm_config_db#(virtual port_if)::set(null, "uvm_test_top*", "request", source_if); + uvm_config_db#(virtual port_if)::set(null, "uvm_test_top*", "response", sink_if); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/tests/test.svh b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/tests/test.svh new file mode 100644 index 000000000..1a31b9a72 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/tests/test.svh @@ -0,0 +1,32 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + virtual port_if request; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + if (!uvm_config_db#(virtual port_if)::get(this, "", "request", request)) + `uvm_fatal("VIF", "Bank test clock interface missing") + endfunction + + task execute(); + access_sequence seq = access_sequence::type_id::create("seq"); + seq.start(test_env.bus.seqr); + test_env.scoreboard.wait_checked(seq.expected_responses); + // Observe the next idle cycles before ending the test. + repeat (2) @(negedge request.clock); + if (test_env.scoreboard.checked != seq.expected_responses || + test_env.bus.mon.retired != seq.expected_responses || + test_env.bus.mon.cancelled != seq.expected_cancellations || + test_env.bus.mon.accepted != seq.expected_responses + seq.expected_cancellations) + `uvm_fatal("BANK_COMPLETE", "Bank transaction accounting mismatch") + `uvm_info("BANK", $sformatf("accepted=%0d checked=%0d cancelled=%0d pending=%0d", + test_env.bus.mon.accepted, test_env.scoreboard.checked, + test_env.bus.mon.cancelled, test_env.scoreboard.pending()), UVM_LOW) + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/agent/agent.svh b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/agent/agent.svh new file mode 100644 index 000000000..82cbf1792 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/agent/agent.svh @@ -0,0 +1,18 @@ +class command_agent extends uvm_agent; + `uvm_component_utils(command_agent) + uvm_sequencer #(command) seqr; + command_driver driver; + command_monitor monitor; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + seqr = new("seqr", this); + driver = command_driver::type_id::create("driver", this); + monitor = command_monitor::type_id::create("monitor", this); + endfunction + function void connect_phase(uvm_phase phase); + driver.seq_item_port.connect(seqr.seq_item_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/agent/command.svh b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/agent/command.svh new file mode 100644 index 000000000..54ce90a1f --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/agent/command.svh @@ -0,0 +1,10 @@ +class command extends uvm_sequence_item; + `uvm_object_utils(command) + bit write; + int unsigned word_addr; + int unsigned beats; + int unsigned completion_stall; + function new(string name = "command"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/agent/driver.svh b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/agent/driver.svh new file mode 100644 index 000000000..cd26758a4 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/agent/driver.svh @@ -0,0 +1,41 @@ +class command_driver extends uvm_driver #(command); + `uvm_component_utils(command_driver) + virtual port_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual port_if)::get(this, "", "control", vif)) + `uvm_fatal("VIF", "BankSet control interface is required") + endfunction + task run_phase(uvm_phase phase); + command req; + vif.valid = 0; + vif.write = 0; + vif.addr = 0; + vif.beats = 0; + vif.done_ready = 0; + forever begin + seq_item_port.get_next_item(req); + do @(negedge vif.clock); while (vif.reset); + vif.valid = 1; + vif.write = req.write; + vif.addr = req.word_addr * 4; + vif.beats = req.beats; + do @(posedge vif.clock); while (!vif.ready); + @(negedge vif.clock); + vif.valid = 0; + wait (vif.done_valid); + repeat (req.completion_stall) @(negedge vif.clock); + @(negedge vif.clock); + vif.done_ready = 1; + @(posedge vif.clock); + @(negedge vif.clock); + vif.done_ready = 0; + evidence::check(vif.ready === 1'b1 && vif.done_valid === 1'b0, + "BankSet did not return to idle after completion"); + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/agent/item.svh b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/agent/item.svh new file mode 100644 index 000000000..c8f6c65cf --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/agent/item.svh @@ -0,0 +1,11 @@ +class operation extends uvm_sequence_item; + bit write; + int unsigned word_addr; + int unsigned data; + byte unsigned mask; + bit last; + `uvm_object_utils(operation) + function new(string name = "operation"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/agent/monitor.svh b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/agent/monitor.svh new file mode 100644 index 000000000..47a86b7ac --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/agent/monitor.svh @@ -0,0 +1,83 @@ +class command_monitor extends uvm_monitor; + `uvm_component_utils(command_monitor) + virtual port_if control; + virtual axis_if write_vif; + virtual axis_if read_vif; + uvm_analysis_port #(operation) ap; + uvm_analysis_port #(command) commands; + int unsigned completed = 0; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + commands = new("commands", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual port_if)::get( + this, "", "control", control + ) || !uvm_config_db#(virtual axis_if)::get( + this, "", "write_vif", write_vif + ) || !uvm_config_db#(virtual axis_if)::get( + this, "", "read_vif", read_vif + )) + `uvm_fatal("VIF", "BankSet monitor interfaces are required") + endfunction + task run_phase(uvm_phase phase); + command active; + operation op; + int unsigned transferred = 0; + forever begin + @(posedge control.clock); + if (control.reset) begin + active = null; + transferred = 0; + end else begin + if (control.valid && control.ready) begin + evidence::check(active == null, "BankSet accepted overlapping commands"); + active = command::type_id::create("observed"); + active.write = control.write; + active.word_addr = control.addr / 4; + active.beats = control.beats; + transferred = 0; + if (!active.write) begin + for (int i = 0; i < active.beats; i++) begin + op = operation::type_id::create("read"); + op.word_addr = active.word_addr + i; + op.last = i == active.beats - 1; + ap.write(op); + end + end + end + if (write_vif.tvalid && write_vif.tready) begin + if (active == null) `uvm_fatal("COMMAND", "Write beat without a command") + evidence::check(active.write && transferred < active.beats, + "Unexpected BankSet write beat"); + op = operation::type_id::create("write"); + op.write = 1; + op.word_addr = active.word_addr + transferred; + op.data = write_vif.tdata; + op.mask = write_vif.tkeep; + ap.write(op); + transferred++; + end + if (read_vif.tvalid && read_vif.tready) begin + if (active == null) `uvm_fatal("COMMAND", "Read beat without a command") + evidence::check(!active.write && transferred < active.beats, + "Unexpected BankSet read beat"); + transferred++; + end + if (control.done_valid) begin + if (active == null) `uvm_fatal("COMMAND", "Completion without a command") + evidence::check( + transferred == active.beats && control.error === 1'b0 && control.ready === 1'b0, + "Invalid BankSet completion"); + if (control.done_ready) begin + commands.write(active); + active = null; + completed++; + end else active.completion_stall++; + end + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/env/coverage.svh b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/env/coverage.svh new file mode 100644 index 000000000..102dce146 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/env/coverage.svh @@ -0,0 +1,25 @@ +class coverage extends uvm_subscriber #(command); + `uvm_component_utils(coverage) + command observed; + covergroup behavior; + option.per_instance = 1; + direction: coverpoint observed.write; + start_bank: coverpoint (observed.word_addr % `BANK_COUNT) { + bins banks[] = {[0 : `BANK_COUNT - 1]}; + } + length: coverpoint observed.beats { + bins single = {1}; bins partial = {[2 : `BANK_WORDS - 1]}; bins full = {`BANK_WORDS}; + } + crossed_row: coverpoint ((observed.word_addr % `BANK_COUNT) + observed.beats > `BANK_COUNT); + stalled: coverpoint (observed.completion_stall > 1); + direction_bank: cross direction, start_bank; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + behavior = new(); + endfunction + function void write(command t); + observed = t; + behavior.sample(); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/env/env.svh b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/env/env.svh new file mode 100644 index 000000000..df09ca487 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/env/env.svh @@ -0,0 +1,27 @@ +class env extends environment #(operation, item, ref_model); + `uvm_component_utils(env) + command_agent commands; + axis_source_agent #(32) source; + axis_sink #(32) sink; + axis_monitor #(32) output_monitor; + reset_controller reset; + coverage cov; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + commands = command_agent::type_id::create("commands", this); + source = axis_source_agent#(32)::type_id::create("source", this); + sink = axis_sink#(32)::type_id::create("sink", this); + output_monitor = axis_monitor#(32)::type_id::create("output_monitor", this); + reset = reset_controller::type_id::create("reset", this); + cov = coverage::type_id::create("coverage", this); + endfunction + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + commands.monitor.ap.connect(input_export); + commands.monitor.commands.connect(cov.analysis_export); + output_monitor.ap.connect(output_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/env/ref_model.svh b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/env/ref_model.svh new file mode 100644 index 000000000..1f70d9e21 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/env/ref_model.svh @@ -0,0 +1,25 @@ +class ref_model extends reference_model #(operation, item); + `uvm_component_utils(ref_model) + chandle model; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + model = bank_ref_create(`BANK_WORDS); + endfunction + function void write(operation t); + int unsigned result = bank_ref_access(model, t.word_addr, t.write, t.data, t.mask); + if (!t.write) begin + item expected = item::type_id::create("expected"); + expected.data = result; + expected.keep = 15; + expected.last = t.last; + expected_ap.write(expected); + end + endfunction + function void final_phase(uvm_phase phase); + bank_ref_destroy(model); + super.final_phase(phase); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/filelist.f b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/filelist.f new file mode 100644 index 000000000..073bd96cb --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/filelist.f @@ -0,0 +1,11 @@ ++incdir+@RTL@ ++incdir+@VERIFY@/uvm/src/core ++incdir+@VERIFY@/uvm/src/protocol/axis ++incdir+@RESOURCES@ +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/protocol/axis/interface.sv +@VERIFY@/uvm/src/protocol/axis/package.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/BankSet/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/interface.sv b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/interface.sv new file mode 100644 index 000000000..6a88956be --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/interface.sv @@ -0,0 +1,20 @@ +`include "parameters.svh" +interface port_if ( + input logic clock, + input logic reset +); + logic valid; + logic ready; + logic write; + logic [`BANK_ADDRESS_BITS-1:0] addr; + logic [31:0] beats; + logic done_valid; + logic done_ready; + logic error; + + assert property (@(posedge clock) disable iff (reset) + done_valid && !done_ready |=> done_valid && $stable( + error + )) + else $fatal(1, "BankSet completion changed under backpressure"); +endinterface diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/package.sv b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/package.sv new file mode 100644 index 000000000..52eb64096 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/package.sv @@ -0,0 +1,28 @@ +`include "parameters.svh" +package bankset_pkg; + import uvm_pkg::*; + import verification_pkg::*; + import axis_vip_pkg::*; + `include "uvm_macros.svh" + typedef axis_item#(32) item; + + import "DPI-C" function chandle bank_ref_create(input int unsigned entries); + import "DPI-C" function void bank_ref_destroy(input chandle model); + import "DPI-C" function int unsigned bank_ref_access( + input chandle model, + input int unsigned addr, + input byte unsigned write, + input int unsigned data, + input byte unsigned mask + ); + `include "agent/item.svh" + `include "agent/command.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/agent.svh" + `include "env/coverage.svh" + `include "env/ref_model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/seq/sequence.svh b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/seq/sequence.svh new file mode 100644 index 000000000..bae1c213d --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/seq/sequence.svh @@ -0,0 +1,77 @@ +class write_sequence extends uvm_sequence #(item); + `uvm_object_utils(write_sequence) + int unsigned beats; + bit masked; + function new(string name = "write_sequence"); + super.new(name); + endfunction + task body(); + item req; + for (int i = 0; i < beats; i++) begin + req = item::type_id::create("req"); + start_item(req); + req.data = 32'h9e3779b9 * (i + 1); + req.keep = masked ? (i % 16) : 15; + req.last = i == beats - 1; + finish_item(req); + end + endtask +endclass + +class command_sequence extends uvm_sequence #(command); + `uvm_object_utils(command_sequence) + command req; + function new(string name = "command_sequence"); + super.new(name); + endfunction + task body(); + start_item(req); + finish_item(req); + endtask +endclass + +class access_sequence extends uvm_sequence; + `uvm_object_utils(access_sequence) + env test_env; + int unsigned reads = 0; + int unsigned commands = 0; + function new(string name = "access_sequence"); + super.new(name); + endfunction + task transfer(bit write, int unsigned word_addr, int unsigned beats, bit masked = 0, + int unsigned stall = 0); + command_sequence cmd = command_sequence::type_id::create("command"); + write_sequence data = write_sequence::type_id::create("data"); + cmd.req = command::type_id::create("req"); + cmd.req.write = write; + cmd.req.word_addr = word_addr; + cmd.req.beats = beats; + cmd.req.completion_stall = stall; + data.beats = beats; + data.masked = masked; + fork + cmd.start(test_env.commands.seqr); + if (write) data.start(test_env.source.seqr); + join + commands++; + if (!write) begin + reads += beats; + test_env.scoreboard.wait_checked(reads); + end + endtask + task body(); + test_env.reset.pulse(4); + transfer(1, 0, `BANK_WORDS); + test_env.sink.stall_cycles = 0; + transfer(0, 0, `BANK_WORDS); + test_env.sink.stall_cycles = 3; + for (int b = 0; b < `BANK_COUNT; b++) begin + transfer(1, b, `BANK_COUNT + 1, 1, 3); + transfer(0, b, `BANK_COUNT + 1, 0, 3); + end + transfer(1, `BANK_WORDS - 1, 1); + transfer(0, `BANK_WORDS - 1, 1); + test_env.reset.pulse(2); + transfer(0, 0, `BANK_WORDS, 0, 3); + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/tb.sv b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/tb.sv new file mode 100644 index 000000000..888283893 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/tb.sv @@ -0,0 +1,49 @@ +module bankset_tb; + import uvm_pkg::*; + import bankset_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + clock_if timing (clock); + port_if control ( + clock, + timing.reset + ); + axis_if source_if (clock); + axis_if sink_if (clock); + assign source_if.reset = control.reset; + assign sink_if.reset = control.reset; + BankSet dut ( + .clock(clock), + .reset(control.reset), + .io_command_valid(control.valid), + .io_command_ready(control.ready), + .io_command_bits_write(control.write), + .io_command_bits_addr(control.addr), + .io_command_bits_beats(control.beats), + .io_write_valid(source_if.tvalid), + .io_write_ready(source_if.tready), + .io_write_bits_tdata(source_if.tdata), + .io_write_bits_tkeep(source_if.tkeep), + .io_write_bits_tlast(source_if.tlast), + .io_read_valid(sink_if.tvalid), + .io_read_ready(sink_if.tready), + .io_read_bits_tdata(sink_if.tdata), + .io_read_bits_tkeep(sink_if.tkeep), + .io_read_bits_tlast(sink_if.tlast), + .io_done_valid(control.done_valid), + .io_done_ready(control.done_ready), + .io_done_bits(control.error) + ); + initial begin + uvm_config_db#(virtual clock_if)::set(null, "uvm_test_top.env.reset", "timing", timing); + uvm_config_db#(virtual port_if)::set(null, "uvm_test_top*", "control", control); + uvm_config_db#(virtual axis_if)::set(null, "uvm_test_top.env.commands.monitor", "write_vif", + source_if); + uvm_config_db#(virtual axis_if)::set(null, "uvm_test_top.env.source.*", "vif", source_if); + uvm_config_db#(virtual axis_if)::set(null, "uvm_test_top.env.sink", "vif", sink_if); + uvm_config_db#(virtual axis_if)::set(null, "uvm_test_top.env.output_monitor", "vif", sink_if); + uvm_config_db#(virtual axis_if)::set(null, "uvm_test_top.env.commands.monitor", "read_vif", + sink_if); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/tests/test.svh b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/tests/test.svh new file mode 100644 index 000000000..956039dc5 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/tests/test.svh @@ -0,0 +1,20 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + access_sequence seq = access_sequence::type_id::create("scenario"); + seq.test_env = test_env; + seq.start(null); + evidence::check(test_env.commands.monitor.completed == seq.commands, + "BankSet command completion count mismatch"); + evidence::check(test_env.scoreboard.checked == seq.reads, + "BankSet read comparison count mismatch"); + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/suite.toml b/arch/src/main/scala/framework/ip/bank/src/main/resources/suite.toml new file mode 100644 index 000000000..009338044 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/suite.toml @@ -0,0 +1,37 @@ +[model] +manifest = "src/csrc/Cargo.toml" +library = "libbank_reference_model" + +[[tests]] +functional_groups = ["bank_pkg::coverage::accesses"] +name = "bank" +top = "bank_tb" +test = "protocol_test" +filelist = "bank/filelist.f" +timeout_ns = 100000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +functional_groups = ["bankset_pkg::coverage::behavior"] +name = "bankset" +top = "bankset_tb" +test = "protocol_test" +filelist = "bankset/filelist.f" +timeout_ns = 100000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/ip/bank/src/main/scala/Bank.scala b/arch/src/main/scala/framework/ip/bank/src/main/scala/Bank.scala new file mode 100644 index 000000000..8f00153a8 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/scala/Bank.scala @@ -0,0 +1,68 @@ +package framework.ip.memory.bank + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} + +class BankRequest(p: BankSetParams) extends Bundle { + val addr = UInt(p.rowBits.W) + val write = Bool() + val data = UInt(p.dataBits.W) + val mask = UInt(p.bytes.W) + val tag = UInt(p.tagBits.W) +} + +class BankResponse(p: BankSetParams) extends Bundle { + val data = UInt(p.dataBits.W) + val tag = UInt(p.tagBits.W) + val error = Bool() +} + +/** One physical SRAM bank. Cache and NoC policy remain outside this module. */ +@instantiable +class Bank(p: BankSetParams) extends Module { + + @public + val io = IO(new Bundle { + val request = Flipped(Decoupled(new BankRequest(p))) + val response = Decoupled(new BankResponse(p)) + }) + + val memory = SyncReadMem(p.entriesPerBank, Vec(p.bytes, UInt(8.W))) + val request = Reg(new BankRequest(p)) + val responseValid = RegInit(false.B) + val readPending = RegInit(false.B) + val responseData = Reg(UInt(p.dataBits.W)) + + val readData = memory.readWrite( + io.request.bits.addr, + io.request.bits.data.asTypeOf(Vec(p.bytes, UInt(8.W))), + io.request.bits.mask.asBools, + io.request.fire, + io.request.bits.write + ) + + io.request.ready := !responseValid && !readPending + when(io.request.fire) { + request := io.request.bits + when(io.request.bits.write) { + responseData := 0.U + responseValid := true.B + }.otherwise { + readPending := true.B + } + } + when(readPending) { + responseData := readData.asUInt + responseValid := true.B + readPending := false.B + } + + io.response.valid := responseValid + io.response.bits.data := responseData + io.response.bits.tag := request.tag + io.response.bits.error := false.B + when(io.response.fire) { + responseValid := false.B + } +} diff --git a/arch/src/main/scala/framework/mem-core/bank/src/main/scala/BankSet.scala b/arch/src/main/scala/framework/ip/bank/src/main/scala/BankSet.scala similarity index 98% rename from arch/src/main/scala/framework/mem-core/bank/src/main/scala/BankSet.scala rename to arch/src/main/scala/framework/ip/bank/src/main/scala/BankSet.scala index 01da682d6..649d13dfc 100644 --- a/arch/src/main/scala/framework/mem-core/bank/src/main/scala/BankSet.scala +++ b/arch/src/main/scala/framework/ip/bank/src/main/scala/BankSet.scala @@ -1,9 +1,9 @@ -package memcore.memory.bank +package framework.ip.memory.bank import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import memcore.bus.axi.Beat +import framework.ip.bus.axi.Beat case class BankSetParams( dataBits: Int, diff --git a/arch/src/main/scala/framework/ip/bank/src/main/scala/Emit.scala b/arch/src/main/scala/framework/ip/bank/src/main/scala/Emit.scala new file mode 100644 index 000000000..9722320e2 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/scala/Emit.scala @@ -0,0 +1,39 @@ +package framework.ip.memory.bank + +import java.nio.file.{Files, Paths} + +object Emit extends App { + val Array("--config", config, "--output-dir", output) = args + val values = ujson.read(Files.readString(Paths.get(config))) + + val p = BankSetParams( + values("dataBits").num.toInt, + values("banks").num.toInt, + values("entriesPerBank").num.toInt, + values("tagBits").num.toInt + ) + + require(p.dataBits == 32, "The Bank reference model uses 32-bit words") + Files.createDirectories(Paths.get(output)) + Files.writeString( + Paths.get(output, "parameters.svh"), + s"""`ifndef BANK_PARAMETERS +`define BANK_PARAMETERS +`define BANK_ROWS ${p.entriesPerBank} +`define BANK_COUNT ${p.banks} +`define BANK_WORDS ${p.entriesPerBank * p.banks} +`define BANK_ROW_BITS ${p.rowBits} +`define BANK_ADDRESS_BITS ${p.addressBits} +`define BANK_TAG_BITS ${p.tagBits} +`endif +""" + ) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Bank(p), + args = Array("--target-dir", s"$output/Bank", "--split-verilog") + ) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new BankSet(p), + args = Array("--target-dir", s"$output/BankSet", "--split-verilog") + ) +} diff --git a/arch/src/main/scala/framework/ip/bank/src/main/scala/SramIO.scala b/arch/src/main/scala/framework/ip/bank/src/main/scala/SramIO.scala new file mode 100644 index 000000000..1a12e62e2 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/scala/SramIO.scala @@ -0,0 +1,36 @@ +package framework.ip.memory.bank + +import chisel3._ +import chisel3.util._ +import framework.acc.memdomain.configs.MemDomainParam + +/** + * Generic SRAM interface definitions + */ +class SramReadReq(val b: MemDomainParam) extends Bundle { + val addr = UInt(16.W) +} + +class SramReadResp(val b: MemDomainParam) extends Bundle { + val data = UInt(b.bankWidth.W) +} + +class SramReadIO(val b: MemDomainParam) extends Bundle { + val req = Flipped(Decoupled(new SramReadReq(b))) + val resp = Decoupled(new SramReadResp(b)) +} + +class SramWriteReq(val b: MemDomainParam) extends Bundle { + val addr = UInt(16.W) + val mask = Vec(b.bankMaskLen, Bool()) + val data = UInt(b.bankWidth.W) +} + +class SramWriteIO(val b: MemDomainParam) extends Bundle { + val req = Flipped(Decoupled(new SramWriteReq(b))) + val resp = Decoupled(new SramWriteResp(b)) +} + +class SramWriteResp(val b: MemDomainParam) extends Bundle { + val ok = Bool() +} diff --git a/arch/src/main/scala/framework/ip/build.sc b/arch/src/main/scala/framework/ip/build.sc new file mode 100644 index 000000000..f24913710 --- /dev/null +++ b/arch/src/main/scala/framework/ip/build.sc @@ -0,0 +1,34 @@ +import mill._ +import $file.axi4.{build => axi4_build} +import $file.axis.{build => axis_build} +import $file.bank.{build => bank_build} +import $file.cache.{build => cache_build} +import $file.chi.{build => chi_build} +import $file.coherence.{build => coherence_build} +import $file.interlock.{build => interlock_build} +import $file.mesh_shm.{build => mesh_shm_build} +import $file.spm.{build => spm_build} + +def sources = T.sources( + axi4_build.sources() ++ + axis_build.sources() ++ + bank_build.sources() ++ + cache_build.sources() ++ + chi_build.sources() ++ + coherence_build.sources() ++ + interlock_build.sources() ++ + mesh_shm_build.sources() ++ + spm_build.sources() +) + +def emitSources = T.sources( + axi4_build.emitSources() ++ + axis_build.emitSources() ++ + bank_build.emitSources() ++ + cache_build.emitSources() ++ + chi_build.emitSources() ++ + coherence_build.emitSources() ++ + interlock_build.emitSources() ++ + mesh_shm_build.emitSources() ++ + spm_build.emitSources() +) diff --git a/arch/src/main/scala/framework/ip/cache/build.sc b/arch/src/main/scala/framework/ip/cache/build.sc new file mode 100644 index 000000000..c377d90c7 --- /dev/null +++ b/arch/src/main/scala/framework/ip/cache/build.sc @@ -0,0 +1,23 @@ +import mill._ +import $file.^.^.^.^.^.^.settings + + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.ip.memory.cache.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("configs/CacheParams.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Cache.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Interface.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/ip/cache/configs/CacheParams.scala b/arch/src/main/scala/framework/ip/cache/configs/CacheParams.scala new file mode 100644 index 000000000..18f4e7a10 --- /dev/null +++ b/arch/src/main/scala/framework/ip/cache/configs/CacheParams.scala @@ -0,0 +1,39 @@ +package framework.ip.memory.cache.configs + +import buckyball.config.CacheHardware + +import chisel3.util.{isPow2, log2Ceil} + +case class CacheParams( + addressBits: Int, + lineBytes: Int, + sets: Int, + ways: Int, + idBits: Int, + metadataBits: Int, + responseDepth: Int) { + require(lineBytes >= 8 && isPow2(lineBytes)) + require(sets >= 2 && isPow2(sets)) + require(ways >= 1 && isPow2(ways)) + require(idBits > 0 && metadataBits > 0 && responseDepth >= 2) + val offsetBits = log2Ceil(lineBytes) + val setBits = log2Ceil(sets) + val wayBits = math.max(1, log2Ceil(ways)) + val tagBits = addressBits - offsetBits - setBits + val lineBits = lineBytes * 8 + require(tagBits > 0) +} + +object CacheParams { + + def fromProto(p: CacheHardware): CacheParams = CacheParams( + p.getAddressBits, + p.getLineBytes, + p.getSets, + p.getWays, + p.getIdBits, + p.getMetadataBits, + p.getResponseDepth + ) + +} diff --git a/arch/src/main/scala/framework/mem-core/cache/configs/default.toml b/arch/src/main/scala/framework/ip/cache/configs/default.toml similarity index 100% rename from arch/src/main/scala/framework/mem-core/cache/configs/default.toml rename to arch/src/main/scala/framework/ip/cache/configs/default.toml diff --git a/arch/src/main/scala/framework/ip/cache/configs/uvm.toml b/arch/src/main/scala/framework/ip/cache/configs/uvm.toml new file mode 100644 index 000000000..e47fe594a --- /dev/null +++ b/arch/src/main/scala/framework/ip/cache/configs/uvm.toml @@ -0,0 +1,10 @@ +[parameters] +addressBits = 44 +lineBytes = 64 +sets = 4 +ways = 2 +idBits = 8 +metadataBits = 4 +responseDepth = 2 + +[defines] diff --git a/arch/src/main/scala/framework/mem-core/cache/src/csrc/Cargo.lock b/arch/src/main/scala/framework/ip/cache/src/csrc/Cargo.lock similarity index 100% rename from arch/src/main/scala/framework/mem-core/cache/src/csrc/Cargo.lock rename to arch/src/main/scala/framework/ip/cache/src/csrc/Cargo.lock diff --git a/arch/src/main/scala/framework/mem-core/cache/src/csrc/Cargo.toml b/arch/src/main/scala/framework/ip/cache/src/csrc/Cargo.toml similarity index 100% rename from arch/src/main/scala/framework/mem-core/cache/src/csrc/Cargo.toml rename to arch/src/main/scala/framework/ip/cache/src/csrc/Cargo.toml diff --git a/arch/src/main/scala/framework/mem-core/coherence/src/csrc/src/lib.rs b/arch/src/main/scala/framework/ip/cache/src/csrc/src/lib.rs similarity index 100% rename from arch/src/main/scala/framework/mem-core/coherence/src/csrc/src/lib.rs rename to arch/src/main/scala/framework/ip/cache/src/csrc/src/lib.rs diff --git a/arch/src/main/scala/framework/mem-core/cache/src/csrc/src/memory.rs b/arch/src/main/scala/framework/ip/cache/src/csrc/src/memory.rs similarity index 100% rename from arch/src/main/scala/framework/mem-core/cache/src/csrc/src/memory.rs rename to arch/src/main/scala/framework/ip/cache/src/csrc/src/memory.rs diff --git a/arch/src/main/scala/framework/ip/cache/src/main/resources/agent/agent.svh b/arch/src/main/scala/framework/ip/cache/src/main/resources/agent/agent.svh new file mode 100644 index 000000000..ee01d47d4 --- /dev/null +++ b/arch/src/main/scala/framework/ip/cache/src/main/resources/agent/agent.svh @@ -0,0 +1,20 @@ +class agent extends uvm_agent; + `uvm_component_utils(agent) + driver drv; + monitor mon; + sink receiver; + uvm_sequencer #(request_item) seqr; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + drv = driver::type_id::create("driver", this); + mon = monitor::type_id::create("monitor", this); + receiver = sink::type_id::create("sink", this); + seqr = new("seqr", this); + endfunction + function void connect_phase(uvm_phase phase); + drv.seq_item_port.connect(seqr.seq_item_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/cache/src/main/resources/agent/driver.svh b/arch/src/main/scala/framework/ip/cache/src/main/resources/agent/driver.svh new file mode 100644 index 000000000..d90b97f2e --- /dev/null +++ b/arch/src/main/scala/framework/ip/cache/src/main/resources/agent/driver.svh @@ -0,0 +1,57 @@ +class driver extends uvm_driver #(request_item); + `uvm_component_utils(driver) + virtual cache_request_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual cache_request_if)::get(this, "", "request", vif)) + `uvm_fatal("VIF", "Cache request interface is required") + endfunction + task run_phase(uvm_phase phase); + request_item req; + int cycles; + vif.reset = 1; + vif.valid = 0; + vif.id = 0; + vif.op = 0; + vif.addr = 0; + vif.way = 0; + vif.data = 0; + vif.mask = 0; + vif.metadata = 0; + vif.eligible = 0; + forever begin + seq_item_port.get_next_item(req); + if (req.reset_command) begin + vif.cb.valid <= 0; + vif.cb.reset <= 1; + repeat (req.reset_cycles) @(vif.cb); + vif.cb.reset <= 0; + @(vif.cb); + end else begin + vif.cb.valid <= 1; + vif.cb.id <= req.id; + vif.cb.op <= req.op; + vif.cb.addr <= req.addr; + vif.cb.way <= req.way; + vif.cb.data <= req.data; + vif.cb.mask <= req.mask; + vif.cb.metadata <= req.metadata; + vif.cb.eligible <= req.eligible; + cycles = 0; + do begin + @(vif.cb); + cycles++; + end while (!vif.cb.ready); + if (req.check_rate) + evidence::check(cycles == 1, + "Cache inserted a request bubble without response backpressure"); + vif.cb.valid <= 0; + if (req.idle_between) @(vif.cb); + end + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/cache/src/main/resources/agent/item.svh b/arch/src/main/scala/framework/ip/cache/src/main/resources/agent/item.svh new file mode 100644 index 000000000..6a24ad548 --- /dev/null +++ b/arch/src/main/scala/framework/ip/cache/src/main/resources/agent/item.svh @@ -0,0 +1,40 @@ +class request_item extends uvm_sequence_item; + bit reset_command; + int unsigned reset_cycles; + bit check_rate; + bit idle_between; + logic [`CACHE_ID_BITS-1:0] id; + logic [2:0] op; + logic [`CACHE_ADDR_BITS-1:0] addr; + logic [`CACHE_WAY_BITS-1:0] way; + bit [`CACHE_LINE_BITS-1:0] data; + bit [`CACHE_LINE_BYTES-1:0] mask; + logic [`CACHE_META_BITS-1:0] metadata; + bit [`CACHE_WAYS-1:0] eligible; + `uvm_object_utils(request_item) + function new(string name = "request_item"); + super.new(name); + endfunction +endclass + +class response_item extends uvm_sequence_item; + logic [`CACHE_ID_BITS-1:0] id; + logic hit, available, entry_valid; + logic [`CACHE_WAY_BITS-1:0] way; + logic [`CACHE_ADDR_BITS-1:0] addr; + logic [`CACHE_LINE_BITS-1:0] data; + logic [`CACHE_META_BITS-1:0] metadata; + `uvm_object_utils_begin(response_item) + `uvm_field_int(id, UVM_ALL_ON) + `uvm_field_int(hit, UVM_ALL_ON) + `uvm_field_int(available, UVM_ALL_ON) + `uvm_field_int(entry_valid, UVM_ALL_ON) + `uvm_field_int(way, UVM_ALL_ON) + `uvm_field_int(addr, UVM_ALL_ON) + `uvm_field_int(data, UVM_ALL_ON) + `uvm_field_int(metadata, UVM_ALL_ON) + `uvm_object_utils_end + function new(string name = "response_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/cache/src/main/resources/agent/monitor.svh b/arch/src/main/scala/framework/ip/cache/src/main/resources/agent/monitor.svh new file mode 100644 index 000000000..6f7e158eb --- /dev/null +++ b/arch/src/main/scala/framework/ip/cache/src/main/resources/agent/monitor.svh @@ -0,0 +1,60 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual cache_request_if request; + virtual cache_response_if response; + uvm_analysis_port #(request_item) request_ap; + uvm_analysis_port #(response_item) response_ap; + uvm_analysis_port #(bit) reset_ap; + int unsigned accepted = 0, retired = 0; + function new(string name, uvm_component parent); + super.new(name, parent); + request_ap = new("request_ap", this); + response_ap = new("response_ap", this); + reset_ap = new("reset_ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual cache_request_if)::get( + this, "", "request", request + ) || !uvm_config_db#(virtual cache_response_if)::get( + this, "", "response", response + )) + `uvm_fatal("VIF", "Cache monitor interfaces are required") + endfunction + task run_phase(uvm_phase phase); + request_item req; + response_item rsp; + forever begin + @(posedge request.clock); + if (request.reset) reset_ap.write(1); + else begin + if (request.valid && request.ready) begin + req = request_item::type_id::create("request"); + req.id = request.id; + req.op = request.op; + req.addr = request.addr; + req.way = request.way; + req.data = request.data; + req.mask = request.mask; + req.metadata = request.metadata; + req.eligible = request.eligible; + request_ap.write(req); + accepted++; + end + if (response.valid && response.ready) begin + rsp = response_item::type_id::create("response"); + rsp.id = response.id; + rsp.hit = response.hit; + rsp.available = response.available; + rsp.entry_valid = response.entry_valid; + rsp.way = response.way; + rsp.addr = response.addr; + rsp.data = response.data; + rsp.metadata = response.metadata; + response_ap.write(rsp); + retired++; + end + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/cache/src/main/resources/agent/sink.svh b/arch/src/main/scala/framework/ip/cache/src/main/resources/agent/sink.svh new file mode 100644 index 000000000..e44a8a52b --- /dev/null +++ b/arch/src/main/scala/framework/ip/cache/src/main/resources/agent/sink.svh @@ -0,0 +1,19 @@ +class sink extends uvm_component; + `uvm_component_utils(sink) + virtual cache_response_if vif; + bit blocked = 0; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual cache_response_if)::get(this, "", "response", vif)) + `uvm_fatal("VIF", "Cache response interface is required") + endfunction + task run_phase(uvm_phase phase); + forever begin + @(vif.cb); + vif.cb.ready <= !vif.reset && !blocked; + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/cache/src/main/resources/env/coverage.svh b/arch/src/main/scala/framework/ip/cache/src/main/resources/env/coverage.svh new file mode 100644 index 000000000..605384f05 --- /dev/null +++ b/arch/src/main/scala/framework/ip/cache/src/main/resources/env/coverage.svh @@ -0,0 +1,23 @@ +class coverage extends uvm_subscriber #(request_item); + `uvm_component_utils(coverage) + covergroup operations with function sample (request_item req); + option.per_instance = 1; + operation: coverpoint req.op {bins supported[] = {[0 : 4]};} + set_index: coverpoint ((req.addr / `CACHE_LINE_BYTES) % `CACHE_SETS) { + bins sets[] = {[0 : `CACHE_SETS - 1]}; + } + way: coverpoint req.way {bins ways[] = {[0 : `CACHE_WAYS - 1]};} + mask: coverpoint req.mask iff (req.op == WRITE) { + bins none = {'0}; bins all = {'1}; bins partial = default; + } + eligibility: coverpoint (req.eligible != 0); + operation_set: cross operation, set_index; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + operations = new(); + endfunction + function void write(request_item t); + operations.sample(t); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/cache/src/main/resources/env/env.svh b/arch/src/main/scala/framework/ip/cache/src/main/resources/env/env.svh new file mode 100644 index 000000000..5ba1235cd --- /dev/null +++ b/arch/src/main/scala/framework/ip/cache/src/main/resources/env/env.svh @@ -0,0 +1,26 @@ +class env extends environment #(request_item, response_item, ref_model); + `uvm_component_utils(env) + agent bus; + coverage cov; + uvm_analysis_imp #(bit, env) reset_export; + function new(string name, uvm_component parent); + super.new(name, parent); + reset_export = new("reset_export", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + bus = agent::type_id::create("bus", this); + cov = coverage::type_id::create("coverage", this); + endfunction + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + bus.mon.request_ap.connect(input_export); + bus.mon.request_ap.connect(cov.analysis_export); + bus.mon.response_ap.connect(output_export); + bus.mon.reset_ap.connect(reset_export); + endfunction + function void write(bit value); + scoreboard.cancel_pending(); + cache_ref_reset(model.model); + endfunction +endclass diff --git a/arch/src/main/scala/framework/mem-core/cache/src/main/resources/ref_model.svh b/arch/src/main/scala/framework/ip/cache/src/main/resources/env/ref_model.svh similarity index 100% rename from arch/src/main/scala/framework/mem-core/cache/src/main/resources/ref_model.svh rename to arch/src/main/scala/framework/ip/cache/src/main/resources/env/ref_model.svh diff --git a/arch/src/main/scala/framework/ip/cache/src/main/resources/filelist.f b/arch/src/main/scala/framework/ip/cache/src/main/resources/filelist.f new file mode 100644 index 000000000..00e97f384 --- /dev/null +++ b/arch/src/main/scala/framework/ip/cache/src/main/resources/filelist.f @@ -0,0 +1,8 @@ ++incdir+@VERIFY@/uvm/src/core ++incdir+@RTL@ ++incdir+@RESOURCES@ +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/Cache/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/mem-core/cache/src/main/resources/interface.sv b/arch/src/main/scala/framework/ip/cache/src/main/resources/interface.sv similarity index 100% rename from arch/src/main/scala/framework/mem-core/cache/src/main/resources/interface.sv rename to arch/src/main/scala/framework/ip/cache/src/main/resources/interface.sv diff --git a/arch/src/main/scala/framework/ip/cache/src/main/resources/package.sv b/arch/src/main/scala/framework/ip/cache/src/main/resources/package.sv new file mode 100644 index 000000000..b5cccf94f --- /dev/null +++ b/arch/src/main/scala/framework/ip/cache/src/main/resources/package.sv @@ -0,0 +1,41 @@ +`include "cache_config.svh" +package cache_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + localparam LOOKUP = 0, READ = 1, WRITE = 2, FILL = 3, INVALIDATE = 4; + import "DPI-C" function chandle cache_ref_create( + input int unsigned sets, + ways, + line_bytes + ); + import "DPI-C" function void cache_ref_destroy(input chandle model); + import "DPI-C" function void cache_ref_reset(input chandle model); + import "DPI-C" function void cache_ref_access( + input chandle model, + input int unsigned op, + input longint unsigned addr, + input int unsigned way, + input bit [`CACHE_LINE_BITS-1:0] data, + input bit [`CACHE_LINE_BYTES-1:0] mask, + input int unsigned metadata, + input bit [`CACHE_WAYS-1:0] eligible, + output byte unsigned hit, + available, + output int unsigned out_way, + output byte unsigned entry_valid, + output longint unsigned out_addr, + output bit [`CACHE_LINE_BITS-1:0] out_data, + output int unsigned out_metadata + ); + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/sink.svh" + `include "agent/agent.svh" + `include "env/ref_model.svh" + `include "env/coverage.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/ip/cache/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/ip/cache/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..02ca880e5 --- /dev/null +++ b/arch/src/main/scala/framework/ip/cache/src/main/resources/seq/sequence.svh @@ -0,0 +1,122 @@ +class access_sequence extends uvm_sequence #(request_item); + `uvm_object_utils(access_sequence) + env test_env; + int unsigned expected = 0; + bit [`CACHE_ID_BITS-1:0] serial = 0; + bit check_rate = 1, idle_between = 0; + function new(string name = "access_sequence"); + super.new(name); + endfunction + task pulse_reset(int unsigned cycles); + request_item req = request_item::type_id::create("reset"); + start_item(req); + req.reset_command = 1; + req.reset_cycles = cycles; + finish_item(req); + endtask + function longint unsigned address(int tag, int set_index); + return (longint'(tag) * `CACHE_SETS + set_index) * `CACHE_LINE_BYTES; + endfunction + task send(int op, longint unsigned addr, int way = 0, bit [`CACHE_LINE_BYTES-1:0] mask = '1, + bit [`CACHE_WAYS-1:0] eligible = '1, bit [`CACHE_LINE_BITS-1:0] data = '0, + int metadata = 0); + request_item req = request_item::type_id::create("access"); + start_item(req); + req.id = serial; + req.op = op; + req.addr = addr; + req.way = way; + req.data = data; + req.mask = mask; + req.metadata = metadata; + req.eligible = eligible; + req.check_rate = check_rate; + req.idle_between = idle_between; + finish_item(req); + expected++; + serial++; + endtask + task drain(); + test_env.scoreboard.wait_checked(expected); + @(test_env.bus.mon.request.cb); + endtask + task body(); + pulse_reset(4); + send(LOOKUP, 0, 0, '0, '0); + send(LOOKUP, 0); + for (int s = 0; s < `CACHE_SETS; s++) begin + for (int w = 0; w < `CACHE_WAYS; w++) begin + send(READ, address(w + 1, s), w); + send(FILL, address(w + 1, s), w, '1, '1, w[0] ? '1 : '0, w); + send(LOOKUP, address(w + 1, s)); + send(WRITE, address(w + 1, s), w, {(`CACHE_LINE_BYTES / 2) {2'b01}}, '1, w[0] ? '0 : '1, + '1); + send(READ, address(w + 1, s), w); + end + end + send(LOOKUP, address(`CACHE_WAYS + 1, 0)); + send(LOOKUP, address(`CACHE_WAYS + 1, 0), 0, '0, '0); + for (int w = 0; w < `CACHE_WAYS; w++) send(LOOKUP, address(`CACHE_WAYS + 1, 0), 0, '0, 1 << w); + send(INVALIDATE, address(1, 0), 0); + send(LOOKUP, address(1, 0)); + send(FILL, address(`CACHE_WAYS + 1, 0), 0, '1, '1, '1, '1); + send(LOOKUP, address(`CACHE_WAYS + 2, 0)); + send(WRITE, address(`CACHE_WAYS + 1, 0), 0, '0, '1, '0, 0); + send(READ, address(`CACHE_WAYS + 1, 0), 0); + send(INVALIDATE, address(`CACHE_WAYS + 1, 0), 0); + send(INVALIDATE, address(`CACHE_WAYS + 1, 0), 0); + drain(); + + idle_between = 1; + for (int s = 0; s < `CACHE_SETS; s++) begin + for (int w = 0; w < `CACHE_WAYS; w++) send(INVALIDATE, address(0, s), w); + for (int w = 0; w < `CACHE_WAYS; w++) begin + longint unsigned high_addr = (((64'h1 << `CACHE_ADDR_BITS) - 1) & + ~(longint'(`CACHE_SETS * `CACHE_LINE_BYTES) - 1)) | (s * `CACHE_LINE_BYTES); + send(FILL, address(0, s), w, '1, '1, '0, 0); + send(LOOKUP, address(0, s), 0, '0, '0); + send(FILL, high_addr, w, '1, '1, '1, '1); + send(LOOKUP, high_addr); + send(WRITE, high_addr, w, '1, '1, '0, 0); + send(READ, high_addr, w); + send(FILL, address(0, s), w, '1, '1, '0, 0); + send(READ, address(0, s), w); + send(INVALIDATE, address(0, s), w); + end + end + drain(); + + idle_between = 0; + serial = '1; + send(READ, address(0, 0), 0, '0, '0); + drain(); + serial = 0; + send(READ, address(0, 0), 0, '0, '0); + drain(); + check_rate = 0; + test_env.bus.receiver.blocked = 1; + repeat (2) @(test_env.bus.mon.request.cb); + fork + begin + for (int i = 0; i < `CACHE_RESPONSE_DEPTH + 2; i++) send(LOOKUP, address(2, 0)); + end + begin + repeat (`CACHE_RESPONSE_DEPTH + 5) @(test_env.bus.mon.request.cb); + test_env.bus.receiver.blocked = 0; + end + join + drain(); + + test_env.bus.receiver.blocked = 1; + repeat (2) @(test_env.bus.mon.request.cb); + for (int i = 0; i < `CACHE_RESPONSE_DEPTH; i++) send(READ, address(1, 1), 0); + repeat (2) @(test_env.bus.mon.request.cb); + pulse_reset(2); + expected = test_env.scoreboard.checked; + test_env.bus.receiver.blocked = 0; + @(test_env.bus.mon.request.cb); + for (int s = 0; s < `CACHE_SETS; s++) send(LOOKUP, address(1, s)); + drain(); + evidence::check(test_env.bus.mon.retired == expected, "Cache response count mismatch"); + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/cache/src/main/resources/suite.toml b/arch/src/main/scala/framework/ip/cache/src/main/resources/suite.toml new file mode 100644 index 000000000..9aa170f00 --- /dev/null +++ b/arch/src/main/scala/framework/ip/cache/src/main/resources/suite.toml @@ -0,0 +1,20 @@ +[model] +manifest = "src/csrc/Cargo.toml" +library = "libcache_reference_model" + +[[tests]] +functional_groups = ["cache_pkg::coverage::operations"] +name = "cache" +top = "cache_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 100000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/mem-core/cache/src/main/resources/cache_tb.sv b/arch/src/main/scala/framework/ip/cache/src/main/resources/tb.sv similarity index 100% rename from arch/src/main/scala/framework/mem-core/cache/src/main/resources/cache_tb.sv rename to arch/src/main/scala/framework/ip/cache/src/main/resources/tb.sv diff --git a/arch/src/main/scala/framework/ip/cache/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/ip/cache/src/main/resources/tests/test.svh new file mode 100644 index 000000000..f24dba40a --- /dev/null +++ b/arch/src/main/scala/framework/ip/cache/src/main/resources/tests/test.svh @@ -0,0 +1,18 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + access_sequence seq = access_sequence::type_id::create("scenario"); + seq.test_env = test_env; + seq.start(test_env.bus.seqr); + evidence::check(test_env.bus.mon.accepted == test_env.scoreboard.checked + evidence::cancelled, + "Cache transaction accounting mismatch"); + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/cache/src/main/scala/Cache.scala b/arch/src/main/scala/framework/ip/cache/src/main/scala/Cache.scala new file mode 100644 index 000000000..eeb087eec --- /dev/null +++ b/arch/src/main/scala/framework/ip/cache/src/main/scala/Cache.scala @@ -0,0 +1,112 @@ +package framework.ip.memory.cache + +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import framework.lib.queue.Queue + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.ip.memory.cache.configs.CacheParams + +@instantiable +class Cache(p: CacheParams) extends Module { + @public + val io = IO(new CacheIO(p)) + + val valid = RegInit(VecInit(Seq.fill(p.ways)(VecInit(Seq.fill(p.sets)(false.B))))) + val tags = Reg(Vec(p.ways, Vec(p.sets, UInt(p.tagBits.W)))) + val metadata = Reg(Vec(p.ways, Vec(p.sets, UInt(p.metadataBits.W)))) + val data = Seq.fill(p.ways)(SyncReadMem(p.sets, Vec(p.lineBytes, UInt(8.W)))) + val nextWay = RegInit(VecInit(Seq.fill(p.sets)(0.U(p.wayBits.W)))) + val responses = Instantiate(new Queue(new CacheResponse(p), p.responseDepth, true)) + val pending = RegInit(false.B) + val pendingRead = Reg(Bool()) + val pendingWay = Reg(UInt(p.wayBits.W)) + val result = Reg(new CacheResponse(p)) + + val req = io.request.bits + val set = req.addr(p.offsetBits + p.setBits - 1, p.offsetBits) + val tag = req.addr(p.addressBits - 1, p.offsetBits + p.setBits) + val hits = VecInit((0 until p.ways).map(w => valid(w)(set) && tags(w)(set) === tag)).asUInt + val invalid = VecInit((0 until p.ways).map(w => !valid(w)(set) && req.eligible(w))).asUInt + + val order = VecInit((0 until p.ways).map { offset => + val way = if (p.ways == 1) 0.U else (nextWay(set) + offset.U)(p.wayBits - 1, 0) + req.eligible(way) + }) + + val replacement = + if (p.ways == 1) 0.U + else + (nextWay(set) + PriorityEncoder(order.asUInt))(p.wayBits - 1, 0) + + val lookupWay = Mux(hits.orR, PriorityEncoder(hits), Mux(invalid.orR, PriorityEncoder(invalid), replacement)) + val lookup = req.op === CacheOp.Lookup.U + val selected = Mux(lookup, lookupWay, req.way) + val available = !lookup || hits.orR || req.eligible.orR + val entryValid = available && valid(selected)(set) + val read = (lookup || req.op === CacheOp.Read.U) && entryValid + + io.request.ready := (responses.io.count +& pending.asUInt) < p.responseDepth.U || responses.io.deq.fire + responses.io.enq.valid := pending + responses.io.enq.bits := result + io.response <> responses.io.deq + pending := io.request.fire + assert(!pending || responses.io.enq.ready, "Cache response capacity reservation failed") + + val readData = Wire(Vec(p.ways, UInt(p.lineBits.W))) + for (way <- 0 until p.ways) { + val write = req.op === CacheOp.Write.U || req.op === CacheOp.Fill.U + val mask = Mux(req.op === CacheOp.Fill.U, Fill(p.lineBytes, 1.U(1.W)), req.mask) + readData(way) := data(way).readWrite( + set, + req.data.asTypeOf(Vec(p.lineBytes, UInt(8.W))), + mask.asBools, + io.request.fire && selected === way.U && (read || write), + write + ).asUInt + } + when(pendingRead) { + responses.io.enq.bits.data := readData(pendingWay) + } + + when(io.request.fire) { + assert(req.op <= CacheOp.Invalidate.U, "Unknown Cache operation") + assert(req.addr(p.offsetBits - 1, 0) === 0.U, "Cache operations require a line-aligned address") + assert(lookup || req.way < p.ways.U, "Cache way exceeds associativity") + assert(PopCount(hits) <= 1.U, "Duplicate Cache tag") + pendingRead := read + pendingWay := selected + result.id := req.id + result.hit := lookup && hits.orR + result.available := available + result.way := Mux(available, selected, 0.U) + result.entryValid := entryValid + result.addr := Mux(entryValid, Cat(tags(selected)(set), set, 0.U(p.offsetBits.W)), 0.U) + result.data := 0.U + result.metadata := Mux(entryValid, metadata(selected)(set), 0.U) + + when(req.op === CacheOp.Fill.U) { + assert(!hits.orR || hits(req.way), "Cache fill would create a duplicate tag") + valid(req.way)(set) := true.B + tags(req.way)(set) := tag + metadata(req.way)(set) := req.metadata + nextWay(set) := (if (p.ways == 1) 0.U else (req.way + 1.U)(p.wayBits - 1, 0)) + result.entryValid := true.B + result.addr := req.addr + result.metadata := req.metadata + } + when(req.op === CacheOp.Write.U) { + assert(entryValid && hits(req.way), "Cache write requires the matching resident line") + metadata(req.way)(set) := req.metadata + result.metadata := req.metadata + } + when(req.op === CacheOp.Invalidate.U) { + valid(req.way)(set) := false.B + result.entryValid := false.B + result.addr := 0.U + result.metadata := 0.U + } + } +} diff --git a/arch/src/main/scala/framework/ip/cache/src/main/scala/Emit.scala b/arch/src/main/scala/framework/ip/cache/src/main/scala/Emit.scala new file mode 100644 index 000000000..6892a1966 --- /dev/null +++ b/arch/src/main/scala/framework/ip/cache/src/main/scala/Emit.scala @@ -0,0 +1,43 @@ +package framework.ip.memory.cache + +import java.nio.file.{Files, Paths} +import framework.ip.memory.cache.configs.CacheParams + +object Emit extends App { + val Array("--config", config, "--output-dir", directory) = args + val values = ujson.read(Files.readString(Paths.get(config))) + val output = Paths.get(directory) + Files.createDirectories(output) + + val p = CacheParams( + values("addressBits").num.toInt, + values("lineBytes").num.toInt, + values("sets").num.toInt, + values("ways").num.toInt, + values("idBits").num.toInt, + values("metadataBits").num.toInt, + values("responseDepth").num.toInt + ) + + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Cache(p), + args = Array("--target-dir", output.resolve("Cache").toString, "--split-verilog") + ) + + val constants = Seq( + "ADDR_BITS" -> p.addressBits, + "LINE_BYTES" -> p.lineBytes, + "LINE_BITS" -> p.lineBits, + "SETS" -> p.sets, + "WAYS" -> p.ways, + "WAY_BITS" -> p.wayBits, + "ID_BITS" -> p.idBits, + "META_BITS" -> p.metadataBits, + "RESPONSE_DEPTH" -> p.responseDepth + ).map { case (name, value) => s"`define CACHE_$name $value" }.mkString("\n") + + Files.writeString( + output.resolve("cache_config.svh"), + s"`ifndef CACHE_CONFIG_SVH\n`define CACHE_CONFIG_SVH\n$constants\n`endif\n" + ) +} diff --git a/arch/src/main/scala/framework/ip/cache/src/main/scala/Interface.scala b/arch/src/main/scala/framework/ip/cache/src/main/scala/Interface.scala new file mode 100644 index 000000000..9fa53bc07 --- /dev/null +++ b/arch/src/main/scala/framework/ip/cache/src/main/scala/Interface.scala @@ -0,0 +1,42 @@ +package framework.ip.memory.cache + +import chisel3._ +import chisel3.util._ +import framework.ip.memory.cache.configs.CacheParams + +object CacheOp { + val Lookup = 0 + val Read = 1 + val Write = 2 + val Fill = 3 + val Invalidate = 4 +} + +class CacheRequest(p: CacheParams) extends Bundle { + val id = UInt(p.idBits.W) + val op = UInt(3.W) + // Read and Invalidate address a physical set/way; all operations are line aligned. + val addr = UInt(p.addressBits.W) + val way = UInt(p.wayBits.W) + val data = UInt(p.lineBits.W) + val mask = UInt(p.lineBytes.W) + val metadata = UInt(p.metadataBits.W) + // Eligibility constrains miss replacement candidates, not tag hits. + val eligible = UInt(p.ways.W) +} + +class CacheResponse(p: CacheParams) extends Bundle { + val id = UInt(p.idBits.W) + val hit = Bool() + val available = Bool() + val way = UInt(p.wayBits.W) + val entryValid = Bool() + val addr = UInt(p.addressBits.W) + val data = UInt(p.lineBits.W) + val metadata = UInt(p.metadataBits.W) +} + +class CacheIO(p: CacheParams) extends Bundle { + val request = Flipped(Decoupled(new CacheRequest(p))) + val response = Decoupled(new CacheResponse(p)) +} diff --git a/arch/src/main/scala/framework/ip/chi/build.sc b/arch/src/main/scala/framework/ip/chi/build.sc new file mode 100644 index 000000000..8847edb58 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/build.sc @@ -0,0 +1,37 @@ +import mill._ +import $file.^.^.^.^.^.^.settings + + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.ip.bus.chi.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Channel.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/CoherenceState.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/DirectedSnoop.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Fabric.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Flit.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/HomeMapping.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/RequestRetry.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/RequesterPort.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Rx.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Tx.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/hnf/ChiHome.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/rnf/AxiCache.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/rnf/BankedChiCache.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/rnf/CachePort.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/rnf/ChiCache.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/snf/MemoryNode.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/snf/SramEndpoint.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/ip/chi/configs/uvm.toml b/arch/src/main/scala/framework/ip/chi/configs/uvm.toml new file mode 100644 index 000000000..1ae6959d9 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/configs/uvm.toml @@ -0,0 +1,19 @@ +[parameters] +nodeIdBits = 7 +addressBits = 44 +dataBits = 256 +linkFlitBits = 32 +credits = 4 +nodeId = 1 +homeId = 64 +records = 4 +banks = 2 +cacheLines = 8 +homeCount = 1 +lineResult = false +probe = false +sramSlots = 8 +sramLines = 256 +sramRxDepth = 4 + +[defines] diff --git a/arch/src/main/scala/framework/mem-core/chi/src/csrc/Cargo.lock b/arch/src/main/scala/framework/ip/chi/src/csrc/Cargo.lock similarity index 100% rename from arch/src/main/scala/framework/mem-core/chi/src/csrc/Cargo.lock rename to arch/src/main/scala/framework/ip/chi/src/csrc/Cargo.lock diff --git a/arch/src/main/scala/framework/mem-core/chi/src/csrc/Cargo.toml b/arch/src/main/scala/framework/ip/chi/src/csrc/Cargo.toml similarity index 100% rename from arch/src/main/scala/framework/mem-core/chi/src/csrc/Cargo.toml rename to arch/src/main/scala/framework/ip/chi/src/csrc/Cargo.toml diff --git a/arch/src/main/scala/framework/mem-core/chi/src/csrc/README.md b/arch/src/main/scala/framework/ip/chi/src/csrc/README.md similarity index 100% rename from arch/src/main/scala/framework/mem-core/chi/src/csrc/README.md rename to arch/src/main/scala/framework/ip/chi/src/csrc/README.md diff --git a/arch/src/main/scala/framework/ip/chi/src/csrc/src/lib.rs b/arch/src/main/scala/framework/ip/chi/src/csrc/src/lib.rs new file mode 100644 index 000000000..d6e5c6d4b --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/csrc/src/lib.rs @@ -0,0 +1,4 @@ +#[path = "../../../../../../../../../../verify/uvm/src/protocol/chi/rnf/model/src/reference.rs"] +mod rnf; +mod memory; +pub use memory::*; diff --git a/arch/src/main/scala/framework/mem-core/chi/src/csrc/src/memory.rs b/arch/src/main/scala/framework/ip/chi/src/csrc/src/memory.rs similarity index 100% rename from arch/src/main/scala/framework/mem-core/chi/src/csrc/src/memory.rs rename to arch/src/main/scala/framework/ip/chi/src/csrc/src/memory.rs diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/flit_types.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/common/flit_types.svh similarity index 78% rename from arch/src/main/scala/framework/mem-core/chi/src/main/resources/flit_types.svh rename to arch/src/main/scala/framework/ip/chi/src/main/resources/common/flit_types.svh index cf3bb675a..3b3ea0c21 100644 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/flit_types.svh +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/common/flit_types.svh @@ -1,5 +1,5 @@ -`include "chi_profile.svh" -// Issue H, NodeID=7, address=44, data=256, all optional buses absent. +`include "profile.svh" +// Issue H base profile with optional buses absent. typedef struct packed { bit trace_tag; bit [1:0] tag_op; @@ -10,15 +10,15 @@ typedef struct packed { bit [1:0] order; bit allow_retry, likely_shared; bit [2:0] pas; - bit [43:0] addr; + bit [`CHI_ADDRESS_BITS-1:0] addr; bit [5:0] size; bit multi_req; bit [6:0] opcode; bit [11:0] return_txn; bit stash_valid; - bit [6:0] return_nid; + bit [`CHI_NODE_BITS-1:0] return_nid; bit [11:0] txn; - bit [6:0] src, tgt; + bit [`CHI_NODE_BITS-1:0] src, tgt; bit [3:0] qos; } req_t; typedef struct packed { @@ -31,7 +31,7 @@ typedef struct packed { bit [1:0] error; bit [4:0] opcode; bit [11:0] txn; - bit [6:0] src, tgt; + bit [`CHI_NODE_BITS-1:0] src, tgt; bit [3:0] qos; } rsp_t; typedef struct packed { @@ -52,8 +52,8 @@ typedef struct packed { bit [2:0] resp; bit [1:0] error; bit [3:0] opcode; - bit [6:0] home; + bit [`CHI_NODE_BITS-1:0] home; bit [11:0] txn; - bit [6:0] src, tgt; + bit [`CHI_NODE_BITS-1:0] src, tgt; bit [3:0] qos; } dat_t; diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/common/profile.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/common/profile.svh new file mode 100644 index 000000000..7e723f930 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/common/profile.svh @@ -0,0 +1,10 @@ +`ifndef CHI_PROFILE_SVH +`define CHI_PROFILE_SVH +`include "parameters.svh" +`define CHI_BE_BITS (`CHI_DATA_BITS / 8) +`define CHI_TAG_BITS (`CHI_DATA_BITS / 32) +`define CHI_TAG_UPDATE_BITS (`CHI_DATA_BITS / 128) +`define CHI_DAT_BITS (91 + `CHI_DATA_BITS + `CHI_BE_BITS + `CHI_TAG_BITS + `CHI_TAG_UPDATE_BITS) +`define CHI_BEATS (512 / `CHI_DATA_BITS) +`define CHI_DATA_ID_STEP (`CHI_DATA_BITS / 128) +`endif diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/agent/agent.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/agent/agent.svh new file mode 100644 index 000000000..d59bb88c0 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/agent/agent.svh @@ -0,0 +1,28 @@ +class agent extends uvm_agent; + `uvm_component_utils(agent) + monitor mon; + endpoint_driver endpoint; + uvm_sequencer #(packet_item) seqr[3]; + source_driver #(0) source0; + source_driver #(1) source1; + source_driver #(2) source2; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + mon = monitor::type_id::create("monitor", this); + endpoint = endpoint_driver::type_id::create("endpoint", this); + seqr[0] = new("seqr0", this); + source0 = source_driver#(0)::type_id::create("source0", this); + seqr[1] = new("seqr1", this); + source1 = source_driver#(1)::type_id::create("source1", this); + seqr[2] = new("seqr2", this); + source2 = source_driver#(2)::type_id::create("source2", this); + endfunction + function void connect_phase(uvm_phase phase); + source0.seq_item_port.connect(seqr[0].seq_item_export); + source1.seq_item_port.connect(seqr[1].seq_item_export); + source2.seq_item_port.connect(seqr[2].seq_item_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/agent/driver.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/agent/driver.svh new file mode 100644 index 000000000..d8b9a4483 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/agent/driver.svh @@ -0,0 +1,44 @@ +class source_driver #( + int PORT = 0 +) extends uvm_driver #(packet_item); + `uvm_component_param_utils(source_driver#(PORT)) + virtual chi_retry_if vif; + protocol_model state; + packet_item pending; + bit driven = 0; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual chi_retry_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "CHI source interface is required") + endfunction + task run_phase(uvm_phase phase); + forever begin + @(negedge vif.clock); + if (pending == null) seq_item_port.try_next_item(pending); + driven = 0; + if (PORT == 0) begin + vif.req_in = pending != null ? pending.payload : state.idle.req_in; + vif.req_in_valid = pending != null; + driven = pending != null; + end else if (PORT == 1) begin + vif.rsp_in = pending != null ? pending.payload : state.idle.rsp_in; + vif.rsp_in_valid = pending != null; + driven = pending != null; + end else begin + vif.accepted_src = pending != null ? pending.payload[18:12] : 0; + vif.accepted_txn = pending != null ? pending.payload[11:0] : 0; + vif.accepted_valid = pending != null; + driven = pending != null; + end + @(posedge vif.clock); + if (!vif.reset && driven && (PORT == 2 || (PORT == 0 ? vif.req_in_ready : vif.rsp_in_ready))) begin + #1; + seq_item_port.item_done(); + pending = null; + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/agent/endpoint.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/agent/endpoint.svh new file mode 100644 index 000000000..24019f9ff --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/agent/endpoint.svh @@ -0,0 +1,20 @@ +class endpoint_driver extends uvm_component; + `uvm_component_utils(endpoint_driver) + virtual chi_retry_if vif; + protocol_model state; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual chi_retry_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "CHI endpoint interface is required") + endfunction + task run_phase(uvm_phase phase); + forever begin + @(negedge vif.clock); + vif.req_out_ready = !vif.reset && state.idle.req_out_ready; + vif.rsp_out_ready = !vif.reset && state.idle.rsp_out_ready; + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/agent/item.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/agent/item.svh new file mode 100644 index 000000000..a1c674ee2 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/agent/item.svh @@ -0,0 +1,37 @@ +class packet_item extends uvm_sequence_item; + `uvm_object_utils(packet_item) + logic [`CHI_DAT_BITS-1:0] payload; + bit sc_success, failed; + function new(string name = "packet_item"); + super.new(name); + endfunction +endclass +class sample_item extends uvm_sequence_item; + `uvm_object_utils(sample_item) + pins_t sampled; + function new(string name = "sample_item"); + super.new(name); + endfunction +endclass +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(data, UVM_ALL_ON) + `uvm_field_int(error, UVM_ALL_ON) + `uvm_object_utils_end + logic [63:0] data; + logic error; + function new(string name = "result_item"); + super.new(name); + endfunction +endclass +class packet_sequence extends uvm_sequence #(packet_item); + `uvm_object_utils(packet_sequence) + packet_item item; + function new(string name = "packet_sequence"); + super.new(name); + endfunction + task body(); + start_item(item); + finish_item(item); + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/agent/monitor.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/agent/monitor.svh new file mode 100644 index 000000000..9723936b5 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/agent/monitor.svh @@ -0,0 +1,39 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual chi_retry_if vif; + uvm_analysis_port #(sample_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual chi_retry_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "CHI monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + sample_item t; + forever begin + @(posedge vif.clock); + t = sample_item::type_id::create("observed"); + t.sampled.reset = vif.reset; + t.sampled.req_in_valid = vif.req_in_valid; + t.sampled.req_in_ready = vif.req_in_ready; + t.sampled.req_out_valid = vif.req_out_valid; + t.sampled.req_out_ready = vif.req_out_ready; + t.sampled.req_in = vif.req_in; + t.sampled.req_out = vif.req_out; + t.sampled.rsp_in_valid = vif.rsp_in_valid; + t.sampled.rsp_in_ready = vif.rsp_in_ready; + t.sampled.rsp_out_valid = vif.rsp_out_valid; + t.sampled.rsp_out_ready = vif.rsp_out_ready; + t.sampled.rsp_in = vif.rsp_in; + t.sampled.rsp_out = vif.rsp_out; + t.sampled.accepted_valid = vif.accepted_valid; + t.sampled.accepted_src = vif.accepted_src; + t.sampled.accepted_txn = vif.accepted_txn; + t.sampled.pending = vif.pending; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/env/env.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/env/env.svh new file mode 100644 index 000000000..6c6bc7b0b --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/env/env.svh @@ -0,0 +1,28 @@ +class env extends uvm_env; + `uvm_component_utils(env) + agent bus; + protocol_model state; + reset_controller reset; + result_board #(result_item, 0) scoreboard; + uvm_analysis_imp #(bit, env) reset_export; + function new(string name, uvm_component parent); + super.new(name, parent); + reset_export = new("reset_export", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + state = protocol_model::type_id::create("model", this); + bus = agent::type_id::create("bus", this); + reset = reset_controller::type_id::create("reset", this); + endfunction + function void connect_phase(uvm_phase phase); + bus.mon.ap.connect(state.analysis_export); + bus.endpoint.state = state; + bus.source0.state = state; + bus.source1.state = state; + bus.source2.state = state; + endfunction + function void write(bit value); + scoreboard.cancel_pending(); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/env/model.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/env/model.svh new file mode 100644 index 000000000..d63819b33 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/env/model.svh @@ -0,0 +1,66 @@ +class protocol_model extends uvm_subscriber #(sample_item); + `uvm_component_utils(protocol_model) + req_t requests[$]; + rsp_t responses[$], original_responses[$]; + int checks = 0; + req_t originals[int]; + int retry_types[int]; + string scenario; + pins_t sampled; + pins_t idle; + uvm_analysis_port #(result_item) expected_ap, actual_ap; + uvm_analysis_port #(bit) reset_ap; + covergroup behavior with function sample (int count, bit stalled, bit reissued); + option.per_instance = 1; + live: coverpoint count { + bins empty = {0}; bins partial = {[1 : `CHI_RECORDS - 1]}; bins full = {`CHI_RECORDS}; + } + backpressure: coverpoint stalled; + retry_request: coverpoint reissued; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + behavior = new(); + expected_ap = new("expected_ap", this); + actual_ap = new("actual_ap", this); + reset_ap = new("reset_ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!$value$plusargs("CASE=%s", scenario)) `uvm_fatal("CASE", "CHI scenario is required") + idle = '0; + + endfunction + function void write(sample_item t); + pins_t s = t.sampled; + sampled = s; + begin + req_t out_req = req_t'(s.req_out); + behavior.sample(s.pending, s.req_out_valid && !s.req_out_ready, + s.req_out_valid && !out_req.allow_retry); + end + + if (s.reset) begin + requests.delete(); + responses.delete(); + original_responses.delete(); + end else begin + if (s.req_in_valid && s.req_in_ready) begin + req_t r = req_t'(s.req_in); + originals[r.txn] = r; + end + if (s.rsp_in_valid && s.rsp_in_ready) begin + rsp_t r = rsp_t'(s.rsp_in); + if (r.opcode == 3) retry_types[r.txn] = r.pcrd_type; + if (r.opcode != 3 && r.opcode != 7) original_responses.push_back(r); + end + if (s.req_out_valid && s.req_out_ready) requests.push_back(req_t'(s.req_out)); + if (s.rsp_out_valid && s.rsp_out_ready) responses.push_back(rsp_t'(s.rsp_out)); + end + + endfunction + function void check_phase(uvm_phase phase); + evidence::check(requests.size() == 0 && responses.size() == 0 && original_responses.size() == 0, + "CHI retry queues did not drain"); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/filelist.f b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/filelist.f new file mode 100644 index 000000000..60c7127cf --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/filelist.f @@ -0,0 +1,10 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@RESOURCES@/../common ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/types.svh +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/RequestRetry/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/interface.sv b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/interface.sv new file mode 100644 index 000000000..0bfe6b217 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/interface.sv @@ -0,0 +1,24 @@ +`include "parameters.svh" +interface chi_retry_if ( + input logic clock +); + logic reset; + logic req_in_valid, req_in_ready, req_out_valid, req_out_ready; + logic [`CHI_REQ_BITS-1:0] req_in, req_out; + logic rsp_in_valid, rsp_in_ready, rsp_out_valid, rsp_out_ready; + logic [`CHI_RSP_BITS-1:0] rsp_in, rsp_out; + logic accepted_valid; + logic [`CHI_NODE_BITS-1:0] accepted_src; + logic [11:0] accepted_txn; + logic [`CHI_PENDING_BITS-1:0] pending; + assert property (@(posedge clock) disable iff (reset) + req_out_valid && !req_out_ready |=> req_out_valid && $stable( + req_out + )) + else $fatal(1, "CHI channel changed under backpressure"); + assert property (@(posedge clock) disable iff (reset) + rsp_out_valid && !rsp_out_ready |=> rsp_out_valid && $stable( + rsp_out + )) + else $fatal(1, "CHI channel changed under backpressure"); +endinterface diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/package.sv b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/package.sv new file mode 100644 index 000000000..8a95541c1 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/package.sv @@ -0,0 +1,16 @@ +package chi_retry_pkg; + import uvm_pkg::*; + import verification_pkg::*; + import observed_types::*; + `include "uvm_macros.svh" + `include "flit_types.svh" + `include "agent/item.svh" + `include "env/model.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/endpoint.svh" + `include "agent/agent.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/seq/sequence.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/seq/sequence.svh new file mode 100644 index 000000000..afb95a3dc --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/seq/sequence.svh @@ -0,0 +1,430 @@ +class scenario_sequence extends uvm_sequence; + `uvm_object_utils(scenario_sequence) + env test_env; + protocol_model state; + virtual chi_retry_if vif; + string scenario; + function new(string name = "scenario_sequence"); + super.new(name); + endfunction + task packet(int port, logic [`CHI_DAT_BITS-1:0] payload, bit sc_success = 0, bit failed = 0); + packet_sequence seq = packet_sequence::type_id::create("packet"); + seq.item = packet_item::type_id::create("item"); + seq.item.payload = payload; + seq.item.sc_success = sc_success; + seq.item.failed = failed; + `uvm_info("SEND", $sformatf("port=%0d payload=%h", port, payload), UVM_HIGH) + seq.start(test_env.bus.seqr[port]); + `uvm_info("ACCEPTED", $sformatf("port=%0d", port), UVM_HIGH) + endtask + task drive_request(req_t r); + packet(0, r); + endtask + + function bit coherent_read(int opcode); + return opcode == 1 || opcode == 'h26 || opcode == 7; + endfunction + function bit coherent_request(int opcode); + return coherent_read(opcode) || opcode == 'hd || opcode == 'h1b || opcode == 8 || opcode == 9 || + opcode == 10; + endfunction + task request(int txn, int opcode = 'hd); + req_t r = '0; + r.src = 1; + r.tgt = 64; + r.txn = txn; + r.opcode = opcode; + r.size = 6; + r.addr = 64 * txn; + r.allow_retry = 1; + if (coherent_request(opcode)) begin + r.mem_attr = 12; + r.snp_attr = 1; + end else begin + r.mem_attr = 4; + r.return_nid = 1; + r.return_txn = txn; + end + r.exp_comp_ack = coherent_read(opcode); + drive_request(r); + endtask + task response(int opcode, int txn, int src = 65, int credit_type = 3, bit contiguous = 0); + rsp_t r = '0; + r.opcode = opcode; + r.txn = txn; + r.src = src; + r.tgt = 1; + r.pcrd_type = (opcode == 3 || opcode == 7) ? credit_type : 0; + r.qos = (txn ^ src) % 16; + r.trace_tag = txn % 2; + if (opcode == 6 || opcode == 5) r.dbid = (txn ^ src ^ 'h555) & 'hfff; + if (opcode == 4 || opcode == 6 || opcode == 5) r.busy = (txn + src) % 8; + if (opcode == 4 && txn % 2) r.error = 3; // Dataless completion may report NDERR. + packet(1, r); + endtask + task expect_request(int txn, bit retry); + req_t r, golden; + `uvm_info("EXPECT_REQ", $sformatf("txn=%0d retry=%0d", txn, retry), UVM_HIGH) + while (!state.requests.size()) @(negedge vif.clock); + r = state.requests.pop_front(); + if (!state.originals.exists(txn)) `uvm_fatal("MODEL", "Missing observed original request") + golden = state.originals[txn]; + if (retry) begin + golden.allow_retry = 0; + golden.pcrd_type = state.retry_types[txn]; + end + if (r !== golden) + `uvm_fatal("RETRY", $sformatf( + "Complete request changed txn=%0h got=%h expected=%h", txn, r, golden)) + state.checks++; + evidence::checks++; + endtask + task receive_response(output rsp_t r); + rsp_t golden; + while (!state.responses.size()) @(negedge vif.clock); + if (!state.original_responses.size()) `uvm_fatal("MODEL", "Missing response/original") + r = state.responses.pop_front(); + golden = state.original_responses.pop_front(); + if (r !== golden) `uvm_fatal("RESPONSE", "Complete response metadata changed") + endtask + task accept_data(int txn, int source); + packet(2, {7'(source), 12'(txn)}); + endtask + + task body(); + rsp_t completed; + state.idle.req_out_ready = 1; + state.idle.rsp_out_ready = 1; + test_env.reset.pulse(4); + if (scenario == "duplicate_txnid") begin + request(12'hf01); + expect_request(12'hf01, 0); + request(12'hf01); + `uvm_fatal("NEGATIVE", "Duplicate live TxnID did not terminate RTL") + end else if (scenario == "bad_opcode") begin + request(0, 'h11); // Separate response/data acceptance is outside this profile. + `uvm_fatal("NEGATIVE", "Unsupported retry opcode did not terminate RTL") + end else begin + // A normal completion passes through and releases the request record. + request(0); + expect_request(0, 0); + state.idle.rsp_out_ready = 0; + fork + response(4, 0); + begin + repeat (12) @(negedge vif.clock); + state.idle.rsp_out_ready = 1; + end + join + receive_response(completed); + if (completed.opcode != 4) `uvm_fatal("PASS", "Completion was swallowed") + for (int i = 0; i < 4; i++) begin + request(i); + expect_request(i, 0); + end + for (int i = 0; i < 4; i++) response(3, i); + repeat (10) @(negedge vif.clock); + if (state.requests.size() || state.responses.size() || state.original_responses.size() || state.sampled.pending != 4) + `uvm_fatal("WAIT", "Retry issued before P-Credit or internal response leaked") + // Home remapped from TgtID 64 to RetryAck SrcID 65; match that credit source. + response(7, 0, 66, 3); + response(7, 0, 65, 2); + repeat (10) @(negedge vif.clock); + if (state.requests.size()) `uvm_fatal("MATCH", "Mismatched protocol credit consumed") + for (int i = 0; i < 4; i++) begin + response(7, 0); + expect_request(i, 1); + response(4, i); + receive_response(completed); + if (completed.txn != i) `uvm_fatal("COMP", "Bad completion"); + end + // An output stall must preserve the queued request and its record. + state.idle.req_out_ready = 0; + request(0); + repeat (12) @(negedge vif.clock); + if (!state.sampled.req_out_valid || state.requests.size()) + `uvm_fatal("STALL", "Queued request lost under stall") + state.idle.req_out_ready = 1; + expect_request(0, 0); + response(4, 0); + receive_response(completed); + if (completed.txn != 0) `uvm_fatal("COMP", "Bad stalled completion"); + // Grant can precede RetryAck and must be retained for its matching retry. + request(0, 4); + expect_request(0, 0); + response(7, 0); + response(3, 0); + expect_request(0, 1); + accept_data(0, 9); + repeat (4) @(negedge vif.clock); + if (state.sampled.pending != 0 || state.requests.size() || state.responses.size() || state.original_responses.size()) + `uvm_fatal("DRAIN", "Retry records did not drain") + // Reset cancels retry-wait records and retained protocol credits. + request(0); + expect_request(0, 0); + response(3, 0); + @(negedge vif.clock); + test_env.reset.pulse(4); + request(0); + expect_request(0, 0); + response(4, 0); + receive_response(completed); + if (completed.txn != 0) `uvm_fatal("RESET", "Completion after reset corrupted"); + repeat (4) @(negedge vif.clock); + if (state.sampled.pending != 0 || state.requests.size() || state.responses.size() || state.original_responses.size()) + `uvm_fatal("DRAIN", "Retry reset did not drain") + // Full protocol IDs are independent of the four local acceptance slots. + // Exercise both polarities of every aligned address, TxnID and routing-ID bit. + for (int phase_index = 0; phase_index < 3; phase_index++) begin + int polarity = phase_index % 2; + for (int address_bit = 6; address_bit < 44; address_bit++) begin + int ids[4], homes[4], types[4], opcodes[4]; + bit [3:0] seen_reissues = '0; + int supported_ops[11] = '{1, 'h26, 7, 'hd, 'h1b, 8, 9, 10, 4, 'h1d, 'h1c}; + for (int slot = 0; slot < 4; slot++) begin + req_t r = '0; + int node_bits = (1 << ((address_bit + slot) % 7)) | 2; + ids[slot] = (1 << ((address_bit + slot) % 12)) ^ (polarity ? 'hfff : 0); + homes[slot] = (address_bit % 3 == 0 ? 65 : node_bits ^ (polarity ? 'h7f : 0)); + types[slot] = (address_bit % 3 == 1 ? address_bit % 16 : (address_bit + slot) % 16); + r.src = 1; + r.tgt = homes[slot]; + r.txn = ids[slot]; + opcodes[slot] = supported_ops[(address_bit+slot+polarity)%11]; + r.opcode = opcodes[slot]; + r.size = 6; + r.lpid = (address_bit + slot) % 32; + r.likely_shared = (address_bit + slot) % 2; + if (coherent_request(opcodes[slot])) begin + r.mem_attr = 12; + r.snp_attr = 1; + end else begin + case ((address_bit + slot) % 3) + 0: r.mem_attr = 0; + 1: r.mem_attr = 4; + 2: r.mem_attr = 12; + endcase + r.mem_attr |= (address_bit + slot + polarity) % 2; // EWA is legal for Normal memory. + r.return_nid = homes[slot]; + r.return_txn = ids[slot] ^ 'hfff; + end + r.exp_comp_ack = coherent_read(opcodes[slot]); + r.addr = ((64'h1 << address_bit) ^ (polarity ? 64'hfffffffffc0 : 0)); + r.qos = (address_bit + 3 * slot) % 16; + r.trace_tag = polarity; + r.allow_retry = 1; + drive_request(r); + expect_request(ids[slot], 0); + end + if (state.sampled.pending != 4) + `uvm_fatal("SLOTS", "Arbitrary IDs did not fill four acceptance slots") + state.idle.rsp_out_ready = 0; // Internal retry traffic must bypass endpoint backpressure. + for (int position = 0; position < 4; position++) begin + int slot = (position + address_bit / 4) % 4; + response(7, 0, homes[slot], types[slot]); + end + if (address_bit % 5 == 0) state.idle.req_out_ready = 0; + for (int position = 0; position < 4; position++) begin + int slot = (address_bit + (address_bit % 2 ? 3 - position : position)) % 4; + response(3, ids[slot], homes[slot], types[slot], position != 0); + end + repeat (3) @(negedge vif.clock); + state.idle.req_out_ready = 1; + state.idle.rsp_out_ready = 1; + for (int position = 0; position < 4; position++) begin + int slot = -1; + while (!state.requests.size()) @(negedge vif.clock); + for (int candidate = 0; candidate < 4; candidate++) + if (state.requests[0].txn == ids[candidate]) slot = candidate; + if (slot < 0 || seen_reissues[slot]) + `uvm_fatal("RETRY_SET", "Unexpected or duplicate reissued transaction") + seen_reissues[slot] = 1; + expect_request(ids[slot], 1); + if (coherent_read(opcodes[slot]) || opcodes[slot] == 4) begin + accept_data(ids[slot], homes[slot]); + end else begin + int acceptance=(opcodes[slot]=='h1b || opcodes[slot]=='h1d || opcodes[slot]=='h1c) ? 6 : 4; + if (acceptance == 6 && (address_bit + slot) % 2) acceptance = 5; + response(acceptance, ids[slot], homes[slot], 0); + receive_response(completed); + if (completed.txn != ids[slot] || completed.opcode != acceptance) + `uvm_fatal("PASS", "Acceptance response corrupted") + end + end + repeat (3) @(negedge vif.clock); + if (state.sampled.pending != 0 || state.requests.size() || state.responses.size() || state.original_responses.size()) + `uvm_fatal("DRAIN", "Full-width request batch did not drain") + end + end + // Fill the two-entry output queue and hold a third request under backpressure. + state.idle.req_out_ready = 0; + fork + begin + for (int slot = 0; slot < 4; slot++) request(slot); + end + begin + repeat (20) @(negedge vif.clock); + state.idle.req_out_ready = 1; + end + join + for (int slot = 0; slot < 4; slot++) expect_request(slot, 0); + for (int slot = 0; slot < 4; slot++) begin + response(4, slot); + receive_response(completed); + end + // Initial allocation overlaps independent RetryAck and completion handshakes. + for (int context_kind = 0; context_kind < 2; context_kind++) begin + for (int slot = 0; slot < 4; slot++) begin + int other = (slot + 1) % 4; + for (int i = 0; i < 4; i++) begin + request(100 + i); + expect_request(100 + i, 0); + end + response(4, 100 + slot); + receive_response(completed); + fork + request(200 + slot); + response(context_kind ? 4 : 3, 100 + other); + join + expect_request(200 + slot, 0); + if (!context_kind) begin + response(7, 0); + expect_request(100 + other, 1); + end else begin + receive_response(completed); + end + for (int i = 0; i < 4; i++) begin + if (context_kind && i == other) continue; + response(4, i == slot ? 200 + slot : 100 + i); + receive_response(completed); + end + end + end + // A new P-Credit may arrive in the cycle a different credit is consumed. + for (int slot = 0; slot < 4; slot++) begin + int next_slot = (slot + 1) % 4; + for (int i = 0; i < 4; i++) begin + request(300 + i); + expect_request(300 + i, 0); + end + response(7, 0); + response(3, 300 + slot); + response(7, 0, 65, 3, 1); + expect_request(300 + slot, 1); + response(3, 300 + next_slot); + expect_request(300 + next_slot, 1); + for (int i = 0; i < 4; i++) begin + response(4, 300 + i); + receive_response(completed); + end + end + // Later split completion/data acceptance may follow record release. + accept_data(3, 65); + repeat (3) @(negedge vif.clock); + if (state.sampled.pending || state.requests.size() || state.responses.size() || state.original_responses.size()) + `uvm_fatal("DRAIN", "Saturated output queue did not drain") + // Acceptance tracking can retire before later data beats or split completion. + request(500, 4); + expect_request(500, 0); + accept_data(500, 9); + request(501, 'h1d); + expect_request(501, 0); + response(6, 501); + receive_response(completed); + for (int i = 0; i < 4; i++) begin + request(600 + i); + expect_request(600 + i, 0); + end + for (int i = 0; i < 4; i++) response(3, 600 + i); + for (int i = 0; i < 4; i++) begin + rsp_t idle = '0; + idle.opcode = 4; + idle.tgt = 1; + idle.txn = 600 + i; + @(negedge vif.clock); + state.idle.rsp_in = idle; + state.idle.accepted_txn = 600 + i; + @(negedge vif.clock); + if (state.sampled.pending != 4 || state.requests.size() || state.responses.size()) + `uvm_fatal("INVALID_CHANNEL", "Inactive matching payload changed a retry-wait record") + end + response(4, 501); + receive_response(completed); + accept_data(500, 9); + if (state.sampled.pending != 4) + `uvm_fatal("LATE_ACCEPT", "Late acceptance retired an unrelated retry") + for (int i = 0; i < 4; i++) begin + response(7, 0); + expect_request(600 + i, 1); + response(4, 600 + i); + receive_response(completed); + end + // VALID held through coordinated reset is accepted once after release. + begin + req_t held = '0; + held.src = 1; + held.tgt = 64; + held.txn = 'ha55; + held.opcode = 'hd; + held.size = 6; + held.addr = 64; + held.allow_retry = 1; + held.mem_attr = 12; + held.snp_attr = 1; + @(negedge vif.clock); + fork + test_env.reset.pulse(4); + begin + wait (vif.reset); + drive_request(held); + end + join + expect_request(held.txn, 0); + response(4, held.txn); + receive_response(completed); + repeat (3) @(negedge vif.clock); + if (state.sampled.pending || state.requests.size() || state.responses.size() || state.original_responses.size()) + `uvm_fatal("RESET", "Held VALID did not complete exactly once") + end + // Invalid channels may carry arbitrary payload; no transaction may be consumed. + begin + req_t idle_req = '0; + rsp_t idle_rsp = '0; + idle_req.src = 1; + idle_req.opcode = 'hd; + idle_req.size = 6; + idle_req.mem_attr = 12; + idle_req.snp_attr = 1; + idle_req.allow_retry = 1; + idle_rsp.tgt = 1; + idle_rsp.opcode = 4; + for (int b = 0; b < $bits(req_t); b++) begin + @(negedge vif.clock); + state.idle.req_in = idle_req ^ (137'b1 << b); + @(negedge vif.clock); + state.idle.req_in = idle_req; + if (state.sampled.pending || state.sampled.req_out_valid || state.sampled.rsp_out_valid) + `uvm_fatal("INVALID_CHANNEL", "Invalid REQ payload changed live transactions") + end + @(negedge vif.clock); + idle_req.snp_attr = 0; + idle_req.mem_attr = 2; + state.idle.req_in = idle_req; + @(negedge vif.clock); + if (state.sampled.pending || state.sampled.req_out_valid) + `uvm_fatal("INVALID_CHANNEL", "Inactive Device payload was consumed") + for (int b = 0; b < $bits(rsp_t); b++) begin + @(negedge vif.clock); + state.idle.rsp_in = idle_rsp ^ (71'b1 << b); + @(negedge vif.clock); + state.idle.rsp_in = idle_rsp; + if (state.sampled.pending || state.sampled.req_out_valid || state.sampled.rsp_out_valid) + `uvm_fatal("INVALID_CHANNEL", "Invalid RSP payload changed live transactions") + end + end + repeat (2) @(negedge vif.clock); + `uvm_info("CHECKS", $sformatf("CHI retry: %0d request/reissue fields checked", state.checks), + UVM_LOW) + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/tb.sv b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/tb.sv new file mode 100644 index 000000000..150c47833 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/tb.sv @@ -0,0 +1,104 @@ +module retry_tb; + import uvm_pkg::*; + import chi_retry_pkg::*; + logic clock = 0; + clock_if timing (clock); + always #5 clock = ~clock; + chi_retry_if source_if (clock); + RequestRetry dut ( + .clock(clock), + .reset(source_if.reset), + .io_reqIn_valid(source_if.req_in_valid), + .io_reqIn_ready(source_if.req_in_ready), + .io_reqIn_bits_qos(source_if.req_in[3:0]), + .io_reqIn_bits_tgtId(source_if.req_in[10:4]), + .io_reqIn_bits_srcId(source_if.req_in[17:11]), + .io_reqIn_bits_txnId(source_if.req_in[29:18]), + .io_reqIn_bits_returnNid(source_if.req_in[36:30]), + .io_reqIn_bits_stashNidValidEndian(source_if.req_in[37:37]), + .io_reqIn_bits_returnTxnId(source_if.req_in[49:38]), + .io_reqIn_bits_opcode(source_if.req_in[56:50]), + .io_reqIn_bits_multiReq(source_if.req_in[57:57]), + .io_reqIn_bits_size(source_if.req_in[63:58]), + .io_reqIn_bits_addr(source_if.req_in[107:64]), + .io_reqIn_bits_pas(source_if.req_in[110:108]), + .io_reqIn_bits_likelyShared(source_if.req_in[111:111]), + .io_reqIn_bits_allowRetry(source_if.req_in[112:112]), + .io_reqIn_bits_order(source_if.req_in[114:113]), + .io_reqIn_bits_pCrdType(source_if.req_in[118:115]), + .io_reqIn_bits_memAttr(source_if.req_in[122:119]), + .io_reqIn_bits_snpAttr(source_if.req_in[123:123]), + .io_reqIn_bits_lpid(source_if.req_in[131:124]), + .io_reqIn_bits_exclSnoopMe(source_if.req_in[132:132]), + .io_reqIn_bits_expCompAck(source_if.req_in[133:133]), + .io_reqIn_bits_tagOp(source_if.req_in[135:134]), + .io_reqIn_bits_traceTag(source_if.req_in[136:136]), + .io_reqOut_valid(source_if.req_out_valid), + .io_reqOut_ready(source_if.req_out_ready), + .io_reqOut_bits_qos(source_if.req_out[3:0]), + .io_reqOut_bits_tgtId(source_if.req_out[10:4]), + .io_reqOut_bits_srcId(source_if.req_out[17:11]), + .io_reqOut_bits_txnId(source_if.req_out[29:18]), + .io_reqOut_bits_returnNid(source_if.req_out[36:30]), + .io_reqOut_bits_stashNidValidEndian(source_if.req_out[37:37]), + .io_reqOut_bits_returnTxnId(source_if.req_out[49:38]), + .io_reqOut_bits_opcode(source_if.req_out[56:50]), + .io_reqOut_bits_multiReq(source_if.req_out[57:57]), + .io_reqOut_bits_size(source_if.req_out[63:58]), + .io_reqOut_bits_addr(source_if.req_out[107:64]), + .io_reqOut_bits_pas(source_if.req_out[110:108]), + .io_reqOut_bits_likelyShared(source_if.req_out[111:111]), + .io_reqOut_bits_allowRetry(source_if.req_out[112:112]), + .io_reqOut_bits_order(source_if.req_out[114:113]), + .io_reqOut_bits_pCrdType(source_if.req_out[118:115]), + .io_reqOut_bits_memAttr(source_if.req_out[122:119]), + .io_reqOut_bits_snpAttr(source_if.req_out[123:123]), + .io_reqOut_bits_lpid(source_if.req_out[131:124]), + .io_reqOut_bits_exclSnoopMe(source_if.req_out[132:132]), + .io_reqOut_bits_expCompAck(source_if.req_out[133:133]), + .io_reqOut_bits_tagOp(source_if.req_out[135:134]), + .io_reqOut_bits_traceTag(source_if.req_out[136:136]), + .io_rspIn_valid(source_if.rsp_in_valid), + .io_rspIn_ready(source_if.rsp_in_ready), + .io_rspIn_bits_qos(source_if.rsp_in[3:0]), + .io_rspIn_bits_tgtId(source_if.rsp_in[10:4]), + .io_rspIn_bits_srcId(source_if.rsp_in[17:11]), + .io_rspIn_bits_txnId(source_if.rsp_in[29:18]), + .io_rspIn_bits_opcode(source_if.rsp_in[34:30]), + .io_rspIn_bits_respErr(source_if.rsp_in[36:35]), + .io_rspIn_bits_resp(source_if.rsp_in[39:37]), + .io_rspIn_bits_fwdState(source_if.rsp_in[42:40]), + .io_rspIn_bits_cBusy(source_if.rsp_in[45:43]), + .io_rspIn_bits_dbid(source_if.rsp_in[57:46]), + .io_rspIn_bits_pCrdType(source_if.rsp_in[61:58]), + .io_rspIn_bits_tagOp(source_if.rsp_in[63:62]), + .io_rspIn_bits_traceTag(source_if.rsp_in[64:64]), + .io_rspIn_bits_cacheLineId(source_if.rsp_in[70:65]), + .io_rspOut_valid(source_if.rsp_out_valid), + .io_rspOut_ready(source_if.rsp_out_ready), + .io_rspOut_bits_qos(source_if.rsp_out[3:0]), + .io_rspOut_bits_tgtId(source_if.rsp_out[10:4]), + .io_rspOut_bits_srcId(source_if.rsp_out[17:11]), + .io_rspOut_bits_txnId(source_if.rsp_out[29:18]), + .io_rspOut_bits_opcode(source_if.rsp_out[34:30]), + .io_rspOut_bits_respErr(source_if.rsp_out[36:35]), + .io_rspOut_bits_resp(source_if.rsp_out[39:37]), + .io_rspOut_bits_fwdState(source_if.rsp_out[42:40]), + .io_rspOut_bits_cBusy(source_if.rsp_out[45:43]), + .io_rspOut_bits_dbid(source_if.rsp_out[57:46]), + .io_rspOut_bits_pCrdType(source_if.rsp_out[61:58]), + .io_rspOut_bits_tagOp(source_if.rsp_out[63:62]), + .io_rspOut_bits_traceTag(source_if.rsp_out[64:64]), + .io_rspOut_bits_cacheLineId(source_if.rsp_out[70:65]), + .io_acceptedData_valid(source_if.accepted_valid), + .io_acceptedData_bits_srcId(source_if.accepted_src), + .io_acceptedData_bits_txnId(source_if.accepted_txn), + .io_pending(source_if.pending) + ); + assign source_if.reset = timing.reset; + initial begin + uvm_config_db#(virtual chi_retry_if)::set(null, "uvm_test_top*", "vif", source_if); + uvm_config_db#(virtual clock_if)::set(null, "uvm_test_top.env.reset", "timing", timing); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/tests/test.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/tests/test.svh new file mode 100644 index 000000000..45c2ab054 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/tests/test.svh @@ -0,0 +1,26 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + virtual chi_retry_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + if (`CHI_NODE_BITS != 7 || `CHI_ADDRESS_BITS != 44 || `CHI_NODE_ID != 1) + `uvm_fatal("PROFILE", + "CHI packet scenarios require 7-bit nodes, 44-bit addresses and requester 1") + if (`CHI_RECORDS != 4) `uvm_fatal("PROFILE", "Retry saturation scenarios require four records") + super.build_phase(phase); + test_env = env::type_id::create("env", this); + if (!uvm_config_db#(virtual chi_retry_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "CHI timing interface is required") + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.test_env = test_env; + seq.state = test_env.state; + seq.vif = vif; + seq.scenario = test_env.state.scenario; + seq.start(null); + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/types.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/types.svh new file mode 100644 index 000000000..0156fee1f --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/types.svh @@ -0,0 +1,14 @@ +`include "profile.svh" +package observed_types; + typedef struct packed { + logic reset; + logic req_in_valid, req_in_ready, req_out_valid, req_out_ready; + logic [`CHI_REQ_BITS-1:0] req_in, req_out; + logic rsp_in_valid, rsp_in_ready, rsp_out_valid, rsp_out_ready; + logic [`CHI_RSP_BITS-1:0] rsp_in, rsp_out; + logic accepted_valid; + logic [`CHI_NODE_BITS-1:0] accepted_src; + logic [11:0] accepted_txn; + logic [`CHI_PENDING_BITS-1:0] pending; + } pins_t; +endpackage diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/rnf/filelist.f b/arch/src/main/scala/framework/ip/chi/src/main/resources/rnf/filelist.f new file mode 100644 index 000000000..47528a5bc --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/rnf/filelist.f @@ -0,0 +1,11 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@RESOURCES@/../common ++incdir+@VERIFY@/uvm/src/protocol/chi ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/protocol/chi/rnf/types.svh +@VERIFY@/uvm/src/protocol/chi/rnf/interface.sv +@RESOURCES@/package.sv +-F @RTL@/BankedChiCache/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/rnf/package.sv b/arch/src/main/scala/framework/ip/chi/src/main/resources/rnf/package.sv new file mode 100644 index 000000000..8d63f8d23 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/rnf/package.sv @@ -0,0 +1,3 @@ +package chi_rnf_pkg; + `include "rnf/body.svh" +endpackage diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/rnf/tb.sv b/arch/src/main/scala/framework/ip/chi/src/main/resources/rnf/tb.sv new file mode 100644 index 000000000..aeb1bee78 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/rnf/tb.sv @@ -0,0 +1,155 @@ +module rnf_tb; + import uvm_pkg::*; + import chi_rnf_pkg::*; + logic clock = 0; + clock_if timing (clock); + always #5 clock = ~clock; + chi_rnf_if vif (clock); + BankedChiCache dut ( + .clock(clock), + .reset(vif.reset), + .io_chi_req_bits_qos(vif.req[0+:4]), + .io_chi_req_bits_tgtId(vif.req[4+:7]), + .io_chi_req_bits_srcId(vif.req[11+:7]), + .io_chi_req_bits_txnId(vif.req[18+:12]), + .io_chi_req_bits_returnNid(vif.req[30+:7]), + .io_chi_req_bits_stashNidValidEndian(vif.req[37+:1]), + .io_chi_req_bits_returnTxnId(vif.req[38+:12]), + .io_chi_req_bits_opcode(vif.req[50+:7]), + .io_chi_req_bits_multiReq(vif.req[57+:1]), + .io_chi_req_bits_size(vif.req[58+:6]), + .io_chi_req_bits_addr(vif.req[64+:44]), + .io_chi_req_bits_pas(vif.req[108+:3]), + .io_chi_req_bits_likelyShared(vif.req[111+:1]), + .io_chi_req_bits_allowRetry(vif.req[112+:1]), + .io_chi_req_bits_order(vif.req[113+:2]), + .io_chi_req_bits_pCrdType(vif.req[115+:4]), + .io_chi_req_bits_memAttr(vif.req[119+:4]), + .io_chi_req_bits_snpAttr(vif.req[123+:1]), + .io_chi_req_bits_lpid(vif.req[124+:8]), + .io_chi_req_bits_exclSnoopMe(vif.req[132+:1]), + .io_chi_req_bits_expCompAck(vif.req[133+:1]), + .io_chi_req_bits_tagOp(vif.req[134+:2]), + .io_chi_req_bits_traceTag(vif.req[136+:1]), + .io_chi_req_valid(vif.req_valid), + .io_chi_req_ready(vif.req_ready), + .io_chi_txRsp_bits_qos(vif.rsp[0+:4]), + .io_chi_txRsp_bits_tgtId(vif.rsp[4+:7]), + .io_chi_txRsp_bits_srcId(vif.rsp[11+:7]), + .io_chi_txRsp_bits_txnId(vif.rsp[18+:12]), + .io_chi_txRsp_bits_opcode(vif.rsp[30+:5]), + .io_chi_txRsp_bits_respErr(vif.rsp[35+:2]), + .io_chi_txRsp_bits_resp(vif.rsp[37+:3]), + .io_chi_txRsp_bits_fwdState(vif.rsp[40+:3]), + .io_chi_txRsp_bits_cBusy(vif.rsp[43+:3]), + .io_chi_txRsp_bits_dbid(vif.rsp[46+:12]), + .io_chi_txRsp_bits_pCrdType(vif.rsp[58+:4]), + .io_chi_txRsp_bits_tagOp(vif.rsp[62+:2]), + .io_chi_txRsp_bits_traceTag(vif.rsp[64+:1]), + .io_chi_txRsp_bits_cacheLineId(vif.rsp[65+:6]), + .io_chi_txRsp_valid(vif.rsp_valid), + .io_chi_txRsp_ready(vif.rsp_ready), + .io_chi_txDat_bits_qos(vif.dat[0+:4]), + .io_chi_txDat_bits_tgtId(vif.dat[4+:7]), + .io_chi_txDat_bits_srcId(vif.dat[11+:7]), + .io_chi_txDat_bits_txnId(vif.dat[18+:12]), + .io_chi_txDat_bits_homeNid(vif.dat[30+:7]), + .io_chi_txDat_bits_opcode(vif.dat[37+:4]), + .io_chi_txDat_bits_respErr(vif.dat[41+:2]), + .io_chi_txDat_bits_resp(vif.dat[43+:3]), + .io_chi_txDat_bits_dataSource(vif.dat[46+:8]), + .io_chi_txDat_bits_dataPull(vif.dat[54+:1]), + .io_chi_txDat_bits_cBusy(vif.dat[55+:3]), + .io_chi_txDat_bits_dbid(vif.dat[58+:16]), + .io_chi_txDat_bits_ccid(vif.dat[74+:2]), + .io_chi_txDat_bits_dataId(vif.dat[76+:2]), + .io_chi_txDat_bits_cacheLineId(vif.dat[78+:6]), + .io_chi_txDat_bits_tagOp(vif.dat[84+:2]), + .io_chi_txDat_bits_tag(vif.dat[86+:8]), + .io_chi_txDat_bits_tagUpdate(vif.dat[94+:2]), + .io_chi_txDat_bits_traceTag(vif.dat[96+:1]), + .io_chi_txDat_bits_copyAtHome(vif.dat[97+:1]), + .io_chi_txDat_bits_numDat(vif.dat[98+:2]), + .io_chi_txDat_bits_replicate(vif.dat[100+:1]), + .io_chi_txDat_bits_be(vif.dat[101+:32]), + .io_chi_txDat_bits_data(vif.dat[133+:256]), + .io_chi_txDat_valid(vif.dat_valid), + .io_chi_txDat_ready(vif.dat_ready), + .io_chi_rxRsp_bits_qos(vif.rx_rsp[0+:4]), + .io_chi_rxRsp_bits_tgtId(vif.rx_rsp[4+:7]), + .io_chi_rxRsp_bits_srcId(vif.rx_rsp[11+:7]), + .io_chi_rxRsp_bits_txnId(vif.rx_rsp[18+:12]), + .io_chi_rxRsp_bits_opcode(vif.rx_rsp[30+:5]), + .io_chi_rxRsp_bits_respErr(vif.rx_rsp[35+:2]), + .io_chi_rxRsp_bits_resp(vif.rx_rsp[37+:3]), + .io_chi_rxRsp_bits_fwdState(vif.rx_rsp[40+:3]), + .io_chi_rxRsp_bits_cBusy(vif.rx_rsp[43+:3]), + .io_chi_rxRsp_bits_dbid(vif.rx_rsp[46+:12]), + .io_chi_rxRsp_bits_pCrdType(vif.rx_rsp[58+:4]), + .io_chi_rxRsp_bits_tagOp(vif.rx_rsp[62+:2]), + .io_chi_rxRsp_bits_traceTag(vif.rx_rsp[64+:1]), + .io_chi_rxRsp_bits_cacheLineId(vif.rx_rsp[65+:6]), + .io_chi_rxRsp_valid(vif.rx_rsp_valid), + .io_chi_rxRsp_ready(vif.rx_rsp_ready), + .io_chi_rxDat_bits_qos(vif.rx_dat[0+:4]), + .io_chi_rxDat_bits_tgtId(vif.rx_dat[4+:7]), + .io_chi_rxDat_bits_srcId(vif.rx_dat[11+:7]), + .io_chi_rxDat_bits_txnId(vif.rx_dat[18+:12]), + .io_chi_rxDat_bits_homeNid(vif.rx_dat[30+:7]), + .io_chi_rxDat_bits_opcode(vif.rx_dat[37+:4]), + .io_chi_rxDat_bits_respErr(vif.rx_dat[41+:2]), + .io_chi_rxDat_bits_resp(vif.rx_dat[43+:3]), + .io_chi_rxDat_bits_dataSource(vif.rx_dat[46+:8]), + .io_chi_rxDat_bits_dataPull(vif.rx_dat[54+:1]), + .io_chi_rxDat_bits_cBusy(vif.rx_dat[55+:3]), + .io_chi_rxDat_bits_dbid(vif.rx_dat[58+:16]), + .io_chi_rxDat_bits_ccid(vif.rx_dat[74+:2]), + .io_chi_rxDat_bits_dataId(vif.rx_dat[76+:2]), + .io_chi_rxDat_bits_cacheLineId(vif.rx_dat[78+:6]), + .io_chi_rxDat_bits_tagOp(vif.rx_dat[84+:2]), + .io_chi_rxDat_bits_tag(vif.rx_dat[86+:8]), + .io_chi_rxDat_bits_tagUpdate(vif.rx_dat[94+:2]), + .io_chi_rxDat_bits_traceTag(vif.rx_dat[96+:1]), + .io_chi_rxDat_bits_copyAtHome(vif.rx_dat[97+:1]), + .io_chi_rxDat_bits_numDat(vif.rx_dat[98+:2]), + .io_chi_rxDat_bits_replicate(vif.rx_dat[100+:1]), + .io_chi_rxDat_bits_be(vif.rx_dat[101+:32]), + .io_chi_rxDat_bits_data(vif.rx_dat[133+:256]), + .io_chi_rxDat_valid(vif.rx_dat_valid), + .io_chi_rxDat_ready(vif.rx_dat_ready), + .io_chi_snp_bits_qos(vif.snp[0+:4]), + .io_chi_snp_bits_srcId(vif.snp[4+:7]), + .io_chi_snp_bits_txnId(vif.snp[11+:12]), + .io_chi_snp_bits_fwdNid(vif.snp[23+:7]), + .io_chi_snp_bits_fwdTxnId(vif.snp[30+:12]), + .io_chi_snp_bits_opcode(vif.snp[42+:5]), + .io_chi_snp_bits_addr(vif.snp[47+:41]), + .io_chi_snp_bits_pas(vif.snp[88+:3]), + .io_chi_snp_bits_doNotGoToSd(vif.snp[91+:1]), + .io_chi_snp_bits_retToSrc(vif.snp[92+:1]), + .io_chi_snp_bits_traceTag(vif.snp[93+:1]), + .io_chi_snp_valid(vif.snp_valid), + .io_chi_snp_ready(vif.snp_ready), + .io_access_valid(vif.access_valid), + .io_access_ready(vif.access_ready), + .io_access_bits_write(vif.access_write), + .io_access_bits_addr(vif.access_addr), + .io_access_bits_data(vif.access_data), + .io_access_bits_mask(vif.access_mask), + .io_access_bits_atomic(vif.access_atomic), + .io_access_bits_atomicWord(vif.access_atomic_word), + .io_result_valid(vif.result_valid), + .io_result_ready(vif.result_ready), + .io_result_bits_error(vif.result_error), + .io_result_bits_data(vif.result_data), + .io_outstanding(vif.outstanding), + .io_hits(), + .io_misses() + ); + assign vif.reset = timing.reset; + initial begin + uvm_config_db#(virtual chi_rnf_if)::set(null, "uvm_test_top*", "vif", vif); + uvm_config_db#(virtual clock_if)::set(null, "uvm_test_top.env.reset", "timing", timing); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/agent/agent.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/agent/agent.svh new file mode 100644 index 000000000..6c18e56b9 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/agent/agent.svh @@ -0,0 +1,18 @@ +class agent extends uvm_agent; + `uvm_component_utils(agent) + driver drv; + monitor mon; + uvm_sequencer #(cycle_item) seqr; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + drv = driver::type_id::create("driver", this); + mon = monitor::type_id::create("monitor", this); + seqr = new("seqr", this); + endfunction + function void connect_phase(uvm_phase phase); + drv.seq_item_port.connect(seqr.seq_item_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/agent/driver.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/agent/driver.svh new file mode 100644 index 000000000..52f749096 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/agent/driver.svh @@ -0,0 +1,29 @@ +class driver extends uvm_driver #(cycle_item); + `uvm_component_utils(driver) + virtual link_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual link_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "CHI link interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item req; + forever begin + seq_item_port.get_next_item(req); + @(negedge vif.clock); + vif.reset = req.reset; + vif.pins.active = req.pins.active; + vif.pins.input_valid = req.pins.input_valid; + vif.pins.input_data = req.pins.input_data; + vif.pins.pend = req.pins.pend; + vif.pins.output_ready = req.pins.output_ready; + @(posedge vif.clock); + req.sampled = vif.pins; + #1; + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/agent/item.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/agent/item.svh new file mode 100644 index 000000000..0147091c1 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/agent/item.svh @@ -0,0 +1,17 @@ +class cycle_item extends uvm_sequence_item; + `uvm_object_utils(cycle_item) + link_signals pins, sampled; + bit reset; + function new(string name = "cycle_item"); + super.new(name); + endfunction +endclass +class flit_item extends uvm_sequence_item; + `uvm_object_utils_begin(flit_item) + `uvm_field_int(data, UVM_ALL_ON) + `uvm_object_utils_end + logic [WIDTH-1:0] data; + function new(string name = "flit_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/agent/monitor.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/agent/monitor.svh new file mode 100644 index 000000000..5ace5103b --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/agent/monitor.svh @@ -0,0 +1,24 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual link_if vif; + uvm_analysis_port #(cycle_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual link_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "CHI link monitor requires an interface") + endfunction + task run_phase(uvm_phase phase); + cycle_item t; + forever begin + @(posedge vif.clock); + t = cycle_item::type_id::create("observed"); + t.reset = vif.reset; + t.sampled = vif.pins; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/env/env.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/env/env.svh new file mode 100644 index 000000000..8f7141d8c --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/env/env.svh @@ -0,0 +1,26 @@ +class env extends uvm_env; + `uvm_component_utils(env) + agent bus; + model reference; + result_board #(flit_item, 0) scoreboard; + uvm_analysis_imp #(bit, env) reset_export; + function new(string name, uvm_component parent); + super.new(name, parent); + reset_export = new("reset_export", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + bus = agent::type_id::create("bus", this); + reference = model::type_id::create("model", this); + scoreboard = result_board#(flit_item, 0)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + bus.mon.ap.connect(reference.analysis_export); + reference.expected_ap.connect(scoreboard.expected_export); + reference.actual_ap.connect(scoreboard.actual_export); + reference.reset_ap.connect(reset_export); + endfunction + function void write(bit value); + scoreboard.cancel_pending(); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/env/model.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/env/model.svh new file mode 100644 index 000000000..e572b250a --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/env/model.svh @@ -0,0 +1,50 @@ +class model extends uvm_subscriber #(cycle_item); + `uvm_component_utils(model) + uvm_analysis_port #(flit_item) expected_ap, actual_ap; + uvm_analysis_port #(bit) reset_ap; + int granted = 0, occupied = 0; + covergroup behavior with function sample (int count, bit stalled, bit simultaneous); + option.per_instance = 1; + occupancy: coverpoint count { + bins empty = {0}; bins partial = {[1 : DEPTH - 1]}; bins full = {DEPTH}; + } + sink_stall: coverpoint stalled; + enqueue_and_dequeue: coverpoint simultaneous; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected_ap = new("expected_ap", this); + actual_ap = new("actual_ap", this); + reset_ap = new("reset_ap", this); + behavior = new(); + endfunction + function void write(cycle_item t); + link_signals s = t.sampled; + flit_item item; + bit retire = s.output_valid && s.output_ready; + if (t.reset) begin + reset_ap.write(1); + granted = 0; + occupied = 0; + return; + end + evidence::check(s.output_valid === (occupied != 0), + "CHI RX valid disagrees with buffered flits"); + if (s.input_valid) evidence::check(granted > 0, "CHI RX consumed an ungranted credit"); + behavior.sample(occupied, s.output_valid && !s.output_ready, s.input_valid && retire); + if (s.input_valid) begin + item = flit_item::type_id::create("expected"); + item.data = s.input_data; + expected_ap.write(item); + end + if (retire) begin + item = flit_item::type_id::create("actual"); + item.data = s.output_data; + actual_ap.write(item); + end + granted += int'(s.credit) - int'(s.input_valid); + occupied += int'(s.input_valid) - int'(retire); + evidence::check(granted >= 0 && occupied >= 0 && granted + occupied <= DEPTH, + "CHI RX credit/storage conservation failed"); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/filelist.f b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/filelist.f new file mode 100644 index 000000000..e545b857d --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/filelist.f @@ -0,0 +1,9 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/types.svh +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/Rx/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/interface.sv b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/interface.sv new file mode 100644 index 000000000..1d93de013 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/interface.sv @@ -0,0 +1,12 @@ +interface link_if ( + input logic clock +); + import link_types::*; + logic reset = 1; + link_signals pins; + assert property (@(posedge clock) disable iff (reset) + pins.output_valid && !pins.output_ready |=> pins.output_valid && $stable( + pins.output_data + )) + else $fatal(1, "CHI RX output changed under backpressure"); +endinterface diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/package.sv b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/package.sv new file mode 100644 index 000000000..824e2712f --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/package.sv @@ -0,0 +1,14 @@ +package link_pkg; + import uvm_pkg::*; + import verification_pkg::*; + import link_types::*; + `include "uvm_macros.svh" + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/agent.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/seq/sequence.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/seq/sequence.svh new file mode 100644 index 000000000..0442df222 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/seq/sequence.svh @@ -0,0 +1,89 @@ +class traffic extends uvm_sequence #(cycle_item); + `uvm_object_utils(traffic) + env test_env; + link_signals pins = '0, observed; + bit reset = 1, vary_ready = 0; + int budget = 0, cycle = 0, sent = 0; + function new(string name = "traffic"); + super.new(name); + endfunction + task tick(int cycles = 1); + cycle_item req; + repeat (cycles) begin + if (vary_ready) pins.output_ready = cycle % 7 < 3; + req = cycle_item::type_id::create("cycle"); + start_item(req); + req.pins = pins; + req.reset = reset; + finish_item(req); + observed = req.sampled; + if (!reset) budget += int'(observed.credit) - int'(pins.input_valid); + cycle++; + end + endtask + task restart(); + pins = '0; + reset = 1; + budget = 0; + vary_ready = 0; + tick(4); + reset = 0; + pins.active = 1; + pins.pend = 1; + endtask + task send(logic [WIDTH-1:0] data); + while (budget == 0) tick(); + pins.input_valid = 1; + pins.input_data = data; + tick(); + pins.input_valid = 0; + sent++; + endtask + task drain(); + while (test_env.scoreboard.checked < sent) tick(); + endtask + task body(); + restart(); + for (int n = 0; n < DEPTH; n++) send(WIDTH'(n)); + tick(8); + evidence::check(budget == 0 && observed.output_valid, "CHI RX did not retain a full buffer"); + pins.output_ready = 1; + drain(); + vary_ready = 1; + for (int n = 0; n < 256; n++) send(WIDTH'(32'h9e3779b9 * (n + 1)) ^ ((n % 2) ? '1 : '0)); + pins.output_ready = 1; + vary_ready = 0; + drain(); + tick(DEPTH + 2); + pins.output_ready = 0; + for (int n = 0; n < DEPTH; n++) send('1); + tick(3); + restart(); + sent -= DEPTH; + pins.output_ready = 1; + for (int n = 0; n < WIDTH; n++) send(WIDTH'(1) << n); + drain(); + tick(2); + evidence::check(evidence::cancelled == DEPTH, "CHI RX reset cancellation count mismatch"); + endtask +endclass +class invalid_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(invalid_sequence) + function new(string name = "invalid_sequence"); + super.new(name); + endfunction + task body(); + cycle_item req; + for (int n = 0; n < 2 * DEPTH + 8; n++) begin + req = cycle_item::type_id::create("overflow"); + start_item(req); + req.pins = '0; + req.reset = n < 4; + req.pins.active = n >= 4; + req.pins.pend = n >= 4; + req.pins.input_valid = n >= DEPTH + 6; + finish_item(req); + end + `uvm_fatal("EXPECTED_ASSERTION", "CHI RX accepted an ungranted flit") + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/tb.sv b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/tb.sv new file mode 100644 index 000000000..6d87af894 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/tb.sv @@ -0,0 +1,23 @@ +module rx_tb; + import uvm_pkg::*; + import link_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + link_if bus (clock); + Rx dut ( + .clock(clock), + .reset(bus.reset), + .io_active(bus.pins.active), + .io_link_flitpend(bus.pins.pend), + .io_link_flitv(bus.pins.input_valid), + .io_link_flit(bus.pins.input_data), + .io_link_lcrdv(bus.pins.credit), + .io_out_valid(bus.pins.output_valid), + .io_out_ready(bus.pins.output_ready), + .io_out_bits(bus.pins.output_data) + ); + initial begin + uvm_config_db#(virtual link_if)::set(null, "uvm_test_top*", "vif", bus); + run_test(); + end +endmodule diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/tests/test.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/tests/test.svh new file mode 100644 index 000000000..38c98c87f --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/tests/test.svh @@ -0,0 +1,31 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + traffic seq = traffic::type_id::create("traffic"); + seq.test_env = test_env; + seq.start(test_env.bus.seqr); + endtask +endclass +class invalid_test extends base_test; + `uvm_component_utils(invalid_test) + agent bus; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + bus = agent::type_id::create("bus", this); + endfunction + task execute(); + invalid_sequence seq = invalid_sequence::type_id::create("violation"); + seq.start(bus.seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/types.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/types.svh new file mode 100644 index 000000000..0f5428c6d --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/types.svh @@ -0,0 +1,9 @@ +`include "parameters.svh" +package link_types; + localparam int WIDTH = `CHI_LINK_BITS, DEPTH = `CHI_CREDITS; + typedef struct packed { + logic active, input_valid, input_ready, output_valid, output_ready, pend, credit; + logic [WIDTH-1:0] input_data, output_data; + logic [`CHI_CREDIT_BITS-1:0] credits; + } link_signals; +endpackage diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/agent/agent.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/agent/agent.svh new file mode 100644 index 000000000..dc28eea7a --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/agent/agent.svh @@ -0,0 +1,24 @@ +class agent extends uvm_agent; + `uvm_component_utils(agent) + monitor mon; + endpoint_driver endpoint; + uvm_sequencer #(packet_item) seqr[2]; + source_driver #(0) source0; + source_driver #(1) source1; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + mon = monitor::type_id::create("monitor", this); + endpoint = endpoint_driver::type_id::create("endpoint", this); + seqr[0] = new("seqr0", this); + source0 = source_driver#(0)::type_id::create("source0", this); + seqr[1] = new("seqr1", this); + source1 = source_driver#(1)::type_id::create("source1", this); + endfunction + function void connect_phase(uvm_phase phase); + source0.seq_item_port.connect(seqr[0].seq_item_export); + source1.seq_item_port.connect(seqr[1].seq_item_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/agent/driver.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/agent/driver.svh new file mode 100644 index 000000000..1b3297f7a --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/agent/driver.svh @@ -0,0 +1,39 @@ +class source_driver #( + int PORT = 0 +) extends uvm_driver #(packet_item); + `uvm_component_param_utils(source_driver#(PORT)) + virtual chi_sram_if vif; + protocol_model state; + packet_item pending; + bit driven = 0; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual chi_sram_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "CHI source interface is required") + endfunction + task run_phase(uvm_phase phase); + forever begin + @(negedge vif.clock); + if (pending == null) seq_item_port.try_next_item(pending); + driven = 0; + if (PORT == 0) begin + vif.req_flit = pending != null ? pending.payload : state.idle.req_flit; + driven = pending != null && !vif.reset && state.req_credits > 0; + vif.req_valid = driven; + end else begin + vif.dat_flit = pending != null ? pending.payload : state.idle.dat_flit; + driven = pending != null && !vif.reset && state.dat_credits > 0; + vif.dat_valid = driven; + end + @(posedge vif.clock); + if (driven) begin + #1; + seq_item_port.item_done(); + pending = null; + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/agent/endpoint.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/agent/endpoint.svh new file mode 100644 index 000000000..2d600813b --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/agent/endpoint.svh @@ -0,0 +1,24 @@ +class endpoint_driver extends uvm_component; + `uvm_component_utils(endpoint_driver) + virtual chi_sram_if vif; + protocol_model state; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual chi_sram_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "CHI endpoint interface is required") + endfunction + task run_phase(uvm_phase phase); + forever begin + @(negedge vif.clock); + vif.rx_active_req = state.idle.rx_active_req; + vif.tx_active_ack = state.idle.tx_active_ack; + vif.req_pend = state.idle.req_pend; + vif.dat_pend = state.idle.dat_pend; + vif.rsp_credit = !vif.reset && vif.tx_active_req && vif.tx_active_ack && state.grant_enabled && state.rsp_granted < 15 && state.cycle % 5 != 0; + vif.out_dat_credit = !vif.reset && vif.tx_active_req && vif.tx_active_ack && state.grant_enabled && state.out_dat_granted < 15 && state.cycle % 7 < 3; + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/agent/item.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/agent/item.svh new file mode 100644 index 000000000..a1c674ee2 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/agent/item.svh @@ -0,0 +1,37 @@ +class packet_item extends uvm_sequence_item; + `uvm_object_utils(packet_item) + logic [`CHI_DAT_BITS-1:0] payload; + bit sc_success, failed; + function new(string name = "packet_item"); + super.new(name); + endfunction +endclass +class sample_item extends uvm_sequence_item; + `uvm_object_utils(sample_item) + pins_t sampled; + function new(string name = "sample_item"); + super.new(name); + endfunction +endclass +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(data, UVM_ALL_ON) + `uvm_field_int(error, UVM_ALL_ON) + `uvm_object_utils_end + logic [63:0] data; + logic error; + function new(string name = "result_item"); + super.new(name); + endfunction +endclass +class packet_sequence extends uvm_sequence #(packet_item); + `uvm_object_utils(packet_sequence) + packet_item item; + function new(string name = "packet_sequence"); + super.new(name); + endfunction + task body(); + start_item(item); + finish_item(item); + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/agent/monitor.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/agent/monitor.svh new file mode 100644 index 000000000..cf177eaed --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/agent/monitor.svh @@ -0,0 +1,44 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual chi_sram_if vif; + uvm_analysis_port #(sample_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual chi_sram_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "CHI monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + sample_item t; + forever begin + @(posedge vif.clock); + t = sample_item::type_id::create("observed"); + t.sampled.reset = vif.reset; + t.sampled.req_pend = vif.req_pend; + t.sampled.req_valid = vif.req_valid; + t.sampled.req_credit = vif.req_credit; + t.sampled.req_flit = vif.req_flit; + t.sampled.dat_pend = vif.dat_pend; + t.sampled.dat_valid = vif.dat_valid; + t.sampled.dat_credit = vif.dat_credit; + t.sampled.dat_flit = vif.dat_flit; + t.sampled.rsp_pend = vif.rsp_pend; + t.sampled.rsp_valid = vif.rsp_valid; + t.sampled.rsp_credit = vif.rsp_credit; + t.sampled.rsp_flit = vif.rsp_flit; + t.sampled.out_dat_pend = vif.out_dat_pend; + t.sampled.out_dat_valid = vif.out_dat_valid; + t.sampled.out_dat_credit = vif.out_dat_credit; + t.sampled.out_dat_flit = vif.out_dat_flit; + t.sampled.rx_active_req = vif.rx_active_req; + t.sampled.rx_active_ack = vif.rx_active_ack; + t.sampled.tx_active_req = vif.tx_active_req; + t.sampled.tx_active_ack = vif.tx_active_ack; + t.sampled.outstanding = vif.outstanding; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/env/env.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/env/env.svh new file mode 100644 index 000000000..5ec2c87e7 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/env/env.svh @@ -0,0 +1,27 @@ +class env extends uvm_env; + `uvm_component_utils(env) + agent bus; + protocol_model state; + reset_controller reset; + result_board #(result_item, 0) scoreboard; + uvm_analysis_imp #(bit, env) reset_export; + function new(string name, uvm_component parent); + super.new(name, parent); + reset_export = new("reset_export", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + state = protocol_model::type_id::create("model", this); + bus = agent::type_id::create("bus", this); + reset = reset_controller::type_id::create("reset", this); + endfunction + function void connect_phase(uvm_phase phase); + bus.mon.ap.connect(state.analysis_export); + bus.endpoint.state = state; + bus.source0.state = state; + bus.source1.state = state; + endfunction + function void write(bit value); + scoreboard.cancel_pending(); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/env/model.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/env/model.svh new file mode 100644 index 000000000..4d8f0c030 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/env/model.svh @@ -0,0 +1,79 @@ +class protocol_model extends uvm_subscriber #(sample_item); + `uvm_component_utils(protocol_model) + chandle reference; + int req_credits = 0, dat_credits = 0, rsp_granted = 0, out_dat_granted = 0; + int cycle = 0, checks = 0; + localparam int STALL_READS = 15 / `CHI_BEATS + 8; + bit grant_enabled = 1; + rsp_t responses[$]; + dat_t data_responses[$]; + string scenario; + pins_t sampled; + pins_t idle; + uvm_analysis_port #(result_item) expected_ap, actual_ap; + uvm_analysis_port #(bit) reset_ap; + covergroup behavior with function sample (int op, bit error); + option.per_instance = 1; + request_opcode: coverpoint op { + bins read = {4}; bins write_full = {'h1d}; bins write_partial = {'h1c}; + } + response_error: coverpoint error; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + behavior = new(); + expected_ap = new("expected_ap", this); + actual_ap = new("actual_ap", this); + reset_ap = new("reset_ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!$value$plusargs("CASE=%s", scenario)) `uvm_fatal("CASE", "CHI scenario is required") + idle = '0; + reference = chi_ref_create(256); + endfunction + function void write(sample_item t); + pins_t s = t.sampled; + sampled = s; + if (s.req_valid) begin + req_t req = req_t'(s.req_flit); + behavior.sample(req.opcode, 0); + end + if (s.rsp_valid) begin + rsp_t rsp = rsp_t'(s.rsp_flit); + behavior.sample(4, rsp.error != 0); + end + if (s.out_dat_valid) begin + dat_t dat = dat_t'(s.out_dat_flit); + behavior.sample(4, dat.error != 0); + end + + if (s.reset) begin + req_credits = 0; + dat_credits = 0; + rsp_granted = 0; + out_dat_granted = 0; + responses.delete(); + data_responses.delete(); + end else begin + cycle++; + req_credits += int'(s.req_credit) - int'(s.req_valid); + dat_credits += int'(s.dat_credit) - int'(s.dat_valid); + rsp_granted += int'(s.rsp_credit) - int'(s.rsp_valid); + out_dat_granted += int'(s.out_dat_credit) - int'(s.out_dat_valid); + if (req_credits < 0 || dat_credits < 0 || rsp_granted < 0 || out_dat_granted < 0) + `uvm_fatal("CREDIT", "Flit without credit") + if (s.rsp_valid) responses.push_back(rsp_t'(s.rsp_flit)); + if (s.out_dat_valid) data_responses.push_back(dat_t'(s.out_dat_flit)); + end + + endfunction + function void final_phase(uvm_phase phase); + chi_ref_destroy(reference); + super.final_phase(phase); + endfunction + function void check_phase(uvm_phase phase); + evidence::check(responses.size() == 0 && data_responses.size() == 0, + "CHI SRAM responses remain unchecked"); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/filelist.f b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/filelist.f new file mode 100644 index 000000000..b48d1618b --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/filelist.f @@ -0,0 +1,10 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@RESOURCES@/../common ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/types.svh +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/SramEndpoint/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/interface.sv b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/interface.sv new file mode 100644 index 000000000..66748e026 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/interface.sv @@ -0,0 +1,17 @@ +`include "parameters.svh" +`include "profile.svh" +interface chi_sram_if ( + input logic clock +); + logic reset; + logic req_pend, req_valid, req_credit; + logic [`CHI_REQ_BITS-1:0] req_flit; + logic dat_pend, dat_valid, dat_credit; + logic [`CHI_DAT_BITS-1:0] dat_flit; + logic rsp_pend, rsp_valid, rsp_credit; + logic [`CHI_RSP_BITS-1:0] rsp_flit; + logic out_dat_pend, out_dat_valid, out_dat_credit; + logic [`CHI_DAT_BITS-1:0] out_dat_flit; + logic rx_active_req, rx_active_ack, tx_active_req, tx_active_ack; + logic [3:0] outstanding; +endinterface diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/package.sv b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/package.sv new file mode 100644 index 000000000..68a20b18a --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/package.sv @@ -0,0 +1,29 @@ +package chi_sram_pkg; + import uvm_pkg::*; + import verification_pkg::*; + import observed_types::*; + `include "uvm_macros.svh" + `include "flit_types.svh" + import "DPI-C" function chandle chi_ref_create(input int unsigned lines); + import "DPI-C" function void chi_ref_destroy(input chandle model); + import "DPI-C" function byte unsigned chi_ref_write( + input chandle model, + input longint unsigned addr, + input bit [511:0] data, + input bit [63:0] mask + ); + import "DPI-C" function byte unsigned chi_ref_read( + input chandle model, + input longint unsigned addr, + output bit [511:0] data + ); + `include "agent/item.svh" + `include "env/model.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/endpoint.svh" + `include "agent/agent.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/seq/sequence.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/seq/sequence.svh new file mode 100644 index 000000000..3a0e27f9d --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/seq/sequence.svh @@ -0,0 +1,450 @@ +class scenario_sequence extends uvm_sequence; + `uvm_object_utils(scenario_sequence) + localparam int STALL_READS = 15 / `CHI_BEATS + 8; + env test_env; + protocol_model state; + virtual chi_sram_if vif; + string scenario; + function new(string name = "scenario_sequence"); + super.new(name); + endfunction + task packet(int port, logic [`CHI_DAT_BITS-1:0] payload, bit sc_success = 0, bit failed = 0); + packet_sequence seq = packet_sequence::type_id::create("packet"); + seq.item = packet_item::type_id::create("item"); + seq.item.payload = payload; + seq.item.sc_success = sc_success; + seq.item.failed = failed; + seq.start(test_env.bus.seqr[port]); + endtask + task reset_link(); + state.idle.rx_active_req = 0; + state.idle.tx_active_ack = 0; + state.idle.req_pend = 0; + state.idle.dat_pend = 0; + test_env.reset.pulse(4); + repeat (3) @(negedge vif.clock); + state.idle.rx_active_req = 1; + wait (state.sampled.tx_active_req); + repeat (3) @(negedge vif.clock); + state.idle.tx_active_ack = 1; + state.idle.req_pend = 1; + state.idle.dat_pend = 1; + repeat (4) @(negedge vif.clock); + evidence::check(state.sampled.rx_active_ack && state.sampled.tx_active_req, + "CHI SRAM activation failed"); + endtask + + task send_req(req_t value); + packet(0, value); + endtask + + task send_dat(dat_t value); + packet(1, value); + endtask + + function req_t request(int opcode, int txn, longint unsigned addr, int source = 3); + req_t r = '0; + r.opcode = opcode; + r.txn = txn; + r.addr = addr; + r.size = 6; + r.src = (txn ^ source) & 127; + r.tgt = 1; + r.return_nid = (txn ^ 85) & 127; + r.return_txn = txn + 100; + r.allow_retry = txn % 2; + r.mem_attr = (txn % 3 == 0 ? 0 : txn % 3 == 1 ? 4 : 12) | (txn % 2); + r.likely_shared = txn % 2; + r.lpid = txn % 32; + r.qos = txn % 16; + r.trace_tag = txn % 2; + return r; + endfunction + task response(req_t r, int opcode, output rsp_t out_rsp); + bit found = 0; + while (!found) begin + @(negedge vif.clock); + foreach (state.responses[i]) begin + if (state.responses[i].txn == r.txn && state.responses[i].tgt == r.src) begin + out_rsp = state.responses[i]; + state.responses.delete(i); + found = 1; + break; + end + end + end + if (out_rsp.opcode != opcode || out_rsp.src != 1 || out_rsp.qos != r.qos || + out_rsp.trace_tag != r.trace_tag) + `uvm_fatal("RESPONSE", $sformatf("Bad response txn=%0d opcode=%0h", r.txn, out_rsp.opcode)) + state.checks++; + evidence::checks++; + endtask + function dat_t write_beat(req_t r, rsp_t dbid_rsp, int beat, bit [`CHI_DATA_BITS-1:0] data, + bit [`CHI_BE_BITS-1:0] mask = '1); + dat_t d = '0; + d.tgt = 1; + d.src = r.src; + d.txn = dbid_rsp.dbid; + d.qos = r.qos; + d.dbid = {4'b0, r.txn}; + d.line_id = r.addr[11:6]; + d.trace_tag = r.trace_tag; + d.opcode = 3; + d.data_id = beat * `CHI_DATA_ID_STEP; + d.data = data; + d.be = mask; + return d; + endfunction + task write_data(req_t r, rsp_t dbid_rsp, bit [511:0] data, bit [63:0] mask); + for (int beat = `CHI_BEATS - 1; beat >= 0; beat--) + send_dat(write_beat( + r, + dbid_rsp, + beat, + data[beat*`CHI_DATA_BITS+:`CHI_DATA_BITS], + mask[beat*`CHI_BE_BITS+:`CHI_BE_BITS] + )); + endtask + task write_line(int txn, longint unsigned addr, bit [511:0] data, bit [63:0] mask = '1); + req_t r = request(mask == '1 ? 'h1d : 'h1c, txn, addr); + rsp_t s; + byte unsigned expected_error; + send_req(r); + response(r, 6, s); + write_data(r, s, data, mask); + response(r, 4, s); + expected_error = chi_ref_write(state.reference, addr, data, mask); + if (s.error != (expected_error ? 3 : 0)) `uvm_fatal("ERROR", "Wrong write error") + state.checks++; + evidence::checks++; + endtask + task read_line(int txn, longint unsigned addr); + req_t r = request(4, txn, addr); + dat_t d; + bit [511:0] golden; + bit [`CHI_BEATS-1:0] seen = 0; + byte unsigned error = chi_ref_read(state.reference, addr, golden); + send_req(r); + while (!(&seen)) begin + bit found = 0; + @(negedge vif.clock); + foreach (state.data_responses[i]) begin + if (state.data_responses[i].txn == r.return_txn) begin + d = state.data_responses[i]; + state.data_responses.delete(i); + found = 1; + break; + end + end + if (found) begin + int b = d.data_id / `CHI_DATA_ID_STEP; + if ((d.data_id % `CHI_DATA_ID_STEP != 0 || b >= `CHI_BEATS) || seen[b] || d.opcode != 4 || + d.tgt != r.return_nid || d.src != 1 || d.home != r.src || d.dbid != r.txn || + d.error != (error ? 3 : 0) || d.qos != r.qos || d.trace_tag != r.trace_tag || + d.data !== golden[b*`CHI_DATA_BITS +: `CHI_DATA_BITS]) + `uvm_fatal("READ", $sformatf("Bad read data txn=%0d DataID=%0d", txn, d.data_id)) + seen[b] = 1; + state.checks++; + evidence::checks++; + end + end + endtask + task scenarios(); + req_t r[8]; + rsp_t dbid[8], s; + bit [511:0] words[8]; + reset_link(); + if (scenario == "bad_opcode") begin + send_req(request('h07, 0, 0)); + repeat (20) @(negedge vif.clock); + `uvm_fatal("NEGATIVE", "Unsupported opcode did not terminate RTL") + end else if (scenario == "duplicate_data") begin + r[0] = request('h1d, 0, 0); + send_req(r[0]); + response(r[0], 6, s); + begin + dat_t d = '0; + d.tgt = 1; + d.src = r[0].src; + d.txn = s.dbid; + d.opcode = 3; + d.data_id = 0; + d.be = '1; + send_dat(d); + send_dat(d); + end + repeat (20) @(negedge vif.clock); + `uvm_fatal("NEGATIVE", "Duplicate DataID did not terminate RTL") + end else begin + // Fill all eight transaction slots, then interleave reverse-order DataIDs. + for (int i = 0; i < 8; i++) begin + for (int b = 0; b < 64; b++) words[i][b*8+:8] = (i * 37 + b); + r[i] = request('h1d, i, i * 64, i % 2 ? 5 : 3); + send_req(r[i]); + response(r[i], 6, dbid[i]); + end + if (state.sampled.outstanding != 8) `uvm_fatal("MSHR", "Failed to occupy all slots") + for (int beat = `CHI_BEATS - 1; beat >= 0; beat--) begin + for (int i = 7; i >= 0; i--) begin + send_dat(write_beat(r[i], dbid[i], beat, words[i][beat*`CHI_DATA_BITS+:`CHI_DATA_BITS], '1 + )); + end + end + for (int i = 0; i < 8; i++) begin + response(r[i], 4, s); + if (s.error != 0 || chi_ref_write(state.reference, i * 64, words[i], '1) != 0) + `uvm_fatal("WRITE", "Initial full write failed") + end + for (int i = 0; i < 8; i++) read_line(32 + i, i * 64); + // Every live slot must merge byte enables independently, including interleaved beats. + for (int phase_index = 0; phase_index < 2; phase_index++) begin + bit [63:0] mask = phase_index ? 64'haaaaaaaaaaaaaaaa : 64'h5555555555555555; + for (int i = 0; i < 8; i++) begin + r[i] = request('h1c, 140 + phase_index * 8 + i, i * 64, i % 2 ? 5 : 3); + send_req(r[i]); + response(r[i], 6, dbid[i]); + end + for (int beat = `CHI_BEATS - 1; beat >= 0; beat--) begin + for (int i = 7; i >= 0; i--) begin + send_dat(write_beat( + r[i], + dbid[i], + beat, + phase_index ? words[i][beat*`CHI_DATA_BITS +: `CHI_DATA_BITS] : ~words[i][beat*`CHI_DATA_BITS +: `CHI_DATA_BITS], + mask[beat*`CHI_BE_BITS+:`CHI_BE_BITS] + )); + end + end + for (int i = 0; i < 8; i++) begin + bit [511:0] data = phase_index ? words[i] : ~words[i]; + response(r[i], 4, s); + if (s.error || chi_ref_write(state.reference, i * 64, data, mask)) + `uvm_fatal("MASK", "Concurrent partial write failed") + end + for (int i = 0; i < 8; i++) read_line(240 + phase_index * 8 + i, i * 64); + end + `uvm_info("PHASE", "Full-width slot vectors", UVM_LOW) + // Exercise each real transaction slot's address, opaque IDs and error status. + // Writes are concurrent, addresses distinct; then verify valid writes by readback. + for (int polarity = 0; polarity < 2; polarity++) begin + for (int address_bit = 9; address_bit < 44; address_bit++) begin + for (int i = 0; i < 8; i++) begin + longint unsigned high = (64'h1 << address_bit) ^ (polarity ? 64'hffffffffe00 : 0); + int tag = (1 << ((address_bit + i) % 12)) ^ (polarity ? 'hfff : 0); + int source = ((1 << ((address_bit + i) % 7)) | 2) ^ (polarity ? 'h7f : 0); + r[i] = request('h1d, tag, high | ((i ^ (1 << ((address_bit + polarity) % 3))) << 6)); + r[i].src = source; + r[i].return_nid = source ^ 'h7f; + r[i].return_txn = tag ^ 'hfff; + r[i].qos = (address_bit + i) % 16; + r[i].trace_tag = polarity; + send_req(r[i]); + response(r[i], 6, dbid[i]); + end + for (int beat = `CHI_BEATS - 1; beat >= 0; beat--) begin + for (int i = 7; i >= 0; i--) begin + send_dat(write_beat( + r[i], + dbid[i], + beat, + polarity ? words[i][beat*`CHI_DATA_BITS +: `CHI_DATA_BITS] : ~words[i][beat*`CHI_DATA_BITS +: `CHI_DATA_BITS], + '1 + )); + end + end + for (int i = 0; i < 8; i++) begin + bit [511:0] data = polarity ? words[i] : ~words[i]; + byte unsigned error; + response(r[i], 4, s); + error = chi_ref_write(state.reference, r[i].addr, data, '1); + if (s.error != (error ? 3 : 0)) + `uvm_fatal("ERROR_SLOT", "Wrong error or opaque transaction attribution") + state.checks++; + evidence::checks++; + end + if (!polarity && address_bit < 14) + for (int i = 0; i < 8; i++) read_line(600 + address_bit * 8 + i, r[i].addr); + end + end + write_line(50, 0, '1, 64'h5555555555555555); + write_line(51, 0, '0, 64'haaaaaaaaaaaaaaaa); + write_line(52, 0, '1, '0); + read_line(53, 0); + write_line(54, 255 * 64, '1); + read_line(55, 255 * 64); + write_line(56, 256 * 64, '1); + read_line(57, 256 * 64); + // Full-width addresses and TxnIDs must not alias into SRAM or DBID slots. + write_line(4095, 0, '1); + read_line(3950, 0); + write_line(58, 64'h80000000000, '1); + read_line(59, 64'h80000000000); + `uvm_info("PHASE", "Data credit stall", UVM_LOW) + // Block output credit return; requests remain live, then resume safely. + state.grant_enabled = 0; + for (int i = 0; i < STALL_READS; i++) send_req(request(4, 80 + i, (i % 8) * 64)); + repeat (25) @(negedge vif.clock); + if (state.sampled.outstanding == 0) + `uvm_fatal("STALL", "No live request under data credit stall") + state.grant_enabled = 1; + // These reads were issued already: compare every returned beat before reset. + for (int i = 0; i < STALL_READS; i++) begin + bit [511:0] golden; + bit [`CHI_BEATS-1:0] seen = 0; + if (chi_ref_read(state.reference, (i % 8) * 64, golden)) + `uvm_fatal("MODEL", "Unexpected error") + while (!(&seen)) begin + @(negedge vif.clock); + foreach (state.data_responses[j]) begin + if (state.data_responses[j].txn == 180 + i) begin + dat_t d = state.data_responses[j]; + int b = d.data_id / `CHI_DATA_ID_STEP; + if (seen[b] || d.data !== golden[b*`CHI_DATA_BITS+:`CHI_DATA_BITS]) + `uvm_fatal("STALL", "Stalled read corrupted") + seen[b] = 1; + state.checks++; + evidence::checks++; + state.data_responses.delete(j); + break; + end + end + end + end + `uvm_info("PHASE", "Read-to-write slot reuse", UVM_LOW) + // Reallocate every slot to a write after the all-slot read drain. + for (int i = 0; i < 8; i++) begin + r[i] = request('h1d, 2800 + i, i * 64); + send_req(r[i]); + response(r[i], 6, dbid[i]); + end + for (int i = 0; i < 8; i++) write_data(r[i], dbid[i], words[i], '1); + for (int i = 0; i < 8; i++) begin + response(r[i], 4, s); + if (s.error || chi_ref_write(state.reference, i * 64, words[i], '1)) + `uvm_fatal("SLOT_REUSE", "Read-to-write slot reuse failed") + end + for (int i = 0; i < 8; i++) read_line(2900 + i, i * 64); + `uvm_info("PHASE", "Response and request queue saturation", UVM_LOW) + // Exhaust all15 RSP credits, then fill all8 node slots and all4 RX entries. + // Distinct TxnIDs from one source and equal TxnIDs from distinct sources are legal. + begin + req_t blocked[11]; + int data_order[8] = '{0, 2, 1, 3, 4, 6, 5, 7}; + while (state.rsp_granted != 15) @(negedge vif.clock); + state.grant_enabled = 0; + for (int i = 0; i < 8; i++) begin + r[i] = request('h1d, 3200 + i, i * 64); + r[i].src = 3; + send_req(r[i]); + response(r[i], 6, dbid[i]); + end + for (int i = 0; i < 8; i++) begin + int slot = data_order[i]; + write_data(r[slot], dbid[slot], words[slot], '1); + end + // Drain seven initial Comps before new DBID state.responses compete for credits. + // One-beat writes can otherwise finish their data input before SRAM replies. + while (state.rsp_granted != 0 || state.responses.size() != 7) @(negedge vif.clock); + for (int i = 0; i < 11; i++) begin + blocked[i] = request('h1d, 3400, i * 64); + blocked[i].src = 10 + i; + send_req(blocked[i]); + end + repeat (30) @(negedge vif.clock); + if (state.sampled.outstanding != 8 || state.req_credits != 0) + `uvm_fatal("RSP_STALL", "RSP credit exhaustion did not saturate node and RX queue") + state.grant_enabled = 1; + for (int i = 0; i < 8; i++) begin + response(r[i], 4, s); + if (s.error || chi_ref_write(state.reference, i * 64, words[i], '1)) + `uvm_fatal("RSP_STALL", "Initial write failed") + end + for (int i = 0; i < 11; i++) begin + response(blocked[i], 6, s); + write_data(blocked[i], s, words[i%8], '1); + response(blocked[i], 4, s); + if (s.error || chi_ref_write(state.reference, i * 64, words[i%8], '1)) + `uvm_fatal("RSP_STALL", "Queued same-TxnID write attributed to wrong source") + end + for (int i = 0; i < 11; i++) read_line(3600 + i, i * 64); + end + `uvm_info("PHASE", "Isolated upper read slots", UVM_LOW) + // Hold lower slots in write-data collection to isolate each upper read slot. + // A second read in that slot exercises arbitration after its own last grant. + for (int target_slot = 4; target_slot < 8; target_slot++) begin + for (int i = 0; i < target_slot; i++) begin + r[i] = request('h1d, 3700 + i, i * 64); + r[i].src = 3; + send_req(r[i]); + response(r[i], 6, dbid[i]); + end + read_line(3800 + target_slot * 2, 0); + read_line(3801 + target_slot * 2, 0); + for (int i = 0; i < target_slot; i++) begin + write_data(r[i], dbid[i], words[i], '1); + response(r[i], 4, s); + if (s.error || chi_ref_write(state.reference, i * 64, words[i], '1)) + `uvm_fatal("ARBITRATION", "Isolated slot damaged held writes") + end + end + `uvm_info("PHASE", "Source and transaction identity", UVM_LOW) + // Slot7 stays live while a lower free slot accepts another transaction from its source. + for (int i = 0; i < 8; i++) begin + r[i] = request('h1d, 3900 + i, i * 64); + r[i].src = 3; + send_req(r[i]); + response(r[i], 6, dbid[i]); + end + for (int i = 0; i < 2; i++) begin + write_data(r[i], dbid[i], words[i], '1); + response(r[i], 4, s); + if (s.error || chi_ref_write(state.reference, i * 64, words[i], '1)) + `uvm_fatal("SOURCE_IDS", "Initial source transaction failed") + end + for (int i = 0; i < 2; i++) begin + r[i] = request('h1d, 3990 + i, i * 64); + r[i].src = 3; + send_req(r[i]); + response(r[i], 6, dbid[i]); + end + repeat (4) @(negedge vif.clock); + for (int i = 0; i < 8; i++) begin + write_data(r[i], dbid[i], words[i], '1); + response(r[i], 4, s); + if (s.error || chi_ref_write(state.reference, i * 64, words[i], '1)) + `uvm_fatal("SOURCE_IDS", "Concurrent source transaction failed") + end + `uvm_info("PHASE", "Reset cancellation", UVM_LOW) + // Cancel a write waiting for payload. SRAM data from completed writes survives. + send_req(request('h1d, 110, 64)); + response(request('h1d, 110, 64), 6, s); + reset_link(); + read_line(111, 64); + write_line(110, 64, '1); + write_line(110, 64, '1); // Reuse is legal only after the first completion. + read_line(112, 64); + repeat (15) @(negedge vif.clock); + if (state.sampled.outstanding != 0 || state.responses.size() || state.data_responses.size()) + `uvm_fatal("DRAIN", "Unmatched transactions remain") + // Payload is unconstrained while FLITV=0 and must not create a transaction. + @(negedge vif.clock); + state.idle.req_flit = '0; + state.idle.dat_flit = '0; + repeat (2) @(negedge vif.clock); + state.idle.req_flit = '1; + state.idle.dat_flit = '1; + repeat (2) @(negedge vif.clock); + state.idle.req_flit = '0; + state.idle.dat_flit = '0; + repeat (2) @(negedge vif.clock); + if (state.sampled.outstanding || state.responses.size() || state.data_responses.size()) + `uvm_fatal("INVALID_CHANNEL", "Inactive physical payload changed transactions") + `uvm_info("CHECKS", $sformatf("CHI SN-F: %0d checked responses/data beats", state.checks), + UVM_LOW) + + end + endtask + task body(); + scenarios(); + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/tb.sv b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/tb.sv new file mode 100644 index 000000000..c46a36c7b --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/tb.sv @@ -0,0 +1,41 @@ +module sram_tb; + import uvm_pkg::*; + import chi_sram_pkg::*; + logic clock = 0; + clock_if timing (clock); + always #5 clock = ~clock; + chi_sram_if source_if (clock); + `CHI_SRAM_MODULE dut ( + .clock(clock), + .reset(source_if.reset), + .io_chi_rxReq_flitpend(source_if.req_pend), + .io_chi_rxReq_flitv(source_if.req_valid), + .io_chi_rxReq_flit(source_if.req_flit), + .io_chi_rxReq_lcrdv(source_if.req_credit), + .io_chi_rxDat_flitpend(source_if.dat_pend), + .io_chi_rxDat_flitv(source_if.dat_valid), + .io_chi_rxDat_flit(source_if.dat_flit), + .io_chi_rxDat_lcrdv(source_if.dat_credit), + .io_chi_txRsp_flitpend(source_if.rsp_pend), + .io_chi_txRsp_flitv(source_if.rsp_valid), + .io_chi_txRsp_flit(source_if.rsp_flit), + .io_chi_txRsp_lcrdv(source_if.rsp_credit), + .io_chi_txDat_flitpend(source_if.out_dat_pend), + .io_chi_txDat_flitv(source_if.out_dat_valid), + .io_chi_txDat_flit(source_if.out_dat_flit), + .io_chi_txDat_lcrdv(source_if.out_dat_credit), + .io_chi_rxLinkActiveReq(source_if.rx_active_req), + .io_chi_rxLinkActiveAck(source_if.rx_active_ack), + .io_chi_txLinkActiveReq(source_if.tx_active_req), + .io_chi_txLinkActiveAck(source_if.tx_active_ack), + .io_chi_rxSActive(source_if.rx_active_req), + .io_chi_txSActive(), + .io_outstanding(source_if.outstanding) + ); + assign source_if.reset = timing.reset; + initial begin + uvm_config_db#(virtual chi_sram_if)::set(null, "uvm_test_top*", "vif", source_if); + uvm_config_db#(virtual clock_if)::set(null, "uvm_test_top.env.reset", "timing", timing); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/tests/test.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/tests/test.svh new file mode 100644 index 000000000..4c5de65de --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/tests/test.svh @@ -0,0 +1,25 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + virtual chi_sram_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + if (`CHI_NODE_BITS != 7 || `CHI_ADDRESS_BITS != 44 || `CHI_NODE_ID != 1) + `uvm_fatal("PROFILE", + "CHI packet scenarios require 7-bit nodes, 44-bit addresses and requester 1") + super.build_phase(phase); + test_env = env::type_id::create("env", this); + if (!uvm_config_db#(virtual chi_sram_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "CHI timing interface is required") + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.test_env = test_env; + seq.state = test_env.state; + seq.vif = vif; + seq.scenario = test_env.state.scenario; + seq.start(null); + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/types.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/types.svh new file mode 100644 index 000000000..95197dd2a --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/types.svh @@ -0,0 +1,16 @@ +`include "profile.svh" +package observed_types; + typedef struct packed { + logic reset; + logic req_pend, req_valid, req_credit; + logic [`CHI_REQ_BITS-1:0] req_flit; + logic dat_pend, dat_valid, dat_credit; + logic [`CHI_DAT_BITS-1:0] dat_flit; + logic rsp_pend, rsp_valid, rsp_credit; + logic [`CHI_RSP_BITS-1:0] rsp_flit; + logic out_dat_pend, out_dat_valid, out_dat_credit; + logic [`CHI_DAT_BITS-1:0] out_dat_flit; + logic rx_active_req, rx_active_ack, tx_active_req, tx_active_ack; + logic [3:0] outstanding; + } pins_t; +endpackage diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/suite.toml b/arch/src/main/scala/framework/ip/chi/src/main/resources/suite.toml new file mode 100644 index 000000000..832f73ad9 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/suite.toml @@ -0,0 +1,241 @@ +[model] +manifest = "src/csrc/Cargo.toml" +library = "libchi_reference_model" + +[[tests]] +functional_groups = ["chi_retry_pkg::protocol_model::behavior"] +name = "retry" +top = "retry_tb" +test = "protocol_test" +filelist = "retry/filelist.f" +timeout_ns = 450000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=normal"] +assertion = "" + +[[tests]] +functional_groups = [] +name = "retry_bad_opcode" +top = "retry_tb" +test = "protocol_test" +filelist = "retry/filelist.f" +timeout_ns = 450000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=bad_opcode"] +assertion = "Unsupported CHI retry request opcode" + +[[tests]] +functional_groups = [] +name = "retry_duplicate_txnid" +top = "retry_tb" +test = "protocol_test" +filelist = "retry/filelist.f" +timeout_ns = 450000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=duplicate_txnid"] +assertion = "CHI retry requester reused a live TxnID" + +[[tests]] +functional_groups = ["chi_rnf_pkg::protocol_model::behavior"] +name = "rnf" +top = "rnf_tb" +test = "protocol_test" +filelist = "rnf/filelist.f" +timeout_ns = 450000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=normal"] +assertion = "" + +[[tests]] +functional_groups = [] +name = "rnf_bad_dbid" +top = "rnf_tb" +test = "protocol_test" +filelist = "rnf/filelist.f" +timeout_ns = 450000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=bad_dbid"] +assertion = "Inconsistent multi-beat fill metadata" + +[[tests]] +functional_groups = [] +name = "rnf_bad_dbid_width" +top = "rnf_tb" +test = "protocol_test" +filelist = "rnf/filelist.f" +timeout_ns = 450000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=bad_dbid_width"] +assertion = "Fill DBID exceeds completion ID width" + +[[tests]] +functional_groups = [] +name = "rnf_bad_resp" +top = "rnf_tb" +test = "protocol_test" +filelist = "rnf/filelist.f" +timeout_ns = 450000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=bad_resp"] +assertion = "Inconsistent multi-beat fill metadata" + +[[tests]] +functional_groups = ["chi_sram_pkg::protocol_model::behavior"] +name = "sram" +top = "sram_tb" +test = "protocol_test" +filelist = "sram/filelist.f" +timeout_ns = 450000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=normal"] +assertion = "" + +[[tests]] +functional_groups = [] +name = "sram_bad_opcode" +top = "sram_tb" +test = "protocol_test" +filelist = "sram/filelist.f" +timeout_ns = 450000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=bad_opcode"] +assertion = "Unsupported CHI request opcode" + +[[tests]] +functional_groups = [] +name = "sram_duplicate_data" +top = "sram_tb" +test = "protocol_test" +filelist = "sram/filelist.f" +timeout_ns = 450000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=duplicate_data"] +assertion = "Duplicate CHI DataID" + +[[tests]] +functional_groups = ["link_pkg::model::behavior"] +name = "tx" +top = "tx_tb" +test = "protocol_test" +filelist = "tx/filelist.f" +timeout_ns = 450000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +functional_groups = [] +name = "tx_bad_credit" +top = "tx_tb" +test = "invalid_test" +filelist = "tx/filelist.f" +timeout_ns = 450000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "CHI TX credit overflow" + +[[tests]] +functional_groups = ["link_pkg::model::behavior"] +name = "rx" +top = "rx_tb" +test = "protocol_test" +filelist = "rx/filelist.f" +timeout_ns = 450000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +functional_groups = [] +name = "rx_bad_credit" +top = "rx_tb" +test = "invalid_test" +filelist = "rx/filelist.f" +timeout_ns = 450000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "CHI RX flit without granted credit" diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/agent/agent.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/agent/agent.svh new file mode 100644 index 000000000..6c18e56b9 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/agent/agent.svh @@ -0,0 +1,18 @@ +class agent extends uvm_agent; + `uvm_component_utils(agent) + driver drv; + monitor mon; + uvm_sequencer #(cycle_item) seqr; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + drv = driver::type_id::create("driver", this); + mon = monitor::type_id::create("monitor", this); + seqr = new("seqr", this); + endfunction + function void connect_phase(uvm_phase phase); + drv.seq_item_port.connect(seqr.seq_item_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/agent/driver.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/agent/driver.svh new file mode 100644 index 000000000..1468e5b41 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/agent/driver.svh @@ -0,0 +1,28 @@ +class driver extends uvm_driver #(cycle_item); + `uvm_component_utils(driver) + virtual link_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual link_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "CHI link interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item req; + forever begin + seq_item_port.get_next_item(req); + @(negedge vif.clock); + vif.reset = req.reset; + vif.pins.active = req.pins.active; + vif.pins.input_valid = req.pins.input_valid; + vif.pins.input_data = req.pins.input_data; + vif.pins.credit = req.pins.credit; + @(posedge vif.clock); + req.sampled = vif.pins; + #1; + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/agent/item.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/agent/item.svh new file mode 100644 index 000000000..0147091c1 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/agent/item.svh @@ -0,0 +1,17 @@ +class cycle_item extends uvm_sequence_item; + `uvm_object_utils(cycle_item) + link_signals pins, sampled; + bit reset; + function new(string name = "cycle_item"); + super.new(name); + endfunction +endclass +class flit_item extends uvm_sequence_item; + `uvm_object_utils_begin(flit_item) + `uvm_field_int(data, UVM_ALL_ON) + `uvm_object_utils_end + logic [WIDTH-1:0] data; + function new(string name = "flit_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/agent/monitor.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/agent/monitor.svh new file mode 100644 index 000000000..5ace5103b --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/agent/monitor.svh @@ -0,0 +1,24 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual link_if vif; + uvm_analysis_port #(cycle_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual link_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "CHI link monitor requires an interface") + endfunction + task run_phase(uvm_phase phase); + cycle_item t; + forever begin + @(posedge vif.clock); + t = cycle_item::type_id::create("observed"); + t.reset = vif.reset; + t.sampled = vif.pins; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/env/env.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/env/env.svh new file mode 100644 index 000000000..8f7141d8c --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/env/env.svh @@ -0,0 +1,26 @@ +class env extends uvm_env; + `uvm_component_utils(env) + agent bus; + model reference; + result_board #(flit_item, 0) scoreboard; + uvm_analysis_imp #(bit, env) reset_export; + function new(string name, uvm_component parent); + super.new(name, parent); + reset_export = new("reset_export", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + bus = agent::type_id::create("bus", this); + reference = model::type_id::create("model", this); + scoreboard = result_board#(flit_item, 0)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + bus.mon.ap.connect(reference.analysis_export); + reference.expected_ap.connect(scoreboard.expected_export); + reference.actual_ap.connect(scoreboard.actual_export); + reference.reset_ap.connect(reset_export); + endfunction + function void write(bit value); + scoreboard.cancel_pending(); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/env/model.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/env/model.svh new file mode 100644 index 000000000..26b63b133 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/env/model.svh @@ -0,0 +1,55 @@ +class model extends uvm_subscriber #(cycle_item); + `uvm_component_utils(model) + uvm_analysis_port #(flit_item) expected_ap, actual_ap; + uvm_analysis_port #(bit) reset_ap; + int balance = 0; + bit delayed_credit = 0, delayed_send = 0; + covergroup behavior with function sample (int credits, bit stalled, bit simultaneous); + option.per_instance = 1; + available: coverpoint credits { + bins empty = {0}; bins partial = {[1 : DEPTH - 1]}; bins full = {DEPTH}; + } + source_stall: coverpoint stalled; + credit_and_send: coverpoint simultaneous; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected_ap = new("expected_ap", this); + actual_ap = new("actual_ap", this); + reset_ap = new("reset_ap", this); + behavior = new(); + endfunction + function void write(cycle_item t); + link_signals s = t.sampled; + flit_item item; + bit fire = s.input_valid && s.input_ready; + if (t.reset) begin + reset_ap.write(1); + balance = 0; + delayed_credit = 0; + delayed_send = 0; + return; + end + evidence::check(s.credits == balance, "CHI TX credit count mismatch"); + evidence::check(s.input_ready === (s.active && balance != 0), + "CHI TX ready disagrees with available credit"); + evidence::check(s.pend === s.active, "CHI TX FLITPEND disagrees with activation"); + evidence::check(s.output_valid === delayed_send, + "CHI TX FLITV did not follow the accepted flit"); + behavior.sample(balance, s.input_valid && !s.input_ready, delayed_credit && fire); + if (fire) begin + item = flit_item::type_id::create("expected"); + item.data = s.input_data; + expected_ap.write(item); + end + if (s.output_valid) begin + item = flit_item::type_id::create("actual"); + item.data = s.output_data; + actual_ap.write(item); + end + balance += int'(delayed_credit) - int'(fire); + evidence::check(balance >= 0 && balance <= DEPTH, "CHI TX credit conservation failed"); + delayed_credit = s.credit; + delayed_send = fire; + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/filelist.f b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/filelist.f new file mode 100644 index 000000000..17aa13b13 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/filelist.f @@ -0,0 +1,9 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/types.svh +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/Tx/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/interface.sv b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/interface.sv new file mode 100644 index 000000000..e74a8c197 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/interface.sv @@ -0,0 +1,7 @@ +interface link_if ( + input logic clock +); + import link_types::*; + logic reset = 1; + link_signals pins; +endinterface diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/package.sv b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/package.sv new file mode 100644 index 000000000..824e2712f --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/package.sv @@ -0,0 +1,14 @@ +package link_pkg; + import uvm_pkg::*; + import verification_pkg::*; + import link_types::*; + `include "uvm_macros.svh" + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/agent.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/seq/sequence.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/seq/sequence.svh new file mode 100644 index 000000000..a9b7ecbf5 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/seq/sequence.svh @@ -0,0 +1,79 @@ +class traffic extends uvm_sequence #(cycle_item); + `uvm_object_utils(traffic) + env test_env; + link_signals pins = '0, observed; + bit reset = 1; + int grants = 0, cycle = 0, sent = 0; + function new(string name = "traffic"); + super.new(name); + endfunction + task tick(int cycles = 1); + cycle_item req; + repeat (cycles) begin + pins.credit = !reset && pins.active && grants > 0 && cycle % 4 != 0; + if (pins.credit) grants--; + req = cycle_item::type_id::create("cycle"); + start_item(req); + req.pins = pins; + req.reset = reset; + finish_item(req); + observed = req.sampled; + if (!reset && observed.output_valid) grants++; + cycle++; + end + endtask + task restart(); + pins = '0; + reset = 1; + grants = 0; + tick(4); + reset = 0; + pins.active = 1; + grants = DEPTH; + do tick(); while (observed.credits != DEPTH); + endtask + task send(logic [WIDTH-1:0] data); + pins.input_valid = 1; + pins.input_data = data; + do tick(); while (!observed.input_ready); + pins.input_valid = 0; + sent++; + endtask + task drain(); + while (test_env.scoreboard.checked < sent) tick(); + endtask + task body(); + restart(); + for (int n = 0; n < 256; n++) send(WIDTH'(32'h9e3779b9 * (n + 1)) ^ ((n % 2) ? '1 : '0)); + drain(); + // Reset cancels the accepted flit before its registered physical output is observed. + pins.input_valid = 1; + pins.input_data = '1; + do tick(); while (!observed.input_ready); + restart(); + for (int n = 0; n < WIDTH; n++) send(WIDTH'(1) << n); + drain(); + tick(3); + evidence::check(evidence::cancelled == 1, + "CHI TX reset did not cancel exactly one in-flight flit"); + endtask +endclass +class invalid_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(invalid_sequence) + function new(string name = "invalid_sequence"); + super.new(name); + endfunction + task body(); + cycle_item req; + for (int n = 0; n < DEPTH + 8; n++) begin + req = cycle_item::type_id::create("overflow"); + start_item(req); + req.pins = '0; + req.reset = n < 4; + req.pins.active = n >= 4; + req.pins.credit = n >= 4; + finish_item(req); + end + `uvm_fatal("EXPECTED_ASSERTION", "CHI TX accepted excess credits") + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/tb.sv b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/tb.sv new file mode 100644 index 000000000..4aa0907ba --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/tb.sv @@ -0,0 +1,24 @@ +module tx_tb; + import uvm_pkg::*; + import link_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + link_if bus (clock); + Tx dut ( + .clock(clock), + .reset(bus.reset), + .io_active(bus.pins.active), + .io_in_valid(bus.pins.input_valid), + .io_in_ready(bus.pins.input_ready), + .io_in_bits(bus.pins.input_data), + .io_link_flitpend(bus.pins.pend), + .io_link_flitv(bus.pins.output_valid), + .io_link_flit(bus.pins.output_data), + .io_link_lcrdv(bus.pins.credit), + .io_credits(bus.pins.credits) + ); + initial begin + uvm_config_db#(virtual link_if)::set(null, "uvm_test_top*", "vif", bus); + run_test(); + end +endmodule diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/tests/test.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/tests/test.svh new file mode 100644 index 000000000..38c98c87f --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/tests/test.svh @@ -0,0 +1,31 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + traffic seq = traffic::type_id::create("traffic"); + seq.test_env = test_env; + seq.start(test_env.bus.seqr); + endtask +endclass +class invalid_test extends base_test; + `uvm_component_utils(invalid_test) + agent bus; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + bus = agent::type_id::create("bus", this); + endfunction + task execute(); + invalid_sequence seq = invalid_sequence::type_id::create("violation"); + seq.start(bus.seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/types.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/types.svh new file mode 100644 index 000000000..0f5428c6d --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/types.svh @@ -0,0 +1,9 @@ +`include "parameters.svh" +package link_types; + localparam int WIDTH = `CHI_LINK_BITS, DEPTH = `CHI_CREDITS; + typedef struct packed { + logic active, input_valid, input_ready, output_valid, output_ready, pend, credit; + logic [WIDTH-1:0] input_data, output_data; + logic [`CHI_CREDIT_BITS-1:0] credits; + } link_signals; +endpackage diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/Channel.scala b/arch/src/main/scala/framework/ip/chi/src/main/scala/Channel.scala similarity index 91% rename from arch/src/main/scala/framework/mem-core/chi/src/main/scala/Channel.scala rename to arch/src/main/scala/framework/ip/chi/src/main/scala/Channel.scala index afb6d5fb3..1b5d8703c 100644 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/Channel.scala +++ b/arch/src/main/scala/framework/ip/chi/src/main/scala/Channel.scala @@ -1,4 +1,4 @@ -package memcore.bus.chi +package framework.ip.bus.chi import chisel3._ diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/CoherenceState.scala b/arch/src/main/scala/framework/ip/chi/src/main/scala/CoherenceState.scala similarity index 78% rename from arch/src/main/scala/framework/mem-core/chi/src/main/scala/CoherenceState.scala rename to arch/src/main/scala/framework/ip/chi/src/main/scala/CoherenceState.scala index 282bc251c..1e074f59f 100644 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/CoherenceState.scala +++ b/arch/src/main/scala/framework/ip/chi/src/main/scala/CoherenceState.scala @@ -1,4 +1,4 @@ -package memcore.bus.chi +package framework.ip.bus.chi object CoherenceState { val I = 0 diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/DirectedSnoop.scala b/arch/src/main/scala/framework/ip/chi/src/main/scala/DirectedSnoop.scala similarity index 83% rename from arch/src/main/scala/framework/mem-core/chi/src/main/scala/DirectedSnoop.scala rename to arch/src/main/scala/framework/ip/chi/src/main/scala/DirectedSnoop.scala index 9c75e8d82..dc33d375c 100644 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/DirectedSnoop.scala +++ b/arch/src/main/scala/framework/ip/chi/src/main/scala/DirectedSnoop.scala @@ -1,4 +1,4 @@ -package memcore.bus.chi +package framework.ip.bus.chi import chisel3._ diff --git a/arch/src/main/scala/framework/ip/chi/src/main/scala/Emit.scala b/arch/src/main/scala/framework/ip/chi/src/main/scala/Emit.scala new file mode 100644 index 000000000..34a551407 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/scala/Emit.scala @@ -0,0 +1,79 @@ +package framework.ip.bus.chi + +import java.nio.file.{Files, Paths} +import framework.ip.bus.chi.snf.SramEndpoint +import framework.ip.bus.chi.rnf.{BankedChiCache, RnfParams} + +object Emit extends App { + val Array("--config", config, "--output-dir", output) = args + val values = ujson.read(Files.readString(Paths.get(config))) + val p = Params(values("nodeIdBits").num.toInt, values("addressBits").num.toInt, values("dataBits").num.toInt) + val flitBits = values("linkFlitBits").num.toInt + val credits = values("credits").num.toInt + val node = values("nodeId").num.toInt + val home = values("homeId").num.toInt + val records = values("records").num.toInt + val banks = values("banks").num.toInt + val cacheLines = values("cacheLines").num.toInt + Files.createDirectories(Paths.get(output)) + Files.writeString( + Paths.get(output, "parameters.svh"), + s"""`ifndef CHI_PARAMETERS +`define CHI_PARAMETERS +`define CHI_NODE_BITS ${p.nodeIdBits} +`define CHI_ADDRESS_BITS ${p.addressBits} +`define CHI_DATA_BITS ${p.dataBits} +`define CHI_SRAM_MODULE SramEndpoint${p.dataBits} +`define CHI_REQ_BITS ${(new RequestFlit(p)).flitWidth} +`define CHI_RSP_BITS ${(new ResponseFlit(p)).flitWidth} +`define CHI_SNP_BITS ${(new SnoopFlit(p)).flitWidth} +`define CHI_LINK_BITS $flitBits +`define CHI_CREDITS $credits +`define CHI_CREDIT_BITS ${chisel3.util.log2Ceil(credits + 1)} +`define CHI_NODE_ID $node +`define CHI_HOME_ID $home +`define CHI_RECORDS $records +`define CHI_PENDING_BITS ${chisel3.util.log2Ceil(records + 1)} +`define CHI_BANKS $banks +`define CHI_CACHE_LINES $cacheLines +`define CHI_SRAM_SLOTS ${values("sramSlots").num.toInt} +`define CHI_SRAM_LINES ${values("sramLines").num.toInt} +`endif +""" + ) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Tx(flitBits, credits), + args = Array("--target-dir", s"$output/Tx", "--split-verilog") + ) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Rx(flitBits, credits), + args = Array("--target-dir", s"$output/Rx", "--split-verilog") + ) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new SramEndpoint( + p, + node, + values("sramSlots").num.toInt, + values("sramLines").num.toInt, + values("sramRxDepth").num.toInt + ), + args = Array("--target-dir", s"$output/SramEndpoint", "--split-verilog") + ) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new RequestRetry(p, node, records), + args = Array("--target-dir", s"$output/RequestRetry", "--split-verilog") + ) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new BankedChiCache(RnfParams( + p, + node, + cacheLines, + home, + values("homeCount").num.toInt, + banks, + values("lineResult").bool, + values("probe").bool + )), + args = Array("--target-dir", s"$output/BankedChiCache", "--split-verilog") + ) +} diff --git a/arch/src/main/scala/framework/ip/chi/src/main/scala/Fabric.scala b/arch/src/main/scala/framework/ip/chi/src/main/scala/Fabric.scala new file mode 100644 index 000000000..97f6b35cc --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/scala/Fabric.scala @@ -0,0 +1,104 @@ +package framework.ip.bus.chi + +import framework.lib.arbiter.{RRArbiter} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} + +/** Explicit requester-to-Home CHI fabric; no caches or backing-memory ownership. */ +@instantiable +class Fabric(p: Params, requesterCount: Int, homeId: Int) extends Module { + require(requesterCount >= 1 && requesterCount < homeId && BigInt(homeId) < (BigInt(1) << p.nodeIdBits)) + val c = p + + @public + val io = IO(new Bundle { + val active = Input(Bool()) + val blockRequesterRsp = Input(Bool()) + val requesters = Vec(requesterCount, Flipped(new RequesterPort(c))) + val req = Decoupled(new RequestFlit(c)) + val rxRsp = Decoupled(new ResponseFlit(c)) + val rxDat = Decoupled(new DataFlit(c)) + val rsp = Flipped(Decoupled(new ResponseFlit(c))) + val dat = Flipped(Decoupled(new DataFlit(c))) + val snp = Flipped(Decoupled(new DirectedSnoop(c))) + }) + + def creditLink[T <: Flit](source: DecoupledIO[T], sink: DecoupledIO[T], blockRx: Bool = false.B): Unit = { + // Backpressure retains complete flits in the real credit-accounted Rx FIFO. + // Active and FLITV retain their channel semantics throughout the stall. + val tx: Instance[Tx] = Instantiate(new Tx(source.bits.flitWidth, 4)) + val rx: Instance[Rx] = Instantiate(new Rx(source.bits.flitWidth, 4)) + tx.io.active := io.active + rx.io.active := io.active + tx.io.in.valid := source.valid + tx.io.in.bits := source.bits.packed + source.ready := tx.io.in.ready + rx.io.link <> tx.io.link + sink.valid := rx.io.out.valid && !blockRx + sink.bits.unpack(rx.io.out.bits) + rx.io.out.ready := sink.ready && !blockRx + } + + // All three arbiters use initialized history: the default RR lastGrant is not + // reset, so simultaneous first requests can otherwise select unknown data. + def requesterArbiter[T <: Data](flit: T): Instance[RRArbiter[T]] = Instantiate(new RRArbiter(flit, requesterCount)) + + val requests = requesterArbiter(new RequestFlit(c)) + val responses = requesterArbiter(new ResponseFlit(c)) + val data = requesterArbiter(new DataFlit(c)) + io.req <> requests.io.out + io.rxRsp <> responses.io.out + io.rxDat <> data.io.out + for (i <- 0 until requesterCount) { + creditLink(io.requesters(i).req, requests.io.in(i)) + creditLink(io.requesters(i).txRsp, responses.io.in(i), io.blockRequesterRsp) + creditLink(io.requesters(i).txDat, data.io.in(i)) + when(requests.io.in(i).valid) { + assert( + requests.io.in(i).bits.srcId === (i + 1).U && requests.io.in(i).bits.tgtId === homeId.U, + "CacheSystem request crossed a wrong node or role" + ) + } + when(responses.io.in(i).valid)(assert( + responses.io.in(i).bits.srcId === (i + 1).U && + responses.io.in(i).bits.tgtId === homeId.U, + "CacheSystem response crossed a wrong node or role" + )) + when(data.io.in(i).valid)(assert( + data.io.in(i).bits.srcId === (i + 1).U && + data.io.in(i).bits.tgtId === homeId.U, + "CacheSystem data crossed a wrong node or role" + )) + } + val rsp = Wire(Vec(requesterCount, Decoupled(new ResponseFlit(c)))) + val dat = Wire(Vec(requesterCount, Decoupled(new DataFlit(c)))) + val snp = Wire(Vec(requesterCount, Decoupled(new SnoopFlit(c)))) + for (i <- 0 until requesterCount) { + rsp(i).valid := io.rsp.valid && io.rsp.bits.tgtId === (i + 1).U + rsp(i).bits := io.rsp.bits + dat(i).valid := io.dat.valid && io.dat.bits.tgtId === (i + 1).U + dat(i).bits := io.dat.bits + snp(i).valid := io.snp.valid && io.snp.bits.targetNode === (i + 1).U + snp(i).bits := io.snp.bits.flit + creditLink(rsp(i), io.requesters(i).rxRsp) + creditLink(dat(i), io.requesters(i).rxDat) + creditLink(snp(i), io.requesters(i).snp) + } + io.rsp.ready := (0 until requesterCount).map(i => rsp(i).ready && io.rsp.bits.tgtId === (i + 1).U).reduce(_ || _) + io.dat.ready := (0 until requesterCount).map(i => dat(i).ready && io.dat.bits.tgtId === (i + 1).U).reduce(_ || _) + io.snp.ready := (0 until requesterCount).map(i => snp(i).ready && io.snp.bits.targetNode === (i + 1).U).reduce(_ || _) + when(io.rsp.valid)(assert( + io.rsp.bits.tgtId >= 1.U && io.rsp.bits.tgtId <= requesterCount.U, + "CacheSystem response target is not a requester" + )) + when(io.dat.valid)(assert( + io.dat.bits.tgtId >= 1.U && io.dat.bits.tgtId <= requesterCount.U, + "CacheSystem data target is not a requester" + )) + when(io.snp.valid)(assert( + io.snp.bits.targetNode >= 1.U && io.snp.bits.targetNode <= requesterCount.U, + "CacheSystem snoop target is not a requester" + )) +} diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/Flit.scala b/arch/src/main/scala/framework/ip/chi/src/main/scala/Flit.scala similarity index 97% rename from arch/src/main/scala/framework/mem-core/chi/src/main/scala/Flit.scala rename to arch/src/main/scala/framework/ip/chi/src/main/scala/Flit.scala index c01f5f3a0..fcbe67a58 100644 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/Flit.scala +++ b/arch/src/main/scala/framework/ip/chi/src/main/scala/Flit.scala @@ -1,4 +1,7 @@ -package memcore.bus.chi +package framework.ip.bus.chi + +import buckyball.config.ChiHardware +import upickle.default.{macroRW, ReadWriter} import chisel3._ import chisel3.util.Cat @@ -30,6 +33,12 @@ case class Params( val commonSecIdBits = 0 } +object Params { + implicit val rw: ReadWriter[Params] = macroRW + + def fromProto(p: ChiHardware): Params = Params(p.getNodeIdBits, p.getAddressBits, p.getDataBits) +} + abstract class Flit extends Bundle { def fieldsLSB: Seq[UInt] diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/HomeMapping.scala b/arch/src/main/scala/framework/ip/chi/src/main/scala/HomeMapping.scala similarity index 93% rename from arch/src/main/scala/framework/mem-core/chi/src/main/scala/HomeMapping.scala rename to arch/src/main/scala/framework/ip/chi/src/main/scala/HomeMapping.scala index fc6c8b382..a12c79628 100644 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/HomeMapping.scala +++ b/arch/src/main/scala/framework/ip/chi/src/main/scala/HomeMapping.scala @@ -1,4 +1,4 @@ -package memcore.bus.chi +package framework.ip.bus.chi import chisel3._ import chisel3.util._ diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/RequestRetry.scala b/arch/src/main/scala/framework/ip/chi/src/main/scala/RequestRetry.scala similarity index 95% rename from arch/src/main/scala/framework/mem-core/chi/src/main/scala/RequestRetry.scala rename to arch/src/main/scala/framework/ip/chi/src/main/scala/RequestRetry.scala index 72462ec87..d98795e8c 100644 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/RequestRetry.scala +++ b/arch/src/main/scala/framework/ip/chi/src/main/scala/RequestRetry.scala @@ -1,6 +1,9 @@ -package memcore.bus.chi +package framework.ip.bus.chi -import memcore.memory.queue.Queue +import framework.lib.arbiter.{RRArbiter} +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import framework.lib.queue.Queue import chisel3._ import chisel3.util._ @@ -41,8 +44,8 @@ class RequestRetry(p: Params, nodeId: Int, records: Int = 4) extends Module { def liveTxn(id: UInt) = VecInit((0 until records).map(i => state(i) =/= free && requests(i).txnId === id)) io.pending := PopCount(state.map(_ =/= free)) - val arb = Module(new RRArbiter(new RequestFlit(p), records + 1)) - val output = Module(new Queue(new RequestFlit(p), 2, pipe = true)) + val arb = Instantiate(new RRArbiter(new RequestFlit(p), records + 1)) + val output = Instantiate(new Queue(new RequestFlit(p), 2, true)) output.io.enq <> arb.io.out io.reqOut <> output.io.deq val initial = arb.io.in(0) diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/RequesterPort.scala b/arch/src/main/scala/framework/ip/chi/src/main/scala/RequesterPort.scala similarity index 94% rename from arch/src/main/scala/framework/mem-core/chi/src/main/scala/RequesterPort.scala rename to arch/src/main/scala/framework/ip/chi/src/main/scala/RequesterPort.scala index 1e425f5d3..2cf160e75 100644 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/RequesterPort.scala +++ b/arch/src/main/scala/framework/ip/chi/src/main/scala/RequesterPort.scala @@ -1,4 +1,4 @@ -package memcore.bus.chi +package framework.ip.bus.chi import chisel3._ import chisel3.util._ diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/Rx.scala b/arch/src/main/scala/framework/ip/chi/src/main/scala/Rx.scala similarity index 98% rename from arch/src/main/scala/framework/mem-core/chi/src/main/scala/Rx.scala rename to arch/src/main/scala/framework/ip/chi/src/main/scala/Rx.scala index d67b5cf2a..2ae50ebaa 100644 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/Rx.scala +++ b/arch/src/main/scala/framework/ip/chi/src/main/scala/Rx.scala @@ -1,4 +1,4 @@ -package memcore.bus.chi +package framework.ip.bus.chi import chisel3._ import chisel3.util._ diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/Tx.scala b/arch/src/main/scala/framework/ip/chi/src/main/scala/Tx.scala similarity index 98% rename from arch/src/main/scala/framework/mem-core/chi/src/main/scala/Tx.scala rename to arch/src/main/scala/framework/ip/chi/src/main/scala/Tx.scala index 9b1074a1e..f29e6d972 100644 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/Tx.scala +++ b/arch/src/main/scala/framework/ip/chi/src/main/scala/Tx.scala @@ -1,4 +1,4 @@ -package memcore.bus.chi +package framework.ip.bus.chi import chisel3._ import chisel3.util._ diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/hnf/ChiHome.scala b/arch/src/main/scala/framework/ip/chi/src/main/scala/hnf/ChiHome.scala similarity index 98% rename from arch/src/main/scala/framework/mem-core/chi/src/main/scala/hnf/ChiHome.scala rename to arch/src/main/scala/framework/ip/chi/src/main/scala/hnf/ChiHome.scala index c8d321537..e4820d140 100644 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/hnf/ChiHome.scala +++ b/arch/src/main/scala/framework/ip/chi/src/main/scala/hnf/ChiHome.scala @@ -1,10 +1,10 @@ -package memcore.bus.chi.hnf +package framework.ip.bus.chi.hnf import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public} -import memcore.bus.chi._ -import memcore.bus.chi.snf.{LineRequest, LineResponse} +import framework.ip.bus.chi._ +import framework.ip.bus.chi.snf.{LineRequest, LineResponse} // A finite-address, precise-directory Home. One serialized transaction per Home; // CompAck closes a read grant before another request can observe its directory state. diff --git a/arch/src/main/scala/framework/ip/chi/src/main/scala/rnf/AxiCache.scala b/arch/src/main/scala/framework/ip/chi/src/main/scala/rnf/AxiCache.scala new file mode 100644 index 000000000..d7eaffe66 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/scala/rnf/AxiCache.scala @@ -0,0 +1,131 @@ +package framework.ip.bus.chi.rnf + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import framework.ip.bus.axi4 +import framework.ip.bus.chi.RequesterPort + +/** + * One independent AXI read and write burst, served through a coherent cache. + * AXI beats are ordinary memory accesses; 128-bit writes are not atomic. + */ +@instantiable +class AxiCache(p: axi4.Params, cacheParam: RnfParams) extends Module { + require(p.dataBits == 128 && p.addressBits == cacheParam.chi.addressBits) + require(cacheParam.lineResult && !cacheParam.probe) + + @public + val io = IO(new Bundle { + val axi = Flipped(new axi4.Port(p)) + val chi = new RequesterPort(cacheParam.chi) + }) + + val cache = Instantiate(new BankedChiCache(cacheParam)) + io.chi <> cache.io.chi + + val reading = RegInit(false.B) + val read = Reg(new axi4.Address(p)) + val readBeat = RegInit(0.U(8.W)) + val readValid = RegInit(false.B) + val readData = Reg(UInt(128.W)) + val readError = Reg(Bool()) + io.axi.ar.ready := !reading + when(io.axi.ar.fire) { + assert(io.axi.ar.bits.size <= 4.U && (io.axi.ar.bits.burst === 1.U || io.axi.ar.bits.len === 0.U)) + assert((io.axi.ar.bits.addr & ((1.U(17.W) << io.axi.ar.bits.size) - 1.U)) === 0.U) + assert(!io.axi.ar.bits.lock && io.axi.ar.bits.region === 0.U && io.axi.ar.bits.cache === 15.U) + read := io.axi.ar.bits + reading := true.B + readBeat := 0.U + } + io.axi.r.valid := readValid + io.axi.r.bits.id := read.id + io.axi.r.bits.data := readData + io.axi.r.bits.resp := Mux(readError, 2.U, 0.U) + io.axi.r.bits.last := readBeat === read.len + when(io.axi.r.fire) { + readValid := false.B + read.addr := read.addr + (1.U << read.size) + readBeat := readBeat + 1.U + when(io.axi.r.bits.last)(reading := false.B) + } + + val writing = RegInit(false.B) + val write = Reg(new axi4.Address(p)) + val writeBeat = RegInit(0.U(8.W)) + val writeValid = RegInit(false.B) + val writeData = Reg(UInt(128.W)) + val writeMask = Reg(UInt(16.W)) + val writeHalf = RegInit(false.B) + val writeError = RegInit(false.B) + val responseValid = RegInit(false.B) + io.axi.aw.ready := !writing && !responseValid + when(io.axi.aw.fire) { + assert(io.axi.aw.bits.size <= 4.U && (io.axi.aw.bits.burst === 1.U || io.axi.aw.bits.len === 0.U)) + assert((io.axi.aw.bits.addr & ((1.U(17.W) << io.axi.aw.bits.size) - 1.U)) === 0.U) + assert(!io.axi.aw.bits.lock && io.axi.aw.bits.region === 0.U && io.axi.aw.bits.cache === 15.U) + write := io.axi.aw.bits + writing := true.B + writeBeat := 0.U + writeError := false.B + } + io.axi.w.ready := writing && !writeValid + when(io.axi.w.fire) { + assert(io.axi.w.bits.last === (writeBeat === write.len)) + val allowed = (("hffff".U(16.W) >> (16.U - (1.U << write.size))) << write.addr(3, 0))(15, 0) + assert((io.axi.w.bits.strb & ~allowed) === 0.U) + writeData := io.axi.w.bits.data + writeMask := io.axi.w.bits.strb + writeValid := true.B + writeHalf := false.B + } + io.axi.b.valid := responseValid + io.axi.b.bits.id := write.id + io.axi.b.bits.resp := Mux(writeError, 2.U, 0.U) + when(io.axi.b.fire)(responseValid := false.B) + + val occupied = RegInit(false.B) + val ownerWrite = Reg(Bool()) + val preferWrite = RegInit(false.B) + val wantsRead = reading && !readValid + val wantsWrite = writeValid + val selectWrite = wantsWrite && (!wantsRead || preferWrite) + val mask = Mux(writeHalf, writeMask(15, 8), writeMask(7, 0)) + val skip = !occupied && selectWrite && !mask.orR + cache.io.access.valid := !occupied && (wantsRead || wantsWrite) && !skip + cache.io.access.bits := 0.U.asTypeOf(cache.io.access.bits) + cache.io.access.bits.addr := Mux( + selectWrite, + Cat(write.addr(p.addressBits - 1, 4), writeHalf, 0.U(3.W)), + Cat(read.addr(p.addressBits - 1, 3), 0.U(3.W)) + ) + cache.io.access.bits.write := selectWrite + cache.io.access.bits.data := Mux(writeHalf, writeData(127, 64), writeData(63, 0)) + cache.io.access.bits.mask := Mux(selectWrite, mask, "hff".U) + when(cache.io.access.fire) { + occupied := true.B + ownerWrite := selectWrite + preferWrite := !selectWrite + } + cache.io.result.ready := occupied + when(cache.io.result.fire) { + occupied := false.B + when(ownerWrite) { + writeError := writeError || cache.io.result.bits.error + }.otherwise { + readData := (cache.io.result.bits.line >> (read.addr(5, 4) << 7))(127, 0) + readError := cache.io.result.bits.error + readValid := true.B + } + } + when(skip || (cache.io.result.fire && ownerWrite)) { + when(writeHalf) { + writeValid := false.B + write.addr := write.addr + (1.U << write.size) + writeBeat := writeBeat + 1.U + when(writeBeat === write.len) { writing := false.B; responseValid := true.B } + }.otherwise(writeHalf := true.B) + when(skip)(preferWrite := false.B) + } +} diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/rnf/BankedChiCache.scala b/arch/src/main/scala/framework/ip/chi/src/main/scala/rnf/BankedChiCache.scala similarity index 84% rename from arch/src/main/scala/framework/mem-core/chi/src/main/scala/rnf/BankedChiCache.scala rename to arch/src/main/scala/framework/ip/chi/src/main/scala/rnf/BankedChiCache.scala index b390bf954..6c9bd0134 100644 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/rnf/BankedChiCache.scala +++ b/arch/src/main/scala/framework/ip/chi/src/main/scala/rnf/BankedChiCache.scala @@ -1,9 +1,15 @@ -package memcore.bus.chi.rnf +package framework.ip.bus.chi.rnf + +import framework.lib.arbiter.{RRArbiter} + +import buckyball.config.RnfHardware + +import framework.ip.bus.chi.rnf.RnfParams import chisel3._ import chisel3.util._ -import memcore.bus.chi._ -import memcore.memory.queue.Queue +import framework.ip.bus.chi._ +import framework.lib.queue.Queue import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} // Each bank executes one demand operation while serving snoops independently. @@ -20,8 +26,24 @@ case class RnfParams( val resultLineBits: Int = if (lineResult) 512 else 0 } +object RnfParams { + + def fromProto(p: RnfHardware, requester: Params): RnfParams = RnfParams( + requester, + p.getNodeId, + p.getLines, + p.getHomeId, + p.getHomeCount, + p.getBanks, + p.getLineResult, + p.getProbe + ) + +} + @instantiable -class BankedChiCache(config: RnfParams) extends Module { +class BankedChiCache(b: RnfParams) extends Module { + val config = b val p = config.chi val nodeId = config.nodeId val cacheLines = config.cacheLines @@ -50,7 +72,7 @@ class BankedChiCache(config: RnfParams) extends Module { def bankOf(address: UInt): UInt = if (banks == 1) 0.U(0.W) else address(log2Ceil(banks) + 5, 6) val selected = bankOf(io.access.bits.addr) - val retirement = Module(new Queue(UInt(bankBits.W), banks, pipe = true)) + val retirement = Instantiate(new Queue(UInt(bankBits.W), banks, true)) val serial = io.access.bits.atomic =/= CacheAtomic.None.U val serialActive = RegInit(false.B) val serialSC = RegInit(false.B) @@ -108,39 +130,15 @@ class BankedChiCache(config: RnfParams) extends Module { val completedSC = io.result.fire && serialActive && serialSC for (cache <- caches) { cache.io.dropReservation := newLR || completedSC } - val reqArb = Module(new RRArbiter(new RequestFlit(p), banks) { + val reqArb = Instantiate(new RRArbiter(new RequestFlit(p), banks)) - override lazy val lastGrant = { - val pointer = RegInit(0.U(math.max(1, log2Ceil(banks)).W)) - when(io.out.fire)(pointer := io.chosen) - pointer - } - - }) + val rspArb = Instantiate(new RRArbiter(new ResponseFlit(p), banks)) - val rspArb = Module(new RRArbiter(new ResponseFlit(p), banks) { - - override lazy val lastGrant = { - val pointer = RegInit(0.U(math.max(1, log2Ceil(banks)).W)) - when(io.out.fire)(pointer := io.chosen) - pointer - } - - }) - - val datArb = Module(new RRArbiter(new DataFlit(p), banks) { - - override lazy val lastGrant = { - val pointer = RegInit(0.U(math.max(1, log2Ceil(banks)).W)) - when(io.out.fire)(pointer := io.chosen) - pointer - } - - }) + val datArb = Instantiate(new RRArbiter(new DataFlit(p), banks)) - val reqQueue = Module(new Queue(new RequestFlit(p), 2, pipe = true)) - val rspQueue = Module(new Queue(new ResponseFlit(p), 2, pipe = true)) - val datQueue = Module(new Queue(new DataFlit(p), 2, pipe = true)) + val reqQueue = Instantiate(new Queue(new RequestFlit(p), 2, true)) + val rspQueue = Instantiate(new Queue(new ResponseFlit(p), 2, true)) + val datQueue = Instantiate(new Queue(new DataFlit(p), 2, true)) for (i <- 0 until banks) { reqArb.io.in(i) <> caches(i).io.chi.req rspArb.io.in(i) <> caches(i).io.chi.txRsp diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/rnf/CachePort.scala b/arch/src/main/scala/framework/ip/chi/src/main/scala/rnf/CachePort.scala similarity index 95% rename from arch/src/main/scala/framework/mem-core/chi/src/main/scala/rnf/CachePort.scala rename to arch/src/main/scala/framework/ip/chi/src/main/scala/rnf/CachePort.scala index c1300502a..431694453 100644 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/rnf/CachePort.scala +++ b/arch/src/main/scala/framework/ip/chi/src/main/scala/rnf/CachePort.scala @@ -1,7 +1,7 @@ -package memcore.bus.chi.rnf +package framework.ip.bus.chi.rnf import chisel3._ -import memcore.bus.chi.Params +import framework.ip.bus.chi.Params class CacheAccess(p: Params) extends Bundle { val addr = UInt(p.addressBits.W) diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/rnf/ChiCache.scala b/arch/src/main/scala/framework/ip/chi/src/main/scala/rnf/ChiCache.scala similarity index 99% rename from arch/src/main/scala/framework/mem-core/chi/src/main/scala/rnf/ChiCache.scala rename to arch/src/main/scala/framework/ip/chi/src/main/scala/rnf/ChiCache.scala index e29de59e7..430f49219 100644 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/rnf/ChiCache.scala +++ b/arch/src/main/scala/framework/ip/chi/src/main/scala/rnf/ChiCache.scala @@ -1,8 +1,8 @@ -package memcore.bus.chi.rnf +package framework.ip.bus.chi.rnf import chisel3._ import chisel3.util._ -import memcore.bus.chi._ +import framework.ip.bus.chi._ import chisel3.experimental.hierarchy.{instantiable, public} // Direct-mapped write-back coherent cache agent. The snoop engine is independent diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/snf/MemoryNode.scala b/arch/src/main/scala/framework/ip/chi/src/main/scala/snf/MemoryNode.scala similarity index 92% rename from arch/src/main/scala/framework/mem-core/chi/src/main/scala/snf/MemoryNode.scala rename to arch/src/main/scala/framework/ip/chi/src/main/scala/snf/MemoryNode.scala index 831fa8086..3520be259 100644 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/snf/MemoryNode.scala +++ b/arch/src/main/scala/framework/ip/chi/src/main/scala/snf/MemoryNode.scala @@ -1,9 +1,12 @@ -package memcore.bus.chi.snf +package framework.ip.bus.chi.snf + +import framework.lib.arbiter.{RRArbiter} +import chisel3.experimental.hierarchy.{Instance, Instantiate} import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public} -import memcore.bus.chi._ +import framework.ip.bus.chi._ class LineRequest(p: Params) extends Bundle { val id = UInt(p.txnIdBits.W) @@ -121,15 +124,7 @@ class MemoryNode(p: Params, nodeId: Int = 1, slots: Int = 8) extends Module { } } - val memArb = Module(new RRArbiter(new LineRequest(p), slots) { - - override lazy val lastGrant = { - val pointer = RegInit(0.U(math.max(1, log2Ceil(slots)).W)) - when(this.io.out.fire)(pointer := this.io.chosen) - pointer - } - - }) + val memArb = Instantiate(new RRArbiter(new LineRequest(p), slots)) for (i <- 0 until slots) { memArb.io.in(i).valid := state(i) === issue @@ -154,15 +149,7 @@ class MemoryNode(p: Params, nodeId: Int = 1, slots: Int = 8) extends Module { } } - val rspArb = Module(new RRArbiter(new ResponseFlit(p), slots) { - - override lazy val lastGrant = { - val pointer = RegInit(0.U(math.max(1, log2Ceil(slots)).W)) - when(this.io.out.fire)(pointer := this.io.chosen) - pointer - } - - }) + val rspArb = Instantiate(new RRArbiter(new ResponseFlit(p), slots)) for (i <- 0 until slots) { val r = rspArb.io.in(i) @@ -180,15 +167,7 @@ class MemoryNode(p: Params, nodeId: Int = 1, slots: Int = 8) extends Module { } io.rsp <> rspArb.io.out - val datArb = Module(new RRArbiter(new DataFlit(p), slots) { - - override lazy val lastGrant = { - val pointer = RegInit(0.U(math.max(1, log2Ceil(slots)).W)) - when(this.io.out.fire)(pointer := this.io.chosen) - pointer - } - - }) + val datArb = Instantiate(new RRArbiter(new DataFlit(p), slots)) for (i <- 0 until slots) { val d = datArb.io.in(i) diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/snf/SramEndpoint.scala b/arch/src/main/scala/framework/ip/chi/src/main/scala/snf/SramEndpoint.scala similarity index 97% rename from arch/src/main/scala/framework/mem-core/chi/src/main/scala/snf/SramEndpoint.scala rename to arch/src/main/scala/framework/ip/chi/src/main/scala/snf/SramEndpoint.scala index 20fcc4568..1db3c8811 100644 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/snf/SramEndpoint.scala +++ b/arch/src/main/scala/framework/ip/chi/src/main/scala/snf/SramEndpoint.scala @@ -1,9 +1,9 @@ -package memcore.bus.chi.snf +package framework.ip.bus.chi.snf import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import memcore.bus.chi._ +import framework.ip.bus.chi._ class SnPort(p: Params) extends Bundle { val rxReq = Flipped(new Channel((new RequestFlit(p)).flitWidth)) diff --git a/arch/src/main/scala/framework/ip/coherence/build.sc b/arch/src/main/scala/framework/ip/coherence/build.sc new file mode 100644 index 000000000..4d02e63b0 --- /dev/null +++ b/arch/src/main/scala/framework/ip/coherence/build.sc @@ -0,0 +1,25 @@ +import mill._ +import $file.^.^.^.^.^.^.settings + + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.ip.memory.coherence.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("configs/CoherenceParams.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Coherence.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Directory.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Interface.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Mshr.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/ip/coherence/configs/CoherenceParams.scala b/arch/src/main/scala/framework/ip/coherence/configs/CoherenceParams.scala new file mode 100644 index 000000000..bf78a86cf --- /dev/null +++ b/arch/src/main/scala/framework/ip/coherence/configs/CoherenceParams.scala @@ -0,0 +1,34 @@ +package framework.ip.memory.coherence.configs + +import buckyball.config.HomeHardware + +import chisel3.util.log2Ceil +import framework.ip.bus.chi.Params +import framework.ip.memory.cache.configs.CacheParams + +case class CoherenceParams( + chi: Params, + cache: CacheParams, + agents: Int, + mshrEntries: Int, + homeId: Int) { + require(cache.lineBytes == 64 && cache.addressBits == chi.addressBits) + require(agents >= 1 && agents < homeId && BigInt(homeId) < (BigInt(1) << chi.nodeIdBits)) + require(mshrEntries >= 1 && BigInt(mshrEntries) <= (BigInt(1) << chi.txnIdBits)) + require(BigInt(mshrEntries) <= (BigInt(1) << cache.idBits)) + val slotBits = math.max(1, log2Ceil(mshrEntries)) + val lineAddressBits = chi.addressBits - cache.offsetBits +} + +object CoherenceParams { + + def fromProto(p: HomeHardware): CoherenceParams = + CoherenceParams( + Params.fromProto(p.getRequesters), + CacheParams.fromProto(p.getCache), + p.getAgents, + p.getMshrEntries, + p.getNodeId + ) + +} diff --git a/arch/src/main/scala/framework/mem-core/coherence/configs/default.toml b/arch/src/main/scala/framework/ip/coherence/configs/default.toml similarity index 100% rename from arch/src/main/scala/framework/mem-core/coherence/configs/default.toml rename to arch/src/main/scala/framework/ip/coherence/configs/default.toml diff --git a/arch/src/main/scala/framework/ip/coherence/configs/requesters.toml b/arch/src/main/scala/framework/ip/coherence/configs/requesters.toml new file mode 100644 index 000000000..2c6ca35c4 --- /dev/null +++ b/arch/src/main/scala/framework/ip/coherence/configs/requesters.toml @@ -0,0 +1,3 @@ +nodeIdBits = 7 +addressBits = 44 +dataBits = 256 diff --git a/arch/src/main/scala/framework/ip/coherence/configs/uvm.toml b/arch/src/main/scala/framework/ip/coherence/configs/uvm.toml new file mode 100644 index 000000000..a0b6176d7 --- /dev/null +++ b/arch/src/main/scala/framework/ip/coherence/configs/uvm.toml @@ -0,0 +1,14 @@ +[parameters] +nodeIdBits = 7 +addressBits = 44 +dataBits = 256 +agents = 2 +mshrEntries = 4 +homeId = 64 +cacheSets = 4 +cacheWays = 2 +cacheIdBits = 8 +metadataBits = 4 +responseDepth = 2 + +[defines] diff --git a/arch/src/main/scala/framework/ip/coherence/src/main/resources/filelist.f b/arch/src/main/scala/framework/ip/coherence/src/main/resources/filelist.f new file mode 100644 index 000000000..e3b3935b9 --- /dev/null +++ b/arch/src/main/scala/framework/ip/coherence/src/main/resources/filelist.f @@ -0,0 +1,10 @@ ++incdir+@VERIFY@/uvm/src/core ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/protocol/chi +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/core/stream_if.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/Coherence/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/ip/coherence/src/main/resources/interface.sv b/arch/src/main/scala/framework/ip/coherence/src/main/resources/interface.sv new file mode 100644 index 000000000..113cb1b8f --- /dev/null +++ b/arch/src/main/scala/framework/ip/coherence/src/main/resources/interface.sv @@ -0,0 +1,12 @@ +`include "coherence_config.svh" +interface coherence_control_if ( + input logic clock +); + logic reset; + logic [`COH_OUTSTANDING_BITS-1:0] outstanding; + clocking cb @(posedge clock); + default input #1step output #0; + input outstanding; + output reset; + endclocking +endinterface diff --git a/arch/src/main/scala/framework/ip/coherence/src/main/resources/package.sv b/arch/src/main/scala/framework/ip/coherence/src/main/resources/package.sv new file mode 100644 index 000000000..e18a425a7 --- /dev/null +++ b/arch/src/main/scala/framework/ip/coherence/src/main/resources/package.sv @@ -0,0 +1,5 @@ +`include "coherence_config.svh" +`define CF(K, F) `COH_``K``_``F``_OFFSET+:`COH_``K``_``F``_WIDTH +package coherence_pkg; + `include "coherence/body.svh" +endpackage diff --git a/arch/src/main/scala/framework/ip/coherence/src/main/resources/suite.toml b/arch/src/main/scala/framework/ip/coherence/src/main/resources/suite.toml new file mode 100644 index 000000000..206e15813 --- /dev/null +++ b/arch/src/main/scala/framework/ip/coherence/src/main/resources/suite.toml @@ -0,0 +1,20 @@ +[model] +manifest = "../../../../../../../verify/uvm/src/protocol/chi/coherence/model/Cargo.toml" +library = "libcoherence_reference_model" + +[[tests]] +functional_groups = ["coherence_pkg::system_model::operations", "coherence_pkg::system_model::occupancy"] +name = "coherence" +top = "coherence_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 300000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/mem-core/coherence/src/main/resources/coherence_tb.sv b/arch/src/main/scala/framework/ip/coherence/src/main/resources/tb.sv similarity index 100% rename from arch/src/main/scala/framework/mem-core/coherence/src/main/resources/coherence_tb.sv rename to arch/src/main/scala/framework/ip/coherence/src/main/resources/tb.sv diff --git a/arch/src/main/scala/framework/mem-core/coherence/src/main/scala/Coherence.scala b/arch/src/main/scala/framework/ip/coherence/src/main/scala/Coherence.scala similarity index 89% rename from arch/src/main/scala/framework/mem-core/coherence/src/main/scala/Coherence.scala rename to arch/src/main/scala/framework/ip/coherence/src/main/scala/Coherence.scala index af313240e..7fd78048f 100644 --- a/arch/src/main/scala/framework/mem-core/coherence/src/main/scala/Coherence.scala +++ b/arch/src/main/scala/framework/ip/coherence/src/main/scala/Coherence.scala @@ -1,14 +1,16 @@ -package memcore.memory.coherence +package framework.ip.memory.coherence -import memcore.memory.queue.Queue +import framework.lib.arbiter.{RRArbiter} + +import framework.lib.queue.Queue import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import memcore.bus.chi._ -import memcore.bus.chi.snf.LineRequest -import memcore.memory.cache.{Cache, CacheOp, CacheRequest} -import memcore.memory.coherence.configs.CoherenceParams +import framework.ip.bus.chi._ +import framework.ip.bus.chi.snf.LineRequest +import framework.ip.memory.cache.{Cache, CacheOp, CacheRequest} +import framework.ip.memory.coherence.configs.CoherenceParams @instantiable class Coherence(p: CoherenceParams) extends Module { @@ -71,64 +73,59 @@ class Coherence(p: CoherenceParams) extends Module { io.req.ready := mshr.io.allocate.ready io.outstanding := mshr.io.outstanding when(io.req.fire) { - val r = io.req.bits - val read = + val r = io.req.bits + val read = r.opcode === Opcode.ReadShared.U || r.opcode === Opcode.ReadNotSharedDirty.U || r.opcode === Opcode.ReadUnique.U + val permission = r.opcode === Opcode.MakeUnique.U assert(r.tgtId === p.homeId.U && r.srcId >= 1.U && r.srcId <= p.agents.U, "Unknown CPU or Home") assert( - read || r.opcode === Opcode.Evict.U || r.opcode === Opcode.WriteBackFull.U || + read || permission || r.opcode === Opcode.Evict.U || r.opcode === Opcode.WriteBackFull.U || r.opcode === Opcode.CleanShared.U || r.opcode === Opcode.CleanInvalid.U || r.opcode === Opcode.MakeInvalid.U, "Unsupported coherence operation" ) assert( r.size === 6.U && r.addr(5, 0) === 0.U && r.order === 0.U && r.pas === 0.U && - r.returnNid === 0.U && r.returnTxnId === 0.U && r.memAttr === "b1100".U && r.snpAttr === 1.U && + r.returnNid === 0.U && r.returnTxnId === 0.U && r.memAttr(2) && !r.memAttr(1) && r.snpAttr === 1.U && r.allowRetry === 1.U && r.exclSnoopMe === 0.U && r.pCrdType === 0.U && r.stashNidValidEndian === 0.U && r.multiReq === 0.U && r.tagOp === 0.U, "Unsupported coherent access attributes" ) - assert(r.expCompAck === read.asUInt, "Coherent read requires CompAck") - val i = mshr.io.allocatedId + assert(r.expCompAck === (read || permission).asUInt, "Coherent grants require CompAck") + val i = mshr.io.allocatedId state(i) := lookupReq failed(i) := false.B sent(i) := 0.U seen(i) := 0.U } - private def arbitrate[T <: Data](gen: T): RRArbiter[T] = Module(new RRArbiter(gen, n) { - - override lazy val lastGrant = { - val pointer = RegInit(0.U(math.max(1, log2Ceil(this.n)).W)) - when(this.io.out.fire)(pointer := this.io.chosen) - pointer - } - - }) + private def arbitrate[T <: Data](gen: T): Instance[RRArbiter[T]] = Instantiate(new RRArbiter(gen, n)) val cacheArb = arbitrate(new CacheRequest(p.cache)) - val cacheQueue = Module(new Queue(new CacheRequest(p.cache), 2, pipe = true)) + val cacheQueue = Instantiate(new Queue(new CacheRequest(p.cache), 2, true)) cacheQueue.io.enq <> cacheArb.io.out cache.io.request <> cacheQueue.io.deq cache.io.response.ready := true.B val memArb = arbitrate(new LineRequest(c)) - val memQueue = Module(new Queue(new LineRequest(c), 2, pipe = true)) + val memQueue = Instantiate(new Queue(new LineRequest(c), 2, true)) memQueue.io.enq <> memArb.io.out io.memoryReq <> memQueue.io.deq io.memoryResp.ready := true.B val snpArb = arbitrate(new DirectedSnoop(c)) - val snpQueue = Module(new Queue(new DirectedSnoop(c), 2, pipe = true)) + val snpQueue = Instantiate(new Queue(new DirectedSnoop(c), 2, true)) snpQueue.io.enq <> snpArb.io.out io.snp <> snpQueue.io.deq val rspArb = arbitrate(new ResponseFlit(c)) - val rspQueue = Module(new Queue(new ResponseFlit(c), 2, pipe = true)) + val rspQueue = Instantiate(new Queue(new ResponseFlit(c), 2, true)) rspQueue.io.enq <> rspArb.io.out io.rsp <> rspQueue.io.deq val datArb = arbitrate(new DataFlit(c)) - val datQueue = Module(new Queue(new DataFlit(c), 2, pipe = true)) + val datQueue = Instantiate(new Queue(new DataFlit(c), 2, true)) datQueue.io.enq <> datArb.io.out io.dat <> datQueue.io.deq // Caller TxnID ends at its final response. The slot/DBID/set stays owned until full protocol completion. mshr.io.releaseCaller := VecInit((0 until n).map { i => (io.rsp.fire && io.rsp.bits.opcode === Opcode.CompDBIDResp.U && io.rsp.bits.dbid === i.U) || + (io.rsp.fire && io.rsp.bits.opcode === Opcode.Comp.U && io.rsp.bits.dbid === i.U && + entries(i).request.opcode === Opcode.MakeUnique.U) || (io.dat.fire && io.dat.bits.opcode === Opcode.CompData.U && io.dat.bits.dbid === i.U && io.dat.bits.dataId === ((c.beatsPerLine - 1) * c.dataBits / 128).U) }).asUInt @@ -158,7 +155,9 @@ class Coherence(p: CoherenceParams) extends Module { update.bits.address := directory.io.read(i) update.bits.entry := directory.io.entries(i) val mask = UIntToOH(r.srcId - 1.U, p.agents) - val exclusive = r.opcode === Opcode.ReadUnique.U + val permission = r.opcode === Opcode.MakeUnique.U + val exclusive = r.opcode === Opcode.ReadUnique.U || permission + val grant = Mux(permission, complete, grantData) val cleanShared = r.opcode === Opcode.CleanShared.U val cleanInvalid = r.opcode === Opcode.CleanInvalid.U val makeInvalid = r.opcode === Opcode.MakeInvalid.U @@ -218,7 +217,7 @@ class Coherence(p: CoherenceParams) extends Module { .otherwise { pending(i) := Mux( !result.hit || maintenance || exclusive || owned, - holders, + holders & Mux(permission && result.hit, ~mask, Fill(p.agents, 1.U(1.W))), 0.U ) state(i) := chooseSnoop @@ -233,14 +232,14 @@ class Coherence(p: CoherenceParams) extends Module { update.bits.entry.sharers := 0.U update.bits.entry.unique := false.B sent(i) := 0.U - state(i) := grantData + state(i) := grant }.elsewhen(state(i) === restoreWait) { sent(i) := 0.U - state(i) := Mux(maintenance, complete, grantData) + state(i) := Mux(maintenance, complete, grant) }.otherwise { assert(state(i) === updateWait, "Unexpected Cache response phase") sent(i) := 0.U - state(i) := Mux(writeback, finished, Mux(maintenance, complete, grantData)) + state(i) := Mux(writeback, finished, Mux(maintenance, complete, grant)) } } @@ -377,7 +376,7 @@ class Coherence(p: CoherenceParams) extends Module { failed(i) := io.memoryResp.bits.error payload(i) := io.memoryResp.bits.data.asTypeOf(payload(i)) sent(i) := 0.U - state(i) := Mux(io.memoryResp.bits.error, grantData, fillReq) + state(i) := Mux(io.memoryResp.bits.error, grant, fillReq) }.otherwise { when(io.memoryResp.bits.error) { failed(i) := true.B @@ -398,10 +397,16 @@ class Coherence(p: CoherenceParams) extends Module { rsp.bits.txnId := r.txnId rsp.bits.dbid := i.U rsp.bits.respErr := Mux(failed(i), 3.U, 0.U) + rsp.bits.resp := Mux(permission && !failed(i), CoherenceState.UC.U, CoherenceState.I.U) rsp.bits.opcode := Mux(state(i) === wbGrant, Opcode.CompDBIDResp.U, Opcode.Comp.U) when(rsp.fire) { seen(i) := 0.U - state(i) := Mux(state(i) === wbGrant, wbData, finished) + state(i) := Mux(state(i) === wbGrant, wbData, Mux(permission, waitAck, finished)) + when(permission && !failed(i)) { + update.valid := true.B + update.bits.entry.sharers := mask + update.bits.entry.unique := true.B + } } val dat = datArb.io.in(i) dat.valid := active && state(i) === grantData diff --git a/arch/src/main/scala/framework/mem-core/coherence/src/main/scala/Directory.scala b/arch/src/main/scala/framework/ip/coherence/src/main/scala/Directory.scala similarity index 94% rename from arch/src/main/scala/framework/mem-core/coherence/src/main/scala/Directory.scala rename to arch/src/main/scala/framework/ip/coherence/src/main/scala/Directory.scala index 7b2b94d86..3540e2e40 100644 --- a/arch/src/main/scala/framework/mem-core/coherence/src/main/scala/Directory.scala +++ b/arch/src/main/scala/framework/ip/coherence/src/main/scala/Directory.scala @@ -1,9 +1,9 @@ -package memcore.memory.coherence +package framework.ip.memory.coherence import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public} -import memcore.memory.coherence.configs.CoherenceParams +import framework.ip.memory.coherence.configs.CoherenceParams class DirectoryEntry(p: CoherenceParams) extends Bundle { val sharers = UInt(p.agents.W) diff --git a/arch/src/main/scala/framework/ip/coherence/src/main/scala/Emit.scala b/arch/src/main/scala/framework/ip/coherence/src/main/scala/Emit.scala new file mode 100644 index 000000000..f0b0bcb8c --- /dev/null +++ b/arch/src/main/scala/framework/ip/coherence/src/main/scala/Emit.scala @@ -0,0 +1,127 @@ +package framework.ip.memory.coherence + +import java.nio.file.{Files, Paths} +import framework.ip.bus.chi._ +import framework.ip.memory.coherence.configs.CoherenceParams + +object Emit extends App { + val Array("--config", config, "--output-dir", directory) = args + val values = ujson.read(Files.readString(Paths.get(config))) + val output = Paths.get(directory) + Files.createDirectories(output) + val chi = Params(values("nodeIdBits").num.toInt, values("addressBits").num.toInt, values("dataBits").num.toInt) + + val cache = framework.ip.memory.cache.configs.CacheParams( + chi.addressBits, + 64, + values("cacheSets").num.toInt, + values("cacheWays").num.toInt, + values("cacheIdBits").num.toInt, + values("metadataBits").num.toInt, + values("responseDepth").num.toInt + ) + + val p = + CoherenceParams(chi, cache, values("agents").num.toInt, values("mshrEntries").num.toInt, values("homeId").num.toInt) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Coherence(p), + args = Array("--target-dir", output.resolve("Coherence").toString, "--split-verilog") + ) + val definitions = scala.collection.mutable.ArrayBuffer[String]() + val ports = scala.collection.mutable.ArrayBuffer[String]() + + def fields(flit: Flit): Seq[(String, Int)] = flit.fieldsLSB.map { field => + flit.elements.find(_._2 eq field).get._1 -> field.getWidth + } + + def channel( + kind: String, + prefix: String, + instance: String, + fs: Seq[(String, Int)] + ): Unit = { + var offset = 0 + for ((name, width) <- fs if width > 0) { + definitions += s"`define COH_${kind}_${name.toUpperCase}_OFFSET $offset" + definitions += s"`define COH_${kind}_${name.toUpperCase}_WIDTH $width" + ports += s".$prefix$name($instance.bits[$offset +: $width])" + offset += width + } + definitions += s"`define COH_${kind}_WIDTH $offset" + } + + channel("REQ", "io_req_bits_", "source_if", fields(new RequestFlit(p.chi))) + channel("RSP", "io_rxRsp_bits_", "rx_rsp", fields(new ResponseFlit(p.chi))) + channel("DAT", "io_rxDat_bits_", "rx_dat", fields(new DataFlit(p.chi))) + channel("TXRSP", "io_rsp_bits_", "tx_rsp", fields(new ResponseFlit(p.chi))) + channel("TXDAT", "io_dat_bits_", "sink_if", fields(new DataFlit(p.chi))) + channel("SNP", "io_snp_bits_flit_", "tx_snp", fields(new SnoopFlit(p.chi))) + val snpWidth = (new SnoopFlit(p.chi)).flitWidth + ports += s".io_snp_bits_targetNode(tx_snp.bits[$snpWidth +: ${p.chi.nodeIdBits}])" + definitions += s"`define COH_SNP_TARGET_OFFSET $snpWidth" + definitions += s"`define COH_SNP_TARGET_WIDTH ${p.chi.nodeIdBits}" + definitions += s"`define COH_SNP_CHANNEL_WIDTH ${snpWidth + p.chi.nodeIdBits}" + channel( + "MEMREQ", + "io_memoryReq_bits_", + "mem_req", + Seq("id" -> p.chi.txnIdBits, "addr" -> p.chi.addressBits, "write" -> 1, "data" -> 512, "mask" -> 64) + ) + channel("MEMRESP", "io_memoryResp_bits_", "mem_resp", Seq("id" -> p.chi.txnIdBits, "data" -> 512, "error" -> 1)) + for ( + (port, instance) <- Seq( + "req" -> "source_if", + "rxRsp" -> "rx_rsp", + "rxDat" -> "rx_dat", + "rsp" -> "tx_rsp", + "dat" -> "sink_if", + "snp" -> "tx_snp", + "memoryReq" -> "mem_req", + "memoryResp" -> "mem_resp" + ) + ) { + ports += s".io_${port}_valid($instance.valid)" + ports += s".io_${port}_ready($instance.ready)" + } + for ( + (name, value) <- Seq( + "AGENTS" -> p.agents, + "MSHRS" -> p.mshrEntries, + "HOME" -> p.homeId, + "SETS" -> p.cache.sets, + "WAYS" -> p.cache.ways, + "BEATS" -> p.chi.beatsPerLine, + "DATA_BITS" -> p.chi.dataBits, + "OUTSTANDING_BITS" -> chisel3.util.log2Ceil(p.mshrEntries + 1), + "READ_SHARED" -> Opcode.ReadShared, + "READ_NSD" -> Opcode.ReadNotSharedDirty, + "READ_UNIQUE" -> Opcode.ReadUnique, + "MAKE_UNIQUE" -> Opcode.MakeUnique, + "EVICT" -> Opcode.Evict, + "WRITEBACK" -> Opcode.WriteBackFull, + "CLEAN_SHARED" -> Opcode.CleanShared, + "CLEAN_INVALID" -> Opcode.CleanInvalid, + "MAKE_INVALID" -> Opcode.MakeInvalid, + "COMP" -> Opcode.Comp, + "COMP_DBID" -> Opcode.CompDBIDResp, + "COMP_ACK" -> Opcode.CompAck, + "COMP_DATA" -> Opcode.CompData, + "SNP_RESP" -> Opcode.SnpResp, + "SNP_DATA" -> Opcode.SnpRespData, + "COPYBACK_DATA" -> Opcode.CopyBackWriteData, + "SNP_UNIQUE" -> Opcode.SnpUnique, + "SNP_INVALID" -> Opcode.SnpCleanInvalid, + "SNP_CLEAN_SHARED" -> Opcode.SnpCleanShared, + "SNP_MAKE_INVALID" -> Opcode.SnpMakeInvalid, + "SNP_SHARED" -> Opcode.SnpNotSharedDirty + ) + ) { + definitions += s"`define COH_$name $value" + } + Files.writeString( + output.resolve("coherence_config.svh"), + "`ifndef COHERENCE_CONFIG_SVH\n`define COHERENCE_CONFIG_SVH\n" + definitions.mkString("\n") + "\n`endif\n" + ) + Files.writeString(output.resolve("coherence_ports.svh"), ports.mkString(",\n") + "\n") + +} diff --git a/arch/src/main/scala/framework/ip/coherence/src/main/scala/Interface.scala b/arch/src/main/scala/framework/ip/coherence/src/main/scala/Interface.scala new file mode 100644 index 000000000..2c850bb55 --- /dev/null +++ b/arch/src/main/scala/framework/ip/coherence/src/main/scala/Interface.scala @@ -0,0 +1,19 @@ +package framework.ip.memory.coherence + +import chisel3._ +import chisel3.util._ +import framework.ip.bus.chi._ +import framework.ip.bus.chi.snf.{LineRequest, LineResponse} +import framework.ip.memory.coherence.configs.CoherenceParams + +class CoherenceIO(p: CoherenceParams) extends Bundle { + val req = Flipped(Decoupled(new RequestFlit(p.chi))) + val rxRsp = Flipped(Decoupled(new ResponseFlit(p.chi))) + val rxDat = Flipped(Decoupled(new DataFlit(p.chi))) + val rsp = Decoupled(new ResponseFlit(p.chi)) + val dat = Decoupled(new DataFlit(p.chi)) + val snp = Decoupled(new DirectedSnoop(p.chi)) + val memoryReq = Decoupled(new LineRequest(p.chi)) + val memoryResp = Flipped(Decoupled(new LineResponse(p.chi))) + val outstanding = Output(UInt(log2Ceil(p.mshrEntries + 1).W)) +} diff --git a/arch/src/main/scala/framework/mem-core/coherence/src/main/scala/Mshr.scala b/arch/src/main/scala/framework/ip/coherence/src/main/scala/Mshr.scala similarity index 94% rename from arch/src/main/scala/framework/mem-core/coherence/src/main/scala/Mshr.scala rename to arch/src/main/scala/framework/ip/coherence/src/main/scala/Mshr.scala index e55193839..fca25a7ed 100644 --- a/arch/src/main/scala/framework/mem-core/coherence/src/main/scala/Mshr.scala +++ b/arch/src/main/scala/framework/ip/coherence/src/main/scala/Mshr.scala @@ -1,10 +1,10 @@ -package memcore.memory.coherence +package framework.ip.memory.coherence import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public} -import memcore.bus.chi.RequestFlit -import memcore.memory.coherence.configs.CoherenceParams +import framework.ip.bus.chi.RequestFlit +import framework.ip.memory.coherence.configs.CoherenceParams class MshrAllocation(p: CoherenceParams) extends Bundle { val request = new RequestFlit(p.chi) diff --git a/arch/src/main/scala/framework/ip/interlock/build.sc b/arch/src/main/scala/framework/ip/interlock/build.sc new file mode 100644 index 000000000..07f0a08c6 --- /dev/null +++ b/arch/src/main/scala/framework/ip/interlock/build.sc @@ -0,0 +1,23 @@ +import mill._ +import $file.^.^.^.^.^.^.settings + + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.ip.memory.interlock.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Interface.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Interlock.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/ip/interlock/configs/uvm.toml b/arch/src/main/scala/framework/ip/interlock/configs/uvm.toml new file mode 100644 index 000000000..9896ccd3d --- /dev/null +++ b/arch/src/main/scala/framework/ip/interlock/configs/uvm.toml @@ -0,0 +1,8 @@ +[parameters] +entries = 4 +addressBits = 44 +idBits = 8 +lineBytes = 64 +maxRanges = 8 + +[defines] diff --git a/arch/src/main/scala/framework/ip/interlock/src/main/resources/filelist.f b/arch/src/main/scala/framework/ip/interlock/src/main/resources/filelist.f new file mode 100644 index 000000000..643c739e2 --- /dev/null +++ b/arch/src/main/scala/framework/ip/interlock/src/main/resources/filelist.f @@ -0,0 +1,9 @@ ++incdir+@RTL@ ++incdir+@VERIFY@/uvm/src/protocol/interlock ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/protocol/interlock/types.svh +@VERIFY@/uvm/src/protocol/interlock/interface.sv +@VERIFY@/uvm/src/protocol/interlock/package.sv +-F @RTL@/Interlock/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/ip/interlock/src/main/resources/suite.toml b/arch/src/main/scala/framework/ip/interlock/src/main/resources/suite.toml new file mode 100644 index 000000000..5864a21c0 --- /dev/null +++ b/arch/src/main/scala/framework/ip/interlock/src/main/resources/suite.toml @@ -0,0 +1,343 @@ +[model] +manifest = "../../../../../../../verify/uvm/src/protocol/interlock/model/Cargo.toml" +library = "libinterlock_reference" + +[[tests]] +functional_groups = ["interlock_pkg::ownership_model::activity", "interlock_pkg::ownership_model::maintenance_kind", "interlock_pkg::ownership_model::probes"] +name = "interlock" +top = "interlock_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +functional_groups = [] +name = "interlock_bad_cancel_info" +top = "interlock_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=cancel_info"] +assertion = "Interlock cancel conflicts with the same command" + +[[tests]] +functional_groups = [] +name = "interlock_bad_cancel_sealed" +top = "interlock_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=cancel_sealed"] +assertion = "Interlock cancel requires an unsealed reservation" + +[[tests]] +functional_groups = [] +name = "interlock_bad_cancel_unknown" +top = "interlock_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=cancel_unknown"] +assertion = "Interlock cancel ID has no reservation" + +[[tests]] +functional_groups = [] +name = "interlock_bad_capacity" +top = "interlock_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=capacity"] +assertion = "Interlock range list must seal within capacity" + +[[tests]] +functional_groups = [] +name = "interlock_bad_cpu_alignment" +top = "interlock_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=cpu_alignment"] +assertion = "Interlock CPU access must be naturally aligned" + +[[tests]] +functional_groups = [] +name = "interlock_bad_cpu_size" +top = "interlock_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=cpu_size"] +assertion = "Interlock CPU size must be 1, 2, 4 or 8 bytes" + +[[tests]] +functional_groups = [] +name = "interlock_bad_direction" +top = "interlock_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=direction"] +assertion = "Interlock ranges must have one access direction" + +[[tests]] +functional_groups = [] +name = "interlock_bad_dma_failed" +top = "interlock_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=dma_failed"] +assertion = "Interlock DMA failed" + +[[tests]] +functional_groups = [] +name = "interlock_bad_dma_ungranted" +top = "interlock_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=dma_ungranted"] +assertion = "Interlock DMA done requires a granted access" + +[[tests]] +functional_groups = [] +name = "interlock_bad_dma_unknown" +top = "interlock_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=dma_unknown"] +assertion = "Interlock DMA done tag has no reservation" + +[[tests]] +functional_groups = [] +name = "interlock_bad_duplicate" +top = "interlock_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=duplicate"] +assertion = "Interlock dispatch ID is already live" + +[[tests]] +functional_groups = [] +name = "interlock_bad_empty_range" +top = "interlock_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=empty_range"] +assertion = "Interlock memory range is empty" + +[[tests]] +functional_groups = [] +name = "interlock_bad_info_duplicate" +top = "interlock_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=info_duplicate"] +assertion = "Interlock access info is already declared" + +[[tests]] +functional_groups = [] +name = "interlock_bad_info_unknown" +top = "interlock_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=info_unknown"] +assertion = "Interlock access info ID has no reservation" + +[[tests]] +functional_groups = [] +name = "interlock_bad_maint_failed" +top = "interlock_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=maint_failed"] +assertion = "Interlock maintenance failed" + +[[tests]] +functional_groups = [] +name = "interlock_bad_maint_tag" +top = "interlock_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=maint_tag"] +assertion = "Interlock maintenance response tag mismatch" + +[[tests]] +functional_groups = [] +name = "interlock_bad_maint_unknown" +top = "interlock_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=maint_unknown"] +assertion = "Interlock maintenance response has no request" + +[[tests]] +functional_groups = [] +name = "interlock_bad_no_memory_seal" +top = "interlock_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=no_memory_seal"] +assertion = "Interlock no-memory declaration must seal an empty command" + +[[tests]] +functional_groups = [] +name = "interlock_bad_overflow_range" +top = "interlock_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=overflow_range"] +assertion = "Interlock memory range exceeds physical width" diff --git a/arch/src/main/scala/framework/ip/interlock/src/main/resources/tb.sv b/arch/src/main/scala/framework/ip/interlock/src/main/resources/tb.sv new file mode 100644 index 000000000..33d8a66c0 --- /dev/null +++ b/arch/src/main/scala/framework/ip/interlock/src/main/resources/tb.sv @@ -0,0 +1,16 @@ +module interlock_tb; + import uvm_pkg::*; + import interlock_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + interlock_if bus (clock); + Interlock dut ( + .clock(clock), + .reset(bus.reset), + `include "ports.svh" + ); + initial begin + uvm_config_db#(virtual interlock_if)::set(null, "uvm_test_top.*", "vif", bus); + run_test(); + end +endmodule diff --git a/arch/src/main/scala/framework/ip/interlock/src/main/scala/Emit.scala b/arch/src/main/scala/framework/ip/interlock/src/main/scala/Emit.scala new file mode 100644 index 000000000..7a0554b66 --- /dev/null +++ b/arch/src/main/scala/framework/ip/interlock/src/main/scala/Emit.scala @@ -0,0 +1,70 @@ +package framework.ip.memory.interlock + +import chisel3._ +import java.nio.file.{Files, Paths} + +object Emit extends App { + val Array("--config", config, "--output-dir", output) = args + val values = ujson.read(Files.readString(Paths.get(config))) + + val p = Params( + values("entries").num.toInt, + values("addressBits").num.toInt, + values("idBits").num.toInt, + values("lineBytes").num.toInt, + values("maxRanges").num.toInt + ) + + Files.createDirectories(Paths.get(output)) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Interlock(p), + args = Array("--target-dir", s"$output/Interlock", "--split-verilog") + ) + + val definitions = scala.collection.mutable.ArrayBuffer[String]( + s"`define INTERLOCK_ENTRIES ${p.entries}", + s"`define INTERLOCK_ADDRESS_BITS ${p.addressBits}", + s"`define INTERLOCK_ID_BITS ${p.idBits}", + s"`define INTERLOCK_LINE_BYTES ${p.lineBytes}", + s"`define INTERLOCK_MAX_RANGES ${p.maxRanges}" + ) + + val ports = scala.collection.mutable.ArrayBuffer[String]() + + def channel( + kind: String, + prefix: String, + signal: String, + data: Bundle, + decoupled: Boolean = true + ): Unit = { + var offset = 0 + for ((name, field) <- data.elements.toSeq) { + val width = field.getWidth + definitions += s"`define INTERLOCK_${kind}_${name.toUpperCase}_OFFSET $offset" + definitions += s"`define INTERLOCK_${kind}_${name.toUpperCase}_WIDTH $width" + ports += s".$prefix$name(bus.signals.${signal}_bits[$offset +: $width])" + offset += width + } + definitions += s"`define INTERLOCK_${kind}_WIDTH $offset" + if (decoupled) { + val channelPrefix = prefix.stripSuffix("bits_") + ports += s".${channelPrefix}valid(bus.signals.${signal}_valid)" + ports += s".${channelPrefix}ready(bus.signals.${signal}_ready)" + } + } + + channel("DISPATCH", "io_dispatch_bits_", "dispatch", new Dispatch(p)) + channel("CANCEL", "io_cancel_bits_", "cancel", new Dispatch(p)) + channel("ACCESS_INFO", "io_accessInfo_bits_", "access_info", new AccessInfo(p)) + channel("MAINTENANCE", "io_maintenance_bits_", "maintenance", new Maintenance(p)) + channel("MAINTAINED", "io_maintained_bits_", "maintained", new Acknowledgement(p)) + channel("GRANT", "io_grant_bits_", "grant", new Tag(p)) + channel("DONE", "io_done_bits_", "done", new Acknowledgement(p)) + channel("COMPLETE", "io_complete_bits_", "complete", new Tag(p)) + channel("CPU_QUERY", "io_cpuQuery_", "cpu_query", new CpuQuery(p), decoupled = false) + ports += ".io_cpuAllow(bus.signals.cpu_allow)" + ports += ".io_cpuProbeAllow(bus.signals.cpu_probe_allow)" + Files.write(Paths.get(output, "parameters.svh"), (definitions.mkString("\n") + "\n").getBytes) + Files.write(Paths.get(output, "ports.svh"), ports.mkString(",\n").getBytes) +} diff --git a/arch/src/main/scala/framework/ip/interlock/src/main/scala/Interface.scala b/arch/src/main/scala/framework/ip/interlock/src/main/scala/Interface.scala new file mode 100644 index 000000000..9d3c8fc0c --- /dev/null +++ b/arch/src/main/scala/framework/ip/interlock/src/main/scala/Interface.scala @@ -0,0 +1,38 @@ +package framework.ip.memory.interlock + +import chisel3._ + +class Dispatch(p: Params) extends Bundle { val id = UInt(p.idBits.W) } + +class AccessInfo(p: Params) extends Bundle { + val id = UInt(p.idBits.W) + val hasMemory = Bool() + val base = UInt(p.addressBits.W) + val bytes = UInt((p.addressBits + 1).W) + val write = Bool() + val last = Bool() +} + +class CpuQuery(p: Params) extends Bundle { + val valid = Bool() + val paddr = UInt(p.addressBits.W) + val sizeLog2 = UInt(3.W) + val write = Bool() + val olderDispatchPending = Bool() +} + +object MaintenanceOp { val Clean = 0; val CleanInvalidate = 1; val Invalidate = 2 } + +class Maintenance(p: Params) extends Bundle { + val tag = UInt(p.idBits.W) + val firstLine = UInt(p.addressBits.W) + val lastLine = UInt(p.addressBits.W) + val op = UInt(2.W) +} + +class Tag(p: Params) extends Bundle { val tag = UInt(p.idBits.W) } + +class Acknowledgement(p: Params) extends Bundle { + val tag = UInt(p.idBits.W) + val ok = Bool() +} diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/scala/Interlock.scala b/arch/src/main/scala/framework/ip/interlock/src/main/scala/Interlock.scala similarity index 96% rename from arch/src/main/scala/framework/mem-core/interlock/src/main/scala/Interlock.scala rename to arch/src/main/scala/framework/ip/interlock/src/main/scala/Interlock.scala index 2dad4115c..d8722c9ec 100644 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/scala/Interlock.scala +++ b/arch/src/main/scala/framework/ip/interlock/src/main/scala/Interlock.scala @@ -1,4 +1,4 @@ -package memcore.memory.interlock +package framework.ip.memory.interlock import chisel3._ import chisel3.util._ @@ -16,7 +16,7 @@ class Interlock(p: Params) extends Module { val cpuQuery = Input(new CpuQuery(p)) val cpuAllow = Output(Bool()) // cpuAllow without the same-cycle dispatch term, for a CPU access that cannot share its cycle - // with a dispatch (the access and the dispatching command occupy the same core stage). + // with a dispatch (the access and the dispatching command occupy the same thread stage). val cpuProbeAllow = Output(Bool()) val maintenance = Decoupled(new Maintenance(p)) val maintained = Flipped(Decoupled(new Acknowledgement(p))) @@ -47,7 +47,8 @@ class Interlock(p: Params) extends Module { MuxCase( 0.U(indexBits.W), (0 until p.entries).map { offset => - val index = (cursor + offset.U)(indexBits - 1, 0) + val sum = cursor +& offset.U + val index = Mux(sum >= p.entries.U, sum - p.entries.U, sum)(indexBits - 1, 0) candidates(index) -> index } ) @@ -179,11 +180,9 @@ class Interlock(p: Params) extends Module { maintenancePayload.tag := ids(index) maintenancePayload.firstLine := rangeFirst(index)(rangeCursor(index)) maintenancePayload.lastLine := rangeLast(index)(rangeCursor(index)) - maintenancePayload.op := Mux( - phase(index) === postReady, - MaintenanceOp.Invalidate.U, - Mux(writes(index), MaintenanceOp.CleanInvalidate.U, MaintenanceOp.Clean.U) - ) + // DMA may leave the newest bytes dirty in a coherent requester. Preserve them + // before releasing the range; discard-only invalidation would lose the write. + maintenancePayload.op := Mux(writes(index), MaintenanceOp.CleanInvalidate.U, MaintenanceOp.Clean.U) } io.maintenance.valid := maintenanceOffer && !protocolError io.maintenance.bits := maintenancePayload @@ -192,7 +191,7 @@ class Interlock(p: Params) extends Module { maintenanceBusy := true.B maintenanceOwner := maintenanceIndex maintenanceCursor := maintenanceIndex + 1.U - phase(maintenanceIndex) := Mux(maintenancePayload.op === MaintenanceOp.Invalidate.U, postWait, preWait) + phase(maintenanceIndex) := Mux(phase(maintenanceIndex) === postReady, postWait, preWait) } io.maintained.ready := maintenanceBusy && !protocolError when(io.maintained.valid && !maintenanceBusy && !io.maintenance.fire) { diff --git a/arch/src/main/scala/framework/ip/interlock/src/main/scala/Params.scala b/arch/src/main/scala/framework/ip/interlock/src/main/scala/Params.scala new file mode 100644 index 000000000..fd5f1529c --- /dev/null +++ b/arch/src/main/scala/framework/ip/interlock/src/main/scala/Params.scala @@ -0,0 +1,24 @@ +package framework.ip.memory.interlock + +import buckyball.config.TrackingHardware + +case class Params( + entries: Int = 4, + addressBits: Int = 44, + idBits: Int = 8, + lineBytes: Int = 64, + maxRanges: Int = 8) { + require(entries >= 2) + require(addressBits >= 4 && idBits >= 1) + require(lineBytes >= 8 && (lineBytes & (lineBytes - 1)) == 0) + require(maxRanges >= 1 && (maxRanges & (maxRanges - 1)) == 0) + val rangeIndexBits: Int = math.max(1, Integer.numberOfTrailingZeros(maxRanges)) + val rangeCountBits: Int = Integer.SIZE - Integer.numberOfLeadingZeros(maxRanges) + val lineBits: Int = Integer.numberOfTrailingZeros(lineBytes) + require(addressBits > lineBits) +} + +object Params { + def fromProto(p: TrackingHardware): Params = + Params(p.getEntries, p.getAddressBits, p.getIdBits, p.getLineBytes, p.getMaxRanges) +} diff --git a/arch/src/main/scala/framework/ip/mesh_shm/build.sc b/arch/src/main/scala/framework/ip/mesh_shm/build.sc new file mode 100644 index 000000000..80489a76e --- /dev/null +++ b/arch/src/main/scala/framework/ip/mesh_shm/build.sc @@ -0,0 +1,32 @@ +import mill._ +import $file.^.^.^.^.^.^.settings + + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.ip.memory.mesh_shm.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/MeshBankNode.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/MeshChannel.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/MeshChannelOrdering.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/MeshControlInterface.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/MeshCreditLink.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/MeshRouter.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/MeshSharedMem.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/MeshSharedMemInterface.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/MeshSharedMemParams.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/MeshThreadEndpoint.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/MeshTransferController.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Geometry.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/csrc/Cargo.lock b/arch/src/main/scala/framework/ip/mesh_shm/src/csrc/Cargo.lock similarity index 100% rename from arch/src/main/scala/framework/mem-core/mesh_shm/src/csrc/Cargo.lock rename to arch/src/main/scala/framework/ip/mesh_shm/src/csrc/Cargo.lock diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/csrc/Cargo.toml b/arch/src/main/scala/framework/ip/mesh_shm/src/csrc/Cargo.toml similarity index 100% rename from arch/src/main/scala/framework/mem-core/mesh_shm/src/csrc/Cargo.toml rename to arch/src/main/scala/framework/ip/mesh_shm/src/csrc/Cargo.toml diff --git a/arch/src/main/scala/framework/ip/mesh_shm/src/csrc/fixed_attachment.cpp b/arch/src/main/scala/framework/ip/mesh_shm/src/csrc/fixed_attachment.cpp new file mode 100644 index 000000000..8e8cb5269 --- /dev/null +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/csrc/fixed_attachment.cpp @@ -0,0 +1,106 @@ +#include "VMeshSharedMem.h" +#include "verilated.h" +#include +#include + +int main(int argc, char **argv) { + Verilated::commandArgs(argc, argv); + Verilated::randReset(0); + VMeshSharedMem dut; + unsigned cycle = 0; + auto check = [](bool condition, const char *message) { + if (!condition) + throw std::runtime_error(message); + }; + auto tick = [&]() { + dut.clock = 0; + dut.eval(); + dut.clock = 1; + dut.eval(); + dut.clock = 0; + dut.eval(); + ++cycle; + }; + auto wait = [&](auto condition) { + for (unsigned n = 0; n < 300; ++n) { + dut.eval(); + if (condition()) + return; + tick(); + } + throw std::runtime_error("Mesh response timed out"); + }; + dut.io_bankWrites_0_ready = 1; + dut.io_bankWrites_1_ready = 1; + dut.io_bankWrites_2_ready = 1; + dut.io_bankWrites_3_ready = 1; + dut.io_bankWrites_4_ready = 1; + dut.io_bankWrites_5_ready = 1; + dut.io_bankWrites_6_ready = 1; + dut.io_bankWrites_7_ready = 1; + dut.io_bankWrites_8_ready = 1; + dut.io_bankWrites_9_ready = 1; + dut.io_bankWrites_10_ready = 1; + dut.io_bankWrites_11_ready = 1; + dut.reset = 1; + for (unsigned i = 0; i < 4; ++i) + tick(); + dut.reset = 0; + try { + // Prototype thread 2 has two ports, both fixed at bank node 4. + // Write through its second port to remote bank 11, then read through its + // first. + dut.io_channels_3_request_bits_tdest = 11; + dut.io_channels_3_request_bits_tid = 9; + dut.io_channels_3_request_bits_tuser = (8u << 1) | 1u; + dut.io_channels_3_request_bits_tkeep = 0xffff; + dut.io_channels_3_request_bits_tlast = 1; + for (unsigned word = 0; word < 4; ++word) + dut.io_channels_3_request_bits_tdata[word] = 0x12340000u + word; + dut.io_channels_3_request_valid = 1; + wait([&] { return dut.io_channels_3_request_ready; }); + std::printf("cycle=%u write remote bank=11 row=8 through thread=2 port=1\n", + cycle); + tick(); + dut.io_channels_3_request_valid = 0; + wait([&] { return dut.io_channels_3_response_valid; }); + check(dut.io_channels_3_response_bits_tid == 9 && + dut.io_channels_3_response_bits_tuser == 2, + "Write response returned to the wrong endpoint"); + for (unsigned i = 0; i < 5; ++i) { + check(dut.io_channels_3_response_valid && + dut.io_channels_3_response_bits_tid == 9, + "Write response changed under backpressure"); + tick(); + } + dut.io_channels_3_response_ready = 1; + tick(); + dut.io_channels_2_request_bits_tdest = 11; + dut.io_channels_2_request_bits_tid = 10; + dut.io_channels_2_request_bits_tuser = 8u << 1; + dut.io_channels_2_request_bits_tkeep = 0xffff; + dut.io_channels_2_request_bits_tlast = 1; + dut.io_channels_2_request_valid = 1; + wait([&] { return dut.io_channels_2_request_ready; }); + std::printf("cycle=%u read remote bank=11 row=8 through thread=2 port=0\n", + cycle); + tick(); + dut.io_channels_2_request_valid = 0; + wait([&] { return dut.io_channels_2_response_valid; }); + check(dut.io_channels_2_response_bits_tid == 10 && + dut.io_channels_2_response_bits_tuser == 0, + "Read response returned to the wrong endpoint"); + for (unsigned word = 0; word < 4; ++word) { + check(dut.io_channels_2_response_bits_tdata[word] == 0x12340000u + word, + "Remote bank read data mismatch"); + } + dut.io_channels_2_response_ready = 1; + tick(); + std::puts("PASS: two ports at one fixed mesh node access a remote bank and " + "preserve response ownership"); + } catch (const std::exception &error) { + std::fprintf(stderr, "FAIL at cycle %u: %s\n", cycle, error.what()); + return 1; + } + dut.final(); +} diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/csrc/src/lib.rs b/arch/src/main/scala/framework/ip/mesh_shm/src/csrc/src/lib.rs similarity index 100% rename from arch/src/main/scala/framework/mem-core/mesh_shm/src/csrc/src/lib.rs rename to arch/src/main/scala/framework/ip/mesh_shm/src/csrc/src/lib.rs diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/csrc/src/transfer.rs b/arch/src/main/scala/framework/ip/mesh_shm/src/csrc/src/transfer.rs similarity index 100% rename from arch/src/main/scala/framework/mem-core/mesh_shm/src/csrc/src/transfer.rs rename to arch/src/main/scala/framework/ip/mesh_shm/src/csrc/src/transfer.rs diff --git a/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/router.f b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/router.f new file mode 100644 index 000000000..cf27953bd --- /dev/null +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/router.f @@ -0,0 +1,7 @@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/protocol/axis/interface.sv ++incdir+@RESOURCES@ +@RESOURCES@/router_package.sv +-F @RTL@/MeshRouter/filelist.f +@RESOURCES@/router_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/router_item.svh b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/router_item.svh similarity index 100% rename from arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/router_item.svh rename to arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/router_item.svh diff --git a/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/router_package.sv b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/router_package.sv new file mode 100644 index 000000000..b949e8a86 --- /dev/null +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/router_package.sv @@ -0,0 +1,13 @@ +package router_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + typedef virtual axis_if #(128, 8, 4, 45) router_vif; + import "DPI-C" function int unsigned mesh_route( + input int unsigned destination, + input int unsigned row, + input int unsigned col + ); + `include "router_item.svh" + `include "router_test.svh" +endpackage diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/router_tb.sv b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/router_tb.sv similarity index 100% rename from arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/router_tb.sv rename to arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/router_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/router_test.svh b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/router_test.svh similarity index 96% rename from arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/router_test.svh rename to arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/router_test.svh index ff62b3a96..5b6541d78 100644 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/router_test.svh +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/router_test.svh @@ -1,13 +1,13 @@ -class protocol_test extends ip_test; +class protocol_test extends base_test; `uvm_component_utils(protocol_test) router_vif source[5], sink[5]; - keyed_scoreboard #(packet) scoreboard; + result_board #(packet, 1) scoreboard; function new(string name, uvm_component parent); super.new(name, parent); endfunction function void build_phase(uvm_phase phase); super.build_phase(phase); - scoreboard = keyed_scoreboard#(packet)::type_id::create("scoreboard", this); + scoreboard = result_board#(packet, 1)::type_id::create("scoreboard", this); for (int i = 0; i < 5; i++) begin if (!uvm_config_db#(router_vif)::get( this, "", $sformatf("source%0d", i), source[i] diff --git a/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/shared.f b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/shared.f new file mode 100644 index 000000000..342a30920 --- /dev/null +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/shared.f @@ -0,0 +1,8 @@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/protocol/axis/interface.sv ++incdir+@RESOURCES@ +@RESOURCES@/transfer_if.sv +@RESOURCES@/shared_package.sv +-F @RTL@/MeshSharedMem/filelist.f +@RESOURCES@/shared_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/shared_item.svh b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/shared_item.svh similarity index 100% rename from arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/shared_item.svh rename to arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/shared_item.svh diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/shared_package.sv b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/shared_package.sv similarity index 95% rename from arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/shared_package.sv rename to arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/shared_package.sv index 7d7197ef8..eb329ee24 100644 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/shared_package.sv +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/shared_package.sv @@ -1,6 +1,6 @@ package shared_pkg; import uvm_pkg::*; - import ip_pkg::*; + import verification_pkg::*; `include "uvm_macros.svh" typedef virtual axis_if #(128, 8, 4, 17) request_vif; typedef virtual axis_if #(128, 8, 0, 2) response_vif; diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/shared_tb.sv b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/shared_tb.sv similarity index 99% rename from arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/shared_tb.sv rename to arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/shared_tb.sv index 62806b4cf..f0e31a0f9 100644 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/shared_tb.sv +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/shared_tb.sv @@ -30,10 +30,10 @@ module shared_tb; .io_transferCompletion_ready(transfer.complete_ready), .io_transferCompletion_bits_error(transfer.error), .io_transferCompletion_bits_tag(transfer.complete_tag), - .io_transferCommand_bits_sourceCore(transfer.source_core), + .io_transferCommand_bits_sourceThread(transfer.source_thread), .io_transferCommand_bits_sourceBank(transfer.source_bank), .io_transferCommand_bits_sourceAddr(transfer.source_row), - .io_transferCommand_bits_targetCore(transfer.target_core), + .io_transferCommand_bits_targetThread(transfer.target_thread), .io_transferCommand_bits_targetBank(transfer.target_bank), .io_transferCommand_bits_targetAddr(transfer.target_row), .io_transferCommand_bits_rows(transfer.rows), diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/shared_test.svh b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/shared_test.svh similarity index 89% rename from arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/shared_test.svh rename to arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/shared_test.svh index bdf3230b4..07d9960ef 100644 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/shared_test.svh +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/shared_test.svh @@ -1,17 +1,17 @@ -class protocol_test extends ip_test; +class protocol_test extends base_test; `uvm_component_utils(protocol_test) request_vif source[10]; response_vif sink[10]; - keyed_scoreboard #(response) scoreboard; + result_board #(response, 1) scoreboard; chandle model; virtual transfer_if transfer; function new(string name, uvm_component parent); super.new(name, parent); - timeout = 500us; + endfunction function void build_phase(uvm_phase phase); super.build_phase(phase); - scoreboard = keyed_scoreboard#(response)::type_id::create("scoreboard", this); + scoreboard = result_board#(response, 1)::type_id::create("scoreboard", this); model = mesh_banks_create(); if (!uvm_config_db#(virtual transfer_if)::get(this, "", "transfer", transfer)) `uvm_fatal("VIF", "transfer interface missing") @@ -159,15 +159,15 @@ class protocol_test extends ip_test; end access (0, 0, 3, 0, 0, 253, 1); endtask - task move_rows(int source_core, int source_bank, int source_row, int target_core, int target_bank, - int target_row, int rows, bit invalid); + task move_rows(int source_thread, int source_bank, int source_row, int target_thread, + int target_bank, int target_row, int rows, bit invalid); bit [7:0] token = $urandom(); @(negedge transfer.clock); transfer.valid = 1; - transfer.source_core = source_core; + transfer.source_thread = source_thread; transfer.source_bank = source_bank; transfer.source_row = source_row; - transfer.target_core = target_core; + transfer.target_thread = target_thread; transfer.target_bank = target_bank; transfer.target_row = target_row; transfer.rows = rows; @@ -175,10 +175,10 @@ class protocol_test extends ip_test; transfer.complete_ready = 0; if (!invalid) for (int i = 0; i < rows; i++) begin - transfer.memory[source_core][source_bank][source_row+i] = { + transfer.memory[source_thread][source_bank][source_row+i] = { $urandom(), $urandom(), $urandom(), $urandom() }; - transfer.initialized[source_core][source_bank][source_row+i] = 1; + transfer.initialized[source_thread][source_bank][source_row+i] = 1; end do @(posedge transfer.clock); while (!transfer.ready); @(negedge transfer.clock); @@ -191,8 +191,8 @@ class protocol_test extends ip_test; end if (!invalid) for (int i = 0; i < rows; i++) - if(!transfer.initialized[target_core][target_bank][target_row+i] || - transfer.memory[target_core][target_bank][target_row+i]!==transfer.memory[source_core][source_bank][source_row+i]) + if(!transfer.initialized[target_thread][target_bank][target_row+i] || + transfer.memory[target_thread][target_bank][target_row+i]!==transfer.memory[source_thread][source_bank][source_row+i]) `uvm_fatal("MVOVER", "target rows differ from source") transfer.complete_ready = 1; @(posedge transfer.clock); @@ -240,10 +240,10 @@ class protocol_test extends ip_test; move_rows(1, 16, 0, 2, 0, 0, 1, 1); move_rows(1, 1, 3, 2, 16, 0, 1, 1); move_rows(1, 1023, 0, 2, 0, 0, 1, 1); - for (int core = 0; core < 5; core++) - move_rows(core, core, 16, (core + 1) % 5, 15 - core, 32, 8, 0); + for (int thread = 0; thread < 5; thread++) + move_rows(thread, thread, 16, (thread + 1) % 5, 15 - thread, 32, 8, 0); move_rows(1, 1, 0, 1, 1, 1, 8, 1); - `uvm_info("MVOVER", "checked core1.bank1 to core2.bank8 and range/error cases", UVM_LOW) + `uvm_info("MVOVER", "checked thread1.bank1 to thread2.bank8 and range/error cases", UVM_LOW) `uvm_info("MESH", $sformatf("checked %0d memory responses", scoreboard.checked), UVM_LOW) endtask function void final_phase(uvm_phase phase); diff --git a/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/suite.toml b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/suite.toml new file mode 100644 index 000000000..17d2625b1 --- /dev/null +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/suite.toml @@ -0,0 +1,37 @@ +[model] +manifest = "src/csrc/Cargo.toml" +library = "libmesh_reference_model" + +[[tests]] +functional_groups = [] +name = "router" +top = "router_tb" +test = "protocol_test" +filelist = "router.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +functional_groups = [] +name = "shared" +top = "shared_tb" +test = "protocol_test" +filelist = "shared.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/transfer_if.sv b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/transfer_if.sv similarity index 96% rename from arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/transfer_if.sv rename to arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/transfer_if.sv index 3aae89b7e..2eba72f6f 100644 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/transfer_if.sv +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/transfer_if.sv @@ -3,7 +3,7 @@ interface transfer_if ( ); logic reset; logic valid, ready, complete_valid, complete_ready, error; - logic [7:0] source_core, target_core, tag, complete_tag; + logic [7:0] source_thread, target_thread, tag, complete_tag; logic [9:0] source_bank, target_bank; logic [15:0] source_row, target_row; logic [16:0] rows; diff --git a/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/Emit.scala b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/Emit.scala new file mode 100644 index 000000000..578c52eaa --- /dev/null +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/Emit.scala @@ -0,0 +1,15 @@ +package framework.ip.memory.mesh_shm + +object Emit extends App { + val p = MeshSharedMemParams.prototype + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new MeshRouter(p, 1, 1), + firtoolOpts = args, + args = Array("--target-dir", "build/MeshRouter", "--split-verilog") + ) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new MeshSharedMem(p), + firtoolOpts = args, + args = Array("--target-dir", "build/MeshSharedMem", "--split-verilog") + ) +} diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshBankNode.scala b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshBankNode.scala similarity index 95% rename from arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshBankNode.scala rename to arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshBankNode.scala index 5fe267592..2a1f8466b 100644 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshBankNode.scala +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshBankNode.scala @@ -1,9 +1,9 @@ -package memcore.memory.mesh_shm +package framework.ip.memory.mesh_shm import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import memcore.memory.bank.{Bank, BankSetParams} +import framework.ip.memory.bank.{Bank, BankSetParams} @instantiable class MeshBankNode(p: MeshSharedMemParams, row: Int, col: Int) extends Module { diff --git a/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshChannel.scala b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshChannel.scala new file mode 100644 index 000000000..bd9a26e49 --- /dev/null +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshChannel.scala @@ -0,0 +1,10 @@ +package framework.ip.memory.mesh_shm + +import chisel3._ +import chisel3.util._ + +/** Thread-facing shared-bank port. Tagged requests support bounded outstanding transactions per channel. */ +class MeshChannel(p: MeshSharedMemParams) extends Bundle { + val request = Flipped(Decoupled(new MeshClientRequest(p))) + val response = Decoupled(new MeshClientResponse(p)) +} diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshChannelOrdering.scala b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshChannelOrdering.scala similarity index 98% rename from arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshChannelOrdering.scala rename to arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshChannelOrdering.scala index ef07fb4cd..e27515a75 100644 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshChannelOrdering.scala +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshChannelOrdering.scala @@ -1,4 +1,4 @@ -package memcore.memory.mesh_shm +package framework.ip.memory.mesh_shm import chisel3._ import chisel3.util._ diff --git a/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshControlInterface.scala b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshControlInterface.scala new file mode 100644 index 000000000..a72b8ea4e --- /dev/null +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshControlInterface.scala @@ -0,0 +1,53 @@ +package framework.ip.memory.mesh_shm + +import chisel3._ +import chisel3.util._ + +/** Contiguous rows from one Thread-local Bank to another Thread-local Bank. */ +class MeshTransferCommand(p: MeshSharedMemParams) extends Bundle { + val sourceThread = UInt(8.W) + val sourceBank = UInt(p.localBankBits.W) + val sourceAddr = UInt(16.W) + val targetThread = UInt(8.W) + val targetBank = UInt(p.localBankBits.W) + val targetAddr = UInt(16.W) + val rows = UInt(17.W) + val tag = UInt(p.tagBits.W) +} + +class MeshTransferCompletion(p: MeshSharedMemParams) extends Bundle { + val tag = UInt(p.tagBits.W) + val error = Bool() +} + +class MeshLocalBankRequest( + addressBits: Int, + bankBits: Int, + dataBits: Int, + tagBits: Int) + extends Bundle { + val addr = UInt(addressBits.W) + val write = Bool() + val data = UInt(dataBits.W) + val mask = UInt((dataBits / 8).W) + val tag = UInt(tagBits.W) + val bank = UInt(bankBits.W) +} + +class MeshLocalBankResponse(dataBits: Int, tagBits: Int) extends Bundle { + val data = UInt(dataBits.W) + val tag = UInt(tagBits.W) + val error = Bool() +} + +/** SharedMem initiates both the source read and target write. */ +class MeshLocalBankPort( + addressBits: Int, + bankBits: Int, + dataBits: Int, + tagBits: Int) + extends Bundle { + + val request = Decoupled(new MeshLocalBankRequest(addressBits, bankBits, dataBits, tagBits)) + val response = Flipped(Decoupled(new MeshLocalBankResponse(dataBits, tagBits))) +} diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshCreditLink.scala b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshCreditLink.scala similarity index 85% rename from arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshCreditLink.scala rename to arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshCreditLink.scala index 2c7bf499c..c19d8fa1f 100644 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshCreditLink.scala +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshCreditLink.scala @@ -1,4 +1,7 @@ -package memcore.memory.mesh_shm +package framework.ip.memory.mesh_shm + +import framework.lib.queue.Queue +import chisel3.experimental.hierarchy.{Instance, Instantiate} import chisel3._ import chisel3.util._ @@ -44,7 +47,7 @@ class MeshCreditToDecoupled(p: MeshSharedMemParams) extends Module { val out = Decoupled(new MeshPacket(p)) }) - val buffer = Module(new Queue(new MeshPacket(p), p.linkBufferDepth, pipe = false, flow = false)) + val buffer = Instantiate(new Queue(new MeshPacket(p), p.linkBufferDepth, false, false)) buffer.io.enq.valid := io.in.valid buffer.io.enq.bits := io.in.bits io.out <> buffer.io.deq diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshRouter.scala b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshRouter.scala similarity index 88% rename from arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshRouter.scala rename to arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshRouter.scala index 3f086ea73..7f5499c78 100644 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshRouter.scala +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshRouter.scala @@ -1,4 +1,7 @@ -package memcore.memory.mesh_shm +package framework.ip.memory.mesh_shm + +import framework.lib.queue.Queue +import framework.lib.arbiter.{RRArbiter} import chisel3._ import chisel3.util._ @@ -40,7 +43,7 @@ class MeshCreditRouter(p: MeshSharedMemParams, row: Int, col: Int) extends Modul } val inputs = Seq.fill(MeshDirection.portCount) { - Module(new Queue(new MeshPacket(p), p.linkBufferDepth, pipe = false, flow = false)) + Instantiate(new Queue(new MeshPacket(p), p.linkBufferDepth, false, false)) } for ((input, port) <- inputs.zip(io.in)) { @@ -54,15 +57,8 @@ class MeshCreditRouter(p: MeshSharedMemParams, row: Int, col: Int) extends Modul val choices = inputs.map(input => direction(input.io.deq.bits)) - val arbiters = Seq.fill(MeshDirection.portCount)(Module(new RRArbiter(new MeshPacket(p), MeshDirection.portCount) { - - override lazy val lastGrant = { - val pointer = RegInit(0.U(math.max(1, log2Ceil(MeshDirection.portCount)).W)) - when(io.out.fire)(pointer := io.chosen) - pointer - } - - })) + val arbiters = + Seq.fill(MeshDirection.portCount)(Instantiate(new RRArbiter(new MeshPacket(p), MeshDirection.portCount))) for (output <- 0 until MeshDirection.portCount) { for (input <- 0 until MeshDirection.portCount) { diff --git a/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshSharedMem.scala b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshSharedMem.scala new file mode 100644 index 000000000..0c9399886 --- /dev/null +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshSharedMem.scala @@ -0,0 +1,259 @@ +package framework.ip.memory.mesh_shm + +import framework.lib.arbiter.{RRArbiter} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} + +class MeshSharedMemIO(p: MeshSharedMemParams) extends Bundle { + val channels = Vec(p.totalChannels, new MeshChannel(p)) + val external = Vec(p.externalChannels, new MeshChannel(p)) + val transferCommand = Flipped(Decoupled(new MeshTransferCommand(p))) + val transferCompletion = Decoupled(new MeshTransferCompletion(p)) + val localBanks = Vec(p.threads.size, new MeshLocalBankPort(p.addressBits, p.localBankBits, p.dataBits, p.tagBits)) + val bankWrites = Vec(p.bankCount, Decoupled(new MeshClientRequest(p))) +} + +@instantiable +class MeshSharedMem(p: MeshSharedMemParams) extends Module { + + @public + val io = IO(new MeshSharedMemIO(p)) + + val transfer: Instance[MeshTransferController] = Instantiate(new MeshTransferController(p)) + transfer.io.command.valid := io.transferCommand.valid + transfer.io.command.bits := io.transferCommand.bits + io.transferCommand.ready := transfer.io.command.ready + io.transferCompletion.valid := transfer.io.completion.valid + io.transferCompletion.bits := transfer.io.completion.bits + transfer.io.completion.ready := io.transferCompletion.ready + + val ordering = Instantiate(new MeshChannelOrdering(p, p.totalChannels + p.externalChannels)) + for (channel <- 0 until p.totalChannels) { + ordering.io.client(channel) <> io.channels(channel) + } + for (channel <- 0 until p.externalChannels) { + ordering.io.client(p.totalChannels + channel) <> io.external(channel) + } + + for (thread <- p.threads.indices if p.threads(thread).ports == 0) { + io.localBanks(thread).request.valid := false.B + io.localBanks(thread).request.bits := 0.U.asTypeOf(io.localBanks(thread).request.bits) + io.localBanks(thread).response.ready := false.B + } + + val requests: Seq[Seq[Instance[MeshCreditRouter]]] = + Seq.tabulate(p.rows, p.cols)((row, col) => Instantiate(new MeshCreditRouter(p, row, col))) + val responses: Seq[Seq[Instance[MeshCreditRouter]]] = + Seq.tabulate(p.rows, p.cols)((row, col) => Instantiate(new MeshCreditRouter(p, row, col))) + val banks: Seq[Seq[Instance[MeshBankNode]]] = + Seq.tabulate(p.rows, p.cols)((row, col) => Instantiate(new MeshBankNode(p, row, col))) + val bankRows = VecInit((0 until p.bankCount).map(i => (i / p.cols).U(p.rowBits.W))) + val bankCols = VecInit((0 until p.bankCount).map(i => (i % p.cols).U(p.colBits.W))) + + for { + row <- 0 until p.rows + col <- 0 until p.cols + } { + val requestRouter = requests(row)(col) + val responseRouter = responses(row)(col) + val bank = banks(row)(col) + val bankIndex = row * p.cols + col + val requestEgress = Instantiate(new MeshCreditToDecoupled(p)) + requestEgress.io.in <> requestRouter.io.out(MeshDirection.endpoint) + val localRequest = requestEgress.io.out + io.bankWrites( + bankIndex + ).valid := localRequest.valid && !localRequest.bits.privateRequest && localRequest.bits.write && bank.io.request.ready + io.bankWrites(bankIndex).bits.bank := bankIndex.U + io.bankWrites(bankIndex).bits.tuser := Cat(bank.io.request.bits.addr, bank.io.request.bits.write) + io.bankWrites(bankIndex).bits.tlast := true.B + io.bankWrites(bankIndex).bits.data := bank.io.request.bits.data + io.bankWrites(bankIndex).bits.mask := bank.io.request.bits.mask + io.bankWrites(bankIndex).bits.tag := bank.io.request.bits.tag + bank.io.request.valid := localRequest.valid && !localRequest.bits.privateRequest && + (!localRequest.bits.write || io.bankWrites(bankIndex).ready) + bank.io.request.bits := localRequest.bits + val privateThreads = + p.threads.indices.filter(thread => p.threads(thread).ports > 0 && p.threadLocations(thread) == (row, col)) + val privateReady = Wire(Vec(privateThreads.size, Bool())) + val replies = Instantiate(new RRArbiter(new MeshPacket(p), privateThreads.size + 1)) + replies.io.in(0) <> bank.io.response + for ((thread, index) <- privateThreads.zipWithIndex) { + val endpoint: Instance[MeshThreadEndpoint] = Instantiate(new MeshThreadEndpoint(p, thread, row, col)) + endpoint.io.request.valid := localRequest.valid && localRequest.bits.privateRequest && localRequest.bits.thread === thread.U + endpoint.io.request.bits := localRequest.bits + privateReady(index) := localRequest.bits.thread === thread.U && endpoint.io.request.ready + io.localBanks(thread) <> endpoint.io.bank + replies.io.in(index + 1) <> endpoint.io.response + } + localRequest.ready := Mux( + localRequest.bits.privateRequest, + (if (privateThreads.nonEmpty) privateReady.asUInt.orR else false.B), + bank.io.request.ready && (!localRequest.bits.write || io.bankWrites(bankIndex).ready) + ) + when(localRequest.valid && localRequest.bits.privateRequest) { + assert( + privateThreads.map(thread => localRequest.bits.thread === thread.U).reduceOption(_ || _).getOrElse(false.B), + "private request reached a node without its Thread" + ) + } + val responseIngress = Instantiate(new MeshDecoupledToCredit(p)) + responseIngress.io.in <> replies.io.out + responseRouter.io.in(MeshDirection.endpoint) <> responseIngress.io.out + val requestIngress = Instantiate(new MeshDecoupledToCredit(p)) + requestRouter.io.in(MeshDirection.endpoint) <> requestIngress.io.out + val responseEgress = Instantiate(new MeshCreditToDecoupled(p)) + responseEgress.io.in <> responseRouter.io.out(MeshDirection.endpoint) + val response = responseEgress.io.out + + val attached = p.channelLocations.zipWithIndex.collect { + case ((r, c), index) if r == row && c == col => index + } ++ (if (row == 0 && col == 0) (p.totalChannels until p.totalChannels + 1 + p.externalChannels) else Seq.empty) + if (attached.nonEmpty) { + val arbiter = Instantiate(new RRArbiter(new MeshPacket(p), attached.size)) + val localResponseReady = Wire(Vec(attached.size, Bool())) + for ((channel, port) <- attached.zipWithIndex) { + if (channel == p.totalChannels) { + arbiter.io.in(port) <> transfer.io.mesh.request + val network = response + transfer.io.mesh.response.valid := network.valid && network.bits.channel === channel.U + transfer.io.mesh.response.bits := network.bits + localResponseReady(port) := network.bits.channel === channel.U && transfer.io.mesh.response.ready + } else { + val client = + if (channel < p.totalChannels) ordering.io.ordered(channel) else ordering.io.ordered(channel - 1) + val requestPort = client.request + val responsePort = client.response + val inFlight = RegInit(0.U(log2Ceil(p.maxInFlight + 1).W)) + val tags = RegInit(0.U((1 << p.tagBits).W)) + val available = inFlight < p.maxInFlight.U && !tags(requestPort.bits.tag) + val errorPending = RegInit(false.B) + val errorTag = Reg(UInt(p.tagBits.W)) + val errorWrite = Reg(Bool()) + val bankValid = requestPort.bits.bank < p.visibleBankCount.U + val packet = Wire(new MeshPacket(p)) + packet := 0.U.asTypeOf(new MeshPacket(p)) + packet.tdest := Cat(bankRows(requestPort.bits.bank), bankCols(requestPort.bits.bank)) + packet.tdata := requestPort.bits.data + packet.tkeep := requestPort.bits.mask + packet.tid := requestPort.bits.tag + packet.tlast := true.B + packet.tuser := Cat( + row.U(p.rowBits.W), + col.U(p.colBits.W), + channel.U(p.channelBits.W), + requestPort.bits.addr, + requestPort.bits.write, + false.B + ) + arbiter.io.in(port).valid := requestPort.valid && available && !errorPending && bankValid + arbiter.io.in(port).bits := packet + requestPort.ready := available && !errorPending && Mux(bankValid, arbiter.io.in(port).ready, true.B) + when(requestPort.fire) { + when(!bankValid) { + errorPending := true.B + errorTag := requestPort.bits.tag + errorWrite := requestPort.bits.write + } + } + + val responseValid = RegInit(false.B) + val responseBits = Reg(new MeshClientResponse(p)) + val responseNext = Wire(Decoupled(new MeshClientResponse(p))) + responsePort.valid := responseValid + responsePort.bits := responseBits + responseNext.ready := !responseValid || responsePort.ready + when(responseNext.ready) { + responseValid := responseNext.valid + when(responseNext.valid)(responseBits := responseNext.bits) + } + responseNext.valid := + errorPending || (response.valid && + response.bits.channel === channel.U) + responseNext.bits := 0.U.asTypeOf(responseNext.bits) + responseNext.bits.data := + Mux(errorPending, 0.U, response.bits.data) + responseNext.bits.tag := + Mux(errorPending, errorTag, response.bits.tag) + responseNext.bits.tkeep := Fill(p.maskBits, 1.U(1.W)) + responseNext.bits.tlast := true.B + responseNext.bits.tuser := Cat( + Mux(errorPending, errorWrite, response.bits.write), + errorPending || response.bits.error + ) + localResponseReady(port) := + response.bits.channel === channel.U && + responseNext.ready && !errorPending + when(response.valid && + response.bits.channel === channel.U) { + assert(inFlight =/= 0.U && tags(response.bits.tag)) + } + when(responsePort.fire) { + assert(inFlight =/= 0.U && tags(responsePort.bits.tag)) + } + when(responseNext.fire && errorPending)(errorPending := false.B) + val issued = Mux(requestPort.fire, 1.U((1 << p.tagBits).W) << requestPort.bits.tag, 0.U) + val completed = Mux(responsePort.fire, 1.U((1 << p.tagBits).W) << responsePort.bits.tag, 0.U) + tags := (tags | issued) & ~completed + when(requestPort.fire =/= responsePort.fire) { + inFlight := Mux(requestPort.fire, inFlight + 1.U, inFlight - 1.U) + } + } + } + requestIngress.io.in <> arbiter.io.out + response.ready := localResponseReady.asUInt.orR + when(response.valid) { + assert( + attached.map(channel => response.bits.channel === channel.U).reduce( + _ || _ + ) + ) + } + } else { + requestIngress.io.in.valid := false.B + requestIngress.io.in.bits := 0.U.asTypeOf(new MeshPacket(p)) + response.ready := false.B + when(response.valid) { + assert(false.B, "response reached a node without an attached channel") + } + } + + if (col + 1 < p.cols) { + requests(row)(col + 1).io.in(MeshDirection.west) <> requestRouter.io.out(MeshDirection.east) + requestRouter.io.in(MeshDirection.east) <> requests(row)(col + 1).io.out(MeshDirection.west) + responses(row)(col + 1).io.in(MeshDirection.west) <> responseRouter.io.out(MeshDirection.east) + responseRouter.io.in(MeshDirection.east) <> responses(row)(col + 1).io.out(MeshDirection.west) + } + if (row + 1 < p.rows) { + requests(row + 1)(col).io.in(MeshDirection.north) <> requestRouter.io.out(MeshDirection.south) + requestRouter.io.in(MeshDirection.south) <> requests(row + 1)(col).io.out(MeshDirection.north) + responses(row + 1)(col).io.in(MeshDirection.north) <> responseRouter.io.out(MeshDirection.south) + responseRouter.io.in(MeshDirection.south) <> responses(row + 1)(col).io.out(MeshDirection.north) + } + + for (router <- Seq(requestRouter, responseRouter)) { + if (col == 0) { + router.io.in(MeshDirection.west).valid := false.B + router.io.in(MeshDirection.west).bits := 0.U.asTypeOf(new MeshPacket(p)) + router.io.out(MeshDirection.west).credit := false.B + } + if (col == p.cols - 1) { + router.io.in(MeshDirection.east).valid := false.B + router.io.in(MeshDirection.east).bits := 0.U.asTypeOf(new MeshPacket(p)) + router.io.out(MeshDirection.east).credit := false.B + } + if (row == 0) { + router.io.in(MeshDirection.north).valid := false.B + router.io.in(MeshDirection.north).bits := 0.U.asTypeOf(new MeshPacket(p)) + router.io.out(MeshDirection.north).credit := false.B + } + if (row == p.rows - 1) { + router.io.in(MeshDirection.south).valid := false.B + router.io.in(MeshDirection.south).bits := 0.U.asTypeOf(new MeshPacket(p)) + router.io.out(MeshDirection.south).credit := false.B + } + } + } +} diff --git a/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshSharedMemInterface.scala b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshSharedMemInterface.scala new file mode 100644 index 000000000..e0306d241 --- /dev/null +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshSharedMemInterface.scala @@ -0,0 +1,37 @@ +package framework.ip.memory.mesh_shm + +import chisel3._ + +class MeshClientRequest(p: MeshSharedMemParams) + extends framework.ip.bus.axi.Beat(p.dataBits, p.tagBits, p.bankBits, p.addressBits + 1) { + def bank = tdest + def addr = tuser(p.addressBits, 1) + def write = tuser(0) + def mask = tkeep + def tag = tid +} + +class MeshClientResponse(p: MeshSharedMemParams) extends framework.ip.bus.axi.Beat(p.dataBits, p.tagBits, 0, 2) { + def tag = tid + def error = tuser(0) + def write = tuser(1) +} + +/** One row transaction: AXI-S payload plus routed event metadata in TUSER. */ +class MeshPacket(p: MeshSharedMemParams) + extends framework.ip.bus.axi.Beat(p.dataBits, p.tagBits, p.rowBits + p.colBits, p.packetUserBits) { + def privateRequest = tuser(p.eventBits) + def localBank = tuser(p.eventBits + p.localBankBits, p.eventBits + 1) + def thread = tuser(p.packetUserBits - 1, p.eventBits + p.localBankBits + 1) + def destRow = tdest(p.rowBits + p.colBits - 1, p.colBits) + def destCol = tdest(p.colBits - 1, 0) + def error = tuser(0) + def write = tuser(1) + def addr = tuser(p.addressBits + 1, 2) + def channel = tuser(p.addressBits + p.channelBits + 1, p.addressBits + 2) + def sourceCol = tuser(p.addressBits + p.channelBits + p.colBits + 1, p.addressBits + p.channelBits + 2) + def sourceRow = + tuser(p.addressBits + p.channelBits + p.colBits + p.rowBits + 1, p.addressBits + p.channelBits + p.colBits + 2) + def mask = tkeep + def tag = tid +} diff --git a/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshSharedMemParams.scala b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshSharedMemParams.scala new file mode 100644 index 000000000..fc0e5430b --- /dev/null +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshSharedMemParams.scala @@ -0,0 +1,79 @@ +package framework.ip.memory.mesh_shm + +import chisel3.util.log2Ceil +import framework.ip.memory.bank.BankSetParams + +case class MeshThreadAttachment(bank: Int, ports: Int) + +case class MeshSharedMemParams( + rows: Int, + cols: Int, + entriesPerBank: Int, + dataBits: Int, + tagBits: Int, + threads: Seq[MeshThreadAttachment], + maxInFlight: Int = 4, + localBankBits: Int = 10, + visibleBanks: Int = 0, + externalChannels: Int = 0, + linkBufferDepth: Int = 2) { + require(rows > 0 && cols > 0) + require(entriesPerBank <= 65536) + require(entriesPerBank >= 2 && (entriesPerBank & (entriesPerBank - 1)) == 0) + require(dataBits > 0 && dataBits % 8 == 0) + require(tagBits > 0 && localBankBits > 0 && threads.nonEmpty) + require(threads.forall(t => t.bank >= 0 && t.bank < rows * cols && t.ports >= 0)) + + require(externalChannels >= 0) + require(linkBufferDepth > 0) + val linkCreditBits: Int = math.max(1, log2Ceil(linkBufferDepth + 1)) + require(maxInFlight > 0 && maxInFlight <= (1 << tagBits)) + val bankParams = BankSetParams(dataBits, 1, entriesPerBank, tagBits) + val bankCount: Int = rows * cols + val visibleBankCount: Int = if (visibleBanks == 0) bankCount else visibleBanks + require(visibleBankCount > 0 && visibleBankCount <= bankCount) + require(threads.exists(_.ports > 0), "mesh requires a data endpoint") + val totalChannels: Int = threads.map(_.ports).sum + val channelLocations: Seq[(Int, Int)] = + threads.flatMap(t => Seq.fill(t.ports)((t.bank / cols, t.bank % cols))) + val rowBits: Int = math.max(1, log2Ceil(rows)) + val colBits: Int = math.max(1, log2Ceil(cols)) + val bankBits: Int = math.max(1, log2Ceil(bankCount)) + val addressBits: Int = 16 + val channelBits: Int = math.max(1, log2Ceil(totalChannels + 1 + externalChannels)) + val threadBits: Int = math.max(1, log2Ceil(threads.size)) + + val threadLocations: Seq[(Int, Int)] = threads.map(t => (t.bank / cols, t.bank % cols)) + + val eventBits: Int = rowBits + colBits + channelBits + addressBits + 2 + val packetUserBits: Int = eventBits + localBankBits + 9 + val maskBits: Int = dataBits / 8 + + // Thread order in this sequence defines the flattened io.channels layout. + def channelsForThread(threadIndex: Int): Range = { + require(threadIndex >= 0 && threadIndex < threads.size) + val start = threads.take(threadIndex).map(_.ports).sum + start until (start + threads(threadIndex).ports) + } + +} + +object MeshSharedMemParams { + + // Five fixed thread attachments on a 3x4 mesh, with 1/1/2/2/4 data ports. + val prototype: MeshSharedMemParams = MeshSharedMemParams( + rows = 3, + cols = 4, + entriesPerBank = 1024, + dataBits = 128, + tagBits = 8, + threads = Seq( + MeshThreadAttachment(0, 1), + MeshThreadAttachment(3, 1), + MeshThreadAttachment(4, 2), + MeshThreadAttachment(7, 2), + MeshThreadAttachment(5, 4) + ) + ) + +} diff --git a/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshThreadEndpoint.scala b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshThreadEndpoint.scala new file mode 100644 index 000000000..1d81a5b3a --- /dev/null +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshThreadEndpoint.scala @@ -0,0 +1,59 @@ +package framework.ip.memory.mesh_shm + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} + +/** The private Bank port stays beside its Thread; only packets cross the Mesh. */ +@instantiable +class MeshThreadEndpoint( + p: MeshSharedMemParams, + thread: Int, + row: Int, + col: Int) + extends Module { + + @public + val io = IO(new Bundle { + val request = Flipped(Decoupled(new MeshPacket(p))) + val response = Decoupled(new MeshPacket(p)) + val bank = new MeshLocalBankPort(p.addressBits, p.localBankBits, p.dataBits, p.tagBits) + }) + + val pending = RegInit(false.B) + val request = Reg(new MeshPacket(p)) + io.bank.request.valid := io.request.valid && !pending + io.bank.request.bits.bank := io.request.bits.localBank + io.bank.request.bits.addr := io.request.bits.addr + io.bank.request.bits.write := io.request.bits.write + io.bank.request.bits.data := io.request.bits.data + io.bank.request.bits.mask := io.request.bits.mask + io.bank.request.bits.tag := io.request.bits.tag + io.request.ready := io.bank.request.ready && !pending + when(io.request.fire) { + assert(io.request.bits.privateRequest && io.request.bits.thread === thread.U) + assert(io.request.bits.destRow === row.U && io.request.bits.destCol === col.U) + request := io.request.bits + pending := true.B + } + io.response.valid := pending && io.bank.response.valid + io.response.bits := request + io.response.bits.tdest := Cat(request.sourceRow, request.sourceCol) + io.response.bits.tdata := Mux(request.write, 0.U, io.bank.response.bits.data) + io.response.bits.tuser := Cat( + request.thread, + request.localBank, + true.B, + row.U(p.rowBits.W), + col.U(p.colBits.W), + request.channel, + request.addr, + request.write, + io.bank.response.bits.error + ) + io.bank.response.ready := pending && io.response.ready + when(io.bank.response.valid) { + assert(pending && io.bank.response.bits.tag === request.tag) + } + when(io.response.fire)(pending := false.B) +} diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshTransferController.scala b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshTransferController.scala similarity index 77% rename from arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshTransferController.scala rename to arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshTransferController.scala index a459a5dd5..6e2d11890 100644 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshTransferController.scala +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshTransferController.scala @@ -1,10 +1,10 @@ -package memcore.memory.mesh_shm +package framework.ip.memory.mesh_shm import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public} -/** A row-range command uses routed private-Core endpoints, without a staging Bank. */ +/** A row-range command uses routed private-Thread endpoints, without a staging Bank. */ @instantiable class MeshTransferController(p: MeshSharedMemParams) extends Module { @@ -26,9 +26,9 @@ class MeshTransferController(p: MeshSharedMemParams) extends Module { val data = Reg(UInt(p.dataBits.W)) val remaining = Reg(UInt(17.W)) val failed = RegInit(false.B) - val endpoints = VecInit(p.cores.map(_.localBankNode.nonEmpty.B)) - val coreRows = VecInit(p.localBankLocations.map(_.map(_._1).getOrElse(0).U(p.rowBits.W))) - val coreCols = VecInit(p.localBankLocations.map(_.map(_._2).getOrElse(0).U(p.colBits.W))) + val endpoints = VecInit(p.threads.map(thread => (thread.ports > 0).B)) + val threadRows = VecInit(p.threadLocations.map(_._1.U(p.rowBits.W))) + val threadCols = VecInit(p.threadLocations.map(_._2.U(p.colBits.W))) io.command.ready := state === idle io.completion.valid := state === complete io.completion.bits.tag := command.tag @@ -38,13 +38,13 @@ class MeshTransferController(p: MeshSharedMemParams) extends Module { remaining := io.command.bits.rows failed := false.B val input = io.command.bits - val source = input.sourceCore(p.coreBits - 1, 0) - val target = input.targetCore(p.coreBits - 1, 0) - when(input.sourceCore >= p.cores.size.U || input.targetCore >= p.cores.size.U || + val source = input.sourceThread(p.threadBits - 1, 0) + val target = input.targetThread(p.threadBits - 1, 0) + when(input.sourceThread >= p.threads.size.U || input.targetThread >= p.threads.size.U || !endpoints(source) || !endpoints(target) || input.rows === 0.U || (input.sourceAddr +& input.rows) > 65536.U || (input.targetAddr +& input.rows) > 65536.U || - (input.sourceCore === input.targetCore && input.sourceBank === input.targetBank && + (input.sourceThread === input.targetThread && input.sourceBank === input.targetBank && input.sourceAddr =/= input.targetAddr && input.sourceAddr < (input.targetAddr +& input.rows) && input.targetAddr < (input.sourceAddr +& input.rows))) { @@ -53,20 +53,20 @@ class MeshTransferController(p: MeshSharedMemParams) extends Module { }.otherwise(state := sourceRequest) } val write = state === targetRequest - val core = Mux(write, command.targetCore, command.sourceCore) - val slot = core(p.coreBits - 1, 0) + val thread = Mux(write, command.targetThread, command.sourceThread) + val slot = thread(p.threadBits - 1, 0) val bank = Mux(write, command.targetBank, command.sourceBank) val address = Mux(write, command.targetAddr, command.sourceAddr) io.mesh.request.valid := state === sourceRequest || state === targetRequest io.mesh.request.bits := 0.U.asTypeOf(new MeshPacket(p)) - io.mesh.request.bits.tdest := Cat(coreRows(slot), coreCols(slot)) + io.mesh.request.bits.tdest := Cat(threadRows(slot), threadCols(slot)) io.mesh.request.bits.tuser := Cat( - core, + thread, bank, true.B, 0.U(p.rowBits.W), 0.U(p.colBits.W), - p.transferChannel.U(p.channelBits.W), + p.totalChannels.U(p.channelBits.W), address(p.addressBits - 1, 0), write, false.B diff --git a/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/configs/Geometry.scala b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/configs/Geometry.scala new file mode 100644 index 000000000..58aa8e360 --- /dev/null +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/configs/Geometry.scala @@ -0,0 +1,11 @@ +package framework.ip.memory.mesh_shm.configs + +import upickle.default.{macroRW, ReadWriter} + +case class Geometry(dataBits: Int, entriesPerBank: Int, banks: Int) { + require(dataBits == 128 && entriesPerBank > 0 && banks > 0) + val bankBytes: Int = entriesPerBank * (dataBits / 8) + val bytes: BigInt = BigInt(banks) * bankBytes +} + +object Geometry { implicit val rw: ReadWriter[Geometry] = macroRW } diff --git a/arch/src/main/scala/framework/ip/spm/build.sc b/arch/src/main/scala/framework/ip/spm/build.sc new file mode 100644 index 000000000..26bc47687 --- /dev/null +++ b/arch/src/main/scala/framework/ip/spm/build.sc @@ -0,0 +1,22 @@ +import mill._ +import $file.^.^.^.^.^.^.settings + +val ipRoot: os.Path = millSourcePath + +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.ip.memory.spm.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Interfaces.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Memory.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/ip/spm/configs/uvm.toml b/arch/src/main/scala/framework/ip/spm/configs/uvm.toml new file mode 100644 index 000000000..669b7cf8c --- /dev/null +++ b/arch/src/main/scala/framework/ip/spm/configs/uvm.toml @@ -0,0 +1,6 @@ +[parameters] +base = "4096" +bytes = 256 +dataBits = 128 + +[defines] diff --git a/arch/src/main/scala/framework/mem-core/spm/src/csrc/src/lib.rs b/arch/src/main/scala/framework/ip/spm/src/csrc/src/lib.rs similarity index 100% rename from arch/src/main/scala/framework/mem-core/spm/src/csrc/src/lib.rs rename to arch/src/main/scala/framework/ip/spm/src/csrc/src/lib.rs diff --git a/arch/src/main/scala/framework/ip/spm/src/main/resources/agent/agent.svh b/arch/src/main/scala/framework/ip/spm/src/main/resources/agent/agent.svh new file mode 100644 index 000000000..6d2a96092 --- /dev/null +++ b/arch/src/main/scala/framework/ip/spm/src/main/resources/agent/agent.svh @@ -0,0 +1,19 @@ +class agent extends uvm_agent; + `uvm_component_utils(agent) + uvm_sequencer #(request_item) seqr; + driver drv; + monitor mon; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + seqr = uvm_sequencer#(request_item)::type_id::create("seqr", this); + drv = driver::type_id::create("driver", this); + mon = monitor::type_id::create("monitor", this); + endfunction + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + drv.seq_item_port.connect(seqr.seq_item_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/spm/src/main/resources/agent/driver.svh b/arch/src/main/scala/framework/ip/spm/src/main/resources/agent/driver.svh new file mode 100644 index 000000000..409427d6a --- /dev/null +++ b/arch/src/main/scala/framework/ip/spm/src/main/resources/agent/driver.svh @@ -0,0 +1,55 @@ +class driver extends uvm_driver #(request_item); + `uvm_component_utils(driver) + virtual port_if vif; + reset_controller resetter; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual port_if)::get( + this, "", "vif", vif + ) || !uvm_config_db#(reset_controller)::get( + this, "", "resetter", resetter + )) + `uvm_fatal("INTERFACE", "SPM driver requires ports and reset controller") + endfunction + task run_phase(uvm_phase phase); + request_item req; + vif.request_valid = 0; + vif.response_ready = 0; + vif.address = 0; + vif.size = 0; + vif.write = 0; + vif.data = 0; + vif.mask = 0; + vif.read_only = 0; + forever begin + seq_item_port.get_next_item(req); + do @(negedge vif.clock); while (vif.reset); + vif.address = req.address; + vif.size = req.size; + vif.write = req.write; + vif.data = req.data; + vif.mask = req.mask; + vif.read_only = req.read_only; + vif.stall = req.stall; + vif.cancel = req.cancel; + vif.request_valid = 1; + do @(posedge vif.clock); while (!vif.request_ready); + @(negedge vif.clock); + vif.request_valid = 0; + if (req.cancel) begin + resetter.pulse(2); + end else begin + while (!vif.response_valid) @(negedge vif.clock); + repeat (req.stall) @(negedge vif.clock); + vif.response_ready = 1; + @(posedge vif.clock); + @(negedge vif.clock); + vif.response_ready = 0; + end + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/spm/src/main/resources/agent/item.svh b/arch/src/main/scala/framework/ip/spm/src/main/resources/agent/item.svh new file mode 100644 index 000000000..ba4693dd3 --- /dev/null +++ b/arch/src/main/scala/framework/ip/spm/src/main/resources/agent/item.svh @@ -0,0 +1,35 @@ +class request_item extends uvm_sequence_item; + logic [63:0] address; + logic [$clog2($clog2(`SPM_BITS/8)+2)-1:0] size; + logic write; + logic [`SPM_BITS-1:0] data; + logic [`SPM_BITS/8-1:0] mask; + logic read_only; + int unsigned stall; + logic cancel; + `uvm_object_utils_begin(request_item) + `uvm_field_int(address, UVM_ALL_ON) + `uvm_field_int(size, UVM_ALL_ON) + `uvm_field_int(write, UVM_ALL_ON) + `uvm_field_int(data, UVM_ALL_ON) + `uvm_field_int(mask, UVM_ALL_ON) + `uvm_field_int(read_only, UVM_ALL_ON) + `uvm_field_int(stall, UVM_ALL_ON) + `uvm_field_int(cancel, UVM_ALL_ON) + `uvm_object_utils_end + function new(string name = "request_item"); + super.new(name); + endfunction +endclass + +class response_item extends uvm_sequence_item; + logic [`SPM_BITS-1:0] value; + logic error; + `uvm_object_utils_begin(response_item) + `uvm_field_int(value, UVM_ALL_ON) + `uvm_field_int(error, UVM_ALL_ON) + `uvm_object_utils_end + function new(string name = "response_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/spm/src/main/resources/agent/monitor.svh b/arch/src/main/scala/framework/ip/spm/src/main/resources/agent/monitor.svh new file mode 100644 index 000000000..b8a8c2317 --- /dev/null +++ b/arch/src/main/scala/framework/ip/spm/src/main/resources/agent/monitor.svh @@ -0,0 +1,56 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual port_if vif; + uvm_analysis_port #(request_item) requests; + uvm_analysis_port #(response_item) responses; + uvm_analysis_port #(bit) reset_ap; + function new(string name, uvm_component parent); + super.new(name, parent); + requests = new("requests", this); + responses = new("responses", this); + reset_ap = new("reset_ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual port_if)::get(this, "", "vif", vif)) + `uvm_fatal("INTERFACE", "SPM monitor requires ports") + endfunction + task run_phase(uvm_phase phase); + bit pending = 0; + request_item req; + response_item rsp; + forever begin + @(posedge vif.clock); + if (vif.reset) begin + pending = 0; + reset_ap.write(1); + end else begin + if (!pending && vif.busy) `uvm_fatal("BUSY", "SPM is busy without an accepted request") + if (vif.request_valid && vif.request_ready) begin + if (pending) `uvm_fatal("OUTSTANDING", "SPM accepted a second request") + req = request_item::type_id::create("request"); + req.address = vif.address; + req.size = vif.size; + req.write = vif.write; + req.data = vif.data; + req.mask = vif.mask; + req.read_only = vif.read_only; + req.stall = vif.stall; + req.cancel = vif.cancel; + requests.write(req); + pending = 1; + end else if (pending && (vif.request_ready || !vif.busy)) + `uvm_fatal("BUSY", "SPM reopened its request port before completion") + if (vif.response_valid && !pending) + `uvm_fatal("EXTRA_RESULT", "SPM produced an unsolicited result") + if (vif.response_valid && vif.response_ready) begin + rsp = response_item::type_id::create("response"); + rsp.value = vif.value; + rsp.error = vif.error; + responses.write(rsp); + pending = 0; + end + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/spm/src/main/resources/env/coverage.svh b/arch/src/main/scala/framework/ip/spm/src/main/resources/env/coverage.svh new file mode 100644 index 000000000..c487cca6d --- /dev/null +++ b/arch/src/main/scala/framework/ip/spm/src/main/resources/env/coverage.svh @@ -0,0 +1,27 @@ +class coverage extends uvm_subscriber #(request_item); + `uvm_component_utils(coverage) + covergroup behavior with function sample (request_item t); + option.per_instance = 1; + operation: coverpoint t.write; + size: coverpoint t.size { + bins legal[] = {[0 : $clog2(`SPM_BITS / 8)]}; + bins oversized = {[$clog2(`SPM_BITS / 8) + 1 : $]}; + } + lower_bound: coverpoint (t.address < `SPM_BASE) {bins seen = {1};} + upper_bound: coverpoint (t.address >= `SPM_BASE + `SPM_BYTES) {bins seen = {1};} + unaligned: coverpoint ((t.address & ((64'd1 << t.size) - 1)) != 0); + read_only: coverpoint t.read_only; + protected_write: coverpoint (t.read_only && t.write) {bins seen = {1};} + empty_mask: coverpoint (t.mask == 0) iff (t.write); + full_mask: coverpoint (&t.mask) iff (t.write); + backpressure: coverpoint t.stall {bins stalls[] = {0, 3};} + cancellation: coverpoint t.cancel; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + behavior = new(); + endfunction + function void write(request_item t); + behavior.sample(t); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/spm/src/main/resources/env/env.svh b/arch/src/main/scala/framework/ip/spm/src/main/resources/env/env.svh new file mode 100644 index 000000000..8bd17acae --- /dev/null +++ b/arch/src/main/scala/framework/ip/spm/src/main/resources/env/env.svh @@ -0,0 +1,26 @@ +`uvm_analysis_imp_decl(_reset) +class env extends environment #(request_item, response_item, model); + `uvm_component_utils(env) + agent bus; + coverage cov; + uvm_analysis_imp_reset #(bit, env) reset_export; + function new(string name, uvm_component parent); + super.new(name, parent); + reset_export = new("reset_export", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + cov = coverage::type_id::create("coverage", this); + bus = agent::type_id::create("agent", this); + endfunction + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + bus.mon.requests.connect(input_export); + bus.mon.requests.connect(cov.analysis_export); + bus.mon.reset_ap.connect(reset_export); + bus.mon.responses.connect(output_export); + endfunction + function void write_reset(bit asserted); + scoreboard.cancel_pending(); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/spm/src/main/resources/env/model.svh b/arch/src/main/scala/framework/ip/spm/src/main/resources/env/model.svh new file mode 100644 index 000000000..74975e1e2 --- /dev/null +++ b/arch/src/main/scala/framework/ip/spm/src/main/resources/env/model.svh @@ -0,0 +1,37 @@ +class model extends reference_model #(request_item, response_item); + `uvm_component_utils(model) + byte unsigned memory[`SPM_BYTES]; + bit initialized[`SPM_BYTES]; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void write(request_item req); + response_item rsp = response_item::type_id::create("expected"); + longint unsigned base = `SPM_BASE; + longint unsigned count = 64'd1 << req.size; + logic [64:0] finish = {1'b0, req.address} + count; + logic [64:0] limit = {1'b0, base} + `SPM_BYTES; + logic [`SPM_BITS/8-1:0] allowed = (`SPM_BITS / 8)'((64'd1 << count) - 1); + bit legal = count <= `SPM_BITS/8 && (req.address & (count-1)) == 0 && + req.address >= base && finish <= limit && + !(req.write && (req.read_only || (req.mask & ~allowed) != 0)); + int offset = req.address - base; + rsp.value = 0; + rsp.error = !legal; + if (legal) begin + for (int i = 0; i < count; i++) begin + if (req.write) begin + if (req.mask[i]) begin + memory[offset+i] = req.data[8*i+:8]; + initialized[offset+i] = 1; + end + end else begin + if (!initialized[offset+i]) + `uvm_fatal("UNINITIALIZED", "SPM read before byte initialization") + rsp.value[8*i+:8] = memory[offset+i]; + end + end + end + expected_ap.write(rsp); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/spm/src/main/resources/filelist.f b/arch/src/main/scala/framework/ip/spm/src/main/resources/filelist.f new file mode 100644 index 000000000..aa324abc6 --- /dev/null +++ b/arch/src/main/scala/framework/ip/spm/src/main/resources/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@/Memory ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv ++incdir+@RESOURCES@ +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/Memory/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/ip/spm/src/main/resources/interface.sv b/arch/src/main/scala/framework/ip/spm/src/main/resources/interface.sv new file mode 100644 index 000000000..63a052ef2 --- /dev/null +++ b/arch/src/main/scala/framework/ip/spm/src/main/resources/interface.sv @@ -0,0 +1,27 @@ +`include "parameters.svh" +interface port_if ( + input logic clock, + input logic reset +); + logic request_valid, request_ready, response_valid, response_ready, busy; + logic [63:0] address; + logic [$clog2($clog2(`SPM_BITS/8)+2)-1:0] size; + logic write; + logic [`SPM_BITS-1:0] data; + logic [`SPM_BITS/8-1:0] mask; + logic read_only; + int unsigned stall; + logic cancel; + logic [`SPM_BITS-1:0] value; + logic error; + assert property (@(posedge clock) disable iff (reset) + request_valid && !request_ready |=> request_valid && $stable( + {address, size, write, data, mask, read_only} + )) + else $fatal(1, "SPM request changed under backpressure"); + assert property (@(posedge clock) disable iff (reset) + response_valid && !response_ready |=> response_valid && $stable( + {value, error} + )) + else $fatal(1, "SPM response changed under backpressure"); +endinterface diff --git a/arch/src/main/scala/framework/ip/spm/src/main/resources/package.sv b/arch/src/main/scala/framework/ip/spm/src/main/resources/package.sv new file mode 100644 index 000000000..d7c844acb --- /dev/null +++ b/arch/src/main/scala/framework/ip/spm/src/main/resources/package.sv @@ -0,0 +1,16 @@ +`include "parameters.svh" +package spm_pkg; + timeunit 1ns; timeprecision 1ps; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/agent.svh" + `include "env/model.svh" + `include "env/coverage.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/ip/spm/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/ip/spm/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..61e840506 --- /dev/null +++ b/arch/src/main/scala/framework/ip/spm/src/main/resources/seq/sequence.svh @@ -0,0 +1,53 @@ +class stimulus extends uvm_sequence #(request_item); + `uvm_object_utils(stimulus) + int unsigned sent = 0; + function new(string name = "stimulus"); + super.new(name); + endfunction + task access (longint unsigned address, int size, bit write, logic [`SPM_BITS-1:0] data, + logic [`SPM_BITS/8-1:0] mask, bit read_only, int stall, bit cancel); + request_item req = request_item::type_id::create("request"); + start_item(req); + req.address = address; + req.size = size; + req.write = write; + req.data = data; + req.mask = mask; + req.read_only = read_only; + req.stall = stall; + req.cancel = cancel; + finish_item(req); + if (!cancel) sent++; + endtask + task body(); + int beat_bytes = `SPM_BITS / 8; + int maximum = $clog2(beat_bytes); + logic [`SPM_BITS-1:0] pattern; + for (int offset = 0; offset < `SPM_BYTES; offset += beat_bytes) begin + for (int byte_ = 0; byte_ < beat_bytes; byte_++) pattern[8*byte_+:8] = offset + byte_; + access (`SPM_BASE + offset, maximum, 1, pattern, '1, 0, 0, 0); + end + for (int size = 0; size <= maximum; size++) begin + for (int offset = 0; offset < `SPM_BYTES; offset += (1 << size)) + access (`SPM_BASE + offset, size, 0, 0, 0, 0, 3, 0); + for (int bit_ = 0; bit_ < (1 << size); bit_++) begin + access (`SPM_BASE + beat_bytes - (1 << size), size, 1, '1, 1 << bit_, 0, 3, 0); + access (`SPM_BASE + beat_bytes - (1 << size), size, 0, 0, 0, 0, 0, 0); + end + end + access (`SPM_BASE, maximum, 1, 0, 0, 0, 0, 0); + access (`SPM_BASE, maximum, 0, 0, 0, 0, 0, 0); + access (`SPM_BASE - 1, 0, 0, 0, 0, 0, 0, 0); + access (`SPM_BASE + `SPM_BYTES, 0, 0, 0, 0, 0, 3, 0); + access (64'hffffffffffffffff, 1, 0, 0, 0, 0, 0, 0); + access (`SPM_BASE + 1, 1, 0, 0, 0, 0, 3, 0); + access (`SPM_BASE, maximum + 1, 0, 0, 0, 0, 0, 0); + access (`SPM_BASE, 0, 1, 0, 2, 0, 0, 0); + access (`SPM_BASE, maximum, 1, 0, '1, 1, 3, 0); + access (`SPM_BASE, maximum, 0, 0, 0, 1, 0, 0); + access (`SPM_BASE, maximum, 1, '1, '1, 0, 0, 1); + access (`SPM_BASE, maximum, 0, 0, 0, 0, 0, 0); + access (`SPM_BASE, maximum, 0, 0, 0, 0, 0, 1); + access (`SPM_BASE, maximum, 0, 0, 0, 0, 3, 0); + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/spm/src/main/resources/suite.toml b/arch/src/main/scala/framework/ip/spm/src/main/resources/suite.toml new file mode 100644 index 000000000..d513c6055 --- /dev/null +++ b/arch/src/main/scala/framework/ip/spm/src/main/resources/suite.toml @@ -0,0 +1,20 @@ +[model] +manifest = "" +library = "" + +[[tests]] +name = "spm" +top = "spm_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 1000000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +functional_groups = ["spm_pkg::coverage::behavior"] +sink_ready = 1 +sink_stall = 0 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/ip/spm/src/main/resources/tb.sv b/arch/src/main/scala/framework/ip/spm/src/main/resources/tb.sv new file mode 100644 index 000000000..f4dcabc18 --- /dev/null +++ b/arch/src/main/scala/framework/ip/spm/src/main/resources/tb.sv @@ -0,0 +1,33 @@ +module spm_tb; + import uvm_pkg::*; + import spm_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + clock_if timing (clock); + port_if bus ( + clock, + timing.reset + ); + Memory dut ( + .clock(clock), + .reset(bus.reset), + .io_port_request_valid(bus.request_valid), + .io_port_request_ready(bus.request_ready), + .io_port_request_bits_address(bus.address), + .io_port_request_bits_size(bus.size), + .io_port_request_bits_write(bus.write), + .io_port_request_bits_data(bus.data), + .io_port_request_bits_mask(bus.mask), + .io_readOnly(bus.read_only), + .io_busy(bus.busy), + .io_port_response_valid(bus.response_valid), + .io_port_response_ready(bus.response_ready), + .io_port_response_bits_data(bus.value), + .io_port_response_bits_error(bus.error) + ); + initial begin + uvm_config_db#(virtual clock_if)::set(null, "uvm_test_top*", "timing", timing); + uvm_config_db#(virtual port_if)::set(null, "uvm_test_top*", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/ip/spm/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/ip/spm/src/main/resources/tests/test.svh new file mode 100644 index 000000000..81a4001e5 --- /dev/null +++ b/arch/src/main/scala/framework/ip/spm/src/main/resources/tests/test.svh @@ -0,0 +1,27 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env bench; + reset_controller resetter; + virtual port_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + resetter = reset_controller::type_id::create("reset", this); + uvm_config_db#(reset_controller)::set(this, "env.agent.driver", "resetter", resetter); + bench = env::type_id::create("env", this); + if (!uvm_config_db#(virtual port_if)::get(this, "", "vif", vif)) + `uvm_fatal("INTERFACE", "Test interface is required") + endfunction + task execute(); + stimulus sequence_ = stimulus::type_id::create("sequence"); + resetter.pulse(4); + sequence_.start(bench.bus.seqr); + bench.scoreboard.wait_checked(sequence_.sent); + @(negedge vif.clock); + evidence::check(evidence::cancelled == 2, + "Reset did not cancel both outstanding SPM responses"); + evidence::check(bench.scoreboard.checked == sequence_.sent, "Comparison count mismatch"); + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/spm/src/main/scala/Emit.scala b/arch/src/main/scala/framework/ip/spm/src/main/scala/Emit.scala new file mode 100644 index 000000000..377c846d0 --- /dev/null +++ b/arch/src/main/scala/framework/ip/spm/src/main/scala/Emit.scala @@ -0,0 +1,24 @@ +package framework.ip.memory.spm + +import java.nio.file.{Files, Paths} + +object Emit extends App { + val Array("--config", config, "--output-dir", output) = args + val values = ujson.read(Files.readString(Paths.get(config))) + val p = Params(BigInt(values("base").str), values("bytes").num.toInt, values("dataBits").num.toInt) + val directory = Paths.get(output, "Memory") + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Memory(p), + args = Array("--target-dir", directory.toString, "--split-verilog") + ) + Files.writeString( + directory.resolve("parameters.svh"), + s"""`ifndef SPM_PARAMETERS +`define SPM_PARAMETERS +`define SPM_BASE 64'h${p.base.toString(16)} +`define SPM_BYTES ${p.bytes} +`define SPM_BITS ${p.dataBits} +`endif +""" + ) +} diff --git a/arch/src/main/scala/framework/ip/spm/src/main/scala/Interfaces.scala b/arch/src/main/scala/framework/ip/spm/src/main/scala/Interfaces.scala new file mode 100644 index 000000000..9a360f405 --- /dev/null +++ b/arch/src/main/scala/framework/ip/spm/src/main/scala/Interfaces.scala @@ -0,0 +1,35 @@ +package framework.ip.memory.spm + +import chisel3._ +import chisel3.util._ +import framework.ip.memory.bank.BankSetParams + +/** One local byte-address window. This does not describe a DDR mapping. */ +case class Params(base: BigInt, bytes: Int, dataBits: Int = 128) { + require(dataBits >= 64 && dataBits % 8 == 0 && isPow2(dataBits / 8)) + val beatBytes = dataBits / 8 + require(base >= 0 && base % beatBytes == 0 && base + bytes <= (BigInt(1) << 64)) + require(bytes >= 2 * beatBytes && isPow2(bytes)) + val laneBits = log2Ceil(beatBytes) + val sizeBits = log2Ceil(laneBits + 2) + val bank = BankSetParams(dataBits, banks = 1, entriesPerBank = bytes / beatBytes) +} + +/** Naturally aligned 1..beatBytes access. Data and mask start at address, not the SRAM row. */ +class Request(p: Params) extends Bundle { + val address = UInt(64.W) + val size = UInt(p.sizeBits.W) + val write = Bool() + val data = UInt(p.dataBits.W) + val mask = UInt(p.beatBytes.W) +} + +class Response(p: Params) extends Bundle { + val data = UInt(p.dataBits.W) + val error = Bool() +} + +class Port(p: Params) extends Bundle { + val request = Decoupled(new Request(p)) + val response = Flipped(Decoupled(new Response(p))) +} diff --git a/arch/src/main/scala/framework/ip/spm/src/main/scala/Memory.scala b/arch/src/main/scala/framework/ip/spm/src/main/scala/Memory.scala new file mode 100644 index 000000000..e28c51887 --- /dev/null +++ b/arch/src/main/scala/framework/ip/spm/src/main/scala/Memory.scala @@ -0,0 +1,60 @@ +package framework.ip.memory.spm + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import framework.ip.memory.bank.Bank + +/** + * One outstanding transaction on one physical read/write port. Writes acknowledge after + * SRAM acceptance; reads return after the synchronous read. Responses hold until accepted. + * Reset cancels protocol state; SRAM contents, including already accepted writes, remain. + */ +@instantiable +class Memory(p: Params) extends Module { + + @public + val io = IO(new Bundle { + val port = Flipped(new Port(p)) + val readOnly = Input(Bool()) + val busy = Output(Bool()) + }) + + val bank = Instantiate(new Bank(p.bank)) + val idle :: memory :: rejected :: Nil = Enum(3) + val state = RegInit(idle) + val offset = Reg(UInt(p.laneBits.W)) + val savedMask = Reg(UInt(p.beatBytes.W)) + val request = io.port.request.bits + val count = 1.U(64.W) << request.size + val end = request.address +& count + + val widthMask = MuxLookup(request.size, 0.U(p.beatBytes.W))( + (0 to p.laneBits).map(size => size.U -> ((BigInt(1) << (1 << size)) - 1).U(p.beatBytes.W)) + ) + + val legal = request.size <= p.laneBits.U && (request.address & (count - 1.U)) === 0.U && + request.address >= p.base.U && end <= (p.base + p.bytes).U && + !(request.write && (io.readOnly || (request.mask & ~widthMask) =/= 0.U)) + + val lane = request.address(p.laneBits - 1, 0) + io.port.request.ready := state === idle && !reset.asBool && Mux(legal, bank.io.request.ready, true.B) + io.busy := state =/= idle + bank.io.request.valid := state === idle && !reset.asBool && io.port.request.valid && legal + bank.io.request.bits.addr := (request.address - p.base.U) >> p.laneBits + bank.io.request.bits.write := request.write + bank.io.request.bits.data := request.data << (lane << 3) + bank.io.request.bits.mask := request.mask << lane + bank.io.request.bits.tag := 0.U + when(io.port.request.fire) { + offset := lane + savedMask := widthMask + state := Mux(legal, memory, rejected) + } + val dataMask = Cat((0 until p.beatBytes).reverse.map(i => Fill(8, savedMask(i)))) + io.port.response.valid := !reset.asBool && (state === rejected || (state === memory && bank.io.response.valid)) + io.port.response.bits.data := Mux(state === rejected, 0.U, (bank.io.response.bits.data >> (offset << 3)) & dataMask) + io.port.response.bits.error := state === rejected || bank.io.response.bits.error + bank.io.response.ready := state === memory && io.port.response.ready && !reset.asBool + when(io.port.response.fire)(state := idle) +} diff --git a/arch/src/main/scala/framework/lib/arbiter/Arbiter.scala b/arch/src/main/scala/framework/lib/arbiter/Arbiter.scala new file mode 100644 index 000000000..50c17eb75 --- /dev/null +++ b/arch/src/main/scala/framework/lib/arbiter/Arbiter.scala @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: Apache-2.0 +// Arbitration follows chisel3.util.Arbiter and RRArbiter. +package framework.lib.arbiter + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} + +@instantiable +class Arbiter[T <: Data](gen: T, n: Int) extends Module { + @public + val io = IO(new ArbiterIO(gen, n)) + + io.chosen := (n - 1).U + io.out.bits := io.in(n - 1).bits + for (i <- n - 2 to 0 by -1) { + when(io.in(i).valid) { + io.chosen := i.U + io.out.bits := io.in(i).bits + } + } + val preceding = io.in.map(_.valid).scanLeft(false.B)(_ || _).dropRight(1) + for ((input, busy) <- io.in.zip(preceding)) input.ready := !busy && io.out.ready + io.out.valid := io.in.map(_.valid).reduce(_ || _) +} + +@instantiable +class RRArbiter[T <: Data](gen: T, n: Int) extends Module { + @public + val io = IO(new ArbiterIO(gen, n)) + + val lastGrant = RegInit(0.U(math.max(1, log2Ceil(n)).W)) + val mask = (0 until n).map(_.U > lastGrant) + val masked = io.in.zip(mask).map { case (input, eligible) => input.valid && eligible } + val requests = masked ++ io.in.map(_.valid) + val grant = requests.scanLeft(false.B)(_ || _).dropRight(1).map(!_) + + io.chosen := (n - 1).U + for (i <- n - 2 to 0 by -1) when(io.in(i).valid)(io.chosen := i.U) + for (i <- n - 1 to 1 by -1) when(masked(i))(io.chosen := i.U) + for (i <- 0 until n) io.in(i).ready := (grant(i) && mask(i) || grant(i + n)) && io.out.ready + io.out.valid := io.in(io.chosen).valid + io.out.bits := io.in(io.chosen).bits + when(io.out.fire)(lastGrant := io.chosen) +} diff --git a/arch/src/main/scala/framework/lib/arbiter/build.sc b/arch/src/main/scala/framework/lib/arbiter/build.sc new file mode 100644 index 000000000..c17f7200e --- /dev/null +++ b/arch/src/main/scala/framework/lib/arbiter/build.sc @@ -0,0 +1,11 @@ +import mill._ + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("Arbiter.scala")) + ) +) + +def emitSources = T.sources( + Seq.empty[PathRef] +) diff --git a/arch/src/main/scala/framework/arith/src/main/scala/DivRem.scala b/arch/src/main/scala/framework/lib/arith/DivRem.scala similarity index 99% rename from arch/src/main/scala/framework/arith/src/main/scala/DivRem.scala rename to arch/src/main/scala/framework/lib/arith/DivRem.scala index 494378649..015963f6a 100644 --- a/arch/src/main/scala/framework/arith/src/main/scala/DivRem.scala +++ b/arch/src/main/scala/framework/lib/arith/DivRem.scala @@ -1,4 +1,4 @@ -package framework.arith +package framework.lib.arith import chisel3._ import chisel3.util._ diff --git a/arch/src/main/scala/framework/lib/arith/Fma.scala b/arch/src/main/scala/framework/lib/arith/Fma.scala new file mode 100644 index 000000000..6f2a765d1 --- /dev/null +++ b/arch/src/main/scala/framework/lib/arith/Fma.scala @@ -0,0 +1,89 @@ +// See LICENSE.Berkeley and LICENSE.SiFive for license details. +package framework.lib.arith + +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} + +@instantiable +class Fma(latency: Int, expWidth: Int, sigWidth: Int) extends Module { + override def desiredName = s"Fma_l${latency}_e${expWidth}_s$sigWidth" + require(latency <= 4) + + @public + val io = IO(new Bundle { + val validin = Input(Bool()) + val op = Input(Bits(2.W)) + val a = Input(Bits((expWidth + sigWidth + 1).W)) + val b = Input(Bits((expWidth + sigWidth + 1).W)) + val c = Input(Bits((expWidth + sigWidth + 1).W)) + val roundingMode = Input(UInt(3.W)) + val detectTininess = Input(UInt(1.W)) + val out = Output(Bits((expWidth + sigWidth + 1).W)) + val exceptionFlags = Output(Bits(5.W)) + val validout = Output(Bool()) + }) + + //------------------------------------------------------------------------ + //------------------------------------------------------------------------ + + val mulAddRecFNToRaw_preMul = Module(new hardfloat.MulAddRecFNToRaw_preMul(expWidth, sigWidth)) + val mulAddRecFNToRaw_postMul = Module(new hardfloat.MulAddRecFNToRaw_postMul(expWidth, sigWidth)) + + mulAddRecFNToRaw_preMul.io.op := io.op + mulAddRecFNToRaw_preMul.io.a := io.a + mulAddRecFNToRaw_preMul.io.b := io.b + mulAddRecFNToRaw_preMul.io.c := io.c + + // Isolate preMul's unpack/alignment logic from the multiplier-add stage. + val preMulRegs = if (latency >= 3) 1 else 0 + val preMulMulAddA = Pipe(io.validin, mulAddRecFNToRaw_preMul.io.mulAddA, preMulRegs) + val preMulMulAddB = Pipe(io.validin, mulAddRecFNToRaw_preMul.io.mulAddB, preMulRegs) + val preMulMulAddC = Pipe(io.validin, mulAddRecFNToRaw_preMul.io.mulAddC, preMulRegs) + val preMulToPostMul = Pipe(io.validin, mulAddRecFNToRaw_preMul.io.toPostMul, preMulRegs) + + // Keep the full-precision product unrounded while separating multiplication + // from the final addition. The aligned addend and post-processing metadata + // must cross this stage with the product to preserve fused FMA semantics. + val mulRegs = if (latency == 4) 1 else 0 + val mulResult = preMulMulAddA.bits * preMulMulAddB.bits + val mulResultPipe = Pipe(preMulToPostMul.valid, mulResult, mulRegs) + val mulAddCPipe = Pipe(preMulToPostMul.valid, preMulMulAddC.bits, mulRegs) + val mulToPostMul = Pipe(preMulToPostMul.valid, preMulToPostMul.bits, mulRegs) + val mulAddResult = mulResultPipe.bits +& mulAddCPipe.bits + + val valid_stage0 = Wire(Bool()) + val roundingMode_stage0 = Wire(UInt(3.W)) + val detectTininess_stage0 = Wire(UInt(1.W)) + + val postmul_regs = if (latency > 0) 1 else 0 + mulAddRecFNToRaw_postMul.io.fromPreMul := Pipe(mulToPostMul.valid, mulToPostMul.bits, postmul_regs).bits + mulAddRecFNToRaw_postMul.io.mulAddResult := Pipe(mulToPostMul.valid, mulAddResult, postmul_regs).bits + mulAddRecFNToRaw_postMul.io.roundingMode := Pipe( + io.validin, + io.roundingMode, + preMulRegs + mulRegs + postmul_regs + ).bits + roundingMode_stage0 := Pipe(io.validin, io.roundingMode, preMulRegs + mulRegs + postmul_regs).bits + detectTininess_stage0 := Pipe(io.validin, io.detectTininess, preMulRegs + mulRegs + postmul_regs).bits + valid_stage0 := Pipe(io.validin, false.B, preMulRegs + mulRegs + postmul_regs).valid + + //------------------------------------------------------------------------ + //------------------------------------------------------------------------ + + val roundRawFNToRecFN = Module(new hardfloat.RoundRawFNToRecFN(expWidth, sigWidth, 0)) + + val round_regs = if (latency >= 2) 1 else 0 + roundRawFNToRecFN.io.invalidExc := Pipe(valid_stage0, mulAddRecFNToRaw_postMul.io.invalidExc, round_regs).bits + roundRawFNToRecFN.io.in := Pipe(valid_stage0, mulAddRecFNToRaw_postMul.io.rawOut, round_regs).bits + roundRawFNToRecFN.io.roundingMode := Pipe(valid_stage0, roundingMode_stage0, round_regs).bits + roundRawFNToRecFN.io.detectTininess := Pipe(valid_stage0, detectTininess_stage0, round_regs).bits + io.validout := Pipe(valid_stage0, false.B, round_regs).valid + + roundRawFNToRecFN.io.infiniteExc := false.B + + io.out := roundRawFNToRecFN.io.out + io.exceptionFlags := roundRawFNToRecFN.io.exceptionFlags +} diff --git a/arch/src/main/scala/framework/lib/arith/IntToFloat.scala b/arch/src/main/scala/framework/lib/arith/IntToFloat.scala new file mode 100644 index 000000000..dc02170cd --- /dev/null +++ b/arch/src/main/scala/framework/lib/arith/IntToFloat.scala @@ -0,0 +1,109 @@ +// See LICENSE.Berkeley and LICENSE.SiFive for license details. +package framework.lib.arith + +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} + +@instantiable +class IntToFloat(intWidth: Int, expWidth: Int, sigWidth: Int) extends Module { + + @public + val io = IO(new Bundle { + val validin = Input(Bool()) + val signedIn = Input(Bool()) + val in = Input(Bits(intWidth.W)) + val roundingMode = Input(UInt(3.W)) + val detectTininess = Input(UInt(1.W)) + val out = Output(Bits((expWidth + sigWidth + 1).W)) + val exceptionFlags = Output(Bits(5.W)) + val validout = Output(Bool()) + }) + + val rawExpWidth = log2Up(intWidth) + 1 + val extIntWidth = 1 << (rawExpWidth - 1) + val normalizationLevels = rawExpWidth - 1 + val firstStageLevels = 2 + val secondStageLevels = normalizationLevels - firstStageLevels + require(secondStageLevels > 0) + + val sign = io.signedIn && io.in(intWidth - 1) + val absIn = Mux(sign, -io.in.asUInt, io.in.asUInt) + val extAbsIn = (0.U(extIntWidth.W) ## absIn)(extIntWidth - 1, 0) + + // Complete the 32- and 16-bit normalization decisions before the cut. + val firstNormStages = Wire(Vec(firstStageLevels + 1, UInt(extIntWidth.W))) + val firstNormDist = Wire(Vec(firstStageLevels, Bool())) + firstNormStages(0) := extAbsIn + for (level <- 0 until firstStageLevels) { + val shiftAmt = extIntWidth >> (level + 1) + val upperHalfIsZero = + !firstNormStages(level)(extIntWidth - 1, extIntWidth - shiftAmt).orR + firstNormStages(level + 1) := Mux( + upperHalfIsZero, + (firstNormStages(level) << shiftAmt)(extIntWidth - 1, 0), + firstNormStages(level) + ) + firstNormDist(firstStageLevels - 1 - level) := upperHalfIsZero + } + + val midNormStage = RegEnable(firstNormStages(firstStageLevels), io.validin) + val midNormDist = RegEnable(firstNormDist.asUInt, io.validin) + val midSign = RegEnable(sign, io.validin) + val midRoundingMode = RegEnable(io.roundingMode, io.validin) + val midDetectTininess = RegEnable(io.detectTininess, io.validin) + val midValid = RegNext(io.validin, false.B) + + val secondNormStages = Wire(Vec(secondStageLevels + 1, UInt(extIntWidth.W))) + val secondNormDist = Wire(Vec(secondStageLevels, Bool())) + secondNormStages(0) := midNormStage + for (offset <- 0 until secondStageLevels) { + val level = firstStageLevels + offset + val shiftAmt = extIntWidth >> (level + 1) + val upperHalfIsZero = + !secondNormStages(offset)(extIntWidth - 1, extIntWidth - shiftAmt).orR + secondNormStages(offset + 1) := Mux( + upperHalfIsZero, + (secondNormStages(offset) << shiftAmt)(extIntWidth - 1, 0), + secondNormStages(offset) + ) + secondNormDist(secondStageLevels - 1 - offset) := upperHalfIsZero + } + + val adjustedNormDist = Cat(midNormDist, secondNormDist.asUInt) + val sig = secondNormStages(secondStageLevels)(extIntWidth - 1, extIntWidth - intWidth) + val intAsRawFloat = Wire(new hardfloat.RawFloat(rawExpWidth, intWidth)) + intAsRawFloat.isNaN := false.B + intAsRawFloat.isInf := false.B + intAsRawFloat.isZero := !sig(intWidth - 1) + intAsRawFloat.sign := midSign + intAsRawFloat.sExp := (2.U(2.W) ## ~adjustedNormDist).zext + intAsRawFloat.sig := sig + + // Isolate the remaining normalization and raw-float construction from the + // target-format rounding and packing logic. + val roundInput = RegEnable(intAsRawFloat, midValid) + val roundRoundingMode = RegEnable(midRoundingMode, midValid) + val roundDetectTininess = RegEnable(midDetectTininess, midValid) + val roundValid = RegNext(midValid, false.B) + + val roundAnyRawFNToRecFN = Module(new hardfloat.RoundAnyRawFNToRecFN( + rawExpWidth, + intWidth, + expWidth, + sigWidth, + hardfloat.consts.flRoundOpt_sigMSBitAlwaysZero | hardfloat.consts.flRoundOpt_neverUnderflows + )) + + roundAnyRawFNToRecFN.io.invalidExc := false.B + roundAnyRawFNToRecFN.io.infiniteExc := false.B + roundAnyRawFNToRecFN.io.in := roundInput + roundAnyRawFNToRecFN.io.roundingMode := roundRoundingMode + roundAnyRawFNToRecFN.io.detectTininess := roundDetectTininess + + io.out := roundAnyRawFNToRecFN.io.out + io.exceptionFlags := roundAnyRawFNToRecFN.io.exceptionFlags + io.validout := roundValid +} diff --git a/arch/src/main/scala/framework/lib/arith/LICENSE.Berkeley b/arch/src/main/scala/framework/lib/arith/LICENSE.Berkeley new file mode 100644 index 000000000..7cff15e40 --- /dev/null +++ b/arch/src/main/scala/framework/lib/arith/LICENSE.Berkeley @@ -0,0 +1,24 @@ +Copyright (c) 2012-2014, The Regents of the University of California +(Regents). All Rights Reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. Neither the name of the Regents nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, +SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING +OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS +BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED +HEREUNDER IS PROVIDED "AS IS". REGENTS HAS NO OBLIGATION TO PROVIDE +MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. diff --git a/arch/src/main/scala/framework/lib/arith/LICENSE.SiFive b/arch/src/main/scala/framework/lib/arith/LICENSE.SiFive new file mode 100644 index 000000000..7e7093375 --- /dev/null +++ b/arch/src/main/scala/framework/lib/arith/LICENSE.SiFive @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2016-2017 SiFive, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/arch/src/main/scala/framework/arith/src/main/scala/Multiply.scala b/arch/src/main/scala/framework/lib/arith/Multiply.scala similarity index 96% rename from arch/src/main/scala/framework/arith/src/main/scala/Multiply.scala rename to arch/src/main/scala/framework/lib/arith/Multiply.scala index 9e025a715..bac5e2960 100644 --- a/arch/src/main/scala/framework/arith/src/main/scala/Multiply.scala +++ b/arch/src/main/scala/framework/lib/arith/Multiply.scala @@ -1,4 +1,4 @@ -package framework.arith +package framework.lib.arith import chisel3._ import chisel3.util._ @@ -16,7 +16,8 @@ class MultiplyRequest extends Bundle { @instantiable class Multiply extends Module { - @public val io = IO(new Bundle { + @public + val io = IO(new Bundle { val request = Flipped(Decoupled(new MultiplyRequest)) val response = Decoupled(UInt(64.W)) }) diff --git a/arch/src/main/scala/framework/lib/arith/build.sc b/arch/src/main/scala/framework/lib/arith/build.sc new file mode 100644 index 000000000..6046890ef --- /dev/null +++ b/arch/src/main/scala/framework/lib/arith/build.sc @@ -0,0 +1,14 @@ +import mill._ + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("DivRem.scala")), + PathRef(millSourcePath / os.RelPath("Fma.scala")), + PathRef(millSourcePath / os.RelPath("IntToFloat.scala")), + PathRef(millSourcePath / os.RelPath("Multiply.scala")) + ) +) + +def emitSources = T.sources( + Seq.empty[PathRef] +) diff --git a/arch/src/main/scala/framework/lib/build.sc b/arch/src/main/scala/framework/lib/build.sc new file mode 100644 index 000000000..b9aee3969 --- /dev/null +++ b/arch/src/main/scala/framework/lib/build.sc @@ -0,0 +1,22 @@ +import mill._ +import $file.arbiter.{build => arbiter_build} +import $file.arith.{build => arith_build} +import $file.dpi.{build => dpi_build} +import $file.hash.{build => hash_build} +import $file.queue.{build => queue_build} + +def sources = T.sources( + arbiter_build.sources() ++ + arith_build.sources() ++ + dpi_build.sources() ++ + hash_build.sources() ++ + queue_build.sources() +) + +def emitSources = T.sources( + arbiter_build.emitSources() ++ + arith_build.emitSources() ++ + dpi_build.emitSources() ++ + hash_build.emitSources() ++ + queue_build.emitSources() +) diff --git a/arch/src/main/scala/framework/dpi/DpiGuard.scala b/arch/src/main/scala/framework/lib/dpi/DpiGuard.scala similarity index 96% rename from arch/src/main/scala/framework/dpi/DpiGuard.scala rename to arch/src/main/scala/framework/lib/dpi/DpiGuard.scala index f85a18c32..7e28200d2 100644 --- a/arch/src/main/scala/framework/dpi/DpiGuard.scala +++ b/arch/src/main/scala/framework/lib/dpi/DpiGuard.scala @@ -1,4 +1,4 @@ -package framework.dpi +package framework.lib.dpi object DpiGuard { private def wrapUnless(macroName: String, body: String): String = diff --git a/arch/src/main/scala/framework/lib/dpi/build.sc b/arch/src/main/scala/framework/lib/dpi/build.sc new file mode 100644 index 000000000..c54afe76c --- /dev/null +++ b/arch/src/main/scala/framework/lib/dpi/build.sc @@ -0,0 +1,11 @@ +import mill._ + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("DpiGuard.scala")) + ) +) + +def emitSources = T.sources( + Seq.empty[PathRef] +) diff --git a/arch/src/main/scala/framework/lib/hash/BankHash.scala b/arch/src/main/scala/framework/lib/hash/BankHash.scala new file mode 100644 index 000000000..21f26b0ad --- /dev/null +++ b/arch/src/main/scala/framework/lib/hash/BankHash.scala @@ -0,0 +1,75 @@ +package framework.lib.hash + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public} +import chisel3.util._ +import framework.acc.memdomain.configs.MemDomainParam + +class BankHashWrite(val b: MemDomainParam) extends Bundle { + val addr = UInt(log2Ceil(b.bankEntries).W) + val mask = Vec(b.bankMaskLen, Bool()) + val data = UInt(b.bankWidth.W) +} + +@instantiable +class BankHashMonitor(val b: MemDomainParam) extends Module { + require(b.bankWidth == 128, "bank hash requires 128-bit bank rows") + require(b.bankMaskLen == 16, "bank hash requires byte write masks") + + @public + val io = IO(new Bundle { + val bind = Input(Bool()) + val write = Flipped(Decoupled(new BankHashWrite(b))) + val statusHash = Output(UInt(32.W)) + }) + + val shadow = SyncReadMem(b.bankEntries, UInt(128.W)) + val validRows = RegInit(0.U(b.bankEntries.W)) + val generation = RegInit(0.U(8.W)) + val statusHash = RegInit(0.U(32.W)) + + def rotateLeft(value: UInt, amount: Int): UInt = + Cat(value(31 - amount, 0), value(31, 32 - amount)) + + def rowHash(addr: UInt, row: UInt): UInt = { + val address = addr.pad(32) + val mixed = row(31, 0) ^ rotateLeft(row(63, 32), 7) ^ rotateLeft(row(95, 64), 13) ^ + rotateLeft(row(127, 96), 21) ^ rotateLeft(address, 11) + Mux(row.orR, mixed, 0.U) + } + + val writeValid = RegInit(false.B) + val writeBits = Reg(new BankHashWrite(b)) + val writeGeneration = Reg(UInt(generation.getWidth.W)) + val commit = writeValid && writeGeneration === generation && !io.bind + io.write.ready := !writeValid && !io.bind + val oldRow = Wire(UInt(128.W)) + val oldBytes = oldRow.asTypeOf(Vec(16, UInt(8.W))) + val writeBytes = writeBits.data.asTypeOf(Vec(16, UInt(8.W))) + val newBytes = VecInit((0 until 16).map(i => Mux(writeBits.mask(i), writeBytes(i), oldBytes(i)))) + val newRow = newBytes.asUInt + val nextHash = statusHash - rowHash(writeBits.addr, oldRow) + rowHash(writeBits.addr, newRow) + + val readData = + shadow.readWrite(Mux(commit, writeBits.addr, io.write.bits.addr), newRow, io.write.fire || commit, commit) + oldRow := Mux(validRows(writeBits.addr), readData, 0.U) + + when(io.write.fire) { + writeValid := true.B + writeBits := io.write.bits + writeGeneration := generation + } + + io.statusHash := statusHash + + when(io.bind) { + generation := generation + 1.U + statusHash := 0.U + writeValid := false.B + validRows := 0.U + }.elsewhen(commit) { + validRows := validRows | UIntToOH(writeBits.addr, b.bankEntries) + statusHash := nextHash + writeValid := false.B + } +} diff --git a/arch/src/main/scala/framework/lib/hash/build.sc b/arch/src/main/scala/framework/lib/hash/build.sc new file mode 100644 index 000000000..ee1b17b54 --- /dev/null +++ b/arch/src/main/scala/framework/lib/hash/build.sc @@ -0,0 +1,11 @@ +import mill._ + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("BankHash.scala")) + ) +) + +def emitSources = T.sources( + Seq.empty[PathRef] +) diff --git a/arch/src/main/scala/framework/lib/hash/src/csrc/monitor.cpp b/arch/src/main/scala/framework/lib/hash/src/csrc/monitor.cpp new file mode 100644 index 000000000..6332142b9 --- /dev/null +++ b/arch/src/main/scala/framework/lib/hash/src/csrc/monitor.cpp @@ -0,0 +1,110 @@ +#include "VBankHashMonitor.h" +#include "verilated.h" +#include +#include +#include +#include +#include +#include + +int main(int argc, char **argv) { + Verilated::commandArgs(argc, argv); + if (argc != 2) + throw std::runtime_error("Expected bank row count"); + const unsigned rows = std::stoul(argv[1]); + VBankHashMonitor dut; + std::vector> contents(rows); + auto tick = [&] { + dut.clock = 0; + dut.eval(); + dut.clock = 1; + dut.eval(); + dut.clock = 0; + dut.eval(); + }; + auto check = [&](bool value, const char *message) { + if (!value) + throw std::runtime_error(message); + }; + auto rotate = [](uint32_t value, unsigned n) { + return (value << n) | (value >> (32 - n)); + }; + auto expected = [&] { + uint32_t hash = 0; + for (unsigned row = 0; row < rows; ++row) { + const auto &v = contents[row]; + if (v[0] | v[1] | v[2] | v[3]) + hash += v[0] ^ rotate(v[1], 7) ^ rotate(v[2], 13) ^ rotate(v[3], 21) ^ + rotate(row, 11); + } + return hash; + }; + auto mask = [&](uint16_t bits) { +#define BYTE(i) dut.io_write_bits_mask_##i = (bits >> i) & 1 + BYTE(0); + BYTE(1); + BYTE(2); + BYTE(3); + BYTE(4); + BYTE(5); + BYTE(6); + BYTE(7); + BYTE(8); + BYTE(9); + BYTE(10); + BYTE(11); + BYTE(12); + BYTE(13); + BYTE(14); + BYTE(15); +#undef BYTE + }; + dut.reset = 1; + dut.io_bind = 0; + dut.io_write_valid = 0; + tick(); + tick(); + dut.reset = 0; + tick(); + uint32_t random = 0x12345678; + for (unsigned round = 0; round < 3; ++round) { + for (unsigned row = 0; row < rows; ++row) { + check(dut.io_write_ready, "Write did not become ready"); + random = random * 1664525 + 1013904223; + const uint16_t bytes = round == 0 ? 0xffff : random; + dut.io_write_bits_addr = row; + for (unsigned word = 0; word < 4; ++word) { + random = random * 1664525 + 1013904223; + dut.io_write_bits_data[word] = random; + for (unsigned byte = 0; byte < 4; ++byte) { + uint32_t m = 0xffu << (byte * 8); + if ((bytes >> (word * 4 + byte)) & 1) + contents[row][word] = (contents[row][word] & ~m) | (random & m); + } + } + mask(bytes); + dut.io_write_valid = 1; + tick(); + check(!dut.io_write_ready, "Pending write did not apply backpressure"); + dut.io_write_valid = 0; + tick(); + check(dut.io_statusHash == expected(), "Masked write hash mismatch"); + } + if (round == 0) + continue; + dut.io_write_valid = 1; + tick(); + dut.io_bind = 1; + dut.eval(); + check(!dut.io_write_ready, "Bind accepted a concurrent write"); + tick(); + dut.io_write_valid = 0; + dut.io_bind = 0; + contents.assign(rows, {}); + tick(); + check(dut.io_statusHash == 0, "Bind did not cancel the pending write"); + } + std::puts("PASS: all rows, masked overwrite, rebind, cancellation and " + "backpressure"); + dut.final(); +} diff --git a/arch/src/main/scala/framework/lib/queue/Queue.scala b/arch/src/main/scala/framework/lib/queue/Queue.scala new file mode 100644 index 000000000..dc670b597 --- /dev/null +++ b/arch/src/main/scala/framework/lib/queue/Queue.scala @@ -0,0 +1,86 @@ +package framework.lib.queue + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public} +import chisel3.util.{log2Ceil, QueueIO} + +@instantiable +class Queue[T <: Data]( + gen: T, + entries: Int, + pipe: Boolean = false, + flow: Boolean = false) + extends Module { + require(entries > 0) + @public + val io = IO(new QueueIO(gen, entries)) + + val count = RegInit(0.U(log2Ceil(entries + 1).W)) + val memoryCount = RegInit(0.U(log2Ceil(entries + 1).W)) + val front = Reg(gen) + val frontValid = RegInit(false.B) + val skid = Reg(gen) + val skidValid = RegInit(false.B) + val readPending = RegInit(false.B) + + io.count := count + io.deq.valid := frontValid || (if (flow) count === 0.U && io.enq.valid else false.B) + io.deq.bits := (if (flow) Mux(frontValid, front, io.enq.bits) else front) + val dequeue = frontValid && io.deq.ready + io.enq.ready := count < entries.U || (if (pipe) dequeue else false.B) + val bypass = if (flow) count === 0.U && io.deq.fire else false.B + val enqueue = io.enq.fire && !bypass + val direct = enqueue && memoryCount === 0.U && !readPending && !skidValid && (!frontValid || dequeue) + val write = enqueue && !direct + + val nextFront = WireDefault(front) + val nextFrontValid = WireDefault(frontValid && !dequeue) + val nextSkid = WireDefault(skid) + val nextSkidValid = WireDefault(skidValid) + when((!frontValid || dequeue) && skidValid) { + nextFront := skid + nextFrontValid := true.B + nextSkidValid := false.B + } + if (entries > 2) { + val memory = SyncReadMem(entries, gen) + val head = RegInit(0.U(log2Ceil(entries).W)) + val tail = RegInit(0.U(log2Ceil(entries).W)) + val occupied = frontValid.asUInt +& skidValid.asUInt +& readPending.asUInt + val read = memoryCount =/= 0.U && !write && occupied < (2.U + dequeue.asUInt) + val readData = memory.readWrite(Mux(write, tail, head), io.enq.bits, write || read, write) + when(readPending) { + when((!frontValid || dequeue) && !skidValid) { + nextFront := readData + nextFrontValid := true.B + }.otherwise { + nextSkid := readData + nextSkidValid := true.B + } + } + readPending := read + when(write =/= read) { + memoryCount := Mux(write, memoryCount + 1.U, memoryCount - 1.U) + } + when(write)(tail := Mux(tail === (entries - 1).U, 0.U, tail + 1.U)) + when(read)(head := Mux(head === (entries - 1).U, 0.U, head + 1.U)) + } else { + // Front and skid registers hold up to two entries, so small queues need no memory + when(write) { + nextSkid := io.enq.bits + nextSkidValid := true.B + } + } + when(direct) { + nextFront := io.enq.bits + nextFrontValid := true.B + } + front := nextFront + frontValid := nextFrontValid + skid := nextSkid + skidValid := nextSkidValid + + when(enqueue =/= dequeue) { + count := Mux(enqueue, count + 1.U, count - 1.U) + } +} diff --git a/arch/src/main/scala/framework/lib/queue/build.sc b/arch/src/main/scala/framework/lib/queue/build.sc new file mode 100644 index 000000000..16493b39a --- /dev/null +++ b/arch/src/main/scala/framework/lib/queue/build.sc @@ -0,0 +1,11 @@ +import mill._ + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("Queue.scala")) + ) +) + +def emitSources = T.sources( + Seq.empty[PathRef] +) diff --git a/arch/src/main/scala/framework/mem-core/axi4/build.sc b/arch/src/main/scala/framework/mem-core/axi4/build.sc deleted file mode 100644 index 54655ecf4..000000000 --- a/arch/src/main/scala/framework/mem-core/axi4/build.sc +++ /dev/null @@ -1,20 +0,0 @@ -import mill._ -import mill.scalalib._ - -object queue extends SbtModule { - override def millSourcePath = os.pwd / os.up / "queue" - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") -} - -object axi4 extends SbtModule { - override def moduleDeps = Seq(queue) - override def millSourcePath = os.pwd - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") - override def mainClass = Some("memcore.bus.axi4.Emit") -} diff --git a/arch/src/main/scala/framework/mem-core/axi4/src/main/scala/Emit.scala b/arch/src/main/scala/framework/mem-core/axi4/src/main/scala/Emit.scala deleted file mode 100644 index c3031c191..000000000 --- a/arch/src/main/scala/framework/mem-core/axi4/src/main/scala/Emit.scala +++ /dev/null @@ -1,9 +0,0 @@ -package memcore.bus.axi4 - -object Emit extends App { - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Interconnect(Params(), masters = 2), - firtoolOpts = args, - args = Array("--target-dir", "build/Interconnect", "--split-verilog") - ) -} diff --git a/arch/src/main/scala/framework/mem-core/axi4/src/main/scala/Interface.scala b/arch/src/main/scala/framework/mem-core/axi4/src/main/scala/Interface.scala deleted file mode 100644 index e259e89bd..000000000 --- a/arch/src/main/scala/framework/mem-core/axi4/src/main/scala/Interface.scala +++ /dev/null @@ -1,44 +0,0 @@ -package memcore.bus.axi4 - -import chisel3._ -import chisel3.util._ - -class Address(p: Params) extends Bundle { - val id = UInt(p.idBits.W) - val addr = UInt(p.addressBits.W) - val len = UInt(8.W) - val size = UInt(3.W) - val burst = UInt(2.W) - val lock = Bool() - val cache = UInt(4.W) - val prot = UInt(3.W) - val qos = UInt(4.W) - val region = UInt(4.W) -} - -class WriteData(p: Params) extends Bundle { - val data = UInt(p.dataBits.W) - val strb = UInt(p.bytes.W) - val last = Bool() -} - -class ReadData(p: Params) extends Bundle { - val id = UInt(p.idBits.W) - val data = UInt(p.dataBits.W) - val resp = UInt(2.W) - val last = Bool() -} - -class WriteResponse(p: Params) extends Bundle { - val id = UInt(p.idBits.W) - val resp = UInt(2.W) -} - -/** AXI4 memory-mapped master. Optional user sidebands are absent. */ -class Port(p: Params) extends Bundle { - val aw = Decoupled(new Address(p)) - val w = Decoupled(new WriteData(p)) - val b = Flipped(Decoupled(new WriteResponse(p))) - val ar = Decoupled(new Address(p)) - val r = Flipped(Decoupled(new ReadData(p))) -} diff --git a/arch/src/main/scala/framework/mem-core/axi4/src/main/scala/Params.scala b/arch/src/main/scala/framework/mem-core/axi4/src/main/scala/Params.scala deleted file mode 100644 index 775a7a17f..000000000 --- a/arch/src/main/scala/framework/mem-core/axi4/src/main/scala/Params.scala +++ /dev/null @@ -1,7 +0,0 @@ -package memcore.bus.axi4 - -case class Params(addressBits: Int = 44, dataBits: Int = 128, idBits: Int = 4) { - require(addressBits > 0 && idBits > 0) - require(dataBits >= 8 && dataBits <= 1024 && (dataBits & (dataBits - 1)) == 0) - val bytes = dataBits / 8 -} diff --git a/arch/src/main/scala/framework/mem-core/axis/build.sc b/arch/src/main/scala/framework/mem-core/axis/build.sc deleted file mode 100644 index c49295076..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/build.sc +++ /dev/null @@ -1,11 +0,0 @@ -import mill._ -import mill.scalalib._ - -object axis extends SbtModule { - override def millSourcePath = os.pwd - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") - override def mainClass = Some("memcore.bus.axi.Emit") -} diff --git a/arch/src/main/scala/framework/mem-core/axis/src/csrc/src/arbiter.rs b/arch/src/main/scala/framework/mem-core/axis/src/csrc/src/arbiter.rs deleted file mode 100644 index bb72e1a9e..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/src/csrc/src/arbiter.rs +++ /dev/null @@ -1,49 +0,0 @@ -#[derive(Default)] -pub(crate) struct ArbiterModel { - selected: Option, -} - -impl ArbiterModel { - pub(crate) fn step(&mut self, valid: u32, ready: bool, last: u32) -> Option { - let selected = self - .selected - .or_else(|| (valid != 0).then(|| valid.trailing_zeros())); - let selected = selected?; - if valid & (1 << selected) == 0 { - return None; - } - - if ready { - if last & (1 << selected) != 0 { - self.selected = None; - } else { - self.selected = Some(selected); - } - } - Some(selected) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn locks_until_last_then_rearbitrates() { - let mut model = ArbiterModel::default(); - - assert_eq!(model.step(0b11, true, 0b00), Some(0)); - assert_eq!(model.step(0b11, true, 0b10), Some(0)); - assert_eq!(model.step(0b11, true, 0b01), Some(0)); - assert_eq!(model.step(0b10, true, 0b10), Some(1)); - assert_eq!(model.step(0b00, true, 0b00), None); - } - - #[test] - fn preserves_selection_during_backpressure() { - let mut model = ArbiterModel::default(); - - assert_eq!(model.step(0b10, false, 0b00), Some(1)); - assert_eq!(model.step(0b11, true, 0b01), Some(0)); - } -} diff --git a/arch/src/main/scala/framework/mem-core/axis/src/csrc/src/model.rs b/arch/src/main/scala/framework/mem-core/axis/src/csrc/src/model.rs deleted file mode 100644 index 9faf1edb3..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/src/csrc/src/model.rs +++ /dev/null @@ -1,50 +0,0 @@ -use std::collections::VecDeque; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) struct Beat { - pub(crate) data: u32, - pub(crate) keep: u8, - pub(crate) last: bool, -} - -#[derive(Default)] -pub(crate) struct AxiSModel { - beats: VecDeque, -} - -impl AxiSModel { - pub(crate) fn push(&mut self, beat: Beat) { - self.beats.push_back(beat); - } - - pub(crate) fn pop(&mut self) -> Option { - self.beats.pop_front() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn preserves_beat_order_and_packet_boundaries() { - let mut model = AxiSModel::default(); - let first = Beat { - data: 0x1234_5678, - keep: 0xf, - last: false, - }; - let second = Beat { - data: 0x9abc_def0, - keep: 0x3, - last: true, - }; - - model.push(first); - model.push(second); - - assert_eq!(model.pop(), Some(first)); - assert_eq!(model.pop(), Some(second)); - assert_eq!(model.pop(), None); - } -} diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter.f b/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter.f deleted file mode 100644 index bad3c5bf4..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter.f +++ /dev/null @@ -1,5 +0,0 @@ -+incdir+@RESOURCES@ -@RESOURCES@/arbiter_interface.sv -@RESOURCES@/arbiter_package.sv --F @RTL@/PacketArbiter/filelist.f -@RESOURCES@/arbiter_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_checker.svh b/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_checker.svh deleted file mode 100644 index a11f577cd..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_checker.svh +++ /dev/null @@ -1,47 +0,0 @@ -class arbiter_checker extends uvm_component; - `uvm_component_utils(arbiter_checker) - - virtual arbiter_if vif; - chandle model; - - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual arbiter_if)::get(this, "", "vif", vif)) begin - `uvm_fatal("VIF", "checker requires vif") - end - model = axis_arbiter_create(); - if (model == null) `uvm_fatal("MODEL", "failed to create arbiter model") - endfunction - - task run_phase(uvm_phase phase); - int unsigned selected; - bit expected_valid; - bit [1:0] expected_ready; - - forever begin - @(posedge vif.clock); - if (!vif.reset) begin - selected = axis_arbiter_step(model, vif.in_valid, vif.out_ready, vif.in_last); - expected_valid = selected != 32'hffffffff; - expected_ready = expected_valid && vif.out_ready ? 2'b01 << selected : 2'b00; - - if (vif.out_valid !== expected_valid) `uvm_fatal("VALID", "output valid mismatch") - if (vif.in_ready !== expected_ready) `uvm_fatal("READY", "input ready mismatch") - if (expected_valid) begin - if (vif.out_data !== vif.in_data[selected]) `uvm_fatal("DATA", "selected data mismatch") - if (vif.out_keep !== vif.in_keep[selected]) `uvm_fatal("KEEP", "selected keep mismatch") - if (vif.out_last !== vif.in_last[selected]) `uvm_fatal("LAST", "selected last mismatch") - end - end - end - endtask - - function void final_phase(uvm_phase phase); - axis_arbiter_destroy(model); - super.final_phase(phase); - endfunction -endclass diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_driver.svh b/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_driver.svh deleted file mode 100644 index 9b8240145..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_driver.svh +++ /dev/null @@ -1,80 +0,0 @@ -class driver extends uvm_component; - `uvm_component_utils(driver) - - virtual arbiter_if vif; - bit done; - - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual arbiter_if)::get(this, "", "vif", vif)) begin - `uvm_fatal("VIF", "driver requires vif") - end - endfunction - - task run_phase(uvm_phase phase); - bit [31:0] data0[3] = '{32'h00000000, 32'hffffffff, 32'haaaaaaaa}; - bit [31:0] data1[3] = '{32'hffffffff, 32'h00000000, 32'h55555555}; - bit [3:0] keep0[3] = '{4'h1, 4'hf, 4'h5}; - bit [3:0] keep1[3] = '{4'hf, 4'h1, 4'ha}; - int unsigned index0 = 0; - int unsigned index1 = 0; - int unsigned cycle = 0; - - vif.in_valid <= 2'b00; - vif.in_data[0] <= 32'h00000000; - vif.in_data[1] <= 32'h00000000; - vif.in_keep[0] <= 4'h0; - vif.in_keep[1] <= 4'h0; - vif.in_last <= 2'b00; - vif.out_ready <= 1'b0; - do @(negedge vif.clock); while (vif.reset); - - while (index0 < 3 || index1 < 3) begin - vif.in_valid[0] <= index0 < 3; - vif.in_valid[1] <= index1 < 3; - vif.in_data[0] <= data0[index0<3?index0 : 2]; - vif.in_data[1] <= data1[index1<3?index1 : 2]; - vif.in_keep[0] <= keep0[index0<3?index0 : 2]; - vif.in_keep[1] <= keep1[index1<3?index1 : 2]; - vif.in_last[0] <= index0 == 2; - vif.in_last[1] <= index1 == 2; - vif.out_ready <= cycle % 3 != 0; - @(posedge vif.clock); - if (vif.in_valid[0] && vif.in_ready[0]) index0++; - if (vif.in_valid[1] && vif.in_ready[1]) index1++; - cycle++; - @(negedge vif.clock); - end - - vif.in_valid <= 2'b10; - vif.in_data[1] <= 32'hffffffff; - vif.in_keep[1] <= 4'hf; - vif.in_last <= 2'b10; - vif.out_ready <= 1'b1; - do @(posedge vif.clock); while (!vif.in_ready[1]); - @(negedge vif.clock); - - index0 = 0; - while (index0 < 2) begin - vif.in_valid <= 2'b01; - vif.in_data[0] <= index0 == 0 ? 32'hffffffff : 32'h00000000; - vif.in_keep[0] <= index0 == 0 ? 4'hf : 4'h1; - vif.in_last <= index0 == 1; - @(posedge vif.clock); - if (vif.in_ready[0]) index0++; - @(negedge vif.clock); - end - - vif.in_valid <= 2'b00; - vif.in_data[0] <= 32'h00000000; - vif.in_data[1] <= 32'h00000000; - vif.in_keep[0] <= 4'h0; - vif.in_keep[1] <= 4'h0; - vif.in_last <= 2'b00; - done = 1'b1; - endtask -endclass diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_interface.sv b/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_interface.sv deleted file mode 100644 index 45f3ad70b..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_interface.sv +++ /dev/null @@ -1,25 +0,0 @@ -interface arbiter_if ( - input logic clock -); - logic reset; - logic [ 1:0] in_valid; - logic [ 1:0] in_ready; - logic [31:0] in_data [2]; - logic [ 3:0] in_keep [2]; - logic [ 1:0] in_last; - logic out_valid; - logic out_ready; - logic [31:0] out_data; - logic [ 3:0] out_keep; - logic out_last; - - assert property (@(posedge clock) disable iff (reset) $onehot0(in_ready)); - assert property (@(posedge clock) disable iff (reset) - out_valid && !out_ready |=> out_valid && $stable( - {out_data, out_keep, out_last} - )); - - cover property (@(posedge clock) disable iff (reset) &in_valid); - cover property (@(posedge clock) disable iff (reset) in_valid[0] && in_ready[0]); - cover property (@(posedge clock) disable iff (reset) in_valid[1] && in_ready[1]); -endinterface diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_package.sv b/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_package.sv deleted file mode 100644 index 0d3a3de07..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_package.sv +++ /dev/null @@ -1,17 +0,0 @@ -package arbiter_pkg; - import uvm_pkg::*; - `include "uvm_macros.svh" - - import "DPI-C" function chandle axis_arbiter_create(); - import "DPI-C" function void axis_arbiter_destroy(input chandle model); - import "DPI-C" function int unsigned axis_arbiter_step( - input chandle model, - input int unsigned valid, - input byte unsigned ready, - input int unsigned last - ); - - `include "arbiter_driver.svh" - `include "arbiter_checker.svh" - `include "arbiter_test.svh" -endpackage diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_tb.sv b/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_tb.sv deleted file mode 100644 index 9d8516136..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_tb.sv +++ /dev/null @@ -1,42 +0,0 @@ -module arbiter_tb; - import uvm_pkg::*; - import arbiter_pkg::*; - - logic clock = 1'b0; - logic reset = 1'b0; - always #5 clock = ~clock; - - arbiter_if source_if (clock); - assign source_if.reset = reset; - - PacketArbiter dut ( - .clock(clock), - .reset(reset), - .io_in_0_ready(source_if.in_ready[0]), - .io_in_0_valid(source_if.in_valid[0]), - .io_in_0_bits_tdata(source_if.in_data[0]), - .io_in_0_bits_tkeep(source_if.in_keep[0]), - .io_in_0_bits_tlast(source_if.in_last[0]), - .io_in_1_ready(source_if.in_ready[1]), - .io_in_1_valid(source_if.in_valid[1]), - .io_in_1_bits_tdata(source_if.in_data[1]), - .io_in_1_bits_tkeep(source_if.in_keep[1]), - .io_in_1_bits_tlast(source_if.in_last[1]), - .io_out_ready(source_if.out_ready), - .io_out_valid(source_if.out_valid), - .io_out_bits_tdata(source_if.out_data), - .io_out_bits_tkeep(source_if.out_keep), - .io_out_bits_tlast(source_if.out_last) - ); - - initial begin - reset = 1'b1; - repeat (4) @(posedge clock); - reset = 1'b0; - end - - initial begin - uvm_config_db#(virtual arbiter_if)::set(null, "*", "vif", source_if); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_test.svh b/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_test.svh deleted file mode 100644 index 76f8ec6a7..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_test.svh +++ /dev/null @@ -1,42 +0,0 @@ -class env extends uvm_env; - `uvm_component_utils(env) - - driver source; - arbiter_checker monitor; - - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - - function void build_phase(uvm_phase phase); - super.build_phase(phase); - source = driver::type_id::create("source", this); - monitor = arbiter_checker::type_id::create("monitor", this); - endfunction -endclass - -class protocol_test extends uvm_test; - `uvm_component_utils(protocol_test) - - env test_env; - virtual arbiter_if vif; - - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual arbiter_if)::get(this, "", "vif", vif)) begin - `uvm_fatal("VIF", "test requires vif") - end - test_env = env::type_id::create("env", this); - endfunction - - task run_phase(uvm_phase phase); - phase.raise_objection(this); - wait (test_env.source.done); - repeat (2) @(posedge vif.clock); - phase.drop_objection(this); - endtask -endclass diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/env.svh b/arch/src/main/scala/framework/mem-core/axis/src/main/resources/env.svh deleted file mode 100644 index 868d407c9..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/env.svh +++ /dev/null @@ -1,37 +0,0 @@ -class env extends checked_env #(item, item, ref_model); - `uvm_component_utils(env) - - source_agent source; - sink sink_driver; - monitor sink_monitor; - - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - - function void build_phase(uvm_phase phase); - virtual axis_if source_vif; - virtual axis_if sink_vif; - - super.build_phase(phase); - if (!uvm_config_db#(virtual axis_if)::get(this, "", "source_vif", source_vif)) begin - `uvm_fatal("AXIS_VIF", "environment requires source_vif") - end - if (!uvm_config_db#(virtual axis_if)::get(this, "", "sink_vif", sink_vif)) begin - `uvm_fatal("AXIS_VIF", "environment requires sink_vif") - end - uvm_config_db#(virtual axis_if)::set(this, "source.*", "vif", source_vif); - uvm_config_db#(virtual axis_if)::set(this, "sink_driver", "vif", sink_vif); - uvm_config_db#(virtual axis_if)::set(this, "sink_monitor", "vif", sink_vif); - - source = source_agent::type_id::create("source", this); - sink_driver = sink::type_id::create("sink_driver", this); - sink_monitor = monitor::type_id::create("sink_monitor", this); - endfunction - - function void connect_phase(uvm_phase phase); - super.connect_phase(phase); - source.mon.ap.connect(input_export); - sink_monitor.ap.connect(output_export); - endfunction -endclass diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/master.f b/arch/src/main/scala/framework/mem-core/axis/src/main/resources/master.f deleted file mode 100644 index f5a1226b1..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/master.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@VERIFY@/uvm/src/protocol/axis -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/protocol/axis/interface.sv -@VERIFY@/uvm/src/protocol/axis/package.sv -@RESOURCES@/package.sv --F @RTL@/Master/filelist.f -@RESOURCES@/master_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/master_tb.sv b/arch/src/main/scala/framework/mem-core/axis/src/main/resources/master_tb.sv deleted file mode 100644 index 3ac27bcde..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/master_tb.sv +++ /dev/null @@ -1,41 +0,0 @@ -module master_tb; - import uvm_pkg::*; - import axis_pkg::*; - - logic clock = 1'b0; - logic reset = 1'b0; - always #5 clock = ~clock; - - axis_if source_if (clock); - axis_if sink_if (clock); - - assign source_if.reset = reset; - assign sink_if.reset = reset; - - Master dut ( - .clock(clock), - .reset(reset), - .io_in_ready(source_if.tready), - .io_in_valid(source_if.tvalid), - .io_in_bits_tdata(source_if.tdata), - .io_in_bits_tkeep(source_if.tkeep), - .io_in_bits_tlast(source_if.tlast), - .io_axis_tvalid(sink_if.tvalid), - .io_axis_tready(sink_if.tready), - .io_axis_tdata(sink_if.tdata), - .io_axis_tkeep(sink_if.tkeep), - .io_axis_tlast(sink_if.tlast) - ); - - initial begin - reset = 1'b1; - repeat (4) @(negedge clock); - reset = 1'b0; - end - - initial begin - uvm_config_db#(virtual axis_if)::set(null, "uvm_test_top.env", "source_vif", source_if); - uvm_config_db#(virtual axis_if)::set(null, "uvm_test_top.env", "sink_vif", sink_if); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/package.sv b/arch/src/main/scala/framework/mem-core/axis/src/main/resources/package.sv deleted file mode 100644 index cd19f48c7..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/package.sv +++ /dev/null @@ -1,29 +0,0 @@ -package axis_pkg; - import uvm_pkg::*; - import ip_pkg::*; - import axis_vip_pkg::*; - typedef axis_item#(32) item; - typedef axis_source_agent#(32) source_agent; - typedef axis_monitor#(32) monitor; - typedef axis_sink#(32) sink; - `include "uvm_macros.svh" - - import "DPI-C" function chandle axis_ref_create(); - import "DPI-C" function void axis_ref_destroy(input chandle model); - import "DPI-C" function void axis_ref_push( - input chandle model, - input int unsigned data, - input byte unsigned keep, - input byte unsigned last - ); - import "DPI-C" function byte unsigned axis_ref_pop( - input chandle model, - output int unsigned data, - output byte unsigned keep, - output byte unsigned last - ); - - `include "ref_model.svh" - `include "env.svh" - `include "test.svh" -endpackage diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/ref_model.svh b/arch/src/main/scala/framework/mem-core/axis/src/main/resources/ref_model.svh deleted file mode 100644 index 89ce490c6..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/ref_model.svh +++ /dev/null @@ -1,39 +0,0 @@ -class ref_model extends reference_model #(item); - `uvm_component_utils(ref_model) - - chandle model; - - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - - function void build_phase(uvm_phase phase); - super.build_phase(phase); - model = axis_ref_create(); - if (model == null) begin - `uvm_fatal("AXIS_MODEL", "failed to create Rust reference model") - end - endfunction - - function void write(item t); - item expected; - int unsigned data; - byte unsigned keep; - byte unsigned last; - - axis_ref_push(model, t.data, t.keep, t.last); - if (!axis_ref_pop(model, data, keep, last)) begin - `uvm_fatal("AXIS_MODEL", "reference model produced no output") - end - expected = item::type_id::create("expected"); - expected.data = data; - expected.keep = keep[3:0]; - expected.last = last[0]; - expected_ap.write(expected); - endfunction - - function void final_phase(uvm_phase phase); - axis_ref_destroy(model); - super.final_phase(phase); - endfunction -endclass diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/slave.f b/arch/src/main/scala/framework/mem-core/axis/src/main/resources/slave.f deleted file mode 100644 index 75928f05e..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/slave.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@VERIFY@/uvm/src/protocol/axis -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/protocol/axis/interface.sv -@VERIFY@/uvm/src/protocol/axis/package.sv -@RESOURCES@/package.sv --F @RTL@/Slave/filelist.f -@RESOURCES@/slave_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/slave_tb.sv b/arch/src/main/scala/framework/mem-core/axis/src/main/resources/slave_tb.sv deleted file mode 100644 index e95331d67..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/slave_tb.sv +++ /dev/null @@ -1,41 +0,0 @@ -module slave_tb; - import uvm_pkg::*; - import axis_pkg::*; - - logic clock = 1'b0; - logic reset = 1'b0; - always #5 clock = ~clock; - - axis_if source_if (clock); - axis_if sink_if (clock); - - assign source_if.reset = reset; - assign sink_if.reset = reset; - - Slave dut ( - .clock(clock), - .reset(reset), - .io_axis_tvalid(source_if.tvalid), - .io_axis_tready(source_if.tready), - .io_axis_tdata(source_if.tdata), - .io_axis_tkeep(source_if.tkeep), - .io_axis_tlast(source_if.tlast), - .io_out_ready(sink_if.tready), - .io_out_valid(sink_if.tvalid), - .io_out_bits_tdata(sink_if.tdata), - .io_out_bits_tkeep(sink_if.tkeep), - .io_out_bits_tlast(sink_if.tlast) - ); - - initial begin - reset = 1'b1; - repeat (4) @(negedge clock); - reset = 1'b0; - end - - initial begin - uvm_config_db#(virtual axis_if)::set(null, "uvm_test_top.env", "source_vif", source_if); - uvm_config_db#(virtual axis_if)::set(null, "uvm_test_top.env", "sink_vif", sink_if); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/test.svh b/arch/src/main/scala/framework/mem-core/axis/src/main/resources/test.svh deleted file mode 100644 index 0d7389911..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/test.svh +++ /dev/null @@ -1,43 +0,0 @@ -class protocol_sequence extends uvm_sequence #(item); - `uvm_object_utils(protocol_sequence) - - function new(string name = "protocol_sequence"); - super.new(name); - endfunction - - task body(); - item req; - for (int i = 0; i < 32; i++) begin - req = item::type_id::create($sformatf("item_%0d", i)); - start_item(req); - req.data = $urandom(); - req.keep = (i % 15) + 1; - req.last = (i % 4) == 3; - finish_item(req); - end - endtask -endclass - -class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - - env test_env; - - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - - function void build_phase(uvm_phase phase); - super.build_phase(phase); - test_env = env::type_id::create("env", this); - endfunction - - task execute(); - protocol_sequence seq; - seq = protocol_sequence::type_id::create("seq"); - seq.start(test_env.source.seqr); - test_env.scoreboard.wait_checked(32); - `uvm_info("AXIS_CHECKED", $sformatf("Checked %0d transfers", test_env.scoreboard.checked), - UVM_LOW) - endtask -endclass diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/scala/Emit.scala b/arch/src/main/scala/framework/mem-core/axis/src/main/scala/Emit.scala deleted file mode 100644 index 543e470bd..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/scala/Emit.scala +++ /dev/null @@ -1,19 +0,0 @@ -package memcore.bus.axi - -object Emit extends App { - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Master(dataBits = 32), - firtoolOpts = args, - args = Array("--target-dir", "build/Master", "--split-verilog") - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Slave(dataBits = 32), - firtoolOpts = args, - args = Array("--target-dir", "build/Slave", "--split-verilog") - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new PacketArbiter(inputs = 2, dataBits = 32), - firtoolOpts = args, - args = Array("--target-dir", "build/PacketArbiter", "--split-verilog") - ) -} diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/scala/Interface.scala b/arch/src/main/scala/framework/mem-core/axis/src/main/scala/Interface.scala deleted file mode 100644 index eba9d6037..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/scala/Interface.scala +++ /dev/null @@ -1,51 +0,0 @@ -package memcore.bus.axi - -import chisel3._ - -/** Physical AXI4-Stream signals, directed from the master's perspective. */ -class Port( - dataBits: Int, - idBits: Int = 0, - destBits: Int = 0, - userBits: Int = 0) - extends Bundle { - require(dataBits > 0 && dataBits % 8 == 0) - val tvalid = Output(Bool()) - val tready = Input(Bool()) - val tdata = Output(UInt(dataBits.W)) - val tkeep = Output(UInt((dataBits / 8).W)) - val tlast = Output(Bool()) - val tid = Output(UInt(idBits.W)) - val tdest = Output(UInt(destBits.W)) - val tuser = Output(UInt(userBits.W)) -} - -/** - * AXI4-Stream payload signals. - * - * A `Decoupled[Beat]` is the full AXI-S channel: `valid` and `ready` - * are TVALID and TREADY. This bundle exposes the remaining standard signal - * names directly in generated RTL. - */ -class Beat( - dataBits: Int, - idBits: Int = 0, - destBits: Int = 0, - userBits: Int = 0) - extends Bundle { - require(dataBits > 0 && dataBits % 8 == 0) - val tdata = UInt(dataBits.W) - val tkeep = UInt((dataBits / 8).W) - val tlast = Bool() - val tid = UInt(idBits.W) - val tdest = UInt(destBits.W) - val tuser = UInt(userBits.W) - - // Internal aliases preserve concise datapath code while retaining AXI-S names. - def data: UInt = tdata - def keep: UInt = tkeep - def last: Bool = tlast - def id: UInt = tid - def dest: UInt = tdest - def user: UInt = tuser -} diff --git a/arch/src/main/scala/framework/mem-core/bank/build.sc b/arch/src/main/scala/framework/mem-core/bank/build.sc deleted file mode 100644 index 381e9d95e..000000000 --- a/arch/src/main/scala/framework/mem-core/bank/build.sc +++ /dev/null @@ -1,22 +0,0 @@ -import mill._ -import mill.scalalib._ - -trait ChiselModule extends SbtModule { - def moduleRoot: os.Path - - override def millSourcePath = moduleRoot - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") -} - -object axis extends ChiselModule { - override def moduleRoot = os.pwd / os.up / "axis" -} - -object bank extends ChiselModule { - override def moduleRoot = os.pwd - override def moduleDeps = Seq(axis) - override def mainClass = Some("memcore.memory.bank.Emit") -} diff --git a/arch/src/main/scala/framework/mem-core/bank/src/csrc/src/memory.rs b/arch/src/main/scala/framework/mem-core/bank/src/csrc/src/memory.rs deleted file mode 100644 index 4968ad3ae..000000000 --- a/arch/src/main/scala/framework/mem-core/bank/src/csrc/src/memory.rs +++ /dev/null @@ -1,80 +0,0 @@ -struct Bank { - words: Vec, - initialized: Vec, -} - -impl Bank { - fn access(&mut self, addr: usize, write: bool, data: u32, mask: u8) -> u32 { - if write { - for byte in 0..4 { - if mask & (1 << byte) != 0 { - let bits = 0xff << (byte * 8); - self.words[addr] = (self.words[addr] & !bits) | (data & bits); - } - } - self.initialized[addr] |= mask; - 0 - } else { - assert_eq!( - self.initialized[addr], 0xf, - "read of uninitialized bank word" - ); - self.words[addr] - } - } -} - -#[no_mangle] -pub extern "C" fn bank_ref_create(entries: u32) -> *mut std::ffi::c_void { - Box::into_raw(Box::new(Bank { - words: vec![0; entries as usize], - initialized: vec![0; entries as usize], - })) - .cast() -} - -#[no_mangle] -pub unsafe extern "C" fn bank_ref_destroy(model: *mut std::ffi::c_void) { - drop(Box::from_raw(model.cast::())); -} - -#[no_mangle] -pub unsafe extern "C" fn bank_ref_access( - model: *mut std::ffi::c_void, - addr: u32, - write: u8, - data: u32, - mask: u8, -) -> u32 { - (*model.cast::()).access(addr as usize, write != 0, data, mask) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn masked_writes_preserve_other_bytes_and_addresses() { - let mut bank = Bank { - words: vec![0; 16], - initialized: vec![0; 16], - }; - bank.access(0, true, 0x12345678, 15); - bank.access(15, true, 0xabcdef01, 15); - bank.access(0, true, 0xaabbccdd, 5); - assert_eq!(bank.access(0, false, 0, 0), 0x12bb56dd); - bank.access(0, true, 0, 0); - assert_eq!(bank.access(0, false, 0, 0), 0x12bb56dd); - assert_eq!(bank.access(15, false, 0, 0), 0xabcdef01); - } - - #[test] - #[should_panic(expected = "uninitialized")] - fn rejects_uninitialized_reads() { - let mut bank = Bank { - words: vec![0; 16], - initialized: vec![0; 16], - }; - bank.access(0, false, 0, 0); - } -} diff --git a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bank.el b/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bank.el deleted file mode 100644 index a7c43228d..000000000 --- a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bank.el +++ /dev/null @@ -1,5 +0,0 @@ -// Format Version: 2 -CHECKSUM: "366479506 2693907090" -INSTANCE: bank_tb.dut -ANNOTATION: "Bank has no error response path; invalid accesses are outside the request contract and error is tied to zero." -Toggle io_response_bits_error "net io_response_bits_error" diff --git a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bank.f b/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bank.f deleted file mode 100644 index c12d863fb..000000000 --- a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bank.f +++ /dev/null @@ -1,7 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -+incdir+@RESOURCES@ -@RESOURCES@/interface.sv -@RESOURCES@/package.sv --F @RTL@/Bank/filelist.f -@RESOURCES@/bank_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bank_tb.sv b/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bank_tb.sv deleted file mode 100644 index b247c06bd..000000000 --- a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bank_tb.sv +++ /dev/null @@ -1,29 +0,0 @@ -module bank_tb; - import uvm_pkg::*; - import bank_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - bank_if source_if (clock); - bank_if sink_if (clock); - Bank dut ( - .clock(clock), - .reset(source_if.reset), - .io_request_valid(source_if.valid), - .io_request_ready(source_if.ready), - .io_request_bits_addr(source_if.addr), - .io_request_bits_write(source_if.write), - .io_request_bits_data(source_if.data), - .io_request_bits_mask(source_if.mask), - .io_request_bits_tag(source_if.tag), - .io_response_valid(sink_if.valid), - .io_response_ready(sink_if.ready), - .io_response_bits_data(sink_if.data), - .io_response_bits_tag(sink_if.tag), - .io_response_bits_error(sink_if.error) - ); - initial begin - uvm_config_db#(virtual bank_if)::set(null, "uvm_test_top*", "request", source_if); - uvm_config_db#(virtual bank_if)::set(null, "uvm_test_top*", "response", sink_if); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bankset.el b/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bankset.el deleted file mode 100644 index 56c580493..000000000 --- a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bankset.el +++ /dev/null @@ -1,79 +0,0 @@ -// Format Version: 2 -// Reviewed for Emit: dataBits=32, banks=4, entriesPerBank=16, tagBits=8. - -CHECKSUM: "1481319037 3871254824" -INSTANCE: bankset_tb.dut -ANNOTATION: "Nonempty-command assertion failure diagnostics; legal commands have beats > 0." -Block 2 "555555334" "if (1)" -ANNOTATION: "Nonempty-command assertion failure diagnostics; legal commands have beats > 0." -Block 3 "3006158740" "$error(\"Assertion failed: BankSet command must be nonempty\n at BankSet.scala:61 assert(io.command.bits.beats =/= 0.U, \\"BankSet command must be nonempty\\")\n\");" -ANNOTATION: "Nonempty-command assertion failure diagnostics; legal commands have beats > 0." -Block 5 "555555334" "if (1)" -ANNOTATION: "Nonempty-command assertion failure diagnostics; legal commands have beats > 0." -Block 6 "1956737205" "$fatal;" -ANNOTATION: "Alignment assertion failure diagnostics; commands are 4-byte aligned." -Block 10 "530868893" "if (1)" -ANNOTATION: "Alignment assertion failure diagnostics; commands are 4-byte aligned." -Block 11 "181752484" "$error(\"Assertion failed: BankSet command must be beat aligned\n at BankSet.scala:62 assert(io.command.bits.addr %%%% p.bytes.U === 0.U, \\"BankSet command must be beat aligned\\")\n\");" -ANNOTATION: "Alignment assertion failure diagnostics; commands are 4-byte aligned." -Block 13 "530868893" "if (1)" -ANNOTATION: "Alignment assertion failure diagnostics; commands are 4-byte aligned." -Block 14 "3298909189" "$fatal;" -ANNOTATION: "Capacity assertion failure diagnostics; command end is at most 256 bytes." -Block 18 "2870240055" "if (1)" -ANNOTATION: "Capacity assertion failure diagnostics; command end is at most 256 bytes." -Block 19 "3417018178" "$error(\"Assertion failed: BankSet command exceeds capacity\n at BankSet.scala:63 assert(\n\");" -ANNOTATION: "Capacity assertion failure diagnostics; command end is at most 256 bytes." -Block 21 "2870240055" "if (1)" -ANNOTATION: "Capacity assertion failure diagnostics; command end is at most 256 bytes." -Block 22 "2272232831" "$fatal;" -ANNOTATION: "TLAST assertion failure diagnostics; last is asserted on exactly the final write beat." -Block 26 "1487852324" "if (1)" -ANNOTATION: "TLAST assertion failure diagnostics; last is asserted on exactly the final write beat." -Block 27 "1783058220" "$error(\"Assertion failed: BankSet write TLAST does not match command length\n at BankSet.scala:87 assert(io.write.bits.last === last, \\"BankSet write TLAST does not match command length\\")\n\");" -ANNOTATION: "TLAST assertion failure diagnostics; last is asserted on exactly the final write beat." -Block 29 "1487852324" "if (1)" -ANNOTATION: "TLAST assertion failure diagnostics; last is asserted on exactly the final write beat." -Block 30 "605788389" "$fatal;" - -CHECKSUM: "1481319037 1542845018" -INSTANCE: bankset_tb.dut -ANNOTATION: "Assertion enable is disabled during reset; no command is issued in reset." -Condition 1 "3864235482" "(_GEN & ((~reset))) 1 -1" (2 "10") -ANNOTATION: "A valid command cannot have zero beats." -Condition 2 "4106865980" "(_GEN_10 & (io_command_bits_beats == 32'b0)) 1 -1" (3 "11") -ANNOTATION: "All driven addresses are 4-byte aligned." -Condition 4 "3972072533" "(_GEN_10 & ((|_GEN_11[2:0]))) 1 -1" (1 "01") -ANNOTATION: "All driven addresses are 4-byte aligned." -Condition 4 "3972072533" "(_GEN_10 & ((|_GEN_11[2:0]))) 1 -1" (3 "11") -ANNOTATION: "All driven commands end within the 256-byte capacity." -Condition 5 "3649435992" "(_GEN_10 & (({28'b0, io_command_bits_addr} + {2'b0, io_command_bits_beats, 2'b0}) > 36'h000000100)) 1 -1" (1 "01") -ANNOTATION: "All driven commands end within the 256-byte capacity." -Condition 5 "3649435992" "(_GEN_10 & (({28'b0, io_command_bits_addr} + {2'b0, io_command_bits_beats, 2'b0}) > 36'h000000100)) 1 -1" (3 "11") -ANNOTATION: "No write handshake in reset; accepted TLAST matches the command length." -Condition 6 "3432548480" "(_GEN_9 & ((~reset)) & (io_write_bits_tlast != last)) 1 -1" (2 "101") -ANNOTATION: "No write handshake in reset; accepted TLAST matches the command length." -Condition 6 "3432548480" "(_GEN_9 & ((~reset)) & (io_write_bits_tlast != last)) 1 -1" (4 "111") -ANNOTATION: "writeResponse follows an accepted Bank write; that Bank holds response.valid until ready." -Condition 9 "618862985" "(_GEN_3 & _GEN_13) 1 -1" (2 "10") -ANNOTATION: "Only one request is outstanding; a read is issued only after the preceding response retires." -Condition 13 "3211095629" "(_GEN_1 & _GEN_8) 1 -1" (2 "10") -ANNOTATION: "Only one request is outstanding; writing selects an idle Bank after the preceding response retires." -Condition 36 "1292920334" "(_io_write_ready_T & _GEN_8) 1 -1" (2 "10") - -CHECKSUM: "1481319037 745494789" -INSTANCE: bankset_tb.dut -ANNOTATION: "Four-byte aligned addresses have low two bits zero." -Toggle io_command_bits_addr [1:0] "net io_command_bits_addr[7:0]" -ANNOTATION: "Four-byte aligned addresses have low two bits zero." -Toggle command_addr [1:0] "reg command_addr[7:0]" -ANNOTATION: "Capacity is 64 words; legal lengths and completed-beat count cannot exceed 64." -Toggle io_command_bits_beats [31:7] "net io_command_bits_beats[31:0]" -ANNOTATION: "Capacity is 64 words; legal lengths and completed-beat count cannot exceed 64." -Toggle command_beats [31:7] "reg command_beats[31:0]" -ANNOTATION: "Capacity is 64 words; legal lengths and completed-beat count cannot exceed 64." -Toggle beat [31:7] "reg beat[31:0]" -ANNOTATION: "Read responses contain a full word; TKEEP is always 4'b1111." -Toggle io_read_bits_tkeep "net io_read_bits_tkeep[3:0]" -ANNOTATION: "Completion error is tied to zero; invalid commands are assertions, not error responses." -Toggle io_done_bits "net io_done_bits" diff --git a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bankset.f b/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bankset.f deleted file mode 100644 index 442fb41c4..000000000 --- a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bankset.f +++ /dev/null @@ -1,10 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@VERIFY@/uvm/src/protocol/axis -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/protocol/axis/interface.sv -@VERIFY@/uvm/src/protocol/axis/package.sv -@RESOURCES@/bankset_interface.sv -@RESOURCES@/bankset_package.sv --F @RTL@/BankSet/filelist.f -@RESOURCES@/bankset_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bankset_interface.sv b/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bankset_interface.sv deleted file mode 100644 index e380df75d..000000000 --- a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bankset_interface.sv +++ /dev/null @@ -1,19 +0,0 @@ -interface bankset_if ( - input logic clock -); - logic reset; - logic valid; - logic ready; - logic write; - logic [7:0] addr; - logic [31:0] beats; - logic done_valid; - logic done_ready; - logic error; - - assert property (@(posedge clock) disable iff (reset) - done_valid && !done_ready |=> done_valid && $stable( - error - )) - else $fatal(1, "BankSet completion changed under backpressure"); -endinterface diff --git a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bankset_package.sv b/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bankset_package.sv deleted file mode 100644 index 99508d625..000000000 --- a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bankset_package.sv +++ /dev/null @@ -1,201 +0,0 @@ -package bankset_pkg; - import uvm_pkg::*; - import ip_pkg::*; - import axis_vip_pkg::*; - `include "uvm_macros.svh" - typedef axis_item#(32) item; - - import "DPI-C" function chandle bank_ref_create(input int unsigned entries); - import "DPI-C" function void bank_ref_destroy(input chandle model); - import "DPI-C" function int unsigned bank_ref_access( - input chandle model, - input int unsigned addr, - input byte unsigned write, - input int unsigned data, - input byte unsigned mask - ); - - class operation extends uvm_sequence_item; - bit write; - int unsigned word_addr; - int unsigned data; - byte unsigned mask; - bit last; - `uvm_object_utils(operation) - function new(string name = "operation"); - super.new(name); - endfunction - endclass - - class ref_model extends reference_model #(operation, item); - `uvm_component_utils(ref_model) - chandle model; - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - model = bank_ref_create(64); - endfunction - function void write(operation t); - int unsigned result = bank_ref_access(model, t.word_addr, t.write, t.data, t.mask); - if (!t.write) begin - item expected = item::type_id::create("expected"); - expected.data = result; - expected.keep = 15; - expected.last = t.last; - expected_ap.write(expected); - end - endfunction - function void final_phase(uvm_phase phase); - bank_ref_destroy(model); - super.final_phase(phase); - endfunction - endclass - - class env extends checked_env #(operation, item, ref_model); - `uvm_component_utils(env) - virtual bankset_if control; - virtual axis_if write_vif; - axis_source_agent #(32) source; - axis_sink #(32) sink; - axis_monitor #(32) output_monitor; - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual bankset_if)::get( - this, "", "control", control - ) || !uvm_config_db#(virtual axis_if)::get( - this, "", "write_vif", write_vif - )) - `uvm_fatal("VIF", "BankSet interfaces missing") - source = axis_source_agent#(32)::type_id::create("source", this); - sink = axis_sink#(32)::type_id::create("sink", this); - output_monitor = axis_monitor#(32)::type_id::create("output_monitor", this); - endfunction - function void connect_phase(uvm_phase phase); - super.connect_phase(phase); - output_monitor.ap.connect(output_export); - endfunction - task run_phase(uvm_phase phase); - int unsigned word_addr; - operation op; - forever begin - @(posedge control.clock); - if (!control.reset) begin - if (control.valid && control.ready) begin - word_addr = control.addr / 4; - if (!control.write) begin - for (int i = 0; i < control.beats; i++) begin - op = operation::type_id::create("read"); - op.write = 0; - op.word_addr = word_addr + i; - op.last = i == control.beats - 1; - input_export.write(op); - end - end - end - if (write_vif.tvalid && write_vif.tready) begin - op = operation::type_id::create("write"); - op.write = 1; - op.word_addr = word_addr; - op.data = write_vif.tdata; - op.mask = write_vif.tkeep; - input_export.write(op); - word_addr++; - end - end - end - endtask - endclass - - class write_sequence extends uvm_sequence #(item); - `uvm_object_utils(write_sequence) - int unsigned beats; - bit masked; - function new(string name = "write_sequence"); - super.new(name); - endfunction - task body(); - item req; - for (int i = 0; i < beats; i++) begin - req = item::type_id::create("req"); - start_item(req); - req.data = $urandom(); - req.keep = masked ? (i % 16) : 15; - req.last = i == beats - 1; - finish_item(req); - end - endtask - endclass - - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - env test_env; - virtual bankset_if control; - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - test_env = env::type_id::create("env", this); - if (!uvm_config_db#(virtual bankset_if)::get(this, "", "control", control)) - `uvm_fatal("VIF", "BankSet control interface missing") - endfunction - task transfer(bit write, int unsigned word_addr, int unsigned beats, bit masked = 0); - write_sequence seq; - int unsigned target = test_env.scoreboard.checked + (write ? 0 : beats); - @(negedge control.clock); - control.valid = 1; - control.write = write; - control.addr = word_addr * 4; - control.beats = beats; - control.done_ready = 0; - do @(posedge control.clock); while (!control.ready); - @(negedge control.clock); - control.valid = 0; - if (write) begin - seq = write_sequence::type_id::create("seq"); - seq.beats = beats; - seq.masked = masked; - seq.start(test_env.source.seqr); - end else begin - test_env.scoreboard.wait_checked(target); - end - do @(negedge control.clock); while (!control.done_valid); - repeat (3) begin - if (control.done_valid !== 1'b1 || control.error !== 1'b0 || control.ready !== 1'b0) - `uvm_fatal("DONE", "invalid BankSet completion under backpressure") - @(negedge control.clock); - end - control.done_ready = 1; - repeat (2) @(negedge control.clock); - control.done_ready = 0; - if (control.done_valid !== 1'b0 || control.ready !== 1'b1) - `uvm_fatal("DONE", "BankSet failed to retire completion") - endtask - task execute(); - control.valid = 0; - control.write = 0; - control.addr = 0; - control.beats = 0; - control.done_ready = 0; - wait (!control.reset); - transfer(1, 0, 64); - test_env.sink.stall_cycles = 0; - transfer(0, 0, 63); - test_env.sink.stall_cycles = 3; - transfer(1, 63, 1); - transfer(0, 63, 1); - transfer(1, 3, 17, 1); - control.reset = 1; - repeat (2) @(negedge control.clock); - control.reset = 0; - transfer(0, 0, 64); - `uvm_info("BANKSET", $sformatf("Checked %0d read beats", test_env.scoreboard.checked), - UVM_LOW) - endtask - endclass -endpackage diff --git a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bankset_tb.sv b/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bankset_tb.sv deleted file mode 100644 index c61cef2e9..000000000 --- a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bankset_tb.sv +++ /dev/null @@ -1,46 +0,0 @@ -module bankset_tb; - import uvm_pkg::*; - import bankset_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - bankset_if control (clock); - axis_if source_if (clock); - axis_if sink_if (clock); - assign source_if.reset = control.reset; - assign sink_if.reset = control.reset; - BankSet dut ( - .clock(clock), - .reset(control.reset), - .io_command_valid(control.valid), - .io_command_ready(control.ready), - .io_command_bits_write(control.write), - .io_command_bits_addr(control.addr), - .io_command_bits_beats(control.beats), - .io_write_valid(source_if.tvalid), - .io_write_ready(source_if.tready), - .io_write_bits_tdata(source_if.tdata), - .io_write_bits_tkeep(source_if.tkeep), - .io_write_bits_tlast(source_if.tlast), - .io_read_valid(sink_if.tvalid), - .io_read_ready(sink_if.tready), - .io_read_bits_tdata(sink_if.tdata), - .io_read_bits_tkeep(sink_if.tkeep), - .io_read_bits_tlast(sink_if.tlast), - .io_done_valid(control.done_valid), - .io_done_ready(control.done_ready), - .io_done_bits(control.error) - ); - initial begin - control.reset = 1; - repeat (4) @(negedge clock); - control.reset = 0; - end - initial begin - uvm_config_db#(virtual bankset_if)::set(null, "uvm_test_top*", "control", control); - uvm_config_db#(virtual axis_if)::set(null, "uvm_test_top.env", "write_vif", source_if); - uvm_config_db#(virtual axis_if)::set(null, "uvm_test_top.env.source.*", "vif", source_if); - uvm_config_db#(virtual axis_if)::set(null, "uvm_test_top.env.sink", "vif", sink_if); - uvm_config_db#(virtual axis_if)::set(null, "uvm_test_top.env.output_monitor", "vif", sink_if); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/env.svh b/arch/src/main/scala/framework/mem-core/bank/src/main/resources/env.svh deleted file mode 100644 index fae0f642d..000000000 --- a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/env.svh +++ /dev/null @@ -1,47 +0,0 @@ -class env extends checked_env #(request_item, response_item, ref_model); - `uvm_component_utils(env) - virtual bank_if request; - virtual bank_if response; - - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual bank_if)::get( - this, "", "request", request - ) || !uvm_config_db#(virtual bank_if)::get( - this, "", "response", response - )) - `uvm_fatal("VIF", "bank interfaces missing") - endfunction - - task run_phase(uvm_phase phase); - request_item req; - response_item rsp; - forever begin - @(posedge request.clock); - if (request.reset) begin - scoreboard.cancel_pending(); - end else begin - if (request.valid && request.ready) begin - req = request_item::type_id::create("request"); - req.addr = request.addr; - req.write = request.write; - req.data = request.data; - req.mask = request.mask; - req.tag = request.tag; - input_export.write(req); - end - if (response.valid && response.ready) begin - rsp = response_item::type_id::create("response"); - rsp.data = response.data; - rsp.tag = response.tag; - rsp.error = response.error; - output_export.write(rsp); - end - end - end - endtask -endclass diff --git a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/interface.sv b/arch/src/main/scala/framework/mem-core/bank/src/main/resources/interface.sv deleted file mode 100644 index 8f1a99a93..000000000 --- a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/interface.sv +++ /dev/null @@ -1,18 +0,0 @@ -interface bank_if ( - input logic clock -); - logic reset; - logic valid; - logic ready; - logic [3:0] addr; - logic write; - logic [31:0] data; - logic [3:0] mask; - logic [7:0] tag; - logic error; - - assert property (@(posedge clock) disable iff (reset) valid && !ready |=> valid && $stable( - {data, tag, error} - )); - cover property (@(posedge clock) disable iff (reset) valid && !ready ##1 valid && ready); -endinterface diff --git a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/item.svh b/arch/src/main/scala/framework/mem-core/bank/src/main/resources/item.svh deleted file mode 100644 index 2c5da1fb2..000000000 --- a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/item.svh +++ /dev/null @@ -1,31 +0,0 @@ -class request_item extends uvm_sequence_item; - logic [3:0] addr; - logic write; - logic [31:0] data; - logic [3:0] mask; - logic [7:0] tag; - `uvm_object_utils_begin(request_item) - `uvm_field_int(addr, UVM_ALL_ON) - `uvm_field_int(write, UVM_ALL_ON) - `uvm_field_int(data, UVM_ALL_ON) - `uvm_field_int(mask, UVM_ALL_ON) - `uvm_field_int(tag, UVM_ALL_ON) - `uvm_object_utils_end - function new(string name = "request_item"); - super.new(name); - endfunction -endclass - -class response_item extends uvm_sequence_item; - logic [31:0] data; - logic [7:0] tag; - logic error; - `uvm_object_utils_begin(response_item) - `uvm_field_int(data, UVM_ALL_ON) - `uvm_field_int(tag, UVM_ALL_ON) - `uvm_field_int(error, UVM_ALL_ON) - `uvm_object_utils_end - function new(string name = "response_item"); - super.new(name); - endfunction -endclass diff --git a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/package.sv b/arch/src/main/scala/framework/mem-core/bank/src/main/resources/package.sv deleted file mode 100644 index 7b19fb001..000000000 --- a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/package.sv +++ /dev/null @@ -1,20 +0,0 @@ -package bank_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - - import "DPI-C" function chandle bank_ref_create(input int unsigned entries); - import "DPI-C" function void bank_ref_destroy(input chandle model); - import "DPI-C" function int unsigned bank_ref_access( - input chandle model, - input int unsigned addr, - input byte unsigned write, - input int unsigned data, - input byte unsigned mask - ); - - `include "item.svh" - `include "ref_model.svh" - `include "env.svh" - `include "test.svh" -endpackage diff --git a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/ref_model.svh b/arch/src/main/scala/framework/mem-core/bank/src/main/resources/ref_model.svh deleted file mode 100644 index 8458a6539..000000000 --- a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/ref_model.svh +++ /dev/null @@ -1,26 +0,0 @@ -class ref_model extends reference_model #(request_item, response_item); - `uvm_component_utils(ref_model) - chandle model; - - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - - function void build_phase(uvm_phase phase); - super.build_phase(phase); - model = bank_ref_create(16); - endfunction - - function void write(request_item t); - response_item expected = response_item::type_id::create("expected"); - expected.data = bank_ref_access(model, t.addr, t.write, t.data, t.mask); - expected.tag = t.tag; - expected.error = 0; - expected_ap.write(expected); - endfunction - - function void final_phase(uvm_phase phase); - bank_ref_destroy(model); - super.final_phase(phase); - endfunction -endclass diff --git a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/test.svh b/arch/src/main/scala/framework/mem-core/bank/src/main/resources/test.svh deleted file mode 100644 index 8a0abae63..000000000 --- a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/test.svh +++ /dev/null @@ -1,106 +0,0 @@ -class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual bank_if request; - virtual bank_if response; - env test_env; - int checked = 0; - - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual bank_if)::get( - this, "", "request", request - ) || !uvm_config_db#(virtual bank_if)::get( - this, "", "response", response - )) - `uvm_fatal("VIF", "bank interfaces missing") - test_env = env::type_id::create("env", this); - endfunction - - task access (bit wr, bit [3:0] addr, bit [31:0] data, bit [3:0] mask, bit [7:0] tag, int stall); - int unsigned target = test_env.scoreboard.checked + 1; - @(negedge request.clock); - request.valid = 1; - request.write = wr; - request.addr = addr; - request.data = data; - request.mask = mask; - request.tag = tag; - response.ready = 0; - do @(posedge request.clock); while (!request.ready); - @(negedge request.clock); - request.valid = 0; - while (!response.valid) @(negedge request.clock); - repeat (stall + 1) begin - if (response.valid !== 1'b1) `uvm_fatal("BANK", "response valid dropped under backpressure") - if (request.ready !== 1'b0) - `uvm_fatal("BANK", "accepted another request with an outstanding response") - @(negedge request.clock); - end - response.ready = 1; - @(negedge request.clock); - if (response.valid !== 1'b0) `uvm_fatal("BANK", "response was not retired") - response.ready = 0; - test_env.scoreboard.wait_checked(target); - checked++; - endtask - - task reset_pending(bit wr); - @(negedge request.clock); - request.valid = 1; - request.write = wr; - request.addr = 15; - request.data = 'hface1234; - request.mask = 15; - request.tag = 255; - response.ready = 0; - do @(posedge request.clock); while (!request.ready); - @(negedge request.clock); - request.valid = 0; - request.reset = 1; - response.reset = 1; - repeat (2) @(negedge request.clock); - request.reset = 0; - response.reset = 0; - repeat (3) begin - @(negedge request.clock); - if (response.valid !== 1'b0 || request.ready !== 1'b1) - `uvm_fatal("RESET", "reset did not discard outstanding transaction") - end - access (0, 15, 0, 0, 42, 3); - endtask - - task execute(); - request.valid = 0; - request.write = 0; - request.addr = 0; - request.data = 0; - request.mask = 0; - request.tag = 0; - request.error = 0; - request.reset = 1; - response.reset = 1; - response.ready = 0; - repeat (4) @(negedge request.clock); - request.reset = 0; - response.reset = 0; - for (int a = 0; a < 16; a++) access (1, a, $urandom(), 15, a, 0); - for (int a = 0; a < 16; a++) access (0, a, 0, 0, 255 - a, 2); - for (int m = 0; m < 16; m++) begin - access (1, m, $urandom(), m, m, 5); - access (0, m, 0, 0, m, 0); - end - reset_pending(1); - reset_pending(0); - for (int a = 0; a < 16; a++) access (0, a, 0, 0, a, 0); - response.ready = 1; - repeat (2) begin - @(negedge request.clock); - if (response.valid !== 1'b0) `uvm_fatal("BANK", "unexpected response while idle") - end - `uvm_info("BANK", $sformatf("Checked %0d responses", checked), UVM_LOW) - endtask -endclass diff --git a/arch/src/main/scala/framework/mem-core/bank/src/main/scala/Bank.scala b/arch/src/main/scala/framework/mem-core/bank/src/main/scala/Bank.scala deleted file mode 100644 index cf7e48059..000000000 --- a/arch/src/main/scala/framework/mem-core/bank/src/main/scala/Bank.scala +++ /dev/null @@ -1,68 +0,0 @@ -package memcore.memory.bank - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} - -class BankRequest(p: BankSetParams) extends Bundle { - val addr = UInt(p.rowBits.W) - val write = Bool() - val data = UInt(p.dataBits.W) - val mask = UInt(p.bytes.W) - val tag = UInt(p.tagBits.W) -} - -class BankResponse(p: BankSetParams) extends Bundle { - val data = UInt(p.dataBits.W) - val tag = UInt(p.tagBits.W) - val error = Bool() -} - -/** One physical SRAM bank. Cache and NoC policy remain outside this module. */ -@instantiable -class Bank(p: BankSetParams) extends Module { - - @public - val io = IO(new Bundle { - val request = Flipped(Decoupled(new BankRequest(p))) - val response = Decoupled(new BankResponse(p)) - }) - - val memory = SyncReadMem(p.entriesPerBank, Vec(p.bytes, UInt(8.W))) - val request = Reg(new BankRequest(p)) - val responseValid = RegInit(false.B) - val readPending = RegInit(false.B) - val responseData = Reg(UInt(p.dataBits.W)) - - val readData = memory.readWrite( - io.request.bits.addr, - io.request.bits.data.asTypeOf(Vec(p.bytes, UInt(8.W))), - io.request.bits.mask.asBools, - io.request.fire, - io.request.bits.write - ) - - io.request.ready := !responseValid && !readPending - when(io.request.fire) { - request := io.request.bits - when(io.request.bits.write) { - responseData := 0.U - responseValid := true.B - }.otherwise { - readPending := true.B - } - } - when(readPending) { - responseData := readData.asUInt - responseValid := true.B - readPending := false.B - } - - io.response.valid := responseValid - io.response.bits.data := responseData - io.response.bits.tag := request.tag - io.response.bits.error := false.B - when(io.response.fire) { - responseValid := false.B - } -} diff --git a/arch/src/main/scala/framework/mem-core/bank/src/main/scala/Emit.scala b/arch/src/main/scala/framework/mem-core/bank/src/main/scala/Emit.scala deleted file mode 100644 index 9d56eab79..000000000 --- a/arch/src/main/scala/framework/mem-core/bank/src/main/scala/Emit.scala +++ /dev/null @@ -1,15 +0,0 @@ -package memcore.memory.bank - -object Emit extends App { - val p = BankSetParams(dataBits = 32, banks = 4, entriesPerBank = 16) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Bank(p), - firtoolOpts = args, - args = Array("--target-dir", "build/Bank", "--split-verilog") - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new BankSet(p), - firtoolOpts = args, - args = Array("--target-dir", "build/BankSet", "--split-verilog") - ) -} diff --git a/arch/src/main/scala/framework/mem-core/cache/build.sc b/arch/src/main/scala/framework/mem-core/cache/build.sc deleted file mode 100644 index b9e3f0dc3..000000000 --- a/arch/src/main/scala/framework/mem-core/cache/build.sc +++ /dev/null @@ -1,21 +0,0 @@ -import mill._ -import mill.scalalib._ - -object queue extends SbtModule { - override def millSourcePath = os.pwd / os.up / "queue" - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") -} - -object cache extends SbtModule { - override def moduleDeps = Seq(queue) - override def millSourcePath = os.pwd - override def sources = T.sources(super.sources() ++ Seq(PathRef(millSourcePath / "configs"))) - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0", ivy"tech.sparse::toml-scala:0.2.2") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") - override def mainClass = Some("memcore.memory.cache.Emit") -} diff --git a/arch/src/main/scala/framework/mem-core/cache/configs/CacheParams.scala b/arch/src/main/scala/framework/mem-core/cache/configs/CacheParams.scala deleted file mode 100644 index ba14f4da2..000000000 --- a/arch/src/main/scala/framework/mem-core/cache/configs/CacheParams.scala +++ /dev/null @@ -1,54 +0,0 @@ -package memcore.memory.cache.configs - -import chisel3.util.{isPow2, log2Ceil} -import scala.io.Source -import scala.util.Using -import toml.{Toml, Value} - -case class CacheParams( - addressBits: Int, - lineBytes: Int, - sets: Int, - ways: Int, - idBits: Int, - metadataBits: Int, - responseDepth: Int) { - require(lineBytes >= 8 && isPow2(lineBytes)) - require(sets >= 2 && isPow2(sets)) - require(ways >= 1 && isPow2(ways)) - require(idBits > 0 && metadataBits > 0 && responseDepth >= 2) - val offsetBits = log2Ceil(lineBytes) - val setBits = log2Ceil(sets) - val wayBits = math.max(1, log2Ceil(ways)) - val tagBits = addressBits - offsetBits - setBits - val lineBits = lineBytes * 8 - require(tagBits > 0) -} - -object CacheParams { - - def load(path: String): CacheParams = { - val text = Using.resource(Source.fromFile(path, "UTF-8"))(_.mkString) - val table = Toml.parse(text) match { - case Right(Value.Tbl(root)) => root("cache") match { - case Value.Tbl(table) => table - case value => throw new IllegalArgumentException(s"Expected [cache] table, got $value") - } - case other => throw new IllegalArgumentException(s"Invalid cache configuration $path: $other") - } - def int(name: String): Int = table(name) match { - case Value.Num(n) if n == n.toInt => n.toInt - case value => throw new IllegalArgumentException(s"Expected integer cache.$name, got $value") - } - CacheParams( - int("addressBits"), - int("lineBytes"), - int("sets"), - int("ways"), - int("idBits"), - int("metadataBits"), - int("responseDepth") - ) - } - -} diff --git a/arch/src/main/scala/framework/mem-core/cache/src/main/resources/cache.el b/arch/src/main/scala/framework/mem-core/cache/src/main/resources/cache.el deleted file mode 100644 index 7df1fa991..000000000 --- a/arch/src/main/scala/framework/mem-core/cache/src/main/resources/cache.el +++ /dev/null @@ -1,113 +0,0 @@ -// Format Version: 2 -// Reviewed for configs/default.toml: 44-bit addresses, 64-byte lines, 4 sets, 2 ways, responseDepth=2. - -CHECKSUM: "1581756980 3097157900" -INSTANCE: cache_tb.dut -ANNOTATION: "Capacity assertion diagnostics: admission reserves count plus pending, so an outstanding pipeline response always has queue space." -Block 2 "3078567325" "if (1)" -ANNOTATION: "Capacity assertion diagnostics: admission reserves count plus pending, so an outstanding pipeline response always has queue space." -Block 3 "1895567494" "$error(\"Assertion failed: Cache response capacity reservation failed\n at Cache.scala:51 assert(!pending || responses.io.enq.ready, \\"Cache response capacity reservation failed\\")\n\");" -ANNOTATION: "Capacity assertion diagnostics: admission reserves count plus pending, so an outstanding pipeline response always has queue space." -Block 5 "3078567325" "if (1)" -ANNOTATION: "Capacity assertion diagnostics: admission reserves count plus pending, so an outstanding pipeline response always has queue space." -Block 6 "2299949409" "$fatal;" -ANNOTATION: "Opcode assertion diagnostics: the legal interface operations are Lookup, Read, Write, Fill, and Invalidate." -Block 10 "422554957" "if (1)" -ANNOTATION: "Opcode assertion diagnostics: the legal interface operations are Lookup, Read, Write, Fill, and Invalidate." -Block 11 "3963963684" "$error(\"Assertion failed: Unknown Cache operation\n at Cache.scala:67 assert(req.op <= CacheOp.Invalidate.U, \\"Unknown Cache operation\\")\n\");" -ANNOTATION: "Opcode assertion diagnostics: the legal interface operations are Lookup, Read, Write, Fill, and Invalidate." -Block 13 "422554957" "if (1)" -ANNOTATION: "Opcode assertion diagnostics: the legal interface operations are Lookup, Read, Write, Fill, and Invalidate." -Block 14 "2766194732" "$fatal;" -ANNOTATION: "Alignment assertion diagnostics: requests use cache-line-aligned addresses." -Block 18 "874237168" "if (1)" -ANNOTATION: "Alignment assertion diagnostics: requests use cache-line-aligned addresses." -Block 19 "1020309457" "$error(\"Assertion failed: Cache operations require a line-aligned address\n at Cache.scala:68 assert(req.addr(p.offsetBits - 1, 0) === 0.U, \\"Cache operations require a line-aligned address\\")\n\");" -ANNOTATION: "Alignment assertion diagnostics: requests use cache-line-aligned addresses." -Block 21 "874237168" "if (1)" -ANNOTATION: "Alignment assertion diagnostics: requests use cache-line-aligned addresses." -Block 22 "1069368929" "$fatal;" -ANNOTATION: "Duplicate-tag assertion diagnostics: reset clears valid, Fill requires an absent tag or its existing way, and Invalidate only clears valid." -Block 26 "1759021865" "if (1)" -ANNOTATION: "Duplicate-tag assertion diagnostics: reset clears valid, Fill requires an absent tag or its existing way, and Invalidate only clears valid." -Block 27 "155016256" "$error(\"Assertion failed: Duplicate Cache tag\n at Cache.scala:70 assert(PopCount(hits) <= 1.U, \\"Duplicate Cache tag\\")\n\");" -ANNOTATION: "Duplicate-tag assertion diagnostics: reset clears valid, Fill requires an absent tag or its existing way, and Invalidate only clears valid." -Block 29 "1759021865" "if (1)" -ANNOTATION: "Duplicate-tag assertion diagnostics: reset clears valid, Fill requires an absent tag or its existing way, and Invalidate only clears valid." -Block 30 "3067504005" "$fatal;" -ANNOTATION: "Fill assertion diagnostics: the coherence controller must not install the same line in two ways of a set." -Block 34 "1166789632" "if (1)" -ANNOTATION: "Fill assertion diagnostics: the coherence controller must not install the same line in two ways of a set." -Block 35 "1252711441" "$error(\"Assertion failed: Cache fill would create a duplicate tag\n at Cache.scala:83 assert(!hits.orR || hits(req.way), \\"Cache fill would create a duplicate tag\\")\n\");" -ANNOTATION: "Fill assertion diagnostics: the coherence controller must not install the same line in two ways of a set." -Block 37 "1166789632" "if (1)" -ANNOTATION: "Fill assertion diagnostics: the coherence controller must not install the same line in two ways of a set." -Block 38 "3120397658" "$fatal;" -ANNOTATION: "Write assertion diagnostics: masked writes require a matching valid resident line in the selected way." -Block 42 "2426821494" "if (1)" -ANNOTATION: "Write assertion diagnostics: masked writes require a matching valid resident line in the selected way." -Block 43 "1714232348" "$error(\"Assertion failed: Cache write requires the matching resident line\n at Cache.scala:93 assert(entryValid && hits(req.way), \\"Cache write requires the matching resident line\\")\n\");" -ANNOTATION: "Write assertion diagnostics: masked writes require a matching valid resident line in the selected way." -Block 45 "2426821494" "if (1)" -ANNOTATION: "Write assertion diagnostics: masked writes require a matching valid resident line in the selected way." -Block 46 "3284370139" "$fatal;" - -CHECKSUM: "1581756980 4074018439" -INSTANCE: cache_tb.dut -ANNOTATION: "Requests are inactive during reset; this condition only gates assertion diagnostics." -Condition 1 "2427994462" "(_readData_1_T & ((~reset))) 1 -1" (2 "10") -ANNOTATION: "Reserved capacity invariant: count plus pending never exceeds responseDepth, so pending implies enq.ready." -Condition 2 "1313259708" "(((~reset)) & ( ~ (((~pending)) | _responses_io_enq_ready) )) 1 -1" (1 "01") -ANNOTATION: "Reserved capacity invariant: count plus pending never exceeds responseDepth, so pending implies enq.ready." -Condition 2 "1313259708" "(((~reset)) & ( ~ (((~pending)) | _responses_io_enq_ready) )) 1 -1" (3 "11") -ANNOTATION: "Reserved capacity invariant: count plus pending never exceeds responseDepth, so pending implies enq.ready." -Condition 4 "1917659308" "(((~pending)) | _responses_io_enq_ready) 1 -1" (1 "00") -ANNOTATION: "Only the five defined operation encodings are driven; invalid-opcode diagnostics are outside legal-operation coverage." -Condition 5 "521367662" "(_GEN_10 & (io_request_bits_op > 3'h4)) 1 -1" (1 "01") -ANNOTATION: "Only the five defined operation encodings are driven; invalid-opcode diagnostics are outside legal-operation coverage." -Condition 5 "521367662" "(_GEN_10 & (io_request_bits_op > 3'h4)) 1 -1" (3 "11") -ANNOTATION: "Addresses are line aligned; misalignment diagnostics are outside legal-operation coverage." -Condition 6 "3741468578" "(_GEN_10 & ((|io_request_bits_addr[5:0]))) 1 -1" (1 "01") -ANNOTATION: "Addresses are line aligned; misalignment diagnostics are outside legal-operation coverage." -Condition 6 "3741468578" "(_GEN_10 & ((|io_request_bits_addr[5:0]))) 1 -1" (3 "11") -ANNOTATION: "Valid tags remain unique: reset clears validity and legal fills cannot duplicate another valid way." -Condition 7 "3021235683" "(_GEN_10 & _GEN_12[1]) 1 -1" (1 "01") -ANNOTATION: "Valid tags remain unique: reset clears validity and legal fills cannot duplicate another valid way." -Condition 7 "3021235683" "(_GEN_10 & _GEN_12[1]) 1 -1" (3 "11") -ANNOTATION: "Legal fills target an absent line or its existing way; failed and reset-gated fill diagnostics are excluded." -Condition 8 "2233729839" "(_readData_1_T & _mask_T_2 & ((~reset)) & ( ~ (((~(|hits))) | _GEN_11[0]) )) 1 -1" (1 "0111") -ANNOTATION: "Legal fills target an absent line or its existing way; failed and reset-gated fill diagnostics are excluded." -Condition 8 "2233729839" "(_readData_1_T & _mask_T_2 & ((~reset)) & ( ~ (((~(|hits))) | _GEN_11[0]) )) 1 -1" (3 "1101") -ANNOTATION: "Legal fills target an absent line or its existing way; failed and reset-gated fill diagnostics are excluded." -Condition 8 "2233729839" "(_readData_1_T & _mask_T_2 & ((~reset)) & ( ~ (((~(|hits))) | _GEN_11[0]) )) 1 -1" (5 "1111") -ANNOTATION: "Legal writes match a valid line; failed and reset-gated write diagnostics are excluded." -Condition 11 "3664488518" "(_readData_1_T & _GEN_8 & ((~reset)) & ( ~ (entryValid & _GEN_11[0]) )) 1 -1" (1 "0111") -ANNOTATION: "Legal writes match a valid line; failed and reset-gated write diagnostics are excluded." -Condition 11 "3664488518" "(_readData_1_T & _GEN_8 & ((~reset)) & ( ~ (entryValid & _GEN_11[0]) )) 1 -1" (3 "1101") -ANNOTATION: "Legal writes match a valid line; failed and reset-gated write diagnostics are excluded." -Condition 11 "3664488518" "(_readData_1_T & _GEN_8 & ((~reset)) & ( ~ (entryValid & _GEN_11[0]) )) 1 -1" (5 "1111") -ANNOTATION: "A tag hit includes its valid bit. Selecting a hit cannot produce entryValid=0." -Condition 13 "2913363430" "(entryValid & _GEN_11[0]) 1 -1" (1 "01") - -CHECKSUM: "784137509 320702925" -INSTANCE: cache_tb.dut.responses -ANNOTATION: "The Cache reserves queue capacity before accepting requests; enq.valid cannot occur with enq.ready=0." -Condition 6 "2766317690" "(io_enq_ready_0 & io_enq_valid) 1 -1" (1 "01") - -CHECKSUM: "1581756980 2991418269" -INSTANCE: cache_tb.dut -ANNOTATION: "Line addresses are 64-byte aligned; low six address bits are zero." -Toggle io_request_bits_addr [5:0] "net io_request_bits_addr[43:0]" -ANNOTATION: "Line addresses are 64-byte aligned; low six address bits are zero." -Toggle io_response_bits_addr [5:0] "net io_response_bits_addr[43:0]" -ANNOTATION: "Line addresses are 64-byte aligned; low six address bits are zero." -Toggle result_addr [5:0] "reg result_addr[43:0]" - -CHECKSUM: "784137509 896502883" -INSTANCE: cache_tb.dut.responses -ANNOTATION: "Line addresses are 64-byte aligned; low six address bits are zero." -Toggle io_enq_bits_addr [5:0] "net io_enq_bits_addr[43:0]" -ANNOTATION: "Line addresses are 64-byte aligned; low six address bits are zero." -Toggle io_deq_bits_addr [5:0] "net io_deq_bits_addr[43:0]" -ANNOTATION: "Line addresses are 64-byte aligned; low six address bits are zero. Queue packing maps address[5:0] to RAM output[17:12]." -Toggle _ram_ext_R0_data [17:12] "net _ram_ext_R0_data[571:0]" diff --git a/arch/src/main/scala/framework/mem-core/cache/src/main/resources/cache.f b/arch/src/main/scala/framework/mem-core/cache/src/main/resources/cache.f deleted file mode 100644 index 3a32dc7dc..000000000 --- a/arch/src/main/scala/framework/mem-core/cache/src/main/resources/cache.f +++ /dev/null @@ -1,8 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/interface.sv -@RESOURCES@/package.sv --F @RTL@/Cache/filelist.f -@RESOURCES@/cache_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/cache/src/main/resources/env.svh b/arch/src/main/scala/framework/mem-core/cache/src/main/resources/env.svh deleted file mode 100644 index cf8d93fd0..000000000 --- a/arch/src/main/scala/framework/mem-core/cache/src/main/resources/env.svh +++ /dev/null @@ -1,62 +0,0 @@ -class env extends checked_env #(request_item, response_item, ref_model); - `uvm_component_utils(env) - virtual cache_request_if request; - virtual cache_response_if response; - bit block_responses = 0; - int unsigned accepted = 0; - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual cache_request_if)::get( - this, "", "request", request - ) || !uvm_config_db#(virtual cache_response_if)::get( - this, "", "response", response - )) - `uvm_fatal("VIF", "Cache interfaces missing") - endfunction - task run_phase(uvm_phase phase); - request_item req; - response_item rsp; - fork - forever begin - @(response.cb); - response.cb.ready <= !response.reset && !block_responses; - end - forever begin - @(posedge request.clock); - if (request.reset) begin - scoreboard.cancel_pending(); - cache_ref_reset(model.model); - end else begin - if (request.valid && request.ready) begin - req = request_item::type_id::create("req"); - req.id = request.id; - req.op = request.op; - req.addr = request.addr; - req.way = request.way; - req.data = request.data; - req.mask = request.mask; - req.metadata = request.metadata; - req.eligible = request.eligible; - input_export.write(req); - accepted++; - end - if (response.valid && response.ready) begin - rsp = response_item::type_id::create("rsp"); - rsp.id = response.id; - rsp.hit = response.hit; - rsp.available = response.available; - rsp.entry_valid = response.entry_valid; - rsp.way = response.way; - rsp.addr = response.addr; - rsp.data = response.data; - rsp.metadata = response.metadata; - output_export.write(rsp); - end - end - end - join - endtask -endclass diff --git a/arch/src/main/scala/framework/mem-core/cache/src/main/resources/item.svh b/arch/src/main/scala/framework/mem-core/cache/src/main/resources/item.svh deleted file mode 100644 index a073b9265..000000000 --- a/arch/src/main/scala/framework/mem-core/cache/src/main/resources/item.svh +++ /dev/null @@ -1,36 +0,0 @@ -class request_item extends uvm_sequence_item; - logic [`CACHE_ID_BITS-1:0] id; - logic [2:0] op; - logic [`CACHE_ADDR_BITS-1:0] addr; - logic [`CACHE_WAY_BITS-1:0] way; - bit [`CACHE_LINE_BITS-1:0] data; - bit [`CACHE_LINE_BYTES-1:0] mask; - logic [`CACHE_META_BITS-1:0] metadata; - bit [`CACHE_WAYS-1:0] eligible; - `uvm_object_utils(request_item) - function new(string name = "request_item"); - super.new(name); - endfunction -endclass - -class response_item extends uvm_sequence_item; - logic [`CACHE_ID_BITS-1:0] id; - logic hit, available, entry_valid; - logic [`CACHE_WAY_BITS-1:0] way; - logic [`CACHE_ADDR_BITS-1:0] addr; - logic [`CACHE_LINE_BITS-1:0] data; - logic [`CACHE_META_BITS-1:0] metadata; - `uvm_object_utils_begin(response_item) - `uvm_field_int(id, UVM_ALL_ON) - `uvm_field_int(hit, UVM_ALL_ON) - `uvm_field_int(available, UVM_ALL_ON) - `uvm_field_int(entry_valid, UVM_ALL_ON) - `uvm_field_int(way, UVM_ALL_ON) - `uvm_field_int(addr, UVM_ALL_ON) - `uvm_field_int(data, UVM_ALL_ON) - `uvm_field_int(metadata, UVM_ALL_ON) - `uvm_object_utils_end - function new(string name = "response_item"); - super.new(name); - endfunction -endclass diff --git a/arch/src/main/scala/framework/mem-core/cache/src/main/resources/package.sv b/arch/src/main/scala/framework/mem-core/cache/src/main/resources/package.sv deleted file mode 100644 index c90d270f7..000000000 --- a/arch/src/main/scala/framework/mem-core/cache/src/main/resources/package.sv +++ /dev/null @@ -1,35 +0,0 @@ -`include "cache_config.svh" -package cache_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - localparam LOOKUP = 0, READ = 1, WRITE = 2, FILL = 3, INVALIDATE = 4; - import "DPI-C" function chandle cache_ref_create( - input int unsigned sets, - ways, - line_bytes - ); - import "DPI-C" function void cache_ref_destroy(input chandle model); - import "DPI-C" function void cache_ref_reset(input chandle model); - import "DPI-C" function void cache_ref_access( - input chandle model, - input int unsigned op, - input longint unsigned addr, - input int unsigned way, - input bit [`CACHE_LINE_BITS-1:0] data, - input bit [`CACHE_LINE_BYTES-1:0] mask, - input int unsigned metadata, - input bit [`CACHE_WAYS-1:0] eligible, - output byte unsigned hit, - available, - output int unsigned out_way, - output byte unsigned entry_valid, - output longint unsigned out_addr, - output bit [`CACHE_LINE_BITS-1:0] out_data, - output int unsigned out_metadata - ); - `include "item.svh" - `include "ref_model.svh" - `include "env.svh" - `include "test.svh" -endpackage diff --git a/arch/src/main/scala/framework/mem-core/cache/src/main/resources/test.svh b/arch/src/main/scala/framework/mem-core/cache/src/main/resources/test.svh deleted file mode 100644 index 6ce2bbb9e..000000000 --- a/arch/src/main/scala/framework/mem-core/cache/src/main/resources/test.svh +++ /dev/null @@ -1,142 +0,0 @@ -class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual cache_request_if request; - env test_env; - int unsigned expected = 0; - bit [`CACHE_ID_BITS-1:0] serial = 0; - bit check_rate = 1; - bit idle_between = 0; - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - test_env = env::type_id::create("env", this); - if (!uvm_config_db#(virtual cache_request_if)::get(this, "", "request", request)) - `uvm_fatal("VIF", "Cache request interface missing") - endfunction - function longint unsigned address(int tag, int set_index); - return (longint'(tag) * `CACHE_SETS + set_index) * `CACHE_LINE_BYTES; - endfunction - task send(int op, longint unsigned addr, int way = 0, bit [`CACHE_LINE_BYTES-1:0] mask = '1, - bit [`CACHE_WAYS-1:0] eligible = '1, bit [`CACHE_LINE_BITS-1:0] data = '0, - int metadata = 0); - int cycles = 0; - request.cb.valid <= 1; - request.cb.id <= serial; - request.cb.op <= op; - request.cb.addr <= addr; - request.cb.way <= way; - request.cb.data <= data; - request.cb.mask <= mask; - request.cb.metadata <= metadata; - request.cb.eligible <= eligible; - do begin - @(request.cb); - cycles++; - end while (!request.cb.ready); - if (check_rate && cycles != 1) - `uvm_fatal("THROUGHPUT", "Cache inserted a request bubble without response backpressure") - request.cb.valid <= 0; - expected++; - serial++; - if (idle_between) @(request.cb); - endtask - task drain(); - test_env.scoreboard.wait_checked(expected); - @(request.cb); - endtask - task execute(); - request.reset = 1; - request.valid = 0; - request.id = 0; - request.op = 0; - request.addr = 0; - request.way = 0; - request.data = 0; - request.mask = 0; - request.metadata = 0; - request.eligible = 0; - repeat (4) @(request.cb); - request.cb.reset <= 0; - @(request.cb); - send(LOOKUP, 0, 0, '0, '0); - send(LOOKUP, 0); - for (int s = 0; s < `CACHE_SETS; s++) begin - for (int w = 0; w < `CACHE_WAYS; w++) begin - send(READ, address(w + 1, s), w); - send(FILL, address(w + 1, s), w, '1, '1, w[0] ? '1 : '0, w); - send(LOOKUP, address(w + 1, s)); - send(WRITE, address(w + 1, s), w, {(`CACHE_LINE_BYTES / 2) {2'b01}}, '1, w[0] ? '0 : '1, - '1); - send(READ, address(w + 1, s), w); - end - end - send(LOOKUP, address(`CACHE_WAYS + 1, 0)); - send(LOOKUP, address(`CACHE_WAYS + 1, 0), 0, '0, '0); - for (int w = 0; w < `CACHE_WAYS; w++) send(LOOKUP, address(`CACHE_WAYS + 1, 0), 0, '0, 1 << w); - send(INVALIDATE, address(1, 0), 0); - send(LOOKUP, address(1, 0)); - send(FILL, address(`CACHE_WAYS + 1, 0), 0, '1, '1, '1, '1); - send(LOOKUP, address(`CACHE_WAYS + 2, 0)); - send(WRITE, address(`CACHE_WAYS + 1, 0), 0, '0, '1, '0, 0); - send(READ, address(`CACHE_WAYS + 1, 0), 0); - send(INVALIDATE, address(`CACHE_WAYS + 1, 0), 0); - send(INVALIDATE, address(`CACHE_WAYS + 1, 0), 0); - drain(); - - idle_between = 1; - for (int s = 0; s < `CACHE_SETS; s++) begin - for (int w = 0; w < `CACHE_WAYS; w++) send(INVALIDATE, address(0, s), w); - for (int w = 0; w < `CACHE_WAYS; w++) begin - longint unsigned high_addr = (((64'h1 << `CACHE_ADDR_BITS) - 1) & - ~(longint'(`CACHE_SETS * `CACHE_LINE_BYTES) - 1)) | (s * `CACHE_LINE_BYTES); - send(FILL, address(0, s), w, '1, '1, '0, 0); - send(LOOKUP, address(0, s), 0, '0, '0); - send(FILL, high_addr, w, '1, '1, '1, '1); - send(LOOKUP, high_addr); - send(WRITE, high_addr, w, '1, '1, '0, 0); - send(READ, high_addr, w); - send(FILL, address(0, s), w, '1, '1, '0, 0); - send(READ, address(0, s), w); - send(INVALIDATE, address(0, s), w); - end - end - drain(); - - idle_between = 0; - serial = '1; - send(READ, address(0, 0), 0, '0, '0); - drain(); - serial = 0; - send(READ, address(0, 0), 0, '0, '0); - drain(); - check_rate = 0; - test_env.block_responses = 1; - repeat (2) @(request.cb); - fork - begin - for (int i = 0; i < `CACHE_RESPONSE_DEPTH + 2; i++) send(LOOKUP, address(2, 0)); - end - begin - repeat (`CACHE_RESPONSE_DEPTH + 5) @(request.cb); - test_env.block_responses = 0; - end - join - drain(); - - test_env.block_responses = 1; - repeat (2) @(request.cb); - for (int i = 0; i < `CACHE_RESPONSE_DEPTH; i++) send(READ, address(1, 1), 0); - repeat (2) @(request.cb); - request.cb.reset <= 1; - repeat (2) @(request.cb); - expected = test_env.scoreboard.checked; - request.cb.reset <= 0; - test_env.block_responses = 0; - @(request.cb); - for (int s = 0; s < `CACHE_SETS; s++) send(LOOKUP, address(1, s)); - drain(); - `uvm_info("CACHE", $sformatf("Checked %0d responses", test_env.scoreboard.checked), UVM_LOW) - endtask -endclass diff --git a/arch/src/main/scala/framework/mem-core/cache/src/main/scala/Cache.scala b/arch/src/main/scala/framework/mem-core/cache/src/main/scala/Cache.scala deleted file mode 100644 index adaac8976..000000000 --- a/arch/src/main/scala/framework/mem-core/cache/src/main/scala/Cache.scala +++ /dev/null @@ -1,110 +0,0 @@ -package memcore.memory.cache - -import memcore.memory.queue.Queue - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import memcore.memory.cache.configs.CacheParams - -@instantiable -class Cache(p: CacheParams) extends Module { - @public - val io = IO(new CacheIO(p)) - - val valid = RegInit(VecInit(Seq.fill(p.ways)(VecInit(Seq.fill(p.sets)(false.B))))) - val tags = Reg(Vec(p.ways, Vec(p.sets, UInt(p.tagBits.W)))) - val metadata = Reg(Vec(p.ways, Vec(p.sets, UInt(p.metadataBits.W)))) - val data = Seq.fill(p.ways)(SyncReadMem(p.sets, Vec(p.lineBytes, UInt(8.W)))) - val nextWay = RegInit(VecInit(Seq.fill(p.sets)(0.U(p.wayBits.W)))) - val responses = Module(new Queue(new CacheResponse(p), p.responseDepth, pipe = true)) - val pending = RegInit(false.B) - val pendingRead = Reg(Bool()) - val pendingWay = Reg(UInt(p.wayBits.W)) - val result = Reg(new CacheResponse(p)) - - val req = io.request.bits - val set = req.addr(p.offsetBits + p.setBits - 1, p.offsetBits) - val tag = req.addr(p.addressBits - 1, p.offsetBits + p.setBits) - val hits = VecInit((0 until p.ways).map(w => valid(w)(set) && tags(w)(set) === tag)).asUInt - val invalid = VecInit((0 until p.ways).map(w => !valid(w)(set) && req.eligible(w))).asUInt - - val order = VecInit((0 until p.ways).map { offset => - val way = if (p.ways == 1) 0.U else (nextWay(set) + offset.U)(p.wayBits - 1, 0) - req.eligible(way) - }) - - val replacement = - if (p.ways == 1) 0.U - else - (nextWay(set) + PriorityEncoder(order.asUInt))(p.wayBits - 1, 0) - - val lookupWay = Mux(hits.orR, PriorityEncoder(hits), Mux(invalid.orR, PriorityEncoder(invalid), replacement)) - val lookup = req.op === CacheOp.Lookup.U - val selected = Mux(lookup, lookupWay, req.way) - val available = !lookup || hits.orR || req.eligible.orR - val entryValid = available && valid(selected)(set) - val read = (lookup || req.op === CacheOp.Read.U) && entryValid - - io.request.ready := (responses.io.count +& pending.asUInt) < p.responseDepth.U || responses.io.deq.fire - responses.io.enq.valid := pending - responses.io.enq.bits := result - io.response <> responses.io.deq - pending := io.request.fire - assert(!pending || responses.io.enq.ready, "Cache response capacity reservation failed") - - val readData = Wire(Vec(p.ways, UInt(p.lineBits.W))) - for (way <- 0 until p.ways) { - val write = req.op === CacheOp.Write.U || req.op === CacheOp.Fill.U - val mask = Mux(req.op === CacheOp.Fill.U, Fill(p.lineBytes, 1.U(1.W)), req.mask) - readData(way) := data(way).readWrite( - set, - req.data.asTypeOf(Vec(p.lineBytes, UInt(8.W))), - mask.asBools, - io.request.fire && selected === way.U && (read || write), - write - ).asUInt - } - when(pendingRead) { - responses.io.enq.bits.data := readData(pendingWay) - } - - when(io.request.fire) { - assert(req.op <= CacheOp.Invalidate.U, "Unknown Cache operation") - assert(req.addr(p.offsetBits - 1, 0) === 0.U, "Cache operations require a line-aligned address") - assert(lookup || req.way < p.ways.U, "Cache way exceeds associativity") - assert(PopCount(hits) <= 1.U, "Duplicate Cache tag") - pendingRead := read - pendingWay := selected - result.id := req.id - result.hit := lookup && hits.orR - result.available := available - result.way := Mux(available, selected, 0.U) - result.entryValid := entryValid - result.addr := Mux(entryValid, Cat(tags(selected)(set), set, 0.U(p.offsetBits.W)), 0.U) - result.data := 0.U - result.metadata := Mux(entryValid, metadata(selected)(set), 0.U) - - when(req.op === CacheOp.Fill.U) { - assert(!hits.orR || hits(req.way), "Cache fill would create a duplicate tag") - valid(req.way)(set) := true.B - tags(req.way)(set) := tag - metadata(req.way)(set) := req.metadata - nextWay(set) := (if (p.ways == 1) 0.U else (req.way + 1.U)(p.wayBits - 1, 0)) - result.entryValid := true.B - result.addr := req.addr - result.metadata := req.metadata - } - when(req.op === CacheOp.Write.U) { - assert(entryValid && hits(req.way), "Cache write requires the matching resident line") - metadata(req.way)(set) := req.metadata - result.metadata := req.metadata - } - when(req.op === CacheOp.Invalidate.U) { - valid(req.way)(set) := false.B - result.entryValid := false.B - result.addr := 0.U - result.metadata := 0.U - } - } -} diff --git a/arch/src/main/scala/framework/mem-core/cache/src/main/scala/Emit.scala b/arch/src/main/scala/framework/mem-core/cache/src/main/scala/Emit.scala deleted file mode 100644 index b0139da49..000000000 --- a/arch/src/main/scala/framework/mem-core/cache/src/main/scala/Emit.scala +++ /dev/null @@ -1,29 +0,0 @@ -package memcore.memory.cache - -import memcore.memory.cache.configs.CacheParams - -object Emit extends App { - val p = CacheParams.load("configs/default.toml") - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Cache(p), - firtoolOpts = args, - args = Array("--target-dir", "build/Cache", "--split-verilog") - ) - - val constants = Seq( - "ADDR_BITS" -> p.addressBits, - "LINE_BYTES" -> p.lineBytes, - "LINE_BITS" -> p.lineBits, - "SETS" -> p.sets, - "WAYS" -> p.ways, - "WAY_BITS" -> p.wayBits, - "ID_BITS" -> p.idBits, - "META_BITS" -> p.metadataBits, - "RESPONSE_DEPTH" -> p.responseDepth - ).map { case (name, value) => s"`define CACHE_$name $value" }.mkString("\n") - - java.nio.file.Files.writeString( - java.nio.file.Paths.get("build/cache_config.svh"), - s"`ifndef CACHE_CONFIG_SVH\n`define CACHE_CONFIG_SVH\n$constants\n`endif\n" - ) -} diff --git a/arch/src/main/scala/framework/mem-core/cache/src/main/scala/Interface.scala b/arch/src/main/scala/framework/mem-core/cache/src/main/scala/Interface.scala deleted file mode 100644 index 78abd4b65..000000000 --- a/arch/src/main/scala/framework/mem-core/cache/src/main/scala/Interface.scala +++ /dev/null @@ -1,42 +0,0 @@ -package memcore.memory.cache - -import chisel3._ -import chisel3.util._ -import memcore.memory.cache.configs.CacheParams - -object CacheOp { - val Lookup = 0 - val Read = 1 - val Write = 2 - val Fill = 3 - val Invalidate = 4 -} - -class CacheRequest(p: CacheParams) extends Bundle { - val id = UInt(p.idBits.W) - val op = UInt(3.W) - // Read and Invalidate address a physical set/way; all operations are line aligned. - val addr = UInt(p.addressBits.W) - val way = UInt(p.wayBits.W) - val data = UInt(p.lineBits.W) - val mask = UInt(p.lineBytes.W) - val metadata = UInt(p.metadataBits.W) - // Eligibility constrains miss replacement candidates, not tag hits. - val eligible = UInt(p.ways.W) -} - -class CacheResponse(p: CacheParams) extends Bundle { - val id = UInt(p.idBits.W) - val hit = Bool() - val available = Bool() - val way = UInt(p.wayBits.W) - val entryValid = Bool() - val addr = UInt(p.addressBits.W) - val data = UInt(p.lineBits.W) - val metadata = UInt(p.metadataBits.W) -} - -class CacheIO(p: CacheParams) extends Bundle { - val request = Flipped(Decoupled(new CacheRequest(p))) - val response = Decoupled(new CacheResponse(p)) -} diff --git a/arch/src/main/scala/framework/mem-core/chi/build.sc b/arch/src/main/scala/framework/mem-core/chi/build.sc deleted file mode 100644 index 17404bc1b..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/build.sc +++ /dev/null @@ -1,20 +0,0 @@ -import mill._ -import mill.scalalib._ - -object queue extends SbtModule { - override def millSourcePath = os.pwd / os.up / "queue" - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") -} - -object chi extends SbtModule { - override def moduleDeps = Seq(queue) - override def millSourcePath = os.pwd - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") - override def mainClass = Some("memcore.bus.chi.Emit") -} diff --git a/arch/src/main/scala/framework/mem-core/chi/src/csrc/src/lib.rs b/arch/src/main/scala/framework/mem-core/chi/src/csrc/src/lib.rs deleted file mode 100644 index 8845fc066..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/csrc/src/lib.rs +++ /dev/null @@ -1,3 +0,0 @@ -mod rnf; -mod memory; -pub use memory::*; diff --git a/arch/src/main/scala/framework/mem-core/chi/src/csrc/src/rnf.rs b/arch/src/main/scala/framework/mem-core/chi/src/csrc/src/rnf.rs deleted file mode 100644 index 9d99db67e..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/csrc/src/rnf.rs +++ /dev/null @@ -1,110 +0,0 @@ -use std::{collections::BTreeMap, ffi::c_void}; - -struct Memory { - limit: u64, - lines: BTreeMap, -} - -#[no_mangle] -pub extern "C" fn rnf_ref_create(address_bits: u32) -> *mut c_void { - assert!((44..=52).contains(&address_bits)); - Box::into_raw(Box::new(Memory { - limit: 1u64 << address_bits, - lines: BTreeMap::new(), - })).cast() -} - -#[no_mangle] -pub unsafe extern "C" fn rnf_ref_destroy(model: *mut c_void) { - drop(Box::from_raw(model.cast::())); -} - -#[no_mangle] -pub unsafe extern "C" fn rnf_ref_write( - model: *mut c_void, addr: u64, data: *const u32, mask: *const u32, -) -> u8 { - let memory = &mut *model.cast::(); - assert_eq!(addr % 64, 0); - if addr >= memory.limit { return 1; } - let data = std::slice::from_raw_parts(data, 16); - let mask = std::slice::from_raw_parts(mask, 2); - let full = mask.iter().all(|&word| word == u32::MAX); - assert!(memory.lines.contains_key(&addr) || full, "Partial write before initialization"); - let line = memory.lines.entry(addr).or_insert([0; 64]); - for b in 0..64 { - if mask[b / 32] & (1 << (b % 32)) != 0 { - line[b] = (data[b / 4] >> (8 * (b % 4))) as u8; - } - } - 0 -} - -#[no_mangle] -pub unsafe extern "C" fn rnf_ref_read( - model: *mut c_void, addr: u64, data: *mut u32, -) -> u8 { - let memory = &*model.cast::(); - assert_eq!(addr % 64, 0); - let output = std::slice::from_raw_parts_mut(data, 16); - output.fill(0); - if addr >= memory.limit { return 1; } - let line = memory.lines.get(&addr).expect("Read of uninitialized line"); - for (word, bytes) in output.iter_mut().zip(line.chunks_exact(4)) { - *word = u32::from_le_bytes(bytes.try_into().unwrap()); - } - 0 -} - -#[no_mangle] -pub unsafe extern "C" fn rnf_ref_access( - model: *mut c_void, addr: u64, write: u8, operand: u64, - atomic: u32, word: u8, mask: u8, sc_success: u8, failed: u8, -) -> u64 { - let memory = &mut *model.cast::(); - assert!(atomic <= 12); - assert!(word == 0 || (1..=11).contains(&atomic)); - assert_eq!(addr % if word != 0 { 4 } else { 8 }, 0); - assert!(atomic == 0 || (write == 0 && mask == 255)); - if failed != 0 { return 0; } - let line = memory.lines.get_mut(&(addr & !63)).expect("CPU access to uninitialized line"); - let offset = (addr & 63) as usize; - let (old, update) = if word != 0 { - let lhs = u32::from_le_bytes(line[offset..offset + 4].try_into().unwrap()); - let rhs = operand as u32; - let result = match atomic { - 2 => lhs.wrapping_add(rhs), - 3 => lhs ^ rhs, - 4 => lhs & rhs, - 5 => lhs | rhs, - 6 => if (lhs as i32) < (rhs as i32) { lhs } else { rhs }, - 7 => if (lhs as i32) > (rhs as i32) { lhs } else { rhs }, - 8 => lhs.min(rhs), - 9 => lhs.max(rhs), - _ => rhs, - }; - ((lhs as i32 as i64) as u64, result as u64) - } else { - let lhs = u64::from_le_bytes(line[offset..offset + 8].try_into().unwrap()); - let rhs = operand; - let result = match atomic { - 2 => lhs.wrapping_add(rhs), - 3 => lhs ^ rhs, - 4 => lhs & rhs, - 5 => lhs | rhs, - 6 => if (lhs as i64) < (rhs as i64) { lhs } else { rhs }, - 7 => if (lhs as i64) > (rhs as i64) { lhs } else { rhs }, - 8 => lhs.min(rhs), - 9 => lhs.max(rhs), - _ => rhs, - }; - (lhs, result) - }; - if write != 0 || (1..=9).contains(&atomic) || (atomic == 11 && sc_success != 0) { - for b in 0..if word != 0 { 4 } else { 8 } { - if word != 0 || mask & (1 << b) != 0 { - line[offset + b] = (update >> (8 * b)) as u8; - } - } - } - match atomic { 11 => u64::from(sc_success == 0), 12 => 0, _ => old } -} diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/chi_profile.svh b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/chi_profile.svh deleted file mode 100644 index fbb04f396..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/chi_profile.svh +++ /dev/null @@ -1,12 +0,0 @@ -`ifndef CHI_PROFILE_SVH -`define CHI_PROFILE_SVH -`ifndef CHI_DATA_BITS -`define CHI_DATA_BITS 256 -`endif -`define CHI_BE_BITS (`CHI_DATA_BITS / 8) -`define CHI_TAG_BITS (`CHI_DATA_BITS / 32) -`define CHI_TAG_UPDATE_BITS (`CHI_DATA_BITS / 128) -`define CHI_DAT_BITS (91 + `CHI_DATA_BITS + `CHI_BE_BITS + `CHI_TAG_BITS + `CHI_TAG_UPDATE_BITS) -`define CHI_BEATS (512 / `CHI_DATA_BITS) -`define CHI_DATA_ID_STEP (`CHI_DATA_BITS / 128) -`endif diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/credit_interface.sv b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/credit_interface.sv deleted file mode 100644 index d9ff4688a..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/credit_interface.sv +++ /dev/null @@ -1,6 +0,0 @@ -interface chi_credit_if ( - input logic clock -); - logic reset, active, pend, valid, lcrdv; - logic [31:0] flit; -endinterface diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/credit_package.sv b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/credit_package.sv deleted file mode 100644 index a1b42639e..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/credit_package.sv +++ /dev/null @@ -1,43 +0,0 @@ -package chi_credit_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual chi_credit_if vif; - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual chi_credit_if)::get(this, "", "vif", vif)) - `uvm_fatal("VIF", "Credit interface missing") - endfunction - task execute(); - vif.reset = 1; - vif.active = 0; -`ifdef CHI_TX_OVERFLOW - vif.lcrdv = 0; -`else - vif.pend = 0; - vif.valid = 0; - vif.flit = 0; -`endif - repeat (4) @(negedge vif.clock); - vif.reset = 0; - vif.active = 1; -`ifdef CHI_TX_OVERFLOW - // Peer returns five credits to a four-credit transmitter with no traffic. - vif.lcrdv = 1; - repeat (8) @(negedge vif.clock); -`else - // Peer uses a fifth flit after exhausting four credits; receiver cannot drain. - vif.pend = 1; - repeat (8) @(negedge vif.clock); - vif.valid = 1; - repeat (6) @(negedge vif.clock); -`endif - `uvm_fatal("NEGATIVE", "Credit violation did not terminate RTL") - endtask - endclass -endpackage diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/credit_tb.sv b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/credit_tb.sv deleted file mode 100644 index 2c4a48591..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/credit_tb.sv +++ /dev/null @@ -1,39 +0,0 @@ -module `CHI_CREDIT_TB; - import uvm_pkg::*; - import chi_credit_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - chi_credit_if source_if (clock); -`ifdef CHI_TX_OVERFLOW - Tx dut ( - .clock(clock), - .reset(source_if.reset), - .io_active(source_if.active), - .io_in_valid(1'b0), - .io_in_ready(), - .io_in_bits(32'b0), - .io_credits(), - .io_link_flitpend(), - .io_link_flitv(), - .io_link_flit(), - .io_link_lcrdv(source_if.lcrdv) - ); -`else - Rx dut ( - .clock(clock), - .reset(source_if.reset), - .io_active(source_if.active), - .io_link_flitpend(source_if.pend), - .io_link_flitv(source_if.valid), - .io_link_flit(source_if.flit), - .io_link_lcrdv(source_if.lcrdv), - .io_out_valid(), - .io_out_ready(1'b0), - .io_out_bits() - ); -`endif - initial begin - uvm_config_db#(virtual chi_credit_if)::set(null, "uvm_test_top", "vif", source_if); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/link.el b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/link.el deleted file mode 100644 index 28e83891b..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/link.el +++ /dev/null @@ -1,45 +0,0 @@ -// Format Version: 2 -// Reviewed against the exact exported object signatures and the declared depth4 always-on profile. - -CHECKSUM: "1668538250 4134186348" -INSTANCE: link_tb.dut -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI TX credit overflow" -Block 2 "239387013" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI TX credit overflow" -Block 3 "4289838947" "$error(\"Assertion failed: CHI TX credit overflow\n at Tx.scala:43 when(returned && !send)(assert(credits < maxCredits.U, \\"CHI TX credit overflow\\"))\n\");" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI TX credit overflow" -Block 5 "239387013" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI TX credit overflow" -Block 6 "1750606003" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI TX credit underflow" -Block 10 "1503763571" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI TX credit underflow" -Block 11 "556320376" "$error(\"Assertion failed: CHI TX credit underflow\n at Tx.scala:44 when(send)(assert(credits =/= 0.U, \\"CHI TX credit underflow\\"))\n\");" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI TX credit underflow" -Block 13 "1503763571" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI TX credit underflow" -Block 14 "2166023300" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI TX requires coordinated reset to stop" -Block 18 "2653430091" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI TX requires coordinated reset to stop" -Block 19 "597017625" "$error(\"Assertion failed: CHI TX requires coordinated reset to stop\n at Tx.scala:45 when(wasActive)(assert(io.active, \\"CHI TX requires coordinated reset to stop\\"))\n\");" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI TX requires coordinated reset to stop" -Block 21 "2653430091" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI TX requires coordinated reset to stop" -Block 22 "2212642507" "$fatal;" - - -CHECKSUM: "1668538250 4069976122" -INSTANCE: link_tb.dut -ANNOTATION: "Tx32/depth4 peer: credit=4 means all four credits are available; another returned credit cannot simultaneously be pending. This vector would violate peer credit conservation." -Condition 1 "284300870" "(returned & ((~send)) & ((~reset)) & credits[2]) 1 -1" (2 "1011") -ANNOTATION: "Tx32/depth4 peer: no returned credit can coexist with all four credits available, including the coordinated reset boundary." -Condition 1 "284300870" "(returned & ((~send)) & ((~reset)) & credits[2]) 1 -1" (3 "1101") -ANNOTATION: "Assertion-failure vector: overgrant violates the declared four-credit peer contract. Independently checked by tx_bad_credit." -Condition 1 "284300870" "(returned & ((~send)) & ((~reset)) & credits[2]) 1 -1" (5 "1111") -ANNOTATION: "Structural invariant: send requires ready, and ready requires nonzero credits; send with zero credits is impossible even on reset." -Condition 2 "3481575346" "(send & ((~reset)) & ((~(|credits)))) 1 -1" (2 "101") -ANNOTATION: "Structural invariant: send implies credits != 0 through io.in.ready." -Condition 2 "3481575346" "(send & ((~reset)) & ((~(|credits)))) 1 -1" (4 "111") -ANNOTATION: "Always-on profile forbids deactivation without coordinated reset; this is the assertion-failure vector, not normal shutdown support." -Condition 3 "3706255149" "(wasActive & ((~reset)) & ((~io_active))) 1 -1" (4 "111") diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/link.f b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/link.f deleted file mode 100644 index e7175b587..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/link.f +++ /dev/null @@ -1,8 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/link_interface.sv -@RESOURCES@/link_package.sv --F @RTL@/Tx/filelist.f --F @RTL@/Rx/filelist.f -@RESOURCES@/link_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/link_interface.sv b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/link_interface.sv deleted file mode 100644 index 6fbdfe9e8..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/link_interface.sv +++ /dev/null @@ -1,9 +0,0 @@ -interface chi_link_if ( - input logic clock -); - logic reset, active, valid, ready; - logic [31:0] data; - logic out_valid, out_ready; - logic [31:0] out_data; - logic [ 2:0] credits; -endinterface diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/link_package.sv b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/link_package.sv deleted file mode 100644 index 7b2a0c97f..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/link_package.sv +++ /dev/null @@ -1,128 +0,0 @@ -package chi_link_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual chi_link_if vif; - bit [31:0] pending[$]; - int accepted = 0, checked = 0, cycle = 0; - bit stalled = 0; - bit [31:0] held; - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual chi_link_if)::get(this, "", "vif", vif)) - `uvm_fatal("VIF", "CHI credit link missing") - endfunction - task monitor(); - forever begin - @(posedge vif.clock); - if (vif.reset) begin - pending.delete(); - stalled = 0; - end else begin - cycle++; - if (vif.valid && vif.ready) begin - pending.push_back(vif.data); - accepted++; - end - if (stalled && (!vif.out_valid || vif.out_data !== held)) - `uvm_fatal("STABLE", "Output changed while stalled") - if (vif.out_valid && vif.out_ready) begin - bit [31:0] golden; - if (!pending.size()) `uvm_fatal("UNEXPECTED", "Output without request") - golden = pending.pop_front(); - if (vif.out_data !== golden) `uvm_fatal("DATA", "Credit link data/order mismatch") - checked++; - end - stalled = vif.out_valid && !vif.out_ready; - held = vif.out_data; - if (vif.credits > 4) `uvm_fatal("CREDIT", "Credit count exceeds depth") - end - end - endtask - task restart(); - @(negedge vif.clock); - vif.reset = 1; - vif.active = 0; - vif.valid = 0; - repeat (4) @(negedge vif.clock); - vif.reset = 0; - repeat (2) @(negedge vif.clock); - vif.active = 1; - repeat (8) @(negedge vif.clock); - endtask - task execute(); - vif.reset = 1; - vif.active = 0; - vif.valid = 0; - vif.data = 0; - vif.out_ready = 0; - fork - monitor(); - join_none - restart(); - for (int i = 0; i < 256; i++) begin - vif.valid = 1; - vif.data = (32'h9e3779b9 * (i + 1)) ^ ((i & 4) ? 32'hffffffff : 32'h0); - do begin - @(posedge vif.clock); - if (vif.ready) begin - @(negedge vif.clock); - break; - end - @(negedge vif.clock); - vif.out_ready = cycle % 7 < 3; - end while (1); - vif.out_ready = cycle % 7 < 3; - end - vif.valid = 0; - vif.out_ready = 1; - while (pending.size()) @(negedge vif.clock); - // Coordinated reset cancels buffered traffic and restores initial credits. - vif.out_ready = 0; - repeat (4) begin - vif.valid = 1; - vif.data = 32'hfedcba98; - do @(posedge vif.clock); while (!vif.ready); - @(negedge vif.clock); - end - vif.valid = 0; - repeat (8) @(negedge vif.clock); - restart(); - vif.out_ready = 1; - for (int i = 0; i < 32; i++) begin - vif.valid = 1; - vif.data = i; - do @(posedge vif.clock); while (!vif.ready); - @(negedge vif.clock); - end - vif.valid = 0; - while (pending.size()) @(negedge vif.clock); - repeat (8) @(negedge vif.clock); - // A coordinated reset can interrupt a physical flit before RX captures it. - vif.valid = 1; - vif.data = 32'h01234567; - do @(posedge vif.clock); while (!vif.ready); - @(negedge vif.clock); - vif.reset = 1; - vif.active = 0; - vif.valid = 0; - repeat (4) @(negedge vif.clock); - vif.reset = 0; - repeat (2) @(negedge vif.clock); - vif.active = 1; - repeat (8) @(negedge vif.clock); - if (checked != 288 || accepted != 293 || vif.credits != 4) - `uvm_fatal("COUNT", $sformatf( - "accepted=%0d checked=%0d credits=%0d", accepted, checked, vif.credits)) - `uvm_info( - "CHECKS", - "CHI credit link: 288 ordered outputs checked, 5 reset-cancelled (4 buffered, 1 in-flight)", - UVM_LOW) - endtask - endclass -endpackage diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/link_tb.sv b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/link_tb.sv deleted file mode 100644 index b1c59c0d8..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/link_tb.sv +++ /dev/null @@ -1,38 +0,0 @@ -module link_tb; - import uvm_pkg::*; - import chi_link_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - chi_link_if source_if (clock); - wire flitpend, flitv, lcrdv; - wire [31:0] flit; - Tx dut ( - .clock(clock), - .reset(source_if.reset), - .io_active(source_if.active), - .io_in_valid(source_if.valid), - .io_in_ready(source_if.ready), - .io_in_bits(source_if.data), - .io_link_flitpend(flitpend), - .io_link_flitv(flitv), - .io_link_flit(flit), - .io_link_lcrdv(lcrdv), - .io_credits(source_if.credits) - ); - Rx receiver ( - .clock(clock), - .reset(source_if.reset), - .io_active(source_if.active), - .io_link_flitpend(flitpend), - .io_link_flitv(flitv), - .io_link_flit(flit), - .io_link_lcrdv(lcrdv), - .io_out_valid(source_if.out_valid), - .io_out_ready(source_if.out_ready), - .io_out_bits(source_if.out_data) - ); - initial begin - uvm_config_db#(virtual chi_link_if)::set(null, "uvm_test_top", "vif", source_if); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry.el b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry.el deleted file mode 100644 index 1b7ca84fc..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry.el +++ /dev/null @@ -1,415 +0,0 @@ -// Format Version: 2 -// Exact reviewed objects; grouped scopes and proofs, with equivalent adjacent bit/transition ranges. - -CHECKSUM: "1565409055 1106117561" -INSTANCE: retry_tb.dut -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Unsupported CHI retry request opcode" -Block 2 "1744103448" "if (1)" -Block 3 "675298159" "$error(\"Assertion failed: Unsupported CHI retry request opcode\n at RequestRetry.scala:64 assert(coherent || noSnoop, \\"Unsupported CHI retry request opcode\\")\n\");" -Block 5 "1744103448" "if (1)" -Block 6 "3310318889" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Unsupported CHI retry request attributes" -Block 10 "3534321110" "if (1)" -Block 11 "3076932440" "$error(\"Assertion failed: Unsupported CHI retry request attributes\n at RequestRetry.scala:65 assert(\n\");" -Block 13 "3534321110" "if (1)" -Block 14 "882429221" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Invalid retry completion-ack profile" -Block 18 "3921772647" "if (1)" -Block 19 "3527252594" "$error(\"Assertion failed: Invalid retry completion-ack profile\n at RequestRetry.scala:71 assert(r.expCompAck === coherentRead.asUInt, \\"Invalid retry completion-ack profile\\")\n\");" -Block 21 "3921772647" "if (1)" -Block 22 "2211796926" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Invalid retry coherent request attributes" -Block 26 "568216364" "if (1)" -Block 27 "439395600" "$error(\"Assertion failed: Invalid retry coherent request attributes\n at RequestRetry.scala:73 assert(r.returnNid === 0.U && r.returnTxnId === 0.U && r.memAttr === 12.U && r.snpAttr === 1.U,\n\");" -Block 29 "568216364" "if (1)" -Block 30 "1967310581" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Invalid retry non-snoop request attributes" -Block 34 "2827715915" "if (1)" -Block 35 "4275880477" "$error(\"Assertion failed: Invalid retry non-snoop request attributes\n at RequestRetry.scala:76 assert(!r.memAttr(1) && r.snpAttr === 0.U, \\"Invalid retry non-snoop request attributes\\")\n\");" -Block 37 "2827715915" "if (1)" -Block 38 "246045839" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Invalid retry requester" -Block 42 "1709660459" "if (1)" -Block 43 "4204837901" "$error(\"Assertion failed: Invalid retry requester\n at RequestRetry.scala:78 assert(io.reqIn.bits.srcId === nodeId.U, \\"Invalid retry requester\\")\n\");" -Block 45 "1709660459" "if (1)" -Block 46 "2514228879" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI retry requester reused a live TxnID" -Block 50 "1112493139" "if (1)" -Block 51 "2040280706" "$error(\"Assertion failed: CHI retry requester reused a live TxnID\n at RequestRetry.scala:79 assert(!duplicate, \\"CHI retry requester reused a live TxnID\\")\n\");" -Block 53 "1112493139" "if (1)" -Block 54 "1089463653" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Initial CHI request must allow retry without a protocol credit" -Block 58 "130518244" "if (1)" -Block 59 "4023718724" "$error(\"Assertion failed: Initial CHI request must allow retry without a protocol credit\n at RequestRetry.scala:80 assert(\n\");" -Block 61 "130518244" "if (1)" -Block 62 "422011914" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Response to wrong retry requester" -Block 66 "297091616" "if (1)" -Block 67 "3803880083" "$error(\"Assertion failed: Response to wrong retry requester\n at RequestRetry.scala:131 assert(r.tgtId === nodeId.U, \\"Response to wrong retry requester\\")\n\");" -Block 69 "297091616" "if (1)" -Block 70 "3411174966" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Unexpected or repeated RetryAck" -Block 74 "2784317146" "if (1)" -Block 75 "3648346299" "$error(\"Assertion failed: Unexpected or repeated RetryAck\n at RequestRetry.scala:135 assert(matches.asUInt.orR && state(i) === firstAttempt && r.respErr === 0.U, \\"Unexpected or repeated RetryAck\\")\n\");" -Block 77 "2784317146" "if (1)" -Block 78 "74728005" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Excess or errored protocol credit" -Block 82 "2447225933" "if (1)" -Block 83 "3962228737" "$error(\"Assertion failed: Excess or errored protocol credit\n at RequestRetry.scala:142 assert(space.asUInt.orR && r.respErr === 0.U, \\"Excess or errored protocol credit\\")\n\");" -Block 85 "2447225933" "if (1)" -Block 86 "891196801" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Response accepted before request reissue" -Block 90 "3397368675" "if (1)" -Block 91 "2039620193" "$error(\"Assertion failed: Response accepted before request reissue\n at RequestRetry.scala:124 assert(state(i) =/= retryWait, \\"Response accepted before request reissue\\")\n\");" -Block 93 "3397368675" "if (1)" -Block 94 "1797854015" "$fatal;" -Block 98 "397495833" "if (1)" -Block 99 "587374098" "$error(\"Assertion failed: Response accepted before request reissue\n at RequestRetry.scala:124 assert(state(i) =/= retryWait, \\"Response accepted before request reissue\\")\n\");" -Block 101 "397495833" "if (1)" -Block 102 "834252620" "$fatal;" - -CHECKSUM: "1565409055 2680414210" -INSTANCE: retry_tb.dut -ANNOTATION: "RequestRetry asserted legal request contract: NodeID1, 64-byte aligned access, supported opcodes below64, normal memory (Device=0), PAS/Order/exclusive/tag/multi-request/stash=0, LPID lower5 only. All higher address and arbitrary TxnID/routing bits stay measured." -Toggle io_reqOut_bits_srcId [6:0] "net io_reqOut_bits_srcId[6:0]" -Toggle io_reqOut_bits_stashNidValidEndian "net io_reqOut_bits_stashNidValidEndian" -Toggle io_reqOut_bits_opcode [6] "net io_reqOut_bits_opcode[6:0]" -Toggle io_reqOut_bits_multiReq "net io_reqOut_bits_multiReq" -Toggle io_reqOut_bits_size [5:0] "net io_reqOut_bits_size[5:0]" -Toggle io_reqOut_bits_addr [5:0] "net io_reqOut_bits_addr[43:0]" -Toggle io_reqOut_bits_pas [2:0] "net io_reqOut_bits_pas[2:0]" -Toggle io_reqOut_bits_order [1:0] "net io_reqOut_bits_order[1:0]" -Toggle io_reqOut_bits_memAttr [1] "net io_reqOut_bits_memAttr[3:0]" -Toggle io_reqOut_bits_lpid [7:5] "net io_reqOut_bits_lpid[7:0]" -Toggle io_reqOut_bits_exclSnoopMe "net io_reqOut_bits_exclSnoopMe" -Toggle io_reqOut_bits_tagOp [1:0] "net io_reqOut_bits_tagOp[1:0]" -Toggle _arb_io_out_bits_srcId [6:0] "net _arb_io_out_bits_srcId[6:0]" -Toggle _arb_io_out_bits_stashNidValidEndian "net _arb_io_out_bits_stashNidValidEndian" -Toggle _arb_io_out_bits_opcode [6] "net _arb_io_out_bits_opcode[6:0]" -Toggle _arb_io_out_bits_multiReq "net _arb_io_out_bits_multiReq" -Toggle _arb_io_out_bits_size [5:0] "net _arb_io_out_bits_size[5:0]" -Toggle _arb_io_out_bits_addr [5:0] "net _arb_io_out_bits_addr[43:0]" -Toggle _arb_io_out_bits_pas [2:0] "net _arb_io_out_bits_pas[2:0]" -Toggle _arb_io_out_bits_order [1:0] "net _arb_io_out_bits_order[1:0]" -Toggle _arb_io_out_bits_memAttr [1] "net _arb_io_out_bits_memAttr[3:0]" -Toggle _arb_io_out_bits_lpid [7:5] "net _arb_io_out_bits_lpid[7:0]" -Toggle _arb_io_out_bits_exclSnoopMe "net _arb_io_out_bits_exclSnoopMe" -Toggle _arb_io_out_bits_tagOp [1:0] "net _arb_io_out_bits_tagOp[1:0]" -Toggle requests_0_srcId [6:0] "reg requests_0_srcId[6:0]" -Toggle requests_0_stashNidValidEndian "reg requests_0_stashNidValidEndian" -Toggle requests_0_opcode [6] "reg requests_0_opcode[6:0]" -Toggle requests_0_multiReq "reg requests_0_multiReq" -Toggle requests_0_size [5:0] "reg requests_0_size[5:0]" -Toggle requests_0_addr [5:0] "reg requests_0_addr[43:0]" -Toggle requests_0_pas [2:0] "reg requests_0_pas[2:0]" -Toggle requests_0_order [1:0] "reg requests_0_order[1:0]" -Toggle requests_0_memAttr [1] "reg requests_0_memAttr[3:0]" -Toggle requests_0_lpid [7:5] "reg requests_0_lpid[7:0]" -Toggle requests_0_exclSnoopMe "reg requests_0_exclSnoopMe" -Toggle requests_0_tagOp [1:0] "reg requests_0_tagOp[1:0]" -Toggle requests_1_srcId [6:0] "reg requests_1_srcId[6:0]" -Toggle requests_1_stashNidValidEndian "reg requests_1_stashNidValidEndian" -Toggle requests_1_opcode [6] "reg requests_1_opcode[6:0]" -Toggle requests_1_multiReq "reg requests_1_multiReq" -Toggle requests_1_size [5:0] "reg requests_1_size[5:0]" -Toggle requests_1_addr [5:0] "reg requests_1_addr[43:0]" -Toggle requests_1_pas [2:0] "reg requests_1_pas[2:0]" -Toggle requests_1_order [1:0] "reg requests_1_order[1:0]" -Toggle requests_1_memAttr [1] "reg requests_1_memAttr[3:0]" -Toggle requests_1_lpid [7:5] "reg requests_1_lpid[7:0]" -Toggle requests_1_exclSnoopMe "reg requests_1_exclSnoopMe" -Toggle requests_1_tagOp [1:0] "reg requests_1_tagOp[1:0]" -Toggle requests_2_srcId [6:0] "reg requests_2_srcId[6:0]" -Toggle requests_2_stashNidValidEndian "reg requests_2_stashNidValidEndian" -Toggle requests_2_opcode [6] "reg requests_2_opcode[6:0]" -Toggle requests_2_multiReq "reg requests_2_multiReq" -Toggle requests_2_size [5:0] "reg requests_2_size[5:0]" -Toggle requests_2_addr [5:0] "reg requests_2_addr[43:0]" -Toggle requests_2_pas [2:0] "reg requests_2_pas[2:0]" -Toggle requests_2_order [1:0] "reg requests_2_order[1:0]" -Toggle requests_2_memAttr [1] "reg requests_2_memAttr[3:0]" -Toggle requests_2_lpid [7:5] "reg requests_2_lpid[7:0]" -Toggle requests_2_exclSnoopMe "reg requests_2_exclSnoopMe" -Toggle requests_2_tagOp [1:0] "reg requests_2_tagOp[1:0]" -Toggle requests_3_srcId [6:0] "reg requests_3_srcId[6:0]" -Toggle requests_3_stashNidValidEndian "reg requests_3_stashNidValidEndian" -Toggle requests_3_opcode [6] "reg requests_3_opcode[6:0]" -Toggle requests_3_multiReq "reg requests_3_multiReq" -Toggle requests_3_size [5:0] "reg requests_3_size[5:0]" -Toggle requests_3_addr [5:0] "reg requests_3_addr[43:0]" -Toggle requests_3_pas [2:0] "reg requests_3_pas[2:0]" -Toggle requests_3_order [1:0] "reg requests_3_order[1:0]" -Toggle requests_3_memAttr [1] "reg requests_3_memAttr[3:0]" -Toggle requests_3_lpid [7:5] "reg requests_3_lpid[7:0]" -Toggle requests_3_exclSnoopMe "reg requests_3_exclSnoopMe" -Toggle requests_3_tagOp [1:0] "reg requests_3_tagOp[1:0]" - -CHECKSUM: "917436931 2269459522" -INSTANCE: retry_tb.dut.arb -ANNOTATION: "RequestRetry asserted legal request contract: NodeID1, 64-byte aligned access, supported opcodes below64, normal memory (Device=0), PAS/Order/exclusive/tag/multi-request/stash=0, LPID lower5 only. All higher address and arbitrary TxnID/routing bits stay measured." -Toggle io_in_1_bits_srcId [6:0] "net io_in_1_bits_srcId[6:0]" -Toggle io_in_1_bits_stashNidValidEndian "net io_in_1_bits_stashNidValidEndian" -Toggle io_in_1_bits_opcode [6] "net io_in_1_bits_opcode[6:0]" -Toggle io_in_1_bits_multiReq "net io_in_1_bits_multiReq" -Toggle io_in_1_bits_size [5:0] "net io_in_1_bits_size[5:0]" -Toggle io_in_1_bits_addr [5:0] "net io_in_1_bits_addr[43:0]" -Toggle io_in_1_bits_pas [2:0] "net io_in_1_bits_pas[2:0]" -Toggle io_in_1_bits_order [1:0] "net io_in_1_bits_order[1:0]" -Toggle io_in_1_bits_memAttr [1] "net io_in_1_bits_memAttr[3:0]" -Toggle io_in_1_bits_lpid [7:5] "net io_in_1_bits_lpid[7:0]" -Toggle io_in_1_bits_exclSnoopMe "net io_in_1_bits_exclSnoopMe" -Toggle io_in_1_bits_tagOp [1:0] "net io_in_1_bits_tagOp[1:0]" -Toggle io_in_2_bits_srcId [6:0] "net io_in_2_bits_srcId[6:0]" -Toggle io_in_2_bits_stashNidValidEndian "net io_in_2_bits_stashNidValidEndian" -Toggle io_in_2_bits_opcode [6] "net io_in_2_bits_opcode[6:0]" -Toggle io_in_2_bits_multiReq "net io_in_2_bits_multiReq" -Toggle io_in_2_bits_size [5:0] "net io_in_2_bits_size[5:0]" -Toggle io_in_2_bits_addr [5:0] "net io_in_2_bits_addr[43:0]" -Toggle io_in_2_bits_pas [2:0] "net io_in_2_bits_pas[2:0]" -Toggle io_in_2_bits_order [1:0] "net io_in_2_bits_order[1:0]" -Toggle io_in_2_bits_memAttr [1] "net io_in_2_bits_memAttr[3:0]" -Toggle io_in_2_bits_lpid [7:5] "net io_in_2_bits_lpid[7:0]" -Toggle io_in_2_bits_exclSnoopMe "net io_in_2_bits_exclSnoopMe" -Toggle io_in_2_bits_tagOp [1:0] "net io_in_2_bits_tagOp[1:0]" -Toggle io_in_3_bits_srcId [6:0] "net io_in_3_bits_srcId[6:0]" -Toggle io_in_3_bits_stashNidValidEndian "net io_in_3_bits_stashNidValidEndian" -Toggle io_in_3_bits_opcode [6] "net io_in_3_bits_opcode[6:0]" -Toggle io_in_3_bits_multiReq "net io_in_3_bits_multiReq" -Toggle io_in_3_bits_size [5:0] "net io_in_3_bits_size[5:0]" -Toggle io_in_3_bits_addr [5:0] "net io_in_3_bits_addr[43:0]" -Toggle io_in_3_bits_pas [2:0] "net io_in_3_bits_pas[2:0]" -Toggle io_in_3_bits_order [1:0] "net io_in_3_bits_order[1:0]" -Toggle io_in_3_bits_memAttr [1] "net io_in_3_bits_memAttr[3:0]" -Toggle io_in_3_bits_lpid [7:5] "net io_in_3_bits_lpid[7:0]" -Toggle io_in_3_bits_exclSnoopMe "net io_in_3_bits_exclSnoopMe" -Toggle io_in_3_bits_tagOp [1:0] "net io_in_3_bits_tagOp[1:0]" -Toggle io_in_4_bits_srcId [6:0] "net io_in_4_bits_srcId[6:0]" -Toggle io_in_4_bits_stashNidValidEndian "net io_in_4_bits_stashNidValidEndian" -Toggle io_in_4_bits_opcode [6] "net io_in_4_bits_opcode[6:0]" -Toggle io_in_4_bits_multiReq "net io_in_4_bits_multiReq" -Toggle io_in_4_bits_size [5:0] "net io_in_4_bits_size[5:0]" -Toggle io_in_4_bits_addr [5:0] "net io_in_4_bits_addr[43:0]" -Toggle io_in_4_bits_pas [2:0] "net io_in_4_bits_pas[2:0]" -Toggle io_in_4_bits_order [1:0] "net io_in_4_bits_order[1:0]" -Toggle io_in_4_bits_memAttr [1] "net io_in_4_bits_memAttr[3:0]" -Toggle io_in_4_bits_lpid [7:5] "net io_in_4_bits_lpid[7:0]" -Toggle io_in_4_bits_exclSnoopMe "net io_in_4_bits_exclSnoopMe" -Toggle io_in_4_bits_tagOp [1:0] "net io_in_4_bits_tagOp[1:0]" -Toggle io_out_bits_srcId [6:0] "net io_out_bits_srcId[6:0]" -Toggle io_out_bits_stashNidValidEndian "net io_out_bits_stashNidValidEndian" -Toggle io_out_bits_opcode [6] "net io_out_bits_opcode[6:0]" -Toggle io_out_bits_multiReq "net io_out_bits_multiReq" -Toggle io_out_bits_size [5:0] "net io_out_bits_size[5:0]" -Toggle io_out_bits_addr [5:0] "net io_out_bits_addr[43:0]" -Toggle io_out_bits_pas [2:0] "net io_out_bits_pas[2:0]" -Toggle io_out_bits_order [1:0] "net io_out_bits_order[1:0]" -Toggle io_out_bits_memAttr [1] "net io_out_bits_memAttr[3:0]" -Toggle io_out_bits_lpid [7:5] "net io_out_bits_lpid[7:0]" -Toggle io_out_bits_exclSnoopMe "net io_out_bits_exclSnoopMe" -Toggle io_out_bits_tagOp [1:0] "net io_out_bits_tagOp[1:0]" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured srcId of replay port1. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_2 [1][6:0] "net [7:0][6:0]_GEN_2" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured srcId of replay port2. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_2 [2][6:0] "net [7:0][6:0]_GEN_2" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured srcId of replay port3. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_2 [3][6:0] "net [7:0][6:0]_GEN_2" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured srcId of replay port4. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_2 [4][6:0] "net [7:0][6:0]_GEN_2" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured stashNidValidEndian of replay port1. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_5 [1] "net _GEN_5[7:0]" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured stashNidValidEndian of replay port2. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_5 [2] "net _GEN_5[7:0]" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured stashNidValidEndian of replay port3. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_5 [3] "net _GEN_5[7:0]" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured stashNidValidEndian of replay port4. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_5 [4] "net _GEN_5[7:0]" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured opcode of replay port1. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_7 [1][6] "net [7:0][6:0]_GEN_7" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured opcode of replay port2. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_7 [2][6] "net [7:0][6:0]_GEN_7" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured opcode of replay port3. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_7 [3][6] "net [7:0][6:0]_GEN_7" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured opcode of replay port4. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_7 [4][6] "net [7:0][6:0]_GEN_7" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured multiReq of replay port1. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_8 [1] "net _GEN_8[7:0]" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured multiReq of replay port2. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_8 [2] "net _GEN_8[7:0]" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured multiReq of replay port3. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_8 [3] "net _GEN_8[7:0]" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured multiReq of replay port4. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_8 [4] "net _GEN_8[7:0]" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured size of replay port1. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_9 [1][5:0] "net [7:0][5:0]_GEN_9" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured size of replay port2. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_9 [2][5:0] "net [7:0][5:0]_GEN_9" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured size of replay port3. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_9 [3][5:0] "net [7:0][5:0]_GEN_9" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured size of replay port4. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_9 [4][5:0] "net [7:0][5:0]_GEN_9" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured addr of replay port1. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_10 [1][5:0] "net [7:0][43:0]_GEN_10" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured addr of replay port2. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_10 [2][5:0] "net [7:0][43:0]_GEN_10" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured addr of replay port3. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_10 [3][5:0] "net [7:0][43:0]_GEN_10" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured addr of replay port4. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_10 [4][5:0] "net [7:0][43:0]_GEN_10" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured pas of replay port1. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_11 [1][2:0] "net [7:0][2:0]_GEN_11" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured pas of replay port2. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_11 [2][2:0] "net [7:0][2:0]_GEN_11" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured pas of replay port3. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_11 [3][2:0] "net [7:0][2:0]_GEN_11" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured pas of replay port4. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_11 [4][2:0] "net [7:0][2:0]_GEN_11" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is literal 1'h0. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_13 [4:1] "net _GEN_13[7:0]" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured order of replay port1. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_14 [1][1:0] "net [7:0][1:0]_GEN_14" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured order of replay port2. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_14 [2][1:0] "net [7:0][1:0]_GEN_14" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured order of replay port3. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_14 [3][1:0] "net [7:0][1:0]_GEN_14" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured order of replay port4. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_14 [4][1:0] "net [7:0][1:0]_GEN_14" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured memAttr of replay port1. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_16 [1][1] "net [7:0][3:0]_GEN_16" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured memAttr of replay port2. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_16 [2][1] "net [7:0][3:0]_GEN_16" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured memAttr of replay port3. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_16 [3][1] "net [7:0][3:0]_GEN_16" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured memAttr of replay port4. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_16 [4][1] "net [7:0][3:0]_GEN_16" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured lpid of replay port1. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_18 [1][7:5] "net [7:0][7:0]_GEN_18" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured lpid of replay port2. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_18 [2][7:5] "net [7:0][7:0]_GEN_18" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured lpid of replay port3. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_18 [3][7:5] "net [7:0][7:0]_GEN_18" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured lpid of replay port4. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_18 [4][7:5] "net [7:0][7:0]_GEN_18" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured exclSnoopMe of replay port1. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_19 [1] "net _GEN_19[7:0]" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured exclSnoopMe of replay port2. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_19 [2] "net _GEN_19[7:0]" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured exclSnoopMe of replay port3. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_19 [3] "net _GEN_19[7:0]" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured exclSnoopMe of replay port4. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_19 [4] "net _GEN_19[7:0]" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured tagOp of replay port1. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_21 [1][1:0] "net [7:0][1:0]_GEN_21" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured tagOp of replay port2. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_21 [2][1:0] "net [7:0][1:0]_GEN_21" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured tagOp of replay port3. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_21 [3][1:0] "net [7:0][1:0]_GEN_21" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured tagOp of replay port4. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_21 [4][1:0] "net [7:0][1:0]_GEN_21" - -CHECKSUM: "3478893755 3492225878" -INSTANCE: retry_tb.dut.output_0 -ANNOTATION: "RequestRetry asserted legal request contract: NodeID1, 64-byte aligned access, supported opcodes below64, normal memory (Device=0), PAS/Order/exclusive/tag/multi-request/stash=0, LPID lower5 only. All higher address and arbitrary TxnID/routing bits stay measured." -Toggle io_enq_bits_srcId [6:0] "net io_enq_bits_srcId[6:0]" -Toggle io_enq_bits_stashNidValidEndian "net io_enq_bits_stashNidValidEndian" -Toggle io_enq_bits_opcode [6] "net io_enq_bits_opcode[6:0]" -Toggle io_enq_bits_multiReq "net io_enq_bits_multiReq" -Toggle io_enq_bits_size [5:0] "net io_enq_bits_size[5:0]" -Toggle io_enq_bits_addr [5:0] "net io_enq_bits_addr[43:0]" -Toggle io_enq_bits_pas [2:0] "net io_enq_bits_pas[2:0]" -Toggle io_enq_bits_order [1:0] "net io_enq_bits_order[1:0]" -Toggle io_enq_bits_memAttr [1] "net io_enq_bits_memAttr[3:0]" -Toggle io_enq_bits_lpid [7:5] "net io_enq_bits_lpid[7:0]" -Toggle io_enq_bits_exclSnoopMe "net io_enq_bits_exclSnoopMe" -Toggle io_enq_bits_tagOp [1:0] "net io_enq_bits_tagOp[1:0]" -Toggle io_deq_bits_srcId [6:0] "net io_deq_bits_srcId[6:0]" -Toggle io_deq_bits_stashNidValidEndian "net io_deq_bits_stashNidValidEndian" -Toggle io_deq_bits_opcode [6] "net io_deq_bits_opcode[6:0]" -Toggle io_deq_bits_multiReq "net io_deq_bits_multiReq" -Toggle io_deq_bits_size [5:0] "net io_deq_bits_size[5:0]" -Toggle io_deq_bits_addr [5:0] "net io_deq_bits_addr[43:0]" -Toggle io_deq_bits_pas [2:0] "net io_deq_bits_pas[2:0]" -Toggle io_deq_bits_order [1:0] "net io_deq_bits_order[1:0]" -Toggle io_deq_bits_memAttr [1] "net io_deq_bits_memAttr[3:0]" -Toggle io_deq_bits_lpid [7:5] "net io_deq_bits_lpid[7:0]" -Toggle io_deq_bits_exclSnoopMe "net io_deq_bits_exclSnoopMe" -Toggle io_deq_bits_tagOp [1:0] "net io_deq_bits_tagOp[1:0]" -ANNOTATION: "Output Queue stores the asserted legal RequestRetry request profile. Exact packed positions are read from generated per-field assignments, preserving all full-width address/routing/TxnID and variable attribute bits." -Toggle _ram_ext_R0_data [17:11] "net _ram_ext_R0_data[136:0]" -Toggle _ram_ext_R0_data [37] "net _ram_ext_R0_data[136:0]" -Toggle _ram_ext_R0_data [69:56] "net _ram_ext_R0_data[136:0]" -Toggle _ram_ext_R0_data [110:108] "net _ram_ext_R0_data[136:0]" -Toggle _ram_ext_R0_data [114:113] "net _ram_ext_R0_data[136:0]" -Toggle _ram_ext_R0_data [120] "net _ram_ext_R0_data[136:0]" -Toggle _ram_ext_R0_data [132:129] "net _ram_ext_R0_data[136:0]" -Toggle _ram_ext_R0_data [135:134] "net _ram_ext_R0_data[136:0]" - -CHECKSUM: "1565409055 2845470659" -INSTANCE: retry_tb.dut -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Invalid retry completion-ack profile. Functional gates and assertion remain enabled." -Condition 29 "4177145288" "(_GEN_25 & (io_reqIn_bits_expCompAck != coherentRead)) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Invalid retry non-snoop request attributes. Functional gates and assertion remain enabled." -Condition 37 "2137277555" "(io_reqIn_valid & ((~coherent)) & ((~reset)) & ( ~ (((~io_reqIn_bits_memAttr[1])) & ((~io_reqIn_bits_snpAttr))) )) 1 -1" (5 "1111") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Invalid retry requester. Functional gates and assertion remain enabled." -Condition 40 "745923699" "(_GEN_25 & (io_reqIn_bits_srcId != 7'b1)) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI retry requester reused a live TxnID. Functional gates and assertion remain enabled." -Condition 42 "3108172069" "(_GEN_25 & ((|_duplicate_T_12))) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Initial CHI request must allow retry without a protocol credit. Functional gates and assertion remain enabled." -Condition 43 "108006917" "(_GEN_25 & ( ~ (io_reqIn_bits_allowRetry & (io_reqIn_bits_pCrdType == 4'b0)) )) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Response to wrong retry requester. Functional gates and assertion remain enabled." -Condition 47 "1101780613" "(_GEN_21 & ((~reset)) & (io_rspIn_bits_tgtId != 7'b1)) 1 -1" (4 "111") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Response accepted before request reissue. Functional gates and assertion remain enabled." -Condition 56 "3214681341" "(_GEN_30 & ((~grant)) & _GEN_22 & ((|_GEN_23)) & ((~reset)) & (_GEN_28[i_1] == 2'h2)) 1 -1" (7 "111111") -Condition 58 "4231516992" "(io_acceptedData_valid & ((|_GEN_24)) & ((~reset)) & (_GEN_28[i_2] == 2'h2)) 1 -1" (5 "1111") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Unsupported CHI retry request opcode. Functional gates and assertion remain enabled." -Condition 14 "2551668066" "(_GEN_25 & ( ~ (coherent | (io_reqIn_bits_opcode == 7'h04) | (io_reqIn_bits_opcode == 7'h1d) | (io_reqIn_bits_opcode == 7'h1c)) )) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Unsupported CHI retry request attributes. Functional gates and assertion remain enabled." -Condition 20 "1109361100" "(_GEN_25 & ( ~ (((~io_reqIn_bits_multiReq)) & (io_reqIn_bits_order == 2'b0) & (io_reqIn_bits_size == 6'h06) & (io_reqIn_bits_addr[5:0] == 6'b0) & (io_reqIn_bits_pas == 3'b0) & ((~io_reqIn_bits_exclSnoopMe)) & (io_reqIn_bits_tagOp == 2'b0) & ((~io_reqIn_bits_stashNidValidEndian)) & (io_reqIn_bits_lpid[7:5] == 3'b0)) )) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Invalid retry coherent request attributes. Functional gates and assertion remain enabled." -Condition 31 "517468456" "(io_reqIn_valid & coherent & ((~reset)) & ( ~ ((io_reqIn_bits_returnNid == 7'b0) & (io_reqIn_bits_returnTxnId == 12'b0) & (io_reqIn_bits_memAttr == 4'hc) & io_reqIn_bits_snpAttr) )) 1 -1" (5 "1111") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Unexpected or repeated RetryAck. Functional gates and assertion remain enabled." -Condition 49 "3899675501" "(_GEN_21 & retryAck & ((~reset)) & ( ~ (((|{((|state_3) & (requests_3_txnId == io_rspIn_bits_txnId)), matches_4_2, matches_4_1, matches_4_0})) & (_GEN_28[i] == 2'b1) & _GEN_29) )) 1 -1" (5 "1111") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Excess or errored protocol credit. Functional gates and assertion remain enabled." -Condition 53 "873833217" "(_GEN_30 & grant & ((~reset)) & ( ~ (((|{(((~creditValid_3) | (_GEN_17 & (&credit_3))) | (_GEN_13 ? (((&credit_2) | _GEN_27) | _GEN_26) : (_GEN_27 | _GEN_26))), space_2, space_1, space_0})) & _GEN_29) )) 1 -1" (5 "1111") -ANNOTATION: "RetryAck decodes RSP opcode3, while each selected credit allocation term decodes PCrdGrant opcode7. One opcode cannot be both; exact vector would require both decodes." -Condition 82 "73012043" "(((~_GEN_21)) | retryAck | ((~_GEN_39))) 1 -1" (3 "010") -Condition 83 "379528980" "(((~_GEN_21)) | retryAck | ((~_GEN_40))) 1 -1" (3 "010") -Condition 84 "2400437582" "(((~_GEN_21)) | retryAck | ((~_GEN_41))) 1 -1" (3 "010") -Condition 85 "3656520365" "(((~_GEN_21)) | retryAck | ((~_GEN_42))) 1 -1" (3 "010") -ANNOTATION: "This subexpression is evaluated only in the ternary branch where a later replay port fires. The second term consumes credit through an earlier replay port. RRArbiter grants exactly one output per cycle, so the two replay fires cannot coexist; exact second-only vector is unreachable in this branch." -Condition 97 "3162203369" "(_GEN_7 | _GEN_47) 1 -1" (2 "01") -Condition 102 "2336271217" "(_GEN_9 | _GEN_48) 1 -1" (2 "01") -Condition 107 "1864735692" "(_GEN_11 | _GEN_49) 1 -1" (2 "01") -Condition 112 "167182624" "(((&credit_1)) | _GEN_50) 1 -1" (2 "01") -Condition 141 "1112285567" "(_GEN_18 | _GEN_55) 1 -1" (2 "01") -Condition 148 "3255526067" "(_GEN_19 | _GEN_56) 1 -1" (2 "01") -Condition 155 "2170964985" "(_GEN_20 | _GEN_57) 1 -1" (2 "01") -Condition 162 "2904700612" "(((&credit_3)) | _GEN_58) 1 -1" (2 "01") -ANNOTATION: "Four-entry PriorityEncoder returns fallback index3 when all live-TxnID matches are0. No-match with selected index0,1 or2 is structurally impossible, independent of payload or initialization." -Condition 114 "91711063" "(io_acceptedData_valid & ((|_GEN_24)) & (i_2 == 2'b0)) 1 -1" (2 "101") -Condition 118 "3535129296" "(_GEN_22 & ((|_GEN_23)) & (i_1 == 2'b0)) 1 -1" (2 "101") -Condition 120 "1021694279" "(io_acceptedData_valid & ((|_GEN_24)) & (i_2 == 2'b1)) 1 -1" (2 "101") -Condition 124 "3944017961" "(_GEN_22 & ((|_GEN_23)) & (i_1 == 2'b1)) 1 -1" (2 "101") -Condition 126 "3486384718" "(io_acceptedData_valid & ((|_GEN_24)) & (i_2 == 2'h2)) 1 -1" (2 "101") -Condition 130 "1247087977" "(_GEN_22 & ((|_GEN_23)) & (i_1 == 2'h2)) 1 -1" (2 "101") -ANNOTATION: "Acceptance opcode decode is DBIDResp6/Comp4/CompDBIDResp5; grant is PCrdGrant7. The exact vector requires grant and acceptance simultaneously, impossible for one RSP opcode." -Condition 116 "1348440713" "(grant | ( ~ (_GEN_22 & ((|_GEN_23)) & (i_1 == 2'b0)) )) 1 -1" (3 "10") -Condition 122 "559201839" "(grant | ( ~ (_GEN_22 & ((|_GEN_23)) & (i_1 == 2'b1)) )) 1 -1" (3 "10") -Condition 128 "2154737347" "(grant | ( ~ (_GEN_22 & ((|_GEN_23)) & (i_1 == 2'h2)) )) 1 -1" (3 "10") -Condition 133 "1731676477" "(grant | ( ~ (_GEN_22 & ((|_GEN_23)) & ((&i_1))) )) 1 -1" (3 "10") -ANNOTATION: "Selected credit allocation implies PCrdGrant opcode7, mutually exclusive with RetryAck opcode3. Exact vector requires both opcodes." -Condition 137 "3997230361" "(_GEN_21 & ((~retryAck)) & _GEN_39) 1 -1" (2 "101") -Condition 144 "312808281" "(_GEN_21 & ((~retryAck)) & _GEN_40) 1 -1" (2 "101") -Condition 151 "2618281630" "(_GEN_21 & ((~retryAck)) & _GEN_41) 1 -1" (2 "101") -Condition 158 "3436279767" "(_GEN_21 & ((~retryAck)) & _GEN_42) 1 -1" (2 "101") -ANNOTATION: "This ternary branch is selected by replay port3 fire. Its earlier-port consumption terms require another replay fire in the same cycle, forbidden by the single-output RRArbiter. Exact earlier-only vectors are unreachable; current-port selection remains measured." -Condition 272 "1429792695" "(_GEN_14 | _GEN_8 | _GEN_1) 1 -1" (2 "001") -Condition 272 "1429792695" "(_GEN_14 | _GEN_8 | _GEN_1) 1 -1" (3 "010") -Condition 277 "3581322673" "(_GEN_15 | _GEN_10 | _GEN_3) 1 -1" (2 "001") -Condition 277 "3581322673" "(_GEN_15 | _GEN_10 | _GEN_3) 1 -1" (3 "010") -Condition 282 "3852876295" "(_GEN_16 | _GEN_12 | _GEN_5) 1 -1" (2 "001") -Condition 282 "3852876295" "(_GEN_16 | _GEN_12 | _GEN_5) 1 -1" (3 "010") - -CHECKSUM: "917436931 1648997668" -INSTANCE: retry_tb.dut.arb -ANNOTATION: "Replay port4 becomes live only after its slot was accepted through initial port0. A port4 grant changes that slot to reissued and removes its valid immediately. Before port4 can become valid again, an initial port0 grant is mandatory and sets lastGrant0. Other intervening grants are ports1..3. Thus live port4 cannot coexist with lastGrant4; values5..7 cannot survive the required initial grant either." -Condition 6 "1359597920" "(io_in_4_valid & ((~ctrl_validMask_grantMask_lastGrant[2]))) 1 -1" (2 "10") diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry.f b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry.f deleted file mode 100644 index 86c8eb77b..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry.f +++ /dev/null @@ -1,7 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/retry_interface.sv -@RESOURCES@/retry_package.sv --F @RTL@/RequestRetry/filelist.f -@RESOURCES@/retry_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_bad_opcode.f b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_bad_opcode.f deleted file mode 100644 index 71d35c4f9..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_bad_opcode.f +++ /dev/null @@ -1,9 +0,0 @@ -+define+CHI_RETRY_BAD_OPCODE -+define+CHI_RETRY_TB=retry_bad_opcode_tb -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/retry_interface.sv -@RESOURCES@/retry_package.sv --F @RTL@/RequestRetry/filelist.f -@RESOURCES@/retry_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_bad_opcode.toml b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_bad_opcode.toml deleted file mode 100644 index e0e04bd9f..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_bad_opcode.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Unsupported CHI retry request opcode" diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_duplicate_txnid.f b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_duplicate_txnid.f deleted file mode 100644 index 81e0cd39c..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_duplicate_txnid.f +++ /dev/null @@ -1,9 +0,0 @@ -+define+CHI_RETRY_DUPLICATE -+define+CHI_RETRY_TB=retry_duplicate_txnid_tb -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/retry_interface.sv -@RESOURCES@/retry_package.sv --F @RTL@/RequestRetry/filelist.f -@RESOURCES@/retry_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_duplicate_txnid.toml b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_duplicate_txnid.toml deleted file mode 100644 index cca6bc008..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_duplicate_txnid.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "CHI retry requester reused a live TxnID" diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_interface.sv b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_interface.sv deleted file mode 100644 index c1059d20b..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_interface.sv +++ /dev/null @@ -1,21 +0,0 @@ -interface chi_retry_if ( - input logic clock -); - logic reset; - logic req_in_valid, req_in_ready, req_out_valid, req_out_ready; - logic [136:0] req_in, req_out; - logic rsp_in_valid, rsp_in_ready, rsp_out_valid, rsp_out_ready; - logic [70:0] rsp_in, rsp_out; - logic accepted_valid; - logic [6:0] accepted_src; - logic [11:0] accepted_txn; - logic [2:0] pending; - assert property (@(posedge clock) disable iff (reset) - req_out_valid && !req_out_ready |=> req_out_valid && $stable( - req_out - )); - assert property (@(posedge clock) disable iff (reset) - rsp_out_valid && !rsp_out_ready |=> rsp_out_valid && $stable( - rsp_out - )); -endinterface diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_package.sv b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_package.sv deleted file mode 100644 index 7e1405f15..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_package.sv +++ /dev/null @@ -1,480 +0,0 @@ -package chi_retry_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - `include "flit_types.svh" - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual chi_retry_if vif; - req_t requests[$]; - rsp_t responses[$], original_responses[$]; - int checks = 0; - req_t originals[int]; - int retry_types[int]; - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual chi_retry_if)::get(this, "", "vif", vif)) - `uvm_fatal("VIF", "Retry interface missing") - endfunction - task monitor(); - forever begin - @(posedge vif.clock); - if (vif.reset) begin - requests.delete(); - responses.delete(); - original_responses.delete(); - end else begin - if (vif.req_out_valid && vif.req_out_ready) requests.push_back(req_t'(vif.req_out)); - if (vif.rsp_out_valid && vif.rsp_out_ready) responses.push_back(rsp_t'(vif.rsp_out)); - end - end - endtask - task send_request(req_t r); - @(negedge vif.clock); - vif.req_in = r; - vif.req_in_valid = 1; - do @(posedge vif.clock); while (!vif.req_in_ready); - originals[r.txn] = r; - @(negedge vif.clock); - vif.req_in_valid = 0; - endtask - function bit coherent_read(int opcode); - return opcode == 1 || opcode == 'h26 || opcode == 7; - endfunction - function bit coherent_request(int opcode); - return coherent_read(opcode) || opcode == 'hd || opcode == 'h1b || opcode == 8 || - opcode == 9 || opcode == 10; - endfunction - task request(int txn, int opcode = 'hd); - req_t r = '0; - r.src = 1; - r.tgt = 64; - r.txn = txn; - r.opcode = opcode; - r.size = 6; - r.addr = 64 * txn; - r.allow_retry = 1; - if (coherent_request(opcode)) begin - r.mem_attr = 12; - r.snp_attr = 1; - end else begin - r.mem_attr = 4; - r.return_nid = 1; - r.return_txn = txn; - end - r.exp_comp_ack = coherent_read(opcode); - send_request(r); - endtask - task response(int opcode, int txn, int src = 65, int credit_type = 3, bit contiguous = 0); - rsp_t r = '0; - r.opcode = opcode; - r.txn = txn; - r.src = src; - r.tgt = 1; - r.pcrd_type = (opcode == 3 || opcode == 7) ? credit_type : 0; - r.qos = (txn ^ src) % 16; - r.trace_tag = txn % 2; - if (opcode == 6 || opcode == 5) r.dbid = (txn ^ src ^ 'h555) & 'hfff; - if (opcode == 4 || opcode == 6 || opcode == 5) r.busy = (txn + src) % 8; - if (opcode == 4 && txn % 2) r.error = 3; // Dataless completion may report NDERR. - if (!contiguous) @(negedge vif.clock); - vif.rsp_in = r; - vif.rsp_in_valid = 1; - do @(posedge vif.clock); while (!vif.rsp_in_ready); - if (opcode == 3) retry_types[txn] = credit_type; - if (opcode != 3 && opcode != 7) original_responses.push_back(r); - @(negedge vif.clock); - vif.rsp_in_valid = 0; - endtask - task expect_request(int txn, bit retry); - req_t r, golden; - while (!requests.size()) @(negedge vif.clock); - r = requests.pop_front(); - if (!originals.exists(txn)) `uvm_fatal("MODEL", "Missing observed original request") - golden = originals[txn]; - if (retry) begin - golden.allow_retry = 0; - golden.pcrd_type = retry_types[txn]; - end - if (r !== golden) - `uvm_fatal("RETRY", $sformatf( - "Complete request changed txn=%0h got=%h expected=%h", txn, r, golden)) - checks++; - endtask - task receive_response(output rsp_t r); - rsp_t golden; - while (!responses.size()) @(negedge vif.clock); - if (!original_responses.size()) `uvm_fatal("MODEL", "Missing response/original") - r = responses.pop_front(); - golden = original_responses.pop_front(); - if (r !== golden) `uvm_fatal("RESPONSE", "Complete response metadata changed") - endtask - task accept_data(int txn, int source); - @(negedge vif.clock); - vif.accepted_valid = 1; - vif.accepted_src = source; - vif.accepted_txn = txn; - @(negedge vif.clock); - vif.accepted_valid = 0; - endtask - task execute(); - rsp_t completed; - vif.reset = 1; - vif.req_in_valid = 0; - vif.rsp_in_valid = 0; - vif.accepted_valid = 0; - vif.req_in = 0; - vif.rsp_in = 0; - vif.accepted_src = 0; - vif.accepted_txn = 0; - vif.req_out_ready = 1; - vif.rsp_out_ready = 1; - fork - monitor(); - join_none - repeat (4) @(negedge vif.clock); - vif.reset = 0; -`ifdef CHI_RETRY_DUPLICATE - request(12'hf01); - expect_request(12'hf01, 0); - request(12'hf01); - `uvm_fatal("NEGATIVE", "Duplicate live TxnID did not terminate RTL") -`elsif CHI_RETRY_BAD_OPCODE - request(0, 'h11); // Separate response/data acceptance is outside this profile. - `uvm_fatal("NEGATIVE", "Unsupported retry opcode did not terminate RTL") -`else - // A normal completion passes through and releases the request record. - request(0); - expect_request(0, 0); - vif.rsp_out_ready = 0; - fork - response(4, 0); - begin - repeat (12) @(negedge vif.clock); - vif.rsp_out_ready = 1; - end - join - receive_response(completed); - if (completed.opcode != 4) `uvm_fatal("PASS", "Completion was swallowed") - for (int i = 0; i < 4; i++) begin - request(i); - expect_request(i, 0); - end - for (int i = 0; i < 4; i++) response(3, i); - repeat (10) @(negedge vif.clock); - if (requests.size() || responses.size() || original_responses.size() || vif.pending != 4) - `uvm_fatal("WAIT", "Retry issued before P-Credit or internal response leaked") - // Home remapped from TgtID 64 to RetryAck SrcID 65; match that credit source. - response(7, 0, 66, 3); - response(7, 0, 65, 2); - repeat (10) @(negedge vif.clock); - if (requests.size()) `uvm_fatal("MATCH", "Mismatched protocol credit consumed") - for (int i = 0; i < 4; i++) begin - response(7, 0); - expect_request(i, 1); - response(4, i); - receive_response(completed); - if (completed.txn != i) `uvm_fatal("COMP", "Bad completion"); - end - // An output stall must preserve the queued request and its record. - vif.req_out_ready = 0; - request(0); - repeat (12) @(negedge vif.clock); - if (!vif.req_out_valid || requests.size()) - `uvm_fatal("STALL", "Queued request lost under stall") - vif.req_out_ready = 1; - expect_request(0, 0); - response(4, 0); - receive_response(completed); - if (completed.txn != 0) `uvm_fatal("COMP", "Bad stalled completion"); - // Grant can precede RetryAck and must be retained for its matching retry. - request(0, 4); - expect_request(0, 0); - response(7, 0); - response(3, 0); - expect_request(0, 1); - @(negedge vif.clock); - vif.accepted_valid = 1; - vif.accepted_src = 9; - vif.accepted_txn = 0; - @(negedge vif.clock); - vif.accepted_valid = 0; - repeat (4) @(negedge vif.clock); - if (vif.pending != 0 || requests.size() || responses.size() || original_responses.size()) - `uvm_fatal("DRAIN", "Retry records did not drain") - // Reset cancels retry-wait records and retained protocol credits. - request(0); - expect_request(0, 0); - response(3, 0); - @(negedge vif.clock); - vif.reset = 1; - repeat (4) @(negedge vif.clock); - vif.reset = 0; - request(0); - expect_request(0, 0); - response(4, 0); - receive_response(completed); - if (completed.txn != 0) `uvm_fatal("RESET", "Completion after reset corrupted"); - repeat (4) @(negedge vif.clock); - if (vif.pending != 0 || requests.size() || responses.size() || original_responses.size()) - `uvm_fatal("DRAIN", "Retry reset did not drain") - // Full protocol IDs are independent of the four local acceptance slots. - // Exercise both polarities of every aligned address, TxnID and routing-ID bit. - for (int phase_index = 0; phase_index < 3; phase_index++) begin - int polarity = phase_index % 2; - for (int address_bit = 6; address_bit < 44; address_bit++) begin - int ids[4], homes[4], types[4], opcodes[4]; - bit [3:0] seen_reissues = '0; - int supported_ops[11] = '{1, 'h26, 7, 'hd, 'h1b, 8, 9, 10, 4, 'h1d, 'h1c}; - for (int slot = 0; slot < 4; slot++) begin - req_t r = '0; - int node_bits = (1 << ((address_bit + slot) % 7)) | 2; - ids[slot] = (1 << ((address_bit + slot) % 12)) ^ (polarity ? 'hfff : 0); - homes[slot] = (address_bit % 3 == 0 ? 65 : node_bits ^ (polarity ? 'h7f : 0)); - types[slot] = (address_bit % 3 == 1 ? address_bit % 16 : (address_bit + slot) % 16); - r.src = 1; - r.tgt = homes[slot]; - r.txn = ids[slot]; - opcodes[slot] = supported_ops[(address_bit+slot+polarity)%11]; - r.opcode = opcodes[slot]; - r.size = 6; - r.lpid = (address_bit + slot) % 32; - r.likely_shared = (address_bit + slot) % 2; - if (coherent_request(opcodes[slot])) begin - r.mem_attr = 12; - r.snp_attr = 1; - end else begin - case ((address_bit + slot) % 3) - 0: r.mem_attr = 0; - 1: r.mem_attr = 4; - 2: r.mem_attr = 12; - endcase - r.mem_attr |= (address_bit + slot + polarity) % 2; // EWA is legal for Normal memory. - r.return_nid = homes[slot]; - r.return_txn = ids[slot] ^ 'hfff; - end - r.exp_comp_ack = coherent_read(opcodes[slot]); - r.addr = ((64'h1 << address_bit) ^ (polarity ? 64'hfffffffffc0 : 0)); - r.qos = (address_bit + 3 * slot) % 16; - r.trace_tag = polarity; - r.allow_retry = 1; - send_request(r); - expect_request(ids[slot], 0); - end - if (vif.pending != 4) - `uvm_fatal("SLOTS", "Arbitrary IDs did not fill four acceptance slots") - vif.rsp_out_ready = 0; // Internal retry traffic must bypass endpoint backpressure. - for (int position = 0; position < 4; position++) begin - int slot = (position + address_bit / 4) % 4; - response(7, 0, homes[slot], types[slot]); - end - if (address_bit % 5 == 0) vif.req_out_ready = 0; - for (int position = 0; position < 4; position++) begin - int slot = (address_bit + (address_bit % 2 ? 3 - position : position)) % 4; - response(3, ids[slot], homes[slot], types[slot], position != 0); - end - repeat (3) @(negedge vif.clock); - vif.req_out_ready = 1; - vif.rsp_out_ready = 1; - for (int position = 0; position < 4; position++) begin - int slot = -1; - while (!requests.size()) @(negedge vif.clock); - for (int candidate = 0; candidate < 4; candidate++) - if (requests[0].txn == ids[candidate]) slot = candidate; - if (slot < 0 || seen_reissues[slot]) - `uvm_fatal("RETRY_SET", "Unexpected or duplicate reissued transaction") - seen_reissues[slot] = 1; - expect_request(ids[slot], 1); - if (coherent_read(opcodes[slot]) || opcodes[slot] == 4) begin - accept_data(ids[slot], homes[slot]); - end else begin - int acceptance=(opcodes[slot]=='h1b || opcodes[slot]=='h1d || opcodes[slot]=='h1c) ? 6 : 4; - if (acceptance == 6 && (address_bit + slot) % 2) acceptance = 5; - response(acceptance, ids[slot], homes[slot], 0); - receive_response(completed); - if (completed.txn != ids[slot] || completed.opcode != acceptance) - `uvm_fatal("PASS", "Acceptance response corrupted") - end - end - repeat (3) @(negedge vif.clock); - if (vif.pending != 0 || requests.size() || responses.size() || original_responses.size()) - `uvm_fatal("DRAIN", "Full-width request batch did not drain") - end - end - // Fill the two-entry output queue and hold a third request under backpressure. - vif.req_out_ready = 0; - fork - begin - for (int slot = 0; slot < 4; slot++) request(slot); - end - begin - repeat (20) @(negedge vif.clock); - vif.req_out_ready = 1; - end - join - for (int slot = 0; slot < 4; slot++) expect_request(slot, 0); - for (int slot = 0; slot < 4; slot++) begin - response(4, slot); - receive_response(completed); - end - // Initial allocation overlaps independent RetryAck and completion handshakes. - for (int context_kind = 0; context_kind < 2; context_kind++) begin - for (int slot = 0; slot < 4; slot++) begin - int other = (slot + 1) % 4; - for (int i = 0; i < 4; i++) begin - request(100 + i); - expect_request(100 + i, 0); - end - response(4, 100 + slot); - receive_response(completed); - fork - request(200 + slot); - response(context_kind ? 4 : 3, 100 + other); - join - expect_request(200 + slot, 0); - if (!context_kind) begin - response(7, 0); - expect_request(100 + other, 1); - end else begin - receive_response(completed); - end - for (int i = 0; i < 4; i++) begin - if (context_kind && i == other) continue; - response(4, i == slot ? 200 + slot : 100 + i); - receive_response(completed); - end - end - end - // A new P-Credit may arrive in the cycle a different credit is consumed. - for (int slot = 0; slot < 4; slot++) begin - int next_slot = (slot + 1) % 4; - for (int i = 0; i < 4; i++) begin - request(300 + i); - expect_request(300 + i, 0); - end - response(7, 0); - response(3, 300 + slot); - response(7, 0, 65, 3, 1); - expect_request(300 + slot, 1); - response(3, 300 + next_slot); - expect_request(300 + next_slot, 1); - for (int i = 0; i < 4; i++) begin - response(4, 300 + i); - receive_response(completed); - end - end - // Later split completion/data acceptance may follow record release. - accept_data(3, 65); - repeat (3) @(negedge vif.clock); - if (vif.pending || requests.size() || responses.size() || original_responses.size()) - `uvm_fatal("DRAIN", "Saturated output queue did not drain") - // Acceptance tracking can retire before later data beats or split completion. - request(500, 4); - expect_request(500, 0); - accept_data(500, 9); - request(501, 'h1d); - expect_request(501, 0); - response(6, 501); - receive_response(completed); - for (int i = 0; i < 4; i++) begin - request(600 + i); - expect_request(600 + i, 0); - end - for (int i = 0; i < 4; i++) response(3, 600 + i); - for (int i = 0; i < 4; i++) begin - rsp_t idle = '0; - idle.opcode = 4; - idle.tgt = 1; - idle.txn = 600 + i; - @(negedge vif.clock); - vif.rsp_in = idle; - vif.accepted_txn = 600 + i; - @(negedge vif.clock); - if (vif.pending != 4 || requests.size() || responses.size()) - `uvm_fatal("INVALID_CHANNEL", "Inactive matching payload changed a retry-wait record") - end - response(4, 501); - receive_response(completed); - accept_data(500, 9); - if (vif.pending != 4) `uvm_fatal("LATE_ACCEPT", "Late acceptance retired an unrelated retry") - for (int i = 0; i < 4; i++) begin - response(7, 0); - expect_request(600 + i, 1); - response(4, 600 + i); - receive_response(completed); - end - // VALID held through coordinated reset is accepted once after release. - begin - req_t held = '0; - held.src = 1; - held.tgt = 64; - held.txn = 'ha55; - held.opcode = 'hd; - held.size = 6; - held.addr = 64; - held.allow_retry = 1; - held.mem_attr = 12; - held.snp_attr = 1; - @(negedge vif.clock); - vif.reset = 1; - vif.req_in = held; - vif.req_in_valid = 1; - repeat (4) @(negedge vif.clock); - vif.reset = 0; - do @(posedge vif.clock); while (!vif.req_in_ready); - originals[held.txn] = held; - @(negedge vif.clock); - vif.req_in_valid = 0; - expect_request(held.txn, 0); - response(4, held.txn); - receive_response(completed); - repeat (3) @(negedge vif.clock); - if (vif.pending || requests.size() || responses.size() || original_responses.size()) - `uvm_fatal("RESET", "Held VALID did not complete exactly once") - end - // Invalid channels may carry arbitrary payload; no transaction may be consumed. - begin - req_t idle_req = '0; - rsp_t idle_rsp = '0; - idle_req.src = 1; - idle_req.opcode = 'hd; - idle_req.size = 6; - idle_req.mem_attr = 12; - idle_req.snp_attr = 1; - idle_req.allow_retry = 1; - idle_rsp.tgt = 1; - idle_rsp.opcode = 4; - for (int b = 0; b < $bits(req_t); b++) begin - @(negedge vif.clock); - vif.req_in = idle_req ^ (137'b1 << b); - @(negedge vif.clock); - vif.req_in = idle_req; - if (vif.pending || vif.req_out_valid || vif.rsp_out_valid) - `uvm_fatal("INVALID_CHANNEL", "Invalid REQ payload changed live transactions") - end - @(negedge vif.clock); - idle_req.snp_attr = 0; - idle_req.mem_attr = 2; - vif.req_in = idle_req; - @(negedge vif.clock); - if (vif.pending || vif.req_out_valid) - `uvm_fatal("INVALID_CHANNEL", "Inactive Device payload was consumed") - for (int b = 0; b < $bits(rsp_t); b++) begin - @(negedge vif.clock); - vif.rsp_in = idle_rsp ^ (71'b1 << b); - @(negedge vif.clock); - vif.rsp_in = idle_rsp; - if (vif.pending || vif.req_out_valid || vif.rsp_out_valid) - `uvm_fatal("INVALID_CHANNEL", "Invalid RSP payload changed live transactions") - end - end - repeat (2) @(negedge vif.clock); - `uvm_info("CHECKS", $sformatf("CHI retry: %0d request/reissue fields checked", checks), - UVM_LOW) -`endif - endtask - endclass -endpackage diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_tb.sv b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_tb.sv deleted file mode 100644 index 2ed3402ed..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_tb.sv +++ /dev/null @@ -1,104 +0,0 @@ -`ifndef CHI_RETRY_TB -`define CHI_RETRY_TB retry_tb -`endif -module `CHI_RETRY_TB; - import uvm_pkg::*; - import chi_retry_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - chi_retry_if source_if (clock); - RequestRetry dut ( - .clock(clock), - .reset(source_if.reset), - .io_reqIn_valid(source_if.req_in_valid), - .io_reqIn_ready(source_if.req_in_ready), - .io_reqIn_bits_qos(source_if.req_in[3:0]), - .io_reqIn_bits_tgtId(source_if.req_in[10:4]), - .io_reqIn_bits_srcId(source_if.req_in[17:11]), - .io_reqIn_bits_txnId(source_if.req_in[29:18]), - .io_reqIn_bits_returnNid(source_if.req_in[36:30]), - .io_reqIn_bits_stashNidValidEndian(source_if.req_in[37:37]), - .io_reqIn_bits_returnTxnId(source_if.req_in[49:38]), - .io_reqIn_bits_opcode(source_if.req_in[56:50]), - .io_reqIn_bits_multiReq(source_if.req_in[57:57]), - .io_reqIn_bits_size(source_if.req_in[63:58]), - .io_reqIn_bits_addr(source_if.req_in[107:64]), - .io_reqIn_bits_pas(source_if.req_in[110:108]), - .io_reqIn_bits_likelyShared(source_if.req_in[111:111]), - .io_reqIn_bits_allowRetry(source_if.req_in[112:112]), - .io_reqIn_bits_order(source_if.req_in[114:113]), - .io_reqIn_bits_pCrdType(source_if.req_in[118:115]), - .io_reqIn_bits_memAttr(source_if.req_in[122:119]), - .io_reqIn_bits_snpAttr(source_if.req_in[123:123]), - .io_reqIn_bits_lpid(source_if.req_in[131:124]), - .io_reqIn_bits_exclSnoopMe(source_if.req_in[132:132]), - .io_reqIn_bits_expCompAck(source_if.req_in[133:133]), - .io_reqIn_bits_tagOp(source_if.req_in[135:134]), - .io_reqIn_bits_traceTag(source_if.req_in[136:136]), - .io_reqOut_valid(source_if.req_out_valid), - .io_reqOut_ready(source_if.req_out_ready), - .io_reqOut_bits_qos(source_if.req_out[3:0]), - .io_reqOut_bits_tgtId(source_if.req_out[10:4]), - .io_reqOut_bits_srcId(source_if.req_out[17:11]), - .io_reqOut_bits_txnId(source_if.req_out[29:18]), - .io_reqOut_bits_returnNid(source_if.req_out[36:30]), - .io_reqOut_bits_stashNidValidEndian(source_if.req_out[37:37]), - .io_reqOut_bits_returnTxnId(source_if.req_out[49:38]), - .io_reqOut_bits_opcode(source_if.req_out[56:50]), - .io_reqOut_bits_multiReq(source_if.req_out[57:57]), - .io_reqOut_bits_size(source_if.req_out[63:58]), - .io_reqOut_bits_addr(source_if.req_out[107:64]), - .io_reqOut_bits_pas(source_if.req_out[110:108]), - .io_reqOut_bits_likelyShared(source_if.req_out[111:111]), - .io_reqOut_bits_allowRetry(source_if.req_out[112:112]), - .io_reqOut_bits_order(source_if.req_out[114:113]), - .io_reqOut_bits_pCrdType(source_if.req_out[118:115]), - .io_reqOut_bits_memAttr(source_if.req_out[122:119]), - .io_reqOut_bits_snpAttr(source_if.req_out[123:123]), - .io_reqOut_bits_lpid(source_if.req_out[131:124]), - .io_reqOut_bits_exclSnoopMe(source_if.req_out[132:132]), - .io_reqOut_bits_expCompAck(source_if.req_out[133:133]), - .io_reqOut_bits_tagOp(source_if.req_out[135:134]), - .io_reqOut_bits_traceTag(source_if.req_out[136:136]), - .io_rspIn_valid(source_if.rsp_in_valid), - .io_rspIn_ready(source_if.rsp_in_ready), - .io_rspIn_bits_qos(source_if.rsp_in[3:0]), - .io_rspIn_bits_tgtId(source_if.rsp_in[10:4]), - .io_rspIn_bits_srcId(source_if.rsp_in[17:11]), - .io_rspIn_bits_txnId(source_if.rsp_in[29:18]), - .io_rspIn_bits_opcode(source_if.rsp_in[34:30]), - .io_rspIn_bits_respErr(source_if.rsp_in[36:35]), - .io_rspIn_bits_resp(source_if.rsp_in[39:37]), - .io_rspIn_bits_fwdState(source_if.rsp_in[42:40]), - .io_rspIn_bits_cBusy(source_if.rsp_in[45:43]), - .io_rspIn_bits_dbid(source_if.rsp_in[57:46]), - .io_rspIn_bits_pCrdType(source_if.rsp_in[61:58]), - .io_rspIn_bits_tagOp(source_if.rsp_in[63:62]), - .io_rspIn_bits_traceTag(source_if.rsp_in[64:64]), - .io_rspIn_bits_cacheLineId(source_if.rsp_in[70:65]), - .io_rspOut_valid(source_if.rsp_out_valid), - .io_rspOut_ready(source_if.rsp_out_ready), - .io_rspOut_bits_qos(source_if.rsp_out[3:0]), - .io_rspOut_bits_tgtId(source_if.rsp_out[10:4]), - .io_rspOut_bits_srcId(source_if.rsp_out[17:11]), - .io_rspOut_bits_txnId(source_if.rsp_out[29:18]), - .io_rspOut_bits_opcode(source_if.rsp_out[34:30]), - .io_rspOut_bits_respErr(source_if.rsp_out[36:35]), - .io_rspOut_bits_resp(source_if.rsp_out[39:37]), - .io_rspOut_bits_fwdState(source_if.rsp_out[42:40]), - .io_rspOut_bits_cBusy(source_if.rsp_out[45:43]), - .io_rspOut_bits_dbid(source_if.rsp_out[57:46]), - .io_rspOut_bits_pCrdType(source_if.rsp_out[61:58]), - .io_rspOut_bits_tagOp(source_if.rsp_out[63:62]), - .io_rspOut_bits_traceTag(source_if.rsp_out[64:64]), - .io_rspOut_bits_cacheLineId(source_if.rsp_out[70:65]), - .io_acceptedData_valid(source_if.accepted_valid), - .io_acceptedData_bits_srcId(source_if.accepted_src), - .io_acceptedData_bits_txnId(source_if.accepted_txn), - .io_pending(source_if.pending) - ); - initial begin - uvm_config_db#(virtual chi_retry_if)::set(null, "uvm_test_top", "vif", source_if); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf.f b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf.f deleted file mode 100644 index c6baa2651..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf.f +++ /dev/null @@ -1,7 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/rnf_interface.sv -@RESOURCES@/rnf_package.sv --F @RTL@/BankedChiCache/filelist.f -@RESOURCES@/rnf_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_dbid.f b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_dbid.f deleted file mode 100644 index ec1749034..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_dbid.f +++ /dev/null @@ -1,9 +0,0 @@ -+define+RNF_TB=rnf_bad_dbid_tb -+define+RNF_BAD_DBID -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/rnf_interface.sv -@RESOURCES@/rnf_package.sv --F @RTL@/BankedChiCache/filelist.f -@RESOURCES@/rnf_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_dbid.toml b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_dbid.toml deleted file mode 100644 index f84f14e9d..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_dbid.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Inconsistent multi-beat fill metadata" diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_dbid_width.f b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_dbid_width.f deleted file mode 100644 index 72df73136..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_dbid_width.f +++ /dev/null @@ -1,9 +0,0 @@ -+define+RNF_TB=rnf_bad_dbid_width_tb -+define+RNF_BAD_DBID_WIDTH -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/rnf_interface.sv -@RESOURCES@/rnf_package.sv --F @RTL@/BankedChiCache/filelist.f -@RESOURCES@/rnf_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_dbid_width.toml b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_dbid_width.toml deleted file mode 100644 index d9d50be48..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_dbid_width.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Fill DBID exceeds completion ID width" diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_resp.f b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_resp.f deleted file mode 100644 index 2719bf861..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_resp.f +++ /dev/null @@ -1,9 +0,0 @@ -+define+RNF_TB=rnf_bad_resp_tb -+define+RNF_BAD_RESP -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/rnf_interface.sv -@RESOURCES@/rnf_package.sv --F @RTL@/BankedChiCache/filelist.f -@RESOURCES@/rnf_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_resp.toml b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_resp.toml deleted file mode 100644 index f84f14e9d..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_resp.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Inconsistent multi-beat fill metadata" diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_interface.sv b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_interface.sv deleted file mode 100644 index 0870c06d6..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_interface.sv +++ /dev/null @@ -1,58 +0,0 @@ -`ifndef RNF_TB -`define RNF_TB rnf_tb -`endif -interface chi_rnf_if ( - input logic clock -); - logic reset; - logic access_valid, access_ready, access_write; - logic [43:0] access_addr; - logic [63:0] access_data; - logic [7:0] access_mask; - logic [3:0] access_atomic; - logic access_atomic_word; - logic result_valid, result_ready, result_error; - logic [63:0] result_data; - logic req_valid, req_ready, rsp_valid, rsp_ready, dat_valid, dat_ready; - logic rx_rsp_valid, rx_rsp_ready, rx_dat_valid, rx_dat_ready, snp_valid, snp_ready; - logic [136:0] req; - logic [70:0] rsp, rx_rsp; - logic [388:0] dat, rx_dat; - logic [93:0] snp; - logic [31:0] outstanding, hits, misses; - // Statistics only: compress a prefix of repeated legal hits/misses. - // Cache contents, coherence state, transactions, and credits are never seeded. - task seed_counter(bit bank, bit miss, int unsigned value); - if (bank) begin - if (miss) $deposit($root.`RNF_TB.dut.caches_1.misses, value); - else $deposit($root.`RNF_TB.dut.caches_1.hits, value); - end else begin - if (miss) $deposit($root.`RNF_TB.dut.caches_0.misses, value); - else $deposit($root.`RNF_TB.dut.caches_0.hits, value); - end - endtask - function int unsigned counter_value(bit bank, bit miss); - if (bank) return miss ? $root.`RNF_TB.dut.caches_1.misses : $root.`RNF_TB.dut.caches_1.hits; - return miss ? $root.`RNF_TB.dut.caches_0.misses : $root.`RNF_TB.dut.caches_0.hits; - endfunction - assert property (@(posedge clock) disable iff (reset) - access_valid && !access_ready |=> access_valid && $stable( - {access_addr, access_write, access_data, access_mask, access_atomic, access_atomic_word} - )); - assert property (@(posedge clock) disable iff (reset) - req_valid && !req_ready |=> req_valid && $stable( - req - )); - assert property (@(posedge clock) disable iff (reset) - rsp_valid && !rsp_ready |=> rsp_valid && $stable( - rsp - )); - assert property (@(posedge clock) disable iff (reset) - dat_valid && !dat_ready |=> dat_valid && $stable( - dat - )); - assert property (@(posedge clock) disable iff (reset) - result_valid && !result_ready |=> result_valid && $stable( - {result_error, result_data} - )); -endinterface diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_package.sv b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_package.sv deleted file mode 100644 index 60d2d9fb8..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_package.sv +++ /dev/null @@ -1,110 +0,0 @@ -package chi_rnf_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - // Issue H, NodeID=7, address=44, data=256, all optional buses absent. - import "DPI-C" function longint unsigned rnf_ref_access( - input chandle model, - input longint unsigned addr, - input byte unsigned write, - input longint unsigned operand, - input int unsigned atomic, - input byte unsigned word, - input byte unsigned mask, - input byte unsigned sc_success, - input byte unsigned failed - ); - typedef struct packed { - bit trace_tag; - bit [1:0] tag_op; - bit exp_comp_ack, excl_snoop_me; - bit [7:0] lpid; - bit snp_attr; - bit [3:0] mem_attr, pcrd_type; - bit [1:0] order; - bit allow_retry, likely_shared; - bit [2:0] pas; - bit [43:0] addr; - bit [5:0] size; - bit multi_req; - bit [6:0] opcode; - bit [11:0] return_txn; - bit stash_valid; - bit [6:0] return_nid; - bit [11:0] txn; - bit [6:0] src, tgt; - bit [3:0] qos; - } req_t; - typedef struct packed { - bit [5:0] line_id; - bit trace_tag; - bit [1:0] tag_op; - bit [3:0] pcrd_type; - bit [11:0] dbid; - bit [2:0] busy, fwd_state, resp; - bit [1:0] error; - bit [4:0] opcode; - bit [11:0] txn; - bit [6:0] src, tgt; - bit [3:0] qos; - } rsp_t; - typedef struct packed { - bit [255:0] data; - bit [31:0] be; - bit replicate; - bit [1:0] num_dat; - bit copy_at_home, trace_tag; - bit [1:0] tag_update; - bit [7:0] tag; - bit [1:0] tag_op; - bit [5:0] line_id; - bit [1:0] data_id, ccid; - bit [15:0] dbid; - bit [2:0] busy; - bit data_pull; - bit [7:0] data_source; - bit [2:0] resp; - bit [1:0] error; - bit [3:0] opcode; - bit [6:0] home; - bit [11:0] txn; - bit [6:0] src, tgt; - bit [3:0] qos; - } dat_t; - import "DPI-C" function chandle rnf_ref_create(input int unsigned address_bits); - import "DPI-C" function void rnf_ref_destroy(input chandle model); - import "DPI-C" function byte unsigned rnf_ref_write( - input chandle model, - input longint unsigned addr, - input bit [511:0] data, - input bit [63:0] mask - ); - import "DPI-C" function byte unsigned rnf_ref_read( - input chandle model, - input longint unsigned addr, - output bit [511:0] data - ); - import "DPI-C" function longint unsigned rnf_ref_access( - input chandle model, - input longint unsigned addr, - input byte unsigned write, - input longint unsigned operand, - input int unsigned atomic, - input byte unsigned word, - input byte unsigned mask, - input byte unsigned sc_success, - input byte unsigned failed - ); - typedef struct packed { - bit trace_tag, ret_to_src, do_not_go_sd; - bit [2:0] pas; - bit [40:0] addr; - bit [4:0] opcode; - bit [11:0] fwd_txn; - bit [6:0] fwd_nid; - bit [11:0] txn; - bit [6:0] src; - bit [3:0] qos; - } snp_t; - `include "rnf_test.svh" -endpackage diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_tb.sv b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_tb.sv deleted file mode 100644 index 11a5ae6f2..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_tb.sv +++ /dev/null @@ -1,155 +0,0 @@ -`ifndef RNF_TB -`define RNF_TB rnf_tb -`endif -module `RNF_TB; - import uvm_pkg::*; - import chi_rnf_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - chi_rnf_if vif (clock); - BankedChiCache dut ( - .clock(clock), - .reset(vif.reset), - .io_chi_req_bits_qos(vif.req[0+:4]), - .io_chi_req_bits_tgtId(vif.req[4+:7]), - .io_chi_req_bits_srcId(vif.req[11+:7]), - .io_chi_req_bits_txnId(vif.req[18+:12]), - .io_chi_req_bits_returnNid(vif.req[30+:7]), - .io_chi_req_bits_stashNidValidEndian(vif.req[37+:1]), - .io_chi_req_bits_returnTxnId(vif.req[38+:12]), - .io_chi_req_bits_opcode(vif.req[50+:7]), - .io_chi_req_bits_multiReq(vif.req[57+:1]), - .io_chi_req_bits_size(vif.req[58+:6]), - .io_chi_req_bits_addr(vif.req[64+:44]), - .io_chi_req_bits_pas(vif.req[108+:3]), - .io_chi_req_bits_likelyShared(vif.req[111+:1]), - .io_chi_req_bits_allowRetry(vif.req[112+:1]), - .io_chi_req_bits_order(vif.req[113+:2]), - .io_chi_req_bits_pCrdType(vif.req[115+:4]), - .io_chi_req_bits_memAttr(vif.req[119+:4]), - .io_chi_req_bits_snpAttr(vif.req[123+:1]), - .io_chi_req_bits_lpid(vif.req[124+:8]), - .io_chi_req_bits_exclSnoopMe(vif.req[132+:1]), - .io_chi_req_bits_expCompAck(vif.req[133+:1]), - .io_chi_req_bits_tagOp(vif.req[134+:2]), - .io_chi_req_bits_traceTag(vif.req[136+:1]), - .io_chi_req_valid(vif.req_valid), - .io_chi_req_ready(vif.req_ready), - .io_chi_txRsp_bits_qos(vif.rsp[0+:4]), - .io_chi_txRsp_bits_tgtId(vif.rsp[4+:7]), - .io_chi_txRsp_bits_srcId(vif.rsp[11+:7]), - .io_chi_txRsp_bits_txnId(vif.rsp[18+:12]), - .io_chi_txRsp_bits_opcode(vif.rsp[30+:5]), - .io_chi_txRsp_bits_respErr(vif.rsp[35+:2]), - .io_chi_txRsp_bits_resp(vif.rsp[37+:3]), - .io_chi_txRsp_bits_fwdState(vif.rsp[40+:3]), - .io_chi_txRsp_bits_cBusy(vif.rsp[43+:3]), - .io_chi_txRsp_bits_dbid(vif.rsp[46+:12]), - .io_chi_txRsp_bits_pCrdType(vif.rsp[58+:4]), - .io_chi_txRsp_bits_tagOp(vif.rsp[62+:2]), - .io_chi_txRsp_bits_traceTag(vif.rsp[64+:1]), - .io_chi_txRsp_bits_cacheLineId(vif.rsp[65+:6]), - .io_chi_txRsp_valid(vif.rsp_valid), - .io_chi_txRsp_ready(vif.rsp_ready), - .io_chi_txDat_bits_qos(vif.dat[0+:4]), - .io_chi_txDat_bits_tgtId(vif.dat[4+:7]), - .io_chi_txDat_bits_srcId(vif.dat[11+:7]), - .io_chi_txDat_bits_txnId(vif.dat[18+:12]), - .io_chi_txDat_bits_homeNid(vif.dat[30+:7]), - .io_chi_txDat_bits_opcode(vif.dat[37+:4]), - .io_chi_txDat_bits_respErr(vif.dat[41+:2]), - .io_chi_txDat_bits_resp(vif.dat[43+:3]), - .io_chi_txDat_bits_dataSource(vif.dat[46+:8]), - .io_chi_txDat_bits_dataPull(vif.dat[54+:1]), - .io_chi_txDat_bits_cBusy(vif.dat[55+:3]), - .io_chi_txDat_bits_dbid(vif.dat[58+:16]), - .io_chi_txDat_bits_ccid(vif.dat[74+:2]), - .io_chi_txDat_bits_dataId(vif.dat[76+:2]), - .io_chi_txDat_bits_cacheLineId(vif.dat[78+:6]), - .io_chi_txDat_bits_tagOp(vif.dat[84+:2]), - .io_chi_txDat_bits_tag(vif.dat[86+:8]), - .io_chi_txDat_bits_tagUpdate(vif.dat[94+:2]), - .io_chi_txDat_bits_traceTag(vif.dat[96+:1]), - .io_chi_txDat_bits_copyAtHome(vif.dat[97+:1]), - .io_chi_txDat_bits_numDat(vif.dat[98+:2]), - .io_chi_txDat_bits_replicate(vif.dat[100+:1]), - .io_chi_txDat_bits_be(vif.dat[101+:32]), - .io_chi_txDat_bits_data(vif.dat[133+:256]), - .io_chi_txDat_valid(vif.dat_valid), - .io_chi_txDat_ready(vif.dat_ready), - .io_chi_rxRsp_bits_qos(vif.rx_rsp[0+:4]), - .io_chi_rxRsp_bits_tgtId(vif.rx_rsp[4+:7]), - .io_chi_rxRsp_bits_srcId(vif.rx_rsp[11+:7]), - .io_chi_rxRsp_bits_txnId(vif.rx_rsp[18+:12]), - .io_chi_rxRsp_bits_opcode(vif.rx_rsp[30+:5]), - .io_chi_rxRsp_bits_respErr(vif.rx_rsp[35+:2]), - .io_chi_rxRsp_bits_resp(vif.rx_rsp[37+:3]), - .io_chi_rxRsp_bits_fwdState(vif.rx_rsp[40+:3]), - .io_chi_rxRsp_bits_cBusy(vif.rx_rsp[43+:3]), - .io_chi_rxRsp_bits_dbid(vif.rx_rsp[46+:12]), - .io_chi_rxRsp_bits_pCrdType(vif.rx_rsp[58+:4]), - .io_chi_rxRsp_bits_tagOp(vif.rx_rsp[62+:2]), - .io_chi_rxRsp_bits_traceTag(vif.rx_rsp[64+:1]), - .io_chi_rxRsp_bits_cacheLineId(vif.rx_rsp[65+:6]), - .io_chi_rxRsp_valid(vif.rx_rsp_valid), - .io_chi_rxRsp_ready(vif.rx_rsp_ready), - .io_chi_rxDat_bits_qos(vif.rx_dat[0+:4]), - .io_chi_rxDat_bits_tgtId(vif.rx_dat[4+:7]), - .io_chi_rxDat_bits_srcId(vif.rx_dat[11+:7]), - .io_chi_rxDat_bits_txnId(vif.rx_dat[18+:12]), - .io_chi_rxDat_bits_homeNid(vif.rx_dat[30+:7]), - .io_chi_rxDat_bits_opcode(vif.rx_dat[37+:4]), - .io_chi_rxDat_bits_respErr(vif.rx_dat[41+:2]), - .io_chi_rxDat_bits_resp(vif.rx_dat[43+:3]), - .io_chi_rxDat_bits_dataSource(vif.rx_dat[46+:8]), - .io_chi_rxDat_bits_dataPull(vif.rx_dat[54+:1]), - .io_chi_rxDat_bits_cBusy(vif.rx_dat[55+:3]), - .io_chi_rxDat_bits_dbid(vif.rx_dat[58+:16]), - .io_chi_rxDat_bits_ccid(vif.rx_dat[74+:2]), - .io_chi_rxDat_bits_dataId(vif.rx_dat[76+:2]), - .io_chi_rxDat_bits_cacheLineId(vif.rx_dat[78+:6]), - .io_chi_rxDat_bits_tagOp(vif.rx_dat[84+:2]), - .io_chi_rxDat_bits_tag(vif.rx_dat[86+:8]), - .io_chi_rxDat_bits_tagUpdate(vif.rx_dat[94+:2]), - .io_chi_rxDat_bits_traceTag(vif.rx_dat[96+:1]), - .io_chi_rxDat_bits_copyAtHome(vif.rx_dat[97+:1]), - .io_chi_rxDat_bits_numDat(vif.rx_dat[98+:2]), - .io_chi_rxDat_bits_replicate(vif.rx_dat[100+:1]), - .io_chi_rxDat_bits_be(vif.rx_dat[101+:32]), - .io_chi_rxDat_bits_data(vif.rx_dat[133+:256]), - .io_chi_rxDat_valid(vif.rx_dat_valid), - .io_chi_rxDat_ready(vif.rx_dat_ready), - .io_chi_snp_bits_qos(vif.snp[0+:4]), - .io_chi_snp_bits_srcId(vif.snp[4+:7]), - .io_chi_snp_bits_txnId(vif.snp[11+:12]), - .io_chi_snp_bits_fwdNid(vif.snp[23+:7]), - .io_chi_snp_bits_fwdTxnId(vif.snp[30+:12]), - .io_chi_snp_bits_opcode(vif.snp[42+:5]), - .io_chi_snp_bits_addr(vif.snp[47+:41]), - .io_chi_snp_bits_pas(vif.snp[88+:3]), - .io_chi_snp_bits_doNotGoToSd(vif.snp[91+:1]), - .io_chi_snp_bits_retToSrc(vif.snp[92+:1]), - .io_chi_snp_bits_traceTag(vif.snp[93+:1]), - .io_chi_snp_valid(vif.snp_valid), - .io_chi_snp_ready(vif.snp_ready), - .io_access_valid(vif.access_valid), - .io_access_ready(vif.access_ready), - .io_access_bits_write(vif.access_write), - .io_access_bits_addr(vif.access_addr), - .io_access_bits_data(vif.access_data), - .io_access_bits_mask(vif.access_mask), - .io_access_bits_atomic(vif.access_atomic), - .io_access_bits_atomicWord(vif.access_atomic_word), - .io_result_valid(vif.result_valid), - .io_result_ready(vif.result_ready), - .io_result_bits_error(vif.result_error), - .io_result_bits_data(vif.result_data), - .io_outstanding(vif.outstanding), - .io_hits(vif.hits), - .io_misses(vif.misses) - ); - initial begin - uvm_config_db#(virtual chi_rnf_if)::set(null, "uvm_test_top", "vif", vif); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_test.svh b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_test.svh deleted file mode 100644 index 614383c3b..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_test.svh +++ /dev/null @@ -1,788 +0,0 @@ -class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual chi_rnf_if vif; - chandle model; - bit [511:0] memory[longint unsigned]; - longint unsigned wb_address[int unsigned], snoop_address; - bit [511:0] snoop_data; - bit [1:0] snoop_seen; - bit [511:0] wb_data[int unsigned]; - bit [1:0] wb_seen[int unsigned]; - bit ack_pending[int unsigned]; - int home_allocations = 0, snoop_allocations = 0; - int home_ids[$]; - int snoop_id; - int fill_packets = 0, partial_goal = 0; - bit hold_dat = 0, forbid_early_snoop = 0, expect_snoop_data = 0; - bit [63:0] expected[$]; - bit expected_errors[$]; - bit [1:0] fail_read = 0; - bit mixed_data_error = 0; - rsp_t rx_rsp_queue[$]; - dat_t rx_dat_queue[$]; - int cycle = 0, submitted = 0, checked = 0, writebacks = 0, snoops = 0, peak = 0; - bit hold_fill = 0, reverse_transactions = 0, hold_req = 0, hold_rsp = 0, hold_output_dat = 0; - int cancelled_writebacks = 0, shared_writebacks = 0, cancelled_cpu = 0; - bit rsp_active = 0, dat_active = 0; - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - timeout = 450us; - if (!uvm_config_db#(virtual chi_rnf_if)::get(this, "", "vif", vif)) - `uvm_fatal("VIF", "RN-F interface missing") - model = rnf_ref_create(44); - home_ids.push_back(0); - home_ids.push_back(4095); - // Odd multiplication and Gray coding form a permutation of all 12-bit IDs. - // This avoids correlating a cache bank or response kind with fixed ID bits. - for (int i = 0; i < 4096; i++) begin - int encoded = (i * 4051) & 4095; - int id = encoded ^ (encoded >> 1); - if (id != 0 && id != 4095) home_ids.push_back(id); - end - endfunction - function int allocate_home_id(); - int id; - if (home_allocations >= 4096) `uvm_fatal("HOME_ID", "Test exhausted the 12-bit Home ID pool") - home_allocations++; - return home_ids.pop_front(); - endfunction - function void consume_req(req_t r); - rsp_t response = '0; - dat_t d = '0; - if (r.src != 1 || r.tgt != 64 || r.txn >= 2 || r.size != 6 || r.addr[5:0] != 0 || !r.allow_retry) - `uvm_fatal("REQ", "RN-F request outside declared no-retry Home contract") - response.src = 64; - response.tgt = 1; - response.txn = r.txn; - if (r.opcode == 'h26 || r.opcode == 'h07) begin - if (!r.exp_comp_ack || !memory.exists(r.addr)) - `uvm_fatal("READ", "Uninitialized read or missing CompAck") - d.src = 64; - d.tgt = 1; - d.home = 64; - d.txn = r.txn; - d.dbid = allocate_home_id(); - ack_pending[d.dbid] = 1; -`ifdef RNF_BAD_DBID_WIDTH - d.dbid = 'h1000; -`endif - // B9.1.3 permits any Resp for NDERR; both packets must retain it. - d.opcode = 4; - d.error = fail_read; - d.resp = fail_read == 3 ? 7 : r.opcode == 'h07 ? 2 : 1; - d.be = '1; - // Return the upper 256b first; DataID units are 128b. - d.data_id = 2; - d.data = fail_read ? 0 : memory[r.addr][511:256]; - rx_dat_queue.push_back(d); - d.data_id = 0; - d.data = fail_read ? 0 : memory[r.addr][255:0]; - if (mixed_data_error) d.error = 0; // first packet error must survive a clean final packet -`ifdef RNF_BAD_DBID - d.dbid = d.dbid ^ 16'h0001; -`endif -`ifdef RNF_BAD_RESP - d.resp = 2; -`endif - rx_dat_queue.push_back(d); - end else if (r.opcode == 'h1b) begin - response.dbid = allocate_home_id(); - response.opcode = 5; - rx_rsp_queue.push_back(response); - wb_address[response.dbid] = r.addr; - wb_seen[response.dbid] = 0; - end else if (r.opcode == 'h0d) begin - response.opcode = 4; - rx_rsp_queue.push_back(response); - end else `uvm_fatal("REQ", "Unexpected request opcode") - endfunction - function void consume_data(dat_t d); - bit [511:0] golden; - int index; - if (d.tgt != 64 || d.src != 1 || !(d.data_id == 0 || d.data_id == 2)) - `uvm_fatal("DAT", "Unexpected RN-F data endpoint/DataID") - index = d.data_id / 2; - if (d.opcode == 2) begin - if (d.resp == 0 && (d.be != 0 || d.data != 0)) - `uvm_fatal("WB_ZERO_DATA", "CopyBack_I must carry zero data and zero byte enables") - if (!wb_address.exists(d.txn) || wb_seen[d.txn][index]) - `uvm_fatal("WB", "Unknown/duplicate writeback") - wb_seen[d.txn][index] = 1; - wb_data[d.txn][index*256+:256] = d.data; - if (!(&d.be) && d.resp != 0) `uvm_fatal("WB", "Valid writeback has empty BE") - if (&wb_seen[d.txn]) begin - if (d.resp == 0) begin - if (d.be != 0) `uvm_fatal("WB", "Cancelled writeback carries valid bytes") - cancelled_writebacks++; - end else begin - if (rnf_ref_read(model, wb_address[d.txn], golden) || wb_data[d.txn] !== golden) - `uvm_fatal("WB", "Dirty writeback differs from independent model") - memory[wb_address[d.txn]] = wb_data[d.txn]; - if (d.resp == 1) shared_writebacks++; - end - writebacks++; - wb_address.delete(d.txn); - wb_data.delete(d.txn); - wb_seen.delete(d.txn); - end - end else if (d.opcode == 1) begin - if (forbid_early_snoop && fill_packets < partial_goal) - `uvm_fatal("SNOOP_EARLY", "Same-line snoop responded before the final fill packet") - if (d.txn != snoop_id || !(&d.be)) `uvm_fatal("SNOOP", "Unexpected snoop data") - if (snoop_seen[index]) `uvm_fatal("SNOOP", "Duplicate snoop DataID") - snoop_data[index*256+:256] = d.data; - snoop_seen[index] = 1; - if (&snoop_seen) begin - if (rnf_ref_read(model, snoop_address, golden) || snoop_data !== golden) - `uvm_fatal("SNOOP", "Snoop differs from golden CPU data") - memory[snoop_address] = snoop_data; - snoops++; - end - end else `uvm_fatal("DAT", "Unexpected RN-F DAT opcode") - endfunction - task monitor(); - forever begin - @(posedge vif.clock); - if (!vif.reset) begin - cycle++; - if (vif.outstanding > peak) peak = vif.outstanding; - if (vif.rx_dat_valid && vif.rx_dat_ready) fill_packets++; - if (vif.req_valid && vif.req_ready) consume_req(req_t'(vif.req)); - if (vif.dat_valid && vif.dat_ready) consume_data(dat_t'(vif.dat)); - if (vif.rsp_valid && vif.rsp_ready) begin - rsp_t r = rsp_t'(vif.rsp); - if (r.opcode != 2 && r.opcode != 1) `uvm_fatal("RSP", "Unexpected RN-F RSP opcode") - if (r.src != 1 || r.tgt != 64) `uvm_fatal("RSP", "RN-F response endpoint mismatch") - if (r.opcode == 2) begin - if (!ack_pending.exists(r.txn)) - `uvm_fatal("ACK", "CompAck did not retain the independent Home DBID") - ack_pending.delete(r.txn); - end else begin - if (forbid_early_snoop && fill_packets < partial_goal) - `uvm_fatal("SNOOP_EARLY", "Same-line snoop responded before the final fill packet") - if (expect_snoop_data) - `uvm_fatal("SNOOP_DATA", "Cached RetToSrc snoop did not return data") - if (r.txn != snoop_id) - `uvm_fatal("SNOOP", "Dataless snoop response has wrong transaction") - snoops++; - end - end - if (vif.result_valid && vif.result_ready) begin - bit [63:0] golden; - bit error; - if (!expected.size()) `uvm_fatal("RESULT", "Unexpected completion") - golden = expected.pop_front(); - error = expected_errors.pop_front(); - if (vif.result_error != error || vif.result_data !== golden) - `uvm_fatal("RESULT", $sformatf( - "completion %0d got=%h expected=%h", checked, vif.result_data, golden)) - checked++; - end - if (rsp_active && vif.rx_rsp_ready) rsp_active = 0; - if (dat_active && vif.rx_dat_ready) dat_active = 0; - end - @(negedge vif.clock); - vif.req_ready = !vif.reset && !hold_req && cycle % 7 >= 2; - vif.rsp_ready = !vif.reset && !hold_rsp && cycle % 11 >= 5; - vif.dat_ready = !vif.reset && !hold_output_dat && cycle % 13 >= 6; - vif.result_ready = !vif.reset && cycle % 9 >= 3; - if (!rsp_active && rx_rsp_queue.size()) begin - vif.rx_rsp = rx_rsp_queue.pop_front(); - rsp_active = 1; - end - if (!dat_active && !hold_fill && !hold_dat && rx_dat_queue.size()) begin - if (reverse_transactions) vif.rx_dat = rx_dat_queue.pop_back(); - else vif.rx_dat = rx_dat_queue.pop_front(); - dat_active = 1; - end - vif.rx_rsp_valid = rsp_active && !vif.reset; - vif.rx_dat_valid = dat_active && !vif.reset; - end - endtask - task issue(longint unsigned address, bit write = 0, bit [63:0] value = 0, int atomic = 0, - bit [7:0] mask = '1, bit sc_success = 0, bit expect_error = 0, bit atomic_word = 0); - bit [63:0] answer; - @(negedge vif.clock); - vif.access_addr = address; - vif.access_write = write; - vif.access_data = value; - vif.access_atomic = atomic; - vif.access_atomic_word = atomic_word; - vif.access_mask = mask; - vif.access_valid = 1; - do @(posedge vif.clock); while (!vif.access_ready); - answer = rnf_ref_access(model, address, write, value, atomic, atomic_word, mask, sc_success, - expect_error); - expected.push_back(answer); - expected_errors.push_back(expect_error); - submitted++; - @(negedge vif.clock); - vif.access_valid = 0; - endtask - task settle(); - wait (checked + cancelled_cpu == submitted); - while (ack_pending.num() != 0 || wb_address.num() != 0) @(negedge vif.clock); - repeat (4) @(negedge vif.clock); - endtask - task access (longint unsigned address, bit write = 0, bit [63:0] value = 0, int atomic = 0, - bit [7:0] mask = '1, bit sc_success = 0, bit expect_error = 0, bit atomic_word = 0); - issue(address, write, value, atomic, mask, sc_success, expect_error, atomic_word); - settle(); - endtask - task snoop_line(longint unsigned address, int opcode, bit ret_source = 0); - snp_t s = '0; - int before_count = snoops; - expect_snoop_data = ret_source; - snoop_id = snoop_allocations % 2 ? snoop_allocations / 2 : 4095 - snoop_allocations / 2; - snoop_allocations++; - s.src = 64; - s.txn = snoop_id; - s.addr = address >> 3; - s.opcode = opcode; - s.do_not_go_sd = 1; - s.ret_to_src = ret_source; - snoop_address = address; - snoop_seen = 0; - @(negedge vif.clock); - vif.snp = s; - vif.snp_valid = 1; - do @(posedge vif.clock); while (!vif.snp_ready); - @(negedge vif.clock); - vif.snp_valid = 0; - wait (snoops > before_count); - repeat (4) @(negedge vif.clock); - endtask - task partial_fill_snoop(longint unsigned target, longint unsigned probe, bit reverse); - int before_packets = fill_packets; - int before_snoops = snoops; - reverse_transactions = reverse; - issue(target); - wait (fill_packets == before_packets + 1); - hold_dat = 1; - partial_goal = before_packets + 2; - forbid_early_snoop = target == probe; - fork - snoop_line(probe, 4, 1); - begin - repeat (20) @(negedge vif.clock); - if (target != probe && snoops == before_snoops) - `uvm_fatal("SNOOP_INDEPENDENT", - "Different-line snoop waited for an unrelated final fill packet") - hold_dat = 0; - end - join - forbid_early_snoop = 0; - reverse_transactions = 0; - settle(); - endtask - task reset_pending_reads(); - bit [511:0] line; - hold_fill = 1; - issue(1024); - issue(1088); - wait (ack_pending.num() == 2); - if (vif.outstanding != 2 || expected.size() != 2) - `uvm_fatal("RESET_PENDING", "Reset did not start with two pending reads") - @(negedge vif.clock); - vif.reset = 1; - // Coordinated reset cancels unretired CPU work and both endpoints' CHI work. - cancelled_cpu += expected.size(); - expected.delete(); - expected_errors.delete(); - ack_pending.delete(); - rx_dat_queue.delete(); - rx_rsp_queue.delete(); - rsp_active = 0; - dat_active = 0; - vif.access_addr = 2048; - vif.access_write = 0; - vif.access_data = 0; - vif.access_atomic = 0; - vif.access_atomic_word = 0; - vif.access_mask = '1; - vif.access_valid = 1; - repeat (4) @(negedge vif.clock); - if (vif.outstanding != 0) `uvm_fatal("RESET_PENDING", "Reset retained CPU retirement entries") - // VALID may remain asserted through reset. Transfers count only outside reset; - // the held command must produce exactly one post-reset completion. - vif.reset = 0; - hold_fill = 0; - do @(posedge vif.clock); while (!vif.access_ready); - if (rnf_ref_read(model, 2048, line)) `uvm_fatal("MODEL", "Reset probe is uninitialized") - expected.push_back(line[63:0]); - expected_errors.push_back(0); - submitted++; - @(negedge vif.clock); - vif.access_valid = 0; - settle(); - endtask - task idle_metadata(); - // Invalid payload is unconstrained; changing it must not create a transaction. - for (int bank = 0; bank < 2; bank++) begin - for (int field = 0; field < 6; field++) begin - rsp_t r = '0; - dat_t d = '0; - snp_t probe = '0; - r.src = 64; - r.tgt = 1; - r.txn = bank; - d.opcode = 4; - d.src = 64; - d.tgt = 1; - d.txn = bank; - d.home = 64; - probe.src = 64; - probe.opcode = 4; - case (field) - 0: begin - r.tgt = 0; - d.opcode = 0; - probe.src = 0; - end - 1: begin - r.src = 0; - d.src = 0; - probe.pas = 1; - end - 2: begin - r.txn = bank ^ 1; - d.tgt = 0; - probe.fwd_nid = 1; - end - 3: begin - r.error = 3; - d.home = 0; - probe.fwd_txn = 1; - end - 4: begin - r.txn = 4095; - d.txn = 4095; - probe.opcode = 0; - end - 5: begin - d.data_id = 3; - d.dbid = 16'hf000; - end - endcase - @(negedge vif.clock); - vif.rx_rsp = r; - vif.rx_dat = d; - vif.snp = probe; - vif.access_addr = bank * 64 + 1; - vif.access_atomic = 15; - vif.access_atomic_word = 0; - vif.access_write = 1; - vif.access_mask = 0; - repeat (2) @(negedge vif.clock); - end - end - for (int bit_index = 0; bit_index < $bits(dat_t); bit_index++) begin - @(negedge vif.clock); - vif.rx_rsp = rsp_t'(rsp_t'(1) << bit_index); - vif.rx_dat = dat_t'(dat_t'(1) << bit_index); - vif.snp = snp_t'(snp_t'(1) << bit_index); - vif.access_addr = 44'(1) << bit_index; - vif.access_data = 64'(1) << bit_index; - vif.access_mask = 8'(1) << bit_index; - vif.access_atomic = 4'(1) << bit_index; - vif.access_atomic_word = bit_index % 2; - vif.access_write = bit_index % 2; - repeat (2) @(negedge vif.clock); - if (vif.req_valid || vif.rsp_valid || vif.dat_valid || vif.result_valid || vif.outstanding) - `uvm_fatal("IDLE_PAYLOAD", "Invalid walking-bit payload created CPU or CHI work") - end - if (vif.req_valid || vif.rsp_valid || vif.dat_valid || vif.result_valid || vif.outstanding) - `uvm_fatal("IDLE_PAYLOAD", "Invalid payload created CPU or CHI work") - vif.rx_rsp = '0; - vif.rx_dat = '0; - vif.snp = '0; - vif.access_addr = 0; - vif.access_atomic = 0; - vif.access_atomic_word = 0; - vif.access_write = 0; - vif.access_mask = '1; - endtask - task execute(); - vif.reset = 1; - vif.access_valid = 0; - vif.access_atomic_word = 0; - vif.snp_valid = 0; - vif.rx_rsp_valid = 0; - vif.rx_dat_valid = 0; - vif.req_ready = 0; - vif.rsp_ready = 0; - vif.dat_ready = 0; - vif.result_ready = 0; - for (int i = 0; i < 256; i++) begin - bit [511:0] initial_line; - for (int j = 0; j < 8; j++) - initial_line[j*64 +: 64] = (i%2 ? 64'h0f0f0f0f0f0f0f0f : 64'hf0f0f0f0f0f0f0f0) ^ 64'(i*8+j); - memory[i*64] = initial_line; - if (rnf_ref_write(model, i * 64, initial_line, '1)) - `uvm_fatal("MODEL", "Initialization failed") - end - repeat (4) @(negedge vif.clock); - vif.reset = 0; - idle_metadata(); - fork - monitor(); - join_none - hold_fill = 1; - issue(0); - issue(64); - repeat (10) @(negedge vif.clock); - if (vif.outstanding != 2) `uvm_fatal("PARALLEL", "Both cache banks did not remain in flight") - reverse_transactions = 1; - hold_fill = 0; - settle(); - reverse_transactions = 0; - for (int line_index = 2; line_index < 8; line_index++) begin - access (line_index * 64); - end - reset_pending_reads(); - for (int word = 1; word < 8; word++) begin - access (word * 8); - access (word * 8, 1, 64'h55aa55aa55aa55aa ^ 64'(word), 0, 8'hff >> word); - end - access (0, 1, 'h1122334455667788, 0, 'h0f); - snoop_line(0, 4); // downgrade dirty to shared - access (0); - access (0, 1, 'h8877665544332211); - access (512); // conflicting dirty victim - access (0); - for (int bank = 0; bank < 2; bank++) begin - for (int op = 1; op <= 9; op++) begin - // Exercise the independent hit and fill AMO datapaths with both comparisons. - access (bank * 64, 1, 0); - access (bank * 64, 0, 3, op); - access (bank * 64, 0, 64'hfffffffffffffffe, op); - snoop_line(bank * 64, 4); - access (bank * 64, 0, 3, op); - snoop_line(bank * 64, 4); - access (bank * 64, 0, 64'hfffffffffffffffe, op); - if (op >= 6) begin - for (int refill = 0; refill < 2; refill++) begin - access (bank * 64, 1, '1); - if (refill) snoop_line(bank * 64, 4); - access (bank * 64, 0, 0, op); - access (bank * 64, 1, 0); - if (refill) snoop_line(bank * 64, 4); - access (bank * 64, 0, '1, op); - end - end - end - access (bank * 64, 0, 0, 10); - access (bank * 64, 0, 99, 11, '1, 1); // successful SC hit - snoop_line(bank * 64, 4); - access (bank * 64, 0, 0, 10); - access (bank * 64, 0, 101, 11, '1, 1); // successful SC fill - access (bank * 64, 0, 0, 12); - end - // AMO.W uses an independent u32/i32 reference and must preserve the other lane. - for (int bank = 0; bank < 2; bank++) begin - for (int lane = 0; lane < 2; lane++) begin - longint unsigned address = bank * 64 + lane * 4; - bit [31:0] lhs_values[4] = '{32'hffffffff, 32'h80000000, 32'h7fffffff, 32'h0}; - bit [31:0] rhs_values[4] = '{32'h1, 32'h7fffffff, 32'h80000000, 32'hffffffff}; - for (int op = 1; op <= 9; op++) begin - for (int boundary = 0; boundary < 4; boundary++) begin - bit [63:0] initial_value = 64'h5aa55aa5a55aa55a; - initial_value[lane*32+:32] = lhs_values[boundary]; - access (bank * 64, 1, initial_value); - if (boundary % 2) snoop_line(bank * 64, 4); // exercise refill permission upgrade - // High operand bits are intentionally unrelated to the 32-bit operation. - access (address, 0, {32'hdeadbeef, rhs_values[boundary]}, op, '1, 0, 0, 1); - access (bank * 64); // independently check both lane update and neighbor preservation - end - end - access (address, 0, 0, 10, '1, 0, 0, 1); - access (address, 0, 64'habcdef0180000001, 11, '1, 1, 0, 1); - access (bank * 64); - snoop_line(bank * 64, 4); - access (address, 0, 0, 10, '1, 0, 0, 1); - access (address, 0, 64'hffffffffffffffff, 11, '1, 1, 0, 1); - access (bank * 64); - access (address, 0, 0, 10, '1, 0, 0, 1); - snoop_line(bank * 64, 7); - access (address, 0, 99, 11, '1, 0, 0, 1); - access (address, 0, 0, 10, '1, 0, 0, 1); - access (bank * 64 + (1 - lane) * 4, 0, 77, 11, '1, 0, 0, 1); - access (address, 0, 88, 11, '1, 0, 0, 1); // failed other-lane SC consumes LR - access (address, 0, 0, 10, '1, 0, 0, 1); - access ((1 - bank) * 64, 0, 77, 11, '1, 0, 0, 1); - access (address, 0, 88, 11, '1, 0, 0, 1); // failed other-bank SC consumes LR - access (address, 0, 0, 10, '1, 0, 0, 1); - access ((1 - bank) * 64, 0, 0, 10); // later LR.D replaces LR.W across banks - access (address, 0, 88, 11, '1, 0, 0, 1); - end - access (bank * 64, 0, 0, 10, '1, 0, 0, 1); - access (bank * 64, 0, 88, 11); // same address, mismatched reservation width - access (bank * 64, 0, 0, 10); - access (bank * 64, 0, 88, 11, '1, 0, 0, 1); - access (bank * 64, 0, 0, 10, '1, 0, 0, 1); - snoop_line((1 - bank) * 64, 7); // unrelated bank invalidation preserves reservation - access (bank * 64, 0, 99, 11, '1, 1, 0, 1); - access (bank * 64); - end - // Every word position uses both lane selectors in the hit and refill merge datapaths. - for (int bank = 0; bank < 2; bank++) begin - for (int word_index = 0; word_index < 8; word_index++) begin - longint unsigned word_address = bank * 64 + word_index * 8; - access (word_address, 1, 64'h800000007fffffff); - for (int lane = 0; lane < 2; lane++) begin - access (word_address + lane * 4, 0, 64'hdeadbeef00000001, 2, '1, 0, 0, 1); - access (word_address); - snoop_line(bank * 64, 4); - access (word_address + lane * 4, 0, 64'hdeadbeefffffffff, 2, '1, 0, 0, 1); - access (word_address); - end - end - end - access (576); // dirty eviction through the second bank - access (64); - access (64, 0, 0, 10); - access (64, 0, 99, 11, '1, 1); - access (64, 0, 0, 10); - snoop_line(64, 7); // invalidate consumes LR - access (64, 0, 123, 11); - access (0, 0, 0, 10); - access (64, 0, 123, 11); // SC to other bank fails and consumes reservation - access (0, 0, 123, 11); - access (0, 0, 0, 12); - snoop_line(448, 4); // clean cached Shared line: dataless response - snoop_line(1536, 9); // absent line: dataless Invalid response - fail_read = 2; - access (768, 0, 0, 0, '1, 0, 1); - fail_read = 0; - access (768); // errored fill must not leave a valid cache line - access (512); // write back the bank0 AMO/SC result before reset invalidates L1 - // Coordinated, quiescent reset invalidates L1; backing contents persist. - @(negedge vif.clock); - vif.reset = 1; - repeat (4) @(negedge vif.clock); - vif.reset = 0; - issue(0); - access (64); - // Every legal high address bit must reach the tag compare and eviction path. - for (int bit_index = 9; bit_index < 44; bit_index++) begin - for (int cache_index = 0; cache_index < 8; cache_index++) begin - longint unsigned high = (64'(1) << bit_index) | (64'(cache_index) << 6); - bit [511:0] high_line = {8{(bit_index % 2 ? 64'h0123456789abcdef : ~64'h0123456789abcdef) ^ high}}; - memory[high] = high_line; - if (rnf_ref_write(model, high, high_line, '1)) `uvm_fatal("MODEL", "High-tag init failed") - access (high, 0, 0, 10); // full-address LR reservation in each bank - snoop_line(high, 4, 1); // all entry indices and high Snoop addresses, full clean data - access (high + 56, 1, - (bit_index % 2 ? 64'h0123456789abcdef : ~64'h0123456789abcdef) ^ high); - access (high + 8, 1, - (bit_index % 2 ? ~64'h0123456789abcdef : 64'h0123456789abcdef) ^ high); // hit-write R3 read port - access (high); // R0 hit-read port must observe the complementary full line - access (cache_index * 64); // dirty high-tag copyback retains the full address - end - end - for (int bank = 0; bank < 2; bank++) begin - access (bank * 64, 1, 64'h1234567890abcdef); - hold_req = 1; - issue(512 + bank * 64); - begin - req_t pending_request; - do begin - @(negedge vif.clock); - pending_request = req_t'(vif.req); - end while (!vif.req_valid || pending_request.opcode != 'h1b); - end - snoop_line(bank * 64, 7); // revoke dirty victim while its WriteBackFull is queued - begin - bit [511:0] replacement = {8{64'hfedcba9876543210}}; - memory[bank*64] = replacement; - if (rnf_ref_write(model, bank * 64, replacement, '1)) - `uvm_fatal("MODEL", "Remote write failed") - end - hold_req = 0; - settle(); - access (bank * 64); // cancelled copyback must not overwrite the newer line - end - if (cancelled_writebacks == 0) - `uvm_fatal("CANCEL", "Queued writeback cancellation did not execute") - fail_read = 2; - access (832, 0, 0, 0, '1, 0, 1); // second bank error path - fail_read = 0; - access (832); - access (448, 0, 0, 10); - snoop_line(448, 4, 1); // clean RetToSrc data must preserve LR reservation - access (448, 0, 44, 11, '1, 1); - for (int bank = 0; bank < 2; bank++) begin - access (bank * 64, 1, 64'h0123456789abcdef); - hold_req = 1; - issue(512 + bank * 64); - begin - req_t pending_request; - do begin - @(negedge vif.clock); - pending_request = req_t'(vif.req); - end while (!vif.req_valid || pending_request.opcode != 'h1b); - end - snoop_line(bank * 64, 4); // queued dirty victim is downgraded to Shared before copyback - hold_req = 0; - settle(); - access (bank * 64); - for (int entry = 0; entry < 4; entry++) begin - longint unsigned address = entry * 128 + bank * 64; - access (address); - snoop_line(address, 4); - access (address, 0, 0, 10); - hold_req = 1; - issue(address, 0, 64'habcdef, - 11); // initially valid LR, SC must fail after the intervening snoop - begin - req_t pending_request; - do begin - @(negedge vif.clock); - pending_request = req_t'(vif.req); - end while (!vif.req_valid || pending_request.opcode != 'h07); - end - snoop_line(address, 7); - hold_req = 0; - settle(); - access (address); // failed SC fill acquired UC but must not modify the word - access (address, 1, 64'h123456789abcdef0); // clean UC hit becomes dirty - end - end - for (int order = 0; order < 2; order++) begin - access (0, 1, 64'h55aa55aa55aa55aa); - access (64, 1, 64'haa55aa55aa55aa55); - hold_output_dat = 1; - issue(3072 + order * 64); - issue(3136 - order * 64); // both bank arrival orders - repeat (60) @(negedge vif.clock); - hold_output_dat = 0; - settle(); - end - // Fill both output request and response queues under sustained legal backpressure. - hold_req = 1; - hold_rsp = 1; - issue(2560); - issue(2624); - repeat (30) @(negedge vif.clock); - hold_req = 0; - repeat (60) @(negedge vif.clock); - fork - snoop_line(16000, 9); // third response meets two queued CompAcks - begin - repeat (20) @(negedge vif.clock); - hold_rsp = 0; - end - join - settle(); - for (int bank = 0; bank < 2; bank++) begin - for (int entry = 0; entry < 4; entry++) begin - longint unsigned address = entry * 128 + bank * 64; - snoop_line(address, 7); - snoop_line(address, 4); // absent Shared snoop at each directory entry - fail_read = entry % 2 ? 3 : 2; - mixed_data_error = entry == 2; - access (address, 0, 0, entry % 2 ? 0 : 10, '1, 0, 1); - fail_read = 0; - mixed_data_error = 0; - access (((entry + 1) % 4) * 128 + bank * 64); // a hit clears the previous response error - access (address); - access (address, 0, 0, 10); - access (address); // ordinary hit must retain LR - access (address, 0, 11, 11, '1, 1); - end - end - for (int bank = 0; bank < 2; bank++) begin - access (bank * 64, 0, 0, 10); - snoop_line(128 + bank * 64, 7); // unrelated-line invalidation must preserve LR - access (bank * 64, 0, 77, 11, '1, 1); - access (bank * 64, 0, 0, 0, 0); // a normal read does not consume byte enables - fork - issue(bank * 64); - begin - repeat (2) @(posedge vif.clock); - snoop_line(256 + bank * 64, 4); // snoop overlaps lookup and temporarily stops it - end - join - settle(); - end - access (128); // independent-line probe must be resident after invalidation scenarios - partial_fill_snoop(1024, 1024, 0); - partial_fill_snoop(1536, 1536, 1); // opposite DataID arrival order - partial_fill_snoop(2048, 128, 0); // same bank, independent cache line - for (int bank = 0; bank < 2; bank++) begin - access (bank * 64 + 56, 1, '1); - access (bank * 64, 1, '1); // full-width hit merge input includes the high word - access (bank * 64 + 56, 1, 0); - access (bank * 64, 1, 0); - for (int word = 0; word < 8; word++) begin - access (bank * 64 + word * 8, 0, 0, 10); - access (bank * 64 + word * 8, 1, '1, 0, 8'hfe); - access (bank * 64 + word * 8, 1, 0, 0, 8'h00); - access (bank * 64 + word * 8); - end - access (bank * 64, 0, 0, 10); // release high word-offset reservation bits - end - for (int bank = 0; bank < 2; bank++) begin - for (int miss = 0; miss < 2; miss++) begin - access (bank * 64); - for (int bit_index = 0; bit_index <= 32; bit_index++) begin - int unsigned seed = bit_index == 32 ? '1 : (32'(1) << bit_index) - 1; - int unsigned other, expected_counter = seed + 1; - longint unsigned address = miss ? (bit_index % 2 ? 1024 : 512) + bank * 64 : bank * 64; - @(negedge vif.clock); - other = vif.counter_value(!bank, miss); - vif.seed_counter(bank, miss, seed); - access (address); - if (vif.counter_value(bank, miss) !== expected_counter) - `uvm_fatal("COUNTER", "Real hit/miss did not increment the seeded statistical boundary") - if ((miss ? vif.misses : vif.hits) !== 32'(other + expected_counter)) - `uvm_fatal("COUNTER_SUM", "Bank sum did not retain modulo-32bit statistics") - end - end - end - begin - longint unsigned a = 24576; - bit [511:0] fresh = '0; - int before_alloc; - memory[a] = fresh; - if (rnf_ref_write(model, a, fresh, '1)) - `uvm_fatal("MODEL", "CMO fixture initialization failed") - access (a, 1, 64'h1122334455667788); - snoop_line(a, 8); // SnpCleanShared: pass dirty data, retain a clean shared copy. - before_alloc = home_allocations; - access (a); - if (home_allocations != before_alloc) - `uvm_fatal("CLEAN_SHARED", "SnpCleanShared invalidated a resident clean copy") - access (a, 1, 64'hdeadbeef01234567); - fresh[63:0] = 64'hcafebabe87654321; - memory[a] = fresh; - if (rnf_ref_write(model, a, fresh, '1)) - `uvm_fatal("GOLDEN", "DMA test write was outside physical memory") - snoop_line(a, 10); // SnpMakeInvalid must not return stale dirty data. - if (memory[a] !== fresh) - `uvm_fatal("MAKE_INVALID", "Stale dirty data overwrote newer DMA memory") - before_alloc = home_allocations; - access (a); - if (home_allocations != before_alloc + 1) - `uvm_fatal("INVALID_REFILL", "SnpMakeInvalid did not invalidate the resident line") - snoop_line(a, 8); // Clean shared copy: dataless retained response. - snoop_line(a, 10); - snoop_line(a, 10); // Absent line: dataless invalid response. - `uvm_info("CMO", "SnpCleanShared retention and SnpMakeInvalid dirty discard checked", UVM_LOW) - end - if (shared_writebacks == 0) - `uvm_fatal("SHARED_WB", "Downgraded victim did not return Shared copyback") - if (peak < 2 || writebacks == 0 || snoops < 2 || expected.size()) - `uvm_fatal("COVERAGE", "Required parallel, dirty eviction or snoop case not exercised") - `uvm_info( - "RNF", - $sformatf( - "Checked %0d CPU completions, peak=%0d writebacks=%0d snoops=%0d cancelled=%0d shared=%0d reset_cancelled=%0d", - checked, peak, writebacks, snoops, cancelled_writebacks, shared_writebacks, - cancelled_cpu), UVM_LOW) - rnf_ref_destroy(model); - endtask -endclass diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rx.el b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rx.el deleted file mode 100644 index cfecac53d..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rx.el +++ /dev/null @@ -1,78 +0,0 @@ -// Format Version: 2 -// Reviewed against the exact exported object signatures and the declared depth4 always-on profile. - -CHECKSUM: "2163818788 95260501" -INSTANCE: rx_tb.dut -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX flit on inactive link" -Block 2 "1857595103" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX flit on inactive link" -Block 3 "971991619" "$error(\"Assertion failed: CHI RX flit on inactive link\n at Rx.scala:64 assert(io.active, \\"CHI RX flit on inactive link\\")\n\");" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX flit on inactive link" -Block 5 "1857595103" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX flit on inactive link" -Block 6 "3528662196" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI FLITPEND must precede FLITV" -Block 10 "3823759045" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI FLITPEND must precede FLITV" -Block 11 "2663934422" "$error(\"Assertion failed: CHI FLITPEND must precede FLITV\n at Rx.scala:65 assert(previousPend, \\"CHI FLITPEND must precede FLITV\\")\n\");" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI FLITPEND must precede FLITV" -Block 13 "3823759045" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI FLITPEND must precede FLITV" -Block 14 "2045170107" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX flit without granted credit" -Block 18 "798018255" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX flit without granted credit" -Block 19 "74350349" "$error(\"Assertion failed: CHI RX flit without granted credit\n at Rx.scala:66 assert(advertised =/= 0.U, \\"CHI RX flit without granted credit\\")\n\");" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX flit without granted credit" -Block 21 "798018255" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX flit without granted credit" -Block 22 "2469915260" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX buffer overflow" -Block 26 "2435529851" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX buffer overflow" -Block 27 "2348576011" "$error(\"Assertion failed: CHI RX buffer overflow\n at Rx.scala:67 assert(enqueueReady, \\"CHI RX buffer overflow\\")\n\");" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX buffer overflow" -Block 29 "2435529851" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX buffer overflow" -Block 30 "773203258" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX credit conservation" -Block 34 "3002688943" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX credit conservation" -Block 35 "2461347743" "$error(\"Assertion failed: CHI RX credit conservation\n at Rx.scala:69 assert(reserved <= depth.U, \\"CHI RX credit conservation\\")\n\");" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX credit conservation" -Block 37 "3002688943" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX credit conservation" -Block 38 "1033059400" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX requires coordinated reset to stop" -Block 42 "2653430091" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX requires coordinated reset to stop" -Block 43 "1798953395" "$error(\"Assertion failed: CHI RX requires coordinated reset to stop\n at Rx.scala:70 when(wasActive)(assert(io.active, \\"CHI RX requires coordinated reset to stop\\"))\n\");" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX requires coordinated reset to stop" -Block 45 "2653430091" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX requires coordinated reset to stop" -Block 46 "2212642507" "$fatal;" - - -CHECKSUM: "2163818788 2138553741" -INSTANCE: rx_tb.dut -ANNOTATION: "Assertion-failure vector: physical FLITV requires the activated link." -Condition 2 "3562691098" "(_GEN_0 & ((~io_active))) 1 -1" (3 "11") -ANNOTATION: "Assertion-failure vector: FLITPEND must precede FLITV by a cycle." -Condition 3 "2541789932" "(_GEN_0 & ((~previousPend))) 1 -1" (3 "11") -ANNOTATION: "Assertion-failure vector: sender must hold a granted credit; independently checked by rx_bad_credit." -Condition 4 "959243910" "(_GEN_0 & (advertised == 3'b0)) 1 -1" (3 "11") -ANNOTATION: "Credit conservation proves any legal arriving flit has a reserved FIFO entry; overflow is outside legal peer behavior." -Condition 6 "183470145" "(_GEN_0 & ((~enqueueReady))) 1 -1" (3 "11") -ANNOTATION: "Depth4 invariant reserved=advertised+occupancy<=4, including coordinated reset boundary." -Condition 7 "311107905" "(((~reset)) & (reserved > 4'h4)) 1 -1" (1 "01") -ANNOTATION: "Depth4 invariant reserved<=4: grant is issued only below depth and legal enqueue replaces one advertised credit with one occupied entry." -Condition 7 "311107905" "(((~reset)) & (reserved > 4'h4)) 1 -1" (3 "11") -ANNOTATION: "Always-on profile forbids runtime deactivation except under coordinated reset; assertion-failure vector only." -Condition 8 "3706255149" "(wasActive & ((~reset)) & ((~io_active))) 1 -1" (4 "111") -ANNOTATION: "A legal arriving flit owns advertised credit>0; advertised+occupancy<=4 implies occupancy<4 and enqueueReady. This functional vector is unreachable." -Condition 9 "241394234" "(io_link_flitv & enqueueReady) 1 -1" (2 "10") - -CHECKSUM: "2163818788 1283625097" -INSTANCE: rx_tb.dut -ANNOTATION: "Only reserved[3]: reset starts at zero; legal FLITV consumes an advertised entry, enqueue moves it to occupancy, and grant cannot exceed depth4. Hence reserved<=4 and bit3 is invariant zero. Bits2:0 remain measured." -Toggle reserved [3] "net reserved[3:0]" diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rx.f b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rx.f deleted file mode 100644 index 75884eb40..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rx.f +++ /dev/null @@ -1,8 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/link_interface.sv -@RESOURCES@/link_package.sv --F @RTL@/Tx/filelist.f --F @RTL@/Rx/filelist.f -@RESOURCES@/rx_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rx_bad_credit.f b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rx_bad_credit.f deleted file mode 100644 index b54ec2380..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rx_bad_credit.f +++ /dev/null @@ -1,9 +0,0 @@ -+define+CHI_RX_OVERFLOW -+define+CHI_CREDIT_TB=rx_bad_credit_tb -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/credit_interface.sv -@RESOURCES@/credit_package.sv --F @RTL@/Rx/filelist.f -@RESOURCES@/credit_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rx_bad_credit.toml b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rx_bad_credit.toml deleted file mode 100644 index f05649ac4..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rx_bad_credit.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "CHI RX flit without granted credit" diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rx_tb.sv b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rx_tb.sv deleted file mode 100644 index e52638c74..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rx_tb.sv +++ /dev/null @@ -1,38 +0,0 @@ -module rx_tb; - import uvm_pkg::*; - import chi_link_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - chi_link_if source_if (clock); - wire flitpend, flitv, lcrdv; - wire [31:0] flit; - Tx transmitter ( - .clock(clock), - .reset(source_if.reset), - .io_active(source_if.active), - .io_in_valid(source_if.valid), - .io_in_ready(source_if.ready), - .io_in_bits(source_if.data), - .io_link_flitpend(flitpend), - .io_link_flitv(flitv), - .io_link_flit(flit), - .io_link_lcrdv(lcrdv), - .io_credits(source_if.credits) - ); - Rx dut ( - .clock(clock), - .reset(source_if.reset), - .io_active(source_if.active), - .io_link_flitpend(flitpend), - .io_link_flitv(flitv), - .io_link_flit(flit), - .io_link_lcrdv(lcrdv), - .io_out_valid(source_if.out_valid), - .io_out_ready(source_if.out_ready), - .io_out_bits(source_if.out_data) - ); - initial begin - uvm_config_db#(virtual chi_link_if)::set(null, "uvm_test_top", "vif", source_if); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram.el b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram.el deleted file mode 100644 index 3fd3dfb7a..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram.el +++ /dev/null @@ -1,971 +0,0 @@ -// Format Version: 2 -// Reviewed SN256 instance-specific diagnostics, legal-input constraints and source invariants. -// Preserve full functional payload, arbitrary valid IDs, idle physical payload and reachable scheduling. - -CHECKSUM: "3483611062 3911945341" -INSTANCE: sram_tb.dut.node -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI request to wrong node" -Block 2 "823506465" "if (1)" -Block 3 "1562418237" "$error(\"Assertion failed: CHI request to wrong node\n at MemoryNode.scala:64 assert(r.tgtId === nodeId.U, \\"CHI request to wrong node\\")\n\");" -Block 5 "823506465" "if (1)" -Block 6 "4007593502" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Unsupported CHI request opcode" -Block 10 "3060840224" "if (1)" -Block 11 "1238176658" "$error(\"Assertion failed: Unsupported CHI request opcode\n at MemoryNode.scala:65 assert(\n\");" -Block 13 "3060840224" "if (1)" -Block 14 "3791457536" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI node requires aligned 64-byte access" -Block 18 "3302013798" "if (1)" -Block 19 "3011646935" "$error(\"Assertion failed: CHI node requires aligned 64-byte access\n at MemoryNode.scala:70 assert(r.size === 6.U && r.addr(5, 0) === 0.U, \\"CHI node requires aligned 64-byte access\\")\n\");" -Block 21 "3302013798" "if (1)" -Block 22 "3453092984" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Unsupported CHI request attributes" -Block 26 "3756387644" "if (1)" -Block 27 "1839675237" "$error(\"Assertion failed: Unsupported CHI request attributes\n at MemoryNode.scala:71 assert(\n\");" -Block 29 "3756387644" "if (1)" -Block 30 "2962064657" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Device memory is outside this CHI node profile" -Block 34 "3061807544" "if (1)" -Block 35 "2656390042" "$error(\"Assertion failed: Device memory is outside this CHI node profile\n at MemoryNode.scala:78 assert(!r.memAttr(1), \\"Device memory is outside this CHI node profile\\")\n\");" -Block 37 "3061807544" "if (1)" -Block 38 "388673725" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI source reused a live TxnID" -Block 42 "2059430060" "if (1)" -Block 43 "1492632624" "$error(\"Assertion failed: CHI source reused a live TxnID\n at MemoryNode.scala:80 assert(\n\");" -Block 45 "2059430060" "if (1)" -Block 46 "2414433873" "$fatal;" -Block 50 "974596391" "if (1)" -Block 51 "1474737040" "$error(\"Assertion failed: CHI source reused a live TxnID\n at MemoryNode.scala:80 assert(\n\");" -Block 53 "974596391" "if (1)" -Block 54 "2163745265" "$fatal;" -Block 58 "4218350522" "if (1)" -Block 59 "1188375408" "$error(\"Assertion failed: CHI source reused a live TxnID\n at MemoryNode.scala:80 assert(\n\");" -Block 61 "4218350522" "if (1)" -Block 62 "2446047505" "$fatal;" -Block 66 "3149507121" "if (1)" -Block 67 "1237605584" "$error(\"Assertion failed: CHI source reused a live TxnID\n at MemoryNode.scala:80 assert(\n\");" -Block 69 "3149507121" "if (1)" -Block 70 "2665139889" "$fatal;" -Block 74 "3221991744" "if (1)" -Block 75 "1864251192" "$error(\"Assertion failed: CHI source reused a live TxnID\n at MemoryNode.scala:80 assert(\n\");" -Block 77 "3221991744" "if (1)" -Block 78 "3087050073" "$fatal;" -Block 82 "2161963211" "if (1)" -Block 83 "1611612312" "$error(\"Assertion failed: CHI source reused a live TxnID\n at MemoryNode.scala:80 assert(\n\");" -Block 85 "2161963211" "if (1)" -Block 86 "3071398649" "$fatal;" -Block 90 "1101345366" "if (1)" -Block 91 "1899812984" "$error(\"Assertion failed: CHI source reused a live TxnID\n at MemoryNode.scala:80 assert(\n\");" -Block 93 "1101345366" "if (1)" -Block 94 "2787265049" "$fatal;" -Block 98 "24277981" "if (1)" -Block 99 "2116923352" "$error(\"Assertion failed: CHI source reused a live TxnID\n at MemoryNode.scala:80 assert(\n\");" -Block 101 "24277981" "if (1)" -Block 102 "2838707641" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Unsupported CHI data opcode" -Block 106 "1895873786" "if (1)" -Block 107 "1082696673" "$error(\"Assertion failed: Unsupported CHI data opcode\n at MemoryNode.scala:97 assert(d.opcode === Opcode.NonCopyBackWriteData.U, \\"Unsupported CHI data opcode\\")\n\");" -Block 109 "1895873786" "if (1)" -Block 110 "4271299145" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Invalid CHI write destination/DBID" -Block 114 "2750264503" "if (1)" -Block 115 "3419488649" "$error(\"Assertion failed: Invalid CHI write destination/DBID\n at MemoryNode.scala:98 assert(d.tgtId === nodeId.U && d.txnId < slots.U, \\"Invalid CHI write destination/DBID\\")\n\");" -Block 117 "2750264503" "if (1)" -Block 118 "1909902123" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Errored write data is outside this CHI node profile" -Block 122 "3516907868" "if (1)" -Block 123 "3490684787" "$error(\"Assertion failed: Errored write data is outside this CHI node profile\n at MemoryNode.scala:99 assert(d.respErr === 0.U, \\"Errored write data is outside this CHI node profile\\")\n\");" -Block 125 "3516907868" "if (1)" -Block 126 "1011725012" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Unsupported CHI data attributes" -Block 130 "2818579679" "if (1)" -Block 131 "2858884155" "$error(\"Assertion failed: Unsupported CHI data attributes\n at MemoryNode.scala:100 assert(\n\");" -Block 133 "2818579679" "if (1)" -Block 134 "3581507328" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Invalid CHI DataID for data width" -Block 138 "3427410875" "if (1)" -Block 139 "401734482" "$error(\"Assertion failed: Invalid CHI DataID for data width\n at MemoryNode.scala:108 assert(legalId, \\"Invalid CHI DataID for data width\\")\n\");" -Block 141 "3427410875" "if (1)" -Block 142 "3587222114" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI write data before DBID or after completion" -Block 146 "974379480" "if (1)" -Block 147 "534939643" "$error(\"Assertion failed: CHI write data before DBID or after completion\n at MemoryNode.scala:110 assert(state(index) === collect, \\"CHI write data before DBID or after completion\\")\n\");" -Block 149 "974379480" "if (1)" -Block 150 "1661163309" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI write data from wrong requester" -Block 154 "8984338" "if (1)" -Block 155 "548569855" "$error(\"Assertion failed: CHI write data from wrong requester\n at MemoryNode.scala:111 assert(d.srcId === requests(index).srcId, \\"CHI write data from wrong requester\\")\n\");" -Block 157 "8984338" "if (1)" -Block 158 "3172387341" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Duplicate CHI DataID" -Block 162 "1067614932" "if (1)" -Block 163 "3638277198" "$error(\"Assertion failed: Duplicate CHI DataID\n at MemoryNode.scala:112 assert(!(received(index) & UIntToOH(beat, p.beatsPerLine)).orR, \\"Duplicate CHI DataID\\")\n\");" -Block 165 "1067614932" "if (1)" -Block 166 "382543761" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: WriteNoSnpFull requires all byte enables" -Block 170 "1678828384" "if (1)" -Block 171 "848044923" "$error(\"Assertion failed: WriteNoSnpFull requires all byte enables\n at MemoryNode.scala:114 assert(d.be.andR, \\"WriteNoSnpFull requires all byte enables\\")\n\");" -Block 173 "1678828384" "if (1)" -Block 174 "1785821791" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Invalid memory response ID" -Block 178 "841885701" "if (1)" -Block 179 "1786497890" "$error(\"Assertion failed: Invalid memory response ID\n at MemoryNode.scala:145 assert(r.id < slots.U, \\"Invalid memory response ID\\")\n\");" -Block 181 "841885701" "if (1)" -Block 182 "4251582268" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Unexpected memory completion" -Block 186 "2406142010" "if (1)" -Block 187 "1700436678" "$error(\"Assertion failed: Unexpected memory completion\n at MemoryNode.scala:147 assert(state(index) === waitMemory, \\"Unexpected memory completion\\")\n\");" -Block 189 "2406142010" "if (1)" -Block 190 "2380214045" "$fatal;" - -CHECKSUM: "1542839461 95260501" -INSTANCE: sram_tb.dut.rxReq -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX flit on inactive link" -Block 2 "1857595103" "if (1)" -Block 3 "971991619" "$error(\"Assertion failed: CHI RX flit on inactive link\n at Rx.scala:64 assert(io.active, \\"CHI RX flit on inactive link\\")\n\");" -Block 5 "1857595103" "if (1)" -Block 6 "3528662196" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI FLITPEND must precede FLITV" -Block 10 "3823759045" "if (1)" -Block 11 "2663934422" "$error(\"Assertion failed: CHI FLITPEND must precede FLITV\n at Rx.scala:65 assert(previousPend, \\"CHI FLITPEND must precede FLITV\\")\n\");" -Block 13 "3823759045" "if (1)" -Block 14 "2045170107" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX flit without granted credit" -Block 18 "798018255" "if (1)" -Block 19 "74350349" "$error(\"Assertion failed: CHI RX flit without granted credit\n at Rx.scala:66 assert(advertised =/= 0.U, \\"CHI RX flit without granted credit\\")\n\");" -Block 21 "798018255" "if (1)" -Block 22 "2469915260" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX buffer overflow" -Block 26 "2435529851" "if (1)" -Block 27 "2348576011" "$error(\"Assertion failed: CHI RX buffer overflow\n at Rx.scala:67 assert(enqueueReady, \\"CHI RX buffer overflow\\")\n\");" -Block 29 "2435529851" "if (1)" -Block 30 "773203258" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX credit conservation" -Block 34 "3002688943" "if (1)" -Block 35 "2461347743" "$error(\"Assertion failed: CHI RX credit conservation\n at Rx.scala:69 assert(reserved <= depth.U, \\"CHI RX credit conservation\\")\n\");" -Block 37 "3002688943" "if (1)" -Block 38 "1033059400" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX requires coordinated reset to stop" -Block 42 "2653430091" "if (1)" -Block 43 "1798953395" "$error(\"Assertion failed: CHI RX requires coordinated reset to stop\n at Rx.scala:70 when(wasActive)(assert(io.active, \\"CHI RX requires coordinated reset to stop\\"))\n\");" -Block 45 "2653430091" "if (1)" -Block 46 "2212642507" "$fatal;" - -CHECKSUM: "1028417357 1379207481" -INSTANCE: sram_tb.dut.txRsp -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI TX credit overflow" -Block 2 "3115401253" "if (1)" -Block 3 "1708991190" "$error(\"Assertion failed: CHI TX credit overflow\n at Tx.scala:43 when(returned && !send)(assert(credits < maxCredits.U, \\"CHI TX credit overflow\\"))\n\");" -Block 5 "3115401253" "if (1)" -Block 6 "4063542534" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI TX credit underflow" -Block 10 "1503763571" "if (1)" -Block 11 "556320376" "$error(\"Assertion failed: CHI TX credit underflow\n at Tx.scala:44 when(send)(assert(credits =/= 0.U, \\"CHI TX credit underflow\\"))\n\");" -Block 13 "1503763571" "if (1)" -Block 14 "2166023300" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI TX requires coordinated reset to stop" -Block 18 "2653430091" "if (1)" -Block 19 "597017625" "$error(\"Assertion failed: CHI TX requires coordinated reset to stop\n at Tx.scala:45 when(wasActive)(assert(io.active, \\"CHI TX requires coordinated reset to stop\\"))\n\");" -Block 21 "2653430091" "if (1)" -Block 22 "2212642507" "$fatal;" - -CHECKSUM: "3712248894 3505880431" -INSTANCE: sram_tb.dut.rxDat -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX flit on inactive link" -Block 2 "1857595103" "if (1)" -Block 3 "971991619" "$error(\"Assertion failed: CHI RX flit on inactive link\n at Rx.scala:64 assert(io.active, \\"CHI RX flit on inactive link\\")\n\");" -Block 5 "1857595103" "if (1)" -Block 6 "3528662196" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI FLITPEND must precede FLITV" -Block 10 "3823759045" "if (1)" -Block 11 "2663934422" "$error(\"Assertion failed: CHI FLITPEND must precede FLITV\n at Rx.scala:65 assert(previousPend, \\"CHI FLITPEND must precede FLITV\\")\n\");" -Block 13 "3823759045" "if (1)" -Block 14 "2045170107" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX flit without granted credit" -Block 18 "798018255" "if (1)" -Block 19 "74350349" "$error(\"Assertion failed: CHI RX flit without granted credit\n at Rx.scala:66 assert(advertised =/= 0.U, \\"CHI RX flit without granted credit\\")\n\");" -Block 21 "798018255" "if (1)" -Block 22 "2469915260" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX buffer overflow" -Block 26 "2435529851" "if (1)" -Block 27 "2348576011" "$error(\"Assertion failed: CHI RX buffer overflow\n at Rx.scala:67 assert(enqueueReady, \\"CHI RX buffer overflow\\")\n\");" -Block 29 "2435529851" "if (1)" -Block 30 "773203258" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX credit conservation" -Block 34 "3002688943" "if (1)" -Block 35 "2461347743" "$error(\"Assertion failed: CHI RX credit conservation\n at Rx.scala:69 assert(reserved <= depth.U, \\"CHI RX credit conservation\\")\n\");" -Block 37 "3002688943" "if (1)" -Block 38 "1033059400" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX requires coordinated reset to stop" -Block 42 "2653430091" "if (1)" -Block 43 "1798953395" "$error(\"Assertion failed: CHI RX requires coordinated reset to stop\n at Rx.scala:70 when(wasActive)(assert(io.active, \\"CHI RX requires coordinated reset to stop\\"))\n\");" -Block 45 "2653430091" "if (1)" -Block 46 "2212642507" "$fatal;" - -CHECKSUM: "2770640837 940463411" -INSTANCE: sram_tb.dut.txDat -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI TX credit overflow" -Block 2 "3115401253" "if (1)" -Block 3 "1708991190" "$error(\"Assertion failed: CHI TX credit overflow\n at Tx.scala:43 when(returned && !send)(assert(credits < maxCredits.U, \\"CHI TX credit overflow\\"))\n\");" -Block 5 "3115401253" "if (1)" -Block 6 "4063542534" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI TX credit underflow" -Block 10 "1503763571" "if (1)" -Block 11 "556320376" "$error(\"Assertion failed: CHI TX credit underflow\n at Tx.scala:44 when(send)(assert(credits =/= 0.U, \\"CHI TX credit underflow\\"))\n\");" -Block 13 "1503763571" "if (1)" -Block 14 "2166023300" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI TX requires coordinated reset to stop" -Block 18 "2653430091" "if (1)" -Block 19 "597017625" "$error(\"Assertion failed: CHI TX requires coordinated reset to stop\n at Tx.scala:45 when(wasActive)(assert(io.active, \\"CHI TX requires coordinated reset to stop\\"))\n\");" -Block 21 "2653430091" "if (1)" -Block 22 "2212642507" "$fatal;" - -CHECKSUM: "2383780593 637638803" -INSTANCE: sram_tb.dut -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Runtime RX link deactivation is unsupported" -Block 2 "337229911" "if (1)" -Block 3 "2663757680" "$error(\"Assertion failed: Runtime RX link deactivation is unsupported\n at SramEndpoint.scala:47 when(rxAck)(assert(io.chi.rxLinkActiveReq, \\"Runtime RX link deactivation is unsupported\\"))\n\");" -Block 5 "337229911" "if (1)" -Block 6 "1648053033" "$fatal;" - -CHECKSUM: "3483611062 1539539244" -INSTANCE: sram_tb.dut.node -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI request to wrong node. Functional gates and assertion remain enabled." -Condition 5 "148920103" "(_GEN_6 & (io_req_bits_tgtId != 7'b1)) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Unsupported CHI request opcode. Functional gates and assertion remain enabled." -Condition 7 "2559978673" "(_GEN_6 & ( ~ (_state_T | (io_req_bits_opcode == 7'h1d) | (io_req_bits_opcode == 7'h1c)) )) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI node requires aligned 64-byte access. Functional gates and assertion remain enabled." -Condition 12 "3277521579" "(_GEN_6 & ( ~ ((io_req_bits_size == 6'h06) & (io_req_bits_addr[5:0] == 6'b0)) )) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Device memory is outside this CHI node profile. Functional gates and assertion remain enabled." -Condition 24 "1834020855" "(_GEN_6 & io_req_bits_memAttr[1]) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI source reused a live TxnID. Functional gates and assertion remain enabled." -Condition 25 "4152540323" "(_GEN_6 & ((|state_0)) & (requests_0_srcId == io_req_bits_srcId) & (requests_0_txnId == io_req_bits_txnId)) 1 -1" (5 "1111") -Condition 28 "2098488880" "(_GEN_6 & ((|state_1)) & (requests_1_srcId == io_req_bits_srcId) & (requests_1_txnId == io_req_bits_txnId)) 1 -1" (5 "1111") -Condition 31 "3865973810" "(_GEN_6 & ((|state_2)) & (requests_2_srcId == io_req_bits_srcId) & (requests_2_txnId == io_req_bits_txnId)) 1 -1" (5 "1111") -Condition 34 "1828245153" "(_GEN_6 & ((|state_3)) & (requests_3_srcId == io_req_bits_srcId) & (requests_3_txnId == io_req_bits_txnId)) 1 -1" (5 "1111") -Condition 37 "3510787249" "(_GEN_6 & ((|state_4)) & (requests_4_srcId == io_req_bits_srcId) & (requests_4_txnId == io_req_bits_txnId)) 1 -1" (5 "1111") -Condition 40 "1540654626" "(_GEN_6 & ((|state_5)) & (requests_5_srcId == io_req_bits_srcId) & (requests_5_txnId == io_req_bits_txnId)) 1 -1" (5 "1111") -Condition 43 "3232691232" "(_GEN_6 & ((|state_6)) & (requests_6_srcId == io_req_bits_srcId) & (requests_6_txnId == io_req_bits_txnId)) 1 -1" (5 "1111") -Condition 46 "1245195955" "(_GEN_6 & ((|state_7)) & (requests_7_srcId == io_req_bits_srcId) & (requests_7_txnId == io_req_bits_txnId)) 1 -1" (5 "1111") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Unsupported CHI data opcode. Functional gates and assertion remain enabled." -Condition 49 "1423762789" "(_GEN_7 & (io_rxDat_bits_opcode != 4'h3)) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Invalid CHI write destination/DBID. Functional gates and assertion remain enabled." -Condition 51 "3130223411" "(_GEN_7 & ( ~ ((io_rxDat_bits_tgtId == 7'b1) & _GEN_0) )) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Errored write data is outside this CHI node profile. Functional gates and assertion remain enabled." -Condition 55 "4258216982" "(_GEN_7 & ((|io_rxDat_bits_respErr))) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Invalid CHI DataID for data width. Functional gates and assertion remain enabled." -Condition 63 "1066065420" "(_GEN_7 & ({(io_rxDat_bits_dataId == 2'h2), (io_rxDat_bits_dataId == 2'b0)} == 2'b0)) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI write data before DBID or after completion. Functional gates and assertion remain enabled." -Condition 65 "1983754605" "(_GEN_10 & (_GEN_9[io_rxDat_bits_txnId[2:0]] != 3'h2)) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI write data from wrong requester. Functional gates and assertion remain enabled." -Condition 67 "1394082298" "(_GEN_10 & (io_rxDat_bits_srcId != _GEN_11[io_rxDat_bits_txnId[2:0]])) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Duplicate CHI DataID. Functional gates and assertion remain enabled." -Condition 69 "3572934112" "(_GEN_10 & ((|(_GEN_4 & (2'b1 << _GEN_2))))) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: WriteNoSnpFull requires all byte enables. Functional gates and assertion remain enabled." -Condition 70 "3806849419" "(_GEN_8 & (_GEN_1[io_rxDat_bits_txnId[2:0]] == 7'h1d) & ((~reset)) & (io_rxDat_bits_be != 32'hffffffff)) 1 -1" (5 "1111") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Invalid memory response ID. Functional gates and assertion remain enabled." -Condition 73 "3254488011" "(io_memoryResp_valid & ((~reset)) & ((~_GEN_5))) 1 -1" (4 "111") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Unexpected memory completion. Functional gates and assertion remain enabled." -Condition 74 "1495455793" "(io_memoryResp_valid & _GEN_5 & ((~reset)) & (_GEN_9[io_memoryResp_bits_id[2:0]] != 3'h4)) 1 -1" (5 "1111") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Unsupported CHI request attributes. Functional gates and assertion remain enabled." -Condition 17 "815973446" "(_GEN_6 & ( ~ ((io_req_bits_order == 2'b0) & ((~io_req_bits_expCompAck)) & ((~io_req_bits_exclSnoopMe)) & ((~io_req_bits_snpAttr)) & ((~io_req_bits_stashNidValidEndian)) & (io_req_bits_pCrdType == 4'b0) & ((~io_req_bits_multiReq)) & (io_req_bits_pas == 3'b0) & (io_req_bits_tagOp == 2'b0)) )) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Unsupported CHI data attributes. Functional gates and assertion remain enabled." -Condition 56 "3320427110" "(_GEN_7 & ( ~ ((io_rxDat_bits_tagOp == 2'b0) & (io_rxDat_bits_tag == 8'b0) & (io_rxDat_bits_tagUpdate == 2'b0) & ((~io_rxDat_bits_dataPull)) & (io_rxDat_bits_numDat == 2'b0) & ((~io_rxDat_bits_replicate))) )) 1 -1" (3 "11") -ANNOTATION: "MemoryNode.scala lines98/109: accepted NonCopyBackWriteData must use an allocated local DBID0..7. The full-width id<8 check is false only for an illegal valid transfer; other vectors remain measured." -Condition 3 "3124372553" "(io_rxDat_valid & _GEN_0) 1 -1" (2 "10") -Condition 98 "1826432495" "(io_rxDat_valid & _GEN_0 & _GEN_20 & ((~io_rxDat_bits_dataId[1]))) 1 -1" (2 "1011") -Condition 99 "2095748577" "(io_rxDat_valid & _GEN_0 & _GEN_20 & io_rxDat_bits_dataId[1]) 1 -1" (2 "1011") -Condition 101 "2689989377" "(io_rxDat_valid & _GEN_0 & _GEN_23 & ((~io_rxDat_bits_dataId[1]))) 1 -1" (2 "1011") -Condition 102 "2539887920" "(io_rxDat_valid & _GEN_0 & _GEN_23 & io_rxDat_bits_dataId[1]) 1 -1" (2 "1011") -Condition 104 "26785311" "(io_rxDat_valid & _GEN_0 & _GEN_26 & ((~io_rxDat_bits_dataId[1]))) 1 -1" (2 "1011") -Condition 105 "1935181793" "(io_rxDat_valid & _GEN_0 & _GEN_26 & io_rxDat_bits_dataId[1]) 1 -1" (2 "1011") -Condition 107 "3885985187" "(io_rxDat_valid & _GEN_0 & _GEN_29 & ((~io_rxDat_bits_dataId[1]))) 1 -1" (2 "1011") -Condition 108 "4186588712" "(io_rxDat_valid & _GEN_0 & _GEN_29 & io_rxDat_bits_dataId[1]) 1 -1" (2 "1011") -Condition 110 "2718711594" "(io_rxDat_valid & _GEN_0 & _GEN_32 & ((~io_rxDat_bits_dataId[1]))) 1 -1" (2 "1011") -Condition 111 "769774406" "(io_rxDat_valid & _GEN_0 & _GEN_32 & io_rxDat_bits_dataId[1]) 1 -1" (2 "1011") -Condition 113 "3872915812" "(io_rxDat_valid & _GEN_0 & _GEN_35 & ((~io_rxDat_bits_dataId[1]))) 1 -1" (2 "1011") -Condition 114 "1054611865" "(io_rxDat_valid & _GEN_0 & _GEN_35 & io_rxDat_bits_dataId[1]) 1 -1" (2 "1011") -Condition 116 "3899578032" "(io_rxDat_valid & _GEN_0 & _GEN_38 & ((~io_rxDat_bits_dataId[1]))) 1 -1" (2 "1011") -Condition 117 "3633996258" "(io_rxDat_valid & _GEN_0 & _GEN_38 & io_rxDat_bits_dataId[1]) 1 -1" (2 "1011") -Condition 118 "265306532" "(io_rxDat_valid & _GEN_0 & ((&io_rxDat_bits_txnId[2:0])) & ((~io_rxDat_bits_dataId[1]))) 1 -1" (2 "1011") -Condition 119 "2039696548" "(io_rxDat_valid & _GEN_0 & ((&io_rxDat_bits_txnId[2:0])) & io_rxDat_bits_dataId[1]) 1 -1" (2 "1011") -Condition 149 "3967989931" "(io_rxDat_valid & _GEN_0 & ((&nextReceived)) & _GEN_20) 1 -1" (2 "1011") -Condition 154 "298828490" "(io_rxDat_valid & _GEN_0 & ((&nextReceived)) & _GEN_23) 1 -1" (2 "1011") -Condition 159 "400676024" "(io_rxDat_valid & _GEN_0 & ((&nextReceived)) & _GEN_26) 1 -1" (2 "1011") -Condition 164 "2421754122" "(io_rxDat_valid & _GEN_0 & ((&nextReceived)) & _GEN_29) 1 -1" (2 "1011") -Condition 169 "563353181" "(io_rxDat_valid & _GEN_0 & ((&nextReceived)) & _GEN_32) 1 -1" (2 "1011") -Condition 174 "3818989856" "(io_rxDat_valid & _GEN_0 & ((&nextReceived)) & _GEN_35) 1 -1" (2 "1011") -Condition 179 "2253940498" "(io_rxDat_valid & _GEN_0 & ((&nextReceived)) & _GEN_38) 1 -1" (2 "1011") -Condition 184 "2397255552" "(io_rxDat_valid & _GEN_0 & ((&nextReceived)) & ((&io_rxDat_bits_txnId[2:0]))) 1 -1" (2 "1011") -Condition 185 "1143475756" "(io_rxDat_valid & _GEN_0 & _GEN_20) 1 -1" (2 "101") -Condition 186 "908938830" "(io_rxDat_valid & _GEN_0 & _GEN_23) 1 -1" (2 "101") -Condition 187 "3991860434" "(io_rxDat_valid & _GEN_0 & _GEN_26) 1 -1" (2 "101") -Condition 188 "3921519957" "(io_rxDat_valid & _GEN_0 & _GEN_29) 1 -1" (2 "101") -Condition 189 "3254585959" "(io_rxDat_valid & _GEN_0 & _GEN_32) 1 -1" (2 "101") -Condition 190 "2792948640" "(io_rxDat_valid & _GEN_0 & _GEN_35) 1 -1" (2 "101") -Condition 191 "510517628" "(io_rxDat_valid & _GEN_0 & _GEN_38) 1 -1" (2 "101") -Condition 192 "2930983340" "(io_rxDat_valid & _GEN_0 & ((&io_rxDat_bits_txnId[2:0]))) 1 -1" (2 "101") -ANNOTATION: "LineSram asserts response valid only after capturing a memArb request with literal local ID0..7; a valid response therefore has id<8, including a coincident reset. Uninitialized invalid response ID remains measured." -Condition 73 "3254488011" "(io_memoryResp_valid & ((~reset)) & ((~_GEN_5))) 1 -1" (2 "101") -ANNOTATION: "LineSram echoes its request ID; MemoryNode memArb supplies only literal local IDs0..7. Therefore a valid backend completion with id>=8 is structurally unreachable." -Condition 74 "1495455793" "(io_memoryResp_valid & _GEN_5 & ((~reset)) & (_GEN_9[io_memoryResp_bits_id[2:0]] != 3'h4)) 1 -1" (2 "1011") -Condition 120 "205294699" "(io_memoryResp_valid & _GEN_5 & (io_memoryResp_bits_id[2:0] == 3'b0)) 1 -1" (2 "101") -Condition 122 "2537288514" "(io_memoryResp_valid & _GEN_5 & (io_memoryResp_bits_id[2:0] == 3'b1)) 1 -1" (2 "101") -Condition 124 "3289281793" "(io_memoryResp_valid & _GEN_5 & (io_memoryResp_bits_id[2:0] == 3'h2)) 1 -1" (2 "101") -Condition 126 "2725440915" "(io_memoryResp_valid & _GEN_5 & (io_memoryResp_bits_id[2:0] == 3'h3)) 1 -1" (2 "101") -Condition 128 "1866493802" "(io_memoryResp_valid & _GEN_5 & (io_memoryResp_bits_id[2:0] == 3'h4)) 1 -1" (2 "101") -Condition 130 "3072721141" "(io_memoryResp_valid & _GEN_5 & (io_memoryResp_bits_id[2:0] == 3'h5)) 1 -1" (2 "101") -Condition 132 "506965470" "(io_memoryResp_valid & _GEN_5 & (io_memoryResp_bits_id[2:0] == 3'h6)) 1 -1" (2 "101") -Condition 134 "3797793362" "(io_memoryResp_valid & _GEN_5 & ((&io_memoryResp_bits_id[2:0]))) 1 -1" (2 "101") - -CHECKSUM: "1542839461 2138553741" -INSTANCE: sram_tb.dut.rxReq -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI RX flit on inactive link. Functional gates and assertion remain enabled." -Condition 2 "3562691098" "(_GEN_0 & ((~io_active))) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI FLITPEND must precede FLITV. Functional gates and assertion remain enabled." -Condition 3 "2541789932" "(_GEN_0 & ((~previousPend))) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI RX flit without granted credit. Functional gates and assertion remain enabled." -Condition 4 "959243910" "(_GEN_0 & (advertised == 3'b0)) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI RX buffer overflow. Functional gates and assertion remain enabled." -Condition 6 "183470145" "(_GEN_0 & ((~enqueueReady))) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI RX credit conservation. Functional gates and assertion remain enabled." -Condition 7 "311107905" "(((~reset)) & (reserved > 4'h4)) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI RX requires coordinated reset to stop. Functional gates and assertion remain enabled." -Condition 8 "3706255149" "(wasActive & ((~reset)) & ((~io_active))) 1 -1" (4 "111") -ANNOTATION: "Depth4 invariant reserved=advertised+occupancy<=4, including coordinated reset boundary." -Condition 7 "311107905" "(((~reset)) & (reserved > 4'h4)) 1 -1" (1 "01") -ANNOTATION: "A legal arriving flit owns advertised credit>0; advertised+occupancy<=4 implies occupancy<4 and enqueueReady. This functional vector is unreachable." -Condition 9 "241394234" "(io_link_flitv & enqueueReady) 1 -1" (2 "10") -ANNOTATION: "SramEndpoint rxAck/txActive and child wasActive all synchronously reset to0. In this always-on profile active remains1 after activation until reset; before the first reset edge both are1, after it both are0. Therefore wasActive1 with active0 cannot coincide even with reset asserted." -Condition 8 "3706255149" "(wasActive & ((~reset)) & ((~io_active))) 1 -1" (2 "101") - -CHECKSUM: "1028417357 1909474316" -INSTANCE: sram_tb.dut.txRsp -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI TX credit overflow. Functional gates and assertion remain enabled." -Condition 1 "3175300005" "(returned & ((~send)) & ((~reset)) & ((&credits))) 1 -1" (5 "1111") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI TX credit underflow. Functional gates and assertion remain enabled." -Condition 2 "3481575346" "(send & ((~reset)) & ((~(|credits)))) 1 -1" (4 "111") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI TX requires coordinated reset to stop. Functional gates and assertion remain enabled." -Condition 3 "3706255149" "(wasActive & ((~reset)) & ((~io_active))) 1 -1" (4 "111") -ANNOTATION: "SN Tx/15-credit peer: credit=15 means all 15 credits are available; another returned credit cannot simultaneously be pending. This vector would violate peer credit conservation." -Condition 1 "3175300005" "(returned & ((~send)) & ((~reset)) & ((&credits))) 1 -1" (2 "1011") -ANNOTATION: "SN Tx/15-credit peer: no returned credit can coexist with all 15 credits available, including the coordinated reset boundary." -Condition 1 "3175300005" "(returned & ((~send)) & ((~reset)) & ((&credits))) 1 -1" (3 "1101") -ANNOTATION: "Structural invariant: send requires ready, and ready requires nonzero credits; send with zero credits is impossible even on reset." -Condition 2 "3481575346" "(send & ((~reset)) & ((~(|credits)))) 1 -1" (2 "101") -ANNOTATION: "Always-on SramEndpoint resets txActive and Tx credits together; peer credits start only after link activation. With no runtime deactivation, inactive implies credits0." -Condition 7 "1112456050" "(io_active & ((|credits))) 1 -1" (1 "01") -ANNOTATION: "SramEndpoint rxAck/txActive and child wasActive all synchronously reset to0. In this always-on profile active remains1 after activation until reset; before the first reset edge both are1, after it both are0. Therefore wasActive1 with active0 cannot coincide even with reset asserted." -Condition 3 "3706255149" "(wasActive & ((~reset)) & ((~io_active))) 1 -1" (2 "101") - -CHECKSUM: "3712248894 2941452375" -INSTANCE: sram_tb.dut.rxDat -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI RX flit on inactive link. Functional gates and assertion remain enabled." -Condition 2 "3562691098" "(_GEN_0 & ((~io_active))) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI FLITPEND must precede FLITV. Functional gates and assertion remain enabled." -Condition 3 "2541789932" "(_GEN_0 & ((~previousPend))) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI RX flit without granted credit. Functional gates and assertion remain enabled." -Condition 4 "959243910" "(_GEN_0 & (advertised == 3'b0)) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI RX buffer overflow. Functional gates and assertion remain enabled." -Condition 6 "183470145" "(_GEN_0 & ((~enqueueReady))) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI RX credit conservation. Functional gates and assertion remain enabled." -Condition 7 "311107905" "(((~reset)) & (reserved > 4'h4)) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI RX requires coordinated reset to stop. Functional gates and assertion remain enabled." -Condition 8 "3706255149" "(wasActive & ((~reset)) & ((~io_active))) 1 -1" (4 "111") -ANNOTATION: "Depth4 invariant reserved=advertised+occupancy<=4, including coordinated reset boundary." -Condition 7 "311107905" "(((~reset)) & (reserved > 4'h4)) 1 -1" (1 "01") -ANNOTATION: "A legal arriving flit owns advertised credit>0; advertised+occupancy<=4 implies occupancy<4 and enqueueReady. This functional vector is unreachable." -Condition 9 "241394234" "(io_link_flitv & enqueueReady) 1 -1" (2 "10") -ANNOTATION: "MemoryNode ties rxDat.ready=true. From reset occupancy0, every occupied entry drains and at most one flit arrives per cycle, so occupancy<=1, occupancy[2]=0, enqueueReady=true. This exact condition vector contradicts that invariant." -Condition 6 "183470145" "(_GEN_0 & ((~enqueueReady))) 1 -1" (1 "01") -Condition 21 "1968069542" "(((~occupancy[2])) | ((|occupancy))) 1 -1" (1 "00") -Condition 21 "1968069542" "(((~occupancy[2])) | ((|occupancy))) 1 -1" (2 "01") -ANNOTATION: "SramEndpoint rxAck/txActive and child wasActive all synchronously reset to0. In this always-on profile active remains1 after activation until reset; before the first reset edge both are1, after it both are0. Therefore wasActive1 with active0 cannot coincide even with reset asserted." -Condition 8 "3706255149" "(wasActive & ((~reset)) & ((~io_active))) 1 -1" (2 "101") - -CHECKSUM: "2770640837 1909474316" -INSTANCE: sram_tb.dut.txDat -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI TX credit overflow. Functional gates and assertion remain enabled." -Condition 1 "3175300005" "(returned & ((~send)) & ((~reset)) & ((&credits))) 1 -1" (5 "1111") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI TX credit underflow. Functional gates and assertion remain enabled." -Condition 2 "3481575346" "(send & ((~reset)) & ((~(|credits)))) 1 -1" (4 "111") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI TX requires coordinated reset to stop. Functional gates and assertion remain enabled." -Condition 3 "3706255149" "(wasActive & ((~reset)) & ((~io_active))) 1 -1" (4 "111") -ANNOTATION: "SN Tx/15-credit peer: credit=15 means all 15 credits are available; another returned credit cannot simultaneously be pending. This vector would violate peer credit conservation." -Condition 1 "3175300005" "(returned & ((~send)) & ((~reset)) & ((&credits))) 1 -1" (2 "1011") -ANNOTATION: "SN Tx/15-credit peer: no returned credit can coexist with all 15 credits available, including the coordinated reset boundary." -Condition 1 "3175300005" "(returned & ((~send)) & ((~reset)) & ((&credits))) 1 -1" (3 "1101") -ANNOTATION: "Structural invariant: send requires ready, and ready requires nonzero credits; send with zero credits is impossible even on reset." -Condition 2 "3481575346" "(send & ((~reset)) & ((~(|credits)))) 1 -1" (2 "101") -ANNOTATION: "Always-on SramEndpoint resets txActive and Tx credits together; peer credits start only after link activation. With no runtime deactivation, inactive implies credits0." -Condition 7 "1112456050" "(io_active & ((|credits))) 1 -1" (1 "01") -ANNOTATION: "SramEndpoint rxAck/txActive and child wasActive all synchronously reset to0. In this always-on profile active remains1 after activation until reset; before the first reset edge both are1, after it both are0. Therefore wasActive1 with active0 cannot coincide even with reset asserted." -Condition 3 "3706255149" "(wasActive & ((~reset)) & ((~io_active))) 1 -1" (2 "101") - -CHECKSUM: "2383780593 3054282342" -INSTANCE: sram_tb.dut -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Runtime RX link deactivation is unsupported. Functional gates and assertion remain enabled." -Condition 1 "929193424" "(rxAck & ((~reset)) & ((~io_chi_rxLinkActiveReq))) 1 -1" (4 "111") - -CHECKSUM: "3483611062 488357000" -INSTANCE: sram_tb.dut.node -ANNOTATION: "MemoryNode.scala source assignments and asserted SN profile: NodeID1; eight local DBIDs; aligned64B ReadNoSnp/WriteNoSnpFull/Ptl; no ordering, exclusive, snoop, tags, PAS or retry credit; incoming write DAT opcode3, clean error, local slot TxnID0..7; outgoing Comp/DBIDResp and CompData with zero optional attributes. Read BE is inapplicable (IssueH B13.9) and emitted zero. RespErr is OK0/NDERR3: neither error bit is excluded. Arbitrary routing, source, return and request TxnIDs and all payload remain measured." -Toggle io_req_bits_tgtId [6:0] "net io_req_bits_tgtId[6:0]" -Toggle io_req_bits_stashNidValidEndian "net io_req_bits_stashNidValidEndian" -Toggle io_req_bits_opcode [2:1] "net io_req_bits_opcode[6:0]" -Toggle io_req_bits_opcode [6:5] "net io_req_bits_opcode[6:0]" -Toggle io_req_bits_multiReq "net io_req_bits_multiReq" -Toggle io_req_bits_size [5:0] "net io_req_bits_size[5:0]" -Toggle io_req_bits_addr [5:0] "net io_req_bits_addr[43:0]" -Toggle io_req_bits_pas [2:0] "net io_req_bits_pas[2:0]" -Toggle io_req_bits_order [1:0] "net io_req_bits_order[1:0]" -Toggle io_req_bits_pCrdType [3:0] "net io_req_bits_pCrdType[3:0]" -Toggle io_req_bits_memAttr [1] "net io_req_bits_memAttr[3:0]" -Toggle io_req_bits_snpAttr "net io_req_bits_snpAttr" -Toggle io_req_bits_exclSnoopMe "net io_req_bits_exclSnoopMe" -Toggle io_req_bits_expCompAck "net io_req_bits_expCompAck" -Toggle io_req_bits_tagOp [1:0] "net io_req_bits_tagOp[1:0]" -Toggle io_rxDat_bits_tgtId [6:0] "net io_rxDat_bits_tgtId[6:0]" -Toggle io_rxDat_bits_txnId [11:3] "net io_rxDat_bits_txnId[11:0]" -Toggle io_rxDat_bits_opcode [3:0] "net io_rxDat_bits_opcode[3:0]" -Toggle io_rxDat_bits_respErr [1:0] "net io_rxDat_bits_respErr[1:0]" -Toggle io_rxDat_bits_dataPull "net io_rxDat_bits_dataPull" -Toggle io_rxDat_bits_dataId [0] "net io_rxDat_bits_dataId[1:0]" -Toggle io_rxDat_bits_tagOp [1:0] "net io_rxDat_bits_tagOp[1:0]" -Toggle io_rxDat_bits_tag [7:0] "net io_rxDat_bits_tag[7:0]" -Toggle io_rxDat_bits_tagUpdate [1:0] "net io_rxDat_bits_tagUpdate[1:0]" -Toggle io_rxDat_bits_numDat [1:0] "net io_rxDat_bits_numDat[1:0]" -Toggle io_rxDat_bits_replicate "net io_rxDat_bits_replicate" -Toggle io_rsp_bits_opcode [0] "net io_rsp_bits_opcode[4:0]" -Toggle io_rsp_bits_opcode [4:2] "net io_rsp_bits_opcode[4:0]" -Toggle io_rsp_bits_dbid [11:3] "net io_rsp_bits_dbid[11:0]" -Toggle io_txDat_bits_dbid [15:12] "net io_txDat_bits_dbid[15:0]" -Toggle io_txDat_bits_dataId [0] "net io_txDat_bits_dataId[1:0]" -Toggle io_memoryReq_bits_id [11:3] "net io_memoryReq_bits_id[11:0]" -Toggle io_memoryReq_bits_addr [5:0] "net io_memoryReq_bits_addr[43:0]" -Toggle io_memoryResp_bits_id [11:3] "net io_memoryResp_bits_id[11:0]" -Toggle requests_0_opcode [2:1] "reg requests_0_opcode[6:0]" -Toggle requests_0_opcode [6:5] "reg requests_0_opcode[6:0]" -Toggle requests_0_addr [5:0] "reg requests_0_addr[43:0]" -Toggle requests_1_opcode [2:1] "reg requests_1_opcode[6:0]" -Toggle requests_1_opcode [6:5] "reg requests_1_opcode[6:0]" -Toggle requests_1_addr [5:0] "reg requests_1_addr[43:0]" -Toggle requests_2_opcode [2:1] "reg requests_2_opcode[6:0]" -Toggle requests_2_opcode [6:5] "reg requests_2_opcode[6:0]" -Toggle requests_2_addr [5:0] "reg requests_2_addr[43:0]" -Toggle requests_3_opcode [2:1] "reg requests_3_opcode[6:0]" -Toggle requests_3_opcode [6:5] "reg requests_3_opcode[6:0]" -Toggle requests_3_addr [5:0] "reg requests_3_addr[43:0]" -Toggle requests_4_opcode [2:1] "reg requests_4_opcode[6:0]" -Toggle requests_4_opcode [6:5] "reg requests_4_opcode[6:0]" -Toggle requests_4_addr [5:0] "reg requests_4_addr[43:0]" -Toggle requests_5_opcode [2:1] "reg requests_5_opcode[6:0]" -Toggle requests_5_opcode [6:5] "reg requests_5_opcode[6:0]" -Toggle requests_5_addr [5:0] "reg requests_5_addr[43:0]" -Toggle requests_6_opcode [2:1] "reg requests_6_opcode[6:0]" -Toggle requests_6_opcode [6:5] "reg requests_6_opcode[6:0]" -Toggle requests_6_addr [5:0] "reg requests_6_addr[43:0]" -Toggle requests_7_opcode [2:1] "reg requests_7_opcode[6:0]" -Toggle requests_7_opcode [6:5] "reg requests_7_opcode[6:0]" -Toggle requests_7_addr [5:0] "reg requests_7_addr[43:0]" -ANNOTATION: "256-bit line return has2beats; sent increments0->1->2 and clears on allocation, so bit2 is unreachable." -Toggle sent_0 [2] "reg sent_0[2:0]" -Toggle sent_1 [2] "reg sent_1[2:0]" -Toggle sent_2 [2] "reg sent_2[2:0]" -Toggle sent_3 [2] "reg sent_3[2:0]" -Toggle sent_4 [2] "reg sent_4[2:0]" -Toggle sent_5 [2] "reg sent_5[2:0]" -Toggle sent_6 [2] "reg sent_6[2:0]" -Toggle sent_7 [2] "reg sent_7[2:0]" -ANNOTATION: "MemoryNode.scala: DAT local DBID check is true for legal stored write DAT; backend response ID is below8 because LineSram echoes literal local slot IDs. Beat index is zero-extended DataID[1]. Stored request opcodes are4/28/29, whose bits2:1 and6:5 are fixed." -Toggle _GEN_1 [7:0][2:1] "net [7:0][6:0]_GEN_1" -Toggle _GEN_1 [7:0][6:5] "net [7:0][6:0]_GEN_1" -Toggle _GEN_2 [1] "net _GEN_2[1:0]" - -CHECKSUM: "1049582285 2762367578" -INSTANCE: sram_tb.dut.memory -ANNOTATION: "Backend receives only eight MemoryNode local slot IDs0..7 and returns the same ID; opaque original CHI TxnIDs remain full width in MemoryNode." -Toggle io_req_bits_id [11:3] "net io_req_bits_id[11:0]" -Toggle io_resp_bits_id [11:3] "net io_resp_bits_id[11:0]" -Toggle request_id [11:3] "reg request_id[11:0]" -ANNOTATION: "The connected MemoryNode asserts64B alignment before issuing backend requests." -Toggle io_req_bits_addr [5:0] "net io_req_bits_addr[43:0]" - -CHECKSUM: "4028718411 2205085749" -INSTANCE: sram_tb.dut.node.memArb -ANNOTATION: "MemoryNode.scala source assignments and asserted SN profile: NodeID1; eight local DBIDs; aligned64B ReadNoSnp/WriteNoSnpFull/Ptl; no ordering, exclusive, snoop, tags, PAS or retry credit; incoming write DAT opcode3, clean error, local slot TxnID0..7; outgoing Comp/DBIDResp and CompData with zero optional attributes. Read BE is inapplicable (IssueH B13.9) and emitted zero. RespErr is OK0/NDERR3: neither error bit is excluded. Arbitrary routing, source, return and request TxnIDs and all payload remain measured." -Toggle io_in_0_bits_addr [5:0] "net io_in_0_bits_addr[43:0]" -Toggle io_in_1_bits_addr [5:0] "net io_in_1_bits_addr[43:0]" -Toggle io_in_2_bits_addr [5:0] "net io_in_2_bits_addr[43:0]" -Toggle io_in_3_bits_addr [5:0] "net io_in_3_bits_addr[43:0]" -Toggle io_in_4_bits_addr [5:0] "net io_in_4_bits_addr[43:0]" -Toggle io_in_5_bits_addr [5:0] "net io_in_5_bits_addr[43:0]" -Toggle io_in_6_bits_addr [5:0] "net io_in_6_bits_addr[43:0]" -Toggle io_in_7_bits_addr [5:0] "net io_in_7_bits_addr[43:0]" -Toggle io_out_bits_id [11:3] "net io_out_bits_id[11:0]" -Toggle io_out_bits_addr [5:0] "net io_out_bits_addr[43:0]" -ANNOTATION: "MemoryNode.scala memArb input(i).id=i for i0..7: every bit of each array element is a compile-time literal, while selected output low3 ID bits remain measured. Address low6 bits inherit the accepted aligned64B request." -Toggle _GEN [7:0][11:0] "net [7:0][11:0]_GEN" -Toggle _GEN_1 [7:0][5:0] "net [7:0][43:0]_GEN_1" - -CHECKSUM: "4063230465 2261056641" -INSTANCE: sram_tb.dut.node.rspArb -ANNOTATION: "MemoryNode.scala source assignments and asserted SN profile: NodeID1; eight local DBIDs; aligned64B ReadNoSnp/WriteNoSnpFull/Ptl; no ordering, exclusive, snoop, tags, PAS or retry credit; incoming write DAT opcode3, clean error, local slot TxnID0..7; outgoing Comp/DBIDResp and CompData with zero optional attributes. Read BE is inapplicable (IssueH B13.9) and emitted zero. RespErr is OK0/NDERR3: neither error bit is excluded. Arbitrary routing, source, return and request TxnIDs and all payload remain measured." -Toggle io_in_0_bits_opcode [0] "net io_in_0_bits_opcode[4:0]" -Toggle io_in_0_bits_opcode [4:2] "net io_in_0_bits_opcode[4:0]" -Toggle io_in_1_bits_opcode [0] "net io_in_1_bits_opcode[4:0]" -Toggle io_in_1_bits_opcode [4:2] "net io_in_1_bits_opcode[4:0]" -Toggle io_in_2_bits_opcode [0] "net io_in_2_bits_opcode[4:0]" -Toggle io_in_2_bits_opcode [4:2] "net io_in_2_bits_opcode[4:0]" -Toggle io_in_3_bits_opcode [0] "net io_in_3_bits_opcode[4:0]" -Toggle io_in_3_bits_opcode [4:2] "net io_in_3_bits_opcode[4:0]" -Toggle io_in_4_bits_opcode [0] "net io_in_4_bits_opcode[4:0]" -Toggle io_in_4_bits_opcode [4:2] "net io_in_4_bits_opcode[4:0]" -Toggle io_in_5_bits_opcode [0] "net io_in_5_bits_opcode[4:0]" -Toggle io_in_5_bits_opcode [4:2] "net io_in_5_bits_opcode[4:0]" -Toggle io_in_6_bits_opcode [0] "net io_in_6_bits_opcode[4:0]" -Toggle io_in_6_bits_opcode [4:2] "net io_in_6_bits_opcode[4:0]" -Toggle io_in_7_bits_opcode [0] "net io_in_7_bits_opcode[4:0]" -Toggle io_in_7_bits_opcode [4:2] "net io_in_7_bits_opcode[4:0]" -Toggle io_out_bits_opcode [0] "net io_out_bits_opcode[4:0]" -Toggle io_out_bits_opcode [4:2] "net io_out_bits_opcode[4:0]" -Toggle io_out_bits_dbid [11:3] "net io_out_bits_dbid[11:0]" -ANNOTATION: "MemoryNode.scala rspArb input(i).dbid=i for i0..7: each array element is a compile-time literal. Input opcode is DBIDResp6 or Comp4, so only bit1 changes; selected output DBID low3bits remain measured." -Toggle _GEN [7:0][11:0] "net [7:0][11:0]_GEN" -Toggle _GEN_4 [7:0][0] "net [7:0][4:0]_GEN_4" -Toggle _GEN_4 [7:0][4:2] "net [7:0][4:0]_GEN_4" - -CHECKSUM: "1047165719 4281785704" -INSTANCE: sram_tb.dut.node.datArb -ANNOTATION: "MemoryNode.scala source assignments and asserted SN profile: NodeID1; eight local DBIDs; aligned64B ReadNoSnp/WriteNoSnpFull/Ptl; no ordering, exclusive, snoop, tags, PAS or retry credit; incoming write DAT opcode3, clean error, local slot TxnID0..7; outgoing Comp/DBIDResp and CompData with zero optional attributes. Read BE is inapplicable (IssueH B13.9) and emitted zero. RespErr is OK0/NDERR3: neither error bit is excluded. Arbitrary routing, source, return and request TxnIDs and all payload remain measured." -Toggle io_in_0_bits_dbid [15:12] "net io_in_0_bits_dbid[15:0]" -Toggle io_in_0_bits_dataId [0] "net io_in_0_bits_dataId[1:0]" -Toggle io_in_1_bits_dbid [15:12] "net io_in_1_bits_dbid[15:0]" -Toggle io_in_1_bits_dataId [0] "net io_in_1_bits_dataId[1:0]" -Toggle io_in_2_bits_dbid [15:12] "net io_in_2_bits_dbid[15:0]" -Toggle io_in_2_bits_dataId [0] "net io_in_2_bits_dataId[1:0]" -Toggle io_in_3_bits_dbid [15:12] "net io_in_3_bits_dbid[15:0]" -Toggle io_in_3_bits_dataId [0] "net io_in_3_bits_dataId[1:0]" -Toggle io_in_4_bits_dbid [15:12] "net io_in_4_bits_dbid[15:0]" -Toggle io_in_4_bits_dataId [0] "net io_in_4_bits_dataId[1:0]" -Toggle io_in_5_bits_dbid [15:12] "net io_in_5_bits_dbid[15:0]" -Toggle io_in_5_bits_dataId [0] "net io_in_5_bits_dataId[1:0]" -Toggle io_in_6_bits_dbid [15:12] "net io_in_6_bits_dbid[15:0]" -Toggle io_in_6_bits_dataId [0] "net io_in_6_bits_dataId[1:0]" -Toggle io_in_7_bits_dbid [15:12] "net io_in_7_bits_dbid[15:0]" -Toggle io_in_7_bits_dataId [0] "net io_in_7_bits_dataId[1:0]" -Toggle io_out_bits_dbid [15:12] "net io_out_bits_dbid[15:0]" -Toggle io_out_bits_dataId [0] "net io_out_bits_dataId[1:0]" -ANNOTATION: "MemoryNode.scala DAT dbid zero-extends the12-bit returnTxnId to16bits, and DataID is beat index shifted left1 at dataBits256. Only the extension and low DataID bit are excluded." -Toggle _GEN_5 [7:0][15:12] "net [7:0][15:0]_GEN_5" -Toggle _GEN_6 [7:0][0] "net [7:0][1:0]_GEN_6" - -CHECKSUM: "1542839461 3642783442" -INSTANCE: sram_tb.dut.rxReq -ANNOTATION: "MemoryNode.scala source assignments and asserted SN profile: NodeID1; eight local DBIDs; aligned64B ReadNoSnp/WriteNoSnpFull/Ptl; no ordering, exclusive, snoop, tags, PAS or retry credit; incoming write DAT opcode3, clean error, local slot TxnID0..7; outgoing Comp/DBIDResp and CompData with zero optional attributes. Read BE is inapplicable (IssueH B13.9) and emitted zero. RespErr is OK0/NDERR3: neither error bit is excluded. Arbitrary routing, source, return and request TxnIDs and all payload remain measured. Exact serialized field slices follow Flit.fieldsLSB; only missing transitions excluded." -Toggle 1to0 io_out_bits [10:4] "net io_out_bits[136:0]" -Toggle 1to0 io_out_bits [37] "net io_out_bits[136:0]" -Toggle 1to0 io_out_bits [52:51] "net io_out_bits[136:0]" -Toggle 1to0 io_out_bits [69:55] "net io_out_bits[136:0]" -Toggle 1to0 io_out_bits [110:108] "net io_out_bits[136:0]" -Toggle 1to0 io_out_bits [118:113] "net io_out_bits[136:0]" -Toggle 1to0 io_out_bits [120] "net io_out_bits[136:0]" -Toggle 1to0 io_out_bits [123] "net io_out_bits[136:0]" -Toggle 1to0 io_out_bits [135:132] "net io_out_bits[136:0]" -Toggle 0to1 io_out_bits [10:4] "net io_out_bits[136:0]" -Toggle 0to1 io_out_bits [37] "net io_out_bits[136:0]" -Toggle 0to1 io_out_bits [52:51] "net io_out_bits[136:0]" -Toggle 0to1 io_out_bits [69:55] "net io_out_bits[136:0]" -Toggle 0to1 io_out_bits [110:108] "net io_out_bits[136:0]" -Toggle 0to1 io_out_bits [118:113] "net io_out_bits[136:0]" -Toggle 0to1 io_out_bits [120] "net io_out_bits[136:0]" -Toggle 0to1 io_out_bits [123] "net io_out_bits[136:0]" -Toggle 0to1 io_out_bits [135:132] "net io_out_bits[136:0]" -Toggle 1to0 storage_0 [10:4] "reg storage_0[136:0]" -Toggle 1to0 storage_0 [37] "reg storage_0[136:0]" -Toggle 1to0 storage_0 [52:51] "reg storage_0[136:0]" -Toggle 1to0 storage_0 [69:55] "reg storage_0[136:0]" -Toggle 1to0 storage_0 [110:108] "reg storage_0[136:0]" -Toggle 1to0 storage_0 [118:113] "reg storage_0[136:0]" -Toggle 1to0 storage_0 [120] "reg storage_0[136:0]" -Toggle 1to0 storage_0 [123] "reg storage_0[136:0]" -Toggle 1to0 storage_0 [135:132] "reg storage_0[136:0]" -Toggle 0to1 storage_0 [10:4] "reg storage_0[136:0]" -Toggle 0to1 storage_0 [37] "reg storage_0[136:0]" -Toggle 0to1 storage_0 [52:51] "reg storage_0[136:0]" -Toggle 0to1 storage_0 [69:55] "reg storage_0[136:0]" -Toggle 0to1 storage_0 [110:108] "reg storage_0[136:0]" -Toggle 0to1 storage_0 [118:113] "reg storage_0[136:0]" -Toggle 0to1 storage_0 [120] "reg storage_0[136:0]" -Toggle 0to1 storage_0 [123] "reg storage_0[136:0]" -Toggle 0to1 storage_0 [135:132] "reg storage_0[136:0]" -Toggle 1to0 storage_1 [10:4] "reg storage_1[136:0]" -Toggle 1to0 storage_1 [37] "reg storage_1[136:0]" -Toggle 1to0 storage_1 [52:51] "reg storage_1[136:0]" -Toggle 1to0 storage_1 [69:55] "reg storage_1[136:0]" -Toggle 1to0 storage_1 [110:108] "reg storage_1[136:0]" -Toggle 1to0 storage_1 [118:113] "reg storage_1[136:0]" -Toggle 1to0 storage_1 [120] "reg storage_1[136:0]" -Toggle 1to0 storage_1 [123] "reg storage_1[136:0]" -Toggle 1to0 storage_1 [135:132] "reg storage_1[136:0]" -Toggle 0to1 storage_1 [10:4] "reg storage_1[136:0]" -Toggle 0to1 storage_1 [37] "reg storage_1[136:0]" -Toggle 0to1 storage_1 [52:51] "reg storage_1[136:0]" -Toggle 0to1 storage_1 [69:55] "reg storage_1[136:0]" -Toggle 0to1 storage_1 [110:108] "reg storage_1[136:0]" -Toggle 0to1 storage_1 [118:113] "reg storage_1[136:0]" -Toggle 0to1 storage_1 [120] "reg storage_1[136:0]" -Toggle 0to1 storage_1 [123] "reg storage_1[136:0]" -Toggle 0to1 storage_1 [135:132] "reg storage_1[136:0]" -Toggle 1to0 storage_2 [10:4] "reg storage_2[136:0]" -Toggle 1to0 storage_2 [37] "reg storage_2[136:0]" -Toggle 1to0 storage_2 [52:51] "reg storage_2[136:0]" -Toggle 1to0 storage_2 [69:55] "reg storage_2[136:0]" -Toggle 1to0 storage_2 [110:108] "reg storage_2[136:0]" -Toggle 1to0 storage_2 [118:113] "reg storage_2[136:0]" -Toggle 1to0 storage_2 [120] "reg storage_2[136:0]" -Toggle 1to0 storage_2 [123] "reg storage_2[136:0]" -Toggle 1to0 storage_2 [135:132] "reg storage_2[136:0]" -Toggle 0to1 storage_2 [10:4] "reg storage_2[136:0]" -Toggle 0to1 storage_2 [37] "reg storage_2[136:0]" -Toggle 0to1 storage_2 [52:51] "reg storage_2[136:0]" -Toggle 0to1 storage_2 [69:55] "reg storage_2[136:0]" -Toggle 0to1 storage_2 [110:108] "reg storage_2[136:0]" -Toggle 0to1 storage_2 [118:113] "reg storage_2[136:0]" -Toggle 0to1 storage_2 [120] "reg storage_2[136:0]" -Toggle 0to1 storage_2 [123] "reg storage_2[136:0]" -Toggle 0to1 storage_2 [135:132] "reg storage_2[136:0]" -Toggle 1to0 storage_3 [10:4] "reg storage_3[136:0]" -Toggle 1to0 storage_3 [37] "reg storage_3[136:0]" -Toggle 1to0 storage_3 [52:51] "reg storage_3[136:0]" -Toggle 1to0 storage_3 [69:55] "reg storage_3[136:0]" -Toggle 1to0 storage_3 [110:108] "reg storage_3[136:0]" -Toggle 1to0 storage_3 [118:113] "reg storage_3[136:0]" -Toggle 1to0 storage_3 [120] "reg storage_3[136:0]" -Toggle 1to0 storage_3 [123] "reg storage_3[136:0]" -Toggle 1to0 storage_3 [135:132] "reg storage_3[136:0]" -Toggle 0to1 storage_3 [10:4] "reg storage_3[136:0]" -Toggle 0to1 storage_3 [37] "reg storage_3[136:0]" -Toggle 0to1 storage_3 [52:51] "reg storage_3[136:0]" -Toggle 0to1 storage_3 [69:55] "reg storage_3[136:0]" -Toggle 0to1 storage_3 [110:108] "reg storage_3[136:0]" -Toggle 0to1 storage_3 [118:113] "reg storage_3[136:0]" -Toggle 0to1 storage_3 [120] "reg storage_3[136:0]" -Toggle 0to1 storage_3 [123] "reg storage_3[136:0]" -Toggle 0to1 storage_3 [135:132] "reg storage_3[136:0]" -ANNOTATION: "Depth4 credit conservation proves advertised+occupancy<=4, so reserved[3] is always0." -Toggle reserved [3] "net reserved[3:0]" -ANNOTATION: "Rx.scala storage(writePointer) captures only on FLITV/enqueue; its output mux array is the exact concatenation of storage0..3. This precise source-field slice matches the reviewed storage slot exclusion. Invalid physical channel payload is never captured; legal stored REQ/DAT profile attributes remain fixed." -Toggle 1to0 _GEN [3:0][10:4] "net [3:0][136:0]_GEN" -Toggle 1to0 _GEN [3:0][37] "net [3:0][136:0]_GEN" -Toggle 1to0 _GEN [3:0][52:51] "net [3:0][136:0]_GEN" -Toggle 1to0 _GEN [3:0][69:55] "net [3:0][136:0]_GEN" -Toggle 1to0 _GEN [3:0][110:108] "net [3:0][136:0]_GEN" -Toggle 1to0 _GEN [3:0][118:113] "net [3:0][136:0]_GEN" -Toggle 1to0 _GEN [3:0][120] "net [3:0][136:0]_GEN" -Toggle 1to0 _GEN [3:0][123] "net [3:0][136:0]_GEN" -Toggle 1to0 _GEN [3:0][135:132] "net [3:0][136:0]_GEN" -Toggle 0to1 _GEN [3:0][10:4] "net [3:0][136:0]_GEN" -Toggle 0to1 _GEN [3:0][37] "net [3:0][136:0]_GEN" -Toggle 0to1 _GEN [3:0][52:51] "net [3:0][136:0]_GEN" -Toggle 0to1 _GEN [3:0][69:55] "net [3:0][136:0]_GEN" -Toggle 0to1 _GEN [3:0][110:108] "net [3:0][136:0]_GEN" -Toggle 0to1 _GEN [3:0][118:113] "net [3:0][136:0]_GEN" -Toggle 0to1 _GEN [3:0][120] "net [3:0][136:0]_GEN" -Toggle 0to1 _GEN [3:0][123] "net [3:0][136:0]_GEN" -Toggle 0to1 _GEN [3:0][135:132] "net [3:0][136:0]_GEN" -ANNOTATION: "Arm IHI0050H: B13.9.1 LPID uses low5bits with upper3 MBZ; C1.13 NonCopyBackWriteData Resp=0 and CBusy=0a; C1.15 HomeNID/DataSource=0a; C1.1 defines0a MUST zero (not recommended). B13.9.4 DBID upper4 padding MBZ, while low12 X remains measured. B13.10.28 CAH is zero for noncopyback writes. B2.9 CCID=original Addr[5:4]=0 for this64B aligned profile. Only legally sampled FIFO storage and internal copies are excluded; invalid physical channel payload remains measured." -Toggle io_out_bits [131:129] "net io_out_bits[136:0]" -Toggle storage_0 [131:129] "reg storage_0[136:0]" -Toggle storage_1 [131:129] "reg storage_1[136:0]" -Toggle storage_2 [131:129] "reg storage_2[136:0]" -Toggle storage_3 [131:129] "reg storage_3[136:0]" -Toggle _GEN [3:0][131:129] "net [3:0][136:0]_GEN" - -CHECKSUM: "1028417357 3487977871" -INSTANCE: sram_tb.dut.txRsp -ANNOTATION: "MemoryNode.scala source assignments and asserted SN profile: NodeID1; eight local DBIDs; aligned64B ReadNoSnp/WriteNoSnpFull/Ptl; no ordering, exclusive, snoop, tags, PAS or retry credit; incoming write DAT opcode3, clean error, local slot TxnID0..7; outgoing Comp/DBIDResp and CompData with zero optional attributes. Read BE is inapplicable (IssueH B13.9) and emitted zero. RespErr is OK0/NDERR3: neither error bit is excluded. Arbitrary routing, source, return and request TxnIDs and all payload remain measured. Exact serialized field slices follow Flit.fieldsLSB; only missing transitions excluded." -Toggle 1to0 io_in_bits [17:11] "net io_in_bits[70:0]" -Toggle 1to0 io_in_bits [30] "net io_in_bits[70:0]" -Toggle 1to0 io_in_bits [34:32] "net io_in_bits[70:0]" -Toggle 1to0 io_in_bits [45:37] "net io_in_bits[70:0]" -Toggle 1to0 io_in_bits [63:49] "net io_in_bits[70:0]" -Toggle 1to0 io_in_bits [70:65] "net io_in_bits[70:0]" -Toggle 0to1 io_in_bits [17:11] "net io_in_bits[70:0]" -Toggle 0to1 io_in_bits [30] "net io_in_bits[70:0]" -Toggle 0to1 io_in_bits [34:32] "net io_in_bits[70:0]" -Toggle 0to1 io_in_bits [45:37] "net io_in_bits[70:0]" -Toggle 0to1 io_in_bits [63:49] "net io_in_bits[70:0]" -Toggle 0to1 io_in_bits [70:65] "net io_in_bits[70:0]" -Toggle 1to0 io_link_flit [17:12] "net io_link_flit[70:0]" -Toggle 1to0 io_link_flit [30] "net io_link_flit[70:0]" -Toggle 1to0 io_link_flit [34:33] "net io_link_flit[70:0]" -Toggle 1to0 io_link_flit [45:37] "net io_link_flit[70:0]" -Toggle 1to0 io_link_flit [63:49] "net io_link_flit[70:0]" -Toggle 1to0 io_link_flit [70:65] "net io_link_flit[70:0]" -Toggle 0to1 io_link_flit [17:12] "net io_link_flit[70:0]" -Toggle 0to1 io_link_flit [30] "net io_link_flit[70:0]" -Toggle 0to1 io_link_flit [34:33] "net io_link_flit[70:0]" -Toggle 0to1 io_link_flit [45:37] "net io_link_flit[70:0]" -Toggle 0to1 io_link_flit [63:49] "net io_link_flit[70:0]" -Toggle 0to1 io_link_flit [70:65] "net io_link_flit[70:0]" -Toggle 1to0 io_link_flit_r [17:12] "reg io_link_flit_r[70:0]" -Toggle 1to0 io_link_flit_r [30] "reg io_link_flit_r[70:0]" -Toggle 1to0 io_link_flit_r [34:33] "reg io_link_flit_r[70:0]" -Toggle 1to0 io_link_flit_r [45:37] "reg io_link_flit_r[70:0]" -Toggle 1to0 io_link_flit_r [63:49] "reg io_link_flit_r[70:0]" -Toggle 1to0 io_link_flit_r [70:65] "reg io_link_flit_r[70:0]" -Toggle 0to1 io_link_flit_r [17:12] "reg io_link_flit_r[70:0]" -Toggle 0to1 io_link_flit_r [30] "reg io_link_flit_r[70:0]" -Toggle 0to1 io_link_flit_r [34:33] "reg io_link_flit_r[70:0]" -Toggle 0to1 io_link_flit_r [45:37] "reg io_link_flit_r[70:0]" -Toggle 0to1 io_link_flit_r [63:49] "reg io_link_flit_r[70:0]" -Toggle 0to1 io_link_flit_r [70:65] "reg io_link_flit_r[70:0]" - -CHECKSUM: "3712248894 2138691777" -INSTANCE: sram_tb.dut.rxDat -ANNOTATION: "MemoryNode.scala source assignments and asserted SN profile: NodeID1; eight local DBIDs; aligned64B ReadNoSnp/WriteNoSnpFull/Ptl; no ordering, exclusive, snoop, tags, PAS or retry credit; incoming write DAT opcode3, clean error, local slot TxnID0..7; outgoing Comp/DBIDResp and CompData with zero optional attributes. Read BE is inapplicable (IssueH B13.9) and emitted zero. RespErr is OK0/NDERR3: neither error bit is excluded. Arbitrary routing, source, return and request TxnIDs and all payload remain measured. Exact serialized field slices follow Flit.fieldsLSB; only missing transitions excluded." -Toggle 1to0 io_out_bits [10:4] "net io_out_bits[388:0]" -Toggle 1to0 io_out_bits [29:21] "net io_out_bits[388:0]" -Toggle 1to0 io_out_bits [42:37] "net io_out_bits[388:0]" -Toggle 1to0 io_out_bits [54] "net io_out_bits[388:0]" -Toggle 1to0 io_out_bits [76] "net io_out_bits[388:0]" -Toggle 1to0 io_out_bits [95:84] "net io_out_bits[388:0]" -Toggle 1to0 io_out_bits [100:98] "net io_out_bits[388:0]" -Toggle 0to1 io_out_bits [10:4] "net io_out_bits[388:0]" -Toggle 0to1 io_out_bits [29:21] "net io_out_bits[388:0]" -Toggle 0to1 io_out_bits [42:37] "net io_out_bits[388:0]" -Toggle 0to1 io_out_bits [54] "net io_out_bits[388:0]" -Toggle 0to1 io_out_bits [76] "net io_out_bits[388:0]" -Toggle 0to1 io_out_bits [95:84] "net io_out_bits[388:0]" -Toggle 0to1 io_out_bits [100:98] "net io_out_bits[388:0]" -Toggle 1to0 storage_0 [10:4] "reg storage_0[388:0]" -Toggle 1to0 storage_0 [29:21] "reg storage_0[388:0]" -Toggle 1to0 storage_0 [42:37] "reg storage_0[388:0]" -Toggle 1to0 storage_0 [54] "reg storage_0[388:0]" -Toggle 1to0 storage_0 [76] "reg storage_0[388:0]" -Toggle 1to0 storage_0 [95:84] "reg storage_0[388:0]" -Toggle 1to0 storage_0 [100:98] "reg storage_0[388:0]" -Toggle 0to1 storage_0 [10:4] "reg storage_0[388:0]" -Toggle 0to1 storage_0 [29:21] "reg storage_0[388:0]" -Toggle 0to1 storage_0 [42:37] "reg storage_0[388:0]" -Toggle 0to1 storage_0 [54] "reg storage_0[388:0]" -Toggle 0to1 storage_0 [76] "reg storage_0[388:0]" -Toggle 0to1 storage_0 [95:84] "reg storage_0[388:0]" -Toggle 0to1 storage_0 [100:98] "reg storage_0[388:0]" -Toggle 1to0 storage_1 [10:4] "reg storage_1[388:0]" -Toggle 1to0 storage_1 [29:21] "reg storage_1[388:0]" -Toggle 1to0 storage_1 [42:37] "reg storage_1[388:0]" -Toggle 1to0 storage_1 [54] "reg storage_1[388:0]" -Toggle 1to0 storage_1 [76] "reg storage_1[388:0]" -Toggle 1to0 storage_1 [95:84] "reg storage_1[388:0]" -Toggle 1to0 storage_1 [100:98] "reg storage_1[388:0]" -Toggle 0to1 storage_1 [10:4] "reg storage_1[388:0]" -Toggle 0to1 storage_1 [29:21] "reg storage_1[388:0]" -Toggle 0to1 storage_1 [42:37] "reg storage_1[388:0]" -Toggle 0to1 storage_1 [54] "reg storage_1[388:0]" -Toggle 0to1 storage_1 [76] "reg storage_1[388:0]" -Toggle 0to1 storage_1 [95:84] "reg storage_1[388:0]" -Toggle 0to1 storage_1 [100:98] "reg storage_1[388:0]" -Toggle 1to0 storage_2 [10:4] "reg storage_2[388:0]" -Toggle 1to0 storage_2 [29:21] "reg storage_2[388:0]" -Toggle 1to0 storage_2 [42:37] "reg storage_2[388:0]" -Toggle 1to0 storage_2 [54] "reg storage_2[388:0]" -Toggle 1to0 storage_2 [76] "reg storage_2[388:0]" -Toggle 1to0 storage_2 [95:84] "reg storage_2[388:0]" -Toggle 1to0 storage_2 [100:98] "reg storage_2[388:0]" -Toggle 0to1 storage_2 [10:4] "reg storage_2[388:0]" -Toggle 0to1 storage_2 [29:21] "reg storage_2[388:0]" -Toggle 0to1 storage_2 [42:37] "reg storage_2[388:0]" -Toggle 0to1 storage_2 [54] "reg storage_2[388:0]" -Toggle 0to1 storage_2 [76] "reg storage_2[388:0]" -Toggle 0to1 storage_2 [95:84] "reg storage_2[388:0]" -Toggle 0to1 storage_2 [100:98] "reg storage_2[388:0]" -Toggle 1to0 storage_3 [10:4] "reg storage_3[388:0]" -Toggle 1to0 storage_3 [29:21] "reg storage_3[388:0]" -Toggle 1to0 storage_3 [42:37] "reg storage_3[388:0]" -Toggle 1to0 storage_3 [54] "reg storage_3[388:0]" -Toggle 1to0 storage_3 [76] "reg storage_3[388:0]" -Toggle 1to0 storage_3 [95:84] "reg storage_3[388:0]" -Toggle 1to0 storage_3 [100:98] "reg storage_3[388:0]" -Toggle 0to1 storage_3 [10:4] "reg storage_3[388:0]" -Toggle 0to1 storage_3 [29:21] "reg storage_3[388:0]" -Toggle 0to1 storage_3 [42:37] "reg storage_3[388:0]" -Toggle 0to1 storage_3 [54] "reg storage_3[388:0]" -Toggle 0to1 storage_3 [76] "reg storage_3[388:0]" -Toggle 0to1 storage_3 [95:84] "reg storage_3[388:0]" -Toggle 0to1 storage_3 [100:98] "reg storage_3[388:0]" -ANNOTATION: "MemoryNode.io.rxDat.ready is tied true, so every occupied entry drains next cycle; at most one incoming flit per cycle gives occupancy<=1 from reset." -Toggle occupancy [2:1] "reg occupancy[2:0]" -ANNOTATION: "Depth4 credit conservation proves advertised+occupancy<=4, so reserved[3] is always0." -Toggle reserved [3] "net reserved[3:0]" -ANNOTATION: "Rx.scala storage(writePointer) captures only on FLITV/enqueue; its output mux array is the exact concatenation of storage0..3. This precise source-field slice matches the reviewed storage slot exclusion. Invalid physical channel payload is never captured; legal stored REQ/DAT profile attributes remain fixed." -Toggle 1to0 _GEN [3:0][10:4] "net [3:0][388:0]_GEN" -Toggle 1to0 _GEN [3:0][29:21] "net [3:0][388:0]_GEN" -Toggle 1to0 _GEN [3:0][42:37] "net [3:0][388:0]_GEN" -Toggle 1to0 _GEN [3:0][54] "net [3:0][388:0]_GEN" -Toggle 1to0 _GEN [3:0][76] "net [3:0][388:0]_GEN" -Toggle 1to0 _GEN [3:0][95:84] "net [3:0][388:0]_GEN" -Toggle 1to0 _GEN [3:0][100:98] "net [3:0][388:0]_GEN" -Toggle 0to1 _GEN [3:0][10:4] "net [3:0][388:0]_GEN" -Toggle 0to1 _GEN [3:0][29:21] "net [3:0][388:0]_GEN" -Toggle 0to1 _GEN [3:0][42:37] "net [3:0][388:0]_GEN" -Toggle 0to1 _GEN [3:0][54] "net [3:0][388:0]_GEN" -Toggle 0to1 _GEN [3:0][76] "net [3:0][388:0]_GEN" -Toggle 0to1 _GEN [3:0][95:84] "net [3:0][388:0]_GEN" -Toggle 0to1 _GEN [3:0][100:98] "net [3:0][388:0]_GEN" -ANNOTATION: "MemoryNode rxDat.ready=true implies occupancy<=1 and therefore enqueueReady=true for depth4." -Toggle enqueueReady "net enqueueReady" -ANNOTATION: "Arm IHI0050H: B13.9.1 LPID uses low5bits with upper3 MBZ; C1.13 NonCopyBackWriteData Resp=0 and CBusy=0a; C1.15 HomeNID/DataSource=0a; C1.1 defines0a MUST zero (not recommended). B13.9.4 DBID upper4 padding MBZ, while low12 X remains measured. B13.10.28 CAH is zero for noncopyback writes. B2.9 CCID=original Addr[5:4]=0 for this64B aligned profile. Only legally sampled FIFO storage and internal copies are excluded; invalid physical channel payload remains measured." -Toggle io_out_bits [36:30] "net io_out_bits[388:0]" -Toggle io_out_bits [53:43] "net io_out_bits[388:0]" -Toggle io_out_bits [57:55] "net io_out_bits[388:0]" -Toggle io_out_bits [75:70] "net io_out_bits[388:0]" -Toggle io_out_bits [97] "net io_out_bits[388:0]" -Toggle storage_0 [36:30] "reg storage_0[388:0]" -Toggle storage_0 [53:43] "reg storage_0[388:0]" -Toggle storage_0 [57:55] "reg storage_0[388:0]" -Toggle storage_0 [75:70] "reg storage_0[388:0]" -Toggle storage_0 [97] "reg storage_0[388:0]" -Toggle storage_1 [36:30] "reg storage_1[388:0]" -Toggle storage_1 [53:43] "reg storage_1[388:0]" -Toggle storage_1 [57:55] "reg storage_1[388:0]" -Toggle storage_1 [75:70] "reg storage_1[388:0]" -Toggle storage_1 [97] "reg storage_1[388:0]" -Toggle storage_2 [36:30] "reg storage_2[388:0]" -Toggle storage_2 [53:43] "reg storage_2[388:0]" -Toggle storage_2 [57:55] "reg storage_2[388:0]" -Toggle storage_2 [75:70] "reg storage_2[388:0]" -Toggle storage_2 [97] "reg storage_2[388:0]" -Toggle storage_3 [36:30] "reg storage_3[388:0]" -Toggle storage_3 [53:43] "reg storage_3[388:0]" -Toggle storage_3 [57:55] "reg storage_3[388:0]" -Toggle storage_3 [75:70] "reg storage_3[388:0]" -Toggle storage_3 [97] "reg storage_3[388:0]" -Toggle _GEN [3:0][36:30] "net [3:0][388:0]_GEN" -Toggle _GEN [3:0][53:43] "net [3:0][388:0]_GEN" -Toggle _GEN [3:0][57:55] "net [3:0][388:0]_GEN" -Toggle _GEN [3:0][75:70] "net [3:0][388:0]_GEN" -Toggle _GEN [3:0][97] "net [3:0][388:0]_GEN" - -CHECKSUM: "2770640837 504484466" -INSTANCE: sram_tb.dut.txDat -ANNOTATION: "MemoryNode.scala source assignments and asserted SN profile: NodeID1; eight local DBIDs; aligned64B ReadNoSnp/WriteNoSnpFull/Ptl; no ordering, exclusive, snoop, tags, PAS or retry credit; incoming write DAT opcode3, clean error, local slot TxnID0..7; outgoing Comp/DBIDResp and CompData with zero optional attributes. Read BE is inapplicable (IssueH B13.9) and emitted zero. RespErr is OK0/NDERR3: neither error bit is excluded. Arbitrary routing, source, return and request TxnIDs and all payload remain measured. Exact serialized field slices follow Flit.fieldsLSB; only missing transitions excluded." -Toggle 1to0 io_in_bits [17:11] "net io_in_bits[388:0]" -Toggle 1to0 io_in_bits [40:37] "net io_in_bits[388:0]" -Toggle 1to0 io_in_bits [57:43] "net io_in_bits[388:0]" -Toggle 1to0 io_in_bits [76:70] "net io_in_bits[388:0]" -Toggle 1to0 io_in_bits [95:78] "net io_in_bits[388:0]" -Toggle 1to0 io_in_bits [132:97] "net io_in_bits[388:0]" -Toggle 0to1 io_in_bits [17:11] "net io_in_bits[388:0]" -Toggle 0to1 io_in_bits [40:37] "net io_in_bits[388:0]" -Toggle 0to1 io_in_bits [57:43] "net io_in_bits[388:0]" -Toggle 0to1 io_in_bits [76:70] "net io_in_bits[388:0]" -Toggle 0to1 io_in_bits [95:78] "net io_in_bits[388:0]" -Toggle 0to1 io_in_bits [132:97] "net io_in_bits[388:0]" -Toggle 1to0 io_link_flit [17:12] "net io_link_flit[388:0]" -Toggle 1to0 io_link_flit [38:37] "net io_link_flit[388:0]" -Toggle 1to0 io_link_flit [40] "net io_link_flit[388:0]" -Toggle 1to0 io_link_flit [57:43] "net io_link_flit[388:0]" -Toggle 1to0 io_link_flit [76:70] "net io_link_flit[388:0]" -Toggle 1to0 io_link_flit [95:78] "net io_link_flit[388:0]" -Toggle 1to0 io_link_flit [132:97] "net io_link_flit[388:0]" -Toggle 0to1 io_link_flit [17:12] "net io_link_flit[388:0]" -Toggle 0to1 io_link_flit [38:37] "net io_link_flit[388:0]" -Toggle 0to1 io_link_flit [40] "net io_link_flit[388:0]" -Toggle 0to1 io_link_flit [57:43] "net io_link_flit[388:0]" -Toggle 0to1 io_link_flit [76:70] "net io_link_flit[388:0]" -Toggle 0to1 io_link_flit [95:78] "net io_link_flit[388:0]" -Toggle 0to1 io_link_flit [132:97] "net io_link_flit[388:0]" -Toggle 1to0 io_link_flit_r [17:12] "reg io_link_flit_r[388:0]" -Toggle 1to0 io_link_flit_r [38:37] "reg io_link_flit_r[388:0]" -Toggle 1to0 io_link_flit_r [40] "reg io_link_flit_r[388:0]" -Toggle 1to0 io_link_flit_r [57:43] "reg io_link_flit_r[388:0]" -Toggle 1to0 io_link_flit_r [76:70] "reg io_link_flit_r[388:0]" -Toggle 1to0 io_link_flit_r [95:78] "reg io_link_flit_r[388:0]" -Toggle 1to0 io_link_flit_r [132:97] "reg io_link_flit_r[388:0]" -Toggle 0to1 io_link_flit_r [17:12] "reg io_link_flit_r[388:0]" -Toggle 0to1 io_link_flit_r [38:37] "reg io_link_flit_r[388:0]" -Toggle 0to1 io_link_flit_r [40] "reg io_link_flit_r[388:0]" -Toggle 0to1 io_link_flit_r [57:43] "reg io_link_flit_r[388:0]" -Toggle 0to1 io_link_flit_r [76:70] "reg io_link_flit_r[388:0]" -Toggle 0to1 io_link_flit_r [95:78] "reg io_link_flit_r[388:0]" -Toggle 0to1 io_link_flit_r [132:97] "reg io_link_flit_r[388:0]" - -CHECKSUM: "2383780593 1281742771" -INSTANCE: sram_tb.dut -ANNOTATION: "MemoryNode.scala source assignments and asserted SN profile: NodeID1; eight local DBIDs; aligned64B ReadNoSnp/WriteNoSnpFull/Ptl; no ordering, exclusive, snoop, tags, PAS or retry credit; incoming write DAT opcode3, clean error, local slot TxnID0..7; outgoing Comp/DBIDResp and CompData with zero optional attributes. Read BE is inapplicable (IssueH B13.9) and emitted zero. RespErr is OK0/NDERR3: neither error bit is excluded. Arbitrary routing, source, return and request TxnIDs and all payload remain measured. Exact serialized field slices follow Flit.fieldsLSB; only missing transitions excluded." -Toggle 1to0 io_chi_txRsp_flit [17:12] "net io_chi_txRsp_flit[70:0]" -Toggle 1to0 io_chi_txRsp_flit [30] "net io_chi_txRsp_flit[70:0]" -Toggle 1to0 io_chi_txRsp_flit [34:33] "net io_chi_txRsp_flit[70:0]" -Toggle 1to0 io_chi_txRsp_flit [45:37] "net io_chi_txRsp_flit[70:0]" -Toggle 1to0 io_chi_txRsp_flit [63:49] "net io_chi_txRsp_flit[70:0]" -Toggle 1to0 io_chi_txRsp_flit [70:65] "net io_chi_txRsp_flit[70:0]" -Toggle 0to1 io_chi_txRsp_flit [17:12] "net io_chi_txRsp_flit[70:0]" -Toggle 0to1 io_chi_txRsp_flit [30] "net io_chi_txRsp_flit[70:0]" -Toggle 0to1 io_chi_txRsp_flit [34:33] "net io_chi_txRsp_flit[70:0]" -Toggle 0to1 io_chi_txRsp_flit [45:37] "net io_chi_txRsp_flit[70:0]" -Toggle 0to1 io_chi_txRsp_flit [63:49] "net io_chi_txRsp_flit[70:0]" -Toggle 0to1 io_chi_txRsp_flit [70:65] "net io_chi_txRsp_flit[70:0]" -Toggle 1to0 io_chi_txDat_flit [17:12] "net io_chi_txDat_flit[388:0]" -Toggle 1to0 io_chi_txDat_flit [38:37] "net io_chi_txDat_flit[388:0]" -Toggle 1to0 io_chi_txDat_flit [40] "net io_chi_txDat_flit[388:0]" -Toggle 1to0 io_chi_txDat_flit [57:43] "net io_chi_txDat_flit[388:0]" -Toggle 1to0 io_chi_txDat_flit [76:70] "net io_chi_txDat_flit[388:0]" -Toggle 1to0 io_chi_txDat_flit [95:78] "net io_chi_txDat_flit[388:0]" -Toggle 1to0 io_chi_txDat_flit [132:97] "net io_chi_txDat_flit[388:0]" -Toggle 0to1 io_chi_txDat_flit [17:12] "net io_chi_txDat_flit[388:0]" -Toggle 0to1 io_chi_txDat_flit [38:37] "net io_chi_txDat_flit[388:0]" -Toggle 0to1 io_chi_txDat_flit [40] "net io_chi_txDat_flit[388:0]" -Toggle 0to1 io_chi_txDat_flit [57:43] "net io_chi_txDat_flit[388:0]" -Toggle 0to1 io_chi_txDat_flit [76:70] "net io_chi_txDat_flit[388:0]" -Toggle 0to1 io_chi_txDat_flit [95:78] "net io_chi_txDat_flit[388:0]" -Toggle 0to1 io_chi_txDat_flit [132:97] "net io_chi_txDat_flit[388:0]" -Toggle 1to0 _rxDat_io_out_bits [10:4] "net _rxDat_io_out_bits[388:0]" -Toggle 1to0 _rxDat_io_out_bits [29:21] "net _rxDat_io_out_bits[388:0]" -Toggle 1to0 _rxDat_io_out_bits [42:37] "net _rxDat_io_out_bits[388:0]" -Toggle 1to0 _rxDat_io_out_bits [54] "net _rxDat_io_out_bits[388:0]" -Toggle 1to0 _rxDat_io_out_bits [76] "net _rxDat_io_out_bits[388:0]" -Toggle 1to0 _rxDat_io_out_bits [95:84] "net _rxDat_io_out_bits[388:0]" -Toggle 1to0 _rxDat_io_out_bits [100:98] "net _rxDat_io_out_bits[388:0]" -Toggle 0to1 _rxDat_io_out_bits [10:4] "net _rxDat_io_out_bits[388:0]" -Toggle 0to1 _rxDat_io_out_bits [29:21] "net _rxDat_io_out_bits[388:0]" -Toggle 0to1 _rxDat_io_out_bits [42:37] "net _rxDat_io_out_bits[388:0]" -Toggle 0to1 _rxDat_io_out_bits [54] "net _rxDat_io_out_bits[388:0]" -Toggle 0to1 _rxDat_io_out_bits [76] "net _rxDat_io_out_bits[388:0]" -Toggle 0to1 _rxDat_io_out_bits [95:84] "net _rxDat_io_out_bits[388:0]" -Toggle 0to1 _rxDat_io_out_bits [100:98] "net _rxDat_io_out_bits[388:0]" -Toggle 1to0 _rxReq_io_out_bits [10:4] "net _rxReq_io_out_bits[136:0]" -Toggle 1to0 _rxReq_io_out_bits [37] "net _rxReq_io_out_bits[136:0]" -Toggle 1to0 _rxReq_io_out_bits [52:51] "net _rxReq_io_out_bits[136:0]" -Toggle 1to0 _rxReq_io_out_bits [69:55] "net _rxReq_io_out_bits[136:0]" -Toggle 1to0 _rxReq_io_out_bits [110:108] "net _rxReq_io_out_bits[136:0]" -Toggle 1to0 _rxReq_io_out_bits [118:113] "net _rxReq_io_out_bits[136:0]" -Toggle 1to0 _rxReq_io_out_bits [120] "net _rxReq_io_out_bits[136:0]" -Toggle 1to0 _rxReq_io_out_bits [123] "net _rxReq_io_out_bits[136:0]" -Toggle 1to0 _rxReq_io_out_bits [135:132] "net _rxReq_io_out_bits[136:0]" -Toggle 0to1 _rxReq_io_out_bits [10:4] "net _rxReq_io_out_bits[136:0]" -Toggle 0to1 _rxReq_io_out_bits [37] "net _rxReq_io_out_bits[136:0]" -Toggle 0to1 _rxReq_io_out_bits [52:51] "net _rxReq_io_out_bits[136:0]" -Toggle 0to1 _rxReq_io_out_bits [69:55] "net _rxReq_io_out_bits[136:0]" -Toggle 0to1 _rxReq_io_out_bits [110:108] "net _rxReq_io_out_bits[136:0]" -Toggle 0to1 _rxReq_io_out_bits [118:113] "net _rxReq_io_out_bits[136:0]" -Toggle 0to1 _rxReq_io_out_bits [120] "net _rxReq_io_out_bits[136:0]" -Toggle 0to1 _rxReq_io_out_bits [123] "net _rxReq_io_out_bits[136:0]" -Toggle 0to1 _rxReq_io_out_bits [135:132] "net _rxReq_io_out_bits[136:0]" -ANNOTATION: "SramEndpoint direct child connections preserve source field invariants: local memory/response IDs0..7, RSP opcode4/6, DAT dbid zero-extension12to16, DataID0/2, backend64B address alignment. Only the listed constant bits are excluded." -Toggle _memory_io_resp_bits_id [11:3] "net _memory_io_resp_bits_id[11:0]" -Toggle _node_io_rsp_bits_opcode [0] "net _node_io_rsp_bits_opcode[4:0]" -Toggle _node_io_rsp_bits_opcode [4:2] "net _node_io_rsp_bits_opcode[4:0]" -Toggle _node_io_rsp_bits_dbid [11:3] "net _node_io_rsp_bits_dbid[11:0]" -Toggle _node_io_txDat_bits_dbid [15:12] "net _node_io_txDat_bits_dbid[15:0]" -Toggle _node_io_txDat_bits_dataId [0] "net _node_io_txDat_bits_dataId[1:0]" -Toggle _node_io_memoryReq_bits_id [11:3] "net _node_io_memoryReq_bits_id[11:0]" -Toggle _node_io_memoryReq_bits_addr [5:0] "net _node_io_memoryReq_bits_addr[43:0]" -ANNOTATION: "Arm IHI0050H: B13.9.1 LPID uses low5bits with upper3 MBZ; C1.13 NonCopyBackWriteData Resp=0 and CBusy=0a; C1.15 HomeNID/DataSource=0a; C1.1 defines0a MUST zero (not recommended). B13.9.4 DBID upper4 padding MBZ, while low12 X remains measured. B13.10.28 CAH is zero for noncopyback writes. B2.9 CCID=original Addr[5:4]=0 for this64B aligned profile. Only legally sampled FIFO storage and internal copies are excluded; invalid physical channel payload remains measured." -Toggle _rxDat_io_out_bits [36:30] "net _rxDat_io_out_bits[388:0]" -Toggle _rxDat_io_out_bits [53:43] "net _rxDat_io_out_bits[388:0]" -Toggle _rxDat_io_out_bits [57:55] "net _rxDat_io_out_bits[388:0]" -Toggle _rxDat_io_out_bits [75:70] "net _rxDat_io_out_bits[388:0]" -Toggle _rxDat_io_out_bits [97] "net _rxDat_io_out_bits[388:0]" -Toggle _rxReq_io_out_bits [131:129] "net _rxReq_io_out_bits[136:0]" diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram.f b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram.f deleted file mode 100644 index b9a0f6e99..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram.f +++ /dev/null @@ -1,7 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/sram_interface.sv -@RESOURCES@/sram_package.sv --F @RTL@/SramEndpoint256/filelist.f -@RESOURCES@/sram_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram128.f b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram128.f deleted file mode 100644 index 641720ca0..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram128.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+CHI_DATA_BITS=128 -+define+CHI_SRAM_DUT=SramEndpoint128 -+define+CHI_SRAM_TB=sram128_tb -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/sram_interface.sv -@RESOURCES@/sram_package.sv --F @RTL@/SramEndpoint128/filelist.f -@RESOURCES@/sram_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram512.f b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram512.f deleted file mode 100644 index c7dca7651..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram512.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+CHI_DATA_BITS=512 -+define+CHI_SRAM_DUT=SramEndpoint512 -+define+CHI_SRAM_TB=sram512_tb -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/sram_interface.sv -@RESOURCES@/sram_package.sv --F @RTL@/SramEndpoint512/filelist.f -@RESOURCES@/sram_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_bad_opcode.f b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_bad_opcode.f deleted file mode 100644 index af75c4f42..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_bad_opcode.f +++ /dev/null @@ -1,9 +0,0 @@ -+define+CHI_BAD_OPCODE -+define+CHI_SRAM_TB=sram_bad_opcode_tb -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/sram_interface.sv -@RESOURCES@/sram_package.sv --F @RTL@/SramEndpoint256/filelist.f -@RESOURCES@/sram_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_bad_opcode.toml b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_bad_opcode.toml deleted file mode 100644 index dd6e02ff9..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_bad_opcode.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Unsupported CHI request opcode" diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_duplicate_data.f b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_duplicate_data.f deleted file mode 100644 index 654192d64..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_duplicate_data.f +++ /dev/null @@ -1,9 +0,0 @@ -+define+CHI_DUPLICATE_DATA -+define+CHI_SRAM_TB=sram_duplicate_data_tb -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/sram_interface.sv -@RESOURCES@/sram_package.sv --F @RTL@/SramEndpoint256/filelist.f -@RESOURCES@/sram_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_duplicate_data.toml b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_duplicate_data.toml deleted file mode 100644 index a4faa37ef..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_duplicate_data.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Duplicate CHI DataID" diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_interface.sv b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_interface.sv deleted file mode 100644 index 6fb9a0df9..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_interface.sv +++ /dev/null @@ -1,16 +0,0 @@ -`include "chi_profile.svh" -interface chi_sram_if ( - input logic clock -); - logic reset; - logic req_pend, req_valid, req_credit; - logic [136:0] req_flit; - logic dat_pend, dat_valid, dat_credit; - logic [`CHI_DAT_BITS-1:0] dat_flit; - logic rsp_pend, rsp_valid, rsp_credit; - logic [70:0] rsp_flit; - logic out_dat_pend, out_dat_valid, out_dat_credit; - logic [`CHI_DAT_BITS-1:0] out_dat_flit; - logic rx_active_req, rx_active_ack, tx_active_req, tx_active_ack; - logic [3:0] outstanding; -endinterface diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_package.sv b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_package.sv deleted file mode 100644 index 7ccad3f88..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_package.sv +++ /dev/null @@ -1,20 +0,0 @@ -package chi_sram_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - `include "flit_types.svh" - import "DPI-C" function chandle chi_ref_create(input int unsigned lines); - import "DPI-C" function void chi_ref_destroy(input chandle model); - import "DPI-C" function byte unsigned chi_ref_write( - input chandle model, - input longint unsigned addr, - input bit [511:0] data, - input bit [63:0] mask - ); - import "DPI-C" function byte unsigned chi_ref_read( - input chandle model, - input longint unsigned addr, - output bit [511:0] data - ); - `include "sram_test.svh" -endpackage diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_tb.sv b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_tb.sv deleted file mode 100644 index 2526b29b8..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_tb.sv +++ /dev/null @@ -1,44 +0,0 @@ -`ifndef CHI_SRAM_TB -`define CHI_SRAM_TB sram_tb -`endif -`ifndef CHI_SRAM_DUT -`define CHI_SRAM_DUT SramEndpoint256 -`endif -module `CHI_SRAM_TB; - import uvm_pkg::*; - import chi_sram_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - chi_sram_if source_if (clock); - `CHI_SRAM_DUT dut ( - .clock(clock), - .reset(source_if.reset), - .io_chi_rxReq_flitpend(source_if.req_pend), - .io_chi_rxReq_flitv(source_if.req_valid), - .io_chi_rxReq_flit(source_if.req_flit), - .io_chi_rxReq_lcrdv(source_if.req_credit), - .io_chi_rxDat_flitpend(source_if.dat_pend), - .io_chi_rxDat_flitv(source_if.dat_valid), - .io_chi_rxDat_flit(source_if.dat_flit), - .io_chi_rxDat_lcrdv(source_if.dat_credit), - .io_chi_txRsp_flitpend(source_if.rsp_pend), - .io_chi_txRsp_flitv(source_if.rsp_valid), - .io_chi_txRsp_flit(source_if.rsp_flit), - .io_chi_txRsp_lcrdv(source_if.rsp_credit), - .io_chi_txDat_flitpend(source_if.out_dat_pend), - .io_chi_txDat_flitv(source_if.out_dat_valid), - .io_chi_txDat_flit(source_if.out_dat_flit), - .io_chi_txDat_lcrdv(source_if.out_dat_credit), - .io_chi_rxLinkActiveReq(source_if.rx_active_req), - .io_chi_rxLinkActiveAck(source_if.rx_active_ack), - .io_chi_txLinkActiveReq(source_if.tx_active_req), - .io_chi_txLinkActiveAck(source_if.tx_active_ack), - .io_chi_rxSActive(source_if.rx_active_req), - .io_chi_txSActive(), - .io_outstanding(source_if.outstanding) - ); - initial begin - uvm_config_db#(virtual chi_sram_if)::set(null, "uvm_test_top", "vif", source_if); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_test.svh b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_test.svh deleted file mode 100644 index d364ca91b..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_test.svh +++ /dev/null @@ -1,499 +0,0 @@ -class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual chi_sram_if vif; - chandle model; - int req_credits = 0, dat_credits = 0, rsp_granted = 0, out_dat_granted = 0; - int cycle = 0, checks = 0; - localparam int STALL_READS = 15 / `CHI_BEATS + 8; - bit grant_enabled = 1; - rsp_t responses[$]; - dat_t data_responses[$]; - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual chi_sram_if)::get(this, "", "vif", vif)) - `uvm_fatal("VIF", "CHI interface missing") - if ($bits(req_t) != 137 || $bits(rsp_t) != 71 || $bits(dat_t) != `CHI_DAT_BITS) - `uvm_fatal("FLIT", "Verification field packing does not match declared profile") - timeout = 300us; - model = chi_ref_create(256); - endfunction - task monitor(); - forever begin - @(posedge vif.clock); - if (vif.reset) begin - req_credits = 0; - dat_credits = 0; - rsp_granted = 0; - out_dat_granted = 0; - responses.delete(); - data_responses.delete(); - end else begin - cycle++; - req_credits += int'(vif.req_credit) - int'(vif.req_valid); - dat_credits += int'(vif.dat_credit) - int'(vif.dat_valid); - rsp_granted += int'(vif.rsp_credit) - int'(vif.rsp_valid); - out_dat_granted += int'(vif.out_dat_credit) - int'(vif.out_dat_valid); - if (req_credits < 0 || dat_credits < 0 || rsp_granted < 0 || out_dat_granted < 0) - `uvm_fatal("CREDIT", "Flit without credit") - if (vif.rsp_valid) responses.push_back(rsp_t'(vif.rsp_flit)); - if (vif.out_dat_valid) data_responses.push_back(dat_t'(vif.out_dat_flit)); - end - @(negedge vif.clock); - vif.rsp_credit = !vif.reset && vif.tx_active_req && vif.tx_active_ack && grant_enabled && rsp_granted < 15 && cycle % 5 != 0; - vif.out_dat_credit = !vif.reset && vif.tx_active_req && vif.tx_active_ack && grant_enabled && out_dat_granted < 15 && cycle % 7 < 3; - end - endtask - task reset_link(); - @(negedge vif.clock); - vif.reset = 1; - vif.req_pend = 0; - vif.dat_pend = 0; - vif.req_valid = 0; - vif.dat_valid = 0; - // Peer levels may settle after the coordinated reset edge; FLITV is already low. - repeat (2) @(negedge vif.clock); - vif.rx_active_req = 0; - vif.tx_active_ack = 0; - repeat (2) @(negedge vif.clock); - vif.reset = 0; - repeat (3) @(negedge vif.clock); - vif.rx_active_req = 1; - while (!vif.tx_active_req) @(negedge vif.clock); - repeat (3) @(negedge vif.clock); - vif.tx_active_ack = 1; - vif.req_pend = 1; - vif.dat_pend = 1; - repeat (4) @(negedge vif.clock); - if (!vif.rx_active_ack || !vif.tx_active_req) `uvm_fatal("LINK", "Activation did not complete") - endtask - task send_req(req_t r); - @(negedge vif.clock); - while (req_credits == 0) @(negedge vif.clock); - vif.req_flit = r; - vif.req_valid = 1; - @(negedge vif.clock); - vif.req_valid = 0; - endtask - task send_dat(dat_t d); - @(negedge vif.clock); - while (dat_credits == 0) @(negedge vif.clock); - vif.dat_flit = d; - vif.dat_valid = 1; - @(negedge vif.clock); - vif.dat_valid = 0; - endtask - function req_t request(int opcode, int txn, longint unsigned addr, int source = 3); - req_t r = '0; - r.opcode = opcode; - r.txn = txn; - r.addr = addr; - r.size = 6; - r.src = (txn ^ source) & 127; - r.tgt = 1; - r.return_nid = (txn ^ 85) & 127; - r.return_txn = txn + 100; - r.allow_retry = txn % 2; - r.mem_attr = (txn % 3 == 0 ? 0 : txn % 3 == 1 ? 4 : 12) | (txn % 2); - r.likely_shared = txn % 2; - r.lpid = txn % 32; - r.qos = txn % 16; - r.trace_tag = txn % 2; - return r; - endfunction - task response(req_t r, int opcode, output rsp_t out_rsp); - bit found = 0; - while (!found) begin - @(negedge vif.clock); - foreach (responses[i]) begin - if (responses[i].txn == r.txn && responses[i].tgt == r.src) begin - out_rsp = responses[i]; - responses.delete(i); - found = 1; - break; - end - end - end - if (out_rsp.opcode != opcode || out_rsp.src != 1 || out_rsp.qos != r.qos || - out_rsp.trace_tag != r.trace_tag) - `uvm_fatal("RESPONSE", $sformatf("Bad response txn=%0d opcode=%0h", r.txn, out_rsp.opcode)) - checks++; - endtask - function dat_t write_beat(req_t r, rsp_t dbid_rsp, int beat, bit [`CHI_DATA_BITS-1:0] data, - bit [`CHI_BE_BITS-1:0] mask = '1); - dat_t d = '0; - d.tgt = 1; - d.src = r.src; - d.txn = dbid_rsp.dbid; - d.qos = r.qos; - d.dbid = {4'b0, r.txn}; - d.line_id = r.addr[11:6]; - d.trace_tag = r.trace_tag; - d.opcode = 3; - d.data_id = beat * `CHI_DATA_ID_STEP; - d.data = data; - d.be = mask; - return d; - endfunction - task write_data(req_t r, rsp_t dbid_rsp, bit [511:0] data, bit [63:0] mask); - for (int beat = `CHI_BEATS - 1; beat >= 0; beat--) - send_dat(write_beat( - r, - dbid_rsp, - beat, - data[beat*`CHI_DATA_BITS+:`CHI_DATA_BITS], - mask[beat*`CHI_BE_BITS+:`CHI_BE_BITS] - )); - endtask - task write_line(int txn, longint unsigned addr, bit [511:0] data, bit [63:0] mask = '1); - req_t r = request(mask == '1 ? 'h1d : 'h1c, txn, addr); - rsp_t s; - byte unsigned expected_error; - send_req(r); - response(r, 6, s); - write_data(r, s, data, mask); - response(r, 4, s); - expected_error = chi_ref_write(model, addr, data, mask); - if (s.error != (expected_error ? 3 : 0)) `uvm_fatal("ERROR", "Wrong write error") - checks++; - endtask - task read_line(int txn, longint unsigned addr); - req_t r = request(4, txn, addr); - dat_t d; - bit [511:0] golden; - bit [`CHI_BEATS-1:0] seen = 0; - byte unsigned error = chi_ref_read(model, addr, golden); - send_req(r); - while (!(&seen)) begin - bit found = 0; - @(negedge vif.clock); - foreach (data_responses[i]) begin - if (data_responses[i].txn == r.return_txn) begin - d = data_responses[i]; - data_responses.delete(i); - found = 1; - break; - end - end - if (found) begin - int b = d.data_id / `CHI_DATA_ID_STEP; - if ((d.data_id % `CHI_DATA_ID_STEP != 0 || b >= `CHI_BEATS) || seen[b] || d.opcode != 4 || - d.tgt != r.return_nid || d.src != 1 || d.home != r.src || d.dbid != r.txn || - d.error != (error ? 3 : 0) || d.qos != r.qos || d.trace_tag != r.trace_tag || - d.data !== golden[b*`CHI_DATA_BITS +: `CHI_DATA_BITS]) - `uvm_fatal("READ", $sformatf("Bad read data txn=%0d DataID=%0d", txn, d.data_id)) - seen[b] = 1; - checks++; - end - end - endtask - task scenarios(); - req_t r[8]; - rsp_t dbid[8], s; - bit [511:0] words[8]; - reset_link(); -`ifdef CHI_BAD_OPCODE - send_req(request('h07, 0, 0)); - repeat (20) @(negedge vif.clock); - `uvm_fatal("NEGATIVE", "Unsupported opcode did not terminate RTL") -`elsif CHI_DUPLICATE_DATA - r[0] = request('h1d, 0, 0); - send_req(r[0]); - response(r[0], 6, s); - begin - dat_t d = '0; - d.tgt = 1; - d.src = r[0].src; - d.txn = s.dbid; - d.opcode = 3; - d.data_id = 0; - d.be = '1; - send_dat(d); - send_dat(d); - end - repeat (20) @(negedge vif.clock); - `uvm_fatal("NEGATIVE", "Duplicate DataID did not terminate RTL") -`else - // Fill all eight transaction slots, then interleave reverse-order DataIDs. - for (int i = 0; i < 8; i++) begin - for (int b = 0; b < 64; b++) words[i][b*8+:8] = (i * 37 + b); - r[i] = request('h1d, i, i * 64, i % 2 ? 5 : 3); - send_req(r[i]); - response(r[i], 6, dbid[i]); - end - if (vif.outstanding != 8) `uvm_fatal("MSHR", "Failed to occupy all slots") - for (int beat = `CHI_BEATS - 1; beat >= 0; beat--) begin - for (int i = 7; i >= 0; i--) begin - send_dat(write_beat(r[i], dbid[i], beat, words[i][beat*`CHI_DATA_BITS+:`CHI_DATA_BITS], '1 - )); - end - end - for (int i = 0; i < 8; i++) begin - response(r[i], 4, s); - if (s.error != 0 || chi_ref_write(model, i * 64, words[i], '1) != 0) - `uvm_fatal("WRITE", "Initial full write failed") - end - for (int i = 0; i < 8; i++) read_line(32 + i, i * 64); - // Every live slot must merge byte enables independently, including interleaved beats. - for (int phase_index = 0; phase_index < 2; phase_index++) begin - bit [63:0] mask = phase_index ? 64'haaaaaaaaaaaaaaaa : 64'h5555555555555555; - for (int i = 0; i < 8; i++) begin - r[i] = request('h1c, 140 + phase_index * 8 + i, i * 64, i % 2 ? 5 : 3); - send_req(r[i]); - response(r[i], 6, dbid[i]); - end - for (int beat = `CHI_BEATS - 1; beat >= 0; beat--) begin - for (int i = 7; i >= 0; i--) begin - send_dat(write_beat( - r[i], - dbid[i], - beat, - phase_index ? words[i][beat*`CHI_DATA_BITS +: `CHI_DATA_BITS] : ~words[i][beat*`CHI_DATA_BITS +: `CHI_DATA_BITS], - mask[beat*`CHI_BE_BITS+:`CHI_BE_BITS] - )); - end - end - for (int i = 0; i < 8; i++) begin - bit [511:0] data = phase_index ? words[i] : ~words[i]; - response(r[i], 4, s); - if (s.error || chi_ref_write(model, i * 64, data, mask)) - `uvm_fatal("MASK", "Concurrent partial write failed") - end - for (int i = 0; i < 8; i++) read_line(240 + phase_index * 8 + i, i * 64); - end - `uvm_info("PHASE", "Full-width slot vectors", UVM_LOW) - // Exercise each real transaction slot's address, opaque IDs and error status. - // Writes are concurrent, addresses distinct; then verify valid writes by readback. - for (int polarity = 0; polarity < 2; polarity++) begin - for (int address_bit = 9; address_bit < 44; address_bit++) begin - for (int i = 0; i < 8; i++) begin - longint unsigned high = (64'h1 << address_bit) ^ (polarity ? 64'hffffffffe00 : 0); - int tag = (1 << ((address_bit + i) % 12)) ^ (polarity ? 'hfff : 0); - int source = ((1 << ((address_bit + i) % 7)) | 2) ^ (polarity ? 'h7f : 0); - r[i] = request('h1d, tag, high | ((i ^ (1 << ((address_bit + polarity) % 3))) << 6)); - r[i].src = source; - r[i].return_nid = source ^ 'h7f; - r[i].return_txn = tag ^ 'hfff; - r[i].qos = (address_bit + i) % 16; - r[i].trace_tag = polarity; - send_req(r[i]); - response(r[i], 6, dbid[i]); - end - for (int beat = `CHI_BEATS - 1; beat >= 0; beat--) begin - for (int i = 7; i >= 0; i--) begin - send_dat(write_beat( - r[i], - dbid[i], - beat, - polarity ? words[i][beat*`CHI_DATA_BITS +: `CHI_DATA_BITS] : ~words[i][beat*`CHI_DATA_BITS +: `CHI_DATA_BITS], - '1 - )); - end - end - for (int i = 0; i < 8; i++) begin - bit [511:0] data = polarity ? words[i] : ~words[i]; - byte unsigned error; - response(r[i], 4, s); - error = chi_ref_write(model, r[i].addr, data, '1); - if (s.error != (error ? 3 : 0)) - `uvm_fatal("ERROR_SLOT", "Wrong error or opaque transaction attribution") - checks++; - end - if (!polarity && address_bit < 14) - for (int i = 0; i < 8; i++) read_line(600 + address_bit * 8 + i, r[i].addr); - end - end - write_line(50, 0, '1, 64'h5555555555555555); - write_line(51, 0, '0, 64'haaaaaaaaaaaaaaaa); - write_line(52, 0, '1, '0); - read_line(53, 0); - write_line(54, 255 * 64, '1); - read_line(55, 255 * 64); - write_line(56, 256 * 64, '1); - read_line(57, 256 * 64); - // Full-width addresses and TxnIDs must not alias into SRAM or DBID slots. - write_line(4095, 0, '1); - read_line(3950, 0); - write_line(58, 64'h80000000000, '1); - read_line(59, 64'h80000000000); - `uvm_info("PHASE", "Data credit stall", UVM_LOW) - // Block output credit return; requests remain live, then resume safely. - grant_enabled = 0; - for (int i = 0; i < STALL_READS; i++) send_req(request(4, 80 + i, (i % 8) * 64)); - repeat (25) @(negedge vif.clock); - if (vif.outstanding == 0) `uvm_fatal("STALL", "No live request under data credit stall") - grant_enabled = 1; - // These reads were issued already: compare every returned beat before reset. - for (int i = 0; i < STALL_READS; i++) begin - bit [511:0] golden; - bit [`CHI_BEATS-1:0] seen = 0; - if (chi_ref_read(model, (i % 8) * 64, golden)) `uvm_fatal("MODEL", "Unexpected error") - while (!(&seen)) begin - @(negedge vif.clock); - foreach (data_responses[j]) begin - if (data_responses[j].txn == 180 + i) begin - dat_t d = data_responses[j]; - int b = d.data_id / `CHI_DATA_ID_STEP; - if (seen[b] || d.data !== golden[b*`CHI_DATA_BITS+:`CHI_DATA_BITS]) - `uvm_fatal("STALL", "Stalled read corrupted") - seen[b] = 1; - checks++; - data_responses.delete(j); - break; - end - end - end - end - `uvm_info("PHASE", "Read-to-write slot reuse", UVM_LOW) - // Reallocate every slot to a write after the all-slot read drain. - for (int i = 0; i < 8; i++) begin - r[i] = request('h1d, 2800 + i, i * 64); - send_req(r[i]); - response(r[i], 6, dbid[i]); - end - for (int i = 0; i < 8; i++) write_data(r[i], dbid[i], words[i], '1); - for (int i = 0; i < 8; i++) begin - response(r[i], 4, s); - if (s.error || chi_ref_write(model, i * 64, words[i], '1)) - `uvm_fatal("SLOT_REUSE", "Read-to-write slot reuse failed") - end - for (int i = 0; i < 8; i++) read_line(2900 + i, i * 64); - `uvm_info("PHASE", "Response and request queue saturation", UVM_LOW) - // Exhaust all15 RSP credits, then fill all8 node slots and all4 RX entries. - // Distinct TxnIDs from one source and equal TxnIDs from distinct sources are legal. - begin - req_t blocked[11]; - int data_order[8] = '{0, 2, 1, 3, 4, 6, 5, 7}; - while (rsp_granted != 15) @(negedge vif.clock); - grant_enabled = 0; - for (int i = 0; i < 8; i++) begin - r[i] = request('h1d, 3200 + i, i * 64); - r[i].src = 3; - send_req(r[i]); - response(r[i], 6, dbid[i]); - end - for (int i = 0; i < 8; i++) begin - int slot = data_order[i]; - write_data(r[slot], dbid[slot], words[slot], '1); - end - // Drain seven initial Comps before new DBID responses compete for credits. - // One-beat writes can otherwise finish their data input before SRAM replies. - while (rsp_granted != 0 || responses.size() != 7) @(negedge vif.clock); - for (int i = 0; i < 11; i++) begin - blocked[i] = request('h1d, 3400, i * 64); - blocked[i].src = 10 + i; - send_req(blocked[i]); - end - repeat (30) @(negedge vif.clock); - if (vif.outstanding != 8 || req_credits != 0) - `uvm_fatal("RSP_STALL", "RSP credit exhaustion did not saturate node and RX queue") - grant_enabled = 1; - for (int i = 0; i < 8; i++) begin - response(r[i], 4, s); - if (s.error || chi_ref_write(model, i * 64, words[i], '1)) - `uvm_fatal("RSP_STALL", "Initial write failed") - end - for (int i = 0; i < 11; i++) begin - response(blocked[i], 6, s); - write_data(blocked[i], s, words[i%8], '1); - response(blocked[i], 4, s); - if (s.error || chi_ref_write(model, i * 64, words[i%8], '1)) - `uvm_fatal("RSP_STALL", "Queued same-TxnID write attributed to wrong source") - end - for (int i = 0; i < 11; i++) read_line(3600 + i, i * 64); - end - `uvm_info("PHASE", "Isolated upper read slots", UVM_LOW) - // Hold lower slots in write-data collection to isolate each upper read slot. - // A second read in that slot exercises arbitration after its own last grant. - for (int target_slot = 4; target_slot < 8; target_slot++) begin - for (int i = 0; i < target_slot; i++) begin - r[i] = request('h1d, 3700 + i, i * 64); - r[i].src = 3; - send_req(r[i]); - response(r[i], 6, dbid[i]); - end - read_line(3800 + target_slot * 2, 0); - read_line(3801 + target_slot * 2, 0); - for (int i = 0; i < target_slot; i++) begin - write_data(r[i], dbid[i], words[i], '1); - response(r[i], 4, s); - if (s.error || chi_ref_write(model, i * 64, words[i], '1)) - `uvm_fatal("ARBITRATION", "Isolated slot damaged held writes") - end - end - `uvm_info("PHASE", "Source and transaction identity", UVM_LOW) - // Slot7 stays live while a lower free slot accepts another transaction from its source. - for (int i = 0; i < 8; i++) begin - r[i] = request('h1d, 3900 + i, i * 64); - r[i].src = 3; - send_req(r[i]); - response(r[i], 6, dbid[i]); - end - for (int i = 0; i < 2; i++) begin - write_data(r[i], dbid[i], words[i], '1); - response(r[i], 4, s); - if (s.error || chi_ref_write(model, i * 64, words[i], '1)) - `uvm_fatal("SOURCE_IDS", "Initial source transaction failed") - end - for (int i = 0; i < 2; i++) begin - r[i] = request('h1d, 3990 + i, i * 64); - r[i].src = 3; - send_req(r[i]); - response(r[i], 6, dbid[i]); - end - repeat (4) @(negedge vif.clock); - for (int i = 0; i < 8; i++) begin - write_data(r[i], dbid[i], words[i], '1); - response(r[i], 4, s); - if (s.error || chi_ref_write(model, i * 64, words[i], '1)) - `uvm_fatal("SOURCE_IDS", "Concurrent source transaction failed") - end - `uvm_info("PHASE", "Reset cancellation", UVM_LOW) - // Cancel a write waiting for payload. SRAM data from completed writes survives. - send_req(request('h1d, 110, 64)); - response(request('h1d, 110, 64), 6, s); - reset_link(); - read_line(111, 64); - write_line(110, 64, '1); - write_line(110, 64, '1); // Reuse is legal only after the first completion. - read_line(112, 64); - repeat (15) @(negedge vif.clock); - if (vif.outstanding != 0 || responses.size() || data_responses.size()) - `uvm_fatal("DRAIN", "Unmatched transactions remain") - // Payload is unconstrained while FLITV=0 and must not create a transaction. - @(negedge vif.clock); - vif.req_flit = '0; - vif.dat_flit = '0; - repeat (2) @(negedge vif.clock); - vif.req_flit = '1; - vif.dat_flit = '1; - repeat (2) @(negedge vif.clock); - vif.req_flit = '0; - vif.dat_flit = '0; - repeat (2) @(negedge vif.clock); - if (vif.outstanding || responses.size() || data_responses.size()) - `uvm_fatal("INVALID_CHANNEL", "Inactive physical payload changed transactions") - `uvm_info("CHECKS", $sformatf("CHI SN-F: %0d checked responses/data beats", checks), UVM_LOW) - chi_ref_destroy(model); -`endif - endtask - task execute(); - vif.reset = 1; - vif.req_pend = 0; - vif.dat_pend = 0; - vif.req_valid = 0; - vif.dat_valid = 0; - vif.req_flit = 0; - vif.dat_flit = 0; - vif.rx_active_req = 0; - vif.tx_active_ack = 0; - vif.rsp_credit = 0; - vif.out_dat_credit = 0; - fork - monitor(); - join_none - scenarios(); - endtask -endclass diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/tx_bad_credit.f b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/tx_bad_credit.f deleted file mode 100644 index 6772bfaeb..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/tx_bad_credit.f +++ /dev/null @@ -1,9 +0,0 @@ -+define+CHI_TX_OVERFLOW -+define+CHI_CREDIT_TB=tx_bad_credit_tb -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/credit_interface.sv -@RESOURCES@/credit_package.sv --F @RTL@/Tx/filelist.f -@RESOURCES@/credit_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/tx_bad_credit.toml b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/tx_bad_credit.toml deleted file mode 100644 index 763078b72..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/tx_bad_credit.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "CHI TX credit overflow" diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/Emit.scala b/arch/src/main/scala/framework/mem-core/chi/src/main/scala/Emit.scala deleted file mode 100644 index 726790415..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/Emit.scala +++ /dev/null @@ -1,42 +0,0 @@ -package memcore.bus.chi - -import memcore.bus.chi.snf.SramEndpoint -import memcore.bus.chi.rnf.{BankedChiCache, RnfParams} - -object Emit extends App { - val p = Params() - require((new RequestFlit(p)).flitWidth == 137) - require((new ResponseFlit(p)).flitWidth == 71) - require((new SnoopFlit(p)).flitWidth == 94) - require((new DataFlit(p)).flitWidth == 389) - - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Tx(flitBits = 32, maxCredits = 4), - firtoolOpts = args, - args = Array("--target-dir", "build/Tx", "--split-verilog") - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Rx(flitBits = 32, depth = 4), - firtoolOpts = args, - args = Array("--target-dir", "build/Rx", "--split-verilog") - ) - for ((dataBits, flitBits) <- Seq(128 -> 240, 256 -> 389, 512 -> 687)) { - val snParams = p.copy(dataBits = dataBits) - require((new DataFlit(snParams)).flitWidth == flitBits) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new SramEndpoint(snParams), - firtoolOpts = args, - args = Array("--target-dir", s"build/SramEndpoint$dataBits", "--split-verilog") - ) - } - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new RequestRetry(p, nodeId = 1), - firtoolOpts = args, - args = Array("--target-dir", "build/RequestRetry", "--split-verilog") - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new BankedChiCache(RnfParams()), - firtoolOpts = args, - args = Array("--target-dir", "build/BankedChiCache", "--split-verilog") - ) -} diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/Fabric.scala b/arch/src/main/scala/framework/mem-core/chi/src/main/scala/Fabric.scala deleted file mode 100644 index ebcbf6a85..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/Fabric.scala +++ /dev/null @@ -1,110 +0,0 @@ -package memcore.bus.chi - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} - -/** Explicit requester-to-Home CHI fabric; no caches or backing-memory ownership. */ -@instantiable -class Fabric(p: Params, requesterCount: Int, homeId: Int) extends Module { - require(requesterCount >= 1 && requesterCount < homeId && BigInt(homeId) < (BigInt(1) << p.nodeIdBits)) - val c = p - - @public - val io = IO(new Bundle { - val active = Input(Bool()) - val blockRequesterRsp = Input(Bool()) - val requesters = Vec(requesterCount, Flipped(new RequesterPort(c))) - val req = Decoupled(new RequestFlit(c)) - val rxRsp = Decoupled(new ResponseFlit(c)) - val rxDat = Decoupled(new DataFlit(c)) - val rsp = Flipped(Decoupled(new ResponseFlit(c))) - val dat = Flipped(Decoupled(new DataFlit(c))) - val snp = Flipped(Decoupled(new DirectedSnoop(c))) - }) - - def creditLink[T <: Flit](source: DecoupledIO[T], sink: DecoupledIO[T], blockRx: Bool = false.B): Unit = { - // Backpressure retains complete flits in the real credit-accounted Rx FIFO. - // Active and FLITV retain their channel semantics throughout the stall. - val tx: Instance[Tx] = Instantiate(new Tx(source.bits.flitWidth, maxCredits = 4)) - val rx: Instance[Rx] = Instantiate(new Rx(source.bits.flitWidth, depth = 4)) - tx.io.active := io.active - rx.io.active := io.active - tx.io.in.valid := source.valid - tx.io.in.bits := source.bits.packed - source.ready := tx.io.in.ready - rx.io.link <> tx.io.link - sink.valid := rx.io.out.valid && !blockRx - sink.bits.unpack(rx.io.out.bits) - rx.io.out.ready := sink.ready && !blockRx - } - - // All three arbiters use initialized history: the default RR lastGrant is not - // reset, so simultaneous first requests can otherwise select unknown data. - def requesterArbiter[T <: Data](flit: T): RRArbiter[T] = Module(new RRArbiter(flit, requesterCount) { - - override lazy val lastGrant = { - val cursor = RegInit(0.U(math.max(1, log2Ceil(requesterCount)).W)) - when(io.out.fire)(cursor := io.chosen) - cursor - } - - }) - - val requests = requesterArbiter(new RequestFlit(c)) - val responses = requesterArbiter(new ResponseFlit(c)) - val data = requesterArbiter(new DataFlit(c)) - io.req <> requests.io.out - io.rxRsp <> responses.io.out - io.rxDat <> data.io.out - for (i <- 0 until requesterCount) { - creditLink(io.requesters(i).req, requests.io.in(i)) - creditLink(io.requesters(i).txRsp, responses.io.in(i), io.blockRequesterRsp) - creditLink(io.requesters(i).txDat, data.io.in(i)) - when(requests.io.in(i).valid) { - assert( - requests.io.in(i).bits.srcId === (i + 1).U && requests.io.in(i).bits.tgtId === homeId.U, - "CacheSystem request crossed a wrong node or role" - ) - } - when(responses.io.in(i).valid)(assert( - responses.io.in(i).bits.srcId === (i + 1).U && - responses.io.in(i).bits.tgtId === homeId.U, - "CacheSystem response crossed a wrong node or role" - )) - when(data.io.in(i).valid)(assert( - data.io.in(i).bits.srcId === (i + 1).U && - data.io.in(i).bits.tgtId === homeId.U, - "CacheSystem data crossed a wrong node or role" - )) - } - val rsp = Wire(Vec(requesterCount, Decoupled(new ResponseFlit(c)))) - val dat = Wire(Vec(requesterCount, Decoupled(new DataFlit(c)))) - val snp = Wire(Vec(requesterCount, Decoupled(new SnoopFlit(c)))) - for (i <- 0 until requesterCount) { - rsp(i).valid := io.rsp.valid && io.rsp.bits.tgtId === (i + 1).U - rsp(i).bits := io.rsp.bits - dat(i).valid := io.dat.valid && io.dat.bits.tgtId === (i + 1).U - dat(i).bits := io.dat.bits - snp(i).valid := io.snp.valid && io.snp.bits.targetNode === (i + 1).U - snp(i).bits := io.snp.bits.flit - creditLink(rsp(i), io.requesters(i).rxRsp) - creditLink(dat(i), io.requesters(i).rxDat) - creditLink(snp(i), io.requesters(i).snp) - } - io.rsp.ready := (0 until requesterCount).map(i => rsp(i).ready && io.rsp.bits.tgtId === (i + 1).U).reduce(_ || _) - io.dat.ready := (0 until requesterCount).map(i => dat(i).ready && io.dat.bits.tgtId === (i + 1).U).reduce(_ || _) - io.snp.ready := (0 until requesterCount).map(i => snp(i).ready && io.snp.bits.targetNode === (i + 1).U).reduce(_ || _) - when(io.rsp.valid)(assert( - io.rsp.bits.tgtId >= 1.U && io.rsp.bits.tgtId <= requesterCount.U, - "CacheSystem response target is not a requester" - )) - when(io.dat.valid)(assert( - io.dat.bits.tgtId >= 1.U && io.dat.bits.tgtId <= requesterCount.U, - "CacheSystem data target is not a requester" - )) - when(io.snp.valid)(assert( - io.snp.bits.targetNode >= 1.U && io.snp.bits.targetNode <= requesterCount.U, - "CacheSystem snoop target is not a requester" - )) -} diff --git a/arch/src/main/scala/framework/mem-core/coherence/build.sc b/arch/src/main/scala/framework/mem-core/coherence/build.sc deleted file mode 100644 index 6d1b6e2c6..000000000 --- a/arch/src/main/scala/framework/mem-core/coherence/build.sc +++ /dev/null @@ -1,39 +0,0 @@ -import mill._ -import mill.scalalib._ - -trait ChiselModule extends SbtModule { - def moduleRoot: os.Path - - override def millSourcePath = moduleRoot - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") -} - -object queue extends SbtModule { - override def millSourcePath = os.pwd / os.up / "queue" - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") -} - -object chi extends ChiselModule { - override def moduleDeps = Seq(queue) - override def moduleRoot = os.pwd / os.up / "chi" -} - -object cache extends ChiselModule { - override def moduleDeps = Seq(queue) - override def moduleRoot = os.pwd / os.up / "cache" - override def sources = T.sources(super.sources() ++ Seq(PathRef(moduleRoot / "configs"))) - override def ivyDeps = super.ivyDeps() ++ Agg(ivy"tech.sparse::toml-scala:0.2.2") -} - -object coherence extends ChiselModule { - override def mainClass = Some("memcore.memory.coherence.Emit") - override def moduleRoot = os.pwd - override def moduleDeps = Seq(chi, cache) - override def sources = T.sources(super.sources() ++ Seq(PathRef(moduleRoot / "configs"))) -} diff --git a/arch/src/main/scala/framework/mem-core/coherence/configs/CoherenceParams.scala b/arch/src/main/scala/framework/mem-core/coherence/configs/CoherenceParams.scala deleted file mode 100644 index 17615701e..000000000 --- a/arch/src/main/scala/framework/mem-core/coherence/configs/CoherenceParams.scala +++ /dev/null @@ -1,53 +0,0 @@ -package memcore.memory.coherence.configs - -import chisel3.util.log2Ceil -import java.nio.file.Paths -import scala.io.Source -import scala.util.Using -import toml.{Toml, Value} -import memcore.bus.chi.Params -import memcore.memory.cache.configs.CacheParams - -case class CoherenceParams( - chi: Params, - cache: CacheParams, - agents: Int, - mshrEntries: Int, - homeId: Int) { - require(cache.lineBytes == 64 && cache.addressBits == chi.addressBits) - require(agents >= 1 && agents < homeId && BigInt(homeId) < (BigInt(1) << chi.nodeIdBits)) - require(mshrEntries >= 1 && BigInt(mshrEntries) <= (BigInt(1) << chi.txnIdBits)) - require(BigInt(mshrEntries) <= (BigInt(1) << cache.idBits)) - val slotBits = math.max(1, log2Ceil(mshrEntries)) - val lineAddressBits = chi.addressBits - cache.offsetBits -} - -object CoherenceParams { - - def load(path: String): CoherenceParams = { - val text = Using.resource(Source.fromFile(path, "UTF-8"))(_.mkString) - val table = Toml.parse(text) match { - case Right(Value.Tbl(root)) => root("coherence") match { - case Value.Tbl(t) => t - case value => throw new IllegalArgumentException(s"Expected [coherence], got $value") - } - case value => throw new IllegalArgumentException(s"Invalid coherence config: $value") - } - def int(name: String): Int = table(name) match { - case Value.Num(n) if n == n.toInt => n.toInt - case value => throw new IllegalArgumentException(s"Expected integer $name, got $value") - } - val cachePath = table("cacheConfig") match { - case Value.Str(value) => Paths.get(path).toAbsolutePath.getParent.resolve(value).normalize.toString - case value => throw new IllegalArgumentException(s"Expected cacheConfig path, got $value") - } - CoherenceParams( - Params(int("nodeIdBits"), int("addressBits"), int("dataBits")), - CacheParams.load(cachePath), - int("agents"), - int("mshrEntries"), - int("homeId") - ) - } - -} diff --git a/arch/src/main/scala/framework/mem-core/coherence/src/csrc/Cargo.lock b/arch/src/main/scala/framework/mem-core/coherence/src/csrc/Cargo.lock deleted file mode 100644 index af0855ac8..000000000 --- a/arch/src/main/scala/framework/mem-core/coherence/src/csrc/Cargo.lock +++ /dev/null @@ -1,7 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "coherence-reference-model" -version = "0.1.0" diff --git a/arch/src/main/scala/framework/mem-core/coherence/src/csrc/Cargo.toml b/arch/src/main/scala/framework/mem-core/coherence/src/csrc/Cargo.toml deleted file mode 100644 index d48bbdc57..000000000 --- a/arch/src/main/scala/framework/mem-core/coherence/src/csrc/Cargo.toml +++ /dev/null @@ -1,7 +0,0 @@ -[package] -name = "coherence-reference-model" -version = "0.1.0" -edition = "2021" - -[lib] -crate-type = ["cdylib"] diff --git a/arch/src/main/scala/framework/mem-core/coherence/src/csrc/src/memory.rs b/arch/src/main/scala/framework/mem-core/coherence/src/csrc/src/memory.rs deleted file mode 100644 index 87260303f..000000000 --- a/arch/src/main/scala/framework/mem-core/coherence/src/csrc/src/memory.rs +++ /dev/null @@ -1,36 +0,0 @@ -use std::collections::HashMap; -use std::ffi::c_void; - -#[no_mangle] -pub extern "C" fn coherence_ref_create() -> *mut c_void { - Box::into_raw(Box::new(HashMap::::new())).cast() -} -#[no_mangle] -pub unsafe extern "C" fn coherence_ref_destroy(model: *mut c_void) { - drop(Box::from_raw(model.cast::>())); -} -#[no_mangle] -pub unsafe extern "C" fn coherence_ref_initial(addr: u64, out: *mut u32) { - let out = std::slice::from_raw_parts_mut(out, 16); - for (i, word) in out.iter_mut().enumerate() { - *word = ((addr as u32) ^ ((addr >> 32) as u32)).rotate_left(i as u32) - ^ (0x10203040u32.wrapping_mul(i as u32 + 1)); - } -} -#[no_mangle] -pub unsafe extern "C" fn coherence_ref_read(model: *mut c_void, addr: u64, out: *mut u32) { - let memory = &mut *model.cast::>(); - if let Some(data) = memory.get(&addr) { - std::slice::from_raw_parts_mut(out, 16).copy_from_slice(data); - } else { - coherence_ref_initial(addr, out); - } -} -#[no_mangle] -pub unsafe extern "C" fn coherence_ref_write(model: *mut c_void, addr: u64, data: *const u32) { - let memory = &mut *model.cast::>(); - memory.insert( - addr, - std::slice::from_raw_parts(data, 16).try_into().unwrap(), - ); -} diff --git a/arch/src/main/scala/framework/mem-core/coherence/src/main/resources/coherence.el b/arch/src/main/scala/framework/mem-core/coherence/src/main/resources/coherence.el deleted file mode 100644 index aa66a6577..000000000 --- a/arch/src/main/scala/framework/mem-core/coherence/src/main/resources/coherence.el +++ /dev/null @@ -1,316 +0,0 @@ -// Format Version: 2 -// Reviewed for 2 CPUs, 4 MSHRs, 4 sets, 2 ways, 64-byte lines, 256-bit CHI data. - -CHECKSUM: "848348206 1645746301" -INSTANCE: coherence_tb.dut.cache -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Unknown Cache operation" -Block 10 "1190204013" "if (1)" -Block 11 "3885794289" "$error(\"Assertion failed: Unknown Cache operation\n at Cache.scala:73 assert(req.op <= CacheOp.Invalidate.U, \\"Unknown Cache operation\\")\n\");" -Block 13 "1190204013" "if (1)" -Block 14 "3662014904" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Cache operations require a line-aligned address" -Block 18 "1317833989" "if (1)" -Block 19 "1722205720" "$error(\"Assertion failed: Cache operations require a line-aligned address\n at Cache.scala:74 assert(req.addr(p.offsetBits - 1, 0) === 0.U, \\"Cache operations require a line-aligned address\\")\n\");" -Block 21 "1317833989" "if (1)" -Block 22 "505164254" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Duplicate Cache tag" -Block 26 "3036749987" "if (1)" -Block 27 "2991635828" "$error(\"Assertion failed: Duplicate Cache tag\n at Cache.scala:76 assert(PopCount(hits) <= 1.U, \\"Duplicate Cache tag\\")\n\");" -Block 29 "3036749987" "if (1)" -Block 30 "3772797571" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Cache fill would create a duplicate tag" -Block 34 "3744269243" "if (1)" -Block 35 "987059305" "$error(\"Assertion failed: Cache fill would create a duplicate tag\n at Cache.scala:89 assert(!hits.orR || hits(req.way), \\"Cache fill would create a duplicate tag\\")\n\");" -Block 37 "3744269243" "if (1)" -Block 38 "629733424" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Cache write requires the matching resident line" -Block 42 "4266678685" "if (1)" -Block 43 "4034066179" "$error(\"Assertion failed: Cache write requires the matching resident line\n at Cache.scala:99 assert(entryValid && hits(req.way), \\"Cache write requires the matching resident line\\")\n\");" -Block 45 "4266678685" "if (1)" -Block 46 "1550685438" "$fatal;" - -CHECKSUM: "25866875 2457151256" -INSTANCE: coherence_tb.dut.directory -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Multiple transactions update one directory entry" -Block 2 "2664146107" "if (1)" -Block 3 "3533917897" "$error(\"Assertion failed: Multiple transactions update one directory entry\n at Directory.scala:43 assert(PopCount(writers) <= 1.U, \\"Multiple transactions update one directory entry\\")\n\");" -Block 5 "2664146107" "if (1)" -Block 6 "3472375403" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Unique ownership requires exactly one CPU" -Block 10 "2881929055" "if (1)" -Block 11 "995660494" "$error(\"Assertion failed: Unique ownership requires exactly one CPU\n at Directory.scala:46 assert(!next.unique || PopCount(next.sharers) === 1.U, \\"Unique ownership requires exactly one CPU\\")\n\");" -Block 13 "2881929055" "if (1)" -Block 14 "1126102520" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Multiple transactions update one directory entry" -Block 18 "1723217575" "if (1)" -Block 19 "3904278350" "$error(\"Assertion failed: Multiple transactions update one directory entry\n at Directory.scala:43 assert(PopCount(writers) <= 1.U, \\"Multiple transactions update one directory entry\\")\n\");" -Block 21 "1723217575" "if (1)" -Block 22 "4109238252" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Unique ownership requires exactly one CPU" -Block 26 "3405939620" "if (1)" -Block 27 "684207841" "$error(\"Assertion failed: Unique ownership requires exactly one CPU\n at Directory.scala:46 assert(!next.unique || PopCount(next.sharers) === 1.U, \\"Unique ownership requires exactly one CPU\\")\n\");" -Block 29 "3405939620" "if (1)" -Block 30 "1351502295" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Multiple transactions update one directory entry" -Block 34 "3919299312" "if (1)" -Block 35 "2315541197" "$error(\"Assertion failed: Multiple transactions update one directory entry\n at Directory.scala:43 assert(PopCount(writers) <= 1.U, \\"Multiple transactions update one directory entry\\")\n\");" -Block 37 "3919299312" "if (1)" -Block 38 "2522819183" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Unique ownership requires exactly one CPU" -Block 42 "539207809" "if (1)" -Block 43 "2227510213" "$error(\"Assertion failed: Unique ownership requires exactly one CPU\n at Directory.scala:46 assert(!next.unique || PopCount(next.sharers) === 1.U, \\"Unique ownership requires exactly one CPU\\")\n\");" -Block 45 "539207809" "if (1)" -Block 46 "4236475635" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Multiple transactions update one directory entry" -Block 50 "4085285969" "if (1)" -Block 51 "2434010258" "$error(\"Assertion failed: Multiple transactions update one directory entry\n at Directory.scala:43 assert(PopCount(writers) <= 1.U, \\"Multiple transactions update one directory entry\\")\n\");" -Block 53 "4085285969" "if (1)" -Block 54 "2370796592" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Unique ownership requires exactly one CPU" -Block 58 "3010328178" "if (1)" -Block 59 "839440352" "$error(\"Assertion failed: Unique ownership requires exactly one CPU\n at Directory.scala:46 assert(!next.unique || PopCount(next.sharers) === 1.U, \\"Unique ownership requires exactly one CPU\\")\n\");" -Block 61 "3010328178" "if (1)" -Block 62 "1246671062" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Multiple transactions update one directory entry" -Block 66 "2091741190" "if (1)" -Block 67 "4087798033" "$error(\"Assertion failed: Multiple transactions update one directory entry\n at Directory.scala:43 assert(PopCount(writers) <= 1.U, \\"Multiple transactions update one directory entry\\")\n\");" -Block 69 "2091741190" "if (1)" -Block 70 "4026378675" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Unique ownership requires exactly one CPU" -Block 74 "2992832868" "if (1)" -Block 75 "730445366" "$error(\"Assertion failed: Unique ownership requires exactly one CPU\n at Directory.scala:46 assert(!next.unique || PopCount(next.sharers) === 1.U, \\"Unique ownership requires exactly one CPU\\")\n\");" -Block 77 "2992832868" "if (1)" -Block 78 "1406128384" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Multiple transactions update one directory entry" -Block 82 "2228281882" "if (1)" -Block 83 "3383957654" "$error(\"Assertion failed: Multiple transactions update one directory entry\n at Directory.scala:43 assert(PopCount(writers) <= 1.U, \\"Multiple transactions update one directory entry\\")\n\");" -Block 85 "2228281882" "if (1)" -Block 86 "3588778036" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Unique ownership requires exactly one CPU" -Block 90 "3031124909" "if (1)" -Block 91 "4279530673" "$error(\"Assertion failed: Unique ownership requires exactly one CPU\n at Directory.scala:46 assert(!next.unique || PopCount(next.sharers) === 1.U, \\"Unique ownership requires exactly one CPU\\")\n\");" -Block 93 "3031124909" "if (1)" -Block 94 "2270303111" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Multiple transactions update one directory entry" -Block 98 "201135693" "if (1)" -Block 99 "2869021973" "$error(\"Assertion failed: Multiple transactions update one directory entry\n at Directory.scala:43 assert(PopCount(writers) <= 1.U, \\"Multiple transactions update one directory entry\\")\n\");" -Block 101 "201135693" "if (1)" -Block 102 "3076177335" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Unique ownership requires exactly one CPU" -Block 106 "3770691673" "if (1)" -Block 107 "389918641" "$error(\"Assertion failed: Unique ownership requires exactly one CPU\n at Directory.scala:46 assert(!next.unique || PopCount(next.sharers) === 1.U, \\"Unique ownership requires exactly one CPU\\")\n\");" -Block 109 "3770691673" "if (1)" -Block 110 "1870383239" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Multiple transactions update one directory entry" -Block 114 "3662072233" "if (1)" -Block 115 "3809633553" "$error(\"Assertion failed: Multiple transactions update one directory entry\n at Directory.scala:43 assert(PopCount(writers) <= 1.U, \\"Multiple transactions update one directory entry\\")\n\");" -Block 117 "3662072233" "if (1)" -Block 118 "4282979763" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Unique ownership requires exactly one CPU" -Block 122 "1829712518" "if (1)" -Block 123 "4024367402" "$error(\"Assertion failed: Unique ownership requires exactly one CPU\n at Directory.scala:46 assert(!next.unique || PopCount(next.sharers) === 1.U, \\"Unique ownership requires exactly one CPU\\")\n\");" -Block 125 "1829712518" "if (1)" -Block 126 "2543361564" "$fatal;" - -CHECKSUM: "3226703875 2930150559" -INSTANCE: coherence_tb.dut.cache.responses -ANNOTATION: "Source-proven Home profile: MSHR IDs 0..3, requester IDs 1..2, aligned 64-byte addresses, full-line masks, all ways eligible, dirty metadata bit 0; CHI optional fields are assigned constants. Only the specified constant bits are excluded." -Toggle io_enq_bits_id [7:2] "net io_enq_bits_id[7:0]" -Toggle io_enq_bits_available "net io_enq_bits_available" -Toggle io_enq_bits_addr [5:0] "net io_enq_bits_addr[43:0]" -Toggle io_enq_bits_metadata [3:1] "net io_enq_bits_metadata[3:0]" -Toggle io_deq_bits_id [7:2] "net io_deq_bits_id[7:0]" -Toggle io_deq_bits_available "net io_deq_bits_available" -Toggle io_deq_bits_addr [5:0] "net io_deq_bits_addr[43:0]" -Toggle io_deq_bits_metadata [3:1] "net io_deq_bits_metadata[3:0]" - -CHECKSUM: "1696518855 210682840" -INSTANCE: coherence_tb.dut.datQueue -ANNOTATION: "Source-proven Home profile: MSHR IDs 0..3, requester IDs 1..2, aligned 64-byte addresses, full-line masks, all ways eligible, dirty metadata bit 0; CHI optional fields are assigned constants. Only the specified constant bits are excluded." -Toggle io_enq_bits_tgtId [6:2] "net io_enq_bits_tgtId[6:0]" -Toggle io_enq_bits_resp [2] "net io_enq_bits_resp[2:0]" -Toggle io_enq_bits_dbid [15:2] "net io_enq_bits_dbid[15:0]" -Toggle io_enq_bits_dataId [0] "net io_enq_bits_dataId[1:0]" -Toggle io_deq_bits_qos [3:0] "net io_deq_bits_qos[3:0]" -Toggle io_deq_bits_tgtId [6:2] "net io_deq_bits_tgtId[6:0]" -Toggle io_deq_bits_srcId [6:0] "net io_deq_bits_srcId[6:0]" -Toggle io_deq_bits_homeNid [6:0] "net io_deq_bits_homeNid[6:0]" -Toggle io_deq_bits_opcode [3:0] "net io_deq_bits_opcode[3:0]" -Toggle io_deq_bits_resp [2] "net io_deq_bits_resp[2:0]" -Toggle io_deq_bits_dataSource [7:0] "net io_deq_bits_dataSource[7:0]" -Toggle io_deq_bits_dataPull "net io_deq_bits_dataPull" -Toggle io_deq_bits_cBusy [2:0] "net io_deq_bits_cBusy[2:0]" -Toggle io_deq_bits_dbid [15:2] "net io_deq_bits_dbid[15:0]" -Toggle io_deq_bits_ccid [1:0] "net io_deq_bits_ccid[1:0]" -Toggle io_deq_bits_dataId [0] "net io_deq_bits_dataId[1:0]" -Toggle io_deq_bits_cacheLineId [5:0] "net io_deq_bits_cacheLineId[5:0]" -Toggle io_deq_bits_tagOp [1:0] "net io_deq_bits_tagOp[1:0]" -Toggle io_deq_bits_tag [7:0] "net io_deq_bits_tag[7:0]" -Toggle io_deq_bits_tagUpdate [1:0] "net io_deq_bits_tagUpdate[1:0]" -Toggle io_deq_bits_traceTag "net io_deq_bits_traceTag" -Toggle io_deq_bits_copyAtHome "net io_deq_bits_copyAtHome" -Toggle io_deq_bits_numDat [1:0] "net io_deq_bits_numDat[1:0]" -Toggle io_deq_bits_replicate "net io_deq_bits_replicate" - -CHECKSUM: "1414150944 1331432527" -INSTANCE: coherence_tb.dut.cacheQueue -ANNOTATION: "Source-proven Home profile: MSHR IDs 0..3, requester IDs 1..2, aligned 64-byte addresses, full-line masks, all ways eligible, dirty metadata bit 0; CHI optional fields are assigned constants. Only the specified constant bits are excluded." -Toggle io_enq_bits_id [7:2] "net io_enq_bits_id[7:0]" -Toggle io_enq_bits_addr [5:0] "net io_enq_bits_addr[43:0]" -Toggle io_enq_bits_metadata [3:1] "net io_enq_bits_metadata[3:0]" -Toggle io_deq_bits_id [7:2] "net io_deq_bits_id[7:0]" -Toggle io_deq_bits_addr [5:0] "net io_deq_bits_addr[43:0]" -Toggle io_deq_bits_mask [63:0] "net io_deq_bits_mask[63:0]" -Toggle io_deq_bits_metadata [3:1] "net io_deq_bits_metadata[3:0]" -Toggle io_deq_bits_eligible [1:0] "net io_deq_bits_eligible[1:0]" - -CHECKSUM: "3112608224 3447389693" -INSTANCE: coherence_tb.dut.memQueue -ANNOTATION: "Source-proven Home profile: MSHR IDs 0..3, requester IDs 1..2, aligned 64-byte addresses, full-line masks, all ways eligible, dirty metadata bit 0; CHI optional fields are assigned constants. Only the specified constant bits are excluded." -Toggle io_enq_bits_id [11:2] "net io_enq_bits_id[11:0]" -Toggle io_enq_bits_addr [5:0] "net io_enq_bits_addr[43:0]" -Toggle io_deq_bits_id [11:2] "net io_deq_bits_id[11:0]" -Toggle io_deq_bits_addr [5:0] "net io_deq_bits_addr[43:0]" -Toggle io_deq_bits_mask [63:0] "net io_deq_bits_mask[63:0]" - -CHECKSUM: "769480049 1064374571" -INSTANCE: coherence_tb.dut.rspQueue -ANNOTATION: "Source-proven Home profile: MSHR IDs 0..3, requester IDs 1..2, aligned 64-byte addresses, full-line masks, all ways eligible, dirty metadata bit 0; CHI optional fields are assigned constants. Only the specified constant bits are excluded." -Toggle io_enq_bits_tgtId [6:2] "net io_enq_bits_tgtId[6:0]" -Toggle io_enq_bits_opcode [4:1] "net io_enq_bits_opcode[4:0]" -Toggle io_enq_bits_dbid [11:2] "net io_enq_bits_dbid[11:0]" -Toggle io_deq_bits_qos [3:0] "net io_deq_bits_qos[3:0]" -Toggle io_deq_bits_tgtId [6:2] "net io_deq_bits_tgtId[6:0]" -Toggle io_deq_bits_srcId [6:0] "net io_deq_bits_srcId[6:0]" -Toggle io_deq_bits_opcode [4:1] "net io_deq_bits_opcode[4:0]" -Toggle io_deq_bits_resp [2:0] "net io_deq_bits_resp[2:0]" -Toggle io_deq_bits_fwdState [2:0] "net io_deq_bits_fwdState[2:0]" -Toggle io_deq_bits_cBusy [2:0] "net io_deq_bits_cBusy[2:0]" -Toggle io_deq_bits_dbid [11:2] "net io_deq_bits_dbid[11:0]" -Toggle io_deq_bits_pCrdType [3:0] "net io_deq_bits_pCrdType[3:0]" -Toggle io_deq_bits_tagOp [1:0] "net io_deq_bits_tagOp[1:0]" -Toggle io_deq_bits_traceTag "net io_deq_bits_traceTag" -Toggle io_deq_bits_cacheLineId [5:0] "net io_deq_bits_cacheLineId[5:0]" - -CHECKSUM: "2391098903 4025403226" -INSTANCE: coherence_tb.dut.snpQueue -ANNOTATION: "Source-proven Home profile: MSHR IDs 0..3, requester IDs 1..2, aligned 64-byte addresses, full-line masks, all ways eligible, dirty metadata bit 0; CHI optional fields are assigned constants. Only the specified constant bits are excluded." -Toggle io_enq_bits_targetNode [6:2] "net io_enq_bits_targetNode[6:0]" -Toggle io_enq_bits_flit_txnId [11:2] "net io_enq_bits_flit_txnId[11:0]" -Toggle io_enq_bits_flit_opcode [4] "net io_enq_bits_flit_opcode[4:0]" -Toggle io_enq_bits_flit_addr [2:0] "net io_enq_bits_flit_addr[40:0]" -Toggle io_deq_bits_targetNode [6:2] "net io_deq_bits_targetNode[6:0]" -Toggle io_deq_bits_flit_qos [3:0] "net io_deq_bits_flit_qos[3:0]" -Toggle io_deq_bits_flit_srcId [6:0] "net io_deq_bits_flit_srcId[6:0]" -Toggle io_deq_bits_flit_txnId [11:2] "net io_deq_bits_flit_txnId[11:0]" -Toggle io_deq_bits_flit_fwdNid [6:0] "net io_deq_bits_flit_fwdNid[6:0]" -Toggle io_deq_bits_flit_fwdTxnId [11:0] "net io_deq_bits_flit_fwdTxnId[11:0]" -Toggle io_deq_bits_flit_opcode [4] "net io_deq_bits_flit_opcode[4:0]" -Toggle io_deq_bits_flit_addr [2:0] "net io_deq_bits_flit_addr[40:0]" -Toggle io_deq_bits_flit_pas [2:0] "net io_deq_bits_flit_pas[2:0]" -Toggle io_deq_bits_flit_doNotGoToSd "net io_deq_bits_flit_doNotGoToSd" -Toggle io_deq_bits_flit_traceTag "net io_deq_bits_flit_traceTag" - -CHECKSUM: "1853676028 3335661445" -INSTANCE: coherence_tb.dut.cacheArb -ANNOTATION: "Source-proven Home profile: MSHR IDs 0..3, requester IDs 1..2, aligned 64-byte addresses, full-line masks, all ways eligible, dirty metadata bit 0; CHI optional fields are assigned constants. Only the specified constant bits are excluded." -Toggle io_in_0_bits_addr [5:0] "net io_in_0_bits_addr[43:0]" -Toggle io_in_0_bits_metadata [3:1] "net io_in_0_bits_metadata[3:0]" -Toggle io_in_1_bits_addr [5:0] "net io_in_1_bits_addr[43:0]" -Toggle io_in_1_bits_metadata [3:1] "net io_in_1_bits_metadata[3:0]" -Toggle io_in_2_bits_addr [5:0] "net io_in_2_bits_addr[43:0]" -Toggle io_in_2_bits_metadata [3:1] "net io_in_2_bits_metadata[3:0]" -Toggle io_in_3_bits_addr [5:0] "net io_in_3_bits_addr[43:0]" -Toggle io_in_3_bits_metadata [3:1] "net io_in_3_bits_metadata[3:0]" -Toggle io_out_bits_id [7:2] "net io_out_bits_id[7:0]" -Toggle io_out_bits_addr [5:0] "net io_out_bits_addr[43:0]" -Toggle io_out_bits_metadata [3:1] "net io_out_bits_metadata[3:0]" - -CHECKSUM: "3130163852 2143613990" -INSTANCE: coherence_tb.dut.memArb -ANNOTATION: "Source-proven Home profile: MSHR IDs 0..3, requester IDs 1..2, aligned 64-byte addresses, full-line masks, all ways eligible, dirty metadata bit 0; CHI optional fields are assigned constants. Only the specified constant bits are excluded." -Toggle io_in_0_bits_addr [5:0] "net io_in_0_bits_addr[43:0]" -Toggle io_in_1_bits_addr [5:0] "net io_in_1_bits_addr[43:0]" -Toggle io_in_2_bits_addr [5:0] "net io_in_2_bits_addr[43:0]" -Toggle io_in_3_bits_addr [5:0] "net io_in_3_bits_addr[43:0]" -Toggle io_out_bits_id [11:2] "net io_out_bits_id[11:0]" -Toggle io_out_bits_addr [5:0] "net io_out_bits_addr[43:0]" - -CHECKSUM: "3716118738 1796417120" -INSTANCE: coherence_tb.dut.snpArb -ANNOTATION: "Source-proven Home profile: MSHR IDs 0..3, requester IDs 1..2, aligned 64-byte addresses, full-line masks, all ways eligible, dirty metadata bit 0; CHI optional fields are assigned constants. Only the specified constant bits are excluded." -Toggle io_in_0_bits_targetNode [6:2] "net io_in_0_bits_targetNode[6:0]" -Toggle io_in_0_bits_flit_opcode [4] "net io_in_0_bits_flit_opcode[4:0]" -Toggle io_in_0_bits_flit_addr [2:0] "net io_in_0_bits_flit_addr[40:0]" -Toggle io_in_1_bits_targetNode [6:2] "net io_in_1_bits_targetNode[6:0]" -Toggle io_in_1_bits_flit_opcode [4] "net io_in_1_bits_flit_opcode[4:0]" -Toggle io_in_1_bits_flit_addr [2:0] "net io_in_1_bits_flit_addr[40:0]" -Toggle io_in_2_bits_targetNode [6:2] "net io_in_2_bits_targetNode[6:0]" -Toggle io_in_2_bits_flit_opcode [4] "net io_in_2_bits_flit_opcode[4:0]" -Toggle io_in_2_bits_flit_addr [2:0] "net io_in_2_bits_flit_addr[40:0]" -Toggle io_in_3_bits_targetNode [6:2] "net io_in_3_bits_targetNode[6:0]" -Toggle io_in_3_bits_flit_opcode [4] "net io_in_3_bits_flit_opcode[4:0]" -Toggle io_in_3_bits_flit_addr [2:0] "net io_in_3_bits_flit_addr[40:0]" -Toggle io_out_bits_targetNode [6:2] "net io_out_bits_targetNode[6:0]" -Toggle io_out_bits_flit_txnId [11:2] "net io_out_bits_flit_txnId[11:0]" -Toggle io_out_bits_flit_opcode [4] "net io_out_bits_flit_opcode[4:0]" -Toggle io_out_bits_flit_addr [2:0] "net io_out_bits_flit_addr[40:0]" - -CHECKSUM: "2004716196 3529883256" -INSTANCE: coherence_tb.dut.rspArb -ANNOTATION: "Source-proven Home profile: MSHR IDs 0..3, requester IDs 1..2, aligned 64-byte addresses, full-line masks, all ways eligible, dirty metadata bit 0; CHI optional fields are assigned constants. Only the specified constant bits are excluded." -Toggle io_in_0_bits_tgtId [6:2] "net io_in_0_bits_tgtId[6:0]" -Toggle io_in_0_bits_opcode [4:1] "net io_in_0_bits_opcode[4:0]" -Toggle io_in_1_bits_tgtId [6:2] "net io_in_1_bits_tgtId[6:0]" -Toggle io_in_1_bits_opcode [4:1] "net io_in_1_bits_opcode[4:0]" -Toggle io_in_2_bits_tgtId [6:2] "net io_in_2_bits_tgtId[6:0]" -Toggle io_in_2_bits_opcode [4:1] "net io_in_2_bits_opcode[4:0]" -Toggle io_in_3_bits_tgtId [6:2] "net io_in_3_bits_tgtId[6:0]" -Toggle io_in_3_bits_opcode [4:1] "net io_in_3_bits_opcode[4:0]" -Toggle io_out_bits_tgtId [6:2] "net io_out_bits_tgtId[6:0]" -Toggle io_out_bits_opcode [4:1] "net io_out_bits_opcode[4:0]" -Toggle io_out_bits_dbid [11:2] "net io_out_bits_dbid[11:0]" - -CHECKSUM: "3496838959 3135397854" -INSTANCE: coherence_tb.dut.datArb -ANNOTATION: "Source-proven Home profile: MSHR IDs 0..3, requester IDs 1..2, aligned 64-byte addresses, full-line masks, all ways eligible, dirty metadata bit 0; CHI optional fields are assigned constants. Only the specified constant bits are excluded." -Toggle io_in_0_bits_tgtId [6:2] "net io_in_0_bits_tgtId[6:0]" -Toggle io_in_0_bits_resp [2] "net io_in_0_bits_resp[2:0]" -Toggle io_in_0_bits_dataId [0] "net io_in_0_bits_dataId[1:0]" -Toggle io_in_1_bits_tgtId [6:2] "net io_in_1_bits_tgtId[6:0]" -Toggle io_in_1_bits_resp [2] "net io_in_1_bits_resp[2:0]" -Toggle io_in_1_bits_dataId [0] "net io_in_1_bits_dataId[1:0]" -Toggle io_in_2_bits_tgtId [6:2] "net io_in_2_bits_tgtId[6:0]" -Toggle io_in_2_bits_resp [2] "net io_in_2_bits_resp[2:0]" -Toggle io_in_2_bits_dataId [0] "net io_in_2_bits_dataId[1:0]" -Toggle io_in_3_bits_tgtId [6:2] "net io_in_3_bits_tgtId[6:0]" -Toggle io_in_3_bits_resp [2] "net io_in_3_bits_resp[2:0]" -Toggle io_in_3_bits_dataId [0] "net io_in_3_bits_dataId[1:0]" -Toggle io_out_bits_tgtId [6:2] "net io_out_bits_tgtId[6:0]" -Toggle io_out_bits_resp [2] "net io_out_bits_resp[2:0]" -Toggle io_out_bits_dbid [15:2] "net io_out_bits_dbid[15:0]" -Toggle io_out_bits_dataId [0] "net io_out_bits_dataId[1:0]" - -CHECKSUM: "848348206 843507997" -INSTANCE: coherence_tb.dut.cache -ANNOTATION: "Source-proven Home profile: MSHR IDs 0..3, requester IDs 1..2, aligned 64-byte addresses, full-line masks, all ways eligible, dirty metadata bit 0; CHI optional fields are assigned constants. Only the specified constant bits are excluded." -Toggle io_request_bits_id [7:2] "net io_request_bits_id[7:0]" -Toggle io_request_bits_addr [5:0] "net io_request_bits_addr[43:0]" -Toggle io_request_bits_mask [63:0] "net io_request_bits_mask[63:0]" -Toggle io_request_bits_metadata [3:1] "net io_request_bits_metadata[3:0]" -Toggle io_request_bits_eligible [1:0] "net io_request_bits_eligible[1:0]" -Toggle io_response_bits_id [7:2] "net io_response_bits_id[7:0]" -Toggle io_response_bits_available "net io_response_bits_available" -Toggle io_response_bits_addr [5:0] "net io_response_bits_addr[43:0]" -Toggle io_response_bits_metadata [3:1] "net io_response_bits_metadata[3:0]" -Toggle metadata_0_0 [3:1] "reg metadata_0_0[3:0]" -Toggle metadata_0_1 [3:1] "reg metadata_0_1[3:0]" -Toggle metadata_0_2 [3:1] "reg metadata_0_2[3:0]" -Toggle metadata_0_3 [3:1] "reg metadata_0_3[3:0]" -Toggle metadata_1_0 [3:1] "reg metadata_1_0[3:0]" -Toggle metadata_1_1 [3:1] "reg metadata_1_1[3:0]" -Toggle metadata_1_2 [3:1] "reg metadata_1_2[3:0]" -Toggle metadata_1_3 [3:1] "reg metadata_1_3[3:0]" -Toggle result_id [7:2] "reg result_id[7:0]" -Toggle result_addr [5:0] "reg result_addr[43:0]" -Toggle result_metadata [3:1] "reg result_metadata[3:0]" diff --git a/arch/src/main/scala/framework/mem-core/coherence/src/main/resources/coherence.f b/arch/src/main/scala/framework/mem-core/coherence/src/main/resources/coherence.f deleted file mode 100644 index 126bf5261..000000000 --- a/arch/src/main/scala/framework/mem-core/coherence/src/main/resources/coherence.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/interface.sv -@RESOURCES@/package.sv --F @RTL@/Coherence/filelist.f -@RESOURCES@/coherence_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/coherence/src/main/resources/env.svh b/arch/src/main/scala/framework/mem-core/coherence/src/main/resources/env.svh deleted file mode 100644 index 1c4c2b110..000000000 --- a/arch/src/main/scala/framework/mem-core/coherence/src/main/resources/env.svh +++ /dev/null @@ -1,393 +0,0 @@ -class env extends uvm_env; - `uvm_component_utils(env) - virtual coherence_control_if control; - virtual stream_if #(`COH_REQ_WIDTH) req; - virtual stream_if #(`COH_RSP_WIDTH) rx_rsp; - virtual stream_if #(`COH_DAT_WIDTH) rx_dat; - virtual stream_if #(`COH_TXRSP_WIDTH) tx_rsp; - virtual stream_if #(`COH_TXDAT_WIDTH) tx_dat; - virtual stream_if #(`COH_SNP_CHANNEL_WIDTH) snp; - virtual stream_if #(`COH_MEMREQ_WIDTH) mem_req; - virtual stream_if #(`COH_MEMRESP_WIDTH) mem_resp; - keyed_scoreboard #(completion) scoreboard; - chandle model; - req_t requests[int]; - logic [511:0] assembled[int]; - bit [`COH_BEATS-1:0] received[int]; - int data_dbid[int]; - logic [2:0] data_permission[int]; - logic [1:0] data_error[int]; - int expected_order[$]; - bit expected_error[int]; - private_line_t cpus[`COH_AGENTS][longint unsigned]; - bit [511:0] memory[longint unsigned]; - bit dirty_addresses[longint unsigned]; - int clean_shared_requests[longint unsigned]; - rsp_t rsp_queue[$], ack_queue[$]; - dat_t dat_queue[$]; - mem_t mem_queue[$]; - bit [`COH_MSHRS-1:0] hold_ack = 0; - bit vary_idle_payload = 0; - bit idle_payload_one = 0; - bit hold_memory = 0, reverse_memory = 0, saw_reorder = 0; - bit rsp_active = 0, dat_active = 0, mem_active = 0; - rsp_t rsp_packet; - dat_t dat_packet; - mem_t mem_packet; - int memory_writes = 0, memory_reads = 0, peak = 0; - int cycle = 0; - int snoops = 0; - bit hold_rsp = 0, hold_snp = 0, hold_mem_req = 0; - bit hold_copyback = 0; - bit fault_enabled = 0, write_fault_enabled = 0; - longint unsigned fault_addr; - - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - scoreboard = keyed_scoreboard#(completion)::type_id::create("scoreboard", this); - model = coherence_ref_create(); - if (!uvm_config_db#(virtual coherence_control_if)::get( - this, "", "control", control - ) || !uvm_config_db#(virtual stream_if #(`COH_REQ_WIDTH))::get( - this, "", "req", req - ) || !uvm_config_db#(virtual stream_if #(`COH_RSP_WIDTH))::get( - this, "", "rx_rsp", rx_rsp - ) || !uvm_config_db#(virtual stream_if #(`COH_DAT_WIDTH))::get( - this, "", "rx_dat", rx_dat - ) || !uvm_config_db#(virtual stream_if #(`COH_TXRSP_WIDTH))::get( - this, "", "tx_rsp", tx_rsp - ) || !uvm_config_db#(virtual stream_if #(`COH_TXDAT_WIDTH))::get( - this, "", "tx_dat", tx_dat - ) || !uvm_config_db#(virtual stream_if #(`COH_SNP_CHANNEL_WIDTH))::get( - this, "", "snp", snp - ) || !uvm_config_db#(virtual stream_if #(`COH_MEMREQ_WIDTH))::get( - this, "", "mem_req", mem_req - ) || !uvm_config_db#(virtual stream_if #(`COH_MEMRESP_WIDTH))::get( - this, "", "mem_resp", mem_resp - )) - `uvm_fatal("VIF", "Coherence interfaces missing") - endfunction - function void cpu_write(int node, longint unsigned addr, bit [511:0] value); - if (!cpus[node-1].exists(addr) || !cpus[node-1][addr].valid || !cpus[node-1][addr].unique_owner) - `uvm_fatal("CPU", "write without unique permission") - cpus[node-1][addr].data = value; - cpus[node-1][addr].dirty = 1; - coherence_ref_write(model, addr, value); - dirty_addresses[addr] = 1; - endfunction - function void finish(completion item); - int key = (item.node << 12) | item.txn; - int position = -1; - item.set_transaction_id(key); - foreach (expected_order[i]) if (expected_order[i] == key) position = i; - if (position < 0) `uvm_fatal("ORDER", "unexpected completion") - if (position != 0) saw_reorder = 1; - expected_order.delete(position); - scoreboard.actual_export.write(item); - if (requests[key][`CF(REQ, OPCODE)] == `COH_CLEAN_SHARED) begin - longint unsigned address = requests[key][`CF(REQ, ADDR)]; - clean_shared_requests[address]--; - if (clean_shared_requests[address] == 0) clean_shared_requests.delete(address); - end - requests.delete(key); - endfunction - - task run_phase(uvm_phase phase); - forever begin - @(posedge control.clock); - if (control.reset) begin - if (dirty_addresses.num() != 0) - `uvm_fatal("RESET", "test reset before dirty data reached memory") - rx_rsp.valid <= 0; - rx_rsp.bits <= '0; - rx_dat.valid <= 0; - rx_dat.bits <= '0; - mem_resp.valid <= 0; - mem_resp.bits <= '0; - tx_rsp.ready <= 0; - tx_dat.ready <= 0; - snp.ready <= 0; - mem_req.ready <= 0; - rsp_active = 0; - dat_active = 0; - mem_active = 0; - rsp_queue.delete(); - ack_queue.delete(); - dat_queue.delete(); - mem_queue.delete(); - foreach (cpus[i]) cpus[i].delete(); - end else begin - cycle++; - tx_rsp.ready <= !hold_rsp && (cycle % 8) >= 4; - tx_dat.ready <= (cycle % 8) >= 4; - snp.ready <= !hold_snp && (cycle % 8) >= 4; - mem_req.ready <= !hold_mem_req && (cycle % 8) >= 4; - if (control.outstanding > peak) peak = control.outstanding; - if (rsp_active && rx_rsp.ready) rsp_active = 0; - if (dat_active && rx_dat.ready) dat_active = 0; - if (mem_active && mem_resp.ready) mem_active = 0; - if (req.valid && req.ready) begin - completion item = completion::type_id::create("expected"); - int node = req.bits[`CF(REQ, SRCID)]; - int txn = req.bits[`CF(REQ, TXNID)]; - int key = (node << 12) | txn; - int opcode = req.bits[`CF(REQ, OPCODE)]; - longint unsigned addr = req.bits[`CF(REQ, ADDR)]; - bit [511:0] value; - requests[key] = req.bits; - if (opcode == `COH_CLEAN_SHARED) begin - if (!clean_shared_requests.exists(addr)) clean_shared_requests[addr] = 0; - clean_shared_requests[addr]++; - end - expected_order.push_back(key); - item.node = node; - item.txn = txn; - item.is_data=opcode==`COH_READ_SHARED || opcode==`COH_READ_NSD || opcode==`COH_READ_UNIQUE; - item.opcode=item.is_data ? `COH_COMP_DATA : (opcode==`COH_WRITEBACK ? `COH_COMP_DBID : `COH_COMP); - item.error = expected_error.exists(key) && expected_error[key] ? 3 : 0; - item.permission=item.is_data && item.error==0 ? (opcode==`COH_READ_UNIQUE ? 2 : 1) : 0; - if (item.is_data && !item.error) begin - coherence_ref_read(model, addr, value); - item.data = value; - end - item.set_transaction_id(key); - scoreboard.expected_export.write(item); - if (opcode == `COH_EVICT && cpus[node-1].exists(addr)) begin - if (cpus[node-1][addr].valid && cpus[node-1][addr].dirty) - `uvm_fatal("CPU", "dirty line sent as Evict") - cpus[node-1][addr].valid = 0; - end - end - if (snp.valid && snp.ready) begin - int node = snp.bits[`CF(SNP, TARGET)]; - int txn = snp.bits[`CF(SNP, TXNID)]; - int opcode = snp.bits[`CF(SNP, OPCODE)]; - longint unsigned addr = longint'(snp.bits[`CF(SNP, ADDR)]) << 3; - private_line_t line; - bit invalidate = opcode == `COH_SNP_UNIQUE || opcode == `COH_SNP_INVALID || opcode == `COH_SNP_MAKE_INVALID; - bit discard = opcode == `COH_SNP_MAKE_INVALID; - if (node < 1 || node > `COH_AGENTS || snp.bits[`CF(SNP, SRCID)] != `COH_HOME) - `uvm_fatal("SNOOP", "invalid destination/source") - snoops++; - line = cpus[node-1].exists(addr) ? cpus[node-1][addr] : '0; - if ((opcode == `COH_SNP_INVALID || opcode == `COH_SNP_CLEAN_SHARED || discard) && snp.bits[ - `CF(SNP, RETTOSRC) - ] !== 0) - `uvm_fatal("SNOOP", "Maintenance snoop must clear RetToSrc") - if (line.valid && !discard && (line.dirty || (!line.unique_owner && snp.bits[ - `CF(SNP, RETTOSRC) - ]))) begin - for (int b = 0; b < `COH_BEATS; b++) begin - dat_t packet = '0; - packet[`CF(DAT, TGTID)] = `COH_HOME; - packet[`CF(DAT, SRCID)] = node; - packet[`CF(DAT, TXNID)] = txn; - packet[`CF(DAT, OPCODE)] = `COH_SNP_DATA; - packet[`CF(DAT, RESP)] = (invalidate ? 0 : 1) | (line.dirty ? 4 : 0); - packet[`CF(DAT, DATAID)] = b * (`COH_DATA_BITS / 128); - packet[`CF(DAT, BE)] = '1; - packet[`CF(DAT, DATA)] = line.data[b*`COH_DATA_BITS+:`COH_DATA_BITS]; - dat_queue.push_back(packet); - end - end else begin - rsp_t packet = '0; - packet[`CF(RSP, TGTID)] = `COH_HOME; - packet[`CF(RSP, SRCID)] = node; - packet[`CF(RSP, TXNID)] = txn; - packet[`CF(RSP, OPCODE)] = `COH_SNP_RESP; - packet[`CF(RSP, RESP)] = line.valid && !invalidate ? 1 : 0; - rsp_queue.push_back(packet); - end - if (line.valid) begin - line.valid = !invalidate; - line.unique_owner = 0; - line.dirty = 0; - cpus[node-1][addr] = line; - end - end - if (tx_rsp.valid && tx_rsp.ready) begin - completion item = completion::type_id::create("rsp"); - int node = tx_rsp.bits[`CF(TXRSP, TGTID)]; - int txn = tx_rsp.bits[`CF(TXRSP, TXNID)]; - int key = (node << 12) | txn; - int dbid = tx_rsp.bits[`CF(TXRSP, DBID)]; - if (!requests.exists( - key - ) || tx_rsp.bits[ - `CF(TXRSP, SRCID) - ] != `COH_HOME || dbid >= `COH_MSHRS) - `uvm_fatal("RSP", "unexpected response routing") - item.node = node; - item.txn = txn; - item.opcode = tx_rsp.bits[`CF(TXRSP, OPCODE)]; - item.error = tx_rsp.bits[`CF(TXRSP, RESPERR)]; - item.permission = tx_rsp.bits[`CF(TXRSP, RESP)]; - if (item.opcode == `COH_COMP_DBID) begin - longint unsigned addr = requests[key][`CF(REQ, ADDR)]; - private_line_t line = cpus[node-1].exists(addr) ? cpus[node-1][addr] : '0; - for (int b = 0; b < `COH_BEATS; b++) begin - dat_t packet = '0; - packet[`CF(DAT, TGTID)] = `COH_HOME; - packet[`CF(DAT, SRCID)] = node; - packet[`CF(DAT, TXNID)] = dbid; - packet[`CF(DAT, OPCODE)] = `COH_COPYBACK_DATA; - packet[ - `CF(DAT, RESP) - ] = line.valid ? (line.unique_owner ? 2 : 1) | (line.dirty ? 4 : 0) : 0; - packet[`CF(DAT, DATAID)] = b * (`COH_DATA_BITS / 128); - packet[`CF(DAT, BE)] = line.valid ? '1 : '0; - packet[`CF(DAT, DATA)] = line.data[b*`COH_DATA_BITS+:`COH_DATA_BITS]; - dat_queue.push_back(packet); - end - if (line.valid) begin - line.valid = 0; - cpus[node-1][addr] = line; - end - end - finish(item); - end - if (tx_dat.valid && tx_dat.ready) begin - int node = tx_dat.bits[`CF(TXDAT, TGTID)]; - int txn = tx_dat.bits[`CF(TXDAT, TXNID)]; - int key = (node << 12) | txn; - int b = tx_dat.bits[`CF(TXDAT, DATAID)] / (`COH_DATA_BITS / 128); - int dbid = tx_dat.bits[`CF(TXDAT, DBID)]; - if (!requests.exists( - key - ) || tx_dat.bits[ - `CF(TXDAT, SRCID) - ] != `COH_HOME || tx_dat.bits[ - `CF(TXDAT, HOMENID) - ] != `COH_HOME || dbid >= `COH_MSHRS || b >= `COH_BEATS) - `uvm_fatal("DATA", "unexpected response routing") - if (!received.exists(key)) begin - received[key] = '0; - assembled[key] = '0; - data_dbid[key] = dbid; - data_permission[key] = tx_dat.bits[`CF(TXDAT, RESP)]; - data_error[key] = tx_dat.bits[`CF(TXDAT, RESPERR)]; - end - if (tx_dat.bits[ - `CF(TXDAT, OPCODE) - ] !== `COH_COMP_DATA || tx_dat.bits[ - `CF(TXDAT, DATAID) - ] % (`COH_DATA_BITS / 128) != 0 || tx_dat.bits[ - `CF(TXDAT, RESP) - ] !== data_permission[key] || tx_dat.bits[ - `CF(TXDAT, RESPERR) - ] !== data_error[key] || tx_dat.bits[ - `CF(TXDAT, BE) - ] !== (data_error[key] != 0 ? `COH_TXDAT_BE_WIDTH'(0) : {`COH_TXDAT_BE_WIDTH{1'b1}})) - `uvm_fatal("DATA", "inconsistent completion data header") - if (received[key][b] || dbid != data_dbid[key]) - `uvm_fatal("DATA", "duplicate beat or inconsistent DBID") - assembled[key][b*`COH_DATA_BITS+:`COH_DATA_BITS] = tx_dat.bits[`CF(TXDAT, DATA)]; - received[key][b] = 1; - if (&received[key]) begin - completion item = completion::type_id::create("data"); - rsp_t ack = '0; - longint unsigned addr = requests[key][`CF(REQ, ADDR)]; - item.node = node; - item.txn = txn; - item.is_data = 1; - item.opcode = tx_dat.bits[`CF(TXDAT, OPCODE)]; - item.error = tx_dat.bits[`CF(TXDAT, RESPERR)]; - item.permission = tx_dat.bits[`CF(TXDAT, RESP)]; - item.data = assembled[key]; - if (!item.error) - cpus[node-1][addr] = '{ - valid: 1, - unique_owner: (item.permission == 2), - dirty: 0, - data: item.data - }; - ack[`CF(RSP, TGTID)] = `COH_HOME; - ack[`CF(RSP, SRCID)] = node; - ack[`CF(RSP, TXNID)] = dbid; - ack[`CF(RSP, OPCODE)] = `COH_COMP_ACK; - ack_queue.push_back(ack); - finish(item); - received.delete(key); - assembled.delete(key); - data_dbid.delete(key); - data_permission.delete(key); - data_error.delete(key); - end - end - if (mem_req.valid && mem_req.ready) begin - mem_t packet = '0; - longint unsigned addr = mem_req.bits[`CF(MEMREQ, ADDR)]; - bit [511:0] value; - packet[`CF(MEMRESP, ID)] = mem_req.bits[`CF(MEMREQ, ID)]; - if (mem_req.bits[`CF(MEMREQ, WRITE)]) begin - coherence_ref_read(model, addr, value); - if (mem_req.bits[`CF(MEMREQ, DATA)] !== value || mem_req.bits[`CF(MEMREQ, MASK)] !== '1) - `uvm_fatal("MEMORY", "dirty victim writeback lost data") - foreach (cpus[i]) - if (cpus[i].exists( - addr - ) && cpus[i][addr].valid && (!clean_shared_requests.exists( - addr - ) || cpus[i][addr].dirty || cpus[i][addr].unique_owner || - cpus[i][addr].data !== value)) - `uvm_fatal( - "INCLUSION", - "DDR clean completed while a private copy was dirty, unique or inconsistent") - if (write_fault_enabled && addr == fault_addr) begin - packet[`CF(MEMRESP, ERROR)] = 1; - end else begin - memory[addr] = value; - dirty_addresses.delete(addr); - end - memory_writes++; - end else begin - if (memory.exists(addr)) value = memory[addr]; - else coherence_ref_initial(addr, value); - packet[`CF(MEMRESP, DATA)] = value; - packet[`CF(MEMRESP, ERROR)] = fault_enabled && addr == fault_addr; - memory_reads++; - end - mem_queue.push_back(packet); - end - if (!rsp_active) begin - if (rsp_queue.size() != 0) begin - rsp_packet = rsp_queue.pop_front(); - rsp_active = 1; - end else begin - int chosen = -1; - foreach (ack_queue[j]) - if (chosen < 0 && !hold_ack[ack_queue[j][`CF(RSP, TXNID)]]) chosen = j; - if (chosen >= 0) begin - rsp_packet = ack_queue[chosen]; - ack_queue.delete(chosen); - rsp_active = 1; - end - end - end - if (!dat_active && dat_queue.size() != 0 && (!hold_copyback || dat_queue[0][ - `CF(DAT, OPCODE) - ] != `COH_COPYBACK_DATA)) begin - dat_packet = dat_queue.pop_front(); - dat_active = 1; - end - if (!mem_active && !hold_memory && mem_queue.size() != 0) begin - mem_packet = reverse_memory ? mem_queue.pop_back() : mem_queue.pop_front(); - mem_active = 1; - end - rx_rsp.valid <= rsp_active; - rx_rsp.bits <= rsp_active || !vary_idle_payload ? rsp_packet : {`COH_RSP_WIDTH{idle_payload_one}}; - rx_dat.valid <= dat_active; - rx_dat.bits <= dat_active || !vary_idle_payload ? dat_packet : {`COH_DAT_WIDTH{idle_payload_one}}; - mem_resp.valid <= mem_active; - mem_resp.bits <= mem_active || !vary_idle_payload ? mem_packet : {`COH_MEMRESP_WIDTH{idle_payload_one}}; - end - end - endtask - function void final_phase(uvm_phase phase); - coherence_ref_destroy(model); - super.final_phase(phase); - endfunction -endclass diff --git a/arch/src/main/scala/framework/mem-core/coherence/src/main/resources/package.sv b/arch/src/main/scala/framework/mem-core/coherence/src/main/resources/package.sv deleted file mode 100644 index 73f77bf30..000000000 --- a/arch/src/main/scala/framework/mem-core/coherence/src/main/resources/package.sv +++ /dev/null @@ -1,54 +0,0 @@ -`include "coherence_config.svh" -`define CF(K, F) `COH_``K``_``F``_OFFSET +: `COH_``K``_``F``_WIDTH -package coherence_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - typedef logic [`COH_REQ_WIDTH-1:0] req_t; - typedef logic [`COH_RSP_WIDTH-1:0] rsp_t; - typedef logic [`COH_DAT_WIDTH-1:0] dat_t; - typedef logic [`COH_MEMRESP_WIDTH-1:0] mem_t; - typedef struct packed { - bit valid, unique_owner, dirty; - bit [511:0] data; - } private_line_t; - import "DPI-C" function chandle coherence_ref_create(); - import "DPI-C" function void coherence_ref_destroy(input chandle model); - import "DPI-C" function void coherence_ref_initial( - input longint unsigned addr, - output bit [511:0] data - ); - import "DPI-C" function void coherence_ref_read( - input chandle model, - input longint unsigned addr, - output bit [511:0] data - ); - import "DPI-C" function void coherence_ref_write( - input chandle model, - input longint unsigned addr, - input bit [511:0] data - ); - - class completion extends uvm_sequence_item; - int node, txn, opcode; - bit is_data; - logic [2:0] permission = 0; - logic [1:0] error = 0; - logic [511:0] data = 0; - `uvm_object_utils_begin(completion) - `uvm_field_int(node, UVM_ALL_ON) - `uvm_field_int(txn, UVM_ALL_ON) - `uvm_field_int(opcode, UVM_ALL_ON) - `uvm_field_int(is_data, UVM_ALL_ON) - `uvm_field_int(permission, UVM_ALL_ON) - `uvm_field_int(error, UVM_ALL_ON) - `uvm_field_int(data, UVM_ALL_ON) - `uvm_object_utils_end - function new(string name = "completion"); - super.new(name); - endfunction - endclass - - `include "env.svh" - `include "test.svh" -endpackage diff --git a/arch/src/main/scala/framework/mem-core/coherence/src/main/resources/test.svh b/arch/src/main/scala/framework/mem-core/coherence/src/main/resources/test.svh deleted file mode 100644 index 321d4415b..000000000 --- a/arch/src/main/scala/framework/mem-core/coherence/src/main/resources/test.svh +++ /dev/null @@ -1,641 +0,0 @@ -class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - env test_env; - virtual coherence_control_if control; - virtual stream_if #(`COH_REQ_WIDTH) req; - int serial = 1; - int reused_txn; - int expected = 0; - longint unsigned eviction_addresses[`COH_MSHRS]; - function new(string name, uvm_component parent); - super.new(name, parent); - timeout = 300us; - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - test_env = env::type_id::create("env", this); - if (!uvm_config_db#(virtual coherence_control_if)::get( - this, "", "control", control - ) || !uvm_config_db#(virtual stream_if #(`COH_REQ_WIDTH))::get( - this, "", "req", req - )) - `uvm_fatal("VIF", "Coherence test interfaces missing") - if (`COH_AGENTS < 2 || `COH_SETS < `COH_MSHRS || `COH_WAYS != 2) - `uvm_fatal("CONFIG", - "This regression requires two CPUs, two ways, and at least one set per MSHR") - endfunction - function longint unsigned addr(int tag, int set_index); - return (longint'(tag) * `COH_SETS + set_index) * 64; - endfunction - function bit [511:0] pattern(int seed); - bit [511:0] value; - for (int word = 0; word < 16; word++) value[word*32+:32] = seed ^ (32'h01010101 * word); - return value; - endfunction - task send(int node, int opcode, longint unsigned address, bit error = 0); - req_t packet = '0; - int txn = serial & 4095; - bit read = opcode == `COH_READ_SHARED || opcode == `COH_READ_NSD || opcode == `COH_READ_UNIQUE; - packet[`CF(REQ, QOS)] = serial & 15; - packet[`CF(REQ, LPID)] = serial & 31; - packet[`CF(REQ, SRCID)] = node; - packet[`CF(REQ, TGTID)] = `COH_HOME; - packet[`CF(REQ, TXNID)] = txn; - packet[`CF(REQ, OPCODE)] = opcode; - packet[`CF(REQ, ADDR)] = address; - packet[`CF(REQ, SIZE)] = 6; - packet[`CF(REQ, EXPCOMPACK)] = read; - packet[`CF(REQ, SNPATTR)] = 1; - packet[`CF(REQ, MEMATTR)] = 12; - packet[`CF(REQ, ALLOWRETRY)] = 1; - test_env.expected_error[(node<<12)|txn] = error; - req.bits <= packet; - req.valid <= 1; - do @(req.sample); while (!req.sample.ready); - req.valid <= 0; - serial++; - expected++; - endtask - task settle(int occupied = 0); - test_env.scoreboard.wait_checked(expected); - do @(control.cb); while (control.cb.outstanding != occupied); - endtask - task execute(); - control.reset = 1; - req.valid = 0; - req.bits = '0; - repeat (4) @(control.cb); - control.cb.reset <= 0; - @(control.cb); - // Idle payload is unconstrained: toggling it must not allocate a transaction or consume a response. - test_env.vary_idle_payload = 1; - test_env.idle_payload_one = 1; - req.bits <= '1; - repeat (3) @(control.cb); - test_env.idle_payload_one = 0; - req.bits <= '0; - repeat (3) @(control.cb); - test_env.vary_idle_payload = 0; - begin - req_t idle = '0; - idle[`CF(REQ, TGTID)] = `COH_HOME; - idle[`CF(REQ, SRCID)] = 1; - idle[`CF(REQ, OPCODE)] = `COH_READ_SHARED; - idle[`CF(REQ, SIZE)] = 6; - idle[`CF(REQ, MEMATTR)] = 12; - idle[`CF(REQ, SNPATTR)] = 1; - idle[`CF(REQ, ALLOWRETRY)] = 1; - idle[`CF(REQ, EXPCOMPACK)] = 1; - // Alter one unconstrained idle bit at a time, independently of the other access attributes. - for (int bit_index = 0; bit_index < `COH_REQ_WIDTH; bit_index++) begin - req.bits <= idle ^ (req_t'(1) << bit_index); - @(control.cb); - end - req.bits <= '0; - @(control.cb); - end - if (control.cb.outstanding != 0 || test_env.scoreboard.checked != 0) - `uvm_fatal("IDLE", "Invalid channel payload changed transaction state") - for (int slot = 0; slot < `COH_MSHRS; slot++) begin - longint unsigned a = addr(8 * slot, slot); - longint unsigned b = addr(8 * slot + 1, slot); - longint unsigned c = addr(8 * slot + 2, slot); - int saved; - test_env.hold_ack = (1 << slot) - 1; - for (int h = 0; h < slot; h++) send(1, `COH_READ_SHARED, addr(100 + slot, h)); - settle(slot); - send(1, `COH_READ_SHARED, a); - settle(slot); - send(2, `COH_READ_NSD, a); - settle(slot); - test_env.cpus[1][a].valid = 0; - send(1, `COH_READ_UNIQUE, a); - settle(slot); - send(2, `COH_READ_SHARED, a); - settle(slot); - send(1, `COH_READ_UNIQUE, a); - settle(slot); - test_env.cpu_write(1, a, '1); - send(2, `COH_READ_SHARED, a); - send(1, `COH_WRITEBACK, a); - settle(slot); - send(1, `COH_READ_UNIQUE, a); - send(2, `COH_EVICT, a); - settle(slot); - test_env.cpu_write(1, a, pattern(32'h12345678)); - send(2, `COH_READ_SHARED, a); - send(1, `COH_WRITEBACK, a); - settle(slot); - send(1, `COH_READ_SHARED, b); - settle(slot); - send(2, `COH_READ_SHARED, c); - settle(slot); - if (test_env.cpus[1][a].valid) - `uvm_fatal("INCLUSION", "inclusive victim survived replacement") - send(1, `COH_READ_SHARED, a); - settle(slot); - send(2, `COH_CLEAN_INVALID, a); - settle(slot); - send(1, `COH_CLEAN_INVALID, a); - settle(slot); - send(1, `COH_EVICT, a); - settle(slot); - send(1, `COH_WRITEBACK, a); - settle(slot); - send(1, `COH_READ_UNIQUE, a); - settle(slot); - test_env.cpu_write(1, a, pattern(32'h87654321)); - send(2, `COH_CLEAN_INVALID, a); - settle(slot); - send(2, `COH_READ_SHARED, a); - settle(slot); - test_env.fault_addr = addr(8 * slot + 3, slot); - test_env.fault_enabled = 1; - send(1, `COH_READ_SHARED, test_env.fault_addr, 1); - settle(slot); - test_env.fault_enabled = 0; - send(1, `COH_READ_SHARED, test_env.fault_addr); - settle(slot); - begin - longint unsigned high = (((64'h1 << `COH_REQ_ADDR_WIDTH) - 1) & - ~(longint'(`COH_SETS * 64) - 1)) | (slot * 64); - send(1, `COH_READ_UNIQUE, high); - settle(slot); - send(1, `COH_EVICT, high); - settle(slot); - send(1, `COH_READ_UNIQUE, high); - settle(slot); - test_env.cpu_write(1, high, '0); - send(1, `COH_WRITEBACK, high); - settle(slot); - send(2, `COH_CLEAN_INVALID, high); - settle(slot); - send(2, `COH_READ_SHARED, high); - settle(slot); - send(1, `COH_CLEAN_INVALID, high); - settle(slot); - send(1, `COH_READ_SHARED, addr(8 * slot + 7, slot)); - settle(slot); - end - send(1, `COH_READ_SHARED, addr(1750 + slot, slot)); - settle(slot); - send(2, `COH_READ_SHARED, addr(1750 + slot, slot)); - settle(slot); - send(1, `COH_EVICT, addr(1750 + slot, slot)); - settle(slot); - send(2, `COH_CLEAN_INVALID, addr(1750 + slot, slot)); - settle(slot); - saved = serial; - serial = 4095; - send(2, `COH_READ_SHARED, a); - settle(slot); - send(2, `COH_READ_SHARED, a); - settle(slot); - serial = 4095; - send(2, `COH_CLEAN_INVALID, a); - settle(slot); - serial = 4095; - send(2, `COH_WRITEBACK, a); - settle(slot); - serial = 4095; - send(2, `COH_EVICT, a); - settle(slot); - serial = saved; - test_env.hold_ack = 0; - settle(); - end - - test_env.hold_ack = '1; - send(1, `COH_READ_SHARED, addr(200, 0)); - test_env.scoreboard.wait_checked(expected); - fork - begin - send(2, `COH_READ_SHARED, addr(201, 0)); - end - begin - repeat (4) @(control.cb); - if (req.ready !== 0 || control.cb.outstanding != 1) - `uvm_fatal("LOCK", "same-set request entered before CompAck") - test_env.hold_ack = 0; - end - join - settle(); - test_env.hold_ack = '1; - test_env.hold_memory = 1; - reused_txn = serial; - send(1, `COH_READ_SHARED, addr(202, 0)); - repeat (8) @(control.cb); - serial = reused_txn; - fork - begin - send(1, `COH_READ_SHARED, addr(203, 1)); - end - begin - repeat (4) @(control.cb); - if (req.ready !== 0 || control.cb.outstanding != 1) - `uvm_fatal("ID_LIFETIME", "Requester TxnID was reused before its final response") - test_env.hold_memory = 0; - end - join - test_env.scoreboard.wait_checked(expected); - do @(control.cb); while (control.cb.outstanding != 2); - test_env.hold_ack = 0; - settle(); - for (int rotation = 0; rotation < `COH_SETS; rotation++) begin - for (int pass = 0; pass < `COH_WAYS; pass++) begin - test_env.hold_memory = 1; - test_env.hold_mem_req = 1; - test_env.reverse_memory = 1; - test_env.saw_reorder = 0; - for (int slot = 0; slot < `COH_MSHRS; slot++) begin - eviction_addresses[slot] = - addr(300 + rotation * `COH_WAYS + pass, (slot + rotation) % `COH_SETS); - send((slot % 2) + 1, `COH_READ_UNIQUE, eviction_addresses[slot]); - end - repeat (20) @(control.cb); - if (control.cb.outstanding != `COH_MSHRS) - `uvm_fatal("PARALLEL", "different sets did not occupy independent MSHRs") - test_env.hold_mem_req = 0; - repeat (16) @(control.cb); - test_env.hold_memory = 0; - settle(); - if (`COH_MSHRS > 1 && !test_env.saw_reorder) - `uvm_fatal("ORDER", "out-of-order response scenario did not execute") - end - end - for (int slot = 0; slot < `COH_MSHRS; slot++) - send((slot % 2) + 1, `COH_EVICT, eviction_addresses[slot]); - settle(); - // Delayed eviction notifications must not remove a newer owner's directory state. - for (int slot = 0; slot < `COH_MSHRS; slot++) - send((slot % 2) + 1, `COH_EVICT, addr( - 8 * ((slot + `COH_SETS - 1) % `COH_SETS), (slot + `COH_SETS - 1) % `COH_SETS)); - settle(); - for (int slot = 0; slot < `COH_MSHRS; slot++) begin - longint unsigned previous_line = eviction_addresses[slot] - `COH_SETS * 64; - test_env.cpu_write((slot % 2) + 1, previous_line, {16{32'h01020300 + slot}}); - send(((slot + 1) % 2) + 1, `COH_READ_SHARED, previous_line); - end - settle(); - for (int slot = 0; slot < `COH_MSHRS; slot++) - send(1, `COH_CLEAN_INVALID, eviction_addresses[slot] - `COH_SETS * 64); - settle(); - - // Fill the completion queue before each selected worker reaches completion. - for (int slot = 0; slot < `COH_MSHRS - 1; slot++) begin - longint unsigned slow = addr(900 + slot, slot); - send(1, `COH_READ_UNIQUE, slow); - settle(); - test_env.cpu_write(1, slow, pattern(32'h76543210)); - test_env.hold_ack = (1 << slot) - 1; - for (int h = 0; h < slot; h++) send(1, `COH_READ_SHARED, addr(950 + slot, h)); - settle(slot); - test_env.hold_memory = 1; - test_env.hold_rsp = 1; - send(2, `COH_CLEAN_INVALID, slow); - send(2, `COH_EVICT, addr(8 * (`COH_SETS - 1), `COH_SETS - 1)); - send(2, `COH_EVICT, addr(8 * (`COH_SETS - 1), `COH_SETS - 1)); - repeat (20) @(control.cb); - test_env.hold_memory = 0; - repeat (20) @(control.cb); - test_env.hold_rsp = 0; - settle(slot); - test_env.hold_ack = 0; - settle(); - end - - // Each victim has two sharers, so a worker can wait to issue its second snoop. - for (int s = 0; s < `COH_MSHRS; s++) begin - send(1, `COH_READ_SHARED, addr(700, s)); - settle(); - send(2, `COH_READ_SHARED, addr(700, s)); - settle(); - send(1, `COH_READ_SHARED, addr(701, s)); - settle(); - end - begin - int before_snoops = test_env.snoops; - fork - begin - for (int s = 0; s < `COH_MSHRS; s++) send(1, `COH_READ_SHARED, addr(702, s)); - end - begin - wait (test_env.snoops > before_snoops); - test_env.hold_snp = 1; - repeat (12) @(control.cb); - test_env.hold_snp = 0; - end - join - end - settle(); - // Align a returned memory operation with a new lookup to exercise every Cache arbiter input. - for (int slot = 0; slot < `COH_MSHRS; slot++) begin - int other; - test_env.hold_ack = (1 << slot) - 1; - for (int h = 0; h < slot; h++) send(1, `COH_READ_SHARED, addr(960 + slot, h)); - settle(slot); - test_env.hold_memory = 1; - send(1, `COH_READ_SHARED, addr(980 + slot, slot)); - wait (test_env.mem_queue.size() != 0); - if (slot == `COH_MSHRS - 1) begin - test_env.hold_ack[0] = 0; - do @(control.cb); while (control.cb.outstanding != slot); - other = 0; - end else other = slot + 1; - test_env.hold_memory = 0; - @(control.cb); - send(2, `COH_READ_SHARED, addr(990 + slot, other)); - settle($countones(test_env.hold_ack)); - test_env.hold_ack = 0; - settle(); - end - - // A recalled dirty line reaches memory after two independent reads fill the request queue. - for (int slot = 0; slot < 2; slot++) begin - longint unsigned slow = addr(1100 + slot, slot); - control.cb.reset <= 1; - repeat (2) @(control.cb); - control.cb.reset <= 0; - @(control.cb); - send(1, `COH_READ_UNIQUE, slow); - settle(); - test_env.cpu_write(1, slow, pattern(32'hFEDCBA98)); - test_env.hold_ack = (1 << slot) - 1; - for (int h = 0; h < slot; h++) send(1, `COH_READ_SHARED, addr(1150 + slot, h)); - settle(slot); - test_env.hold_snp = 1; - test_env.hold_mem_req = 1; - send(2, `COH_CLEAN_INVALID, slow); - send(1, `COH_READ_SHARED, addr(1200 + slot, slot + 1)); - send(2, `COH_READ_SHARED, addr(1250 + slot, slot + 2)); - repeat (16) @(control.cb); - test_env.hold_snp = 0; - repeat (16) @(control.cb); - test_env.hold_mem_req = 0; - settle(slot); - test_env.hold_ack = 0; - settle(); - end - for (int slot = 0; slot < `COH_MSHRS; slot++) begin - longint unsigned a = addr(1500 + 3 * slot, slot); - longint unsigned b = addr(1501 + 3 * slot, slot); - longint unsigned c = addr(1502 + 3 * slot, slot); - test_env.hold_ack = (1 << slot) - 1; - for (int h = 0; h < slot; h++) send(1, `COH_READ_SHARED, addr(1600 + slot, h)); - settle(slot); - send(1, `COH_READ_UNIQUE, a); - settle(slot); - test_env.cpu_write(1, a, pattern(32'hDEADBEEF)); - send(2, `COH_READ_SHARED, b); - settle(slot); - test_env.fault_addr = a; - test_env.write_fault_enabled = 1; - send(2, `COH_READ_SHARED, c, 1); - settle(slot); - test_env.write_fault_enabled = 0; - send(1, `COH_READ_SHARED, a); - settle(slot); - send(2, `COH_READ_SHARED, c); - settle(slot); - send(2, `COH_CLEAN_INVALID, a); - settle(slot); - send(1, `COH_READ_UNIQUE, a); - settle(slot); - test_env.cpu_write(1, a, pattern(32'h01234567)); - test_env.write_fault_enabled = 1; - send(1, `COH_CLEAN_INVALID, a, 1); - settle(slot); - test_env.write_fault_enabled = 0; - send(2, `COH_READ_SHARED, a); - settle(slot); - send(1, `COH_CLEAN_INVALID, a); - settle(slot); - test_env.hold_ack = 0; - settle(); - end - for (int slot = 0; slot < `COH_MSHRS; slot++) begin - test_env.hold_ack = (1 << slot) - 1; - for (int h = 0; h < slot; h++) send(1, `COH_READ_SHARED, addr(1700 + slot, h)); - settle(slot); - `uvm_info("ADDR_SWEEP", $sformatf( - "begin slot=%0d checked=%0d", slot, test_env.scoreboard.checked), UVM_LOW) - for (int bit_index = 8; bit_index < `COH_REQ_ADDR_WIDTH; bit_index++) begin - longint unsigned a = (64'h1 << bit_index) | (longint'(slot) << 6); - bit [511:0] value = {8{64'h1 << ((bit_index - 8) % 64)}}; - send(1, `COH_READ_UNIQUE, a); - settle(slot); - test_env.cpu_write(1, a, value); - send(2, `COH_READ_SHARED, a); - settle(slot); - send(1, `COH_CLEAN_INVALID, a); - settle(slot); - send(2, `COH_READ_SHARED, a); - settle(slot); - end - test_env.hold_ack = 0; - settle(); - end - // Both resident ways must see the high address bits, independently of replacement parity. - for (int set_index = 0; set_index < `COH_SETS; set_index++) begin - longint unsigned high = (((64'h1 << `COH_REQ_ADDR_WIDTH) - 1) & - ~(longint'(`COH_SETS * 64) - 1)) | (longint'(set_index) << 6); - send(1, `COH_READ_SHARED, high); - settle(); - send(2, `COH_READ_SHARED, high ^ (`COH_SETS * 64)); - settle(); - send(1, `COH_READ_SHARED, addr(2000, set_index)); - settle(); - send(2, `COH_READ_SHARED, addr(2001, set_index)); - settle(); - send(1, `COH_READ_SHARED, addr(2001, set_index)); - settle(); - send(1, `COH_EVICT, addr(2001, set_index)); - settle(); - send(2, `COH_CLEAN_INVALID, addr(2001, set_index)); - settle(); - end - // True CMO: clean retains cache copies; invalidate-after-DMA must discard stale dirty bytes. - begin - longint unsigned a = addr(2300, 0); - bit [511:0] dirty_value = pattern(32'habbacd01), dma_value = pattern(32'hd00dbeef); - int before_write, before_read; - send(1, `COH_READ_UNIQUE, a); - settle(); - test_env.cpu_write(1, a, dirty_value); - before_write = test_env.memory_writes; - send(2, `COH_CLEAN_SHARED, a); - settle(); - if (test_env.memory_writes != before_write + 1 || test_env.memory[a] !== dirty_value || - !test_env.cpus[0][a].valid || test_env.cpus[0][a].dirty || test_env.cpus[0][a].unique_owner) - `uvm_fatal("CLEAN_SHARED", "Dirty clean did not retain SC data and make DDR visible") - before_read = test_env.memory_reads; - before_write = test_env.memory_writes; - send(1, `COH_CLEAN_SHARED, a); - settle(); - send(2, `COH_READ_SHARED, a); - settle(); - if (test_env.memory_reads != before_read || test_env.memory_writes != before_write) - `uvm_fatal("CLEAN_HIT", "CleanShared evicted a valid line or wrote an already-clean line") - send(1, `COH_READ_UNIQUE, a); - settle(); - test_env.cpu_write(1, a, pattern(32'hfa11ed00)); - test_env.fault_addr = a; - test_env.write_fault_enabled = 1; - send(2, `COH_CLEAN_SHARED, a, 1); - settle(); - test_env.write_fault_enabled = 0; - if (!test_env.dirty_addresses.exists(a) || test_env.memory[a] !== dirty_value) - `uvm_fatal("CLEAN_ERROR", "Failed clean lost dirty data or falsely changed DDR") - send(2, `COH_READ_SHARED, a); - settle(); - send(2, `COH_CLEAN_SHARED, a); - settle(); - if (test_env.memory[a] !== pattern(32'hfa11ed00)) - `uvm_fatal("CLEAN_RETRY", "Restored dirty line did not reach DDR on later clean") - send(1, `COH_READ_UNIQUE, a); - settle(); - test_env.cpu_write(1, a, dirty_value); - test_env.memory[a] = dma_value; - coherence_ref_write(test_env.model, a, dma_value); - test_env.dirty_addresses.delete(a); - before_write = test_env.memory_writes; - send(2, `COH_MAKE_INVALID, a); - settle(); - if (test_env.memory_writes != before_write || test_env.memory[a] !== dma_value || - test_env.cpus[0][a].valid || test_env.cpus[1][a].valid) - `uvm_fatal("MAKE_INVALID", - "Post-DMA invalidation wrote stale cache data or retained a copy") - before_read = test_env.memory_reads; - send(1, `COH_READ_SHARED, a); - settle(); - if (test_env.memory_reads != before_read + 1) - `uvm_fatal("INVALID_REFILL", "Invalidated line did not refill fresh DDR data") - send(1, `COH_MAKE_INVALID, a); - settle(); - send(1, `COH_CLEAN_SHARED, a); - settle(); - send(1, `COH_MAKE_INVALID, a); - settle(); - `uvm_info( - "CMO", - "CleanShared DDR visibility/retention/error restoration and MakeInvalid discard checked", - UVM_LOW) - end - // Exercise the new maintenance paths in every real Home transaction slot. - begin - longint unsigned addresses[`COH_MSHRS]; - bit [511:0] values[`COH_MSHRS]; - int before_write; - for (int i = 0; i < `COH_MSHRS; i++) begin - addresses[i] = addr(2400 + i, i); - values[i] = pattern(32'h13572468 ^ i); - send(1, `COH_READ_UNIQUE, addresses[i]); - settle(); - test_env.cpu_write(1, addresses[i], values[i]); - end - before_write = test_env.memory_writes; - test_env.hold_memory = 1; - for (int i = 0; i < `COH_MSHRS; i++) send(2, `COH_CLEAN_SHARED, addresses[i]); - do @(control.cb); while (control.cb.outstanding != `COH_MSHRS); - repeat (8) @(control.cb); - test_env.hold_memory = 0; - settle(); - if (test_env.memory_writes != before_write + `COH_MSHRS) - `uvm_fatal("CMO_PARALLEL", "Every dirty clean did not receive a real DDR write") - for (int i = 0; i < `COH_MSHRS; i++) begin - if (test_env.memory[addresses[i]] !== values[i] || !test_env.cpus[0][addresses[i]].valid) - `uvm_fatal("CMO_PARALLEL", - "Parallel clean lost dirty data or invalidated a retained copy") - send(1, `COH_READ_UNIQUE, addresses[i]); - settle(); - test_env.cpu_write(1, addresses[i], pattern(32'hdeadcafe ^ i)); - values[i] = pattern(32'h02468ace ^ i); - test_env.memory[addresses[i]] = values[i]; - coherence_ref_write(test_env.model, addresses[i], values[i]); - test_env.dirty_addresses.delete(addresses[i]); - end - before_write = test_env.memory_writes; - test_env.hold_snp = 1; - for (int i = 0; i < `COH_MSHRS; i++) send(2, `COH_MAKE_INVALID, addresses[i]); - do @(control.cb); while (control.cb.outstanding != `COH_MSHRS); - repeat (8) @(control.cb); - test_env.hold_snp = 0; - settle(); - if (test_env.memory_writes != before_write) - `uvm_fatal("CMO_DISCARD", "Parallel MakeInvalid overwrote new DDR with stale dirty data") - for (int i = 0; i < `COH_MSHRS; i++) begin - send(1, `COH_READ_SHARED, addresses[i]); - settle(); - send(2, `COH_MAKE_INVALID, addresses[i]); - settle(); - end - end - // Initial caller IDs can be reused after their final response while old DBID/set ownership survives. - begin - longint unsigned a = addr(2600, 0), b = addr(2601, 1); - longint unsigned c = addr(2602, 2), d = addr(2603, 3); - int caller; - req_t probe = '0; - send(1, `COH_READ_UNIQUE, a); - settle(); - test_env.cpu_write(1, a, pattern(32'hc011ab1e)); - test_env.hold_copyback = 1; - caller = serial; - send(1, `COH_WRITEBACK, a); - settle(1); - serial = caller; - send(1, `COH_READ_SHARED, b); - settle(1); - // The old DBID is still live: even an unrelated caller cannot allocate the old set. - probe[`CF(REQ, ADDR)] = addr(2700, 0); - probe[`CF(REQ, SRCID)] = 2; - probe[`CF(REQ, TXNID)] = serial & 4095; - req.bits <= probe; - repeat (4) begin - @(req.sample); - if (req.sample.ready) - `uvm_fatal("MSHR_SET", "Caller retirement released the old DBID/set too early") - end - test_env.hold_copyback = 0; - do @(control.cb); while (control.cb.outstanding != 0); - send(1, `COH_READ_SHARED, a); - settle(); - send(1, `COH_CLEAN_INVALID, a); - settle(); - test_env.hold_ack = '1; - caller = serial; - send(1, `COH_READ_SHARED, c); - settle(1); - serial = caller; - send(1, `COH_READ_SHARED, d); - settle(2); - test_env.hold_ack = 0; - do @(control.cb); while (control.cb.outstanding != 0); - `uvm_info( - "CALLER_TXN", - "WB caller reuse before late DBID data and read caller reuse before late CompAck checked", - UVM_LOW) - end - if (test_env.memory_writes == 0) `uvm_fatal("WRITEBACK", "no dirty victim reached memory") - // Reset at retirement after all returned data was checked; the clean read has no dirty state. - test_env.hold_ack = '1; - send(1, `COH_READ_SHARED, addr(2100, 0)); - settle(1); - test_env.hold_ack = 0; - do - @(control.cb); - while (!test_env.rx_rsp.valid || test_env.rx_rsp.bits[ - `CF(RSP, OPCODE) - ] != `COH_COMP_ACK); - @(negedge control.clock); - control.reset <= 1; - repeat (2) @(control.cb); - control.cb.reset <= 0; - @(control.cb); - send(1, `COH_READ_SHARED, addr(0, 0)); - settle(); - `uvm_info("COHERENCE", $sformatf("Checked %0d completions, peak MSHRs=%0d, dirty writes=%0d", - test_env.scoreboard.checked, test_env.peak, - test_env.memory_writes), UVM_LOW) - endtask -endclass diff --git a/arch/src/main/scala/framework/mem-core/coherence/src/main/scala/Emit.scala b/arch/src/main/scala/framework/mem-core/coherence/src/main/scala/Emit.scala deleted file mode 100644 index 6dc07f0ed..000000000 --- a/arch/src/main/scala/framework/mem-core/coherence/src/main/scala/Emit.scala +++ /dev/null @@ -1,110 +0,0 @@ -package memcore.memory.coherence - -import java.nio.file.{Files, Paths} -import memcore.bus.chi._ -import memcore.memory.coherence.configs.CoherenceParams - -object Emit extends App { - val p = CoherenceParams.load("configs/default.toml") - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Coherence(p), - firtoolOpts = args, - args = Array("--target-dir", "build/Coherence", "--split-verilog") - ) - val definitions = scala.collection.mutable.ArrayBuffer[String]() - val ports = scala.collection.mutable.ArrayBuffer[String]() - - def fields(flit: Flit): Seq[(String, Int)] = flit.fieldsLSB.map { field => - flit.elements.find(_._2 eq field).get._1 -> field.getWidth - } - - def channel( - kind: String, - prefix: String, - instance: String, - fs: Seq[(String, Int)] - ): Unit = { - var offset = 0 - for ((name, width) <- fs if width > 0) { - definitions += s"`define COH_${kind}_${name.toUpperCase}_OFFSET $offset" - definitions += s"`define COH_${kind}_${name.toUpperCase}_WIDTH $width" - ports += s".$prefix$name($instance.bits[$offset +: $width])" - offset += width - } - definitions += s"`define COH_${kind}_WIDTH $offset" - } - - channel("REQ", "io_req_bits_", "source_if", fields(new RequestFlit(p.chi))) - channel("RSP", "io_rxRsp_bits_", "rx_rsp", fields(new ResponseFlit(p.chi))) - channel("DAT", "io_rxDat_bits_", "rx_dat", fields(new DataFlit(p.chi))) - channel("TXRSP", "io_rsp_bits_", "tx_rsp", fields(new ResponseFlit(p.chi))) - channel("TXDAT", "io_dat_bits_", "sink_if", fields(new DataFlit(p.chi))) - channel("SNP", "io_snp_bits_flit_", "tx_snp", fields(new SnoopFlit(p.chi))) - val snpWidth = (new SnoopFlit(p.chi)).flitWidth - ports += s".io_snp_bits_targetNode(tx_snp.bits[$snpWidth +: ${p.chi.nodeIdBits}])" - definitions += s"`define COH_SNP_TARGET_OFFSET $snpWidth" - definitions += s"`define COH_SNP_TARGET_WIDTH ${p.chi.nodeIdBits}" - definitions += s"`define COH_SNP_CHANNEL_WIDTH ${snpWidth + p.chi.nodeIdBits}" - channel( - "MEMREQ", - "io_memoryReq_bits_", - "mem_req", - Seq("id" -> p.chi.txnIdBits, "addr" -> p.chi.addressBits, "write" -> 1, "data" -> 512, "mask" -> 64) - ) - channel("MEMRESP", "io_memoryResp_bits_", "mem_resp", Seq("id" -> p.chi.txnIdBits, "data" -> 512, "error" -> 1)) - for ( - (port, instance) <- Seq( - "req" -> "source_if", - "rxRsp" -> "rx_rsp", - "rxDat" -> "rx_dat", - "rsp" -> "tx_rsp", - "dat" -> "sink_if", - "snp" -> "tx_snp", - "memoryReq" -> "mem_req", - "memoryResp" -> "mem_resp" - ) - ) { - ports += s".io_${port}_valid($instance.valid)" - ports += s".io_${port}_ready($instance.ready)" - } - for ( - (name, value) <- Seq( - "AGENTS" -> p.agents, - "MSHRS" -> p.mshrEntries, - "HOME" -> p.homeId, - "SETS" -> p.cache.sets, - "WAYS" -> p.cache.ways, - "BEATS" -> p.chi.beatsPerLine, - "DATA_BITS" -> p.chi.dataBits, - "OUTSTANDING_BITS" -> chisel3.util.log2Ceil(p.mshrEntries + 1), - "READ_SHARED" -> Opcode.ReadShared, - "READ_NSD" -> Opcode.ReadNotSharedDirty, - "READ_UNIQUE" -> Opcode.ReadUnique, - "EVICT" -> Opcode.Evict, - "WRITEBACK" -> Opcode.WriteBackFull, - "CLEAN_SHARED" -> Opcode.CleanShared, - "CLEAN_INVALID" -> Opcode.CleanInvalid, - "MAKE_INVALID" -> Opcode.MakeInvalid, - "COMP" -> Opcode.Comp, - "COMP_DBID" -> Opcode.CompDBIDResp, - "COMP_ACK" -> Opcode.CompAck, - "COMP_DATA" -> Opcode.CompData, - "SNP_RESP" -> Opcode.SnpResp, - "SNP_DATA" -> Opcode.SnpRespData, - "COPYBACK_DATA" -> Opcode.CopyBackWriteData, - "SNP_UNIQUE" -> Opcode.SnpUnique, - "SNP_INVALID" -> Opcode.SnpCleanInvalid, - "SNP_CLEAN_SHARED" -> Opcode.SnpCleanShared, - "SNP_MAKE_INVALID" -> Opcode.SnpMakeInvalid, - "SNP_SHARED" -> Opcode.SnpNotSharedDirty - ) - ) { - definitions += s"`define COH_$name $value" - } - Files.writeString( - Paths.get("build/coherence_config.svh"), - "`ifndef COHERENCE_CONFIG_SVH\n`define COHERENCE_CONFIG_SVH\n" + definitions.mkString("\n") + "\n`endif\n" - ) - Files.writeString(Paths.get("build/coherence_ports.svh"), ports.mkString(",\n") + "\n") - -} diff --git a/arch/src/main/scala/framework/mem-core/coherence/src/main/scala/Interface.scala b/arch/src/main/scala/framework/mem-core/coherence/src/main/scala/Interface.scala deleted file mode 100644 index 02373d235..000000000 --- a/arch/src/main/scala/framework/mem-core/coherence/src/main/scala/Interface.scala +++ /dev/null @@ -1,19 +0,0 @@ -package memcore.memory.coherence - -import chisel3._ -import chisel3.util._ -import memcore.bus.chi._ -import memcore.bus.chi.snf.{LineRequest, LineResponse} -import memcore.memory.coherence.configs.CoherenceParams - -class CoherenceIO(p: CoherenceParams) extends Bundle { - val req = Flipped(Decoupled(new RequestFlit(p.chi))) - val rxRsp = Flipped(Decoupled(new ResponseFlit(p.chi))) - val rxDat = Flipped(Decoupled(new DataFlit(p.chi))) - val rsp = Decoupled(new ResponseFlit(p.chi)) - val dat = Decoupled(new DataFlit(p.chi)) - val snp = Decoupled(new DirectedSnoop(p.chi)) - val memoryReq = Decoupled(new LineRequest(p.chi)) - val memoryResp = Flipped(Decoupled(new LineResponse(p.chi))) - val outstanding = Output(UInt(log2Ceil(p.mshrEntries + 1).W)) -} diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/build.sc b/arch/src/main/scala/framework/mem-core/cpu_mem/build.sc deleted file mode 100644 index d78b5cebf..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/build.sc +++ /dev/null @@ -1,35 +0,0 @@ -import mill._ -import mill.scalalib._ - -trait ChiselModule extends SbtModule { - def moduleRoot: os.Path - override def millSourcePath = moduleRoot - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") -} - -object queue extends SbtModule { - override def millSourcePath = os.pwd / os.up / "queue" - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") -} - -object chi extends ChiselModule { - override def moduleDeps = Seq(queue) - override def moduleRoot = os.pwd / os.up / "chi" -} - -object mmu extends ChiselModule { - override def moduleRoot = os.pwd / os.up / "mmu" - override def moduleDeps = Seq(chi) -} - -object cpu_mem extends ChiselModule { - override def mainClass = Some("memcore.memory.cpu.Emit") - override def moduleRoot = os.pwd - override def moduleDeps = Seq(chi, mmu) -} diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/csrc/Cargo.lock b/arch/src/main/scala/framework/mem-core/cpu_mem/src/csrc/Cargo.lock deleted file mode 100644 index 33d3bc9f5..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/csrc/Cargo.lock +++ /dev/null @@ -1,7 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "cpu-mem-reference-model" -version = "0.1.0" diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/csrc/Cargo.toml b/arch/src/main/scala/framework/mem-core/cpu_mem/src/csrc/Cargo.toml deleted file mode 100644 index 870e514dd..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/csrc/Cargo.toml +++ /dev/null @@ -1,7 +0,0 @@ -[package] -name = "cpu-mem-reference-model" -version = "0.1.0" -edition = "2021" - -[lib] -crate-type = ["cdylib"] diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/csrc/src/lib.rs b/arch/src/main/scala/framework/mem-core/cpu_mem/src/csrc/src/lib.rs deleted file mode 100644 index 34957a641..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/csrc/src/lib.rs +++ /dev/null @@ -1,2 +0,0 @@ -mod access; -pub use access::*; diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem.f b/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem.f deleted file mode 100644 index 037da65d2..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/interface.sv -@RESOURCES@/package.sv --F @RTL@/CpuMem/filelist.f -@RESOURCES@/cpu_mem_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_atomic.f b/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_atomic.f deleted file mode 100644 index 587362b3e..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_atomic.f +++ /dev/null @@ -1,11 +0,0 @@ -+define+CPU_MEM_TOP=cpu_mem_bad_atomic_tb -+define+CPU_BAD_CASE=3 -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/interface.sv -@RESOURCES@/package.sv --F @RTL@/CpuMem/filelist.f -@RESOURCES@/cpu_mem_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_atomic.toml b/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_atomic.toml deleted file mode 100644 index 3e461fd78..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_atomic.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "CPU memory port does not accept Fence or unknown atomics" diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_atomic_write.f b/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_atomic_write.f deleted file mode 100644 index 9c3eff1c4..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_atomic_write.f +++ /dev/null @@ -1,11 +0,0 @@ -+define+CPU_MEM_TOP=cpu_mem_bad_atomic_write_tb -+define+CPU_BAD_CASE=5 -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/interface.sv -@RESOURCES@/package.sv --F @RTL@/CpuMem/filelist.f -@RESOURCES@/cpu_mem_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_atomic_write.toml b/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_atomic_write.toml deleted file mode 100644 index 72c087483..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_atomic_write.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "CPU atomic requires a 32 or 64 bit operand" diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_fence.f b/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_fence.f deleted file mode 100644 index 5e7cdeba1..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_fence.f +++ /dev/null @@ -1,11 +0,0 @@ -+define+CPU_MEM_TOP=cpu_mem_bad_fence_tb -+define+CPU_BAD_CASE=2 -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/interface.sv -@RESOURCES@/package.sv --F @RTL@/CpuMem/filelist.f -@RESOURCES@/cpu_mem_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_fence.toml b/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_fence.toml deleted file mode 100644 index 3e461fd78..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_fence.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "CPU memory port does not accept Fence or unknown atomics" diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_operand.f b/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_operand.f deleted file mode 100644 index 44169ca32..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_operand.f +++ /dev/null @@ -1,11 +0,0 @@ -+define+CPU_MEM_TOP=cpu_mem_bad_operand_tb -+define+CPU_BAD_CASE=4 -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/interface.sv -@RESOURCES@/package.sv --F @RTL@/CpuMem/filelist.f -@RESOURCES@/cpu_mem_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_operand.toml b/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_operand.toml deleted file mode 100644 index 72c087483..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_operand.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "CPU atomic requires a 32 or 64 bit operand" diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_size.f b/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_size.f deleted file mode 100644 index 2511563fd..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_size.f +++ /dev/null @@ -1,11 +0,0 @@ -+define+CPU_MEM_TOP=cpu_mem_bad_size_tb -+define+CPU_BAD_CASE=1 -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/interface.sv -@RESOURCES@/package.sv --F @RTL@/CpuMem/filelist.f -@RESOURCES@/cpu_mem_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_size.toml b/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_size.toml deleted file mode 100644 index d3ba2a56f..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_size.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "CPU memory size must be 1, 2, 4 or 8 bytes" diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_tag.f b/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_tag.f deleted file mode 100644 index c793b9f0f..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_tag.f +++ /dev/null @@ -1,11 +0,0 @@ -+define+CPU_MEM_TOP=cpu_mem_bad_tag_tb -+define+CPU_BAD_TAG -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/interface.sv -@RESOURCES@/package.sv --F @RTL@/CpuMem/filelist.f -@RESOURCES@/cpu_mem_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_tag.toml b/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_tag.toml deleted file mode 100644 index 090ec048b..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_tag.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Uncached response tag mismatch" diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_tb.sv b/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_tb.sv deleted file mode 100644 index d04d9df99..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_tb.sv +++ /dev/null @@ -1,89 +0,0 @@ -`include "cpu_mem_config.svh" -`ifndef CPU_MEM_TOP -`define CPU_MEM_TOP cpu_mem_tb -`endif -module `CPU_MEM_TOP; - import uvm_pkg::*; - import cpu_mem_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - cpu_control_if control (clock); - stream_if #(`CPU_REQ_WIDTH) request ( - clock, - control.reset - ); - stream_if #(`CPU_RESP_WIDTH) response ( - clock, - control.reset - ); - stream_if #(`CPU_CACHE_WIDTH) cache_request ( - clock, - control.reset - ); - stream_if #(`CPU_CRESULT_WIDTH) cache_response ( - clock, - control.reset - ); - stream_if #(`CPU_UNCACHED_WIDTH) uncached_request ( - clock, - control.reset - ); - stream_if #(`CPU_URESULT_WIDTH) uncached_response ( - clock, - control.reset - ); - int unsigned in_flight = 0; - bit issued = 0, returned = 0, uncached = 0; - always @(posedge clock) begin - if (control.reset) begin - in_flight = 0; - issued = 0; - returned = 0; - end else begin - if (in_flight != 0 && request.ready) - $fatal(1, "CPU port became ready before consuming its response"); - if (request.valid && request.ready) begin - if (in_flight != 0) $fatal(1, "CPU memory accepted overlapping requests"); - in_flight = 1; - issued = 0; - returned = 0; - end - if ((cache_request.valid && cache_request.ready) || (uncached_request.valid && uncached_request.ready)) begin - if (in_flight != 1 || issued) $fatal(1, "Unexpected or duplicate backend request"); - issued = 1; - uncached = uncached_request.valid && uncached_request.ready; - end - if ((cache_response.valid && cache_response.ready) || (uncached_response.valid && uncached_response.ready)) begin - if (in_flight != 1 || !issued || returned || - uncached != (uncached_response.valid && uncached_response.ready)) - $fatal(1, "Unexpected or duplicate backend response"); - returned = 1; - end - if (response.valid && response.ready) begin - if (in_flight != 1 || (issued && !returned)) $fatal(1, "Unexpected CPU result"); - in_flight = 0; - end - end - end - CpuMem dut ( - .clock(clock), - .reset(control.reset), - `include "cpu_mem_ports.svh" - ); - initial begin - uvm_config_db#(virtual cpu_control_if)::set(null, "uvm_test_top*", "control", control); - uvm_config_db#(virtual stream_if #(`CPU_REQ_WIDTH))::set(null, "uvm_test_top*", "request", - request); - uvm_config_db#(virtual stream_if #(`CPU_RESP_WIDTH))::set(null, "uvm_test_top*", "response", - response); - uvm_config_db#(virtual stream_if #(`CPU_CACHE_WIDTH))::set(null, "uvm_test_top*", - "cache_request", cache_request); - uvm_config_db#(virtual stream_if #(`CPU_CRESULT_WIDTH))::set(null, "uvm_test_top*", - "cache_response", cache_response); - uvm_config_db#(virtual stream_if #(`CPU_UNCACHED_WIDTH))::set( - null, "uvm_test_top*", "uncached_request", uncached_request); - uvm_config_db#(virtual stream_if #(`CPU_URESULT_WIDTH))::set( - null, "uvm_test_top*", "uncached_response", uncached_response); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/package.sv b/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/package.sv deleted file mode 100644 index 8aa10cff2..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/package.sv +++ /dev/null @@ -1,36 +0,0 @@ -`include "cpu_mem_config.svh" -package cpu_mem_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - `define CPUF(kind, field) `CPU_``kind``_``field``_OFFSET +: `CPU_``kind``_``field``_WIDTH - typedef bit [`CPU_REQ_WIDTH-1:0] req_t; - import "DPI-C" function void cpu_mem_ref_prepare( - input longint unsigned addr, - input int unsigned size, - write, - input longint unsigned data, - input int unsigned atomic_op, - cacheable, - normal, - address_bits, - output int unsigned target, - misaligned, - access_fault, - output longint unsigned bus_addr, - bus_data, - output int unsigned bus_mask, - atomic_word - ); - import "DPI-C" function longint unsigned cpu_mem_ref_result( - input longint unsigned addr, - input int unsigned size, - write, - signed_load, - atomic_op, - cacheable, - input longint unsigned raw, - input int unsigned error - ); - `include "test.svh" -endpackage diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/test.svh b/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/test.svh deleted file mode 100644 index e1aea4692..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/test.svh +++ /dev/null @@ -1,350 +0,0 @@ -class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual cpu_control_if control; - virtual stream_if #(`CPU_REQ_WIDTH) request; - virtual stream_if #(`CPU_RESP_WIDTH) response; - virtual stream_if #(`CPU_CACHE_WIDTH) cache_request; - virtual stream_if #(`CPU_CRESULT_WIDTH) cache_response; - virtual stream_if #(`CPU_UNCACHED_WIDTH) uncached_request; - virtual stream_if #(`CPU_URESULT_WIDTH) uncached_response; - int checked = 0, cache_checked = 0, uncached_checked = 0, faults_checked = 0, cancelled = 0; - function new(string name, uvm_component parent); - super.new(name, parent); - timeout = 200us; - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual cpu_control_if)::get( - this, "", "control", control - ) || !uvm_config_db#(virtual stream_if #(`CPU_REQ_WIDTH))::get( - this, "", "request", request - ) || !uvm_config_db#(virtual stream_if #(`CPU_RESP_WIDTH))::get( - this, "", "response", response - ) || !uvm_config_db#(virtual stream_if #(`CPU_CACHE_WIDTH))::get( - this, "", "cache_request", cache_request - ) || !uvm_config_db#(virtual stream_if #(`CPU_CRESULT_WIDTH))::get( - this, "", "cache_response", cache_response - ) || !uvm_config_db#(virtual stream_if #(`CPU_UNCACHED_WIDTH))::get( - this, "", "uncached_request", uncached_request - ) || !uvm_config_db#(virtual stream_if #(`CPU_URESULT_WIDTH))::get( - this, "", "uncached_response", uncached_response - )) - `uvm_fatal("VIF", "CPU memory interfaces missing") - endfunction - task transact(longint unsigned addr, int size, bit write, bit signed_load, longint unsigned data, - int atomic_op, bit cacheable, longint unsigned raw = 64'h80ff7f0181fe8000, - bit error = 0, bit normal_ram = 0, bit device = 0); - req_t packet = '0; - int unsigned target, misaligned, access_fault, bus_mask, atomic_word; - longint unsigned bus_addr, bus_data, expected_data; - bit [`CPU_RESP_WIDTH-1:0] expected_response; - int tag = checked & ((1 << `CPU_TAG_BITS) - 1); - cpu_mem_ref_prepare(addr, size, write, data, atomic_op, cacheable, - !device && (cacheable || normal_ram), `CPU_PHYSICAL_BITS, target, - misaligned, access_fault, bus_addr, bus_data, bus_mask, atomic_word); - expected_data = target == 0 ? 0 : - cpu_mem_ref_result(addr, size, write, signed_load, atomic_op, cacheable, raw, error); - expected_response = '0; - expected_response[`CPUF(RESP, TAG)] = tag; - expected_response[`CPUF(RESP, DATA)] = expected_data; - expected_response[`CPUF(RESP, MISALIGNED)] = misaligned; - expected_response[`CPUF(RESP, ACCESSFAULT)] = access_fault || (target != 0 && error); - packet[`CPUF(REQ, ADDR)] = addr; - packet[`CPUF(REQ, TAG)] = tag; - packet[`CPUF(REQ, SIZE)] = size; - packet[`CPUF(REQ, WRITE)] = write; - packet[`CPUF(REQ, SIGNED)] = signed_load; - packet[`CPUF(REQ, DATA)] = data; - packet[`CPUF(REQ, ATOMIC)] = atomic_op; - packet[`CPUF(REQ, CACHEABLE)] = cacheable; - packet[`CPUF(REQ, NORMAL)] = !device && (cacheable || normal_ram); - @(control.cb); - response.ready <= 0; - cache_request.ready <= 0; - uncached_request.ready <= 0; - request.bits <= packet; - request.valid <= 1; - do @(request.sample); while (!request.sample.ready); - request.valid <= 0; - if (target == 1) begin - do @(cache_request.sample); while (!cache_request.sample.valid); - if (cache_request.sample.bits[ - `CPUF(CACHE, ADDR) - ] !== bus_addr || cache_request.sample.bits[ - `CPUF(CACHE, DATA) - ] !== bus_data || cache_request.sample.bits[ - `CPUF(CACHE, MASK) - ] !== bus_mask || cache_request.sample.bits[ - `CPUF(CACHE, WRITE) - ] !== write || cache_request.sample.bits[ - `CPUF(CACHE, ATOMIC) - ] !== atomic_op || cache_request.sample.bits[ - `CPUF(CACHE, ATOMICWORD) - ] !== atomic_word || uncached_request.valid) - `uvm_fatal("CACHE", $sformatf( - "Cache translation mismatch addr=%h size=%0d atomic=%0d got=%h", - addr, - size, - atomic_op, - cache_request.sample.bits - )) - repeat (3) @(control.cb); - cache_request.ready <= 1; - @(cache_request.sample); - cache_request.ready <= 0; - repeat (2) @(control.cb); - cache_response.bits <= '0; - cache_response.bits[`CPUF(CRESULT, DATA)] <= raw; - cache_response.bits[`CPUF(CRESULT, ERROR)] <= error; - cache_response.valid <= 1; - do @(cache_response.sample); while (!cache_response.sample.ready); - cache_response.valid <= 0; - cache_checked++; - end else if (target == 2) begin - cache_response.bits[`CPUF(CRESULT, DATA)] <= ~raw; - do @(uncached_request.sample); while (!uncached_request.sample.valid); - if (uncached_request.sample.bits[ - `CPUF(UNCACHED, ADDR) - ] !== addr || uncached_request.sample.bits[ - `CPUF(UNCACHED, SIZE) - ] !== size || uncached_request.sample.bits[ - `CPUF(UNCACHED, WRITE) - ] !== write || uncached_request.sample.bits[ - `CPUF(UNCACHED, DATA) - ] !== data || uncached_request.sample.bits[ - `CPUF(UNCACHED, TAG) - ] !== tag || cache_request.valid) - `uvm_fatal("MMIO", "Uncached access expanded or changed its address/size/data/tag") - repeat (3) @(control.cb); - uncached_request.ready <= 1; - @(uncached_request.sample); - uncached_request.ready <= 0; - repeat (2) @(control.cb); - uncached_response.bits <= '0; -`ifdef CPU_BAD_TAG - uncached_response.bits[`CPUF(URESULT, TAG)] <= tag ^ 1; -`else - uncached_response.bits[`CPUF(URESULT, TAG)] <= tag; -`endif - uncached_response.bits[`CPUF(URESULT, DATA)] <= raw; - uncached_response.bits[`CPUF(URESULT, ERROR)] <= error; - uncached_response.valid <= 1; - do @(uncached_response.sample); while (!uncached_response.sample.ready); - uncached_response.valid <= 0; - uncached_checked++; - end else begin - repeat (3) begin - @(control.cb); - if (cache_request.valid || uncached_request.valid) - `uvm_fatal("FAULT", "Faulting request reached an external memory port") - end - faults_checked++; - end - do @(response.sample); while (!response.sample.valid); - if (response.sample.bits !== expected_response) - `uvm_fatal("RESULT", $sformatf( - "CPU result mismatch addr=%h size=%0d atomic=%0d got=%h expected=%h", - addr, - size, - atomic_op, - response.sample.bits, - expected_response - )) - repeat (3) begin - @(control.cb); - if (request.ready || response.bits !== expected_response || !response.valid || - cache_request.valid || uncached_request.valid) - `uvm_fatal("SINGLE", "Outstanding CPU response changed or accepted another operation") - end - response.ready <= 1; - @(response.sample); - response.ready <= 0; - checked++; - endtask - // Reset jointly cancels the CPU operation and its memory response producer; stores are not rolled back. - task cancel_at(int phase_index, bit cacheable); - req_t packet = '0; - packet[`CPUF(REQ, ADDR)] = 64'h5000; - packet[`CPUF(REQ, SIZE)] = 3; - packet[`CPUF(REQ, TAG)] = 63; - packet[`CPUF(REQ, CACHEABLE)] = cacheable; - packet[`CPUF(REQ, NORMAL)] = cacheable; - @(control.cb); - request.bits <= packet; - request.valid <= 1; - cache_request.ready <= 0; - uncached_request.ready <= 0; - response.ready <= 0; - do @(request.sample); while (!request.sample.ready); - request.valid <= 0; - if (cacheable) begin - do @(cache_request.sample); while (!cache_request.sample.valid); - if (phase_index != 0) begin - cache_request.ready <= 1; - @(cache_request.sample); - cache_request.ready <= 0; - if (phase_index == 2) begin - cache_response.bits <= '0; - cache_response.valid <= 1; - do @(cache_response.sample); while (!cache_response.sample.ready); - cache_response.valid <= 0; - do @(response.sample); while (!response.sample.valid); - end - end - end else begin - do @(uncached_request.sample); while (!uncached_request.sample.valid); - if (phase_index != 0) begin - uncached_request.ready <= 1; - @(uncached_request.sample); - uncached_request.ready <= 0; - if (phase_index == 2) begin - uncached_response.bits <= '0; - uncached_response.bits[`CPUF(URESULT, TAG)] <= 63; - uncached_response.valid <= 1; - do @(uncached_response.sample); while (!uncached_response.sample.ready); - uncached_response.valid <= 0; - do @(response.sample); while (!response.sample.valid); - end - end - end - control.cb.reset <= 1; - cache_response.valid <= 0; - // Payload and VALID are ignored during joint reset, including an unrelated tag. - uncached_response.bits <= '0; - uncached_response.valid <= !cacheable && phase_index == 1; - repeat (2) @(control.cb); - uncached_response.valid <= 0; - control.cb.reset <= 0; - repeat (2) @(control.cb); - if (response.valid || cache_request.valid || uncached_request.valid || !request.ready) - `uvm_fatal("RESET", "Reset retained a cancelled transaction") - cancelled++; - endtask - task execute(); - control.reset = 1; - request.valid = 1; - request.bits = '1; - response.ready = 0; - cache_request.ready = 0; - uncached_request.ready = 0; - cache_response.valid = 0; - cache_response.bits = '0; - uncached_response.valid = 0; - uncached_response.bits = '0; - repeat (4) @(control.cb); - request.valid <= 0; - control.cb.reset <= 0; -`ifdef CPU_BAD_CASE - begin - req_t bad = '0; - bad[`CPUF(REQ, SIZE)] = `CPU_BAD_CASE == 1 ? 4 : (`CPU_BAD_CASE == 4 ? 1 : 3); - bad[`CPUF(REQ, ATOMIC)] = `CPU_BAD_CASE == 2 ? 12 : (`CPU_BAD_CASE == 3 ? 15 : 1); - bad[`CPUF(REQ, WRITE)] = `CPU_BAD_CASE == 5; - bad[`CPUF(REQ, CACHEABLE)] = 1; - @(control.cb); - request.bits <= bad; - request.valid <= 1; - repeat (10) @(control.cb); - `uvm_fatal("NO_ASSERT", "Invalid CPU memory contract was accepted") - end -`elsif CPU_BAD_TAG - transact(64'h60020000, 0, 0, 0, 0, 0, 0); - `uvm_fatal("NO_ASSERT", "Incorrect uncached response tag was accepted") -`else - for (int phase_index = 0; phase_index < 3; phase_index++) - for (int cacheable = 0; cacheable < 2; cacheable++) cancel_at(phase_index, cacheable); - for (int cacheable = 0; cacheable < 2; cacheable++) - for (int size = 0; size < 4; size++) - for (int offset = 0; offset < 8; offset++) - for (int sign_bit = 0; sign_bit < 2; sign_bit++) begin - transact(64'h60020000 + offset, size, 0, sign_bit, 0, 0, cacheable); - transact(64'h60020000 + offset, size, 1, 0, 64'hfedcba9876543210, 0, cacheable); - end - // Every aligned upper/lower Word atomic lane, including sign-expanded results and SC status. - for (int op = 1; op <= 11; op++) - for (int size = 2; size < 4; size++) - for (int offset = 0; offset < 8; offset += 4) begin - longint unsigned raw = op == 11 ? 1 : (size == 2 ? 64'hffffffff80000081 : 64'h8123456789abcdef); - transact(64'h1000 + offset, size, 0, 0, 64'h8877665544332211, op, 1, raw); - transact(64'h1000 + offset, size, 0, 1, 64'h1122334455667788, op, 1, raw, 1); - transact(64'h1000 + offset, size, 0, 0, 1, op, 0, raw); - end - for (int size = 2; size < 4; size++) - for (int offset = 1; offset < 8; offset++) - if (offset % (1 << size) != 0) transact(64'h1000 + offset, size, 0, 0, 1, 1, 1); - for (int size = 0; size < 4; size++) begin - transact(64'h2000, size, 0, 1, 0, 0, 1, '1); - transact(64'h2000, size, 0, 0, 0, 0, 1, 0); - transact(64'h2000, size, 0, 0, 0, 0, 1, '1, 1); - transact(64'h60020004, size, 0, 1, 0, 0, 0, '1, 1); - transact(64'h60020004, size, 1, 0, '1, 0, 0, 0, 1); - transact(64'h60020000, size, 0, 1, 0, 0, 0, '1); - transact(64'h60020000, size, 0, 0, 0, 0, 0, '1); - transact(64'h60020000, size, 1, 0, '1, 0, 0, 0, 1); - end - for (int bit_index = 3; bit_index < 64; bit_index++) begin - transact(64'h1 << bit_index, 3, 0, 0, 0, 0, 1); - transact(64'h1 << bit_index, 3, 1, 0, '1, 0, 0); - end - transact(64'h1004, 2, 0, 1, 0, 0, 1, 0); - transact(64'h60020004, 2, 0, 1, 0, 0, 0, 0); - transact(64'h1000, 3, 1, 0, 0, 0, 1, 0, 1); - transact(64'h1000, 2, 0, 0, 1, 11, 1, 0); - transact(64'h1004, 2, 0, 0, 1, 11, 1, 0); - transact(64'h1000, 3, 0, 0, 1, 11, 1, 0); - transact(64'h1000, 3, 1, 0, 0, 0, 1); - transact(64'h1000, 3, 1, 0, '1, 0, 1); - transact((64'h1 << `CPU_PHYSICAL_BITS) - 8, 3, 0, 0, 0, 0, 1); - transact((64'h1 << `CPU_PHYSICAL_BITS) - 1, 0, 1, 0, '1, 0, 1); - // Noncacheable normal memory, including every atomic, faults without issuing traffic. - for (int size = 0; size < 4; size++) begin - transact(64'h3000, size, 0, 0, 0, 0, 0, 0, 0, 1); - transact(64'h3000, size, 1, 0, 1, 0, 0, 0, 0, 1); - transact(64'h3001, size, 0, 0, 0, 0, 0, 0, 0, 1); - transact(64'h60020000, size, 0, 0, 0, 0, 1, 0, 0, 0, 1); - transact(64'h60020000, size, 1, 0, 1, 0, 1, 0, 0, 0, 1); - transact(64'h60020001, size, 0, 0, 0, 0, 1, 0, 0, 0, 1); - end - for (int op = 1; op <= 11; op++) - for (int size = 2; size <= 3; size++) begin - transact(64'h3000 + (size == 2 ? 4 : 0), size, 0, 0, 64'h76543210, op, 0, - op == 11 ? 0 : 64'hffffffff81234567, 0, 1); - transact(64'h3000, size, 0, 0, 1, op, 0, 64'hdeadbeef12345678, 1, 1); - end - transact(64'h3004, 2, 0, 0, 1, 11, 0, 1, 0, 1); - // Reset ignores even VALID=1 garbage payload; no transaction may escape reset. - control.cb.reset <= 1; - request.valid <= 1; - request.bits <= '1; - repeat (2) @(control.cb); - request.valid <= 0; - request.bits <= '0; - control.cb.reset <= 0; - repeat (2) @(control.cb); - if (response.valid || cache_request.valid || uncached_request.valid || !request.ready) - `uvm_fatal("RESET_PAYLOAD", "Reset payload escaped as a transaction") - // VALID=0 payload is unconstrained; it must not create an operation. - response.ready <= 1; - cache_request.ready <= 1; - uncached_request.ready <= 1; - request.bits <= '1; - cache_response.bits <= '1; - uncached_response.bits <= '1; - repeat (2) @(control.cb); - request.bits <= '0; - cache_response.bits <= '0; - uncached_response.bits <= '0; - repeat (2) @(control.cb); - if (response.valid || cache_request.valid || uncached_request.valid || !request.ready) - `uvm_fatal("IDLE", "Idle payload generated an operation") - `uvm_info("CPU_MEM", $sformatf( - "Checked %0d CPU results, %0d cache requests, %0d exact uncached requests, %0d suppressed faults, %0d joint-reset cancellations", - checked, - cache_checked, - uncached_checked, - faults_checked, - cancelled - ), UVM_LOW) -`endif - endtask -endclass diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/scala/CpuMem.scala b/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/scala/CpuMem.scala deleted file mode 100644 index acf22eee6..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/scala/CpuMem.scala +++ /dev/null @@ -1,135 +0,0 @@ -package memcore.memory.cpu - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import memcore.bus.chi.Params -import memcore.bus.chi.rnf.{CacheAccess, CacheAtomic, CacheResult} - -case class CpuMemParams(chi: Params = Params(), tagBits: Int = 6) { - require(chi.addressBits < 64 && tagBits > 0) -} - -class CpuMemoryRequest(p: CpuMemParams) extends Bundle { - val addr = UInt(64.W) - val tag = UInt(p.tagBits.W) - val size = UInt(3.W) - val write = Bool() - val signed = Bool() - val data = UInt(64.W) - val atomic = UInt(4.W) - val cacheable = Bool() - val normal = Bool() -} - -/** `line` is the whole cached line for full-line clients; it is absent by default. */ -class CpuMemoryResponse(p: CpuMemParams, lineBits: Int = 0) extends Bundle { - val tag = UInt(p.tagBits.W) - val data = UInt(64.W) - val misaligned = Bool() - val accessFault = Bool() - val line = UInt(lineBits.W) -} - -class UncachedRequest(p: CpuMemParams) extends Bundle { - val addr = UInt(64.W) - val tag = UInt(p.tagBits.W) - val size = UInt(3.W) - val write = Bool() - val data = UInt(64.W) -} - -class UncachedResponse(p: CpuMemParams) extends Bundle { - val tag = UInt(p.tagBits.W) - val data = UInt(64.W) - val error = Bool() -} - -@instantiable -class CpuMem(p: CpuMemParams, lineBits: Int = 0) extends Module { - - @public - val io = IO(new Bundle { - val request = Flipped(Decoupled(new CpuMemoryRequest(p))) - val response = Decoupled(new CpuMemoryResponse(p, lineBits)) - val cacheRequest = Decoupled(new CacheAccess(p.chi)) - val cacheResponse = Flipped(Decoupled(new CacheResult(lineBits))) - val uncachedRequest = Decoupled(new UncachedRequest(p)) - val uncachedResponse = Flipped(Decoupled(new UncachedResponse(p))) - }) - - val idle :: issueCache :: waitCache :: issueUncached :: waitUncached :: respond :: Nil = Enum(6) - val state = RegInit(idle) - val command = Reg(new CpuMemoryRequest(p)) - val answer = Reg(new CpuMemoryResponse(p, lineBits)) - - val mask = MuxLookup(command.size, 255.U(8.W))(Seq( - 0.U -> 1.U(8.W), - 1.U -> 3.U(8.W), - 2.U -> 15.U(8.W) - )) - - io.request.ready := state === idle - when(io.request.fire) { - val r = io.request.bits - assert(r.size <= 3.U, "CPU memory size must be 1, 2, 4 or 8 bytes") - assert(r.atomic <= CacheAtomic.SC.U, "CPU memory port does not accept Fence or unknown atomics") - assert(r.atomic === CacheAtomic.None.U || (!r.write && r.size >= 2.U), "CPU atomic requires a 32 or 64 bit operand") - val misaligned = (r.addr & ((1.U(64.W) << r.size) - 1.U)).orR - val accessFault = r.addr(63, p.chi.addressBits).orR || (r.normal && !r.cacheable) || - (!r.normal && (r.cacheable || r.atomic =/= CacheAtomic.None.U)) - command := r - answer.tag := r.tag - answer.data := 0.U - if (lineBits > 0) answer.line := 0.U - answer.misaligned := misaligned - answer.accessFault := !misaligned && accessFault - state := Mux(misaligned || accessFault, respond, Mux(r.cacheable, issueCache, issueUncached)) - } - - val atomic = command.atomic =/= CacheAtomic.None.U - io.cacheRequest.valid := state === issueCache - io.cacheRequest.bits.addr := Mux(atomic, command.addr, command.addr & ~7.U(64.W))(p.chi.addressBits - 1, 0) - io.cacheRequest.bits.write := command.write - io.cacheRequest.bits.data := Mux(atomic, command.data, command.data << (command.addr(2, 0) << 3)) - io.cacheRequest.bits.mask := Mux(atomic, 255.U, Mux(command.write, mask << command.addr(2, 0), 0.U)) - io.cacheRequest.bits.atomic := command.atomic - io.cacheRequest.bits.atomicWord := atomic && command.size === 2.U - when(io.cacheRequest.fire)(state := waitCache) - io.cacheResponse.ready := state === waitCache - io.uncachedRequest.valid := state === issueUncached - io.uncachedRequest.bits.addr := command.addr - io.uncachedRequest.bits.tag := command.tag - io.uncachedRequest.bits.size := command.size - io.uncachedRequest.bits.write := command.write - io.uncachedRequest.bits.data := command.data - when(io.uncachedRequest.fire)(state := waitUncached) - io.uncachedResponse.ready := state === waitUncached - - val raw = - Mux(state === waitUncached, io.uncachedResponse.bits.data, io.cacheResponse.bits.data >> (command.addr(2, 0) << 3)) - - val load = MuxLookup(command.size, raw)(Seq( - 0.U -> Cat(Fill(56, command.signed && raw(7)), raw(7, 0)), - 1.U -> Cat(Fill(48, command.signed && raw(15)), raw(15, 0)), - 2.U -> Cat(Fill(32, command.signed && raw(31)), raw(31, 0)) - )) - - when(io.cacheResponse.fire || io.uncachedResponse.fire) { - val error = Mux(state === waitUncached, io.uncachedResponse.bits.error, io.cacheResponse.bits.error) - when(io.uncachedResponse.fire) { - assert(io.uncachedResponse.bits.tag === command.tag, "Uncached response tag mismatch") - } - answer.data := Mux( - error || command.write, - 0.U, - Mux(atomic, io.cacheResponse.bits.data, load) - ) - answer.accessFault := error - if (lineBits > 0) answer.line := Mux(state === waitCache && !error, io.cacheResponse.bits.line, 0.U) - state := respond - } - io.response.valid := state === respond - io.response.bits := answer - when(io.response.fire)(state := idle) -} diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/scala/Emit.scala b/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/scala/Emit.scala deleted file mode 100644 index a55ff637a..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/scala/Emit.scala +++ /dev/null @@ -1,92 +0,0 @@ -package memcore.memory.cpu - -import java.nio.file.{Files, Paths} - -object Emit extends App { - val p = CpuMemParams() - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new CpuMem(p), - firtoolOpts = args, - args = Array("--target-dir", "build/CpuMem", "--split-verilog") - ) - - val definitions = scala.collection.mutable.ArrayBuffer[String]( - s"`define CPU_PHYSICAL_BITS ${p.chi.addressBits}", - s"`define CPU_TAG_BITS ${p.tagBits}" - ) - - val ports = scala.collection.mutable.ArrayBuffer[String]() - - def channel( - kind: String, - prefix: String, - instance: String, - fields: Seq[(String, Int)] - ): Unit = { - var offset = 0 - for ((field, width) <- fields) { - definitions += s"`define CPU_${kind}_${field.toUpperCase}_OFFSET $offset" - definitions += s"`define CPU_${kind}_${field.toUpperCase}_WIDTH $width" - ports += s".$prefix$field($instance.bits[$offset +: $width])" - offset += width - } - definitions += s"`define CPU_${kind}_WIDTH $offset" - } - - channel( - "REQ", - "io_request_bits_", - "request", - Seq( - "addr" -> 64, - "tag" -> p.tagBits, - "size" -> 3, - "write" -> 1, - "signed" -> 1, - "data" -> 64, - "atomic" -> 4, - "cacheable" -> 1, - "normal" -> 1 - ) - ) - channel( - "RESP", - "io_response_bits_", - "response", - Seq("tag" -> p.tagBits, "data" -> 64, "misaligned" -> 1, "accessFault" -> 1) - ) - channel( - "CACHE", - "io_cacheRequest_bits_", - "cache_request", - Seq("addr" -> p.chi.addressBits, "write" -> 1, "data" -> 64, "mask" -> 8, "atomic" -> 4, "atomicWord" -> 1) - ) - channel("CRESULT", "io_cacheResponse_bits_", "cache_response", Seq("data" -> 64, "error" -> 1)) - channel( - "UNCACHED", - "io_uncachedRequest_bits_", - "uncached_request", - Seq("addr" -> 64, "tag" -> p.tagBits, "size" -> 3, "write" -> 1, "data" -> 64) - ) - channel( - "URESULT", - "io_uncachedResponse_bits_", - "uncached_response", - Seq("tag" -> p.tagBits, "data" -> 64, "error" -> 1) - ) - for ( - (prefix, instance) <- Seq( - "request" -> "request", - "response" -> "response", - "cacheRequest" -> "cache_request", - "cacheResponse" -> "cache_response", - "uncachedRequest" -> "uncached_request", - "uncachedResponse" -> "uncached_response" - ) - ) { - ports += s".io_${prefix}_valid($instance.valid)" - ports += s".io_${prefix}_ready($instance.ready)" - } - Files.write(Paths.get("build/cpu_mem_config.svh"), (definitions.mkString("\n") + "\n").getBytes) - Files.write(Paths.get("build/cpu_mem_ports.svh"), ports.mkString(",\n").getBytes) -} diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/scala/VirtualMemory.scala b/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/scala/VirtualMemory.scala deleted file mode 100644 index 9cf9a104a..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/scala/VirtualMemory.scala +++ /dev/null @@ -1,306 +0,0 @@ -package memcore.memory.cpu - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import memcore.bus.chi.rnf.{CacheAccess, CacheAtomic, CacheResult} -import memcore.memory.mmu.Walker - -case class PhysicalRegion( - base: BigInt, - bytes: BigInt, - cacheable: Boolean, - executable: Boolean, - readable: Boolean, - writable: Boolean, - atomic: Boolean, - normal: Boolean) - -class PhysicalAuthorization extends Bundle { - val paddr = UInt(64.W) - val size = UInt(3.W) - val read = Bool() - val write = Bool() - val execute = Bool() - val isPte = Bool() - val privilege = UInt(2.W) -} - -class VirtualMemoryRequest(p: CpuMemParams) extends Bundle { - val vaddr = UInt(64.W) - val tag = UInt(p.tagBits.W) - val size = UInt(3.W) - val write = Bool() - val execute = Bool() - val signed = Bool() - val data = UInt(64.W) - val atomic = UInt(4.W) - val privilege = UInt(2.W) - val sum = Bool() - val mxr = Bool() - val satpMode = UInt(4.W) - val rootPpn = UInt(44.W) -} - -class VirtualMemoryResponse(p: CpuMemParams, lineBits: Int = 0) extends Bundle { - val tag = UInt(p.tagBits.W) - val data = UInt(64.W) - val misaligned = Bool() - val pageFault = Bool() - val accessFault = Bool() - // Whole cached line of the translated word, for full-line clients only. - val line = UInt(lineBits.W) -} - -/** Single-outstanding virtual data/instruction-word frontend; cache ownership is external. */ -@instantiable -class VirtualMemory( - cp: CpuMemParams, - regions: Seq[PhysicalRegion], - lineBits: Int = 0, - translationEntries: Int = 0) - extends Module { - require(regions.nonEmpty) - for ((region, i) <- regions.zipWithIndex) { - require(region.base >= 0 && region.bytes > 0 && region.base + region.bytes <= (BigInt(1) << cp.chi.addressBits)) - require(!region.cacheable || (region.base % 64 == 0 && region.bytes % 64 == 0)) - require( - region.normal || (!region.cacheable && !region.executable && !region.atomic), - "Device regions cannot be cached, executable or atomic" - ) - for (other <- regions.drop(i + 1)) { - require(region.base + region.bytes <= other.base || other.base + other.bytes <= region.base) - } - } - - @public - val io = IO(new Bundle { - val active = Input(Bool()) - // SFENCE.VMA: drop every cached translation. - val flushTranslation = Input(Bool()) - - // Combinational translation through the TLB only, for a caller's fast path. - val lookup = new Bundle { - val config = Input(new memcore.memory.mmu.Config) - val req = Input(new memcore.memory.mmu.Request) - val hit = Output(Bool()) - val paddr = Output(UInt(cp.chi.addressBits.W)) - } - - val request = Flipped(Decoupled(new VirtualMemoryRequest(cp))) - val response = Decoupled(new VirtualMemoryResponse(cp, lineBits)) - val authorizationRequest = Decoupled(new PhysicalAuthorization) - val authorizationResponse = Flipped(Decoupled(Bool())) - val cacheRequest = Decoupled(new CacheAccess(cp.chi)) - val cacheResponse = Flipped(Decoupled(new CacheResult(lineBits))) - val uncachedRequest = Decoupled(new UncachedRequest(cp)) - val uncachedResponse = Flipped(Decoupled(new UncachedResponse(cp))) - val observedTranslation = Output(Valid(new memcore.memory.mmu.Response(cp.chi))) - val observedPhysical = Output(Valid(new CpuMemoryRequest(cp))) - - val observedCache = Output(Valid(new Bundle { - val pte = Bool() - val access = new CacheAccess(cp.chi) - })) - - }) - - val walker: Instance[Walker] = Instantiate(new Walker(cp.chi, translationEntries)) - val physical: Instance[CpuMem] = Instantiate(new CpuMem(cp, lineBits)) - - val idle :: startWalk :: waitWalk :: authorize :: waitAuthorization :: startAccess :: waitAccess :: respond :: Nil = - Enum(8) - val state = RegInit(idle) - val command = Reg(new VirtualMemoryRequest(cp)) - val answer = Reg(new VirtualMemoryResponse(cp, lineBits)) - val paddr = Reg(UInt(64.W)) - val cacheable = Reg(Bool()) - val normal = Reg(Bool()) - val pteInFlight = RegInit(false.B) - val pteError = RegInit(false.B) - val pteAuthorized = RegInit(false.B) - val pteAuthorizationPending = RegInit(false.B) - val regionAllowed = Reg(Bool()) - def matches(first: UInt, last: UInt): Seq[Bool] = - regions.map(region => first >= region.base.U(65.W) && last < (region.base + region.bytes).U(65.W)) - io.request.ready := state === idle && io.active - when(io.request.fire) { - val r = io.request.bits - assert(r.size <= 3.U, "Virtual LSU size must be 1, 2, 4 or 8 bytes") - assert(r.atomic <= CacheAtomic.SC.U, "Virtual LSU does not accept Fence or unknown atomics") - assert( - r.atomic === CacheAtomic.None.U || (!r.write && r.size >= 2.U), - "Virtual LSU atomic requires a 32 or 64 bit operand" - ) - assert(r.satpMode === 0.U || r.satpMode === 8.U, "Virtual LSU requires Bare or Sv39 mode") - assert( - r.privilege === 0.U || r.privilege === 1.U || r.privilege === 3.U, - "Virtual LSU effective privilege must be U, S or M" - ) - assert( - !r.execute || (!r.write && r.atomic === CacheAtomic.None.U && r.size === 3.U && !r.signed), - "Virtual instruction word requires unsigned 64 bit read without atomic" - ) - command := r - answer := 0.U.asTypeOf(answer) - answer.tag := r.tag - val misaligned = (r.vaddr & ((1.U(64.W) << r.size) - 1.U)).orR - answer.misaligned := misaligned - state := Mux(misaligned, respond, startWalk) - } - - walker.io.flush := io.flushTranslation - walker.io.lookup.config := io.lookup.config - walker.io.lookup.req := io.lookup.req - io.lookup.hit := walker.io.lookup.hit - io.lookup.paddr := walker.io.lookup.paddr - walker.io.config.mode := command.satpMode - walker.io.config.rootPpn := command.rootPpn - walker.io.req.valid := state === startWalk - walker.io.req.bits.vaddr := command.vaddr - walker.io.req.bits.write := command.write || - (command.atomic =/= CacheAtomic.None.U && command.atomic =/= CacheAtomic.LR.U) - walker.io.req.bits.execute := command.execute - walker.io.req.bits.privilege := command.privilege - walker.io.req.bits.sum := command.sum - walker.io.req.bits.mxr := command.mxr - when(walker.io.req.fire)(state := waitWalk) - walker.io.resp.ready := state === waitWalk - - val lastByte = walker.io.resp.bits.paddr.pad(65) + MuxLookup(command.size, 0.U(4.W))( - Seq(1.U -> 1.U(4.W), 2.U -> 3.U(4.W), 3.U -> 7.U(4.W)) - ) - - val dataRegions = matches(walker.io.resp.bits.paddr.pad(65), lastByte) - val isAtomic = command.atomic =/= CacheAtomic.None.U - val readsData = !command.execute && !command.write && command.atomic =/= CacheAtomic.SC.U - val writesData = command.write || (isAtomic && command.atomic =/= CacheAtomic.LR.U) - - val permittedRegion = dataRegions.zip(regions).map { case (hit, region) => - hit && (!region.normal || region.cacheable).B && (!command.execute || (region.normal && region.executable).B) && - (!readsData || region.readable.B) && (!writesData || region.writable.B) && - (!isAtomic || region.atomic.B) - }.reduce(_ || _) - - when(walker.io.resp.fire) { - when(walker.io.resp.bits.pageFault || walker.io.resp.bits.accessFault) { - answer.pageFault := walker.io.resp.bits.pageFault - answer.accessFault := walker.io.resp.bits.accessFault - state := respond - }.otherwise { - paddr := walker.io.resp.bits.paddr - cacheable := dataRegions.zip(regions).map { case (hit, region) => hit && region.cacheable.B }.reduce(_ || _) - normal := dataRegions.zip(regions).map { case (hit, region) => hit && region.normal.B }.reduce(_ || _) - regionAllowed := permittedRegion - state := authorize - } - } - - physical.io.request.valid := state === startAccess - physical.io.request.bits.addr := paddr - physical.io.request.bits.tag := command.tag - physical.io.request.bits.size := command.size - physical.io.request.bits.write := command.write - physical.io.request.bits.signed := command.signed - physical.io.request.bits.data := command.data - physical.io.request.bits.atomic := command.atomic - physical.io.request.bits.cacheable := cacheable - physical.io.request.bits.normal := normal - when(physical.io.request.fire)(state := waitAccess) - physical.io.response.ready := state === waitAccess - when(physical.io.response.fire) { - assert(physical.io.response.bits.tag === command.tag, "Virtual LSU physical response tag mismatch") - answer.data := physical.io.response.bits.data - answer.misaligned := physical.io.response.bits.misaligned - answer.accessFault := physical.io.response.bits.accessFault - if (lineBits > 0) answer.line := physical.io.response.bits.line - state := respond - } - - val walking = state === startWalk || state === waitWalk - val accessing = state === startAccess || state === waitAccess - val pteFirst = walker.io.access.bits.addr.pad(65) - val pteRegions = matches(pteFirst, pteFirst + 7.U) - - val pteAllowed = pteRegions.zip(regions).map { case (hit, region) => - hit && (region.normal && region.readable && region.cacheable).B - }.reduce(_ || _) - - // The Core owns a per-command PMP snapshot. Each permission decision is a - // single-outstanding handshake; neither a stalled query nor its response - // allows speculative cache or MMIO traffic. PTE accesses use S privilege. - io.authorizationRequest.valid := state === authorize || - (walking && walker.io.access.valid && !pteAuthorized && !pteAuthorizationPending && !pteError) - io.authorizationRequest.bits.paddr := Mux(walking, walker.io.access.bits.addr.pad(64), paddr) - io.authorizationRequest.bits.size := Mux(walking, 3.U, command.size) - io.authorizationRequest.bits.read := walking || readsData - io.authorizationRequest.bits.write := !walking && writesData - io.authorizationRequest.bits.execute := !walking && command.execute - io.authorizationRequest.bits.isPte := walking - io.authorizationRequest.bits.privilege := Mux(walking, 1.U, command.privilege) - when(io.authorizationRequest.fire) { - when(walking) { - pteAuthorizationPending := true.B - } - .otherwise(state := waitAuthorization) - } - io.authorizationResponse.ready := pteAuthorizationPending || state === waitAuthorization - when(io.authorizationResponse.fire) { - when(pteAuthorizationPending) { - pteAuthorizationPending := false.B - pteAuthorized := true.B - pteError := !io.authorizationResponse.bits || !pteAllowed - }.otherwise { - when(io.authorizationResponse.bits && regionAllowed)(state := startAccess) - .otherwise { answer.accessFault := true.B; state := respond } - } - } - io.cacheRequest.valid := Mux( - walking, - walker.io.access.valid && pteAuthorized && !pteError, - accessing && physical.io.cacheRequest.valid - ) - io.cacheRequest.bits := Mux(walking, walker.io.access.bits, physical.io.cacheRequest.bits) - io.uncachedRequest.valid := accessing && physical.io.uncachedRequest.valid - io.uncachedRequest.bits := physical.io.uncachedRequest.bits - walker.io.access.ready := walking && pteAuthorized && - (pteError || io.cacheRequest.ready) - when(walker.io.access.fire) { - pteAuthorized := false.B - pteInFlight := !pteError - } - walker.io.result.valid := walking && ((!pteAuthorized && pteError) || (pteInFlight && io.cacheResponse.valid)) - walker.io.result.bits := 0.U.asTypeOf(walker.io.result.bits) - walker.io.result.bits.data := Mux(pteError || io.cacheResponse.bits.error, 0.U, io.cacheResponse.bits.data) - walker.io.result.bits.error := pteError || io.cacheResponse.bits.error - when(walker.io.result.fire) { pteError := false.B; pteInFlight := false.B } - physical.io.cacheRequest.ready := accessing && io.cacheRequest.ready - physical.io.cacheResponse.valid := accessing && io.cacheResponse.valid - physical.io.cacheResponse.bits := io.cacheResponse.bits - physical.io.uncachedRequest.ready := accessing && io.uncachedRequest.ready - physical.io.uncachedResponse.valid := accessing && io.uncachedResponse.valid - physical.io.uncachedResponse.bits := io.uncachedResponse.bits - io.cacheResponse.ready := Mux( - walking, - pteInFlight && walker.io.result.ready, - accessing && physical.io.cacheResponse.ready - ) - io.uncachedResponse.ready := accessing && physical.io.uncachedResponse.ready - when(io.cacheResponse.valid) { - assert((walking && pteInFlight) || accessing, "Virtual LSU cache result has no owner") - } - when(io.uncachedResponse.valid) { - assert(accessing, "Virtual LSU uncached result has no owner") - } - - io.response.valid := state === respond - io.response.bits := answer - when(io.response.fire)(state := idle) - io.observedTranslation.valid := walker.io.resp.fire - io.observedTranslation.bits := walker.io.resp.bits - io.observedPhysical.valid := physical.io.request.fire - io.observedPhysical.bits := physical.io.request.bits - io.observedCache.valid := io.cacheRequest.fire - io.observedCache.bits.pte := walking - io.observedCache.bits.access := io.cacheRequest.bits -} diff --git a/arch/src/main/scala/framework/mem-core/ddr/build.sc b/arch/src/main/scala/framework/mem-core/ddr/build.sc deleted file mode 100644 index 60f1b35d0..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/build.sc +++ /dev/null @@ -1,35 +0,0 @@ -import mill._ -import mill.scalalib._ - -trait ChiselModule extends SbtModule { - def moduleRoot: os.Path - override def millSourcePath = moduleRoot - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") -} - -object queue extends SbtModule { - override def millSourcePath = os.pwd / os.up / "queue" - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") -} - -object chi extends ChiselModule { - override def moduleDeps = Seq(queue) - override def moduleRoot = os.pwd / os.up / "chi" -} - -object axi4 extends ChiselModule { - override def moduleDeps = Seq(queue) - override def moduleRoot = os.pwd / os.up / "axi4" -} - -object ddr extends ChiselModule { - override def moduleRoot = os.pwd - override def moduleDeps = Seq(chi, axi4) - override def mainClass = Some("memcore.memory.ddr.Emit") -} diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/csrc/src/lib.rs b/arch/src/main/scala/framework/mem-core/ddr/src/csrc/src/lib.rs deleted file mode 100644 index 879e3a09c..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/csrc/src/lib.rs +++ /dev/null @@ -1,2 +0,0 @@ -mod memory; -pub use memory::*; diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_alignment.f b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_alignment.f deleted file mode 100644 index a0b44121a..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_alignment.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+DDR_TB=bad_alignment_tb -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Bridge/filelist.f -@RESOURCES@/bridge_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_alignment.toml b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_alignment.toml deleted file mode 100644 index 75bd23a9d..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_alignment.toml +++ /dev/null @@ -1,2 +0,0 @@ -expected_assertion = "DDR line request must be 64 byte aligned" -invalid_case = 0 diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_early_b.f b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_early_b.f deleted file mode 100644 index bb2e41872..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_early_b.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+DDR_TB=bad_early_b_tb -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Bridge/filelist.f -@RESOURCES@/bridge_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_early_b.toml b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_early_b.toml deleted file mode 100644 index cfe95574d..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_early_b.toml +++ /dev/null @@ -1,2 +0,0 @@ -expected_assertion = "DDR received B before AW and complete W or for inactive ID" -invalid_case = 2 diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_early_rlast.f b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_early_rlast.f deleted file mode 100644 index 38ced94f3..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_early_rlast.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+DDR_TB=bad_early_rlast_tb -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Bridge/filelist.f -@RESOURCES@/bridge_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_early_rlast.toml b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_early_rlast.toml deleted file mode 100644 index 25e224c8a..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_early_rlast.toml +++ /dev/null @@ -1,2 +0,0 @@ -expected_assertion = "DDR AXI RLAST does not match line beat count" -invalid_case = 3 diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_late_rlast.f b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_late_rlast.f deleted file mode 100644 index 4e063c528..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_late_rlast.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+DDR_TB=bad_late_rlast_tb -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Bridge/filelist.f -@RESOURCES@/bridge_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_late_rlast.toml b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_late_rlast.toml deleted file mode 100644 index 5a24795e0..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_late_rlast.toml +++ /dev/null @@ -1,2 +0,0 @@ -expected_assertion = "DDR AXI RLAST does not match line beat count" -invalid_case = 4 diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_read_exokay.f b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_read_exokay.f deleted file mode 100644 index 44dd5c1e8..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_read_exokay.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+DDR_TB=bad_read_exokay_tb -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Bridge/filelist.f -@RESOURCES@/bridge_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_read_exokay.toml b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_read_exokay.toml deleted file mode 100644 index e2994b234..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_read_exokay.toml +++ /dev/null @@ -1,2 +0,0 @@ -expected_assertion = "DDR nonexclusive read received EXOKAY" -invalid_case = 5 diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_rid.f b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_rid.f deleted file mode 100644 index ed36259a9..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_rid.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+DDR_TB=bad_rid_tb -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Bridge/filelist.f -@RESOURCES@/bridge_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_rid.toml b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_rid.toml deleted file mode 100644 index 13b50d759..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_rid.toml +++ /dev/null @@ -1,2 +0,0 @@ -expected_assertion = "DDR received R for an inactive AXI ID" -invalid_case = 1 diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_write_exokay.f b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_write_exokay.f deleted file mode 100644 index aba96f4a5..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_write_exokay.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+DDR_TB=bad_write_exokay_tb -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Bridge/filelist.f -@RESOURCES@/bridge_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_write_exokay.toml b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_write_exokay.toml deleted file mode 100644 index 266196316..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_write_exokay.toml +++ /dev/null @@ -1,2 +0,0 @@ -expected_assertion = "DDR nonexclusive write received EXOKAY" -invalid_case = 6 diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge.f b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge.f deleted file mode 100644 index 58c2ae4cc..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Bridge/filelist.f -@RESOURCES@/bridge_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge256.f b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge256.f deleted file mode 100644 index 08805a0d1..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge256.f +++ /dev/null @@ -1,11 +0,0 @@ -+define+DDR_TB=bridge256_tb -+define+DDR_PROFILE256 -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Bridge256/filelist.f -@RESOURCES@/bridge_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge512.f b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge512.f deleted file mode 100644 index 92cd77dcf..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge512.f +++ /dev/null @@ -1,11 +0,0 @@ -+define+DDR_TB=bridge512_tb -+define+DDR_PROFILE512 -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Bridge512/filelist.f -@RESOURCES@/bridge_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge64.f b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge64.f deleted file mode 100644 index 31a72de29..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge64.f +++ /dev/null @@ -1,11 +0,0 @@ -+define+DDR_TB=bridge64_tb -+define+DDR_PROFILE64 -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Bridge64/filelist.f -@RESOURCES@/bridge_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge_six.f b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge_six.f deleted file mode 100644 index 85fa55b93..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge_six.f +++ /dev/null @@ -1,11 +0,0 @@ -+define+DDR_TB=bridge_six_tb -+define+DDR_PROFILESIX -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/BridgeSix/filelist.f -@RESOURCES@/bridge_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge_tb.sv b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge_tb.sv deleted file mode 100644 index 85dd485bf..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge_tb.sv +++ /dev/null @@ -1,71 +0,0 @@ -`include "profile.svh" -`ifndef DDR_TB -`define DDR_TB bridge_tb -`endif -module `DDR_TB; - import uvm_pkg::*; - import ddr_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - ddr_control_if ctl (clock); - stream_if #(`DDR_REQ_WIDTH) - req0 ( - clock, - ctl.reset - ), - req1 ( - clock, - ctl.reset - ); - stream_if #(`DDR_RESP_WIDTH) - resp0 ( - clock, - ctl.reset - ), - resp1 ( - clock, - ctl.reset - ); - stream_if #(`DDR_A_WIDTH) - aw ( - clock, - ctl.reset - ), - ar ( - clock, - ctl.reset - ); - stream_if #(`DDR_W_WIDTH) w ( - clock, - ctl.reset - ); - stream_if #(`DDR_B_WIDTH) b ( - clock, - ctl.reset - ); - stream_if #(`DDR_R_WIDTH) r ( - clock, - ctl.reset - ); - Bridge dut ( - .clock(clock), - .reset(ctl.reset), - `include `DDR_PORTS - ); - initial begin - int invalid_case; - uvm_config_db#(virtual ddr_control_if)::set(null, "uvm_test_top", "ctl", ctl); - uvm_config_db#(virtual stream_if #(`DDR_REQ_WIDTH))::set(null, "uvm_test_top", "req0", req0); - uvm_config_db#(virtual stream_if #(`DDR_REQ_WIDTH))::set(null, "uvm_test_top", "req1", req1); - uvm_config_db#(virtual stream_if #(`DDR_RESP_WIDTH))::set(null, "uvm_test_top", "resp0", resp0); - uvm_config_db#(virtual stream_if #(`DDR_RESP_WIDTH))::set(null, "uvm_test_top", "resp1", resp1); - uvm_config_db#(virtual stream_if #(`DDR_A_WIDTH))::set(null, "uvm_test_top", "aw", aw); - uvm_config_db#(virtual stream_if #(`DDR_A_WIDTH))::set(null, "uvm_test_top", "ar", ar); - uvm_config_db#(virtual stream_if #(`DDR_W_WIDTH))::set(null, "uvm_test_top", "w", w); - uvm_config_db#(virtual stream_if #(`DDR_B_WIDTH))::set(null, "uvm_test_top", "b", b); - uvm_config_db#(virtual stream_if #(`DDR_R_WIDTH))::set(null, "uvm_test_top", "r", r); - if ($value$plusargs("invalid_case=%d", invalid_case)) - uvm_config_db#(int)::set(null, "uvm_test_top", "invalid_case", invalid_case); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/package.sv b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/package.sv deleted file mode 100644 index 146228cff..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/package.sv +++ /dev/null @@ -1,599 +0,0 @@ -`include "profile.svh" -`define DF(K, F) `DDR_``K``_``F``_OFFSET +: `DDR_``K``_``F``_WIDTH -package ddr_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - import "DPI-C" function chandle ddr_ref_create(); - import "DPI-C" function void ddr_ref_destroy(input chandle model); - import "DPI-C" function void ddr_ref_program( - input chandle model, - input longint unsigned address, - input bit [511:0] data - ); - import "DPI-C" function int unsigned ddr_ref_read( - input chandle model, - input longint unsigned address, - output bit [511:0] data - ); - import "DPI-C" function int unsigned ddr_ref_write( - input chandle model, - input longint unsigned address, - input bit [511:0] data, - input longint unsigned mask - ); - typedef longint unsigned u64; - typedef bit [`DDR_REQ_WIDTH-1:0] req_t; - class result_item extends uvm_sequence_item; - bit [511:0] data; - bit error; - int tag, client; - `uvm_object_utils_begin(result_item) - - `uvm_field_int(data, UVM_DEFAULT) - `uvm_field_int(error, UVM_DEFAULT) - - `uvm_field_int(tag, UVM_DEFAULT) - `uvm_field_int(client, UVM_DEFAULT) - - `uvm_object_utils_end - function new(string name = "result_item"); - super.new(name); - endfunction - endclass - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual ddr_control_if ctl; - virtual stream_if #(`DDR_REQ_WIDTH) req[2]; - virtual stream_if #(`DDR_RESP_WIDTH) resp[2]; - virtual stream_if #(`DDR_A_WIDTH) aw, ar; - virtual stream_if #(`DDR_W_WIDTH) w; - virtual stream_if #(`DDR_B_WIDTH) b; - virtual stream_if #(`DDR_R_WIDTH) r; - keyed_scoreboard #(result_item) scoreboard; - chandle model; - typedef struct { - req_t packet; - int client; - bit done; - } command; - typedef struct { - int key, beat, serial; - bit [511:0] line; - } reading; - typedef struct {int key, due, serial;} writing; - typedef struct packed { - bit [511:0] data; - bit [63:0] mask; - } wline; - command commands[int]; - reading reading_ids[int]; - writing writing_ids[int]; - int aw_ids[$], aw_keys[$], aw_serials[$]; - wline w_lines[$]; - wline assembling; - int assembling_beat = 0; - int read_error[u64], write_error[u64]; - bit hold_aw = 0, hold_w = 0, hold_ar = 0, hold_b = 0, hold_r = 0, hold_resp[2] = '{0, 0}; - bit r_active = 0, b_active = 0; - int r_id = 0, b_id = 0, last_r_id = -1, last_r_serial = -1, r_cursor = 0, r_beats = 0; - logic [`DDR_R_WIDTH-1:0] r_packet; - logic [`DDR_B_WIDTH-1:0] b_packet; - int - cycle = 0, - accepted = 0, - returned[2] = '{0, 0}, - cancelled = 0, - peak = 0, - aw_count = 0, - w_bursts = 0, - ar_count = 0; - int - request_stalls = 0, - response_stalls = 0, - aw_stalls = 0, - w_stalls = 0, - interleaves = 0, - error_results = 0; - int invalid_case = -1; - int b_reorders = 0, r_reorders = 0, max_b_serial = -1, max_r_serial = -1; - function new(string name, uvm_component parent); - super.new(name, parent); - timeout = 1ms; - endfunction - function void verify_contract(bit ok, string message); - if (!ok) `uvm_fatal("DDR_CONTRACT", message) - endfunction - function bit [511:0] pattern(int seed); - bit [511:0] data; - for (int i = 0; i < 8; i++) - data[i*64+:64]=64'h0123456789abcdef^(64'h0101010101010101*(seed+i))^(64'h1<<((seed+i*7)%64)); - return data; - endfunction - function void initialize(u64 address, int seed); - ddr_ref_program(model, address, pattern(seed)); - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - model = ddr_ref_create(); - scoreboard = keyed_scoreboard#(result_item)::type_id::create("scoreboard", this); - verify_contract(uvm_config_db#(virtual ddr_control_if)::get(this, "", "ctl", ctl), - "Missing control"); - for (int c = 0; c < 2; c++) begin - verify_contract(uvm_config_db#(virtual stream_if #(`DDR_REQ_WIDTH))::get( - this, "", $sformatf("req%0d", c), req[c]), "Missing request"); - verify_contract(uvm_config_db#(virtual stream_if #(`DDR_RESP_WIDTH))::get( - this, "", $sformatf("resp%0d", c), resp[c]), "Missing response"); - end - verify_contract(uvm_config_db#(virtual stream_if #(`DDR_A_WIDTH))::get(this, "", "aw", aw), - "Missing AW"); - verify_contract(uvm_config_db#(virtual stream_if #(`DDR_A_WIDTH))::get(this, "", "ar", ar), - "Missing AR"); - verify_contract(uvm_config_db#(virtual stream_if #(`DDR_W_WIDTH))::get(this, "", "w", w), - "Missing W"); - verify_contract(uvm_config_db#(virtual stream_if #(`DDR_B_WIDTH))::get(this, "", "b", b), - "Missing B"); - verify_contract(uvm_config_db#(virtual stream_if #(`DDR_R_WIDTH))::get(this, "", "r", r), - "Missing R"); - void'(uvm_config_db#(int)::get(this, "", "invalid_case", invalid_case)); - endfunction - function int address_owner(u64 address, bit write); - int found = -1; - foreach (commands[key]) - if (commands[key].packet[ - `DF(REQ, ADDR) - ] == address && commands[key].packet[ - `DF(REQ, WRITE) - ] == write) begin - verify_contract(found == -1, "Ambiguous fixture address"); - found = key; - end - verify_contract(found != -1, "AXI address without accepted line command"); - return found; - endfunction - function int observe_address(logic [`DDR_A_WIDTH-1:0] a, bit write); - int key, id; - verify_contract(!$isunknown(a), "Unknown AXI address payload"); - key = address_owner(a[`DF(A, ADDR)], write); - id = a[`DF(A, ID)]; - verify_contract(id >= `DDR_ID_BASE && id < `DDR_ID_BASE + `DDR_SLOTS, - "AXI ID outside configured namespace"); - verify_contract((id - `DDR_ID_BASE) / `DDR_SLOTS_PER_CLIENT == commands[key].client, - "AXI ID crossed client namespace"); - verify_contract(a[`DF(A, LEN)] == `DDR_BEATS - 1 && a[`DF(A, SIZE)] == $clog2( - (`DDR_BEAT_BYTES)) && a[`DF(A, BURST)] == 1, "Wrong AXI burst geometry"); - verify_contract(a[`DF(A, LOCK)] == 0 && a[`DF(A, CACHE)] == 0 && a[`DF(A, PROT)] == 0 && a[ - `DF(A, QOS)] == 0 && a[`DF(A, REGION)] == 0, "Unexpected AXI attributes"); - return key; - endfunction - task service(); - forever begin - @(ctl.sample); - if (!ctl.sample.reset) begin - cycle++; - for (int c = 0; c < 2; c++) begin - if (req[c].sample.valid && !req[c].sample.ready) request_stalls++; - if (resp[c].sample.valid && !resp[c].sample.ready) response_stalls++; - if (req[c].sample.valid && req[c].sample.ready) begin - result_item expected = new; - req_t q = req[c].sample.bits; - int key = (c << 12) | int'(q[`DF(REQ, ID)]); - bit [511:0] line; - verify_contract(!commands.exists(key), "Accepted duplicate live client tag"); - verify_contract(ddr_ref_read(model, q[`DF(REQ, ADDR)], line), - "Uninitialized fixture line"); - commands[key] = '{q, c, 0}; - expected.set_transaction_id(key); - expected.client = c; - expected.tag = q[`DF(REQ, ID)]; - expected.error = q[`DF(REQ, WRITE)] ? write_error.exists(q[`DF(REQ, ADDR)]) : - read_error.exists(q[`DF(REQ, ADDR)]); - expected.data = (q[`DF(REQ, WRITE)] || expected.error) ? '0 : line; - scoreboard.expected_export.write(expected); - accepted++; - if (commands.num() > peak) peak = commands.num(); - end - end - if (aw.sample.valid && !aw.sample.ready) aw_stalls++; - if (w.sample.valid && !w.sample.ready) w_stalls++; - if (aw.sample.valid && aw.sample.ready) begin - int key = observe_address(aw.sample.bits, 1); - int id = aw.sample.bits[`DF(A, ID)]; - aw_ids.push_back(id); - aw_keys.push_back(key); - aw_serials.push_back(aw_count); - aw_count++; - end - if (w.sample.valid && w.sample.ready) begin - verify_contract(!$isunknown(w.sample.bits), "Unknown W payload"); - verify_contract(w.sample.bits[`DF(W, LAST)] == (assembling_beat == `DDR_BEATS - 1), - "Wrong WLAST"); - assembling.data[assembling_beat*`DDR_DATA_BITS+:`DDR_DATA_BITS] = w.sample.bits[ - `DF(W, DATA) - ]; - assembling.mask[assembling_beat*(`DDR_BEAT_BYTES)+:(`DDR_BEAT_BYTES)] = w.sample.bits[ - `DF(W, STRB) - ]; - if (assembling_beat == `DDR_BEATS - 1) begin - w_lines.push_back(assembling); - assembling = '0; - assembling_beat = 0; - w_bursts++; - end else assembling_beat++; - end - if (aw_ids.size() && w_lines.size()) begin - int id = aw_ids.pop_front(); - int key = aw_keys.pop_front(); - int serial = aw_serials.pop_front(); - wline line = w_lines.pop_front(); - req_t q = commands[key].packet; - verify_contract(!writing_ids.exists(id) && !reading_ids.exists(id), - "AXI ID reused while active"); - verify_contract(line.data === q[`DF(REQ, DATA)] && line.mask === q[`DF(REQ, MASK)], - "AW/W order, payload or byte mask mismatch"); - verify_contract(ddr_ref_write(model, q[`DF(REQ, ADDR)], line.data, line.mask), - "Write outside initialized DDR"); - writing_ids[id] = '{key, cycle + 7, serial}; - end - if (ar.sample.valid && ar.sample.ready) begin - int key = observe_address(ar.sample.bits, 0); - int id = ar.sample.bits[`DF(A, ID)]; - bit [511:0] line; - verify_contract(!reading_ids.exists(id) && !writing_ids.exists(id), - "AXI ID reused while active"); - verify_contract(ddr_ref_read(model, ar.sample.bits[`DF(A, ADDR)], line), - "Read outside initialized DDR"); - reading_ids[id] = '{key, 0, ar_count, line}; - ar_count++; - end - if (r.sample.valid && r.sample.ready) begin - int key = reading_ids[r_id].key; - r_beats++; - if (last_r_id >= 0 && last_r_id != r_id && reading_ids.exists( - last_r_id - ) && reading_ids[last_r_id].serial == last_r_serial) - interleaves++; - last_r_id = r_id; - last_r_serial = reading_ids[r_id].serial; - if (reading_ids[r_id].beat == `DDR_BEATS - 1) begin - if (reading_ids[r_id].serial < max_r_serial) r_reorders++; - else max_r_serial = reading_ids[r_id].serial; - commands[key].done = 1; - reading_ids.delete(r_id); - end else reading_ids[r_id].beat++; - r_active = 0; - r_cursor = (r_id + 1) % 16; - end - if (b.sample.valid && b.sample.ready) begin - if (writing_ids[b_id].serial < max_b_serial) b_reorders++; - else max_b_serial = writing_ids[b_id].serial; - commands[writing_ids[b_id].key].done = 1; - writing_ids.delete(b_id); - b_active = 0; - end - for (int c = 0; c < 2; c++) - if (resp[c].sample.valid && resp[c].sample.ready) begin - result_item actual = new; - int key = (c << 12) | int'(resp[c].sample.bits[`DF(RESP, ID)]); - verify_contract(!$isunknown(resp[c].sample.bits) && commands.exists(key - ) && commands[key].done, - "Early/unknown line response before final R or B"); - actual.set_transaction_id(key); - actual.client = c; - actual.tag = resp[c].sample.bits[`DF(RESP, ID)]; - actual.data = resp[c].sample.bits[`DF(RESP, DATA)]; - actual.error = resp[c].sample.bits[`DF(RESP, ERROR)]; - scoreboard.actual_export.write(actual); - if (actual.error) error_results++; - returned[c]++; - commands.delete(key); - end - end - @(negedge ctl.clock); - aw.ready = !ctl.reset && !hold_aw && cycle % 5 != 0; - w.ready = !ctl.reset && !hold_w && cycle % 7 >= 2; - ar.ready = !ctl.reset && !hold_ar && cycle % 4 != 0; - for (int c = 0; c < 2; c++) resp[c].ready = !ctl.reset && !hold_resp[c] && cycle % 5 != c; - if (!r_active && !hold_r) - for (int off = 0; off < 16; off++) begin - int id = (r_cursor + off) % 16; - if (!r_active && reading_ids.exists(id)) begin - int key = reading_ids[id].key; - u64 address = commands[key].packet[`DF(REQ, ADDR)]; - int beat = reading_ids[id].beat; - r_active = 1; - r_id = id; - r_packet = '0; - r_packet[`DF(R, ID)] = id; - r_packet[`DF(R, DATA)] = reading_ids[id].line[beat*`DDR_DATA_BITS+:`DDR_DATA_BITS]; - r_packet[`DF(R, LAST)] = beat == `DDR_BEATS - 1; - if (read_error.exists(address) && beat == read_error[address]) - r_packet[`DF(R, RESP)] = beat % 2 ? 3 : 2; - end - end - if (!b_active && !hold_b) - for (int id = 15; id >= 0; id--) - if (!b_active && writing_ids.exists(id) && writing_ids[id].due <= cycle) begin - b_active = 1; - b_id = id; - b_packet = '0; - b_packet[`DF(B, ID)] = id; - if (write_error.exists(commands[writing_ids[id].key].packet[`DF(REQ, ADDR)])) - b_packet[ - `DF(B, RESP) - ] = write_error[commands[writing_ids[id].key].packet[ - `DF(REQ, ADDR) - ]]; - end - r.valid = !ctl.reset && r_active; - r.bits = r_packet; - b.valid = !ctl.reset && b_active; - b.bits = b_packet; - end - endtask - task submit(int client, int id, u64 address, bit write, bit [63:0] mask = '1); - req_t q = '0; - q[`DF(REQ, ID)] = id; - q[`DF(REQ, ADDR)] = address; - q[`DF(REQ, WRITE)] = write; - q[`DF(REQ, DATA)] = pattern(id + client * 17 + int'(address >> 6)); - q[`DF(REQ, MASK)] = mask; - @(negedge ctl.clock); - req[client].bits = q; - req[client].valid = 1; - do @(req[client].sample); while (!req[client].sample.ready); - @(negedge ctl.clock); - req[client].valid = 0; - endtask - task batch(u64 base, bit write); - for (int c = 0; c < 2; c++) - for (int i = 0; i < `DDR_SLOTS_PER_CLIENT; i++) - initialize(base + c * 4096 + i * 64, i + c * 7 + int'(base >> 6)); - fork - begin - for (int i = 0; i < `DDR_SLOTS_PER_CLIENT; i++) - submit(0, 12'h800 + i, base + i * 64, write, - i == 0 ? 0 : i == 1 ? '1 : i == 2 ? 64'h5555555555555555 : 64'haaaaaaaaaaaaaaaa); - end - begin - for (int i = 0; i < `DDR_SLOTS_PER_CLIENT; i++) - submit(1, 12'h800 + i, base + 4096 + i * 64, write); - end - join - endtask - task drain(); - wait (commands.num() == 0); - @(ctl.sample); - verify_contract( - !reading_ids.num()&&!writing_ids.num()&&!aw_ids.size()&&!w_lines.size()&&!assembling_beat&&!r_active&&!b_active, - "Backend did not drain"); - endtask - task joint_reset(); - @(negedge ctl.clock); - ctl.reset = 1; - req[0].valid = 0; - req[1].valid = 0; - repeat (3) @(ctl.sample); - cancelled += commands.num(); - commands.delete(); - reading_ids.delete(); - writing_ids.delete(); - aw_ids.delete(); - aw_keys.delete(); - aw_serials.delete(); - w_lines.delete(); - scoreboard.expected.delete(); - scoreboard.actual.delete(); - assembling = '0; - assembling_beat = 0; - r_active = 0; - b_active = 0; - @(negedge ctl.clock); - ctl.reset = 0; - endtask - function int finished_commands(); - int count = 0; - foreach (commands[key]) if (commands[key].done) count++; - return count; - endfunction - task invalid_scenario(); - int id, beats; - bit aw_seen; - if (invalid_case == 0) begin - submit(0, 0, 'h10001, 0); - end else if (invalid_case == 1) begin - @(negedge ctl.clock); - r.bits = '0; - r.bits[`DF(R, ID)] = 0; - r.valid = 1; - end else if (invalid_case == 2 || invalid_case == 6) begin - aw.ready = 1; - w.ready = invalid_case == 6; - submit(0, 0, 'h10000, 1); - beats = 0; - aw_seen = 0; - do begin - @(ctl.sample); - if (aw.sample.valid && aw.sample.ready) begin - id = aw.sample.bits[`DF(A, ID)]; - aw_seen = 1; - end - if (w.sample.valid && w.sample.ready) beats++; - end while (!aw_seen || (invalid_case == 6 && beats != `DDR_BEATS)); - @(negedge ctl.clock); - b.bits = '0; - b.bits[`DF(B, ID)] = id; - b.bits[`DF(B, RESP)] = invalid_case == 6 ? 1 : 0; - b.valid = 1; - end else begin - ar.ready = 1; - submit(0, 0, 'h10000, 0); - do @(ctl.sample); while (!ar.sample.valid || !ar.sample.ready); - id = ar.sample.bits[`DF(A, ID)]; - for (int beat = 0; beat < (invalid_case == 4 ? `DDR_BEATS : 1); beat++) begin - @(negedge ctl.clock); - r.bits = '0; - r.bits[`DF(R, ID)] = id; - r.bits[`DF(R, LAST)] = invalid_case == 3; - r.bits[`DF(R, RESP)] = invalid_case == 5 ? 1 : 0; - r.valid = 1; - do @(r.sample); while (!r.sample.ready); - end - end - repeat (8) @(ctl.sample); - `uvm_fatal("MISSING_ASSERTION", "Invalid protocol did not trigger expected DUT assertion") - endtask - task execute(); - int before_aw, before_w, before_ar, before_r; - int initial_returned; - ctl.reset = 1; - req[0].valid = 0; - req[1].valid = 0; - req[0].bits = '0; - req[1].bits = '0; - resp[0].ready = 0; - resp[1].ready = 0; - aw.ready = 0; - ar.ready = 0; - w.ready = 0; - b.valid = 0; - b.bits = '0; - r.valid = 0; - r.bits = '0; - repeat (5) @(negedge ctl.clock); - ctl.reset = 0; - if (invalid_case >= 0) begin - invalid_scenario(); - return; - end - fork - service(); - join_none - // W must progress even when AW has never been accepted; all eight slots stay live until B. - hold_aw = 1; - hold_b = 1; - hold_resp[0] = 1; - batch('h10000, 1); - wait (w_bursts == `DDR_SLOTS); - verify_contract(aw_count == 0 && returned[0] == 0 && returned[1] == 0, - "W-before-AW or B-delayed behavior failed"); - hold_aw = 0; - wait (aw_count == `DDR_SLOTS); - initialize('h18000, 99); - fork - submit(0, 12'h800, 'h18000, 1); - begin - hold_b = 0; - wait (returned[1] == `DDR_SLOTS_PER_CLIENT); - verify_contract(returned[0] == 0, "Client0 backpressure leaked"); - hold_resp[0] = 0; - end - join - drain(); - for (int c = 0; c < 2; c++) - for (int i = 0; i < `DDR_SLOTS_PER_CLIENT; i++) - submit(c, 12'h900 + i, 'h10000 + c * 4096 + i * 64, 0); - drain(); - // Conversely allow addresses to run ahead of every W beat. - before_aw = aw_count; - before_w = w_bursts; - hold_w = 1; - batch('h20000, 1); - wait (aw_count == before_aw + `DDR_SLOTS); - verify_contract(w_bursts == before_w, "AW-before-W stalled ordering failed"); - hold_w = 0; - drain(); - // Concurrent reads and writes: hold B until all independent reads return. - hold_b = 1; - initial_returned = returned[0] + returned[1]; - for (int c = 0; c < 2; c++) - for (int i = 0; i < `DDR_SLOTS_PER_CLIENT; i++) - initialize('h30000 + c * 4096 + i * 64, 30 + i); - fork - begin - for (int i = 0; i < `DDR_SLOTS_PER_CLIENT; i++) - submit(0, 12'ha00 + i, 'h30000 + i * 64, i[0]); - end - begin - for (int i = 0; i < `DDR_SLOTS_PER_CLIENT; i++) - submit(1, 12'ha00 + i, 'h31000 + i * 64, i[0]); - end - join - wait (returned[0] + returned[1] == initial_returned + 2 * ((`DDR_SLOTS_PER_CLIENT + 1) / 2)); - hold_b = 0; - drain(); - for (int i = 0; i < 4; i++) begin - initialize('h40000 + i * 64, 40 + i); - read_error['h40000+i*64] = i % `DDR_BEATS; - submit(i % 2, 12'hb00 + i, 'h40000 + i * 64, 0); - end - initialize('h41000, 44); - write_error['h41000] = 2; - submit(1, 12'hb10, 'h41000, 1); - initialize('h41040, 45); - write_error['h41040] = 3; - submit(0, 12'hb10, 'h41040, 1); - drain(); - before_ar = ar_count; - hold_r = 1; - batch('h48000, 0); - wait (ar_count == before_ar + `DDR_SLOTS); - r_cursor = `DDR_ID_BASE; - hold_r = 0; - drain(); - // Each byte strobe is independently checked, then read back through the reference. - for (int i = 0; i < 64; i++) begin - initialize('h50000 + i * 64, 50 + i); - submit(i % 2, 12'hc00 + i, 'h50000 + i * 64, 1, 64'h1 << i); - drain(); - submit(i % 2, 12'hd00 + i, 'h50000 + i * 64, 0); - drain(); - end - // Coordinated reset cancels queued AW-only, W-only and stalled line completions. - before_aw = aw_count; - hold_w = 1; - batch('h60000, 1); - wait (aw_count == before_aw + `DDR_SLOTS); - joint_reset(); - hold_w = 0; - before_w = w_bursts; - hold_aw = 1; - batch('h70000, 1); - wait (w_bursts == before_w + `DDR_SLOTS); - joint_reset(); - hold_aw = 0; - before_ar = ar_count; - before_r = r_beats; - hold_r = 1; - batch('h78000, 0); - wait (ar_count == before_ar + `DDR_SLOTS); - hold_r = 0; - wait (r_beats >= before_r + 1); - joint_reset(); - hold_resp[0] = 1; - hold_resp[1] = 1; - batch('h80000, 0); - do @(ctl.sample); while (finished_commands() != `DDR_SLOTS); - joint_reset(); - hold_resp[0] = 0; - hold_resp[1] = 0; - batch('h90000, 0); - drain(); - verify_contract( - peak==`DDR_SLOTS&&(`DDR_BEATS==1?interleaves==0:interleaves>0)&&b_reorders>0&&r_reorders>0&&request_stalls>0&&response_stalls>0&&aw_stalls>0&&w_stalls>0&&error_results==6&&cancelled==4*`DDR_SLOTS, - "Missing concurrency, stalls, errors or reset coverage"); - verify_contract(accepted == returned[0] + returned[1] + cancelled, "Lost line response"); - `uvm_info( - "DDR_PASS", - $sformatf( - "accepted=%0d returned=%0d/%0d cancelled=%0d peak=%0d errors=%0d readInterleave=%0d R/Breorder=%0d/%0d stalls req/resp/AW/W=%0d/%0d/%0d/%0d; full64B payload/mask/tag and all queues drained", - accepted, returned[0], returned[1], cancelled, peak, error_results, interleaves, - r_reorders, b_reorders, request_stalls, response_stalls, aw_stalls, w_stalls), - UVM_LOW) - endtask - function void final_phase(uvm_phase phase); - ddr_ref_destroy(model); - super.final_phase(phase); - endfunction - endclass -endpackage diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/profile.svh b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/profile.svh deleted file mode 100644 index 6e72f7a55..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/profile.svh +++ /dev/null @@ -1,20 +0,0 @@ -`ifndef DDR_PROFILE_SVH -`define DDR_PROFILE_SVH -`ifdef DDR_PROFILE64 -`define DDR_CONFIG "ddr64_config.svh" -`define DDR_PORTS "ddr64_ports.svh" -`elsif DDR_PROFILE256 -`define DDR_CONFIG "ddr256_config.svh" -`define DDR_PORTS "ddr256_ports.svh" -`elsif DDR_PROFILE512 -`define DDR_CONFIG "ddr512_config.svh" -`define DDR_PORTS "ddr512_ports.svh" -`elsif DDR_PROFILESIX -`define DDR_CONFIG "ddr_six_config.svh" -`define DDR_PORTS "ddr_six_ports.svh" -`else -`define DDR_CONFIG "ddr_config.svh" -`define DDR_PORTS "ddr_ports.svh" -`endif -`include `DDR_CONFIG -`endif diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/scala/Emit.scala b/arch/src/main/scala/framework/mem-core/ddr/src/main/scala/Emit.scala deleted file mode 100644 index a30caeda8..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/scala/Emit.scala +++ /dev/null @@ -1,68 +0,0 @@ -package memcore.memory.ddr - -import chisel3._ -import java.nio.file.{Files, Paths} -import memcore.bus.axi4 -import memcore.bus.chi.snf.{LineRequest, LineResponse} - -object Emit extends App { - - def emit(p: Params, name: String, header: String): Unit = { - Files.createDirectories(Paths.get("build")) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Bridge(p), - firtoolOpts = args, - args = Array("--target-dir", s"build/$name", "--split-verilog") - ) - val defs = scala.collection.mutable.ArrayBuffer[String]( - s"`define DDR_DATA_BITS ${p.dataBits}", - s"`define DDR_BEAT_BYTES ${p.axi.bytes}", - s"`define DDR_BEATS ${p.beats}", - s"`define DDR_SLOTS ${p.slots}", - s"`define DDR_SLOTS_PER_CLIENT ${p.slotsPerClient}", - s"`define DDR_ID_BASE ${p.idBase}" - ) - val ports = scala.collection.mutable.ArrayBuffer[String]() - def channel( - kind: String, - prefix: String, - signal: String, - bundle: Bundle, - define: Boolean = true - ): Unit = { - var offset = 0 - for ((name, field) <- bundle.elements.toSeq) { - val width = field.getWidth - if (define) { - defs += s"`define DDR_${kind}_${name.toUpperCase}_OFFSET $offset" - defs += s"`define DDR_${kind}_${name.toUpperCase}_WIDTH $width" - } - ports += s".${prefix}_bits_$name($signal.bits[$offset +: $width])" - offset += width - } - if (define) defs += s"`define DDR_${kind}_WIDTH $offset" - ports += s".${prefix}_valid($signal.valid)" - ports += s".${prefix}_ready($signal.ready)" - } - for (c <- 0 until p.clients) { - channel("REQ", s"io_request_$c", s"req$c", new LineRequest(p.line), c == 0) - channel("RESP", s"io_response_$c", s"resp$c", new LineResponse(p.line), c == 0) - } - channel("A", "io_axi_aw", "aw", new axi4.Address(p.axi)) - channel("A", "io_axi_ar", "ar", new axi4.Address(p.axi), false) - channel("W", "io_axi_w", "w", new axi4.WriteData(p.axi)) - channel("B", "io_axi_b", "b", new axi4.WriteResponse(p.axi)) - channel("R", "io_axi_r", "r", new axi4.ReadData(p.axi)) - Files.writeString( - Paths.get(s"build/${header}_config.svh"), - "`ifndef DDR_CONFIG_SVH\n`define DDR_CONFIG_SVH\n" + defs.mkString("\n") + "\n`endif\n" - ) - Files.writeString(Paths.get(s"build/${header}_ports.svh"), ports.mkString(",\n") + "\n") - } - - emit(Params(idBase = 4), "Bridge", "ddr") - emit(Params(dataBits = 64, idBase = 4), "Bridge64", "ddr64") - emit(Params(dataBits = 256, idBase = 4), "Bridge256", "ddr256") - emit(Params(dataBits = 512, idBase = 4), "Bridge512", "ddr512") - emit(Params(slotsPerClient = 3, idBase = 4), "BridgeSix", "ddr_six") -} diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/scala/Params.scala b/arch/src/main/scala/framework/mem-core/ddr/src/main/scala/Params.scala deleted file mode 100644 index 2aa1eed33..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/scala/Params.scala +++ /dev/null @@ -1,20 +0,0 @@ -package memcore.memory.ddr - -import memcore.bus.chi -import memcore.bus.axi4 - -case class Params( - line: chi.Params = chi.Params(), - clients: Int = 2, - slotsPerClient: Int = 4, - dataBits: Int = 128, - idBits: Int = 4, - idBase: Int = 0) { - require(line.addressBits >= 6) - require(clients > 0 && slotsPerClient > 0) - require(Set(64, 128, 256, 512).contains(dataBits)) - val slots = clients * slotsPerClient - require(slots > 1 && idBase >= 0 && BigInt(idBase) + slots <= (BigInt(1) << idBits)) - val axi = axi4.Params(line.addressBits, dataBits, idBits) - val beats = 512 / dataBits -} diff --git a/arch/src/main/scala/framework/mem-core/fetch/build.sc b/arch/src/main/scala/framework/mem-core/fetch/build.sc deleted file mode 100644 index bccef01f1..000000000 --- a/arch/src/main/scala/framework/mem-core/fetch/build.sc +++ /dev/null @@ -1,10 +0,0 @@ -import mill._ -import mill.scalalib._ -object fetch extends SbtModule { - override def millSourcePath = os.pwd - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") - override def mainClass = Some("memcore.memory.fetch.Emit") -} diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch.el b/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch.el deleted file mode 100644 index b72d899ed..000000000 --- a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch.el +++ /dev/null @@ -1,50 +0,0 @@ -// Format Version: 2 -// 4-byte groups, 16-bit halfword/RVC alignment, one 64-bit word request. -CHECKSUM: "1717534490 3493169012" -INSTANCE: fetch_tb.dut -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Fetch privilege must be U, S or M" -Block 2 "3303546747" "if (1)" -Block 3 "784000845" "$error(\"Assertion failed: Fetch privilege must be U, S or M\n at Fetch.scala:73 assert(io.context.privilege =/= 2.U, \\"Fetch privilege must be U, S or M\\")\n\");" -Block 5 "3303546747" "if (1)" -Block 6 "2616558672" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Fetch reset vector must be instruction aligned" -Block 10 "1637879550" "if (1)" -Block 11 "1142822639" "$error(\"Assertion failed: Fetch reset vector must be instruction aligned\n at Fetch.scala:74 assert(pc(log2Ceil(p.instructionBytes) - 1, 0) === 0.U, \\"Fetch reset vector must be instruction aligned\\")\n\");" -Block 13 "1637879550" "if (1)" -Block 14 "2917930836" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Fetch maintenance response must acknowledge completion" -Block 18 "2033796801" "if (1)" -Block 19 "2717912749" "$error(\"Assertion failed: Fetch maintenance response must acknowledge completion\n at Fetch.scala:105 assert(io.maintained.bits, \\"Fetch maintenance response must acknowledge completion\\")\n\");" -Block 21 "2033796801" "if (1)" -Block 22 "2015233156" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Fetch redirect must be instruction aligned" -Block 26 "2247434341" "if (1)" -Block 27 "1751189408" "$error(\"Assertion failed: Fetch redirect must be instruction aligned\n at Fetch.scala:109 assert(io.redirect.bits(log2Ceil(p.instructionBytes) - 1, 0) === 0.U, \\"Fetch redirect must be instruction aligned\\")\n\");" -Block 29 "2247434341" "if (1)" -Block 30 "1248359523" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Fetch flush requires a restart redirect" -Block 34 "1762317919" "if (1)" -Block 35 "4112421248" "$error(\"Assertion failed: Fetch flush requires a restart redirect\n at Fetch.scala:115 assert(io.redirect.valid, \\"Fetch flush requires a restart redirect\\")\n\");" -Block 37 "1762317919" "if (1)" -Block 38 "564188382" "$fatal;" - -CHECKSUM: "1717534490 1587966699" -INSTANCE: fetch_tb.dut -ANNOTATION: "Only full enabled contract-failure vectors: privilege 2, odd start/redirect PC, incomplete maintenance, or flush without restart. Each is independently tested by an exact negative target; all idle/reset/pure comparator vectors remain measured." -Condition 2 "1830893280" "(_GEN_1 & (io_context_privilege == 2'h2)) 1 -1" (3 "11") -Condition 4 "2086071732" "(_GEN_1 & pc[0]) 1 -1" (3 "11") -Condition 5 "1305965348" "(_GEN_0 & ((~reset)) & ((~io_maintained_bits))) 1 -1" (4 "111") -Condition 6 "1743659560" "(io_redirect_valid & ((~reset)) & io_redirect_bits[0]) 1 -1" (4 "111") -Condition 7 "2120892050" "(io_flush & ((~reset)) & ((~io_redirect_valid))) 1 -1" (4 "111") - -CHECKSUM: "1717534490 202402385" -INSTANCE: fetch_tb.dut -ANNOTATION: "Word addresses are assigned pc & ~7; execute and maintenance request bits are literal true. Every deliverable packet comes from an accepted halfword-aligned fetch, so PC bit 0 is zero; its full 4-byte group mask is 11 or 10, so mask bit 1 is one. Reset-only PC/readPc/input values remain measured, and no instruction data is waived." -Toggle io_packet_bits_pc [0] "net io_packet_bits_pc[63:0]" -Toggle io_packet_bits_mask [1] "net io_packet_bits_mask[1:0]" -Toggle io_request_bits_addr [2:0] "net io_request_bits_addr[63:0]" -Toggle io_request_bits_execute "net io_request_bits_execute" -Toggle io_maintenance_bits "net io_maintenance_bits" -Toggle command_addr [2:0] "reg command_addr[63:0]" -Toggle answer_pc [0] "reg answer_pc[63:0]" -Toggle answer_mask [1] "reg answer_mask[1:0]" diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch.f b/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch.f deleted file mode 100644 index 2102a5b94..000000000 --- a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/interface.sv -@RESOURCES@/package.sv --F @RTL@/Fetch/filelist.f -@RESOURCES@/fetch_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_ack.f b/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_ack.f deleted file mode 100644 index 49acb9a66..000000000 --- a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_ack.f +++ /dev/null @@ -1,11 +0,0 @@ -+define+FETCH_TOP=fetch_bad_ack_tb -+define+FETCH_BAD_ACK -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/interface.sv -@RESOURCES@/package.sv --F @RTL@/Fetch/filelist.f -@RESOURCES@/fetch_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_ack.toml b/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_ack.toml deleted file mode 100644 index b4785e84f..000000000 --- a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_ack.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Fetch maintenance response must acknowledge completion" diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_flush.f b/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_flush.f deleted file mode 100644 index e80751594..000000000 --- a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_flush.f +++ /dev/null @@ -1,11 +0,0 @@ -+define+FETCH_TOP=fetch_bad_flush_tb -+define+FETCH_BAD_FLUSH -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/interface.sv -@RESOURCES@/package.sv --F @RTL@/Fetch/filelist.f -@RESOURCES@/fetch_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_flush.toml b/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_flush.toml deleted file mode 100644 index d829076ff..000000000 --- a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_flush.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Fetch flush requires a restart redirect" diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_privilege.f b/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_privilege.f deleted file mode 100644 index 036ddd1f2..000000000 --- a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_privilege.f +++ /dev/null @@ -1,11 +0,0 @@ -+define+FETCH_TOP=fetch_bad_privilege_tb -+define+FETCH_BAD_PRIVILEGE -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/interface.sv -@RESOURCES@/package.sv --F @RTL@/Fetch/filelist.f -@RESOURCES@/fetch_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_privilege.toml b/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_privilege.toml deleted file mode 100644 index b6e96cd3f..000000000 --- a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_privilege.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Fetch privilege must be U, S or M" diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_redirect.f b/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_redirect.f deleted file mode 100644 index 80ea2e2f3..000000000 --- a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_redirect.f +++ /dev/null @@ -1,11 +0,0 @@ -+define+FETCH_TOP=fetch_bad_redirect_tb -+define+FETCH_BAD_REDIRECT -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/interface.sv -@RESOURCES@/package.sv --F @RTL@/Fetch/filelist.f -@RESOURCES@/fetch_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_redirect.toml b/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_redirect.toml deleted file mode 100644 index 1f743d176..000000000 --- a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_redirect.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Fetch redirect must be instruction aligned" diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_reset.f b/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_reset.f deleted file mode 100644 index 598dcc455..000000000 --- a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_reset.f +++ /dev/null @@ -1,11 +0,0 @@ -+define+FETCH_TOP=fetch_bad_reset_tb -+define+FETCH_BAD_RESET -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/interface.sv -@RESOURCES@/package.sv --F @RTL@/Fetch/filelist.f -@RESOURCES@/fetch_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_reset.toml b/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_reset.toml deleted file mode 100644 index b6a254790..000000000 --- a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_reset.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Fetch reset vector must be instruction aligned" diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_tb.sv b/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_tb.sv deleted file mode 100644 index 6ac96319b..000000000 --- a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_tb.sv +++ /dev/null @@ -1,87 +0,0 @@ -`include "fetch_config.svh" -`ifndef FETCH_TOP -`define FETCH_TOP fetch_tb -`endif -module `FETCH_TOP; - import uvm_pkg::*; - import fetch_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - fetch_control_if control (clock); - stream_if #(`FETCH_REQUEST_WIDTH) request ( - clock, - control.reset - ); - stream_if #(`FETCH_RESPONSE_WIDTH) response ( - clock, - control.reset - ); - // CPU redirection explicitly cancels a held frontend packet; it never cancels the word port. - stream_if #(`FETCH_PACKET_WIDTH) sink_if ( - clock, - control.reset || control.redirect_valid || control.flush - ); - stream_if #(1) maintenance ( - clock, - control.reset - ); - stream_if #(1) maintained ( - clock, - control.reset - ); - bit read_pending = 0, maintenance_pending = 0; - always @(posedge clock) begin - if (control.reset) begin - read_pending = 0; - maintenance_pending = 0; - end else begin - if (request.valid && request.ready) begin - if (read_pending || maintenance_pending) $fatal(1, "Fetch overlapped backend transactions"); - read_pending = 1; - end - if (response.valid && response.ready) begin - if (!read_pending) $fatal(1, "Fetch accepted an unsolicited word response"); - read_pending = 0; - end - if (maintenance.valid && maintenance.ready) begin - if (read_pending || maintenance_pending) - $fatal(1, "Maintenance began before draining the word"); - maintenance_pending = 1; - end - if (maintained.valid && maintained.ready) begin - if (!maintenance_pending) - $fatal(1, "Fetch accepted an unsolicited maintenance acknowledgement"); - maintenance_pending = 0; - end - if ((control.redirect_valid || control.flush) && sink_if.valid) - $fatal(1, "Cancelled packet was visible to the CPU"); - end - end - Fetch dut ( - .clock(clock), - .reset(control.reset), - .io_resetVector(control.reset_vector), - .io_context_privilege(control.privilege), - .io_context_satp(control.satp), - .io_context_sum(control.sum), - .io_context_mxr(control.mxr), - .io_redirect_valid(control.redirect_valid), - .io_redirect_bits(control.redirect_pc), - .io_flush(control.flush), - .io_invalidate(1'b0), - .io_npc(control.npc), - `include "fetch_ports.svh" - ); - initial begin - uvm_config_db#(virtual fetch_control_if)::set(null, "uvm_test_top*", "control", control); - uvm_config_db#(virtual stream_if #(`FETCH_REQUEST_WIDTH))::set(null, "uvm_test_top*", "request", - request); - uvm_config_db#(virtual stream_if #(`FETCH_RESPONSE_WIDTH))::set(null, "uvm_test_top*", - "response", response); - uvm_config_db#(virtual stream_if #(`FETCH_PACKET_WIDTH))::set(null, "uvm_test_top*", "packet", - sink_if); - uvm_config_db#(virtual stream_if #(1))::set(null, "uvm_test_top*", "maintenance", maintenance); - uvm_config_db#(virtual stream_if #(1))::set(null, "uvm_test_top*", "maintained", maintained); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/interface.sv b/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/interface.sv deleted file mode 100644 index 459a4b770..000000000 --- a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/interface.sv +++ /dev/null @@ -1,14 +0,0 @@ -interface fetch_control_if ( - input logic clock -); - logic reset, redirect_valid, flush; - logic [63:0] reset_vector, redirect_pc, satp, npc; - logic [1:0] privilege; - logic sum, mxr; - clocking cb @(posedge clock); - default input #1step output #0; - input npc; - output reset, redirect_valid, redirect_pc, flush, privilege, satp, mxr; - output context_sum = sum; - endclocking -endinterface diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/package.sv b/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/package.sv deleted file mode 100644 index 52ad5b29b..000000000 --- a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/package.sv +++ /dev/null @@ -1,20 +0,0 @@ -`include "fetch_config.svh" -package fetch_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - `define FF(kind, field) `FETCH_``kind``_``field``_OFFSET +: `FETCH_``kind``_``field``_WIDTH - import "DPI-C" function longint unsigned fetch_ref_word( - input longint unsigned address, - input int unsigned version - ); - import "DPI-C" function int unsigned fetch_ref_group( - input longint unsigned address, - input int unsigned version - ); - import "DPI-C" function int unsigned fetch_ref_instruction( - input longint unsigned address, - input int unsigned version - ); - `include "test.svh" -endpackage diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/test.svh b/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/test.svh deleted file mode 100644 index 5bce60020..000000000 --- a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/test.svh +++ /dev/null @@ -1,410 +0,0 @@ -class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual fetch_control_if control; - virtual stream_if #(`FETCH_REQUEST_WIDTH) request; - virtual stream_if #(`FETCH_RESPONSE_WIDTH) response; - virtual stream_if #(`FETCH_PACKET_WIDTH) packet; - virtual stream_if #(1) maintenance, maintained; - int checked = 0, instructions = 0, reads = 0, drains = 0, barriers = 0, cancellations = 0; - longint unsigned cursor; - bit pending_half = 0; - bit [15:0] first_half; - longint unsigned partial_pc; - bit [63:0] saved_word; - bit [`FETCH_REQUEST_WIDTH-1:0] saved_request; - int version = 0; - function new(string name, uvm_component parent); - super.new(name, parent); - timeout = 200us; - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual fetch_control_if)::get( - this, "", "control", control - ) || !uvm_config_db#(virtual stream_if #(`FETCH_REQUEST_WIDTH))::get( - this, "", "request", request - ) || !uvm_config_db#(virtual stream_if #(`FETCH_RESPONSE_WIDTH))::get( - this, "", "response", response - ) || !uvm_config_db#(virtual stream_if #(`FETCH_PACKET_WIDTH))::get( - this, "", "packet", packet - ) || !uvm_config_db#(virtual stream_if #(1))::get( - this, "", "maintenance", maintenance - ) || !uvm_config_db#(virtual stream_if #(1))::get( - this, "", "maintained", maintained - )) - `uvm_fatal("VIF", "Fetch interfaces missing") - endfunction - task redirect(longint unsigned target, bit flush = 0); - control.cb.redirect_pc <= target; - control.cb.redirect_valid <= 1; - control.cb.flush <= flush; - @(control.cb); - control.cb.redirect_valid <= 0; - control.cb.flush <= 0; - cursor = target; - pending_half = 0; - endtask - task offered(longint unsigned expected_pc); - do @(request.sample); while (!request.sample.valid); - saved_request = request.sample.bits; - if (saved_request[ - `FF(REQUEST, ADDR) - ] !== (expected_pc & ~64'h7) || saved_request[ - `FF(REQUEST, CONTEXT_PRIVILEGE) - ] !== control.privilege || saved_request[ - `FF(REQUEST, CONTEXT_SATP) - ] !== control.satp || saved_request[ - `FF(REQUEST, CONTEXT_SUM) - ] !== control.sum || saved_request[ - `FF(REQUEST, CONTEXT_MXR) - ] !== control.mxr || saved_request[ - `FF(REQUEST, EXECUTE) - ] !== 1) - `uvm_fatal("READ", "Word request changed PC/context/execute metadata") - saved_word = fetch_ref_word(expected_pc & ~64'h7, version); - endtask - task accept_word(); - request.ready <= 1; - @(request.sample); - request.ready <= 0; - reads++; - endtask - task reply(bit pf = 0, bit af = 0); - bit [511:0] line; - response.bits <= '0; - response.bits[`FF(RESPONSE, DATA)] <= saved_word; - for (int word = 0; word < 8; word++) - line[word*64+:64] = - fetch_ref_word((saved_request[`FF(REQUEST, ADDR)] & ~64'h3f) + word * 8, version); - response.bits[`FF(RESPONSE, LINE)] <= line; - response.bits[`FF(RESPONSE, LINEEXECUTABLE)] <= 0; - response.bits[`FF(RESPONSE, PAGEFAULT)] <= pf; - response.bits[`FF(RESPONSE, ACCESSFAULT)] <= af; - response.valid <= 1; - do @(response.sample); while (!response.sample.ready); - response.valid <= 0; - endtask - task check_packet(bit pf = 0, bit af = 0); - bit [31:0] data; - bit [ 1:0] mask; - bit [63:0] pc; - do @(packet.sample); while (!packet.sample.valid); - pc = packet.sample.bits[`FF(PACKET, PC)]; - data = packet.sample.bits[`FF(PACKET, DATA)]; - mask = packet.sample.bits[`FF(PACKET, MASK)]; - if (pc !== cursor || data !== (pf || af ? 0 : fetch_ref_group( - cursor, version - )) || mask !== (cursor[1] ? 2'b10 : 2'b11) || packet.sample.bits[ - `FF(PACKET, PAGEFAULT) - ] !== pf || packet.sample.bits[ - `FF(PACKET, ACCESSFAULT) - ] !== af) - `uvm_fatal("PACKET", $sformatf( - "Fetch packet mismatch cursor=%h got=%h data=%h", cursor, pc, data)) - // Independent instruction reconstruction exercises the same packet boundaries that IBuf consumes. - for (int lane = 0; lane < 2; lane++) - if (mask[lane]) begin - longint unsigned here = (cursor & ~64'h3) + lane * 2; - bit [15:0] halfword = data[lane*16+:16]; - if (pf || af) begin - if (pending_half && here !== partial_pc + 2) - `uvm_fatal("SECOND_HALF", "Fault did not apply to the second halfword") - pending_half = 0; - end else if (pending_half) begin - if ({halfword, first_half} !== fetch_ref_instruction( - partial_pc, version - ) || here !== partial_pc + 2) - `uvm_fatal("RVC", - "32-bit instruction split across groups was reconstructed incorrectly") - pending_half = 0; - instructions++; - end else if (halfword[1:0] != 2'b11) begin - if (halfword !== fetch_ref_instruction(here, version)) - `uvm_fatal("RVC", "Compressed instruction mismatch") - instructions++; - end else begin - first_half = halfword; - partial_pc = here; - pending_half = 1; - end - end - repeat (3) @(control.cb); - packet.ready <= 1; - @(packet.sample); - packet.ready <= 0; - checked++; - cursor = (cursor & ~64'h3) + 4; - endtask - task fetch_one(bit pf = 0, bit af = 0); - offered(cursor); - repeat (3) @(control.cb); - accept_word(); - repeat (2) @(control.cb); - reply(pf, af); - check_packet(pf, af); - endtask - task discarded_reply(); - reply(); - @(control.cb); - if (packet.valid) `uvm_fatal("CANCEL", "Old fetch response escaped redirection") - drains++; - endtask - task fence_ack(); - do @(maintenance.sample); while (!maintenance.sample.valid); - repeat (3) begin - @(control.cb); - if (request.valid || packet.valid) - `uvm_fatal("FENCE", "Fetch proceeded before maintenance request") - end - maintenance.ready <= 1; - @(maintenance.sample); - maintenance.ready <= 0; - repeat (3) begin - @(control.cb); - if (request.valid || packet.valid) - `uvm_fatal("FENCE", "Fetch proceeded before maintenance completed") - end -`ifdef FETCH_BAD_ACK - maintained.bits <= 0; -`else - maintained.bits <= 1; -`endif - maintained.valid <= 1; - do @(maintained.sample); while (!maintained.sample.ready); - maintained.valid <= 0; - maintained.bits <= 0; - barriers++; - endtask - task reset_fetch(int phase_index); - offered(cursor); - if (phase_index != 0) begin - accept_word(); - if (phase_index == 2) begin - reply(); - do @(packet.sample); while (!packet.sample.valid); - end - end - control.cb.reset <= 1; - response.valid <= 0; - maintained.valid <= 1; - maintained.bits <= 0; - control.cb.redirect_valid <= 0; - control.cb.flush <= 0; - control.cb.privilege <= 2; - control.cb.satp <= '1; - control.reset_vector <= '1; - repeat (2) @(control.cb); - control.cb.redirect_valid <= 1; - control.cb.redirect_pc <= '1; - control.cb.flush <= 1; - repeat (2) @(control.cb); - control.cb.redirect_valid <= 0; - control.cb.flush <= 1; - control.reset_vector <= 0; - repeat (2) @(control.cb); - control.cb.flush <= 0; - maintained.valid <= 0; - control.cb.privilege <= 3; - control.cb.satp <= 0; - repeat (2) @(control.cb); - control.cb.reset <= 0; - cursor = control.reset_vector; - pending_half = 0; - cancellations++; - fetch_one(); - endtask - task execute(); - control.reset = 1; -`ifdef FETCH_BAD_RESET - control.reset_vector = 1; -`else - control.reset_vector = 0; -`endif - control.redirect_valid = 0; - control.redirect_pc = 0; - control.flush = 0; -`ifdef FETCH_BAD_PRIVILEGE - control.privilege = 2; -`else - control.privilege = 3; -`endif - control.satp = 0; - control.sum = 0; - control.mxr = 0; - request.ready = 0; - response.valid = 0; - response.bits = '0; - packet.ready = 0; - maintenance.ready = 0; - maintained.valid = 0; - maintained.bits = 1; - cursor = 0; - repeat (4) @(control.cb); - control.cb.reset <= 0; -`ifdef FETCH_BAD_REDIRECT - redirect(1); - repeat (10) @(control.cb); - `uvm_fatal("NO_ASSERT", "Odd instruction PC was accepted") -`elsif FETCH_BAD_FLUSH - control.cb.flush <= 1; - repeat (10) @(control.cb); - `uvm_fatal("NO_ASSERT", "Flush without restart was accepted") -`else - for (int i = 0; i < 8; i++) fetch_one(); - // Zero-latency word owner: response VALID begins in the request handshake cycle. - offered(cursor); - fork - accept_word(); - reply(); - join - check_packet(); - // A stalled offer is immutable even after redirect and live context changes. - offered(cursor); - control.cb.privilege <= 2; - control.reset_vector <= 1; - @(control.cb); - control.cb.privilege <= 0; - control.reset_vector <= 0; - control.cb.satp <= 64'h8000123456789000; - control.cb.context_sum <= 1; - control.cb.mxr <= 1; - redirect(2); - repeat (3) @(control.cb); - if (request.bits !== saved_request || !request.valid) - `uvm_fatal("OFFER", "Redirect withdrew or changed a backpressured word offer") - accept_word(); - redirect(6); - discarded_reply(); - for (int i = 0; i < 4; i++) fetch_one(); - // Same-cycle response and redirect must drain the old word and suppress the packet. - offered(cursor); - accept_word(); - fork - redirect(62); - begin - discarded_reply(); - end - join - for (int i = 0; i < 4; i++) fetch_one(); - // A redirect in the same cycle as accepting the old word still drains that word. - offered(cursor); - fork - accept_word(); - redirect(66); - join - discarded_reply(); - for (int i = 0; i < 3; i++) fetch_one(); - // Cancel an already buffered packet under CPU backpressure. - offered(cursor); - accept_word(); - reply(); - do @(packet.sample); while (!packet.sample.valid); - packet.ready <= 1; - redirect(126); - packet.ready <= 0; - for (int i = 0; i < 3; i++) fetch_one(); - // At PC=4094 a 32-bit instruction starts at the last halfword of the page. - redirect(4094); - fetch_one(); - if (!pending_half) - `uvm_fatal("SECOND_HALF", "Page boundary did not leave a partial instruction") - fetch_one(1, 0); - redirect(8192); - fetch_one(0, 1); - // Flush must drain an old accepted read before requesting external maintenance. - offered(cursor); - accept_word(); - version = 1; - redirect(0, 1); - discarded_reply(); - fence_ack(); - for (int i = 0; i < 4; i++) fetch_one(); - // A fence arriving with a maintenance handshake needs another ordered acknowledgement. - offered(cursor); - accept_word(); - reply(); - do @(packet.sample); while (!packet.sample.valid); - redirect(18, 1); - do @(maintenance.sample); while (!maintenance.sample.valid); - fork - begin - maintenance.ready <= 1; - @(maintenance.sample); - maintenance.ready <= 0; - end - redirect(30, 1); - join - repeat (3) @(control.cb); - maintained.valid <= 1; - maintained.bits <= 1; - do @(maintained.sample); while (!maintained.sample.ready); - maintained.valid <= 0; - maintained.bits <= 0; - barriers++; - fence_ack(); - for (int i = 0; i < 3; i++) fetch_one(); - // Byte-memory patterns vary all transported instruction bits without duplicating the execution decoder. - for (int image = 2; image < 36; image++) begin - version = image; - control.cb.privilege <= image % 3 == 0 ? 3 : (image % 3 == 1 ? 1 : 0); - control.cb.satp <= image[0] ? 64'h8fffffffffffffff : 0; - control.cb.context_sum <= image[0]; - control.cb.mxr <= !image[0]; - redirect(image * 64); - for (int i = 0; i < 3; i++) fetch_one(); - end - control.cb.privilege <= 3; - control.cb.satp <= 0; - redirect(64'h2000); - fetch_one(); - for (int bit_index = 3; bit_index < 64; bit_index++) begin - redirect(64'h1 << bit_index); - fetch_one(0, bit_index >= 44); - end - for (int phase_index = 0; phase_index < 3; phase_index++) reset_fetch(phase_index); - // Joint reset cancels an accepted maintenance transaction and ignores its reset-cycle payload. - offered(cursor); - accept_word(); - reply(); - do @(packet.sample); while (!packet.sample.valid); - redirect(0, 1); - do @(maintenance.sample); while (!maintenance.sample.valid); - maintenance.ready <= 1; - @(maintenance.sample); - maintenance.ready <= 0; - control.cb.reset <= 1; - maintained.valid <= 1; - maintained.bits <= 0; - repeat (2) @(control.cb); - maintained.valid <= 0; - control.cb.reset <= 0; - cursor = control.reset_vector; - pending_half = 0; - cancellations++; - fetch_one(); - // READY can be high with VALID=0; unrelated payload cannot create a transaction. - request.ready <= 1; - maintenance.ready <= 1; - control.cb.redirect_pc <= '1; - @(control.cb); - control.cb.redirect_pc <= 0; - // Explicitly terminate the autonomous producer and cancel any trailing halfword by joint reset. - control.cb.reset <= 1; - response.valid <= 0; - maintained.valid <= 0; - pending_half = 0; - repeat (2) @(control.cb); - if (request.valid || packet.valid || maintenance.valid) - `uvm_fatal("RESET", "Reset retained a live frontend transaction") - `uvm_info("FETCH", $sformatf( - "Checked %0d packets, reconstructed %0d instructions, %0d accepted reads, %0d stale drains, %0d maintenance acknowledgements, %0d joint-reset cancellations", - checked, - instructions, - reads, - drains, - barriers, - cancellations - ), UVM_LOW) -`endif - endtask -endclass diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/main/scala/Emit.scala b/arch/src/main/scala/framework/mem-core/fetch/src/main/scala/Emit.scala deleted file mode 100644 index b4a8f6420..000000000 --- a/arch/src/main/scala/framework/mem-core/fetch/src/main/scala/Emit.scala +++ /dev/null @@ -1,72 +0,0 @@ -package memcore.memory.fetch - -import java.nio.file.{Files, Paths} - -object Emit extends App { - val p = Params() - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Fetch(p), - firtoolOpts = args, - args = Array("--target-dir", "build/Fetch", "--split-verilog") - ) - val definitions = scala.collection.mutable.ArrayBuffer[String]() - val ports = scala.collection.mutable.ArrayBuffer[String]() - - def fields( - kind: String, - prefix: String, - instance: String, - fs: Seq[(String, Int)] - ): Unit = { - var offset = 0 - for ((name, width) <- fs) { - definitions += s"`define FETCH_${kind}_${name.toUpperCase}_OFFSET $offset" - definitions += s"`define FETCH_${kind}_${name.toUpperCase}_WIDTH $width" - ports += s".$prefix$name($instance.bits[$offset +: $width])" - offset += width - } - definitions += s"`define FETCH_${kind}_WIDTH $offset" - } - - fields( - "REQUEST", - "io_request_bits_", - "request", - Seq( - "addr" -> 64, - "context_privilege" -> 2, - "context_satp" -> 64, - "context_sum" -> 1, - "context_mxr" -> 1, - "execute" -> 1 - ) - ) - fields( - "RESPONSE", - "io_response_bits_", - "response", - Seq("data" -> 64, "line" -> 512, "lineExecutable" -> 1, "pageFault" -> 1, "accessFault" -> 1) - ) - fields( - "PACKET", - "io_packet_bits_", - "sink_if", - Seq("pc" -> 64, "data" -> 32, "mask" -> p.lanes, "pageFault" -> 1, "accessFault" -> 1) - ) - for ( - (prefix, instance) <- Seq( - "request" -> "request", - "response" -> "response", - "packet" -> "sink_if", - "maintenance" -> "maintenance", - "maintained" -> "maintained" - ) - ) { - ports += s".io_${prefix}_valid($instance.valid)" - ports += s".io_${prefix}_ready($instance.ready)" - } - ports += ".io_maintenance_bits(maintenance.bits[0])" - ports += ".io_maintained_bits(maintained.bits[0])" - Files.write(Paths.get("build/fetch_config.svh"), (definitions.mkString("\n") + "\n").getBytes) - Files.write(Paths.get("build/fetch_ports.svh"), ports.mkString(",\n").getBytes) -} diff --git a/arch/src/main/scala/framework/mem-core/interlock/build.sc b/arch/src/main/scala/framework/mem-core/interlock/build.sc deleted file mode 100644 index 90648d244..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/build.sc +++ /dev/null @@ -1,10 +0,0 @@ -import mill._ -import mill.scalalib._ -object interlock extends SbtModule { - override def millSourcePath = os.pwd - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") - override def mainClass = Some("memcore.memory.interlock.Emit") -} diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/csrc/Cargo.lock b/arch/src/main/scala/framework/mem-core/interlock/src/csrc/Cargo.lock deleted file mode 100644 index a2560a2a9..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/csrc/Cargo.lock +++ /dev/null @@ -1,7 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "interlock-reference" -version = "0.1.0" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/csrc/Cargo.toml b/arch/src/main/scala/framework/mem-core/interlock/src/csrc/Cargo.toml deleted file mode 100644 index ce45bc7a7..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/csrc/Cargo.toml +++ /dev/null @@ -1,6 +0,0 @@ -[package] -name = "interlock-reference" -version = "0.1.0" -edition = "2021" -[lib] -crate-type = ["cdylib", "staticlib"] diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/csrc/src/lib.rs b/arch/src/main/scala/framework/mem-core/interlock/src/csrc/src/lib.rs deleted file mode 100644 index a61a9079a..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/csrc/src/lib.rs +++ /dev/null @@ -1,2 +0,0 @@ -mod ownership; -pub use ownership::*; diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/csrc/src/ownership.rs b/arch/src/main/scala/framework/mem-core/interlock/src/csrc/src/ownership.rs deleted file mode 100644 index 3b3a906ee..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/csrc/src/ownership.rs +++ /dev/null @@ -1,99 +0,0 @@ -//! Transaction facts from real handshakes. No hardware cursor or phase FSM is mirrored. -use std::ffi::c_void; -#[derive(Clone, Copy)] -struct Access { base: u64, bytes: u64, write: bool } -struct Transaction { - id: u32, age: u64, sealed: bool, ranges: Vec, - pre: Vec, post: Vec, granted: bool, dma_ack: bool, -} -struct Model { - capacity: usize, address_bits: u32, line_bytes: u64, max_ranges: usize, - age: u64, live: Vec, maintenance: Option<(u32, u32, usize)>, -} -impl Model { - fn transaction(&self,id:u32)->&Transaction { self.live.iter().find(|t|t.id==id).expect("unknown transaction") } - fn transaction_mut(&mut self,id:u32)->&mut Transaction { self.live.iter_mut().find(|t|t.id==id).expect("unknown transaction") } - fn overlap(a:Access,b:Access)->bool { a.baseOption { - t.ranges.first().map(|first| { - let base=t.ranges.iter().map(|a|a.base).min().unwrap(); - let end=t.ranges.iter().map(|a|a.base+a.bytes).max().unwrap(); - Access{base,bytes:end-base,write:first.write} - }) - } - fn older_conflict(&self,t:&Transaction)->bool { - let a=Self::bounds(t).expect("memory grant without ranges"); - self.live.iter().filter(|old|old.age(p:*mut c_void)->&'a mut Model { &mut *p.cast::() } -#[no_mangle] -pub extern "C" fn interlock_ref_create(entries:u32,address_bits:u32,line_bytes:u32,max_ranges:u32)->*mut c_void { - assert!(entries>0&&address_bits<64&&line_bytes.is_power_of_two()&&max_ranges>0); - Box::into_raw(Box::new(Model{capacity:entries as usize,address_bits,line_bytes:line_bytes as u64,max_ranges:max_ranges as usize,age:0,live:Vec::new(),maintenance:None})).cast() -} -#[no_mangle] -pub unsafe extern "C" fn interlock_ref_destroy(p:*mut c_void) { drop(Box::from_raw(p.cast::())); } -#[no_mangle] -pub unsafe extern "C" fn interlock_ref_reset(p:*mut c_void) { let m=model(p);m.live.clear();m.maintenance=None;m.age=0; } -#[no_mangle] -pub unsafe extern "C" fn interlock_ref_live(p:*mut c_void)->u32 { model(p).live.len() as u32 } -#[no_mangle] -pub unsafe extern "C" fn interlock_ref_reserve(p:*mut c_void,id:u32) { - let m=model(p);assert!(m.live.len()0&&base.checked_add(bytes).is_some_and(|end|end<=(1<u8 { - let m=model(p);assert!(bytes.is_power_of_two()&&bytes<=8&&addr%bytes as u64==0); - assert!(addr/m.line_bytes==(addr+bytes as u64-1)/m.line_bytes); - if older!=0||reserve_now!=0{return 0;} - let cpu=Access{base:addr,bytes:bytes as u64,write:write!=0}; - u8::from(!m.live.iter().any(|t|!t.sealed||t.ranges.iter().any(|&a|Model::overlap(cpu,a)&&(cpu.write||a.write)))) -} -#[no_mangle] -pub unsafe extern "C" fn interlock_ref_maint_offer(p:*mut c_void,id:u32,op:u32,first:u64,last:u64) { - let m=model(p);assert!(m.maintenance.is_none());let t=m.transaction(id);assert!(t.sealed&&!t.ranges.is_empty()); - let write=t.ranges[0].write; - if op==2 { assert!(write&&t.granted&&t.dma_ack); } - else { assert!(op==u32::from(write)&&!t.granted&&!m.older_conflict(t)); } - let done=if op==2{&t.post}else{&t.pre}; - let index=t.ranges.iter().enumerate().position(|(i,a)|!done[i]&&a.base==first&&a.base+a.bytes-m.line_bytes==last).expect("wrong or duplicate maintenance extent"); - m.maintenance=Some((id,op,index)); -} -#[no_mangle] -pub unsafe extern "C" fn interlock_ref_maint_ack(p:*mut c_void,id:u32) { - let m=model(p);let (owner,op,index)=m.maintenance.take().expect("ACK without maintenance request");assert_eq!(owner,id); - let t=m.transaction_mut(id);if op==2{t.post[index]=true;}else{t.pre[index]=true;} -} -#[no_mangle] -pub unsafe extern "C" fn interlock_ref_grant(p:*mut c_void,id:u32) { - let m=model(p);let t=m.transaction(id);assert!(t.sealed&&!t.ranges.is_empty()&&t.pre.iter().all(|b|*b)&&!t.granted&&!m.older_conflict(t)); - m.transaction_mut(id).granted=true; -} -#[no_mangle] -pub unsafe extern "C" fn interlock_ref_dma_ack(p:*mut c_void,id:u32) { let t=model(p).transaction_mut(id);assert!(t.granted&&!t.dma_ack);t.dma_ack=true; } -#[no_mangle] -pub unsafe extern "C" fn interlock_ref_complete(p:*mut c_void,id:u32) { - let m=model(p);let t=m.transaction(id);assert!(t.sealed); - assert!(t.ranges.is_empty()||(t.granted&&t.dma_ack&&(!t.ranges[0].write||t.post.iter().all(|b|*b))),"premature completion"); - m.live.retain(|t|t.id!=id); -} diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/control_if.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/control_if.sv deleted file mode 100644 index 758c6018e..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/control_if.sv +++ /dev/null @@ -1,5 +0,0 @@ -interface interlock_control_if ( - input logic clock -); - bit start = 0, done = 0; -endinterface diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock.f deleted file mode 100644 index c6f620e16..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_info.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_info.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_info.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_info.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_info.f deleted file mode 100644 index c0dd50220..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_info.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_bad_cancel_info_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_info.toml b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_info.toml deleted file mode 100644 index 07bf6ed6b..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_info.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Interlock cancel conflicts with the same command" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_info_scenario.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_info_scenario.svh deleted file mode 100644 index 7cbcd7f94..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_info_scenario.svh +++ /dev/null @@ -1,8 +0,0 @@ -// verilog_syntax: parse-as-statements -reserve(7); -@(negedge clock); -cancel.valid = 1; -`ILF(cancel.bits, CANCEL, ID) = 7; -access_info.valid = 1; -`ILF(access_info.bits, ACCESS_INFO, ID) = 7; -`ILF(access_info.bits, ACCESS_INFO, LAST) = 1; diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_info_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_info_tb.sv deleted file mode 100644 index 2f719bd40..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_info_tb.sv +++ /dev/null @@ -1,3 +0,0 @@ -`define INTERLOCK_NEGATIVE_TB interlock_bad_cancel_info_tb -`define INTERLOCK_NEGATIVE_SCENARIO "interlock_bad_cancel_info_scenario.svh" -`include "negative_tb.svh" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_sealed.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_sealed.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_sealed.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_sealed.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_sealed.f deleted file mode 100644 index c3fdbb8f5..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_sealed.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_bad_cancel_sealed_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_sealed.toml b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_sealed.toml deleted file mode 100644 index dc66a7902..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_sealed.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Interlock cancel requires an unsealed reservation" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_sealed_scenario.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_sealed_scenario.svh deleted file mode 100644 index ee4ca4cec..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_sealed_scenario.svh +++ /dev/null @@ -1,6 +0,0 @@ -// verilog_syntax: parse-as-statements -reserve(7); -info(7, 1, 'h4000, 64, 0, 1); -@(negedge clock); -cancel.valid = 1; -`ILF(cancel.bits, CANCEL, ID) = 7; diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_sealed_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_sealed_tb.sv deleted file mode 100644 index 7550d92a2..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_sealed_tb.sv +++ /dev/null @@ -1,3 +0,0 @@ -`define INTERLOCK_NEGATIVE_TB interlock_bad_cancel_sealed_tb -`define INTERLOCK_NEGATIVE_SCENARIO "interlock_bad_cancel_sealed_scenario.svh" -`include "negative_tb.svh" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_unknown.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_unknown.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_unknown.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_unknown.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_unknown.f deleted file mode 100644 index ea5d2bb9d..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_unknown.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_bad_cancel_unknown_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_unknown.toml b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_unknown.toml deleted file mode 100644 index ebac80229..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_unknown.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Interlock cancel ID has no reservation" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_unknown_scenario.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_unknown_scenario.svh deleted file mode 100644 index a4a8587f2..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_unknown_scenario.svh +++ /dev/null @@ -1,4 +0,0 @@ -// verilog_syntax: parse-as-statements -@(negedge clock); -cancel.valid = 1; -`ILF(cancel.bits, CANCEL, ID) = 7; diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_unknown_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_unknown_tb.sv deleted file mode 100644 index 298015b8f..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_unknown_tb.sv +++ /dev/null @@ -1,3 +0,0 @@ -`define INTERLOCK_NEGATIVE_TB interlock_bad_cancel_unknown_tb -`define INTERLOCK_NEGATIVE_SCENARIO "interlock_bad_cancel_unknown_scenario.svh" -`include "negative_tb.svh" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_capacity.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_capacity.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_capacity.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_capacity.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_capacity.f deleted file mode 100644 index 7755f7b96..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_capacity.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_bad_capacity_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_capacity.toml b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_capacity.toml deleted file mode 100644 index 286cf7f26..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_capacity.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Interlock range list must seal within capacity" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_capacity_scenario.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_capacity_scenario.svh deleted file mode 100644 index f5f2761d3..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_capacity_scenario.svh +++ /dev/null @@ -1,3 +0,0 @@ -// verilog_syntax: parse-as-statements -reserve(7); -for (int i = 0; i < 8; i++) info(7, 1, 'h4000 + i * 128, 64, 0, 0); diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_capacity_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_capacity_tb.sv deleted file mode 100644 index 179a82e97..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_capacity_tb.sv +++ /dev/null @@ -1,3 +0,0 @@ -`define INTERLOCK_NEGATIVE_TB interlock_bad_capacity_tb -`define INTERLOCK_NEGATIVE_SCENARIO "interlock_bad_capacity_scenario.svh" -`include "negative_tb.svh" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_alignment.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_alignment.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_alignment.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_alignment.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_alignment.f deleted file mode 100644 index 6248b967c..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_alignment.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_bad_cpu_alignment_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_alignment.toml b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_alignment.toml deleted file mode 100644 index 34829bec4..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_alignment.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Interlock CPU access must be naturally aligned" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_alignment_scenario.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_alignment_scenario.svh deleted file mode 100644 index e41448a14..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_alignment_scenario.svh +++ /dev/null @@ -1,5 +0,0 @@ -// verilog_syntax: parse-as-statements -@(negedge clock); -`ILF(cpu_query.bits, CPU_QUERY, VALID) = 1; -`ILF(cpu_query.bits, CPU_QUERY, SIZELOG2) = 3; -`ILF(cpu_query.bits, CPU_QUERY, PADDR) = 'h4001; diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_alignment_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_alignment_tb.sv deleted file mode 100644 index 59c2bc926..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_alignment_tb.sv +++ /dev/null @@ -1,3 +0,0 @@ -`define INTERLOCK_NEGATIVE_TB interlock_bad_cpu_alignment_tb -`define INTERLOCK_NEGATIVE_SCENARIO "interlock_bad_cpu_alignment_scenario.svh" -`include "negative_tb.svh" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_size.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_size.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_size.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_size.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_size.f deleted file mode 100644 index 5b860ef1a..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_size.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_bad_cpu_size_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_size.toml b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_size.toml deleted file mode 100644 index 75d374e71..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_size.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Interlock CPU size must be 1, 2, 4 or 8 bytes" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_size_scenario.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_size_scenario.svh deleted file mode 100644 index 42fc29d8b..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_size_scenario.svh +++ /dev/null @@ -1,4 +0,0 @@ -// verilog_syntax: parse-as-statements -@(negedge clock); -`ILF(cpu_query.bits, CPU_QUERY, VALID) = 1; -`ILF(cpu_query.bits, CPU_QUERY, SIZELOG2) = 4; diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_size_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_size_tb.sv deleted file mode 100644 index ddccba87c..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_size_tb.sv +++ /dev/null @@ -1,3 +0,0 @@ -`define INTERLOCK_NEGATIVE_TB interlock_bad_cpu_size_tb -`define INTERLOCK_NEGATIVE_SCENARIO "interlock_bad_cpu_size_scenario.svh" -`include "negative_tb.svh" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_direction.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_direction.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_direction.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_direction.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_direction.f deleted file mode 100644 index 02ddcb98c..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_direction.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_bad_direction_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_direction.toml b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_direction.toml deleted file mode 100644 index 6f78b4bfd..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_direction.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Interlock ranges must have one access direction" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_direction_scenario.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_direction_scenario.svh deleted file mode 100644 index 34d02d785..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_direction_scenario.svh +++ /dev/null @@ -1,4 +0,0 @@ -// verilog_syntax: parse-as-statements -reserve(7); -info(7, 1, 'h4000, 64, 0, 0); -info(7, 1, 'h5000, 64, 1, 1); diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_direction_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_direction_tb.sv deleted file mode 100644 index 6e55aee8b..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_direction_tb.sv +++ /dev/null @@ -1,3 +0,0 @@ -`define INTERLOCK_NEGATIVE_TB interlock_bad_direction_tb -`define INTERLOCK_NEGATIVE_SCENARIO "interlock_bad_direction_scenario.svh" -`include "negative_tb.svh" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_failed.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_failed.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_failed.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_failed.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_failed.f deleted file mode 100644 index 31dbfb80c..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_failed.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_bad_dma_failed_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_failed.toml b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_failed.toml deleted file mode 100644 index 10a115658..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_failed.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Interlock DMA failed" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_failed_scenario.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_failed_scenario.svh deleted file mode 100644 index 3b5b1b04d..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_failed_scenario.svh +++ /dev/null @@ -1,15 +0,0 @@ -// verilog_syntax: parse-as-statements -reserve(7); -info(7, 1, 'h4000, 64, 1); -maintenance_accept(); -maintenance_ack(7, 1); -do @(posedge clock); while (!maintained.ready); -@(negedge clock); -maintained.valid = 0; -grant.ready = 1; -do @(posedge clock); while (!grant.valid); -@(negedge clock); -grant.ready = 0; -done.valid = 1; -`ILF(done.bits, DONE, TAG) = 7; -`ILF(done.bits, DONE, OK) = 0; diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_failed_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_failed_tb.sv deleted file mode 100644 index 8c8a0c957..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_failed_tb.sv +++ /dev/null @@ -1,3 +0,0 @@ -`define INTERLOCK_NEGATIVE_TB interlock_bad_dma_failed_tb -`define INTERLOCK_NEGATIVE_SCENARIO "interlock_bad_dma_failed_scenario.svh" -`include "negative_tb.svh" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_ungranted.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_ungranted.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_ungranted.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_ungranted.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_ungranted.f deleted file mode 100644 index 40a4d9123..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_ungranted.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_bad_dma_ungranted_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_ungranted.toml b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_ungranted.toml deleted file mode 100644 index c03f2dc8f..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_ungranted.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Interlock DMA done requires a granted access" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_ungranted_scenario.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_ungranted_scenario.svh deleted file mode 100644 index 4b6151177..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_ungranted_scenario.svh +++ /dev/null @@ -1,7 +0,0 @@ -// verilog_syntax: parse-as-statements -reserve(7); -info(7, 1, 'h4000, 64, 1); -@(negedge clock); -done.valid = 1; -`ILF(done.bits, DONE, TAG) = 7; -`ILF(done.bits, DONE, OK) = 1; diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_ungranted_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_ungranted_tb.sv deleted file mode 100644 index f803fa281..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_ungranted_tb.sv +++ /dev/null @@ -1,3 +0,0 @@ -`define INTERLOCK_NEGATIVE_TB interlock_bad_dma_ungranted_tb -`define INTERLOCK_NEGATIVE_SCENARIO "interlock_bad_dma_ungranted_scenario.svh" -`include "negative_tb.svh" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_unknown.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_unknown.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_unknown.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_unknown.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_unknown.f deleted file mode 100644 index a4ff5ef32..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_unknown.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_bad_dma_unknown_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_unknown.toml b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_unknown.toml deleted file mode 100644 index c8de0ff0b..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_unknown.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Interlock DMA done tag has no reservation" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_unknown_scenario.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_unknown_scenario.svh deleted file mode 100644 index 05f9f5e37..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_unknown_scenario.svh +++ /dev/null @@ -1,5 +0,0 @@ -// verilog_syntax: parse-as-statements -@(negedge clock); -done.valid = 1; -`ILF(done.bits, DONE, TAG) = 7; -`ILF(done.bits, DONE, OK) = 1; diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_unknown_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_unknown_tb.sv deleted file mode 100644 index 2095fc13a..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_unknown_tb.sv +++ /dev/null @@ -1,3 +0,0 @@ -`define INTERLOCK_NEGATIVE_TB interlock_bad_dma_unknown_tb -`define INTERLOCK_NEGATIVE_SCENARIO "interlock_bad_dma_unknown_scenario.svh" -`include "negative_tb.svh" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_duplicate.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_duplicate.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_duplicate.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_duplicate.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_duplicate.f deleted file mode 100644 index 3120d9da8..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_duplicate.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_bad_duplicate_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_duplicate.toml b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_duplicate.toml deleted file mode 100644 index 5b2a37f26..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_duplicate.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Interlock dispatch ID is already live" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_duplicate_scenario.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_duplicate_scenario.svh deleted file mode 100644 index e99e77c88..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_duplicate_scenario.svh +++ /dev/null @@ -1,5 +0,0 @@ -// verilog_syntax: parse-as-statements -reserve(7); -@(negedge clock); -dispatch.valid = 1; -`ILF(dispatch.bits, DISPATCH, ID) = 7; diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_duplicate_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_duplicate_tb.sv deleted file mode 100644 index c175edc15..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_duplicate_tb.sv +++ /dev/null @@ -1,3 +0,0 @@ -`define INTERLOCK_NEGATIVE_TB interlock_bad_duplicate_tb -`define INTERLOCK_NEGATIVE_SCENARIO "interlock_bad_duplicate_scenario.svh" -`include "negative_tb.svh" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_empty_range.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_empty_range.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_empty_range.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_empty_range.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_empty_range.f deleted file mode 100644 index 0f4e5672c..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_empty_range.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_bad_empty_range_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_empty_range.toml b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_empty_range.toml deleted file mode 100644 index b228dd89f..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_empty_range.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Interlock memory range is empty" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_empty_range_scenario.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_empty_range_scenario.svh deleted file mode 100644 index a8fe960d9..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_empty_range_scenario.svh +++ /dev/null @@ -1,3 +0,0 @@ -// verilog_syntax: parse-as-statements -reserve(7); -info(7, 1, 'h4000, 0, 0); diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_empty_range_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_empty_range_tb.sv deleted file mode 100644 index b5d405352..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_empty_range_tb.sv +++ /dev/null @@ -1,3 +0,0 @@ -`define INTERLOCK_NEGATIVE_TB interlock_bad_empty_range_tb -`define INTERLOCK_NEGATIVE_SCENARIO "interlock_bad_empty_range_scenario.svh" -`include "negative_tb.svh" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_duplicate.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_duplicate.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_duplicate.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_duplicate.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_duplicate.f deleted file mode 100644 index a1500033b..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_duplicate.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_bad_info_duplicate_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_duplicate.toml b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_duplicate.toml deleted file mode 100644 index 8d437efdb..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_duplicate.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Interlock access info is already declared" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_duplicate_scenario.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_duplicate_scenario.svh deleted file mode 100644 index 2e1c233b5..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_duplicate_scenario.svh +++ /dev/null @@ -1,5 +0,0 @@ -// verilog_syntax: parse-as-statements -reserve(7); -info(7, 1, 'h4000, 64, 0); -@(negedge clock); -access_info.valid = 1; diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_duplicate_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_duplicate_tb.sv deleted file mode 100644 index ea5583f8d..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_duplicate_tb.sv +++ /dev/null @@ -1,3 +0,0 @@ -`define INTERLOCK_NEGATIVE_TB interlock_bad_info_duplicate_tb -`define INTERLOCK_NEGATIVE_SCENARIO "interlock_bad_info_duplicate_scenario.svh" -`include "negative_tb.svh" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_unknown.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_unknown.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_unknown.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_unknown.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_unknown.f deleted file mode 100644 index c188f0811..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_unknown.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_bad_info_unknown_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_unknown.toml b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_unknown.toml deleted file mode 100644 index fd97b4bd0..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_unknown.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Interlock access info ID has no reservation" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_unknown_scenario.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_unknown_scenario.svh deleted file mode 100644 index 74f805ad4..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_unknown_scenario.svh +++ /dev/null @@ -1,4 +0,0 @@ -// verilog_syntax: parse-as-statements -@(negedge clock); -access_info.valid = 1; -`ILF(access_info.bits, ACCESS_INFO, ID) = 7; diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_unknown_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_unknown_tb.sv deleted file mode 100644 index 833111221..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_unknown_tb.sv +++ /dev/null @@ -1,3 +0,0 @@ -`define INTERLOCK_NEGATIVE_TB interlock_bad_info_unknown_tb -`define INTERLOCK_NEGATIVE_SCENARIO "interlock_bad_info_unknown_scenario.svh" -`include "negative_tb.svh" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_failed.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_failed.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_failed.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_failed.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_failed.f deleted file mode 100644 index e477f0ec8..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_failed.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_bad_maint_failed_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_failed.toml b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_failed.toml deleted file mode 100644 index ae0c93089..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_failed.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Interlock maintenance failed" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_failed_scenario.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_failed_scenario.svh deleted file mode 100644 index 54db9679d..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_failed_scenario.svh +++ /dev/null @@ -1,5 +0,0 @@ -// verilog_syntax: parse-as-statements -reserve(7); -info(7, 1, 'h4000, 64, 1); -maintenance_accept(); -maintenance_ack(7, 0); diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_failed_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_failed_tb.sv deleted file mode 100644 index e01336b5f..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_failed_tb.sv +++ /dev/null @@ -1,3 +0,0 @@ -`define INTERLOCK_NEGATIVE_TB interlock_bad_maint_failed_tb -`define INTERLOCK_NEGATIVE_SCENARIO "interlock_bad_maint_failed_scenario.svh" -`include "negative_tb.svh" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_tag.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_tag.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_tag.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_tag.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_tag.f deleted file mode 100644 index 6e82b0296..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_tag.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_bad_maint_tag_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_tag.toml b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_tag.toml deleted file mode 100644 index 03417e409..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_tag.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Interlock maintenance response tag mismatch" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_tag_scenario.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_tag_scenario.svh deleted file mode 100644 index 4c06f89cb..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_tag_scenario.svh +++ /dev/null @@ -1,5 +0,0 @@ -// verilog_syntax: parse-as-statements -reserve(7); -info(7, 1, 'h4000, 64, 1); -maintenance_accept(); -maintenance_ack(8, 1); diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_tag_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_tag_tb.sv deleted file mode 100644 index 34f465476..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_tag_tb.sv +++ /dev/null @@ -1,3 +0,0 @@ -`define INTERLOCK_NEGATIVE_TB interlock_bad_maint_tag_tb -`define INTERLOCK_NEGATIVE_SCENARIO "interlock_bad_maint_tag_scenario.svh" -`include "negative_tb.svh" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_unknown.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_unknown.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_unknown.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_unknown.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_unknown.f deleted file mode 100644 index 4969360ce..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_unknown.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_bad_maint_unknown_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_unknown.toml b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_unknown.toml deleted file mode 100644 index 46923a720..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_unknown.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Interlock maintenance response has no request" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_unknown_scenario.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_unknown_scenario.svh deleted file mode 100644 index 2c5a2779e..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_unknown_scenario.svh +++ /dev/null @@ -1,2 +0,0 @@ -// verilog_syntax: parse-as-statements -maintenance_ack(7, 1); diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_unknown_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_unknown_tb.sv deleted file mode 100644 index 8d557bd0e..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_unknown_tb.sv +++ /dev/null @@ -1,3 +0,0 @@ -`define INTERLOCK_NEGATIVE_TB interlock_bad_maint_unknown_tb -`define INTERLOCK_NEGATIVE_SCENARIO "interlock_bad_maint_unknown_scenario.svh" -`include "negative_tb.svh" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_no_memory_seal.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_no_memory_seal.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_no_memory_seal.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_no_memory_seal.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_no_memory_seal.f deleted file mode 100644 index f101005ad..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_no_memory_seal.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_bad_no_memory_seal_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_no_memory_seal.toml b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_no_memory_seal.toml deleted file mode 100644 index 5fee3763a..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_no_memory_seal.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Interlock no-memory declaration must seal an empty command" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_no_memory_seal_scenario.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_no_memory_seal_scenario.svh deleted file mode 100644 index 401845ab8..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_no_memory_seal_scenario.svh +++ /dev/null @@ -1,3 +0,0 @@ -// verilog_syntax: parse-as-statements -reserve(7); -info(7, 0, 0, 0, 0, 0); diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_no_memory_seal_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_no_memory_seal_tb.sv deleted file mode 100644 index a16997398..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_no_memory_seal_tb.sv +++ /dev/null @@ -1,3 +0,0 @@ -`define INTERLOCK_NEGATIVE_TB interlock_bad_no_memory_seal_tb -`define INTERLOCK_NEGATIVE_SCENARIO "interlock_bad_no_memory_seal_scenario.svh" -`include "negative_tb.svh" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_overflow_range.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_overflow_range.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_overflow_range.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_overflow_range.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_overflow_range.f deleted file mode 100644 index af0de0558..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_overflow_range.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_bad_overflow_range_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_overflow_range.toml b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_overflow_range.toml deleted file mode 100644 index 5b613055b..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_overflow_range.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Interlock memory range exceeds physical width" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_overflow_range_scenario.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_overflow_range_scenario.svh deleted file mode 100644 index aa5305eed..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_overflow_range_scenario.svh +++ /dev/null @@ -1,3 +0,0 @@ -// verilog_syntax: parse-as-statements -reserve(7); -info(7, 1, (64'd1 << 44) - 64, 65, 0); diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_overflow_range_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_overflow_range_tb.sv deleted file mode 100644 index 41a5eaef3..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_overflow_range_tb.sv +++ /dev/null @@ -1,3 +0,0 @@ -`define INTERLOCK_NEGATIVE_TB interlock_bad_overflow_range_tb -`define INTERLOCK_NEGATIVE_SCENARIO "interlock_bad_overflow_range_scenario.svh" -`include "negative_tb.svh" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_tb.sv deleted file mode 100644 index 1bc7f983f..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_tb.sv +++ /dev/null @@ -1,743 +0,0 @@ -module interlock_tb; - import uvm_pkg::*; - import interlock_pkg::*; - `include "uvm_macros.svh" - `include "interlock_config.svh" - `define ILF(B, K, F) B[`INTERLOCK_``K``_``F``_OFFSET +: `INTERLOCK_``K``_``F``_WIDTH] - logic clock = 0, reset = 1, cpu_allow; - always #5 clock = ~clock; - interlock_control_if ctl (clock); - stream_if #( - .WIDTH(`INTERLOCK_DISPATCH_WIDTH) - ) dispatch ( - clock, - reset - ); - stream_if #( - .WIDTH(`INTERLOCK_CANCEL_WIDTH) - ) cancel ( - clock, - reset - ); - stream_if #( - .WIDTH(`INTERLOCK_ACCESS_INFO_WIDTH) - ) access_info ( - clock, - reset - ); - stream_if #( - .WIDTH(`INTERLOCK_MAINTENANCE_WIDTH) - ) maintenance ( - clock, - reset - ); - stream_if #( - .WIDTH(`INTERLOCK_MAINTAINED_WIDTH) - ) maintained ( - clock, - reset - ); - stream_if #( - .WIDTH(`INTERLOCK_GRANT_WIDTH) - ) grant ( - clock, - reset - ); - stream_if #( - .WIDTH(`INTERLOCK_DONE_WIDTH) - ) done ( - clock, - reset - ); - stream_if #( - .WIDTH(`INTERLOCK_COMPLETE_WIDTH) - ) complete ( - clock, - reset - ); - stream_if #( - .WIDTH(`INTERLOCK_CPU_QUERY_WIDTH) - ) cpu_query ( - clock, - reset - ); - Interlock dut ( - .clock(clock), - .reset(reset), - `include "interlock_ports.svh" - ); - chandle model; - int grants[256], completions[256], maintenance_ops[256][3]; - int checked_queries = 0, reserved = 0, acknowledged = 0, released = 0; - bit model_ready = 0; - int cache_pending_refills = 0; - function automatic void check(bit value, string message); - if (!value) `uvm_fatal("INTERLOCK_CONTRACT", message) - endfunction - always @(posedge clock) begin - if (model_ready) begin - if (reset) begin - interlock_ref_reset(model); - foreach (grants[i]) begin - grants[i] = 0; - completions[i] = 0; - for (int j = 0; j < 3; j++) maintenance_ops[i][j] = 0; - end - end else begin - if (`ILF(cpu_query.bits, CPU_QUERY, VALID)) begin - check(cpu_allow === interlock_ref_cpu_allow( - model, - `ILF(cpu_query.bits, CPU_QUERY, PADDR), - 1 << - `ILF(cpu_query.bits, CPU_QUERY, SIZELOG2), - `ILF(cpu_query.bits, CPU_QUERY, WRITE), - `ILF(cpu_query.bits, CPU_QUERY, OLDERDISPATCHPENDING), - dispatch.valid && dispatch.ready - ), "CPU permit disagrees with transaction oracle"); - checked_queries++; - end - if (dispatch.valid && dispatch.ready) begin - int id; - id = `ILF(dispatch.bits, DISPATCH, ID); - interlock_ref_reserve(model, id); - reserved++; - grants[id] = 0; - completions[id] = 0; - for (int j = 0; j < 3; j++) maintenance_ops[id][j] = 0; - end - if (cancel.valid && cancel.ready) - interlock_ref_cancel(model, `ILF(cancel.bits, CANCEL, ID)); - if (access_info.valid && access_info.ready) - interlock_ref_info(model, `ILF(access_info.bits, ACCESS_INFO, ID), - `ILF(access_info.bits, ACCESS_INFO, HASMEMORY), - `ILF(access_info.bits, ACCESS_INFO, BASE), - `ILF(access_info.bits, ACCESS_INFO, BYTES), - `ILF(access_info.bits, ACCESS_INFO, WRITE), - `ILF(access_info.bits, ACCESS_INFO, LAST)); - if (maintenance.valid && maintenance.ready) begin - interlock_ref_maint_offer(model, `ILF(maintenance.bits, MAINTENANCE, TAG), - `ILF(maintenance.bits, MAINTENANCE, OP), - `ILF(maintenance.bits, MAINTENANCE, FIRSTLINE), - `ILF(maintenance.bits, MAINTENANCE, LASTLINE)); - maintenance_ops[ - `ILF(maintenance.bits, MAINTENANCE, TAG) - ][ - `ILF(maintenance.bits, MAINTENANCE, OP) - ]++; - end - if (maintained.valid && maintained.ready) begin - check(`ILF(maintained.bits, MAINTAINED, OK), "normal cache BFM failed ACK"); - interlock_ref_maint_ack(model, `ILF(maintained.bits, MAINTAINED, TAG)); - end - if (grant.valid && grant.ready) begin - interlock_ref_grant(model, `ILF(grant.bits, GRANT, TAG)); - grants[`ILF(grant.bits, GRANT, TAG)]++; - end - if (done.valid && done.ready) begin - check(`ILF(done.bits, DONE, OK), "normal DMA BFM failed ACK"); - interlock_ref_dma_ack(model, `ILF(done.bits, DONE, TAG)); - acknowledged++; - end - if (complete.valid && complete.ready) begin - interlock_ref_complete(model, `ILF(complete.bits, COMPLETE, TAG)); - completions[`ILF(complete.bits, COMPLETE, TAG)]++; - released++; - end - end - end - end - task automatic query(longint unsigned addr, bit write, bit permit, int cycles = 3, bit older = 0, - int size_log2 = 3); - @(negedge clock); - cpu_query.bits = '0; - `ILF(cpu_query.bits, CPU_QUERY, VALID) = 1; - `ILF(cpu_query.bits, CPU_QUERY, PADDR) = addr; - `ILF(cpu_query.bits, CPU_QUERY, SIZELOG2) = size_log2; - `ILF(cpu_query.bits, CPU_QUERY, WRITE) = write; - `ILF(cpu_query.bits, CPU_QUERY, OLDERDISPATCHPENDING) = older; - #1; - check(cpu_allow === permit, "directed CPU query incorrect or exceeded combinational timing"); - repeat (cycles) begin - @(posedge clock); - #1; - check(cpu_allow === permit, "CPU permit changed during directed query"); - end - @(negedge clock); - cpu_query.bits = '0; - endtask - task automatic reserve(int id); - @(negedge clock); - dispatch.bits = '0; - `ILF(dispatch.bits, DISPATCH, ID) = id; - dispatch.valid = 1; - do @(posedge clock); while (!dispatch.ready); - @(negedge clock); - dispatch.valid = 0; - endtask - task automatic declare_access(int id, bit memory, longint unsigned base, bytes, bit write, - bit last = 1); - @(negedge clock); - access_info.bits = '0; - `ILF(access_info.bits, ACCESS_INFO, ID) = id; - `ILF(access_info.bits, ACCESS_INFO, HASMEMORY) = memory; - `ILF(access_info.bits, ACCESS_INFO, BASE) = base; - `ILF(access_info.bits, ACCESS_INFO, BYTES) = bytes; - `ILF(access_info.bits, ACCESS_INFO, WRITE) = write; - `ILF(access_info.bits, ACCESS_INFO, LAST) = last; - access_info.valid = 1; - do @(posedge clock); while (!access_info.ready); - @(negedge clock); - access_info.valid = 0; - endtask - // External cache BFM: maintenance ACK is delayed until old request/refill work drains. - task automatic maintain(int expected_tag = -1, int expected_op = -1, int old_refill_delay = 7); - int tag, op; - logic [`INTERLOCK_MAINTENANCE_WIDTH-1:0] packet; - wait (maintenance.valid); - packet = maintenance.bits; - tag = `ILF(packet, MAINTENANCE, TAG); - op = `ILF(packet, MAINTENANCE, OP); - if (expected_tag >= 0) check(tag == expected_tag, "unexpected maintenance owner"); - if (expected_op >= 0) check(op == expected_op, "unexpected maintenance operation"); - @(negedge clock); - cpu_query.bits = '0; - `ILF(cpu_query.bits, CPU_QUERY, VALID) = 1; - `ILF(cpu_query.bits, CPU_QUERY, PADDR) = `ILF(packet, MAINTENANCE, FIRSTLINE); - `ILF(cpu_query.bits, CPU_QUERY, SIZELOG2) = 3; - `ILF(cpu_query.bits, CPU_QUERY, WRITE) = (op == 0); - repeat (5) begin - @(posedge clock); - #1; - check(maintenance.valid && maintenance.bits === packet, - "maintenance offer changed under backpressure"); - end - @(negedge clock); - maintenance.ready = 1; - @(posedge clock); - check(maintenance.valid, "maintenance request disappeared"); - @(negedge clock); - maintenance.ready = 0; - cache_pending_refills = 1; - // Independent external refill work completes later; ACK waits for that event. - fork - begin - repeat (old_refill_delay) @(posedge clock); - #1; - cache_pending_refills = 0; - end - join_none - wait (cache_pending_refills == 0); - @(negedge clock); - check(!maintained.valid, "cache ACK preceded old refill drain"); - maintained.bits = '0; - `ILF(maintained.bits, MAINTAINED, TAG) = tag; - `ILF(maintained.bits, MAINTAINED, OK) = 1; - maintained.valid = 1; - do @(posedge clock); while (!maintained.ready); - @(negedge clock); - maintained.valid = 0; - maintained.bits = '0; - cpu_query.bits = '0; - endtask - task automatic finish_dma(int id); - @(negedge clock); - done.bits = '0; - `ILF(done.bits, DONE, TAG) = id; - `ILF(done.bits, DONE, OK) = 1; - done.valid = 1; - do @(posedge clock); while (!done.ready); - @(negedge clock); - done.valid = 0; - done.bits = '0; - endtask - task automatic await_grant(int id); - wait (grants[id] == 1); - @(negedge clock); - endtask - task automatic await_complete(int id); - wait (completions[id] == 1); - @(negedge clock); - endtask - task automatic joint_reset(); - @(negedge clock); - reset = 1; - dispatch.valid = 0; - access_info.valid = 0; - maintained.valid = 0; - done.valid = 0; - maintenance.ready = 0; - grant.ready = 0; - complete.ready = 0; - cpu_query.bits = '0; - repeat (4) @(posedge clock); - @(negedge clock); - reset = 0; - endtask - initial begin - uvm_config_db#(virtual interlock_control_if)::set(null, "*", "vif", ctl); - run_test("protocol_test"); - end - initial begin - dispatch.valid = 0; - dispatch.bits = '0; - cancel.valid = 0; - cancel.bits = '0; - access_info.valid = 0; - access_info.bits = '0; - maintenance.ready = 0; - maintained.valid = 0; - maintained.bits = '0; - grant.ready = 0; - done.valid = 0; - done.bits = '0; - complete.ready = 0; - cpu_query.valid = 0; - cpu_query.ready = 0; - cpu_query.bits = '0; - wait (ctl.start); - check( - `INTERLOCK_ENTRIES==4&&`INTERLOCK_ADDRESS_BITS==44&&`INTERLOCK_ID_BITS==8&&`INTERLOCK_LINE_BYTES==64, - "unexpected declared verification profile"); - model = interlock_ref_create(`INTERLOCK_ENTRIES, `INTERLOCK_ADDRESS_BITS, `INTERLOCK_LINE_BYTES, - `INTERLOCK_MAX_RANGES); - model_ready = 1; - joint_reset(); - query('h4000, 0, 0, 4, 1); - query('h4000, 0, 1); - query(0, 1, 1); - // Same-cycle dispatch must deny even a read/read CPU query until access info is known. - @(negedge clock); - dispatch.bits = '0; - `ILF(dispatch.bits, DISPATCH, ID) = 255; - dispatch.valid = 1; - cpu_query.bits = '0; - `ILF(cpu_query.bits, CPU_QUERY, VALID) = 1; - `ILF(cpu_query.bits, CPU_QUERY, PADDR) = 'h4000; - `ILF(cpu_query.bits, CPU_QUERY, SIZELOG2) = 3; - #1; - check(!cpu_allow, "same-cycle dispatch was invisible to CPU"); - @(posedge clock); - check(dispatch.ready, "first reservation blocked"); - @(negedge clock); - dispatch.valid = 0; - cpu_query.bits = '0; - query('h9000, 0, 0, 6); - declare_access(255, 0, 0, 0, 0); - wait (complete.valid); - repeat (8) begin - @(posedge clock); - #1; - check(complete.valid && `ILF(complete.bits, COMPLETE, TAG) == 255, - "non-memory completion unstable"); - end - @(negedge clock); - complete.ready = 1; - await_complete(255); - check(grants[255] == 0, "non-memory registration was sent to DMA"); - query('h9000, 1, 1); - // Fill all register entries, show R/R parallel grants and reverse-order DMA retirement. - @(negedge clock); - complete.ready = 0; - for (int id = 1; id <= 4; id++) begin - reserve(id); - declare_access(id, 1, 'h4000, 128, 0); - end - query('h4000, 0, 1); - query('h4000, 1, 0); - query('h4080, 1, 1); - @(negedge clock); - dispatch.valid = 1; - `ILF(dispatch.bits, DISPATCH, ID) = 5; - repeat (8) begin - @(posedge clock); - #1; - check(!dispatch.ready, "full table accepted a fifth reservation"); - end - // Keep the fifth offer valid until a real slot is released and it handshakes. - // Grant offers must remain stable while the external DMA consumer stalls. - maintain(); - wait (grant.valid); - repeat (8) begin - @(posedge clock); - #1; - check(grant.valid, "grant vanished under backpressure"); - end - @(negedge clock); - grant.ready = 1; - for (int i = 0; i < 3; i++) maintain(); - for (int id = 1; id <= 4; id++) await_grant(id); - @(negedge clock); - complete.ready = 1; - finish_dma(4); - await_complete(4); - do @(posedge clock); while (!dispatch.ready); - @(negedge clock); - dispatch.valid = 0; - declare_access(5, 0, 0, 0, 0); - await_complete(5); - for (int id = 3; id >= 1; id--) begin - finish_dma(id); - await_complete(id); - end - query('h4000, 1, 1); - check(interlock_ref_live(model) == 0, "R/R retirement leaked reservations"); - // RAW: CPU read and dependent NPU read must wait for write ACK AND post-invalidate. - reserve(20); - declare_access(20, 1, 'h6003, 61, 1); - reserve(21); - declare_access(21, 1, 'h6000, 64, 0); - query('h6000, 0, 0); - query('h6040, 1, 1); - maintain(20, 1, 15); - await_grant(20); - @(negedge clock); - done.bits = '0; - `ILF(done.bits, DONE, TAG) = 20; - `ILF(done.bits, DONE, OK) = 1; - query('h6000, 0, 0, 32); - check(grants[21] == 0, "dependent RAW grant issued before final DMA ACK"); - finish_dma(20); - query('h6000, 0, 0, 8); - check(completions[20] == 0 && grants[21] == 0, "write retired before post-invalidate"); - @(negedge clock); - complete.ready = 0; - maintain(20, 2, 13); - wait (complete.valid); - query('h6000, 0, 0, 8); - check(grants[21] == 0, "RAW grant overtook stalled predecessor completion"); - @(negedge clock); - complete.ready = 1; - await_complete(20); - maintain(21, 0); - await_grant(21); - query('h6000, 0, 1); - query('h6000, 1, 0); - finish_dma(21); - await_complete(21); - query('h6000, 1, 1); - // WAR then WAW enforce age while independent ranges retain forward progress. - reserve(30); - declare_access(30, 1, 'h7000, 64, 0); - reserve(31); - declare_access(31, 1, 'h7000, 64, 1); - reserve(32); - declare_access(32, 1, 'h7000, 64, 1); - reserve(33); - declare_access(33, 1, 'h8000, 64, 0); - maintain(); - maintain(); - await_grant(30); - await_grant(33); - check(grants[31] == 0 && grants[32] == 0, "WAR/WAW bypassed older operation"); - query('h9000, 1, 1); - finish_dma(33); - await_complete(33); - finish_dma(30); - await_complete(30); - maintain(31, 1); - await_grant(31); - check(grants[32] == 0, "WAW bypassed older writer"); - finish_dma(31); - maintain(31, 2); - await_complete(31); - maintain(32, 1); - await_grant(32); - finish_dma(32); - maintain(32, 2); - await_complete(32); - // Unknown older reservation blocks a younger known nonoverlapping memory access. - reserve(40); - reserve(41); - declare_access(41, 1, 'hA000, 64, 0); - query('hB000, 0, 0, 6); - check(!maintenance.valid && grants[41] == 0, "younger access overtook unknown reservation"); - declare_access(40, 0, 0, 0, 0); - await_complete(40); - maintain(41, 0); - await_grant(41); - finish_dma(41); - await_complete(41); - // Reuse a fully retired ID at the highest legal PA line. - reserve(255); - declare_access(255, 1, (64'd1 << 44) - 64, 64, 1); - query((64'd1 << 44) - 8, 0, 0); - query((64'd1 << 44) - 72, 1, 1); - maintain(255, 1); - await_grant(255); - finish_dma(255); - maintain(255, 2); - await_complete(255); - query((64'd1 << 44) - 8, 0, 1); - // Joint reset cancels an unknown reservation and a granted DMA operation externally. - reserve(50); - declare_access(50, 1, 'hB000, 64, 0); - maintain(50, 0); - await_grant(50); - reserve(51); - query('hC000, 0, 0); - joint_reset(); - query('hB000, 1, 1); - check(interlock_ref_live(model) == 0, "joint reset did not cancel outstanding operations"); - @(negedge clock); - grant.ready = 1; - complete.ready = 1; - reserve(50); - declare_access(50, 1, 'hB000, 64, 0); - maintain(50, 0); - await_grant(50); - finish_dma(50); - await_complete(50); - // Reachable report holes: same-cycle dispatch+info holds info until its later handshake. - @(negedge clock); - dispatch.bits = '0; - access_info.bits = '0; - `ILF(dispatch.bits, DISPATCH, ID) = 254; - dispatch.valid = 1; - `ILF(access_info.bits, ACCESS_INFO, ID) = 254; - `ILF(access_info.bits, ACCESS_INFO, LAST) = 1; - access_info.valid = 1; - cpu_query.bits = '0; - `ILF(cpu_query.bits, CPU_QUERY, VALID) = 1; - `ILF(cpu_query.bits, CPU_QUERY, PADDR) = 'h4000; - `ILF(cpu_query.bits, CPU_QUERY, SIZELOG2) = 3; - #1; - check(!cpu_allow, "same-cycle info bypassed reservation hazard"); - @(posedge clock); - check(dispatch.ready && !access_info.ready, "new info must await its reservation handshake"); - @(negedge clock); - dispatch.valid = 0; - do @(posedge clock); while (!access_info.ready); - @(negedge clock); - access_info.valid = 0; - cpu_query.bits = '0; - await_complete(254); - // Each physical register slot sees complementary full-width IDs and high PA tags. - for (int slot = 0; slot < 4; slot++) begin - reserve(8'hf0 ^ slot); - declare_access(8'hf0 ^ slot, 1, (64'd1 << 44) - 64, 64, 0); - end - for (int slot = 0; slot < 4; slot++) maintain(); - for (int slot = 0; slot < 4; slot++) await_grant(8'hf0 ^ slot); - for (int slot = 3; slot >= 0; slot--) begin - finish_dma(8'hf0 ^ slot); - await_complete(8'hf0 ^ slot); - end - for (int slot = 0; slot < 4; slot++) begin - reserve(8'h0f ^ slot); - declare_access(8'h0f ^ slot, 1, 0, 64'd1 << 44, 1); - end - for (int slot = 0; slot < 4; slot++) begin - maintain(8'h0f ^ slot, 1); - await_grant(8'h0f ^ slot); - finish_dma(8'h0f ^ slot); - maintain(8'h0f ^ slot, 2); - await_complete(8'h0f ^ slot); - end - // Extent input high bits are legal: no artificial small-range fixture restriction. - for (int bit_index = 0; bit_index < 44; bit_index++) begin - reserve(170); - declare_access(170, 1, 0, 64'd1 << bit_index, 0); - maintain(170, 0, 2); - await_grant(170); - finish_dma(170); - await_complete(170); - end - // Sub-line extents are rounded to the cache-maintenance line ownership unit. - reserve(171); - declare_access(171, 1, 'h3003f, 1, 0); - query('h30000, 1, 0); - query('h30040, 1, 1); - maintain(171, 0); - await_grant(171); - finish_dma(171); - await_complete(171); - // CPU byte/half/word accesses are legal and naturally aligned, up to one line. - for (int size_log2 = 0; size_log2 <= 3; size_log2++) - for (int offset = 0; offset < 8; offset += (1 << size_log2)) - query('h20000 + offset, 1, 1, 1, 0, size_log2); - // Recycle low slots while higher older slots remain live; dependencies must not alias a new occupant. - for (int slot = 0; slot < 4; slot++) begin - reserve(100 + slot); - declare_access(100 + slot, 1, 'hD000, 64, 0); - end - for (int slot = 0; slot < 4; slot++) maintain(); - for (int slot = 0; slot < 4; slot++) await_grant(100 + slot); - for (int slot = 0; slot < 4; slot++) begin - finish_dma(100 + slot); - await_complete(100 + slot); - reserve(200 + slot); - declare_access(200 + slot, 1, 'hD000, 64, 1); - end - for (int slot = 0; slot < 4; slot++) begin - maintain(200 + slot, 1); - await_grant(200 + slot); - finish_dma(200 + slot); - maintain(200 + slot, 2); - await_complete(200 + slot); - end - // Joint synchronous reset cancels legal in-flight producer offers at its first edge. - reserve(60); - declare_access(60, 1, 'hF000, 64, 0); - maintain(60, 0); - await_grant(60); - reserve(61); - declare_access(61, 1, 'h10000, 64, 1); - wait (maintenance.valid); - @(negedge clock); - maintenance.ready = 1; - @(posedge clock); - @(negedge clock); - maintenance.ready = 0; - reserve(62); - @(negedge clock); - reset = 1; - dispatch.bits = '0; - `ILF(dispatch.bits, DISPATCH, ID) = 63; - dispatch.valid = 1; - access_info.bits = '0; - `ILF(access_info.bits, ACCESS_INFO, ID) = 62; - `ILF(access_info.bits, ACCESS_INFO, LAST) = 1; - access_info.valid = 1; - maintained.bits = '0; - `ILF(maintained.bits, MAINTAINED, TAG) = 61; - `ILF(maintained.bits, MAINTAINED, OK) = 1; - maintained.valid = 1; - done.bits = '0; - `ILF(done.bits, DONE, TAG) = 60; - `ILF(done.bits, DONE, OK) = 1; - done.valid = 1; - cpu_query.bits = '0; - `ILF(cpu_query.bits, CPU_QUERY, VALID) = 1; - `ILF(cpu_query.bits, CPU_QUERY, PADDR) = 'hF000; - `ILF(cpu_query.bits, CPU_QUERY, SIZELOG2) = 3; - @(posedge clock); - @(negedge clock); - dispatch.valid = 0; - access_info.valid = 0; - maintained.valid = 0; - done.valid = 0; - cpu_query.bits = '0; - repeat (3) @(posedge clock); - @(negedge clock); - reset = 0; - query(0, 1, 1); - check(interlock_ref_live(model) == 0, "joint reset left producer-offer reservations live"); - // Fast external endpoints may offer ACK in the same cycle as request/grant acceptance. - @(negedge clock); - grant.ready = 0; - reserve(0); - declare_access(0, 1, 'hE000, 64, 0); - wait (maintenance.valid); - @(negedge clock); - maintenance.ready = 1; - maintained.bits = '0; - `ILF(maintained.bits, MAINTAINED, TAG) = 0; - `ILF(maintained.bits, MAINTAINED, OK) = 1; - maintained.valid = 1; - @(posedge clock); - check(maintenance.valid && !maintained.ready, - "fast cache response did not wait for accepted request"); - @(negedge clock); - maintenance.ready = 0; - do @(posedge clock); while (!maintained.ready); - @(negedge clock); - maintained.valid = 0; - maintained.bits = '0; - wait (grant.valid); - @(negedge clock); - grant.ready = 1; - done.bits = '0; - `ILF(done.bits, DONE, TAG) = 0; - `ILF(done.bits, DONE, OK) = 1; - done.valid = 1; - @(posedge clock); - check(grant.valid && !done.ready, "early DMA completion did not wait for accepted grant"); - @(negedge clock); - grant.ready = 0; - do @(posedge clock); while (!done.ready); - @(negedge clock); - done.valid = 0; - done.bits = '0; - grant.ready = 1; - await_complete(0); - // An unsealed eight-extent list blocks all CPU queries; sealing exposes exact holes. - reserve(80); - for (int extent = 0; extent < 8; extent++) begin - declare_access(80, 1, 'h200000 + extent * 128, 64, 1, extent == 7); - if (extent != 7) begin - query('h300000, 0, 0, 1); - check(!maintenance.valid && grants[80] == 0, - "partial unsealed list reached maintenance/grant"); - end - end - query('h200040, 1, 1); - query('h200000, 0, 0); - for (int extent = 0; extent < 8; extent++) begin - maintain(80, 1, 2); - check(grants[80] == 0 || extent == 7, "grant before every pre-maintenance ACK"); - end - await_grant(80); - finish_dma(80); - for (int extent = 0; extent < 8; extent++) begin - maintain(80, 2, 2); - if (extent != 7) check(completions[80] == 0, "complete before every post-invalidate ACK"); - end - await_complete(80); - query('h200000, 1, 1); - check(maintenance_ops[80][1] == 8 && maintenance_ops[80][2] == 8, - "eight extents were not all maintained"); - // Preflight failure cancels unknown/partial reservations without producing a fake completion. - reserve(81); - query('h300000, 0, 0, 1); - @(negedge clock); - cancel.bits = '0; - `ILF(cancel.bits, CANCEL, ID) = 81; - cancel.valid = 1; - do @(posedge clock); while (!cancel.ready); - @(negedge clock); - cancel.valid = 0; - query('h300000, 0, 1, 1); - check(completions[81] == 0 && grants[81] == 0, "empty cancel became grant/complete"); - reserve(82); - declare_access(82, 1, 'h400000, 64, 0, 0); - query('h500000, 0, 0, 1); - @(negedge clock); - cancel.bits = '0; - `ILF(cancel.bits, CANCEL, ID) = 82; - cancel.valid = 1; - do @(posedge clock); while (!cancel.ready); - @(negedge clock); - cancel.valid = 0; - query('h500000, 0, 1, 1); - check(completions[82] == 0 && grants[82] == 0, "partial cancel became grant/complete"); - reserve(82); - declare_access(82, 0, 0, 0, 0); - await_complete(82); - // Inactive payloads are unconstrained by valid: exercise arithmetic carry without an illegal handshake. - @(negedge clock); - access_info.valid = 0; - access_info.bits = '1; - cpu_query.bits = '0; - `ILF(cpu_query.bits, CPU_QUERY, SIZELOG2) = 4; - repeat (2) begin - @(posedge clock); - #1; - check(cpu_allow, "inactive CPU query affected permit"); - end - @(negedge clock); - access_info.bits = '0; - cpu_query.bits = '0; - repeat (2) @(posedge clock); - check(interlock_ref_live(model) == 0, "normal test left unmatched transactions"); - `uvm_info("CHECKED", $sformatf( - "queries=%0d reserved=%0d dmaACK=%0d completions=%0d", - checked_queries, - reserved, - acknowledged, - released - ), UVM_LOW) - interlock_ref_destroy(model); - model_ready = 0; - ctl.done = 1; - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/negative_tb.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/negative_tb.svh deleted file mode 100644 index b5a74c3c4..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/negative_tb.svh +++ /dev/null @@ -1,136 +0,0 @@ -module `INTERLOCK_NEGATIVE_TB; - import uvm_pkg::*; - import interlock_pkg::*; - `include "uvm_macros.svh" - `include "interlock_config.svh" - `define ILF(B, K, F) B[`INTERLOCK_``K``_``F``_OFFSET +: `INTERLOCK_``K``_``F``_WIDTH] - logic clock = 0, reset = 1, cpu_allow; - always #5 clock = ~clock; - interlock_control_if ctl (clock); - stream_if #( - .WIDTH(`INTERLOCK_DISPATCH_WIDTH) - ) dispatch ( - clock, - reset - ); - stream_if #( - .WIDTH(`INTERLOCK_CANCEL_WIDTH) - ) cancel ( - clock, - reset - ); - stream_if #( - .WIDTH(`INTERLOCK_ACCESS_INFO_WIDTH) - ) access_info ( - clock, - reset - ); - stream_if #( - .WIDTH(`INTERLOCK_MAINTENANCE_WIDTH) - ) maintenance ( - clock, - reset - ); - stream_if #( - .WIDTH(`INTERLOCK_MAINTAINED_WIDTH) - ) maintained ( - clock, - reset - ); - stream_if #( - .WIDTH(`INTERLOCK_GRANT_WIDTH) - ) grant ( - clock, - reset - ); - stream_if #( - .WIDTH(`INTERLOCK_DONE_WIDTH) - ) done ( - clock, - reset - ); - stream_if #( - .WIDTH(`INTERLOCK_COMPLETE_WIDTH) - ) complete ( - clock, - reset - ); - stream_if #( - .WIDTH(`INTERLOCK_CPU_QUERY_WIDTH) - ) cpu_query ( - clock, - reset - ); - Interlock dut ( - .clock(clock), - .reset(reset), - `include "interlock_ports.svh" - ); - task automatic reserve(int id); - @(negedge clock); - dispatch.bits = '0; - `ILF(dispatch.bits, DISPATCH, ID) = id; - dispatch.valid = 1; - do @(posedge clock); while (!dispatch.ready); - @(negedge clock); - dispatch.valid = 0; - endtask - task automatic info(int id, bit memory, longint unsigned base, bytes, bit write, bit last = 1); - @(negedge clock); - access_info.bits = '0; - `ILF(access_info.bits, ACCESS_INFO, ID) = id; - `ILF(access_info.bits, ACCESS_INFO, HASMEMORY) = memory; - `ILF(access_info.bits, ACCESS_INFO, BASE) = base; - `ILF(access_info.bits, ACCESS_INFO, BYTES) = bytes; - `ILF(access_info.bits, ACCESS_INFO, WRITE) = write; - `ILF(access_info.bits, ACCESS_INFO, LAST) = last; - access_info.valid = 1; - do @(posedge clock); while (!access_info.ready); - @(negedge clock); - access_info.valid = 0; - endtask - task automatic maintenance_accept(); - wait (maintenance.valid); - @(negedge clock); - maintenance.ready = 1; - @(posedge clock); - @(negedge clock); - maintenance.ready = 0; - endtask - task automatic maintenance_ack(int id, bit ok); - @(negedge clock); - maintained.bits = '0; - `ILF(maintained.bits, MAINTAINED, TAG) = id; - `ILF(maintained.bits, MAINTAINED, OK) = ok; - maintained.valid = 1; - endtask - initial begin - uvm_config_db#(virtual interlock_control_if)::set(null, "*", "vif", ctl); - run_test("protocol_test"); - end - initial begin - dispatch.valid = 0; - dispatch.bits = '0; - cancel.valid = 0; - cancel.bits = '0; - access_info.valid = 0; - access_info.bits = '0; - maintenance.ready = 0; - maintained.valid = 0; - maintained.bits = '0; - grant.ready = 0; - done.valid = 0; - done.bits = '0; - complete.ready = 0; - cpu_query.bits = '0; - cpu_query.valid = 0; - cpu_query.ready = 0; - wait (ctl.start); - repeat (4) @(posedge clock); - @(negedge clock); - reset = 0; - `include `INTERLOCK_NEGATIVE_SCENARIO - repeat (20) @(posedge clock); - `uvm_fatal("MISSING_ASSERTION", "illegal input did not produce the declared RTL assertion") - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/package.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/package.sv deleted file mode 100644 index 57425508c..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/package.sv +++ /dev/null @@ -1,84 +0,0 @@ -package interlock_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - import "DPI-C" function chandle interlock_ref_create( - input int unsigned entries, - address_bits, - line_bytes, - max_ranges - ); - import "DPI-C" function void interlock_ref_destroy(input chandle model); - import "DPI-C" function void interlock_ref_reset(input chandle model); - import "DPI-C" function int unsigned interlock_ref_live(input chandle model); - import "DPI-C" function void interlock_ref_reserve( - input chandle model, - input int unsigned id - ); - import "DPI-C" function void interlock_ref_info( - input chandle model, - input int unsigned id, - input byte unsigned has_mem, - input longint unsigned base, - bytes, - input byte unsigned write, - last - ); - import "DPI-C" function byte unsigned interlock_ref_cpu_allow( - input chandle model, - input longint unsigned addr, - input int unsigned bytes, - input byte unsigned write, - older_pending, - reserve_now - ); - import "DPI-C" function void interlock_ref_maint_offer( - input chandle model, - input int unsigned id, - op, - input longint unsigned first_line, - last_line - ); - import "DPI-C" function void interlock_ref_maint_ack( - input chandle model, - input int unsigned id - ); - import "DPI-C" function void interlock_ref_grant( - input chandle model, - input int unsigned id - ); - import "DPI-C" function void interlock_ref_dma_ack( - input chandle model, - input int unsigned id - ); - import "DPI-C" function void interlock_ref_complete( - input chandle model, - input int unsigned id - ); - import "DPI-C" function void interlock_ref_cancel( - input chandle model, - input int unsigned id - ); - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual interlock_control_if vif; - function new(string name, uvm_component parent); - super.new(name, parent); - timeout = 500us; - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual interlock_control_if)::get(this, "", "vif", vif)) - `uvm_fatal("VIF", "missing control interface") - endfunction - task execute(); - @(negedge vif.clock); - vif.start = 1; - wait (vif.done); - `uvm_info( - "INTERLOCK_PASS", - "Checked live reservations, CPU permits, maintenance/grant/ACK/completion handshakes with independent transaction oracle", - UVM_LOW) - endtask - endclass -endpackage diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/scala/Emit.scala b/arch/src/main/scala/framework/mem-core/interlock/src/main/scala/Emit.scala deleted file mode 100644 index e617b117a..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/scala/Emit.scala +++ /dev/null @@ -1,60 +0,0 @@ -package memcore.memory.interlock - -import chisel3._ -import java.nio.file.{Files, Paths} - -object Emit extends App { - val p = Params() - Files.createDirectories(Paths.get("build")) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Interlock(p), - firtoolOpts = args, - args = Array("--target-dir", "build/Interlock", "--split-verilog") - ) - - val definitions = scala.collection.mutable.ArrayBuffer[String]( - s"`define INTERLOCK_ENTRIES ${p.entries}", - s"`define INTERLOCK_ADDRESS_BITS ${p.addressBits}", - s"`define INTERLOCK_ID_BITS ${p.idBits}", - s"`define INTERLOCK_LINE_BYTES ${p.lineBytes}", - s"`define INTERLOCK_MAX_RANGES ${p.maxRanges}" - ) - - val ports = scala.collection.mutable.ArrayBuffer[String]() - - def channel( - kind: String, - prefix: String, - signal: String, - data: Bundle, - decoupled: Boolean = true - ): Unit = { - var offset = 0 - for ((name, field) <- data.elements.toSeq) { - val width = field.getWidth - definitions += s"`define INTERLOCK_${kind}_${name.toUpperCase}_OFFSET $offset" - definitions += s"`define INTERLOCK_${kind}_${name.toUpperCase}_WIDTH $width" - ports += s".$prefix$name($signal.bits[$offset +: $width])" - offset += width - } - definitions += s"`define INTERLOCK_${kind}_WIDTH $offset" - if (decoupled) { - val channelPrefix = prefix.stripSuffix("bits_") - ports += s".${channelPrefix}valid($signal.valid)" - ports += s".${channelPrefix}ready($signal.ready)" - } - } - - channel("DISPATCH", "io_dispatch_bits_", "dispatch", new Dispatch(p)) - channel("CANCEL", "io_cancel_bits_", "cancel", new Dispatch(p)) - channel("ACCESS_INFO", "io_accessInfo_bits_", "access_info", new AccessInfo(p)) - channel("MAINTENANCE", "io_maintenance_bits_", "maintenance", new Maintenance(p)) - channel("MAINTAINED", "io_maintained_bits_", "maintained", new Acknowledgement(p)) - channel("GRANT", "io_grant_bits_", "grant", new Tag(p)) - channel("DONE", "io_done_bits_", "done", new Acknowledgement(p)) - channel("COMPLETE", "io_complete_bits_", "complete", new Tag(p)) - channel("CPU_QUERY", "io_cpuQuery_", "cpu_query", new CpuQuery(p), decoupled = false) - ports += ".io_cpuAllow(cpu_allow)" - Files.write(Paths.get("build/interlock_config.svh"), (definitions.mkString("\n") + "\n").getBytes) - Files.write(Paths.get("build/interlock_ports.svh"), ports.mkString(",\n").getBytes) -} diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/scala/Interface.scala b/arch/src/main/scala/framework/mem-core/interlock/src/main/scala/Interface.scala deleted file mode 100644 index c4aaa94d7..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/scala/Interface.scala +++ /dev/null @@ -1,38 +0,0 @@ -package memcore.memory.interlock - -import chisel3._ - -class Dispatch(p: Params) extends Bundle { val id = UInt(p.idBits.W) } - -class AccessInfo(p: Params) extends Bundle { - val id = UInt(p.idBits.W) - val hasMemory = Bool() - val base = UInt(p.addressBits.W) - val bytes = UInt((p.addressBits + 1).W) - val write = Bool() - val last = Bool() -} - -class CpuQuery(p: Params) extends Bundle { - val valid = Bool() - val paddr = UInt(p.addressBits.W) - val sizeLog2 = UInt(3.W) - val write = Bool() - val olderDispatchPending = Bool() -} - -object MaintenanceOp { val Clean = 0; val CleanInvalidate = 1; val Invalidate = 2 } - -class Maintenance(p: Params) extends Bundle { - val tag = UInt(p.idBits.W) - val firstLine = UInt(p.addressBits.W) - val lastLine = UInt(p.addressBits.W) - val op = UInt(2.W) -} - -class Tag(p: Params) extends Bundle { val tag = UInt(p.idBits.W) } - -class Acknowledgement(p: Params) extends Bundle { - val tag = UInt(p.idBits.W) - val ok = Bool() -} diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/scala/Params.scala b/arch/src/main/scala/framework/mem-core/interlock/src/main/scala/Params.scala deleted file mode 100644 index 8c2c69735..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/scala/Params.scala +++ /dev/null @@ -1,17 +0,0 @@ -package memcore.memory.interlock - -case class Params( - entries: Int = 4, - addressBits: Int = 44, - idBits: Int = 8, - lineBytes: Int = 64, - maxRanges: Int = 8) { - require(entries >= 2 && (entries & (entries - 1)) == 0) - require(addressBits >= 4 && idBits >= 1) - require(lineBytes >= 8 && (lineBytes & (lineBytes - 1)) == 0) - require(maxRanges >= 1 && (maxRanges & (maxRanges - 1)) == 0) - val rangeIndexBits: Int = math.max(1, Integer.numberOfTrailingZeros(maxRanges)) - val rangeCountBits: Int = Integer.SIZE - Integer.numberOfLeadingZeros(maxRanges) - val lineBits: Int = Integer.numberOfTrailingZeros(lineBytes) - require(addressBits > lineBits) -} diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/build.sc b/arch/src/main/scala/framework/mem-core/mesh_shm/build.sc deleted file mode 100644 index 9b0de30f9..000000000 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/build.sc +++ /dev/null @@ -1,28 +0,0 @@ -import mill._ -import mill.scalalib._ - -trait ChiselModule extends SbtModule { - def moduleRoot: os.Path - - override def millSourcePath = moduleRoot - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") -} - -object axis extends ChiselModule { - override def moduleRoot = os.pwd / os.up / "axis" -} - -object bank extends ChiselModule { - override def moduleRoot = os.pwd / os.up / "bank" - override def moduleDeps = Seq(axis) -} - -object mesh_shm extends ChiselModule { - override def moduleRoot = os.pwd - override def moduleDeps = Seq(bank) - override def mainClass = Some("memcore.memory.mesh_shm.Emit") - -} diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/router.f b/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/router.f deleted file mode 100644 index 935fe9d7d..000000000 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/router.f +++ /dev/null @@ -1,7 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/protocol/axis/interface.sv -+incdir+@RESOURCES@ -@RESOURCES@/router_package.sv --F @RTL@/MeshRouter/filelist.f -@RESOURCES@/router_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/router_package.sv b/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/router_package.sv deleted file mode 100644 index 4a91ab97c..000000000 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/router_package.sv +++ /dev/null @@ -1,13 +0,0 @@ -package router_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - typedef virtual axis_if #(128, 8, 4, 45) router_vif; - import "DPI-C" function int unsigned mesh_route( - input int unsigned destination, - input int unsigned row, - input int unsigned col - ); - `include "router_item.svh" - `include "router_test.svh" -endpackage diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/shared.f b/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/shared.f deleted file mode 100644 index 52096c242..000000000 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/shared.f +++ /dev/null @@ -1,8 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/protocol/axis/interface.sv -+incdir+@RESOURCES@ -@RESOURCES@/transfer_if.sv -@RESOURCES@/shared_package.sv --F @RTL@/MeshSharedMem/filelist.f -@RESOURCES@/shared_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/Emit.scala b/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/Emit.scala deleted file mode 100644 index 8102772ae..000000000 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/Emit.scala +++ /dev/null @@ -1,15 +0,0 @@ -package memcore.memory.mesh_shm - -object Emit extends App { - val p = MeshSharedMemParams.prototype - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new MeshRouter(p, 1, 1), - firtoolOpts = args, - args = Array("--target-dir", "build/MeshRouter", "--split-verilog") - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new MeshSharedMem(p), - firtoolOpts = args, - args = Array("--target-dir", "build/MeshSharedMem", "--split-verilog") - ) -} diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshChannel.scala b/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshChannel.scala deleted file mode 100644 index 10b55ccbe..000000000 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshChannel.scala +++ /dev/null @@ -1,10 +0,0 @@ -package memcore.memory.mesh_shm - -import chisel3._ -import chisel3.util._ - -/** Core-facing shared-bank port. Non-conflicting requests may be outstanding together. */ -class MeshChannel(p: MeshSharedMemParams) extends Bundle { - val request = Flipped(Decoupled(new MeshClientRequest(p))) - val response = Decoupled(new MeshClientResponse(p)) -} diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshClientEndpoint.scala b/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshClientEndpoint.scala deleted file mode 100644 index 18279b39c..000000000 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshClientEndpoint.scala +++ /dev/null @@ -1,110 +0,0 @@ -package memcore.memory.mesh_shm - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} - -/** Converts one client operation class into packets injected at one Mesh node. */ -@instantiable -class MeshClientEndpoint( - p: MeshSharedMemParams, - row: Int, - col: Int, - channel: Int, - acceptedWrite: Option[Boolean]) - extends Module { - require(row >= 0 && row < p.rows && col >= 0 && col < p.cols) - require(channel >= 0 && channel < p.networkClientChannels) - - @public - val io = IO(new Bundle { - val client = new MeshChannel(p) - val inFlight = Output(UInt(log2Ceil(p.maxInFlight + 1).W)) - val tags = Output(UInt((1 << p.tagBits).W)) - - val mesh = new Bundle { - val request = Decoupled(new MeshPacket(p)) - val response = Flipped(Decoupled(new MeshPacket(p))) - } - - }) - - val inFlight = RegInit(0.U(log2Ceil(p.maxInFlight + 1).W)) - val tags = RegInit(0.U((1 << p.tagBits).W)) - val errorPending = RegInit(false.B) - val errorTag = Reg(UInt(p.tagBits.W)) - val errorWrite = Reg(Bool()) - val request = io.client.request - val response = io.client.response - val available = inFlight < p.maxInFlight.U && !tags(request.bits.tag) - val bankValid = request.bits.bank < p.visibleBankCount.U - val bankRows = VecInit((0 until p.bankCount).map(i => (i / p.cols).U(p.rowBits.W))) - val bankCols = VecInit((0 until p.bankCount).map(i => (i % p.cols).U(p.colBits.W))) - io.inFlight := inFlight - io.tags := tags - - acceptedWrite.foreach { write => - when(request.valid) { - assert(request.bits.write === write.B, "Mesh client operation used the wrong directional endpoint") - } - } - - io.mesh.request.valid := request.valid && available && !errorPending && bankValid - io.mesh.request.bits := 0.U.asTypeOf(new MeshPacket(p)) - io.mesh.request.bits.tdest := Cat(bankRows(request.bits.bank), bankCols(request.bits.bank)) - io.mesh.request.bits.tdata := request.bits.data - io.mesh.request.bits.tkeep := request.bits.mask - io.mesh.request.bits.tid := request.bits.tag - io.mesh.request.bits.tlast := true.B - io.mesh.request.bits.tuser := Cat( - row.U(p.rowBits.W), - col.U(p.colBits.W), - channel.U(p.channelBits.W), - request.bits.addr, - request.bits.write, - false.B - ) - request.ready := available && !errorPending && Mux(bankValid, io.mesh.request.ready, true.B) - when(request.fire && !bankValid) { - errorPending := true.B - errorTag := request.bits.tag - errorWrite := request.bits.write - } - - val responseValid = RegInit(false.B) - val responseBits = Reg(new MeshClientResponse(p)) - val responseNext = Wire(Decoupled(new MeshClientResponse(p))) - response.valid := responseValid - response.bits := responseBits - responseNext.ready := !responseValid || response.ready - when(responseNext.ready) { - responseValid := responseNext.valid - when(responseNext.valid)(responseBits := responseNext.bits) - } - responseNext.valid := errorPending || io.mesh.response.valid - responseNext.bits := 0.U.asTypeOf(responseNext.bits) - responseNext.bits.data := Mux(errorPending, 0.U, io.mesh.response.bits.data) - responseNext.bits.tag := Mux(errorPending, errorTag, io.mesh.response.bits.tag) - responseNext.bits.tkeep := Fill(p.maskBits, 1.U(1.W)) - responseNext.bits.tlast := true.B - responseNext.bits.tuser := Cat( - Mux(errorPending, errorWrite, io.mesh.response.bits.write), - errorPending || io.mesh.response.bits.error - ) - io.mesh.response.ready := responseNext.ready && !errorPending - - when(io.mesh.response.valid) { - assert(io.mesh.response.bits.channel === channel.U) - assert(inFlight =/= 0.U && tags(io.mesh.response.bits.tag)) - } - when(response.fire) { - assert(inFlight =/= 0.U && tags(response.bits.tag)) - } - when(responseNext.fire && errorPending)(errorPending := false.B) - val issued = Mux(request.fire, 1.U((1 << p.tagBits).W) << request.bits.tag, 0.U) - val completed = Mux(response.fire, 1.U((1 << p.tagBits).W) << response.bits.tag, 0.U) - tags := (tags | issued) & ~completed - when(request.fire =/= response.fire) { - inFlight := Mux(request.fire, inFlight + 1.U, inFlight - 1.U) - } -} diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshControlInterface.scala b/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshControlInterface.scala deleted file mode 100644 index 045f60001..000000000 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshControlInterface.scala +++ /dev/null @@ -1,53 +0,0 @@ -package memcore.memory.mesh_shm - -import chisel3._ -import chisel3.util._ - -/** Contiguous rows from one Core-local Bank to another Core-local Bank. */ -class MeshTransferCommand(p: MeshSharedMemParams) extends Bundle { - val sourceCore = UInt(8.W) - val sourceBank = UInt(p.localBankBits.W) - val sourceAddr = UInt(16.W) - val targetCore = UInt(8.W) - val targetBank = UInt(p.localBankBits.W) - val targetAddr = UInt(16.W) - val rows = UInt(17.W) - val tag = UInt(p.tagBits.W) -} - -class MeshTransferCompletion(p: MeshSharedMemParams) extends Bundle { - val tag = UInt(p.tagBits.W) - val error = Bool() -} - -class MeshLocalBankRequest( - addressBits: Int, - bankBits: Int, - dataBits: Int, - tagBits: Int) - extends Bundle { - val addr = UInt(addressBits.W) - val write = Bool() - val data = UInt(dataBits.W) - val mask = UInt((dataBits / 8).W) - val tag = UInt(tagBits.W) - val bank = UInt(bankBits.W) -} - -class MeshLocalBankResponse(dataBits: Int, tagBits: Int) extends Bundle { - val data = UInt(dataBits.W) - val tag = UInt(tagBits.W) - val error = Bool() -} - -/** SharedMem initiates both the source read and target write. */ -class MeshLocalBankPort( - addressBits: Int, - bankBits: Int, - dataBits: Int, - tagBits: Int) - extends Bundle { - - val request = Decoupled(new MeshLocalBankRequest(addressBits, bankBits, dataBits, tagBits)) - val response = Flipped(Decoupled(new MeshLocalBankResponse(dataBits, tagBits))) -} diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshCoreEndpoint.scala b/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshCoreEndpoint.scala deleted file mode 100644 index 523dff63f..000000000 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshCoreEndpoint.scala +++ /dev/null @@ -1,59 +0,0 @@ -package memcore.memory.mesh_shm - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} - -/** The private Bank port stays beside its Core; only packets cross the Mesh. */ -@instantiable -class MeshCoreEndpoint( - p: MeshSharedMemParams, - core: Int, - row: Int, - col: Int) - extends Module { - - @public - val io = IO(new Bundle { - val request = Flipped(Decoupled(new MeshPacket(p))) - val response = Decoupled(new MeshPacket(p)) - val bank = new MeshLocalBankPort(p.addressBits, p.localBankBits, p.dataBits, p.tagBits) - }) - - val pending = RegInit(false.B) - val request = Reg(new MeshPacket(p)) - io.bank.request.valid := io.request.valid && !pending - io.bank.request.bits.bank := io.request.bits.localBank - io.bank.request.bits.addr := io.request.bits.addr - io.bank.request.bits.write := io.request.bits.write - io.bank.request.bits.data := io.request.bits.data - io.bank.request.bits.mask := io.request.bits.mask - io.bank.request.bits.tag := io.request.bits.tag - io.request.ready := io.bank.request.ready && !pending - when(io.request.fire) { - assert(io.request.bits.privateRequest && io.request.bits.core === core.U) - assert(io.request.bits.destRow === row.U && io.request.bits.destCol === col.U) - request := io.request.bits - pending := true.B - } - io.response.valid := pending && io.bank.response.valid - io.response.bits := request - io.response.bits.tdest := Cat(request.sourceRow, request.sourceCol) - io.response.bits.tdata := Mux(request.write, 0.U, io.bank.response.bits.data) - io.response.bits.tuser := Cat( - request.core, - request.localBank, - true.B, - row.U(p.rowBits.W), - col.U(p.colBits.W), - request.channel, - request.addr, - request.write, - io.bank.response.bits.error - ) - io.bank.response.ready := pending && io.response.ready - when(io.bank.response.valid) { - assert(pending && io.bank.response.bits.tag === request.tag) - } - when(io.response.fire)(pending := false.B) -} diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshSharedMem.scala b/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshSharedMem.scala deleted file mode 100644 index b5148c73c..000000000 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshSharedMem.scala +++ /dev/null @@ -1,246 +0,0 @@ -package memcore.memory.mesh_shm - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} - -@instantiable -class MeshSharedMem(p: MeshSharedMemParams) extends Module { - - @public - val io = IO(new Bundle { - val channels = Vec(p.totalChannels, new MeshChannel(p)) - val external = Vec(p.externalChannels, new MeshChannel(p)) - val transferCommand = Flipped(Decoupled(new MeshTransferCommand(p))) - val transferCompletion = Decoupled(new MeshTransferCompletion(p)) - val localBanks = Vec(p.cores.size, new MeshLocalBankPort(p.addressBits, p.localBankBits, p.dataBits, p.tagBits)) - val bankWrites = Vec(p.bankCount, Decoupled(new MeshClientRequest(p))) - }) - - val transfer: Instance[MeshTransferController] = Instantiate(new MeshTransferController(p)) - transfer.io.command.valid := io.transferCommand.valid - transfer.io.command.bits := io.transferCommand.bits - io.transferCommand.ready := transfer.io.command.ready - io.transferCompletion.valid := transfer.io.completion.valid - io.transferCompletion.bits := transfer.io.completion.bits - transfer.io.completion.ready := io.transferCompletion.ready - - val ordering = Instantiate(new MeshChannelOrdering(p, p.totalChannels + p.externalChannels)) - for (channel <- 0 until p.totalChannels) { - ordering.io.client(channel).request.valid := io.channels(channel).request.valid - ordering.io.client(channel).request.bits := io.channels(channel).request.bits - io.channels(channel).request.ready := ordering.io.client(channel).request.ready - io.channels(channel).response <> ordering.io.client(channel).response - } - for (channel <- 0 until p.externalChannels) { - val index = p.totalChannels + channel - ordering.io.client(index).request.valid := io.external(channel).request.valid - ordering.io.client(index).request.bits := io.external(channel).request.bits - io.external(channel).request.ready := ordering.io.client(index).request.ready - io.external(channel).response <> ordering.io.client(index).response - } - - val directionalEndpoints = p.inputChannelLocations.zip(p.outputChannelLocations).zipWithIndex.flatMap { - case ((inputLocation, outputLocation), channel) => - val read = Instantiate(new MeshClientEndpoint( - p, - inputLocation._1, - inputLocation._2, - 2 * channel, - Some(false) - )) - val write = Instantiate(new MeshClientEndpoint( - p, - outputLocation._1, - outputLocation._2, - 2 * channel + 1, - Some(true) - )) - val clientRequest = ordering.io.ordered(channel).request - val available = (read.io.inFlight +& write.io.inFlight) < p.maxInFlight.U && - !read.io.tags(clientRequest.bits.tag) && !write.io.tags(clientRequest.bits.tag) - read.io.client.request.valid := clientRequest.valid && available && !clientRequest.bits.write - read.io.client.request.bits := clientRequest.bits - write.io.client.request.valid := clientRequest.valid && available && clientRequest.bits.write - write.io.client.request.bits := clientRequest.bits - clientRequest.ready := available && Mux( - clientRequest.bits.write, - write.io.client.request.ready, - read.io.client.request.ready - ) - - val responses = Module(new RRArbiter(new MeshClientResponse(p), 2)) - responses.io.in(0) <> read.io.client.response - responses.io.in(1) <> write.io.client.response - ordering.io.ordered(channel).response <> responses.io.out - Seq( - (inputLocation._1, inputLocation._2, 2 * channel, read), - (outputLocation._1, outputLocation._2, 2 * channel + 1, write) - ) - } - - val externalEndpoints = (0 until p.externalChannels).map { index => - val channel = 2 * p.totalChannels + index - val orderingChannel = p.totalChannels + index - val endpoint = Instantiate(new MeshClientEndpoint(p, 0, 0, channel, None)) - endpoint.io.client <> ordering.io.ordered(orderingChannel) - (0, 0, channel, endpoint) - } - - val clientEndpoints = directionalEndpoints ++ externalEndpoints - - for (core <- p.cores.indices if p.cores(core).localBankNode.isEmpty) { - io.localBanks(core).request.valid := false.B - io.localBanks(core).request.bits := 0.U.asTypeOf(io.localBanks(core).request.bits) - io.localBanks(core).response.ready := false.B - } - - val requests: Seq[Seq[Instance[MeshCreditRouter]]] = - Seq.tabulate(p.rows, p.cols)((row, col) => Instantiate(new MeshCreditRouter(p, row, col))) - val responses: Seq[Seq[Instance[MeshCreditRouter]]] = - Seq.tabulate(p.rows, p.cols)((row, col) => Instantiate(new MeshCreditRouter(p, row, col))) - val banks: Seq[Seq[Instance[MeshBankNode]]] = - Seq.tabulate(p.rows, p.cols)((row, col) => Instantiate(new MeshBankNode(p, row, col))) - for { - row <- 0 until p.rows - col <- 0 until p.cols - } { - val requestRouter = requests(row)(col) - val responseRouter = responses(row)(col) - val bank = banks(row)(col) - val bankIndex = row * p.cols + col - val requestEgress = Instantiate(new MeshCreditToDecoupled(p)) - requestEgress.io.in <> requestRouter.io.out(MeshDirection.endpoint) - val localRequest = requestEgress.io.out - io.bankWrites( - bankIndex - ).valid := localRequest.valid && !localRequest.bits.privateRequest && localRequest.bits.write && bank.io.request.ready - io.bankWrites(bankIndex).bits.bank := bankIndex.U - io.bankWrites(bankIndex).bits.tuser := Cat(bank.io.request.bits.addr, bank.io.request.bits.write) - io.bankWrites(bankIndex).bits.tlast := true.B - io.bankWrites(bankIndex).bits.data := bank.io.request.bits.data - io.bankWrites(bankIndex).bits.mask := bank.io.request.bits.mask - io.bankWrites(bankIndex).bits.tag := bank.io.request.bits.tag - bank.io.request.valid := localRequest.valid && !localRequest.bits.privateRequest && - (!localRequest.bits.write || io.bankWrites(bankIndex).ready) - bank.io.request.bits := localRequest.bits - val privateCores = - p.cores.indices.filter(core => p.localBankLocations(core).contains((row, col))) - val privateReady = Wire(Vec(privateCores.size, Bool())) - val replies = Module(new RRArbiter(new MeshPacket(p), privateCores.size + 1) { - override lazy val lastGrant = { - val pointer = RegInit(0.U(math.max(1, log2Ceil(privateCores.size + 1)).W)) - when(io.out.fire)(pointer := io.chosen) - pointer - } - }) - replies.io.in(0) <> bank.io.response - for ((core, index) <- privateCores.zipWithIndex) { - val endpoint: Instance[MeshCoreEndpoint] = Instantiate(new MeshCoreEndpoint(p, core, row, col)) - endpoint.io.request.valid := localRequest.valid && localRequest.bits.privateRequest && localRequest.bits.core === core.U - endpoint.io.request.bits := localRequest.bits - privateReady(index) := localRequest.bits.core === core.U && endpoint.io.request.ready - io.localBanks(core) <> endpoint.io.bank - replies.io.in(index + 1) <> endpoint.io.response - } - localRequest.ready := Mux( - localRequest.bits.privateRequest, - (if (privateCores.nonEmpty) privateReady.asUInt.orR else false.B), - bank.io.request.ready && (!localRequest.bits.write || io.bankWrites(bankIndex).ready) - ) - when(localRequest.valid && localRequest.bits.privateRequest) { - assert( - privateCores.map(core => localRequest.bits.core === core.U).reduceOption(_ || _).getOrElse(false.B), - "private request reached a node without its Core" - ) - } - val responseIngress = Instantiate(new MeshDecoupledToCredit(p)) - responseIngress.io.in <> replies.io.out - responseRouter.io.in(MeshDirection.endpoint) <> responseIngress.io.out - - val attached = clientEndpoints.filter { case (r, c, _, _) => r == row && c == col } - val transferAttached = row == 0 && col == 0 - val attachmentCount = attached.size + (if (transferAttached) 1 else 0) - val requestIngress = Instantiate(new MeshDecoupledToCredit(p)) - requestRouter.io.in(MeshDirection.endpoint) <> requestIngress.io.out - val responseEgress = Instantiate(new MeshCreditToDecoupled(p)) - responseEgress.io.in <> responseRouter.io.out(MeshDirection.endpoint) - val network = responseEgress.io.out - if (attachmentCount > 0) { - val arbiter = Module(new RRArbiter(new MeshPacket(p), attachmentCount) { - override lazy val lastGrant = { - val pointer = RegInit(0.U(math.max(1, log2Ceil(attachmentCount)).W)) - when(io.out.fire)(pointer := io.chosen) - pointer - } - }) - val localResponseReady = Wire(Vec(attachmentCount, Bool())) - for (((_, _, channel, endpoint), port) <- attached.zipWithIndex) { - arbiter.io.in(port) <> endpoint.io.mesh.request - endpoint.io.mesh.response.valid := network.valid && network.bits.channel === channel.U - endpoint.io.mesh.response.bits := network.bits - localResponseReady(port) := network.bits.channel === channel.U && endpoint.io.mesh.response.ready - } - if (transferAttached) { - val port = attached.size - arbiter.io.in(port) <> transfer.io.mesh.request - transfer.io.mesh.response.valid := network.valid && network.bits.channel === p.transferChannel.U - transfer.io.mesh.response.bits := network.bits - localResponseReady(port) := network.bits.channel === p.transferChannel.U && transfer.io.mesh.response.ready - } - requestIngress.io.in <> arbiter.io.out - network.ready := localResponseReady.asUInt.orR - when(network.valid) { - assert( - (attached.map { case (_, _, channel, _) => - network.bits.channel === channel.U - } ++ - (if (transferAttached) Seq(network.bits.channel === p.transferChannel.U) else Seq.empty)).reduce(_ || _) - ) - } - } else { - requestIngress.io.in.valid := false.B - requestIngress.io.in.bits := 0.U.asTypeOf(new MeshPacket(p)) - network.ready := false.B - when(network.valid) { - assert(false.B, "response reached a node without an attached channel") - } - } - - if (col + 1 < p.cols) { - requests(row)(col + 1).io.in(MeshDirection.west) <> requestRouter.io.out(MeshDirection.east) - requestRouter.io.in(MeshDirection.east) <> requests(row)(col + 1).io.out(MeshDirection.west) - responses(row)(col + 1).io.in(MeshDirection.west) <> responseRouter.io.out(MeshDirection.east) - responseRouter.io.in(MeshDirection.east) <> responses(row)(col + 1).io.out(MeshDirection.west) - } - if (row + 1 < p.rows) { - requests(row + 1)(col).io.in(MeshDirection.north) <> requestRouter.io.out(MeshDirection.south) - requestRouter.io.in(MeshDirection.south) <> requests(row + 1)(col).io.out(MeshDirection.north) - responses(row + 1)(col).io.in(MeshDirection.north) <> responseRouter.io.out(MeshDirection.south) - responseRouter.io.in(MeshDirection.south) <> responses(row + 1)(col).io.out(MeshDirection.north) - } - - for (router <- Seq(requestRouter, responseRouter)) { - if (col == 0) { - router.io.in(MeshDirection.west).valid := false.B - router.io.in(MeshDirection.west).bits := 0.U.asTypeOf(new MeshPacket(p)) - router.io.out(MeshDirection.west).credit := false.B - } - if (col == p.cols - 1) { - router.io.in(MeshDirection.east).valid := false.B - router.io.in(MeshDirection.east).bits := 0.U.asTypeOf(new MeshPacket(p)) - router.io.out(MeshDirection.east).credit := false.B - } - if (row == 0) { - router.io.in(MeshDirection.north).valid := false.B - router.io.in(MeshDirection.north).bits := 0.U.asTypeOf(new MeshPacket(p)) - router.io.out(MeshDirection.north).credit := false.B - } - if (row == p.rows - 1) { - router.io.in(MeshDirection.south).valid := false.B - router.io.in(MeshDirection.south).bits := 0.U.asTypeOf(new MeshPacket(p)) - router.io.out(MeshDirection.south).credit := false.B - } - } - } -} diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshSharedMemInterface.scala b/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshSharedMemInterface.scala deleted file mode 100644 index 1d591f544..000000000 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshSharedMemInterface.scala +++ /dev/null @@ -1,37 +0,0 @@ -package memcore.memory.mesh_shm - -import chisel3._ - -class MeshClientRequest(p: MeshSharedMemParams) - extends memcore.bus.axi.Beat(p.dataBits, p.tagBits, p.bankBits, p.addressBits + 1) { - def bank = tdest - def addr = tuser(p.addressBits, 1) - def write = tuser(0) - def mask = tkeep - def tag = tid -} - -class MeshClientResponse(p: MeshSharedMemParams) extends memcore.bus.axi.Beat(p.dataBits, p.tagBits, 0, 2) { - def tag = tid - def error = tuser(0) - def write = tuser(1) -} - -/** One row transaction: AXI-S payload plus routed event metadata in TUSER. */ -class MeshPacket(p: MeshSharedMemParams) - extends memcore.bus.axi.Beat(p.dataBits, p.tagBits, p.rowBits + p.colBits, p.packetUserBits) { - def privateRequest = tuser(p.eventBits) - def localBank = tuser(p.eventBits + p.localBankBits, p.eventBits + 1) - def core = tuser(p.packetUserBits - 1, p.eventBits + p.localBankBits + 1) - def destRow = tdest(p.rowBits + p.colBits - 1, p.colBits) - def destCol = tdest(p.colBits - 1, 0) - def error = tuser(0) - def write = tuser(1) - def addr = tuser(p.addressBits + 1, 2) - def channel = tuser(p.addressBits + p.channelBits + 1, p.addressBits + 2) - def sourceCol = tuser(p.addressBits + p.channelBits + p.colBits + 1, p.addressBits + p.channelBits + 2) - def sourceRow = - tuser(p.addressBits + p.channelBits + p.colBits + p.rowBits + 1, p.addressBits + p.channelBits + p.colBits + 2) - def mask = tkeep - def tag = tid -} diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshSharedMemParams.scala b/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshSharedMemParams.scala deleted file mode 100644 index b9fc26d98..000000000 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshSharedMemParams.scala +++ /dev/null @@ -1,151 +0,0 @@ -package memcore.memory.mesh_shm - -import chisel3.util.log2Ceil -import memcore.memory.bank.BankSetParams - -case class MeshCoreAttachment( - inputBankIds: Seq[Int], - outputBankIds: Seq[Int], - localBankNode: Option[Int]) { - require(inputBankIds.size == outputBankIds.size, "mesh core input/output channel counts must match") -} - -case class MeshSharedMemParams( - rows: Int, - cols: Int, - entriesPerBank: Int, - dataBits: Int, - tagBits: Int, - cores: Seq[MeshCoreAttachment], - maxInFlight: Int = 4, - linkBufferDepth: Int = 2, - localBankBits: Int = 10, - visibleBanks: Int = 0, - externalChannels: Int = 0) { - require(rows > 0 && cols > 0) - require(entriesPerBank <= 65536) - require(entriesPerBank >= 2 && (entriesPerBank & (entriesPerBank - 1)) == 0) - require(dataBits > 0 && dataBits % 8 == 0) - require(tagBits > 0 && localBankBits > 0 && cores.nonEmpty) - require(cores.forall(c => - (c.inputBankIds ++ c.outputBankIds ++ c.localBankNode).forall(id => id >= 0 && id < rows * cols) - )) - - require(externalChannels >= 0) - require(linkBufferDepth > 0) - require(maxInFlight > 0 && maxInFlight <= (1 << tagBits)) - val bankParams = BankSetParams(dataBits, 1, entriesPerBank, tagBits) - val bankCount: Int = rows * cols - val visibleBankCount: Int = if (visibleBanks == 0) bankCount else visibleBanks - require(visibleBankCount > 0 && visibleBankCount <= bankCount) - require(cores.exists(_.inputBankIds.nonEmpty), "mesh requires a data endpoint") - require(cores.forall(core => core.inputBankIds.nonEmpty == core.localBankNode.nonEmpty)) - val totalChannels: Int = cores.map(_.inputBankIds.size).sum - val inputChannelLocations: Seq[(Int, Int)] = - cores.flatMap(c => c.inputBankIds.map(id => (id / cols, id % cols))) - val outputChannelLocations: Seq[(Int, Int)] = - cores.flatMap(c => c.outputBankIds.map(id => (id / cols, id % cols))) - val rowBits: Int = math.max(1, log2Ceil(rows)) - val colBits: Int = math.max(1, log2Ceil(cols)) - val bankBits: Int = math.max(1, log2Ceil(bankCount)) - val addressBits: Int = 16 - val networkClientChannels: Int = 2 * totalChannels + externalChannels - val transferChannel: Int = networkClientChannels - val channelBits: Int = math.max(1, log2Ceil(networkClientChannels + 1)) - val coreBits: Int = math.max(1, log2Ceil(cores.size)) - val linkCreditBits: Int = math.max(1, log2Ceil(linkBufferDepth + 1)) - - val localBankLocations: Seq[Option[(Int, Int)]] = - cores.map(_.localBankNode.map(id => (id / cols, id % cols))) - - val eventBits: Int = rowBits + colBits + channelBits + addressBits + 2 - val packetUserBits: Int = eventBits + localBankBits + 9 - val maskBits: Int = dataBits / 8 - - // Core order in this sequence defines the flattened io.channels layout. - def channelsForCore(coreIndex: Int): Range = { - require(coreIndex >= 0 && coreIndex < cores.size) - val start = cores.take(coreIndex).map(_.inputBankIds.size).sum - start until (start + cores(coreIndex).inputBankIds.size) - } - -} - -object MeshSharedMemParams { - - def dimensions(bankCount: Int): (Int, Int) = { - require(bankCount > 1) - val rows = (math.sqrt(bankCount.toDouble).toInt to 1 by -1).find(bankCount % _ == 0).get - (rows, bankCount / rows) - } - - /** - * Assigns each channel lane to a directed chain of adjacent Mesh nodes. - * - * Core order defines the stages within each lane; a Core may own fewer lanes - * than its neighbors. A two-dimensional Mesh with an even dimension uses a - * Hamiltonian cycle, so the last stage and reused lanes remain adjacent. - */ - def directionalAttachments( - rows: Int, - cols: Int, - channelsPerCore: Seq[Int] - ): Seq[MeshCoreAttachment] = { - require(rows > 0 && cols > 0) - require(channelsPerCore.nonEmpty && channelsPerCore.forall(_ > 0)) - val bankCount = rows * cols - val channelCount = channelsPerCore.sum - val canWrap = bankCount == 2 || rows > 1 && cols > 1 && (rows % 2 == 0 || cols % 2 == 0) - require(bankCount > 1) - require( - canWrap || channelCount < bankCount, - "a Mesh without a Hamiltonian cycle cannot wrap directional channels" - ) - - def cycle(r: Int, c: Int): Seq[(Int, Int)] = { - require(c % 2 == 0 && r > 1) - val firstColumn = (0 until r).map(row => (row, 0)) - val inner = (1 until c).flatMap { col => - val rowOrder = if (col % 2 == 1) (r - 1 to 1 by -1) else (1 until r) - rowOrder.map(row => (row, col)) - } - val firstRow = (c - 1 to 1 by -1).map(col => (0, col)) - firstColumn ++ inner ++ firstRow - } - - val coordinates = - if (rows % 2 == 0 && cols > 1) cycle(cols, rows).map { case (row, col) => (col, row) } - else if (cols % 2 == 0 && rows > 1) cycle(rows, cols) - else (0 until rows).flatMap { row => - val columns = if (row % 2 == 0) (0 until cols) else (cols - 1 to 0 by -1) - columns.map(col => (row, col)) - } - val bankOrder = coordinates.map { case (row, col) => row * cols + col } - val channelOrder = (0 until channelsPerCore.max).flatMap { channel => - channelsPerCore.indices.collect { case core if channel < channelsPerCore(core) => (core, channel) } - } - val locations = channelOrder.zipWithIndex.map { case (owner, index) => - owner -> ( - bankOrder(index % bankOrder.size), - bankOrder((index + 1) % bankOrder.size) - ) - }.toMap - - channelsPerCore.indices.map { core => - val inputs = (0 until channelsPerCore(core)).map(channel => locations(core -> channel)._1) - val outputs = (0 until channelsPerCore(core)).map(channel => locations(core -> channel)._2) - MeshCoreAttachment(inputs, outputs, Some(inputs.head)) - } - } - - // Experimental 3x4 mesh: type 0/1/2 use 1/2/4 channels, with 2/2/1 cores. - val prototype: MeshSharedMemParams = MeshSharedMemParams( - rows = 3, - cols = 4, - entriesPerBank = 1024, - dataBits = 128, - tagBits = 8, - cores = directionalAttachments(3, 4, Seq(1, 1, 2, 2, 4)) - ) - -} diff --git a/arch/src/main/scala/framework/mem-core/mmu/build.sc b/arch/src/main/scala/framework/mem-core/mmu/build.sc deleted file mode 100644 index 7266ec37d..000000000 --- a/arch/src/main/scala/framework/mem-core/mmu/build.sc +++ /dev/null @@ -1,30 +0,0 @@ -import mill._ -import mill.scalalib._ - -trait ChiselModule extends SbtModule { - def moduleRoot: os.Path - override def millSourcePath = moduleRoot - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") -} - -object queue extends SbtModule { - override def millSourcePath = os.pwd / os.up / "queue" - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") -} - -object chi extends ChiselModule { - override def moduleDeps = Seq(queue) - override def moduleRoot = os.pwd / os.up / "chi" -} - -object mmu extends ChiselModule { - override def mainClass = Some("memcore.memory.mmu.Emit") - override def moduleRoot = os.pwd - override def moduleDeps = Seq(chi) -} diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/csrc/src/translation.rs b/arch/src/main/scala/framework/mem-core/mmu/src/csrc/src/translation.rs deleted file mode 100644 index f878e8ae7..000000000 --- a/arch/src/main/scala/framework/mem-core/mmu/src/csrc/src/translation.rs +++ /dev/null @@ -1,209 +0,0 @@ -use std::collections::{HashMap, VecDeque}; -use std::ffi::c_void; - -#[derive(Clone, Copy)] -struct Pte { - value: u64, - error: bool, -} - -struct Model { - address_bits: u32, - memory: HashMap, - expected_reads: VecDeque, -} - -#[derive(Default)] -struct Translation { - address: u64, - page_fault: bool, - access_fault: bool, - level: u32, -} - -impl Translation { - fn fault(page: bool, level: u32) -> Self { - Self { - page_fault: page, - access_fault: !page, - level, - ..Self::default() - } - } -} - -struct Request { - address: u64, - mode: u32, - root: u64, - privilege: u32, - write: bool, - execute: bool, - sum: bool, - mxr: bool, -} - -impl Model { - // Sequential architectural reference. Memory has only explicitly programmed - // PTEs; Err(address) identifies an incomplete fixture, never an invented PTE. - fn walk(&self, r: &Request, reads: &mut Vec) -> Result { - let max_pa = (1u64 << self.address_bits) - 1; - if r.privilege == 3 || r.mode == 0 { - return Ok(if r.address > max_pa { - Translation::fault(false, 0) - } else { - Translation { address: r.address, ..Translation::default() } - }); - } - if r.address > 0x0000_003f_ffff_ffff && r.address < 0xffff_ffc0_0000_0000 { - return Ok(Translation::fault(true, 2)); - } - let mut ppn = r.root; - for level in (0..=2u32).rev() { - let vpn = (r.address >> (12 + 9 * level)) & 511; - let address = ppn * 4096 + vpn * 8; - if address > max_pa { - return Ok(Translation::fault(false, level)); - } - reads.push(address); - let pte = *self.memory.get(&address).ok_or(address)?; - if pte.error { - return Ok(Translation::fault(false, level)); - } - let value = pte.value; - let valid = value & 1 != 0; - let readable = value & 2 != 0; - let writable = value & 4 != 0; - let executable = value & 8 != 0; - if !valid || (writable && !readable) || value >> 54 != 0 { - return Ok(Translation::fault(true, level)); - } - ppn = (value >> 10) & ((1u64 << 44) - 1); - if readable || executable { - let user_page = value & 16 != 0; - let privilege_ok = if r.privilege == 0 { - user_page - } else { - !user_page || (r.sum && !r.execute) - }; - let operation_ok = if r.execute { - executable - } else if r.write { - writable - } else { - readable || (r.mxr && executable) - }; - let accessed = value & 64 != 0; - let dirty = value & 128 != 0; - let aligned = ppn & ((1u64 << (9 * level)) - 1) == 0; - if !privilege_ok || !operation_ok || !accessed || (r.write && !dirty) || !aligned { - return Ok(Translation::fault(true, level)); - } - let page_bytes = 4096u64 << (9 * level); - let address = ppn * 4096 + (r.address & (page_bytes - 1)); - return Ok(if address > max_pa { - Translation::fault(false, level) - } else { - Translation { address, level, ..Translation::default() } - }); - } - if level == 0 || value & 0xd0 != 0 { - return Ok(Translation::fault(true, level)); - } - } - unreachable!("the bottom-level non-leaf faults before leaving the loop") - } -} - -#[no_mangle] -pub extern "C" fn mmu_ref_create(address_bits: u32) -> *mut c_void { - assert!((44..=52).contains(&address_bits)); - Box::into_raw(Box::new(Model { - address_bits, - memory: HashMap::new(), - expected_reads: VecDeque::new(), - })).cast() -} - -#[no_mangle] -pub unsafe extern "C" fn mmu_ref_destroy(model: *mut c_void) { - drop(Box::from_raw(model.cast::())); -} - -#[no_mangle] -pub unsafe extern "C" fn mmu_ref_clear(model: *mut c_void) { - let model = &mut *model.cast::(); - assert!(model.expected_reads.is_empty(), "fixture changed with unmatched PTE reads"); - model.memory.clear(); -} - -#[no_mangle] -pub unsafe extern "C" fn mmu_ref_program(model: *mut c_void, address: u64, value: u64, error: u32) { - assert_eq!(address % 8, 0); - (*model.cast::()).memory.insert(address, Pte { value, error: error != 0 }); -} - -#[no_mangle] -pub unsafe extern "C" fn mmu_ref_read(model: *mut c_void, address: u64, value: *mut u64, error: *mut u32) -> u32 { - if let Some(pte) = (*model.cast::()).memory.get(&address) { - *value = pte.value; - *error = pte.error as u32; - 1 - } else { - 0 - } -} - -#[no_mangle] -pub unsafe extern "C" fn mmu_ref_pending(model: *mut c_void) -> u32 { - (*model.cast::()).expected_reads.len() as u32 -} - -#[no_mangle] -pub unsafe extern "C" fn mmu_ref_peek(model: *mut c_void, address: *mut u64) -> u32 { - if let Some(next) = (*model.cast::()).expected_reads.front() { - *address = *next; - 1 - } else { - 0 - } -} - -#[no_mangle] -pub unsafe extern "C" fn mmu_ref_consume(model: *mut c_void, address: u64) -> u32 { - let expected = &mut (*model.cast::()).expected_reads; - if expected.front() != Some(&address) { - return 0; - } - expected.pop_front(); - 1 -} - -#[no_mangle] -pub unsafe extern "C" fn mmu_ref_translate( - model: *mut c_void, address: u64, mode: u32, root: u64, privilege: u32, - write: u32, execute: u32, sum: u32, mxr: u32, - paddr: *mut u64, page_fault: *mut u32, access_fault: *mut u32, level: *mut u32, -) -> u32 { - assert!(mode == 0 || mode == 8); - assert!(matches!(privilege, 0 | 1 | 3)); - assert!(write == 0 || execute == 0); - assert!(root < (1u64 << 44)); - let model = &mut *model.cast::(); - let request = Request { address, mode, root, privilege, write: write != 0, execute: execute != 0, sum: sum != 0, mxr: mxr != 0 }; - let mut reads = Vec::new(); - match model.walk(&request, &mut reads) { - Ok(result) => { - model.expected_reads.extend(reads); - *paddr = result.address; - *page_fault = result.page_fault as u32; - *access_fault = result.access_fault as u32; - *level = result.level; - 0 - } - Err(address) => { - *paddr = address; - 1 - } - } -} diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker.f b/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker.f deleted file mode 100644 index 35b035a74..000000000 --- a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/walker_interface.sv -@RESOURCES@/walker_package.sv --F @RTL@/Walker/filelist.f -@RESOURCES@/walker_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_access.f b/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_access.f deleted file mode 100644 index 258b768a9..000000000 --- a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_access.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/walker_interface.sv -@RESOURCES@/walker_negative_package.sv --F @RTL@/Walker/filelist.f -@RESOURCES@/walker_bad_access_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_access.toml b/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_access.toml deleted file mode 100644 index 9a665fd6f..000000000 --- a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_access.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Walker access cannot both write and execute" diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_access_tb.sv b/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_access_tb.sv deleted file mode 100644 index 77977e2a4..000000000 --- a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_access_tb.sv +++ /dev/null @@ -1,44 +0,0 @@ -`include "walker_config.svh" -module walker_bad_access_tb; - import uvm_pkg::*; - import walker_negative_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - walker_control_if control (clock); - stream_if #(`WALK_REQ_WIDTH) source_if ( - clock, - control.reset - ); - stream_if #(`WALK_RESP_WIDTH) sink_if ( - clock, - control.reset - ); - stream_if #(`WALK_ACCESS_WIDTH) access_if ( - clock, - control.reset - ); - stream_if #(`WALK_RESULT_WIDTH) result_if ( - clock, - control.reset - ); - Walker dut ( - .clock(clock), - .reset(control.reset), - .io_config_mode(control.mode), - .io_config_rootPpn(control.root_ppn), - `include "walker_ports.svh" - ); - initial begin - uvm_config_db#(virtual walker_control_if)::set(null, "uvm_test_top", "control", control); - uvm_config_db#(virtual stream_if #(`WALK_REQ_WIDTH))::set(null, "uvm_test_top", "req", - source_if); - uvm_config_db#(virtual stream_if #(`WALK_RESP_WIDTH))::set(null, "uvm_test_top", "resp", - sink_if); - uvm_config_db#(virtual stream_if #(`WALK_ACCESS_WIDTH))::set(null, "uvm_test_top", "access", - access_if); - uvm_config_db#(virtual stream_if #(`WALK_RESULT_WIDTH))::set(null, "uvm_test_top", "result", - result_if); - uvm_config_db#(int)::set(null, "uvm_test_top", "invalid_case", 2); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_mode.f b/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_mode.f deleted file mode 100644 index 3e9e6d826..000000000 --- a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_mode.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/walker_interface.sv -@RESOURCES@/walker_negative_package.sv --F @RTL@/Walker/filelist.f -@RESOURCES@/walker_bad_mode_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_mode.toml b/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_mode.toml deleted file mode 100644 index fbd6a3acf..000000000 --- a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_mode.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Walker requires Bare or Sv39 mode" diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_mode_tb.sv b/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_mode_tb.sv deleted file mode 100644 index 21e941816..000000000 --- a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_mode_tb.sv +++ /dev/null @@ -1,44 +0,0 @@ -`include "walker_config.svh" -module walker_bad_mode_tb; - import uvm_pkg::*; - import walker_negative_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - walker_control_if control (clock); - stream_if #(`WALK_REQ_WIDTH) source_if ( - clock, - control.reset - ); - stream_if #(`WALK_RESP_WIDTH) sink_if ( - clock, - control.reset - ); - stream_if #(`WALK_ACCESS_WIDTH) access_if ( - clock, - control.reset - ); - stream_if #(`WALK_RESULT_WIDTH) result_if ( - clock, - control.reset - ); - Walker dut ( - .clock(clock), - .reset(control.reset), - .io_config_mode(control.mode), - .io_config_rootPpn(control.root_ppn), - `include "walker_ports.svh" - ); - initial begin - uvm_config_db#(virtual walker_control_if)::set(null, "uvm_test_top", "control", control); - uvm_config_db#(virtual stream_if #(`WALK_REQ_WIDTH))::set(null, "uvm_test_top", "req", - source_if); - uvm_config_db#(virtual stream_if #(`WALK_RESP_WIDTH))::set(null, "uvm_test_top", "resp", - sink_if); - uvm_config_db#(virtual stream_if #(`WALK_ACCESS_WIDTH))::set(null, "uvm_test_top", "access", - access_if); - uvm_config_db#(virtual stream_if #(`WALK_RESULT_WIDTH))::set(null, "uvm_test_top", "result", - result_if); - uvm_config_db#(int)::set(null, "uvm_test_top", "invalid_case", 0); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_privilege.f b/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_privilege.f deleted file mode 100644 index b19f01b46..000000000 --- a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_privilege.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/walker_interface.sv -@RESOURCES@/walker_negative_package.sv --F @RTL@/Walker/filelist.f -@RESOURCES@/walker_bad_privilege_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_privilege.toml b/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_privilege.toml deleted file mode 100644 index 9a567c024..000000000 --- a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_privilege.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Walker effective privilege must be U, S or M" diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_privilege_tb.sv b/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_privilege_tb.sv deleted file mode 100644 index cf19dbc56..000000000 --- a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_privilege_tb.sv +++ /dev/null @@ -1,44 +0,0 @@ -`include "walker_config.svh" -module walker_bad_privilege_tb; - import uvm_pkg::*; - import walker_negative_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - walker_control_if control (clock); - stream_if #(`WALK_REQ_WIDTH) source_if ( - clock, - control.reset - ); - stream_if #(`WALK_RESP_WIDTH) sink_if ( - clock, - control.reset - ); - stream_if #(`WALK_ACCESS_WIDTH) access_if ( - clock, - control.reset - ); - stream_if #(`WALK_RESULT_WIDTH) result_if ( - clock, - control.reset - ); - Walker dut ( - .clock(clock), - .reset(control.reset), - .io_config_mode(control.mode), - .io_config_rootPpn(control.root_ppn), - `include "walker_ports.svh" - ); - initial begin - uvm_config_db#(virtual walker_control_if)::set(null, "uvm_test_top", "control", control); - uvm_config_db#(virtual stream_if #(`WALK_REQ_WIDTH))::set(null, "uvm_test_top", "req", - source_if); - uvm_config_db#(virtual stream_if #(`WALK_RESP_WIDTH))::set(null, "uvm_test_top", "resp", - sink_if); - uvm_config_db#(virtual stream_if #(`WALK_ACCESS_WIDTH))::set(null, "uvm_test_top", "access", - access_if); - uvm_config_db#(virtual stream_if #(`WALK_RESULT_WIDTH))::set(null, "uvm_test_top", "result", - result_if); - uvm_config_db#(int)::set(null, "uvm_test_top", "invalid_case", 1); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_interface.sv b/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_interface.sv deleted file mode 100644 index 895f6d148..000000000 --- a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_interface.sv +++ /dev/null @@ -1,7 +0,0 @@ -interface walker_control_if ( - input logic clock -); - logic reset; - logic [3:0] mode; - logic [43:0] root_ppn; -endinterface diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_negative_package.sv b/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_negative_package.sv deleted file mode 100644 index e4ff511f8..000000000 --- a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_negative_package.sv +++ /dev/null @@ -1,64 +0,0 @@ -`include "walker_config.svh" -package walker_negative_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual walker_control_if control; - virtual stream_if #(`WALK_REQ_WIDTH) req; - virtual stream_if #(`WALK_RESP_WIDTH) resp; - virtual stream_if #(`WALK_ACCESS_WIDTH) access; - virtual stream_if #(`WALK_RESULT_WIDTH) result; - int invalid_case; - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual walker_control_if)::get( - this, "", "control", control - ) || !uvm_config_db#(virtual stream_if #(`WALK_REQ_WIDTH))::get( - this, "", "req", req - ) || !uvm_config_db#(virtual stream_if #(`WALK_RESP_WIDTH))::get( - this, "", "resp", resp - ) || !uvm_config_db#(virtual stream_if #(`WALK_ACCESS_WIDTH))::get( - this, "", "access", access - ) || !uvm_config_db#(virtual stream_if #(`WALK_RESULT_WIDTH))::get( - this, "", "result", result - ) || !uvm_config_db#(int)::get( - this, "", "invalid_case", invalid_case - )) - `uvm_fatal("VIF", "walker negative interface missing") - endfunction - task execute(); - control.reset = 1; - control.mode = 8; - control.root_ppn = 16; - req.valid = 0; - req.bits = '0; - resp.ready = 1; - access.ready = 1; - result.valid = 0; - result.bits = '0; - repeat (4) @(negedge control.clock); - control.reset = 0; - req.bits[`WALK_REQ_PRIVILEGE_OFFSET+:`WALK_REQ_PRIVILEGE_WIDTH] = 1; - case (invalid_case) - 0: control.mode = 9; - 1: req.bits[`WALK_REQ_PRIVILEGE_OFFSET+:`WALK_REQ_PRIVILEGE_WIDTH] = 2; - 2: begin - req.bits[`WALK_REQ_WRITE_OFFSET] = 1; - req.bits[`WALK_REQ_EXECUTE_OFFSET] = 1; - end - default: `uvm_fatal("CASE", "Unknown invalid walker request") - endcase - req.valid = 1; - `uvm_info("INVALID_INPUT", $sformatf("Driving walker contract violation %0d", invalid_case), - UVM_LOW) - repeat (10) @(negedge control.clock); - `uvm_fatal("MISSING_ASSERTION", - "Illegal walker input did not trigger the specified contract assertion") - endtask - endclass -endpackage diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_package.sv b/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_package.sv deleted file mode 100644 index 35f7a04e7..000000000 --- a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_package.sv +++ /dev/null @@ -1,458 +0,0 @@ -`include "walker_config.svh" -`define WF(K, F) `WALK_``K``_``F``_OFFSET +: `WALK_``K``_``F``_WIDTH -package walker_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - - import "DPI-C" function chandle mmu_ref_create(input int unsigned address_bits); - import "DPI-C" function void mmu_ref_destroy(input chandle model); - import "DPI-C" function void mmu_ref_clear(input chandle model); - import "DPI-C" function void mmu_ref_program( - input chandle model, - input longint unsigned address, - value, - input int unsigned error - ); - import "DPI-C" function int unsigned mmu_ref_read( - input chandle model, - input longint unsigned address, - output longint unsigned value, - output int unsigned error - ); - import "DPI-C" function int unsigned mmu_ref_pending(input chandle model); - import "DPI-C" function int unsigned mmu_ref_peek( - input chandle model, - output longint unsigned address - ); - import "DPI-C" function int unsigned mmu_ref_consume( - input chandle model, - input longint unsigned address - ); - import "DPI-C" function int unsigned mmu_ref_translate( - input chandle model, - input longint unsigned address, - input int unsigned mode, - input longint unsigned root, - input int unsigned privilege, - write, - execute, - sum, - mxr, - output longint unsigned paddr, - output int unsigned page_fault, - access_fault, - level - ); - typedef bit [63:0] u64; - typedef logic [`WALK_REQ_WIDTH-1:0] req_t; - class translation extends uvm_sequence_item; - bit [63:0] paddr; - bit page_fault, access_fault; - bit [1:0] level; - `uvm_object_utils_begin(translation) - `uvm_field_int(paddr, UVM_DEFAULT) - `uvm_field_int(page_fault, UVM_DEFAULT) - `uvm_field_int(access_fault, UVM_DEFAULT) - `uvm_field_int(level, UVM_DEFAULT) - `uvm_object_utils_end - function new(string name = "translation"); - super.new(name); - endfunction - endclass - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual walker_control_if control; - virtual stream_if #(`WALK_REQ_WIDTH) req; - virtual stream_if #(`WALK_RESP_WIDTH) resp; - virtual stream_if #(`WALK_ACCESS_WIDTH) access; - virtual stream_if #(`WALK_RESULT_WIDTH) result; - in_order_scoreboard #(translation) scoreboard; - // Only explicitly programmed PTEs exist. There is no implicit zero/pattern RAM. - chandle model; - int - expected = 0, - accepted = 0, - returned = 0, - reads = 0, - cycle = 0, - req_stalls = 0, - access_stalls = 0, - resp_stalls = 0; - int page_faults = 0, access_faults = 0, successes[3] = '{0, 0, 0}; - int due = 0; - bit pending_result = 0, result_active = 0, hold_access = 0, hold_response = 0; - logic [`WALK_RESULT_WIDTH-1:0] result_packet; - string current_case; - localparam u64 MAX_PA = (64'h1 << `WALK_RESP_PADDR_WIDTH) - 1; - localparam u64 HIGH_PPN = 64'h1 << (`WALK_RESP_PADDR_WIDTH - 12); - function new(string name, uvm_component parent); - super.new(name, parent); - timeout = 1ms; - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - scoreboard = in_order_scoreboard#(translation)::type_id::create("scoreboard", this); - model = mmu_ref_create(`WALK_RESP_PADDR_WIDTH); - if (!uvm_config_db#(virtual walker_control_if)::get( - this, "", "control", control - ) || !uvm_config_db#(virtual stream_if #(`WALK_REQ_WIDTH))::get( - this, "", "req", req - ) || !uvm_config_db#(virtual stream_if #(`WALK_RESP_WIDTH))::get( - this, "", "resp", resp - ) || !uvm_config_db#(virtual stream_if #(`WALK_ACCESS_WIDTH))::get( - this, "", "access", access - ) || !uvm_config_db#(virtual stream_if #(`WALK_RESULT_WIDTH))::get( - this, "", "result", result - )) - `uvm_fatal("VIF", "Walker interfaces missing") - endfunction - function req_t request(u64 va, int operation = 0, int privilege = 1, bit sum = 0, bit mxr = 0); - req_t r = '0; - r[`WF(REQ, VADDR)] = va; - r[`WF(REQ, WRITE)] = operation == 1; - r[`WF(REQ, EXECUTE)] = operation == 2; - r[`WF(REQ, PRIVILEGE)] = privilege; - r[`WF(REQ, SUM)] = sum; - r[`WF(REQ, MXR)] = mxr; - return r; - endfunction - function void program_pte(u64 address, u64 value, bit error = 0); - mmu_ref_program(model, address, value, error); - endfunction - function u64 pte(u64 address); - longint unsigned value; - int unsigned error; - if (!mmu_ref_read(model, address, value, error)) - `uvm_fatal("FIXTURE", "Attempt to modify an unprogrammed PTE") - return value; - endfunction - function translation golden(req_t r, int mode, u64 root); - longint unsigned address; - int unsigned page_fault, access_fault, level; - translation t = new(current_case); - if (mmu_ref_translate( - model, - r[ - `WF(REQ, VADDR) - ], - mode, - root, - r[ - `WF(REQ, PRIVILEGE) - ], - r[ - `WF(REQ, WRITE) - ], - r[ - `WF(REQ, EXECUTE) - ], - r[ - `WF(REQ, SUM) - ], - r[ - `WF(REQ, MXR) - ], - address, - page_fault, - access_fault, - level - ) != 0) - `uvm_fatal("FIXTURE", $sformatf("%s uninitialized golden PTE %h", current_case, address)) - t.paddr = address; - t.page_fault = page_fault; - t.access_fault = access_fault; - t.level = level; - return t; - endfunction - function u64 pte_address(u64 va, int level); - return (64'h12 - level) * 4096 + ((va >> (12 + 9 * level)) & 511) * 8; - endfunction - function void prepare(u64 va, int leaf_level, u64 flags = 64'hcf, u64 mapped = 64'h80000000); - mmu_ref_clear(model); - for (int level = 2; level >= leaf_level; level--) begin - if (level == leaf_level) - program_pte(pte_address(va, level), ((mapped >> 12) << 10) | flags); - else program_pte(pte_address(va, level), ((64'h13 - level) << 10) | 1); - end - endfunction - task service(); - forever begin - @(req.sample); - if (!control.reset) begin - cycle++; - if (req.sample.valid && req.sample.ready) accepted++; - if (req.sample.valid && !req.sample.ready) req_stalls++; - if (access.sample.valid && !access.sample.ready) access_stalls++; - if (resp.sample.valid && !resp.sample.ready) resp_stalls++; - if (result_active && result.sample.valid && result.sample.ready) begin - result_active = 0; - pending_result = 0; - end - if (access.sample.valid) begin - u64 address; - longint unsigned next_address, value; - int unsigned error; - if ($isunknown(access.sample.bits)) `uvm_fatal("PTE_X", "Unknown PTE memory request") - address = access.sample.bits[`WF(ACCESS, ADDR)]; - if (!mmu_ref_peek(model, next_address) || address !== next_address) - `uvm_fatal("PTE_SEQUENCE", $sformatf( - "%s unexpected PTE address %h (remaining=%0d)", - current_case, - address, - mmu_ref_pending( - model - ) - )) - if (access.sample.bits[ - `WF(ACCESS, WRITE) - ] || access.sample.bits[ - `WF(ACCESS, DATA) - ] != 0 || access.sample.bits[ - `WF(ACCESS, MASK) - ] != 0 || access.sample.bits[ - `WF(ACCESS, ATOMIC) - ] != 0 || access.sample.bits[ - `WF(ACCESS, ATOMICWORD) - ] != 0) - `uvm_fatal("PTE_WRITE", "Svade profile permits only ordinary 64-bit PTE reads") - if (!mmu_ref_read(model, address, value, error)) - `uvm_fatal("UNINITIALIZED_PTE", "DUT read an unprogrammed memory location") - if (access.sample.ready) begin - if (pending_result) - `uvm_fatal("OUTSTANDING", "Walker issued more than one PTE access") - if (!mmu_ref_consume(model, address)) - `uvm_fatal("PTE_SEQUENCE", "Reference PTE queue changed before handshake") - reads++; - result_packet = '0; - result_packet[`WF(RESULT, DATA)] = value; - result_packet[`WF(RESULT, ERROR)] = error; - due = cycle + 2 + reads % 4; - pending_result = 1; - end - end - if (resp.sample.valid && resp.sample.ready) begin - translation actual = new(current_case); - if ($isunknown(resp.sample.bits)) - `uvm_fatal("RESPONSE_X", "Unknown translation response") - actual.paddr = resp.sample.bits[`WF(RESP, PADDR)]; - actual.page_fault = resp.sample.bits[`WF(RESP, PAGEFAULT)]; - actual.access_fault = resp.sample.bits[`WF(RESP, ACCESSFAULT)]; - actual.level = resp.sample.bits[`WF(RESP, LEVEL)]; - if (actual.page_fault && actual.access_fault) - `uvm_fatal("FAULT_KIND", "Fault kinds must be mutually exclusive") - if ((actual.page_fault || actual.access_fault) && actual.paddr != 0) - `uvm_fatal("FAULT_PA", "Fault response exposed a physical address") - scoreboard.actual_export.write(actual); - returned++; - if (actual.page_fault) page_faults++; - else if (actual.access_fault) access_faults++; - else successes[actual.level]++; - end - end - @(negedge control.clock); - access.ready = !control.reset && !hold_access && cycle % 5 >= 2; - resp.ready = !control.reset && !hold_response && cycle % 7 >= 3; - if (pending_result && !result_active && cycle >= due) result_active = 1; - result.valid = result_active; - result.bits = result_packet; - end - endtask - task send(req_t r, int mode = 8, u64 root = 16, bit change_inputs = 0); - @(negedge control.clock); - control.mode = mode; - control.root_ppn = root; - req.bits = r; - req.valid = 1; - do @(req.sample); while (!req.sample.ready); - @(negedge control.clock); - req.valid = 0; - if (change_inputs) begin - control.mode = 0; - control.root_ppn = 0; - req.bits = ~r; - end - endtask - task run_case(string name, req_t r, int mode = 8, u64 root = 16, bit change_inputs = 0); - translation prediction; - current_case = name; - if (mmu_ref_pending(model) || pending_result) - `uvm_fatal("PENDING", "Previous translation not drained") - prediction = golden(r, mode, root); - scoreboard.expected_export.write(prediction); - expected++; - send(r, mode, root, change_inputs); - scoreboard.wait_checked(expected); - @(negedge control.clock); - if (mmu_ref_pending(model) || pending_result) - `uvm_fatal("PENDING", "Translation skipped expected PTE reads") - endtask - task execute(); - u64 va = 64'h123456789, flags, mapped; - int permissions[5] = '{1, 3, 4, 5, 7}; - control.reset = 1; - control.mode = 0; - control.root_ppn = 0; - req.valid = 0; - req.bits = '0; - resp.ready = 0; - access.ready = 0; - result.valid = 0; - result.bits = '0; - repeat (5) @(negedge control.clock); - control.reset = 0; - fork - service(); - join_none - // Explicitly force a stalled PTE request, then hold the response while a - // second request is pending. Both expected paths use initialized table words. - prepare(va, 0, 64'h43, 64'h81234000); - current_case = "back_to_back_read_then_execute_fault"; - scoreboard.expected_export.write(golden(request(va), 8, 16)); - expected++; - scoreboard.expected_export.write(golden(request(va, 2), 8, 16)); - expected++; - hold_access = 1; - hold_response = 1; - send(request(va)); - fork - send(request(va, 2)); - begin - wait (access.sample.valid); - repeat (8) @(req.sample); - hold_access = 0; - wait (resp.sample.valid); - repeat (8) @(req.sample); - hold_response = 0; - end - join - scoreboard.wait_checked(expected); - @(negedge control.clock); - if (mmu_ref_pending(model) || pending_result) - `uvm_fatal("PENDING", "Back-to-back translations failed to drain") - // Three page sizes and both canonical sign regions, including page edges. - for (int level = 0; level < 3; level++) begin - for (int position = 0; position < 3; position++) begin - u64 size = 64'h1000 << (9 * level); - u64 address = (va & ~(size - 1)) + (position == 0 ? 0 : position == 1 ? 123 : size - 1); - prepare(address, level); - run_case($sformatf("page_size_%0d_edge_%0d", level, position), request(address)); - address |= 64'hffffffc000000000; - prepare(address, level); - run_case($sformatf("negative_canonical_%0d_%0d", level, position), request(address)); - end - end - // Permissions: all valid RWX forms, U/S, U-page, SUM, MXR, load/store/fetch. - foreach (permissions[i]) - for (int privilege = 0; privilege < 2; privilege++) - for (int user_page = 0; user_page < 2; user_page++) - for (int sum = 0; sum < 2; sum++) - for (int mxr = 0; mxr < 2; mxr++) - for (int operation = 0; operation < 3; operation++) begin - flags = 64'hc1 | (u64'(permissions[i]) << 1) | (u64'(user_page) << 4); - prepare(va, 0, flags, 64'h81234000); - run_case($sformatf( - "perm_%0d_prv%0d_u%0d_sum%0d_mxr%0d_op%0d", - permissions[i], - privilege, - user_page, - sum, - mxr, - operation - ), request(va, operation, privilege, sum, mxr)); - end - for (int level = 0; level < 3; level++) - for (int ad = 0; ad < 4; ad++) - for (int operation = 0; operation < 3; operation++) begin - flags = 64'hf | (u64'(ad) << 6); - prepare(va, level, flags); - run_case($sformatf("svade_level%0d_ad%0d_op%0d", level, ad, operation), request( - va, operation)); - end - for (int level = 0; level < 3; level++) begin - for (int reserved_bit = 54; reserved_bit < 64; reserved_bit++) begin - prepare(va, level); - program_pte(pte_address(va, level), pte(pte_address(va, level) - ) | (64'h1 << reserved_bit)); - run_case($sformatf("reserved_leaf_l%0d_bit%0d", level, reserved_bit), request(va)); - end - for (int encoding = 0; encoding < 3; encoding++) begin - prepare(va, level, encoding == 0 ? 64'hce : encoding == 1 ? 64'hc5 : 64'hcd); - run_case($sformatf("invalid_pte_l%0d_encoding%0d", level, encoding), request(va)); - end - prepare(va, level); - program_pte(pte_address(va, level), '1, 1); - run_case($sformatf("backend_error_l%0d", level), request(va)); - prepare(va, level, 64'hcf, HIGH_PPN * 4096); - run_case($sformatf("physical_leaf_overflow_l%0d", level), request(va)); - prepare(va, level, 64'hf, HIGH_PPN * 4096); - run_case($sformatf("pte_fault_precedes_final_pa_overflow_l%0d", level), request(va)); - end - for (int level = 1; level <= 2; level++) begin - for (int bit_index = 0; bit_index < 9 * level; bit_index++) begin - prepare(va, level); - program_pte(pte_address(va, level), pte(pte_address(va, level) - ) | (64'h1 << (10 + bit_index))); - run_case($sformatf("superpage_alignment_l%0d_bit%0d", level, bit_index), request(va)); - end - for (int bad = 0; bad < 3; bad++) begin - prepare(va, 0); - program_pte(pte_address(va, level), pte(pte_address(va, level) - ) | (64'h1 << (bad == 0 ? 4 : bad == 1 ? 6 : 7))); - run_case($sformatf("nonleaf_reserved_l%0d_%0d", level, bad), request(va)); - end - for (int bit_index = 54; bit_index < 64; bit_index++) begin - prepare(va, 0); - program_pte(pte_address(va, level), pte(pte_address(va, level)) | (64'h1 << bit_index)); - run_case($sformatf("nonleaf_upper_reserved_l%0d_%0d", level, bit_index), request(va)); - end - prepare(va, 0); - program_pte(pte_address(va, level), (HIGH_PPN << 10) | 1); - run_case($sformatf("next_table_address_overflow_l%0d", level), request(va)); - end - prepare(va, 0); - program_pte(pte_address(va, 0), (64'h13 << 10) | 1); - run_case("nonleaf_at_bottom", request(va)); - prepare(va, 0, 64'h3ef); // G and both RSW bits are permitted and ignored for translation. - for (int level = 1; level <= 2; level++) - program_pte(pte_address(va, level), pte(pte_address(va, level)) | 64'h320); - run_case("software_and_global_bits", request(va)); - prepare(va, 0, 64'h59); - run_case("latched_root_sum_mxr_privilege", request(va, 0, 1, 1, 1), 8, 16, 1); - mmu_ref_clear(model); - run_case("root_physical_overflow", request(va), 8, HIGH_PPN); - run_case("positive_noncanonical", request(64'h0000004000000000)); - run_case("upper_noncanonical", request(64'hffffffbfffffffff)); - for (int operation = 0; operation < 3; operation++) begin - for (int privilege = 0; privilege < 2; privilege++) begin - run_case("bare_noncanonical_physical", request(64'h8000000000, operation, privilege), 0, - 0); - run_case("bare_last_physical_byte", request(MAX_PA, operation, privilege), 0, 0); - run_case("bare_physical_overflow", request(MAX_PA + 1, operation, privilege), 0, 0); - end - run_case("machine_ignores_sv39", request(64'h8000000000, operation, 3), 8, HIGH_PPN); - run_case("machine_physical_overflow", request(MAX_PA + 1, operation, 3), 8, 16); - end - // Single-address translations on opposite sides of a base-page boundary. - prepare(64'hfff, 0, 64'hcf, MAX_PA & ~64'hfff); - run_case("last_byte_before_physical_limit", request(64'hfff)); - prepare(64'h1000, 0, 64'hcf, MAX_PA + 1); - run_case("next_page_physical_overflow", request(64'h1000)); - // The last PTE word of the highest implemented physical table page is legal. - mmu_ref_clear(model); - program_pte(MAX_PA - 7, 64'hcf); - run_case("last_physical_pte_word", request(64'hffffffffc0000000), 8, MAX_PA >> 12); - if(expected!=accepted||expected!=returned||!req_stalls||!access_stalls||!resp_stalls||!page_faults||!access_faults||!successes[0]||!successes[1]||!successes[2]) - `uvm_fatal("SCENARIOS", "Walker scenarios or handshake checks were not exercised") - `uvm_info( - "CHECKS", - $sformatf( - "Sv39/Svade walker checked=%0d PTEreads=%0d successLevels=%0d/%0d/%0d PF=%0d AF=%0d stalls(req/access/resp)=%0d/%0d/%0d; explicit PTE dictionary, no implicit writes", - expected, reads, successes[0], successes[1], successes[2], page_faults, - access_faults, req_stalls, access_stalls, resp_stalls), UVM_LOW) - endtask - function void final_phase(uvm_phase phase); - mmu_ref_destroy(model); - endfunction - endclass -endpackage diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_tb.sv b/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_tb.sv deleted file mode 100644 index 646a52c6c..000000000 --- a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_tb.sv +++ /dev/null @@ -1,43 +0,0 @@ -`include "walker_config.svh" -module walker_tb; - import uvm_pkg::*; - import walker_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - walker_control_if control (clock); - stream_if #(`WALK_REQ_WIDTH) source_if ( - clock, - control.reset - ); - stream_if #(`WALK_RESP_WIDTH) sink_if ( - clock, - control.reset - ); - stream_if #(`WALK_ACCESS_WIDTH) access_if ( - clock, - control.reset - ); - stream_if #(`WALK_RESULT_WIDTH) result_if ( - clock, - control.reset - ); - Walker dut ( - .clock(clock), - .reset(control.reset), - .io_config_mode(control.mode), - .io_config_rootPpn(control.root_ppn), - `include "walker_ports.svh" - ); - initial begin - uvm_config_db#(virtual walker_control_if)::set(null, "uvm_test_top", "control", control); - uvm_config_db#(virtual stream_if #(`WALK_REQ_WIDTH))::set(null, "uvm_test_top", "req", - source_if); - uvm_config_db#(virtual stream_if #(`WALK_RESP_WIDTH))::set(null, "uvm_test_top", "resp", - sink_if); - uvm_config_db#(virtual stream_if #(`WALK_ACCESS_WIDTH))::set(null, "uvm_test_top", "access", - access_if); - uvm_config_db#(virtual stream_if #(`WALK_RESULT_WIDTH))::set(null, "uvm_test_top", "result", - result_if); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/main/scala/Emit.scala b/arch/src/main/scala/framework/mem-core/mmu/src/main/scala/Emit.scala deleted file mode 100644 index 95c87ffd3..000000000 --- a/arch/src/main/scala/framework/mem-core/mmu/src/main/scala/Emit.scala +++ /dev/null @@ -1,58 +0,0 @@ -package memcore.memory.mmu - -import java.nio.file.{Files, Paths} -import memcore.bus.chi.Params - -object Emit extends App { - val p = Params() - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Walker(p), - firtoolOpts = args, - args = Array("--target-dir", "build/Walker", "--split-verilog") - ) - val walkerDefinitions = scala.collection.mutable.ArrayBuffer[String]() - val walkerPorts = scala.collection.mutable.ArrayBuffer[String]() - - def walkerChannel( - kind: String, - port: String, - signal: String, - fs: Seq[(String, Int)] - ): Unit = { - var offset = 0 - for ((field, width) <- fs) { - walkerDefinitions += s"`define WALK_${kind}_${field.toUpperCase}_OFFSET $offset" - walkerDefinitions += s"`define WALK_${kind}_${field.toUpperCase}_WIDTH $width" - walkerPorts += s".io_${port}_bits_$field($signal.bits[$offset +: $width])" - offset += width - } - walkerDefinitions += s"`define WALK_${kind}_WIDTH $offset" - walkerPorts += s".io_${port}_valid($signal.valid)" - walkerPorts += s".io_${port}_ready($signal.ready)" - } - - walkerChannel( - "REQ", - "req", - "source_if", - Seq("vaddr" -> 64, "write" -> 1, "execute" -> 1, "privilege" -> 2, "sum" -> 1, "mxr" -> 1) - ) - walkerChannel( - "RESP", - "resp", - "sink_if", - Seq("paddr" -> p.addressBits, "pageFault" -> 1, "accessFault" -> 1, "level" -> 2) - ) - walkerChannel( - "ACCESS", - "access", - "access_if", - Seq("addr" -> p.addressBits, "write" -> 1, "data" -> 64, "mask" -> 8, "atomic" -> 4, "atomicWord" -> 1) - ) - walkerChannel("RESULT", "result", "result_if", Seq("data" -> 64, "error" -> 1)) - Files.writeString( - Paths.get("build/walker_config.svh"), - "`ifndef WALKER_CONFIG_SVH\n`define WALKER_CONFIG_SVH\n" + walkerDefinitions.mkString("\n") + "\n`endif\n" - ) - Files.writeString(Paths.get("build/walker_ports.svh"), walkerPorts.mkString(",\n") + "\n") -} diff --git a/arch/src/main/scala/framework/mem-core/preflight/build.sc b/arch/src/main/scala/framework/mem-core/preflight/build.sc deleted file mode 100644 index b28c2c7e0..000000000 --- a/arch/src/main/scala/framework/mem-core/preflight/build.sc +++ /dev/null @@ -1,35 +0,0 @@ -import mill._ -import mill.scalalib._ - -trait ChiselModule extends SbtModule { - def moduleRoot: os.Path - override def millSourcePath = moduleRoot - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") -} - -object queue extends SbtModule { - override def millSourcePath = os.pwd / os.up / "queue" - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") -} - -object chi extends ChiselModule { - override def moduleDeps = Seq(queue) - override def moduleRoot = os.pwd / os.up / "chi" -} - -object mmu extends ChiselModule { - override def moduleRoot = os.pwd / os.up / "mmu" - override def moduleDeps = Seq(chi) -} - -object preflight extends ChiselModule { - override def moduleRoot = os.pwd - override def moduleDeps = Seq(chi, mmu) - override def mainClass = Some("memcore.memory.preflight.Emit") -} diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/package.sv b/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/package.sv deleted file mode 100644 index 3ab987bbb..000000000 --- a/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/package.sv +++ /dev/null @@ -1,543 +0,0 @@ -`ifdef PREFLIGHT_LINE_READ -`include "preflight64_config.svh" -`else -`include "preflight_config.svh" -`endif -`define PF(K, F) `PF_``K``_``F``_OFFSET +: `PF_``K``_``F``_WIDTH -package preflight_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - typedef longint unsigned u64; - typedef bit [`PF_CMD_WIDTH-1:0] command_t; - import "DPI-C" function chandle preflight_ref_create( - input int unsigned address_bits, - beat - ); - import "DPI-C" function void preflight_ref_destroy(input chandle model); - import "DPI-C" function void preflight_ref_program( - input chandle model, - input longint unsigned address, - value, - input int unsigned error - ); - import "DPI-C" function void preflight_ref_policy( - input chandle model, - input int unsigned pte_valid, - input longint unsigned pte, - input int unsigned range_valid, - input longint unsigned first, - last - ); - import "DPI-C" function int unsigned preflight_ref_submit( - input chandle model, - input int unsigned id, - input longint unsigned base, - input int unsigned rows, - cols, - span, - col_stride, - row_stride, - write, - mode, - input longint unsigned root, - input int unsigned privilege, - sum, - mxr - ); - import "DPI-C" function int unsigned preflight_ref_count( - input chandle model, - input int unsigned id - ); - import "DPI-C" function void preflight_ref_expected( - input chandle model, - input int unsigned id, - index, - output longint unsigned va, - pa, - output int unsigned bytes, - write, - last, - error - ); - import "DPI-C" function int unsigned preflight_ref_authorize( - input chandle model, - input int unsigned id, - input longint unsigned pa, - input int unsigned bytes, - write, - pte, - privilege, - output int unsigned allow - ); - import "DPI-C" function int unsigned preflight_ref_pte( - input chandle model, - input int unsigned id, - input longint unsigned address, - output longint unsigned value, - output int unsigned error - ); - import "DPI-C" function int unsigned preflight_ref_pending( - input chandle model, - input int unsigned id - ); - import "DPI-C" function void preflight_ref_retire( - input chandle model, - input int unsigned id - ); - import "DPI-C" function void preflight_ref_reset(input chandle model); - class result_item extends uvm_sequence_item; - bit [`PF_OUT_WIDTH-1:0] packet; - `uvm_object_utils_begin(result_item) - `uvm_field_int(packet, UVM_DEFAULT) - `uvm_object_utils_end - function new(string name = "result_item"); - super.new(name); - endfunction - endclass - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual preflight_control_if ctl; - virtual stream_if #(`PF_CMD_WIDTH) cmd; - virtual stream_if #(`PF_OUT_WIDTH) prepared; - virtual stream_if #(`PF_AUTH_WIDTH) authorization; - virtual stream_if #(`PF_PERMIT_WIDTH) permission; - virtual stream_if #(`PF_PTE_WIDTH) pte_req; - virtual stream_if #(`PF_PTERESP_WIDTH) pte_resp; - keyed_scoreboard #(result_item) scoreboard; - chandle model; - bit live[int]; - int output_index[int]; - int cycle = 0, submitted = 0, retired = 0, segments = 0, failures[7] = '{default: 0}, peak = 0; - int - auth_count = 0, - pte_count = 0, - command_stalls = 0, - output_stalls = 0, - auth_stalls = 0, - cancelled = 0; - bit hold_output = 0, hold_permission = 0, auth_pending = 0, pte_pending = 0, pte_grant = 0; - int auth_id, pte_id, grant_id, auth_due, pte_due; - bit auth_allow, auth_pte; - u64 grant_pa; - logic [`PF_PERMIT_WIDTH-1:0] permit_packet; - logic [`PF_PTERESP_WIDTH-1:0] pte_packet; - u64 next_table = 'h10000; - function new(string name, uvm_component parent); - super.new(name, parent); - timeout = 1ms; - endfunction - function void verify_contract(bit ok, string message); - if (!ok) `uvm_fatal("PREFLIGHT", message) - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - model = preflight_ref_create(`PF_ADDRESS_BITS, `PF_BEAT_BYTES); - scoreboard = keyed_scoreboard#(result_item)::type_id::create("scoreboard", this); - verify_contract(uvm_config_db#(virtual preflight_control_if)::get(this, "", "ctl", ctl), - "Missing control"); - verify_contract(uvm_config_db#(virtual stream_if #(`PF_CMD_WIDTH))::get(this, "", "cmd", cmd), - "Missing command"); - verify_contract(uvm_config_db#(virtual stream_if #(`PF_OUT_WIDTH))::get( - this, "", "prepared", prepared), "Missing output"); - verify_contract(uvm_config_db#(virtual stream_if #(`PF_AUTH_WIDTH))::get( - this, "", "authorization", authorization), "Missing authorization"); - verify_contract(uvm_config_db#(virtual stream_if #(`PF_PERMIT_WIDTH))::get( - this, "", "permission", permission), "Missing permission"); - verify_contract(uvm_config_db#(virtual stream_if #(`PF_PTE_WIDTH))::get( - this, "", "pte_req", pte_req), "Missing PTE request"); - verify_contract(uvm_config_db#(virtual stream_if #(`PF_PTERESP_WIDTH))::get( - this, "", "pte_resp", pte_resp), "Missing PTE response"); - endfunction - function command_t request(int id, u64 base, int rows, int span, int row_stride, int cols = 1, - int col_stride = 0, bit write = 0, int mode = 0, u64 root = 0, - int privilege = 1, bit sum = 0, bit mxr = 0); - command_t q = '0; - q[`PF(CMD, ID)] = id; - q[`PF(CMD, BASEVA)] = base; - q[`PF(CMD, ROWS)] = rows; - q[`PF(CMD, COLUMNS)] = cols; - q[`PF(CMD, SPANBYTES)] = span; - q[`PF(CMD, ROWSTRIDE)] = row_stride; - q[`PF(CMD, COLUMNSTRIDE)] = col_stride; - q[`PF(CMD, WRITE)] = write; - q[`PF(CMD, MODE)] = mode; - q[`PF(CMD, ROOTPPN)] = root; - q[`PF(CMD, PRIVILEGE)] = privilege; - q[`PF(CMD, SUM)] = sum; - q[`PF(CMD, MXR)] = mxr; - return q; - endfunction - function u64 mapping(u64 va, int pages, u64 flags = 'hc7, int backend_error_page = -1); - u64 root = next_table; - next_table += 'h3000; - preflight_ref_program(model, root + ((va >> 30) & 511) * 8, ((root + 4096) >> 12) << 10 | 1, - 0); - preflight_ref_program(model, root + 4096 + ((va >> 21) & 511) * 8, - ((root + 8192) >> 12) << 10 | 1, 0); - for (int i = 0; i < pages; i++) begin - u64 a = (va & ~64'hfff) + i * 4096; - // Deliberately noncontiguous 4KiB physical pages; no identity-map shortcut. - preflight_ref_program(model, root + 8192 + ((a >> 12) & 511) * 8, - ((64'h800000 + i * 8192) >> 12) << 10 | flags, - i == backend_error_page); - end - return root >> 12; - endfunction - function void predict(command_t q); - int id = q[`PF(CMD, ID)]; - int count; - verify_contract(!live.exists(id), "Duplicate admitted command ID"); - verify_contract(preflight_ref_submit( - model, - id, - q[ - `PF(CMD, BASEVA) - ], - q[ - `PF(CMD, ROWS) - ], - q[ - `PF(CMD, COLUMNS) - ], - q[ - `PF(CMD, SPANBYTES) - ], - q[ - `PF(CMD, COLUMNSTRIDE) - ], - q[ - `PF(CMD, ROWSTRIDE) - ], - q[ - `PF(CMD, WRITE) - ], - q[ - `PF(CMD, MODE) - ], - q[ - `PF(CMD, ROOTPPN) - ], - q[ - `PF(CMD, PRIVILEGE) - ], - q[ - `PF(CMD, SUM) - ], - q[ - `PF(CMD, MXR) - ] - ) == 0, "Uninitialized PTE in reference fixture"); - live[id] = 1; - output_index[id] = 0; - submitted++; - if (live.num() > peak) peak = live.num(); - count = preflight_ref_count(model, id); - for (int i = 0; i < count; i++) begin - result_item expected = new; - u64 va, pa; - int unsigned bytes, write, last, error; - preflight_ref_expected(model, id, i, va, pa, bytes, write, last, error); - expected.packet = '0; - expected.packet[`PF(OUT, ID)] = id; - expected.packet[`PF(OUT, VA)] = va; - expected.packet[`PF(OUT, PA)] = pa; - expected.packet[`PF(OUT, BYTES)] = bytes; - expected.packet[`PF(OUT, WRITE)] = write; - expected.packet[`PF(OUT, LAST)] = last; - expected.packet[`PF(OUT, ERROR)] = error; - expected.set_transaction_id((id << 4) | i); - scoreboard.expected_export.write(expected); - end - endfunction - task service(); - forever begin - @(ctl.sample); - if (!ctl.sample.reset) begin - cycle++; - if (cmd.sample.valid && !cmd.sample.ready) command_stalls++; - if (prepared.sample.valid && !prepared.sample.ready) output_stalls++; - if (authorization.sample.valid && !authorization.sample.ready) auth_stalls++; - if (cmd.sample.valid && cmd.sample.ready) predict(cmd.sample.bits); - if (permission.sample.valid && permission.sample.ready) begin - verify_contract(auth_pending, "Unsolicited permission consumed"); - if (auth_pte && auth_allow) begin - pte_grant = 1; - grant_id = auth_id; - end - auth_pending = 0; - end - if (authorization.sample.valid && authorization.sample.ready) begin - int unsigned allow; - int id = authorization.sample.bits[`PF(AUTH, ID)]; - verify_contract(!$isunknown(authorization.sample.bits) && !auth_pending, - "Overlapping/unknown authorization"); - verify_contract(preflight_ref_authorize( - model, - id, - authorization.sample.bits[ - `PF(AUTH, PA) - ], - authorization.sample.bits[ - `PF(AUTH, BYTES) - ], - authorization.sample.bits[ - `PF(AUTH, WRITE) - ], - authorization.sample.bits[ - `PF(AUTH, ISPTE) - ], - authorization.sample.bits[ - `PF(AUTH, PRIVILEGE) - ], - allow - ) == 1, "Authorization lost snapshot, range or PTE S privilege"); - auth_pending = 1; - auth_id = id; - auth_allow = allow; - auth_pte = authorization.sample.bits[`PF(AUTH, ISPTE)]; - grant_pa = authorization.sample.bits[`PF(AUTH, PA)]; - auth_due = cycle + 3; - permit_packet = '0; - permit_packet[`PF(PERMIT, ID)] = id; - permit_packet[`PF(PERMIT, ALLOW)] = allow; - auth_count++; - end - if (pte_resp.sample.valid && pte_resp.sample.ready) pte_pending = 0; - if (pte_req.sample.valid && pte_req.sample.ready) begin - u64 value; - int unsigned error; - verify_contract(pte_grant && !pte_pending && !$isunknown(pte_req.sample.bits), - "PTE accessed before authorization"); - verify_contract(pte_req.sample.bits[`PF(PTE, ADDR)] == grant_pa && !pte_req.sample.bits[ - `PF(PTE, WRITE)] && pte_req.sample.bits[`PF(PTE, ATOMIC) - ] == 0 && !pte_req.sample.bits[`PF(PTE, ATOMICWORD) - ] && pte_req.sample.bits[`PF(PTE, MASK)] == 0, - "PTE request changed address or attempted writeback"); - verify_contract(preflight_ref_pte(model, grant_id, grant_pa, value, error) == 1, - "PTE address chain differs from architectural walker"); - pte_packet = '0; - pte_packet[`PF(PTERESP, DATA)] = value; - pte_packet[`PF(PTERESP, ERROR)] = error; - pte_id = grant_id; - pte_pending = 1; - pte_due = cycle + 2 + (pte_count % 4); - pte_grant = 0; - pte_count++; - end - if (prepared.sample.valid) begin - int id = prepared.sample.bits[`PF(OUT, ID)]; - verify_contract(live.exists(id) && !$isunknown(prepared.sample.bits), - "Unsolicited/unknown prepared plan"); - verify_contract( - preflight_ref_pending(model, id - ) == 0 && !(auth_pending && auth_id == id) && !(pte_pending && pte_id == id), - "Partial plan exposed before all translations/permissions completed"); - if (prepared.sample.ready) begin - result_item actual = new; - actual.packet = prepared.sample.bits; - actual.set_transaction_id((id << 4) | output_index[id]); - scoreboard.actual_export.write(actual); - output_index[id]++; - segments++; - if (prepared.sample.bits[`PF(OUT, LAST)]) begin - failures[int'(prepared.sample.bits[`PF(OUT, ERROR)])]++; - retired++; - live.delete(id); - output_index.delete(id); - preflight_ref_retire(model, id); - end - end - end - end - @(negedge ctl.clock); - authorization.ready = !ctl.reset && !auth_pending && cycle % 5 >= 2; - permission.valid = !ctl.reset && auth_pending && !hold_permission && cycle >= auth_due; - permission.bits = permit_packet; - pte_req.ready = !ctl.reset && !pte_pending && cycle % 4 != 0; - pte_resp.valid = !ctl.reset && pte_pending && cycle >= pte_due; - pte_resp.bits = pte_packet; - prepared.ready = !ctl.reset && !hold_output && cycle % 7 >= 3; - end - endtask - task submit(command_t q); - @(negedge ctl.clock); - cmd.bits = q; - cmd.valid = 1; - do @(cmd.sample); while (!cmd.sample.ready); - @(negedge ctl.clock); - cmd.valid = 0; - cmd.bits = ~q; - endtask - task drain(); - wait (live.num() == 0); - @(ctl.sample); - verify_contract(!auth_pending && !pte_pending && !pte_grant, - "Preparation endpoints did not drain"); - endtask - task execute(); - u64 root, root2; - int prior, gap_stride; - command_t q; - ctl.reset = 1; - cmd.valid = 0; - cmd.bits = '0; - prepared.ready = 0; - authorization.ready = 0; - permission.valid = 0; - permission.bits = '0; - pte_req.ready = 0; - pte_resp.valid = 0; - pte_resp.bits = '0; - repeat (5) @(negedge ctl.clock); - ctl.reset = 0; - fork - service(); - join_none - // Current BERT maximum: 1024 dense 16B rows normalize to five pages. - root = mapping('h400000f0, 9); - root2 = mapping('h50000000, 9); - hold_output = 1; - submit(request(1, 'h400000f0, 1024, 16, 16, 1, 0, 0, 8, root)); - submit(request(2, 'h50000000, 2048, 16, 16, 1, 0, 1, 8, root2)); // exactly eight pages - submit(request(3, 'h60000ff8, 1, 16, 16)); // alignment-expanded beat crosses page - submit(request(4, 'h70000000, 1, 16, 16)); - wait (prepared.sample.valid); - repeat (8) @(ctl.sample); - fork - submit(request(5, 'h70002000, 1, 16, 16)); - begin - repeat (12) @(ctl.sample); - hold_output = 0; - end - join - drain(); - submit(request(6, 'h50000000, 2304, 16, 16, 1, 0, 0, 8, root2)); - drain(); // ninth page: one error, zero successful output - submit(request(7, 'h60000000, 2, 16, 8192, 2, 4096)); - drain(); // sparse 2D, four extents - submit(request(8, 'h60000000, 1, 16, 16)); - drain(); // progress immediately after capacity error - root = mapping('h40010000, 1, 'h43); - submit(request(9, 'h40010000, 1, 16, 16, 1, 0, 1, 8, root)); - drain(); - root = mapping('h40020000, 1, 'h7); - submit(request(10, 'h40020000, 1, 16, 16, 1, 0, 0, 8, root)); - drain(); - root = mapping('h40030000, 1, 'hc7, 0); - submit(request(11, 'h40030000, 1, 16, 16, 1, 0, 0, 8, root)); - drain(); - root = mapping('h40040000, 1); - preflight_ref_policy(model, 1, (root << 12) + ((64'h40040000 >> 30) & 511) * 8, 0, 0, 0); - submit(request(12, 'h40040000, 1, 16, 16, 1, 0, 0, 8, root)); - drain(); - preflight_ref_policy(model, 0, 0, 1, 'h800000, 'h80000f); - submit(request(13, 'h40040000, 1, 16, 16, 1, 0, 0, 8, root)); - drain(); - preflight_ref_policy(model, 0, 0, 1, 'h71000000, 'h71000003); - // Denied bytes precede the user span: the aligned transport range is still authorized. - submit(request(14, 'h71000004, 1, 4, 4)); - drain(); - preflight_ref_policy(model, 0, 0, 0, 0, 0); - submit(request(15, (64'h1 << `PF_ADDRESS_BITS) - 16, 1, 32, 32)); - drain(); - submit(request(16, 64'hfffffffffffffff8, 1, 16, 16)); - drain(); - submit(request(17, 'h80000000, 0, 16, 16)); - drain(); - q = request(18, 'h80000000, 1, 16, 16); - q[`PF(CMD, MODE)] = 9; - submit(q); - drain(); - root = mapping('h40050000, 1, 'h59); - submit(request(19, 'h40050000, 1, 16, 16, 1, 0, 0, 8, root, 1, 1, 1)); - drain(); - // External snapshot service captures policy at command acceptance, not from later live settings. - hold_permission = 1; - submit(request(20, 'h72000000, 1, 16, 16)); - preflight_ref_policy(model, 0, 0, 1, 'h72000000, 'h7200000f); - wait (auth_pending); - repeat (8) @(ctl.sample); - hold_permission = 0; - drain(); - preflight_ref_policy(model, 0, 0, 0, 0, 0); - submit(request(21, 'h73000000, 1, 16, 256, 16, 16)); - drain(); // columns retains all 32 input bits - q = request(22, 64'hfffffffffffff000, -1, -1, -1, -1, -1); - submit(q); - drain(); - // Write authorization covers actual bytes; read authorization covers transport padding. - prior = failures[4]; - preflight_ref_policy(model, 0, 0, 1, 'h74000000, 'h74000003); - submit(request(23, 'h74000004, 1, 4, 4, 1, 0, 1)); - drain(); - verify_contract(failures[4] == prior, "write was denied for untouched prefix padding"); - submit(request(24, 'h74000004, 1, 4, 4)); - drain(); - verify_contract(failures[4] == prior + 1, "read failed to authorize actual prefix overread"); - prior = failures[4]; - preflight_ref_policy(model, 0, 0, 1, 'h74000108, 'h7400013f); - submit(request(25, 'h74000104, 1, 4, 4, 1, 0, 1)); - drain(); - verify_contract(failures[4] == prior, "write was denied for untouched tail padding"); - submit(request(26, 'h74000104, 1, 4, 4)); - drain(); - verify_contract(failures[4] == prior + 1, "read failed to authorize actual tail overread"); - prior = failures[4]; - preflight_ref_policy(model, 0, 0, 1, 'h74002ff0, 'h74002ffb); - submit(request(27, 'h74002ffc, 1, 8, 8, 1, 0, 1)); - drain(); - verify_contract(failures[4] == prior, "cross-page write included untouched bytes"); - submit(request(28, 'h74002ffc, 1, 8, 8)); - drain(); - verify_contract(failures[4] == prior + 1, "cross-page read omitted padded transport bytes"); - preflight_ref_policy(model, 0, 0, 0, 0, 0); - // MobileNet's gather has nine aligned segments on two noncontiguous PA pages. - // Every same-page gap must be explicitly authorized before the plan retains a union. - root = mapping('h74003e00, 2); - prior = failures[5]; - submit(request(29, 'h74003e00, 1, 16, 4032, 8, 72, 0, 8, root)); - drain(); - verify_contract(failures[5] == prior, "Authorized MobileNet gather exceeded range capacity"); - // Splitting preserves the exact accessed bytes and keeps each list within eight records. - prior = failures[5]; - submit(request(30, 'h74003e00, 1, 16, 4032, 4, 72, 0, 8, root)); - drain(); - submit(request(31, 'h74003f20, 1, 16, 4032, 4, 72, 0, 8, root)); - drain(); - verify_contract(failures[5] == prior, "Split MobileNet gather exceeded range capacity"); - // A denied hole must keep both individually legal spans, rather than poison the command. - gap_stride = `PF_BEAT_BYTES == 16 ? 72 : 2 * `PF_BEAT_BYTES + 8; - preflight_ref_policy(model, 0, 0, 1, 'h800e00 + `PF_BEAT_BYTES, - 'h800e00 + (gap_stride / `PF_BEAT_BYTES) * `PF_BEAT_BYTES - 1); - prior = failures[4]; - submit(request(32, 'h74003e00, 1, 16, 4032, 2, gap_stride, 0, 8, root)); - drain(); - verify_contract(failures[4] == prior, "Denied union rejected individually authorized pixels"); - // Denial of an actual transport byte must still yield one terminal fault, no partial plan. - preflight_ref_policy(model, 0, 0, 1, 'h800e00 + gap_stride, 'h800e00 + gap_stride + 7); - submit(request(33, 'h74003e00, 1, 16, 4032, 2, gap_stride, 0, 8, root)); - drain(); - verify_contract(failures[4] == prior + 1, - "Actual MobileNet pixel bytes escaped authorization"); - preflight_ref_policy(model, 0, 0, 0, 0, 0); - verify_contract( - peak==4&&submitted==retired&&command_stalls>0&&output_stalls>0&&auth_stalls>0, - "Missing concurrency/backpressure/drain coverage"); - for (int error = 1; error < 7; error++) - verify_contract(failures[error] > 0, "Missing typed error scenario"); - verify_contract(scoreboard.checked == segments, "Not every output descriptor was compared"); - `uvm_info( - "PREFLIGHT_PASS", - $sformatf( - "commands=%0d descriptors=%0d peak=%0d PTE=%0d auth=%0d errors shape/overflow/PF/AF/capacity/context=%0d/%0d/%0d/%0d/%0d/%0d stalls cmd/out/auth=%0d/%0d/%0d; all plans/permissions drained", - retired, segments, peak, pte_count, auth_count, failures[1], failures[2], failures[3], - failures[4], failures[5], failures[6], command_stalls, output_stalls, auth_stalls), - UVM_LOW) - endtask - function void final_phase(uvm_phase phase); - preflight_ref_destroy(model); - super.final_phase(phase); - endfunction - endclass -endpackage diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/preflight.f b/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/preflight.f deleted file mode 100644 index 23dc0b937..000000000 --- a/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/preflight.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Preflight/filelist.f -@RESOURCES@/preflight_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/preflight64.f b/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/preflight64.f deleted file mode 100644 index d6c9d5270..000000000 --- a/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/preflight64.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+PREFLIGHT_LINE_READ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Preflight64/filelist.f -@RESOURCES@/preflight_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/preflight_tb.sv b/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/preflight_tb.sv deleted file mode 100644 index 0453c4e45..000000000 --- a/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/preflight_tb.sv +++ /dev/null @@ -1,59 +0,0 @@ -`ifdef PREFLIGHT_LINE_READ -`include "preflight64_config.svh" -`define PREFLIGHT_TOP preflight64_tb -`else -`include "preflight_config.svh" -`define PREFLIGHT_TOP preflight_tb -`endif -module `PREFLIGHT_TOP; - import uvm_pkg::*; - import preflight_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - preflight_control_if ctl (clock); - stream_if #(`PF_CMD_WIDTH) cmd ( - clock, - ctl.reset - ); - stream_if #(`PF_OUT_WIDTH) prepared ( - clock, - ctl.reset - ); - stream_if #(`PF_AUTH_WIDTH) authorization ( - clock, - ctl.reset - ); - stream_if #(`PF_PERMIT_WIDTH) permission ( - clock, - ctl.reset - ); - stream_if #(`PF_PTE_WIDTH) pte_req ( - clock, - ctl.reset - ); - stream_if #(`PF_PTERESP_WIDTH) pte_resp ( - clock, - ctl.reset - ); - Preflight dut ( - .clock(clock), - .reset(ctl.reset), - `include "preflight_ports.svh" - ); - initial begin - uvm_config_db#(virtual preflight_control_if)::set(null, "uvm_test_top", "ctl", ctl); - uvm_config_db#(virtual stream_if #(`PF_CMD_WIDTH))::set(null, "uvm_test_top", "cmd", cmd); - uvm_config_db#(virtual stream_if #(`PF_OUT_WIDTH))::set(null, "uvm_test_top", "prepared", - prepared); - uvm_config_db#(virtual stream_if #(`PF_AUTH_WIDTH))::set(null, "uvm_test_top", "authorization", - authorization); - uvm_config_db#(virtual stream_if #(`PF_PERMIT_WIDTH))::set(null, "uvm_test_top", "permission", - permission); - uvm_config_db#(virtual stream_if #(`PF_PTE_WIDTH))::set(null, "uvm_test_top", "pte_req", - pte_req); - uvm_config_db#(virtual stream_if #(`PF_PTERESP_WIDTH))::set(null, "uvm_test_top", "pte_resp", - pte_resp); - run_test("protocol_test"); - end -endmodule -`undef PREFLIGHT_TOP diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/prepared_map.cm_hier b/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/prepared_map.cm_hier deleted file mode 100644 index f77ff296d..000000000 --- a/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/prepared_map.cm_hier +++ /dev/null @@ -1,5 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.reserve -+tree @TOP@.prepared -+tree @TOP@.mapped -+tree @TOP@.retire diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/prepared_map.f b/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/prepared_map.f deleted file mode 100644 index 2739ee5eb..000000000 --- a/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/prepared_map.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/prepared_map_control_if.sv -@RESOURCES@/prepared_map_package.sv --F @RTL@/PreparedMap/filelist.f -@RESOURCES@/prepared_map_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/prepared_map_control_if.sv b/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/prepared_map_control_if.sv deleted file mode 100644 index eb7f6d0c7..000000000 --- a/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/prepared_map_control_if.sv +++ /dev/null @@ -1,12 +0,0 @@ -`include "preflight_config.svh" -interface prepared_map_control_if ( - input logic clock -); - logic reset, hold_ready; - logic [`PF_QUERY_WIDTH-1:0] query_0, query_1; - logic [`PF_RESULT_WIDTH-1:0] result_0, result_1; - clocking sample @(posedge clock); - default input #1step; - input reset, query_0, query_1, result_0, result_1; - endclocking -endinterface diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/prepared_map_package.sv b/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/prepared_map_package.sv deleted file mode 100644 index 85a9280d1..000000000 --- a/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/prepared_map_package.sv +++ /dev/null @@ -1,336 +0,0 @@ -`include "preflight_config.svh" -`define PF(K, F) `PF_``K``_``F``_OFFSET +: `PF_``K``_``F``_WIDTH -package prepared_map_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - typedef longint unsigned u64; - import "DPI-C" function chandle pmap_ref_create(input int unsigned address_bits); - import "DPI-C" function void pmap_ref_destroy(input chandle model); - import "DPI-C" function void pmap_ref_reset(input chandle model); - import "DPI-C" function void pmap_ref_reserve( - input chandle model, - input int unsigned id - ); - import "DPI-C" function void pmap_ref_prepared( - input chandle model, - input int unsigned id, - input u64 va, - pa, - input int unsigned bytes, - write, - last, - error - ); - import "DPI-C" function int unsigned pmap_ref_ready( - input chandle model, - input int unsigned id, - fire, - output int unsigned error, - output u64 va - ); - import "DPI-C" function void pmap_ref_query( - input chandle model, - input int unsigned valid, - id, - input u64 va, - input int unsigned bytes, - write, - retiring, - retiring_id, - output int unsigned hit, - output u64 pa, - output int unsigned error - ); - import "DPI-C" function void pmap_ref_release( - input chandle model, - input int unsigned id - ); - import "DPI-C" function int unsigned pmap_ref_pending(input chandle model); - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual prepared_map_control_if ctl; - virtual stream_if #(`PF_RESERVE_WIDTH) reserve; - virtual stream_if #(`PF_OUT_WIDTH) prepared; - virtual stream_if #(`PF_MAP_READY_WIDTH) mapped; - virtual stream_if #(`PF_RELEASE_WIDTH) retire; - chandle model; - bit notified[int]; - int status[int]; - int - queries = 0, - hits = 0, - misses = 0, - allocations = 0, - notifications = 0, - retired = 0, - stalls = 0, - dual = 0, - cancelled = 0; - function new(string name, uvm_component parent); - super.new(name, parent); - timeout = 1ms; - endfunction - function void verify_contract(bit ok, string message); - if (!ok) `uvm_fatal("PREPARED_MAP", message) - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - verify_contract(uvm_config_db#(virtual prepared_map_control_if)::get(this, "", "ctl", ctl), - "ctl missing"); - verify_contract(uvm_config_db#(virtual stream_if #(`PF_RESERVE_WIDTH))::get( - this, "", "reserve", reserve), "reserve missing"); - verify_contract(uvm_config_db#(virtual stream_if #(`PF_OUT_WIDTH))::get( - this, "", "prepared", prepared), "prepared missing"); - verify_contract(uvm_config_db#(virtual stream_if #(`PF_MAP_READY_WIDTH))::get( - this, "", "mapped", mapped), "mapped missing"); - verify_contract(uvm_config_db#(virtual stream_if #(`PF_RELEASE_WIDTH))::get( - this, "", "retire", retire), "retire missing"); - model = pmap_ref_create(`PF_ADDRESS_BITS); - endfunction - function void query_check(logic [`PF_QUERY_WIDTH-1:0] q, logic [`PF_RESULT_WIDTH-1:0] actual); - int hit, error; - u64 pa; - pmap_ref_query(model, q[`PF(QUERY, VALID)], q[`PF(QUERY, ID)], q[`PF(QUERY, VA)], q[ - `PF(QUERY, BYTES)], q[`PF(QUERY, WRITE)], - retire.sample.valid && retire.sample.ready, retire.sample.bits[`PF(RELEASE, ID) - ], hit, pa, error); - verify_contract(actual[`PF(RESULT, HIT)] === 1'(hit) && actual[`PF(RESULT, PA) - ] === `PF_ADDRESS_BITS'(pa) && actual[`PF(RESULT, ERROR)] === 3'(error), - $sformatf( - "query id%0d VA%h bytes%0d expected hit%0d PA%h error%0d got %h", - q[ - `PF(QUERY, ID) - ], - q[ - `PF(QUERY, VA) - ], - q[ - `PF(QUERY, BYTES) - ], - hit, - pa, - error, - actual - )); - if (q[`PF(QUERY, VALID)]) begin - queries++; - if (hit) hits++; - else misses++; - end - endfunction - task monitor(); - forever begin - @(ctl.sample); - if (ctl.sample.reset) begin - pmap_ref_reset(model); - notified.delete(); - status.delete(); - end else begin - query_check(ctl.sample.query_0, ctl.sample.result_0); - query_check(ctl.sample.query_1, ctl.sample.result_1); - if (ctl.sample.query_0[`PF(QUERY, VALID)] && ctl.sample.query_1[`PF(QUERY, VALID)]) - dual++; - if (mapped.sample.valid) begin - int error, id; - u64 va; - id = mapped.sample.bits[`PF(MAP_READY, ID)]; - verify_contract(pmap_ref_ready(model, id, mapped.sample.ready, error, va) == 1, - "unexpected notification"); - verify_contract(mapped.sample.bits[`PF(MAP_READY, ERROR) - ] === 3'(error) && mapped.sample.bits[`PF(MAP_READY, VA)] === va, - "notification mismatch"); - if (mapped.sample.ready) begin - notified[id] = 1; - status[id] = error; - notifications++; - end else stalls++; - end - if (retire.sample.valid && retire.sample.ready) begin - int id; - id = retire.sample.bits[`PF(RELEASE, ID)]; - pmap_ref_release(model, id); - notified.delete(id); - retired++; - end - if (reserve.sample.valid && reserve.sample.ready) begin - pmap_ref_reserve(model, reserve.sample.bits[`PF(RESERVE, ID)]); - allocations++; - end - if (prepared.sample.valid && prepared.sample.ready) - pmap_ref_prepared(model, prepared.sample.bits[`PF(OUT, ID)], prepared.sample.bits[ - `PF(OUT, VA)], prepared.sample.bits[`PF(OUT, PA)], - prepared.sample.bits[`PF(OUT, BYTES)], prepared.sample.bits[ - `PF(OUT, WRITE)], prepared.sample.bits[`PF(OUT, LAST)], - prepared.sample.bits[`PF(OUT, ERROR)]); - end - end - endtask - task reserve_tag(int id); - @(negedge ctl.clock); - reserve.bits = '0; - reserve.bits[`PF(RESERVE, ID)] = id; - reserve.valid = 1; - do @(ctl.sample); while (!reserve.sample.ready); - @(negedge ctl.clock); - reserve.valid = 0; - endtask - task record(int id, u64 va, u64 pa, int bytes, bit write, bit last, int error = 0); - @(negedge ctl.clock); - prepared.bits = '0; - prepared.bits[`PF(OUT, ID)] = id; - prepared.bits[`PF(OUT, VA)] = va; - prepared.bits[`PF(OUT, PA)] = pa; - prepared.bits[`PF(OUT, BYTES)] = bytes; - prepared.bits[`PF(OUT, WRITE)] = write; - prepared.bits[`PF(OUT, LAST)] = last; - prepared.bits[`PF(OUT, ERROR)] = error; - prepared.valid = 1; - do @(ctl.sample); while (!prepared.sample.ready); - @(negedge ctl.clock); - prepared.valid = 0; - endtask - function logic [`PF_QUERY_WIDTH-1:0] query(int id, u64 va, int bytes, bit write); - logic [`PF_QUERY_WIDTH-1:0] q = '0; - q[`PF(QUERY, VALID)] = 1; - q[`PF(QUERY, ID)] = id; - q[`PF(QUERY, VA)] = va; - q[`PF(QUERY, BYTES)] = bytes; - q[`PF(QUERY, WRITE)] = write; - return q; - endfunction - task probes(logic [`PF_QUERY_WIDTH-1:0] a, logic [`PF_QUERY_WIDTH-1:0] b, int clocks = 3); - @(negedge ctl.clock); - ctl.query_0 = a; - ctl.query_1 = b; - repeat (clocks) @(ctl.sample); - @(negedge ctl.clock); - ctl.query_0 = '0; - ctl.query_1 = '0; - endtask - task wait_ready(int id, int error = 0); - while (!notified.exists(id)) @(negedge ctl.clock); - verify_contract(status[id] == error, "wrong terminal status"); - endtask - task retire_tag(int id, bit probe_release = 0); - @(negedge ctl.clock); - retire.bits = '0; - retire.bits[`PF(RELEASE, ID)] = id; - retire.valid = 1; - if (probe_release) begin - ctl.query_0 = query(id, 'h1010, 8, 0); - ctl.query_1 = query(2, 'h4004, 8, 1); - end - do @(ctl.sample); while (!retire.sample.ready); - @(negedge ctl.clock); - retire.valid = 0; - ctl.query_0 = '0; - ctl.query_1 = '0; - endtask - task execute(); - ctl.reset = 1; - ctl.hold_ready = 0; - ctl.query_0 = '0; - ctl.query_1 = '0; - reserve.valid = 0; - reserve.bits = '0; - prepared.valid = 0; - prepared.bits = '0; - retire.valid = 0; - retire.bits = '0; - fork - monitor(); - join_none - repeat (3) @(negedge ctl.clock); - ctl.reset = 0; - reserve_tag(1); - record(1, 'h1000, 'h8000, 4096, 0, 0); - probes(query(1, 'h1010, 16, 0), query(1, 'h1010, 16, 1)); - @(negedge ctl.clock); - ctl.hold_ready = 1; - record(1, 'h2000, 'hc000, 4096, 0, 1); - probes(query(1, 'h1010, 16, 0), query(1, 'h2010, 16, 0), 8); - @(negedge ctl.clock); - ctl.hold_ready = 0; - wait_ready(1); - probes(query(1, 'h1ffe, 2, 0), query(1, 'h2ffe, 2, 0)); - probes(query(1, 'h1fff, 2, 0), query(1, 'h3000, 4, 0)); - probes(query(1, 'h1000, 16, 1), query(1, 64'hffffffffffffffff, 2, 0)); - probes(query(1, 'h1000, 0, 0), query(99, 'h1000, 8, 0)); - reserve_tag(2); - record(2, 'h4000, 'h9000, 64, 1, 0); - record(2, 64'hffffffffffffffe0, (64'd1 << `PF_ADDRESS_BITS) - 32, 32, 1, 1); - wait_ready(2); - probes(query(1, 'h1f00, 16, 0), query(2, 'h4004, 16, 1)); - probes(query(2, 64'hfffffffffffffff8, 8, 1), query(2, 64'hfffffffffffffff8, 9, 1)); - retire_tag(1, 1); - probes(query(1, 'h1010, 16, 0), query(2, 'h4004, 16, 1)); - reserve_tag(1); - record(1, 'h6000, 'ha000, 128, 0, 1); - wait_ready(1); - probes(query(1, 'h1010, 16, 0), query(1, 'h6040, 64, 0)); - reserve_tag(3); - record(3, 'h7000, 'hb000, 64, 0, 0); - record(3, 'h8000, 0, 0, 0, 1, 3); - wait_ready(3, 3); - probes(query(3, 'h7004, 8, 0), query(3, 'h8000, 8, 0)); - reserve_tag(4); - fork - reserve_tag(5); - begin - repeat (6) begin - @(negedge ctl.clock); - verify_contract(!reserve.ready, "full table accepted fifth reserve"); - end - record(4, 'h9000, 0, 0, 0, 1, 4); - wait_ready(4, 4); - retire_tag(4); - end - join - for (int i = 0; i < 8; i++) begin - record(5, 'h10000 + i * 4096, 'h30000 + (7 - i) * 4096, 32, 0, i == 7); - end - wait_ready(5); - probes(query(5, 'h17018, 8, 0), query(5, 'h10008, 8, 0)); - retire_tag(1); - retire_tag(2); - retire_tag(3); - retire_tag(5); - // Retain an explicitly authorized same-page union and a discontiguous next PA page. - reserve_tag(10); - record(10, 'h74003e00, 'h800e00, 512, 0, 0); - record(10, 'h74004000, 'h802000, 16, 0, 1); - wait_ready(10); - probes(query(10, 'h74003e10, 16, 0), query(10, 'h74004000, 16, 0)); - probes(query(10, 'h74003df0, 16, 0), query(10, 'h74004010, 8, 0)); - probes(query(10, 'h74003ff8, 16, 0), query(10, 'h74003e48, 16, 1)); - retire_tag(10); - reserve_tag(9); - record(9, 'h9000, 'h19000, 64, 0, 1); - // Reset cancels a terminal notification still under backpressure. - @(negedge ctl.clock); - ctl.hold_ready = 1; - repeat (2) @(negedge ctl.clock); - cancelled += pmap_ref_pending(model); - ctl.reset = 1; - repeat (3) @(negedge ctl.clock); - ctl.reset = 0; - ctl.hold_ready = 0; - probes(query(9, 'h9000, 8, 0), query(1, 'h6000, 8, 0)); - repeat (3) @(negedge ctl.clock); - verify_contract(pmap_ref_pending(model) == 0, "mapping remained after retirement/reset"); - verify_contract(hits > 0 && misses > 0 && dual > 0 && stalls >= 4 && cancelled == 1, - "required checks did not execute"); - `uvm_info( - "PREPARED_MAP", - $sformatf( - "Checked %0d queries, %0d hits, %0d misses; reserve=%0d ready=%0d retire=%0d stalls=%0d dual=%0d reset_cancel=%0d", - queries, hits, misses, allocations, notifications, retired, stalls, dual, cancelled), - UVM_LOW) - endtask - function void final_phase(uvm_phase phase); - pmap_ref_destroy(model); - super.final_phase(phase); - endfunction - endclass -endpackage diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/prepared_map_tb.sv b/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/prepared_map_tb.sv deleted file mode 100644 index ab5d4426c..000000000 --- a/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/prepared_map_tb.sv +++ /dev/null @@ -1,42 +0,0 @@ -`include "preflight_config.svh" -module prepared_map_tb; - import uvm_pkg::*; - import prepared_map_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - prepared_map_control_if ctl (clock); - stream_if #(`PF_RESERVE_WIDTH) reserve ( - clock, - ctl.reset - ); - stream_if #(`PF_OUT_WIDTH) prepared ( - clock, - ctl.reset - ); - stream_if #(`PF_MAP_READY_WIDTH) mapped ( - clock, - ctl.reset - ); - stream_if #(`PF_RELEASE_WIDTH) retire ( - clock, - ctl.reset - ); - assign mapped.ready = !ctl.reset && !ctl.hold_ready; - PreparedMap dut ( - .clock(clock), - .reset(ctl.reset), - `include "prepared_map_ports.svh" - ); - initial begin - uvm_config_db#(virtual prepared_map_control_if)::set(null, "uvm_test_top", "ctl", ctl); - uvm_config_db#(virtual stream_if #(`PF_RESERVE_WIDTH))::set(null, "uvm_test_top", "reserve", - reserve); - uvm_config_db#(virtual stream_if #(`PF_OUT_WIDTH))::set(null, "uvm_test_top", "prepared", - prepared); - uvm_config_db#(virtual stream_if #(`PF_MAP_READY_WIDTH))::set(null, "uvm_test_top", "mapped", - mapped); - uvm_config_db#(virtual stream_if #(`PF_RELEASE_WIDTH))::set(null, "uvm_test_top", "retire", - retire); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/main/scala/Emit.scala b/arch/src/main/scala/framework/mem-core/preflight/src/main/scala/Emit.scala deleted file mode 100644 index d09224e96..000000000 --- a/arch/src/main/scala/framework/mem-core/preflight/src/main/scala/Emit.scala +++ /dev/null @@ -1,94 +0,0 @@ -package memcore.memory.preflight - -import chisel3._ -import java.nio.file.{Files, Paths} -import memcore.bus.chi.rnf.{CacheAccess, CacheResult} - -object Emit extends App { - val p = Params() - Files.createDirectories(Paths.get("build")) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Preflight(p), - firtoolOpts = args, - args = Array("--target-dir", "build/Preflight", "--split-verilog") - ) - - val defs = scala.collection.mutable.ArrayBuffer[String]( - s"`define PF_ADDRESS_BITS ${p.bus.addressBits}", - s"`define PF_BEAT_BYTES ${p.beatBytes}" - ) - - val ports = scala.collection.mutable.ArrayBuffer[String]() - - def channel( - kind: String, - prefix: String, - signal: String, - bundle: Bundle - ): Unit = { - var offset = 0 - for ((name, field) <- bundle.elements.toSeq) { - val width = field.getWidth - defs += s"`define PF_${kind}_${name.toUpperCase}_OFFSET $offset" - defs += s"`define PF_${kind}_${name.toUpperCase}_WIDTH $width" - // firtool drops zero-width fields from the module ports. - if (width > 0) ports += s".${prefix}_bits_$name($signal.bits[$offset +: $width])" - offset += width - } - defs += s"`define PF_${kind}_WIDTH $offset" - ports += s".${prefix}_valid($signal.valid)"; ports += s".${prefix}_ready($signal.ready)" - } - - channel("CMD", "io_command", "cmd", new Command(p)) - channel("OUT", "io_prepared", "prepared", new PreparedSegment(p)) - channel("AUTH", "io_authorization", "authorization", new Authorization(p)) - channel("PERMIT", "io_permission", "permission", new Permission(p)) - channel("PTE", "io_pteRequest", "pte_req", new CacheAccess(p.bus)) - channel("PTERESP", "io_pteResponse", "pte_resp", new CacheResult) - Files.writeString(Paths.get("build/preflight_config.svh"), defs.mkString("\n") + "\n") - Files.writeString(Paths.get("build/preflight_ports.svh"), ports.mkString(",\n") + "\n") - val lineRead = p.copy(beatBytes = 64) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Preflight(lineRead), - firtoolOpts = args, - args = Array("--target-dir", "build/Preflight64", "--split-verilog") - ) - Files.writeString( - Paths.get("build/preflight64_config.svh"), - defs.map(d => if (d.startsWith("`define PF_BEAT_BYTES ")) s"`define PF_BEAT_BYTES ${lineRead.beatBytes}" else d) - .mkString("\n") + "\n" - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new PreparedMap(p), - firtoolOpts = args, - args = Array("--target-dir", "build/PreparedMap", "--split-verilog") - ) - ports.clear() - channel("RESERVE", "io_reserve", "reserve", new MapTag(p)) - channel("MAP_READY", "io_ready", "mapped", new MapReady(p)) - channel("RELEASE", "io_release", "retire", new MapTag(p)) - // PreparedSegment uses the existing PF_OUT layout. - var preparedOffset = 0 - for ((name, field) <- (new PreparedSegment(p)).elements.toSeq) { - ports += s".io_prepared_bits_$name(prepared.bits[$preparedOffset +: ${field.getWidth}])" - preparedOffset += field.getWidth - } - ports += ".io_prepared_valid(prepared.valid)"; ports += ".io_prepared_ready(prepared.ready)" - for ( - (kind, prefix, signal, bundle) <- Seq( - ("QUERY", "io_queries", "query", new MapQuery(p)), - ("RESULT", "io_results", "result", new MapResult(p)) - ) - ) { - var offset = 0 - for ((name, field) <- bundle.elements.toSeq) { - defs += s"`define PF_${kind}_${name.toUpperCase}_OFFSET $offset" - defs += s"`define PF_${kind}_${name.toUpperCase}_WIDTH ${field.getWidth}" - for (port <- 0 until 2) ports += s".${prefix}_${port}_$name(ctl.${signal}_$port[$offset +: ${field.getWidth}])" - offset += field.getWidth - } - defs += s"`define PF_${kind}_WIDTH $offset" - } - Files.writeString(Paths.get("build/preflight_config.svh"), defs.mkString("\n") + "\n") - Files.writeString(Paths.get("build/prepared_map_ports.svh"), ports.mkString(",\n") + "\n") -} diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/main/scala/Interface.scala b/arch/src/main/scala/framework/mem-core/preflight/src/main/scala/Interface.scala deleted file mode 100644 index c2ec8605d..000000000 --- a/arch/src/main/scala/framework/mem-core/preflight/src/main/scala/Interface.scala +++ /dev/null @@ -1,50 +0,0 @@ -package memcore.memory.preflight - -import chisel3._ - -/** Shape describes transport bytes, including any read beyond the user's useful payload. */ -class Command(p: Params) extends Bundle { - val id = UInt(p.idBits.W) - val baseVA = UInt(64.W) - val rows = UInt(32.W) - val columns = UInt(32.W) - val spanBytes = UInt(32.W) - val columnStride = UInt(32.W) - val rowStride = UInt(32.W) - val write = Bool() - val mode = UInt(4.W) - val rootPpn = UInt(44.W) - val privilege = UInt(2.W) - val sum = Bool() - val mxr = Bool() -} - -object Error { - val Ok = 0; val Shape = 1; val Overflow = 2; val PageFault = 3 - val AccessFault = 4; val Capacity = 5; val Context = 6 -} - -/** Failure returns one terminal zero-PA record. The caller retains successful maps and IDs until DMA retirement. */ -class PreparedSegment(p: Params) extends Bundle { - val id = UInt(p.idBits.W) - val va = UInt(64.W) - val pa = UInt(p.bus.addressBits.W) - val bytes = UInt(13.W) - val write = Bool() - val last = Bool() - val error = UInt(3.W) -} - -class Authorization(p: Params) extends Bundle { - val id = UInt(p.idBits.W) - val pa = UInt(p.bus.addressBits.W) - val bytes = UInt(13.W) - val write = Bool() - val isPte = Bool() - val privilege = UInt(2.W) -} - -class Permission(p: Params) extends Bundle { - val id = UInt(p.idBits.W) - val allow = Bool() -} diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/main/scala/Params.scala b/arch/src/main/scala/framework/mem-core/preflight/src/main/scala/Params.scala deleted file mode 100644 index 8651b8464..000000000 --- a/arch/src/main/scala/framework/mem-core/preflight/src/main/scala/Params.scala +++ /dev/null @@ -1,15 +0,0 @@ -package memcore.memory.preflight - -import memcore.bus.chi - -case class Params( - bus: chi.Params = chi.Params(), - contexts: Int = 4, - maxRanges: Int = 8, - idBits: Int = 8, - beatBytes: Int = 16) { - require(contexts >= 2 && (contexts & (contexts - 1)) == 0) - require(maxRanges == 8) - require(idBits > 0 && bus.addressBits >= 12 && bus.addressBits < 64) - require(beatBytes >= 8 && beatBytes <= 64 && (beatBytes & (beatBytes - 1)) == 0) -} diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/main/scala/Preflight.scala b/arch/src/main/scala/framework/mem-core/preflight/src/main/scala/Preflight.scala deleted file mode 100644 index 162e94943..000000000 --- a/arch/src/main/scala/framework/mem-core/preflight/src/main/scala/Preflight.scala +++ /dev/null @@ -1,236 +0,0 @@ -package memcore.memory.preflight - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import memcore.bus.chi.rnf.{CacheAccess, CacheResult} -import memcore.memory.mmu.Walker - -/** Preparation only. A consumer retains the entire successful mapping until DMA retires. */ -@instantiable -class Preflight(p: Params) extends Module { - val contextBits = log2Ceil(p.contexts) - - @public - val io = IO(new Bundle { - val command = Flipped(Decoupled(new Command(p))) - val prepared = Decoupled(new PreparedSegment(p)) - val authorization = Decoupled(new Authorization(p)) - val permission = Flipped(Decoupled(new Permission(p))) - val pteRequest = Decoupled(new CacheAccess(p.bus)) - val pteResponse = Flipped(Decoupled(new CacheResult)) - }) - - val free :: queued :: working :: ready :: Nil = Enum(4) - val phase = RegInit(VecInit(Seq.fill(p.contexts)(free))) - val commands = Reg(Vec(p.contexts, new Command(p))) - val counts = RegInit(VecInit(Seq.fill(p.contexts)(0.U(4.W)))) - val errors = Reg(Vec(p.contexts, UInt(3.W))) - val faultVA = Reg(Vec(p.contexts, UInt(64.W))) - val vas = Reg(Vec(p.contexts, Vec(p.maxRanges, UInt(64.W)))) - val pas = Reg(Vec(p.contexts, Vec(p.maxRanges, UInt(p.bus.addressBits.W)))) - val sizes = Reg(Vec(p.contexts, Vec(p.maxRanges, UInt(13.W)))) - val available = VecInit(phase.map(_ === free)) - val duplicate = - (0 until p.contexts).map(i => phase(i) =/= free && commands(i).id === io.command.bits.id).reduce(_ || _) - io.command.ready := available.asUInt.orR && !duplicate && !reset.asBool - when(io.command.fire) { - val slot = PriorityEncoder(available.asUInt) - commands(slot) := io.command.bits; counts(slot) := 0.U; errors(slot) := Error.Ok.U; phase(slot) := queued - } - - def select(candidates: Vec[Bool], cursor: UInt): UInt = { - MuxCase( - 0.U(contextBits.W), - (0 until p.contexts).map { off => - val index = (cursor + off.U)(contextBits - 1, 0) - candidates(index) -> index - } - ) - } - - val idle :: checkShape :: span :: sendSpan :: segment :: walk :: waitWalk :: authorize :: waitPermission :: save :: Nil = - Enum(10) - val state = RegInit(idle) - val worker = Reg(UInt(contextBits.W)) - val cursor = RegInit(0.U(contextBits.W)) - val row = Reg(UInt(32.W)); val column = Reg(UInt(32.W)) - val dense = Reg(Bool()); val wholeBytes = Reg(UInt(32.W)) - val spanAddress = Reg(UInt(64.W)); val spanBytes = Reg(UInt(32.W)) - val spanLast = Reg(Bool()) - val currentVA = Reg(UInt(64.W)); val currentPA = Reg(UInt(p.bus.addressBits.W)) - val currentBytes = Reg(UInt(13.W)); val segmentLast = Reg(Bool()) - val unionAuthorization = RegInit(false.B) - val previous = (counts(worker) - 1.U)(2, 0) - val previousVA = vas(worker)(previous) - val previousPA = pas(worker)(previous) - val previousEnd = previousVA +& sizes(worker)(previous) - val currentEnd = currentVA +& currentBytes - val cancelSegmenter = WireDefault(false.B) - val splitter: Instance[Segmenter] = Instantiate(new Segmenter(4096, 4096)) - splitter.io.cancel := cancelSegmenter - val walker: Instance[Walker] = Instantiate(new Walker(p.bus)) - val cmd = commands(worker) - - def finish(code: UInt, address: UInt): Unit = { - errors(worker) := code; faultVA(worker) := address; phase(worker) := ready; state := idle - when(code =/= Error.Ok.U) { counts(worker) := 0.U; cancelSegmenter := true.B } - } - - val waiting = VecInit(phase.map(_ === queued)) - when(state === idle && waiting.asUInt.orR) { - val slot = select(waiting, cursor) - worker := slot; cursor := slot + 1.U; phase(slot) := working; state := checkShape - } - when(state === checkShape) { - val rowExtent = cmd.columns * cmd.spanBytes - val total = cmd.rows * rowExtent - val isDense = (cmd.columns === 1.U || cmd.columnStride === cmd.spanBytes) && - (cmd.rows === 1.U || cmd.rowStride === rowExtent) - val end = cmd.baseVA.pad(67) + ((cmd.rows - 1.U) * cmd.rowStride).pad(67) + - ((cmd.columns - 1.U) * cmd.columnStride).pad(67) + cmd.spanBytes.pad(67) - 1.U - when(cmd.rows === 0.U || cmd.columns === 0.U || cmd.spanBytes === 0.U || - (cmd.rows > 1.U && cmd.rowStride === 0.U) || (cmd.columns > 1.U && cmd.columnStride === 0.U)) { - finish(Error.Shape.U, cmd.baseVA) - }.elsewhen(cmd.mode =/= 0.U && cmd.mode =/= 8.U || cmd.privilege === 2.U) { - finish(Error.Context.U, cmd.baseVA) - }.elsewhen(end(66, 64).orR || (isDense && total > "hffffffff".U)) { - finish(Error.Overflow.U, cmd.baseVA) - }.otherwise { - dense := isDense; wholeBytes := total(31, 0); row := 0.U; column := 0.U; state := span - } - } - when(state === span) { - val start = cmd.baseVA.pad(67) + (row * cmd.rowStride).pad(67) + (column * cmd.columnStride).pad(67) - val bytes = Mux(dense, wholeBytes, cmd.spanBytes) - val mask = (~(BigInt(p.beatBytes) - 1) & ((BigInt(1) << 67) - 1)).U(67.W) - val first = Mux(cmd.write, start, start & mask) - val end = Mux(cmd.write, start + bytes.pad(67), (start + bytes.pad(67) + (p.beatBytes - 1).U) & mask) - val length = end - first - when(length > "hffffffff".U)(finish(Error.Overflow.U, start(63, 0))) - .otherwise { - spanAddress := first(63, 0); spanBytes := length(31, 0) - spanLast := dense || (row === cmd.rows - 1.U && column === cmd.columns - 1.U) - state := sendSpan - } - } - splitter.io.descriptor.valid := state === sendSpan - splitter.io.descriptor.bits.addr := spanAddress - splitter.io.descriptor.bits.bytes := spanBytes - splitter.io.descriptor.bits.rowBytes := spanBytes - splitter.io.descriptor.bits.rowStride := spanBytes - when(splitter.io.descriptor.fire)(state := segment) - splitter.io.segment.ready := state === segment - when(splitter.io.segment.fire) { - currentVA := splitter.io.segment.bits.addr; currentBytes := splitter.io.segment.bits.bytes(12, 0) - segmentLast := splitter.io.segment.bits.last; state := walk - } - walker.io.flush := false.B - walker.io.lookup.config := 0.U.asTypeOf(walker.io.lookup.config) - walker.io.lookup.req := 0.U.asTypeOf(walker.io.lookup.req) - walker.io.config.mode := cmd.mode; walker.io.config.rootPpn := cmd.rootPpn - walker.io.req.valid := state === walk - walker.io.req.bits.vaddr := currentVA; walker.io.req.bits.write := cmd.write; walker.io.req.bits.execute := false.B - walker.io.req.bits.privilege := cmd.privilege; walker.io.req.bits.sum := cmd.sum; walker.io.req.bits.mxr := cmd.mxr - when(walker.io.req.fire)(state := waitWalk) - walker.io.resp.ready := state === waitWalk - when(walker.io.resp.fire) { - val end = walker.io.resp.bits.paddr.pad(p.bus.addressBits + 2) + currentBytes.pad(p.bus.addressBits + 2) - 1.U - when(walker.io.resp.bits.pageFault)(finish(Error.PageFault.U, currentVA)) - .elsewhen(walker.io.resp.bits.accessFault || end(p.bus.addressBits + 1, p.bus.addressBits).orR) { - finish(Error.AccessFault.U, currentVA) - }.otherwise { - currentPA := walker.io.resp.bits.paddr - unionAuthorization := counts(worker) =/= 0.U && previousVA(63, 12) === currentVA(63, 12) && - currentVA > previousEnd && previousPA +& (currentVA - previousVA) === walker.io.resp.bits.paddr - state := authorize - } - } - val pteIdle :: pteAsk :: pteWait :: pteIssue :: pteReturn :: pteReject :: pteRejectReturn :: Nil = Enum(7) - val pteState = RegInit(pteIdle) - when(pteState === pteIdle && walker.io.access.valid)(pteState := pteAsk) - io.authorization.valid := (pteState === pteAsk || state === authorize) && !reset.asBool - io.authorization.bits.id := cmd.id - io.authorization.bits.isPte := pteState === pteAsk - io.authorization.bits.pa := Mux( - pteState === pteAsk, - walker.io.access.bits.addr, - Mux(unionAuthorization, previousPA, currentPA) - ) - io.authorization.bits.bytes := Mux( - pteState === pteAsk, - 8.U, - Mux(unionAuthorization, currentEnd - previousVA, currentBytes) - ) - io.authorization.bits.write := pteState =/= pteAsk && cmd.write - io.authorization.bits.privilege := Mux(pteState === pteAsk, 1.U, cmd.privilege) - when(io.authorization.fire) { - when(pteState === pteAsk)(pteState := pteWait).otherwise(state := waitPermission) - } - io.permission.ready := (pteState === pteWait || state === waitPermission) && !reset.asBool - when(io.permission.fire) { - assert(io.permission.bits.id === cmd.id, "Preflight permission response tag mismatch") - when(pteState === pteWait)(pteState := Mux(io.permission.bits.allow, pteIssue, pteReject)) - .otherwise { - when(io.permission.bits.allow)(state := save) - .elsewhen(unionAuthorization) { - unionAuthorization := false.B - state := authorize - }.otherwise(finish(Error.AccessFault.U, currentVA)) - } - } - io.pteRequest.valid := pteState === pteIssue && walker.io.access.valid && !reset.asBool - io.pteRequest.bits := walker.io.access.bits - walker.io.access.ready := pteState === pteReject || pteState === pteIssue && io.pteRequest.ready - when(walker.io.access.fire)(pteState := Mux(pteState === pteReject, pteRejectReturn, pteReturn)) - walker.io.result.valid := pteState === pteRejectReturn || pteState === pteReturn && io.pteResponse.valid - walker.io.result.bits := 0.U.asTypeOf(walker.io.result.bits) - walker.io.result.bits.data := Mux(pteState === pteRejectReturn, 0.U, io.pteResponse.bits.data) - walker.io.result.bits.error := pteState === pteRejectReturn || io.pteResponse.bits.error - io.pteResponse.ready := pteState === pteReturn && walker.io.result.ready && !reset.asBool - when(walker.io.result.fire)(pteState := pteIdle) - // Touching spans already cover their union. Gapped spans merge only after that whole - // same-page physical union was explicitly authorized; a denied union keeps exact spans. - val merge = counts(worker) =/= 0.U && previousVA(63, 12) === currentVA(63, 12) && - currentVA >= previousVA && (currentVA <= previousEnd || unionAuthorization) - val saveCapacity = merge || counts(worker) =/= p.maxRanges.U - when(state === save && !saveCapacity)(finish(Error.Capacity.U, currentVA)) - when(state === save && saveCapacity) { - when(merge) { - assert( - pas(worker)(previous) +& (currentVA - previousVA) === currentPA, - "Preflight same-page ranges disagree on PA" - ) - when(currentEnd > previousEnd)(sizes(worker)(previous) := currentEnd - previousVA) - }.otherwise { - val index = counts(worker)(2, 0) - vas(worker)(index) := currentVA; pas(worker)(index) := currentPA; sizes(worker)(index) := currentBytes - counts(worker) := counts(worker) + 1.U - } - when(segmentLast && spanLast)(finish(Error.Ok.U, currentVA)) - .elsewhen(segmentLast) { - when(column === cmd.columns - 1.U) { column := 0.U; row := row + 1.U } - .otherwise(column := column + 1.U) - state := span - }.otherwise(state := segment) - } - val offer = RegInit(false.B); val outputSlot = Reg(UInt(contextBits.W)); val outputIndex = Reg(UInt(3.W)) - val outputCursor = RegInit(0.U(contextBits.W)) - val prepared = VecInit(phase.map(_ === ready)) - when(!offer && prepared.asUInt.orR) { - offer := true.B; outputSlot := select(prepared, outputCursor); outputIndex := 0.U - } - io.prepared.valid := offer && !reset.asBool - val failed = errors(outputSlot) =/= Error.Ok.U - io.prepared.bits.id := commands(outputSlot).id - io.prepared.bits.va := Mux(failed, faultVA(outputSlot), vas(outputSlot)(outputIndex)) - io.prepared.bits.pa := Mux(failed, 0.U, pas(outputSlot)(outputIndex)) - io.prepared.bits.bytes := Mux(failed, 0.U, sizes(outputSlot)(outputIndex)) - io.prepared.bits.write := commands(outputSlot).write - io.prepared.bits.error := errors(outputSlot) - io.prepared.bits.last := failed || outputIndex === counts(outputSlot) - 1.U - when(io.prepared.fire) { - when(io.prepared.bits.last) { phase(outputSlot) := free; offer := false.B; outputCursor := outputSlot + 1.U } - .otherwise(outputIndex := outputIndex + 1.U) - } -} diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/main/scala/PreparedMap.scala b/arch/src/main/scala/framework/mem-core/preflight/src/main/scala/PreparedMap.scala deleted file mode 100644 index 33e243798..000000000 --- a/arch/src/main/scala/framework/mem-core/preflight/src/main/scala/PreparedMap.scala +++ /dev/null @@ -1,160 +0,0 @@ -package memcore.memory.preflight - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} - -class MapTag(p: Params) extends Bundle { val id = UInt(p.idBits.W) } - -class MapReady(p: Params) extends Bundle { - val id = UInt(p.idBits.W) - val error = UInt(3.W) - val va = UInt(64.W) -} - -class MapQuery(p: Params) extends Bundle { - val valid = Bool() - val id = UInt(p.idBits.W) - val va = UInt(64.W) - val bytes = UInt(13.W) - val write = Bool() -} - -class MapResult(p: Params) extends Bundle { - val hit = Bool() - val pa = UInt(p.bus.addressBits.W) - val error = UInt(3.W) -} - -@instantiable -class PreparedMap(p: Params) extends Module { - - @public - val io = IO(new Bundle { - val reserve = Flipped(Decoupled(new MapTag(p))) - val prepared = Flipped(Decoupled(new PreparedSegment(p))) - val ready = Decoupled(new MapReady(p)) - val release = Flipped(Decoupled(new MapTag(p))) - val queries = Input(Vec(2, new MapQuery(p))) - val results = Output(Vec(2, new MapResult(p))) - }) - - val slotBits = log2Ceil(p.contexts) - val live = RegInit(VecInit(Seq.fill(p.contexts)(false.B))) - val sealedList = RegInit(VecInit(Seq.fill(p.contexts)(false.B))) - val notified = RegInit(VecInit(Seq.fill(p.contexts)(false.B))) - val tags = Reg(Vec(p.contexts, UInt(p.idBits.W))) - val counts = RegInit(VecInit(Seq.fill(p.contexts)(0.U(log2Ceil(p.maxRanges + 1).W)))) - val errors = RegInit(VecInit(Seq.fill(p.contexts)(Error.Ok.U(3.W)))) - val faultVA = Reg(Vec(p.contexts, UInt(64.W))) - val direction = Reg(Vec(p.contexts, Bool())) - val vas = Reg(Vec(p.contexts, Vec(p.maxRanges, UInt(64.W)))) - val pas = Reg(Vec(p.contexts, Vec(p.maxRanges, UInt(p.bus.addressBits.W)))) - val sizes = Reg(Vec(p.contexts, Vec(p.maxRanges, UInt(13.W)))) - - def matches(id: UInt): Vec[Bool] = VecInit((0 until p.contexts).map(i => live(i) && tags(i) === id)) - val free = VecInit(live.map(!_)) - val reserveMatch = matches(io.reserve.bits.id) - io.reserve.ready := free.asUInt.orR && !reserveMatch.asUInt.orR && !reset.asBool - when(io.reserve.fire) { - val slot = PriorityEncoder(free) - live(slot) := true.B; tags(slot) := io.reserve.bits.id - sealedList(slot) := false.B; notified(slot) := false.B - counts(slot) := 0.U; errors(slot) := Error.Ok.U; faultVA(slot) := 0.U - } - - val preparedMatch = matches(io.prepared.bits.id) - val preparedSlot = PriorityEncoder(preparedMatch) - io.prepared.ready := preparedMatch.asUInt.orR && !sealedList(preparedSlot) && !reset.asBool - val reservingPrepared = io.reserve.fire && io.reserve.bits.id === io.prepared.bits.id - when(io.prepared.valid && !reset.asBool && !reservingPrepared) { - assert(preparedMatch.asUInt.orR, "PreparedMap record has no reservation") - assert(!sealedList(preparedSlot), "PreparedMap record follows the terminal record") - } - when(io.prepared.fire) { - val record = io.prepared.bits - val lastVA = record.va +& (record.bytes - 1.U) - val lastPA = record.pa +& (record.bytes - 1.U) - val shapeOK = record.bytes =/= 0.U && record.bytes <= 4096.U && - (record.va(11, 0) +& record.bytes) <= 4096.U && - (record.pa(11, 0) +& record.bytes) <= 4096.U && - !lastVA(64) && !lastPA(p.bus.addressBits) - val capacityOK = counts(preparedSlot) < p.maxRanges.U && - (counts(preparedSlot) =/= (p.maxRanges - 1).U || record.last) - val directionOK = counts(preparedSlot) === 0.U || direction(preparedSlot) === record.write - when(record.error =/= Error.Ok.U) { - assert(record.last, "PreparedMap failure must be terminal") - errors(preparedSlot) := record.error; faultVA(preparedSlot) := record.va - sealedList(preparedSlot) := true.B; counts(preparedSlot) := 0.U - }.otherwise { - assert(shapeOK, "PreparedMap successful record must fit one VA and PA page") - assert(capacityOK, "PreparedMap ranges must seal within capacity") - assert(directionOK, "PreparedMap ranges must have one direction") - when(!shapeOK || !capacityOK || !directionOK) { - errors(preparedSlot) := Mux(!capacityOK, Error.Capacity.U, Error.Shape.U) - faultVA(preparedSlot) := record.va; sealedList(preparedSlot) := true.B - counts(preparedSlot) := 0.U - }.otherwise { - val index = counts(preparedSlot)(log2Ceil(p.maxRanges) - 1, 0) - vas(preparedSlot)(index) := record.va; pas(preparedSlot)(index) := record.pa - sizes(preparedSlot)(index) := record.bytes; direction(preparedSlot) := record.write - counts(preparedSlot) := counts(preparedSlot) + 1.U - when(record.last)(sealedList(preparedSlot) := true.B) - } - } - } - - val offer = RegInit(false.B) - val offeredSlot = Reg(UInt(slotBits.W)) - val awaiting = VecInit((0 until p.contexts).map(i => live(i) && sealedList(i) && !notified(i))) - when(!offer && awaiting.asUInt.orR) { - offer := true.B; offeredSlot := PriorityEncoder(awaiting) - } - io.ready.valid := offer && !reset.asBool - io.ready.bits.id := tags(offeredSlot) - io.ready.bits.error := errors(offeredSlot) - io.ready.bits.va := faultVA(offeredSlot) - when(io.ready.fire) { notified(offeredSlot) := true.B; offer := false.B } - - val releaseMatch = matches(io.release.bits.id) - val releaseSlot = PriorityEncoder(releaseMatch) - io.release.ready := releaseMatch.asUInt.orR && notified(releaseSlot) && !reset.asBool - when(io.release.fire) { - live(releaseSlot) := false.B; sealedList(releaseSlot) := false.B - notified(releaseSlot) := false.B; counts(releaseSlot) := 0.U - errors(releaseSlot) := Error.Ok.U - } - - for (port <- 0 until 2) { - val query = io.queries(port) - val owner = matches(query.id) - val slot = PriorityEncoder(owner) - val retiring = io.release.fire && io.release.bits.id === query.id - val last = query.va +& (query.bytes - 1.U) - val shapeOK = query.bytes =/= 0.U && !last(64) && - (query.va(11, 0) +& query.bytes) <= 4096.U - val authorized = owner.asUInt.orR && sealedList(slot) && notified(slot) && - errors(slot) === Error.Ok.U && !retiring && !reset.asBool - val hits = VecInit((0 until p.maxRanges).map { index => - index.U < counts(slot) && query.va >= vas(slot)(index) && - last < (vas(slot)(index) +& sizes(slot)(index)) && direction(slot) === query.write - }) - val index = PriorityEncoder(hits) - val translated = pas(slot)(index) +& (query.va - vas(slot)(index)) - val hit = query.valid && shapeOK && authorized && hits.asUInt.orR - io.results(port).hit := hit - io.results(port).pa := Mux(hit, translated(p.bus.addressBits - 1, 0), 0.U) - io.results(port).error := Mux(!query.valid || hit, Error.Ok.U, Mux(!shapeOK, Error.Shape.U, Error.AccessFault.U)) - when(hit) { - assert(!translated(translated.getWidth - 1, p.bus.addressBits).orR, "PreparedMap translated address overflows PA") - for (other <- 0 until p.maxRanges) { - when(hits(other)) { - assert( - (pas(slot)(other) +& (query.va - vas(slot)(other))) === translated, - "PreparedMap overlapping records disagree on PA" - ) - } - } - } - } -} diff --git a/arch/src/main/scala/framework/mem-core/queue/build.sc b/arch/src/main/scala/framework/mem-core/queue/build.sc deleted file mode 100644 index 3fffcdc99..000000000 --- a/arch/src/main/scala/framework/mem-core/queue/build.sc +++ /dev/null @@ -1,10 +0,0 @@ -import mill._ -import mill.scalalib._ - -object queue extends SbtModule { - override def millSourcePath = os.pwd - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") -} diff --git a/arch/src/main/scala/framework/mem-core/queue/src/main/scala/Queue.scala b/arch/src/main/scala/framework/mem-core/queue/src/main/scala/Queue.scala deleted file mode 100644 index 7d39d32da..000000000 --- a/arch/src/main/scala/framework/mem-core/queue/src/main/scala/Queue.scala +++ /dev/null @@ -1,79 +0,0 @@ -package memcore.memory.queue - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public} -import chisel3.util.{log2Ceil, QueueIO} - -@instantiable -class Queue[T <: Data]( - gen: T, - entries: Int, - pipe: Boolean = false, - flow: Boolean = false) - extends Module { - require(entries > 0) - @public - val io = IO(new QueueIO(gen, entries)) - - val memory = SyncReadMem(entries, gen) - val count = RegInit(0.U(log2Ceil(entries + 1).W)) - val memoryCount = RegInit(0.U(log2Ceil(entries + 1).W)) - val head = RegInit(0.U(math.max(1, log2Ceil(entries)).W)) - val tail = RegInit(0.U(math.max(1, log2Ceil(entries)).W)) - val front = Reg(gen) - val frontValid = RegInit(false.B) - val skid = Reg(gen) - val skidValid = RegInit(false.B) - val readPending = RegInit(false.B) - - io.count := count - io.deq.valid := frontValid || (if (flow) count === 0.U && io.enq.valid else false.B) - io.deq.bits := (if (flow) Mux(frontValid, front, io.enq.bits) else front) - val dequeue = frontValid && io.deq.ready - io.enq.ready := count < entries.U || (if (pipe) dequeue else false.B) - val bypass = if (flow) count === 0.U && io.deq.fire else false.B - val enqueue = io.enq.fire && !bypass - val direct = enqueue && memoryCount === 0.U && !readPending && !skidValid && (!frontValid || dequeue) - val write = enqueue && !direct - val held = frontValid.asUInt +& skidValid.asUInt - val occupied = held +& readPending.asUInt - val read = memoryCount =/= 0.U && !write && occupied < (2.U + dequeue.asUInt) - val readData = memory.readWrite(Mux(write, tail, head), io.enq.bits, write || read, write) - - val nextFront = WireDefault(front) - val nextFrontValid = WireDefault(frontValid && !dequeue) - val nextSkid = WireDefault(skid) - val nextSkidValid = WireDefault(skidValid) - when((!frontValid || dequeue) && skidValid) { - nextFront := skid - nextFrontValid := true.B - nextSkidValid := false.B - } - when(readPending) { - when((!frontValid || dequeue) && !skidValid) { - nextFront := readData - nextFrontValid := true.B - }.otherwise { - nextSkid := readData - nextSkidValid := true.B - } - } - when(direct) { - nextFront := io.enq.bits - nextFrontValid := true.B - } - front := nextFront - frontValid := nextFrontValid - skid := nextSkid - skidValid := nextSkidValid - readPending := read - - when(enqueue =/= dequeue) { - count := Mux(enqueue, count + 1.U, count - 1.U) - } - when(write =/= read) { - memoryCount := Mux(write, memoryCount + 1.U, memoryCount - 1.U) - } - when(write)(tail := Mux(tail === (entries - 1).U, 0.U, tail + 1.U)) - when(read)(head := Mux(head === (entries - 1).U, 0.U, head + 1.U)) -} diff --git a/arch/src/main/scala/framework/mem-core/spm/src/main/resources/interface.sv b/arch/src/main/scala/framework/mem-core/spm/src/main/resources/interface.sv deleted file mode 100644 index b00b96c00..000000000 --- a/arch/src/main/scala/framework/mem-core/spm/src/main/resources/interface.sv +++ /dev/null @@ -1,81 +0,0 @@ -interface spm_if ( - input logic clock, - reset -); - import uvm_pkg::*; - import ip_pkg::*; - import spm_pkg::*; - `include "uvm_macros.svh" - logic valid = 0, ready, write = 0; - logic [ 63:0] address = 0; - logic [ 2:0] size = 0; - logic [127:0] data = 0; - logic [ 15:0] mask = 0; - logic response_valid, response_ready = 0, error; - logic [127:0] result; - bit cancel = 0, read_only = 0; - chandle model; - in_order_scoreboard #(response_item) scoreboard; - bit enabled = 0, held = 0; - logic [128:0] held_response; - - always @(posedge clock) - if (enabled) begin - if (reset || cancel) begin - scoreboard.cancel_pending(); - held = 0; - if (response_valid) `uvm_fatal("CANCEL", "response survived reset/cancel") - end else begin - if (held && (!response_valid || {error, result} !== held_response)) - `uvm_fatal("HOLD", "response changed under backpressure") - held = response_valid && !response_ready; - held_response = {error, result}; - if (valid && ready) begin - response_item item; - longint unsigned low, high; - int status; - item = new; - status = spm_access(model, address, size, write, mask, data[63:0], data[127:64], - read_only, low, high); - if (status == 2) `uvm_fatal("REFERENCE", "read before explicit SRAM initialization") - item.data = {high, low}; - item.error = status != 0; - scoreboard.write_expected(item); - end - if (response_valid && response_ready) begin - response_item item; - item = new; - item.data = result; - item.error = error; - scoreboard.write_actual(item); - end - end - end - task send(input logic [63:0] addr, input logic [2:0] width, input bit wr, - input logic [127:0] value = 0, input logic [15:0] keep = 0); - @(negedge clock); - address = addr; - size = width; - write = wr; - data = value; - mask = keep; - valid = 1; - do @(posedge clock); while (!ready); - @(negedge clock); - valid = 0; - endtask - task take(output logic [127:0] value); - @(negedge clock); - response_ready = 1; - do @(posedge clock); while (!response_valid); - value = result; - @(negedge clock); - response_ready = 0; - endtask - task access (input logic [63:0] addr, input logic [2:0] width, input bit wr, - input logic [127:0] value = 0, input logic [15:0] keep = 0); - logic [127:0] ignored; - send(addr, width, wr, value, keep); - take(ignored); - endtask -endinterface diff --git a/arch/src/main/scala/framework/mem-core/spm/src/main/resources/package.sv b/arch/src/main/scala/framework/mem-core/spm/src/main/resources/package.sv deleted file mode 100644 index 32e9788e8..000000000 --- a/arch/src/main/scala/framework/mem-core/spm/src/main/resources/package.sv +++ /dev/null @@ -1,33 +0,0 @@ -package spm_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - import "DPI-C" function chandle spm_create( - input longint unsigned base, - input int unsigned bytes - ); - import "DPI-C" function void spm_destroy(input chandle model); - import "DPI-C" function int unsigned spm_access( - input chandle model, - input longint unsigned address, - input int unsigned size, - input byte unsigned write, - input int unsigned mask, - input longint unsigned low, - high, - input byte unsigned read_only, - output longint unsigned out_low, - out_high - ); - class response_item extends uvm_sequence_item; - logic [127:0] data; - logic error; - `uvm_object_utils_begin(response_item) - `uvm_field_int(data, UVM_ALL_ON) - `uvm_field_int(error, UVM_ALL_ON) - `uvm_object_utils_end - function new(string name = "response_item"); - super.new(name); - endfunction - endclass -endpackage diff --git a/arch/src/main/scala/framework/mem-core/spm/src/main/scala/Interfaces.scala b/arch/src/main/scala/framework/mem-core/spm/src/main/scala/Interfaces.scala deleted file mode 100644 index f9c6c7531..000000000 --- a/arch/src/main/scala/framework/mem-core/spm/src/main/scala/Interfaces.scala +++ /dev/null @@ -1,35 +0,0 @@ -package memcore.memory.spm - -import chisel3._ -import chisel3.util._ -import memcore.memory.bank.BankSetParams - -/** One local byte-address window. This does not describe a DDR mapping. */ -case class Params(base: BigInt, bytes: Int, dataBits: Int = 128) { - require(dataBits >= 64 && dataBits % 8 == 0 && isPow2(dataBits / 8)) - val beatBytes = dataBits / 8 - require(base >= 0 && base % beatBytes == 0 && base + bytes <= (BigInt(1) << 64)) - require(bytes >= 2 * beatBytes && isPow2(bytes)) - val laneBits = log2Ceil(beatBytes) - val sizeBits = log2Ceil(laneBits + 2) - val bank = BankSetParams(dataBits, banks = 1, entriesPerBank = bytes / beatBytes) -} - -/** Naturally aligned 1..beatBytes access. Data and mask start at address, not the SRAM row. */ -class Request(p: Params) extends Bundle { - val address = UInt(64.W) - val size = UInt(p.sizeBits.W) - val write = Bool() - val data = UInt(p.dataBits.W) - val mask = UInt(p.beatBytes.W) -} - -class Response(p: Params) extends Bundle { - val data = UInt(p.dataBits.W) - val error = Bool() -} - -class Port(p: Params) extends Bundle { - val request = Decoupled(new Request(p)) - val response = Flipped(Decoupled(new Response(p))) -} diff --git a/arch/src/main/scala/framework/mem-core/spm/src/main/scala/Memory.scala b/arch/src/main/scala/framework/mem-core/spm/src/main/scala/Memory.scala deleted file mode 100644 index 3e90a98fa..000000000 --- a/arch/src/main/scala/framework/mem-core/spm/src/main/scala/Memory.scala +++ /dev/null @@ -1,59 +0,0 @@ -package memcore.memory.spm - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import memcore.memory.bank.Bank - -/** - * One outstanding transaction on one physical read/write port. Writes acknowledge after - * SRAM acceptance; reads return after the synchronous read. Responses hold until accepted. - * Reset cancels protocol state; SRAM contents, including already accepted writes, remain. - */ -@instantiable -class Memory(p: Params) extends Module { - - @public val io = IO(new Bundle { - val port = Flipped(new Port(p)) - val readOnly = Input(Bool()) - val busy = Output(Bool()) - }) - - val bank = Instantiate(new Bank(p.bank)) - val idle :: memory :: rejected :: Nil = Enum(3) - val state = RegInit(idle) - val offset = Reg(UInt(p.laneBits.W)) - val savedMask = Reg(UInt(p.beatBytes.W)) - val request = io.port.request.bits - val count = 1.U(64.W) << request.size - val end = request.address +& count - - val widthMask = MuxLookup(request.size, 0.U(p.beatBytes.W))( - (0 to p.laneBits).map(size => size.U -> ((BigInt(1) << (1 << size)) - 1).U(p.beatBytes.W)) - ) - - val legal = request.size <= p.laneBits.U && (request.address & (count - 1.U)) === 0.U && - request.address >= p.base.U && end <= (p.base + p.bytes).U && - !(request.write && (io.readOnly || (request.mask & ~widthMask) =/= 0.U)) - - val lane = request.address(p.laneBits - 1, 0) - io.port.request.ready := state === idle && !reset.asBool && Mux(legal, bank.io.request.ready, true.B) - io.busy := state =/= idle - bank.io.request.valid := state === idle && !reset.asBool && io.port.request.valid && legal - bank.io.request.bits.addr := (request.address - p.base.U) >> p.laneBits - bank.io.request.bits.write := request.write - bank.io.request.bits.data := request.data << (lane << 3) - bank.io.request.bits.mask := request.mask << lane - bank.io.request.bits.tag := 0.U - when(io.port.request.fire) { - offset := lane - savedMask := widthMask - state := Mux(legal, memory, rejected) - } - val dataMask = Cat((0 until p.beatBytes).reverse.map(i => Fill(8, savedMask(i)))) - io.port.response.valid := !reset.asBool && (state === rejected || (state === memory && bank.io.response.valid)) - io.port.response.bits.data := Mux(state === rejected, 0.U, (bank.io.response.bits.data >> (offset << 3)) & dataMask) - io.port.response.bits.error := state === rejected || bank.io.response.bits.error - bank.io.response.ready := state === memory && io.port.response.ready && !reset.asBool - when(io.port.response.fire)(state := idle) -} diff --git a/arch/src/main/scala/framework/mem-core/tls/src/csrc/Cargo.lock b/arch/src/main/scala/framework/mem-core/tls/src/csrc/Cargo.lock deleted file mode 100644 index 73011b8a8..000000000 --- a/arch/src/main/scala/framework/mem-core/tls/src/csrc/Cargo.lock +++ /dev/null @@ -1,7 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "tls_ref" -version = "0.1.0" diff --git a/arch/src/main/scala/framework/mem-core/tls/src/csrc/Cargo.toml b/arch/src/main/scala/framework/mem-core/tls/src/csrc/Cargo.toml deleted file mode 100644 index 4591b837f..000000000 --- a/arch/src/main/scala/framework/mem-core/tls/src/csrc/Cargo.toml +++ /dev/null @@ -1,8 +0,0 @@ -[package] -name = "tls_ref" -version = "0.1.0" -edition = "2021" - -[lib] -crate-type = ["cdylib"] -path = "../../../spm/src/csrc/src/lib.rs" diff --git a/arch/src/main/scala/framework/mem-core/tls/src/main/resources/tls.f b/arch/src/main/scala/framework/mem-core/tls/src/main/resources/tls.f deleted file mode 100644 index 13249c547..000000000 --- a/arch/src/main/scala/framework/mem-core/tls/src/main/resources/tls.f +++ /dev/null @@ -1,10 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@VERIFY@/uvm/src/ip/test -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/test/control_if.sv -@VERIFY@/uvm/src/ip/test/control_test_package.sv -@RESOURCES@/../../../../spm/src/main/resources/package.sv -@RESOURCES@/../../../../spm/src/main/resources/interface.sv -+incdir+@RTL@ --F @RTL@/Store/filelist.f -@RESOURCES@/tls_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/tls/src/main/resources/tls_tb.sv b/arch/src/main/scala/framework/mem-core/tls/src/main/resources/tls_tb.sv deleted file mode 100644 index f1064cb1b..000000000 --- a/arch/src/main/scala/framework/mem-core/tls/src/main/resources/tls_tb.sv +++ /dev/null @@ -1,121 +0,0 @@ -`include "config.svh" -module tls_tb; - import uvm_pkg::*; - import ip_pkg::*; - import ip_control_test_pkg::*; - import spm_pkg::*; - `include "uvm_macros.svh" - logic clock = 0; - always #5 clock = ~clock; - logic [1:0] reset = '1, active = '0; - ip_control_if control (clock); - virtual spm_if ports[4]; - chandle models[2]; - for (genvar i = 0; i < 4; i++) begin : endpoints - spm_if bus ( - clock, - reset[i/2] - ); - initial ports[i] = bus; - end - for (genvar tile = 0; tile < 2; tile++) begin : tiles - Store dut ( - .clock(clock), - .reset(reset[tile]), - .io_running(active[tile]), - .io_cancel(endpoints[tile*2+1].bus.cancel), - .io_busy(), - .io_host_request_valid(endpoints[tile*2+0].bus.valid), - .io_host_request_ready(endpoints[tile*2+0].bus.ready), - .io_host_request_bits_address(endpoints[tile*2+0].bus.address), - .io_host_request_bits_size(endpoints[tile*2+0].bus.size), - .io_host_request_bits_write(endpoints[tile*2+0].bus.write), - .io_host_request_bits_data(endpoints[tile*2+0].bus.data), - .io_host_request_bits_mask(endpoints[tile*2+0].bus.mask), - .io_host_response_valid(endpoints[tile*2+0].bus.response_valid), - .io_host_response_ready(endpoints[tile*2+0].bus.response_ready), - .io_host_response_bits_data(endpoints[tile*2+0].bus.result), - .io_host_response_bits_error(endpoints[tile*2+0].bus.error), - .io_local_request_valid(endpoints[tile*2+1].bus.valid), - .io_local_request_ready(endpoints[tile*2+1].bus.ready), - .io_local_request_bits_address(endpoints[tile*2+1].bus.address), - .io_local_request_bits_size(endpoints[tile*2+1].bus.size), - .io_local_request_bits_write(endpoints[tile*2+1].bus.write), - .io_local_request_bits_data(endpoints[tile*2+1].bus.data), - .io_local_request_bits_mask(endpoints[tile*2+1].bus.mask), - .io_local_response_valid(endpoints[tile*2+1].bus.response_valid), - .io_local_response_ready(endpoints[tile*2+1].bus.response_ready), - .io_local_response_bits_data(endpoints[tile*2+1].bus.result), - .io_local_response_bits_error(endpoints[tile*2+1].bus.error) - ); - end - initial begin - foreach (models[tile]) models[tile] = spm_create(`LOCAL_BASE, `LOCAL_BYTES); - #0; - foreach (ports[i]) begin - ports[i].model = models[i/2]; - ports[i].scoreboard = new($sformatf("port_%0d", i), null); - ports[i].enabled = 1; - end - uvm_config_db#(virtual ip_control_if)::set(null, "uvm_test_top", "vif", control); - uvm_config_db#(time)::set(null, "uvm_test_top", "timeout", 100us); - run_test("protocol_test"); - end - final foreach (models[i]) spm_destroy(models[i]); - initial begin : exercise - logic [127:0] value; - wait (control.start); - repeat (3) @(negedge clock); - reset = 0; - for (int row = 0; row < `LOCAL_BYTES; row += 16) begin - ports[0].access(`LOCAL_BASE + row, 4, 1, 0, '1); - ports[2].access(`LOCAL_BASE + row, 4, 1, '1, '1); - end - @(negedge clock); - active = 3; - ports[1].access(`LOCAL_BASE, 3, 1, 64'h123456789abcdef0, 16'hff); - ports[3].access(`LOCAL_BASE, 4, 0); - ports[1].access(`LOCAL_BASE + 15, 0, 1, 8'ha5, 1); - ports[1].access(`LOCAL_BASE, 4, 0); - ports[1].send(`LOCAL_BASE, 4, 0); - wait (ports[1].response_valid); - repeat (5) @(negedge clock); - ports[3].access(`LOCAL_BASE + 16, 4, 0); - ports[1].take(value); - ports[1].access(`LOCAL_BASE + 3, 2, 0); - ports[1].access(`LOCAL_BASE + `LOCAL_BYTES, 3, 0); - ports[1].access(64'h80000000, 3, 0); - ports[1].access(`LOCAL_BASE, 5, 0); - ports[1].access(`LOCAL_BASE + 8, 3, 1, 64'h1122334455667788, 16'ha5); - ports[1].access(`LOCAL_BASE + 8, 3, 0); - ports[1].access((64'h1 << 32) | `LOCAL_BASE, 3, 0); - ports[1].access(64'hfffffffffffffff8, 3, 0); - ports[1].access(`LOCAL_BASE, 0, 1, '1, 3); - ports[1].send(`LOCAL_BASE, 3, 0); - ports[1].cancel = 1; - @(negedge clock); - ports[1].cancel = 0; - repeat (5) begin - @(negedge clock); - if (ports[1].response_valid) `uvm_fatal("CANCEL", "old TLS response survived cancellation") - end - ports[1].access(`LOCAL_BASE, 3, 0); - ports[1].send(`LOCAL_BASE + 16, 0, 1, 8'h5a, 1); - wait (ports[1].response_valid); - @(negedge clock); - reset[0] = 1; - active[0] = 0; - repeat (2) @(negedge clock); - reset[0] = 0; - ports[0].access(`LOCAL_BASE + 16, 0, 0); - ports[3].access(`LOCAL_BASE + 16, 0, 0); - repeat (2) @(negedge clock); - foreach (ports[i]) begin - if (ports[i].scoreboard.expected_queue.size() || ports[i].scoreboard.actual_queue.size()) - `uvm_fatal("DRAIN", "local storage still has unmatched accesses") - `uvm_info("CHECKED", $sformatf("port %0d: %0d comparisons", i, ports[i].scoreboard.checked), - UVM_LOW) - end - control.done = 1; - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/tls/src/main/scala/Emit.scala b/arch/src/main/scala/framework/mem-core/tls/src/main/scala/Emit.scala deleted file mode 100644 index 721fc9b24..000000000 --- a/arch/src/main/scala/framework/mem-core/tls/src/main/scala/Emit.scala +++ /dev/null @@ -1,18 +0,0 @@ -package memcore.memory.tls - -import java.nio.file.{Files, Paths} - -object Emit extends App { - val memory = memcore.memory.spm.Params(base = BigInt("10000", 16), bytes = 64) - val output = Paths.get("src/main/scala/framework/mem-core/tls/build") - Files.createDirectories(output) - Files.writeString( - output.resolve("config.svh"), - s"`define LOCAL_BASE 64'h${memory.base.toString(16)}\n`define LOCAL_BYTES ${memory.bytes}\n" - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Store(memory), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("Store").toString, "--split-verilog") - ) -} diff --git a/arch/src/main/scala/framework/mem-core/tls/src/main/scala/Store.scala b/arch/src/main/scala/framework/mem-core/tls/src/main/scala/Store.scala deleted file mode 100644 index 0e03879ba..000000000 --- a/arch/src/main/scala/framework/mem-core/tls/src/main/scala/Store.scala +++ /dev/null @@ -1,48 +0,0 @@ -package memcore.memory.tls - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import memcore.memory.spm.{Memory, Params, Port} - -/** - * A context-private local store. Loader and execution access it in separate ownership phases. - * Cancellation drains an accepted local access and discards its response before reuse. - */ -@instantiable -class Store(p: Params, localReadOnly: Boolean = false) extends Module { - - @public val io = IO(new Bundle { - val running = Input(Bool()) - val cancel = Input(Bool()) - val host = Flipped(new Port(p)) - val local = Flipped(new Port(p)) - val busy = Output(Bool()) - }) - - val memory = Instantiate(new Memory(p)) - val busy = RegInit(false.B) - val owner = Reg(Bool()) - val cancelled = RegInit(false.B) - val discard = owner && (cancelled || io.cancel) - memory.io.port.request.valid := !busy && !reset.asBool && - Mux(io.running, io.local.request.valid && !io.cancel, io.host.request.valid) - memory.io.port.request.bits := Mux(io.running, io.local.request.bits, io.host.request.bits) - memory.io.readOnly := io.running && localReadOnly.B - io.local.request.ready := !busy && io.running && !io.cancel && memory.io.port.request.ready - io.host.request.ready := !busy && !io.running && memory.io.port.request.ready - when(memory.io.port.request.fire) { - busy := true.B - owner := io.running - cancelled := false.B - } - when(busy && owner && io.cancel)(cancelled := true.B) - io.local.response.valid := busy && owner && !discard && memory.io.port.response.valid - io.host.response.valid := busy && !owner && memory.io.port.response.valid - io.local.response.bits := memory.io.port.response.bits - io.host.response.bits := memory.io.port.response.bits - memory.io.port.response.ready := busy && - (discard || Mux(owner, io.local.response.ready, io.host.response.ready)) - when(memory.io.port.response.fire) { busy := false.B; cancelled := false.B } - io.busy := busy -} diff --git a/arch/src/main/scala/framework/mem-core/tss/src/csrc/Cargo.lock b/arch/src/main/scala/framework/mem-core/tss/src/csrc/Cargo.lock deleted file mode 100644 index fdcc5b948..000000000 --- a/arch/src/main/scala/framework/mem-core/tss/src/csrc/Cargo.lock +++ /dev/null @@ -1,7 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "tss_ref" -version = "0.1.0" diff --git a/arch/src/main/scala/framework/mem-core/tss/src/csrc/Cargo.toml b/arch/src/main/scala/framework/mem-core/tss/src/csrc/Cargo.toml deleted file mode 100644 index 146688e0a..000000000 --- a/arch/src/main/scala/framework/mem-core/tss/src/csrc/Cargo.toml +++ /dev/null @@ -1,8 +0,0 @@ -[package] -name = "tss_ref" -version = "0.1.0" -edition = "2021" - -[lib] -crate-type = ["cdylib"] -path = "../../../spm/src/csrc/src/lib.rs" diff --git a/arch/src/main/scala/framework/mem-core/tss/src/main/resources/tss.cm_hier b/arch/src/main/scala/framework/mem-core/tss/src/main/resources/tss.cm_hier deleted file mode 100644 index 80afe3c28..000000000 --- a/arch/src/main/scala/framework/mem-core/tss/src/main/resources/tss.cm_hier +++ /dev/null @@ -1,2 +0,0 @@ -+tree @TOP@.tiles[0].dut -+tree @TOP@.tiles[1].dut diff --git a/arch/src/main/scala/framework/mem-core/tss/src/main/resources/tss.f b/arch/src/main/scala/framework/mem-core/tss/src/main/resources/tss.f deleted file mode 100644 index 7459e2657..000000000 --- a/arch/src/main/scala/framework/mem-core/tss/src/main/resources/tss.f +++ /dev/null @@ -1,10 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@VERIFY@/uvm/src/ip/test -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/test/control_if.sv -@VERIFY@/uvm/src/ip/test/control_test_package.sv -@RESOURCES@/../../../../spm/src/main/resources/package.sv -@RESOURCES@/../../../../spm/src/main/resources/interface.sv -+incdir+@RTL@ --F @RTL@/Store/filelist.f -@RESOURCES@/tss_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/tss/src/main/resources/tss.toml b/arch/src/main/scala/framework/mem-core/tss/src/main/resources/tss.toml deleted file mode 100644 index c18e5753d..000000000 --- a/arch/src/main/scala/framework/mem-core/tss/src/main/resources/tss.toml +++ /dev/null @@ -1 +0,0 @@ -rtl_scopes = ["tiles[0].dut", "tiles[1].dut"] diff --git a/arch/src/main/scala/framework/mem-core/tss/src/main/resources/tss_tb.sv b/arch/src/main/scala/framework/mem-core/tss/src/main/resources/tss_tb.sv deleted file mode 100644 index 4d895911e..000000000 --- a/arch/src/main/scala/framework/mem-core/tss/src/main/resources/tss_tb.sv +++ /dev/null @@ -1,151 +0,0 @@ -`include "config.svh" -module tss_tb; - import uvm_pkg::*; - import ip_pkg::*; - import ip_control_test_pkg::*; - import spm_pkg::*; - `include "uvm_macros.svh" - logic clock = 0; - always #5 clock = ~clock; - logic [1:0] reset = '1, active = '0; - ip_control_if control (clock); - virtual spm_if ports[6]; - chandle models[2]; - for (genvar i = 0; i < 6; i++) begin : endpoints - spm_if bus ( - clock, - reset[i/3] - ); - initial ports[i] = bus; - end - for (genvar tile = 0; tile < 2; tile++) begin : tiles - Store dut ( - .clock(clock), - .reset(reset[tile]), - .io_inUse(active[tile]), - .io_busy_0(), - .io_busy_1(), - .io_hostBusy(), - .io_cancel_0(endpoints[tile*3].bus.cancel), - .io_cancel_1(endpoints[tile*3+1].bus.cancel), - .io_clients_0_request_valid(endpoints[tile*3+0].bus.valid), - .io_clients_0_request_ready(endpoints[tile*3+0].bus.ready), - .io_clients_0_request_bits_address(endpoints[tile*3+0].bus.address), - .io_clients_0_request_bits_size(endpoints[tile*3+0].bus.size), - .io_clients_0_request_bits_write(endpoints[tile*3+0].bus.write), - .io_clients_0_request_bits_data(endpoints[tile*3+0].bus.data), - .io_clients_0_request_bits_mask(endpoints[tile*3+0].bus.mask), - .io_clients_0_response_valid(endpoints[tile*3+0].bus.response_valid), - .io_clients_0_response_ready(endpoints[tile*3+0].bus.response_ready), - .io_clients_0_response_bits_data(endpoints[tile*3+0].bus.result), - .io_clients_0_response_bits_error(endpoints[tile*3+0].bus.error), - .io_clients_1_request_valid(endpoints[tile*3+1].bus.valid), - .io_clients_1_request_ready(endpoints[tile*3+1].bus.ready), - .io_clients_1_request_bits_address(endpoints[tile*3+1].bus.address), - .io_clients_1_request_bits_size(endpoints[tile*3+1].bus.size), - .io_clients_1_request_bits_write(endpoints[tile*3+1].bus.write), - .io_clients_1_request_bits_data(endpoints[tile*3+1].bus.data), - .io_clients_1_request_bits_mask(endpoints[tile*3+1].bus.mask), - .io_clients_1_response_valid(endpoints[tile*3+1].bus.response_valid), - .io_clients_1_response_ready(endpoints[tile*3+1].bus.response_ready), - .io_clients_1_response_bits_data(endpoints[tile*3+1].bus.result), - .io_clients_1_response_bits_error(endpoints[tile*3+1].bus.error), - .io_host_request_valid(endpoints[tile*3+2].bus.valid), - .io_host_request_ready(endpoints[tile*3+2].bus.ready), - .io_host_request_bits_address(endpoints[tile*3+2].bus.address), - .io_host_request_bits_size(endpoints[tile*3+2].bus.size), - .io_host_request_bits_write(endpoints[tile*3+2].bus.write), - .io_host_request_bits_data(endpoints[tile*3+2].bus.data), - .io_host_request_bits_mask(endpoints[tile*3+2].bus.mask), - .io_host_response_valid(endpoints[tile*3+2].bus.response_valid), - .io_host_response_ready(endpoints[tile*3+2].bus.response_ready), - .io_host_response_bits_data(endpoints[tile*3+2].bus.result), - .io_host_response_bits_error(endpoints[tile*3+2].bus.error) - ); - end - initial begin - foreach (models[tile]) models[tile] = spm_create(`LOCAL_BASE, `LOCAL_BYTES); - #0; - foreach (ports[i]) begin - ports[i].model = models[i/3]; - ports[i].scoreboard = new($sformatf("port_%0d", i), null); - ports[i].enabled = 1; - end - uvm_config_db#(virtual ip_control_if)::set(null, "uvm_test_top", "vif", control); - uvm_config_db#(time)::set(null, "uvm_test_top", "timeout", 100us); - run_test("protocol_test"); - end - final foreach (models[i]) spm_destroy(models[i]); - initial begin : exercise - logic [127:0] value; - wait (control.start); - repeat (3) @(negedge clock); - reset = 0; - for (int row = 0; row < `LOCAL_BYTES; row += 16) begin - ports[2].access(`LOCAL_BASE + row, 4, 1, 0, '1); - ports[5].access(`LOCAL_BASE + row, 4, 1, '1, '1); - end - @(negedge clock); - active = 3; - fork - begin - ports[0].access(`LOCAL_BASE, 3, 1, 64'h123456789abcdef0, 16'hff); - ports[0].access(`LOCAL_BASE + 8, 3, 1, 1, 16'hff); - end - begin - do begin - ports[1].send(`LOCAL_BASE + 8, 3, 0); - ports[1].take(value); - end while (value != 1); - ports[1].access(`LOCAL_BASE, 3, 0); - end - join - ports[3].access(`LOCAL_BASE, 4, 0); - ports[0].send(`LOCAL_BASE + 16, 3, 1, 64'h1122, 16'hff); - wait (ports[0].response_valid); - ports[1].access(`LOCAL_BASE + 16, 3, 0); - repeat (5) @(negedge clock); - ports[0].take(value); - fork - for (int n = 0; n < 8; n++) ports[0].access(`LOCAL_BASE + 32, 0, 1, n, 1); - for (int n = 0; n < 8; n++) ports[1].access(`LOCAL_BASE + 32, 0, 1, n + 16, 1); - join - ports[1].access(`LOCAL_BASE + 32, 0, 0); - ports[0].access(`LOCAL_BASE + 1, 3, 0); - ports[0].access(`LOCAL_BASE + `LOCAL_BYTES, 3, 0); - ports[0].access(64'h80000000, 3, 0); - ports[0].access(`LOCAL_BASE, 5, 0); - ports[0].access(`LOCAL_BASE + 8, 3, 1, 64'h1122334455667788, 16'ha5); - ports[1].access(`LOCAL_BASE + 8, 3, 0); - ports[0].access((64'h1 << 32) | `LOCAL_BASE, 3, 0); - ports[0].access(64'hfffffffffffffff8, 3, 0); - ports[0].access(`LOCAL_BASE, 0, 1, '1, 3); - ports[0].send(`LOCAL_BASE, 3, 0); - ports[0].cancel = 1; - @(negedge clock); - ports[0].cancel = 0; - ports[1].access(`LOCAL_BASE + 16, 3, 0); - repeat (5) begin - @(negedge clock); - if (ports[0].response_valid) `uvm_fatal("CANCEL", "old TSS response survived cancellation") - end - ports[0].access(`LOCAL_BASE, 3, 0); - ports[0].send(`LOCAL_BASE + 16, 0, 1, 8'ha5, 1); - wait (ports[0].response_valid); - @(negedge clock); - reset[0] = 1; - active[0] = 0; - repeat (2) @(negedge clock); - reset[0] = 0; - ports[2].access(`LOCAL_BASE + 16, 0, 0); - ports[3].access(`LOCAL_BASE + 16, 0, 0); - repeat (2) @(negedge clock); - foreach (ports[i]) begin - if (ports[i].scoreboard.expected_queue.size() || ports[i].scoreboard.actual_queue.size()) - `uvm_fatal("DRAIN", "local storage still has unmatched accesses") - `uvm_info("CHECKED", $sformatf("port %0d: %0d comparisons", i, ports[i].scoreboard.checked), - UVM_LOW) - end - control.done = 1; - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/tss/src/main/scala/Emit.scala b/arch/src/main/scala/framework/mem-core/tss/src/main/scala/Emit.scala deleted file mode 100644 index c84d552a9..000000000 --- a/arch/src/main/scala/framework/mem-core/tss/src/main/scala/Emit.scala +++ /dev/null @@ -1,18 +0,0 @@ -package memcore.memory.tss - -import java.nio.file.{Files, Paths} - -object Emit extends App { - val memory = memcore.memory.spm.Params(base = BigInt("20000", 16), bytes = 64) - val output = Paths.get("src/main/scala/framework/mem-core/tss/build") - Files.createDirectories(output) - Files.writeString( - output.resolve("config.svh"), - s"`define LOCAL_BASE 64'h${memory.base.toString(16)}\n`define LOCAL_BYTES ${memory.bytes}\n" - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Store(Params(memory, ports = 2)), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("Store").toString, "--split-verilog") - ) -} diff --git a/arch/src/main/scala/framework/mem-core/tss/src/main/scala/Store.scala b/arch/src/main/scala/framework/mem-core/tss/src/main/scala/Store.scala deleted file mode 100644 index 93f543ee7..000000000 --- a/arch/src/main/scala/framework/mem-core/tss/src/main/scala/Store.scala +++ /dev/null @@ -1,86 +0,0 @@ -package memcore.memory.tss - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import memcore.memory.spm.{Memory, Params => MemoryParams, Port, Response} - -case class Params(memory: MemoryParams, ports: Int) { - require(ports > 0) -} - -/** - * One shared SRAM, fair arbitration, one outstanding request and one response slot per port. - * SRAM service is serialized; a held response does not block other clients. inUse covers the - * lifetime of the cooperating group, not just one Ant. Cancel drains only its owning port. - */ -@instantiable -class Store(p: Params) extends Module { - val count = p.ports + 1 - val host = p.ports - val indexBits = log2Ceil(count) - - @public val io = IO(new Bundle { - val inUse = Input(Bool()) - val cancel = Input(Vec(p.ports, Bool())) - val clients = Vec(p.ports, Flipped(new Port(p.memory))) - val host = Flipped(new Port(p.memory)) - val busy = Output(Vec(p.ports, Bool())) - val hostBusy = Output(Bool()) - }) - - val memory = Instantiate(new Memory(p.memory)) - val ports = io.clients.toSeq :+ io.host - val occupied = RegInit(VecInit(Seq.fill(count)(false.B))) - val available = RegInit(VecInit(Seq.fill(count)(false.B))) - val cancelled = RegInit(VecInit(Seq.fill(count)(false.B))) - val answers = Reg(Vec(count, new Response(p.memory))) - val active = RegInit(false.B) - val owner = Reg(UInt(indexBits.W)) - val next = RegInit(0.U(indexBits.W)) - val cancel = VecInit(io.cancel.toSeq :+ false.B) - - val eligible = VecInit(ports.zipWithIndex.map { case (port, i) => - val allowed = - if (i == host) !io.inUse && !occupied.take(p.ports).reduce(_ || _) - else io.inUse && !occupied(host) - port.request.valid && !occupied(i) && !cancel(i) && allowed && !reset.asBool - }) - - val after = VecInit((0 until count).map(i => eligible(i) && i.U >= next)).asUInt - val chosen = Mux(after.orR, PriorityEncoder(after), PriorityEncoder(eligible.asUInt)) - memory.io.readOnly := false.B - memory.io.port.request.valid := !active && eligible.asUInt.orR - memory.io.port.request.bits := VecInit(ports.map(_.request.bits))(chosen) - memory.io.port.response.ready := active - when(memory.io.port.request.fire) { - active := true.B - owner := chosen - occupied(chosen) := true.B - cancelled(chosen) := false.B - next := Mux(chosen === (count - 1).U, 0.U, chosen + 1.U) - } - for ((port, i) <- ports.zipWithIndex) { - port.request.ready := !active && eligible(i) && chosen === i.U && memory.io.port.request.ready - port.response.valid := available(i) && !cancelled(i) && !cancel(i) && !reset.asBool - port.response.bits := answers(i) - when(cancel(i) && occupied(i))(cancelled(i) := true.B) - when(port.response.fire || (cancel(i) && available(i))) { - available(i) := false.B - occupied(i) := false.B - cancelled(i) := false.B - } - } - when(memory.io.port.response.fire) { - active := false.B - when(cancelled(owner) || cancel(owner)) { - occupied(owner) := false.B - cancelled(owner) := false.B - }.otherwise { - answers(owner) := memory.io.port.response.bits - available(owner) := true.B - } - } - io.busy := VecInit(occupied.take(p.ports)) - io.hostBusy := occupied(host) -} diff --git a/arch/src/main/scala/framework/memdomain/MemDomain.scala b/arch/src/main/scala/framework/memdomain/MemDomain.scala deleted file mode 100644 index 740010090..000000000 --- a/arch/src/main/scala/framework/memdomain/MemDomain.scala +++ /dev/null @@ -1,332 +0,0 @@ -package framework.memdomain - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.balldomain.blink.{BankRead, BankWrite} -import framework.balldomain.blink.mmio.{MmioRead, MmioWrite} -import framework.frontend.globalrs.{GlobalSchedComplete, GlobalSchedIssue} -import framework.top.GlobalConfig -import framework.memdomain.backend.MemRequestIO -import framework.memdomain.backend.mmio.MmioPool -import framework.memdomain.backend.shared.SharedMemLayout -import framework.memdomain.frontend.MemFrontend -import framework.memdomain.frontend.mem.{Footprint, KernelMemoryBridge, MemConfigerIO} -import framework.memdomain.frontend.mem.dma.DmaPort -import framework.memdomain.midend.MemMidend -import framework.memdomain.backend.MemBackend -import framework.memdomain.backend.banks.btrace.PhysicalBankHash -import framework.memdomain.isa.{MvoverISA, MvoverPort} -import memcore.memory.mesh_shm.MeshLocalBankPort - -@instantiable -class MemDomain(val b: GlobalConfig) extends Module { - val totalMmioRead = b.ballDomain.ballIdMappings.map(_.mmioReadBW).sum - val totalMmioWrite = b.ballDomain.ballIdMappings.map(_.mmioWriteBW).sum - val totalBallRead = b.ballDomain.ballIdMappings.map(_.inBW).sum - val totalBallWrite = b.ballDomain.ballIdMappings.map(_.outBW).sum - val kernelPorts = if (b.rvv.enable) b.rvv.memoryPorts else 0 - val sharedHashCount = if (b.memDomain.sharedEnable) SharedMemLayout.totalBank(b) else 0 - - @public - val io = IO(new Bundle { -// Command Channel - val global_issue_i = Flipped(Decoupled(new GlobalSchedIssue(b))) - val global_complete_o = Decoupled(new GlobalSchedComplete(b)) - val busy = Output(Bool()) - val inst_ids = Input(Vec(b.frontend.rob_entries, UInt(64.W))) - val footprints = Output(Vec(3, new Footprint(b))) - -// Inside Channel - val ballChannelActive = Input(Vec(b.ballDomain.ballNum, Bool())) - val ballChannelReady = Output(Vec(b.ballDomain.ballNum, Bool())) - - val ballDomain = new Bundle { - val bankRead = Vec(totalBallRead, new BankRead(b)) - val bankWrite = Vec(totalBallWrite, new BankWrite(b)) - val mmioRead = Vec(totalMmioRead, new MmioRead(b)) - val mmioWrite = Vec(totalMmioWrite, new MmioWrite(b)) - } - - val kernel = - if (b.rvv.enable) Some(new KernelMemoryBridge(b)) - else None - -// Outside Channel - val dma = new DmaPort(b.memDomain.dma_buswidth) - val hartid = Input(UInt(b.tile.xLen.W)) - -// Shared memory path - val shared_mem_req = Vec(SharedMemLayout.channelPerHart(b), new MemRequestIO(b)) - val mvover = new MvoverPort - - val meshLocalBank = Flipped(new MeshLocalBankPort( - MvoverISA.AddressBits, - MvoverISA.BankBits, - b.memDomain.bankWidth, - math.max(1, log2Ceil(b.frontend.rob_entries)) - )) - - val shared_config = Decoupled(new MemConfigerIO(b)) - val shared_query_valid = Output(Bool()) - val shared_query_vbank_id = Output(UInt(b.memDomain.vbankIdWidth.W)) - val shared_query_group_count = Input(UInt(b.memDomain.groupCountWidth.W)) - - val bank_hashes = - if (b.sim.diffTest) { - Some(Output(Vec(b.memDomain.bankNum + sharedHashCount, new PhysicalBankHash(b)))) - } else { - None - } - - val shared_bank_hashes = - if (b.sim.diffTest && b.memDomain.sharedEnable) { - Some(Input(Vec(sharedHashCount, new PhysicalBankHash(b)))) - } else { - None - } - - }) - - val frontend: Instance[MemFrontend] = Instantiate(new MemFrontend(b)) - val midend: Instance[MemMidend] = Instantiate(new MemMidend(b)) - val backend: Instance[MemBackend] = Instantiate(new MemBackend(b)) - - io.bank_hashes.foreach(_ := backend.io.bank_hashes.get) - backend.io.shared_bank_hashes.foreach(_ := io.shared_bank_hashes.get) - - // Connect query interface from frontend to backend - backend.io.query_vbank_id := frontend.io.query_vbank_id - backend.io.query_is_shared := frontend.io.query_is_shared - frontend.io.query_group_count := backend.io.query_group_count - frontend.io.clearBusy := backend.io.clearBusy - frontend.io.hartid := io.hartid - midend.io.inst_ids := io.inst_ids - - // Shared query: backend delegates shared query to external SharedMemBackend - backend.io.shared_query_group_count := io.shared_query_group_count - io.shared_query_valid := backend.io.shared_query_valid - io.shared_query_vbank_id := backend.io.shared_query_vbank_id - -//===----------------------------------------------------------------------===// -// Connection with outside (all in frontend) -//===----------------------------------------------------------------------===// - frontend.io.global_issue_i <> io.global_issue_i - frontend.io.global_complete_o <> io.global_complete_o - io.mvover <> frontend.io.mvover - io.busy := frontend.io.busy - io.footprints := frontend.io.footprints - - io.dma <> frontend.io.dma - - // Ball Domain interface connects to midend unified bankRead/bankWrite - // Indices [0, totalBallRead) are balldomain; last index is frontend (DMA) - for (i <- 0 until totalBallRead) { - midend.io.bankRead(i).bankRead <> io.ballDomain.bankRead(i) - midend.io.bankRead(i).is_shared := false.B - } - - midend.io.ballChannelActive := io.ballChannelActive - io.ballChannelReady := midend.io.ballChannelReady - - for (i <- 0 until totalBallWrite) { - midend.io.bankWrite(i).bankWrite <> io.ballDomain.bankWrite(i) - midend.io.bankWrite(i).is_shared := false.B - } - - if (b.rvv.enable) { - val kernel = io.kernel.get - val dma = frontend.io.kernel.get - dma.load <> kernel.load - kernel.image <> dma.image - kernel.result <> dma.result - dma.abort := kernel.abort - kernel.busy := dma.busy - kernel.ready := kernel.active - for (port <- 0 until kernelPorts) { - val read = backend.io.kernel_req(port) - read.read <> kernel.bankRead(port).io - read.write.req.valid := false.B - read.write.req.bits := 0.U.asTypeOf(read.write.req.bits) - read.write.resp.ready := true.B - read.bank_id := kernel.bankRead(port).bank_id - read.group_id := kernel.bankRead(port).group_id - read.rob_id := kernel.bankRead(port).rob_id - read.inst_id := io.inst_ids(kernel.bankRead(port).rob_id) - read.hart_id := io.hartid - read.is_shared := false.B - - val write = backend.io.kernel_req(kernelPorts + port) - write.write <> kernel.bankWrite(port).io - write.read.req.valid := false.B - write.read.req.bits := 0.U.asTypeOf(write.read.req.bits) - write.read.resp.ready := true.B - write.bank_id := kernel.bankWrite(port).bank_id - write.group_id := kernel.bankWrite(port).group_id - write.rob_id := kernel.bankWrite(port).rob_id - write.inst_id := io.inst_ids(kernel.bankWrite(port).rob_id) - write.hart_id := io.hartid - write.is_shared := false.B - } - } - - midend.io.bankRead(totalBallRead).bankRead <> frontend.io.interdma.bankRead - midend.io.bankRead(totalBallRead).is_shared := frontend.io.interdma.read_is_shared - midend.io.hartid := io.hartid - - for (i <- 1 until b.memDomain.bankChannel) { - midend.io.mem_req(i) <> backend.io.mem_req(i) - } - if (b.memDomain.sharedEnable && b.memDomain.bankWidth == 128) { - // Reserve channel 0 for Mesh-initiated private-Bank access only when its - // ordinary request/response pair has drained. This preserves the normal - // AccPipe route and prevents one-cycle SRAM responses from being lost. - val normal = midend.io.mem_req(0) - val routed = backend.io.mem_req(0) - val local = io.meshLocalBank - val normalReadPending = RegInit(false.B) - val normalWritePending = RegInit(false.B) - val localPending = RegInit(false.B) - val localIsWrite = RegInit(false.B) - val localTag = Reg(chiselTypeOf(local.request.bits.tag)) - val normalActive = normalReadPending || normalWritePending || - normal.read.req.valid || normal.write.req.valid - val localRequest = local.request.valid && !localPending && !normalActive - val localRead = localRequest && !local.request.bits.write - val localWrite = localRequest && local.request.bits.write - - routed.bank_id := Mux(localRequest, local.request.bits.bank, normal.bank_id) - routed.group_id := Mux(localRequest, 0.U, normal.group_id) - routed.is_shared := Mux(localRequest, false.B, normal.is_shared) - routed.hart_id := Mux(localRequest, io.hartid, normal.hart_id) - routed.rob_id := Mux(localRequest, 0.U, normal.rob_id) - routed.inst_id := Mux(localRequest, 0.U, normal.inst_id) - routed.read.req.valid := localRead || (normal.read.req.valid && !localPending) - routed.read.req.bits.addr := Mux(localRead, local.request.bits.addr, normal.read.req.bits.addr) - routed.write.req.valid := localWrite || (normal.write.req.valid && !localPending) - routed.write.req.bits.addr := Mux(localWrite, local.request.bits.addr, normal.write.req.bits.addr) - routed.write.req.bits.data := Mux(localWrite, local.request.bits.data, normal.write.req.bits.data) - routed.write.req.bits.mask := Mux(localWrite, VecInit(local.request.bits.mask.asBools), normal.write.req.bits.mask) - normal.read.req.ready := !localPending && routed.read.req.ready - normal.write.req.ready := !localPending && routed.write.req.ready - local.request.ready := !normalActive && !localPending && - Mux(local.request.bits.write, routed.write.req.ready, routed.read.req.ready) - - routed.read.resp.ready := Mux(localPending && !localIsWrite, local.response.ready, normal.read.resp.ready) - routed.write.resp.ready := Mux(localPending && localIsWrite, local.response.ready, normal.write.resp.ready) - normal.read.resp.valid := routed.read.resp.valid && !localPending - normal.read.resp.bits := routed.read.resp.bits - normal.write.resp.valid := routed.write.resp.valid && !localPending - normal.write.resp.bits := routed.write.resp.bits - local.response.valid := localPending && Mux(localIsWrite, routed.write.resp.valid, routed.read.resp.valid) - local.response.bits.data := routed.read.resp.bits.data - local.response.bits.tag := localTag - local.response.bits.error := false.B - - when(normal.read.req.fire)(normalReadPending := true.B) - when(normal.read.resp.fire)(normalReadPending := false.B) - when(normal.write.req.fire)(normalWritePending := true.B) - when(normal.write.resp.fire)(normalWritePending := false.B) - when(local.request.fire) { - assert(local.request.bits.bank <= b.frontend.vbank_id_upper_bound.U) - assert(local.request.bits.addr < b.memDomain.bankEntries.U) - assert(!local.request.bits.write || local.request.bits.mask.andR) - localPending := true.B - localIsWrite := local.request.bits.write - localTag := local.request.bits.tag - } - when(local.response.fire)(localPending := false.B) - } else { - midend.io.mem_req(0) <> backend.io.mem_req(0) - io.meshLocalBank.request.ready := false.B - io.meshLocalBank.response.valid := false.B - io.meshLocalBank.response.bits := 0.U.asTypeOf(io.meshLocalBank.response.bits) - } - backend.io.config <> frontend.io.config - -//===----------------------------------------------------------------------===// -// MMIO subsystem wiring -//===----------------------------------------------------------------------===// - val loaderBankWrite = frontend.io.interdma.bankWrite - val dmaBankWrite = midend.io.bankWrite(totalBallWrite).bankWrite - midend.io.bankWrite(totalBallWrite).is_shared := frontend.io.interdma.write_is_shared - - if (b.memDomain.mmioEnable) { - val mmioPool: Instance[MmioPool] = Instantiate(new MmioPool(b)) - - // Write path: route MemLoader's bankWrite to MmioPool when is_mvin_mmio_active - val destIsMmio = frontend.io.is_mvin_mmio_active - - // MMIO is one globally encoded byte space. Each DMA beat is 16 bytes and - // MmioPool stripes those bytes across the five physical byte banks. - val mmioWriteAddr = frontend.io.mmio_addr + loaderBankWrite.io.req.bits.addr * (b.memDomain.bankWidth / 8).U - val mmioByteMask = Wire(Vec(b.memDomain.bankMaskLen, Bool())) - for (k <- 0 until b.memDomain.bankMaskLen) { - mmioByteMask(k) := k.U < frontend.io.mmio_col - } - - dmaBankWrite.bank_id := loaderBankWrite.bank_id - dmaBankWrite.rob_id := loaderBankWrite.rob_id - dmaBankWrite.ball_id := loaderBankWrite.ball_id - dmaBankWrite.group_id := loaderBankWrite.group_id - - // Route write to MMIO or main bank based on is_mvin_mmio_active - mmioPool.io.write.req.valid := loaderBankWrite.io.req.valid && destIsMmio - mmioPool.io.write.req.bits.addr := loaderBankWrite.io.req.bits.addr - mmioPool.io.write.req.bits.data := loaderBankWrite.io.req.bits.data - mmioPool.io.write.req.bits.mask := mmioByteMask - mmioPool.io.writeAddr := mmioWriteAddr - - // Main bank write (when NOT mvin_mmio). - dmaBankWrite.io.req.valid := loaderBankWrite.io.req.valid && !destIsMmio - dmaBankWrite.io.req.bits := loaderBankWrite.io.req.bits - - // Request ready mux: select MMIO or main bank ready - loaderBankWrite.io.req.ready := Mux( - destIsMmio, - mmioPool.io.write.req.ready, - dmaBankWrite.io.req.ready - ) - - // Response mux: select MMIO or main bank response - loaderBankWrite.io.resp.valid := Mux( - destIsMmio, - mmioPool.io.write.resp.valid, - dmaBankWrite.io.resp.valid - ) - loaderBankWrite.io.resp.bits := Mux( - destIsMmio, - mmioPool.io.write.resp.bits, - dmaBankWrite.io.resp.bits - ) - - // Ready signals - mmioPool.io.write.resp.ready := loaderBankWrite.io.resp.ready && destIsMmio - dmaBankWrite.io.resp.ready := loaderBankWrite.io.resp.ready && !destIsMmio - - // Ball read path: connect every configured Blink MMIO line to MmioPool. - for (i <- 0 until totalMmioRead) { - mmioPool.io.ballReq(i) <> io.ballDomain.mmioRead(i).req - io.ballDomain.mmioRead(i).resp <> mmioPool.io.ballResp(i) - } - for (i <- 0 until totalMmioWrite) { - mmioPool.io.ballWriteReq(i) <> io.ballDomain.mmioWrite(i).req - } - } else { - dmaBankWrite <> loaderBankWrite - assert(!frontend.io.is_mvin_mmio_active, "MemDomain MMIO is disabled, but mvin_mmio was issued") - - for (i <- 0 until totalMmioRead) { - io.ballDomain.mmioRead(i).req.ready := false.B - io.ballDomain.mmioRead(i).resp.valid := false.B - io.ballDomain.mmioRead(i).resp.bits := 0.U.asTypeOf(io.ballDomain.mmioRead(i).resp.bits) - } - for (i <- 0 until totalMmioWrite) { - io.ballDomain.mmioWrite(i).req.ready := false.B - } - } - - // Shared path passthrough - io.shared_mem_req <> backend.io.shared_mem_req - io.shared_config <> backend.io.shared_config -} diff --git a/arch/src/main/scala/framework/memdomain/backend/MTrace.scala b/arch/src/main/scala/framework/memdomain/backend/MTrace.scala deleted file mode 100644 index f8c551536..000000000 --- a/arch/src/main/scala/framework/memdomain/backend/MTrace.scala +++ /dev/null @@ -1,115 +0,0 @@ -package framework.memdomain.backend - -import chisel3._ -import chisel3.util._ -import framework.dpi.DpiGuard - -class MTraceDPI extends BlackBox with HasBlackBoxInline { - - val io = IO(new Bundle { - val clock = Input(Clock()) - val reset = Input(Bool()) - val is_write = Input(UInt(8.W)) - val is_shared = Input(UInt(8.W)) - val channel = Input(UInt(32.W)) - val hart_id = Input(UInt(64.W)) - val rob_id = Input(UInt(32.W)) - val inst_id = Input(UInt(64.W)) - val vbank_id = Input(UInt(32.W)) - val pbank_id = Input(UInt(32.W)) - val group_id = Input(UInt(32.W)) - val addr = Input(UInt(32.W)) - val write_mask = Input(UInt(32.W)) - val data_lo = Input(UInt(64.W)) - val data_hi = Input(UInt(64.W)) - val enable = Input(Bool()) - }) - - setInline( - "MTraceDPI.v", - """ - |module MTraceDPI( - | input clock, - | input reset, - | input [7:0] is_write, - | input [7:0] is_shared, - | input [31:0] channel, - | input [63:0] hart_id, - | input [31:0] rob_id, - | input [63:0] inst_id, - | input [31:0] vbank_id, - | input [31:0] pbank_id, - | input [31:0] group_id, - | input [31:0] addr, - | input [31:0] write_mask, - | input [63:0] data_lo, - | input [63:0] data_hi, - | input enable - |); - |""".stripMargin + DpiGuard.wrapMTrace(""" - | import "DPI-C" context function void dpi_mtrace( - | input int unsigned is_write, - | input int unsigned is_shared, - | input int unsigned channel, - | input int unsigned hart_id_lo, - | input int unsigned hart_id_hi, - | input int unsigned rob_id, - | input int unsigned inst_id_lo, - | input int unsigned inst_id_hi, - | input int unsigned vbank_id, - | input int unsigned pbank_id, - | input int unsigned group_id, - | input int unsigned addr, - | input int unsigned write_mask, - | input int unsigned data_lo_lo, - | input int unsigned data_lo_hi, - | input int unsigned data_hi_lo, - | input int unsigned data_hi_hi - | ); - | reg [7:0] is_write_reg; - | reg [7:0] is_shared_reg; - | reg [31:0] channel_reg; - | reg [63:0] hart_id_reg; - | reg [31:0] rob_id_reg; - | reg [63:0] inst_id_reg; - | reg [31:0] vbank_id_reg; - | reg [31:0] pbank_id_reg; - | reg [31:0] group_id_reg; - | reg [31:0] addr_reg; - | reg [31:0] write_mask_reg; - | reg [63:0] data_lo_reg; - | reg [63:0] data_hi_reg; - | reg valid_reg; - | - | always @(posedge clock) begin - | if (reset) begin - | valid_reg <= 1'b0; - | end else begin - | if (valid_reg) begin - | dpi_mtrace(is_write_reg, is_shared_reg, channel_reg, hart_id_reg[31:0], hart_id_reg[63:32], rob_id_reg, inst_id_reg[31:0], inst_id_reg[63:32], vbank_id_reg, pbank_id_reg, group_id_reg, addr_reg, write_mask_reg, data_lo_reg[31:0], data_lo_reg[63:32], data_hi_reg[31:0], data_hi_reg[63:32]); - | end - | - | valid_reg <= enable; - | if (enable) begin - | is_write_reg <= is_write; - | is_shared_reg <= is_shared; - | channel_reg <= channel; - | hart_id_reg <= hart_id; - | rob_id_reg <= rob_id; - | inst_id_reg <= inst_id; - | vbank_id_reg <= vbank_id; - | pbank_id_reg <= pbank_id; - | group_id_reg <= group_id; - | addr_reg <= addr; - | write_mask_reg <= write_mask; - | data_lo_reg <= data_lo; - | data_hi_reg <= data_hi; - | end - | end - | end - |""".stripMargin) + - """ - |endmodule - """.stripMargin - ) -} diff --git a/arch/src/main/scala/framework/memdomain/backend/MemBackend.scala b/arch/src/main/scala/framework/memdomain/backend/MemBackend.scala deleted file mode 100644 index 1faa814c0..000000000 --- a/arch/src/main/scala/framework/memdomain/backend/MemBackend.scala +++ /dev/null @@ -1,292 +0,0 @@ -package framework.memdomain.backend - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import chisel3.util._ -import framework.memdomain.frontend.mem.MemConfigerIO -import framework.top.GlobalConfig -import framework.memdomain.backend.privatepath.PrivateMemBackend -import framework.memdomain.backend.shared.SharedMemLayout -import framework.memdomain.backend.banks.btrace.PhysicalBankHash -import framework.memdomain.backend.banks.{SramReadIO, SramWriteIO} - -class MemRequestIO(b: GlobalConfig) extends Bundle { - val write = Flipped(new SramWriteIO(b)) // midend sends write req into backend - val read = Flipped(new SramReadIO(b)) // midend sends read req into backend - val bank_id = Output(UInt(b.memDomain.vbankIdWidth.W)) - val group_id = Output(UInt(b.memDomain.groupIdWidth.W)) - val is_shared = Output(Bool()) - val hart_id = Output(UInt(b.tile.xLen.W)) - val rob_id = Output(UInt(log2Up(b.frontend.rob_entries).W)) - val inst_id = Output(UInt(64.W)) -} - -@instantiable -class MemBackend(val b: GlobalConfig) extends Module { - val kernelRequests = if (b.rvv.enable) 2 * b.rvv.memoryPorts else 0 - - val sharedHashCount = if (b.memDomain.sharedEnable) SharedMemLayout.totalBank(b) else 0 - - @public - val io = IO(new Bundle { - val mem_req = Vec(b.memDomain.bankChannel, Flipped(new MemRequestIO(b))) - val kernel_req = Vec(kernelRequests, Flipped(new MemRequestIO(b))) - val config = Flipped(Decoupled(new MemConfigerIO(b))) - - // Shared path — exposed to tile level for multi-core sharing. - // Number of shared ports per core comes from TOML sharedInputChannels / computeCoreIds.size. - val shared_mem_req = Vec(SharedMemLayout.channelPerHart(b), new MemRequestIO(b)) - val shared_config = Decoupled(new MemConfigerIO(b)) - - // Query interface: shared query goes out, private query handled internally. - val shared_query_valid = Output(Bool()) - val shared_query_vbank_id = Output(UInt(b.memDomain.vbankIdWidth.W)) - val shared_query_group_count = Input(UInt(b.memDomain.groupCountWidth.W)) - - // Original query interface from frontend - val query_vbank_id = Input(UInt(b.memDomain.vbankIdWidth.W)) - val query_is_shared = Input(Bool()) - val query_group_count = Output(UInt(b.memDomain.groupCountWidth.W)) - val clearBusy = Output(Bool()) - - val bank_hashes = - if (b.sim.diffTest) { - Some(Output(Vec(b.memDomain.bankNum + sharedHashCount, new PhysicalBankHash(b)))) - } else { - None - } - - val shared_bank_hashes = - if (b.sim.diffTest && b.memDomain.sharedEnable) { - Some(Input(Vec(SharedMemLayout.totalBank(b), new PhysicalBankHash(b)))) - } else { - None - } - - }) - - // Keep the private backend datapath unchanged and isolate it in a dedicated module. - val privateBackend: Instance[PrivateMemBackend] = Instantiate(new PrivateMemBackend(b)) - for (i <- 0 until kernelRequests) { - privateBackend.io.kernel_req(i) <> io.kernel_req(i) - when(io.kernel_req(i).read.req.valid || io.kernel_req(i).write.req.valid) { - assert(!io.kernel_req(i).is_shared) - assert(io.kernel_req(i).bank_id <= b.frontend.vbank_id_upper_bound.U) - } - } - val sharedChannelPerHart = SharedMemLayout.channelPerHart(b) - - io.bank_hashes.foreach { states => - for (i <- 0 until b.memDomain.bankNum) { - states(i) := privateBackend.io.bank_hashes.get(i) - } - for (i <- 0 until sharedHashCount) { - states(b.memDomain.bankNum + i) := io.shared_bank_hashes.get(i) - } - } - - // Route config to the selected backend only. - val cfgToShared = io.config.bits.is_shared - privateBackend.io.config.valid := io.config.valid && !cfgToShared - privateBackend.io.config.bits := io.config.bits - if (b.memDomain.sharedEnable) { - io.shared_config.valid := io.config.valid && cfgToShared - io.shared_config.bits := io.config.bits - io.config.ready := Mux(cfgToShared, io.shared_config.ready, privateBackend.io.config.ready) - } else { - io.shared_config.valid := false.B - io.shared_config.bits := DontCare - io.config.ready := Mux(cfgToShared, false.B, privateBackend.io.config.ready) - when(io.config.valid && cfgToShared) { - assert(false.B, "MemBackend shared config received while sharedMem is disabled\n") - } - } - - privateBackend.io.query_vbank_id := io.query_vbank_id - io.clearBusy := privateBackend.io.clearBusy - - if (b.memDomain.sharedEnable) { - // Shared query and request routing are only elaborated for shared chips. - io.shared_query_valid := io.query_is_shared - io.shared_query_vbank_id := io.query_vbank_id - io.query_group_count := Mux( - io.query_is_shared, - io.shared_query_group_count, - privateBackend.io.query_group_count - ) - - // Per-channel request routing: is_shared=0 -> private, is_shared=1 -> shared IO. - // Route selection is latched at request fire to keep response demux stable. - val readPending = RegInit(VecInit(Seq.fill(b.memDomain.bankChannel)(false.B))) - val writePending = RegInit(VecInit(Seq.fill(b.memDomain.bankChannel)(false.B))) - val readRouteShared = RegInit(VecInit(Seq.fill(b.memDomain.bankChannel)(false.B))) - val writeRouteShared = RegInit(VecInit(Seq.fill(b.memDomain.bankChannel)(false.B))) - - // Shared IO defaults - for (i <- 0 until sharedChannelPerHart) { - io.shared_mem_req(i).bank_id := 0.U - io.shared_mem_req(i).group_id := 0.U - io.shared_mem_req(i).is_shared := false.B - io.shared_mem_req(i).hart_id := 0.U - io.shared_mem_req(i).rob_id := 0.U - io.shared_mem_req(i).inst_id := 0.U - - io.shared_mem_req(i).read.req.valid := false.B - io.shared_mem_req(i).read.req.bits := DontCare - io.shared_mem_req(i).read.resp.ready := false.B - - io.shared_mem_req(i).write.req.valid := false.B - io.shared_mem_req(i).write.req.bits := DontCare - io.shared_mem_req(i).write.resp.ready := false.B - } - - for (i <- 0 until b.memDomain.bankChannel) { - val useSharedReqRaw = io.mem_req(i).bank_id >= b.frontend.shared_bank_id_base.U && - io.mem_req(i).bank_id < b.memDomain.virtualBankCount.U - val canUseSharedPort = i < sharedChannelPerHart - val hasReq = io.mem_req(i).read.req.valid || io.mem_req(i).write.req.valid - when(useSharedReqRaw && !canUseSharedPort.B && hasReq) { - assert( - false.B, - p"shared request on unsupported channel i=$i (sharedChannelPerHart=$sharedChannelPerHart, bankChannel=${b.memDomain.bankChannel})" - ) - } - val useSharedReq = useSharedReqRaw && canUseSharedPort.B - val useSharedReadResp = Mux(readPending(i), readRouteShared(i), useSharedReq) - val useSharedWriteResp = Mux(writePending(i), writeRouteShared(i), useSharedReq) - - when(io.mem_req(i).read.req.fire) { - readPending(i) := true.B - readRouteShared(i) := useSharedReq - } - when(io.mem_req(i).read.resp.fire) { - readPending(i) := false.B - } - - when(io.mem_req(i).write.req.fire) { - writePending(i) := true.B - writeRouteShared(i) := useSharedReq - } - when(io.mem_req(i).write.resp.fire) { - writePending(i) := false.B - } - - // Metadata is passed to both backends; only selected backend receives valid req/resp-ready. - privateBackend.io.mem_req(i).bank_id := io.mem_req(i).bank_id - privateBackend.io.mem_req(i).group_id := io.mem_req(i).group_id - privateBackend.io.mem_req(i).is_shared := useSharedReq - privateBackend.io.mem_req(i).hart_id := io.mem_req(i).hart_id - privateBackend.io.mem_req(i).rob_id := io.mem_req(i).rob_id - privateBackend.io.mem_req(i).inst_id := io.mem_req(i).inst_id - - // Read request route - privateBackend.io.mem_req(i).read.req.valid := io.mem_req(i).read.req.valid && !useSharedReq - privateBackend.io.mem_req(i).read.req.bits := io.mem_req(i).read.req.bits - - // Write request route - privateBackend.io.mem_req(i).write.req.valid := io.mem_req(i).write.req.valid && !useSharedReq - privateBackend.io.mem_req(i).write.req.bits := io.mem_req(i).write.req.bits - - // Response ready route (selected by latched request route when pending). - privateBackend.io.mem_req(i).read.resp.ready := io.mem_req(i).read.resp.ready && !useSharedReadResp - privateBackend.io.mem_req(i).write.resp.ready := io.mem_req(i).write.resp.ready && !useSharedWriteResp - - if (i < sharedChannelPerHart) { - io.shared_mem_req(i).bank_id := io.mem_req(i).bank_id - io.shared_mem_req(i).group_id := io.mem_req(i).group_id - io.shared_mem_req(i).is_shared := useSharedReq - io.shared_mem_req(i).hart_id := io.mem_req(i).hart_id - io.shared_mem_req(i).rob_id := io.mem_req(i).rob_id - io.shared_mem_req(i).inst_id := io.mem_req(i).inst_id - - io.shared_mem_req(i).read.req.valid := io.mem_req(i).read.req.valid && useSharedReq - io.shared_mem_req(i).read.req.bits := io.mem_req(i).read.req.bits - io.mem_req(i).read.req.ready := Mux( - useSharedReq, - io.shared_mem_req(i).read.req.ready, - privateBackend.io.mem_req(i).read.req.ready - ) - - io.shared_mem_req(i).write.req.valid := io.mem_req(i).write.req.valid && useSharedReq - io.shared_mem_req(i).write.req.bits := io.mem_req(i).write.req.bits - io.mem_req(i).write.req.ready := Mux( - useSharedReq, - io.shared_mem_req(i).write.req.ready, - privateBackend.io.mem_req(i).write.req.ready - ) - - io.shared_mem_req(i).read.resp.ready := io.mem_req(i).read.resp.ready && useSharedReadResp - io.shared_mem_req(i).write.resp.ready := io.mem_req(i).write.resp.ready && useSharedWriteResp - - io.mem_req(i).read.resp.valid := Mux( - useSharedReadResp, - io.shared_mem_req(i).read.resp.valid, - privateBackend.io.mem_req(i).read.resp.valid - ) - io.mem_req(i).read.resp.bits := Mux( - useSharedReadResp, - io.shared_mem_req(i).read.resp.bits, - privateBackend.io.mem_req(i).read.resp.bits - ) - - io.mem_req(i).write.resp.valid := Mux( - useSharedWriteResp, - io.shared_mem_req(i).write.resp.valid, - privateBackend.io.mem_req(i).write.resp.valid - ) - io.mem_req(i).write.resp.bits := Mux( - useSharedWriteResp, - io.shared_mem_req(i).write.resp.bits, - privateBackend.io.mem_req(i).write.resp.bits - ) - } else { - io.mem_req(i).read.req.ready := privateBackend.io.mem_req(i).read.req.ready - io.mem_req(i).write.req.ready := privateBackend.io.mem_req(i).write.req.ready - io.mem_req(i).read.resp.valid := privateBackend.io.mem_req(i).read.resp.valid - io.mem_req(i).read.resp.bits := privateBackend.io.mem_req(i).read.resp.bits - io.mem_req(i).write.resp.valid := privateBackend.io.mem_req(i).write.resp.valid - io.mem_req(i).write.resp.bits := privateBackend.io.mem_req(i).write.resp.bits - } - } - } else { - io.shared_query_valid := false.B - io.shared_query_vbank_id := 0.U - io.query_group_count := privateBackend.io.query_group_count - - for (i <- 0 until SharedMemLayout.channelPerHart(b)) { - io.shared_mem_req(i).bank_id := 0.U - io.shared_mem_req(i).group_id := 0.U - io.shared_mem_req(i).is_shared := false.B - io.shared_mem_req(i).hart_id := 0.U - io.shared_mem_req(i).rob_id := 0.U - io.shared_mem_req(i).inst_id := 0.U - io.shared_mem_req(i).read.req.valid := false.B - io.shared_mem_req(i).read.req.bits := DontCare - io.shared_mem_req(i).read.resp.ready := false.B - io.shared_mem_req(i).write.req.valid := false.B - io.shared_mem_req(i).write.req.bits := DontCare - io.shared_mem_req(i).write.resp.ready := false.B - } - - for (i <- 0 until b.memDomain.bankChannel) { - privateBackend.io.mem_req(i).bank_id := io.mem_req(i).bank_id - privateBackend.io.mem_req(i).group_id := io.mem_req(i).group_id - privateBackend.io.mem_req(i).is_shared := false.B - privateBackend.io.mem_req(i).hart_id := io.mem_req(i).hart_id - privateBackend.io.mem_req(i).rob_id := io.mem_req(i).rob_id - privateBackend.io.mem_req(i).inst_id := io.mem_req(i).inst_id - privateBackend.io.mem_req(i).read.req.valid := io.mem_req(i).read.req.valid - privateBackend.io.mem_req(i).read.req.bits := io.mem_req(i).read.req.bits - privateBackend.io.mem_req(i).read.resp.ready := io.mem_req(i).read.resp.ready - io.mem_req(i).read.req.ready := privateBackend.io.mem_req(i).read.req.ready - io.mem_req(i).read.resp.valid := privateBackend.io.mem_req(i).read.resp.valid - io.mem_req(i).read.resp.bits := privateBackend.io.mem_req(i).read.resp.bits - privateBackend.io.mem_req(i).write.req.valid := io.mem_req(i).write.req.valid - privateBackend.io.mem_req(i).write.req.bits := io.mem_req(i).write.req.bits - privateBackend.io.mem_req(i).write.resp.ready := io.mem_req(i).write.resp.ready - io.mem_req(i).write.req.ready := privateBackend.io.mem_req(i).write.req.ready - io.mem_req(i).write.resp.valid := privateBackend.io.mem_req(i).write.resp.valid - io.mem_req(i).write.resp.bits := privateBackend.io.mem_req(i).write.resp.bits - } - } -} diff --git a/arch/src/main/scala/framework/memdomain/backend/accpipe/AccPipe.scala b/arch/src/main/scala/framework/memdomain/backend/accpipe/AccPipe.scala deleted file mode 100644 index ecf7c1035..000000000 --- a/arch/src/main/scala/framework/memdomain/backend/accpipe/AccPipe.scala +++ /dev/null @@ -1,95 +0,0 @@ -package framework.memdomain.backend.accpipe - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} - -import framework.top.GlobalConfig -import framework.memdomain.backend.banks.{SramReadIO, SramWriteIO} -import framework.memdomain.backend.MemRequestIO - -@instantiable -class AccPipe(val b: GlobalConfig) extends Module { - - @public - val io = IO(new Bundle { - val sramRead = Flipped(new SramReadIO(b)) - val sramWrite = Flipped(new SramWriteIO(b)) - - val mem_req = Flipped(new MemRequestIO(b)) - - val busy = Output(Bool()) - val group_id = Output(UInt(b.memDomain.groupIdWidth.W)) - val bank_id = Output(UInt(b.memDomain.vbankIdWidth.W)) - val hart_id = Output(UInt(b.tile.xLen.W)) - }) - - // Each group has its own physical bank, so no address shifting is needed. - - //group_id output - val group_id_reg = RegInit(0.U(b.memDomain.groupIdWidth.W)) - io.group_id := group_id_reg - - //Bank_id output - val bank_id_reg = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - io.bank_id := bank_id_reg - val hart_id_reg = RegInit(0.U(b.tile.xLen.W)) - io.hart_id := hart_id_reg - - val rd_hold = RegInit(false.B) - val rd_data_reg = RegInit(0.U(b.memDomain.bankWidth.W)) - val wr_hold = RegInit(false.B) - val wr_ok_reg = RegInit(false.B) - val busy_reg = RegInit(false.B) - - val canStart = !busy_reg && !rd_hold && !wr_hold - val hasWriteReq = io.mem_req.write.req.valid - val wrReq = canStart && hasWriteReq - val rdReq = canStart && !hasWriteReq && io.mem_req.read.req.valid - - io.sramRead.req.valid := rdReq - io.sramRead.req.bits.addr := io.mem_req.read.req.bits.addr - io.sramRead.resp.ready := !rd_hold - - io.sramWrite.req.valid := wrReq - io.sramWrite.req.bits.addr := io.mem_req.write.req.bits.addr - io.sramWrite.req.bits.data := io.mem_req.write.req.bits.data - io.sramWrite.req.bits.mask := io.mem_req.write.req.bits.mask - io.sramWrite.resp.ready := !wr_hold - - io.mem_req.read.req.ready := canStart && !hasWriteReq && io.sramRead.req.ready - io.mem_req.read.resp.valid := rd_hold - io.mem_req.read.resp.bits.data := rd_data_reg - - io.mem_req.write.req.ready := canStart && io.sramWrite.req.ready - io.mem_req.write.resp.valid := wr_hold - io.mem_req.write.resp.bits.ok := wr_ok_reg - - when(rd_hold && io.mem_req.read.resp.ready) { - rd_hold := false.B - }.elsewhen(io.sramRead.resp.fire) { - rd_hold := true.B - rd_data_reg := io.sramRead.resp.bits.data - } - - when(wr_hold && io.mem_req.write.resp.ready) { - wr_hold := false.B - }.elsewhen(io.sramWrite.resp.fire) { - wr_hold := true.B - wr_ok_reg := io.sramWrite.resp.bits.ok - } - - when(io.mem_req.read.req.fire || io.mem_req.write.req.fire) { - group_id_reg := io.mem_req.group_id - bank_id_reg := io.mem_req.bank_id - hart_id_reg := io.mem_req.hart_id - } - when(io.mem_req.read.req.fire || io.mem_req.write.req.fire) { - busy_reg := true.B - } - when(io.mem_req.read.resp.fire || io.mem_req.write.resp.fire) { - busy_reg := false.B - } - - io.busy := busy_reg -} diff --git a/arch/src/main/scala/framework/memdomain/backend/banks/SramBank.scala b/arch/src/main/scala/framework/memdomain/backend/banks/SramBank.scala deleted file mode 100644 index 9a3f2ebf3..000000000 --- a/arch/src/main/scala/framework/memdomain/backend/banks/SramBank.scala +++ /dev/null @@ -1,91 +0,0 @@ -package framework.memdomain.backend.banks - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.top.GlobalConfig - -/** - * SramBank: Pure SRAM bank - * Simple read/write memory without any accumulation logic - * Each bank is a single-port SRAM. `clear` zeroes every row locally, one row per cycle; - * the ports refuse requests while `clearing`, so banks clear in parallel. - */ -@instantiable -class SramBank(val b: GlobalConfig) extends Module { - require(b.memDomain.bankEntries >= 2 && b.memDomain.bankEntries <= 65536) - val indexBits = log2Ceil(b.memDomain.bankEntries) - val mask_len = b.memDomain.bankMaskLen - val mask_elem = UInt((b.memDomain.bankWidth / mask_len).W) - - @public - val io = IO(new Bundle { - val sramRead = new SramReadIO(b) - val sramWrite = new SramWriteIO(b) - val clear = Input(Bool()) - val clearing = Output(Bool()) - }) - - val mem = SyncReadMem(b.memDomain.bankEntries, Vec(mask_len, mask_elem)) - - // ----------------------------------------------------------------------------- - // Local clear - // ----------------------------------------------------------------------------- - val clearing = RegInit(false.B) - val clearRow = RegInit(0.U(log2Ceil(b.memDomain.bankEntries).W)) - io.clearing := clearing - when(io.clear) { - clearing := true.B - clearRow := 0.U - }.elsewhen(clearing) { - clearRow := clearRow + 1.U - when(clearRow === (b.memDomain.bankEntries - 1).U)(clearing := false.B) - } - - val readPending = RegNext(io.sramRead.req.fire, false.B) - val readHeld = RegInit(false.B) - val readData = Reg(UInt(b.memDomain.bankWidth.W)) - val writeValid = RegInit(false.B) - - io.sramRead.resp.valid := readPending || readHeld - io.sramWrite.resp.valid := writeValid - io.sramWrite.resp.bits.ok := true.B - io.sramWrite.req.ready := !io.clear && !clearing && (!writeValid || io.sramWrite.resp.ready) - io.sramRead.req.ready := !io.clear && !clearing && !io.sramWrite.req.fire && - (!io.sramRead.resp.valid || io.sramRead.resp.ready) - - val ren = io.sramRead.req.fire - val wen = clearing || io.sramWrite.req.fire - - when(io.sramRead.req.fire) { - assert(io.sramRead.req.bits.addr < b.memDomain.bankEntries.U, "SRAM read exceeds bank depth") - } - when(io.sramWrite.req.fire) { - assert(io.sramWrite.req.bits.addr < b.memDomain.bankEntries.U, "SRAM write exceeds bank depth") - } - - val rdata = mem.readWrite( - Mux( - clearing, - clearRow, - Mux(wen, io.sramWrite.req.bits.addr(indexBits - 1, 0), io.sramRead.req.bits.addr(indexBits - 1, 0)) - ), - Mux(clearing, 0.U.asTypeOf(Vec(mask_len, mask_elem)), io.sramWrite.req.bits.data.asTypeOf(Vec(mask_len, mask_elem))), - Mux(clearing, VecInit(Seq.fill(mask_len)(true.B)), io.sramWrite.req.bits.mask), - ren || wen, - wen - ) - - io.sramRead.resp.bits.data := Mux(readHeld, readData, rdata.asUInt) - when(readPending && !io.sramRead.resp.ready) { - readHeld := true.B - readData := rdata.asUInt - }.elsewhen(io.sramRead.resp.fire) { - readHeld := false.B - } - when(io.sramWrite.req.fire) { - writeValid := true.B - }.elsewhen(io.sramWrite.resp.fire) { - writeValid := false.B - } -} diff --git a/arch/src/main/scala/framework/memdomain/backend/banks/SramIO.scala b/arch/src/main/scala/framework/memdomain/backend/banks/SramIO.scala deleted file mode 100644 index 488db7ae5..000000000 --- a/arch/src/main/scala/framework/memdomain/backend/banks/SramIO.scala +++ /dev/null @@ -1,36 +0,0 @@ -package framework.memdomain.backend.banks - -import chisel3._ -import chisel3.util._ -import framework.top.GlobalConfig - -/** - * Generic SRAM interface definitions - */ -class SramReadReq(val b: GlobalConfig) extends Bundle { - val addr = UInt(16.W) -} - -class SramReadResp(val b: GlobalConfig) extends Bundle { - val data = UInt(b.memDomain.bankWidth.W) -} - -class SramReadIO(val b: GlobalConfig) extends Bundle { - val req = Flipped(Decoupled(new SramReadReq(b))) - val resp = Decoupled(new SramReadResp(b)) -} - -class SramWriteReq(val b: GlobalConfig) extends Bundle { - val addr = UInt(16.W) - val mask = Vec(b.memDomain.bankMaskLen, Bool()) - val data = UInt(b.memDomain.bankWidth.W) -} - -class SramWriteIO(val b: GlobalConfig) extends Bundle { - val req = Flipped(Decoupled(new SramWriteReq(b))) - val resp = Decoupled(new SramWriteResp(b)) -} - -class SramWriteResp(val b: GlobalConfig) extends Bundle { - val ok = Bool() -} diff --git a/arch/src/main/scala/framework/memdomain/backend/banks/btrace/BTraceDPI.scala b/arch/src/main/scala/framework/memdomain/backend/banks/btrace/BTraceDPI.scala deleted file mode 100644 index 52db28039..000000000 --- a/arch/src/main/scala/framework/memdomain/backend/banks/btrace/BTraceDPI.scala +++ /dev/null @@ -1,91 +0,0 @@ -package framework.memdomain.backend.banks.btrace - -import chisel3._ -import chisel3.util._ -import framework.dpi.DpiGuard -import framework.top.GlobalConfig - -class BTraceRecord extends Bundle { - val instId = UInt(64.W) - val hartId = UInt(64.W) - val w0Vbank = UInt(32.W) - val w0Hash = UInt(32.W) -} - -class PhysicalBankHash(val b: GlobalConfig) extends Bundle { - val valid = Bool() - val hartId = UInt(b.tile.xLen.W) - val vbankId = UInt(b.memDomain.vbankIdWidth.W) - val pbankId = UInt(32.W) - val groupId = UInt(32.W) - val statusHash = UInt(32.W) -} - -class BTraceDPI extends BlackBox with HasBlackBoxInline { - - val io = IO(new Bundle { - val clock = Input(Clock()) - val reset = Input(Bool()) - val instId = Input(UInt(64.W)) - val hartId = Input(UInt(64.W)) - val ownerHartId = Input(UInt(64.W)) - val w0Vbank = Input(UInt(32.W)) - val w0Hash = Input(UInt(32.W)) - val fire = Input(Bool()) - val produced = Input(UInt(64.W)) - val idle = Input(Bool()) - }) - - setInline( - "BTraceDPI.v", - """ - |module BTraceDPI( - | input clock, - | input reset, - | input [63:0] instId, - | input [63:0] hartId, - | input [63:0] ownerHartId, - | input [31:0] w0Vbank, - | input [31:0] w0Hash, - | input fire, - | input [63:0] produced, - | input idle - |); - |""".stripMargin + DpiGuard.wrapBTrace( - """ - | export "DPI-C" function btrace_snapshot; - | function void btrace_snapshot( - | output int unsigned hart_lo, - | output int unsigned hart_hi, - | output int unsigned produced_lo, - | output int unsigned produced_hi, - | output int unsigned is_idle - | ); - | hart_lo = hartId[31:0]; - | hart_hi = hartId[63:32]; - | produced_lo = produced[31:0]; - | produced_hi = produced[63:32]; - | is_idle = {31'b0, idle}; - | endfunction - | import "DPI-C" context function void dpi_btrace( - | input int unsigned inst_id_lo, - | input int unsigned inst_id_hi, - | input int unsigned hart_id_lo, - | input int unsigned hart_id_hi, - | input int unsigned owner_hart_id_lo, - | input int unsigned owner_hart_id_hi, - | input int unsigned w0_vbank, - | input int unsigned w0_hash - | ); - | always @(posedge clock) begin - | if (!reset && fire) begin - | dpi_btrace(instId[31:0], instId[63:32], hartId[31:0], hartId[63:32], ownerHartId[31:0], ownerHartId[63:32], w0Vbank, w0Hash); - | end - | end - |""".stripMargin - ) + - """ - |endmodule - |""".stripMargin - ) -} diff --git a/arch/src/main/scala/framework/memdomain/backend/mmio/MmioBank.scala b/arch/src/main/scala/framework/memdomain/backend/mmio/MmioBank.scala deleted file mode 100644 index 4864c5257..000000000 --- a/arch/src/main/scala/framework/memdomain/backend/mmio/MmioBank.scala +++ /dev/null @@ -1,62 +0,0 @@ -package framework.memdomain.backend.mmio - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.top.GlobalConfig - -/** - * MmioBank: single MMIO SRAM bank. - * - * Internal storage : mmioBankEntries x mmioReadWidth bits (default 1024 x 8 = 1 KB) - * Write port : one byte; MmioPool distributes DMA beats across banks. - * Read port : one byte. - * - * Read latency : 1 cycle. Write/read uses single-port semantics; write wins. - */ -@instantiable -class MmioBank(val b: GlobalConfig) extends Module { - - val numEntries = b.memDomain.mmioBankEntries - require( - b.memDomain.mmioBankWidth == b.memDomain.mmioReadWidth, - "MmioBank requires one physical MMIO element per read" - ) - require( - b.memDomain.mmioReadWidth == 8, - "MmioBank requires 8-bit MMIO elements" - ) - - @public - val io = IO(new Bundle { - val write = new MmioBankWriteIO(b) - val read = new MmioBankReadIO(b) - }) - - val mem = SyncReadMem(numEntries, UInt(b.memDomain.mmioReadWidth.W)) - - val readPending = RegNext(io.read.req.fire, false.B) - val readHeld = RegInit(false.B) - val readData = Reg(UInt(b.memDomain.mmioReadWidth.W)) - - io.read.resp.valid := readPending || readHeld - io.write.req.ready := true.B - io.read.req.ready := !io.write.req.valid && (!io.read.resp.valid || io.read.resp.ready) - val ren = io.read.req.fire - val wen = io.write.req.fire - - val rdata = mem.readWrite( - Mux(wen, io.write.req.bits.addr, io.read.req.bits.addr), - io.write.req.bits.data, - ren || wen, - wen - ) - - io.read.resp.bits.data := Mux(readHeld, readData, rdata) - when(readPending && !io.read.resp.ready) { - readHeld := true.B - readData := rdata - }.elsewhen(io.read.resp.fire) { - readHeld := false.B - } -} diff --git a/arch/src/main/scala/framework/memdomain/backend/mmio/MmioIO.scala b/arch/src/main/scala/framework/memdomain/backend/mmio/MmioIO.scala deleted file mode 100644 index 345990326..000000000 --- a/arch/src/main/scala/framework/memdomain/backend/mmio/MmioIO.scala +++ /dev/null @@ -1,51 +0,0 @@ -package framework.memdomain.backend.mmio - -import chisel3._ -import chisel3.util._ -import framework.top.GlobalConfig - -/** - * Internal bundles for the MMIO subsystem. - * - * Naming convention: - * *Req / *Resp = Bundles that travel through Decoupled channels - * *Port = Bundles that group req+resp together - */ - -// ===== Per-MmioBank internal read port ===== -class MmioBankReadReq(b: GlobalConfig) extends Bundle { - val addr = UInt(log2Ceil(b.memDomain.mmioBankEntries).W) -} - -class MmioBankReadResp(b: GlobalConfig) extends Bundle { - val data = UInt(b.memDomain.mmioReadWidth.W) -} - -class MmioBankReadIO(b: GlobalConfig) extends Bundle { - val req = Flipped(Decoupled(new MmioBankReadReq(b))) - val resp = Decoupled(new MmioBankReadResp(b)) -} - -class MmioBankWriteReq(b: GlobalConfig) extends Bundle { - val addr = UInt(log2Ceil(b.memDomain.mmioBankEntries).W) - val data = UInt(b.memDomain.mmioReadWidth.W) -} - -class MmioBankWriteIO(b: GlobalConfig) extends Bundle { - val req = Flipped(Decoupled(new MmioBankWriteReq(b))) -} - -// ===== Ball-facing MMIO ports (carried over BlinkIO) ===== -class MmioReadReq(b: GlobalConfig) extends Bundle { - // MMIO is one globally encoded byte address space. - val addr = UInt(log2Ceil(b.memDomain.mmioTotalBytes).W) -} - -class MmioReadResp(b: GlobalConfig) extends Bundle { - val data = UInt(b.memDomain.mmioReadWidth.W) -} - -class MmioWriteReq(b: GlobalConfig) extends Bundle { - val addr = UInt(log2Ceil(b.memDomain.mmioTotalBytes).W) - val data = UInt(b.memDomain.mmioReadWidth.W) -} diff --git a/arch/src/main/scala/framework/memdomain/backend/mmio/MmioPool.scala b/arch/src/main/scala/framework/memdomain/backend/mmio/MmioPool.scala deleted file mode 100644 index 48dd02d4e..000000000 --- a/arch/src/main/scala/framework/memdomain/backend/mmio/MmioPool.scala +++ /dev/null @@ -1,165 +0,0 @@ -package framework.memdomain.backend.mmio - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.memdomain.backend.banks.SramWriteIO -import framework.top.GlobalConfig - -/** - * MmioPool: top-level MMIO subsystem wrapper. - * - * Instantiates: - * - Vec of MmioBanks (one per mmioBankNum) - * - MmioRouter (translates Ball requests to bank requests) - * - * External interfaces: - * - write: from MemLoader (mvin_mmio DMA path) - * - ballReq/ballResp/ballWriteReq: from Balls (via MemDomain wiring) - */ -@instantiable -class MmioPool(val b: GlobalConfig) extends Module { - - val mmioReadPorts = b.ballDomain.ballIdMappings.map(_.mmioReadBW).sum - val mmioWritePorts = b.ballDomain.ballIdMappings.map(_.mmioWriteBW).sum - - @public - val io = IO(new Bundle { - // DMA write path (from MemLoader) - val write = new SramWriteIO(b) - val writeAddr = Input(UInt(log2Ceil(b.memDomain.mmioTotalBytes).W)) - - // Ball read path (from MemDomain, one port per configured Blink line) - val ballReq = - Vec(mmioReadPorts, Flipped(Decoupled(new MmioReadReq(b)))) - val ballResp = Vec(mmioReadPorts, Decoupled(new MmioReadResp(b))) - val ballWriteReq = Vec(mmioWritePorts, Flipped(Decoupled(new MmioWriteReq(b)))) - }) - - // Instantiate components - val banks: Seq[Instance[MmioBank]] = Seq.fill(b.memDomain.mmioBankNum)(Instantiate(new MmioBank(b))) - val router: Instance[MmioRouter] = Instantiate(new MmioRouter(b)) - - // MMIO read addresses are already globally encoded; no region translation. - router.io.ballReq <> io.ballReq - io.ballResp <> router.io.ballResp - - // Wire router to banks (read path) - for (i <- 0 until b.memDomain.mmioBankNum) { - banks(i).io.read.req <> router.io.bankReadReq(i) - banks(i).io.read.resp <> router.io.bankReadResp(i) - } - - // A DMA beat is 128 bits. The globally encoded byte address is striped over - // the homogeneous 8-bit MMIO banks, at most one byte per bank each cycle. - val dmaBytes = b.memDomain.bankWidth / 8 - val writeActive = RegInit(false.B) - val writeAddrReg = RegInit(0.U(log2Ceil(b.memDomain.mmioTotalBytes).W)) - val writeDataReg = RegInit(0.U(b.memDomain.bankWidth.W)) - val writeMaskReg = RegInit(VecInit(Seq.fill(b.memDomain.bankMaskLen)(false.B))) - val writeCursorReg = RegInit(0.U(log2Ceil(dmaBytes).W)) - - io.write.req.ready := !writeActive - when(io.write.req.fire) { - assert(io.writeAddr + dmaBytes.U <= b.memDomain.mmioTotalBytes.U, "MmioPool: DMA write exceeds MMIO address space") - writeActive := true.B - writeAddrReg := io.writeAddr - writeDataReg := io.write.req.bits.data - writeMaskReg := io.write.req.bits.mask - writeCursorReg := 0.U - } - - val finalWriteGroup = writeActive && - (writeCursorReg +& b.memDomain.mmioBankNum.U >= dmaBytes.U) - - if (mmioWritePorts == 0) { - for (bankIdx <- 0 until b.memDomain.mmioBankNum) { - val byteMatch = VecInit((0 until b.memDomain.mmioBankNum).map { offset => - val byteIdx = writeCursorReg +& offset.U - val byteAddr = writeAddrReg + byteIdx - byteIdx < dmaBytes.U && (byteAddr % b.memDomain.mmioBankNum.U === bankIdx.U) - }) - val selectedOffset = PriorityEncoder(byteMatch) - val selectedByte = writeCursorReg +& selectedOffset - val selectedAddr = writeAddrReg + selectedByte - val dmaWriteValid = writeActive && byteMatch.asUInt.orR - val dmaWriteAddr = (selectedAddr / b.memDomain.mmioBankNum.U)(log2Ceil(b.memDomain.mmioBankEntries) - 1, 0) - val selectedData = (writeDataReg >> (selectedByte << 3.U))(7, 0) - val dmaWriteData = Mux(writeMaskReg(selectedByte), selectedData, 0.U) - banks(bankIdx).io.write.req.valid := dmaWriteValid - banks(bankIdx).io.write.req.bits.addr := dmaWriteAddr - banks(bankIdx).io.write.req.bits.data := dmaWriteData - } - } else { - val bwV = RegInit(VecInit(Seq.fill(mmioWritePorts)(false.B))) - val bwA = Reg(Vec(mmioWritePorts, UInt(log2Ceil(b.memDomain.mmioTotalBytes).W))) - val bwD = Reg(Vec(mmioWritePorts, UInt(b.memDomain.mmioReadWidth.W))) - - for (bankIdx <- 0 until b.memDomain.mmioBankNum) { - val byteMatch = VecInit((0 until b.memDomain.mmioBankNum).map { offset => - val byteIdx = writeCursorReg +& offset.U - val byteAddr = writeAddrReg + byteIdx - byteIdx < dmaBytes.U && (byteAddr % b.memDomain.mmioBankNum.U === bankIdx.U) - }) - val selectedOffset = PriorityEncoder(byteMatch) - val selectedByte = writeCursorReg +& selectedOffset - val selectedAddr = writeAddrReg + selectedByte - val dmaWriteValid = writeActive && byteMatch.asUInt.orR - val dmaWriteAddr = (selectedAddr / b.memDomain.mmioBankNum.U)(log2Ceil(b.memDomain.mmioBankEntries) - 1, 0) - val selectedData = (writeDataReg >> (selectedByte << 3.U))(7, 0) - val dmaWriteData = Mux(writeMaskReg(selectedByte), selectedData, 0.U) - - val ballMatches = VecInit((0 until mmioWritePorts).map { i => - bwV(i) && (bwA(i) % b.memDomain.mmioBankNum.U === bankIdx.U) - }) - val ballIdx = PriorityEncoder(ballMatches) - val ballAddr = bwA(ballIdx) - assert( - PopCount(ballMatches) <= 1.U, - "MmioPool: multiple Ball writes target one MMIO bank in a cycle" - ) - assert( - !(writeActive && ballMatches.asUInt.orR), - "MmioPool: DMA and Ball write target one MMIO bank in a cycle" - ) - banks(bankIdx).io.write.req.valid := dmaWriteValid || ballMatches.asUInt.orR - banks(bankIdx).io.write.req.bits.addr := Mux( - dmaWriteValid, - dmaWriteAddr, - (ballAddr / b.memDomain.mmioBankNum.U)(log2Ceil(b.memDomain.mmioBankEntries) - 1, 0) - ) - banks(bankIdx).io.write.req.bits.data := Mux(dmaWriteValid, dmaWriteData, bwD(ballIdx)) - } - - for (i <- 0 until mmioWritePorts) { - val targetBank = bwA(i) % b.memDomain.mmioBankNum.U - val targetReady = Mux1H( - UIntToOH(targetBank, b.memDomain.mmioBankNum), - banks.map(_.io.write.req.ready) - ) - val taken = bwV(i) && !writeActive && targetReady - io.ballWriteReq(i).ready := !bwV(i) || taken - when(taken && !io.ballWriteReq(i).fire) { - bwV(i) := false.B - } - when(io.ballWriteReq(i).fire) { - bwV(i) := true.B - bwA(i) := io.ballWriteReq(i).bits.addr - bwD(i) := io.ballWriteReq(i).bits.data - } - assert( - !(io.ballWriteReq(i).valid && - (io.ballWriteReq(i).bits.addr >= b.memDomain.mmioTotalBytes.U)), - "MmioPool: Ball write exceeds the MMIO address space" - ) - } - } - - io.write.resp.valid := finalWriteGroup - io.write.resp.bits.ok := true.B - when(io.write.resp.fire) { - writeActive := false.B - }.elsewhen(writeActive && !finalWriteGroup) { - writeCursorReg := writeCursorReg + b.memDomain.mmioBankNum.U - } -} diff --git a/arch/src/main/scala/framework/memdomain/backend/mmio/MmioRouter.scala b/arch/src/main/scala/framework/memdomain/backend/mmio/MmioRouter.scala deleted file mode 100644 index 0b3dc30e0..000000000 --- a/arch/src/main/scala/framework/memdomain/backend/mmio/MmioRouter.scala +++ /dev/null @@ -1,137 +0,0 @@ -package framework.memdomain.backend.mmio - -import memcore.memory.queue.Queue - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.top.GlobalConfig - -/** - * MmioRouter: translates Ball mmioRead requests into MmioBank requests. - * - * Per configured Blink MMIO line: - * 1. Receive a globally encoded MMIO byte address. - * 2. Decode the byte address: - * bank_id = absAddr % mmioBankNum - * rowInBank = absAddr / mmioBankNum - * 3. Route req to bankPorts(bank_id). - * - * Concurrency rule (enforced by software): - * At any time, each mmio_bank is accessed by at most one requester. - * Therefore no arbitration is required — bank ports are driven by exactly the matching Ball. - */ -@instantiable -class MmioRouter(val b: GlobalConfig) extends Module { - - val clientNum = b.ballDomain.ballIdMappings.map(_.mmioReadBW).sum - val mmioBankNum = b.memDomain.mmioBankNum - - @public - val io = IO(new Bundle { - // From Balls (via MemDomain wiring) - val ballReq = Vec(clientNum, Flipped(Decoupled(new MmioReadReq(b)))) - val ballResp = Vec(clientNum, Decoupled(new MmioReadResp(b))) - - // Bank ports (one per MmioBank) - val bankReadReq = Vec(mmioBankNum, Decoupled(new MmioBankReadReq(b))) - val bankReadResp = - Vec(mmioBankNum, Flipped(Decoupled(new MmioBankReadResp(b)))) - }) - - val ballAbsAddr = Wire( - Vec(clientNum, UInt(log2Ceil(b.memDomain.mmioTotalBytes).W)) - ) - - val ballBankId = Wire(Vec(clientNum, UInt(log2Ceil(mmioBankNum).W))) - - val ballRowInBank = Wire( - Vec(clientNum, UInt(log2Ceil(b.memDomain.mmioBankEntries).W)) - ) - - for (i <- 0 until clientNum) { - ballAbsAddr(i) := io.ballReq(i).bits.addr - ballBankId(i) := (ballAbsAddr(i) % mmioBankNum.U)( - log2Ceil(mmioBankNum) - 1, - 0 - ) - ballRowInBank(i) := (ballAbsAddr(i) / mmioBankNum.U)( - log2Ceil(b.memDomain.mmioBankEntries) - 1, - 0 - ) - - assert( - !(io.ballReq(i).valid && (ballAbsAddr(i) >= b.memDomain.mmioTotalBytes.U)), - "MmioRouter: read exceeds the MMIO address space\n" - ) - } - - // Per-bank request routing - for (bankIdx <- 0 until mmioBankNum) { - if (clientNum == 0) { - io.bankReadReq(bankIdx).valid := false.B - io.bankReadReq(bankIdx).bits.addr := 0.U - } else { - val ballMatches = VecInit((0 until clientNum).map { i => - io.ballReq(i).valid && (ballBankId(i) === bankIdx.U) - }) - val matchedBall = PriorityEncoder(ballMatches) - val anyMatch = ballMatches.asUInt.orR - - io.bankReadReq(bankIdx).valid := anyMatch - io.bankReadReq(bankIdx).bits.addr := ballRowInBank(matchedBall) - - assert( - PopCount(ballMatches) <= 1.U, - "MmioRouter: bank %d hit by multiple requesters in the same cycle (software invariant violated)\n", - bankIdx.U - ) - } - } - - // Per-Ball req.ready - for (i <- 0 until clientNum) { - io.ballReq(i).ready := io.bankReadReq(ballBankId(i)).ready - } - - // Response routing: each Ball has a small skid buffer holding in-flight bank_id - val inFlightBankPerBall = Seq.fill(clientNum)( - Module( - new Queue( - UInt(log2Ceil(mmioBankNum).W), - entries = 2, - pipe = true, - flow = false - ) - ) - ) - - for (i <- 0 until clientNum) { - inFlightBankPerBall(i).io.enq.valid := io.ballReq(i).fire - inFlightBankPerBall(i).io.enq.bits := ballBankId(i) - } - - for (i <- 0 until clientNum) { - val q = inFlightBankPerBall(i) - val srcBank = q.io.deq.bits - val srcRespVal = io.bankReadResp(srcBank).valid - - io.ballResp(i).valid := q.io.deq.valid && srcRespVal - io.ballResp(i).bits.data := io.bankReadResp(srcBank).bits.data - - q.io.deq.ready := io.ballResp(i).ready && srcRespVal - } - - for (bankIdx <- 0 until mmioBankNum) { - if (clientNum == 0) { - io.bankReadResp(bankIdx).ready := false.B - } else { - val anyExpect = VecInit((0 until clientNum).map { i => - inFlightBankPerBall(i).io.deq.valid && - (inFlightBankPerBall(i).io.deq.bits === bankIdx.U) && - io.ballResp(i).ready - }).asUInt.orR - io.bankReadResp(bankIdx).ready := anyExpect - } - } -} diff --git a/arch/src/main/scala/framework/memdomain/backend/privatepath/PrivateMemBackend.scala b/arch/src/main/scala/framework/memdomain/backend/privatepath/PrivateMemBackend.scala deleted file mode 100644 index d8874f672..000000000 --- a/arch/src/main/scala/framework/memdomain/backend/privatepath/PrivateMemBackend.scala +++ /dev/null @@ -1,440 +0,0 @@ -package framework.memdomain.backend.privatepath - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.memdomain.frontend.mem.MemConfigerIO -import framework.memdomain.backend.{MTraceDPI, MemRequestIO} -import sims.hash.BankHashMonitor -import framework.memdomain.backend.accpipe.AccPipe -import framework.memdomain.backend.banks.SramBank -import framework.memdomain.backend.banks.btrace.PhysicalBankHash -import framework.top.GlobalConfig - -@instantiable -class PrivateMemBackend(val b: GlobalConfig) extends Module { - val kernelRequests = if (b.rvv.enable) 2 * b.rvv.memoryPorts else 0 - val requestCount = b.memDomain.bankChannel + kernelRequests - - @public - val io = IO(new Bundle { - val mem_req = Vec(b.memDomain.bankChannel, Flipped(new MemRequestIO(b))) - val kernel_req = Vec(kernelRequests, Flipped(new MemRequestIO(b))) - val config = Flipped(Decoupled(new MemConfigerIO(b))) - - // Query interface for frontend to get group count - val query_vbank_id = Input(UInt(b.memDomain.vbankIdWidth.W)) - val query_group_count = Output(UInt(b.memDomain.groupCountWidth.W)) - val clearBusy = Output(Bool()) - val bank_hashes = if (b.sim.diffTest) Some(Output(Vec(b.memDomain.bankNum, new PhysicalBankHash(b)))) else None - }) - - val requests = io.mem_req.toSeq ++ io.kernel_req.toSeq - - val banks: Seq[Instance[SramBank]] = Seq.fill(b.memDomain.bankNum)(Instantiate(new SramBank(b))) - val accPipes: Seq[Instance[AccPipe]] = Seq.fill(requestCount)(Instantiate(new AccPipe(b))) - - val hashMonitors: Option[Seq[Instance[BankHashMonitor]]] = - if (b.sim.diffTest) { - Some(Seq.fill(b.memDomain.bankNum)(Instantiate(new BankHashMonitor(b)))) - } else { - None - } - - // Per-channel memory trace DPI-C modules to avoid losing simultaneous events - val mtraces = Seq.fill(requestCount)(Module(new MTraceDPI)) - for (mt <- mtraces) { - mt.io.clock := clock - mt.io.reset := reset.asBool - mt.io.is_write := 0.U - mt.io.is_shared := 0.U - mt.io.channel := 0.U - mt.io.hart_id := 0.U - mt.io.rob_id := 0.U - mt.io.inst_id := 0.U - mt.io.vbank_id := 0.U - mt.io.pbank_id := 0.U - mt.io.group_id := 0.U - mt.io.addr := 0.U - mt.io.write_mask := 0.U - mt.io.data_lo := 0.U - mt.io.data_hi := 0.U - mt.io.enable := false.B - } - - // ----------------------------------------------------------------------------- - // Mapping table - // ----------------------------------------------------------------------------- - class MappingTableEntry extends Bundle { - val valid = Bool() - val hart_id = UInt(b.tile.xLen.W) - val vbank_id = UInt(b.memDomain.vbankIdWidth.W) - val is_multi = Bool() - val group_id = UInt(b.memDomain.groupIdWidth.W) - } - - val mappingTable = RegInit(VecInit(Seq.fill(b.memDomain.bankNum)(0.U.asTypeOf(new MappingTableEntry)))) - // The frontend contract reserves [0, vbank_id_upper_bound] for private - // banks. Keep the direct route table at that architectural size. - val privateVbankCount = b.frontend.vbank_id_upper_bound + 1 - require( - privateVbankCount > 0 && privateVbankCount <= b.memDomain.virtualBankCount, - s"private vbank table size ($privateVbankCount) exceeds virtualBankCount(${b.memDomain.virtualBankCount})" - ) - val groupIndexWidth = log2Up(b.memDomain.bankNum) - val pbankIndexWidth = log2Up(b.memDomain.bankNum) - val privateVbankIdWidth = log2Up(privateVbankCount) - - // A non-multi vbank has exactly one route, so keep that common case as a - // small indexed table. Multi-bank mappings retain their per-group entries - // in mappingTable and use the fallback scan below. This avoids the much - // larger 32x24 valid+pbank register matrix while still removing the full - // mapping scan from the overwhelmingly common single-bank path. - val singleRouteValid = RegInit(VecInit(Seq.fill(privateVbankCount)(false.B))) - - val singleRoutePbank = RegInit( - VecInit(Seq.fill(privateVbankCount)(0.U(pbankIndexWidth.W))) - ) - - // The private namespace is bounded by the frontend contract, while - // multi-bank groups remain represented individually in mappingTable. - val groupCountByVbank = RegInit( - VecInit(Seq.fill(privateVbankCount)(0.U(b.memDomain.groupCountWidth.W))) - ) - - def addEntry( - hart_id: UInt, - vbank_id: UInt, - pbank_id: UInt, - is_multi: Bool, - group_id: UInt - ): Unit = { - val entry = mappingTable(pbank_id) - entry.valid := true.B - entry.hart_id := hart_id - entry.vbank_id := vbank_id - entry.is_multi := is_multi - entry.group_id := group_id - } - - def deleteEntry(vbank_id: UInt): Unit = { - for (i <- 0 until b.memDomain.bankNum) { - when(mappingTable(i).valid && mappingTable(i).vbank_id === vbank_id) { - mappingTable(i).valid := false.B - } - } - } - - // ----------------------------------------------------------------------------- - // Default Value - // ----------------------------------------------------------------------------- - - for (i <- 0 until requestCount) { - accPipes(i).io.mem_req.write <> requests(i).write - accPipes(i).io.mem_req.read <> requests(i).read - accPipes(i).io.mem_req.bank_id := requests(i).bank_id - accPipes(i).io.mem_req.group_id := requests(i).group_id - accPipes(i).io.mem_req.is_shared := requests(i).is_shared - accPipes(i).io.mem_req.hart_id := requests(i).hart_id - accPipes(i).io.mem_req.rob_id := requests(i).rob_id - accPipes(i).io.mem_req.inst_id := requests(i).inst_id - - // Bank-side defaults (only driven when a bank is actually connected) - accPipes(i).io.sramRead.req.ready := false.B - accPipes(i).io.sramRead.resp.valid := false.B - accPipes(i).io.sramRead.resp.bits := DontCare - - accPipes(i).io.sramWrite.req.ready := false.B - accPipes(i).io.sramWrite.resp.valid := false.B - accPipes(i).io.sramWrite.resp.bits := DontCare - } - - banks.zipWithIndex.foreach { - case (bank, _) => - bank.io.sramRead.req.valid := false.B - bank.io.sramRead.req.bits := DontCare - bank.io.sramRead.resp.ready := true.B - - bank.io.sramWrite.req.valid := false.B - bank.io.sramWrite.req.bits := DontCare - bank.io.sramWrite.resp.ready := true.B - bank.io.clear := false.B - } - io.clearBusy := VecInit(banks.map(_.io.clearing)).asUInt.orR - - val realloc = io.config.bits.group_id === 0.U - val freePbankMask = - VecInit(mappingTable.map(entry => !entry.valid || (realloc && entry.vbank_id === io.config.bits.vbank_id))) - val hasFreePbank = freePbankMask.asUInt.orR - io.config.ready := !io.config.bits.alloc || hasFreePbank - when(io.config.valid && io.config.bits.alloc && !hasFreePbank) { - assert(false.B, "PrivateMemBackend allocation failed: no free physical bank\n") - } - - // ----------------------------------------------------------------------------- - // Bank Alloc/Release - // ----------------------------------------------------------------------------- - - hashMonitors.foreach(_.foreach(_.io.bind := false.B)) - - when(io.config.fire) { - val vbank = io.config.bits.vbank_id - val vbankIdx = vbank(privateVbankIdWidth - 1, 0) - when(io.config.bits.transfer) { - val source = io.config.bits.source_bank_id - val sourceIdx = source(privateVbankIdWidth - 1, 0) - val sourceCount = groupCountByVbank(sourceIdx) - val targetCount = groupCountByVbank(vbankIdx) - val totalCount = targetCount +& sourceCount - assert(source =/= vbank && sourceCount =/= 0.U, "Private bank transfer requires a distinct allocated source") - assert(totalCount <= b.memDomain.bankNum.U, "Private bank transfer exceeds physical bank count") - for (entry <- mappingTable) { - when(entry.valid && entry.vbank_id === source) { - entry.vbank_id := vbank - entry.group_id := targetCount + entry.group_id - entry.is_multi := totalCount > 1.U - } - when(entry.valid && entry.vbank_id === vbank) { - entry.is_multi := totalCount > 1.U - } - } - singleRouteValid(sourceIdx) := false.B - singleRouteValid(vbankIdx) := totalCount === 1.U - when(totalCount === 1.U)(singleRoutePbank(vbankIdx) := singleRoutePbank(sourceIdx)) - groupCountByVbank(sourceIdx) := 0.U - groupCountByVbank(vbankIdx) := totalCount - }.elsewhen(io.config.bits.alloc) { - val freePbank = PriorityEncoder(freePbankMask) - hashMonitors.foreach { hashes => - for (i <- 0 until b.memDomain.bankNum) { - hashes(i).io.bind := freePbank === i.U - } - } - // The bank hash already treats a newly bound bank as zero, so the clear writes need no hash update. - for (i <- 0 until b.memDomain.bankNum) { - when(io.config.bits.clear && freePbank === i.U)(banks(i).io.clear := true.B) - } - // Match bemu mset: realloc of the same vbank frees prior physical banks first. - // MemConfiger emits one fire per group; only group 0 drops the old mapping. - when(io.config.bits.group_id === 0.U) { - deleteEntry(io.config.bits.vbank_id) - singleRouteValid(vbankIdx) := false.B - } - addEntry( - io.config.bits.hart_id, - io.config.bits.vbank_id, - freePbank, - io.config.bits.is_multi, - io.config.bits.group_id - ) - when(!io.config.bits.is_multi) { - singleRoutePbank(vbankIdx) := freePbank - singleRouteValid(vbankIdx) := true.B - } - groupCountByVbank(vbankIdx) := Mux(io.config.bits.is_multi, io.config.bits.group_id +& 1.U, 1.U) - }.otherwise { - deleteEntry(io.config.bits.vbank_id) - singleRouteValid(vbankIdx) := false.B - groupCountByVbank(vbankIdx) := 0.U - } - } - - // ----------------------------------------------------------------------------- - // Query interface: return group count for a given vbank_id - // ----------------------------------------------------------------------------- - val queryVbankId = RegNext(io.query_vbank_id, 0.U) - val queryVbank = queryVbankId(privateVbankIdWidth - 1, 0) - io.query_group_count := RegNext(groupCountByVbank(queryVbank), 0.U) - - // ----------------------------------------------------------------------------- - // Connect AccPipe and Banks - // ----------------------------------------------------------------------------- - private def emitTrace( - ch: Int, - isWrite: UInt, - pbankId: UInt, - addr: UInt, - writeMask: UInt, - dataLo: UInt, - dataHi: UInt, - en: Bool - ): Unit = { - mtraces(ch).io.is_write := isWrite - mtraces(ch).io.is_shared := requests(ch).is_shared.asUInt - mtraces(ch).io.channel := ch.U - mtraces(ch).io.hart_id := requests(ch).hart_id - mtraces(ch).io.rob_id := requests(ch).rob_id - mtraces(ch).io.inst_id := requests(ch).inst_id - mtraces(ch).io.vbank_id := requests(ch).bank_id - mtraces(ch).io.pbank_id := pbankId - mtraces(ch).io.group_id := requests(ch).group_id - mtraces(ch).io.addr := addr - mtraces(ch).io.write_mask := writeMask - mtraces(ch).io.data_lo := dataLo - mtraces(ch).io.data_hi := dataHi - mtraces(ch).io.enable := en - } - - // The route is held for the complete AccPipe transaction. Keep the route - // vectors in one place so the bank-side command mux and response demux share - // exactly the same ownership predicate. - val activeRouteOHs = Seq.fill(requestCount)(Wire(UInt(b.memDomain.bankNum.W))) - - for (i <- 0 until requestCount) { - val routePbankReg = RegInit(0.U(pbankIndexWidth.W)) - val routeValidReg = RegInit(false.B) - val routePending = RegInit(false.B) - val requestVbank = requests(i).bank_id - val requestVbankIdx = requestVbank(privateVbankIdWidth - 1, 0) - val requestGroup = requests(i).group_id(groupIndexWidth - 1, 0) - val singleRoute = singleRoutePbank(requestVbankIdx) - val singleValid = singleRouteValid(requestVbankIdx) - val multiRouteMatch = VecInit(mappingTable.map(entry => - entry.valid && entry.is_multi && - entry.vbank_id === requestVbank && entry.group_id === requestGroup - )) - val multiRoute = PriorityEncoder(multiRouteMatch) - val multiValid = multiRouteMatch.asUInt.orR - val requestRoute = Mux(singleValid, singleRoute, multiRoute) - val requestRouteValid = singleValid || multiValid - // Route lookup is deliberately one cycle ahead of the SRAM request. The - // route is captured while the AccPipe is idle, then held through the - // entire request/response lifetime. This removes the live mapping-table - // lookup from every wide bank-side mux select while preserving one request - // per channel per cycle once the pipe is running. - val requestSeen = requests(i).read.req.valid || requests(i).write.req.valid - val activeRoute = routePbankReg - val requestActive = routePending || accPipes(i).io.busy - val activeRouteValid = routeValidReg && requestActive - val activeRouteOH = UIntToOH(activeRoute, b.memDomain.bankNum) & - Fill(b.memDomain.bankNum, activeRouteValid) - - activeRouteOHs(i) := activeRouteOH - - when(!routePending && !accPipes(i).io.busy && requestSeen && requestRouteValid) { - routePbankReg := requestRoute - routeValidReg := true.B - routePending := true.B - } - when(requests(i).read.req.fire || requests(i).write.req.fire) { - routePending := false.B - } - - // Requests are attributed once the selected SRAM accepts them. - when(accPipes(i).io.sramRead.req.fire) { - emitTrace(i, 0.U, activeRoute, accPipes(i).io.sramRead.req.bits.addr, 0.U, 0.U, 0.U, true.B) - } - - // Arrival trace: the write has crossed arbitration and is accepted by the - // selected SPM SRAM port. Keep attribution metadata from the same AccPipe - // transaction and data from the bank-side request. - when(accPipes(i).io.sramWrite.req.fire) { - emitTrace( - i, - 1.U, - activeRoute, - accPipes(i).io.sramWrite.req.bits.addr, - accPipes(i).io.sramWrite.req.bits.mask.asUInt, - accPipes(i).io.sramWrite.req.bits.data(63, 0), - accPipes(i).io.sramWrite.req.bits.data(127, 64), - true.B - ) - } - - } - - // Fixed RVV request slots share the physical-bank arbitration with ordinary - // channels. Capture the accepted request's owner for the one-cycle SRAM - // response; a stalled requester must never receive another owner's response. - val responseRoutes = Seq.fill(requestCount)(WireDefault(0.U(b.memDomain.bankNum.W))) - val responseRouteBits = Seq.fill(requestCount)(Wire(Vec(b.memDomain.bankNum, Bool()))) - - for (j <- 0 until b.memDomain.bankNum) { - val arb = Module(new RRArbiter( - new Bundle { - val write = Bool() - val addr = UInt(16.W) - val data = UInt(b.memDomain.bankWidth.W) - val mask = Vec(b.memDomain.bankMaskLen, Bool()) - }, - requestCount - )) - - for (i <- 0 until requestCount) { - val pipe = accPipes(i) - arb.io.in(i).valid := activeRouteOHs(i)(j) && - (pipe.io.sramRead.req.valid || pipe.io.sramWrite.req.valid) - arb.io.in(i).bits.write := pipe.io.sramWrite.req.valid - arb.io.in(i).bits.addr := Mux( - pipe.io.sramWrite.req.valid, - pipe.io.sramWrite.req.bits.addr, - pipe.io.sramRead.req.bits.addr - ) - arb.io.in(i).bits.data := pipe.io.sramWrite.req.bits.data - arb.io.in(i).bits.mask := pipe.io.sramWrite.req.bits.mask - when(activeRouteOHs(i)(j)) { - pipe.io.sramRead.req.ready := arb.io.in(i).ready && !arb.io.in(i).bits.write - pipe.io.sramWrite.req.ready := arb.io.in(i).ready && arb.io.in(i).bits.write - } - responseRouteBits(i)(j) := RegNext(arb.io.in(i).fire, false.B) - } - - val bank = banks(j) - bank.io.sramRead.req.valid := arb.io.out.valid && !arb.io.out.bits.write - bank.io.sramRead.req.bits.addr := arb.io.out.bits.addr - bank.io.sramWrite.req.valid := arb.io.out.valid && arb.io.out.bits.write - bank.io.sramWrite.req.bits.addr := arb.io.out.bits.addr - bank.io.sramWrite.req.bits.data := arb.io.out.bits.data - bank.io.sramWrite.req.bits.mask := arb.io.out.bits.mask - arb.io.out.ready := Mux(arb.io.out.bits.write, bank.io.sramWrite.req.ready, bank.io.sramRead.req.ready) - - hashMonitors.foreach { hashes => - val monitor = hashes(j) - bank.io.sramWrite.req.valid := arb.io.out.valid && arb.io.out.bits.write && monitor.io.write.ready - monitor.io.write.valid := arb.io.out.valid && arb.io.out.bits.write && bank.io.sramWrite.req.ready - arb.io.out.ready := Mux( - arb.io.out.bits.write, - bank.io.sramWrite.req.ready && monitor.io.write.ready, - bank.io.sramRead.req.ready - ) - monitor.io.write.bits.addr := arb.io.out.bits.addr - monitor.io.write.bits.mask := arb.io.out.bits.mask - monitor.io.write.bits.data := arb.io.out.bits.data - } - bank.io.sramRead.resp.ready := VecInit((0 until requestCount).map { i => - responseRouteBits(i)(j) && accPipes(i).io.sramRead.resp.ready - }).asUInt.orR - bank.io.sramWrite.resp.ready := VecInit((0 until requestCount).map { i => - responseRouteBits(i)(j) && accPipes(i).io.sramWrite.resp.ready - }).asUInt.orR - when(bank.io.sramRead.resp.valid)(assert(bank.io.sramRead.resp.ready)) - when(bank.io.sramWrite.resp.valid)(assert(bank.io.sramWrite.resp.ready)) - } - - for (i <- 0 until requestCount) { - responseRoutes(i) := responseRouteBits(i).asUInt - val readRespSel = VecInit((0 until b.memDomain.bankNum).map { j => - responseRoutes(i)(j) && banks(j).io.sramRead.resp.valid - }).asUInt - val writeRespSel = VecInit((0 until b.memDomain.bankNum).map { j => - responseRoutes(i)(j) && banks(j).io.sramWrite.resp.valid - }).asUInt - accPipes(i).io.sramRead.resp.valid := readRespSel.orR - accPipes(i).io.sramRead.resp.bits.data := (0 until b.memDomain.bankNum).map { j => - Fill(b.memDomain.bankWidth, responseRoutes(i)(j)) & banks(j).io.sramRead.resp.bits.data - }.reduce(_ | _) - accPipes(i).io.sramWrite.resp.valid := writeRespSel.orR - accPipes(i).io.sramWrite.resp.bits.ok := writeRespSel.orR - } - - io.bank_hashes.foreach { states => - for (i <- 0 until b.memDomain.bankNum) { - states(i).valid := mappingTable(i).valid - states(i).hartId := mappingTable(i).hart_id - states(i).vbankId := mappingTable(i).vbank_id - states(i).pbankId := i.U - states(i).groupId := mappingTable(i).group_id - states(i).statusHash := hashMonitors.get(i).io.statusHash - } - } -} diff --git a/arch/src/main/scala/framework/memdomain/backend/shared/PhysicalPort.scala b/arch/src/main/scala/framework/memdomain/backend/shared/PhysicalPort.scala deleted file mode 100644 index 64ab7da5b..000000000 --- a/arch/src/main/scala/framework/memdomain/backend/shared/PhysicalPort.scala +++ /dev/null @@ -1,23 +0,0 @@ -package framework.memdomain.backend.shared - -import chisel3._ -import chisel3.util._ - -/** Tile shared SRAM byte space; not a Core virtual-bank or private-bank address. */ -class SharedPhysicalRequest(dataBits: Int) extends Bundle { - val address = UInt(64.W) - val write = Bool() - val data = UInt(dataBits.W) - val mask = UInt((dataBits / 8).W) -} - -class SharedPhysicalResponse(dataBits: Int) extends Bundle { - val data = UInt(dataBits.W) - val error = Bool() -} - -/** One outstanding transaction per port; a write response acknowledges actual SRAM completion. */ -class SharedPhysicalPort(dataBits: Int) extends Bundle { - val request = Decoupled(new SharedPhysicalRequest(dataBits)) - val response = Flipped(Decoupled(new SharedPhysicalResponse(dataBits))) -} diff --git a/arch/src/main/scala/framework/memdomain/backend/shared/SharedLease.scala b/arch/src/main/scala/framework/memdomain/backend/shared/SharedLease.scala deleted file mode 100644 index 39959d48b..000000000 --- a/arch/src/main/scala/framework/memdomain/backend/shared/SharedLease.scala +++ /dev/null @@ -1,17 +0,0 @@ -package framework.memdomain.backend.shared - -import chisel3._ -import chisel3.util._ - -class SharedLeaseRequest extends Bundle { - val endpoint = UInt(32.W) - val vbank = UInt(16.W) - val group = UInt(16.W) - val release = Bool() -} - -/** One outstanding mapping lease operation; export returns its physical byte base. */ -class SharedLeasePort extends Bundle { - val request = Decoupled(new SharedLeaseRequest) - val response = Flipped(Decoupled(UInt(64.W))) -} diff --git a/arch/src/main/scala/framework/memdomain/backend/shared/SharedMem.scala b/arch/src/main/scala/framework/memdomain/backend/shared/SharedMem.scala deleted file mode 100644 index c874b4fac..000000000 --- a/arch/src/main/scala/framework/memdomain/backend/shared/SharedMem.scala +++ /dev/null @@ -1,39 +0,0 @@ -package framework.memdomain.backend.shared - -import framework.top.GlobalConfig - -object SharedMemLayout { - - def totalBank(b: GlobalConfig): Int = { - require(b.memDomain.sharedEnable, "shared memory is disabled") - require(b.memDomain.sharedBankNum > 0, "sharedBankNum must be > 0") - b.memDomain.sharedBankNum - } - - def channelPerHart(b: GlobalConfig): Int = { - if (!b.memDomain.sharedEnable) { - return 0 - } - val computeCores = b.memDomain.computeCoreIds.size - require(computeCores > 0, "shared memory requires compute cores") - require( - b.memDomain.sharedInputChannels > 0, - s"sharedInputChannels(${b.memDomain.sharedInputChannels}) must be > 0" - ) - require( - b.memDomain.sharedInputChannels % computeCores == 0, - s"sharedInputChannels(${b.memDomain.sharedInputChannels}) must be divisible by computeCores($computeCores)" - ) - if (b.memDomain.sharedInputChannels > 32) { - require( - b.memDomain.sharedInputChannels == computeCores, - s"sharedInputChannels(${b.memDomain.sharedInputChannels}) must equal computeCores($computeCores) when > 32" - ) - } - val ch = b.memDomain.sharedInputChannels / computeCores - require(ch > 0, s"channelPerHart($ch) must be > 0") - ch - } - - def totalChannel(b: GlobalConfig): Int = if (b.memDomain.sharedEnable) b.memDomain.sharedInputChannels else 0 -} diff --git a/arch/src/main/scala/framework/memdomain/backend/shared/SharedMemBackend.scala b/arch/src/main/scala/framework/memdomain/backend/shared/SharedMemBackend.scala deleted file mode 100644 index e9f8209af..000000000 --- a/arch/src/main/scala/framework/memdomain/backend/shared/SharedMemBackend.scala +++ /dev/null @@ -1,588 +0,0 @@ -package framework.memdomain.backend.shared - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.memdomain.backend.{MTraceDPI, MemRequestIO} -import sims.hash.BankHashMonitor -import framework.memdomain.backend.accpipe.AccPipe -import framework.memdomain.backend.banks.SramBank -import framework.memdomain.backend.banks.btrace.PhysicalBankHash -import framework.memdomain.frontend.mem.MemConfigerIO -import framework.top.GlobalConfig -import memcore.memory.mesh_shm.{MeshCoreAttachment, MeshLocalBankPort, MeshSharedMem, MeshSharedMemParams} -import framework.memdomain.isa.{MvoverISA, MvoverPort} - -@instantiable -class SharedMemBackend(val b: GlobalConfig, useMesh: Boolean = false, externalPhysicalPorts: Int = 0) extends Module { - require(externalPhysicalPorts >= 0 && (externalPhysicalPorts == 0 || useMesh)) - val nCores = b.memDomain.nCores - val totalBanks = SharedMemLayout.totalBank(b) - val totalChannel = SharedMemLayout.totalChannel(b) - val sharedConfig = b.copy(memDomain = b.memDomain.copy(bankEntries = b.memDomain.sharedBankEntries)) - val tagBits = math.max(1, log2Ceil(b.frontend.rob_entries)) - - @public - val io = IO(new Bundle { - val mem_req = Vec(totalChannel, Flipped(new MemRequestIO(b))) - val lease = Option.when(externalPhysicalPorts > 0)(Flipped(new SharedLeasePort)) - val leaseOwnerHartId = Option.when(externalPhysicalPorts > 0)(Input(UInt(b.tile.xLen.W))) - val physical = Vec(externalPhysicalPorts, Flipped(new SharedPhysicalPort(b.memDomain.bankWidth))) - val physicalOwnerHartId = if (externalPhysicalPorts > 0) Some(Input(UInt(b.tile.xLen.W))) else None - val mvover = Flipped(new MvoverPort) - val localBanks = - Vec(nCores, new MeshLocalBankPort(MvoverISA.AddressBits, MvoverISA.BankBits, b.memDomain.bankWidth, tagBits)) - val config = Flipped(Decoupled(new MemConfigerIO(b))) - - // Query interface for frontend to get group count - val query_valid = Input(Vec(nCores, Bool())) - val query_hart_id = Input(Vec(nCores, UInt(b.tile.xLen.W))) - val query_vbank_id = Input(Vec(nCores, UInt(b.memDomain.vbankIdWidth.W))) - val query_group_count = Output(Vec(nCores, UInt(b.memDomain.groupCountWidth.W))) - val bank_hashes = if (b.sim.diffTest) Some(Output(Vec(totalBanks, new PhysicalBankHash(b)))) else None - }) - - val meshParams = - if (useMesh) { - require(b.memDomain.bankWidth == 128 && b.memDomain.bankMaskLen == 16) - require(b.memDomain.sharedInputChannels % b.memDomain.computeCoreIds.size == 0) - val physicalBanks = totalBanks - val (rows, columns) = MeshSharedMemParams.dimensions(physicalBanks) - val perCore = SharedMemLayout.channelPerHart(b) - val computeCores = b.memDomain.computeCoreIds - val attachments = MeshSharedMemParams.directionalAttachments( - rows, - columns, - Seq.fill(computeCores.size)(perCore) - ) - Some(MeshSharedMemParams( - rows = rows, - cols = columns, - entriesPerBank = b.memDomain.sharedBankEntries, - dataBits = b.memDomain.bankWidth, - tagBits = tagBits, - localBankBits = MvoverISA.BankBits, - cores = (0 until nCores).map { core => - val computeIndex = computeCores.indexOf(core) - if (computeIndex < 0) MeshCoreAttachment(Seq.empty, Seq.empty, None) - else attachments(computeIndex) - }, - externalChannels = externalPhysicalPorts, - visibleBanks = totalBanks - )) - } else None - - val mesh: Option[Instance[MeshSharedMem]] = meshParams.map(p => Instantiate(new MeshSharedMem(p))) - mesh.foreach { network => - require(network.io.transferCommand.bits.sourceAddr.getWidth <= 16) - require(network.io.transferCommand.bits.sourceCore.getWidth <= 8) - network.io.transferCommand.valid := io.mvover.command.valid - network.io.transferCommand.bits.sourceCore := io.mvover.command.bits.sourceCore - network.io.transferCommand.bits.targetCore := io.mvover.command.bits.targetCore - network.io.transferCommand.bits.sourceBank := io.mvover.command.bits.sourceBank - network.io.transferCommand.bits.targetBank := io.mvover.command.bits.targetBank - network.io.transferCommand.bits.sourceAddr := io.mvover.command.bits.sourceAddr - network.io.transferCommand.bits.targetAddr := io.mvover.command.bits.targetAddr - network.io.transferCommand.bits.rows := io.mvover.command.bits.rows - network.io.transferCommand.bits.tag := 0.U - io.mvover.command.ready := network.io.transferCommand.ready - io.mvover.completion.valid := network.io.transferCompletion.valid - io.mvover.completion.bits := network.io.transferCompletion.bits.error - network.io.transferCompletion.ready := io.mvover.completion.ready - for ((local, index) <- network.io.localBanks.zipWithIndex) { - io.localBanks(index).request.valid := local.request.valid - io.localBanks(index).request.bits := local.request.bits - local.request.ready := io.localBanks(index).request.ready - local.response.valid := io.localBanks(index).response.valid - local.response.bits := io.localBanks(index).response.bits - io.localBanks(index).response.ready := local.response.ready - } - } - if (!useMesh) { - io.mvover.command.ready := false.B - io.mvover.completion.valid := false.B - io.mvover.completion.bits := false.B - for (local <- io.localBanks) { - local.request.valid := false.B - local.request.bits := 0.U.asTypeOf(local.request.bits) - local.response.ready := false.B - } - } - - val banks: Seq[Instance[SramBank]] = - if (useMesh) Seq.empty else Seq.fill(totalBanks)(Instantiate(new SramBank(sharedConfig))) - val accPipes: Seq[Instance[AccPipe]] = Seq.fill(totalChannel)(Instantiate(new AccPipe(b))) - - val hashMonitors: Option[Seq[Instance[BankHashMonitor]]] = - if (b.sim.diffTest) { - Some(Seq.fill(totalBanks)(Instantiate(new BankHashMonitor(sharedConfig)))) - } else { - None - } - - // Per-channel memory trace DPI-C modules to avoid losing simultaneous events - val mtraces = Seq.fill(totalChannel + externalPhysicalPorts)(Module(new MTraceDPI)) - for (mt <- mtraces) { - mt.io.clock := clock - mt.io.reset := reset.asBool - mt.io.is_write := 0.U - mt.io.is_shared := 0.U - mt.io.channel := 0.U - mt.io.hart_id := 0.U - mt.io.rob_id := 0.U - mt.io.inst_id := 0.U - mt.io.vbank_id := 0.U - mt.io.pbank_id := 0.U - mt.io.group_id := 0.U - mt.io.addr := 0.U - mt.io.write_mask := 0.U - mt.io.data_lo := 0.U - mt.io.data_hi := 0.U - mt.io.enable := false.B - } - - // ----------------------------------------------------------------------------- - // Mapping table - // ----------------------------------------------------------------------------- - class MappingTableEntry extends Bundle { - val valid = Bool() - val hart_id = UInt(b.tile.xLen.W) - val vbank_id = UInt(b.memDomain.vbankIdWidth.W) - val is_multi = Bool() - val group_id = UInt(b.memDomain.groupIdWidth.W) - } - - val mappingTable = RegInit(VecInit(Seq.fill(totalBanks)(0.U.asTypeOf(new MappingTableEntry)))) - val leases = Option.when(externalPhysicalPorts > 0)(RegInit(VecInit(Seq.fill(totalBanks)(0.U(32.W))))) - io.lease.foreach { port => - val pending = RegInit(false.B) - val result = Reg(UInt(64.W)) - val matches = VecInit(mappingTable.map(entry => - entry.valid && - entry.hart_id === io.leaseOwnerHartId.get && entry.vbank_id === port.request.bits.vbank && - entry.group_id === port.request.bits.group - )) - val index = PriorityEncoder(matches) - port.request.ready := !pending && !io.config.valid && !reset.asBool - port.response.valid := pending && !reset.asBool - port.response.bits := result - when(port.request.fire) { - assert(PopCount(matches) === 1.U, "Shared lease requires one allocated owner/bank/group") - val count = leases.get(index) - when(port.request.bits.release) { - assert(count =/= 0.U, "Shared lease release has no export") - count := count - 1.U - result := 0.U - }.otherwise { - assert(count =/= "hffffffff".U, "Shared lease count overflow") - count := count + 1.U - result := index * (b.memDomain.sharedBankEntries * (b.memDomain.bankWidth / 8)).U(64.W) - } - pending := true.B - } - when(port.response.fire)(pending := false.B) - } - - mesh.foreach { network => - val rowBits = log2Ceil(b.memDomain.sharedBankEntries) - val bytesPerRow = b.memDomain.bankWidth / 8 - val byteBits = log2Ceil(bytesPerRow) - for (i <- 0 until externalPhysicalPorts) { - val physical = io.physical(i) - val channel = network.io.external(i) - val bank = physical.request.bits.address >> (rowBits + byteBits) - val legal = bank < totalBanks.U && physical.request.bits.address(byteBits - 1, 0) === 0.U - val bankIndex = bank(math.max(1, log2Ceil(totalBanks)) - 1, 0) - val allocated = VecInit(mappingTable.map(_.valid))(bankIndex) - when(physical.request.valid && !reset.asBool) { - assert(legal && allocated, "T2T physical access requires an allocated shared bank and aligned address") - } - channel.request.valid := physical.request.valid && legal && allocated - channel.request.bits := 0.U.asTypeOf(channel.request.bits) - channel.request.bits.bank := bankIndex - channel.request.bits.tuser := Cat( - physical.request.bits.address(rowBits + byteBits - 1, byteBits).pad(16), - physical.request.bits.write - ) - channel.request.bits.data := physical.request.bits.data - channel.request.bits.mask := physical.request.bits.mask - channel.request.bits.tag := 0.U - channel.request.bits.tlast := true.B - physical.request.ready := channel.request.ready && legal && allocated - physical.response.valid := channel.response.valid - physical.response.bits.data := channel.response.bits.data - physical.response.bits.error := channel.response.bits.error - channel.response.ready := physical.response.ready - } - } - for (i <- 0 until externalPhysicalPorts) { - val physical = io.physical(i) - val held = Reg(new SharedPhysicalRequest(b.memDomain.bankWidth)) - val entry = Reg(new MappingTableEntry) - val pending = RegInit(false.B) - val bankBits = log2Ceil(b.memDomain.sharedBankEntries * (b.memDomain.bankWidth / 8)) - when(physical.request.fire) { - assert(!pending, "Physical shared port accepts one outstanding request") - held := physical.request.bits - entry := mappingTable((physical.request.bits.address >> bankBits)(math.max(1, log2Ceil(totalBanks)) - 1, 0)) - pending := true.B - } - when(physical.response.valid && !reset.asBool)(assert(pending)) - when(physical.response.fire)(pending := false.B) - val trace = mtraces(totalChannel + i) - val value = Mux(held.write, held.data, physical.response.bits.data) - trace.io.is_write := held.write.asUInt - trace.io.is_shared := 1.U - trace.io.channel := (totalChannel + i).U - trace.io.hart_id := io.physicalOwnerHartId.get - trace.io.vbank_id := entry.vbank_id - trace.io.pbank_id := held.address >> bankBits - trace.io.group_id := entry.group_id - trace.io.addr := (held.address >> 4) & (b.memDomain.sharedBankEntries - 1).U - trace.io.write_mask := Mux(held.write, held.mask, 0.U) - trace.io.data_lo := value(63, 0) - trace.io.data_hi := value(127, 64) - trace.io.enable := physical.response.fire && !physical.response.bits.error - } - - def addEntry( - hart_id: UInt, - vbank_id: UInt, - pbank_id: UInt, - is_multi: Bool, - group_id: UInt - ): Unit = { - val duplicate = mappingTable.map(entry => - entry.valid && - !(group_id === 0.U && entry.hart_id === hart_id && entry.vbank_id === vbank_id) && - (entry.hart_id === hart_id) && - (entry.vbank_id === vbank_id) && - (entry.group_id === group_id) - ).reduce(_ || _) - when(duplicate) { - assert(false.B, "SharedMemBackend duplicate allocation: hart=%d vbank=%d group=%d\n", hart_id, vbank_id, group_id) - } - - val entry = mappingTable(pbank_id) - leases.foreach(counts => assert(counts(pbank_id) === 0.U, "Shared bank allocation replaces live exports")) - entry.valid := true.B - entry.hart_id := hart_id - entry.vbank_id := vbank_id - entry.is_multi := is_multi - entry.group_id := group_id - } - - def deleteEntry(hart_id: UInt, vbank_id: UInt): Unit = { - val found = - mappingTable.map(entry => entry.valid && entry.vbank_id === vbank_id && entry.hart_id === hart_id).reduce(_ || _) - when(!found) { - assert(false.B, "SharedMemBackend release missing allocation: hart=%d vbank=%d\n", hart_id, vbank_id) - } - clearVbank(hart_id, vbank_id) - } - - def clearVbank(hart_id: UInt, vbank_id: UInt): Unit = { - for (i <- 0 until totalBanks) { - when(mappingTable(i).valid && mappingTable(i).vbank_id === vbank_id && mappingTable(i).hart_id === hart_id) { - leases.foreach(counts => assert(counts(i) === 0.U, "Shared bank mapping changed with live exports")) - mappingTable(i).valid := false.B - } - } - } - - // ----------------------------------------------------------------------------- - // Default Value - // ----------------------------------------------------------------------------- - - for (i <- 0 until totalChannel) { - accPipes(i).io.mem_req.write <> io.mem_req(i).write - accPipes(i).io.mem_req.read <> io.mem_req(i).read - accPipes(i).io.mem_req.bank_id := io.mem_req(i).bank_id - accPipes(i).io.mem_req.group_id := io.mem_req(i).group_id - accPipes(i).io.mem_req.is_shared := io.mem_req(i).is_shared - accPipes(i).io.mem_req.hart_id := io.mem_req(i).hart_id - accPipes(i).io.mem_req.rob_id := io.mem_req(i).rob_id - accPipes(i).io.mem_req.inst_id := io.mem_req(i).inst_id - - // Bank-side defaults (only driven when a bank is actually connected) - accPipes(i).io.sramRead.req.ready := false.B - accPipes(i).io.sramRead.resp.valid := false.B - accPipes(i).io.sramRead.resp.bits := DontCare - - accPipes(i).io.sramWrite.req.ready := false.B - accPipes(i).io.sramWrite.resp.valid := false.B - accPipes(i).io.sramWrite.resp.bits := DontCare - } - - banks.zipWithIndex.foreach { - case (bank, _) => - bank.io.sramRead.req.valid := false.B - bank.io.sramRead.req.bits := DontCare - bank.io.sramRead.resp.ready := true.B - - bank.io.sramWrite.req.valid := false.B - bank.io.sramWrite.req.bits := DontCare - bank.io.sramWrite.resp.ready := true.B - bank.io.clear := false.B - } - - val realloc = io.config.bits.group_id === 0.U - - val freePbankMask = VecInit(mappingTable.map(entry => - !entry.valid || - (realloc && entry.hart_id === io.config.bits.hart_id && entry.vbank_id === io.config.bits.vbank_id) - )) - - val hasFreePbank = freePbankMask.asUInt.orR - io.config.ready := !io.config.bits.alloc || hasFreePbank - when(io.config.valid && io.config.bits.alloc && !hasFreePbank) { - assert(false.B, "SharedMemBackend allocation failed: no free physical shared bank\n") - } - - // ----------------------------------------------------------------------------- - // Bank Alloc/Release - // ----------------------------------------------------------------------------- - - if (!useMesh) { - hashMonitors.foreach { hashes => - for (j <- 0 until totalBanks) { - hashes(j).io.write.valid := false.B - hashes(j).io.write.bits.addr := banks(j).io.sramWrite.req.bits.addr - hashes(j).io.write.bits.mask := banks(j).io.sramWrite.req.bits.mask - hashes(j).io.write.bits.data := banks(j).io.sramWrite.req.bits.data - } - } - } - - hashMonitors.foreach(_.foreach(_.io.bind := false.B)) - - when(io.config.fire) { - when(io.config.bits.transfer) { - val source = io.config.bits.source_bank_id - val target = io.config.bits.vbank_id - val owner = io.config.bits.hart_id - val sourceCount = PopCount(mappingTable.map(e => e.valid && e.hart_id === owner && e.vbank_id === source)) - val targetCount = PopCount(mappingTable.map(e => e.valid && e.hart_id === owner && e.vbank_id === target)) - val totalCount = sourceCount +& targetCount - assert(source =/= target && sourceCount =/= 0.U, "Shared bank transfer requires a distinct allocated source") - assert(totalCount <= totalBanks.U, "Shared bank transfer exceeds physical bank count") - for ((entry, physical) <- mappingTable.zipWithIndex) { - when(entry.valid && entry.hart_id === owner && entry.vbank_id === source) { - leases.foreach(counts => assert(counts(physical) === 0.U, "Shared bank transfer source has live exports")) - entry.vbank_id := target - entry.group_id := targetCount + entry.group_id - entry.is_multi := totalCount > 1.U - } - when(entry.valid && entry.hart_id === owner && entry.vbank_id === target) { - entry.is_multi := totalCount > 1.U - } - } - }.elsewhen(io.config.bits.alloc) { - when(io.config.bits.group_id === 0.U) { - clearVbank(io.config.bits.hart_id, io.config.bits.vbank_id) - } - val pbankId = PriorityEncoder(freePbankMask) - hashMonitors.foreach { hashes => - for (i <- 0 until totalBanks) { - hashes(i).io.bind := pbankId === i.U - } - } - printf( - p"[SharedMemBackend][ALLOC] hart=${io.config.bits.hart_id} vbank=0x${Hexadecimal(io.config.bits.vbank_id)} " + - p"group=${io.config.bits.group_id} pbank=$pbankId is_multi=${io.config.bits.is_multi}\n" - ) - addEntry( - io.config.bits.hart_id, - io.config.bits.vbank_id, - pbankId, - io.config.bits.is_multi, - io.config.bits.group_id - ) - }.otherwise { - printf( - p"[SharedMemBackend][RELEASE] hart=${io.config.bits.hart_id} vbank=0x${Hexadecimal(io.config.bits.vbank_id)}\n" - ) - deleteEntry(io.config.bits.hart_id, io.config.bits.vbank_id) - } - } - - // ----------------------------------------------------------------------------- - // Query interface: return group count for a given vbank_id - // ----------------------------------------------------------------------------- - for (q <- 0 until nCores) { - val queryValid = RegNext(io.query_valid(q), false.B) - val queryHart = RegNext(io.query_hart_id(q), 0.U) - val queryVbank = RegNext(io.query_vbank_id(q), 0.U) - val groupCounts = mappingTable.map { entry => - val matches = queryValid && - entry.valid && - (entry.hart_id === queryHart) && - (entry.vbank_id === queryVbank) - val count = Mux(entry.is_multi, entry.group_id +& 1.U, 1.U) - Mux(matches, count, 0.U) - } - - io.query_group_count(q) := RegNext(groupCounts.reduce((a, b) => Mux(a > b, a, b)), 0.U) - val queryHit = groupCounts.map(_ =/= 0.U).reduce(_ || _) - when(queryValid && !queryHit) { - printf( - p"[SharedMemBackend][QUERY_MISS] q=$q hart=$queryHart " + - p"vbank=0x${Hexadecimal(queryVbank)} returned group_count=0\n" - ) - } - } - - // ----------------------------------------------------------------------------- - // Connect AccPipe and Banks - // ----------------------------------------------------------------------------- - private def emitTrace( - ch: Int, - isWrite: UInt, - pbankId: UInt, - addr: UInt, - writeMask: UInt, - dataLo: UInt, - dataHi: UInt, - en: Bool - ): Unit = { - mtraces(ch).io.is_write := isWrite - mtraces(ch).io.is_shared := io.mem_req(ch).is_shared.asUInt - mtraces(ch).io.channel := ch.U - mtraces(ch).io.hart_id := io.mem_req(ch).hart_id - mtraces(ch).io.rob_id := io.mem_req(ch).rob_id - mtraces(ch).io.inst_id := io.mem_req(ch).inst_id - mtraces(ch).io.vbank_id := io.mem_req(ch).bank_id - mtraces(ch).io.pbank_id := pbankId - mtraces(ch).io.group_id := io.mem_req(ch).group_id - mtraces(ch).io.addr := addr - mtraces(ch).io.write_mask := writeMask - mtraces(ch).io.data_lo := dataLo - mtraces(ch).io.data_hi := dataHi - mtraces(ch).io.enable := en - } - - for (i <- 0 until totalChannel) { - val activeHart = Mux(accPipes(i).io.busy, accPipes(i).io.hart_id, io.mem_req(i).hart_id) - val activeBank = Mux(accPipes(i).io.busy, accPipes(i).io.bank_id, io.mem_req(i).bank_id) - val activeGroup = Mux(accPipes(i).io.busy, accPipes(i).io.group_id, io.mem_req(i).group_id) - val req_valid = io.mem_req(i).read.req.valid || io.mem_req(i).write.req.valid || accPipes(i).io.busy - - val tracePbankId = Wire(UInt(32.W)) - tracePbankId := 0.U - for (j <- 0 until totalBanks) { - val trace_hit_bank = mappingTable(j).valid && - (mappingTable(j).hart_id === activeHart) && - (mappingTable(j).vbank_id === activeBank) && - (!mappingTable(j).is_multi || - (mappingTable(j).is_multi && (mappingTable(j).group_id === activeGroup))) - when(trace_hit_bank) { - tracePbankId := j.U - } - } - - // Memory trace: read request - when(io.mem_req(i).read.req.fire) { - emitTrace(i, 0.U, tracePbankId, io.mem_req(i).read.req.bits.addr, 0.U, 0.U, 0.U, true.B) - } - - // Arrival trace: observe acceptance at the selected shared SPM SRAM port. - when(accPipes(i).io.sramWrite.req.fire) { - emitTrace( - i, - 1.U, - tracePbankId, - accPipes(i).io.sramWrite.req.bits.addr, - accPipes(i).io.sramWrite.req.bits.mask.asUInt, - accPipes(i).io.sramWrite.req.bits.data(63, 0), - accPipes(i).io.sramWrite.req.bits.data(127, 64), - true.B - ) - } - - if (useMesh) { - val channel = mesh.get.io.channels(i) - val mapped = mappingTable.map(entry => - entry.valid && - entry.hart_id === activeHart && - entry.vbank_id === activeBank && - (!entry.is_multi || entry.group_id === activeGroup) - ).reduce(_ || _) - val writeReq = accPipes(i).io.sramWrite.req.valid - val nextTag = RegInit(0.U(tagBits.W)) - - channel.request.valid := mapped && (writeReq || accPipes(i).io.sramRead.req.valid) - channel.request.bits.bank := tracePbankId - channel.request.bits.tuser := Cat( - Mux(writeReq, accPipes(i).io.sramWrite.req.bits.addr, accPipes(i).io.sramRead.req.bits.addr), - writeReq - ) - channel.request.bits.tlast := true.B - channel.request.bits.data := accPipes(i).io.sramWrite.req.bits.data - channel.request.bits.mask := accPipes(i).io.sramWrite.req.bits.mask.asUInt - channel.request.bits.tag := nextTag - accPipes(i).io.sramRead.req.ready := mapped && !writeReq && channel.request.ready - accPipes(i).io.sramWrite.req.ready := mapped && channel.request.ready - accPipes(i).io.sramRead.resp.valid := channel.response.valid && !channel.response.bits.write - accPipes(i).io.sramRead.resp.bits.data := channel.response.bits.data - accPipes(i).io.sramWrite.resp.valid := channel.response.valid && channel.response.bits.write - accPipes(i).io.sramWrite.resp.bits.ok := !channel.response.bits.error - channel.response.ready := Mux( - channel.response.bits.write, - accPipes(i).io.sramWrite.resp.ready, - accPipes(i).io.sramRead.resp.ready - ) - when(channel.request.fire) { - nextTag := nextTag + 1.U - } - } else { - for (j <- 0 until totalBanks) { - val hit_bank = mappingTable(j).valid && - (mappingTable(j).hart_id === activeHart) && - (mappingTable(j).vbank_id === activeBank) && - (!mappingTable(j).is_multi || - (mappingTable(j).is_multi && (mappingTable(j).group_id === activeGroup))) - - when(hit_bank && req_valid) { - banks(j).io.sramRead <> accPipes(i).io.sramRead - banks(j).io.sramWrite <> accPipes(i).io.sramWrite - hashMonitors.foreach { hashes => - val monitor = hashes(j) - val physical = banks(j).io.sramWrite.req - val offered = accPipes(i).io.sramWrite.req - physical.valid := offered.valid && monitor.io.write.ready - monitor.io.write.valid := offered.valid && physical.ready - offered.ready := physical.ready && monitor.io.write.ready - } - } - } - } - } - - mesh.foreach(_.io.bankWrites.foreach(_.ready := true.B)) - - hashMonitors.foreach { hashes => - for (j <- 0 until totalBanks) { - val monitor = hashes(j) - if (useMesh) { - val write = mesh.get.io.bankWrites(j) - write.ready := monitor.io.write.ready - monitor.io.write.valid := write.valid - monitor.io.write.bits.addr := write.bits.addr - monitor.io.write.bits.mask := write.bits.mask.asBools - monitor.io.write.bits.data := write.bits.data - - } - } - } - - io.bank_hashes.foreach { states => - for (i <- 0 until totalBanks) { - states(i).valid := mappingTable(i).valid - states(i).hartId := mappingTable(i).hart_id - states(i).vbankId := mappingTable(i).vbank_id - states(i).pbankId := i.U - states(i).groupId := mappingTable(i).group_id - states(i).statusHash := hashMonitors.get(i).io.statusHash - } - } -} diff --git a/arch/src/main/scala/framework/memdomain/boot/MemBoot.scala b/arch/src/main/scala/framework/memdomain/boot/MemBoot.scala deleted file mode 100644 index d583b491d..000000000 --- a/arch/src/main/scala/framework/memdomain/boot/MemBoot.scala +++ /dev/null @@ -1,16 +0,0 @@ -package framework.memdomain.boot - -import framework.memdomain.isa.MemISA -import framework.system.core.rocket.BuckyballCommand -import framework.top.GlobalConfig - -/** Internal MemDomain boot command program for one accelerator instance. */ -object MemBoot { - val BootBankId = 0 - - def initializationCommands(b: GlobalConfig): Seq[BuckyballCommand] = - Seq(MemISA.mset(MemISA.MsetArgs(BootBankId, columns = b.memDomain.bankNum, alloc = true, clear = true))) - - def releaseCommands(b: GlobalConfig): Seq[BuckyballCommand] = - Seq(MemISA.mset(MemISA.MsetArgs(BootBankId, columns = 0, alloc = false, clear = false))) -} diff --git a/arch/src/main/scala/framework/memdomain/configs/MemDomainParam.scala b/arch/src/main/scala/framework/memdomain/configs/MemDomainParam.scala deleted file mode 100644 index 4ab76565b..000000000 --- a/arch/src/main/scala/framework/memdomain/configs/MemDomainParam.scala +++ /dev/null @@ -1,89 +0,0 @@ -package framework.memdomain.configs - -import upickle.default._ - -/** - * MemDomain Parameter - * - * Real values are injected by `ChipLoader`. The no-arg `apply()` is a - * stub used only by `GlobalConfig()` before bundle overrides land — matching - * how `BallDomainParam()` returns an empty default. - */ -case class MemDomainParam( - bankNum: Int, - bankWidth: Int, - bankEntries: Int, - bankMaskLen: Int, - virtualBankCount: Int, - sharedEnable: Boolean, - sharedEntries: Int, - sharedBankEntries: Int, - sharedBankNum: Int, - sharedInputChannels: Int, - sharedDefaultGroupCount: Int, - nCores: Int, - computeCoreIds: Seq[Int], - tlb_size: Int, - dma_n_xacts: Int, - dma_burst_maxbytes: Int, - bankChannel: Int, - max_in_flight_mem_reqs: Int, - dma_buswidth: Int, - memAddrLen: Int, - tmaReadChannel: Int, - tmaWriteChannel: Int, - mmioEnable: Boolean, - mmioBankNum: Int, - mmioBankEntries: Int, - mmioBankWidth: Int, - mmioReadWidth: Int) { - - if (sharedEnable) { - require(sharedBankEntries >= 2 && sharedBankEntries <= 65536) - require(sharedEntries == sharedBankNum * sharedBankEntries, "Shared pool size must equal banks times rows per bank") - } - - // MMIO derived values - val mmioBankBytes: Int = mmioBankEntries * (mmioBankWidth / 8) - val mmioTotalBytes: Int = mmioBankNum * mmioBankBytes - val vbankIdWidth: Int = chisel3.util.log2Up(virtualBankCount) - val maxGroupCount: Int = math.max(bankNum, if (sharedEnable) sharedBankNum else bankNum) - val groupCountWidth: Int = chisel3.util.log2Up(maxGroupCount + 1) - val groupIdWidth: Int = chisel3.util.log2Up(maxGroupCount) -} - -object MemDomainParam { - implicit val rw: ReadWriter[MemDomainParam] = macroRW - - /** Stub default; real values come from ChipLoader. */ - def apply(): MemDomainParam = MemDomainParam( - bankNum = 0, - bankWidth = 0, - bankEntries = 0, - bankMaskLen = 0, - virtualBankCount = 0, - sharedEnable = false, - sharedEntries = 0, - sharedBankEntries = 0, - sharedBankNum = 0, - sharedInputChannels = 0, - sharedDefaultGroupCount = 0, - nCores = 0, - computeCoreIds = Seq.empty, - tlb_size = 0, - dma_n_xacts = 0, - dma_burst_maxbytes = 0, - bankChannel = 0, - max_in_flight_mem_reqs = 0, - dma_buswidth = 0, - memAddrLen = 0, - tmaReadChannel = 0, - tmaWriteChannel = 0, - mmioEnable = false, - mmioBankNum = 0, - mmioBankEntries = 0, - mmioBankWidth = 0, - mmioReadWidth = 0 - ) - -} diff --git a/arch/src/main/scala/framework/memdomain/frontend/MemFrontend.scala b/arch/src/main/scala/framework/memdomain/frontend/MemFrontend.scala deleted file mode 100644 index 6961a069a..000000000 --- a/arch/src/main/scala/framework/memdomain/frontend/MemFrontend.scala +++ /dev/null @@ -1,263 +0,0 @@ -package framework.memdomain.frontend - -import memcore.memory.queue.Queue - -import chisel3._ -import chisel3.util._ -import framework.memdomain.frontend.mem.dma.{DmaError, DmaPort, DmaReadCommand} -import framework.memdomain.frontend.mem.{ - Footprint, - KernelDma, - KernelDmaPort, - MemConfiger, - MemConfigerIO, - MemLoader, - MemStorer -} -import framework.frontend.globalrs.{GlobalSchedComplete, GlobalSchedIssue} -import framework.frontend.decoder.GISA.MVIN_KERNEL_BITPAT -import framework.balldomain.blink.{BankRead, BankWrite} -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.top.GlobalConfig -import framework.memdomain.frontend.cmd.decoder.MemDomainDecoder -import framework.memdomain.frontend.cmd.rs.MemReservationStation -import framework.memdomain.utils.pmc.MemCyclePMC -import framework.memdomain.isa.{MvoverISA, MvoverPort} - -/** - * MemFrontend: - * Provides DMA interface and Ball Domain interface - */ -@instantiable -class MemFrontend(val b: GlobalConfig) extends Module { - - @public - val io = IO(new Bundle { - // Issue interface from global RS (single channel) - val global_issue_i = Flipped(Decoupled(new GlobalSchedIssue(b))) - // Report completion to global RS (single channel) - val global_complete_o = Decoupled(new GlobalSchedComplete(b)) - val mvover = new MvoverPort - - // Bank read/write interface - used by load/store - val interdma = new Bundle { - val bankRead = Flipped(new BankRead(b)) - val bankWrite = Flipped(new BankWrite(b)) - val read_is_shared = Output(Bool()) - val write_is_shared = Output(Bool()) - } - - val dma = new DmaPort(b.memDomain.dma_buswidth) - - val config = Decoupled(new MemConfigerIO(b)) - - // MMIO outputs (to MmioPool via MemDomain) - val is_mvin_mmio_active = Output(Bool()) - val mmio_addr = Output(UInt(17.W)) - val mmio_col = Output(UInt(8.W)) - - // Query interface to backend for group count - val query_vbank_id = Output(UInt(b.memDomain.vbankIdWidth.W)) - val query_is_shared = Output(Bool()) - val query_group_count = Input(UInt(b.memDomain.groupCountWidth.W)) - val clearBusy = Input(Bool()) - - val hartid = Input(UInt(b.tile.xLen.W)) - val footprints = Output(Vec(3, new Footprint(b))) - - val kernel = if (b.rvv.enable) Some(new KernelDmaPort(b)) else None - - // Busy signal - val busy = Output(Bool()) - }) - - val memDecoder: Instance[MemDomainDecoder] = Instantiate(new MemDomainDecoder(b)) - val memRs: Instance[MemReservationStation] = Instantiate(new MemReservationStation(b)) - val memLoader: Instance[MemLoader] = Instantiate(new MemLoader(b)) - val memStorer: Instance[MemStorer] = Instantiate(new MemStorer(b)) - val pmc: Instance[MemCyclePMC] = Instantiate(new MemCyclePMC(b)) - - val configer: Instance[MemConfiger] = Instantiate(new MemConfiger(b)) - -// ----------------------------------------------------------------------------- -// Global RS -> MemDecoder -// ----------------------------------------------------------------------------- - val isMvover = io.global_issue_i.bits.cmd.cmd.funct === MvoverISA.Funct.U - val mvoverPending = RegInit(false.B) - val mvoverRobId = Reg(chiselTypeOf(io.global_issue_i.bits.rob_id)) - val mvoverIsSub = Reg(Bool()) - val mvoverSubRobId = Reg(chiselTypeOf(io.global_issue_i.bits.sub_rob_id)) - - io.mvover.command.valid := io.global_issue_i.valid && isMvover && !mvoverPending - io.mvover.command.bits.sourceCore := io.global_issue_i.bits.cmd.cmd.rs1Data(7, 0) - io.mvover.command.bits.targetCore := io.global_issue_i.bits.cmd.cmd.rs1Data(15, 8) - io.mvover.command.bits.sourceBank := io.global_issue_i.bits.cmd.cmd.rs1Data(25, 16) - io.mvover.command.bits.targetBank := io.global_issue_i.bits.cmd.cmd.rs1Data(35, 26) - io.mvover.command.bits.sourceAddr := io.global_issue_i.bits.cmd.cmd.rs2Data(15, 0) - io.mvover.command.bits.targetAddr := io.global_issue_i.bits.cmd.cmd.rs2Data(31, 16) - io.mvover.command.bits.rows := io.global_issue_i.bits.cmd.cmd.rs2Data(47, 32) +& 1.U - when(io.mvover.command.fire) { - mvoverPending := true.B - mvoverRobId := io.global_issue_i.bits.rob_id - mvoverIsSub := io.global_issue_i.bits.is_sub - mvoverSubRobId := io.global_issue_i.bits.sub_rob_id - } - memDecoder.io.cmd_i.valid := io.global_issue_i.valid && !isMvover && !mvoverPending - memDecoder.io.cmd_i.bits := io.global_issue_i.bits.cmd - io.global_issue_i.ready := !mvoverPending && - Mux(isMvover, io.mvover.command.ready, memDecoder.io.cmd_i.ready) - - // Config signal goes to backend - io.config <> configer.io.config - - val selectStorerQuery = memStorer.io.query_valid - io.query_vbank_id := Mux(selectStorerQuery, memStorer.io.query_vbank_id, memLoader.io.query_vbank_id) - io.query_is_shared := Mux(selectStorerQuery, memStorer.io.query_is_shared, memLoader.io.query_is_shared) - memLoader.io.query_group_count := io.query_group_count - memStorer.io.query_group_count := io.query_group_count - - when( - memLoader.io.query_valid && memStorer.io.query_valid && - memLoader.io.query_is_shared && memStorer.io.query_is_shared && - memLoader.io.query_vbank_id =/= memStorer.io.query_vbank_id - ) { - assert(false.B, "MemFrontend shared query conflict: loader and storer query different vbanks in the same cycle\n") - } - -// ----------------------------------------------------------------------------- -// MemDecoder -> MemReservationStation -// ----------------------------------------------------------------------------- - // Connect decoded instruction and global rob_id - memRs.io.mem_decode_cmd_i.valid := memDecoder.io.mem_decode_cmd_o.valid - memRs.io.mem_decode_cmd_i.bits.cmd := memDecoder.io.mem_decode_cmd_o.bits - memRs.io.mem_decode_cmd_i.bits.rob_id := io.global_issue_i.bits.rob_id - memRs.io.mem_decode_cmd_i.bits.is_sub := io.global_issue_i.bits.is_sub - memRs.io.mem_decode_cmd_i.bits.sub_rob_id := io.global_issue_i.bits.sub_rob_id - memDecoder.io.mem_decode_cmd_o.ready := memRs.io.mem_decode_cmd_i.ready - -// ----------------------------------------------------------------------------- -// MemReservationStation -> MemLoader/MemStorer -// ----------------------------------------------------------------------------- - memLoader.io.cmdReq <> memRs.io.issue_o.ld - memStorer.io.cmdReq <> memRs.io.issue_o.st - configer.io.cmdReq <> memRs.io.issue_o.cf - configer.io.hartid := io.hartid - configer.io.clearBusy := io.clearBusy - memRs.io.commit_i.ld <> memLoader.io.cmdResp - memRs.io.commit_i.st <> memStorer.io.cmdResp - memRs.io.commit_i.cf <> configer.io.cmdResp - -//===-------------------------------------------------------------------===//-- -// PMC - Performance Monitor Counter -// ----------------------------------------------------------------------------- - pmc.io.ldReq_i.valid := memRs.io.issue_o.ld.fire - pmc.io.ldReq_i.bits := memRs.io.issue_o.ld.bits - pmc.io.stReq_i.valid := memRs.io.issue_o.st.fire - pmc.io.stReq_i.bits := memRs.io.issue_o.st.bits - pmc.io.ldResp_o.valid := memLoader.io.cmdResp.fire - pmc.io.ldResp_o.bits := memLoader.io.cmdResp.bits - pmc.io.stResp_o.valid := memStorer.io.cmdResp.fire - pmc.io.stResp_o.bits := memStorer.io.cmdResp.bits - - // A logical request is accepted into this queue before its external transport fires. - // Keep both producer identity and transfer stable while the transport is backpressured. - val readQueue = Module(new Queue(new DmaReadCommand, 1)) - val readerOwned = RegInit(false.B) - val kernelOwner = RegInit(false.B) - io.dma.read.valid := readQueue.io.deq.valid && !readerOwned - io.dma.read.bits := readQueue.io.deq.bits - readQueue.io.deq.ready := io.dma.read.ready && !readerOwned - when(io.dma.read.fire) { - readerOwned := true.B - kernelOwner := io.dma.read.bits.producer === 2.U - } - when(io.dma.readResult.fire && io.dma.readResult.bits.last)(readerOwned := false.B) - when(io.dma.readResult.valid) { - assert(readerOwned || io.dma.read.fire, "MemFrontend DMA read result has no accepted transport owner") - } - io.footprints(0) := memLoader.io.footprint - io.footprints(1) := memStorer.io.footprint - io.footprints(2) := 0.U.asTypeOf(new Footprint(b)) - if (b.rvv.enable) { - val kernelDma: Instance[KernelDma] = Instantiate(new KernelDma(b)) - io.footprints(2) := kernelDma.io.footprint - kernelDma.io.kernel <> io.kernel.get - val selectKernel = kernelDma.io.request.valid - readQueue.io.enq.valid := kernelDma.io.request.valid || memLoader.io.dmaReq.valid - readQueue.io.enq.bits.transfer := Mux(selectKernel, kernelDma.io.request.bits, memLoader.io.dmaReq.bits) - readQueue.io.enq.bits.producer := Mux(selectKernel, 2.U, 0.U) - kernelDma.io.request.ready := selectKernel && readQueue.io.enq.ready - memLoader.io.dmaReq.ready := !selectKernel && readQueue.io.enq.ready - // KernelDma treats enqueue acceptance as an obligation to drain even after abort. - kernelDma.io.response.valid := readerOwned && kernelOwner && io.dma.readResult.valid - kernelDma.io.response.bits := io.dma.readResult.bits - memLoader.io.dmaResp.valid := readerOwned && !kernelOwner && io.dma.readResult.valid - memLoader.io.dmaResp.bits := io.dma.readResult.bits - io.dma.readResult.ready := readerOwned && Mux(kernelOwner, kernelDma.io.response.ready, memLoader.io.dmaResp.ready) - } else { - readQueue.io.enq.valid := memLoader.io.dmaReq.valid - readQueue.io.enq.bits.transfer := memLoader.io.dmaReq.bits - readQueue.io.enq.bits.producer := 0.U - memLoader.io.dmaReq.ready := readQueue.io.enq.ready - memLoader.io.dmaResp.valid := readerOwned && io.dma.readResult.valid - memLoader.io.dmaResp.bits := io.dma.readResult.bits - io.dma.readResult.ready := readerOwned && memLoader.io.dmaResp.ready - } - io.dma.write <> memStorer.io.dmaReq - io.dma.writeData <> memStorer.io.dmaData - memStorer.io.dmaResp <> io.dma.writeResult - - // MemLoader owns the frontend write channel until its write response returns, - // preserving BankWrite's request/response pairing. - val frontendWriteBusy = RegInit(false.B) - - io.interdma.bankWrite.io.req.valid := !frontendWriteBusy && memLoader.io.bankWrite.io.req.valid - io.interdma.bankWrite.io.req.bits := memLoader.io.bankWrite.io.req.bits - io.interdma.bankWrite.bank_id := memLoader.io.bankWrite.bank_id - io.interdma.bankWrite.rob_id := memLoader.io.bankWrite.rob_id - io.interdma.bankWrite.ball_id := memLoader.io.bankWrite.ball_id - io.interdma.bankWrite.group_id := memLoader.io.bankWrite.group_id - memLoader.io.bankWrite.io.req.ready := !frontendWriteBusy && io.interdma.bankWrite.io.req.ready - - io.interdma.bankWrite.io.resp.ready := frontendWriteBusy && memLoader.io.bankWrite.io.resp.ready - memLoader.io.bankWrite.io.resp.valid := frontendWriteBusy && io.interdma.bankWrite.io.resp.valid - memLoader.io.bankWrite.io.resp.bits := io.interdma.bankWrite.io.resp.bits - - when(io.interdma.bankWrite.io.req.fire)(frontendWriteBusy := true.B) - when(io.interdma.bankWrite.io.resp.fire)(frontendWriteBusy := false.B) - - memStorer.io.bankRead <> io.interdma.bankRead - io.interdma.read_is_shared := memStorer.io.is_shared - io.interdma.write_is_shared := memLoader.io.is_shared - - // MMIO signals from MemConfiger and MemLoader, exposed to MemDomain - io.is_mvin_mmio_active := memLoader.io.is_mvin_mmio_active - io.mmio_addr := memLoader.io.mmio_addr - io.mmio_col := memLoader.io.mmio_col - - // Completion signal connected to global RS - val mvoverComplete = mvoverPending && io.mvover.completion.valid - io.global_complete_o.valid := mvoverComplete || memRs.io.complete_o.valid - io.global_complete_o.bits.rob_id := Mux(mvoverComplete, mvoverRobId, memRs.io.complete_o.bits.rob_id) - io.global_complete_o.bits.is_sub := Mux(mvoverComplete, mvoverIsSub, memRs.io.complete_o.bits.is_sub) - io.global_complete_o.bits.sub_rob_id := Mux(mvoverComplete, mvoverSubRobId, memRs.io.complete_o.bits.sub_rob_id) - io.global_complete_o.bits.fault := Mux( - mvoverComplete, - 0.U.asTypeOf(io.global_complete_o.bits.fault), - memRs.io.complete_o.bits.fault - ) - when(mvoverComplete && io.mvover.completion.bits) { - io.global_complete_o.bits.fault.error := DmaError.Bank.U - } - - memRs.io.complete_o.ready := io.global_complete_o.ready && !mvoverComplete - io.mvover.completion.ready := io.global_complete_o.ready && mvoverPending - when(io.mvover.completion.fire) { - mvoverPending := false.B - } - - // Busy signal - // Simple busy signal - io.busy := mvoverPending || !memRs.io.complete_o.ready || io.kernel.map(_.busy).getOrElse(false.B) || - readQueue.io.deq.valid || readerOwned || io.dma.readBusy || io.dma.writeBusy -} diff --git a/arch/src/main/scala/framework/memdomain/frontend/cmd/decoder/DISA.scala b/arch/src/main/scala/framework/memdomain/frontend/cmd/decoder/DISA.scala deleted file mode 100644 index ea91c3892..000000000 --- a/arch/src/main/scala/framework/memdomain/frontend/cmd/decoder/DISA.scala +++ /dev/null @@ -1,17 +0,0 @@ -package framework.memdomain.frontend.cmd.decoder - -import chisel3._ -import chisel3.util._ -import framework.memdomain.isa.MemISA - -object DISA { - val MVOVER_BITPAT = BitPat("b0001101") - val MSET_FUNCT = MemISA.MsetFunct - // enable=010 (1 write), opcode group - val MSET_BITPAT = BitPat("b0100000") // 32 (0x20) — enable=010, opcode=0 - val MVIN_BITPAT = BitPat("b0100001") // 33 (0x21) — enable=010, opcode=1 - val MVIN_2D_BITPAT = BitPat("b0100010") // 34 (0x22) — 2D DMA into bank rows - val MVIN_MMIO_BITPAT = BitPat("b0100011") // 35 (0x23) — enable=010, opcode=3 (DMA into MMIO) - // enable=001 (1 read), opcode group - val MVOUT_BITPAT = BitPat("b0010000") // 16 (0x10) — enable=001, opcode=0 -} diff --git a/arch/src/main/scala/framework/memdomain/frontend/cmd/decoder/DomainDecoder.scala b/arch/src/main/scala/framework/memdomain/frontend/cmd/decoder/DomainDecoder.scala deleted file mode 100644 index cce719409..000000000 --- a/arch/src/main/scala/framework/memdomain/frontend/cmd/decoder/DomainDecoder.scala +++ /dev/null @@ -1,162 +0,0 @@ -package framework.memdomain.frontend.cmd.decoder - -import chisel3._ -import chisel3.util._ -import framework.frontend.decoder.{DomainId, PostGDCmd} -import framework.top.GlobalConfig -import framework.memdomain.frontend.cmd.decoder.DISA._ -import freechips.rocketchip.tile._ -import chisel3.experimental.hierarchy.{instantiable, public} - -// Detailed decode output for Mem domain -class MemDecodeCmd(b: GlobalConfig) extends Bundle { - // Shared memory access marker: raw BB_BANK0 id >= frontend.sharedBankIdBase. - val is_shared = Bool() - val is_load = Bool() - val is_store = Bool() - val is_config = Bool() - val clear = Bool() - val transfer = Bool() - val source_bank_id = UInt(b.memDomain.vbankIdWidth.W) - - val is_mvin_mmio = Bool() - val is_mvin_2d = Bool() - - val mem_addr = UInt(b.memDomain.memAddrLen.W) - val iter = UInt(b.frontend.iter_len.W) - val bank_id = UInt(b.memDomain.vbankIdWidth.W) - val special = UInt(64.W) -} - -// LS decode fields (iter removed from decode table — always from rs1[63:48]) -object LSDecodeFields extends Enumeration { - type Field = Value - val LD_EN, ST_EN, MEMADDR, BANK_ID, SPECIAL, VALID = Value -} - -// Default constants for Mem decoder -object MemDefaultConstants { - val Y = true.B - val N = false.B - val DADDR = 0.U(15.W) - val DSPECIAL = 0.U(64.W) -} - -@instantiable -class MemDomainDecoder(val b: GlobalConfig) extends Module { - import MemDefaultConstants._ - - val sharedBankIdBase = b.frontend.shared_bank_id_base - require(sharedBankIdBase > 0, s"sharedBankIdBase($sharedBankIdBase) must be > 0") - require( - sharedBankIdBase < (1 << b.frontend.bank_id_len), - s"sharedBankIdBase($sharedBankIdBase) must fit in bankIdLen(${b.frontend.bank_id_len})" - ) - require( - sharedBankIdBase > b.frontend.vbank_id_upper_bound, - s"sharedBankIdBase($sharedBankIdBase) must be > vbankIdUpperBound(${b.frontend.vbank_id_upper_bound})" - ) - - @public - val io = IO(new Bundle { - val cmd_i = Flipped(Decoupled(new PostGDCmd(b))) - val mem_decode_cmd_o = Decoupled(new MemDecodeCmd(b)) - }) - - val memAddrLen = b.memDomain.memAddrLen - val bankIdLen = b.frontend.bank_id_len - val iterLen = b.frontend.iter_len - - // Only process Mem instructions - io.cmd_i.ready := io.mem_decode_cmd_o.ready - - val func7 = io.cmd_i.bits.cmd.funct - val rs1 = io.cmd_i.bits.cmd.rs1Data - val rs2 = io.cmd_i.bits.cmd.rs2Data - - // Unified encoding: - // rs1[9:0] = read/config bank (BANK0) - // rs1[29:20] = write bank (BANK2) - // rs1[63:30] = iter (34-bit) - // funct7[6:4] = enable (bank access flags, decoded by GlobalDecoder) - // rs2[38:0] = mem_addr (for MVIN/MVOUT, 39-bit) - // - // mvin_mmio: rs1[63:30]=row (iter), rs2[38:0]=dram_addr, rs2[55:39]=mmio_addr, rs2[63:56]=col - import LSDecodeFields._ - // format: off - val ls_default_decode = List(N, N, DADDR, DADDR, DSPECIAL, N) - val ls_decode_list = ListLookup(func7, ls_default_decode, Array( - MSET_BITPAT -> List(N, N, 0.U(memAddrLen.W), rs1(bankIdLen - 1, 0), rs2, Y), - MVIN_BITPAT -> List(Y, N, rs2(memAddrLen - 1, 0), rs1(bankIdLen + 19, 20), rs2, Y), - MVIN_2D_BITPAT -> List(Y, N, Cat(rs2(35, 0), 0.U(3.W)), rs1(bankIdLen + 19, 20), rs2, Y), - MVOUT_BITPAT -> List(N, Y, rs2(memAddrLen - 1, 0), rs1(bankIdLen - 1, 0), rs2, Y), - MVIN_MMIO_BITPAT -> List(Y, N, rs2(memAddrLen - 1, 0), 0.U(bankIdLen.W), rs2, Y) - ) - ) - // format: on - - assert( - !(io.cmd_i.fire && !ls_decode_list(LSDecodeFields.VALID.id).asBool), - s"MemDomainDecoder: Invalid command opcode, func7 = 0x%x\n", - func7 - ) - -// ----------------------------------------------------------------------------- -// Output assignment -// ----------------------------------------------------------------------------- - io.mem_decode_cmd_o.valid := io.cmd_i.valid && (io.cmd_i.bits.domain_id === DomainId.MEM) - - val transfer = func7 === MSET_BITPAT && rs2(12) - val raw_bank_id = Mux(func7 === MVIN_BITPAT || func7 === MVIN_2D_BITPAT || transfer, rs1(29, 20), rs1(9, 0)) - val shared_bank_id = b.memDomain.sharedEnable.B && raw_bank_id >= sharedBankIdBase.U - // format: off - // BB_BANK0 is encoded in rs1[9:0]. Keep the configured architectural ID; - // private and shared banks occupy disjoint ranges in that namespace. - io.mem_decode_cmd_o.bits.is_shared := io.mem_decode_cmd_o.valid && shared_bank_id - io.mem_decode_cmd_o.bits.is_load := Mux(io.mem_decode_cmd_o.valid, ls_decode_list(LSDecodeFields.LD_EN.id).asBool, false.B) - io.mem_decode_cmd_o.bits.is_store := Mux(io.mem_decode_cmd_o.valid, ls_decode_list(LSDecodeFields.ST_EN.id).asBool, false.B) - io.mem_decode_cmd_o.bits.is_config := Mux(io.mem_decode_cmd_o.valid, func7 === MSET_BITPAT, false.B) - io.mem_decode_cmd_o.bits.transfer := io.mem_decode_cmd_o.valid && transfer - io.mem_decode_cmd_o.bits.source_bank_id := rs1(bankIdLen - 1, 0) - io.mem_decode_cmd_o.bits.clear := io.mem_decode_cmd_o.valid && (func7 === MSET_BITPAT) && rs2(11) - io.mem_decode_cmd_o.bits.is_mvin_mmio := Mux(io.mem_decode_cmd_o.valid, func7 === MVIN_MMIO_BITPAT, false.B) - io.mem_decode_cmd_o.bits.is_mvin_2d := Mux(io.mem_decode_cmd_o.valid, func7 === MVIN_2D_BITPAT, false.B) - io.mem_decode_cmd_o.bits.mem_addr := Mux(io.mem_decode_cmd_o.valid, ls_decode_list(LSDecodeFields.MEMADDR.id).asUInt, 0.U(b.memDomain.memAddrLen.W)) - io.mem_decode_cmd_o.bits.iter := Mux(io.mem_decode_cmd_o.valid, rs1(63, 30), 0.U(iterLen.W)) - - when(io.cmd_i.fire && func7 === MSET_BITPAT) { - when(transfer) { - assert(rs1(63, 30) === 0.U && rs1(19, 10) === 0.U, "MSET transfer reserves rs1 outside BANK0/BANK2") - assert(rs2 === (1.U << 12), "MSET transfer reserves other rs2 bits") - assert(rs1(9, 0) =/= rs1(29, 20), "MSET transfer requires distinct source and target") - assert((rs1(9, 0) >= sharedBankIdBase.U) === (rs1(29, 20) >= sharedBankIdBase.U), "MSET transfer requires one bank namespace") - }.otherwise { - assert(rs1(63, 10) === 0.U, "MSET reserves rs1[63:10]") - assert(rs2(63, 12) === 0.U, "MSET reserves rs2[63:12]") - } - assert(!(rs2(11) && !rs2(10)), "MSET clear requires alloc=1") - } - when(io.cmd_i.fire && (func7 === MVIN_BITPAT || func7 === MVOUT_BITPAT)) { - assert(rs2(63) || rs2(62, 58) === 0.U, "Whole-bank DMA reserves selected group bits") - } - when(io.cmd_i.fire && func7 === MVIN_2D_BITPAT) { - val height = rs1(63, 30) - val pixelBytes = rs2(42, 36) << 3 - val width = rs2(61, 59) +& 1.U - val validBytes = Mux(rs2(62), 8.U(5.W), 16.U(5.W)) - assert(height =/= 0.U, "MVIN_2D height must be non-zero") - assert(rs2(42, 36) =/= 0.U, "MVIN_2D pixel bytes must be non-zero") - assert(rs2(52, 43) =/= 0.U, "MVIN_2D source width must be non-zero") - assert(validBytes <= pixelBytes, "MVIN_2D valid bytes exceed pixel bytes") - assert((rs2(58, 53) +& (height * width)) <= Mux(shared_bank_id, b.memDomain.sharedBankEntries.U, b.memDomain.bankEntries.U), "MVIN_2D destination exceeds bank") - assert(!rs2(63), "MVIN_2D reserves rs2[63]") - } - when(io.cmd_i.fire && (func7 === MVIN_BITPAT || func7 === MVIN_2D_BITPAT)) { - assert(rs1(19, 0) === 0.U, "MVIN write-bank encoding reserves rs1[19:0]") - } - - val ls_bank_id = ls_decode_list(LSDecodeFields.BANK_ID.id).asUInt - io.mem_decode_cmd_o.bits.bank_id := Mux(io.mem_decode_cmd_o.valid, Mux(transfer, raw_bank_id, ls_bank_id), 0.U(b.memDomain.vbankIdWidth.W)) - io.mem_decode_cmd_o.bits.special := Mux(io.mem_decode_cmd_o.valid, ls_decode_list(LSDecodeFields.SPECIAL.id).asUInt, 0.U(64.W)) - // format: on -} diff --git a/arch/src/main/scala/framework/memdomain/frontend/cmd/rs/README.md b/arch/src/main/scala/framework/memdomain/frontend/cmd/rs/README.md deleted file mode 100644 index d72da7ced..000000000 --- a/arch/src/main/scala/framework/memdomain/frontend/cmd/rs/README.md +++ /dev/null @@ -1,54 +0,0 @@ -# Reservation Station (RS) Module - -## Overview - -Reservation station module for memory domain instruction scheduling, located at `framework/memdomain/frontend/cmd/rs`. Local Mem RS is a small FIFO scheduler; global reorder lives in Frontend `GlobalROB`. - -## File Structure - -``` -rs/ -└── reservationStation.scala - Memory reservation station -``` - -## MemReservationStation - -### Interface - -```scala -class MemReservationStation(implicit b: CustomBuckyballConfig, p: Parameters) extends Module { - val io = IO(new Bundle { - val mem_decode_cmd_i = Flipped(Decoupled(new MemDecodeCmd)) - val rs_rocc_o = new Bundle { - val resp = Decoupled(new RoCCResponseBB) - val busy = Output(Bool()) - } - val issue_o = new MemIssueInterface - val commit_i = new MemCommitInterface - }) -} -``` - -### Issue Interface - -```scala -class MemIssueInterface(implicit b: CustomBuckyballConfig, p: Parameters) extends Bundle { - val ld = Decoupled(new MemRsIssue) // Load instruction issue - val st = Decoupled(new MemRsIssue) // Store instruction issue -} -``` - -### Commit Interface - -```scala -class MemCommitInterface(implicit b: CustomBuckyballConfig, p: Parameters) extends Bundle { - val ld = Flipped(Decoupled(new MemRsComplete)) // Load completion - val st = Flipped(Decoupled(new MemRsComplete)) // Store completion -} -``` - -## Related Modules - -- [Memory Domain](../README.md) -- [Memory Controller](../mem/README.md) -- [DMA Engines](../dma/README.md) diff --git a/arch/src/main/scala/framework/memdomain/frontend/cmd/rs/reservationStation.scala b/arch/src/main/scala/framework/memdomain/frontend/cmd/rs/reservationStation.scala deleted file mode 100644 index a049aef47..000000000 --- a/arch/src/main/scala/framework/memdomain/frontend/cmd/rs/reservationStation.scala +++ /dev/null @@ -1,136 +0,0 @@ -package framework.memdomain.frontend.cmd.rs - -import memcore.memory.queue.Queue - -import chisel3._ -import chisel3.util._ -import chisel3.experimental._ -import framework.memdomain.frontend.cmd.decoder.MemDecodeCmd -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.top.GlobalConfig - -// Mem domain issue interface - includes global rob_id -class MemRsIssue(val b: GlobalConfig) extends Bundle { - val cmd = new MemDecodeCmd(b) - // Global ROB ID - val rob_id = UInt(log2Up(b.frontend.rob_entries).W) - val is_sub = Bool() - val sub_rob_id = UInt(log2Up(b.frontend.sub_rob_depth * 4).W) -} - -// Mem domain completion interface -class MemRsComplete(val b: GlobalConfig) extends Bundle { - val fault = new framework.memdomain.frontend.mem.dma.DmaStatus - val rob_id = UInt(log2Up(b.frontend.rob_entries).W) - val is_sub = Bool() - val sub_rob_id = UInt(log2Up(b.frontend.sub_rob_depth * 4).W) -} - -// Mem domain issue interface combination (Load + Store) -class MemIssueInterface(val b: GlobalConfig) extends Bundle { - val ld = Decoupled(new MemRsIssue(b)) - val st = Decoupled(new MemRsIssue(b)) - val cf = Decoupled(new MemRsIssue(b)) -} - -// Mem domain completion interface combination (Load + Store) -class MemCommitInterface(val b: GlobalConfig) extends Bundle { - val ld = Flipped(Decoupled(new MemRsComplete(b))) - val st = Flipped(Decoupled(new MemRsComplete(b))) - val cf = Flipped(Decoupled(new MemRsComplete(b))) -} - -// Local Mem reservation station - simple FIFO scheduler -@instantiable -class MemReservationStation(val b: GlobalConfig) extends Module { - - @public - val io = IO(new Bundle { - - // Decoded instruction input (with global rob_id) - val mem_decode_cmd_i = Flipped(new DecoupledIO(new Bundle { - val cmd = new MemDecodeCmd(b) - // Global ROB ID - val rob_id = UInt(log2Up(b.frontend.rob_entries).W) - val is_sub = Bool() - val sub_rob_id = UInt(log2Up(b.frontend.sub_rob_depth * 4).W) - })) - - // Rs -> MemLoader/MemStorer - val issue_o = new MemIssueInterface(b) - val commit_i = new MemCommitInterface(b) - - // Output completion signal (with global rob_id, single channel) - val complete_o = Decoupled(new MemRsComplete(b)) - }) - - // Simple FIFO queue, only for buffering - val fifo = Module(new Queue( - new Bundle { - val cmd = new MemDecodeCmd(b) - val rob_id = UInt(log2Up(b.frontend.rob_entries).W) - val is_sub = Bool() - val sub_rob_id = UInt(log2Up(b.frontend.sub_rob_depth * 4).W) - }, - entries = 4 - )) // Small buffer is sufficient - -// ----------------------------------------------------------------------------- -// Inbound - FIFO enqueue -// ----------------------------------------------------------------------------- - fifo.io.enq <> io.mem_decode_cmd_i - -// ----------------------------------------------------------------------------- -// Outbound - instruction issue (dispatch based on is_load/is_store) -// ----------------------------------------------------------------------------- - val headEntry = fifo.io.deq.bits - - // Load issue - io.issue_o.ld.valid := fifo.io.deq.valid && headEntry.cmd.is_load - io.issue_o.ld.bits.cmd := headEntry.cmd - io.issue_o.ld.bits.rob_id := headEntry.rob_id - io.issue_o.ld.bits.is_sub := headEntry.is_sub - io.issue_o.ld.bits.sub_rob_id := headEntry.sub_rob_id - - // Store issue - io.issue_o.st.valid := fifo.io.deq.valid && headEntry.cmd.is_store - io.issue_o.st.bits.cmd := headEntry.cmd - io.issue_o.st.bits.rob_id := headEntry.rob_id - io.issue_o.st.bits.is_sub := headEntry.is_sub - io.issue_o.st.bits.sub_rob_id := headEntry.sub_rob_id - - // Config issue - io.issue_o.cf.valid := fifo.io.deq.valid && headEntry.cmd.is_config - io.issue_o.cf.bits.cmd := headEntry.cmd - io.issue_o.cf.bits.rob_id := headEntry.rob_id - io.issue_o.cf.bits.is_sub := headEntry.is_sub - io.issue_o.cf.bits.sub_rob_id := headEntry.sub_rob_id - - // FIFO deq.ready - can only dequeue when target unit is ready - fifo.io.deq.ready := - (headEntry.cmd.is_load && io.issue_o.ld.ready) || - (headEntry.cmd.is_store && io.issue_o.st.ready) || - (headEntry.cmd.is_config && io.issue_o.cf.ready) - -// ----------------------------------------------------------------------------- -// Completion signal processing - directly forward to global RS -// ----------------------------------------------------------------------------- - val completeArb = Module(new Arbiter(new MemRsComplete(b), 3)) - - completeArb.io.in(0).valid := io.commit_i.ld.valid - completeArb.io.in(0).bits := io.commit_i.ld.bits - io.commit_i.ld.ready := completeArb.io.in(0).ready - - completeArb.io.in(1).valid := io.commit_i.st.valid - completeArb.io.in(1).bits := io.commit_i.st.bits - io.commit_i.st.ready := completeArb.io.in(1).ready - - completeArb.io.in(2).valid := io.commit_i.cf.valid - completeArb.io.in(2).bits := io.commit_i.cf.bits - io.commit_i.cf.ready := completeArb.io.in(2).ready - - // Forward completion signal (with global rob_id) - io.complete_o.valid := completeArb.io.out.valid - io.complete_o.bits := completeArb.io.out.bits - completeArb.io.out.ready := io.complete_o.ready -} diff --git a/arch/src/main/scala/framework/memdomain/frontend/mem/Footprint.scala b/arch/src/main/scala/framework/memdomain/frontend/mem/Footprint.scala deleted file mode 100644 index cb9f83bab..000000000 --- a/arch/src/main/scala/framework/memdomain/frontend/mem/Footprint.scala +++ /dev/null @@ -1,21 +0,0 @@ -package framework.memdomain.frontend.mem - -import chisel3._ -import chisel3.util._ -import framework.top.GlobalConfig -import framework.memdomain.frontend.mem.dma.DmaStatus - -class Footprint(b: GlobalConfig) extends Bundle { - val valid = Bool() - val rob_id = UInt(log2Up(b.frontend.rob_entries).W) - val is_sub = Bool() - val sub_rob_id = UInt(log2Up(b.frontend.sub_rob_depth * 4).W) - val baseVA = UInt(64.W) - val rows = UInt(64.W) - val columns = UInt(64.W) - val spanBytes = UInt(64.W) - val columnStride = UInt(64.W) - val rowStride = UInt(64.W) - val write = Bool() - val fault = new DmaStatus -} diff --git a/arch/src/main/scala/framework/memdomain/frontend/mem/KernelDma.scala b/arch/src/main/scala/framework/memdomain/frontend/mem/KernelDma.scala deleted file mode 100644 index de0033ab5..000000000 --- a/arch/src/main/scala/framework/memdomain/frontend/mem/KernelDma.scala +++ /dev/null @@ -1,145 +0,0 @@ -package framework.memdomain.frontend.mem - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.memdomain.frontend.mem.dma.{BBReadRequest, BBReadResponse, DmaError, DmaStatus} -import framework.top.GlobalConfig -import framework.balldomain.blink.{BankRead, BankWrite} - -class KernelDmaRequest(b: GlobalConfig) extends Bundle { - val rob_id = UInt(log2Ceil(b.frontend.rob_entries).W) - val address = UInt(64.W) - val bytes = UInt(32.W) -} - -class KernelDmaPort(b: GlobalConfig) extends Bundle { - val load = Flipped(Decoupled(new KernelDmaRequest(b))) - val image = Decoupled(UInt(32.W)) - val result = Decoupled(new DmaStatus) - val abort = Input(Bool()) - val busy = Output(Bool()) -} - -class KernelMemoryBridge(b: GlobalConfig) extends KernelDmaPort(b) { - val bankRead = Vec(b.rvv.memoryPorts, new BankRead(b)) - val bankWrite = Vec(b.rvv.memoryPorts, new BankWrite(b)) - val active = Input(Bool()) - val ready = Output(Bool()) -} - -@instantiable -class KernelDma(val b: GlobalConfig) extends Module { - val beatWords = b.memDomain.dma_buswidth / 32 - val beatBytes = b.memDomain.dma_buswidth / 8 - require(beatWords > 0 && b.memDomain.dma_buswidth % 32 == 0) - - @public - val io = IO(new Bundle { - val footprint = Output(new Footprint(b)) - val kernel = new KernelDmaPort(b) - val request = Decoupled(new BBReadRequest) - val response = Flipped(Decoupled(new BBReadResponse(b.memDomain.dma_buswidth))) - }) - - val idle :: request :: receive :: output :: terminal :: Nil = Enum(5) - val state = RegInit(idle) - val descriptor = Reg(new KernelDmaRequest(b)) - val remaining = Reg(UInt(32.W)) - val data = Reg(UInt(b.memDomain.dma_buswidth.W)) - val words = Reg(UInt(math.max(1, log2Ceil(beatWords + 1)).W)) - val index = Reg(UInt(math.max(1, log2Ceil(beatWords)).W)) - val last = Reg(Bool()) - val discarded = RegInit(false.B) - val result = RegInit(0.U.asTypeOf(new DmaStatus)) - val owner = Reg(UInt(log2Ceil(b.frontend.rob_entries).W)) - - def readSpan(bytes: UInt): UInt = - (bytes.pad(64) + (beatBytes - 1).U) & (~(BigInt(beatBytes) - 1) & ((BigInt(1) << 64) - 1)).U(64.W) - - def invalidShape(address: UInt, bytes: UInt): Bool = { - val end = address +& (readSpan(bytes) - 1.U) - bytes === 0.U || bytes(1, 0) =/= 0.U || end(64) - } - - // The read DMA fills a complete output beat even for a short final payload. - // The preflight layer then aligns this span around the original VA. - io.footprint := 0.U.asTypeOf(new Footprint(b)) - io.footprint.valid := state =/= idle - io.footprint.rob_id := owner - io.footprint.baseVA := descriptor.address - io.footprint.rows := 1.U - io.footprint.columns := 1.U - io.footprint.spanBytes := readSpan(descriptor.bytes) - io.footprint.fault.error := Mux(invalidShape(descriptor.address, descriptor.bytes), DmaError.Shape.U, DmaError.None.U) - io.footprint.fault.address := Mux(io.footprint.fault.error =/= 0.U, descriptor.address, 0.U) - - io.kernel.load.ready := state === idle && !io.kernel.abort - io.kernel.busy := state =/= idle - io.request.valid := state === request && !io.kernel.abort - io.request.bits := 0.U.asTypeOf(io.request.bits) - io.request.bits.vaddr := descriptor.address - io.request.bits.len := descriptor.bytes - io.request.bits.groups := 1.U - io.request.bits.stride := 1.U - io.response.ready := state === receive - io.kernel.image.valid := state === output && !discarded && !io.kernel.abort - io.kernel.image.bits := (data >> (index * 32.U))(31, 0) - io.kernel.result.valid := state === terminal - io.kernel.result.bits := result - - // Abort cancels a load, never an execution. Once offered, the terminal result - // is immutable even if the image parser subsequently rejects the image. - when(io.kernel.abort && state =/= idle && state =/= terminal) { - discarded := true.B - when(result.error === DmaError.None.U) { - result.error := DmaError.Cancelled.U - result.address := descriptor.address - } - when(state === request)(state := terminal) - when(state === output)(state := Mux(last, terminal, receive)) - } - - when(io.kernel.load.fire) { - val invalid = invalidShape(io.kernel.load.bits.address, io.kernel.load.bits.bytes) - owner := io.kernel.load.bits.rob_id - descriptor := io.kernel.load.bits - remaining := io.kernel.load.bits.bytes >> 2 - discarded := false.B - result.error := Mux(invalid, DmaError.Shape.U, DmaError.None.U) - result.address := Mux(invalid, io.kernel.load.bits.address, 0.U) - state := Mux(invalid, terminal, request) - } - when(io.request.fire)(state := receive) - when(io.response.fire) { - data := io.response.bits.data - words := Mux(remaining < beatWords.U, remaining, beatWords.U) - index := 0.U - last := io.response.bits.last - remaining := remaining - Mux(remaining < beatWords.U, remaining, beatWords.U) - when(io.response.bits.fault.error =/= DmaError.None.U) { - when(result.error === DmaError.None.U || result.error === DmaError.Cancelled.U) { - result := io.response.bits.fault - } - discarded := true.B - when(io.response.bits.last)(state := terminal) - }.elsewhen(discarded || io.kernel.abort) { - when(io.response.bits.last)(state := terminal) - }.elsewhen(io.response.bits.last =/= (remaining <= beatWords.U) || remaining === 0.U) { - result.error := DmaError.Protocol.U - result.address := descriptor.address + (descriptor.bytes - (remaining << 2)) - discarded := true.B - when(io.response.bits.last)(state := terminal) - }.otherwise { - state := output - } - } - when(state === output && io.kernel.image.fire) { - when(index +& 1.U === words) { - state := Mux(last, terminal, receive) - }.otherwise { - index := index + 1.U - } - } - when(io.kernel.result.fire)(state := idle) -} diff --git a/arch/src/main/scala/framework/memdomain/frontend/mem/MemConfiger.scala b/arch/src/main/scala/framework/memdomain/frontend/mem/MemConfiger.scala deleted file mode 100644 index 9bf49fb88..000000000 --- a/arch/src/main/scala/framework/memdomain/frontend/mem/MemConfiger.scala +++ /dev/null @@ -1,141 +0,0 @@ -package framework.memdomain.frontend.mem - -import chisel3._ -import chisel3.util._ -import framework.top.GlobalConfig -import framework.memdomain.backend.shared.SharedMemLayout -import framework.memdomain.frontend.cmd.rs.{MemRsComplete, MemRsIssue} -import chisel3.experimental.hierarchy.{instantiable, public} - -class MemConfigerIO(val b: GlobalConfig) extends Bundle { - val vbank_id = Output(UInt(b.memDomain.vbankIdWidth.W)) - val is_shared = Output(Bool()) - val is_multi = Output(Bool()) - val alloc = Output(Bool()) - val transfer = Output(Bool()) - val source_bank_id = Output(UInt(b.memDomain.vbankIdWidth.W)) - // Zero the physical bank this allocation binds; the backend clears it locally. - val clear = Output(Bool()) - val group_id = Output(UInt(b.memDomain.groupIdWidth.W)) - val hart_id = Output(UInt(b.tile.xLen.W)) -} - -@instantiable -class MemConfiger(val b: GlobalConfig) extends Module { - val rob_id_width = log2Up(b.frontend.rob_entries) - - @public - val io = IO(new Bundle { - val cmdReq = Flipped(Decoupled(new MemRsIssue(b))) - val cmdResp = Decoupled(new MemRsComplete(b)) - val config = Decoupled(new MemConfigerIO(b)) - val hartid = Input(UInt(b.tile.xLen.W)) - // A backend bank is still clearing; MSET.clear completes only after it drains. - val clearBusy = Input(Bool()) - }) - - val idle :: config :: clearWait :: resp :: Nil = Enum(4) - - val state = RegInit(idle) - val transfer_reg = RegInit(false.B) - val source_bank_id_reg = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - val alloc_reg = RegInit(false.B) - val is_shared_reg = RegInit(false.B) - val col_reg = RegInit(0.U(b.memDomain.groupCountWidth.W)) - val clear_reg = RegInit(false.B) - val vbank_id_reg = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - val rob_id_reg = RegInit(0.U(rob_id_width.W)) - val is_sub_reg = RegInit(false.B) - val sub_rob_id_reg = RegInit(0.U(log2Up(b.frontend.sub_rob_depth * 4).W)) - val counter = RegInit(0.U(b.memDomain.groupCountWidth.W)) - - io.config.bits.is_multi := false.B - io.config.bits.is_shared := false.B - io.config.bits.transfer := false.B - io.config.bits.source_bank_id := 0.U - io.config.bits.alloc := false.B - io.config.bits.clear := false.B - io.config.bits.vbank_id := 0.U(b.memDomain.vbankIdWidth.W) - io.config.bits.group_id := 0.U - io.config.bits.hart_id := io.hartid - io.config.valid := false.B - io.cmdResp.valid := false.B - io.cmdResp.bits := 0.U.asTypeOf(io.cmdResp.bits) - io.cmdResp.bits.rob_id := 0.U(rob_id_width.W) - io.cmdResp.bits.is_sub := false.B - io.cmdResp.bits.sub_rob_id := 0.U - - io.cmdReq.ready := state === idle - - when(state === idle) { - when(io.cmdReq.valid) { - when(io.cmdReq.fire) { - val rawCol = io.cmdReq.bits.cmd.special(9, 5) - val alloc = io.cmdReq.bits.cmd.special(10) - val fullCol = - if (b.memDomain.sharedEnable) { - Mux( - io.cmdReq.bits.cmd.is_shared, - SharedMemLayout.totalBank(b).U(col_reg.getWidth.W), - b.memDomain.bankNum.U(col_reg.getWidth.W) - ) - } else { - b.memDomain.bankNum.U(col_reg.getWidth.W) - } - - state := config - col_reg := Mux( - io.cmdReq.bits.cmd.transfer, - 1.U, - Mux(alloc && rawCol === 0.U, fullCol, Mux(rawCol > 1.U, rawCol, 1.U)) - ) - transfer_reg := io.cmdReq.bits.cmd.transfer - source_bank_id_reg := io.cmdReq.bits.cmd.source_bank_id - alloc_reg := alloc - clear_reg := io.cmdReq.bits.cmd.clear - is_shared_reg := io.cmdReq.bits.cmd.is_shared - vbank_id_reg := io.cmdReq.bits.cmd.bank_id - rob_id_reg := io.cmdReq.bits.rob_id - is_sub_reg := io.cmdReq.bits.is_sub - sub_rob_id_reg := io.cmdReq.bits.sub_rob_id - assert( - !(io.cmdReq.bits.cmd.clear && io.cmdReq.bits.cmd.is_shared), - "MSET clear is currently supported for private banks only" - ) - } - } - - }.elsewhen(state === config) { - io.config.bits.is_multi := col_reg > 1.U - io.config.bits.is_shared := is_shared_reg - io.config.bits.transfer := transfer_reg - io.config.bits.source_bank_id := source_bank_id_reg - io.config.bits.alloc := alloc_reg - io.config.bits.clear := clear_reg && alloc_reg - io.config.bits.vbank_id := vbank_id_reg - io.config.bits.group_id := counter(b.memDomain.groupIdWidth - 1, 0) - io.config.valid := true.B - - when(io.config.fire) { - when(counter === col_reg - 1.U) { - counter := 0.U - state := Mux(clear_reg && alloc_reg, clearWait, resp) - }.otherwise { - counter := counter + 1.U - } - } - }.elsewhen(state === clearWait) { - // Each bound bank started clearing when its allocation fired. - when(!io.clearBusy)(state := resp) - }.elsewhen(state === resp) { - io.cmdResp.valid := true.B - io.cmdResp.bits.rob_id := rob_id_reg - io.cmdResp.bits.is_sub := is_sub_reg - io.cmdResp.bits.sub_rob_id := sub_rob_id_reg - - when(io.cmdResp.fire) { - state := idle - counter := 0.U - } - } -} diff --git a/arch/src/main/scala/framework/memdomain/frontend/mem/MemLoader.scala b/arch/src/main/scala/framework/memdomain/frontend/mem/MemLoader.scala deleted file mode 100644 index ab74d4abb..000000000 --- a/arch/src/main/scala/framework/memdomain/frontend/mem/MemLoader.scala +++ /dev/null @@ -1,303 +0,0 @@ -package framework.memdomain.frontend.mem - -import chisel3._ -import chisel3.util._ -import framework.memdomain.frontend.cmd.rs.{MemRsComplete, MemRsIssue} -import framework.memdomain.frontend.mem.dma.{BBReadRequest, BBReadResponse, DmaError, DmaStatus} -import freechips.rocketchip.rocket.MStatus -import framework.balldomain.blink.BankWrite -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.top.GlobalConfig - -@instantiable -class MemLoader(val b: GlobalConfig) extends Module { - require( - b.memDomain.bankWidth == 128 && b.memDomain.dma_buswidth == 128, - "MemLoader requires 128-bit bank and DMA beats" - ) - val rob_id_width = log2Up(b.frontend.rob_entries) - - @public - val io = IO(new Bundle { - val cmdReq = Flipped(Decoupled(new MemRsIssue(b))) - val cmdResp = Decoupled(new MemRsComplete(b)) - val footprint = Output(new Footprint(b)) - - val dmaReq = Decoupled(new BBReadRequest()) - val dmaResp = Flipped(Decoupled(new BBReadResponse(b.memDomain.bankWidth))) - - val bankWrite = Flipped(new BankWrite(b)) - - // MMIO routing hint: tells upper level (MemDomain) to route bankWrite to MmioPool - // When is_mvin_mmio_active is true, mmio_addr/col carry MMIO destination info - val is_mvin_mmio_active = Output(Bool()) - val mmio_addr = Output(UInt(17.W)) // rs2[55:39]: MMIO byte address - val mmio_col = Output(UInt(8.W)) // rs2[63:56]: valid bytes per row - - val query_valid = Output(Bool()) - val query_vbank_id = Output(UInt(b.memDomain.vbankIdWidth.W)) - val query_is_shared = Output(Bool()) - val query_group_count = Input(UInt(b.memDomain.groupCountWidth.W)) - - // Propagate decoded shared/private access intent. - val is_shared = Output(Bool()) - }) - - val s_idle :: s_setup :: s_query :: s_query_wait :: s_mul :: s_dma_req :: s_dma_wait :: s_wait_write_resp :: s_done :: Nil = - Enum(9) - val state = RegInit(s_idle) - - val rob_id_reg = RegInit(0.U(rob_id_width.W)) - val is_sub_reg = RegInit(false.B) - val sub_rob_id_reg = RegInit(0.U(log2Up(b.frontend.sub_rob_depth * 4).W)) - val mem_addr_reg = Reg(UInt(b.memDomain.memAddrLen.W)) - val iter_cmd = Reg(UInt(b.frontend.iter_len.W)) - val iter_reg = Reg(UInt(b.frontend.iter_len.W)) - val resp_count = RegInit(0.U(log2Up(16).W)) - val wr_bank_reg = Reg(UInt(b.memDomain.vbankIdWidth.W)) - val stride_reg = Reg(UInt(19.W)) - val is_shared_reg = RegInit(false.B) - val fault = RegInit(0.U.asTypeOf(new DmaStatus)) - val footprintValid = RegInit(false.B) - - // Group counter for multi-bank writes - val group_counter = RegInit(0.U(b.memDomain.groupCountWidth.W)) - val group_count_reg = RegInit(0.U(b.memDomain.groupCountWidth.W)) - val selected_group = RegInit(false.B) - val group_base = RegInit(0.U(5.W)) - val rowAddr = RegInit(0.U(16.W)) - - // MMIO routing info (latched at cmdReq.fire, exposed to upper level) - val is_mvin_mmio_reg = RegInit(false.B) - val is_mvin_2d_reg = RegInit(false.B) - val valid_bytes_reg = RegInit(16.U(5.W)) - val pixel_bytes_reg = RegInit(0.U(10.W)) - val source_width_reg = RegInit(0.U(10.W)) - val tile_width_reg = RegInit(0.U(4.W)) - val mmio_addr_reg = RegInit(0.U(17.W)) - val mmio_col_reg = RegInit(0.U(8.W)) - - io.is_mvin_mmio_active := is_mvin_mmio_reg - io.mmio_addr := mmio_addr_reg - io.mmio_col := mmio_col_reg - - val shapeRows = iter_cmd.pad(64) - val shapeColumns = Mux(is_mvin_2d_reg, tile_width_reg.pad(64), 1.U(64.W)) - val shapeSpan = - Mux(is_mvin_2d_reg, (b.memDomain.dma_buswidth / 8).U(64.W), (group_count_reg * (b.memDomain.bankWidth / 8).U).pad(64)) - val shapeColumnStride = Mux(is_mvin_2d_reg, pixel_bytes_reg.pad(64), 0.U(64.W)) - val linearRowStride = group_count_reg * stride_reg * (b.memDomain.bankWidth / 8).U - val shapeRowStride = Mux(is_mvin_2d_reg, (source_width_reg * pixel_bytes_reg).pad(64), linearRowStride.pad(64)) - val shapeBeats = iter_cmd * Mux(is_mvin_2d_reg, tile_width_reg.pad(group_count_reg.getWidth), group_count_reg) - val shapeEnd = mem_addr_reg.pad(64) +& ((shapeRows - 1.U) * shapeRowStride) +& - ((shapeColumns - 1.U) * shapeColumnStride) +& (shapeSpan - 1.U) - - val storageRows = Mux(is_shared_reg, b.memDomain.sharedBankEntries.U, b.memDomain.bankEntries.U) - val destinationRows = iter_cmd * Mux(is_mvin_2d_reg, tile_width_reg, 1.U) - val destinationStart = Reg(UInt(16.W)) - when(io.cmdReq.fire) { - destinationStart := Mux(io.cmdReq.bits.cmd.is_mvin_2d, io.cmdReq.bits.cmd.special(58, 53), 0.U) - } - - val shapeError = - (!is_mvin_mmio_reg && (destinationStart +& destinationRows) > storageRows) || iter_cmd === 0.U || group_count_reg === 0.U || stride_reg === 0.U || - (shapeBeats >> b.frontend.iter_len).orR || (shapeEnd >> b.memDomain.memAddrLen).orR || - (is_mvin_2d_reg && (pixel_bytes_reg === 0.U || source_width_reg < tile_width_reg)) - - io.footprint := 0.U.asTypeOf(new Footprint(b)) - io.footprint.valid := footprintValid - io.footprint.rob_id := rob_id_reg - io.footprint.is_sub := is_sub_reg - io.footprint.sub_rob_id := sub_rob_id_reg - io.footprint.baseVA := mem_addr_reg - io.footprint.rows := shapeRows - io.footprint.columns := shapeColumns - io.footprint.spanBytes := shapeSpan - io.footprint.columnStride := shapeColumnStride - io.footprint.rowStride := shapeRowStride - io.footprint.write := false.B - io.footprint.fault.error := Mux(shapeError, DmaError.Shape.U, DmaError.None.U) - io.footprint.fault.address := Mux(shapeError, mem_addr_reg.pad(64), 0.U) - - // ----------------------------- - // pending latch for 1-beat DMA -> bankWrite - // ----------------------------- - val pending = RegInit(false.B) - val latData = Reg(UInt(b.memDomain.bankWidth.W)) - val latLast = RegInit(false.B) - - // ----------------------------- - // defaults - // ----------------------------- - io.cmdReq.ready := (state === s_idle) - - io.dmaReq.valid := (state === s_dma_req) - io.dmaReq.bits.vaddr := mem_addr_reg - io.dmaReq.bits.len := iter_reg * (b.memDomain.bankWidth / 8).U - io.dmaReq.bits.status := 0.U.asTypeOf(new MStatus) - io.dmaReq.bits.stride := stride_reg - io.dmaReq.bits.groups := group_count_reg - io.dmaReq.bits.is_2d := is_mvin_2d_reg - io.dmaReq.bits.pixel_bytes := pixel_bytes_reg - io.dmaReq.bits.source_width := source_width_reg - io.dmaReq.bits.tile_width := tile_width_reg - - // only accept DMA beat when waiting AND no pending beat buffered - io.dmaResp.ready := (state === s_dma_wait) && !pending - - // bank write request driven from pending - io.bankWrite.io.req.valid := pending - io.bankWrite.io.req.bits.addr := rowAddr - - val validDataMask = VecInit( - (0 until b.memDomain.bankWidth / 8).map(i => Mux(i.U < valid_bytes_reg, "hff".U(8.W), 0.U(8.W))) - ).asUInt - - io.bankWrite.io.req.bits.data := Mux(is_mvin_2d_reg, latData & validDataMask, latData) - io.bankWrite.io.req.bits.mask := VecInit(Seq.fill(b.memDomain.bankMaskLen)(true.B)) - - // IMPORTANT: always ready for write response (avoid deadlock) - io.bankWrite.io.resp.ready := true.B - - io.bankWrite.rob_id := rob_id_reg - io.bankWrite.bank_id := wr_bank_reg - io.bankWrite.ball_id := 0.U - io.bankWrite.group_id := group_base + group_counter - io.is_shared := is_shared_reg - - // cmdResp (Decoupled): hold valid until accepted - io.cmdResp.valid := (state === s_done) - io.cmdResp.bits := 0.U.asTypeOf(new MemRsComplete(b)) - io.cmdResp.bits.rob_id := rob_id_reg - io.cmdResp.bits.is_sub := is_sub_reg - io.cmdResp.bits.sub_rob_id := sub_rob_id_reg - io.cmdResp.bits.fault := fault - - // ----------------------------- - // Receive load instruction (both mvin and mvin_mmio go through is_load path) - // ----------------------------- - when(io.cmdReq.fire && io.cmdReq.bits.cmd.is_load) { - rob_id_reg := io.cmdReq.bits.rob_id - is_sub_reg := io.cmdReq.bits.is_sub - sub_rob_id_reg := io.cmdReq.bits.sub_rob_id - mem_addr_reg := io.cmdReq.bits.cmd.mem_addr - wr_bank_reg := io.cmdReq.bits.cmd.bank_id - resp_count := 0.U - pending := false.B - fault := 0.U.asTypeOf(new DmaStatus) - footprintValid := io.cmdReq.bits.cmd.is_mvin_mmio || io.cmdReq.bits.cmd.is_mvin_2d - iter_cmd := io.cmdReq.bits.cmd.iter - latLast := false.B - group_counter := 0.U - rowAddr := Mux( - io.cmdReq.bits.cmd.is_mvin_2d, - io.cmdReq.bits.cmd.special(58, 53), - 0.U - ) - is_shared_reg := io.cmdReq.bits.cmd.is_shared - - val regular = !io.cmdReq.bits.cmd.is_mvin_mmio && !io.cmdReq.bits.cmd.is_mvin_2d - selected_group := regular && io.cmdReq.bits.cmd.special(63) - group_base := Mux(regular && io.cmdReq.bits.cmd.special(63), io.cmdReq.bits.cmd.special(62, 58), 0.U) - is_mvin_mmio_reg := io.cmdReq.bits.cmd.is_mvin_mmio - is_mvin_2d_reg := io.cmdReq.bits.cmd.is_mvin_2d - valid_bytes_reg := Mux(io.cmdReq.bits.cmd.special(62), 8.U(5.W), 16.U(5.W)) - pixel_bytes_reg := io.cmdReq.bits.cmd.special(42, 36) << 3 - source_width_reg := io.cmdReq.bits.cmd.special(52, 43) - tile_width_reg := io.cmdReq.bits.cmd.special(61, 59) +& 1.U - when(io.cmdReq.bits.cmd.is_mvin_mmio) { - mmio_addr_reg := io.cmdReq.bits.cmd.special(55, 39) - mmio_col_reg := io.cmdReq.bits.cmd.special(63, 56) - iter_reg := io.cmdReq.bits.cmd.iter - group_count_reg := 1.U - stride_reg := 1.U - state := s_dma_req - }.elsewhen(io.cmdReq.bits.cmd.is_mvin_2d) { - iter_reg := io.cmdReq.bits.cmd.iter * (io.cmdReq.bits.cmd.special(61, 59) +& 1.U) - group_count_reg := 1.U - stride_reg := 1.U - state := s_dma_req - }.otherwise { - stride_reg := io.cmdReq.bits.cmd.special(57, 39) - iter_cmd := io.cmdReq.bits.cmd.iter - state := s_setup - } - } - - io.query_valid := state === s_setup || state === s_query || state === s_query_wait - io.query_vbank_id := wr_bank_reg - io.query_is_shared := is_shared_reg && io.query_valid - - when(state === s_setup) { - state := s_query - } - - when(state === s_query) { - state := s_query_wait - } - - when(state === s_query_wait) { - assert(io.query_group_count >= 1.U, "MemLoader groups must be >= 1") - when(selected_group) { - assert(group_base < io.query_group_count, "MemLoader selected group is outside bank allocation") - } - group_count_reg := Mux(selected_group, 1.U, io.query_group_count) - state := s_mul - } - - when(state === s_mul) { - iter_reg := iter_cmd * group_count_reg - state := s_dma_req - footprintValid := true.B - } - - // DMA req accepted - when(io.dmaReq.fire) { - state := s_dma_wait - resp_count := 0.U - } - - // Latch DMA beat into pending buffer - when(io.dmaResp.fire) { - when(io.dmaResp.bits.fault.error =/= DmaError.None.U || fault.error =/= DmaError.None.U) { - when(fault.error === DmaError.None.U)(fault := io.dmaResp.bits.fault) - when(io.dmaResp.bits.last)(state := s_done) - }.otherwise { - pending := true.B - latData := io.dmaResp.bits.data - latLast := io.dmaResp.bits.last - } - } - - // When bankWrite request handshakes, consume pending beat - when(io.bankWrite.io.req.fire) { - pending := false.B - resp_count := resp_count + 1.U - - when(latLast) { - state := s_wait_write_resp - }.otherwise { - state := s_wait_write_resp - } - } - - // Wait for each write response before accepting the next DMA beat. - when(state === s_wait_write_resp && io.bankWrite.io.resp.fire) { - when(!io.bankWrite.io.resp.bits.ok && fault.error === DmaError.None.U) { - fault.error := DmaError.Bank.U - fault.address := rowAddr * (b.memDomain.bankWidth / 8).U - } - when(group_counter + 1.U < group_count_reg) { - group_counter := group_counter + 1.U - }.otherwise { - group_counter := 0.U - rowAddr := rowAddr + 1.U - } - state := Mux(latLast, s_done, s_dma_wait) - } - - when(state === s_done && io.cmdResp.fire) { - state := s_idle - footprintValid := false.B - } -} diff --git a/arch/src/main/scala/framework/memdomain/frontend/mem/MemStorer.scala b/arch/src/main/scala/framework/memdomain/frontend/mem/MemStorer.scala deleted file mode 100644 index 18ad106b2..000000000 --- a/arch/src/main/scala/framework/memdomain/frontend/mem/MemStorer.scala +++ /dev/null @@ -1,212 +0,0 @@ -package framework.memdomain.frontend.mem - -import memcore.memory.queue.Queue - -import chisel3._ -import chisel3.util._ -import framework.top.GlobalConfig -import framework.memdomain.frontend.cmd.rs.{MemRsComplete, MemRsIssue} -import framework.memdomain.frontend.mem.dma.{BBWriteCommand, BBWriteData, BBWriteResponse, DmaError, DmaStatus} -import framework.balldomain.blink.BankRead -import chisel3.experimental.hierarchy.{instantiable, public} - -@instantiable -class MemStorer(val b: GlobalConfig) extends Module { - require( - b.memDomain.bankWidth == 128 && b.memDomain.dma_buswidth == 128, - "MemStorer requires 128-bit bank and DMA beats" - ) - val lineBytes = b.memDomain.bankWidth / 8 - val lgLine = log2Ceil(lineBytes) - val robBits = log2Up(b.frontend.rob_entries) - - @public - val io = IO(new Bundle { - val cmdReq = Flipped(Decoupled(new MemRsIssue(b))) - val cmdResp = Decoupled(new MemRsComplete(b)) - val footprint = Output(new Footprint(b)) - val dmaReq = Decoupled(new BBWriteCommand) - val dmaData = Decoupled(new BBWriteData(b.memDomain.bankWidth)) - val dmaResp = Flipped(Decoupled(new BBWriteResponse)) - val bankRead = Flipped(new BankRead(b)) - val query_valid = Output(Bool()) - val query_vbank_id = Output(UInt(b.memDomain.vbankIdWidth.W)) - val query_is_shared = Output(Bool()) - val query_group_count = Input(UInt(b.memDomain.groupCountWidth.W)) - val is_shared = Output(Bool()) - }) - - val idle :: setup :: query :: queryWait :: prepare :: descriptor :: stream :: waitResponse :: drain :: done :: Nil = - Enum(10) - val state = RegInit(idle) - val rob = RegInit(0.U(robBits.W)) - val isSub = RegInit(false.B) - val subRob = RegInit(0.U(log2Up(b.frontend.sub_rob_depth * 4).W)) - val iterations = RegInit(0.U(b.frontend.iter_len.W)) - val bank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - val stride = RegInit(1.U(19.W)) - val groups = RegInit(1.U(b.memDomain.groupCountWidth.W)) - val shared = RegInit(false.B) - val baseVA = RegInit(0.U(64.W)) - val cursor = RegInit(0.U(64.W)) - val fault = RegInit(0.U.asTypeOf(new DmaStatus)) - val footprintValid = RegInit(false.B) - val readRow = RegInit(0.U(b.frontend.iter_len.W)) - val readGroup = RegInit(0.U(b.memDomain.groupCountWidth.W)) - val readsFinished = RegInit(false.B) - val finalDescriptor = RegInit(false.B) - - val selectedGroup = RegInit(false.B) - val groupBase = RegInit(0.U(5.W)) - - // Each accepted bank read reserves a data slot until its beat is sent. - // Ordered last tags include requests whose synchronous response has not yet arrived. - val lastTags = Module(new Queue(Bool(), 2, flow = true)) - val data = Module(new Queue(new BBWriteData(b.memDomain.bankWidth), 2, pipe = true)) - val reserved = lastTags.io.count +& data.io.count - val empty = lastTags.io.count === 0.U && data.io.count === 0.U - - val span = groups * lineBytes.U - val rowStride = groups * stride * lineBytes.U - val totalBytes = iterations * span - val descriptorBytes = Mux(stride === 1.U, totalBytes, span) - val shapeEnd = baseVA +& ((iterations - 1.U) * rowStride) +& (span - 1.U) - - val strideOverflow = - if (rowStride.getWidth > b.memDomain.memAddrLen) - (rowStride >> b.memDomain.memAddrLen).orR - else false.B - - val lengthOverflow = if (descriptorBytes.getWidth > 32) (descriptorBytes >> 32).orR else false.B - val storageRows = Mux(shared, b.memDomain.sharedBankEntries.U, b.memDomain.bankEntries.U) - - val shapeError = iterations === 0.U || iterations > storageRows || - groups === 0.U || groups > (BigInt(1) << b.memDomain.groupIdWidth).U || stride === 0.U || - (shapeEnd >> b.memDomain.memAddrLen).orR || strideOverflow || lengthOverflow - - io.cmdReq.ready := state === idle - when(io.cmdReq.fire) { - assert(io.cmdReq.bits.cmd.is_store, "MemStorer received a non-store command") - assert(empty, "MemStorer reused a command with pending bank reads or data") - rob := io.cmdReq.bits.rob_id - isSub := io.cmdReq.bits.is_sub - subRob := io.cmdReq.bits.sub_rob_id - iterations := io.cmdReq.bits.cmd.iter - bank := io.cmdReq.bits.cmd.bank_id - selectedGroup := io.cmdReq.bits.cmd.special(63) - groupBase := Mux(io.cmdReq.bits.cmd.special(63), io.cmdReq.bits.cmd.special(62, 58), 0.U) - stride := io.cmdReq.bits.cmd.special(57, 39) - shared := io.cmdReq.bits.cmd.is_shared - baseVA := io.cmdReq.bits.cmd.mem_addr - cursor := io.cmdReq.bits.cmd.mem_addr - readRow := 0.U - readGroup := 0.U - readsFinished := false.B - fault := 0.U.asTypeOf(new DmaStatus) - footprintValid := false.B - state := setup - } - io.query_valid := state === setup || state === query || state === queryWait - io.query_vbank_id := bank - io.query_is_shared := shared && io.query_valid - when(state === setup)(state := query) - when(state === query)(state := queryWait) - when(state === queryWait) { - when(selectedGroup) { - assert(groupBase < io.query_group_count, "MemStorer selected group is outside bank allocation") - } - groups := Mux(selectedGroup, 1.U, io.query_group_count) - state := prepare - } - when(state === prepare) { - footprintValid := true.B - when(shapeError) { - fault.error := DmaError.Shape.U - fault.address := baseVA - state := drain - }.otherwise(state := descriptor) - } - - io.footprint := 0.U.asTypeOf(new Footprint(b)) - io.footprint.valid := footprintValid - io.footprint.rob_id := rob - io.footprint.is_sub := isSub - io.footprint.sub_rob_id := subRob - io.footprint.baseVA := baseVA - io.footprint.rows := iterations - io.footprint.columns := 1.U - io.footprint.spanBytes := span - io.footprint.columnStride := 0.U - io.footprint.rowStride := rowStride - io.footprint.write := true.B - io.footprint.fault.error := Mux(shapeError, DmaError.Shape.U, DmaError.None.U) - io.footprint.fault.address := Mux(shapeError, baseVA, 0.U) - - io.dmaReq.valid := state === descriptor - io.dmaReq.bits.vaddr := cursor - io.dmaReq.bits.len := descriptorBytes - when(io.dmaReq.fire) { - assert(empty, "MemStorer started a descriptor with pending bank reads or data") - finalDescriptor := stride === 1.U || readRow === iterations - 1.U - readsFinished := false.B - state := stream - } - - val lastGroup = readGroup === groups - 1.U - val lastRow = readRow === iterations - 1.U - val lastRead = lastGroup && (stride =/= 1.U || lastRow) - io.bankRead.rob_id := rob - io.bankRead.bank_id := bank - io.bankRead.ball_id := 0.U - io.bankRead.group_id := groupBase + readGroup - io.bankRead.io.req.bits.addr := readRow - io.is_shared := shared - io.bankRead.io.req.valid := state === stream && !readsFinished && - (reserved < 2.U || io.dmaData.fire) - lastTags.io.enq.valid := io.bankRead.io.req.fire - lastTags.io.enq.bits := lastRead - when(io.bankRead.io.req.fire) { - assert(lastTags.io.enq.ready, "MemStorer bank read has no reserved last-tag slot") - when(lastGroup) { - readGroup := 0.U - when(!lastRow)(readRow := readRow + 1.U) - }.otherwise(readGroup := readGroup + 1.U) - when(lastRead)(readsFinished := true.B) - } - - data.io.enq.valid := io.bankRead.io.resp.valid && lastTags.io.deq.valid - data.io.enq.bits.data := io.bankRead.io.resp.bits.data - data.io.enq.bits.last := lastTags.io.deq.bits - io.bankRead.io.resp.ready := data.io.enq.ready && lastTags.io.deq.valid - lastTags.io.deq.ready := io.bankRead.io.resp.fire - when(io.bankRead.io.resp.valid) { - assert(lastTags.io.deq.valid, "MemStorer received an unreserved bank response") - assert(data.io.enq.ready, "MemStorer reserved bank response exceeded its data buffer") - } - assert(reserved <= 2.U, "MemStorer exceeded its two reserved data slots") - - io.dmaData.valid := state === stream && data.io.deq.valid - io.dmaData.bits := data.io.deq.bits - data.io.deq.ready := state === drain || (state === stream && io.dmaData.ready) - when(io.dmaData.fire && io.dmaData.bits.last)(state := waitResponse) - - // WriteDma consumes the complete descriptor even after an AXI/map error, and - // returns exactly one terminal response. Do not advance a strided row before it. - io.dmaResp.ready := state === waitResponse - when(io.dmaResp.fire) { - when(io.dmaResp.bits.fault.error =/= DmaError.None.U || !io.dmaResp.bits.done) { - when(io.dmaResp.bits.fault.error =/= DmaError.None.U)(fault := io.dmaResp.bits.fault) - .otherwise { fault.error := DmaError.Protocol.U; fault.address := cursor } - state := drain - }.elsewhen(finalDescriptor)(state := drain) - .otherwise { cursor := cursor + rowStride; state := descriptor } - } - when(state === drain && empty)(state := done) - io.cmdResp.valid := state === done - io.cmdResp.bits := 0.U.asTypeOf(new MemRsComplete(b)) - io.cmdResp.bits.rob_id := rob - io.cmdResp.bits.is_sub := isSub - io.cmdResp.bits.sub_rob_id := subRob - io.cmdResp.bits.fault := fault - when(io.cmdResp.fire) { footprintValid := false.B; state := idle } -} diff --git a/arch/src/main/scala/framework/memdomain/frontend/mem/dma/Bundles.scala b/arch/src/main/scala/framework/memdomain/frontend/mem/dma/Bundles.scala deleted file mode 100644 index 261beaf26..000000000 --- a/arch/src/main/scala/framework/memdomain/frontend/mem/dma/Bundles.scala +++ /dev/null @@ -1,38 +0,0 @@ -package framework.memdomain.frontend.mem.dma - -import chisel3._ -import freechips.rocketchip.rocket.MStatus - -class BBReadRequest extends Bundle { - val vaddr = UInt(64.W) - val len = UInt(32.W) - val status = new MStatus - val stride = UInt(19.W) - val groups = UInt(6.W) - val is_2d = Bool() - val pixel_bytes = UInt(10.W) - val source_width = UInt(10.W) - val tile_width = UInt(4.W) -} - -class BBReadResponse(dataWidth: Int) extends Bundle { - val data = UInt(dataWidth.W) - val last = Bool() - val addrcounter = UInt(16.W) - val fault = new DmaStatus -} - -class BBWriteCommand extends Bundle { - val vaddr = UInt(64.W) - val len = UInt(32.W) -} - -class BBWriteData(dataWidth: Int) extends Bundle { - val data = UInt(dataWidth.W) - val last = Bool() -} - -class BBWriteResponse extends Bundle { - val done = Bool() - val fault = new DmaStatus -} diff --git a/arch/src/main/scala/framework/memdomain/frontend/mem/dma/Dma.scala b/arch/src/main/scala/framework/memdomain/frontend/mem/dma/Dma.scala deleted file mode 100644 index 4c78f78e4..000000000 --- a/arch/src/main/scala/framework/memdomain/frontend/mem/dma/Dma.scala +++ /dev/null @@ -1,69 +0,0 @@ -package framework.memdomain.frontend.mem.dma - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.top.GlobalConfig -import memcore.memory.preflight.{MapQuery, MapResult, Params} -import memcore.bus.axi4 - -class DmaDecision(p: Params) extends Bundle { - val valid = Bool() - val parentId = UInt(p.idBits.W) - val fault = new DmaStatus -} - -@instantiable -class Dma(b: GlobalConfig, prepared: Params, axiParams: axi4.Params) extends Module { - - @public - val io = IO(new Bundle { - val dma = Flipped(new DmaPort(b.memDomain.dma_buswidth)) - // One decision for each physical producer: loader, storer, kernel image. - val decisions = Input(Vec(3, new DmaDecision(prepared))) - val queries = Output(Vec(2, new MapQuery(prepared))) - val mappings = Input(Vec(2, new MapResult(prepared))) - val axi = new axi4.Port(axiParams) - }) - - val reader: Instance[ReadDma] = Instantiate(new ReadDma(b, prepared, axiParams)) - val writer: Instance[WriteDma] = Instantiate(new WriteDma(b, prepared, axiParams)) - val kernel = io.dma.read.bits.producer === 2.U - val readDecision = Mux(kernel, io.decisions(2), io.decisions(0)) - when(io.dma.read.valid) { - assert(io.dma.read.bits.producer === 0.U || kernel, "DMA read has an unknown producer") - } - reader.io.req.valid := io.dma.read.valid - reader.io.req.bits := io.dma.read.bits.transfer - io.dma.read.ready := reader.io.req.ready - io.dma.readResult <> reader.io.resp - reader.io.parentId := readDecision.parentId - reader.io.decisionValid := readDecision.valid - reader.io.decisionFault := readDecision.fault - writer.io.req <> io.dma.write - writer.io.data <> io.dma.writeData - io.dma.writeResult <> writer.io.resp - writer.io.parentId := io.decisions(1).parentId - writer.io.decisionValid := io.decisions(1).valid - writer.io.decisionFault := io.decisions(1).fault - io.dma.readBusy := reader.io.busy - io.dma.writeBusy := writer.io.busy - io.queries(0) := reader.io.query - io.queries(1) := writer.io.query - reader.io.mapping := io.mappings(0) - writer.io.mapping := io.mappings(1) - - // AXI read and write channels retain independent command owners. - io.axi.ar <> reader.io.axi.ar - reader.io.axi.r <> io.axi.r - io.axi.aw <> writer.io.axi.aw - io.axi.w <> writer.io.axi.w - writer.io.axi.b <> io.axi.b - reader.io.axi.aw.ready := false.B - reader.io.axi.w.ready := false.B - reader.io.axi.b.valid := false.B - reader.io.axi.b.bits := 0.U.asTypeOf(new axi4.WriteResponse(axiParams)) - writer.io.axi.ar.ready := false.B - writer.io.axi.r.valid := false.B - writer.io.axi.r.bits := 0.U.asTypeOf(new axi4.ReadData(axiParams)) -} diff --git a/arch/src/main/scala/framework/memdomain/frontend/mem/dma/DmaPort.scala b/arch/src/main/scala/framework/memdomain/frontend/mem/dma/DmaPort.scala deleted file mode 100644 index bafde2c10..000000000 --- a/arch/src/main/scala/framework/memdomain/frontend/mem/dma/DmaPort.scala +++ /dev/null @@ -1,21 +0,0 @@ -package framework.memdomain.frontend.mem.dma - -import chisel3._ -import chisel3.util._ - -class DmaReadCommand extends Bundle { - val transfer = new BBReadRequest - // Indices of MemFrontend's physical footprints: loader=0, storer=1, kernel image=2. - val producer = UInt(2.W) -} - -/** Accelerator-side logical transfers. Translation and external-memory transport belong to the system. */ -class DmaPort(dataBits: Int) extends Bundle { - val read = Decoupled(new DmaReadCommand) - val readResult = Flipped(Decoupled(new BBReadResponse(dataBits))) - val write = Decoupled(new BBWriteCommand) - val writeData = Decoupled(new BBWriteData(dataBits)) - val writeResult = Flipped(Decoupled(new BBWriteResponse)) - val readBusy = Input(Bool()) - val writeBusy = Input(Bool()) -} diff --git a/arch/src/main/scala/framework/memdomain/frontend/mem/dma/ReadDma.scala b/arch/src/main/scala/framework/memdomain/frontend/mem/dma/ReadDma.scala deleted file mode 100644 index 6974395ff..000000000 --- a/arch/src/main/scala/framework/memdomain/frontend/mem/dma/ReadDma.scala +++ /dev/null @@ -1,220 +0,0 @@ -package framework.memdomain.frontend.mem.dma - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.top.GlobalConfig -import memcore.memory.preflight.{Error, MapQuery, MapResult, Params} -import memcore.bus.axi4 - -@instantiable -class ReadDma(b: GlobalConfig, prepared: Params, axiParams: axi4.Params) extends Module { - require(b.memDomain.bankWidth == 128 && b.memDomain.dma_buswidth == 128 && axiParams.dataBits == 128) - require(prepared.beatBytes == 16 && prepared.bus.addressBits == axiParams.addressBits) - - @public - val io = IO(new Bundle { - val req = Flipped(Decoupled(new BBReadRequest)) - val resp = Decoupled(new BBReadResponse(128)) - val parentId = Input(UInt(prepared.idBits.W)) - val decisionValid = Input(Bool()) - val decisionFault = Input(new DmaStatus) - val busy = Output(Bool()) - val query = Output(new MapQuery(prepared)) - val mapping = Input(new MapResult(prepared)) - val axi = new axi4.Port(axiParams) - }) - - val idle :: shape :: divide :: products :: offset :: check :: plan :: query :: offer :: receive :: respond :: drain :: failed :: Nil = - Enum(13) - val state = RegInit(idle) - val command = Reg(new BBReadRequest) - val parent = Reg(UInt(prepared.idBits.W)) - val total = Reg(UInt(17.W)) - val progress = RegInit(0.U(17.W)) - val group = RegInit(0.U(6.W)) - val column = RegInit(0.U(4.W)) - val address = Reg(UInt(64.W)) - val queryVA = Reg(UInt(64.W)) - val physical = Reg(UInt(axiParams.addressBits.W)) - val candidate = Reg(UInt(9.W)) - val remaining = Reg(UInt(9.W)) - val second = RegInit(false.B) - val firstPart = Reg(UInt(128.W)) - val result = Reg(UInt(128.W)) - val fault = RegInit(0.U.asTypeOf(new DmaStatus)) - - io.req.ready := state === idle && io.decisionValid - io.busy := state =/= idle - io.resp.valid := state === respond || state === failed - io.resp.bits.data := Mux(state === failed, 0.U, result) - io.resp.bits.last := state === failed || progress +& 1.U === total - io.resp.bits.addrcounter := progress(15, 0) - io.resp.bits.fault := Mux(state === failed, fault, 0.U.asTypeOf(new DmaStatus)) - io.query := 0.U.asTypeOf(new MapQuery(prepared)) - io.query.valid := state === query - io.query.id := parent - io.query.va := queryVA - io.query.bytes := candidate << 4 - io.query.write := false.B - io.axi.ar.valid := state === offer - io.axi.ar.bits := 0.U.asTypeOf(new axi4.Address(axiParams)) - io.axi.ar.bits.addr := physical - io.axi.ar.bits.len := candidate - 1.U - io.axi.ar.bits.size := 4.U - io.axi.ar.bits.burst := 1.U - io.axi.r.ready := state === receive || state === drain - io.axi.aw.valid := false.B - io.axi.aw.bits := 0.U.asTypeOf(new axi4.Address(axiParams)) - io.axi.w.valid := false.B - io.axi.w.bits := 0.U.asTypeOf(new axi4.WriteData(axiParams)) - io.axi.b.ready := false.B - - def fail(error: UInt, at: UInt): Unit = { - fault.error := error - fault.address := at - state := failed - } - - val count = (io.req.bits.len.pad(33) + 15.U) >> 4 - when(io.req.fire) { - command := io.req.bits; parent := io.parentId - address := io.req.bits.vaddr; progress := 0.U; group := 0.U; column := 0.U; second := false.B - fault := 0.U.asTypeOf(new DmaStatus) - val bad = io.req.bits.len === 0.U || count > 65536.U || - io.req.bits.groups === 0.U || io.req.bits.stride === 0.U || - (io.req.bits.is_2d && (io.req.bits.pixel_bytes === 0.U || io.req.bits.tile_width === 0.U || - io.req.bits.source_width < io.req.bits.tile_width)) - when(io.decisionFault.error =/= DmaError.None.U) { - fail(io.decisionFault.error, io.decisionFault.address) - }.elsewhen(bad)(fail(DmaError.Shape.U, io.req.bits.vaddr)).otherwise { - total := count(16, 0); state := shape - } - } - - // At most 65536 beats: divide the 16-bit last index without a combinational divider. - val quotient = Reg(UInt(16.W)) - val remainder = Reg(UInt(6.W)) - val divisor = Reg(UInt(6.W)) - val divideLeft = Reg(UInt(5.W)) - val rowStride = Reg(UInt(29.W)) - val columnStride = Reg(UInt(10.W)) - val rowOffset = Reg(UInt(45.W)) - val columnOffset = Reg(UInt(16.W)) - val lastOffset = Reg(UInt(46.W)) - val shapeDivisor = Mux(command.is_2d, command.tile_width.pad(6), command.groups) - when(state === shape) { - quotient := (total - 1.U)(15, 0) - remainder := 0.U - divisor := shapeDivisor - divideLeft := 16.U - rowStride := Mux(command.is_2d, command.source_width * command.pixel_bytes, (command.groups * command.stride) << 4) - columnStride := Mux(command.is_2d, command.pixel_bytes, 16.U) - state := Mux(shapeDivisor === 1.U, products, divide) - } - val shiftedRemainder = Cat(remainder, quotient(15)) - val subtractDivisor = shiftedRemainder >= divisor - when(state === divide) { - quotient := Cat(quotient(14, 0), subtractDivisor) - remainder := Mux(subtractDivisor, shiftedRemainder - divisor, shiftedRemainder)(5, 0) - divideLeft := divideLeft - 1.U - when(divideLeft === 1.U)(state := products) - } - when(state === products) { - rowOffset := quotient * rowStride - columnOffset := remainder * columnStride - state := offset - } - when(state === offset) { - lastOffset := rowOffset +& columnOffset +& 15.U - state := check - } - val lastByte = command.vaddr +& lastOffset - when(state === check) { - when(lastByte(64))(fail(DmaError.Shape.U, command.vaddr)).otherwise(state := plan) - } - - val contiguous = Mux( - command.is_2d, - Mux( - command.pixel_bytes =/= 16.U, - 1.U, - Mux(command.source_width === command.tile_width, total - progress, command.tile_width - column) - ), - Mux(command.stride === 1.U, total - progress, command.groups - group) - ) - - val pageBeats = (4096.U(13.W) - address(11, 0)) >> 4 - val commandBeats = Mux(contiguous < (total - progress), contiguous, total - progress) - val limited = Mux(commandBeats < 256.U, commandBeats, 256.U) - when(state === plan) { - queryVA := address & ~15.U(64.W) - candidate := Mux(address(3, 0) =/= 0.U, 1.U, Mux(limited < pageBeats, limited, pageBeats)) - state := query - } - when(state === query) { - when(!io.mapping.hit || io.mapping.error =/= Error.Ok.U) { - val error = MuxLookup(io.mapping.error, DmaError.AccessFault.U)(Seq( - Error.Shape.U -> DmaError.Shape.U, - Error.Overflow.U -> DmaError.Shape.U, - Error.PageFault.U -> DmaError.PageFault.U, - Error.Capacity.U -> DmaError.Capacity.U, - Error.Context.U -> DmaError.Context.U - )) - fail(error, queryVA) - }.elsewhen(io.mapping.pa(3, 0) =/= 0.U || (io.mapping.pa(11, 0) +& (candidate << 4)) > 4096.U) { - fail(DmaError.AccessFault.U, queryVA) - }.otherwise { physical := io.mapping.pa; state := offer } - } - when(io.axi.ar.fire) { remaining := candidate; state := receive } - when(io.axi.r.fire) { - when(state === drain) { - when(io.axi.r.bits.last)(state := failed) - }.otherwise { - val badId = io.axi.r.bits.id =/= 0.U - val badLast = io.axi.r.bits.last =/= (remaining === 1.U) - val badResponse = io.axi.r.bits.resp =/= 0.U - assert(!badLast, "ReadDma AXI RLAST does not match ARLEN") - when(badId || badLast || badResponse) { - fault.error := Mux(badId || badLast || io.axi.r.bits.resp === 1.U, DmaError.Protocol.U, DmaError.AccessFault.U) - fault.address := physical - state := Mux(badLast || io.axi.r.bits.last, failed, drain) - }.otherwise { - remaining := remaining - 1.U - physical := physical + 16.U - when(address(3, 0) =/= 0.U && !second) { - firstPart := (io.axi.r.bits.data >> (address(3, 0) * 8.U))(127, 0) - second := true.B; queryVA := queryVA + 16.U; candidate := 1.U; state := query - }.otherwise { - result := Mux( - second, - firstPart | (io.axi.r.bits.data << ((16.U - address(3, 0)) * 8.U))(127, 0), - io.axi.r.bits.data - ) - state := respond - } - } - } - } - when(io.resp.fire) { - when(state === failed || progress +& 1.U === total)(state := idle).otherwise { - val next = Mux( - command.is_2d, - Mux( - column +& 1.U === command.tile_width, - address + (command.source_width - command.tile_width +& 1.U) * command.pixel_bytes, - address + command.pixel_bytes - ), - Mux( - group +& 1.U === command.groups, - address + 16.U + command.groups * (command.stride - 1.U) * 16.U, - address + 16.U - ) - ) - when(command.is_2d)(column := Mux(column +& 1.U === command.tile_width, 0.U, column + 1.U)) - .otherwise(group := Mux(group +& 1.U === command.groups, 0.U, group + 1.U)) - address := next; progress := progress + 1.U; second := false.B - state := Mux(remaining =/= 0.U, receive, plan) - } - } -} diff --git a/arch/src/main/scala/framework/memdomain/frontend/mem/dma/WriteDma.scala b/arch/src/main/scala/framework/memdomain/frontend/mem/dma/WriteDma.scala deleted file mode 100644 index 159948db2..000000000 --- a/arch/src/main/scala/framework/memdomain/frontend/mem/dma/WriteDma.scala +++ /dev/null @@ -1,203 +0,0 @@ -package framework.memdomain.frontend.mem.dma - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.top.GlobalConfig -import memcore.memory.preflight.{Error, MapQuery, MapResult, Params} -import memcore.bus.axi4 - -@instantiable -class WriteDma(b: GlobalConfig, prepared: Params, axiParams: axi4.Params) extends Module { - require(b.memDomain.bankWidth == 128 && b.memDomain.dma_buswidth == 128 && axiParams.dataBits == 128) - require(prepared.beatBytes == 16 && prepared.bus.addressBits == axiParams.addressBits) - - @public - val io = IO(new Bundle { - val req = Flipped(Decoupled(new BBWriteCommand)) - val data = Flipped(Decoupled(new BBWriteData(128))) - val resp = Decoupled(new BBWriteResponse) - val parentId = Input(UInt(prepared.idBits.W)) - val decisionValid = Input(Bool()) - val decisionFault = Input(new DmaStatus) - val busy = Output(Bool()) - val query = Output(new MapQuery(prepared)) - val mapping = Input(new MapResult(prepared)) - val axi = new axi4.Port(axiParams) - }) - - val idle :: plan :: query :: stream :: waitB :: drain :: respond :: Nil = Enum(7) - val state = RegInit(idle) - val command = Reg(new BBWriteCommand) - val parent = Reg(UInt(prepared.idBits.W)) - val endVA = Reg(UInt(64.W)) - val cursorVA = Reg(UInt(64.W)) - val inputTotal = Reg(UInt(29.W)) - val consumed = RegInit(0.U(29.W)) - val sourceEnded = RegInit(false.B) - val remaining = Reg(UInt(29.W)) - val firstBeat = RegInit(true.B) - val beats = Reg(UInt(9.W)) - val beatIndex = RegInit(0.U(9.W)) - val awSent = RegInit(false.B) - val wDone = RegInit(false.B) - val queryVA = Reg(UInt(64.W)) - val queryBytes = Reg(UInt(13.W)) - val headOffset = Reg(UInt(4.W)) - val physical = Reg(UInt(axiParams.addressBits.W)) - val carry = RegInit(0.U(128.W)) - val outputValid = RegInit(false.B) - val outputData = Reg(UInt(128.W)) - val outputMask = Reg(UInt(16.W)) - val outputLast = Reg(Bool()) - val fault = RegInit(0.U.asTypeOf(new DmaStatus)) - val offset = command.vaddr(3, 0) - val expectedLast = consumed +& 1.U === inputTotal - val outputRoom = !outputValid || (io.axi.w.ready && !outputLast) - val outputIndex = beatIndex + io.axi.w.fire.asUInt - val outputHead = firstBeat && !io.axi.w.fire - - io.req.ready := state === idle && io.decisionValid - io.busy := state =/= idle - io.data.ready := (state === stream && !wDone && outputRoom && fault.error === DmaError.None.U && consumed < inputTotal) || - (state === drain && !sourceEnded && consumed < inputTotal) - io.resp.valid := state === respond - io.resp.bits.done := fault.error === DmaError.None.U - io.resp.bits.fault := fault - io.query := 0.U.asTypeOf(new MapQuery(prepared)) - io.query.valid := state === query - io.query.id := parent - io.query.va := queryVA - io.query.bytes := queryBytes - io.query.write := true.B - io.axi.aw.valid := state === stream && !awSent - io.axi.aw.bits := 0.U.asTypeOf(new axi4.Address(axiParams)) - io.axi.aw.bits.id := 1.U - io.axi.aw.bits.addr := physical - io.axi.aw.bits.len := beats - 1.U - io.axi.aw.bits.size := 4.U - io.axi.aw.bits.burst := 1.U - io.axi.w.valid := state === stream && outputValid - io.axi.w.bits.data := outputData - io.axi.w.bits.strb := outputMask - io.axi.w.bits.last := outputLast - io.axi.b.ready := state === waitB - io.axi.ar.valid := false.B - io.axi.ar.bits := 0.U.asTypeOf(new axi4.Address(axiParams)) - io.axi.r.ready := false.B - - def terminate(error: UInt, at: UInt): Unit = { - fault.error := error - fault.address := at - state := Mux(sourceEnded || consumed === inputTotal, respond, drain) - } - - when(io.req.fire) { - command := io.req.bits; parent := io.parentId - consumed := 0.U; sourceEnded := false.B; firstBeat := true.B - carry := 0.U; outputValid := false.B; fault := 0.U.asTypeOf(new DmaStatus) - val finalByte = io.req.bits.vaddr +& (io.req.bits.len - 1.U) - val malformed = io.req.bits.len === 0.U || io.req.bits.len(3, 0) =/= 0.U || finalByte(64) - inputTotal := io.req.bits.len >> 4 - remaining := (io.req.bits.len >> 4) +& (io.req.bits.vaddr(3, 0) =/= 0.U).asUInt - cursorVA := io.req.bits.vaddr & ~15.U(64.W) - endVA := finalByte(63, 0) - when(malformed) { - fault.error := DmaError.Shape.U; fault.address := io.req.bits.vaddr; state := respond - }.elsewhen(io.decisionFault.error =/= DmaError.None.U) { - fault := io.decisionFault; state := drain - }.otherwise(state := plan) - } - - when(state === plan) { - val pageBeats = (4096.U(13.W) - cursorVA(11, 0)) >> 4 - val capped = Mux(remaining < 256.U, remaining, 256.U) - val extent = Mux(capped < pageBeats, capped, pageBeats) - val head = Mux(firstBeat, offset, 0.U) - val start = cursorVA + head - val covered = (extent << 4) - head - val commandBytes = endVA - start + 1.U - beats := extent; beatIndex := 0.U; awSent := false.B; wDone := false.B - headOffset := head; queryVA := start - queryBytes := Mux(commandBytes < covered, commandBytes, covered) - state := query - } - when(state === query) { - val base = io.mapping.pa.pad(axiParams.addressBits + 1) - headOffset - val lastPA = base +& ((beats << 4) - 1.U) - val invalid = !io.mapping.hit || io.mapping.error =/= Error.Ok.U || - base(axiParams.addressBits) || base(3, 0) =/= 0.U || - lastPA(lastPA.getWidth - 1, axiParams.addressBits).orR || - (base >> 12) =/= (lastPA >> 12) - when(invalid) { - val error = MuxLookup(io.mapping.error, DmaError.AccessFault.U)(Seq( - Error.Shape.U -> DmaError.Shape.U, - Error.Overflow.U -> DmaError.Shape.U, - Error.PageFault.U -> DmaError.PageFault.U, - Error.Capacity.U -> DmaError.Capacity.U, - Error.Context.U -> DmaError.Context.U - )) - terminate(error, queryVA) - }.otherwise { physical := base(axiParams.addressBits - 1, 0); state := stream } - } - when(io.axi.aw.fire)(awSent := true.B) - - // An accepted old W can be replaced immediately; refills take priority over clearing valid. - when(io.axi.w.fire)(outputValid := false.B) - when(io.data.fire) { - val properLast = io.data.bits.last === expectedLast - assert(properLast, "WriteDma input LAST does not match command length") - consumed := consumed + 1.U - when(io.data.bits.last || expectedLast)(sourceEnded := true.B) - when(state === stream) { - outputData := Mux( - properLast, - Mux(offset === 0.U, io.data.bits.data, carry | (io.data.bits.data << (offset * 8.U))(127, 0)), - 0.U - ) - outputMask := Mux(properLast, Mux(outputHead, ("hffff".U(16.W) << offset)(15, 0), "hffff".U), 0.U) - outputLast := outputIndex +& 1.U === beats - outputValid := true.B - carry := (io.data.bits.data >> ((16.U - offset) * 8.U))(127, 0) - when(!properLast && fault.error === DmaError.None.U) { - fault.error := DmaError.Protocol.U - fault.address := command.vaddr + (consumed << 4) - } - } - } - when(state === stream && !wDone && outputRoom) { - when(fault.error =/= DmaError.None.U) { - // Complete an already accepted AW even if the producer ended early. - outputData := 0.U; outputMask := 0.U - outputLast := outputIndex +& 1.U === beats; outputValid := true.B - }.elsewhen(consumed === inputTotal) { - outputData := carry - outputMask := ((1.U(17.W) << offset) - 1.U)(15, 0) - outputLast := outputIndex +& 1.U === beats; outputValid := true.B - } - } - when(io.axi.w.fire) { - firstBeat := false.B - cursorVA := cursorVA + 16.U; remaining := remaining - 1.U - beatIndex := beatIndex + 1.U - when(outputLast)(wDone := true.B) - } - when(state === stream && (awSent || io.axi.aw.fire) && - (wDone || (io.axi.w.fire && outputLast)))(state := waitB) - when(io.axi.b.fire) { - val bad = io.axi.b.bits.id =/= 1.U || io.axi.b.bits.resp =/= 0.U - when(bad && fault.error === DmaError.None.U) { - fault.error := Mux( - io.axi.b.bits.id =/= 1.U || io.axi.b.bits.resp === 1.U, - DmaError.Protocol.U, - DmaError.AccessFault.U - ) - fault.address := physical - } - when(bad || fault.error =/= DmaError.None.U) { - state := Mux(sourceEnded || consumed === inputTotal, respond, drain) - }.elsewhen(remaining === 0.U)(state := respond).otherwise(state := plan) - } - when(state === drain && (sourceEnded || consumed === inputTotal))(state := respond) - when(io.resp.fire)(state := idle) -} diff --git a/arch/src/main/scala/framework/memdomain/isa/MemISA.scala b/arch/src/main/scala/framework/memdomain/isa/MemISA.scala deleted file mode 100644 index d056b1bf9..000000000 --- a/arch/src/main/scala/framework/memdomain/isa/MemISA.scala +++ /dev/null @@ -1,31 +0,0 @@ -package framework.memdomain.isa - -import framework.system.core.rocket.BuckyballCommand - -/** Shared MemDomain ISA encoding helpers. */ -object MemISA { - val MsetFunct = 0x20 - - final case class MsetArgs( - bankId: Int, - row: Int = 0, - columns: Int, - alloc: Boolean, - clear: Boolean) - - def mset(args: MsetArgs): BuckyballCommand = { - require(args.bankId >= 0 && args.bankId < (1 << 10), s"MSET bank ID out of range: ${args.bankId}") - require(args.row >= 0 && args.row < (1 << 5), s"MSET row out of range: ${args.row}") - require(args.columns >= 0 && args.columns <= 32, s"MSET column count out of range: ${args.columns}") - require(!args.clear || args.alloc, "MSET clear requires allocation") - - val rs1 = BigInt(args.bankId) - val rs2 = - BigInt(args.row) | - (BigInt(args.columns) << 5) | - (if (args.alloc) BigInt(1) << 10 else 0) | - (if (args.clear) BigInt(1) << 11 else 0) - BuckyballCommand(MsetFunct, rs1, rs2) - } - -} diff --git a/arch/src/main/scala/framework/memdomain/isa/MvoverIO.scala b/arch/src/main/scala/framework/memdomain/isa/MvoverIO.scala deleted file mode 100644 index 14e231dcf..000000000 --- a/arch/src/main/scala/framework/memdomain/isa/MvoverIO.scala +++ /dev/null @@ -1,27 +0,0 @@ -package framework.memdomain.isa - -import chisel3._ -import chisel3.util._ - -object MvoverISA { - val Funct = 0x0d - val CoreBits = 8 - val BankBits = 10 - val AddressBits = 16 -} - -/** Contiguous rows between logical Core-local Banks in the same Tile. */ -class MvoverCommand extends Bundle { - val sourceCore = UInt(MvoverISA.CoreBits.W) - val sourceBank = UInt(MvoverISA.BankBits.W) - val sourceAddr = UInt(MvoverISA.AddressBits.W) - val targetCore = UInt(MvoverISA.CoreBits.W) - val targetBank = UInt(MvoverISA.BankBits.W) - val targetAddr = UInt(MvoverISA.AddressBits.W) - val rows = UInt((MvoverISA.AddressBits + 1).W) -} - -class MvoverPort extends Bundle { - val command = Decoupled(new MvoverCommand) - val completion = Flipped(Decoupled(Bool())) -} diff --git a/arch/src/main/scala/framework/memdomain/midend/MemMidend.scala b/arch/src/main/scala/framework/memdomain/midend/MemMidend.scala deleted file mode 100644 index 971dfc3ee..000000000 --- a/arch/src/main/scala/framework/memdomain/midend/MemMidend.scala +++ /dev/null @@ -1,277 +0,0 @@ -package framework.memdomain.midend - -import chisel3._ -import chisel3.util._ -import framework.top.GlobalConfig -import framework.balldomain.blink.{BankRead, BankWrite} -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.memdomain.backend.MemRequestIO - -// BankRead/BankWrite with is_shared flag, used for unified midend interface -class BankReadWithShared(val b: GlobalConfig) extends Bundle { - val bankRead = new BankRead(b) - val is_shared = Input(Bool()) -} - -class BankWriteWithShared(val b: GlobalConfig) extends Bundle { - val bankWrite = new BankWrite(b) - val is_shared = Input(Bool()) -} - -/** - * MemMidend: Midend module for memory scheduling - * Connects MemFrontend to MemManager - * - * Unified interface: bankRead/bankWrite Vecs include both balldomain and frontend requests. - * The last entry is frontend DMA. - * All requests go through the same mapping table and channel allocation logic. - */ -@instantiable -class MemMidend(val b: GlobalConfig) extends Module { - val totalBallRead = b.ballDomain.ballIdMappings.map(_.inBW).sum - val totalBallWrite = b.ballDomain.ballIdMappings.map(_.outBW).sum - - // Total slots: balldomain entries + 1 frontend entry - val ownerCount = b.ballDomain.ballNum - val totalRead = totalBallRead + 1 - val totalWrite = totalBallWrite + 1 - - @public - val io = IO(new Bundle { - // Unified read/write interfaces: indices [0, totalBallRead) are balldomain, - // The final port belongs to frontend DMA. - val bankRead = Vec(totalRead, new BankReadWithShared(b)) - val bankWrite = Vec(totalWrite, new BankWriteWithShared(b)) - val ballChannelActive = Input(Vec(b.ballDomain.ballNum, Bool())) - val ballChannelReady = Output(Vec(b.ballDomain.ballNum, Bool())) - - val hartid = Input(UInt(b.tile.xLen.W)) - val inst_ids = Input(Vec(b.frontend.rob_entries, UInt(64.W))) - - // Output to backend (MemManager) - val mem_req = Vec(b.memDomain.bankChannel, new MemRequestIO(b)) - }) - - // ----------------------------------------------------------------------------- - // Mapping table for tracking all requests (balldomain + frontend) - // ----------------------------------------------------------------------------- - class MappingTableEntry extends Bundle { - val valid = Bool() - val isRead = Bool() - val id = UInt(log2Ceil(math.max(totalRead, totalWrite)).W) - val leased = Bool() - val owner = UInt(math.max(1, log2Ceil(ownerCount)).W) - } - - val mappingTable = RegInit(VecInit(Seq.fill(b.memDomain.bankChannel)(0.U.asTypeOf(new MappingTableEntry)))) - - // A route stays assigned until its Ball releases it (or a frontend channel - // drains), so maintain the reverse ownership at allocation time. This keeps - // readiness and pending detection from repeatedly scanning every backend - // channel for the same port. - val portSlots = math.max(totalRead, totalWrite) - val readPortAllocated = RegInit(VecInit(Seq.fill(portSlots)(false.B))) - val writePortAllocated = RegInit(VecInit(Seq.fill(portSlots)(false.B))) - - def isAllocated(isRead: Bool, id: UInt): Bool = - Mux(isRead, readPortAllocated(id), writePortAllocated(id)) - - val readPortOwner = b.ballDomain.ballIdMappings.zipWithIndex.flatMap { case (mapping, ball) => - Seq.fill(mapping.inBW)(ball) - } - - val writePortOwner = b.ballDomain.ballIdMappings.zipWithIndex.flatMap { case (mapping, ball) => - Seq.fill(mapping.outBW)(ball) - } - - val ownerActive = io.ballChannelActive - val ownerReady = Wire(Vec(ownerCount, Bool())) - for (ball <- 0 until b.ballDomain.ballNum) { - val readReady = readPortOwner.zipWithIndex - .filter(_._1 == ball) - .map { case (_, port) => isAllocated(true.B, port.U) } - .foldLeft(true.B)(_ && _) - val writeReady = writePortOwner.zipWithIndex - .filter(_._1 == ball) - .map { case (_, port) => isAllocated(false.B, port.U) } - .foldLeft(true.B)(_ && _) - ownerReady(ball) := ownerActive(ball) && readReady && writeReady - } - - io.ballChannelReady := VecInit(ownerReady.take(b.ballDomain.ballNum)) - - // Allocate exactly one channel per cycle. Ball channels are allocated before - // their data traffic can start; frontend DMA channels are demand-allocated. - for (i <- 0 until totalRead) { - io.bankRead(i).bankRead.io.req.ready := false.B - io.bankRead(i).bankRead.io.resp.valid := false.B - io.bankRead(i).bankRead.io.resp.bits := DontCare - - } - - for (i <- 0 until totalWrite) { - io.bankWrite(i).bankWrite.io.req.ready := false.B - io.bankWrite(i).bankWrite.io.resp.valid := false.B - io.bankWrite(i).bankWrite.io.resp.bits := DontCare - } - - val pendingReads = VecInit((0 until totalRead).map { i => - val active = - if (i < totalBallRead) { - ownerActive(readPortOwner(i)) - - } else { - io.bankRead(i).bankRead.io.req.valid - } - active && !isAllocated(true.B, i.U) - }) - - val pendingWrites = VecInit((0 until totalWrite).map { i => - val active = - if (i < totalBallWrite) { - ownerActive(writePortOwner(i)) - - } else { - io.bankWrite(i).bankWrite.io.req.valid - } - active && !isAllocated(false.B, i.U) - }) - - val pending = VecInit(pendingReads ++ pendingWrites) - val freeChannels = mappingTable.map(entry => !entry.valid) - - when(pending.asUInt.orR && freeChannels.reduce(_ || _)) { - val channel = PriorityEncoder(freeChannels) - val port = PriorityEncoder(pending) - mappingTable(channel).valid := true.B - mappingTable(channel).isRead := port < totalRead.U - mappingTable(channel).id := Mux(port < totalRead.U, port, port - totalRead.U) - mappingTable(channel).leased := Mux( - port < totalRead.U, - MuxLookup(port, false.B)((0 until totalBallRead).map(i => i.U -> true.B).toSeq), - MuxLookup(port - totalRead.U, false.B)((0 until totalBallWrite).map(i => i.U -> true.B).toSeq) - ) - mappingTable(channel).owner := Mux( - port < totalRead.U, - MuxLookup(port, 0.U)(readPortOwner.zipWithIndex.map { case (ball, i) => i.U -> ball.U }.toSeq), - MuxLookup(port - totalRead.U, 0.U)(writePortOwner.zipWithIndex.map { case (ball, i) => i.U -> ball.U }.toSeq) - ) - - when(port < totalRead.U) { - readPortAllocated(port) := true.B - }.otherwise { - writePortAllocated(port - totalRead.U) := true.B - } - } - - // Connect mapped entries to backend channels - for (i <- 0 until b.memDomain.bankChannel) { - io.mem_req(i).read.req.valid := false.B - io.mem_req(i).read.req.bits := DontCare - io.mem_req(i).read.resp.ready := false.B - io.mem_req(i).write.req.valid := false.B - io.mem_req(i).write.req.bits := DontCare - io.mem_req(i).write.resp.ready := false.B - io.mem_req(i).bank_id := 0.U - io.mem_req(i).group_id := 0.U - io.mem_req(i).is_shared := false.B - io.mem_req(i).hart_id := io.hartid - io.mem_req(i).rob_id := 0.U - io.mem_req(i).inst_id := 0.U - - val isRead = mappingTable(i).isRead - val rid = mappingTable(i).id - val wid = mappingTable(i).id - val ballRead = io.bankRead(rid).bankRead.io - val ballWrite = io.bankWrite(wid).bankWrite.io - val rbank_id = io.bankRead(rid).bankRead.bank_id - val wbank_id = io.bankWrite(wid).bankWrite.bank_id - val rgroup_id = io.bankRead(rid).bankRead.group_id - val wgroup_id = io.bankWrite(wid).bankWrite.group_id - val r_shared = io.bankRead(rid).is_shared - val w_shared = io.bankWrite(wid).is_shared - val rrob_id = io.bankRead(rid).bankRead.rob_id - val wrob_id = io.bankWrite(wid).bankWrite.rob_id - // A released Ball cannot issue new traffic, but its final bank response - // still needs the existing route until the physical channel drains. - val ballRouteOpen = !mappingTable(i).leased || - ownerReady(mappingTable(i).owner) || - !ownerActive(mappingTable(i).owner) - - when(mappingTable(i).valid) { - when(isRead) { - when(ballRouteOpen) { - io.mem_req(i).read <> ballRead - io.mem_req(i).bank_id := rbank_id - io.mem_req(i).group_id := rgroup_id - io.mem_req(i).is_shared := r_shared - io.mem_req(i).rob_id := rrob_id - io.mem_req(i).inst_id := io.inst_ids(rrob_id) - } - }.otherwise { - when(ballRouteOpen) { - io.mem_req(i).write <> ballWrite - io.mem_req(i).bank_id := wbank_id - io.mem_req(i).group_id := wgroup_id - io.mem_req(i).is_shared := w_shared - io.mem_req(i).rob_id := wrob_id - io.mem_req(i).inst_id := io.inst_ids(wrob_id) - } - } - } - } - - // Mapping table release - for (i <- 0 until b.memDomain.bankChannel) { - val releaseCounter = RegInit(0.U(5.W)) - val readPending = RegInit(0.U(32.W)) - val writePending = RegInit(0.U(32.W)) - when(io.mem_req(i).read.req.fire =/= io.mem_req(i).read.resp.fire) { - readPending := Mux(io.mem_req(i).read.req.fire, readPending + 1.U, readPending - 1.U) - } - when(io.mem_req(i).write.req.fire =/= io.mem_req(i).write.resp.fire) { - writePending := Mux(io.mem_req(i).write.req.fire, writePending + 1.U, writePending - 1.U) - } - - // Releasing a Ball's logical channel can precede the final bank response. - // Keep its physical route until that response has been consumed; otherwise - // the response loses its consumer and leaves the AccPipe busy. - val ballReleased = mappingTable(i).leased && - !ownerActive(mappingTable(i).owner) - val channelDrained = (readPending === 0.U) && (writePending === 0.U) && !(io.mem_req(i).read.resp.valid || - io.mem_req(i).write.resp.valid || io.mem_req(i).read.req.valid || - io.mem_req(i).write.req.valid) - - when(mappingTable(i).valid && ballReleased && channelDrained) { - mappingTable(i).valid := false.B - mappingTable(i).isRead := false.B - mappingTable(i).id := 0.U - mappingTable(i).leased := false.B - mappingTable(i).owner := 0.U - when(mappingTable(i).isRead) { - readPortAllocated(mappingTable(i).id) := false.B - }.otherwise { - writePortAllocated(mappingTable(i).id) := false.B - } - releaseCounter := 0.U - }.elsewhen(mappingTable(i).valid && !mappingTable(i).leased && channelDrained) { - releaseCounter := releaseCounter + 1.U - - when(releaseCounter === 16.U) { - releaseCounter := 0.U - mappingTable(i).valid := false.B - mappingTable(i).isRead := false.B - mappingTable(i).id := 0.U - mappingTable(i).leased := false.B - mappingTable(i).owner := 0.U - when(mappingTable(i).isRead) { - readPortAllocated(mappingTable(i).id) := false.B - }.otherwise { - writePortAllocated(mappingTable(i).id) := false.B - } - } - }.otherwise { - releaseCounter := 0.U - } - } -} diff --git a/arch/src/main/scala/framework/memdomain/utils/pmc/MemCyclePMC.scala b/arch/src/main/scala/framework/memdomain/utils/pmc/MemCyclePMC.scala deleted file mode 100644 index 5f9c02239..000000000 --- a/arch/src/main/scala/framework/memdomain/utils/pmc/MemCyclePMC.scala +++ /dev/null @@ -1,79 +0,0 @@ -package framework.memdomain.utils.pmc - -import chisel3._ -import chisel3.util._ -import framework.top.GlobalConfig -import framework.memdomain.frontend.cmd.rs.{MemRsComplete, MemRsIssue} -import chisel3.experimental.hierarchy.{instantiable, public} - -@instantiable -class MemCyclePMC(val b: GlobalConfig) extends Module { - - @public - val io = IO(new Bundle { - val ldReq_i = Input(Valid(new MemRsIssue(b))) - val stReq_i = Input(Valid(new MemRsIssue(b))) - val ldResp_o = Input(Valid(new MemRsComplete(b))) - val stResp_o = Input(Valid(new MemRsComplete(b))) - val ldTotalCycles = Output(UInt(64.W)) - val stTotalCycles = Output(UInt(64.W)) - }) - - val cycleCounter = RegInit(0.U(64.W)) - cycleCounter := cycleCounter + 1.U - - val startTime = Reg(Vec(b.frontend.rob_entries, UInt(64.W))) - val ldTotalCycles = RegInit(0.U(64.W)) - val stTotalCycles = RegInit(0.U(64.W)) - - // DPI-C trace modules for load and store - val ldPmcTrace = Module(new MemPMCTraceDPI) - val stPmcTrace = Module(new MemPMCTraceDPI) - - ldPmcTrace.io.clock := clock - ldPmcTrace.io.reset := reset.asBool - ldPmcTrace.io.is_store := 0.U - ldPmcTrace.io.rob_id := 0.U - ldPmcTrace.io.elapsed := 0.U - ldPmcTrace.io.enable := false.B - - stPmcTrace.io.clock := clock - stPmcTrace.io.reset := reset.asBool - stPmcTrace.io.is_store := 1.U - stPmcTrace.io.rob_id := 0.U - stPmcTrace.io.elapsed := 0.U - stPmcTrace.io.enable := false.B - - when(io.ldReq_i.valid) { - startTime(io.ldReq_i.bits.rob_id) := cycleCounter - } - - when(io.stReq_i.valid) { - startTime(io.stReq_i.bits.rob_id) := cycleCounter - } - - when(io.ldResp_o.valid) { - val robId = io.ldResp_o.bits.rob_id - val elapsed = cycleCounter - startTime(robId) - ldTotalCycles := ldTotalCycles + elapsed - - // DPI-C trace output - ldPmcTrace.io.rob_id := robId - ldPmcTrace.io.elapsed := elapsed - ldPmcTrace.io.enable := true.B - } - - when(io.stResp_o.valid) { - val robId = io.stResp_o.bits.rob_id - val elapsed = cycleCounter - startTime(robId) - stTotalCycles := stTotalCycles + elapsed - - // DPI-C trace output - stPmcTrace.io.rob_id := robId - stPmcTrace.io.elapsed := elapsed - stPmcTrace.io.enable := true.B - } - - io.ldTotalCycles := ldTotalCycles - io.stTotalCycles := stTotalCycles -} diff --git a/arch/src/main/scala/framework/memdomain/utils/pmc/MemPMCTraceDPI.scala b/arch/src/main/scala/framework/memdomain/utils/pmc/MemPMCTraceDPI.scala deleted file mode 100644 index 69ba68b8c..000000000 --- a/arch/src/main/scala/framework/memdomain/utils/pmc/MemPMCTraceDPI.scala +++ /dev/null @@ -1,64 +0,0 @@ -package framework.memdomain.utils.pmc - -import chisel3._ -import chisel3.util._ -import framework.dpi.DpiGuard - -class MemPMCTraceDPI extends BlackBox with HasBlackBoxInline { - - val io = IO(new Bundle { - val clock = Input(Clock()) - val reset = Input(Bool()) - val is_store = Input(UInt(8.W)) - val rob_id = Input(UInt(32.W)) - val elapsed = Input(UInt(64.W)) - val enable = Input(Bool()) - }) - - setInline( - "MemPMCTraceDPI.v", - """ - |module MemPMCTraceDPI( - | input clock, - | input reset, - | input [7:0] is_store, - | input [31:0] rob_id, - | input [63:0] elapsed, - | input enable - |); - |""".stripMargin + DpiGuard.wrapPMCTrace( - """ - | import "DPI-C" context function void dpi_mem_pmctrace( - | input int unsigned is_store, - | input int unsigned rob_id, - | input int unsigned elapsed_lo, - | input int unsigned elapsed_hi - | ); - | reg [7:0] is_store_reg; - | reg [31:0] rob_id_reg; - | reg [63:0] elapsed_reg; - | reg valid_reg; - | - | always @(posedge clock) begin - | if (reset) begin - | valid_reg <= 1'b0; - | end else begin - | if (valid_reg) begin - | dpi_mem_pmctrace(is_store_reg, rob_id_reg, elapsed_reg[31:0], elapsed_reg[63:32]); - | end - | - | valid_reg <= enable; - | if (enable) begin - | is_store_reg <= is_store; - | rob_id_reg <= rob_id; - | elapsed_reg <= elapsed; - | end - | end - | end - |""".stripMargin - ) + - """ - |endmodule - """.stripMargin - ) -} diff --git a/arch/src/main/scala/framework/memdomain/verification/.gitignore b/arch/src/main/scala/framework/memdomain/verification/.gitignore deleted file mode 100644 index 5efea52c3..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -/build/ -/src/csrc/target/ -/cm.log -/ucli.key diff --git a/arch/src/main/scala/framework/memdomain/verification/src/csrc/Cargo.lock b/arch/src/main/scala/framework/memdomain/verification/src/csrc/Cargo.lock deleted file mode 100644 index 2c71631e6..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/csrc/Cargo.lock +++ /dev/null @@ -1,268 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "bebop-bank-hash" -version = "0.1.0" -dependencies = [ - "serde", - "serde_json", - "snafu", -] - -[[package]] -name = "bebop-elf" -version = "0.1.0" -dependencies = [ - "bebop-memory", -] - -[[package]] -name = "bebop-memory" -version = "0.1.0" -dependencies = [ - "log", - "serde", - "serde_json", - "sha2", -] - -[[package]] -name = "bebop-rtl-trace" -version = "0.1.0" -dependencies = [ - "bebop-bank-hash", -] - -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" -dependencies = [ - "generic-array", -] - -[[package]] -name = "cfg-if" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" - -[[package]] -name = "cpufeatures" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" -dependencies = [ - "libc", -] - -[[package]] -name = "crypto-common" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" -dependencies = [ - "generic-array", - "typenum", -] - -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer", - "crypto-common", -] - -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - -[[package]] -name = "itoa" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" - -[[package]] -name = "libc" -version = "0.2.190" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" - -[[package]] -name = "log" -version = "0.4.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" - -[[package]] -name = "memchr" -version = "2.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" - -[[package]] -name = "memdomain-ack-reference" -version = "0.1.0" -dependencies = [ - "bebop-elf", - "bebop-memory", - "bebop-rtl-trace", -] - -[[package]] -name = "proc-macro2" -version = "1.0.107" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quote" -version = "1.0.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "serde" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" -dependencies = [ - "serde_core", - "serde_derive", -] - -[[package]] -name = "serde_core" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.6", -] - -[[package]] -name = "serde_json" -version = "1.0.151" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" -dependencies = [ - "itoa", - "memchr", - "serde", - "serde_core", - "zmij", -] - -[[package]] -name = "sha2" -version = "0.10.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - -[[package]] -name = "snafu" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" -dependencies = [ - "snafu-derive", -] - -[[package]] -name = "snafu-derive" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "syn" -version = "2.0.119" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "3.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "typenum" -version = "1.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" - -[[package]] -name = "unicode-ident" -version = "1.0.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" - -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - -[[package]] -name = "zmij" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/arch/src/main/scala/framework/memdomain/verification/src/csrc/Cargo.toml b/arch/src/main/scala/framework/memdomain/verification/src/csrc/Cargo.toml deleted file mode 100644 index b335a27c0..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/csrc/Cargo.toml +++ /dev/null @@ -1,11 +0,0 @@ -[package] -name = "memdomain-ack-reference" -version = "0.1.0" -edition = "2021" -[lib] -crate-type = ["cdylib", "staticlib"] - -[dependencies] -bebop-rtl-trace = { path = "../../../../../../../../../bebop/src/nodes/lib/rtl-trace" } -bebop-elf = { path = "../../../../../../../../../bebop/src/nodes/lib/elf" } -bebop-memory = { path = "../../../../../../../../../bebop/src/nodes/lib/memory" } diff --git a/arch/src/main/scala/framework/memdomain/verification/src/csrc/src/lib.rs b/arch/src/main/scala/framework/memdomain/verification/src/csrc/src/lib.rs deleted file mode 100644 index e256fc4f7..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/csrc/src/lib.rs +++ /dev/null @@ -1,12 +0,0 @@ -mod memory; -mod tile_peer; -pub use memory::*; -#[path = "../../../../../mem-core/ddr/src/csrc/src/lib.rs"] -mod ddr_reference; - -#[no_mangle] -pub extern "C" fn admission_trace_init() { - bebop_rtl_trace::init_trace(std::path::Path::new("build/admission_trace"), - bebop_rtl_trace::TraceConfig { itrace: true, mtrace: true, pmctrace: true, - ctrace: false, banktrace: false }).expect("initialize AdmissionSystem trace"); -} diff --git a/arch/src/main/scala/framework/memdomain/verification/src/csrc/src/memory.rs b/arch/src/main/scala/framework/memdomain/verification/src/csrc/src/memory.rs deleted file mode 100644 index db01c630f..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/csrc/src/memory.rs +++ /dev/null @@ -1,18 +0,0 @@ -//! Byte-addressed golden storage, independent of DUT state and completion timing. -use std::collections::BTreeMap; -#[derive(Default)] -pub struct Memory(BTreeMap); -#[no_mangle] -pub extern "C" fn ack_ref_create() -> *mut Memory { Box::into_raw(Box::new(Memory::default())) } -#[no_mangle] -pub unsafe extern "C" fn ack_ref_destroy(m: *mut Memory) { drop(Box::from_raw(m)); } -#[no_mangle] -pub unsafe extern "C" fn ack_ref_write(m: *mut Memory, addr: u64, lo: u64, hi: u64, mask: u32) { - let data = (lo as u128 | ((hi as u128) << 64)).to_le_bytes(); - for (i, byte) in data.into_iter().enumerate() { if mask & (1 << i) != 0 { (*m).0.insert(addr + i as u64, byte); } } -} -#[no_mangle] -pub unsafe extern "C" fn ack_ref_check(m: *mut Memory, addr: u64, lo: u64, hi: u64) -> u32 { - let data = (lo as u128 | ((hi as u128) << 64)).to_le_bytes(); - u32::from(data.iter().enumerate().all(|(i, byte)| (*m).0.get(&(addr + i as u64)) == Some(byte))) -} diff --git a/arch/src/main/scala/framework/memdomain/verification/src/csrc/src/tile_peer.rs b/arch/src/main/scala/framework/memdomain/verification/src/csrc/src/tile_peer.rs deleted file mode 100644 index cc62540c5..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/csrc/src/tile_peer.rs +++ /dev/null @@ -1,147 +0,0 @@ -use bebop_memory::Memory; -use std::cell::RefCell; -use std::collections::BTreeSet; -use std::ffi::{c_char, c_void, CStr}; -use std::path::Path; - -const BASE: u64 = 0x8000_0000; -const BYTES: usize = 16usize << 30; - -// External DDR starts zeroed. Storage is allocated one line at a time, using -// the maintained DDR reference for every program/read/write operation. -struct TileMemory { - model: *mut c_void, - initialized: RefCell>, -} -impl TileMemory { - fn new() -> Self { - Self { model: crate::ddr_reference::ddr_ref_create(), initialized: RefCell::new(BTreeSet::new()) } - } - fn line(&self, address: u64) -> [u32; 16] { - let mut data = [0u32; 16]; - if self.initialized.borrow_mut().insert(address) { - unsafe { crate::ddr_reference::ddr_ref_program(self.model, address, data.as_ptr()); } - } - assert_eq!(unsafe { crate::ddr_reference::ddr_ref_read(self.model, address, data.as_mut_ptr()) }, 1, - "DDR reference line must be initialized"); - data - } - fn check(offset: usize, bytes: usize) { - assert!(offset.checked_add(bytes).is_some_and(|end| end <= BYTES), "Tile DDR access outside 16 GiB"); - } -} -impl Drop for TileMemory { - fn drop(&mut self) { unsafe { crate::ddr_reference::ddr_ref_destroy(self.model); } } -} -impl Memory for TileMemory { - fn len(&self) -> usize { BYTES } - fn read_buffer(&self, offset: usize, output: &mut [u8]) { - Self::check(offset, output.len()); - let mut cursor = 0; - while cursor < output.len() { - let address = BASE + (offset + cursor) as u64; - let line = self.line(address & !63); - let start = (address & 63) as usize; - let count = (64 - start).min(output.len() - cursor); - for i in 0..count { - output[cursor + i] = (line[(start + i) / 4] >> (((start + i) % 4) * 8)) as u8; - } - cursor += count; - } - } - fn write_buffer(&self, offset: usize, input: &[u8]) { - Self::check(offset, input.len()); - let mut cursor = 0; - while cursor < input.len() { - let address = BASE + (offset + cursor) as u64; - let mut line = self.line(address & !63); - let start = (address & 63) as usize; - let count = (64 - start).min(input.len() - cursor); - for i in 0..count { - let shift = ((start + i) % 4) * 8; - line[(start + i) / 4] = (line[(start + i) / 4] & !(255 << shift)) | ((input[cursor + i] as u32) << shift); - } - unsafe { crate::ddr_reference::ddr_ref_program(self.model, address & !63, line.as_ptr()); } - cursor += count; - } - } - fn fill(&self, offset: usize, bytes: usize, value: u8) { - Self::check(offset, bytes); - let block = [value; 64]; - for cursor in (0..bytes).step_by(64) { self.write_buffer(offset + cursor, &block[..64.min(bytes - cursor)]); } - } -} -thread_local! { static DDR: RefCell> = const { RefCell::new(None) }; } -thread_local! { static EXPECTED: RefCell> = const { RefCell::new(None) }; } - -#[no_mangle] -pub unsafe extern "C" fn tile_memory_init(image_path: *const c_char, expected_path: *const c_char) -> u64 { - let image = CStr::from_ptr(image_path).to_str().expect("Tile image path UTF-8"); - let expected = CStr::from_ptr(expected_path).to_str().expect("Tile expected bytes path UTF-8"); - assert!(!image.is_empty() && !expected.is_empty(), "Explicit Tile image and expected-byte paths are required"); - let memory = TileMemory::new(); - let entry = bebop_elf::load_elf(image, &memory, BASE, 0).expect("load Tile ELF").entry; - assert!((BASE..BASE + BYTES as u64).contains(&entry), "Tile entry outside DDR"); - DDR.with(|ddr| { assert!(ddr.borrow().is_none(), "Tile peer initialized twice"); *ddr.borrow_mut() = Some(memory); }); - EXPECTED.with(|path| { *path.borrow_mut() = Some(expected.to_owned()); }); - bebop_rtl_trace::init_trace(Path::new("build/tile_system_trace"), bebop_rtl_trace::TraceConfig { - itrace: true, mtrace: true, pmctrace: true, ctrace: true, banktrace: true, - }).expect("initialize actual Tile RTL traces"); - println!("TILE_PEER_IMAGE image={image} entry={entry:#x} DDR=16GiB zero-initialized"); - entry -} - -#[no_mangle] -pub extern "C" fn tile_peer_read64(address: u64) -> u64 { - DDR.with(|ddr| { - let memory = ddr.borrow(); - let mut data = [0; 8]; - memory.as_ref().expect("Tile peer not initialized").read_buffer(address.checked_sub(BASE).expect("AXI address below DDR") as usize, &mut data); - u64::from_le_bytes(data) - }) -} -#[no_mangle] -pub extern "C" fn tile_peer_write128(address: u64, lo: u64, hi: u64, mask: u32) { - assert_eq!(mask & !0xffff, 0, "AXI 128-bit write strobe width"); - assert_eq!(address & 15, 0, "AXI DDR write beat alignment"); - DDR.with(|ddr| { - let memory = ddr.borrow(); - let memory = memory.as_ref().expect("Tile peer not initialized"); - let offset = address.checked_sub(BASE).expect("AXI address below DDR") as usize; - TileMemory::check(offset, 16); - let mut data = memory.line(address & !63); - let incoming = (lo as u128 | ((hi as u128) << 64)).to_le_bytes(); - let start = (address & 63) as usize; - for i in 0..16 { - let shift = ((start + i) % 4) * 8; - data[(start + i) / 4] = (data[(start + i) / 4] & !(255 << shift)) | ((incoming[i] as u32) << shift); - } - assert_eq!(unsafe { crate::ddr_reference::ddr_ref_write(memory.model, address & !63, - data.as_ptr(), (mask as u64) << start) }, 1, "DDR AXI write to initialized reference line"); - }); -} -#[no_mangle] -pub extern "C" fn tile_peer_finish() { - let expected = EXPECTED.with(|path| path.borrow().as_ref().expect("Explicit Tile expected bytes path required").clone()); - let text = std::fs::read_to_string(&expected).expect("read expected Tile DDR output bytes"); - let mut compared = 0; - DDR.with(|ddr| { - let memory = ddr.borrow(); - let memory = memory.as_ref().expect("Tile peer not initialized"); - for line in text.lines().filter(|line| !line.trim().is_empty()) { - let fields: Vec<_> = line.split_whitespace().collect(); - assert_eq!(fields.len(), 2, "Expected DDR row must contain address and hex bytes"); - let address = u64::from_str_radix(fields[0].trim_start_matches("0x"), 16).expect("expected DDR address"); - assert!(!fields[1].is_empty() && fields[1].len() % 2 == 0, "expected DDR byte string"); - let bytes: Vec<_> = (0..fields[1].len()).step_by(2).map(|i| - u8::from_str_radix(&fields[1][i..i + 2], 16).expect("expected DDR byte")).collect(); - let mut actual = vec![0; bytes.len()]; - memory.read_buffer(address.checked_sub(BASE).expect("expected DDR address below RAM") as usize, &mut actual); - assert_eq!(actual, bytes, "Actual Tile NPU DDR bytes differ at {address:#x}"); - compared += actual.len(); - } - }); - assert!(compared > 0, "No independent Tile DDR output comparisons"); - println!("TILE_PEER_DDR_COMPARE bytes={compared} expected={expected}"); - bebop_rtl_trace::write_trace_summary(Path::new("build/tile_system_trace")).expect("write Tile RTL trace summary"); -} diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/ack_package.sv b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/ack_package.sv deleted file mode 100644 index 47a9f7bb8..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/ack_package.sv +++ /dev/null @@ -1,17 +0,0 @@ -package ack_pkg; - import "DPI-C" function chandle ack_ref_create(); - import "DPI-C" function void ack_ref_destroy(input chandle model); - import "DPI-C" function void ack_ref_write( - input chandle model, - input longint unsigned addr, - lo, - hi, - input int unsigned mask - ); - import "DPI-C" function int unsigned ack_ref_check( - input chandle model, - input longint unsigned addr, - lo, - hi - ); -endpackage diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/admission_system.cm_hier b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/admission_system.cm_hier deleted file mode 100644 index cdc734792..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/admission_system.cm_hier +++ /dev/null @@ -1,3 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.command_if -+tree @TOP@.complete_if diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/admission_system.f b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/admission_system.f deleted file mode 100644 index 3022d4f8c..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/admission_system.f +++ /dev/null @@ -1,8 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip/test -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/test/control_if.sv -@VERIFY@/uvm/src/ip/test/control_test_package.sv --F @RTL@/AdmissionSystem/filelist.f -@RESOURCES@/admission_system_tb.sv diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/admission_system.toml b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/admission_system.toml deleted file mode 100644 index eb0e075b4..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/admission_system.toml +++ /dev/null @@ -1 +0,0 @@ -chips = ["goban"] diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/admission_system_tb.sv b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/admission_system_tb.sv deleted file mode 100644 index ab66f8308..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/admission_system_tb.sv +++ /dev/null @@ -1,479 +0,0 @@ -`include "admission_system_config.svh" -module admission_system_tb; - import uvm_pkg::*; - import ip_control_test_pkg::*; - `include "uvm_macros.svh" - import "DPI-C" function chandle ddr_ref_create(); - import "DPI-C" function void admission_trace_init(); - import "DPI-C" function void dpi_bdb_set_clk(input longint unsigned cycle); - import "DPI-C" function void ddr_ref_destroy(input chandle model); - import "DPI-C" function void ddr_ref_program( - input chandle model, - input longint unsigned address, - input bit [511:0] data - ); - import "DPI-C" function int unsigned ddr_ref_read( - input chandle model, - input longint unsigned address, - output bit [511:0] data - ); - import "DPI-C" function int unsigned ddr_ref_write( - input chandle model, - input longint unsigned address, - input bit [511:0] data, - input longint unsigned mask - ); - logic clock = 0, reset = 1; - always #5 clock = ~clock; - ip_control_if ctl (clock); - `include "admission_system_signals.svh" -AdmissionSystem dut ( - `include "admission_system_ports.svh" - ); - `include "admission_system_clocking.svh" -stream_if #(64 + 64 + 8) command_if ( - clock, - reset - ); - assign command_if.valid = io_core_command_valid; - assign command_if.ready = io_core_command_ready; - assign command_if.bits = { - io_core_command_bits_instruction_rs1Data, - io_core_command_bits_instruction_rs2Data, - io_core_command_bits_tag - }; - stream_if #(8) complete_if ( - clock, - reset - ); - assign complete_if.valid = io_core_complete_valid; - assign complete_if.ready = io_core_complete_ready; - assign complete_if.bits = io_core_complete_bits_tag; - chandle model; - bit returned[4] = '{default: 0}; - bit [511:0] line, wline; - bit [63:0] wmask; - int - cycle = 0, - completed = 0, - reads = 0, - writes = 0, - backs = 0, - aw_id = 0, - ar_id = 0, - rbeat = 0, - rbeats = 0, - wbeat = 0, - wbeats = 0, - r_due = 0, - b_due = 0, - maint_due = 0; - bit reading = 0, writing = 0, committed = 0, roff = 0, boff = 0, maint_pending = 0, allow_b = 1; - localparam int ROWS = `ADMISSION_ROWS; - localparam int BYTES = ROWS * 16; - localparam int EXPECTED_BURSTS = (ROWS + 255) / 256; - localparam longint unsigned INPUT_BASE = 'h80010000; - localparam longint unsigned OUTPUT_BASE = 'h80020000; - localparam longint unsigned UNALIGNED_BASE = 'h80024ff3; - int expected_total_b = EXPECTED_BURSTS, expected_store_tag = 2; - bit unaligned_phase = 0, expectedFaultPhase = 0; - int fault_start_aw = 0, fault_start_b = 0, fault_start_completed = 0; - longint unsigned aw_address = 0, ar_address = 0; - int maint_tag = 0; - bit finish_wait = 0; - function automatic bit [511:0] pattern(int line_index = 0); - bit [511:0] data; - for (int i = 0; i < 64; i++) data[i*8+:8] = line_index * 37 + i * 11 + 3; - return data; - endfunction - task automatic verify_contract(bit ok, string message); - if (!ok) `uvm_fatal("ADMISSION_SYSTEM", message) - endtask - task automatic memory_and_checks(); - forever begin - @(sample); - if (!sample.reset) begin - cycle++; - dpi_bdb_set_clk(cycle); - if (!expectedFaultPhase) - verify_contract( - !sample.io_halted, $sformatf( - "halted error%0d address%h", sample.io_fault_error, sample.io_fault_address)); - else begin - verify_contract(!sample.io_core_complete_valid, - "errored store published Core completion"); - if (writes > fault_start_aw || sample.io_halted) - verify_contract(!sample.io_core_cpuAllow, "errored write range was released to CPU"); - if (sample.io_halted) - verify_contract( - sample.io_fault_error == 2 && sample.io_fault_address == OUTPUT_BASE && - sample.io_faultTag == 0, - "late BERROR halted with incorrect typed fault or tag"); - end - if (sample.io_core_complete_valid && sample.io_core_complete_ready) begin - int id; - id = sample.io_core_complete_bits_tag; - verify_contract(!returned[id], "duplicate Core completion"); - returned[id] = 1; - completed++; - if (id == expected_store_tag) - verify_contract( - writes == expected_total_b && - backs + int'(sample.io_axi_b_valid && sample.io_axi_b_ready) == expected_total_b && - (!writing || (sample.io_axi_b_valid && sample.io_axi_b_ready)), - "MVOUT Core release preceded final B for the whole bank"); - end - if (sample.io_core_maintenance_valid && sample.io_core_maintenance_ready) begin - verify_contract(!maint_pending, "maintenance overlap"); - maint_pending = 1; - maint_tag = sample.io_core_maintenance_bits_tag; - maint_due = cycle + 8; - end - if (sample.io_core_maintained_valid && sample.io_core_maintained_ready) maint_pending = 0; - if (sample.io_axi_ar_valid && sample.io_axi_ar_ready) begin - verify_contract( - !reading && sample.io_axi_ar_bits_size == 4 && sample.io_axi_ar_bits_burst == 1 && - sample.io_axi_ar_bits_addr[3:0] == 0 && sample.io_axi_ar_bits_len + 1 <= 256 && - sample.io_axi_ar_bits_addr[11:0] + (sample.io_axi_ar_bits_len + 1) * 16 <= 4096, - "wrong AR"); - verify_contract( - sample.io_axi_ar_bits_addr == INPUT_BASE + reads * 4096 && - sample.io_axi_ar_bits_len + 1 == ((ROWS - reads * 256) < 256 ? ROWS - reads * 256 : 256), - "read did not fill a legal 256-beat/page segment"); - reading = 1; - ar_id = sample.io_axi_ar_bits_id; - rbeat = 0; - rbeats = sample.io_axi_ar_bits_len + 1; - ar_address = sample.io_axi_ar_bits_addr; - r_due = cycle + 4; - reads++; - end - if (sample.io_axi_r_valid && sample.io_axi_r_ready) begin - roff = 0; - rbeat++; - r_due = cycle + 3; - if (rbeat == rbeats) reading = 0; - end - if (sample.io_axi_aw_valid && sample.io_axi_aw_ready) begin - verify_contract( - !writing && sample.io_axi_aw_bits_size == 4 && sample.io_axi_aw_bits_burst == 1 && - sample.io_axi_aw_bits_addr[3:0] == 0 && sample.io_axi_aw_bits_len + 1 <= 256 && - sample.io_axi_aw_bits_addr[11:0] + (sample.io_axi_aw_bits_len + 1) * 16 <= 4096, - "wrong AW"); - if (expectedFaultPhase) - verify_contract( - writes == fault_start_aw && sample.io_axi_aw_bits_addr == OUTPUT_BASE && - sample.io_axi_aw_bits_len == 255, - "BERROR case emitted a later page or lost its first 256-beat burst"); - else if (!unaligned_phase) - verify_contract( - sample.io_axi_aw_bits_addr == OUTPUT_BASE + writes * 4096 && - sample.io_axi_aw_bits_len + 1 == ((ROWS - writes * 256) < 256 ? ROWS - writes * 256 : 256), - "write did not fill a legal 256-beat/page segment"); - else begin - int segment; - segment = writes - EXPECTED_BURSTS; - verify_contract( - (segment == 0 && sample.io_axi_aw_bits_addr == (UNALIGNED_BASE & ~64'd15) && sample.io_axi_aw_bits_len == 0) || - (segment == 1 && sample.io_axi_aw_bits_addr == ((UNALIGNED_BASE + 13) & ~64'd15) && sample.io_axi_aw_bits_len == 3), - "unaligned cross-page AW segments wrong"); - end - writing = 1; - aw_id = sample.io_axi_aw_bits_id; - aw_address = sample.io_axi_aw_bits_addr; - wbeats = sample.io_axi_aw_bits_len + 1; - wbeat = 0; - wline = '0; - wmask = '0; - committed = 0; - boff = 0; - writes++; - end - if (sample.io_axi_w_valid && sample.io_axi_w_ready) begin - verify_contract(writing && sample.io_axi_w_bits_last == (wbeat + 1 == wbeats), - "W lacks actual AW or WLAST incorrect"); - if (unaligned_phase) - verify_contract( - sample.io_axi_w_bits_strb == (writes == EXPECTED_BURSTS + 1 ? 'hfff8 : wbeat == wbeats - 1 ? 'h0007 : 'hffff), - "unaligned cross-page first/tail WSTRB incorrect"); - begin - longint unsigned beat_address; - int line_offset; - beat_address = aw_address + wbeat * 16; - line_offset = beat_address & 63; - wline = '0; - wmask = '0; - wline[line_offset*8+:128] = sample.io_axi_w_bits_data; - wmask[line_offset+:16] = sample.io_axi_w_bits_strb; - verify_contract(ddr_ref_write(model, beat_address & ~64'd63, wline, wmask) == 1, - "DDR beat commit failed across reference-line mapping"); - end - wbeat++; - if (wbeat == wbeats) begin - committed = 1; - b_due = cycle + 8; - end - end - if (sample.io_axi_b_valid && sample.io_axi_b_ready) begin - writing = 0; - boff = 0; - backs++; - end - if(sample.io_core_command_valid && sample.io_core_command_bits_instruction_opcode=='h2b && sample.io_core_command_bits_instruction_funct==2 && - (!returned[expected_store_tag] || backs < expected_total_b || writing)) - verify_contract(!sample.io_core_command_ready, - "Task finish published with pending DDR/ROB work"); - end - @(negedge clock); - io_axi_ar_ready = !reading && cycle % 5 != 0; - io_axi_aw_ready = !writing && cycle % 4 != 0; - io_axi_w_ready = writing && wbeat < wbeats && cycle % 3 != 0; - if (reading && !roff && cycle >= r_due) begin - io_axi_r_valid = 1; - io_axi_r_bits_id = ar_id; - begin - longint unsigned beat_address; - int line_offset; - beat_address = ar_address + rbeat * 16; - line_offset = beat_address & 63; - verify_contract(ddr_ref_read(model, beat_address & ~64'd63, line) == 1, - "unprogrammed DDR line during long read burst"); - io_axi_r_bits_data = line[line_offset*8+:128]; - end - io_axi_r_bits_resp = 0; - io_axi_r_bits_last = rbeat + 1 == rbeats; - roff = 1; - end - if (!roff) io_axi_r_valid = 0; - io_axi_b_valid = writing && committed && cycle >= b_due && allow_b; - io_axi_b_bits_id = aw_id; - io_axi_b_bits_resp = expectedFaultPhase ? 2 : 0; - io_core_maintenance_ready = !maint_pending; - io_core_maintained_valid = maint_pending && cycle >= maint_due; - io_core_maintained_bits_tag = maint_tag; - io_core_maintained_bits_ok = 1; - end - endtask - task automatic controller(int fn, longint unsigned a, longint unsigned b, - output longint unsigned result); - @(negedge clock); - io_controller_cmd_bits_opcode = 'h2b; - io_controller_cmd_bits_funct = fn; - io_controller_cmd_bits_rs1Data = a; - io_controller_cmd_bits_rs2Data = b; - io_controller_cmd_bits_rd = 10; - io_controller_cmd_valid = 1; - io_controller_resp_ready = 1; - do @(sample); while (!sample.io_controller_cmd_ready); - @(negedge clock); - io_controller_cmd_valid = 0; - do @(sample); while (!sample.io_controller_resp_valid); - result = sample.io_controller_resp_bits_data; - @(negedge clock); - io_controller_resp_ready = 0; - endtask - task automatic command(int tag, int opcode, int fn, longint unsigned a, longint unsigned b, - bit permission = 1); - @(negedge clock); - io_core_reserve_bits_id = tag; - io_core_reserve_valid = 1; - do @(sample); while (!sample.io_core_reserve_ready); - @(negedge clock); - io_core_reserve_valid = 0; - io_core_command_bits_tag = tag; - io_core_command_bits_satp = 0; - io_core_command_bits_effectivePrivilege = 1; - io_core_command_bits_instruction_opcode = opcode; - io_core_command_bits_instruction_funct = fn; - io_core_command_bits_instruction_funct3 = 3; - io_core_command_bits_instruction_xs1 = 1; - io_core_command_bits_instruction_xs2 = 1; - io_core_command_bits_instruction_rd = 10; - io_core_command_bits_instruction_xd = opcode == 'h2b; - io_core_command_bits_instruction_rs1Data = a; - io_core_command_bits_instruction_rs2Data = b; - io_core_command_bits_pmp_0_cfg_a = 1; - io_core_command_bits_pmp_0_cfg_r = permission; - io_core_command_bits_pmp_0_cfg_w = permission; - io_core_command_bits_pmp_0_addr = 'h20400000; - io_core_command_valid = 1; - do @(sample); while (!sample.io_core_command_ready); - @(negedge clock); - io_core_command_valid = 0; - // The live input is now denied. Permission must keep the accepted snapshot. - io_core_command_bits_pmp_0_cfg_r = 0; - io_core_command_bits_pmp_0_cfg_w = 0; - endtask - initial begin - uvm_config_db#(virtual ip_control_if)::set(null, "*", "vif", ctl); - uvm_config_db#(time)::set(null, "uvm_test_top", "timeout", 1ms); - run_test("protocol_test"); - end - initial begin - longint unsigned value; - bit [511:0] actual, first_expected, second_expected, source_data; - `include "admission_system_init.svh" -admission_trace_init(); - model = ddr_ref_create(); - verify_contract(ROWS > 256 && BYTES % 64 == 0, - "fixture must exercise a full bank across pages"); - for (int index = 0; index < BYTES / 64; index++) begin - ddr_ref_program(model, INPUT_BASE + index * 64, pattern(index)); - ddr_ref_program(model, OUTPUT_BASE + index * 64, '0); - end - wait (ctl.start); - repeat (5) @(negedge clock); - reset = 0; - io_core_complete_ready = 1; - io_core_response_ready = 1; - fork - memory_and_checks(); - join_none - wait (io_npuIdle); - command(3, 'h7b, 32, 0, 'h421, 0); - wait (returned[3]); - returned[3] = 0; - for (int i = 0; i < 7; i++) controller(7, i, i == 6 ? `ADMISSION_SIGNATURE : 0, value); - controller(0, 0, `ADMISSION_SIGNATURE, value); - verify_contract(value == 0, "actual signature task submit failed"); - command(3, 'h2b, 8, 0, 0); - wait (returned[3]); - returned[3] = 0; - io_blockedResponse = 1; - io_core_cpuQuery_valid = 1; - io_core_cpuQuery_paddr = INPUT_BASE; - io_core_cpuQuery_sizeLog2 = 3; - io_core_cpuQuery_write = 1; - command(0, 'h7b, 33, 64'(ROWS) << 30, INPUT_BASE | (64'd1 << 39)); - @(sample); - verify_contract(!sample.io_core_cpuAllow, "CPU write passed reserved NPU read range"); - repeat (12) @(sample); - verify_contract(!returned[0], "context released at request acceptance"); - @(negedge clock); - io_blockedResponse = 0; - wait (returned[0]); - @(sample); - verify_contract(sample.io_core_cpuAllow, "CPU remained blocked after read completion"); - @(negedge clock); - allow_b = 0; - io_core_cpuQuery_paddr = OUTPUT_BASE; - io_core_cpuQuery_write = 0; - command(2, 'h7b, 16, 64'(ROWS) << 30, OUTPUT_BASE | (64'd1 << 39)); - wait (committed); - @(sample); - verify_contract(!sample.io_core_cpuAllow, - "CPU read passed NPU write before real DDR B and post-maintenance"); - fork - command(3, 'h2b, 2, 0, 0, 0); - begin - finish_wait = 1; - repeat (12) @(sample); - verify_contract(!returned[2], "MVOUT released before real B"); - @(negedge clock); - finish_wait = 0; - allow_b = 1; - end - join - wait (returned[2] && returned[3] && io_workDrained); - @(sample); - verify_contract(sample.io_core_cpuAllow, - "CPU read did not resume after actual write completion"); - verify_contract( - reads == EXPECTED_BURSTS && writes == EXPECTED_BURSTS && - backs == writes && !writing && !reading, - "full-bank AXI segments not drained"); - for (int index = 0; index < BYTES / 64; index++) - verify_contract(ddr_ref_read(model, OUTPUT_BASE + index * 64, actual) == 1 && actual == pattern( - index), "full-bank MVIN/MVOUT DDR golden mismatch"); - controller(1, 0, 0, value); - verify_contract(value == 1, "Task finish failed to publish actual completion"); - // Reuse the released tag for a tiny real WriteDma carry/mask transaction. - ddr_ref_program(model, UNALIGNED_BASE & ~64'd63, {64{8'ha5}}); - ddr_ref_program(model, (UNALIGNED_BASE + 13) & ~64'd63, {64{8'ha5}}); - @(negedge clock); - returned[0] = 0; - expected_store_tag = 0; - expected_total_b = EXPECTED_BURSTS + 2; - unaligned_phase = 1; - allow_b = 0; - committed = 0; - io_core_cpuQuery_paddr = UNALIGNED_BASE & ~64'd7; - io_core_cpuQuery_write = 0; - command(0, 'h7b, 16, 64'd4 << 30, UNALIGNED_BASE | (64'd1 << 39)); - wait (committed); - repeat (12) begin - @(sample); - verify_contract(!returned[0] && !sample.io_core_cpuAllow, - "unaligned store released before its delayed first B"); - end - @(negedge clock); - allow_b = 1; - wait (returned[0] && io_workDrained); - @(sample); - verify_contract( - backs == expected_total_b && writes == expected_total_b && !writing && !reading && - sample.io_core_cpuAllow, - "unaligned Core release did not join both final B responses"); - source_data = pattern(0); - first_expected = {64{8'ha5}}; - second_expected = {64{8'ha5}}; - first_expected[51*8+:13*8] = source_data[0+:13*8]; - second_expected[0+:51*8] = source_data[13*8+:51*8]; - verify_contract(ddr_ref_read(model, UNALIGNED_BASE & ~64'd63, actual - ) == 1 && actual == first_expected, - "unaligned head carry corrupted requested data or three sentinel bytes"); - verify_contract(ddr_ref_read(model, (UNALIGNED_BASE + 13) & ~64'd63, actual - ) == 1 && actual == second_expected, - "cross-page carry/tail corrupted requested data or thirteen sentinel bytes"); - // First-page SLVERR is terminal: drain the remaining raw input without - // issuing later AWs, and retain the Admission reservation instead of release. - @(negedge clock); - fault_start_aw = writes; - fault_start_b = backs; - fault_start_completed = completed; - expectedFaultPhase = 1; - unaligned_phase = 0; - allow_b = 1; - committed = 0; - returned[0] = 0; - io_core_cpuQuery_paddr = OUTPUT_BASE; - io_core_cpuQuery_write = 0; - command(0, 'h7b, 16, 64'(ROWS) << 30, OUTPUT_BASE | (64'd1 << 39)); - do @(sample); while (!(sample.io_halted && sample.io_npuIdle)); - repeat (24) begin - @(sample); - verify_contract( - sample.io_halted && sample.io_npuIdle && sample.io_fault_error == 2 && - sample.io_fault_address == OUTPUT_BASE && sample.io_faultTag == 0, - $sformatf( - "BERROR terminal error=%0d addr=%h tag=%0d idle=%0d halted=%0d expected=%h", - sample.io_fault_error, - sample.io_fault_address, - sample.io_faultTag, - sample.io_npuIdle, - sample.io_halted, - OUTPUT_BASE - )); - verify_contract( - writes == fault_start_aw + 1 && backs == fault_start_b + 1 && - !writing && !reading && !sample.io_axi_aw_valid, - "BERROR emitted an extra AW/B or left an AXI transaction live"); - verify_contract( - !returned[0] && completed == fault_start_completed && - !sample.io_core_complete_valid && !sample.io_core_cpuAllow && !sample.io_workDrained, - "BERROR released its Core/CPU reservation instead of retaining the failed context"); - end - `uvm_info("ADMISSION_BERROR", - "First 256-beat B=SLVERR: remaining input drained, one AW/B only, fault tag/address retained and no Core/CPU release", - UVM_LOW) - `uvm_info("ADMISSION_SYSTEM", $sformatf( - "Completed%0d AR%0d AW%0d B%0d; actual Goban signature/PMP frozen, Task finish joined", - completed, - reads, - writes, - backs - ), UVM_LOW) - ddr_ref_destroy(model); - ctl.done = 1; - end -endmodule diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/controller_system.cm_hier b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/controller_system.cm_hier deleted file mode 100644 index 122b5840e..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/controller_system.cm_hier +++ /dev/null @@ -1,4 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.command_if -+tree @TOP@.response_if -+tree @TOP@.complete_if diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/controller_system.f b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/controller_system.f deleted file mode 100644 index 5e931ccdb..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/controller_system.f +++ /dev/null @@ -1,8 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip/test -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/test/control_if.sv -@VERIFY@/uvm/src/ip/test/control_test_package.sv --F @RTL@/ControllerSystem/filelist.f -@RESOURCES@/controller_system_tb.sv diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/controller_system.toml b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/controller_system.toml deleted file mode 100644 index eb0e075b4..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/controller_system.toml +++ /dev/null @@ -1 +0,0 @@ -chips = ["goban"] diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/controller_system_tb.sv b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/controller_system_tb.sv deleted file mode 100644 index ba533dd70..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/controller_system_tb.sv +++ /dev/null @@ -1,305 +0,0 @@ -`include "controller_system_config.svh" -module controller_system_tb; - import uvm_pkg::*; - import ip_control_test_pkg::*; - `include "uvm_macros.svh" - import "DPI-C" function void admission_trace_init(); - import "DPI-C" function void dpi_bdb_set_clk(input longint unsigned cycle); - import "DPI-C" function chandle ack_ref_create(); - import "DPI-C" function void ack_ref_destroy(input chandle model); - import "DPI-C" function void ack_ref_write( - input chandle model, - input longint unsigned address, - input longint unsigned lo, - input longint unsigned hi, - input int unsigned mask - ); - import "DPI-C" function int unsigned ack_ref_check( - input chandle model, - input longint unsigned address, - input longint unsigned lo, - input longint unsigned hi - ); - logic clock = 0, reset = 1; - always #5 clock = ~clock; - ip_control_if ctl (clock); - `include "controller_system_signals.svh" -ControllerSystem dut ( - `include "controller_system_ports.svh" - ); - `include "controller_system_clocking.svh" -stream_if #(136) command_if ( - clock, - reset - ); - assign command_if.valid = io_core_command_valid; - assign command_if.ready = io_core_command_ready; - assign command_if.bits = { - io_core_command_bits_instruction_rs1Data, - io_core_command_bits_instruction_rs2Data, - io_core_command_bits_tag - }; - stream_if #(69) response_if ( - clock, - reset - ); - assign response_if.valid = io_core_response_valid; - assign response_if.ready = io_core_response_ready; - assign response_if.bits = {io_core_response_bits_rd, io_core_response_bits_data}; - stream_if #(8) complete_if ( - clock, - reset - ); - assign complete_if.valid = io_core_complete_valid; - assign complete_if.ready = io_core_complete_ready; - assign complete_if.bits = io_core_complete_bits_tag; - chandle model; - int cycle = 0, commands = 0, completions = 0, moves = 0, move_done = 0, target_writes = 0; - bit protect = 0; - function automatic bit [127:0] row_data(int row); - bit [127:0] data; - for (int j = 0; j < 16; j++) data[j*8+:8] = row * 31 + j * 7 + 3; - return data; - endfunction - task automatic verify_contract(bit ok, string message); - if (!ok) `uvm_fatal("CONTROLLER_SYSTEM", message) - endtask - task automatic monitor_contract(); - forever begin - @(sample); - if (!sample.reset) begin - cycle++; - dpi_bdb_set_clk(cycle); - verify_contract(!sample.io_moveFault_valid, "real bank move reported failure"); - if (sample.io_moveAccepted) moves++; - if (sample.io_moveCompleted) move_done++; - if (sample.io_bankWriteAccepted_1) target_writes++; - if (sample.io_core_command_valid && sample.io_core_command_ready) commands++; - if (sample.io_core_complete_valid && sample.io_core_complete_ready) begin - verify_contract(sample.io_core_complete_bits_tag == 0, "wrong completion owner"); - completions++; - end - if (protect) - verify_contract(!sample.io_core_cpuAllow, "pending move allowed CPU publication"); - end - end - endtask - task automatic issue(int opcode, int fn, longint unsigned a, longint unsigned b, - longint unsigned root = 'h8000000000012345); - @(negedge clock); - io_core_reserve_bits_id = 0; - io_core_reserve_valid = 1; - do @(sample); while (!sample.io_core_reserve_ready); - @(negedge clock); - io_core_reserve_valid = 0; - io_core_command_bits_tag = 0; - io_core_command_bits_satp = root; - io_core_command_bits_instruction_opcode = opcode; - io_core_command_bits_instruction_funct = fn; - io_core_command_bits_instruction_funct3 = 3; - io_core_command_bits_instruction_xs1 = 1; - io_core_command_bits_instruction_xs2 = 1; - io_core_command_bits_instruction_xd = opcode == 'h2b; - io_core_command_bits_instruction_rd = 11; - io_core_command_bits_instruction_rs1Data = a; - io_core_command_bits_instruction_rs2Data = b; - io_core_command_valid = 1; - if (opcode == 'h7b && (fn == 13 || fn == 0)) protect = 1; - do @(sample); while (!sample.io_core_command_ready); - @(negedge clock); - io_core_command_valid = 0; - endtask - task automatic receive_response(output longint unsigned value); - do @(sample); while (!sample.io_core_response_valid); - repeat (5) begin - @(sample); - verify_contract(sample.io_core_response_valid && sample.io_core_response_bits_rd == 11, - "response withdrawn/wrong rd"); - end - value = sample.io_core_response_bits_data; - @(negedge clock); - io_core_response_ready = 1; - do @(sample); while (!sample.io_core_response_valid); - @(negedge clock); - io_core_response_ready = 0; - endtask - task automatic complete_command(); - do @(sample); while (!sample.io_core_complete_valid); - repeat (5) begin - @(sample); - verify_contract(sample.io_core_complete_valid && sample.io_core_complete_bits_tag == 0, - "completion withdrawn/wrong tag"); - end - @(negedge clock); - io_core_complete_ready = 1; - do @(sample); while (!sample.io_core_complete_valid); - @(negedge clock); - io_core_complete_ready = 0; - protect = 0; - endtask - task automatic task_call(int fn, longint unsigned a, longint unsigned b, - output longint unsigned value); - issue('h2b, fn, a, b); - @(sample); - verify_contract(sample.io_core_cpuAllow, "task RPC unnecessarily blocked CPU memory"); - receive_response(value); - complete_command(); - endtask - task automatic worker_call(int fn, longint unsigned a, output longint unsigned value); - @(negedge clock); - io_workers_0_cmd_bits_opcode = 'h2b; - io_workers_0_cmd_bits_funct = fn; - io_workers_0_cmd_bits_rs1Data = a; - io_workers_0_cmd_bits_rs2Data = 0; - io_workers_0_cmd_bits_rd = 12; - io_workers_0_cmd_valid = 1; - io_workers_0_resp_ready = 0; - do @(sample); while (!sample.io_workers_0_cmd_ready); - @(negedge clock); - io_workers_0_cmd_valid = 0; - wait (io_workers_0_resp_valid); - repeat (3) @(sample); - value = sample.io_workers_0_resp_bits_data; - @(negedge clock); - io_workers_0_resp_ready = 1; - do @(sample); while (!sample.io_workers_0_resp_valid); - @(negedge clock); - io_workers_0_resp_ready = 0; - endtask - task automatic bank0(int row, bit write, bit [127:0] data, output bit [127:0] result); - @(negedge clock); - io_access_0_request_bits_bank = 0; - io_access_0_request_bits_addr = row; - io_access_0_request_bits_write = write; - io_access_0_request_bits_data = data; - io_access_0_request_bits_mask = '1; - io_access_0_request_bits_tag = 3; - io_access_0_request_valid = 1; - io_access_0_response_ready = 1; - do @(sample); while (!sample.io_access_0_request_ready); - @(negedge clock); - io_access_0_request_valid = 0; - do @(sample); while (!sample.io_access_0_response_valid); - verify_contract( - sample.io_access_0_response_bits_tag == 3 && !sample.io_access_0_response_bits_error, - "bank0 response owner/error"); - result = sample.io_access_0_response_bits_data; - @(negedge clock); - io_access_0_response_ready = 0; - endtask - task automatic bank1(int row, output bit [127:0] result); - @(negedge clock); - io_access_1_request_bits_bank = 0; - io_access_1_request_bits_addr = row; - io_access_1_request_bits_write = 0; - io_access_1_request_bits_mask = '1; - io_access_1_request_bits_tag = 4; - io_access_1_request_valid = 1; - io_access_1_response_ready = 1; - do @(sample); while (!sample.io_access_1_request_ready); - @(negedge clock); - io_access_1_request_valid = 0; - do @(sample); while (!sample.io_access_1_response_valid); - verify_contract( - sample.io_access_1_response_bits_tag == 4 && !sample.io_access_1_response_bits_error, - "bank1 response owner/error"); - result = sample.io_access_1_response_bits_data; - @(negedge clock); - io_access_1_response_ready = 0; - endtask - initial begin - uvm_config_db#(virtual ip_control_if)::set(null, "*", "vif", ctl); - uvm_config_db#(time)::set(null, "uvm_test_top", "timeout", 1ms); - run_test("protocol_test"); - end - initial begin - longint unsigned value; - bit [127:0] actual; - `include "controller_system_init.svh" -admission_trace_init(); - model = ack_ref_create(); - io_allowBankResponse_0 = 1; - io_allowBankResponse_1 = 1; - io_allowBankResponse_2 = 1; - io_allowBankResponse_3 = 1; - wait (ctl.start); - repeat (5) @(negedge clock); - reset = 0; - io_core_cpuQuery_valid = 1; - io_core_cpuQuery_paddr = 'h80001000; - io_core_cpuQuery_sizeLog2 = 3; - io_core_cpuQuery_write = 1; - fork - monitor_contract(); - join_none - task_call(3, 0, 0, value); - verify_contract(value == 4, "actual worker count"); - task_call(4, 0, 0, value); - verify_contract(value == `CONTROLLER_SIGNATURE, "actual signature mismatch"); - for (int i = 0; i < 7; i++) begin - task_call(7, i, i == 6 ? `CONTROLLER_SIGNATURE : 'h80000000 + i * 64, value); - verify_contract(value == 0, "task stage failed"); - end - task_call(0, 0, `CONTROLLER_SIGNATURE, value); - verify_contract(value == 0, "task submit failed"); - worker_call(8, 0, value); - verify_contract(value == 1, "worker did not receive real task"); - worker_call(9, 7, value); - verify_contract(value == 'h8000000000012345, "TaskController did not capture controller satp"); - worker_call(9, 0, value); - verify_contract(value == 'h80000000, "worker context corrupted"); - worker_call(2, 0, value); - verify_contract(value == 0, "worker finish failed"); - task_call(1, 0, 0, value); - verify_contract(value == 1, "controller did not observe success"); - for (int r = 3; r < 6; r++) begin - bit [127:0] expected; - expected = row_data(r); - bank0(r, 1, expected, actual); - ack_ref_write(model, r * 16, expected[63:0], expected[127:64], 'hffff); - end - fork - begin - wait (target_writes == 3); - @(negedge clock); - io_allowBankResponse_1 = 0; - repeat (12) begin - @(sample); - verify_contract( - !sample.io_core_response_valid && !sample.io_core_complete_valid && move_done == 0, - "controller completed before final actual bank ACK"); - end - @(negedge clock); - io_allowBankResponse_1 = 1; - end - begin - issue('h7b, 13, 64'd1 << 8, 64'd3 | (64'd7 << 16) | (64'd2 << 32)); - receive_response(value); - verify_contract(value == 0, "MVO response failed"); - complete_command(); - end - join - verify_contract(moves == 1 && move_done == 1 && target_writes == 3, - "real MVO request/completion count"); - for (int r = 7; r < 10; r++) begin - bank1(r, actual); - verify_contract(ack_ref_check(model, (r - 4) * 16, actual[63:0], actual[127:64]) == 1, - "MVO row data mismatch"); - end - @(sample); - verify_contract(sample.io_core_cpuAllow, - "CPU publication did not resume after move completion"); - verify_contract(commands == completions, - "accepted controller command not completed exactly once"); - `uvm_info("CONTROLLER_SYSTEM", $sformatf( - "Commands%0d completions%0d actual MVO%0d/%0d targetRows%0d; Task/satp/Bank/finalACK checked", - commands, - completions, - moves, - move_done, - target_writes - ), UVM_LOW) - ack_ref_destroy(model); - ctl.done = 1; - end -endmodule diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/kernel_dma.cm_hier b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/kernel_dma.cm_hier deleted file mode 100644 index e8cdffc65..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/kernel_dma.cm_hier +++ /dev/null @@ -1,4 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.source_if -+tree @TOP@.sink_if -+tree @TOP@.result_if diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/kernel_dma.f b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/kernel_dma.f deleted file mode 100644 index 05a60c3bb..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/kernel_dma.f +++ /dev/null @@ -1,8 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip/test -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/test/control_if.sv -@VERIFY@/uvm/src/ip/test/control_test_package.sv --F @RTL@/KernelDma/filelist.f -@RESOURCES@/kernel_dma_tb.sv diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/kernel_dma_tb.sv b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/kernel_dma_tb.sv deleted file mode 100644 index 801952487..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/kernel_dma_tb.sv +++ /dev/null @@ -1,348 +0,0 @@ -module kernel_dma_tb; - import uvm_pkg::*; - import ip_control_test_pkg::*; - `include "uvm_macros.svh" - logic clock = 0, reset = 1; - always #5 clock = ~clock; - ip_control_if ctl (clock); - `include "kernel_dma_signals.svh" -KernelDma dut ( - `include "kernel_dma_ports.svh" - ); - `include "kernel_dma_clocking.svh" -stream_if #(64 + 32) source_if ( - clock, - reset || io_kernel_abort - ); - assign source_if.valid = io_request_valid; - assign source_if.ready = io_request_ready; - assign source_if.bits = {io_request_bits_vaddr, io_request_bits_len}; - stream_if #(32) sink_if ( - clock, - reset || io_kernel_abort - ); - assign sink_if.valid = io_kernel_image_valid; - assign sink_if.ready = io_kernel_image_ready; - assign sink_if.bits = io_kernel_image_bits; - stream_if #(68) result_if ( - clock, - reset - ); - assign result_if.valid = io_kernel_result_valid; - assign result_if.ready = io_kernel_result_ready; - assign result_if.bits = {io_kernel_result_bits_error, io_kernel_result_bits_address}; - int images = 0, results = 0, requests = 0, cases = 0, checked = 0; - int expected_images = 0, expected_error = 0; - int expected_rob = 0; - longint unsigned base = 0, fault_address = 0; - int length = 0; - function automatic logic [31:0] word_value(int index); - return 32'hc1700081 ^ (index * 32'h1020305); - endfunction - function automatic logic [127:0] beat_value(int first); - logic [127:0] data; - for (int i = 0; i < 4; i++) data[32*i+:32] = word_value(first + i); - return data; - endfunction - task automatic verify_contract(bit condition, string message); - if (!condition) `uvm_fatal("KERNEL_DMA", message) - endtask - task automatic observe(); - forever begin - @(sample); - if (!sample.reset) begin - verify_contract(sample.io_footprint_valid == sample.io_kernel_busy, - "footprint lifetime differs from transaction ownership"); - if (sample.io_footprint_valid) begin - longint unsigned rounded_span; - rounded_span = (64'(length) + 15) & ~64'd15; - verify_contract( - sample.io_footprint_rob_id==expected_rob && - !sample.io_footprint_is_sub && sample.io_footprint_sub_rob_id==0 && - sample.io_footprint_baseVA==base && sample.io_footprint_rows==1 && - sample.io_footprint_columns==1 && sample.io_footprint_spanBytes==rounded_span && - sample.io_footprint_columnStride==0 && sample.io_footprint_rowStride==0 && - !sample.io_footprint_write, - "captured footprint changed or lost full VA/rounded transport span"); - verify_contract( - sample.io_footprint_fault_error==(expected_error==7 ? 7 : 0) && - sample.io_footprint_fault_address==(expected_error==7 ? base : 0), - "footprint shape fault changed while stalled"); - checked++; - end - if (sample.io_request_valid && sample.io_request_ready) begin - requests++; - verify_contract( - requests==1 && sample.io_request_bits_vaddr==base && - sample.io_request_bits_len==length && sample.io_request_bits_groups==1 && - sample.io_request_bits_stride==1 && !sample.io_request_bits_is_2d, - "wrong DMA request or duplicate dispatch"); - end - if (sample.io_kernel_image_valid && sample.io_kernel_image_ready) begin - verify_contract(sample.io_kernel_busy && images < expected_images, - "bad/later image word escaped"); - verify_contract(sample.io_kernel_image_bits === word_value(images), - "image word order/value differs from independent sequence"); - images++; - checked++; - end - if (sample.io_kernel_result_valid) begin - verify_contract(sample.io_kernel_busy && !sample.io_kernel_load_ready, - "terminal result did not retain ownership"); - verify_contract(images == expected_images, - "terminal result before all permitted image words consumed"); - verify_contract( - sample.io_kernel_result_bits_error==expected_error && - sample.io_kernel_result_bits_address==fault_address, - "wrong terminal full status/address"); - if (sample.io_kernel_result_ready) results++; - checked++; - end - end - end - endtask - task automatic configure(longint unsigned address, int bytes, int words, int error = 0, - longint unsigned error_address = 0); - @(negedge clock); - verify_contract(!io_kernel_busy && io_kernel_load_ready, - "previous transaction did not release"); - base = address; - length = bytes; - expected_images = words; - expected_error = error; - fault_address = error_address; - images = 0; - results = 0; - requests = 0; - cases++; - expected_rob = cases % (1 << $bits(io_rob_id)); - io_rob_id = expected_rob; - io_kernel_image_ready = 0; - io_kernel_result_ready = 0; - io_request_ready = 0; - io_response_valid = 0; - io_response_bits_fault_error = 0; - io_response_bits_fault_address = 0; - io_kernel_load_bits_address = address; - io_kernel_load_bits_bytes = bytes; - io_kernel_load_valid = 1; - do @(sample); while (!sample.io_kernel_load_ready); - @(negedge clock); - io_kernel_load_valid = 0; - io_rob_id = ~expected_rob; - endtask - task automatic accept_request(); - wait (io_request_valid); - repeat (4) begin - @(sample); - verify_contract( - sample.io_kernel_busy && !sample.io_kernel_load_ready && !sample.io_kernel_result_valid, - "request stalled ownership wrong"); - end - @(negedge clock); - io_request_ready = 1; - do @(sample); while (!sample.io_request_valid); - @(negedge clock); - io_request_ready = 0; - endtask - task automatic response(int first, bit last, int error = 0, longint unsigned address = 0); - @(negedge clock); - io_response_bits_data = beat_value(first); - io_response_bits_last = last; - io_response_bits_addrcounter = first / 4; - io_response_bits_fault_error = error; - io_response_bits_fault_address = address; - io_response_valid = 1; - do @(sample); while (!sample.io_response_ready); - @(negedge clock); - io_response_valid = 0; - endtask - task automatic consume_words(int count); - @(negedge clock); - io_kernel_image_ready = 1; - wait (images == count); - @(negedge clock); - io_kernel_image_ready = 0; - endtask - task automatic no_terminal(int cycles); - repeat (cycles) begin - @(sample); - verify_contract( - sample.io_kernel_busy && !sample.io_kernel_result_valid && - !sample.io_kernel_load_ready && results==0, - "early terminal before accepted read drained"); - end - endtask - task automatic retire_result(int required_requests); - wait (io_kernel_result_valid); - repeat (8) @(sample); - @(negedge clock); - io_kernel_abort = 1; // Held terminal must be immutable, including success. - repeat (4) @(sample); - @(negedge clock); - io_kernel_abort = 0; - io_kernel_result_ready = 1; - do @(sample); while (!sample.io_kernel_result_valid); - @(negedge clock); - io_kernel_result_ready = 0; - repeat (4) @(sample); - @(negedge clock); - verify_contract( - results==1 && requests==required_requests && !io_kernel_busy && io_kernel_load_ready && - !io_kernel_result_valid && !io_kernel_image_valid && !io_footprint_valid, - "duplicate result or ownership/footprint not released"); - endtask - task automatic recovery(); - configure('h9000 + cases * 64, 4, 1); - accept_request(); - response(0, 1); - consume_words(1); - retire_result(1); - endtask - initial begin - uvm_config_db#(virtual ip_control_if)::set(null, "*", "vif", ctl); - uvm_config_db#(time)::set(null, "uvm_test_top", "timeout", 100us); - run_test("protocol_test"); - end - initial begin - `include "kernel_dma_init.svh" - wait (ctl.start); - repeat (4) @(sample); - @(negedge clock); - reset = 0; - fork - observe(); - join_none - // A partial final beat emits exactly five words; success requires the final image handshake. - configure('h1000, 20, 5); - accept_request(); - response(0, 0); - no_terminal(8); - consume_words(4); - response(4, 1); - no_terminal(8); - consume_words(5); - retire_result(1); - // Original unaligned VA is preserved; preflight later aligns [1,17) to [0,32). - configure(1, 4, 1); - accept_request(); - response(0, 1); - no_terminal(8); - consume_words(1); - retire_result(1); - // First and middle faults preserve all 64 address bits and drop later data while draining. - configure('h2000, 48, 0, 3, 64'h123456789abcde00); - accept_request(); - response(0, 0, 3, 64'h123456789abcde00); - no_terminal(8); - response(4, 0); - no_terminal(8); - response(8, 1, 4, 'h2222); - retire_result(1); - recovery(); - configure('h3000, 48, 4, 4, 64'hfedcba9876543210); - accept_request(); - response(0, 0); - consume_words(4); - response(4, 0, 4, 64'hfedcba9876543210); - no_terminal(8); - response(8, 1); - retire_result(1); - recovery(); - // Abort an offered, unaccepted DMA request: cancellation, no request handshake. - configure('h4000, 16, 0, 10, 'h4000); - wait (io_request_valid); - @(negedge clock); - io_kernel_abort = 1; - repeat (4) @(sample); - @(negedge clock); - io_kernel_abort = 0; - retire_result(0); - recovery(); - // Abort an accepted read: keep ownership until its terminal response is actually consumed. - configure('h5000, 32, 0, 10, 'h5000); - accept_request(); - @(negedge clock); - io_kernel_abort = 1; - no_terminal(8); - @(negedge clock); - io_kernel_abort = 0; - response(0, 0); - no_terminal(8); - response(4, 1); - retire_result(1); - recovery(); - // Abort a buffered non-final output after one word: flush the remaining words and drain. - configure('h6000, 32, 1, 10, 'h6000); - accept_request(); - response(0, 0); - consume_words(1); - @(negedge clock); - io_kernel_abort = 1; - no_terminal(4); - @(negedge clock); - io_kernel_abort = 0; - response(4, 1); - retire_result(1); - recovery(); - // Abort a fully buffered final beat with zero output acceptance. - configure('h6100, 16, 0, 10, 'h6100); - accept_request(); - response(0, 1); - @(negedge clock); - io_kernel_abort = 1; - repeat (4) @(sample); - @(negedge clock); - io_kernel_abort = 0; - retire_result(1); - recovery(); - // An abort asserted before load acceptance is not a transaction and produces no result. - @(negedge clock); - io_kernel_abort = 1; - io_kernel_load_valid = 1; - repeat (8) begin - @(sample); - verify_contract( - !sample.io_kernel_load_ready && !sample.io_kernel_busy && - !sample.io_kernel_result_valid && !sample.io_request_valid, - "unaccepted cancelled load generated work"); - end - @(negedge clock); - io_kernel_load_valid = 0; - io_kernel_abort = 0; - recovery(); - // Shape errors never dispatch a DMA request. - configure('h7000, 0, 0, 7, 'h7000); - retire_result(0); - recovery(); - configure('h7100, 6, 0, 7, 'h7100); - retire_result(0); - recovery(); - configure(64'hfffffffffffffffc, 8, 0, 7, 64'hfffffffffffffffc); - retire_result(0); - recovery(); - // Useful [VA,VA+4) fits, but the mandatory sixteen-byte transport span wraps. - configure(64'hfffffffffffffffc, 4, 0, 7, 64'hfffffffffffffffc); - retire_result(0); - recovery(); - // Early last and missing last are protocol failures, never fabricated success. - configure('h8000, 32, 0, 6, 'h8000); - accept_request(); - response(0, 1); - retire_result(1); - recovery(); - configure('h8100, 16, 0, 6, 'h8100); - accept_request(); - response(0, 0); - no_terminal(8); - response(4, 1); - retire_result(1); - recovery(); - `uvm_info("KERNEL_DMA_GATE", $sformatf( - "%0d cases, %0d status/word checks; fault/cancel/drain/held terminal and recovery", - cases, - checked - ), UVM_LOW) - ctl.done = 1; - end -endmodule diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/loader_ack.f b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/loader_ack.f deleted file mode 100644 index 7fadeefec..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/loader_ack.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip/test -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/test/control_if.sv -@VERIFY@/uvm/src/ip/test/control_test_package.sv -@RESOURCES@/ack_package.sv --F @RTL@/MemLoader/filelist.f -@RESOURCES@/loader_ack_tb.sv diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/loader_ack_tb.sv b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/loader_ack_tb.sv deleted file mode 100644 index c6c99f81c..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/loader_ack_tb.sv +++ /dev/null @@ -1,412 +0,0 @@ -module loader_ack_tb; - import uvm_pkg::*; - import ack_pkg::*; - import ip_control_test_pkg::*; - `include "uvm_macros.svh" - logic clock = 0, reset = 0; - always #5 clock = ~clock; - ip_control_if ctl (clock); - `include "loader_ack_signals.svh" -stream_if #( - .WIDTH($bits( - io_cmdResp_bits_rob_id - ) + $bits( - io_cmdResp_bits_is_sub - ) + $bits( - io_cmdResp_bits_sub_rob_id - ) + $bits( - io_cmdResp_bits_fault_error - ) + $bits( - io_cmdResp_bits_fault_address - )) - ) sink_if ( - clock, - reset - ); - assign sink_if.valid = io_cmdResp_valid; - assign sink_if.ready = io_cmdResp_ready; - assign sink_if.bits = { - io_cmdResp_bits_rob_id, - io_cmdResp_bits_is_sub, - io_cmdResp_bits_sub_rob_id, - io_cmdResp_bits_fault_error, - io_cmdResp_bits_fault_address - }; - MemLoader dut ( - `include "loader_ack_ports.svh" - ); - stream_if #( - .WIDTH($bits( - io_cmdReq_bits_cmd_special - ) + $bits( - io_cmdReq_bits_cmd_mem_addr - ) + $bits( - io_cmdReq_bits_cmd_iter - ) + $bits( - io_cmdReq_bits_cmd_bank_id - )) - ) source_if ( - clock, - reset - ); - assign source_if.valid = io_cmdReq_valid; - assign source_if.ready = io_cmdReq_ready; - assign source_if.bits = { - io_cmdReq_bits_cmd_special, - io_cmdReq_bits_cmd_mem_addr, - io_cmdReq_bits_cmd_iter, - io_cmdReq_bits_cmd_bank_id - }; - stream_if #( - .WIDTH($bits( - io_footprint_rob_id - ) + $bits( - io_footprint_is_sub - ) + $bits( - io_footprint_sub_rob_id - ) + $bits( - io_footprint_baseVA - ) + $bits( - io_footprint_rows - ) + $bits( - io_footprint_columns - ) + $bits( - io_footprint_spanBytes - ) + $bits( - io_footprint_columnStride - ) + $bits( - io_footprint_rowStride - ) + $bits( - io_footprint_write - ) + $bits( - io_footprint_fault_error - ) + $bits( - io_footprint_fault_address - )) - ) footprint_if ( - clock, - reset - ); - assign footprint_if.valid = io_footprint_valid; - assign footprint_if.ready = io_cmdResp_valid && io_cmdResp_ready; - assign footprint_if.bits = { - io_footprint_rob_id, - io_footprint_is_sub, - io_footprint_sub_rob_id, - io_footprint_baseVA, - io_footprint_rows, - io_footprint_columns, - io_footprint_spanBytes, - io_footprint_columnStride, - io_footprint_rowStride, - io_footprint_write, - io_footprint_fault_error, - io_footprint_fault_address - }; - chandle model; - int completions = 0; - logic [127:0] expected, observed; - longint unsigned observed_addr; - int unsigned observed_mask; - function automatic logic [127:0] row_data(int row); - logic [127:0] data; - for (int j = 0; j < 16; j++) data[j*8+:8] = (row * 37 + j * 11 + 3) & 255; - return data; - endfunction - task automatic check(bit condition, string message); - if (!condition) `uvm_fatal("ACK_CONTRACT", message) - endtask - task automatic no_completion(int cycles); - repeat (cycles) begin - @(posedge clock); - #1; - check(!io_cmdResp_valid, "cmdResp.valid asserted before final ACK"); - check(!io_cmdReq_ready, "accepted next command before completion"); - check(completions == 0, "unexpected completion count before ACK"); - end - endtask - task automatic complete_command(int error = 0, longint unsigned address = 0); - wait (io_cmdResp_valid); - repeat (8) begin - @(posedge clock); - #1; - check( - io_cmdResp_valid && io_cmdResp_bits_rob_id==3 && io_cmdResp_bits_is_sub==1 && io_cmdResp_bits_sub_rob_id==2, - "completion changed under backpressure"); - check(io_cmdResp_bits_fault_error == error && io_cmdResp_bits_fault_address == address, - "completion fault changed under backpressure"); - check(!io_cmdReq_ready, "command accepted while completion stalled"); - end - @(negedge clock); - io_cmdResp_ready = 1; - @(posedge clock); - check(io_cmdResp_valid, "completion lost before handshake"); - completions++; - @(negedge clock); - io_cmdResp_ready = 0; - repeat (8) begin - @(posedge clock); - #1; - check(!io_cmdResp_valid, "duplicate completion"); - end - check(completions == 1 && io_cmdReq_ready, "completion count or idle readiness wrong"); - endtask - task automatic begin_load(longint unsigned address, int rows, int groups = 1, int stride = 1); - completions = 0; - @(negedge clock); - io_cmdReq_bits_cmd_mem_addr = address; - io_cmdReq_bits_cmd_iter = rows; - io_query_group_count = groups; - io_cmdReq_bits_cmd_special = 64'(stride) << 39; - io_dmaResp_bits_fault_error = 0; - io_dmaResp_bits_fault_address = 0; - io_cmdReq_valid = 1; - do @(posedge clock); while (!io_cmdReq_ready); - @(negedge clock); - io_cmdReq_valid = 0; - wait (io_dmaReq_valid); - check( - io_footprint_valid && io_footprint_baseVA==address && io_footprint_rows==rows && io_footprint_columns==1 && io_footprint_spanBytes==groups*16 && io_footprint_columnStride==0 && io_footprint_rowStride==groups*stride*16 && io_footprint_fault_error==0 && io_footprint_write==0, - "production footprint differs from actual command"); - check(io_dmaReq_bits_vaddr == address && io_dmaReq_bits_len == rows * groups * 16, - "followup load DMA shape wrong"); - @(negedge clock); - io_dmaReq_ready = 1; - @(posedge clock); - check(io_dmaReq_valid, "followup DMA request missing"); - @(negedge clock); - io_dmaReq_ready = 0; - endtask - task automatic read_beat(int row, bit last, int error = 0, longint unsigned address = 0); - @(negedge clock); - io_dmaResp_bits_data = row_data(row); - io_dmaResp_bits_last = last; - io_dmaResp_bits_addrcounter = row; - io_dmaResp_bits_fault_error = error; - io_dmaResp_bits_fault_address = address; - io_dmaResp_valid = 1; - do @(posedge clock); while (!io_dmaResp_ready); - @(negedge clock); - io_dmaResp_valid = 0; - endtask - task automatic bank_ack(int row, bit ok); - wait (io_bankWrite_io_req_valid); - check(io_bankWrite_io_req_bits_addr == row && io_bankWrite_io_req_bits_data === row_data(row), - "followup bank data/address wrong"); - @(negedge clock); - io_bankWrite_io_req_ready = 1; - @(posedge clock); - check(io_bankWrite_io_req_valid, "followup bank request missing"); - @(negedge clock); - io_bankWrite_io_req_ready = 0; - no_completion(16); - @(negedge clock); - io_bankWrite_io_resp_bits_ok = ok; - io_bankWrite_io_resp_valid = 1; - do @(posedge clock); while (!io_bankWrite_io_resp_ready); - @(negedge clock); - io_bankWrite_io_resp_valid = 0; - endtask - task automatic no_bank_write(int cycles); - repeat (cycles) begin - @(posedge clock); - #1; - check(!io_bankWrite_io_req_valid, "bank writes continued after first fault"); - end - endtask - initial begin - uvm_config_db#(virtual ip_control_if)::set(null, "*", "vif", ctl); - uvm_config_db#(time)::set(null, "uvm_test_top", "timeout", 100us); - run_test("protocol_test"); - end - initial begin - `include "loader_ack_init.svh" - wait (ctl.start); - model = ack_ref_create(); - @(negedge clock); - reset = 1; - repeat (4) @(posedge clock); - @(negedge clock); - reset = 0; - io_query_group_count = 1; - io_cmdReq_bits_cmd_mem_addr = 64'h1000; - io_cmdReq_bits_cmd_iter = 4; - io_cmdReq_bits_cmd_bank_id = 1; - io_cmdReq_bits_cmd_special = 64'd1 << 39; - io_cmdReq_bits_rob_id = 3; - io_cmdReq_bits_is_sub = 1; - io_cmdReq_bits_sub_rob_id = 2; - io_cmdReq_bits_cmd_is_load = 1; - io_cmdReq_valid = 1; - do @(posedge clock); while (!io_cmdReq_ready); - @(negedge clock); - io_cmdReq_valid = 0; - wait (io_dmaReq_valid); - check( - io_footprint_valid && io_footprint_baseVA=='h1000 && io_footprint_rows==4 && io_footprint_columns==1 && io_footprint_spanBytes==16 && io_footprint_rowStride==16 && !io_footprint_write && io_footprint_fault_error==0, - "initial loader footprint wrong"); - check( - io_dmaReq_bits_vaddr==64'h1000 && io_dmaReq_bits_len==64 && io_dmaReq_bits_stride==1 && io_dmaReq_bits_groups==1, - "load DMA request wrong"); - repeat (3) begin - @(posedge clock); - #1; - check(io_dmaReq_valid && io_dmaReq_bits_vaddr == 64'h1000, - "DMA request changed while stalled"); - end - @(negedge clock); - io_dmaReq_ready = 1; - @(posedge clock); - check(io_dmaReq_valid, "missing DMA handshake"); - @(negedge clock); - io_dmaReq_ready = 0; - for (int row = 0; row < 4; row++) begin - expected = row_data(row); - io_dmaResp_bits_data = expected; - io_dmaResp_bits_last = (row == 3); - io_dmaResp_bits_addrcounter = row; - io_dmaResp_valid = 1; - do @(posedge clock); while (!io_dmaResp_ready); - @(negedge clock); - io_dmaResp_valid = 0; - wait (io_bankWrite_io_req_valid); - check( - io_bankWrite_io_req_bits_addr==row && io_bankWrite_io_req_bits_data===expected && (&{io_bankWrite_io_req_bits_mask_0, io_bankWrite_io_req_bits_mask_1, io_bankWrite_io_req_bits_mask_2, io_bankWrite_io_req_bits_mask_3, io_bankWrite_io_req_bits_mask_4, io_bankWrite_io_req_bits_mask_5, io_bankWrite_io_req_bits_mask_6, io_bankWrite_io_req_bits_mask_7, io_bankWrite_io_req_bits_mask_8, io_bankWrite_io_req_bits_mask_9, io_bankWrite_io_req_bits_mask_10, io_bankWrite_io_req_bits_mask_11, io_bankWrite_io_req_bits_mask_12, io_bankWrite_io_req_bits_mask_13, io_bankWrite_io_req_bits_mask_14, io_bankWrite_io_req_bits_mask_15}) && io_bankWrite_bank_id==1 && io_bankWrite_rob_id==3 && io_bankWrite_group_id==0, - "bank write address/data/mask/tag wrong"); - repeat (2) begin - @(posedge clock); - #1; - check(io_bankWrite_io_req_valid && io_bankWrite_io_req_bits_data === expected, - "bank write changed under backpressure"); - end - @(negedge clock); - io_bankWrite_io_req_ready = 1; - @(posedge clock); - check(io_bankWrite_io_req_valid, "missing bank write handshake"); - ack_ref_write(model, io_bankWrite_io_req_bits_addr * 16, io_bankWrite_io_req_bits_data[63:0], - io_bankWrite_io_req_bits_data[127:64], 65535); - @(negedge clock); - io_bankWrite_io_req_ready = 0; - no_completion(row == 3 ? 32 : 2); - if (row == 3) - `uvm_info("FINAL_ACK_DELAY", - "bankWrite request accepted; cmdResp.valid remained zero for 32 clocks without bankWrite ACK", - UVM_LOW) - @(negedge clock); - io_bankWrite_io_resp_bits_ok = 1; - io_bankWrite_io_resp_valid = 1; - do @(posedge clock); while (!io_bankWrite_io_resp_ready); - @(negedge clock); - io_bankWrite_io_resp_valid = 0; - end - complete_command(); - for (int row = 0; row < 4; row++) begin - expected = row_data(row); - check(ack_ref_check(model, row * 16, expected[63:0], expected[127:64]) == 1, - "bank golden read after completion failed"); - end - // A failed bank ACK preserves the already-written prefix and drains all DMA beats. - begin_load('h2000, 4); - read_beat(0, 0); - bank_ack(0, 1); - read_beat(1, 0); - bank_ack(1, 0); - read_beat(2, 0, 4, 'hdead); - no_bank_write(8); - no_completion(16); - read_beat(3, 1); - no_bank_write(4); - complete_command(5, 16); - begin_load('h2100, 1); - read_beat(0, 1); - bank_ack(0, 1); - complete_command(); - // Failed DMA data must never reach a bank; subsequent errors cannot replace the first. - begin_load('h3000, 3); - read_beat(0, 0, 3, 'h3030); - no_bank_write(8); - read_beat(1, 0); - no_bank_write(8); - no_completion(16); - read_beat(2, 1, 2, 'h3040); - no_bank_write(4); - complete_command(3, 'h3030); - begin_load('h3100, 1); - read_beat(0, 1); - bank_ack(0, 1); - complete_command(); - begin_load('h8000, 2, 2, 3); - read_beat(0, 1, 3, 'h8000); - complete_command(3, 'h8000); - // 2D footprint retains original rows instead of flattened DMA beat count. - completions = 0; - @(negedge clock); - io_cmdReq_bits_cmd_mem_addr = 'ha000; - io_cmdReq_bits_cmd_iter = 2; - io_cmdReq_bits_cmd_is_mvin_2d = 1; - io_cmdReq_bits_cmd_special = (64'd1 << 59) | (64'd1 << 36) | (64'd4 << 43); - io_cmdReq_valid = 1; - do @(posedge clock); while (!io_cmdReq_ready); - @(negedge clock); - io_cmdReq_valid = 0; - wait (io_dmaReq_valid); - check( - io_footprint_valid && io_footprint_rows==2 && io_footprint_columns==2 && - io_footprint_spanBytes==16 && io_footprint_columnStride==8 && io_footprint_rowStride==32 && - io_footprint_fault_error==0 && io_dmaReq_bits_len==64, - "2D padded footprint wrong"); - @(negedge clock); - io_dmaReq_ready = 1; - @(posedge clock); - @(negedge clock); - io_dmaReq_ready = 0; - for (int row = 0; row < 4; row++) begin - read_beat(row, row == 3); - bank_ack(row, 1); - end - complete_command(); - @(negedge clock); - io_cmdReq_bits_cmd_is_mvin_2d = 0; - begin_load('ha100, 1); - read_beat(0, 1); - bank_ack(0, 1); - complete_command(); - // MMIO explicitly uses one group despite an unrelated query value of two. - completions = 0; - @(negedge clock); - io_query_group_count = 2; - io_cmdReq_bits_cmd_mem_addr = 'hb000; - io_cmdReq_bits_cmd_iter = 2; - io_cmdReq_bits_cmd_is_mvin_mmio = 1; - io_cmdReq_bits_cmd_special = (64'd16 << 56) | (64'h100 << 39); - io_cmdReq_valid = 1; - do @(posedge clock); while (!io_cmdReq_ready); - @(negedge clock); - io_cmdReq_valid = 0; - wait (io_dmaReq_valid); - check( - io_footprint_valid && io_footprint_rows==2 && io_footprint_columns==1 && - io_footprint_spanBytes==16 && io_footprint_columnStride==0 && io_footprint_rowStride==16 && - io_footprint_fault_error==0 && io_dmaReq_bits_groups==1, - "MMIO footprint must use one group"); - @(negedge clock); - io_dmaReq_ready = 1; - @(posedge clock); - @(negedge clock); - io_dmaReq_ready = 0; - read_beat(0, 1, 3, 'hb000); - complete_command(3, 'hb000); - @(negedge clock); - io_cmdReq_bits_cmd_is_mvin_mmio = 0; - begin_load('hb100, 1); - read_beat(0, 1); - bank_ack(0, 1); - complete_command(); - ack_ref_destroy(model); - `uvm_info( - "ACK_GATE", - "Late ACK, typed DMA/bank fault, drain, stable completion and successful followup checked", - UVM_LOW) - ctl.done = 1; - end -endmodule diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/memory_system.f b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/memory_system.f deleted file mode 100644 index 7d45d7cc2..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/memory_system.f +++ /dev/null @@ -1,7 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip/test -@VERIFY@/uvm/src/ip/test/control_if.sv -@VERIFY@/uvm/src/ip/test/control_test_package.sv --F @RTL@/Memory/filelist.f -@RESOURCES@/memory_system_tb.sv diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/memory_system.toml b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/memory_system.toml deleted file mode 100644 index eb0e075b4..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/memory_system.toml +++ /dev/null @@ -1 +0,0 @@ -chips = ["goban"] diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/memory_system_tb.sv b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/memory_system_tb.sv deleted file mode 100644 index 004333104..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/memory_system_tb.sv +++ /dev/null @@ -1,355 +0,0 @@ -module memory_system_tb; - import uvm_pkg::*; - import ip_control_test_pkg::*; - `include "uvm_macros.svh" - logic clock = 0, reset = 1; - always #5 clock = ~clock; - ip_control_if ctl (clock); - `include "memory_system_signals.svh" -Memory dut ( - `include "memory_system_ports.svh" - ); - bit line_valid[2], line_write[2], line_ready_out[2]; - bit [ 11:0] line_id [2]; - bit [ 43:0] line_addr[2]; - bit [ 63:0] line_mask[2]; - bit [511:0] line_data[2]; - logic line_ready[2], reply_valid[2], reply_error[2]; - logic [ 11:0] reply_id [2]; - logic [511:0] reply_data[2]; - assign io_lineRequest_0_valid = line_valid[0]; - assign io_lineRequest_0_bits_write = line_write[0]; - assign io_lineRequest_0_bits_id = line_id[0]; - assign io_lineRequest_0_bits_addr = line_addr[0]; - assign io_lineRequest_0_bits_data = line_data[0]; - assign io_lineRequest_0_bits_mask = line_mask[0]; - assign line_ready[0] = io_lineRequest_0_ready; - assign io_lineResponse_0_ready = line_ready_out[0]; - assign reply_valid[0] = io_lineResponse_0_valid; - assign reply_id[0] = io_lineResponse_0_bits_id; - assign reply_data[0] = io_lineResponse_0_bits_data; - assign reply_error[0] = io_lineResponse_0_bits_error; - assign io_lineRequest_1_valid = line_valid[1]; - assign io_lineRequest_1_bits_write = line_write[1]; - assign io_lineRequest_1_bits_id = line_id[1]; - assign io_lineRequest_1_bits_addr = line_addr[1]; - assign io_lineRequest_1_bits_data = line_data[1]; - assign io_lineRequest_1_bits_mask = line_mask[1]; - assign line_ready[1] = io_lineRequest_1_ready; - assign io_lineResponse_1_ready = line_ready_out[1]; - assign reply_valid[1] = io_lineResponse_1_valid; - assign reply_id[1] = io_lineResponse_1_bits_id; - assign reply_data[1] = io_lineResponse_1_bits_data; - assign reply_error[1] = io_lineResponse_1_bits_error; - `include "memory_system_clocking.svh" - typedef struct { - bit [63:0] addr; - int beat; - } read_t; - typedef struct { - bit [63:0] addr; - bit error; - } write_t; - read_t reads[int]; - write_t writes[int]; - bit [511:0] memory[bit [63:0]]; - int aw_ids[$]; - bit [63:0] aw_addr[$]; - bit [511:0] w_data[$]; - bit [63:0] w_mask[$]; - bit [511:0] assembly; - bit [63:0] assembly_mask; - int beat = 0, cycles = 0, read_bursts = 0, write_bursts = 0, checks = 0; - int dma_checks = 0; - bit hold_b = 0, read_error = 0, write_error = 0; - int r_id = 0, b_id = 0; - bit r_active = 0, b_active = 0; - function automatic void ck(bit ok, string message); - if (!ok) `uvm_fatal("MEMORY_SYSTEM", message) - endfunction - task automatic axi_service(); - forever begin - @(sample); - if (!sample.reset) begin - cycles++; - if (sample.io_axi_aw_valid && sample.io_axi_aw_ready) begin - ck( - sample.io_axi_aw_bits_len==3&&sample.io_axi_aw_bits_size==4&&sample.io_axi_aw_bits_burst==1&&!sample.io_axi_aw_bits_lock, - "AXI AW contract"); - aw_ids.push_back(sample.io_axi_aw_bits_id); - aw_addr.push_back(sample.io_axi_aw_bits_addr); - end - if (sample.io_axi_w_valid && sample.io_axi_w_ready) begin - ck(sample.io_axi_w_bits_last == (beat == 3), "AXI WLAST"); - assembly[128*beat+:128] = sample.io_axi_w_bits_data; - assembly_mask[16*beat+:16] = sample.io_axi_w_bits_strb; - if (beat == 3) begin - w_data.push_back(assembly); - w_mask.push_back(assembly_mask); - beat = 0; - end else beat++; - end - if (aw_ids.size() && w_data.size()) begin - int id = aw_ids.pop_front(); - bit [63:0] addr = aw_addr.pop_front(); - bit [511:0] data = w_data.pop_front(); - bit [63:0] mask = w_mask.pop_front(); - ck(!writes.exists(id), "AXI active write ID reused"); - if (!write_error) - for (int j = 0; j < 64; j++) if (mask[j]) memory[addr][j*8+:8] = data[j*8+:8]; - writes[id] = '{addr, write_error}; - write_bursts++; - end - if (sample.io_axi_ar_valid && sample.io_axi_ar_ready) begin - int id = sample.io_axi_ar_bits_id; - bit [63:0] addr = sample.io_axi_ar_bits_addr; - ck( - sample.io_axi_ar_bits_len==3&&sample.io_axi_ar_bits_size==4&&sample.io_axi_ar_bits_burst==1&&!sample.io_axi_ar_bits_lock, - "AXI AR contract"); - ck(!reads.exists(id), "AXI active read ID reused"); - ck(memory.exists(addr), "Read of uninitialized external memory"); - reads[id] = '{addr, 0}; - read_bursts++; - end - if (sample.io_axi_r_valid && sample.io_axi_r_ready) begin - if (reads[r_id].beat == 3) reads.delete(r_id); - else reads[r_id].beat++; - r_active = 0; - end - if (sample.io_axi_b_valid && sample.io_axi_b_ready) begin - writes.delete(b_id); - b_active = 0; - end - end - @(negedge clock); - io_axi_aw_ready = !reset && cycles % 5 != 0; - io_axi_w_ready = !reset && cycles % 7 >= 2; - io_axi_ar_ready = !reset && cycles % 4 != 0; - if (!r_active) - foreach (reads[id]) - if (!r_active) begin - r_active = 1; - r_id = id; - io_axi_r_bits_id = id; - io_axi_r_bits_data = memory[reads[id].addr][128*reads[id].beat+:128]; - io_axi_r_bits_last = reads[id].beat == 3; - io_axi_r_bits_resp = read_error && reads[id].beat == 1 ? 2 : 0; - end - if (!b_active && !hold_b) - foreach (writes[id]) - if (!b_active) begin - b_active = 1; - b_id = id; - io_axi_b_bits_id = id; - io_axi_b_bits_resp = writes[id].error ? 2 : 0; - end - io_axi_r_valid = !reset && r_active; - io_axi_b_valid = !reset && b_active; - end - endtask - task automatic issue_line(int agent, bit write, bit [63:0] addr, bit [63:0] data, int id); - @(negedge clock); - line_valid[agent] = 1; - line_write[agent] = write; - line_addr[agent] = addr; - line_data[agent] = '0; - line_data[agent][63:0] = data; - line_mask[agent] = 'hff; - line_id[agent] = id; - do @(sample); while (!sample.line_ready[agent]); - @(negedge clock); - line_valid[agent] = 0; - endtask - task automatic receive_line(int agent, int id, bit [511:0] data, bit write = 0); - do @(sample); while (!sample.reply_valid[agent]); - repeat (4) begin - ck(sample.reply_valid[agent] && sample.reply_id[agent] == id && !sample.reply_error[agent], - "Line owner/error or backpressure stability"); - if (!write) ck(sample.reply_data[agent] == data, "DDR line readback data"); - @(sample); - end - @(negedge clock); - line_ready_out[agent] = 1; - do @(sample); while (!sample.reply_valid[agent]); - checks++; - @(negedge clock); - line_ready_out[agent] = 0; - endtask - initial begin - uvm_config_db#(virtual ip_control_if)::set(null, "*", "vif", ctl); - uvm_config_db#(time)::set(null, "uvm_test_top", "timeout", 1ms); - run_test("protocol_test"); - end - initial begin - io_dma_0_aw_valid = 0; - io_dma_0_aw_bits_id = 6; - io_dma_0_aw_bits_addr = 'h80000080; - io_dma_0_aw_bits_len = 3; - io_dma_0_aw_bits_size = 4; - io_dma_0_aw_bits_burst = 1; - io_dma_0_aw_bits_lock = 0; - io_dma_0_aw_bits_cache = 0; - io_dma_0_aw_bits_prot = 0; - io_dma_0_aw_bits_qos = 0; - io_dma_0_aw_bits_region = 0; - io_dma_0_w_valid = 0; - io_dma_0_w_bits_data = 0; - io_dma_0_w_bits_strb = '1; - io_dma_0_w_bits_last = 0; - io_dma_0_b_ready = 0; - io_dma_0_ar_valid = 0; - io_dma_0_ar_bits_id = 6; - io_dma_0_ar_bits_addr = 'h80000080; - io_dma_0_ar_bits_len = 3; - io_dma_0_ar_bits_size = 4; - io_dma_0_ar_bits_burst = 1; - io_dma_0_ar_bits_lock = 0; - io_dma_0_ar_bits_cache = 0; - io_dma_0_ar_bits_prot = 0; - io_dma_0_ar_bits_qos = 0; - io_dma_0_ar_bits_region = 0; - io_dma_0_r_ready = 0; - for (int i = 0; i < 2; i++) begin - line_valid[i] = 0; - line_write[i] = 0; - line_addr[i] = 0; - line_data[i] = 0; - line_mask[i] = 0; - line_id[i] = 0; - line_ready_out[i] = 0; - end - io_axi_aw_ready = 0; - io_axi_ar_ready = 0; - io_axi_w_ready = 0; - io_axi_r_valid = 0; - io_axi_b_valid = 0; - io_axi_r_bits_id = 0; - io_axi_r_bits_data = 0; - io_axi_r_bits_last = 0; - io_axi_r_bits_resp = 0; - io_axi_b_bits_id = 0; - io_axi_b_bits_resp = 0; - memory['h80000000] = '0; - memory['h80000040] = '0; - wait (ctl.start); - repeat (5) @(negedge clock); - reset = 0; - fork - axi_service(); - join_none - // A retained NPU write response must not block independent coherence backing reads. - hold_b = 1; - @(negedge clock); - io_dma_0_aw_valid = 1; - do @(sample); while (!sample.io_dma_0_aw_ready); - @(negedge clock); - io_dma_0_aw_valid = 0; - for (int i = 0; i < 4; i++) begin - io_dma_0_w_valid = 1; - io_dma_0_w_bits_data = 128'('h100 + i); - io_dma_0_w_bits_last = i == 3; - do @(sample); while (!sample.io_dma_0_w_ready); - @(negedge clock); - end - io_dma_0_w_valid = 0; - wait (writes.num() != 0); - issue_line(0, 0, 'h80000000, 0, 19); - receive_line(0, 19, 0); - ck(io_outstanding == 1, "NPU write completed before its final AXI B"); - @(negedge clock); - hold_b = 0; - io_dma_0_b_ready = 1; - do @(sample); while (!sample.io_dma_0_b_valid); - ck(sample.io_dma_0_b_bits_id == 6 && sample.io_dma_0_b_bits_resp == 0, - "NPU write response owner or error"); - @(negedge clock); - io_dma_0_b_ready = 0; - io_dma_0_ar_valid = 1; - do @(sample); while (!sample.io_dma_0_ar_ready); - @(negedge clock); - io_dma_0_ar_valid = 0; - io_dma_0_r_ready = 1; - for (int i = 0; i < 4; i++) begin - do @(sample); while (!sample.io_dma_0_r_valid); - ck( - sample.io_dma_0_r_bits_id == 6 && sample.io_dma_0_r_bits_resp == 0 && - sample.io_dma_0_r_bits_data == 128'('h100 + i) && sample.io_dma_0_r_bits_last == (i == 3), - "NPU readback data, owner, error or RLAST"); - dma_checks++; - end - @(negedge clock); - io_dma_0_r_ready = 0; - fork - begin - issue_line(0, 1, 'h80000000, 23, 20); - receive_line(0, 20, 0, 1); - end - begin - issue_line(1, 1, 'h80000040, 31, 20); - receive_line(1, 20, 0, 1); - end - join - fork - begin - issue_line(0, 0, 'h80000000, 0, 21); - receive_line(0, 21, 23); - end - begin - issue_line(1, 0, 'h80000040, 0, 21); - receive_line(1, 21, 31); - end - join - // Neither client observes write completion before the final AXI B. - hold_b = 1; - issue_line(0, 1, 'h80000000, 41, 22); - wait (writes.num() != 0); - repeat (12) begin - @(sample); - ck(!sample.reply_valid[0] && sample.io_outstanding != 0, - "Backing write completed before final AXI B"); - end - @(negedge clock); - hold_b = 0; - receive_line(0, 22, 0, 1); - // Transport errors return to the originating coherence client. - read_error = 1; - issue_line(1, 0, 'h80000040, 0, 23); - do @(sample); while (!sample.reply_valid[1]); - ck(sample.reply_id[1] == 23 && sample.reply_error[1], "DDR R error owner"); - @(negedge clock); - line_ready_out[1] = 1; - do @(sample); while (!sample.reply_valid[1]); - checks++; - @(negedge clock); - line_ready_out[1] = 0; - read_error = 0; - write_error = 1; - issue_line(0, 1, 'h80000000, 99, 24); - do @(sample); while (!sample.reply_valid[0]); - ck(sample.reply_id[0] == 24 && sample.reply_error[0], "DDR B error owner"); - @(negedge clock); - line_ready_out[0] = 1; - do @(sample); while (!sample.reply_valid[0]); - checks++; - @(negedge clock); - line_ready_out[0] = 0; - write_error = 0; - issue_line(0, 0, 'h80000000, 0, 25); - receive_line(0, 25, 41); - repeat (8) @(sample); - ck( - !reads.num() && !writes.num() && !aw_ids.size() && !aw_addr.size() && - !w_data.size() && !w_mask.size() && !r_active && !b_active && sample.io_outstanding == 0, - "DDR transport did not drain"); - for (int i = 0; i < 2; i++) ck(!sample.reply_valid[i], "Unmatched backing response"); - ck(checks == 9 && dma_checks == 4, "Missing checked backing/DMA transactions"); - `uvm_info("MEMORY_SYSTEM_PASS", $sformatf( - "backing checked=%0d DMA beats=%0d AXI reads/writes=%0d/%0d; owners, backpressure, final B, R/B errors and drain checked", - checks, - dma_checks, - read_bursts, - write_bursts - ), UVM_LOW) - ctl.done = 1; - end -endmodule diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/read_dma.cm_hier b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/read_dma.cm_hier deleted file mode 100644 index 772e1cfc5..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/read_dma.cm_hier +++ /dev/null @@ -1 +0,0 @@ -+tree @TOP@.dut diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/read_dma.f b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/read_dma.f deleted file mode 100644 index 179db17e9..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/read_dma.f +++ /dev/null @@ -1,7 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip/test -@VERIFY@/uvm/src/ip/test/control_if.sv -@VERIFY@/uvm/src/ip/test/control_test_package.sv --F @RTL@/ReadDma/filelist.f -@RESOURCES@/read_dma_tb.sv diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/read_dma_tb.sv b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/read_dma_tb.sv deleted file mode 100644 index 7ffc5b3fa..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/read_dma_tb.sv +++ /dev/null @@ -1,185 +0,0 @@ -module read_dma_tb; - import uvm_pkg::*; - import ip_control_test_pkg::*; - `include "uvm_macros.svh" - logic clock = 0, reset = 1; - always #5 clock = ~clock; - ip_control_if ctl (clock); - `include "read_dma_signals.svh" -ReadDma dut ( - `include "read_dma_ports.svh" - ); - `include "read_dma_clocking.svh" - int cases = 0, beats = 0, bursts = 0, checks = 0; - function automatic bit [127:0] data_at(longint unsigned address); - bit [127:0] value; - for (int i = 0; i < 16; ++i) value[i*8+:8] = 8'((address + i) ^ ((address + i) >> 8)); - return value; - endfunction - task automatic check(bit good, string reason); - checks++; - if (!good) `uvm_fatal("READ_DMA", reason) - endtask - task automatic run_case(longint unsigned base, int length, int groups, int stride, bit mode2d = 0, - int pixel = 16, int width = 1, int source = 1, int expected_error = 0, - bit bus_error = 0, int cancel_after = 0); - int count, seen, pending, issued, tick, fault_seen; - longint unsigned read_address, expected_address, query_address; - int query_bytes; - bit [127:0] held_data; - bit held, held_last; - bit [ 3:0] held_error; - bit [63:0] held_address; - count = (length + 15) / 16; - seen = 0; - pending = 0; - issued = 0; - fault_seen = 0; - held = 0; - cases++; - @(negedge clock); - io_req_bits_vaddr = base; - io_req_bits_len = length; - io_req_bits_groups = groups; - io_req_bits_stride = stride; - io_req_bits_is_2d = mode2d; - io_req_bits_pixel_bytes = pixel; - io_req_bits_tile_width = width; - io_req_bits_source_width = source; - io_req_valid = 1; - do @(sample); while (!sample.io_req_ready); - @(negedge clock); - io_req_valid = 0; - for (tick = 0; tick < 20000; ++tick) begin - io_mapping_hit = 1; - io_mapping_error = 0; - io_mapping_pa = io_query_va; - io_axi_ar_ready = pending == 0 && tick % 3 != 0; - io_axi_r_valid = pending != 0; - io_axi_r_bits_data = data_at(read_address); - io_axi_r_bits_last = pending == 1; - io_axi_r_bits_resp = bus_error && issued == 1 ? 2 : 0; - io_resp_ready = tick % 5 == 0; - @(sample); - if (held) - check( - sample.io_resp_valid && sample.io_resp_bits_data == held_data && - sample.io_resp_bits_last == held_last && sample.io_resp_bits_fault_error == held_error && - sample.io_resp_bits_fault_address == held_address, - "response changed under backpressure"); - held = sample.io_resp_valid && !sample.io_resp_ready; - held_data = sample.io_resp_bits_data; - held_last = sample.io_resp_bits_last; - held_error = sample.io_resp_bits_fault_error; - held_address = sample.io_resp_bits_fault_address; - if (sample.io_query_valid) begin - check( - !sample.io_query_write && sample.io_query_va[3:0] == 0 && - sample.io_query_bytes != 0 && sample.io_query_bytes % 16 == 0, - "invalid prepared footprint"); - query_address = sample.io_mapping_pa; - query_bytes = sample.io_query_bytes; - end - if (sample.io_axi_ar_valid && sample.io_axi_ar_ready) begin - check( - sample.io_axi_ar_bits_addr == query_address && - (int'(sample.io_axi_ar_bits_len)+1)*16 == query_bytes, - "burst differs from prepared footprint"); - check(sample.io_axi_ar_bits_size == 4 && sample.io_axi_ar_bits_burst == 1, - "AXI request must be 128-bit INCR"); - check( - sample.io_axi_ar_bits_addr[3:0] == 0 && - int'(sample.io_axi_ar_bits_addr[11:0]) + (int'(sample.io_axi_ar_bits_len)+1)*16 <= 4096, - "burst alignment/page boundary"); - pending = int'(sample.io_axi_ar_bits_len) + 1; - read_address = sample.io_axi_ar_bits_addr; - bursts++; - end - if (sample.io_axi_r_valid && sample.io_axi_r_ready) begin - pending--; - issued++; - read_address += 16; - end - if (sample.io_resp_valid && sample.io_resp_ready) begin - if (sample.io_resp_bits_fault_error != 0) begin - check(sample.io_resp_bits_fault_error == expected_error && sample.io_resp_bits_last, - "wrong terminal fault"); - check(pending == 0, "fault completed before accepted burst drained"); - fault_seen++; - end else begin - check(expected_error == 0 || bus_error, "shape failure dispatched data"); - expected_address = mode2d ? base + (seen/width)*source*pixel + (seen%width)*pixel : - base + (seen/groups)*groups*stride*16 + (seen%groups)*16; - check(sample.io_resp_bits_data == data_at(expected_address), - "data/stride/unaligned assembly mismatch"); - check(sample.io_resp_bits_addrcounter == seen, "beat index mismatch"); - if (!bus_error) check(sample.io_resp_bits_last == (seen + 1 == count), "last mismatch"); - seen++; - beats++; - end - if (sample.io_resp_bits_last) begin - check(fault_seen == (expected_error != 0), "missing/unexpected terminal error"); - if (!expected_error) check(seen == count, "incomplete response"); - @(negedge clock); - io_resp_ready = 0; - io_axi_r_valid = 0; - io_axi_ar_ready = 0; - return; - end - end - @(negedge clock); - if (cancel_after != 0 && tick == cancel_after) begin - check(issued == 0, "reset fixture must precede AXI issue"); - reset = 1; - io_axi_r_valid = 0; - io_axi_ar_ready = 0; - repeat (3) @(sample); - @(negedge clock); - reset = 0; - repeat (4) begin - @(sample); - check(!sample.io_resp_valid && !sample.io_axi_ar_valid && !sample.io_busy, - "reset leaked old work"); - end - return; - end - end - `uvm_fatal("READ_DMA", "request timed out") - endtask - initial begin - uvm_config_db#(virtual ip_control_if)::set(null, "*", "vif", ctl); - uvm_config_db#(time)::set(null, "uvm_test_top", "timeout", 2ms); - run_test("protocol_test"); - end - initial begin - `include "read_dma_init.svh" - wait (ctl.start); - repeat (4) @(sample); - @(negedge clock); - reset = 0; - io_decisionValid = 1; - run_case('h1000, 16, 1, 1); - run_case('h1ff0, 16 * 257, 1, 1); - run_case('h3003, 53, 3, 5); - run_case('h4ff9, 17, 1, 1); - run_case('h6000, 16 * 19, 3, 7); - run_case('h8000, 16 * 64, 63, 524287); - run_case('h8000, 16 * 63, 63, 524287); - run_case('h9000, 16 * 20, 1, 1, 1, 7, 3, 7); - run_case('haff0, 16 * 20, 1, 1, 1, 16, 3, 7); - run_case('hb000, 16 * 17, 1, 1, 1, 1023, 15, 1023); - run_case(64'hfffffffffffffff0, 16, 1, 1, 0, 16, 1, 1, 0); - run_case(64'hfffffffffffffff1, 16, 1, 1, 0, 16, 1, 1, 7); - run_case(64'hfffffff000000000, 16 * 65536, 63, 524287, 0, 16, 1, 1, 7); - run_case(64'hfffffffffff00000, 16 * 65536, 1, 1, 1, 1023, 15, 1023, 7); - run_case('hc000, 0, 1, 1, 0, 16, 1, 1, 7); - run_case('hc000, 16, 0, 1, 0, 16, 1, 1, 7); - run_case('hc000, 16, 1, 1, 1, 0, 3, 3, 7); - run_case('hd000, 64, 1, 1, 0, 16, 1, 1, 2, 1); - run_case('he000, 256, 3, 1, 0, 16, 1, 1, 0, 0, 5); - run_case('hf000, 32, 1, 1); - `uvm_info("READ_DMA", $sformatf( - "%0d cases %0d checks %0d beats %0d bursts", cases, checks, beats, bursts), UVM_LOW) - ctl.done = 1; - end -endmodule diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/shared_lease.cm_hier b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/shared_lease.cm_hier deleted file mode 100644 index 772e1cfc5..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/shared_lease.cm_hier +++ /dev/null @@ -1 +0,0 @@ -+tree @TOP@.dut diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/shared_lease.f b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/shared_lease.f deleted file mode 100644 index 8e2244b16..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/shared_lease.f +++ /dev/null @@ -1,7 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip/test -@VERIFY@/uvm/src/ip/test/control_if.sv -@VERIFY@/uvm/src/ip/test/control_test_package.sv --F @RTL@/SharedMemBackend/filelist.f -@RESOURCES@/shared_lease_tb.sv diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/shared_lease_tb.sv b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/shared_lease_tb.sv deleted file mode 100644 index 3052eaa84..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/shared_lease_tb.sv +++ /dev/null @@ -1,163 +0,0 @@ -module shared_lease_tb; - import uvm_pkg::*; - import ip_control_test_pkg::*; - `include "uvm_macros.svh" - logic clock = 0, reset = 1; - always #5 clock = ~clock; - ip_control_if ctl (clock); - `include "shared_lease_signals.svh" -SharedMemBackend dut ( - `include "shared_lease_ports.svh" - ); - `include "shared_lease_clocking.svh" - int checks = 0, invalid_case = 0; - task automatic check(bit good, string reason); - checks++; - if (!good) `uvm_fatal("SHARED_LEASE", reason) - endtask - task automatic configure(bit alloc, int bank, int group_id = 0, bit transfer = 0, int source = 0); - @(negedge clock); - io_config_bits_alloc = alloc; - io_config_bits_vbank_id = bank; - io_config_bits_group_id = group_id; - io_config_bits_transfer = transfer; - io_config_bits_source_bank_id = source; - io_config_valid = 1; - do @(sample); while (!sample.io_config_ready); - @(negedge clock); - io_config_valid = 0; - endtask - task automatic request(bit release_lease, int bank = 8, int group_id = 0); - @(negedge clock); - io_lease_request_bits_release = release_lease; - io_lease_request_bits_vbank = bank; - io_lease_request_bits_group = group_id; - io_lease_request_valid = 1; - do @(sample); while (!sample.io_lease_request_ready); - @(negedge clock); - io_lease_request_valid = 0; - endtask - task automatic reply(longint unsigned expected); - repeat (5) begin - @(sample); - check(sample.io_lease_response_valid && sample.io_lease_response_bits == expected, - "lease response must remain stable under backpressure"); - check(!sample.io_lease_request_ready, "accepted second lease before response"); - end - @(negedge clock); - io_lease_response_ready = 1; - @(sample); - check(sample.io_lease_response_valid, "response lost"); - @(negedge clock); - io_lease_response_ready = 0; - endtask - task automatic data_access(bit write, bit [127:0] value); - @(negedge clock); - io_physical_0_request_valid = 1; - io_physical_0_request_bits_address = 128; - io_physical_0_request_bits_write = write; - io_physical_0_request_bits_data = value; - io_physical_0_request_bits_mask = '1; - do @(sample); while (!sample.io_physical_0_request_ready); - @(negedge clock); - io_physical_0_request_valid = 0; - do @(sample); while (!sample.io_physical_0_response_valid); - check(!sample.io_physical_0_response_bits_error, "leased bank data access failed"); - if (!write) - check(sample.io_physical_0_response_bits_data == value, "leased bank data mismatch"); - @(negedge clock); - io_physical_0_response_ready = 1; - @(sample); - @(negedge clock); - io_physical_0_response_ready = 0; - endtask - initial begin - uvm_config_db#(virtual ip_control_if)::set(null, "*", "vif", ctl); - uvm_config_db#(time)::set(null, "uvm_test_top", "timeout", 100us); - run_test("protocol_test"); - end - initial begin - `include "shared_lease_init.svh" - void'($value$plusargs("invalid_case=%d", invalid_case)); - wait (ctl.start); - repeat (4) @(sample); - @(negedge clock); - reset = 0; - io_config_bits_hart_id = 64'h100000002; - io_config_bits_is_shared = 1; - io_leaseOwnerHartId = 64'h100000002; - configure(1, 8); - configure(1, 8, 1); - request(0, 8, 1); - reply(128); - if (invalid_case == 1) configure(0, 8); - if (invalid_case == 2) configure(1, 8); - if (invalid_case == 3) configure(0, 9, 0, 1, 8); - if (invalid_case == 4) begin - request(1, 8, 0); - end - if (invalid_case == 5) begin - request(0, 10, 0); - end - if (invalid_case) begin - repeat (8) @(sample); - `uvm_fatal("SHARED_LEASE", "expected RTL assertion absent") - end - request(0, 8, 1); - reply(128); - request(1, 8, 1); - reply(0); - data_access(1, 128'hffeeddccbbaa99887766554433221100); - data_access(0, 128'hffeeddccbbaa99887766554433221100); - request(1, 8, 1); - reply(0); - // Config wins when both are offered; lease then observes the new mapping. - @(negedge clock); - io_config_valid = 1; - io_config_bits_alloc = 1; - io_config_bits_vbank_id = 9; - io_config_bits_group_id = 0; - io_lease_request_valid = 1; - io_lease_request_bits_release = 0; - io_lease_request_bits_vbank = 9; - io_lease_request_bits_group = 0; - @(sample); - check(sample.io_config_ready && !sample.io_lease_request_ready, "config priority deadlock"); - @(negedge clock); - io_config_valid = 0; - @(sample); - check(sample.io_lease_request_ready, "lease not accepted after config"); - @(negedge clock); - io_lease_request_valid = 0; - reply(256); - request(1, 9); - reply(0); - configure(0, 9, 0, 1, 8); - request(0, 9, 2); - reply(128); - request(1, 9, 2); - reply(0); - configure(0, 9); - configure(1, 8); - request(0); // Reset discards an unconsumed response and its lease. - @(negedge clock); - reset = 1; - repeat (3) @(sample); - @(negedge clock); - reset = 0; - repeat (3) begin - @(sample); - check(!sample.io_lease_response_valid, "old response survived reset"); - end - configure(1, 8); - request(0); - reply(0); - request(1); - reply(0); - configure(0, 8); - `uvm_info("SHARED_LEASE", $sformatf( - "%0d checks: leases, data sharing, config priority, backpressure, reset", checks), - UVM_LOW) - ctl.done = 1; - end -endmodule diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/storer_ack.f b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/storer_ack.f deleted file mode 100644 index 4f677251e..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/storer_ack.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip/test -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/test/control_if.sv -@VERIFY@/uvm/src/ip/test/control_test_package.sv -@RESOURCES@/ack_package.sv --F @RTL@/MemStorer/filelist.f -@RESOURCES@/storer_ack_tb.sv diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/storer_ack_tb.sv b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/storer_ack_tb.sv deleted file mode 100644 index cebf09d58..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/storer_ack_tb.sv +++ /dev/null @@ -1,366 +0,0 @@ -module storer_ack_tb; - import uvm_pkg::*; - import ack_pkg::*; - import ip_control_test_pkg::*; - `include "uvm_macros.svh" - logic clock = 0, reset = 0; - always #5 clock = ~clock; - ip_control_if ctl (clock); - `include "storer_ack_signals.svh" -stream_if #( - .WIDTH($bits( - io_cmdResp_bits_rob_id - ) + $bits( - io_cmdResp_bits_is_sub - ) + $bits( - io_cmdResp_bits_sub_rob_id - ) + $bits( - io_cmdResp_bits_fault_error - ) + $bits( - io_cmdResp_bits_fault_address - )) - ) sink_if ( - clock, - reset - ); - assign sink_if.valid = io_cmdResp_valid; - assign sink_if.ready = io_cmdResp_ready; - assign sink_if.bits = { - io_cmdResp_bits_rob_id, - io_cmdResp_bits_is_sub, - io_cmdResp_bits_sub_rob_id, - io_cmdResp_bits_fault_error, - io_cmdResp_bits_fault_address - }; - MemStorer dut ( - `include "storer_ack_ports.svh" - ); - stream_if #( - .WIDTH($bits( - io_cmdReq_bits_cmd_special - ) + $bits( - io_cmdReq_bits_cmd_mem_addr - ) + $bits( - io_cmdReq_bits_cmd_iter - ) + $bits( - io_cmdReq_bits_cmd_bank_id - )) - ) source_if ( - clock, - reset - ); - assign source_if.valid = io_cmdReq_valid; - assign source_if.ready = io_cmdReq_ready; - assign source_if.bits = { - io_cmdReq_bits_cmd_special, - io_cmdReq_bits_cmd_mem_addr, - io_cmdReq_bits_cmd_iter, - io_cmdReq_bits_cmd_bank_id - }; - stream_if #( - .WIDTH($bits( - io_footprint_rob_id - ) + $bits( - io_footprint_is_sub - ) + $bits( - io_footprint_sub_rob_id - ) + $bits( - io_footprint_baseVA - ) + $bits( - io_footprint_rows - ) + $bits( - io_footprint_columns - ) + $bits( - io_footprint_spanBytes - ) + $bits( - io_footprint_columnStride - ) + $bits( - io_footprint_rowStride - ) + $bits( - io_footprint_write - ) + $bits( - io_footprint_fault_error - ) + $bits( - io_footprint_fault_address - )) - ) footprint_if ( - clock, - reset - ); - assign footprint_if.valid = io_footprint_valid; - assign footprint_if.ready = io_cmdResp_valid && io_cmdResp_ready; - assign footprint_if.bits = { - io_footprint_rob_id, - io_footprint_is_sub, - io_footprint_sub_rob_id, - io_footprint_baseVA, - io_footprint_rows, - io_footprint_columns, - io_footprint_spanBytes, - io_footprint_columnStride, - io_footprint_rowStride, - io_footprint_write, - io_footprint_fault_error, - io_footprint_fault_address - }; - chandle model; - int completions = 0, store_cases = 0, descriptor_count = 0, data_count = 0; - logic [127:0] expected; - longint unsigned descriptor_address; - int descriptor_bytes; - logic [127:0] descriptor_data[$]; - function automatic logic [127:0] row_data(int row); - logic [127:0] data; - for (int j = 0; j < 16; j++) data[j*8+:8] = (row * 37 + j * 11 + 3) & 255; - return data; - endfunction - task automatic check(bit condition, string message); - if (!condition) `uvm_fatal("ACK_CONTRACT", message) - endtask - task automatic no_completion(int cycles); - repeat (cycles) begin - @(posedge clock); - #1; - check(!io_cmdResp_valid, "cmdResp.valid asserted before final descriptor ACK"); - check(!io_cmdReq_ready, "accepted next command before completion"); - check(completions == 0, "unexpected completion count before descriptor ACK"); - end - endtask - task automatic complete_command(int error = 0, longint unsigned address = 0); - wait (io_cmdResp_valid); - repeat (8) begin - @(posedge clock); - #1; - check( - io_cmdResp_valid && io_cmdResp_bits_rob_id == 3 && io_cmdResp_bits_is_sub == 1 && io_cmdResp_bits_sub_rob_id == 2, - "completion changed under backpressure"); - check(io_cmdResp_bits_fault_error == error && io_cmdResp_bits_fault_address == address, - "completion fault changed under backpressure"); - check(!io_cmdReq_ready, "command accepted while completion stalled"); - end - @(negedge clock); - io_cmdResp_ready = 1; - @(posedge clock); - check(io_cmdResp_valid, "completion lost before handshake"); - completions++; - @(negedge clock); - io_cmdResp_ready = 0; - repeat (8) begin - @(posedge clock); - #1; - check(!io_cmdResp_valid, "duplicate completion"); - end - check(completions == 1 && io_cmdReq_ready, "completion count or idle readiness wrong"); - endtask - task automatic begin_store(longint unsigned address, int rows, int groups = 1, int stride = 1); - completions = 0; - store_cases++; - descriptor_count = 0; - data_count = 0; - @(negedge clock); - io_cmdReq_bits_cmd_mem_addr = address; - io_cmdReq_bits_cmd_iter = rows; - io_query_group_count = groups; - io_cmdReq_bits_cmd_special = 64'(stride) << 39; - io_dmaResp_bits_fault_error = 0; - io_dmaResp_bits_fault_address = 0; - io_cmdReq_valid = 1; - do @(posedge clock); while (!io_cmdReq_ready); - @(negedge clock); - io_cmdReq_valid = 0; - endtask - task automatic accept_descriptor(longint unsigned address, int bytes); - wait (io_dmaReq_valid); - repeat (4) begin - @(posedge clock); - #1; - check(io_dmaReq_valid && io_dmaReq_bits_vaddr == address && io_dmaReq_bits_len == bytes, - "whole-descriptor address/length changed under backpressure"); - check(!io_cmdResp_valid, "descriptor offer released Core completion"); - end - @(negedge clock); - io_dmaReq_ready = 1; - @(posedge clock); - check(io_dmaReq_valid, "missing descriptor handshake"); - descriptor_address = io_dmaReq_bits_vaddr; - descriptor_bytes = io_dmaReq_bits_len; - descriptor_data.delete(); - descriptor_count++; - @(negedge clock); - io_dmaReq_ready = 0; - endtask - task automatic bank_and_data(int row, int group, logic [127:0] data, bit last); - wait (io_bankRead_io_req_valid); - check( - io_bankRead_io_req_bits_addr == row && io_bankRead_group_id == group && - io_bankRead_bank_id == 1 && io_bankRead_rob_id == 3, - "bank row/group/tag wrong"); - repeat (2) begin - @(posedge clock); - #1; - check( - io_bankRead_io_req_valid && io_bankRead_io_req_bits_addr == row && - io_bankRead_group_id == group, - "bank read changed under backpressure"); - end - @(negedge clock); - io_bankRead_io_req_ready = 1; - @(posedge clock); - check(io_bankRead_io_req_valid, "missing bank read handshake"); - @(negedge clock); - io_bankRead_io_req_ready = 0; - no_completion(2); - @(negedge clock); - io_bankRead_io_resp_bits_data = data; - io_bankRead_io_resp_valid = 1; - do @(posedge clock); while (!io_bankRead_io_resp_ready); - @(negedge clock); - io_bankRead_io_resp_valid = 0; - wait (io_dmaData_valid); - repeat (3) begin - @(posedge clock); - #1; - check(io_dmaData_valid && io_dmaData_bits_data === data && io_dmaData_bits_last == last, - "raw 128-bit bank stream changed under backpressure or was shifted for alignment"); - check(!io_dmaReq_valid, "duplicate descriptor emitted during its stream"); - check(!io_cmdResp_valid, "Core completion preceded final descriptor ACK"); - end - @(negedge clock); - io_dmaData_ready = 1; - @(posedge clock); - check(io_dmaData_valid, "missing raw DMA data handshake"); - descriptor_data.push_back(io_dmaData_bits_data); - data_count++; - @(negedge clock); - io_dmaData_ready = 0; - endtask - task automatic feed_descriptor(longint unsigned base, int rows, int groups = 1, int stride = 1, - int first_row = 0, int descriptor_rows = 0); - int count; - count = descriptor_rows == 0 ? rows : descriptor_rows; - accept_descriptor(base + first_row * groups * stride * 16, count * groups * 16); - check( - io_footprint_valid && io_footprint_baseVA == base && io_footprint_rows == rows && - io_footprint_columns == 1 && io_footprint_spanBytes == groups * 16 && - io_footprint_columnStride == 0 && io_footprint_rowStride == groups * stride * 16 && - io_footprint_fault_error == 0 && io_footprint_write, - "footprint differs from the complete store command"); - for (int row = first_row; row < first_row + count; row++) - for (int group = 0; group < groups; group++) - bank_and_data(row, group, row_data(row * groups + group), - row == first_row + count - 1 && group == groups - 1); - check(descriptor_data.size() * 16 == descriptor_bytes, "stream length differs from descriptor"); - endtask - task automatic dma_ack(bit done, int error = 0, longint unsigned address = 0); - no_completion(32); - if (done && error == 0) - foreach (descriptor_data[index]) - ack_ref_write(model, descriptor_address + index * 16, descriptor_data[index][63:0], - descriptor_data[index][127:64], 'hffff); - @(negedge clock); - io_dmaResp_bits_done = done; - io_dmaResp_bits_fault_error = error; - io_dmaResp_bits_fault_address = address; - io_dmaResp_valid = 1; - do @(posedge clock); while (!io_dmaResp_ready); - @(negedge clock); - io_dmaResp_valid = 0; - endtask - task automatic no_future_requests(); - repeat (8) begin - @(posedge clock); - #1; - check(!io_dmaReq_valid && !io_dmaData_valid && !io_bankRead_io_req_valid, - "failed final ACK launched residual DMA data/descriptor or a future bank read"); - end - endtask - task automatic recovery(longint unsigned address); - begin_store(address, 1); - feed_descriptor(address, 1); - dma_ack(1); - complete_command(); - check(descriptor_count == 1 && data_count == 1, - "recovery reused previous row/descriptor state"); - endtask - initial begin - uvm_config_db#(virtual ip_control_if)::set(null, "*", "vif", ctl); - uvm_config_db#(time)::set(null, "uvm_test_top", "timeout", 100us); - run_test("protocol_test"); - end - initial begin - logic [127:0] partial; - `include "storer_ack_init.svh" - wait (ctl.start); - model = ack_ref_create(); - @(negedge clock); - reset = 1; - repeat (4) @(posedge clock); - @(negedge clock); - reset = 0; - io_cmdReq_bits_cmd_bank_id = 1; - io_cmdReq_bits_rob_id = 3; - io_cmdReq_bits_is_sub = 1; - io_cmdReq_bits_sub_rob_id = 2; - io_cmdReq_bits_cmd_is_store = 1; - begin_store('h1000, 4); - feed_descriptor('h1000, 4); - check(descriptor_count == 1 && data_count == 4, "stride-one rows were not one long descriptor"); - dma_ack(1); - complete_command(); - for (int row = 0; row < 4; row++) begin - expected = row_data(row); - check(ack_ref_check(model, 'h1000 + row * 16, expected[63:0], expected[127:64]) == 1, - "DDR golden after final completion failed"); - end - // The packing/partial-burst failure belongs to WriteDma, not MemStorer. - begin_store('h4003, 4); - feed_descriptor('h4003, 4); - dma_ack(0, 3, 'h4000); - no_future_requests(); - complete_command(3, 'h4000); - recovery('h4100); - // Preserve already committed bytes when a later DMA segment faults. - begin_store('h5003, 4); - feed_descriptor('h5003, 4); - partial = ~row_data(0); - ack_ref_write(model, 'h5003, partial[63:0], partial[127:64], 'hffff); - expected = row_data(0); - ack_ref_write(model, 'h5003, expected[63:0], expected[127:64], 'h1fff); - partial[103:0] = expected[103:0]; - dma_ack(0, 4, 'h5010); - no_future_requests(); - complete_command(4, 'h5010); - check(ack_ref_check(model, 'h5003, partial[63:0], partial[127:64]) == 1, - "later-segment fault rolled back prior committed bytes"); - recovery('h5100); - begin_store('h6000, 1); - feed_descriptor('h6000, 1); - dma_ack(0); - no_future_requests(); - complete_command(6, 'h6000); - recovery('h6100); - // The descriptor keeps the unaligned VA, and data is sent unshifted once. - begin_store('h7003, 1); - feed_descriptor('h7003, 1); - dma_ack(1); - complete_command(); - expected = row_data(0); - check(descriptor_count == 1 && data_count == 1 && ack_ref_check( - model, 'h7003, expected[63:0], expected[127:64]) == 1, - "unaligned store changed raw bytes or emitted a second descriptor"); - // Each strided row has one groups*16-byte descriptor; fault suppresses row1. - begin_store('h8000, 2, 2, 3); - feed_descriptor('h8000, 2, 2, 3, 0, 1); - dma_ack(0, 3, 'h8000); - no_future_requests(); - complete_command(3, 'h8000); - check(descriptor_count == 1 && data_count == 2, "strided row did not group its raw bank beats"); - recovery('h8100); - check(store_cases == 10, "original store/recovery case count changed"); - ack_ref_destroy(model); - `uvm_info("ACK_GATE", - "10 existing cases: descriptor-final ACK delay, raw128 stable stream, typed fault/protocol fault, strided drain, stable completion and recovery", - UVM_LOW) - ctl.done = 1; - end -endmodule diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_axi_peer.svh b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_axi_peer.svh deleted file mode 100644 index 062488e99..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_axi_peer.svh +++ /dev/null @@ -1,103 +0,0 @@ -typedef struct { - bit [63:0] addr; - int beat; -} axi_read_t; -typedef struct { - bit [63:0] addr; - int due; -} axi_write_t; -axi_read_t reads[int]; -axi_write_t writes[int]; -int aw_ids[$]; -bit [63:0] aw_addresses[$]; -bit [511:0] w_lines[$]; -bit [63:0] w_masks[$]; -bit [511:0] assembly; -bit [63:0] assembly_mask; -int w_beat = 0, r_id = 0, b_id = 0, r_cursor = 0; -bit r_active = 0, b_active = 0; -int axi_reads = 0, axi_writes = 0, final_b_waits = 0; -task automatic axi_service(); - forever begin - @(sample); - if (!sample.reset) begin - if (sample.io_axi_aw_valid && sample.io_axi_aw_ready) begin - ck( - sample.io_axi_aw_bits_len==3&&sample.io_axi_aw_bits_size==4&&sample.io_axi_aw_bits_burst==1&&!sample.io_axi_aw_bits_lock, - "Actual DDR AW line burst"); - aw_ids.push_back(sample.io_axi_aw_bits_id); - aw_addresses.push_back(sample.io_axi_aw_bits_addr); - end - if (sample.io_axi_w_valid && sample.io_axi_w_ready) begin - ck(sample.io_axi_w_bits_last == (w_beat == 3), "Actual DDR WLAST"); - assembly[w_beat*128+:128] = sample.io_axi_w_bits_data; - assembly_mask[w_beat*16+:16] = sample.io_axi_w_bits_strb; - if (w_beat == 3) begin - w_lines.push_back(assembly); - w_masks.push_back(assembly_mask); - w_beat = 0; - end else w_beat++; - end - if (aw_ids.size() && w_lines.size()) begin - int id = aw_ids.pop_front(); - bit [63:0] addr = aw_addresses.pop_front(); - bit [511:0] data = w_lines.pop_front(); - bit [63:0] mask = w_masks.pop_front(); - ck(!reads.exists(id) && !writes.exists(id), "Actual DDR active AXI ID reused for write"); - for (int beat = 0; beat < 4; beat++) - tile_peer_write128(addr + 16 * beat, data[128*beat+:64], data[128*beat+64+:64], - int'(mask[16*beat+:16])); - writes[id] = '{addr, cycles + 12}; - axi_writes++; - end - if (sample.io_axi_ar_valid && sample.io_axi_ar_ready) begin - int id = sample.io_axi_ar_bits_id; - bit [63:0] addr = sample.io_axi_ar_bits_addr; - ck( - sample.io_axi_ar_bits_len==3&&sample.io_axi_ar_bits_size==4&&sample.io_axi_ar_bits_burst==1&&!sample.io_axi_ar_bits_lock, - "Actual DDR AR line burst"); - ck(!reads.exists(id) && !writes.exists(id), "Actual DDR active AXI ID reused for read"); - reads[id] = '{addr, 0}; - axi_reads++; - end - if (sample.io_axi_r_valid && sample.io_axi_r_ready) begin - if (reads[r_id].beat == 3) reads.delete(r_id); - else reads[r_id].beat++; - r_active = 0; - r_cursor = (r_id + 1) % 16; - end - if (sample.io_axi_b_valid && sample.io_axi_b_ready) begin - writes.delete(b_id); - b_active = 0; - end - foreach (writes[id]) if (writes[id].due > cycles) final_b_waits++; - end - @(negedge clock); - io_axi_aw_ready = !reset && cycles % 5 != 0; - io_axi_w_ready = !reset && cycles % 7 >= 2; - io_axi_ar_ready = !reset && cycles % 4 != 0; - if (!r_active) - for (int off = 0; off < 16; off++) begin - int id = (r_cursor + off) % 16; - if (!r_active && reads.exists(id)) begin - bit [63:0] addr = reads[id].addr + 16 * reads[id].beat; - r_active = 1; - r_id = id; - io_axi_r_bits_id = id; - io_axi_r_bits_data = {tile_peer_read64(addr + 8), tile_peer_read64(addr)}; - io_axi_r_bits_last = reads[id].beat == 3; - io_axi_r_bits_resp = 0; - end - end - if (!b_active) - for (int id = 15; id >= 0; id--) - if (!b_active && writes.exists(id) && writes[id].due <= cycles) begin - b_active = 1; - b_id = id; - io_axi_b_bits_id = id; - io_axi_b_bits_resp = 0; - end - io_axi_r_valid = !reset && r_active; - io_axi_b_valid = !reset && b_active; - end -endtask diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_four.cm_hier b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_four.cm_hier deleted file mode 100644 index 772e1cfc5..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_four.cm_hier +++ /dev/null @@ -1 +0,0 @@ -+tree @TOP@.dut diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_four.f b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_four.f deleted file mode 100644 index 6cb2ed1e1..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_four.f +++ /dev/null @@ -1,7 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip/test -@VERIFY@/uvm/src/ip/test/control_if.sv -@VERIFY@/uvm/src/ip/test/control_test_package.sv --F @RTL@/TileSystem/filelist.f -@RESOURCES@/tile_system_four_tb.sv diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_four.toml b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_four.toml deleted file mode 100644 index eb0e075b4..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_four.toml +++ /dev/null @@ -1 +0,0 @@ -chips = ["goban"] diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_four_config.svh b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_four_config.svh deleted file mode 100644 index c10702927..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_four_config.svh +++ /dev/null @@ -1,2 +0,0 @@ -`define TILE_IMAGE_PATH "../../../../../../../bb-tests/output/goban/workloads/src/CTest/chip/goban/goban-goban-ctest-chip-task_dma_four-multicore-baremetal" -`define TILE_EXPECT_PATH "../../../../../../../bb-tests/output/goban/workloads/src/CTest/chip/goban/goban-goban-ctest-chip-task_dma_four-multicore-baremetal.expect" diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_four_tb.sv b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_four_tb.sv deleted file mode 100644 index acfb1599b..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_four_tb.sv +++ /dev/null @@ -1,4 +0,0 @@ -`include "tile_system_four_config.svh" -module tile_system_four_tb; - `include "tile_system_tb.svh" -endmodule diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_single.cm_hier b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_single.cm_hier deleted file mode 100644 index 772e1cfc5..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_single.cm_hier +++ /dev/null @@ -1 +0,0 @@ -+tree @TOP@.dut diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_single.f b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_single.f deleted file mode 100644 index 6c26f8d5d..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_single.f +++ /dev/null @@ -1,7 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip/test -@VERIFY@/uvm/src/ip/test/control_if.sv -@VERIFY@/uvm/src/ip/test/control_test_package.sv --F @RTL@/TileSystem/filelist.f -@RESOURCES@/tile_system_single_tb.sv diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_single.toml b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_single.toml deleted file mode 100644 index eb0e075b4..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_single.toml +++ /dev/null @@ -1 +0,0 @@ -chips = ["goban"] diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_single_config.svh b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_single_config.svh deleted file mode 100644 index 6553cf780..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_single_config.svh +++ /dev/null @@ -1,2 +0,0 @@ -`define TILE_IMAGE_PATH "../../../../../../../bb-tests/output/goban/workloads/src/CTest/chip/goban/goban-goban-ctest-chip-task_dma_single-multicore-baremetal" -`define TILE_EXPECT_PATH "../../../../../../../bb-tests/output/goban/workloads/src/CTest/chip/goban/goban-goban-ctest-chip-task_dma_single-multicore-baremetal.expect" diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_single_tb.sv b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_single_tb.sv deleted file mode 100644 index 49883f777..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_single_tb.sv +++ /dev/null @@ -1,4 +0,0 @@ -`include "tile_system_single_config.svh" -module tile_system_single_tb; - `include "tile_system_tb.svh" -endmodule diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_tb.svh b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_tb.svh deleted file mode 100644 index ec2e42a74..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_tb.svh +++ /dev/null @@ -1,318 +0,0 @@ -`include "tile_system_config.svh" -import uvm_pkg::*; -import ip_control_test_pkg::*; -`include "uvm_macros.svh" -import "DPI-C" function longint unsigned tile_memory_init( - input string imagePath, - input string expectedPath -); -import "DPI-C" function longint unsigned tile_peer_read64(input longint unsigned address); -import "DPI-C" function void tile_peer_write128( - input longint unsigned address, - input longint unsigned lo, - input longint unsigned hi, - input int unsigned mask -); -import "DPI-C" function void tile_peer_finish(); -import "DPI-C" function void dpi_bdb_set_clk(input longint unsigned cycle); -logic clock = 0, reset = 1; -always #5 clock = ~clock; -ip_control_if ctl (clock); -`include "tile_system_signals.svh" -TileSystem dut ( - `include "tile_system_ports.svh" -); -logic device_valid[5], device_ready[5], device_write[5], device_normal[5]; -logic [63:0] device_addr[5], device_data[5]; -logic [5:0] device_tag[5]; -logic [2:0] device_size[5]; -logic [3:0] device_atomic[5]; -logic dev_response_valid[5], dev_response_ready[5]; -logic [ 5:0] dev_response_tag [5]; -logic [63:0] dev_response_data[5]; -logic retired[5], trapped[5], failure[5]; -logic [63:0] retired_pc[5], trap_pc[5], trap_cause[5], trap_value[5]; -assign device_valid[0] = io_deviceRequest_0_valid; -assign device_addr[0] = io_deviceRequest_0_bits_addr; -assign device_data[0] = io_deviceRequest_0_bits_data; -assign device_tag[0] = io_deviceRequest_0_bits_tag; -assign device_size[0] = io_deviceRequest_0_bits_size; -assign device_atomic[0] = io_deviceRequest_0_bits_atomic; -assign device_write[0] = io_deviceRequest_0_bits_write; -assign device_normal[0] = io_deviceRequest_0_bits_normal; -assign io_deviceRequest_0_ready = device_ready[0]; -assign dev_response_ready[0] = io_deviceResponse_0_ready; -assign io_deviceResponse_0_valid = dev_response_valid[0]; -assign io_deviceResponse_0_bits_tag = dev_response_tag[0]; -assign io_deviceResponse_0_bits_data = dev_response_data[0]; -assign io_deviceResponse_0_bits_error = 0; -assign retired[0] = io_retired_0; -assign retired_pc[0] = io_retiredPc_0; -assign trapped[0] = io_trapped_0; -assign trap_pc[0] = io_trapPc_0; -assign trap_cause[0] = io_trapCause_0; -assign trap_value[0] = io_trapValue_0; -assign failure[0] = io_failure_0_valid; -assign device_valid[1] = io_deviceRequest_1_valid; -assign device_addr[1] = io_deviceRequest_1_bits_addr; -assign device_data[1] = io_deviceRequest_1_bits_data; -assign device_tag[1] = io_deviceRequest_1_bits_tag; -assign device_size[1] = io_deviceRequest_1_bits_size; -assign device_atomic[1] = io_deviceRequest_1_bits_atomic; -assign device_write[1] = io_deviceRequest_1_bits_write; -assign device_normal[1] = io_deviceRequest_1_bits_normal; -assign io_deviceRequest_1_ready = device_ready[1]; -assign dev_response_ready[1] = io_deviceResponse_1_ready; -assign io_deviceResponse_1_valid = dev_response_valid[1]; -assign io_deviceResponse_1_bits_tag = dev_response_tag[1]; -assign io_deviceResponse_1_bits_data = dev_response_data[1]; -assign io_deviceResponse_1_bits_error = 0; -assign retired[1] = io_retired_1; -assign retired_pc[1] = io_retiredPc_1; -assign trapped[1] = io_trapped_1; -assign trap_pc[1] = io_trapPc_1; -assign trap_cause[1] = io_trapCause_1; -assign trap_value[1] = io_trapValue_1; -assign failure[1] = io_failure_1_valid; -assign device_valid[2] = io_deviceRequest_2_valid; -assign device_addr[2] = io_deviceRequest_2_bits_addr; -assign device_data[2] = io_deviceRequest_2_bits_data; -assign device_tag[2] = io_deviceRequest_2_bits_tag; -assign device_size[2] = io_deviceRequest_2_bits_size; -assign device_atomic[2] = io_deviceRequest_2_bits_atomic; -assign device_write[2] = io_deviceRequest_2_bits_write; -assign device_normal[2] = io_deviceRequest_2_bits_normal; -assign io_deviceRequest_2_ready = device_ready[2]; -assign dev_response_ready[2] = io_deviceResponse_2_ready; -assign io_deviceResponse_2_valid = dev_response_valid[2]; -assign io_deviceResponse_2_bits_tag = dev_response_tag[2]; -assign io_deviceResponse_2_bits_data = dev_response_data[2]; -assign io_deviceResponse_2_bits_error = 0; -assign retired[2] = io_retired_2; -assign retired_pc[2] = io_retiredPc_2; -assign trapped[2] = io_trapped_2; -assign trap_pc[2] = io_trapPc_2; -assign trap_cause[2] = io_trapCause_2; -assign trap_value[2] = io_trapValue_2; -assign failure[2] = io_failure_2_valid; -assign device_valid[3] = io_deviceRequest_3_valid; -assign device_addr[3] = io_deviceRequest_3_bits_addr; -assign device_data[3] = io_deviceRequest_3_bits_data; -assign device_tag[3] = io_deviceRequest_3_bits_tag; -assign device_size[3] = io_deviceRequest_3_bits_size; -assign device_atomic[3] = io_deviceRequest_3_bits_atomic; -assign device_write[3] = io_deviceRequest_3_bits_write; -assign device_normal[3] = io_deviceRequest_3_bits_normal; -assign io_deviceRequest_3_ready = device_ready[3]; -assign dev_response_ready[3] = io_deviceResponse_3_ready; -assign io_deviceResponse_3_valid = dev_response_valid[3]; -assign io_deviceResponse_3_bits_tag = dev_response_tag[3]; -assign io_deviceResponse_3_bits_data = dev_response_data[3]; -assign io_deviceResponse_3_bits_error = 0; -assign retired[3] = io_retired_3; -assign retired_pc[3] = io_retiredPc_3; -assign trapped[3] = io_trapped_3; -assign trap_pc[3] = io_trapPc_3; -assign trap_cause[3] = io_trapCause_3; -assign trap_value[3] = io_trapValue_3; -assign failure[3] = io_failure_3_valid; -assign device_valid[4] = io_deviceRequest_4_valid; -assign device_addr[4] = io_deviceRequest_4_bits_addr; -assign device_data[4] = io_deviceRequest_4_bits_data; -assign device_tag[4] = io_deviceRequest_4_bits_tag; -assign device_size[4] = io_deviceRequest_4_bits_size; -assign device_atomic[4] = io_deviceRequest_4_bits_atomic; -assign device_write[4] = io_deviceRequest_4_bits_write; -assign device_normal[4] = io_deviceRequest_4_bits_normal; -assign io_deviceRequest_4_ready = device_ready[4]; -assign dev_response_ready[4] = io_deviceResponse_4_ready; -assign io_deviceResponse_4_valid = dev_response_valid[4]; -assign io_deviceResponse_4_bits_tag = dev_response_tag[4]; -assign io_deviceResponse_4_bits_data = dev_response_data[4]; -assign io_deviceResponse_4_bits_error = 0; -assign retired[4] = io_retired_4; -assign retired_pc[4] = io_retiredPc_4; -assign trapped[4] = io_trapped_4; -assign trap_pc[4] = io_trapPc_4; -assign trap_cause[4] = io_trapCause_4; -assign trap_value[4] = io_trapValue_4; -assign failure[4] = io_failure_4_valid; -clocking sample @(posedge clock); - default input #1step; - input reset; - input device_valid,device_ready,device_write,device_normal,device_addr,device_data,device_tag,device_size,device_atomic; - input dev_response_valid,dev_response_ready,retired,retired_pc,trapped,trap_pc,trap_cause,trap_value,failure; - input io_memoryOutstanding; - input io_axi_aw_valid,io_axi_aw_ready,io_axi_aw_bits_id,io_axi_aw_bits_addr,io_axi_aw_bits_len,io_axi_aw_bits_size,io_axi_aw_bits_burst,io_axi_aw_bits_lock; - input io_axi_w_valid, io_axi_w_ready, io_axi_w_bits_data, io_axi_w_bits_strb, io_axi_w_bits_last; - input io_axi_ar_valid,io_axi_ar_ready,io_axi_ar_bits_id,io_axi_ar_bits_addr,io_axi_ar_bits_len,io_axi_ar_bits_size,io_axi_ar_bits_burst,io_axi_ar_bits_lock; - input io_axi_r_valid, io_axi_r_ready, io_axi_b_valid, io_axi_b_ready; -endclocking -int cycles = 0, retire_count[5], device_requests = 0, device_responses = 0; -bit exit_seen = 0, exit_ack = 0; -int exit_status = 0; -bit device_pending[5], device_exit[5]; -int device_due[5]; -bit [5:0] pending_tag[5]; -bit [63:0] pending_data[5]; -function automatic void ck(bit ok, string message); - if (!ok) `uvm_fatal("TILE_SYSTEM", message) -endfunction -`include "tile_axi_peer.svh" -task automatic device_service(); - forever begin - @(sample); - if (!sample.reset) - for (int i = 0; i < 5; i++) begin - if (sample.device_valid[i] && sample.device_ready[i]) begin - ck(!device_pending[i] && !sample.device_normal[i] && sample.device_atomic[i] == 0, - "Illegal actual Tile Device request/owner"); - device_pending[i] = 1; - pending_tag[i] = sample.device_tag[i]; - pending_data[i] = 0; - device_due[i] = cycles + 4; - device_exit[i] = 0; - device_requests++; - case (sample.device_addr[i]) - 'h60000000: begin - ck(i == 0 && sample.device_write[i] && sample.device_size[i] == 2 && !exit_seen, - "Completion must be one controller 32-bit Device write"); - exit_seen = 1; - exit_status = int'(sample.device_data[i][31:0]); - device_exit[i] = 1; - ck(exit_status == 0, $sformatf( - "CPU firmware exited status%0d pc%h", exit_status, sample.retired_pc[i])); - end - 'h10000000: begin - if (sample.device_write[i]) $write("%c", sample.device_data[i][7:0]); - end - 'h10000005: begin - ck(!sample.device_write[i], "UART status is read only"); - pending_data[i] = 'h60; - end - default: - ck(0, $sformatf( - "Unsupported actual Device address hart%0d addr%h", i, sample.device_addr[i])); - endcase - end - if (sample.dev_response_valid[i] && sample.dev_response_ready[i]) begin - ck(device_pending[i], "Device response without retained CPU owner"); - if (device_exit[i]) exit_ack = 1; - device_pending[i] = 0; - device_responses++; - end - end - @(negedge clock); - for (int i = 0; i < 5; i++) begin - device_ready[i] = !reset && !device_pending[i] && cycles % 5 != 0; - dev_response_valid[i] = !reset && device_pending[i] && device_due[i] <= cycles; - dev_response_tag[i] = pending_tag[i]; - dev_response_data[i] = pending_data[i]; - end - end -endtask -task automatic monitor_cpu(); - forever begin - @(sample); - if (!sample.reset) begin - cycles++; - dpi_bdb_set_clk(cycles); - for (int i = 0; i < 5; i++) begin - ck(!sample.failure[i], $sformatf("Actual NPU DMA failure hart%0d", i)); - if (sample.retired[i]) retire_count[i]++; - if (sample.trapped[i]) begin - ck(i != 0 && sample.trap_cause[i] == 8, $sformatf( - "Unexpected CPU trap hart%0d pc%h cause%h value%h", - i, - sample.trap_pc[i], - sample.trap_cause[i], - sample.trap_value[i] - )); - end - end - end - end -endtask -initial begin - uvm_config_db#(virtual ip_control_if)::set(null, "*", "vif", ctl); - uvm_config_db#(time)::set(null, "uvm_test_top", "timeout", 1ms); - run_test("protocol_test"); -end -initial begin - longint unsigned entry; - ck(`TILE_CPU_COUNT == 5, "Tile gate requires actual Goban five-CPU profile"); - for (int i = 0; i < 5; i++) begin - device_ready[i] = 0; - dev_response_valid[i] = 0; - dev_response_tag[i] = 0; - dev_response_data[i] = 0; - device_pending[i] = 0; - device_exit[i] = 0; - device_due[i] = 0; - pending_tag[i] = 0; - pending_data[i] = 0; - retire_count[i] = 0; - end - io_axi_aw_ready = 0; - io_axi_ar_ready = 0; - io_axi_w_ready = 0; - io_axi_r_valid = 0; - io_axi_b_valid = 0; - io_axi_r_bits_id = 0; - io_axi_r_bits_data = 0; - io_axi_r_bits_last = 0; - io_axi_r_bits_resp = 0; - io_axi_b_bits_id = 0; - io_axi_b_bits_resp = 0; - io_interrupts_0_external = 0; - io_interrupts_0_software = 0; - io_interrupts_0_timer = 0; - io_interrupts_1_external = 0; - io_interrupts_1_software = 0; - io_interrupts_1_timer = 0; - io_interrupts_2_external = 0; - io_interrupts_2_software = 0; - io_interrupts_2_timer = 0; - io_interrupts_3_external = 0; - io_interrupts_3_software = 0; - io_interrupts_3_timer = 0; - io_interrupts_4_external = 0; - io_interrupts_4_software = 0; - io_interrupts_4_timer = 0; - entry = tile_memory_init(`TILE_IMAGE_PATH, `TILE_EXPECT_PATH); - io_resetVector_0 = entry; - io_resetVector_1 = entry; - io_resetVector_2 = entry; - io_resetVector_3 = entry; - io_resetVector_4 = entry; - wait (ctl.start); - repeat (5) @(negedge clock); - reset = 0; - fork - axi_service(); - device_service(); - monitor_cpu(); - join_none - wait (exit_ack); - do - @(sample); - while (reads.num() || writes.num() || aw_ids.size() || aw_addresses.size() || w_lines.size() || - w_masks.size() || w_beat || r_active || b_active || sample.io_memoryOutstanding != 0); - ck(device_requests == device_responses, "Device completions not drained"); - for (int i = 0; i < 5; i++) - ck(retire_count[i] > 0, $sformatf("Configured CPU%0d never executed", i)); - ck(axi_reads > 0 && axi_writes > 0 && final_b_waits > 0, - "Firmware did not exercise actual DDR read/write/finalB"); - tile_peer_finish(); - `uvm_info("TILE_SYSTEM_PASS", $sformatf( - "actual five CPU firmware cycles%0d DDR read/write%0d/%0d finalBwait%0d Device%0d/%0d status%0d; no RoCC BFM", - cycles, - axi_reads, - axi_writes, - final_b_waits, - device_requests, - device_responses, - exit_status - ), UVM_LOW) - ctl.done = 1; -end diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/scala/AdmissionSystem.scala b/arch/src/main/scala/framework/memdomain/verification/src/main/scala/AdmissionSystem.scala deleted file mode 100644 index cdbaa8e1c..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/scala/AdmissionSystem.scala +++ /dev/null @@ -1,117 +0,0 @@ -package framework.memdomain.verification - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.system.core.rocket.{CpuParams, HasCpuParameters} -import framework.top.GlobalConfig -import framework.system.core.accelerator.{Admission, BuckyballAccelerator} -import framework.system.core.rocket.RoCCIO -import framework.memdomain.backend.shared.{SharedMemBackend, SharedMemLayout} -import hier.core.rocket.AdmissionPorts -import hier.tile.TaskController -import memcore.memory.interlock.{Params => TrackingParams} -import memcore.memory.preflight.Params -import memcore.memory.cpu.PhysicalRegion -import memcore.bus.axi4 - -/** Actual NPU/ROB/task/translation/AXI join. Core and CMO peer use the declared BFM contract. */ -@instantiable -class AdmissionSystem( - b: GlobalConfig, - prepared: Params, - coreIndex: Int, - workerSignatures: Seq[BigInt], - regions: Seq[PhysicalRegion] -)( - implicit val cpuParams: CpuParams) - extends Module - with HasCpuParameters { - val tracking = TrackingParams(addressBits = prepared.bus.addressBits) - val axiParams = axi4.Params(prepared.bus.addressBits, 128, 4) - - @public - val io = IO(new Bundle { - val core = Flipped(new AdmissionPorts(tracking, nPMPs, prepared.bus)) - val controller = new RoCCIO(64) - val controllerSatp = Input(UInt(64.W)) - val axi = new memcore.bus.axi4.Port(axiParams) - val npuIdle = Output(Bool()) - val halted = Output(Bool()) - val fault = Output(new framework.memdomain.frontend.mem.dma.DmaStatus) - val faultTag = Output(UInt(tracking.idBits.W)) - val workDrained = Output(Bool()) - val blockedResponse = Input(Bool()) - }) - - val admission: Instance[Admission] = Instantiate(new Admission(b, tracking, prepared, regions, axiParams)) - val accelerator: Instance[BuckyballAccelerator] = Instantiate(new BuckyballAccelerator(b)) - val tasks: Instance[TaskController] = - Instantiate(new TaskController((1 to workerSignatures.size), workerSignatures, b.memDomain.nCores)) - admission.io.core <> io.core - accelerator.io.cmd <> admission.io.npuCommand - admission.io.npuResponse <> accelerator.io.resp - admission.io.allocation.valid := accelerator.io.allocation.valid - admission.io.allocation.bits := accelerator.io.allocation.bits.rob_id - admission.io.retired := accelerator.io.retired - admission.io.npuFault := accelerator.io.fault - admission.io.npuBusy := !accelerator.io.idle - admission.io.npuInterrupt := accelerator.io.interrupt - admission.io.footprints := accelerator.io.footprints - admission.io.dma <> accelerator.io.dma - accelerator.io.hartid := coreIndex.U - accelerator.io.sharedBankOwnerHartId := 0.U - accelerator.io.barrier_release := accelerator.io.barrier_arrive - admission.io.task <> tasks.io.ports(coreIndex) - tasks.io.ports(0) <> io.controller - tasks.io.satp := io.controllerSatp - for (i <- 1 until b.memDomain.nCores if i != coreIndex) { - tasks.io.ports(i).cmd.valid := false.B - tasks.io.ports(i).cmd.bits := 0.U.asTypeOf(tasks.io.ports(i).cmd.bits) - tasks.io.ports(i).resp.ready := true.B - tasks.io.ports(i).exception := false.B - } - io.npuIdle := accelerator.io.idle - io.halted := admission.io.halted; io.fault := admission.io.fault - io.faultTag := admission.io.faultTag; io.workDrained := admission.io.workDrained - - val shared: Instance[SharedMemBackend] = Instantiate(new SharedMemBackend(b, useMesh = true)) - shared.io.config <> accelerator.io.shared_config - shared.io.mvover <> accelerator.io.mvover - val channels = SharedMemLayout.channelPerHart(b) - val endpoint = b.memDomain.computeCoreIds.indexOf(coreIndex) - require(endpoint >= 0) - for (i <- shared.io.mem_req.indices) { - if (i >= endpoint * channels && i < (endpoint + 1) * channels) { - shared.io.mem_req(i) <> accelerator.io.shared_mem_req(i - endpoint * channels) - } else { - val q = shared.io.mem_req(i) - q.read.req.valid := false.B; q.read.req.bits := 0.U.asTypeOf(q.read.req.bits); q.read.resp.ready := true.B - q.write.req.valid := false.B; q.write.req.bits := 0.U.asTypeOf(q.write.req.bits); q.write.resp.ready := true.B - q.bank_id := 0.U; q.group_id := 0.U; q.is_shared := false.B; q.hart_id := 0.U; q.rob_id := 0.U; q.inst_id := 0.U - } - } - for (i <- 0 until b.memDomain.nCores) { - shared.io.query_valid(i) := (if (i == coreIndex) accelerator.io.shared_query_valid else false.B) - shared.io.query_hart_id(i) := i.U - shared.io.query_vbank_id(i) := (if (i == coreIndex) accelerator.io.shared_query_vbank_id else 0.U) - if (i == coreIndex) shared.io.localBanks(i) <> accelerator.io.meshLocalBank - else { - shared.io.localBanks(i).request.ready := false.B - shared.io.localBanks(i).response.valid := false.B - shared.io.localBanks(i).response.bits := 0.U.asTypeOf(shared.io.localBanks(i).response.bits) - } - } - accelerator.io.shared_query_group_count := shared.io.query_group_count(coreIndex) - accelerator.io.shared_bank_hashes.foreach(_ := shared.io.bank_hashes.get) - - io.axi.aw <> admission.io.axi.aw - io.axi.w <> admission.io.axi.w - io.axi.ar <> admission.io.axi.ar - admission.io.axi.r.valid := io.axi.r.valid && !io.blockedResponse - admission.io.axi.r.bits := io.axi.r.bits - io.axi.r.ready := admission.io.axi.r.ready && !io.blockedResponse - admission.io.axi.b.valid := io.axi.b.valid && !io.blockedResponse - admission.io.axi.b.bits := io.axi.b.bits - io.axi.b.ready := admission.io.axi.b.ready && !io.blockedResponse -} diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/scala/ControllerSystem.scala b/arch/src/main/scala/framework/memdomain/verification/src/main/scala/ControllerSystem.scala deleted file mode 100644 index 210ff7769..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/scala/ControllerSystem.scala +++ /dev/null @@ -1,107 +0,0 @@ -package framework.memdomain.verification - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.system.core.rocket.{CpuParams, HasCpuParameters} -import framework.top.GlobalConfig -import framework.system.core.ControllerAdmission -import framework.system.core.rocket.RoCCIO -import framework.system.tile.BankNetwork -import hier.core.rocket.AdmissionPorts -import hier.tile.TaskController -import memcore.memory.interlock.{Params => TrackingParams} -import memcore.bus.chi.{Params => ChiParams} -import memcore.memory.bank.{Bank, BankRequest, BankSetParams} - -/** Real controller/task/bank-move composition; CPU command and bank initialization are test peers. */ -@instantiable -class ControllerSystem(b: GlobalConfig, signatures: Seq[BigInt])(implicit val cpuParams: CpuParams) - extends Module - with HasCpuParameters { - val tracking = TrackingParams() - val bankParams = - BankSetParams(b.memDomain.bankWidth, 1, b.memDomain.bankEntries, math.max(1, log2Ceil(b.frontend.rob_entries))) - val cores = b.memDomain.computeCoreIds - - @public - val io = IO(new Bundle { - val core = Flipped(new AdmissionPorts(tracking, nPMPs, ChiParams())) - val workers = Vec(signatures.size, new RoCCIO(64)) - - val access = Vec( - cores.size, - Flipped(new memcore.memory.mesh_shm.MeshLocalBankPort( - 16, - 10, - bankParams.dataBits, - bankParams.tagBits - )) - ) - - val allowBankResponse = Input(Vec(cores.size, Bool())) - val moveFault = Valid(new framework.memdomain.frontend.mem.dma.DmaStatus) - val moveAccepted = Output(Bool()) - val moveCompleted = Output(Bool()) - val bankWriteAccepted = Output(Vec(cores.size, Bool())) - }) - - val admission: Instance[ControllerAdmission] = - Instantiate(new ControllerAdmission(tracking, ChiParams(), moves = true)) - val tasks: Instance[TaskController] = - Instantiate(new TaskController(1 to signatures.size, signatures, b.memDomain.nCores)) - val network: Instance[BankNetwork] = Instantiate(new BankNetwork(b, cores, useMesh = true, controllerMove = true)) - admission.io.core <> io.core - admission.io.task <> tasks.io.ports(0) - tasks.io.satp := admission.io.taskSatp - for (i <- signatures.indices) { tasks.io.ports(i + 1) <> io.workers(i) } - admission.io.move <> network.io.controllerMvover.get - io.moveFault := admission.io.moveFault - io.moveAccepted := admission.io.move.command.fire - io.moveCompleted := admission.io.move.completion.fire - network.io.hartIds.zipWithIndex.foreach { case (hart, i) => hart := i.U } - for (i <- cores.indices) { - val port = network.io.compute(i) - port.config.valid := false.B; port.config.bits := 0.U.asTypeOf(port.config.bits) - port.queryValid := false.B; port.queryVbank := 0.U; port.barrierArrive := false.B - port.move.command.valid := false.B; port.move.command.bits := 0.U.asTypeOf(port.move.command.bits) - port.move.completion.ready := true.B - for (q <- port.requests) { - q.read.req.valid := false.B; q.read.req.bits := 0.U.asTypeOf(q.read.req.bits); q.read.resp.ready := true.B - q.write.req.valid := false.B; q.write.req.bits := 0.U.asTypeOf(q.write.req.bits); q.write.resp.ready := true.B - q.bank_id := 0.U; q.group_id := 0.U; q.is_shared := false.B - q.hart_id := 0.U; q.rob_id := 0.U; q.inst_id := 0.U - } - // Bank zero is the declared private endpoint used by this gate. Storage and ACK are the existing Bank IP. - val bank: Instance[Bank] = Instantiate(new Bank(bankParams)) - val choose = Module(new Arbiter(chiselTypeOf(port.local.request.bits), 2)) - choose.io.in(0) <> port.local.request - choose.io.in(1) <> io.access(i).request - bank.io.request.valid := choose.io.out.valid - choose.io.out.ready := bank.io.request.ready - bank.io.request.bits := 0.U.asTypeOf(new BankRequest(bankParams)) - bank.io.request.bits.addr := choose.io.out.bits.addr - bank.io.request.bits.write := choose.io.out.bits.write - bank.io.request.bits.data := choose.io.out.bits.data - bank.io.request.bits.mask := choose.io.out.bits.mask - bank.io.request.bits.tag := choose.io.out.bits.tag - val owner = Reg(UInt(1.W)) - io.bankWriteAccepted(i) := choose.io.in(0).fire && choose.io.in(0).bits.write - when(choose.io.out.fire) { - assert( - choose.io.out.bits.bank === 0.U && choose.io.out.bits.addr < bankParams.entriesPerBank.U, - "ControllerSystem endpoint is private bank zero within the actual bank capacity" - ) - owner := choose.io.chosen - } - port.local.response.bits := bank.io.response.bits - io.access(i).response.bits := bank.io.response.bits - port.local.response.valid := bank.io.response.valid && owner === 0.U && io.allowBankResponse(i) - io.access(i).response.valid := bank.io.response.valid && owner === 1.U - bank.io.response.ready := Mux( - owner === 0.U, - port.local.response.ready && io.allowBankResponse(i), - io.access(i).response.ready - ) - } -} diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/scala/Emit.scala b/arch/src/main/scala/framework/memdomain/verification/src/main/scala/Emit.scala deleted file mode 100644 index c37e15172..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/scala/Emit.scala +++ /dev/null @@ -1,321 +0,0 @@ -package framework.memdomain.verification - -import chisel3._ -import chisel3.reflect.DataMirror -import chisel3.ActualDirection -import framework.system.core.rocket.CpuParams -import java.nio.file.{Files, Paths} -import framework.system.configloader.{ChipLoader, RocketTileCore} -import framework.memdomain.frontend.mem.{KernelDma, MemConfiger, MemLoader, MemStorer} -import framework.memdomain.backend.privatepath.PrivateMemBackend -import framework.memdomain.backend.shared.SharedMemBackend - -object Emit extends App { - require( - args.length >= 4 && args(0) == "--chip" && args(2) == "--targets", - "Usage: memdomain_ack.run --chip --targets [firtool options]" - ) - val chip = args(1) - val targets = args(3).split(",").toSet - - val known = Set( - "mset_private", - "mset_shared", - "shared_lease", - "mset_configer", - "program_rob", - "program_decoder", - "loader_ack", - "read_dma", - "storer_ack", - "kernel_dma", - "admission_system", - "controller_system", - "memory_system", - "tile_system_single", - "tile_system_four" - ) - - require(targets.nonEmpty && targets.subsetOf(known), s"Unknown verification targets: ${targets -- known}") - val firtoolOptions = args.drop(4) - require(chip == "toy" || chip == "goban", s"MemDomain gates declare Toy or Goban profiles, got $chip") - val repo = Paths.get("..").toAbsolutePath.normalize - lazy val topology = ChipLoader.load(repo.resolve(s"examples/chips/$chip/configs/generated/chip.pb").toString) - - lazy val b = { - val configs = topology.tiles.flatMap(_.cores).collect { case RocketTileCore(_, Some(config)) => config } - require(configs.nonEmpty, "MemDomain gate requires a configured accelerator") - if (chip == "toy") require(configs.size == 1, "Toy ACK gate requires exactly one compute configuration") - val config = configs.head - require( - config.memDomain.bankWidth == 128 && config.memDomain.bankMaskLen == 16, - "ACK gate profile requires 16-byte bank rows and byte masks" - ) - config - } - - val output = repo.resolve("arch/src/main/scala/framework/memdomain/verification/build") - Files.createDirectories(output) - - def leaves(d: Data, name: String): Seq[(String, Data)] = d match { - case r: Record => r.elements.toSeq.flatMap { case (n, v) => leaves(v, s"${name}_$n") } - case v: Vec[_] => v.zipWithIndex.flatMap { case (e, i) => leaves(e, s"${name}_$i") }.toSeq - case e => if (e.getWidth > 0) Seq(name -> e) else Seq.empty - } - - lazy val msetConfig = { - val defaults = framework.top.GlobalConfig() - defaults.copy( - memDomain = defaults.memDomain.copy( - bankNum = 4, - bankWidth = 128, - bankEntries = 8, - bankMaskLen = 16, - virtualBankCount = 16, - sharedEnable = true, - sharedEntries = 32, - sharedBankEntries = 8, - sharedBankNum = 4, - sharedInputChannels = 2, - sharedDefaultGroupCount = 1, - nCores = 2, - computeCoreIds = Seq(0, 1), - bankChannel = 2, - dma_buswidth = 128, - memAddrLen = 39 - ), - frontend = defaults.frontend.copy( - rob_entries = 8, - bank_id_len = 10, - vbank_id_upper_bound = 7, - shared_bank_id_base = 8, - iter_len = 34, - sub_rob_depth = 4 - ), - tile = defaults.tile.copy(xLen = 64), - sim = defaults.sim.copy(diffTest = true) - ) - } - - lazy val programConfig = msetConfig.copy( - rvv = msetConfig.rvv.copy(enable = true), - sim = msetConfig.sim.copy(diffTest = false) - ) - - for ( - target <- Seq("mset_private", "mset_shared", "shared_lease", "mset_configer", "program_rob", "program_decoder") - if targets(target) - ) { - var interface: Data = null - val moduleName = target match { - case "mset_private" => "PrivateMemBackend" - case "mset_shared" => "SharedMemBackend" - case "shared_lease" => "SharedMemBackend" - case "mset_configer" => "MemConfiger" - case "program_rob" => "GlobalROB" - case "program_decoder" => "GlobalDecoder" - } - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - target match { - case "mset_private" => val m = new PrivateMemBackend(msetConfig); interface = m.io; m - case "mset_shared" => val m = new SharedMemBackend(msetConfig); interface = m.io; m - case "shared_lease" => - val m = new SharedMemBackend(msetConfig, useMesh = true, externalPhysicalPorts = 1); interface = m.io; m - case "mset_configer" => val m = new MemConfiger(msetConfig); interface = m.io; m - case "program_rob" => val m = new framework.frontend.globalrs.GlobalROB(programConfig); interface = m.io; m - case "program_decoder" => - val m = new framework.frontend.decoder.GlobalDecoder(programConfig); interface = m.io; m - }, - args = Array("--target-dir", output.resolve(moduleName).toString, "--split-verilog"), - firtoolOpts = firtoolOptions - ) - val fields = leaves(interface, "io") - if (target == "shared_lease") Files.writeString( - output.resolve("shared_lease_clocking.svh"), - "clocking sample @(posedge clock);\ndefault input #1step;\ninput reset;\n" + - fields.map { case (n, _) => s"input $n;" }.mkString("\n") + "\nendclocking\n" - ) - Files.writeString( - output.resolve(s"${target}_signals.svh"), - fields.map { case (name, data) => s"logic [${data.getWidth - 1}:0] $name;" }.mkString("\n") + "\n" - ) - Files.writeString( - output.resolve(s"${target}_ports.svh"), - (Seq(".clock(clock)", ".reset(reset)") ++ fields.map { case (name, _) => s".$name($name)" }).mkString( - ",\n" - ) + "\n" - ) - Files.writeString( - output.resolve(s"${target}_init.svh"), - fields.collect { - case (name, data) if DataMirror.directionOf(data) == ActualDirection.Input => s"$name = '0;" - }.mkString("\n") + "\n" - ) - } - - for (load <- Seq(true, false) if targets(if (load) "loader_ack" else "storer_ack")) { - var io: Data = null - val stem = if (load) "loader_ack" else "storer_ack" - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - if (load) { val m = new MemLoader(msetConfig); io = m.io; m } - else { val m = new MemStorer(msetConfig); io = m.io; m }, - args = Array("--target-dir", output.resolve(if (load) "MemLoader" else "MemStorer").toString, "--split-verilog"), - firtoolOpts = firtoolOptions - ) - val fs = leaves(io, "io") - Files.writeString( - output.resolve(s"${stem}_signals.svh"), - fs.map { case (n, d) => s"logic [${d.getWidth - 1}:0] $n;" }.mkString("\n") + "\n" - ) - Files.writeString( - output.resolve(s"${stem}_ports.svh"), - (Seq(".clock(clock)", ".reset(reset)") ++ fs.map { case (n, _) => s".$n($n)" }).mkString(",\n") + "\n" - ) - Files.writeString( - output.resolve(s"${stem}_init.svh"), - fs.collect { case (n, d) if DataMirror.directionOf(d) == ActualDirection.Input => s"$n = '0;" }.mkString( - "\n" - ) + "\n" - ) - } - - if (targets("read_dma")) { - var readIo: Data = null - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - { - val m = new framework.memdomain.frontend.mem.dma.ReadDma( - msetConfig, - memcore.memory.preflight.Params(beatBytes = 16), - memcore.bus.axi4.Params() - ); readIo = m.io; m - }, - args = Array("--target-dir", output.resolve("ReadDma").toString, "--split-verilog"), - firtoolOpts = firtoolOptions - ) - val fields = leaves(readIo, "io") - Files.writeString( - output.resolve("read_dma_signals.svh"), - fields.map { case (n, d) => s"logic [${d.getWidth - 1}:0] $n;" }.mkString("\n") + "\n" - ) - Files.writeString( - output.resolve("read_dma_ports.svh"), - (Seq(".clock(clock)", ".reset(reset)") ++ fields.map { case (n, _) => s".$n($n)" }).mkString(",\n") + "\n" - ) - Files.writeString( - output.resolve("read_dma_init.svh"), - fields.collect { case (n, d) if DataMirror.directionOf(d) == ActualDirection.Input => s"$n = '0;" }.mkString( - "\n" - ) + "\n" - ) - Files.writeString( - output.resolve("read_dma_clocking.svh"), - "clocking sample @(posedge clock);\ndefault input #1step;\ninput reset;\n" + - fields.map { case (n, _) => s"input $n;" }.mkString("\n") + "\nendclocking\n" - ) - } - - if (targets("kernel_dma")) { - var kernelIo: Data = null - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - { val m = new KernelDma(b); kernelIo = m.io; m }, - args = Array("--target-dir", output.resolve("KernelDma").toString, "--split-verilog"), - firtoolOpts = firtoolOptions - ) - val kernelFields = leaves(kernelIo, "io") - Files.writeString( - output.resolve("kernel_dma_signals.svh"), - kernelFields.map { case (n, d) => s"logic [${d.getWidth - 1}:0] $n;" }.mkString("\n") + "\n" - ) - Files.writeString( - output.resolve("kernel_dma_ports.svh"), - (Seq(".clock(clock)", ".reset(reset)") ++ kernelFields.map { case (n, _) => s".$n($n)" }).mkString(",\n") + "\n" - ) - Files.writeString( - output.resolve("kernel_dma_init.svh"), - kernelFields.collect { - case (n, d) if DataMirror.directionOf(d) == ActualDirection.Input => s"$n = '0;" - }.mkString("\n") + "\n" - ) - Files.writeString( - output.resolve("kernel_dma_clocking.svh"), - "clocking sample @(posedge clock);\ndefault input #1step;\ninput reset;\n" + - kernelFields.map { case (n, _) => s"input $n;" }.mkString("\n") + "\nendclocking\n" - ) - } - if (targets("admission_system") || targets("controller_system")) { - // Task admission and the controller belong to compute tiles; the main tile has neither. - val tile = topology.mountedTiles.find(_.controller.isDefined).getOrElse( - throw new IllegalArgumentException(s"$chip: admission/controller gates need a compute tile") - ) - val selected = tile.cores.zipWithIndex.collectFirst { case (RocketTileCore(cpu, Some(config)), i) => - (cpu, config, i) - }.get - val cpu = selected._1; val config = selected._2; val coreIndex = selected._3 - val signatures = tile.signatures.tail - require( - signatures.size == tile.cores.size - 1 && config.memDomain.sharedEnable, - "AdmissionSystem requires the actual shared-bank/task-worker configuration" - ) - implicit val coreParameters: CpuParams = CpuParams( - core = framework.system.core.rocket.configs.RocketCpuParam.toRocketCoreParams( - cpu, - tile.param.xLen, - tile.param.pgLevels - ), - dcache = Some(freechips.rocketchip.rocket.DCacheParams()), - icache = Some(freechips.rocketchip.rocket.ICacheParams()), - btb = None, - physicalAddressBits = tile.param.paddrBits, - hartIdBits = 6, - beatBytes = 16, - blockBytes = 64 - ) - val prepared = memcore.memory.preflight.Params(beatBytes = 16) - val regions = - Seq(memcore.memory.cpu.PhysicalRegion(BigInt("80000000", 16), BigInt(16) << 20, true, true, true, true, true, true)) - if (targets("admission_system")) { - var interface: Data = null - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - { val m = new AdmissionSystem(config, prepared, coreIndex, signatures, regions); interface = m.io; m }, - args = Array("--target-dir", output.resolve("AdmissionSystem").toString, "--split-verilog"), - firtoolOpts = firtoolOptions - ) - val fields = leaves(interface, "io") - Files.writeString( - output.resolve("admission_system_signals.svh"), - fields.map { case (n, d) => s"logic [${d.getWidth - 1}:0] $n;" }.mkString("\n") + "\n" - ) - Files.writeString( - output.resolve("admission_system_ports.svh"), - (Seq(".clock(clock)", ".reset(reset)") ++ fields.map { case (n, _) => s".$n($n)" }).mkString(",\n") + "\n" - ) - Files.writeString( - output.resolve("admission_system_init.svh"), - fields.collect { case (n, d) if DataMirror.directionOf(d) == ActualDirection.Input => s"$n = '0;" }.mkString( - "\n" - ) + "\n" - ) - Files.writeString( - output.resolve("admission_system_clocking.svh"), - "clocking sample @(posedge clock);\ndefault input #1step;\ninput reset;\n" + - fields.map { case (n, _) => s"input $n;" }.mkString("\n") + "\nendclocking\n" - ) - Files.writeString( - output.resolve("admission_system_config.svh"), - s"`define ADMISSION_ROWS ${config.memDomain.bankEntries}\n`define ADMISSION_SIGNATURE 64'h${config.coreSignature.toString(16)}\n`define ADMISSION_CORE $coreIndex\n" - ) - } - if (targets("controller_system")) EmitController(output, config, signatures, firtoolOptions) - } - - if (targets.exists(_.startsWith("tile_system_"))) { - require(chip == "goban", "Tile verification profile requires Goban") - val description = buckyball.config.Chip.parseFrom( - Files.readAllBytes(repo.resolve(s"examples/chips/$chip/configs/generated/chip.pb")) - ) - val platform = Class.forName(description.getMill.getVerilatorConfig).getDeclaredConstructor() - .newInstance().asInstanceOf[sims.soc.SystemTarget] - EmitTile(topology, output, firtoolOptions, platform.instantiate) - } - if (targets("memory_system")) EmitMemory(output, firtoolOptions) -} diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/scala/EmitController.scala b/arch/src/main/scala/framework/memdomain/verification/src/main/scala/EmitController.scala deleted file mode 100644 index 0f746f709..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/scala/EmitController.scala +++ /dev/null @@ -1,62 +0,0 @@ -package framework.memdomain.verification - -import chisel3._ -import chisel3.reflect.DataMirror -import java.nio.file.{Files, Path} -import framework.system.core.rocket.CpuParams -import framework.top.GlobalConfig - -object EmitController { - - def apply( - output: Path, - b: GlobalConfig, - signatures: Seq[BigInt], - firtoolOptions: Array[String] - )( - implicit cpuParams: CpuParams - ): Unit = { - var interface: Data = null - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - { val m = new ControllerSystem(b, signatures); interface = m.io; m }, - args = Array("--target-dir", output.resolve("ControllerSystem").toString, "--split-verilog"), - firtoolOpts = firtoolOptions - ) - def leaves(data: Data, name: String): Seq[(String, Data)] = data match { - case bundle: Record => bundle.elements.toSeq.flatMap { case (field, value) => leaves(value, s"${name}_$field") } - case vec: Vec[_] => vec.zipWithIndex.flatMap { case (value, i) => leaves(value, s"${name}_$i") }.toSeq - case value => Seq(name -> value) - } - val fields = leaves(interface, "io") - Files.writeString( - output.resolve("controller_system_signals.svh"), - fields.map { - case (name, data) => s"logic [${data.getWidth - 1}:0] $name;" - }.mkString("\n") + "\n" - ) - Files.writeString( - output.resolve("controller_system_ports.svh"), - (Seq(".clock(clock)", ".reset(reset)") ++ fields.map { case (name, _) => s".$name($name)" }).mkString( - ",\n" - ) + "\n" - ) - Files.writeString( - output.resolve("controller_system_init.svh"), - fields.collect { - case (name, data) if DataMirror.directionOf(data) == ActualDirection.Input => s"$name = '0;" - }.mkString("\n") + "\n" - ) - Files.writeString( - output.resolve("controller_system_clocking.svh"), - "clocking sample @(posedge clock);\ndefault input #1step;\ninput reset;\n" + - fields.map { - case (name, _) => s"input $name;" - }.mkString("\n") + "\nendclocking\n" - ) - Files.writeString( - output.resolve("controller_system_config.svh"), - s"`define CONTROLLER_SIGNATURE 64'h${signatures.head.toString(16)}\n" - ) - } - -} diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/scala/EmitMemory.scala b/arch/src/main/scala/framework/memdomain/verification/src/main/scala/EmitMemory.scala deleted file mode 100644 index 430c0470e..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/scala/EmitMemory.scala +++ /dev/null @@ -1,54 +0,0 @@ -package framework.memdomain.verification - -import chisel3._ -import chisel3.reflect.DataMirror -import java.nio.file.{Files, Path} -import framework.system.memory.Memory -import memcore.memory.ddr.{Params => DdrParams} - -object EmitMemory { - - def apply(output: Path, firtoolOptions: Array[String]): Unit = { - val ddr = DdrParams() - var interface: Data = null - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - { val m = new Memory(ddr, dmaMasters = 1); interface = m.io; m }, - args = Array("--target-dir", output.resolve("Memory").toString, "--split-verilog"), - firtoolOpts = firtoolOptions - ) - def leaves(data: Data, name: String): Seq[(String, Data)] = data match { - case bundle: Record => bundle.elements.toSeq.flatMap { case (field, value) => leaves(value, s"${name}_$field") } - case vec: Vec[_] => vec.zipWithIndex.flatMap { case (value, i) => leaves(value, s"${name}_$i") }.toSeq - case value => Seq(name -> value) - } - val fields = leaves(interface, "io") - Files.writeString( - output.resolve("memory_system_signals.svh"), - fields.map { - case (name, data) => s"logic [${data.getWidth - 1}:0] $name;" - }.mkString("\n") + "\n" - ) - Files.writeString( - output.resolve("memory_system_ports.svh"), - (Seq(".clock(clock)", ".reset(reset)") ++ fields.map { case (name, _) => s".$name($name)" }).mkString( - ",\n" - ) + "\n" - ) - Files.writeString( - output.resolve("memory_system_init.svh"), - fields.collect { - case (name, data) if DataMirror.directionOf(data) == ActualDirection.Input => s"$name = '0;" - }.mkString("\n") + "\n" - ) - Files.writeString( - output.resolve("memory_system_clocking.svh"), - "clocking sample @(posedge clock);\ndefault input #1step;\ninput reset;\n" + - "" + - "input line_ready,reply_valid,reply_id,reply_data,reply_error;\n" + - fields.map { - case (name, _) => s"input $name;" - }.mkString("\n") + "\nendclocking\n" - ) - } - -} diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/scala/EmitTile.scala b/arch/src/main/scala/framework/memdomain/verification/src/main/scala/EmitTile.scala deleted file mode 100644 index 0798016e8..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/scala/EmitTile.scala +++ /dev/null @@ -1,88 +0,0 @@ -package framework.memdomain.verification - -import java.nio.file.{Files, Path} -import chisel3._ -import chisel3.experimental.hierarchy.Instance -import framework.system.{System, SystemParams} -import framework.system.configloader.ExampleTopology -import memcore.bus.chi.{Params => ChiParams} -import memcore.bus.chi.rnf.RnfParams -import memcore.memory.cache.configs.CacheParams -import memcore.memory.coherence.configs.CoherenceParams -import memcore.memory.cpu.PhysicalRegion -import memcore.memory.interlock.{Params => TrackingParams} - -/** Elaboration of the actual single-tile composition, with explicit verification cache geometry. */ -object EmitTile { - - def apply( - topology: ExampleTopology, - output: Path, - firtoolOptions: Array[String], - build: SystemParams => Instance[System] - ): Unit = { - require(topology.tiles.size == 1, "Tile gate requires one configured tile") - val t = topology.tiles.head - val chi = ChiParams() - val memory = CoherenceParams( - chi, - CacheParams(chi.addressBits, 64, 4, 2, 8, 4, 2), - agents = 2 * t.cores.size, - mshrEntries = 8, - homeId = 64 - ) - val regions = Seq( - PhysicalRegion(BigInt("80000000", 16), BigInt(256) << 20, true, true, true, true, true, true), - PhysicalRegion(BigInt("90000000", 16), BigInt(16) << 20, false, true, true, true, true, true), - PhysicalRegion(BigInt("10000000", 16), BigInt(4096), false, false, true, true, false, false), - PhysicalRegion(BigInt("60000000", 16), BigInt(4096), false, false, true, true, false, false) - ) - val slotsPerClient = 16 - val ddr = memcore.memory.ddr.Params( - chi, - clients = 1, - slotsPerClient = slotsPerClient, - dataBits = 128, - idBits = 4 - ) - var interface: Data = null - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - { - val system = new TileSystem( - topology, - 36, - memory, - RnfParams(chi, cacheLines = 8, banks = 2), - regions, - TrackingParams(addressBits = chi.addressBits), - ddr, - build - ) - interface = system.io - system - }, - args = Array("--target-dir", output.resolve("TileSystem").toString, "--split-verilog"), - firtoolOpts = firtoolOptions - ) - def leaves(data: Data, name: String): Seq[(String, Data)] = data match { - case record: Record => record.elements.toSeq.flatMap { case (n, v) => leaves(v, s"${name}_$n") } - case vector: Vec[_] => vector.zipWithIndex.flatMap { case (value, i) => leaves(value, s"${name}_$i") }.toSeq - case leaf => if (leaf.getWidth > 0) Seq(name -> leaf) else Seq.empty - } - val fields = leaves(interface, "io") - Files.writeString( - output.resolve("tile_system_signals.svh"), - fields.map { - case (name, data) => s"logic [${data.getWidth - 1}:0] $name;" - }.mkString("\n") + "\n" - ) - Files.writeString( - output.resolve("tile_system_ports.svh"), - (Seq(".clock(clock)", ".reset(reset)") ++ fields.map { case (name, _) => s".$name($name)" }).mkString( - ",\n" - ) + "\n" - ) - Files.writeString(output.resolve("tile_system_config.svh"), s"`define TILE_CPU_COUNT ${t.cores.size}\n") - } - -} diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/scala/TileSystem.scala b/arch/src/main/scala/framework/memdomain/verification/src/main/scala/TileSystem.scala deleted file mode 100644 index c15909e4d..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/scala/TileSystem.scala +++ /dev/null @@ -1,34 +0,0 @@ -package framework.memdomain.verification - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.system.configloader.ExampleTopology -import framework.system.{System, SystemParams} -import framework.system.memory.Memory -import framework.memdomain.frontend.mem.dma.DmaStatus -import hier.tile.memory.CoreInterrupts -import memcore.bus.chi.rnf.RnfParams -import memcore.memory.coherence.configs.CoherenceParams -import memcore.memory.cpu.{CpuMemParams, PhysicalRegion, UncachedRequest, UncachedResponse} -import memcore.memory.interlock.{Params => TrackingParams} -import memcore.memory.ddr.{Params => DdrParams} - -/** Verification names the same production Tile/DDR composition. */ -@instantiable -class TileSystem( - topology: ExampleTopology, - cpuPhysicalBits: Int, - memory: CoherenceParams, - l1: RnfParams, - regions: Seq[PhysicalRegion], - tracking: TrackingParams, - ddr: DdrParams, - build: SystemParams => Instance[System]) - extends Module { - val system: Instance[framework.system.System] = - build(SystemParams(topology, cpuPhysicalBits, memory, l1, regions, tracking, ddr)) - @public - val io = IO(chiselTypeOf(system.dlink)) - io <> system.dlink -} diff --git a/arch/src/main/scala/framework/root/.gitignore b/arch/src/main/scala/framework/root/.gitignore deleted file mode 100644 index 36093f7bf..000000000 --- a/arch/src/main/scala/framework/root/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -out/ -build/ -test_run_dir/ -generated/ diff --git a/arch/src/main/scala/framework/root/README.md b/arch/src/main/scala/framework/root/README.md deleted file mode 100644 index 852c39e73..000000000 --- a/arch/src/main/scala/framework/root/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# root -ROOT: Rigid Organisation, Open Topology diff --git a/arch/src/main/scala/framework/root/root-chip/build.sc b/arch/src/main/scala/framework/root/root-chip/build.sc deleted file mode 100644 index 48c609b05..000000000 --- a/arch/src/main/scala/framework/root/root-chip/build.sc +++ /dev/null @@ -1,43 +0,0 @@ -import mill._ -import mill.scalalib._ - -trait ChiselModule extends SbtModule { - def moduleRoot: os.Path - - override def millSourcePath = moduleRoot - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") -} - -val frameworkRoot = os.pwd / os.up / os.up -val memCoreRoot = frameworkRoot / "mem-core" - -object queue extends SbtModule { - override def millSourcePath = os.pwd / os.up / os.up / "mem-core" / "queue" - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") -} - -object axis extends ChiselModule { - override def moduleRoot = memCoreRoot / "axis" -} - -object chi extends ChiselModule { - override def moduleDeps = Seq(queue) - override def moduleRoot = memCoreRoot / "chi" -} - -object bank extends ChiselModule { - override def moduleRoot = memCoreRoot / "bank" - override def moduleDeps = Seq(axis) -} - -object root_chip extends ChiselModule { - override def moduleRoot = os.pwd - override def mainClass = Some("hier.chip.mesh.Emit") - override def moduleDeps = Seq(axis, chi, bank) -} diff --git a/arch/src/main/scala/framework/root/root-chip/src/csrc/Cargo.lock b/arch/src/main/scala/framework/root/root-chip/src/csrc/Cargo.lock deleted file mode 100644 index a561fabf0..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/csrc/Cargo.lock +++ /dev/null @@ -1,7 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "root-chip-reference-model" -version = "0.1.0" diff --git a/arch/src/main/scala/framework/root/root-chip/src/csrc/Cargo.toml b/arch/src/main/scala/framework/root/root-chip/src/csrc/Cargo.toml deleted file mode 100644 index 631fdb590..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/csrc/Cargo.toml +++ /dev/null @@ -1,6 +0,0 @@ -[package] -name="root-chip-reference-model" -version="0.1.0" -edition="2021" -[lib] -crate-type=["cdylib"] diff --git a/arch/src/main/scala/framework/root/root-chip/src/csrc/src/lib.rs b/arch/src/main/scala/framework/root/root-chip/src/csrc/src/lib.rs deleted file mode 100644 index 56353a299..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/csrc/src/lib.rs +++ /dev/null @@ -1,2 +0,0 @@ -mod routing; -pub use routing::*; diff --git a/arch/src/main/scala/framework/root/root-chip/src/csrc/src/routing.rs b/arch/src/main/scala/framework/root/root-chip/src/csrc/src/routing.rs deleted file mode 100644 index ef20027e9..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/csrc/src/routing.rs +++ /dev/null @@ -1,4 +0,0 @@ -#[no_mangle] -pub extern "C" fn mesh_route(x: i32, y: i32, dst_x: i32, dst_y: i32) -> i32 { - if dst_x < x { 4 } else if dst_x > x { 3 } else if dst_y < y { 1 } else if dst_y > y { 2 } else { 0 } -} diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_node.f b/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_node.f deleted file mode 100644 index d33202880..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_node.f +++ /dev/null @@ -1,7 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/codec_interface.sv -@RESOURCES@/bad_node_package.sv --F @RTL@/ChiMeshCodecLoopback/filelist.f -@RESOURCES@/bad_node_tb.sv diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_node.toml b/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_node.toml deleted file mode 100644 index 4a4cc5ad7..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_node.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "CHI Mesh target NodeID is not placed" diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_node_package.sv b/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_node_package.sv deleted file mode 100644 index eda5ed51f..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_node_package.sv +++ /dev/null @@ -1,30 +0,0 @@ -package chi_bad_node_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual chi_codec_if vif; - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual chi_codec_if)::get(this, "", "vif", vif)) - `uvm_fatal("VIF", "codec missing") - endfunction - task execute(); - vif.reset = 1; - vif.pause = 0; - vif.valid = 0; - vif.target = 0; - vif.data = 0; - vif.out_ready = 1; - repeat (4) @(negedge vif.clock); - vif.reset = 0; - vif.valid = 1; - repeat (8) @(negedge vif.clock); - `uvm_fatal("MISSING_ASSERTION", "unplaced NodeID was not rejected") - endtask - endclass -endpackage diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_node_tb.sv b/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_node_tb.sv deleted file mode 100644 index 54a913b26..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_node_tb.sv +++ /dev/null @@ -1,28 +0,0 @@ -module bad_node_tb; - import uvm_pkg::*; - import chi_bad_node_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - chi_codec_if vif (clock); - ChiMeshCodecLoopback dut ( - .clock(clock), - .reset(vif.reset), - .io_pause(vif.pause), - .io_in_valid(vif.valid), - .io_in_ready(vif.ready), - .io_in_bits_targetNode(vif.target), - .io_in_bits_flit(vif.data), - .io_out_valid(vif.out_valid), - .io_out_ready(vif.out_ready), - .io_out_bits(vif.out_data), - .io_observedFire(vif.chunk_fire), - .io_observed_head(vif.chunk_head), - .io_observed_tail(vif.chunk_tail), - .io_observed_dstX(vif.chunk_x), - .io_observed_dstY(vif.chunk_y) - ); - initial begin - uvm_config_db#(virtual chi_codec_if)::set(null, "uvm_test_top", "vif", vif); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_requester_req.f b/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_requester_req.f deleted file mode 100644 index 7df8ad137..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_requester_req.f +++ /dev/null @@ -1,7 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/fabric_interface.sv -@RESOURCES@/bad_requester_req_package.sv --F @RTL@/ChiMeshRequesterEndpoint/filelist.f -@RESOURCES@/bad_requester_req_tb.sv diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_requester_req.toml b/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_requester_req.toml deleted file mode 100644 index 0cc7e830b..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_requester_req.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "CHI requester endpoint received a REQ" diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_requester_req_package.sv b/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_requester_req_package.sv deleted file mode 100644 index 369fc4d1d..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_requester_req_package.sv +++ /dev/null @@ -1,57 +0,0 @@ -package chi_bad_requester_req_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual mesh_fabric_if vif; - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - timeout = 10us; - if (!uvm_config_db#(virtual mesh_fabric_if)::get(this, "", "vif", vif)) - `uvm_fatal("VIF", "requester mesh interface missing") - endfunction - task execute(); - // Issue-H base-profile REQ is 137 bits, carried as five 32-bit mesh chunks. - bit [159:0] request; - request = '0; - request[4+:7] = 1; // TgtID: populated RN-F, deliberately the wrong role. - request[11+:7] = 2; // SrcID: another populated requester. - request[18+:12] = 7; // TxnID is opaque to transport. - request[50+:7] = 7'h01; // ReadShared. - request[58+:6] = 6; - request[64+:44] = 44'h100; - request[112] = 1; - request[119+:4] = 4'b1100; - request[123] = 1; - request[133] = 1; - vif.reset = 1; - vif.active = 0; - for (int vc = 0; vc < 4; vc++) begin - vif.valid[0][vc] = 0; - vif.data[0][vc] = 0; - vif.head[0][vc] = 0; - vif.tail[0][vc] = 0; - end - repeat (4) @(negedge vif.clock); - vif.reset = 0; - for (int chunk = 0; chunk < 5; chunk++) begin - vif.valid[0][0] = 1; - vif.data[0][0] = request[chunk*32+:32]; - vif.head[0][0] = chunk == 0; - vif.tail[0][0] = chunk == 4; - do @(posedge vif.clock); while (!vif.ready[0][0]); - @(negedge vif.clock); - end - vif.valid[0][0] = 0; - `uvm_info("ROLE_INPUT", - "Delivered all five REQ chunks: SrcID2, TgtID1, REQ VC0, correct HEAD/TAIL", - UVM_LOW) - repeat (8) @(negedge vif.clock); - `uvm_fatal("MISSING_ASSERTION", "requester accepted or discarded a wrong-role REQ") - endtask - endclass -endpackage diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_requester_req_tb.sv b/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_requester_req_tb.sv deleted file mode 100644 index 7be33bbea..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_requester_req_tb.sv +++ /dev/null @@ -1,126 +0,0 @@ -module bad_requester_req_tb; - import uvm_pkg::*; - import chi_bad_requester_req_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - mesh_fabric_if vif (clock); - ChiMeshRequesterEndpoint dut ( - .clock(clock), - .reset(vif.reset), - .io_chi_req_valid(1'b0), - .io_chi_req_bits_qos('0), - .io_chi_req_bits_tgtId('0), - .io_chi_req_bits_srcId('0), - .io_chi_req_bits_txnId('0), - .io_chi_req_bits_returnNid('0), - .io_chi_req_bits_stashNidValidEndian('0), - .io_chi_req_bits_returnTxnId('0), - .io_chi_req_bits_opcode('0), - .io_chi_req_bits_multiReq('0), - .io_chi_req_bits_size('0), - .io_chi_req_bits_addr('0), - .io_chi_req_bits_pas('0), - .io_chi_req_bits_likelyShared('0), - .io_chi_req_bits_allowRetry('0), - .io_chi_req_bits_order('0), - .io_chi_req_bits_pCrdType('0), - .io_chi_req_bits_memAttr('0), - .io_chi_req_bits_snpAttr('0), - .io_chi_req_bits_lpid('0), - .io_chi_req_bits_exclSnoopMe('0), - .io_chi_req_bits_expCompAck('0), - .io_chi_req_bits_tagOp('0), - .io_chi_req_bits_traceTag('0), - .io_chi_txRsp_valid(1'b0), - .io_chi_txRsp_bits_qos('0), - .io_chi_txRsp_bits_tgtId('0), - .io_chi_txRsp_bits_srcId('0), - .io_chi_txRsp_bits_txnId('0), - .io_chi_txRsp_bits_opcode('0), - .io_chi_txRsp_bits_respErr('0), - .io_chi_txRsp_bits_resp('0), - .io_chi_txRsp_bits_fwdState('0), - .io_chi_txRsp_bits_cBusy('0), - .io_chi_txRsp_bits_dbid('0), - .io_chi_txRsp_bits_pCrdType('0), - .io_chi_txRsp_bits_tagOp('0), - .io_chi_txRsp_bits_traceTag('0), - .io_chi_txRsp_bits_cacheLineId('0), - .io_chi_txDat_valid(1'b0), - .io_chi_txDat_bits_qos('0), - .io_chi_txDat_bits_tgtId('0), - .io_chi_txDat_bits_srcId('0), - .io_chi_txDat_bits_txnId('0), - .io_chi_txDat_bits_homeNid('0), - .io_chi_txDat_bits_opcode('0), - .io_chi_txDat_bits_respErr('0), - .io_chi_txDat_bits_resp('0), - .io_chi_txDat_bits_dataSource('0), - .io_chi_txDat_bits_dataPull('0), - .io_chi_txDat_bits_cBusy('0), - .io_chi_txDat_bits_dbid('0), - .io_chi_txDat_bits_ccid('0), - .io_chi_txDat_bits_dataId('0), - .io_chi_txDat_bits_cacheLineId('0), - .io_chi_txDat_bits_tagOp('0), - .io_chi_txDat_bits_tag('0), - .io_chi_txDat_bits_tagUpdate('0), - .io_chi_txDat_bits_traceTag('0), - .io_chi_txDat_bits_copyAtHome('0), - .io_chi_txDat_bits_numDat('0), - .io_chi_txDat_bits_replicate('0), - .io_chi_txDat_bits_be('0), - .io_chi_txDat_bits_data('0), - .io_chi_rxRsp_ready(1'b1), - .io_chi_rxDat_ready(1'b1), - .io_chi_snp_ready(1'b1), - .io_meshOut_0_ready(1'b1), - .io_meshIn_0_valid(vif.valid[0][0]), - .io_meshIn_0_ready(vif.ready[0][0]), - .io_meshIn_0_bits_srcX(1'b0), - .io_meshIn_0_bits_srcY(1'b1), - .io_meshIn_0_bits_dstX(1'b1), - .io_meshIn_0_bits_dstY(1'b0), - .io_meshIn_0_bits_vc(2'd0), - .io_meshIn_0_bits_head(vif.head[0][0]), - .io_meshIn_0_bits_tail(vif.tail[0][0]), - .io_meshIn_0_bits_payload(vif.data[0][0]), - .io_meshOut_1_ready(1'b1), - .io_meshIn_1_valid(vif.valid[0][1]), - .io_meshIn_1_ready(vif.ready[0][1]), - .io_meshIn_1_bits_srcX(1'b0), - .io_meshIn_1_bits_srcY(1'b1), - .io_meshIn_1_bits_dstX(1'b1), - .io_meshIn_1_bits_dstY(1'b0), - .io_meshIn_1_bits_vc(2'd1), - .io_meshIn_1_bits_head(vif.head[0][1]), - .io_meshIn_1_bits_tail(vif.tail[0][1]), - .io_meshIn_1_bits_payload(vif.data[0][1]), - .io_meshOut_2_ready(1'b1), - .io_meshIn_2_valid(vif.valid[0][2]), - .io_meshIn_2_ready(vif.ready[0][2]), - .io_meshIn_2_bits_srcX(1'b0), - .io_meshIn_2_bits_srcY(1'b1), - .io_meshIn_2_bits_dstX(1'b1), - .io_meshIn_2_bits_dstY(1'b0), - .io_meshIn_2_bits_vc(2'd2), - .io_meshIn_2_bits_head(vif.head[0][2]), - .io_meshIn_2_bits_tail(vif.tail[0][2]), - .io_meshIn_2_bits_payload(vif.data[0][2]), - .io_meshOut_3_ready(1'b1), - .io_meshIn_3_valid(vif.valid[0][3]), - .io_meshIn_3_ready(vif.ready[0][3]), - .io_meshIn_3_bits_srcX(1'b0), - .io_meshIn_3_bits_srcY(1'b1), - .io_meshIn_3_bits_dstX(1'b1), - .io_meshIn_3_bits_dstY(1'b0), - .io_meshIn_3_bits_vc(2'd3), - .io_meshIn_3_bits_head(vif.head[0][3]), - .io_meshIn_3_bits_tail(vif.tail[0][3]), - .io_meshIn_3_bits_payload(vif.data[0][3]) - ); - initial begin - uvm_config_db#(virtual mesh_fabric_if)::set(null, "uvm_test_top", "vif", vif); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/codec.el b/arch/src/main/scala/framework/root/root-chip/src/main/resources/codec.el deleted file mode 100644 index 8cc9bd3d6..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/codec.el +++ /dev/null @@ -1,180 +0,0 @@ -// Format Version: 2 -// Reviewed for 389-bit DAT / 32-bit chunks / populated NodeIDs {1,2,3,64}, coordinated reset. - -CHECKSUM: "3171296859 1964414076" -INSTANCE: codec_tb.dut.tx -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI Mesh target NodeID is not placed" -Block 2 "3889754848" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI Mesh target NodeID is not placed" -Block 3 "2159607100" "$error(\"Assertion failed: CHI Mesh target NodeID is not placed\n at ChiMesh.scala:81 when(io.in.valid) { assert(placed, \\"CHI Mesh target NodeID is not placed\\") }\n\");" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI Mesh target NodeID is not placed" -Block 5 "3889754848" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI Mesh target NodeID is not placed" -Block 6 "4027875381" "$fatal;" - -CHECKSUM: "1704682488 3830468848" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI mesh packet started without head" -Block 2 "4264478414" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI mesh packet started without head" -Block 3 "1178357294" "$error(\"Assertion failed: CHI mesh packet started without head\n at ChiMesh.scala:133 assert(io.in.bits.head, \\"CHI mesh packet started without head\\")\n\");" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI mesh packet started without head" -Block 5 "4264478414" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI mesh packet started without head" -Block 6 "520366355" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI mesh packet has a second head" -Block 10 "991107315" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI mesh packet has a second head" -Block 11 "1926190078" "$error(\"Assertion failed: CHI mesh packet has a second head\n at ChiMesh.scala:135 assert(!io.in.bits.head, \\"CHI mesh packet has a second head\\")\n\");" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI mesh packet has a second head" -Block 13 "991107315" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI mesh packet has a second head" -Block 14 "1882298097" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI mesh packet has wrong length" -Block 18 "1840269416" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI mesh packet has wrong length" -Block 19 "74803256" "$error(\"Assertion failed: CHI mesh packet has wrong length\n at ChiMesh.scala:137 assert(io.in.bits.tail === (beat === (chunks - 1).U), \\"CHI mesh packet has wrong length\\")\n\");" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI mesh packet has wrong length" -Block 21 "1840269416" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI mesh packet has wrong length" -Block 22 "1745333983" "$fatal;" - -CHECKSUM: "3171296859 4271561183" -INSTANCE: codec_tb.dut.tx -ANNOTATION: "NodeID rejection failure vector; a normally accepted request must target populated NodeIDs {1,2,3,64}; bad_node checks rejection separately" -Condition 1 "690909136" "(io_in_valid & ((~reset)) & ((~_GEN_1[io_in_bits_targetNode]))) 1 -1" (4 "111") - -CHECKSUM: "1704682488 2938369825" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "First accepted fragment has HEAD; shared reset keeps both codec counters aligned, including the reset-disabled failure vector" -Condition 1 "2778753700" "(_GEN & ((~collecting)) & ((~reset)) & ((~io_in_bits_head))) 1 -1" (3 "1101") - -CHECKSUM: "1704682488 2938369825" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "First accepted fragment has HEAD; shared reset keeps both codec counters aligned, including the reset-disabled failure vector" -Condition 1 "2778753700" "(_GEN & ((~collecting)) & ((~reset)) & ((~io_in_bits_head))) 1 -1" (5 "1111") - -CHECKSUM: "1704682488 2938369825" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "During collection packetizer HEAD remains false even if paused; shared reset cancels both endpoints together" -Condition 2 "142507406" "(_GEN & collecting & ((~reset)) & io_in_bits_head) 1 -1" (1 "0111") - -CHECKSUM: "1704682488 2938369825" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "During collection packetizer HEAD remains false even if paused; shared reset cancels both endpoints together" -Condition 2 "142507406" "(_GEN & collecting & ((~reset)) & io_in_bits_head) 1 -1" (3 "1101") - -CHECKSUM: "1704682488 2938369825" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "During collection packetizer HEAD remains false even if paused; shared reset cancels both endpoints together" -Condition 2 "142507406" "(_GEN & collecting & ((~reset)) & io_in_bits_head) 1 -1" (5 "1111") - -CHECKSUM: "1704682488 2938369825" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "Packetizer and receiver counters advance on exactly the same handshake; accepted TAIL always matches beat 12, including coordinated reset" -Condition 3 "1479293268" "(_GEN & ((~reset)) & (io_in_bits_tail != (beat == 4'hc))) 1 -1" (2 "101") - -CHECKSUM: "1704682488 2938369825" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "Packetizer and receiver counters advance on exactly the same handshake; accepted TAIL always matches beat 12, including coordinated reset" -Condition 3 "1479293268" "(_GEN & ((~reset)) & (io_in_bits_tail != (beat == 4'hc))) 1 -1" (4 "111") - -CHECKSUM: "1704682488 2938369825" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "Stored words are beats 0 through 11; legal TAIL can only accompany beat 12, so tail-at-stored-word vector is unreachable" -Condition 10 "291367532" "(_GEN & ((~io_in_bits_tail)) & (beat == 4'b0)) 1 -1" (2 "101") - -CHECKSUM: "1704682488 2938369825" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "Stored words are beats 0 through 11; legal TAIL can only accompany beat 12, so tail-at-stored-word vector is unreachable" -Condition 12 "684678524" "(_GEN & ((~io_in_bits_tail)) & (beat == 4'b1)) 1 -1" (2 "101") - -CHECKSUM: "1704682488 2938369825" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "Stored words are beats 0 through 11; legal TAIL can only accompany beat 12, so tail-at-stored-word vector is unreachable" -Condition 14 "3689188981" "(_GEN & ((~io_in_bits_tail)) & (beat == 4'h2)) 1 -1" (2 "101") - -CHECKSUM: "1704682488 2938369825" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "Stored words are beats 0 through 11; legal TAIL can only accompany beat 12, so tail-at-stored-word vector is unreachable" -Condition 16 "3799398757" "(_GEN & ((~io_in_bits_tail)) & (beat == 4'h3)) 1 -1" (2 "101") - -CHECKSUM: "1704682488 2938369825" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "Stored words are beats 0 through 11; legal TAIL can only accompany beat 12, so tail-at-stored-word vector is unreachable" -Condition 18 "2405101270" "(_GEN & ((~io_in_bits_tail)) & (beat == 4'h4)) 1 -1" (2 "101") - -CHECKSUM: "1704682488 2938369825" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "Stored words are beats 0 through 11; legal TAIL can only accompany beat 12, so tail-at-stored-word vector is unreachable" -Condition 20 "3066583494" "(_GEN & ((~io_in_bits_tail)) & (beat == 4'h5)) 1 -1" (2 "101") - -CHECKSUM: "1704682488 2938369825" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "Stored words are beats 0 through 11; legal TAIL can only accompany beat 12, so tail-at-stored-word vector is unreachable" -Condition 22 "3217531303" "(_GEN & ((~io_in_bits_tail)) & (beat == 4'h6)) 1 -1" (2 "101") - -CHECKSUM: "1704682488 2938369825" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "Stored words are beats 0 through 11; legal TAIL can only accompany beat 12, so tail-at-stored-word vector is unreachable" -Condition 24 "2253723319" "(_GEN & ((~io_in_bits_tail)) & (beat == 4'h7)) 1 -1" (2 "101") - -CHECKSUM: "1704682488 2938369825" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "Stored words are beats 0 through 11; legal TAIL can only accompany beat 12, so tail-at-stored-word vector is unreachable" -Condition 26 "759940555" "(_GEN & ((~io_in_bits_tail)) & (beat == 4'h8)) 1 -1" (2 "101") - -CHECKSUM: "1704682488 2938369825" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "Stored words are beats 0 through 11; legal TAIL can only accompany beat 12, so tail-at-stored-word vector is unreachable" -Condition 28 "349795035" "(_GEN & ((~io_in_bits_tail)) & (beat == 4'h9)) 1 -1" (2 "101") - -CHECKSUM: "1704682488 2938369825" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "Stored words are beats 0 through 11; legal TAIL can only accompany beat 12, so tail-at-stored-word vector is unreachable" -Condition 30 "811415760" "(_GEN & ((~io_in_bits_tail)) & (beat == 4'ha)) 1 -1" (2 "101") - -CHECKSUM: "1704682488 2938369825" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "Stored words are beats 0 through 11; legal TAIL can only accompany beat 12, so tail-at-stored-word vector is unreachable" -Condition 32 "961724593" "(_GEN & ((~io_in_bits_tail)) & (beat == 4'hb)) 1 -1" (2 "101") - -CHECKSUM: "2536693556 3895022188" -INSTANCE: codec_tb.dut -ANNOTATION: "Codec local injection coordinate is fixed (0,0) in this declared top" -Toggle io_observed_srcX "net io_observed_srcX" - -CHECKSUM: "2536693556 3895022188" -INSTANCE: codec_tb.dut -ANNOTATION: "Codec local injection coordinate is fixed (0,0) in this declared top" -Toggle io_observed_srcY "net io_observed_srcY" - -CHECKSUM: "2536693556 3895022188" -INSTANCE: codec_tb.dut -ANNOTATION: "This codec verification top transports only CHI DAT, VC2" -Toggle io_observed_vc "net io_observed_vc[1:0]" - -CHECKSUM: "3171296859 1555586989" -INSTANCE: codec_tb.dut.tx -ANNOTATION: "Elaboration-produced coordinate/presence lookup table consists entirely of literal constants; no datapath object selected by generated-name prefix" -Toggle _GEN "net _GEN[127:0]" - -CHECKSUM: "3171296859 1555586989" -INSTANCE: codec_tb.dut.tx -ANNOTATION: "Elaboration-produced coordinate/presence lookup table consists entirely of literal constants; no datapath object selected by generated-name prefix" -Toggle _GEN_0 "net _GEN_0[127:0]" - -CHECKSUM: "3171296859 1555586989" -INSTANCE: codec_tb.dut.tx -ANNOTATION: "Elaboration-produced coordinate/presence lookup table consists entirely of literal constants; no datapath object selected by generated-name prefix" -Toggle _GEN_1 "net _GEN_1[127:0]" - -CHECKSUM: "3171296859 1555586989" -INSTANCE: codec_tb.dut.tx -ANNOTATION: "Only NodeIDs 1,2,3,64 are admitted: accepted target bits 5 through 2 are always zero; bits 6,1,0 remain measured" -Toggle target [5:2] "reg target[6:0]" - -CHECKSUM: "3171296859 1555586989" -INSTANCE: codec_tb.dut.tx -ANNOTATION: "Only the upper 27 bits of the 416-bit zero-extended 389-bit flit are excluded; all useful bits remain checked" -Toggle _io_out_bits_payload_T_1 [415:389] "net _io_out_bits_payload_T_1[415:0]" diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/codec.f b/arch/src/main/scala/framework/root/root-chip/src/main/resources/codec.f deleted file mode 100644 index d64509fc6..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/codec.f +++ /dev/null @@ -1,7 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/codec_interface.sv -@RESOURCES@/codec_package.sv --F @RTL@/ChiMeshCodecLoopback/filelist.f -@RESOURCES@/codec_tb.sv diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/codec_interface.sv b/arch/src/main/scala/framework/root/root-chip/src/main/resources/codec_interface.sv deleted file mode 100644 index 7848dde2b..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/codec_interface.sv +++ /dev/null @@ -1,8 +0,0 @@ -interface chi_codec_if ( - input logic clock -); - logic reset, pause, valid, ready, out_valid, out_ready; - logic [6:0] target; - logic [388:0] data, out_data; - logic chunk_fire, chunk_head, chunk_tail, chunk_x, chunk_y; -endinterface diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/codec_package.sv b/arch/src/main/scala/framework/root/root-chip/src/main/resources/codec_package.sv deleted file mode 100644 index 39df7c68a..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/codec_package.sv +++ /dev/null @@ -1,124 +0,0 @@ -package chi_codec_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual chi_codec_if vif; - bit [388:0] expected[$]; - bit [6:0] targets[$]; - int checked = 0; - int cycles = 0; - bit flow = 0; - bit stalled; - bit [388:0] held; - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual chi_codec_if)::get(this, "", "vif", vif)) - `uvm_fatal("VIF", "codec missing") - endfunction - task monitor(); - forever begin - @(posedge vif.clock); - if (vif.reset) begin - expected.delete(); - targets.delete(); - stalled = 0; - end else begin - if (vif.valid && vif.ready) begin - expected.push_back(vif.data); - targets.push_back(vif.target); - end - if (vif.chunk_fire) begin - bit [6:0] target; - if (!targets.size()) `uvm_fatal("NODEMAP_EXTRA", "chunk without accepted target") - target = targets[0]; - if (vif.chunk_x !== (target & 1) || vif.chunk_y !== ((target >> 1) & 1)) - `uvm_fatal("NODEMAP", "CHI NodeID coordinate lookup mismatch") - if (vif.chunk_tail) target = targets.pop_front(); - end - if (stalled && (!vif.out_valid || vif.out_data !== held)) - `uvm_fatal("STABLE", "reassembled flit changed under stall") - if (vif.out_valid && vif.out_ready) begin - bit [388:0] golden; - if (!expected.size()) `uvm_fatal("EXTRA", "unexpected reassembled CHI DAT") - golden = expected.pop_front(); - if (vif.out_data !== golden) - `uvm_fatal("DATA", "CHI DAT bit packing/reassembly mismatch"); - checked++; - end - stalled = vif.out_valid && !vif.out_ready; - held = vif.out_data; - end - end - endtask - task flow_control(); - forever begin - @(negedge vif.clock); - cycles++; - if (flow) begin - vif.pause = cycles % 7 < 2; - vif.out_ready = cycles % 11 >= 4; - end else begin - vif.pause = 0; - vif.out_ready = 1; - end - end - endtask - task execute(); - vif.reset = 1; - vif.pause = 0; - vif.valid = 1; - vif.target = 0; - vif.data = 0; - vif.out_ready = 0; - fork - monitor(); - flow_control(); - join_none - repeat (4) @(negedge vif.clock); - vif.reset = 0; - vif.valid = 0; - // Invalid target metadata is legal when no transaction is offered. - for (int bitnum = 0; bitnum < 7; bitnum++) begin - vif.target = 1 << bitnum; - @(negedge vif.clock); - vif.target = 0; - @(negedge vif.clock); - end - vif.reset = 1; - vif.valid = 1; - vif.target = 0; - repeat (2) @(negedge vif.clock); - vif.reset = 0; - vif.valid = 0; - vif.target = 1; - flow = 1; - for (int packet = 0; packet < 32; packet++) begin - vif.target = packet % 4 == 0 ? 64 : packet % 4; - vif.valid = 1; - for (int bitnum = 0; bitnum < 389; bitnum++) vif.data[bitnum] = $urandom_range(0, 1); - do begin - @(posedge vif.clock); - if (vif.ready) begin - @(negedge vif.clock); - break; - end - @(negedge vif.clock); - end while (1); - - end - vif.valid = 0; - flow = 0; - while (checked != 32) @(negedge vif.clock); - repeat (8) @(negedge vif.clock); - `uvm_info( - "CHECKS", - "32 real 389-bit CHI DAT flits serialized to 13 mesh chunks and reassembled under backpressure", - UVM_LOW) - endtask - endclass -endpackage diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/codec_tb.sv b/arch/src/main/scala/framework/root/root-chip/src/main/resources/codec_tb.sv deleted file mode 100644 index cfe70d6d7..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/codec_tb.sv +++ /dev/null @@ -1,28 +0,0 @@ -module codec_tb; - import uvm_pkg::*; - import chi_codec_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - chi_codec_if vif (clock); - ChiMeshCodecLoopback dut ( - .clock(clock), - .reset(vif.reset), - .io_pause(vif.pause), - .io_in_valid(vif.valid), - .io_in_ready(vif.ready), - .io_in_bits_targetNode(vif.target), - .io_in_bits_flit(vif.data), - .io_out_valid(vif.out_valid), - .io_out_ready(vif.out_ready), - .io_out_bits(vif.out_data), - .io_observedFire(vif.chunk_fire), - .io_observed_head(vif.chunk_head), - .io_observed_tail(vif.chunk_tail), - .io_observed_dstX(vif.chunk_x), - .io_observed_dstY(vif.chunk_y) - ); - initial begin - uvm_config_db#(virtual chi_codec_if)::set(null, "uvm_test_top", "vif", vif); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/fabric_interface.sv b/arch/src/main/scala/framework/root/root-chip/src/main/resources/fabric_interface.sv deleted file mode 100644 index c855cd1ab..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/fabric_interface.sv +++ /dev/null @@ -1,11 +0,0 @@ -interface mesh_fabric_if ( - input logic clock -); - logic reset, active; - logic valid[5][4], ready[5][4], out_valid[5][4], out_ready[5][4]; - logic [1:0] x[5][4], y[5][4], src_x[5][4], src_y[5][4], vc[5][4]; - logic head[5][4], tail[5][4]; - logic [31:0] data[5][4], out_data[5][4]; - logic [1:0] out_src_x[5][4], out_src_y[5][4]; - logic out_head[5][4], out_tail[5][4]; -endinterface diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/network.f b/arch/src/main/scala/framework/root/root-chip/src/main/resources/network.f deleted file mode 100644 index fc8e0c60e..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/network.f +++ /dev/null @@ -1,7 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/fabric_interface.sv -@RESOURCES@/network_package.sv --F @RTL@/MeshCreditNetwork/filelist.f -@RESOURCES@/network_tb.sv diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/network_package.sv b/arch/src/main/scala/framework/root/root-chip/src/main/resources/network_package.sv deleted file mode 100644 index 58dc740c9..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/network_package.sv +++ /dev/null @@ -1,386 +0,0 @@ -package mesh_network_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual mesh_fabric_if vif; - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - timeout = 1ms; - if (!uvm_config_db#(virtual mesh_fabric_if)::get(this, "", "vif", vif)) - `uvm_fatal("VIF", "fabric interface missing") - endfunction - typedef bit [33:0] golden_beat; - typedef golden_beat beat_queue[$]; - task route_matrix(); - beat_queue expected[4][4][4]; - int submitted[4][4]; - int checked = 0; - bit done; - bit [31:0] payload; - for (int i = 0; i < 4; i++) for (int v = 0; v < 4; v++) submitted[i][v] = 0; - for (int tick = 0; tick < 60000; tick++) begin - for (int source = 0; source < 4; source++) - for (int v = 0; v < 4; v++) begin - int index, destination, pattern; - index = submitted[source][v]; - destination = index / (64 * 3); - pattern = (index / 3) % 64; - payload = 32'h1 << (pattern % 32); - if (pattern >= 32) payload = ~payload; - payload=payload^(source*32'h9e3779b9)^(v*32'h7f4a7c15)^(destination*32'h85ebca6b); - vif.valid[source][v] = index < 768; - vif.x[source][v] = destination % 2; - vif.y[source][v] = destination / 2; - vif.data[source][v] = payload; - vif.head[source][v] = index % 3 == 0; - vif.tail[source][v] = index % 3 == 2; - vif.out_ready[source][v] = tick % 17 >= ((source + v) % 5); - end - @(posedge vif.clock); - for (int source = 0; source < 4; source++) - for (int v = 0; v < 4; v++) - if (vif.valid[source][v] && vif.ready[source][v]) begin - int destination; - destination = vif.y[source][v] * 2 + vif.x[source][v]; - expected[source][destination][v].push_back( - {vif.tail[source][v], vif.head[source][v], vif.data[source][v]}); - submitted[source][v]++; - end - for (int destination = 0; destination < 4; destination++) - for (int v = 0; v < 4; v++) - if (vif.out_valid[destination][v] && vif.out_ready[destination][v]) begin - int source; - golden_beat golden; - source = vif.out_src_y[destination][v] * 2 + vif.out_src_x[destination][v]; - if (source >= 4 || !expected[source][destination][v].size()) - `uvm_fatal("MATRIX_EXTRA", "unmatched node/VC output") - golden = expected[source][destination][v].pop_front(); - if({vif.out_tail[destination][v],vif.out_head[destination][v],vif.out_data[destination][v]}!==golden) - `uvm_fatal("MATRIX_ORDER", "all-node source/destination/VC FIFO mismatch") - checked++; - end - @(negedge vif.clock); - done = checked == 12288; - if (done) begin - for (int source = 0; source < 4; source++) - for (int destination = 0; destination < 4; destination++) - for (int v = 0; v < 4; v++) - if (expected[source][destination][v].size()) - `uvm_fatal("MATRIX_PENDING", "unmatched accepted all-node request"); - break; - end - if (tick == 59999) - `uvm_fatal("MATRIX_TIMEOUT", "all-node walking/covering payload failed to drain") - end - for (int i = 0; i < 4; i++) - for (int v = 0; v < 4; v++) begin - vif.valid[i][v] = 0; - vif.out_ready[i][v] = 1; - end - repeat (8) @(negedge vif.clock); - `uvm_info( - "CHECKS", - "12288 all-node/all-VC walking+complement payload flits compared by per(source,destination,VC) FIFO including packet boundaries", - UVM_LOW) - endtask - task isolated_paths(); - isolated_reset(); - // Reset cancels packet ownership; local input metadata is unconstrained while reset is asserted. - vif.reset = 1; - for (int i = 0; i < 4; i++) - for (int v = 0; v < 4; v++) begin - vif.valid[i][v] = 1; - vif.head[i][v] = 0; - vif.tail[i][v] = 1; - vif.x[i][v] = i % 2; - vif.y[i][v] = i / 2; - vif.out_ready[i][v] = 1; - end - repeat (3) @(negedge vif.clock); - for (int i = 0; i < 4; i++) for (int v = 0; v < 4; v++) vif.valid[i][v] = 0; - vif.active = 0; - repeat (3) @(negedge vif.clock); - vif.reset = 0; - vif.active = 1; - repeat (20) begin - @(posedge vif.clock); - for (int i = 0; i < 4; i++) - for (int v = 0; v < 4; v++) - if (vif.out_valid[i][v]) - `uvm_fatal("RESET_FINAL", "reset-time metadata survived as a packet"); - @(negedge vif.clock); - end - // Exercise each physical route independently: fill its buffers, pause packet - // bodies while an owner is locked, then resume and compare every accepted flit. - for (int source = 0; source < 4; source++) - for (int path = 0; path < 6; path++) - for (int vc = 0; vc < 4; vc++) begin - int destination; - int submitted = 0, checked = 0, next_cycle = 0; - golden_beat expected[$]; - bit done = 0; - destination = path < 4 ? path : path == 4 ? (source ^ 3) : (source ^ 1); - for (int node = 0; node < 4; node++) - for (int v = 0; v < 4; v++) begin - vif.valid[node][v] = 0; - vif.out_ready[node][v] = 1; - end - for (int tick = 0; tick < 300; tick++) begin - vif.valid[source][vc] = submitted < 13 && tick >= next_cycle; - vif.x[source][vc] = vif.valid[source][vc] ? destination % 2 : (destination ^ 3) % 2; - vif.y[source][vc] = vif.valid[source][vc] ? destination / 2 : (destination ^ 3) / 2; - vif.head[source][vc] = submitted % 4 == 0; - vif.tail[source][vc] = submitted == 12 || submitted % 4 == 3; - vif.data[source][vc]=32'h61900000|(source<<16)|(destination<<12)|(vc<<8)|submitted; - vif.out_ready[destination][vc]=path>=4 || submitted>=12 || (tick>=45 && tick%13>=3); - @(posedge vif.clock); - if (vif.valid[source][vc] && vif.ready[source][vc]) begin - expected.push_back({vif.tail[source][vc], vif.head[source][vc], vif.data[source][vc] - }); - submitted++; - next_cycle = tick + 1 + (submitted == 12 ? 30 : submitted % 4 == 1 ? 7 : 0); - end - for (int node = 0; node < 4; node++) - for (int v = 0; v < 4; v++) - if (vif.out_valid[node][v] && vif.out_ready[node][v]) begin - golden_beat golden; - if (node != destination || v != vc || !expected.size()) - `uvm_fatal("PATH_EXTRA", "isolated path delivered unexpected flit") - golden = expected.pop_front(); - if({vif.out_tail[node][v],vif.out_head[node][v],vif.out_data[node][v]}!==golden || vif.out_src_x[node][v]!=source%2 || vif.out_src_y[node][v]!=source/2) - `uvm_fatal("PATH_DATA", "isolated packet path mismatch") - checked++; - end - @(negedge vif.clock); - if (checked == 13) begin - done = 1; - break; - end - end - if (!done || expected.size()) - `uvm_fatal("PATH_TIMEOUT", "isolated route failed to drain") - vif.valid[source][vc] = 0; - vif.out_ready[destination][vc] = 1; - repeat (8) @(negedge vif.clock); - end - `uvm_info( - "CHECKS", - "1248 isolated-path packet flits checked with owner gaps and per-route FIFO saturation", - UVM_LOW) - endtask - task isolated_reset(); - // Reset each physical horizontal link/VC after a packet head, before its tail. - for (int source = 0; source < 4; source++) - for (int vc = 0; vc < 4; vc++) begin - repeat (12) @(negedge vif.clock); - vif.valid[source][vc] = 1; - vif.head[source][vc] = 1; - vif.tail[source][vc] = 0; - vif.x[source][vc] = (source ^ 1) % 2; - vif.y[source][vc] = source / 2; - vif.data[source][vc] = 32'hca110000 | (source << 8) | vc; - @(posedge vif.clock); - if (!vif.ready[source][vc]) - `uvm_fatal("RESET_CREDIT", "isolated reset launch did not have its initialized credit") - @(negedge vif.clock); - vif.reset = 1; - vif.active = 0; - vif.valid[source][vc] = 0; - repeat (3) @(negedge vif.clock); - vif.reset = 0; - vif.active = 1; - repeat (12) begin - @(posedge vif.clock); - for (int i = 0; i < 4; i++) - for (int v = 0; v < 4; v++) - if (vif.out_valid[i][v]) - `uvm_fatal("RESET_ISOLATED", "cancelled per-VC flit survived reset"); - @(negedge vif.clock); - end - end - endtask - task reset_sweep(); - for (int step = 2; step < 13; step++) begin - for (int source = 0; source < 4; source++) - for (int v = 0; v < 4; v++) begin - vif.valid[source][v] = 1; - vif.x[source][v] = (source ^ 3) % 2; - vif.y[source][v] = (source ^ 3) / 2; - vif.head[source][v] = 1; - vif.tail[source][v] = 1; - vif.data[source][v] = 32'hcc000000 | (source << 8) | v; - vif.out_ready[source][v] = 0; - end - repeat (step) @(negedge vif.clock); - vif.reset = 1; - vif.active = 0; - for (int source = 0; source < 4; source++) - for (int v = 0; v < 4; v++) vif.valid[source][v] = 0; - repeat (4) @(negedge vif.clock); - vif.reset = 0; - vif.active = 1; - for (int source = 0; source < 4; source++) - for (int v = 0; v < 4; v++) vif.out_ready[source][v] = 1; - repeat (20) begin - @(posedge vif.clock); - for (int source = 0; source < 4; source++) - for (int v = 0; v < 4; v++) - if (vif.out_valid[source][v]) - `uvm_fatal("RESET_STAGE", "cancelled pipeline-stage flit survived reset"); - @(negedge vif.clock); - end - end - endtask - task execute(); - int sent[4][4], received[4][4]; - bit complete; - bit stalled[4][4]; - bit [31:0] held[4][4]; - vif.reset = 1; - vif.active = 0; - for (int i = 0; i < 4; i++) - for (int v = 0; v < 4; v++) begin - sent[i][v] = 0; - received[i][v] = 0; - stalled[i][v] = 0; - vif.valid[i][v] = 0; - vif.x[i][v] = 1; - vif.y[i][v] = 1; - vif.out_ready[i][v] = 1; - end - repeat (4) @(negedge vif.clock); - vif.reset = 0; - for (int i = 0; i < 4; i++) - for (int v = 0; v < 4; v++) begin - vif.valid[i][v] = 1; - vif.head[i][v] = 1; - vif.tail[i][v] = 1; - vif.data[i][v] = 32'habcd; - end - repeat (4) begin - @(posedge vif.clock); - for (int i = 0; i < 4; i++) - for (int v = 0; v < 4; v++) - if (vif.ready[i][v] || vif.out_valid[i][v]) - `uvm_fatal("INACTIVE", "inactive local path accepted traffic"); - @(negedge vif.clock); - end - for (int i = 0; i < 4; i++) for (int v = 0; v < 4; v++) vif.valid[i][v] = 0; - vif.active = 1; - vif.out_ready[3][0] = 0; - for (int tick = 0; tick < 1000; tick++) begin - for (int i = 0; i < 4; i++) - for (int v = 0; v < 4; v++) begin - vif.valid[i][v] = sent[i][v] < 24; - vif.head[i][v] = sent[i][v] % 3 == 0; - vif.tail[i][v] = sent[i][v] % 3 == 2; - vif.data[i][v] = (i << 24) | (v << 16) | sent[i][v]; - end - @(posedge vif.clock); - for (int i = 0; i < 4; i++) - for (int v = 0; v < 4; v++) if (vif.valid[i][v] && vif.ready[i][v]) sent[i][v]++; - for (int destination = 0; destination < 4; destination++) - for (int v = 0; v < 4; v++) begin - if(stalled[destination][v]&&(!vif.out_valid[destination][v]||vif.out_data[destination][v]!==held[destination][v])) - `uvm_fatal("STABLE", "network VC changed under stall") - stalled[destination][v] = vif.out_valid[destination][v] && !vif.out_ready[destination][v]; - held[destination][v] = vif.out_data[destination][v]; - end - for (int destination = 0; destination < 4; destination++) - for (int v = 0; v < 4; v++) - if (vif.out_valid[destination][v] && vif.out_ready[destination][v]) begin - int source, seq; - source = vif.out_data[destination][v] >> 24; - seq = vif.out_data[destination][v] & 65535; - if(destination!=3||source>=4||seq!=received[source][v]||((vif.out_data[destination][v]>>16)&255)!=v) - `uvm_fatal("ORDER", "network lost, duplicated or reordered source/VC flit") - received[source][v]++; - end - @(negedge vif.clock); - if (tick == 350) begin - for (int i = 0; i < 4; i++) - for (int v = 1; v < 4; v++) - if (received[i][v] != 24) `uvm_fatal("ISOLATION", "REQ blocked unrelated CHI protocol VC") - vif.out_ready[3][0] = 1; - end - complete = 1; - for (int i = 0; i < 4; i++) - for (int v = 0; v < 4; v++) if (received[i][v] != 24) complete = 0; - if (complete) break; - if (tick == 999) `uvm_fatal("TIMEOUT", "multi-hop network did not drain") - end - for (int i = 0; i < 4; i++) for (int v = 0; v < 4; v++) vif.valid[i][v] = 0; - repeat (10) @(negedge vif.clock); - // Reset cancels buffered traffic and starts a fresh per-VC credit epoch. - vif.out_ready[3][0] = 0; - vif.valid[0][0] = 1; - vif.head[0][0] = 1; - vif.tail[0][0] = 1; - vif.data[0][0] = 32'hdeadbeef; - do @(posedge vif.clock); while (!vif.ready[0][0]); - @(negedge vif.clock); - vif.valid[0][0] = 0; - repeat (12) @(negedge vif.clock); - vif.reset = 1; - vif.active = 0; - repeat (4) @(negedge vif.clock); - vif.reset = 0; - vif.active = 1; - vif.out_ready[3][0] = 1; - repeat (20) begin - @(posedge vif.clock); - for (int i = 0; i < 4; i++) - for (int v = 0; v < 4; v++) - if (vif.out_valid[i][v]) `uvm_fatal("RESET", "pre-reset flit survived coordinated reset"); - @(negedge vif.clock); - end - vif.valid[0][0] = 1; - vif.data[0][0] = 32'h5eedca11; - do @(posedge vif.clock); while (!vif.ready[0][0]); - @(negedge vif.clock); - vif.valid[0][0] = 0; - do @(posedge vif.clock); while (!vif.out_valid[3][0]); - if (vif.out_data[3][0] !== 32'h5eedca11) `uvm_fatal("RESET", "fresh epoch payload mismatch"); - @(negedge vif.clock); - repeat (8) @(negedge vif.clock); - reset_sweep(); - route_matrix(); - isolated_paths(); - isolated_reset(); - // Reset cancels packet ownership; local input metadata is unconstrained while reset is asserted. - vif.reset = 1; - for (int i = 0; i < 4; i++) - for (int v = 0; v < 4; v++) begin - vif.valid[i][v] = 1; - vif.head[i][v] = 0; - vif.tail[i][v] = 1; - vif.x[i][v] = i % 2; - vif.y[i][v] = i / 2; - vif.out_ready[i][v] = 1; - end - repeat (3) @(negedge vif.clock); - for (int i = 0; i < 4; i++) for (int v = 0; v < 4; v++) vif.valid[i][v] = 0; - vif.active = 0; - repeat (3) @(negedge vif.clock); - vif.reset = 0; - vif.active = 1; - repeat (20) begin - @(posedge vif.clock); - for (int i = 0; i < 4; i++) - for (int v = 0; v < 4; v++) - if (vif.out_valid[i][v]) - `uvm_fatal("RESET_FINAL", "reset-time metadata survived as a packet"); - @(negedge vif.clock); - end - `uvm_info( - "CHECKS", - "384 multi-hop packet flits checked by source/VC; RSP/DAT/SNP finish while REQ blocked", - UVM_LOW) - endtask - endclass -endpackage diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/network_tb.sv b/arch/src/main/scala/framework/root/root-chip/src/main/resources/network_tb.sv deleted file mode 100644 index e5c897afc..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/network_tb.sv +++ /dev/null @@ -1,288 +0,0 @@ -module network_tb; - import uvm_pkg::*; - import mesh_network_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - mesh_fabric_if vif (clock); - MeshCreditNetwork dut ( - .clock(clock), - .reset(vif.reset), - .io_active(vif.active), - .io_localIn_0_0_valid(vif.valid[0][0]), - .io_localIn_0_0_ready(vif.ready[0][0]), - .io_localIn_0_0_bits_dstX(vif.x[0][0]), - .io_localIn_0_0_bits_dstY(vif.y[0][0]), - .io_localIn_0_0_bits_head(vif.head[0][0]), - .io_localIn_0_0_bits_tail(vif.tail[0][0]), - .io_localIn_0_0_bits_payload(vif.data[0][0]), - .io_localIn_0_0_bits_srcX(1'd0), - .io_localIn_0_0_bits_srcY(1'd0), - .io_localIn_0_0_bits_vc(2'd0), - .io_localOut_0_0_valid(vif.out_valid[0][0]), - .io_localOut_0_0_ready(vif.out_ready[0][0]), - .io_localOut_0_0_bits_payload(vif.out_data[0][0]), - .io_localOut_0_0_bits_srcX(vif.out_src_x[0][0]), - .io_localOut_0_0_bits_srcY(vif.out_src_y[0][0]), - .io_localOut_0_0_bits_head(vif.out_head[0][0]), - .io_localOut_0_0_bits_tail(vif.out_tail[0][0]), - .io_localIn_0_1_valid(vif.valid[0][1]), - .io_localIn_0_1_ready(vif.ready[0][1]), - .io_localIn_0_1_bits_dstX(vif.x[0][1]), - .io_localIn_0_1_bits_dstY(vif.y[0][1]), - .io_localIn_0_1_bits_head(vif.head[0][1]), - .io_localIn_0_1_bits_tail(vif.tail[0][1]), - .io_localIn_0_1_bits_payload(vif.data[0][1]), - .io_localIn_0_1_bits_srcX(1'd0), - .io_localIn_0_1_bits_srcY(1'd0), - .io_localIn_0_1_bits_vc(2'd1), - .io_localOut_0_1_valid(vif.out_valid[0][1]), - .io_localOut_0_1_ready(vif.out_ready[0][1]), - .io_localOut_0_1_bits_payload(vif.out_data[0][1]), - .io_localOut_0_1_bits_srcX(vif.out_src_x[0][1]), - .io_localOut_0_1_bits_srcY(vif.out_src_y[0][1]), - .io_localOut_0_1_bits_head(vif.out_head[0][1]), - .io_localOut_0_1_bits_tail(vif.out_tail[0][1]), - .io_localIn_0_2_valid(vif.valid[0][2]), - .io_localIn_0_2_ready(vif.ready[0][2]), - .io_localIn_0_2_bits_dstX(vif.x[0][2]), - .io_localIn_0_2_bits_dstY(vif.y[0][2]), - .io_localIn_0_2_bits_head(vif.head[0][2]), - .io_localIn_0_2_bits_tail(vif.tail[0][2]), - .io_localIn_0_2_bits_payload(vif.data[0][2]), - .io_localIn_0_2_bits_srcX(1'd0), - .io_localIn_0_2_bits_srcY(1'd0), - .io_localIn_0_2_bits_vc(2'd2), - .io_localOut_0_2_valid(vif.out_valid[0][2]), - .io_localOut_0_2_ready(vif.out_ready[0][2]), - .io_localOut_0_2_bits_payload(vif.out_data[0][2]), - .io_localOut_0_2_bits_srcX(vif.out_src_x[0][2]), - .io_localOut_0_2_bits_srcY(vif.out_src_y[0][2]), - .io_localOut_0_2_bits_head(vif.out_head[0][2]), - .io_localOut_0_2_bits_tail(vif.out_tail[0][2]), - .io_localIn_0_3_valid(vif.valid[0][3]), - .io_localIn_0_3_ready(vif.ready[0][3]), - .io_localIn_0_3_bits_dstX(vif.x[0][3]), - .io_localIn_0_3_bits_dstY(vif.y[0][3]), - .io_localIn_0_3_bits_head(vif.head[0][3]), - .io_localIn_0_3_bits_tail(vif.tail[0][3]), - .io_localIn_0_3_bits_payload(vif.data[0][3]), - .io_localIn_0_3_bits_srcX(1'd0), - .io_localIn_0_3_bits_srcY(1'd0), - .io_localIn_0_3_bits_vc(2'd3), - .io_localOut_0_3_valid(vif.out_valid[0][3]), - .io_localOut_0_3_ready(vif.out_ready[0][3]), - .io_localOut_0_3_bits_payload(vif.out_data[0][3]), - .io_localOut_0_3_bits_srcX(vif.out_src_x[0][3]), - .io_localOut_0_3_bits_srcY(vif.out_src_y[0][3]), - .io_localOut_0_3_bits_head(vif.out_head[0][3]), - .io_localOut_0_3_bits_tail(vif.out_tail[0][3]), - .io_localIn_1_0_valid(vif.valid[1][0]), - .io_localIn_1_0_ready(vif.ready[1][0]), - .io_localIn_1_0_bits_dstX(vif.x[1][0]), - .io_localIn_1_0_bits_dstY(vif.y[1][0]), - .io_localIn_1_0_bits_head(vif.head[1][0]), - .io_localIn_1_0_bits_tail(vif.tail[1][0]), - .io_localIn_1_0_bits_payload(vif.data[1][0]), - .io_localIn_1_0_bits_srcX(1'd1), - .io_localIn_1_0_bits_srcY(1'd0), - .io_localIn_1_0_bits_vc(2'd0), - .io_localOut_1_0_valid(vif.out_valid[1][0]), - .io_localOut_1_0_ready(vif.out_ready[1][0]), - .io_localOut_1_0_bits_payload(vif.out_data[1][0]), - .io_localOut_1_0_bits_srcX(vif.out_src_x[1][0]), - .io_localOut_1_0_bits_srcY(vif.out_src_y[1][0]), - .io_localOut_1_0_bits_head(vif.out_head[1][0]), - .io_localOut_1_0_bits_tail(vif.out_tail[1][0]), - .io_localIn_1_1_valid(vif.valid[1][1]), - .io_localIn_1_1_ready(vif.ready[1][1]), - .io_localIn_1_1_bits_dstX(vif.x[1][1]), - .io_localIn_1_1_bits_dstY(vif.y[1][1]), - .io_localIn_1_1_bits_head(vif.head[1][1]), - .io_localIn_1_1_bits_tail(vif.tail[1][1]), - .io_localIn_1_1_bits_payload(vif.data[1][1]), - .io_localIn_1_1_bits_srcX(1'd1), - .io_localIn_1_1_bits_srcY(1'd0), - .io_localIn_1_1_bits_vc(2'd1), - .io_localOut_1_1_valid(vif.out_valid[1][1]), - .io_localOut_1_1_ready(vif.out_ready[1][1]), - .io_localOut_1_1_bits_payload(vif.out_data[1][1]), - .io_localOut_1_1_bits_srcX(vif.out_src_x[1][1]), - .io_localOut_1_1_bits_srcY(vif.out_src_y[1][1]), - .io_localOut_1_1_bits_head(vif.out_head[1][1]), - .io_localOut_1_1_bits_tail(vif.out_tail[1][1]), - .io_localIn_1_2_valid(vif.valid[1][2]), - .io_localIn_1_2_ready(vif.ready[1][2]), - .io_localIn_1_2_bits_dstX(vif.x[1][2]), - .io_localIn_1_2_bits_dstY(vif.y[1][2]), - .io_localIn_1_2_bits_head(vif.head[1][2]), - .io_localIn_1_2_bits_tail(vif.tail[1][2]), - .io_localIn_1_2_bits_payload(vif.data[1][2]), - .io_localIn_1_2_bits_srcX(1'd1), - .io_localIn_1_2_bits_srcY(1'd0), - .io_localIn_1_2_bits_vc(2'd2), - .io_localOut_1_2_valid(vif.out_valid[1][2]), - .io_localOut_1_2_ready(vif.out_ready[1][2]), - .io_localOut_1_2_bits_payload(vif.out_data[1][2]), - .io_localOut_1_2_bits_srcX(vif.out_src_x[1][2]), - .io_localOut_1_2_bits_srcY(vif.out_src_y[1][2]), - .io_localOut_1_2_bits_head(vif.out_head[1][2]), - .io_localOut_1_2_bits_tail(vif.out_tail[1][2]), - .io_localIn_1_3_valid(vif.valid[1][3]), - .io_localIn_1_3_ready(vif.ready[1][3]), - .io_localIn_1_3_bits_dstX(vif.x[1][3]), - .io_localIn_1_3_bits_dstY(vif.y[1][3]), - .io_localIn_1_3_bits_head(vif.head[1][3]), - .io_localIn_1_3_bits_tail(vif.tail[1][3]), - .io_localIn_1_3_bits_payload(vif.data[1][3]), - .io_localIn_1_3_bits_srcX(1'd1), - .io_localIn_1_3_bits_srcY(1'd0), - .io_localIn_1_3_bits_vc(2'd3), - .io_localOut_1_3_valid(vif.out_valid[1][3]), - .io_localOut_1_3_ready(vif.out_ready[1][3]), - .io_localOut_1_3_bits_payload(vif.out_data[1][3]), - .io_localOut_1_3_bits_srcX(vif.out_src_x[1][3]), - .io_localOut_1_3_bits_srcY(vif.out_src_y[1][3]), - .io_localOut_1_3_bits_head(vif.out_head[1][3]), - .io_localOut_1_3_bits_tail(vif.out_tail[1][3]), - .io_localIn_2_0_valid(vif.valid[2][0]), - .io_localIn_2_0_ready(vif.ready[2][0]), - .io_localIn_2_0_bits_dstX(vif.x[2][0]), - .io_localIn_2_0_bits_dstY(vif.y[2][0]), - .io_localIn_2_0_bits_head(vif.head[2][0]), - .io_localIn_2_0_bits_tail(vif.tail[2][0]), - .io_localIn_2_0_bits_payload(vif.data[2][0]), - .io_localIn_2_0_bits_srcX(1'd0), - .io_localIn_2_0_bits_srcY(1'd1), - .io_localIn_2_0_bits_vc(2'd0), - .io_localOut_2_0_valid(vif.out_valid[2][0]), - .io_localOut_2_0_ready(vif.out_ready[2][0]), - .io_localOut_2_0_bits_payload(vif.out_data[2][0]), - .io_localOut_2_0_bits_srcX(vif.out_src_x[2][0]), - .io_localOut_2_0_bits_srcY(vif.out_src_y[2][0]), - .io_localOut_2_0_bits_head(vif.out_head[2][0]), - .io_localOut_2_0_bits_tail(vif.out_tail[2][0]), - .io_localIn_2_1_valid(vif.valid[2][1]), - .io_localIn_2_1_ready(vif.ready[2][1]), - .io_localIn_2_1_bits_dstX(vif.x[2][1]), - .io_localIn_2_1_bits_dstY(vif.y[2][1]), - .io_localIn_2_1_bits_head(vif.head[2][1]), - .io_localIn_2_1_bits_tail(vif.tail[2][1]), - .io_localIn_2_1_bits_payload(vif.data[2][1]), - .io_localIn_2_1_bits_srcX(1'd0), - .io_localIn_2_1_bits_srcY(1'd1), - .io_localIn_2_1_bits_vc(2'd1), - .io_localOut_2_1_valid(vif.out_valid[2][1]), - .io_localOut_2_1_ready(vif.out_ready[2][1]), - .io_localOut_2_1_bits_payload(vif.out_data[2][1]), - .io_localOut_2_1_bits_srcX(vif.out_src_x[2][1]), - .io_localOut_2_1_bits_srcY(vif.out_src_y[2][1]), - .io_localOut_2_1_bits_head(vif.out_head[2][1]), - .io_localOut_2_1_bits_tail(vif.out_tail[2][1]), - .io_localIn_2_2_valid(vif.valid[2][2]), - .io_localIn_2_2_ready(vif.ready[2][2]), - .io_localIn_2_2_bits_dstX(vif.x[2][2]), - .io_localIn_2_2_bits_dstY(vif.y[2][2]), - .io_localIn_2_2_bits_head(vif.head[2][2]), - .io_localIn_2_2_bits_tail(vif.tail[2][2]), - .io_localIn_2_2_bits_payload(vif.data[2][2]), - .io_localIn_2_2_bits_srcX(1'd0), - .io_localIn_2_2_bits_srcY(1'd1), - .io_localIn_2_2_bits_vc(2'd2), - .io_localOut_2_2_valid(vif.out_valid[2][2]), - .io_localOut_2_2_ready(vif.out_ready[2][2]), - .io_localOut_2_2_bits_payload(vif.out_data[2][2]), - .io_localOut_2_2_bits_srcX(vif.out_src_x[2][2]), - .io_localOut_2_2_bits_srcY(vif.out_src_y[2][2]), - .io_localOut_2_2_bits_head(vif.out_head[2][2]), - .io_localOut_2_2_bits_tail(vif.out_tail[2][2]), - .io_localIn_2_3_valid(vif.valid[2][3]), - .io_localIn_2_3_ready(vif.ready[2][3]), - .io_localIn_2_3_bits_dstX(vif.x[2][3]), - .io_localIn_2_3_bits_dstY(vif.y[2][3]), - .io_localIn_2_3_bits_head(vif.head[2][3]), - .io_localIn_2_3_bits_tail(vif.tail[2][3]), - .io_localIn_2_3_bits_payload(vif.data[2][3]), - .io_localIn_2_3_bits_srcX(1'd0), - .io_localIn_2_3_bits_srcY(1'd1), - .io_localIn_2_3_bits_vc(2'd3), - .io_localOut_2_3_valid(vif.out_valid[2][3]), - .io_localOut_2_3_ready(vif.out_ready[2][3]), - .io_localOut_2_3_bits_payload(vif.out_data[2][3]), - .io_localOut_2_3_bits_srcX(vif.out_src_x[2][3]), - .io_localOut_2_3_bits_srcY(vif.out_src_y[2][3]), - .io_localOut_2_3_bits_head(vif.out_head[2][3]), - .io_localOut_2_3_bits_tail(vif.out_tail[2][3]), - .io_localIn_3_0_valid(vif.valid[3][0]), - .io_localIn_3_0_ready(vif.ready[3][0]), - .io_localIn_3_0_bits_dstX(vif.x[3][0]), - .io_localIn_3_0_bits_dstY(vif.y[3][0]), - .io_localIn_3_0_bits_head(vif.head[3][0]), - .io_localIn_3_0_bits_tail(vif.tail[3][0]), - .io_localIn_3_0_bits_payload(vif.data[3][0]), - .io_localIn_3_0_bits_srcX(1'd1), - .io_localIn_3_0_bits_srcY(1'd1), - .io_localIn_3_0_bits_vc(2'd0), - .io_localOut_3_0_valid(vif.out_valid[3][0]), - .io_localOut_3_0_ready(vif.out_ready[3][0]), - .io_localOut_3_0_bits_payload(vif.out_data[3][0]), - .io_localOut_3_0_bits_srcX(vif.out_src_x[3][0]), - .io_localOut_3_0_bits_srcY(vif.out_src_y[3][0]), - .io_localOut_3_0_bits_head(vif.out_head[3][0]), - .io_localOut_3_0_bits_tail(vif.out_tail[3][0]), - .io_localIn_3_1_valid(vif.valid[3][1]), - .io_localIn_3_1_ready(vif.ready[3][1]), - .io_localIn_3_1_bits_dstX(vif.x[3][1]), - .io_localIn_3_1_bits_dstY(vif.y[3][1]), - .io_localIn_3_1_bits_head(vif.head[3][1]), - .io_localIn_3_1_bits_tail(vif.tail[3][1]), - .io_localIn_3_1_bits_payload(vif.data[3][1]), - .io_localIn_3_1_bits_srcX(1'd1), - .io_localIn_3_1_bits_srcY(1'd1), - .io_localIn_3_1_bits_vc(2'd1), - .io_localOut_3_1_valid(vif.out_valid[3][1]), - .io_localOut_3_1_ready(vif.out_ready[3][1]), - .io_localOut_3_1_bits_payload(vif.out_data[3][1]), - .io_localOut_3_1_bits_srcX(vif.out_src_x[3][1]), - .io_localOut_3_1_bits_srcY(vif.out_src_y[3][1]), - .io_localOut_3_1_bits_head(vif.out_head[3][1]), - .io_localOut_3_1_bits_tail(vif.out_tail[3][1]), - .io_localIn_3_2_valid(vif.valid[3][2]), - .io_localIn_3_2_ready(vif.ready[3][2]), - .io_localIn_3_2_bits_dstX(vif.x[3][2]), - .io_localIn_3_2_bits_dstY(vif.y[3][2]), - .io_localIn_3_2_bits_head(vif.head[3][2]), - .io_localIn_3_2_bits_tail(vif.tail[3][2]), - .io_localIn_3_2_bits_payload(vif.data[3][2]), - .io_localIn_3_2_bits_srcX(1'd1), - .io_localIn_3_2_bits_srcY(1'd1), - .io_localIn_3_2_bits_vc(2'd2), - .io_localOut_3_2_valid(vif.out_valid[3][2]), - .io_localOut_3_2_ready(vif.out_ready[3][2]), - .io_localOut_3_2_bits_payload(vif.out_data[3][2]), - .io_localOut_3_2_bits_srcX(vif.out_src_x[3][2]), - .io_localOut_3_2_bits_srcY(vif.out_src_y[3][2]), - .io_localOut_3_2_bits_head(vif.out_head[3][2]), - .io_localOut_3_2_bits_tail(vif.out_tail[3][2]), - .io_localIn_3_3_valid(vif.valid[3][3]), - .io_localIn_3_3_ready(vif.ready[3][3]), - .io_localIn_3_3_bits_dstX(vif.x[3][3]), - .io_localIn_3_3_bits_dstY(vif.y[3][3]), - .io_localIn_3_3_bits_head(vif.head[3][3]), - .io_localIn_3_3_bits_tail(vif.tail[3][3]), - .io_localIn_3_3_bits_payload(vif.data[3][3]), - .io_localIn_3_3_bits_srcX(1'd1), - .io_localIn_3_3_bits_srcY(1'd1), - .io_localIn_3_3_bits_vc(2'd3), - .io_localOut_3_3_valid(vif.out_valid[3][3]), - .io_localOut_3_3_ready(vif.out_ready[3][3]), - .io_localOut_3_3_bits_payload(vif.out_data[3][3]), - .io_localOut_3_3_bits_srcX(vif.out_src_x[3][3]), - .io_localOut_3_3_bits_srcY(vif.out_src_y[3][3]), - .io_localOut_3_3_bits_head(vif.out_head[3][3]), - .io_localOut_3_3_bits_tail(vif.out_tail[3][3]) - ); - initial begin - uvm_config_db#(virtual mesh_fabric_if)::set(null, "uvm_test_top", "vif", vif); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/router.el b/arch/src/main/scala/framework/root/root-chip/src/main/resources/router.el deleted file mode 100644 index 8962495b3..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/router.el +++ /dev/null @@ -1,502 +0,0 @@ -// Format Version: 2 -// Reviewed exact objects; native URG checksums/signatures and bit/transition scope preserved. - -CHECKSUM: "1802735833 964344264" -INSTANCE: router_tb.dut -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Mesh flit arrived on wrong VC port" -Block 2 "2419507893" "if (1)" -Block 3 "3668136291" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 5 "2419507893" "if (1)" -Block 6 "2132654405" "$fatal;" -Block 26 "1224552289" "if (1)" -Block 27 "409298446" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 29 "1224552289" "if (1)" -Block 30 "3185282600" "$fatal;" -Block 50 "1934849908" "if (1)" -Block 51 "1166025150" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 53 "1934849908" "if (1)" -Block 54 "3762211224" "$fatal;" -Block 74 "65211867" "if (1)" -Block 75 "4087705017" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 77 "65211867" "if (1)" -Block 78 "1444644255" "$fatal;" -Block 98 "1600740343" "if (1)" -Block 99 "2104806949" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 101 "1600740343" "if (1)" -Block 102 "3637126659" "$fatal;" -Block 122 "2168220930" "if (1)" -Block 123 "2564218234" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 125 "2168220930" "if (1)" -Block 126 "1030239580" "$fatal;" -Block 146 "3814076770" "if (1)" -Block 147 "2257455454" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 149 "3814076770" "if (1)" -Block 150 "590406008" "$fatal;" -Block 170 "619444456" "if (1)" -Block 171 "938657885" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 173 "619444456" "if (1)" -Block 174 "2454462587" "$fatal;" -Block 194 "2288343919" "if (1)" -Block 195 "2897478908" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 197 "2288343919" "if (1)" -Block 198 "151710938" "$fatal;" -Block 218 "2768801095" "if (1)" -Block 219 "3659715001" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 221 "2768801095" "if (1)" -Block 222 "2140944799" "$fatal;" -Block 242 "147697006" "if (1)" -Block 243 "253650898" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 245 "147697006" "if (1)" -Block 246 "2862649332" "$fatal;" -Block 266 "1789391118" "if (1)" -Block 267 "289846262" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 269 "1789391118" "if (1)" -Block 270 "3036175312" "$fatal;" -Block 290 "3616373105" "if (1)" -Block 291 "2174420298" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 293 "3616373105" "if (1)" -Block 294 "606465388" "$fatal;" -Block 314 "2063850230" "if (1)" -Block 315 "450742763" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 317 "2063850230" "if (1)" -Block 318 "3210944973" "$fatal;" -Block 338 "1450052830" "if (1)" -Block 339 "1816861870" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 341 "1450052830" "if (1)" -Block 342 "3388339336" "$fatal;" -Block 362 "171706947" "if (1)" -Block 363 "2243434851" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 365 "171706947" "if (1)" -Block 366 "537325893" "$fatal;" -Block 386 "1750488611" "if (1)" -Block 387 "2615201095" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 389 "1750488611" "if (1)" -Block 390 "1046357345" "$fatal;" -Block 410 "2951454633" "if (1)" -Block 411 "714869828" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 413 "2951454633" "if (1)" -Block 414 "2401428578" "$fatal;" -Block 434 "56966190" "if (1)" -Block 435 "2983903461" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 437 "56966190" "if (1)" -Block 438 "342108355" "$fatal;" -Block 458 "772336134" "if (1)" -Block 459 "3343652256" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 461 "772336134" "if (1)" -Block 462 "1660091782" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Mesh destination outside topology" -Block 10 "3524853549" "if (1)" -Block 11 "3668459727" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 13 "3524853549" "if (1)" -Block 14 "127688172" "$fatal;" -Block 34 "624517000" "if (1)" -Block 35 "572034051" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 37 "624517000" "if (1)" -Block 38 "4281110816" "$fatal;" -Block 58 "2799459455" "if (1)" -Block 59 "65291866" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 61 "2799459455" "if (1)" -Block 62 "3738196857" "$fatal;" -Block 82 "2466509191" "if (1)" -Block 83 "537394510" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 85 "2466509191" "if (1)" -Block 86 "4248035437" "$fatal;" -Block 106 "243241476" "if (1)" -Block 107 "429169505" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 109 "243241476" "if (1)" -Block 110 "3298854466" "$fatal;" -Block 130 "2931492635" "if (1)" -Block 131 "1613403034" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 133 "2931492635" "if (1)" -Block 134 "3172896441" "$fatal;" -Block 154 "1503307710" "if (1)" -Block 155 "2560253782" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 157 "1503307710" "if (1)" -Block 158 "1169048181" "$fatal;" -Block 178 "3473843463" "if (1)" -Block 179 "3677482553" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 181 "3473843463" "if (1)" -Block 182 "101053210" "$fatal;" -Block 202 "4208003327" "if (1)" -Block 203 "4174533933" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 205 "4208003327" "if (1)" -Block 206 "635848718" "$fatal;" -Block 226 "1739418492" "if (1)" -Block 227 "3242275586" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 229 "1739418492" "if (1)" -Block 230 "477441569" "$fatal;" -Block 250 "3594899581" "if (1)" -Block 251 "4186328242" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 253 "3594899581" "if (1)" -Block 254 "615666065" "$fatal;" -Block 274 "560230616" "if (1)" -Block 275 "20349054" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 277 "560230616" "if (1)" -Block 278 "3691158877" "$fatal;" -Block 298 "1461373702" "if (1)" -Block 299 "4071637836" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 301 "1461373702" "if (1)" -Block 302 "797204079" "$fatal;" -Block 322 "1657104126" "if (1)" -Block 323 "3511941208" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 325 "1657104126" "if (1)" -Block 326 "208135547" "$fatal;" -Block 346 "4290315645" "if (1)" -Block 347 "3904937591" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 349 "4290315645" "if (1)" -Block 350 "905216852" "$fatal;" -Block 370 "1602064482" "if (1)" -Block 371 "2440992396" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 373 "1602064482" "if (1)" -Block 374 "1279953839" "$fatal;" -Block 394 "2824614087" "if (1)" -Block 395 "1775131200" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 397 "2824614087" "if (1)" -Block 398 "3036301155" "$fatal;" -Block 418 "1053307518" "if (1)" -Block 419 "711322415" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 421 "1053307518" "if (1)" -Block 422 "4149343756" "$fatal;" -Block 442 "186097542" "if (1)" -Block 443 "159797307" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 445 "186097542" "if (1)" -Block 446 "3568454936" "$fatal;" -Block 466 "2523659269" "if (1)" -Block 467 "806696468" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 469 "2523659269" "if (1)" -Block 470 "3978373943" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Mesh packet body without ownership" -Block 18 "1479776755" "if (1)" -Block 19 "3696211781" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 21 "1479776755" "if (1)" -Block 22 "2048665000" "$fatal;" -Block 42 "1003602289" "if (1)" -Block 43 "1541976043" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 45 "1003602289" "if (1)" -Block 46 "4256906502" "$fatal;" -Block 66 "1944574667" "if (1)" -Block 67 "1172873431" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 69 "1944574667" "if (1)" -Block 70 "3820696122" "$fatal;" -Block 90 "4087040474" "if (1)" -Block 91 "2177760972" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 93 "4087040474" "if (1)" -Block 94 "664693793" "$fatal;" -Block 114 "2364201432" "if (1)" -Block 115 "965280461" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 117 "2364201432" "if (1)" -Block 118 "2681901088" "$fatal;" -Block 138 "575965992" "if (1)" -Block 139 "662191603" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 141 "575965992" "if (1)" -Block 142 "2167123742" "$fatal;" -Block 162 "2780275615" "if (1)" -Block 163 "1919327153" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 165 "2780275615" "if (1)" -Block 166 "3560245596" "$fatal;" -Block 186 "3620965993" "if (1)" -Block 187 "829912204" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 189 "3620965993" "if (1)" -Block 190 "2535777889" "$fatal;" -Block 210 "1583181966" "if (1)" -Block 211 "1087336062" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 213 "1583181966" "if (1)" -Block 214 "3869047955" "$fatal;" -Block 234 "1962668773" "if (1)" -Block 235 "588553913" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 237 "1962668773" "if (1)" -Block 238 "2236025940" "$fatal;" -Block 258 "1046265680" "if (1)" -Block 259 "3929352611" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 261 "1046265680" "if (1)" -Block 262 "1281801038" "$fatal;" -Block 282 "1608998260" "if (1)" -Block 283 "764998666" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 285 "1608998260" "if (1)" -Block 286 "2345363175" "$fatal;" -Block 306 "574513879" "if (1)" -Block 307 "1302867378" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 309 "574513879" "if (1)" -Block 310 "3959144287" "$fatal;" -Block 330 "1689751111" "if (1)" -Block 331 "1315827385" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 333 "1689751111" "if (1)" -Block 334 "3896409172" "$fatal;" -Block 354 "2345279621" "if (1)" -Block 355 "1865992791" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 357 "2345279621" "if (1)" -Block 358 "3379255482" "$fatal;" -Block 378 "3602701937" "if (1)" -Block 379 "664587898" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 381 "3602701937" "if (1)" -Block 382 "2177850519" "$fatal;" -Block 402 "2372097411" "if (1)" -Block 403 "739169264" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 405 "2372097411" "if (1)" -Block 406 "2321367325" "$fatal;" -Block 426 "2353113249" "if (1)" -Block 427 "2459213730" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 429 "2353113249" "if (1)" -Block 430 "885460303" "$fatal;" -Block 450 "1647330288" "if (1)" -Block 451 "289923572" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 453 "1647330288" "if (1)" -Block 454 "3071570713" "$fatal;" -Block 474 "3712945988" "if (1)" -Block 475 "2331052331" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 477 "3712945988" "if (1)" -Block 478 "748854214" "$fatal;" - -CHECKSUM: "1802735833 3870451462" -INSTANCE: router_tb.dut -ANNOTATION: "Input READY implies selected input VALID: READY requires output VALID and this input selected, and output VALID is selected input VALID" -Condition 1 "3092155993" "(io_in_0_0_ready_0 & io_in_0_0_valid) 1 -1" (2 "10") -Condition 3 "3850033337" "(io_in_1_0_ready_0 & io_in_1_0_valid) 1 -1" (2 "10") -Condition 5 "1752407670" "(io_in_2_0_ready_0 & io_in_2_0_valid) 1 -1" (2 "10") -Condition 7 "1756263612" "(io_in_3_0_ready_0 & io_in_3_0_valid) 1 -1" (2 "10") -Condition 9 "3996989854" "(io_in_4_0_ready_0 & io_in_4_0_valid) 1 -1" (2 "10") -Condition 11 "2231740155" "(io_in_0_1_ready_0 & io_in_0_1_valid) 1 -1" (2 "10") -Condition 13 "3627151387" "(io_in_1_1_ready_0 & io_in_1_1_valid) 1 -1" (2 "10") -Condition 15 "1429780180" "(io_in_2_1_ready_0 & io_in_2_1_valid) 1 -1" (2 "10") -Condition 17 "4175394999" "(io_in_3_1_ready_0 & io_in_3_1_valid) 1 -1" (2 "10") -Condition 19 "2118923669" "(io_in_4_1_ready_0 & io_in_4_1_valid) 1 -1" (2 "10") -Condition 21 "3932843002" "(io_in_0_2_ready_0 & io_in_0_2_valid) 1 -1" (2 "10") -Condition 23 "3076406554" "(io_in_1_2_ready_0 & io_in_1_2_valid) 1 -1" (2 "10") -Condition 25 "978811861" "(io_in_2_2_ready_0 & io_in_2_2_valid) 1 -1" (2 "10") -Condition 27 "1711358307" "(io_in_3_2_ready_0 & io_in_3_2_valid) 1 -1" (2 "10") -Condition 29 "3767667777" "(io_in_4_2_ready_0 & io_in_4_2_valid) 1 -1" (2 "10") -Condition 31 "2343211812" "(io_in_0_3_ready_0 & io_in_0_3_valid) 1 -1" (2 "10") -Condition 33 "3600729540" "(io_in_1_3_ready_0 & io_in_1_3_valid) 1 -1" (2 "10") -Condition 35 "1536656139" "(io_in_2_3_ready_0 & io_in_2_3_valid) 1 -1" (2 "10") -Condition 37 "4134554984" "(io_in_3_3_ready_0 & io_in_3_3_valid) 1 -1" (2 "10") -Condition 39 "1893924938" "(io_in_4_3_ready_0 & io_in_4_3_valid) 1 -1" (2 "10") -ANNOTATION: "Accepted flit must match the explicit VC port; this vector is the VC-mismatch assertion failure" -Condition 41 "4070260215" "(_GEN_76 & ((|io_in_0_0_bits_vc))) 1 -1" (3 "11") -Condition 60 "3671467409" "(_GEN_78 & ((|io_in_1_0_bits_vc))) 1 -1" (3 "11") -Condition 79 "3127070248" "(_GEN_80 & ((|io_in_2_0_bits_vc))) 1 -1" (3 "11") -Condition 98 "4085502683" "(_GEN_82 & ((|io_in_3_0_bits_vc))) 1 -1" (3 "11") -Condition 117 "2704255070" "(_GEN_84 & ((|io_in_4_0_bits_vc))) 1 -1" (3 "11") -Condition 136 "4209678335" "(_GEN_86 & (io_in_0_1_bits_vc != 2'b1)) 1 -1" (3 "11") -Condition 156 "3794511059" "(_GEN_88 & (io_in_1_1_bits_vc != 2'b1)) 1 -1" (3 "11") -Condition 176 "2865133531" "(_GEN_90 & (io_in_2_1_bits_vc != 2'b1)) 1 -1" (3 "11") -Condition 196 "725272976" "(_GEN_92 & (io_in_3_1_bits_vc != 2'b1)) 1 -1" (3 "11") -Condition 216 "2694844148" "(_GEN_94 & (io_in_4_1_bits_vc != 2'b1)) 1 -1" (3 "11") -Condition 236 "4160240023" "(_GEN_96 & (io_in_0_2_bits_vc != 2'h2)) 1 -1" (3 "11") -Condition 256 "4013492923" "(_GEN_98 & (io_in_1_2_bits_vc != 2'h2)) 1 -1" (3 "11") -Condition 276 "1238580357" "(_GEN_100 & (io_in_2_2_bits_vc != 2'h2)) 1 -1" (3 "11") -Condition 296 "3358556878" "(_GEN_102 & (io_in_3_2_bits_vc != 2'h2)) 1 -1" (3 "11") -Condition 316 "1135953322" "(_GEN_104 & (io_in_4_2_bits_vc != 2'h2)) 1 -1" (3 "11") -Condition 336 "2633490133" "(_GEN_106 & (io_in_0_3_bits_vc != 2'h3)) 1 -1" (3 "11") -Condition 356 "2218192377" "(_GEN_108 & (io_in_1_3_bits_vc != 2'h3)) 1 -1" (3 "11") -Condition 376 "3436920561" "(_GEN_110 & (io_in_2_3_bits_vc != 2'h3)) 1 -1" (3 "11") -Condition 396 "1294434490" "(_GEN_112 & (io_in_3_3_bits_vc != 2'h3)) 1 -1" (3 "11") -Condition 416 "3334293470" "(_GEN_114 & (io_in_4_3_bits_vc != 2'h3)) 1 -1" (3 "11") -ANNOTATION: "Accepted destination must be inside the declared 3x3 topology; this vector is the range-assertion failure" -Condition 42 "1924237034" "(_GEN_76 & ( ~ ((io_in_0_0_bits_dstX != 2'h3) & (io_in_0_0_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -Condition 61 "1403403801" "(_GEN_78 & ( ~ ((io_in_1_0_bits_dstX != 2'h3) & (io_in_1_0_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -Condition 80 "254368599" "(_GEN_80 & ( ~ ((io_in_2_0_bits_dstX != 2'h3) & (io_in_2_0_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -Condition 99 "1476507923" "(_GEN_82 & ( ~ ((io_in_3_0_bits_dstX != 2'h3) & (io_in_3_0_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -Condition 118 "2495079852" "(_GEN_84 & ( ~ ((io_in_4_0_bits_dstX != 2'h3) & (io_in_4_0_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -Condition 138 "26791419" "(_GEN_86 & ( ~ ((io_in_0_1_bits_dstX != 2'h3) & (io_in_0_1_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -Condition 158 "546272520" "(_GEN_88 & ( ~ ((io_in_1_1_bits_dstX != 2'h3) & (io_in_1_1_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -Condition 178 "2228200946" "(_GEN_90 & ( ~ ((io_in_2_1_bits_dstX != 2'h3) & (io_in_2_1_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -Condition 198 "3555149750" "(_GEN_92 & ( ~ ((io_in_3_1_bits_dstX != 2'h3) & (io_in_3_1_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -Condition 218 "525409033" "(_GEN_94 & ( ~ ((io_in_4_1_bits_dstX != 2'h3) & (io_in_4_1_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -Condition 238 "3572317890" "(_GEN_96 & ( ~ ((io_in_0_2_bits_dstX != 2'h3) & (io_in_0_2_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -Condition 258 "4126836273" "(_GEN_98 & ( ~ ((io_in_1_2_bits_dstX != 2'h3) & (io_in_1_2_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -Condition 278 "3777107480" "(_GEN_100 & ( ~ ((io_in_2_2_bits_dstX != 2'h3) & (io_in_2_2_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -Condition 298 "3054171228" "(_GEN_102 & ( ~ ((io_in_3_2_bits_dstX != 2'h3) & (io_in_3_2_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -Condition 318 "2059184355" "(_GEN_104 & ( ~ ((io_in_4_2_bits_dstX != 2'h3) & (io_in_4_2_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -Condition 338 "4019421364" "(_GEN_106 & ( ~ ((io_in_0_3_bits_dstX != 2'h3) & (io_in_0_3_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -Condition 358 "3464771655" "(_GEN_108 & ( ~ ((io_in_1_3_bits_dstX != 2'h3) & (io_in_1_3_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -Condition 378 "1791289533" "(_GEN_110 & ( ~ ((io_in_2_3_bits_dstX != 2'h3) & (io_in_2_3_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -Condition 398 "1038879481" "(_GEN_112 & ( ~ ((io_in_3_3_bits_dstX != 2'h3) & (io_in_3_3_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -Condition 418 "4049229382" "(_GEN_114 & ( ~ ((io_in_4_3_bits_dstX != 2'h3) & (io_in_4_3_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -ANNOTATION: "Accepted packet body must retain output/VC ownership; this vector is the body-without-ownership assertion failure" -Condition 47 "3947639187" "(_GEN_75 & ((~io_in_0_0_bits_head)) & ((~reset)) & ( ~ ((_owned_T & owners_0_3 & (owners_0_4 == 3'b0)) | (_owned_T_4 & owners_1_3 & (owners_1_4 == 3'b0)) | (_owned_T_8 & owners_2_3 & (owners_2_4 == 3'b0)) | (_owned_T_12 & owners_3_3 & (owners_3_4 == 3'b0)) | (_owned_T_16 & owners_4_3 & (owners_4_4 == 3'b0))) )) 1 -1" (5 "1111") -Condition 66 "1972422343" "(_GEN_77 & ((~io_in_1_0_bits_head)) & ((~reset)) & ( ~ ((_owned_T_23 & owners_0_3 & (owners_0_4 == 3'b1)) | (_owned_T_27 & owners_1_3 & (owners_1_4 == 3'b1)) | (_owned_T_31 & owners_2_3 & (owners_2_4 == 3'b1)) | (_owned_T_35 & owners_3_3 & (owners_3_4 == 3'b1)) | (_owned_T_39 & owners_4_3 & (owners_4_4 == 3'b1))) )) 1 -1" (5 "1111") -Condition 85 "3114650597" "(_GEN_79 & ((~io_in_2_0_bits_head)) & ((~reset)) & ( ~ ((_owned_T_46 & owners_0_3 & (owners_0_4 == 3'h2)) | (_owned_T_50 & owners_1_3 & (owners_1_4 == 3'h2)) | (_owned_T_54 & owners_2_3 & (owners_2_4 == 3'h2)) | (_owned_T_58 & owners_3_3 & (owners_3_4 == 3'h2)) | (_owned_T_62 & owners_4_3 & (owners_4_4 == 3'h2))) )) 1 -1" (5 "1111") -Condition 104 "3746486545" "(_GEN_81 & ((~io_in_3_0_bits_head)) & ((~reset)) & ( ~ ((_owned_T_69 & owners_0_3 & (owners_0_4 == 3'h3)) | (_owned_T_73 & owners_1_3 & (owners_1_4 == 3'h3)) | (_owned_T_77 & owners_2_3 & (owners_2_4 == 3'h3)) | (_owned_T_81 & owners_3_3 & (owners_3_4 == 3'h3)) | (_owned_T_85 & owners_4_3 & (owners_4_4 == 3'h3))) )) 1 -1" (5 "1111") -Condition 123 "1611394691" "(_GEN_83 & ((~io_in_4_0_bits_head)) & ((~reset)) & ( ~ ((_owned_T_92 & owners_0_3 & (owners_0_4 == 3'h4)) | (_owned_T_96 & owners_1_3 & (owners_1_4 == 3'h4)) | (_owned_T_100 & owners_2_3 & (owners_2_4 == 3'h4)) | (_owned_T_104 & owners_3_3 & (owners_3_4 == 3'h4)) | (_owned_T_108 & owners_4_3 & (owners_4_4 == 3'h4))) )) 1 -1" (5 "1111") -Condition 143 "1457370676" "(_GEN_85 & ((~io_in_0_1_bits_head)) & ((~reset)) & ( ~ ((_owned_T_115 & owners_0_3_1 & (owners_0_4_1 == 3'b0)) | (_owned_T_119 & owners_1_3_1 & (owners_1_4_1 == 3'b0)) | (_owned_T_123 & owners_2_3_1 & (owners_2_4_1 == 3'b0)) | (_owned_T_127 & owners_3_3_1 & (owners_3_4_1 == 3'b0)) | (_owned_T_131 & owners_4_3_1 & (owners_4_4_1 == 3'b0))) )) 1 -1" (5 "1111") -Condition 163 "4245520960" "(_GEN_87 & ((~io_in_1_1_bits_head)) & ((~reset)) & ( ~ ((_owned_T_138 & owners_0_3_1 & (owners_0_4_1 == 3'b1)) | (_owned_T_142 & owners_1_3_1 & (owners_1_4_1 == 3'b1)) | (_owned_T_146 & owners_2_3_1 & (owners_2_4_1 == 3'b1)) | (_owned_T_150 & owners_3_3_1 & (owners_3_4_1 == 3'b1)) | (_owned_T_154 & owners_4_3_1 & (owners_4_4_1 == 3'b1))) )) 1 -1" (5 "1111") -Condition 183 "486659993" "(_GEN_89 & ((~io_in_2_1_bits_head)) & ((~reset)) & ( ~ ((_owned_T_161 & owners_0_3_1 & (owners_0_4_1 == 3'h2)) | (_owned_T_165 & owners_1_3_1 & (owners_1_4_1 == 3'h2)) | (_owned_T_169 & owners_2_3_1 & (owners_2_4_1 == 3'h2)) | (_owned_T_173 & owners_3_3_1 & (owners_3_4_1 == 3'h2)) | (_owned_T_177 & owners_4_3_1 & (owners_4_4_1 == 3'h2))) )) 1 -1" (5 "1111") -Condition 203 "1083250543" "(_GEN_91 & ((~io_in_3_1_bits_head)) & ((~reset)) & ( ~ ((_owned_T_184 & owners_0_3_1 & (owners_0_4_1 == 3'h3)) | (_owned_T_188 & owners_1_3_1 & (owners_1_4_1 == 3'h3)) | (_owned_T_192 & owners_2_3_1 & (owners_2_4_1 == 3'h3)) | (_owned_T_196 & owners_3_3_1 & (owners_3_4_1 == 3'h3)) | (_owned_T_200 & owners_4_3_1 & (owners_4_4_1 == 3'h3))) )) 1 -1" (5 "1111") -Condition 223 "355838805" "(_GEN_93 & ((~io_in_4_1_bits_head)) & ((~reset)) & ( ~ ((_owned_T_207 & owners_0_3_1 & (owners_0_4_1 == 3'h4)) | (_owned_T_211 & owners_1_3_1 & (owners_1_4_1 == 3'h4)) | (_owned_T_215 & owners_2_3_1 & (owners_2_4_1 == 3'h4)) | (_owned_T_219 & owners_3_3_1 & (owners_3_4_1 == 3'h4)) | (_owned_T_223 & owners_4_3_1 & (owners_4_4_1 == 3'h4))) )) 1 -1" (5 "1111") -Condition 243 "3816971184" "(_GEN_95 & ((~io_in_0_2_bits_head)) & ((~reset)) & ( ~ ((_owned_T_230 & owners_0_3_2 & (owners_0_4_2 == 3'b0)) | (_owned_T_234 & owners_1_3_2 & (owners_1_4_2 == 3'b0)) | (_owned_T_238 & owners_2_3_2 & (owners_2_4_2 == 3'b0)) | (_owned_T_242 & owners_3_3_2 & (owners_3_4_2 == 3'b0)) | (_owned_T_246 & owners_4_3_2 & (owners_4_4_2 == 3'b0))) )) 1 -1" (5 "1111") -Condition 263 "226870005" "(_GEN_97 & ((~io_in_1_2_bits_head)) & ((~reset)) & ( ~ ((_owned_T_253 & owners_0_3_2 & (owners_0_4_2 == 3'b1)) | (_owned_T_257 & owners_1_3_2 & (owners_1_4_2 == 3'b1)) | (_owned_T_261 & owners_2_3_2 & (owners_2_4_2 == 3'b1)) | (_owned_T_265 & owners_3_3_2 & (owners_3_4_2 == 3'b1)) | (_owned_T_269 & owners_4_3_2 & (owners_4_4_2 == 3'b1))) )) 1 -1" (5 "1111") -Condition 283 "2142325087" "(_GEN_99 & ((~io_in_2_2_bits_head)) & ((~reset)) & ( ~ ((_owned_T_276 & owners_0_3_2 & (owners_0_4_2 == 3'h2)) | (_owned_T_280 & owners_1_3_2 & (owners_1_4_2 == 3'h2)) | (_owned_T_284 & owners_2_3_2 & (owners_2_4_2 == 3'h2)) | (_owned_T_288 & owners_3_3_2 & (owners_3_4_2 == 3'h2)) | (_owned_T_292 & owners_4_3_2 & (owners_4_4_2 == 3'h2))) )) 1 -1" (5 "1111") -Condition 303 "168759350" "(_GEN_101 & ((~io_in_3_2_bits_head)) & ((~reset)) & ( ~ ((_owned_T_299 & owners_0_3_2 & (owners_0_4_2 == 3'h3)) | (_owned_T_303 & owners_1_3_2 & (owners_1_4_2 == 3'h3)) | (_owned_T_307 & owners_2_3_2 & (owners_2_4_2 == 3'h3)) | (_owned_T_311 & owners_3_3_2 & (owners_3_4_2 == 3'h3)) | (_owned_T_315 & owners_4_3_2 & (owners_4_4_2 == 3'h3))) )) 1 -1" (5 "1111") -Condition 323 "1234006216" "(_GEN_103 & ((~io_in_4_2_bits_head)) & ((~reset)) & ( ~ ((_owned_T_322 & owners_0_3_2 & (owners_0_4_2 == 3'h4)) | (_owned_T_326 & owners_1_3_2 & (owners_1_4_2 == 3'h4)) | (_owned_T_330 & owners_2_3_2 & (owners_2_4_2 == 3'h4)) | (_owned_T_334 & owners_3_3_2 & (owners_3_4_2 == 3'h4)) | (_owned_T_338 & owners_4_3_2 & (owners_4_4_2 == 3'h4))) )) 1 -1" (5 "1111") -Condition 343 "3686430807" "(_GEN_105 & ((~io_in_0_3_bits_head)) & ((~reset)) & ( ~ ((_owned_T_345 & owners_0_3_3 & (owners_0_4_3 == 3'b0)) | (_owned_T_349 & owners_1_3_3 & (owners_1_4_3 == 3'b0)) | (_owned_T_353 & owners_2_3_3 & (owners_2_4_3 == 3'b0)) | (_owned_T_357 & owners_3_3_3 & (owners_3_4_3 == 3'b0)) | (_owned_T_361 & owners_4_3_3 & (owners_4_4_3 == 3'b0))) )) 1 -1" (5 "1111") -Condition 363 "3447495620" "(_GEN_107 & ((~io_in_1_3_bits_head)) & ((~reset)) & ( ~ ((_owned_T_368 & owners_0_3_3 & (owners_0_4_3 == 3'b1)) | (_owned_T_372 & owners_1_3_3 & (owners_1_4_3 == 3'b1)) | (_owned_T_376 & owners_2_3_3 & (owners_2_4_3 == 3'b1)) | (_owned_T_380 & owners_3_3_3 & (owners_3_4_3 == 3'b1)) | (_owned_T_384 & owners_4_3_3 & (owners_4_4_3 == 3'b1))) )) 1 -1" (5 "1111") -Condition 383 "2754832380" "(_GEN_109 & ((~io_in_2_3_bits_head)) & ((~reset)) & ( ~ ((_owned_T_391 & owners_0_3_3 & (owners_0_4_3 == 3'h2)) | (_owned_T_395 & owners_1_3_3 & (owners_1_4_3 == 3'h2)) | (_owned_T_399 & owners_2_3_3 & (owners_2_4_3 == 3'h2)) | (_owned_T_403 & owners_3_3_3 & (owners_3_4_3 == 3'h2)) | (_owned_T_407 & owners_4_3_3 & (owners_4_4_3 == 3'h2))) )) 1 -1" (5 "1111") -Condition 403 "2964337691" "(_GEN_111 & ((~io_in_3_3_bits_head)) & ((~reset)) & ( ~ ((_owned_T_414 & owners_0_3_3 & (owners_0_4_3 == 3'h3)) | (_owned_T_418 & owners_1_3_3 & (owners_1_4_3 == 3'h3)) | (_owned_T_422 & owners_2_3_3 & (owners_2_4_3 == 3'h3)) | (_owned_T_426 & owners_3_3_3 & (owners_3_4_3 == 3'h3)) | (_owned_T_430 & owners_4_3_3 & (owners_4_4_3 == 3'h3))) )) 1 -1" (5 "1111") -Condition 423 "531256894" "(_GEN_113 & ((~io_in_4_3_bits_head)) & ((~reset)) & ( ~ ((_owned_T_437 & owners_0_3_3 & (owners_0_4_3 == 3'h4)) | (_owned_T_441 & owners_1_3_3 & (owners_1_4_3 == 3'h4)) | (_owned_T_445 & owners_2_3_3 & (owners_2_4_3 == 3'h4)) | (_owned_T_449 & owners_3_3_3 & (owners_3_4_3 == 3'h4)) | (_owned_T_453 & owners_4_3_3 & (owners_4_4_3 == 3'h4))) )) 1 -1" (5 "1111") -ANNOTATION: "Tail/capture subexpression is evaluated only in output-FIRE branch; FIRE needs READY while capture needs !READY, so capture cannot be asserted in this branch" -Condition 518 "2956550500" "(_GEN_8[owners_0_2] & ((~_owners_T_3))) 1 -1" (2 "10") -Condition 522 "708526675" "(_GEN_8[owners_1_2] & ((~_owners_T_8))) 1 -1" (2 "10") -Condition 526 "1710407644" "(_GEN_8[owners_2_2] & ((~_owners_T_13))) 1 -1" (2 "10") -Condition 530 "855132190" "(_GEN_8[owners_3_2] & ((~_owners_T_18))) 1 -1" (2 "10") -Condition 534 "1570867619" "(_GEN_8[owners_4_2] & ((~_owners_T_23))) 1 -1" (2 "10") -Condition 538 "3016253282" "(_GEN_27[owners_0_2_1] & ((~_owners_T_28))) 1 -1" (2 "10") -Condition 542 "2555122219" "(_GEN_27[owners_1_2_1] & ((~_owners_T_33))) 1 -1" (2 "10") -Condition 546 "2458396873" "(_GEN_27[owners_2_2_1] & ((~_owners_T_38))) 1 -1" (2 "10") -Condition 550 "3558935641" "(_GEN_27[owners_3_2_1] & ((~_owners_T_43))) 1 -1" (2 "10") -Condition 554 "64083681" "(_GEN_27[owners_4_2_1] & ((~_owners_T_48))) 1 -1" (2 "10") -Condition 558 "1343425993" "(_GEN_46[owners_0_2_2] & ((~_owners_T_53))) 1 -1" (2 "10") -Condition 562 "721917084" "(_GEN_46[owners_1_2_2] & ((~_owners_T_58))) 1 -1" (2 "10") -Condition 566 "1790583670" "(_GEN_46[owners_2_2_2] & ((~_owners_T_63))) 1 -1" (2 "10") -Condition 570 "4116221512" "(_GEN_46[owners_3_2_2] & ((~_owners_T_68))) 1 -1" (2 "10") -Condition 574 "3731495116" "(_GEN_46[owners_4_2_2] & ((~_owners_T_73))) 1 -1" (2 "10") -Condition 578 "834226852" "(_GEN_65[owners_0_2_3] & ((~_owners_T_78))) 1 -1" (2 "10") -Condition 582 "3439348919" "(_GEN_65[owners_1_2_3] & ((~_owners_T_83))) 1 -1" (2 "10") -Condition 586 "3351541333" "(_GEN_65[owners_2_2_3] & ((~_owners_T_88))) 1 -1" (2 "10") -Condition 590 "2733104725" "(_GEN_65[owners_3_2_3] & ((~_owners_T_93))) 1 -1" (2 "10") -Condition 594 "632686821" "(_GEN_65[owners_4_2_3] & ((~_owners_T_98))) 1 -1" (2 "10") -ANNOTATION: "Capture is VALID & !READY & !lockValid; capture and an existing lock cannot both be true, so !capture=0 / lockValid=1 is unreachable" -Condition 519 "4125160278" "(((~_owners_T_3)) | owners_0_3) 1 -1" (2 "01") -Condition 523 "3698058731" "(((~_owners_T_8)) | owners_1_3) 1 -1" (2 "01") -Condition 527 "2771202367" "(((~_owners_T_13)) | owners_2_3) 1 -1" (2 "01") -Condition 531 "229547046" "(((~_owners_T_18)) | owners_3_3) 1 -1" (2 "01") -Condition 535 "2665684153" "(((~_owners_T_23)) | owners_4_3) 1 -1" (2 "01") -Condition 539 "2806063833" "(((~_owners_T_28)) | owners_0_3_1) 1 -1" (2 "01") -Condition 543 "3170010768" "(((~_owners_T_33)) | owners_1_3_1) 1 -1" (2 "01") -Condition 547 "3394554053" "(((~_owners_T_38)) | owners_2_3_1) 1 -1" (2 "01") -Condition 551 "950194497" "(((~_owners_T_43)) | owners_3_3_1) 1 -1" (2 "01") -Condition 555 "2952743903" "(((~_owners_T_48)) | owners_4_3_1) 1 -1" (2 "01") -Condition 559 "1705755745" "(((~_owners_T_53)) | owners_0_3_2) 1 -1" (2 "01") -Condition 563 "3367075034" "(((~_owners_T_58)) | owners_1_3_2) 1 -1" (2 "01") -Condition 567 "2469408776" "(((~_owners_T_63)) | owners_2_3_2) 1 -1" (2 "01") -Condition 571 "3671899352" "(((~_owners_T_68)) | owners_3_3_2) 1 -1" (2 "01") -Condition 575 "1467654804" "(((~_owners_T_73)) | owners_4_3_2) 1 -1" (2 "01") -Condition 579 "1545107880" "(((~_owners_T_78)) | owners_0_3_3) 1 -1" (2 "01") -Condition 583 "2118368442" "(((~_owners_T_83)) | owners_1_3_3) 1 -1" (2 "01") -Condition 587 "149287663" "(((~_owners_T_88)) | owners_2_3_3) 1 -1" (2 "01") -Condition 591 "1576121327" "(((~_owners_T_93)) | owners_3_3_3) 1 -1" (2 "01") -Condition 595 "2584552825" "(((~_owners_T_98)) | owners_4_3_3) 1 -1" (2 "01") - -CHECKSUM: "1802735833 384767911" -INSTANCE: router_tb.dut -ANNOTATION: "Literal zero prepended to the 3-bit round-robin cursor; only padded bit 3 is excluded" -Toggle _GEN [3] "net _GEN[3:0]" -Toggle _GEN_10 [3] "net _GEN_10[3:0]" -Toggle _GEN_12 [3] "net _GEN_12[3:0]" -Toggle _GEN_14 [3] "net _GEN_14[3:0]" -Toggle _GEN_16 [3] "net _GEN_16[3:0]" -Toggle _GEN_18 [3] "net _GEN_18[3:0]" -Toggle _GEN_29 [3] "net _GEN_29[3:0]" -Toggle _GEN_31 [3] "net _GEN_31[3:0]" -Toggle _GEN_33 [3] "net _GEN_33[3:0]" -Toggle _GEN_35 [3] "net _GEN_35[3:0]" -Toggle _GEN_37 [3] "net _GEN_37[3:0]" -Toggle _GEN_48 [3] "net _GEN_48[3:0]" -Toggle _GEN_50 [3] "net _GEN_50[3:0]" -Toggle _GEN_52 [3] "net _GEN_52[3:0]" -Toggle _GEN_54 [3] "net _GEN_54[3:0]" -Toggle _GEN_56 [3] "net _GEN_56[3:0]" -Toggle _GEN_67 [3] "net _GEN_67[3:0]" -Toggle _GEN_69 [3] "net _GEN_69[3:0]" -Toggle _GEN_71 [3] "net _GEN_71[3:0]" -Toggle _GEN_73 [3] "net _GEN_73[3:0]" -ANNOTATION: "Result is explicitly modulo the 5 input directions; only values 0..4 are possible, so only bit 3 is constant zero" -Toggle _owners_ordered_T_1 [3] "net _owners_ordered_T_1[3:0]" -Toggle _owners_ordered_T_3 [3] "net _owners_ordered_T_3[3:0]" -Toggle _owners_ordered_T_5 [3] "net _owners_ordered_T_5[3:0]" -Toggle _owners_ordered_T_7 [3] "net _owners_ordered_T_7[3:0]" -Toggle owners_fairChoice [3] "net owners_fairChoice[3:0]" -Toggle _owners_ordered_T_11 [3] "net _owners_ordered_T_11[3:0]" -Toggle _owners_ordered_T_13 [3] "net _owners_ordered_T_13[3:0]" -Toggle _owners_ordered_T_15 [3] "net _owners_ordered_T_15[3:0]" -Toggle _owners_ordered_T_17 [3] "net _owners_ordered_T_17[3:0]" -Toggle owners_fairChoice_1 [3] "net owners_fairChoice_1[3:0]" -Toggle _owners_ordered_T_21 [3] "net _owners_ordered_T_21[3:0]" -Toggle _owners_ordered_T_23 [3] "net _owners_ordered_T_23[3:0]" -Toggle _owners_ordered_T_25 [3] "net _owners_ordered_T_25[3:0]" -Toggle _owners_ordered_T_27 [3] "net _owners_ordered_T_27[3:0]" -Toggle owners_fairChoice_2 [3] "net owners_fairChoice_2[3:0]" -Toggle _owners_ordered_T_31 [3] "net _owners_ordered_T_31[3:0]" -Toggle _owners_ordered_T_33 [3] "net _owners_ordered_T_33[3:0]" -Toggle _owners_ordered_T_35 [3] "net _owners_ordered_T_35[3:0]" -Toggle _owners_ordered_T_37 [3] "net _owners_ordered_T_37[3:0]" -Toggle owners_fairChoice_3 [3] "net owners_fairChoice_3[3:0]" -Toggle _owners_ordered_T_41 [3] "net _owners_ordered_T_41[3:0]" -Toggle _owners_ordered_T_43 [3] "net _owners_ordered_T_43[3:0]" -Toggle _owners_ordered_T_45 [3] "net _owners_ordered_T_45[3:0]" -Toggle _owners_ordered_T_47 [3] "net _owners_ordered_T_47[3:0]" -Toggle owners_fairChoice_4 [3] "net owners_fairChoice_4[3:0]" -Toggle _owners_ordered_T_51 [3] "net _owners_ordered_T_51[3:0]" -Toggle _owners_ordered_T_53 [3] "net _owners_ordered_T_53[3:0]" -Toggle _owners_ordered_T_55 [3] "net _owners_ordered_T_55[3:0]" -Toggle _owners_ordered_T_57 [3] "net _owners_ordered_T_57[3:0]" -Toggle owners_fairChoice_5 [3] "net owners_fairChoice_5[3:0]" -Toggle _owners_ordered_T_61 [3] "net _owners_ordered_T_61[3:0]" -Toggle _owners_ordered_T_63 [3] "net _owners_ordered_T_63[3:0]" -Toggle _owners_ordered_T_65 [3] "net _owners_ordered_T_65[3:0]" -Toggle _owners_ordered_T_67 [3] "net _owners_ordered_T_67[3:0]" -Toggle owners_fairChoice_6 [3] "net owners_fairChoice_6[3:0]" -Toggle _owners_ordered_T_71 [3] "net _owners_ordered_T_71[3:0]" -Toggle _owners_ordered_T_73 [3] "net _owners_ordered_T_73[3:0]" -Toggle _owners_ordered_T_75 [3] "net _owners_ordered_T_75[3:0]" -Toggle _owners_ordered_T_77 [3] "net _owners_ordered_T_77[3:0]" -Toggle owners_fairChoice_7 [3] "net owners_fairChoice_7[3:0]" -Toggle _owners_ordered_T_81 [3] "net _owners_ordered_T_81[3:0]" -Toggle _owners_ordered_T_83 [3] "net _owners_ordered_T_83[3:0]" -Toggle _owners_ordered_T_85 [3] "net _owners_ordered_T_85[3:0]" -Toggle _owners_ordered_T_87 [3] "net _owners_ordered_T_87[3:0]" -Toggle owners_fairChoice_8 [3] "net owners_fairChoice_8[3:0]" -Toggle _owners_ordered_T_91 [3] "net _owners_ordered_T_91[3:0]" -Toggle _owners_ordered_T_93 [3] "net _owners_ordered_T_93[3:0]" -Toggle _owners_ordered_T_95 [3] "net _owners_ordered_T_95[3:0]" -Toggle _owners_ordered_T_97 [3] "net _owners_ordered_T_97[3:0]" -Toggle owners_fairChoice_9 [3] "net owners_fairChoice_9[3:0]" -Toggle _owners_ordered_T_101 [3] "net _owners_ordered_T_101[3:0]" -Toggle _owners_ordered_T_103 [3] "net _owners_ordered_T_103[3:0]" -Toggle _owners_ordered_T_105 [3] "net _owners_ordered_T_105[3:0]" -Toggle _owners_ordered_T_107 [3] "net _owners_ordered_T_107[3:0]" -Toggle owners_fairChoice_10 [3] "net owners_fairChoice_10[3:0]" -Toggle _owners_ordered_T_111 [3] "net _owners_ordered_T_111[3:0]" -Toggle _owners_ordered_T_113 [3] "net _owners_ordered_T_113[3:0]" -Toggle _owners_ordered_T_115 [3] "net _owners_ordered_T_115[3:0]" -Toggle _owners_ordered_T_117 [3] "net _owners_ordered_T_117[3:0]" -Toggle owners_fairChoice_11 [3] "net owners_fairChoice_11[3:0]" -Toggle _owners_ordered_T_121 [3] "net _owners_ordered_T_121[3:0]" -Toggle _owners_ordered_T_123 [3] "net _owners_ordered_T_123[3:0]" -Toggle _owners_ordered_T_125 [3] "net _owners_ordered_T_125[3:0]" -Toggle _owners_ordered_T_127 [3] "net _owners_ordered_T_127[3:0]" -Toggle owners_fairChoice_12 [3] "net owners_fairChoice_12[3:0]" -Toggle _owners_ordered_T_131 [3] "net _owners_ordered_T_131[3:0]" -Toggle _owners_ordered_T_133 [3] "net _owners_ordered_T_133[3:0]" -Toggle _owners_ordered_T_135 [3] "net _owners_ordered_T_135[3:0]" -Toggle _owners_ordered_T_137 [3] "net _owners_ordered_T_137[3:0]" -Toggle owners_fairChoice_13 [3] "net owners_fairChoice_13[3:0]" -Toggle _owners_ordered_T_141 [3] "net _owners_ordered_T_141[3:0]" -Toggle _owners_ordered_T_143 [3] "net _owners_ordered_T_143[3:0]" -Toggle _owners_ordered_T_145 [3] "net _owners_ordered_T_145[3:0]" -Toggle _owners_ordered_T_147 [3] "net _owners_ordered_T_147[3:0]" -Toggle owners_fairChoice_14 [3] "net owners_fairChoice_14[3:0]" -Toggle _owners_ordered_T_151 [3] "net _owners_ordered_T_151[3:0]" -Toggle _owners_ordered_T_153 [3] "net _owners_ordered_T_153[3:0]" -Toggle _owners_ordered_T_155 [3] "net _owners_ordered_T_155[3:0]" -Toggle _owners_ordered_T_157 [3] "net _owners_ordered_T_157[3:0]" -Toggle owners_fairChoice_15 [3] "net owners_fairChoice_15[3:0]" -Toggle _owners_ordered_T_161 [3] "net _owners_ordered_T_161[3:0]" -Toggle _owners_ordered_T_163 [3] "net _owners_ordered_T_163[3:0]" -Toggle _owners_ordered_T_165 [3] "net _owners_ordered_T_165[3:0]" -Toggle _owners_ordered_T_167 [3] "net _owners_ordered_T_167[3:0]" -Toggle owners_fairChoice_16 [3] "net owners_fairChoice_16[3:0]" -Toggle _owners_ordered_T_171 [3] "net _owners_ordered_T_171[3:0]" -Toggle _owners_ordered_T_173 [3] "net _owners_ordered_T_173[3:0]" -Toggle _owners_ordered_T_175 [3] "net _owners_ordered_T_175[3:0]" -Toggle _owners_ordered_T_177 [3] "net _owners_ordered_T_177[3:0]" -Toggle owners_fairChoice_17 [3] "net owners_fairChoice_17[3:0]" -Toggle _owners_ordered_T_181 [3] "net _owners_ordered_T_181[3:0]" -Toggle _owners_ordered_T_183 [3] "net _owners_ordered_T_183[3:0]" -Toggle _owners_ordered_T_185 [3] "net _owners_ordered_T_185[3:0]" -Toggle _owners_ordered_T_187 [3] "net _owners_ordered_T_187[3:0]" -Toggle owners_fairChoice_18 [3] "net owners_fairChoice_18[3:0]" -Toggle _owners_ordered_T_191 [3] "net _owners_ordered_T_191[3:0]" -Toggle _owners_ordered_T_193 [3] "net _owners_ordered_T_193[3:0]" -Toggle _owners_ordered_T_195 [3] "net _owners_ordered_T_195[3:0]" -Toggle _owners_ordered_T_197 [3] "net _owners_ordered_T_197[3:0]" -Toggle owners_fairChoice_19 [3] "net owners_fairChoice_19[3:0]" diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/router.f b/arch/src/main/scala/framework/root/root-chip/src/main/resources/router.f deleted file mode 100644 index 5f48e24aa..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/router.f +++ /dev/null @@ -1,7 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/fabric_interface.sv -@RESOURCES@/router_package.sv --F @RTL@/MeshRouter/filelist.f -@RESOURCES@/router_tb.sv diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/router_package.sv b/arch/src/main/scala/framework/root/root-chip/src/main/resources/router_package.sv deleted file mode 100644 index 9a459f26a..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/router_package.sv +++ /dev/null @@ -1,211 +0,0 @@ -package mesh_router_pkg; - import "DPI-C" function int mesh_route( - int x, - int y, - int dst_x, - int dst_y - ); - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual mesh_fabric_if vif; - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - timeout = 300us; - if (!uvm_config_db#(virtual mesh_fabric_if)::get(this, "", "vif", vif)) - `uvm_fatal("VIF", "fabric interface missing") - endfunction - task packet(int source, int direction, int channel, bit stall_head = 0, int length = 3); - int dx, dy; - dx = direction == 4 ? 0 : direction == 3 ? 2 : 1; - dy = direction == 1 ? 0 : direction == 2 ? 2 : 1; - vif.src_x[source][channel] = (source + direction) % 3; - vif.src_y[source][channel] = (source + channel) % 3; - for (int beat = 0; beat < length; beat++) begin - if (beat == 1) begin - vif.valid[source][channel] = 0; - vif.out_ready[direction][channel] = 1; - // VALID-low packet gap may carry unrelated metadata; ownership must survive. - vif.x[source][channel] = dx == 0 ? 2 : 0; - vif.y[source][channel] = dy == 0 ? 2 : 0; - repeat (2) @(negedge vif.clock); - end - vif.valid[source][channel] = 1; - vif.x[source][channel] = dx; - vif.y[source][channel] = dy; - vif.head[source][channel] = beat == 0; - vif.tail[source][channel] = beat == length - 1; - vif.data[source][channel] = $urandom; - vif.out_ready[direction][channel] = beat == 0 && !stall_head; - if (beat != 0 || stall_head) - repeat (2) begin - @(posedge vif.clock); - if(vif.ready[source][channel]||vif.out_data[direction][channel]!==vif.data[source][channel]) - `uvm_fatal("STABLE", "packet beat unstable while blocked"); - @(negedge vif.clock); - end - vif.out_ready[direction][channel] = 1; - @(posedge vif.clock); - if (!vif.ready[source][channel] || !vif.out_valid[mesh_route( - 1, 1, dx, dy - )][channel] || vif.out_data[direction][channel] !== vif.data[source][channel] || - vif.out_src_x[direction][channel] !== vif.src_x[source][channel] || - vif.out_src_y[direction][channel] !== vif.src_y[source][channel]) - `uvm_fatal("PACKET", "route or metadata mismatch") - @(negedge vif.clock); - end - vif.valid[source][channel] = 0; - endtask - task execute(); - int counts [5][4]; - int fired; - bit accepted[5][4]; - vif.reset = 1; - vif.active = 0; - for (int i = 0; i < 5; i++) - for (int v = 0; v < 4; v++) begin - vif.valid[i][v] = 0; - vif.src_x[i][v] = (i + v) % 3; - vif.src_y[i][v] = (i + v) / 3 % 3; - vif.vc[i][v] = v; - vif.x[i][v] = 1; - vif.y[i][v] = 1; - vif.head[i][v] = 1; - vif.tail[i][v] = 1; - vif.data[i][v] = 32'h100 + i; - vif.out_ready[i][v] = 1; - end - repeat (4) @(negedge vif.clock); - // Reset cancels transactions; metadata is unconstrained during reset. - for (int i = 0; i < 5; i++) - for (int v = 0; v < 4; v++) begin - vif.valid[i][v] = 1; - vif.x[i][v] = 3; - vif.y[i][v] = 3; - vif.vc[i][v] = (v + 1) % 4; - vif.head[i][v] = 0; - end - for (int source = 0; source < 5; source++) begin - for (int i = 0; i < 5; i++) for (int v = 0; v < 4; v++) vif.valid[i][v] = i == source; - repeat (2) @(negedge vif.clock); - end - for (int i = 0; i < 5; i++) for (int v = 0; v < 4; v++) vif.valid[i][v] = 0; - vif.reset = 0; - repeat (4) @(negedge vif.clock); - for (int i = 0; i < 5; i++) - for (int v = 0; v < 4; v++) begin - vif.x[i][v] = 3; - vif.y[i][v] = 1; - end - repeat (2) @(negedge vif.clock); - for (int i = 0; i < 5; i++) - for (int v = 0; v < 4; v++) begin - vif.x[i][v] = 1; - vif.y[i][v] = 3; - end - repeat (2) @(negedge vif.clock); - for (int i = 0; i < 5; i++) - for (int v = 0; v < 4; v++) begin - vif.x[i][v] = 1; - vif.y[i][v] = 1; - vif.vc[i][v] = v; - vif.head[i][v] = 1; - end - for (int i = 0; i < 5; i++) - for (int v = 0; v < 4; v++) begin - counts[i][v] = 0; - vif.valid[i][v] = 1; - end - for (int tick = 0; tick < 100; tick++) begin - @(posedge vif.clock); - fired = 0; - for (int v = 0; v < 4; v++) begin - fired = 0; - for (int i = 0; i < 5; i++) begin - accepted[i][v] = vif.ready[i][v]; - if (vif.ready[i][v]) begin - fired++; - counts[i][v]++; - if (vif.out_data[0][v] !== vif.data[i][v]) - `uvm_fatal("DATA", "selected source mismatch"); - end - end - if (fired != 1) - `uvm_fatal("CONTENTION", $sformatf("%0d inputs retired for one output", fired)); - end - @(negedge vif.clock); - for (int i = 0; i < 5; i++) - for (int v = 0; v < 4; v++) - if (accepted[i][v]) begin - if (counts[i][v] == 20) vif.valid[i][v] = 0; - else begin - vif.data[i][v] = $urandom; - vif.src_x[i][v] = counts[i][v] % 3; - vif.src_y[i][v] = (counts[i][v] / 3) % 3; - end - end - end - for (int i = 0; i < 5; i++) - for (int v = 0; v < 4; v++) begin - if (counts[i][v] != 20) `uvm_fatal("FAIR", "persistent contender starved"); - vif.valid[i][v] = 0; - end - // A blocked REQ must not stop independent RSP, DAT and SNP ports. - vif.valid[4][0] = 1; - vif.data[4][0] = 32'h104; - vif.out_ready[0][0] = 0; - for (int v = 1; v < 4; v++) vif.valid[0][v] = 1; - repeat (20) begin - @(posedge vif.clock); - if (vif.ready[4][0]) `uvm_fatal("STALL", "blocked REQ retired") - for (int v = 1; v < 4; v++) - if (!vif.ready[0][v] || !vif.out_valid[0][v]) - `uvm_fatal("ISOLATION", "other CHI VC blocked by REQ") - @(negedge vif.clock); - end - for (int v = 1; v < 4; v++) vif.valid[0][v] = 0; - vif.out_ready[0][0] = 1; - @(posedge vif.clock); - if (!vif.ready[4][0] || vif.out_data[0][0] !== 32'h104) - `uvm_fatal("RESUME", "stalled request was not preserved") - @(negedge vif.clock); - vif.valid[4][0] = 0; - // Unlocked and empty output may be backpressured independently of a source. - for (int direction = 0; direction < 5; direction++) - for (int channel = 0; channel < 4; channel++) vif.out_ready[direction][channel] = 0; - repeat (2) @(negedge vif.clock); - for (int direction = 0; direction < 5; direction++) - for (int channel = 0; channel < 4; channel++) vif.out_ready[direction][channel] = 1; - // Each source seed and each offset from the round-robin cursor is exercised. - for (int channel = 0; channel < 4; channel++) - for (int direction = 0; direction < 5; direction++) - for (int seed = 0; seed < 5; seed++) - for (int offset = 0; offset < 5; offset++) begin - packet(seed, direction, channel, offset % 2); - packet((seed + 1 + offset) % 5, direction, channel); - end - for (int channel = 0; channel < 4; channel++) - for (int direction = 0; direction < 5; direction++) - for (int source = 0; source < 5; source++) packet(source, direction, channel, 0, 1); - // Input metadata is unconstrained with VALID low; no packet may retire. - for (int value = 0; value < 4; value++) begin - for (int i = 0; i < 5; i++) for (int v = 0; v < 4; v++) vif.vc[i][v] = value; - repeat (2) @(negedge vif.clock); - end - vif.reset = 1; - repeat (4) @(negedge vif.clock); - vif.reset = 0; - for (int i = 0; i < 5; i++) for (int v = 0; v < 4; v++) vif.vc[i][v] = v; - repeat (4) @(negedge vif.clock); - `uvm_info( - "CHECKS", - "400 fair contended flits, 60 VC-isolation flits, 3100 routed packet flits; all cursor offsets and source coordinates with per-beat backpressure", - UVM_LOW) - endtask - endclass -endpackage diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/router_tb.sv b/arch/src/main/scala/framework/root/root-chip/src/main/resources/router_tb.sv deleted file mode 100644 index f5627697d..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/router_tb.sv +++ /dev/null @@ -1,355 +0,0 @@ -module router_tb; - import uvm_pkg::*; - import mesh_router_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - mesh_fabric_if vif (clock); - MeshRouter dut ( - .clock(clock), - .reset(vif.reset), - .io_in_0_0_valid(vif.valid[0][0]), - .io_in_0_0_ready(vif.ready[0][0]), - .io_in_0_0_bits_dstX(vif.x[0][0]), - .io_in_0_0_bits_dstY(vif.y[0][0]), - .io_in_0_0_bits_head(vif.head[0][0]), - .io_in_0_0_bits_tail(vif.tail[0][0]), - .io_in_0_0_bits_payload(vif.data[0][0]), - .io_in_0_0_bits_srcX(vif.src_x[0][0]), - .io_in_0_0_bits_srcY(vif.src_y[0][0]), - .io_in_0_0_bits_vc(vif.vc[0][0]), - .io_out_0_0_valid(vif.out_valid[0][0]), - .io_out_0_0_ready(vif.out_ready[0][0]), - .io_out_0_0_bits_payload(vif.out_data[0][0]), - .io_out_0_0_bits_srcX(vif.out_src_x[0][0]), - .io_out_0_0_bits_srcY(vif.out_src_y[0][0]), - .io_out_0_0_bits_head(vif.out_head[0][0]), - .io_out_0_0_bits_tail(vif.out_tail[0][0]), - .io_in_0_1_valid(vif.valid[0][1]), - .io_in_0_1_ready(vif.ready[0][1]), - .io_in_0_1_bits_dstX(vif.x[0][1]), - .io_in_0_1_bits_dstY(vif.y[0][1]), - .io_in_0_1_bits_head(vif.head[0][1]), - .io_in_0_1_bits_tail(vif.tail[0][1]), - .io_in_0_1_bits_payload(vif.data[0][1]), - .io_in_0_1_bits_srcX(vif.src_x[0][1]), - .io_in_0_1_bits_srcY(vif.src_y[0][1]), - .io_in_0_1_bits_vc(vif.vc[0][1]), - .io_out_0_1_valid(vif.out_valid[0][1]), - .io_out_0_1_ready(vif.out_ready[0][1]), - .io_out_0_1_bits_payload(vif.out_data[0][1]), - .io_out_0_1_bits_srcX(vif.out_src_x[0][1]), - .io_out_0_1_bits_srcY(vif.out_src_y[0][1]), - .io_out_0_1_bits_head(vif.out_head[0][1]), - .io_out_0_1_bits_tail(vif.out_tail[0][1]), - .io_in_0_2_valid(vif.valid[0][2]), - .io_in_0_2_ready(vif.ready[0][2]), - .io_in_0_2_bits_dstX(vif.x[0][2]), - .io_in_0_2_bits_dstY(vif.y[0][2]), - .io_in_0_2_bits_head(vif.head[0][2]), - .io_in_0_2_bits_tail(vif.tail[0][2]), - .io_in_0_2_bits_payload(vif.data[0][2]), - .io_in_0_2_bits_srcX(vif.src_x[0][2]), - .io_in_0_2_bits_srcY(vif.src_y[0][2]), - .io_in_0_2_bits_vc(vif.vc[0][2]), - .io_out_0_2_valid(vif.out_valid[0][2]), - .io_out_0_2_ready(vif.out_ready[0][2]), - .io_out_0_2_bits_payload(vif.out_data[0][2]), - .io_out_0_2_bits_srcX(vif.out_src_x[0][2]), - .io_out_0_2_bits_srcY(vif.out_src_y[0][2]), - .io_out_0_2_bits_head(vif.out_head[0][2]), - .io_out_0_2_bits_tail(vif.out_tail[0][2]), - .io_in_0_3_valid(vif.valid[0][3]), - .io_in_0_3_ready(vif.ready[0][3]), - .io_in_0_3_bits_dstX(vif.x[0][3]), - .io_in_0_3_bits_dstY(vif.y[0][3]), - .io_in_0_3_bits_head(vif.head[0][3]), - .io_in_0_3_bits_tail(vif.tail[0][3]), - .io_in_0_3_bits_payload(vif.data[0][3]), - .io_in_0_3_bits_srcX(vif.src_x[0][3]), - .io_in_0_3_bits_srcY(vif.src_y[0][3]), - .io_in_0_3_bits_vc(vif.vc[0][3]), - .io_out_0_3_valid(vif.out_valid[0][3]), - .io_out_0_3_ready(vif.out_ready[0][3]), - .io_out_0_3_bits_payload(vif.out_data[0][3]), - .io_out_0_3_bits_srcX(vif.out_src_x[0][3]), - .io_out_0_3_bits_srcY(vif.out_src_y[0][3]), - .io_out_0_3_bits_head(vif.out_head[0][3]), - .io_out_0_3_bits_tail(vif.out_tail[0][3]), - .io_in_1_0_valid(vif.valid[1][0]), - .io_in_1_0_ready(vif.ready[1][0]), - .io_in_1_0_bits_dstX(vif.x[1][0]), - .io_in_1_0_bits_dstY(vif.y[1][0]), - .io_in_1_0_bits_head(vif.head[1][0]), - .io_in_1_0_bits_tail(vif.tail[1][0]), - .io_in_1_0_bits_payload(vif.data[1][0]), - .io_in_1_0_bits_srcX(vif.src_x[1][0]), - .io_in_1_0_bits_srcY(vif.src_y[1][0]), - .io_in_1_0_bits_vc(vif.vc[1][0]), - .io_out_1_0_valid(vif.out_valid[1][0]), - .io_out_1_0_ready(vif.out_ready[1][0]), - .io_out_1_0_bits_payload(vif.out_data[1][0]), - .io_out_1_0_bits_srcX(vif.out_src_x[1][0]), - .io_out_1_0_bits_srcY(vif.out_src_y[1][0]), - .io_out_1_0_bits_head(vif.out_head[1][0]), - .io_out_1_0_bits_tail(vif.out_tail[1][0]), - .io_in_1_1_valid(vif.valid[1][1]), - .io_in_1_1_ready(vif.ready[1][1]), - .io_in_1_1_bits_dstX(vif.x[1][1]), - .io_in_1_1_bits_dstY(vif.y[1][1]), - .io_in_1_1_bits_head(vif.head[1][1]), - .io_in_1_1_bits_tail(vif.tail[1][1]), - .io_in_1_1_bits_payload(vif.data[1][1]), - .io_in_1_1_bits_srcX(vif.src_x[1][1]), - .io_in_1_1_bits_srcY(vif.src_y[1][1]), - .io_in_1_1_bits_vc(vif.vc[1][1]), - .io_out_1_1_valid(vif.out_valid[1][1]), - .io_out_1_1_ready(vif.out_ready[1][1]), - .io_out_1_1_bits_payload(vif.out_data[1][1]), - .io_out_1_1_bits_srcX(vif.out_src_x[1][1]), - .io_out_1_1_bits_srcY(vif.out_src_y[1][1]), - .io_out_1_1_bits_head(vif.out_head[1][1]), - .io_out_1_1_bits_tail(vif.out_tail[1][1]), - .io_in_1_2_valid(vif.valid[1][2]), - .io_in_1_2_ready(vif.ready[1][2]), - .io_in_1_2_bits_dstX(vif.x[1][2]), - .io_in_1_2_bits_dstY(vif.y[1][2]), - .io_in_1_2_bits_head(vif.head[1][2]), - .io_in_1_2_bits_tail(vif.tail[1][2]), - .io_in_1_2_bits_payload(vif.data[1][2]), - .io_in_1_2_bits_srcX(vif.src_x[1][2]), - .io_in_1_2_bits_srcY(vif.src_y[1][2]), - .io_in_1_2_bits_vc(vif.vc[1][2]), - .io_out_1_2_valid(vif.out_valid[1][2]), - .io_out_1_2_ready(vif.out_ready[1][2]), - .io_out_1_2_bits_payload(vif.out_data[1][2]), - .io_out_1_2_bits_srcX(vif.out_src_x[1][2]), - .io_out_1_2_bits_srcY(vif.out_src_y[1][2]), - .io_out_1_2_bits_head(vif.out_head[1][2]), - .io_out_1_2_bits_tail(vif.out_tail[1][2]), - .io_in_1_3_valid(vif.valid[1][3]), - .io_in_1_3_ready(vif.ready[1][3]), - .io_in_1_3_bits_dstX(vif.x[1][3]), - .io_in_1_3_bits_dstY(vif.y[1][3]), - .io_in_1_3_bits_head(vif.head[1][3]), - .io_in_1_3_bits_tail(vif.tail[1][3]), - .io_in_1_3_bits_payload(vif.data[1][3]), - .io_in_1_3_bits_srcX(vif.src_x[1][3]), - .io_in_1_3_bits_srcY(vif.src_y[1][3]), - .io_in_1_3_bits_vc(vif.vc[1][3]), - .io_out_1_3_valid(vif.out_valid[1][3]), - .io_out_1_3_ready(vif.out_ready[1][3]), - .io_out_1_3_bits_payload(vif.out_data[1][3]), - .io_out_1_3_bits_srcX(vif.out_src_x[1][3]), - .io_out_1_3_bits_srcY(vif.out_src_y[1][3]), - .io_out_1_3_bits_head(vif.out_head[1][3]), - .io_out_1_3_bits_tail(vif.out_tail[1][3]), - .io_in_2_0_valid(vif.valid[2][0]), - .io_in_2_0_ready(vif.ready[2][0]), - .io_in_2_0_bits_dstX(vif.x[2][0]), - .io_in_2_0_bits_dstY(vif.y[2][0]), - .io_in_2_0_bits_head(vif.head[2][0]), - .io_in_2_0_bits_tail(vif.tail[2][0]), - .io_in_2_0_bits_payload(vif.data[2][0]), - .io_in_2_0_bits_srcX(vif.src_x[2][0]), - .io_in_2_0_bits_srcY(vif.src_y[2][0]), - .io_in_2_0_bits_vc(vif.vc[2][0]), - .io_out_2_0_valid(vif.out_valid[2][0]), - .io_out_2_0_ready(vif.out_ready[2][0]), - .io_out_2_0_bits_payload(vif.out_data[2][0]), - .io_out_2_0_bits_srcX(vif.out_src_x[2][0]), - .io_out_2_0_bits_srcY(vif.out_src_y[2][0]), - .io_out_2_0_bits_head(vif.out_head[2][0]), - .io_out_2_0_bits_tail(vif.out_tail[2][0]), - .io_in_2_1_valid(vif.valid[2][1]), - .io_in_2_1_ready(vif.ready[2][1]), - .io_in_2_1_bits_dstX(vif.x[2][1]), - .io_in_2_1_bits_dstY(vif.y[2][1]), - .io_in_2_1_bits_head(vif.head[2][1]), - .io_in_2_1_bits_tail(vif.tail[2][1]), - .io_in_2_1_bits_payload(vif.data[2][1]), - .io_in_2_1_bits_srcX(vif.src_x[2][1]), - .io_in_2_1_bits_srcY(vif.src_y[2][1]), - .io_in_2_1_bits_vc(vif.vc[2][1]), - .io_out_2_1_valid(vif.out_valid[2][1]), - .io_out_2_1_ready(vif.out_ready[2][1]), - .io_out_2_1_bits_payload(vif.out_data[2][1]), - .io_out_2_1_bits_srcX(vif.out_src_x[2][1]), - .io_out_2_1_bits_srcY(vif.out_src_y[2][1]), - .io_out_2_1_bits_head(vif.out_head[2][1]), - .io_out_2_1_bits_tail(vif.out_tail[2][1]), - .io_in_2_2_valid(vif.valid[2][2]), - .io_in_2_2_ready(vif.ready[2][2]), - .io_in_2_2_bits_dstX(vif.x[2][2]), - .io_in_2_2_bits_dstY(vif.y[2][2]), - .io_in_2_2_bits_head(vif.head[2][2]), - .io_in_2_2_bits_tail(vif.tail[2][2]), - .io_in_2_2_bits_payload(vif.data[2][2]), - .io_in_2_2_bits_srcX(vif.src_x[2][2]), - .io_in_2_2_bits_srcY(vif.src_y[2][2]), - .io_in_2_2_bits_vc(vif.vc[2][2]), - .io_out_2_2_valid(vif.out_valid[2][2]), - .io_out_2_2_ready(vif.out_ready[2][2]), - .io_out_2_2_bits_payload(vif.out_data[2][2]), - .io_out_2_2_bits_srcX(vif.out_src_x[2][2]), - .io_out_2_2_bits_srcY(vif.out_src_y[2][2]), - .io_out_2_2_bits_head(vif.out_head[2][2]), - .io_out_2_2_bits_tail(vif.out_tail[2][2]), - .io_in_2_3_valid(vif.valid[2][3]), - .io_in_2_3_ready(vif.ready[2][3]), - .io_in_2_3_bits_dstX(vif.x[2][3]), - .io_in_2_3_bits_dstY(vif.y[2][3]), - .io_in_2_3_bits_head(vif.head[2][3]), - .io_in_2_3_bits_tail(vif.tail[2][3]), - .io_in_2_3_bits_payload(vif.data[2][3]), - .io_in_2_3_bits_srcX(vif.src_x[2][3]), - .io_in_2_3_bits_srcY(vif.src_y[2][3]), - .io_in_2_3_bits_vc(vif.vc[2][3]), - .io_out_2_3_valid(vif.out_valid[2][3]), - .io_out_2_3_ready(vif.out_ready[2][3]), - .io_out_2_3_bits_payload(vif.out_data[2][3]), - .io_out_2_3_bits_srcX(vif.out_src_x[2][3]), - .io_out_2_3_bits_srcY(vif.out_src_y[2][3]), - .io_out_2_3_bits_head(vif.out_head[2][3]), - .io_out_2_3_bits_tail(vif.out_tail[2][3]), - .io_in_3_0_valid(vif.valid[3][0]), - .io_in_3_0_ready(vif.ready[3][0]), - .io_in_3_0_bits_dstX(vif.x[3][0]), - .io_in_3_0_bits_dstY(vif.y[3][0]), - .io_in_3_0_bits_head(vif.head[3][0]), - .io_in_3_0_bits_tail(vif.tail[3][0]), - .io_in_3_0_bits_payload(vif.data[3][0]), - .io_in_3_0_bits_srcX(vif.src_x[3][0]), - .io_in_3_0_bits_srcY(vif.src_y[3][0]), - .io_in_3_0_bits_vc(vif.vc[3][0]), - .io_out_3_0_valid(vif.out_valid[3][0]), - .io_out_3_0_ready(vif.out_ready[3][0]), - .io_out_3_0_bits_payload(vif.out_data[3][0]), - .io_out_3_0_bits_srcX(vif.out_src_x[3][0]), - .io_out_3_0_bits_srcY(vif.out_src_y[3][0]), - .io_out_3_0_bits_head(vif.out_head[3][0]), - .io_out_3_0_bits_tail(vif.out_tail[3][0]), - .io_in_3_1_valid(vif.valid[3][1]), - .io_in_3_1_ready(vif.ready[3][1]), - .io_in_3_1_bits_dstX(vif.x[3][1]), - .io_in_3_1_bits_dstY(vif.y[3][1]), - .io_in_3_1_bits_head(vif.head[3][1]), - .io_in_3_1_bits_tail(vif.tail[3][1]), - .io_in_3_1_bits_payload(vif.data[3][1]), - .io_in_3_1_bits_srcX(vif.src_x[3][1]), - .io_in_3_1_bits_srcY(vif.src_y[3][1]), - .io_in_3_1_bits_vc(vif.vc[3][1]), - .io_out_3_1_valid(vif.out_valid[3][1]), - .io_out_3_1_ready(vif.out_ready[3][1]), - .io_out_3_1_bits_payload(vif.out_data[3][1]), - .io_out_3_1_bits_srcX(vif.out_src_x[3][1]), - .io_out_3_1_bits_srcY(vif.out_src_y[3][1]), - .io_out_3_1_bits_head(vif.out_head[3][1]), - .io_out_3_1_bits_tail(vif.out_tail[3][1]), - .io_in_3_2_valid(vif.valid[3][2]), - .io_in_3_2_ready(vif.ready[3][2]), - .io_in_3_2_bits_dstX(vif.x[3][2]), - .io_in_3_2_bits_dstY(vif.y[3][2]), - .io_in_3_2_bits_head(vif.head[3][2]), - .io_in_3_2_bits_tail(vif.tail[3][2]), - .io_in_3_2_bits_payload(vif.data[3][2]), - .io_in_3_2_bits_srcX(vif.src_x[3][2]), - .io_in_3_2_bits_srcY(vif.src_y[3][2]), - .io_in_3_2_bits_vc(vif.vc[3][2]), - .io_out_3_2_valid(vif.out_valid[3][2]), - .io_out_3_2_ready(vif.out_ready[3][2]), - .io_out_3_2_bits_payload(vif.out_data[3][2]), - .io_out_3_2_bits_srcX(vif.out_src_x[3][2]), - .io_out_3_2_bits_srcY(vif.out_src_y[3][2]), - .io_out_3_2_bits_head(vif.out_head[3][2]), - .io_out_3_2_bits_tail(vif.out_tail[3][2]), - .io_in_3_3_valid(vif.valid[3][3]), - .io_in_3_3_ready(vif.ready[3][3]), - .io_in_3_3_bits_dstX(vif.x[3][3]), - .io_in_3_3_bits_dstY(vif.y[3][3]), - .io_in_3_3_bits_head(vif.head[3][3]), - .io_in_3_3_bits_tail(vif.tail[3][3]), - .io_in_3_3_bits_payload(vif.data[3][3]), - .io_in_3_3_bits_srcX(vif.src_x[3][3]), - .io_in_3_3_bits_srcY(vif.src_y[3][3]), - .io_in_3_3_bits_vc(vif.vc[3][3]), - .io_out_3_3_valid(vif.out_valid[3][3]), - .io_out_3_3_ready(vif.out_ready[3][3]), - .io_out_3_3_bits_payload(vif.out_data[3][3]), - .io_out_3_3_bits_srcX(vif.out_src_x[3][3]), - .io_out_3_3_bits_srcY(vif.out_src_y[3][3]), - .io_out_3_3_bits_head(vif.out_head[3][3]), - .io_out_3_3_bits_tail(vif.out_tail[3][3]), - .io_in_4_0_valid(vif.valid[4][0]), - .io_in_4_0_ready(vif.ready[4][0]), - .io_in_4_0_bits_dstX(vif.x[4][0]), - .io_in_4_0_bits_dstY(vif.y[4][0]), - .io_in_4_0_bits_head(vif.head[4][0]), - .io_in_4_0_bits_tail(vif.tail[4][0]), - .io_in_4_0_bits_payload(vif.data[4][0]), - .io_in_4_0_bits_srcX(vif.src_x[4][0]), - .io_in_4_0_bits_srcY(vif.src_y[4][0]), - .io_in_4_0_bits_vc(vif.vc[4][0]), - .io_out_4_0_valid(vif.out_valid[4][0]), - .io_out_4_0_ready(vif.out_ready[4][0]), - .io_out_4_0_bits_payload(vif.out_data[4][0]), - .io_out_4_0_bits_srcX(vif.out_src_x[4][0]), - .io_out_4_0_bits_srcY(vif.out_src_y[4][0]), - .io_out_4_0_bits_head(vif.out_head[4][0]), - .io_out_4_0_bits_tail(vif.out_tail[4][0]), - .io_in_4_1_valid(vif.valid[4][1]), - .io_in_4_1_ready(vif.ready[4][1]), - .io_in_4_1_bits_dstX(vif.x[4][1]), - .io_in_4_1_bits_dstY(vif.y[4][1]), - .io_in_4_1_bits_head(vif.head[4][1]), - .io_in_4_1_bits_tail(vif.tail[4][1]), - .io_in_4_1_bits_payload(vif.data[4][1]), - .io_in_4_1_bits_srcX(vif.src_x[4][1]), - .io_in_4_1_bits_srcY(vif.src_y[4][1]), - .io_in_4_1_bits_vc(vif.vc[4][1]), - .io_out_4_1_valid(vif.out_valid[4][1]), - .io_out_4_1_ready(vif.out_ready[4][1]), - .io_out_4_1_bits_payload(vif.out_data[4][1]), - .io_out_4_1_bits_srcX(vif.out_src_x[4][1]), - .io_out_4_1_bits_srcY(vif.out_src_y[4][1]), - .io_out_4_1_bits_head(vif.out_head[4][1]), - .io_out_4_1_bits_tail(vif.out_tail[4][1]), - .io_in_4_2_valid(vif.valid[4][2]), - .io_in_4_2_ready(vif.ready[4][2]), - .io_in_4_2_bits_dstX(vif.x[4][2]), - .io_in_4_2_bits_dstY(vif.y[4][2]), - .io_in_4_2_bits_head(vif.head[4][2]), - .io_in_4_2_bits_tail(vif.tail[4][2]), - .io_in_4_2_bits_payload(vif.data[4][2]), - .io_in_4_2_bits_srcX(vif.src_x[4][2]), - .io_in_4_2_bits_srcY(vif.src_y[4][2]), - .io_in_4_2_bits_vc(vif.vc[4][2]), - .io_out_4_2_valid(vif.out_valid[4][2]), - .io_out_4_2_ready(vif.out_ready[4][2]), - .io_out_4_2_bits_payload(vif.out_data[4][2]), - .io_out_4_2_bits_srcX(vif.out_src_x[4][2]), - .io_out_4_2_bits_srcY(vif.out_src_y[4][2]), - .io_out_4_2_bits_head(vif.out_head[4][2]), - .io_out_4_2_bits_tail(vif.out_tail[4][2]), - .io_in_4_3_valid(vif.valid[4][3]), - .io_in_4_3_ready(vif.ready[4][3]), - .io_in_4_3_bits_dstX(vif.x[4][3]), - .io_in_4_3_bits_dstY(vif.y[4][3]), - .io_in_4_3_bits_head(vif.head[4][3]), - .io_in_4_3_bits_tail(vif.tail[4][3]), - .io_in_4_3_bits_payload(vif.data[4][3]), - .io_in_4_3_bits_srcX(vif.src_x[4][3]), - .io_in_4_3_bits_srcY(vif.src_y[4][3]), - .io_in_4_3_bits_vc(vif.vc[4][3]), - .io_out_4_3_valid(vif.out_valid[4][3]), - .io_out_4_3_ready(vif.out_ready[4][3]), - .io_out_4_3_bits_payload(vif.out_data[4][3]), - .io_out_4_3_bits_srcX(vif.out_src_x[4][3]), - .io_out_4_3_bits_srcY(vif.out_src_y[4][3]), - .io_out_4_3_bits_head(vif.out_head[4][3]), - .io_out_4_3_bits_tail(vif.out_tail[4][3]) - ); - initial begin - uvm_config_db#(virtual mesh_fabric_if)::set(null, "uvm_test_top", "vif", vif); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/ChiMesh.scala b/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/ChiMesh.scala deleted file mode 100644 index ec2e28ddf..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/ChiMesh.scala +++ /dev/null @@ -1,475 +0,0 @@ -package hier.chip.mesh - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import memcore.bus.chi._ - -/** - * CHI uses four non-overlapping Mesh VCs. A production tile mesh reserves at - * least four more VCs for bulk DMA and future system traffic. - */ -object ChiMeshVirtualChannel { - val Request = 0 - val Response = 1 - val Data = 2 - val Snoop = 3 - val Required = 4 -} - -/** - * Static placement of CHI NodeIDs with an explicit set of populated endpoints. - */ -case class ChiMeshNodeMap( - xByNode: Seq[Int], - yByNode: Seq[Int], - mesh: MeshParams, - presentNodes: Set[Int]) { - require(xByNode.nonEmpty && xByNode.size == yByNode.size) - require(presentNodes.nonEmpty && presentNodes.forall(node => node >= 0 && node < xByNode.size)) - require(xByNode.forall(x => x >= 0 && x < mesh.xNodes)) - require(yByNode.forall(y => y >= 0 && y < mesh.yNodes)) - def coordinateBits: Int = math.max(1, log2Ceil(xByNode.size)) -} - -/** A packed CHI flit with the routing destination kept outside protocol bits. */ -class ChiMeshMessage(flitBits: Int, nodeIdBits: Int) extends Bundle { - val targetNode = UInt(nodeIdBits.W) - val flit = UInt(flitBits.W) -} - -/** - * Turns one atomic CHI flit into a fixed-length wormhole packet. - * - * CHI channel identity is carried by the mesh VC, so a packetizer instance is - * dedicated to one channel. `targetNode` is routing metadata, never serialized - * into the CHI flit. This makes request, response, data and snoop packetizers - * identical even though their target fields differ. - */ -@instantiable -class ChiMeshPacketizer( - mesh: MeshParams, - flitBits: Int, - virtualChannel: Int, - localX: Int, - localY: Int, - nodeMap: ChiMeshNodeMap) - extends Module { - require(flitBits > 0) - require(virtualChannel >= 0 && virtualChannel < mesh.virtualChannels) - require(localX >= 0 && localX < mesh.xNodes && localY >= 0 && localY < mesh.yNodes) - require(nodeMap.mesh == mesh) - - val chunks = (flitBits + mesh.payloadBits - 1) / mesh.payloadBits - val beatBits = math.max(1, log2Ceil(chunks)) - val paddedBits = chunks * mesh.payloadBits - val message = new ChiMeshMessage(flitBits, nodeMap.coordinateBits) - - @public - val io = IO(new Bundle { - val in = Flipped(Decoupled(message)) - val out = Decoupled(new MeshFlit(mesh)) - }) - - val busy = RegInit(false.B) - val beat = RegInit(0.U(beatBits.W)) - val flit = Reg(UInt(flitBits.W)) - val target = Reg(UInt(nodeMap.coordinateBits.W)) - val xLookup = VecInit(nodeMap.xByNode.map(_.U(mesh.xBits.W))) - val yLookup = VecInit(nodeMap.yByNode.map(_.U(mesh.yBits.W))) - val present = VecInit(nodeMap.xByNode.indices.map(node => nodeMap.presentNodes.contains(node).B)) - val padded = Wire(UInt(paddedBits.W)) - padded := flit - - val placed = io.in.bits.targetNode < nodeMap.xByNode.size.U && present(io.in.bits.targetNode) - io.in.ready := !busy && placed - when(io.in.valid)(assert(placed, "CHI Mesh target NodeID is not placed")) - when(io.in.fire) { - flit := io.in.bits.flit - target := io.in.bits.targetNode - beat := 0.U - busy := true.B - } - - io.out.valid := busy - io.out.bits.srcX := localX.U - io.out.bits.srcY := localY.U - io.out.bits.dstX := xLookup(target) - io.out.bits.dstY := yLookup(target) - io.out.bits.vc := virtualChannel.U - io.out.bits.head := beat === 0.U - io.out.bits.tail := beat === (chunks - 1).U - io.out.bits.payload := (padded >> (beat * mesh.payloadBits.U))(mesh.payloadBits - 1, 0) - when(io.out.fire) { - when(io.out.bits.tail) { - busy := false.B - }.otherwise { - beat := beat + 1.U - } - } -} - -/** Reassembles one fixed-length CHI packet after the mesh VC has been demultiplexed. */ -@instantiable -class ChiMeshDepacketizer(mesh: MeshParams, flitBits: Int, virtualChannel: Int) extends Module { - require(flitBits > 0) - require(virtualChannel >= 0 && virtualChannel < mesh.virtualChannels) - val chunks = (flitBits + mesh.payloadBits - 1) / mesh.payloadBits - val beatBits = math.max(1, log2Ceil(chunks)) - - @public - val io = IO(new Bundle { - val in = Flipped(Decoupled(new MeshFlit(mesh))) - val out = Decoupled(UInt(flitBits.W)) - }) - - val collecting = RegInit(false.B) - val beat = RegInit(0.U(beatBits.W)) - val words = Reg(Vec(math.max(1, chunks - 1), UInt(mesh.payloadBits.W))) - val result = Reg(UInt(flitBits.W)) - val resultValid = RegInit(false.B) - - io.in.ready := !resultValid - io.out.valid := resultValid - io.out.bits := result - when(io.in.fire) { - assert(io.in.bits.vc === virtualChannel.U, "CHI packet arrived on wrong mesh VC") - when(!collecting) { - assert(io.in.bits.head, "CHI mesh packet started without head") - }.otherwise { - assert(!io.in.bits.head, "CHI mesh packet has a second head") - } - assert(io.in.bits.tail === (beat === (chunks - 1).U), "CHI mesh packet has wrong length") - if (chunks > 1) { when(!io.in.bits.tail)(words(beat) := io.in.bits.payload) } - when(io.in.bits.tail) { - result := (if (chunks == 1) io.in.bits.payload else Cat(Seq(io.in.bits.payload) ++ words.reverse))(flitBits - 1, 0) - resultValid := true.B - collecting := false.B - beat := 0.U - }.otherwise { - collecting := true.B - beat := beat + 1.U - } - } - when(io.out.fire) { - resultValid := false.B - } -} - -/** - * One tile's CHI-facing Mesh endpoint. - * - * It owns packet boundaries and CHI channel-to-VC assignment. Its Mesh ports - * carry opaque packets, so a router can be changed without changing cache or - * Home protocol state. Snoop destinations are explicit sideband because CHI - * Snp flits identify their source but do not contain a target NodeID. - */ -@instantiable -class ChiMeshEndpoint( - p: Params, - mesh: MeshParams, - localX: Int, - localY: Int, - nodeMap: ChiMeshNodeMap) - extends Module { - require(mesh.virtualChannels >= ChiMeshVirtualChannel.Required) - require(nodeMap.mesh == mesh && nodeMap.coordinateBits == p.nodeIdBits) - val reqBits = (new RequestFlit(p)).flitWidth - val rspBits = (new ResponseFlit(p)).flitWidth - val datBits = (new DataFlit(p)).flitWidth - val snpBits = (new SnoopFlit(p)).flitWidth - val message = (bits: Int) => new ChiMeshMessage(bits, p.nodeIdBits) - - @public - val io = IO(new Bundle { - val txReq = Flipped(Decoupled(message(reqBits))) - val txRsp = Flipped(Decoupled(message(rspBits))) - val txDat = Flipped(Decoupled(message(datBits))) - val txSnp = Flipped(Decoupled(message(snpBits))) - val rxReq = Decoupled(UInt(reqBits.W)) - val rxRsp = Decoupled(UInt(rspBits.W)) - val rxDat = Decoupled(UInt(datBits.W)) - val rxSnp = Decoupled(UInt(snpBits.W)) - val meshOut = Vec(mesh.virtualChannels, Decoupled(new MeshFlit(mesh))) - val meshIn = Vec(mesh.virtualChannels, Flipped(Decoupled(new MeshFlit(mesh)))) - }) - - val txReq: Instance[ChiMeshPacketizer] = - Instantiate(new ChiMeshPacketizer(mesh, reqBits, ChiMeshVirtualChannel.Request, localX, localY, nodeMap)) - val txRsp: Instance[ChiMeshPacketizer] = - Instantiate(new ChiMeshPacketizer(mesh, rspBits, ChiMeshVirtualChannel.Response, localX, localY, nodeMap)) - val txDat: Instance[ChiMeshPacketizer] = - Instantiate(new ChiMeshPacketizer(mesh, datBits, ChiMeshVirtualChannel.Data, localX, localY, nodeMap)) - val txSnp: Instance[ChiMeshPacketizer] = - Instantiate(new ChiMeshPacketizer(mesh, snpBits, ChiMeshVirtualChannel.Snoop, localX, localY, nodeMap)) - txReq.io.in <> io.txReq - txRsp.io.in <> io.txRsp - txDat.io.in <> io.txDat - txSnp.io.in <> io.txSnp - io.meshOut(ChiMeshVirtualChannel.Request) <> txReq.io.out - io.meshOut(ChiMeshVirtualChannel.Response) <> txRsp.io.out - io.meshOut(ChiMeshVirtualChannel.Data) <> txDat.io.out - io.meshOut(ChiMeshVirtualChannel.Snoop) <> txSnp.io.out - for (vc <- ChiMeshVirtualChannel.Required until mesh.virtualChannels) { - io.meshOut(vc).valid := false.B - io.meshOut(vc).bits := 0.U.asTypeOf(new MeshFlit(mesh)) - io.meshIn(vc).ready := false.B - } - val rxReq: Instance[ChiMeshDepacketizer] = - Instantiate(new ChiMeshDepacketizer(mesh, reqBits, ChiMeshVirtualChannel.Request)) - val rxRsp: Instance[ChiMeshDepacketizer] = - Instantiate(new ChiMeshDepacketizer(mesh, rspBits, ChiMeshVirtualChannel.Response)) - val rxDat: Instance[ChiMeshDepacketizer] = - Instantiate(new ChiMeshDepacketizer(mesh, datBits, ChiMeshVirtualChannel.Data)) - val rxSnp: Instance[ChiMeshDepacketizer] = - Instantiate(new ChiMeshDepacketizer(mesh, snpBits, ChiMeshVirtualChannel.Snoop)) - val receivers = Seq(rxReq, rxRsp, rxDat, rxSnp) - for ((receiver, vc) <- receivers.zipWithIndex) { receiver.io.in <> io.meshIn(vc) } - io.rxReq <> rxReq.io.out - io.rxRsp <> rxRsp.io.out - io.rxDat <> rxDat.io.out - io.rxSnp <> rxSnp.io.out -} - -/** Typed adapter for a cache or NPU CHI requester port. */ -@instantiable -class ChiMeshRequesterEndpoint( - p: Params, - mesh: MeshParams, - localX: Int, - localY: Int, - nodeMap: ChiMeshNodeMap) - extends Module { - - @public - val io = IO(new Bundle { - val chi = Flipped(new RequesterPort(p)) - val meshOut = Vec(mesh.virtualChannels, Decoupled(new MeshFlit(mesh))) - val meshIn = Vec(mesh.virtualChannels, Flipped(Decoupled(new MeshFlit(mesh)))) - }) - - val endpoint: Instance[ChiMeshEndpoint] = Instantiate(new ChiMeshEndpoint(p, mesh, localX, localY, nodeMap)) - endpoint.io.txReq.valid := io.chi.req.valid - endpoint.io.txReq.bits.targetNode := io.chi.req.bits.tgtId - endpoint.io.txReq.bits.flit := io.chi.req.bits.packed - io.chi.req.ready := endpoint.io.txReq.ready - endpoint.io.txRsp.valid := io.chi.txRsp.valid - endpoint.io.txRsp.bits.targetNode := io.chi.txRsp.bits.tgtId - endpoint.io.txRsp.bits.flit := io.chi.txRsp.bits.packed - io.chi.txRsp.ready := endpoint.io.txRsp.ready - endpoint.io.txDat.valid := io.chi.txDat.valid - endpoint.io.txDat.bits.targetNode := io.chi.txDat.bits.tgtId - endpoint.io.txDat.bits.flit := io.chi.txDat.bits.packed - io.chi.txDat.ready := endpoint.io.txDat.ready - endpoint.io.txSnp.valid := false.B - endpoint.io.txSnp.bits := 0.U.asTypeOf(endpoint.io.txSnp.bits) - endpoint.io.rxRsp.ready := io.chi.rxRsp.ready - io.chi.rxRsp.valid := endpoint.io.rxRsp.valid - io.chi.rxRsp.bits.unpack(endpoint.io.rxRsp.bits) - endpoint.io.rxDat.ready := io.chi.rxDat.ready - io.chi.rxDat.valid := endpoint.io.rxDat.valid - io.chi.rxDat.bits.unpack(endpoint.io.rxDat.bits) - endpoint.io.rxSnp.ready := io.chi.snp.ready - io.chi.snp.valid := endpoint.io.rxSnp.valid - io.chi.snp.bits.unpack(endpoint.io.rxSnp.bits) - endpoint.io.rxReq.ready := false.B - when(endpoint.io.rxReq.valid)(assert(false.B, "CHI requester endpoint received a REQ")) - io.meshOut <> endpoint.io.meshOut - endpoint.io.meshIn <> io.meshIn -} - -/** Typed adapter for one directory Home and all of its snoop targets. */ -@instantiable -class ChiMeshHomeEndpoint( - p: Params, - mesh: MeshParams, - localX: Int, - localY: Int, - nodeMap: ChiMeshNodeMap, - agents: Int) - extends Module { - require(agents >= 1 && agents < (1 << p.nodeIdBits)) - val snpBits = (new SnoopFlit(p)).flitWidth - - @public - val io = IO(new Bundle { - val req = Decoupled(new RequestFlit(p)) - val rxRsp = Decoupled(new ResponseFlit(p)) - val rxDat = Decoupled(new DataFlit(p)) - val rsp = Flipped(Decoupled(new ResponseFlit(p))) - val dat = Flipped(Decoupled(new DataFlit(p))) - val snp = Vec(agents, Flipped(Decoupled(new SnoopFlit(p)))) - val meshOut = Vec(mesh.virtualChannels, Decoupled(new MeshFlit(mesh))) - val meshIn = Vec(mesh.virtualChannels, Flipped(Decoupled(new MeshFlit(mesh)))) - }) - - val endpoint: Instance[ChiMeshEndpoint] = Instantiate(new ChiMeshEndpoint(p, mesh, localX, localY, nodeMap)) - endpoint.io.txReq.valid := false.B - endpoint.io.txReq.bits := 0.U.asTypeOf(endpoint.io.txReq.bits) - endpoint.io.txRsp.valid := io.rsp.valid - endpoint.io.txRsp.bits.targetNode := io.rsp.bits.tgtId - endpoint.io.txRsp.bits.flit := io.rsp.bits.packed - io.rsp.ready := endpoint.io.txRsp.ready - endpoint.io.txDat.valid := io.dat.valid - endpoint.io.txDat.bits.targetNode := io.dat.bits.tgtId - endpoint.io.txDat.bits.flit := io.dat.bits.packed - io.dat.ready := endpoint.io.txDat.ready - val snpArbiter = Module(new RRArbiter(new ChiMeshMessage(snpBits, p.nodeIdBits), agents)) - for (agent <- 0 until agents) { - snpArbiter.io.in(agent).valid := io.snp(agent).valid - snpArbiter.io.in(agent).bits.targetNode := (agent + 1).U - snpArbiter.io.in(agent).bits.flit := io.snp(agent).bits.packed - io.snp(agent).ready := snpArbiter.io.in(agent).ready - } - endpoint.io.txSnp <> snpArbiter.io.out - endpoint.io.rxReq.ready := io.req.ready - io.req.valid := endpoint.io.rxReq.valid - io.req.bits.unpack(endpoint.io.rxReq.bits) - endpoint.io.rxRsp.ready := io.rxRsp.ready - io.rxRsp.valid := endpoint.io.rxRsp.valid - io.rxRsp.bits.unpack(endpoint.io.rxRsp.bits) - endpoint.io.rxDat.ready := io.rxDat.ready - io.rxDat.valid := endpoint.io.rxDat.valid - io.rxDat.bits.unpack(endpoint.io.rxDat.bits) - endpoint.io.rxSnp.ready := true.B - when(endpoint.io.rxSnp.valid)(assert(false.B, "Home received a CHI snoop")) - io.meshOut <> endpoint.io.meshOut - endpoint.io.meshIn <> io.meshIn -} - -/** Native verification target for Home snoop target-sideband routing. */ -class ChiMeshHomeSnoopLoopback extends Module { - val chi = Params() - val mesh = MeshParams(xNodes = 2, yNodes = 1, payloadBits = 32, virtualChannels = 8) - - val map = ChiMeshNodeMap( - Seq.tabulate(1 << chi.nodeIdBits)(node => if (node == 1) 1 else 0), - Seq.fill(1 << chi.nodeIdBits)(0), - mesh, - presentNodes = (0 until (1 << chi.nodeIdBits)).toSet - ) - - val snpBits = (new SnoopFlit(chi)).flitWidth - - @public - val io = IO(new Bundle { - val in = Flipped(Decoupled(UInt(snpBits.W))) - val out = Decoupled(UInt(snpBits.W)) - }) - - val home: Instance[ChiMeshHomeEndpoint] = - Instantiate(new ChiMeshHomeEndpoint(chi, mesh, localX = 0, localY = 0, nodeMap = map, agents = 2)) - val receiver: Instance[ChiMeshEndpoint] = - Instantiate(new ChiMeshEndpoint(chi, mesh, localX = 1, localY = 0, nodeMap = map)) - home.io.snp(0).valid := io.in.valid - home.io.snp(0).bits.unpack(io.in.bits) - io.in.ready := home.io.snp(0).ready - home.io.snp(1).valid := false.B - home.io.snp(1).bits := 0.U.asTypeOf(home.io.snp(1).bits) - home.io.rsp.valid := false.B - home.io.rsp.bits := 0.U.asTypeOf(home.io.rsp.bits) - home.io.dat.valid := false.B - home.io.dat.bits := 0.U.asTypeOf(home.io.dat.bits) - home.io.req.ready := true.B - home.io.rxRsp.ready := true.B - home.io.rxDat.ready := true.B - receiver.io.txReq.valid := false.B - receiver.io.txReq.bits := 0.U.asTypeOf(receiver.io.txReq.bits) - receiver.io.txRsp.valid := false.B - receiver.io.txRsp.bits := 0.U.asTypeOf(receiver.io.txRsp.bits) - receiver.io.txDat.valid := false.B - receiver.io.txDat.bits := 0.U.asTypeOf(receiver.io.txDat.bits) - receiver.io.txSnp.valid := false.B - receiver.io.txSnp.bits := 0.U.asTypeOf(receiver.io.txSnp.bits) - receiver.io.rxReq.ready := true.B - receiver.io.rxRsp.ready := true.B - receiver.io.rxDat.ready := true.B - io.out <> receiver.io.rxSnp - receiver.io.meshIn <> home.io.meshOut - for (vc <- 0 until mesh.virtualChannels) { - home.io.meshIn(vc).valid := false.B - home.io.meshIn(vc).bits := 0.U.asTypeOf(new MeshFlit(mesh)) - receiver.io.meshOut(vc).ready := true.B - } -} - -/** Native endpoint verification target: a real CHI Dat channel traverses its Mesh endpoint. */ -class ChiMeshEndpointLoopback extends Module { - val chi = Params() - val mesh = MeshParams(xNodes = 2, yNodes = 2, payloadBits = 32, virtualChannels = 8) - - val map = ChiMeshNodeMap( - Seq.tabulate(1 << chi.nodeIdBits)(_ & 1), - Seq.tabulate(1 << chi.nodeIdBits)(node => (node >> 1) & 1), - mesh, - presentNodes = (0 until (1 << chi.nodeIdBits)).toSet - ) - - val datBits = (new DataFlit(chi)).flitWidth - - @public - val io = IO(new Bundle { - val in = Flipped(Decoupled(new ChiMeshMessage(datBits, chi.nodeIdBits))) - val out = Decoupled(UInt(datBits.W)) - }) - - val endpoint: Instance[ChiMeshEndpoint] = - Instantiate(new ChiMeshEndpoint(chi, mesh, localX = 0, localY = 0, nodeMap = map)) - endpoint.io.txDat <> io.in - endpoint.io.txReq.valid := false.B - endpoint.io.txReq.bits := 0.U.asTypeOf(endpoint.io.txReq.bits) - endpoint.io.txRsp.valid := false.B - endpoint.io.txRsp.bits := 0.U.asTypeOf(endpoint.io.txRsp.bits) - endpoint.io.txSnp.valid := false.B - endpoint.io.txSnp.bits := 0.U.asTypeOf(endpoint.io.txSnp.bits) - endpoint.io.rxReq.ready := true.B - endpoint.io.rxRsp.ready := true.B - endpoint.io.rxSnp.ready := true.B - io.out <> endpoint.io.rxDat - endpoint.io.meshIn <> endpoint.io.meshOut -} - -/** Native verification target for packetization and reassembly without a mesh topology. */ -class ChiMeshCodecLoopback extends Module { - val mesh = MeshParams(xNodes = 2, yNodes = 2, payloadBits = 32, virtualChannels = 4) - val chi = Params() - - val map = ChiMeshNodeMap( - Seq.tabulate(1 << chi.nodeIdBits)(_ & 1), - Seq.tabulate(1 << chi.nodeIdBits)(node => (node >> 1) & 1), - mesh, - presentNodes = Set(1, 2, 3, 64) - ) - - val flitBits = (new DataFlit(chi)).flitWidth - - @public - val io = IO(new Bundle { - val in = Flipped(Decoupled(new ChiMeshMessage(flitBits, map.coordinateBits))) - val out = Decoupled(UInt(flitBits.W)) - val pause = Input(Bool()) - val observedFire = Output(Bool()) - val observedValid = Output(Bool()) - val observed = Output(new MeshFlit(mesh)) - }) - - val tx: Instance[ChiMeshPacketizer] = Instantiate(new ChiMeshPacketizer( - mesh, - flitBits, - virtualChannel = ChiMeshVirtualChannel.Data, - localX = 0, - localY = 0, - nodeMap = map - )) - - val rx: Instance[ChiMeshDepacketizer] = - Instantiate(new ChiMeshDepacketizer(mesh, flitBits, virtualChannel = ChiMeshVirtualChannel.Data)) - io.in <> tx.io.in - rx.io.in.valid := tx.io.out.valid && !io.pause - rx.io.in.bits := tx.io.out.bits - tx.io.out.ready := rx.io.in.ready && !io.pause - io.out <> rx.io.out - io.observedFire := tx.io.out.fire - io.observedValid := tx.io.out.valid - io.observed := tx.io.out.bits -} diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/Emit.scala b/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/Emit.scala deleted file mode 100644 index f80e06dee..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/Emit.scala +++ /dev/null @@ -1,35 +0,0 @@ -package hier.chip.mesh - -object Emit extends App { - val p = MeshParams(xNodes = 3, yNodes = 3, payloadBits = 32, virtualChannels = 4) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new MeshRouter(p, 1, 1), - firtoolOpts = args, - args = Array("--target-dir", "build/MeshRouter", "--split-verilog") - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new MeshCreditNetwork(MeshParams(2, 2, 32, 4), linkDepth = 2), - firtoolOpts = args, - args = Array("--target-dir", "build/MeshCreditNetwork", "--split-verilog") - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new ChiMeshCodecLoopback, - firtoolOpts = args, - args = Array("--target-dir", "build/ChiMeshCodecLoopback", "--split-verilog") - ) - val chi = memcore.bus.chi.Params() - val endpointMesh = MeshParams(2, 2, 32, 4) - - val endpointNodes = ChiMeshNodeMap( - Seq.tabulate(1 << chi.nodeIdBits)(_ & 1), - Seq.tabulate(1 << chi.nodeIdBits)(node => (node >> 1) & 1), - endpointMesh, - presentNodes = Set(1, 2, 64) - ) - - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new ChiMeshRequesterEndpoint(chi, endpointMesh, 1, 0, endpointNodes), - firtoolOpts = args, - args = Array("--target-dir", "build/ChiMeshRequesterEndpoint", "--split-verilog") - ) -} diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/Mesh.scala b/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/Mesh.scala deleted file mode 100644 index 02fd950a9..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/Mesh.scala +++ /dev/null @@ -1,126 +0,0 @@ -package hier.chip.mesh - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} - -object MeshDirection { - val Local = 0 - val North = 1 - val South = 2 - val East = 3 - val West = 4 - val Count = 5 -} - -case class MeshParams( - xNodes: Int, - yNodes: Int, - payloadBits: Int, - virtualChannels: Int = 2) { - require(xNodes >= 1 && yNodes >= 1) - require(payloadBits > 0) - require(virtualChannels >= 1 && isPow2(virtualChannels)) - val xBits = math.max(1, log2Ceil(xNodes)) - val yBits = math.max(1, log2Ceil(yNodes)) - val vcBits = math.max(1, log2Ceil(virtualChannels)) -} - -/** Generic wormhole flit. Packet content is opaque to the mesh. */ -class MeshFlit(p: MeshParams) extends Bundle { - val srcX = UInt(p.xBits.W) - val srcY = UInt(p.yBits.W) - val dstX = UInt(p.xBits.W) - val dstY = UInt(p.yBits.W) - val vc = UInt(p.vcBits.W) - val head = Bool() - val tail = Bool() - val payload = UInt(p.payloadBits.W) -} - -/** Deterministic XY routing; only physically connected directions own state. */ -@instantiable -class MeshRouter(p: MeshParams, x: Int, y: Int) extends Module { - require(x >= 0 && x < p.xNodes && y >= 0 && y < p.yNodes) - val flit = new MeshFlit(p) - - val connected = Seq(MeshDirection.Local) ++ - Seq(MeshDirection.North).filter(_ => y > 0) ++ - Seq(MeshDirection.South).filter(_ => y < p.yNodes - 1) ++ - Seq(MeshDirection.East).filter(_ => x < p.xNodes - 1) ++ - Seq(MeshDirection.West).filter(_ => x > 0) - - val count = connected.size - val inputBits = math.max(1, log2Ceil(count)) - - @public - val io = IO(new Bundle { - val in = Vec(MeshDirection.Count, Vec(p.virtualChannels, Flipped(Decoupled(flit)))) - val out = Vec(MeshDirection.Count, Vec(p.virtualChannels, Decoupled(flit))) - }) - - def route(value: MeshFlit): UInt = Mux( - value.dstX < x.U, - MeshDirection.West.U, - Mux( - value.dstX > x.U, - MeshDirection.East.U, - Mux(value.dstY < y.U, MeshDirection.North.U, Mux(value.dstY > y.U, MeshDirection.South.U, MeshDirection.Local.U)) - ) - ) - - for { - direction <- (0 until MeshDirection.Count).filterNot(connected.contains) - vc <- 0 until p.virtualChannels - } { - io.in(direction)(vc).ready := false.B - io.out(direction)(vc).valid := false.B - io.out(direction)(vc).bits := 0.U.asTypeOf(flit) - } - for (vc <- 0 until p.virtualChannels) { - val inputs = connected.map(direction => io.in(direction)(vc)) - val payloads = VecInit(inputs.map(_.bits)) - val routes = inputs.map(input => route(input.bits)) - val owners = connected.map { out => - val lockValid = RegInit(false.B) - val lockInput = Reg(UInt(inputBits.W)) - val cursor = RegInit(0.U(inputBits.W)) - val request = VecInit(inputs.indices.map(in => inputs(in).valid && routes(in) === out.U)) - val ordered = VecInit(inputs.indices.map(offset => request((cursor +& offset.U) % count.U))) - val fairChoice = (cursor +& PriorityEncoder(ordered)) % count.U - val selected = Mux(lockValid, lockInput, fairChoice) - val valid = Mux(lockValid, VecInit(inputs.map(_.valid))(lockInput), request.asUInt.orR) - io.out(out)(vc).valid := valid - io.out(out)(vc).bits := payloads(selected) - when(valid && !io.out(out)(vc).ready && !lockValid) { - lockValid := true.B - lockInput := selected - } - when(io.out(out)(vc).fire) { - when(io.out(out)(vc).bits.tail) { - lockValid := false.B - cursor := Mux(selected === (count - 1).U, 0.U, selected + 1.U) - }.otherwise { - lockValid := true.B - lockInput := selected - } - } - (out, selected, lockValid, lockInput) - } - for ((input, index) <- inputs.zipWithIndex) { - input.ready := owners.map { case (out, selected, _, _) => - io.out(out)(vc).ready && io.out(out)(vc).valid && selected === index.U - }.reduce(_ || _) - when(input.fire) { - assert(input.bits.vc === vc.U, "Mesh flit arrived on wrong VC port") - assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, "Mesh destination outside topology") - when(!input.bits.head) { - val owned = owners.map { case (out, _, locked, owner) => - routes(index) === out.U && locked && owner === index.U - }.reduce(_ || _) - assert(owned, "Mesh packet body without ownership") - } - } - } - } -} diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/MeshBulk.scala b/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/MeshBulk.scala deleted file mode 100644 index a8c5199d9..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/MeshBulk.scala +++ /dev/null @@ -1,192 +0,0 @@ -package hier.chip.mesh - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import memcore.bus.axi.Beat - -/** One AXI-stream beat plus the Mesh destination selected at packet start. */ -class MeshBulkBeat(dataBits: Int, nodeIdBits: Int) extends Bundle { - require(dataBits > 0 && dataBits % 8 == 0) - val targetNode = UInt(nodeIdBits.W) - val data = UInt(dataBits.W) - val keep = UInt((dataBits / 8).W) - val last = Bool() -} - -/** VC5 control descriptor for one remote scratchpad DMA transaction. */ -class BulkDescriptor(nodeIdBits: Int, addressBits: Int = 32) extends Bundle { - val sourceNode = UInt(nodeIdBits.W) - val targetNode = UInt(nodeIdBits.W) - val write = Bool() - val scratchpadAddr = UInt(addressBits.W) - val bytes = UInt(32.W) - val txnId = UInt(8.W) -} - -/** VC6 acknowledgement or terminal completion for a bulk descriptor. */ -class BulkCompletion(nodeIdBits: Int) extends Bundle { - val targetNode = UInt(nodeIdBits.W) - val txnId = UInt(8.W) - val ready = Bool() - val error = Bool() -} - -/** One-flit control packet for VC5 descriptors and VC6 acknowledgements. */ -class MeshBulkControlTx( - mesh: MeshParams, - payloadBits: Int, - virtualChannel: Int, - localX: Int, - localY: Int, - nodeMap: ChiMeshNodeMap) - extends Module { - require(payloadBits > 0 && payloadBits <= mesh.payloadBits) - - @public - val io = IO(new Bundle { - val in = Flipped(Decoupled(new MeshBulkBeat(payloadBits, nodeMap.coordinateBits))) - val out = Decoupled(new MeshFlit(mesh)) - }) - - val xLookup = VecInit(nodeMap.xByNode.map(_.U(mesh.xBits.W))) - val yLookup = VecInit(nodeMap.yByNode.map(_.U(mesh.yBits.W))) - io.out.valid := io.in.valid - io.out.bits.srcX := localX.U - io.out.bits.srcY := localY.U - io.out.bits.dstX := xLookup(io.in.bits.targetNode) - io.out.bits.dstY := yLookup(io.in.bits.targetNode) - io.out.bits.vc := virtualChannel.U - io.out.bits.head := true.B - io.out.bits.tail := true.B - io.out.bits.payload := io.in.bits.data - io.in.ready := io.out.ready -} - -class MeshBulkControlRx(mesh: MeshParams, payloadBits: Int, virtualChannel: Int) extends Module { - require(payloadBits > 0 && payloadBits <= mesh.payloadBits) - - @public - val io = IO(new Bundle { - val in = Flipped(Decoupled(new MeshFlit(mesh))) - val out = Decoupled(UInt(payloadBits.W)) - }) - - io.out.valid := io.in.valid - io.out.bits := io.in.bits.payload(payloadBits - 1, 0) - io.in.ready := io.out.ready - when(io.in.fire) { - assert( - io.in.bits.vc === virtualChannel.U && io.in.bits.head && io.in.bits.tail, - "Bulk control packet must be one flit on its assigned VC" - ) - } -} - -/** - * Maps one AXI-stream transaction to one non-interleaved Mesh packet. - * - * Mesh `head` is the first AXI beat and `tail` is the beat carrying AXI - * `last`. The Mesh router therefore holds the selected output across the full - * long stream, rather than arbitrating again at every beat. - */ -@instantiable -class MeshBulkPacketizer( - mesh: MeshParams, - dataBits: Int, - virtualChannel: Int, - localX: Int, - localY: Int, - nodeMap: ChiMeshNodeMap) - extends Module { - require(mesh.payloadBits >= dataBits + dataBits / 8) - require(virtualChannel >= 0 && virtualChannel < mesh.virtualChannels) - require(nodeMap.mesh == mesh) - - @public - val io = IO(new Bundle { - val in = Flipped(Decoupled(new MeshBulkBeat(dataBits, nodeMap.coordinateBits))) - val out = Decoupled(new MeshFlit(mesh)) - }) - - val first = RegInit(true.B) - val target = Reg(UInt(nodeMap.coordinateBits.W)) - val xLookup = VecInit(nodeMap.xByNode.map(_.U(mesh.xBits.W))) - val yLookup = VecInit(nodeMap.yByNode.map(_.U(mesh.yBits.W))) - val selectedTarget = Mux(first, io.in.bits.targetNode, target) - val payload = Wire(UInt(mesh.payloadBits.W)) - payload := Cat(io.in.bits.keep, io.in.bits.data) - io.out.valid := io.in.valid - io.out.bits.srcX := localX.U - io.out.bits.srcY := localY.U - io.out.bits.dstX := xLookup(selectedTarget) - io.out.bits.dstY := yLookup(selectedTarget) - io.out.bits.vc := virtualChannel.U - io.out.bits.head := first - io.out.bits.tail := io.in.bits.last - io.out.bits.payload := payload - io.in.ready := io.out.ready - when(io.in.fire) { - assert(io.in.bits.targetNode < nodeMap.xByNode.size.U, "Bulk Mesh target NodeID is not placed") - when(!first)(assert(io.in.bits.targetNode === target, "Bulk Mesh target changed inside AXI packet")) - target := io.in.bits.targetNode - first := io.in.bits.last - } -} - -/** Recreates AXI-stream beat boundaries from a single bulk Mesh VC. */ -@instantiable -class MeshBulkDepacketizer(mesh: MeshParams, dataBits: Int, virtualChannel: Int) extends Module { - require(mesh.payloadBits >= dataBits + dataBits / 8) - require(virtualChannel >= 0 && virtualChannel < mesh.virtualChannels) - - @public - val io = IO(new Bundle { - val in = Flipped(Decoupled(new MeshFlit(mesh))) - val out = Decoupled(new Beat(dataBits)) - }) - - val first = RegInit(true.B) - io.out.valid := io.in.valid - io.out.bits.data := io.in.bits.payload(dataBits - 1, 0) - io.out.bits.keep := io.in.bits.payload(dataBits + dataBits / 8 - 1, dataBits) - io.out.bits.last := io.in.bits.tail - io.out.bits.id := 0.U - io.out.bits.dest := 0.U - io.out.bits.user := 0.U - io.in.ready := io.out.ready - when(io.in.fire) { - assert(io.in.bits.vc === virtualChannel.U, "Bulk packet arrived on wrong Mesh VC") - assert(io.in.bits.head === first, "Bulk Mesh packet head mismatch") - first := io.in.bits.tail - } -} - -/** Native verification target for long AXI-stream packet ownership over Mesh. */ -class MeshBulkLoopback extends Module { - val mesh = MeshParams(xNodes = 2, yNodes = 2, payloadBits = 320, virtualChannels = 8) - - val map = ChiMeshNodeMap( - Seq.tabulate(128)(_ & 1), - Seq.tabulate(128)(node => (node >> 1) & 1), - mesh, - presentNodes = (0 until 128).toSet - ) - - @public - val io = IO(new Bundle { - val in = Flipped(Decoupled(new MeshBulkBeat(256, 7))) - val out = Decoupled(new Beat(256)) - val observed = Output(new MeshFlit(mesh)) - val observedValid = Output(Bool()) - }) - - val tx: Instance[MeshBulkPacketizer] = - Instantiate(new MeshBulkPacketizer(mesh, 256, virtualChannel = 4, localX = 0, localY = 0, map)) - val rx: Instance[MeshBulkDepacketizer] = Instantiate(new MeshBulkDepacketizer(mesh, 256, virtualChannel = 4)) - tx.io.in <> io.in - rx.io.in <> tx.io.out - io.out <> rx.io.out - io.observed := tx.io.out.bits - io.observedValid := tx.io.out.valid -} diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/MeshCredit.scala b/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/MeshCredit.scala deleted file mode 100644 index 2f57f3d42..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/MeshCredit.scala +++ /dev/null @@ -1,165 +0,0 @@ -package hier.chip.mesh - -import memcore.memory.queue.Queue - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} - -/** One directed physical Mesh link. Each VC returns its own buffer credits. */ -class MeshCreditLink(p: MeshParams) extends Bundle { - val valid = Output(Bool()) - val flit = Output(new MeshFlit(p)) - val credit = Input(Vec(p.virtualChannels, Bool())) -} - -/** - * Sender half of a credit-based Mesh link. - * - * Credits start at zero and are supplied by the receiver after link activation. - * A coordinated reset is the only supported way to stop an active link. - */ -@instantiable -class MeshCreditTx(p: MeshParams, maxCredits: Int) extends Module { - require(maxCredits >= 1 && maxCredits <= 255) - val creditBits = math.max(1, log2Ceil(maxCredits + 1)) - - @public - val io = IO(new Bundle { - val active = Input(Bool()) - val in = Vec(p.virtualChannels, Flipped(Decoupled(new MeshFlit(p)))) - val link = new MeshCreditLink(p) - val credits = Output(UInt(creditBits.W)) - }) - - val credits = RegInit(VecInit(Seq.fill(p.virtualChannels)(0.U(creditBits.W)))) - val returned = RegNext(io.link.credit, VecInit(Seq.fill(p.virtualChannels)(false.B))) - val cursor = RegInit(0.U(p.vcBits.W)) - val eligible = VecInit((0 until p.virtualChannels).map(vc => io.active && io.in(vc).valid && credits(vc) =/= 0.U)) - val ordered = VecInit((0 until p.virtualChannels).map(offset => eligible((cursor +& offset.U) % p.virtualChannels.U))) - val selected = (cursor +& PriorityEncoder(ordered)) % p.virtualChannels.U - val send = eligible.asUInt.orR - for (vc <- 0 until p.virtualChannels) { - io.in(vc).ready := send && selected === vc.U - val consumed = send && selected === vc.U - when(returned(vc) =/= consumed)(credits(vc) := Mux(returned(vc), credits(vc) + 1.U, credits(vc) - 1.U)) - when(returned(vc) && !consumed)(assert(credits(vc) < maxCredits.U, "Mesh TX credit overflow")) - when(consumed)(assert(io.in(vc).bits.vc === vc.U, "Mesh TX wrong VC port")) - } - when(send)(cursor := Mux(selected === (p.virtualChannels - 1).U, 0.U, selected + 1.U)) - io.link.valid := RegNext(send, false.B) - io.link.flit := RegEnable(io.in(selected).bits, 0.U.asTypeOf(new MeshFlit(p)), send) - io.credits := credits(selected) - val wasActive = RegNext(io.active, false.B) - when(wasActive)(assert(io.active, "Mesh TX requires coordinated reset to stop")) -} - -/** Receiver half of a credit-based Mesh link with independent FIFO and credit pool per VC. */ -@instantiable -class MeshCreditRx(p: MeshParams, depth: Int) extends Module { - require(depth >= 1 && depth <= 255) - val creditBits = math.max(1, log2Ceil(depth + 1)) - - @public - val io = IO(new Bundle { - val active = Input(Bool()) - val link = Flipped(new MeshCreditLink(p)) - val out = Vec(p.virtualChannels, Decoupled(new MeshFlit(p))) - val occupancy = Output(Vec(p.virtualChannels, UInt(creditBits.W))) - }) - - val queues = Seq.fill(p.virtualChannels)(Module(new Queue(new MeshFlit(p), depth, pipe = true))) - val advertised = RegInit(VecInit(Seq.fill(p.virtualChannels)(0.U(creditBits.W)))) - val enqueueReady = Wire(Vec(p.virtualChannels, Bool())) - for (vc <- 0 until p.virtualChannels) { - val queue = queues(vc) - val accepted = io.link.valid && io.link.flit.vc === vc.U - val grant = io.active && (advertised(vc) +& queue.io.count) < depth.U - io.link.credit(vc) := grant - queue.io.enq.valid := accepted - queue.io.enq.bits := io.link.flit - enqueueReady(vc) := queue.io.enq.ready - io.out(vc) <> queue.io.deq - when(grant =/= accepted) { - advertised(vc) := Mux(grant, advertised(vc) + 1.U, advertised(vc) - 1.U) - } - io.occupancy(vc) := queue.io.count - } - when(io.link.valid) { - assert(io.active, "Mesh RX flit on inactive link") - assert(advertised(io.link.flit.vc) =/= 0.U, "Mesh RX flit without returned credit") - assert(enqueueReady(io.link.flit.vc), "Mesh RX buffer overflow") - } - for (vc <- 0 until p.virtualChannels) { - assert((advertised(vc) +& queues(vc).io.count) <= depth.U, "Mesh RX credit conservation") - } - val wasActive = RegNext(io.active, false.B) - when(wasActive)(assert(io.active, "Mesh RX requires coordinated reset to stop")) -} - -/** 2D Mesh with explicit independent local VC ports and per-VC physical credits. */ -@instantiable -class MeshCreditNetwork(p: MeshParams, linkDepth: Int = 2) extends Module { - require(linkDepth >= 1 && linkDepth <= 255) - val flit = new MeshFlit(p) - val routers: Seq[Seq[Instance[MeshRouter]]] = - Seq.tabulate(p.yNodes, p.xNodes)((y, x) => Instantiate(new MeshRouter(p, x, y))) - private def index(x: Int, y: Int): Int = y * p.xNodes + x - - @public - val io = IO(new Bundle { - val active = Input(Bool()) - val localIn = Vec(p.xNodes * p.yNodes, Vec(p.virtualChannels, Flipped(Decoupled(flit)))) - val localOut = Vec(p.xNodes * p.yNodes, Vec(p.virtualChannels, Decoupled(flit))) - }) - - def wireLink(source: Vec[DecoupledIO[MeshFlit]], sink: Vec[DecoupledIO[MeshFlit]]): Unit = { - val tx: Instance[MeshCreditTx] = Instantiate(new MeshCreditTx(p, linkDepth)); - val rx: Instance[MeshCreditRx] = Instantiate(new MeshCreditRx(p, linkDepth)) - tx.io.active := io.active; rx.io.active := io.active - tx.io.in <> source; rx.io.link <> tx.io.link; sink <> rx.io.out - } - - for { - y <- 0 until p.yNodes - x <- 0 until p.xNodes - } { - val router = routers(y)(x) - for (vc <- 0 until p.virtualChannels) { - val input = io.localIn(index(x, y))(vc) - val output = io.localOut(index(x, y))(vc) - router.io.in(MeshDirection.Local)(vc).valid := io.active && input.valid - router.io.in(MeshDirection.Local)(vc).bits := input.bits - input.ready := io.active && router.io.in(MeshDirection.Local)(vc).ready - output.valid := io.active && router.io.out(MeshDirection.Local)(vc).valid - output.bits := router.io.out(MeshDirection.Local)(vc).bits - router.io.out(MeshDirection.Local)(vc).ready := io.active && output.ready - } - for (vc <- 0 until p.virtualChannels) { - for ( - direction <- Seq(MeshDirection.West).filter(_ => x == 0) ++ Seq(MeshDirection.East).filter(_ => - x == p.xNodes - 1 - ) ++ Seq(MeshDirection.North).filter(_ => y == 0) ++ Seq(MeshDirection.South).filter(_ => - y == p.yNodes - 1 - ) - ) { - router.io.in(direction)(vc).valid := false.B; router.io.in(direction)(vc).bits := 0.U.asTypeOf(flit) - router.io.out(direction)(vc).ready := false.B - } - } - } - for { - y <- 0 until p.yNodes - x <- 0 until p.xNodes - 1 - } { - wireLink(routers(y)(x).io.out(MeshDirection.East), routers(y)(x + 1).io.in(MeshDirection.West)) - wireLink(routers(y)(x + 1).io.out(MeshDirection.West), routers(y)(x).io.in(MeshDirection.East)) - } - for { - y <- 0 until p.yNodes - 1 - x <- 0 until p.xNodes - } { - wireLink(routers(y)(x).io.out(MeshDirection.South), routers(y + 1)(x).io.in(MeshDirection.North)) - wireLink(routers(y + 1)(x).io.out(MeshDirection.North), routers(y)(x).io.in(MeshDirection.South)) - } -} diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/ScratchpadBulkWriteSink.scala b/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/ScratchpadBulkWriteSink.scala deleted file mode 100644 index 3c8c9984e..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/ScratchpadBulkWriteSink.scala +++ /dev/null @@ -1,50 +0,0 @@ -package hier.chip.mesh - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{Instance, Instantiate} -import memcore.bus.axi.Beat -import memcore.memory.bank.{BankSet, BankSetParams} - -/** NoC extension that binds VC5/VC4/VC6 DMA traffic to a local scratchpad. */ -class ScratchpadBulkWriteSink(p: BankSetParams, nodeIdBits: Int) extends Module { - - val io = IO(new Bundle { - val descriptor = Flipped(Decoupled(new BulkDescriptor(nodeIdBits))) - val bulkIn = Flipped(Decoupled(new Beat(p.dataBits))) - val completion = Decoupled(new BulkCompletion(nodeIdBits)) - }) - - val scratchpad: Instance[BankSet] = Instantiate(new BankSet(p)) - val idle :: issue :: streaming :: complete :: Nil = Enum(4) - val state = RegInit(idle) - val descriptor = Reg(new BulkDescriptor(nodeIdBits)) - val beats = descriptor.bytes / p.bytes.U - io.descriptor.ready := state === idle - when(io.descriptor.fire) { - assert( - io.descriptor.bits.write && io.descriptor.bits.bytes =/= 0.U && - io.descriptor.bits.bytes % p.bytes.U === 0.U, - "Scratchpad bulk sink accepts aligned nonempty writes only" - ) - descriptor := io.descriptor.bits - state := issue - } - scratchpad.io.command.valid := state === issue - scratchpad.io.command.bits.write := true.B - scratchpad.io.command.bits.addr := descriptor.scratchpadAddr(p.addressBits - 1, 0) - scratchpad.io.command.bits.beats := beats - when(scratchpad.io.command.fire)(state := streaming) - scratchpad.io.write.valid := state === streaming && io.bulkIn.valid - scratchpad.io.write.bits := io.bulkIn.bits - io.bulkIn.ready := state === streaming && scratchpad.io.write.ready - scratchpad.io.read.ready := true.B - scratchpad.io.done.ready := state === streaming - when(scratchpad.io.done.fire)(state := complete) - io.completion.valid := state === complete - io.completion.bits.targetNode := descriptor.sourceNode - io.completion.bits.txnId := descriptor.txnId - io.completion.bits.ready := false.B - io.completion.bits.error := scratchpad.io.done.bits - when(io.completion.fire)(state := idle) -} diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/TileMeshEndpoint.scala b/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/TileMeshEndpoint.scala deleted file mode 100644 index 6c50290a3..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/TileMeshEndpoint.scala +++ /dev/null @@ -1,95 +0,0 @@ -package hier.chip.mesh - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import memcore.bus.chi._ -import memcore.bus.axi.Beat - -/** - * One requester tile's combined coherent and bulk Mesh endpoint. - * - * CHI uses VC0--VC3 through `ChiMeshRequesterEndpoint`; bulk AXI streams use - * VC4. Each VC has independent flow control; physical links arbitrate flits - * using the receiver credit pool for that VC. - */ -@instantiable -class TileMeshEndpoint( - p: Params, - mesh: MeshParams, - localX: Int, - localY: Int, - nodeMap: ChiMeshNodeMap, - bulkDataBits: Int = 256) - extends Module { - require(mesh.virtualChannels >= 5) - require(mesh.payloadBits >= bulkDataBits + bulkDataBits / 8) - require(nodeMap.coordinateBits == p.nodeIdBits) - - @public - val io = IO(new Bundle { - val chi = Flipped(new RequesterPort(p)) - val bulkIn = Flipped(Decoupled(new MeshBulkBeat(bulkDataBits, p.nodeIdBits))) - val bulkOut = Decoupled(new Beat(bulkDataBits)) - val meshOut = Vec(mesh.virtualChannels, Decoupled(new MeshFlit(mesh))) - val meshIn = Vec(mesh.virtualChannels, Flipped(Decoupled(new MeshFlit(mesh)))) - }) - - val chiEndpoint: Instance[ChiMeshRequesterEndpoint] = - Instantiate(new ChiMeshRequesterEndpoint(p, mesh, localX, localY, nodeMap)) - val bulkTx: Instance[MeshBulkPacketizer] = - Instantiate(new MeshBulkPacketizer(mesh, bulkDataBits, virtualChannel = 4, localX, localY, nodeMap)) - val bulkRx: Instance[MeshBulkDepacketizer] = - Instantiate(new MeshBulkDepacketizer(mesh, bulkDataBits, virtualChannel = 4)) - chiEndpoint.io.chi <> io.chi - bulkTx.io.in <> io.bulkIn - for (vc <- 0 until mesh.virtualChannels) { - if (vc < 4) { - io.meshOut(vc) <> chiEndpoint.io.meshOut(vc) - chiEndpoint.io.meshIn(vc) <> io.meshIn(vc) - } else { - chiEndpoint.io.meshOut(vc).ready := false.B - chiEndpoint.io.meshIn(vc).valid := false.B - chiEndpoint.io.meshIn(vc).bits := 0.U.asTypeOf(new MeshFlit(mesh)) - if (vc == 4) { io.meshOut(vc) <> bulkTx.io.out; bulkRx.io.in <> io.meshIn(vc) } - else { - io.meshOut(vc).valid := false.B; io.meshOut(vc).bits := 0.U.asTypeOf(new MeshFlit(mesh)); - io.meshIn(vc).ready := false.B - } - } - } - io.bulkOut <> bulkRx.io.out -} - -/** Native bulk-path verification target for the combined tile endpoint. */ -class TileMeshBulkLoopback extends Module { - val chi = Params() - val mesh = MeshParams(xNodes = 2, yNodes = 2, payloadBits = 320, virtualChannels = 8) - - val map = ChiMeshNodeMap( - Seq.tabulate(1 << chi.nodeIdBits)(_ & 1), - Seq.tabulate(1 << chi.nodeIdBits)(node => (node >> 1) & 1), - mesh, - presentNodes = (0 until (1 << chi.nodeIdBits)).toSet - ) - - @public - val io = IO(new Bundle { - val in = Flipped(Decoupled(new MeshBulkBeat(256, chi.nodeIdBits))) - val out = Decoupled(new Beat(256)) - }) - - val endpoint: Instance[TileMeshEndpoint] = Instantiate(new TileMeshEndpoint(chi, mesh, 0, 0, map)) - endpoint.io.bulkIn <> io.in - io.out <> endpoint.io.bulkOut - endpoint.io.chi.req.valid := false.B - endpoint.io.chi.req.bits := 0.U.asTypeOf(endpoint.io.chi.req.bits) - endpoint.io.chi.txRsp.valid := false.B - endpoint.io.chi.txRsp.bits := 0.U.asTypeOf(endpoint.io.chi.txRsp.bits) - endpoint.io.chi.txDat.valid := false.B - endpoint.io.chi.txDat.bits := 0.U.asTypeOf(endpoint.io.chi.txDat.bits) - endpoint.io.chi.snp.ready := true.B - endpoint.io.chi.rxRsp.ready := true.B - endpoint.io.chi.rxDat.ready := true.B - endpoint.io.meshIn <> endpoint.io.meshOut -} diff --git a/arch/src/main/scala/framework/root/root-core/src/csrc/Cargo.lock b/arch/src/main/scala/framework/root/root-core/src/csrc/Cargo.lock deleted file mode 100644 index 4a1308c3e..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/csrc/Cargo.lock +++ /dev/null @@ -1,7 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "root-core-reference-model" -version = "0.1.0" diff --git a/arch/src/main/scala/framework/root/root-core/src/csrc/Cargo.toml b/arch/src/main/scala/framework/root/root-core/src/csrc/Cargo.toml deleted file mode 100644 index f9c892cdb..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/csrc/Cargo.toml +++ /dev/null @@ -1,6 +0,0 @@ -[package] -name = "root-core-reference-model" -version = "0.1.0" -edition = "2021" -[lib] -crate-type = ["cdylib"] diff --git a/arch/src/main/scala/framework/root/root-core/src/csrc/README.md b/arch/src/main/scala/framework/root/root-core/src/csrc/README.md deleted file mode 100644 index e69de29bb..000000000 diff --git a/arch/src/main/scala/framework/root/root-core/src/csrc/src/lib.rs b/arch/src/main/scala/framework/root/root-core/src/csrc/src/lib.rs deleted file mode 100644 index 879e3a09c..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/csrc/src/lib.rs +++ /dev/null @@ -1,2 +0,0 @@ -mod memory; -pub use memory::*; diff --git a/arch/src/main/scala/framework/root/root-core/src/csrc/src/memory.rs b/arch/src/main/scala/framework/root/root-core/src/csrc/src/memory.rs deleted file mode 100644 index 61d5cf2de..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/csrc/src/memory.rs +++ /dev/null @@ -1,43 +0,0 @@ -use std::ffi::{c_char, c_void, CStr}; -const BASE: u64 = 0x8000_0000; -const BYTES: usize = 16 * 1024 * 1024; - -#[no_mangle] -pub unsafe extern "C" fn core_ref_create(path: *const c_char) -> *mut c_void { - let path = CStr::from_ptr(path).to_str().unwrap(); - let text = std::fs::read_to_string(path).unwrap(); - let mut memory = vec![0u8; BYTES]; - let mut address = BASE; - for token in text.split_whitespace() { - if let Some(value) = token.strip_prefix('@') { address = u64::from_str_radix(value, 16).unwrap(); } - else { - memory[usize::try_from(address - BASE).unwrap()] = u8::from_str_radix(token, 16).unwrap(); - address += 1; - } - } - Box::into_raw(Box::new(memory)).cast() -} -#[no_mangle] -pub unsafe extern "C" fn core_ref_destroy(model: *mut c_void) { - drop(Box::from_raw(model.cast::>())); -} -#[no_mangle] -pub unsafe extern "C" fn core_ref_read(model: *mut c_void, address: u64, output: *mut u32) { - assert!(address % 64 == 0); - let bytes = &*model.cast::>(); - let start = usize::try_from(address - BASE).unwrap(); - for i in 0..16 { *output.add(i) = u32::from_le_bytes(bytes[start + i*4..start + i*4 + 4].try_into().unwrap()); } -} -#[no_mangle] -pub unsafe extern "C" fn core_ref_write(model: *mut c_void, address: u64, data: *const u32, mask: *const u32) { - assert!(address % 64 == 0); - let bytes = &mut *model.cast::>(); - let start = usize::try_from(address - BASE).unwrap(); - let mask = u64::from(*mask) | (u64::from(*mask.add(1)) << 32); - for i in 0..16 { - let word = (*data.add(i)).to_le_bytes(); - for (b, value) in word.into_iter().enumerate() { - if mask & (1u64 << (i*4+b)) != 0 { bytes[start + i*4 + b] = value; } - } - } -} diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/README.md b/arch/src/main/scala/framework/root/root-core/src/main/resources/README.md deleted file mode 100644 index e69de29bb..000000000 diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission.S b/arch/src/main/scala/framework/root/root-core/src/main/resources/admission.S deleted file mode 100644 index 08870aa0a..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission.S +++ /dev/null @@ -1,84 +0,0 @@ -.option norelax -.section .text.start,"ax",@progbits -.globl _start -_start: - la t0,trap_handler - csrw mtvec,t0 - li s0,0x10000000 - li t0,0x20400000 - csrw pmpaddr0,t0 - li t0,0x0f - csrw pmpcfg0,t0 - li t0,0x8000000000080006 - csrw satp,t0 - li t0,3<<11 - csrc mstatus,t0 - li t0,(1<<11)|(1<<17)|(1<<18)|(1<<19) - csrs mstatus,t0 - li s1,0x400040c0 - li t0,0x1111222233334444 - mv t1,s1 -.globl admit_old_load -admit_old_load: - ld t3,0(s1) -.globl admit_cmd0 -admit_cmd0: - .insn r 0x7b,3,0x11,zero,t0,t1 -.globl admit_young_load -admit_young_load: - ld t4,8(s1) - li t5,0x0123456789abcdef - bne t3,t5,fail - li t5,0x8877665544332211 - bne t4,t5,fail - li t0,(1<<17)|(1<<18)|(1<<19) - csrc mstatus,t0 - li t0,0x20800000 - csrw pmpaddr0,t0 - li t0,0x0b - csrw pmpcfg0,t0 - li t0,0x8000300000080009 - csrw satp,t0 - li t0,0x5555666677778888 - li t1,0x400040c8 -.globl admit_cmd1 -admit_cmd1: - .insn r 0x7b,3,0x12,zero,t0,t1 -.globl admit_fault -admit_fault: - .insn r 0x7b,1,0x13,zero,t0,t1 -.globl admit_after_fault -admit_after_fault: - sd zero,0(s0) -1: j 1b -fail: - li t0,1 - sd t0,0(s0) -2: j 2b -.section .text.trap,"ax",@progbits -.option push -.option norvc -trap_handler: - csrr t0,mcause - li t1,2 - bne t0,t1,fail - csrr t0,mepc - la t1,admit_fault - bne t0,t1,fail - addi t0,t0,4 - csrw mepc,t0 - mret -.option pop -.section .data,"aw",@progbits -.quad 0x0123456789abcdef,0x8877665544332211 -.section .pt.root,"aw",@progbits -.zero 8 -.quad 0x20001c01 -.zero 4080 -.section .pt.mid,"aw",@progbits -.quad 0x20002001 -.zero 4088 -.section .pt.leaf,"aw",@progbits -.zero 32 -.quad 0x200010c7 -.zero 4056 diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission.cm_hier b/arch/src/main/scala/framework/root/root-core/src/main/resources/admission.cm_hier deleted file mode 100644 index b8cd95204..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission.cm_hier +++ /dev/null @@ -1,12 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.tracker -+tree @TOP@.reserve -+tree @TOP@.command -+tree @TOP@.complete -+tree @TOP@.cancelled -+tree @TOP@.info -+tree @TOP@.cancel_request -+tree @TOP@.mem_req -+tree @TOP@.mem_resp -+tree @TOP@.uncached_req -+tree @TOP@.uncached_resp diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission.f b/arch/src/main/scala/framework/root/root-core/src/main/resources/admission.f deleted file mode 100644 index 4bb12da19..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission.f +++ /dev/null @@ -1,10 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/admission_control.sv -@RESOURCES@/admission_package.sv --F @RTL@/AdmissionCore/filelist.f --F @RTL@/AdmissionInterlock/filelist.f -@RESOURCES@/admission_tb.sv diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission.ld b/arch/src/main/scala/framework/root/root-core/src/main/resources/admission.ld deleted file mode 100644 index ea50912c3..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission.ld +++ /dev/null @@ -1,18 +0,0 @@ -OUTPUT_ARCH(riscv) -ENTRY(_start) -SECTIONS -{ - . = 0x80000000; - .text.start : { *(.text.start) } - . = 0x80000200; - .text.trap : { *(.text.trap) } - . = 0x800040c0; - .data : { *(.data) } - . = 0x80006000; - .pt.root : { *(.pt.root) } - . = 0x80007000; - .pt.mid : { *(.pt.mid) } - . = 0x80008000; - .pt.leaf : { *(.pt.leaf) } - /DISCARD/ : { *(.riscv.attributes) *(.comment) } -} diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge.f b/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge.f deleted file mode 100644 index cbf89d5fb..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/admission_bridge_control.sv -@RESOURCES@/admission_bridge_package.sv --F @RTL@/AdmissionBridge/filelist.f -@RESOURCES@/admission_bridge_tb.sv diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge_bad_late_fault.f b/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge_bad_late_fault.f deleted file mode 100644 index 301acda12..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge_bad_late_fault.f +++ /dev/null @@ -1,11 +0,0 @@ -+define+AB_TB=admission_bridge_bad_late_fault_tb -+define+AB_LATE_FAULT -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/admission_bridge_control.sv -@RESOURCES@/admission_bridge_package.sv --F @RTL@/AdmissionBridge/filelist.f -@RESOURCES@/admission_bridge_tb.sv diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge_bad_late_fault.toml b/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge_bad_late_fault.toml deleted file mode 100644 index ccb662d7d..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge_bad_late_fault.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "AdmissionBridge fault arrived after retirement offer" diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge_control.sv b/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge_control.sv deleted file mode 100644 index 151b47d51..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge_control.sv +++ /dev/null @@ -1,17 +0,0 @@ -`include "admission_bridge_config.svh" -interface admission_bridge_control_if ( - input logic clock -); - logic reset, boot_allocation, allocation_valid; - logic fault_valid, unbound_fault_valid; - logic [`AB_FAULT_WIDTH-1:0] fault_bits, unbound_fault_bits; - logic [`AB_ROB_BITS-1:0] allocation_id, lookup_id[3]; - logic [`AB_ROB_ENTRIES-1:0] retired; - logic lookup_valid[3]; - logic [`AB_SNAP_WIDTH-1:0] lookup_bits[3]; - clocking sample @(posedge clock); - default input #1step; - input fault_valid, unbound_fault_valid, fault_bits, unbound_fault_bits; - input reset, allocation_valid, allocation_id, retired, lookup_id, lookup_valid, lookup_bits; - endclocking -endinterface diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge_package.sv b/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge_package.sv deleted file mode 100644 index abe45361e..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge_package.sv +++ /dev/null @@ -1,354 +0,0 @@ -`include "admission_bridge_config.svh" -package admission_bridge_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - `define RF(F) `AB_RET_``F``_OFFSET +: `AB_RET_``F``_WIDTH - `define FF(F) `AB_FAULT_``F``_OFFSET +: `AB_FAULT_``F``_WIDTH - `define SF(F) `AB_SNAP_``F``_OFFSET +: `AB_SNAP_``F``_WIDTH - typedef bit [`AB_SNAP_WIDTH-1:0] snapshot_t; - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual admission_bridge_control_if control; - virtual stream_if #(`AB_SNAP_WIDTH) command; - virtual stream_if #(`AB_RET_WIDTH) retirement; - virtual stream_if #(`AB_NPU_WIDTH) npu; - typedef struct { - snapshot_t snapshot; - int rob; - bit pending; - bit [3:0] error; - bit [63:0] address; - } context_t; - context_t contexts[int]; - int accepted = 0, notices = 0, cancelled = 0, peak = 0, lookup_hits = 0, lookup_misses = 0; - bit late_fault = 0; - int bound_faults = 0, repeated_faults = 0, unbound_faults = 0, error_notices = 0; - int multi_retire = 0, boot_ignored = 0, nonrob = 0, stalls = 0, cycle = 0; - function new(string name, uvm_component parent); - super.new(name, parent); - timeout = 100us; - endfunction - function void verify_contract(bit ok, string message); - if (!ok) `uvm_fatal("ADMISSION_BRIDGE", message) - endfunction - function snapshot_t packet(int tag, int funct, int seed); - snapshot_t value; - // All CSR/PMP/instruction bits participate in independent exact comparisons. - for (int i = 0; i < `AB_SNAP_WIDTH; i++) - value[i] = ((i * 17 + seed * 7) % (seed % 19 + 3)) % 2; - value[`SF(TAG)] = tag; - value[`SF(INSTRUCTION_FUNCT)] = funct; - return value; - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - void'(uvm_config_db#(bit)::get(this, "", "late_fault", late_fault)); - verify_contract(uvm_config_db#(virtual admission_bridge_control_if)::get( - this, "", "control", control), "Missing control"); - verify_contract(uvm_config_db#(virtual stream_if #(`AB_SNAP_WIDTH))::get( - this, "", "command", command), "Missing command"); - verify_contract(uvm_config_db#(virtual stream_if #(`AB_RET_WIDTH))::get( - this, "", "retirement", retirement), "Missing retirement"); - verify_contract(uvm_config_db#(virtual stream_if #(`AB_NPU_WIDTH))::get(this, "", "npu", npu), - "Missing NPU"); - endfunction - task monitor(); - forever begin - @(control.sample); - if (control.sample.reset) begin - cancelled += contexts.num(); - contexts.delete(); - end else begin - int retiring = 0; - cycle++; - foreach (contexts[tag]) - if (contexts[tag].rob >= 0 && control.sample.retired[contexts[tag].rob]) begin - contexts[tag].pending = 1; - retiring++; - end - if (retiring > 1) multi_retire++; - begin - int bound = -1; - foreach (contexts[tag]) - if (contexts[tag].rob == control.sample.fault_bits[`FF(ROB_ID)]) bound = tag; - verify_contract( - control.sample.unbound_fault_valid == (control.sample.fault_valid && bound < 0), - "Unknown/boot fault was lost or misrouted"); - if (control.sample.unbound_fault_valid) begin - verify_contract(control.sample.unbound_fault_bits === control.sample.fault_bits, - "Unbound fault payload changed"); - unbound_faults++; - end - if (control.sample.fault_valid && bound >= 0 && control.sample.fault_bits[ - `FF(ERROR) - ] != 0 && !late_fault) begin - if (contexts[bound].error == 0) begin - contexts[bound].error = control.sample.fault_bits[`FF(ERROR)]; - contexts[bound].address = control.sample.fault_bits[`FF(ADDRESS)]; - bound_faults++; - end else repeated_faults++; - end - end - for (int port = 0; port < 3; port++) begin - int found = -1; - foreach (contexts[tag]) - if (!contexts[tag].pending && contexts[tag].rob == control.sample.lookup_id[port]) begin - verify_contract(found == -1, "Reference duplicate ROB"); - found = tag; - end - verify_contract(control.sample.lookup_valid[port] == (found >= 0), - "Lookup live/retired visibility mismatch"); - if (found >= 0) begin - verify_contract(control.sample.lookup_bits[port] === contexts[found].snapshot, - "Lookup lost snapshot or changed CSR/PMP context"); - lookup_hits++; - end else lookup_misses++; - end - verify_contract( - (command.sample.valid&&command.sample.ready)==(npu.sample.valid&&npu.sample.ready), - "CPU/NPU acceptance not atomic"); - if (command.sample.valid && !command.sample.ready) stalls++; - if (npu.sample.valid) - verify_contract( - npu.sample.bits === command.sample.bits[`AB_INSTRUCTION_OFFSET+:`AB_NPU_WIDTH], - "NPU instruction differs from CPU snapshot"); - if (control.sample.allocation_valid && !(npu.sample.valid && npu.sample.ready)) begin - verify_contract(contexts.num() == 0, "Boot overlapped external contexts"); - boot_ignored++; - end - // Check retirement while stalled as well as when accepted. - if (retirement.sample.valid) begin - int tag = retirement.sample.bits[`RF(SNAPSHOT_TAG)]; - verify_contract(contexts.exists(tag) && contexts[tag].pending, - "Unexpected retirement notice"); - verify_contract( - retirement.sample.bits[`AB_RET_SNAPSHOT_OFFSET+:`AB_SNAP_WIDTH]===contexts[tag].snapshot, - "Retirement changed full snapshot"); - verify_contract(retirement.sample.bits[`RF(ERROR) - ] === contexts[tag].error && retirement.sample.bits[`RF(ADDRESS) - ] === contexts[tag].address, - "Retirement lost first fault/error/full64 address"); - end - if (command.sample.valid && command.sample.ready) begin - int tag = command.sample.bits[`SF(TAG)]; - int funct = command.sample.bits[`SF(INSTRUCTION_FUNCT)]; - int rob = funct <= 1 ? -1 : int'(control.sample.allocation_id); - verify_contract(!contexts.exists(tag) && contexts.num() < `AB_ENTRIES, - "Reused tag/context in retirement acceptance cycle"); - if (rob >= 0) - foreach (contexts[old]) - verify_contract(contexts[old].rob != rob, - "ROB reused before old retirement acknowledgement"); - verify_contract(control.sample.allocation_valid == (rob >= 0), - "Wrong allocation ownership"); - contexts[tag] = '{command.sample.bits, rob, rob < 0, 0, 0}; - accepted++; - if (rob < 0) nonrob++; - if (contexts.num() > peak) peak = contexts.num(); - end - if (retirement.sample.valid && retirement.sample.ready) begin - int tag = retirement.sample.bits[`RF(SNAPSHOT_TAG)]; - if (contexts[tag].error != 0) error_notices++; - contexts.delete(tag); - notices++; - end - end - @(negedge control.clock); - for (int port = 0; port < 3; port++) - control.lookup_id[port] = (cycle + port * 5) % `AB_ROB_ENTRIES; - end - endtask - task submit(int tag, int rob, int funct = 2, int seed = 3); - @(negedge control.clock); - control.allocation_id = rob; - command.bits = packet(tag, funct, seed); - command.valid = 1; - do @(command.sample); while (!command.sample.ready); - @(negedge control.clock); - command.valid = 0; - command.bits = ~command.bits; - endtask - task retire(bit [`AB_ROB_ENTRIES-1:0] mask); - @(negedge control.clock); - control.retired = mask; - @(control.sample); - @(negedge control.clock); - control.retired = 0; - endtask - task fault(int rob, int error, bit [63:0] address, bit [`AB_ROB_ENTRIES-1:0] mask = 0); - @(negedge control.clock); - control.fault_bits = '0; - control.fault_bits[`FF(ROB_ID)] = rob; - control.fault_bits[`FF(ERROR)] = error; - control.fault_bits[`FF(ADDRESS)] = address; - control.fault_valid = 1; - control.retired = mask; - @(control.sample); - @(negedge control.clock); - control.fault_valid = 0; - control.retired = 0; - control.fault_bits = ~control.fault_bits; - endtask - task blocked(int cycles); - repeat (cycles) begin - @(control.sample); - verify_contract(!command.sample.ready && !npu.sample.valid, "Blocked binding reached NPU"); - end - endtask - task drain(); - wait (contexts.num() == 0); - repeat (3) @(control.sample); - verify_contract(!retirement.sample.valid, "Notice remained after all acknowledgements"); - endtask - task execute(); - control.fault_valid = 0; - control.fault_bits = 0; - control.reset = 1; - control.boot_allocation = 0; - control.retired = 0; - control.allocation_id = 0; - for (int i = 0; i < 3; i++) control.lookup_id[i] = 0; - command.valid = 0; - command.bits = 0; - npu.ready = 0; - retirement.ready = 0; - fork - monitor(); - join_none - repeat (4) @(negedge control.clock); - control.reset = 0; - if (late_fault) begin - npu.ready = 1; - submit(86, 8, 2, 9); - retire(16'h100); - wait (retirement.valid); - fault(8, 7, 64'hf123456789abcdef); - repeat (4) @(control.sample); - `uvm_fatal("MISSING_ASSERTION", "Late fault did not trigger DUT ordering assertion") - return; - end - for (int i = 0; i < 3; i++) begin - @(negedge control.clock); - control.boot_allocation = 1; - control.allocation_id = i; - @(control.sample); - end - @(negedge control.clock); - control.boot_allocation = 0; - fault(14, 7, 64'hfedcba9876543210); - // A stable external offer must survive NPU backpressure without reservation. - fork - submit(0, 0, 2, 7); - begin - repeat (5) @(control.sample); - verify_contract(contexts.num() == 0, "NPU stall reserved a context"); - @(negedge control.clock); - npu.ready = 1; - end - join - submit(127, 3, 3, 11); - submit(255, 7, 4, 17); - submit(37, 15, 5, 23); - fault(0, 3, 64'h8123456789abcdef); - fault(0, 4, 64'hfedcba9876543210); - fault(7, 0, 64'hffffffffffffffff); - fault(14, 5, 64'habcdef0123456789); - fork - submit(2, 9, 6, 29); - begin - @(negedge control.clock); - repeat (2) @(control.sample); - blocked(4); - fault(3, 9, 64'hfedcba9876543210, 16'h8089); - wait (retirement.valid); - blocked(4); - @(negedge control.clock); - retirement.ready = 1; - end - join - wait (contexts.num() == 1); - repeat (18) @(control.sample); - retire(16'h200); - drain(); - // A duplicated caller tag remains blocked after ROB retire until notice acceptance. - @(negedge control.clock); - retirement.ready = 0; - submit(42, 5, 8, 31); - fork - submit(42, 6, 9, 37); - begin - repeat (2) @(control.sample); - blocked(4); - retire(16'h20); - wait (retirement.valid); - blocked(4); - @(negedge control.clock); - retirement.ready = 1; - end - join - repeat (18) @(control.sample); - retire(16'h40); - drain(); - // Old pending notices keep their ROB binding even with a different caller tag. - @(negedge control.clock); - retirement.ready = 0; - submit(55, 11, 10, 41); - retire(16'h800); - wait (retirement.valid); - fork - submit(56, 11, 11, 43); - begin - repeat (2) @(control.sample); - blocked(5); - @(negedge control.clock); - retirement.ready = 1; - end - join - repeat (18) @(control.sample); - retire(16'h800); - drain(); - @(negedge control.clock); - retirement.ready = 0; - submit(73, 0, 15, 45); - submit(70, 0, 0, 47); - submit(71, 0, 1, 53); - repeat (5) @(control.sample); - @(negedge control.clock); - retirement.ready = 1; - retire(16'h1); - drain(); - // Coordinated reset cancels both a held notice and another unretired binding. - @(negedge control.clock); - retirement.ready = 0; - submit(80, 2, 12, 59); - submit(81, 4, 13, 61); - retire(16'h4); - wait (retirement.valid); - @(negedge control.clock); - control.reset = 1; - repeat (3) @(negedge control.clock); - control.reset = 0; - retirement.ready = 1; - repeat (18) @(control.sample); - submit(80, 2, 14, 67); - repeat (18) @(control.sample); - retire(16'h4); - drain(); - verify_contract( - accepted==notices+cancelled&&cancelled==2&&peak==4&&boot_ignored==3&&nonrob==2&&multi_retire>0&&lookup_hits>0&&lookup_misses>0&&stalls>0&&bound_faults==2&&repeated_faults==1&&unbound_faults==2&&error_notices==2, - "Missing lifecycle coverage or lost notice"); - `uvm_info( - "ADMISSION_FAULT_PASS", - $sformatf( - "firstFaults=%0d repeatedIgnored=%0d unboundForwarded=%0d errorNotices=%0d; same-cycle fault+retire and full64 addresses checked", - bound_faults, repeated_faults, unbound_faults, error_notices), UVM_LOW) - `uvm_info( - "ADMISSION_BRIDGE_PASS", - $sformatf( - "accepted=%0d notices=%0d resetCancelled=%0d peak=%0d bootIgnored=%0d nonROB=%0d multiRetire=%0d lookup hit/miss=%0d/%0d stalls=%0d; complete snapshot/tag/PMP retained and all contexts drained", - accepted, notices, cancelled, peak, boot_ignored, nonrob, multi_retire, lookup_hits, - lookup_misses, stalls), UVM_LOW) - endtask - endclass -endpackage diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge_tb.sv b/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge_tb.sv deleted file mode 100644 index 93cdff5fb..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge_tb.sv +++ /dev/null @@ -1,43 +0,0 @@ -`include "admission_bridge_config.svh" -`ifndef AB_TB -`define AB_TB admission_bridge_tb -`endif -module `AB_TB; - import uvm_pkg::*; - import admission_bridge_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - admission_bridge_control_if control (clock); - stream_if #(`AB_SNAP_WIDTH) command ( - clock, - control.reset - ); - stream_if #(`AB_RET_WIDTH) retirement ( - clock, - control.reset - ); - stream_if #(`AB_NPU_WIDTH) npu ( - clock, - control.reset - ); - assign control.allocation_valid=control.boot_allocation || - (npu.valid&&npu.ready&&npu.bits[`AB_NPU_FUNCT_OFFSET+:`AB_NPU_FUNCT_WIDTH]>1); - AdmissionBridge dut ( - .clock(clock), - .reset(control.reset), - `include "admission_bridge_ports.svh" - ); - initial begin - uvm_config_db#(virtual admission_bridge_control_if)::set(null, "uvm_test_top", "control", - control); - uvm_config_db#(virtual stream_if #(`AB_SNAP_WIDTH))::set(null, "uvm_test_top", "command", - command); - uvm_config_db#(virtual stream_if #(`AB_RET_WIDTH))::set(null, "uvm_test_top", "retirement", - retirement); - uvm_config_db#(virtual stream_if #(`AB_NPU_WIDTH))::set(null, "uvm_test_top", "npu", npu); -`ifdef AB_LATE_FAULT - uvm_config_db#(bit)::set(null, "uvm_test_top", "late_fault", 1); -`endif - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_control.sv b/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_control.sv deleted file mode 100644 index ee6908e7e..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_control.sv +++ /dev/null @@ -1,13 +0,0 @@ -`include "admission_config.svh" -interface admission_control_if ( - input logic clock -); - logic reset, retired, trapped, cancelled; - logic [63:0] retired_pc, trap_pc, trap_cause, trap_value; - logic [`ADMIT_MEMORY_OUTSTANDING_BITS-1:0] outstanding; - logic [`ADMIT_OUTSTANDING_BITS-1:0] admission_outstanding; - logic [`ADMIT_QUERY_WIDTH-1:0] query; - logic cpu_allow, accelerator_irq; - logic block_maintenance_response, maintenance_response_pending; - bit start = 0, finished = 0, hold_complete = 1; -endinterface diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_package.sv b/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_package.sv deleted file mode 100644 index 9fab23374..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_package.sv +++ /dev/null @@ -1,40 +0,0 @@ -package admission_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - import "DPI-C" function chandle core_ref_create(input string path); - import "DPI-C" function void core_ref_destroy(input chandle model); - import "DPI-C" function void core_ref_read( - input chandle model, - input longint unsigned address, - output bit [511:0] data - ); - import "DPI-C" function void core_ref_write( - input chandle model, - input longint unsigned address, - input bit [511:0] data, - input bit [63:0] mask - ); - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual admission_control_if control; - function new(string name, uvm_component parent); - super.new(name, parent); - timeout = 1ms; - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual admission_control_if)::get(this, "", "control", control)) - `uvm_fatal("VIF", "missing admission interface") - endfunction - task execute(); - @(negedge control.clock); - control.start = 1; - wait (control.finished); - `uvm_info( - "ADMISSION_PASS", - "Actual Rocket/Core reservations, context snapshots, younger request ordering and completion/cancel verified", - UVM_LOW) - endtask - endclass -endpackage diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_tb.sv b/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_tb.sv deleted file mode 100644 index 23b9ca853..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_tb.sv +++ /dev/null @@ -1,605 +0,0 @@ -module admission_tb; - import uvm_pkg::*; - import admission_pkg::*; - `include "uvm_macros.svh" - `include "admission_config.svh" - `include "admission_fixture.svh" - `define AF(B, K, F) B[`ADMIT_``K``_``F``_OFFSET +: `ADMIT_``K``_``F``_WIDTH] - logic clock = 0; - always #5 clock = ~clock; - admission_control_if control (clock); - stream_if #(`ADMIT_PTE_WIDTH) pte_req ( - clock, - control.reset - ); - stream_if #(`ADMIT_PTERESP_WIDTH) pte_resp ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`ADMIT_RESERVE_WIDTH) - ) reserve ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`ADMIT_COMMAND_WIDTH) - ) command ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`ADMIT_COMPLETE_WIDTH) - ) complete ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`ADMIT_CANCELLED_WIDTH) - ) cancelled ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`ADMIT_RESPONSE_WIDTH) - ) response ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`ADMIT_MAINTENANCE_WIDTH) - ) maintenance ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`ADMIT_MAINTAINED_WIDTH) - ) maintained ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`ADMIT_MEMREQ_WIDTH) - ) mem_req ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`ADMIT_MEMRESP_WIDTH) - ) mem_resp ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`ADMIT_UNCACHED_WIDTH) - ) uncached_req ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`ADMIT_URESP_WIDTH) - ) uncached_resp ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`ADMIT_RESERVE_WIDTH) - ) info ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`ADMIT_RESERVE_WIDTH) - ) cancel_request ( - clock, - control.reset - ); - logic tracker_complete_valid, tracker_complete_ready, tracker_cancel_ready; - logic [`ADMIT_COMPLETE_WIDTH-1:0] tracker_complete_tag; - logic maintenance_valid, grant_valid; - AdmissionCore dut ( - .clock(clock), - .reset(control.reset), - .io_resetVector(64'h80000000), - .io_time(64'b0), - .io_timerInterrupt(1'b0), - .io_softwareInterrupt(1'b0), - .io_externalInterrupt(1'b0), - .io_retired(control.retired), - .io_retiredPc(control.retired_pc), - .io_trapped(control.trapped), - .io_trapCause(control.trap_cause), - .io_trapValue(control.trap_value), - .io_trapPc(control.trap_pc), - .io_cancelledData(control.cancelled), - .io_memoryOutstanding(control.outstanding), - .io_blockMaintenanceResponse(control.block_maintenance_response), - .io_maintenanceResponsePending(control.maintenance_response_pending), - `include "admission_ports.svh" - ); - // The first opaque command owns one write range; real maintenance uses this Core and Home. - logic tracker_allow, tracker_maint_ready, tracker_done_ready, tracker_maintained_ready; - logic [7:0] tracker_grant_tag; - logic [7:0] tracker_maint_tag; - logic [43:0] tracker_maint_first, tracker_maint_last; - logic [1:0] tracker_maint_op; - logic dma_done_valid = 0; - logic [7:0] dma_done_tag = 0; - bit use_tracker_maintenance = 0; - assign control.cpu_allow = tracker_allow; - Interlock tracker ( - .clock(clock), - .reset(control.reset), - .io_dispatch_valid(reserve.valid), - .io_dispatch_ready(reserve.ready), - .io_dispatch_bits_id(reserve.bits), - .io_cancel_valid(cancel_request.valid && cancelled.ready), - .io_cancel_ready(tracker_cancel_ready), - .io_cancel_bits_id(cancel_request.bits), - .io_accessInfo_valid(info.valid), - .io_accessInfo_ready(info.ready), - .io_accessInfo_bits_id(info.bits), - .io_accessInfo_bits_hasMemory(1'b1), - .io_accessInfo_bits_base(44'h800040c0), - .io_accessInfo_bits_bytes(45'd64), - .io_accessInfo_bits_write(1'b1), - .io_accessInfo_bits_last(1'b1), - .io_cpuQuery_valid(`AF(control.query, QUERY, VALID)), - .io_cpuQuery_paddr(`AF(control.query, QUERY, PADDR)), - .io_cpuQuery_sizeLog2(`AF(control.query, QUERY, SIZELOG2)), - .io_cpuQuery_write(`AF(control.query, QUERY, WRITE)), - .io_cpuQuery_olderDispatchPending(`AF(control.query, QUERY, OLDERDISPATCHPENDING)), - .io_cpuAllow(tracker_allow), - .io_maintenance_valid(maintenance_valid), - .io_maintenance_bits_tag(tracker_maint_tag), - .io_maintenance_bits_firstLine(tracker_maint_first), - .io_maintenance_bits_lastLine(tracker_maint_last), - .io_maintenance_bits_op(tracker_maint_op), - .io_maintenance_ready(tracker_maint_ready), - .io_maintained_valid(use_tracker_maintenance && maintained.valid), - .io_maintained_ready(tracker_maintained_ready), - .io_maintained_bits_tag(`AF(maintained.bits, MAINTAINED, TAG)), - .io_maintained_bits_ok(`AF(maintained.bits, MAINTAINED, OK)), - .io_grant_valid(grant_valid), - .io_grant_bits_tag(tracker_grant_tag), - .io_grant_ready(1'b1), - .io_done_valid(dma_done_valid), - .io_done_ready(tracker_done_ready), - .io_done_bits_tag(dma_done_tag), - .io_done_bits_ok(1'b1), - .io_complete_valid(tracker_complete_valid), - .io_complete_ready(tracker_complete_ready), - .io_complete_bits_tag(tracker_complete_tag) - ); - assign complete.valid = tracker_complete_valid && !control.hold_complete; - assign complete.bits = tracker_complete_tag; - assign tracker_complete_ready = complete.ready && !control.hold_complete; - assign cancelled.valid = cancel_request.valid && tracker_cancel_ready; - assign cancelled.bits = cancel_request.bits; - assign cancel_request.ready = tracker_cancel_ready && cancelled.ready; - typedef struct { - int id, due; - longint unsigned addr; - bit write; - bit [511:0] data; - bit [63:0] mask; - } memory_entry; - memory_entry pending_memory[$], active_memory; - bit - memory_active = 0, - uncached_active = 0, - model_ready = 0, - old_retired = 0, - young_retired = 0, - exit_seen = 0, - exit_retired = 0; - bit cmo_done = 0; - int cmo_requests = 0, cmo_acks = 0, cmo_drain_waits = 0, cmo_late_rsp_waits = 0; - int - cycle = 0, - reservations = 0, - commands_seen = 0, - traps = 0, - completed = 0, - cancelled_count = 0, - old_pte_waits = 0; - logic [`ADMIT_URESP_WIDTH-1:0] uncached_result; - int uncached_due = 0; - int external_ptes = 0, pte_stalls = 0, grants = 0, dma_done_count = 0, blocked_cmo = 0; - bit live_tags[`ADMIT_ENTRIES]; - logic [`ADMIT_COMMAND_WIDTH-1:0] snapshots[2]; - chandle memory; - function automatic void check(bit condition, string message); - if (!condition) `uvm_fatal("ADMISSION", message) - endfunction - always @(posedge clock) begin - if (model_ready && !control.reset) begin - cycle++; - if (pte_req.valid && pte_req.ready) external_ptes++; - if (pte_resp.valid && !pte_resp.ready) pte_stalls++; - if (grant_valid) begin - check(tracker_grant_tag == `AF(snapshots[0], COMMAND, TAG), "wrong DMA grant tag"); - grants++; - end - if (dma_done_valid && tracker_done_ready) dma_done_count++; - if (maintenance.valid && !maintenance.ready) cmo_drain_waits++; - if (maintenance.valid && maintenance.ready) begin - check( - dut.core.cache.io_olderRequestsDrained && !dut.core.cacheOffer && !dut.core.cacheOwner && - !dut.core.uncachedOffer && !dut.core.uncachedOwner && !dut.core.pteDenied, - "CMO accepted before cache/PTE owners drained"); - if (!use_tracker_maintenance) check(dut.core.lsu.io_idle, "old CPU owner bypassed drain"); - else if (dut.core.state != 0 || !dut.core.lsu.io_idle) begin - check(!control.cpu_allow && `AF(control.query, QUERY, VALID), - "CMO bypassed a nonblocked busy broker"); - blocked_cmo++; - end - check(old_pte_waits > 0, "CMO fixture did not overlap actual page-table read"); - cmo_requests++; - end - if (control.maintenance_response_pending && control.block_maintenance_response) begin - cmo_late_rsp_waits++; - check(!maintained.valid, "CMO acknowledged before delayed real CHI completion"); - end - if (maintained.valid) begin - check(`AF(maintained.bits, MAINTAINED, TAG) == (use_tracker_maintenance ? - `AF(snapshots[0], COMMAND, TAG) : 8'he1) && `AF(maintained.bits, MAINTAINED, OK), - "Actual Core CMO failed or changed tag/status"); - end - if (maintained.valid && maintained.ready) cmo_acks++; - if (!use_tracker_maintenance) - check(!maintenance_valid && !grant_valid, "early tracker maintenance/grant"); - if (control.retired) begin - if (control.retired_pc == `ADMIT_OLD_LOAD_PC) old_retired = 1; - if (control.retired_pc == `ADMIT_YOUNG_LOAD_PC) begin - check(commands_seen >= 1, "young load retired before descriptor delivery"); - young_retired = 1; - end - if (control.retired_pc == `CORE_EXIT_PC) exit_retired = 1; - end - if (control.trapped) begin - check(traps == 0 && control.trap_cause == 2 && control.trap_pc == `ADMIT_FAULT_PC, - "unexpected precise trap"); - traps++; - end - if(dut.core.lsu.io_cpu_req_valid&&dut.core.lsu.io_cpu_req_ready&&dut.core.lsu.io_pc==`ADMIT_YOUNG_LOAD_PC) - check(reservations >= 1, "young EX memory accepted ahead of older RoCC reservation"); - if (reserve.valid && reserve.ready) begin - int tag; - tag = reserve.bits; - check(old_retired && tag < `ADMIT_ENTRIES && !live_tags[tag] && reservations < 2, - "early/duplicate/faulting reservation"); - live_tags[tag] = 1; - reservations++; - `uvm_info("RESERVE", $sformatf("tag%0d pc=%h", tag, dut.core.cpu.io_rocc_cmd_bits_pc), - UVM_LOW) - end - if (command.valid && command.ready) begin - int tag; - tag = `AF(command.bits, COMMAND, TAG); - check(commands_seen < 2 && live_tags[tag], "snapshot has no live reservation"); - snapshots[commands_seen] = command.bits; - commands_seen++; - `uvm_info("COMMAND", $sformatf("tag%0d pc=%h satp=%h priv%0d", tag, - `AF(command.bits, COMMAND, INSTRUCTION_PC), - `AF(command.bits, COMMAND, SATP), `AF(command.bits, COMMAND, - EFFECTIVEPRIVILEGE)), - UVM_LOW) - end - if (complete.valid && complete.ready) begin - check(live_tags[complete.bits], "duplicate completion"); - live_tags[complete.bits] = 0; - completed++; - end - if (cancelled.valid && cancelled.ready) begin - check(live_tags[cancelled.bits], "duplicate cancellation"); - live_tags[cancelled.bits] = 0; - cancelled_count++; - end - if (mem_resp.valid && mem_resp.ready) begin - check(memory_active && `AF(mem_resp.bits, MEMRESP, ID) == active_memory.id, - "DDR response identity changed"); - if (active_memory.write) - core_ref_write(memory, active_memory.addr, active_memory.data, active_memory.mask); - memory_active = 0; - end - if (mem_req.valid && mem_req.ready) begin - memory_entry item; - item.id = `AF(mem_req.bits, MEMREQ, ID); - item.addr = `AF(mem_req.bits, MEMREQ, ADDR); - item.write = `AF(mem_req.bits, MEMREQ, WRITE); - item.data = `AF(mem_req.bits, MEMREQ, DATA); - item.mask = `AF(mem_req.bits, MEMREQ, MASK); - if (!item.write) core_ref_read(memory, item.addr, item.data); - item.due = cycle + (item.addr == 64'h80006000 ? 40 : 9); - pending_memory.push_back(item); - if (item.addr == 64'h80006000) old_pte_waits++; - end - if (uncached_resp.valid && uncached_resp.ready) uncached_active = 0; - if (uncached_req.valid && uncached_req.ready) begin - check(!uncached_active, "uncached owner overlapped"); - uncached_result = '0; - `AF(uncached_result, URESP, TAG) = `AF(uncached_req.bits, UNCACHED, TAG); - uncached_due = cycle + 11; - check(`AF(uncached_req.bits, UNCACHED, WRITE) && `AF(uncached_req.bits, UNCACHED, ADDR) - == 64'h10000000, "unexpected device/PTE side effect"); - check(`AF(uncached_req.bits, UNCACHED, DATA) == 0, "firmware reported admission failure"); - exit_seen = 1; - uncached_active = 1; - end - end - end - always @(negedge clock) begin - if (model_ready && !control.reset) begin - mem_req.ready = cycle % 5 != 0; - if (!memory_active) begin - mem_resp.valid = 0; - for (int i = pending_memory.size() - 1; i >= 0; i--) - if (!memory_active && pending_memory[i].due <= cycle) begin - active_memory = pending_memory[i]; - pending_memory.delete(i); - memory_active = 1; - mem_resp.bits = '0; - `AF(mem_resp.bits, MEMRESP, ID) = active_memory.id; - `AF(mem_resp.bits, MEMRESP, DATA) = active_memory.write ? '0 : active_memory.data; - mem_resp.valid = 1; - end - end - uncached_req.ready = !uncached_active && cycle % 5 != 0; - uncached_resp.valid = uncached_active && cycle >= uncached_due; - uncached_resp.bits = uncached_result; - end - end - task automatic external_pte(input longint unsigned addr, expected, input bit error); - @(negedge clock); - pte_req.bits = '0; - `AF(pte_req.bits, PTE, ADDR) = addr; - pte_req.valid = 1; - do @(posedge clock); while (!pte_req.ready); - @(negedge clock); - pte_req.valid = 0; - wait (pte_resp.valid); - repeat (4) begin - @(posedge clock); - check(pte_resp.valid && `AF(pte_resp.bits, PTERESP, ERROR) == error && - `AF(pte_resp.bits, PTERESP, DATA) == expected, - "external PTE response owner/data/error unstable"); - end - @(negedge clock); - pte_resp.ready = 1; - do @(posedge clock); while (!pte_resp.valid); - @(negedge clock); - pte_resp.ready = 0; - endtask - task automatic check_context(logic [`ADMIT_COMMAND_WIDTH-1:0] packet, bit second); - check(`AF(packet, COMMAND, INSTRUCTION_OPCODE) == 7'h7b && - `AF(packet, COMMAND, INSTRUCTION_FUNCT3) == 3 && `AF(packet, COMMAND, INSTRUCTION_XS1) && - `AF(packet, COMMAND, INSTRUCTION_XS2) && !`AF(packet, COMMAND, INSTRUCTION_XD), - "stock RoCC classification/operands altered"); - check(`AF(packet, COMMAND, INSTRUCTION_PC) == (second ? `ADMIT_CMD1_PC : `ADMIT_CMD0_PC), - "command PC changed"); - check(`AF(packet, COMMAND, INSTRUCTION_RS1DATA) - == (second ? 64'h5555666677778888 : 64'h1111222233334444), "rs1 snapshot changed"); - check(`AF(packet, COMMAND, INSTRUCTION_RS2DATA) == (second ? 64'h400040c8 : 64'h400040c0), - "rs2 snapshot changed"); - check(`AF(packet, COMMAND, SATP) == (second ? 64'h8000300000080009 : 64'h8000000000080006), - "satp snapshot changed"); - check(`AF(packet, COMMAND, EFFECTIVEPRIVILEGE) == 1 && `AF(packet, COMMAND, SUM) - == !second && `AF(packet, COMMAND, MXR) == !second, "privilege/SUM/MXR snapshot changed"); - check(`AF(packet, COMMAND, PMP_0_ADDR) == (second ? 64'h20800000 : 64'h20400000) && - `AF(packet, COMMAND, PMP_0_CFG_A) == 1 && `AF(packet, COMMAND, PMP_0_CFG_R) && - `AF(packet, COMMAND, PMP_0_CFG_W) && `AF(packet, COMMAND, PMP_0_CFG_X) == !second && ! - `AF(packet, COMMAND, PMP_0_CFG_L), "PMP snapshot changed"); - endtask - always @(negedge clock) begin - if (use_tracker_maintenance) begin - maintenance.valid = maintenance_valid; - maintenance.bits = '0; - `AF(maintenance.bits, MAINTENANCE, TAG) = tracker_maint_tag; - `AF(maintenance.bits, MAINTENANCE, FIRSTLINE) = tracker_maint_first; - `AF(maintenance.bits, MAINTENANCE, LASTLINE) = tracker_maint_last; - `AF(maintenance.bits, MAINTENANCE, OP) = tracker_maint_op; - maintained.ready = tracker_maintained_ready; - end - end - always_comb tracker_maint_ready = use_tracker_maintenance && maintenance.ready; - initial begin - uvm_config_db#(virtual admission_control_if)::set(null, "*", "control", control); - run_test("protocol_test"); - end - initial begin - pte_req.valid = 0; - pte_req.bits = '0; - pte_resp.ready = 0; - maintenance.valid = 0; - maintenance.bits = '0; - maintained.ready = 0; - control.block_maintenance_response = 1; - control.reset = 1; - control.accelerator_irq = 0; - command.ready = 0; - response.valid = 0; - response.bits = '0; - info.valid = 0; - info.bits = '0; - cancel_request.valid = 0; - cancel_request.bits = '0; - mem_req.ready = 0; - mem_resp.valid = 0; - mem_resp.bits = '0; - uncached_req.ready = 0; - uncached_resp.valid = 0; - uncached_resp.bits = '0; - foreach (live_tags[i]) live_tags[i] = 0; - wait (control.start); - memory = core_ref_create(`ADMIT_IMAGE); - model_ready = 1; - repeat (4) @(posedge clock); - @(negedge clock); - control.reset = 0; - fork - begin - wait (old_pte_waits > 0); - @(negedge clock); - maintenance.bits = '0; - `AF(maintenance.bits, MAINTENANCE, TAG) = 8'he1; - `AF(maintenance.bits, MAINTENANCE, FIRSTLINE) = 44'h800040c0; - `AF(maintenance.bits, MAINTENANCE, LASTLINE) = 44'h800040c0; - `AF(maintenance.bits, MAINTENANCE, OP) = 0; - maintenance.valid = 1; - do @(posedge clock); while (!maintenance.ready); - @(negedge clock); - maintenance.valid = 0; - wait (control.maintenance_response_pending); - repeat (8) begin - @(posedge clock); - #1; - check(!maintained.valid && reservations == 0, - "Late CMO completion released new admission"); - end - @(negedge clock); - control.block_maintenance_response = 0; - wait (maintained.valid); - begin - logic [`ADMIT_MAINTAINED_WIDTH-1:0] held; - held = maintained.bits; - repeat (8) begin - @(posedge clock); - #1; - check(maintained.valid && maintained.bits === held && reservations == 0, - "Held CMO ACK changed or released new admission"); - end - end - @(negedge clock); - maintained.ready = 1; - do @(posedge clock); while (!maintained.valid); - @(negedge clock); - maintained.ready = 0; - cmo_done = 1; - end - join_none - wait (command.valid); - begin - logic [`ADMIT_COMMAND_WIDTH-1:0] held; - held = command.bits; - check_context(held, 0); - repeat (8) begin - @(posedge clock); - #1; - check(command.valid && command.bits === held && reservations == 1 && !young_retired, - "snapshot/young request escaped admission stall"); - end - end - @(negedge clock); - command.ready = 1; - wait (commands_seen == 1); - @(negedge clock); - command.ready = 0; - // A younger authorized physical query waits behind the unknown reservation. - wait (!control.cpu_allow && `AF(control.query, QUERY, VALID)); - external_pte(64'h80006008, 64'h20001c01, 0); - external_pte(64'ha0000008, 0, 1); - external_pte(64'h10000000, 0, 1); - @(negedge clock); - pte_req.bits = '0; - `AF(pte_req.bits, PTE, ADDR) = 44'h80006008; - pte_req.valid = 1; - do @(posedge clock); while (!pte_req.ready); - @(negedge clock); - pte_req.valid = 0; - wait (pte_resp.valid); - check(!`AF(pte_resp.bits, PTERESP, ERROR) && `AF(pte_resp.bits, PTERESP, DATA) - == 64'h20001c01, "external PTE did not return the coherent cached value"); - use_tracker_maintenance = 1; - info.bits = `AF(snapshots[0], COMMAND, TAG); - info.valid = 1; - do @(posedge clock); while (!info.ready); - @(negedge clock); - info.valid = 0; - wait (maintenance_valid); - repeat (8) begin - @(posedge clock); - #1; - check(pte_resp.valid && !tracker_maint_ready && !young_retired, - "held external PTE did not block maintenance or younger access"); - end - @(negedge clock); - pte_resp.ready = 1; - do @(posedge clock); while (!pte_resp.valid); - @(negedge clock); - pte_resp.ready = 0; - wait (grants == 1); - wait (!control.cpu_allow && - `AF(control.query, QUERY, VALID) - && - `AF(control.query, QUERY, PADDR) - == 44'h800040c8 && dut.core.state != 0); - repeat (8) begin - @(posedge clock); - #1; - check(!young_retired, "young CPU escaped live write range"); - end - @(negedge clock); - control.hold_complete = 0; - dma_done_tag = `AF(snapshots[0], COMMAND, TAG); - dma_done_valid = 1; - do @(posedge clock); while (!tracker_done_ready); - @(negedge clock); - dma_done_valid = 0; - wait (command.valid); - check_context(command.bits, 1); - check(completed == 1, "second command proceeded before real first completion"); - @(negedge clock); - command.ready = 1; - wait (commands_seen == 2); - @(negedge clock); - command.ready = 0; - cancel_request.bits = `AF(snapshots[1], COMMAND, TAG); - cancel_request.valid = 1; - do @(posedge clock); while (!cancel_request.ready); - @(negedge clock); - cancel_request.valid = 0; - control.hold_complete = 0; - wait(cmo_done&&exit_seen&&exit_retired&&completed==1&&cancelled_count==1&&control.admission_outstanding==0&&control.outstanding==0); - repeat (8) @(posedge clock); - check( - reservations == 2 && commands_seen == 2 && traps == 1 && old_pte_waits > 0 && young_retired, - "actual Core admission fixture did not complete required observations"); - check(!memory_active && pending_memory.size() == 0 && !uncached_active, - "memory responses not drained"); - check( - cmo_requests == 3 && cmo_acks == 3 && cmo_drain_waits > 0 && cmo_late_rsp_waits >= 8 && - external_ptes==4 && pte_stalls>=8 && grants==1 && dma_done_count==1 && blocked_cmo>0, - $sformatf( - "CMO requirements req%0d ack%0d oldwait%0d latewait%0d pte%0d stalls%0d grant%0d done%0d blocked%0d", - cmo_requests, - cmo_acks, - cmo_drain_waits, - cmo_late_rsp_waits, - external_ptes, - pte_stalls, - grants, - dma_done_count, - blocked_cmo - )); - `uvm_info("CORE_MAINTENANCE_PASS", $sformatf( - "request=%0d ack=%0d olderOwnerWait=%0d lateCHIWait=%0d; actual Core CMO and held ACK checked", - cmo_requests, - cmo_acks, - cmo_drain_waits, - cmo_late_rsp_waits - ), UVM_LOW) - `uvm_info("CHECKED", $sformatf( - "reserve%0d cmd%0d complete%0d cancel%0d preciseTrap%0d oldPTE%0d", - reservations, - commands_seen, - completed, - cancelled_count, - traps, - old_pte_waits - ), UVM_LOW) - core_ref_destroy(memory); - model_ready = 0; - control.finished = 1; - end -endmodule diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/build_fixture.py b/arch/src/main/scala/framework/root/root-core/src/main/resources/build_fixture.py deleted file mode 100644 index 6c909b30a..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/build_fixture.py +++ /dev/null @@ -1,108 +0,0 @@ -from pathlib import Path -import argparse -import subprocess - -parser = argparse.ArgumentParser() -parser.add_argument("destination") -parser.add_argument( - "fixture", - choices=("core", "supervisor", "multicore", "irq", "admission", "fpu"), - nargs="?", - default="core", -) -options = parser.parse_args() -output = Path(options.destination) -fixture = options.fixture -output.mkdir(parents=True, exist_ok=True) -source = Path(__file__).resolve().parent -subprocess.run( - [ - "riscv64-none-elf-gcc", - ( - "-march=rv64imafdc_zicsr_zifencei" - if fixture == "fpu" - else "-march=rv64imac_zicsr_zifencei" - ), - "-mabi=lp64", - "-nostdlib", - "-nostartfiles", - "-Wl,--build-id=none", - "-Wl,--no-relax", - "-T", - str(source / f"{fixture}.ld"), - str(source / f"{fixture}.S"), - "-o", - str(output / f"{fixture}.elf"), - ], - check=True, -) -subprocess.run( - [ - "riscv64-none-elf-objcopy", - "-O", - "verilog", - str(output / f"{fixture}.elf"), - str(output / f"{fixture}.hex"), - ], - check=True, -) -image = output / f"{fixture}.hex" -image.write_bytes(image.read_bytes().replace(b"\r\n", b"\n")) -disassembly = subprocess.run( - [ - "riscv64-none-elf-objdump", - "-d", - str(output / f"{fixture}.elf"), - ], - check=True, - capture_output=True, - text=True, -).stdout -(output / f"{fixture}.dis").write_text(disassembly) -import re - -exits = re.findall(r"^\s*([0-9a-f]+):[^\n]*\bsd\s+zero,0\(s0\)", disassembly, re.M) -if len(exits) != 1: - raise RuntimeError(f"Expected one successful exit SD in firmware, found {exits}") -definitions = [f"`define CORE_EXIT_PC 64'h{exits[0]}"] -if fixture in ("supervisor", "irq", "admission", "fpu"): - symbols = subprocess.run( - [ - "riscv64-none-elf-nm", - "--defined-only", - str(output / f"{fixture}.elf"), - ], - check=True, - capture_output=True, - text=True, - ).stdout - addresses = { - name: int(address, 16) - for address, _, name in (line.split() for line in symbols.splitlines()) - } -if fixture == "irq": - for name in ("irq_load", "irq_amo", "irq_store"): - definitions.append(f"`define {name.upper()}_PC 64'h{addresses[name]:x}") -if fixture == "fpu": - definitions.append(f"`define FPU_ILLEGAL_PC 64'h{addresses['fpu_illegal']:x}") -if fixture == "admission": - for name, address in sorted(addresses.items()): - if name.startswith("admit_"): - definitions.append(f"`define {name.upper()}_PC 64'h{address:x}") -if fixture == "supervisor": - traps = [ - ("trap_boundary", 1, 0x4000A008), - ("trap_store", 15, 0x40008000), - ("trap_sum", 13, 0x40006000), - ("trap_user_load", 13, 0x40008000), - ("trap_user_ecall", 8, 0), - ("trap_supervisor_ecall", 9, 0), - ] - definitions.append(f"`define SUPERVISOR_TRAP_COUNT {len(traps)}") - for index, (name, cause, value) in enumerate(traps): - definitions += [ - f"`define SUPERVISOR_TRAP_{index}_PC 64'h{addresses[name] - 0x40000000:x}", - f"`define SUPERVISOR_TRAP_{index}_CAUSE 64'd{cause}", - f"`define SUPERVISOR_TRAP_{index}_VALUE 64'h{value:x}", - ] -(output / f"{fixture}_fixture.svh").write_text("\n".join(definitions) + "\n") diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/core.S b/arch/src/main/scala/framework/root/root-core/src/main/resources/core.S deleted file mode 100644 index 5cde0ed34..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/core.S +++ /dev/null @@ -1,203 +0,0 @@ -.option norelax -.section .text.start,"ax",@progbits -.globl _start -_start: - la t0, trap_handler - csrw mtvec, t0 - li s0, 0x10000000 - la s1, data - li s9, -1 - j tests - -.section .text.trap,"ax",@progbits -.option push -.option norvc -trap_handler: - csrr t0, mcause - bne t0, s9, fail - csrr t0, mtval - bne t0, s10, fail - csrr t0, mepc - addi t0, t0, 4 - csrw mepc, t0 - li s9, -1 - mret -.option pop - -.section .text.tests,"ax",@progbits -tests: - li s2, 1 - sd s2, 8(s0) - li t0, 0x8877665544332211 - sd t0, 0(s1) - ld t1, 0(s1) - bne t0, t1, fail - lb t0, 7(s1) - li t1, -120 - bne t0, t1, fail - lbu t0, 7(s1) - li t1, 0x88 - bne t0, t1, fail - lh t0, 6(s1) - li t1, -30601 - bne t0, t1, fail - lhu t0, 6(s1) - li t1, 0x8877 - bne t0, t1, fail - lw t0, 4(s1) - li t1, 0xffffffff88776655 - bne t0, t1, fail - lwu t0, 4(s1) - li t1, 0x88776655 - bne t0, t1, fail - li t0, 0xaa - sb t0, 1(s1) - li t0, 0xbeef - sh t0, 2(s1) - li t0, 0xcafebabe - sw t0, 4(s1) - ld t0, 0(s1) - li t1, 0xcafebabebeefaa11 - bne t0, t1, fail - - li s2, 2 - sd s2, 8(s0) - li t0, 0x7fffffff - sw t0, 0(s1) - li t0, 1 - amoadd.w t1, t0, (s1) - li t2, 0x7fffffff - bne t1, t2, fail - ld t1, 0(s1) - li t2, 0xcafebabe80000000 - bne t1, t2, fail - addi t3, s1, 4 - amoadd.w t1, t0, (t3) - li t2, 0xffffffffcafebabe - bne t1, t2, fail - ld t1, 0(s1) - li t2, 0xcafebabf80000000 - bne t1, t2, fail - li t0, 0x1122334455667788 - amoswap.d t1, t0, (s1) - bne t1, t2, fail - ld t1, 0(s1) - bne t1, t0, fail - - li s2, 3 - sd s2, 8(s0) - li t4, 32 - j lr_word - .balign 512 -lr_word: - lr.w t0, (s1) - addi t0, t0, 1 - sc.w t1, t0, (s1) - beqz t1, word_done - addi t4, t4, -1 - bnez t4, lr_word - j fail -word_done: - ld t0, 0(s1) - li t1, 0x1122334455667789 - bne t0, t1, fail - li t4, 32 - j lr_double - .balign 512 -lr_double: - lr.d t0, (s1) - addi t0, t0, 1 - sc.d t1, t0, (s1) - beqz t1, double_done - addi t4, t4, -1 - bnez t4, lr_double - j fail -double_done: - ld t0, 0(s1) - li t1, 0x112233445566778a - bne t0, t1, fail - - li s2, 4 - sd s2, 8(s0) - li t0, 1 - beqz t0, fail - li t0, 0 - bnez t0, fail - li t0, 0xdeadbeef - beq zero, zero, branch_done - sd t0, 0(s1) -branch_done: - ld t0, 0(s1) - li t1, 0x112233445566778a - bne t0, t1, fail - jal ra, patch_target - li t0, 7 - bne a0, t0, fail - la t0, patch_target - li t1, 0x00900513 - sw t1, 0(t0) - fence.i - jal ra, patch_target - li t0, 9 - bne a0, t0, fail - - li s2, 5 - sd s2, 8(s0) -.option push -.option norvc - li s9, 11 - li s10, 0 - ecall - li t1, -1 - bne s9, t1, fail - li t3, 0x20000000 - li s9, 5 - mv s10, t3 - ld t0, 0(t3) - li t1, -1 - bne s9, t1, fail - li s9, 7 - sd t0, 0(t3) - li t1, -1 - bne s9, t1, fail -.option pop - - li s2, 6 - sd s2, 8(s0) - srli t0, s1, 2 - csrw pmpaddr0, t0 - addi t0, t0, 8 - csrw pmpaddr1, t0 - li t0, 0x8800 - csrw pmpcfg0, t0 -.option push -.option norvc - li s9, 5 - mv s10, s1 - ld t0, 0(s1) - li t1, -1 - bne s9, t1, fail - li s9, 7 - sd t0, 0(s1) - li t1, -1 - bne s9, t1, fail -.option pop - li s2, 7 - sd s2, 8(s0) - sd zero, 0(s0) -1: j 1b - -fail: - ori t0, s2, 0x100 - sd t0, 0(s0) -2: j 2b - -.balign 8 -patch_target: - .word 0x00700513 - ret - -.section .data,"aw",@progbits -.balign 8 -data: - .quad 0 diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/core.cm_hier b/arch/src/main/scala/framework/root/root-core/src/main/resources/core.cm_hier deleted file mode 100644 index 0f4845487..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/core.cm_hier +++ /dev/null @@ -1,5 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.mem_req -+tree @TOP@.mem_resp -+tree @TOP@.uncached_req -+tree @TOP@.uncached_resp diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/core.f b/arch/src/main/scala/framework/root/root-core/src/main/resources/core.f deleted file mode 100644 index 2fb40b16b..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/core.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/core_interface.sv -@RESOURCES@/core_package.sv --F @RTL@/Core/filelist.f -@RESOURCES@/core_tb.sv diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/core.ld b/arch/src/main/scala/framework/root/root-core/src/main/resources/core.ld deleted file mode 100644 index 7b1347647..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/core.ld +++ /dev/null @@ -1,14 +0,0 @@ -OUTPUT_ARCH(riscv) -ENTRY(_start) -SECTIONS -{ - . = 0x80000000; - .text.start : { *(.text.start) } - . = 0x80000200; - .text.trap : { *(.text.trap) } - . = 0x80000400; - .text.tests : { *(.text.tests) } - . = 0x800040c0; - .data : { *(.data) } - /DISCARD/ : { *(.riscv.attributes) *(.comment) } -} diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/core_interface.sv b/arch/src/main/scala/framework/root/root-core/src/main/resources/core_interface.sv deleted file mode 100644 index 0fa5a6b79..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/core_interface.sv +++ /dev/null @@ -1,17 +0,0 @@ -interface core_control_if ( - input logic clock -); - logic reset; - logic timer_irq, software_irq, external_irq; - logic load_waiting, amo_completed, store_completed; - logic [2:0] target_access; - logic cancelled, retired, trapped; - logic [63:0] retired_pc, trap_cause, trap_value, trap_pc; - logic [2:0] outstanding; - clocking cb @(posedge clock); - default input #1step output #0; - output reset, timer_irq, software_irq, external_irq; - input load_waiting, amo_completed, store_completed, target_access; - input cancelled, outstanding, retired, retired_pc, trapped, trap_cause, trap_value, trap_pc; - endclocking -endinterface diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/core_package.sv b/arch/src/main/scala/framework/root/root-core/src/main/resources/core_package.sv deleted file mode 100644 index ae084188b..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/core_package.sv +++ /dev/null @@ -1,34 +0,0 @@ -`include "core_config.svh" -`ifdef IRQ_PROFILE -`define TEST_IMAGE `IRQ_IMAGE -`include "irq_fixture.svh" -`elsif SUPERVISOR_PROFILE -`define TEST_IMAGE `SUPERVISOR_IMAGE -`include "supervisor_fixture.svh" -`elsif FPU_PROFILE -`define TEST_IMAGE `FPU_IMAGE -`include "fpu_fixture.svh" -`else -`define TEST_IMAGE `CORE_IMAGE -`include "core_fixture.svh" -`endif -package core_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - `define CF(kind, field) `CORE_``kind``_``field``_OFFSET +: `CORE_``kind``_``field``_WIDTH - import "DPI-C" function chandle core_ref_create(input string path); - import "DPI-C" function void core_ref_destroy(input chandle model); - import "DPI-C" function void core_ref_read( - input chandle model, - input longint unsigned address, - output bit [511:0] data - ); - import "DPI-C" function void core_ref_write( - input chandle model, - input longint unsigned address, - input bit [511:0] data, - input bit [63:0] mask - ); - `include "core_test.svh" -endpackage diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/core_tb.sv b/arch/src/main/scala/framework/root/root-core/src/main/resources/core_tb.sv deleted file mode 100644 index c95275f47..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/core_tb.sv +++ /dev/null @@ -1,85 +0,0 @@ -`include "core_config.svh" -`ifdef IRQ_PROFILE -`define CORE_TB irq_tb -`define CORE_DUT Irq -`elsif SUPERVISOR_PROFILE -`define CORE_TB supervisor_tb -`define CORE_DUT Supervisor -`elsif FPU_PROFILE -`define CORE_TB fpu_tb -`define CORE_DUT Fpu -`else -`define CORE_TB core_tb -`define CORE_DUT Core -`endif -module `CORE_TB; - import uvm_pkg::*; - import core_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - logic [63:0] time_value = 0; - always @(posedge clock) time_value <= control.reset ? 0 : time_value + 1; - core_control_if control (clock); - stream_if #(`CORE_MEMREQ_WIDTH) mem_req ( - clock, - control.reset - ); - stream_if #(`CORE_MEMRESP_WIDTH) mem_resp ( - clock, - control.reset - ); - stream_if #(`CORE_UNCACHED_WIDTH) uncached_req ( - clock, - control.reset - ); - stream_if #(`CORE_URESP_WIDTH) uncached_resp ( - clock, - control.reset - ); - `CORE_DUT dut ( - .clock(clock), - .reset(control.reset), - .io_resetVector(64'h80000000), - .io_time(time_value), -`ifdef IRQ_PROFILE - .io_timerInterrupt(control.timer_irq), - .io_softwareInterrupt(control.software_irq), - .io_externalInterrupt(control.external_irq), -`else - .io_timerInterrupt(1'b0), - .io_softwareInterrupt(1'b0), - .io_externalInterrupt(1'b0), -`endif - .io_retired(control.retired), - .io_retiredPc(control.retired_pc), - .io_trapped(control.trapped), - .io_trapCause(control.trap_cause), - .io_trapValue(control.trap_value), - .io_trapPc(control.trap_pc), - .io_cancelledData(control.cancelled), - .io_memoryOutstanding(control.outstanding), - `include "core_ports.svh" - ); -`ifdef IRQ_PROFILE - `include "irq_fixture.svh" - wire demand_fire = dut.core.cache.io_access_valid && dut.core.cache.io_access_ready; - assign control.target_access[0] = demand_fire && dut.core.cache.io_access_bits_addr == 44'h800040c0 && !dut.core.cache.io_access_bits_write && dut.core.cache.io_access_bits_atomic == 0; - assign control.target_access[1] = demand_fire && dut.core.cache.io_access_bits_addr == 44'h800040c8 && dut.core.cache.io_access_bits_atomic == 2; - assign control.target_access[2] = demand_fire && dut.core.cache.io_access_bits_addr == 44'h800040d0 && dut.core.cache.io_access_bits_write && dut.core.cache.io_access_bits_atomic == 0; - assign control.load_waiting = dut.core.lsu.io_response_ready && dut.core.lsu.instructionPc == `IRQ_LOAD_PC; - assign control.amo_completed = dut.core.lsu.io_cpu_req_ready && !dut.core.lsu.io_idle && dut.core.lsu.instructionPc == `IRQ_AMO_PC; - assign control.store_completed = dut.core.lsu.io_cpu_req_ready && !dut.core.lsu.io_idle && dut.core.lsu.instructionPc == `IRQ_STORE_PC; -`endif - initial begin - uvm_config_db#(virtual core_control_if)::set(null, "uvm_test_top*", "control", control); - uvm_config_db#(virtual stream_if #(`CORE_MEMREQ_WIDTH))::set(null, "uvm_test_top*", "mem_req", - mem_req); - uvm_config_db#(virtual stream_if #(`CORE_MEMRESP_WIDTH))::set(null, "uvm_test_top*", "mem_resp", - mem_resp); - uvm_config_db#(virtual stream_if #(`CORE_UNCACHED_WIDTH))::set(null, "uvm_test_top*", - "uncached_req", uncached_req); - uvm_config_db#(virtual stream_if #(`CORE_URESP_WIDTH))::set(null, "uvm_test_top*", - "uncached_resp", uncached_resp); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/core_test.svh b/arch/src/main/scala/framework/root/root-core/src/main/resources/core_test.svh deleted file mode 100644 index abf0426c5..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/core_test.svh +++ /dev/null @@ -1,321 +0,0 @@ -class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual core_control_if control; - virtual stream_if #(`CORE_MEMREQ_WIDTH) mem_req; - virtual stream_if #(`CORE_MEMRESP_WIDTH) mem_resp; - virtual stream_if #(`CORE_UNCACHED_WIDTH) uncached_req; - virtual stream_if #(`CORE_URESP_WIDTH) uncached_resp; - chandle model; - bit [`CORE_MEMRESP_WIDTH-1:0] memory_queue[$]; - bit [`CORE_URESP_WIDTH-1:0] mmio_queue[$]; - bit memory_active = 0, mmio_active = 0, exited = 0; - int stage = 0, cycles = 0, reads = 0, writes = 0, cancellations = 0; - longint unsigned exit_code; -`ifdef IRQ_PROFILE - localparam int STAGES = 3; - localparam int TRAPS = 3; - bit [2:0] raised = 0, target_retired = 0; - int acknowledgements = 0, memory_release = 0; - int target_retire_count[3] = '{default: 0}; - int target_access_count[3] = '{default: 0}; -`elsif SUPERVISOR_PROFILE - localparam int STAGES = 3; - localparam int TRAPS = `SUPERVISOR_TRAP_COUNT; -`elsif FPU_PROFILE - localparam int STAGES = 4; - localparam int TRAPS = 1; -`else - localparam int STAGES = 7; - localparam int TRAPS = 5; -`endif - int traps = 0, retired = 0; - bit exit_retired = 0; - longint unsigned exit_pc; - function new(string name, uvm_component parent); - super.new(name, parent); - timeout = 5ms; - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual core_control_if)::get( - this, "", "control", control - ) || !uvm_config_db#(virtual stream_if #(`CORE_MEMREQ_WIDTH))::get( - this, "", "mem_req", mem_req - ) || !uvm_config_db#(virtual stream_if #(`CORE_MEMRESP_WIDTH))::get( - this, "", "mem_resp", mem_resp - ) || !uvm_config_db#(virtual stream_if #(`CORE_UNCACHED_WIDTH))::get( - this, "", "uncached_req", uncached_req - ) || !uvm_config_db#(virtual stream_if #(`CORE_URESP_WIDTH))::get( - this, "", "uncached_resp", uncached_resp - )) - `uvm_fatal("VIF", "Core verification interfaces missing") - model = core_ref_create(`TEST_IMAGE); - endfunction - task drive_memory(); - forever begin - @(posedge control.clock); - if (control.reset) begin - mem_req.ready <= 0; - uncached_req.ready <= 0; - mem_resp.valid <= 0; - mem_resp.bits <= '0; - uncached_resp.valid <= 0; - uncached_resp.bits <= '0; - memory_queue.delete(); - mmio_queue.delete(); - memory_active = 0; - mmio_active = 0; - end else begin - cycles++; -`ifdef IRQ_PROFILE - foreach (target_access_count[i]) if (control.target_access[i]) target_access_count[i]++; -`endif - if (control.cancelled) cancellations++; - if (control.retired) begin - retired++; -`ifdef IRQ_PROFILE - if (control.retired_pc == `IRQ_LOAD_PC) begin - target_retired[0] = 1; - target_retire_count[0]++; - end - if (control.retired_pc == `IRQ_AMO_PC) begin - target_retired[1] = 1; - target_retire_count[1]++; - end - if (control.retired_pc == `IRQ_STORE_PC) begin - target_retired[2] = 1; - target_retire_count[2]++; - end -`endif - if (exited && control.retired_pc == exit_pc) exit_retired = 1; - end - if (control.trapped) begin - int expected_cause = traps == 0 ? 11 : (traps % 2 == 1 ? 5 : 7); - longint unsigned expected_value = traps == 0 ? 0 : (traps <= 2 ? 64'h20000000 : 64'h800040c0); -`ifdef IRQ_PROFILE - expected_cause = traps == 0 ? 7 : (traps == 1 ? 3 : 11); - expected_value = 0; - if (traps >= 3 || !raised[traps] || !target_retired[traps] || - control.trap_cause != (64'h8000000000000000 | expected_cause) || control.trap_value != 0) - `uvm_fatal("IRQ", - "Interrupt preceded original retirement or had an unexpected cause/value") -`elsif SUPERVISOR_PROFILE - longint unsigned expected_pc; - case (traps) - 0: begin - expected_cause = `SUPERVISOR_TRAP_0_CAUSE; - expected_value = `SUPERVISOR_TRAP_0_VALUE; - expected_pc = `SUPERVISOR_TRAP_0_PC; - end - 1: begin - expected_cause = `SUPERVISOR_TRAP_1_CAUSE; - expected_value = `SUPERVISOR_TRAP_1_VALUE; - expected_pc = `SUPERVISOR_TRAP_1_PC; - end - 2: begin - expected_cause = `SUPERVISOR_TRAP_2_CAUSE; - expected_value = `SUPERVISOR_TRAP_2_VALUE; - expected_pc = `SUPERVISOR_TRAP_2_PC; - end - 3: begin - expected_cause = `SUPERVISOR_TRAP_3_CAUSE; - expected_value = `SUPERVISOR_TRAP_3_VALUE; - expected_pc = `SUPERVISOR_TRAP_3_PC; - end - 4: begin - expected_cause = `SUPERVISOR_TRAP_4_CAUSE; - expected_value = `SUPERVISOR_TRAP_4_VALUE; - expected_pc = `SUPERVISOR_TRAP_4_PC; - end - 5: begin - expected_cause = `SUPERVISOR_TRAP_5_CAUSE; - expected_value = `SUPERVISOR_TRAP_5_VALUE; - expected_pc = `SUPERVISOR_TRAP_5_PC; - end - endcase - if (control.trap_pc != expected_pc) - `uvm_fatal("TRAP_PC", "Supervisor trap PC differs from fixture symbol") -`elsif FPU_PROFILE - expected_cause = 2; - expected_value = 64'h53; - if (control.trap_pc != `FPU_ILLEGAL_PC) - `uvm_fatal("TRAP_PC", "FPU FS-off trap PC differs from fixture symbol") -`endif -`ifndef IRQ_PROFILE - if (traps >= TRAPS || control.trap_cause != expected_cause || control.trap_value != expected_value) - `uvm_fatal("TRAP", $sformatf( - "Unexpected precise trap count=%0d cause=%h value=%h pc=%h", - traps, - control.trap_cause, - control.trap_value, - control.trap_pc - )) -`endif - traps++; - end -`ifdef IRQ_PROFILE - if (!raised[1] && acknowledgements == 1 && control.amo_completed) begin - control.software_irq <= 1; - raised[1] = 1; - `uvm_info("IRQ_RAISE", "Software level asserted while AMO result was buffered", UVM_LOW) - end - if (!raised[2] && acknowledgements == 2 && control.store_completed) begin - control.external_irq <= 1; - raised[2] = 1; - `uvm_info("IRQ_RAISE", "External level asserted while store completion was buffered", - UVM_LOW) - end -`endif - mem_req.ready <= cycles % 7 > 1; - uncached_req.ready <= cycles % 7 > 1; - if (mem_req.valid && mem_req.ready) begin - bit [`CORE_MEMRESP_WIDTH-1:0] result = '0; - bit [511:0] data; - longint unsigned address = mem_req.bits[`CF(MEMREQ, ADDR)]; -`ifdef IRQ_PROFILE - if (!raised[0] && address == 64'h800040c0 && !mem_req.bits[`CF(MEMREQ, WRITE)]) begin - if (!control.load_waiting) - `uvm_fatal("IRQ_TIMING", "Cold data DDR request did not belong to receiving load") - control.timer_irq <= 1; - raised[0] = 1; - memory_release = cycles + 20; - `uvm_info("IRQ_RAISE", - "Timer level asserted with cold-load DDR response held for 20 cycles", - UVM_LOW) - end -`endif - result[`CF(MEMRESP, ID)] = mem_req.bits[`CF(MEMREQ, ID)]; - if (mem_req.bits[`CF(MEMREQ, WRITE)]) begin - core_ref_write(model, address, mem_req.bits[`CF(MEMREQ, DATA)], mem_req.bits[ - `CF(MEMREQ, MASK)]); - writes++; - end else begin - core_ref_read(model, address, data); - result[`CF(MEMRESP, DATA)] = data; - reads++; - end - memory_queue.push_back(result); - end - if (uncached_req.valid && uncached_req.ready) begin - bit [`CORE_URESP_WIDTH-1:0] result = '0; - longint unsigned address = uncached_req.bits[`CF(UNCACHED, ADDR)]; - longint unsigned value = uncached_req.bits[`CF(UNCACHED, DATA)]; - bit bad_mmio, mmio_handled = 0; - result[`CF(URESP, TAG)] = uncached_req.bits[`CF(UNCACHED, TAG)]; - bad_mmio = !uncached_req.bits[ - `CF(UNCACHED, WRITE) - ] || uncached_req.bits[ - `CF(UNCACHED, SIZE) - ] != 3 || (address != 64'h10000000 && address != 64'h10000008); -`ifdef IRQ_PROFILE - if (uncached_req.bits[ - `CF(UNCACHED, WRITE) - ] && uncached_req.bits[ - `CF(UNCACHED, SIZE) - ] == 3 && address == 64'h10000010) - bad_mmio = 0; -`endif - if (bad_mmio) `uvm_fatal("MMIO", "Unexpected or expanded Core MMIO operation") -`ifdef IRQ_PROFILE - if (address == 64'h10000010) begin - longint unsigned expected = acknowledgements == 0 ? 64'h80 : (acknowledgements == 1 ? 64'h8 : 64'h800); - if (acknowledgements >= 3 || traps != acknowledgements + 1 || !raised[acknowledgements] || value != expected) - `uvm_fatal("IRQ_ACK", "IRQ ACK did not match its pending level/cause") - case (acknowledgements) - 0: control.timer_irq <= 0; - 1: control.software_irq <= 0; - 2: control.external_irq <= 0; - endcase - acknowledgements++; - mmio_handled = 1; - end -`endif - if (!mmio_handled && address == 64'h10000008) begin - if (value != stage + 1 || value > STAGES) - `uvm_fatal("STAGE", "Firmware stage signature out of order") - stage = value; - `uvm_info("STAGE", $sformatf("Completed firmware stage %0d", stage), UVM_LOW) - end else if (!mmio_handled) begin - exit_code = value; - // The final exit SD has rd/tag zero; observe its later precise architectural retirement. - exit_pc = `CORE_EXIT_PC; - if (value != 0 || stage != STAGES) - `uvm_fatal("FIRMWARE", $sformatf("Firmware failed: stage=%0d exit=%h", stage, value)) - exited = 1; - end - mmio_queue.push_back(result); - end - if (memory_active && mem_resp.ready) memory_active = 0; - if (mmio_active && uncached_resp.ready) mmio_active = 0; - begin - bit release_mem; - release_mem = !memory_active && memory_queue.size() != 0 && cycles % 5 == 0; -`ifdef IRQ_PROFILE - release_mem = release_mem && cycles >= memory_release; -`endif - if (release_mem) begin - mem_resp.bits <= memory_queue.pop_front(); - memory_active = 1; - end - end - if (!mmio_active && mmio_queue.size() != 0 && cycles % 5 == 0) begin - uncached_resp.bits <= mmio_queue.pop_front(); - mmio_active = 1; - end - mem_resp.valid <= memory_active; - uncached_resp.valid <= mmio_active; - end - end - endtask - task execute(); - control.reset = 1; - control.timer_irq = 0; - control.software_irq = 0; - control.external_irq = 0; - mem_req.ready = 0; - uncached_req.ready = 0; - mem_resp.valid = 0; - mem_resp.bits = '0; - uncached_resp.valid = 0; - uncached_resp.bits = '0; - repeat (5) @(control.cb); - control.cb.reset <= 0; - fork - drive_memory(); - join_none - wait (exited && exit_retired); - if (traps != TRAPS || retired < 100) - `uvm_fatal("COMPLETION", "Firmware ended without expected architectural execution/traps") -`ifdef IRQ_PROFILE - if (raised != 3'b111 || target_retired != 3'b111 || acknowledgements != 3 || - control.timer_irq || control.software_irq || control.external_irq) - `uvm_fatal("IRQ_COMPLETION", - "IRQ gate finished without all held levels, retirements and acknowledgements") - foreach (target_access_count[i]) - if (target_access_count[i] != 1 || target_retire_count[i] != 1) - `uvm_fatal("IRQ_ONCE", $sformatf( - "Target %0d executed %0d cache demands and retired %0d times", - i, - target_access_count[i], - target_retire_count[i] - )) - `uvm_info("IRQ_CHECKED", - "All three deferred targets executed and retired exactly once; three held-level ACKs drained", - UVM_LOW) -`endif - do - @(control.cb); - while (memory_queue.size() != 0 || mmio_queue.size() != 0 || memory_active || mmio_active); - `uvm_info( - "CORE", - $sformatf( - "Firmware checked %0d stages, exit=0; cycles=%0d memory reads=%0d writes=%0d S1/S2 cancellations=%0d precise traps=%0d retired=%0d", - STAGES, cycles, reads, writes, cancellations, traps, retired), UVM_LOW) - control.cb.reset <= 1; - repeat (3) @(control.cb); - endtask - function void final_phase(uvm_phase phase); - core_ref_destroy(model); - super.final_phase(phase); - endfunction -endclass diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/fpu.S b/arch/src/main/scala/framework/root/root-core/src/main/resources/fpu.S deleted file mode 100644 index f2a7e20a8..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/fpu.S +++ /dev/null @@ -1,148 +0,0 @@ -.option norelax -.section .text.start,"ax",@progbits -.globl _start -_start: - la t0,trap_handler - csrw mtvec,t0 - li s0,0x10000000 - la s1,data - li s9,-1 - li t0,0x6000 - csrs mstatus,t0 - csrw fcsr,zero - j tests -.section .text.trap,"ax",@progbits -.option push -.option norvc -trap_handler: - csrr t0,mcause - bne t0,s9,fail - csrr t0,mtval - li t1,0x53 - bne t0,t1,fail - csrr t0,mepc - addi t0,t0,4 - csrw mepc,t0 - li s9,-1 - mret -.option pop -.section .text.tests,"ax",@progbits -tests: - li t0,1 - sd t0,8(s0) - flw ft0,0(s1) - fmv.x.w t0,ft0 - li t1,0x3fc00000 - bne t0,t1,fail - fld ft4,8(s1) - fmv.x.d t0,ft4 - li t1,0x3ff8000000000000 - bne t0,t1,fail - flw ft1,4(s1) - fld ft5,16(s1) - fsw ft0,32(s1) - fsd ft4,40(s1) - lw t0,32(s1) - li t1,0x3fc00000 - bne t0,t1,fail - ld t0,40(s1) - li t1,0x3ff8000000000000 - bne t0,t1,fail - li t0,2 - sd t0,8(s0) - fadd.s ft2,ft0,ft1 - fmul.s ft2,ft2,ft0 - fsw ft2,36(s1) - lw t0,36(s1) - li t1,0x40b40000 - bne t0,t1,fail - li s2,16 -fp_dependencies: - fmul.d ft6,ft4,ft5 - fadd.d ft6,ft6,ft4 - fsub.d ft6,ft6,ft5 - fdiv.d ft6,ft6,ft4 - fsd ft6,48(s1) - fld ft7,48(s1) - fmv.x.d t0,ft7 - li t1,0x3ffc000000000000 - bne t0,t1,fail - addi s2,s2,-1 - bnez s2,fp_dependencies - li t0,3 - sd t0,8(s0) - fsflags zero - li t0,0x3f800000 - fmv.w.x ft0,t0 - li t0,0x40400000 - fmv.w.x ft1,t0 - fdiv.s ft2,ft0,ft1 - fmv.x.w t0,ft2 - li t1,0x3eaaaaab - bne t0,t1,fail - frflags t0 - li t1,1 - bne t0,t1,fail - fsflags zero - li t0,0x3ff0000000000000 - fmv.d.x ft4,t0 - fmv.d.x ft5,zero - fdiv.d ft6,ft4,ft5 - fmv.x.d t0,ft6 - li t1,0x7ff0000000000000 - bne t0,t1,fail - frflags t0 - li t1,8 - bne t0,t1,fail - fsflags zero - fmv.w.x ft0,zero - fdiv.s ft1,ft0,ft0 - fmv.x.w t0,ft1 - li t1,0x7fc00000 - bne t0,t1,fail - frflags t0 - li t1,16 - bne t0,t1,fail - fsflags zero - li t0,1 - fsrm t0 - li t0,0x3fc00000 - fmv.w.x ft0,t0 - fcvt.w.s t0,ft0,dyn - li t1,1 - bne t0,t1,fail - frflags t0 - li t1,1 - bne t0,t1,fail - fsrm zero - li t0,4 - sd t0,8(s0) - li s9,2 - li t0,0x6000 - csrc mstatus,t0 -.globl fpu_illegal -.option push -.option norvc -fpu_illegal: - .word 0x00000053 -.option pop - li t0,0x6000 - csrs mstatus,t0 - li t0,0x3f800000 - fmv.w.x ft0,t0 - fadd.s ft1,ft0,ft0 - fmv.x.w t0,ft1 - li t1,0x40000000 - bne t0,t1,fail - sd zero,0(s0) -1:j 1b -fail: - li t0,1 - sd t0,0(s0) -1:j 1b -.section .data,"aw",@progbits -.balign 64 -data: - .word 0x3fc00000,0x40100000 - .dword 0x3ff8000000000000,0x4002000000000000 - .zero 40 diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/fpu.f b/arch/src/main/scala/framework/root/root-core/src/main/resources/fpu.f deleted file mode 100644 index beb8e9085..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/fpu.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+FPU_PROFILE -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/core_interface.sv -@RESOURCES@/core_package.sv --F @RTL@/Fpu/filelist.f -@RESOURCES@/core_tb.sv diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/fpu.ld b/arch/src/main/scala/framework/root/root-core/src/main/resources/fpu.ld deleted file mode 100644 index 7b1347647..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/fpu.ld +++ /dev/null @@ -1,14 +0,0 @@ -OUTPUT_ARCH(riscv) -ENTRY(_start) -SECTIONS -{ - . = 0x80000000; - .text.start : { *(.text.start) } - . = 0x80000200; - .text.trap : { *(.text.trap) } - . = 0x80000400; - .text.tests : { *(.text.tests) } - . = 0x800040c0; - .data : { *(.data) } - /DISCARD/ : { *(.riscv.attributes) *(.comment) } -} diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/irq.S b/arch/src/main/scala/framework/root/root-core/src/main/resources/irq.S deleted file mode 100644 index 6a79ac32a..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/irq.S +++ /dev/null @@ -1,97 +0,0 @@ -.option norelax -.option norvc -.section .text.start,"ax",@progbits -.globl _start -_start: - li s0, 0x10000000 - la s1, data - li s2, 0 - li s3, 0 - la t0, trap_handler - csrw mtvec, t0 - li s7, 7 - li t0, 0x80 - csrw mie, t0 - li t0, 8 - csrs mstatus, t0 - j tests - -.section .text.trap,"ax",@progbits -trap_handler: - // t4/t5/t6 are reserved to this handler, so interrupted operands survive. - csrr t4, mcause - li t5, 0x8000000000000000 - or t5, t5, s7 - bne t4, t5, fail - csrr t4, mtval - bnez t4, fail - li t5, 1 - sll t5, t5, s7 - csrr t6, mip - and t6, t6, t5 - beqz t6, fail - ld t4, 64(s1) - addi t4, t4, 1 - sd t4, 64(s1) - sd t5, 16(s0) - addi s3, s3, 1 - mret - -.section .text.tests,"ax",@progbits -tests: -irq_load: - ld s8, 0(s1) - li t0, 0x8877665544332211 - bne s8, t0, fail -timer_wait: - li t0, 1 - bne s3, t0, timer_wait - li s2, 1 - sd s2, 8(s0) - li s7, 3 - li t0, 8 - csrw mie, t0 - addi t2, s1, 8 - li t0, 1 -irq_amo: - amoadd.d.aqrl s8, t0, (t2) - bnez s8, fail - ld t1, 8(s1) - bne t1, t0, fail -software_wait: - li t0, 2 - bne s3, t0, software_wait - li s2, 2 - sd s2, 8(s0) - li s7, 11 - li t0, 0x800 - csrw mie, t0 - li t0, 0x55 -irq_store: - sd t0, 16(s1) -external_wait: - li t0, 3 - bne s3, t0, external_wait - ld t0, 16(s1) - li t1, 0x55 - bne t0, t1, fail - ld t0, 64(s1) - li t1, 3 - bne t0, t1, fail - ld t0, 8(s1) - li t1, 1 - bne t0, t1, fail - li s2, 3 - sd s2, 8(s0) - sd zero, 0(s0) -1: j 1b -fail: - ori t0, s2, 0x100 - sd t0, 0(s0) -2: j 2b - -.section .data,"aw",@progbits -data: - .dword 0x8877665544332211 - .dword 0, 0, 0, 0, 0, 0, 0 - .dword 0 diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/irq.cm_hier b/arch/src/main/scala/framework/root/root-core/src/main/resources/irq.cm_hier deleted file mode 100644 index 0f4845487..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/irq.cm_hier +++ /dev/null @@ -1,5 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.mem_req -+tree @TOP@.mem_resp -+tree @TOP@.uncached_req -+tree @TOP@.uncached_resp diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/irq.f b/arch/src/main/scala/framework/root/root-core/src/main/resources/irq.f deleted file mode 100644 index b9b3967a8..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/irq.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+IRQ_PROFILE -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/core_interface.sv -@RESOURCES@/core_package.sv --F @RTL@/Irq/filelist.f -@RESOURCES@/core_tb.sv diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/irq.ld b/arch/src/main/scala/framework/root/root-core/src/main/resources/irq.ld deleted file mode 100644 index 7b1347647..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/irq.ld +++ /dev/null @@ -1,14 +0,0 @@ -OUTPUT_ARCH(riscv) -ENTRY(_start) -SECTIONS -{ - . = 0x80000000; - .text.start : { *(.text.start) } - . = 0x80000200; - .text.trap : { *(.text.trap) } - . = 0x80000400; - .text.tests : { *(.text.tests) } - . = 0x800040c0; - .data : { *(.data) } - /DISCARD/ : { *(.riscv.attributes) *(.comment) } -} diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/lsu.f b/arch/src/main/scala/framework/root/root-core/src/main/resources/lsu.f deleted file mode 100644 index ccd73eebf..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/lsu.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/lsu_interface.sv -@RESOURCES@/lsu_package.sv --F @RTL@/Lsu/filelist.f -@RESOURCES@/lsu_tb.sv diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/lsu_interface.sv b/arch/src/main/scala/framework/root/root-core/src/main/resources/lsu_interface.sv deleted file mode 100644 index 11210ed35..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/lsu_interface.sv +++ /dev/null @@ -1,14 +0,0 @@ -interface lsu_control_if ( - input logic clock -); - logic reset, s1_kill, s2_kill, cancel_offer; - logic [45:0] pc; - logic [63:0] s1_data; - logic nack, ordered, idle, cancelled; - logic ma_ld, ma_st, pf_ld, pf_st, ae_ld, ae_st; - clocking cb @(posedge clock); - default input #1step output #0; - output reset, s1_kill, s2_kill, pc, s1_data, cancel_offer; - input nack, ordered, idle, cancelled, ma_ld, ma_st, pf_ld, pf_st, ae_ld, ae_st; - endclocking -endinterface diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/lsu_package.sv b/arch/src/main/scala/framework/root/root-core/src/main/resources/lsu_package.sv deleted file mode 100644 index 460760b35..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/lsu_package.sv +++ /dev/null @@ -1,8 +0,0 @@ -`include "lsu_config.svh" -package lsu_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - `define LF(kind, field) `CORE_``kind``_``field``_OFFSET +: `CORE_``kind``_``field``_WIDTH - `include "lsu_test.svh" -endpackage diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/lsu_tb.sv b/arch/src/main/scala/framework/root/root-core/src/main/resources/lsu_tb.sv deleted file mode 100644 index c7afb3d4c..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/lsu_tb.sv +++ /dev/null @@ -1,59 +0,0 @@ -`include "lsu_config.svh" -module lsu_tb; - import uvm_pkg::*; - import lsu_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - lsu_control_if control (clock); - // EX structural replay may withdraw an unaccepted HellaCache offer; backend streams remain irrevocable. - stream_if #(`CORE_CPU_WIDTH) source_if ( - clock, - control.reset || control.cancel_offer - ); - stream_if #(`CORE_RETURN_WIDTH) sink_if ( - clock, - control.reset - ); - stream_if #(`CORE_VIRTUAL_WIDTH) virtual_req ( - clock, - control.reset - ); - stream_if #(`CORE_RESULT_WIDTH) virtual_resp ( - clock, - control.reset - ); - assign sink_if.ready = 1; - Lsu dut ( - .clock(clock), - .reset(control.reset), - .io_pc(control.pc), - .io_cpu_s1_kill(control.s1_kill), - .io_cpu_s2_kill(control.s2_kill), - .io_cpu_s1_data_data(control.s1_data), - .io_cpu_s1_data_mask(8'hff), - .io_cpu_keep_clock_enabled(1'b1), - .io_cpu_s2_nack(control.nack), - .io_cpu_ordered(control.ordered), - .io_idle(control.idle), - .io_cancelled(control.cancelled), - .io_cpu_s2_xcpt_ma_ld(control.ma_ld), - .io_cpu_s2_xcpt_ma_st(control.ma_st), - .io_cpu_s2_xcpt_pf_ld(control.pf_ld), - .io_cpu_s2_xcpt_pf_st(control.pf_st), - .io_cpu_s2_xcpt_ae_ld(control.ae_ld), - .io_cpu_s2_xcpt_ae_st(control.ae_st), - `include "lsu_ports.svh" - ); - initial begin - uvm_config_db#(virtual lsu_control_if)::set(null, "uvm_test_top*", "control", control); - uvm_config_db#(virtual stream_if #(`CORE_CPU_WIDTH))::set(null, "uvm_test_top*", "cpu", - source_if); - uvm_config_db#(virtual stream_if #(`CORE_RETURN_WIDTH))::set(null, "uvm_test_top*", "result", - sink_if); - uvm_config_db#(virtual stream_if #(`CORE_VIRTUAL_WIDTH))::set(null, "uvm_test_top*", - "virtual_req", virtual_req); - uvm_config_db#(virtual stream_if #(`CORE_RESULT_WIDTH))::set(null, "uvm_test_top*", - "virtual_resp", virtual_resp); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/lsu_test.svh b/arch/src/main/scala/framework/root/root-core/src/main/resources/lsu_test.svh deleted file mode 100644 index ff30cab48..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/lsu_test.svh +++ /dev/null @@ -1,193 +0,0 @@ -class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual lsu_control_if control; - virtual stream_if #(`CORE_CPU_WIDTH) cpu; - virtual stream_if #(`CORE_RETURN_WIDTH) result; - virtual stream_if #(`CORE_VIRTUAL_WIDTH) virtual_req; - virtual stream_if #(`CORE_RESULT_WIDTH) virtual_resp; - int issued = 0, checked = 0, killed = 0, rejected = 0; - function new(string name, uvm_component parent); - super.new(name, parent); - timeout = 50us; - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual lsu_control_if)::get( - this, "", "control", control - ) || !uvm_config_db#(virtual stream_if #(`CORE_CPU_WIDTH))::get( - this, "", "cpu", cpu - ) || !uvm_config_db#(virtual stream_if #(`CORE_RETURN_WIDTH))::get( - this, "", "result", result - ) || !uvm_config_db#(virtual stream_if #(`CORE_VIRTUAL_WIDTH))::get( - this, "", "virtual_req", virtual_req - ) || !uvm_config_db#(virtual stream_if #(`CORE_RESULT_WIDTH))::get( - this, "", "virtual_resp", virtual_resp - )) - `uvm_fatal("VIF", "LSU interfaces missing") - endfunction - task start(int cmd = 1, longint unsigned address = 64'h800040c0); - bit [`CORE_CPU_WIDTH-1:0] packet = '0; - packet[`LF(CPU, ADDR)] = address; - packet[`LF(CPU, TAG)] = 2; - packet[`LF(CPU, CMD)] = cmd; - packet[`LF(CPU, SIZE)] = 3; - packet[`LF(CPU, DPRV)] = 3; - packet[`LF(CPU, SIGNED)] = 1; - packet[`LF(CPU, NO_RESP)] = cmd == 1; - packet[`LF(CPU, DATA)] = 64'hdeadbeef; // EX req.data is not the later S1 operand. - control.cb.pc <= 46'h80000100; - control.cb.s1_kill <= 0; - control.cb.s2_kill <= 0; - control.cb.s1_data <= 64'h0123456789abcdef; - cpu.bits <= packet; - cpu.valid <= 1; - do @(cpu.sample); while (!cpu.sample.ready); - cpu.valid <= 0; - endtask - task no_backend(); - repeat (3) begin - @(control.cb); - if (virtual_req.valid || result.valid) - `uvm_fatal("KILL", "Killed instruction had a memory/CPU side effect") - end - killed++; - endtask - task execute_operation(int cmd = 1, bit fault = 0); - start(cmd); - @(control.cb); - @(control.cb); - if (!control.cb.nack) `uvm_fatal("S2", "Original operation did not nack at its precise S2") - do @(virtual_req.sample); while (!virtual_req.sample.valid); - if (virtual_req.sample.bits[ - `LF(VIRTUAL, TAG) - ] != 2 || virtual_req.sample.bits[ - `LF(VIRTUAL, VADDR) - ] != 64'h800040c0 || virtual_req.sample.bits[ - `LF(VIRTUAL, DATA) - ] != 64'h0123456789abcdef || virtual_req.sample.bits[ - `LF(VIRTUAL, WRITE) - ] != (cmd == 1) || virtual_req.sample.bits[ - `LF(VIRTUAL, EXECUTE) - ] != 0) - `uvm_fatal("OPERAND", "S1 operand/tag/address incorrectly captured") - repeat (3) @(control.cb); - virtual_req.ready <= 1; - @(virtual_req.sample); - virtual_req.ready <= 0; - issued++; - repeat (4) @(control.cb); - virtual_resp.bits <= '0; - virtual_resp.bits[`LF(RESULT, TAG)] <= 2; - virtual_resp.bits[`LF(RESULT, DATA)] <= cmd == 0 ? 64'h89abcdef01234567 : 0; - virtual_resp.bits[`LF(RESULT, ACCESSFAULT)] <= fault; - virtual_resp.valid <= 1; - do @(virtual_resp.sample); while (!virtual_resp.sample.ready); - virtual_resp.valid <= 0; - @(control.cb); - if (!control.cb.ordered || result.valid) - `uvm_fatal("ORDER", - "Buffered completed operation still blocked ordered decode or returned early") - endtask - task reject_identity(bit wrong_pc); - bit [`CORE_CPU_WIDTH-1:0] packet = '0; - packet[`LF(CPU, ADDR)] = 64'h800040c0; - packet[`LF(CPU, TAG)] = wrong_pc ? 2 : 4; - packet[`LF(CPU, CMD)] = 1; - packet[`LF(CPU, SIZE)] = 3; - packet[`LF(CPU, DPRV)] = 3; - control.cb.pc <= wrong_pc ? 46'h80000104 : 46'h80000100; - cpu.bits <= packet; - cpu.valid <= 1; - repeat (3) begin - @(cpu.sample); - if (cpu.sample.ready || result.valid || virtual_req.valid) - `uvm_fatal("IDENTITY", "A different tag/PC consumed the buffered completion") - end - control.cb.cancel_offer <= 1; - cpu.valid <= 0; - @(control.cb); - control.cb.cancel_offer <= 0; - rejected++; - endtask - task replay(int cmd = 1, bit fault = 0, bit misaligned = 0, - longint unsigned address = 64'h800040c0); - start(cmd, address); - @(control.cb); - do @(result.sample); while (!result.sample.valid); - if (result.sample.bits[ - `LF(RETURN, TAG) - ] != 2 || result.sample.bits[ - `LF(RETURN, CMD) - ] != cmd || result.sample.bits[ - `LF(RETURN, ADDR) - ] != address || result.sample.bits[ - `LF(RETURN, HAS_DATA) - ] != (cmd == 0 && !fault && !misaligned) || control.cb.ae_ld != (cmd == 0 && fault) || - control.cb.ae_st != (cmd == 1 && fault) || control.cb.ma_ld != (cmd == 0 && misaligned) - || control.cb.ma_st != (cmd == 1 && misaligned) || control.cb.nack || virtual_req.valid) - `uvm_fatal("REPLAY", "Replay return/exception/tag was not aligned with its own S2") - if (cmd == 0 && !fault && !misaligned && result.sample.bits[ - `LF(RETURN, DATA) - ] != 64'h89abcdef01234567) - `uvm_fatal("DATA", "Replay data changed") - checked++; - @(control.cb); - endtask - task execute(); - control.reset = 1; - control.cancel_offer = 0; - control.s1_kill = 0; - control.s2_kill = 0; - control.pc = 0; - control.s1_data = 0; - cpu.valid = 0; - cpu.bits = '0; - virtual_req.ready = 0; - virtual_resp.valid = 0; - virtual_resp.bits = '0; - repeat (4) @(control.cb); - control.cb.reset <= 0; - @(control.cb); - start(); - control.cb.s1_kill <= 1; - no_backend(); - control.cb.s1_kill <= 0; - start(); - @(control.cb); - control.cb.s2_kill <= 1; - no_backend(); - control.cb.s2_kill <= 0; - execute_operation(); - reject_identity(0); - reject_identity(1); - start(); - control.cb.s1_kill <= 1; - no_backend(); - control.cb.s1_kill <= 0; - start(); - @(control.cb); - control.cb.s2_kill <= 1; - no_backend(); - control.cb.s2_kill <= 0; - replay(); - execute_operation(0); - replay(0); - execute_operation(0, 1); - replay(0, 1); - execute_operation(1, 1); - replay(1, 1); - start(0, 64'h800040c1); - @(control.cb); - @(control.cb); - repeat (3) @(control.cb); - if (virtual_req.valid) `uvm_fatal("MISALIGN", "Misaligned request reached backend") - replay(0, 0, 1, 64'h800040c1); - if (issued != 4 || checked != 5 || killed != 4 || rejected != 2) - `uvm_fatal("COUNTS", "LSU contract cases incomplete") - `uvm_info( - "LSU", - $sformatf( - "Checked %0d precise replay completions, %0d real backend executions, %0d original/replay S1/S2 kills, %0d rejected tag/PC offers", - checked, issued, killed, rejected), UVM_LOW) - endtask -endclass diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance.cm_hier b/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance.cm_hier deleted file mode 100644 index 6bd7e07ab..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance.cm_hier +++ /dev/null @@ -1,5 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.source_if -+tree @TOP@.sink_if -+tree @TOP@.chi_req -+tree @TOP@.chi_rsp diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance.f b/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance.f deleted file mode 100644 index 716a51f6a..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/maintenance_interface.sv -@RESOURCES@/maintenance_package.sv --F @RTL@/Maintenance/filelist.f -@RESOURCES@/maintenance_tb.sv diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_bad_response.cm_hier b/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_bad_response.cm_hier deleted file mode 100644 index 6bd7e07ab..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_bad_response.cm_hier +++ /dev/null @@ -1,5 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.source_if -+tree @TOP@.sink_if -+tree @TOP@.chi_req -+tree @TOP@.chi_rsp diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_bad_response.f b/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_bad_response.f deleted file mode 100644 index 49e36bc22..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_bad_response.f +++ /dev/null @@ -1,11 +0,0 @@ -+define+MAINTENANCE_TB=maintenance_bad_response_tb -+define+MAINTENANCE_BAD_RESPONSE -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/maintenance_interface.sv -@RESOURCES@/maintenance_package.sv --F @RTL@/Maintenance/filelist.f -@RESOURCES@/maintenance_tb.sv diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_bad_response.toml b/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_bad_response.toml deleted file mode 100644 index 33bb516f3..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_bad_response.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Maintenance response has no issued request" diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_interface.sv b/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_interface.sv deleted file mode 100644 index 7a062bbb3..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_interface.sv +++ /dev/null @@ -1,10 +0,0 @@ -interface maintenance_control_if ( - input logic clock -); - logic reset, drained, idle; - clocking cb @(posedge clock); - default input #1step output #0; - output reset, drained; - input idle; - endclocking -endinterface diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_package.sv b/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_package.sv deleted file mode 100644 index 0374a573e..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_package.sv +++ /dev/null @@ -1,166 +0,0 @@ -`include "maintenance_config.svh" -package maintenance_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - `define MF(kind, field) `CORE_``kind``_``field``_OFFSET +: `CORE_``kind``_``field``_WIDTH - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual maintenance_control_if control; - virtual stream_if #(`CORE_RANGE_WIDTH) request; - virtual stream_if #(`CORE_ACK_WIDTH) response; - virtual stream_if #(`CORE_REQ_WIDTH) req; - virtual stream_if #(`CORE_RSP_WIDTH) rsp; - int checked_lines = 0, checked_ranges = 0; - function new(string name, uvm_component parent); - super.new(name, parent); - timeout = 50us; - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual maintenance_control_if)::get( - this, "", "control", control - ) || !uvm_config_db#(virtual stream_if #(`CORE_RANGE_WIDTH))::get( - this, "", "request", request - ) || !uvm_config_db#(virtual stream_if #(`CORE_ACK_WIDTH))::get( - this, "", "response", response - ) || !uvm_config_db#(virtual stream_if #(`CORE_REQ_WIDTH))::get( - this, "", "req", req - ) || !uvm_config_db#(virtual stream_if #(`CORE_RSP_WIDTH))::get( - this, "", "rsp", rsp - )) - `uvm_fatal("VIF", "Maintenance interfaces missing") - endfunction - task reset_all(); - control.cb.reset <= 1; - control.cb.drained <= 0; - request.valid <= 0; - request.bits <= '0; - response.ready <= 0; - req.ready <= 0; - rsp.valid <= 0; - rsp.bits <= '0; - repeat (3) @(control.cb); - control.cb.reset <= 0; - repeat (2) @(control.cb); - if (!control.cb.idle || req.valid || response.valid) - `uvm_fatal("RESET", "Joint reset retained work") - endtask - task start_range(int tag, int op, longint unsigned first, int lines); - bit [`CORE_RANGE_WIDTH-1:0] bits = '0; - bits[`MF(RANGE, TAG)] = tag; - bits[`MF(RANGE, OP)] = op; - bits[`MF(RANGE, FIRSTLINE)] = first; - bits[`MF(RANGE, LASTLINE)] = first + (lines - 1) * 64; - request.bits <= bits; - request.valid <= 1; - do @(request.sample); while (!request.sample.ready); - request.valid <= 0; - endtask - task line_completion(longint unsigned address, int opcode, bit error = 0); - bit [`CORE_REQ_WIDTH-1:0] expected = '0; - bit [`CORE_RSP_WIDTH-1:0] reply = '0; - int home = `MAINTENANCE_HOME_BASE + ((address >> 6) % `MAINTENANCE_HOME_COUNT); - expected[`MF(REQ, SRCID)] = `MAINTENANCE_NODE; - expected[`MF(REQ, TGTID)] = home; - expected[`MF(REQ, TXNID)] = `MAINTENANCE_TXN; - expected[`MF(REQ, ADDR)] = address; - expected[`MF(REQ, SIZE)] = 6; - expected[`MF(REQ, OPCODE)] = opcode; - expected[`MF(REQ, SNPATTR)] = 1; - expected[`MF(REQ, MEMATTR)] = 12; - expected[`MF(REQ, ALLOWRETRY)] = 1; - do @(req.sample); while (!req.sample.valid); - if (req.sample.bits !== expected) - `uvm_fatal("REQ", $sformatf( - "CMO mismatch at %h: %h != %h", address, req.sample.bits, expected)) - repeat (3) begin - @(req.sample); - if (!req.sample.valid || req.sample.bits !== expected || response.valid) - `uvm_fatal("STALL", "CMO changed or completed while request was stalled") - end - req.ready <= 1; - @(req.sample); - req.ready <= 0; - repeat (5) begin - @(rsp.sample); - if (req.valid || response.valid) - `uvm_fatal("EARLY", "Range advanced before actual Home completion") - end - reply[`MF(RSP, SRCID)] = home; - reply[`MF(RSP, TGTID)] = `MAINTENANCE_NODE; - reply[`MF(RSP, TXNID)] = `MAINTENANCE_TXN; - reply[`MF(RSP, OPCODE)] = 4; - reply[`MF(RSP, RESPERR)] = error ? 3 : 0; - rsp.bits <= reply; - rsp.valid <= 1; - do @(rsp.sample); while (!rsp.sample.ready); - rsp.valid <= 0; - checked_lines++; - endtask - task finish_range(int tag, bit ok); - bit [`CORE_ACK_WIDTH-1:0] expected = '0; - expected[`MF(ACK, TAG)] = tag; - expected[`MF(ACK, OK)] = ok; - do @(response.sample); while (!response.sample.valid); - repeat (4) begin - if (response.sample.bits !== expected || req.valid || request.ready) - `uvm_fatal("ACK", "Bad completion or accepted new range before retirement") - @(response.sample); - end - response.ready <= 1; - @(response.sample); - response.ready <= 0; - @(control.cb); - if (!control.cb.idle) `uvm_fatal("IDLE", "Completed range did not retire") - checked_ranges++; - endtask - task execute(); - reset_all(); -`ifdef MAINTENANCE_BAD_RESPONSE - rsp.valid <= 1; - repeat (3) @(control.cb); - `uvm_fatal("MISSING_ASSERT", "Unissued Home response was not rejected") -`else - for (int op = 0; op < 3; op++) begin - int lines = 3 - op; - control.cb.drained <= 0; - start_range(16 + op, op, 64'h80000000, lines); - repeat (4) begin - @(control.cb); - if (req.valid || response.valid) `uvm_fatal("DRAIN", "CMO overtook older CPU accesses") - end - control.cb.drained <= 1; - for (int i = 0; i < lines; i++) line_completion(64'h80000000 + i * 64, 8 + op); - finish_range(16 + op, 1); - end - start_range(255, 1, 64'h80000100, 4); - line_completion(64'h80000100, 9); - line_completion(64'h80000140, 9, 1); - finish_range(255, 0); - start_range(0, 0, 64'hfffffffffc0, 1); - line_completion(64'hfffffffffc0, 8); - finish_range(0, 1); - control.cb.drained <= 0; - start_range(42, 0, 64'h80000000, 1); - reset_all(); - control.cb.drained <= 1; - start_range(43, 2, 64'h80000000, 1); - do @(req.sample); while (!req.sample.valid); - req.ready <= 1; - @(req.sample); - req.ready <= 0; - reset_all(); - control.cb.drained <= 1; - start_range(44, 2, 64'h80000040, 1); - line_completion(64'h80000040, 10); - finish_range(44, 1); - if (checked_lines != 10 || checked_ranges != 6) - `uvm_fatal("COUNTS", "Maintenance scenarios incomplete") - `uvm_info("MAINTENANCE", - "Checked 10 CMO lines, 6 ranges, delayed Home completion, errors and joint reset", - UVM_LOW) -`endif - endtask - endclass -endpackage diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_tb.sv b/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_tb.sv deleted file mode 100644 index 073ec3e26..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_tb.sv +++ /dev/null @@ -1,46 +0,0 @@ -`include "maintenance_config.svh" -`ifndef MAINTENANCE_TB -`define MAINTENANCE_TB maintenance_tb -`endif -module `MAINTENANCE_TB; - import uvm_pkg::*; - import maintenance_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - maintenance_control_if control (clock); - stream_if #(`CORE_RANGE_WIDTH) source_if ( - clock, - control.reset - ); - stream_if #(`CORE_ACK_WIDTH) sink_if ( - clock, - control.reset - ); - stream_if #(`CORE_REQ_WIDTH) chi_req ( - clock, - control.reset - ); - stream_if #(`CORE_RSP_WIDTH) chi_rsp ( - clock, - control.reset - ); - Maintenance dut ( - .clock(clock), - .reset(control.reset), - .io_drained(control.drained), - .io_idle(control.idle), - `include "maintenance_ports.svh" - ); - initial begin - uvm_config_db#(virtual maintenance_control_if)::set(null, "uvm_test_top*", "control", control); - uvm_config_db#(virtual stream_if #(`CORE_RANGE_WIDTH))::set(null, "uvm_test_top*", "request", - source_if); - uvm_config_db#(virtual stream_if #(`CORE_ACK_WIDTH))::set(null, "uvm_test_top*", "response", - sink_if); - uvm_config_db#(virtual stream_if #(`CORE_REQ_WIDTH))::set(null, "uvm_test_top*", "req", - chi_req); - uvm_config_db#(virtual stream_if #(`CORE_RSP_WIDTH))::set(null, "uvm_test_top*", "rsp", - chi_rsp); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/multicore.S b/arch/src/main/scala/framework/root/root-core/src/main/resources/multicore.S deleted file mode 100644 index 17aa3bb5a..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/multicore.S +++ /dev/null @@ -1,108 +0,0 @@ -.option norelax -.option norvc -.macro check_payload - ld t0, 0(s1) - li t1, 0x8877665544332211 - bne t0, t1, fail - ld t0, 32(s1) - li t1, 0x0102030405060708 - bne t0, t1, fail - ld t0, 40(s1) - li t1, 0x1122334455667788 - bne t0, t1, fail - ld t0, 48(s1) - li t1, 0x8000000000000001 - bne t0, t1, fail - ld t0, 56(s1) - li t1, 0xffeeddccbbaa9988 - bne t0, t1, fail -.endm -.section .text.start,"ax",@progbits -.globl _start -_start: - li s0, 0x10000000 - la s1, shared - li s2, 0 - la t0, trap_handler - csrw mtvec, t0 - csrw mie, zero - csrr s5, mhartid - beqz s5, producer - li t0, 1 - beq s5, t0, consumer - j fail - -.section .text.trap,"ax",@progbits -trap_handler: - j fail - -.section .text.tests,"ax",@progbits -producer: - li t0, 0x8877665544332211 - sd t0, 0(s1) - sd zero, 24(s1) - li t0, 0x0102030405060708 - sd t0, 32(s1) - li t0, 0x1122334455667788 - sd t0, 40(s1) - li t0, 0x8000000000000001 - sd t0, 48(s1) - li t0, 0xffeeddccbbaa9988 - sd t0, 56(s1) - fence rw, rw - li t0, 1 - sd t0, 8(s1) - li s2, 1 - sd s2, 8(s0) - li s3, 4096 -producer_wait: - ld t0, 16(s1) - bnez t0, producer_check - addi s3, s3, -1 - bnez s3, producer_wait - j fail -producer_check: - li t1, 1 - bne t0, t1, fail - fence r, rw - ld t0, 24(s1) - bne t0, t1, fail - check_payload - j finish -consumer: - li s3, 4096 -consumer_wait: - ld t0, 8(s1) - bnez t0, consumer_check - addi s3, s3, -1 - bnez s3, consumer_wait - j fail -consumer_check: - li t1, 1 - bne t0, t1, fail - fence r, rw - check_payload - li s2, 1 - sd s2, 8(s0) - addi t2, s1, 24 - li t0, 1 - amoadd.d.aqrl t1, t0, (t2) - bnez t1, fail - ld t1, 24(s1) - bne t1, t0, fail - fence rw, w - sd t0, 16(s1) -finish: - li s2, 2 - sd s2, 8(s0) - sd zero, 0(s0) -1: j 1b -fail: - ori t0, s2, 0x100 - sd t0, 0(s0) -2: j 2b - -.section .data,"aw",@progbits -.balign 64 -shared: - .dword 0, 0, 0, 0, 0, 0, 0, 0 diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/multicore.ld b/arch/src/main/scala/framework/root/root-core/src/main/resources/multicore.ld deleted file mode 100644 index 7b1347647..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/multicore.ld +++ /dev/null @@ -1,14 +0,0 @@ -OUTPUT_ARCH(riscv) -ENTRY(_start) -SECTIONS -{ - . = 0x80000000; - .text.start : { *(.text.start) } - . = 0x80000200; - .text.trap : { *(.text.trap) } - . = 0x80000400; - .text.tests : { *(.text.tests) } - . = 0x800040c0; - .data : { *(.data) } - /DISCARD/ : { *(.riscv.attributes) *(.comment) } -} diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/permission.cm_hier b/arch/src/main/scala/framework/root/root-core/src/main/resources/permission.cm_hier deleted file mode 100644 index 772e1cfc5..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/permission.cm_hier +++ /dev/null @@ -1 +0,0 @@ -+tree @TOP@.dut diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/permission.f b/arch/src/main/scala/framework/root/root-core/src/main/resources/permission.f deleted file mode 100644 index 8081d7f26..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/permission.f +++ /dev/null @@ -1,5 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv --F @RTL@/PermissionVerification/filelist.f -@RESOURCES@/permission_tb.sv diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/permission_tb.sv b/arch/src/main/scala/framework/root/root-core/src/main/resources/permission_tb.sv deleted file mode 100644 index f77e4a83c..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/permission_tb.sv +++ /dev/null @@ -1,216 +0,0 @@ -interface permission_control_if ( - input logic clock -); - logic reset, context_valid, request_valid, request_ready, response_valid, response_ready; - logic write, is_pte, allow; - logic [1:0] privilege; - logic [7:0] id, context_id, response_id; - logic [12:0] bytes; - logic [43:0] address; - logic [31:0] pmp_config; - logic [63:0] pmp_addr[4]; - clocking cb @(posedge clock); - default input #1step output #0; - output reset, context_valid, request_valid, response_ready, write, is_pte; - output privilege, id, context_id, bytes, address, pmp_config, pmp_addr; - input request_ready, response_valid, response_id, allow; - endclocking -endinterface - -package permission_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual permission_control_if control; - int checked = 0; - localparam longint unsigned BASE = 'h80000000; - function new(string name, uvm_component parent); - super.new(name, parent); - timeout = 100us; - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual permission_control_if)::get(this, "", "control", control)) - `uvm_fatal("VIF", "Permission interface missing") - endfunction - task initialize(); - control.cb.reset <= 1; - control.cb.context_valid <= 0; - control.cb.request_valid <= 0; - control.cb.response_ready <= 0; - control.cb.pmp_config <= 0; - control.cb.context_id <= 0; - control.cb.id <= 0; - control.cb.address <= 0; - control.cb.bytes <= 0; - control.cb.write <= 0; - control.cb.is_pte <= 0; - control.cb.privilege <= 0; - foreach (control.pmp_addr[i]) control.cb.pmp_addr[i] <= 0; - repeat (3) @(control.cb); - control.cb.reset <= 0; - repeat (2) @(control.cb); - endtask - task check_range(longint unsigned address, int bytes, bit write, int privilege, bit expected, - bit is_pte = 0, bit context_valid = 1, bit wrong_tag = 0, bit mutate = 0, - int hold = 0); - int tag = checked + 1; - control.cb.id <= tag; - control.cb.context_id <= wrong_tag ? tag + 1 : tag; - control.cb.context_valid <= context_valid; - control.cb.address <= address; - control.cb.bytes <= bytes; - control.cb.write <= write; - control.cb.is_pte <= is_pte; - control.cb.privilege <= privilege; - control.cb.request_valid <= 1; - do @(control.cb); while (!control.cb.request_ready); - control.cb.request_valid <= 0; - if (mutate) begin - control.cb.pmp_config <= 0; - foreach (control.pmp_addr[i]) control.cb.pmp_addr[i] <= 0; - control.cb.context_valid <= 0; - control.cb.context_id <= tag + 1; - end - do @(control.cb); while (!control.cb.response_valid); - for (int cycle = 0; cycle <= hold; ++cycle) begin - if (!control.cb.response_valid || control.cb.response_id !== tag || - control.cb.allow !== expected || control.cb.request_ready) - `uvm_fatal("PERMISSION", $sformatf( - "case%0d PA=%h bytes=%0d write=%0d prv=%0d expected=%b got=%b", - tag, - address, - bytes, - write, - privilege, - expected, - control.cb.allow - )) - if (cycle < hold) @(control.cb); - end - control.cb.response_ready <= 1; - @(control.cb); - control.cb.response_ready <= 0; - @(control.cb); - ++checked; - endtask - task execute(); - initialize(); - check_range(BASE, 64, 0, 3, 1); - check_range(BASE, 64, 0, 1, 0); - check_range('h90000000, 64, 0, 3, 1); - check_range('h90000000, 64, 1, 3, 0); - check_range(BASE + 'h2000000, 64, 0, 3, 0); - check_range(BASE, 0, 0, 3, 0); - check_range(BASE, 4097, 0, 3, 0); - check_range(BASE, 64, 0, 2, 0); - check_range(BASE, 64, 0, 3, 0, 0, 0); - check_range(BASE, 64, 0, 3, 0, 0, 1, 1); - check_range('h1000000000, 64, 0, 3, 0); - - control.cb.pmp_config <= 'h1b; // NAPOT R/W, 8 KiB at BASE. - control.cb.pmp_addr[0] <= BASE + 'hffc; - check_range(BASE, 4096, 0, 1, 1); - check_range(BASE + 1, 127, 0, 1, 1); - check_range(BASE + 8192 - 64, 64, 1, 1, 1); - check_range(BASE + 8192, 64, 0, 1, 0); - check_range(BASE, 8, 0, 1, 1, 1); - check_range(BASE, 8, 1, 1, 0, 1); - check_range(BASE, 8, 0, 0, 0, 1); - check_range(BASE, 64, 0, 1, 0, 1); - - control.cb.pmp_config <= 'h99; // Locked NAPOT read-only, applies in M-mode too. - check_range(BASE, 64, 0, 3, 1); - check_range(BASE, 64, 1, 3, 0); - check_range(BASE, 4096, 0, 1, 1, 0, 1, 0, 1, 9); // Captured PMP survives changes. - - // Higher-priority 64-byte read-only entry in the middle of a writable region. - control.cb.pmp_config <= 'h1b19; - control.cb.pmp_addr[0] <= BASE + 64 + 28; - control.cb.pmp_addr[1] <= BASE + 'hffc; - check_range(BASE, 192, 0, 1, 1); - check_range(BASE, 192, 1, 1, 0); - check_range(BASE, 64, 1, 1, 1); - check_range(BASE + 64, 64, 1, 1, 0); - check_range(BASE + 128, 64, 1, 1, 1); - - // Two adjacent TOR entries: check each actual line rather than rejecting a page envelope. - control.cb.pmp_config <= 'h0b0900; - control.cb.pmp_addr[0] <= BASE; - control.cb.pmp_addr[1] <= BASE + 64; - control.cb.pmp_addr[2] <= BASE + 128; - check_range(BASE, 128, 0, 1, 1); - check_range(BASE, 64, 1, 1, 0); - check_range(BASE + 64, 64, 1, 1, 1); - check_range(BASE, 128, 1, 1, 0); - - // A denied hole blocks a proposed union while both original pixels remain permitted. - control.cb.pmp_config <= 'h1b18; - control.cb.pmp_addr[0] <= BASE + 20; // Higher-priority 16-byte denied NAPOT hole at BASE+16. - control.cb.pmp_addr[1] <= BASE + 'hffc; - check_range(BASE, 96, 0, 1, 0); - check_range(BASE, 16, 0, 1, 1); - check_range(BASE + 64, 32, 0, 1, 1); - - // Cancel an accepted long check by joint reset. No stale response may escape. - control.cb.pmp_config <= 'h1b; - control.cb.pmp_addr[0] <= BASE + 'hffc; - control.cb.context_valid <= 1; - control.cb.context_id <= 99; - control.cb.id <= 99; - control.cb.address <= BASE; - control.cb.bytes <= 4096; - control.cb.write <= 0; - control.cb.is_pte <= 0; - control.cb.privilege <= 1; - control.cb.request_valid <= 1; - do @(control.cb); while (!control.cb.request_ready); - control.cb.request_valid <= 0; - repeat (4) @(control.cb); - initialize(); - repeat (8) begin - @(control.cb); - if (control.cb.response_valid) `uvm_fatal("RESET", "Old permission response survived reset") - end - check_range(BASE, 64, 0, 3, 1); - `uvm_info("PASS", $sformatf("%0d frozen-context PMP/PMA checks passed", checked), UVM_LOW) - endtask - endclass -endpackage - -module permission_tb; - import uvm_pkg::*; - import permission_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - permission_control_if control (clock); - PermissionVerification dut ( - .clock(clock), - .reset(control.reset), - .io_request_valid(control.request_valid), - .io_request_ready(control.request_ready), - .io_request_bits_id(control.id), - .io_request_bits_pa(control.address), - .io_request_bits_bytes(control.bytes), - .io_request_bits_write(control.write), - .io_request_bits_isPte(control.is_pte), - .io_request_bits_privilege(control.privilege), - .io_response_valid(control.response_valid), - .io_response_ready(control.response_ready), - .io_response_bits_id(control.response_id), - .io_response_bits_allow(control.allow), - .io_contextValid(control.context_valid), - .io_contextId(control.context_id), - .io_pmpConfig(control.pmp_config), - .io_pmpAddresses_0(control.pmp_addr[0]), - .io_pmpAddresses_1(control.pmp_addr[1]), - .io_pmpAddresses_2(control.pmp_addr[2]), - .io_pmpAddresses_3(control.pmp_addr[3]) - ); - initial begin - uvm_config_db#(virtual permission_control_if)::set(null, "uvm_test_top*", "control", control); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/supervisor.S b/arch/src/main/scala/framework/root/root-core/src/main/resources/supervisor.S deleted file mode 100644 index caf82779d..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/supervisor.S +++ /dev/null @@ -1,221 +0,0 @@ -.option norelax -.option norvc - -.macro expect_trap cause, value - li s9, \cause - li s10, \value -.endm - -.macro check_trap failure - li t0, -1 - bne s9, t0, \failure -.endm - -.section .text.start,"ax",@progbits -.globl _start -_start: - li s0, 0x10000000 - li s2, 1 - li s9, -1 - li s11, 0 - la t0, trap_handler - csrw mtvec, t0 - csrw medeleg, zero - csrw mideleg, zero - csrw mie, zero - // The standalone Core has no RoCC accelerator; TIME must still be readable. - csrr t0, time - beqz t0, fail - li t0, 2 - csrw mcounteren, t0 - csrw scounteren, t0 - li t0, 0x20002800 - csrw pmpaddr0, t0 - li t0, 0x20002803 - csrw pmpaddr1, t0 - li t0, -1 - csrw pmpaddr2, t0 - li t0, 0x1f191d - csrw pmpcfg0, t0 - li t0, 0x8000000000080010 - csrw satp, t0 - sfence.vma - - // MPRV=S translates data while machine instruction fetch remains physical. - li t0, 0x21800 - csrc mstatus, t0 - li t0, 0x20800 - csrs mstatus, t0 - li t1, 0x40008000 - ld t2, 0(t1) - li t0, 0x20000 - csrc mstatus, t0 - li t0, 0x1122334455667788 - bne t2, t0, fail - sd s2, 8(s0) - li t0, 0x40004000 - csrw mepc, t0 - mret - -.section .text.trap,"ax",@progbits -trap_handler: - csrr t3, mcause - bne t3, s9, fail - csrr t0, mtval - bne t0, s10, fail - addi s11, s11, 1 - li s9, -1 - li t1, 1 - beq t3, t1, return_boundary - li t1, 8 - beq t3, t1, return_supervisor - li t1, 9 - beq t3, t1, finish - csrr t0, mepc - addi t0, t0, 4 - csrw mepc, t0 - mret -return_boundary: - la t0, boundary_return - li t1, 0x40000000 - sub t0, t0, t1 - csrw mepc, t0 - mret -return_supervisor: - sd s2, 8(s0) - // U ECALL returns through M to an explicit S continuation. - li t0, 0x1800 - csrc mstatus, t0 - li t0, 0x800 - csrs mstatus, t0 - la t0, supervisor_return - li t1, 0x40000000 - sub t0, t0, t1 - csrw mepc, t0 - mret -finish: - li t0, 6 - bne s11, t0, fail - li s2, 3 - sd s2, 8(s0) - sd zero, 0(s0) -1: j 1b -fail: - ori t0, s2, 0x100 - sd t0, 0(s0) -2: j 2b - -.section .text.supervisor,"ax",@progbits -supervisor: - csrr t0, time - beqz t0, supervisor_fail - li s2, 2 - li s1, 0x40008000 - li t0, 0x8877665544332211 - sd t0, 0(s1) - ld t1, 0(s1) - bne t0, t1, supervisor_fail - // Exercise both fetch and data through a canonical upper-half Sv39 alias. - li s3, 0xffffffffc0008000 - la t0, high_alias - li t1, 0xffffffff80000000 - add t0, t0, t1 - jalr ra, 0(t0) - // The allowed RX range is exactly one 8-byte fetch word. - li s4, 0 - li t0, 0x4000a000 - jalr ra, 0(t0) - li t0, 1 - bne s4, t0, supervisor_fail - expect_trap 1, 0x4000a008 - li t0, 0x4000a008 - jr t0 -boundary_return: - check_trap supervisor_fail - // Change the actual leaf via a writable page-table alias, then fence. - li t0, 0x40012040 - li t1, 0x200020c3 - sd t1, 0(t0) - sfence.vma s1, zero - expect_trap 15, 0x40008000 -trap_store: - sd zero, 0(s1) - check_trap supervisor_fail - ld t0, 0(s1) - li t1, 0x8877665544332211 - bne t0, t1, supervisor_fail - // Supervisor cannot read a user page with SUM clear. - expect_trap 13, 0x40006000 - li t0, 0x40006000 -trap_sum: - ld t1, 0(t0) - check_trap supervisor_fail - li t0, 0x40006000 - csrw sepc, t0 - li t0, 0x100 - csrc sstatus, t0 - sret -supervisor_return: - check_trap supervisor_fail - expect_trap 9, 0 -trap_supervisor_ecall: - ecall -3: j 3b -supervisor_fail: - ebreak - j supervisor_fail -high_alias: - ld t1, 0(s3) - li t2, 0x8877665544332211 - bne t1, t2, supervisor_fail - ret - -.section .text.user,"ax",@progbits -user: - csrr t0, time - beqz t0, user_fail - // User access to a supervisor-only leaf must fault. - expect_trap 13, 0x40008000 - li t0, 0x40008000 -trap_user_load: - ld t1, 0(t0) - check_trap user_fail - expect_trap 8, 0 -trap_user_ecall: - ecall -4: j 4b -user_fail: - ebreak - j user_fail - -.section .data,"aw",@progbits - .dword 0x1122334455667788 - -.section .text.boundary,"ax",@progbits - addi s4, s4, 1 - ret -trap_boundary: - nop - -.section .pagetables,"aw",@progbits -.balign 4096 -root: - .dword 0 - .dword 0x20004401 - .zero 4072 - .dword 0x20004401 -middle: - .dword 0x20004801 - .zero 4088 -leaf: - .zero 32 - .dword 0x200010cb - .dword 0 - .dword 0x200018db - .dword 0 - .dword 0x200020c7 - .dword 0 - .dword 0x200028cb - .zero 56 - .dword 0x200048c7 - .zero 3944 diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/supervisor.cm_hier b/arch/src/main/scala/framework/root/root-core/src/main/resources/supervisor.cm_hier deleted file mode 100644 index 0f4845487..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/supervisor.cm_hier +++ /dev/null @@ -1,5 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.mem_req -+tree @TOP@.mem_resp -+tree @TOP@.uncached_req -+tree @TOP@.uncached_resp diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/supervisor.f b/arch/src/main/scala/framework/root/root-core/src/main/resources/supervisor.f deleted file mode 100644 index 1ae752420..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/supervisor.f +++ /dev/null @@ -1,13 +0,0 @@ -+define+SUPERVISOR_PROFILE -// SATP PPN/ASID have no architectural reset value; select concrete bits for this 4-state run. -+define+RANDOMIZE_REG_INIT -+define+RANDOM=32'b0 -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/core_interface.sv -@RESOURCES@/core_package.sv --F @RTL@/Supervisor/filelist.f -@RESOURCES@/core_tb.sv diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/supervisor.ld b/arch/src/main/scala/framework/root/root-core/src/main/resources/supervisor.ld deleted file mode 100644 index 8db53fc3c..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/supervisor.ld +++ /dev/null @@ -1,20 +0,0 @@ -OUTPUT_ARCH(riscv) -ENTRY(_start) -SECTIONS -{ - . = 0x80000000; - .text.start : { *(.text.start) } - . = 0x80002000; - .text.trap : { *(.text.trap) } - . = 0x80004000; - .text.supervisor : { *(.text.supervisor) } - . = 0x80006000; - .text.user : { *(.text.user) } - . = 0x80008000; - .data : { *(.data) } - . = 0x8000a000; - .text.boundary : { *(.text.boundary) } - . = 0x80010000; - .pagetables : { *(.pagetables) } - /DISCARD/ : { *(.riscv.attributes) *(.comment) } -} diff --git a/arch/src/main/scala/framework/root/root-core/src/main/scala/rocket/Emit.scala b/arch/src/main/scala/framework/root/root-core/src/main/scala/rocket/Emit.scala deleted file mode 100644 index e8628b215..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/scala/rocket/Emit.scala +++ /dev/null @@ -1,481 +0,0 @@ -package hier.core.rocket - -import java.nio.file.{Files, Paths} -import scala.sys.process._ -import framework.system.core.rocket.{CpuParams, RocketParameters} -import freechips.rocketchip.tile.FPUParams -import freechips.rocketchip.rocket.{ - ASIdBits, - DCacheParams, - HellaCacheReq, - HellaCacheResp, - ICacheParams, - RocketCoreParams -} -import memcore.memory.cpu.{CpuMemParams, PhysicalRegion, VirtualMemoryRequest, VirtualMemoryResponse} -import chisel3._ -import memcore.memory.cache.configs.CacheParams -import memcore.memory.coherence.configs.CoherenceParams -import memcore.bus.chi.{Params => ChiParams} - -object Emit extends App { - val root = Paths.get("src/main/scala/framework/root/root-core").toAbsolutePath - val output = root.resolve("build") - Files.createDirectories(output) - require( - Process(Seq("python", root.resolve("src/main/resources/build_fixture.py").toString, output.toString)).! == 0, - "Core fixture generation failed" - ) - - implicit val cpuParams: CpuParams = CpuParams( - core = new RocketCoreParams( - useVM = false, - useUser = false, - useSupervisor = false, - useHypervisor = false, - useDebug = false, - fpu = None, - nPMPs = 4, - clockGate = false, - haveCFlush = false, - haveCease = false, - haveSimTimeout = false - ) { override val pmpGranularity = 8 }, - dcache = Some(DCacheParams(nSets = 64, nWays = 2)), - icache = Some(ICacheParams()), - btb = None, - physicalAddressBits = 44, - hartIdBits = 1, - beatBytes = 8, - blockBytes = 64 - ) - - implicit val upstream = RocketParameters(cpuParams) - - val memory = CoherenceParams(ChiParams(), CacheParams(44, 64, 4, 2, 8, 4, 2), 2, 4, 64) - - val floatingParameters = cpuParams.copy(core = cpuParams.core.copy(fpu = Some(FPUParams(minFLen = 16, fLen = 64)))) - - val regions = Seq( - PhysicalRegion(BigInt("80000000", 16), BigInt(16) << 20, true, true, true, true, true, true), - PhysicalRegion(BigInt("10000000", 16), BigInt(4096), false, false, true, true, false, false) - ) - - require( - Process(Seq("python", root.resolve("src/main/resources/build_fixture.py").toString, output.toString, "fpu")).! == 0, - "FPU fixture generation failed" - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Verification(memory, regions, "Fpu")(floatingParameters), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("Fpu").toString, "--split-verilog") - ) - - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Verification(memory, regions, "Core"), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("Core").toString, "--split-verilog") - ) - - val supervisorParameters = cpuParams.copy( - core = new RocketCoreParams( - useVM = true, - useUser = true, - useSupervisor = true, - useHypervisor = false, - useDebug = false, - fpu = None, - nPMPs = 4, - clockGate = false, - haveCFlush = false, - haveCease = false, - haveSimTimeout = false - ) { override val pmpGranularity = 8 }, - dcache = Some(DCacheParams(nSets = 64, nWays = 2)), - icache = Some(ICacheParams()), - btb = None, - physicalAddressBits = 36, - asIdBits = 16 - ) - - require( - Process( - Seq("python", root.resolve("src/main/resources/build_fixture.py").toString, output.toString, "supervisor") - ).! == 0, - "Supervisor fixture generation failed" - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Verification(memory, regions, "Supervisor")(supervisorParameters), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("Supervisor").toString, "--split-verilog") - ) - - require( - Process(Seq("python", root.resolve("src/main/resources/build_fixture.py").toString, output.toString, "irq")).! == 0, - "IRQ fixture generation failed" - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Verification(memory, regions, "Irq"), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("Irq").toString, "--split-verilog") - ) - - val definitions = scala.collection.mutable.ArrayBuffer[String]( - s"""`define IRQ_IMAGE "${output.resolve("irq.hex")}"""", - s"""`define FPU_IMAGE "${output.resolve("fpu.hex")}"""", - s"""`define SUPERVISOR_IMAGE "${output.resolve("supervisor.hex")}"""", - s"""`define CORE_IMAGE "${output.resolve("core.hex")}"""" - ) - - val ports = scala.collection.mutable.ArrayBuffer[String]() - - def channel( - kind: String, - prefix: String, - instance: String, - fields: Seq[(String, Int)] - ): Unit = { - var offset = 0 - for ((name, width) <- fields) { - definitions += s"`define CORE_${kind}_${name.toUpperCase}_OFFSET $offset" - definitions += s"`define CORE_${kind}_${name.toUpperCase}_WIDTH $width" - ports += s".$prefix$name($instance.bits[$offset +: $width])" - offset += width - } - definitions += s"`define CORE_${kind}_WIDTH $offset" - } - - channel( - "MEMREQ", - "io_memoryRequest_bits_", - "mem_req", - Seq("id" -> 12, "addr" -> 44, "write" -> 1, "data" -> 512, "mask" -> 64) - ) - channel("MEMRESP", "io_memoryResponse_bits_", "mem_resp", Seq("id" -> 12, "data" -> 512, "error" -> 1)) - channel( - "UNCACHED", - "io_uncachedRequest_bits_", - "uncached_req", - Seq("addr" -> 64, "tag" -> 6, "size" -> 3, "write" -> 1, "data" -> 64) - ) - channel("URESP", "io_uncachedResponse_bits_", "uncached_resp", Seq("tag" -> 6, "data" -> 64, "error" -> 1)) - for ( - (prefix, instance) <- Seq( - "memoryRequest" -> "mem_req", - "memoryResponse" -> "mem_resp", - "uncachedRequest" -> "uncached_req", - "uncachedResponse" -> "uncached_resp" - ) - ) { - ports += s".io_${prefix}_valid($instance.valid)" - ports += s".io_${prefix}_ready($instance.ready)" - } - Files.write(output.resolve("core_config.svh"), (definitions.mkString("\n") + "\n").getBytes) - Files.write(output.resolve("core_ports.svh"), ports.mkString(",\n").getBytes) - val cp = CpuMemParams(memory.chi, tagBits = 6) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Lsu(cp), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("Lsu").toString, "--split-verilog") - ) - definitions.clear() - ports.clear() - - def elements(bundle: Bundle): Seq[(String, Int)] = bundle.elements.toSeq.collect { - case (name, field) if field.getWidth > 0 => name -> field.getWidth - } - - channel("CPU", "io_cpu_req_bits_", "source_if", elements(new HellaCacheReq)) - channel("RETURN", "io_cpu_resp_bits_", "sink_if", elements(new HellaCacheResp)) - channel("VIRTUAL", "io_request_bits_", "virtual_req", elements(new VirtualMemoryRequest(cp))) - channel("RESULT", "io_response_bits_", "virtual_resp", elements(new VirtualMemoryResponse(cp))) - for ((prefix, instance) <- Seq("cpu_req" -> "source_if", "request" -> "virtual_req", "response" -> "virtual_resp")) { - ports += s".io_${prefix}_valid($instance.valid)" - ports += s".io_${prefix}_ready($instance.ready)" - } - ports += ".io_cpu_resp_valid(sink_if.valid)" - for (i <- 0 until 4) { - for (name <- Seq("l", "res", "a", "x", "w", "r")) ports += s".io_pmp_${i}_cfg_$name('0)" - ports += s".io_pmp_${i}_addr('0)" - ports += s".io_pmp_${i}_mask('0)" - } - ports += ".io_context_privilege(2'd3)" - ports += ".io_context_satp(64'd0)" - ports += ".io_context_sum(1'b0)" - ports += ".io_context_mxr(1'b0)" - ports += ".io_maintenance_ready(1'b0)" - ports += ".io_maintained_valid(1'b0)" - ports += ".io_maintained_bits(1'b0)" - Files.write(output.resolve("lsu_config.svh"), (definitions.mkString("\n") + "\n").getBytes) - Files.write(output.resolve("lsu_ports.svh"), ports.mkString(",\n").getBytes) - - val tracking = memcore.memory.interlock.Params() - val rnf = memcore.bus.chi.rnf.RnfParams(memory.chi, homeCount = 2) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Maintenance(rnf, tracking), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("Maintenance").toString, "--split-verilog") - ) - definitions.clear() - ports.clear() - definitions += s"`define MAINTENANCE_HOME_BASE ${rnf.homeId}" - definitions += s"`define MAINTENANCE_HOME_COUNT ${rnf.homeCount}" - definitions += s"`define MAINTENANCE_NODE ${rnf.nodeId}" - definitions += s"`define MAINTENANCE_TXN ${rnf.banks}" - channel("RANGE", "io_request_bits_", "source_if", elements(new memcore.memory.interlock.Maintenance(tracking))) - channel("ACK", "io_response_bits_", "sink_if", elements(new memcore.memory.interlock.Acknowledgement(tracking))) - channel("REQ", "io_req_bits_", "chi_req", elements(new memcore.bus.chi.RequestFlit(memory.chi))) - channel("RSP", "io_rsp_bits_", "chi_rsp", elements(new memcore.bus.chi.ResponseFlit(memory.chi))) - for ( - (prefix, instance) <- Seq("request" -> "source_if", "response" -> "sink_if", "req" -> "chi_req", "rsp" -> "chi_rsp") - ) { - ports += s".io_${prefix}_valid($instance.valid)" - ports += s".io_${prefix}_ready($instance.ready)" - } - Files.write(output.resolve("maintenance_config.svh"), (definitions.mkString("\n") + "\n").getBytes) - Files.write(output.resolve("maintenance_ports.svh"), ports.mkString(",\n").getBytes) - - require( - Process( - Seq("python", root.resolve("src/main/resources/build_fixture.py").toString, output.toString, "admission") - ).! == 0, - "Admission fixture generation failed" - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Verification( - memory, - regions :+ PhysicalRegion(BigInt("a0000000", 16), BigInt(4096), false, false, true, true, true, true), - "AdmissionCore", - Some(Commands(compute = true, scheduler = true, tracking = tracking)) - )(supervisorParameters), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("AdmissionCore").toString, "--split-verilog") - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new memcore.memory.interlock.Interlock(tracking), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("AdmissionInterlock").toString, "--split-verilog") - ) - - val admissionDefinitions = scala.collection.mutable.ArrayBuffer[String]( - s"""`define ADMIT_IMAGE "${output.resolve("admission.hex")}"""", - s"`define ADMIT_ENTRIES ${tracking.entries}", - s"`define ADMIT_PMP_COUNT ${supervisorParameters.core.nPMPs}", - s"`define ADMIT_OUTSTANDING_BITS ${chisel3.util.log2Ceil(tracking.entries + 1)}", - s"`define ADMIT_MEMORY_OUTSTANDING_BITS ${chisel3.util.log2Ceil(memory.mshrEntries + 1)}" - ) - - val admissionPorts = scala.collection.mutable.ArrayBuffer[String]() - - def flatten(data: Data, prefix: String = ""): Seq[(String, Int)] = data match { - case record: Record => record.elements.toSeq.flatMap { case (name, value) => - flatten(value, if (prefix.isEmpty) name else s"${prefix}_$name") - } - case vector: Vec[_] => vector.zipWithIndex.flatMap { case (value, i) => flatten(value, s"${prefix}_$i") } - case field if field.getWidth > 0 => Seq(prefix -> field.getWidth) - case _ => Seq.empty - } - - def admissionLayout( - kind: String, - prefix: String, - signal: String, - bundle: Data - ): Unit = { - var offset = 0 - for ((name, width) <- flatten(bundle)) { - admissionDefinitions += s"`define ADMIT_${kind}_${name.toUpperCase}_OFFSET $offset" - admissionDefinitions += s"`define ADMIT_${kind}_${name.toUpperCase}_WIDTH $width" - admissionPorts += s".$prefix$name($signal[$offset +: $width])" - offset += width - } - admissionDefinitions += s"`define ADMIT_${kind}_WIDTH $offset" - } - - for ( - (kind, port, signal, bundle) <- Seq( - ( - "RESERVE", - "admission_reserve", - "reserve", - new memcore.memory.interlock.Dispatch(tracking) - ), - ( - "COMMAND", - "admission_command", - "command", - new CommandSnapshot(tracking, supervisorParameters.core.nPMPs)( - supervisorParameters - ) - ), - ( - "COMPLETE", - "admission_complete", - "complete", - new memcore.memory.interlock.Tag(tracking) - ), - ( - "CANCELLED", - "admission_cancelled", - "cancelled", - new memcore.memory.interlock.Tag(tracking) - ), - ( - "RESPONSE", - "admission_response", - "response", - new framework.system.core.rocket.RoCCResponseBB - ), - ( - "MAINTENANCE", - "admission_maintenance", - "maintenance", - new memcore.memory.interlock.Maintenance(tracking) - ), - ( - "MAINTAINED", - "admission_maintained", - "maintained", - new memcore.memory.interlock.Acknowledgement(tracking) - ), - ( - "PTE", - "admission_pteRequest", - "pte_req", - new memcore.bus.chi.rnf.CacheAccess(memory.chi) - ), - ( - "PTERESP", - "admission_pteResponse", - "pte_resp", - new memcore.bus.chi.rnf.CacheResult - ), - ( - "MEMREQ", - "memoryRequest", - "mem_req", - new memcore.bus.chi.snf.LineRequest(memory.chi) - ), - ( - "MEMRESP", - "memoryResponse", - "mem_resp", - new memcore.bus.chi.snf.LineResponse(memory.chi) - ), - ( - "UNCACHED", - "uncachedRequest", - "uncached_req", - new memcore.memory.cpu.UncachedRequest(cp) - ), - ( - "URESP", - "uncachedResponse", - "uncached_resp", - new memcore.memory.cpu.UncachedResponse(cp) - ) - ) - ) { - admissionLayout(kind, s"io_${port}_bits_", s"$signal.bits", bundle) - admissionPorts += s".io_${port}_valid($signal.valid)" - admissionPorts += s".io_${port}_ready($signal.ready)" - } - admissionLayout("QUERY", "io_admission_cpuQuery_", "control.query", new memcore.memory.interlock.CpuQuery(tracking)) - admissionPorts ++= Seq( - ".io_admission_cpuAllow(control.cpu_allow)", - ".io_admission_cpuProbeAllow(control.cpu_allow)", - ".io_admission_interrupt(control.accelerator_irq)", - ".io_admission_outstanding(control.admission_outstanding)" - ) - Files.write(output.resolve("admission_config.svh"), (admissionDefinitions.mkString("\n") + "\n").getBytes) - Files.write(output.resolve("admission_ports.svh"), admissionPorts.mkString(",\n").getBytes) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new AdmissionBridge(16, tracking, supervisorParameters.core.nPMPs, Seq(1))(supervisorParameters), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("AdmissionBridge").toString, "--split-verilog") - ) - - val bridgeDefinitions = scala.collection.mutable.ArrayBuffer[String]( - "`define AB_ROB_ENTRIES 16", - "`define AB_ROB_BITS 4", - s"`define AB_ENTRIES ${tracking.entries}" - ) - - val bridgePorts = scala.collection.mutable.ArrayBuffer[String]() - val bridgeSnapshot = new CommandSnapshot(tracking, supervisorParameters.core.nPMPs)(supervisorParameters) - - def bridgeLayout( - kind: String, - prefix: String, - signal: String, - data: Data, - define: Boolean - ): Unit = { - var offset = 0 - for ((name, width) <- flatten(data)) { - if (define) { - bridgeDefinitions += s"`define AB_${kind}_${name.toUpperCase}_OFFSET $offset" - bridgeDefinitions += s"`define AB_${kind}_${name.toUpperCase}_WIDTH $width" - } - bridgePorts += s".$prefix$name($signal[$offset +: $width])" - offset += width - } - if (define) bridgeDefinitions += s"`define AB_${kind}_WIDTH $offset" - } - - bridgeLayout("SNAP", "io_command_bits_", "command.bits", bridgeSnapshot, true) - val bridgeRetirement = - new AdmissionRetirement(tracking, supervisorParameters.core.nPMPs)(supervisorParameters) - bridgeLayout("RET", "io_retirement_bits_", "retirement.bits", bridgeRetirement, true) - bridgeLayout("FAULT", "io_fault_bits_", "control.fault_bits", new RobFault(16), true) - bridgeLayout("FAULT", "io_unboundFault_bits_", "control.unbound_fault_bits", new RobFault(16), false) - bridgePorts ++= Seq(".io_fault_valid(control.fault_valid)", ".io_unboundFault_valid(control.unbound_fault_valid)") - val snapshotStart = - flatten(bridgeRetirement).takeWhile { case (name, _) => !name.startsWith("snapshot_") }.map(_._2).sum - bridgeDefinitions += s"`define AB_RET_SNAPSHOT_OFFSET $snapshotStart" - bridgeLayout("NPU", "io_npuCommand_bits_", "npu.bits", new framework.system.core.rocket.RoCCCommandBB, true) - for ((port, signal) <- Seq("command" -> "command", "retirement" -> "retirement", "npuCommand" -> "npu")) { - bridgePorts += s".io_${port}_valid($signal.valid)" - bridgePorts += s".io_${port}_ready($signal.ready)" - } - for (i <- 0 until 3) { - bridgeLayout("SNAP", s"io_lookup_${i}_snapshot_bits_", s"control.lookup_bits[$i]", bridgeSnapshot, false) - bridgePorts += s".io_lookup_${i}_snapshot_valid(control.lookup_valid[$i])" - bridgePorts += s".io_lookup_${i}_robId(control.lookup_id[$i])" - } - bridgePorts += ".io_lookupTag_tag(8'd0)" - bridgePorts ++= Seq( - ".io_allocation_valid(control.allocation_valid)", - ".io_allocation_bits(control.allocation_id)", - ".io_retired(control.retired)" - ) - val instructionStart = - flatten(bridgeSnapshot).takeWhile { case (name, _) => !name.startsWith("instruction_") }.map(_._2).sum - bridgeDefinitions += s"`define AB_INSTRUCTION_OFFSET $instructionStart" - Files.writeString( - output.resolve("admission_bridge_config.svh"), - "`ifndef ADMISSION_BRIDGE_CONFIG\n`define ADMISSION_BRIDGE_CONFIG\n" + bridgeDefinitions.mkString( - "\n" - ) + "\n`endif\n" - ) - Files.writeString(output.resolve("admission_bridge_ports.svh"), bridgePorts.mkString(",\n") + "\n") - - val preparationConfig = memcore.memory.preflight.Params(bus = memory.chi, beatBytes = 64) - - val preparationRegions = Seq( - PhysicalRegion(BigInt("80000000", 16), BigInt(16) << 20, true, true, true, true, true, true), - PhysicalRegion(BigInt("90000000", 16), BigInt(4096), false, false, true, false, false, true) - ) - - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new PermissionVerification(preparationConfig, preparationRegions)(supervisorParameters), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("PermissionVerification").toString, "--split-verilog") - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Preparation(preparationConfig, preparationRegions)(supervisorParameters), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("Preparation").toString, "--split-verilog") - ) - -} diff --git a/arch/src/main/scala/framework/root/root-core/src/main/scala/rocket/PermissionVerification.scala b/arch/src/main/scala/framework/root/root-core/src/main/scala/rocket/PermissionVerification.scala deleted file mode 100644 index dc7afaa6d..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/scala/rocket/PermissionVerification.scala +++ /dev/null @@ -1,39 +0,0 @@ -package hier.core.rocket - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.system.core.rocket.{CpuParams, HasCpuParameters} -import freechips.rocketchip.rocket.PMPConfig -import memcore.memory.cpu.PhysicalRegion -import memcore.memory.preflight.{Authorization, Permission, Params => PreparationParams} -import chisel3.util.Decoupled - -@instantiable -class PermissionVerification(config: PreparationParams, regions: Seq[PhysicalRegion])(implicit val cpuParams: CpuParams) - extends Module - with HasCpuParameters { - require(nPMPs == 4) - - @public - val io = IO(new Bundle { - val request = Flipped(Decoupled(new Authorization(config))) - val response = Decoupled(new Permission(config)) - val contextValid = Input(Bool()) - val contextId = Input(UInt(config.idBits.W)) - val pmpConfig = Input(UInt(32.W)) - // PMP address CSRs expressed as CSR << 2, including their NAPOT encoding. - val pmpAddresses = Input(Vec(4, UInt(64.W))) - }) - - val permission: Instance[PermissionCheck] = Instantiate(new PermissionCheck(config, regions)) - permission.io.request <> io.request - io.response <> permission.io.response - permission.io.contextValid := io.contextValid - permission.io.contextId := io.contextId - for (i <- 0 until 4) { - val pmp = permission.io.pmp(i) - pmp.cfg := io.pmpConfig(8 * i + 7, 8 * i).asTypeOf(new PMPConfig) - pmp.addr := io.pmpAddresses(i)(paddrBits - 1, 2) - pmp.mask := pmp.computeMask - } -} diff --git a/arch/src/main/scala/framework/root/root-core/src/main/scala/rocket/Verification.scala b/arch/src/main/scala/framework/root/root-core/src/main/scala/rocket/Verification.scala deleted file mode 100644 index cbfd4aa17..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/scala/rocket/Verification.scala +++ /dev/null @@ -1,98 +0,0 @@ -package hier.core.rocket - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.system.core.rocket.{CpuParams, HasCpuParameters} -import freechips.rocketchip.tile.TraceBundle -import memcore.memory.cpu.{CpuMemParams, PhysicalRegion, UncachedRequest, UncachedResponse} -import memcore.memory.coherence.Coherence -import memcore.memory.coherence.configs.CoherenceParams -import memcore.bus.chi.Fabric -import memcore.bus.chi.rnf.RnfParams -import memcore.bus.chi.snf.{LineRequest, LineResponse} - -/** Verification system: the production Core's data and instruction L1s share a canonical Home. */ -@instantiable -class Verification( - memory: CoherenceParams, - regions: Seq[PhysicalRegion], - topName: String, - commands: Option[Commands] = None -)( - implicit val cpuParams: CpuParams) - extends Module - with HasCpuParameters { - override def desiredName: String = topName - require(memory.agents == 2) - val cp = CpuMemParams(memory.chi, tagBits = 6) - - @public - val io = IO(new Bundle { - val timerInterrupt = Input(Bool()) - val softwareInterrupt = Input(Bool()) - val externalInterrupt = Input(Bool()) - val resetVector = Input(UInt(64.W)) - val time = Input(UInt(64.W)) - val memoryRequest = Decoupled(new LineRequest(memory.chi)) - val memoryResponse = Flipped(Decoupled(new LineResponse(memory.chi))) - val uncachedRequest = Decoupled(new UncachedRequest(cp)) - val uncachedResponse = Flipped(Decoupled(new UncachedResponse(cp))) - val trace = Output(new TraceBundle) - val retired = Output(Bool()) - val retiredPc = Output(UInt(64.W)) - val trapped = Output(Bool()) - val trapCause = Output(UInt(64.W)) - val trapValue = Output(UInt(64.W)) - val trapPc = Output(UInt(64.W)) - val cancelledData = Output(Bool()) - val memoryOutstanding = Output(UInt(log2Ceil(memory.mshrEntries + 1).W)) - val blockMaintenanceResponse = commands.map(_ => Input(Bool())) - val maintenanceResponsePending = commands.map(_ => Output(Bool())) - val admission = commands.map(mode => new AdmissionPorts(mode.tracking, cpuParams.core.nPMPs, memory.chi)) - }) - - val config = RnfParams(memory.chi, nodeId = 1, cacheLines = 8, homeId = memory.homeId, homeCount = 1, banks = 2) - val core: Instance[Core] = Instantiate(new Core(config, config.copy(nodeId = 2), regions, commands)) - val home: Instance[Coherence] = Instantiate(new Coherence(memory)) - val fabric: Instance[Fabric] = Instantiate(new Fabric(memory.chi, memory.agents, memory.homeId)) - commands.foreach(_ => io.admission.get <> core.clink.admission.get) - core.clink.resetVector := io.resetVector - core.clink.time := io.time - core.clink.timerInterrupt := io.timerInterrupt - core.clink.softwareInterrupt := io.softwareInterrupt - core.clink.externalInterrupt := io.externalInterrupt - core.clink.supervisorExternalInterrupt := false.B - core.clink.hartId := 0.U - io.uncachedRequest <> core.clink.uncachedRequest - core.clink.uncachedResponse <> io.uncachedResponse - io.trace := core.clink.trace - io.retired := core.clink.retired - io.retiredPc := core.clink.retiredPc - io.trapped := core.clink.trapped - io.trapCause := core.clink.trapCause - io.trapValue := core.clink.trapValue - io.trapPc := core.clink.trapPc - io.cancelledData := core.clink.cancelledData - fabric.io.active := true.B - fabric.io.blockRequesterRsp := false.B - fabric.io.requesters(0) <> core.clink.chi - commands.foreach { _ => - val response = fabric.io.requesters(0).rxRsp - val isMaintenance = response.bits.txnId === config.banks.U - val hold = io.blockMaintenanceResponse.get && isMaintenance - core.clink.chi.rxRsp.valid := response.valid && !hold - response.ready := core.clink.chi.rxRsp.ready && !hold - io.maintenanceResponsePending.get := response.valid && isMaintenance - } - fabric.io.requesters(1) <> core.clink.instructionChi - home.io.req <> fabric.io.req - home.io.rxRsp <> fabric.io.rxRsp - home.io.rxDat <> fabric.io.rxDat - fabric.io.rsp <> home.io.rsp - fabric.io.dat <> home.io.dat - fabric.io.snp <> home.io.snp - io.memoryRequest <> home.io.memoryReq - home.io.memoryResp <> io.memoryResponse - io.memoryOutstanding := home.io.outstanding -} diff --git a/arch/src/main/scala/framework/root/root-tile/src/csrc/Cargo.lock b/arch/src/main/scala/framework/root/root-tile/src/csrc/Cargo.lock deleted file mode 100644 index e2fc5f03e..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/csrc/Cargo.lock +++ /dev/null @@ -1,7 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "tile-memory-reference-model" -version = "0.1.0" diff --git a/arch/src/main/scala/framework/root/root-tile/src/csrc/Cargo.toml b/arch/src/main/scala/framework/root/root-tile/src/csrc/Cargo.toml deleted file mode 100644 index 445a6b832..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/csrc/Cargo.toml +++ /dev/null @@ -1,8 +0,0 @@ -[package] -name = "tile-memory-reference-model" -version = "0.1.0" -edition = "2021" - -[lib] -crate-type = ["cdylib"] -path = "src/lib.rs" diff --git a/arch/src/main/scala/framework/root/root-tile/src/csrc/README.md b/arch/src/main/scala/framework/root/root-tile/src/csrc/README.md deleted file mode 100644 index e69de29bb..000000000 diff --git a/arch/src/main/scala/framework/root/root-tile/src/csrc/src/consistency.rs b/arch/src/main/scala/framework/root/root-tile/src/csrc/src/consistency.rs deleted file mode 100644 index 04b5b4f44..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/csrc/src/consistency.rs +++ /dev/null @@ -1,80 +0,0 @@ -//! Physical DDR and CPU-visible architectural memory are distinct golden stores. -//! CPU dirty writes do not commit DDR. Only observed Home write ACKs or the mock DMA do. -use std::{collections::BTreeMap, ffi::c_void}; -struct Memory { - ddr: BTreeMap, - visible: BTreeMap, -} -fn initial(address: u64) -> [u8; 64] { - let mut line = [0; 64]; - for (i, byte) in line.iter_mut().enumerate() { - *byte = ((address >> 6) as u8).wrapping_mul(19).wrapping_add((i as u8).wrapping_mul(13)) ^ 0x5a; - } - line -} -fn line(map: &BTreeMap, address: u64) -> [u8; 64] { - *map.get(&address).unwrap_or(&initial(address)) -} -fn write_word(map: &mut BTreeMap, address: u64, data: u64, mask: u32) { - assert!(address < 1 << 44 && address & 7 == 0); - let base = address & !63; - let target = map.entry(base).or_insert_with(|| initial(base)); - for (i, byte) in data.to_le_bytes().iter().enumerate() { - if mask & (1 << i) != 0 { target[(address & 63) as usize + i] = *byte; } - } -} -unsafe fn memory<'a>(ptr: *mut c_void) -> &'a mut Memory { &mut *ptr.cast::() } -#[no_mangle] -pub extern "C" fn cons_ref_create() -> *mut c_void { - Box::into_raw(Box::new(Memory { ddr: BTreeMap::new(), visible: BTreeMap::new() })).cast() -} -#[no_mangle] -pub unsafe extern "C" fn cons_ref_destroy(ptr: *mut c_void) { drop(Box::from_raw(ptr.cast::())); } -#[no_mangle] -pub unsafe extern "C" fn cons_ref_expected64(ptr: *mut c_void, address: u64) -> u64 { - assert!(address & 7 == 0); - let bytes = line(&memory(ptr).visible, address & !63); - u64::from_le_bytes(bytes[(address & 63) as usize..(address & 63) as usize + 8].try_into().unwrap()) -} -#[no_mangle] -pub unsafe extern "C" fn cons_ref_cpu_commit(ptr: *mut c_void, address: u64, data: u64, mask: u32) { - write_word(&mut memory(ptr).visible, address, data, mask); -} -#[no_mangle] -pub unsafe extern "C" fn cons_ref_ddr_read(ptr: *mut c_void, address: u64, output: *mut u32) { - assert!(address & 63 == 0 && address < 1 << 44); - let bytes = line(&memory(ptr).ddr, address); - for (i, word) in std::slice::from_raw_parts_mut(output, 16).iter_mut().enumerate() { - *word = u32::from_le_bytes(bytes[i * 4..i * 4 + 4].try_into().unwrap()); - } -} -#[no_mangle] -pub unsafe extern "C" fn cons_ref_write_matches(ptr: *mut c_void, address: u64, data: *const u32, mask: *const u32) -> u32 { - assert!(address & 63 == 0 && address < 1 << 44); - let wanted = line(&memory(ptr).visible, address); - let data = std::slice::from_raw_parts(data, 16); - let mask = std::slice::from_raw_parts(mask, 2); - u32::from((0..64).all(|i| mask[i / 32] & (1 << (i % 32)) == 0 || wanted[i] == (data[i / 4] >> (8 * (i % 4))) as u8)) -} -#[no_mangle] -pub unsafe extern "C" fn cons_ref_ddr_commit(ptr: *mut c_void, address: u64, data: *const u32, mask: *const u32) { - assert!(address & 63 == 0 && address < 1 << 44); - let data = std::slice::from_raw_parts(data, 16); - let mask = std::slice::from_raw_parts(mask, 2); - let target = memory(ptr).ddr.entry(address).or_insert_with(|| initial(address)); - for (i, byte) in target.iter_mut().enumerate() { - if mask[i / 32] & (1 << (i % 32)) != 0 { *byte = (data[i / 4] >> (8 * (i % 4))) as u8; } - } -} -#[no_mangle] -pub unsafe extern "C" fn cons_ref_ddr_visible(ptr: *mut c_void, address: u64) -> u32 { - assert!(address & 63 == 0); - let m = memory(ptr); - u32::from(line(&m.ddr, address) == line(&m.visible, address)) -} -#[no_mangle] -pub unsafe extern "C" fn cons_ref_dma_write(ptr: *mut c_void, address: u64, data: u64, mask: u32) { - let m = memory(ptr); - write_word(&mut m.ddr, address, data, mask); - write_word(&mut m.visible, address, data, mask); -} diff --git a/arch/src/main/scala/framework/root/root-tile/src/csrc/src/lib.rs b/arch/src/main/scala/framework/root/root-tile/src/csrc/src/lib.rs deleted file mode 100644 index e1552417d..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/csrc/src/lib.rs +++ /dev/null @@ -1,15 +0,0 @@ -// Reuse the actual leaf reference implementations, including their exported DPI API. -#[path = "../../../../../mem-core/coherence/src/csrc/src/memory.rs"] -mod line_memory; -#[path = "../../../../../mem-core/mmu/src/csrc/src/translation.rs"] -mod mmu; -#[path = "../../../../../mem-core/cpu_mem/src/csrc/src/access.rs"] -mod cpu_mem; - -#[path = "../../../../root-core/src/csrc/src/memory.rs"] -mod core_memory; - -mod consistency; - -#[path = "../../../../../mem-core/ddr/src/csrc/src/memory.rs"] -mod ddr_memory; diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/README.md b/arch/src/main/scala/framework/root/root-tile/src/main/resources/README.md deleted file mode 100644 index e69de29bb..000000000 diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/cache_system.f b/arch/src/main/scala/framework/root/root-tile/src/main/resources/cache_system.f deleted file mode 100644 index f75e2acf5..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/cache_system.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/cache_system_interface.sv -@RESOURCES@/cache_system_package.sv --F @RTL@/CacheSystem/filelist.f -@RESOURCES@/cache_system_tb.sv diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/cache_system_interface.sv b/arch/src/main/scala/framework/root/root-tile/src/main/resources/cache_system_interface.sv deleted file mode 100644 index 3c33cdf15..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/cache_system_interface.sv +++ /dev/null @@ -1,35 +0,0 @@ -`include "cache_system_config.svh" -interface cache_system_if ( - input logic clock -); - logic reset, active, block_requester_rsp; - logic access_valid[2], access_ready[2], access_write[2], access_atomicWord[2]; - logic [`COH_REQ_ADDR_WIDTH-1:0] access_addr[2]; - logic [63:0] access_data[2], result_data[2]; - logic [7:0] access_mask [2]; - logic [3:0] access_atomic[2]; - logic result_valid[2], result_ready[2], result_error[2]; - logic [`COH_OUTSTANDING_BITS-1:0] outstanding; - logic req_valid, rsp_valid, dat_valid, snp_valid, rx_rsp_valid, rx_dat_valid; - logic [`COH_REQ_WIDTH-1:0] req_bits; - logic [`COH_RSP_WIDTH-1:0] rsp_bits, rx_rsp_bits; - logic [`COH_DAT_WIDTH-1:0] dat_bits, rx_dat_bits; - logic [`COH_SNP_CHANNEL_WIDTH-1:0] snp_bits; - clocking sample @(posedge clock); - default input #1step; - input reset,active,block_requester_rsp,access_atomicWord,access_valid,access_ready,access_write,access_addr,access_data,access_mask,access_atomic; - input result_valid, result_ready, result_error, result_data, outstanding; - input req_valid, rsp_valid, dat_valid, snp_valid, rx_rsp_valid, rx_dat_valid; - input req_bits, rsp_bits, dat_bits, snp_bits, rx_rsp_bits, rx_dat_bits; - endclocking - for (genvar core = 0; core < 2; core++) begin - assert property (@(posedge clock) disable iff(reset) access_valid[core]&&!access_ready[core] |=> access_valid[core]&&$stable( - {access_addr[core],access_write[core],access_data[core],access_mask[core],access_atomic[core],access_atomicWord[core]} - )) - else $fatal(1, "CacheSystem client changed stalled command"); - assert property (@(posedge clock) disable iff(reset) result_valid[core]&&!result_ready[core] |=> result_valid[core]&&$stable( - {result_data[core], result_error[core]} - )) - else $fatal(1, "CacheSystem changed stalled result"); - end -endinterface diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/cache_system_package.sv b/arch/src/main/scala/framework/root/root-tile/src/main/resources/cache_system_package.sv deleted file mode 100644 index 1cfdac6a0..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/cache_system_package.sv +++ /dev/null @@ -1,606 +0,0 @@ -`include "cache_system_config.svh" -`define CSF(K, F) `COH_``K``_``F``_OFFSET +: `COH_``K``_``F``_WIDTH -package cache_system_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - typedef logic [`COH_REQ_WIDTH-1:0] req_t; - typedef logic [`COH_RSP_WIDTH-1:0] rsp_t; - typedef logic [`COH_DAT_WIDTH-1:0] dat_t; - typedef bit [`COH_MEMRESP_WIDTH-1:0] mem_t; - import "DPI-C" function chandle coherence_ref_create(); - import "DPI-C" function void coherence_ref_destroy(input chandle model); - import "DPI-C" function void coherence_ref_read( - input chandle model, - input longint unsigned addr, - output bit [511:0] data - ); - import "DPI-C" function void coherence_ref_write( - input chandle model, - input longint unsigned addr, - input bit [511:0] data - ); - class result_item extends uvm_sequence_item; - bit [63:0] data; - bit error; - `uvm_object_utils_begin(result_item) - `uvm_field_int(data, UVM_DEFAULT) - `uvm_field_int(error, UVM_DEFAULT) - `uvm_object_utils_end - function new(string name = "result_item"); - super.new(name); - endfunction - endclass - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual cache_system_if vif; - virtual stream_if #(`COH_MEMREQ_WIDTH) mem_req; - virtual stream_if #(`COH_MEMRESP_WIDTH) mem_resp; - in_order_scoreboard #(result_item) scoreboard[2]; - chandle architectural, backing; - typedef struct packed { - bit write; - bit [63:0] addr, data; - bit [7:0] mask; - } command_t; - typedef command_t command_queue[$]; - command_queue commands[2]; - typedef struct { - mem_t bits; - int due, serial; - } memory_entry; - memory_entry pending[$]; - req_t requests[int]; - bit [511:0] read_expected[int], snoop_expected[int]; - longint unsigned wb_address[int]; - int - fill_seen[int], - fill_dbid[int], - snoop_seen[int], - snoop_target[int], - wb_seen[int], - wb_owner[int], - slot_owner[int]; - bit ack_pending[int]; - int submitted[2] = '{0, 0}; - int result_stalls[2] = '{0, 0}; - int home_requests = 0, rsp_block_cycles = 0, release_cycle = -1, ack_resume_cycles = -1; - int - cycle = 0, - peak = 0, - memory_reads = 0, - memory_writes = 0, - memory_serial = 0, - last_memory_serial = -1; - int full_lines = 0, snoop_lines = 0, copybacks = 0, dirty_snoops = 0, acks = 0, errors = 0; - bit - hold_memory = 0, - hold_results = 0, - mem_active = 0, - saw_reorder = 0, - saw_dbid_mismatch = 0, - saw_large_snoop = 0; - bit [`COH_MEMRESP_WIDTH-1:0] mem_packet; - localparam bit [63:0] ERROR_ADDRESS = 64'hf000; - function new(string name, uvm_component parent); - super.new(name, parent); - timeout = 1ms; - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if(`COH_AGENTS!=2||`COH_MSHRS!=4||`COH_SETS!=4||`COH_WAYS!=2||`COH_DATA_BITS!=256) - `uvm_fatal( - "CONFIG", - "CacheSystem regression requires 2 requesters, 4 MSHRs/sets, 2 ways and 256-bit DAT") - if (!uvm_config_db#(virtual cache_system_if)::get( - this, "", "control", vif - ) || !uvm_config_db#(virtual stream_if #(`COH_MEMREQ_WIDTH))::get( - this, "", "mem_req", mem_req - ) || !uvm_config_db#(virtual stream_if #(`COH_MEMRESP_WIDTH))::get( - this, "", "mem_resp", mem_resp - )) - `uvm_fatal("VIF", "CacheSystem interfaces missing") - for (int i = 0; i < 2; i++) - scoreboard[i] = - in_order_scoreboard#(result_item)::type_id::create($sformatf("scoreboard_%0d", i), this); - architectural = coherence_ref_create(); - backing = coherence_ref_create(); - endfunction - function void observe_cpu(); - for (int core = 0; core < 2; core++) begin - if (vif.sample.access_valid[core] && vif.sample.access_ready[core]) begin - command_t cmd; - bit [511:0] line; - result_item expected; - cmd = '{ - vif.sample.access_write[core], - 64'(vif.sample.access_addr[core]), - vif.sample.access_data[core], - vif.sample.access_mask[core] - }; - if (vif.sample.access_atomicWord[core] !== 1'b0) - `uvm_fatal("PROFILE", "This regression drives atomicWord=False") - commands[core].push_back(cmd); - coherence_ref_read(architectural, cmd.addr & ~64'h3f, line); - expected = new; - expected.error = cmd.addr == ERROR_ADDRESS; - expected.data = expected.error ? 0 : line[int'(cmd.addr&63)*8+:64]; - scoreboard[core].expected_export.write(expected); - submitted[core]++; - end - if (vif.sample.result_valid[core] && vif.sample.result_ready[core]) begin - command_t cmd; - bit [511:0] line; - result_item actual; - if ($isunknown({vif.sample.result_data[core], vif.sample.result_error[core]})) - `uvm_fatal("RESULT_X", "Unknown CPU result") - if (!commands[core].size()) - `uvm_fatal("EXTRA_RESULT", "CPU result without an accepted command") - cmd = commands[core].pop_front(); - actual = new; - actual.data = vif.sample.result_data[core]; - actual.error = vif.sample.result_error[core]; - scoreboard[core].actual_export.write(actual); - if (actual.error) errors++; - if (cmd.write && !actual.error) begin - coherence_ref_read(architectural, cmd.addr & ~64'h3f, line); - for (int b = 0; b < 8; b++) - if (cmd.mask[b]) line[(int'(cmd.addr&63)+b)*8+:8] = cmd.data[b*8+:8]; - coherence_ref_write(architectural, cmd.addr & ~64'h3f, line); - end - end - end - endfunction - function void observe_protocol(); - if (vif.sample.req_valid && $isunknown(vif.sample.req_bits)) - `uvm_fatal("REQ_X", "Unknown accepted RequestFlit") - if (vif.sample.rsp_valid && $isunknown(vif.sample.rsp_bits)) - `uvm_fatal("RSP_X", "Unknown accepted ResponseFlit") - if (vif.sample.dat_valid && $isunknown(vif.sample.dat_bits)) - `uvm_fatal("DAT_X", "Unknown accepted DataFlit") - if (vif.sample.snp_valid && $isunknown(vif.sample.snp_bits)) - `uvm_fatal("SNP_X", "Unknown accepted DirectedSnoop") - if (vif.sample.rx_rsp_valid && $isunknown(vif.sample.rx_rsp_bits)) - `uvm_fatal("RX_RSP_X", "Unknown requester response") - if (vif.sample.rx_dat_valid && $isunknown(vif.sample.rx_dat_bits)) - `uvm_fatal("RX_DAT_X", "Unknown requester data") - if (vif.sample.req_valid) begin - req_t r; - int key; - home_requests++; - r = vif.sample.req_bits; - key = (int'(r[`CSF(REQ, SRCID)]) << 1) | int'(r[`CSF(REQ, TXNID)]); - if (r[ - `CSF(REQ, TGTID) - ] != 64 || r[ - `CSF(REQ, SRCID) - ] < 1 || r[ - `CSF(REQ, SRCID) - ] > 2 || r[ - `CSF(REQ, TXNID) - ] >= 2) - `uvm_fatal("REQ_ID", $sformatf( - "Requester identity or bank TxnID corrupted src=%0d tgt=%0d txn=%0d packet=%h", - r[ - `CSF(REQ, SRCID) - ], - r[ - `CSF(REQ, TGTID) - ], - r[ - `CSF(REQ, TXNID) - ], - r - )) - requests[key] = r; - if (r[`CSF(REQ, OPCODE)] == `COH_READ_UNIQUE || r[`CSF(REQ, OPCODE)] == `COH_READ_NSD) begin - coherence_ref_read(architectural, r[`CSF(REQ, ADDR)], read_expected[key]); - fill_seen[key] = 0; - end - end - if (vif.sample.rsp_valid) begin - rsp_t r; - int key, id; - r = vif.sample.rsp_bits; - key = (int'(r[`CSF(RSP, TGTID)]) << 1) | int'(r[`CSF(RSP, TXNID)]); - id = r[`CSF(RSP, DBID)]; - if (!requests.exists(key) || r[`CSF(RSP, SRCID)] != 64 || r[`CSF(RSP, RESPERR)] != 0) - `uvm_fatal("RSP_ID", "Unexpected completion identity or error") - if (r[`CSF(RSP, OPCODE)] == `COH_COMP_DBID) begin - if (id >= 4 || requests[key][`CSF(REQ, OPCODE)] != `COH_WRITEBACK) - `uvm_fatal("WB_ID", "Writeback grant does not match request") - wb_address[id] = requests[key][`CSF(REQ, ADDR)]; - wb_owner[id] = r[`CSF(RSP, TGTID)]; - wb_seen[id] = 0; - end else if (r[`CSF(RSP, OPCODE)] != `COH_COMP) - `uvm_fatal("RETRY", "Canonical Home profile must backpressure REQ, not generate Retry") - end - if (vif.sample.dat_valid) begin - dat_t d; - int key, b, id; - bit failed; - bit [255:0] expected; - d = vif.sample.dat_bits; - key = (int'(d[`CSF(DAT, TGTID)]) << 1) | int'(d[`CSF(DAT, TXNID)]); - id = d[`CSF(DAT, DBID)]; - b = int'(d[`CSF(DAT, DATAID)]) >> 1; - failed = d[`CSF(DAT, RESPERR)] != 0; - if (!requests.exists( - key - ) || !read_expected.exists( - key - ) || id >= 4 || d[ - `CSF(DAT, SRCID) - ] != 64 || d[ - `CSF(DAT, HOMENID) - ] != 64 || d[ - `CSF(DAT, OPCODE) - ] != `COH_COMP_DATA) - `uvm_fatal("DAT_ID", "CompData bank TxnID, HomeNID or DBID mismatch") - if (d[`CSF(DAT, DATAID)] != 0 && d[`CSF(DAT, DATAID)] != 2) - `uvm_fatal("DATAID", "Invalid 256-bit DataID") - if (fill_seen[key] & (1 << b)) `uvm_fatal("DUPLICATE", "Duplicate CompData beat") - if (fill_seen[key] && fill_dbid[key] != id) - `uvm_fatal("DBID", "CompData DBID changed within line") - if (failed != (requests[key][`CSF(REQ, ADDR)] == ERROR_ADDRESS)) - `uvm_fatal("ERROR", "Unexpected backend error status") - expected = failed ? '0 : read_expected[key][b*256+:256]; - if (d[`CSF(DAT, DATA)] !== expected || d[`CSF(DAT, BE)] !== (failed ? 32'h0 : 32'hffffffff)) - `uvm_fatal("LINE_DATA", "Full 64-byte CompData payload/BE mismatch") - if (failed && d[`CSF(DAT, RESPERR)] != 3) `uvm_fatal("NDERR", "Read fault was not NDERR") - if (!fill_seen[key]) begin - fill_dbid[key] = id; - slot_owner[id] = key; - end - fill_seen[key] |= 1 << b; - if (id != d[`CSF(DAT, TXNID)]) saw_dbid_mismatch = 1; - if (fill_seen[key] == 3) begin - full_lines++; - ack_pending[id] = 1; - end - end - if (vif.sample.snp_valid) begin - int id, target; - bit [63:0] address; - id = vif.sample.snp_bits[`CSF(SNP, TXNID)]; - target = vif.sample.snp_bits[`CSF(SNP, TARGET)]; - address = 64'(vif.sample.snp_bits[`CSF(SNP, ADDR)]) << 3; - if (id >= 4 || target < 1 || target > 2 || vif.sample.snp_bits[`CSF(SNP, SRCID)] != 64) - `uvm_fatal("SNP_ID", "Directed snoop lost target/slot/address semantics") - coherence_ref_read(architectural, address, snoop_expected[id]); - snoop_seen[id] = 0; - snoop_target[id] = target; - if (id >= 2) saw_large_snoop = 1; - end - if (vif.sample.rx_rsp_valid) begin - rsp_t r; - int id; - r = vif.sample.rx_rsp_bits; - id = r[`CSF(RSP, TXNID)]; - if (r[`CSF(RSP, OPCODE)] == `COH_COMP_ACK) begin - if (!ack_pending.exists( - id - ) || !ack_pending[id] || slot_owner[id] >> 1 != r[ - `CSF(RSP, SRCID) - ] || r[ - `CSF(RSP, TGTID) - ] != 64) - `uvm_fatal("ACK_ID", "CompAck must return the Home DBID, not original bank TxnID") - ack_pending[id] = 0; - acks++; - end else begin - if (r[ - `CSF(RSP, OPCODE) - ] != `COH_SNP_RESP || !snoop_target.exists( - id - ) || r[ - `CSF(RSP, SRCID) - ] != snoop_target[id] || r[ - `CSF(RSP, TGTID) - ] != 64) - `uvm_fatal("RX_RSP", "Snoop response identity mismatch") - end - end - if (vif.sample.rx_dat_valid) begin - dat_t d; - int id, b; - bit [511:0] expected; - d = vif.sample.rx_dat_bits; - id = d[`CSF(DAT, TXNID)]; - b = int'(d[`CSF(DAT, DATAID)]) >> 1; - if (id >= 4 || (d[`CSF(DAT, DATAID)] != 0 && d[`CSF(DAT, DATAID)] != 2)) - `uvm_fatal("RETURN_ID", "Snoop/writeback must return full Home slot ID") - if (d[`CSF(DAT, OPCODE)] == `COH_SNP_DATA) begin - if (!snoop_expected.exists( - id - ) || d[ - `CSF(DAT, SRCID) - ] != snoop_target[id] || d[ - `CSF(DAT, TGTID) - ] != 64 || (snoop_seen[id] & (1 << b))) - `uvm_fatal("SNP_DATA", "Unmatched or duplicate snoop data") - expected = snoop_expected[id]; - snoop_seen[id] |= 1 << b; - if (snoop_seen[id] == 3) begin - snoop_lines++; - if (d[`CSF(DAT, RESP)] & 4) dirty_snoops++; - end - end else if (d[`CSF(DAT, OPCODE)] == `COH_COPYBACK_DATA) begin - if (!wb_address.exists( - id - ) || d[ - `CSF(DAT, SRCID) - ] != wb_owner[id] || d[ - `CSF(DAT, TGTID) - ] != 64 || (wb_seen[id] & (1 << b))) - `uvm_fatal("WB_DATA", "Unmatched or duplicate copyback") - coherence_ref_read(architectural, wb_address[id], expected); - wb_seen[id] |= 1 << b; - if (wb_seen[id] == 3) copybacks++; - if (d[`CSF(DAT, RESP)] == 0) expected = '0; - end else `uvm_fatal("RETURN_OPCODE", "Unexpected requester data opcode") - if (d[`CSF(DAT, DATA)] !== expected[b*256+:256]) - `uvm_fatal("RETURN_PAYLOAD", "Full-line snoop/copyback data mismatch") - end - endfunction - task service(); - forever begin - @(vif.sample); - if (!vif.sample.reset) begin - cycle++; - if (vif.sample.outstanding > peak) peak = vif.sample.outstanding; - observe_cpu(); - observe_protocol(); - for (int core = 0; core < 2; core++) - if (vif.sample.result_valid[core] && !vif.sample.result_ready[core]) - result_stalls[core]++; - if (vif.sample.block_requester_rsp) begin - rsp_block_cycles++; - if (!vif.sample.active || vif.sample.rx_rsp_valid) - `uvm_fatal("RSP_BLOCK", "RSP stall changed active or allowed a blocked response") - end - if (release_cycle >= 0 && ack_resume_cycles < 0 && acks >= 4) - ack_resume_cycles = cycle - release_cycle; - if (mem_active && mem_resp.sample.valid && mem_resp.sample.ready) mem_active = 0; - if (mem_req.sample.valid && mem_req.sample.ready) begin - memory_entry entry; - bit [511:0] line, expected; - bit write; - longint unsigned addr; - if ($isunknown( - { - mem_req.sample.bits[`CSF(MEMREQ, ID)], - mem_req.sample.bits[`CSF(MEMREQ, ADDR)], - mem_req.sample.bits[`CSF(MEMREQ, WRITE)] - } - )) - `uvm_fatal("MEM_X", "Unknown backing command") - entry.bits = '0; - entry.bits[`CSF(MEMRESP, ID)] = mem_req.sample.bits[`CSF(MEMREQ, ID)]; - addr = mem_req.sample.bits[`CSF(MEMREQ, ADDR)]; - write = mem_req.sample.bits[`CSF(MEMREQ, WRITE)]; - if (addr[5:0] != 0) `uvm_fatal("MEM_ALIGN", "Backing request not line aligned") - foreach (pending[i]) - if (pending[i].bits[`CSF(MEMRESP, ID)] == entry.bits[`CSF(MEMRESP, ID)]) - `uvm_fatal("MEM_ID", "Reused live backing ID") - if (write) begin - if ($isunknown( - { - mem_req.sample.bits[`CSF(MEMREQ, DATA)], - mem_req.sample.bits[`CSF(MEMREQ, MASK)] - } - )) - `uvm_fatal("MEM_WRITE_X", "Unknown backing write payload") - line = mem_req.sample.bits[`CSF(MEMREQ, DATA)]; - coherence_ref_read(architectural, addr, expected); - if (mem_req.sample.bits[`CSF(MEMREQ, MASK)] !== '1 || line !== expected) - `uvm_fatal("MEM_WRITE", "Backing write differs from architectural full line") - coherence_ref_write(backing, addr, line); - memory_writes++; - end else begin - coherence_ref_read(backing, addr, line); - memory_reads++; - end - entry.bits[`CSF(MEMRESP, DATA)] = line; - entry.bits[`CSF(MEMRESP, ERROR)] = !write && addr == ERROR_ADDRESS; - entry.due = cycle + 9 + (3 - int'(entry.bits[`CSF(MEMRESP, ID)])) * 3; - entry.serial = memory_serial++; - pending.push_back(entry); - end - end - @(negedge vif.clock); - mem_req.ready = !vif.reset && cycle % 7 != 0; - for (int core = 0; core < 2; core++) - vif.result_ready[core] = !vif.reset && !hold_results && (cycle + core) % 11 >= 3; - if (!mem_active && !hold_memory) begin - for (int i = pending.size() - 1; i >= 0; i--) - if (pending[i].due <= cycle) begin - memory_entry entry; - entry = pending[i]; - pending.delete(i); - mem_packet = entry.bits; - mem_active = 1; - if (entry.serial < last_memory_serial) saw_reorder = 1; - last_memory_serial = entry.serial; - break; - end - end - mem_resp.valid = mem_active; - mem_resp.bits = mem_packet; - end - endtask - task send(int core, longint unsigned address, bit write = 0, bit [63:0] data = 0, - bit [7:0] mask = 8'hff); - @(negedge vif.clock); - vif.access_addr[core] = address; - vif.access_write[core] = write; - vif.access_data[core] = data; - vif.access_mask[core] = mask; - vif.access_atomic[core] = 0; - vif.access_atomicWord[core] = 0; - vif.access_valid[core] = 1; - do @(vif.sample); while (!vif.sample.access_ready[core]); - @(negedge vif.clock); - vif.access_valid[core] = 0; - endtask - task settle(); - for (int core = 0; core < 2; core++) scoreboard[core].wait_checked(submitted[core]); - do @(vif.sample); while (vif.sample.outstanding != 0 || pending.size() != 0 || mem_active); - repeat (8) @(vif.sample); - endtask - task execute(); - vif.reset = 1; - vif.active = 0; - vif.block_requester_rsp = 0; - mem_req.ready = 0; - mem_resp.valid = 0; - mem_resp.bits = '0; - for (int i = 0; i < 2; i++) begin - vif.access_valid[i] = 0; - vif.access_addr[i] = 0; - vif.access_write[i] = 0; - vif.access_data[i] = 0; - vif.access_mask[i] = '1; - vif.access_atomic[i] = 0; - vif.access_atomicWord[i] = 0; - vif.result_ready[i] = 0; - end - repeat (5) @(negedge vif.clock); - vif.reset = 0; - vif.active = 1; - fork - service(); - join_none - repeat (12) @(vif.sample); - // Completion ACKs remain in the physical Rx FIFOs while DAT and CPU result - // channels make progress. Hold CPU results first, then issue more REQs while - // all four Home slots still wait for their ACKs. - hold_memory = 1; - hold_results = 1; - @(negedge vif.clock); - vif.block_requester_rsp = 1; - fork - begin - send(0, 'h0); - send(0, 'h40); - end - begin - send(1, 'h80); - send(1, 'hc0); - end - join - wait (pending.size() == 4); - if (peak != 4) - `uvm_fatal("CONCURRENCY", "Four independent misses did not occupy all Home MSHRs") - hold_memory = 0; - wait (vif.sample.result_valid[0] && vif.sample.result_valid[1]); - repeat (24) begin - @(vif.sample); - if(vif.sample.outstanding!=4||acks!=0||scoreboard[0].checked!=0||scoreboard[1].checked!=0) - `uvm_fatal("ACK_LIFETIME", "Blocked ACK or CPU result prematurely retired") - end - hold_results = 0; - scoreboard[0].wait_checked(2); - scoreboard[1].wait_checked(2); - repeat (24) begin - @(vif.sample); - if (vif.sample.outstanding != 4 || acks != 0 || full_lines != 4) - `uvm_fatal("ACK_LIFETIME", "Home released a slot before receiving CompAck") - end - fork - send(0, 'h800); - send(1, 'h880); - join - repeat (16) begin - @(vif.sample); - if(vif.sample.outstanding!=4||home_requests!=4||scoreboard[0].checked!=2||scoreboard[1].checked!=2) - `uvm_fatal("REQ_PRESSURE", "New request advanced while all slots awaited blocked ACKs") - end - @(negedge vif.clock); - release_cycle = cycle; - vif.block_requester_rsp = 0; - begin - bit drained = 0; - for (int elapsed = 0; elapsed < 256; elapsed++) begin - @(vif.sample); - if(scoreboard[0].checked==submitted[0]&&scoreboard[1].checked==submitted[1]&& - vif.sample.outstanding==0&&pending.size()==0&&!mem_active)begin - drained = 1; - break; - end - end - if(!drained||ack_resume_cycles<0||ack_resume_cycles>32||result_stalls[0]<20||result_stalls[1]<20) - `uvm_fatal("ACK_PROGRESS", - "CompAck/blocked REQ/result channels did not drain within the progress bound") - end - settle(); - `uvm_info( - "BACKPRESSURE", - $sformatf( - "4 MSHRs held for CompAck with live credit links; CPU stall cycles=%0d/%0d RSP blocked=%0d first4ACK resume=%0d cycles; pending REQs and all results drained", - result_stalls[0], result_stalls[1], rsp_block_cycles, ack_resume_cycles), UVM_LOW) - // Prime two RN1 lines, then keep slots0/1 waiting for unrelated backing reads. - send(0, 'h80); - settle(); - send(0, 'hc0); - settle(); - hold_memory = 1; - send(0, 'h400); - send(0, 'h440); - wait (pending.size() == 2); - send(1, 'h80, 1, 64'h0123456789abcdef); - send(1, 'hc0, 1, 64'h8877665544332211); - wait (scoreboard[1].checked == submitted[1]); - if (!saw_large_snoop) - `uvm_fatal("SNP_SLOT", "No snoop used a Home slot beyond RN bank-ID range") - hold_memory = 0; - settle(); - send(0, 'h80); - settle(); // Dirty owner at RN2 supplies the newest data and downgrades. - send(0, 'h80, 1, 64'hfedcba9876543210); - settle(); - send(0, 'h88, 1, 64'hdecafbad76543210, 8'h3c); - settle(); - send(1, 'h80); - settle(); - send(0, 'h80, 1, 64'h445566778899aabb); - settle(); - send(0, 'h280); - settle(); // Same private index: CopyBackWriteData into shared L2. - send(1, 'h480); - settle(); - send(1, 'h680); - settle(); // L2 same-set replacement flushes dirty data. - send(1, 'h80); - settle(); - send(0, ERROR_ADDRESS); - settle(); - if(!saw_dbid_mismatch||!saw_reorder||!dirty_snoops||!copybacks||!memory_writes||errors!=1||acks!=full_lines) - `uvm_fatal("SCENARIOS", $sformatf( - "dbid=%0b reorder=%0b dirty=%0d copybacks=%0d writes=%0d errors=%0d ack/lines=%0d/%0d", - saw_dbid_mismatch, - saw_reorder, - dirty_snoops, - copybacks, - memory_writes, - errors, - acks, - full_lines - )) - foreach (ack_pending[id]) - if (ack_pending[id]) `uvm_fatal("PENDING_ACK", "Unmatched completion acknowledgement") - for (int core = 0; core < 2; core++) - if (commands[core].size()) `uvm_fatal("PENDING_CPU", "Unmatched accepted CPU command") - `uvm_info( - "CHECKS", - $sformatf( - "canonical 2RN-F x2banks + shared L2: CPU checked=%0d/%0d full64B=%0d snoopLines=%0d dirtySnoops=%0d copybacks=%0d memoryWrites=%0d peakMSHR=%0d NDERR=%0d", - scoreboard[0].checked, scoreboard[1].checked, full_lines, snoop_lines, dirty_snoops, - copybacks, memory_writes, peak, errors), UVM_LOW) - endtask - function void final_phase(uvm_phase phase); - coherence_ref_destroy(architectural); - coherence_ref_destroy(backing); - endfunction - endclass -endpackage diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/cache_system_tb.sv b/arch/src/main/scala/framework/root/root-tile/src/main/resources/cache_system_tb.sv deleted file mode 100644 index 7e16d4ad2..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/cache_system_tb.sv +++ /dev/null @@ -1,32 +0,0 @@ -`include "cache_system_config.svh" -module cache_system_tb; - import uvm_pkg::*; - import cache_system_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - cache_system_if control (clock); - stream_if #(`COH_MEMREQ_WIDTH) mem_req ( - clock, - control.reset - ); - stream_if #(`COH_MEMRESP_WIDTH) mem_resp ( - clock, - control.reset - ); - CacheSystem dut ( - .clock(clock), - .reset(control.reset), - .io_active(control.active), - .io_blockRequesterRsp(control.block_requester_rsp), - .io_outstanding(control.outstanding), - `include "cache_system_ports.svh" - ); - initial begin - uvm_config_db#(virtual cache_system_if)::set(null, "uvm_test_top", "control", control); - uvm_config_db#(virtual stream_if #(`COH_MEMREQ_WIDTH))::set(null, "uvm_test_top", "mem_req", - mem_req); - uvm_config_db#(virtual stream_if #(`COH_MEMRESP_WIDTH))::set(null, "uvm_test_top", "mem_resp", - mem_resp); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/consistency.cm_hier b/arch/src/main/scala/framework/root/root-tile/src/main/resources/consistency.cm_hier deleted file mode 100644 index 140ec3277..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/consistency.cm_hier +++ /dev/null @@ -1,12 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.access_0 -+tree @TOP@.access_1 -+tree @TOP@.result_0 -+tree @TOP@.result_1 -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.mem_req -+tree @TOP@.mem_resp diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/consistency.f b/arch/src/main/scala/framework/root/root-tile/src/main/resources/consistency.f deleted file mode 100644 index ea556227a..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/consistency.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/consistency_control.sv -@RESOURCES@/consistency_package.sv --F @RTL@/Consistency/filelist.f -@RESOURCES@/consistency_tb.sv diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/consistency_control.sv b/arch/src/main/scala/framework/root/root-tile/src/main/resources/consistency_control.sv deleted file mode 100644 index a82d901ab..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/consistency_control.sv +++ /dev/null @@ -1,13 +0,0 @@ -`include "consistency_config.svh" -interface consistency_control_if ( - input logic clock -); - logic reset; - logic cpu_allow, older_dispatch_pending, older_requests_drained; - logic block_requester_rsp, block_requester_data; - logic [`CONS_OUTSTANDING_BITS-1:0] outstanding; - logic req_valid, snp_valid; - logic [`CONS_REQ_WIDTH-1:0] req_bits; - logic [`CONS_SNP_CHANNEL_WIDTH-1:0] snp_bits; - bit start = 0, finished = 0; -endinterface diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/consistency_package.sv b/arch/src/main/scala/framework/root/root-tile/src/main/resources/consistency_package.sv deleted file mode 100644 index b486df7f6..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/consistency_package.sv +++ /dev/null @@ -1,66 +0,0 @@ -package consistency_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - import "DPI-C" function chandle cons_ref_create(); - import "DPI-C" function void cons_ref_destroy(input chandle model); - import "DPI-C" function longint unsigned cons_ref_expected64( - input chandle model, - input longint unsigned address - ); - import "DPI-C" function void cons_ref_cpu_commit( - input chandle model, - input longint unsigned address, - data, - input int unsigned mask - ); - import "DPI-C" function void cons_ref_ddr_read( - input chandle model, - input longint unsigned address, - output bit [511:0] data - ); - import "DPI-C" function int unsigned cons_ref_write_matches( - input chandle model, - input longint unsigned address, - input bit [511:0] data, - input bit [63:0] mask - ); - import "DPI-C" function void cons_ref_ddr_commit( - input chandle model, - input longint unsigned address, - input bit [511:0] data, - input bit [63:0] mask - ); - import "DPI-C" function int unsigned cons_ref_ddr_visible( - input chandle model, - input longint unsigned address - ); - import "DPI-C" function void cons_ref_dma_write( - input chandle model, - input longint unsigned address, - data, - input int unsigned mask - ); - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual consistency_control_if control; - function new(string name, uvm_component parent); - super.new(name, parent); - timeout = 1ms; - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual consistency_control_if)::get(this, "", "control", control)) - `uvm_fatal("VIF", "missing consistency control") - endfunction - task execute(); - @(negedge control.clock); - control.start = 1; - wait (control.finished); - `uvm_info( - "CONSISTENCY_PASS", - "Real L1/CMO/Home DDR data, partial DMA bytes, delayed ACK/old fill and CPU permit checks completed", - UVM_LOW) - endtask - endclass -endpackage diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/consistency_tb.sv b/arch/src/main/scala/framework/root/root-tile/src/main/resources/consistency_tb.sv deleted file mode 100644 index 2bfad093a..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/consistency_tb.sv +++ /dev/null @@ -1,564 +0,0 @@ -module consistency_tb; - import uvm_pkg::*; - import consistency_pkg::*; - `include "uvm_macros.svh" - `include "consistency_config.svh" - `define CF(B, K, F) B[`CONS_``K``_``F``_OFFSET +: `CONS_``K``_``F``_WIDTH] - logic clock = 0; - always #5 clock = ~clock; - consistency_control_if control (clock); - stream_if #(.WIDTH(`CONS_CPU_WIDTH)) - access_0 ( - clock, - control.reset - ), - access_1 ( - clock, - control.reset - ); - stream_if #(.WIDTH(`CONS_RESULT_WIDTH)) - result_0 ( - clock, - control.reset - ), - result_1 ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`CONS_DISPATCH_WIDTH) - ) dispatch ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`CONS_INFO_WIDTH) - ) access_info ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`CONS_GRANT_WIDTH) - ) grant ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`CONS_DONE_WIDTH) - ) done ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`CONS_COMPLETE_WIDTH) - ) complete ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`CONS_MEMREQ_WIDTH) - ) mem_req ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`CONS_MEMRESP_WIDTH) - ) mem_resp ( - clock, - control.reset - ); - Consistency dut ( - .clock(clock), - .reset(control.reset), - `include "consistency_ports.svh" - ); - localparam longint unsigned A='h4000,B='h4040,P='h10000,X='hC000,Y='h8040,E='h20000,R='h30000+3*`CONS_LINE_BYTES; - localparam longint unsigned EVICT = E + `CONS_L1_LINES * `CONS_LINE_BYTES; - typedef struct { - longint unsigned addr, data, expected; - int unsigned mask; - bit write; - } cpu_entry; - typedef struct { - int unsigned id; - longint unsigned addr; - bit write; - bit [511:0] data; - bit [63:0] mask; - int due; - } memory_entry; - cpu_entry cpu_pending[2][$]; - memory_entry memory_pending[$], active_memory; - bit memory_active = 0, memory_live[`CONS_MSHRS]; - chandle model; - bit model_ready = 0, hold_write = 0, hold_read = 0, old_fill_seen = 0; - longint unsigned hold_write_addr, hold_read_addr; - int cycle = 0, checked[2], submitted[2], grants[256], completions[256]; - int memory_reads = 0, memory_writes = 0, memory_acks = 0, peak = 0, cmo_count[3], peer_snoops = 0; - function automatic void check(bit condition, string message); - if (!condition) `uvm_fatal("CONSISTENCY", message) - endfunction - function automatic void observe_cpu(int core, bit valid, ready, - logic [`CONS_CPU_WIDTH-1:0] request, bit response_valid, - response_ready, logic [`CONS_RESULT_WIDTH-1:0] response); - if (valid && ready) begin - cpu_entry item; - check(!$isunknown(request), "unknown CPU request"); - check(`CF(request, CPU, ATOMIC) == 0 && !`CF(request, CPU, ATOMICWORD), - "physical consistency profile uses ordinary 64-bit CPU accesses"); - item.addr = `CF(request, CPU, ADDR); - item.data = `CF(request, CPU, DATA); - item.mask = `CF(request, CPU, MASK); - item.write = `CF(request, CPU, WRITE); - item.expected = cons_ref_expected64(model, item.addr); - cpu_pending[core].push_back(item); - submitted[core]++; - `uvm_info("CPU_ACCEPT", $sformatf("core%0d addr=%h write%0d", core, item.addr, item.write), - UVM_LOW) - end - if (response_valid && response_ready) begin - cpu_entry item; - check(cpu_pending[core].size() != 0, "CPU response without accepted access"); - item = cpu_pending[core].pop_front(); - check(!$isunknown(response) && !`CF(response, RESULT, ERROR), "CPU memory access failed"); - check(`CF(response, RESULT, DATA) === item.expected, $sformatf( - "core%0d addr=%h result=%h expected=%h", - core, - item.addr, - `CF(response, RESULT, DATA), - item.expected - )); - if (item.write) cons_ref_cpu_commit(model, item.addr, item.data, item.mask); - checked[core]++; - `uvm_info("CPU_RESULT", $sformatf("core%0d addr=%h data=%h", core, item.addr, - `CF(response, RESULT, DATA)), UVM_LOW) - end - endfunction - always @(posedge clock) begin - if (model_ready && !control.reset) begin - cycle++; - if (control.outstanding > peak) peak = control.outstanding; - observe_cpu(0, access_0.valid, access_0.ready, access_0.bits, result_0.valid, result_0.ready, - result_0.bits); - observe_cpu(1, access_1.valid, access_1.ready, access_1.bits, result_1.valid, result_1.ready, - result_1.bits); - if (dispatch.valid && dispatch.ready) begin - int id; - id = `CF(dispatch.bits, DISPATCH, ID); - grants[id] = 0; - completions[id] = 0; - end - if (grant.valid && grant.ready) begin - grants[`CF(grant.bits, GRANT, TAG)]++; - `uvm_info("NPU_GRANT", $sformatf("tag%0d", `CF(grant.bits, GRANT, TAG)), UVM_LOW) - end - if (complete.valid && complete.ready) begin - completions[`CF(complete.bits, COMPLETE, TAG)]++; - `uvm_info("NPU_COMPLETE", $sformatf("tag%0d", `CF(complete.bits, COMPLETE, TAG)), UVM_LOW) - end - if (control.req_valid) - `uvm_info("HOME_REQUEST", $sformatf( - "node%0d txn%0d op%h addr=%h", - `CF(control.req_bits, REQ, SRCID), - `CF(control.req_bits, REQ, TXNID), - `CF(control.req_bits, REQ, OPCODE), - `CF(control.req_bits, REQ, ADDR) - ), UVM_LOW) - if (control.snp_valid) - `uvm_info("HOME_SNOOP", $sformatf( - "node%0d op%h addr=%h", - control.snp_bits[`CONS_SNP_TARGET_OFFSET+:`CONS_SNP_TARGET_WIDTH], - `CF(control.snp_bits, SNP, OPCODE), - 64'( - `CF(control.snp_bits, SNP, ADDR) - ) << 3 - ), UVM_LOW) - if (control.req_valid && - `CF(control.req_bits, REQ, SRCID) - == 1 && - `CF(control.req_bits, REQ, TXNID) - == 2) begin - int opcode; - opcode = `CF(control.req_bits, REQ, OPCODE); - check(opcode inside {'h08, 'h09, 'h0a}, "maintenance requester emitted a non-CMO opcode"); - check(`CF(control.req_bits, REQ, TGTID) == `CONS_HOME, "CMO routed to wrong Home"); - cmo_count[opcode-'h08]++; - `uvm_info("CMO_REQUEST", $sformatf("op%h addr=%h", opcode, `CF(control.req_bits, REQ, - ADDR)), UVM_LOW) - end - if(control.snp_valid&&control.snp_bits[`CONS_SNP_TARGET_OFFSET+:`CONS_SNP_TARGET_WIDTH]==2) - peer_snoops++; - if (mem_resp.valid && mem_resp.ready) begin - check(memory_active && `CF(mem_resp.bits, MEMRESP, ID) == active_memory.id, - "memory response identity mismatch"); - if (active_memory.write) - cons_ref_ddr_commit(model, active_memory.addr, active_memory.data, active_memory.mask); - memory_live[active_memory.id] = 0; - memory_active = 0; - memory_acks++; - `uvm_info("DDR_ACK", $sformatf("id%0d addr=%h write%0d", active_memory.id, - active_memory.addr, active_memory.write), UVM_LOW) - end - if (mem_req.valid && mem_req.ready) begin - memory_entry item; - check(!$isunknown(mem_req.bits), "unknown real Home memory request"); - item.id = `CF(mem_req.bits, MEMREQ, ID); - item.addr = `CF(mem_req.bits, MEMREQ, ADDR); - item.write = `CF(mem_req.bits, MEMREQ, WRITE); - item.data = `CF(mem_req.bits, MEMREQ, DATA); - item.mask = `CF(mem_req.bits, MEMREQ, MASK); - check(item.id < `CONS_MSHRS && !memory_live[item.id], "Home reused a live DDR ID"); - check((item.addr & 63) == 0, "Home DDR request not a 64-byte line"); - memory_live[item.id] = 1; - if (item.write) begin - check(item.mask === '1 && cons_ref_write_matches(model, item.addr, item.data, item.mask - ) == 1, "Home writeback lost CPU-visible dirty bytes"); - memory_writes++; - end else begin - cons_ref_ddr_read(model, item.addr, item.data); - memory_reads++; - if (item.addr == X) old_fill_seen = 1; - end - item.due = cycle + 7 + (3 - item.id) * 3; - memory_pending.push_back(item); - `uvm_info("DDR_REQUEST", $sformatf("id%0d addr=%h write%0d", item.id, item.addr, - item.write), UVM_LOW) - end - end - end - // DDR BFM: reads and write ACKs can reorder across distinct real Home MSHR IDs. - always @(negedge clock) begin - if (model_ready && !control.reset) begin - mem_req.ready = (cycle % 7) != 0; - if (!memory_active) begin - mem_resp.valid = 0; - for (int i = memory_pending.size() - 1; i >= 0; i--) begin - if(!memory_active&&memory_pending[i].due<=cycle&& - !(hold_write&&memory_pending[i].write&&memory_pending[i].addr==hold_write_addr)&& - !(hold_read&&!memory_pending[i].write&&memory_pending[i].addr==hold_read_addr)) begin - active_memory = memory_pending[i]; - memory_pending.delete(i); - memory_active = 1; - mem_resp.bits = '0; - `CF(mem_resp.bits, MEMRESP, ID) = active_memory.id; - `CF(mem_resp.bits, MEMRESP, DATA) = active_memory.write ? '0 : active_memory.data; - mem_resp.valid = 1; - end - end - end - end - end - task automatic cpu_request(int core, longint unsigned address, bit write = 0, - longint unsigned data = 0, int unsigned mask = 255); - logic [`CONS_CPU_WIDTH-1:0] packet; - packet = '0; - `CF(packet, CPU, ADDR) = address; - `CF(packet, CPU, WRITE) = write; - `CF(packet, CPU, DATA) = data; - `CF(packet, CPU, MASK) = write ? mask : 0; - @(negedge clock); - if (core == 0) begin - access_0.bits = packet; - access_0.valid = 1; - do @(posedge clock); while (!access_0.ready); - @(negedge clock); - access_0.valid = 0; - end else begin - access_1.bits = packet; - access_1.valid = 1; - do @(posedge clock); while (!access_1.ready); - @(negedge clock); - access_1.valid = 0; - end - endtask - task automatic cpu_access(int core, longint unsigned address, bit write = 0, - longint unsigned data = 0, int unsigned mask = 255); - int goal; - goal = checked[core] + 1; - cpu_request(core, address, write, data, mask); - wait (checked[core] == goal); - @(negedge clock); - endtask - task automatic reserve_access(int tag, longint unsigned base, bytes, bit write); - @(negedge clock); - dispatch.bits = '0; - `CF(dispatch.bits, DISPATCH, ID) = tag; - dispatch.valid = 1; - do @(posedge clock); while (!dispatch.ready); - @(negedge clock); - dispatch.valid = 0; - access_info.bits = '0; - `CF(access_info.bits, INFO, ID) = tag; - `CF(access_info.bits, INFO, HASMEMORY) = 1; - `CF(access_info.bits, INFO, BASE) = base; - `CF(access_info.bits, INFO, BYTES) = bytes; - `CF(access_info.bits, INFO, WRITE) = write; - `CF(access_info.bits, INFO, LAST) = 1; - access_info.valid = 1; - do @(posedge clock); while (!access_info.ready); - @(negedge clock); - access_info.valid = 0; - endtask - task automatic dma_done(int tag); - @(negedge clock); - done.bits = '0; - `CF(done.bits, DONE, TAG) = tag; - `CF(done.bits, DONE, OK) = 1; - done.valid = 1; - do @(posedge clock); while (!done.ready); - @(negedge clock); - done.valid = 0; - done.bits = '0; - endtask - task automatic await_complete(int tag); - wait (completions[tag] == 1); - @(negedge clock); - endtask - task automatic block_cpu_read(longint unsigned address, int clocks = 32); - @(negedge clock); - access_0.bits = '0; - `CF(access_0.bits, CPU, ADDR) = address; - access_0.valid = 1; - repeat (clocks) begin - @(posedge clock); - #1; - check(!control.cpu_allow && !access_0.ready, - "CPU target read passed before DMA ACK/post-invalidate"); - end - endtask - task automatic release_cpu_read(); - int goal; - goal = checked[0] + 1; - do @(posedge clock); while (!access_0.ready); - @(negedge clock); - access_0.valid = 0; - wait (checked[0] == goal); - @(negedge clock); - endtask - initial begin - uvm_config_db#(virtual consistency_control_if)::set(null, "*", "control", control); - run_test("protocol_test"); - end - initial begin - control.reset = 1; - control.older_dispatch_pending = 0; - control.older_requests_drained = 1; - control.block_requester_rsp = 0; - control.block_requester_data = 0; - access_0.valid = 0; - access_0.bits = '0; - access_1.valid = 0; - access_1.bits = '0; - result_0.ready = 1; - result_1.ready = 1; - dispatch.valid = 0; - dispatch.bits = '0; - access_info.valid = 0; - access_info.bits = '0; - grant.ready = 1; - done.valid = 0; - done.bits = '0; - complete.ready = 1; - mem_req.ready = 0; - mem_resp.valid = 0; - mem_resp.bits = '0; - foreach (memory_live[i]) memory_live[i] = 0; - foreach (checked[i]) begin - checked[i] = 0; - submitted[i] = 0; - end - foreach (grants[i]) begin - grants[i] = 0; - completions[i] = 0; - end - foreach (cmo_count[i]) cmo_count[i] = 0; - wait (control.start); - check( - `CONS_AGENTS==2&&`CONS_SLOTS==4&&`CONS_PA_BITS==44&&`CONS_LINE_BYTES==64&&`CONS_L1_LINES==8&&`CONS_L2_SETS==16, - "unexpected frozen physical profile"); - model = cons_ref_create(); - model_ready = 1; - repeat (4) @(posedge clock); - @(negedge clock); - control.reset = 0; - // CPU publication hint stalls only until the older unregistered dispatch is recorded. - @(negedge clock); - control.older_dispatch_pending = 1; - access_0.bits = '0; - `CF(access_0.bits, CPU, ADDR) = B; - access_0.valid = 1; - repeat (8) begin - @(posedge clock); - #1; - check(!control.cpu_allow && !access_0.ready, "older dispatch hint did not block CPU"); - end - @(negedge clock); - control.older_dispatch_pending = 0; - release_cpu_read(); - // Real dirty L1 data must reach DDR before a mock NPU read is granted. - cpu_access(0, A, 1, 64'h1122334455667788); - check(cons_ref_ddr_visible(model, A) == 0, "CPU dirty store incorrectly updated DDR golden"); - hold_write = 1; - hold_write_addr = A; - reserve_access(10, A, 64, 0); - wait (memory_writes > 0 || grants[10] > 0); - check(memory_writes > 0, "dirty pre-Clean granted without a real DDR writeback"); - repeat (32) begin - @(posedge clock); - #1; - check(grants[10] == 0 && completions[10] == 0, "NPU read granted before real DDR write ACK"); - end - cpu_access(1, B); // Independent requester/range progresses while dirty write ACK is held. - @(negedge clock); - hold_write = 0; - wait (grants[10] == 1); - check(cons_ref_ddr_visible(model, A) == 1, "pre-Clean did not publish dirty CPU bytes to DDR"); - cpu_access(0, A); // R/R overlap completes while NPU read has not sent done. - check(completions[10] == 0, "NPU read unexpectedly completed without done"); - dma_done(10); - await_complete(10); - // Preserve dirty neighbour bytes around a partial NPU write and invalidate all real holders. - cpu_access(0, P, 1, 64'h0102030405060708); - cpu_access(0, P + 16, 1, 64'h9192939495969798); - cpu_access(1, P); - cpu_access(1, P + 16); - control.older_requests_drained = 0; - reserve_access(20, P, 64, 1); - begin - int prior_cmo_count; - prior_cmo_count = cmo_count[1]; - repeat (12) begin - @(posedge clock); - #1; - check(grants[20] == 0 && cmo_count[1] == prior_cmo_count, - "maintenance admitted before external old requests drained"); - end - end - @(negedge clock); - control.older_requests_drained = 1; - wait (grants[20] == 1); - check(cons_ref_ddr_visible(model, P) == 1, "pre-CleanInvalidate lost dirty neighbour bytes"); - cons_ref_dma_write(model, P + 8, 64'hfeedface89abcdef, 15); - block_cpu_read(P + 8, 32); - check(completions[20] == 0, "write completed before DMA done ACK"); - @(negedge clock); - complete.ready = 0; - dma_done(20); - wait (complete.valid); - repeat (8) begin - @(posedge clock); - #1; - check(!control.cpu_allow && !access_0.ready && complete.valid && - `CF(complete.bits, COMPLETE, TAG) == 20, - "CPU read bypassed stalled final range completion"); - end - @(negedge clock); - complete.ready = 1; - await_complete(20); - release_cpu_read(); - for (int word = 0; word < 8; word++) begin - cpu_access(0, P + word * 8); - cpu_access(1, P + word * 8); - end - // An old dirty CopyBack can leave L1 yet remain in the explicit transport FIFO. - cpu_access(0, E, 1, 64'h12345678badc0ffe); - @(negedge clock); - control.block_requester_data = 1; - cpu_access(0, EVICT); // Same L1 slot, different real Home set. - check(cons_ref_ddr_visible(model, E) == 0, - "old dirty CopyBack reached DDR despite transport stall"); - reserve_access(40, E, 64, 0); - repeat (32) begin - @(posedge clock); - #1; - check(grants[40] == 0 && completions[40] == 0, "CMO granted ahead of old delayed CopyBack"); - end - @(negedge clock); - control.block_requester_data = 0; - wait (grants[40] == 1); - check(cons_ref_ddr_visible(model, E) == 1, "old CopyBack did not publish DDR before grant"); - dma_done(40); - await_complete(40); - // A CompAck accepted by L1 can still be queued at the real Home RX credit boundary. - @(negedge clock); - control.block_requester_rsp = 1; - cpu_access(0, R); - reserve_access(50, R, 64, 0); - repeat (24) begin - @(posedge clock); - #1; - check(grants[50] == 0 && completions[50] == 0, "CMO overtook old undelivered CompAck"); - end - @(negedge clock); - control.block_requester_rsp = 0; - wait (grants[50] == 1); - check(cons_ref_ddr_visible(model, R) == 1, "CompAck drain changed read-only DDR data"); - dma_done(50); - await_complete(50); - // Delay an actual older CPU miss/fill and its result, then admit maintenance only after both drain. - hold_read = 1; - hold_read_addr = X; - result_0.ready = 0; - control.older_requests_drained = 0; - cpu_request(0, X); - wait (old_fill_seen); - reserve_access(30, X, 64, 1); - cpu_access(1, Y); - repeat (16) begin - @(posedge clock); - #1; - check(grants[30] == 0, "maintenance overtook an older delayed fill"); - end - @(negedge clock); - hold_read = 0; - wait (result_0.valid); - repeat (8) begin - @(posedge clock); - #1; - check(grants[30] == 0, "maintenance overtook unaccepted older CPU result"); - end - @(negedge clock); - result_0.ready = 1; - wait (checked[0] == submitted[0]); - repeat (8) begin - @(posedge clock); - #1; - check(grants[30] == 0, "maintenance ignored olderRequestsDrained=0 after fill"); - end - @(negedge clock); - control.older_requests_drained = 1; - wait (grants[30] == 1); - check(cons_ref_ddr_visible(model, X) == 1, "old-fill pre-maintenance altered DDR bytes"); - cons_ref_dma_write(model, X, 64'hd0d1d2d3d4d5d6d7, 255); - block_cpu_read(X, 32); - dma_done(30); - await_complete(30); - release_cpu_read(); - cpu_access(1, X); - wait (control.outstanding == 0 && !memory_active && memory_pending.size() == 0); - repeat (8) @(posedge clock); - check( - checked[0]==submitted[0]&&checked[1]==submitted[1]&&cpu_pending[0].size()==0&&cpu_pending[1].size()==0, - "unmatched CPU accesses remain"); - check( - memory_writes>0&&cmo_count[0]>0&&cmo_count[1]>=2&&cmo_count[2]>=2&&peer_snoops>0, - "required real CMO/writeback/holder snoops were absent"); - check(peak >= 2, "independent old-fill/peer traffic did not exercise multiple Home MSHRs"); - `uvm_info("CHECKED", $sformatf( - "CPU%0d/%0d DDRread%0d/write%0d/ACK%0d CMO%0d/%0d/%0d peerSnp%0d peak%0d", - checked[0], - checked[1], - memory_reads, - memory_writes, - memory_acks, - cmo_count[0], - cmo_count[1], - cmo_count[2], - peer_snoops, - peak - ), UVM_LOW) - cons_ref_destroy(model); - model_ready = 0; - control.finished = 1; - end -endmodule diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/tile.f b/arch/src/main/scala/framework/root/root-tile/src/main/resources/tile.f deleted file mode 100644 index 3c3a49006..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/tile.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/tile_interface.sv -@RESOURCES@/tile_package.sv --F @RTL@/Tile/filelist.f -@RESOURCES@/tile_tb.sv diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/tile_interface.sv b/arch/src/main/scala/framework/root/root-tile/src/main/resources/tile_interface.sv deleted file mode 100644 index b22250e83..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/tile_interface.sv +++ /dev/null @@ -1,20 +0,0 @@ -`include "cache_system_config.svh" -interface tile_if ( - input logic clock -); - logic reset; - logic retired[2], trapped[2]; - logic [63:0] retiredPc[2], trapCause[2], trapValue[2], trapPc[2]; - logic [2:0] outstanding; - logic req_valid, rsp_valid, dat_valid, snp_valid, rx_rsp_valid, rx_dat_valid; - logic [`COH_REQ_WIDTH-1:0] req_bits; - logic [`COH_RSP_WIDTH-1:0] rsp_bits, rx_rsp_bits; - logic [`COH_DAT_WIDTH-1:0] dat_bits, rx_dat_bits; - logic [`COH_SNP_CHANNEL_WIDTH-1:0] snp_bits; - clocking sample @(posedge clock); - default input #1step; - input reset, retired, retiredPc, trapped, trapCause, trapValue, trapPc, outstanding; - input req_valid, rsp_valid, dat_valid, snp_valid, rx_rsp_valid, rx_dat_valid; - input req_bits, rsp_bits, dat_bits, snp_bits, rx_rsp_bits, rx_dat_bits; - endclocking -endinterface diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/tile_package.sv b/arch/src/main/scala/framework/root/root-tile/src/main/resources/tile_package.sv deleted file mode 100644 index bb4f142b8..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/tile_package.sv +++ /dev/null @@ -1,415 +0,0 @@ -`include "tile_config.svh" -`include "cache_system_config.svh" -`include "multicore_fixture.svh" -`define TF(K, F) `TILE_``K``_``F``_OFFSET +: `TILE_``K``_``F``_WIDTH -`define TH(K, F) `COH_``K``_``F``_OFFSET +: `COH_``K``_``F``_WIDTH -package tile_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - import "DPI-C" function chandle core_ref_create(input string path); - import "DPI-C" function void core_ref_destroy(input chandle model); - import "DPI-C" function void core_ref_read( - input chandle model, - input longint unsigned address, - output bit [511:0] data - ); - import "DPI-C" function void core_ref_write( - input chandle model, - input longint unsigned address, - input bit [511:0] data, - input bit [63:0] mask - ); - typedef logic [`COH_REQ_WIDTH-1:0] req_t; - typedef logic [`COH_RSP_WIDTH-1:0] rsp_t; - typedef logic [`COH_DAT_WIDTH-1:0] dat_t; - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual tile_if control; - virtual stream_if #(`TILE_MEMREQ_WIDTH) mem_req; - virtual stream_if #(`TILE_MEMRESP_WIDTH) mem_resp; - virtual stream_if #(`TILE_UNCACHED_WIDTH) uncached_req[2]; - virtual stream_if #(`TILE_URESP_WIDTH) uncached_resp[2]; - chandle model; - typedef struct { - bit [`TILE_MEMRESP_WIDTH-1:0] packet; - int due; - } memory_entry; - memory_entry memory_queue[$]; - bit memory_active = 0, mmio_pending[2] = '{0, 0}; - bit [`TILE_MEMRESP_WIDTH-1:0] memory_packet; - bit [`TILE_URESP_WIDTH-1:0] mmio_packet[2]; - int mmio_due[2], mmio_wait[2] = '{0, 0}, memory_wait = 0; - int stages[2] = '{0, 0}, retired[2] = '{0, 0}, mmio_stalls[2] = '{0, 0}; - bit exited[2] = '{0, 0}, exit_retired[2] = '{0, 0}; - int cycles = 0, reads = 0, writes = 0, memory_stalls = 0, node_requests[2] = '{0, 0}; - req_t requests[int]; - int fill_seen[int], fill_dbid[int], ack_owner[int]; - bit [511:0] fill_line[int], return_line[int]; - longint unsigned return_address[int]; - int return_owner[int], return_seen[int]; - bit return_is_snoop[int]; - int grants = 0, snoops = 0, writebacks = 0, acks = 0, dirty_snoops = 0; - bit final_line_seen = 0; - localparam longint unsigned SHARED = 64'h800040c0; - function new(string name, uvm_component parent); - super.new(name, parent); - timeout = 5ms; - endfunction - function bit [511:0] final_line(); - return { - 64'hffeeddccbbaa9988, - 64'h8000000000000001, - 64'h1122334455667788, - 64'h0102030405060708, - 64'd1, - 64'd1, - 64'd1, - 64'h8877665544332211 - }; - endfunction - function void build_phase(uvm_phase phase); - bit [511:0] initial_line; - super.build_phase(phase); - if (!uvm_config_db#(virtual tile_if)::get( - this, "", "control", control - ) || !uvm_config_db#(virtual stream_if #(`TILE_MEMREQ_WIDTH))::get( - this, "", "mem_req", mem_req - ) || !uvm_config_db#(virtual stream_if #(`TILE_MEMRESP_WIDTH))::get( - this, "", "mem_resp", mem_resp - )) - `uvm_fatal("VIF", "Tile interfaces missing") - for (int h = 0; h < 2; h++) begin - if (!uvm_config_db#(virtual stream_if #(`TILE_UNCACHED_WIDTH))::get( - this, "", $sformatf("uncached_req_%0d", h), uncached_req[h] - ) || !uvm_config_db#(virtual stream_if #(`TILE_URESP_WIDTH))::get( - this, "", $sformatf("uncached_resp_%0d", h), uncached_resp[h] - )) - `uvm_fatal("VIF", "Per-hart MMIO interface missing") - end - model = core_ref_create(`TILE_IMAGE); - core_ref_read(model, SHARED, initial_line); - if (initial_line !== 512'b0) - `uvm_fatal("FIXTURE", "Shared 64-byte fixture must be explicitly initialized to zero") - endfunction - function void observe_protocol(); - if (control.sample.req_valid) begin - req_t r = control.sample.req_bits; - int node, key; - if ($isunknown(r)) `uvm_fatal("CHI_X", "Unknown accepted request") - node = r[`TH(REQ, SRCID)]; - key = (node << 1) | int'(r[`TH(REQ, TXNID)]); - if (node < 1 || node > 2 || r[ - `TH(REQ, TGTID) - ] != 64 || r[ - `TH(REQ, TXNID) - ] >= 2 || requests.exists( - key - )) - `uvm_fatal("REQ_ID", "Requester node/bank identity or duplicate transaction") - requests[key] = r; - node_requests[node-1]++; - end - if (control.sample.rsp_valid) begin - rsp_t r = control.sample.rsp_bits; - int key, id; - key = (int'(r[`TH(RSP, TGTID)]) << 1) | int'(r[`TH(RSP, TXNID)]); - id = r[`TH(RSP, DBID)]; - if ($isunknown( - r - ) || !requests.exists( - key - ) || r[ - `TH(RSP, SRCID) - ] != 64 || r[ - `TH(RSP, RESPERR) - ] != 0) - `uvm_fatal("RSP_ID", "Unmatched Home response or error") - if (r[`TH(RSP, OPCODE)] == `COH_COMP_DBID) begin - if (id >= 4 || return_owner.exists( - id - ) || requests[key][ - `TH(REQ, OPCODE) - ] != `COH_WRITEBACK) - `uvm_fatal("WB_ID", "Invalid writeback DBID/descriptor") - return_owner[id] = r[`TH(RSP, TGTID)]; - return_address[id] = requests[key][`TH(REQ, ADDR)]; - return_seen[id] = 0; - return_line[id] = '0; - return_is_snoop[id] = 0; - end else if (r[`TH(RSP, OPCODE)] != `COH_COMP) - `uvm_fatal("RSP_OPCODE", "Unexpected retry/response") - requests.delete(key); - end - if (control.sample.dat_valid) begin - dat_t d = control.sample.dat_bits; - int key, id, beat; - key = (int'(d[`TH(DAT, TGTID)]) << 1) | int'(d[`TH(DAT, TXNID)]); - id = d[`TH(DAT, DBID)]; - beat = int'(d[`TH(DAT, DATAID)]) >> 1; - if ($isunknown( - d - ) || !requests.exists( - key - ) || id >= 4 || d[ - `TH(DAT, SRCID) - ] != 64 || d[ - `TH(DAT, HOMENID) - ] != 64 || d[ - `TH(DAT, OPCODE) - ] != `COH_COMP_DATA || d[ - `TH(DAT, RESPERR) - ] != 0 || d[ - `TH(DAT, BE) - ] != 32'hffffffff || (d[ - `TH(DAT, DATAID) - ] != 0 && d[ - `TH(DAT, DATAID) - ] != 2)) - `uvm_fatal("COMP_DATA", "Invalid coherent grant") - if (!fill_seen.exists(key)) begin - fill_seen[key] = 0; - fill_line[key] = '0; - fill_dbid[key] = id; - end - if ((fill_seen[key] & (1 << beat)) || fill_dbid[key] != id) - `uvm_fatal("COMP_BEAT", "Duplicate beat or changed DBID") - fill_line[key][beat*256+:256] = d[`TH(DAT, DATA)]; - fill_seen[key] |= 1 << beat; - if (fill_seen[key] == 3) begin - if (requests[key][`TH(REQ, ADDR)] == SHARED && fill_line[key] === final_line()) - final_line_seen = 1; - if (ack_owner.exists(id)) `uvm_fatal("ACK_SLOT", "Home DBID reused before CompAck") - ack_owner[id] = d[`TH(DAT, TGTID)]; - grants++; - requests.delete(key); - fill_seen.delete(key); - fill_dbid.delete(key); - fill_line.delete(key); - end - end - if (control.sample.snp_valid) begin - int id = control.sample.snp_bits[`TH(SNP, TXNID)]; - int target = control.sample.snp_bits[`TH(SNP, TARGET)]; - if ($isunknown( - control.sample.snp_bits - ) || id >= 4 || target < 1 || target > 2 || return_owner.exists( - id - ) || control.sample.snp_bits[ - `TH(SNP, SRCID) - ] != 64) - `uvm_fatal("SNOOP_ID", "Invalid directed snoop descriptor") - return_owner[id] = target; - return_address[id] = 64'(control.sample.snp_bits[`TH(SNP, ADDR)]) << 3; - return_seen[id] = 0; - return_line[id] = '0; - return_is_snoop[id] = 1; - end - if (control.sample.rx_rsp_valid) begin - rsp_t r = control.sample.rx_rsp_bits; - int id = r[`TH(RSP, TXNID)]; - if ($isunknown(r) || r[`TH(RSP, TGTID)] != 64) - `uvm_fatal("RX_RSP", "Invalid requester response") - if (r[`TH(RSP, OPCODE)] == `COH_COMP_ACK) begin - if (!ack_owner.exists(id) || ack_owner[id] != r[`TH(RSP, SRCID)]) - `uvm_fatal("ACK", "CompAck did not return Home DBID/owner") - ack_owner.delete(id); - acks++; - end else begin - if (r[ - `TH(RSP, OPCODE) - ] != `COH_SNP_RESP || !return_owner.exists( - id - ) || !return_is_snoop[id] || return_owner[id] != r[ - `TH(RSP, SRCID) - ] || return_seen[id] != 0) - `uvm_fatal("SNOOP_RSP", "Unmatched snoop response") - return_owner.delete(id); - return_seen.delete(id); - return_address.delete(id); - return_line.delete(id); - return_is_snoop.delete(id); - end - end - if (control.sample.rx_dat_valid) begin - dat_t d = control.sample.rx_dat_bits; - int id = d[`TH(DAT, TXNID)]; - int beat = int'(d[`TH(DAT, DATAID)]) >> 1; - if ($isunknown( - d - ) || !return_owner.exists( - id - ) || return_owner[id] != d[ - `TH(DAT, SRCID) - ] || d[ - `TH(DAT, TGTID) - ] != 64 || d[ - `TH(DAT, RESPERR) - ] != 0 || (d[ - `TH(DAT, DATAID) - ] != 0 && d[ - `TH(DAT, DATAID) - ] != 2) || (return_seen[id] & (1 << beat)) || d[ - `TH(DAT, BE) - ] != 32'hffffffff) - `uvm_fatal("RETURN_DATA", "Unmatched/duplicate/partial return data") - if (d[`TH(DAT, OPCODE)] != (return_is_snoop[id] ? `COH_SNP_DATA : `COH_COPYBACK_DATA)) - `uvm_fatal("RETURN_OPCODE", "Wrong return role") - return_line[id][beat*256+:256] = d[`TH(DAT, DATA)]; - return_seen[id] |= 1 << beat; - if (return_seen[id] == 3) begin - if(return_is_snoop[id]&&return_address[id]==SHARED&&return_line[id]===final_line()) - final_line_seen = 1; - if (return_is_snoop[id]) begin - snoops++; - if (d[`TH(DAT, RESP)] & 4) dirty_snoops++; - end else writebacks++; - return_owner.delete(id); - return_seen.delete(id); - return_address.delete(id); - return_line.delete(id); - return_is_snoop.delete(id); - end - end - endfunction - task service(); - forever begin - @(control.sample); - if (!control.sample.reset) begin - cycles++; - observe_protocol(); - if (mem_resp.sample.valid && mem_resp.sample.ready) memory_active = 0; - for (int h = 0; h < 2; h++) begin - if (uncached_resp[h].sample.valid && uncached_resp[h].sample.ready) mmio_pending[h] = 0; - if (control.sample.trapped[h]) - `uvm_fatal("TRAP", $sformatf( - "hart%0d cause=%h value=%h pc=%h", - h, - control.sample.trapCause[h], - control.sample.trapValue[h], - control.sample.trapPc[h] - )) - if (control.sample.retired[h]) begin - retired[h]++; - if (control.sample.retiredPc[h] == `CORE_EXIT_PC) begin - if (!exited[h] || exit_retired[h]) - `uvm_fatal("EXIT_RETIRE", "Exit retired before MMIO success or twice") - exit_retired[h] = 1; - end - end - if (uncached_req[h].sample.valid && !uncached_req[h].sample.ready) begin - mmio_stalls[h]++; - mmio_wait[h]++; - end - if (uncached_req[h].sample.valid && uncached_req[h].sample.ready) begin - longint unsigned address = uncached_req[h].sample.bits[`TF(UNCACHED, ADDR)]; - longint unsigned value = uncached_req[h].sample.bits[`TF(UNCACHED, DATA)]; - if (mmio_pending[h] || exited[h] || $isunknown( - uncached_req[h].sample.bits - ) || !uncached_req[h].sample.bits[ - `TF(UNCACHED, WRITE) - ] || uncached_req[h].sample.bits[ - `TF(UNCACHED, SIZE) - ] != 3 || (address != 64'h10000000 && address != 64'h10000008)) - `uvm_fatal("MMIO", "Unexpected per-hart MMIO operation") - if (address == 64'h10000008) begin - if (value != stages[h] + 1 || value > 2) - `uvm_fatal("STAGE", "Per-hart stage out of order") - stages[h] = value; - end else begin - if (value != 0 || stages[h] != 2) - `uvm_fatal("FIRMWARE", $sformatf( - "hart%0d failed stage%0d code%h", h, stages[h], value)) - exited[h] = 1; - end - mmio_packet[h] = '0; - mmio_packet[h][`TF(URESP, TAG)] = uncached_req[h].sample.bits[`TF(UNCACHED, TAG)]; - mmio_pending[h] = 1; - mmio_due[h] = cycles + 5 + h; - mmio_wait[h] = 0; - end - end - if (mem_req.sample.valid && !mem_req.sample.ready) begin - memory_stalls++; - memory_wait++; - end - if (mem_req.sample.valid && mem_req.sample.ready) begin - memory_entry e; - bit [511:0] line; - longint unsigned address = mem_req.sample.bits[`TF(MEMREQ, ADDR)]; - if ($isunknown(mem_req.sample.bits) || address[5:0] != 0) - `uvm_fatal("MEMORY", "Unknown/unaligned backend request") - e.packet = '0; - e.packet[`TF(MEMRESP, ID)] = mem_req.sample.bits[`TF(MEMREQ, ID)]; - if (mem_req.sample.bits[`TF(MEMREQ, WRITE)]) begin - core_ref_write(model, address, mem_req.sample.bits[`TF(MEMREQ, DATA)], - mem_req.sample.bits[`TF(MEMREQ, MASK)]); - writes++; - end else begin - core_ref_read(model, address, line); - e.packet[`TF(MEMRESP, DATA)] = line; - reads++; - end - e.due = cycles + 5 + (reads + writes) % 7; - memory_queue.push_back(e); - memory_wait = 0; - end - end - @(negedge control.clock); - mem_req.ready = !control.reset && memory_wait >= 2; - for (int h = 0; h < 2; h++) begin - uncached_req[h].ready = !control.reset && !mmio_pending[h] && mmio_wait[h] >= 2; - uncached_resp[h].valid = !control.reset && mmio_pending[h] && cycles >= mmio_due[h]; - uncached_resp[h].bits = mmio_packet[h]; - end - if (!memory_active) - for (int q = memory_queue.size() - 1; q >= 0; q--) - if (memory_queue[q].due <= cycles) begin - memory_packet = memory_queue[q].packet; - memory_queue.delete(q); - memory_active = 1; - break; - end - mem_resp.valid = !control.reset && memory_active; - mem_resp.bits = memory_packet; - end - endtask - task execute(); - control.reset = 1; - mem_req.ready = 0; - mem_resp.valid = 0; - mem_resp.bits = '0; - for (int h = 0; h < 2; h++) begin - uncached_req[h].ready = 0; - uncached_resp[h].valid = 0; - uncached_resp[h].bits = '0; - end - repeat (5) @(negedge control.clock); - control.reset = 0; - fork - service(); - join_none - wait (exit_retired[0] && exit_retired[1]); - do - @(control.sample); - while (control.sample.outstanding || memory_active || memory_queue.size() || - mmio_pending[0] || mmio_pending[1] || requests.num() || fill_seen.num() || - ack_owner.num() || return_owner.num()); - if(!final_line_seen||!dirty_snoops||!node_requests[0]||!node_requests[1]||!memory_stalls||!mmio_stalls[0]||!mmio_stalls[1]) - `uvm_fatal( - "SCENARIOS", - "Missing final full64B coherent line, dirty snoop, distinct nodes or backpressure") - `uvm_info( - "TILE", - $sformatf( - "2 real harts stages=2/2 exits=0/0 retired=%0d/%0d; CHI req=%0d/%0d grants=%0d snoopLines=%0d dirty=%0d copybacks=%0d acks=%0d; DDR R/W=%0d/%0d stalls=%0d MMIO stalls=%0d/%0d; final64B checked and all descriptors drained", - retired[0], retired[1], node_requests[0], node_requests[1], grants, snoops, - dirty_snoops, writebacks, acks, reads, writes, memory_stalls, mmio_stalls[0], - mmio_stalls[1]), UVM_LOW) - endtask - function void final_phase(uvm_phase phase); - core_ref_destroy(model); - super.final_phase(phase); - endfunction - endclass -endpackage diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/tile_tb.sv b/arch/src/main/scala/framework/root/root-tile/src/main/resources/tile_tb.sv deleted file mode 100644 index be26d3420..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/tile_tb.sv +++ /dev/null @@ -1,58 +0,0 @@ -`include "tile_config.svh" -module tile_tb; - import uvm_pkg::*; - import tile_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - tile_if control (clock); - stream_if #(`TILE_MEMREQ_WIDTH) mem_req ( - clock, - control.reset - ); - stream_if #(`TILE_MEMRESP_WIDTH) mem_resp ( - clock, - control.reset - ); - stream_if #(`TILE_UNCACHED_WIDTH) - uncached_req_0 ( - clock, - control.reset - ), - uncached_req_1 ( - clock, - control.reset - ); - stream_if #(`TILE_URESP_WIDTH) - uncached_resp_0 ( - clock, - control.reset - ), - uncached_resp_1 ( - clock, - control.reset - ); - Tile dut ( - .clock(clock), - .reset(control.reset), - .io_resetVector(64'h80000000), - .io_time(64'b0), - .io_outstanding(control.outstanding), - `include "tile_ports.svh" - ); - initial begin - uvm_config_db#(virtual tile_if)::set(null, "uvm_test_top", "control", control); - uvm_config_db#(virtual stream_if #(`TILE_MEMREQ_WIDTH))::set(null, "uvm_test_top", "mem_req", - mem_req); - uvm_config_db#(virtual stream_if #(`TILE_MEMRESP_WIDTH))::set(null, "uvm_test_top", "mem_resp", - mem_resp); - uvm_config_db#(virtual stream_if #(`TILE_UNCACHED_WIDTH))::set( - null, "uvm_test_top", "uncached_req_0", uncached_req_0); - uvm_config_db#(virtual stream_if #(`TILE_UNCACHED_WIDTH))::set( - null, "uvm_test_top", "uncached_req_1", uncached_req_1); - uvm_config_db#(virtual stream_if #(`TILE_URESP_WIDTH))::set(null, "uvm_test_top", - "uncached_resp_0", uncached_resp_0); - uvm_config_db#(virtual stream_if #(`TILE_URESP_WIDTH))::set(null, "uvm_test_top", - "uncached_resp_1", uncached_resp_1); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache.f b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache.f deleted file mode 100644 index 33340ae79..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/virtual_cache_interface.sv -@RESOURCES@/virtual_cache_package.sv --F @RTL@/VirtualCacheSystem/filelist.f -@RESOURCES@/virtual_cache_tb.sv diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_atomic.f b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_atomic.f deleted file mode 100644 index ee4b21ef4..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_atomic.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+VIRTUAL_CACHE_TB=virtual_cache_bad_atomic_tb -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/virtual_cache_interface.sv -@RESOURCES@/virtual_cache_package.sv --F @RTL@/VirtualCacheSystem/filelist.f -@RESOURCES@/virtual_cache_tb.sv diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_atomic.toml b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_atomic.toml deleted file mode 100644 index 366502210..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_atomic.toml +++ /dev/null @@ -1,2 +0,0 @@ -expected_assertion = "Virtual LSU does not accept Fence or unknown atomics" -invalid_case = 1 diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_atomic_write.f b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_atomic_write.f deleted file mode 100644 index 793f41de9..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_atomic_write.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+VIRTUAL_CACHE_TB=virtual_cache_bad_atomic_write_tb -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/virtual_cache_interface.sv -@RESOURCES@/virtual_cache_package.sv --F @RTL@/VirtualCacheSystem/filelist.f -@RESOURCES@/virtual_cache_tb.sv diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_atomic_write.toml b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_atomic_write.toml deleted file mode 100644 index 5c3b59521..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_atomic_write.toml +++ /dev/null @@ -1,2 +0,0 @@ -expected_assertion = "Virtual LSU atomic requires a 32 or 64 bit operand" -invalid_case = 2 diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_atomic.f b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_atomic.f deleted file mode 100644 index 17d40a9c5..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_atomic.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+VIRTUAL_CACHE_TB=virtual_cache_bad_execute_atomic_tb -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/virtual_cache_interface.sv -@RESOURCES@/virtual_cache_package.sv --F @RTL@/VirtualCacheSystem/filelist.f -@RESOURCES@/virtual_cache_tb.sv diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_atomic.toml b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_atomic.toml deleted file mode 100644 index 24accfd7f..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_atomic.toml +++ /dev/null @@ -1,2 +0,0 @@ -expected_assertion = "Virtual instruction word requires unsigned 64 bit read without atomic" -invalid_case = 7 diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_signed.f b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_signed.f deleted file mode 100644 index 5f1106e80..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_signed.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+VIRTUAL_CACHE_TB=virtual_cache_bad_execute_signed_tb -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/virtual_cache_interface.sv -@RESOURCES@/virtual_cache_package.sv --F @RTL@/VirtualCacheSystem/filelist.f -@RESOURCES@/virtual_cache_tb.sv diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_signed.toml b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_signed.toml deleted file mode 100644 index 75aaca461..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_signed.toml +++ /dev/null @@ -1,2 +0,0 @@ -expected_assertion = "Virtual instruction word requires unsigned 64 bit read without atomic" -invalid_case = 9 diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_size.f b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_size.f deleted file mode 100644 index 8e4146c1d..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_size.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+VIRTUAL_CACHE_TB=virtual_cache_bad_execute_size_tb -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/virtual_cache_interface.sv -@RESOURCES@/virtual_cache_package.sv --F @RTL@/VirtualCacheSystem/filelist.f -@RESOURCES@/virtual_cache_tb.sv diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_size.toml b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_size.toml deleted file mode 100644 index 73034135d..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_size.toml +++ /dev/null @@ -1,2 +0,0 @@ -expected_assertion = "Virtual instruction word requires unsigned 64 bit read without atomic" -invalid_case = 8 diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_write.f b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_write.f deleted file mode 100644 index 83023cc9a..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_write.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+VIRTUAL_CACHE_TB=virtual_cache_bad_execute_write_tb -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/virtual_cache_interface.sv -@RESOURCES@/virtual_cache_package.sv --F @RTL@/VirtualCacheSystem/filelist.f -@RESOURCES@/virtual_cache_tb.sv diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_write.toml b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_write.toml deleted file mode 100644 index acc1d20e0..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_write.toml +++ /dev/null @@ -1,2 +0,0 @@ -expected_assertion = "Virtual instruction word requires unsigned 64 bit read without atomic" -invalid_case = 6 diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_mode.f b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_mode.f deleted file mode 100644 index 496ef0a7c..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_mode.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+VIRTUAL_CACHE_TB=virtual_cache_bad_mode_tb -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/virtual_cache_interface.sv -@RESOURCES@/virtual_cache_package.sv --F @RTL@/VirtualCacheSystem/filelist.f -@RESOURCES@/virtual_cache_tb.sv diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_mode.toml b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_mode.toml deleted file mode 100644 index 4eba68ed5..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_mode.toml +++ /dev/null @@ -1,2 +0,0 @@ -expected_assertion = "Virtual LSU requires Bare or Sv39 mode" -invalid_case = 4 diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_operand.f b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_operand.f deleted file mode 100644 index f2c5dba67..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_operand.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+VIRTUAL_CACHE_TB=virtual_cache_bad_operand_tb -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/virtual_cache_interface.sv -@RESOURCES@/virtual_cache_package.sv --F @RTL@/VirtualCacheSystem/filelist.f -@RESOURCES@/virtual_cache_tb.sv diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_operand.toml b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_operand.toml deleted file mode 100644 index 401130c64..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_operand.toml +++ /dev/null @@ -1,2 +0,0 @@ -expected_assertion = "Virtual LSU atomic requires a 32 or 64 bit operand" -invalid_case = 3 diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_privilege.f b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_privilege.f deleted file mode 100644 index c0895f988..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_privilege.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+VIRTUAL_CACHE_TB=virtual_cache_bad_privilege_tb -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/virtual_cache_interface.sv -@RESOURCES@/virtual_cache_package.sv --F @RTL@/VirtualCacheSystem/filelist.f -@RESOURCES@/virtual_cache_tb.sv diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_privilege.toml b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_privilege.toml deleted file mode 100644 index 3666d1758..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_privilege.toml +++ /dev/null @@ -1,2 +0,0 @@ -expected_assertion = "Virtual LSU effective privilege must be U, S or M" -invalid_case = 5 diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_size.f b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_size.f deleted file mode 100644 index 897a4abc0..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_size.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+VIRTUAL_CACHE_TB=virtual_cache_bad_size_tb -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/virtual_cache_interface.sv -@RESOURCES@/virtual_cache_package.sv --F @RTL@/VirtualCacheSystem/filelist.f -@RESOURCES@/virtual_cache_tb.sv diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_size.toml b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_size.toml deleted file mode 100644 index b51b400fc..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_size.toml +++ /dev/null @@ -1,2 +0,0 @@ -expected_assertion = "Virtual LSU size must be 1, 2, 4 or 8 bytes" -invalid_case = 0 diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_interface.sv b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_interface.sv deleted file mode 100644 index 3d8d7d9ee..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_interface.sv +++ /dev/null @@ -1,18 +0,0 @@ -`include "virtual_cache_config.svh" -interface virtual_cache_if ( - input logic clock -); - logic reset, active, conflict_trace; - int conflict_wb, conflict_dbid, conflict_beats, conflict_ack; - logic [`VM_OUTSTANDING_BITS-1:0] outstanding; - logic translation_valid, physical_valid, cache_valid, cache_pte, eviction_valid; - logic [`VM_TRANSLATION_WIDTH-1:0] translation_bits; - logic [`VM_PHYSICAL_WIDTH-1:0] physical_bits; - logic [`VM_CACHE_WIDTH-1:0] cache_bits; - logic [`VM_CACHE_ADDR_WIDTH-1:0] eviction_addr; - clocking sample @(posedge clock); - default input #1step; - input reset,active,outstanding,translation_valid,physical_valid,cache_valid,cache_pte,eviction_valid; - input translation_bits, physical_bits, cache_bits, eviction_addr; - endclocking -endinterface diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_package.sv b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_package.sv deleted file mode 100644 index bfcceade4..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_package.sv +++ /dev/null @@ -1,1127 +0,0 @@ -`include "virtual_cache_config.svh" -`define VF(K, F) `VM_``K``_``F``_OFFSET +: `VM_``K``_``F``_WIDTH -package virtual_cache_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - typedef bit [63:0] u64; - typedef logic [`VM_REQ_WIDTH-1:0] req_t; - typedef logic [`VM_MEMRESP_WIDTH-1:0] mem_t; - import "DPI-C" function chandle mmu_ref_create(input int unsigned address_bits); - import "DPI-C" function void mmu_ref_destroy(input chandle model); - import "DPI-C" function void mmu_ref_program( - input chandle model, - input longint unsigned address, - value, - input int unsigned error - ); - import "DPI-C" function int unsigned mmu_ref_pending(input chandle model); - import "DPI-C" function int unsigned mmu_ref_peek( - input chandle model, - output longint unsigned address - ); - import "DPI-C" function int unsigned mmu_ref_consume( - input chandle model, - input longint unsigned address - ); - import "DPI-C" function int unsigned mmu_ref_translate( - input chandle model, - input longint unsigned address, - input int unsigned mode, - input longint unsigned root, - input int unsigned privilege, - write, - execute, - sum, - mxr, - output longint unsigned paddr, - output int unsigned page_fault, - access_fault, - level - ); - import "DPI-C" function void cpu_mem_ref_prepare( - input longint unsigned addr, - input int unsigned size, - write, - input longint unsigned data, - input int unsigned atomic, - cacheable, - normal, - address_bits, - output int unsigned target, - misaligned, - access_fault, - output longint unsigned bus_addr, - bus_data, - output int unsigned bus_mask, - atomic_word - ); - import "DPI-C" function longint unsigned cpu_mem_ref_result( - input longint unsigned addr, - input int unsigned size, - write, - signed_load, - atomic, - cacheable, - input longint unsigned raw, - input int unsigned error - ); - class result_item extends uvm_sequence_item; - bit [5:0] tag; - u64 data; - bit misaligned, page_fault, access_fault; - `uvm_object_utils_begin(result_item) - `uvm_field_int(tag, UVM_DEFAULT) - `uvm_field_int(data, UVM_DEFAULT) - `uvm_field_int(misaligned, UVM_DEFAULT) - `uvm_field_int(page_fault, UVM_DEFAULT) - `uvm_field_int(access_fault, UVM_DEFAULT) - `uvm_object_utils_end - function new(string name = "result_item"); - super.new(name); - endfunction - endclass - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual virtual_cache_if control; - virtual stream_if #(`VM_AUTH_WIDTH) auth_req; - virtual stream_if #(1) auth_resp; - virtual stream_if #(`VM_REQ_WIDTH) req; - virtual stream_if #(`VM_RESP_WIDTH) resp; - virtual stream_if #(`VM_MEMREQ_WIDTH) mem_req; - virtual stream_if #(`VM_MEMRESP_WIDTH) mem_resp; - virtual stream_if #(`VM_UNCACHED_WIDTH) uncached_req; - virtual stream_if #(`VM_URESP_WIDTH) uncached_resp; - in_order_scoreboard #(result_item) scoreboard; - chandle mmu; - bit [511:0] backing[u64], architectural[u64]; - bit [7:0] mmio_backing[u64], mmio_arch[u64]; - bit line_error[u64], mmio_error[u64]; - int evictions[u64]; - typedef struct { - mem_t bits; - int due; - } memory_entry; - memory_entry pending[$]; - bit mem_active = 0, uncached_active = 0, uncached_pending = 0; - mem_t memory_packet; - logic [`VM_URESP_WIDTH-1:0] uncached_packet; - int - uncached_due, - cycle = 0, - expected = 0, - accepted = 0, - returned = 0, - pte_reads = 0, - data_accesses = 0, - uncached_accesses = 0; - int - memory_reads = 0, - memory_writes = 0, - req_stalls = 0, - resp_stalls = 0, - mem_stalls = 0, - uncached_stalls = 0; - int page_faults = 0, access_faults = 0, misalignments = 0, sc_successes = 0, sc_evictions = 0; - int invalid_case = -1; - bit deny_final = 0, deny_pte = 0; - u64 denied_pte = 0; - bit - final_queried = 0, - auth_pending = 0, - auth_allow = 0, - auth_is_pte = 0, - pte_permit = 0, - final_permit = 0; - u64 auth_address = 0, pte_permitted_address = 0; - int auth_due = 0, auth_queries = 0, auth_denials = 0, auth_stalls = 0; - - bit - hold_response = 0, - case_active = 0, - expect_translation = 0, - expect_physical = 0, - expect_cache = 0, - expect_uncached = 0; - bit saw_translation = 0, saw_physical = 0, saw_cache = 0, saw_uncached = 0, commit_write = 0; - int uncached_pte_reads = 0; - int expected_region = 0, commit_bytes = 0; - u64 expected_pa = 0, commit_data = 0; - logic [`VM_TRANSLATION_WIDTH-1:0] translation_expected; - logic [`VM_PHYSICAL_WIDTH-1:0] physical_expected; - logic [`VM_CACHE_WIDTH-1:0] cache_expected; - logic [`VM_UNCACHED_WIDTH-1:0] uncached_expected; - req_t current; - string current_case; - u64 next_table = `VM_DDR_BASE + 64'h10000; - localparam u64 DATA = `VM_DDR_BASE + 64'h800180; - localparam u64 EVICT_DATA = `VM_DDR_BASE + 64'h810000; - localparam u64 ERROR_DATA = `VM_DDR_BASE + 64'h900180; - localparam u64 MAX_PA = (64'h1 << `VM_CACHE_ADDR_WIDTH) - 1; - function new(string name, uvm_component parent); - super.new(name, parent); - timeout = 2ms; - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - scoreboard = in_order_scoreboard#(result_item)::type_id::create("scoreboard", this); - mmu = mmu_ref_create(`VM_CACHE_ADDR_WIDTH); - if (!uvm_config_db#(virtual stream_if #(`VM_AUTH_WIDTH))::get( - this, "", "auth_req", auth_req - ) || !uvm_config_db#(virtual stream_if #(1))::get( - this, "", "auth_resp", auth_resp - ) || !uvm_config_db#(virtual virtual_cache_if)::get( - this, "", "control", control - ) || !uvm_config_db#(virtual stream_if #(`VM_REQ_WIDTH))::get( - this, "", "req", req - ) || !uvm_config_db#(virtual stream_if #(`VM_RESP_WIDTH))::get( - this, "", "resp", resp - ) || !uvm_config_db#(virtual stream_if #(`VM_MEMREQ_WIDTH))::get( - this, "", "mem_req", mem_req - ) || !uvm_config_db#(virtual stream_if #(`VM_MEMRESP_WIDTH))::get( - this, "", "mem_resp", mem_resp - ) || !uvm_config_db#(virtual stream_if #(`VM_UNCACHED_WIDTH))::get( - this, "", "uncached_req", uncached_req - ) || !uvm_config_db#(virtual stream_if #(`VM_URESP_WIDTH))::get( - this, "", "uncached_resp", uncached_resp - )) - `uvm_fatal("VIF", "Virtual cache interfaces missing") - void'(uvm_config_db#(int)::get(this, "", "invalid_case", invalid_case)); - endfunction - function int region(u64 address, int bytes); - if (address > MAX_PA || bytes - 1 > MAX_PA - address) return 0; - if(address>=`VM_READ_ONLY_BASE&&address+bytes<=`VM_WRITE_ONLY_BASE+`VM_WRITE_ONLY_BYTES) - return 1; - if(address>=`VM_DDR_BASE&&address-`VM_DDR_BASE<`VM_DDR_BYTES&&bytes<=`VM_DDR_BYTES-(address-`VM_DDR_BASE)) - return 1; - if(address>=`VM_MMIO_BASE&&address-`VM_MMIO_BASE<`VM_MMIO_BYTES&&bytes<=`VM_MMIO_BYTES-(address-`VM_MMIO_BASE)) - return 2; - if(address>=`VM_SHORT_MMIO_BASE&&address-`VM_SHORT_MMIO_BASE<`VM_SHORT_MMIO_BYTES&&bytes<=`VM_SHORT_MMIO_BYTES-(address-`VM_SHORT_MMIO_BASE)) - return 2; - if(address>=`VM_NORMAL_RAM_BASE&&address+bytes<=`VM_NORMAL_RAM_BASE+`VM_NORMAL_RAM_BYTES) - return 3; - return 0; - endfunction - function void initialize_line(u64 address, bit [511:0] value); - u64 base = address & ~64'h3f; - if (backing.exists(base)) `uvm_fatal("FIXTURE", "Attempt to reinitialize a live DDR line") - backing[base] = value; - architectural[base] = value; - endfunction - function void program_pte(u64 address, u64 value, bit error = 0); - u64 base = address & ~64'h3f; - if (region(address, 8) == 1 || region(address, 8) == 3) begin - if (!backing.exists(base)) initialize_line(base, '0); - backing[base][int'(address&63)*8+:64] = value; - architectural[base] = backing[base]; - if (error) line_error[base] = 1; - end else if (!error) `uvm_fatal("FIXTURE", "Non-DDR PTE must be an explicit access error") - mmu_ref_program(mmu, address, value, error); - endfunction - function u64 mapping(u64 va, u64 pa, u64 flags = 64'hcf, int error_level = -1); - u64 root = next_table; - next_table += 64'h3000; - for (int level = 2; level >= 0; level--) begin - u64 table = root + (2 - level) * 4096; - u64 address = table + ((va >> (12 + 9 * level)) & 511) * 8; - program_pte(address, - level == 0 ? ((pa >> 12) << 10) | flags : (((table + 4096) >> 12) << 10) | 1, - level == error_level); - end - return root >> 12; - endfunction - function req_t command(u64 address, int size = 3, bit write = 0, bit signed_load = 0, - u64 data = 0, int atomic = 0, int mode = 0, u64 root = 0, - int privilege = 1, bit sum = 0, bit mxr = 0, bit execute = 0); - req_t r = '0; - r[`VF(REQ, EXECUTE)] = execute; - r[`VF(REQ, VADDR)] = address; - r[`VF(REQ, SIZE)] = size; - r[`VF(REQ, WRITE)] = write; - r[`VF(REQ, SIGNED)] = signed_load; - r[`VF(REQ, DATA)] = data; - r[`VF(REQ, ATOMIC)] = atomic; - r[`VF(REQ, SATPMODE)] = mode; - r[`VF(REQ, ROOTPPN)] = root; - r[`VF(REQ, PRIVILEGE)] = privilege; - r[`VF(REQ, SUM)] = sum; - r[`VF(REQ, MXR)] = mxr; - return r; - endfunction - function u64 read_arch(u64 address, int bytes, int kind); - u64 value = 0; - for (int i = 0; i < bytes; i++) begin - u64 a = address + i; - if (kind == 1 || kind == 3) begin - if (!architectural.exists(a & ~64'h3f)) - `uvm_fatal("FIXTURE", $sformatf("Uninitialized architecture DDR %h", a)) - value |= u64'(architectural[a&~64'h3f][int'(a&63)*8+:8]) << (i * 8); - end else begin - if (!mmio_arch.exists(a)) - `uvm_fatal("FIXTURE", $sformatf("Uninitialized architecture MMIO %h", a)) - value |= u64'(mmio_arch[a]) << (i * 8); - end - end - return value; - endfunction - function void predict(req_t r, int sc_result); - int unsigned target, misaligned, af, mask, word, pf, level; - longint unsigned pa, bus_addr, bus_data; - u64 raw, old, operand, next_value, saved_pte; - int size, atom, bytes; - bit write, failed; - result_item answer = new(current_case); - if (mmu_ref_pending(mmu)) - `uvm_fatal("PTE_PENDING", "Previous reference PTE chain not drained") - current = r; - case_active = 1; - commit_write = 0; - pte_permit = 0; - final_permit = 0; - final_queried = 0; - expect_translation = 0; - expect_physical = 0; - expect_cache = 0; - expect_uncached = 0; - saw_translation = 0; - saw_physical = 0; - saw_cache = 0; - saw_uncached = 0; - translation_expected = '0; - physical_expected = '0; - cache_expected = '0; - uncached_expected = '0; - size = r[`VF(REQ, SIZE)]; - bytes = 1 << size; - atom = r[`VF(REQ, ATOMIC)]; - write = r[`VF(REQ, WRITE)]; - answer.tag = r[`VF(REQ, TAG)]; - // Before translation only alignment is meaningful; use physical reference - // again with the translated PA for its actual range/classification result. - cpu_mem_ref_prepare(r[`VF(REQ, VADDR)], size, write, r[`VF(REQ, DATA)], atom, 1, 1, - `VM_CACHE_ADDR_WIDTH, target, misaligned, af, bus_addr, bus_data, mask, - word); - answer.misaligned = misaligned; - if (!misaligned) begin - expect_translation = 1; - if (deny_pte) begin - saved_pte = read_arch(denied_pte, 8, 1); - mmu_ref_program(mmu, denied_pte, saved_pte, 1); - end - if (mmu_ref_translate( - mmu, - r[ - `VF(REQ, VADDR) - ], - r[ - `VF(REQ, SATPMODE) - ], - r[ - `VF(REQ, ROOTPPN) - ], - r[ - `VF(REQ, PRIVILEGE) - ], - write || (atom >= 1 && atom <= 9) || atom == 11, - r[ - `VF(REQ, EXECUTE) - ], - r[ - `VF(REQ, SUM) - ], - r[ - `VF(REQ, MXR) - ], - pa, - pf, - af, - level - ) != 0) - `uvm_fatal("FIXTURE", $sformatf("Reference requested an unprogrammed PTE %h", pa)) - if (deny_pte) mmu_ref_program(mmu, denied_pte, saved_pte, 0); - translation_expected[`VF(TRANSLATION, PADDR)] = pa; - translation_expected[`VF(TRANSLATION, PAGEFAULT)] = pf; - translation_expected[`VF(TRANSLATION, ACCESSFAULT)] = af; - translation_expected[`VF(TRANSLATION, LEVEL)] = level; - answer.page_fault = pf; - answer.access_fault = af; - if (!pf && !af) begin - expected_pa = pa; - expected_region = region(pa, bytes); - if(deny_final||(pa>=`VM_READ_ONLY_BASE&&pa<`VM_READ_ONLY_BASE+64&&(write||atom!=0||r[ - `VF(REQ, EXECUTE) - ])) || (pa >= `VM_WRITE_ONLY_BASE && pa < `VM_WRITE_ONLY_BASE + 64 && - (!write || atom != 0 || r[ - `VF(REQ, EXECUTE) - ])) || expected_region == 0 || - (atom != 0 && expected_region != 1 && expected_region != 3) || (r[ - `VF(REQ, EXECUTE) - ] && expected_region != 1 && expected_region != 3)) - answer.access_fault = 1; - else begin - expect_physical = 1; - physical_expected[`VF(PHYSICAL, ADDR)] = pa; - physical_expected[`VF(PHYSICAL, TAG)] = r[`VF(REQ, TAG)]; - physical_expected[`VF(PHYSICAL, SIZE)] = size; - physical_expected[`VF(PHYSICAL, WRITE)] = write; - physical_expected[`VF(PHYSICAL, SIGNED)] = r[`VF(REQ, SIGNED)]; - physical_expected[`VF(PHYSICAL, DATA)] = r[`VF(REQ, DATA)]; - physical_expected[`VF(PHYSICAL, ATOMIC)] = atom; - physical_expected[`VF(PHYSICAL, CACHEABLE)] = expected_region == 1; - physical_expected[`VF(PHYSICAL, NORMAL)] = expected_region == 1 || expected_region == 3; - cpu_mem_ref_prepare(pa, size, write, r[`VF(REQ, DATA)], atom, expected_region == 1, - expected_region == 1 || expected_region == 3, `VM_CACHE_ADDR_WIDTH, - target, misaligned, af, bus_addr, bus_data, mask, word); - answer.access_fault = af; - if (target != 0) begin - expect_cache = target == 1; - expect_uncached = target == 2; - cache_expected[`VF(CACHE, ADDR)] = bus_addr; - cache_expected[`VF(CACHE, WRITE)] = write; - cache_expected[`VF(CACHE, DATA)] = bus_data; - cache_expected[`VF(CACHE, MASK)] = mask; - cache_expected[`VF(CACHE, ATOMIC)] = atom; - cache_expected[`VF(CACHE, ATOMICWORD)] = word; - uncached_expected[`VF(UNCACHED, ADDR)] = pa; - uncached_expected[`VF(UNCACHED, TAG)] = r[`VF(REQ, TAG)]; - uncached_expected[`VF(UNCACHED, SIZE)] = size; - uncached_expected[`VF(UNCACHED, WRITE)] = write; - uncached_expected[`VF(UNCACHED, DATA)] = r[`VF(REQ, DATA)]; - uncached_expected[`VF(UNCACHED, ATOMIC)] = atom; - uncached_expected[ - `VF(UNCACHED, NORMAL) - ] = expected_region == 1 || expected_region == 3; - failed = expected_region != 2 ? - (line_error.exists(pa & ~64'h3f) && line_error[pa&~64'h3f]) : - (mmio_error.exists(pa) && mmio_error[pa]); - answer.access_fault = failed; - if (!failed) begin - old = read_arch(pa, bytes, expected_region); - operand = r[`VF(REQ, DATA)]; - next_value = operand; - if (atom != 0) begin - raw = (bytes == 4 && old[31]) ? old | 64'hffffffff00000000 : old; - case (atom) - 1: next_value = operand; - 2: next_value = old + operand; - 3: next_value = old ^ operand; - 4: next_value = old & operand; - 5: next_value = old | operand; - 6: - next_value = bytes == 4 ? - ($signed(old[31:0]) < $signed(operand[31:0]) ? old : operand) : - ($signed(old) < $signed(operand) ? old : operand); - 7: - next_value = bytes == 4 ? - ($signed(old[31:0]) > $signed(operand[31:0]) ? old : operand) : - ($signed(old) > $signed(operand) ? old : operand); - 8: - next_value=bytes==4?(old[31:0]operand[31:0]?old:operand):(old>operand?old:operand); - 10: next_value = old; - 11: begin - if (sc_result < 0 || sc_result > 1) - `uvm_fatal("SC_ORACLE", - "SC requires an explicit architectural scenario outcome") - raw = sc_result; - end - default: `uvm_fatal("ATOMIC", "Unsupported test operation") - endcase - end else raw = target == 1 ? read_arch(pa & ~64'h7, 8, 1) : old; - answer.data = cpu_mem_ref_result(pa, size, write, r[`VF(REQ, SIGNED)], atom, - target == 1, raw, 0); - commit_write = write || (atom >= 1 && atom <= 9) || (atom == 11 && sc_result == 0); - commit_bytes = bytes; - commit_data = next_value; - end - end - end - end - end - scoreboard.expected_export.write(answer); - expected++; - endfunction - function void observations(); - if (control.sample.eviction_valid) evictions[u64'(control.sample.eviction_addr)]++; - if (control.sample.cache_valid) begin - u64 address; - longint unsigned next_pte; - logic [`VM_CACHE_WIDTH-1:0] wanted; - if (!case_active || $isunknown({control.sample.cache_pte, control.sample.cache_bits})) - `uvm_fatal("CACHE_EVENT", "Unknown or unsolicited cache operation") - address = control.sample.cache_bits[`VF(CACHE, ADDR)]; - if (control.sample.cache_pte) begin - if (!pte_permit || pte_permitted_address != address || region( - address, 8 - ) != 1 || !mmu_ref_peek( - mmu, next_pte - ) || address != next_pte) - `uvm_fatal("PTE_CHAIN", $sformatf( - "%s wrong real-cache PTE address %h", current_case, address)) - wanted = '0; - wanted[`VF(CACHE, ADDR)] = next_pte; - if (control.sample.cache_bits !== wanted) - `uvm_fatal("PTE_WRITE", - "PTE access must be an ordinary cache read, never A/D writeback") - if (!mmu_ref_consume(mmu, address)) - `uvm_fatal("PTE_CHAIN", "PTE reference queue mismatch") - pte_permit = 0; - pte_reads++; - end else begin - if(!expect_cache||saw_cache||!saw_translation||!saw_physical||control.sample.cache_bits!==cache_expected) - `uvm_fatal("CACHE_DATA", $sformatf( - "%s wrong PA/lane/mask/operand/atomic cache request", current_case)) - saw_cache = 1; - data_accesses++; - end - end - if (control.sample.translation_valid) begin - longint unsigned denied; - if (!expect_translation || saw_translation || $isunknown( - control.sample.translation_bits - ) || control.sample.translation_bits !== translation_expected) - `uvm_fatal("TRANSLATION", $sformatf( - "%s wrong translation result got=%h expected=%h", - current_case, - control.sample.translation_bits, - translation_expected - )) - if (mmu_ref_pending(mmu)) begin - if (!mmu_ref_peek( - mmu, denied - ) || (region( - denied, 8 - ) != 2 && region( - denied, 8 - ) != 0 && (!deny_pte || denied != denied_pte)) || !translation_expected[ - `VF(TRANSLATION, ACCESSFAULT) - ]) - `uvm_fatal("PTE_CHAIN", "Translation skipped a required cache PTE read") - if (!mmu_ref_consume(mmu, denied) || mmu_ref_pending(mmu)) - `uvm_fatal("PMA", "Unexpected denied-PTE reference chain") - end - saw_translation = 1; - end - if (control.sample.physical_valid) begin - if (!final_permit || !expect_physical || saw_physical || !saw_translation || $isunknown( - control.sample.physical_bits - ) || control.sample.physical_bits !== physical_expected) - `uvm_fatal("PHYSICAL", $sformatf( - "%s PA classification or captured command changed", current_case)) - saw_physical = 1; - end - endfunction - task service(); - forever begin - @(control.sample); - if (!control.sample.reset) begin - cycle++; - if (req.sample.valid && req.sample.ready) accepted++; - if (req.sample.valid && !req.sample.ready) req_stalls++; - if (resp.sample.valid && !resp.sample.ready) resp_stalls++; - if (mem_req.sample.valid && !mem_req.sample.ready) mem_stalls++; - if (uncached_req.sample.valid && !uncached_req.sample.ready) uncached_stalls++; - if (auth_req.sample.valid && !auth_req.sample.ready) auth_stalls++; - if (auth_resp.sample.valid && auth_resp.sample.ready) begin - if (!auth_pending) `uvm_fatal("AUTH_RESPONSE", "Authorization response without query") - if (auth_allow) begin - if (auth_is_pte) begin - pte_permit = 1; - pte_permitted_address = auth_address; - end else final_permit = 1; - end else auth_denials++; - auth_pending = 0; - end - if (auth_req.sample.valid && auth_req.sample.ready) begin - logic [`VM_AUTH_WIDTH-1:0] wanted = '0; - longint unsigned pte; - if (auth_pending || $isunknown(auth_req.sample.bits)) - `uvm_fatal("AUTH_QUERY", "Overlapping or unknown authorization query") - auth_is_pte = auth_req.sample.bits[`VF(AUTH, ISPTE)]; - if (auth_is_pte) begin - if (!mmu_ref_peek(mmu, pte)) `uvm_fatal("AUTH_PTE", "Unsolicited PTE authorization") - wanted[`VF(AUTH, PADDR)] = pte; - wanted[`VF(AUTH, SIZE)] = 3; - wanted[`VF(AUTH, READ)] = 1; - wanted[`VF(AUTH, PRIVILEGE)] = 1; - wanted[`VF(AUTH, ISPTE)] = 1; - end else begin - if (final_queried) `uvm_fatal("AUTH_FINAL", "Repeated final query") - final_queried = 1; - if (!saw_translation || translation_expected[ - `VF(TRANSLATION, PAGEFAULT) - ] || translation_expected[ - `VF(TRANSLATION, ACCESSFAULT) - ]) - `uvm_fatal("AUTH_FINAL", "Final authorization before valid translation") - wanted[`VF(AUTH, PADDR)] = translation_expected[`VF(TRANSLATION, PADDR)]; - wanted[`VF(AUTH, SIZE)] = current[`VF(REQ, SIZE)]; - wanted[ - `VF(AUTH, READ) - ] = !current[ - `VF(REQ, EXECUTE) - ] && !current[ - `VF(REQ, WRITE) - ] && current[ - `VF(REQ, ATOMIC) - ] != 11; - wanted[ - `VF(AUTH, WRITE) - ] = current[ - `VF(REQ, WRITE) - ] || (current[ - `VF(REQ, ATOMIC) - ] != 0 && current[ - `VF(REQ, ATOMIC) - ] != 10); - wanted[`VF(AUTH, EXECUTE)] = current[`VF(REQ, EXECUTE)]; - wanted[`VF(AUTH, PRIVILEGE)] = current[`VF(REQ, PRIVILEGE)]; - end - if (auth_req.sample.bits !== wanted) - `uvm_fatal("AUTH_FIELDS", $sformatf( - "%s got=%h expected=%h", current_case, auth_req.sample.bits, wanted)) - auth_address = auth_req.sample.bits[`VF(AUTH, PADDR)]; - auth_allow = !(auth_is_pte ? (deny_pte && auth_address == denied_pte) : deny_final); - auth_pending = 1; - auth_due = cycle + 3 + (auth_queries % 5); - auth_queries++; - end - observations(); - if (mem_active && mem_resp.sample.valid && mem_resp.sample.ready) mem_active = 0; - if (uncached_active && uncached_resp.sample.valid && uncached_resp.sample.ready) begin - uncached_active = 0; - uncached_pending = 0; - end - if (mem_req.sample.valid && mem_req.sample.ready) begin - memory_entry e; - u64 address; - bit write, failed; - bit [511:0] line; - if ($isunknown( - { - mem_req.sample.bits[`VF(MEMREQ, ID)], - mem_req.sample.bits[`VF(MEMREQ, ADDR)], - mem_req.sample.bits[`VF(MEMREQ, WRITE)] - } - )) - `uvm_fatal("DDR_X", "Unknown DDR request") - address = mem_req.sample.bits[`VF(MEMREQ, ADDR)]; - write = mem_req.sample.bits[`VF(MEMREQ, WRITE)]; - if (address[5:0] != 0 || region(address, 64) != 1 || !backing.exists(address)) - `uvm_fatal("DDR_DICTIONARY", $sformatf("Unexpected/uninitialized DDR line %h", address - )) - failed = !write && line_error.exists(address) && line_error[address]; - if (write) begin - if ($isunknown( - { - mem_req.sample.bits[`VF(MEMREQ, DATA)], - mem_req.sample.bits[`VF(MEMREQ, MASK)] - } - )) - `uvm_fatal("DDR_WRITE_X", "Unknown DDR write") - line = mem_req.sample.bits[`VF(MEMREQ, DATA)]; - if (mem_req.sample.bits[`VF(MEMREQ, MASK)] !== '1 || line !== architectural[address]) - `uvm_fatal("DDR_WRITE", "Cache writeback corrupted full initialized line") - backing[address] = line; - memory_writes++; - end else memory_reads++; - e.bits = '0; - e.bits[`VF(MEMRESP, ID)] = mem_req.sample.bits[`VF(MEMREQ, ID)]; - e.bits[`VF(MEMRESP, DATA)] = backing[address]; - e.bits[`VF(MEMRESP, ERROR)] = failed; - e.due = cycle + 5 + (int'(e.bits[`VF(MEMRESP, ID)]) % 4); - pending.push_back(e); - end - if (uncached_req.sample.valid && uncached_req.sample.ready) begin - u64 address, value, next_pte; - int bytes, kind; - bit failed, write, is_pte; - address = uncached_req.sample.bits[`VF(UNCACHED, ADDR)]; - bytes = 1 << uncached_req.sample.bits[`VF(UNCACHED, SIZE)]; - write = uncached_req.sample.bits[`VF(UNCACHED, WRITE)]; - kind = region(address, bytes); - is_pte = !saw_physical && mmu_ref_pending(mmu) != 0; - if (uncached_pending || $isunknown(uncached_req.sample.bits)) - `uvm_fatal("UNCACHED_OWNER", "Overlapping or unknown uncached request") - if (is_pte) begin - logic [`VM_UNCACHED_WIDTH-1:0] wanted = '0; - if (!pte_permit || pte_permitted_address != address || kind != 3 || !mmu_ref_peek( - mmu, next_pte - ) || next_pte != address) - `uvm_fatal("PTE_CHAIN", "Wrong authorized uncached PTE address") - wanted[`VF(UNCACHED, ADDR)] = address; - wanted[`VF(UNCACHED, TAG)] = current[`VF(REQ, TAG)]; - wanted[`VF(UNCACHED, SIZE)] = 3; - wanted[`VF(UNCACHED, NORMAL)] = 1; - if (uncached_req.sample.bits !== wanted || !mmu_ref_consume(mmu, address)) - `uvm_fatal("PTE_REQUEST", "Uncached PTE contract mismatch") - pte_permit = 0; - pte_reads++; - uncached_pte_reads++; - end else begin - if(!expect_uncached||saw_uncached||!saw_physical||uncached_req.sample.bits!==uncached_expected) - `uvm_fatal("UNCACHED_REQUEST", $sformatf( - "%s unexpected uncached request", current_case)) - saw_uncached = 1; - end - if (kind != 2 && kind != 3) `uvm_fatal("UNCACHED_REGION", "Invalid uncached region") - failed = kind == 3 ? (line_error.exists(address & ~64'h3f) && line_error[address&~64'h3f - ]) : (mmio_error.exists(address) && mmio_error[address]); - value = 0; - for (int i = 0; i < bytes; i++) begin - if (kind == 3) begin - if (!backing.exists(address & ~64'h3f)) - `uvm_fatal("RAM_DICTIONARY", "Uninitialized normal RAM") - value |= u64'(backing[address&~64'h3f][int'((address+i)&63)*8+:8]) << (i * 8); - if (write && !failed) - backing[address&~64'h3f][int'((address+i)&63)*8+:8]=uncached_req.sample.bits[`VM_UNCACHED_DATA_OFFSET+i*8+:8]; - end else begin - if (!mmio_backing.exists(address + i)) - `uvm_fatal("MMIO_DICTIONARY", "Uninitialized device") - value |= u64'(mmio_backing[address+i]) << (i * 8); - if (write && !failed) - mmio_backing[address+i]=uncached_req.sample.bits[`VM_UNCACHED_DATA_OFFSET+i*8+:8]; - end - end - uncached_packet = '0; - uncached_packet[`VF(URESP, TAG)] = uncached_req.sample.bits[`VF(UNCACHED, TAG)]; - uncached_packet[`VF(URESP, DATA)] = write ? 0 : value; - uncached_packet[`VF(URESP, ERROR)] = failed; - uncached_due = cycle + 7; - uncached_pending = 1; - uncached_accesses++; - end - if (resp.sample.valid && resp.sample.ready) begin - result_item actual = new(current_case); - if (!case_active || $isunknown(resp.sample.bits)) - `uvm_fatal("RESULT", "Unknown or unsolicited virtual result") - actual.tag = resp.sample.bits[`VF(RESP, TAG)]; - actual.data = resp.sample.bits[`VF(RESP, DATA)]; - actual.misaligned = resp.sample.bits[`VF(RESP, MISALIGNED)]; - actual.page_fault = resp.sample.bits[`VF(RESP, PAGEFAULT)]; - actual.access_fault = resp.sample.bits[`VF(RESP, ACCESSFAULT)]; - if(saw_translation!=expect_translation||saw_physical!=expect_physical||saw_cache!=expect_cache||saw_uncached!=expect_uncached||mmu_ref_pending( - mmu - )) - `uvm_fatal("SIDE_EFFECT", - "Fault suppressed or successful operation skipped a required stage") - if (auth_pending || final_queried != (expect_translation && !translation_expected[ - `VF(TRANSLATION, PAGEFAULT) - ] && !translation_expected[ - `VF(TRANSLATION, ACCESSFAULT) - ])) - `uvm_fatal("AUTH_DRAIN", "Missing final authorization or response before decision") - scoreboard.actual_export.write(actual); - returned++; - case_active = 0; - if (actual.page_fault) page_faults++; - if (actual.access_fault) access_faults++; - if (actual.misaligned) misalignments++; - if (commit_write) - for (int i = 0; i < commit_bytes; i++) begin - u64 a = expected_pa + i; - if (expected_region == 1 || expected_region == 3) - architectural[a&~64'h3f][int'(a&63)*8+:8] = commit_data[i*8+:8]; - else mmio_arch[a] = commit_data[i*8+:8]; - end - end - end - @(negedge control.clock); - auth_req.ready = !control.reset && !auth_pending && cycle % 7 >= 4; - auth_resp.valid = !control.reset && auth_pending && cycle >= auth_due; - auth_resp.bits = auth_allow; - mem_req.ready = !control.reset && cycle % 7 >= 2; - uncached_req.ready = !control.reset && cycle % 9 >= 4; - resp.ready = !control.reset && !hold_response && cycle % 11 >= 4; - if (!mem_active) - for (int i = pending.size() - 1; i >= 0; i--) - if (pending[i].due <= cycle) begin - memory_packet = pending[i].bits; - pending.delete(i); - mem_active = 1; - break; - end - mem_resp.valid = mem_active; - mem_resp.bits = memory_packet; - if (uncached_pending && cycle >= uncached_due) uncached_active = 1; - uncached_resp.valid = uncached_active; - uncached_resp.bits = uncached_packet; - end - endtask - task run_case(string name, req_t packet, int sc_result = -1, bit perturb = 0, bit drain = 1); - current_case = name; - packet[`VF(REQ, TAG)] = expected & 63; - predict(packet, sc_result); - @(negedge control.clock); - req.bits = packet; - req.valid = 1; - do @(req.sample); while (!req.sample.ready); - @(negedge control.clock); - req.valid = 0; - if (perturb) req.bits = ~packet; - scoreboard.wait_checked(expected); - if (drain) begin - do - @(control.sample); - while (control.sample.outstanding != 0 || pending.size() || mem_active || uncached_pending); - repeat (8) @(control.sample); - end - endtask - task execute(); - bit [511:0] pattern; - u64 root, va, prior; - int sizes[4] = '{0, 1, 2, 3}; - auth_req.ready = 0; - auth_resp.valid = 0; - auth_resp.bits = 0; - control.conflict_trace = 0; - control.reset = 1; - control.active = 0; - req.valid = 0; - req.bits = '0; - resp.ready = 0; - mem_req.ready = 0; - mem_resp.valid = 0; - mem_resp.bits = '0; - uncached_req.ready = 0; - uncached_resp.valid = 0; - uncached_resp.bits = '0; - repeat (5) @(negedge control.clock); - control.reset = 0; - control.active = 1; - if (invalid_case >= 0) begin - req_t bad = command(1); - bad[`VF(REQ, TAG)] = 23; - case (invalid_case) - 0: bad[`VF(REQ, SIZE)] = 4; - 1: bad[`VF(REQ, ATOMIC)] = 12; - 2: begin - bad[`VF(REQ, ATOMIC)] = 2; - bad[`VF(REQ, WRITE)] = 1; - end - 3: begin - bad[`VF(REQ, ATOMIC)] = 2; - bad[`VF(REQ, SIZE)] = 1; - end - 4: bad[`VF(REQ, SATPMODE)] = 9; - 5: bad[`VF(REQ, PRIVILEGE)] = 2; - 6: begin - bad[`VF(REQ, EXECUTE)] = 1; - bad[`VF(REQ, WRITE)] = 1; - end - 7: begin - bad[`VF(REQ, EXECUTE)] = 1; - bad[`VF(REQ, ATOMIC)] = 10; - end - 8: begin - bad[`VF(REQ, EXECUTE)] = 1; - bad[`VF(REQ, SIZE)] = 2; - end - 9: begin - bad[`VF(REQ, EXECUTE)] = 1; - bad[`VF(REQ, SIGNED)] = 1; - end - default: `uvm_fatal("CASE", "Unknown virtual negative case") - endcase - req.bits = bad; - req.valid = 1; - `uvm_info("INVALID_INPUT", $sformatf( - "Driving illegal virtual command %0d with misaligned VA", - invalid_case), UVM_LOW) - repeat (10) @(negedge control.clock); - `uvm_fatal("MISSING_ASSERTION", "Illegal command hidden by early fault response") - end - for (int w = 0; w < 8; w++) - pattern[w*64+:64] = 64'h88776655fedcba80 ^ (64'h0101010101010101 * w); - initialize_line(`VM_DDR_BASE + 64'h4c0, pattern); - begin - bit [511:0] conflict_data = pattern; - conflict_data[63:0] = 64'hcafebabebeefaa11; - initialize_line(`VM_DDR_BASE + 64'h40c0, conflict_data); - end - initialize_line(`VM_READ_ONLY_BASE, pattern); - initialize_line(`VM_WRITE_ONLY_BASE, pattern); - initialize_line(DATA, pattern); - initialize_line(DATA + 4096, pattern); - initialize_line(DATA + 8192, pattern); - initialize_line(EVICT_DATA, pattern); - initialize_line(ERROR_DATA, pattern); - initialize_line(`VM_DDR_BASE + `VM_DDR_BYTES - 64, pattern); - line_error[ERROR_DATA&~64'h3f] = 1; - for (int i = 0; i < 32; i++) begin - mmio_arch[`VM_MMIO_BASE+i] = 8'h80 + i; - mmio_backing[`VM_MMIO_BASE+i] = 8'h80 + i; - end - for (int i = 0; i < 5; i++) begin - mmio_arch[`VM_SHORT_MMIO_BASE+i] = 8'hf0 + i; - mmio_backing[`VM_SHORT_MMIO_BASE+i] = 8'hf0 + i; - end - mmio_error[`VM_MMIO_BASE+24] = 1; - fork - service(); - join_none - repeat (12) @(control.sample); - // Match the actual Core conflict sequence without idle cycles or HN drain - // between returned requests: code and data have the same bank/index. - control.conflict_trace = 1; - run_case("conflict_code_first", command( - `VM_DDR_BASE + 64'h4c0, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1), -1, 0, 0); - run_case("conflict_store_W", command(`VM_DDR_BASE + 64'h40c0, 2, 1, 0, 64'h7fffffff), -1, 0, - 0); - run_case("conflict_code_after_store", command( - `VM_DDR_BASE + 64'h4d8, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1), -1, 0, 0); - run_case("conflict_code_AMO_word", command( - `VM_DDR_BASE + 64'h4e0, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1), -1, 0, 0); - run_case("conflict_AMO_W", command(`VM_DDR_BASE + 64'h40c0, 2, 0, 0, 1, 2), -1, 0, 0); - run_case("conflict_code_after_AMO", command( - `VM_DDR_BASE + 64'h4e0, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1), -1, 0, 0); - run_case("conflict_data_readback", command(`VM_DDR_BASE + 64'h40c0), -1, 0, 0); - do - @(control.sample); - while (control.sample.outstanding || pending.size() || mem_active || uncached_pending); - if(control.conflict_wb!=2||control.conflict_dbid!=2||control.conflict_beats!=4||control.conflict_ack!=6) - `uvm_fatal("CONFLICT_CHI", $sformatf( - "Missing WB/DBID/data/ACK: %0d/%0d/%0d/%0d", - control.conflict_wb, - control.conflict_dbid, - control.conflict_beats, - control.conflict_ack - )) - control.conflict_trace = 0; - `uvm_info( - "CONFLICT", - "Code/store/code/AMO.W/code collision completed without idle-cycle insertion; all queues drained", - UVM_LOW) - // Hold the second request unchanged until the first response retires, then - // accept and check it normally; never withdraw a stalled VALID. - current_case = "single_outstanding_snapshot"; - begin - req_t first = command(DATA, 0, 0, 1); - req_t second = command(`VM_MMIO_BASE, 3, 1, 0, 64'hdeadbeef); - first[`VF(REQ, TAG)] = 31; - second[`VF(REQ, TAG)] = 17; - predict(first, -1); - hold_response = 1; - @(negedge control.clock); - req.bits = first; - req.valid = 1; - do @(req.sample); while (!req.sample.ready); - @(negedge control.clock); - req.bits = second; - wait (resp.sample.valid); - repeat (12) begin - @(req.sample); - if (req.sample.ready) - `uvm_fatal("OUTSTANDING", "Accepted a second command before retirement"); - end - @(negedge control.clock); - hold_response = 0; - scoreboard.wait_checked(expected); - @(negedge control.clock); - current_case = "pending_request_after_retirement"; - predict(second, -1); - do @(req.sample); while (!req.sample.ready); - @(negedge control.clock); - req.valid = 0; - scoreboard.wait_checked(expected); - do - @(control.sample); - while (control.sample.outstanding || pending.size() || mem_active || uncached_pending); - repeat (8) @(control.sample); - end - foreach (sizes[s]) - for (int offset = 0; offset < 8; offset += (1 << sizes[s])) - for (int sign_load = 0; sign_load < 2; sign_load++) - run_case("bare_lane_sign", command(DATA + offset, sizes[s], 0, sign_load)); - foreach (sizes[s]) begin - run_case("store_mask", command( - DATA + (sizes[s] == 0 ? 7 : sizes[s] == 1 ? 2 : sizes[s] == 2 ? 4 : 0), - sizes[s], - 1, - 0, - 64'hfe12345689abcdef - )); - run_case("masked_store_readback", command(DATA)); - end - // The numerical VA deliberately belongs to the opposite physical class. - va = 64'h10000180; - root = mapping(va, DATA); - run_case("VA_MMIO_maps_DDR", command(va, 3, 0, 0, 0, 0, 8, root), -1, 1); - run_case("LR_D", command(va, 3, 0, 0, 0, 10, 8, root)); - run_case("SC_D_success", command(va, 3, 0, 0, 64'h8899aabbccddeeff, 11, 8, root), 0); - sc_successes++; - run_case("LR_W_upper", command(va + 4, 2, 0, 0, 0, 10, 8, root)); - run_case("SC_W_success", command(va + 4, 2, 0, 0, 64'h7fffffff, 11, 8, root), 0); - sc_successes++; - for (int op = 1; op <= 9; op++) begin - run_case("AMO_W_upper", command(va + 4, 2, 0, 0, 64'h80000001 + op, op, 8, root)); - run_case("AMO_neighbor_preserved", command(va, 2, 0, 0, 0, 0, 8, root)); - end - run_case("dirty_private_eviction", command(DATA + 4096)); - run_case("dirty_shared_eviction_to_DDR", command(DATA + 8192)); - if (memory_writes == 0) - `uvm_fatal("DDR_WRITEBACK", "Dirty data never reached the explicit DDR dictionary") - run_case("DDR_writeback_readback", command(DATA)); - va = 64'h80000010; - root = mapping(va, `VM_MMIO_BASE + 16); - run_case("VA_DDR_maps_MMIO", command(va, 1, 0, 1, 0, 0, 8, root), -1, 1); - run_case("MMIO_store", command(va, 1, 1, 0, 64'hf123, 0, 8, root)); - run_case("MMIO_readback", command(va, 1, 0, 1, 0, 0, 8, root)); - run_case("MMIO_backend_error", command(`VM_MMIO_BASE + 24, 3)); - run_case("DDR_backend_error", command(ERROR_DATA)); - run_case("MMIO_atomic_denied", command(`VM_MMIO_BASE, 2, 0, 0, 1, 2)); - run_case("unknown_PA", command(64'h40000000)); - run_case("short_MMIO_last_byte", command(`VM_SHORT_MMIO_BASE + 4, 0, 0, 1)); - run_case("cross_region_word", command(`VM_SHORT_MMIO_BASE + 4, 2)); - run_case("cross_region_double", command(`VM_SHORT_MMIO_BASE, 3)); - run_case("DDR_last_byte", command(`VM_DDR_BASE + `VM_DDR_BYTES - 1, 0)); - run_case("DDR_after_end", command(`VM_DDR_BASE + `VM_DDR_BYTES, 0)); - run_case("misalign_before_missing_PTE", command(1, 3, 0, 0, 0, 0, 8, 0)); - run_case("noncanonical", command(64'h4000000000, 0, 0, 0, 0, 0, 8, 0)); - va = 64'h4180; - root = mapping(va, DATA, 64'h43); - run_case("LR_read_only", command(va, 3, 0, 0, 0, 10, 8, root)); - for (int op = 1; op <= 9; op++) - run_case("AMO_needs_store_permission", command(va, 3, 0, 0, 1, op, 8, root)); - run_case("SC_needs_store_permission", command(va, 3, 0, 0, 1, 11, 8, root)); - root = mapping(va, DATA, 64'h4f); - run_case("Svade_dirty_fault", command(va, 3, 1, 0, 1, 0, 8, root)); - root = mapping(va, DATA, 64'hf); - run_case("Svade_A_bit_page_fault", command(va, 3, 0, 0, 0, 0, 8, root)); - root = mapping(va, DATA, 64'h59); - run_case("SUM_denied", command(va, 3, 0, 0, 0, 0, 8, root, 1, 0, 1)); - run_case("SUM_MXR_allowed", command(va, 3, 0, 0, 0, 0, 8, root, 1, 1, 1), -1, 1); - run_case("U_load_ignores_SUM", command(va, 3, 0, 0, 0, 0, 8, root, 0, 0, 1)); - root = mapping(va, DATA, 64'hcf); - run_case("U_cannot_read_supervisor_page", command(va, 3, 0, 0, 0, 0, 8, root, 0)); - run_case("M_bypasses_satp", command(DATA, 3, 0, 0, 0, 0, 8, 0, 3)); - root = mapping(64'hffffffc000000180, DATA); - run_case("negative_canonical_kernel_VA", command(64'hffffffc000000180, 3, 0, 0, 0, 0, 8, root - )); - root = mapping(va, DATA, 64'hcf, 1); - run_case("PTE_backend_error", command(va, 3, 0, 0, 0, 0, 8, root)); - mmu_ref_program(mmu, `VM_MMIO_BASE, 0, 1); - run_case("PTE_cannot_access_MMIO", command(va, 3, 0, 0, 0, 0, 8, `VM_MMIO_BASE >> 12)); - mmu_ref_program(mmu, 64'h20000000, 0, 1); - run_case("unknown_PTE_region", command(va, 3, 0, 0, 0, 0, 8, 64'h20000)); - root = mapping(va, 64'h40000180); - run_case("translated_unknown_PA", command(va, 3, 0, 0, 0, 0, 8, root)); - root = mapping(va, 64'h1 << `VM_CACHE_ADDR_WIDTH); - run_case("translated_PA_overflow", command(va, 3, 0, 0, 0, 0, 8, root)); - // Instruction words share the real PTE/data cache path; Fetch owns RVC assembly. - va = 64'h4180; - root = mapping(va, DATA, 64'h49); - run_case("X_only_data_denied", command(va, 3, 0, 0, 0, 0, 8, root)); - run_case("X_only_instruction_word", command(va, 3, 0, 0, 0, 0, 8, root, 1, 0, 0, 1), -1, 1); - run_case("U_execute_supervisor_denied", command(va, 3, 0, 0, 0, 0, 8, root, 0, 1, 1, 1)); - root = mapping(va, DATA, 64'h59); - run_case("U_execute_user_word", command(va, 3, 0, 0, 0, 0, 8, root, 0, 0, 0, 1)); - run_case("S_execute_user_SUM_denied", command(va, 3, 0, 0, 0, 0, 8, root, 1, 1, 1, 1)); - root = mapping(va, DATA, 64'h43); - run_case("readable_nonX_instruction_denied", command(va, 3, 0, 0, 0, 0, 8, root, 1, 0, 1, 1)); - run_case("instruction_MMIO_no_side_effect", command( - `VM_MMIO_BASE, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1)); - root = mapping(va, `VM_MMIO_BASE + 384, 64'h49); - run_case("translated_instruction_MMIO_denied", command(va, 3, 0, 0, 0, 0, 8, root, 1, 0, 0, 1 - )); - // Two explicit aligned requests straddle a page; this owner never splits accesses. - root = mapping(64'h4ff8, DATA, 64'h49); - initialize_line((DATA & ~64'hfff) + 64'hfc0, pattern); - run_case("page_last_instruction_word", command(64'h4ff8, 3, 0, 0, 0, 0, 8, root, 1, 0, 0, 1)); - // The next PTE is explicitly initialized invalid in the same fixture line. - mmu_ref_program(mmu, (root << 12) + 8192 + 5 * 8, 0, 0); - run_case("page_second_instruction_PF", command(64'h5000, 3, 0, 0, 0, 0, 8, root, 1, 0, 0, 1)); - root = mapping(64'h5000, 64'h40000000, 64'h49); - run_case("page_second_instruction_AF", command(64'h5000, 3, 0, 0, 0, 0, 8, root, 1, 0, 0, 1)); - run_case("PMA_read_only_load", command(`VM_READ_ONLY_BASE)); - run_case("PMA_read_only_store_denied", command(`VM_READ_ONLY_BASE, 3, 1, 0, 64'hbadbad)); - run_case("PMA_read_only_AMO_denied", command(`VM_READ_ONLY_BASE, 2, 0, 0, 1, 2)); - run_case("PMA_cached_nonexecutable", command( - `VM_READ_ONLY_BASE, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1)); - run_case("PMA_write_only_read_denied", command(`VM_WRITE_ONLY_BASE)); - // Authorization uses a policy captured for this accepted command. Denials - // reuse the MMU backend-error result contract without modifying DDR data. - deny_final = 1; - run_case("authorization_deny_load", command(DATA), -1, 1); - run_case("authorization_deny_store", command(DATA, 3, 1, 0, 64'hdeadbeef), -1, 1); - run_case("authorization_deny_AMO", command(DATA, 2, 0, 0, 1, 2), -1, 1); - run_case("authorization_deny_execute", command(DATA, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1), -1, 1); - run_case("authorization_deny_MMIO", command(`VM_MMIO_BASE, 3, 1, 0, 64'hbadbad), -1, 1); - deny_final = 0; - run_case("denied_writes_preserve_DDR", command(DATA)); - run_case("denied_write_preserves_MMIO", command(`VM_MMIO_BASE)); - for (int denied_level = 0; denied_level < 3; denied_level++) begin - va = 64'h4180; - root = mapping(va, DATA); - denied_pte=(root<<12)+(2-denied_level)*4096+((va>>(12+9*denied_level))&511)*8; - deny_pte = 1; - run_case("authorization_deny_PTE", command(va, 3, 0, 0, 0, 0, 8, root), -1, 1); - deny_pte = 0; - run_case("PTE_authorization_reenabled", command(va, 3, 0, 0, 0, 0, 8, root)); - end - if (auth_denials != 8 || !auth_stalls || auth_pending) - `uvm_fatal("AUTH_SCENARIOS", "Authorization denials/stalls/drain missing") - `uvm_info( - "AUTHORIZATION", - $sformatf( - "queries=%0d denies=%0d queryStalls=%0d; PTE/final fields and no-before-grant backend checked", - auth_queries, auth_denials, auth_stalls), UVM_LOW) - // No reservation timing model: require a real clean Evict of the LR line - // during the subsequent walk and independently check SC's architectural 1. - va = 64'h4000; - root = mapping(va, EVICT_DATA); - run_case("LR_before_PTE_eviction", command(EVICT_DATA, 3, 0, 0, 0, 10)); - prior = evictions.exists(EVICT_DATA) ? evictions[EVICT_DATA] : 0; - run_case("SC_after_real_PTE_eviction", command(va, 3, 0, 0, 64'h1234, 11, 8, root), 1); - if (!evictions.exists(EVICT_DATA) || evictions[EVICT_DATA] <= prior) - `uvm_fatal("SC_EVICTION", "No real cache eviction witnessed between LR and failed SC") - sc_evictions++; - run_case("failed_SC_preserves_data", command(EVICT_DATA)); - initialize_line(`VM_NORMAL_RAM_BASE, 512'hfedcba98765432100123456789abcdef); - run_case("normal_RAM_load", command(`VM_NORMAL_RAM_BASE)); - run_case("normal_RAM_execute", command(`VM_NORMAL_RAM_BASE, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1)); - run_case("normal_RAM_store", command(`VM_NORMAL_RAM_BASE + 8, 3, 1, 0, 64'h8123456789abcdef)); - run_case("normal_RAM_store_readback", command(`VM_NORMAL_RAM_BASE + 8)); - next_table = `VM_NORMAL_RAM_BASE + 64'h10000; - va = 64'h12345000; - root = mapping(va, `VM_NORMAL_RAM_BASE); - run_case("uncached_PTE_uncached_data", command(va, 3, 0, 0, 0, 0, 8, root), -1, 1); - run_case("uncached_PTE_execute", command(va, 3, 0, 0, 0, 0, 8, root, 1, 0, 0, 1)); - va = 64'h12345180; - root = mapping(va, DATA); - run_case("uncached_PTE_cached_data", command(va, 3, 0, 0, 0, 0, 8, root)); - root = mapping(va, DATA, 8'h0f); - run_case("uncached_PTE_missing_AD_fault", command(va, 3, 0, 0, 0, 0, 8, root)); - root = mapping(va, DATA); - begin - u64 address = (root << 12) + ((va >> 30) & 511) * 8; - program_pte(address, read_arch(address, 8, 3), 1); - end - run_case("uncached_PTE_backend_error", command(va, 3, 0, 0, 0, 0, 8, root)); - root = mapping(va, DATA); - denied_pte = (root << 12) + ((va >> 30) & 511) * 8; - deny_pte = 1; - run_case("uncached_PTE_authorization_denied", command(va, 3, 0, 0, 0, 0, 8, root), -1, 1); - deny_pte = 0; - line_error[`VM_NORMAL_RAM_BASE] = 1; - run_case("normal_RAM_execute_error", command( - `VM_NORMAL_RAM_BASE, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1)); - run_case("normal_RAM_load_error", command(`VM_NORMAL_RAM_BASE)); - if (uncached_pte_reads < 10) - `uvm_fatal("UNCACHED_PTE", "Uncached page walk cases were not exercised") - if(expected!=accepted||expected!=returned||!pte_reads||!data_accesses||!uncached_accesses||!req_stalls||!resp_stalls||!mem_stalls||!uncached_stalls||!page_faults||!access_faults||!misalignments||!sc_successes||!sc_evictions) - `uvm_fatal("SCENARIOS", "Virtual LSU scenarios not all exercised") - `uvm_info( - "CHECKS", - $sformatf( - "virtual LSU checked=%0d realCachePTE=%0d dataCache=%0d MMIO=%0d DDRreads/writes=%0d/%0d PF/AF/misaligned=%0d/%0d/%0d SCsuccess/evicted=%0d/%0d stalls(req/resp/DDR/MMIO)=%0d/%0d/%0d/%0d; all queues drained", - expected, pte_reads, data_accesses, uncached_accesses, memory_reads, memory_writes, - page_faults, access_faults, misalignments, sc_successes, sc_evictions, req_stalls, - resp_stalls, mem_stalls, uncached_stalls), UVM_LOW) - endtask - function void final_phase(uvm_phase phase); - mmu_ref_destroy(mmu); - endfunction - endclass -endpackage diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_tb.sv b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_tb.sv deleted file mode 100644 index d5b4387ba..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_tb.sv +++ /dev/null @@ -1,102 +0,0 @@ -`include "virtual_cache_config.svh" -`ifndef VIRTUAL_CACHE_TB -`define VIRTUAL_CACHE_TB virtual_cache_tb -`endif -module `VIRTUAL_CACHE_TB; - import uvm_pkg::*; - import virtual_cache_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - virtual_cache_if control (clock); - stream_if #(`VM_AUTH_WIDTH) auth_req ( - clock, - control.reset - ); - stream_if #(1) auth_resp ( - clock, - control.reset - ); - stream_if #(`VM_REQ_WIDTH) source_if ( - clock, - control.reset - ); - stream_if #(`VM_RESP_WIDTH) sink_if ( - clock, - control.reset - ); - stream_if #(`VM_MEMREQ_WIDTH) mem_req ( - clock, - control.reset - ); - stream_if #(`VM_MEMRESP_WIDTH) mem_resp ( - clock, - control.reset - ); - stream_if #(`VM_UNCACHED_WIDTH) uncached_req ( - clock, - control.reset - ); - stream_if #(`VM_URESP_WIDTH) uncached_resp ( - clock, - control.reset - ); - VirtualCacheSystem dut ( - .clock(clock), - .reset(control.reset), - .io_active(control.active), - .io_outstanding(control.outstanding), - `include "virtual_cache_ports.svh" - ); - // Observe existing HN boundary; no DUT logic is added. rxDat/rxRsp ready - // are structurally true and optimized out of emitted RTL. - always @(posedge clock) begin - if (control.reset) begin - control.conflict_wb = 0; - control.conflict_dbid = 0; - control.conflict_beats = 0; - control.conflict_ack = 0; - end else if (control.conflict_trace) begin - if (dut.caches.home.home.io_req_valid && dut.caches.home.home.io_req_ready) begin - if (dut.caches.home.home.io_req_bits_opcode == 7'h1b) control.conflict_wb++; - $display("[CONFLICT_CHI] REQ op=%h txn=%h addr=%h", dut.caches.home.home.io_req_bits_opcode, - dut.caches.home.home.io_req_bits_txnId, dut.caches.home.home.io_req_bits_addr); - end - if (dut.caches.home.home.io_rsp_valid && dut.caches.home.home.io_rsp_ready) begin - if (dut.caches.home.home.io_rsp_bits_opcode == 5'h05) control.conflict_dbid++; - $display("[CONFLICT_CHI] RSP op=%h txn=%h dbid=%h", dut.caches.home.home.io_rsp_bits_opcode, - dut.caches.home.home.io_rsp_bits_txnId, dut.caches.home.home.io_rsp_bits_dbid); - end - if (dut.caches.home.home.io_rxDat_valid && dut.caches.home.home.io_rxDat_bits_opcode==4'h2) begin - control.conflict_beats++; - $display( - "[CONFLICT_CHI] RXDAT op=%h txn=%h beat=%h", dut.caches.home.home.io_rxDat_bits_opcode, - dut.caches.home.home.io_rxDat_bits_txnId, dut.caches.home.home.io_rxDat_bits_dataId); - end - if (dut.caches.home.home.io_rxRsp_valid && dut.caches.home.home.io_rxRsp_bits_opcode==5'h02) begin - control.conflict_ack++; - $display("[CONFLICT_CHI] RXRSP op=%h txn=%h", dut.caches.home.home.io_rxRsp_bits_opcode, - dut.caches.home.home.io_rxRsp_bits_txnId); - end - end - end - initial begin - int invalid_case; - uvm_config_db#(virtual stream_if #(`VM_AUTH_WIDTH))::set(null, "uvm_test_top", "auth_req", - auth_req); - uvm_config_db#(virtual stream_if #(1))::set(null, "uvm_test_top", "auth_resp", auth_resp); - uvm_config_db#(virtual virtual_cache_if)::set(null, "uvm_test_top", "control", control); - uvm_config_db#(virtual stream_if #(`VM_REQ_WIDTH))::set(null, "uvm_test_top", "req", source_if); - uvm_config_db#(virtual stream_if #(`VM_RESP_WIDTH))::set(null, "uvm_test_top", "resp", sink_if); - uvm_config_db#(virtual stream_if #(`VM_MEMREQ_WIDTH))::set(null, "uvm_test_top", "mem_req", - mem_req); - uvm_config_db#(virtual stream_if #(`VM_MEMRESP_WIDTH))::set(null, "uvm_test_top", "mem_resp", - mem_resp); - uvm_config_db#(virtual stream_if #(`VM_UNCACHED_WIDTH))::set(null, "uvm_test_top", - "uncached_req", uncached_req); - uvm_config_db#(virtual stream_if #(`VM_URESP_WIDTH))::set(null, "uvm_test_top", "uncached_resp", - uncached_resp); - if ($value$plusargs("invalid_case=%d", invalid_case)) - uvm_config_db#(int)::set(null, "uvm_test_top", "invalid_case", invalid_case); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/scala/TaskAdmission.scala b/arch/src/main/scala/framework/root/root-tile/src/main/scala/TaskAdmission.scala deleted file mode 100644 index 64c67e690..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/scala/TaskAdmission.scala +++ /dev/null @@ -1,55 +0,0 @@ -package hier.tile - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.system.core.rocket.{CpuParams, HasCpuParameters} -import hier.core.rocket.CommandSnapshot -import framework.system.core.rocket.{RoCCIO, RoCCResponseBB} -import memcore.memory.interlock.{Params => TrackingParams, Tag} - -/** Routes an accepted task command without allocating an accelerator ROB entry. */ -@instantiable -class TaskAdmission(tracking: TrackingParams, pmps: Int)(implicit val cpuParams: CpuParams) - extends Module - with HasCpuParameters { - - @public - val io = IO(new Bundle { - val command = Flipped(Decoupled(new CommandSnapshot(tracking, pmps))) - val task = Flipped(new RoCCIO(64)) - val response = Decoupled(new RoCCResponseBB(64)) - val release = Decoupled(new Tag(tracking)) - val satp = Output(UInt(64.W)) - // Worker completion cannot publish a result while its accepted NPU/DMA/cache work remains live. - val workDrained = Input(Bool()) - }) - - val idle :: responding :: releasing :: Nil = Enum(3) - val state = RegInit(idle) - val tag = Reg(UInt(tracking.idBits.W)) - val satp = Reg(UInt(64.W)) - val operation = io.command.bits.instruction.funct - val finishTask = operation === 2.U || operation === 14.U || - (operation >= 16.U && operation <= 19.U) - val eligible = state === idle && (!finishTask || io.workDrained) && !reset.asBool - io.task.cmd.valid := io.command.valid && eligible - io.task.cmd.bits := io.command.bits.instruction - io.command.ready := io.task.cmd.ready && eligible - io.task.exception := false.B - when(io.command.fire) { - assert(io.command.bits.instruction.opcode === "h2b".U, "TaskAdmission received a compute instruction") - tag := io.command.bits.tag - satp := io.command.bits.satp - state := responding - } - // TaskController samples the root on the command handshake, including the first staged field. - io.satp := Mux(state === idle && io.command.valid, io.command.bits.satp, satp) - io.response.valid := state === responding && io.task.resp.valid && !reset.asBool - io.response.bits := io.task.resp.bits - io.task.resp.ready := state === responding && io.response.ready && !reset.asBool - when(io.response.fire)(state := releasing) - io.release.valid := state === releasing && !reset.asBool - io.release.bits.tag := tag - when(io.release.fire)(state := idle) -} diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/scala/TaskController.scala b/arch/src/main/scala/framework/root/root-tile/src/main/scala/TaskController.scala deleted file mode 100644 index f62d0a3dc..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/scala/TaskController.scala +++ /dev/null @@ -1,108 +0,0 @@ -package hier.tile - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.system.core.rocket.RoCCIO - -/** Linux submits compiled kernels; workers consume contexts without joining Linux SMP. */ -@instantiable -class TaskController(coreIds: Seq[Int], signatures: Seq[BigInt], cpuCount: Int) extends Module { - require(coreIds == (1 to signatures.size) && signatures.nonEmpty) - require(cpuCount == coreIds.size + 1, "task workers must include every non-controller CPU") - - @public - val io = IO(new Bundle { - val ports = Vec(cpuCount, new RoCCIO(64)) - val satp = Input(UInt(64.W)) - }) - - val count = coreIds.size - val kinds = VecInit(signatures.map(_.U(64.W))) - val contexts = Reg(Vec(count, Vec(7, UInt(64.W)))) - val roots = Reg(Vec(count, UInt(64.W))) - val active = RegInit(VecInit(Seq.fill(count)(false.B))) - val available = RegInit(VecInit(Seq.fill(count)(false.B))) - val results = RegInit(VecInit(Seq.fill(count)(1.U(2.W)))) - val staged = Reg(Vec(7, UInt(64.W))) - val stagedRoot = Reg(UInt(64.W)) - val stagedFields = RegInit(0.U(3.W)) - val workspace = RegInit(0.U(64.W)) - - for (cpu <- 0 until cpuCount) { - val port = io.ports(cpu) - val worker = coreIds.indexOf(cpu) - val pending = RegInit(false.B) - val rd = Reg(UInt(5.W)) - val response = Reg(UInt(64.W)) - val command = port.cmd.bits - val op = command.funct - val index = command.rs1Data - val value = command.rs2Data - val slot = index(math.max(1, log2Ceil(count)) - 1, 0) - val field = index(2, 0) - val inRange = index < count.U - val free = VecInit((0 until count).map(i => !active(i) && kinds(i) === value)).asUInt.orR - val waitReady = - if (cpu == 0) - Mux(op === 1.U, Mux(index === count.U, free, !inRange || !active(slot)), true.B) - else if (worker >= 0) Mux(op === 8.U, available(worker), true.B) - else true.B - port.cmd.ready := !pending && waitReady - port.resp.valid := pending - port.resp.bits.rd := rd - port.resp.bits.data := response - port.busy := pending - port.interrupt := false.B - when(port.resp.fire)(pending := false.B) - when(port.cmd.fire) { - pending := true.B - rd := command.rd - response := 2.U - if (cpu == 0) { - switch(op) { - is(0.U) { - when(inRange && !active(slot) && stagedFields === 7.U && - stagedRoot === io.satp && kinds(slot) === value && staged(6) === value) { - contexts(slot) := staged - roots(slot) := stagedRoot - active(slot) := true.B - available(slot) := true.B - results(slot) := 0.U - stagedFields := 0.U - response := 0.U - } - } - is(1.U)(response := Mux(index === count.U, 0.U, Mux(inRange, results(slot), 2.U))) - is(3.U)(response := count.U) - is(4.U)(response := Mux(inRange, kinds(slot), 0.U)) - is(5.U)(response := workspace) - is(6.U) { workspace := index; response := 0.U } - is(7.U) { - when(index === 0.U || (index < 7.U && index === stagedFields && stagedRoot === io.satp)) { - staged(field) := value - stagedFields := field + 1.U - when(index === 0.U)(stagedRoot := io.satp) - response := 0.U - } - } - is(10.U)(response := Mux(inRange, results(slot), 2.U)) - } - } else if (worker >= 0) { - switch(op) { - is(2.U) { - when(active(worker) && !available(worker)) { - active(worker) := false.B - results(worker) := Mux(index === 0.U, 1.U, 2.U) - response := 0.U - } - } - is(5.U)(response := contexts(worker)(4)) - is(11.U)(when(active(worker) && !available(worker))(response := 0.U)) - is(8.U) { available(worker) := false.B; response := 1.U } - is(9.U)(response := Mux(index === 7.U, roots(worker), Mux(index < 7.U, contexts(worker)(field), 0.U))) - } - } - } - } -} diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/CacheSystem.scala b/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/CacheSystem.scala deleted file mode 100644 index 16bc528b3..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/CacheSystem.scala +++ /dev/null @@ -1,59 +0,0 @@ -package hier.tile.memory - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import memcore.bus.chi._ -import memcore.bus.chi.rnf.{BankedChiCache, CacheAccess, CacheResult, RnfParams} -import memcore.bus.chi.snf.{LineRequest, LineResponse} -import memcore.memory.coherence.configs.CoherenceParams - -/** Independent canonical cache composition with physical active-link CHI channels. */ -@instantiable -class CacheSystem(p: CoherenceParams) extends Module { - require(p.agents == 2 && p.homeId == 64 && p.mshrEntries == 4) - val c = p.chi - - @public - val io = IO(new Bundle { - val active = Input(Bool()) - val blockRequesterRsp = Input(Bool()) - val access = Vec(p.agents, Flipped(Decoupled(new CacheAccess(c)))) - val result = Vec(p.agents, Decoupled(new CacheResult)) - val memoryReq = Decoupled(new LineRequest(c)) - val memoryResp = Flipped(Decoupled(new LineResponse(c))) - val outstanding = Output(UInt(log2Ceil(p.mshrEntries + 1).W)) - val observedReq = Output(Valid(new RequestFlit(c))) - val observedRsp = Output(Valid(new ResponseFlit(c))) - val observedDat = Output(Valid(new DataFlit(c))) - val observedSnp = Output(Valid(new DirectedSnoop(c))) - val observedRxRsp = Output(Valid(new ResponseFlit(c))) - val observedRxDat = Output(Valid(new DataFlit(c))) - }) - - val home: Instance[Home] = Instantiate(new Home(p)) - - val cores: Seq[Instance[BankedChiCache]] = Seq.tabulate(p.agents)(i => - Instantiate(new BankedChiCache( - RnfParams(c, nodeId = i + 1, cacheLines = 8, homeId = p.homeId, homeCount = 1, banks = 2) - )) - ) - - home.io.active := io.active - home.io.blockRequesterRsp := io.blockRequesterRsp - io.memoryReq <> home.io.memoryReq - home.io.memoryResp <> io.memoryResp - io.outstanding := home.io.outstanding - for (i <- 0 until p.agents) { - cores(i).io.access <> io.access(i) - io.result(i) <> cores(i).io.result - home.io.requesters(i) <> cores(i).io.chi - } - io.observedReq := home.io.observedReq - io.observedRsp := home.io.observedRsp - io.observedDat := home.io.observedDat - io.observedSnp := home.io.observedSnp - io.observedRxRsp := home.io.observedRxRsp - io.observedRxDat := home.io.observedRxDat - -} diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Composition.scala b/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Composition.scala deleted file mode 100644 index 73d8bc78e..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Composition.scala +++ /dev/null @@ -1,175 +0,0 @@ -package hier.tile.memory - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.system.core.rocket.CpuParams -import hier.core.rocket.{AdmissionPorts, Commands, CpuCLink} -import hier.tile.TaskController -import framework.system.core.rocket.RoCCIO -import memcore.bus.chi.RequesterPort -import memcore.bus.chi.rnf.RnfParams -import memcore.bus.chi.snf.{LineRequest, LineResponse} -import memcore.memory.cpu.{CpuMemParams, PhysicalRegion, UncachedRequest, UncachedResponse} -import memcore.memory.coherence.configs.CoherenceParams -import memcore.memory.interlock.{Params => TrackingParams} - -/** Roles describe command capabilities; a scheduler may also have compute capability. */ -case class CoreRole(compute: Boolean, scheduler: Boolean) { - require(compute || scheduler) -} - -object CoreRole { - val Controller = CoreRole(compute = false, scheduler = true) - val Compute = CoreRole(compute = true, scheduler = false) -} - -case class CorePlacement( - l1: RnfParams, - cpu: CpuParams, - role: CoreRole) - -class CoreInterrupts extends Bundle { - val timer = Bool() - val software = Bool() - val external = Bool() - val supervisorExternal = Bool() -} - -/** Explicit CPU and task assembly; the Chip owner supplies each Core admission. */ -@instantiable -class Composition( - memory: CoherenceParams, - placements: Seq[CorePlacement], - regions: Seq[PhysicalRegion], - tracking: TrackingParams, - workerCoreIds: Seq[Int], - signatures: Seq[BigInt], - controlCoreIds: Seq[Int] = Nil) - extends Module { - val localCoreIds = placements.indices.filterNot(controlCoreIds.contains) - require(controlCoreIds.distinct.size == controlCoreIds.size) - require(controlCoreIds.forall(placements.indices.contains)) - // Control requesters leave the tile; the remaining cores use the local Home. - require(placements.nonEmpty && 2 * placements.size == memory.agents) - require(workerCoreIds.size == signatures.size && signatures.forall(s => s >= 0 && s < (BigInt(1) << 64))) - require( - workerCoreIds.forall(i => i > 0 && i < placements.size), - "Task worker IDs must explicitly name non-controller placements" - ) - require(placements.head.role.scheduler, "TaskController port zero must be a scheduler-capable placement") - val c = memory.chi - val cp = CpuMemParams(c, tagBits = 6) - require(tracking.addressBits == c.addressBits) - placements.zipWithIndex.foreach { case (entry, index) => - require(entry.l1.chi == c && entry.l1.homeId == memory.homeId && entry.l1.homeCount == 1) - // Memory/Fabric routes node i+1. This is a CHI node contract, never a hart-ID derivation. - require(entry.l1.nodeId == index + 1, "Core CHI node must match its Memory requester port") - } - - @public - val io = IO(new Bundle { - - // Caller routes opcode 0x2b commands and supplies their captured controller SATP. - val cores = MixedVec(placements.zipWithIndex.map { case (entry, index) => - val controlIndex = controlCoreIds.indexOf(index) - val localIndex = localCoreIds.indexOf(index) - val node = if (controlIndex >= 0) controlIndex + 1 else localIndex + 1 - new CpuCLink(entry.l1.copy(nodeId = node), Some(Commands(entry.role.compute, entry.role.scheduler, tracking)))( - entry.cpu - ) - }.map(Flipped(_))) - - val taskControl = Vec(placements.size, new RoCCIO(64)) - val controllerSatp = Input(UInt(64.W)) - val hartIds = Input(Vec(placements.size, UInt(64.W))) - val resetVector = Input(Vec(placements.size, UInt(64.W))) - val time = Input(UInt(64.W)) - val interrupts = Input(Vec(placements.size, new CoreInterrupts)) - val admission = - MixedVec(placements.map(entry => new AdmissionPorts(tracking, entry.cpu.core.nPMPs, c)(entry.cpu))) - val uncachedRequest = Vec(placements.size, Decoupled(new UncachedRequest(cp))) - val uncachedResponse = Vec(placements.size, Flipped(Decoupled(new UncachedResponse(cp)))) - val control = Vec(2 * controlCoreIds.size, new RequesterPort(c)) - val backingReq = Vec(if (localCoreIds.nonEmpty) 1 else 0, Decoupled(new LineRequest(c))) - val backingResp = Vec(if (localCoreIds.nonEmpty) 1 else 0, Flipped(Decoupled(new LineResponse(c)))) - val homeOutstanding = Output(UInt(log2Ceil(memory.mshrEntries + 1).W)) - val retired = Output(Vec(placements.size, Bool())) - val retiredPc = Output(Vec(placements.size, UInt(64.W))) - val trapped = Output(Vec(placements.size, Bool())) - val trapCause = Output(Vec(placements.size, UInt(64.W))) - val trapValue = Output(Vec(placements.size, UInt(64.W))) - val trapPc = Output(Vec(placements.size, UInt(64.W))) - }) - - if (workerCoreIds.nonEmpty) { - val taskController: Instance[TaskController] = - Instantiate(new TaskController(workerCoreIds, signatures, placements.size)) - taskController.io.ports <> io.taskControl - taskController.io.satp := io.controllerSatp - } else { - // A Tile without task workers reports zero workers (op 3) and an invalid index (2) otherwise. - for (port <- io.taskControl) { - val pending = RegInit(false.B) - val rd = Reg(UInt(5.W)) - val response = Reg(UInt(64.W)) - port.cmd.ready := !pending - port.resp.valid := pending - port.resp.bits.rd := rd - port.resp.bits.data := response - port.busy := pending - port.interrupt := false.B - when(port.cmd.fire) { - pending := true.B - rd := port.cmd.bits.rd - response := Mux(port.cmd.bits.funct === 3.U, 0.U, 2.U) - } - when(port.resp.fire)(pending := false.B) - } - } - for (port <- io.taskControl) { - when(port.cmd.valid) { - assert(port.cmd.bits.opcode === "h2b".U, "Composition task port requires an opcode 0x2b command") - } - } - - val memorySystem = Option.when(localCoreIds.nonEmpty)( - Instantiate(new Memory(memory.copy(agents = 2 * localCoreIds.size))) - ) - - io.homeOutstanding := 0.U - memorySystem.foreach { m => - io.backingReq <> m.io.backingReq - m.io.backingResp <> io.backingResp - io.homeOutstanding := m.io.homeOutstanding - } - - for (index <- placements.indices) { - val controlIndex = controlCoreIds.indexOf(index) - val localIndex = localCoreIds.indexOf(index) - val core = io.cores(index) - core.hartId := io.hartIds(index) - core.resetVector := io.resetVector(index) - core.time := io.time - core.timerInterrupt := io.interrupts(index).timer - core.softwareInterrupt := io.interrupts(index).software - core.externalInterrupt := io.interrupts(index).external - core.supervisorExternalInterrupt := io.interrupts(index).supervisorExternal - io.admission(index) <> core.admission.get - io.uncachedRequest(index) <> core.uncachedRequest - core.uncachedResponse <> io.uncachedResponse(index) - if (controlIndex >= 0) { - io.control(controlIndex) <> core.chi - io.control(controlCoreIds.size + controlIndex) <> core.instructionChi - } else { - memorySystem.get.io.coherent(localIndex) <> core.chi - memorySystem.get.io.coherent(localCoreIds.size + localIndex) <> core.instructionChi - } - io.retired(index) := core.retired - io.retiredPc(index) := core.retiredPc - io.trapped(index) := core.trapped - io.trapCause(index) := core.trapCause - io.trapValue(index) := core.trapValue - io.trapPc(index) := core.trapPc - } -} diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Consistency.scala b/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Consistency.scala deleted file mode 100644 index 08b91ef24..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Consistency.scala +++ /dev/null @@ -1,91 +0,0 @@ -package hier.tile.memory - -import memcore.memory.queue.Queue - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import hier.core.rocket.{Cache => CoreCache} -import memcore.bus.chi.{DataFlit, DirectedSnoop, RequestFlit} -import memcore.bus.chi.rnf.{BankedChiCache, CacheAccess, CacheAtomic, CacheResult, RnfParams} -import memcore.bus.chi.snf.{LineRequest, LineResponse} -import memcore.memory.coherence.configs.CoherenceParams -import memcore.memory.interlock.{AccessInfo, Acknowledgement, Dispatch, Interlock, Params, Tag} - -/** Physical-access verification composition: actual L1s, one shared L2 and one core-local interlock. */ -@instantiable -class Consistency(memory: CoherenceParams, l1: RnfParams, tracking: Params) extends Module { - require(memory.agents >= 2 && l1.homeCount == 1 && l1.homeId == memory.homeId) - require(l1.chi == memory.chi && tracking.addressBits == memory.chi.addressBits) - val c = memory.chi - - @public - val io = IO(new Bundle { - val access = Flipped(Vec(memory.agents, Decoupled(new CacheAccess(c)))) - val result = Vec(memory.agents, Decoupled(new CacheResult)) - val dispatch = Flipped(Decoupled(new Dispatch(tracking))) - val accessInfo = Flipped(Decoupled(new AccessInfo(tracking))) - val grant = Decoupled(new Tag(tracking)) - val done = Flipped(Decoupled(new Acknowledgement(tracking))) - val complete = Decoupled(new Tag(tracking)) - val olderDispatchPending = Input(Bool()) - val olderRequestsDrained = Input(Bool()) - val blockRequesterRsp = Input(Bool()) - val blockRequesterData = Input(Bool()) - val cpuAllow = Output(Bool()) - val memoryReq = Decoupled(new LineRequest(c)) - val memoryResp = Flipped(Decoupled(new LineResponse(c))) - val observedReq = Output(Valid(new RequestFlit(c))) - val observedSnp = Output(Valid(new DirectedSnoop(c))) - val outstanding = Output(UInt(log2Ceil(memory.mshrEntries + 1).W)) - }) - - val home: Instance[Home] = Instantiate(new Home(memory)) - home.io.active := true.B - home.io.blockRequesterRsp := io.blockRequesterRsp - io.memoryReq <> home.io.memoryReq - home.io.memoryResp <> io.memoryResp - io.observedReq := home.io.observedReq - io.observedSnp := home.io.observedSnp - io.outstanding := home.io.outstanding - - val interlock: Instance[Interlock] = Instantiate(new Interlock(tracking)) - interlock.io.cancel.valid := false.B - interlock.io.cancel.bits := 0.U.asTypeOf(new Dispatch(tracking)) - interlock.io.dispatch <> io.dispatch - interlock.io.accessInfo <> io.accessInfo - io.grant <> interlock.io.grant - interlock.io.done <> io.done - io.complete <> interlock.io.complete - val access = io.access(0).bits - interlock.io.cpuQuery.valid := io.access(0).valid && access.atomic =/= CacheAtomic.Fence.U - interlock.io.cpuQuery.paddr := access.addr - interlock.io.cpuQuery.sizeLog2 := Mux(access.atomicWord, 2.U, 3.U) - interlock.io.cpuQuery.write := access.write || access.atomic === CacheAtomic.SC.U || - (access.atomic >= CacheAtomic.Swap.U && access.atomic <= CacheAtomic.MaxU.U) - interlock.io.cpuQuery.olderDispatchPending := io.olderDispatchPending - io.cpuAllow := interlock.io.cpuAllow - - val first: Instance[CoreCache] = Instantiate(new CoreCache(l1.copy(nodeId = 1), tracking)) - first.io.access.valid := io.access(0).valid && interlock.io.cpuAllow - first.io.access.bits := access - io.access(0).ready := first.io.access.ready && interlock.io.cpuAllow - io.result(0) <> first.io.result - first.io.olderRequestsDrained := io.olderRequestsDrained - first.io.maintenance <> interlock.io.maintenance - interlock.io.maintained <> first.io.maintained - home.io.requesters(0) <> first.io.chi - // Verification-only delayed transport, including a complete old writeback - // accepted by L1 but not yet delivered to Home. - val delayedData = Module(new Queue(new DataFlit(c), c.beatsPerLine)) - delayedData.io.enq <> first.io.chi.txDat - home.io.requesters(0).txDat.valid := delayedData.io.deq.valid && !io.blockRequesterData - home.io.requesters(0).txDat.bits := delayedData.io.deq.bits - delayedData.io.deq.ready := home.io.requesters(0).txDat.ready && !io.blockRequesterData - for (i <- 1 until memory.agents) { - val cache: Instance[BankedChiCache] = Instantiate(new BankedChiCache(l1.copy(nodeId = i + 1))) - cache.io.access <> io.access(i) - io.result(i) <> cache.io.result - home.io.requesters(i) <> cache.io.chi - } -} diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Emit.scala b/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Emit.scala deleted file mode 100644 index d9135dd3a..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Emit.scala +++ /dev/null @@ -1,468 +0,0 @@ -package hier.tile.memory - -import java.nio.file.{Files, Paths} -import scala.sys.process._ -import framework.system.core.rocket.CpuParams -import freechips.rocketchip.rocket.{DCacheParams, ICacheParams, RocketCoreParams} -import memcore.bus.chi.rnf.RnfParams -import memcore.bus.chi._ -import memcore.bus.chi.snf.{LineRequest, LineResponse} -import memcore.memory.cpu.PhysicalRegion -import memcore.memory.cache.configs.CacheParams -import memcore.memory.coherence.configs.CoherenceParams - -object Emit extends App { - val root = Paths.get("src/main/scala/framework/root/root-tile").toAbsolutePath - val output = root.resolve("build") - Files.createDirectories(output) - // Frozen verification profile: two identical 2-bank/8-line RN-Fs, one Home64. - val chi = Params() - val p = CoherenceParams(chi, CacheParams(chi.addressBits, 64, 4, 2, 8, 4, 2), 2, 4, 64) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new CacheSystem(p), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("CacheSystem").toString, "--split-verilog") - ) - val definitions = scala.collection.mutable.ArrayBuffer[String]() - val ports = scala.collection.mutable.ArrayBuffer[String]() - def fields(flit: Flit): Seq[(String, Int)] = - flit.fieldsLSB.map(field => flit.elements.find(_._2 eq field).get._1 -> field.getWidth) - - def layout( - namespace: String, - definitions: scala.collection.mutable.Buffer[String], - kind: String, - fs: Seq[(String, Int)] - ): Unit = { - var offset = 0 - for ((name, width) <- fs if width > 0) { - definitions += s"`define ${namespace}_${kind}_${name.toUpperCase}_OFFSET $offset" - definitions += s"`define ${namespace}_${kind}_${name.toUpperCase}_WIDTH $width" - offset += width - } - definitions += s"`define ${namespace}_${kind}_WIDTH $offset" - } - - def bind( - ports: scala.collection.mutable.Buffer[String], - prefix: String, - signal: String, - fs: Seq[(String, Int)] - ): Unit = { - var offset = 0 - for ((name, width) <- fs if width > 0) { - ports += s".$prefix$name($signal[$offset +: $width])" - offset += width - } - } - - val req = fields(new RequestFlit(chi)) - val rsp = fields(new ResponseFlit(chi)) - val dat = fields(new DataFlit(chi)) - val snp = fields(new SnoopFlit(chi)) - val lineReq = new LineRequest(chi) - val lineResp = new LineResponse(chi) - val memReq = Seq("id", "addr", "write", "data", "mask").map(name => name -> lineReq.elements(name).getWidth) - val memResp = Seq("id", "data", "error").map(name => name -> lineResp.elements(name).getWidth) - for ( - (kind, fs) <- Seq("REQ" -> req, "RSP" -> rsp, "DAT" -> dat, "SNP" -> snp, "MEMREQ" -> memReq, "MEMRESP" -> memResp) - ) { layout("COH", definitions, kind, fs) } - val snpWidth = snp.map(_._2).sum - definitions += s"`define COH_SNP_TARGET_OFFSET $snpWidth" - definitions += s"`define COH_SNP_TARGET_WIDTH ${chi.nodeIdBits}" - definitions += s"`define COH_SNP_CHANNEL_WIDTH ${snpWidth + chi.nodeIdBits}" - for ( - (name, value) <- Seq( - "AGENTS" -> p.agents, - "MSHRS" -> p.mshrEntries, - "HOME" -> p.homeId, - "SETS" -> p.cache.sets, - "WAYS" -> p.cache.ways, - "DATA_BITS" -> chi.dataBits, - "OUTSTANDING_BITS" -> chisel3.util.log2Ceil(p.mshrEntries + 1), - "READ_NSD" -> Opcode.ReadNotSharedDirty, - "READ_UNIQUE" -> Opcode.ReadUnique, - "WRITEBACK" -> Opcode.WriteBackFull, - "COMP" -> Opcode.Comp, - "COMP_DBID" -> Opcode.CompDBIDResp, - "COMP_ACK" -> Opcode.CompAck, - "COMP_DATA" -> Opcode.CompData, - "SNP_RESP" -> Opcode.SnpResp, - "SNP_DATA" -> Opcode.SnpRespData, - "COPYBACK_DATA" -> Opcode.CopyBackWriteData - ) - ) { - definitions += s"`define COH_$name $value" - } - - val observedChannels = Seq( - ("observedReq", "req", req), - ("observedRsp", "rsp", rsp), - ("observedDat", "dat", dat), - ("observedRxRsp", "rx_rsp", rsp), - ("observedRxDat", "rx_dat", dat) - ) - - for (core <- 0 until p.agents) { - for (field <- Seq("addr", "write", "data", "mask", "atomic", "atomicWord")) { - ports += s".io_access_${core}_bits_$field(control.access_$field[$core])" - } - ports += s".io_access_${core}_valid(control.access_valid[$core])" - ports += s".io_access_${core}_ready(control.access_ready[$core])" - ports += s".io_result_${core}_valid(control.result_valid[$core])" - ports += s".io_result_${core}_ready(control.result_ready[$core])" - ports += s".io_result_${core}_bits_data(control.result_data[$core])" - ports += s".io_result_${core}_bits_error(control.result_error[$core])" - } - for ((port, signal, fs) <- observedChannels) { - bind(ports, s"io_${port}_bits_", s"control.${signal}_bits", fs) - ports += s".io_${port}_valid(control.${signal}_valid)" - } - bind(ports, "io_observedSnp_bits_flit_", "control.snp_bits", snp) - ports += s".io_observedSnp_bits_targetNode(control.snp_bits[$snpWidth +: ${chi.nodeIdBits}])" - ports += ".io_observedSnp_valid(control.snp_valid)" - bind(ports, "io_memoryReq_bits_", "mem_req.bits", memReq) - bind(ports, "io_memoryResp_bits_", "mem_resp.bits", memResp) - ports ++= Seq( - ".io_memoryReq_valid(mem_req.valid)", - ".io_memoryReq_ready(mem_req.ready)", - ".io_memoryResp_valid(mem_resp.valid)", - ".io_memoryResp_ready(mem_resp.ready)" - ) - Files.writeString( - output.resolve("cache_system_config.svh"), - "`ifndef CACHE_SYSTEM_CONFIG_SVH\n`define CACHE_SYSTEM_CONFIG_SVH\n" + definitions.mkString("\n") + "\n`endif\n" - ) - Files.writeString(output.resolve("cache_system_ports.svh"), ports.mkString(",\n") + "\n") - - // The short uncached window makes aligned cross-region accesses testable. - val regions = Seq( - PhysicalRegion(BigInt("80000000", 16), 0x1000000, true, true, true, true, true, true), - PhysicalRegion(BigInt("10000000", 16), 0x1000, false, false, true, true, false, false), - PhysicalRegion(BigInt("10002000", 16), 5, false, false, true, true, false, false), - PhysicalRegion(BigInt("90000000", 16), 64, true, false, true, false, false, true), - PhysicalRegion(BigInt("90000040", 16), 64, true, false, false, true, false, true), - PhysicalRegion(BigInt("a0000000", 16), 0x100000, false, true, true, true, true, true) - ) - - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new VirtualCacheSystem(p, regions), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("VirtualCacheSystem").toString, "--split-verilog") - ) - val vd = scala.collection.mutable.ArrayBuffer[String]() - val vp = scala.collection.mutable.ArrayBuffer[String]() - - def virtualFields( - kind: String, - prefix: String, - signal: String, - fs: Seq[(String, Int)] - ): Unit = { - layout("VM", vd, kind, fs) - bind(vp, prefix, signal, fs) - } - - virtualFields( - "REQ", - "io_request_bits_", - "source_if.bits", - Seq( - "vaddr" -> 64, - "tag" -> 6, - "size" -> 3, - "write" -> 1, - "execute" -> 1, - "signed" -> 1, - "data" -> 64, - "atomic" -> 4, - "privilege" -> 2, - "sum" -> 1, - "mxr" -> 1, - "satpMode" -> 4, - "rootPpn" -> 44 - ) - ) - virtualFields( - "AUTH", - "io_authorizationRequest_bits_", - "auth_req.bits", - Seq("paddr" -> 64, "size" -> 3, "read" -> 1, "write" -> 1, "execute" -> 1, "isPte" -> 1, "privilege" -> 2) - ) - vp += ".io_authorizationResponse_bits(auth_resp.bits)" - virtualFields( - "RESP", - "io_response_bits_", - "sink_if.bits", - Seq("tag" -> 6, "data" -> 64, "misaligned" -> 1, "pageFault" -> 1, "accessFault" -> 1) - ) - virtualFields("MEMREQ", "io_memoryReq_bits_", "mem_req.bits", memReq) - virtualFields("MEMRESP", "io_memoryResp_bits_", "mem_resp.bits", memResp) - virtualFields( - "UNCACHED", - "io_uncachedRequest_bits_", - "uncached_req.bits", - Seq("addr" -> 64, "tag" -> 6, "size" -> 3, "write" -> 1, "data" -> 64, "atomic" -> 4, "normal" -> 1) - ) - virtualFields("URESP", "io_uncachedResponse_bits_", "uncached_resp.bits", Seq("tag" -> 6, "data" -> 64, "error" -> 1)) - virtualFields( - "TRANSLATION", - "io_observedTranslation_bits_", - "control.translation_bits", - Seq( - "paddr" -> chi.addressBits, - "pageFault" -> 1, - "accessFault" -> 1, - "level" -> 2 - ) - ) - virtualFields( - "PHYSICAL", - "io_observedPhysical_bits_", - "control.physical_bits", - Seq( - "addr" -> 64, - "tag" -> 6, - "size" -> 3, - "write" -> 1, - "signed" -> 1, - "data" -> 64, - "atomic" -> 4, - "cacheable" -> 1, - "normal" -> 1 - ) - ) - virtualFields( - "CACHE", - "io_observedCache_bits_access_", - "control.cache_bits", - Seq("addr" -> chi.addressBits, "write" -> 1, "data" -> 64, "mask" -> 8, "atomic" -> 4, "atomicWord" -> 1) - ) - vp ++= Seq( - ".io_observedTranslation_valid(control.translation_valid)", - ".io_observedPhysical_valid(control.physical_valid)", - ".io_observedCache_valid(control.cache_valid)", - ".io_observedCache_bits_pte(control.cache_pte)", - ".io_observedEviction_valid(control.eviction_valid)", - ".io_observedEviction_bits(control.eviction_addr)" - ) - for ( - (port, signal) <- Seq( - "request" -> "source_if", - "response" -> "sink_if", - "memoryReq" -> "mem_req", - "memoryResp" -> "mem_resp", - "authorizationRequest" -> "auth_req", - "authorizationResponse" -> "auth_resp", - "uncachedRequest" -> "uncached_req", - "uncachedResponse" -> "uncached_resp" - ) - ) { - vp += s".io_${port}_valid($signal.valid)" - vp += s".io_${port}_ready($signal.ready)" - } - for ( - (name, region) <- Seq( - "DDR" -> regions(0), - "MMIO" -> regions(1), - "SHORT_MMIO" -> regions(2), - "READ_ONLY" -> regions(3), - "WRITE_ONLY" -> regions(4), - "NORMAL_RAM" -> regions(5) - ) - ) { - vd += s"`define VM_${name}_BASE 64'h${region.base.toString(16)}" - vd += s"`define VM_${name}_BYTES 64'h${region.bytes.toString(16)}" - } - vd += s"`define VM_OUTSTANDING_BITS ${chisel3.util.log2Ceil(p.mshrEntries + 1)}" - Files.writeString( - output.resolve("virtual_cache_config.svh"), - "`ifndef VIRTUAL_CACHE_CONFIG_SVH\n`define VIRTUAL_CACHE_CONFIG_SVH\n" + vd.mkString("\n") + "\n`endif\n" - ) - Files.writeString(output.resolve("virtual_cache_ports.svh"), vp.mkString(",\n") + "\n") - - implicit val cpu: CpuParams = CpuParams( - core = new RocketCoreParams( - useVM = false, - useUser = false, - useSupervisor = false, - useHypervisor = false, - useDebug = false, - fpu = None, - nPMPs = 4, - clockGate = false, - haveCFlush = false, - haveCease = false, - haveSimTimeout = false - ) { override val pmpGranularity = 8 }, - dcache = Some(DCacheParams(nSets = 64, nWays = 2)), - icache = Some(ICacheParams()), - btb = None, - physicalAddressBits = 44, - hartIdBits = 1, - beatBytes = 8, - blockBytes = 64 - ) - - val builder = Paths.get("src/main/scala/framework/root/root-core/src/main/resources/build_fixture.py").toAbsolutePath - require( - Process(Seq("python", builder.toString, output.toString, "multicore")).! == 0, - "Two-Core fixture generation failed" - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Tile(p.copy(agents = 2 * p.agents), RnfParams(chi, cacheLines = 8, banks = 2), regions), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("Tile").toString, "--split-verilog") - ) - - val td = scala.collection.mutable.ArrayBuffer[String]( - s"""`define TILE_IMAGE "${output.resolve("multicore.hex")}"""" - ) - - val tp = scala.collection.mutable.ArrayBuffer[String]() - - bind(tp, "io_memoryReq_bits_", "mem_req.bits", memReq) - bind(tp, "io_memoryResp_bits_", "mem_resp.bits", memResp) - tp ++= Seq( - ".io_memoryReq_valid(mem_req.valid)", - ".io_memoryReq_ready(mem_req.ready)", - ".io_memoryResp_valid(mem_resp.valid)", - ".io_memoryResp_ready(mem_resp.ready)" - ) - val uncached = Seq("addr" -> 64, "tag" -> 6, "size" -> 3, "write" -> 1, "data" -> 64) - val uncachedResp = Seq("tag" -> 6, "data" -> 64, "error" -> 1) - for ((kind, fs) <- Seq("MEMREQ" -> memReq, "MEMRESP" -> memResp, "UNCACHED" -> uncached, "URESP" -> uncachedResp)) { - layout("TILE", td, kind, fs) - } - for (i <- 0 until p.agents) { - bind(tp, s"io_uncachedRequest_${i}_bits_", s"uncached_req_$i.bits", uncached) - bind(tp, s"io_uncachedResponse_${i}_bits_", s"uncached_resp_$i.bits", uncachedResp) - tp ++= Seq( - s".io_uncachedRequest_${i}_valid(uncached_req_$i.valid)", - s".io_uncachedRequest_${i}_ready(uncached_req_$i.ready)", - s".io_uncachedResponse_${i}_valid(uncached_resp_$i.valid)", - s".io_uncachedResponse_${i}_ready(uncached_resp_$i.ready)" - ) - for (field <- Seq("retired", "retiredPc", "trapped", "trapCause", "trapValue", "trapPc")) - tp += s".io_${field}_$i(control.$field[$i])" - } - for ((port, signal, fs) <- observedChannels) { - bind(tp, s"io_${port}_bits_", s"control.${signal}_bits", fs) - tp += s".io_${port}_valid(control.${signal}_valid)" - } - bind(tp, "io_observedSnp_bits_flit_", "control.snp_bits", snp) - tp += s".io_observedSnp_bits_targetNode(control.snp_bits[$snpWidth +: ${chi.nodeIdBits}])" - tp += ".io_observedSnp_valid(control.snp_valid)" - Files.writeString(output.resolve("tile_config.svh"), td.mkString("\n") + "\n") - Files.writeString(output.resolve("tile_ports.svh"), tp.mkString(",\n") + "\n") - - val tracking = memcore.memory.interlock.Params() - val consistencyMemory = p.copy(cache = p.cache.copy(sets = 16)) - val consistencyL1 = RnfParams(chi, cacheLines = 8, banks = 2) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Consistency(consistencyMemory, consistencyL1, tracking), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("Consistency").toString, "--split-verilog") - ) - - val cd = scala.collection.mutable.ArrayBuffer[String]( - s"`define CONS_AGENTS ${p.agents}", - s"`define CONS_HOME ${p.homeId}", - s"`define CONS_MSHRS ${p.mshrEntries}", - s"`define CONS_OUTSTANDING_BITS ${chisel3.util.log2Ceil(p.mshrEntries + 1)}", - s"`define CONS_PA_BITS ${tracking.addressBits}", - s"`define CONS_LINE_BYTES ${tracking.lineBytes}", - s"`define CONS_L1_LINES ${consistencyL1.cacheLines}", - s"`define CONS_L2_SETS ${consistencyMemory.cache.sets}", - s"`define CONS_SLOTS ${tracking.entries}" - ) - - val cp = scala.collection.mutable.ArrayBuffer[String]() - def members(bundle: chisel3.Bundle): Seq[(String, Int)] = - bundle.elements.toSeq.collect { case (name, field) if field.getWidth > 0 => name -> field.getWidth } - val accessFields = members(new memcore.bus.chi.rnf.CacheAccess(chi)) - val resultFields = members(new memcore.bus.chi.rnf.CacheResult) - layout("CONS", cd, "CPU", accessFields) - layout("CONS", cd, "RESULT", resultFields) - - def connect(port: String, signal: String, fs: Seq[(String, Int)]): Unit = { - bind(cp, s"io_${port}_bits_", s"$signal.bits", fs) - cp += s".io_${port}_valid($signal.valid)" - cp += s".io_${port}_ready($signal.ready)" - } - - for (i <- 0 until p.agents) { - connect(s"access_$i", s"access_$i", accessFields) - connect(s"result_$i", s"result_$i", resultFields) - } - for ( - (kind, port, signal, fs) <- Seq( - ( - "DISPATCH", - "dispatch", - "dispatch", - members(new memcore.memory.interlock.Dispatch(tracking)) - ), - ( - "INFO", - "accessInfo", - "access_info", - members(new memcore.memory.interlock.AccessInfo(tracking)) - ), - ("GRANT", "grant", "grant", members(new memcore.memory.interlock.Tag(tracking))), - ("DONE", "done", "done", members(new memcore.memory.interlock.Acknowledgement(tracking))), - ("COMPLETE", "complete", "complete", members(new memcore.memory.interlock.Tag(tracking))), - ("MEMREQ", "memoryReq", "mem_req", memReq), - ("MEMRESP", "memoryResp", "mem_resp", memResp) - ) - ) { - layout("CONS", cd, kind, fs) - connect(port, signal, fs) - } - layout("CONS", cd, "REQ", req) - layout("CONS", cd, "SNP", snp) - cd += s"`define CONS_SNP_TARGET_OFFSET $snpWidth" - cd += s"`define CONS_SNP_TARGET_WIDTH ${chi.nodeIdBits}" - cd += s"`define CONS_SNP_CHANNEL_WIDTH ${snpWidth + chi.nodeIdBits}" - bind(cp, "io_observedReq_bits_", "control.req_bits", req) - bind(cp, "io_observedSnp_bits_flit_", "control.snp_bits", snp) - cp ++= Seq( - ".io_observedReq_valid(control.req_valid)", - ".io_observedSnp_valid(control.snp_valid)", - s".io_observedSnp_bits_targetNode(control.snp_bits[$snpWidth +: ${chi.nodeIdBits}])", - ".io_olderDispatchPending(control.older_dispatch_pending)", - ".io_olderRequestsDrained(control.older_requests_drained)", - ".io_blockRequesterRsp(control.block_requester_rsp)", - ".io_blockRequesterData(control.block_requester_data)", - ".io_cpuAllow(control.cpu_allow)", - ".io_outstanding(control.outstanding)" - ) - Files.writeString(output.resolve("consistency_config.svh"), cd.mkString("\n") + "\n") - Files.writeString(output.resolve("consistency_ports.svh"), cp.mkString(",\n") + "\n") - // Five-core Goban shape elaboration only: explicit placement and opaque test signatures. - // This does not load a PB profile; the production Tile is assembled in framework.system.tile. - val compositionCpu = cpu.copy(hartIdBits = 3) - val compositionMemory = p.copy(agents = 10) - - val compositionPlacements = Seq( - CorePlacement(RnfParams(chi, nodeId = 1, cacheLines = 8, banks = 2), compositionCpu, CoreRole.Controller), - CorePlacement(RnfParams(chi, nodeId = 2, cacheLines = 8, banks = 2), compositionCpu, CoreRole.Compute), - CorePlacement(RnfParams(chi, nodeId = 3, cacheLines = 8, banks = 2), compositionCpu, CoreRole.Compute), - CorePlacement(RnfParams(chi, nodeId = 4, cacheLines = 8, banks = 2), compositionCpu, CoreRole.Compute), - CorePlacement(RnfParams(chi, nodeId = 5, cacheLines = 8, banks = 2), compositionCpu, CoreRole.Compute) - ) - - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Composition( - compositionMemory, - compositionPlacements, - regions, - memcore.memory.interlock.Params(addressBits = chi.addressBits), - workerCoreIds = Seq(1, 2, 3, 4), - signatures = Seq(BigInt(1), BigInt(2), BigInt(3), BigInt(4)) - ), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("Composition").toString, "--split-verilog") - ) - -} diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Home.scala b/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Home.scala deleted file mode 100644 index 9b245bc1c..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Home.scala +++ /dev/null @@ -1,52 +0,0 @@ -package hier.tile.memory - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import memcore.bus.chi._ -import memcore.bus.chi.snf.{LineRequest, LineResponse} -import memcore.memory.coherence.Coherence -import memcore.memory.coherence.configs.CoherenceParams - -/** One Tile-owned shared L2/Home and its explicit requester fabric. */ -@instantiable -class Home(p: CoherenceParams) extends Module { - val c = p.chi - - @public - val io = IO(new Bundle { - val active = Input(Bool()) - val blockRequesterRsp = Input(Bool()) - val requesters = Vec(p.agents, Flipped(new RequesterPort(c))) - val memoryReq = Decoupled(new LineRequest(c)) - val memoryResp = Flipped(Decoupled(new LineResponse(c))) - val outstanding = Output(UInt(log2Ceil(p.mshrEntries + 1).W)) - val observedReq = Output(Valid(new RequestFlit(c))) - val observedRsp = Output(Valid(new ResponseFlit(c))) - val observedDat = Output(Valid(new DataFlit(c))) - val observedSnp = Output(Valid(new DirectedSnoop(c))) - val observedRxRsp = Output(Valid(new ResponseFlit(c))) - val observedRxDat = Output(Valid(new DataFlit(c))) - }) - - val home: Instance[Coherence] = Instantiate(new Coherence(p)) - val fabric: Instance[Fabric] = Instantiate(new Fabric(c, p.agents, p.homeId)) - fabric.io.active := io.active - fabric.io.blockRequesterRsp := io.blockRequesterRsp - fabric.io.requesters <> io.requesters - home.io.req <> fabric.io.req - home.io.rxRsp <> fabric.io.rxRsp - home.io.rxDat <> fabric.io.rxDat - fabric.io.rsp <> home.io.rsp - fabric.io.dat <> home.io.dat - fabric.io.snp <> home.io.snp - io.memoryReq <> home.io.memoryReq - home.io.memoryResp <> io.memoryResp - io.outstanding := home.io.outstanding - io.observedReq.valid := home.io.req.fire; io.observedReq.bits := home.io.req.bits - io.observedRsp.valid := home.io.rsp.fire; io.observedRsp.bits := home.io.rsp.bits - io.observedDat.valid := home.io.dat.fire; io.observedDat.bits := home.io.dat.bits - io.observedSnp.valid := home.io.snp.fire; io.observedSnp.bits := home.io.snp.bits - io.observedRxRsp.valid := home.io.rxRsp.fire; io.observedRxRsp.bits := home.io.rxRsp.bits - io.observedRxDat.valid := home.io.rxDat.fire; io.observedRxDat.bits := home.io.rxDat.bits -} diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Memory.scala b/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Memory.scala deleted file mode 100644 index 6119e39bc..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Memory.scala +++ /dev/null @@ -1,30 +0,0 @@ -package hier.tile.memory - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import memcore.bus.chi.RequesterPort -import memcore.bus.chi.snf.{LineRequest, LineResponse} -import memcore.memory.coherence.configs.CoherenceParams - -/** CPU coherent Home and shared L2 with one backing-memory client. */ -@instantiable -class Memory(p: CoherenceParams) extends Module { - val c = p.chi - - @public - val io = IO(new Bundle { - val coherent = Vec(p.agents, Flipped(new RequesterPort(c))) - val backingReq = Vec(1, Decoupled(new LineRequest(c))) - val backingResp = Vec(1, Flipped(Decoupled(new LineResponse(c)))) - val homeOutstanding = Output(UInt(log2Ceil(p.mshrEntries + 1).W)) - }) - - val home: Instance[Home] = Instantiate(new Home(p)) - home.io.active := true.B - home.io.blockRequesterRsp := false.B - home.io.requesters <> io.coherent - io.backingReq(0) <> home.io.memoryReq - home.io.memoryResp <> io.backingResp(0) - io.homeOutstanding := home.io.outstanding -} diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Tile.scala b/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Tile.scala deleted file mode 100644 index 2aafa747a..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Tile.scala +++ /dev/null @@ -1,82 +0,0 @@ -package hier.tile.memory - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.system.core.rocket.{CpuParams, HasCpuParameters} -import hier.core.rocket.Core -import memcore.bus.chi._ -import memcore.bus.chi.rnf.RnfParams -import memcore.bus.chi.snf.{LineRequest, LineResponse} -import memcore.memory.cpu.{CpuMemParams, PhysicalRegion, UncachedRequest, UncachedResponse} -import memcore.memory.coherence.configs.CoherenceParams - -@instantiable -class Tile(memory: CoherenceParams, l1: RnfParams, regions: Seq[PhysicalRegion])(implicit val cpuParams: CpuParams) - extends Module - with HasCpuParameters { - require(l1.chi == memory.chi && l1.homeId == memory.homeId && l1.homeCount == 1) - val c = memory.chi - val cp = CpuMemParams(c, tagBits = 6) - - val cores = memory.agents / 2 - require(memory.agents == 2 * cores) - - @public - val io = IO(new Bundle { - val resetVector = Input(UInt(64.W)) - val time = Input(UInt(64.W)) - val memoryReq = Decoupled(new LineRequest(c)) - val memoryResp = Flipped(Decoupled(new LineResponse(c))) - val uncachedRequest = Vec(cores, Decoupled(new UncachedRequest(cp))) - val uncachedResponse = Vec(cores, Flipped(Decoupled(new UncachedResponse(cp)))) - val retired = Output(Vec(cores, Bool())) - val retiredPc = Output(Vec(cores, UInt(64.W))) - val trapped = Output(Vec(cores, Bool())) - val trapCause = Output(Vec(cores, UInt(64.W))) - val trapValue = Output(Vec(cores, UInt(64.W))) - val trapPc = Output(Vec(cores, UInt(64.W))) - val outstanding = Output(UInt(log2Ceil(memory.mshrEntries + 1).W)) - val observedReq = Output(Valid(new RequestFlit(c))) - val observedRsp = Output(Valid(new ResponseFlit(c))) - val observedDat = Output(Valid(new DataFlit(c))) - val observedSnp = Output(Valid(new DirectedSnoop(c))) - val observedRxRsp = Output(Valid(new ResponseFlit(c))) - val observedRxDat = Output(Valid(new DataFlit(c))) - }) - - val home: Instance[Home] = Instantiate(new Home(memory)) - home.io.active := true.B - home.io.blockRequesterRsp := false.B - io.memoryReq <> home.io.memoryReq - home.io.memoryResp <> io.memoryResp - io.outstanding := home.io.outstanding - io.observedReq := home.io.observedReq - io.observedRsp := home.io.observedRsp - io.observedDat := home.io.observedDat - io.observedSnp := home.io.observedSnp - io.observedRxRsp := home.io.observedRxRsp - io.observedRxDat := home.io.observedRxDat - // Requester i is core i's data L1; requester cores + i is its instruction L1. - val core: Seq[Instance[Core]] = - Seq.tabulate(cores)(i => Instantiate(new Core(l1.copy(nodeId = i + 1), l1.copy(nodeId = cores + i + 1), regions))) - for (i <- 0 until cores) { - core(i).clink.timerInterrupt := false.B - core(i).clink.softwareInterrupt := false.B - core(i).clink.externalInterrupt := false.B - core(i).clink.supervisorExternalInterrupt := false.B - core(i).clink.hartId := i.U - core(i).clink.resetVector := io.resetVector - core(i).clink.time := io.time - home.io.requesters(i) <> core(i).clink.chi - home.io.requesters(cores + i) <> core(i).clink.instructionChi - io.uncachedRequest(i) <> core(i).clink.uncachedRequest - core(i).clink.uncachedResponse <> io.uncachedResponse(i) - io.retired(i) := core(i).clink.retired - io.retiredPc(i) := core(i).clink.retiredPc - io.trapped(i) := core(i).clink.trapped - io.trapCause(i) := core(i).clink.trapCause - io.trapValue(i) := core(i).clink.trapValue - io.trapPc(i) := core(i).clink.trapPc - } -} diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/VirtualCacheSystem.scala b/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/VirtualCacheSystem.scala deleted file mode 100644 index 2a07408a1..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/VirtualCacheSystem.scala +++ /dev/null @@ -1,78 +0,0 @@ -package hier.tile.memory - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import memcore.bus.chi.Opcode -import memcore.bus.chi.rnf.CacheAccess -import memcore.bus.chi.snf.{LineRequest, LineResponse} -import memcore.memory.cpu.{ - CpuMemParams, - CpuMemoryRequest, - PhysicalAuthorization, - PhysicalRegion, - UncachedRequest, - UncachedResponse, - VirtualMemory, - VirtualMemoryRequest, - VirtualMemoryResponse -} -import memcore.memory.coherence.configs.CoherenceParams - -/** Independent verification composition: one virtual client and the two-RN-F shared-L2 cache system. */ -@instantiable -class VirtualCacheSystem(p: CoherenceParams, regions: Seq[PhysicalRegion]) extends Module { - val cp = CpuMemParams(p.chi, tagBits = 6) - - @public - val io = IO(new Bundle { - val active = Input(Bool()) - val request = Flipped(Decoupled(new VirtualMemoryRequest(cp))) - val response = Decoupled(new VirtualMemoryResponse(cp)) - val authorizationRequest = Decoupled(new PhysicalAuthorization) - val authorizationResponse = Flipped(Decoupled(Bool())) - val memoryReq = Decoupled(new LineRequest(p.chi)) - val memoryResp = Flipped(Decoupled(new LineResponse(p.chi))) - val uncachedRequest = Decoupled(new UncachedRequest(cp)) - val uncachedResponse = Flipped(Decoupled(new UncachedResponse(cp))) - val observedTranslation = Output(Valid(new memcore.memory.mmu.Response(p.chi))) - val observedPhysical = Output(Valid(new CpuMemoryRequest(cp))) - - val observedCache = Output(Valid(new Bundle { - val pte = Bool() - val access = new CacheAccess(p.chi) - })) - - val observedEviction = Output(Valid(UInt(p.chi.addressBits.W))) - val outstanding = Output(UInt(log2Ceil(p.mshrEntries + 1).W)) - }) - - val virtualMemory: Instance[VirtualMemory] = Instantiate(new VirtualMemory(cp, regions)) - val caches: Instance[CacheSystem] = Instantiate(new CacheSystem(p)) - virtualMemory.io.active := io.active - virtualMemory.io.flushTranslation := false.B - virtualMemory.io.lookup.config := 0.U.asTypeOf(virtualMemory.io.lookup.config) - virtualMemory.io.lookup.req := 0.U.asTypeOf(virtualMemory.io.lookup.req) - virtualMemory.io.request <> io.request - io.response <> virtualMemory.io.response - io.authorizationRequest <> virtualMemory.io.authorizationRequest - virtualMemory.io.authorizationResponse <> io.authorizationResponse - io.uncachedRequest <> virtualMemory.io.uncachedRequest - virtualMemory.io.uncachedResponse <> io.uncachedResponse - caches.io.active := io.active - caches.io.blockRequesterRsp := false.B - caches.io.access(0) <> virtualMemory.io.cacheRequest - virtualMemory.io.cacheResponse <> caches.io.result(0) - caches.io.access(1).valid := false.B - caches.io.access(1).bits := 0.U.asTypeOf(caches.io.access(1).bits) - caches.io.result(1).ready := false.B - when(caches.io.result(1).valid)(assert(false.B, "Virtual LSU inactive requester returned data")) - io.memoryReq <> caches.io.memoryReq - caches.io.memoryResp <> io.memoryResp - io.outstanding := caches.io.outstanding - io.observedTranslation := virtualMemory.io.observedTranslation - io.observedPhysical := virtualMemory.io.observedPhysical - io.observedCache := virtualMemory.io.observedCache - io.observedEviction.valid := caches.io.observedReq.valid && caches.io.observedReq.bits.opcode === Opcode.Evict.U - io.observedEviction.bits := caches.io.observedReq.bits.addr -} diff --git a/arch/src/main/scala/framework/rvv/README.md b/arch/src/main/scala/framework/rvv/README.md deleted file mode 100644 index 3ea42ae69..000000000 --- a/arch/src/main/scala/framework/rvv/README.md +++ /dev/null @@ -1 +0,0 @@ -# rvv diff --git a/arch/src/main/scala/framework/rvv/build.sc b/arch/src/main/scala/framework/rvv/build.sc deleted file mode 100644 index 9645d6c4e..000000000 --- a/arch/src/main/scala/framework/rvv/build.sc +++ /dev/null @@ -1,64 +0,0 @@ -import mill._ -import mill.scalalib._ - -val archRoot = os.pwd / os.up / os.up / os.up / os.up / os.up - -object cde extends SbtModule { - override def millSourcePath = archRoot / "thirdparty" / "rocket-chip" / "dependencies" / "cde" - override def scalaVersion = "2.13.16" - - override def sources = T.sources { - super.sources() ++ Seq(PathRef(millSourcePath / "cde" / "src" / "chipsalliance")) - } - -} - -object hardfloat extends SbtModule { - override def millSourcePath = archRoot / "thirdparty" / "berkeley-hardfloat" - override def scalaVersion = "2.13.16" - override def moduleDeps = Seq(cde) - - override def sources = T.sources { - super.sources() ++ Seq(PathRef(millSourcePath / "hardfloat" / "src" / "main" / "scala")) - } - - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") -} - -object rvv extends SbtModule { - override def millSourcePath = os.pwd - override def scalaVersion = "2.13.16" - override def moduleDeps = Seq(hardfloat) - - override def sources = T.sources { - super.sources() ++ Seq( - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "arith" / "src" / "main" / "scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "top" / "GlobalConfig.scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "top" / "configs" / "SimParam.scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "memdomain" / "configs" / "MemDomainParam.scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "frontend" / "configs" / "FrontendParam.scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "balldomain" / "configs" / "BallDomainParam.scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "system" / "tile" / "configs" / "TileParam.scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "balldomain" / "blink" / "blink.scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "balldomain" / "blink" / "bank.scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "balldomain" / "blink" / "status.scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "balldomain" / "blink" / "baseball.scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "balldomain" / "blink" / "SubRobRow.scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "balldomain" / "blink" / "mmio" / "MmioRead.scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "balldomain" / "blink" / "mmio" / "MmioWrite.scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "memdomain" / "backend" / "banks" / "SramIO.scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "memdomain" / "backend" / "mmio" / "MmioIO.scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "balldomain" / "rs" / "interfaces.scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "balldomain" / "decoder" / "BallDecodeCmd.scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "frontend" / "decoder" / "PostGDCmd.scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "frontend" / "scoreboard" / "BankAccessInfo.scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "system" / "core" / "rocket" / "RoCCCommandBB.scala") - ) - } - - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0", ivy"com.lihaoyi::upickle:3.3.1") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") - override def mainClass = Some("framework.rvv.Emit") -} diff --git a/arch/src/main/scala/framework/rvv/src/csrc/Cargo.lock b/arch/src/main/scala/framework/rvv/src/csrc/Cargo.lock deleted file mode 100644 index c99fadbc9..000000000 --- a/arch/src/main/scala/framework/rvv/src/csrc/Cargo.lock +++ /dev/null @@ -1,135 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "bitflags" -version = "2.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" - -[[package]] -name = "itoa" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" - -[[package]] -name = "memchr" -version = "2.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" - -[[package]] -name = "proc-macro2" -version = "1.0.107" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quote" -version = "1.0.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "rustc_apfloat" -version = "0.2.3+llvm-462a31f5a5ab" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "486c2179b4796f65bfe2ee33679acf0927ac83ecf583ad6c91c3b4570911b9ad" -dependencies = [ - "bitflags", - "smallvec", -] - -[[package]] -name = "rvv" -version = "0.1.0" -dependencies = [ - "rustc_apfloat", -] - -[[package]] -name = "rvv-reference-model" -version = "0.1.0" -dependencies = [ - "rvv", - "serde_json", -] - -[[package]] -name = "serde" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" -dependencies = [ - "serde_core", -] - -[[package]] -name = "serde_core" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "serde_json" -version = "1.0.151" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" -dependencies = [ - "itoa", - "memchr", - "serde", - "serde_core", - "zmij", -] - -[[package]] -name = "smallvec" -version = "1.16.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e" - -[[package]] -name = "syn" -version = "3.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "unicode-ident" -version = "1.0.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" - -[[package]] -name = "zmij" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/arch/src/main/scala/framework/rvv/src/csrc/Cargo.toml b/arch/src/main/scala/framework/rvv/src/csrc/Cargo.toml deleted file mode 100644 index 7162fbd21..000000000 --- a/arch/src/main/scala/framework/rvv/src/csrc/Cargo.toml +++ /dev/null @@ -1,15 +0,0 @@ -[package] -name = "rvv-reference-model" -version = "0.1.0" -edition = "2021" - -[lib] -crate-type = ["cdylib", "staticlib"] - -[dependencies] -rvv = { path = "../../../../../../../../bebop/src/nodes/bemu/rvv" } - -[build-dependencies] -serde_json = "1" - -[workspace] diff --git a/arch/src/main/scala/framework/rvv/src/csrc/README.md b/arch/src/main/scala/framework/rvv/src/csrc/README.md deleted file mode 100644 index e69de29bb..000000000 diff --git a/arch/src/main/scala/framework/rvv/src/csrc/build.rs b/arch/src/main/scala/framework/rvv/src/csrc/build.rs deleted file mode 100644 index 80727754a..000000000 --- a/arch/src/main/scala/framework/rvv/src/csrc/build.rs +++ /dev/null @@ -1,62 +0,0 @@ -use std::{env, fs, path::PathBuf, process::Command}; -fn main() { - let root = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()) - .ancestors() - .nth(8) - .unwrap() - .to_path_buf(); - let ip = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()).parent().unwrap().parent().unwrap().to_path_buf(); - let parameters = ip.join("src/main/resources/rvv.json"); - println!("cargo:rerun-if-changed={}", parameters.display()); - let config: serde_json::Value = serde_json::from_str(&fs::read_to_string(parameters).unwrap()).unwrap(); - let value = |key: &str| config[key].as_u64().unwrap() as usize; - let constants = format!("const VLEN: usize = {};\nconst ELEN: usize = {};\nconst IBUF_BYTES: usize = {};\n", - value("vLen"), value("eLen"), value("iBufWords") * 4); - fs::write(PathBuf::from(env::var_os("OUT_DIR").unwrap()).join("params.rs"), constants).unwrap(); - let rtl = ip.join("build/KernelEngine"); - fs::create_dir_all(&rtl).unwrap(); - fs::write(rtl.join("rvv_config.svh"), format!("`ifndef RVV_MEMORY_PORTS\n`define RVV_MEMORY_PORTS {}\n`endif\n", value("memoryPorts"))).unwrap(); - let mut connections = String::new(); - for port in 0..value("memoryPorts") { - for (direction, fields) in [ - ("Read", vec![("bank_id", "read_bank"), ("group_id", "read_group"), ("io_req_valid", "read_valid"), ("io_req_ready", "read_ready"), ("io_req_bits_addr", "read_row"), ("io_resp_valid", "read_response_valid"), ("io_resp_ready", "read_response_ready"), ("io_resp_bits_data", "read_data")]), - ("Write", vec![("bank_id", "write_bank"), ("group_id", "write_group"), ("io_req_valid", "write_valid"), ("io_req_ready", "write_ready"), ("io_req_bits_addr", "write_row"), ("io_resp_valid", "write_response_valid"), ("io_resp_ready", "write_response_ready"), ("io_req_bits_data", "write_data"), ("io_resp_bits_ok", "write_ok")]), - ] { - for (field, signal) in fields { - connections += &format!(".io_bank{direction}_{port}_{field}(vif.{signal}[{port}]),\n"); - } - } - for bit in 0..16 { - connections += &format!(".io_bankWrite_{port}_io_req_bits_mask_{bit}(vif.write_mask[{port}][{bit}]),\n"); - } - } - connections.truncate(connections.trim_end().len() - 1); - fs::write(rtl.join("rvv_ports.svh"), connections).unwrap(); - let source = root.join("bb-tests/workloads/lib/bbsw/kernels/rvv"); - for name in [ - "build_images.py", - "kernel.ld", - "kernels.h", - "silu.cpp", - "snake.cpp", - "quant.cpp", - "math/exp.cpp", - "math/sin.cpp", - "math/cos.cpp", - "math/erf.cpp", - "rope.cpp", - "trig.cpp", - "math/math.h", - "math/constants.h", - ] { - println!("cargo:rerun-if-changed={}", source.join(name).display()); - } - let output = PathBuf::from(env::var_os("OUT_DIR").unwrap()).join("kernels"); - assert!(Command::new("python3") - .arg(source.join("build_images.py")) - .arg("--output") - .arg(output) - .status() - .unwrap() - .success()); -} diff --git a/arch/src/main/scala/framework/rvv/src/csrc/src/lib.rs b/arch/src/main/scala/framework/rvv/src/csrc/src/lib.rs deleted file mode 100644 index 9cce67869..000000000 --- a/arch/src/main/scala/framework/rvv/src/csrc/src/lib.rs +++ /dev/null @@ -1,2 +0,0 @@ -mod ffi; -mod model; diff --git a/arch/src/main/scala/framework/rvv/src/csrc/src/model.rs b/arch/src/main/scala/framework/rvv/src/csrc/src/model.rs deleted file mode 100644 index fcc242e65..000000000 --- a/arch/src/main/scala/framework/rvv/src/csrc/src/model.rs +++ /dev/null @@ -1,903 +0,0 @@ -include!(concat!(env!("OUT_DIR"), "/params.rs")); - -use rvv::{Engine, Fault, Memory, MemoryError}; -use std::path::Path; - -pub const BANK_BYTES: usize = 4096; -pub const BANKS: usize = 6; -#[derive(Clone)] -pub struct Banks(pub Vec); -impl Banks { - fn range(&self, address: u64, bytes: usize) -> Result, MemoryError> { - let bank = (address >> 32) as usize; - let offset = (address & 0xffff_ffff) as usize; - if bank >= BANKS || offset + bytes > BANK_BYTES { - return Err(MemoryError); - } - Ok(bank * BANK_BYTES + offset..bank * BANK_BYTES + offset + bytes) - } - fn put(&mut self, bank: usize, offset: usize, data: &[u8]) { - self.0[bank * BANK_BYTES + offset..bank * BANK_BYTES + offset + data.len()] - .copy_from_slice(data); - } -} -impl Memory for Banks { - fn ball_command(&mut self, _: u32, _: u64, _: u64) -> Result { - Err(MemoryError) - } - fn read(&mut self, address: u64, bytes: usize) -> Result { - let mut value = [0u8; 8]; - value[..bytes].copy_from_slice(&self.0[self.range(address, bytes)?]); - Ok(u64::from_le_bytes(value)) - } - fn write(&mut self, address: u64, bytes: usize, value: u64) -> Result<(), MemoryError> { - let range = self.range(address, bytes)?; - self.0[range].copy_from_slice(&value.to_le_bytes()[..bytes]); - Ok(()) - } -} -pub struct Case { - pub name: String, - pub program: Vec, - pub constants: Vec, - pub entry: u32, - pub args: [u64; 8], - pub initial: Banks, - pub expected_cause: u32, - pub fixed_expected: Option, -} -pub struct Model { - pub cases: Vec, - pub current: usize, - pub actual: Banks, - pub expected: Banks, - pub engine: Engine, - pub fault: Option, -} -fn i(op: u32, rd: u32, f: u32, rs: u32, imm: i32) -> u32 { - ((imm as u32 & 4095) << 20) | rs << 15 | f << 12 | rd << 7 | op -} -fn s(op: u32, f: u32, rs1: u32, rs2: u32, imm: u32) -> u32 { - (imm >> 5) << 25 | rs2 << 20 | rs1 << 15 | f << 12 | (imm & 31) << 7 | op -} -fn r(op: u32, rd: u32, f: u32, rs1: u32, rs2: u32, f7: u32) -> u32 { - f7 << 25 | rs2 << 20 | rs1 << 15 | f << 12 | rd << 7 | op -} -fn initial() -> Banks { - let mut b = Banks(vec![0; BANKS * BANK_BYTES]); - for offset in 0..BANK_BYTES { - b.0[BANK_BYTES + offset] = (offset as u8).wrapping_mul(73).wrapping_add(19); - } - for n in 0..256 { - let x = (n as f32 - 123.0) / 31.0; - b.put(2, n * 4, &x.to_le_bytes()); - b.put(3, n * 4, &(0.25 + n as f32 / 128.0).to_le_bytes()); - } - b -} -fn micro(name: &str, words: &[u32], args: [u64; 8]) -> Case { - Case { - name: name.into(), - program: words.iter().flat_map(|w| w.to_le_bytes()).collect(), - constants: vec![0; 512], - entry: 0, - args, - initial: initial(), - expected_cause: 0, - fixed_expected: None, - } -} -fn elf(path: &Path, name: &str, args: [u64; 8]) -> Case { - let bytes = std::fs::read(path).unwrap(); - assert_eq!(&bytes[..7], b"\x7fELF\x02\x01\x01"); - let u64_at = |p| u64::from_le_bytes(bytes[p..p + 8].try_into().unwrap()); - let u32_at = |p| u32::from_le_bytes(bytes[p..p + 4].try_into().unwrap()); - let u16_at = |p| u16::from_le_bytes(bytes[p..p + 2].try_into().unwrap()); - assert_eq!(u16_at(18), 243); - let mut c = micro(name, &[], args); - c.entry = u32::try_from(u64_at(24)).expect("kernel entry exceeds local address width"); - assert_eq!(u16_at(58), 64, "invalid ELF64 section header size"); - let sections = usize::try_from(u64_at(40)).unwrap(); - for n in 0..u16_at(60) as usize { - let p = sections.checked_add(n.checked_mul(64).unwrap()).unwrap(); - let flags = u64_at(p + 8); - let address = u32::try_from(u64_at(p + 16)).expect("kernel section exceeds local address width"); - let size = usize::try_from(u64_at(p + 32)).unwrap(); - if flags & 2 == 0 || size == 0 { - continue; - } - let offset = usize::try_from(u64_at(p + 24)).unwrap(); - if flags & 4 != 0 { - assert_eq!(address, 0); - c.program = bytes[offset..offset.checked_add(size).unwrap()].to_vec(); - } else { - assert_ne!( - u32_at(p + 4), - 8, - "allocated NOBITS requires explicit initialization" - ); - assert_eq!(address, 0x40000000); - assert_eq!(flags & 1, 0, "kernel globals must be read-only"); - c.constants.resize(size.max(512), 0); - c.constants[..size].copy_from_slice(&bytes[offset..offset.checked_add(size).unwrap()]); - } - } - assert!(!c.program.is_empty() && c.program.len() <= 4096 && c.program.len() % 4 == 0); - c -} -// Independent integer arithmetic gold: the BEMU Engine does not implement these fixed-point opcodes. -fn fixed_gold(case: &Case, bits: usize, op: u32, mode: u32) -> Banks { - let mut gold = case.initial.clone(); - let bytes = bits / 8; - let mask = (1i128 << bits) - 1; - let minimum = -(1i128 << (bits - 1)); - let maximum = (1i128 << (bits - 1)) - 1; - let read = |offset: usize, size: usize| { - let mut value = [0; 8]; - value[..size].copy_from_slice(&case.initial.0[2 * BANK_BYTES + offset..2 * BANK_BYTES + offset + size]); - u64::from_le_bytes(value) as i128 - }; - let signed = |value: i128, width: usize| if value & (1i128 << (width - 1)) != 0 { value - (1i128 << width) } else { value }; - let mut saturated = false; - for lane in 0..5 { - let a = read(lane * bytes, bytes); - let b = read(128 + lane * bytes, bytes); - let mut result = a; - if mode != 3 || lane >= 2 { - let (value, shift) = match op { - 8 => (a + b, 1), 9 => (signed(a, bits) + signed(b, bits), 1), - 10 => (a - b, 1), 11 => (signed(a, bits) - signed(b, bits), 1), - 39 => (signed(a, bits) * signed(b, bits), bits - 1), - 42 => (a, b as usize & (bits - 1)), - 43 => (signed(a, bits), b as usize & (bits - 1)), - 46 => (read(lane * bytes * 2, bytes * 2), b as usize & (bits * 2 - 1)), - 47 => (signed(read(lane * bytes * 2, bytes * 2), bits * 2), b as usize & (bits * 2 - 1)), - _ => unreachable!(), - }; - let truncated = value >> shift; - let half = shift != 0 && value & (1i128 << (shift - 1)) != 0; - let low = shift != 0 && value & ((1i128 << (shift - 1)) - 1) != 0; - let increment = match mode { - 0 => half, 1 => half && (low || truncated & 1 != 0), 2 => false, - 3 => truncated & 1 == 0 && value & ((1i128 << shift) - 1) != 0, - _ => unreachable!(), - }; - result = truncated + i128::from(increment); - if op == 39 || op == 47 { - saturated |= result < minimum || result > maximum; - result = result.clamp(minimum, maximum); - } else if op == 46 { - saturated |= result > mask; - result = result.min(mask); - } - } - gold.put(1, lane * bytes, &((result & mask) as u64).to_le_bytes()[..bytes]); - } - gold.put(1, 128, &u32::from(saturated).to_le_bytes()); - gold -} - -// Additional coverage for balanced integer reductions and bounded rounding carry. -fn integer_depth_cases(args: [u64; 8]) -> Vec { - let mut cases = vec![]; - for sew in 0..3 { - let bits = 8usize << sew; - let mask = u64::MAX >> (64 - bits); - let values = [1u64 << (bits - 1), mask >> 1, 0, 1, mask]; - let width = [0, 5, 6, 7][sew as usize]; - for reduction in [true, false] { - let operations: Vec<(u32, u32)> = if reduction { - (0..8).map(|op| (op, 2)).collect() - } else { - let mut ops = vec![(8, 2), (9, 2), (10, 2), (11, 2), (39, 0), (42, 0), (43, 0)]; - if sew < 2 { ops.extend([(46, 0), (47, 0)]); } - ops - }; - for (op, kind) in operations { - for mode in 0..(if reduction { 5 } else { 4 }) { - let mut a = args; - a[2] = 5; - let lmul = if 5 * bits > VLEN { 1 } else { 0 }; - let mut words = vec![ - i(0x57, 5, 7, 12, ((sew << 3) | lmul) as i32), - (1 << 25) | (11 << 15) | (width << 12) | (8 << 7) | 7, - i(0x13, 6, 0, 11, 128), - (1 << 25) | (6 << 15) | (width << 12) | (16 << 7) | 7, - (1 << 25) | (11 << 15) | (width << 12) | (24 << 7) | 7, - ]; - let mut vm = 1; - if reduction { - if mode == 1 { - words.push((0x19 << 26) | (1 << 25) | (8 << 20) | (3 << 12) | 0x57); - vm = 0; - } else if mode == 2 { - words.push((0x1b << 26) | (1 << 25) | (2 << 12) | 0x57); - vm = 0; - } else if mode == 3 { - words.push(i(0x57, 5, 7, 0, 0xc00 | (sew << 3) as i32)); - } else if mode == 4 { - words.push(i(0x73, 0, 5, 1, 8)); - } - } else { - words.push(i(0x73, 0, 5, mode, 0xa)); - words.push(i(0x73, 0, 5, 0, 9)); - if op == 46 || op == 47 { - words.push((1 << 25) | (11 << 15) | ([5, 6, 7][sew as usize] << 12) | (8 << 7) | 7); - } - } - if !reduction && mode == 3 { words.push(i(0x73, 0, 5, 2, 8)); } - words.push((op << 26) | (vm << 25) | (8 << 20) | (16 << 15) | (kind << 12) | (24 << 7) | 0x57); - words.push(i(0x57, 5, 7, 12, ((sew << 3) | lmul) as i32)); - words.push((1 << 25) | (10 << 15) | (width << 12) | (24 << 7) | 0x27); - if !reduction { - words.push(i(0x73, 6, 2, 0, 9)); - words.push(s(0x23, 2, 10, 6, 128)); - } - words.push(0x8067); - let mut case = micro(&format!("depth_{}_e{bits}_op{op}_mode{mode}", if reduction {"reduce"} else {"round"}), &words, a); - for (index, value) in values.into_iter().enumerate() { - let bytes = bits / 8; - let rhs = if reduction || op == 39 || kind == 2 { value } else { [0, 1, bits as u64 - 1, bits as u64 / 2, 3][index] }; - case.initial.put(2, index * bytes, &value.to_le_bytes()[..bytes]); - case.initial.put(2, 128 + index * bytes, &rhs.to_le_bytes()[..bytes]); - } - if !reduction { case.fixed_expected = Some(fixed_gold(&case, bits, op, mode)); } - if reduction && mode == 4 { case.expected_cause = 2; } - cases.push(case); - } - } - } - } - cases -} - -impl Model { - pub fn new(kernel_dir: &Path) -> Self { - let mut cases = vec![]; - let mut args = [0; 8]; - args[0] = 1 << 32; - args[1] = 2 << 32; - for kernel in ["silu", "swiglu", "snake", "quant"] { - let lengths: &[u32] = if kernel == "quant" { - &[32, 64] - } else { - &[1, 31, 33, 65] - }; - for &n in lengths { - let mut a = args; - match kernel { - "swiglu" => { - a[2] = 3 << 32; - a[3] = u64::from(n); - } - "snake" => { - a[2] = 2; - a[3] = u64::from(n); - a[4] = 48; - a[5] = 56; - } - "quant" => { - a[1] = args[0] + 1024; - a[2] = args[1]; - a[3] = u64::from(n); - a[4] = args[0] + 1536; - } - _ => a[2] = u64::from(n), - } - if kernel == "silu" { a[3] = args[0] + 1024; } - if kernel == "swiglu" { a[4] = args[0] + 1024; } - let mut c = elf( - &kernel_dir.join(format!("{kernel}.elf")), - &format!("{kernel}_{n}"), - a, - ); - if kernel == "snake" { - c.initial.put( - 0, - 48, - &[ - 0.0f32.to_le_bytes(), - 0.5f32.to_le_bytes(), - 0.25f32.to_le_bytes(), - (-0.25f32).to_le_bytes(), - ] - .concat(), - ); - } - cases.push(c); - } - } - // Unaligned loads/stores and every RV32M operation, including divide by zero and overflow. - for &(a, b) in &[ - (17u32, 3u32), - (0x80000000, 0xffffffff), - (0xffffffef, 0), - (0xffffffff, 7), - ] { - let mut aargs = args; - aargs[2] = u64::from(a); - aargs[3] = u64::from(b); - let mut words = vec![]; - for f in 0..8 { - words.push(r(0x33, 5, f, 12, 13, 1)); - words.push(s(0x23, 2, 10, 5, f * 4 + 1)); - } - for f in [0, 1, 2, 4, 5] { - words.push(i(3, 6, f, 10, 1)); - words.push(s(0x23, 2, 10, 6, 48 + f * 4)); - } - words.push(0x8067); - cases.push(micro(&format!("scalar_m_{a:x}_{b:x}"), &words, aargs)); - } - for funct in [0, 1, 4, 5, 6, 7] { - for &(left, right) in &[(0xffffffff, 1), (7, 7)] { - let mut a = args; - a[2] = left; - a[3] = right; - let words = [ - i(0x13, 5, 0, 0, 0), - (13 << 20) | (12 << 15) | (funct << 12) | (4 << 8) | 0x63, - i(0x13, 5, 0, 0, 1), - s(0x23, 2, 10, 5, 0), - 0x8067, - ]; - cases.push(micro( - &format!("branch_{funct}_{left:x}_{right:x}"), - &words, - a, - )); - } - } - for sew in [2, 3] { - for (vl, start) in [(0, 0), (4, 0), (4, 1), (4, 4)] { - let width = if sew == 2 { 6 } else { 7 }; - let words = [ - i(0x57, 5, 7, 4, 0xc00 | (sew << 3)), - (1 << 25) | (11 << 15) | (width << 12) | (9 << 7) | 7, - i(0x57, 5, 7, 12, (sew << 3) | 3), - i(0x73, 0, 5, start, 8), - r(0x57, 9, 6, 13, 0, 33), - i(0x57, 5, 7, 4, 0xc00 | (sew << 3)), - (1 << 25) | (10 << 15) | (width << 12) | (9 << 7) | 0x27, - i(0x73, 7, 2, 0, 1), - s(0x23, 2, 10, 7, 64), - 0x8067, - ]; - let mut a = args; - a[2] = vl; - a[3] = 0x80000001; - let mut c = micro(&format!("scalar_insert_e{}_vl{vl}_start{start}", 8 << sew), &words, a); - if sew == 3 && ELEN == 32 { c.expected_cause = 2; } - cases.push(c); - } - } - for scalar in [0x3f800000u64, 0x7f800123] { - for (vl, start) in [(0, 0), (4, 0), (4, 1), (4, 4)] { - let words = [ - i(0x57, 5, 7, 4, 0xc00 | (2 << 3)), - (1 << 25) | (11 << 15) | (6 << 12) | (9 << 7) | 7, - i(7, 3, 2, 13, 0), - i(0x57, 5, 7, 12, (2 << 3) | 3), - i(0x73, 0, 5, start, 8), - r(0x57, 9, 5, 3, 0, 33), - i(0x57, 5, 7, 4, 0xc00 | (2 << 3)), - (1 << 25) | (10 << 15) | (6 << 12) | (9 << 7) | 0x27, - i(0x73, 7, 2, 0, 1), - s(0x23, 2, 10, 7, 64), - 0x8067, - ]; - let mut a = args; - a[2] = vl; - a[3] = 3 << 32; - let mut c = micro(&format!("floating_insert_{scalar:x}_vl{vl}_start{start}"), &words, a); - c.initial.put(3, 0, &scalar.to_le_bytes()); - cases.push(c); - } - } - let mut a = args; - a[2] = 17; - cases.push(micro( - "vset_avl_register_rules", - &[ - i(0x57, 5, 7, 12, 16), - s(0x23, 2, 10, 5, 0), - i(0x57, 5, 7, 0, 16), - s(0x23, 2, 10, 5, 4), - i(0x57, 0, 7, 0, 16), - i(0x73, 5, 2, 0, 0xc20), - s(0x23, 2, 10, 5, 8), - i(0x57, 5, 7, 3, 0xc10u32 as i32), - s(0x23, 2, 10, 5, 12), - 0x8067, - ], - a, - )); - // SEW and integer/fractional LMUL, VL=0, mask/tail preservation, CSR visibility. - for sew in 0..4 { - for lmul in [0, 1, 2, 3, 5, 6, 7] { - for avl in [0, 1, 17, 63] { - let vt = (sew << 3) | lmul; - let mut a = args; - a[2] = avl; - let width = [0, 5, 6, 7][sew as usize]; - let words = [ - i(0x57, 5, 7, 12, vt), - (1 << 25) | (11 << 15) | (width << 12) | (8 << 7) | 7, - r(0x57, 16, 0, 8, 8, 0) | (1 << 25), - (1 << 25) | (10 << 15) | (width << 12) | (16 << 7) | 0x27, - i(0x73, 6, 2, 0, 0xc20), - s(0x23, 2, 10, 6, 1536), - i(0x73, 6, 2, 0, 0xc21), - s(0x23, 2, 10, 6, 1540), - 0x8067, - ]; - let mut c = micro( - &format!("vector_sew{}_lmul{lmul}_avl{avl}", 8 << sew), - &words, - a, - ); - if (8 << sew) > ELEN as i32 || (lmul >= 5 && (8 << sew) > (ELEN as i32 >> (8 - lmul))) { - c.expected_cause = 2; - } - cases.push(c); - } - } - } - for sew in 0..4 { - let mut a = args; - a[2] = 17; - let width = [0, 5, 6, 7][sew as usize]; - let words = [ - i(0x57, 5, 7, 12, sew << 3), - (1 << 25) | (11 << 15) | (width << 12) | (8 << 7) | 7, - (0x18 << 26) | (1 << 25) | (8 << 20) | (3 << 12) | 0x57, - (8 << 20) | (8 << 15) | (16 << 7) | 0x57, - (1 << 25) | (10 << 15) | (width << 12) | (16 << 7) | 0x27, - 0x8067, - ]; - let mut c = micro(&format!("masked_sew{}", 8 << sew), &words, a); - if sew == 3 && ELEN == 32 { c.expected_cause = 2; } - cases.push(c); - } - for sew in [2, 3] { - let mut a = args; - a[2] = 17; - let width = if sew == 2 { 6 } else { 7 }; - let mut c = micro( - &format!("vector_floating_{}", 8 << sew), - &[ - i(0x57, 5, 7, 12, sew << 3), - (1 << 25) | (11 << 15) | (width << 12) | (8 << 7) | 7, - (1 << 25) | (8 << 20) | (8 << 15) | (1 << 12) | (16 << 7) | 0x57, - (0x2c << 26) | (1 << 25) | (8 << 20) | (8 << 15) | (1 << 12) | (16 << 7) | 0x57, - (1 << 25) | (10 << 15) | (width << 12) | (16 << 7) | 0x27, - (0x12 << 26) - | (1 << 25) - | (16 << 20) - | (1 << 15) - | (1 << 12) - | (24 << 7) - | 0x57, - i(0x13, 7, 0, 10, 1024), - (1 << 25) | (7 << 15) | (width << 12) | (24 << 7) | 0x27, - i(0x73, 6, 2, 0, 1), - s(0x23, 2, 10, 6, 1536), - 0x8067, - ], - a, - ); - if sew == 3 { - if ELEN == 32 { c.expected_cause = 2; } - for n in 0..17 { - c.initial - .put(2, n * 8, &(n as f64 / 4.0 - 2.0).to_le_bytes()); - } - } - cases.push(c); - } - // Scalar IEEE arithmetic/FMA/conversion flags are stored through CSR reads. - for fmt in 0..2 { - let load_f = if fmt == 0 { 2 } else { 3 }; - let step = if fmt == 0 { 4 } else { 8 }; - let mut c = micro( - &format!("floating_{}", 32 << fmt), - &[ - i(7, 1, load_f, 11, 0), - i(7, 2, load_f, 11, step), - r(0x53, 3, 0, 1, 2, fmt), - (2 << 27) | (fmt << 25) | (2 << 20) | (1 << 15) | (4 << 7) | 0x43, - s(0x27, load_f, 10, 3, 0), - s(0x27, load_f, 10, 4, 8), - r(0x53, 5, 0, 1, 0, 0x60 | fmt), - s(0x23, 2, 10, 5, 16), - i(0x73, 6, 2, 0, 1), - s(0x23, 2, 10, 6, 20), - 0x8067, - ], - args, - ); - if fmt == 0 { - c.initial.put( - 2, - 0, - &[1.5f32.to_le_bytes(), (-2.25f32).to_le_bytes()].concat(), - ); - } else { - if ELEN == 32 { c.expected_cause = 2; } - c.initial.put( - 2, - 0, - &[1.5f64.to_le_bytes(), (-2.25f64).to_le_bytes()].concat(), - ); - } - cases.push(c); - } - // Raw IEEE encodings preserve signaling NaNs and signed zero in the inputs. - for fmt in 0..2 { - let fixtures: &[(&str, [u64; 3])] = if fmt == 0 { - &[ - ("zeros", [0, 0x80000000, 0x3f800000]), - ("division_by_zero", [0x3f800000, 0, 0xbf800000]), - ("negative_zero", [0x80000000, 0x40000000, 0]), - ("qnan", [0x7fc12345, 0x3f800000, 0xbf800000]), - ("snan", [0x7f812345, 0x3f800000, 0]), - ("infinities", [0x7f800000, 0, 0xff800000]), - ("subnormal", [1, 0x3f000000, 0x00800000]), - ("overflow", [0x7f7fffff, 0x40000000, 0xff7fffff]), - ("conversion_ties", [0x3fc00000, 0xc0200000, 0x3f800000]), - ("division_inexact", [0x3f800000, 0x40400000, 0x80000000]), - ("fused_cancellation", [0x3f800001, 0x3f800001, 0xbf800000]), - ] - } else { - &[ - ("zeros", [0, 0x8000000000000000, 0x3ff0000000000000]), - ( - "division_by_zero", - [0x3ff0000000000000, 0, 0xbff0000000000000], - ), - ("negative_zero", [0x8000000000000000, 0x4000000000000000, 0]), - ( - "qnan", - [0x7ff8123456789abc, 0x3ff0000000000000, 0xbff0000000000000], - ), - ("snan", [0x7ff0123456789abc, 0x3ff0000000000000, 0]), - ("infinities", [0x7ff0000000000000, 0, 0xfff0000000000000]), - ("subnormal", [1, 0x3fe0000000000000, 0x0010000000000000]), - ( - "overflow", - [0x7fefffffffffffff, 0x4000000000000000, 0xffefffffffffffff], - ), - ( - "conversion_ties", - [0x3ff8000000000000, 0xc004000000000000, 0x3ff0000000000000], - ), - ( - "division_inexact", - [0x3ff0000000000000, 0x4008000000000000, 0x8000000000000000], - ), - ( - "fused_cancellation", - [0x3ff0000000000001, 0x3ff0000000000001, 0xbff0000000000000], - ), - ("narrowing_tie", [0x3ff0000010000000, 0x3ff0000000000000, 0]), - ] - }; - let width = if fmt == 0 { 2 } else { 3 }; - let bytes = 1usize << width; - for frm in 0..5 { - for &(name, values) in fixtures { - if frm != 0 - && !matches!( - name, - "conversion_ties" | "fused_cancellation" | "narrowing_tie" - ) - { - continue; - } - let mut a = args; - a[2] = 0x01000001; - a[3] = 0xffffffff; - let mut words = vec![i(0x73, 0, 5, frm, 2)]; - for f in 1..=3 { - words.push(i(7, f, width, 11, (f as usize - 1) as i32 * bytes as i32)); - } - // Clear and sample fflags for every operation; result and flags share a 32-byte record. - let mut operations = vec![]; - for op in [0, 4, 8, 12] { - operations.push((r(0x53, 4, 7, 1, 2, op | fmt), s(0x27, width, 10, 4, 0))); - } - for op in [0x43, 0x47, 0x4b, 0x4f] { - operations.push(( - (3 << 27) - | (fmt << 25) - | (2 << 20) - | (1 << 15) - | (7 << 12) - | (4 << 7) - | op, - s(0x27, width, 10, 4, 0), - )); - } - for source in [1, 2] { - for unsigned in 0..2 { - operations.push(( - r(0x53, 5, 7, source, unsigned, 0x60 | fmt), - s(0x23, 2, 10, 5, 0), - )); - } - } - for unsigned in 0..2 { - operations.push(( - r(0x53, 4, 7, 12 + unsigned, unsigned, 0x68 | fmt), - s(0x27, width, 10, 4, 0), - )); - } - let target = 1 - fmt; - if fmt == 1 { operations.push(( - r(0x53, 4, 7, 1, fmt, 0x20 | target), - s(0x27, if target == 0 { 2 } else { 3 }, 10, 4, 0), - )); } - for (n, (operation, store)) in operations.into_iter().enumerate() { - let offset = n as u32 * 32; - words.extend([ - i(0x73, 0, 5, 0, 1), - operation, - store | ((offset >> 5) << 25), - i(0x73, 6, 2, 0, 1), - s(0x23, 2, 10, 6, offset + 16), - ]); - } - words.extend([i(0x73, 6, 2, 0, 2), s(0x23, 2, 10, 6, 1536), 0x8067]); - let mut c = micro( - &format!("scalar_ieee_{}_frm{frm}_{name}", 32 << fmt), - &words, - a, - ); - if fmt == 1 && ELEN == 32 { c.expected_cause = 2; } - for (n, bits) in values.into_iter().enumerate() { - c.initial.put(2, n * bytes, &bits.to_le_bytes()[..bytes]); - } - cases.push(c); - } - // The vector case runs the same exceptional inputs lane by lane. - let mut a = args; - a[2] = fixtures.len() as u64; - a[3] = 3 << 32; - a[4] = 5 << 32; - let vwidth: u32 = if fmt == 0 { 6 } else { 7 }; - let load = - |vd: u32, rs: u32| (1 << 25) | (rs << 15) | (vwidth << 12) | (vd << 7) | 7; - let store = |vd: u32| (1 << 25) | (7 << 15) | (vwidth << 12) | (vd << 7) | 0x27; - let mut words = vec![ - i(0x73, 0, 5, frm, 2), - i(0x57, 5, 7, 12, (fmt as i32 + 2) << 3), - load(8, 11), - load(12, 13), - ]; - for (n, operation) in [0, 2, 0x24, 0x20, 0x2c].into_iter().enumerate() { - let offset = n as u32 * 128; - words.extend([ - i(0x73, 0, 5, 0, 1), - load(20, 14), - (operation << 26) - | (1 << 25) - | (12 << 20) - | (8 << 15) - | (1 << 12) - | (20 << 7) - | 0x57, - i(0x13, 7, 0, 10, offset as i32), - store(20), - i(0x73, 6, 2, 0, 1), - s(0x23, 2, 10, 6, 1024 + n as u32 * 4), - ]); - } - for unsigned in 0..2 { - words.extend([ - i(0x73, 0, 5, 0, 1), - (0x12 << 26) - | (1 << 25) - | (8 << 20) - | ((1 - unsigned) << 15) - | (1 << 12) - | (24 << 7) - | 0x57, - i(0x13, 7, 0, 10, 640 + unsigned as i32 * 128), - store(24), - i(0x73, 6, 2, 0, 1), - s(0x23, 2, 10, 6, 1044 + unsigned * 4), - ]); - } - words.push(0x8067); - let mut c = micro(&format!("vector_ieee_{}_frm{frm}", 32 << fmt), &words, a); - if fmt == 1 && ELEN == 32 { c.expected_cause = 2; } - for (n, (_, values)) in fixtures.iter().enumerate() { - for (bank, bits) in [(2, values[0]), (3, values[1]), (5, values[2])] { - c.initial.put(bank, n * bytes, &bits.to_le_bytes()[..bytes]); - } - } - cases.push(c); - } - } - cases.push(micro( - "bank_line_crossing_word", - &[i(3, 5, 2, 11, 15), s(0x23, 2, 10, 5, 15), 0x8067], - args, - )); - let mut double_load = micro( - "bank_line_crossing_double", - &[i(7, 5, 3, 11, 11), s(0x27, 3, 10, 5, 11), 0x8067], - args, - ); - if ELEN == 32 { double_load.expected_cause = 2; } - cases.push(double_load); - let mut a = args; - a[2] = 17; - let mut c = micro( - "illegal_group_alignment", - &[ - i(0x57, 5, 7, 12, 17), - (1 << 25) | (11 << 15) | (6 << 12) | (9 << 7) | 7, - ], - a, - ); - c.expected_cause = 2; - cases.push(c); - cases.push(micro("illegal_opcode", &[0xffffffff], args)); - cases.push(micro("load_fault", &[i(3, 5, 2, 11, 0)], { - let mut a = args; - a[1] = 6 << 32; - a - })); - cases.push(micro("store_fault", &[s(0x23, 2, 10, 11, 0)], { - let mut a = args; - a[0] = (1 << 32) + 4094; - a - })); - for (n, cause) in [ - (cases.len() - 3, 2), - (cases.len() - 2, 5), - (cases.len() - 1, 7), - ] { - cases[n].expected_cause = cause; - } - cases.extend(integer_depth_cases(args)); - let mut local = micro( - "local_stack_and_constants", - &[ - i(0x13, 2, 0, 2, -16), - s(0x23, 2, 2, 11, 0), - s(0x23, 2, 2, 12, 4), - i(3, 5, 2, 2, 0), - i(3, 6, 2, 2, 4), - s(0x23, 2, 10, 5, 0), - s(0x23, 2, 10, 6, 4), - 0x400003b7, - i(3, 5, 2, 7, 0), - s(0x23, 2, 10, 5, 8), - i(0x13, 2, 0, 2, 16), - 0x8067, - ], - args, - ); - local.constants[..4].copy_from_slice(&0x72cafe31u32.to_le_bytes()); - cases.push(local); - let mut readonly = micro( - "store_readonly_constants", - &[0x400002b7, s(0x23, 2, 5, 11, 0), 0x8067], - args, - ); - readonly.expected_cause = 7; - cases.push(readonly); - for kernel in ["sin", "cos"] { - let values: Vec = (0..65).map(|n| (n as f32 - 32.0) * 0.75).chain([ - 120.0, -120.0, 10000.0, -10000.0, 1e10, -1e10, 1e30, -1e30, - f32::MAX, -f32::MAX, f32::from_bits(1), -f32::from_bits(1), - 0.0, -0.0, f32::INFINITY, f32::NEG_INFINITY, f32::NAN, - ]).collect(); - let mut a = args; - a[2] = values.len() as u64; - a[3] = a[0] + 1024; - let mut c = elf(&kernel_dir.join(format!("{kernel}.elf")), &format!("{kernel}_fp32_range"), a); - for (n, value) in values.iter().enumerate() { c.initial.put(2, n * 4, &value.to_le_bytes()); } - cases.push(c); - } - for kernel in ["gelu", "tanh"] { - let mut a = args; - a[2] = 37; - a[3] = a[0] + 1024; - cases.push(elf(&kernel_dir.join(format!("{kernel}.elf")), &format!("{kernel}_fp32"), a)); - } - let b = initial(); - Self { - cases, - current: 0, - actual: b.clone(), - expected: b, - engine: Engine::new(VLEN, ELEN, IBUF_BYTES), - fault: None, - } - } - pub fn select(&mut self, n: usize) { - self.current = n; - let c = &self.cases[n]; - self.actual = c.initial.clone(); - let descriptor = [ - u64::from(c.entry), - c.program.len() as u64, - 0x40002000, - u64::from(c.args[0]), - u64::from(c.args[1]), - u64::from(c.args[2]), - u64::from(c.args[3]), - u64::from(c.args[4]), - u64::from(c.args[5]), - u64::from(c.args[6]), - u64::from(c.args[7]), - 0, - ]; - for (index, value) in descriptor.iter().enumerate() { - self.actual.put(1, 2048 + index * 8, &value.to_le_bytes()); - } - self.expected = match &c.fixed_expected { - Some(gold) => { - let mut expected = gold.clone(); - let descriptor = BANK_BYTES + 2048..BANK_BYTES + 2144; - expected.0[descriptor.clone()].copy_from_slice(&self.actual.0[descriptor]); - expected - } - None => self.actual.clone(), - }; - self.engine.load_constants(n % 2, &c.constants); - self.engine.load_program(n % 2, &c.program); - self.fault = if c.fixed_expected.is_some() { None } else { self - .engine - .run( - n % 2, - c.entry, - c.program.len() as u32, - c.args.map(u64::from), - 0x40002000, - &mut self.expected, - ) - .err() }; - assert_eq!( - self.fault.map(|f| f.cause).unwrap_or(0), - c.expected_cause, - "unexpected reference outcome for {}: {:?}", - c.name, self.fault - ); - let read_float = |banks: &Banks, address: u64| { - f32::from_le_bytes(banks.0[banks.range(address, 4).unwrap()].try_into().unwrap()) - }; - let operation = c.name.split('_').next().unwrap(); - if matches!(operation, "silu" | "swiglu" | "snake" | "sin" | "cos") { - let length = c.args[if matches!(operation, "swiglu" | "snake") { 3 } else { 2 }] as usize; - let channels = if operation == "snake" { c.args[2] as usize } else { 1 }; - for channel in 0..channels { - for index in 0..length { - let offset = ((channel * length + index) * 4) as u64; - let x = read_float(&c.initial, c.args[1] + offset); - let expected = match operation { - "silu" => x / (1.0 + (-x).exp()), - "swiglu" => x / (1.0 + (-x).exp()) * read_float(&c.initial, c.args[2] + offset), - "snake" => { - let alpha = read_float(&c.initial, c.args[4] + channel as u64 * 4).exp(); - let beta = read_float(&c.initial, c.args[5] + channel as u64 * 4).exp() + 1e-9; - let sine = (x * alpha).sin(); - x + sine * sine / beta - } - "sin" => x.sin(), - "cos" => x.cos(), - _ => unreachable!(), - }; - let actual = read_float(&self.expected, c.args[0] + offset); - assert!(if expected.is_nan() { actual.is_nan() } else { - actual == expected || (actual - expected).abs() <= 2e-6 * (1.0 + expected.abs()) - }, "{} element {index}: {actual} != libm {expected}", c.name); - } - } - } - eprintln!("RVV case {n}: {}", c.name); - } -} diff --git a/arch/src/main/scala/framework/rvv/src/main/resources/README.md b/arch/src/main/scala/framework/rvv/src/main/resources/README.md deleted file mode 100644 index e69de29bb..000000000 diff --git a/arch/src/main/scala/framework/rvv/src/main/resources/package.sv b/arch/src/main/scala/framework/rvv/src/main/resources/package.sv deleted file mode 100644 index 9395630e6..000000000 --- a/arch/src/main/scala/framework/rvv/src/main/resources/package.sv +++ /dev/null @@ -1,97 +0,0 @@ -package rvv_pkg; - timeunit 1ns; timeprecision 1ps; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - import "DPI-C" function void rvv_model_init(); - import "DPI-C" function int unsigned rvv_case_count(); - import "DPI-C" function void rvv_case_select(int unsigned n); - import "DPI-C" function longint unsigned rvv_case_meta(int unsigned field); - import "DPI-C" function int unsigned rvv_image_bytes(); - import "DPI-C" function int unsigned rvv_program_word(int unsigned n); - import "DPI-C" function int unsigned rvv_image_word(int unsigned n); - import "DPI-C" function void rvv_memory_masked_write( - longint unsigned address, - longint unsigned data, - int unsigned mask - ); - import "DPI-C" function longint unsigned rvv_expected_status(int unsigned field); - import "DPI-C" function longint unsigned rvv_memory_read( - longint unsigned address, - int unsigned size - ); - import "DPI-C" function int unsigned rvv_memory_error( - longint unsigned address, - int unsigned size - ); - import "DPI-C" function void rvv_memory_write( - longint unsigned address, - int unsigned size, - longint unsigned data, - int unsigned mask - ); - import "DPI-C" function int unsigned rvv_compare_memory(); - import "DPI-C" function int unsigned rvv_compare_image(); - - class launch_item extends uvm_sequence_item; - bit execute; - bit protocol_fault; - bit [31:0] protocol_cause, protocol_pc, protocol_instruction; - bit [63:0] protocol_tval; - bit [3:0] rob; - `uvm_object_utils(launch_item) - function new(string name = "launch_item"); - super.new(name); - endfunction - endclass - class completion_item extends uvm_sequence_item; - bit fault; - bit [15:0] write_bank; - bit [3:0] rob; - bit [31:0] pc, instruction, cause; - bit [63:0] tval; - `uvm_object_utils_begin(completion_item) - `uvm_field_int(fault, UVM_DEFAULT) - `uvm_field_int(rob, UVM_DEFAULT) - `uvm_field_int(write_bank, UVM_DEFAULT) - `uvm_field_int(pc, UVM_DEFAULT) - `uvm_field_int(instruction, UVM_DEFAULT | UVM_NOCOMPARE) - `uvm_field_int(cause, UVM_DEFAULT) - `uvm_field_int(tval, UVM_DEFAULT) - `uvm_object_utils_end - function new(string name = "completion_item"); - super.new(name); - endfunction - function bit do_compare(uvm_object rhs, uvm_comparer comparer); - completion_item other; - if (!$cast(other, rhs)) return 0; - return super.do_compare(rhs, comparer) && (!fault || instruction == other.instruction); - endfunction - endclass - class kernel_model extends reference_model #(launch_item, completion_item); - `uvm_component_utils(kernel_model) - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - function void write(launch_item t); - completion_item expected = completion_item::type_id::create("expected"); - expected.rob = t.rob; - expected.fault = t.execute ? rvv_expected_status(0) : 0; - expected.pc = t.execute ? rvv_expected_status(1) : 0; - expected.instruction = t.execute ? rvv_expected_status(2) : 0; - expected.cause = t.execute ? rvv_expected_status(3) : 0; - expected.tval = t.execute ? rvv_expected_status(4) : 0; - if (t.protocol_fault) begin - expected.fault = 1; - expected.pc = t.protocol_pc; - expected.instruction = t.protocol_instruction; - expected.cause = t.protocol_cause; - expected.tval = t.protocol_tval; - end - expected.write_bank = t.execute && !expected.fault ? 1 : 0; - expected_ap.write(expected); - endfunction - endclass - typedef checked_env#(launch_item, completion_item, kernel_model) kernel_env; - `include "test.svh" -endpackage diff --git a/arch/src/main/scala/framework/rvv/src/main/resources/test.svh b/arch/src/main/scala/framework/rvv/src/main/resources/test.svh deleted file mode 100644 index 2d7e7bf8c..000000000 --- a/arch/src/main/scala/framework/rvv/src/main/resources/test.svh +++ /dev/null @@ -1,265 +0,0 @@ -class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual rvv_if vif; - kernel_env test_env; - int unsigned completed = 0; - bit protocol_fault = 0; - bit reject_write = 0; - bit [31:0] protocol_cause, protocol_pc, protocol_instruction; - bit [63:0] protocol_tval; - int unsigned requests[`RVV_MEMORY_PORTS]; - function new(string name, uvm_component parent); - super.new(name, parent); - timeout = 40ms; - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual rvv_if)::get(this, "", "vif", vif)) - `uvm_fatal("VIF", "RVV interface missing") - test_env = kernel_env::type_id::create("env", this); - endfunction - task memory_port(int p); - bit read_pending = 0, write_pending = 0; - int read_delay, write_delay; - bit [31:0] seed = 32'h629731a5 ^ (p * 32'h1234567); - longint unsigned address; - forever begin - @(negedge vif.clock); - seed = {seed[30:0], seed[31] ^ seed[21] ^ seed[1] ^ seed[0]}; - if (vif.reset) begin - read_pending = 0; - write_pending = 0; - vif.read_ready[p] = 0; - vif.write_ready[p] = 0; - vif.read_response_valid[p] = 0; - vif.write_response_valid[p] = 0; - end else begin - vif.read_ready[p] = !read_pending && seed[0]; - vif.write_ready[p] = !write_pending && seed[1]; - if (read_pending && read_delay != 0) read_delay--; - if (write_pending && write_delay != 0) write_delay--; - vif.read_response_valid[p] = read_pending && read_delay == 0; - vif.write_response_valid[p] = write_pending && write_delay == 0; - end - @(posedge vif.clock); - if (!vif.reset) begin - if (vif.read_valid[p] && vif.read_ready[p]) begin - requests[p]++; - read_pending = 1; - read_delay = 1 + seed[4:2]; - address = (longint'(vif.read_bank[p] + vif.read_group[p]) << 32) | (int'(vif.read_row[p]) << 4); - vif.read_data[p] = {rvv_memory_read(address + 8, 3), rvv_memory_read(address, 3)}; - end - if (vif.write_valid[p] && vif.write_ready[p]) begin - requests[p]++; - write_pending = 1; - write_delay = 1 + seed[7:5]; - address = (longint'(vif.write_bank[p] + vif.write_group[p]) << 32) | (int'(vif.write_row[p]) << 4); - if (!reject_write) begin - rvv_memory_masked_write(address, vif.write_data[p][63:0], vif.write_mask[p][7:0]); - rvv_memory_masked_write(address + 8, vif.write_data[p][127:64], - vif.write_mask[p][15:8]); - end - vif.write_ok[p] = !reject_write; - end - if (vif.read_response_valid[p] && vif.read_response_ready[p]) read_pending = 0; - if (vif.write_response_valid[p] && vif.write_response_ready[p]) write_pending = 0; - end - end - endtask - task monitor(); - forever begin - @(posedge vif.clock); - if (!vif.reset) begin - if (vif.command_valid && vif.command_ready) begin - launch_item item = launch_item::type_id::create("launch"); - item.execute = vif.command_funct7 == 79; - item.protocol_fault = protocol_fault; - item.protocol_cause = protocol_cause; - item.protocol_tval = protocol_tval; - item.protocol_pc = protocol_pc; - item.protocol_instruction = protocol_instruction; - item.rob = vif.command_rob; - test_env.input_export.write(item); - end - if (vif.done_valid && vif.done_ready) begin - completion_item item = completion_item::type_id::create("completion"); - item.fault = vif.done_fault; - item.pc = vif.done_pc; - item.instruction = vif.done_instruction; - item.cause = vif.done_cause; - item.tval = vif.done_tval; - item.rob = vif.done_rob; - item.write_bank = vif.done_write_bank; - test_env.output_export.write(item); - end - end - end - endtask - task command(int funct7, longint unsigned rs1, longint unsigned rs2); - @(negedge vif.clock); - vif.command_funct7 = funct7; - vif.command_rs1 = rs1; - vif.command_rs2 = rs2; - vif.command_rob = completed % 16; - vif.command_valid = 1; - do @(posedge vif.clock); while (!vif.command_ready); - @(negedge vif.clock); - vif.command_valid = 0; - endtask - task retire(); - bit [250:0] held; - while (!vif.done_valid) @(negedge vif.clock); - held = { - vif.done_rob, - vif.done_write_bank, - vif.done_fault, - vif.done_pc, - vif.done_instruction, - vif.done_cycles, - vif.done_cause, - vif.done_tval, - vif.done_fflags, - vif.done_vxsat - }; - repeat (5) begin - @(negedge vif.clock); - if(!vif.done_valid || held !== {vif.done_rob,vif.done_write_bank,vif.done_fault,vif.done_pc, - vif.done_instruction,vif.done_cycles,vif.done_cause,vif.done_tval,vif.done_fflags,vif.done_vxsat}) - `uvm_fatal("COMPLETION", "completion changed under backpressure") - if (vif.command_ready || !vif.busy) - `uvm_fatal("COMPLETION", "completion allowed command before retirement") - end - vif.done_ready = 1; - @(negedge vif.clock); - vif.done_ready = 0; - completed++; - test_env.scoreboard.wait_checked(completed); - endtask - task upload(); - command(44, (longint'(rvv_image_bytes()) << 30) | ((6 + rvv_case_meta(0)) << 20), 64'h10000000); - for (int word_index = 0; word_index < rvv_image_bytes() / 4; word_index++) begin - @(negedge vif.clock); - vif.image_valid = 1; - vif.image_data = rvv_image_word(word_index); - do @(posedge vif.clock); while (!vif.image_ready); - end - @(negedge vif.clock); - vif.image_valid = 0; - retire(); - if (!rvv_compare_image()) `uvm_fatal("IMAGE", "image load modified compute banks") - endtask - task execute(); - vif.reset = 1; - vif.image_valid = 0; - vif.command_valid = 0; - vif.done_ready = 0; - foreach (vif.read_ready[p]) begin - vif.read_ready[p] = 0; - vif.write_ready[p] = 0; - vif.read_response_valid[p] = 0; - vif.write_response_valid[p] = 0; - vif.read_data[p] = 0; - vif.write_ok[p] = 0; - end - rvv_model_init(); - fork - begin - for (int p = 0; p < `RVV_MEMORY_PORTS; p++) begin - automatic int port = p; - fork - memory_port(port); - join_none - end - wait fork; - end - monitor(); - join_none - repeat (4) @(negedge vif.clock); - vif.reset = 0; - for (int case_index = 0; case_index < rvv_case_count(); case_index++) begin - rvv_case_select(case_index); - upload(); - command(79, (64'd1 << 20) | ((6 + rvv_case_meta(0)) << 10), 2048); - while (!vif.done_valid) @(negedge vif.clock); - if (!rvv_compare_memory()) - `uvm_fatal("REFERENCE", $sformatf("case %0d bank contents differ", case_index)) - retire(); - `uvm_info("RVV", $sformatf("case %0d completed and compared", case_index), UVM_LOW) - end - foreach (requests[p]) - if (requests[p] == 0) `uvm_fatal("PORTS", $sformatf("bank port %0d was never exercised", p)) - // A malformed header reports a command fault before any code/data is written. - protocol_fault = 1; - protocol_cause = 2; - protocol_tval = 32'hdeadbeef; - command(44, (longint'(rvv_image_bytes()) << 30) | (64'd6 << 20), 0); - for (int index = 0; index < 6; index++) begin - @(negedge vif.clock); - vif.image_valid = 1; - vif.image_data = index == 0 ? 32'hdeadbeef : rvv_image_word(index); - do @(posedge vif.clock); while (!vif.image_ready); - end - @(negedge vif.clock); - vif.image_valid = 0; - retire(); - protocol_tval = 25; - command(44, (64'd25 << 30) | (64'd6 << 20), 0); - retire(); - protocol_cause = 1; - protocol_tval = (64'd1 << 20) | (64'd6 << 10); - command(79, protocol_tval, 2048); - retire(); - protocol_cause = 2; - protocol_tval = 127; - command(127, 0, 0); - retire(); - // Writable globals and BSS are forbidden in a kernel image. - for (int invalid_field = 3; invalid_field <= 5; invalid_field += 2) begin - protocol_cause = 2; - protocol_tval = invalid_field == 3 ? 32'h50000 : 4; - command(44, (longint'(rvv_image_bytes()) << 30) | (64'd6 << 20), 0); - for (int index = 0; index < 6; index++) begin - @(negedge vif.clock); - vif.image_valid = 1; - vif.image_data = index == invalid_field ? - (invalid_field == 3 ? 32'h50000 : 4) : rvv_image_word(index); - do @(posedge vif.clock); while (!vif.image_ready); - end - @(negedge vif.clock); - vif.image_valid = 0; - retire(); - end - // A failed compute-bank write acknowledgement is an execution fault. - protocol_fault = 0; - rvv_case_select(14); - upload(); - protocol_fault = 1; - protocol_cause = 7; - protocol_tval = (64'd1 << 32) + 1; - protocol_pc = 4; - protocol_instruction = rvv_program_word(1); - reject_write = 1; - command(79, (64'd1 << 20) | ((6 + rvv_case_meta(0)) << 10), 2048); - retire(); - reject_write = 0; - protocol_pc = 0; - protocol_instruction = 0; - protocol_fault = 0; - // Reset cancels a command while it waits for a descriptor bank response. - rvv_case_select(rvv_case_count() - 2); - upload(); - command(79, (64'd1 << 20) | ((6 + rvv_case_meta(0)) << 10), 2048); - do @(posedge vif.clock); while (!(vif.read_valid[0] && vif.read_ready[0])); - @(negedge vif.clock); - vif.reset = 1; - test_env.scoreboard.cancel_pending(); - repeat (3) @(negedge vif.clock); - vif.reset = 0; - repeat (5) begin - @(negedge vif.clock); - if (vif.done_valid || vif.busy) `uvm_fatal("RESET", "reset did not cancel pending kernel") - end - `uvm_info("RVV", $sformatf("checked %0d Blink commands", completed), UVM_LOW) - endtask -endclass diff --git a/arch/src/main/scala/framework/rvv/src/main/resources/topkernel.f b/arch/src/main/scala/framework/rvv/src/main/resources/topkernel.f deleted file mode 100644 index 975cf783a..000000000 --- a/arch/src/main/scala/framework/rvv/src/main/resources/topkernel.f +++ /dev/null @@ -1,8 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -+incdir+@RESOURCES@ -+incdir+@RTL@/KernelEngine -@RESOURCES@/interface.sv -@RESOURCES@/package.sv --F @RTL@/KernelEngine/filelist.f -@RESOURCES@/topkernel_tb.sv diff --git a/arch/src/main/scala/framework/rvv/src/main/scala/Emit.scala b/arch/src/main/scala/framework/rvv/src/main/scala/Emit.scala deleted file mode 100644 index c153ea413..000000000 --- a/arch/src/main/scala/framework/rvv/src/main/scala/Emit.scala +++ /dev/null @@ -1,35 +0,0 @@ -package framework.rvv - -import framework.top.GlobalConfig -import framework.rvv.configs.RvvParam -import upickle.default.read - -object Emit extends App { - val defaults = GlobalConfig() - - val b = defaults.copy( - rvv = read[RvvParam](os.read(os.pwd / "src" / "main" / "resources" / "rvv.json")), - memDomain = defaults.memDomain.copy( - bankNum = 6, - bankWidth = 128, - bankEntries = 256, - bankMaskLen = 16, - virtualBankCount = 6, - nCores = 1, - computeCoreIds = Seq(0) - ), - frontend = defaults.frontend.copy( - rob_entries = 16, - bank_id_len = 3, - vbank_id_upper_bound = 5, - iter_len = 16 - ), - tile = defaults.tile.copy(xLen = 64) - ) - - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new KernelEngine(b), - firtoolOpts = args, - args = Array("--target-dir", "build/KernelEngine", "--split-verilog") - ) -} diff --git a/arch/src/main/scala/framework/rvv/src/main/scala/configs/RvvParam.scala b/arch/src/main/scala/framework/rvv/src/main/scala/configs/RvvParam.scala deleted file mode 100644 index 345b157fc..000000000 --- a/arch/src/main/scala/framework/rvv/src/main/scala/configs/RvvParam.scala +++ /dev/null @@ -1,45 +0,0 @@ -package framework.rvv.configs - -import chisel3.util.{isPow2, log2Ceil} -import upickle.default._ - -case class RvvParam( - enable: Boolean, - laneNumber: Int, - vLen: Int, - eLen: Int, - iBufWords: Int, - memoryPorts: Int) { - require(laneNumber > 0) - require(vLen >= 128 && vLen >= eLen && isPow2(vLen)) - require(eLen == 32 || eLen == 64) - require(iBufWords > 0 && isPow2(iBufWords)) - require(memoryPorts > 0) - require(laneNumber <= memoryPorts) - - val wordBits: Int = eLen - val wordOffsetBits: Int = log2Ceil(wordBits) - val maxSew: Int = log2Ceil(eLen / 8) - val wordsPerRegister: Int = vLen / wordBits - val registerBanks: Int = math.min(laneNumber, wordsPerRegister) - require(isPow2(registerBanks), "RVV register bank count must be a power of two") - - val elementsPerRegister: Int = vLen / eLen - val constBytes: Int = 4096 - val stackBytes: Int = 4096 -} - -object RvvParam { - - def apply(): RvvParam = RvvParam( - enable = false, - laneNumber = 4, - vLen = 1024, - eLen = 32, - iBufWords = 1024, - memoryPorts = 4 - ) - - implicit val rw: ReadWriter[RvvParam] = macroRW[RvvParam] - -} diff --git a/arch/src/main/scala/framework/rvv/src/main/scala/core/VectorCore.scala b/arch/src/main/scala/framework/rvv/src/main/scala/core/VectorCore.scala deleted file mode 100644 index 39df8c3c3..000000000 --- a/arch/src/main/scala/framework/rvv/src/main/scala/core/VectorCore.scala +++ /dev/null @@ -1,921 +0,0 @@ -package framework.rvv - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.top.GlobalConfig - -@instantiable -class VectorCore(val b: GlobalConfig) extends Module { - val p = b.rvv - - @public - val io = IO(new Bundle { - val initialize = Input(Bool()) - val roundingMode = Input(UInt(3.W)) - val vxrm = Input(UInt(2.W)) - val vstartWrite = Flipped(Valid(UInt(64.W))) - val vl = Output(UInt(32.W)) - val vtype = Output(UInt(64.W)) - val vstart = Output(UInt(64.W)) - val issue = Flipped(Decoupled(new VectorIssue)) - val result = Decoupled(new VectorResult) - val memoryRequest = Vec(p.memoryPorts, Decoupled(new VectorMemoryRequest)) - val memoryResponse = - Vec(p.memoryPorts, Flipped(Decoupled(new VectorMemoryResponse))) - val busy = Output(Bool()) - }) - - val idle :: snapshot :: capture :: execute :: commit :: Nil = Enum(5) - val state = RegInit(idle) - val resultValid = RegInit(false.B) - val result = RegInit(0.U.asTypeOf(new VectorResult)) - val instruction = Reg(UInt(32.W)) - val scalar1 = Reg(UInt(64.W)) - val scalar2 = Reg(UInt(64.W)) - val floatScalar = Reg(UInt(64.W)) - val vl = RegInit(0.U(32.W)) - val vtype = RegInit("h8000000000000000".U(64.W)) - val vstart = RegInit(0.U(64.W)) - val base = Reg(UInt(32.W)) - val readPortCount = math.max(p.laneNumber, p.memoryPorts) - val wordsPerRegister = p.wordsPerRegister - val maskWords = Reg(Vec(readPortCount, UInt(p.wordBits.W))) - val operandWords = Reg(Vec(readPortCount, UInt(p.wordBits.W))) - val sourceWords = Reg(Vec(readPortCount, UInt(p.wordBits.W))) - val destinationWords = Reg(Vec(readPortCount, UInt(p.wordBits.W))) - val readPhase = RegInit(0.U(3.W)) - val readSent = RegInit(false.B) - val snapshotSent = RegInit(false.B) - val firstBatch = RegInit(true.B) - val writeSent = RegInit(false.B) - val resumeState = Reg(UInt(3.W)) - val finishRequested = WireDefault(false.B) - val accumulator = Reg(UInt(p.wordBits.W)) - val count = Reg(UInt(32.W)) - val offered = RegInit(VecInit(Seq.fill(p.memoryPorts)(false.B))) - val issued = RegInit(VecInit(Seq.fill(p.memoryPorts)(false.B))) - val received = RegInit(VecInit(Seq.fill(p.memoryPorts)(false.B))) - val memoryData = Reg(Vec(p.memoryPorts, UInt(p.wordBits.W))) - val memoryFailed = RegInit(VecInit(Seq.fill(p.memoryPorts)(false.B))) - val divBatchValid = RegInit(false.B) - val divActive = RegInit(VecInit(Seq.fill(p.laneNumber)(false.B))) - val divStarted = RegInit(VecInit(Seq.fill(p.laneNumber)(false.B))) - val divDone = RegInit(VecInit(Seq.fill(p.laneNumber)(false.B))) - val divData = Reg(Vec(p.laneNumber, UInt(p.wordBits.W))) - val fpStarted = RegInit(VecInit(Seq.fill(p.laneNumber)(false.B))) - val fpDone = RegInit(VecInit(Seq.fill(p.laneNumber)(false.B))) - val fpData = Reg(Vec(p.laneNumber, UInt(p.wordBits.W))) - val fpFlags = Reg(Vec(p.laneNumber, UInt(5.W))) - val fpLegal = Reg(Vec(p.laneNumber, Bool())) - - val opcode = instruction(6, 0) - val rd = instruction(11, 7) - val kind = instruction(14, 12) - val rs1 = instruction(19, 15) - val rs2 = instruction(24, 20) - val vm = instruction(25) - val op = instruction(31, 26) - val sew = vtype(4, 3) - val lmul = Cat(vtype(2), vtype(2, 0)).asSInt - val configure = opcode === "h57".U && kind === 7.U - val load = opcode === "h07".U - val store = opcode === "h27".U - val memory = load || store - val fp = opcode === "h57".U && (kind === 1.U || kind === 5.U) - val integer = - opcode === "h57".U && (kind === 0.U || kind === 2.U || kind === 3.U || kind === 4.U || kind === 6.U) - val divide = integer && (kind === 2.U || kind === 6.U) && op >= 32.U && op <= 35.U - val multiply = kind === 2.U || kind === 6.U - val vectorOperand = kind === 0.U || kind === 1.U || kind === 2.U - val maskLogical = integer && kind === 2.U && op >= 24.U && op <= 31.U - - val compare = - (integer && !multiply && (op >= 24.U && op <= 31.U || op === 17.U || op === 19.U)) || - (fp && Seq(24, 25, 27, 28, 29, 31).map(n => op === n.U).reduce(_ || _)) - - val carryOperation = integer && !multiply && op >= 16.U && op <= 19.U - val carryMask = carryOperation && op(0) - val merge = op === 23.U && (fp || integer && !multiply) - val scalarMoveOut = - op === 16.U && rs1 === 0.U && vm && (kind === 2.U || kind === 1.U) - val scalarMoveIn = - op === 16.U && rs2 === 0.U && vm && (kind === 6.U || kind === 5.U) - val population = integer && kind === 2.U && op === 16.U && rs1 === 16.U - val first = integer && kind === 2.U && op === 16.U && rs1 === 17.U - val identity = - integer && kind === 2.U && op === 20.U && rs1 === 17.U && rs2 === 0.U - val intReduction = integer && kind === 2.U && op <= 7.U - val fpReduction = - fp && kind === 1.U && (op === 1.U || op === 3.U || op === 5.U || op === 7.U) - val reduction = intReduction || fpReduction - val wholeMove = integer && kind === 3.U && op === 39.U - val memMode = instruction(27, 26) - - val memWidth = MuxLookup(kind, 0.U(2.W))( - Seq(0.U -> 0.U, 5.U -> 1.U, 6.U -> 2.U, 7.U -> 3.U) - ) - - val wholeMemory = memory && memMode === 0.U && rs2 === 8.U - val indexed = memMode(0) - val memSew = Mux(indexed && !wholeMemory, sew, memWidth) - val intWide = integer && multiply && op >= 48.U - val intNarrow = integer && !multiply && op >= 44.U && op <= 47.U - val floatWide = fp && op === 18.U && rs1 >= 8.U && rs1 <= 15.U - val floatNarrow = fp && op === 18.U && rs1 >= 16.U && rs1 <= 23.U - val extension = integer && multiply && op === 18.U - - val sourceSew = Mux( - memory, - Mux(indexed, memWidth, memSew), - Mux( - intNarrow || floatNarrow || intWide && op >= 52.U && op <= 55.U, - sew +& 1.U, - Mux( - extension, - sew - Mux(rs1(2, 1) === 1.U, 3.U, Mux(rs1(2, 1) === 2.U, 2.U, 1.U)), - sew - ) - ) - ) - - val destinationSew = - Mux(memory, memSew, Mux(intWide || floatWide, sew +& 1.U, sew)) - - val limit = Mux( - wholeMemory, - ((instruction(31, 29) +& 1.U) * p.vLen.U) >> (memWidth +& 3.U), - Mux(wholeMove, ((rs1 +& 1.U) * p.vLen.U) >> (sew +& 3.U), vl) - ) - - def element(word: UInt, index: UInt, width: UInt): UInt = { - val shift = (index << (width.pad(3) + 3.U))(p.wordOffsetBits - 1, 0) - val mask = MuxLookup(width, Fill(p.wordBits, 1.U(1.W)))( - Seq(0.U -> 255.U, 1.U -> 65535.U, 2.U -> "hffffffff".U) - ) - ((word >> shift) & mask)(p.eLen - 1, 0) - } - - def maskElement(word: UInt, index: UInt): Bool = (word >> index(p.wordOffsetBits - 1, 0))(0) - - def wordAddress( - register: UInt, - index: UInt, - width: UInt, - maskBit: Bool = false.B - ): UInt = { - val shift = Mux(maskBit, index, index << (width.pad(3) + 3.U)) - val address = Mux(maskBit, register, register + (shift >> log2Ceil(p.vLen)))(4, 0) - val within = shift.pad(log2Ceil(p.vLen))(log2Ceil(p.vLen) - 1, 0) - address * wordsPerRegister.U + (within >> p.wordOffsetBits) - } - - def groupLegal(register: UInt, width: UInt): Bool = { - val exponent = lmul.pad(6) + width.zext - sew.zext - val registers = 1.U(6.W) << Mux(exponent > 0.S, exponent.asUInt, 0.U) - exponent >= -3.S && exponent <= 3.S && (register & (registers - 1.U)) === 0.U && register +& registers <= 32.U - } - - def overlapLegal( - destination: UInt, - destinationWidth: UInt, - operand: UInt, - operandWidth: UInt - ): Bool = { - val destinationExponent = lmul.pad(6) + destinationWidth.zext - sew.zext - val operandExponent = lmul.pad(6) + operandWidth.zext - sew.zext - val destinationCount = 1 - .U(6.W) << Mux(destinationExponent > 0.S, destinationExponent.asUInt, 0.U) - val operandCount = - 1.U(6.W) << Mux(operandExponent > 0.S, operandExponent.asUInt, 0.U) - val overlap = - destination < operand +& operandCount && operand < destination +& destinationCount - !overlap || destinationWidth === operandWidth || - (destinationWidth < operandWidth && destination === operand) || - (destinationWidth > operandWidth && operandExponent >= 0.S && destination +& destinationCount === operand +& operandCount) - } - - def maximum(width: UInt, multiplier: UInt): UInt = { - val elements = p.vLen.U(32.W) >> (width.pad(3) + 3.U) - Mux(multiplier(2), elements >> (8.U - multiplier), elements << multiplier) - } - - val vlmax = maximum(sew, vtype(2, 0)) - val registers: Instance[Operands] = Instantiate(new Operands(b)) - registers.io.initialize := io.initialize - val writes = Wire(chiselTypeOf(registers.io.write.bits)) - val pendingWrites = Reg(chiselTypeOf(registers.io.write.bits)) - for (lane <- 0 until p.laneNumber) { - writes(lane) := 0.U.asTypeOf(writes(lane)) - } - registers.io.write.valid := state === commit && !writeSent && !io.initialize - registers.io.write.bits := pendingWrites - registers.io.writeDone.ready := state === commit && writeSent && !io.initialize - when(registers.io.write.fire)(writeSent := true.B) - when(registers.io.writeDone.fire) { - writeSent := false.B - state := resumeState - } - - def writeElement( - lane: Int, - index: UInt, - width: UInt, - data: UInt, - maskBit: Bool = false.B - ): Unit = { - val bitOffset = Mux(maskBit, index, index << (width.pad(3) + 3.U)) - val within = bitOffset.pad(log2Ceil(p.vLen))(log2Ceil(p.vLen) - 1, 0) - val mask = Mux( - maskBit, - 1.U(p.wordBits.W), - MuxLookup(width, Fill(p.wordBits, 1.U(1.W)))( - Seq(0.U -> 255.U, 1.U -> 65535.U, 2.U -> "hffffffff".U) - ) - ) - writes(lane).valid := true.B - writes(lane).address := wordAddress(rd, index, width, maskBit) - writes(lane).mask := (mask << within(p.wordOffsetBits - 1, 0))(p.wordBits - 1, 0) - writes(lane).data := (data << within(p.wordOffsetBits - 1, 0))(p.wordBits - 1, 0) - } - - def finish( - fault: Bool = false.B, - cause: UInt = 0.U, - value: UInt = 0.U - ): Unit = { - finishRequested := true.B - state := idle - resultValid := true.B - result.fault := fault - result.cause := cause - result.tval := value - when(!fault)(vstart := 0.U) - } - - def nextBatch(step: UInt): Unit = { - when(base + step >= limit)(finish()).otherwise { - base := base + step - state := capture - readPhase := 0.U - } - offered.foreach(_ := false.B) - issued.foreach(_ := false.B) - received.foreach(_ := false.B) - memoryFailed.foreach(_ := false.B) - divBatchValid := false.B - divStarted.foreach(_ := false.B) - divDone.foreach(_ := false.B) - fpStarted.foreach(_ := false.B) - fpDone.foreach(_ := false.B) - } - - io.vl := vl - io.vtype := vtype - io.vstart := vstart - io.busy := state =/= idle || resultValid || registers.io.busy - io.issue.ready := state === idle && !resultValid && !registers.io.busy && !io.initialize - io.result.valid := resultValid && state === idle && !registers.io.busy - io.result.bits := result - when(io.result.fire)(resultValid := false.B) - when(io.vstartWrite.valid)(vstart := io.vstartWrite.bits) - when(io.issue.fire) { - instruction := io.issue.bits.instruction - scalar1 := io.issue.bits.scalar1 - scalar2 := io.issue.bits.scalar2 - floatScalar := io.issue.bits.floating - state := snapshot - readPhase := 0.U - readSent := false.B - snapshotSent := false.B - firstBatch := true.B - writeSent := false.B - result := 0.U.asTypeOf(new VectorResult) - base := vstart - count := 0.U - offered.foreach(_ := false.B) - issued.foreach(_ := false.B) - received.foreach(_ := false.B) - memoryFailed.foreach(_ := false.B) - divBatchValid := false.B - divStarted.foreach(_ := false.B) - divDone.foreach(_ := false.B) - fpStarted.foreach(_ := false.B) - fpDone.foreach(_ := false.B) - } - - val instructionLegal = Wire(Bool()) - val alu: Seq[Instance[IALU]] = Seq.fill(p.laneNumber)(Instantiate(new IALU(p.eLen))) - val falu: Seq[Instance[FALU]] = Seq.fill(p.laneNumber)(Instantiate(new FALU(p.eLen))) - val laneA = Wire(Vec(p.laneNumber, UInt(p.wordBits.W))) - val laneB = Wire(Vec(p.laneNumber, UInt(p.wordBits.W))) - val laneSelected = Wire(Vec(p.laneNumber, Bool())) - val laneValue = Wire(Vec(p.laneNumber, UInt(p.wordBits.W))) - val laneFlags = Wire(Vec(p.laneNumber, UInt(5.W))) - val laneLegal = Wire(Vec(p.laneNumber, Bool())) - val fpComplete = Wire(Vec(p.laneNumber, Bool())) - val gather = - integer && !multiply && (op === 12.U || op === 14.U && kind === 0.U) - val slide = - integer && !multiply && (op === 14.U || op === 15.U) && kind =/= 0.U - val slideOne = integer && multiply && (op === 14.U || op === 15.U) - - val operandReads = (0 until readPortCount).map { port => - val index = base + port.U - val scalar = Mux(kind === 3.U, Cat(Fill(59, rs1(4)), rs1), scalar1) - val floatingScalar = Mux(sew === 2.U && !floatScalar(63, 32).andR, "h7fc00000".U, floatScalar) - val b = - Mux( - vectorOperand, - element(operandWords(port), index, Mux(gather && op === 14.U, 1.U, sew)), - Mux(fp, floatingScalar, scalar) - ) - val slideOffset = Mux(slideOne, 1.U, Mux(kind === 3.U, rs1, scalar1)) - val sourceIndex = Mux(gather && !slide, b, Mux(op === 14.U, index - slideOffset, index + slideOffset)) - val permuteRead = gather || slide || slideOne - // element already keeps only the low group bit offset; upper index bits cannot affect its read. - val a = element( - sourceWords(port), - Mux(scalarMoveOut, 0.U, Mux(permuteRead, sourceIndex(31, 0), index)), - Mux(permuteRead, sew, sourceSew) - ) - val c = element(destinationWords(port), index, Mux(fp, sew, destinationSew)) - (a, b, c, sourceIndex) - } - - val permuteRead = gather || slide || slideOne - registers.io.snapshot.valid := state === snapshot && permuteRead && !snapshotSent && !io.initialize - registers.io.snapshot.bits := rs2 - registers.io.snapshotDone.ready := state === snapshot && snapshotSent && !io.initialize - when(registers.io.snapshot.fire)(snapshotSent := true.B) - when(state === snapshot && !permuteRead || registers.io.snapshotDone.fire) { - state := Mux(configure, execute, capture) - readPhase := 0.U - readSent := false.B - } - - registers.io.read.valid := state === capture && !readSent && !io.initialize - registers.io.readResult.ready := state === capture && readSent && !io.initialize - for (port <- 0 until readPortCount) { - val index = base + port.U - val maskSource = maskLogical || population || first - val sourceIndex = Mux(scalarMoveOut, 0.U, Mux(permuteRead, operandReads(port)._4(31, 0), index)) - val sourceWidth = Mux(scalarMoveOut || permuteRead, sew, sourceSew) - val shift = sourceIndex << (sourceWidth.pad(3) + 3.U) - registers.io.read.bits(port).address := MuxLookup(readPhase, wordAddress(rs1, 0.U, sew))( - Seq( - 0.U -> wordAddress(0.U, index, sew, true.B), - 1.U -> wordAddress(rs1, index, Mux(gather && op === 14.U, 1.U, sew), maskLogical), - 2.U -> Mux(permuteRead, shift >> p.wordOffsetBits, wordAddress(rs2, sourceIndex, sourceWidth, maskSource)), - 3.U -> wordAddress(rd, index, Mux(fp, sew, destinationSew)) - ) - ) - registers.io.read.bits(port).snapshot := readPhase === 2.U && permuteRead - } - when(registers.io.read.fire)(readSent := true.B) - when(registers.io.readResult.fire) { - readSent := false.B - switch(readPhase) { - is(0.U)(maskWords := registers.io.readResult.bits) - is(1.U)(operandWords := registers.io.readResult.bits) - is(2.U)(sourceWords := registers.io.readResult.bits) - is(3.U)(destinationWords := registers.io.readResult.bits) - is(4.U) { - accumulator := registers.io.readResult.bits(0) & MuxLookup(sew, Fill(p.eLen, 1.U(1.W)))( - Seq(0.U -> 255.U, 1.U -> 65535.U, 2.U -> "hffffffff".U) - ) - firstBatch := false.B - } - } - when(readPhase === 4.U || readPhase === 3.U && !firstBatch) { - state := execute - }.otherwise(readPhase := readPhase + 1.U) - } - - for (lane <- 0 until p.laneNumber) { - val index = base + lane.U - val selected = maskElement(maskWords(lane), index) - val scalar = Mux( - kind === 3.U, - Cat(Fill(59, rs1(4)), rs1), - scalar1 - ) - val floatingScalar = - Mux(sew === 2.U && !floatScalar(63, 32).andR, "h7fc00000".U, floatScalar) - val b = operandReads(lane)._2 - val slideOffset = Mux(slideOne, 1.U, Mux(kind === 3.U, rs1, scalar1)) - val up = op === 14.U - val sourceIndex = operandReads(lane)._4 - val a = operandReads(lane)._1 - laneA(lane) := a - laneB(lane) := b - laneSelected( - lane - ) := index < limit && (wholeMove || vm || selected || merge || carryOperation) && - (!slide || !up || index >= slideOffset) - alu(lane).io.divRequest.valid := state === execute && instructionLegal && divide && - divBatchValid && divActive(lane) && !divStarted(lane) && !io.initialize - alu(lane).io.divRequest.bits.a := a - alu(lane).io.divRequest.bits.b := b - alu(lane).io.divRequest.bits.sew := sew - alu(lane).io.divRequest.bits.signed := op(0) - alu(lane).io.divRequest.bits.remainder := op(1) - alu(lane).io.divResponse.ready := state === execute && divide && divBatchValid && - divStarted(lane) && !divDone(lane) && !io.initialize - alu(lane).io.clear := io.initialize - when(alu(lane).io.divRequest.fire)(divStarted(lane) := true.B) - when(alu(lane).io.divResponse.fire) { - divData(lane) := alu(lane).io.divResponse.bits - divDone(lane) := true.B - } - alu(lane).io.a := a - alu(lane).io.b := b - alu(lane).io.c := operandReads(lane)._3 - alu(lane).io.carry := (selected && (!carryMask || !vm)) || merge && vm - alu(lane).io.vxrm := io.vxrm - alu(lane).io.selector := rs1 - alu(lane).io.sew := sew - alu(lane).io.funct6 := op - alu(lane).io.multiplyClass := multiply - falu(lane).io.a := a - falu(lane).io.b := Mux(fpReduction, accumulator, b) - falu(lane).io.c := operandReads(lane)._3 - falu(lane).io.sew := sew - falu(lane).io.op := Mux( - fpReduction, - Mux(op === 5.U, 4.U, Mux(op === 7.U, 6.U, 0.U)), - op - ) - falu(lane).io.subop := rs1 - falu(lane).io.roundingMode := io.roundingMode - val fpRequired = - laneSelected(lane) && !merge && (!fpReduction || (lane == 0).B) - falu( - lane - ).io.start := state === execute && instructionLegal && fp && fpRequired && !fpStarted( - lane - ) && falu( - lane - ).io.ready - fpComplete(lane) := !fpRequired || fpDone(lane) || falu(lane).io.valid - laneFlags(lane) := Mux( - merge, - 0.U, - Mux(fpDone(lane), fpFlags(lane), falu(lane).io.flags) - ) - laneLegal(lane) := merge || Mux( - fpDone(lane), - fpLegal(lane), - falu(lane).io.legal - ) - val floatingValue = Mux(fpDone(lane), fpData(lane), falu(lane).io.result) - val gathered = Mux(sourceIndex < vlmax, a, 0.U) - val slid = Mux( - slideOne && Mux(up, index === 0.U, index === vl - 1.U), - scalar, - Mux(!up && sourceIndex >= vlmax, 0.U, a) - ) - val maskA = maskElement(sourceWords(lane), index) - val maskB = maskElement(operandWords(lane), index) - val maskResult = MuxLookup(op, false.B)( - Seq( - 24.U -> (maskA && !maskB), - 25.U -> (maskA && maskB), - 26.U -> (maskA || maskB), - 27.U -> (maskA ^ maskB), - 28.U -> (maskA || !maskB), - 29.U -> !(maskA && maskB), - 30.U -> !(maskA || maskB), - 31.U -> !(maskA ^ maskB) - ) - ) - laneValue(lane) := Mux( - fp, - Mux(merge, Mux(vm || selected, b, a), floatingValue), - Mux( - wholeMove, - a, - Mux( - maskLogical, - maskResult, - Mux( - identity, - index, - Mux( - slide || slideOne, - slid, - Mux(gather, gathered, alu(lane).io.result) - ) - ) - ) - ) - ) - when(falu(lane).io.start)(fpStarted(lane) := true.B) - when(state === execute && falu(lane).io.valid) { - fpDone(lane) := true.B - fpData(lane) := falu(lane).io.result - fpFlags(lane) := falu(lane).io.flags - fpLegal(lane) := falu(lane).io.legal - } - } - - val destinationGroupLegal = - compare || maskLogical || population || first || reduction || scalarMoveOut || scalarMoveIn || - groupLegal(rd, destinationSew) - - val sourceGroupLegal = - maskLogical || population || first || scalarMoveIn || scalarMoveOut || groupLegal( - rs2, - sourceSew - ) - - val operandGroupLegal = - !vectorOperand || op === 18.U || reduction || scalarMoveOut || maskLogical || population || first || - groupLegal(rs1, sew) - - val aluLegal = alu - .map(_.io.legal) - .reduce(_ && _) || maskLogical || identity || gather || slide || slideOne || - scalarMoveOut || scalarMoveIn || population || first || intReduction || wholeMove - - val fpOperationLegal = Seq(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 16, 18, 19, 23, 24, 25, 27, 28, 29, 31, 32, 33, 36, 39, - 40, 41, 42, 43, 44, 45, 46, 47) - .map(n => op === n.U) - .reduce(_ || _) - - val ordinaryLegal = - !vtype( - 63 - ) && sourceSew <= p.maxSew.U && destinationSew <= p.maxSew.U && destinationGroupLegal && sourceGroupLegal && operandGroupLegal && - (!(intWide || floatWide || intNarrow || floatNarrow || extension) || - overlapLegal(rd, destinationSew, rs2, sourceSew) && - (!vectorOperand || op === 18.U || overlapLegal( - rd, - destinationSew, - rs1, - sew - ))) && - (vm || rd =/= 0.U || compare || maskLogical || population || first || reduction) && - (!merge || !vm || rs2 === 0.U) && (!maskLogical || vm) && - (!carryOperation || carryMask || !vm) && - (!(reduction || population || first) || vstart === 0.U) && - (!fp || sew >= 2.U && fpOperationLegal && io.roundingMode <= 4.U && - (scalarMoveOut || scalarMoveIn || merge || falu(0).io.legal) && - (op =/= 16.U || scalarMoveOut || scalarMoveIn) && - (op =/= 18.U && op =/= 19.U || kind === 1.U) && (!merge || kind === 5.U)) && - (!integer || aluLegal) - - val wholeMoveLegal = vm && vstart === 0.U && Seq(0, 1, 3, 7) - .map(n => rs1 === n.U) - .reduce(_ || _) && - (rd & rs1) === 0.U && (rs2 & rs1) === 0.U && rd +& rs1 < 32.U && rs2 +& rs1 < 32.U - - val memoryLegal = memWidth <= p.maxSew.U && memSew <= p.maxSew.U && Seq( - 0, - 5, - 6, - 7 - ).map(n => kind === n.U).reduce(_ || _) && !instruction(28) && - Mux( - wholeMemory, - vm && Seq(0, 1, 3, 7) - .map(n => instruction(31, 29) === n.U) - .reduce(_ || _) && - (rd & instruction(31, 29)) === 0.U && rd +& instruction( - 31, - 29 - ) < 32.U && (!store || memWidth === 0.U), - !vtype(63) && instruction( - 31, - 29 - ) === 0.U && (memMode =/= 0.U || rs2 === 0.U) && - groupLegal(rd, memSew) && (!indexed || groupLegal( - rs2, - memWidth - )) && (vm || store || rd =/= 0.U) - ) - - val legal = Mux( - memory, - memoryLegal, - Mux(wholeMove, wholeMoveLegal, (integer || fp) && ordinaryLegal) - ) - - instructionLegal := legal - - val required = Wire(Vec(p.memoryPorts, Bool())) - val addresses = Wire(Vec(p.memoryPorts, UInt(64.W))) - val canceled = Wire(Vec(p.memoryPorts, Bool())) - for (port <- 0 until p.memoryPorts) { - val index = base + port.U - required( - port - ) := (port < p.laneNumber).B && index < limit && (wholeMemory || vm || maskElement( - maskWords(port), - index - )) - addresses(port) := scalar1 + Mux( - indexed && !wholeMemory, - operandReads(port)._1, - Mux(memMode === 2.U, index * scalar2, index << memSew) - ) - val earlierFailure = (0 until port) - .map { prior => - required(prior) && (memoryFailed(prior) || io - .memoryResponse(prior) - .fire && io.memoryResponse(prior).bits.error) - } - .foldLeft(false.B)(_ || _) - val earlierPending = (0 until port) - .map { prior => - val distance = addresses(port) - addresses(prior) - val reverseDistance = addresses(prior) - addresses(port) - val collision = - store && (distance < (1.U << memSew) || reverseDistance < (1.U << memSew)) - required(prior) && (memMode === 3.U || collision) && !received(prior) - } - .foldLeft(false.B)(_ || _) - canceled(port) := !issued(port) && !offered(port) && earlierFailure - io.memoryRequest(port) - .valid := state === execute && memory && legal && required( - port - ) && !issued( - port - ) && !earlierPending && !canceled(port) - io.memoryRequest(port).bits.address := addresses(port) - io.memoryRequest(port).bits.write := store - io.memoryRequest(port).bits.data := operandReads(port)._3 - io.memoryRequest(port).bits.mask := MuxLookup(memSew, 255.U(8.W))( - Seq(0.U -> 1.U, 1.U -> 3.U, 2.U -> 15.U) - ) - io.memoryRequest(port).bits.size := memSew - io.memoryResponse(port).ready := state === execute && memory && issued( - port - ) && !received(port) - when(io.memoryRequest(port).valid)(offered(port) := true.B) - when(io.memoryRequest(port).fire)(issued(port) := true.B) - when(io.memoryResponse(port).fire) { - received(port) := true.B - memoryData(port) := io.memoryResponse(port).bits.data - memoryFailed(port) := io.memoryResponse(port).bits.error - } - } - - val memoryComplete = (0 until p.memoryPorts) - .map(i => !required(i) || canceled(i) || received(i) || io.memoryResponse(i).fire) - .reduce(_ && _) - - val failures = VecInit( - (0 until p.memoryPorts).map(i => - required(i) && (memoryFailed(i) || io - .memoryResponse(i) - .fire && io.memoryResponse(i).bits.error) - ) - ) - - val failureIndex = PriorityEncoder(failures.asUInt) - - when(state === execute) { - when(configure) { - val immediate = instruction(31, 30) === 3.U - val registerType = instruction(31, 25) === 64.U - val requested = Mux( - registerType, - scalar2, - Mux(immediate, instruction(29, 20), instruction(30, 20)) - ) - val requestedSew = requested(5, 3) - val requestedLmul = requested(2, 0) - val typeLegal = - requested(63, 8) === 0.U && requestedSew <= p.maxSew.U && requestedLmul =/= 4.U && - (!requestedLmul( - 2 - ) || (8.U << requestedSew) <= (p.eLen.U >> (8.U - requestedLmul))) - val newMaximum = maximum(requestedSew, requestedLmul) - val keepLength = !immediate && rs1 === 0.U && rd === 0.U - when( - !(immediate || registerType || !instruction( - 31 - )) || typeLegal && keepLength && newMaximum =/= vlmax - ) { - finish(true.B, 2.U, instruction) - }.otherwise { - val avl = Mux( - immediate, - rs1, - Mux(rs1 =/= 0.U, scalar1, Mux(rd =/= 0.U, newMaximum, vl)) - ) - val length = Mux(typeLegal, Mux(avl < newMaximum, avl, newMaximum), 0.U) - vl := length - vtype := Mux(typeLegal, requested, "h8000000000000000".U) - result.scalarWrite := rd =/= 0.U - result.scalarData := length - finish() - } - }.elsewhen(!legal) { - finish(true.B, 2.U, instruction) - }.elsewhen(scalarMoveOut) { - val value = element(sourceWords(0), 0.U, sew) - when(fp) { - result.floatWrite := true.B - result.floatData := Mux( - sew === 2.U, - Cat("hffffffff".U(32.W), value(31, 0)), - value - ) - }.otherwise { - result.scalarWrite := rd =/= 0.U - result.scalarData := MuxLookup(sew, value)( - Seq( - 0.U -> Cat(Fill(56, value(7)), value(7, 0)), - 1.U -> Cat(Fill(48, value(15)), value(15, 0)), - 2.U -> Cat(Fill(32, value(31)), value(31, 0)) - ) - ) - } - finish() - }.elsewhen(scalarMoveIn) { - when(vl =/= 0.U && vstart < vl) { - val scalar = Mux( - sew === 2.U && !floatScalar(63, 32).andR, - "h7fc00000".U, - floatScalar - ) - writeElement( - 0, - 0.U, - sew, - Mux(fp, scalar, scalar1) - ) - } - finish() - }.elsewhen(base >= limit) { - when(population || first) { - result.scalarWrite := rd =/= 0.U - result.scalarData := Mux(first, "hffffffffffffffff".U, count) - } - finish() - }.elsewhen(memory) { - when(memoryComplete) { - for (lane <- 0 until p.laneNumber) { - when( - load && required( - lane - ) && (!failures.asUInt.orR || lane.U < failureIndex) - ) { - writeElement( - lane, - base + lane.U, - memSew, - Mux( - io.memoryResponse(lane).fire, - io.memoryResponse(lane).bits.data, - memoryData(lane) - ) - ) - } - } - when(failures.asUInt.orR) { - vstart := base + failureIndex - finish(true.B, Mux(load, 5.U, 7.U), addresses(failureIndex)) - }.otherwise(nextBatch(p.laneNumber.U)) - } - }.elsewhen(population || first) { - val selected = VecInit((0 until p.laneNumber).map { lane => - val index = base + lane.U - index < vl && maskElement(sourceWords(lane), index) && - (vm || maskElement(maskWords(lane), index)) - }) - val total = count + PopCount(selected) - count := total - when(first && selected.asUInt.orR || base + p.laneNumber.U >= vl) { - result.scalarWrite := rd =/= 0.U - result.scalarData := Mux( - first, - Mux( - selected.asUInt.orR, - base + PriorityEncoder(selected.asUInt), - "hffffffff".U - ), - total - ) - finish() - }.otherwise { - base := base + p.laneNumber.U - readPhase := 0.U - state := capture - } - }.elsewhen(intReduction) { - // All integer reductions are associative at the destination width. Keep - // the original leaf order and ignore masked lanes instead of serializing them. - def combine(left: (Bool, UInt), right: (Bool, UInt)): (Bool, UInt) = { - val (leftValid, a) = left - val (rightValid, b) = right - val width = 8.U(7.W) << sew - val sa = (a << (p.eLen.U - width))(p.eLen - 1, 0).asSInt - val sb = (b << (p.eLen.U - width))(p.eLen - 1, 0).asSInt - val value = MuxLookup(op, a + b)(Seq( - 1.U -> (a & b), - 2.U -> (a | b), - 3.U -> (a ^ b), - 4.U -> Mux(b < a, b, a), - 5.U -> Mux(sb < sa, b, a), - 6.U -> Mux(b > a, b, a), - 7.U -> Mux(sb > sa, b, a) - )) - (leftValid || rightValid, Mux(!leftValid, b, Mux(!rightValid, a, value))) - } - def tree(values: Seq[(Bool, UInt)]): (Bool, UInt) = { - if (values.size == 1) values.head - else { - val (left, right) = values.splitAt(values.size / 2) - combine(tree(left), tree(right)) - } - } - val reduced = tree(Seq(true.B -> accumulator) ++ - (0 until p.laneNumber).map(lane => laneSelected(lane) -> laneA(lane)))._2 - accumulator := reduced - when(base + p.laneNumber.U >= vl)(writeElement(0, 0.U, sew, reduced)) - nextBatch(p.laneNumber.U) - }.elsewhen(fpReduction) { - when(fpComplete(0)) { - when(laneSelected(0) && !laneLegal(0))(finish(true.B, 2.U, instruction)) - .otherwise { - val value = Mux(laneSelected(0), laneValue(0), accumulator) - accumulator := value - when(laneSelected(0))(result.flags := result.flags | laneFlags(0)) - when(base + 1.U >= vl)(writeElement(0, 0.U, sew, value)) - nextBatch(1.U) - } - } - }.elsewhen(divide) { - when(!io.initialize) { - when(!divBatchValid) { - divActive := laneSelected - divBatchValid := true.B - }.elsewhen((0 until p.laneNumber).map(i => !divActive(i) || divDone(i)).reduce(_ && _)) { - for (lane <- 0 until p.laneNumber) { - when(divActive(lane)) { - writeElement(lane, base + lane.U, destinationSew, divData(lane)) - } - } - nextBatch(p.laneNumber.U) - } - } - }.elsewhen(!fp || fpComplete.reduce(_ && _)) { - val floatingLegal = (0 until p.laneNumber) - .map(i => !laneSelected(i) || laneLegal(i)) - .reduce(_ && _) - when(fp && !floatingLegal)(finish(true.B, 2.U, instruction)).otherwise { - for (lane <- 0 until p.laneNumber) { - when(laneSelected(lane)) { - writeElement( - lane, - base + lane.U, - destinationSew, - laneValue(lane), - compare || maskLogical - ) - } - } - when(fp) { - result.flags := result.flags | (0 until p.laneNumber) - .map(i => Mux(laneSelected(i), laneFlags(i), 0.U)) - .reduce(_ | _) - } - when(integer && !wholeMove) { - result.saturated := result.saturated || (0 until p.laneNumber) - .map(i => laneSelected(i) && alu(i).io.saturated) - .reduce(_ || _) - } - nextBatch(p.laneNumber.U) - } - } - } - when(state === execute && writes.map(_.valid).reduce(_ || _) && !io.initialize) { - pendingWrites := writes - resumeState := Mux(finishRequested, idle, capture) - state := commit - writeSent := false.B - } - when(io.initialize) { - state := idle - readSent := false.B - snapshotSent := false.B - writeSent := false.B - firstBatch := true.B - readPhase := 0.U - resultValid := false.B - vl := 0.U - vtype := "h8000000000000000".U - vstart := 0.U - divBatchValid := false.B - divStarted.foreach(_ := false.B) - divDone.foreach(_ := false.B) - fpStarted.foreach(_ := false.B) - fpDone.foreach(_ := false.B) - offered.foreach(_ := false.B) - issued.foreach(_ := false.B) - received.foreach(_ := false.B) - } -} diff --git a/arch/src/main/scala/framework/rvv/src/main/scala/interfaces/KernelBlinkIO.scala b/arch/src/main/scala/framework/rvv/src/main/scala/interfaces/KernelBlinkIO.scala deleted file mode 100644 index b7d38a14c..000000000 --- a/arch/src/main/scala/framework/rvv/src/main/scala/interfaces/KernelBlinkIO.scala +++ /dev/null @@ -1,41 +0,0 @@ -package framework.rvv - -import chisel3._ -import chisel3.util._ -import framework.balldomain.blink.BlinkPortIO -import framework.top.GlobalConfig - -class KernelBlinkIO(b: GlobalConfig) - extends BlinkPortIO(b, b.rvv.memoryPorts, b.rvv.memoryPorts, new KernelRequest(b), new KernelResponse(b)) { - // Images contain instructions and read-only constants; mutable globals are - // rejected. Temporary state stays inside the kernel; data outputs use banks. - val ballRequest = Decoupled(new RvvBallCommand) - val ballResponse = Flipped(Decoupled(UInt(64.W))) - val image = Flipped(Decoupled(UInt(32.W))) - val imageTerminal = Flipped(Decoupled(new ImageTerminal)) - // Sample diagnostics with cmdResp; no separate completion handshake. - val result = Output(new KernelCompletion) -} - -class KernelCommand extends Bundle { - val funct7 = UInt(7.W) - val rs1 = UInt(64.W) - val rs2 = UInt(64.W) -} - -class KernelRequest(b: GlobalConfig) extends Bundle { - val cmd = new KernelCommand - val rob_id = UInt(log2Up(b.frontend.rob_entries).W) - val read_groups = UInt(b.memDomain.groupCountWidth.W) - val write_groups = UInt(b.memDomain.groupCountWidth.W) -} - -class KernelResponse(b: GlobalConfig) extends Bundle { - val rob_id = UInt(log2Up(b.frontend.rob_entries).W) - val write_bank = UInt(16.W) -} - -class ImageTerminal extends Bundle { - val error = UInt(4.W) - val address = UInt(64.W) -} diff --git a/arch/src/main/scala/framework/rvv/src/main/scala/interfaces/RvvInterface.scala b/arch/src/main/scala/framework/rvv/src/main/scala/interfaces/RvvInterface.scala deleted file mode 100644 index 6411b07ef..000000000 --- a/arch/src/main/scala/framework/rvv/src/main/scala/interfaces/RvvInterface.scala +++ /dev/null @@ -1,88 +0,0 @@ -package framework.rvv - -import chisel3._ - -class ProgramWrite extends Bundle { - val buffer = Bool() - val first = Bool() - val address = UInt(32.W) - val data = UInt(32.W) -} - -class KernelLaunch extends Bundle { - val iBuffer = Bool() - val entry = UInt(32.W) - val end = UInt(32.W) - val stack = UInt(64.W) - val args = Vec(8, UInt(64.W)) -} - -class KernelCompletion extends Bundle { - val fault = Bool() - val pc = UInt(32.W) - val instruction = UInt(32.W) - val cycles = UInt(64.W) - val cause = UInt(32.W) - val tval = UInt(64.W) - val fflags = UInt(5.W) - val vxsat = Bool() -} - -class VectorMemoryRequest extends Bundle { - val address = UInt(64.W) - val write = Bool() - val data = UInt(64.W) - val mask = UInt(8.W) - val size = UInt(2.W) -} - -class VectorMemoryResponse extends Bundle { - val data = UInt(64.W) - val error = Bool() -} - -class VectorIssue extends Bundle { - val instruction = UInt(32.W) - val scalar1 = UInt(64.W) - val scalar2 = UInt(64.W) - val floating = UInt(64.W) -} - -class VectorResult extends Bundle { - val fault = Bool() - val cause = UInt(32.W) - val tval = UInt(64.W) - val scalarWrite = Bool() - val scalarData = UInt(64.W) - val floatWrite = Bool() - val floatData = UInt(64.W) - val flags = UInt(5.W) - val saturated = Bool() -} - -class ImageLoad extends Bundle { - val buffer = Bool() - val bytes = UInt(32.W) -} - -class ImageResult extends Bundle { - val fault = Bool() - val cause = UInt(32.W) - val tval = UInt(32.W) - val entry = UInt(32.W) - val textBytes = UInt(32.W) - val constBytes = UInt(32.W) -} - -class RvvBallCommand extends Bundle { - val funct7 = UInt(7.W) - val rs1 = UInt(64.W) - val rs2 = UInt(64.W) -} - -class BankLayout(b: framework.top.GlobalConfig) extends Bundle { - val readBank = UInt(b.memDomain.vbankIdWidth.W) - val writeBank = UInt(b.memDomain.vbankIdWidth.W) - val readGroups = UInt(b.memDomain.groupCountWidth.W) - val writeGroups = UInt(b.memDomain.groupCountWidth.W) -} diff --git a/arch/src/main/scala/framework/rvv/src/main/scala/memory/BankPort.scala b/arch/src/main/scala/framework/rvv/src/main/scala/memory/BankPort.scala deleted file mode 100644 index dd9f4d84e..000000000 --- a/arch/src/main/scala/framework/rvv/src/main/scala/memory/BankPort.scala +++ /dev/null @@ -1,109 +0,0 @@ -package framework.rvv - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.balldomain.blink.{BankRead, BankWrite} -import framework.top.GlobalConfig - -@instantiable -class BankPort(val b: GlobalConfig) extends Module { - val lineBytes = b.memDomain.bankWidth / 8 - val offsetBits = log2Ceil(lineBytes) - require(b.memDomain.bankWidth >= 64 && isPow2(b.memDomain.bankWidth)) - require(lineBytes <= 65536) - require(b.memDomain.bankMaskLen == lineBytes) - - @public - val io = IO(new Bundle { - val request = Flipped(Decoupled(new VectorMemoryRequest)) - val response = Decoupled(new VectorMemoryResponse) - val read = Flipped(new BankRead(b)) - val write = Flipped(new BankWrite(b)) - val readBank = Input(UInt(b.memDomain.vbankIdWidth.W)) - val writeBank = Input(UInt(b.memDomain.vbankIdWidth.W)) - val readGroups = Input(UInt(b.memDomain.groupCountWidth.W)) - val writeGroups = Input(UInt(b.memDomain.groupCountWidth.W)) - val robId = Input(UInt(log2Up(b.frontend.rob_entries).W)) - }) - - val idle :: readRequest :: readResponse :: writeRequest :: writeResponse :: complete :: Nil = Enum(6) - val state = RegInit(idle) - val request = Reg(new VectorMemoryRequest) - val robId = Reg(UInt(log2Up(b.frontend.rob_entries).W)) - val consumed = RegInit(0.U(4.W)) - val result = RegInit(0.U(64.W)) - val error = RegInit(false.B) - - val byteCount = (1.U(4.W) << request.size)(3, 0) - val byteAddress = request.address(31, 0) +& consumed - val lineOffset = byteAddress(offsetBits - 1, 0) - val remaining = byteCount - consumed - val lineRemaining = lineBytes.U - lineOffset - val chunk = Mux(remaining <= lineRemaining, remaining, lineRemaining)(3, 0) - val nextConsumed = consumed + chunk - val lineAddress = byteAddress >> offsetBits - val readShifted = io.read.io.resp.bits.data >> (lineOffset << 3) - val chunkMask = ((1.U(65.W) << (chunk << 3)) - 1.U)(63, 0) - val readChunk = readShifted.pad(64)(63, 0) & chunkMask - - io.request.ready := state === idle - io.response.valid := state === complete - io.response.bits.data := result - io.response.bits.error := error - val localBank = request.address(63, 32) - val fromRead = localBank < io.readGroups - val parentBank = Mux(fromRead, io.readBank, io.writeBank) - val parentGroup = Mux(fromRead, localBank, localBank - io.readGroups) - io.read.bank_id := parentBank - io.read.rob_id := robId - io.read.ball_id := 0.U - io.read.group_id := parentGroup - io.read.io.req.valid := state === readRequest - io.read.io.req.bits.addr := lineAddress - io.read.io.resp.ready := state === readResponse - io.write.bank_id := parentBank - io.write.rob_id := robId - io.write.ball_id := 0.U - io.write.group_id := parentGroup - io.write.io.req.valid := state === writeRequest - io.write.io.req.bits.addr := lineAddress - io.write.io.req.bits.data := (request.data >> (consumed << 3)).pad(b.memDomain.bankWidth) << (lineOffset << 3) - io.write.io.resp.ready := state === writeResponse - for (i <- 0 until lineBytes) { - val sourceByte = consumed + i.U - lineOffset - io.write.io.req.bits.mask(i) := i.U >= lineOffset && i.U < lineOffset +& chunk && request.mask(sourceByte(2, 0)) - } - - when(io.request.fire) { - request := io.request.bits - robId := io.robId - consumed := 0.U - result := 0.U - val end = io.request.bits.address(31, 0) +& (1.U(33.W) << io.request.bits.size) - val invalid = io.request.bits.address(63, 32) >= (io.readGroups +& io.writeGroups) || - (io.request.bits.write && io.request.bits.address(63, 32) < io.readGroups) || - end > (BigInt(b.memDomain.bankEntries) * lineBytes).U - error := invalid - state := Mux(invalid, complete, Mux(io.request.bits.write, writeRequest, readRequest)) - } - when(io.read.io.req.fire) { - state := readResponse - } - when(io.read.io.resp.fire) { - result := result | (readChunk << (consumed << 3)) - consumed := nextConsumed - state := Mux(nextConsumed === byteCount, complete, readRequest) - } - when(io.write.io.req.fire) { - state := writeResponse - } - when(io.write.io.resp.fire) { - consumed := nextConsumed - error := !io.write.io.resp.bits.ok - state := Mux(!io.write.io.resp.bits.ok || nextConsumed === byteCount, complete, writeRequest) - } - when(io.response.fire) { - state := idle - } -} diff --git a/arch/src/main/scala/framework/rvv/src/main/scala/top/Execution.scala b/arch/src/main/scala/framework/rvv/src/main/scala/top/Execution.scala deleted file mode 100644 index 0a35e7296..000000000 --- a/arch/src/main/scala/framework/rvv/src/main/scala/top/Execution.scala +++ /dev/null @@ -1,354 +0,0 @@ -package framework.rvv - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.top.GlobalConfig - -@instantiable -class Execution(val b: GlobalConfig) extends Module { - val p = b.rvv - - @public - val io = IO(new Bundle { - val program = Flipped(Decoupled(new ProgramWrite)) - val launch = Flipped(Decoupled(new KernelLaunch)) - val done = Decoupled(new KernelCompletion) - val busy = Output(Bool()) - val ballRequest = Decoupled(new RvvBallCommand) - val ballResponse = Flipped(Decoupled(UInt(64.W))) - val memoryRequest = Vec(p.memoryPorts, Decoupled(new VectorMemoryRequest)) - val memoryResponse = Vec(p.memoryPorts, Flipped(Decoupled(new VectorMemoryResponse))) - }) - - val iBuf: Seq[Instance[IBuf]] = Seq.fill(2)(Instantiate(new IBuf(b))) - val registers: Instance[ScalarRF] = Instantiate(new ScalarRF) - val scalar: Instance[ScalarExecution] = Instantiate(new ScalarExecution) - val floating: Instance[ScalarFloating] = Instantiate(new ScalarFloating(p.eLen)) - val vector: Instance[VectorCore] = Instantiate(new VectorCore(b)) - - val idle :: fetch :: fetchWait :: execute :: vectorWait :: scalarWait :: floatWait :: memorySend :: memoryWait :: ballSend :: ballWait :: completed :: Nil = - Enum(12) - - val state = RegInit(idle) - val activeBuffer = RegInit(false.B) - val end = Reg(UInt(32.W)) - val pc = Reg(UInt(32.W)) - val instruction = RegInit(0.U(32.W)) - val cycles = RegInit(0.U(64.W)) - val fault = RegInit(false.B) - val cause = RegInit(0.U(32.W)) - val tval = RegInit(0.U(64.W)) - val fflags = RegInit(0.U(5.W)) - val frm = RegInit(0.U(3.W)) - val vxrm = RegInit(0.U(2.W)) - val vxsat = RegInit(false.B) - val scalarTarget = Reg(UInt(5.W)) - val scalarNextPc = Reg(UInt(32.W)) - - val scalarDivide = (instruction(6, 0) === "h33".U || instruction(6, 0) === "h3b".U) && instruction( - 31, - 25 - ) === 1.U && instruction(14, 12) >= 4.U - - val memory = Reg(new VectorMemoryRequest) - val memoryFloating = Reg(Bool()) - val memoryUnsigned = Reg(Bool()) - - def finish(isFault: Bool, code: UInt, value: UInt): Unit = { - state := completed - fault := isFault - cause := code - tval := value - } - - io.ballRequest.valid := state === ballSend - io.ballRequest.bits.funct7 := instruction(31, 25) - io.ballRequest.bits.rs1 := registers.io.xReadData1 - io.ballRequest.bits.rs2 := registers.io.xReadData2 - io.ballResponse.ready := state === ballWait - when(io.ballRequest.fire)(state := ballWait) - - val running = state =/= idle && state =/= completed - io.busy := state =/= idle - io.launch.ready := state === idle - io.done.valid := state === completed - io.done.bits.fault := fault - io.done.bits.pc := pc - io.done.bits.instruction := instruction - io.done.bits.cycles := cycles - io.done.bits.cause := cause - io.done.bits.tval := tval - io.done.bits.fflags := fflags - io.done.bits.vxsat := vxsat - - for (i <- 0 until 2) { - iBuf(i).io.upload.valid := io.program.valid && io.program.bits.buffer === i.U - iBuf(i).io.upload.bits := io.program.bits - iBuf(i).io.uploadEnabled := !io.launch.valid && (!running || activeBuffer =/= i.U) - iBuf(i).io.fetchEnabled := state === fetch && activeBuffer === i.U && pc < end - iBuf(i).io.fetchAddress := pc - } - io.program.ready := Mux(io.program.bits.buffer, iBuf(1).io.upload.ready, iBuf(0).io.upload.ready) - val imageWords = Mux(io.launch.bits.iBuffer, iBuf(1).io.wordCount, iBuf(0).io.wordCount) - val fetchLoaded = Mux(activeBuffer, iBuf(1).io.loaded, iBuf(0).io.loaded) - val fetchedInstruction = Mux(activeBuffer, iBuf(1).io.instruction, iBuf(0).io.instruction) - - registers.io.initialize.valid := io.launch.fire - registers.io.initialize.bits := io.launch.bits - registers.io.xReadAddress1 := instruction(19, 15) - registers.io.xReadAddress2 := instruction(24, 20) - registers.io.fReadAddress1 := instruction(19, 15) - registers.io.fReadAddress2 := instruction(24, 20) - registers.io.fReadAddress3 := instruction(31, 27) - registers.io.xWrite.valid := false.B - registers.io.xWrite.bits.address := instruction(11, 7) - registers.io.xWrite.bits.data := 0.U - registers.io.fWrite.valid := false.B - registers.io.fWrite.bits.address := instruction(11, 7) - registers.io.fWrite.bits.data := 0.U - scalar.io.divRequest.valid := false.B - scalar.io.divRequest.bits.a := registers.io.xReadData1 - scalar.io.divRequest.bits.b := registers.io.xReadData2 - scalar.io.divRequest.bits.sew := Mux(instruction(6, 0) === "h3b".U, 2.U, 3.U) - scalar.io.divRequest.bits.signed := !instruction(12) - scalar.io.divRequest.bits.remainder := instruction(13) - scalar.io.divResponse.ready := state === scalarWait && !io.launch.fire - scalar.io.clear := io.launch.fire - scalar.io.instruction := instruction - scalar.io.pc := pc - scalar.io.source1 := registers.io.xReadData1 - scalar.io.source2 := registers.io.xReadData2 - floating.io.instruction := instruction - floating.io.source1 := registers.io.fReadData1 - floating.io.source2 := registers.io.fReadData2 - floating.io.source3 := registers.io.fReadData3 - floating.io.xSource := registers.io.xReadData1 - floating.io.frm := frm - floating.io.start := false.B - vector.io.initialize := false.B - vector.io.roundingMode := frm - vector.io.vxrm := vxrm - vector.io.vstartWrite.valid := false.B - vector.io.vstartWrite.bits := 0.U - vector.io.issue.valid := false.B - vector.io.issue.bits.instruction := instruction - vector.io.issue.bits.scalar1 := registers.io.xReadData1 - vector.io.issue.bits.scalar2 := registers.io.xReadData2 - vector.io.issue.bits.floating := registers.io.fReadData1 - vector.io.result.ready := state === vectorWait - - for (i <- 0 until p.memoryPorts) { - io.memoryRequest(i) <> vector.io.memoryRequest(i) - vector.io.memoryResponse(i) <> io.memoryResponse(i) - } - when(io.ballResponse.fire) { - registers.io.xWrite.valid := instruction(11, 7) =/= 0.U - registers.io.xWrite.bits.data := io.ballResponse.bits - pc := pc + 4.U - state := fetch - } - when(state === scalarWait && scalar.io.divResponse.fire) { - registers.io.xWrite.valid := true.B - registers.io.xWrite.bits.address := scalarTarget - registers.io.xWrite.bits.data := Mux( - instruction(6, 0) === "h3b".U, - Cat(Fill(32, scalar.io.divResponse.bits(31)), scalar.io.divResponse.bits(31, 0)), - scalar.io.divResponse.bits - ) - pc := scalarNextPc - state := fetch - } - when(state === memorySend || state === memoryWait) { - io.memoryRequest(0).valid := state === memorySend - io.memoryRequest(0).bits := memory - vector.io.memoryRequest(0).ready := false.B - vector.io.memoryResponse(0).valid := false.B - io.memoryResponse(0).ready := state === memoryWait - } - - val opcode = instruction(6, 0) - val funct3 = instruction(14, 12) - val vectorInstruction = opcode === "h57".U || - (opcode === "h07".U || opcode === "h27".U) && (funct3 === 0.U || funct3 >= 5.U) - val floatingInstruction = opcode === "h53".U || opcode === "h43".U || opcode === "h47".U || - opcode === "h4b".U || opcode === "h4f".U - val load = opcode === "h03".U || opcode === "h07".U - val store = opcode === "h23".U || opcode === "h27".U - val csrAddress = instruction(31, 20) - val csrKnown = WireDefault(true.B) - val csrValue = WireDefault(0.U(64.W)) - switch(csrAddress) { - is("h001".U)(csrValue := fflags) - is("h002".U)(csrValue := frm) - is("h003".U)(csrValue := Cat(frm, fflags)) - is("h008".U)(csrValue := vector.io.vstart) - is("h009".U)(csrValue := vxsat) - is("h00a".U)(csrValue := vxrm) - is("h00f".U)(csrValue := Cat(vxrm, vxsat)) - is("hc20".U)(csrValue := vector.io.vl) - is("hc21".U)(csrValue := vector.io.vtype) - is("hc22".U)(csrValue := (p.vLen / 8).U) - } - csrKnown := Seq(0x001, 0x002, 0x003, 0x008, 0x009, 0x00a, 0x00f, 0xc20, 0xc21, 0xc22) - .map(a => csrAddress === a.U).reduce(_ || _) - val csrSource = Mux(funct3(2), instruction(19, 15), registers.io.xReadData1) - val csrWrite = funct3(1, 0) === 1.U || instruction(19, 15) =/= 0.U - - val csrNew = MuxLookup(funct3(1, 0), csrValue)(Seq( - 1.U -> csrSource, - 2.U -> (csrValue | csrSource), - 3.U -> (csrValue & ~csrSource) - )) - - when(running)(cycles := cycles + 1.U) - when(io.done.fire)(state := idle) - when(io.launch.fire) { - activeBuffer := io.launch.bits.iBuffer - pc := io.launch.bits.entry - end := io.launch.bits.end - instruction := 0.U - cycles := 0.U - fflags := 0.U - frm := 0.U - fault := false.B - cause := 0.U - tval := 0.U - state := fetch - when(io.launch.bits.entry(1, 0).orR || io.launch.bits.end(1, 0).orR) { - finish(true.B, 0.U, io.launch.bits.entry) - }.elsewhen(io.launch.bits.entry >= io.launch.bits.end || io.launch.bits.end > imageWords * 4.U) { - finish(true.B, 1.U, io.launch.bits.entry) - } - } - when(state === fetch) { - when(pc === end)(finish(false.B, 0.U, 0.U)) - .elsewhen(pc(1, 0).orR)(finish(true.B, 0.U, pc)) - .elsewhen(pc >= end || !fetchLoaded)(finish(true.B, 1.U, pc)) - .otherwise(state := fetchWait) - } - when(state === fetchWait) { instruction := fetchedInstruction; state := execute } - when(state === execute) { - when(opcode === "h7b".U && funct3 === 3.U) { - state := ballSend - }.elsewhen(vectorInstruction) { - vector.io.issue.valid := true.B - when(vector.io.issue.fire)(state := vectorWait) - }.elsewhen(floatingInstruction) { - when(!floating.io.legal)(finish(true.B, 2.U, instruction)) - .otherwise { - floating.io.start := floating.io.ready - when(floating.io.ready)(state := floatWait) - } - }.elsewhen(load || store) { - val fp = opcode === "h07".U || opcode === "h27".U - val size = Mux(fp, Mux(funct3 === 3.U, 3.U, 2.U), funct3(1, 0)) - val legal = Mux( - fp, - funct3 === 2.U || (p.eLen == 64).B && funct3 === 3.U, - Mux(store, funct3 <= 3.U, funct3 <= 6.U) - ) - val immediate = Mux( - store, - Cat(Fill(52, instruction(31)), instruction(31, 25), instruction(11, 7)), - Cat(Fill(52, instruction(31)), instruction(31, 20)) - ) - val address = registers.io.xReadData1 + immediate - memory.address := address - memory.write := store - memory.data := Mux(fp, registers.io.fReadData2, registers.io.xReadData2) - memory.size := size - memory.mask := MuxLookup(size, 1.U(8.W))(Seq(0.U -> 1.U, 1.U -> 3.U, 2.U -> 15.U, 3.U -> 255.U)) - memoryFloating := fp - memoryUnsigned := funct3(2) - when(!legal)(finish(true.B, 2.U, instruction)) - .otherwise(state := memorySend) - }.elsewhen(opcode === "h73".U) { - when(funct3 === 0.U) { - when(instruction === "h00100073".U)(finish(true.B, 3.U, pc)) - .elsewhen(instruction === "h00000073".U)(finish(true.B, 11.U, 0.U)) - .otherwise(finish(true.B, 2.U, instruction)) - }.elsewhen(funct3 === 4.U || !csrKnown || csrWrite && csrAddress(11, 10) === 3.U) { - finish(true.B, 2.U, instruction) - }.otherwise { - registers.io.xWrite.valid := true.B - registers.io.xWrite.bits.data := csrValue - when(csrWrite) { - switch(csrAddress) { - is("h001".U)(fflags := csrNew(4, 0)) - is("h002".U)(frm := csrNew(2, 0)) - is("h003".U) { frm := csrNew(7, 5); fflags := csrNew(4, 0) } - is("h008".U) { vector.io.vstartWrite.valid := true.B; vector.io.vstartWrite.bits := csrNew } - is("h009".U)(vxsat := csrNew(0)) - is("h00a".U)(vxrm := csrNew(1, 0)) - is("h00f".U) { vxrm := csrNew(2, 1); vxsat := csrNew(0) } - } - } - pc := pc + 4.U - state := fetch - } - }.elsewhen(!scalar.io.legal)(finish(true.B, 2.U, instruction)) - .elsewhen(scalar.io.nextPc(63, 32).orR)(finish(true.B, 1.U, scalar.io.nextPc)) - .elsewhen(scalar.io.nextPc(1, 0).orR)(finish(true.B, 0.U, scalar.io.nextPc)) - .elsewhen(scalarDivide) { - scalar.io.divRequest.valid := true.B - when(scalar.io.divRequest.fire) { - scalarTarget := scalar.io.destination - scalarNextPc := scalar.io.nextPc - state := scalarWait - } - }.otherwise { - registers.io.xWrite.valid := scalar.io.write - registers.io.xWrite.bits.data := scalar.io.result - pc := scalar.io.nextPc - state := fetch - } - } - when(state === memorySend && io.memoryRequest(0).fire)(state := memoryWait) - when(state === memoryWait && io.memoryResponse(0).fire) { - when(io.memoryResponse(0).bits.error)(finish(true.B, Mux(memory.write, 7.U, 5.U), memory.address)) - .otherwise { - when(!memory.write) { - val data = io.memoryResponse(0).bits.data - when(memoryFloating) { - registers.io.fWrite.valid := true.B - registers.io.fWrite.bits.data := Mux(memory.size === 2.U, Cat("hffffffff".U(32.W), data(31, 0)), data) - }.otherwise { - registers.io.xWrite.valid := true.B - registers.io.xWrite.bits.data := MuxLookup(memory.size, data)(Seq( - 0.U -> Cat(Fill(56, !memoryUnsigned && data(7)), data(7, 0)), - 1.U -> Cat(Fill(48, !memoryUnsigned && data(15)), data(15, 0)), - 2.U -> Cat(Fill(32, !memoryUnsigned && data(31)), data(31, 0)) - )) - } - } - pc := pc + 4.U - state := fetch - } - } - when((state === floatWait || state === execute && floatingInstruction) && floating.io.valid) { - when(!floating.io.legal)(finish(true.B, 2.U, instruction)) - .otherwise { - registers.io.fWrite.valid := floating.io.floatWrite - registers.io.fWrite.bits.data := floating.io.result - registers.io.xWrite.valid := floating.io.integerWrite - registers.io.xWrite.bits.data := floating.io.result - fflags := fflags | floating.io.flags - pc := pc + 4.U - state := fetch - } - } - when(vector.io.result.fire) { - when(vector.io.result.bits.fault)(finish(true.B, vector.io.result.bits.cause, vector.io.result.bits.tval)) - .otherwise { - registers.io.xWrite.valid := vector.io.result.bits.scalarWrite - registers.io.xWrite.bits.data := vector.io.result.bits.scalarData - registers.io.fWrite.valid := vector.io.result.bits.floatWrite - registers.io.fWrite.bits.data := vector.io.result.bits.floatData - fflags := fflags | vector.io.result.bits.flags - vxsat := vxsat || vector.io.result.bits.saturated - pc := pc + 4.U - state := fetch - } - } -} diff --git a/arch/src/main/scala/framework/rvv/src/main/scala/top/KernelEngine.scala b/arch/src/main/scala/framework/rvv/src/main/scala/top/KernelEngine.scala deleted file mode 100644 index 05f361cdf..000000000 --- a/arch/src/main/scala/framework/rvv/src/main/scala/top/KernelEngine.scala +++ /dev/null @@ -1,241 +0,0 @@ -package framework.rvv - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.balldomain.blink.{BallStatus, HasBallStatus} -import framework.top.GlobalConfig - -@instantiable -class KernelEngine(val b: GlobalConfig) extends Module with HasBallStatus { - val p = b.rvv - require(p.enable, "RVV must be explicitly enabled") - - @public - val io = IO(new KernelBlinkIO(b)) - def status: BallStatus = io.status - - val execution: Instance[Execution] = Instantiate(new Execution(b)) - val loader: Instance[ImageLoader] = Instantiate(new ImageLoader(b)) - val local: Instance[KernelMemory] = Instantiate(new KernelMemory(b)) - val ports: Seq[Instance[BankPort]] = Seq.fill(p.memoryPorts)(Instantiate(new BankPort(b))) - val idle :: loading :: descriptorSend :: descriptorWait :: launch :: executing :: completed :: Nil = Enum(7) - - val state = RegInit(idle) - val command = Reg(new KernelRequest(b)) - val channels = RegInit(false.B) - val validImage = RegInit(VecInit(Seq.fill(2)(false.B))) - val imageEntry = Reg(Vec(2, UInt(32.W))) - val imageBytes = Reg(Vec(2, UInt(32.W))) - val imageConstants = Reg(Vec(2, UInt(32.W))) - val buffer = Reg(Bool()) - val descriptor = Reg(Vec(12, UInt(64.W))) - val word = Reg(UInt(4.W)) - val terminalSeen = RegInit(false.B) - val terminalFailed = RegInit(false.B) - val terminalCancelled = RegInit(false.B) - val result = RegInit(0.U.asTypeOf(new KernelCompletion)) - - io.cmdReq.ready := state === idle - io.cmdResp.valid := state === completed && (command.cmd.funct7 =/= 44.U || result.fault || terminalSeen) - io.cmdResp.bits.rob_id := command.rob_id - io.cmdResp.bits.write_bank := Mux(command.cmd.funct7 === 79.U && !result.fault, command.cmd.rs1(29, 20), 0.U) - io.result := result - io.status.idle := state === idle - io.status.running := state =/= idle - when(state =/= idle && io.channelReady)(channels := true.B) - - val loadBank = io.cmdReq.bits.cmd.rs1(29, 20) - val loadBankValid = loadBank >= b.memDomain.virtualBankCount.U && loadBank < (b.memDomain.virtualBankCount + 2).U - val loadBuffer = loadBank - b.memDomain.virtualBankCount.U - loader.io.start.valid := io.cmdReq.fire && io.cmdReq.bits.cmd.funct7 === 44.U && loadBankValid - loader.io.start.bits.buffer := loadBuffer(0) - loader.io.start.bits.bytes := io.cmdReq.bits.cmd.rs1(63, 30) - loader.io.image <> io.image - io.imageTerminal.ready := state =/= idle && command.cmd.funct7 === 44.U && !terminalSeen - val terminalError = io.imageTerminal.fire && io.imageTerminal.bits.error =/= 0.U - loader.io.abort := terminalError - when(io.imageTerminal.fire) { - terminalSeen := true.B - terminalFailed := io.imageTerminal.bits.error =/= 0.U - terminalCancelled := io.imageTerminal.bits.error === 10.U - when(terminalError) { - validImage(buffer) := false.B - when(!result.fault) { - result.fault := true.B - result.cause := 5.U - result.tval := io.imageTerminal.bits.address - } - } - } - loader.io.done.ready := state === loading - io.ballRequest <> execution.io.ballRequest - execution.io.ballResponse <> io.ballResponse - execution.io.program <> loader.io.program - execution.io.launch.valid := state === launch - execution.io.launch.bits.iBuffer := buffer - execution.io.launch.bits.entry := descriptor(0)(31, 0) - execution.io.launch.bits.end := descriptor(1)(31, 0) - execution.io.launch.bits.stack := descriptor(2) - for (i <- 0 until 8) (execution.io.launch.bits.args(i) := descriptor(i + 3)) - execution.io.done.ready := state === executing - - local.io.buffer := buffer - local.io.loading := state === loading - local.io.constBytes := imageConstants(buffer) - for (i <- 0 until p.memoryPorts) { - io.bankRead(i) <> ports(i).io.read - io.bankWrite(i) <> ports(i).io.write - ports(i).io.robId := command.rob_id - ports(i).io.readBank := command.cmd.rs1(9, 0) - ports(i).io.writeBank := command.cmd.rs1(29, 20) - ports(i).io.readGroups := command.read_groups - ports(i).io.writeGroups := command.write_groups - val address = execution.io.memoryRequest(i).bits.address - val internal = address >= "h40000000".U && address < "h40002000".U - ports(i).io.request.valid := state === executing && channels && !internal && execution.io.memoryRequest(i).valid - ports(i).io.request.bits := execution.io.memoryRequest(i).bits - local.io.request(i).valid := state === executing && internal && execution.io.memoryRequest(i).valid - local.io.request(i).bits := execution.io.memoryRequest(i).bits - execution.io.memoryRequest(i).ready := state === executing && Mux( - internal, - local.io.request(i).ready, - channels && ports(i).io.request.ready - ) - execution.io.memoryResponse(i).valid := state === executing && (local.io.response(i).valid || ports( - i - ).io.response.valid) - execution.io.memoryResponse(i).bits := Mux( - local.io.response(i).valid, - local.io.response(i).bits, - ports(i).io.response.bits - ) - ports(i).io.response.ready := state === executing && execution.io.memoryResponse(i).ready - local.io.response(i).ready := state === executing && execution.io.memoryResponse(i).ready - } - loader.io.memoryRequest.ready := state === loading && local.io.request(0).ready - loader.io.memoryResponse.valid := state === loading && local.io.response(0).valid - loader.io.memoryResponse.bits := local.io.response(0).bits - when(state === loading) { - local.io.request(0).valid := loader.io.memoryRequest.valid - local.io.request(0).bits := loader.io.memoryRequest.bits - local.io.response(0).ready := loader.io.memoryResponse.ready - } - when(state === descriptorSend) { - ports(0).io.request.valid := channels - ports(0).io.request.bits.address := (command.read_groups.pad(64) << 32) + command.cmd.rs2 + word * 8.U - ports(0).io.request.bits.write := false.B - ports(0).io.request.bits.data := 0.U - ports(0).io.request.bits.mask := 255.U - ports(0).io.request.bits.size := 3.U - } - when(state === descriptorWait)(ports(0).io.response.ready := true.B) - - when(io.cmdReq.fire) { - command := io.cmdReq.bits - channels := io.channelReady - result := 0.U.asTypeOf(new KernelCompletion) - terminalSeen := false.B - terminalFailed := false.B - terminalCancelled := false.B - switch(io.cmdReq.bits.cmd.funct7) { - is(44.U) { - buffer := loadBuffer(0) - validImage(loadBuffer(0)) := false.B - state := loading - when(!loadBankValid || io.cmdReq.bits.cmd.rs1(19, 0).orR) { - result.fault := true.B - result.cause := 2.U - result.tval := io.cmdReq.bits.cmd.rs1 - state := completed - } - } - is(79.U) { - val program = io.cmdReq.bits.cmd.rs1(19, 10) - val programValid = program >= b.memDomain.virtualBankCount.U && program < (b.memDomain.virtualBankCount + 2).U - val slot = program - b.memDomain.virtualBankCount.U - buffer := slot(0) - word := 0.U - state := descriptorSend - when(!programValid || !validImage( - slot(0) - ) || io.cmdReq.bits.read_groups === 0.U || io.cmdReq.bits.write_groups === 0.U || - io.cmdReq.bits.cmd.rs1(63, 30).orR || io.cmdReq.bits.cmd.rs1(9, 0) === io.cmdReq.bits.cmd.rs1(29, 20) || - io.cmdReq.bits.cmd.rs2(63, 32).orR || (io.cmdReq.bits.cmd.rs2( - 31, - 0 - ) +& 96.U) > (b.memDomain.bankEntries * (b.memDomain.bankWidth / 8)).U) { - result.fault := true.B - result.cause := 1.U - result.tval := io.cmdReq.bits.cmd.rs1 - state := completed - } - } - is(32.U) { - val program = io.cmdReq.bits.cmd.rs1(9, 0) - val slot = program - b.memDomain.virtualBankCount.U - when(program < b.memDomain.virtualBankCount.U || program >= (b.memDomain.virtualBankCount + 2).U || - io.cmdReq.bits.cmd.rs1(63, 10).orR || io.cmdReq.bits.cmd.rs2.orR) { - result.fault := true.B - result.cause := 2.U - result.tval := io.cmdReq.bits.cmd.rs1 - }.otherwise(validImage(slot(0)) := false.B) - state := completed - } - - } - when(io.cmdReq.bits.cmd.funct7 =/= 44.U && io.cmdReq.bits.cmd.funct7 =/= 79.U && io.cmdReq.bits.cmd.funct7 =/= 32.U) { - result.fault := true.B - result.cause := 2.U - result.tval := io.cmdReq.bits.cmd.funct7 - state := completed - } - } - when(loader.io.done.fire) { - val nativeFault = loader.io.done.bits.fault && loader.io.done.bits.cause =/= 5.U - when((!terminalFailed && !terminalError) || - (nativeFault && (terminalCancelled || (io.imageTerminal.fire && io.imageTerminal.bits.error === 10.U)))) { - result.fault := loader.io.done.bits.fault - result.cause := loader.io.done.bits.cause - result.tval := loader.io.done.bits.tval - } - imageEntry(buffer) := loader.io.done.bits.entry - imageBytes(buffer) := loader.io.done.bits.textBytes - imageConstants(buffer) := loader.io.done.bits.constBytes - state := completed - } - when(state === descriptorSend && ports(0).io.request.fire)(state := descriptorWait) - when(state === descriptorWait && ports(0).io.response.fire) { - when(ports(0).io.response.bits.error) { - result.fault := true.B - result.cause := 5.U - result.tval := (command.read_groups.pad(64) << 32) + command.cmd.rs2 + word * 8.U - state := completed - }.otherwise { - descriptor(word) := ports(0).io.response.bits.data - word := word + 1.U - state := Mux(word === 11.U, launch, descriptorSend) - } - } - when(state === launch) { - when(descriptor(0) =/= imageEntry(buffer) || descriptor(1) =/= imageBytes(buffer) || descriptor( - 11 - ) =/= 0.U || descriptor(2) =/= "h40002000".U) { - execution.io.launch.valid := false.B - result.fault := true.B - result.cause := 1.U - result.tval := descriptor(0) - state := completed - }.elsewhen(execution.io.launch.fire)(state := executing) - } - when(execution.io.done.fire) { - result := execution.io.done.bits - state := completed - } - when(io.cmdResp.fire) { - when(command.cmd.funct7 === 44.U && !result.fault && terminalSeen && !terminalFailed) { - validImage(buffer) := true.B - } - state := idle - } -} diff --git a/arch/src/main/scala/framework/system/Device.scala b/arch/src/main/scala/framework/system/Device.scala deleted file mode 100644 index e78cb95e0..000000000 --- a/arch/src/main/scala/framework/system/Device.scala +++ /dev/null @@ -1,13 +0,0 @@ -package framework.system - -import chisel3.experimental.hierarchy.{instantiable, Instantiate} -import framework.top.GlobalConfig -import framework.system.GlobalConfigOps._ -import framework.system.tile.{MainTile, TileFactory} - -@instantiable -class Device(b: GlobalConfig) extends System(b.systemParams) { - val main = Instantiate(new MainTile(b.forTile(0))) - val tiles = b.systemDesign.get.tiles.indices.drop(1).map(i => TileFactory.instantiate(b.forTile(i))) - DeviceConnection.connect(b.systemParams, dlink, Seq(main.tlink) ++ tiles.map(_.tlink)) -} diff --git a/arch/src/main/scala/framework/system/DeviceConnection.scala b/arch/src/main/scala/framework/system/DeviceConnection.scala deleted file mode 100644 index a642bae32..000000000 --- a/arch/src/main/scala/framework/system/DeviceConnection.scala +++ /dev/null @@ -1,131 +0,0 @@ -package framework.system - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{Instance, Instantiate} -import framework.system.dlink.DLinkIO -import framework.system.tile.tlink.{Fabric, TLinkIO} -import framework.system.memory.Memory -import framework.system.device.{BootRomLines, Devices} -import memcore.memory.cpu.CpuMemParams - -object DeviceConnection { - - def connect(p: SystemParams, dlink: DLinkIO, tilePorts: Seq[TLinkIO]): Unit = { - val topology = p.topology - val memory = p.memory - val ddr = p.ddr - val devices = p.devices - val tiles = topology.tiles - val hartIds = tiles.flatMap(_.hartIds) - require(ddr.clients == 1 && ddr.line == memory.chi) - val n = hartIds.size - - val dmaMasters = tiles.size - - val cp = CpuMemParams(memory.chi, tagBits = 6) - - val controlCounts = tiles.map(_.hartIds.size) - val controlBase = controlCounts.scanLeft(0)(_ + _) - val controlCount = controlCounts.sum - - require(tilePorts.size == tiles.size, "Designed tile count must match software topology") - val linkedTiles = tilePorts.indices.filter(i => tilePorts(i).t2t.isDefined) - if (linkedTiles.nonEmpty) { - val geometries = linkedTiles.map(i => tiles(i).sharedStorage.get.bytes) - require(geometries.distinct.size == 1, "TLink tiles must share the same public shared-storage geometry") - val fabric = Instantiate(new Fabric(ddr.axi, linkedTiles, geometries.head)) - for ((tile, port) <- linkedTiles.zipWithIndex) { - fabric.io.source(port) <> tilePorts(tile).t2t.get.tx - tilePorts(tile).t2t.get.rx <> fabric.io.target(port) - } - } - - val backing: Instance[Memory] = Instantiate(new Memory(ddr, dmaMasters + 1)) - val chipDevices = Instantiate(new Devices(devices, cp, hartIds)) - chipDevices.io.sources := 0.U - val external = Instantiate(new framework.system.memory.UncachedAxi(cp, ddr.axi, n)) - val scuParams = devices.scu - val scu = Instantiate(new sims.scu.SystemControl(n, cp, scuParams)) - scu.io.hartIds := VecInit(hartIds.map(_.U(32.W))) - for (core <- hartIds.indices) { - val request = chipDevices.io.externalRequest(core) - val response = chipDevices.io.externalResponse(core) - val inScu = request.bits.addr >= scuParams.baseAddress.U && - request.bits.addr < (scuParams.baseAddress + scuParams.totalSizeBytes).U - val selectedScu = RegInit(false.B) - when(request.fire)(selectedScu := inScu) - scu.io.request(core).valid := request.valid && inScu - scu.io.request(core).bits := request.bits - external.io.request(core).valid := request.valid && !inScu - external.io.request(core).bits := request.bits - request.ready := Mux(inScu, scu.io.request(core).ready, external.io.request(core).ready) - response.valid := Mux(selectedScu, scu.io.response(core).valid, external.io.response(core).valid) - response.bits := Mux(selectedScu, scu.io.response(core).bits, external.io.response(core).bits) - scu.io.response(core).ready := selectedScu && response.ready - external.io.response(core).ready := !selectedScu && response.ready - } - scu.io.failure := VecInit(tilePorts.flatMap(_.failure)) - scu.io.workDrained := VecInit(tilePorts.flatMap(_.workDrained)) - backing.io.dma(dmaMasters) <> external.io.axi - dlink.axi <> backing.io.axi - - val coreBase = tiles.scanLeft(0)(_ + _.hartIds.size) - for ((tile, index) <- tilePorts.zipWithIndex) { - tile.tileId := index.U - for ((port, id) <- tile.hartIds.zip(tiles(index).hartIds)) port := id.U(64.W) - for ((port, id) <- tile.executionIds.zip(tiles(index).executionIds)) port := id.U(64.W) - tile.time := chipDevices.io.time - for (local <- tiles(index).hartIds.indices) { - val core = coreBase(index) + local - tile.resetVector(local) := devices.bootrom.get.base.U - tile.interrupts(local) := chipDevices.io.interrupts(core) - chipDevices.io.request(core) <> tile.uncachedRequest(local) - tile.uncachedResponse(local) <> chipDevices.io.response(core) - } - // Tile-local requester IDs keep homogeneous tiles identical. Only endpoint identity - // changes at the shared control fabric; clocks, transactions and backpressure do not. - for (lane <- 0 until (if (index == 0) 0 else 2 * controlCounts(index))) { - val localNode = lane + 1 - val globalIndex = controlBase(index) + lane % controlCounts(index) + - (if (lane >= controlCounts(index)) controlCount else 0) - val globalNode = globalIndex + 1 - val source = tile.control(lane) - val remoteCount = controlCount - controlCounts.head - val remoteIndex = controlBase(index) - controlCounts.head + lane % controlCounts(index) + - (if (lane >= controlCounts(index)) remoteCount else 0) - val target = tilePorts.head.remoteControl(remoteIndex) - target <> source - target.req.bits.srcId := globalNode.U - target.txRsp.bits.srcId := globalNode.U - target.txDat.bits.srcId := globalNode.U - source.rxRsp.bits.tgtId := localNode.U - source.rxDat.bits.tgtId := localNode.U - when(source.req.valid) { - assert(source.req.bits.srcId === localNode.U && source.req.bits.returnNid === 0.U) - } - when(source.txRsp.valid)(assert(source.txRsp.bits.srcId === localNode.U)) - when(source.txDat.valid)(assert(source.txDat.bits.srcId === localNode.U)) - when(target.rxRsp.valid)(assert(target.rxRsp.bits.tgtId === globalNode.U)) - when(target.rxDat.valid)(assert(target.rxDat.bits.tgtId === globalNode.U)) - } - if (index == 0) { - val request = tile.backingRequest(0) - val response = tile.backingResponse(0) - devices.bootrom match { - case Some(rom) => - val lines = Instantiate(new BootRomLines(rom, memory.chi)) - lines.io.request <> request - response <> lines.io.response - backing.io.lineRequest(0) <> lines.io.memoryRequest - lines.io.memoryResponse <> backing.io.lineResponse(0) - case None => - backing.io.lineRequest(0) <> request - response <> backing.io.lineResponse(0) - } - } - backing.io.dma(index) <> tile.mem - } - } - -} diff --git a/arch/src/main/scala/framework/system/GlobalConfigOps.scala b/arch/src/main/scala/framework/system/GlobalConfigOps.scala deleted file mode 100644 index 4db33ce60..000000000 --- a/arch/src/main/scala/framework/system/GlobalConfigOps.scala +++ /dev/null @@ -1,186 +0,0 @@ -package framework.system - -import framework.top.GlobalConfig -import framework.top.configs._ -import framework.system.configloader._ -import framework.system.core.{AntCoreParams, RocketCoreParams} -import framework.system.tile.{TileParameters, TileParams} -import framework.system.device._ - -/** Pure configuration projection and IP-parameter derivation; no hardware is instantiated here. */ -object GlobalConfigOps { - def accelerator(b: GlobalConfig): AcceleratorConfig = - AcceleratorConfig(b.memDomain, b.frontend, b.rvv, b.ballDomain, b.tile, b.sim, b.coreSignature) - def accelerator(a: AcceleratorConfig): GlobalConfig = - GlobalConfig(a.memDomain, a.frontend, a.rvv, a.ballDomain, a.tile, a.sim, a.coreSignature) - - def core(name: String, c: TileCore): CoreDesign = c match { - case RocketTileCore(cpu, b) => CoreDesign(name, Some(cpu), None, b.map(accelerator)) - case AntTileCore(local, b) => CoreDesign(name, None, Some(local), Some(accelerator(b))) - case _ => throw new IllegalArgumentException("Unsupported core design") - } - - def core(c: CoreDesign): TileCore = c.rocket match { - case Some(cpu) => RocketTileCore(cpu, c.accelerator.map(accelerator)) - case None => AntTileCore(c.local.get, accelerator(c.accelerator.get)) - } - - def shared(s: SharedConfig): SharedStorageParams = SharedStorageParams(s.bankBits, s.bankEntries, s.banks) - - def design(t: TileTopology): TileDesign = { - require(t.coreFactories.size == t.cores.size, "Every core requires a configured design") - TileDesign( - t.factory, - t.param, - t.main, - t.coreFactories.zip(t.cores).map { case (n, c) => core(n, c) }, - t.signatures, - t.controller, - t.sharedStorage.map(s => SharedConfig(s.bankBits, s.bankEntries, s.banks)) - ) - } - - def fromSystem(p: SystemParams): GlobalConfig = { - val d = p.devices - val devices = DevicesConfig( - d.clint.base, - d.clint.bytes, - d.clint.tickCycles, - d.plic.base, - d.plic.bytes, - d.plic.sources, - d.plic.priorityBits, - d.bootrom.map(r => RomConfig(r.base, r.bytes, r.image)), - d.scu.baseAddress, - d.scu.strideBytes, - d.scu.totalSizeBytes, - d.scu.maxHarts - ) - val platform = PlatformConfig(p.cpuPhysicalBits, p.memory, p.l1, p.regions, p.tracking, p.ddr, devices) - GlobalConfig().copy(systemDesign = - Some(SystemDesign( - p.topology.tiles.map(t => framework.top.configs.TilePlacement(design(t), t.hartIds, t.executionIds)), - platform - )) - ) - } - - implicit class DesignParameters(val b: GlobalConfig) extends AnyVal { - - def systemParams: SystemParams = { - val s = b.systemDesign.get - val p = s.platform - val d = p.devices - val topology = ExampleTopology(s.tiles.map { t => - val v = t.design - TileTopology( - v.param, - v.main, - v.cores.map(core), - t.hartIds, - t.executionIds, - v.signatures, - v.controller, - v.sharedStorage.map(shared), - v.cores.map(_.factory), - v.factory - ) - }) - val devices = DeviceParams( - ClintParams(d.clintBase, d.clintBytes, d.tickCycles), - PlicParams(d.plicBase, d.plicBytes, d.sources, d.priorityBits), - d.bootrom.map(r => BootRomParams(r.base, r.bytes, r.image)), - sims.scu.SCUParams(d.scuBase, d.scuStride, d.scuBytes, d.scuMaxHarts) - ) - SystemParams(topology, p.cpuPhysicalBits, p.memory, p.l1, p.regions, p.tracking, p.ddr, devices) - } - - def forTile(index: Int): GlobalConfig = { - val s = b.systemDesign.get - val ids = s.tiles.flatMap(_.hartIds) - GlobalConfig().copy(tileDesign = Some(TileScope(s.tiles(index).design, s.platform, ids.max, ids.size, s.tiles.size))) - } - - def tileLinkParams: framework.system.tile.TileLinkParams = { - val t = b.tileDesign.get - framework.system.tile.TileLinkParams( - t.design.main, - t.design.cores.count(_.rocket.isDefined), - t.design.cores.size, - if (t.design.main) t.controls else 0, - t.platform.memory.chi, - t.platform.ddr.axi, - t.design.sharedStorage.isDefined - ) - } - - def tileParams: TileParams = { - val t = b.tileDesign.get - val p = t.platform - val cpus = t.design.cores.flatMap(_.rocket).map(cpu => - TileParameters.cpuParameters(cpu, t.design.param, t.maxHartId, p.cpuPhysicalBits) - ) - TileParams( - t.design.main, - t.design.cores.map(core), - t.design.signatures, - t.design.controller, - cpus, - p.memory.copy(agents = 2 * cpus.size), - p.l1, - p.regions, - p.tracking, - t.controls, - p.ddr.axi, - t.design.sharedStorage.map(shared), - t.tiles - ) - } - - def forCore(index: Int): GlobalConfig = { - val t = b.tileDesign.get - val c = t.design.cores(index) - val cpus = t.design.cores.flatMap(_.rocket) - val selected = c.accelerator.map(accelerator).getOrElse(GlobalConfig()) - val signature = if (c.local.isDefined) selected.coreSignature else BigInt(0) - selected.copy(coreDesign = Some(CoreScope( - c.factory, - c.rocket, - c.local, - c.accelerator.isDefined, - t.design.controller.map(i => t.design.cores(i).rocket.get), - t.design.param, - t.platform, - t.maxHartId, - if (c.rocket.isDefined) t.design.cores.take(index).count(_.rocket.isDefined) else 0, - cpus.size, - signature - ))) - } - - def rocketParams: RocketCoreParams = { - val c = b.coreDesign.get - require(c.rocket.isDefined) - val p = c.platform - val cpu = TileParameters.cpuParameters(c.rocket.get, c.tile, c.maxHartId, p.cpuPhysicalBits) - RocketCoreParams( - cpu, - p.l1.copy(nodeId = c.cpuIndex + 1), - p.l1.copy(nodeId = c.cpuCount + c.cpuIndex + 1), - p.regions, - hier.core.rocket.Commands(c.hasAccelerator, true, p.tracking), - Option.when(c.hasAccelerator)(b) - ) - } - - def antParams: AntCoreParams = { - val c = b.coreDesign.get - require(c.local.isDefined && c.hasAccelerator) - val p = c.platform - val cpu = TileParameters.cpuParameters(c.controller.get, c.tile, c.maxHartId, p.cpuPhysicalBits) - AntCoreParams(b, c.local.get, p.tracking, p.memory.chi, c.signature, cpu) - } - - } - -} diff --git a/arch/src/main/scala/framework/system/System.scala b/arch/src/main/scala/framework/system/System.scala deleted file mode 100644 index 8c1b798f6..000000000 --- a/arch/src/main/scala/framework/system/System.scala +++ /dev/null @@ -1,31 +0,0 @@ -package framework.system - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.system.configloader.ExampleTopology -import framework.system.device.DeviceParams -import framework.system.dlink.{DLinkIO, HasDLink} -import memcore.bus.chi.rnf.RnfParams -import memcore.memory.coherence.configs.CoherenceParams -import memcore.memory.cpu.PhysicalRegion -import memcore.memory.interlock.{Params => TrackingParams} -import memcore.memory.ddr.{Params => DdrParams} - -case class SystemParams( - topology: ExampleTopology, - cpuPhysicalBits: Int, - memory: CoherenceParams, - l1: RnfParams, - regions: Seq[PhysicalRegion], - tracking: TrackingParams, - ddr: DdrParams, - devices: DeviceParams = DeviceParams()) { - - def toGlobalConfig: framework.top.GlobalConfig = GlobalConfigOps.fromSystem(this) - -} - -@instantiable -abstract class System(p: SystemParams) extends Module with HasDLink { - @public val dlink = IO(new DLinkIO(p)) -} diff --git a/arch/src/main/scala/framework/system/configloader/ChipLoader.scala b/arch/src/main/scala/framework/system/configloader/ChipLoader.scala deleted file mode 100644 index 93d9a507a..000000000 --- a/arch/src/main/scala/framework/system/configloader/ChipLoader.scala +++ /dev/null @@ -1,499 +0,0 @@ -package framework.system.configloader - -import buckyball.config.{ - BoomCpuConfig, - Chip, - CoreInstance, - CpuConfig, - FrontendConfig, - MemDomainConfig, - RocketCpuConfig, - RvvConfig, - SharedMemConfig, - SpmConfig, - TileKind, - TileParamConfig, - TilePlacement -} -import java.nio.file.{Files, Path, Paths} -import framework.balldomain.configs.{BallDomainParam, BallISAEntry, BallIdMapping} -import framework.frontend.configs.FrontendParam -import framework.rvv.configs.RvvParam -import framework.memdomain.configs.MemDomainParam -import framework.system.core.boom.configs.{BoomCpuParam, BoomDCacheParam, BoomICacheParam} -import framework.system.core.rocket.configs._ -import framework.system.tile.configs.TileParam -import framework.top.GlobalConfig -import scala.jdk.CollectionConverters._ - -/** Load ExampleTopology from chip.pb. */ -object ChipLoader { - - def load(pbPath: String): ExampleTopology = { - val path = Paths.get(pbPath) - if (!Files.isRegularFile(path)) { - throw new RuntimeException(s"chip.pb does not exist: $pbPath") - } - val repo = repoRoot(path) - val chip = Chip.parseFrom(Files.readAllBytes(path)) - val cores = chip.getCoresList.asScala.toSeq - if (cores.isEmpty) { - throw new RuntimeException(s"chip.pb has no cores: $pbPath") - } - val tiles = chip.getTilesList.asScala.map(parseTile(_, cores, repo)).toSeq - require( - tiles.size == chip.getNTiles, - s"chip.pb declares n_tiles=${chip.getNTiles} but defines ${tiles.size} tile(s) in $pbPath" - ) - val hartIds = tiles.flatMap(_.hartIds) - require(hartIds.forall(_ >= 0), "hart_id must fit a nonnegative Scala Int") - require(hartIds.distinct.size == hartIds.size, "chip hart_id values must be globally unique") - ExampleTopology(tiles) - } - - def repoRoot(pb: Path): Path = { - val abs = pb.toAbsolutePath.normalize - val s = abs.toString - val marker = "/examples/chips/" - val i = s.lastIndexOf(marker) - if (i < 0) { - throw new RuntimeException(s"chip.pb is not under examples/chips: $pb") - } - Paths.get(s.substring(0, i)) - } - - def repoFile(repo: Path, rel: String, what: String): String = { - if (rel.isEmpty) { - throw new RuntimeException(s"$what is empty") - } - val p = Paths.get(rel) - if (p.isAbsolute) { - throw new RuntimeException(s"$what must be repo-relative: $rel") - } - val resolved = repo.resolve(p).normalize() - if (!Files.isRegularFile(resolved)) { - throw new RuntimeException(s"$what missing: $resolved") - } - resolved.toString - } - - def parseTile(tile: TilePlacement, cores: Seq[CoreInstance], repo: Path): TileTopology = { - val indices = tile.getCoreIndicesList.asScala.map(_.toInt).toSeq - val nCores = indices.size - val computeCoreIds = - indices.zipWithIndex.collect { case (index, slot) if cores(index).getBalldomain.getBallNum > 0 => slot } - val hasBuckyball = computeCoreIds.nonEmpty - val tileParam = parseTileParam(tile.getParam) - - val shared = tile.getSharedMem - val virtualBankCount = tile.getVirtualBankCount - if (hasBuckyball) { - require(virtualBankCount > 0, s"tile ${tile.getPath}: virtual_bank_count must be > 0") - } - val sharedBankNum = - if (shared.getEnable) { - require(shared.getBankWidth == 128, s"tile ${tile.getPath}: shared bank width must be 128 bits") - indices.iterator.map(idx => cores(idx)).filter(_.getBalldomain.getBallNum > 0).foreach { core => - require( - core.getMem.getBank.getWidth == shared.getBankWidth, - s"tile ${tile.getPath}: all Buckyball cores must use shared bank width ${shared.getBankWidth}" - ) - } - require(shared.getEntries > 0, s"tile ${tile.getPath}: shared entries must be > 0") - require( - shared.getBankEntries > 0 && shared.getEntries % shared.getBankEntries == 0, - s"tile ${tile.getPath}: shared entries ${shared.getEntries} must be divisible by shared bank entries ${shared.getBankEntries}" - ) - shared.getEntries / shared.getBankEntries - } else 0 - val tss = Option.when(tile.hasTss)(parseSpm(tile.getTss)) - val tileCores = indices.map { idx => - parseCore(cores(idx), tileParam, shared, sharedBankNum, virtualBankCount, nCores, computeCoreIds, tss, repo) - } - val ants = indices.filter(i => cores(i).getCpu.getKind == "ant") - ants.zipWithIndex.foreach { case (idx, context) => - require( - !cores(idx).hasHartId && cores(idx).hasAntContextId && cores(idx).getAntContextId == context, - s"core $idx: Ant requires explicit local context, without a CPU hart" - ) - } - val hartIds = indices.filterNot(ants.contains).map { idx => - require(cores(idx).hasHartId && !cores(idx).hasAntContextId, s"core $idx: CPU requires hart_id only") - cores(idx).getHartId - } - val main = tile.getKind match { - case TileKind.TILE_KIND_MAIN => true - case TileKind.TILE_KIND_TILE => false - case kind => throw new IllegalArgumentException(s"Unknown tile kind: $kind") - } - require(!main || !tile.hasControllerCoreIndex, "The main tile has no task controller") - val controller = Option.when(tile.hasControllerCoreIndex) { - val local = indices.indexOf(tile.getControllerCoreIndex) - require(local >= 0, s"tile ${tile.getPath}: controller must belong to the tile") - local - } - require(tile.getFactoryClass.isEmpty == main, "Only mounted tiles must specify a factory") - TileTopology( - tileParam, - main, - tileCores, - hartIds, - indices, - if (controller.isDefined) indices.map(idx => coreSignature(cores(idx), tss, shared)) else Nil, - controller, - Option.when(shared.getEnable)(SharedStorageParams(shared.getBankWidth, shared.getBankEntries, sharedBankNum)), - indices.map(idx => cores(idx).getFactoryClass), - tile.getFactoryClass - ) - } - - def parseCore( - core: CoreInstance, - tile: TileParam, - shared: SharedMemConfig, - sharedBankNum: Int, - virtualBankCount: Int, - nCores: Int, - computeCoreIds: Seq[Int], - tss: Option[memcore.memory.spm.Params], - repo: Path - ): TileCore = { - require(core.hasCpu, s"core ${core.getPkg}: missing cpu config") - val cpu = core.getCpu - val kind = cpu.getKind - kind match { - case "rocket" => - require(cpu.hasRocket, s"core ${core.getPkg}: missing cpu.rocket") - RocketTileCore( - parseRocketCpu(cpu.getRocket), - parseAccelerator(core, tile, shared, sharedBankNum, virtualBankCount, nCores, computeCoreIds, tss, repo) - ) - case "ant" => - require(cpu.hasAnt && tss.isDefined, s"core ${core.getPkg}: Ant requires local geometry and tile TSS") - val a = cpu.getAnt - require(a.hasTls, s"core ${core.getPkg}: missing Ant TLS") - val local = framework.ant.Params(a.getCodeBytes, parseSpm(a.getTls), tss.get, a.getTaskBits) - val accelerator = - parseAccelerator(core, tile, shared, sharedBankNum, virtualBankCount, nCores, computeCoreIds, tss, repo) - require(accelerator.isDefined, "A compute Ant must have an NPU endpoint") - AntTileCore(local, accelerator.get) - case "boom" => - if (core.getBalldomain.getBallNum > 0) { - throw new RuntimeException(s"core ${core.getPkg}: kind=boom forbids balldomain") - } - if (!cpu.hasBoom) { - throw new RuntimeException(s"core ${core.getPkg}: kind=boom missing cpu.boom") - } - BoomTileCore(parseBoomCpu(cpu.getBoom)) - case other => - throw new RuntimeException(s"core ${core.getPkg}: unsupported kind '$other'") - } - } - - def parseAccelerator( - core: CoreInstance, - tile: TileParam, - shared: SharedMemConfig, - sharedBankNum: Int, - virtualBankCount: Int, - nCores: Int, - computeCoreIds: Seq[Int], - tss: Option[memcore.memory.spm.Params], - repo: Path - ): Option[GlobalConfig] = { - val domain = core.getBalldomain - if (domain.getBallNum == 0) { - return None - } - require(core.hasFrontend, s"core ${core.getPkg} missing frontend config") - require(core.hasRvv, s"core ${core.getPkg} missing rvv config") - require(core.getRvv.hasEnable, s"core ${core.getPkg} must explicitly set rvv.enable") - require(tile.coreDataBytes > 0, s"core ${core.getPkg} missing tile config") - val frontend = core.getFrontend - require( - virtualBankCount <= (1 << frontend.getBankIdLen), - s"core ${core.getPkg}: virtual_bank_count $virtualBankCount does not fit bank_id_len ${frontend.getBankIdLen}" - ) - require( - frontend.getVbankIdUpperBound < virtualBankCount, - s"core ${core.getPkg}: private vbank upper bound ${frontend.getVbankIdUpperBound} must be below virtual_bank_count $virtualBankCount" - ) - if (shared.getEnable) { - require( - frontend.getSharedBankIdBase > frontend.getVbankIdUpperBound, - s"core ${core.getPkg}: shared bank base must be above the private vbank range" - ) - require( - frontend.getSharedBankIdBase <= virtualBankCount, - s"core ${core.getPkg}: shared bank base ${frontend.getSharedBankIdBase} exceeds virtual_bank_count $virtualBankCount" - ) - } - - val buckyball = GlobalConfig().copy( - coreSignature = coreSignature(core, tss, shared), - ballDomain = parseBallDomain(core, repo), - frontend = parseFrontend(core.getFrontend), - rvv = parseRvv(core.getRvv), - tile = tile, - memDomain = parseMemDomain(core.getMem, shared, sharedBankNum, virtualBankCount, nCores, computeCoreIds) - ) - Some(buckyball) - } - - def parseSpm(value: SpmConfig): memcore.memory.spm.Params = - memcore.memory.spm.Params(BigInt(java.lang.Long.toUnsignedString(value.getBase)), value.getBytes, value.getDataBits) - - def coreSignature( - core: CoreInstance, - tss: Option[memcore.memory.spm.Params], - sharedMemory: SharedMemConfig - ): BigInt = { - require(core.hasMem && core.getMem.hasBank, s"core ${core.getPkg}: task signature requires explicit mem.bank") - val bytes = new java.io.ByteArrayOutputStream() - def text(value: String): Unit = { - bytes.write(value.getBytes(java.nio.charset.StandardCharsets.UTF_8)); bytes.write(0) - } - def integer(value: Long, width: Int = 8): Unit = - (0 until width).foreach(i => bytes.write(((value >>> (8 * i)) & 255).toInt)) - text(core.getPkg) - val bank = core.getMem.getBank - Seq(bank.getNum, bank.getWidth, bank.getEntries).foreach(value => integer(value.toLong)) - core.getBalldomain.getIsaList.asScala.sortBy(_.getFunct7).foreach { entry => - text(entry.getMnemonic); integer(entry.getFunct7.toLong, 4) - } - core.getBalldomain.getMappingsList.asScala.sortBy(_.getBallClass.split('.').last).foreach { mapping => - text(mapping.getBallClass.split('.').last) - integer(mapping.getInBw.toLong); integer(mapping.getOutBw.toLong) - mapping.getBallParamsMap.asScala.toSeq.sortBy(_._1).foreach { case (name, value) => text(name); text(value) } - } - if (core.getCpu.getKind == "ant") { - val a = core.getCpu.getAnt - val shared = tss.get - text("ant") - Seq( - a.getCodeBytes.toLong, - a.getTls.getBase, - a.getTls.getBytes.toLong, - a.getTls.getDataBits.toLong, - a.getTaskBits.toLong, - shared.base.toLong, - shared.bytes.toLong, - shared.dataBits.toLong, - sharedMemory.getBankEntries.toLong, - sharedMemory.getEntries.toLong - ).foreach(v => integer(v)) - } - val mask = (BigInt(1) << 64) - 1 - bytes.toByteArray.foldLeft(BigInt("cbf29ce484222325", 16)) { (hash, value) => - ((hash ^ BigInt(value & 255)) * BigInt("100000001b3", 16)) & mask - } - } - - def parseBallDomain(core: CoreInstance, repo: Path): BallDomainParam = { - val domain = core.getBalldomain - val mappings = domain.getMappingsList.asScala.map { m => - val params = m.getBallParamsMap.asScala.toMap - require(m.getInBw > 0 && m.getOutBw > 0, s"Ball ${m.getBallName}: BBus widths must be positive") - val config = Some(repoFile(repo, m.getConfigPath, s"Ball ${m.getBallName} config")) - - BallIdMapping( - ballId = m.getBallId, - ballName = m.getBallName, - ballClass = m.getBallClass, - config = config, - inBW = m.getInBw, - outBW = m.getOutBw, - mmioReadBW = m.getMmioReadBw, - mmioWriteBW = m.getMmioWriteBw, - ballParams = params - ) - }.toSeq - val isa = domain.getIsaList.asScala.map { e => - BallISAEntry(mnemonic = e.getMnemonic, funct7 = e.getFunct7, bid = e.getBid) - }.toSeq - BallDomainParam(ballNum = domain.getBallNum, ballIdMappings = mappings, ballISA = isa) - } - - def parseMemDomain( - mem: MemDomainConfig, - shared: SharedMemConfig, - sharedBankNum: Int, - virtualBankCount: Int, - nCores: Int, - computeCoreIds: Seq[Int] - ): MemDomainParam = { - val bank = mem.getBank - val dma = mem.getDma - val tlb = mem.getTlb - val tma = mem.getTma - val mmio = mem.getMmio - MemDomainParam( - bankNum = bank.getNum, - bankWidth = bank.getWidth, - bankEntries = bank.getEntries, - bankMaskLen = bank.getMaskLen, - virtualBankCount = virtualBankCount, - sharedEnable = shared.getEnable, - sharedEntries = shared.getEntries, - sharedBankEntries = shared.getBankEntries, - sharedBankNum = sharedBankNum, - sharedInputChannels = shared.getInputChannels, - sharedDefaultGroupCount = shared.getDefaultGroupCount, - nCores = nCores, - computeCoreIds = computeCoreIds, - tlb_size = tlb.getSize, - dma_n_xacts = dma.getNXacts, - dma_burst_maxbytes = dma.getBurstMaxBytes, - bankChannel = bank.getChannel, - max_in_flight_mem_reqs = dma.getMaxInFlightMemReqs, - dma_buswidth = dma.getBusWidth, - memAddrLen = mem.getMem.getAddrLen, - tmaReadChannel = tma.getReadChannel, - tmaWriteChannel = tma.getWriteChannel, - mmioEnable = mmio.getEnable, - mmioBankNum = mmio.getBankNum, - mmioBankEntries = mmio.getBankEntries, - mmioBankWidth = mmio.getBankWidth, - mmioReadWidth = mmio.getReadWidth - ) - } - - def parseFrontend(frontend: FrontendConfig): FrontendParam = - FrontendParam( - rob_entries = frontend.getRobEntries, - rs_out_of_order_response = frontend.getRsOutOfOrderResponse, - bank_id_len = frontend.getBankIdLen, - vbank_id_upper_bound = frontend.getVbankIdUpperBound, - shared_bank_id_base = frontend.getSharedBankIdBase, - iter_len = frontend.getIterLen, - sub_rob_enable = frontend.getSubRobEnable, - sub_rob_depth = frontend.getSubRobDepth - ) - - def parseRvv(rvv: RvvConfig): RvvParam = - RvvParam( - enable = rvv.getEnable, - laneNumber = rvv.getLaneNumber, - vLen = rvv.getVLen, - eLen = rvv.getELen, - iBufWords = rvv.getIBufWords, - memoryPorts = rvv.getMemoryPorts - ) - - def parseTileParam(param: TileParamConfig): TileParam = - TileParam( - coreDataBytes = param.getCoreDataBytes, - xLen = param.getXLen, - vaddrBits = param.getVaddrBits, - paddrBits = param.getPaddrBits, - pgIdxBits = param.getPgIdxBits, - pgLevels = param.getPgLevels, - nPMPs = param.getNPmps - ) - - def parseRocketCpu(rocket: RocketCpuConfig): RocketCpuParam = { - val mulDiv = rocket.getMulDiv - val fpu = rocket.getFpu - val dcache = rocket.getDcache - val icache = rocket.getIcache - val btb = rocket.getBtb - RocketCpuParam( - mtvecInitEnable = rocket.getMtvecInitEnable, - useUser = rocket.getUseUser, - useSupervisor = rocket.getUseSupervisor, - useHypervisor = rocket.getUseHypervisor, - useDebug = rocket.getUseDebug, - useAtomics = rocket.getUseAtomics, - useAtomicsOnlyForIO = rocket.getUseAtomicsOnlyForIO, - useCompressed = rocket.getUseCompressed, - useRVE = rocket.getUseRVE, - useConditionalZero = rocket.getUseConditionalZero, - nLocalInterrupts = rocket.getNLocalInterrupts, - useNMI = rocket.getUseNMI, - nBreakpoints = rocket.getNBreakpoints, - useBPWatch = rocket.getUseBPWatch, - mcontextWidth = rocket.getMcontextWidth, - scontextWidth = rocket.getScontextWidth, - nPerfCounters = rocket.getNPerfCounters, - haveBasicCounters = rocket.getHaveBasicCounters, - misaWritable = rocket.getMisaWritable, - mtvecInit = BigInt(java.lang.Long.toUnsignedString(rocket.getMtvecInit)), - mtvecWritable = rocket.getMtvecWritable, - fastLoadWord = rocket.getFastLoadWord, - fastLoadByte = rocket.getFastLoadByte, - branchPredictionModeCSR = rocket.getBranchPredictionModeCSR, - clockGate = rocket.getClockGate, - mvendorid = rocket.getMvendorid, - mimpid = rocket.getMimpid, - haveCease = rocket.getHaveCease, - haveSimTimeout = rocket.getHaveSimTimeout, - useVM = rocket.getUseVm, - useZba = rocket.getUseZba, - useZbb = rocket.getUseZbb, - useZbs = rocket.getUseZbs, - haveCFlush = rocket.getHaveCFlush, - mulDiv = MulDivParam( - divUnroll = mulDiv.getDivUnroll, - divEarlyOutGranularity = mulDiv.getDivEarlyOutGranularity, - enable = mulDiv.getEnable, - mulUnroll = mulDiv.getMulUnroll, - mulEarlyOut = mulDiv.getMulEarlyOut, - divEarlyOut = mulDiv.getDivEarlyOut - ), - fpu = FPUParam( - divSqrt = fpu.getDivSqrt, - sfmaLatency = fpu.getSfmaLatency, - dfmaLatency = fpu.getDfmaLatency, - fpmuLatency = fpu.getFpmuLatency, - ifpuLatency = fpu.getIfpuLatency, - enable = fpu.getEnable, - minFLen = fpu.getMinFLen, - fLen = fpu.getFLen - ), - dcache = DCacheParam( - clockGate = dcache.getClockGate, - nSets = dcache.getNSets, - nWays = dcache.getNWays, - nMSHRs = dcache.getNMshrs - ), - icache = ICacheParam( - prefetch = icache.getPrefetch, - nSets = icache.getNSets, - nWays = icache.getNWays - ), - btb = BTBParam( - nMatchBits = btb.getNMatchBits, - nPages = btb.getNPages, - updatesOutOfOrder = btb.getUpdatesOutOfOrder, - bhtEnable = btb.getBhtEnable, - bhtEntries = btb.getBhtEntries, - bhtCounterLength = btb.getBhtCounterLength, - bhtHistoryLength = btb.getBhtHistoryLength, - bhtHistoryBits = btb.getBhtHistoryBits, - enable = btb.getEnable, - nEntries = btb.getNEntries, - nRAS = btb.getNRas - ) - ) - } - - def parseBoomCpu(boom: BoomCpuConfig): BoomCpuParam = { - val dcache = boom.getDcache - val icache = boom.getIcache - BoomCpuParam( - fetchWidth = boom.getFetchWidth, - decodeWidth = boom.getDecodeWidth, - numRobEntries = boom.getNumRobEntries, - dcache = BoomDCacheParam( - nSets = dcache.getNSets, - nWays = dcache.getNWays, - nMSHRs = dcache.getNMshrs - ), - icache = BoomICacheParam( - nSets = icache.getNSets, - nWays = icache.getNWays - ) - ) - } - -} diff --git a/arch/src/main/scala/framework/system/configloader/Topology.scala b/arch/src/main/scala/framework/system/configloader/Topology.scala deleted file mode 100644 index 035f8b5e0..000000000 --- a/arch/src/main/scala/framework/system/configloader/Topology.scala +++ /dev/null @@ -1,50 +0,0 @@ -package framework.system.configloader - -import framework.system.core.boom.configs.BoomCpuParam -import framework.system.core.rocket.configs.RocketCpuParam -import framework.system.tile.configs.TileParam -import framework.top.GlobalConfig - -/** Chip topology loaded from chip.pb: the main tile first, then configured TLink tiles. */ -case class ExampleTopology(tiles: Seq[TileTopology]) { - require(tiles.nonEmpty && tiles.head.main && tiles.tail.forall(!_.main), "Tile 0 alone is the main tile") - def mainTile: TileTopology = tiles.head - def mountedTiles: Seq[TileTopology] = tiles.tail -} - -sealed trait TileCore - -case class RocketTileCore( - cpu: RocketCpuParam, - buckyball: Option[GlobalConfig]) - extends TileCore - -case class BoomTileCore(cpu: BoomCpuParam) extends TileCore - -case class AntTileCore(local: framework.ant.Params, buckyball: GlobalConfig) extends TileCore - -case class SharedStorageParams(bankBits: Int, bankEntries: Int, banks: Int) { - require(bankBits == 128 && bankEntries > 0 && banks > 0) - val bankBytes: Int = bankEntries * (bankBits / 8) - val bytes: BigInt = BigInt(banks) * bankBytes -} - -/** - * Per-tile topology: ordered cores. CPU kind is per-core; shared memory is tile-level. - * CPU cores have system hart IDs; Ant contexts remain tile-local resources. - */ -case class TileTopology( - param: TileParam, - main: Boolean, - cores: Seq[TileCore], - hartIds: Seq[Int], - executionIds: Seq[Int], - signatures: Seq[BigInt], - controller: Option[Int], - sharedStorage: Option[SharedStorageParams], - coreFactories: Seq[String], - factory: String) { - val cpuCoreIds: Seq[Int] = cores.indices.filter(i => !cores(i).isInstanceOf[AntTileCore]) - require(hartIds.size == cpuCoreIds.size && executionIds.size == cores.size) - require(!main || controller.isEmpty, "The main tile has no task controller") -} diff --git a/arch/src/main/scala/framework/system/core/AntEndpoint.scala b/arch/src/main/scala/framework/system/core/AntEndpoint.scala deleted file mode 100644 index ccd8d19e6..000000000 --- a/arch/src/main/scala/framework/system/core/AntEndpoint.scala +++ /dev/null @@ -1,19 +0,0 @@ -package framework.system.core - -import framework.system.GlobalConfigOps._ -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.system.core.clink.{CLinkIO, HasCLink} -import framework.system.core.rocket.CpuParams - -/** Typed Ant/NPU boundary; implementations own Ant and TLS, while the tile owns TSS. */ -@instantiable -abstract class AntEndpoint(val b: framework.top.GlobalConfig) extends Module with HasCLink { - val p = b.antParams - val local = p.local - val tracking = p.tracking - val bus = p.bus - val signature = p.signature - implicit val cpuParams: CpuParams = p.cpu - @public val clink = IO(new CLinkIO(b, local, tracking, bus)) -} diff --git a/arch/src/main/scala/framework/system/core/ControllerAdmission.scala b/arch/src/main/scala/framework/system/core/ControllerAdmission.scala deleted file mode 100644 index f5282bb43..000000000 --- a/arch/src/main/scala/framework/system/core/ControllerAdmission.scala +++ /dev/null @@ -1,104 +0,0 @@ -package framework.system.core - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.system.core.rocket.{CpuParams, HasCpuParameters} -import framework.system.core.rocket.RoCCIO -import framework.memdomain.isa.{MvoverISA, MvoverPort} -import framework.memdomain.frontend.mem.dma.{DmaError, DmaStatus} -import hier.core.rocket.{AdmissionPorts, CommandSnapshot} -import hier.tile.TaskAdmission -import memcore.bus.chi.{Params => ChiParams} -import memcore.memory.interlock.{Params => TrackingParams} - -/** CPU-only controller command path: task RPC and bank moves have explicit completion owners. */ -@instantiable -class ControllerAdmission(tracking: TrackingParams, bus: ChiParams, moves: Boolean)(implicit val cpuParams: CpuParams) - extends Module - with HasCpuParameters { - require(xLen == 64) - - @public - val io = IO(new Bundle { - val core = Flipped(new AdmissionPorts(tracking, nPMPs, bus)) - val task = Flipped(new RoCCIO(64)) - val taskSatp = Output(UInt(64.W)) - val taskContext = Output(new CommandSnapshot(tracking, nPMPs)) - val move = new MvoverPort - val moveFault = Valid(new DmaStatus) - }) - - val idle :: task :: moving :: moveResponse :: releasing :: Nil = Enum(5) - val state = RegInit(idle) - val tag = Reg(UInt(tracking.idBits.W)) - val rd = Reg(UInt(5.W)) - val failedMove = Reg(Bool()) - val instruction = io.core.command.bits.instruction - val isTask = instruction.opcode === "h2b".U - val isMove = moves.B && instruction.opcode === "h7b".U && instruction.funct === MvoverISA.Funct.U - val controller: Instance[TaskAdmission] = Instantiate(new TaskAdmission(tracking, nPMPs)) - controller.io.task <> io.task - io.taskSatp := controller.io.satp - // The Ant launch handshake is the task command handshake; freeze this complete snapshot there. - io.taskContext := io.core.command.bits - controller.io.workDrained := true.B // This endpoint has no NPU or DMA client. - controller.io.command.valid := state === idle && io.core.command.valid && isTask - controller.io.command.bits := io.core.command.bits - - io.core.reserve.ready := !reset.asBool - io.core.command.ready := state === idle && !reset.asBool && - Mux(isTask, controller.io.command.ready, isMove && io.move.command.ready) - when(io.core.command.valid && state === idle && !reset.asBool) { - assert(isTask || isMove, "CPU-only controller received an unsupported compute instruction") - } - io.move.command.valid := state === idle && io.core.command.valid && isMove && !reset.asBool - io.move.command.bits.sourceCore := instruction.rs1Data(7, 0) - io.move.command.bits.targetCore := instruction.rs1Data(15, 8) - io.move.command.bits.sourceBank := instruction.rs1Data(25, 16) - io.move.command.bits.targetBank := instruction.rs1Data(35, 26) - io.move.command.bits.sourceAddr := instruction.rs2Data(15, 0) - io.move.command.bits.targetAddr := instruction.rs2Data(31, 16) - io.move.command.bits.rows := instruction.rs2Data(47, 32) +& 1.U - when(io.core.command.fire) { - tag := io.core.command.bits.tag - rd := instruction.rd - state := Mux(isTask, task, moving) - } - io.move.completion.ready := state === moving && !reset.asBool - when(io.move.completion.fire) { - failedMove := io.move.completion.bits - state := moveResponse - } - io.moveFault.valid := io.move.completion.fire && io.move.completion.bits - io.moveFault.bits.error := DmaError.Bank.U - io.moveFault.bits.address := 0.U // Existing move completion supplies no byte address. - - io.core.response.valid := Mux(state === task, controller.io.response.valid, state === moveResponse) && !reset.asBool - io.core.response.bits := controller.io.response.bits - when(state === moveResponse) { - io.core.response.bits.rd := rd - io.core.response.bits.data := failedMove.asUInt - } - controller.io.response.ready := state === task && io.core.response.ready - when(state === moveResponse && io.core.response.fire)(state := releasing) - io.core.complete.valid := (state === releasing || (state === task && controller.io.release.valid)) && !reset.asBool - io.core.complete.bits.tag := Mux(state === task, controller.io.release.bits.tag, tag) - controller.io.release.ready := state === task && io.core.complete.ready - when(io.core.complete.fire)(state := idle) - io.core.cancelled.valid := false.B - io.core.cancelled.bits := 0.U.asTypeOf(io.core.cancelled.bits) - io.core.interrupt := false.B - - // A move may have retired at CPU dispatch while its work is still pending here. - // Do not let a following CPU store publish a completion flag before the bank move ends. - val pendingMove = io.core.command.valid && isMove - io.core.cpuAllow := !(pendingMove || state === moving || state === moveResponse || state === releasing) - io.core.cpuProbeAllow := io.core.cpuAllow - io.core.maintenance.valid := false.B - io.core.maintenance.bits := 0.U.asTypeOf(io.core.maintenance.bits) - io.core.maintained.ready := true.B - io.core.pteRequest.valid := false.B - io.core.pteRequest.bits := 0.U.asTypeOf(io.core.pteRequest.bits) - io.core.pteResponse.ready := true.B -} diff --git a/arch/src/main/scala/framework/system/core/CoreConnection.scala b/arch/src/main/scala/framework/system/core/CoreConnection.scala deleted file mode 100644 index cf6d7bd92..000000000 --- a/arch/src/main/scala/framework/system/core/CoreConnection.scala +++ /dev/null @@ -1,80 +0,0 @@ -package framework.system.core - -import chisel3._ -import chisel3.experimental.hierarchy.Instance -import framework.ant.LocalExecution -import framework.system.core.accelerator.{AntAdmission, BuckyballAccelerator} -import framework.system.core.clink.CLinkIO - -object CoreConnection { - - def accelerator(link: AcceleratorCLink, accelerator: Instance[BuckyballAccelerator]): Unit = { - accelerator.io.cmd <> link.npu.command - link.npu.response <> accelerator.io.resp - link.npu.allocation := accelerator.io.allocation - link.npu.retired := accelerator.io.retired - link.npu.fault := accelerator.io.fault - link.npu.busy := !accelerator.io.idle - link.npu.interrupt := accelerator.io.interrupt - link.npu.footprints := accelerator.io.footprints - link.mem <> accelerator.io.dma - accelerator.io.hartid := link.hartId - accelerator.io.sharedBankOwnerHartId := link.shmOwner - link.shm.requests <> accelerator.io.shared_mem_req - link.shm.move <> accelerator.io.mvover - accelerator.io.meshLocalBank <> link.shm.local - link.shm.config <> accelerator.io.shared_config - link.shm.queryValid := accelerator.io.shared_query_valid - link.shm.queryVbank := accelerator.io.shared_query_vbank_id - accelerator.io.shared_query_group_count := link.shm.queryGroups - link.shm.barrierArrive := accelerator.io.barrier_arrive - accelerator.io.barrier_release := link.shm.barrierRelease - accelerator.io.shared_bank_hashes.foreach(_ := link.sharedHashes.get) - } - - def ant( - clink: CLinkIO, - execution: Instance[LocalExecution], - accelerator: Instance[BuckyballAccelerator], - issuer: Instance[AntAdmission], - signature: BigInt - ): Unit = { - execution.io.local <> clink.local - clink.ctrl.fingerprint := signature.U(64.W) - clink.ctrl.online := true.B - issuer.io.bind.valid := clink.ctrl.bind.valid - issuer.io.bind.bits.task := clink.ctrl.bind.bits.task - issuer.io.bind.bits.snapshot := clink.ctrl.bind.bits.snapshot - issuer.io.command <> execution.io.command - execution.io.response <> issuer.io.response - issuer.io.cancel := clink.local.cancel - issuer.io.workDrained := clink.ctrl.workDrained - issuer.io.halted := clink.ctrl.halted - clink.ctrl.drained := issuer.io.drained - execution.io.npuDrained := issuer.io.drained - clink.admission <> issuer.io.admission - clink.memory <> issuer.io.memory - accelerator.io.cmd <> clink.npu.command - clink.npu.response <> accelerator.io.resp - clink.npu.allocation := accelerator.io.allocation - clink.npu.retired := accelerator.io.retired - clink.npu.fault := accelerator.io.fault - clink.npu.busy := !accelerator.io.idle - clink.npu.interrupt := accelerator.io.interrupt - clink.npu.footprints := accelerator.io.footprints - clink.mem <> accelerator.io.dma - accelerator.io.hartid := clink.ctrl.hartId - accelerator.io.sharedBankOwnerHartId := clink.ctrl.shmOwner - clink.shm.requests <> accelerator.io.shared_mem_req - clink.shm.move <> accelerator.io.mvover - accelerator.io.meshLocalBank <> clink.shm.local - clink.shm.config <> accelerator.io.shared_config - clink.shm.queryValid := accelerator.io.shared_query_valid - clink.shm.queryVbank := accelerator.io.shared_query_vbank_id - accelerator.io.shared_query_group_count := clink.shm.queryGroups - clink.shm.barrierArrive := accelerator.io.barrier_arrive - accelerator.io.barrier_release := clink.shm.barrierRelease - accelerator.io.shared_bank_hashes.foreach(_ := clink.sharedHashes.get) - } - -} diff --git a/arch/src/main/scala/framework/system/core/CoreFactory.scala b/arch/src/main/scala/framework/system/core/CoreFactory.scala deleted file mode 100644 index 7b65ded35..000000000 --- a/arch/src/main/scala/framework/system/core/CoreFactory.scala +++ /dev/null @@ -1,24 +0,0 @@ -package framework.system.core - -import chisel3.experimental.hierarchy.Instance -import framework.top.GlobalConfig - -trait RocketCoreFactory { - def instantiate(b: GlobalConfig): Instance[RocketEndpoint] -} - -trait AntCoreFactory { - def instantiate(b: GlobalConfig): Instance[AntEndpoint] -} - -object CoreFactory { - - def rocket(b: GlobalConfig): Instance[RocketEndpoint] = - Class.forName(b.coreDesign.get.factory + "$") - .getField("MODULE$").get(null).asInstanceOf[RocketCoreFactory].instantiate(b) - - def ant(b: GlobalConfig): Instance[AntEndpoint] = - Class.forName(b.coreDesign.get.factory + "$") - .getField("MODULE$").get(null).asInstanceOf[AntCoreFactory].instantiate(b) - -} diff --git a/arch/src/main/scala/framework/system/core/CoreParams.scala b/arch/src/main/scala/framework/system/core/CoreParams.scala deleted file mode 100644 index 9fce8d980..000000000 --- a/arch/src/main/scala/framework/system/core/CoreParams.scala +++ /dev/null @@ -1,15 +0,0 @@ -package framework.system.core - -import framework.ant.{Params => AntParams} -import framework.system.core.rocket.CpuParams -import framework.top.GlobalConfig -import memcore.bus.chi -import memcore.memory.interlock.{Params => TrackingParams} - -case class AntCoreParams( - buckyball: GlobalConfig, - local: AntParams, - tracking: TrackingParams, - bus: chi.Params, - signature: BigInt, - cpu: CpuParams) diff --git a/arch/src/main/scala/framework/system/core/RocketEndpoint.scala b/arch/src/main/scala/framework/system/core/RocketEndpoint.scala deleted file mode 100644 index aa8aa0ac9..000000000 --- a/arch/src/main/scala/framework/system/core/RocketEndpoint.scala +++ /dev/null @@ -1,49 +0,0 @@ -package framework.system.core - -import framework.system.GlobalConfigOps._ -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.system.core.clink.{CLink, HasCLink, NpuEvents, ShmPort} -import framework.system.core.rocket.CpuParams -import framework.top.GlobalConfig -import framework.memdomain.frontend.mem.dma.DmaPort -import framework.memdomain.backend.banks.btrace.PhysicalBankHash -import framework.memdomain.backend.shared.SharedMemLayout -import hier.core.rocket.{Commands, CpuCLink} -import memcore.bus.chi.rnf.RnfParams -import memcore.memory.cpu.PhysicalRegion - -case class RocketCoreParams( - cpu: CpuParams, - data: RnfParams, - instruction: RnfParams, - regions: Seq[PhysicalRegion], - commands: Commands, - buckyball: Option[GlobalConfig]) - -class AcceleratorCLink(b: GlobalConfig) extends Bundle { - val hartId = Input(UInt(64.W)) - val shmOwner = Input(UInt(64.W)) - val npu = new NpuEvents(b) - val shm = new ShmPort(b) - val mem = new DmaPort(b.memDomain.dma_buswidth) - - val sharedHashes = - if (b.sim.diffTest && b.memDomain.sharedEnable) - Some(Input(Vec(SharedMemLayout.totalBank(b), new PhysicalBankHash(b)))) - else None - -} - -class RocketCLink(p: RocketCoreParams) extends CLink { - val cpu = new CpuCLink(p.data, Some(p.commands))(p.cpu) - val accelerator = p.buckyball.map(new AcceleratorCLink(_)) -} - -/** Reusable implementation; the concrete design and class selection live in examples/cores. */ -@instantiable -abstract class RocketEndpoint(b: framework.top.GlobalConfig) extends Module with HasCLink { - val p = b.rocketParams - @public val clink = IO(new RocketCLink(p)) -} diff --git a/arch/src/main/scala/framework/system/core/accelerator/Admission.scala b/arch/src/main/scala/framework/system/core/accelerator/Admission.scala deleted file mode 100644 index cb5940491..000000000 --- a/arch/src/main/scala/framework/system/core/accelerator/Admission.scala +++ /dev/null @@ -1,420 +0,0 @@ -package framework.system.core.accelerator - -import memcore.memory.queue.Queue - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.system.core.rocket.{CpuParams, HasCpuParameters} -import framework.top.GlobalConfig -import framework.system.core.rocket.{RoCCCommandBB, RoCCIO, RoCCResponseBB} -import framework.frontend.decoder.GISA -import framework.memdomain.frontend.cmd.decoder.DISA -import framework.memdomain.frontend.mem.Footprint -import framework.memdomain.frontend.mem.dma.{Dma, DmaDecision, DmaError, DmaPort, DmaStatus} -import hier.core.rocket.{AdmissionBridge, AdmissionPorts, Preparation, RobFault} -import hier.tile.TaskAdmission -import memcore.memory.cpu.PhysicalRegion -import memcore.memory.interlock.{AccessInfo, Interlock, Params => TrackingParams} -import memcore.memory.preflight.{Command, Error, Params => PreparationParams} -import memcore.bus.axi4 - -/** Per-compute-core admission, physical preparation and completion join. */ -@instantiable -class Admission( - b: GlobalConfig, - tracking: TrackingParams, - prepared: PreparationParams, - regions: Seq[PhysicalRegion], - axiParams: axi4.Params -)( - implicit val cpuParams: CpuParams) - extends Module - with HasCpuParameters { - require(tracking.entries == 4 && tracking.idBits == prepared.idBits) - require(tracking.addressBits == prepared.bus.addressBits && tracking.maxRanges == prepared.maxRanges) - require(prepared.beatBytes == axiParams.bytes && b.memDomain.dma_buswidth == axiParams.dataBits) - val parents = tracking.entries - val n = 8 - val internalTracking = tracking.copy(entries = n) - val internalPrepared = prepared.copy(contexts = n) - val indexBits = log2Ceil(n) - require(tracking.idBits >= indexBits) - - @public - val io = IO(new Bundle { - val core = Flipped(new AdmissionPorts(tracking, nPMPs, prepared.bus)) - val npuCommand = Decoupled(new RoCCCommandBB) - val npuResponse = Flipped(Decoupled(new RoCCResponseBB)) - val allocation = Flipped(Valid(UInt(log2Ceil(b.frontend.rob_entries).W))) - val retired = Input(UInt(b.frontend.rob_entries.W)) - val npuFault = Flipped(Valid(new RobFault(b.frontend.rob_entries))) - // Actual scheduler/boot work, not merely RoCC ROB backpressure. - val npuBusy = Input(Bool()) - val npuInterrupt = Input(Bool()) - val footprints = Input(Vec(3, new Footprint(b))) - val dma = Flipped(new DmaPort(b.memDomain.dma_buswidth)) - val task = Flipped(new RoCCIO(64)) - val taskSatp = Output(UInt(64.W)) - val axi = new axi4.Port(axiParams) - val halted = Output(Bool()) - val fault = Output(new DmaStatus) - val faultTag = Output(UInt(tracking.idBits.W)) - val faultHasTag = Output(Bool()) - val workDrained = Output(Bool()) - }) - - val bridge: Instance[AdmissionBridge] = Instantiate(new AdmissionBridge( - b.frontend.rob_entries, - tracking, - nPMPs, - Seq(GISA.BARRIER_BITPAT.value.toInt) - )) - - val interlock: Instance[Interlock] = Instantiate(new Interlock(internalTracking)) - val preparation: Instance[Preparation] = Instantiate(new Preparation(internalPrepared, regions)) - val dma: Instance[Dma] = Instantiate(new Dma(b, internalPrepared, axiParams)) - val task: Instance[TaskAdmission] = Instantiate(new TaskAdmission(tracking, nPMPs)) - - val live = RegInit(VecInit(Seq.fill(n)(false.B))) - val memory = RegInit(VecInit(Seq.fill(n)(false.B))) - val barrier = RegInit(VecInit(Seq.fill(n)(false.B))) - val retirementSeen = RegInit(VecInit(Seq.fill(n)(false.B))) - val interlockDone = RegInit(VecInit(Seq.fill(n)(false.B))) - val noMemoryPending = RegInit(VecInit(Seq.fill(n)(false.B))) - val shapeClaimed = RegInit(VecInit(Seq.fill(n)(false.B))) - val preparedStarted = RegInit(VecInit(Seq.fill(n)(false.B))) - val mapReady = RegInit(VecInit(Seq.fill(n)(false.B))) - val grantSeen = RegInit(VecInit(Seq.fill(n)(false.B))) - val doneIssued = RegInit(VecInit(Seq.fill(n)(false.B))) - val errors = RegInit(VecInit(Seq.fill(n)(0.U.asTypeOf(new DmaStatus)))) - val halted = RegInit(false.B) - val firstFault = RegInit(0.U.asTypeOf(new DmaStatus)) - val firstTag = RegInit(0.U(tracking.idBits.W)) - val firstHasTag = RegInit(false.B) - val parentTags = Reg(Vec(n, UInt(tracking.idBits.W))) - val childDispatchPending = RegInit(VecInit(Seq.fill(3)(false.B))) - def index(tag: UInt): UInt = tag(indexBits - 1, 0) - def known(tag: UInt): Bool = tag < n.U && live(index(tag)) - - def stop( - error: UInt, - address: UInt, - tag: UInt, - hasTag: Bool - ): Unit = { - when(!halted) { - halted := true.B; firstFault.error := error; firstFault.address := address - firstTag := Mux(tag >= parents.U && known(tag), parentTags(index(tag)), tag) - firstHasTag := hasTag - } - } - - def dmaError(error: UInt): UInt = MuxLookup(error, DmaError.Context.U)(Seq( - Error.Ok.U -> DmaError.None.U, - Error.Shape.U -> DmaError.Shape.U, - Error.Overflow.U -> DmaError.Shape.U, - Error.PageFault.U -> DmaError.PageFault.U, - Error.AccessFault.U -> DmaError.AccessFault.U, - Error.Capacity.U -> DmaError.Capacity.U, - Error.Context.U -> DmaError.Context.U - )) - - val command = io.core.command - val isTask = command.bits.instruction.opcode === "h2b".U - val isBarrier = !isTask && command.bits.instruction.funct === GISA.BARRIER_BITPAT - - val isMemory = !isTask && (command.bits.instruction.funct === DISA.MVIN_BITPAT || - command.bits.instruction.funct === DISA.MVIN_2D_BITPAT || - command.bits.instruction.funct === DISA.MVIN_MMIO_BITPAT || - command.bits.instruction.funct === DISA.MVOUT_BITPAT || - command.bits.instruction.funct === GISA.MVIN_KERNEL_BITPAT) - - val barrierProtect = barrier.asUInt.orR || (command.valid && isBarrier) - - val unknownSubMemory = - if (b.frontend.sub_rob_enable) - VecInit((0 until parents).map(i => live(i) && !memory(i) && !retirementSeen(i) && !barrier(i))).asUInt.orR - else false.B - - val childDispatch = childDispatchPending.asUInt.orR - val dispatchProducer = PriorityEncoder(childDispatchPending) - interlock.io.dispatch.valid := !halted && (childDispatch || - (io.core.reserve.valid && !barrierProtect && !unknownSubMemory)) - interlock.io.dispatch.bits := io.core.reserve.bits - when(childDispatch)(interlock.io.dispatch.bits.id := parents.U + dispatchProducer) - io.core.reserve.ready := interlock.io.dispatch.ready && !halted && - !barrierProtect && !unknownSubMemory && !childDispatch - when(interlock.io.dispatch.fire && childDispatch)(childDispatchPending(dispatchProducer) := false.B) - when(io.core.reserve.fire) { - val tag = io.core.reserve.bits.id - assert(tag < parents.U && !live(index(tag)), "Admission reused a live Core tag") - parentTags(index(tag)) := tag - live(index(tag)) := true.B; memory(index(tag)) := false.B; barrier(index(tag)) := false.B - retirementSeen(index(tag)) := false.B; interlockDone(index(tag)) := false.B - noMemoryPending(index(tag)) := false.B; shapeClaimed(index(tag)) := false.B; preparedStarted(index(tag)) := false.B - mapReady(index(tag)) := false.B; grantSeen(index(tag)) := false.B; doneIssued(index(tag)) := false.B - errors(index(tag)) := 0.U.asTypeOf(new DmaStatus) - } - bridge.io.command.valid := command.valid && !isTask && !halted - bridge.io.command.bits := command.bits - task.io.command.valid := command.valid && isTask && !halted - task.io.command.bits := command.bits - command.ready := !halted && Mux(isTask, task.io.command.ready, bridge.io.command.ready) - when(command.fire) { - assert(command.bits.tag < parents.U && known(command.bits.tag), "Admission command has no reserved Core tag") - val slot = index(command.bits.tag) - memory(slot) := isMemory; barrier(slot) := isBarrier - noMemoryPending(slot) := !isMemory - } - io.npuCommand <> bridge.io.npuCommand - bridge.io.allocation := io.allocation; bridge.io.retired := io.retired; bridge.io.fault := io.npuFault - io.task <> task.io.task; io.taskSatp := task.io.satp - val responses = Module(new Arbiter(new RoCCResponseBB, 2)) - responses.io.in(0) <> task.io.response; responses.io.in(1) <> io.npuResponse - io.core.response <> responses.io.out - io.core.interrupt := io.npuInterrupt || io.task.interrupt - interlock.io.cpuQuery := io.core.cpuQuery - io.core.cpuAllow := interlock.io.cpuAllow && !halted && !barrierProtect && !unknownSubMemory - io.core.cpuProbeAllow := interlock.io.cpuProbeAllow && !halted && !barrierProtect && !unknownSubMemory - io.core.maintenance <> interlock.io.maintenance - interlock.io.maintained <> io.core.maintained - io.core.pteRequest <> preparation.io.pteRequest - preparation.io.pteResponse <> io.core.pteResponse - interlock.io.cancel.valid := false.B; interlock.io.cancel.bits := 0.U.asTypeOf(interlock.io.cancel.bits) - io.core.cancelled.valid := false.B; io.core.cancelled.bits := 0.U.asTypeOf(io.core.cancelled.bits) - - // Notices never wait for other tags: otherwise a high-priority barrier notice - // could prevent older ROB retirements from reaching the completion join. - bridge.io.retirement.ready := true.B - when(bridge.io.retirement.fire) { - val notice = bridge.io.retirement.bits - assert(known(notice.snapshot.tag), "Admission retirement has no live Core tag") - retirementSeen(index(notice.snapshot.tag)) := true.B - when(notice.error =/= DmaError.None.U) { - errors(index(notice.snapshot.tag)).error := notice.error - errors(index(notice.snapshot.tag)).address := notice.address - stop(notice.error, notice.address, notice.snapshot.tag, true.B) - } - } - task.io.release.ready := true.B - when(task.io.release.fire) { - assert(known(task.io.release.bits.tag), "Task release has no live Core tag") - retirementSeen(index(task.io.release.bits.tag)) := true.B - } - interlock.io.complete.ready := true.B - when(interlock.io.complete.fire) { - assert(known(interlock.io.complete.bits.tag), "Interlock complete has no live Core tag") - interlockDone(index(interlock.io.complete.bits.tag)) := true.B - } - when(bridge.io.unboundFault.valid) { - stop(bridge.io.unboundFault.bits.error, bridge.io.unboundFault.bits.address, 0.U, false.B) - } - - val producerBound = RegInit(VecInit(Seq.fill(3)(false.B))) - val producerTag = Reg(Vec(3, UInt(tracking.idBits.W))) - val producerSub = RegInit(VecInit(Seq.fill(3)(false.B))) - val producerRob = Reg(Vec(3, UInt(log2Ceil(b.frontend.rob_entries).W))) - val producerSubRob = Reg(Vec(3, UInt(log2Ceil(b.frontend.sub_rob_depth * 4).W))) - val producerMatches = Wire(Vec(3, Bool())) - val shapes = Reg(Vec(3, new Command(prepared))) - val shapePending = RegInit(VecInit(Seq.fill(3)(false.B))) - for (i <- 0 until 3) { - bridge.io.lookup(i).robId := io.footprints(i).rob_id - val footprint = io.footprints(i) - val snapshot = bridge.io.lookup(i).snapshot - val childTag = (parents + i).U(tracking.idBits.W) - producerMatches(i) := footprint.valid && footprint.rob_id === producerRob(i) && - footprint.is_sub === producerSub(i) && (!producerSub(i) || footprint.sub_rob_id === producerSubRob(i)) - when(producerBound(i) && !producerMatches(i)) { - when(producerSub(i))(retirementSeen(parents + i) := true.B) - .otherwise(producerBound(i) := false.B) - } - when(footprint.valid && !producerBound(i) && (!footprint.is_sub || !live(parents + i)) && - !halted && !io.npuFault.valid) { - when(!snapshot.valid || !known(snapshot.bits.tag) || - (!footprint.is_sub && !memory(index(snapshot.bits.tag))) || - (footprint.is_sub && !b.frontend.sub_rob_enable.B)) { - stop(DmaError.Context.U, footprint.baseVA, snapshot.bits.tag, snapshot.valid) - }.otherwise { - val tag = Mux(footprint.is_sub, childTag, snapshot.bits.tag); val slot = index(tag) - when(footprint.is_sub) { - live(slot) := true.B; memory(slot) := true.B; barrier(slot) := false.B - parentTags(slot) := snapshot.bits.tag - retirementSeen(slot) := false.B; interlockDone(slot) := false.B - noMemoryPending(slot) := false.B; preparedStarted(slot) := false.B - mapReady(slot) := false.B; grantSeen(slot) := false.B; doneIssued(slot) := false.B - errors(slot) := 0.U.asTypeOf(new DmaStatus) - childDispatchPending(i) := true.B - } - producerSub(i) := footprint.is_sub - producerRob(i) := footprint.rob_id; producerSubRob(i) := footprint.sub_rob_id - val narrow = footprint.rows(63, 32).orR || footprint.columns(63, 32).orR || - footprint.spanBytes(63, 32).orR || footprint.columnStride(63, 32).orR || footprint.rowStride(63, 32).orR - producerBound(i) := true.B; producerTag(i) := tag - shapes(i).id := tag; shapes(i).baseVA := footprint.baseVA - shapes(i).rows := footprint.rows(31, 0); shapes(i).columns := footprint.columns(31, 0) - shapes(i).spanBytes := footprint.spanBytes(31, 0); shapes(i).columnStride := footprint.columnStride(31, 0) - shapes(i).rowStride := footprint.rowStride(31, 0); shapes(i).write := footprint.write - shapes(i).mode := snapshot.bits.satp(63, 60); shapes(i).rootPpn := snapshot.bits.satp(43, 0) - shapes(i).privilege := snapshot.bits.effectivePrivilege - shapes(i).sum := snapshot.bits.sum; shapes(i).mxr := snapshot.bits.mxr - when(narrow || footprint.fault.error =/= DmaError.None.U) { - val error = Mux(narrow, DmaError.Shape.U, footprint.fault.error) - errors(slot).error := error - errors(slot).address := Mux(narrow, footprint.baseVA, footprint.fault.address) - stop(error, Mux(narrow, footprint.baseVA, footprint.fault.address), snapshot.bits.tag, true.B) - }.otherwise { - when(!footprint.is_sub && shapeClaimed(slot)) { - errors(slot).error := DmaError.Context.U; errors(slot).address := footprint.baseVA - stop(DmaError.Context.U, footprint.baseVA, tag, true.B) - }.otherwise { shapeClaimed(slot) := true.B; shapePending(i) := true.B } - } - } - } - } - val prepareOffer = RegInit(false.B) - val prepareProducer = Reg(UInt(2.W)) - when(!prepareOffer && shapePending.asUInt.orR && !halted) { - prepareOffer := true.B; prepareProducer := PriorityEncoder(shapePending) - } - preparation.io.command.valid := prepareOffer && !halted - preparation.io.command.bits := shapes(prepareProducer) - when(preparation.io.command.fire) { - prepareOffer := false.B; shapePending(prepareProducer) := false.B - preparedStarted(index(shapes(prepareProducer).id)) := true.B - } - val preparationTag = preparation.io.contextTag - bridge.io.lookupTag.tag := Mux(preparationTag >= parents.U, parentTags(index(preparationTag)), preparationTag) - preparation.io.contextValid := bridge.io.lookupTag.snapshot.valid - preparation.io.contextId := preparationTag - preparation.io.contextPmp := bridge.io.lookupTag.snapshot.bits.pmp - - val infos = Module(new Queue(new AccessInfo(internalTracking), 2)) - interlock.io.accessInfo <> infos.io.deq - val range = preparation.io.ranges - val infoOffer = RegInit(false.B) - val infoRange = RegInit(false.B) - val infoPacket = Reg(new AccessInfo(internalTracking)) - val pure = PriorityEncoder(noMemoryPending) - when(!infoOffer && !halted && - ((range.valid && range.bits.error === Error.Ok.U) || (!range.valid && noMemoryPending.asUInt.orR))) { - infoOffer := true.B; infoRange := range.valid && range.bits.error === Error.Ok.U - infoPacket := 0.U.asTypeOf(infoPacket) - infoPacket.id := Mux(range.valid, range.bits.id, pure) - infoPacket.hasMemory := range.valid - infoPacket.base := Mux(range.valid, range.bits.pa, 0.U) - infoPacket.bytes := Mux(range.valid, range.bits.bytes, 0.U) - infoPacket.write := range.valid && range.bits.write - infoPacket.last := Mux(range.valid, range.bits.last, true.B) - } - infos.io.enq.valid := infoOffer && !halted - infos.io.enq.bits := infoPacket - range.ready := Mux(range.bits.error =/= Error.Ok.U, true.B, infoOffer && infoRange && infos.io.enq.ready && !halted) - when(infos.io.enq.fire) { - infoOffer := false.B - when(!infoRange)(noMemoryPending(index(infoPacket.id)) := false.B) - } - when(range.fire && range.bits.error =/= Error.Ok.U) { - val error = dmaError(range.bits.error) - errors(index(range.bits.id)).error := error; errors(index(range.bits.id)).address := range.bits.va - stop(error, range.bits.va, range.bits.id, true.B) - } - preparation.io.ready.ready := true.B - when(preparation.io.ready.fire) { - val value = preparation.io.ready.bits - mapReady(index(value.id)) := true.B - when(value.error =/= Error.Ok.U) { - errors(index(value.id)).error := dmaError(value.error); errors(index(value.id)).address := value.va - stop(dmaError(value.error), value.va, value.id, true.B) - } - } - interlock.io.grant.ready := true.B - when(interlock.io.grant.fire)(grantSeen(index(interlock.io.grant.bits.tag)) := true.B) - - preparation.io.queries := dma.io.queries - for (i <- 0 until 2) { - dma.io.mappings(i) := preparation.io.results(i) - when(halted) { - dma.io.mappings(i).hit := false.B; dma.io.mappings(i).pa := 0.U - dma.io.mappings(i).error := Error.Context.U - } - } - when(io.npuFault.valid && io.npuFault.bits.error =/= DmaError.None.U) { - stop(io.npuFault.bits.error, io.npuFault.bits.address, bridge.io.faultTag.bits, bridge.io.faultTag.valid) - } - val readOwnerTag = Reg(UInt(tracking.idBits.W)) - val writeOwnerTag = Reg(UInt(tracking.idBits.W)) - when(io.dma.read.fire)(readOwnerTag := Mux(io.dma.read.bits.producer === 2.U, producerTag(2), producerTag(0))) - when(io.dma.write.fire)(writeOwnerTag := producerTag(1)) - dma.io.dma <> io.dma - io.axi <> dma.io.axi - for (i <- 0 until 3) { - val tag = producerTag(i); val slot = index(tag) - dma.io.decisions(i) := 0.U.asTypeOf(new DmaDecision(prepared)) - dma.io.decisions(i).parentId := tag - dma.io.decisions(i).valid := producerBound(i) && producerMatches(i) && - (!producerSub(i) || !retirementSeen(slot)) && - ((mapReady(slot) && grantSeen(slot)) || errors(slot).error =/= DmaError.None.U || halted) - dma.io.decisions(i).fault := Mux(halted, firstFault, errors(slot)) - } - when(io.dma.readResult.fire && io.dma.readResult.bits.fault.error =/= DmaError.None.U) { - stop(io.dma.readResult.bits.fault.error, io.dma.readResult.bits.fault.address, readOwnerTag, true.B) - } - when(io.dma.writeResult.fire && io.dma.writeResult.bits.fault.error =/= DmaError.None.U) { - stop(io.dma.writeResult.bits.fault.error, io.dma.writeResult.bits.fault.address, writeOwnerTag, true.B) - } - - val doneCandidates = VecInit((0 until n).map(i => - live(i) && memory(i) && retirementSeen(i) && - mapReady(i) && grantSeen(i) && !doneIssued(i) && errors(i).error === DmaError.None.U - )) - - val doneOffer = RegInit(false.B) - val doneTag = Reg(UInt(indexBits.W)) - when(!doneOffer && doneCandidates.asUInt.orR && !halted) { - doneOffer := true.B; doneTag := PriorityEncoder(doneCandidates) - } - interlock.io.done.valid := doneOffer && !halted - interlock.io.done.bits.tag := doneTag; interlock.io.done.bits.ok := true.B - when(interlock.io.done.fire) { doneIssued(doneTag) := true.B; doneOffer := false.B } - - val releasable = VecInit((0 until n).map(i => - live(i) && retirementSeen(i) && interlockDone(i) && - errors(i).error === DmaError.None.U && (!barrier(i) || - (PopCount(live) === 1.U && !io.npuBusy && !io.dma.readBusy && !io.dma.writeBusy)) && - (if (i < parents) !VecInit((parents until parents + 3).map(j => live(j) && parentTags(j) === i.U)).asUInt.orR - else true.B) - )) - - val releaseOffer = RegInit(false.B) - val releaseTag = Reg(UInt(indexBits.W)) - when(!releaseOffer && releasable.asUInt.orR && !halted) { - releaseOffer := true.B; releaseTag := PriorityEncoder(releasable) - } - val releasingMap = memory(releaseTag) && preparedStarted(releaseTag) - val releasingChild = releaseTag >= parents.U - val releaseReady = releasingChild || io.core.complete.ready - preparation.io.release.valid := releaseOffer && releasingMap && releaseReady && !halted - preparation.io.release.bits.id := releaseTag - io.core.complete.valid := releaseOffer && !releasingChild && !halted && (!releasingMap || preparation.io.release.ready) - io.core.complete.bits.tag := releaseTag - val releaseFire = releaseOffer && releaseReady && !halted && (!releasingMap || preparation.io.release.ready) - when(releaseFire) { - assert(!releasingMap || preparation.io.release.fire, "Admission map/Core release is not atomic") - live(releaseTag) := false.B; barrier(releaseTag) := false.B; releaseOffer := false.B - for (i <- 0 until 3) { - when(releaseTag === (parents + i).U)(producerBound(i) := false.B) - } - } - val otherLive = (live.asUInt & ~(1.U(n.W) << index(command.bits.tag))).orR - - val drained = !otherLive && !io.npuBusy && !io.dma.readBusy && !io.dma.writeBusy && - !prepareOffer && !shapePending.asUInt.orR && - !interlock.io.maintenance.valid && !halted - - task.io.workDrained := drained - io.workDrained := !live.asUInt.orR && !io.npuBusy && !io.dma.readBusy && !io.dma.writeBusy && - !halted - io.halted := halted; io.fault := firstFault; io.faultTag := firstTag; io.faultHasTag := firstHasTag -} diff --git a/arch/src/main/scala/framework/system/core/accelerator/AntAdmission.scala b/arch/src/main/scala/framework/system/core/accelerator/AntAdmission.scala deleted file mode 100644 index 46188b2c4..000000000 --- a/arch/src/main/scala/framework/system/core/accelerator/AntAdmission.scala +++ /dev/null @@ -1,147 +0,0 @@ -package framework.system.core.accelerator - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.ant -import framework.system.core.rocket.CpuParams -import hier.core.rocket.{AdmissionPorts, CommandSnapshot} -import memcore.bus.chi -import memcore.bus.chi.rnf.{CacheAccess, CacheResult} -import memcore.memory.interlock.{Acknowledgement, CpuQuery, Maintenance, Params => TrackingParams} - -/** Controller-side cache services; requests retain their original admission tag. */ -class AntMemoryPort(tracking: TrackingParams, bus: chi.Params) extends Bundle { - val cpuQuery = Input(new CpuQuery(tracking)) - val cpuAllow = Output(Bool()) - val cpuProbeAllow = Output(Bool()) - val maintenance = Decoupled(new Maintenance(tracking)) - val maintained = Flipped(Decoupled(new Acknowledgement(tracking))) - val pteRequest = Decoupled(new CacheAccess(bus)) - val pteResponse = Flipped(Decoupled(new CacheResult)) -} - -/** Binds local Ant execution to the controller's immutable task address-space snapshot. */ -@instantiable -class AntAdmission( - p: ant.Params, - tracking: TrackingParams, - pmps: Int, - bus: chi.Params -)( - implicit val cpuParams: CpuParams) - extends Module { - require(tracking.entries == 4) - - @public val io = IO(new Bundle { - - val bind = Flipped(Valid(new Bundle { - val task = UInt(p.taskBits.W) - val snapshot = new CommandSnapshot(tracking, pmps) - })) - - val command = Flipped(Decoupled(new ant.Command(p))) - val response = Decoupled(new ant.Response(p)) - val cancel = Input(Bool()) - val workDrained = Input(Bool()) - val halted = Input(Bool()) - val drained = Output(Bool()) - val admission = new AdmissionPorts(tracking, pmps, bus) - val memory = new AntMemoryPort(tracking, bus) - }) - - val bound = RegInit(false.B) - val task = Reg(UInt(p.taskBits.W)) - val context = Reg(new CommandSnapshot(tracking, pmps)) - val live = RegInit(VecInit(Seq.fill(tracking.entries)(false.B))) - val sent = RegInit(VecInit(Seq.fill(tracking.entries)(false.B))) - val pending = RegInit(false.B) - val packet = Reg(new CommandSnapshot(tracking, pmps)) - val responsePending = RegInit(false.B) - val responseRd = Reg(UInt(5.W)) - val responseDelivered = RegInit(false.B) - val free = VecInit(live.map(!_)).asUInt - val selected = PriorityEncoder(free) - val port = io.admission - io.drained := !live.asUInt.orR && !pending && !responsePending && io.workDrained - when(io.bind.valid) { - assert(io.drained && !io.command.valid, "Ant context rebound before previous task drained") - bound := true.B - task := io.bind.bits.task - context := io.bind.bits.snapshot - } - val capacity = bound && free.orR && !pending && !responsePending && !io.cancel && !io.bind.valid && !reset.asBool - port.reserve.valid := io.command.valid && capacity - port.reserve.bits.id := selected - io.command.ready := capacity && port.reserve.ready - port.command.valid := pending - port.command.bits := packet - when(io.command.fire) { - val instruction = io.command.bits.instruction - assert(io.command.bits.task === task, "Ant command uses another task context") - assert(instruction(6, 0) === "h7b".U, "Ant admission only accepts NPU commands") - packet := context - packet.tag := selected - packet.instruction.raw_inst := instruction - packet.instruction.pc := io.command.bits.pc - packet.instruction.funct := instruction(31, 25) - packet.instruction.funct3 := instruction(14, 12) - packet.instruction.rs2 := instruction(24, 20) - packet.instruction.rs1 := instruction(19, 15) - packet.instruction.xd := instruction(14) - packet.instruction.xs1 := instruction(13) - packet.instruction.xs2 := instruction(12) - packet.instruction.rd := instruction(11, 7) - packet.instruction.opcode := instruction(6, 0) - packet.instruction.rs1Data := io.command.bits.rs1 - packet.instruction.rs2Data := io.command.bits.rs2 - live(selected) := true.B - sent(selected) := false.B - pending := true.B - when(instruction(14)) { - responsePending := true.B - responseDelivered := false.B - responseRd := instruction(11, 7) - } - } - when(port.command.fire) { - pending := false.B - sent(packet.tag(1, 0)) := true.B - when(packet.instruction.xd)(responseDelivered := true.B) - } - port.complete.ready := true.B - when(port.complete.fire) { - val tag = port.complete.bits.tag - assert( - tag < tracking.entries.U && live(tag(1, 0)) && sent(tag(1, 0)), - "Ant completion has no delivered live command" - ) - live(tag(1, 0)) := false.B - sent(tag(1, 0)) := false.B - } - port.cancelled.ready := true.B - assert(!port.cancelled.valid, "Ant cancellation must drain accepted NPU commands") - assert(!io.halted && !port.interrupt, "Ant NPU backend fault") - port.outstanding := PopCount(live) - io.response.valid := port.response.valid && responsePending - io.response.bits.task := task - io.response.bits.rd := port.response.bits.rd - io.response.bits.data := port.response.bits.data - io.response.bits.error := false.B - port.response.ready := io.response.ready && responsePending - when(port.response.valid) { - assert( - responsePending && (responseDelivered || (port.command.fire && packet.instruction.xd)), - "Ant received an unsolicited NPU response" - ) - assert(port.response.bits.rd === responseRd, "Ant NPU response destination changed") - } - when(port.response.fire)(responsePending := false.B) - port.cpuQuery := io.memory.cpuQuery - io.memory.cpuAllow := port.cpuAllow - io.memory.cpuProbeAllow := port.cpuProbeAllow - io.memory.maintenance <> port.maintenance - port.maintained <> io.memory.maintained - io.memory.pteRequest <> port.pteRequest - port.pteResponse <> io.memory.pteResponse -} diff --git a/arch/src/main/scala/framework/system/core/accelerator/AntMemory.scala b/arch/src/main/scala/framework/system/core/accelerator/AntMemory.scala deleted file mode 100644 index a374fd1ba..000000000 --- a/arch/src/main/scala/framework/system/core/accelerator/AntMemory.scala +++ /dev/null @@ -1,73 +0,0 @@ -package framework.system.core.accelerator - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import memcore.bus.chi -import memcore.memory.interlock.{Params => TrackingParams} - -/** Independent fair, single-owner cache-maintenance and page-table service arbiters. */ -@instantiable -class AntMemory(ports: Int, tracking: TrackingParams, bus: chi.Params) extends Module { - require(ports > 0) - - @public val io = IO(new Bundle { - val clients = Vec(ports, Flipped(new AntMemoryPort(tracking, bus))) - val controller = new AntMemoryPort(tracking, bus) - }) - - io.controller.cpuAllow := io.clients.map(_.cpuAllow).reduce(_ && _) - io.controller.cpuProbeAllow := io.clients.map(_.cpuProbeAllow).reduce(_ && _) - io.clients.foreach(_.cpuQuery := io.controller.cpuQuery) - - def arbitrate[T <: Data, R <: Data]( - requests: Seq[DecoupledIO[T]], - responses: Seq[DecoupledIO[R]], - request: DecoupledIO[T], - response: DecoupledIO[R] - ): Unit = { - val ownerBits = math.max(1, log2Ceil(ports)) - val next = RegInit(0.U(ownerBits.W)) - val owner = Reg(UInt(ownerBits.W)) - val occupied = RegInit(false.B) - val sent = RegInit(false.B) - val packet = Reg(chiselTypeOf(request.bits)) - val candidates = VecInit(requests.map(_.valid)).asUInt - val later = VecInit(requests.zipWithIndex.map { case (r, i) => r.valid && i.U >= next }).asUInt - val selected = PriorityEncoder(Mux(later.orR, later, candidates)) - for (i <- 0 until ports) { - requests(i).ready := !occupied && selected === i.U && !reset.asBool - responses(i).valid := occupied && sent && owner === i.U && response.valid - responses(i).bits := response.bits - when(requests(i).fire) { - packet := requests(i).bits - owner := i.U - occupied := true.B - sent := false.B - next := (if (i + 1 == ports) 0 else i + 1).U - } - } - request.valid := occupied && !sent && !reset.asBool - request.bits := packet - when(request.fire)(sent := true.B) - response.ready := occupied && sent && VecInit(responses.map(_.ready))(owner) && !reset.asBool - when(response.valid)(assert(occupied && sent, "Ant cache service returned a response without a sent owner")) - when(response.fire) { - occupied := false.B - sent := false.B - } - } - - arbitrate( - io.clients.map(_.maintenance), - io.clients.map(_.maintained), - io.controller.maintenance, - io.controller.maintained - ) - arbitrate( - io.clients.map(_.pteRequest), - io.clients.map(_.pteResponse), - io.controller.pteRequest, - io.controller.pteResponse - ) -} diff --git a/arch/src/main/scala/framework/system/core/accelerator/BuckyballAccelerator.scala b/arch/src/main/scala/framework/system/core/accelerator/BuckyballAccelerator.scala deleted file mode 100644 index 186a95d42..000000000 --- a/arch/src/main/scala/framework/system/core/accelerator/BuckyballAccelerator.scala +++ /dev/null @@ -1,219 +0,0 @@ -package framework.system.core.accelerator - -import memcore.memory.queue.Queue - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} - -import framework.top.GlobalConfig -import framework.frontend.Frontend -import framework.frontend.globalrs.RobAllocation -import hier.core.rocket.RobFault -import framework.system.core.rocket.{RoCCCommandBB, RoCCResponseBB} -import framework.memdomain.MemDomain -import framework.memdomain.backend.MemRequestIO -import framework.memdomain.backend.shared.SharedMemLayout -import framework.memdomain.frontend.mem.{Footprint, MemConfigerIO} -import framework.memdomain.frontend.mem.dma.DmaPort -import framework.balldomain.BallDomain -import framework.memdomain.backend.banks.btrace.PhysicalBankHash -import framework.memdomain.isa.{MvoverISA, MvoverPort} -import memcore.memory.mesh_shm.MeshLocalBankPort - -/** Accelerator domains with an explicit DMA transport boundary owned by the enclosing system. */ -@instantiable -class BuckyballAccelerator(val b: GlobalConfig) extends Module { - val totalBallRead = b.ballDomain.ballIdMappings.map(_.inBW).sum - val totalBallWrite = b.ballDomain.ballIdMappings.map(_.outBW).sum - val sharedHashCount = if (b.memDomain.sharedEnable) SharedMemLayout.totalBank(b) else 0 - - @public - val io = IO(new Bundle { - // RoCC command/response (connected to Rocket core inside tile) - val cmd = Flipped(Decoupled(new RoCCCommandBB(b.tile.xLen))) - val resp = Decoupled(new RoCCResponseBB(b.tile.xLen)) - val busy = Output(Bool()) - val idle = Output(Bool()) - val interrupt = Output(Bool()) - val hartid = Input(UInt(b.tile.xLen.W)) - val sharedBankOwnerHartId = Input(UInt(b.tile.xLen.W)) - // Includes boot allocations; external context binds only on io.cmd.fire. - val allocation = Valid(new RobAllocation(b)) - val retired = Output(UInt(b.frontend.rob_entries.W)) - val fault = Valid(new RobFault(b.frontend.rob_entries)) - val firstFault = Output(Valid(new RobFault(b.frontend.rob_entries))) - val footprints = Output(Vec(3, new Footprint(b))) - - val dma = new DmaPort(b.memDomain.dma_buswidth) - - // Shared memory path — exposed to tile level for multi-core SharedMemBackend - val shared_mem_req = Vec(SharedMemLayout.channelPerHart(b), new MemRequestIO(b)) - val mvover = new MvoverPort - - val meshLocalBank = Flipped(new MeshLocalBankPort( - MvoverISA.AddressBits, - MvoverISA.BankBits, - b.memDomain.bankWidth, - math.max(1, log2Ceil(b.frontend.rob_entries)) - )) - - val shared_config = Decoupled(new MemConfigerIO(b)) - val shared_query_valid = Output(Bool()) - val shared_query_vbank_id = Output(UInt(b.memDomain.vbankIdWidth.W)) - val shared_query_group_count = Input(UInt(b.memDomain.groupCountWidth.W)) - - val shared_bank_hashes = - if (b.sim.diffTest && b.memDomain.sharedEnable) { - Some(Input(Vec(sharedHashCount, new PhysicalBankHash(b)))) - } else { - None - } - - // Barrier interface — connected to tile-level BarrierUnit - val barrier_arrive = Output(Bool()) - val barrier_release = Input(Bool()) - }) - - // --- Instantiate domains --- - val frontend: Instance[Frontend] = Instantiate(new Frontend(b)) - val ballDomain: Instance[BallDomain] = Instantiate(new BallDomain(b)) - val memDomain: Instance[MemDomain] = Instantiate(new MemDomain(b)) - val rvv = if (b.rvv.enable) Some(Instantiate(new Rvv(b))) else None - frontend.io.hartid := io.hartid - frontend.io.sharedBankOwnerHartId := io.sharedBankOwnerHartId - frontend.io.bank_hashes.foreach(_ := memDomain.io.bank_hashes.get) - memDomain.io.shared_bank_hashes.foreach(_ := io.shared_bank_hashes.get) - io.allocation := frontend.io.allocation - io.retired := frontend.io.retired - io.footprints := memDomain.io.footprints - - // --- Frontend <- cmd --- - frontend.io.cmd.valid := io.cmd.valid - frontend.io.cmd.bits.cmd := io.cmd.bits - io.cmd.ready := frontend.io.cmd.ready - - // --- Frontend -> BallDomain --- - ballDomain.global_issue_i <> frontend.io.ball_issue_o - frontend.io.ball_complete_i <> ballDomain.global_complete_o - - // --- BallDomain -> Frontend (SubROB requests) --- - for (i <- 0 until b.ballDomain.ballNum) { - frontend.io.ball_subrob_req_i(i) <> ballDomain.subRobReq(i) - } - - memDomain.io.ballChannelActive := ballDomain.ballChannelActive - ballDomain.ballChannelReady := memDomain.io.ballChannelReady - - // --- Frontend -> MemDomain --- - memDomain.io.global_issue_i <> frontend.io.mem_issue_o - frontend.io.mem_complete_i <> memDomain.io.global_complete_o - // Observe the accepted completion before the scheduler's ID-only queue. - // Its arbiter accepts at most one of these domain completions per cycle. - val memoryFault = memDomain.io.global_complete_o.fire && memDomain.io.global_complete_o.bits.fault.error =/= 0.U - val ballFault = ballDomain.global_complete_o.fire && ballDomain.global_complete_o.bits.fault.error =/= 0.U - val rvvFault = if (b.rvv.enable) rvv.get.complete.fire && rvv.get.complete.bits.fault.error =/= 0.U else false.B - - val faultCompletion = Mux( - memoryFault, - memDomain.io.global_complete_o.bits, - if (b.rvv.enable) Mux(rvvFault, rvv.get.complete.bits, ballDomain.global_complete_o.bits) - else ballDomain.global_complete_o.bits - ) - - io.fault.valid := memoryFault || ballFault || rvvFault - io.fault.bits.rob_id := faultCompletion.rob_id - io.fault.bits.error := faultCompletion.fault.error - io.fault.bits.address := faultCompletion.fault.address - val firstFault = RegInit(0.U.asTypeOf(Valid(new RobFault(b.frontend.rob_entries)))) - when(io.fault.valid && !firstFault.valid)(firstFault := io.fault) - io.firstFault := firstFault - memDomain.io.hartid := io.hartid - memDomain.io.inst_ids := frontend.io.inst_ids - - if (b.rvv.enable) { - val controller = rvv.get - controller.command <> frontend.io.kernel_issue_o.get - frontend.io.kernel_complete_i.get <> controller.complete - controller.memory <> memDomain.io.kernel.get - ballDomain.internalCommand.get <> controller.ballRequest - controller.ballResponse <> ballDomain.internalComplete.get - ballDomain.ownerRobId.get := controller.ownerRobId - ballDomain.kernelBanks.get := controller.banks - } - - // --- BallDomain <-> MemDomain (bankRead with pipeline register to break comb loops) --- - for (i <- 0 until totalBallRead) { - val bankReadReqWithIds = Wire(Decoupled(new Bundle { - val bank_id = chiselTypeOf(ballDomain.bankRead(i).bank_id) - val rob_id = chiselTypeOf(ballDomain.bankRead(i).rob_id) - val ball_id = chiselTypeOf(ballDomain.bankRead(i).ball_id) - val group_id = chiselTypeOf(ballDomain.bankRead(i).group_id) - val req = chiselTypeOf(ballDomain.bankRead(i).io.req.bits) - })) - - bankReadReqWithIds.valid := ballDomain.bankRead(i).io.req.valid - bankReadReqWithIds.bits.bank_id := ballDomain.bankRead(i).bank_id - bankReadReqWithIds.bits.rob_id := ballDomain.bankRead(i).rob_id - bankReadReqWithIds.bits.ball_id := ballDomain.bankRead(i).ball_id - bankReadReqWithIds.bits.group_id := ballDomain.bankRead(i).group_id - bankReadReqWithIds.bits.req := ballDomain.bankRead(i).io.req.bits - ballDomain.bankRead(i).io.req.ready := bankReadReqWithIds.ready - - val bankReadReqQueue = Module(new Queue(chiselTypeOf(bankReadReqWithIds.bits), 8)) - bankReadReqQueue.io.enq <> bankReadReqWithIds - val bankReadReqQ = bankReadReqQueue.io.deq - - memDomain.io.ballDomain.bankRead(i).io.req.valid := bankReadReqQ.valid - memDomain.io.ballDomain.bankRead(i).io.req.bits := bankReadReqQ.bits.req - memDomain.io.ballDomain.bankRead(i).bank_id := bankReadReqQ.bits.bank_id - memDomain.io.ballDomain.bankRead(i).rob_id := bankReadReqQ.bits.rob_id - memDomain.io.ballDomain.bankRead(i).ball_id := bankReadReqQ.bits.ball_id - memDomain.io.ballDomain.bankRead(i).group_id := bankReadReqQ.bits.group_id - bankReadReqQ.ready := memDomain.io.ballDomain.bankRead(i).io.req.ready - - ballDomain.bankRead(i).io.resp <> memDomain.io.ballDomain.bankRead(i).io.resp - } - - ballDomain.bankWrite <> memDomain.io.ballDomain.bankWrite - - // --- BallDomain <-> MemDomain (MMIO read path) --- - ballDomain.mmioRead <> memDomain.io.ballDomain.mmioRead - ballDomain.mmioWrite <> memDomain.io.ballDomain.mmioWrite - - io.dma <> memDomain.io.dma - - // --- Shared memory passthrough --- - io.shared_mem_req <> memDomain.io.shared_mem_req - io.mvover <> memDomain.io.mvover - io.meshLocalBank <> memDomain.io.meshLocalBank - io.shared_config <> memDomain.io.shared_config - io.shared_query_valid := memDomain.io.shared_query_valid - io.shared_query_vbank_id := memDomain.io.shared_query_vbank_id - memDomain.io.shared_query_group_count := io.shared_query_group_count - - // --- Barrier passthrough --- - io.barrier_arrive := frontend.io.barrier_arrive - frontend.io.barrier_release := io.barrier_release - - // --- Response & status --- - io.resp <> frontend.io.resp - io.busy := frontend.io.busy - io.idle := frontend.io.idle && !io.dma.readBusy && !io.dma.writeBusy - // Typed completion failures are delivered through the admission fault path. - // firstFault is retained diagnostic state, not an additional sticky IRQ. - io.interrupt := (if (b.rvv.enable) rvv.get.fault else false.B) - - // --- Busy watchdog --- - // BootRom clears and initializes the local memories before the external - // command interface becomes ready. That initialization can legitimately - // exceed the runtime watchdog limit, so only start monitoring after the - // first external command has handshaken. - val runtime_started = RegInit(false.B) - when(io.cmd.fire) { - runtime_started := true.B - } - val busy_counter = RegInit(0.U(32.W)) - busy_counter := Mux(runtime_started && frontend.io.busy, busy_counter + 1.U, 0.U) - assert(busy_counter < 10000000.U, "BuckyballAccelerator: busy for too long!") -} diff --git a/arch/src/main/scala/framework/system/core/accelerator/Rvv.scala b/arch/src/main/scala/framework/system/core/accelerator/Rvv.scala deleted file mode 100644 index a7e685f78..000000000 --- a/arch/src/main/scala/framework/system/core/accelerator/Rvv.scala +++ /dev/null @@ -1,120 +0,0 @@ -package framework.system.core.accelerator - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.top.GlobalConfig -import framework.rvv.{KernelEngine, RvvBallCommand} - -import framework.frontend.globalrs.{GlobalSchedComplete, GlobalSchedIssue} -import framework.memdomain.frontend.mem.{KernelDmaRequest, KernelMemoryBridge} - -import framework.memdomain.frontend.mem.dma.{DmaError, DmaStatus} - -@instantiable -class Rvv(val b: GlobalConfig) extends Module { - require(b.rvv.enable) - @public val command = IO(Flipped(Decoupled(new GlobalSchedIssue(b)))) - @public val complete = IO(Decoupled(new GlobalSchedComplete(b))) - @public val memory = IO(Flipped(new KernelMemoryBridge(b))) - @public val fault = IO(Output(Bool())) - @public val ballRequest = IO(Decoupled(new RvvBallCommand)) - @public val ballResponse = IO(Flipped(Decoupled(UInt(64.W)))) - @public val ownerRobId = IO(Output(UInt(log2Up(b.frontend.rob_entries).W))) - @public val banks = IO(Output(new framework.rvv.BankLayout(b))) - @public val busy = IO(Output(Bool())) - - val engine: Instance[KernelEngine] = Instantiate(new KernelEngine(b)) - val loadOwner = RegInit(false.B) - val loadPending = RegInit(false.B) - val load = Reg(new KernelDmaRequest(b)) - val loadAccepted = RegInit(false.B) - val terminalSeen = RegInit(false.B) - val terminalStatus = Reg(new DmaStatus) - val failed = RegInit(false.B) - val runOwner = RegInit(false.B) - val isLoad = command.bits.cmd.cmd.funct === 44.U - engine.io.ballRequest <> ballRequest - engine.io.ballResponse <> ballResponse - engine.io.cmdReq.valid := command.valid - engine.io.cmdReq.bits.cmd.funct7 := command.bits.cmd.cmd.funct - engine.io.cmdReq.bits.cmd.rs1 := command.bits.cmd.cmd.rs1Data - engine.io.cmdReq.bits.cmd.rs2 := command.bits.cmd.cmd.rs2Data - engine.io.cmdReq.bits.rob_id := command.bits.rob_id - engine.io.cmdReq.bits.read_groups := command.bits.cmd.op1_col - engine.io.cmdReq.bits.write_groups := command.bits.cmd.wr_col - command.ready := engine.io.cmdReq.ready - busy := engine.io.status.running - val owner = Reg(UInt(log2Up(b.frontend.rob_entries).W)) - ownerRobId := owner - val layout = Reg(new framework.rvv.BankLayout(b)) - banks := layout - when(command.fire && command.bits.cmd.cmd.funct === 79.U) { - layout.readBank := command.bits.cmd.cmd.rs1Data(9, 0) - layout.writeBank := command.bits.cmd.cmd.rs1Data(29, 20) - layout.readGroups := command.bits.cmd.op1_col - layout.writeGroups := command.bits.cmd.wr_col - } - when(command.fire) { - assert(!command.bits.is_sub, "RVV command must belong to the main ROB") - owner := command.bits.rob_id - } - when(engine.io.cmdReq.fire) { - loadOwner := isLoad - runOwner := command.bits.cmd.cmd.funct === 79.U - when(isLoad) { - load.address := command.bits.cmd.cmd.rs2Data - load.bytes := command.bits.cmd.cmd.rs1Data(63, 30) - load.rob_id := command.bits.rob_id - loadPending := true.B - loadAccepted := false.B - terminalSeen := false.B - } - } - - val cancel = loadOwner && engine.io.cmdResp.valid && engine.io.result.fault && !terminalSeen - memory.load.valid := loadPending && !cancel - memory.load.bits := load - when(memory.load.fire || cancel)(loadPending := false.B) - memory.abort := cancel - when(memory.load.fire)(loadAccepted := true.B) - engine.io.imageTerminal.valid := loadOwner && loadAccepted && !terminalSeen && memory.result.valid - engine.io.imageTerminal.bits.error := memory.result.bits.error - engine.io.imageTerminal.bits.address := memory.result.bits.address - memory.result.ready := loadOwner && loadAccepted && !terminalSeen && engine.io.imageTerminal.ready - when(memory.result.fire) { - terminalSeen := true.B - terminalStatus := memory.result.bits - } - engine.io.image <> memory.image - memory.active := engine.io.status.running && !loadOwner - engine.io.channelReady := memory.ready - for (port <- 0 until b.rvv.memoryPorts) { - memory.bankRead(port) <> engine.io.bankRead(port) - memory.bankWrite(port) <> engine.io.bankWrite(port) - } - - val drained = !loadOwner || (!memory.busy && !loadPending && (!loadAccepted || terminalSeen)) - complete.valid := engine.io.cmdResp.valid && drained - complete.bits.rob_id := engine.io.cmdResp.bits.rob_id - complete.bits.is_sub := false.B - complete.bits.sub_rob_id := 0.U - complete.bits.fault := 0.U.asTypeOf(new DmaStatus) - when( - loadOwner && terminalSeen && terminalStatus.error =/= DmaError.None.U && terminalStatus.error =/= DmaError.Cancelled.U - ) { - complete.bits.fault := terminalStatus - }.elsewhen(engine.io.result.fault && (engine.io.result.cause === 7.U || !loadOwner && engine.io.result.cause === 5.U)) { - complete.bits.fault.error := DmaError.Bank.U - complete.bits.fault.address := engine.io.result.tval - }.elsewhen(engine.io.result.fault && engine.io.result.cause =/= 5.U) { - complete.bits.fault.error := DmaError.Shape.U - complete.bits.fault.address := Mux(loadOwner, load.address, engine.io.result.tval) - }.elsewhen(loadOwner && terminalSeen && terminalStatus.error =/= DmaError.None.U) { - complete.bits.fault := terminalStatus - } - - engine.io.cmdResp.ready := complete.ready && drained - when(engine.io.cmdResp.fire && !loadOwner && engine.io.result.fault)(failed := true.B) - fault := failed -} diff --git a/arch/src/main/scala/framework/system/core/ant/src/csrc/src/lib.rs b/arch/src/main/scala/framework/system/core/ant/src/csrc/src/lib.rs deleted file mode 100644 index 679bed2f2..000000000 --- a/arch/src/main/scala/framework/system/core/ant/src/csrc/src/lib.rs +++ /dev/null @@ -1,20 +0,0 @@ -#[path = "../../../../../../mem-core/spm/src/csrc/src/lib.rs"] -mod storage; - -#[no_mangle] -pub extern "C" fn ant_arithmetic(a: u64, b: u64, operation: u32, word: bool) -> u64 { - let (a, b) = if word { (a as u32 as u64, b as u32 as u64) } else { (a, b) }; - let signed = |x: u64| if word { x as u32 as i32 as i64 } else { x as i64 }; - let value = match operation { - 0 => a.wrapping_mul(b), - 1 => (((a as i64 as i128) * (b as i64 as i128)) >> 64) as u64, - 2 => (((a as i64 as i128) * (b as i128)) >> 64) as u64, - 3 => (((a as u128) * (b as u128)) >> 64) as u64, - 4 => if b == 0 { u64::MAX } else { signed(a).wrapping_div(signed(b)) as u64 }, - 5 => if b == 0 { u64::MAX } else { a / b }, - 6 => if b == 0 { a } else { signed(a).wrapping_rem(signed(b)) as u64 }, - 7 => if b == 0 { a } else { a % b }, - _ => unreachable!(), - }; - if word { value as u32 as i32 as i64 as u64 } else { value } -} diff --git a/arch/src/main/scala/framework/system/core/ant/src/main/resources/ant.f b/arch/src/main/scala/framework/system/core/ant/src/main/resources/ant.f deleted file mode 100644 index 028d3899d..000000000 --- a/arch/src/main/scala/framework/system/core/ant/src/main/resources/ant.f +++ /dev/null @@ -1,10 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@VERIFY@/uvm/src/ip/test -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/test/control_if.sv -@VERIFY@/uvm/src/ip/test/control_test_package.sv -@RESOURCES@/../../../../../../mem-core/spm/src/main/resources/package.sv -@RESOURCES@/../../../../../../mem-core/spm/src/main/resources/interface.sv -+incdir+@RTL@ --F @RTL@/Execution/filelist.f -@RESOURCES@/ant_tb.sv diff --git a/arch/src/main/scala/framework/system/core/ant/src/main/resources/ant_tb.sv b/arch/src/main/scala/framework/system/core/ant/src/main/resources/ant_tb.sv deleted file mode 100644 index cdbe4e93d..000000000 --- a/arch/src/main/scala/framework/system/core/ant/src/main/resources/ant_tb.sv +++ /dev/null @@ -1,314 +0,0 @@ -`include "config.svh" -module ant_tb; - import uvm_pkg::*; - import ip_pkg::*; - import spm_pkg::*; - import ip_control_test_pkg::*; - `include "uvm_macros.svh" - import "DPI-C" function longint unsigned ant_arithmetic( - longint unsigned a, - b, - int unsigned operation, - bit word_mode - ); - logic clock = 0, reset = 1, in_use = 0; - always #5 clock = ~clock; - ip_control_if control (clock); - virtual spm_if ports[5]; - for (genvar i = 0; i < 5; i++) begin : endpoints - spm_if bus ( - clock, - reset - ); - initial ports[i] = bus; - end - logic [1:0] start_valid = 0, start_ready, result_valid, result_ready = 0; - logic [1:0] cancel = 0, drained = '1, was_cancelled, running; - logic [31:0] task_id[2] = '{1, 2}, result_task[2]; - logic [63:0] value[2]; - logic [63:0] entry[2] = '{0, 0}, code_end[2] = '{16, 16}; - logic [1:0] command_valid, command_ready = '1, response_valid = 0, response_ready; - logic [31:0] command_instruction[2], command_task[2], response_task[2] = '{1, 2}; - logic [63:0] command_a[2], command_b[2], response_data[2] = '{64'd55, 64'd55}; - logic [4:0] response_rd[2] = '{5'd10, 5'd10}; - logic [1:0] response_error = 0, retired_valid; - logic [63:0] retired_pc[2]; - int checks = 0; - Execution dut ( - .clock(clock), - .reset(reset), - .io_inUse(in_use), - .io_start_0_valid(start_valid[0]), - .io_start_0_ready(start_ready[0]), - .io_start_0_bits_task(task_id[0]), - .io_start_0_bits_entry(entry[0]), - .io_start_0_bits_codeEnd(code_end[0]), - .io_start_0_bits_argument(`TLS_BASE), - .io_start_0_bits_stack(`TLS_END), - .io_result_0_valid(result_valid[0]), - .io_result_0_ready(result_ready[0]), - .io_result_0_bits_task(result_task[0]), - .io_result_0_bits_cancelled(was_cancelled[0]), - .io_result_0_bits_value(value[0]), - .io_cancel_0(cancel[0]), - .io_npuDrained_0(drained[0]), - .io_running_0(running[0]), - .io_command_0_ready(command_ready[0]), - .io_command_0_valid(command_valid[0]), - .io_command_0_bits_instruction(command_instruction[0]), - .io_command_0_bits_task(command_task[0]), - .io_command_0_bits_rs1(command_a[0]), - .io_command_0_bits_rs2(command_b[0]), - .io_retired_0_valid(retired_valid[0]), - .io_retired_0_bits_pc(retired_pc[0]), - .io_response_0_valid(response_valid[0]), - .io_response_0_ready(response_ready[0]), - .io_response_0_bits_task(response_task[0]), - .io_response_0_bits_rd(response_rd[0]), - .io_response_0_bits_data(response_data[0]), - .io_response_0_bits_error(response_error[0]), - .io_start_1_valid(start_valid[1]), - .io_start_1_ready(start_ready[1]), - .io_start_1_bits_task(task_id[1]), - .io_start_1_bits_entry(entry[1]), - .io_start_1_bits_codeEnd(code_end[1]), - .io_start_1_bits_argument(`TLS_BASE), - .io_start_1_bits_stack(`TLS_END), - .io_result_1_valid(result_valid[1]), - .io_result_1_ready(result_ready[1]), - .io_result_1_bits_task(result_task[1]), - .io_result_1_bits_cancelled(was_cancelled[1]), - .io_result_1_bits_value(value[1]), - .io_cancel_1(cancel[1]), - .io_npuDrained_1(drained[1]), - .io_running_1(running[1]), - .io_command_1_ready(command_ready[1]), - .io_command_1_valid(command_valid[1]), - .io_command_1_bits_instruction(command_instruction[1]), - .io_command_1_bits_task(command_task[1]), - .io_command_1_bits_rs1(command_a[1]), - .io_command_1_bits_rs2(command_b[1]), - .io_retired_1_valid(retired_valid[1]), - .io_retired_1_bits_pc(retired_pc[1]), - .io_response_1_valid(response_valid[1]), - .io_response_1_ready(response_ready[1]), - .io_response_1_bits_task(response_task[1]), - .io_response_1_bits_rd(response_rd[1]), - .io_response_1_bits_data(response_data[1]), - .io_response_1_bits_error(response_error[1]), - .io_code_0_request_valid(endpoints[0].bus.valid), - .io_code_0_request_ready(endpoints[0].bus.ready), - .io_code_0_request_bits_address(endpoints[0].bus.address), - .io_code_0_request_bits_size(endpoints[0].bus.size), - .io_code_0_request_bits_write(endpoints[0].bus.write), - .io_code_0_request_bits_data(endpoints[0].bus.data), - .io_code_0_request_bits_mask(endpoints[0].bus.mask), - .io_code_0_response_valid(endpoints[0].bus.response_valid), - .io_code_0_response_ready(endpoints[0].bus.response_ready), - .io_code_0_response_bits_data(endpoints[0].bus.result), - .io_code_0_response_bits_error(endpoints[0].bus.error), - .io_data_0_request_valid(endpoints[1].bus.valid), - .io_data_0_request_ready(endpoints[1].bus.ready), - .io_data_0_request_bits_address(endpoints[1].bus.address), - .io_data_0_request_bits_size(endpoints[1].bus.size), - .io_data_0_request_bits_write(endpoints[1].bus.write), - .io_data_0_request_bits_data(endpoints[1].bus.data), - .io_data_0_request_bits_mask(endpoints[1].bus.mask), - .io_data_0_response_valid(endpoints[1].bus.response_valid), - .io_data_0_response_ready(endpoints[1].bus.response_ready), - .io_data_0_response_bits_data(endpoints[1].bus.result), - .io_data_0_response_bits_error(endpoints[1].bus.error), - .io_code_1_request_valid(endpoints[2].bus.valid), - .io_code_1_request_ready(endpoints[2].bus.ready), - .io_code_1_request_bits_address(endpoints[2].bus.address), - .io_code_1_request_bits_size(endpoints[2].bus.size), - .io_code_1_request_bits_write(endpoints[2].bus.write), - .io_code_1_request_bits_data(endpoints[2].bus.data), - .io_code_1_request_bits_mask(endpoints[2].bus.mask), - .io_code_1_response_valid(endpoints[2].bus.response_valid), - .io_code_1_response_ready(endpoints[2].bus.response_ready), - .io_code_1_response_bits_data(endpoints[2].bus.result), - .io_code_1_response_bits_error(endpoints[2].bus.error), - .io_data_1_request_valid(endpoints[3].bus.valid), - .io_data_1_request_ready(endpoints[3].bus.ready), - .io_data_1_request_bits_address(endpoints[3].bus.address), - .io_data_1_request_bits_size(endpoints[3].bus.size), - .io_data_1_request_bits_write(endpoints[3].bus.write), - .io_data_1_request_bits_data(endpoints[3].bus.data), - .io_data_1_request_bits_mask(endpoints[3].bus.mask), - .io_data_1_response_valid(endpoints[3].bus.response_valid), - .io_data_1_response_ready(endpoints[3].bus.response_ready), - .io_data_1_response_bits_data(endpoints[3].bus.result), - .io_data_1_response_bits_error(endpoints[3].bus.error), - .io_shared_request_valid(endpoints[4].bus.valid), - .io_shared_request_ready(endpoints[4].bus.ready), - .io_shared_request_bits_address(endpoints[4].bus.address), - .io_shared_request_bits_size(endpoints[4].bus.size), - .io_shared_request_bits_write(endpoints[4].bus.write), - .io_shared_request_bits_data(endpoints[4].bus.data), - .io_shared_request_bits_mask(endpoints[4].bus.mask), - .io_shared_response_valid(endpoints[4].bus.response_valid), - .io_shared_response_ready(endpoints[4].bus.response_ready), - .io_shared_response_bits_data(endpoints[4].bus.result), - .io_shared_response_bits_error(endpoints[4].bus.error) - ); - initial begin - #0; - uvm_config_db#(virtual ip_control_if)::set(null, "uvm_test_top", "vif", control); - uvm_config_db#(time)::set(null, "uvm_test_top", "timeout", 2ms); - run_test("protocol_test"); - end - task automatic put(int port, longint unsigned address, logic [127:0] data, bit wide = 0); - logic [127:0] reply; - ports[port].send(address, wide ? 4 : 2, 1, data, wide ? 16'hffff : 16'h000f); - ports[port].take(reply); - if (ports[port].error) `uvm_fatal("LOAD", "management write failed") - endtask - task automatic launch(int id); - @(negedge clock); - start_valid[id] = 1; - do @(posedge clock); while (!start_ready[id]); - @(negedge clock); - start_valid[id] = 0; - endtask - task automatic finish(int id, longint unsigned expected, bit cancelled = 0); - wait (result_valid[id]); - if (was_cancelled[id] !== cancelled || value[id] !== expected || result_task[id] !== task_id[id]) - `uvm_fatal("RESULT", $sformatf( - "context %0d task %0d: cancelled %b value %h expected %h", - id, - result_task[id], - was_cancelled[id], - value[id], - expected - )) - repeat (3) begin - @(negedge clock); - if (!result_valid[id] || value[id] !== expected) `uvm_fatal("HOLD", "completion changed") - end - result_ready[id] = 1; - @(negedge clock); - result_ready[id] = 0; - checks++; - endtask - initial begin : exercise - longint unsigned operands[8] = '{ - 0, - 1, - 64'hffffffffffffffff, - 64'h8000000000000000, - 64'h7fffffffffffffff, - 64'h80000000, - 64'hffffffff, - 37 - }; - logic [31:0] operation; - wait (control.start); - repeat (3) @(negedge clock); - reset = 0; - // Real RV64IM programs: LD operands, arithmetic, ECALL returning a0. - // Golden arithmetic uses independent Rust i128/u128 and defined division corners. - for (int w = 0; w < 2; w++) - for (int op = 0; op < 8; op++) - if (!w || op == 0 || op >= 4) - for (int sample = 0; sample < 64; sample ++) begin - operation = 32'h02b50533 | (op << 12) | (w << 3); - put(0, 0, {32'h00000073, operation, 32'h00053503, 32'h00853583}, 1); - put(1, `TLS_BASE, {operands[sample%8], operands[sample/8]}, 1); - in_use = 1; - launch(0); - finish(0, ant_arithmetic(operands[sample/8], operands[sample%8], op, w)); - in_use = 0; - end - // One producer and one consumer share data then publish/observe a flag. - put(4, `TSS_BASE, 0, 1); - put(0, 0, {32'h0062b023, 32'h02a00313, 32'h000202b7, 32'h00000013}, 1); - put(0, 16, {32'h02a00513, 32'h0062b423, 32'h00100313, 32'h00000013}, 1); - put(0, 32, 32'h00000073); - // Consumer loops while flag is zero, then returns shared data. - put(2, 0, {32'h0002b503, 32'hfe030ee3, 32'h0082b303, 32'h000202b7}, 1); - put(2, 16, 32'h00000073); - code_end[0] = 36; - code_end[1] = 20; - in_use = 1; - fork - launch(0); - launch(1); - join - fork - finish(0, 42); - finish(1, 42); - join - in_use = 0; - // Identical TLS addresses hold different values in the two contexts. - put(0, 0, {96'b0, 32'h00053503}, 1); - put(0, 4, 32'h00000073); - put(2, 0, {96'b0, 32'h00053503}, 1); - put(2, 4, 32'h00000073); - put(1, `TLS_BASE, 128'd17, 1); - put(3, `TLS_BASE, 128'd29, 1); - code_end[0] = 8; - code_end[1] = 8; - in_use = 1; - fork - launch(0); - launch(1); - join - fork - finish(0, 17); - finish(1, 29); - join - in_use = 0; - code_end[0] = 16; - // Raw custom instruction and opaque DDR pointer survive submission backpressure. - put(0, 0, {32'h00000073, 32'h12b5757b, 32'h00053503, 32'h00853583}, 1); - put(1, `TLS_BASE, {64'hfedcba9876543210, 64'h80001000}, 1); - command_ready[0] = 0; - in_use = 1; - launch(0); - wait (command_valid[0]); - // A busy Ant must not lock the other Ant's private loader. - put(3, `TLS_BASE, 128'd91, 1); - repeat (7) begin - @(negedge clock); - if (!command_valid[0] || command_instruction[0] !== 32'h12b5757b || - command_task[0] !== task_id[0] || command_a[0] !== 64'h80001000 || - command_b[0] !== 64'hfedcba9876543210) - `uvm_fatal("COMMAND", "raw NPU command changed under backpressure") - end - drained[0] = 0; - command_ready[0] = 1; - @(negedge clock); - response_valid[0] = 1; - do @(posedge clock); while (!response_ready[0]); - @(negedge clock); - response_valid[0] = 0; - repeat (20) begin - @(negedge clock); - if (result_valid[0]) `uvm_fatal("DRAIN", "task completed before NPU drained") - end - drained[0] = 1; - finish(0, 55); - in_use = 0; - // Cancel an actual in-flight DIV; reusing the context must not consume its old result. - put(0, 0, {32'h00000073, 32'h02b54533, 32'h00053503, 32'h00853583}, 1); - put(1, `TLS_BASE, {64'd37, 64'h7fffffffffffffff}, 1); - in_use = 1; - launch(0); - do @(posedge clock); while (!(retired_valid[0] && retired_pc[0] == 4)); - repeat (8) @(negedge clock); - cancel[0] = 1; - @(negedge clock); - cancel[0] = 0; - finish(0, 0, 1); - in_use = 0; - put(0, 0, {64'b0, 32'h00000073, 32'h00b00513}, 1); - code_end[0] = 8; - in_use = 1; - launch(0); - finish(0, 11); - in_use = 0; - `uvm_info("ANT", $sformatf("%0d task completions checked", checks), UVM_LOW) - control.done = 1; - end -endmodule diff --git a/arch/src/main/scala/framework/system/core/ant/src/main/resources/control.f b/arch/src/main/scala/framework/system/core/ant/src/main/resources/control.f deleted file mode 100644 index 7c06037d8..000000000 --- a/arch/src/main/scala/framework/system/core/ant/src/main/resources/control.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@VERIFY@/uvm/src/ip/test -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/test/control_if.sv -@VERIFY@/uvm/src/ip/test/control_test_package.sv -+incdir+@RTL@ --F @RTL@/Control/filelist.f -@RESOURCES@/control_if.sv -@RESOURCES@/control_tb.sv diff --git a/arch/src/main/scala/framework/system/core/ant/src/main/resources/control_tb.sv b/arch/src/main/scala/framework/system/core/ant/src/main/resources/control_tb.sv deleted file mode 100644 index 966d9c0b0..000000000 --- a/arch/src/main/scala/framework/system/core/ant/src/main/resources/control_tb.sv +++ /dev/null @@ -1,141 +0,0 @@ -`include "config.svh" -module control_tb; - import uvm_pkg::*; - import ip_control_test_pkg::*; - `include "uvm_macros.svh" - logic clock = 0, reset = 1; - always #5 clock = ~clock; - ip_control_if control (clock); - ant_control_if management ( - clock, - reset - ); - logic [1:0] online = '1, start_valid, start_ready = '1, result_valid = 0, result_ready, cancel; - logic storage_busy = 0, in_use; - logic [31:0] task_id[2], result_task[2] = '{7, 8}; - logic [63:0] entry[2], code_end[2], argument[2], stack[2]; - int checked = 0, launches = 0; - Control dut ( - .clock(clock), - .reset(reset), - .io_request_valid(management.valid), - .io_request_ready(management.ready), - .io_request_bits_operation(management.operation), - .io_request_bits_context(management.context_id), - .io_request_bits_field(management.field), - .io_request_bits_data(management.data), - .io_response_valid(management.reply_valid), - .io_response_ready(management.reply_ready), - .io_response_bits(management.reply), - .io_storageBusy(storage_busy), - .io_inUse(in_use), - .io_signatures_0(64'h1234), - .io_online_0(online[0]), - .io_cancel_0(cancel[0]), - .io_start_0_valid(start_valid[0]), - .io_start_0_ready(start_ready[0]), - .io_start_0_bits_task(task_id[0]), - .io_start_0_bits_entry(entry[0]), - .io_start_0_bits_codeEnd(code_end[0]), - .io_start_0_bits_argument(argument[0]), - .io_start_0_bits_stack(stack[0]), - .io_result_0_valid(result_valid[0]), - .io_result_0_ready(result_ready[0]), - .io_result_0_bits_task(result_task[0]), - .io_result_0_bits_cancelled(1'b0), - .io_result_0_bits_value(64'h42), - .io_signatures_1(64'h1235), - .io_online_1(online[1]), - .io_cancel_1(cancel[1]), - .io_start_1_valid(start_valid[1]), - .io_start_1_ready(start_ready[1]), - .io_start_1_bits_task(task_id[1]), - .io_start_1_bits_entry(entry[1]), - .io_start_1_bits_codeEnd(code_end[1]), - .io_start_1_bits_argument(argument[1]), - .io_start_1_bits_stack(stack[1]), - .io_result_1_valid(result_valid[1]), - .io_result_1_ready(result_ready[1]), - .io_result_1_bits_task(result_task[1]), - .io_result_1_bits_cancelled(1'b0), - .io_result_1_bits_value(64'h42) - ); - always @(posedge clock) - if (!reset) begin - for (int i = 0; i < 2; i++) - if (start_valid[i] && start_ready[i]) begin - if(task_id[i] !== 7+i || entry[i] !== 0 || code_end[i] !== 16 || - argument[i] !== `TLS_BASE || stack[i] !== `TLS_END) - `uvm_fatal("START", "local descriptor changed") - launches++; - end - end - initial begin - uvm_config_db#(virtual ip_control_if)::set(null, "uvm_test_top", "vif", control); - uvm_config_db#(time)::set(null, "uvm_test_top", "timeout", 100us); - run_test("protocol_test"); - end - task automatic transact(int op, int ctx, int f, longint unsigned value, - longint unsigned expected = 0); - longint unsigned actual; - management.transfer(op, ctx, f, value, actual); - if (actual !== expected) - `uvm_fatal("CONTROL", $sformatf("data %h expected %h", actual, expected)) - checked++; - endtask - task automatic descriptor(int ctx); - transact(1, ctx, 0, 7 + ctx); - transact(1, ctx, 1, 0); - transact(1, ctx, 2, 16); - transact(1, ctx, 3, `TLS_BASE); - transact(1, ctx, 4, `TLS_END); - transact(1, ctx, 5, 64'h1234 + ctx); - endtask - initial begin - wait (control.start); - repeat (3) @(negedge clock); - reset = 0; - transact(0, 0, 7, 0, 2); - transact(0, 1, 0, 0, 64'h1235); - transact(0, 0, 1, 0, 1); - transact(0, 0, 2, 0, 256); - transact(0, 0, 3, 0, `TLS_BASE); - transact(0, 0, 4, 0, 256); - transact(0, 0, 5, 0, `TSS_BASE); - transact(0, 0, 6, 0, 256); - transact(5, 0, 0, 0); - descriptor(0); - transact(1, 0, 5, 64'h1234); - start_ready[0] = 0; - fork - transact(2, 0, 0, 0); - begin - wait (start_valid[0]); - repeat (5) begin - @(negedge clock); - if (!start_valid[0] || management.ready) `uvm_fatal("STALL", "launch was not held") - end - start_ready[0] = 1; - end - join - transact(0, 0, 1, 0, 3); - descriptor(1); - transact(2, 1, 0, 0); - @(negedge clock); - result_valid = 3; - do @(posedge clock); while (result_ready != 3); - @(negedge clock); - result_valid = 0; - transact(0, 0, 1, 0, 5); - transact(0, 0, 8, 0, 7); - transact(0, 0, 9, 0, 64'h42); - transact(0, 1, 8, 0, 8); - transact(3, 0, 0, 0); - transact(3, 1, 0, 0); - transact(6, 0, 0, 0); - if (in_use || launches != 2) `uvm_fatal("GROUP", "group lifetime or launch count wrong") - `uvm_info("CONTROL", $sformatf("%0d transactions checked; %0d launches", checked, launches), - UVM_LOW) - control.done = 1; - end -endmodule diff --git a/arch/src/main/scala/framework/system/core/ant/src/main/resources/service.f b/arch/src/main/scala/framework/system/core/ant/src/main/resources/service.f deleted file mode 100644 index 04e91c71c..000000000 --- a/arch/src/main/scala/framework/system/core/ant/src/main/resources/service.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@VERIFY@/uvm/src/ip/test -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/test/control_if.sv -@VERIFY@/uvm/src/ip/test/control_test_package.sv -+incdir+@RTL@ --F @RTL@/Service/filelist.f -@RESOURCES@/control_if.sv -@RESOURCES@/service_tb.sv diff --git a/arch/src/main/scala/framework/system/core/ant/src/main/resources/service_tb.sv b/arch/src/main/scala/framework/system/core/ant/src/main/resources/service_tb.sv deleted file mode 100644 index a5a2d0ae1..000000000 --- a/arch/src/main/scala/framework/system/core/ant/src/main/resources/service_tb.sv +++ /dev/null @@ -1,160 +0,0 @@ -`include "config.svh" -module service_tb; - import uvm_pkg::*; - import ip_pkg::*; - import ip_control_test_pkg::*; - `include "uvm_macros.svh" - logic clock = 0, reset = 1; - always #5 clock = ~clock; - ip_control_if control (clock); - ant_control_if management ( - clock, - reset - ); - logic [1:0] drained = '1, command_valid, command_ready = '1, response_valid = 0, response_ready; - logic [31:0] command_instruction[2], command_task[2], response_task[2] = '{0, 0}; - logic [63:0] command_a[2], command_b[2], response_data[2] = '{0, 0}, retired_pc[2]; - logic [4:0] response_rd[2] = '{0, 0}; - logic [1:0] response_error = 0, retired_valid, cancel_npu; - int completed = 0; - ServiceVerification dut ( - .clock(clock), - .reset(reset), - .io_npuDrained_0(drained[0]), - .io_command_0_ready(command_ready[0]), - .io_command_0_valid(command_valid[0]), - .io_command_0_bits_instruction(command_instruction[0]), - .io_command_0_bits_task(command_task[0]), - .io_command_0_bits_rs1(command_a[0]), - .io_command_0_bits_rs2(command_b[0]), - .io_retired_0_valid(retired_valid[0]), - .io_retired_0_bits_pc(retired_pc[0]), - .io_response_0_valid(response_valid[0]), - .io_response_0_ready(response_ready[0]), - .io_response_0_bits_task(response_task[0]), - .io_response_0_bits_rd(response_rd[0]), - .io_response_0_bits_data(response_data[0]), - .io_response_0_bits_error(response_error[0]), - .io_npuDrained_1(drained[1]), - .io_command_1_ready(command_ready[1]), - .io_command_1_valid(command_valid[1]), - .io_command_1_bits_instruction(command_instruction[1]), - .io_command_1_bits_task(command_task[1]), - .io_command_1_bits_rs1(command_a[1]), - .io_command_1_bits_rs2(command_b[1]), - .io_retired_1_valid(retired_valid[1]), - .io_retired_1_bits_pc(retired_pc[1]), - .io_response_1_valid(response_valid[1]), - .io_response_1_ready(response_ready[1]), - .io_response_1_bits_task(response_task[1]), - .io_response_1_bits_rd(response_rd[1]), - .io_response_1_bits_data(response_data[1]), - .io_response_1_bits_error(response_error[1]), - .io_request_valid(management.valid), - .io_request_ready(management.ready), - .io_request_bits_operation(management.operation), - .io_request_bits_context(management.context_id), - .io_request_bits_field(management.field), - .io_request_bits_data(management.data), - .io_reply_valid(management.reply_valid), - .io_reply_ready(management.reply_ready), - .io_reply_bits(management.reply), - .io_signatures_0(64'h1234), - .io_online_0(1'b1), - .io_cancelNpu_0(cancel_npu[0]), - .io_signatures_1(64'h1235), - .io_online_1(1'b1), - .io_cancelNpu_1(cancel_npu[1]) - ); - initial begin - #0; - uvm_config_db#(virtual ip_control_if)::set(null, "uvm_test_top", "vif", control); - uvm_config_db#(time)::set(null, "uvm_test_top", "timeout", 100us); - run_test("protocol_test"); - end - task automatic command(int op, int ctx, int field, longint unsigned value = 0); - longint unsigned reply; - management.transfer(op, ctx, field, value, reply); - endtask - task automatic put(int port, longint unsigned address, logic [127:0] data); - longint unsigned result, offset; - int space, ctx; - space = port == 4 ? 2 : port % 2; - ctx = port == 4 ? 0 : port / 2; - offset = address - (space == 0 ? 0 : space == 1 ? `TLS_BASE : `TSS_BASE); - for (int half = 0; half < 2; half++) begin - command(8 + 2 * space, ctx, offset + 8 * half, data[64*half+:64]); - management.transfer(7 + 2 * space, ctx, offset + 8 * half, 0, result); - if (result !== data[64*half+:64]) `uvm_fatal("LOAD", "local storage readback mismatch") - end - endtask - task automatic launch(int ctx, int task_id); - command(1, ctx, 0, task_id); - command(1, ctx, 1, 0); - command(1, ctx, 2, 8); - command(1, ctx, 3, `TLS_BASE); - command(1, ctx, 4, `TLS_END); - command(1, ctx, 5, 64'h1234 + ctx); - command(2, ctx, 0); - endtask - task automatic finish(int ctx, int task_id, longint unsigned value, bit cancelled = 0); - longint unsigned reply, status; - do management.transfer(0, ctx, 1, 0, status); while (!(status & 4)); - if (status !== (cancelled ? 13 : 5)) `uvm_fatal("RESULT", "wrong terminal state") - management.transfer(0, ctx, 8, 0, reply); - if (reply !== task_id) `uvm_fatal("RESULT", "wrong task ID") - management.transfer(0, ctx, 9, 0, reply); - if (reply !== value) `uvm_fatal("RESULT", "wrong terminal value") - command(3, ctx, 0); - completed++; - endtask - initial begin - longint unsigned reply; - wait (control.start); - repeat (3) @(negedge clock); - reset = 0; - put(1, `TLS_BASE, 128'h1111); - put(3, `TLS_BASE, 128'h2222); - put(4, `TSS_BASE, 128'h3333); - put(0, 0, {64'b0, 32'h00000073, 32'h02500513}); - put(2, 0, {64'b0, 32'h00000073, 32'h01d00513}); - command(5, 0, 0); - launch(0, 1); - launch(1, 2); - finish(0, 1, 37); - finish(1, 2, 29); - command(6, 0, 0); - put(0, 0, {64'b0, 32'h00000073, 32'h0000007b}); - command_ready[0] = 0; - command(5, 0, 0); - launch(0, 3); - drained[0] = 0; - wait (command_valid[0]); - command(4, 0, 0); - management.transfer(0, 0, 1, 0, reply); - if (reply !== 3) `uvm_fatal("DRAIN", "task completed before external work drained") - drained[0] = 1; - finish(0, 3, 0, 1); - command(6, 0, 0); - // A new task must not inherit the old task's cancellation. - command_ready[0] = 1; - put(0, 0, {64'b0, 32'h00000073, 32'h00900513}); - command(5, 0, 0); - launch(0, 4); - finish(0, 4, 9); - command(6, 0, 0); - // Cancellation remains effective after ECALL while backend work is still pending. - command(5, 0, 0); - launch(0, 5); - drained[0] = 0; - repeat (30) @(negedge clock); - command(4, 0, 0); - drained[0] = 1; - finish(0, 5, 0, 1); - command(6, 0, 0); - `uvm_info("SERVICE", $sformatf( - "%0d real Ant tasks; %0d control transactions checked", completed, management.checked - ), UVM_LOW) - control.done = 1; - end -endmodule diff --git a/arch/src/main/scala/framework/system/core/ant/src/main/scala/Ant.scala b/arch/src/main/scala/framework/system/core/ant/src/main/scala/Ant.scala deleted file mode 100644 index 944f6cc73..000000000 --- a/arch/src/main/scala/framework/system/core/ant/src/main/scala/Ant.scala +++ /dev/null @@ -1,282 +0,0 @@ -package framework.ant - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import framework.arith.{DivRem, Multiply} -import memcore.memory.spm - -/** - * One RV64IM execution context. Ordinary memory is strictly local. NPU work may remain - * in flight after a command is accepted; the owner supplies drained for final completion. - * Cancellation also reaches the local stores, whose accepted operations must drain before reuse. - */ -@instantiable -class Ant(p: Params) extends Module { - val code = spm.Params(0, p.codeBytes, p.data.dataBits) - - @public val io = IO(new Bundle { - val start = Flipped(Decoupled(new Start(p))) - val result = Decoupled(new Completion(p)) - val cancel = Input(Bool()) - val drained = Input(Bool()) - val running = Output(Bool()) - val localDone = Output(Bool()) - val imem = new spm.Port(code) - val tls = new spm.Port(p.data) - val tss = new spm.Port(p.shared) - val command = Decoupled(new Command(p)) - val response = Flipped(Decoupled(new Response(p))) - val retired = Output(Valid(new Retire)) - }) - - val (idle :: fetch :: fetchWait :: execute :: memory :: memoryWait :: - multiply :: multiplyWait :: divide :: divideWait :: issue :: response :: complete :: Nil) = Enum(13) - val state = RegInit(idle) - val registers = Reg(Vec(32, UInt(64.W))) - val pc = Reg(UInt(64.W)) - val instruction = Reg(UInt(32.W)) - val task = Reg(UInt(p.taskBits.W)) - val codeEnd = Reg(UInt(64.W)) - val completion = Reg(new Completion(p)) - val cancelled = RegInit(false.B) - val multiplyPending = RegInit(false.B) - val dividePending = RegInit(false.B) - val live = !io.cancel && !reset.asBool - val memoryAddress = Reg(UInt(64.W)) - val memoryShared = Reg(Bool()) - val decoder = Instantiate(new Decode) - val multiplier = Instantiate(new Multiply) - val divider = Instantiate(new DivRem(64)) - decoder.io.instruction := instruction - val d = decoder.io.decoded - val a = Mux(d.rs1 === 0.U, 0.U, registers(d.rs1)) - val b = Mux(d.rs2 === 0.U, 0.U, registers(d.rs2)) - val nextPc = pc + 4.U - - io.start.ready := state === idle && io.drained && !io.cancel && !reset.asBool - io.result.valid := state === complete && io.drained && !multiplyPending && !dividePending && !reset.asBool - io.result.bits := completion - io.running := state =/= idle - io.localDone := state === complete - io.retired.valid := false.B - io.retired.bits := 0.U.asTypeOf(new Retire) - - def retire(rd: UInt, value: UInt, next: UInt): Unit = { - when(live) { - when(rd =/= 0.U)(registers(rd) := value) - io.retired.valid := true.B - io.retired.bits.pc := pc - io.retired.bits.instruction := instruction - io.retired.bits.rd := rd - io.retired.bits.data := Mux(rd === 0.U, 0.U, value) - pc := next - state := fetch - } - } - - def finish(value: UInt, wasCancelled: Bool = false.B): Unit = { - completion.task := task - completion.value := value - completion.cancelled := wasCancelled - state := complete - } - - def word(value: UInt): UInt = Mux(d.word, value(31, 0).asSInt.pad(64).asUInt, value) - - when(io.start.fire) { - for (r <- registers) r := 0.U - registers(2) := io.start.bits.stack - registers(4) := p.data.base.U - registers(10) := io.start.bits.argument - pc := io.start.bits.entry - codeEnd := io.start.bits.codeEnd - task := io.start.bits.task - cancelled := false.B - val valid = io.start.bits.entry(1, 0) === 0.U && io.start.bits.codeEnd(1, 0) === 0.U && - io.start.bits.codeEnd <= p.codeBytes.U && (io.start.bits.entry +& 4.U) <= io.start.bits.codeEnd && - io.start.bits.argument >= p.data.base.U && io.start.bits.argument < (p.data.base + p.data.bytes).U && - io.start.bits.stack > p.data.base.U && io.start.bits.stack <= (p.data.base + p.data.bytes).U && - io.start.bits.stack(3, 0) === 0.U - assert(valid, "Invalid Ant local execution descriptor") - when(valid)(state := fetch) - } - - val fetchAllowed = pc(1, 0) === 0.U && (pc +& 4.U) <= codeEnd - io.imem.request.valid := state === fetch && fetchAllowed && live - io.imem.request.bits := 0.U.asTypeOf(io.imem.request.bits) - io.imem.request.bits.address := pc - io.imem.request.bits.size := 2.U - io.imem.response.ready := state === fetchWait || (state === complete && cancelled) - when(state === fetch && live) { - assert(fetchAllowed, "Ant instruction address invalid: pc=0x%x", pc) - when(io.imem.request.fire)(state := fetchWait) - } - when(state === fetchWait && io.imem.response.fire && live) { - assert(!io.imem.response.bits.error, "Ant instruction storage access failed: pc=0x%x", pc) - when(!io.imem.response.bits.error) { - instruction := io.imem.response.bits.data(31, 0) - state := execute - } - } - - val operand = Mux(d.immediateAlu, d.immediate, b) - val shift = Mux(d.word, Cat(0.U(1.W), operand(4, 0)), operand(5, 0)) - val logical = Mux(d.word, Cat(0.U(32.W), a(31, 0)), a) - val signed = Mux(d.word, a(31, 0).asSInt.pad(64).asUInt, a) - - val alu = MuxLookup(d.funct3, 0.U(64.W))(Seq( - 0.U -> Mux(d.subtract, a - operand, a + operand), - 1.U -> (a << shift)(63, 0), - 2.U -> (a.asSInt < operand.asSInt).asUInt, - 3.U -> (a < operand).asUInt, - 4.U -> (a ^ operand), - 5.U -> Mux(d.arithmeticShift, (signed.asSInt >> shift).asUInt, logical >> shift), - 6.U -> (a | operand), - 7.U -> (a & operand) - )) - - val branchTaken = MuxLookup(d.funct3, false.B)(Seq( - 0.U -> (a === b), - 1.U -> (a =/= b), - 4.U -> (a.asSInt < b.asSInt), - 5.U -> (a.asSInt >= b.asSInt), - 6.U -> (a < b), - 7.U -> (a >= b) - )) - - val address = a + d.immediate - val size = d.funct3(1, 0) - val bytes = 1.U(64.W) << size - val end = address +& bytes - val inTls = address >= p.data.base.U && end <= (p.data.base + p.data.bytes).U - val inTss = address >= p.shared.base.U && end <= (p.shared.base + p.shared.bytes).U - when(state === execute && live) { - switch(d.kind) { - is(Kind.illegal)(assert(false.B, "Ant illegal instruction: pc=0x%x instruction=0x%x", pc, instruction)) - is(Kind.lui)(retire(d.rd, d.immediate, nextPc)) - is(Kind.auipc)(retire(d.rd, pc + d.immediate, nextPc)) - is(Kind.alu)(retire(d.rd, word(alu), nextPc)) - is(Kind.jal, Kind.jalr, Kind.branch) { - val target = Mux( - d.kind === Kind.jalr, - (a + d.immediate) & ~1.U(64.W), - Mux(d.kind === Kind.branch && !branchTaken, nextPc, pc + d.immediate) - ) - assert(target(1, 0) === 0.U, "Ant branch target misaligned: pc=0x%x target=0x%x", pc, target) - retire(Mux(d.kind === Kind.branch, 0.U, d.rd), nextPc, target) - } - is(Kind.load, Kind.store) { - assert((address & (bytes - 1.U)) === 0.U, "Ant load/store misaligned: pc=0x%x address=0x%x", pc, address) - assert(inTls || inTss, "Ant load/store outside TLS/TSS: pc=0x%x address=0x%x", pc, address) - when((address & (bytes - 1.U)) === 0.U && (inTls || inTss)) { - memoryAddress := address - memoryShared := inTss - state := memory - } - } - is(Kind.multiply)(state := multiply) - is(Kind.divide)(state := divide) - is(Kind.custom)(state := issue) - is(Kind.fence)(retire(0.U, 0.U, nextPc)) // Ordinary LSU operations already complete in order. - is(Kind.exit) { - assert(registers(17) === 0.U, "Ant unsupported ECALL: pc=0x%x a7=0x%x", pc, registers(17)) - when(registers(17) === 0.U)(finish(registers(10))) - } - is(Kind.breakpoint)(assert(false.B, "Ant breakpoint: pc=0x%x", pc)) - } - } - - val mask = MuxLookup(size, 0.U(p.data.beatBytes.W))(Seq( - 0.U -> 1.U, - 1.U -> 3.U, - 2.U -> 15.U, - 3.U -> 255.U - )) - - for ((port, shared) <- Seq((io.tls, false), (io.tss, true))) { - port.request.valid := state === memory && memoryShared === shared.B && live - port.request.bits := 0.U.asTypeOf(port.request.bits) - port.request.bits.address := memoryAddress - port.request.bits.size := size - port.request.bits.write := d.kind === Kind.store - port.request.bits.data := b - port.request.bits.mask := Mux(d.kind === Kind.store, mask, 0.U) - port.response.ready := (state === memoryWait && memoryShared === shared.B) || (state === complete && cancelled) - } - when(io.tls.request.fire || io.tss.request.fire)(state := memoryWait) - val memoryResponse = Mux(memoryShared, io.tss.response.bits, io.tls.response.bits) - val memoryValid = Mux(memoryShared, io.tss.response.valid, io.tls.response.valid) - - val loaded = MuxLookup(d.funct3, memoryResponse.data(63, 0))(Seq( - 0.U -> memoryResponse.data(7, 0).asSInt.pad(64).asUInt, - 1.U -> memoryResponse.data(15, 0).asSInt.pad(64).asUInt, - 2.U -> memoryResponse.data(31, 0).asSInt.pad(64).asUInt, - 4.U -> memoryResponse.data(7, 0).pad(64), - 5.U -> memoryResponse.data(15, 0).pad(64), - 6.U -> memoryResponse.data(31, 0).pad(64) - )) - - when(state === memoryWait && memoryValid && live) { - assert(!memoryResponse.error, "Ant local memory access failed: pc=0x%x address=0x%x", pc, memoryAddress) - when(!memoryResponse.error)(retire(Mux(d.kind === Kind.store, 0.U, d.rd), loaded, nextPc)) - } - - multiplier.io.request.valid := state === multiply && live - multiplier.io.request.bits.a := a - multiplier.io.request.bits.b := b - multiplier.io.request.bits.high := d.funct3 =/= 0.U - multiplier.io.request.bits.signedA := d.funct3 === 1.U || d.funct3 === 2.U - multiplier.io.request.bits.signedB := d.funct3 === 1.U - multiplier.io.response.ready := state === multiplyWait || (state === complete && cancelled) - when(multiplier.io.request.fire) { state := multiplyWait; multiplyPending := true.B } - when(multiplier.io.response.fire)(multiplyPending := false.B) - when(state === multiplyWait && multiplier.io.response.fire && live) { - retire(d.rd, word(multiplier.io.response.bits), nextPc) - } - divider.io.clear := false.B - divider.io.request.valid := state === divide && live - divider.io.request.bits.a := a - divider.io.request.bits.b := b - divider.io.request.bits.sew := Mux(d.word, 2.U, 3.U) - divider.io.request.bits.signed := !d.funct3(0) - divider.io.request.bits.remainder := d.funct3(1) - divider.io.response.ready := state === divideWait || (state === complete && cancelled) - when(divider.io.request.fire) { state := divideWait; dividePending := true.B } - when(divider.io.response.fire)(dividePending := false.B) - when(state === divideWait && divider.io.response.fire && live) { - retire(d.rd, word(divider.io.response.bits), nextPc) - } - - io.command.valid := state === issue && live - io.command.bits.task := task - io.command.bits.pc := pc - io.command.bits.instruction := instruction - io.command.bits.rs1 := Mux(instruction(13), a, 0.U) - io.command.bits.rs2 := Mux(instruction(12), b, 0.U) - io.response.ready := state === response || (state === complete && cancelled) - when(io.command.fire) { - when(instruction(14))(state := response) - .otherwise(retire(0.U, 0.U, nextPc)) - } - when(state === response && io.response.fire && live) { - assert(io.response.bits.task === task && io.response.bits.rd === d.rd, "Ant NPU response does not match its task") - assert(!io.response.bits.error, "Ant NPU response failed: pc=0x%x data=0x%x", pc, io.response.bits.data) - when(io.response.bits.task === task && io.response.bits.rd === d.rd && !io.response.bits.error) { - retire(d.rd, io.response.bits.data, nextPc) - } - } - when(io.result.fire)(state := idle) - when(io.cancel && state =/= idle && state =/= complete) { - finish(0.U, true.B) - cancelled := true.B - io.retired.valid := false.B - } - // Local execution may have ended while the task still waits for NPU/DMA drain. - // Once a completion is published, preserve it under response backpressure. - when(io.cancel && state === complete && !io.result.valid) { - completion.cancelled := true.B - completion.value := 0.U - cancelled := true.B - } -} diff --git a/arch/src/main/scala/framework/system/core/ant/src/main/scala/Control.scala b/arch/src/main/scala/framework/system/core/ant/src/main/scala/Control.scala deleted file mode 100644 index 782e51664..000000000 --- a/arch/src/main/scala/framework/system/core/ant/src/main/scala/Control.scala +++ /dev/null @@ -1,143 +0,0 @@ -package framework.ant - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} - -object ControlOp { - val Query = 0 - val Stage = 1 - val Start = 2 - val Acknowledge = 3 - val Cancel = 4 - val Acquire = 5 - val Release = 6 -} - -class ControlRequest extends Bundle { - val operation = UInt(7.W) - val context = UInt(32.W) - val field = UInt(32.W) - val data = UInt(64.W) -} - -/** - * Task registers for one software owner. Invalid descriptors and resource reuse are - * programming errors. Execution contract violations assert at their source. - */ -@instantiable -class Control(p: Params, contexts: Int) extends Module { - require(contexts > 0 && p.taskBits <= 64) - - @public val io = IO(new Bundle { - val request = Flipped(Decoupled(new ControlRequest)) - val response = Decoupled(UInt(64.W)) - val signatures = Input(Vec(contexts, UInt(64.W))) - val online = Input(Vec(contexts, Bool())) - val storageBusy = Input(Bool()) - val start = Vec(contexts, Decoupled(new Start(p))) - val result = Vec(contexts, Flipped(Decoupled(new Completion(p)))) - val cancel = Output(Vec(contexts, Bool())) - val inUse = Output(Bool()) - val available = Output(Vec(contexts, Bool())) - }) - - val group = RegInit(false.B) - val owned = RegInit(VecInit(Seq.fill(contexts)(false.B))) - val done = RegInit(VecInit(Seq.fill(contexts)(false.B))) - val staged = Reg(Vec(contexts, Vec(6, UInt(64.W)))) - val fields = RegInit(VecInit(Seq.fill(contexts)(0.U(6.W)))) - val results = Reg(Vec(contexts, new Completion(p))) - val pending = RegInit(false.B) - val answer = Reg(UInt(64.W)) - val req = io.request.bits - val index = req.context(math.max(1, log2Ceil(contexts)) - 1, 0) - val inRange = req.context < contexts.U - val free = !owned(index) && !done(index) - val canStart = inRange && io.online(index) && free && group && - fields(index) === 63.U && staged(index)(5) === io.signatures(index) - val launching = req.operation === ControlOp.Start.U && canStart - io.request.ready := !pending && !reset.asBool && (!launching || io.start(index).ready) - io.response.valid := pending && !reset.asBool - io.response.bits := answer - io.inUse := group - when(io.response.fire)(pending := false.B) - for (i <- 0 until contexts) { - io.available(i) := io.online(i) && !owned(i) && !done(i) - io.start(i).valid := io.request.valid && !pending && !reset.asBool && launching && req.context === i.U - io.start(i).bits.task := staged(i)(0) - io.start(i).bits.entry := staged(i)(1) - io.start(i).bits.codeEnd := staged(i)(2) - io.start(i).bits.argument := staged(i)(3) - io.start(i).bits.stack := staged(i)(4) - // Cancel accepts immediately, independently of the start-ready path. - io.cancel(i) := io.request.valid && !pending && !reset.asBool && - req.operation === ControlOp.Cancel.U && req.context === i.U && owned(i) - io.result(i).ready := owned(i) && !done(i) && !reset.asBool - when(io.result(i).valid && !reset.asBool) { - assert(owned(i) && !done(i), "Ant result arrived without a live task") - assert(io.result(i).bits.task === staged(i)(0), "Ant completion task does not match its submission") - } - when(io.result(i).fire) { - results(i) := io.result(i).bits - owned(i) := false.B - done(i) := true.B - } - } - when(io.request.fire) { - pending := true.B - answer := 0.U - assert(req.operation <= ControlOp.Release.U, "Invalid Ant control operation") - when(req.operation <= ControlOp.Cancel.U && !(req.operation === ControlOp.Query.U && req.field === 7.U)) { - assert(inRange, "Invalid Ant context index") - } - switch(req.operation) { - is(ControlOp.Query.U) { - assert(req.field < 10.U, "Invalid Ant query field") - when(req.field >= 8.U)(assert(done(index), "Ant result read before completion")) - answer := MuxLookup(req.field, 0.U)(Seq( - 0.U -> io.signatures(index), - 1.U -> Cat(done(index) && results(index).cancelled, done(index), owned(index), io.online(index)), - 2.U -> p.codeBytes.U, - 3.U -> p.data.base.U, - 4.U -> p.data.bytes.U, - 5.U -> p.shared.base.U, - 6.U -> p.shared.bytes.U, - 7.U -> contexts.U, - 8.U -> results(index).task, - 9.U -> results(index).value - )) - } - is(ControlOp.Stage.U) { - assert(free, "Ant descriptor overwritten before resource release") - assert(req.field < 6.U, "Invalid Ant descriptor field") - when(req.field === 0.U)(assert((req.data >> p.taskBits) === 0.U, "Ant task ID overflow")) - staged(index)(req.field(2, 0)) := req.data - fields(index) := fields(index) | UIntToOH(req.field, 6) - } - is(ControlOp.Start.U) { - assert(io.online(index), "Offline Ant context submitted") - assert(free, "Ant context reused before completion acknowledgement") - assert(group, "Ant submitted without TSS group ownership") - assert(fields(index) === 63.U, "Incomplete Ant descriptor") - assert(staged(index)(5) === io.signatures(index), "Ant program fingerprint mismatch") - when(canStart) { owned(index) := true.B; fields(index) := 0.U } - } - is(ControlOp.Acknowledge.U) { - assert(done(index), "Ant completion acknowledged before result") - done(index) := false.B - } - is(ControlOp.Acquire.U) { - assert(!group && !io.storageBusy, "Ant group acquired before loader completion or previous release") - group := true.B - } - is(ControlOp.Release.U) { - assert( - group && !owned.asUInt.orR && !done.asUInt.orR && !io.storageBusy, - "Ant group released before tasks and storage drained" - ) - group := false.B - } - } - } -} diff --git a/arch/src/main/scala/framework/system/core/ant/src/main/scala/Decode.scala b/arch/src/main/scala/framework/system/core/ant/src/main/scala/Decode.scala deleted file mode 100644 index d2dc96c51..000000000 --- a/arch/src/main/scala/framework/system/core/ant/src/main/scala/Decode.scala +++ /dev/null @@ -1,93 +0,0 @@ -package framework.ant - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} - -object Kind extends ChiselEnum { - val illegal, lui, auipc, jal, jalr, branch, load, store, alu, multiply, divide, custom, fence, exit, breakpoint = - Value -} - -class Decoded extends Bundle { - val kind = Kind() - val rs1 = UInt(5.W) - val rs2 = UInt(5.W) - val rd = UInt(5.W) - val funct3 = UInt(3.W) - val immediate = UInt(64.W) - val immediateAlu = Bool() - val word = Bool() - val subtract = Bool() - val arithmeticShift = Bool() -} - -@instantiable -class Decode extends Module { - - @public val io = IO(new Bundle { - val instruction = Input(UInt(32.W)) - val decoded = Output(new Decoded) - }) - - val i = io.instruction - val f3 = i(14, 12) - val f7 = i(31, 25) - val d = WireDefault(0.U.asTypeOf(new Decoded)) - d.kind := Kind.illegal - d.rs1 := i(19, 15) - d.rs2 := i(24, 20) - d.rd := i(11, 7) - d.funct3 := f3 - d.immediate := i(31, 20).asSInt.pad(64).asUInt - d.word := i(3) - d.subtract := f7 === 32.U - d.arithmeticShift := i(30) - switch(i(6, 0)) { - is("h37".U) { d.kind := Kind.lui; d.immediate := Cat(i(31, 12), 0.U(12.W)).asSInt.pad(64).asUInt } - is("h17".U) { d.kind := Kind.auipc; d.immediate := Cat(i(31, 12), 0.U(12.W)).asSInt.pad(64).asUInt } - is("h6f".U) { - d.kind := Kind.jal - d.immediate := Cat(i(31), i(19, 12), i(20), i(30, 21), 0.U(1.W)).asSInt.pad(64).asUInt - } - is("h67".U)(when(f3 === 0.U)(d.kind := Kind.jalr)) - is("h63".U) { - when(f3 === 0.U || f3 === 1.U || f3 >= 4.U)(d.kind := Kind.branch) - d.immediate := Cat(i(31), i(7), i(30, 25), i(11, 8), 0.U(1.W)).asSInt.pad(64).asUInt - } - is("h03".U)(when(f3 <= 6.U)(d.kind := Kind.load)) - is("h23".U) { - when(f3 <= 3.U)(d.kind := Kind.store) - d.immediate := Cat(i(31, 25), i(11, 7)).asSInt.pad(64).asUInt - } - is("h13".U, "h1b".U) { - d.immediateAlu := true.B - d.subtract := false.B - val shiftLeft = f3 === 1.U - val shiftRight = f3 === 5.U - val shiftEncoding = - Mux(i(3), f7 === 0.U || (shiftRight && f7 === 32.U), i(31, 26) === 0.U || (shiftRight && i(31, 26) === 16.U)) - val operation = !i(3) || f3 === 0.U || shiftLeft || shiftRight - when(operation && (!(shiftLeft || shiftRight) || shiftEncoding))(d.kind := Kind.alu) - } - is("h33".U, "h3b".U) { - when(f7 === 1.U) { - when(!i(3) || f3 === 0.U || f3 >= 4.U) { - d.kind := Mux(f3 < 4.U, Kind.multiply, Kind.divide) - } - }.otherwise { - val operation = !i(3) || f3 === 0.U || f3 === 1.U || f3 === 5.U - when(operation && (f7 === 0.U || (f7 === 32.U && (f3 === 0.U || f3 === 5.U)))) { - d.kind := Kind.alu - } - } - } - is("h7b".U)(d.kind := Kind.custom) - is("h0f".U)(when(f3 === 0.U)(d.kind := Kind.fence)) - is("h73".U) { - when(i === "h00000073".U)(d.kind := Kind.exit) - when(i === "h00100073".U)(d.kind := Kind.breakpoint) - } - } - io.decoded := d -} diff --git a/arch/src/main/scala/framework/system/core/ant/src/main/scala/Emit.scala b/arch/src/main/scala/framework/system/core/ant/src/main/scala/Emit.scala deleted file mode 100644 index 4ae83658c..000000000 --- a/arch/src/main/scala/framework/system/core/ant/src/main/scala/Emit.scala +++ /dev/null @@ -1,69 +0,0 @@ -package framework.ant - -import java.nio.file.{Files, Paths} -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import memcore.memory.spm - -object Emit extends App { - val p = Params(256, spm.Params(0x10000, 256), spm.Params(0x20000, 256)) - val output = Paths.get("src/main/scala/framework/system/core/ant/build") - Files.createDirectories(output) - Files.writeString( - output.resolve("config.svh"), - s"`define TLS_BASE 64'h${p.data.base.toString(16)}\n" + - s"`define TSS_BASE 64'h${p.shared.base.toString(16)}\n" + - s"`define TLS_END 64'h${(p.data.base + p.data.bytes).toString(16)}\n" - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Execution(p, contexts = 2), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("Execution").toString, "--split-verilog") - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Control(p, contexts = 2), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("Control").toString, "--split-verilog") - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new ServiceVerification(p, contexts = 2), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("Service").toString, "--split-verilog") - ) -} - -@instantiable -class ServiceVerification(p: Params, contexts: Int) extends Module { - - @public val io = IO(new Bundle { - val request = Flipped(Decoupled(new ControlRequest)) - val reply = Decoupled(UInt(64.W)) - val signatures = Input(Vec(contexts, UInt(64.W))) - val online = Input(Vec(contexts, Bool())) - val command = Vec(contexts, Decoupled(new Command(p))) - val response = Vec(contexts, Flipped(Decoupled(new Response(p)))) - val npuDrained = Input(Vec(contexts, Bool())) - val cancelNpu = Output(Vec(contexts, Bool())) - val launched = Output(Vec(contexts, Valid(new Start(p)))) - val retired = Output(Vec(contexts, Valid(new Retire))) - val idle = Output(Bool()) - }) - - val service = Instantiate(new Service(p, contexts)) - service.io.request <> io.request - io.reply <> service.io.reply - service.io.signatures := io.signatures - service.io.online := io.online - io.launched := service.io.launched - io.retired := service.io.retired - io.idle := service.io.idle - for (i <- 0 until contexts) { - val execution = Instantiate(new LocalExecution(p)) - execution.io.local <> service.io.locals(i) - execution.io.npuDrained := io.npuDrained(i) - io.command(i) <> execution.io.command - execution.io.response <> io.response(i) - io.cancelNpu(i) := service.io.locals(i).cancel - } -} diff --git a/arch/src/main/scala/framework/system/core/ant/src/main/scala/Execution.scala b/arch/src/main/scala/framework/system/core/ant/src/main/scala/Execution.scala deleted file mode 100644 index 91d5497d1..000000000 --- a/arch/src/main/scala/framework/system/core/ant/src/main/scala/Execution.scala +++ /dev/null @@ -1,55 +0,0 @@ -package framework.ant - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import memcore.memory.{spm, tss} - -/** A tile-local group. The controller holds inUse until all cooperating tasks have finished. */ -@instantiable -class Execution(p: Params, contexts: Int) extends Module { - require(contexts > 0) - val codeParams = spm.Params(0, p.codeBytes, p.data.dataBits) - - @public val io = IO(new Bundle { - val inUse = Input(Bool()) - val start = Vec(contexts, Flipped(Decoupled(new Start(p)))) - val result = Vec(contexts, Decoupled(new Completion(p))) - val cancel = Input(Vec(contexts, Bool())) - val npuDrained = Input(Vec(contexts, Bool())) - val running = Output(Vec(contexts, Bool())) - val storageBusy = Output(Bool()) - val command = Vec(contexts, Decoupled(new Command(p))) - val response = Vec(contexts, Flipped(Decoupled(new Response(p)))) - val retired = Output(Vec(contexts, Valid(new Retire))) - val code = Vec(contexts, Flipped(new spm.Port(codeParams))) - val data = Vec(contexts, Flipped(new spm.Port(p.data))) - val shared = Flipped(new spm.Port(p.shared)) - }) - - val shared = Instantiate(new tss.Store(tss.Params(p.shared, contexts))) - // A premature group release cannot grant management access over a running task. - shared.io.inUse := io.inUse || io.running.asUInt.orR - shared.io.host <> io.shared - shared.io.cancel := io.cancel - val privateBusy = Wire(Vec(contexts, Bool())) - io.storageBusy := privateBusy.asUInt.orR || shared.io.busy.asUInt.orR || shared.io.hostBusy - for (i <- 0 until contexts) { - val execution = Instantiate(new LocalExecution(p)) - privateBusy(i) := execution.io.local.storageBusy - execution.io.local.inUse := io.inUse - execution.io.local.start <> io.start(i) - io.result(i) <> execution.io.local.result - io.running(i) := execution.io.local.running - io.retired(i) := execution.io.local.retired - execution.io.local.cancel := io.cancel(i) - execution.io.local.sharedBusy := shared.io.busy(i) - execution.io.local.sharedHostBusy := shared.io.hostBusy - execution.io.npuDrained := io.npuDrained(i) - io.command(i) <> execution.io.command - execution.io.response <> io.response(i) - execution.io.local.code <> io.code(i) - execution.io.local.data <> io.data(i) - shared.io.clients(i) <> execution.io.local.shared - } -} diff --git a/arch/src/main/scala/framework/system/core/ant/src/main/scala/Interfaces.scala b/arch/src/main/scala/framework/system/core/ant/src/main/scala/Interfaces.scala deleted file mode 100644 index 794982dba..000000000 --- a/arch/src/main/scala/framework/system/core/ant/src/main/scala/Interfaces.scala +++ /dev/null @@ -1,70 +0,0 @@ -package framework.ant - -import chisel3._ -import chisel3.util._ -import memcore.memory.spm - -/** RV64IM local execution; no compressed, floating-point, atomic or privileged instructions. */ -case class Params( - codeBytes: Int, - data: spm.Params, - shared: spm.Params, - taskBits: Int = 32) { - require(codeBytes >= 32 && isPow2(codeBytes)) - require(data.base >= codeBytes && shared.base >= codeBytes) - require(data.base + data.bytes <= shared.base || shared.base + shared.bytes <= data.base) - require(data.dataBits == shared.dataBits) - require(taskBits > 0) -} - -class Start(p: Params) extends Bundle { - val task = UInt(p.taskBits.W) - val entry = UInt(64.W) - val codeEnd = UInt(64.W) - val argument = UInt(64.W) - val stack = UInt(64.W) -} - -class Completion(p: Params) extends Bundle { - val task = UInt(p.taskBits.W) - val value = UInt(64.W) - val cancelled = Bool() -} - -class Command(p: Params) extends Bundle { - val task = UInt(p.taskBits.W) - val pc = UInt(64.W) - val instruction = UInt(32.W) - val rs1 = UInt(64.W) - val rs2 = UInt(64.W) -} - -class Response(p: Params) extends Bundle { - val task = UInt(p.taskBits.W) - val rd = UInt(5.W) - val data = UInt(64.W) - val error = Bool() -} - -class Retire extends Bundle { - val pc = UInt(64.W) - val instruction = UInt(32.W) - val rd = UInt(5.W) - val data = UInt(64.W) -} - -/** Core-side local execution and uncached management boundary. */ -class LocalPort(p: Params) extends Bundle { - val inUse = Input(Bool()) - val start = Flipped(Decoupled(new Start(p))) - val result = Decoupled(new Completion(p)) - val cancel = Input(Bool()) - val sharedBusy = Input(Bool()) - val sharedHostBusy = Input(Bool()) - val running = Output(Bool()) - val storageBusy = Output(Bool()) - val retired = Output(Valid(new Retire)) - val code = Flipped(new spm.Port(spm.Params(0, p.codeBytes, p.data.dataBits))) - val data = Flipped(new spm.Port(p.data)) - val shared = new spm.Port(p.shared) -} diff --git a/arch/src/main/scala/framework/system/core/ant/src/main/scala/LocalExecution.scala b/arch/src/main/scala/framework/system/core/ant/src/main/scala/LocalExecution.scala deleted file mode 100644 index 781064686..000000000 --- a/arch/src/main/scala/framework/system/core/ant/src/main/scala/LocalExecution.scala +++ /dev/null @@ -1,42 +0,0 @@ -package framework.ant - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import memcore.memory.{spm, tls} - -@instantiable -class LocalExecution(p: Params) extends Module { - - @public val io = IO(new Bundle { - val local = new LocalPort(p) - val npuDrained = Input(Bool()) - val command = Decoupled(new Command(p)) - val response = Flipped(Decoupled(new Response(p))) - }) - - val ant = Instantiate(new Ant(p)) - val image = Instantiate(new tls.Store(spm.Params(0, p.codeBytes, p.data.dataBits), localReadOnly = true)) - val data = Instantiate(new tls.Store(p.data)) - io.local.storageBusy := image.io.busy || data.io.busy - ant.io.start.valid := io.local.start.valid && io.local.inUse && !io.local.sharedHostBusy - ant.io.start.bits := io.local.start.bits - io.local.start.ready := ant.io.start.ready && io.local.inUse && !io.local.sharedHostBusy - io.local.result <> ant.io.result - io.local.running := ant.io.running - io.local.retired := ant.io.retired - ant.io.cancel := io.local.cancel - ant.io.drained := io.npuDrained && !image.io.busy && !data.io.busy && !io.local.sharedBusy - io.command <> ant.io.command - ant.io.response <> io.response - val claimed = ant.io.running || (io.local.inUse && io.local.start.valid) - image.io.running := claimed - data.io.running := claimed - image.io.cancel := io.local.cancel - data.io.cancel := io.local.cancel - image.io.host <> io.local.code - data.io.host <> io.local.data - image.io.local <> ant.io.imem - data.io.local <> ant.io.tls - io.local.shared <> ant.io.tss -} diff --git a/arch/src/main/scala/framework/system/core/ant/src/main/scala/Service.scala b/arch/src/main/scala/framework/system/core/ant/src/main/scala/Service.scala deleted file mode 100644 index 876efbf41..000000000 --- a/arch/src/main/scala/framework/system/core/ant/src/main/scala/Service.scala +++ /dev/null @@ -1,102 +0,0 @@ -package framework.ant - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import memcore.memory.{spm, tss} - -/** - * Tile execution service. The caller supplies the uncached management transport and - * task-bound NPU/DMA owner; neither CPU cache nor address translation lives here. - */ -@instantiable -class Service(p: Params, contexts: Int) extends Module { - - @public val io = IO(new Bundle { - val request = Flipped(Decoupled(new ControlRequest)) - val reply = Decoupled(UInt(64.W)) - val signatures = Input(Vec(contexts, UInt(64.W))) - val online = Input(Vec(contexts, Bool())) - val locals = Vec(contexts, Flipped(new LocalPort(p))) - val launched = Output(Vec(contexts, Valid(new Start(p)))) - val retired = Output(Vec(contexts, Valid(new Retire))) - val idle = Output(Bool()) - }) - - val control = Instantiate(new Control(p, contexts)) - val sharedStore = Instantiate(new tss.Store(tss.Params(p.shared, contexts))) - val running = VecInit(io.locals.map(_.running)) - val storageBusy = io.locals.map(_.storageBusy).reduce(_ || _) || - sharedStore.io.busy.asUInt.orR || sharedStore.io.hostBusy - control.io.signatures := io.signatures - control.io.online := io.online - control.io.storageBusy := storageBusy - sharedStore.io.inUse := control.io.inUse || running.asUInt.orR - sharedStore.io.cancel := control.io.cancel - for (i <- 0 until contexts) { - io.locals(i).inUse := control.io.inUse - io.locals(i).start <> control.io.start(i) - control.io.result(i) <> io.locals(i).result - io.locals(i).cancel := control.io.cancel(i) - io.locals(i).sharedBusy := sharedStore.io.busy(i) - io.locals(i).sharedHostBusy := sharedStore.io.hostBusy - sharedStore.io.clients(i) <> io.locals(i).shared - io.retired(i) := io.locals(i).retired - } - val idle :: controlReply :: loading :: loadReply :: Nil = Enum(4) - val state = RegInit(idle) - val request = io.request.bits - val isLoad = request.operation >= 7.U - val space = (request.operation - 7.U) >> 1 - val write = !request.operation(0) - val shared = space === 2.U - val localIndex = request.context(math.max(1, log2Ceil(contexts)) - 1, 0) - val offset = request.field - val bytes = MuxLookup(space, 0.U)(Seq(0.U -> p.codeBytes.U, 1.U -> p.data.bytes.U, 2.U -> p.shared.bytes.U)) - val localBase = MuxLookup(space, 0.U(64.W))(Seq(0.U -> 0.U, 1.U -> p.data.base.U, 2.U -> p.shared.base.U)) - val valid = request.operation <= 12.U && Mux(shared, request.context === 0.U, request.context < contexts.U) && - offset(2, 0) === 0.U && (offset +& 8.U) <= bytes - val permitted = - Mux(shared, !control.io.inUse, !running(localIndex) && (!write || control.io.available(localIndex))) - val ports = io.locals.map(_.code).toSeq ++ io.locals.map(_.data).toSeq :+ sharedStore.io.host - val selected = Mux(shared, (2 * contexts).U, Mux(space === 0.U, request.context, request.context +& contexts.U)) - val owner = Reg(UInt(log2Ceil(ports.size).W)) - val value = Reg(UInt(64.W)) - for ((port, i) <- ports.zipWithIndex) { - port.request.valid := state === idle && io.request.valid && isLoad && valid && permitted && selected === i.U && !reset.asBool - port.request.bits := 0.U.asTypeOf(port.request.bits) - port.request.bits.address := localBase + offset - port.request.bits.size := 3.U - port.request.bits.write := write - port.request.bits.data := request.data - port.request.bits.mask := Mux(write, 255.U, 0.U) - port.response.ready := state === loading && owner === i.U && !reset.asBool - } - when(state === idle && io.request.valid && isLoad && !reset.asBool) { - assert(valid, "Invalid Ant local loader address or operation") - assert(permitted, "Ant local storage loader conflicts with execution ownership") - } - control.io.request.valid := state === idle && io.request.valid && !isLoad && !reset.asBool - control.io.request.bits := request - io.request.ready := state === idle && !reset.asBool && - Mux(isLoad, valid && permitted && VecInit(ports.map(_.request.ready))(selected), control.io.request.ready) - when(io.request.fire) { - state := Mux(isLoad, loading, controlReply) - owner := selected - } - val response = VecInit(ports.map(_.response.bits))(owner) - when(state === loading && VecInit(ports.map(_.response.valid))(owner)) { - assert(!response.error, "Ant local loader storage access failed") - value := response.data(63, 0) - state := loadReply - } - io.reply.valid := (state === loadReply || (state === controlReply && control.io.response.valid)) && !reset.asBool - io.reply.bits := Mux(state === loadReply, value, control.io.response.bits) - control.io.response.ready := state === controlReply && io.reply.ready && !reset.asBool - when(io.reply.fire)(state := idle) - io.idle := state === idle && !running.asUInt.orR && !storageBusy - for (i <- 0 until contexts) { - io.launched(i).valid := control.io.start(i).fire - io.launched(i).bits := control.io.start(i).bits - } -} diff --git a/arch/src/main/scala/framework/system/core/clink/Shm.scala b/arch/src/main/scala/framework/system/core/clink/Shm.scala deleted file mode 100644 index 929c53e4e..000000000 --- a/arch/src/main/scala/framework/system/core/clink/Shm.scala +++ /dev/null @@ -1,30 +0,0 @@ -package framework.system.core.clink - -import chisel3._ -import chisel3.util._ -import framework.top.GlobalConfig -import framework.memdomain.backend.MemRequestIO -import framework.memdomain.backend.shared.SharedMemLayout -import framework.memdomain.frontend.mem.MemConfigerIO -import framework.memdomain.isa.{MvoverISA, MvoverPort} -import memcore.memory.mesh_shm.MeshLocalBankPort - -/** Core-facing access to the tile's shared bank network. */ -class ShmPort(b: GlobalConfig) extends Bundle { - val requests = Vec(SharedMemLayout.channelPerHart(b), new MemRequestIO(b)) - val move = new MvoverPort - - val local = Flipped(new MeshLocalBankPort( - MvoverISA.AddressBits, - MvoverISA.BankBits, - b.memDomain.bankWidth, - math.max(1, log2Ceil(b.frontend.rob_entries)) - )) - - val config = Decoupled(new MemConfigerIO(b)) - val queryValid = Output(Bool()) - val queryVbank = Output(UInt(b.memDomain.vbankIdWidth.W)) - val queryGroups = Input(UInt(b.memDomain.groupCountWidth.W)) - val barrierArrive = Output(Bool()) - val barrierRelease = Input(Bool()) -} diff --git a/arch/src/main/scala/framework/system/core/clink/base.scala b/arch/src/main/scala/framework/system/core/clink/base.scala deleted file mode 100644 index 009619c6e..000000000 --- a/arch/src/main/scala/framework/system/core/clink/base.scala +++ /dev/null @@ -1,7 +0,0 @@ -package framework.system.core.clink - -import chisel3.Bundle - -class CLink extends Bundle - -trait HasCLink { def clink: CLink } diff --git a/arch/src/main/scala/framework/system/core/clink/clink.scala b/arch/src/main/scala/framework/system/core/clink/clink.scala deleted file mode 100644 index 4939e46a0..000000000 --- a/arch/src/main/scala/framework/system/core/clink/clink.scala +++ /dev/null @@ -1,69 +0,0 @@ -package framework.system.core.clink - -import chisel3._ -import chisel3.util._ -import framework.ant.{Params => AntParams, LocalPort} -import framework.top.GlobalConfig -import framework.frontend.globalrs.RobAllocation -import framework.memdomain.frontend.mem.Footprint -import framework.memdomain.frontend.mem.dma.DmaPort -import framework.memdomain.backend.banks.btrace.PhysicalBankHash -import framework.memdomain.backend.shared.SharedMemLayout -import framework.system.core.accelerator.AntMemoryPort -import framework.system.core.rocket.{CpuParams, RoCCCommandBB, RoCCResponseBB} -import hier.core.rocket.{AdmissionPorts, CommandSnapshot, RobFault} -import memcore.bus.chi -import memcore.memory.interlock.{Params => TrackingParams} - -/** Task address-space binding and identity; accepted work drains before context reuse. */ -class Control(p: AntParams, tracking: TrackingParams, pmps: Int)(implicit val cpuParams: CpuParams) extends Bundle { - val hartId = Input(UInt(64.W)) - val shmOwner = Input(UInt(64.W)) - val fingerprint = Output(UInt(64.W)) - val online = Output(Bool()) - - val bind = Flipped(Valid(new Bundle { - val task = UInt(p.taskBits.W) - val snapshot = new CommandSnapshot(tracking, pmps) - })) - - val drained = Output(Bool()) - val workDrained = Input(Bool()) - val halted = Input(Bool()) -} - -/** The tile admits commands and joins their actual ROB, memory and fault lifetimes. */ -class NpuEvents(b: GlobalConfig) extends Bundle { - val command = Flipped(Decoupled(new RoCCCommandBB(b.tile.xLen))) - val response = Decoupled(new RoCCResponseBB(b.tile.xLen)) - val allocation = Valid(new RobAllocation(b)) - val retired = Output(UInt(b.frontend.rob_entries.W)) - val fault = Valid(new RobFault(b.frontend.rob_entries)) - val busy = Output(Bool()) - val interrupt = Output(Bool()) - val footprints = Output(Vec(3, new Footprint(b))) -} - -/** Real NPU/issuer boundary; translation, cache maintenance and task-group storage stay in Tile. */ -class CLinkIO( - b: GlobalConfig, - ant: AntParams, - tracking: TrackingParams, - bus: chi.Params -)( - implicit val cpuParams: CpuParams) - extends CLink { - val ctrl = new Control(ant, tracking, cpuParams.core.nPMPs) - val local = new LocalPort(ant) - val admission = new AdmissionPorts(tracking, cpuParams.core.nPMPs, bus) - val memory = new AntMemoryPort(tracking, bus) - val npu = new NpuEvents(b) - val shm = new ShmPort(b) - val mem = new DmaPort(b.memDomain.dma_buswidth) - - val sharedHashes = - if (b.sim.diffTest && b.memDomain.sharedEnable) - Some(Input(Vec(SharedMemLayout.totalBank(b), new PhysicalBankHash(b)))) - else None - -} diff --git a/arch/src/main/scala/framework/system/core/rocket/BuckyballCommand.scala b/arch/src/main/scala/framework/system/core/rocket/BuckyballCommand.scala deleted file mode 100644 index eba52e452..000000000 --- a/arch/src/main/scala/framework/system/core/rocket/BuckyballCommand.scala +++ /dev/null @@ -1,13 +0,0 @@ -package framework.system.core.rocket - -/** - * Elaboration-time representation of one Buckyball custom instruction. - * Domain ISA builders own the funct7 and operand encodings; boot only streams - * these already-formed commands. - */ -case class BuckyballCommand(funct: Int, rs1: BigInt = 0, rs2: BigInt = 0) - -object BuckyballCommand { - val Custom3Opcode = 0x7b - val Custom3Funct3 = 3 -} diff --git a/arch/src/main/scala/framework/system/core/rocket/CoreParameters.scala b/arch/src/main/scala/framework/system/core/rocket/CoreParameters.scala deleted file mode 100644 index c963e31cb..000000000 --- a/arch/src/main/scala/framework/system/core/rocket/CoreParameters.scala +++ /dev/null @@ -1,175 +0,0 @@ -// See LICENSE.SiFive for license details. - -package freechips.rocketchip.tile - -import chisel3._ -import chisel3.util.isPow2 -import org.chipsalliance.cde.config._ -import freechips.rocketchip.rocket._ -import freechips.rocketchip.util._ - -case object MaxHartIdBits extends Field[Int] -case object CorePAddrBits extends Field[Int] - -// These parameters can be varied per-core -trait CoreParams { - val bootFreqHz: BigInt - val useVM: Boolean - val useHypervisor: Boolean - val useUser: Boolean - val useSupervisor: Boolean - val useDebug: Boolean - val useAtomics: Boolean - val useAtomicsOnlyForIO: Boolean - val useCompressed: Boolean - val useVector: Boolean = false - val vectorUseDCache: Boolean = false - val useRVE: Boolean - val useConditionalZero: Boolean - val useZba: Boolean - val useZbb: Boolean - val useZbs: Boolean - val mulDiv: Option[MulDivParams] - val fpu: Option[FPUParams] - val fetchWidth: Int - val decodeWidth: Int - val retireWidth: Int - val instBits: Int - val nLocalInterrupts: Int - val useNMI: Boolean - val nPMPs: Int - val pmpGranularity: Int - val nBreakpoints: Int - val useBPWatch: Boolean - val mcontextWidth: Int - val scontextWidth: Int - val nPerfCounters: Int - val haveBasicCounters: Boolean - val haveFSDirty: Boolean - val misaWritable: Boolean - val haveCFlush: Boolean - val nL2TLBEntries: Int - val nL2TLBWays: Int - val nPTECacheEntries: Int - val mtvecInit: Option[BigInt] - val mtvecWritable: Boolean - val traceHasWdata: Boolean - val xLen: Int - val pgLevels: Int - def traceCustom: Option[Data] = None - def customIsaExt: Option[String] = None - def customCSRs(implicit p: Parameters): CustomCSRs = new CustomCSRs - - def hasSupervisorMode: Boolean = useSupervisor || useVM - def instBytes: Int = instBits / 8 - def fetchBytes: Int = fetchWidth * instBytes - def lrscCycles: Int - - def dcacheReqTagBits: Int = 6 - - def minFLen: Int = 32 - - def vLen: Int = 0 - def eLen: Int = 0 - def vfLen: Int = 0 - def vfh: Boolean = false - def vExts: Seq[String] = Nil - def hasV: Boolean = vLen >= 128 && eLen >= 64 && vfLen >= 64 - def vMemDataBits: Int = 0 - - def useBitmanip = useZba && useZbb && useZbs -} - -trait HasCoreParameters extends HasTileParameters { - - override lazy val paddrBits: Int = { - val bits = p(CorePAddrBits) - require(bits <= maxPAddrBits) - bits - } - - val coreParams: CoreParams = tileParams.core - - val minFLen = coreParams.fpu.map(_ => coreParams.minFLen).getOrElse(0) - val fLen = coreParams.fpu.map(_.fLen).getOrElse(0) - - val usingMulDiv = coreParams.mulDiv.nonEmpty - val usingFPU = coreParams.fpu.nonEmpty - val usingAtomics = coreParams.useAtomics - val usingAtomicsOnlyForIO = coreParams.useAtomicsOnlyForIO - val usingAtomicsInCache = usingAtomics && !usingAtomicsOnlyForIO - val usingCompressed = coreParams.useCompressed - val usingVector = coreParams.useVector - val usingNMI = coreParams.useNMI - val usingConditionalZero = coreParams.useConditionalZero - - val retireWidth = coreParams.retireWidth - val fetchWidth = coreParams.fetchWidth - val decodeWidth = coreParams.decodeWidth - - val fetchBytes = coreParams.fetchBytes - val coreInstBits = coreParams.instBits - val coreInstBytes = coreInstBits / 8 - val coreDataBits = xLen max fLen max vMemDataBits - val coreDataBytes = coreDataBits / 8 - def coreMaxAddrBits = paddrBits max vaddrBitsExtended - - val nBreakpoints = coreParams.nBreakpoints - val nPMPs = coreParams.nPMPs - val pmpGranularity = coreParams.pmpGranularity - val nPerfCounters = coreParams.nPerfCounters - val mtvecInit = coreParams.mtvecInit - val mtvecWritable = coreParams.mtvecWritable - val customIsaExt = coreParams.customIsaExt - val traceHasWdata = coreParams.traceHasWdata - - def vLen = coreParams.vLen - def eLen = coreParams.eLen - def vfLen = coreParams.vfLen - def vMemDataBits = if (usingVector) coreParams.vMemDataBits else 0 - def maxVLMax = vLen - - if (usingVector) { - require(isPow2(vLen), s"vLen ($vLen) must be a power of 2") - require(eLen >= 32 && vLen % eLen == 0, s"eLen must divide vLen ($vLen) and be no less than 32") - require(eLen == 32 || eLen == 64) - require(vfLen <= eLen) - require(!coreParams.vfh || (vfLen >= 32 && coreParams.minFLen <= 16)) - } - - if (coreParams.useVM) { - if (coreParams.xLen == 32) { - require(coreParams.pgLevels == 2) - } else { - require(coreParams.pgLevels >= 3) - } - } - - lazy val hartIdLen: Int = p(MaxHartIdBits) - - lazy val resetVectorLen: Int = { - val externalLen = paddrBits - require(externalLen <= xLen, s"External reset vector length ($externalLen) must be <= XLEN ($xLen)") - require( - externalLen <= vaddrBitsExtended, - s"External reset vector length ($externalLen) must be <= virtual address bit width ($vaddrBitsExtended)" - ) - externalLen - } - - // Print out log of committed instructions and their writeback values. - // Requires post-processing due to out-of-order writebacks. - val enableCommitLog = false - -} - -abstract class CoreModule(implicit val p: Parameters) extends Module with HasCoreParameters - -abstract class CoreBundle(implicit val p: Parameters) extends ParameterizedBundle()(p) with HasCoreParameters - -// This is a raw commit trace from the core, not the TraceCoreInterface -class TraceBundle(implicit val p: Parameters) extends Bundle with HasCoreParameters { - val insns = Vec(coreParams.retireWidth, new TracedInstruction) - val time = UInt(64.W) - val custom = coreParams.traceCustom -} diff --git a/arch/src/main/scala/framework/system/core/rocket/CpuInterfaces.scala b/arch/src/main/scala/framework/system/core/rocket/CpuInterfaces.scala deleted file mode 100644 index 7d63d73eb..000000000 --- a/arch/src/main/scala/framework/system/core/rocket/CpuInterfaces.scala +++ /dev/null @@ -1,170 +0,0 @@ -// See LICENSE.Berkeley and LICENSE.SiFive for the Rocket interface definitions. -package freechips.rocketchip.rocket - -import chisel3._ -import chisel3.util._ -import org.chipsalliance.cde.config.Parameters -import freechips.rocketchip.tile.{CoreBundle, HasCoreParameters, TileInterrupts} - -/** CPU-facing interfaces, independent of the upstream cache and TileLink implementations. */ -class FrontendReq(implicit p: Parameters) extends CoreBundle()(p) { - val pc = UInt(vaddrBitsExtended.W) - val speculative = Bool() -} - -class FrontendExceptions extends Bundle { - val pf = new Bundle { val inst = Bool() } - val gf = new Bundle { val inst = Bool() } - val ae = new Bundle { val inst = Bool() } -} - -class FrontendResp(implicit p: Parameters) extends CoreBundle()(p) { - val btb = new BTBResp - val pc = UInt(vaddrBitsExtended.W) - val data = UInt((fetchWidth * coreInstBits).W) - val mask = Bits(fetchWidth.W) - val xcpt = new FrontendExceptions - val replay = Bool() -} - -class FrontendPerfEvents extends Bundle { - val acquire = Bool() - val tlbMiss = Bool() -} - -class SFenceReq(implicit p: Parameters) extends CoreBundle()(p) { - val rs1 = Bool() - val rs2 = Bool() - val addr = UInt(vaddrBits.W) - val asid = UInt((asIdBits max 1).W) - val hv = Bool() - val hg = Bool() -} - -class FrontendIO(implicit p: Parameters) extends CoreBundle()(p) { - val might_request = Output(Bool()) - val clock_enabled = Input(Bool()) - val req = Valid(new FrontendReq) - val sfence = Valid(new SFenceReq) - val resp = Flipped(Decoupled(new FrontendResp)) - val gpa = Flipped(Valid(UInt(vaddrBitsExtended.W))) - val gpa_is_pte = Input(Bool()) - val btb_update = Valid(new BTBUpdate) - val bht_update = Valid(new BHTUpdate) - val ras_update = Valid(new RASUpdate) - val flush_icache = Output(Bool()) - val npc = Input(UInt(vaddrBitsExtended.W)) - val perf = Input(new FrontendPerfEvents) - val progress = Output(Bool()) -} - -trait HasCoreMemOp extends HasCoreParameters { - val addr = UInt(coreMaxAddrBits.W) - - val idx = Option.when(usingVM && lgCacheBlockBytes + log2Up(tileParams.dcache.get.nSets) > pgIdxBits)( - UInt(coreMaxAddrBits.W) - ) - - val tag = UInt((coreParams.dcacheReqTagBits + log2Ceil(dcacheArbPorts)).W) - val cmd = UInt(M_SZ.W) - val size = UInt(log2Ceil(log2Ceil(coreDataBytes) + 1).W) - val signed = Bool() - val dprv = UInt(PRV.SZ.W) - val dv = Bool() -} - -trait HasCoreData extends HasCoreParameters { - val data = UInt(coreDataBits.W) - val mask = UInt(coreDataBytes.W) -} - -class HellaCacheReqInternal(implicit p: Parameters) extends CoreBundle()(p) with HasCoreMemOp { - val phys = Bool() - val no_resp = Bool() - val no_alloc = Bool() - val no_xcpt = Bool() -} - -class HellaCacheReq(implicit p: Parameters) extends HellaCacheReqInternal()(p) with HasCoreData - -class HellaCacheResp(implicit p: Parameters) extends CoreBundle()(p) with HasCoreMemOp with HasCoreData { - val replay = Bool() - val has_data = Bool() - val data_word_bypass = UInt(coreDataBits.W) - val data_raw = UInt(coreDataBits.W) - val store_data = UInt(coreDataBits.W) -} - -class AlignmentExceptions extends Bundle { - val ld = Bool() - val st = Bool() -} - -class HellaCacheExceptions extends Bundle { - val ma = new AlignmentExceptions - val pf = new AlignmentExceptions - val gf = new AlignmentExceptions - val ae = new AlignmentExceptions -} - -class HellaCacheWriteData(implicit p: Parameters) extends CoreBundle()(p) with HasCoreData - -class HellaCachePerfEvents extends Bundle { - val acquire = Bool() - val release = Bool() - val grant = Bool() - val tlbMiss = Bool() - val blocked = Bool() - val canAcceptStoreThenLoad = Bool() - val canAcceptStoreThenRMW = Bool() - val canAcceptLoadThenLoad = Bool() - val storeBufferEmptyAfterLoad = Bool() - val storeBufferEmptyAfterStore = Bool() -} - -class HellaCacheIO(implicit p: Parameters) extends CoreBundle()(p) { - val req = Decoupled(new HellaCacheReq) - val s1_kill = Output(Bool()) - val s1_data = Output(new HellaCacheWriteData) - val s2_nack = Input(Bool()) - val s2_nack_cause_raw = Input(Bool()) - val s2_kill = Output(Bool()) - val s2_uncached = Input(Bool()) - val s2_paddr = Input(UInt(paddrBits.W)) - val resp = Flipped(Valid(new HellaCacheResp)) - val replay_next = Input(Bool()) - val s2_xcpt = Input(new HellaCacheExceptions) - val s2_gpa = Input(UInt(vaddrBitsExtended.W)) - val s2_gpa_is_pte = Input(Bool()) - val uncached_resp = Option.when(tileParams.dcache.get.separateUncachedResp)(Flipped(Decoupled(new HellaCacheResp))) - val ordered = Input(Bool()) - val store_pending = Input(Bool()) - val perf = Input(new HellaCachePerfEvents) - val keep_clock_enabled = Output(Bool()) - val clock_enabled = Input(Bool()) -} - -class PTWPerfEvents extends Bundle { - val l2miss = Bool() - val l2hit = Bool() - val pte_miss = Bool() - val pte_hit = Bool() -} - -class DatapathPTWIO(implicit p: Parameters) extends CoreBundle()(p) { - val ptbr = Input(new PTBR) - val hgatp = Input(new PTBR) - val vsatp = Input(new PTBR) - val sfence = Flipped(Valid(new SFenceReq)) - val status = Input(new MStatus) - val hstatus = Input(new HStatus) - val gstatus = Input(new MStatus) - val pmp = Input(Vec(nPMPs, new PMP)) - val perf = Output(new PTWPerfEvents) - val customCSRs = Flipped(coreParams.customCSRs) - val clock_enabled = Output(Bool()) -} - -class CoreInterrupts(val hasBeu: Boolean)(implicit p: Parameters) extends TileInterrupts()(p) { - val buserror = Option.when(hasBeu)(Bool()) -} diff --git a/arch/src/main/scala/framework/system/core/rocket/CpuParams.scala b/arch/src/main/scala/framework/system/core/rocket/CpuParams.scala deleted file mode 100644 index 4c2ef32ae..000000000 --- a/arch/src/main/scala/framework/system/core/rocket/CpuParams.scala +++ /dev/null @@ -1,15 +0,0 @@ -package framework.system.core.rocket - -import freechips.rocketchip.rocket.{BTBParams, DCacheParams, ICacheParams, RocketCoreParams} - -/** Explicit CPU metadata; the upstream parameter conversion stays in the Rocket adapter. */ -case class CpuParams( - core: RocketCoreParams, - dcache: Option[DCacheParams], - icache: Option[ICacheParams], - btb: Option[BTBParams], - physicalAddressBits: Int, - hartIdBits: Int, - beatBytes: Int = 8, - blockBytes: Int = 64, - asIdBits: Int = 0) diff --git a/arch/src/main/scala/framework/system/core/rocket/Parameters.scala b/arch/src/main/scala/framework/system/core/rocket/Parameters.scala deleted file mode 100644 index 732867221..000000000 --- a/arch/src/main/scala/framework/system/core/rocket/Parameters.scala +++ /dev/null @@ -1,24 +0,0 @@ -package framework.system.core.rocket - -import org.chipsalliance.cde.config.Parameters -import freechips.rocketchip.tile.{CoreBeatBytes, CorePAddrBits, CpuTileParams, HasCoreParameters, MaxHartIdBits, TileKey} -import freechips.rocketchip.rocket.ASIdBits -import freechips.rocketchip.subsystem.CacheBlockBytes - -object RocketParameters { - - def apply(cpu: CpuParams): Parameters = Parameters.empty.alterPartial { - case TileKey => CpuTileParams(core = cpu.core, dcache = cpu.dcache, icache = cpu.icache, btb = cpu.btb) - case CorePAddrBits => cpu.physicalAddressBits - case MaxHartIdBits => cpu.hartIdBits - case CoreBeatBytes => cpu.beatBytes - case CacheBlockBytes => cpu.blockBytes - case ASIdBits => cpu.asIdBits - } - -} - -trait HasCpuParameters extends HasCoreParameters { - implicit val cpuParams: CpuParams - implicit lazy val p: Parameters = RocketParameters(cpuParams) -} diff --git a/arch/src/main/scala/framework/system/core/rocket/RoCCCommandBB.scala b/arch/src/main/scala/framework/system/core/rocket/RoCCCommandBB.scala deleted file mode 100644 index da6f1194e..000000000 --- a/arch/src/main/scala/framework/system/core/rocket/RoCCCommandBB.scala +++ /dev/null @@ -1,36 +0,0 @@ -package framework.system.core.rocket - -import chisel3._ -import chisel3.util._ - -/** RoCC response bundle */ -class RoCCResponseBB(xLen: Int = 64) extends Bundle { - val rd = Bits(5.W) - val data = Bits(xLen.W) -} - -/** RoCC interface between a core and an accelerator. */ -class RoCCIO(xLen: Int = 64) extends Bundle { - val cmd = Flipped(Decoupled(new RoCCCommandBB(xLen))) - val resp = Decoupled(new RoCCResponseBB(xLen)) - val busy = Output(Bool()) - val interrupt = Output(Bool()) - val exception = Input(Bool()) -} - -/** RoCC command bundle */ -class RoCCCommandBB(xLen: Int = 64) extends Bundle { - val raw_inst = UInt(32.W) - val pc = UInt(xLen.W) - val funct = UInt(7.W) - val funct3 = UInt(3.W) - val rs2 = Bits(5.W) - val rs1 = Bits(5.W) - val xd = Bool() - val xs1 = Bool() - val xs2 = Bool() - val rd = Bits(5.W) - val opcode = UInt(7.W) - val rs1Data = UInt(xLen.W) - val rs2Data = UInt(xLen.W) -} diff --git a/arch/src/main/scala/framework/system/core/rocket/RocketCoreBB.scala b/arch/src/main/scala/framework/system/core/rocket/RocketCoreBB.scala deleted file mode 100644 index 441caf85c..000000000 --- a/arch/src/main/scala/framework/system/core/rocket/RocketCoreBB.scala +++ /dev/null @@ -1,1486 +0,0 @@ -// See LICENSE.Berkeley for license details. -// See LICENSE.SiFive for license details. - -package framework.system.core.rocket - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public} -import chisel3.util._ -import chisel3.withClock -import freechips.rocketchip.tile._ -import freechips.rocketchip.util._ -import freechips.rocketchip.util.property -import scala.collection.mutable.ArrayBuffer -import freechips.rocketchip.rocket._ - -/** RoCC core IO — used inside Rocket core. */ -class RoCCCoreIOBB(val nRoCCCSRs: Int = 0)(implicit val cpuParams: CpuParams) extends Bundle with HasCpuParameters { - val cmd = Flipped(Decoupled(new RoCCCommandBB)) - val resp = Decoupled(new RoCCResponseBB) - val mem = new HellaCacheIO - val busy = Output(Bool()) - val interrupt = Output(Bool()) - val exception = Input(Bool()) - val csrs = Flipped(Vec(nRoCCCSRs, new CustomCSRIO)) -} - -@instantiable -class RocketBB( - roccCSRs: Seq[CustomCSR], - coreHasBuckyball: Boolean, - hasTaskControl: Boolean, - val hasBusError: Boolean, - dcacheFlushOnFenceI: Boolean, - dcacheCanSupportCFlushLine: Boolean -)( - implicit val cpuParams: CpuParams) - extends Module - with HasRocketCoreParameters - with HasCpuParameters { - def nTotalRoCCCSRs = roccCSRs.size - - override val usingRoCC = coreHasBuckyball || hasTaskControl - - @public val io = IO(new CoreBundle()(p) { - val hartid = Input(UInt(hartIdLen.W)) - val reset_vector = Input(UInt(resetVectorLen.W)) - val interrupts = Input(new CoreInterrupts(hasBusError)) - val interruptReady = Input(Bool()) - val imem = new FrontendIO - val dmem = new HellaCacheIO - val dmemPc = Output(UInt(vaddrBitsExtended.W)) - val roccOlderPending = Output(Bool()) - val roccCommitEligible = Output(Bool()) - val ptw = Flipped(new DatapathPTWIO()) - val fpu = Flipped(new FPUCoreIO()) - val rocc = Flipped(new RoCCCoreIOBB(nTotalRoCCCSRs)) - val trace = Output(new TraceBundle) - val bpwatch = Output(Vec(coreParams.nBreakpoints, new BPWatch(coreParams.retireWidth))) - val cease = Output(Bool()) - val wfi = Output(Bool()) - val traceStall = Input(Bool()) - val vector = if (usingVector) Some(Flipped(new VectorCoreIO)) else None - }) - - import ALU._ - - val clock_en_reg = RegInit(true.B) - val long_latency_stall = Reg(Bool()) - val id_reg_pause = Reg(Bool()) - val imem_might_request_reg = Reg(Bool()) - val clock_en = WireDefault(true.B) - - val gated_clock = - if (!rocketParams.clockGate) clock - else ClockGate(clock, clock_en, "rocket_clock_gate") - - class RocketImpl { // entering gated-clock domain - - // performance counters - def pipelineIDToWB[T <: Data](x: T): T = - RegEnable(RegEnable(RegEnable(x, !ctrl_killd), ex_pc_valid), mem_pc_valid) - - val perfEvents = new EventSets(Seq( - new EventSet( - (mask, hits) => Mux(wb_xcpt, mask(0), wb_valid && pipelineIDToWB((mask & hits).orR)), - Seq( - ("exception", () => false.B), - ("load", () => id_ctrl.mem && id_ctrl.mem_cmd === M_XRD && !id_ctrl.fp), - ("store", () => id_ctrl.mem && id_ctrl.mem_cmd === M_XWR && !id_ctrl.fp), - ("amo", () => usingAtomics.B && id_ctrl.mem && (isAMO(id_ctrl.mem_cmd) || id_ctrl.mem_cmd.isOneOf(M_XLR, M_XSC))), - ("system", () => id_ctrl.csr =/= CSR.N), - ( - "arith", - () => - id_ctrl.wxd && !(id_ctrl.jal || id_ctrl.jalr || id_ctrl.mem || id_ctrl.fp || id_ctrl.mul || id_ctrl.div || id_ctrl.csr =/= CSR.N) - ), - ("branch", () => id_ctrl.branch), - ("jal", () => id_ctrl.jal), - ("jalr", () => id_ctrl.jalr) - ) - ++ (if (!usingMulDiv) Seq() - else - Seq( - ("mul", () => if (pipelinedMul) id_ctrl.mul else id_ctrl.div && (id_ctrl.alu_fn & FN_DIV) =/= FN_DIV), - ("div", () => if (pipelinedMul) id_ctrl.div else id_ctrl.div && (id_ctrl.alu_fn & FN_DIV) === FN_DIV) - )) - ++ (if (!usingFPU) Seq() - else - Seq( - ("fp load", () => id_ctrl.fp && io.fpu.dec.ldst && io.fpu.dec.wen), - ("fp store", () => id_ctrl.fp && io.fpu.dec.ldst && !io.fpu.dec.wen), - ("fp add", () => id_ctrl.fp && io.fpu.dec.fma && io.fpu.dec.swap23), - ("fp mul", () => id_ctrl.fp && io.fpu.dec.fma && !io.fpu.dec.swap23 && !io.fpu.dec.ren3), - ("fp mul-add", () => id_ctrl.fp && io.fpu.dec.fma && io.fpu.dec.ren3), - ("fp div/sqrt", () => id_ctrl.fp && (io.fpu.dec.div || io.fpu.dec.sqrt)), - ( - "fp other", - () => id_ctrl.fp && !(io.fpu.dec.ldst || io.fpu.dec.fma || io.fpu.dec.div || io.fpu.dec.sqrt) - ) - )) - ), - new EventSet( - (mask, hits) => (mask & hits).orR, - Seq( - ( - "load-use interlock", - () => id_ex_hazard && ex_ctrl.mem || id_mem_hazard && mem_ctrl.mem || id_wb_hazard && wb_ctrl.mem - ), - ("long-latency interlock", () => id_sboard_hazard), - ( - "csr interlock", - () => - id_ex_hazard && ex_ctrl.csr =/= CSR.N || id_mem_hazard && mem_ctrl.csr =/= CSR.N || id_wb_hazard && wb_ctrl.csr =/= CSR.N - ), - ("I$ blocked", () => icache_blocked), - ("D$ blocked", () => id_ctrl.mem && dcache_blocked), - ("branch misprediction", () => take_pc_mem && mem_direction_misprediction), - ( - "control-flow target misprediction", - () => take_pc_mem && mem_misprediction && mem_cfi && !mem_direction_misprediction && !icache_blocked - ), - ("flush", () => wb_reg_flush_pipe), - ("replay", () => replay_wb) - ) - ++ (if (!usingMulDiv) Seq() - else - Seq( - ( - "mul/div interlock", - () => - id_ex_hazard && (ex_ctrl.mul || ex_ctrl.div) || id_mem_hazard && (mem_ctrl.mul || mem_ctrl.div) || id_wb_hazard && wb_ctrl.div - ) - )) - ++ (if (!usingFPU) Seq() - else - Seq( - ( - "fp interlock", - () => - id_ex_hazard && ex_ctrl.fp || id_mem_hazard && mem_ctrl.fp || id_wb_hazard && wb_ctrl.fp || id_ctrl.fp && id_stall_fpu - ) - )) - ), - new EventSet( - (mask, hits) => (mask & hits).orR, - Seq( - ("I$ miss", () => io.imem.perf.acquire), - ("D$ miss", () => io.dmem.perf.acquire), - ("D$ release", () => io.dmem.perf.release), - ("ITLB miss", () => io.imem.perf.tlbMiss), - ("DTLB miss", () => io.dmem.perf.tlbMiss), - ("L2 TLB miss", () => io.ptw.perf.l2miss) - ) - ) - )) - - val pipelinedMul = usingMulDiv && mulDivParams.mulUnroll == xLen - - val decode_table = { - (if (usingMulDiv) new MDecode(pipelinedMul) +: (xLen > 32).option(new M64Decode(pipelinedMul)).toSeq else Nil) ++: - (if (usingAtomics) new ADecode +: (xLen > 32).option(new A64Decode).toSeq else Nil) ++: - (if (fLen >= 32) new FDecode +: (xLen > 32).option(new F64Decode).toSeq else Nil) ++: - (if (fLen >= 64) new DDecode +: (xLen > 32).option(new D64Decode).toSeq else Nil) ++: - (if (minFLen == 16) - new HDecode +: (xLen > 32).option(new H64Decode).toSeq ++: (fLen >= 64).option(new HDDecode).toSeq - else Nil) ++: - (usingRoCC.option(new RoCCDecode)) ++: - (if (xLen == 32) new I32Decode else new I64Decode) +: - (usingVM.option(new SVMDecode)) ++: - (usingSupervisor.option(new SDecode)) ++: - (usingHypervisor.option(new HypervisorDecode)) ++: - ((usingHypervisor && (xLen == 64)).option(new Hypervisor64Decode)) ++: - (usingDebug.option(new DebugDecode)) ++: - (usingNMI.option(new NMIDecode)) ++: - (usingConditionalZero.option(new ConditionalZeroDecode)) ++: - Seq(new FenceIDecode(dcacheFlushOnFenceI || coreParams.haveCFlush)) ++: - coreParams.haveCFlush.option(new CFlushDecode(dcacheCanSupportCFlushLine)) ++: - rocketParams.haveCease.option(new CeaseDecode) ++: - usingVector.option(new VCFGDecode) ++: - (if (coreParams.useZba) new ZbaDecode +: (xLen > 32).option(new Zba64Decode).toSeq else Nil) ++: - (if (coreParams.useZbb) Seq(new ZbbDecode, if (xLen == 32) new Zbb32Decode else new Zbb64Decode) else Nil) ++: - coreParams.useZbs.option(new ZbsDecode) ++: - Seq(new IDecode) - } flatMap (_.table) - - val ex_ctrl = Reg(new IntCtrlSigs) - val mem_ctrl = Reg(new IntCtrlSigs) - val wb_ctrl = Reg(new IntCtrlSigs) - - val ex_reg_xcpt_interrupt = Reg(Bool()) - val ex_reg_valid = Reg(Bool()) - val ex_reg_rvc = Reg(Bool()) - val ex_reg_btb_resp = Reg(new BTBResp) - val ex_reg_xcpt = Reg(Bool()) - val ex_reg_flush_pipe = Reg(Bool()) - val ex_reg_load_use = Reg(Bool()) - val ex_reg_cause = Reg(UInt()) - val ex_reg_replay = Reg(Bool()) - val ex_reg_pc = Reg(UInt()) - val ex_reg_mem_size = Reg(UInt()) - val ex_reg_hls = Reg(Bool()) - val ex_reg_inst = Reg(Bits()) - val ex_reg_raw_inst = Reg(UInt()) - val ex_reg_wphit = Reg(Vec(nBreakpoints, Bool())) - val ex_reg_set_vconfig = Reg(Bool()) - - val mem_reg_xcpt_interrupt = Reg(Bool()) - val mem_reg_valid = Reg(Bool()) - val mem_reg_rvc = Reg(Bool()) - val mem_reg_btb_resp = Reg(new BTBResp) - val mem_reg_xcpt = Reg(Bool()) - val mem_reg_replay = Reg(Bool()) - val mem_reg_flush_pipe = Reg(Bool()) - val mem_reg_cause = Reg(UInt()) - val mem_reg_slow_bypass = Reg(Bool()) - val mem_reg_load = Reg(Bool()) - val mem_reg_store = Reg(Bool()) - val mem_reg_set_vconfig = Reg(Bool()) - val mem_reg_sfence = Reg(Bool()) - val mem_reg_pc = Reg(UInt()) - val mem_reg_inst = Reg(Bits()) - val mem_reg_mem_size = Reg(UInt()) - val mem_reg_hls_or_dv = Reg(Bool()) - val mem_reg_raw_inst = Reg(UInt()) - val mem_reg_wdata = Reg(Bits()) - val mem_reg_rs2 = Reg(Bits()) - val mem_br_taken = Reg(Bool()) - val take_pc_mem = Wire(Bool()) - val mem_reg_wphit = Reg(Vec(nBreakpoints, Bool())) - - val wb_reg_valid = Reg(Bool()) - val wb_reg_xcpt = Reg(Bool()) - val wb_reg_replay = Reg(Bool()) - val wb_reg_flush_pipe = Reg(Bool()) - val wb_reg_cause = Reg(UInt()) - val wb_reg_set_vconfig = Reg(Bool()) - val wb_reg_sfence = Reg(Bool()) - val wb_reg_pc = Reg(UInt()) - val wb_reg_mem_size = Reg(UInt()) - val wb_reg_hls_or_dv = Reg(Bool()) - val wb_reg_hfence_v = Reg(Bool()) - val wb_reg_hfence_g = Reg(Bool()) - val wb_reg_inst = Reg(Bits()) - val wb_reg_raw_inst = Reg(UInt()) - val wb_reg_wdata = Reg(Bits()) - val wb_reg_rs2 = Reg(Bits()) - val take_pc_wb = Wire(Bool()) - val wb_reg_wphit = Reg(Vec(nBreakpoints, Bool())) - - val take_pc_mem_wb = take_pc_wb || take_pc_mem - val take_pc = take_pc_mem_wb - - // decode stage - val ibuf = Module(new IBuf) - val id_expanded_inst = ibuf.io.inst.map(_.bits.inst) - val id_raw_inst = ibuf.io.inst.map(_.bits.raw) - val id_inst = id_expanded_inst.map(_.bits) - ibuf.io.imem <> io.imem.resp - ibuf.io.kill := take_pc - - require(decodeWidth == 1 /* TODO */ && retireWidth == decodeWidth) - require(!(coreParams.useRVE && coreParams.fpu.nonEmpty), "Can't select both RVE and floating-point") - require(!(coreParams.useRVE && coreParams.useHypervisor), "Can't select both RVE and Hypervisor") - val id_ctrl = Wire(new IntCtrlSigs).decode(id_inst(0), decode_table) - - val lgNXRegs = if (coreParams.useRVE) 4 else 5 - val regAddrMask = (1 << lgNXRegs) - 1 - - def decodeReg(x: UInt) = (x.extract(x.getWidth - 1, lgNXRegs).asBool, x(lgNXRegs - 1, 0)) - val (id_raddr3_illegal, id_raddr3) = decodeReg(id_expanded_inst(0).rs3) - val (id_raddr2_illegal, id_raddr2) = decodeReg(id_expanded_inst(0).rs2) - val (id_raddr1_illegal, id_raddr1) = decodeReg(id_expanded_inst(0).rs1) - val (id_waddr_illegal, id_waddr) = decodeReg(id_expanded_inst(0).rd) - - val id_load_use = Wire(Bool()) - val id_reg_fence = RegInit(false.B) - val id_ren = IndexedSeq(id_ctrl.rxs1, id_ctrl.rxs2) - val id_raddr = IndexedSeq(id_raddr1, id_raddr2) - val rf = new RegFile(regAddrMask, xLen) - val id_rs = id_raddr.map(rf.read _) - val ctrl_killd = Wire(Bool()) - - // A scheduler needs the custom ISA even without compute Balls. - val csrP = - if (usingRoCC) p - else p.alterPartial { case CoreRoCCCount => 0 } - - val csr = Module(new CSRFile( - perfEvents, - coreParams.customCSRs.decls, - roccCSRs, - hasBusError - )(csrP) { - override def usingRoCC: Boolean = RocketBB.this.usingRoCC - }) - - val id_csr_en = id_ctrl.csr.isOneOf(CSR.S, CSR.C, CSR.W) - val id_system_insn = id_ctrl.csr === CSR.I - val id_csr_ren = id_ctrl.csr.isOneOf(CSR.S, CSR.C) && id_expanded_inst(0).rs1 === 0.U - val id_csr = Mux(id_system_insn && id_ctrl.mem, CSR.N, Mux(id_csr_ren, CSR.R, id_ctrl.csr)) - val id_csr_flush = id_system_insn || (id_csr_en && !id_csr_ren && csr.io.decode(0).write_flush) - val id_set_vconfig = - Seq(Instructions.VSETVLI, Instructions.VSETIVLI, Instructions.VSETVL).map(_ === id_inst(0)).orR && usingVector.B - - id_ctrl.vec := false.B - if (usingVector) { - val v_decode = rocketParams.vector.get.decoder(p) - v_decode.io.inst := id_inst(0) - v_decode.io.vconfig := csr.io.vector.get.vconfig - when(v_decode.io.legal) { - id_ctrl.legal := !csr.io.vector.get.vconfig.vtype.vill - id_ctrl.fp := v_decode.io.fp - id_ctrl.rocc := false.B - id_ctrl.branch := false.B - id_ctrl.jal := false.B - id_ctrl.jalr := false.B - id_ctrl.rxs2 := v_decode.io.read_rs2 - id_ctrl.rxs1 := v_decode.io.read_rs1 - id_ctrl.mem := false.B - id_ctrl.rfs1 := v_decode.io.read_frs1 - id_ctrl.rfs2 := false.B - id_ctrl.rfs3 := false.B - id_ctrl.wfd := v_decode.io.write_frd - id_ctrl.mul := false.B - id_ctrl.div := false.B - id_ctrl.wxd := v_decode.io.write_rd - id_ctrl.csr := CSR.N - id_ctrl.fence_i := false.B - id_ctrl.fence := false.B - id_ctrl.amo := false.B - id_ctrl.dp := false.B - id_ctrl.vec := true.B - } - } - - val id_illegal_insn = !id_ctrl.legal || - (id_ctrl.mul || id_ctrl.div) && !csr.io.status.isa('m' - 'a') || - id_ctrl.amo && !csr.io.status.isa('a' - 'a') || - id_ctrl.fp && (csr.io.decode(0).fp_illegal || (io.fpu.illegal_rm && !id_ctrl.vec)) || - (id_ctrl.vec) && (csr.io.decode(0).vector_illegal || csr.io.vector.map(_.vconfig.vtype.vill).getOrElse( - false.B - )) || - id_ctrl.dp && !csr.io.status.isa('d' - 'a') || - ibuf.io.inst(0).bits.rvc && !csr.io.status.isa('c' - 'a') || - id_raddr2_illegal && id_ctrl.rxs2 || - id_raddr1_illegal && id_ctrl.rxs1 || - id_waddr_illegal && id_ctrl.wxd || - id_ctrl.rocc && csr.io.decode(0).rocc_illegal || - id_csr_en && (csr.io.decode(0).read_illegal || !id_csr_ren && csr.io.decode(0).write_illegal) || - !ibuf.io.inst(0).bits.rvc && (id_system_insn && csr.io.decode(0).system_illegal) - - val id_virtual_insn = id_ctrl.legal && - ((id_csr_en && !(!id_csr_ren && csr.io.decode(0).write_illegal) && csr.io.decode(0).virtual_access_illegal) || - (!ibuf.io.inst(0).bits.rvc && id_system_insn && csr.io.decode(0).virtual_system_illegal)) - - // stall decode for fences (now, for AMO.rl; later, for AMO.aq and FENCE) - val id_amo_aq = id_inst(0)(26) - val id_amo_rl = id_inst(0)(25) - val id_fence_succ = id_inst(0)(23, 20) - val id_fence_next = id_ctrl.fence || id_ctrl.amo && id_amo_aq - val id_mem_busy = !io.dmem.ordered || io.dmem.req.valid - when(!id_mem_busy)(id_reg_fence := false.B) - - val id_rocc_busy = usingRoCC.B && - (io.rocc.busy || ex_reg_valid && ex_ctrl.rocc || - mem_reg_valid && mem_ctrl.rocc || wb_reg_valid && wb_ctrl.rocc) - - val id_csr_rocc_write = roccCSRs.map(_.id.U === id_inst(0)(31, 20)).orR && id_csr_en && !id_csr_ren - val id_vec_busy = io.vector.map(v => v.backend_busy || v.trap_check_busy).getOrElse(false.B) - - val id_do_fence = WireDefault(id_rocc_busy && (id_ctrl.fence || id_csr_rocc_write) || - id_vec_busy && id_ctrl.fence || - id_mem_busy && (id_ctrl.amo && id_amo_rl || id_ctrl.fence_i || id_reg_fence && (id_ctrl.mem || id_ctrl.rocc))) - - val bpu = Module(new BreakpointUnit(nBreakpoints)) - bpu.io.status := csr.io.status - bpu.io.bp := csr.io.bp - bpu.io.pc := ibuf.io.pc - bpu.io.ea := mem_reg_wdata - bpu.io.mcontext := csr.io.mcontext - bpu.io.scontext := csr.io.scontext - - val id_xcpt0 = ibuf.io.inst(0).bits.xcpt0 - val id_xcpt1 = ibuf.io.inst(0).bits.xcpt1 - - val deliveredInterrupt = csr.io.interrupt && io.interruptReady - - val (id_xcpt, id_cause) = checkExceptions(List( - (deliveredInterrupt, csr.io.interrupt_cause), - (bpu.io.debug_if, CSR.debugTriggerCause.U), - (bpu.io.xcpt_if, Causes.breakpoint.U), - (id_xcpt0.pf.inst, Causes.fetch_page_fault.U), - (id_xcpt0.gf.inst, Causes.fetch_guest_page_fault.U), - (id_xcpt0.ae.inst, Causes.fetch_access.U), - (id_xcpt1.pf.inst, Causes.fetch_page_fault.U), - (id_xcpt1.gf.inst, Causes.fetch_guest_page_fault.U), - (id_xcpt1.ae.inst, Causes.fetch_access.U), - (id_virtual_insn, Causes.virtual_instruction.U), - (id_illegal_insn, Causes.illegal_instruction.U) - )) - - val idCoverCauses = List( - (CSR.debugTriggerCause, "DEBUG_TRIGGER"), - (Causes.breakpoint, "BREAKPOINT"), - (Causes.fetch_access, "FETCH_ACCESS"), - (Causes.illegal_instruction, "ILLEGAL_INSTRUCTION") - ) ++ (if (usingVM) - List( - (Causes.fetch_page_fault, "FETCH_PAGE_FAULT") - ) - else Nil) - - coverExceptions(id_xcpt, id_cause, "DECODE", idCoverCauses) - - val dcache_bypass_data = - if (fastLoadByte) io.dmem.resp.bits.data(xLen - 1, 0) - else if (fastLoadWord) io.dmem.resp.bits.data_word_bypass(xLen - 1, 0) - else wb_reg_wdata - - // detect bypass opportunities - val ex_waddr = ex_reg_inst(11, 7) & regAddrMask.U - val mem_waddr = mem_reg_inst(11, 7) & regAddrMask.U - val wb_waddr = wb_reg_inst(11, 7) & regAddrMask.U - - val bypass_sources = IndexedSeq( - (true.B, 0.U, 0.U), // treat reading x0 as a bypass - (ex_reg_valid && ex_ctrl.wxd, ex_waddr, mem_reg_wdata), - (mem_reg_valid && mem_ctrl.wxd && !mem_ctrl.mem, mem_waddr, wb_reg_wdata), - (mem_reg_valid && mem_ctrl.wxd, mem_waddr, dcache_bypass_data) - ) - - val id_bypass_src = id_raddr.map(raddr => bypass_sources.map(s => s._1 && s._2 === raddr)) - - // execute stage - val bypass_mux = bypass_sources.map(_._3) - val ex_reg_rs_bypass = Reg(Vec(id_raddr.size, Bool())) - val ex_reg_rs_lsb = Reg(Vec(id_raddr.size, UInt(log2Ceil(bypass_sources.size).W))) - val ex_reg_rs_msb = Reg(Vec(id_raddr.size, UInt())) - - val ex_rs = - for (i <- 0 until id_raddr.size) - yield Mux(ex_reg_rs_bypass(i), bypass_mux(ex_reg_rs_lsb(i)), Cat(ex_reg_rs_msb(i), ex_reg_rs_lsb(i))) - - val ex_imm = ImmGen(ex_ctrl.sel_imm, ex_reg_inst) - val ex_rs1shl = Mux(ex_reg_inst(3), ex_rs(0)(31, 0), ex_rs(0)) << ex_reg_inst(14, 13) - - val ex_op1 = MuxLookup(ex_ctrl.sel_alu1, 0.S)(Seq( - A1_RS1 -> ex_rs(0).asSInt, - A1_PC -> ex_reg_pc.asSInt, - A1_RS1SHL -> (if (rocketParams.useZba) ex_rs1shl.asSInt else 0.S) - )) - - val ex_op2_oh = - UIntToOH(Mux(ex_ctrl.sel_alu2(0), (ex_reg_inst >> 20).asUInt, ex_rs(1))(log2Ceil(xLen) - 1, 0)).asSInt - - val ex_op2 = MuxLookup(ex_ctrl.sel_alu2, 0.S)(Seq( - A2_RS2 -> ex_rs(1).asSInt, - A2_IMM -> ex_imm, - A2_SIZE -> Mux(ex_reg_rvc, 2.S, 4.S) - ) ++ (if (coreParams.useZbs) - Seq( - A2_RS2OH -> ex_op2_oh, - A2_IMMOH -> ex_op2_oh - ) - else Nil)) - - val (ex_new_vl, ex_new_vconfig) = - if (usingVector) { - val ex_new_vtype = VType.fromUInt(MuxCase( - ex_rs(1), - Seq( - ex_reg_inst(31, 30).andR -> ex_reg_inst(29, 20), - !ex_reg_inst(31) -> ex_reg_inst(30, 20) - ) - )) - val ex_avl = Mux( - ex_ctrl.rxs1, - Mux( - ex_reg_inst(19, 15) === 0.U, - Mux(ex_reg_inst(11, 7) === 0.U, csr.io.vector.get.vconfig.vl, ex_new_vtype.vlMax), - ex_rs(0) - ), - ex_reg_inst(19, 15) - ) - val ex_new_vl = ex_new_vtype.vl(ex_avl, csr.io.vector.get.vconfig.vl, false.B, false.B, false.B) - val ex_new_vconfig = Wire(new VConfig) - ex_new_vconfig.vtype := ex_new_vtype - ex_new_vconfig.vl := ex_new_vl - (Some(ex_new_vl), Some(ex_new_vconfig)) - } else { (None, None) } - - val alu = Module(new ALU) - alu.io.dw := ex_ctrl.alu_dw - alu.io.fn := ex_ctrl.alu_fn - alu.io.in2 := ex_op2.asUInt - alu.io.in1 := ex_op1.asUInt - - // multiplier and divider - val div = Module(new MulDiv(if (pipelinedMul) mulDivParams.copy(mulUnroll = 0) else mulDivParams, width = xLen)) - div.io.req.valid := ex_reg_valid && ex_ctrl.div - div.io.req.bits.dw := ex_ctrl.alu_dw - div.io.req.bits.fn := ex_ctrl.alu_fn - div.io.req.bits.in1 := ex_rs(0) - div.io.req.bits.in2 := ex_rs(1) - div.io.req.bits.tag := ex_waddr - - val mul = pipelinedMul.option { - val m = Module(new PipelinedMultiplier(xLen, 2)) - m.io.req.valid := ex_reg_valid && ex_ctrl.mul - m.io.req.bits := div.io.req.bits - m - } - - ex_reg_valid := !ctrl_killd - ex_reg_replay := !take_pc && ibuf.io.inst(0).valid && ibuf.io.inst(0).bits.replay - ex_reg_xcpt := !ctrl_killd && id_xcpt - ex_reg_xcpt_interrupt := !take_pc && ibuf.io.inst(0).valid && deliveredInterrupt - - when(!ctrl_killd) { - ex_ctrl := id_ctrl - ex_reg_rvc := ibuf.io.inst(0).bits.rvc - ex_ctrl.csr := id_csr - when(id_ctrl.fence && id_fence_succ === 0.U)(id_reg_pause := true.B) - when(id_fence_next)(id_reg_fence := true.B) - when(id_xcpt) { // pass PC down ALU writeback pipeline for badaddr - ex_ctrl.alu_fn := FN_ADD - ex_ctrl.alu_dw := DW_XPR - ex_ctrl.sel_alu1 := A1_RS1 // badaddr := instruction - ex_ctrl.sel_alu2 := A2_ZERO - when(id_xcpt1.asUInt.orR) { // badaddr := PC+2 - ex_ctrl.sel_alu1 := A1_PC - ex_ctrl.sel_alu2 := A2_SIZE - ex_reg_rvc := true.B - } - when(bpu.io.xcpt_if || id_xcpt0.asUInt.orR) { // badaddr := PC - ex_ctrl.sel_alu1 := A1_PC - ex_ctrl.sel_alu2 := A2_ZERO - } - } - ex_reg_flush_pipe := id_ctrl.fence_i || id_csr_flush - ex_reg_load_use := id_load_use - ex_reg_hls := usingHypervisor.B && id_system_insn && id_ctrl.mem_cmd.isOneOf(M_XRD, M_XWR, M_HLVX) - ex_reg_mem_size := Mux(usingHypervisor.B && id_system_insn, id_inst(0)(27, 26), id_inst(0)(13, 12)) - when(id_ctrl.mem_cmd.isOneOf(M_SFENCE, M_HFENCEV, M_HFENCEG, M_FLUSH_ALL)) { - ex_reg_mem_size := Cat(id_raddr2 =/= 0.U, id_raddr1 =/= 0.U) - } - when(id_ctrl.mem_cmd === M_SFENCE && csr.io.status.v) { - ex_ctrl.mem_cmd := M_HFENCEV - } - if (dcacheFlushOnFenceI || coreParams.haveCFlush) { - when(id_ctrl.fence_i) { - ex_reg_mem_size := 0.U - } - } - - for (i <- 0 until id_raddr.size) { - val do_bypass = id_bypass_src(i).reduce(_ || _) - val bypass_src = PriorityEncoder(id_bypass_src(i)) - ex_reg_rs_bypass(i) := do_bypass - ex_reg_rs_lsb(i) := bypass_src - when(id_ren(i) && !do_bypass) { - ex_reg_rs_lsb(i) := id_rs(i)(log2Ceil(bypass_sources.size) - 1, 0) - ex_reg_rs_msb(i) := id_rs(i) >> log2Ceil(bypass_sources.size) - } - } - when(id_illegal_insn || id_virtual_insn) { - val inst = Mux(ibuf.io.inst(0).bits.rvc, id_raw_inst(0)(15, 0), id_raw_inst(0)) - ex_reg_rs_bypass(0) := false.B - ex_reg_rs_lsb(0) := inst(log2Ceil(bypass_sources.size) - 1, 0) - ex_reg_rs_msb(0) := inst >> log2Ceil(bypass_sources.size) - } - } - when(!ctrl_killd || deliveredInterrupt || ibuf.io.inst(0).bits.replay) { - ex_reg_cause := id_cause - ex_reg_inst := id_inst(0) - ex_reg_raw_inst := id_raw_inst(0) - ex_reg_pc := ibuf.io.pc - ex_reg_btb_resp := ibuf.io.btb_resp - ex_reg_wphit := bpu.io.bpwatch.map(bpw => bpw.ivalid(0)) - ex_reg_set_vconfig := id_set_vconfig && !id_xcpt - } - - // replay inst in ex stage? - val ex_pc_valid = ex_reg_valid || ex_reg_replay || ex_reg_xcpt_interrupt - val wb_dcache_miss = wb_ctrl.mem && !io.dmem.resp.valid - - val replay_ex_structural = ex_ctrl.mem && !io.dmem.req.ready || - ex_ctrl.div && !div.io.req.ready || - ex_ctrl.vec && !io.vector.map(_.ex.ready).getOrElse(true.B) - - val replay_ex_load_use = wb_dcache_miss && ex_reg_load_use - val replay_ex = ex_reg_replay || (ex_reg_valid && (replay_ex_structural || replay_ex_load_use)) - val ctrl_killx = take_pc_mem_wb || replay_ex || !ex_reg_valid - // detect 2-cycle load-use delay for LB/LH/SC - val ex_slow_bypass = ex_ctrl.mem_cmd === M_XSC || ex_reg_mem_size < 2.U - val ex_sfence = - usingVM.B && ex_ctrl.mem && (ex_ctrl.mem_cmd === M_SFENCE || ex_ctrl.mem_cmd === M_HFENCEV || ex_ctrl.mem_cmd === M_HFENCEG) - - val (ex_xcpt, ex_cause) = checkExceptions(List( - (ex_reg_xcpt_interrupt || ex_reg_xcpt, ex_reg_cause) - )) - - val exCoverCauses = idCoverCauses - coverExceptions(ex_xcpt, ex_cause, "EXECUTE", exCoverCauses) - - // memory stage - val mem_pc_valid = mem_reg_valid || mem_reg_replay || mem_reg_xcpt_interrupt - - val mem_br_target = mem_reg_pc.asSInt + - Mux( - mem_ctrl.branch && mem_br_taken, - ImmGen(IMM_SB, mem_reg_inst), - Mux(mem_ctrl.jal, ImmGen(IMM_UJ, mem_reg_inst), Mux(mem_reg_rvc, 2.S, 4.S)) - ) - - val mem_npc = (Mux( - mem_ctrl.jalr || mem_reg_sfence, - encodeVirtualAddress(mem_reg_wdata, mem_reg_wdata).asSInt, - mem_br_target - ) & -2.S).asUInt - - val mem_wrong_npc = - Mux( - ex_pc_valid, - mem_npc =/= ex_reg_pc, - Mux(ibuf.io.inst(0).valid || ibuf.io.imem.valid, mem_npc =/= ibuf.io.pc, true.B) - ) - - val mem_npc_misaligned = !csr.io.status.isa('c' - 'a') && mem_npc(1) && !mem_reg_sfence - val mem_int_wdata = - Mux(!mem_reg_xcpt && (mem_ctrl.jalr ^ mem_npc_misaligned), mem_br_target, mem_reg_wdata.asSInt).asUInt - val mem_cfi = mem_ctrl.branch || mem_ctrl.jalr || mem_ctrl.jal - val mem_cfi_taken = (mem_ctrl.branch && mem_br_taken) || mem_ctrl.jalr || mem_ctrl.jal - val mem_direction_misprediction = mem_ctrl.branch && mem_br_taken =/= (usingBTB.B && mem_reg_btb_resp.taken) - val mem_misprediction = if (usingBTB) mem_wrong_npc else mem_cfi_taken - take_pc_mem := mem_reg_valid && !mem_reg_xcpt && (mem_misprediction || mem_reg_sfence) - - mem_reg_valid := !ctrl_killx - mem_reg_replay := !take_pc_mem_wb && replay_ex - mem_reg_xcpt := !ctrl_killx && ex_xcpt - mem_reg_xcpt_interrupt := !take_pc_mem_wb && ex_reg_xcpt_interrupt - - // on pipeline flushes, cause mem_npc to hold the sequential npc, which - // will drive the W-stage npc mux - when(mem_reg_valid && mem_reg_flush_pipe) { - mem_reg_sfence := false.B - }.elsewhen(ex_pc_valid) { - mem_ctrl := ex_ctrl - mem_reg_rvc := ex_reg_rvc - mem_reg_load := ex_ctrl.mem && isRead(ex_ctrl.mem_cmd) - mem_reg_store := ex_ctrl.mem && isWrite(ex_ctrl.mem_cmd) - mem_reg_sfence := ex_sfence - mem_reg_btb_resp := ex_reg_btb_resp - mem_reg_flush_pipe := ex_reg_flush_pipe - mem_reg_slow_bypass := ex_slow_bypass - mem_reg_wphit := ex_reg_wphit - mem_reg_set_vconfig := ex_reg_set_vconfig - - mem_reg_cause := ex_cause - mem_reg_inst := ex_reg_inst - mem_reg_raw_inst := ex_reg_raw_inst - mem_reg_mem_size := ex_reg_mem_size - mem_reg_hls_or_dv := io.dmem.req.bits.dv - mem_reg_pc := ex_reg_pc - // IDecode ensured they are 1H - mem_reg_wdata := Mux(ex_reg_set_vconfig, ex_new_vl.getOrElse(alu.io.out), alu.io.out) - mem_br_taken := alu.io.cmp_out - - when(ex_ctrl.rxs2 && (ex_ctrl.mem || ex_ctrl.rocc || ex_sfence)) { - val size = Mux(ex_ctrl.rocc, log2Ceil(xLen / 8).U, ex_reg_mem_size) - mem_reg_rs2 := new StoreGen(size, 0.U, ex_rs(1), coreDataBytes).data - } - if (usingVector) { - when(ex_reg_set_vconfig) { - mem_reg_rs2 := ex_new_vconfig.get.asUInt - } - } - when(ex_ctrl.jalr && csr.io.status.debug) { - // flush I$ on D-mode JALR to effect uncached fetch without D$ flush - mem_ctrl.fence_i := true.B - mem_reg_flush_pipe := true.B - } - } - - val mem_breakpoint = (mem_reg_load && bpu.io.xcpt_ld) || (mem_reg_store && bpu.io.xcpt_st) - val mem_debug_breakpoint = (mem_reg_load && bpu.io.debug_ld) || (mem_reg_store && bpu.io.debug_st) - - val (mem_ldst_xcpt, mem_ldst_cause) = checkExceptions(List( - (mem_debug_breakpoint, CSR.debugTriggerCause.U), - (mem_breakpoint, Causes.breakpoint.U) - )) - - val (mem_xcpt, mem_cause) = checkExceptions(List( - (mem_reg_xcpt_interrupt || mem_reg_xcpt, mem_reg_cause), - (mem_reg_valid && mem_npc_misaligned, Causes.misaligned_fetch.U), - (mem_reg_valid && mem_ldst_xcpt, mem_ldst_cause) - )) - - val memCoverCauses = (exCoverCauses ++ List( - (CSR.debugTriggerCause, "DEBUG_TRIGGER"), - (Causes.breakpoint, "BREAKPOINT"), - (Causes.misaligned_fetch, "MISALIGNED_FETCH") - )).distinct - - coverExceptions(mem_xcpt, mem_cause, "MEMORY", memCoverCauses) - - val dcache_kill_mem = mem_reg_valid && mem_ctrl.wxd && io.dmem.replay_next // structural hazard on writeback port - val fpu_kill_mem = mem_reg_valid && mem_ctrl.fp && io.fpu.nack_mem - val vec_kill_mem = mem_reg_valid && mem_ctrl.mem && io.vector.map(_.mem.block_mem).getOrElse(false.B) - val vec_kill_all = mem_reg_valid && io.vector.map(_.mem.block_all).getOrElse(false.B) - val replay_mem = dcache_kill_mem || mem_reg_replay || fpu_kill_mem || vec_kill_mem || vec_kill_all - val killm_common = dcache_kill_mem || take_pc_wb || mem_reg_xcpt || !mem_reg_valid - div.io.kill := killm_common && RegNext(div.io.req.fire) - val ctrl_killm = killm_common || mem_xcpt || fpu_kill_mem || vec_kill_mem - - // writeback stage - wb_reg_valid := !ctrl_killm - wb_reg_replay := replay_mem && !take_pc_wb - wb_reg_xcpt := mem_xcpt && !take_pc_wb && !io.vector.map(_.mem.block_all).getOrElse(false.B) - wb_reg_flush_pipe := !ctrl_killm && mem_reg_flush_pipe - when(mem_pc_valid) { - wb_ctrl := mem_ctrl - wb_reg_sfence := mem_reg_sfence - wb_reg_wdata := Mux(!mem_reg_xcpt && mem_ctrl.fp && mem_ctrl.wxd, io.fpu.toint_data, mem_int_wdata) - when(mem_ctrl.rocc || mem_reg_sfence || mem_reg_set_vconfig) { - wb_reg_rs2 := mem_reg_rs2 - } - wb_reg_cause := mem_cause - wb_reg_inst := mem_reg_inst - wb_reg_raw_inst := mem_reg_raw_inst - wb_reg_mem_size := mem_reg_mem_size - wb_reg_hls_or_dv := mem_reg_hls_or_dv - wb_reg_hfence_v := mem_ctrl.mem_cmd === M_HFENCEV - wb_reg_hfence_g := mem_ctrl.mem_cmd === M_HFENCEG - wb_reg_pc := mem_reg_pc - wb_reg_wphit := mem_reg_wphit | bpu.io.bpwatch.map { bpw => - (bpw.rvalid(0) && mem_reg_load) || (bpw.wvalid(0) && mem_reg_store) - } - wb_reg_set_vconfig := mem_reg_set_vconfig - } - - val (wb_xcpt, wb_cause) = checkExceptions(List( - (wb_reg_xcpt, wb_reg_cause), - (wb_reg_valid && wb_ctrl.mem && io.dmem.s2_xcpt.pf.st, Causes.store_page_fault.U), - (wb_reg_valid && wb_ctrl.mem && io.dmem.s2_xcpt.pf.ld, Causes.load_page_fault.U), - (wb_reg_valid && wb_ctrl.mem && io.dmem.s2_xcpt.gf.st, Causes.store_guest_page_fault.U), - (wb_reg_valid && wb_ctrl.mem && io.dmem.s2_xcpt.gf.ld, Causes.load_guest_page_fault.U), - (wb_reg_valid && wb_ctrl.mem && io.dmem.s2_xcpt.ae.st, Causes.store_access.U), - (wb_reg_valid && wb_ctrl.mem && io.dmem.s2_xcpt.ae.ld, Causes.load_access.U), - (wb_reg_valid && wb_ctrl.mem && io.dmem.s2_xcpt.ma.st, Causes.misaligned_store.U), - (wb_reg_valid && wb_ctrl.mem && io.dmem.s2_xcpt.ma.ld, Causes.misaligned_load.U) - )) - - val wbCoverCauses = List( - (Causes.misaligned_store, "MISALIGNED_STORE"), - (Causes.misaligned_load, "MISALIGNED_LOAD"), - (Causes.store_access, "STORE_ACCESS"), - (Causes.load_access, "LOAD_ACCESS") - ) ++ (if (usingVM) - List( - (Causes.store_page_fault, "STORE_PAGE_FAULT"), - (Causes.load_page_fault, "LOAD_PAGE_FAULT") - ) - else Nil) ++ (if (usingHypervisor) - List( - (Causes.store_guest_page_fault, "STORE_GUEST_PAGE_FAULT"), - (Causes.load_guest_page_fault, "LOAD_GUEST_PAGE_FAULT") - ) - else Nil) - - coverExceptions(wb_xcpt, wb_cause, "WRITEBACK", wbCoverCauses) - - val wb_pc_valid = wb_reg_valid || wb_reg_replay || wb_reg_xcpt - val wb_wxd = wb_reg_valid && wb_ctrl.wxd - val wb_set_sboard = wb_ctrl.div || wb_dcache_miss || wb_ctrl.rocc || wb_ctrl.vec - val replay_wb_common = io.dmem.s2_nack || wb_reg_replay - val replay_wb_rocc = wb_reg_valid && wb_ctrl.rocc && !io.rocc.cmd.ready - val replay_wb_csr: Bool = wb_reg_valid && csr.io.rw_stall - val replay_wb_vec = wb_reg_valid && io.vector.map(_.wb.replay).getOrElse(false.B) - val replay_wb = replay_wb_common || replay_wb_rocc || replay_wb_csr || replay_wb_vec - take_pc_wb := replay_wb || wb_xcpt || csr.io.eret || wb_reg_flush_pipe - - // writeback arbitration - val dmem_resp_xpu = !io.dmem.resp.bits.tag(0).asBool - val dmem_resp_fpu = io.dmem.resp.bits.tag(0).asBool - val dmem_resp_waddr = io.dmem.resp.bits.tag(5, 1) - val dmem_resp_valid = io.dmem.resp.valid && io.dmem.resp.bits.has_data - val dmem_resp_replay = dmem_resp_valid && io.dmem.resp.bits.replay - - class LLWB extends Bundle { - val data = UInt(xLen.W) - val tag = UInt(5.W) - } - - val ll_arb = Module(new Arbiter(new LLWB, 3)) // div, rocc, vec - ll_arb.io.in.foreach(_.valid := false.B) - ll_arb.io.in.foreach(_.bits := DontCare) - val ll_wdata = WireInit(ll_arb.io.out.bits.data) - val ll_waddr = WireInit(ll_arb.io.out.bits.tag) - val ll_wen = WireInit(ll_arb.io.out.fire) - ll_arb.io.out.ready := !wb_wxd - - div.io.resp.ready := ll_arb.io.in(0).ready - ll_arb.io.in(0).valid := div.io.resp.valid - ll_arb.io.in(0).bits.data := div.io.resp.bits.data - ll_arb.io.in(0).bits.tag := div.io.resp.bits.tag - - if (usingRoCC) { - io.rocc.resp.ready := ll_arb.io.in(1).ready - ll_arb.io.in(1).valid := io.rocc.resp.valid - ll_arb.io.in(1).bits.data := io.rocc.resp.bits.data - ll_arb.io.in(1).bits.tag := io.rocc.resp.bits.rd - } else { - // tie off RoCC - io.rocc.resp.ready := false.B - io.rocc.mem.req.ready := false.B - } - - io.vector.map { v => - v.resp.ready := Mux(v.resp.bits.fp, !(dmem_resp_valid && dmem_resp_fpu), ll_arb.io.in(2).ready) - ll_arb.io.in(2).valid := v.resp.valid && !v.resp.bits.fp - ll_arb.io.in(2).bits.data := v.resp.bits.data - ll_arb.io.in(2).bits.tag := v.resp.bits.rd - } - // Dont care mem since not all RoCC need accessing memory - io.rocc.mem := DontCare - - when(dmem_resp_replay && dmem_resp_xpu) { - ll_arb.io.out.ready := false.B - ll_waddr := dmem_resp_waddr - ll_wen := true.B - } - - val wb_valid = wb_reg_valid && !replay_wb && !wb_xcpt - val wb_wen = wb_valid && wb_ctrl.wxd - val rf_wen = wb_wen || ll_wen - val rf_waddr = Mux(ll_wen, ll_waddr, wb_waddr) - - val rf_wdata = Mux( - dmem_resp_valid && dmem_resp_xpu, - io.dmem.resp.bits.data(xLen - 1, 0), - Mux( - ll_wen, - ll_wdata, - Mux( - wb_ctrl.csr =/= CSR.N, - csr.io.rw.rdata, - Mux(wb_ctrl.mul, mul.map(_.io.resp.bits.data).getOrElse(wb_reg_wdata), wb_reg_wdata) - ) - ) - ) - - when(rf_wen)(rf.write(rf_waddr, rf_wdata)) - - // hook up control/status regfile - csr.io.ungated_clock := clock - csr.io.decode(0).inst := id_inst(0) - csr.io.exception := wb_xcpt - csr.io.cause := wb_cause - csr.io.retire := wb_valid - csr.io.inst(0) := (if (usingCompressed) - Cat(Mux(wb_reg_raw_inst(1, 0).andR, wb_reg_inst >> 16, 0.U), wb_reg_raw_inst(15, 0)) - else wb_reg_inst) - csr.io.interrupts := io.interrupts - csr.io.hartid := io.hartid - io.fpu.fcsr_rm := csr.io.fcsr_rm - val vector_fcsr_flags = io.vector.map(_.set_fflags.bits).getOrElse(0.U(5.W)) - val vector_fcsr_flags_valid = io.vector.map(_.set_fflags.valid).getOrElse(false.B) - csr.io.fcsr_flags.valid := io.fpu.fcsr_flags.valid | vector_fcsr_flags_valid - csr.io.fcsr_flags.bits := (io.fpu.fcsr_flags.bits & Fill(5, io.fpu.fcsr_flags.valid)) | (vector_fcsr_flags & Fill( - 5, - vector_fcsr_flags_valid - )) - io.fpu.time := csr.io.time(31, 0) - io.fpu.hartid := io.hartid - csr.io.rocc_interrupt := io.rocc.interrupt - csr.io.pc := wb_reg_pc - - val tval_dmem_addr = !wb_reg_xcpt - - val tval_any_addr = tval_dmem_addr || - wb_reg_cause.isOneOf( - Causes.breakpoint.U, - Causes.fetch_access.U, - Causes.fetch_page_fault.U, - Causes.fetch_guest_page_fault.U - ) - - val tval_inst = wb_reg_cause === Causes.illegal_instruction.U - val tval_valid = wb_xcpt && (tval_any_addr || tval_inst) - csr.io.gva := wb_xcpt && (tval_any_addr && csr.io.status.v || tval_dmem_addr && wb_reg_hls_or_dv) - csr.io.tval := Mux(tval_valid, encodeVirtualAddress(wb_reg_wdata, wb_reg_wdata), 0.U) - - val (htval, mhtinst_read_pseudo) = { - val htval_valid_imem = wb_reg_xcpt && wb_reg_cause === Causes.fetch_guest_page_fault.U - val htval_imem = Mux(htval_valid_imem, io.imem.gpa.bits, 0.U) - assert(!htval_valid_imem || io.imem.gpa.valid) - - val htval_valid_dmem = - wb_xcpt && tval_dmem_addr && io.dmem.s2_xcpt.gf.asUInt.orR && !io.dmem.s2_xcpt.pf.asUInt.orR - val htval_dmem = Mux(htval_valid_dmem, io.dmem.s2_gpa, 0.U) - - val htval = (htval_dmem | htval_imem) >> hypervisorExtraAddrBits - // read pseudoinstruction if a guest-page fault is caused by an implicit memory access for VS-stage address - // translation - val mhtinst_read_pseudo = (io.imem.gpa_is_pte && htval_valid_imem) || (io.dmem.s2_gpa_is_pte && htval_valid_dmem) - (htval, mhtinst_read_pseudo) - } - - csr.io.vector.foreach { v => - v.set_vconfig.valid := wb_reg_set_vconfig && wb_reg_valid - v.set_vconfig.bits := wb_reg_rs2.asTypeOf(new VConfig) - v.set_vs_dirty := wb_valid && wb_ctrl.vec - v.set_vstart.valid := wb_valid && wb_reg_set_vconfig - v.set_vstart.bits := 0.U - } - - io.vector.foreach { v => - when(v.wb.retire || v.wb.xcpt || wb_ctrl.vec) { - csr.io.pc := v.wb.pc - csr.io.retire := v.wb.retire - csr.io.inst(0) := v.wb.inst - when(v.wb.xcpt && !wb_reg_xcpt) { - wb_xcpt := true.B - wb_cause := v.wb.cause - csr.io.tval := v.wb.tval - } - } - v.wb.store_pending := io.dmem.store_pending - v.wb.vxrm := csr.io.vector.get.vxrm - v.wb.frm := csr.io.fcsr_rm - csr.io.vector.get.set_vxsat := v.set_vxsat - when(v.set_vconfig.valid) { - csr.io.vector.get.set_vconfig.valid := true.B - csr.io.vector.get.set_vconfig.bits := v.set_vconfig.bits - } - when(v.set_vstart.valid) { - csr.io.vector.get.set_vstart.valid := true.B - csr.io.vector.get.set_vstart.bits := v.set_vstart.bits - } - } - - csr.io.htval := htval - csr.io.mhtinst_read_pseudo := mhtinst_read_pseudo - io.ptw.ptbr := csr.io.ptbr - io.ptw.hgatp := csr.io.hgatp - io.ptw.vsatp := csr.io.vsatp - (io.ptw.customCSRs.csrs zip csr.io.customCSRs).map { case (lhs, rhs) => lhs <> rhs } - io.ptw.status := csr.io.status - io.ptw.hstatus := csr.io.hstatus - io.ptw.gstatus := csr.io.gstatus - io.ptw.pmp := csr.io.pmp - csr.io.rw.addr := wb_reg_inst(31, 20) - csr.io.rw.cmd := CSR.maskCmd(wb_reg_valid, wb_ctrl.csr) - csr.io.rw.wdata := wb_reg_wdata - - io.rocc.csrs <> csr.io.roccCSRs - io.trace.time := csr.io.time - io.trace.insns := csr.io.trace - if (rocketParams.debugROB.isDefined) { - val sz = rocketParams.debugROB.get.size - if (sz < 1) { // use unsynthesizable ROB - val csr_trace_with_wdata = WireInit(csr.io.trace(0)) - csr_trace_with_wdata.wdata.get := rf_wdata - val should_wb = WireInit((wb_ctrl.wfd || (wb_ctrl.wxd && wb_waddr =/= 0.U)) && !csr.io.trace(0).exception) - val has_wb = WireInit(wb_ctrl.wxd && wb_wen && !wb_set_sboard) - val wb_addr = WireInit(wb_waddr + Mux(wb_ctrl.wfd, 32.U, 0.U)) - - io.vector.foreach { v => - when(v.wb.retire) { - should_wb := v.wb.rob_should_wb - has_wb := false.B - wb_addr := Cat(v.wb.rob_should_wb_fp, csr_trace_with_wdata.insn(11, 7)) - } - } - - DebugROB.pushTrace(clock, reset, io.hartid, csr_trace_with_wdata, should_wb, has_wb, wb_addr) - - io.trace.insns(0) := DebugROB.popTrace(clock, reset, io.hartid) - - DebugROB.pushWb(clock, reset, io.hartid, ll_wen, rf_waddr, rf_wdata) - } else { // synthesizable ROB (no FPRs) - require(!usingVector, "Synthesizable ROB does not support vector implementations") - val csr_trace_with_wdata = WireInit(csr.io.trace(0)) - csr_trace_with_wdata.wdata.get := rf_wdata - - val debug_rob = Module(new HardDebugROB(sz, 32)) - debug_rob.io.i_insn := csr_trace_with_wdata - debug_rob.io.should_wb := (wb_ctrl.wfd || (wb_ctrl.wxd && wb_waddr =/= 0.U)) && - !csr.io.trace(0).exception - debug_rob.io.has_wb := wb_ctrl.wxd && wb_wen && !wb_set_sboard - debug_rob.io.tag := wb_waddr + Mux(wb_ctrl.wfd, 32.U, 0.U) - - debug_rob.io.wb_val := ll_wen - debug_rob.io.wb_tag := rf_waddr - debug_rob.io.wb_data := rf_wdata - - io.trace.insns(0) := debug_rob.io.o_insn - } - } else { - io.trace.insns := csr.io.trace - } - for (((iobpw, wphit), bp) <- io.bpwatch zip wb_reg_wphit zip csr.io.bp) { - iobpw.valid(0) := wphit - iobpw.action := bp.control.action - // tie off bpwatch valids - iobpw.rvalid.foreach(_ := false.B) - iobpw.wvalid.foreach(_ := false.B) - iobpw.ivalid.foreach(_ := false.B) - } - - val hazard_targets = Seq( - (id_ctrl.rxs1 && id_raddr1 =/= 0.U, id_raddr1), - (id_ctrl.rxs2 && id_raddr2 =/= 0.U, id_raddr2), - (id_ctrl.wxd && id_waddr =/= 0.U, id_waddr) - ) - - val fp_hazard_targets = Seq( - (io.fpu.dec.ren1, id_raddr1), - (io.fpu.dec.ren2, id_raddr2), - (io.fpu.dec.ren3, id_raddr3), - (io.fpu.dec.wen, id_waddr) - ) - - val sboard = new Scoreboard(32, true) - sboard.clear(ll_wen, ll_waddr) - - def id_sboard_clear_bypass(r: UInt) = - // ll_waddr arrives late when D$ has ECC, so reshuffle the hazard check - if (!tileParams.dcache.get.dataECC.isDefined) ll_wen && ll_waddr === r - else div.io.resp.fire && div.io.resp.bits.tag === r || dmem_resp_replay && dmem_resp_xpu && dmem_resp_waddr === r - - val id_sboard_hazard = checkHazards(hazard_targets, rd => sboard.read(rd) && !id_sboard_clear_bypass(rd)) - sboard.set(wb_set_sboard && wb_wen, wb_waddr) - - // stall for RAW/WAW hazards on CSRs, loads, AMOs, and mul/div in execute stage. - val ex_cannot_bypass = - ex_ctrl.csr =/= CSR.N || ex_ctrl.jalr || ex_ctrl.mem || ex_ctrl.mul || ex_ctrl.div || ex_ctrl.fp || ex_ctrl.rocc || ex_ctrl.vec - val data_hazard_ex = ex_ctrl.wxd && checkHazards(hazard_targets, _ === ex_waddr) - val fp_data_hazard_ex = id_ctrl.fp && ex_ctrl.wfd && checkHazards(fp_hazard_targets, _ === ex_waddr) - val id_ex_hazard = ex_reg_valid && (data_hazard_ex && ex_cannot_bypass || fp_data_hazard_ex) - - // stall for RAW/WAW hazards on CSRs, LB/LH, and mul/div in memory stage. - val mem_mem_cmd_bh = - if (fastLoadWord) (!fastLoadByte).B && mem_reg_slow_bypass - else true.B - - val mem_cannot_bypass = - mem_ctrl.csr =/= CSR.N || mem_ctrl.mem && mem_mem_cmd_bh || mem_ctrl.mul || mem_ctrl.div || mem_ctrl.fp || mem_ctrl.rocc || mem_ctrl.vec - val data_hazard_mem = mem_ctrl.wxd && checkHazards(hazard_targets, _ === mem_waddr) - val fp_data_hazard_mem = id_ctrl.fp && mem_ctrl.wfd && checkHazards(fp_hazard_targets, _ === mem_waddr) - val id_mem_hazard = mem_reg_valid && (data_hazard_mem && mem_cannot_bypass || fp_data_hazard_mem) - id_load_use := mem_reg_valid && data_hazard_mem && mem_ctrl.mem - - val id_vconfig_hazard = id_ctrl.vec && ( - (ex_reg_valid && ex_reg_set_vconfig) || - (mem_reg_valid && mem_reg_set_vconfig) || - (wb_reg_valid && wb_reg_set_vconfig) - ) - - // stall for RAW/WAW hazards on load/AMO misses and mul/div in writeback. - val data_hazard_wb = wb_ctrl.wxd && checkHazards(hazard_targets, _ === wb_waddr) - val fp_data_hazard_wb = id_ctrl.fp && wb_ctrl.wfd && checkHazards(fp_hazard_targets, _ === wb_waddr) - val id_wb_hazard = wb_reg_valid && (data_hazard_wb && wb_set_sboard || fp_data_hazard_wb) - - val id_stall_fpu = - if (usingFPU) { - val fp_sboard = new Scoreboard(32) - fp_sboard.set(((wb_dcache_miss || wb_ctrl.vec) && wb_ctrl.wfd || io.fpu.sboard_set) && wb_valid, wb_waddr) - val v_ll = io.vector.map(v => v.resp.fire && v.resp.bits.fp).getOrElse(false.B) - fp_sboard.clear((dmem_resp_replay && dmem_resp_fpu) || v_ll, io.fpu.ll_resp_tag) - fp_sboard.clear(io.fpu.sboard_clr, io.fpu.sboard_clra) - - checkHazards(fp_hazard_targets, fp_sboard.read _) - } else false.B - - val dcache_blocked = { - // speculate that a blocked D$ will unblock the cycle after a Grant - val blocked = Reg(Bool()) - blocked := !io.dmem.req.ready && io.dmem.clock_enabled && !io.dmem.perf.grant && (blocked || io.dmem.req.valid || io.dmem.s2_nack) - blocked && !io.dmem.perf.grant - } - - val rocc_blocked = RegInit(false.B) - rocc_blocked := !wb_xcpt && !io.rocc.cmd.ready && (io.rocc.cmd.valid || rocc_blocked) - - val ctrl_stalld = - id_ex_hazard || id_mem_hazard || id_wb_hazard || id_sboard_hazard || - id_vconfig_hazard || - csr.io.singleStep && (ex_reg_valid || mem_reg_valid || wb_reg_valid) || - id_csr_en && csr.io.decode(0).fp_csr && !io.fpu.fcsr_rdy || - id_csr_en && csr.io.decode(0).vector_csr && id_vec_busy || - id_ctrl.fp && id_stall_fpu || - id_ctrl.mem && dcache_blocked || // reduce activity during D$ misses - id_ctrl.rocc && rocc_blocked || // reduce activity while RoCC is busy - id_ctrl.div && (!(div.io.req.ready || (div.io.resp.valid && !wb_wxd)) || div.io.req.valid) || // reduce odds of replay - !clock_en || - id_do_fence || io.rocc.busy || - csr.io.csr_stall || - id_reg_pause || - io.traceStall - - ctrl_killd := !ibuf.io.inst(0).valid || ibuf.io.inst( - 0 - ).bits.replay || take_pc_mem_wb || ctrl_stalld || deliveredInterrupt - - io.imem.req.valid := take_pc - io.imem.req.bits.speculative := !take_pc_wb - io.imem.req.bits.pc := - Mux( - wb_xcpt || csr.io.eret, - csr.io.evec, // exception or [m|s]ret - Mux( - replay_wb, - wb_reg_pc, // replay - mem_npc - ) - ) // flush or branch misprediction - io.imem.flush_icache := wb_reg_valid && wb_ctrl.fence_i && !io.dmem.s2_nack - io.imem.might_request := { - imem_might_request_reg := ex_pc_valid || mem_pc_valid || io.ptw.customCSRs.disableICacheClockGate || io.vector.map( - _.trap_check_busy - ).getOrElse(false.B) - imem_might_request_reg - } - io.imem.progress := RegNext(wb_reg_valid && !replay_wb_common) - io.imem.sfence.valid := wb_reg_valid && wb_reg_sfence - io.imem.sfence.bits.rs1 := wb_reg_mem_size(0) - io.imem.sfence.bits.rs2 := wb_reg_mem_size(1) - io.imem.sfence.bits.addr := wb_reg_wdata - io.imem.sfence.bits.asid := wb_reg_rs2 - io.imem.sfence.bits.hv := wb_reg_hfence_v - io.imem.sfence.bits.hg := wb_reg_hfence_g - io.ptw.sfence := io.imem.sfence - - ibuf.io.inst(0).ready := !ctrl_stalld - - io.imem.btb_update.valid := mem_reg_valid && !take_pc_wb && mem_wrong_npc && (!mem_cfi || mem_cfi_taken) - io.imem.btb_update.bits.isValid := mem_cfi - io.imem.btb_update.bits.cfiType := - Mux( - (mem_ctrl.jal || mem_ctrl.jalr) && mem_waddr(0), - CFIType.call, - Mux( - mem_ctrl.jalr && (mem_reg_inst(19, 15) & regAddrMask.U) === BitPat("b00?01"), - CFIType.ret, - Mux(mem_ctrl.jal || mem_ctrl.jalr, CFIType.jump, CFIType.branch) - ) - ) - io.imem.btb_update.bits.target := io.imem.req.bits.pc - io.imem.btb_update.bits.br_pc := (if (usingCompressed) mem_reg_pc + Mux(mem_reg_rvc, 0.U, 2.U) else mem_reg_pc) - io.imem.btb_update.bits.pc := ~(~io.imem.btb_update.bits.br_pc | (coreInstBytes * fetchWidth - 1).U) - io.imem.btb_update.bits.prediction := mem_reg_btb_resp - io.imem.btb_update.bits.taken := DontCare - - io.imem.bht_update.valid := mem_reg_valid && !take_pc_wb - io.imem.bht_update.bits.pc := io.imem.btb_update.bits.pc - io.imem.bht_update.bits.taken := mem_br_taken - io.imem.bht_update.bits.mispredict := mem_wrong_npc - io.imem.bht_update.bits.branch := mem_ctrl.branch - io.imem.bht_update.bits.prediction := mem_reg_btb_resp.bht - - // Connect RAS in Frontend - io.imem.ras_update := DontCare - - io.fpu.valid := !ctrl_killd && id_ctrl.fp - io.fpu.killx := ctrl_killx - io.fpu.killm := killm_common - io.fpu.inst := id_inst(0) - io.fpu.fromint_data := ex_rs(0) - io.fpu.ll_resp_val := dmem_resp_valid && dmem_resp_fpu - io.fpu.ll_resp_data := (if (minFLen == 32) io.dmem.resp.bits.data_word_bypass else io.dmem.resp.bits.data) - io.fpu.ll_resp_type := io.dmem.resp.bits.size - io.fpu.ll_resp_tag := dmem_resp_waddr - io.fpu.keep_clock_enabled := io.ptw.customCSRs.disableCoreClockGate - - io.fpu.v_sew := csr.io.vector.map(_.vconfig.vtype.vsew).getOrElse(0.U) - - io.vector.map { v => - when(!(dmem_resp_valid && dmem_resp_fpu)) { - io.fpu.ll_resp_val := v.resp.valid && v.resp.bits.fp - io.fpu.ll_resp_data := v.resp.bits.data - io.fpu.ll_resp_type := v.resp.bits.size - io.fpu.ll_resp_tag := v.resp.bits.rd - } - } - - io.vector.foreach { v => - v.ex.valid := ex_reg_valid && (ex_ctrl.vec || rocketParams.vector.get.issueVConfig.B && ex_reg_set_vconfig) && !ctrl_killx - v.ex.inst := ex_reg_inst - v.ex.vconfig := csr.io.vector.get.vconfig - v.ex.vstart := Mux(mem_reg_valid && mem_ctrl.vec || wb_reg_valid && wb_ctrl.vec, 0.U, csr.io.vector.get.vstart) - v.ex.rs1 := ex_rs(0) - v.ex.rs2 := ex_rs(1) - v.ex.pc := ex_reg_pc - v.mem.frs1 := io.fpu.store_data - v.killm := killm_common - v.status := csr.io.status - } - - io.dmemPc := ex_reg_pc - io.dmem.req.valid := ex_reg_valid && ex_ctrl.mem - val ex_dcache_tag = Cat(ex_waddr, ex_ctrl.fp) - require(coreParams.dcacheReqTagBits >= ex_dcache_tag.getWidth) - io.dmem.req.bits.tag := ex_dcache_tag - io.dmem.req.bits.cmd := ex_ctrl.mem_cmd - io.dmem.req.bits.size := ex_reg_mem_size - io.dmem.req.bits.signed := !Mux(ex_reg_hls, ex_reg_inst(20), ex_reg_inst(14)) - io.dmem.req.bits.phys := false.B - io.dmem.req.bits.addr := encodeVirtualAddress(ex_rs(0), alu.io.adder_out) - io.dmem.req.bits.idx.foreach(_ := io.dmem.req.bits.addr) - io.dmem.req.bits.dprv := Mux(ex_reg_hls, csr.io.hstatus.spvp, csr.io.status.dprv) - io.dmem.req.bits.dv := ex_reg_hls || csr.io.status.dv - io.dmem.req.bits.no_resp := !isRead(ex_ctrl.mem_cmd) || (!ex_ctrl.fp && ex_waddr === 0.U) - io.dmem.req.bits.no_alloc := DontCare - io.dmem.req.bits.no_xcpt := DontCare - io.dmem.req.bits.data := DontCare - io.dmem.req.bits.mask := DontCare - - io.dmem.s1_data.data := (if (fLen == 0) mem_reg_rs2 - else Mux(mem_ctrl.fp, Fill(coreDataBits / fLen, io.fpu.store_data), mem_reg_rs2)) - io.dmem.s1_data.mask := DontCare - - io.dmem.s1_kill := killm_common || mem_ldst_xcpt || fpu_kill_mem || vec_kill_mem - io.dmem.s2_kill := false.B - // don't let D$ go to sleep if we're probably going to use it soon - io.dmem.keep_clock_enabled := ibuf.io.inst(0).valid && id_ctrl.mem && !csr.io.csr_stall - - // Pipeline classification, not a second accelerator decoder. Readiness is deliberately absent. - io.roccOlderPending := usingRoCC.B && - ((mem_reg_valid && mem_ctrl.rocc) || (wb_reg_valid && wb_ctrl.rocc)) - io.roccCommitEligible := usingRoCC.B && wb_reg_valid && wb_ctrl.rocc && !replay_wb_common && !wb_xcpt - io.rocc.cmd.valid := wb_reg_valid && wb_ctrl.rocc && !replay_wb_common - io.rocc.exception := wb_xcpt && csr.io.status.xs.orR - io.rocc.cmd.bits.raw_inst := wb_reg_inst - io.rocc.cmd.bits.funct := wb_reg_inst(31, 25) - io.rocc.cmd.bits.funct3 := wb_reg_inst(14, 12) - io.rocc.cmd.bits.rs2 := wb_reg_inst(24, 20) - io.rocc.cmd.bits.rs1 := wb_reg_inst(19, 15) - io.rocc.cmd.bits.pc := wb_reg_pc - io.rocc.cmd.bits.xd := wb_reg_inst(14) - io.rocc.cmd.bits.xs1 := wb_reg_inst(13) - io.rocc.cmd.bits.xs2 := wb_reg_inst(12) - io.rocc.cmd.bits.rd := wb_reg_inst(11, 7) - io.rocc.cmd.bits.opcode := wb_reg_inst(6, 0) - io.rocc.cmd.bits.rs1Data := wb_reg_wdata - io.rocc.cmd.bits.rs2Data := wb_reg_rs2 - - // gate the clock - val unpause = - csr.io.time(rocketParams.lgPauseCycles - 1, 0) === 0.U || csr.io.inhibit_cycle || io.dmem.perf.release || take_pc - when(unpause)(id_reg_pause := false.B) - io.cease := csr.io.status.cease && !clock_en_reg - io.wfi := csr.io.status.wfi - if (rocketParams.clockGate) { - long_latency_stall := csr.io.csr_stall || io.dmem.perf.blocked || id_reg_pause && !unpause - clock_en := clock_en_reg || ex_pc_valid || (!long_latency_stall && io.imem.resp.valid) - clock_en_reg := - ex_pc_valid || mem_pc_valid || wb_pc_valid || // instruction in flight - io.ptw.customCSRs.disableCoreClockGate || // chicken bit - !div.io.req.ready || // mul/div in flight - usingFPU.B && !io.fpu.fcsr_rdy || // long-latency FPU in flight - io.dmem.replay_next || // long-latency load replaying - (!long_latency_stall && (ibuf.io.inst(0).valid || io.imem.resp.valid)) // instruction pending - - assert(!(ex_pc_valid || mem_pc_valid || wb_pc_valid) || clock_en) - } - - // evaluate performance counters - val icache_blocked = !(io.imem.resp.valid || RegNext(io.imem.resp.valid)) - csr.io.counters foreach { c => c.inc := RegNext(perfEvents.evaluate(c.eventSel)) } - - val coreMonitorBundle = Wire(new CoreMonitorBundle(xLen, fLen)) - - coreMonitorBundle.clock := clock - coreMonitorBundle.reset := reset - coreMonitorBundle.hartid := io.hartid - coreMonitorBundle.timer := csr.io.time(31, 0) - coreMonitorBundle.valid := csr.io.trace(0).valid && !csr.io.trace(0).exception - coreMonitorBundle.pc := csr.io.trace(0).iaddr(vaddrBitsExtended - 1, 0).sextTo(xLen) - coreMonitorBundle.wrenx := wb_wen && !wb_set_sboard - coreMonitorBundle.wrenf := false.B - coreMonitorBundle.wrdst := wb_waddr - coreMonitorBundle.wrdata := rf_wdata - coreMonitorBundle.rd0src := wb_reg_inst(19, 15) - coreMonitorBundle.rd0val := RegNext(RegNext(ex_rs(0))) - coreMonitorBundle.rd1src := wb_reg_inst(24, 20) - coreMonitorBundle.rd1val := RegNext(RegNext(ex_rs(1))) - coreMonitorBundle.inst := csr.io.trace(0).insn - coreMonitorBundle.excpt := csr.io.trace(0).exception - coreMonitorBundle.priv_mode := csr.io.trace(0).priv - - if (enableCommitLog) { - val t = csr.io.trace(0) - val rd = wb_waddr - val wfd = wb_ctrl.wfd - val wxd = wb_ctrl.wxd - val has_data = wb_wen && !wb_set_sboard - - when(t.valid && !t.exception) { - when(wfd) { - printf("%d 0x%x (0x%x) f%d p%d 0xXXXXXXXXXXXXXXXX\n", t.priv, t.iaddr, t.insn, rd, rd + 32.U) - } - .elsewhen(wxd && rd =/= 0.U && has_data) { - printf("%d 0x%x (0x%x) x%d 0x%x\n", t.priv, t.iaddr, t.insn, rd, rf_wdata) - } - .elsewhen(wxd && rd =/= 0.U && !has_data) { - printf("%d 0x%x (0x%x) x%d p%d 0xXXXXXXXXXXXXXXXX\n", t.priv, t.iaddr, t.insn, rd, rd) - } - .otherwise { - printf("%d 0x%x (0x%x)\n", t.priv, t.iaddr, t.insn) - } - } - - when(ll_wen && rf_waddr =/= 0.U) { - printf("x%d p%d 0x%x\n", rf_waddr, rf_waddr, rf_wdata) - } - } else { - when(csr.io.trace(0).valid) { - printf( - "C%d: %d [%d] pc=[%x] W[r%d=%x][%d] R[r%d=%x] R[r%d=%x] inst=[%x] DASM(%x) wb_xcpt:%d\n", - io.hartid, - coreMonitorBundle.timer, - coreMonitorBundle.valid, - coreMonitorBundle.pc, - Mux(wb_ctrl.wxd || wb_ctrl.wfd, coreMonitorBundle.wrdst, 0.U), - Mux(coreMonitorBundle.wrenx, coreMonitorBundle.wrdata, 0.U), - coreMonitorBundle.wrenx, - Mux(wb_ctrl.rxs1 || wb_ctrl.rfs1, coreMonitorBundle.rd0src, 0.U), - Mux(wb_ctrl.rxs1 || wb_ctrl.rfs1, coreMonitorBundle.rd0val, 0.U), - Mux(wb_ctrl.rxs2 || wb_ctrl.rfs2, coreMonitorBundle.rd1src, 0.U), - Mux(wb_ctrl.rxs2 || wb_ctrl.rfs2, coreMonitorBundle.rd1val, 0.U), - coreMonitorBundle.inst, - coreMonitorBundle.inst, - wb_xcpt - ) - } - } - - // CoreMonitorBundle for late latency writes - val xrfWriteBundle = Wire(new CoreMonitorBundle(xLen, fLen)) - - xrfWriteBundle.clock := clock - xrfWriteBundle.reset := reset - xrfWriteBundle.hartid := io.hartid - xrfWriteBundle.timer := csr.io.time(31, 0) - xrfWriteBundle.valid := false.B - xrfWriteBundle.pc := 0.U - xrfWriteBundle.wrdst := rf_waddr - xrfWriteBundle.wrenx := rf_wen && !(csr.io.trace(0).valid && wb_wen && (wb_waddr === rf_waddr)) - xrfWriteBundle.wrenf := false.B - xrfWriteBundle.wrdata := rf_wdata - xrfWriteBundle.rd0src := 0.U - xrfWriteBundle.rd0val := 0.U - xrfWriteBundle.rd1src := 0.U - xrfWriteBundle.rd1val := 0.U - xrfWriteBundle.inst := 0.U - xrfWriteBundle.excpt := false.B - xrfWriteBundle.priv_mode := csr.io.trace(0).priv - - if (rocketParams.haveSimTimeout) - PlusArg.timeout( - name = "max_core_cycles", - docstring = "Kill the emulation after INT rdtime cycles. Off if 0." - )(csr.io.time) - - } // leaving gated-clock domain - val rocketImpl = withClock(gated_clock)(new RocketImpl) - - def checkExceptions(x: Seq[(Bool, UInt)]) = - (WireInit(x.map(_._1).reduce(_ || _)), WireInit(PriorityMux(x))) - - def coverExceptions( - exceptionValid: Bool, - cause: UInt, - labelPrefix: String, - coverCausesLabels: Seq[(Int, String)] - ): Unit = { - for ((coverCause, label) <- coverCausesLabels) { - property.cover(exceptionValid && (cause === coverCause.U), s"${labelPrefix}_$label") - } - } - - def checkHazards(targets: Seq[(Bool, UInt)], cond: UInt => Bool) = - targets.map(h => h._1 && cond(h._2)).reduce(_ || _) - - def encodeVirtualAddress(a0: UInt, ea: UInt) = - if (vaddrBitsExtended == vaddrBits) ea - else { - // efficient means to compress 64-bit VA into vaddrBits+1 bits - // (VA is bad if VA(vaddrBits) != VA(vaddrBits-1)) - val b = vaddrBitsExtended - 1 - val a = (a0 >> b).asSInt - val msb = Mux(a === 0.S || a === -1.S, ea(b), !ea(b - 1)) - Cat(msb, ea(b - 1, 0)) - } - - class Scoreboard(n: Int, zero: Boolean = false) { - def set(en: Bool, addr: UInt): Unit = update(en, _next | mask(en, addr)) - def clear(en: Bool, addr: UInt): Unit = update(en, _next & ~mask(en, addr)) - def read(addr: UInt): Bool = r(addr) - def readBypassed(addr: UInt): Bool = _next(addr) - - val _r = RegInit(0.U(n.W)) - val r = if (zero) (_r >> 1 << 1) else _r - private var _next = r - private var ens = false.B - def mask(en: Bool, addr: UInt) = Mux(en, 1.U << addr, 0.U) - - def update(en: Bool, update: UInt) = { - _next = update - ens = ens || en - when(ens)(_r := _next) - } - - } - -} - -class RegFile(n: Int, w: Int, zero: Boolean = false) { - val rf = Reg(Vec(n, UInt(w.W))) - def access(addr: UInt) = rf(~addr(log2Up(n) - 1, 0)) - val reads = ArrayBuffer[(UInt, UInt)]() - private var canRead = true - - def read(addr: UInt) = { - require(canRead) - reads += addr -> Wire(UInt()) - reads.last._2 := Mux(zero.B && addr === 0.U, 0.U, access(addr)) - reads.last._2 - } - - def write(addr: UInt, data: UInt) = { - canRead = false - when(addr =/= 0.U) { - access(addr) := data - for ((raddr, rdata) <- reads) - when(addr === raddr)(rdata := data) - } - } - -} - -object ImmGen { - - def apply(sel: UInt, inst: UInt) = { - val sign = Mux(sel === IMM_Z, 0.S, inst(31).asSInt) - val b30_20 = Mux(sel === IMM_U, inst(30, 20).asSInt, sign) - val b19_12 = Mux(sel =/= IMM_U && sel =/= IMM_UJ, sign, inst(19, 12).asSInt) - val b11 = Mux( - sel === IMM_U || sel === IMM_Z, - 0.S, - Mux(sel === IMM_UJ, inst(20).asSInt, Mux(sel === IMM_SB, inst(7).asSInt, sign)) - ) - val b10_5 = Mux(sel === IMM_U || sel === IMM_Z, 0.U, inst(30, 25)) - val b4_1 = Mux( - sel === IMM_U, - 0.U, - Mux(sel === IMM_S || sel === IMM_SB, inst(11, 8), Mux(sel === IMM_Z, inst(19, 16), inst(24, 21))) - ) - val b0 = Mux(sel === IMM_S, inst(7), Mux(sel === IMM_I, inst(20), Mux(sel === IMM_Z, inst(15), 0.U))) - - Cat(sign, b30_20, b19_12, b11, b10_5, b4_1, b0).asSInt - } - -} diff --git a/arch/src/main/scala/framework/system/core/rocket/RocketCoreParameters.scala b/arch/src/main/scala/framework/system/core/rocket/RocketCoreParameters.scala deleted file mode 100644 index 607096033..000000000 --- a/arch/src/main/scala/framework/system/core/rocket/RocketCoreParameters.scala +++ /dev/null @@ -1,34 +0,0 @@ -// See LICENSE.Berkeley and LICENSE.SiFive for Rocket CSR semantics. -package freechips.rocketchip.rocket - -import chisel3._ -import org.chipsalliance.cde.config.Parameters -import freechips.rocketchip.tile._ -import freechips.rocketchip.util._ - -trait HasRocketCoreParameters extends HasCoreParameters { - lazy val rocketParams = tileParams.core.asInstanceOf[RocketCoreParams] - val fastLoadWord = rocketParams.fastLoadWord - val fastLoadByte = rocketParams.fastLoadByte - val mulDivParams = rocketParams.mulDiv.getOrElse(MulDivParams()) - require(!fastLoadByte || fastLoadWord) - require(!rocketParams.haveFSDirty) -} - -class RocketCustomCSRs(implicit p: Parameters) extends CustomCSRs with HasRocketCoreParameters { - override def bpmCSR = rocketParams.branchPredictionModeCSR.option(CustomCSR(bpmCSRId, 1, Some(BigInt(0)))) - - override def chickenCSR = { - val dc = tileParams.dcache.get - val mask = BigInt(dc.clockGate.toInt | (rocketParams.clockGate.toInt << 1) | - (rocketParams.clockGate.toInt << 2) | (1 << 3) | (dc.scratch.isEmpty.toInt << 9) | - (tileParams.icache.get.prefetch.toInt << 17)) - Some(CustomCSR(chickenCSRId, mask, Some(mask))) - } - - def disableICachePrefetch = getOrElse(chickenCSR, _.value(17), true.B) - def marchid = CustomCSR.constant(CSRs.marchid, BigInt(1)) - def mvendorid = CustomCSR.constant(CSRs.mvendorid, BigInt(rocketParams.mvendorid)) - def mimpid = CustomCSR.constant(CSRs.mimpid, BigInt(rocketParams.mimpid)) - override def decls = super.decls :+ marchid :+ mvendorid :+ mimpid -} diff --git a/arch/src/main/scala/framework/system/core/rocket/TileInterrupts.scala b/arch/src/main/scala/framework/system/core/rocket/TileInterrupts.scala deleted file mode 100644 index 4e3ea2a1b..000000000 --- a/arch/src/main/scala/framework/system/core/rocket/TileInterrupts.scala +++ /dev/null @@ -1,21 +0,0 @@ -// See LICENSE.SiFive for the Rocket interface definitions. -package freechips.rocketchip.tile - -import chisel3._ -import org.chipsalliance.cde.config.Parameters - -class NMI(val w: Int) extends Bundle { - val rnmi = Bool() - val rnmi_interrupt_vector = UInt(w.W) - val rnmi_exception_vector = UInt(w.W) -} - -class TileInterrupts(implicit p: Parameters) extends CoreBundle()(p) { - val debug = Bool() - val mtip = Bool() - val msip = Bool() - val meip = Bool() - val seip = Option.when(usingSupervisor)(Bool()) - val lip = Vec(coreParams.nLocalInterrupts, Bool()) - val nmi = Option.when(usingNMI)(new NMI(resetVectorLen)) -} diff --git a/arch/src/main/scala/framework/system/core/rocket/TileParameters.scala b/arch/src/main/scala/framework/system/core/rocket/TileParameters.scala deleted file mode 100644 index 5e93631ad..000000000 --- a/arch/src/main/scala/framework/system/core/rocket/TileParameters.scala +++ /dev/null @@ -1,78 +0,0 @@ -// See LICENSE.Berkeley and LICENSE.SiFive for Rocket interface definitions. -package freechips.rocketchip.tile { - - import org.chipsalliance.cde.config.{Field, Parameters} - import chisel3.util.{log2Ceil, log2Up} - import freechips.rocketchip.rocket._ - - case object TileKey extends Field[CpuTileParams] - case object CoreBeatBytes extends Field[Int] - case object CoreRoCCCount extends Field[Int](0) - - trait HasTileParameters { - implicit val p: Parameters - def tileParams: CpuTileParams = p(TileKey) - def usingVM = tileParams.core.useVM - def usingSupervisor = tileParams.core.hasSupervisorMode - def usingUser = tileParams.core.useUser || usingSupervisor - def usingHypervisor = usingVM && tileParams.core.useHypervisor - def usingDebug = tileParams.core.useDebug - def usingRoCC = p(CoreRoCCCount) > 0 - def usingBTB = tileParams.btb.exists(_.nEntries > 0) - def usingPTW = usingVM - def usingDataScratchpad = tileParams.dcache.flatMap(_.scratch).isDefined - def xLen = tileParams.core.xLen - def xBytes = xLen / 8 - def iLen = 32 - def pgIdxBits = 12 - def pgLevelBits = 10 - log2Ceil(xLen / 32) - def pgLevels = tileParams.core.pgLevels - def minPgLevels = { val levels = if (xLen == 32) 2 else 3; require(pgLevels >= levels); levels } - def maxSVAddrBits = pgIdxBits + pgLevels * pgLevelBits - def maxHypervisorExtraAddrBits = 2 - def hypervisorExtraAddrBits = if (usingHypervisor) maxHypervisorExtraAddrBits else 0 - def maxHVAddrBits = maxSVAddrBits + hypervisorExtraAddrBits - def asIdBits = p(ASIdBits) - def vmIdBits = p(VMIdBits) - - lazy val maxPAddrBits: Int = { - require(xLen == 32 || xLen == 64); if (!usingVM) xLen else if (xLen == 32) 34 else 56 - } - - lazy val paddrBits: Int = p(CorePAddrBits) - - def vaddrBits: Int = - if (usingVM) { - val bits = maxHVAddrBits - require(bits == xLen || xLen > bits && bits > paddrBits) - bits - } else (paddrBits + 1).min(xLen) - - def vpnBits = vaddrBits - pgIdxBits - def ppnBits = paddrBits - pgIdxBits - def vpnBitsExtended = vpnBits + (if (vaddrBits < xLen) 1 + (if (usingHypervisor) 1 else 0) else 0) - def vaddrBitsExtended = vpnBitsExtended + pgIdxBits - def cacheBlockBytes = p(freechips.rocketchip.subsystem.CacheBlockBytes) - def lgCacheBlockBytes = log2Up(cacheBlockBytes) - def masterPortBeatBytes = p(CoreBeatBytes) - def dcacheArbPorts = 1 + (if (usingVM) 1 else 0) + (if (usingDataScratchpad) 1 else 0) + p(CoreRoCCCount) + - (if (tileParams.core.useVector && tileParams.core.vectorUseDCache) 1 else 0) - } - -} - -package freechips.rocketchip.rocket { - import org.chipsalliance.cde.config.Field - case object ASIdBits extends Field[Int](0) - case object VMIdBits extends Field[Int](0) -} - -package freechips.rocketchip.subsystem { - import org.chipsalliance.cde.config.Field - case object CacheBlockBytes extends Field[Int] -} - -package freechips.rocketchip.devices.debug { - import org.chipsalliance.cde.config.Field - case object DebugModuleKey extends Field[Option[DebugModuleParams]](Some(DebugModuleParams())) -} diff --git a/arch/src/main/scala/framework/system/core/rocket/VectorInterfaces.scala b/arch/src/main/scala/framework/system/core/rocket/VectorInterfaces.scala deleted file mode 100644 index e2377278e..000000000 --- a/arch/src/main/scala/framework/system/core/rocket/VectorInterfaces.scala +++ /dev/null @@ -1,53 +0,0 @@ -// See LICENSE.Berkeley for Rocket's CPU/vector interface contract. -package freechips.rocketchip.rocket - -import chisel3._ -import chisel3.util._ -import org.chipsalliance.cde.config.Parameters -import freechips.rocketchip.tile._ - -class VectorCoreIO(implicit p: Parameters) extends CoreBundle { - val status = Input(new MStatus) - - val ex = new Bundle { - val valid = Input(Bool()); val ready = Output(Bool()); val inst = Input(UInt(32.W)) - val pc = Input(UInt(vaddrBitsExtended.W)); val vconfig = Input(new VConfig) - val vstart = Input(UInt(log2Ceil(maxVLMax).W)); val rs1 = Input(UInt(xLen.W)); val rs2 = Input(UInt(xLen.W)) - } - - val killm = Input(Bool()) - - val mem = new Bundle { - val frs1 = Input(UInt(fLen.W)); val block_mem = Output(Bool()); val block_all = Output(Bool()) - } - - val wb = new Bundle { - val store_pending = Input(Bool()); val replay = Output(Bool()); val retire = Output(Bool()) - val inst = Output(UInt(32.W)); val rob_should_wb = Output(Bool()); val rob_should_wb_fp = Output(Bool()) - val pc = Output(UInt(vaddrBitsExtended.W)); val xcpt = Output(Bool()) - val cause = Output(UInt(log2Ceil(Causes.all.max).W)); val tval = Output(UInt(coreMaxAddrBits.W)) - val vxrm = Input(UInt(2.W)); val frm = Input(UInt(3.W)) - } - - val resp = Decoupled(new Bundle { - val fp = Bool(); val size = UInt(2.W); val rd = UInt(5.W); val data = UInt((xLen max fLen).W) - }) - - val set_vstart = Valid(UInt(log2Ceil(maxVLMax).W)) - val set_vxsat = Output(Bool()) - val set_vconfig = Valid(new VConfig) - val set_fflags = Valid(UInt(5.W)) - val trap_check_busy = Output(Bool()) - val backend_busy = Output(Bool()) -} - -abstract class RocketVectorDecoder(implicit p: Parameters) extends CoreModule { - - val io = IO(new Bundle { - val inst = Input(UInt(32.W)); val vconfig = Input(new VConfig) - val vector = Output(Bool()); val legal = Output(Bool()); val fp = Output(Bool()) - val read_rs1 = Output(Bool()); val read_rs2 = Output(Bool()); val read_frs1 = Output(Bool()) - val write_rd = Output(Bool()); val write_frd = Output(Bool()) - }) - -} diff --git a/arch/src/main/scala/framework/system/core/rocket/configs/RocketCpuParam.scala b/arch/src/main/scala/framework/system/core/rocket/configs/RocketCpuParam.scala deleted file mode 100644 index b36f8d2f9..000000000 --- a/arch/src/main/scala/framework/system/core/rocket/configs/RocketCpuParam.scala +++ /dev/null @@ -1,378 +0,0 @@ -package framework.system.core.rocket.configs { - - import upickle.default._ - import freechips.rocketchip.rocket.{BHTParams, BTBParams, DCacheParams, ICacheParams, MulDivParams, RocketCoreParams} - import freechips.rocketchip.tile.FPUParams - - /** - * JSON-serializable Rocket CPU configuration parameters. - * - * Optional features (mulDiv, fpu, btb) use an `enable` flag instead of Option[T], - * so JSON stays as a plain dictionary (upickle serializes Option as array). - */ - - case class MulDivParam( - divUnroll: Int, - divEarlyOutGranularity: Int, - enable: Boolean, - mulUnroll: Int, - mulEarlyOut: Boolean, - divEarlyOut: Boolean) - - object MulDivParam { - implicit val rw: ReadWriter[MulDivParam] = macroRW - - def toMulDivParams(p: MulDivParam): Option[MulDivParams] = - if (p.enable) Some(MulDivParams( - divUnroll = p.divUnroll, - divEarlyOutGranularity = p.divEarlyOutGranularity, - mulUnroll = p.mulUnroll, - mulEarlyOut = p.mulEarlyOut, - divEarlyOut = p.divEarlyOut - )) - else None - - } - - case class FPUParam( - divSqrt: Boolean, - sfmaLatency: Int, - dfmaLatency: Int, - fpmuLatency: Int, - ifpuLatency: Int, - enable: Boolean, - minFLen: Int, - fLen: Int) - - object FPUParam { - implicit val rw: ReadWriter[FPUParam] = macroRW - - def toFPUParams(p: FPUParam): Option[FPUParams] = - if (p.enable) Some(FPUParams( - divSqrt = p.divSqrt, - sfmaLatency = p.sfmaLatency, - dfmaLatency = p.dfmaLatency, - fpmuLatency = p.fpmuLatency, - ifpuLatency = p.ifpuLatency, - minFLen = p.minFLen, - fLen = p.fLen - )) - else None - - } - - case class DCacheParam( - clockGate: Boolean, - nSets: Int, - nWays: Int, - nMSHRs: Int) - - object DCacheParam { - implicit val rw: ReadWriter[DCacheParam] = macroRW - - def toDCacheParams(p: DCacheParam, rowBits: Int, blockBytes: Int): DCacheParams = DCacheParams( - clockGate = p.clockGate, - nSets = p.nSets, - nWays = p.nWays, - rowBits = rowBits, - nMSHRs = p.nMSHRs, - blockBytes = blockBytes - ) - - } - - case class ICacheParam( - prefetch: Boolean, - nSets: Int, - nWays: Int) - - object ICacheParam { - implicit val rw: ReadWriter[ICacheParam] = macroRW - - def toICacheParams(p: ICacheParam, rowBits: Int, blockBytes: Int): ICacheParams = ICacheParams( - prefetch = p.prefetch, - nSets = p.nSets, - nWays = p.nWays, - rowBits = rowBits, - blockBytes = blockBytes - ) - - } - - case class BTBParam( - nMatchBits: Int, - nPages: Int, - updatesOutOfOrder: Boolean, - bhtEnable: Boolean, - bhtEntries: Int, - bhtCounterLength: Int, - bhtHistoryLength: Int, - bhtHistoryBits: Int, - enable: Boolean, - nEntries: Int, - nRAS: Int) - - object BTBParam { - implicit val rw: ReadWriter[BTBParam] = macroRW - - def toBTBParams(p: BTBParam): Option[BTBParams] = - if (p.enable) Some(BTBParams( - nMatchBits = p.nMatchBits, - nPages = p.nPages, - updatesOutOfOrder = p.updatesOutOfOrder, - bhtParams = - if (p.bhtEnable) Some(BHTParams(p.bhtEntries, p.bhtCounterLength, p.bhtHistoryLength, p.bhtHistoryBits)) - else None, - nEntries = p.nEntries, - nRAS = p.nRAS - )) - else None - - } - - case class RocketCpuParam( - mtvecInitEnable: Boolean, - useUser: Boolean, - useSupervisor: Boolean, - useHypervisor: Boolean, - useDebug: Boolean, - useAtomics: Boolean, - useAtomicsOnlyForIO: Boolean, - useCompressed: Boolean, - useRVE: Boolean, - useConditionalZero: Boolean, - nLocalInterrupts: Int, - useNMI: Boolean, - nBreakpoints: Int, - useBPWatch: Boolean, - mcontextWidth: Int, - scontextWidth: Int, - nPerfCounters: Int, - haveBasicCounters: Boolean, - misaWritable: Boolean, - mtvecInit: BigInt, - mtvecWritable: Boolean, - fastLoadWord: Boolean, - fastLoadByte: Boolean, - branchPredictionModeCSR: Boolean, - clockGate: Boolean, - mvendorid: Int, - mimpid: Int, - haveCease: Boolean, - haveSimTimeout: Boolean, - useVM: Boolean, - useZba: Boolean, - useZbb: Boolean, - useZbs: Boolean, - haveCFlush: Boolean, - mulDiv: MulDivParam, - fpu: FPUParam, - dcache: DCacheParam, - icache: ICacheParam, - btb: BTBParam) - - object RocketCpuParam { - implicit val rw: ReadWriter[RocketCpuParam] = macroRW - - /** - * Convert to rocket-chip RocketCoreParams. - * Cache geometry, physical address width and bus widths are supplied explicitly by the core. - */ - def toRocketCoreParams(p: RocketCpuParam, xLen: Int, pgLevels: Int): RocketCoreParams = RocketCoreParams( - xLen = xLen, - pgLevels = pgLevels, - useUser = p.useUser, - useSupervisor = p.useSupervisor, - useHypervisor = p.useHypervisor, - useDebug = p.useDebug, - useAtomics = p.useAtomics, - useAtomicsOnlyForIO = p.useAtomicsOnlyForIO, - useCompressed = p.useCompressed, - useRVE = p.useRVE, - useConditionalZero = p.useConditionalZero, - nLocalInterrupts = p.nLocalInterrupts, - useNMI = p.useNMI, - nBreakpoints = p.nBreakpoints, - useBPWatch = p.useBPWatch, - mcontextWidth = p.mcontextWidth, - scontextWidth = p.scontextWidth, - nPerfCounters = p.nPerfCounters, - haveBasicCounters = p.haveBasicCounters, - misaWritable = p.misaWritable, - mtvecInit = if (p.mtvecInitEnable) Some(p.mtvecInit) else None, - mtvecWritable = p.mtvecWritable, - fastLoadWord = p.fastLoadWord, - fastLoadByte = p.fastLoadByte, - branchPredictionModeCSR = p.branchPredictionModeCSR, - clockGate = p.clockGate, - mvendorid = p.mvendorid, - mimpid = p.mimpid, - haveCease = p.haveCease, - haveSimTimeout = p.haveSimTimeout, - useVM = p.useVM, - useZba = p.useZba, - useZbb = p.useZbb, - useZbs = p.useZbs, - haveCFlush = p.haveCFlush, - mulDiv = MulDivParam.toMulDivParams(p.mulDiv), - fpu = FPUParam.toFPUParams(p.fpu) - ) - - def toDCacheParams(p: RocketCpuParam, rowBits: Int, blockBytes: Int): DCacheParams = - DCacheParam.toDCacheParams(p.dcache, rowBits, blockBytes) - - def toICacheParams(p: RocketCpuParam, rowBits: Int, blockBytes: Int): ICacheParams = - ICacheParam.toICacheParams(p.icache, rowBits, blockBytes) - - def toBTBParams(p: RocketCpuParam): Option[BTBParams] = - BTBParam.toBTBParams(p.btb) - } - -} - -package freechips.rocketchip.rocket { - import chisel3._ - import chisel3.util._ - import org.chipsalliance.cde.config.Parameters - import freechips.rocketchip.tile._ - - case class DCacheParams( - nSets: Int = 64, - nWays: Int = 4, - rowBits: Int = 64, - subWordBits: Option[Int] = None, - replacementPolicy: String = "random", - nTLBSets: Int = 1, - nTLBWays: Int = 32, - nTLBBasePageSectors: Int = 4, - nTLBSuperpages: Int = 4, - tagECC: Option[String] = None, - dataECC: Option[String] = None, - dataECCBytes: Int = 1, - nMSHRs: Int = 1, - nSDQ: Int = 17, - nRPQ: Int = 16, - nMMIOs: Int = 1, - blockBytes: Int = 64, - separateUncachedResp: Boolean = false, - acquireBeforeRelease: Boolean = false, - pipelineWayMux: Boolean = false, - clockGate: Boolean = false, - scratch: Option[BigInt] = None) { - require(scratch.isEmpty || nWays == 1) - require(scratch.isEmpty || nMSHRs == 0) - if (scratch.isEmpty) require(isPow2(nSets)) - } - - case class ICacheParams( - nSets: Int = 64, - nWays: Int = 4, - rowBits: Int = 128, - nTLBSets: Int = 1, - nTLBWays: Int = 32, - nTLBBasePageSectors: Int = 4, - nTLBSuperpages: Int = 4, - cacheIdBits: Int = 0, - tagECC: Option[String] = None, - dataECC: Option[String] = None, - itimAddr: Option[BigInt] = None, - prefetch: Boolean = false, - blockBytes: Int = 64, - latency: Int = 2, - fetchBytes: Int = 4) - - case class RocketCoreParams( - xLen: Int = 64, - pgLevels: Int = 3, - bootFreqHz: BigInt = 0, - useVM: Boolean = true, - useUser: Boolean = false, - useSupervisor: Boolean = false, - useHypervisor: Boolean = false, - useDebug: Boolean = true, - useAtomics: Boolean = true, - useAtomicsOnlyForIO: Boolean = false, - useCompressed: Boolean = true, - useRVE: Boolean = false, - useConditionalZero: Boolean = false, - useZba: Boolean = false, - useZbb: Boolean = false, - useZbs: Boolean = false, - nLocalInterrupts: Int = 0, - useNMI: Boolean = false, - nBreakpoints: Int = 1, - useBPWatch: Boolean = false, - mcontextWidth: Int = 0, - scontextWidth: Int = 0, - nPMPs: Int = 8, - nPerfCounters: Int = 0, - haveBasicCounters: Boolean = true, - haveCFlush: Boolean = false, - misaWritable: Boolean = true, - nL2TLBEntries: Int = 0, - nL2TLBWays: Int = 1, - nPTECacheEntries: Int = 8, - mtvecInit: Option[BigInt] = Some(BigInt(0)), - mtvecWritable: Boolean = true, - fastLoadWord: Boolean = true, - fastLoadByte: Boolean = false, - branchPredictionModeCSR: Boolean = false, - clockGate: Boolean = false, - mvendorid: Int = 0, - mimpid: Int = 0x20181004, - mulDiv: Option[MulDivParams] = Some(MulDivParams()), - fpu: Option[FPUParams] = Some(FPUParams()), - debugROB: Option[DebugROBParams] = None, - haveCease: Boolean = true, - haveSimTimeout: Boolean = true, - vector: Option[RocketCoreVectorParams] = None, - enableTraceCoreIngress: Boolean = false) - extends CoreParams { - val lgPauseCycles = 5 - val haveFSDirty = false - val pmpGranularity = if (useHypervisor) 4096 else 4 - val fetchWidth = if (useCompressed) 2 else 1 - val decodeWidth = fetchWidth / (if (useCompressed) 2 else 1) - val retireWidth = 1 - val instBits = if (useCompressed) 16 else 32 - val lrscCycles = 80 - val traceHasWdata = debugROB.isDefined - override def minFLen = fpu.map(_.minFLen).getOrElse(32) - override def customCSRs(implicit p: Parameters) = new RocketCustomCSRs - override val useVector = vector.isDefined - override val vectorUseDCache = vector.exists(_.useDCache) - override def vLen = vector.map(_.vLen).getOrElse(0) - override def eLen = vector.map(_.eLen).getOrElse(0) - override def vfLen = vector.map(_.vfLen).getOrElse(0) - override def vfh = vector.exists(_.vfh) - override def vMemDataBits = vector.map(_.vMemDataBits).getOrElse(0) - override def vExts = vector.map(_.vExts).getOrElse(Nil) - } - - case class RocketCoreVectorParams( - vLen: Int, - eLen: Int, - vfLen: Int, - vfh: Boolean, - vMemDataBits: Int, - decoder: Parameters => RocketVectorDecoder, - useDCache: Boolean, - issueVConfig: Boolean, - vExts: Seq[String]) - -} - -package freechips.rocketchip.tile { - import freechips.rocketchip.rocket._ - - case class CpuTileParams( - core: CoreParams, - dcache: Option[DCacheParams], - icache: Option[ICacheParams], - btb: Option[BTBParams]) - -} - -package freechips.rocketchip.devices.debug { - case class DebugModuleParams(nDscratch: Int = 1, debugEntry: BigInt = 0x800, debugException: BigInt = 0x808) -} diff --git a/arch/src/main/scala/framework/system/core/seed b/arch/src/main/scala/framework/system/core/seed deleted file mode 160000 index f8ba87b5c..000000000 --- a/arch/src/main/scala/framework/system/core/seed +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f8ba87b5c73c2c78aa9e08e3b92324cbed8e72de diff --git a/arch/src/main/scala/framework/system/device/BootRom.scala b/arch/src/main/scala/framework/system/device/BootRom.scala deleted file mode 100644 index 7cdc799f1..000000000 --- a/arch/src/main/scala/framework/system/device/BootRom.scala +++ /dev/null @@ -1,86 +0,0 @@ -package framework.system.device - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import chisel3.util._ -import memcore.bus.chi.{Params => ChiParams} -import memcore.bus.chi.snf.{LineRequest, LineResponse} - -/** - * The BootROM image (`bootrom/bootrom.rv64.img`, built from `bootrom.S`) with its two config dwords - * filled: the hart count at offset 0x8 and the DRAM entry at offset 0x10. - */ -case class BootRomParams(base: BigInt, bytes: BigInt, image: Seq[Byte]) { - require(base % 64 == 0 && bytes % 64 == 0 && image.size <= bytes, "BootROM must be whole cache lines") -} - -object BootRom { - - def apply( - base: BigInt, - bytes: BigInt, - hartCount: Int, - entry: BigInt - ): BootRomParams = { - val stream = getClass.getResourceAsStream("/bootrom/bootrom.rv64.img") - require(stream != null, "missing BootROM resource bootrom/bootrom.rv64.img") - val image = - try stream.readAllBytes().toSeq - finally stream.close() - require(image.size >= 24 && image.slice(8, 24).forall(_ == 0), "BootROM image lacks its zeroed config dwords") - def dword(value: BigInt): Seq[Byte] = (0 until 8).map(i => ((value >> (8 * i)) & 0xff).toByte) - BootRomParams(base, bytes, image.take(8) ++ dword(hartCount) ++ dword(entry) ++ image.drop(24)) - } - -} - -/** - * Serves BootROM lines on a tile's L2 backing path and forwards every other line to memory. The - * ROM is a cacheable, executable, read-only region, so fetches and loads reach it as line reads; - * a write to a ROM line can only come from a faulty requester and answers with an error. - */ -@instantiable -class BootRomLines(p: BootRomParams, chi: ChiParams) extends Module { - - @public val io = IO(new Bundle { - val request = Flipped(Decoupled(new LineRequest(chi))) - val response = Decoupled(new LineResponse(chi)) - val memoryRequest = Decoupled(new LineRequest(chi)) - val memoryResponse = Flipped(Decoupled(new LineResponse(chi))) - }) - - val lines = p.image.padTo(((p.image.size + 63) / 64) * 64, 0.toByte).grouped(64).map { line => - line.zipWithIndex.map { case (byte, i) => BigInt(byte & 0xff) << (8 * i) }.sum - }.toSeq - - val rom = VecInit(lines.map(_.U(512.W))) - - val offset = io.request.bits.addr - p.base.U - val inRom = io.request.bits.addr >= p.base.U && io.request.bits.addr < (p.base + p.bytes).U - val line = offset >> 6 - val pending = RegInit(false.B) - val id = Reg(UInt(chi.txnIdBits.W)) - val data = Reg(UInt(512.W)) - val error = Reg(Bool()) - - io.memoryRequest.valid := io.request.valid && !inRom - io.memoryRequest.bits := io.request.bits - io.request.ready := Mux(inRom, !pending, io.memoryRequest.ready) - when(io.request.fire && inRom) { - pending := true.B - id := io.request.bits.id - data := Mux(line < lines.size.U, rom(line(log2Ceil(lines.size.max(2)) - 1, 0)), 0.U) - error := io.request.bits.write - } - - // Memory responses keep priority; the ROM answer waits at most until memory is idle. - io.response.valid := io.memoryResponse.valid || pending - io.response.bits := Mux(io.memoryResponse.valid, io.memoryResponse.bits, 0.U.asTypeOf(io.response.bits)) - when(!io.memoryResponse.valid) { - io.response.bits.id := id - io.response.bits.data := data - io.response.bits.error := error - } - io.memoryResponse.ready := io.response.ready - when(io.response.fire && !io.memoryResponse.valid)(pending := false.B) -} diff --git a/arch/src/main/scala/framework/system/device/Clint.scala b/arch/src/main/scala/framework/system/device/Clint.scala deleted file mode 100644 index 613233947..000000000 --- a/arch/src/main/scala/framework/system/device/Clint.scala +++ /dev/null @@ -1,86 +0,0 @@ -package framework.system.device - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import chisel3.util._ - -/** One register access presented for a single cycle; data is right-aligned at `addr`. */ -class DeviceAccess extends Bundle { - val addr = UInt(64.W) - val write = Bool() - val size = UInt(3.W) - val data = UInt(64.W) -} - -/** Register-block interface: `read` and `error` answer the access presented in the same cycle. */ -class DevicePort extends Bundle { - val access = Flipped(Valid(new DeviceAccess)) - val read = Output(UInt(64.W)) - val error = Output(Bool()) -} - -/** `mtime` advances once every `tickCycles` clock cycles; the device tree timebase must match. */ -case class ClintParams(base: BigInt = BigInt("2000000", 16), bytes: BigInt = BigInt("10000", 16), tickCycles: Int = 1000) { - require(tickCycles >= 1) -} - -/** Standard CLINT layout: msip at 4*hart, mtimecmp at 0x4000 + 8*hart, mtime at 0xbff8. */ -@instantiable -class Clint(p: ClintParams, hartIds: Seq[Int]) extends Module { - require(hartIds.distinct.size == hartIds.size && hartIds.max < 4095) - val n = hartIds.size - - @public val io = IO(new Bundle { - val port = new DevicePort - val msip = Output(Vec(n, Bool())) - val mtip = Output(Vec(n, Bool())) - val time = Output(UInt(64.W)) - }) - - val msip = RegInit(VecInit(Seq.fill(n)(false.B))) - val mtimecmp = RegInit(VecInit(Seq.fill(n)(~0.U(64.W)))) - val mtime = RegInit(0.U(64.W)) - val ticks = RegInit(0.U(log2Ceil(p.tickCycles + 1).W)) - io.time := mtime - - val access = io.port.access.bits - val offset = access.addr - p.base.U - val write = io.port.access.valid && access.write - val word = access.size === 2.U - val double = access.size === 3.U - - // Each register answers a 32-bit access to either half, and a 64-bit access to its base. - val hits = Wire(Vec(2 * n + 1, Bool())) - val reads = Wire(Vec(2 * n + 1, UInt(64.W))) - - def register( - slot: Int, - at: BigInt, - state: UInt, - update: UInt => Unit, - wide: Boolean - ): Unit = { - val value = WireDefault(UInt(64.W), state) - val low = word && offset === at.U - val high = wide.B && word && offset === (at + 4).U - val full = wide.B && double && offset === at.U - hits(slot) := low || high || full - reads(slot) := Mux(high, value(63, 32), Mux(low, value(31, 0), value)) - when(write && low)(update(if (wide) Cat(value(63, 32), access.data(31, 0)) else access.data(31, 0))) - when(write && high)(update(Cat(access.data(31, 0), value(31, 0)))) - when(write && full)(update(access.data)) - } - - ticks := Mux(ticks === (p.tickCycles - 1).U, 0.U, ticks + 1.U) - when(ticks === (p.tickCycles - 1).U)(mtime := mtime + 1.U) - for ((hart, i) <- hartIds.zipWithIndex) { - register(i, 4 * hart, msip(i).asUInt, v => msip(i) := v(0), wide = false) - register(n + i, 0x4000 + 8 * BigInt(hart), mtimecmp(i), v => mtimecmp(i) := v, wide = true) - io.msip(i) := msip(i) - io.mtip(i) := mtime >= mtimecmp(i) - } - register(2 * n, 0xbff8, mtime, v => mtime := v, wide = true) - - io.port.read := Mux1H(hits, reads) - io.port.error := !hits.asUInt.orR -} diff --git a/arch/src/main/scala/framework/system/device/Devices.scala b/arch/src/main/scala/framework/system/device/Devices.scala deleted file mode 100644 index 598b4c3ea..000000000 --- a/arch/src/main/scala/framework/system/device/Devices.scala +++ /dev/null @@ -1,101 +0,0 @@ -package framework.system.device - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import chisel3.util._ -import hier.tile.memory.CoreInterrupts -import memcore.memory.cpu.{CpuMemParams, UncachedRequest, UncachedResponse} - -/** Chip devices; the BootROM, when present, is served on the tiles' line paths rather than here. */ -case class DeviceParams( - clint: ClintParams = ClintParams(), - plic: PlicParams = PlicParams(), - bootrom: Option[BootRomParams] = None, - scu: sims.scu.SCUParams = sims.scu.SCUParams()) - -/** - * Chip device decode for the per-core uncached device ports. CLINT and PLIC are served inside; - * every other device address leaves on the same core's external port. Each core port carries one - * outstanding request, and the internal register blocks take one access per cycle in round robin. - */ -@instantiable -class Devices(p: DeviceParams, cp: CpuMemParams, hartIds: Seq[Int]) extends Module { - val n = hartIds.size - - @public val io = IO(new Bundle { - val request = Vec(n, Flipped(Decoupled(new UncachedRequest(cp)))) - val response = Vec(n, Decoupled(new UncachedResponse(cp))) - val externalRequest = Vec(n, Decoupled(new UncachedRequest(cp))) - val externalResponse = Vec(n, Flipped(Decoupled(new UncachedResponse(cp)))) - val sources = Input(UInt(p.plic.sources.W)) - val interrupts = Output(Vec(n, new CoreInterrupts)) - val time = Output(UInt(64.W)) - }) - - val clint = Instantiate(new Clint(p.clint, hartIds)) - val plic = Instantiate(new Plic(p.plic, hartIds)) - io.time := clint.io.time - plic.io.sources := io.sources - for (i <- 0 until n) { - io.interrupts(i).timer := clint.io.mtip(i) - io.interrupts(i).software := clint.io.msip(i) - io.interrupts(i).external := plic.io.meip(i) - io.interrupts(i).supervisorExternal := plic.io.seip(i) - } - - def within(addr: UInt, base: BigInt, bytes: BigInt): Bool = addr >= base.U && addr < (base + bytes).U - val inClint = io.request.map(r => within(r.bits.addr, p.clint.base, p.clint.bytes)) - val inPlic = io.request.map(r => within(r.bits.addr, p.plic.base, p.plic.bytes)) - val internal = inClint.zip(inPlic).map { case (c, l) => c || l } - - val pending = RegInit(VecInit(Seq.fill(n)(false.B))) - val outside = RegInit(VecInit(Seq.fill(n)(false.B))) - val tag = Reg(Vec(n, UInt(cp.tagBits.W))) - val data = Reg(Vec(n, UInt(64.W))) - val error = Reg(Vec(n, Bool())) - val idle = (0 until n).map(i => !pending(i) && !outside(i)) - - // Round-robin grant among the cores presenting an internal access this cycle. - val wants = VecInit((0 until n).map(i => io.request(i).valid && internal(i) && idle(i))).asUInt - val last = RegInit(0.U(log2Ceil(n + 1).W)) - val rotated = (0 until n).map(k => (k.U +& last +& 1.U) % n.U) - val grant = PriorityMux(rotated.map(i => wants(i)), rotated) - val granted = wants.orR - when(granted)(last := grant) - - val access = io.request(grant).bits - for (port <- Seq(clint.io.port, plic.io.port)) { - port.access.bits.addr := access.addr - port.access.bits.write := access.write - port.access.bits.size := access.size - port.access.bits.data := access.data - } - clint.io.port.access.valid := granted && VecInit(inClint)(grant) - plic.io.port.access.valid := granted && VecInit(inPlic)(grant) - val internalRead = Mux(VecInit(inClint)(grant), clint.io.port.read, plic.io.port.read) - val internalError = Mux(VecInit(inClint)(grant), clint.io.port.error, plic.io.port.error) - - for (i <- 0 until n) { - val request = io.request(i) - val response = io.response(i) - val external = io.externalRequest(i) - val mine = granted && grant === i.U - external.valid := request.valid && !internal(i) && idle(i) - external.bits := request.bits - request.ready := idle(i) && Mux(internal(i), mine, external.ready) - when(request.fire) { - when(internal(i)) { - pending(i) := true.B - tag(i) := request.bits.tag - data(i) := Mux(request.bits.write, 0.U, internalRead) - error(i) := internalError - }.otherwise(outside(i) := true.B) - } - response.valid := pending(i) || (outside(i) && io.externalResponse(i).valid) - response.bits.tag := Mux(pending(i), tag(i), io.externalResponse(i).bits.tag) - response.bits.data := Mux(pending(i), data(i), io.externalResponse(i).bits.data) - response.bits.error := Mux(pending(i), error(i), io.externalResponse(i).bits.error) - io.externalResponse(i).ready := outside(i) && response.ready - when(response.fire) { pending(i) := false.B; outside(i) := false.B } - } -} diff --git a/arch/src/main/scala/framework/system/dlink/dlink.scala b/arch/src/main/scala/framework/system/dlink/dlink.scala deleted file mode 100644 index e75e5772c..000000000 --- a/arch/src/main/scala/framework/system/dlink/dlink.scala +++ /dev/null @@ -1,11 +0,0 @@ -package framework.system.dlink - -import chisel3._ -import framework.system.SystemParams - -/** Unified memory access; address decoding belongs to the platform. */ -class DLinkIO(p: SystemParams) extends Bundle { - val axi = new memcore.bus.axi4.Port(p.ddr.axi) -} - -trait HasDLink { def dlink: DLinkIO } diff --git a/arch/src/main/scala/framework/system/memory/Memory.scala b/arch/src/main/scala/framework/system/memory/Memory.scala deleted file mode 100644 index 3f38b277d..000000000 --- a/arch/src/main/scala/framework/system/memory/Memory.scala +++ /dev/null @@ -1,33 +0,0 @@ -package framework.system.memory - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import memcore.bus.chi.snf.{LineRequest, LineResponse} -import memcore.memory.ddr.{Params => DdrParams, Bridge} - -/** CPU coherence backing traffic and NPU DMA meet at the DDR AXI arbiter. */ -@instantiable -class Memory(ddr: DdrParams, dmaMasters: Int = 0) extends Module { - - @public - val io = IO(new Bundle { - val lineRequest = Vec(ddr.clients, Flipped(Decoupled(new LineRequest(ddr.line)))) - val lineResponse = Vec(ddr.clients, Decoupled(new LineResponse(ddr.line))) - val dma = Vec(dmaMasters, Flipped(new memcore.bus.axi4.Port(ddr.axi))) - val axi = new memcore.bus.axi4.Port(ddr.axi) - val outstanding = Output(UInt(log2Ceil(ddr.slots + 2 * (1 << ddr.axi.idBits) + 1).W)) - }) - - val bridge: Instance[Bridge] = Instantiate(new Bridge(ddr)) - bridge.io.request <> io.lineRequest - io.lineResponse <> bridge.io.response - val fabric: Instance[memcore.bus.axi4.Interconnect] = - Instantiate(new memcore.bus.axi4.Interconnect(ddr.axi, dmaMasters + 1)) - fabric.io.in(0) <> bridge.io.axi - io.axi <> fabric.io.out - for (i <- 0 until dmaMasters) { - fabric.io.in(i + 1) <> io.dma(i) - } - io.outstanding := bridge.io.outstanding +& fabric.io.outstanding -} diff --git a/arch/src/main/scala/framework/system/memory/UncachedAxi.scala b/arch/src/main/scala/framework/system/memory/UncachedAxi.scala deleted file mode 100644 index 7380988f8..000000000 --- a/arch/src/main/scala/framework/system/memory/UncachedAxi.scala +++ /dev/null @@ -1,73 +0,0 @@ -package framework.system.memory - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import memcore.bus.axi4 -import memcore.memory.cpu.{CpuMemParams, UncachedRequest, UncachedResponse} - -/** Uncached CPU accesses share one AXI master; responses retain the CPU port and tag. */ -@instantiable -class UncachedAxi(cp: CpuMemParams, ap: axi4.Params, ports: Int) extends Module { - - @public - val io = IO(new Bundle { - val request = Vec(ports, Flipped(Decoupled(new UncachedRequest(cp)))) - val response = Vec(ports, Decoupled(new UncachedResponse(cp))) - val axi = new axi4.Port(ap) - }) - - val indexBits = math.max(1, log2Ceil(ports)) - val idle :: issue :: waitResponse :: respond :: Nil = Enum(4) - val state = RegInit(idle) - val cursor = RegInit(0.U(indexBits.W)) - val owner = Reg(UInt(indexBits.W)) - val command = Reg(new UncachedRequest(cp)) - val answer = Reg(new UncachedResponse(cp)) - val addressSent = RegInit(false.B) - val dataSent = RegInit(false.B) - val ordered = (0 until ports).map(i => (cursor +& i.U) % ports.U) - val selected = if (ports == 1) 0.U else PriorityMux(ordered.map(i => io.request(i).valid), ordered) - val present = io.request.map(_.valid).reduce(_ || _) - for (i <- 0 until ports) { - io.request(i).ready := state === idle && present && selected === i.U - io.response(i).valid := state === respond && owner === i.U - io.response(i).bits := answer - when(io.response(i).fire)(state := idle) - } - when(state === idle && present) { - owner := selected - command := (if (ports == 1) io.request.head.bits else io.request(selected).bits) - addressSent := false.B - dataSent := false.B - cursor := Mux(selected === (ports - 1).U, 0.U, selected + 1.U) - state := issue - } - for (address <- Seq(io.axi.aw, io.axi.ar)) { - address.bits := 0.U.asTypeOf(address.bits) - address.bits.addr := command.addr - address.bits.size := command.size - address.bits.burst := 1.U - } - io.axi.aw.valid := state === issue && command.write && !addressSent - io.axi.ar.valid := state === issue && !command.write - val shift = command.addr(log2Ceil(ap.bytes) - 1, 0) - io.axi.w.valid := state === issue && command.write && !dataSent - io.axi.w.bits.data := command.data << (shift << 3) - io.axi.w.bits.strb := (((1.U((ap.bytes + 1).W) << (1.U << command.size)) - 1.U) << shift) - io.axi.w.bits.last := true.B - when(io.axi.aw.fire)(addressSent := true.B) - when(io.axi.w.fire)(dataSent := true.B) - when(io.axi.ar.fire || (command.write && (addressSent || io.axi.aw.fire) && (dataSent || io.axi.w.fire))) { - when(state === issue)(state := waitResponse) - } - io.axi.r.ready := state === waitResponse && !command.write - io.axi.b.ready := state === waitResponse && command.write - when(io.axi.r.fire || io.axi.b.fire) { - when(io.axi.r.fire)(assert(io.axi.r.bits.last)) - answer.tag := command.tag - answer.data := Mux(command.write, 0.U, io.axi.r.bits.data >> (shift << 3)) - answer.error := Mux(command.write, io.axi.b.bits.resp =/= 0.U, io.axi.r.bits.resp =/= 0.U) - state := respond - } -} diff --git a/arch/src/main/scala/framework/system/tile/BankNetwork.scala b/arch/src/main/scala/framework/system/tile/BankNetwork.scala deleted file mode 100644 index 6946efdd6..000000000 --- a/arch/src/main/scala/framework/system/tile/BankNetwork.scala +++ /dev/null @@ -1,113 +0,0 @@ -package framework.system.tile - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.top.GlobalConfig -import framework.system.core.clink.ShmPort -import framework.memdomain.backend.shared.{SharedLeasePort, SharedMemBackend, SharedMemLayout, SharedPhysicalPort} -import framework.memdomain.backend.banks.btrace.PhysicalBankHash -import framework.memdomain.frontend.mem.MemConfigerIO -import framework.memdomain.isa.{MvoverCommand, MvoverPort} - -/** Shared banks and inter-core bank movement; transport and CPU command decoding stay outside. */ -@instantiable -class BankNetwork( - b: GlobalConfig, - enabledCoreIds: Seq[Int], - useMesh: Boolean, - controllerMove: Boolean, - physicalPorts: Int = 0) - extends Module { - require(b.memDomain.sharedEnable) - require(enabledCoreIds.nonEmpty && enabledCoreIds == b.memDomain.computeCoreIds) - require(enabledCoreIds.distinct.size == enabledCoreIds.size && - enabledCoreIds.forall(i => i >= 0 && i < b.memDomain.nCores)) - val nCores = b.memDomain.nCores - val channels = SharedMemLayout.channelPerHart(b) - val controllerPorts = if (controllerMove) 1 else 0 - val movePorts = enabledCoreIds.size + controllerPorts - - @public - val io = IO(new Bundle { - val lease = Option.when(physicalPorts > 0)(Flipped(new SharedLeasePort)) - val physical = Vec(physicalPorts, Flipped(new SharedPhysicalPort(b.memDomain.bankWidth))) - val compute = Vec(enabledCoreIds.size, Flipped(new ShmPort(b))) - val hartIds = Input(Vec(nCores, UInt(b.tile.xLen.W))) - val controllerMvover = if (controllerMove) Some(Flipped(new MvoverPort)) else None - val bankHashes = - if (b.sim.diffTest) Some(Output(Vec(SharedMemLayout.totalBank(b), new PhysicalBankHash(b)))) else None - }) - - val sharedBackend: Instance[SharedMemBackend] = Instantiate(new SharedMemBackend(b, useMesh, physicalPorts)) - io.lease.foreach { lease => - sharedBackend.io.lease.get <> lease - val endpoint = lease.request.bits.endpoint - val owner = MuxLookup(endpoint, 0.U(b.tile.xLen.W))( - enabledCoreIds.zipWithIndex.map { case (physical, logical) => logical.U -> io.hartIds(physical) } - ) - sharedBackend.io.leaseOwnerHartId.get := owner - when(lease.request.valid && !reset.asBool) { - assert(endpoint < enabledCoreIds.size.U, "Shared lease endpoint is not a configured NPU") - } - } - sharedBackend.io.physical <> io.physical - sharedBackend.io.physicalOwnerHartId.foreach(_ := io.hartIds(0)) - io.bankHashes.foreach(_ := sharedBackend.io.bank_hashes.get) - for ((physical, compute) <- enabledCoreIds.zipWithIndex) { - val port = io.compute(compute) - for (channel <- 0 until channels) { - sharedBackend.io.mem_req(compute * channels + channel) <> port.requests(channel) - } - sharedBackend.io.localBanks(physical) <> port.local - sharedBackend.io.query_valid(physical) := port.queryValid - sharedBackend.io.query_hart_id(physical) := io.hartIds(physical) - sharedBackend.io.query_vbank_id(physical) := port.queryVbank - port.queryGroups := sharedBackend.io.query_group_count(physical) - } - for (physical <- 0 until nCores if !enabledCoreIds.contains(physical)) { - val local = sharedBackend.io.localBanks(physical) - local.request.ready := false.B - local.response.valid := false.B - local.response.bits := 0.U.asTypeOf(local.response.bits) - sharedBackend.io.query_valid(physical) := false.B - sharedBackend.io.query_hart_id(physical) := 0.U - sharedBackend.io.query_vbank_id(physical) := 0.U - } - - val moveArb = Module(new Arbiter(new MvoverCommand, movePorts)) - val moveOwner = RegInit(0.U(math.max(1, log2Ceil(movePorts)).W)) - val completionReady = Wire(Vec(movePorts, Bool())) - io.controllerMvover.foreach { port => - moveArb.io.in(0) <> port.command - port.completion.valid := sharedBackend.io.mvover.completion.valid && moveOwner === 0.U - port.completion.bits := sharedBackend.io.mvover.completion.bits - completionReady(0) := port.completion.ready - } - for (compute <- enabledCoreIds.indices) { - val port = compute + controllerPorts - moveArb.io.in(port) <> io.compute(compute).move.command - io.compute(compute).move.completion.valid := sharedBackend.io.mvover.completion.valid && moveOwner === port.U - io.compute(compute).move.completion.bits := sharedBackend.io.mvover.completion.bits - completionReady(port) := io.compute(compute).move.completion.ready - } - sharedBackend.io.mvover.command <> moveArb.io.out - - def physicalComputeCore(logical: UInt): UInt = MuxLookup(logical, nCores.U(8.W))( - enabledCoreIds.zipWithIndex.map { case (physical, logicalIndex) => logicalIndex.U -> physical.U(8.W) } - ) - - sharedBackend.io.mvover.command.bits.sourceCore := physicalComputeCore(moveArb.io.out.bits.sourceCore) - sharedBackend.io.mvover.command.bits.targetCore := physicalComputeCore(moveArb.io.out.bits.targetCore) - sharedBackend.io.mvover.completion.ready := completionReady(moveOwner) - when(moveArb.io.out.fire)(moveOwner := moveArb.io.chosen) - - val configArb = Module(new Arbiter(new MemConfigerIO(b), enabledCoreIds.size)) - for (compute <- enabledCoreIds.indices) { configArb.io.in(compute) <> io.compute(compute).config } - sharedBackend.io.config <> configArb.io.out - val barrier: Instance[BarrierUnit] = Instantiate(new BarrierUnit(enabledCoreIds.size)) - for (compute <- enabledCoreIds.indices) { - barrier.io.arrive(compute) := io.compute(compute).barrierArrive - io.compute(compute).barrierRelease := barrier.io.release(compute) - } -} diff --git a/arch/src/main/scala/framework/system/tile/BarrierUnit.scala b/arch/src/main/scala/framework/system/tile/BarrierUnit.scala deleted file mode 100644 index 5f54584a8..000000000 --- a/arch/src/main/scala/framework/system/tile/BarrierUnit.scala +++ /dev/null @@ -1,35 +0,0 @@ -package framework.system.tile - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public} - -/** - * BarrierUnit — tile-level hardware barrier for multi-core synchronization. - * - * Each core's BuckyballAccelerator raises arrive(i) after its ROB drains. - * When all N cores have arrived, release(i) is asserted for one cycle, - * allowing all cores to proceed simultaneously. - * - * @param nCores number of cores in this tile - */ -@instantiable -class BarrierUnit(val nCores: Int) extends Module { - - @public - val io = IO(new Bundle { - val arrive = Input(Vec(nCores, Bool())) - val release = Output(Vec(nCores, Bool())) - }) - - val arrived = RegInit(VecInit(Seq.fill(nCores)(false.B))) - val allArrived = arrived.asUInt.andR - - for (i <- 0 until nCores) { - when(io.arrive(i))(arrived(i) := true.B) - io.release(i) := allArrived - } - - when(allArrived) { - for (i <- 0 until nCores) { arrived(i) := false.B } - } -} diff --git a/arch/src/main/scala/framework/system/tile/ComputeTilePlatform.scala b/arch/src/main/scala/framework/system/tile/ComputeTilePlatform.scala deleted file mode 100644 index 42bb2e90e..000000000 --- a/arch/src/main/scala/framework/system/tile/ComputeTilePlatform.scala +++ /dev/null @@ -1,228 +0,0 @@ -package framework.system.tile - -import chisel3._ -import framework.system.tile.{BankNetwork, TileEndpoint, TileParams} -import framework.system.tile.tlink.{ - HasTLink, - TLinkIO, - T2TParams, - T2TTransfer, - SharedAxiEndpoint, - Control => TransferControl -} -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.ant.Service -import framework.system.configloader.{AntTileCore, RocketTileCore} -import framework.system.core.{ControllerAdmission, RocketCLink} -import hier.core.rocket.Commands -import framework.system.core.rocket.CpuParams -import framework.system.core.accelerator.{Admission, AntMemory} -import framework.memdomain.frontend.mem.dma.DmaStatus -import framework.memdomain.backend.shared.SharedMemLayout -import hier.tile.memory.{Composition, CorePlacement, CoreRole} -import memcore.bus.axi4 -import memcore.bus.chi.rnf.RnfParams -import memcore.memory.cpu.PhysicalRegion -import memcore.memory.coherence.configs.CoherenceParams -import memcore.memory.interlock.{Params => TrackingParams} -import memcore.memory.preflight.{Params => PreparationParams} - -/** One coherent Linux controller, local Ant execution contexts, and their NPU endpoints. */ -@instantiable -class ComputeTilePlatform(p: TileParams) extends TileEndpoint(p.linkParams, false) { - require(!p.main && p.controller.contains(0) && p.cpus.size == 1) - - val controller = p.cores.head match { - case core: RocketTileCore => core - case _ => throw new IllegalArgumentException("Compute controller must be Rocket") - } - - val workers = p.cores.tail.map { - case core: AntTileCore => core - case _ => throw new IllegalArgumentException("Compute workers must be Ant") - } - - val signatures = p.signatures.tail - val cpu = p.cpus.head - val memory = p.memory - val l1 = p.l1 - val regions = p.regions - val tracking = p.tracking - val axiParams = p.axi - val tiles = p.tiles - require(controller.buckyball.isEmpty && workers.nonEmpty && workers.size == signatures.size) - val local = workers.head.local - require(workers.forall(_.local == local), "Ant local storage geometry must be homogeneous within a tile") - val base = workers.head.buckyball - val enabled = 1 to workers.size - require(workers.forall(w => - w.buckyball.memDomain.sharedEnable && - w.buckyball.memDomain.computeCoreIds == enabled && w.buckyball.memDomain.nCores == workers.size + 1 - )) - val c = memory.chi - val prepared = - PreparationParams(c, tracking.entries, tracking.maxRanges, tracking.idBits, beatBytes = axiParams.bytes) - - val tileMemory = Instantiate(new axi4.Interconnect(axiParams, workers.size)) - tlink.mem <> tileMemory.io.out - - val composition = Instantiate(new Composition( - memory.copy(agents = 2), - Seq(CorePlacement(l1.copy(nodeId = 1), cpu, CoreRole.Controller)), - regions, - tracking, - Nil, - Nil, - Seq(0) - )) - - @public val controllerLink = IO(Flipped(new RocketCLink(p.rocket(0)))) - - @public val workerLinks = IO(MixedVec((1 until p.cores.size).map { i => - val params = p.ant(i) - Flipped(new framework.system.core.clink.CLinkIO( - params.buckyball, - params.local, - params.tracking, - params.bus - )(params.cpu)) - })) - - composition.io.cores(0) <> controllerLink.cpu - - val controllerAdmission = Instantiate(new ControllerAdmission(tracking, c, moves = true)(cpu)) - val service = Instantiate(new Service(local, workers.size)) - val broker = Instantiate(new AntMemory(workers.size, tracking, c)) - val banks = Instantiate(new BankNetwork(base, enabled, useMesh = true, controllerMove = true, physicalPorts = 3)) - - val transferParams = T2TParams( - sharedBytes = BigInt(SharedMemLayout.totalBank(base)) * base.memDomain.sharedBankEntries * axiParams.bytes, - bankBytes = base.memDomain.sharedBankEntries * axiParams.bytes, - tiles = tiles, - axi = axiParams - ) - - val transferControl = Instantiate(new TransferControl(transferParams)) - val transfer = Instantiate(new T2TTransfer(transferParams)) - val sharedEndpoint = Instantiate(new SharedAxiEndpoint(transferParams)) - transferControl.io.tileId := tlink.tileId - transfer.io.command <> transferControl.io.command - transferControl.io.completion <> transfer.io.completion - banks.io.physical(0) <> transfer.io.source - banks.io.physical(1) <> sharedEndpoint.io.read - banks.io.physical(2) <> sharedEndpoint.io.write - banks.io.lease.get <> transferControl.io.lease.get - tlink.t2t.get.tx <> transfer.io.mem - sharedEndpoint.io.mem <> tlink.t2t.get.rx - composition.io.hartIds := tlink.hartIds - composition.io.resetVector := tlink.resetVector - composition.io.time := tlink.time - composition.io.interrupts := tlink.interrupts - tlink.control <> composition.io.control - tlink.uncachedRequest <> composition.io.uncachedRequest - composition.io.uncachedResponse <> tlink.uncachedResponse - tlink.retired := composition.io.retired - tlink.retiredPc := composition.io.retiredPc - tlink.trapped := composition.io.trapped - tlink.trapCause := composition.io.trapCause - tlink.trapValue := composition.io.trapValue - tlink.trapPc := composition.io.trapPc - - // No hidden CPU polls the legacy task port. Only the controller drives this service. - composition.io.controllerSatp := 0.U - for (port <- composition.io.taskControl) { - port.cmd.valid := false.B - port.cmd.bits := 0.U.asTypeOf(port.cmd.bits) - port.resp.ready := true.B - port.exception := false.B - } - val core = composition.io.admission(0) - controllerAdmission.io.core <> core - val task = controllerAdmission.io.task - val rd = Reg(UInt(5.W)) - val isTransfer = task.cmd.bits.funct >= 13.U - transferControl.io.request.bits.operation := task.cmd.bits.funct - transferControl.io.request.bits.context := task.cmd.bits.rs1Data(63, 32) - transferControl.io.request.bits.field := task.cmd.bits.rs1Data(31, 0) - transferControl.io.request.bits.data := task.cmd.bits.rs2Data - transferControl.io.request.valid := task.cmd.valid && isTransfer - service.io.request.valid := task.cmd.valid && !isTransfer - service.io.request.bits.operation := task.cmd.bits.funct - service.io.request.bits.context := task.cmd.bits.rs1Data(63, 32) - service.io.request.bits.field := task.cmd.bits.rs1Data(31, 0) - service.io.request.bits.data := task.cmd.bits.rs2Data - task.cmd.ready := Mux(isTransfer, transferControl.io.request.ready, service.io.request.ready) - when(task.cmd.fire) { - assert(task.cmd.bits.opcode === "h2b".U && task.cmd.bits.funct3 === 7.U, "Invalid Ant management instruction") - rd := task.cmd.bits.rd - } - task.resp.valid := service.io.reply.valid || transferControl.io.reply.valid - task.resp.bits.rd := rd - task.resp.bits.data := Mux(transferControl.io.reply.valid, transferControl.io.reply.bits, service.io.reply.bits) - transferControl.io.reply.ready := task.resp.ready - assert(!(service.io.reply.valid && transferControl.io.reply.valid), "Concurrent tile management responses") - service.io.reply.ready := task.resp.ready - task.busy := !service.io.idle || !transferControl.io.idle - task.interrupt := false.B - banks.io.hartIds := tlink.executionIds - banks.io.controllerMvover.get <> controllerAdmission.io.move - - broker.io.controller.cpuQuery := core.cpuQuery - core.cpuAllow := controllerAdmission.io.core.cpuAllow && broker.io.controller.cpuAllow - core.cpuProbeAllow := controllerAdmission.io.core.cpuProbeAllow && broker.io.controller.cpuProbeAllow - core.maintenance <> broker.io.controller.maintenance - broker.io.controller.maintained <> core.maintained - core.pteRequest <> broker.io.controller.pteRequest - broker.io.controller.pteResponse <> core.pteResponse - - val failures = Wire(Vec(workers.size, Valid(new DmaStatus))) - val drained = Wire(Vec(workers.size, Bool())) - for ((worker, i) <- workers.zipWithIndex) { - val b = worker.buckyball - val link = workerLinks(i) - val admission = Instantiate(new Admission(b, tracking, prepared, regions, axiParams)(cpu)) - link.ctrl.bind.valid := service.io.launched(i).valid - link.ctrl.bind.bits.task := service.io.launched(i).bits.task - link.ctrl.bind.bits.snapshot := controllerAdmission.io.taskContext - service.io.locals(i) <> link.local - link.ctrl.workDrained := admission.io.workDrained - link.ctrl.halted := admission.io.halted - drained(i) := link.ctrl.drained - service.io.signatures(i) := link.ctrl.fingerprint - service.io.online(i) := link.ctrl.online - link.admission <> admission.io.core - link.memory <> broker.io.clients(i) - admission.io.task.cmd.ready := false.B - admission.io.task.resp.valid := false.B - admission.io.task.resp.bits := 0.U.asTypeOf(admission.io.task.resp.bits) - admission.io.task.busy := false.B - admission.io.task.interrupt := false.B - link.npu.command <> admission.io.npuCommand - admission.io.npuResponse <> link.npu.response - admission.io.allocation.valid := link.npu.allocation.valid - admission.io.allocation.bits := link.npu.allocation.bits.rob_id - admission.io.retired := link.npu.retired - admission.io.npuFault := link.npu.fault - admission.io.npuBusy := link.npu.busy - admission.io.npuInterrupt := link.npu.interrupt - admission.io.footprints := link.npu.footprints - admission.io.dma <> link.mem - tileMemory.io.in(i) <> admission.io.axi - failures(i).valid := admission.io.halted - failures(i).bits := admission.io.fault - link.ctrl.hartId := tlink.executionIds(i + 1) - link.ctrl.shmOwner := tlink.executionIds(0) - banks.io.compute(i) <> link.shm - link.sharedHashes.foreach(_ := banks.io.bankHashes.get) - } - tlink.failure(0).valid := failures.map(_.valid).reduce(_ || _) || controllerAdmission.io.moveFault.valid - tlink.failure(0).bits := Mux( - controllerAdmission.io.moveFault.valid, - controllerAdmission.io.moveFault.bits, - PriorityMux(failures.map(f => f.valid -> f.bits)) - ) - tlink.workDrained( - 0 - ) := service.io.idle && transferControl.io.idle && !sharedEndpoint.io.busy && drained.asUInt.andR && core.outstanding === 0.U -} diff --git a/arch/src/main/scala/framework/system/tile/MainTile.scala b/arch/src/main/scala/framework/system/tile/MainTile.scala deleted file mode 100644 index df8744801..000000000 --- a/arch/src/main/scala/framework/system/tile/MainTile.scala +++ /dev/null @@ -1,15 +0,0 @@ -package framework.system.tile - -import chisel3.experimental.hierarchy.{instantiable, Instantiate} -import framework.top.GlobalConfig -import framework.system.GlobalConfigOps._ -import framework.system.core.CoreFactory - -@instantiable -class MainTile(b: GlobalConfig) extends TileEndpoint(b.tileLinkParams, true) { - require(b.tileDesign.get.design.factory.isEmpty, "The main tile is built into the device") - val p = b.tileParams - val platform = Instantiate(new MainTilePlatform(p)) - val cores = p.cores.indices.map(i => CoreFactory.rocket(b.forCore(i))) - TileConnection.main(platform, tlink, cores.map(_.clink)) -} diff --git a/arch/src/main/scala/framework/system/tile/MainTilePlatform.scala b/arch/src/main/scala/framework/system/tile/MainTilePlatform.scala deleted file mode 100644 index d9c428a99..000000000 --- a/arch/src/main/scala/framework/system/tile/MainTilePlatform.scala +++ /dev/null @@ -1,406 +0,0 @@ -package framework.system.tile - -import chisel3._ -import framework.system.tile.{BankNetwork, TileEndpoint, TileParams} -import framework.system.tile.tlink.{ - HasTLink, - TLinkIO, - T2TParams, - T2TTransfer, - SharedAxiEndpoint, - Control => TransferControl -} -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.system.core.rocket.{CpuParams, RoCCIO} -import framework.system.configloader.{AntTileCore, RocketTileCore, SharedStorageParams, TileCore, TileTopology} -import framework.system.core.rocket.configs.RocketCpuParam -import framework.system.core.{ControllerAdmission, RocketCLink} -import hier.core.rocket.Commands -import framework.system.core.accelerator.Admission -import framework.memdomain.frontend.mem.dma.DmaStatus -import framework.memdomain.backend.shared.{SharedMemLayout, SharedPhysicalPort, SharedPhysicalRequest} -import framework.memdomain.backend.MTraceDPI -import memcore.memory.mesh_shm.{MeshSharedMem, MeshSharedMemParams} -import hier.tile.memory.{Composition, CoreInterrupts, CorePlacement, CoreRole} -import memcore.bus.chi.RequesterPort -import memcore.bus.chi.rnf.RnfParams -import memcore.bus.chi.snf.{LineRequest, LineResponse} -import memcore.memory.cpu.{CpuMemParams, PhysicalRegion, UncachedRequest, UncachedResponse} -import memcore.memory.coherence.configs.CoherenceParams -import memcore.memory.interlock.{Params => TrackingParams} -import memcore.memory.preflight.{Params => PreparationParams} - -/** A PB-defined Tile assembled explicitly from its CPU, cache, accelerator and bank IPs. */ -@instantiable -class MainTilePlatform(p: TileParams) extends TileEndpoint(p.linkParams, true) { - val coreTypes = p.cores - val cpuParameters = p.cpus - val memory = p.memory - val l1 = p.l1 - val regions = p.regions - val tracking = p.tracking - val controlCores = p.controls - val axiParams = p.axi - val sharedStorage = p.sharedStorage - val tiles = p.tiles - require(p.controller.isEmpty, "Main tile has no hidden task controller") - require(coreTypes.size == cpuParameters.size) - require(memory.agents == 2 * coreTypes.size) - - val cores = coreTypes.map { - case core: RocketTileCore => core - case _ => throw new IllegalArgumentException("This Tile requires Rocket CPUs") - } - - // A CPU-only Tile (for example a main tile without Buckyball cores) has no banks or NPU DMA. - val enabled = cores.indices.filter(i => cores(i).buckyball.isDefined) - val base = enabled.headOption.map(i => cores(i).buckyball.get) - // Shared banks form the Tile bank mesh; otherwise every accelerator keeps its private banks. - val shared = base.exists(_.memDomain.sharedEnable) - for (b <- base) { - require(b.memDomain.nCores == cores.size) - require(b.memDomain.computeCoreIds == enabled) - } - for { - i <- enabled - base <- base - } { - val b = cores(i).buckyball.get - require(b.tile.xLen == 64 && b.memDomain.nCores == cores.size && b.memDomain.sharedEnable == shared) - require(b.memDomain.bankWidth == base.memDomain.bankWidth && - b.memDomain.dma_buswidth == base.memDomain.dma_buswidth && - b.memDomain.bankChannel == base.memDomain.bankChannel && - b.frontend.rob_entries == base.frontend.rob_entries) - } - val c = memory.chi - val cp = CpuMemParams(c, tagBits = 6) - val prepared = - PreparationParams(c, tracking.entries, tracking.maxRanges, tracking.idBits, beatBytes = axiParams.bytes) - - val placements = cores.indices.map { i => - CorePlacement( - l1.copy(nodeId = i + 1), - cpuParameters(i), - CoreRole(compute = cores(i).buckyball.isDefined, scheduler = true) - ) - } - - val controlCoreIds = cores.indices - - // TLink and NPU requests share the same bank network when main has an accelerator. - val controllerMove = shared && cores.head.buckyball.isEmpty - - val banks: Option[Instance[BankNetwork]] = Option.when(shared)( - Instantiate(new BankNetwork( - base.get, - enabled, - useMesh = true, - controllerMove = controllerMove, - physicalPorts = if (sharedStorage.isDefined) 4 else 0 - )) - ) - - val mainTransfer = sharedStorage.map { geometry => - require(enabled.isEmpty || shared, "Main NPU and TLink require one shared bank network") - val p = T2TParams(geometry.bytes, geometry.bankBytes, tiles, axiParams) - val control = Instantiate(new TransferControl(p, localAccess = true, leaseAccess = banks.isDefined)) - val transfer = Instantiate(new T2TTransfer(p)) - val endpoint = Instantiate(new SharedAxiEndpoint(p)) - val physical = Wire(Vec(4, Flipped(new SharedPhysicalPort(128)))) - val pending = RegInit(VecInit(Seq.fill(4)(false.B))) - banks match { - case Some(network) => - val b = base.get - require( - b.memDomain.bankWidth == geometry.bankBits && - b.memDomain.sharedBankEntries == geometry.bankEntries && - SharedMemLayout.totalBank(b) == geometry.banks, - "Main NPU and TLink storage geometry must agree" - ) - network.io.physical <> physical - network.io.lease.get <> control.io.lease.get - for ((port, index) <- physical.zipWithIndex) { - when(port.request.fire) { - assert(!pending(index), "Main SHM port accepts one outstanding request") - pending(index) := true.B - } - when(port.response.fire) { - assert(pending(index) && !port.response.bits.error, "Main SHM response has no owner or failed") - pending(index) := false.B - } - } - case None => - val (rows, columns) = MeshSharedMemParams.dimensions(geometry.banks) - val core = MeshSharedMemParams.directionalAttachments(rows, columns, Seq(1)).head - val shm = Instantiate(new MeshSharedMem(MeshSharedMemParams( - rows = rows, - cols = columns, - entriesPerBank = geometry.bankEntries, - dataBits = geometry.bankBits, - tagBits = 8, - cores = Seq(core), - externalChannels = 4, - visibleBanks = geometry.banks - ))) - // Main has no private NPU endpoint. Its CPU management and T2T use physical SHM ports. - for (channel <- shm.io.channels) { - channel.request.valid := false.B - channel.request.bits := 0.U.asTypeOf(channel.request.bits) - channel.response.ready := true.B - } - shm.io.transferCommand.valid := false.B - shm.io.transferCommand.bits := 0.U.asTypeOf(shm.io.transferCommand.bits) - shm.io.transferCompletion.ready := true.B - for (local <- shm.io.localBanks) { - local.request.ready := false.B - local.response.valid := false.B - local.response.bits := 0.U.asTypeOf(local.response.bits) - } - shm.io.bankWrites.foreach(_.ready := true.B) - val bankBits = log2Ceil(geometry.bankBytes) - for ((port, index) <- physical.zipWithIndex) { - val channel = shm.io.external(index) - val held = Reg(new SharedPhysicalRequest(128)) - when(port.request.valid && !reset.asBool) { - assert( - port.request.bits.address(3, 0) === 0.U && port.request.bits.address < geometry.bytes.U, - "Main SHM physical access is unaligned or exceeds capacity" - ) - } - channel.request.valid := port.request.valid - channel.request.bits := 0.U.asTypeOf(channel.request.bits) - channel.request.bits.bank := port.request.bits.address >> bankBits - channel.request.bits.tuser := Cat(port.request.bits.address(bankBits - 1, 4).pad(16), port.request.bits.write) - channel.request.bits.data := port.request.bits.data - channel.request.bits.mask := port.request.bits.mask - channel.request.bits.tag := 0.U - channel.request.bits.tlast := true.B - port.request.ready := channel.request.ready - port.response.valid := channel.response.valid - port.response.bits.data := channel.response.bits.data - port.response.bits.error := channel.response.bits.error - channel.response.ready := port.response.ready - when(port.request.fire) { - assert(!pending(index), "Main SHM port accepts one outstanding request") - pending(index) := true.B - held := port.request.bits - } - when(port.response.fire) { - assert(pending(index) && !port.response.bits.error, "Main SHM response has no owner or failed") - pending(index) := false.B - } - val trace = Module(new MTraceDPI) - val value = Mux(held.write, held.data, port.response.bits.data) - trace.io.clock := clock - trace.io.reset := reset.asBool - trace.io.hart_id := tlink.executionIds(0) - trace.io.channel := (shm.io.channels.size + index).U - trace.io.is_shared := 1.U - trace.io.is_write := held.write.asUInt - trace.io.rob_id := 0.U - trace.io.inst_id := 0.U - trace.io.vbank_id := 0.U - trace.io.group_id := 0.U - trace.io.pbank_id := held.address >> bankBits - trace.io.addr := held.address(bankBits - 1, 4) - trace.io.write_mask := Mux(held.write, held.mask, 0.U) - trace.io.data_lo := value(63, 0) - trace.io.data_hi := value(127, 64) - trace.io.enable := port.response.fire - } - } - control.io.tileId := tlink.tileId - transfer.io.command <> control.io.command - control.io.completion <> transfer.io.completion - physical(0) <> transfer.io.source - physical(1) <> endpoint.io.read - physical(2) <> endpoint.io.write - physical(3) <> control.io.local.get - tlink.t2t.get.tx <> transfer.io.mem - endpoint.io.mem <> tlink.t2t.get.rx - (control, endpoint, pending.asUInt.orR) - } - - val tileMemory = Option.when(enabled.nonEmpty)( - Instantiate(new memcore.bus.axi4.Interconnect(axiParams, enabled.size)) - ) - - tileMemory match { - case Some(fabric) => tlink.mem <> fabric.io.out - case None => - tlink.mem.aw.valid := false.B - tlink.mem.aw.bits := 0.U.asTypeOf(tlink.mem.aw.bits) - tlink.mem.w.valid := false.B - tlink.mem.w.bits := 0.U.asTypeOf(tlink.mem.w.bits) - tlink.mem.ar.valid := false.B - tlink.mem.ar.bits := 0.U.asTypeOf(tlink.mem.ar.bits) - tlink.mem.b.ready := true.B - tlink.mem.r.ready := true.B - } - - val composition: Instance[Composition] = Instantiate(new Composition( - memory, - placements, - regions, - tracking, - Nil, - Nil, - controlCoreIds - )) - - composition.io.hartIds := tlink.hartIds - composition.io.resetVector := tlink.resetVector - composition.io.time := tlink.time - composition.io.interrupts := tlink.interrupts - tlink.uncachedRequest <> composition.io.uncachedRequest - composition.io.uncachedResponse <> tlink.uncachedResponse - locally { - val memorySystem = Instantiate(new hier.tile.memory.Memory(memory.copy(agents = 2 * controlCores))) - for (lane <- 0 until 2 * cores.size) { - val localNode = lane + 1 - val globalIndex = lane % cores.size + (if (lane >= cores.size) controlCores else 0) - val source = composition.io.control(lane) - val target = memorySystem.io.coherent(globalIndex) - target <> source - target.req.bits.srcId := (globalIndex + 1).U - target.txRsp.bits.srcId := (globalIndex + 1).U - target.txDat.bits.srcId := (globalIndex + 1).U - source.rxRsp.bits.tgtId := localNode.U - source.rxDat.bits.tgtId := localNode.U - } - val remoteCount = controlCores - cores.size - for (lane <- 0 until 2 * remoteCount) { - val globalIndex = cores.size + lane % remoteCount + (if (lane >= remoteCount) controlCores else 0) - memorySystem.io.coherent(globalIndex) <> tlink.remoteControl(lane) - } - tlink.backingRequest <> memorySystem.io.backingReq - memorySystem.io.backingResp <> tlink.backingResponse - } - tlink.retired := composition.io.retired - tlink.retiredPc := composition.io.retiredPc - tlink.trapped := composition.io.trapped - tlink.trapCause := composition.io.trapCause - tlink.trapValue := composition.io.trapValue - tlink.trapPc := composition.io.trapPc - banks.foreach(_.io.hartIds := tlink.executionIds) - tlink.failure := 0.U.asTypeOf(tlink.failure) - - def connectTask(i: Int, task: RoCCIO, npuIdle: Bool = true.B): Unit = { - val target = composition.io.taskControl(i) - if (i == 0 && mainTransfer.isDefined) { - val (control, _, _) = mainTransfer.get - val transfer = task.cmd.bits.funct >= 13.U - val accessesStorage = task.cmd.bits.funct === 14.U || - (task.cmd.bits.funct >= 16.U && task.cmd.bits.funct <= 19.U) - val transferReady = !accessesStorage || npuIdle - val rd = Reg(UInt(5.W)) - control.io.request.valid := task.cmd.valid && transfer && transferReady - control.io.request.bits.operation := task.cmd.bits.funct - control.io.request.bits.context := task.cmd.bits.rs1Data(63, 32) - control.io.request.bits.field := task.cmd.bits.rs1Data(31, 0) - control.io.request.bits.data := task.cmd.bits.rs2Data - target.cmd.valid := task.cmd.valid && !transfer - target.cmd.bits := task.cmd.bits - target.exception := task.exception - task.cmd.ready := Mux(transfer, control.io.request.ready && transferReady, target.cmd.ready) - when(task.cmd.fire) { - assert(task.cmd.bits.opcode === "h2b".U && task.cmd.bits.funct3 === 7.U, "Invalid main management instruction") - rd := task.cmd.bits.rd - } - assert(!(control.io.reply.valid && target.resp.valid), "Concurrent main management responses") - task.resp.valid := control.io.reply.valid || target.resp.valid - task.resp.bits.rd := Mux(control.io.reply.valid, rd, target.resp.bits.rd) - task.resp.bits.data := Mux(control.io.reply.valid, control.io.reply.bits, target.resp.bits.data) - control.io.reply.ready := task.resp.ready - target.resp.ready := task.resp.ready - task.busy := !control.io.idle || target.busy - task.interrupt := target.interrupt - } else task <> target - } - - @public val coreLinks = IO(MixedVec(cores.indices.map(i => Flipped(new RocketCLink(p.rocket(i)))))) - for (i <- cores.indices) { - composition.io.cores(i) <> coreLinks(i).cpu - cores(i).buckyball match { - case Some(b) => - val accelerator = coreLinks(i).accelerator.get - val admission: Instance[Admission] = - Instantiate(new Admission(b, tracking, prepared, regions, axiParams)(cpuParameters(i))) - admission.io.core <> composition.io.admission(i) - connectTask(i, admission.io.task, !accelerator.npu.busy) - if (i == 0) composition.io.controllerSatp := admission.io.taskSatp - if (i == 0 && mainTransfer.isDefined) { - // A younger NPU command cannot pass an active local storage-management command. - val storageIdle = mainTransfer.get._1.io.idle - accelerator.npu.command.valid := admission.io.npuCommand.valid && storageIdle - accelerator.npu.command.bits := admission.io.npuCommand.bits - admission.io.npuCommand.ready := accelerator.npu.command.ready && storageIdle - } else accelerator.npu.command <> admission.io.npuCommand - admission.io.npuResponse <> accelerator.npu.response - admission.io.allocation.valid := accelerator.npu.allocation.valid - admission.io.allocation.bits := accelerator.npu.allocation.bits.rob_id - admission.io.retired := accelerator.npu.retired - admission.io.npuFault := accelerator.npu.fault - admission.io.npuBusy := accelerator.npu.busy - admission.io.npuInterrupt := accelerator.npu.interrupt - admission.io.footprints := accelerator.npu.footprints - admission.io.dma <> accelerator.mem - val bankEndpoint = enabled.indexOf(i) - tileMemory.get.io.in(bankEndpoint) <> admission.io.axi - accelerator.hartId := tlink.executionIds(i) - accelerator.shmOwner := tlink.executionIds(0) - banks match { - case Some(network) => - val port = network.io.compute(bankEndpoint) - port.requests <> accelerator.shm.requests - port.move <> accelerator.shm.move - accelerator.shm.local <> port.local - port.config <> accelerator.shm.config - port.queryValid := accelerator.shm.queryValid - port.queryVbank := accelerator.shm.queryVbank - accelerator.shm.queryGroups := port.queryGroups - port.barrierArrive := accelerator.shm.barrierArrive - accelerator.shm.barrierRelease := port.barrierRelease - accelerator.sharedHashes.foreach(_ := network.io.bankHashes.get) - case None => - // Private banks: no shared channels exist, and mesh moves and barriers stay idle. - accelerator.shm.move.command.ready := false.B - accelerator.shm.move.completion.valid := false.B - accelerator.shm.move.completion.bits := false.B - accelerator.shm.local.request.valid := false.B - accelerator.shm.local.request.bits := 0.U.asTypeOf(accelerator.shm.local.request.bits) - accelerator.shm.local.response.ready := true.B - accelerator.shm.config.ready := false.B - accelerator.shm.queryGroups := 0.U - accelerator.shm.barrierRelease := false.B - } - tlink.failure(i).valid := admission.io.halted - tlink.failure(i).bits := admission.io.fault - tlink.workDrained(i) := admission.io.workDrained && - (if (i == 0) mainTransfer.map { case (control, endpoint, storageBusy) => - control.io.idle && !endpoint.io.busy && !storageBusy - }.getOrElse(true.B) - else true.B) - case None => - val admission: Instance[ControllerAdmission] = - Instantiate(new ControllerAdmission(tracking, c, moves = i == 0 && controllerMove)(cpuParameters(i))) - admission.io.core <> composition.io.admission(i) - connectTask(i, admission.io.task) - if (i == 0) composition.io.controllerSatp := admission.io.taskSatp - if (i == 0 && controllerMove) { - banks.get.io.controllerMvover.get <> admission.io.move - } else { - admission.io.move.command.ready := false.B - admission.io.move.completion.valid := false.B - admission.io.move.completion.bits := false.B - } - tlink.failure(i) := admission.io.moveFault - tlink.workDrained(i) := composition.io.admission(i).outstanding === 0.U && - (if (i == 0) mainTransfer.map { case (control, endpoint, storageBusy) => - control.io.idle && !endpoint.io.busy && !storageBusy - }.getOrElse(true.B) - else true.B) - } - } -} diff --git a/arch/src/main/scala/framework/system/tile/TileConnection.scala b/arch/src/main/scala/framework/system/tile/TileConnection.scala deleted file mode 100644 index 2c27661f5..000000000 --- a/arch/src/main/scala/framework/system/tile/TileConnection.scala +++ /dev/null @@ -1,30 +0,0 @@ -package framework.system.tile - -import chisel3._ -import chisel3.experimental.hierarchy.Instance -import framework.system.core.RocketCLink -import framework.system.core.clink.CLinkIO -import framework.system.tile.tlink.TLinkIO - -/** Cross-level binding is shared by all tile designs. */ -object TileConnection { - - def main(platform: Instance[MainTilePlatform], link: TLinkIO, cores: Seq[RocketCLink]): Unit = { - require(cores.size == platform.coreLinks.size) - cores.zipWithIndex.foreach { case (core, i) => platform.coreLinks(i) <> core } - link <> platform.tlink - } - - def compute( - platform: Instance[ComputeTilePlatform], - link: TLinkIO, - controller: RocketCLink, - workers: Seq[CLinkIO] - ): Unit = { - require(workers.size == platform.workerLinks.size) - platform.controllerLink <> controller - workers.zipWithIndex.foreach { case (core, i) => platform.workerLinks(i) <> core } - link <> platform.tlink - } - -} diff --git a/arch/src/main/scala/framework/system/tile/TileEndpoint.scala b/arch/src/main/scala/framework/system/tile/TileEndpoint.scala deleted file mode 100644 index b87580867..000000000 --- a/arch/src/main/scala/framework/system/tile/TileEndpoint.scala +++ /dev/null @@ -1,74 +0,0 @@ -package framework.system.tile - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance} -import framework.system.configloader.{AntTileCore, RocketTileCore, SharedStorageParams, TileCore} -import framework.system.core.rocket.CpuParams -import framework.system.core.{AntCoreParams, RocketCoreParams} -import hier.core.rocket.Commands -import framework.system.tile.tlink.{HasTLink, TLinkIO} -import memcore.bus.axi4 -import memcore.bus.chi.rnf.RnfParams -import memcore.memory.cpu.PhysicalRegion -import memcore.memory.coherence.configs.CoherenceParams -import memcore.memory.interlock.{Params => TrackingParams} - -case class TileLinkParams( - main: Boolean, - cpus: Int, - executions: Int, - controls: Int, - bus: memcore.bus.chi.Params, - axi: axi4.Params, - sharedStorage: Boolean) - -/** Structural parameters only; tile IDs and hart IDs arrive through TLink. */ -case class TileParams( - main: Boolean, - cores: Seq[TileCore], - signatures: Seq[BigInt], - controller: Option[Int], - cpus: Seq[CpuParams], - memory: CoherenceParams, - l1: RnfParams, - regions: Seq[PhysicalRegion], - tracking: TrackingParams, - controls: Int, - axi: axi4.Params, - sharedStorage: Option[SharedStorageParams], - tiles: Int) { - - def linkParams: TileLinkParams = - TileLinkParams(main, cpus.size, cores.size, if (main) controls else 0, memory.chi, axi, sharedStorage.isDefined) - - def rocket(index: Int): RocketCoreParams = { - val core = cores(index) match { - case value: RocketTileCore => value - case _ => throw new IllegalArgumentException("Rocket parameters require a Rocket slot") - } - val cpuIndex = cores.take(index).count(_.isInstanceOf[RocketTileCore]) - RocketCoreParams( - cpus(cpuIndex), - l1.copy(nodeId = cpuIndex + 1), - l1.copy(nodeId = cpus.size + cpuIndex + 1), - regions, - Commands(core.buckyball.isDefined, true, tracking), - core.buckyball - ) - } - - def ant(index: Int): AntCoreParams = { - val core = cores(index) match { - case value: AntTileCore => value - case _ => throw new IllegalArgumentException("Ant parameters require an Ant slot") - } - AntCoreParams(core.buckyball, core.local, tracking, memory.chi, signatures(index), cpus.head) - } - -} - -@instantiable -abstract class TileEndpoint(p: TileLinkParams, main: Boolean) extends Module with HasTLink { - require(p.main == main, "Software tile role must match the designed TLink role") - @public val tlink = IO(new TLinkIO(p.main, p.cpus, p.executions, p.controls, p.bus, p.axi, p.sharedStorage)) -} diff --git a/arch/src/main/scala/framework/system/tile/TileFactory.scala b/arch/src/main/scala/framework/system/tile/TileFactory.scala deleted file mode 100644 index 5abafe0ad..000000000 --- a/arch/src/main/scala/framework/system/tile/TileFactory.scala +++ /dev/null @@ -1,18 +0,0 @@ -package framework.system.tile - -import chisel3.experimental.hierarchy.Instance -import framework.top.GlobalConfig - -trait TileFactory { - def instantiate(b: GlobalConfig): Instance[TileEndpoint] -} - -object TileFactory { - - def instantiate(b: GlobalConfig): Instance[TileEndpoint] = { - require(!b.tileDesign.get.design.main, "The main tile is built into the device") - Class.forName(b.tileDesign.get.design.factory + "$") - .getField("MODULE$").get(null).asInstanceOf[TileFactory].instantiate(b) - } - -} diff --git a/arch/src/main/scala/framework/system/tile/TileParameters.scala b/arch/src/main/scala/framework/system/tile/TileParameters.scala deleted file mode 100644 index b1b4a592b..000000000 --- a/arch/src/main/scala/framework/system/tile/TileParameters.scala +++ /dev/null @@ -1,34 +0,0 @@ -package framework.system.tile - -import chisel3.util.log2Ceil -import framework.system.core.rocket.CpuParams -import framework.system.core.rocket.configs.RocketCpuParam - -object TileParameters { - - def cpuParameters( - cpu: RocketCpuParam, - t: framework.system.tile.configs.TileParam, - maxHartId: Int, - physicalAddressBits: Int - ): CpuParams = { - require(physicalAddressBits > t.pgIdxBits && physicalAddressBits <= t.paddrBits) - require( - !cpu.useVM || physicalAddressBits < t.vaddrBits, - "Rocket requires platform physical addresses narrower than its virtual address format" - ) - val core = RocketCpuParam.toRocketCoreParams(cpu, t.xLen, t.pgLevels).copy( - nPMPs = t.nPMPs - ) - CpuParams( - core = core, - dcache = Some(RocketCpuParam.toDCacheParams(cpu, t.coreDataBytes * 8, 64)), - icache = Some(RocketCpuParam.toICacheParams(cpu, t.coreDataBytes * 8, 64)), - btb = RocketCpuParam.toBTBParams(cpu), - physicalAddressBits = physicalAddressBits, - hartIdBits = math.max(1, log2Ceil(maxHartId + 1)), - beatBytes = t.coreDataBytes - ) - } - -} diff --git a/arch/src/main/scala/framework/system/tile/configs/TileParam.scala b/arch/src/main/scala/framework/system/tile/configs/TileParam.scala deleted file mode 100644 index bb59efd4d..000000000 --- a/arch/src/main/scala/framework/system/tile/configs/TileParam.scala +++ /dev/null @@ -1,27 +0,0 @@ -package framework.system.tile.configs - -import upickle.default._ - -case class TileParam( - coreDataBytes: Int, - xLen: Int, - vaddrBits: Int, - paddrBits: Int, - pgIdxBits: Int, - pgLevels: Int, - nPMPs: Int) // Physical Memory Protection entries, typically 8 or 16 - -object TileParam { - implicit val rw: ReadWriter[TileParam] = macroRW - - def apply(): TileParam = TileParam( - coreDataBytes = 0, - xLen = 0, - vaddrBits = 0, - paddrBits = 0, - pgIdxBits = 0, - pgLevels = 0, - nPMPs = 0 - ) - -} diff --git a/arch/src/main/scala/framework/system/tile/tlink/Control.scala b/arch/src/main/scala/framework/system/tile/tlink/Control.scala deleted file mode 100644 index d52264a50..000000000 --- a/arch/src/main/scala/framework/system/tile/tlink/Control.scala +++ /dev/null @@ -1,129 +0,0 @@ -package framework.system.tile.tlink - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.ant.ControlRequest -import framework.memdomain.backend.shared.{SharedLeasePort, SharedPhysicalPort} - -/** The controller stages one transfer; the start instruction returns after destination commit. */ -@instantiable -class Control(p: T2TParams, localAccess: Boolean = false, leaseAccess: Boolean = true) extends Module { - - @public val io = IO(new Bundle { - val request = Flipped(Decoupled(new ControlRequest)) - val reply = Decoupled(UInt(64.W)) - val tileId = Input(UInt(p.tileBits.W)) - val command = Decoupled(new T2TCommand(p)) - val completion = Flipped(Decoupled(Bool())) - val idle = Output(Bool()) - val local = Option.when(localAccess)(new SharedPhysicalPort(128)) - val lease = Option.when(leaseAccess)(new SharedLeasePort) - }) - - val idle :: running :: reading :: leasing :: responding :: Nil = Enum(5) - val state = RegInit(idle) - val fields = RegInit(0.U(4.W)) - val staged = Reg(Vec(4, UInt(64.W))) - val answer = Reg(UInt(64.W)) - val request = io.request.bits - val start = request.operation === 14.U - val access = request.operation === 16.U || request.operation === 17.U - val lease = request.operation === 18.U || request.operation === 19.U - val upperHalf = Reg(Bool()) - val localWrite = Reg(Bool()) - val localReady = io.local.map(_.request.ready).getOrElse(false.B) - val leaseReady = io.lease.map(_.request.ready).getOrElse(false.B) - io.lease.foreach { port => - port.request.valid := state === idle && io.request.valid && lease && !reset.asBool - port.request.bits.endpoint := request.context - port.request.bits.vbank := request.field(31, 16) - port.request.bits.group := request.field(15, 0) - port.request.bits.release := request.operation === 19.U - port.response.ready := state === leasing && !reset.asBool - when(port.response.fire) { - answer := port.response.bits - state := responding - } - } - io.local.foreach { port => - val address = Cat(request.context, request.field) - port.request.valid := state === idle && io.request.valid && access && !reset.asBool - port.request.bits.address := address & ~15.U(64.W) - port.request.bits.write := request.operation === 17.U - port.request.bits.data := Cat(request.data, request.data) - port.request.bits.mask := Mux(address(3), "hff00".U, "h00ff".U) - port.response.ready := state === reading && !reset.asBool - when(port.response.fire) { - assert(!port.response.bits.error, "Main shared-storage access failed") - answer := Mux(localWrite, 0.U, Mux(upperHalf, port.response.bits.data(127, 64), port.response.bits.data(63, 0))) - state := responding - } - } - io.command.valid := state === idle && io.request.valid && start && !reset.asBool - io.command.bits.sourceByteAddress := staged(0) - io.command.bits.targetTile := staged(1) - io.command.bits.targetByteAddress := staged(2) - io.command.bits.bytes := staged(3) - io.request.ready := state === idle && !reset.asBool && - Mux(lease, leaseReady, Mux(access, localReady, !start || io.command.ready)) - io.reply.valid := state === responding && !reset.asBool - io.reply.bits := answer - io.completion.ready := state === running && !reset.asBool - io.idle := state === idle - when(io.request.valid && state === idle && !reset.asBool) { - assert( - (request.operation >= 13.U && request.operation <= 15.U) || - (localAccess.B && access) || (leaseAccess.B && lease), - "Invalid TLink management operation" - ) - } - when(io.request.fire) { - when(!access && !lease)(assert(request.context === 0.U, "TLink management has one tile-owned transfer context")) - answer := 0.U - state := responding - when(lease) { - assert(request.data === 0.U, "TLink shared lease reserves rs2") - state := leasing - } - when(access) { - val address = Cat(request.context, request.field) - assert(address(2, 0) === 0.U && (address +& 8.U) <= p.sharedBytes.U, "Invalid main shared-storage access") - upperHalf := address(3) - localWrite := request.operation === 17.U - state := reading - } - switch(request.operation) { - is(13.U) { - assert(request.field < 4.U, "Invalid TLink transfer descriptor field") - staged(request.field(1, 0)) := request.data - fields := fields | UIntToOH(request.field, 4) - } - is(14.U) { - assert(request.field === 0.U, "TLink start reserves the field selector") - assert(fields.andR, "Incomplete TLink transfer descriptor") - assert( - staged(1) < p.tiles.U && staged(1) =/= io.tileId, - "TLink transfer must target another tile" - ) - assert(staged(3) > 0.U && (staged(3) >> 32) === 0.U, "Invalid TLink transfer size") - fields := 0.U - state := running - } - is(15.U) { - assert(request.field < 4.U, "Invalid TLink geometry query") - answer := MuxLookup(request.field, 0.U)(Seq( - 0.U -> p.sharedBytes.U(64.W), - 1.U -> p.bankBytes.U(64.W), - 2.U -> io.tileId.pad(64), - 3.U -> p.tiles.U(64.W) - )) - } - } - } - when(io.completion.fire) { - assert(!io.completion.bits, "TLink transfer failed") - state := responding - } - when(io.reply.fire)(state := idle) -} diff --git a/arch/src/main/scala/framework/system/tile/tlink/Fabric.scala b/arch/src/main/scala/framework/system/tile/tlink/Fabric.scala deleted file mode 100644 index 5ad7d0088..000000000 --- a/arch/src/main/scala/framework/system/tile/tlink/Fabric.scala +++ /dev/null @@ -1,80 +0,0 @@ -package framework.system.tile.tlink - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import memcore.bus.axi4 - -/** Tile-addressed shared-storage fabric, separate from the DDR and CPU coherence paths. */ -@instantiable -class Fabric(p: axi4.Params, tileIds: Seq[Int], sharedBytes: BigInt) extends Module { - require(tileIds.nonEmpty && tileIds.distinct.size == tileIds.size && tileIds.forall(_ >= 0)) - require(p.dataBits == 128 && sharedBytes >= p.bytes) - val localBits = log2Ceil(sharedBytes) - val indexBits = math.max(1, log2Ceil(tileIds.size)) - require(p.addressBits >= localBits + log2Ceil(tileIds.max + 1)) - - @public val io = IO(new Bundle { - val source = Vec(tileIds.size, Flipped(new axi4.Port(p))) - val target = Vec(tileIds.size, new axi4.Port(p)) - }) - - val targets = tileIds.indices.map(_ => Instantiate(new axi4.Interconnect(p, tileIds.size))) - for (i <- tileIds.indices) { io.target(i) <> targets(i).io.out } - - for (source <- tileIds.indices) { - val port = io.source(source) - val reading = RegInit(false.B) - val writing = RegInit(false.B) - val readTarget = Reg(UInt(indexBits.W)) - val writeTarget = Reg(UInt(indexBits.W)) - val readMatches = VecInit(tileIds.map(id => (port.ar.bits.addr >> localBits) === id.U)) - val writeMatches = VecInit(tileIds.map(id => (port.aw.bits.addr >> localBits) === id.U)) - val readSelection = PriorityEncoder(readMatches) - val writeSelection = PriorityEncoder(writeMatches) - - def check(address: axi4.Address, matched: Bool): Unit = { - val offset = address.addr(localBits - 1, 0) - val bytes = (address.len +& 1.U) << log2Ceil(p.bytes) - assert(matched, "TLink address does not select a shared-storage tile") - assert( - address.size === log2Ceil(p.bytes).U && address.burst === 1.U && !address.lock, - "TLink requires full-width INCR bursts" - ) - assert( - offset(log2Ceil(p.bytes) - 1, 0) === 0.U && (offset +& bytes) <= sharedBytes.U, - "TLink burst exceeds shared storage or is unaligned" - ) - assert((offset.pad(12)(11, 0) +& bytes) <= 4096.U, "TLink burst crosses a 4KiB boundary") - } - when(port.ar.valid && !reading)(check(port.ar.bits, readMatches.asUInt.orR)) - when(port.aw.valid && !writing)(check(port.aw.bits, writeMatches.asUInt.orR)) - when(port.ar.fire) { reading := true.B; readTarget := readSelection } - when(port.r.fire && port.r.bits.last)(reading := false.B) - when(port.aw.fire) { writing := true.B; writeTarget := writeSelection } - when(port.b.fire)(writing := false.B) - - port.ar.ready := !reading && readMatches.asUInt.orR && VecInit(targets.map(_.io.in(source).ar.ready))(readSelection) - port.aw.ready := !writing && writeMatches.asUInt.orR && VecInit(targets.map(_.io.in(source).aw.ready))( - writeSelection - ) - port.w.ready := writing && VecInit(targets.map(_.io.in(source).w.ready))(writeTarget) - port.r.valid := reading && VecInit(targets.map(_.io.in(source).r.valid))(readTarget) - port.r.bits := VecInit(targets.map(_.io.in(source).r.bits))(readTarget) - port.b.valid := writing && VecInit(targets.map(_.io.in(source).b.valid))(writeTarget) - port.b.bits := VecInit(targets.map(_.io.in(source).b.bits))(writeTarget) - for (target <- tileIds.indices) { - val route = targets(target).io.in(source) - route.ar.valid := port.ar.valid && !reading && readMatches(target) - route.ar.bits := port.ar.bits - route.ar.bits.addr := port.ar.bits.addr(localBits - 1, 0) - route.aw.valid := port.aw.valid && !writing && writeMatches(target) - route.aw.bits := port.aw.bits - route.aw.bits.addr := port.aw.bits.addr(localBits - 1, 0) - route.w.valid := port.w.valid && writing && writeTarget === target.U - route.w.bits := port.w.bits - route.r.ready := port.r.ready && reading && readTarget === target.U - route.b.ready := port.b.ready && writing && writeTarget === target.U - } - } -} diff --git a/arch/src/main/scala/framework/system/tile/tlink/T2T.scala b/arch/src/main/scala/framework/system/tile/tlink/T2T.scala deleted file mode 100644 index 05b5b0177..000000000 --- a/arch/src/main/scala/framework/system/tile/tlink/T2T.scala +++ /dev/null @@ -1,127 +0,0 @@ -package framework.system.tile.tlink - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import framework.memdomain.backend.shared.SharedPhysicalPort -import memcore.bus.axi4 -import memcore.memory.queue.Queue - -case class T2TParams( - sharedBytes: BigInt, - bankBytes: Int, - tiles: Int, - axi: axi4.Params) { - require(sharedBytes > 0 && bankBytes >= 16 && isPow2(bankBytes) && sharedBytes % bankBytes == 0) - require(tiles > 0 && axi.dataBits == 128) - val sharedAddressBits: Int = math.max(1, log2Ceil(sharedBytes)) - val tileBits: Int = math.max(1, log2Ceil(tiles)) - require(sharedAddressBits + tileBits <= axi.addressBits && sharedAddressBits <= 32) -} - -class T2TCommand(p: T2TParams) extends Bundle { - val sourceByteAddress = UInt(64.W) - val targetTile = UInt(p.tileBits.W) - val targetByteAddress = UInt(64.W) - val bytes = UInt(32.W) -} - -/** - * Push shared SRAM bytes to another Tile. Starts are 16-byte aligned; the final beat may be partial. - * One command is active. Bank mappings remain owned by the caller through completion. - * Reset is coordinated with the fabric and SRAM ports, not an independent transaction cancellation. - */ -@instantiable -class T2TTransfer(p: T2TParams) extends Module { - - @public val io = IO(new Bundle { - val command = Flipped(Decoupled(new T2TCommand(p))) - val completion = Decoupled(Bool()) // false = no error, after the final destination write response - val source = new SharedPhysicalPort(128) - val mem = new axi4.Port(p.axi) - }) - - val idle :: plan :: stream :: reply :: complete :: Nil = Enum(5) - val state = RegInit(idle) - val source = Reg(UInt(p.sharedAddressBits.W)) - val target = Reg(UInt(p.sharedAddressBits.W)) - val tile = Reg(UInt(p.tileBits.W)) - val remaining = Reg(UInt((p.sharedAddressBits + 1).W)) - val burstBytes = Reg(UInt(13.W)) - val beats = Reg(UInt(9.W)) - val issued = RegInit(0.U(9.W)) - val sent = RegInit(0.U(9.W)) - val pending = RegInit(false.B) - val awSent = RegInit(false.B) - val data = Instantiate(new Queue(UInt(128.W), 2)) - val live = !reset.asBool - io.command.ready := state === idle && live - io.completion.valid := state === complete && live - io.completion.bits := false.B - when(io.completion.fire)(state := idle) - when(io.command.fire) { - val cmd = io.command.bits - assert(cmd.bytes =/= 0.U && cmd.sourceByteAddress(3, 0) === 0.U && cmd.targetByteAddress(3, 0) === 0.U) - assert(cmd.targetTile < p.tiles.U, "T2T target must be a Tile system ID") - assert( - cmd.sourceByteAddress +& cmd.bytes <= p.sharedBytes.U && cmd.targetByteAddress +& cmd.bytes <= p.sharedBytes.U, - "T2T command exceeds Tile shared SRAM" - ) - source := cmd.sourceByteAddress; target := cmd.targetByteAddress - tile := cmd.targetTile; remaining := cmd.bytes; state := plan - } - def minimum(a: UInt, b: UInt): UInt = Mux(a < b, a, b) - val sourcePage = 4096.U(13.W) - source.pad(12)(11, 0) - val targetPage = 4096.U(13.W) - target.pad(12)(11, 0) - val sourceBank = p.bankBytes.U - (source & (p.bankBytes - 1).U) - val targetBank = p.bankBytes.U - (target & (p.bankBytes - 1).U) - val nextBytes = Seq(remaining, sourcePage, targetPage, sourceBank, targetBank, 4096.U).reduce(minimum) - when(state === plan) { - burstBytes := nextBytes; beats := (nextBytes +& 15.U) >> 4 - issued := 0.U; sent := 0.U; pending := false.B; awSent := false.B; state := stream - assert(data.io.count === 0.U && nextBytes =/= 0.U) - } - io.mem.aw.valid := state === stream && !awSent && live - io.mem.aw.bits := 0.U.asTypeOf(new axi4.Address(p.axi)) - io.mem.aw.bits.addr := Cat(tile, target(p.sharedAddressBits - 1, 0)) - io.mem.aw.bits.len := beats - 1.U - io.mem.aw.bits.size := 4.U - io.mem.aw.bits.burst := 1.U - when(io.mem.aw.fire)(awSent := true.B) - io.source.request.valid := state === stream && issued < beats && !pending && data.io.count < 2.U && live - io.source.request.bits.address := source + (issued << 4) - io.source.request.bits.write := false.B - io.source.request.bits.data := 0.U - io.source.request.bits.mask := 0.U - when(io.source.request.fire) { pending := true.B; issued := issued + 1.U } - data.io.enq.valid := state === stream && pending && io.source.response.valid && !io.source.response.bits.error && live - data.io.enq.bits := io.source.response.bits.data - io.source.response.ready := state === stream && pending && data.io.enq.ready && live - when(io.source.response.valid && live) { - assert(pending && !io.source.response.bits.error, "T2T shared source read failed") - } - when(io.source.response.fire)(pending := false.B) - val last = sent +& 1.U === beats - val tail = burstBytes(3, 0) - io.mem.w.valid := state === stream && awSent && data.io.deq.valid && live - io.mem.w.bits.data := data.io.deq.bits - io.mem.w.bits.strb := Mux(last && tail =/= 0.U, (1.U(17.W) << tail) - 1.U, "hffff".U) - io.mem.w.bits.last := last - data.io.deq.ready := state === stream && awSent && io.mem.w.ready && live - when(io.mem.w.fire) { sent := sent + 1.U; when(last)(state := reply) } - io.mem.b.ready := state === reply && live - when(io.mem.b.fire) { - assert(io.mem.b.bits.id === 0.U && io.mem.b.bits.resp === 0.U, "T2T destination write failed") - assert(!pending && data.io.count === 0.U && issued === beats && sent === beats) - when(io.mem.b.bits.id === 0.U && io.mem.b.bits.resp === 0.U) { - when(remaining === burstBytes)(state := complete).otherwise { - source := source + burstBytes; target := target + burstBytes - remaining := remaining - burstBytes; state := plan - } - } - } - io.mem.ar.valid := false.B - io.mem.ar.bits := 0.U.asTypeOf(new axi4.Address(p.axi)) - io.mem.r.ready := false.B - when(live)(assert(!io.mem.r.valid, "T2T push engine received an unsolicited AXI read response")) -} diff --git a/arch/src/main/scala/framework/system/tile/tlink/tlink.scala b/arch/src/main/scala/framework/system/tile/tlink/tlink.scala deleted file mode 100644 index 203f52eae..000000000 --- a/arch/src/main/scala/framework/system/tile/tlink/tlink.scala +++ /dev/null @@ -1,54 +0,0 @@ -package framework.system.tile.tlink - -import chisel3._ -import chisel3.util._ -import hier.tile.memory.CoreInterrupts -import framework.memdomain.frontend.mem.dma.DmaStatus -import memcore.bus.{axi4, chi} -import memcore.bus.chi.RequesterPort -import memcore.bus.chi.snf.{LineRequest, LineResponse} -import memcore.memory.cpu.{CpuMemParams, UncachedRequest, UncachedResponse} - -class SharedPort(p: axi4.Params) extends Bundle { - val tx = new axi4.Port(p) - val rx = Flipped(new axi4.Port(p)) -} - -/** Tile boundary: coherent CPU control traffic and one independent DDR data port. */ -class TLinkIO( - main: Boolean, - cpus: Int, - executions: Int, - controls: Int, - bus: chi.Params, - axi: axi4.Params, - sharedStorage: Boolean) - extends Bundle { - val cp = CpuMemParams(bus, tagBits = 6) - val tileId = Input(UInt(32.W)) - val t2t = Option.when(sharedStorage)(new SharedPort(axi)) - val hartIds = Input(Vec(cpus, UInt(64.W))) - val executionIds = Input(Vec(executions, UInt(64.W))) - val resetVector = Input(Vec(cpus, UInt(64.W))) - val time = Input(UInt(64.W)) - val interrupts = Input(Vec(cpus, new CoreInterrupts)) - val uncachedRequest = Vec(cpus, Decoupled(new UncachedRequest(cp))) - val uncachedResponse = Vec(cpus, Flipped(Decoupled(new UncachedResponse(cp)))) - val control = Vec(if (main) 0 else 2 * cpus, new RequesterPort(bus)) - val remoteControl = Vec(if (main) 2 * (controls - cpus) else 0, Flipped(new RequesterPort(bus))) - val backingRequest = Vec(if (main) 1 else 0, Decoupled(new LineRequest(bus))) - val backingResponse = Vec(if (main) 1 else 0, Flipped(Decoupled(new LineResponse(bus)))) - val mem = new axi4.Port(axi) - val failure = Output(Vec(cpus, Valid(new DmaStatus))) - val workDrained = Output(Vec(cpus, Bool())) - val retired = Output(Vec(cpus, Bool())) - val retiredPc = Output(Vec(cpus, UInt(64.W))) - val trapped = Output(Vec(cpus, Bool())) - val trapCause = Output(Vec(cpus, UInt(64.W))) - val trapValue = Output(Vec(cpus, UInt(64.W))) - val trapPc = Output(Vec(cpus, UInt(64.W))) -} - -trait HasTLink { - def tlink: TLinkIO -} diff --git a/arch/src/main/scala/framework/thread/ant/build.sc b/arch/src/main/scala/framework/thread/ant/build.sc new file mode 100644 index 000000000..d06fa8611 --- /dev/null +++ b/arch/src/main/scala/framework/thread/ant/build.sc @@ -0,0 +1,25 @@ +import mill._ +import $file.^.^.^.^.^.^.settings + + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.thread.ant.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("configs/AntParam.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Ant.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Decode.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Executor.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/LocalStore.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/thread/ant/configs/AntParam.scala b/arch/src/main/scala/framework/thread/ant/configs/AntParam.scala new file mode 100644 index 000000000..8b8f53043 --- /dev/null +++ b/arch/src/main/scala/framework/thread/ant/configs/AntParam.scala @@ -0,0 +1,35 @@ +package framework.thread.ant.configs + +import buckyball.config.AntConfig +import chisel3.util.isPow2 +import framework.ip.memory.spm +import upickle.default.{macroRW, ReadWriter} + +case class AntParam( + codeBytes: Int, + data: spm.Params, + shared: spm.Params, + taskBits: Int) { + require(codeBytes >= 32 && isPow2(codeBytes)) + require(data.base >= codeBytes && shared.base >= codeBytes) + require(data.base + data.bytes <= shared.base || shared.base + shared.bytes <= data.base) + require(data.dataBits == shared.dataBits) + require(taskBits > 0) +} + +object AntParam { + + def fromProto(p: AntConfig, shared: spm.Params): AntParam = { + require(p.hasTls, "Ant TLS configuration is required") + val tls = p.getTls + AntParam( + p.getCodeBytes, + spm.Params(BigInt(java.lang.Long.toUnsignedString(tls.getBase)), tls.getBytes, tls.getDataBits), + shared, + p.getTaskBits + ) + } + + implicit val spmRW: ReadWriter[spm.Params] = macroRW + implicit val rw: ReadWriter[AntParam] = macroRW +} diff --git a/arch/src/main/scala/framework/thread/ant/configs/default.toml b/arch/src/main/scala/framework/thread/ant/configs/default.toml new file mode 100644 index 000000000..dd065ba9d --- /dev/null +++ b/arch/src/main/scala/framework/thread/ant/configs/default.toml @@ -0,0 +1,7 @@ +codeBytes = 0x4000 +taskBits = 32 + +[tls] +base = 0x10000 +bytes = 0x4000 +dataBits = 128 diff --git a/arch/src/main/scala/framework/system/core/ant/src/csrc/Cargo.lock b/arch/src/main/scala/framework/thread/ant/src/csrc/Cargo.lock similarity index 100% rename from arch/src/main/scala/framework/system/core/ant/src/csrc/Cargo.lock rename to arch/src/main/scala/framework/thread/ant/src/csrc/Cargo.lock diff --git a/arch/src/main/scala/framework/system/core/ant/src/csrc/Cargo.toml b/arch/src/main/scala/framework/thread/ant/src/csrc/Cargo.toml similarity index 100% rename from arch/src/main/scala/framework/system/core/ant/src/csrc/Cargo.toml rename to arch/src/main/scala/framework/thread/ant/src/csrc/Cargo.toml diff --git a/arch/src/main/scala/framework/thread/ant/src/csrc/src/lib.rs b/arch/src/main/scala/framework/thread/ant/src/csrc/src/lib.rs new file mode 100644 index 000000000..e85ae86dc --- /dev/null +++ b/arch/src/main/scala/framework/thread/ant/src/csrc/src/lib.rs @@ -0,0 +1,20 @@ +#[path = "../../../../../ip/spm/src/csrc/src/lib.rs"] +mod storage; + +#[no_mangle] +pub extern "C" fn ant_arithmetic(a: u64, b: u64, operation: u32, word: bool) -> u64 { + let (a, b) = if word { (a as u32 as u64, b as u32 as u64) } else { (a, b) }; + let signed = |x: u64| if word { x as u32 as i32 as i64 } else { x as i64 }; + let value = match operation { + 0 => a.wrapping_mul(b), + 1 => (((a as i64 as i128) * (b as i64 as i128)) >> 64) as u64, + 2 => (((a as i64 as i128) * (b as i128)) >> 64) as u64, + 3 => (((a as u128) * (b as u128)) >> 64) as u64, + 4 => if b == 0 { u64::MAX } else { signed(a).wrapping_div(signed(b)) as u64 }, + 5 => if b == 0 { u64::MAX } else { a / b }, + 6 => if b == 0 { a } else { signed(a).wrapping_rem(signed(b)) as u64 }, + 7 => if b == 0 { a } else { a % b }, + _ => unreachable!(), + }; + if word { value as u32 as i32 as i64 as u64 } else { value } +} diff --git a/arch/src/main/scala/framework/system/core/ant/src/main/resources/ant.cm_hier b/arch/src/main/scala/framework/thread/ant/src/main/resources/ant.cm_hier similarity index 100% rename from arch/src/main/scala/framework/system/core/ant/src/main/resources/ant.cm_hier rename to arch/src/main/scala/framework/thread/ant/src/main/resources/ant.cm_hier diff --git a/arch/src/main/scala/framework/thread/ant/src/main/resources/ant.f b/arch/src/main/scala/framework/thread/ant/src/main/resources/ant.f new file mode 100644 index 000000000..b27fde31e --- /dev/null +++ b/arch/src/main/scala/framework/thread/ant/src/main/resources/ant.f @@ -0,0 +1,10 @@ ++incdir+@VERIFY@/uvm/src/core ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/core/scenario_if.sv +@VERIFY@/uvm/src/core/scenario_package.sv +@RESOURCES@/../../../../../ip/spm/src/main/resources/package.sv +@RESOURCES@/../../../../../ip/spm/src/main/resources/interface.sv ++incdir+@RTL@ +-F @RTL@/Ant/filelist.f +@RESOURCES@/ant_tb.sv diff --git a/arch/src/main/scala/framework/thread/ant/src/main/resources/ant_tb.sv b/arch/src/main/scala/framework/thread/ant/src/main/resources/ant_tb.sv new file mode 100644 index 000000000..e62ab19ae --- /dev/null +++ b/arch/src/main/scala/framework/thread/ant/src/main/resources/ant_tb.sv @@ -0,0 +1,315 @@ +`include "config.svh" +module ant_tb; + import uvm_pkg::*; + import verification_pkg::*; + import spm_pkg::*; + import scenario_pkg::*; + `include "uvm_macros.svh" + import "DPI-C" function longint unsigned ant_arithmetic( + longint unsigned a, + b, + int unsigned operation, + bit word_mode + ); + logic clock = 0, reset = 1, in_use = 0; + always #5 clock = ~clock; + scenario_if control (clock); + spm_if host ( + clock, + reset + ); + logic [1:0] start_valid = 0, start_ready, result_valid, result_ready = 0; + logic [1:0] cancel = 0, drained = '1, was_cancelled, running; + logic [31:0] task_id[2] = '{1, 2}, result_task[2]; + logic [63:0] value[2]; + logic [63:0] entry[2] = '{0, 0}, code_end[2] = '{16, 16}; + logic [1:0] command_valid, command_ready = '1, retired_valid; + logic [31:0] command_instruction[2]; + logic [63:0] command_a[2], command_b[2], retired_pc[2]; + logic [1:0] transfer_valid = 0, transfer_ready, transferred_valid, transferred_ready = 0; + logic [1:0] transfer_code = 0, transfer_to_local = 0, transferred_result; + logic [31:0] transfer_local[2], transfer_shared[2], transfer_bytes[2]; + int checks = 0; + AntVerification dut ( + .clock(clock), + .reset(reset), + .io_inUse(in_use), + .io_rocc_0_cmd_valid(command_valid[0]), + .io_rocc_0_cmd_ready(command_ready[0]), + .io_rocc_0_cmd_bits_instruction_raw_inst(command_instruction[0]), + .io_rocc_0_cmd_bits_instruction_rs1Data(command_a[0]), + .io_rocc_0_cmd_bits_instruction_rs2Data(command_b[0]), + .io_rocc_0_busy(!drained[0]), + .io_rocc_0_interrupt(1'b0), + .io_rocc_0_resp_valid(1'b0), + .io_rocc_0_resp_bits_rd(5'd0), + .io_rocc_0_resp_bits_data(64'd0), + .io_retired_0_valid(retired_valid[0]), + .io_retired_0_bits_pc(retired_pc[0]), + .io_rocc_1_cmd_valid(command_valid[1]), + .io_rocc_1_cmd_ready(command_ready[1]), + .io_rocc_1_cmd_bits_instruction_raw_inst(command_instruction[1]), + .io_rocc_1_cmd_bits_instruction_rs1Data(command_a[1]), + .io_rocc_1_cmd_bits_instruction_rs2Data(command_b[1]), + .io_rocc_1_busy(!drained[1]), + .io_rocc_1_interrupt(1'b0), + .io_rocc_1_resp_valid(1'b0), + .io_rocc_1_resp_bits_rd(5'd0), + .io_rocc_1_resp_bits_data(64'd0), + .io_retired_1_valid(retired_valid[1]), + .io_retired_1_bits_pc(retired_pc[1]), + .io_start_0_valid(start_valid[0]), + .io_start_0_ready(start_ready[0]), + .io_start_0_bits_task(task_id[0]), + .io_start_0_bits_entry(entry[0]), + .io_start_0_bits_codeEnd(code_end[0]), + .io_start_0_bits_argument(`TLS_BASE), + .io_start_0_bits_stack(`TLS_END), + .io_result_0_valid(result_valid[0]), + .io_result_0_ready(result_ready[0]), + .io_result_0_bits_task(result_task[0]), + .io_result_0_bits_cancelled(was_cancelled[0]), + .io_result_0_bits_value(value[0]), + .io_cancel_0(cancel[0]), + .io_running_0(running[0]), + .io_transfer_0_valid(transfer_valid[0]), + .io_transfer_0_ready(transfer_ready[0]), + .io_transfer_0_bits_code(transfer_code[0]), + .io_transfer_0_bits_toLocal(transfer_to_local[0]), + .io_transfer_0_bits_localOffset(transfer_local[0]), + .io_transfer_0_bits_sharedOffset(transfer_shared[0]), + .io_transfer_0_bits_bytes(transfer_bytes[0]), + .io_transferred_0_valid(transferred_valid[0]), + .io_transferred_0_ready(transferred_ready[0]), + .io_transferred_0_bits(transferred_result[0]), + .io_start_1_valid(start_valid[1]), + .io_start_1_ready(start_ready[1]), + .io_start_1_bits_task(task_id[1]), + .io_start_1_bits_entry(entry[1]), + .io_start_1_bits_codeEnd(code_end[1]), + .io_start_1_bits_argument(`TLS_BASE), + .io_start_1_bits_stack(`TLS_END), + .io_result_1_valid(result_valid[1]), + .io_result_1_ready(result_ready[1]), + .io_result_1_bits_task(result_task[1]), + .io_result_1_bits_cancelled(was_cancelled[1]), + .io_result_1_bits_value(value[1]), + .io_cancel_1(cancel[1]), + .io_running_1(running[1]), + .io_transfer_1_valid(transfer_valid[1]), + .io_transfer_1_ready(transfer_ready[1]), + .io_transfer_1_bits_code(transfer_code[1]), + .io_transfer_1_bits_toLocal(transfer_to_local[1]), + .io_transfer_1_bits_localOffset(transfer_local[1]), + .io_transfer_1_bits_sharedOffset(transfer_shared[1]), + .io_transfer_1_bits_bytes(transfer_bytes[1]), + .io_transferred_1_valid(transferred_valid[1]), + .io_transferred_1_ready(transferred_ready[1]), + .io_transferred_1_bits(transferred_result[1]), + .io_host_request_valid(host.valid), + .io_host_request_ready(host.ready), + .io_host_request_bits_address(host.address), + .io_host_request_bits_size(host.size), + .io_host_request_bits_write(host.write), + .io_host_request_bits_data(host.data), + .io_host_request_bits_mask(host.mask), + .io_host_response_valid(host.response_valid), + .io_host_response_ready(host.response_ready), + .io_host_response_bits_data(host.result), + .io_host_response_bits_error(host.error) + ); + initial begin + #0; + uvm_config_db#(virtual scenario_if)::set(null, "uvm_test_top", "vif", control); + uvm_config_db#(time)::set(null, "uvm_test_top", "timeout", 10ms); + run_test("protocol_test"); + end + task automatic copy_local(int ctx, bit code, bit to_local, int offset, int bytes); + @(negedge clock); + transfer_code[ctx] = code; + transfer_to_local[ctx] = to_local; + transfer_local[ctx] = offset; + transfer_shared[ctx] = `TSS_SCRATCH - `TSS_BASE; + transfer_bytes[ctx] = bytes; + transfer_valid[ctx] = 1; + do @(posedge clock); while (!transfer_ready[ctx]); + @(negedge clock); + transfer_valid[ctx] = 0; + wait (transferred_valid[ctx]); + repeat (3) begin + @(negedge clock); + if (!transferred_valid[ctx] || !transferred_result[ctx]) + `uvm_fatal("TRANSFER", "transfer completion changed under backpressure") + end + transferred_ready[ctx] = 1; + @(negedge clock); + transferred_ready[ctx] = 0; + endtask + task automatic put(int port, longint unsigned address, logic [127:0] data, bit wide = 0); + logic [127:0] word; + int ctx, offset, bytes; + if (port == 4) begin + host.access(address, wide ? 4 : 2, 1, data, wide ? 16'hffff : 16'h000f); + end else begin + ctx = port / 2; + offset = address - (port % 2 ? `TLS_BASE : 0); + bytes = wide ? 16 : 8; + word = data; + if (!wide) begin + copy_local(ctx, !(port % 2), 0, offset & ~7, 8); + host.send(`TSS_SCRATCH, 3, 0); + host.take(word); + word[(offset%8)*8+:32] = data[31:0]; + end + host.access(`TSS_SCRATCH, wide ? 4 : 3, 1, word, wide ? 16'hffff : 16'h00ff); + copy_local(ctx, !(port % 2), 1, offset & ~7, bytes); + end + if (host.error) `uvm_fatal("LOAD", "TSS access failed") + endtask + task automatic launch(int id); + @(negedge clock); + start_valid[id] = 1; + do @(posedge clock); while (!start_ready[id]); + @(negedge clock); + start_valid[id] = 0; + endtask + task automatic finish(int id, longint unsigned expected, bit cancelled = 0); + wait (result_valid[id]); + if (was_cancelled[id] !== cancelled || value[id] !== expected || result_task[id] !== task_id[id]) + `uvm_fatal("RESULT", $sformatf( + "context %0d task %0d: cancelled %b value %h expected %h", + id, + result_task[id], + was_cancelled[id], + value[id], + expected + )) + repeat (3) begin + @(negedge clock); + if (!result_valid[id] || value[id] !== expected) `uvm_fatal("HOLD", "completion changed") + end + result_ready[id] = 1; + @(negedge clock); + result_ready[id] = 0; + checks++; + endtask + initial begin : exercise + longint unsigned operands[8] = '{ + 0, + 1, + 64'hffffffffffffffff, + 64'h8000000000000000, + 64'h7fffffffffffffff, + 64'h80000000, + 64'hffffffff, + 37 + }; + logic [31:0] operation; + wait (control.start); + repeat (3) @(negedge clock); + reset = 0; + // Real RV64IM programs: LD operands, arithmetic, ECALL returning a0. + // Golden arithmetic uses independent Rust i128/u128 and defined division corners. + for (int w = 0; w < 2; w++) + for (int op = 0; op < 8; op++) + if (!w || op == 0 || op >= 4) + for (int sample = 0; sample < 64; sample ++) begin + operation = 32'h02b50533 | (op << 12) | (w << 3); + put(0, 0, {32'h00000073, operation, 32'h00053503, 32'h00853583}, 1); + put(1, `TLS_BASE, {operands[sample%8], operands[sample/8]}, 1); + in_use = 1; + launch(0); + finish(0, ant_arithmetic(operands[sample/8], operands[sample%8], op, w)); + in_use = 0; + end + // One producer and one consumer share data then publish/observe a flag. + put(4, `TSS_BASE, 0, 1); + put(0, 0, {32'h0062b023, 32'h02a00313, 32'h000202b7, 32'h00000013}, 1); + put(0, 16, {32'h02a00513, 32'h0062b423, 32'h00100313, 32'h00000013}, 1); + put(0, 32, 32'h00000073); + // Consumer loops while flag is zero, then returns shared data. + put(2, 0, {32'h0002b503, 32'hfe030ee3, 32'h0082b303, 32'h000202b7}, 1); + put(2, 16, 32'h00000073); + code_end[0] = 36; + code_end[1] = 20; + in_use = 1; + fork + launch(0); + launch(1); + join + fork + finish(0, 42); + finish(1, 42); + join + in_use = 0; + // Identical TLS addresses hold different values in the two contexts. + put(0, 0, {96'b0, 32'h00053503}, 1); + put(0, 4, 32'h00000073); + put(2, 0, {96'b0, 32'h00053503}, 1); + put(2, 4, 32'h00000073); + put(1, `TLS_BASE, 128'd17, 1); + put(3, `TLS_BASE, 128'd29, 1); + code_end[0] = 8; + code_end[1] = 8; + in_use = 1; + fork + launch(0); + launch(1); + join + fork + finish(0, 17); + finish(1, 29); + join + in_use = 0; + code_end[0] = 16; + // Raw custom instruction and opaque DDR pointer survive submission backpressure. + put(0, 0, {32'h03700513, 32'h12b5307b, 32'h00053503, 32'h00853583}, 1); + put(0, 16, 32'h00000073); + code_end[0] = 20; + put(1, `TLS_BASE, {64'hfedcba9876543210, 64'h80001000}, 1); + command_ready[0] = 0; + in_use = 1; + launch(0); + wait (command_valid[0]); + // A busy Ant must not lock the other Ant's private loader. + put(3, `TLS_BASE, 128'd91, 1); + repeat (7) begin + @(negedge clock); + if (!command_valid[0] || command_instruction[0] !== 32'h12b5307b || + command_a[0] !== 64'h80001000 || + command_b[0] !== 64'hfedcba9876543210) + `uvm_fatal("COMMAND", "raw NPU command changed under backpressure") + end + drained[0] = 0; + command_ready[0] = 1; + @(negedge clock); + repeat (20) begin + @(negedge clock); + if (result_valid[0]) `uvm_fatal("DRAIN", "task completed before NPU drained") + end + drained[0] = 1; + finish(0, 55); + in_use = 0; + code_end[0] = 16; + // Cancel an actual in-flight DIV; reusing the context must not consume its old result. + put(0, 0, {32'h00000073, 32'h02b54533, 32'h00053503, 32'h00853583}, 1); + put(1, `TLS_BASE, {64'd37, 64'h7fffffffffffffff}, 1); + in_use = 1; + launch(0); + do @(posedge clock); while (!(retired_valid[0] && retired_pc[0] == 4)); + repeat (8) @(negedge clock); + cancel[0] = 1; + @(negedge clock); + cancel[0] = 0; + finish(0, 0, 1); + in_use = 0; + put(0, 0, {64'b0, 32'h00000073, 32'h00b00513}, 1); + code_end[0] = 8; + in_use = 1; + launch(0); + finish(0, 11); + in_use = 0; + `uvm_info("ANT", $sformatf("%0d task completions checked", checks), UVM_LOW) + control.done = 1; + end +endmodule diff --git a/arch/src/main/scala/framework/system/core/ant/src/main/resources/control.cm_hier b/arch/src/main/scala/framework/thread/ant/src/main/resources/control.cm_hier similarity index 100% rename from arch/src/main/scala/framework/system/core/ant/src/main/resources/control.cm_hier rename to arch/src/main/scala/framework/thread/ant/src/main/resources/control.cm_hier diff --git a/arch/src/main/scala/framework/thread/ant/src/main/resources/control.f b/arch/src/main/scala/framework/thread/ant/src/main/resources/control.f new file mode 100644 index 000000000..8e955d5fa --- /dev/null +++ b/arch/src/main/scala/framework/thread/ant/src/main/resources/control.f @@ -0,0 +1,9 @@ ++incdir+@VERIFY@/uvm/src/core ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/core/scenario_if.sv +@VERIFY@/uvm/src/core/scenario_package.sv ++incdir+@RTL@ +-F @RTL@/Control/filelist.f +@RESOURCES@/scenario_if.sv +@RESOURCES@/control_tb.sv diff --git a/arch/src/main/scala/framework/system/core/ant/src/main/resources/control_if.sv b/arch/src/main/scala/framework/thread/ant/src/main/resources/control_if.sv similarity index 100% rename from arch/src/main/scala/framework/system/core/ant/src/main/resources/control_if.sv rename to arch/src/main/scala/framework/thread/ant/src/main/resources/control_if.sv diff --git a/arch/src/main/scala/framework/thread/ant/src/main/resources/control_tb.sv b/arch/src/main/scala/framework/thread/ant/src/main/resources/control_tb.sv new file mode 100644 index 000000000..f2e395c99 --- /dev/null +++ b/arch/src/main/scala/framework/thread/ant/src/main/resources/control_tb.sv @@ -0,0 +1,176 @@ +`include "config.svh" +module control_tb; + import uvm_pkg::*; + import scenario_pkg::*; + `include "uvm_macros.svh" + logic clock = 0, reset = 1; + always #5 clock = ~clock; + scenario_if control (clock); + ant_control_if management ( + clock, + reset + ); + logic [1:0] start_valid, start_ready = '1, result_valid = 0, result_ready, cancel; + logic storage_busy = 0, in_use; + logic [31:0] task_id[2], result_task[2] = '{7, 8}; + logic [63:0] entry[2], code_end[2], argument[2], stack[2]; + logic [63:0] signature[2]; + logic [63:0] request_satp = 64'h8000000000000bad, launch_satp[2]; + logic [1:0] request_privilege = 0, launch_privilege[2]; + logic request_sum = 0, request_mxr = 0; + logic [1:0] launch_sum, launch_mxr; + logic [63:0] expected_satp[2] = '{64'h8000000000000123, 64'h8000000000000456}; + logic [1:0] expected_privilege[2] = '{2'd1, 2'd0}; + logic [1:0] expected_sum = 2'b01, expected_mxr = 2'b10; + int checked = 0, launches = 0; + Control dut ( + .clock(clock), + .reset(reset), + .io_request_valid(management.valid), + .io_request_ready(management.ready), + .io_request_bits_control_operation(management.operation), + .io_request_bits_control_context(management.context_id), + .io_request_bits_control_field(management.field), + .io_request_bits_control_data(management.data), + .io_request_bits_context_satp(request_satp), + .io_request_bits_context_effectivePrivilege(request_privilege), + .io_request_bits_context_sum(request_sum), + .io_request_bits_context_mxr(request_mxr), + .io_response_valid(management.reply_valid), + .io_response_ready(management.reply_ready), + .io_response_bits(management.reply), + .io_storageBusy(storage_busy), + .io_inUse(in_use), + .io_cancel_0(cancel[0]), + .io_start_0_valid(start_valid[0]), + .io_start_0_ready(start_ready[0]), + .io_start_0_bits_descriptor_task(task_id[0]), + .io_start_0_bits_descriptor_entry(entry[0]), + .io_start_0_bits_descriptor_codeEnd(code_end[0]), + .io_start_0_bits_descriptor_argument(argument[0]), + .io_start_0_bits_descriptor_stack(stack[0]), + .io_start_0_bits_context_satp(launch_satp[0]), + .io_start_0_bits_context_effectivePrivilege(launch_privilege[0]), + .io_start_0_bits_context_sum(launch_sum[0]), + .io_start_0_bits_context_mxr(launch_mxr[0]), + .io_result_0_valid(result_valid[0]), + .io_result_0_ready(result_ready[0]), + .io_result_0_bits_task(result_task[0]), + .io_result_0_bits_cancelled(1'b0), + .io_result_0_bits_value(64'h42), + .io_cancel_1(cancel[1]), + .io_start_1_valid(start_valid[1]), + .io_start_1_ready(start_ready[1]), + .io_start_1_bits_descriptor_task(task_id[1]), + .io_start_1_bits_descriptor_entry(entry[1]), + .io_start_1_bits_descriptor_codeEnd(code_end[1]), + .io_start_1_bits_descriptor_argument(argument[1]), + .io_start_1_bits_descriptor_stack(stack[1]), + .io_start_1_bits_context_satp(launch_satp[1]), + .io_start_1_bits_context_effectivePrivilege(launch_privilege[1]), + .io_start_1_bits_context_sum(launch_sum[1]), + .io_start_1_bits_context_mxr(launch_mxr[1]), + .io_result_1_valid(result_valid[1]), + .io_result_1_ready(result_ready[1]), + .io_result_1_bits_task(result_task[1]), + .io_result_1_bits_cancelled(1'b0), + .io_result_1_bits_value(64'h42) + `include "control_pmp.svh" + ); + always @(posedge clock) + if (!reset) begin + for (int i = 0; i < 2; i++) + if (start_valid[i] && start_ready[i]) begin + if(task_id[i] !== 7+i || entry[i] !== 0 || code_end[i] !== 16 || + argument[i] !== `TLS_BASE || stack[i] !== `TLS_END) + `uvm_fatal("START", "local descriptor changed") + if (launch_satp[i] !== expected_satp[i] || launch_privilege[i] !== expected_privilege[i] || + launch_sum[i] !== expected_sum[i] || launch_mxr[i] !== expected_mxr[i]) + `uvm_fatal("CONTEXT", "launch context does not belong to the accepted start command") + launches++; + end + end + initial begin + uvm_config_db#(virtual scenario_if)::set(null, "uvm_test_top", "vif", control); + uvm_config_db#(time)::set(null, "uvm_test_top", "timeout", 100us); + run_test("protocol_test"); + end + task automatic transact(int op, int ctx, int f, longint unsigned value, + longint unsigned expected = 0); + longint unsigned actual; + management.transfer(op, ctx, f, value, actual); + if (actual !== expected) + `uvm_fatal("CONTROL", $sformatf("data %h expected %h", actual, expected)) + checked++; + endtask + task automatic descriptor(int ctx); + transact(1, ctx, 0, 7 + ctx); + transact(1, ctx, 1, 0); + transact(1, ctx, 2, 16); + transact(1, ctx, 3, `TLS_BASE); + transact(1, ctx, 4, `TLS_END); + transact(1, ctx, 5, signature[ctx]); + endtask + initial begin + wait (control.start); + repeat (3) @(negedge clock); + reset = 0; + transact(0, 0, 7, 0, 2); + management.transfer(0, 0, 0, 0, signature[0]); + management.transfer(0, 1, 0, 0, signature[1]); + transact(0, 0, 1, 0, 1); + transact(0, 0, 2, 0, `CODE_BYTES); + transact(0, 0, 3, 0, `TLS_BASE); + transact(0, 0, 4, 0, `TLS_BYTES); + transact(0, 0, 5, 0, `TSS_BASE); + transact(0, 0, 6, 0, `TSS_BYTES); + transact(5, 0, 0, 0); + descriptor(0); + transact(1, 0, 5, signature[0]); + // Staging was performed with another address context. Only Start selects the task's context. + request_satp = expected_satp[0]; + request_privilege = expected_privilege[0]; + request_sum = expected_sum[0]; + request_mxr = expected_mxr[0]; + start_ready[0] = 0; + fork + transact(2, 0, 0, 0); + begin + wait (start_valid[0]); + repeat (5) begin + @(negedge clock); + if (!start_valid[0] || management.ready || launches != 0) + `uvm_fatal("STALL", "launch was accepted before the thread became ready") + if (task_id[0] !== 7 || launch_satp[0] !== expected_satp[0] || + launch_privilege[0] !== expected_privilege[0] || + launch_sum[0] !== expected_sum[0] || launch_mxr[0] !== expected_mxr[0]) + `uvm_fatal("STALL", "launch descriptor and address context changed under backpressure") + end + start_ready[0] = 1; + end + join + transact(0, 0, 1, 0, 3); + descriptor(1); + request_satp = expected_satp[1]; + request_privilege = expected_privilege[1]; + request_sum = expected_sum[1]; + request_mxr = expected_mxr[1]; + transact(2, 1, 0, 0); + @(negedge clock); + result_valid = 3; + do @(posedge clock); while (result_ready != 3); + @(negedge clock); + result_valid = 0; + transact(0, 0, 1, 0, 5); + transact(0, 0, 8, 0, 7); + transact(0, 0, 9, 0, 64'h42); + transact(0, 1, 8, 0, 8); + transact(3, 0, 0, 0); + transact(3, 1, 0, 0); + transact(6, 0, 0, 0); + if (in_use || launches != 2) `uvm_fatal("GROUP", "group lifetime or launch count wrong") + `uvm_info("CONTROL", $sformatf("%0d transactions checked; %0d launches", checked, launches), + UVM_LOW) + control.done = 1; + end +endmodule diff --git a/arch/src/main/scala/framework/system/core/ant/src/main/resources/service.cm_hier b/arch/src/main/scala/framework/thread/ant/src/main/resources/service.cm_hier similarity index 100% rename from arch/src/main/scala/framework/system/core/ant/src/main/resources/service.cm_hier rename to arch/src/main/scala/framework/thread/ant/src/main/resources/service.cm_hier diff --git a/arch/src/main/scala/framework/thread/ant/src/main/resources/service.f b/arch/src/main/scala/framework/thread/ant/src/main/resources/service.f new file mode 100644 index 000000000..4a391fa4e --- /dev/null +++ b/arch/src/main/scala/framework/thread/ant/src/main/resources/service.f @@ -0,0 +1,9 @@ ++incdir+@VERIFY@/uvm/src/core ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/core/scenario_if.sv +@VERIFY@/uvm/src/core/scenario_package.sv ++incdir+@RTL@ +-F @RTL@/Service/filelist.f +@RESOURCES@/scenario_if.sv +@RESOURCES@/service_tb.sv diff --git a/arch/src/main/scala/framework/thread/ant/src/main/resources/service_tb.sv b/arch/src/main/scala/framework/thread/ant/src/main/resources/service_tb.sv new file mode 100644 index 000000000..286e24c1d --- /dev/null +++ b/arch/src/main/scala/framework/thread/ant/src/main/resources/service_tb.sv @@ -0,0 +1,153 @@ +`include "config.svh" +module service_tb; + import uvm_pkg::*; + import verification_pkg::*; + import scenario_pkg::*; + `include "uvm_macros.svh" + logic clock = 0, reset = 1; + always #5 clock = ~clock; + scenario_if control (clock); + ant_control_if management ( + clock, + reset + ); + logic [1:0] drained = '1, command_valid, command_ready = '1, retired_valid; + logic [31:0] command_instruction[2]; + logic [63:0] command_a[2], command_b[2], retired_pc[2]; + int completed = 0; + ServiceVerification dut ( + .clock(clock), + .reset(reset), + .io_rocc_0_cmd_valid(command_valid[0]), + .io_rocc_0_cmd_ready(command_ready[0]), + .io_rocc_0_cmd_bits_instruction_raw_inst(command_instruction[0]), + .io_rocc_0_cmd_bits_instruction_rs1Data(command_a[0]), + .io_rocc_0_cmd_bits_instruction_rs2Data(command_b[0]), + .io_rocc_0_busy(!drained[0]), + .io_rocc_0_interrupt(1'b0), + .io_rocc_0_resp_valid(1'b0), + .io_rocc_0_resp_bits_rd(5'd0), + .io_rocc_0_resp_bits_data(64'd0), + .io_retired_0_valid(retired_valid[0]), + .io_retired_0_bits_pc(retired_pc[0]), + .io_rocc_1_cmd_valid(command_valid[1]), + .io_rocc_1_cmd_ready(command_ready[1]), + .io_rocc_1_cmd_bits_instruction_raw_inst(command_instruction[1]), + .io_rocc_1_cmd_bits_instruction_rs1Data(command_a[1]), + .io_rocc_1_cmd_bits_instruction_rs2Data(command_b[1]), + .io_rocc_1_busy(!drained[1]), + .io_rocc_1_interrupt(1'b0), + .io_rocc_1_resp_valid(1'b0), + .io_rocc_1_resp_bits_rd(5'd0), + .io_rocc_1_resp_bits_data(64'd0), + .io_retired_1_valid(retired_valid[1]), + .io_retired_1_bits_pc(retired_pc[1]), + .io_request_valid(management.valid), + .io_request_ready(management.ready), + .io_request_bits_operation(management.operation), + .io_request_bits_context(management.context_id), + .io_request_bits_field(management.field), + .io_request_bits_data(management.data), + .io_reply_valid(management.reply_valid), + .io_reply_ready(management.reply_ready), + .io_reply_bits(management.reply) + ); + initial begin + #0; + uvm_config_db#(virtual scenario_if)::set(null, "uvm_test_top", "vif", control); + uvm_config_db#(time)::set(null, "uvm_test_top", "timeout", 100us); + run_test("protocol_test"); + end + task automatic command(int op, int ctx, int field, longint unsigned value = 0); + longint unsigned reply; + management.transfer(op, ctx, field, value, reply); + endtask + task automatic put(int port, longint unsigned address, logic [127:0] data); + longint unsigned result, offset, shared_offset, descriptor; + int space, ctx; + space = port == 4 ? 2 : port % 2; + ctx = port == 4 ? 0 : port / 2; + offset = address - (space == 0 ? 0 : space == 1 ? `TLS_BASE : `TSS_BASE); + shared_offset = space == 2 ? offset : `TSS_SCRATCH - `TSS_BASE; + for (int half = 0; half < 2; half++) + command(12, 0, shared_offset + 8 * half, data[64*half+:64]); + if (space != 2) begin + descriptor = (64'd16 << 32) | shared_offset; + command(8 + 2 * space, ctx, offset, descriptor); + command(7 + 2 * space, ctx, offset, descriptor); + end + for (int half = 0; half < 2; half++) begin + management.transfer(11, 0, shared_offset + 8 * half, 0, result); + if (result !== data[64*half+:64]) `uvm_fatal("LOAD", "TSS transfer readback mismatch") + end + endtask + task automatic launch(int ctx, int task_id); + command(1, ctx, 0, task_id); + command(1, ctx, 1, 0); + command(1, ctx, 2, 8); + command(1, ctx, 3, `TLS_BASE); + command(1, ctx, 4, `TLS_END); + command(1, ctx, 5, ctx ? `SIGNATURE_1 : `SIGNATURE_0); + command(2, ctx, 0); + endtask + task automatic finish(int ctx, int task_id, longint unsigned value, bit cancelled = 0); + longint unsigned reply, status; + do management.transfer(0, ctx, 1, 0, status); while (!(status & 4)); + if (status !== (cancelled ? 13 : 5)) `uvm_fatal("RESULT", "wrong terminal state") + management.transfer(0, ctx, 8, 0, reply); + if (reply !== task_id) `uvm_fatal("RESULT", "wrong task ID") + management.transfer(0, ctx, 9, 0, reply); + if (reply !== value) `uvm_fatal("RESULT", "wrong terminal value") + command(3, ctx, 0); + completed++; + endtask + initial begin + longint unsigned reply; + wait (control.start); + repeat (3) @(negedge clock); + reset = 0; + put(1, `TLS_BASE, 128'h1111); + put(3, `TLS_BASE, 128'h2222); + put(4, `TSS_BASE, 128'h3333); + put(0, 0, {64'b0, 32'h00000073, 32'h02500513}); + put(2, 0, {64'b0, 32'h00000073, 32'h01d00513}); + command(5, 0, 0); + launch(0, 1); + launch(1, 2); + finish(0, 1, 37); + finish(1, 2, 29); + command(6, 0, 0); + put(0, 0, {64'b0, 32'h00000073, 32'h0000307b}); + command_ready[0] = 0; + command(5, 0, 0); + launch(0, 3); + drained[0] = 0; + wait (command_valid[0]); + command(4, 0, 0); + management.transfer(0, 0, 1, 0, reply); + if (reply !== 3) `uvm_fatal("DRAIN", "task completed before external work drained") + drained[0] = 1; + finish(0, 3, 0, 1); + command(6, 0, 0); + // A new task must not inherit the old task's cancellation. + command_ready[0] = 1; + put(0, 0, {64'b0, 32'h00000073, 32'h00900513}); + command(5, 0, 0); + launch(0, 4); + finish(0, 4, 9); + command(6, 0, 0); + // Cancellation remains effective after ECALL while backend work is still pending. + command(5, 0, 0); + launch(0, 5); + drained[0] = 0; + repeat (30) @(negedge clock); + command(4, 0, 0); + drained[0] = 1; + finish(0, 5, 0, 1); + command(6, 0, 0); + `uvm_info("SERVICE", $sformatf( + "%0d real Ant tasks; %0d control transactions checked", completed, management.checked + ), UVM_LOW) + control.done = 1; + end +endmodule diff --git a/arch/src/main/scala/framework/thread/ant/src/main/resources/suite.toml b/arch/src/main/scala/framework/thread/ant/src/main/resources/suite.toml new file mode 100644 index 000000000..e5b06b45d --- /dev/null +++ b/arch/src/main/scala/framework/thread/ant/src/main/resources/suite.toml @@ -0,0 +1,71 @@ +[model] +manifest = "src/csrc/Cargo.toml" +library = "libant_ref" + +[[tests]] +functional_groups = [] +name = "ant" +top = "ant_tb" +test = "protocol_test" +filelist = "ant.f" +timeout_ns = 100000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +functional_groups = [] +name = "control" +top = "control_tb" +test = "protocol_test" +filelist = "control.f" +timeout_ns = 100000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +functional_groups = [] +name = "service" +top = "service_tb" +test = "protocol_test" +filelist = "service.f" +timeout_ns = 100000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +functional_groups = [] +name = "tls" +top = "tls_tb" +test = "protocol_test" +filelist = "tls.f" +timeout_ns = 100000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["tiles[0].dut", "tiles[1].dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/mem-core/tls/src/main/resources/tls.cm_hier b/arch/src/main/scala/framework/thread/ant/src/main/resources/tls.cm_hier similarity index 100% rename from arch/src/main/scala/framework/mem-core/tls/src/main/resources/tls.cm_hier rename to arch/src/main/scala/framework/thread/ant/src/main/resources/tls.cm_hier diff --git a/arch/src/main/scala/framework/thread/ant/src/main/resources/tls.f b/arch/src/main/scala/framework/thread/ant/src/main/resources/tls.f new file mode 100644 index 000000000..fcce13b16 --- /dev/null +++ b/arch/src/main/scala/framework/thread/ant/src/main/resources/tls.f @@ -0,0 +1,10 @@ ++incdir+@VERIFY@/uvm/src/core ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/core/scenario_if.sv +@VERIFY@/uvm/src/core/scenario_package.sv +@RESOURCES@/../../../../../ip/spm/src/main/resources/package.sv +@RESOURCES@/../../../../../ip/spm/src/main/resources/interface.sv ++incdir+@RTL@ +-F @RTL@/Store/filelist.f +@RESOURCES@/tls_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/tls/src/main/resources/tls.toml b/arch/src/main/scala/framework/thread/ant/src/main/resources/tls.toml similarity index 100% rename from arch/src/main/scala/framework/mem-core/tls/src/main/resources/tls.toml rename to arch/src/main/scala/framework/thread/ant/src/main/resources/tls.toml diff --git a/arch/src/main/scala/framework/thread/ant/src/main/resources/tls_tb.sv b/arch/src/main/scala/framework/thread/ant/src/main/resources/tls_tb.sv new file mode 100644 index 000000000..7fe8d8de1 --- /dev/null +++ b/arch/src/main/scala/framework/thread/ant/src/main/resources/tls_tb.sv @@ -0,0 +1,121 @@ +`include "config.svh" +module tls_tb; + import uvm_pkg::*; + import verification_pkg::*; + import scenario_pkg::*; + import spm_pkg::*; + `include "uvm_macros.svh" + logic clock = 0; + always #5 clock = ~clock; + logic [1:0] reset = '1, active = '0; + scenario_if control (clock); + virtual spm_if ports[4]; + chandle models[2]; + for (genvar i = 0; i < 4; i++) begin : endpoints + spm_if bus ( + clock, + reset[i/2] + ); + initial ports[i] = bus; + end + for (genvar core = 0; core < 2; core++) begin : cores + LocalStore dut ( + .clock(clock), + .reset(reset[core]), + .io_running(active[core]), + .io_cancel(endpoints[core*2+1].bus.cancel), + .io_busy(), + .io_host_request_valid(endpoints[core*2+0].bus.valid), + .io_host_request_ready(endpoints[core*2+0].bus.ready), + .io_host_request_bits_address(endpoints[core*2+0].bus.address), + .io_host_request_bits_size(endpoints[core*2+0].bus.size), + .io_host_request_bits_write(endpoints[core*2+0].bus.write), + .io_host_request_bits_data(endpoints[core*2+0].bus.data), + .io_host_request_bits_mask(endpoints[core*2+0].bus.mask), + .io_host_response_valid(endpoints[core*2+0].bus.response_valid), + .io_host_response_ready(endpoints[core*2+0].bus.response_ready), + .io_host_response_bits_data(endpoints[core*2+0].bus.result), + .io_host_response_bits_error(endpoints[core*2+0].bus.error), + .io_local_request_valid(endpoints[core*2+1].bus.valid), + .io_local_request_ready(endpoints[core*2+1].bus.ready), + .io_local_request_bits_address(endpoints[core*2+1].bus.address), + .io_local_request_bits_size(endpoints[core*2+1].bus.size), + .io_local_request_bits_write(endpoints[core*2+1].bus.write), + .io_local_request_bits_data(endpoints[core*2+1].bus.data), + .io_local_request_bits_mask(endpoints[core*2+1].bus.mask), + .io_local_response_valid(endpoints[core*2+1].bus.response_valid), + .io_local_response_ready(endpoints[core*2+1].bus.response_ready), + .io_local_response_bits_data(endpoints[core*2+1].bus.result), + .io_local_response_bits_error(endpoints[core*2+1].bus.error) + ); + end + initial begin + foreach (models[core]) models[core] = spm_create(`LOCAL_BASE, `LOCAL_BYTES); + #0; + foreach (ports[i]) begin + ports[i].model = models[i/2]; + ports[i].scoreboard = new($sformatf("port_%0d", i), null); + ports[i].enabled = 1; + end + uvm_config_db#(virtual scenario_if)::set(null, "uvm_test_top", "vif", control); + uvm_config_db#(time)::set(null, "uvm_test_top", "timeout", 100us); + run_test("protocol_test"); + end + final foreach (models[i]) spm_destroy(models[i]); + initial begin : exercise + logic [127:0] value; + wait (control.start); + repeat (3) @(negedge clock); + reset = 0; + for (int row = 0; row < `LOCAL_BYTES; row += 16) begin + ports[0].access(`LOCAL_BASE + row, 4, 1, 0, '1); + ports[2].access(`LOCAL_BASE + row, 4, 1, '1, '1); + end + @(negedge clock); + active = 3; + ports[1].access(`LOCAL_BASE, 3, 1, 64'h123456789abcdef0, 16'hff); + ports[3].access(`LOCAL_BASE, 4, 0); + ports[1].access(`LOCAL_BASE + 15, 0, 1, 8'ha5, 1); + ports[1].access(`LOCAL_BASE, 4, 0); + ports[1].send(`LOCAL_BASE, 4, 0); + wait (ports[1].response_valid); + repeat (5) @(negedge clock); + ports[3].access(`LOCAL_BASE + 16, 4, 0); + ports[1].take(value); + ports[1].access(`LOCAL_BASE + 3, 2, 0); + ports[1].access(`LOCAL_BASE + `LOCAL_BYTES, 3, 0); + ports[1].access(64'h80000000, 3, 0); + ports[1].access(`LOCAL_BASE, 5, 0); + ports[1].access(`LOCAL_BASE + 8, 3, 1, 64'h1122334455667788, 16'ha5); + ports[1].access(`LOCAL_BASE + 8, 3, 0); + ports[1].access((64'h1 << 32) | `LOCAL_BASE, 3, 0); + ports[1].access(64'hfffffffffffffff8, 3, 0); + ports[1].access(`LOCAL_BASE, 0, 1, '1, 3); + ports[1].send(`LOCAL_BASE, 3, 0); + ports[1].cancel = 1; + @(negedge clock); + ports[1].cancel = 0; + repeat (5) begin + @(negedge clock); + if (ports[1].response_valid) `uvm_fatal("CANCEL", "old TLS response survived cancellation") + end + ports[1].access(`LOCAL_BASE, 3, 0); + ports[1].send(`LOCAL_BASE + 16, 0, 1, 8'h5a, 1); + wait (ports[1].response_valid); + @(negedge clock); + reset[0] = 1; + active[0] = 0; + repeat (2) @(negedge clock); + reset[0] = 0; + ports[0].access(`LOCAL_BASE + 16, 0, 0); + ports[3].access(`LOCAL_BASE + 16, 0, 0); + repeat (2) @(negedge clock); + foreach (ports[i]) begin + if (ports[i].scoreboard.pending() || ports[i].scoreboard.pending()) + `uvm_fatal("DRAIN", "local storage still has unmatched accesses") + `uvm_info("CHECKED", $sformatf("port %0d: %0d comparisons", i, ports[i].scoreboard.checked), + UVM_LOW) + end + control.done = 1; + end +endmodule diff --git a/arch/src/main/scala/framework/thread/ant/src/main/scala/Ant.scala b/arch/src/main/scala/framework/thread/ant/src/main/scala/Ant.scala new file mode 100644 index 000000000..9f7f73d9d --- /dev/null +++ b/arch/src/main/scala/framework/thread/ant/src/main/scala/Ant.scala @@ -0,0 +1,163 @@ +package framework.thread.ant + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.thread.ant.configs.AntParam +import framework.core.memory.access.preflight.configs.PmpParams +import framework.thread.ant.{Completion, Executor, Retire, Start} +import framework.core.memory.access.preflight.PmpEntry +import framework.acc.frontend.RoCCIO +import framework.ip.memory.spm + +/** Address space and permissions captured from the task's launch command. */ +class TaskContext(permission: PmpParams) extends Bundle { + val satp = UInt(64.W) + val effectivePrivilege = UInt(2.W) + val sum = Bool() + val mxr = Bool() + val pmp = + Vec(permission.entries, new PmpEntry(permission)) +} + +/** One accepted launch binds the task context and starts its local program together. */ +class Launch(b: AntParam, permission: PmpParams) extends Bundle { + val descriptor = new Start(b) + val context = new TaskContext(permission) +} + +/** Byte offsets into the TSS and the selected private store; transfers use 8-byte beats. */ +class Transfer extends Bundle { + val code = Bool() + val toLocal = Bool() + val localOffset = UInt(32.W) + val sharedOffset = UInt(32.W) + val bytes = UInt(32.W) +} + +/** One thread task: launch, cancellation and completion. Storage accesses use independent interfaces. */ +class Task(b: AntParam, permission: PmpParams) extends Bundle { + val transfer = Flipped(Decoupled(new Transfer)) + val transferred = Decoupled(Bool()) + val inUse = Input(Bool()) + val start = Flipped(Decoupled(new Launch(b, permission))) + val result = Decoupled(new Completion(b)) + val cancel = Input(Bool()) + val running = Output(Bool()) + val storageBusy = Output(Bool()) + +} + +@instantiable +class Ant(val b: AntParam, permission: PmpParams) extends Module { + + @public + val io = IO(new Bundle { + val trace = Output(Valid(new Retire)) + val task = new Task(b, permission) + + val shared = new spm.Port(b.shared) + val rocc = Flipped(new RoCCIO(64, permission)) + + }) + + val executor: Instance[Executor] = Instantiate(new Executor(b, permission)) + val image = Instantiate(new LocalStore(b, 0)) + val data = Instantiate(new LocalStore(b, 1)) + val loader = Instantiate(new Loader(b)) + + io.trace := executor.io.retired + io.task.storageBusy := executor.io.storageBusy || loader.io.busy + executor.io.start.valid := io.task.start.valid && !loader.io.busy + executor.io.start.bits := io.task.start.bits + io.task.start.ready := executor.io.start.ready && !loader.io.busy + loader.io.transfer.valid := io.task.transfer.valid && !executor.io.storageClaimed && !io.task.start.valid + loader.io.transfer.bits := io.task.transfer.bits + io.task.transfer.ready := loader.io.transfer.ready && !executor.io.storageClaimed && !io.task.start.valid + io.task.transferred <> loader.io.transferred + io.task.result <> executor.io.result + io.task.running := executor.io.running + executor.io.cancel := io.task.cancel + executor.io.inUse := io.task.inUse + executor.io.localBusy(0) := image.io.busy + executor.io.localBusy(1) := data.io.busy + image.io.running := executor.io.storageClaimed + data.io.running := executor.io.storageClaimed + image.io.cancel := io.task.cancel + data.io.cancel := io.task.cancel + image.io.host <> loader.io.code + data.io.host <> loader.io.data + image.io.local <> executor.io.imem + data.io.local <> executor.io.tls + io.shared.request.valid := Mux(loader.io.busy, loader.io.shared.request.valid, executor.io.tss.request.valid) + io.shared.request.bits := Mux(loader.io.busy, loader.io.shared.request.bits, executor.io.tss.request.bits) + loader.io.shared.request.ready := loader.io.busy && io.shared.request.ready + executor.io.tss.request.ready := !loader.io.busy && io.shared.request.ready + loader.io.shared.response.valid := loader.io.busy && io.shared.response.valid + loader.io.shared.response.bits := io.shared.response.bits + executor.io.tss.response.valid := !loader.io.busy && io.shared.response.valid + executor.io.tss.response.bits := io.shared.response.bits + io.shared.response.ready := Mux(loader.io.busy, loader.io.shared.response.ready, executor.io.tss.response.ready) + io.rocc <> executor.io.rocc +} + +/** Moves data between TSS and private storage without exposing private SRAM ports. */ +@instantiable +class Loader(b: AntParam) extends Module { + + @public + val io = IO(new Bundle { + val transfer = Flipped(Decoupled(new Transfer)) + val transferred = Decoupled(Bool()) + val busy = Output(Bool()) + val code = new spm.Port(spm.Params(0, b.codeBytes, b.data.dataBits)) + val data = new spm.Port(b.data) + val shared = new spm.Port(b.shared) + }) + + val idle :: reading :: readResponse :: writing :: writeResponse :: completed :: Nil = Enum(6) + val state = RegInit(idle) + val transfer = Reg(new Transfer) + val offset = Reg(UInt(32.W)) + val value = Reg(UInt(64.W)) + io.transfer.ready := state === idle + io.transferred.valid := state === completed + io.transferred.bits := true.B + io.busy := state =/= idle + when(io.transfer.fire) { + val t = io.transfer.bits + assert(t.bytes =/= 0.U && !(t.localOffset | t.sharedOffset | t.bytes)(2, 0).orR) + assert((t.localOffset +& t.bytes) <= Mux(t.code, b.codeBytes.U, b.data.bytes.U)) + assert((t.sharedOffset +& t.bytes) <= b.shared.bytes.U) + transfer := t + offset := 0.U + state := reading + } + val local = Mux(transfer.code, 0.U, 1.U) + val source = Mux(transfer.toLocal, 2.U, local) + val destination = Mux(transfer.toLocal, local, 2.U) + val ports = Seq(io.code, io.data, io.shared) + for ((port, i) <- ports.zipWithIndex) { + port.request.valid := (state === reading && source === i.U) || (state === writing && destination === i.U) + port.request.bits := 0.U.asTypeOf(port.request.bits) + val base = Seq(BigInt(0), b.data.base, b.shared.base)(i) + port.request.bits.address := base.U + (if (i == 2) transfer.sharedOffset else transfer.localOffset) + offset + port.request.bits.size := 3.U + port.request.bits.write := state === writing + port.request.bits.data := value + port.request.bits.mask := Mux(state === writing, 255.U, 0.U) + port.response.ready := (state === readResponse && source === i.U) || (state === writeResponse && destination === i.U) + when(port.request.fire)(state := Mux(state === reading, readResponse, writeResponse)) + when(port.response.fire) { + assert(!port.response.bits.error, "Ant TSS transfer failed") + when(state === readResponse) { + value := port.response.bits.data(63, 0) + state := writing + }.otherwise { + when(offset + 8.U === transfer.bytes)(state := completed) + .otherwise { offset := offset + 8.U; state := reading } + } + } + } + when(io.transferred.fire)(state := idle) +} diff --git a/arch/src/main/scala/framework/thread/ant/src/main/scala/Decode.scala b/arch/src/main/scala/framework/thread/ant/src/main/scala/Decode.scala new file mode 100644 index 000000000..d35f5ebc3 --- /dev/null +++ b/arch/src/main/scala/framework/thread/ant/src/main/scala/Decode.scala @@ -0,0 +1,94 @@ +package framework.thread.ant + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} + +object Kind extends ChiselEnum { + val illegal, lui, auipc, jal, jalr, branch, load, store, alu, multiply, divide, custom, fence, exit, breakpoint = + Value +} + +class Decoded extends Bundle { + val kind = Kind() + val rs1 = UInt(5.W) + val rs2 = UInt(5.W) + val rd = UInt(5.W) + val funct3 = UInt(3.W) + val immediate = UInt(64.W) + val immediateAlu = Bool() + val word = Bool() + val subtract = Bool() + val arithmeticShift = Bool() +} + +@instantiable +class Decode extends Module { + + @public + val io = IO(new Bundle { + val instruction = Input(UInt(32.W)) + val decoded = Output(new Decoded) + }) + + val i = io.instruction + val f3 = i(14, 12) + val f7 = i(31, 25) + val d = WireDefault(0.U.asTypeOf(new Decoded)) + d.kind := Kind.illegal + d.rs1 := i(19, 15) + d.rs2 := i(24, 20) + d.rd := i(11, 7) + d.funct3 := f3 + d.immediate := i(31, 20).asSInt.pad(64).asUInt + d.word := i(3) + d.subtract := f7 === 32.U + d.arithmeticShift := i(30) + switch(i(6, 0)) { + is("h37".U) { d.kind := Kind.lui; d.immediate := Cat(i(31, 12), 0.U(12.W)).asSInt.pad(64).asUInt } + is("h17".U) { d.kind := Kind.auipc; d.immediate := Cat(i(31, 12), 0.U(12.W)).asSInt.pad(64).asUInt } + is("h6f".U) { + d.kind := Kind.jal + d.immediate := Cat(i(31), i(19, 12), i(20), i(30, 21), 0.U(1.W)).asSInt.pad(64).asUInt + } + is("h67".U)(when(f3 === 0.U)(d.kind := Kind.jalr)) + is("h63".U) { + when(f3 === 0.U || f3 === 1.U || f3 >= 4.U)(d.kind := Kind.branch) + d.immediate := Cat(i(31), i(7), i(30, 25), i(11, 8), 0.U(1.W)).asSInt.pad(64).asUInt + } + is("h03".U)(when(f3 <= 6.U)(d.kind := Kind.load)) + is("h23".U) { + when(f3 <= 3.U)(d.kind := Kind.store) + d.immediate := Cat(i(31, 25), i(11, 7)).asSInt.pad(64).asUInt + } + is("h13".U, "h1b".U) { + d.immediateAlu := true.B + d.subtract := false.B + val shiftLeft = f3 === 1.U + val shiftRight = f3 === 5.U + val shiftEncoding = + Mux(i(3), f7 === 0.U || (shiftRight && f7 === 32.U), i(31, 26) === 0.U || (shiftRight && i(31, 26) === 16.U)) + val operation = !i(3) || f3 === 0.U || shiftLeft || shiftRight + when(operation && (!(shiftLeft || shiftRight) || shiftEncoding))(d.kind := Kind.alu) + } + is("h33".U, "h3b".U) { + when(f7 === 1.U) { + when(!i(3) || f3 === 0.U || f3 >= 4.U) { + d.kind := Mux(f3 < 4.U, Kind.multiply, Kind.divide) + } + }.otherwise { + val operation = !i(3) || f3 === 0.U || f3 === 1.U || f3 === 5.U + when(operation && (f7 === 0.U || (f7 === 32.U && (f3 === 0.U || f3 === 5.U)))) { + d.kind := Kind.alu + } + } + } + is("h7b".U)(d.kind := Kind.custom) + is("h0f".U)(when(f3 === 0.U)(d.kind := Kind.fence)) + is("h73".U) { + when(i === "h00000073".U)(d.kind := Kind.exit) + when(i === "h00100073".U)(d.kind := Kind.breakpoint) + } + } + io.decoded := d +} diff --git a/arch/src/main/scala/framework/thread/ant/src/main/scala/Emit.scala b/arch/src/main/scala/framework/thread/ant/src/main/scala/Emit.scala new file mode 100644 index 000000000..b64d05625 --- /dev/null +++ b/arch/src/main/scala/framework/thread/ant/src/main/scala/Emit.scala @@ -0,0 +1,148 @@ +package framework.thread.ant + +import buckyball.config.Design +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.Instantiate +import framework.core.controller.configs.TasksParam +import framework.core.controller.task.{Control, Service} +import framework.thread.ant.{Completion, Retire, Start} +import framework.acc.frontend.RoCCIO +import framework.core.tss.{Store => TssStore} +import framework.core.tss.configs.{Params => TssParam} +import framework.core.memory.access.preflight.configs.PmpParams +import framework.device.cbus.ControlRequest +import framework.ip.memory.spm +import java.nio.file.{Files, Paths} +import scala.jdk.CollectionConverters._ + +/** Direct task/transfer verification around real Ants and their shared storage. */ +class AntVerification(p: TasksParam) extends Module { + require(p.locals.size == 2 && p.locals.distinct.size == 1) + val local = p.locals.head + + val io = IO(new Bundle { + val inUse = Input(Bool()) + val start = Vec(2, Flipped(Decoupled(new Start(local)))) + val result = Vec(2, Decoupled(new Completion(local))) + val transfer = Vec(2, Flipped(Decoupled(new Transfer))) + val transferred = Vec(2, Decoupled(Bool())) + val cancel = Input(Vec(2, Bool())) + val running = Output(Vec(2, Bool())) + val rocc = Vec(2, Flipped(new RoCCIO(64, p.permission))) + val retired = Output(Vec(2, Valid(new Retire))) + val host = Flipped(new spm.Port(p.tss.memory)) + }) + + val ants = Seq.fill(2)(Instantiate(new Ant(local, p.permission))) + val shared = Instantiate(new TssStore(p.tss)) + shared.io.host <> io.host + shared.io.inUse := io.inUse || ants.map(_.io.task.storageBusy).reduce(_ || _) + shared.io.cancel := io.cancel + for ((ant, i) <- ants.zipWithIndex) { + ant.io.task.inUse := io.inUse + ant.io.task.start.valid := io.start(i).valid && !shared.io.busy(i) && !shared.io.hostBusy + ant.io.task.start.bits.descriptor := io.start(i).bits + ant.io.task.start.bits.context := 0.U.asTypeOf(ant.io.task.start.bits.context) + io.start(i).ready := ant.io.task.start.ready && !shared.io.busy(i) && !shared.io.hostBusy + io.result(i).valid := ant.io.task.result.valid && !shared.io.busy(i) + io.result(i).bits := ant.io.task.result.bits + ant.io.task.result.ready := io.result(i).ready && !shared.io.busy(i) + ant.io.task.transfer <> io.transfer(i) + io.transferred(i) <> ant.io.task.transferred + ant.io.task.cancel := io.cancel(i) + io.running(i) := ant.io.task.running + io.rocc(i) <> ant.io.rocc + io.retired(i) := ant.io.trace + shared.io.clients(i) <> ant.io.shared + } +} + +/** Production management service connected to real Ant/TSS instances. */ +class ServiceVerification(p: TasksParam) extends Module { + + val io = IO(new Bundle { + val request = Flipped(Decoupled(new ControlRequest)) + val reply = Decoupled(UInt(64.W)) + val rocc = Vec(p.count, Flipped(new RoCCIO(64, p.permission))) + val retired = Output(Vec(p.count, Valid(new Retire))) + }) + + val service = Instantiate(new Service(p)) + val shared = Instantiate(new TssStore(p.tss)) + service.io.request.valid := io.request.valid + service.io.request.bits.control := io.request.bits + service.io.request.bits.context := 0.U.asTypeOf(service.io.request.bits.context) + io.request.ready := service.io.request.ready + io.reply <> service.io.reply + shared.io.host <> service.io.tss.host + shared.io.inUse := service.io.tss.inUse + shared.io.cancel := service.io.tss.cancel + service.io.tss.busy := shared.io.busy + service.io.tss.hostBusy := shared.io.hostBusy + for (i <- 0 until p.count) { + val ant = Instantiate(new Ant(p.locals(i), p.permission)) + ant.io.task <> service.io.tasks(i) + shared.io.clients(i) <> ant.io.shared + io.rocc(i) <> ant.io.rocc + io.retired(i) := ant.io.trace + } +} + +object Emit extends App { + require(args.length == 4 && args(0) == "--config" && args(2) == "--core") + val design = Design.parseFrom(Files.readAllBytes(Paths.get(args(1)))) + val core = design.getDevice.getCoresList.asScala.find(_.getHartId == args(3).toInt).get + val shared = core.getTss + val memory = spm.Params(BigInt(java.lang.Long.toUnsignedString(shared.getBase)), shared.getBytes, shared.getDataBits) + require(core.getThreadIndicesCount >= 2, "Ant verification requires two configured hardware threads") + val threads = core.getThreadIndicesList.asScala.take(2).map(i => design.getThreads(i.intValue)).toSeq + val locals = threads.map(t => configs.AntParam.fromProto(t.getCpu.getAnt, memory)) + + val p = TasksParam( + locals, + threads.map(t => BigInt(java.lang.Long.toUnsignedString(t.getSignature))), + TssParam(memory, 2), + PmpParams(core.getParam.getPaddrBits, core.getParam.getNPmps) + ) + + require(locals.distinct.size == 1 && locals.head.taskBits == 32 && memory.dataBits == 128) + require(locals.head.data.base == 0x10000 && memory.base == 0x20000, "Shared-program fixture address encoding mismatch") + val directory = Paths.get("build") + Files.createDirectories(directory) + Files.writeString( + directory.resolve("config.svh"), + s"""`define LOCAL_BASE 64'h${locals.head.data.base.toString(16)} +`define LOCAL_BYTES ${locals.head.data.bytes} +`define TLS_BASE 64'h${locals.head.data.base.toString(16)} +`define TLS_END 64'h${(locals.head.data.base + locals.head.data.bytes).toString(16)} +`define TSS_BASE 64'h${memory.base.toString(16)} +`define TSS_SCRATCH 64'h${(memory.base + memory.bytes - 32).toString(16)} +`define CODE_BYTES ${locals.head.codeBytes} +`define TLS_BYTES ${locals.head.data.bytes} +`define TSS_BYTES ${memory.bytes} +`define SIGNATURE_0 64'h${p.signatures(0).toString(16)} +`define SIGNATURE_1 64'h${p.signatures(1).toString(16)} +""" + ) + val fields = Seq("cfg_l", "cfg_res", "cfg_a", "cfg_x", "cfg_w", "cfg_r", "addr", "mask") + Files.writeString( + directory.resolve("control_pmp.svh"), + (for { + i <- 0 until p.permission.entries + field <- fields + } yield s", .io_request_bits_context_pmp_${i}_$field('0)").mkString("\n") + ) + Seq[(String, () => RawModule)]( + "Store" -> (() => new LocalStore(locals.head, 1)), + "Ant" -> (() => new AntVerification(p)), + "Control" -> (() => new Control(p)), + "Service" -> (() => new ServiceVerification(p)) + ).foreach { case (name, top) => + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + top(), + Array("--target-dir", s"build/$name", "--split-verilog"), + firtoolOpts = Array("--disable-all-randomization") + ) + } +} diff --git a/arch/src/main/scala/framework/thread/ant/src/main/scala/Executor.scala b/arch/src/main/scala/framework/thread/ant/src/main/scala/Executor.scala new file mode 100644 index 000000000..780e3d1fa --- /dev/null +++ b/arch/src/main/scala/framework/thread/ant/src/main/scala/Executor.scala @@ -0,0 +1,329 @@ +package framework.thread.ant + +import framework.thread.ant.configs.AntParam +import framework.core.memory.access.preflight.configs.PmpParams + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import framework.lib.arith.{DivRem, Multiply} +import framework.ip.memory.spm +import framework.thread.ant.{Launch, TaskContext} +import framework.acc.frontend.RoCCIO + +class Start(b: AntParam) extends Bundle { + val p = b + + val task = UInt(p.taskBits.W) + val entry = UInt(64.W) + val codeEnd = UInt(64.W) + val argument = UInt(64.W) + val stack = UInt(64.W) +} + +class Completion(b: AntParam) extends Bundle { + val p = b + + val task = UInt(p.taskBits.W) + val value = UInt(64.W) + val cancelled = Bool() +} + +class Retire extends Bundle { + val pc = UInt(64.W) + val instruction = UInt(32.W) + val rd = UInt(5.W) + val data = UInt(64.W) +} + +/** + * One RV64IM execution context. Ordinary memory is strictly local. NPU work may remain + * in flight after a command is accepted; completion waits for accelerator and storage drain. + * Cancellation also reaches the local stores, whose accepted operations must drain before reuse. + */ +@instantiable +class Executor(val b: AntParam, permission: PmpParams) extends Module { + val p = b + val code = spm.Params(0, p.codeBytes, p.data.dataBits) + + @public + val io = IO(new Bundle { + + val start = Flipped(Decoupled(new Launch(b, permission))) + + val result = Decoupled(new Completion(b)) + + val cancel = Input(Bool()) + val inUse = Input(Bool()) + val localBusy = Input(Vec(2, Bool())) + val storageBusy = Output(Bool()) + val storageClaimed = Output(Bool()) + val running = Output(Bool()) + + val imem = new spm.Port(code) + + val tls = new spm.Port(p.data) + + val tss = new spm.Port(p.shared) + + val rocc = Flipped(new RoCCIO(64, permission)) + + val retired = Output(Valid(new Retire)) + }) + + val (idle :: fetch :: fetchWait :: execute :: memory :: memoryWait :: + multiply :: multiplyWait :: divide :: divideWait :: issue :: complete :: Nil) = Enum(12) + val state = RegInit(idle) + val registers = Reg(Vec(32, UInt(64.W))) + val pc = Reg(UInt(64.W)) + val instruction = Reg(UInt(32.W)) + val task = Reg(UInt(p.taskBits.W)) + val codeEnd = Reg(UInt(64.W)) + + val completion = Reg(new Completion(b)) + val context = Reg(new TaskContext(permission)) + io.storageBusy := io.localBusy.asUInt.orR + io.storageClaimed := state =/= idle || (io.inUse && io.start.valid) + val drained = !io.storageBusy && !io.rocc.busy + + val cancelled = RegInit(false.B) + val multiplyPending = RegInit(false.B) + val dividePending = RegInit(false.B) + val live = !io.cancel && !reset.asBool + val memoryAddress = Reg(UInt(64.W)) + val memoryShared = Reg(Bool()) + val decoder = Instantiate(new Decode) + val multiplier = Instantiate(new Multiply) + val divider = + Instantiate(new DivRem(64)) + decoder.io.instruction := instruction + val d = decoder.io.decoded + val a = Mux(d.rs1 === 0.U, 0.U, registers(d.rs1)) + val rhs = Mux(d.rs2 === 0.U, 0.U, registers(d.rs2)) + val nextPc = pc + 4.U + + io.start.ready := state === idle && io.inUse && drained && !io.cancel && !reset.asBool + io.result.valid := state === complete && drained && !multiplyPending && !dividePending && !reset.asBool + io.result.bits := completion + io.running := state =/= idle + io.retired.valid := false.B + io.retired.bits := 0.U.asTypeOf(new Retire) + + def retire(rd: UInt, value: UInt, next: UInt): Unit = { + when(live) { + when(rd =/= 0.U)(registers(rd) := value) + io.retired.valid := true.B + io.retired.bits.pc := pc + io.retired.bits.instruction := instruction + io.retired.bits.rd := rd + io.retired.bits.data := Mux(rd === 0.U, 0.U, value) + pc := next + state := fetch + } + } + + def finish(value: UInt, wasCancelled: Bool = false.B): Unit = { + completion.task := task + completion.value := value + completion.cancelled := wasCancelled + state := complete + } + + def word(value: UInt): UInt = Mux(d.word, value(31, 0).asSInt.pad(64).asUInt, value) + + when(io.start.fire) { + context := io.start.bits.context + for (r <- registers) r := 0.U + registers(2) := io.start.bits.descriptor.stack + registers(4) := p.data.base.U + registers(10) := io.start.bits.descriptor.argument + pc := io.start.bits.descriptor.entry + codeEnd := io.start.bits.descriptor.codeEnd + task := io.start.bits.descriptor.task + cancelled := false.B + val valid = io.start.bits.descriptor.entry(1, 0) === 0.U && io.start.bits.descriptor.codeEnd(1, 0) === 0.U && + io.start.bits.descriptor.codeEnd <= p.codeBytes.U && (io.start.bits.descriptor.entry +& 4.U) <= io.start.bits.descriptor.codeEnd && + io.start.bits.descriptor.argument >= p.data.base.U && io.start.bits.descriptor.argument < (p.data.base + p.data.bytes).U && + io.start.bits.descriptor.stack > p.data.base.U && io.start.bits.descriptor.stack <= (p.data.base + p.data.bytes).U && + io.start.bits.descriptor.stack(3, 0) === 0.U + assert(valid, "Invalid Ant local execution descriptor") + when(valid)(state := fetch) + } + + val fetchAllowed = pc(1, 0) === 0.U && (pc +& 4.U) <= codeEnd + io.imem.request.valid := state === fetch && fetchAllowed && live + io.imem.request.bits := 0.U.asTypeOf(io.imem.request.bits) + io.imem.request.bits.address := pc + io.imem.request.bits.size := 2.U + io.imem.response.ready := state === fetchWait || (state === complete && cancelled) + when(state === fetch && live) { + assert(fetchAllowed, "Ant instruction address invalid: pc=0x%x", pc) + when(io.imem.request.fire)(state := fetchWait) + } + when(state === fetchWait && io.imem.response.fire && live) { + assert(!io.imem.response.bits.error, "Ant instruction storage access failed: pc=0x%x", pc) + when(!io.imem.response.bits.error) { + instruction := io.imem.response.bits.data(31, 0) + state := execute + } + } + + val operand = Mux(d.immediateAlu, d.immediate, rhs) + val shift = Mux(d.word, Cat(0.U(1.W), operand(4, 0)), operand(5, 0)) + val logical = Mux(d.word, Cat(0.U(32.W), a(31, 0)), a) + val signed = Mux(d.word, a(31, 0).asSInt.pad(64).asUInt, a) + + val alu = MuxLookup(d.funct3, 0.U(64.W))(Seq( + 0.U -> Mux(d.subtract, a - operand, a + operand), + 1.U -> (a << shift)(63, 0), + 2.U -> (a.asSInt < operand.asSInt).asUInt, + 3.U -> (a < operand).asUInt, + 4.U -> (a ^ operand), + 5.U -> Mux(d.arithmeticShift, (signed.asSInt >> shift).asUInt, logical >> shift), + 6.U -> (a | operand), + 7.U -> (a & operand) + )) + + val branchTaken = MuxLookup(d.funct3, false.B)(Seq( + 0.U -> (a === rhs), + 1.U -> (a =/= rhs), + 4.U -> (a.asSInt < rhs.asSInt), + 5.U -> (a.asSInt >= rhs.asSInt), + 6.U -> (a < rhs), + 7.U -> (a >= rhs) + )) + + val address = a + d.immediate + val size = d.funct3(1, 0) + val bytes = 1.U(64.W) << size + val end = address +& bytes + val inTls = address >= p.data.base.U && end <= (p.data.base + p.data.bytes).U + val inTss = address >= p.shared.base.U && end <= (p.shared.base + p.shared.bytes).U + when(state === execute && live) { + switch(d.kind) { + is(Kind.illegal)(assert(false.B, "Ant illegal instruction: pc=0x%x instruction=0x%x", pc, instruction)) + is(Kind.lui)(retire(d.rd, d.immediate, nextPc)) + is(Kind.auipc)(retire(d.rd, pc + d.immediate, nextPc)) + is(Kind.alu)(retire(d.rd, word(alu), nextPc)) + is(Kind.jal, Kind.jalr, Kind.branch) { + val target = Mux( + d.kind === Kind.jalr, + (a + d.immediate) & ~1.U(64.W), + Mux(d.kind === Kind.branch && !branchTaken, nextPc, pc + d.immediate) + ) + assert(target(1, 0) === 0.U, "Ant branch target misaligned: pc=0x%x target=0x%x", pc, target) + retire(Mux(d.kind === Kind.branch, 0.U, d.rd), nextPc, target) + } + is(Kind.load, Kind.store) { + assert((address & (bytes - 1.U)) === 0.U, "Ant load/store misaligned: pc=0x%x address=0x%x", pc, address) + assert(inTls || inTss, "Ant load/store outside TLS/TSS: pc=0x%x address=0x%x", pc, address) + when((address & (bytes - 1.U)) === 0.U && (inTls || inTss)) { + memoryAddress := address + memoryShared := inTss + state := memory + } + } + is(Kind.multiply)(state := multiply) + is(Kind.divide)(state := divide) + is(Kind.custom)(state := issue) + is(Kind.fence)(retire(0.U, 0.U, nextPc)) // Ordinary LSU operations already complete in order. + is(Kind.exit) { + assert(registers(17) === 0.U, "Ant unsupported ECALL: pc=0x%x a7=0x%x", pc, registers(17)) + when(registers(17) === 0.U)(finish(registers(10))) + } + is(Kind.breakpoint)(assert(false.B, "Ant breakpoint: pc=0x%x", pc)) + } + } + + val mask = MuxLookup(size, 0.U(p.data.beatBytes.W))(Seq( + 0.U -> 1.U, + 1.U -> 3.U, + 2.U -> 15.U, + 3.U -> 255.U + )) + + for ((port, shared) <- Seq((io.tls, false), (io.tss, true))) { + port.request.valid := state === memory && memoryShared === shared.B && live + port.request.bits := 0.U.asTypeOf(port.request.bits) + port.request.bits.address := memoryAddress + port.request.bits.size := size + port.request.bits.write := d.kind === Kind.store + port.request.bits.data := rhs + port.request.bits.mask := Mux(d.kind === Kind.store, mask, 0.U) + port.response.ready := (state === memoryWait && memoryShared === shared.B) || (state === complete && cancelled) + } + when(io.tls.request.fire || io.tss.request.fire)(state := memoryWait) + val memoryResponse = Mux(memoryShared, io.tss.response.bits, io.tls.response.bits) + val memoryValid = Mux(memoryShared, io.tss.response.valid, io.tls.response.valid) + + val loaded = MuxLookup(d.funct3, memoryResponse.data(63, 0))(Seq( + 0.U -> memoryResponse.data(7, 0).asSInt.pad(64).asUInt, + 1.U -> memoryResponse.data(15, 0).asSInt.pad(64).asUInt, + 2.U -> memoryResponse.data(31, 0).asSInt.pad(64).asUInt, + 4.U -> memoryResponse.data(7, 0).pad(64), + 5.U -> memoryResponse.data(15, 0).pad(64), + 6.U -> memoryResponse.data(31, 0).pad(64) + )) + + when(state === memoryWait && memoryValid && live) { + assert(!memoryResponse.error, "Ant local memory access failed: pc=0x%x address=0x%x", pc, memoryAddress) + when(!memoryResponse.error)(retire(Mux(d.kind === Kind.store, 0.U, d.rd), loaded, nextPc)) + } + + multiplier.io.request.valid := state === multiply && live + multiplier.io.request.bits.a := a + multiplier.io.request.bits.b := rhs + multiplier.io.request.bits.high := d.funct3 =/= 0.U + multiplier.io.request.bits.signedA := d.funct3 === 1.U || d.funct3 === 2.U + multiplier.io.request.bits.signedB := d.funct3 === 1.U + multiplier.io.response.ready := state === multiplyWait || (state === complete && cancelled) + when(multiplier.io.request.fire) { state := multiplyWait; multiplyPending := true.B } + when(multiplier.io.response.fire)(multiplyPending := false.B) + when(state === multiplyWait && multiplier.io.response.fire && live) { + retire(d.rd, word(multiplier.io.response.bits), nextPc) + } + divider.io.clear := false.B + divider.io.request.valid := state === divide && live + divider.io.request.bits.a := a + divider.io.request.bits.b := rhs + divider.io.request.bits.sew := Mux(d.word, 2.U, 3.U) + divider.io.request.bits.signed := !d.funct3(0) + divider.io.request.bits.remainder := d.funct3(1) + divider.io.response.ready := state === divideWait || (state === complete && cancelled) + when(divider.io.request.fire) { state := divideWait; dividePending := true.B } + when(divider.io.response.fire)(dividePending := false.B) + when(state === divideWait && divider.io.response.fire && live) { + retire(d.rd, word(divider.io.response.bits), nextPc) + } + + io.rocc.cmd.valid := state === issue && live + io.rocc.cmd.bits.instruction.raw_inst := instruction + io.rocc.cmd.bits.instruction.pc := pc + io.rocc.cmd.bits.instruction.rs1Data := Mux(instruction(13), a, 0.U) + io.rocc.cmd.bits.instruction.rs2Data := Mux(instruction(12), rhs, 0.U) + io.rocc.cmd.bits.satp := context.satp + io.rocc.cmd.bits.effectivePrivilege := context.effectivePrivilege + io.rocc.cmd.bits.sum := context.sum + io.rocc.cmd.bits.mxr := context.mxr + io.rocc.cmd.bits.pmp := context.pmp + io.rocc.resp.ready := false.B + when(io.rocc.cmd.valid) { + assert(!instruction(14), "Ant accelerator commands do not write integer registers") + } + when(io.rocc.cmd.fire)(retire(0.U, 0.U, nextPc)) + assert(!io.rocc.resp.valid, "Ant received an unsolicited RoCC response") + assert(!io.rocc.interrupt, "Ant accelerator fault") + when(io.result.fire)(state := idle) + when(io.cancel && state =/= idle && state =/= complete) { + finish(0.U, true.B) + cancelled := true.B + io.retired.valid := false.B + } + // Local execution may have ended while the task still waits for NPU/DMA drain. + // Once a completion is published, preserve it under response backpressure. + when(io.cancel && state === complete && !io.result.valid) { + completion.cancelled := true.B + completion.value := 0.U + cancelled := true.B + } +} diff --git a/arch/src/main/scala/framework/thread/ant/src/main/scala/LocalStore.scala b/arch/src/main/scala/framework/thread/ant/src/main/scala/LocalStore.scala new file mode 100644 index 000000000..382011997 --- /dev/null +++ b/arch/src/main/scala/framework/thread/ant/src/main/scala/LocalStore.scala @@ -0,0 +1,61 @@ +package framework.thread.ant + +import framework.thread.ant.configs.AntParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import framework.ip.memory.spm.{Memory, Params, Port} + +/** + * A context-private local store. Loader and execution access it in separate ownership phases. + * Cancellation drains an accepted local access and discards its response before reuse. + */ +@instantiable +class LocalStore(val b: AntParam, index: Int) extends Module { + val local = b + + val (p, localReadOnly) = index match { + case 0 => (Params(0, local.codeBytes, local.data.dataBits), true) + case 1 => (local.data, false) + } + + @public + val io = IO(new Bundle { + val running = Input(Bool()) + val cancel = Input(Bool()) + + val host = Flipped(new Port(p)) + + val local = Flipped(new Port(p)) + + val busy = Output(Bool()) + }) + + val memory = Instantiate(new Memory(p)) + + val busy = RegInit(false.B) + val owner = Reg(Bool()) + val cancelled = RegInit(false.B) + val discard = owner && (cancelled || io.cancel) + memory.io.port.request.valid := !busy && !reset.asBool && + Mux(io.running, io.local.request.valid && !io.cancel, io.host.request.valid) + memory.io.port.request.bits := Mux(io.running, io.local.request.bits, io.host.request.bits) + memory.io.readOnly := io.running && localReadOnly.B + io.local.request.ready := !busy && io.running && !io.cancel && memory.io.port.request.ready + io.host.request.ready := !busy && !io.running && memory.io.port.request.ready + when(memory.io.port.request.fire) { + busy := true.B + owner := io.running + cancelled := false.B + } + when(busy && owner && io.cancel)(cancelled := true.B) + io.local.response.valid := busy && owner && !discard && memory.io.port.response.valid + io.host.response.valid := busy && !owner && memory.io.port.response.valid + io.local.response.bits := memory.io.port.response.bits + io.host.response.bits := memory.io.port.response.bits + memory.io.port.response.ready := busy && + (discard || Mux(owner, io.local.response.ready, io.host.response.ready)) + when(memory.io.port.response.fire) { busy := false.B; cancelled := false.B } + io.busy := busy +} diff --git a/arch/src/main/scala/framework/thread/build.sc b/arch/src/main/scala/framework/thread/build.sc new file mode 100644 index 000000000..a1ccd93a1 --- /dev/null +++ b/arch/src/main/scala/framework/thread/build.sc @@ -0,0 +1,16 @@ +import mill._ +import $file.ant.{build => ant_build} +import $file.tlink.{build => tlink_build} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("configs/ThreadParam.scala")) + ) ++ + ant_build.sources() ++ + tlink_build.sources() +) + +def emitSources = T.sources( + ant_build.emitSources() ++ + tlink_build.emitSources() +) diff --git a/arch/src/main/scala/framework/thread/configs/ThreadParam.scala b/arch/src/main/scala/framework/thread/configs/ThreadParam.scala new file mode 100644 index 000000000..ac361a1bc --- /dev/null +++ b/arch/src/main/scala/framework/thread/configs/ThreadParam.scala @@ -0,0 +1,39 @@ +package framework.thread.configs + +import buckyball.config.{CorePlacement, ThreadInstance} +import framework.ip.memory.spm +import framework.acc.configs.AccParam +import framework.thread.ant.configs.AntParam + +case class ThreadParam( + id: Int, + name: String, + component: String, + rtlClass: String, + signature: BigInt, + commandBits: Int, + acc: Option[AccParam], + ant: Option[AntParam]) { + require(ant.isEmpty || commandBits == 64, "Ant implements a 64-bit RoCC command interface") +} + +object ThreadParam { + + def fromProto(thread: ThreadInstance, core: CorePlacement): ThreadParam = { + require(thread.hasLocalId && thread.hasSignature) + require(core.hasTss && thread.hasCpu && thread.getCpu.getKind == "ant" && thread.getCpu.hasAnt) + val tss = core.getTss + val shared = spm.Params(BigInt(java.lang.Long.toUnsignedString(tss.getBase)), tss.getBytes, tss.getDataBits) + ThreadParam( + id = thread.getLocalId, + name = thread.getRole, + component = thread.getPkg, + rtlClass = thread.getRtlClass, + signature = BigInt(java.lang.Long.toUnsignedString(thread.getSignature)), + commandBits = core.getParam.getXLen, + acc = Some(AccParam.fromProto(thread, core)), + ant = Some(AntParam.fromProto(thread.getCpu.getAnt, shared)) + ) + } + +} diff --git a/arch/src/main/scala/framework/thread/tlink/basethread.scala b/arch/src/main/scala/framework/thread/tlink/basethread.scala new file mode 100644 index 000000000..e60207d8a --- /dev/null +++ b/arch/src/main/scala/framework/thread/tlink/basethread.scala @@ -0,0 +1,9 @@ +package framework.thread.tlink + +import chisel3.experimental.hierarchy.{instantiable, public} + +@instantiable +trait HasTLink { + @public + val tlink: TLinkIO +} diff --git a/arch/src/main/scala/framework/thread/tlink/build.sc b/arch/src/main/scala/framework/thread/tlink/build.sc new file mode 100644 index 000000000..295276ed8 --- /dev/null +++ b/arch/src/main/scala/framework/thread/tlink/build.sc @@ -0,0 +1,12 @@ +import mill._ + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("basethread.scala")), + PathRef(millSourcePath / os.RelPath("tlink.scala")) + ) +) + +def emitSources = T.sources( + Seq.empty[PathRef] +) diff --git a/arch/src/main/scala/framework/thread/tlink/tlink.scala b/arch/src/main/scala/framework/thread/tlink/tlink.scala new file mode 100644 index 000000000..e998e34a5 --- /dev/null +++ b/arch/src/main/scala/framework/thread/tlink/tlink.scala @@ -0,0 +1,82 @@ +package framework.thread.tlink + +import sims.configs.SimParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.core.tbus.configs.{Params => TBusParam} + +import chisel3._ +import chisel3.util._ +import framework.core.tbus.router.{LeasePort, PhysicalPort} +import framework.acc.memdomain.{BankPort, LayoutConfig, MvoverISA, MvoverPort} +import framework.acc.memdomain.bank_router.BankMapping +import framework.ip.memory.mesh_shm.MeshLocalBankPort +import framework.thread.configs.ThreadParam +import framework.thread.ant.Task +import framework.core.memory.MemoryPort +import framework.core.memory.access.preflight.configs.PmpParams +import framework.core.memory.access.mmu.configs.{Params => WalkerParam} +import framework.ip.memory.interlock.{Params => TrackingParam} +import framework.acc.memdomain.PhysicalBankHash +import framework.ip.memory.spm +import framework.ip.bus.axi4 + +/** Thread-facing shared-bank access, lifetime and synchronization. */ +class BankAccess(frontendParam: FrontendParam, memParam: MemDomainParam) extends Bundle { + val sharedPorts = memParam.sharedPorts + require(sharedPorts >= 0 && (memParam.sharedEnable || sharedPorts == 0)) + val requests = Vec(sharedPorts, new BankPort(frontendParam, memParam)) + + val local = Flipped(new MeshLocalBankPort( + MvoverISA.AddressBits, + MvoverISA.BankBits, + memParam.bankWidth, + math.max(1, log2Ceil(frontendParam.rob_entries)) + )) + + val config = Decoupled(new LayoutConfig(frontendParam, memParam)) + val mapping = Decoupled(new BankMapping(frontendParam)) +} + +class SharedLink(frontendParam: FrontendParam, memParam: MemDomainParam) extends Bundle { + val sharedPorts = memParam.sharedPorts + val bank = new BankAccess(frontendParam, memParam) + val move = new MvoverPort + val barrierArrive = Output(Bool()) + val barrierRelease = Input(Bool()) +} + +/** Complete thread boundary; each service retains its independent handshake. */ +class TLinkIO( + thread: ThreadParam, + permission: PmpParams, + tracking: TrackingParam, + walker: WalkerParam, + simp: SimParam) + extends Bundle { + val sharedPorts = thread.acc.get.memDomain.sharedPorts + val diffTest = simp.diffTest + val task = new Task(thread.ant.get, permission) + val tss = new spm.Port(thread.ant.get.shared) + val memory = new MemoryPort(tracking, walker) + val ddr = new axi4.Port(thread.acc.get.transport.axi) + val shm = new SharedLink(thread.acc.get.frontend, thread.acc.get.memDomain) + + val sharedHashes = Option.when(diffTest && thread.acc.get.memDomain.sharedEnable)( + Input(Vec(thread.acc.get.memDomain.sharedBankNum, new PhysicalBankHash(thread.acc.get.frontend))) + ) + +} + +/** External shared-memory access and optional bank-movement commands. */ +class ExternalLink(memParam: MemDomainParam, tbusParam: TBusParam) extends Bundle { + val p = tbusParam + val lease = Option.when(p.router.physicalPorts > 0)(Flipped(new LeasePort)) + + val memory = Vec( + p.router.physicalPorts, + Flipped(new PhysicalPort(memParam.bankWidth)) + ) + + val move = Option.when(p.externalMove)(Flipped(new MvoverPort)) +} diff --git a/arch/src/main/scala/framework/top/GlobalConfig.scala b/arch/src/main/scala/framework/top/GlobalConfig.scala deleted file mode 100644 index 887b3ad3c..000000000 --- a/arch/src/main/scala/framework/top/GlobalConfig.scala +++ /dev/null @@ -1,40 +0,0 @@ -package framework.top - -import upickle.default.{macroRW, ReadWriter} -import chisel3.experimental.SerializableModuleParameter -import framework.memdomain.configs.MemDomainParam -import framework.frontend.configs.FrontendParam -import framework.rvv.configs.RvvParam -import framework.balldomain.configs.BallDomainParam -import framework.system.tile.configs.TileParam -import framework.top.configs.SimParam - -case class GlobalConfig( - memDomain: MemDomainParam, - frontend: FrontendParam, - rvv: RvvParam, - ballDomain: BallDomainParam, - tile: TileParam, - sim: SimParam, - coreSignature: BigInt = 0, - systemDesign: Option[framework.top.configs.SystemDesign] = None, - tileDesign: Option[framework.top.configs.TileScope] = None, - coreDesign: Option[framework.top.configs.CoreScope] = None) - extends SerializableModuleParameter - -object GlobalConfig { - - def apply(): GlobalConfig = { - GlobalConfig( - memDomain = MemDomainParam(), - frontend = FrontendParam(), - rvv = RvvParam(), - ballDomain = BallDomainParam(), - tile = TileParam(), - sim = SimParam() - ) - } - - implicit val rw: ReadWriter[GlobalConfig] = macroRW[GlobalConfig] - -} diff --git a/arch/src/main/scala/framework/top/configs/DesignConfig.scala b/arch/src/main/scala/framework/top/configs/DesignConfig.scala deleted file mode 100644 index 7a1fc5ef6..000000000 --- a/arch/src/main/scala/framework/top/configs/DesignConfig.scala +++ /dev/null @@ -1,130 +0,0 @@ -package framework.top.configs - -import upickle.default.{macroRW, ReadWriter} -import framework.frontend.configs.FrontendParam -import framework.memdomain.configs.MemDomainParam -import framework.rvv.configs.RvvParam -import framework.balldomain.configs.BallDomainParam -import framework.system.core.rocket.configs.RocketCpuParam -import framework.system.tile.configs.TileParam -import memcore.bus.chi -import memcore.bus.chi.rnf.RnfParams -import memcore.memory.cpu.PhysicalRegion -import memcore.memory.coherence.configs.CoherenceParams -import memcore.memory.cache.configs.CacheParams -import memcore.memory.{interlock, spm} -import memcore.memory.ddr.{Params => DdrParams} - -case class AcceleratorConfig( - memDomain: MemDomainParam, - frontend: FrontendParam, - rvv: RvvParam, - ballDomain: BallDomainParam, - tile: TileParam, - sim: SimParam, - coreSignature: BigInt) - -object AcceleratorConfig { implicit val rw: ReadWriter[AcceleratorConfig] = macroRW } - -case class CoreDesign( - factory: String, - rocket: Option[RocketCpuParam], - local: Option[framework.ant.Params], - accelerator: Option[AcceleratorConfig]) { - require(rocket.isDefined != local.isDefined) -} - -object CoreDesign { - import DesignCodecs._ - implicit val rw: ReadWriter[CoreDesign] = macroRW -} - -case class SharedConfig(bankBits: Int, bankEntries: Int, banks: Int) -object SharedConfig { implicit val rw: ReadWriter[SharedConfig] = macroRW } - -case class TileDesign( - factory: String, - param: TileParam, - main: Boolean, - cores: Seq[CoreDesign], - signatures: Seq[BigInt], - controller: Option[Int], - sharedStorage: Option[SharedConfig]) - -object TileDesign { implicit val rw: ReadWriter[TileDesign] = macroRW } - -case class DevicesConfig( - clintBase: BigInt, - clintBytes: BigInt, - tickCycles: Int, - plicBase: BigInt, - plicBytes: BigInt, - sources: Int, - priorityBits: Int, - bootrom: Option[RomConfig], - scuBase: BigInt, - scuStride: BigInt, - scuBytes: BigInt, - scuMaxHarts: Int) - -object DevicesConfig { implicit val rw: ReadWriter[DevicesConfig] = macroRW } -case class RomConfig(base: BigInt, bytes: BigInt, image: Seq[Byte]) -object RomConfig { implicit val rw: ReadWriter[RomConfig] = macroRW } - -case class PlatformConfig( - cpuPhysicalBits: Int, - memory: CoherenceParams, - l1: RnfParams, - regions: Seq[PhysicalRegion], - tracking: interlock.Params, - ddr: DdrParams, - devices: DevicesConfig) - -object PlatformConfig { - import DesignCodecs._ - implicit val rw: ReadWriter[PlatformConfig] = macroRW -} - -case class TilePlacement(design: TileDesign, hartIds: Seq[Int], executionIds: Seq[Int]) -object TilePlacement { implicit val rw: ReadWriter[TilePlacement] = macroRW } -case class SystemDesign(tiles: Seq[TilePlacement], platform: PlatformConfig) -object SystemDesign { implicit val rw: ReadWriter[SystemDesign] = macroRW } - -case class TileScope( - design: TileDesign, - platform: PlatformConfig, - maxHartId: Int, - controls: Int, - tiles: Int) - -object TileScope { implicit val rw: ReadWriter[TileScope] = macroRW } - -case class CoreScope( - factory: String, - rocket: Option[RocketCpuParam], - local: Option[framework.ant.Params], - hasAccelerator: Boolean, - controller: Option[RocketCpuParam], - tile: TileParam, - platform: PlatformConfig, - maxHartId: Int, - cpuIndex: Int, - cpuCount: Int, - signature: BigInt) - -object CoreScope { - import DesignCodecs._ - implicit val rw: ReadWriter[CoreScope] = macroRW -} - -object DesignCodecs { - implicit val chiRW: ReadWriter[chi.Params] = macroRW - implicit val cacheRW: ReadWriter[CacheParams] = macroRW - implicit val coherenceRW: ReadWriter[CoherenceParams] = macroRW - implicit val rnfRW: ReadWriter[RnfParams] = macroRW - implicit val regionRW: ReadWriter[PhysicalRegion] = macroRW - implicit val trackingRW: ReadWriter[interlock.Params] = macroRW - implicit val ddrRW: ReadWriter[DdrParams] = macroRW - implicit val spmRW: ReadWriter[spm.Params] = macroRW - implicit val antRW: ReadWriter[framework.ant.Params] = macroRW -} diff --git a/arch/src/main/scala/framework/top/configs/SimParam.scala b/arch/src/main/scala/framework/top/configs/SimParam.scala deleted file mode 100644 index 2dc810c81..000000000 --- a/arch/src/main/scala/framework/top/configs/SimParam.scala +++ /dev/null @@ -1,9 +0,0 @@ -package framework.top.configs - -import upickle.default._ - -case class SimParam(diffTest: Boolean = false) - -object SimParam { - implicit val rw: ReadWriter[SimParam] = macroRW -} diff --git a/arch/src/main/scala/sims/build.sc b/arch/src/main/scala/sims/build.sc new file mode 100644 index 000000000..9432aef2b --- /dev/null +++ b/arch/src/main/scala/sims/build.sc @@ -0,0 +1,17 @@ +import mill._ +import mill.scalalib._ +import $file.^.framework.{build => framework_build} + +val simRoot: os.Path = millSourcePath + +object library extends SbtModule { + override def millSourcePath = simRoot + override def scalaVersion = "2.13.16" + override def moduleDeps = Seq(framework_build.library) + override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0", ivy"com.lihaoyi::upickle:3.3.1") + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-language:reflectiveCalls", "-deprecation", "-feature", "-Ymacro-annotations") + override def sources = T.sources { + os.walk(simRoot).filter(path => path.ext == "scala" && !path.toString.contains("/out/")).map(PathRef(_)) + } +} diff --git a/arch/src/main/scala/sims/configs/SimParam.scala b/arch/src/main/scala/sims/configs/SimParam.scala new file mode 100644 index 000000000..92353a3c1 --- /dev/null +++ b/arch/src/main/scala/sims/configs/SimParam.scala @@ -0,0 +1,31 @@ +package sims.configs + +import buckyball.config.SimHardware + +import upickle.default._ + +case class MemoryParam( + lineBytes: Int, + clockHz: BigInt) + +object MemoryParam { implicit val rw: ReadWriter[MemoryParam] = macroRW } + +/** Differential debugging may delay completion; passive trace does not use this flag. */ +case class SimParam(diffTest: Boolean, memory: MemoryParam) + +object SimParam { + + def fromProto(p: SimHardware): SimParam = { + require(p.isInitialized, "Simulation configuration is incomplete") + val memory = p.getMemory + SimParam( + p.getDiffTest, + MemoryParam( + memory.getLineBytes, + BigInt(java.lang.Long.toUnsignedString(memory.getClockHz)) + ) + ) + } + + implicit val rw: ReadWriter[SimParam] = macroRW +} diff --git a/arch/src/main/scala/sims/core/CoreTarget.scala b/arch/src/main/scala/sims/core/CoreTarget.scala new file mode 100644 index 000000000..e31c0d89e --- /dev/null +++ b/arch/src/main/scala/sims/core/CoreTarget.scala @@ -0,0 +1,71 @@ +package sims.core + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, Instance, Instantiate} +import _root_.configs.GlobalConfig +import framework.core.clink.CLinkIO +import framework.device.dlink.BaseDevice +import framework.device.ddr.Bridge +import framework.device.peripherals.Controller +import framework.device.peripherals.bootrom.BootRomLines +import framework.ip.bus.axi4 +import framework.ip.bus.chi.snf.LineResponse +import framework.lib.arbiter.RRArbiter +import framework.lib.queue.{Queue => ResponseQueue} +import sims.soc.SystemTarget + +/** A registered core supplies its CLink; the simulator owns its external environment. */ +abstract class CoreTarget(b: GlobalConfig) extends SystemTarget(b) { + def instantiateCore(): CLinkIO + def instantiate(b: GlobalConfig): Instance[BaseDevice] = + Instantiate(new CoreEnvironment(b, () => instantiateCore())) +} + +@instantiable +class CoreEnvironment(b: GlobalConfig, instantiateCore: () => CLinkIO) extends BaseDevice(b.device.ddr.axi) { + val Seq(corep) = b.device.cores + val core = instantiateCore() + require(core.p.harts == 1 && core.p.backing && core.p.requesters == 0) + require(core.c2c.isEmpty && core.banks.isEmpty) + require(b.device.ddr.clients == 1) + + val peripherals = Instantiate(new Controller(b.device, b.device.ddr.axi, corep.memory.memoryAccess.memory)) + + val decoder = Instantiate(new axi4.Decoder( + Seq( + (b.device.clint.base, b.device.clint.bytes), + (b.device.plic.base, b.device.plic.bytes), + (b.device.scu.baseAddress, b.device.scu.totalSizeBytes) + ), + b.device.ddr.axi + )) + + decoder.io.in <> core.memory.uncached + peripherals.io.axi <> decoder.io.matched + peripherals.io.hardwareInterrupt.sources := 0.U + core.interrupts := peripherals.io.hardwareInterrupt.harts + + val bridge = Instantiate(new Bridge(b.device.ddr)) + val memory = Instantiate(new axi4.Interconnect(axi4.InterconnectParam(b.device.ddr.axi, 3, None))) + memory.io.in(0) <> core.memory.ddr + memory.io.in(1) <> decoder.io.other + memory.io.in(2) <> bridge.io.axi + dlink.axi <> memory.io.out + + val rom = Instantiate(new BootRomLines(b.device.bootrom, corep.memory.home.memory.chi)) + val backing = core.memory.backing.get + val inRom = backing.request.bits.addr >= b.device.bootrom.base.U && + backing.request.bits.addr < (b.device.bootrom.base + b.device.bootrom.bytes).U + rom.io.request.valid := backing.request.valid && inRom + rom.io.request.bits := backing.request.bits + bridge.io.request(0).valid := backing.request.valid && !inRom + bridge.io.request(0).bits := backing.request.bits + backing.request.ready := Mux(inRom, rom.io.request.ready, bridge.io.request(0).ready) + val answers = Instantiate(new RRArbiter(new LineResponse(corep.memory.home.memory.chi), 2)) + answers.io.in(0) <> rom.io.response + answers.io.in(1) <> bridge.io.response(0) + val response = Instantiate(new ResponseQueue(new LineResponse(corep.memory.home.memory.chi), 1, false, false)) + response.io.enq <> answers.io.out + backing.response <> response.io.deq +} diff --git a/arch/src/main/scala/sims/hash/BankHash.scala b/arch/src/main/scala/sims/hash/BankHash.scala deleted file mode 100644 index 32d4482a5..000000000 --- a/arch/src/main/scala/sims/hash/BankHash.scala +++ /dev/null @@ -1,75 +0,0 @@ -package sims.hash - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public} -import chisel3.util._ -import framework.top.GlobalConfig - -class BankHashWrite(val b: GlobalConfig) extends Bundle { - val addr = UInt(log2Ceil(b.memDomain.bankEntries).W) - val mask = Vec(b.memDomain.bankMaskLen, Bool()) - val data = UInt(b.memDomain.bankWidth.W) -} - -@instantiable -class BankHashMonitor(val b: GlobalConfig) extends Module { - require(b.memDomain.bankWidth == 128, "bank hash requires 128-bit bank rows") - require(b.memDomain.bankMaskLen == 16, "bank hash requires byte write masks") - - @public - val io = IO(new Bundle { - val bind = Input(Bool()) - val write = Flipped(Decoupled(new BankHashWrite(b))) - val statusHash = Output(UInt(32.W)) - }) - - val shadow = SyncReadMem(b.memDomain.bankEntries, UInt(128.W)) - val validRows = RegInit(VecInit(Seq.fill(b.memDomain.bankEntries)(false.B))) - val generation = RegInit(0.U(8.W)) - val statusHash = RegInit(0.U(32.W)) - - def rotateLeft(value: UInt, amount: Int): UInt = - Cat(value(31 - amount, 0), value(31, 32 - amount)) - - def rowHash(addr: UInt, row: UInt): UInt = { - val address = addr.pad(32) - val mixed = row(31, 0) ^ rotateLeft(row(63, 32), 7) ^ rotateLeft(row(95, 64), 13) ^ - rotateLeft(row(127, 96), 21) ^ rotateLeft(address, 11) - Mux(row.orR, mixed, 0.U) - } - - val writeValid = RegInit(false.B) - val writeBits = Reg(new BankHashWrite(b)) - val writeGeneration = Reg(UInt(generation.getWidth.W)) - val commit = writeValid && writeGeneration === generation && !io.bind - io.write.ready := !writeValid && !io.bind - val oldRow = Wire(UInt(128.W)) - val oldBytes = oldRow.asTypeOf(Vec(16, UInt(8.W))) - val writeBytes = writeBits.data.asTypeOf(Vec(16, UInt(8.W))) - val newBytes = VecInit((0 until 16).map(i => Mux(writeBits.mask(i), writeBytes(i), oldBytes(i)))) - val newRow = newBytes.asUInt - val nextHash = statusHash - rowHash(writeBits.addr, oldRow) + rowHash(writeBits.addr, newRow) - - val readData = - shadow.readWrite(Mux(commit, writeBits.addr, io.write.bits.addr), newRow, io.write.fire || commit, commit) - oldRow := Mux(validRows(writeBits.addr), readData, 0.U) - - when(io.write.fire) { - writeValid := true.B - writeBits := io.write.bits - writeGeneration := generation - } - - io.statusHash := statusHash - - when(io.bind) { - generation := generation + 1.U - statusHash := 0.U - writeValid := false.B - validRows.foreach(_ := false.B) - }.elsewhen(commit) { - validRows(writeBits.addr) := true.B - statusHash := nextHash - writeValid := false.B - } -} diff --git a/arch/src/main/scala/sims/p2e/P2ETop.scala b/arch/src/main/scala/sims/p2e/P2ETop.scala index d83f7e650..421c2838e 100644 --- a/arch/src/main/scala/sims/p2e/P2ETop.scala +++ b/arch/src/main/scala/sims/p2e/P2ETop.scala @@ -1,16 +1,22 @@ package sims.p2e +import chisel3.experimental.hierarchy.public + +import chisel3.experimental.hierarchy.instantiable + +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import _root_.configs.GlobalConfig + import chisel3._ import chisel3.experimental.Analog import chisel3.util._ -import memcore.bus.axi4 -import memcore.memory.queue.Queue +import framework.ip.bus.axi4 +import framework.lib.queue.Queue import sims.soc.{SimSoc, SystemTarget} -/** A chip on the P2E board: the System over the 16-GiB DDR4 macro at 0x80000000. */ -abstract class P2ETarget(pb: String) extends SystemTarget(pb) { - override val dramBytes: BigInt = BigInt(16) << 30 -} +/** A design on the P2E board: the System over the 16-GiB DDR4 macro at 0x80000000. */ +abstract class P2ETarget(b: GlobalConfig) extends SystemTarget(b) /** Board pins of the P2E top; bebop's Tcl drives `sys_rstn`/`soc_hold` and reads the status pins. */ class P2ETopIO extends Bundle { @@ -46,7 +52,9 @@ class P2ETopIO extends Bundle { } /** AXI slave port of the DDR4 macro, as named on the macro (`_ddr4_s_axi_*`). */ -class Ddr4AxiSlave(idBits: Int) extends Bundle { +class Ddr4AxiSlave(port: Int) extends Bundle { + val idBits = XepicDdr4Dc1.portIdBits(port) + val awid = Input(UInt(idBits.W)) val awaddr = Input(UInt(64.W)) val awlen = Input(UInt(8.W)) @@ -86,10 +94,14 @@ class Ddr4AxiSlave(idBits: Int) extends Bundle { val rvalid = Output(Bool()) } +object XepicDdr4Dc1 { val portIdBits = Seq(11, 4) } + /** The board DDR4 controller; vcom replaces this stub with the `xepic_ddr4_dc1` netlist macro. */ +@instantiable class XepicDdr4Dc1 extends BlackBox { override def desiredName = "xepic_ddr4_dc1" + @public val io = IO(new Bundle { val sys_rstn = Input(Bool()) val c0_sys_clk_p = Input(Bool()) @@ -121,8 +133,10 @@ class XepicDdr4Dc1 extends BlackBox { val axi_clk = Input(Clock()) val c0_ddr4_ui_clk = Output(Clock()) // Flattened as the macro's `s0_ddr4_s_axi_*` and `s1_ddr4_s_axi_*` ports. - val s0_ddr4_s_axi = new Ddr4AxiSlave(11) - val s1_ddr4_s_axi = new Ddr4AxiSlave(4) + val s0_ddr4_s_axi = + new Ddr4AxiSlave(0) + val s1_ddr4_s_axi = + new Ddr4AxiSlave(1) }) } @@ -132,19 +146,26 @@ class XepicDdr4Dc1 extends BlackBox { * narrow bursts, so only lanes move: beat k of a burst starting at A uses lane (A + 16k)[4]. * CPU 0x80000000 is DDR offset 0, and the System's IDs widen to the macro's 11 bits. */ -class P2EDdrAdapter(system: axi4.Params, base: BigInt) extends Module { +@instantiable +class P2EDdrAdapter(b: GlobalConfig) extends Module { + val system = b.device.ddr.axi + val base = b.device.ddr.base + require(system.dataBits == 128, "P2E DDR adapter maps 16-byte beats onto the 32-byte macro bus") - require(system.idBits <= 11) + require(system.idBits <= XepicDdr4Dc1.portIdBits.head) + @public val io = IO(new Bundle { - val in = Flipped(new axi4.Port(system)) - val out = Flipped(new Ddr4AxiSlave(11)) + val in = Flipped(new axi4.Port(b.device.ddr.axi)) + val out = + Flipped(new Ddr4AxiSlave(0)) }) val lanes = 1 << system.idBits // Write lanes: W beats follow AW order, so queue each accepted burst's first lane. - val writeLane = Module(new Queue(Bool(), 8)) + val writeLane = Instantiate(new Queue(Bool(), 8, false, false)) + val writeOdd = RegInit(false.B) val writeUpper = writeLane.io.deq.bits ^ writeOdd io.out.awid := io.in.aw.bits.id @@ -222,10 +243,14 @@ class P2EDdrAdapter(system: axi4.Params, base: BigInt) extends Module { * DDR adapter and the System, all on `user_clk`. The SoC leaves reset only after calibration and * after the host releases `soc_hold`. */ -class P2EHarness(target: SystemTarget, diffTest: Boolean) extends RawModule { +@instantiable +class P2EHarness(b: GlobalConfig, target: SystemTarget) extends RawModule { + val diffTest = b.sim.diffTest + + @public val io = FlatIO(new P2ETopIO) - val ddr = Module(new XepicDdr4Dc1) + val ddr = Instantiate(new XepicDdr4Dc1) ddr.io.sys_rstn := io.sys_rstn ddr.io.c0_sys_clk_p := io.c0_sys_clk_p ddr.io.c0_sys_clk_n := io.c0_sys_clk_n @@ -257,8 +282,11 @@ class P2EHarness(target: SystemTarget, diffTest: Boolean) extends RawModule { val socReset = !io.sys_rstn || !ddr.io.c0_init_calib_complete || io.soc_hold withClockAndReset(io.user_clk, socReset) { - val soc = Module(new SimSoc(target, diffTest)) - val adapter = Module(new P2EDdrAdapter(soc.axiParams, target.dramBase)) + val soc = Instantiate(new SimSoc( + b, + target + )) + val adapter = Instantiate(new P2EDdrAdapter(b)) adapter.io.in <> soc.io.axi ddr.io.s0_ddr4_s_axi <> adapter.io.out } @@ -273,9 +301,16 @@ class P2EHarness(target: SystemTarget, diffTest: Boolean) extends RawModule { } /** The P2E top module bebop's vvac/vcom flow builds; `P2ETop.top` is the harness. */ -class P2ETop(target: SystemTarget, diffTest: Boolean) extends RawModule { - val io = IO(new P2ETopIO) - val top = Module(new P2EHarness(target, diffTest)) +class P2ETop(b: GlobalConfig, target: SystemTarget) extends RawModule { + val diffTest = b.sim.diffTest + + val io = IO(new P2ETopIO) + + val top = Instantiate(new P2EHarness( + b, + target + )) + // The DDR4 data pins stay on the macro inside `top`: vcom binds the macro's own pins, and a top // inout driving the macro is rejected. for ((name, port) <- io.elements) port match { diff --git a/arch/src/main/scala/sims/p2e/TargetConfig.scala b/arch/src/main/scala/sims/p2e/TargetConfig.scala index eb0812a48..6d6b2a51d 100644 --- a/arch/src/main/scala/sims/p2e/TargetConfig.scala +++ b/arch/src/main/scala/sims/p2e/TargetConfig.scala @@ -1,46 +1,28 @@ package sims.p2e import _root_.circt.stage.ChiselStage +import sims.soc.SystemTarget -//===----------------------------------------------------------------------===// -object Elaborate extends App { - if (args.isEmpty) { - println("Usage: Elaborate [--difftest] [firtool-opts...] -o=") - println("Example: Elaborate sims.p2e.PebbleP2ETarget") - sys.exit(1) - } - val targetClassName = args(0) - println(s"Elaborating P2ETop with target: $targetClassName") - - val target: P2ETarget = - try { - Class.forName(targetClassName).getDeclaredConstructor().newInstance() match { - case p2e: P2ETarget => p2e - case _ => - println(s"Error: $targetClassName is not a P2ETarget") - sys.exit(1) - } - } catch { - case e: ClassNotFoundException => - println(s"Error: target class not found: $targetClassName") - sys.exit(1) - } - - val rawFirtoolOpts = args.drop(1) +/** The design owns configuration decoding and concrete component types. */ +object Elaborate { - val outDir = rawFirtoolOpts.collectFirst { - case opt if opt.startsWith("-o=") => opt.stripPrefix("-o=") - }.getOrElse { - throw new Exception("missing -o= in firtool opts") + def main(args: Array[String]): Unit = { + require(args.length >= 6 && args(0) == "--config" && args(2) == "--target-class" && args(4) == "--target-dir") + val original = buckyball.config.Design.parseFrom(java.nio.file.Files.readAllBytes(java.nio.file.Paths.get(args(1)))) + val options = args.drop(6) + val design = original.toBuilder.setSims(original.getSims.toBuilder + .setDiffTest(original.getSims.getDiffTest || options.contains("--difftest"))).build() + val target = Class.forName(args(3)).getConstructor(classOf[buckyball.config.Design]) + .newInstance(design).asInstanceOf[SystemTarget] + emit(target, args(5), options.filterNot(_ == "--difftest")) } - val firtoolOpts = rawFirtoolOpts.filterNot { opt => - opt == "--split-verilog" || opt == "--difftest" || opt.startsWith("-o=") + def emit(target: SystemTarget, directory: String, firtoolOpts: Array[String]): Unit = { + ChiselStage.emitSystemVerilogFile( + new P2ETop(target.b, target), + args = Array("--target-dir", directory, "--split-verilog"), + firtoolOpts = firtoolOpts + ) } - ChiselStage.emitSystemVerilogFile( - new P2ETop(target, diffTest = rawFirtoolOpts.contains("--difftest")), - firtoolOpts = firtoolOpts, - args = Array("--target-dir", outDir, "--split-verilog") - ) } diff --git a/arch/src/main/scala/sims/scu/SCU.scala b/arch/src/main/scala/sims/scu/SCU.scala deleted file mode 100644 index bf4f8587d..000000000 --- a/arch/src/main/scala/sims/scu/SCU.scala +++ /dev/null @@ -1,162 +0,0 @@ -package sims.scu - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public} -import chisel3.util.HasBlackBoxInline - -/** - * SCU is a global multi-hart System Control Unit (see SystemControl). It - * provides a per-hart sub-region for UART output and simulation exit. Each hart accesses its own sub-region via - * `baseAddress + hartId * strideBytes`. - * - * Address layout (default values): - * baseAddress = 0x60000000 - * strideBytes = 0x40000 (256 KiB per hart) - * totalSizeBytes = 0x10000000 (256 MiB total, supports up to 1024 harts) - * maxHarts = 64 - * - * Within each hart's sub-region: - * +0x00000: simExit register (write triggers $finish with exit code) - * +0x20000: uartTx register (write outputs a byte via DPI-C) - * +0x20004: uartRx register (read pops a byte from host input) - * +0x20005: status register (read-only, bit0 rx-valid, bit5/6 tx-ready) - * +0x20006: ready register (read/write software-visible byte) - * - * @param baseAddress physical base address (must be aligned to totalSizeBytes) - * @param strideBytes bytes per hart (must be power of two) - * @param totalSizeBytes total address window (must be power of two, - * >= maxHarts * strideBytes) - * @param maxHarts number of hart sub-regions. Accesses targeting - * hartId >= maxHarts return an access error. - */ -case class SCUParams( - baseAddress: BigInt = BigInt("60000000", 16), - strideBytes: BigInt = BigInt("40000", 16), - totalSizeBytes: BigInt = BigInt("10000000", 16), - maxHarts: Int = 64) { - require( - strideBytes > 0 && (strideBytes & (strideBytes - 1)) == 0, - s"SCU strideBytes ($strideBytes) must be a positive power of two" - ) - require( - totalSizeBytes > 0 && (totalSizeBytes & (totalSizeBytes - 1)) == 0, - s"SCU totalSizeBytes ($totalSizeBytes) must be a positive power of two" - ) - require(maxHarts > 0, s"SCU maxHarts ($maxHarts) must be positive") - require( - BigInt(maxHarts) * strideBytes <= totalSizeBytes, - s"SCU maxHarts ($maxHarts) * strideBytes ($strideBytes) = " + - s"${BigInt(maxHarts) * strideBytes} exceeds totalSizeBytes ($totalSizeBytes)" - ) -} - -/** - * Single DPI-C bridge module for all harts. The hart_id is supplied as an - * input signal rather than being baked into the module name, so only one - * BlackBox is generated and DPI-C imports are not duplicated. - * - * Has separate hart_id inputs for uart and exit operations because different - * harts may write uart and exit simultaneously. - */ -@instantiable -class SCUWriteDPI extends BlackBox with HasBlackBoxInline { - override def desiredName = "SCUWriteDPI" - - @public - val io = IO(new Bundle { - val clock = Input(Clock()) - val reset = Input(Bool()) - val uart_hart_id = Input(UInt(32.W)) - val uart_valid = Input(Bool()) - val uart_data = Input(UInt(8.W)) - val exit_hart_id = Input(UInt(32.W)) - val exit_valid = Input(Bool()) - val exit_code = Input(UInt(32.W)) - }) - - setInline( - "SCUWriteDPI.v", - s""" - |import "DPI-C" context function void scu_uart_write(input int unsigned hart_id, input int unsigned ch); - |import "DPI-C" context function void scu_sim_exit(input int unsigned hart_id, input int unsigned code); - | - |module SCUWriteDPI( - | input clock, - | input reset, - | input [31:0] uart_hart_id, - | input uart_valid, - | input [7:0] uart_data, - | input [31:0] exit_hart_id, - | input exit_valid, - | input [31:0] exit_code - |); - | always @(posedge clock) begin - | if (!reset) begin - | if (uart_valid) begin - | scu_uart_write(uart_hart_id, {24'h0, uart_data}); - | end - | - | if (exit_valid) begin - | scu_sim_exit(exit_hart_id, exit_code); - | end - | end - | end - |endmodule - """.stripMargin - ) -} - -@instantiable -class SCUReadDPI extends BlackBox with HasBlackBoxInline { - override def desiredName = "SCUReadDPI" - - @public - val io = IO(new Bundle { - val clock = Input(Clock()) - val reset = Input(Bool()) - val hart_id = Input(UInt(32.W)) - val enable = Input(Bool()) - val pop = Input(Bool()) - val rx_valid = Output(Bool()) - val rx_data = Output(UInt(8.W)) - }) - - setInline( - "SCUReadDPI.v", - s""" - |import "DPI-C" context function void scu_uart_rx_sample( - | input int unsigned hart_id, - | input int unsigned pop, - | output int unsigned valid, - | output int unsigned data - |); - | - |module SCUReadDPI( - | input clock, - | input reset, - | input [31:0] hart_id, - | input enable, - | input pop, - | output reg rx_valid, - | output reg [7:0] rx_data - |); - | integer valid; - | integer data; - | always @(posedge clock) begin - | if (reset) begin - | rx_valid <= 1'b0; - | rx_data <= 8'h0; - | end else if (enable) begin - | scu_uart_rx_sample(hart_id, {31'h0, pop}, valid, data); - | if (pop) begin - | rx_valid <= 1'b0; - | end else begin - | rx_valid <= (valid != 0); - | rx_data <= data[7:0]; - | end - | end - | end - |endmodule - """.stripMargin - ) -} diff --git a/arch/src/main/scala/sims/scu/SystemControl.scala b/arch/src/main/scala/sims/scu/SystemControl.scala deleted file mode 100644 index b6316f324..000000000 --- a/arch/src/main/scala/sims/scu/SystemControl.scala +++ /dev/null @@ -1,114 +0,0 @@ -package sims.scu - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import chisel3.util._ -import memcore.memory.cpu.{CpuMemParams, UncachedRequest, UncachedResponse} - -/** - * SCU on the System device ports, with the register layout documented on SCUParams. The hart sub-region is - * selected by address, so software keeps using `base + hartId * stride` regardless of port. - * Data is right-aligned at `addr`, as on every uncached CPU port. - */ -@instantiable -class SystemControl(ports: Int, p: CpuMemParams, params: SCUParams = SCUParams()) extends Module { - val strideBits = log2Ceil(params.strideBytes) - - @public - val io = IO(new Bundle { - val request = Vec(ports, Flipped(Decoupled(new UncachedRequest(p)))) - val response = Vec(ports, Decoupled(new UncachedResponse(p))) - val hartIds = Input(Vec(ports, UInt(32.W))) - val failure = Input(Vec(ports, Valid(new framework.memdomain.frontend.mem.dma.DmaStatus))) - val workDrained = Input(Vec(ports, Bool())) - }) - - val faulted = SystemControl.observe(io.failure.toSeq, io.hartIds.toSeq) - - for (i <- 0 until ports) { - val request = io.request(i) - val response = io.response(i) - val write: Instance[SCUWriteDPI] = Instantiate(new SCUWriteDPI) - val read: Instance[SCUReadDPI] = Instantiate(new SCUReadDPI) - val ready = RegInit(0.U(8.W)) - - val pending = RegInit(false.B) - val tag = Reg(UInt(p.tagBits.W)) - val error = Reg(Bool()) - val data = Reg(UInt(64.W)) - val status = Reg(Bool()) - - val offset = request.bits.addr - params.baseAddress.U - val hart = offset(31, strideBits) - val register = offset(strideBits - 1, 0) - val inside = request.bits.addr >= params.baseAddress.U && - request.bits.addr < (params.baseAddress + params.totalSizeBytes).U && hart < params.maxHarts.U - val fire = request.fire - val wr = request.bits.write - val exit = inside && register === 0x00000.U && wr - val uartTx = inside && register === 0x20000.U && wr - val uartRx = inside && register === 0x20004.U && !wr - val uartSt = inside && register === 0x20005.U && !wr - val readyReg = inside && register === 0x20006.U - - write.io.clock := clock - write.io.reset := reset.asBool - write.io.exit_hart_id := hart - write.io.exit_valid := fire && exit && !faulted - write.io.exit_code := request.bits.data(31, 0) - write.io.uart_hart_id := hart - write.io.uart_valid := fire && uartTx - write.io.uart_data := request.bits.data(7, 0) - - // The DPI samples on this edge; status reads answer with the fresh sample next cycle. - read.io.clock := clock - read.io.reset := reset.asBool - read.io.hart_id := hart - read.io.enable := fire && (uartRx || uartSt) - read.io.pop := fire && uartRx - - request.ready := !pending && (!request.valid || !exit || io.workDrained(i)) - when(fire) { - pending := true.B - tag := request.bits.tag - error := !(exit || uartTx || uartRx || uartSt || readyReg) - status := uartSt - data := Mux(uartRx, read.io.rx_data, Mux(readyReg && !wr, ready, 0.U)) - when(readyReg && wr)(ready := request.bits.data(7, 0)) - } - - response.valid := pending - response.bits.tag := tag - response.bits.error := error - response.bits.data := Mux(status, "h60".U | read.io.rx_valid.asUInt, data) - when(response.fire)(pending := false.B) - } -} - -object SystemControl { - - def observe(failures: Seq[ValidIO[framework.memdomain.frontend.mem.dma.DmaStatus]], hartIds: Seq[UInt]): Bool = { - val failed = RegInit(false.B) - val pending = VecInit(failures.map(_.valid)) - val first = PriorityEncoder(pending) - val exit = Instantiate(new SCUWriteDPI) - exit.io.clock := Module.clock - exit.io.reset := Module.reset.asBool - exit.io.uart_hart_id := 0.U - exit.io.uart_valid := false.B - exit.io.uart_data := 0.U - exit.io.exit_hart_id := VecInit(hartIds)(first) - exit.io.exit_valid := pending.asUInt.orR && !failed - exit.io.exit_code := 0x200.U | VecInit(failures.map(_.bits.error))(first) - when(pending.asUInt.orR)(failed := true.B) - for ((failure, index) <- failures.zipWithIndex) { - when(failure.valid && !failed && first === index.U) { - printf( - cf"[SCU] hart ${hartIds(index)} NPU failure: error=${failure.bits.error} address=0x${failure.bits.address}%x\n" - ) - } - } - pending.asUInt.orR || failed - } - -} diff --git a/arch/src/main/scala/sims/soc/SimSoc.scala b/arch/src/main/scala/sims/soc/SimSoc.scala index e4f37085f..e9bafb0b4 100644 --- a/arch/src/main/scala/sims/soc/SimSoc.scala +++ b/arch/src/main/scala/sims/soc/SimSoc.scala @@ -1,99 +1,38 @@ package sims.soc -import chisel3._ -import chisel3.experimental.hierarchy.Instance -import framework.system.{System, SystemParams} -import framework.system.configloader.{AntTileCore, ChipLoader, RocketTileCore} -import framework.system.device.{BootRom, DeviceParams} -import memcore.bus.axi4 -import memcore.bus.chi.{Params => ChiParams} -import memcore.bus.chi.rnf.RnfParams -import memcore.memory.cache.configs.CacheParams -import memcore.memory.coherence.configs.CoherenceParams -import memcore.memory.cpu.{CpuMemParams, PhysicalRegion} -import memcore.memory.ddr.{Params => DdrParams} -import memcore.memory.interlock.{Params => TrackingParams} -import sims.scu.{SCUParams} - -/** A chip on the explicit System: one PB plus the platform's DRAM window. */ -abstract class SystemTarget(val pb: String) { - def instantiate(p: SystemParams): Instance[System] - val dramBase: BigInt = BigInt("80000000", 16) - val dramBytes: BigInt = BigInt(512) << 20 -} - -/** - * The explicit System with its SCU and NPU-fault abort. Every platform (Verilator, P2E) supplies - * the DRAM behind `io.axi` and nothing else. - */ -class SimSoc(target: SystemTarget, diffTest: Boolean, mainOnly: Boolean = false) extends Module { - - val loaded = ChipLoader.load(target.pb) - - val topology = loaded.copy(tiles = loaded.tiles.filter(tile => !mainOnly || tile.main).map(tile => - tile.copy(cores = tile.cores.map { - case RocketTileCore(cpu, Some(config)) => - RocketTileCore(cpu, Some(config.copy(sim = config.sim.copy(diffTest = diffTest)))) - case AntTileCore(local, config) => - AntTileCore(local, config.copy(sim = config.sim.copy(diffTest = diffTest))) - case core => core - }) - )) +import chisel3.experimental.hierarchy.public - val scu = SCUParams() +import chisel3.experimental.hierarchy.instantiable - val chi = ChiParams() +import _root_.configs.GlobalConfig - // Per-tile template (System sets each tile's agent count). Tags and directories are still - // registers in these IPs; sizes stay modest until they move to SRAM. - val memory = CoherenceParams( - chi, - CacheParams(chi.addressBits, 64, 64, 4, 8, 4, 2), - agents = 2, - mshrEntries = 8, - homeId = math.max(64, 2 * topology.tiles.map(_.hartIds.size).sum + 1) - ) - - // Every hart starts in the BootROM, which validates mhartid and jumps to the DRAM entry. - val hartIdList = topology.tiles.flatMap(_.hartIds) - require(hartIdList.sorted == hartIdList.indices, "BootROM validates harts as 0 until the hart count") - val bootrom = BootRom(BigInt("10000", 16), BigInt("10000", 16), hartIdList.size, target.dramBase) - val devices = DeviceParams(bootrom = Some(bootrom), scu = scu) +import chisel3._ +import chisel3.experimental.hierarchy.Instance +import framework.device.dlink.BaseDevice +import framework.device.configs.DeviceParam +import framework.ip.bus.axi4 +import sims.configs.SimParam + +/** A design supplies its parameters and explicitly instantiates its DLink implementation. */ +abstract class SystemTarget(val b: GlobalConfig) { + def instantiate(b: GlobalConfig): Instance[BaseDevice] + val dramBase: BigInt = b.device.ddr.base + val dramBytes: BigInt = b.device.ddr.bytes +} - val regions = Seq( - PhysicalRegion(target.dramBase, target.dramBytes, true, true, true, true, true, true), - PhysicalRegion(bootrom.base, bootrom.bytes, true, true, true, false, false, true), - PhysicalRegion(devices.clint.base, devices.clint.bytes, false, false, true, true, false, false), - PhysicalRegion(devices.plic.base, devices.plic.bytes, false, false, true, true, false, false), - PhysicalRegion(scu.baseAddress, scu.totalSizeBytes, false, false, true, true, false, false) - ) +@instantiable +class SimSoc(b: GlobalConfig, target: SystemTarget) extends Module { + val diffTest = b.sim.diffTest - val slotsPerClient = 16 + val p = b.device - val ddr = DdrParams( - chi, - clients = 1, - slotsPerClient = slotsPerClient, - dataBits = 128, - idBits = 4 - ) + @public + val io = IO(new Bundle { - val axiParams: axi4.Params = ddr.axi + val axi = new axi4.Port(p.ddr.axi) - val io = IO(new Bundle { - val axi = new axi4.Port(axiParams) }) - val system: Instance[System] = target.instantiate(SystemParams( - topology, - 36, - memory, - RnfParams(chi, cacheLines = 64, banks = 2, homeId = memory.homeId), - regions, - TrackingParams(addressBits = chi.addressBits), - ddr, - devices - )) - + val system: Instance[BaseDevice] = target.instantiate(b) io.axi <> system.dlink.axi } diff --git a/arch/src/main/scala/sims/verilator/BBsimDRAM.scala b/arch/src/main/scala/sims/verilator/BBsimDRAM.scala index 5d8c1f9b1..6b0f9277a 100644 --- a/arch/src/main/scala/sims/verilator/BBsimDRAM.scala +++ b/arch/src/main/scala/sims/verilator/BBsimDRAM.scala @@ -1,48 +1,49 @@ package sims.verilator +import _root_.configs.GlobalConfig + import chisel3._ import chisel3.experimental.hierarchy.{instantiable, public} import chisel3.experimental.IntParam import chisel3.util.HasBlackBoxInline -import memcore.bus.axi4.{Params => AxiParams, Port => AxiPort} +import framework.ip.bus.axi4.{Params => AxiParams, Port => AxiPort} @instantiable -class BBSimDRAM( - memSize: BigInt, - lineSize: Int, - clockFreqHz: BigInt, - memBase: BigInt, - params: AxiParams, - chipId: Int) +class BBSimDRAM(b: GlobalConfig, deviceId: Int) extends BlackBox( Map( - "MEM_SIZE" -> IntParam(memSize), - "LINE_SIZE" -> IntParam(lineSize), - "ADDR_BITS" -> IntParam(params.addressBits), - "DATA_BITS" -> IntParam(params.dataBits), - "ID_BITS" -> IntParam(params.idBits), - "CLOCK_HZ" -> IntParam(clockFreqHz), - "MEM_BASE" -> IntParam(memBase), - "CHIP_ID" -> IntParam(chipId) + "MEM_SIZE" -> IntParam(b.device.ddr.bytes), + "LINE_SIZE" -> IntParam(b.sim.memory.lineBytes), + "ADDR_BITS" -> IntParam(b.device.ddr.axi.addressBits), + "DATA_BITS" -> IntParam(b.device.ddr.axi.dataBits), + "ID_BITS" -> IntParam(b.device.ddr.axi.idBits), + "CLOCK_HZ" -> IntParam(b.sim.memory.clockHz), + "MEM_BASE" -> IntParam(b.device.ddr.base), + "DEVICE_ID" -> IntParam(deviceId) ) ) with HasBlackBoxInline { + val memSize = b.device.ddr.bytes + val lineSize = b.sim.memory.lineBytes + val clockFreqHz = b.sim.memory.clockHz + val memBase = b.device.ddr.base + val axiParams = b.device.ddr.axi @public val io = IO(new Bundle { val clock = Input(Clock()) val reset = Input(Reset()) - val axi = Flipped(new AxiPort(params)) + val axi = Flipped(new AxiPort(b.device.ddr.axi)) }) - require(Set(64, 128).contains(params.dataBits), "BBSimDRAM supports 64 or 128 bit AXI beats") + require(Set(64, 128).contains(axiParams.dataBits), "BBSimDRAM supports 64 or 128 bit AXI beats") setInline( "BBSimDRAM.v", """ |import "DPI-C" function chandle bbsim_memory_init |( - | input int chip_id, + | input int device_id, | input longint mem_size, | input longint word_size, | input longint line_size, @@ -98,7 +99,7 @@ class BBSimDRAM( | CLOCK_HZ = 100000, | STRB_BITS = DATA_BITS/8, | MEM_BASE = 0, - | CHIP_ID = 0)( + | DEVICE_ID = 0)( | input clock, | input reset, | output axi_aw_ready, @@ -191,7 +192,7 @@ class BBSimDRAM( | | always @(posedge clock) begin | if (!initialized) begin - | channel = bbsim_memory_init(CHIP_ID, MEM_SIZE, WORD_SIZE, LINE_SIZE, ID_BITS, CLOCK_HZ, MEM_BASE); + | channel = bbsim_memory_init(DEVICE_ID, MEM_SIZE, WORD_SIZE, LINE_SIZE, ID_BITS, CLOCK_HZ, MEM_BASE); | initialized = 1'b1; | end | diff --git a/arch/src/main/scala/sims/verilator/BdbClkDPI.scala b/arch/src/main/scala/sims/verilator/BdbClkDPI.scala index 65cf198dc..685d76c96 100644 --- a/arch/src/main/scala/sims/verilator/BdbClkDPI.scala +++ b/arch/src/main/scala/sims/verilator/BdbClkDPI.scala @@ -25,6 +25,7 @@ class BdbClkDPI extends BlackBox with HasBlackBoxInline { | input wire clock, | input wire reset |); + |`ifndef BEBOP_PARALLEL | reg [63:0] cnt; | always @(posedge clock) begin | if (reset) begin @@ -35,6 +36,7 @@ class BdbClkDPI extends BlackBox with HasBlackBoxInline { | cnt <= cnt + 64'd1; | end | end + |`endif |endmodule """.stripMargin ) diff --git a/arch/src/main/scala/sims/verilator/SystemHarness.scala b/arch/src/main/scala/sims/verilator/SystemHarness.scala index 7e341971f..c024ce447 100644 --- a/arch/src/main/scala/sims/verilator/SystemHarness.scala +++ b/arch/src/main/scala/sims/verilator/SystemHarness.scala @@ -1,22 +1,33 @@ package sims.verilator +import _root_.configs.GlobalConfig + import chisel3._ import chisel3.experimental.hierarchy.Instantiate import chisel3.experimental.hierarchy.Instance import sims.soc.{SimSoc, SystemTarget} /** Verilator top for the explicit System. Keeps the BBSimHarness name bebop builds against. */ -class SystemHarness(target: SystemTarget, diffTest: Boolean, mainOnly: Boolean = false) extends Module { +class SystemHarness(b: GlobalConfig, target: SystemTarget) extends Module { + val diffTest = b.sim.diffTest + override def desiredName = "BBSimHarness" val bdbClkDpi: Instance[BdbClkDPI] = Instantiate(new BdbClkDPI) bdbClkDpi.io.clock := clock bdbClkDpi.io.reset := reset.asBool - val soc = Module(new SimSoc(target, diffTest, mainOnly)) + val soc = Instantiate(new SimSoc( + b, + target + )) val dram: Instance[BBSimDRAM] = - Instantiate(new BBSimDRAM(target.dramBytes, 64, 1000, target.dramBase, soc.axiParams, chipId = 0)) + Instantiate(new BBSimDRAM( + b, + 0 + )) + dram.io.clock := clock dram.io.reset := reset dram.io.axi <> soc.io.axi diff --git a/arch/src/main/scala/sims/verilator/TargetConfig.scala b/arch/src/main/scala/sims/verilator/TargetConfig.scala index 06d2affb4..62f6a0d3a 100644 --- a/arch/src/main/scala/sims/verilator/TargetConfig.scala +++ b/arch/src/main/scala/sims/verilator/TargetConfig.scala @@ -2,55 +2,27 @@ package sims.verilator import _root_.circt.stage.ChiselStage import sims.soc.SystemTarget -import framework.system.configloader.ChipLoader -import java.nio.file.{Files, Paths} -//===----------------------------------------------------------------------===// -object Elaborate extends App { - if (args.isEmpty) { - println("Usage: Elaborate [firtool-opts...]") - println("Example: Elaborate sims.verilator.GobanVerilatorTarget") - sys.exit(1) - } - val targetClassName = args(0) - println(s"Elaborating BBSimHarness with target: $targetClassName") - - val target: SystemTarget = - try { - Class.forName(targetClassName).getDeclaredConstructor().newInstance() match { - case system: SystemTarget => system - case _ => - println(s"Error: $targetClassName is not a SystemTarget") - sys.exit(1) - } - } catch { - case e: ClassNotFoundException => - println(s"Error: target class not found: $targetClassName") - sys.exit(1) - } +/** The design owns configuration decoding and concrete component types. */ +object Elaborate { - val rawFirtoolOpts = args.drop(1) - val mainOnly = rawFirtoolOpts.contains("--main-only") - - val outDir = rawFirtoolOpts.collectFirst { - case opt if opt.startsWith("-o=") => opt.stripPrefix("-o=") - }.getOrElse { - throw new Exception("missing -o= in firtool opts") + def main(args: Array[String]): Unit = { + require(args.length >= 6 && args(0) == "--config" && args(2) == "--target-class" && args(4) == "--target-dir") + val original = buckyball.config.Design.parseFrom(java.nio.file.Files.readAllBytes(java.nio.file.Paths.get(args(1)))) + val options = args.drop(6) + val design = original.toBuilder.setSims(original.getSims.toBuilder + .setDiffTest(original.getSims.getDiffTest || options.contains("--difftest"))).build() + val target = Class.forName(args(3)).getConstructor(classOf[buckyball.config.Design]) + .newInstance(design).asInstanceOf[SystemTarget] + emit(target, args(5), options.filterNot(_ == "--difftest")) } - val firtoolOpts = rawFirtoolOpts.filterNot { opt => - opt == "--split-verilog" || opt == "--difftest" || opt == "--main-only" || opt.startsWith("-o=") + def emit(target: SystemTarget, directory: String, firtoolOpts: Array[String]): Unit = { + ChiselStage.emitSystemVerilogFile( + new SystemHarness(target.b, target), + args = Array("--target-dir", directory, "--split-verilog"), + firtoolOpts = firtoolOpts + ) } - ChiselStage.emitSystemVerilogFile( - new SystemHarness(target, diffTest = rawFirtoolOpts.contains("--difftest"), mainOnly = mainOnly), - firtoolOpts = firtoolOpts, - args = Array("--target-dir", outDir, "--split-verilog") - ) - val hierarchy = Paths.get(outDir).resolve("hierarchy.vlt") - if (!mainOnly && ChipLoader.load(target.pb).tiles.exists(!_.main)) { - Files.writeString(hierarchy, "`verilator_config\nhier_block -module \"ComputeTile\"\n") - } else { - Files.deleteIfExists(hierarchy) - } } diff --git a/arch/thirdparty/build.sc b/arch/thirdparty/build.sc new file mode 100644 index 000000000..c0f3f7ba6 --- /dev/null +++ b/arch/thirdparty/build.sc @@ -0,0 +1,101 @@ +import mill._ +import mill.scalalib._ +val archRoot: os.Path = millSourcePath / os.up + +object protoJava extends JavaModule { + + def protoDir = T { + archRoot / os.up / "bbdev" / "api" / "steps" / "config" / "scripts" / "proto" + } + + def protoSources = T.sources { + Seq(PathRef(protoDir() / "design.proto"), PathRef(protoDir() / "hardware.proto")) ++ + Seq("balls", "cores").flatMap(kind => + os.walk(archRoot / os.up / "examples" / "prefab" / kind).filter(p => p.ext == "proto" && !p.toString.contains("/thirdparty/") && !p.toString.contains("/out/")).map(PathRef(_))) + } + + def generatedSources = T { + val dir = protoDir() + val protos = protoSources().map(_.path) + val command = Seq("protoc", s"--java_out=${T.dest}") ++ + protos.map(path => s"-I${path / os.up}").distinct ++ protos.map(_.toString) + os.proc(command).call() + Seq(PathRef(T.dest)) + } + + override def ivyDeps = Agg(ivy"com.google.protobuf:protobuf-java:4.35.1") +} + +object hardfloat extends SbtModule { + override def millSourcePath = + archRoot / "thirdparty" / "berkeley-hardfloat" + override def scalaVersion = "2.13.16" + override def scalacOptions = Seq("-Ymacro-annotations") + + // Override sources to match build.sbt behavior + override def sources = T.sources { + super.sources() ++ Seq( + PathRef(millSourcePath / "hardfloat" / "src" / "main" / "scala") + ) + } + + override def ivyDeps = Agg( + ivy"org.chipsalliance::chisel:6.7.0" + ) + + override def scalacPluginIvyDeps = Agg( + ivy"org.chipsalliance:::chisel-plugin:6.7.0" + ) + +} + +object midas_target_utils extends SbtModule { + override def millSourcePath = + archRoot / os.up / "thirdparty" / "firesim" / "sim" / "midas" / "targetutils" + override def scalaVersion = "2.13.16" + + override def ivyDeps = Agg( + ivy"org.chipsalliance::chisel:6.7.0" + ) + + override def scalacPluginIvyDeps = Agg( + ivy"org.chipsalliance:::chisel-plugin:6.7.0" + ) + +} + +object gemmini extends SbtModule { + override def millSourcePath = + archRoot / "thirdparty" / "gemmini" + override def scalaVersion = "2.13.16" + override def scalacOptions = Seq("-Ymacro-annotations") + + override def sources = T.sources { + Seq( + "Arithmetic", + "Dataflow", + "MeshWithDelays", + "Mesh", + "Tile", + "PE", + "Util", + "Transposer", + "TagQueue", + "Pipeline", + "Shifter", + "SyncMem" + ) + .map(name => PathRef(millSourcePath / "src" / "main" / "scala" / "gemmini" / s"$name.scala")) + } + + override def moduleDeps = Seq(hardfloat) + + override def ivyDeps = Agg( + ivy"org.chipsalliance::chisel:6.7.0" + ) + + override def scalacPluginIvyDeps = Agg( + ivy"org.chipsalliance:::chisel-plugin:6.7.0" + ) + +} diff --git a/arch/thirdparty/common_cells b/arch/thirdparty/common_cells new file mode 160000 index 000000000..128154569 --- /dev/null +++ b/arch/thirdparty/common_cells @@ -0,0 +1 @@ +Subproject commit 1281545696eb3fcba50ec5b4275993476a3c710e diff --git a/arch/thirdparty/cvfpu b/arch/thirdparty/cvfpu new file mode 160000 index 000000000..a8e0cba6d --- /dev/null +++ b/arch/thirdparty/cvfpu @@ -0,0 +1 @@ +Subproject commit a8e0cba6dd50f357ece73c2c955d96efc3c6c315 diff --git a/arch/thirdparty/hci b/arch/thirdparty/hci new file mode 160000 index 000000000..a8d8132b7 --- /dev/null +++ b/arch/thirdparty/hci @@ -0,0 +1 @@ +Subproject commit a8d8132b7f7839fd55f92df65cdc3e361ea80a6b diff --git a/arch/thirdparty/hwpe-ctrl b/arch/thirdparty/hwpe-ctrl new file mode 160000 index 000000000..dfcf1cb74 --- /dev/null +++ b/arch/thirdparty/hwpe-ctrl @@ -0,0 +1 @@ +Subproject commit dfcf1cb746709de206377388bbcb4a3571e24f7b diff --git a/arch/thirdparty/hwpe-stream b/arch/thirdparty/hwpe-stream new file mode 160000 index 000000000..05cb83036 --- /dev/null +++ b/arch/thirdparty/hwpe-stream @@ -0,0 +1 @@ +Subproject commit 05cb8303693065812e9d6bfe10976220fb455666 diff --git a/arch/thirdparty/redmule b/arch/thirdparty/redmule new file mode 160000 index 000000000..7fa9fbe8a --- /dev/null +++ b/arch/thirdparty/redmule @@ -0,0 +1 @@ +Subproject commit 7fa9fbe8a29e8572810ae12b92c19749045ac860 diff --git a/arch/thirdparty/tech_cells_generic b/arch/thirdparty/tech_cells_generic new file mode 160000 index 000000000..3a3de7363 --- /dev/null +++ b/arch/thirdparty/tech_cells_generic @@ -0,0 +1 @@ +Subproject commit 3a3de73632a06826b1bd9c65a0a2e92b32016845 diff --git a/arch/uvm.toml b/arch/uvm.toml index caeae8009..be18d381a 100644 --- a/arch/uvm.toml +++ b/arch/uvm.toml @@ -1,49 +1,259 @@ -ips = ["ant", "tls", "tss", "preflight", "ddr", "interlock", "memdomain", "root_core", "root_tile", "fetch", "mmu", "cpu_mem", "axis", "bank", "cache", "coherence", "chi", "root_chip", "mesh_shm", "rvv"] - [paths] -ant = "arch/src/main/scala/framework/system/core/ant" -tls = "arch/src/main/scala/framework/mem-core/tls" -tss = "arch/src/main/scala/framework/mem-core/tss" -preflight = "arch/src/main/scala/framework/mem-core/preflight" -ddr = "arch/src/main/scala/framework/mem-core/ddr" -interlock = "arch/src/main/scala/framework/mem-core/interlock" -memdomain = "arch/src/main/scala/framework/memdomain/verification" -axis = "arch/src/main/scala/framework/mem-core/axis" -bank = "arch/src/main/scala/framework/mem-core/bank" -cache = "arch/src/main/scala/framework/mem-core/cache" -coherence = "arch/src/main/scala/framework/mem-core/coherence" -mesh_shm = "arch/src/main/scala/framework/mem-core/mesh_shm" -rvv = "arch/src/main/scala/framework/rvv" -chi = "arch/src/main/scala/framework/mem-core/chi" -root_chip = "arch/src/main/scala/framework/root/root-chip" -cpu_mem = "arch/src/main/scala/framework/mem-core/cpu_mem" -mmu = "arch/src/main/scala/framework/mem-core/mmu" -fetch = "arch/src/main/scala/framework/mem-core/fetch" -root_tile = "arch/src/main/scala/framework/root/root-tile" - -root_core = "arch/src/main/scala/framework/root/root-core" - -[build.root_core] -directory = "arch" -module = "root_core" - -[build.root_tile] -directory = "arch" -module = "root_tile" - -[build.memdomain] -directory = "arch" -module = "memdomain_ack" -arguments = ["--chip", "{chip}", "--targets", "{targets}"] - -[build.tls] -directory = "arch" -module = "tls" +tbus_router="arch/src/main/scala/framework/core/tbus/src/main/scala/router" +core_cache="arch/src/main/scala/framework/core/memory/access/src/main/scala/cache" +home="arch/src/main/scala/framework/core/memory/home" +admission="arch/src/main/scala/framework/core/controller/admission" +tol1 = "arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1" +toddr = "arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr" +tomem = "arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem" +prm = "arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm" +bank_router = "arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router" +global_rob = "arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global" +boot = "arch/src/main/scala/framework/acc/frontend/src/main/scala/boot" +bat = "arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat" +scoreboard = "arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard" +tss = "arch/src/main/scala/framework/core/tss" +preflight = "arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight" +ddr = "arch/src/main/scala/framework/device/ddr" +interlock = "arch/src/main/scala/framework/ip/interlock" +axis = "arch/src/main/scala/framework/ip/axis" +axi4 = "arch/src/main/scala/framework/ip/axi4" +bank = "arch/src/main/scala/framework/ip/bank" +spm = "arch/src/main/scala/framework/ip/spm" +cache = "arch/src/main/scala/framework/ip/cache" +coherence = "arch/src/main/scala/framework/ip/coherence" +chi = "arch/src/main/scala/framework/ip/chi" +mem_access = "arch/src/main/scala/framework/core/memory/access" +mmu = "arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu" +fetch = "arch/src/main/scala/framework/core/execution/fetch" + + +instruction = "arch/src/main/scala/framework/core/execution/instruction" +fpu = "arch/src/main/scala/framework/core/execution/fpu" +muldiv = "arch/src/main/scala/framework/core/execution/muldiv" +privileged = "arch/src/main/scala/framework/core/execution/privileged" +integer = "arch/src/main/scala/framework/core/execution/integer" +register_file = "arch/src/main/scala/framework/core/execution/register_file" [build.tss] -directory = "arch" -module = "tss" +directory = "arch/src/main/scala/framework" +module = "core.tss" +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] + +[build.global_rob] +directory = "arch/src/main/scala/framework" +module = "acc.frontend.scheduler.rob.global" +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] + + +[build.boot] +directory = "arch/src/main/scala/framework" +module = "acc.frontend.boot" +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] + + +[build.bank_router] +directory = "arch/src/main/scala/framework" +module = "acc.memdomain.bank_router" +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] + +[build.axis] +directory = "arch/src/main/scala/framework" +module = "ip.axis" +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] + +[build.axi4] +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] +directory = "arch/src/main/scala/framework" +module = "ip.axi4" + +[build.chi] +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] +directory = "arch/src/main/scala/framework" +module = "ip.chi" + +[build.bank] +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] +directory = "arch/src/main/scala/framework" +module = "ip.bank" + +[build.cache] +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] +directory = "arch/src/main/scala/framework" +module = "ip.cache" + +[build.coherence] +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] +directory = "arch/src/main/scala/framework" +module = "ip.coherence" + +[build.interlock] +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] +directory = "arch/src/main/scala/framework" +module = "ip.interlock" + +[build.spm] +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] +directory = "arch/src/main/scala/framework" +module = "ip.spm" + +[build.ddr] +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] +directory = "arch/src/main/scala/framework" +module = "device.ddr" + +[build.mmu] +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] +directory = "arch/src/main/scala/framework" +module = "core.memory.access.mmu" + +[build.fetch] +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] +directory = "arch/src/main/scala/framework" +module = "core.execution.fetch" + +[build.mem_access] +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] +directory = "arch/src/main/scala/framework" +module = "core.memory.access" + +[build.preflight] +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] +directory = "arch/src/main/scala/framework" +module = "core.memory.access.preflight" + + +[build.bat] +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] +directory = "arch/src/main/scala/framework" +module = "acc.frontend.scheduler.bat" + +[build.scoreboard] +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] +directory = "arch/src/main/scala/framework" +module = "acc.frontend.scheduler.scoreboard" + + +[build.instruction] +directory = "arch/src/main/scala/framework" +module = "core.execution.instruction" +arguments = ["--config", "{config}"] + +[build.fpu] +directory = "arch/src/main/scala/framework" +module = "core.execution.fpu" +arguments = ["--config", "{config}"] + +[build.muldiv] +directory = "arch/src/main/scala/framework" +module = "core.execution.muldiv" +arguments = ["--config", "{config}"] + +[build.privileged] +directory = "arch/src/main/scala/framework" +module = "core.execution.privileged" +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] + +[build.integer] +directory = "arch/src/main/scala/framework" +module = "core.execution.integer" +arguments = ["--config", "{config}"] + +[build.register_file] +directory = "arch/src/main/scala/framework" +module = "core.execution.register_file" +arguments = ["--config", "{config}"] + +[modules.frontend] +level = "domain" +path = "arch/src/main/scala/framework/acc/frontend" + +[modules.frontend.build] +directory = "arch/src/main/scala/framework" +module = "acc.frontend" +arguments = ["--config", "{repo}/examples/designs/{design}/output/sims/device/generated/design.pb"] + +[modules.memdomain] +level = "domain" +path = "arch/src/main/scala/framework/acc/memdomain" + +[modules.memdomain.build] +directory = "arch/src/main/scala/framework" +module = "acc.memdomain" +arguments = ["--design", "{design}"] + +[modules.gpdomain] +level = "domain" +path = "arch/src/main/scala/framework/acc/gpdomain" + +[modules.gpdomain.build] +directory = "arch/src/main/scala/framework" +module = "acc.gpdomain" +arguments = ["--config", "{repo}/examples/designs/{design}/output/sims/device/generated/design.pb"] + +[modules.balldomain] +level = "domain" +path = "arch/src/main/scala/framework/acc/balldomain" + +[modules.balldomain.build] +runtime = [{ directory = "examples", module = "library" }] +directory = "arch/src/main/scala/framework" +module = "acc.balldomain" +arguments = ["--config", "{repo}/examples/designs/{design}/output/sims/device/generated/design.pb"] + +[modules.ant] +level = "thread" +path = "arch/src/main/scala/framework/thread/ant" + +[modules.ant.build] +directory = "arch/src/main/scala/framework" +module = "thread.ant" +arguments = ["--config", "{repo}/examples/designs/{design}/output/sims/device/generated/design.pb", "--core", "1"] + +[modules.mesh_shm] +level = "subsystem" +path = "arch/src/main/scala/framework/ip/mesh_shm" + +[modules.mesh_shm.build] +directory = "arch/src/main/scala/framework" +module = "ip.mesh_shm" + +[build.prm] +directory = "arch/src/main/scala/framework" +module = "acc.memdomain.prm" +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] + +[build.tol1] +directory = "arch/src/main/scala/framework" +module = "acc.memdomain.tomem.tol1" +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] + +[build.toddr] +directory="arch/src/main/scala/framework" +module="acc.memdomain.tomem.toddr" +arguments=["--config","{config}","--output-dir","{rtl}"] + +[build.tomem] +directory="arch/src/main/scala/framework" +module="acc.memdomain.tomem" +arguments=["--config","{config}","--output-dir","{rtl}"] + +[build.admission] +directory="arch/src/main/scala/framework" +module="core.controller.admission" +arguments=["--config","{config}","--output-dir","{rtl}"] + +[build.home] +directory="arch/src/main/scala/framework" +module="core.memory.home" +arguments=["--config","{config}","--output-dir","{rtl}"] + +[build.core_cache] +directory="arch/src/main/scala/framework" +module="core.memory.access.cache" +arguments=["--config","{config}","--output-dir","{rtl}"] -[build.ant] -directory = "arch" -module = "ant" +[build.tbus_router] +directory="arch/src/main/scala/framework" +module="core.tbus.router" +arguments=["--config","{config}","--output-dir","{rtl}"] diff --git a/bb-tests/CMakeLists.txt b/bb-tests/CMakeLists.txt index df6afe5a9..7dc5f899b 100644 --- a/bb-tests/CMakeLists.txt +++ b/bb-tests/CMakeLists.txt @@ -1,10 +1,4 @@ -project(bb-tests) cmake_minimum_required(VERSION 3.26) -cmake_policy(SET CMP0003 NEW) - -set(CMAKE_EXPORT_COMPILE_COMMANDS ON) - -set(BB_TESTS_DIR ${CMAKE_CURRENT_SOURCE_DIR}) -set(WORKLOADS_DIR ${BB_TESTS_DIR}/workloads) +project(BuckyballTests C CXX) add_subdirectory(workloads) diff --git a/bb-tests/thirdparty/busybox b/bb-tests/thirdparty/busybox deleted file mode 160000 index 1a64f6a20..000000000 --- a/bb-tests/thirdparty/busybox +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 1a64f6a20aaf6ea4dbba68bbfa8cc1ab7e5c57c4 diff --git a/bb-tests/thirdparty/linux b/bb-tests/thirdparty/linux deleted file mode 160000 index ffc253263..000000000 --- a/bb-tests/thirdparty/linux +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ffc253263a1375a65fa6c9f62a893e9767fbebfa diff --git a/bb-tests/thirdparty/opensbi b/bb-tests/thirdparty/opensbi deleted file mode 160000 index c0f87f10d..000000000 --- a/bb-tests/thirdparty/opensbi +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c0f87f10d1bfb9e72a84ddfafb5604ee1bfe9d04 diff --git a/bb-tests/workloads/CMakeLists.txt b/bb-tests/workloads/CMakeLists.txt index 588799bc1..a6d9dbdbd 100644 --- a/bb-tests/workloads/CMakeLists.txt +++ b/bb-tests/workloads/CMakeLists.txt @@ -1,10 +1,85 @@ -cmake_minimum_required(VERSION 3.12) +# ===----------------------------------------------------------------------=== +# The workloads of one simulation of a design, configured with DESIGN_CMAKE, +# generated from its design.pb by `bbdev stack --build`: +# +# cmake -G Ninja -S bb-tests/workloads -B -DDESIGN_CMAKE= +# ninja -C sync-bin | sync-ctest-bin | sync-mlirtest-bin | sync-soctest-bin +# +# Products are copied to ${DESIGN_OUTPUT}/workloads. +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.26) +set(CMAKE_C_COMPILER riscv64-unknown-linux-gnu-gcc) +set(CMAKE_CXX_COMPILER riscv64-unknown-linux-gnu-g++) project(BuckyballWorkloads C CXX) +include("${DESIGN_CMAKE}") set(CMAKE_EXPORT_COMPILE_COMMANDS ON) -include(common/CMakeLists.txt) +set(CMAKE_BUILD_TYPE Release) +set(CMAKE_C_FLAGS_RELEASE "-O2 -UNDEBUG") +set(CMAKE_CXX_FLAGS_RELEASE "-O2 -UNDEBUG") +set(WORKLOAD_DIR ${CMAKE_CURRENT_LIST_DIR}) +set(WORKLOAD_SRC_DIR ${WORKLOAD_DIR}/src) + +set(BUCKYBALL_REPO_ROOT ${DESIGN_REPO}) +set(WORKLOAD_LIB_DIR ${WORKLOAD_DIR}/lib) +set(BAREMETAL_DIR ${BUCKYBALL_REPO_ROOT}/stack/os/baremetal) +# bb_cores.h: the cores of DESIGN_DEFS. +set(RUNTIME_INCLUDE_DIR ${BUCKYBALL_REPO_ROOT}/stack/runtime/include) + +find_program(ELF_CC riscv64-unknown-elf-gcc REQUIRED) +find_program(ELF_CXX riscv64-unknown-elf-g++ REQUIRED) +find_program(ELF_OBJCOPY riscv64-unknown-elf-objcopy REQUIRED) +set(LINUX_CC ${CMAKE_C_COMPILER}) +set(LINUX_CXX ${CMAKE_CXX_COMPILER}) + +# runtime, runtime-baremetal, runtime-linux and engine of every program here: +# built by `bbdev stack --build "--design --device "`. +set(BUCKYBALL_CONTROL_TARGET "core${DESIGN_WORKLOAD_CONTROL_HART}") +# The definitions of every program of the design, as compiler flags. +set(DESIGN_DEFINES ${DESIGN_DEFS}) +list(TRANSFORM DESIGN_DEFINES PREPEND "-D") + +set(BBSIM_LD ${BAREMETAL_DIR}/bbsim.ld) +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=rv64gc") +set(CMAKE_EXE_LINKER_FLAGS + "${CMAKE_EXE_LINKER_FLAGS} -static -Wl,--no-dynamic-linker" +) + +set(OUTPUT_BIN_DIR "${DESIGN_OUTPUT}/workloads") + +set(CTEST_MAX_LINES 100) +set(BBHW_MEM_C ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/dma.c) +set(BUCKYBALL_CTEST_C_FLAGS + -g + -fno-common + -O2 + -static + -march=rv64gc + -mcmodel=medany + -fno-builtin-printf + -specs=nano.specs + -specs=nosys.specs + -nostartfiles + -Wl,-T,${BBSIM_LD} +) +file( + GLOB BUCKYBALL_BALL_WORKLOAD_DIRS + CONFIGURE_DEPENDS + "${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/*/workloads" +) +file( + GLOB BUCKYBALL_BALL_ISA_DEPS + CONFIGURE_DEPENDS + "${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/*/workloads/isa/*" +) + +set(BUDDY_MLIR_DIR ${BUCKYBALL_REPO_ROOT}/stack/thirdparty/buddy-mlir) + +set(BUCKYBALL_RISCV_MATTR "+xbuckyball,+m,+D") -add_subdirectory(${WORKLOAD_LIB_DIR} ${CMAKE_CURRENT_BINARY_DIR}/lib) add_subdirectory(src) -add_custom_target(sync-bin DEPENDS sync-native-bin sync-mlirtest-bin sync-soctest-bin) +add_custom_target( + sync-bin + DEPENDS sync-native-bin sync-mlirtest-bin sync-soctest-bin +) add_custom_target(build-all ALL DEPENDS sync-bin) diff --git a/bb-tests/workloads/common/CMakeLists.txt b/bb-tests/workloads/common/CMakeLists.txt deleted file mode 100644 index c9f866308..000000000 --- a/bb-tests/workloads/common/CMakeLists.txt +++ /dev/null @@ -1,233 +0,0 @@ -get_filename_component(WORKLOAD_DIR ${CMAKE_CURRENT_LIST_DIR}/.. ABSOLUTE) -set(WORKLOAD_SRC_DIR ${WORKLOAD_DIR}/src) - -get_filename_component(BUCKYBALL_REPO_ROOT ${WORKLOAD_DIR}/../.. ABSOLUTE) -set(WORKLOAD_LIB_DIR ${WORKLOAD_DIR}/lib) -include_directories(${BUCKYBALL_REPO_ROOT}/stack/runtime/include) -set(BBSW_BAREMETAL_DIR ${WORKLOAD_LIB_DIR}/bbsw/baremetal) - -function(buckyball_ctest_name OUT CORE TEST) - if("${CORE}" STREQUAL "" OR "${TEST}" STREQUAL "") - message(FATAL_ERROR "buckyball_ctest_name requires core and test") - endif() - set(${OUT} "${BUCKYBALL_WORKLOAD_CHIP}-${CORE}-ctest-${TEST}" PARENT_SCOPE) -endfunction() - -function(buckyball_mlirtest_name OUT CORE TEST) - if("${CORE}" STREQUAL "" OR "${TEST}" STREQUAL "") - message(FATAL_ERROR "buckyball_mlirtest_name requires core and test") - endif() - set(${OUT} "${BUCKYBALL_WORKLOAD_CHIP}-${CORE}-mlirtest-${TEST}" PARENT_SCOPE) -endfunction() - -function(buckyball_soctest_name OUT CORE TEST) - if("${CORE}" STREQUAL "" OR "${TEST}" STREQUAL "") - message(FATAL_ERROR "buckyball_soctest_name requires core and test") - endif() - set(${OUT} "${BUCKYBALL_WORKLOAD_CHIP}-${CORE}-soctest-${TEST}" PARENT_SCOPE) -endfunction() - -set(RISCV_GNU_TOOLCHAIN $ENV{RISCV}) - -set(BUCKYBALL_WORKLOAD_CHIP "" CACHE STRING "Buckyball chip for workloads") -if(BUCKYBALL_WORKLOAD_CHIP STREQUAL "") - message(FATAL_ERROR - "BUCKYBALL_WORKLOAD_CHIP is required. Pass -DBUCKYBALL_WORKLOAD_CHIP=.") -endif() -set(BUCKYBALL_WORKLOAD_SOURCE_CHIP "${BUCKYBALL_WORKLOAD_CHIP}" CACHE STRING - "Buckyball chip source tree for workloads") -if(NOT IS_DIRECTORY - "${BUCKYBALL_REPO_ROOT}/examples/chips/${BUCKYBALL_WORKLOAD_SOURCE_CHIP}") - message(FATAL_ERROR - "Buckyball workload source chip does not exist: ${BUCKYBALL_WORKLOAD_SOURCE_CHIP}") -endif() -message(STATUS - "Buckyball workload instance: ${BUCKYBALL_WORKLOAD_CHIP} (source: ${BUCKYBALL_WORKLOAD_SOURCE_CHIP})") - -if(NOT IS_ABSOLUTE "${Python3_EXECUTABLE}") - find_program(BUCKYBALL_PYTHON3_EXECUTABLE NAMES python3) - if(NOT BUCKYBALL_PYTHON3_EXECUTABLE) - message(FATAL_ERROR "python3 is required to build workloads") - endif() - set(Python3_EXECUTABLE "${BUCKYBALL_PYTHON3_EXECUTABLE}") -endif() - -foreach(_BUCKYBALL_REQ IN ITEMS - BUCKYBALL_CHIP_PB) - if(NOT DEFINED ${_BUCKYBALL_REQ} OR "${${_BUCKYBALL_REQ}}" STREQUAL "") - message(FATAL_ERROR - "${_BUCKYBALL_REQ} must be passed by bbdev config --install (cmake.defs)") - endif() -endforeach() - -if(NOT EXISTS "${BUCKYBALL_CHIP_PB}") - message(FATAL_ERROR "missing Chip.pb: ${BUCKYBALL_CHIP_PB}") -endif() -set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS - "${BUCKYBALL_CHIP_PB}" - "${BUCKYBALL_REPO_ROOT}/stack/compiler/scripts/pb_to_target_registry.py") -set(BUCKYBALL_ISA_DIR "${CMAKE_BINARY_DIR}/generated/isa") -execute_process( - COMMAND "${Python3_EXECUTABLE}" - "${BUCKYBALL_REPO_ROOT}/stack/compiler/scripts/pb_to_target_registry.py" - --repo "${BUCKYBALL_REPO_ROOT}" - --chip-pb "${BUCKYBALL_CHIP_PB}" - --isa-dir "${BUCKYBALL_ISA_DIR}" - --print-targets - --print-target-balls - RESULT_VARIABLE BUCKYBALL_TARGET_REGISTRY_RESULT - OUTPUT_VARIABLE BUCKYBALL_TARGETS_RAW - ERROR_VARIABLE BUCKYBALL_TARGET_REGISTRY_ERROR) -if(NOT BUCKYBALL_TARGET_REGISTRY_RESULT EQUAL 0) - message(FATAL_ERROR - "failed to generate target ISA headers: ${BUCKYBALL_TARGET_REGISTRY_ERROR}") -endif() -string(REPLACE "\n" ";" BUCKYBALL_TARGETS "${BUCKYBALL_TARGETS_RAW}") -execute_process( - COMMAND "${Python3_EXECUTABLE}" - "${BUCKYBALL_REPO_ROOT}/stack/compiler/scripts/pb_to_target_registry.py" - --repo "${BUCKYBALL_REPO_ROOT}" - --chip-pb "${BUCKYBALL_CHIP_PB}" - --print-bank-targets - RESULT_VARIABLE BUCKYBALL_BANK_TARGETS_RESULT - OUTPUT_VARIABLE BUCKYBALL_BANK_TARGETS_RAW - ERROR_VARIABLE BUCKYBALL_BANK_TARGETS_ERROR) -if(NOT BUCKYBALL_BANK_TARGETS_RESULT EQUAL 0) - message(FATAL_ERROR - "failed to get compiler targets with banks: ${BUCKYBALL_BANK_TARGETS_ERROR}") -endif() -string(REPLACE "\n" ";" BUCKYBALL_BANK_TARGETS - "${BUCKYBALL_BANK_TARGETS_RAW}") -list(REMOVE_ITEM BUCKYBALL_BANK_TARGETS "") -execute_process( - COMMAND "${Python3_EXECUTABLE}" - "${BUCKYBALL_REPO_ROOT}/stack/compiler/scripts/pb_to_target_registry.py" - --repo "${BUCKYBALL_REPO_ROOT}" - --chip-pb "${BUCKYBALL_CHIP_PB}" - --print-core-workload-targets - RESULT_VARIABLE BUCKYBALL_CORE_TARGETS_RESULT - OUTPUT_VARIABLE BUCKYBALL_CORE_TARGETS_RAW - ERROR_VARIABLE BUCKYBALL_CORE_TARGETS_ERROR) -if(NOT BUCKYBALL_CORE_TARGETS_RESULT EQUAL 0) - message(FATAL_ERROR - "failed to get Core workload targets: ${BUCKYBALL_CORE_TARGETS_ERROR}") -endif() -string(REPLACE "\n" ";" BUCKYBALL_CORE_TARGETS - "${BUCKYBALL_CORE_TARGETS_RAW}") -list(REMOVE_ITEM BUCKYBALL_CORE_TARGETS "") -set(BUCKYBALL_TARGET_BALL_LINES) -foreach(BUCKYBALL_TARGET_LINE ${BUCKYBALL_TARGETS}) - if(BUCKYBALL_TARGET_LINE MATCHES "^([^:]+):(.*)$") - set(BUCKYBALL_TARGET_BALLS_${CMAKE_MATCH_1} "${CMAKE_MATCH_2}") - list(APPEND BUCKYBALL_TARGET_BALL_LINES ${BUCKYBALL_TARGET_LINE}) - endif() -endforeach() -list(REMOVE_ITEM BUCKYBALL_TARGETS ${BUCKYBALL_TARGET_BALL_LINES} "") -if(NOT BUCKYBALL_TARGETS) - message(FATAL_ERROR "Chip.pb has no compiler targets") -endif() -foreach(_BUCKYBALL_TARGET IN LISTS BUCKYBALL_TARGETS) - if(NOT EXISTS "${BUCKYBALL_ISA_DIR}/${_BUCKYBALL_TARGET}/params.h") - message(FATAL_ERROR - "missing generated params.h for target ${_BUCKYBALL_TARGET}") - endif() -endforeach() -message(STATUS "Buckyball compiler targets: ${BUCKYBALL_TARGETS}") -message(STATUS "Buckyball bank targets: ${BUCKYBALL_BANK_TARGETS}") -message(STATUS "Buckyball Core workload targets: ${BUCKYBALL_CORE_TARGETS}") -execute_process( - COMMAND "${Python3_EXECUTABLE}" - "${BUCKYBALL_REPO_ROOT}/stack/compiler/scripts/pb_to_target_registry.py" - --repo "${BUCKYBALL_REPO_ROOT}" --chip-pb "${BUCKYBALL_CHIP_PB}" - --print-cpu-targets - RESULT_VARIABLE _cpu_result OUTPUT_VARIABLE _cpu_targets ERROR_VARIABLE _cpu_error) -if(NOT _cpu_result EQUAL 0) - message(FATAL_ERROR "failed to get CPU workload targets: ${_cpu_error}") -endif() -string(REPLACE "\n" ";" BUCKYBALL_CPU_TARGETS "${_cpu_targets}") -list(REMOVE_ITEM BUCKYBALL_CPU_TARGETS "") - -function(buckyball_assign_physical_banks TARGET OUT) - list(FIND BUCKYBALL_BANK_TARGETS "${TARGET}" _bank_target_index) - if(_bank_target_index EQUAL -1) - message(FATAL_ERROR "${TARGET}: no physical banks") - endif() - set(${OUT} "-assign-physical-banks" PARENT_SCOPE) -endfunction() - -set(BBSIM_LD ${BBSW_BAREMETAL_DIR}/bbsim.ld) -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=rv64gc") -set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static -Wl,--no-dynamic-linker") - -function(buckyball_chip_ball_dirs OUT_VAR TARGET) - set(_raw "${BUCKYBALL_TARGET_BALLS_${TARGET}}") - if(_raw STREQUAL "") - set(${OUT_VAR} "" PARENT_SCOPE) - return() - endif() - string(REPLACE "," ";" _dirs "${_raw}") - foreach(_dir IN LISTS _dirs) - if(_dir STREQUAL "") - message(FATAL_ERROR "empty ball_dir on target ${TARGET}") - endif() - endforeach() - set(${OUT_VAR} "${_dirs}" PARENT_SCOPE) -endfunction() - -function(buckyball_add_ball_workload_subdirs KIND) - if(NOT KIND STREQUAL "ctests" AND NOT KIND STREQUAL "mlir_tests") - message(FATAL_ERROR "unknown ball workload kind: ${KIND}") - endif() - foreach(BUCKYBALL_TARGET IN LISTS BUCKYBALL_TARGETS) - if(NOT BUCKYBALL_TARGET IN_LIST BUCKYBALL_CPU_TARGETS) - continue() - endif() - buckyball_chip_ball_dirs(_ball_dirs ${BUCKYBALL_TARGET}) - foreach(BUCKYBALL_BALL_DIR IN LISTS _ball_dirs) - set(_src - ${BUCKYBALL_REPO_ROOT}/examples/balls/${BUCKYBALL_BALL_DIR}/workloads/${KIND}) - if(NOT IS_DIRECTORY ${_src}) - if(KIND STREQUAL "mlir_tests") - continue() - endif() - message(FATAL_ERROR "missing Ball ${KIND} directory: ${_src}") - endif() - if(KIND STREQUAL "ctests") - set(BUCKYBALL_CTEST_TARGET ${BUCKYBALL_TARGET}) - add_subdirectory( - ${_src} - ${CMAKE_CURRENT_BINARY_DIR}/balls/${BUCKYBALL_TARGET}/${BUCKYBALL_BALL_DIR}/ctests) - else() - set(BUCKYBALL_MLIR_ACTIVE_TARGET ${BUCKYBALL_TARGET}) - add_subdirectory( - ${_src} - ${CMAKE_CURRENT_BINARY_DIR}/balls/${BUCKYBALL_TARGET}/${BUCKYBALL_BALL_DIR}) - endif() - endforeach() - endforeach() -endfunction() - -function(buckyball_add_core_workload_subdirs KIND) - foreach(_entry IN LISTS BUCKYBALL_CORE_TARGETS) - if(NOT _entry MATCHES "^([^:]+):([^:]+):([0-9]+)$") - message(FATAL_ERROR "invalid Core workload target: ${_entry}") - endif() - set(BUCKYBALL_CORE_WORKLOAD_TARGET "${CMAKE_MATCH_1}") - if(KIND STREQUAL "ctests" AND NOT BUCKYBALL_CORE_WORKLOAD_TARGET IN_LIST BUCKYBALL_CPU_TARGETS) - continue() - endif() - set(_pkg "${CMAKE_MATCH_2}") - set(BUCKYBALL_CORE_WORKLOAD_PROFILE "${CMAKE_MATCH_3}") - set(_src - "${BUCKYBALL_REPO_ROOT}/examples/cores/${_pkg}/workloads/${KIND}") - if(IS_DIRECTORY "${_src}") - add_subdirectory("${_src}" - "${CMAKE_CURRENT_BINARY_DIR}/cores/${BUCKYBALL_CORE_WORKLOAD_TARGET}/${KIND}") - endif() - endforeach() -endfunction() - -set(ELF_CC ${RISCV_GNU_TOOLCHAIN}/bin/riscv64-unknown-elf-gcc) -set(LINUX_CC ${RISCV_GNU_TOOLCHAIN}/bin/riscv64-unknown-linux-gnu-gcc) -set(BUILD_BIN_DIR ${CMAKE_BINARY_DIR}) -set(OUTPUT_BIN_DIR ${WORKLOAD_DIR}/../output/${BUCKYBALL_WORKLOAD_CHIP}/workloads) diff --git a/bb-tests/workloads/lib/CMakeLists.txt b/bb-tests/workloads/lib/CMakeLists.txt deleted file mode 100644 index 04e30376e..000000000 --- a/bb-tests/workloads/lib/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -add_subdirectory(bbhw) -add_subdirectory(${BUCKYBALL_REPO_ROOT}/stack/runtime runtime EXCLUDE_FROM_ALL) - -if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/kernel/CMakeLists.txt") - add_subdirectory(kernel) -endif() diff --git a/bb-tests/workloads/lib/bbhw/CMakeLists.txt b/bb-tests/workloads/lib/bbhw/CMakeLists.txt deleted file mode 100644 index 195ef1c6e..000000000 --- a/bb-tests/workloads/lib/bbhw/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -# bbhw: ISA macros (headers) + dma.c (dma_touch). -# Include path is WORKLOAD_LIB_DIR (workloads/lib), so #include works. -# dma.c is compiled into each ctest via bb-tests/workloads/CMakeLists.txt. diff --git a/bb-tests/workloads/lib/bbhw/isa/01_barrier.c b/bb-tests/workloads/lib/bbhw/isa/01_barrier.c index 02aef6904..e8bc9b8a4 100644 --- a/bb-tests/workloads/lib/bbhw/isa/01_barrier.c +++ b/bb-tests/workloads/lib/bbhw/isa/01_barrier.c @@ -6,14 +6,14 @@ #define BB_BARRIER_FUNC7 1 /** - * bb_barrier() - hardware multi-core barrier synchronization. + * bb_barrier() - hardware multi-thread barrier synchronization. * * Semantics: - * 1. Waits for this core's own instruction ROB to drain. - * 2. Signals arrive to the tile-level BarrierUnit. - * 3. Stalls until all nCores cores have arrived (hardware all-reduce). + * 1. Waits for this thread's own instruction ROB to drain. + * 2. Signals arrive to the core-level BarrierUnit. + * 3. Stalls until all nThreads threads have arrived (hardware all-reduce). * - * All cores in the same BBTile must call bb_barrier() at the same point. + * All threads in the same BBCore must call bb_barrier() at the same point. */ #define bb_barrier() BUCKYBALL_INSTRUCTION_R_R(0, 0, BB_BARRIER_FUNC7) diff --git a/bb-tests/workloads/lib/bbhw/isa/13_mvover.c b/bb-tests/workloads/lib/bbhw/isa/13_mvover.c index 82579abe0..c77023ddb 100644 --- a/bb-tests/workloads/lib/bbhw/isa/13_mvover.c +++ b/bb-tests/workloads/lib/bbhw/isa/13_mvover.c @@ -5,20 +5,20 @@ enum { BB_MVOVER_FUNC7 = 0x0d }; -/* Move contiguous rows from a private Bank of one Core to another in the same - * Tile. Bank IDs are Core-local virtual IDs; row addresses are not byte +/* Move contiguous rows from a private Bank of one Thread to another in the same + * Core. Bank IDs are Thread-local virtual IDs; row addresses are not byte * addresses. */ -static inline void bb_mvover(uint32_t source_core, uint32_t source_bank, - uint32_t source_row, uint32_t target_core, +static inline void bb_mvover(uint32_t source_thread, uint32_t source_bank, + uint32_t source_row, uint32_t target_thread, uint32_t target_bank, uint32_t target_row, uint32_t rows) { - if (source_core > 255 || target_core > 255 || source_bank > 1023 || + if (source_thread > 255 || target_thread > 255 || source_bank > 1023 || target_bank > 1023 || source_row > 65535 || target_row > 65535 || !rows || rows > 65536) __builtin_trap(); - uint64_t rs1 = FIELD(source_core, 0, 7) | FIELD(target_core, 8, 15) | + uint64_t rs1 = FIELD(source_thread, 0, 7) | FIELD(target_thread, 8, 15) | FIELD(source_bank, 16, 25) | FIELD(target_bank, 26, 35); uint64_t rs2 = FIELD(source_row, 0, 15) | FIELD(target_row, 16, 31) | FIELD(rows - 1, 32, 47); diff --git a/bb-tests/workloads/lib/bbhw/isa/14_task.c b/bb-tests/workloads/lib/bbhw/isa/14_task.c deleted file mode 100644 index 382ffb65b..000000000 --- a/bb-tests/workloads/lib/bbhw/isa/14_task.c +++ /dev/null @@ -1,52 +0,0 @@ -#ifndef BB_TASK_ISA_H -#define BB_TASK_ISA_H - -#include "isa.h" - -static inline uintptr_t bb_task_submit(uintptr_t worker, uintptr_t signature) { - uintptr_t result; - asm volatile(".insn r 0x2b, 7, 0, %0, %1, %2" - : "=r"(result) - : "r"(worker), "r"(signature) - : "memory"); - return result; -} - -static inline uintptr_t bb_task_status(uintptr_t worker) { - uintptr_t result; - asm volatile(".insn r 0x2b, 7, 1, %0, %1, %2" - : "=r"(result) - : "r"(worker), "r"((uintptr_t)0) - : "memory"); - return result; -} - -static inline uintptr_t bb_task_worker_count(void) { - uintptr_t result; - asm volatile(".insn r 0x2b, 7, 3, %0, %1, %2" - : "=r"(result) - : "r"((uintptr_t)0), "r"((uintptr_t)0) - : "memory"); - return result; -} - -static inline uintptr_t bb_task_core_signature(uintptr_t core) { - uintptr_t result; - asm volatile(".insn r 0x2b, 7, 4, %0, %1, %2" - : "=r"(result) - : "r"(core), "r"((uintptr_t)0) - : "memory"); - return result; -} - -static inline uintptr_t bb_task_write_descriptor(uintptr_t field, - uintptr_t value) { - uintptr_t result; - asm volatile(".insn r 0x2b, 7, 7, %0, %1, %2" - : "=r"(result) - : "r"(field), "r"(value) - : "memory"); - return result; -} - -#endif diff --git a/bb-tests/workloads/lib/bbhw/isa/16_mvout.c b/bb-tests/workloads/lib/bbhw/isa/16_mvout.c index ca92b8987..4a2d9efb0 100644 --- a/bb-tests/workloads/lib/bbhw/isa/16_mvout.c +++ b/bb-tests/workloads/lib/bbhw/isa/16_mvout.c @@ -2,7 +2,9 @@ #define _BB_MVOUT_H_ #include "isa.h" +#if defined(__linux__) #include +#endif #define BB_MVOUT_FUNC7 16 diff --git a/bb-tests/workloads/lib/bbhw/isa/32_mset.c b/bb-tests/workloads/lib/bbhw/isa/32_mset.c index d7df73012..34fd5c675 100644 --- a/bb-tests/workloads/lib/bbhw/isa/32_mset.c +++ b/bb-tests/workloads/lib/bbhw/isa/32_mset.c @@ -2,7 +2,9 @@ #define _BB_MSET_H_ #include "isa.h" +#if defined(__linux__) #include +#endif #define BB_MSET_FUNC7 32 diff --git a/bb-tests/workloads/lib/bbhw/isa/34_mvin_2d.c b/bb-tests/workloads/lib/bbhw/isa/34_mvin_2d.c index 57971cb41..d993f2567 100644 --- a/bb-tests/workloads/lib/bbhw/isa/34_mvin_2d.c +++ b/bb-tests/workloads/lib/bbhw/isa/34_mvin_2d.c @@ -5,12 +5,12 @@ #define BB_MVIN_2D_FUNC7 34 -// rs1: write bank in BANK2 and tile height; [19:0] reserved. +// rs1: write bank in BANK2 and core height; [19:0] reserved. // rs2: [35:0] address / 8, [42:36] physical pixel bytes / 8, // [52:43] source width, [58:53] destination row base, -// [61:59] tile width - 1, [62] valid bytes (0 = 16, 1 = 8). +// [61:59] core width - 1, [62] valid bytes (0 = 16, 1 = 8). #define bb_mvin_2d(mem_addr, bank_id, height, pixel_bytes, source_width, \ - dst_base, tile_width, valid_bytes) \ + dst_base, core_width, valid_bytes) \ do { \ const uintptr_t bb_mvin_2d_addr = (uintptr_t)(mem_addr); \ const uint64_t bb_mvin_2d_valid = (valid_bytes); \ @@ -21,7 +21,7 @@ (BB_BANK2(bank_id) | BB_ITER(height)), \ (FIELD(bb_mvin_2d_addr >> 3, 0, 35) | \ FIELD((pixel_bytes) / 8, 36, 42) | FIELD(source_width, 43, 52) | \ - FIELD(dst_base, 53, 58) | FIELD((tile_width) - 1, 59, 61) | \ + FIELD(dst_base, 53, 58) | FIELD((core_width) - 1, 59, 61) | \ FIELD(bb_mvin_2d_valid == 8, 62, 62)), \ BB_MVIN_2D_FUNC7); \ } while (0) diff --git a/bb-tests/workloads/lib/bbhw/isa/bb_func7.h b/bb-tests/workloads/lib/bbhw/isa/bb_func7.h index 5b712fd4b..855065681 100644 --- a/bb-tests/workloads/lib/bbhw/isa/bb_func7.h +++ b/bb-tests/workloads/lib/bbhw/isa/bb_func7.h @@ -1,8 +1,9 @@ -#ifndef BB_FUNC7_H -#define BB_FUNC7_H +#ifndef FUNCT7_H +#define FUNCT7_H -#include "ballISA.h" - -#define BB_FUNC7(mne) BB_FUNC7_##mne +// The funct7 of instruction `mne` of the Ball in directory `ball`, from the +// compile definitions of the target (DESIGN_TARGET_DEFS in design.cmake). +#define FUNCT7(ball, mne) \ + THREAD_BALLDOMAIN_BALLS_##ball##_INSTRUCTIONS_##mne##_FUNCT7 #endif diff --git a/bb-tests/workloads/lib/bbhw/isa/isa.c b/bb-tests/workloads/lib/bbhw/isa/isa.c deleted file mode 100644 index 283156402..000000000 --- a/bb-tests/workloads/lib/bbhw/isa/isa.c +++ /dev/null @@ -1 +0,0 @@ -#include "isa.h" diff --git a/bb-tests/workloads/lib/bbhw/isa/isa.h b/bb-tests/workloads/lib/bbhw/isa/isa.h index c733d66f9..2d2923360 100644 --- a/bb-tests/workloads/lib/bbhw/isa/isa.h +++ b/bb-tests/workloads/lib/bbhw/isa/isa.h @@ -21,23 +21,26 @@ typedef int32_t result_t; #define FIELD(val, start_bit, end_bit) \ (((val) & ((2UL << ((end_bit) - (start_bit))) - 1)) << (start_bit)) -// rs1 bank field helpers (10-bit each) +// rs1 bank fields: 10-bit sources and 9-bit architectural destination. #define BB_BANK0(id) FIELD(id, 0, 9) #define BB_BANK1(id) FIELD(id, 10, 19) -#define BB_BANK2(id) FIELD(id, 20, 29) +#define BB_BANK2(id) FIELD(id, 20, 28) +// The architectural destination also supplies an old-value read operand. +#define BB_READ_WR_OLD (1ULL << 29) +#define BB_UPDATE_WR (1ULL << 63) -// rs1 iter field (34-bit, bits 30-63) -#define BB_ITER(n) FIELD(n, 30, 63) +// rs1 iter field (33-bit, bits 30-62); bit 63 declares an in-place update. +#define BB_ITER(n) FIELD(n, 30, 62) // funct7 encoding: [6:4]=enable, [3:0]=opcode // enable: 000=none, 001=1rd, 010=1wr, 011=1rd+1wr, 100=2rd+1wr // 101/110/111 = none (extended opcode space) // Generic RISC-V custom instruction macro (funct3 always 0x3 = CUSTOM3_RS1_RS2) -#ifdef BUCKYBALL_ANT_HOST -#include +#ifdef BUCKYBALL_STREAM +#include #define BUCKYBALL_INSTRUCTION_R_R(rs1_val, rs2_val, func7) \ - ant_emit((func7), (uint64_t)(rs1_val), (uint64_t)(rs2_val)) + stream_emit((func7), (uint64_t)(rs1_val), (uint64_t)(rs2_val)) #else #define BUCKYBALL_INSTRUCTION_R_R(rs1_val, rs2_val, func7) \ asm volatile(".insn r " STR(CUSTOM_3) ", 3, %c2, x0, %0, %1" \ @@ -51,7 +54,6 @@ typedef int32_t result_t; #include "01_barrier.c" #include "12_kernel.c" #include "13_mvover.c" -#include "14_task.c" #include "16_mvout.c" #include "32_mset.c" #include "33_mvin.c" diff --git a/bb-tests/workloads/lib/bbhw/mmio/mmio_allocator.h b/bb-tests/workloads/lib/bbhw/mmio/mmio_allocator.h index 786a13513..8c8614541 100644 --- a/bb-tests/workloads/lib/bbhw/mmio/mmio_allocator.h +++ b/bb-tests/workloads/lib/bbhw/mmio/mmio_allocator.h @@ -1,13 +1,13 @@ #ifndef BBHW_MMIO_ALLOCATOR_H #define BBHW_MMIO_ALLOCATOR_H -#include #include #include -#define MMIO_BANK_BYTES ((MMIO_BANK_ENTRIES) * ((MMIO_BANK_WIDTH_BITS) / 8)) -#define MMIO_TOTAL_BYTES ((MMIO_BANK_NUM) * (MMIO_BANK_BYTES)) -#define MMIO_LOGICAL_ROW_BYTES ((BANK_WIDTH) / 8) +#define MMIO_BANK_BYTES \ + ((THREAD_MEM_MMIO_BANK_ENTRIES) * ((THREAD_MEM_MMIO_BANK_WIDTH) / 8)) +#define MMIO_TOTAL_BYTES ((THREAD_MEM_MMIO_BANK_NUM) * (MMIO_BANK_BYTES)) +#define MMIO_LOGICAL_ROW_BYTES ((THREAD_MEM_BANK_WIDTH) / 8) #define MMIO_LOGICAL_ROWS ((MMIO_TOTAL_BYTES) / (MMIO_LOGICAL_ROW_BYTES)) _Static_assert((MMIO_TOTAL_BYTES) % (MMIO_LOGICAL_ROW_BYTES) == 0, "MMIO bytes must be a multiple of SRAM row bytes"); diff --git a/bb-tests/workloads/lib/bbsw/CRunnerUtils/CRunnerUtils.cpp b/bb-tests/workloads/lib/bbsw/CRunnerUtils/CRunnerUtils.cpp deleted file mode 100644 index 33821e886..000000000 --- a/bb-tests/workloads/lib/bbsw/CRunnerUtils/CRunnerUtils.cpp +++ /dev/null @@ -1,75 +0,0 @@ -// The code here is from the upstream llvm, you can check -// llvm/mlir/include/mlir/ExecutionEngine/CRunnerUtils.cpp - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -extern "C" void memrefCopy(int64_t elemSize, UnrankedMemRefType *srcArg, - UnrankedMemRefType *dstArg) { - DynamicMemRefType src(*srcArg); - DynamicMemRefType dst(*dstArg); - - int64_t rank = src.rank; - MLIR_MSAN_MEMORY_IS_INITIALIZED(src.sizes, rank * sizeof(int64_t)); - - // Handle empty shapes -> nothing to copy. - for (int rankp = 0; rankp < rank; ++rankp) - if (src.sizes[rankp] == 0) - return; - - char *srcPtr = src.data + src.offset * elemSize; - char *dstPtr = dst.data + dst.offset * elemSize; - - if (rank == 0) { - memcpy(dstPtr, srcPtr, elemSize); - return; - } - - int64_t *indices = static_cast(alloca(sizeof(int64_t) * rank)); - int64_t *srcStrides = static_cast(alloca(sizeof(int64_t) * rank)); - int64_t *dstStrides = static_cast(alloca(sizeof(int64_t) * rank)); - - // Initialize index and scale strides. - for (int rankp = 0; rankp < rank; ++rankp) { - indices[rankp] = 0; - srcStrides[rankp] = src.strides[rankp] * elemSize; - dstStrides[rankp] = dst.strides[rankp] * elemSize; - } - - int64_t readIndex = 0, writeIndex = 0; - for (;;) { - // Copy over the element, byte by byte. - memcpy(dstPtr + writeIndex, srcPtr + readIndex, elemSize); - // Advance index and read position. - for (int64_t axis = rank - 1; axis >= 0; --axis) { - // Advance at current axis. - auto newIndex = ++indices[axis]; - readIndex += srcStrides[axis]; - writeIndex += dstStrides[axis]; - // If this is a valid index, we have our next index, so continue copying. - if (src.sizes[axis] != newIndex) - break; - // We reached the end of this axis. If this is axis 0, we are done. - if (axis == 0) - return; - // Else, reset to 0 and undo the advancement of the linear index that - // this axis had. Then continue with the axis one outer. - indices[axis] = 0; - readIndex -= src.sizes[axis] * srcStrides[axis]; - writeIndex -= dst.sizes[axis] * dstStrides[axis]; - } - } -} - -extern "C" void printF64(double d) { fprintf(stdout, "%lg", d); } - -extern "C" void printNewline() { fputc('\n', stdout); } diff --git a/bb-tests/workloads/lib/bbsw/baremetal/bbsim.ld b/bb-tests/workloads/lib/bbsw/baremetal/bbsim.ld deleted file mode 100644 index a33fda064..000000000 --- a/bb-tests/workloads/lib/bbsw/baremetal/bbsim.ld +++ /dev/null @@ -1,63 +0,0 @@ -/* bbsim.ld — baremetal linker script for BBSimHarness (DRAM at 0x8000_0000) */ -/* Bootrom jumps directly to 0x80000000 (ELF entry), _start must be first. */ -OUTPUT_ARCH("riscv") -ENTRY(_start) - -SECTIONS { - . = 0x80000000; - - .text : { - *(.text.init) /* crt0.S _start must be first */ - *(.text.startup .text.startup.*) - *(.text .text.*) - *(.gnu.linkonce.t.*) - } - - .rodata : { - *(.rodata .rodata.*) - *(.srodata .srodata.*) - *(.gnu.linkonce.r.*) - } - - .data : { - *(.data .data.*) - *(.sdata .sdata.*) - *(.gnu.linkonce.d.*) - *(.gnu.linkonce.s.*) - PROVIDE(_edata = .); - } - - .tdata ALIGN(16) : { - PROVIDE(__tls_start = .); - *(.tdata .tdata.* .gnu.linkonce.td.*) - PROVIDE(__tdata_end = .); - } - - .tbss ALIGN(8) (NOLOAD) : { - PROVIDE(__bss_start = .); - *(.tbss .tbss.* .gnu.linkonce.tb.*) - . = ALIGN(8); - PROVIDE(__tls_end = .); - } - . = ADDR(.tbss) + SIZEOF(.tbss); - - /* All zero-init sections: crt0.S clears __bss_start .. __bss_end, - so heap (at _end) starts after ALL bss including sbss. */ - .bss (NOLOAD) : { - *(.bss .bss.*) - *(.sbss .sbss.*) - *(.gnu.linkonce.b.*) - *(.gnu.linkonce.sb.*) - *(COMMON) - . = ALIGN(8); - PROVIDE(__bss_end = .); - } - - .noinit (NOLOAD) : { - *(.noinit .noinit.*) - } - - . = ALIGN(16); - PROVIDE(end = .); - PROVIDE(_end = .); -} diff --git a/bb-tests/workloads/lib/bbsw/baremetal/crt0.S b/bb-tests/workloads/lib/bbsw/baremetal/crt0.S deleted file mode 100644 index 81ec0f2aa..000000000 --- a/bb-tests/workloads/lib/bbsw/baremetal/crt0.S +++ /dev/null @@ -1,71 +0,0 @@ -/* crt0.S — minimal startup for BBSimHarness baremetal - * Bootrom jumps to 0x80000000 (this code). - * Sets up stack and TLS, clears BSS, then calls main. */ -#include -.section .text.init,"ax",@progbits -.global _start -_start: - /* The generated target profile selects its test hart before boot. */ - csrr t0, mhartid - li t1, BB_TEST_HART - bne t0, t1, park_hart - - /* Enable FPU: set mstatus.FS = 01 (Initial) */ - li t0, 0x2000 # mstatus.FS = 01 (bit 13) - csrs mstatus, t0 - - /* Set up trap vector (mtvec) to trap_entry */ - la t0, trap_entry - csrw mtvec, t0 - - /* stack pointer: top of the 256 MiB BBSim DRAM window */ - li sp, 0x90000000 - - .option push - .option norelax - la tp, __tls_start - .option pop - - /* clear BSS */ - la a0, __bss_start - la a1, __bss_end -bss_loop: - bgeu a0, a1, bss_done - sd zero, 0(a0) - addi a0, a0, 8 - j bss_loop -bss_done: - - li a0, 0 - li a1, 0 - call main - - /* if main returns, write its return code to sim_exit */ - mv t0, a0 - li a0, 0x60000000 - sw t0, 0(a0) -1: j 1b - -/* Trap handler: print mcause and mepc, then exit */ -.align 4 -trap_entry: - /* Save context */ - addi sp, sp, -16 - sd ra, 8(sp) - sd a0, 0(sp) - - /* Read mcause and mepc for debugging */ - csrr a0, mcause - csrr a1, mepc - - /* Exit simulation with error code */ - li a0, 0x60000000 - li t0, 1 - sw t0, 0(a0) - - /* Infinite loop */ -1: j 1b - -park_hart: - wfi - j park_hart diff --git a/bb-tests/workloads/lib/bbsw/baremetal/multicore/multicore.h b/bb-tests/workloads/lib/bbsw/baremetal/multicore/multicore.h deleted file mode 100644 index 134a49c8c..000000000 --- a/bb-tests/workloads/lib/bbsw/baremetal/multicore/multicore.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef BBSW_BAREMETAL_MULTICORE_H -#define BBSW_BAREMETAL_MULTICORE_H - -#include -#include - -static inline uint32_t bb_get_hart_id(void) { - uint32_t hart; - asm volatile("csrr %0, mhartid" : "=r"(hart)); - return hart; -} - -static inline core_id_t bb_get_core_id(void) { - return bb_topology_core_id(bb_get_hart_id()); -} - -#endif diff --git a/bb-tests/workloads/lib/bbsw/baremetal/start.S b/bb-tests/workloads/lib/bbsw/baremetal/start.S deleted file mode 100644 index 3b943056c..000000000 --- a/bb-tests/workloads/lib/bbsw/baremetal/start.S +++ /dev/null @@ -1,100 +0,0 @@ -.section .text.init -.global _start - -#include - -_start: - # Get hart ID - csrr t0, mhartid - - # Enable the FPU on every hart: mstatus.FS = Initial - li t1, 0x2000 - csrs mstatus, t1 - - # Set up stack pointer for each hart - # Each hart gets 4KB stack space (0x1000 bytes) - li sp, 0x90000000 - li t1, 0x1000 - mul t1, t0, t1 - sub sp, sp, t1 - # Each hart owns a TLS copy at the top of its 4KB stack slot. - la t2, __tls_start - la t3, __tdata_end - la t4, __tls_end - sub t5, t4, t2 - addi t5, t5, 15 - andi t5, t5, -16 - sub sp, sp, t5 - mv tp, sp - mv t6, tp -.Lcopy_tls: - bgeu t2, t3, .Lzero_tls - lbu a0, 0(t2) - sb a0, 0(t6) - addi t2, t2, 1 - addi t6, t6, 1 - j .Lcopy_tls -.Lzero_tls: - bgeu t2, t4, .Lrun_main - sb zero, 0(t6) - addi t2, t2, 1 - addi t6, t6, 1 - j .Lzero_tls -.Lrun_main: - # Loaders write only the file image, so .bss can hold a previous run's values on P2E DDR. - # The reporting hart clears it, then releases the others through the last flag write. - csrr t0, mhartid - la t1, .Lbss_ready - li t2, BB_REPORT_HART - bne t0, t2, .Lwait_bss - la t2, __bss_start - la t3, __bss_end - addi t4, t3, -63 -.Lclear_bss_blocks: - bgeu t2, t4, .Lclear_bss - sd zero, 0(t2) - sd zero, 8(t2) - sd zero, 16(t2) - sd zero, 24(t2) - sd zero, 32(t2) - sd zero, 40(t2) - sd zero, 48(t2) - sd zero, 56(t2) - addi t2, t2, 64 - j .Lclear_bss_blocks -.Lclear_bss: - bgeu t2, t3, .Lbss_cleared - sd zero, 0(t2) - addi t2, t2, 8 - j .Lclear_bss -.Lbss_cleared: - li t2, 1 - fence rw, w - sw t2, 0(t1) - j .Lcall_main -.Lwait_bss: - lw t2, 0(t1) - beqz t2, .Lwait_bss - fence r, rw -.Lcall_main: - # Jump to main for all harts; the workload dispatches by core id. - li a0, 0 - li a1, 0 - call main - - # Core 0 of the work tile reports the workload result. Other harts remain parked. - csrr t0, mhartid - li t1, BB_REPORT_HART - bne t0, t1, 1f - li t1, 0x60000000 - sw a0, 0(t1) -1: - wfi - j 1b - -# Outside .bss so the clear cannot reset it; a stale 1 from a previous run is -# overwritten by the image load because .data is part of the file image. -.section .data -.balign 4 -.Lbss_ready: - .word 0 diff --git a/bb-tests/workloads/lib/bbsw/baremetal/syscalls.c b/bb-tests/workloads/lib/bbsw/baremetal/syscalls.c deleted file mode 100644 index a37464db6..000000000 --- a/bb-tests/workloads/lib/bbsw/baremetal/syscalls.c +++ /dev/null @@ -1,18 +0,0 @@ -#include - -#define MMIO_SIM_EXIT ((volatile uint32_t *)0x60000000UL) -#define MMIO_UART_TX ((volatile uint32_t *)0x60020000UL) - -int _write(int fd, const char *buf, int len) { - (void)fd; - for (int i = 0; i < len; ++i) { - *MMIO_UART_TX = (uint32_t)(unsigned char)buf[i]; - } - return len; -} - -void __attribute__((noreturn)) _exit(int code) { - *MMIO_SIM_EXIT = (uint32_t)code; - while (1) { - } -} diff --git a/bb-tests/workloads/lib/bbsw/baremetal/test.c b/bb-tests/workloads/lib/bbsw/baremetal/test.c deleted file mode 100644 index fa2b77579..000000000 --- a/bb-tests/workloads/lib/bbsw/baremetal/test.c +++ /dev/null @@ -1,12 +0,0 @@ -#include -#include - -#define MMIO_SIM_EXIT ((volatile uint32_t *)0x60000000UL) - -void bb_test_pass(void) { *MMIO_SIM_EXIT = 0; } - -void bb_test_fail(void) { - *MMIO_SIM_EXIT = 1; - while (1) { - } -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/CMakeLists.txt b/bb-tests/workloads/lib/bbsw/kernels/CMakeLists.txt deleted file mode 100644 index aa9a73446..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -get_filename_component(REPO_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../../../../.." ABSOLUTE) -add_library(scalar_kernels STATIC mxfp8/pack.cpp mxfp8/quant.cpp) -target_compile_options(scalar_kernels PRIVATE -O2) -target_compile_features(scalar_kernels PRIVATE cxx_std_20) -target_include_directories(scalar_kernels PRIVATE - "${REPO_ROOT}/stack/compiler/thirdparty/buddy-mlir/llvm/mlir/include/mlir/ExecutionEngine") diff --git a/bb-tests/workloads/lib/bbsw/kernels/mxfp8/CMakeLists.txt b/bb-tests/workloads/lib/bbsw/kernels/mxfp8/CMakeLists.txt deleted file mode 100644 index 09552e568..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/mxfp8/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -get_filename_component(REPO_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../.." ABSOLUTE) -add_library(mxfp8_pack STATIC pack.cpp) -target_compile_options(mxfp8_pack PRIVATE -O2) -target_compile_features(mxfp8_pack PRIVATE cxx_std_20) -target_include_directories(mxfp8_pack PRIVATE - "${REPO_ROOT}/stack/compiler/thirdparty/buddy-mlir/llvm/mlir/include/mlir/ExecutionEngine") - -add_library(mxfp8_pack_ant STATIC EXCLUDE_FROM_ALL pack.cpp) -target_link_libraries(mxfp8_pack_ant PRIVATE runtime_ant_tasks) -target_compile_options(mxfp8_pack_ant PRIVATE -O2) -target_compile_features(mxfp8_pack_ant PRIVATE cxx_std_20) -target_include_directories(mxfp8_pack_ant PRIVATE - "${REPO_ROOT}/stack/compiler/thirdparty/buddy-mlir/llvm/mlir/include/mlir/ExecutionEngine") diff --git a/bb-tests/workloads/lib/bbsw/kernels/mxfp8/pack.cpp b/bb-tests/workloads/lib/bbsw/kernels/mxfp8/pack.cpp deleted file mode 100644 index 4fbc78c13..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/mxfp8/pack.cpp +++ /dev/null @@ -1,69 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include "quant.h" - -extern "C" void _mlir_ciface_mxfp8_quant(StridedMemRefType *input, - StridedMemRefType *output, - int64_t tileRows, int64_t tileK, - int64_t panelStride) { - int64_t m = input->sizes[0], k = input->sizes[1]; - int64_t mt = (m + tileRows - 1) / tileRows, kt = (k + tileK - 1) / tileK; - int64_t bytes = panelStride; - if (k % 32 || tileK % 32 || output->sizes[0] != mt * kt * bytes) { - fprintf(stderr, "mxfp8: invalid activation packing shape\n"); - abort(); - } - auto *source = input->data + input->offset; - auto *dest = output->data + output->offset; - const int64_t rowStride = input->strides[0], columnStride = input->strides[1]; - const int64_t outputStride = output->strides[0]; - for (int64_t mr = 0; mr < mt; ++mr) - for (int64_t kr = 0; kr < kt; ++kr) - for (int64_t row = 0; row < tileRows; ++row) { - int64_t width = std::min(tileK, k - kr * tileK); - int64_t base = (mr * kt + kr) * bytes; - if (mr * tileRows + row >= m && outputStride == 1) { - std::memset(dest + base + row * width, 0, width); - std::memset(dest + base + tileRows * width + row * (width / 32), 127, - width / 32); - continue; - } - alignas(16) uint32_t staging[1024]; - alignas(16) uint8_t packed[1056]; - for (int64_t begin = 0; begin < width; begin += 1024) { - int64_t count = std::min(1024, width - begin); - int64_t r = mr * tileRows + row, c = kr * tileK + begin; - int64_t valid = - r < m ? std::max(0, std::min(count, k - c)) : 0; - const uint32_t *values; - if (valid == count && columnStride == 1) { - values = - reinterpret_cast(source + r * rowStride + c); - } else { - for (int64_t i = 0; i < valid; ++i) - staging[i] = __builtin_bit_cast( - uint32_t, source[r * rowStride + (c + i) * columnStride]); - std::fill(staging + valid, staging + count, 0); - values = staging; - } - mxfp8_quant(values, packed, count); - int64_t codes = base + row * width + begin; - int64_t scales = - base + tileRows * width + row * (width / 32) + begin / 32; - if (outputStride == 1) { - std::memcpy(dest + codes, packed, count); - std::memcpy(dest + scales, packed + count, count / 32); - } else { - for (int64_t i = 0; i < count; ++i) - dest[(codes + i) * outputStride] = packed[i]; - for (int64_t i = 0; i < count / 32; ++i) - dest[(scales + i) * outputStride] = packed[count + i]; - } - } - } -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/mxfp8/quant.cpp b/bb-tests/workloads/lib/bbsw/kernels/mxfp8/quant.cpp deleted file mode 100644 index 258d63c20..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/mxfp8/quant.cpp +++ /dev/null @@ -1,46 +0,0 @@ -#include "quant.h" -#include -#include -#include - -static uint32_t roundedShift(uint32_t value, unsigned shift) { - if (shift > 31) - return 0; - uint32_t half = uint32_t(1) << (shift - 1); - return (value + half - 1 + ((value >> shift) & 1)) >> shift; -} - -static uint8_t encode(uint32_t bits, int blockExponent) { - uint8_t sign = (bits >> 24) & 128; - if (!(bits & 0x7fffffff)) - return sign; - int exponent = int((bits >> 23) & 255); - uint32_t fraction = bits & 0x7fffff; - if (exponent == 0) { - int shift = __builtin_clz(fraction) - 8; - fraction = (fraction << shift) & 0x7fffff; - exponent = 1 - shift; - } - exponent -= blockExponent + 120; - uint32_t code = - exponent <= 0 ? roundedShift(fraction | 0x800000, unsigned(21 - exponent)) - : unsigned(exponent * 8) + roundedShift(fraction, 20); - return sign | std::min(code, 126u); -} - -extern "C" void mxfp8_quant(const uint32_t *input, uint8_t *output, - uint32_t count) { - for (uint32_t begin = 0; begin < count; begin += 32) { - uint32_t maximum = 0; - for (uint32_t i = 0; i < 32; ++i) - maximum = std::max(maximum, input[begin + i] & 0x7fffffff); - if (maximum >= 0x7f800000) { - fputs("mxfp8: non-finite activation\n", stderr); - abort(); - } - int exponent = maximum == 0 ? 0 : std::max(-127, int(maximum >> 23) - 135); - output[count + begin / 32] = uint8_t(exponent + 127); - for (uint32_t i = 0; i < 32; ++i) - output[begin + i] = encode(input[begin + i], exponent); - } -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/mxfp8/quant.h b/bb-tests/workloads/lib/bbsw/kernels/mxfp8/quant.h deleted file mode 100644 index 4d82efd3e..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/mxfp8/quant.h +++ /dev/null @@ -1,5 +0,0 @@ -#pragma once -#include - -extern "C" void mxfp8_quant(const uint32_t *input, uint8_t *output, - uint32_t count); diff --git a/bb-tests/workloads/lib/bbsw/kernels/mxquant/CMakeLists.txt b/bb-tests/workloads/lib/bbsw/kernels/mxquant/CMakeLists.txt deleted file mode 100644 index fcb5dc2fe..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/mxquant/CMakeLists.txt +++ /dev/null @@ -1,16 +0,0 @@ -get_filename_component(REPO_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../.." ABSOLUTE) -add_library(mxquant_backend STATIC dispatch.cpp) -target_link_libraries(mxquant_backend PRIVATE runtime) -target_compile_options(mxquant_backend PRIVATE -O2) -target_compile_features(mxquant_backend PRIVATE cxx_std_20) -target_include_directories(mxquant_backend PRIVATE - ${RUNTIME_PARAMS} "${REPO_ROOT}/bb-tests/workloads/lib" - "${REPO_ROOT}/examples/balls/mxquant/workloads") - -add_library(mxquant_backend_ant STATIC EXCLUDE_FROM_ALL dispatch.cpp) -target_link_libraries(mxquant_backend_ant PRIVATE runtime_ant_tasks) -target_compile_options(mxquant_backend_ant PRIVATE -O2) -target_compile_features(mxquant_backend_ant PRIVATE cxx_std_20) -target_include_directories(mxquant_backend_ant PRIVATE - ${RUNTIME_PARAMS} "${REPO_ROOT}/bb-tests/workloads/lib" - "${REPO_ROOT}/examples/balls/mxquant/workloads") diff --git a/bb-tests/workloads/lib/bbsw/kernels/mxquant/dispatch.cpp b/bb-tests/workloads/lib/bbsw/kernels/mxquant/dispatch.cpp deleted file mode 100644 index a99ca82b3..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/mxquant/dispatch.cpp +++ /dev/null @@ -1,15 +0,0 @@ -#include "../mxfp8/quant.h" -#include -#include - -extern "C" void mxfp8_quant(const uint32_t *input, uint8_t *output, - uint32_t count) { - constexpr unsigned inputBank = 0, outputBank = 1; - bb_mem_alloc(inputBank, 1, 1); - bb_mem_alloc(outputBank, 1, 1); - bb_mvin((uintptr_t)input, inputBank, count / 4, 1); - bb_mxquant(inputBank, outputBank, count); - bb_mvout((uintptr_t)output, outputBank, (count + count / 32 + 15) / 16, 1); - bb_mem_release(inputBank); - bb_mem_release(outputBank); -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/CMakeLists.txt b/bb-tests/workloads/lib/bbsw/kernels/rvv/CMakeLists.txt deleted file mode 100644 index 746874b0c..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/CMakeLists.txt +++ /dev/null @@ -1,57 +0,0 @@ -get_filename_component(REPO_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../.." ABSOLUTE) -add_library(rvv_kernels STATIC dispatch.cpp matmul_dispatch.cpp cache_dispatch.cpp normalization_dispatch.cpp pointwise_dispatch.cpp activation_dispatch.cpp flash_attention_dispatch.cpp flash_attention_mxfp8_dispatch.cpp) -target_link_libraries(rvv_kernels PRIVATE runtime rvv_images) -target_compile_options(rvv_kernels PRIVATE -O2) -target_compile_features(rvv_kernels PRIVATE cxx_std_20) -target_include_directories(rvv_kernels PRIVATE - ${RUNTIME_PARAMS} - "${REPO_ROOT}/bb-tests/workloads/lib" - "${REPO_ROOT}/stack/compiler/thirdparty/buddy-mlir/llvm/mlir/include/mlir/ExecutionEngine") - -set(IMAGE_SOURCE "${CMAKE_CURRENT_BINARY_DIR}/images.cpp") -set(IMAGE_HEADER "${CMAKE_CURRENT_BINARY_DIR}/images.h") -add_custom_command( - OUTPUT "${IMAGE_SOURCE}" "${IMAGE_HEADER}" - BYPRODUCTS silu.o snake.o quant.o dequant.o matmul.o norm.o softmax.o rope.o exp.o sin.o cos.o - layernorm.o gelu.o tanh.o erf.o pointwise.o layernorm.elf gelu.elf tanh.elf pointwise.elf - trig.o sin.elf cos.elf flash_attention.o flash_attention.elf - flash_attention_mxfp8.o flash_attention_mxfp8.elf - silu.elf swiglu.elf snake.elf quant.elf cache_quant.elf dequant.elf matmul.elf norm.elf norm_no_weight.elf softmax.elf attention_softmax.elf logsumexp_softmax.elf rope.elf - COMMAND "${PYTHON}" "${CMAKE_CURRENT_SOURCE_DIR}/build_images.py" - --output "${CMAKE_CURRENT_BINARY_DIR}" - DEPENDS build_images.py kernel.ld kernels.h silu.cpp snake.cpp quant.cpp dequant.cpp matmul.cpp - norm.cpp softmax.cpp rope.cpp layernorm.cpp gelu.cpp tanh.cpp trig.cpp pointwise.cpp math/erf.cpp - flash_attention.cpp flash_attention.h flash_attention_mxfp8.cpp math/mxfp8.h - math/exp.cpp math/sin.cpp math/cos.cpp math/math.h math/constants.h - VERBATIM) -add_custom_target(rvv-images DEPENDS "${IMAGE_SOURCE}" "${IMAGE_HEADER}") -add_library(rvv_images STATIC "${IMAGE_SOURCE}") -target_compile_features(rvv_images PRIVATE cxx_std_20) -add_dependencies(rvv_images rvv-images) -target_include_directories(rvv_images PRIVATE "${CMAKE_CURRENT_BINARY_DIR}") -target_include_directories(rvv_kernels PRIVATE "${CMAKE_CURRENT_BINARY_DIR}") - -add_library(rvv_quant_backend STATIC quant_dispatch.cpp) -target_link_libraries(rvv_quant_backend PRIVATE runtime rvv_images) -target_compile_options(rvv_quant_backend PRIVATE -O2) -target_compile_features(rvv_quant_backend PRIVATE cxx_std_20) -target_include_directories(rvv_quant_backend PRIVATE - ${RUNTIME_PARAMS} "${REPO_ROOT}/bb-tests/workloads/lib" - "${CMAKE_CURRENT_BINARY_DIR}") -add_dependencies(rvv_quant_backend rvv-images) - -add_library(rvv_kernels_ant STATIC EXCLUDE_FROM_ALL dispatch.cpp matmul_dispatch.cpp cache_dispatch.cpp normalization_dispatch.cpp pointwise_dispatch.cpp activation_dispatch.cpp flash_attention_dispatch.cpp flash_attention_mxfp8_dispatch.cpp) -target_link_libraries(rvv_kernels_ant PRIVATE runtime_ant_tasks rvv_images) -target_compile_options(rvv_kernels_ant PRIVATE -O2) -target_compile_features(rvv_kernels_ant PRIVATE cxx_std_20) -target_include_directories(rvv_kernels_ant PRIVATE ${RUNTIME_PARAMS} - "${REPO_ROOT}/bb-tests/workloads/lib" "${CMAKE_CURRENT_BINARY_DIR}" - "${REPO_ROOT}/stack/compiler/thirdparty/buddy-mlir/llvm/mlir/include/mlir/ExecutionEngine") -add_dependencies(rvv_kernels_ant rvv-images) -add_library(rvv_quant_backend_ant STATIC EXCLUDE_FROM_ALL quant_dispatch.cpp) -target_link_libraries(rvv_quant_backend_ant PRIVATE runtime_ant_tasks rvv_images) -target_compile_options(rvv_quant_backend_ant PRIVATE -O2) -target_compile_features(rvv_quant_backend_ant PRIVATE cxx_std_20) -target_include_directories(rvv_quant_backend_ant PRIVATE ${RUNTIME_PARAMS} - "${REPO_ROOT}/bb-tests/workloads/lib" "${CMAKE_CURRENT_BINARY_DIR}") -add_dependencies(rvv_quant_backend_ant rvv-images) diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/activation_dispatch.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/activation_dispatch.cpp deleted file mode 100644 index d6f74ee35..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/activation_dispatch.cpp +++ /dev/null @@ -1,86 +0,0 @@ -#include "images.h" -#include -#include -#include -#include -#include -#include -#include - -namespace { -constexpr unsigned readBank = 3, writeBank = 4; -constexpr unsigned programBank = VIRTUAL_BANK_NUM; -constexpr size_t bankBytes = BANK_LINES * (BANK_WIDTH / 8); - -int64_t offset(const DynamicMemRefType &value, int64_t index) { - int64_t result = value.offset; - for (int64_t axis = value.rank - 1; axis >= 0; --axis) { - result += (index % value.sizes[axis]) * value.strides[axis]; - index /= value.sizes[axis]; - } - return result; -} -} // namespace - -static void unary(UnrankedMemRefType *output, - UnrankedMemRefType *input, - const images::KernelImage &image) { - DynamicMemRefType out(*output), in(*input); - if (out.rank != in.rank) - abort(); - int64_t count = 1; - for (int64_t axis = in.rank - 1; axis >= 0; --axis) { - if (in.sizes[axis] <= 0 || out.sizes[axis] != in.sizes[axis]) - abort(); - count *= in.sizes[axis]; - } - alignas(16) float source[bankBytes / sizeof(float)]; - alignas(16) float result[bankBytes / sizeof(float)]; - for (unsigned bank = 0; bank < 3; ++bank) - bb_mem_alloc(bank, 1, 1); - bb_mem_transfer(0, readBank); - bb_mem_transfer(1, writeBank); - bb_mem_transfer(2, writeBank); - mvin_kernel(image.bytes, image.size, programBank); - uint32_t rounding; - asm volatile("csrr %0, frm" : "=r"(rounding)::"memory"); - for (int64_t begin = 0; begin < count; begin += bankBytes / sizeof(float)) { - uint32_t length = - std::min(bankBytes / sizeof(float), count - begin); - for (uint32_t i = 0; i < length; ++i) - std::memcpy(source + i, in.data + offset(in, begin + i), sizeof(float)); - std::fill(source + length, source + (length + 3) / 4 * 4, 0.0f); - bb_mvin_group((uintptr_t)source, readBank, 0, (length + 3) / 4, 1); - alignas(16) kernel_launch packet{ - image.entry, - image.text_bytes, - 0x40002000, - {uint64_t(2) << 32, 0, length, - (uint64_t(1) << 32) + offsetof(kernel_launch, reserved), - rounding << 5}, - 0}; - bb_mvin_group((uintptr_t)&packet, writeBank, 0, sizeof(packet) / 16, 1); - run_kernel(readBank, programBank, writeBank, 0); - bb_mvout_group((uintptr_t)result, writeBank, 1, (length + 3) / 4, 1); - bb_mvout_group((uintptr_t)&packet, writeBank, 0, sizeof(packet) / 16, 1); - asm volatile("csrs fflags, %0" ::"r"(packet.reserved) : "memory"); - for (uint32_t i = 0; i < length; ++i) - std::memcpy(out.data + offset(out, begin + i), result + i, sizeof(float)); - } - bb_mem_release(readBank); - bb_mem_release(writeBank); - release_kernel(programBank); -} - -extern "C" void _mlir_ciface_rvv_tanh(UnrankedMemRefType *output, - UnrankedMemRefType *input) { - unary(output, input, images::tanh); -} -extern "C" void _mlir_ciface_rvv_sin(UnrankedMemRefType *output, - UnrankedMemRefType *input) { - unary(output, input, images::sin); -} -extern "C" void _mlir_ciface_rvv_cos(UnrankedMemRefType *output, - UnrankedMemRefType *input) { - unary(output, input, images::cos); -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/build_images.py b/bb-tests/workloads/lib/bbsw/kernels/rvv/build_images.py deleted file mode 100644 index 5f4537eb6..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/build_images.py +++ /dev/null @@ -1,198 +0,0 @@ -import argparse -from pathlib import Path -import struct -import subprocess - -KERNELS = { - "silu": "rvv_silu_launch", - "swiglu": "rvv_swiglu_launch", - "snake": "rvv_snake", - "quant": "rvv_quant", - "cache_quant": "rvv_cache_quant", - "dequant": "rvv_dequant", - "matmul": "rvv_matmul", - "norm": "rvv_norm_launch", - "norm_no_weight": "rvv_norm_no_weight_launch", - "softmax": "rvv_softmax_launch", - "attention_softmax": "rvv_attention_softmax_launch", - "logsumexp_softmax": "rvv_logsumexp_softmax_launch", - "rope": "rvv_rope_launch", - "layernorm": "rvv_layernorm", - "gelu": "rvv_gelu", - "tanh": "rvv_tanh_launch", - "sin": "rvv_sin_launch", - "cos": "rvv_cos_launch", - "flash_attention": "rvv_flash_attention_launch", - "flash_attention_mxfp8": "rvv_flash_attention_mxfp8_launch", - "pointwise": "rvv_pointwise", -} - - -def read_segments(path): - elf = path.read_bytes() - header = struct.unpack_from("<16sHHIQQQIHHHHHH", elf) - ident, kind, machine = header[:3] - entry, shoff, shsize, shnum, names_index = ( - header[4], - header[6], - header[11], - header[12], - header[13], - ) - if ident[:6] != b"\x7fELF\x02\x01" or kind != 2 or machine != 243: - raise ValueError(f"{path}: expected a linked little-endian RISC-V ELF64") - sections = [ - struct.unpack_from(" 4096 - or text[5] % 4 - or entry >= text[5] - or entry % 4 - ): - raise ValueError(f"{path}: invalid RVV instruction memory layout") - data_address = 0x40000000 - initialized = b"" - if ".rodata" in sections and sections[".rodata"][5]: - section = sections[".rodata"] - if section[1] != 1 or section[3] != data_address or section[5] > 4096: - raise ValueError(f"{path}: invalid RVV constant memory layout") - initialized = elf[section[4] : section[4] + section[5]] - initialized += bytes((-len(initialized)) % 4) - return elf[text[4] : text[4] + text[5]], initialized, entry - - -def array(name, content): - rows = [ - " " + ", ".join(f"0x{value:02x}" for value in content[offset : offset + 16]) - for offset in range(0, len(content), 16) - ] - return f"alignas(4) const uint8_t {name}[] = {{\n" + ",\n".join(rows) + "\n};\n" - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--output", required=True, type=Path) - args = parser.parse_args() - source = Path(__file__).resolve().parent - tools = source.parents[5] / "result/bin" - args.output.mkdir(parents=True, exist_ok=True) - objects = [] - for filename in ( - "silu.cpp", - "snake.cpp", - "quant.cpp", - "dequant.cpp", - "matmul.cpp", - "norm.cpp", - "softmax.cpp", - "rope.cpp", - "layernorm.cpp", - "gelu.cpp", - "tanh.cpp", - "trig.cpp", - "flash_attention.cpp", - "flash_attention_mxfp8.cpp", - "pointwise.cpp", - "math/erf.cpp", - "math/exp.cpp", - "math/sin.cpp", - "math/cos.cpp", - ): - output = args.output / (Path(filename).stem + ".o") - subprocess.run( - [ - str(tools / "clang++"), - "--target=riscv64", - "-march=rv64imf_zve32f_zvl128b", - "-mabi=lp64f", - "-mcmodel=medany", - "-ffreestanding", - "-O2", - "-ffp-contract=off", - "-ffunction-sections", - "-fdata-sections", - *( - ["-fno-vectorize", "-fno-slp-vectorize"] - if filename == "math/erf.cpp" - else [] - ), - "-c", - str(source / filename), - "-o", - str(output), - ], - check=True, - ) - objects.append(output) - arrays = [] - images = [] - for name, symbol in KERNELS.items(): - path = args.output / (name + ".elf") - subprocess.run( - [ - str(tools / "riscv64-unknown-elf-ld"), - "-m", - "elf64lriscv", - "--gc-sections", - "-e", - symbol, - "-T", - str(source / "kernel.ld"), - *map(str, objects), - "-o", - str(path), - ], - check=True, - ) - text, data, entry = read_segments(path) - image = ( - struct.pack("<6I", 0x31564B52, len(text), entry, 0x40000000, len(data), 0) - + text - + data - ) - arrays.append(array(name + "_image", image)) - images.append( - f"const KernelImage {name}{{{name}_image, sizeof({name}_image), " - f"{entry}, {len(text)}}};" - ) - print( - f"{name}: image={len(image)} text={len(text)} data={len(data)} " - f"bss=0 entry={entry}" - ) - (args.output / "images.h").write_text( - "#pragma once\n#include \n\nnamespace images {\n" - "struct KernelImage {\n" - " const uint8_t *bytes;\n uint32_t size;\n" - " uint32_t entry;\n uint32_t text_bytes;\n};\n" - + "".join(f"extern const KernelImage {name};\n" for name in KERNELS) - + "}\n" - ) - (args.output / "images.cpp").write_text( - '#include "images.h"\n\nnamespace images {\nnamespace {\n' - + "\n".join(arrays) - + "}\n" - + "\n".join(images) - + "\n}\n" - ) - - -if __name__ == "__main__": - main() diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/cache_dispatch.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/cache_dispatch.cpp deleted file mode 100644 index a78d85c93..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/cache_dispatch.cpp +++ /dev/null @@ -1,179 +0,0 @@ -#include "images.h" -#include -#include -#include -#include -#include -#include -#include - -namespace { -constexpr unsigned readBank = 4, writeBank = 5; -constexpr unsigned programBank = VIRTUAL_BANK_NUM; -constexpr uint32_t bankBytes = BANK_LINES * (BANK_WIDTH / 8); -constexpr int64_t chunk = bankBytes / 128 * 32; -static_assert(chunk > 0 && bankBytes <= 65536); - -template int64_t elements(const DynamicMemRefType &value) { - if (!value.rank || value.sizes[value.rank - 1] <= 0) - abort(); - int64_t count = 1; - for (int64_t axis = 0; axis < value.rank; ++axis) { - if (value.sizes[axis] <= 0) - abort(); - count *= value.sizes[axis]; - } - return count; -} - -template -void copyRows(DynamicMemRefType &value, T *packed, int64_t first, - int64_t count, bool write) { - const int64_t width = value.sizes[value.rank - 1]; - const int64_t stride = value.strides[value.rank - 1]; - for (int64_t done = 0; done < count;) { - int64_t index = first + done; - const int64_t length = std::min(count - done, width - index % width); - int64_t offset = value.offset; - for (int64_t axis = value.rank - 1; axis >= 0; --axis) { - offset += index % value.sizes[axis] * value.strides[axis]; - index /= value.sizes[axis]; - } - T *row = value.data + offset; - if (stride == 1) { - if (write) - std::memcpy(row, packed + done, length * sizeof(T)); - else - std::memcpy(packed + done, row, length * sizeof(T)); - } else { - for (int64_t column = 0; column < length; ++column) { - if (write) - row[column * stride] = packed[done + column]; - else - packed[done + column] = row[column * stride]; - } - } - done += length; - } -} - -void check(DynamicMemRefType &codes, - DynamicMemRefType &scales) { - if (codes.rank != scales.rank || codes.sizes[codes.rank - 1] % 32) - abort(); - for (int64_t axis = 0; axis < codes.rank; ++axis) - if (scales.sizes[axis] != - codes.sizes[axis] / (axis == codes.rank - 1 ? 32 : 1)) - abort(); -} - -void load(const images::KernelImage &image, bool encoding) { - for (unsigned bank = 0; bank < 4; ++bank) - bb_mem_alloc(bank, 1, 1); - if (encoding) { - bb_mem_transfer(3, readBank); - bb_mem_transfer(0, writeBank); - bb_mem_transfer(1, writeBank); - bb_mem_transfer(2, writeBank); - } else { - bb_mem_transfer(2, readBank); - bb_mem_transfer(3, readBank); - bb_mem_transfer(0, writeBank); - bb_mem_transfer(1, writeBank); - } - mvin_kernel(image.bytes, image.size, programBank); -} - -void launch(const images::KernelImage &image, uint32_t count, bool encoding) { - const uint64_t descriptor = uint64_t(encoding ? 1 : 2) << 32; - alignas(16) kernel_launch packet{}; - packet.entry = image.entry; - packet.end = image.text_bytes; - packet.stack = 0x40002000; - packet.args[0] = uint64_t(encoding ? 2 : 3) << 32; - packet.args[1] = encoding ? uint64_t(3) << 32 : 0; - packet.args[2] = encoding ? 0 : uint64_t(1) << 32; - packet.args[3] = count; - packet.args[4] = descriptor + offsetof(kernel_launch, reserved); - bb_mvin_group((uintptr_t)&packet, writeBank, 0, sizeof(packet) / 16, 1); - run_kernel(readBank, programBank, writeBank, 0); - alignas(16) kernel_launch completed; - bb_mvout_group((uintptr_t)&completed, writeBank, 0, sizeof(completed) / 16, - 1); - if (std::memcmp(&packet, &completed, offsetof(kernel_launch, reserved)) || - completed.reserved) { - fputs("MXFP8 cache kernel rejected non-finite values, NaN encoding, or " - "FP32 overflow\n", - stderr); - abort(); - } -} - -void release() { - bb_mem_release(readBank); - bb_mem_release(writeBank); - release_kernel(programBank); -} -} // namespace - -extern "C" void -_mlir_ciface_rvv_mxfp8_encode(UnrankedMemRefType *codes, - UnrankedMemRefType *scales, - UnrankedMemRefType *input) { - DynamicMemRefType c(*codes), s(*scales); - DynamicMemRefType in(*input); - check(c, s); - int64_t count = elements(in); - if (elements(c) != count || elements(s) != count / 32 || in.rank != c.rank) - abort(); - for (int64_t axis = 0; axis < in.rank; ++axis) - if (in.sizes[axis] != c.sizes[axis]) - abort(); - alignas(16) float staging[chunk]; - alignas(16) int8_t codeData[chunk]; - alignas(16) int8_t scaleData[(chunk / 32 + 15) / 16 * 16]; - load(images::cache_quant, true); - for (int64_t begin = 0; begin < count; begin += chunk) { - uint32_t length = std::min(chunk, count - begin); - copyRows(in, staging, begin, length, false); - bb_mvin_group((uintptr_t)staging, readBank, 0, length / 4, 1); - launch(images::cache_quant, length, true); - bb_mvout_group((uintptr_t)codeData, writeBank, 1, length / 16, 1); - bb_mvout_group((uintptr_t)scaleData, writeBank, 2, (length / 32 + 15) / 16, - 1); - copyRows(c, codeData, begin, length, true); - copyRows(s, scaleData, begin / 32, length / 32, true); - } - release(); -} - -extern "C" void -_mlir_ciface_rvv_mxfp8_decode(UnrankedMemRefType *output, - UnrankedMemRefType *codes, - UnrankedMemRefType *scales) { - DynamicMemRefType out(*output); - DynamicMemRefType c(*codes), s(*scales); - check(c, s); - int64_t count = elements(c); - if (elements(out) != count || elements(s) != count / 32 || out.rank != c.rank) - abort(); - for (int64_t axis = 0; axis < out.rank; ++axis) - if (out.sizes[axis] != c.sizes[axis]) - abort(); - alignas(16) float staging[chunk]; - alignas(16) int8_t codeData[chunk]; - alignas(16) int8_t scaleData[(chunk / 32 + 15) / 16 * 16]{}; - load(images::dequant, false); - for (int64_t begin = 0; begin < count; begin += chunk) { - uint32_t length = std::min(chunk, count - begin); - copyRows(c, codeData, begin, length, false); - copyRows(s, scaleData, begin / 32, length / 32, false); - bb_mvin_group((uintptr_t)codeData, readBank, 0, length / 16, 1); - bb_mvin_group((uintptr_t)scaleData, readBank, 1, (length / 32 + 15) / 16, - 1); - launch(images::dequant, length, false); - bb_mvout_group((uintptr_t)staging, writeBank, 1, length / 4, 1); - copyRows(out, staging, begin, length, true); - } - release(); -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/dequant.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/dequant.cpp deleted file mode 100644 index c83e0ea40..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/dequant.cpp +++ /dev/null @@ -1,73 +0,0 @@ -#include "kernels.h" -#include - -extern "C" void rvv_dequant(uint32_t *output, const uint8_t *input, - const uint8_t *scales, size_t count, - uint32_t *status) { - *status = 0; - for (size_t block = 0; block < count / 32; ++block) { - unsigned scale = scales[block]; - if (scale == 255) { - *status = 1; - return; - } - for (size_t begin = 0; begin < 32;) { - size_t vl = __riscv_vsetvl_e32m1(32 - begin); - auto bytes = __riscv_vle8_v_u8mf4(input + block * 32 + begin, vl); - auto code = - __riscv_vzext_vf2_u32m1(__riscv_vzext_vf2_u16mf2(bytes, vl), vl); - auto sign = - __riscv_vsll_vx_u32m1(__riscv_vand_vx_u32m1(code, 128, vl), 24, vl); - auto magnitude = __riscv_vand_vx_u32m1(code, 127, vl); - if (__riscv_vcpop_m_b32(__riscv_vmseq_vx_u32m1_b32(magnitude, 127, vl), - vl)) { - *status = 1; - return; - } - auto exponent = __riscv_vsrl_vx_u32m1(magnitude, 3, vl); - auto fraction = __riscv_vand_vx_u32m1(code, 7, vl); - auto subnormal = __riscv_vmseq_vx_u32m1_b32(exponent, 0, vl); - auto highest = __riscv_vmv_v_x_u32m1(0, vl); - highest = __riscv_vmerge_vxm_u32m1( - highest, 1, __riscv_vmsgeu_vx_u32m1_b32(fraction, 2, vl), vl); - highest = __riscv_vmerge_vxm_u32m1( - highest, 2, __riscv_vmsgeu_vx_u32m1_b32(fraction, 4, vl), vl); - auto normalExponent = __riscv_vadd_vx_i32m1( - __riscv_vreinterpret_v_u32m1_i32m1(exponent), int(scale) - 7, vl); - auto subExponent = __riscv_vadd_vx_i32m1( - __riscv_vreinterpret_v_u32m1_i32m1(highest), int(scale) - 9, vl); - auto resultExponent = - __riscv_vmerge_vvm_i32m1(normalExponent, subExponent, subnormal, vl); - auto overflow = __riscv_vmand_mm_b32( - __riscv_vmsge_vx_i32m1_b32(resultExponent, 255, vl), - __riscv_vmsne_vx_u32m1_b32(magnitude, 0, vl), vl); - if (__riscv_vcpop_m_b32(overflow, vl)) { - *status = 2; - return; - } - auto normalSignificand = - __riscv_vsll_vx_u32m1(__riscv_vor_vx_u32m1(fraction, 8, vl), 20, vl); - auto subSignificand = __riscv_vsll_vv_u32m1( - fraction, __riscv_vrsub_vx_u32m1(highest, 23, vl), vl); - auto significand = __riscv_vmerge_vvm_u32m1( - normalSignificand, subSignificand, subnormal, vl); - auto normalBits = __riscv_vor_vv_u32m1( - __riscv_vsll_vx_u32m1( - __riscv_vreinterpret_v_i32m1_u32m1(resultExponent), 23, vl), - __riscv_vand_vx_u32m1(significand, 0x7fffff, vl), vl); - auto tinyBits = __riscv_vsrl_vv_u32m1( - significand, - __riscv_vreinterpret_v_i32m1_u32m1( - __riscv_vrsub_vx_i32m1(resultExponent, 1, vl)), - vl); - auto bits = __riscv_vmerge_vvm_u32m1( - normalBits, tinyBits, - __riscv_vmsle_vx_i32m1_b32(resultExponent, 0, vl), vl); - bits = __riscv_vmerge_vxm_u32m1( - bits, 0, __riscv_vmseq_vx_u32m1_b32(magnitude, 0, vl), vl); - __riscv_vse32_v_u32m1(output + block * 32 + begin, - __riscv_vor_vv_u32m1(bits, sign, vl), vl); - begin += vl; - } - } -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/dispatch.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/dispatch.cpp deleted file mode 100644 index 28ab8222d..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/dispatch.cpp +++ /dev/null @@ -1,442 +0,0 @@ -#include "images.h" -#include -#include -#include -#include -#include -#include -#include - -namespace { -constexpr unsigned readBank = 4, writeBank = 5; -constexpr unsigned outputBank = 3, inputBank = 0, upBank = 1; -constexpr uint64_t descriptorAddress = uint64_t(2) << 32; -constexpr unsigned programBank = VIRTUAL_BANK_NUM; -constexpr int64_t chunk = 1024; -constexpr uint32_t bankBytes = BANK_LINES * (BANK_WIDTH / 8); -static_assert(bankBytes >= 4096 && bankBytes <= 65536 && bankBytes % 16 == 0); - -int64_t elements(const DynamicMemRefType &value) { - int64_t count = 1; - for (int64_t axis = value.rank - 1; axis >= 0; --axis) { - if (value.sizes[axis] <= 0 || - (value.sizes[axis] != 1 && value.strides[axis] != count)) { - fputs("RVV kernel requires a positive contiguous FP32 tensor\n", stderr); - abort(); - } - count *= value.sizes[axis]; - } - return count; -} - -void load(const images::KernelImage &image, unsigned buffer) { - for (unsigned bank = 0; bank < 4; ++bank) - bb_mem_alloc(bank, 1, 1); - bb_mem_transfer(2, readBank); - bb_mem_transfer(3, readBank); - bb_mem_transfer(0, writeBank); - bb_mem_transfer(1, writeBank); - mvin_kernel(image.bytes, image.size, programBank + buffer); -} - -uint32_t roundingMode() { - uint32_t rounding; - asm volatile("csrr %0, frm" : "=r"(rounding)::"memory"); - return rounding << 5; -} - -void launch(const void *descriptor, uint32_t bytes, unsigned buffer) { - bb_mvin_group((uintptr_t)descriptor, writeBank, 0, bytes / 16, 1); - run_kernel(readBank, programBank + buffer, writeBank, 0); -} - -void send(const float *input, unsigned bank, uint32_t count, float *staging) { - if (count % 4) { - std::memcpy(staging, input, count * sizeof(float)); - std::fill_n(staging + count, 4 - count % 4, 0.0f); - input = staging; - } - bb_mvin_group((uintptr_t)input, readBank, bank, (count + 3) / 4, 1); -} - -void receive(float *output, uint32_t count, float *staging) { - bb_mvout_group((uintptr_t)(count % 4 ? staging : output), writeBank, 1, - (count + 3) / 4, 1); - if (count % 4) - std::memcpy(output, staging, count * sizeof(float)); -} - -// Tensor launch entries return private exception flags in the launch descriptor -// reserved field. -void receiveTensor(float *output, uint32_t count, float *staging) { - receive(output, count, staging); - alignas(16) kernel_launch descriptor; - bb_mvout_group((uintptr_t)&descriptor, writeBank, 0, sizeof(descriptor) / 16, - 1); - asm volatile("csrs fflags, %0" ::"r"(descriptor.reserved) : "memory"); -} - -void release(unsigned buffer) { - bb_mem_release(readBank); - bb_mem_release(writeBank); - release_kernel(programBank + buffer); -} -} // namespace - -extern "C" void _mlir_ciface_rvv_silu(UnrankedMemRefType *output, - UnrankedMemRefType *input) { - DynamicMemRefType out(*output), in(*input); - int64_t count = elements(in); - if (elements(out) != count) - abort(); - alignas(16) float staging[chunk]; - const uint32_t rounding = roundingMode(); - load(images::silu, 0); - for (int64_t begin = 0; begin < count; begin += chunk) { - uint32_t length = std::min(chunk, count - begin); - send(in.data + in.offset + begin, inputBank, length, staging); - kernel_launch call{ - images::silu.entry, - images::silu.text_bytes, - 0x40002000, - {uint64_t(outputBank) << 32, uint64_t(inputBank) << 32, length, - descriptorAddress + offsetof(kernel_launch, reserved), rounding}, - 0}; - launch(&call, sizeof(call), 0); - receiveTensor(out.data + out.offset + begin, length, staging); - } - release(0); -} - -extern "C" void _mlir_ciface_rvv_swiglu(UnrankedMemRefType *output, - UnrankedMemRefType *gate, - UnrankedMemRefType *up) { - DynamicMemRefType out(*output), a(*gate), b(*up); - int64_t count = elements(a); - if (elements(out) != count || elements(b) != count) - abort(); - alignas(16) float staging[chunk]; - const uint32_t rounding = roundingMode(); - load(images::swiglu, 0); - for (int64_t begin = 0; begin < count; begin += chunk) { - uint32_t length = std::min(chunk, count - begin); - send(a.data + a.offset + begin, inputBank, length, staging); - send(b.data + b.offset + begin, upBank, length, staging); - kernel_launch call{images::swiglu.entry, - images::swiglu.text_bytes, - 0x40002000, - {uint64_t(outputBank) << 32, uint64_t(inputBank) << 32, - uint64_t(upBank) << 32, length, - descriptorAddress + offsetof(kernel_launch, reserved), - rounding}, - 0}; - launch(&call, sizeof(call), 0); - receiveTensor(out.data + out.offset + begin, length, staging); - } - release(0); -} - -extern "C" void _mlir_ciface_rvv_snake(UnrankedMemRefType *output, - UnrankedMemRefType *input, - UnrankedMemRefType *logAlpha, - UnrankedMemRefType *logBeta) { - DynamicMemRefType out(*output), in(*input), alpha(*logAlpha), - beta(*logBeta); - int64_t channels = elements(alpha), count = elements(in); - if (elements(beta) != channels || elements(out) != count || count % channels) - abort(); - int64_t width = count / channels; - alignas(16) float staging[chunk]; - load(images::snake, 1); - // Short channels fit together in one input bank and one coefficient packet. - const int64_t batch = - width <= chunk - ? std::min(chunk / width, - (bankBytes - sizeof(kernel_launch)) / 8) - : 1; - alignas(16) uint8_t descriptor[bankBytes]; - for (int64_t channel = 0; channel < channels; channel += batch) { - uint32_t grouped = std::min(batch, channels - channel); - for (int64_t begin = 0; begin < width; begin += chunk) { - uint32_t length = std::min(chunk, width - begin); - int64_t offset = channel * width + begin; - uint32_t elements = grouped * length; - send(in.data + in.offset + offset, inputBank, elements, staging); - kernel_launch call{ - images::snake.entry, - images::snake.text_bytes, - 0x40002000, - {uint64_t(outputBank) << 32, uint64_t(inputBank) << 32, grouped, - length, descriptorAddress + sizeof(kernel_launch), - descriptorAddress + sizeof(kernel_launch) + grouped * 4}, - 0}; - uint32_t bytes = (sizeof(call) + grouped * 2 * sizeof(float) + 15) & ~15U; - std::memset(descriptor, 0, bytes); - std::memcpy(descriptor, &call, sizeof(call)); - std::memcpy(descriptor + sizeof(call), - alpha.data + alpha.offset + channel, grouped * sizeof(float)); - std::memcpy(descriptor + sizeof(call) + grouped * sizeof(float), - beta.data + beta.offset + channel, grouped * sizeof(float)); - launch(descriptor, bytes, 1); - receive(out.data + out.offset + offset, elements, staging); - } - } - release(1); -} - -extern "C" void _mlir_ciface_rvv_norm(UnrankedMemRefType *output, - UnrankedMemRefType *input, - UnrankedMemRefType *weight, - uint32_t meanMultiplierBits, - uint32_t epsilonBits) { - DynamicMemRefType out(*output), in(*input), w(*weight); - if (in.rank < 1 || in.strides[in.rank - 1] != 1) - abort(); - int64_t count = 1; - for (int64_t axis = 0; axis < in.rank; ++axis) { - if (in.sizes[axis] <= 0) - abort(); - count *= in.sizes[axis]; - } - if (elements(out) != count) - abort(); - int64_t width = in.sizes[in.rank - 1]; - if (elements(w) != width || width > bankBytes / sizeof(float)) { - fputs("RVV norm requires one complete row and its weights in banks\n", - stderr); - abort(); - } - alignas(16) float staging[bankBytes / sizeof(float)]; - const uint32_t rounding = roundingMode(); - load(images::norm, 0); - send(w.data + w.offset, upBank, width, staging); - for (int64_t begin = 0; begin < count; begin += width) { - int64_t row = begin / width, offset = in.offset; - for (int64_t axis = in.rank - 2; axis >= 0; --axis) { - offset += (row % in.sizes[axis]) * in.strides[axis]; - row /= in.sizes[axis]; - } - send(in.data + offset, inputBank, width, staging); - kernel_launch call{images::norm.entry, - images::norm.text_bytes, - 0x40002000, - {uint64_t(outputBank) << 32, uint64_t(inputBank) << 32, - uint64_t(upBank) << 32, uint32_t(width), - meanMultiplierBits, epsilonBits, - descriptorAddress + offsetof(kernel_launch, reserved), - rounding}, - 0}; - launch(&call, sizeof(call), 0); - receiveTensor(out.data + out.offset + begin, width, staging); - } - release(0); -} - -extern "C" void _mlir_ciface_rvv_norm_no_weight( - UnrankedMemRefType *output, UnrankedMemRefType *input, - uint32_t meanMultiplierBits, uint32_t epsilonBits) { - DynamicMemRefType out(*output), in(*input); - if (in.rank < 1 || in.strides[in.rank - 1] != 1) - abort(); - int64_t count = 1; - for (int64_t axis = 0; axis < in.rank; ++axis) { - if (in.sizes[axis] <= 0) - abort(); - count *= in.sizes[axis]; - } - if (elements(out) != count) - abort(); - int64_t width = in.sizes[in.rank - 1]; - if (width > bankBytes / sizeof(float)) { - fputs("RVV norm requires one complete row in a bank\n", stderr); - abort(); - } - alignas(16) float staging[bankBytes / sizeof(float)]; - const uint32_t rounding = roundingMode(); - load(images::norm_no_weight, 0); - for (int64_t begin = 0; begin < count; begin += width) { - int64_t row = begin / width, offset = in.offset; - for (int64_t axis = in.rank - 2; axis >= 0; --axis) { - offset += (row % in.sizes[axis]) * in.strides[axis]; - row /= in.sizes[axis]; - } - send(in.data + offset, inputBank, width, staging); - kernel_launch call{images::norm_no_weight.entry, - images::norm_no_weight.text_bytes, - 0x40002000, - {uint64_t(outputBank) << 32, uint64_t(inputBank) << 32, - uint32_t(width), meanMultiplierBits, epsilonBits, - descriptorAddress + offsetof(kernel_launch, reserved), - rounding}, - 0}; - launch(&call, sizeof(call), 0); - receiveTensor(out.data + out.offset + begin, width, staging); - } - release(0); -} - -static void tensorSoftmax(UnrankedMemRefType *output, - UnrankedMemRefType *input, - const images::KernelImage &image) { - DynamicMemRefType out(*output), in(*input); - int64_t count = elements(in); - if (in.rank < 1 || elements(out) != count) - abort(); - int64_t width = in.sizes[in.rank - 1]; - if (width > bankBytes / sizeof(float)) { - fputs("RVV softmax requires one complete row in a bank\n", stderr); - abort(); - } - int64_t rows = count / width, batch = bankBytes / sizeof(float) / width; - alignas(16) float staging[bankBytes / sizeof(float)]; - const uint32_t rounding = roundingMode(); - load(image, 0); - for (int64_t row = 0; row < rows; row += batch) { - uint32_t grouped = std::min(batch, rows - row), length = grouped * width; - send(in.data + in.offset + row * width, inputBank, length, staging); - kernel_launch call{image.entry, - image.text_bytes, - 0x40002000, - {uint64_t(outputBank) << 32, uint64_t(inputBank) << 32, - grouped, uint32_t(width), - descriptorAddress + offsetof(kernel_launch, reserved), - rounding}, - 0}; - launch(&call, sizeof(call), 0); - receiveTensor(out.data + out.offset + row * width, length, staging); - } - release(0); -} - -extern "C" void _mlir_ciface_rvv_softmax(UnrankedMemRefType *output, - UnrankedMemRefType *input) { - tensorSoftmax(output, input, images::softmax); -} - -extern "C" void -_mlir_ciface_rvv_logsumexp_softmax(UnrankedMemRefType *output, - UnrankedMemRefType *input) { - tensorSoftmax(output, input, images::logsumexp_softmax); -} - -extern "C" void -_mlir_ciface_rvv_attention_softmax(UnrankedMemRefType *output, - UnrankedMemRefType *input, - UnrankedMemRefType *mask, - uint32_t scaleBits, uint32_t maskedBits) { - DynamicMemRefType out(*output), in(*input); - DynamicMemRefType condition(*mask); - int64_t count = elements(in); - if (in.rank != 4 || in.sizes[0] != 1 || elements(out) != count || - condition.rank != 4 || condition.sizes[0] != 1 || - condition.sizes[1] != 1 || - (condition.sizes[2] != 1 && condition.sizes[2] != in.sizes[2]) || - condition.sizes[3] != in.sizes[3] || condition.strides[3] != 1 || - (condition.sizes[2] != 1 && condition.strides[2] != in.sizes[3])) - abort(); - int64_t width = in.sizes[3]; - int64_t maskQueries = condition.sizes[2]; - if (width > bankBytes / sizeof(float) || maskQueries * width > bankBytes) { - fputs("RVV attention softmax requires one complete row in a bank\n", - stderr); - abort(); - } - int64_t rows = count / width, batch = bankBytes / sizeof(float) / width; - alignas(16) float staging[bankBytes / sizeof(float)]; - alignas(16) uint8_t maskStaging[bankBytes]; - const uint32_t rounding = roundingMode(); - load(images::attention_softmax, 0); - uint32_t maskBytes = maskQueries * width; - std::memcpy(maskStaging, condition.data + condition.offset, maskBytes); - std::memset(maskStaging + maskBytes, 0, (16 - maskBytes % 16) % 16); - bb_mvin_group((uintptr_t)maskStaging, readBank, upBank, (maskBytes + 15) / 16, - 1); - for (int64_t row = 0; row < rows;) { - uint32_t grouped = std::min(batch, rows - row); - uint32_t length = grouped * width; - send(in.data + in.offset + row * width, inputBank, length, staging); - alignas(16) - uint8_t descriptor[sizeof(kernel_launch) + 4 * sizeof(uint32_t)]{}; - kernel_launch call{images::attention_softmax.entry, - images::attention_softmax.text_bytes, - 0x40002000, - {uint64_t(outputBank) << 32, uint64_t(inputBank) << 32, - uint64_t(upBank) << 32, grouped, uint32_t(width), - descriptorAddress + sizeof(kernel_launch), - descriptorAddress + offsetof(kernel_launch, reserved), - rounding}, - 0}; - std::memcpy(descriptor, &call, sizeof(call)); - uint32_t parameters[]{scaleBits, maskedBits, uint32_t(maskQueries), - uint32_t(row % maskQueries)}; - std::memcpy(descriptor + sizeof(call), parameters, sizeof(parameters)); - launch(descriptor, sizeof(descriptor), 0); - receiveTensor(out.data + out.offset + row * width, length, staging); - row += grouped; - } - release(0); -} - -extern "C" void _mlir_ciface_rvv_rope(UnrankedMemRefType *output, - UnrankedMemRefType *input, - UnrankedMemRefType *frequencies, - UnrankedMemRefType *positions, - uint32_t scaleBits) { - DynamicMemRefType out(*output), in(*input), freq(*frequencies); - DynamicMemRefType pos(*positions); - int64_t count = elements(in); - if (in.rank != 4 || in.sizes[0] != 1 || elements(out) != count) - abort(); - int64_t heads = in.sizes[1], rows = in.sizes[2], width = in.sizes[3]; - if (width % 2 || elements(freq) != width / 2 || pos.rank != 1 || - pos.sizes[0] != rows || pos.strides[0] != 1 || - width > bankBytes / sizeof(float)) { - fputs( - "RVV RoPE requires complete even-width heads and matching positions\n", - stderr); - abort(); - } - int64_t batch = - std::min(bankBytes / sizeof(float) / width, - (bankBytes - sizeof(kernel_launch)) / sizeof(int32_t)); - alignas(16) float staging[bankBytes / sizeof(float)]; - alignas(16) uint8_t descriptor[bankBytes]; - const uint32_t rounding = roundingMode(); - load(images::rope, 0); - send(freq.data + freq.offset, upBank, width / 2, staging); - const int64_t rowChunk = std::min(batch, rows); - const int64_t headChunk = bankBytes / sizeof(float) / (rowChunk * width); - for (int64_t head = 0; head < heads; head += headChunk) - for (int64_t row = 0; row < rows; row += rowChunk) { - uint32_t groupedRows = std::min(rowChunk, rows - row); - uint32_t groupedHeads = std::min(headChunk, heads - head); - uint32_t length = groupedHeads * groupedRows * width; - int64_t begin = (head * rows + row) * width; - send(in.data + in.offset + begin, inputBank, length, staging); - kernel_launch call{ - images::rope.entry, - images::rope.text_bytes, - 0x40002000, - {uint64_t(outputBank) << 32, uint64_t(inputBank) << 32, - uint64_t(upBank) << 32, descriptorAddress + sizeof(kernel_launch), - groupedRows, - (uint64_t(scaleBits) << 32) | (groupedHeads << 16) | uint32_t(width), - descriptorAddress + offsetof(kernel_launch, reserved), rounding}, - 0}; - uint32_t bytes = - (sizeof(call) + groupedRows * sizeof(int32_t) + 15) & ~15U; - std::memset(descriptor, 0, bytes); - std::memcpy(descriptor, &call, sizeof(call)); - auto packedPositions = - reinterpret_cast(descriptor + sizeof(call)); - for (size_t index = 0; index < groupedRows; ++index) { - const int64_t position = pos.data[pos.offset + row + index]; - if (position < INT32_MIN || position > INT32_MAX) - abort(); - packedPositions[index] = static_cast(position); - } - launch(descriptor, bytes, 0); - receiveTensor(out.data + out.offset + begin, length, staging); - } - release(0); -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/flash_attention.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/flash_attention.cpp deleted file mode 100644 index 55ecfa411..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/flash_attention.cpp +++ /dev/null @@ -1,131 +0,0 @@ -#include "flash_attention.h" -#include "math/math.h" - -static float exponential(float value) { - return __riscv_vfmv_f_s_f32m1_f32( - rvv_exp(__riscv_vfmv_v_f_f32m1(value, 1), 1)); -} - -extern "C" void -rvv_flash_attention_launch(const float *query, const float *panel, - unsigned char *descriptor, float *output, - float *blockOutput, uint64_t dimensions, - uint64_t parameters, uint32_t *flags) { - const unsigned phase = dimensions & 255, rows = (dimensions >> 8) & 255; - const unsigned width = (dimensions >> 16) & 65535; - const unsigned begin = (dimensions >> 32) & 255; - const unsigned count = (dimensions >> 40) & 255; - const unsigned keyCount = (dimensions >> 48) & 255; - const uint32_t rounding = parameters >> 32; - asm volatile("csrw fcsr, %0" ::"r"(rounding) : "memory"); - union { - uint32_t bits; - float value; - } scale{uint32_t(parameters)}; - auto *state = - reinterpret_cast(descriptor + rvv_flash::stateOffset); - auto *mask = reinterpret_cast(descriptor + rvv_flash::maskOffset); - if (phase == 0) { - for (unsigned row = 0; row < rows; ++row) { - state->sum[row] = 0; - state->maximum[row] = -1.0e30f; - for (unsigned d = 0; d < width; d += rvv_flash::lanes) - __riscv_vse32_v_f32m2(output + row * width + d, - __riscv_vfmv_v_f_f32m2(0, rvv_flash::lanes), - rvv_flash::lanes); - } - } else if (phase == 1) { - for (unsigned row = 0; row < rows; ++row) - for (unsigned k = 0; k < count; ++k) { - auto partial = __riscv_vfmv_v_f_f32m2(0, rvv_flash::lanes); - for (unsigned d = 0; d < width; d += rvv_flash::lanes) { - auto q = - __riscv_vle32_v_f32m2(query + row * width + d, rvv_flash::lanes); - auto key = - __riscv_vle32_v_f32m2(panel + k * width + d, rvv_flash::lanes); - partial = __riscv_vfmacc_vv_f32m2(partial, q, key, rvv_flash::lanes); - } - auto sum = __riscv_vfredosum_vs_f32m2_f32m1( - partial, __riscv_vfmv_v_f_f32m1(0, 1), rvv_flash::lanes); - float score = __riscv_vfmv_f_s_f32m1_f32(sum) * scale.value; - state->scores[row * rvv_flash::keys + begin + k] = - score + mask[row * rvv_flash::lanes + k]; - } - } else if (phase == 2) { - for (unsigned row = 0; row < rows; ++row) { - float maximum = -1.0e30f; - for (unsigned k = 0; k < keyCount; ++k) { - float score = state->scores[row * rvv_flash::keys + k]; - if (score > maximum) - maximum = score; - } - float sum = 0; - for (unsigned k = 0; k < keyCount; ++k) { - float p = - exponential(state->scores[row * rvv_flash::keys + k] - maximum); - state->scores[row * rvv_flash::keys + k] = p; - sum = sum + p; - } - state->blockMaximum[row] = maximum; - state->blockSum[row] = sum; - float merged = - maximum > state->maximum[row] ? maximum : state->maximum[row]; - state->mergedMaximum[row] = merged; - state->alpha[row] = exponential(state->maximum[row] - merged); - state->beta[row] = exponential(maximum - merged); - } - } else if (phase == 3) { - for (unsigned row = 0; row < rows; ++row) - for (unsigned d = 0; d < width; d += rvv_flash::lanes) { - auto partial = - begin == 0 ? __riscv_vfmv_v_f_f32m2(0, rvv_flash::lanes) - : __riscv_vle32_v_f32m2(blockOutput + row * width + d, - rvv_flash::lanes); - for (unsigned k = 0; k < count; ++k) { - auto value = - __riscv_vle32_v_f32m2(panel + k * width + d, rvv_flash::lanes); - float p = state->scores[row * rvv_flash::keys + begin + k]; - partial = - __riscv_vfmacc_vf_f32m2(partial, p, value, rvv_flash::lanes); - } - __riscv_vse32_v_f32m2(blockOutput + row * width + d, partial, - rvv_flash::lanes); - } - } else if (phase == 4) { - for (unsigned row = 0; row < rows; ++row) { - for (unsigned d = 0; d < width; d += rvv_flash::lanes) { - auto old = - __riscv_vle32_v_f32m2(output + row * width + d, rvv_flash::lanes); - auto block = __riscv_vle32_v_f32m2(blockOutput + row * width + d, - rvv_flash::lanes); - old = __riscv_vfmul_vf_f32m2(old, state->alpha[row], rvv_flash::lanes); - block = - __riscv_vfmul_vf_f32m2(block, state->beta[row], rvv_flash::lanes); - auto merged = __riscv_vfadd_vv_f32m2(old, block, rvv_flash::lanes); - __riscv_vse32_v_f32m2(output + row * width + d, merged, - rvv_flash::lanes); - } - float old = state->sum[row] * state->alpha[row]; - float block = state->blockSum[row] * state->beta[row]; - state->sum[row] = old + block; - state->maximum[row] = state->mergedMaximum[row]; - } - } else if (phase == 5) { - for (unsigned row = 0; row < rows; ++row) { - for (unsigned d = 0; d < width; d += rvv_flash::lanes) { - auto value = - __riscv_vle32_v_f32m2(output + row * width + d, rvv_flash::lanes); - value = - __riscv_vfdiv_vf_f32m2(value, state->sum[row], rvv_flash::lanes); - __riscv_vse32_v_f32m2(output + row * width + d, value, - rvv_flash::lanes); - } - mask[row] = state->sum[row]; - } - } else { - asm volatile("ebreak" ::: "memory"); - } - uint32_t status; - asm volatile("csrr %0, fflags" : "=r"(status)::"memory"); - *flags = status; -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/flash_attention.h b/bb-tests/workloads/lib/bbsw/kernels/rvv/flash_attention.h deleted file mode 100644 index 9d1a8c7f3..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/flash_attention.h +++ /dev/null @@ -1,15 +0,0 @@ -#pragma once -#include -#include - -namespace rvv_flash { -constexpr unsigned queries = 16, keys = 64, lanes = 16; -constexpr unsigned maskOffset = 128; -constexpr unsigned stateOffset = maskOffset + queries * lanes * sizeof(float); -struct State { - float sum[queries], maximum[queries]; - float scores[queries * keys]; - float blockMaximum[queries], blockSum[queries]; - float alpha[queries], beta[queries], mergedMaximum[queries]; -}; -} // namespace rvv_flash diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/flash_attention_dispatch.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/flash_attention_dispatch.cpp deleted file mode 100644 index 9c13df397..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/flash_attention_dispatch.cpp +++ /dev/null @@ -1,153 +0,0 @@ -#include "flash_attention.h" -#include "images.h" -#include -#include -#include -#include -#include -#include -#include - -namespace { -constexpr unsigned readBank = 5, writeBank = 6; -constexpr unsigned programBank = VIRTUAL_BANK_NUM; -constexpr size_t bankBytes = BANK_LINES * (BANK_WIDTH / 8); -struct alignas(16) Packet { - kernel_launch launch; - uint64_t padding[4]; - float mask[rvv_flash::queries * rvv_flash::lanes]; -}; -static_assert(offsetof(Packet, mask) == rvv_flash::maskOffset); -static_assert(sizeof(Packet) == rvv_flash::stateOffset); -static_assert(rvv_flash::stateOffset + sizeof(rvv_flash::State) <= bankBytes); - -void panel(const DynamicMemRefType &source, int64_t batch, int64_t head, - int64_t begin, unsigned rows, unsigned width, float *staging) { - for (unsigned row = 0; row < rows; ++row) - std::memcpy(staging + row * width, - source.data + source.offset + batch * source.strides[0] + - head * source.strides[1] + - (begin + row) * source.strides[2], - width * sizeof(float)); -} -} // namespace - -extern "C" void _mlir_ciface_rvv_flash_attention( - UnrankedMemRefType *output, UnrankedMemRefType *scores, - UnrankedMemRefType *query, UnrankedMemRefType *key, - UnrankedMemRefType *value, UnrankedMemRefType *mask, - uint32_t scaleBits) { - DynamicMemRefType out(*output), score(*scores), q(*query), k(*key), - v(*value), condition(*mask); - if (q.rank != 4 || k.rank != 4 || v.rank != 4 || out.rank != 4 || - score.rank != 3 || condition.rank != 4) - abort(); - const int64_t batches = q.sizes[0], heads = q.sizes[1], queries = q.sizes[2]; - const int64_t keys = k.sizes[2], width = q.sizes[3]; - if (batches <= 0 || heads <= 0 || queries <= 0 || keys <= 0 || width <= 0 || - width % rvv_flash::lanes || width * sizeof(float) > bankBytes || - k.sizes[0] != batches || v.sizes[0] != batches || k.sizes[1] != heads || - v.sizes[1] != heads || v.sizes[2] != keys || k.sizes[3] != width || - v.sizes[3] != width || condition.sizes[0] != batches || - condition.sizes[1] != 1 || condition.sizes[2] != queries || - condition.sizes[3] != keys || q.strides[3] != 1 || k.strides[3] != 1 || - v.strides[3] != 1 || condition.strides[3] != 1 || out.strides[3] != 1 || - score.strides[2] != 1) - abort(); - for (unsigned axis = 0; axis < 4; ++axis) - if (out.sizes[axis] != q.sizes[axis] || - (axis < 3 && score.sizes[axis] != q.sizes[axis])) - abort(); - const unsigned tileRows = - std::min(rvv_flash::queries, bankBytes / (width * sizeof(float))); - const unsigned tileKeys = - std::min(rvv_flash::lanes, bankBytes / (width * sizeof(float))); - alignas(16) float staging[bankBytes / sizeof(float)]; - Packet packet{}; - packet.launch.entry = images::flash_attention.entry; - packet.launch.end = images::flash_attention.text_bytes; - packet.launch.stack = 0x40002000; - packet.launch.args[0] = 0; - packet.launch.args[1] = uint64_t(1) << 32; - packet.launch.args[2] = uint64_t(2) << 32; - packet.launch.args[3] = uint64_t(3) << 32; - packet.launch.args[4] = uint64_t(4) << 32; - packet.launch.args[7] = - (uint64_t(2) << 32) + offsetof(kernel_launch, reserved); - uint32_t rounding; - asm volatile("csrr %0, frm" : "=r"(rounding)::"memory"); - packet.launch.args[6] = uint64_t(scaleBits) | (uint64_t(rounding << 5) << 32); - for (unsigned bank = 0; bank < 5; ++bank) - bb_mem_alloc(bank, 1, 1); - bb_mem_transfer(0, readBank); - bb_mem_transfer(1, readBank); - for (unsigned bank = 2; bank < 5; ++bank) - bb_mem_transfer(bank, writeBank); - mvin_kernel(images::flash_attention.bytes, images::flash_attention.size, - programBank); - auto launch = [&](unsigned phase, unsigned rows, unsigned keyCount, - unsigned begin = 0, unsigned count = 0) { - packet.launch.args[5] = uint64_t(phase) | (uint64_t(rows) << 8) | - (uint64_t(width) << 16) | (uint64_t(begin) << 32) | - (uint64_t(count) << 40) | - (uint64_t(keyCount) << 48); - packet.launch.reserved = 0; - bb_mvin_group((uintptr_t)&packet, writeBank, 0, sizeof(packet) / 16, 1); - run_kernel(readBank, programBank, writeBank, 0); - bb_mvout_group( - (uintptr_t)&packet, writeBank, 0, - phase == 5 ? sizeof(packet) / 16 : sizeof(kernel_launch) / 16, 1); - asm volatile("csrs fflags, %0" ::"r"(packet.launch.reserved) : "memory"); - }; - for (int64_t batch = 0; batch < batches; ++batch) - for (int64_t head = 0; head < heads; ++head) - for (int64_t queryBegin = 0; queryBegin < queries; - queryBegin += tileRows) { - unsigned rows = std::min(tileRows, queries - queryBegin); - panel(q, batch, head, queryBegin, rows, width, staging); - bb_mvin_group((uintptr_t)staging, readBank, 0, rows * width / 4, 1); - launch(0, rows, 0); - for (int64_t keyBegin = 0; keyBegin < keys; - keyBegin += rvv_flash::keys) { - unsigned keyCount = - std::min(rvv_flash::keys, keys - keyBegin); - for (unsigned begin = 0; begin < keyCount; begin += tileKeys) { - unsigned count = std::min(tileKeys, keyCount - begin); - panel(k, batch, head, keyBegin + begin, count, width, staging); - bb_mvin_group((uintptr_t)staging, readBank, 1, count * width / 4, - 1); - for (unsigned row = 0; row < rows; ++row) - std::memcpy(packet.mask + row * rvv_flash::lanes, - condition.data + condition.offset + - batch * condition.strides[0] + - (queryBegin + row) * condition.strides[2] + - keyBegin + begin, - count * sizeof(float)); - launch(1, rows, keyCount, begin, count); - } - launch(2, rows, keyCount); - for (unsigned begin = 0; begin < keyCount; begin += tileKeys) { - unsigned count = std::min(tileKeys, keyCount - begin); - panel(v, batch, head, keyBegin + begin, count, width, staging); - bb_mvin_group((uintptr_t)staging, readBank, 1, count * width / 4, - 1); - launch(3, rows, keyCount, begin, count); - } - launch(4, rows, keyCount); - } - launch(5, rows, 0); - bb_mvout_group((uintptr_t)staging, writeBank, 1, rows * width / 4, 1); - for (unsigned row = 0; row < rows; ++row) { - std::memcpy(out.data + out.offset + batch * out.strides[0] + - head * out.strides[1] + - (queryBegin + row) * out.strides[2], - staging + row * width, width * sizeof(float)); - std::memcpy(score.data + score.offset + batch * score.strides[0] + - head * score.strides[1] + queryBegin + row, - packet.mask + row, sizeof(float)); - } - } - bb_mem_release(readBank); - bb_mem_release(writeBank); - release_kernel(programBank); -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/flash_attention_mxfp8.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/flash_attention_mxfp8.cpp deleted file mode 100644 index 619a7dd7a..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/flash_attention_mxfp8.cpp +++ /dev/null @@ -1,144 +0,0 @@ -#include "flash_attention.h" -#include "math/math.h" -#include "math/mxfp8.h" - -static float exponential(float value) { - return __riscv_vfmv_f_s_f32m1_f32( - rvv_exp(__riscv_vfmv_v_f_f32m1(value, 1), 1)); -} - -extern "C" void -rvv_flash_attention_mxfp8_launch(const float *query, const uint8_t *panel, - unsigned char *descriptor, float *output, - float *blockOutput, uint64_t dimensions, - uint64_t parameters, uint32_t *flags) { - const unsigned phase = dimensions & 255, rows = (dimensions >> 8) & 255; - const unsigned width = (dimensions >> 16) & 65535; - const unsigned begin = (dimensions >> 32) & 255; - const unsigned count = (dimensions >> 40) & 255; - const unsigned keyCount = (dimensions >> 48) & 255; - const uint32_t rounding = parameters >> 32; - uint32_t decodeStatus = 0; - const auto *scales = panel + count * width; - asm volatile("csrw fcsr, %0" ::"r"(rounding) : "memory"); - union { - uint32_t bits; - float value; - } scale{uint32_t(parameters)}; - auto *state = - reinterpret_cast(descriptor + rvv_flash::stateOffset); - auto *mask = reinterpret_cast(descriptor + rvv_flash::maskOffset); - if (phase == 0) { - for (unsigned row = 0; row < rows; ++row) { - state->sum[row] = 0; - state->maximum[row] = -1.0e30f; - for (unsigned d = 0; d < width; d += rvv_flash::lanes) - __riscv_vse32_v_f32m2(output + row * width + d, - __riscv_vfmv_v_f_f32m2(0, rvv_flash::lanes), - rvv_flash::lanes); - } - } else if (phase == 1) { - for (unsigned row = 0; row < rows; ++row) - for (unsigned k = 0; k < count; ++k) { - auto partial = __riscv_vfmv_v_f_f32m2(0, rvv_flash::lanes); - for (unsigned d = 0; d < width; d += rvv_flash::lanes) { - auto q = - __riscv_vle32_v_f32m2(query + row * width + d, rvv_flash::lanes); - auto key = - rvv_mxfp8_load16(panel + k * width + d, - scales[k * (width / 32) + d / 32], decodeStatus); - if (decodeStatus) { - *flags = decodeStatus; - return; - } - partial = __riscv_vfmacc_vv_f32m2(partial, q, key, rvv_flash::lanes); - } - auto sum = __riscv_vfredosum_vs_f32m2_f32m1( - partial, __riscv_vfmv_v_f_f32m1(0, 1), rvv_flash::lanes); - float score = __riscv_vfmv_f_s_f32m1_f32(sum) * scale.value; - state->scores[row * rvv_flash::keys + begin + k] = - score + mask[row * rvv_flash::lanes + k]; - } - } else if (phase == 2) { - for (unsigned row = 0; row < rows; ++row) { - float maximum = -1.0e30f; - for (unsigned k = 0; k < keyCount; ++k) { - float score = state->scores[row * rvv_flash::keys + k]; - if (score > maximum) - maximum = score; - } - float sum = 0; - for (unsigned k = 0; k < keyCount; ++k) { - float p = - exponential(state->scores[row * rvv_flash::keys + k] - maximum); - state->scores[row * rvv_flash::keys + k] = p; - sum = sum + p; - } - state->blockMaximum[row] = maximum; - state->blockSum[row] = sum; - float merged = - maximum > state->maximum[row] ? maximum : state->maximum[row]; - state->mergedMaximum[row] = merged; - state->alpha[row] = exponential(state->maximum[row] - merged); - state->beta[row] = exponential(maximum - merged); - } - } else if (phase == 3) { - for (unsigned row = 0; row < rows; ++row) - for (unsigned d = 0; d < width; d += rvv_flash::lanes) { - auto partial = - begin == 0 ? __riscv_vfmv_v_f_f32m2(0, rvv_flash::lanes) - : __riscv_vle32_v_f32m2(blockOutput + row * width + d, - rvv_flash::lanes); - for (unsigned k = 0; k < count; ++k) { - auto value = - rvv_mxfp8_load16(panel + k * width + d, - scales[k * (width / 32) + d / 32], decodeStatus); - if (decodeStatus) { - *flags = decodeStatus; - return; - } - float p = state->scores[row * rvv_flash::keys + begin + k]; - partial = - __riscv_vfmacc_vf_f32m2(partial, p, value, rvv_flash::lanes); - } - __riscv_vse32_v_f32m2(blockOutput + row * width + d, partial, - rvv_flash::lanes); - } - } else if (phase == 4) { - for (unsigned row = 0; row < rows; ++row) { - for (unsigned d = 0; d < width; d += rvv_flash::lanes) { - auto old = - __riscv_vle32_v_f32m2(output + row * width + d, rvv_flash::lanes); - auto block = __riscv_vle32_v_f32m2(blockOutput + row * width + d, - rvv_flash::lanes); - old = __riscv_vfmul_vf_f32m2(old, state->alpha[row], rvv_flash::lanes); - block = - __riscv_vfmul_vf_f32m2(block, state->beta[row], rvv_flash::lanes); - auto merged = __riscv_vfadd_vv_f32m2(old, block, rvv_flash::lanes); - __riscv_vse32_v_f32m2(output + row * width + d, merged, - rvv_flash::lanes); - } - float old = state->sum[row] * state->alpha[row]; - float block = state->blockSum[row] * state->beta[row]; - state->sum[row] = old + block; - state->maximum[row] = state->mergedMaximum[row]; - } - } else if (phase == 5) { - for (unsigned row = 0; row < rows; ++row) { - for (unsigned d = 0; d < width; d += rvv_flash::lanes) { - auto value = - __riscv_vle32_v_f32m2(output + row * width + d, rvv_flash::lanes); - value = - __riscv_vfdiv_vf_f32m2(value, state->sum[row], rvv_flash::lanes); - __riscv_vse32_v_f32m2(output + row * width + d, value, - rvv_flash::lanes); - } - mask[row] = state->sum[row]; - } - } else { - asm volatile("ebreak" ::: "memory"); - } - uint32_t status; - asm volatile("csrr %0, fflags" : "=r"(status)::"memory"); - *flags = status; -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/flash_attention_mxfp8_dispatch.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/flash_attention_mxfp8_dispatch.cpp deleted file mode 100644 index 384295e5a..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/flash_attention_mxfp8_dispatch.cpp +++ /dev/null @@ -1,187 +0,0 @@ -#include "flash_attention.h" -#include "images.h" -#include -#include -#include -#include -#include -#include -#include - -namespace { -constexpr unsigned readBank = 5, writeBank = 6; -constexpr unsigned programBank = VIRTUAL_BANK_NUM; -constexpr size_t bankBytes = BANK_LINES * (BANK_WIDTH / 8); -struct alignas(16) Packet { - kernel_launch launch; - uint64_t padding[4]; - float mask[rvv_flash::queries * rvv_flash::lanes]; -}; -static_assert(offsetof(Packet, mask) == rvv_flash::maskOffset); -static_assert(sizeof(Packet) == rvv_flash::stateOffset); -static_assert(rvv_flash::stateOffset + sizeof(rvv_flash::State) <= bankBytes); - -void panel(const DynamicMemRefType &source, int64_t batch, int64_t head, - int64_t begin, unsigned rows, unsigned width, float *staging) { - for (unsigned row = 0; row < rows; ++row) - std::memcpy(staging + row * width, - source.data + source.offset + batch * source.strides[0] + - head * source.strides[1] + - (begin + row) * source.strides[2], - width * sizeof(float)); -} - -void packedPanel(const DynamicMemRefType &codes, - const DynamicMemRefType &scales, int64_t batch, - int64_t head, int64_t begin, unsigned rows, unsigned width, - int8_t *staging) { - for (unsigned row = 0; row < rows; ++row) { - std::memcpy(staging + row * width, - codes.data + codes.offset + batch * codes.strides[0] + - head * codes.strides[1] + (begin + row) * codes.strides[2], - width); - std::memcpy(staging + rows * width + row * (width / 32), - scales.data + scales.offset + batch * scales.strides[0] + - head * scales.strides[1] + - (begin + row) * scales.strides[2], - width / 32); - } -} -} // namespace - -extern "C" void _mlir_ciface_rvv_flash_attention_mxfp8( - UnrankedMemRefType *output, UnrankedMemRefType *scores, - UnrankedMemRefType *query, UnrankedMemRefType *keyCodes, - UnrankedMemRefType *keyScales, - UnrankedMemRefType *valueCodes, - UnrankedMemRefType *valueScales, UnrankedMemRefType *mask, - uint32_t scaleBits) { - DynamicMemRefType out(*output), score(*scores), q(*query), - condition(*mask); - DynamicMemRefType k(*keyCodes), ks(*keyScales), v(*valueCodes), - vs(*valueScales); - if (q.rank != 4 || k.rank != 4 || v.rank != 4 || out.rank != 4 || - score.rank != 3 || condition.rank != 4 || ks.rank != 4 || vs.rank != 4) - abort(); - const int64_t batches = q.sizes[0], heads = q.sizes[1], queries = q.sizes[2]; - const int64_t keys = k.sizes[2], width = q.sizes[3]; - if (batches <= 0 || heads <= 0 || queries <= 0 || keys <= 0 || width <= 0 || - width % 32 || width * sizeof(float) > bankBytes || - k.sizes[0] != batches || v.sizes[0] != batches || k.sizes[1] <= 0 || - heads % k.sizes[1] || v.sizes[1] != k.sizes[1] || v.sizes[2] != keys || - k.sizes[3] != width || v.sizes[3] != width || - condition.sizes[0] != batches || condition.sizes[1] != 1 || - condition.sizes[2] != queries || condition.sizes[3] != keys || - q.strides[3] != 1 || k.strides[3] != 1 || v.strides[3] != 1 || - condition.strides[3] != 1 || out.strides[3] != 1 || score.strides[2] != 1) - abort(); - for (unsigned axis = 0; axis < 4; ++axis) - if (ks.sizes[axis] != (axis == 3 ? width / 32 : k.sizes[axis]) || - vs.sizes[axis] != ks.sizes[axis]) - abort(); - if (ks.strides[3] != 1 || vs.strides[3] != 1) - abort(); - for (unsigned axis = 0; axis < 4; ++axis) - if (out.sizes[axis] != q.sizes[axis] || - (axis < 3 && score.sizes[axis] != q.sizes[axis])) - abort(); - const unsigned tileRows = - std::min(rvv_flash::queries, bankBytes / (width * sizeof(float))); - const unsigned tileKeys = - std::min(rvv_flash::lanes, bankBytes / (width * sizeof(float))); - alignas(16) float staging[bankBytes / sizeof(float)]; - alignas(16) int8_t packed[bankBytes]; - Packet packet{}; - packet.launch.entry = images::flash_attention_mxfp8.entry; - packet.launch.end = images::flash_attention_mxfp8.text_bytes; - packet.launch.stack = 0x40002000; - packet.launch.args[0] = 0; - packet.launch.args[1] = uint64_t(1) << 32; - packet.launch.args[2] = uint64_t(2) << 32; - packet.launch.args[3] = uint64_t(3) << 32; - packet.launch.args[4] = uint64_t(4) << 32; - packet.launch.args[7] = - (uint64_t(2) << 32) + offsetof(kernel_launch, reserved); - uint32_t rounding; - asm volatile("csrr %0, frm" : "=r"(rounding)::"memory"); - packet.launch.args[6] = uint64_t(scaleBits) | (uint64_t(rounding << 5) << 32); - for (unsigned bank = 0; bank < 5; ++bank) - bb_mem_alloc(bank, 1, 1); - bb_mem_transfer(0, readBank); - bb_mem_transfer(1, readBank); - for (unsigned bank = 2; bank < 5; ++bank) - bb_mem_transfer(bank, writeBank); - mvin_kernel(images::flash_attention_mxfp8.bytes, - images::flash_attention_mxfp8.size, programBank); - auto launch = [&](unsigned phase, unsigned rows, unsigned keyCount, - unsigned begin = 0, unsigned count = 0) { - packet.launch.args[5] = uint64_t(phase) | (uint64_t(rows) << 8) | - (uint64_t(width) << 16) | (uint64_t(begin) << 32) | - (uint64_t(count) << 40) | - (uint64_t(keyCount) << 48); - packet.launch.reserved = 0; - bb_mvin_group((uintptr_t)&packet, writeBank, 0, sizeof(packet) / 16, 1); - run_kernel(readBank, programBank, writeBank, 0); - bb_mvout_group( - (uintptr_t)&packet, writeBank, 0, - phase == 5 ? sizeof(packet) / 16 : sizeof(kernel_launch) / 16, 1); - if (packet.launch.reserved & ~uint64_t(31)) { - fputs("flash_attention_mxfp8: invalid packed KV data\n", stderr); - abort(); - } - asm volatile("csrs fflags, %0" ::"r"(packet.launch.reserved) : "memory"); - }; - for (int64_t batch = 0; batch < batches; ++batch) - for (int64_t head = 0; head < heads; ++head) - for (int64_t queryBegin = 0; queryBegin < queries; - queryBegin += tileRows) { - unsigned rows = std::min(tileRows, queries - queryBegin); - panel(q, batch, head, queryBegin, rows, width, staging); - bb_mvin_group((uintptr_t)staging, readBank, 0, rows * width / 4, 1); - launch(0, rows, 0); - for (int64_t keyBegin = 0; keyBegin < keys; - keyBegin += rvv_flash::keys) { - unsigned keyCount = - std::min(rvv_flash::keys, keys - keyBegin); - for (unsigned begin = 0; begin < keyCount; begin += tileKeys) { - unsigned count = std::min(tileKeys, keyCount - begin); - packedPanel(k, ks, batch, head / (heads / k.sizes[1]), - keyBegin + begin, count, width, packed); - bb_mvin_group((uintptr_t)packed, readBank, 1, - (count * width + count * (width / 32) + 15) / 16, 1); - for (unsigned row = 0; row < rows; ++row) - std::memcpy(packet.mask + row * rvv_flash::lanes, - condition.data + condition.offset + - batch * condition.strides[0] + - (queryBegin + row) * condition.strides[2] + - keyBegin + begin, - count * sizeof(float)); - launch(1, rows, keyCount, begin, count); - } - launch(2, rows, keyCount); - for (unsigned begin = 0; begin < keyCount; begin += tileKeys) { - unsigned count = std::min(tileKeys, keyCount - begin); - packedPanel(v, vs, batch, head / (heads / k.sizes[1]), - keyBegin + begin, count, width, packed); - bb_mvin_group((uintptr_t)packed, readBank, 1, - (count * width + count * (width / 32) + 15) / 16, 1); - launch(3, rows, keyCount, begin, count); - } - launch(4, rows, keyCount); - } - launch(5, rows, 0); - bb_mvout_group((uintptr_t)staging, writeBank, 1, rows * width / 4, 1); - for (unsigned row = 0; row < rows; ++row) { - std::memcpy(out.data + out.offset + batch * out.strides[0] + - head * out.strides[1] + - (queryBegin + row) * out.strides[2], - staging + row * width, width * sizeof(float)); - std::memcpy(score.data + score.offset + batch * score.strides[0] + - head * score.strides[1] + queryBegin + row, - packet.mask + row, sizeof(float)); - } - } - bb_mem_release(readBank); - bb_mem_release(writeBank); - release_kernel(programBank); -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/gelu.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/gelu.cpp deleted file mode 100644 index cc54994e9..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/gelu.cpp +++ /dev/null @@ -1,19 +0,0 @@ -#include "kernels.h" - -extern "C" float rvv_erff(float value); - -extern "C" void rvv_gelu(float *output, const float *input, size_t count, - uint32_t *flags, uint32_t rounding) { - asm volatile("csrw fcsr, %0" ::"r"(rounding) : "memory"); - for (size_t index = 0; index < count; ++index) { - const float value = input[index]; - const float scaled = value * 0x1.6a09e6p-1f; - const float error = rvv_erff(scaled); - const float shifted = error + 1.0f; - const float halved = value * 0.5f; - output[index] = halved * shifted; - } - uint32_t result; - asm volatile("csrr %0, fflags" : "=r"(result)::"memory"); - *flags = result; -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/kernel.ld b/bb-tests/workloads/lib/bbsw/kernels/rvv/kernel.ld deleted file mode 100644 index b43527119..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/kernel.ld +++ /dev/null @@ -1,13 +0,0 @@ -SECTIONS { - . = 0; - .text : { *(.text*) } - ASSERT(SIZEOF(.text) <= 4096, "RVV instruction memory overflow") - . = 0x40000000; - .rodata : { *(.rodata*) *(.srodata*) } - ASSERT(SIZEOF(.rodata) <= 4096, "RVV constant memory overflow") - .data : { *(.data*) *(.sdata*) *(.tdata*) } - .bss (NOLOAD) : { *(.bss*) *(.sbss*) *(.tbss*) *(COMMON) } - ASSERT(SIZEOF(.data) == 0, "RVV kernels cannot define writable global data") - ASSERT(SIZEOF(.bss) == 0, "RVV kernels cannot define writable global data") - /DISCARD/ : { *(.eh_frame*) *(.comment*) *(.riscv.attributes*) } -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/kernels.h b/bb-tests/workloads/lib/bbsw/kernels/rvv/kernels.h deleted file mode 100644 index 93a1134f4..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/kernels.h +++ /dev/null @@ -1,40 +0,0 @@ -#pragma once -#include -#include - -extern "C" { -void rvv_flash_attention_mxfp8_launch(const float *query, const uint8_t *panel, - unsigned char *descriptor, float *output, - float *blockOutput, uint64_t dimensions, - uint64_t parameters, uint32_t *flags); -void rvv_layernorm(float *output, const float *input, const float *weight, - const float *bias, uint64_t widthAndBias, - uint64_t parameters, uint32_t *flags, uint32_t rounding); -void rvv_gelu(float *output, const float *input, size_t count, uint32_t *flags, - uint32_t rounding); -void rvv_pointwise(float *output, const float *lhs, const float *rhs, - size_t count, uint32_t opcode, uint32_t *flags, - uint32_t rounding); -void rvv_matmul(float *output, const float *lhs, const float *rhs, size_t rows, - size_t cols, size_t inner); -void rvv_quant(uint8_t *output, uint8_t *scales, const uint32_t *input, - size_t count, uint32_t *status); -void rvv_cache_quant(uint8_t *output, uint8_t *scales, const uint32_t *input, - size_t count, uint32_t *status); -void rvv_dequant(uint32_t *output, const uint8_t *input, const uint8_t *scales, - size_t count, uint32_t *status); -void rvv_silu(float *output, const float *input, size_t count); -void rvv_swiglu(float *output, const float *gate, const float *up, - size_t count); -void rvv_snake(float *output, const float *input, size_t channels, - size_t length, const float *log_alpha, const float *log_beta); -void rvv_norm(float *output, const float *input, const float *weight, - size_t width, uint32_t meanMultiplierBits, uint32_t epsilonBits); -void rvv_norm_no_weight(float *output, const float *input, size_t width, - uint32_t meanMultiplierBits, uint32_t epsilonBits); -void rvv_softmax(float *output, const float *input, size_t rows, size_t width); -void rvv_logsumexp_softmax(float *output, const float *input, size_t rows, - size_t width); -void rvv_rope(float *output, const float *input, const float *frequencies, - const int32_t *positions, size_t rows, size_t headDimAndHeads); -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/layernorm.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/layernorm.cpp deleted file mode 100644 index 15f3e00fc..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/layernorm.cpp +++ /dev/null @@ -1,52 +0,0 @@ -#include "kernels.h" -#include - -extern "C" void rvv_layernorm(float *output, const float *input, - const float *weight, const float *bias, - uint64_t widthAndBias, uint64_t parameters, - uint32_t *flags, uint32_t rounding) { - asm volatile("csrw fcsr, %0" ::"r"(rounding) : "memory"); - const bool hasBias = widthAndBias >> 32; - const size_t width = uint32_t(widthAndBias); - const float multiplier = __builtin_bit_cast(float, uint32_t(parameters)); - const float epsilon = __builtin_bit_cast(float, uint32_t(parameters >> 32)); - auto sum = __riscv_vfmv_v_f_f32m1(0, 1); - for (size_t offset = 0; offset < width;) { - const size_t vl = __riscv_vsetvl_e32m1(width - offset); - auto values = __riscv_vle32_v_f32m1(input + offset, vl); - sum = __riscv_vfredosum_vs_f32m1_f32m1(values, sum, vl); - offset += vl; - } - const float mean = __riscv_vfmv_f_s_f32m1_f32(sum) * multiplier; - sum = __riscv_vfmv_v_f_f32m1(0, 1); - for (size_t offset = 0; offset < width;) { - const size_t vl = __riscv_vsetvl_e32m1(width - offset); - auto values = __riscv_vle32_v_f32m1(input + offset, vl); - auto centered = __riscv_vfsub_vf_f32m1(values, mean, vl); - auto squares = __riscv_vfmul_vv_f32m1(centered, centered, vl); - sum = __riscv_vfredosum_vs_f32m1_f32m1(squares, sum, vl); - offset += vl; - } - const float variance = __riscv_vfmv_f_s_f32m1_f32(sum) * multiplier; - const float biased = variance + epsilon; - float root; - asm volatile("fsqrt.s %0, %1" : "=f"(root) : "f"(biased)); - const float scale = 1.0f / root; - for (size_t offset = 0; offset < width;) { - const size_t vl = __riscv_vsetvl_e32m1(width - offset); - auto values = __riscv_vle32_v_f32m1(input + offset, vl); - auto centered = __riscv_vfsub_vf_f32m1(values, mean, vl); - auto normalized = __riscv_vfmul_vf_f32m1(centered, scale, vl); - auto weights = __riscv_vle32_v_f32m1(weight + offset, vl); - auto weighted = __riscv_vfmul_vv_f32m1(normalized, weights, vl); - if (hasBias) { - auto biases = __riscv_vle32_v_f32m1(bias + offset, vl); - weighted = __riscv_vfadd_vv_f32m1(weighted, biases, vl); - } - __riscv_vse32_v_f32m1(output + offset, weighted, vl); - offset += vl; - } - uint32_t result; - asm volatile("csrr %0, fflags" : "=r"(result)::"memory"); - *flags = result; -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/math/constants.h b/bb-tests/workloads/lib/bbsw/kernels/rvv/math/constants.h deleted file mode 100644 index b82cc5c26..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/math/constants.h +++ /dev/null @@ -1,39 +0,0 @@ -/* Single-precision e^x function. - Copyright (C) 2017-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ - -#pragma once -#include -static const float exp_table[32] = { - 0x1.0000000000000p+0f, 0x1.059b0e0000000p+0f, 0x1.0b55860000000p+0f, - 0x1.11301e0000000p+0f, 0x1.172b840000000p+0f, 0x1.1d48740000000p+0f, - 0x1.2387a60000000p+0f, 0x1.29e9e00000000p+0f, 0x1.306fe00000000p+0f, - 0x1.371a740000000p+0f, 0x1.3dea640000000p+0f, 0x1.44e0860000000p+0f, - 0x1.4bfdae0000000p+0f, 0x1.5342b60000000p+0f, 0x1.5ab07e0000000p+0f, - 0x1.6247ec0000000p+0f, 0x1.6a09e60000000p+0f, 0x1.71f75e0000000p+0f, - 0x1.7a11480000000p+0f, 0x1.82589a0000000p+0f, 0x1.8ace540000000p+0f, - 0x1.93737c0000000p+0f, 0x1.9c49180000000p+0f, 0x1.a5503c0000000p+0f, - 0x1.ae89fa0000000p+0f, 0x1.b7f7700000000p+0f, 0x1.c199be0000000p+0f, - 0x1.cb720e0000000p+0f, 0x1.d5818e0000000p+0f, 0x1.dfc9740000000p+0f, - 0x1.ea4afa0000000p+0f, 0x1.f507660000000p+0f, -}; -static const uint32_t inv_pio4[24] = { - - 0xa2, 0xa2f9, 0xa2f983, 0xa2f9836e, 0xf9836e4e, 0x836e4e44, - 0x6e4e4415, 0x4e441529, 0x441529fc, 0x1529fc27, 0x29fc2757, 0xfc2757d1, - 0x2757d1f5, 0x57d1f534, 0xd1f534dd, 0xf534ddc0, 0x34ddc0db, 0xddc0db62, - 0xc0db6295, 0xdb629599, 0x6295993c, 0x95993c43, 0x993c4390, 0x3c439041}; diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/math/cos.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/math/cos.cpp deleted file mode 100644 index 6b9785b66..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/math/cos.cpp +++ /dev/null @@ -1,99 +0,0 @@ -// Vector adaptation of glibc 2.42 s_cosf.c and s_sincosf.h/sincosf_poly.h. -// Copyright (C) 2018-2025 Free Software Foundation, Inc. -// LGPL-2.1-or-later; see ../../../thirdparty/glibc/COPYING.LIB. -#include "constants.h" -#include "math.h" - -extern "C" vfloat32m1_t rvv_cos(vfloat32m1_t value, size_t vl) { - auto bits = __riscv_vreinterpret_u32m1(value); - auto top = - __riscv_vand_vx_u32m1(__riscv_vsrl_vx_u32m1(bits, 20, vl), 0x7ff, vl); - auto tiny = __riscv_vmsltu_vx_u32m1_b32(top, 0x398, vl); - auto fast = __riscv_vmsltu_vx_u32m1_b32(top, 0x42f, vl); - fast = __riscv_vmandn_mm_b32(fast, tiny, vl); - auto finite = __riscv_vmsltu_vx_u32m1_b32(top, 0x7f8, vl); - auto input = - __riscv_vmerge_vvm_f32m1(__riscv_vfmv_v_f_f32m1(0, vl), value, fast, vl); - auto x = input; - auto r = __riscv_vfmul_vf_f32m1(x, 0x1.45f306dc9c883p23, vl); - auto n = __riscv_vfcvt_rtz_x_f_v_i32m1(r, vl); - n = __riscv_vsra_vx_i32m1(__riscv_vadd_vx_i32m1(n, 0x800000, vl), 24, vl); - auto small = __riscv_vmsltu_vx_u32m1_b32(top, 0x3f4, vl); - n = __riscv_vmerge_vxm_i32m1(n, 0, small, vl); - auto nd = __riscv_vfcvt_f_x_v_f32m1(n, vl); - x = __riscv_vfnmsac_vf_f32m1(x, 0x1.921fb4p0f, nd, vl); - x = __riscv_vfnmsac_vf_f32m1(x, 0x1.4442d2p-24f, nd, vl); - auto quadrant = n; - auto large = __riscv_vmandn_mm_b32(finite, fast, vl); - large = __riscv_vmandn_mm_b32(large, tiny, vl); - if (__riscv_vcpop_m_b32(large, vl)) { - auto index = - __riscv_vand_vx_u32m1(__riscv_vsrl_vx_u32m1(bits, 26, vl), 15, vl); - index = __riscv_vsll_vx_u32m1(index, 2, vl); - auto a0 = __riscv_vluxei32_v_u32m1(inv_pio4, index, vl); - auto a1 = __riscv_vluxei32_v_u32m1(inv_pio4 + 4, index, vl); - auto a2 = __riscv_vluxei32_v_u32m1(inv_pio4 + 8, index, vl); - auto shift = - __riscv_vand_vx_u32m1(__riscv_vsrl_vx_u32m1(bits, 23, vl), 7, vl); - auto xi = __riscv_vor_vx_u32m1(__riscv_vand_vx_u32m1(bits, 0xffffff, vl), - 0x800000, vl); - xi = __riscv_vsll_vv_u32m1(xi, shift, vl); - auto low0 = __riscv_vmulhu_vv_u32m1(xi, a2, vl); - auto low1 = __riscv_vmul_vv_u32m1(xi, a1, vl); - auto low = __riscv_vadd_vv_u32m1(low0, low1, vl); - auto carry_mask = __riscv_vmsltu_vv_u32m1_b32(low, low1, vl); - auto high = __riscv_vadd_vv_u32m1(__riscv_vmul_vv_u32m1(xi, a0, vl), - __riscv_vmulhu_vv_u32m1(xi, a1, vl), vl); - auto carry = __riscv_vmerge_vxm_u32m1(__riscv_vmv_v_x_u32m1(0, vl), 1, - carry_mask, vl); - high = __riscv_vadd_vv_u32m1(high, carry, vl); - auto q = __riscv_vsrl_vx_u32m1(__riscv_vadd_vx_u32m1(high, 0x20000000, vl), - 30, vl); - high = __riscv_vsub_vv_u32m1(high, __riscv_vsll_vx_u32m1(q, 30, vl), vl); - auto reduced = - __riscv_vfcvt_f_x_v_f32m1(__riscv_vreinterpret_i32m1(high), vl); - reduced = __riscv_vfmul_vf_f32m1(reduced, 0x1.921fb4p-30f, vl); - auto fraction = __riscv_vfcvt_f_xu_v_f32m1(low, vl); - reduced = __riscv_vfmacc_vf_f32m1(reduced, 0x1.921fb4p-62f, fraction, vl); - auto q32 = __riscv_vreinterpret_i32m1(q); - n = __riscv_vmerge_vvm_i32m1(n, q32, large, vl); - auto sign = __riscv_vreinterpret_i32m1(__riscv_vsrl_vx_u32m1(bits, 31, vl)); - quadrant = __riscv_vmerge_vvm_i32m1( - quadrant, __riscv_vadd_vv_i32m1(q32, sign, vl), large, vl); - x = __riscv_vmerge_vvm_f32m1(x, reduced, large, vl); - } - auto x2 = __riscv_vfmul_vv_f32m1(x, x, vl); - auto negative = - __riscv_vmsne_vx_i32m1_b32(__riscv_vand_vx_i32m1(quadrant, 2, vl), 0, vl); - // sign[] is {1,-1,-1,1}; cosine polynomial sign depends on bit 1. - auto signbit = __riscv_vxor_vv_i32m1( - quadrant, __riscv_vsra_vx_i32m1(quadrant, 1, vl), vl); - auto sine_negative = - __riscv_vmsne_vx_i32m1_b32(__riscv_vand_vx_i32m1(signbit, 1, vl), 0, vl); - auto sx = __riscv_vmerge_vvm_f32m1(x, __riscv_vfneg_v_f32m1(x, vl), - sine_negative, vl); - auto x3 = __riscv_vfmul_vv_f32m1(sx, x2, vl); - auto s1 = - __riscv_vfmacc_vf_f32m1(__riscv_vfmv_v_f_f32m1(0x1.1107605230bc4p-7, vl), - -0x1.994eb3774cf24p-13, x2, vl); - auto s = __riscv_vfmacc_vf_f32m1(sx, -0x1.555545995a603p-3, x3, vl); - auto x7 = __riscv_vfmul_vv_f32m1(x3, x2, vl); - s = __riscv_vfmacc_vv_f32m1(s, s1, x7, vl); - auto x4 = __riscv_vfmul_vv_f32m1(x2, x2, vl); - auto c2 = __riscv_vfmacc_vf_f32m1( - __riscv_vfmv_v_f_f32m1(-0x1.6c087e89a359dp-10, vl), 0x1.99343027bf8c3p-16, - x2, vl); - auto c1 = __riscv_vfmacc_vf_f32m1(__riscv_vfmv_v_f_f32m1(1, vl), - -0x1.ffffffd0c621cp-2, x2, vl); - auto c = __riscv_vfmacc_vf_f32m1(c1, 0x1.55553e1068f19p-5, x4, vl); - auto x6 = __riscv_vfmul_vv_f32m1(x4, x2, vl); - c = __riscv_vfmacc_vv_f32m1(c, c2, x6, vl); - c = __riscv_vmerge_vvm_f32m1(c, __riscv_vfneg_v_f32m1(c, vl), negative, vl); - auto odd = __riscv_vmsne_vx_i32m1_b32(__riscv_vand_vx_i32m1(n, 1, vl), 0, vl); - auto result = __riscv_vmerge_vvm_f32m1(c, s, odd, vl); - result = __riscv_vfmerge_vfm_f32m1(result, 1, tiny, vl); - auto invalid = __riscv_vmnot_m_b32(finite, vl); - auto difference = __riscv_vfsub_vv_f32m1_m(invalid, value, value, vl); - auto nan = __riscv_vfdiv_vv_f32m1_m(invalid, difference, difference, vl); - return __riscv_vmerge_vvm_f32m1(result, nan, invalid, vl); -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/math/erf.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/math/erf.cpp deleted file mode 100644 index 1ae8a2201..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/math/erf.cpp +++ /dev/null @@ -1,248 +0,0 @@ -/* Error function evaluated in binary32. - -Copyright (c) 2022-2024 Alexei Sibidanov. - -This file is part of the CORE-MATH project -project (file src/binary32/erf/erff.c revision bc385c2). - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - -#include -#include -// GNU glibc 2.42 sysdeps/ieee754/flt-32/s_erff.c. -static uint32_t asuint(float x) { return __builtin_bit_cast(uint32_t, x); } - -extern "C" float rvv_erff(float x) { - /* for 7 <= i < 63, C[i-7] is a degree-7 polynomial approximation of - erf(i/16+1/32+x) for -1/32 <= x <= 1/32 */ - static const float C[56][8] = { - {0x1.f86faa9428f9cp-2f, 0x1.cfc41e36c7dfap-1f, -0x1.b2c7dc53508b9p-2f, - -0x1.5a9de93fa556ep-3f, 0x1.731793dbb01b5p-3f, 0x1.133e06426cf18p-6f, - -0x1.a12a6289cafd8p-5f, 0x1.717d6f1d6f557p-9f}, - {0x1.1855a5fd3dd50p-1f, 0x1.b3aafcc27502fp-1f, -0x1.cee5ac8e92bb2p-2f, - -0x1.fa02983ca2d79p-4f, 0x1.77cd746cb1922p-3f, -0x1.fa6f277886487p-10f, - -0x1.8de75458db416p-5f, 0x1.00899c98551c9p-7f}, - {0x1.32a54cb8db67ap-1f, 0x1.96164fafd8de5p-1f, -0x1.e23a7ea0c9ad3p-2f, - -0x1.3f5ee15671cf4p-4f, 0x1.70e468a3d72d9p-3f, -0x1.3da68037cfc99p-6f, - -0x1.69ed9ba1f9839p-5f, 0x1.8cab9244a4ff4p-7f}, - {0x1.4b13713ad3513p-1f, 0x1.7791b886e7405p-1f, -0x1.ecef423109bf5p-2f, - -0x1.15c3c5cec6847p-5f, 0x1.5f688fc931ba6p-3f, -0x1.1da63ed190037p-5f, - -0x1.38427ca63cca4p-5f, 0x1.fa00e52525e17p-7f}, - {0x1.61955607dd15dp-1f, 0x1.58a445da7c74ep-1f, -0x1.ef6c246a0f66cp-2f, - 0x1.e83e0d9d61330p-8f, 0x1.44cc65535bc9fp-3f, -0x1.87d3c4860435dp-5f, - -0x1.f90b10501169bp-6f, 0x1.22295856d427ap-6f}, - {0x1.762870f720c6fp-1f, 0x1.39ccc1b136d5cp-1f, -0x1.ea4feea4e4744p-2f, - 0x1.715e5952ebfbap-5f, 0x1.22cdbd83c75c4p-3f, -0x1.da50aa1d925b6p-5f, - -0x1.754dc0a29b4ddp-6f, 0x1.350b6bef9392cp-6f}, - {0x1.88d1cd474a2e0p-1f, 0x1.1b7e98fe26219p-1f, -0x1.de65a22ce1419p-2f, - 0x1.40686a3f16400p-4f, 0x1.f6b0cbb216b2bp-4f, -0x1.09c7c903edd57p-4f, - -0x1.da7529fde641p-7f, 0x1.362a7a0588eabp-6f}, - {0x1.999d4192a5717p-1f, 0x1.fc3ee5d1524b3p-2f, -0x1.cc990045b55c8p-2f, - 0x1.b37338e68b37dp-4f, 0x1.a0d120c872ea7p-4f, -0x1.19bb2b07ecff6p-4f, - -0x1.a110f5f593aafp-8f, 0x1.272c15a57720ep-6f}, - {0x1.a89c850b7d54dp-1f, 0x1.c40b0729ed54ap-2f, -0x1.b5eaaef0a2346p-2f, - 0x1.0847c7dacbae1p-3f, 0x1.47de0ba6d18fbp-4f, -0x1.1d9de77a4b648p-4f, - 0x1.30ffbe56f0726p-10f, 0x1.0a9cb99feea01p-6f}, - {0x1.b5e62fce16096p-1f, 0x1.8eed36b886d95p-2f, -0x1.9b64a06e50705p-2f, - 0x1.2bb6e2c744df5p-3f, 0x1.dee3261ca61bcp-5f, -0x1.16996004f7da5p-4f, - 0x1.fdff37bae983ep-8f, 0x1.c750083e65f9ap-7f}, - {0x1.c194b1d49a184p-1f, 0x1.5d4fd33729015p-2f, -0x1.7e0f4f045addbp-2f, - 0x1.444bc66c31a1bp-3f, 0x1.356dbf16ec8f1p-5f, -0x1.0643de0906cd8p-4f, - 0x1.b281af7bd3a2cp-7f, 0x1.6b97eaa2c6abdp-7f}, - {0x1.cbc54b476248ep-1f, 0x1.2f7cc3fe6f423p-2f, -0x1.5ee8429e36de8p-2f, - 0x1.52a8395f96177p-3f, 0x1.313761ba257dcp-6f, -0x1.dcf844d5fed8fp-5f, - 0x1.1e1420f475fa9p-6f, 0x1.091c7dc1e18b2p-7f}, - {0x1.d4970f9ce00d9p-1f, 0x1.059f59af7a905p-2f, -0x1.3eda354de36c3p-2f, - 0x1.57b85ad439779p-3f, 0x1.8e913b9778136p-10f, -0x1.a2893bd3435f4p-5f, - 0x1.4d3a90e37164ap-6f, 0x1.4ce7f6e19a902p-8f}, - {0x1.dc29fb60715b0p-1f, 0x1.bf8e1b1ca2277p-3f, -0x1.1eb7095e5d6d2p-2f, - 0x1.549ea6f7a64f4p-3f, -0x1.b10f12f3877a3p-7f, -0x1.61420c8f7156ap-5f, - 0x1.674f1f92a8812p-6f, 0x1.25543ffd74d52p-9f}, - {0x1.e29e22a89d767p-1f, 0x1.7bd5c7df3fe99p-3f, -0x1.fe674494077bfp-3f, - 0x1.4a9feacf86578p-3f, -0x1.a008269076644p-6f, -0x1.1cf0e8fb4f1cbp-5f, - 0x1.6e0d2ef105fb3p-6f, -0x1.367205fbd7876p-12f}, - {0x1.e812fc64db36ap-1f, 0x1.3fda6bc016991p-3f, -0x1.c1cb278627920p-3f, - 0x1.3b10512314f1ep-3f, -0x1.1e6457bb1b9a9p-5f, -0x1.b1f6474e2388cp-6f, - 0x1.640a5345f7ec7p-6f, -0x1.3dae5a997fdbp-9f}, - {0x1.eca6ccd709544p-1f, 0x1.0b3f52ce8c380p-3f, -0x1.8885019f63c6dp-3f, - 0x1.274275fc91a05p-3f, -0x1.57f73699a8372p-5f, -0x1.3076a305fc7cep-6f, - 0x1.4c6ae04843a41p-6f, -0x1.0be5fcf5ecc91p-8f}, - {0x1.f0762fde45ee7p-1f, 0x1.bb1c972f23e4ap-4f, -0x1.5341e3c01b58dp-3f, - 0x1.107929f6f0b60p-3f, -0x1.7e1b34f976c02p-5f, -0x1.73b62589c234ap-7f, - 0x1.2a97ee1876486p-6f, -0x1.595f40a3150fep-8f}, - {0x1.f39bc242e43e6p-1f, 0x1.6c7e64e7281c5p-4f, -0x1.2274b86835fd3p-3f, - 0x1.efb890e5c770dp-4f, -0x1.92c7db16847e0p-5f, -0x1.45477db5e2dd4p-8f, - 0x1.01fc6165fc866p-6f, -0x1.8845509030c2cp-8f}, - {0x1.f62fe80272419p-1f, 0x1.297db960e4f5dp-4f, -0x1.ecb83b087c04fp-4f, - 0x1.bce18363ca3d1p-4f, -0x1.985aaf776482cp-5f, 0x1.cd953efdae886p-12f, - 0x1.ab9a0b89b54ffp-7f, -0x1.9b5e576ccc31cp-8f}, - {0x1.f848acb544e95p-1f, 0x1.e1d4cf1e24501p-5f, -0x1.9e12e1fde5552p-4f, - 0x1.8a27806df3d1bp-4f, -0x1.91674e5eb3319p-5f, 0x1.3bc75595b2db8p-8f, - 0x1.51bc537ac61afp-7f, -0x1.96b23b19ea04dp-8f}, - {0x1.f9f9ba8d3c733p-1f, 0x1.83298d7172108p-5f, -0x1.58d101f905a75p-4f, - 0x1.58f1456f8639bp-4f, -0x1.808d1850b8231p-5f, 0x1.0c1bd99c348a7p-7f, - 0x1.f61e9d7bc48cap-8f, -0x1.7f07c13441774p-8f}, - {0x1.fb54641aebbc9p-1f, 0x1.34ac36ad8dafap-5f, -0x1.1c8ec267f9405p-4f, - 0x1.2a52c5d841848p-4f, -0x1.68541c02b3b6bp-5f, 0x1.5afe400196379p-7f, - 0x1.565b2d6eda3d6p-8f, -0x1.596aaff29e739p-8f}, - {0x1.fc67bcf2d7b8fp-1f, 0x1.e85c449e377efp-6f, -0x1.d177f166c07c6p-5f, - 0x1.fe23b7584b504p-5f, -0x1.4b12109613313p-5f, 0x1.8d9905c0acf7dp-7f, - 0x1.9265032a669dap-9f, -0x1.2ac4a6dbcbf3ep-8f}, - {0x1.fd40bd6d7a785p-1f, 0x1.7f5188610ddc7p-6f, -0x1.7954423f7c998p-5f, - 0x1.af5baae33887fp-5f, -0x1.2ad77c7cbc474p-5f, 0x1.a7b8c47ec2a51p-7f, - 0x1.46646ee094bccp-10f, -0x1.ef19d8db8673p-9f}, - {0x1.fdea6e062d0c9p-1f, 0x1.2a875b5ffab58p-6f, -0x1.2f3178cd6dcd5p-5f, - 0x1.68d1c45b94182p-5f, -0x1.09648ed3aeaefp-5f, 0x1.ad8b150d38164p-7f, - -0x1.e9a6023d9429fp-13f, -0x1.8722d19ee2e8ep-9f}, - {0x1.fe6e1742f7cf5p-1f, 0x1.cd5ec93c1243ap-7f, -0x1.e2ff3aaacb386p-6f, - 0x1.2aa4e5823cc89p-5f, -0x1.d049842dbe399p-6f, 0x1.a34edb21ab302p-7f, - -0x1.676e5996c7f9bp-10f, -0x1.23b01a35140bfp-9f}, - {0x1.fed37386190fbp-1f, 0x1.61beae53b72c2p-7f, -0x1.7d6193f22c3c1p-6f, - 0x1.e947279e3bb7dp-6f, -0x1.906031b97ca97p-6f, 0x1.8d14d62561755p-7f, - -0x1.1f245e7178882p-9f, -0x1.9257d4eb47685p-10f}, - {0x1.ff20e0a7ba8c2p-1f, 0x1.0d1d69569b839p-7f, -0x1.2a8ca0dc02752p-6f, - 0x1.8cc071b709751p-6f, -0x1.54a149f1b070cp-6f, 0x1.6e9137b13412cp-7f, - -0x1.6577ed3d8e83bp-9f, -0x1.e9c1a5178a289p-11f}, - {0x1.ff5b8fb26f5f6p-1f, 0x1.9646f35a7663cp-8f, -0x1.cf68ed9311b0bp-7f, - 0x1.3e8735b5a694fp-6f, -0x1.1e1612d026fdfp-6f, 0x1.4afd8e6ca636dp-7f, - -0x1.8c375170ccb22p-9f, -0x1.c799443c4fd3bp-12f}, - {0x1.ff87b1913e853p-1f, 0x1.30499b5039596p-8f, -0x1.64964201ec8bap-7f, - 0x1.fa73d7eafba98p-7f, -0x1.daa3022141fbbp-7f, 0x1.2509444c063b7p-7f, - -0x1.99482a2f8a0a1p-9f, -0x1.403d1f76c9454p-15f}, - {0x1.ffa89fe5b3625p-1f, 0x1.c4412bf4b8f35p-9f, -0x1.100f347126cf0p-7f, - 0x1.8ebda07671d40p-7f, -0x1.850c6a31c98c1p-7f, 0x1.fdac860c67d21p-8f, - -0x1.927d03d2ba12cp-9f, 0x1.0ff620b4190fep-12f}, - {0x1.ffc10194fcb64p-1f, 0x1.4d78bba8ca621p-9f, -0x1.9ba107a443e02p-8f, - 0x1.36f273fbc04ccp-7f, -0x1.3b38716ac7e6fp-7f, 0x1.b3fe0181914acp-8f, - -0x1.7d3fe7de98c5cp-9f, 0x1.ea31f8e5317f7p-12f}, - {0x1.ffd2eae369a07p-1f, 0x1.e7f232d9e266cp-10f, -0x1.34c7442dd48d9p-8f, - 0x1.e066bed070a0bp-8f, -0x1.f914f3c42fc0dp-8f, 0x1.6f4664ed2260fp-8f, - -0x1.5e59910761d24p-9f, 0x1.39cbb6e84c126p-11f}, - {0x1.ffdff92db56e5p-1f, 0x1.6235fbd7a4373p-10f, -0x1.cb5e029b9e56ap-9f, - 0x1.6fa4c7ef274dap-8f, -0x1.903a089a835f3p-8f, 0x1.30f12e0ca1901p-8f, - -0x1.39d21b6957f99p-9f, 0x1.5d3f8495a703cp-11f}, - {0x1.ffe96a78a04a9p-1f, 0x1.fe41cd9bb4f2cp-11f, -0x1.52d7b28966c0cp-9f, - 0x1.16c192d86a1a7p-8f, -0x1.39bfce951100cp-8f, 0x1.f376a7869f9e3p-9f, - -0x1.12e6cef999c4fp-9f, 0x1.66acd4d667b5p-11f}, - {0x1.fff0312b010b5p-1f, 0x1.6caa0d3583018p-11f, -0x1.efb729f4cf75bp-10f, - 0x1.a2da7cebe12acp-9f, -0x1.e6c27a24bc759p-9f, 0x1.93b1f4d8ea65p-9f, - -0x1.d82050aa94a08p-10f, 0x1.5cd7dc75d6cbap-11f}, - {0x1.fff50456dab8cp-1f, 0x1.0295ef6591865p-11f, -0x1.679880e95a4dap-10f, - 0x1.37d38e3a5c8ebp-9f, -0x1.75b3708aebb8fp-9f, 0x1.4231c4b4b0296p-9f, - -0x1.8e26476489318p-10f, 0x1.45c3b570dd924p-11f}, - {0x1.fff86cfd3e657p-1f, 0x1.6be02102b353dp-12f, -0x1.02b157780d6aep-10f, - 0x1.cc1d886861133p-10f, -0x1.1bff6f12ec9abp-9f, 0x1.fc0f77bd9c736p-10f, - -0x1.4a3320bd0959dp-10f, 0x1.267f8b4f95d2p-11f}, - {0x1.fffad0b901755p-1f, 0x1.fc0d55470cf5ep-13f, -0x1.7121aff5e820ep-11f, - 0x1.506d6992f7de5p-10f, -0x1.ab595d3ecd0d6p-10f, 0x1.8bdd79daaf754p-10f, - -0x1.0d9b090f997c1p-10f, 0x1.031ab9fd1c7dap-11f}, - {0x1.fffc7a37857d2p-1f, 0x1.5feada379d8a5p-13f, -0x1.05304df58f3aap-11f, - 0x1.e79c081b8600fp-11f, -0x1.3e5dbe33232e0p-10f, 0x1.30eb208200729p-10f, - -0x1.b1d493b147945p-11f, 0x1.bd587bbc071bep-12f}, - {0x1.fffd9fdeabccep-1f, 0x1.e3bcf436a1a49p-14f, -0x1.6e953111ef0a1p-12f, - 0x1.5e3edf6768654p-11f, -0x1.d5be67c0547a4p-11f, 0x1.d07d9ffa1d435p-11f, - -0x1.58328f5f358cap-11f, 0x1.76d42d95c42c4p-12f}, - {0x1.fffe68f4fa777p-1f, 0x1.49e17724f4cddp-14f, -0x1.fe48c44e229c1p-13f, - 0x1.f2bd95d76f188p-12f, -0x1.57388cb12d011p-11f, 0x1.5decc25c5c079p-11f, - -0x1.0d7499d1b0d2dp-11f, 0x1.359332c94ecdcp-12f}, - {0x1.fffef1960d85dp-1f, 0x1.be6abbb10a4cdp-15f, -0x1.6040381a8c313p-13f, - 0x1.5fff1dde9ee9dp-12f, -0x1.f0c933efa9971p-12f, 0x1.04cbf4a5cd760p-11f, - -0x1.a07f150af6dadp-12f, 0x1.f68dd183426bap-13f}, - {0x1.ffff4db27f146p-1f, 0x1.2bb5cc22e5cd8p-15f, -0x1.e25894899f526p-14f, - 0x1.ec8a8e5a72757p-13f, -0x1.64256ae0a3cf9p-12f, 0x1.80a836c18c46cp-12f, - -0x1.3dea401af6775p-12f, 0x1.915ddff3fe0d1p-13f}, - {0x1.ffff8b500e77cp-1f, 0x1.8f4ccca7fc769p-16f, -0x1.478cffe305946p-14f, - 0x1.559f04adde504p-13f, -0x1.f9e1577d6961dp-13f, 0x1.18bda53c14716p-12f, - -0x1.df8634c35541cp-13f, 0x1.3bb5c6b616337p-13f}, - {0x1.ffffb43555b5fp-1f, 0x1.07ebd2a2d26c8p-16f, -0x1.b93e442a37f2bp-15f, - 0x1.d5cf15159ce28p-14f, -0x1.63f5e1469c006p-13f, 0x1.95a03acebac18p-13f, - -0x1.656e5e2a1f8e2p-13f, 0x1.e98c437189bdep-14f}, - {0x1.ffffcf23ff5fcp-1f, 0x1.5a2adfa0b492cp-17f, -0x1.26c88270759f0p-15f, - 0x1.40473572b99a8p-14f, -0x1.f057cbde578a5p-14f, 0x1.22178d1c3c948p-13f, - -0x1.0765b61a0d859p-13f, 0x1.765b3ea03ddbep-14f}, - {0x1.ffffe0bd3e852p-1f, 0x1.c282cd3957a72p-18f, -0x1.86ad6dfa44faap-16f, - 0x1.b0f313f03a029p-15f, -0x1.56e44abecd255p-14f, 0x1.9ad1ecfe34a89p-14f, - -0x1.7fe4033478618p-14f, 0x1.1a8184e049fbfp-14f}, - {0x1.ffffec2641a9ep-1f, 0x1.22df29821407ep-18f, -0x1.00c902a6cfd98p-16f, - 0x1.22234eb88671fp-15f, -0x1.d57a181c9e6e1p-15f, 0x1.200c283b54a90p-14f, - -0x1.14b4c3295a7d0p-14f, 0x1.a4f966f713bdep-15f}, - {0x1.fffff37d63a36p-1f, 0x1.74adc8f405eecp-19f, -0x1.4ed4228e44858p-17f, - 0x1.81918baea92bap-16f, -0x1.3e81b17a0009cp-15f, 0x1.9004a36116436p-15f, - -0x1.8aa1ba400e076p-15f, 0x1.35cd4e2340a9ep-15f}, - {0x1.fffff82cdcf1bp-1f, 0x1.d9c73698fa87dp-20f, -0x1.b11017ec67115p-18f, - 0x1.fc0dfadf653f8p-17f, -0x1.ac4e03cd2dfc2p-16f, 0x1.131806b5abbc5p-15f, - -0x1.1672ef66fcaafp-15f, 0x1.c2882c7debed7p-16f}, - {0x1.fffffb248c39dp-1f, 0x1.2acee2f5ec66ap-20f, -0x1.15cc570408a36p-18f, - 0x1.4be757bbb75a3p-17f, -0x1.1d6aa5f8d2940p-16f, 0x1.76c5937d5105ep-16f, - -0x1.84dffc3ca9302p-16f, 0x1.43c8315f2c30ap-16f}, - {0x1.fffffd01f36afp-1f, 0x1.75fa8dbc840bap-21f, -0x1.6186da0133f5ap-19f, - 0x1.ae023231e1af5p-18f, -0x1.790812f7ca394p-17f, 0x1.f9c25656d0ef2p-17f, - -0x1.0cc66682e304cp-16f, 0x1.cc170a75d6f9cp-17f}, - {0x1.fffffe2ba0ea5p-1f, 0x1.d06ad6ecde88ep-22f, -0x1.be46aa8edc9a1p-20f, - 0x1.143860c7840b8p-18f, -0x1.edaba78fb1260p-18f, 0x1.52138a96ecee2p-17f, - -0x1.6fca538c4e2eep-17f, 0x1.434040640bcefp-17f}, - {0x1.fffffee3cc32cp-1f, 0x1.1e1e857adb8ddp-22f, -0x1.1769ce5f2a6e8p-20f, - 0x1.5fe5d479b0543p-19f, -0x1.405d865c94c2ap-18f, 0x1.bfc94feb96afcp-18f, - -0x1.f245d5f3e8358p-18f, 0x1.c142456acf443p-18f}, - }; - float ax = __builtin_fabsf(x); - uint32_t ux = asuint(ax); - float s = x; - float z = ax; - /* 0x407ad444 corresponds to x = 0x1.f5a888p+1 = 3.91921..., which is the - largest float such that erf(x) does not round to 1 (to nearest). */ - if ((ux > 0x407ad444u)) { - float os = __builtin_copysignf(1.0f, x); - if (ux > (0xffu << 23)) - return x + x; /* nan */ - if (ux == (0xffu << 23)) - return os; /* +-inf */ - return os - 0x1p-25f * os; - } - uint32_t i = 16.0f * ax; - float v = static_cast(i); - /* 0x3ee00000 corresponds to x = 0.4375, for smaller x we have i < 7. */ - if ((ux < 0x3ee00000u)) { - static const float c[] = {0x1.20dd750429b6dp+0f, -0x1.812746b0375fbp-2f, - 0x1.ce2f219fd6f45p-4f, -0x1.b82ce2cbf0838p-6f, - 0x1.565bb655adb85p-8f, -0x1.c025bfc879c94p-11f, - 0x1.f81718f61309cp-14f, -0x1.cc67bd88f5867p-17f}; - float z2 = s * s, z4 = z2 * z2, z8 = z4 * z4; - float c0 = c[0] + z2 * c[1]; - float c2 = c[2] + z2 * c[3]; - float c4 = c[4] + z2 * c[5]; - float c6 = c[6] + z2 * c[7]; - c0 += z4 * c2; - c4 += z4 * c6; - c0 += z8 * c4; - return s * c0; - } - z = (z - 0.03125f) - 0.0625f * v; - const float *c = C[i - 7]; - float z2 = z * z, z4 = z2 * z2; - float c0 = c[0] + z * c[1]; - float c2 = c[2] + z * c[3]; - float c4 = c[4] + z * c[5]; - float c6 = c[6] + z * c[7]; - c0 += z2 * c2; - c4 += z2 * c6; - c0 += z4 * c4; - return __builtin_copysignf(c0, s); -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/math/exp.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/math/exp.cpp deleted file mode 100644 index 8acc0b093..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/math/exp.cpp +++ /dev/null @@ -1,66 +0,0 @@ -// Vector adaptation of glibc 2.42 sysdeps/ieee754/flt-32/e_expf.c. -// Copyright (C) 2017-2025 Free Software Foundation, Inc. -// LGPL-2.1-or-later; see ../../../thirdparty/glibc/COPYING.LIB. -#include "constants.h" -#include "math.h" - -extern "C" vfloat32m1_t rvv_exp(vfloat32m1_t value, size_t vl) { - auto bits = __riscv_vreinterpret_u32m1(value); - auto magnitude = __riscv_vand_vx_u32m1(bits, 0x7fffffff, vl); - auto finite = __riscv_vmsltu_vx_u32m1_b32(magnitude, 0x7f800000, vl); - auto ordered = __riscv_vmerge_vvm_f32m1(__riscv_vfmv_v_f_f32m1(0, vl), value, - finite, vl); - auto normal = __riscv_vmfge_vf_f32m1_b32(ordered, -0x1.9d1d9ep6f, vl); - normal = __riscv_vmand_mm_b32( - normal, __riscv_vmfle_vf_f32m1_b32(ordered, 0x1.62e42ep6f, vl), vl); - normal = __riscv_vmand_mm_b32(normal, finite, vl); - auto input = __riscv_vmerge_vvm_f32m1(__riscv_vfmv_v_f_f32m1(0, vl), value, - normal, vl); - auto scaled = __riscv_vfmul_vf_f32m1(input, 0x1.715476p5f, vl); - auto kd = __riscv_vfadd_vf_f32m1(scaled, 0x1.8p23f, vl); - kd = __riscv_vfsub_vf_f32m1(kd, 0x1.8p23f, vl); - auto ki = __riscv_vfcvt_rtz_x_f_v_i32m1(kd, vl); - auto r = __riscv_vfnmsac_vf_f32m1(input, 0x1.62e4p-6f, kd, vl); - r = __riscv_vfnmsac_vf_f32m1(r, 0x1.7f7d1cp-25f, kd, vl); - auto index = __riscv_vand_vx_u32m1(__riscv_vreinterpret_u32m1(ki), 31, vl); - index = __riscv_vsll_vx_u32m1(index, 2, vl); - auto table = __riscv_vluxei32_v_f32m1(exp_table, index, vl); - auto exponent = __riscv_vsra_vx_i32m1(ki, 5, vl); - auto bounded = __riscv_vmax_vx_i32m1(exponent, -126, vl); - bounded = __riscv_vmin_vx_i32m1(bounded, 127, vl); - auto tail = __riscv_vsub_vv_i32m1(exponent, bounded, vl); - auto scale_bits = - __riscv_vsll_vx_i32m1(__riscv_vadd_vx_i32m1(bounded, 127, vl), 23, vl); - auto tail_bits = - __riscv_vsll_vx_i32m1(__riscv_vadd_vx_i32m1(tail, 127, vl), 23, vl); - auto polynomial = __riscv_vfmacc_vf_f32m1(__riscv_vfmv_v_f_f32m1(0.5f, vl), - 1.0f / 6.0f, r, vl); - auto y = __riscv_vfadd_vf_f32m1(r, 1.0f, vl); - y = __riscv_vfmacc_vv_f32m1(y, polynomial, __riscv_vfmul_vv_f32m1(r, r, vl), - vl); - y = __riscv_vfmul_vv_f32m1(y, table, vl); - y = __riscv_vfmul_vv_f32m1( - y, __riscv_vreinterpret_f32m1(__riscv_vreinterpret_u32m1(tail_bits)), vl); - auto result = __riscv_vfmul_vv_f32m1( - y, __riscv_vreinterpret_f32m1(__riscv_vreinterpret_u32m1(scale_bits)), - vl); - auto under = __riscv_vmflt_vf_f32m1_b32(ordered, -0x1.9d1d9ep6f, vl); - under = __riscv_vmand_mm_b32(under, finite, vl); - auto helper = __riscv_vfmv_v_f_f32m1(0x1.4p-75f, vl); - helper = __riscv_vfmul_vv_f32m1_m(under, helper, helper, vl); - result = __riscv_vmerge_vvm_f32m1(result, helper, under, vl); - auto zero = __riscv_vmflt_vf_f32m1_b32(ordered, -0x1.9fe368p6f, vl); - result = __riscv_vfmerge_vfm_f32m1(result, 0, zero, vl); - auto over = __riscv_vmfgt_vf_f32m1_b32(ordered, 0x1.62e42ep6f, vl); - auto overflow = __riscv_vmand_mm_b32(over, finite, vl); - helper = __riscv_vfmv_v_f_f32m1(0x1p97f, vl); - helper = __riscv_vfmul_vv_f32m1_m(overflow, helper, helper, vl); - result = __riscv_vmerge_vvm_f32m1(result, helper, overflow, vl); - auto infinity = __riscv_vmseq_vx_u32m1_b32(bits, 0x7f800000, vl); - result = __riscv_vfmerge_vfm_f32m1(result, __builtin_inff(), infinity, vl); - auto negative_infinity = __riscv_vmseq_vx_u32m1_b32(bits, 0xff800000, vl); - result = __riscv_vfmerge_vfm_f32m1(result, 0, negative_infinity, vl); - auto nan = __riscv_vmsgtu_vx_u32m1_b32(magnitude, 0x7f800000, vl); - return __riscv_vmerge_vvm_f32m1( - result, __riscv_vfadd_vv_f32m1_m(nan, value, value, vl), nan, vl); -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/math/math.h b/bb-tests/workloads/lib/bbsw/kernels/rvv/math/math.h deleted file mode 100644 index ef5af1af0..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/math/math.h +++ /dev/null @@ -1,6 +0,0 @@ -#pragma once -#include - -extern "C" vfloat32m1_t rvv_exp(vfloat32m1_t value, size_t vl); -extern "C" vfloat32m1_t rvv_sin(vfloat32m1_t value, size_t vl); -extern "C" vfloat32m1_t rvv_cos(vfloat32m1_t value, size_t vl); diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/math/mxfp8.h b/bb-tests/workloads/lib/bbsw/kernels/rvv/math/mxfp8.h deleted file mode 100644 index 89299bc7e..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/math/mxfp8.h +++ /dev/null @@ -1,64 +0,0 @@ -#pragma once -#include -#include - -static inline vfloat32m2_t rvv_mxfp8_load16(const uint8_t *input, - unsigned scale, uint32_t &status) { - constexpr size_t vl = 16; - if (scale == 255) { - status = 1U << 8; - return __riscv_vfmv_v_f_f32m2(0, vl); - } - auto bytes = __riscv_vle8_v_u8mf2(input, vl); - auto code = __riscv_vzext_vf2_u32m2(__riscv_vzext_vf2_u16m1(bytes, vl), vl); - auto sign = - __riscv_vsll_vx_u32m2(__riscv_vand_vx_u32m2(code, 128, vl), 24, vl); - auto magnitude = __riscv_vand_vx_u32m2(code, 127, vl); - if (__riscv_vcpop_m_b16(__riscv_vmseq_vx_u32m2_b16(magnitude, 127, vl), vl)) { - status = 1U << 8; - return __riscv_vfmv_v_f_f32m2(0, vl); - } - auto exponent = __riscv_vsrl_vx_u32m2(magnitude, 3, vl); - auto fraction = __riscv_vand_vx_u32m2(code, 7, vl); - auto subnormal = __riscv_vmseq_vx_u32m2_b16(exponent, 0, vl); - auto highest = __riscv_vmv_v_x_u32m2(0, vl); - highest = __riscv_vmerge_vxm_u32m2( - highest, 1, __riscv_vmsgeu_vx_u32m2_b16(fraction, 2, vl), vl); - highest = __riscv_vmerge_vxm_u32m2( - highest, 2, __riscv_vmsgeu_vx_u32m2_b16(fraction, 4, vl), vl); - auto normalExponent = __riscv_vadd_vx_i32m2( - __riscv_vreinterpret_v_u32m2_i32m2(exponent), int(scale) - 7, vl); - auto subExponent = __riscv_vadd_vx_i32m2( - __riscv_vreinterpret_v_u32m2_i32m2(highest), int(scale) - 9, vl); - auto resultExponent = - __riscv_vmerge_vvm_i32m2(normalExponent, subExponent, subnormal, vl); - auto overflow = - __riscv_vmand_mm_b16(__riscv_vmsge_vx_i32m2_b16(resultExponent, 255, vl), - __riscv_vmsne_vx_u32m2_b16(magnitude, 0, vl), vl); - if (__riscv_vcpop_m_b16(overflow, vl)) { - status = 2U << 8; - return __riscv_vfmv_v_f_f32m2(0, vl); - } - auto normalSignificand = - __riscv_vsll_vx_u32m2(__riscv_vor_vx_u32m2(fraction, 8, vl), 20, vl); - auto subSignificand = __riscv_vsll_vv_u32m2( - fraction, __riscv_vrsub_vx_u32m2(highest, 23, vl), vl); - auto significand = __riscv_vmerge_vvm_u32m2(normalSignificand, subSignificand, - subnormal, vl); - auto normalBits = __riscv_vor_vv_u32m2( - __riscv_vsll_vx_u32m2(__riscv_vreinterpret_v_i32m2_u32m2(resultExponent), - 23, vl), - __riscv_vand_vx_u32m2(significand, 0x7fffff, vl), vl); - auto tinyBits = - __riscv_vsrl_vv_u32m2(significand, - __riscv_vreinterpret_v_i32m2_u32m2( - __riscv_vrsub_vx_i32m2(resultExponent, 1, vl)), - vl); - auto bits = __riscv_vmerge_vvm_u32m2( - normalBits, tinyBits, __riscv_vmsle_vx_i32m2_b16(resultExponent, 0, vl), - vl); - bits = __riscv_vmerge_vxm_u32m2( - bits, 0, __riscv_vmseq_vx_u32m2_b16(magnitude, 0, vl), vl); - return __riscv_vreinterpret_v_u32m2_f32m2( - __riscv_vor_vv_u32m2(bits, sign, vl)); -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/math/sin.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/math/sin.cpp deleted file mode 100644 index 63212a3fd..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/math/sin.cpp +++ /dev/null @@ -1,107 +0,0 @@ -// Vector adaptation of glibc 2.42 s_sinf.c and s_sincosf.h/sincosf_poly.h. -// Copyright (C) 2018-2025 Free Software Foundation, Inc. -// LGPL-2.1-or-later; see ../../../thirdparty/glibc/COPYING.LIB. -#include "constants.h" -#include "math.h" - -extern "C" vfloat32m1_t rvv_sin(vfloat32m1_t value, size_t vl) { - auto bits = __riscv_vreinterpret_u32m1(value); - auto top = - __riscv_vand_vx_u32m1(__riscv_vsrl_vx_u32m1(bits, 20, vl), 0x7ff, vl); - auto tiny = __riscv_vmsltu_vx_u32m1_b32(top, 0x398, vl); - auto fast = __riscv_vmsltu_vx_u32m1_b32(top, 0x42f, vl); - fast = __riscv_vmandn_mm_b32(fast, tiny, vl); - auto finite = __riscv_vmsltu_vx_u32m1_b32(top, 0x7f8, vl); - auto input = - __riscv_vmerge_vvm_f32m1(__riscv_vfmv_v_f_f32m1(0, vl), value, fast, vl); - auto x = input; - auto r = __riscv_vfmul_vf_f32m1(x, 0x1.45f306dc9c883p23, vl); - auto n = __riscv_vfcvt_rtz_x_f_v_i32m1(r, vl); - n = __riscv_vsra_vx_i32m1(__riscv_vadd_vx_i32m1(n, 0x800000, vl), 24, vl); - auto small = __riscv_vmsltu_vx_u32m1_b32(top, 0x3f4, vl); - n = __riscv_vmerge_vxm_i32m1(n, 0, small, vl); - auto nd = __riscv_vfcvt_f_x_v_f32m1(n, vl); - x = __riscv_vfnmsac_vf_f32m1(x, 0x1.921fb4p0f, nd, vl); - x = __riscv_vfnmsac_vf_f32m1(x, 0x1.4442d2p-24f, nd, vl); - auto quadrant = n; - auto large = __riscv_vmandn_mm_b32(finite, fast, vl); - large = __riscv_vmandn_mm_b32(large, tiny, vl); - if (__riscv_vcpop_m_b32(large, vl)) { - auto index = - __riscv_vand_vx_u32m1(__riscv_vsrl_vx_u32m1(bits, 26, vl), 15, vl); - index = __riscv_vsll_vx_u32m1(index, 2, vl); - auto a0 = __riscv_vluxei32_v_u32m1(inv_pio4, index, vl); - auto a1 = __riscv_vluxei32_v_u32m1(inv_pio4 + 4, index, vl); - auto a2 = __riscv_vluxei32_v_u32m1(inv_pio4 + 8, index, vl); - auto shift = - __riscv_vand_vx_u32m1(__riscv_vsrl_vx_u32m1(bits, 23, vl), 7, vl); - auto xi = __riscv_vor_vx_u32m1(__riscv_vand_vx_u32m1(bits, 0xffffff, vl), - 0x800000, vl); - xi = __riscv_vsll_vv_u32m1(xi, shift, vl); - auto low0 = __riscv_vmulhu_vv_u32m1(xi, a2, vl); - auto low1 = __riscv_vmul_vv_u32m1(xi, a1, vl); - auto low = __riscv_vadd_vv_u32m1(low0, low1, vl); - auto carry_mask = __riscv_vmsltu_vv_u32m1_b32(low, low1, vl); - auto high = __riscv_vadd_vv_u32m1(__riscv_vmul_vv_u32m1(xi, a0, vl), - __riscv_vmulhu_vv_u32m1(xi, a1, vl), vl); - auto carry = __riscv_vmerge_vxm_u32m1(__riscv_vmv_v_x_u32m1(0, vl), 1, - carry_mask, vl); - high = __riscv_vadd_vv_u32m1(high, carry, vl); - auto q = __riscv_vsrl_vx_u32m1(__riscv_vadd_vx_u32m1(high, 0x20000000, vl), - 30, vl); - high = __riscv_vsub_vv_u32m1(high, __riscv_vsll_vx_u32m1(q, 30, vl), vl); - auto reduced = - __riscv_vfcvt_f_x_v_f32m1(__riscv_vreinterpret_i32m1(high), vl); - reduced = __riscv_vfmul_vf_f32m1(reduced, 0x1.921fb4p-30f, vl); - auto fraction = __riscv_vfcvt_f_xu_v_f32m1(low, vl); - reduced = __riscv_vfmacc_vf_f32m1(reduced, 0x1.921fb4p-62f, fraction, vl); - auto q32 = __riscv_vreinterpret_i32m1(q); - n = __riscv_vmerge_vvm_i32m1(n, q32, large, vl); - auto sign = __riscv_vreinterpret_i32m1(__riscv_vsrl_vx_u32m1(bits, 31, vl)); - quadrant = __riscv_vmerge_vvm_i32m1( - quadrant, __riscv_vadd_vv_i32m1(q32, sign, vl), large, vl); - x = __riscv_vmerge_vvm_f32m1(x, reduced, large, vl); - } - auto x2 = __riscv_vfmul_vv_f32m1(x, x, vl); - auto negative = - __riscv_vmsne_vx_i32m1_b32(__riscv_vand_vx_i32m1(quadrant, 2, vl), 0, vl); - // sign[] is {1,-1,-1,1}; cosine polynomial sign depends on bit 1. - auto signbit = __riscv_vxor_vv_i32m1( - quadrant, __riscv_vsra_vx_i32m1(quadrant, 1, vl), vl); - auto sine_negative = - __riscv_vmsne_vx_i32m1_b32(__riscv_vand_vx_i32m1(signbit, 1, vl), 0, vl); - auto sx = __riscv_vmerge_vvm_f32m1(x, __riscv_vfneg_v_f32m1(x, vl), - sine_negative, vl); - auto x3 = __riscv_vfmul_vv_f32m1(sx, x2, vl); - auto s1 = - __riscv_vfmacc_vf_f32m1(__riscv_vfmv_v_f_f32m1(0x1.1107605230bc4p-7, vl), - -0x1.994eb3774cf24p-13, x2, vl); - auto s = __riscv_vfmacc_vf_f32m1(sx, -0x1.555545995a603p-3, x3, vl); - auto x7 = __riscv_vfmul_vv_f32m1(x3, x2, vl); - s = __riscv_vfmacc_vv_f32m1(s, s1, x7, vl); - auto x4 = __riscv_vfmul_vv_f32m1(x2, x2, vl); - auto c2 = __riscv_vfmacc_vf_f32m1( - __riscv_vfmv_v_f_f32m1(-0x1.6c087e89a359dp-10, vl), 0x1.99343027bf8c3p-16, - x2, vl); - auto c1 = __riscv_vfmacc_vf_f32m1(__riscv_vfmv_v_f_f32m1(1, vl), - -0x1.ffffffd0c621cp-2, x2, vl); - auto c = __riscv_vfmacc_vf_f32m1(c1, 0x1.55553e1068f19p-5, x4, vl); - auto x6 = __riscv_vfmul_vv_f32m1(x4, x2, vl); - c = __riscv_vfmacc_vv_f32m1(c, c2, x6, vl); - c = __riscv_vmerge_vvm_f32m1(c, __riscv_vfneg_v_f32m1(c, vl), negative, vl); - auto odd = __riscv_vmsne_vx_i32m1_b32(__riscv_vand_vx_i32m1(n, 1, vl), 0, vl); - auto result = __riscv_vmerge_vvm_f32m1(s, c, odd, vl); - auto subnormal = __riscv_vmsltu_vx_u32m1_b32(top, 0x008, vl); - if (__riscv_vcpop_m_b32(subnormal, vl)) { - auto extended = value; - auto square = __riscv_vfmul_vv_f32m1_m(subnormal, extended, extended, vl); - auto rounded = square; - // glibc forces this conversion for its underflow/inexact side effects. - asm volatile("" : : "vr"(rounded)); - } - result = __riscv_vmerge_vvm_f32m1(result, value, tiny, vl); - auto invalid = __riscv_vmnot_m_b32(finite, vl); - auto difference = __riscv_vfsub_vv_f32m1_m(invalid, value, value, vl); - auto nan = __riscv_vfdiv_vv_f32m1_m(invalid, difference, difference, vl); - return __riscv_vmerge_vvm_f32m1(result, nan, invalid, vl); -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/matmul.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/matmul.cpp deleted file mode 100644 index 5932082b3..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/matmul.cpp +++ /dev/null @@ -1,19 +0,0 @@ -#include "kernels.h" -#include - -extern "C" void rvv_matmul(float *output, const float *lhs, const float *rhs, - size_t rows, size_t cols, size_t inner) { - for (size_t row = 0; row < rows; ++row) { - for (size_t col = 0; col < cols;) { - const size_t vl = __riscv_vsetvl_e32m1(cols - col); - auto sum = __riscv_vle32_v_f32m1(output + row * cols + col, vl); - for (size_t k = 0; k < inner; ++k) { - auto weight = __riscv_vle32_v_f32m1(rhs + k * cols + col, vl); - auto product = __riscv_vfmul_vf_f32m1(weight, lhs[row * inner + k], vl); - sum = __riscv_vfadd_vv_f32m1(sum, product, vl); - } - __riscv_vse32_v_f32m1(output + row * cols + col, sum, vl); - col += vl; - } - } -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/matmul_dispatch.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/matmul_dispatch.cpp deleted file mode 100644 index 5a9476713..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/matmul_dispatch.cpp +++ /dev/null @@ -1,212 +0,0 @@ -#include "images.h" -#include -#include -#include -#include -#include -#include -#include - -namespace { -constexpr unsigned readBank = 4, writeBank = 5; -constexpr unsigned programBank = VIRTUAL_BANK_NUM; -constexpr uint32_t bankBytes = BANK_LINES * (BANK_WIDTH / 8); -static_assert(bankBytes >= 4096 && bankBytes <= 65536 && bankBytes % 16 == 0); -} // namespace - -extern "C" void _mlir_ciface_rvv_matmul(UnrankedMemRefType *output, - UnrankedMemRefType *lhs, - UnrankedMemRefType *rhs) { - DynamicMemRefType out(*output), a(*lhs), b(*rhs); - if ((out.rank != 2 && out.rank != 3) || a.rank != out.rank || - b.rank != out.rank) { - fputs("RVV matmul requires matching rank-two or rank-three tensors\n", - stderr); - abort(); - } - const int64_t axis = out.rank - 2; - const int64_t rows = a.sizes[axis], inner = a.sizes[axis + 1]; - const int64_t cols = b.sizes[axis + 1]; - const int64_t batches = axis ? out.sizes[0] : 1; - if (rows <= 0 || inner <= 0 || cols <= 0 || batches <= 0 || - b.sizes[axis] != inner || out.sizes[axis] != rows || - out.sizes[axis + 1] != cols || - (axis && (a.sizes[0] != batches || b.sizes[0] != batches))) { - fprintf(stderr, - "RVV matmul shape mismatch: output=[%lld,%lld,%lld], " - "lhs=[%lld,%lld,%lld], rhs=[%lld,%lld,%lld]\n", - (long long)(axis ? out.sizes[0] : 1), (long long)out.sizes[axis], - (long long)out.sizes[axis + 1], (long long)(axis ? a.sizes[0] : 1), - (long long)a.sizes[axis], (long long)a.sizes[axis + 1], - (long long)(axis ? b.sizes[0] : 1), (long long)b.sizes[axis], - (long long)b.sizes[axis + 1]); - abort(); - } - alignas(16) float aTile[bankBytes / sizeof(float)]; - alignas(16) float bTile[bankBytes / sizeof(float)]; - alignas(16) float cTile[16 * 16]; - for (unsigned bank = 0; bank < 4; ++bank) - bb_mem_alloc(bank, 1, 1); - bb_mem_transfer(2, readBank); - bb_mem_transfer(3, readBank); - bb_mem_transfer(0, writeBank); - bb_mem_transfer(1, writeBank); - mvin_kernel(images::matmul.bytes, images::matmul.size, programBank); - for (int64_t batch = 0; batch < batches; ++batch) { - const float *aSource = - a.data + a.offset + (axis ? batch * a.strides[0] : 0); - const float *bSource = - b.data + b.offset + (axis ? batch * b.strides[0] : 0); - float *cSource = - out.data + out.offset + (axis ? batch * out.strides[0] : 0); - for (int64_t row = 0; row < rows; row += 16) { - const uint32_t m = std::min(16, rows - row); - for (int64_t col = 0; col < cols; col += 16) { - const uint32_t n = std::min(16, cols - col); - const uint32_t count = m * n; - float *cDirect = - cSource + row * out.strides[axis] + col * out.strides[axis + 1]; - const bool directC = - out.strides[axis + 1] == 1 && (m == 1 || out.strides[axis] == n) && - count % 4 == 0 && (reinterpret_cast(cDirect) & 15) == 0; - if (directC) { - bb_mvin_group((uintptr_t)cDirect, writeBank, 1, count / 4, 1); - } else { - for (uint32_t r = 0; r < m; ++r) - for (uint32_t c = 0; c < n; ++c) - cTile[r * n + c] = cSource[(row + r) * out.strides[axis] + - (col + c) * out.strides[axis + 1]]; - std::fill(cTile + count, cTile + (count + 3) / 4 * 4, 0.0f); - bb_mvin_group((uintptr_t)cTile, writeBank, 1, (count + 3) / 4, 1); - } - const int64_t tileK = bankBytes / sizeof(float) / std::max(m, n); - for (int64_t begin = 0; begin < inner; begin += tileK) { - const uint32_t k = std::min(tileK, inner - begin); - const float *aDirect = - aSource + row * a.strides[axis] + begin * a.strides[axis + 1]; - const float *bDirect = - bSource + begin * b.strides[axis] + col * b.strides[axis + 1]; - const bool directA = a.strides[axis + 1] == 1 && - (m == 1 || a.strides[axis] == k) && - m * k % 4 == 0 && - (reinterpret_cast(aDirect) & 15) == 0; - const bool directB = b.strides[axis + 1] == 1 && - (k == 1 || b.strides[axis] == n) && - k * n % 4 == 0 && - (reinterpret_cast(bDirect) & 15) == 0; - if (directA) { - bb_mvin_group((uintptr_t)aDirect, readBank, 0, m * k / 4, 1); - } else { - for (uint32_t r = 0; r < m; ++r) { - const float *source = aSource + (row + r) * a.strides[axis] + - begin * a.strides[axis + 1]; - if (a.strides[axis + 1] == 1) - std::memcpy(aTile + r * k, source, k * sizeof(float)); - else - for (uint32_t i = 0; i < k; ++i) - aTile[r * k + i] = source[i * a.strides[axis + 1]]; - } - std::fill(aTile + m * k, aTile + (m * k + 3) / 4 * 4, 0.0f); - bb_mvin_group((uintptr_t)aTile, readBank, 0, (m * k + 3) / 4, 1); - } - if (directB) { - bb_mvin_group((uintptr_t)bDirect, readBank, 1, k * n / 4, 1); - } else { - for (uint32_t i = 0; i < k; ++i) { - const float *source = bSource + (begin + i) * b.strides[axis] + - col * b.strides[axis + 1]; - if (b.strides[axis + 1] == 1) - std::memcpy(bTile + i * n, source, n * sizeof(float)); - else - for (uint32_t c = 0; c < n; ++c) - bTile[i * n + c] = source[c * b.strides[axis + 1]]; - } - std::fill(bTile + k * n, bTile + (k * n + 3) / 4 * 4, 0.0f); - bb_mvin_group((uintptr_t)bTile, readBank, 1, (k * n + 3) / 4, 1); - } - kernel_launch call{images::matmul.entry, - images::matmul.text_bytes, - 0x40002000, - {uint64_t(3) << 32, 0, uint64_t(1) << 32, m, n, k}, - 0}; - bb_mvin_group((uintptr_t)&call, writeBank, 0, sizeof(call) / 16, 1); - run_kernel(readBank, programBank, writeBank, 0); - } - if (directC) { - bb_mvout_group((uintptr_t)cDirect, writeBank, 1, count / 4, 1); - } else { - bb_mvout_group((uintptr_t)cTile, writeBank, 1, (count + 3) / 4, 1); - for (uint32_t r = 0; r < m; ++r) - for (uint32_t c = 0; c < n; ++c) - cSource[(row + r) * out.strides[axis] + - (col + c) * out.strides[axis + 1]] = cTile[r * n + c]; - } - } - } - } - bb_mem_release(readBank); - bb_mem_release(writeBank); - release_kernel(programBank); -} - -extern "C" void -_mlir_ciface_rvv_matmul_transpose_rhs(UnrankedMemRefType *output, - UnrankedMemRefType *lhs, - UnrankedMemRefType *rhs) { - DynamicMemRefType b(*rhs); - if (b.rank == 2) { - StridedMemRefType view{b.basePtr, - b.data, - b.offset, - {b.sizes[1], b.sizes[0]}, - {b.strides[1], b.strides[0]}}; - UnrankedMemRefType transposed{2, &view}; - _mlir_ciface_rvv_matmul(output, lhs, &transposed); - } else if (b.rank == 3) { - StridedMemRefType view{ - b.basePtr, - b.data, - b.offset, - {b.sizes[0], b.sizes[2], b.sizes[1]}, - {b.strides[0], b.strides[2], b.strides[1]}}; - UnrankedMemRefType transposed{3, &view}; - _mlir_ciface_rvv_matmul(output, lhs, &transposed); - } else { - fputs("RVV transposed matmul requires rank-two or rank-three RHS\n", - stderr); - abort(); - } -} - -namespace { -void broadcastMatmul(UnrankedMemRefType *output, - UnrankedMemRefType *lhs, - UnrankedMemRefType *rhs, bool transpose) { - DynamicMemRefType out(*output), b(*rhs); - if (out.rank != 3 || b.rank != 4 || b.sizes[0] != 1 || b.sizes[1] != 1) { - fputs("RVV broadcast matmul requires one rank-four RHS head\n", stderr); - abort(); - } - StridedMemRefType view{ - b.basePtr, - b.data, - b.offset, - {out.sizes[0], b.sizes[transpose ? 3 : 2], b.sizes[transpose ? 2 : 3]}, - {0, b.strides[transpose ? 3 : 2], b.strides[transpose ? 2 : 3]}}; - UnrankedMemRefType repeated{3, &view}; - _mlir_ciface_rvv_matmul(output, lhs, &repeated); -} -} // namespace - -extern "C" void -_mlir_ciface_rvv_matmul_broadcast_rhs(UnrankedMemRefType *output, - UnrankedMemRefType *lhs, - UnrankedMemRefType *rhs) { - broadcastMatmul(output, lhs, rhs, false); -} - -extern "C" void _mlir_ciface_rvv_matmul_broadcast_transpose_rhs( - UnrankedMemRefType *output, UnrankedMemRefType *lhs, - UnrankedMemRefType *rhs) { - broadcastMatmul(output, lhs, rhs, true); -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/norm.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/norm.cpp deleted file mode 100644 index 43259fd60..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/norm.cpp +++ /dev/null @@ -1,69 +0,0 @@ -#include "kernels.h" -#include - -template -static void norm(float *output, const float *input, const float *weight, - size_t width, uint32_t meanMultiplierBits, - uint32_t epsilonBits) { - auto sum = __riscv_vfmv_v_f_f32m1(0, 1); - for (size_t offset = 0; offset < width;) { - const size_t vl = __riscv_vsetvl_e32m1(width - offset); - auto values = __riscv_vle32_v_f32m1(input + offset, vl); - auto squares = __riscv_vfmul_vv_f32m1(values, values, vl); - sum = __riscv_vfredosum_vs_f32m1_f32m1(squares, sum, vl); - offset += vl; - } - auto meanMultiplier = - __riscv_vreinterpret_f32m1(__riscv_vmv_v_x_u32m1(meanMultiplierBits, 1)); - auto epsilon = - __riscv_vreinterpret_f32m1(__riscv_vmv_v_x_u32m1(epsilonBits, 1)); - auto mean = __riscv_vfmul_vv_f32m1(sum, meanMultiplier, 1); - auto biased = __riscv_vfadd_vv_f32m1(mean, epsilon, 1); - auto root = __riscv_vfsqrt_v_f32m1(biased, 1); - auto reciprocal = __riscv_vfrdiv_vf_f32m1(root, 1, 1); - const float scale = __riscv_vfmv_f_s_f32m1_f32(reciprocal); - for (size_t offset = 0; offset < width;) { - const size_t vl = __riscv_vsetvl_e32m1(width - offset); - auto values = __riscv_vle32_v_f32m1(input + offset, vl); - auto scaled = __riscv_vfmul_vf_f32m1(values, scale, vl); - if constexpr (weighted) { - auto weights = __riscv_vle32_v_f32m1(weight + offset, vl); - scaled = __riscv_vfmul_vv_f32m1(weights, scaled, vl); - } - __riscv_vse32_v_f32m1(output + offset, scaled, vl); - offset += vl; - } -} - -extern "C" void rvv_norm(float *output, const float *input, const float *weight, - size_t width, uint32_t mean, uint32_t epsilon) { - norm(output, input, weight, width, mean, epsilon); -} - -extern "C" void rvv_norm_no_weight(float *output, const float *input, - size_t width, uint32_t mean, - uint32_t epsilon) { - norm(output, input, nullptr, width, mean, epsilon); -} - -extern "C" void rvv_norm_launch(float *output, const float *input, - const float *weight, size_t width, - uint32_t mean, uint32_t epsilon, - uint32_t *flags, uint32_t rounding) { - asm volatile("csrw fcsr, %0" ::"r"(rounding) : "memory"); - rvv_norm(output, input, weight, width, mean, epsilon); - uint32_t result; - asm volatile("csrr %0, fflags" : "=r"(result)::"memory"); - *flags = result; -} - -extern "C" void rvv_norm_no_weight_launch(float *output, const float *input, - size_t width, uint32_t mean, - uint32_t epsilon, uint32_t *flags, - uint32_t rounding) { - asm volatile("csrw fcsr, %0" ::"r"(rounding) : "memory"); - rvv_norm_no_weight(output, input, width, mean, epsilon); - uint32_t result; - asm volatile("csrr %0, fflags" : "=r"(result)::"memory"); - *flags = result; -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/normalization_dispatch.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/normalization_dispatch.cpp deleted file mode 100644 index fa69cf8da..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/normalization_dispatch.cpp +++ /dev/null @@ -1,178 +0,0 @@ -#include "images.h" -#include -#include -#include -#include -#include -#include -#include - -namespace { -constexpr unsigned readBank = 5, writeBank = 6; -constexpr unsigned inputBank = 0, weightBank = 1, biasBank = 2; -constexpr unsigned descriptorBank = 3, outputBank = 4; -constexpr uint64_t descriptorAddress = uint64_t(descriptorBank) << 32; -constexpr unsigned programBank = VIRTUAL_BANK_NUM; -constexpr uint32_t bankBytes = BANK_LINES * (BANK_WIDTH / 8); - -int64_t elements(const DynamicMemRefType &value) { - int64_t count = 1; - for (int64_t axis = value.rank - 1; axis >= 0; --axis) { - if (value.sizes[axis] <= 0 || - (value.sizes[axis] != 1 && value.strides[axis] != count)) { - fputs("RVV normalization requires contiguous FP32 tensors\n", stderr); - abort(); - } - count *= value.sizes[axis]; - } - return count; -} -void load(const images::KernelImage &image) { - for (unsigned bank = 0; bank < 5; ++bank) - bb_mem_alloc(bank, 1, 1); - for (unsigned bank = 0; bank < 3; ++bank) - bb_mem_transfer(bank, readBank); - bb_mem_transfer(descriptorBank, writeBank); - bb_mem_transfer(outputBank, writeBank); - mvin_kernel(image.bytes, image.size, programBank); -} -void send(const float *input, unsigned bank, uint32_t count, float *staging) { - if (count % 4) { - std::memcpy(staging, input, count * sizeof(float)); - std::fill_n(staging + count, 4 - count % 4, 0.0f); - input = staging; - } - bb_mvin_group((uintptr_t)input, readBank, bank, (count + 3) / 4, 1); -} -void launch(kernel_launch &descriptor) { - bb_mvin_group((uintptr_t)&descriptor, writeBank, 0, sizeof(descriptor) / 16, - 1); - run_kernel(readBank, programBank, writeBank, 0); -} -void receive(float *output, uint32_t count, float *staging) { - bb_mvout_group((uintptr_t)(count % 4 ? staging : output), writeBank, 1, - (count + 3) / 4, 1); - alignas(16) kernel_launch descriptor; - bb_mvout_group((uintptr_t)&descriptor, writeBank, 0, sizeof(descriptor) / 16, - 1); - if (count % 4) - std::memcpy(output, staging, count * sizeof(float)); - asm volatile("csrs fflags, %0" ::"r"(descriptor.reserved) : "memory"); -} -void release() { - bb_mem_release(readBank); - bb_mem_release(writeBank); - release_kernel(programBank); -} -uint32_t roundingMode() { - uint32_t rounding; - asm volatile("csrr %0, frm" : "=r"(rounding)::"memory"); - return rounding << 5; -} -} // namespace - -static void layernorm(UnrankedMemRefType *output, - UnrankedMemRefType *input, - UnrankedMemRefType *weight, - UnrankedMemRefType *bias, uint32_t multiplier, - uint32_t epsilon) { - DynamicMemRefType out(*output), in(*input), w(*weight); - const int64_t count = elements(in); - const int64_t width = in.sizes[in.rank - 1]; - if (elements(out) != count || elements(w) != width || - width > bankBytes / sizeof(float)) { - fputs("RVV LayerNorm requires one complete row and weights in banks\n", - stderr); - abort(); - } - alignas(16) float staging[bankBytes / sizeof(float)]; - const uint32_t rounding = roundingMode(); - load(images::layernorm); - send(w.data + w.offset, weightBank, width, staging); - if (bias) { - DynamicMemRefType b(*bias); - if (elements(b) != width) - abort(); - send(b.data + b.offset, biasBank, width, staging); - } - for (int64_t begin = 0; begin < count; begin += width) { - send(in.data + in.offset + begin, inputBank, width, staging); - alignas(16) kernel_launch call{ - images::layernorm.entry, - images::layernorm.text_bytes, - 0x40002000, - {uint64_t(outputBank) << 32, uint64_t(inputBank) << 32, - uint64_t(weightBank) << 32, uint64_t(biasBank) << 32, - uint64_t(width) | (uint64_t(bias != nullptr) << 32), - (uint64_t(epsilon) << 32) | multiplier, - descriptorAddress + offsetof(kernel_launch, reserved), rounding}, - 0}; - launch(call); - receive(out.data + out.offset + begin, width, staging); - } - release(); -} - -extern "C" void _mlir_ciface_rvv_layernorm(UnrankedMemRefType *output, - UnrankedMemRefType *input, - UnrankedMemRefType *weight, - UnrankedMemRefType *bias, - uint32_t multiplier, - uint32_t epsilon) { - layernorm(output, input, weight, bias, multiplier, epsilon); -} - -extern "C" void _mlir_ciface_rvv_layernorm_no_bias( - UnrankedMemRefType *output, UnrankedMemRefType *input, - UnrankedMemRefType *weight, uint32_t multiplier, uint32_t epsilon) { - layernorm(output, input, weight, nullptr, multiplier, epsilon); -} - -extern "C" void _mlir_ciface_rvv_gelu(UnrankedMemRefType *output, - UnrankedMemRefType *input) { - DynamicMemRefType out(*output), in(*input); - const int64_t count = elements(out); - int64_t inputCount = 1; - bool contiguous = true; - for (int64_t axis = in.rank - 1; axis >= 0; --axis) { - if (in.sizes[axis] != 1 && in.strides[axis] != inputCount) - contiguous = false; - inputCount *= in.sizes[axis]; - } - if (inputCount != count) - abort(); - alignas(16) float staging[bankBytes / sizeof(float)]; - const uint32_t rounding = roundingMode(); - load(images::gelu); - for (int64_t begin = 0; begin < count;) { - const uint32_t length = - std::min(bankBytes / sizeof(float), count - begin); - if (contiguous) { - send(in.data + in.offset + begin, inputBank, length, staging); - } else { - for (uint32_t index = 0; index < length; ++index) { - int64_t remaining = begin + index, offset = in.offset; - for (int64_t axis = in.rank - 1; axis >= 0; --axis) { - offset += (remaining % in.sizes[axis]) * in.strides[axis]; - remaining /= in.sizes[axis]; - } - staging[index] = in.data[offset]; - } - std::fill_n(staging + length, (4 - length % 4) % 4, 0.0f); - bb_mvin_group((uintptr_t)staging, readBank, inputBank, (length + 3) / 4, - 1); - } - alignas(16) kernel_launch call{ - images::gelu.entry, - images::gelu.text_bytes, - 0x40002000, - {uint64_t(outputBank) << 32, uint64_t(inputBank) << 32, length, - descriptorAddress + offsetof(kernel_launch, reserved), rounding, 0, 0, - 0}, - 0}; - launch(call); - receive(out.data + out.offset + begin, length, staging); - begin += length; - } - release(); -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/pack.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/pack.cpp deleted file mode 100644 index 1d4161321..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/pack.cpp +++ /dev/null @@ -1,19 +0,0 @@ -#include -#include -#include - -extern "C" void rvv_pack(uint32_t *output, const uint32_t *input, size_t fullK, - size_t fullN, size_t startK, size_t startN, - size_t countK, size_t columns) { - for (size_t column = 0; column < columns; ++column) - for (size_t offset = 0; offset < countK;) { - const size_t vl = __riscv_vsetvl_e32m1(countK - offset); - auto value = __riscv_vmv_v_x_u32m1(0, vl); - if (startN + column < fullN) - value = __riscv_vlse32_v_u32m1(input + (startK + offset) * fullN + - startN + column, - fullN * sizeof(uint32_t), vl); - __riscv_vse32_v_u32m1(output + column * countK + offset, value, vl); - offset += vl; - } -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/pointwise.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/pointwise.cpp deleted file mode 100644 index e8d1b99b2..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/pointwise.cpp +++ /dev/null @@ -1,52 +0,0 @@ -#include "kernels.h" -#include - -extern "C" void rvv_pointwise(float *output, const float *lhs, const float *rhs, - size_t count, uint32_t operation, uint32_t *flags, - uint32_t rounding) { - asm volatile("csrw fcsr, %0" ::"r"(rounding) : "memory"); - for (size_t first = 0; first < count;) { - size_t vl = __riscv_vsetvl_e32m1(count - first); - auto a = __riscv_vle32_v_f32m1(lhs + first, vl); - vfloat32m1_t result; - if (operation == 4) { - result = __riscv_vfneg_v_f32m1(a, vl); - } else if (operation == 5) { - result = __riscv_vfabs_v_f32m1(a, vl); - } else { - auto b = __riscv_vle32_v_f32m1(rhs + first, vl); - switch (operation) { - case 0: - result = __riscv_vfadd_vv_f32m1(a, b, vl); - break; - case 1: - result = __riscv_vfsub_vv_f32m1(a, b, vl); - break; - case 2: - result = __riscv_vfmul_vv_f32m1(a, b, vl); - break; - case 3: - result = __riscv_vfdiv_vv_f32m1(a, b, vl); - break; - case 6: - case 7: { - result = operation == 6 ? __riscv_vfmax_vv_f32m1(a, b, vl) - : __riscv_vfmin_vv_f32m1(a, b, vl); - auto nan = - __riscv_vmor_mm_b32(__riscv_vmfne_vv_f32m1_b32(a, a, vl), - __riscv_vmfne_vv_f32m1_b32(b, b, vl), vl); - auto quiet = __riscv_vfadd_vv_f32m1_m(nan, a, b, vl); - result = __riscv_vmerge_vvm_f32m1(result, quiet, nan, vl); - break; - } - default: - __builtin_trap(); - } - } - __riscv_vse32_v_f32m1(output + first, result, vl); - first += vl; - } - uint32_t status; - asm volatile("csrr %0, fflags" : "=r"(status)::"memory"); - *flags = status; -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/pointwise_dispatch.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/pointwise_dispatch.cpp deleted file mode 100644 index ee31ac579..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/pointwise_dispatch.cpp +++ /dev/null @@ -1,145 +0,0 @@ -#include "images.h" -#include -#include -#include -#include -#include -#include - -namespace { -constexpr unsigned readBank = 4, writeBank = 5; -constexpr unsigned programBank = VIRTUAL_BANK_NUM; -constexpr size_t bankBytes = BANK_LINES * (BANK_WIDTH / 8); -constexpr size_t chunk = bankBytes / sizeof(float); - -bool linear(const DynamicMemRefType &value, - const DynamicMemRefType &shape) { - if (value.rank != shape.rank) - return false; - int64_t stride = 1; - for (int64_t axis = value.rank - 1; axis >= 0; --axis) { - if (value.sizes[axis] != shape.sizes[axis] || - (value.sizes[axis] != 1 && value.strides[axis] != stride)) - return false; - stride *= value.sizes[axis]; - } - return true; -} - -int64_t offset(const DynamicMemRefType &value, - const DynamicMemRefType &shape, int64_t index) { - int64_t result = value.offset; - for (int64_t axis = shape.rank - 1; axis >= 0; --axis) { - int64_t coordinate = index % shape.sizes[axis]; - index /= shape.sizes[axis]; - int64_t inputAxis = axis - (shape.rank - value.rank); - if (inputAxis >= 0 && value.sizes[inputAxis] != 1) - result += coordinate * value.strides[inputAxis]; - } - return result; -} - -void run(UnrankedMemRefType *output, UnrankedMemRefType *lhs, - UnrankedMemRefType *rhs, uint32_t operation) { - DynamicMemRefType out(*output), a(*lhs); - int64_t count = 1; - for (int64_t axis = 0; axis < out.rank; ++axis) { - if (out.sizes[axis] <= 0) - abort(); - count *= out.sizes[axis]; - } - for (auto *input : {lhs, rhs}) { - if (!input) - continue; - DynamicMemRefType value(*input); - if (value.rank > out.rank) - abort(); - for (int64_t axis = 0; axis < value.rank; ++axis) - if (value.sizes[axis] != 1 && - value.sizes[axis] != out.sizes[axis + out.rank - value.rank]) - abort(); - } - alignas(16) float aTile[chunk], bTile[chunk], result[chunk]; - const bool outputLinear = linear(out, out); - uint32_t rounding; - asm volatile("csrr %0, frm" : "=r"(rounding)::"memory"); - for (unsigned bank = 0; bank < 4; ++bank) - bb_mem_alloc(bank, 1, 1); - bb_mem_transfer(2, readBank); - bb_mem_transfer(3, readBank); - bb_mem_transfer(0, writeBank); - bb_mem_transfer(1, writeBank); - mvin_kernel(images::pointwise.bytes, images::pointwise.size, programBank); - for (int64_t first = 0; first < count; first += chunk) { - uint32_t length = std::min(chunk, count - first); - auto pack = [&](const DynamicMemRefType &value, float *staging) { - if (linear(value, out)) { - const float *source = value.data + value.offset + first; - if (length % 4 == 0) - return source; - std::memcpy(staging, source, length * sizeof(float)); - } else { - bool scalar = true; - for (int64_t axis = 0; axis < value.rank; ++axis) - scalar &= value.sizes[axis] == 1; - if (scalar) - std::fill_n(staging, length, value.data[value.offset]); - else - for (uint32_t index = 0; index < length; ++index) - staging[index] = value.data[offset(value, out, first + index)]; - } - std::fill(staging + length, staging + (length + 3) / 4 * 4, 0.0f); - return static_cast(staging); - }; - bb_mvin_group((uintptr_t)pack(a, aTile), readBank, 0, (length + 3) / 4, 1); - if (rhs) { - DynamicMemRefType b(*rhs); - bb_mvin_group((uintptr_t)pack(b, bTile), readBank, 1, (length + 3) / 4, - 1); - } - alignas(16) kernel_launch call{ - images::pointwise.entry, - images::pointwise.text_bytes, - 0x40002000, - {uint64_t(3) << 32, 0, uint64_t(1) << 32, length, operation, - (uint64_t(2) << 32) + offsetof(kernel_launch, reserved), - rounding << 5}, - 0}; - bb_mvin_group((uintptr_t)&call, writeBank, 0, sizeof(call) / 16, 1); - run_kernel(readBank, programBank, writeBank, 0); - float *destination = outputLinear && length % 4 == 0 - ? out.data + out.offset + first - : result; - bb_mvout_group((uintptr_t)destination, writeBank, 1, (length + 3) / 4, 1); - bb_mvout_group((uintptr_t)&call, writeBank, 0, sizeof(call) / 16, 1); - asm volatile("csrs fflags, %0" ::"r"(call.reserved) : "memory"); - if (destination == result) { - if (outputLinear) - std::memcpy(out.data + out.offset + first, result, - length * sizeof(float)); - else - for (uint32_t index = 0; index < length; ++index) - out.data[offset(out, out, first + index)] = result[index]; - } - } - bb_mem_release(readBank); - bb_mem_release(writeBank); - release_kernel(programBank); -} -} // namespace - -extern "C" void _mlir_ciface_rvv_binary(UnrankedMemRefType *output, - UnrankedMemRefType *lhs, - UnrankedMemRefType *rhs, - uint32_t operation) { - if (operation > 7 || operation == 4 || operation == 5) - abort(); - run(output, lhs, rhs, operation); -} -extern "C" void _mlir_ciface_rvv_unary(UnrankedMemRefType *output, - UnrankedMemRefType *input, - uint32_t operation) { - if (operation < 4 || operation > 5) - abort(); - run(output, input, nullptr, operation); -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/quant.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/quant.cpp deleted file mode 100644 index 180c2540a..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/quant.cpp +++ /dev/null @@ -1,131 +0,0 @@ -#include "kernels.h" -#include -#include - -template -static void quant(uint8_t *output, uint8_t *scales, const uint32_t *input, - size_t count, uint32_t *status) { - *status = 0; - for (size_t block = 0; block < count / 32; ++block) { - uint32_t maximum = 0; - for (size_t begin = 0; begin < 32;) { - size_t vl = __riscv_vsetvl_e32m1(32 - begin); - auto bits = __riscv_vle32_v_u32m1(input + block * 32 + begin, vl); - auto magnitude = __riscv_vand_vx_u32m1(bits, 0x7fffffff, vl); - auto seed = __riscv_vmv_v_x_u32m1(maximum, vl); - maximum = __riscv_vmv_x_s_u32m1_u32( - __riscv_vredmaxu_vs_u32m1_u32m1(magnitude, seed, vl)); - begin += vl; - } - if (maximum >= 0x7f800000) { - *status = 1; - return; - } - int blockExponent = maximum == 0 ? 0 : int(maximum >> 23) - 135; - if constexpr (bounded) { - if ((maximum & 0x7fffff) > 0x600000) - ++blockExponent; - if (blockExponent > 119) - blockExponent = 119; - } - if (blockExponent < -127) - blockExponent = -127; - scales[block] = uint8_t(blockExponent + 127); - for (size_t begin = 0; begin < 32;) { - size_t vl = __riscv_vsetvl_e32m1(32 - begin); - auto bits = __riscv_vle32_v_u32m1(input + block * 32 + begin, vl); - auto sign = - __riscv_vand_vx_u32m1(__riscv_vsrl_vx_u32m1(bits, 24, vl), 128, vl); - auto fraction = __riscv_vand_vx_u32m1(bits, 0x7fffff, vl); - auto exponent = __riscv_vreinterpret_v_u32m1_i32m1( - __riscv_vand_vx_u32m1(__riscv_vsrl_vx_u32m1(bits, 23, vl), 255, vl)); - auto subnormal = - __riscv_vmand_mm_b32(__riscv_vmseq_vx_i32m1_b32(exponent, 0, vl), - __riscv_vmsne_vx_u32m1_b32(fraction, 0, vl), vl); - if (__riscv_vcpop_m_b32(subnormal, vl)) { - auto normalized = fraction; - auto shift = __riscv_vmv_v_x_u32m1(0, vl); - for (unsigned step = 16; step; step >>= 1) { - auto selected = - __riscv_vmsltu_vx_u32m1_b32(normalized, 1U << (24 - step), vl); - auto moved = __riscv_vsll_vx_u32m1(normalized, step, vl); - normalized = - __riscv_vmerge_vvm_u32m1(normalized, moved, selected, vl); - auto increment = __riscv_vadd_vx_u32m1(shift, step, vl); - shift = __riscv_vmerge_vvm_u32m1(shift, increment, selected, vl); - } - fraction = __riscv_vmerge_vvm_u32m1( - fraction, __riscv_vand_vx_u32m1(normalized, 0x7fffff, vl), - subnormal, vl); - auto adjusted = __riscv_vrsub_vx_i32m1( - __riscv_vreinterpret_v_u32m1_i32m1(shift), 1, vl); - exponent = __riscv_vmerge_vvm_i32m1(exponent, adjusted, subnormal, vl); - } - exponent = __riscv_vsub_vx_i32m1(exponent, blockExponent + 120, vl); - auto underflow = __riscv_vmsle_vx_i32m1_b32(exponent, 0, vl); - vuint32m1_t code; - if (__riscv_vcpop_m_b32(underflow, vl) == 0) { - auto parity = __riscv_vand_vx_u32m1( - __riscv_vsrl_vx_u32m1(fraction, 20, vl), 1, vl); - auto rounded = __riscv_vsrl_vx_u32m1( - __riscv_vadd_vv_u32m1(__riscv_vadd_vx_u32m1(fraction, 0x7ffff, vl), - parity, vl), - 20, vl); - code = - __riscv_vadd_vv_u32m1(rounded, - __riscv_vreinterpret_v_i32m1_u32m1( - __riscv_vsll_vx_i32m1(exponent, 3, vl)), - vl); - } else { - auto shift = __riscv_vmv_v_x_u32m1(20, vl); - auto underShift = __riscv_vreinterpret_v_i32m1_u32m1( - __riscv_vrsub_vx_i32m1(exponent, 21, vl)); - shift = __riscv_vmerge_vvm_u32m1(shift, underShift, underflow, vl); - auto tooSmall = __riscv_vmsgtu_vx_u32m1_b32(shift, 31, vl); - shift = __riscv_vminu_vx_u32m1(shift, 31, vl); - auto value = __riscv_vmerge_vvm_u32m1( - fraction, __riscv_vor_vx_u32m1(fraction, 0x800000, vl), underflow, - vl); - auto half = - __riscv_vsll_vv_u32m1(__riscv_vmv_v_x_u32m1(1, vl), - __riscv_vsub_vx_u32m1(shift, 1, vl), vl); - auto parity = __riscv_vand_vx_u32m1( - __riscv_vsrl_vv_u32m1(value, shift, vl), 1, vl); - auto rounded = __riscv_vsrl_vv_u32m1( - __riscv_vadd_vv_u32m1( - __riscv_vsub_vx_u32m1(__riscv_vadd_vv_u32m1(value, half, vl), 1, - vl), - parity, vl), - shift, vl); - rounded = __riscv_vmerge_vxm_u32m1(rounded, 0, tooSmall, vl); - auto normal = - __riscv_vadd_vv_u32m1(rounded, - __riscv_vreinterpret_v_i32m1_u32m1( - __riscv_vsll_vx_i32m1(exponent, 3, vl)), - vl); - code = __riscv_vmerge_vvm_u32m1(normal, rounded, underflow, vl); - } - code = __riscv_vminu_vx_u32m1(code, 126, vl); - auto zero = __riscv_vmseq_vx_u32m1_b32( - __riscv_vand_vx_u32m1(bits, 0x7fffffff, vl), 0, vl); - code = __riscv_vmerge_vxm_u32m1(code, 0, zero, vl); - code = __riscv_vor_vv_u32m1(code, sign, vl); - auto narrow = __riscv_vnsrl_wx_u16mf2(code, 0, vl); - __riscv_vse8_v_u8mf4(output + block * 32 + begin, - __riscv_vnsrl_wx_u8mf4(narrow, 0, vl), vl); - begin += vl; - } - } -} - -extern "C" void rvv_quant(uint8_t *output, uint8_t *scales, - const uint32_t *input, size_t count, - uint32_t *status) { - quant(output, scales, input, count, status); -} - -extern "C" void rvv_cache_quant(uint8_t *output, uint8_t *scales, - const uint32_t *input, size_t count, - uint32_t *status) { - quant(output, scales, input, count, status); -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/quant_dispatch.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/quant_dispatch.cpp deleted file mode 100644 index 682fe9238..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/quant_dispatch.cpp +++ /dev/null @@ -1,41 +0,0 @@ -#include "../mxfp8/quant.h" -#include "images.h" -#include -#include -#include -#include - -extern "C" void mxfp8_quant(const uint32_t *input, uint8_t *output, - uint32_t count) { - constexpr unsigned readBank = 4, writeBank = 5; - constexpr unsigned programBank = VIRTUAL_BANK_NUM; - for (unsigned bank = 0; bank < 3; ++bank) - bb_mem_alloc(bank, 1, 1); - bb_mem_transfer(2, readBank); - bb_mem_transfer(0, writeBank); - bb_mem_transfer(1, writeBank); - mvin_kernel(images::quant.bytes, images::quant.size, programBank); - bb_mvin_group((uintptr_t)input, readBank, 0, count / 4, 1); - alignas(16) uint8_t descriptor[sizeof(kernel_launch)]{}; - kernel_launch call{images::quant.entry, - images::quant.text_bytes, - 0x40002000, - {uint64_t(2) << 32, (uint64_t(2) << 32) + count, 0, count, - (uint64_t(1) << 32) + offsetof(kernel_launch, reserved)}, - 0}; - std::memcpy(descriptor, &call, sizeof(call)); - bb_mvin_group((uintptr_t)descriptor, writeBank, 0, sizeof(descriptor) / 16, - 1); - run_kernel(readBank, programBank, writeBank, 0); - bb_mvout_group((uintptr_t)descriptor, writeBank, 0, sizeof(descriptor) / 16, - 1); - bb_mvout_group((uintptr_t)output, writeBank, 1, - (count + count / 32 + 15) / 16, 1); - if (descriptor[offsetof(kernel_launch, reserved)]) { - fputs("mxfp8: non-finite activation\n", stderr); - abort(); - } - bb_mem_release(readBank); - bb_mem_release(writeBank); - release_kernel(programBank); -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/rope.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/rope.cpp deleted file mode 100644 index d3e58ee3a..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/rope.cpp +++ /dev/null @@ -1,72 +0,0 @@ -#include "kernels.h" -#include "math/math.h" - -extern "C" void rvv_rope(float *output, const float *input, - const float *frequencies, const int32_t *positions, - size_t rows, size_t headDimAndHeads) { - const size_t headDim = headDimAndHeads & 65535; - const size_t heads = (headDimAndHeads >> 16) & 65535; - const uint32_t scaleBits = headDimAndHeads >> 32; - const float scale = __riscv_vfmv_f_s_f32m1_f32( - __riscv_vreinterpret_f32m1(__riscv_vmv_v_x_u32m1(scaleBits, 1))); - const size_t half = headDim / 2; - for (size_t row = 0; row < rows; ++row) { - auto integer = __riscv_vle32_v_i32m1(positions + row, 1); - auto converted = __riscv_vfcvt_f_x_v_f32m1(integer, 1); - const float position = __riscv_vfmv_f_s_f32m1_f32(converted); - for (size_t offset = 0; offset < half;) { - const size_t vl = __riscv_vsetvl_e32m1(half - offset); - auto frequency = __riscv_vle32_v_f32m1(frequencies + offset, vl); - auto angle = __riscv_vfmul_vf_f32m1(frequency, position, vl); - angle = __riscv_vfadd_vf_f32m1(angle, 0, vl); - auto cosine = rvv_cos(angle, vl); - cosine = __riscv_vfmul_vf_f32m1(cosine, scale, vl); - for (size_t head = 0; head < heads; ++head) { - const float *source = input + (head * rows + row) * headDim; - float *destination = output + (head * rows + row) * headDim; - auto lower = __riscv_vle32_v_f32m1(source + offset, vl); - auto upper = __riscv_vle32_v_f32m1(source + half + offset, vl); - __riscv_vse32_v_f32m1(destination + offset, - __riscv_vfmul_vv_f32m1(lower, cosine, vl), vl); - __riscv_vse32_v_f32m1(destination + half + offset, - __riscv_vfmul_vv_f32m1(upper, cosine, vl), vl); - } - frequency = __riscv_vle32_v_f32m1(frequencies + offset, vl); - angle = __riscv_vfmul_vf_f32m1(frequency, position, vl); - angle = __riscv_vfadd_vf_f32m1(angle, 0, vl); - auto sine = rvv_sin(angle, vl); - sine = __riscv_vfmul_vf_f32m1(sine, scale, vl); - for (size_t head = 0; head < heads; ++head) { - const float *source = input + (head * rows + row) * headDim; - float *destination = output + (head * rows + row) * headDim; - auto lower = __riscv_vle32_v_f32m1(source + offset, vl); - auto upper = __riscv_vle32_v_f32m1(source + half + offset, vl); - auto rotatedLower = - __riscv_vfmul_vv_f32m1(__riscv_vfneg_v_f32m1(upper, vl), sine, vl); - auto rotatedUpper = __riscv_vfmul_vv_f32m1(lower, sine, vl); - auto lowerProduct = __riscv_vle32_v_f32m1(destination + offset, vl); - auto upperProduct = - __riscv_vle32_v_f32m1(destination + half + offset, vl); - __riscv_vse32_v_f32m1( - destination + offset, - __riscv_vfadd_vv_f32m1(lowerProduct, rotatedLower, vl), vl); - __riscv_vse32_v_f32m1( - destination + half + offset, - __riscv_vfadd_vv_f32m1(upperProduct, rotatedUpper, vl), vl); - } - offset += vl; - } - } -} - -extern "C" void rvv_rope_launch(float *output, const float *input, - const float *frequencies, - const int32_t *positions, size_t rows, - size_t headDimAndHeads, uint32_t *flags, - uint32_t rounding) { - asm volatile("csrw fcsr, %0" ::"r"(rounding) : "memory"); - rvv_rope(output, input, frequencies, positions, rows, headDimAndHeads); - uint32_t result; - asm volatile("csrr %0, fflags" : "=r"(result)::"memory"); - *flags = result; -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/silu.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/silu.cpp deleted file mode 100644 index 6c5fe1795..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/silu.cpp +++ /dev/null @@ -1,50 +0,0 @@ -#include "kernels.h" -#include "math/math.h" - -extern "C" void rvv_silu(float *output, const float *input, size_t count) { - for (size_t offset = 0; offset < count;) { - const size_t vl = __riscv_vsetvl_e32m1(count - offset); - auto value = __riscv_vle32_v_f32m1(input + offset, vl); - auto denominator = __riscv_vfadd_vf_f32m1( - rvv_exp(__riscv_vfneg_v_f32m1(value, vl), vl), 1, vl); - auto sigmoid = __riscv_vfrdiv_vf_f32m1(denominator, 1, vl); - auto result = __riscv_vfmul_vv_f32m1(value, sigmoid, vl); - __riscv_vse32_v_f32m1(output + offset, result, vl); - offset += vl; - } -} - -extern "C" void rvv_swiglu(float *output, const float *gate, const float *up, - size_t count) { - for (size_t offset = 0; offset < count;) { - const size_t vl = __riscv_vsetvl_e32m1(count - offset); - auto value = __riscv_vle32_v_f32m1(gate + offset, vl); - auto other = __riscv_vle32_v_f32m1(up + offset, vl); - auto denominator = __riscv_vfadd_vf_f32m1( - rvv_exp(__riscv_vfneg_v_f32m1(value, vl), vl), 1, vl); - auto sigmoid = __riscv_vfrdiv_vf_f32m1(denominator, 1, vl); - auto result = __riscv_vfmul_vv_f32m1( - __riscv_vfmul_vv_f32m1(value, sigmoid, vl), other, vl); - __riscv_vse32_v_f32m1(output + offset, result, vl); - offset += vl; - } -} - -extern "C" void rvv_silu_launch(float *output, const float *input, size_t count, - uint32_t *flags, uint32_t rounding) { - asm volatile("csrw fcsr, %0" ::"r"(rounding) : "memory"); - rvv_silu(output, input, count); - uint32_t result; - asm volatile("csrr %0, fflags" : "=r"(result)::"memory"); - *flags = result; -} - -extern "C" void rvv_swiglu_launch(float *output, const float *gate, - const float *up, size_t count, - uint32_t *flags, uint32_t rounding) { - asm volatile("csrw fcsr, %0" ::"r"(rounding) : "memory"); - rvv_swiglu(output, gate, up, count); - uint32_t result; - asm volatile("csrr %0, fflags" : "=r"(result)::"memory"); - *flags = result; -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/snake.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/snake.cpp deleted file mode 100644 index 49d3b193a..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/snake.cpp +++ /dev/null @@ -1,27 +0,0 @@ -#include "kernels.h" -#include "math/math.h" - -extern "C" void rvv_snake(float *output, const float *input, size_t channels, - size_t length, const float *log_alpha, - const float *log_beta) { - for (size_t channel = 0; channel < channels; ++channel) { - auto alpha = rvv_exp(__riscv_vfmv_v_f_f32m1(log_alpha[channel], 1), 1); - auto beta = __riscv_vfadd_vf_f32m1( - rvv_exp(__riscv_vfmv_v_f_f32m1(log_beta[channel], 1), 1), 1e-9f, 1); - auto reciprocal = __riscv_vfrdiv_vf_f32m1(beta, 1, 1); - const float scale = __riscv_vfmv_f_s_f32m1_f32(alpha); - const float inverse = __riscv_vfmv_f_s_f32m1_f32(reciprocal); - for (size_t offset = 0; offset < length;) { - const size_t vl = __riscv_vsetvl_e32m1(length - offset); - const size_t index = channel * length + offset; - auto value = __riscv_vle32_v_f32m1(input + index, vl); - auto angle = __riscv_vfmul_vf_f32m1(value, scale, vl); - auto sine = rvv_sin(angle, vl); - auto square = __riscv_vfmul_vv_f32m1(sine, sine, vl); - auto scaled = __riscv_vfmul_vf_f32m1(square, inverse, vl); - auto result = __riscv_vfadd_vv_f32m1(value, scaled, vl); - __riscv_vse32_v_f32m1(output + index, result, vl); - offset += vl; - } - } -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/softmax.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/softmax.cpp deleted file mode 100644 index f859cf04e..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/softmax.cpp +++ /dev/null @@ -1,183 +0,0 @@ -#include "kernels.h" -#include "math/math.h" - -extern "C" void rvv_softmax(float *output, const float *input, size_t rows, - size_t width) { - for (size_t row = 0; row < rows; ++row) { - const float *source = input + row * width; - float *destination = output + row * width; - auto maximum = __riscv_vfmv_v_f_f32m1(-0x1.fffffep127f, 1); - bool hasNaN = false; - for (size_t offset = 0; offset < width;) { - const size_t vl = __riscv_vsetvl_e32m1(width - offset); - auto values = __riscv_vle32_v_f32m1(source + offset, vl); - auto magnitude = __riscv_vand_vx_u32m1(__riscv_vreinterpret_u32m1(values), - 0x7fffffff, vl); - hasNaN |= - __riscv_vcpop_m_b32( - __riscv_vmsgtu_vx_u32m1_b32(magnitude, 0x7f800000, vl), vl) != 0; - maximum = __riscv_vfredmax_vs_f32m1_f32m1(values, maximum, vl); - offset += vl; - } - if (hasNaN) - maximum = - __riscv_vreinterpret_f32m1(__riscv_vmv_v_x_u32m1(0x7fc00000, 1)); - const float peak = __riscv_vfmv_f_s_f32m1_f32(maximum); - auto sum = __riscv_vfmv_v_f_f32m1(0, 1); - for (size_t offset = 0; offset < width;) { - const size_t vl = __riscv_vsetvl_e32m1(width - offset); - auto values = __riscv_vle32_v_f32m1(source + offset, vl); - auto centered = __riscv_vfsub_vf_f32m1(values, peak, vl); - auto exponential = rvv_exp(centered, vl); - __riscv_vse32_v_f32m1(destination + offset, exponential, vl); - sum = __riscv_vfredosum_vs_f32m1_f32m1(exponential, sum, vl); - offset += vl; - } - auto reciprocal = __riscv_vfrdiv_vf_f32m1(sum, 1, 1); - const float scale = __riscv_vfmv_f_s_f32m1_f32(reciprocal); - for (size_t offset = 0; offset < width;) { - const size_t vl = __riscv_vsetvl_e32m1(width - offset); - auto values = __riscv_vle32_v_f32m1(destination + offset, vl); - auto result = __riscv_vfmul_vf_f32m1(values, scale, vl); - __riscv_vse32_v_f32m1(destination + offset, result, vl); - offset += vl; - } - } -} - -extern "C" void rvv_softmax_launch(float *output, const float *input, - size_t rows, size_t width, uint32_t *flags, - uint32_t rounding) { - asm volatile("csrw fcsr, %0" ::"r"(rounding) : "memory"); - rvv_softmax(output, input, rows, width); - uint32_t result; - asm volatile("csrr %0, fflags" : "=r"(result)::"memory"); - *flags = result; -} - -extern "C" void rvv_attention_softmax_launch(float *output, const float *input, - const uint8_t *mask, size_t rows, - size_t width, - const uint32_t *parameters, - uint32_t *flags, - uint32_t rounding) { - asm volatile("csrw fcsr, %0" ::"r"(rounding) : "memory"); - union { - uint32_t bits; - float value; - } scale{parameters[0]}, masked{parameters[1]}; - for (size_t row = 0; row < rows; ++row) - for (size_t offset = 0; offset < width;) { - const size_t vl = __riscv_vsetvl_e32m1(width - offset); - auto values = __riscv_vle32_v_f32m1(input + row * width + offset, vl); - auto scaled = __riscv_vfmul_vf_f32m1(values, scale.value, vl); - auto bytes = __riscv_vle8_v_u8mf4( - mask + ((row + parameters[3]) % parameters[2]) * width + offset, vl); - auto selected = __riscv_vmsne_vx_u8mf4_b32(bytes, 0, vl); - auto result = - __riscv_vfmerge_vfm_f32m1(scaled, masked.value, selected, vl); - __riscv_vse32_v_f32m1(output + row * width + offset, result, vl); - offset += vl; - } - rvv_softmax(output, output, rows, width); - uint32_t result; - asm volatile("csrr %0, fflags" : "=r"(result)::"memory"); - *flags = result; -} - -// Softmax sums are positive and finite for finite scores. Preserve IEEE special -// values when the input row contains infinities or NaNs. -static vfloat32m1_t logSum(vfloat32m1_t value) { - constexpr size_t vl = 1; - auto bits = __riscv_vreinterpret_u32m1(value); - auto fraction = __riscv_vand_vx_u32m1(bits, 0x007fffff, vl); - auto mantissa = __riscv_vreinterpret_f32m1( - __riscv_vor_vx_u32m1(fraction, 0x3f800000, vl)); - auto exponent = __riscv_vsub_vx_i32m1( - __riscv_vreinterpret_i32m1(__riscv_vsrl_vx_u32m1(bits, 23, vl)), 127, vl); - auto upper = __riscv_vmfge_vf_f32m1_b32(mantissa, 0x1.6a09e6p0f, vl); - mantissa = __riscv_vmerge_vvm_f32m1( - mantissa, __riscv_vfmul_vf_f32m1(mantissa, 0.5f, vl), upper, vl); - exponent = __riscv_vmerge_vvm_i32m1( - exponent, __riscv_vadd_vx_i32m1(exponent, 1, vl), upper, vl); - auto numerator = __riscv_vfsub_vf_f32m1(mantissa, 1, vl); - auto denominator = __riscv_vfadd_vf_f32m1(mantissa, 1, vl); - auto t = __riscv_vfdiv_vv_f32m1(numerator, denominator, vl); - auto square = __riscv_vfmul_vv_f32m1(t, t, vl); - auto term = t, series = t; - for (unsigned odd = 3; odd <= 15; odd += 2) { - term = __riscv_vfmul_vv_f32m1(term, square, vl); - series = __riscv_vfadd_vv_f32m1( - series, __riscv_vfdiv_vf_f32m1(term, float(odd), vl), vl); - } - auto e = __riscv_vfcvt_f_x_v_f32m1(exponent, vl); - auto result = - __riscv_vfadd_vv_f32m1(__riscv_vfmul_vf_f32m1(e, 0x1.62e4p-1f, vl), - __riscv_vfmul_vf_f32m1(series, 2, vl), vl); - result = __riscv_vfadd_vv_f32m1( - result, __riscv_vfmul_vf_f32m1(e, 0x1.7f7d1cp-20f, vl), vl); - auto zero = __riscv_vmseq_vx_u32m1_b32(bits, 0, vl); - auto infinity = __riscv_vmseq_vx_u32m1_b32(bits, 0x7f800000, vl); - auto nan = __riscv_vmsgtu_vx_u32m1_b32(bits, 0x7f800000, vl); - auto negativeInfinity = - __riscv_vreinterpret_f32m1(__riscv_vmv_v_x_u32m1(0xff800000, vl)); - auto quietNaN = - __riscv_vreinterpret_f32m1(__riscv_vmv_v_x_u32m1(0x7fc00000, vl)); - result = __riscv_vmerge_vvm_f32m1(result, negativeInfinity, zero, vl); - result = __riscv_vmerge_vvm_f32m1(result, value, infinity, vl); - return __riscv_vmerge_vvm_f32m1(result, quietNaN, nan, vl); -} - -extern "C" void rvv_logsumexp_softmax(float *output, const float *input, - size_t rows, size_t width) { - for (size_t row = 0; row < rows; ++row) { - const float *source = input + row * width; - float *destination = output + row * width; - auto maximum = __riscv_vfmv_v_f_f32m1(-0x1.fffffep127f, 1); - bool hasNaN = false; - for (size_t offset = 0; offset < width;) { - const size_t vl = __riscv_vsetvl_e32m1(width - offset); - auto values = __riscv_vle32_v_f32m1(source + offset, vl); - auto magnitude = __riscv_vand_vx_u32m1(__riscv_vreinterpret_u32m1(values), - 0x7fffffff, vl); - hasNaN |= - __riscv_vcpop_m_b32( - __riscv_vmsgtu_vx_u32m1_b32(magnitude, 0x7f800000, vl), vl) != 0; - maximum = __riscv_vfredmax_vs_f32m1_f32m1(values, maximum, vl); - offset += vl; - } - if (hasNaN) - maximum = - __riscv_vreinterpret_f32m1(__riscv_vmv_v_x_u32m1(0x7fc00000, 1)); - const float peak = __riscv_vfmv_f_s_f32m1_f32(maximum); - auto sum = __riscv_vfmv_v_f_f32m1(0, 1); - for (size_t offset = 0; offset < width;) { - const size_t vl = __riscv_vsetvl_e32m1(width - offset); - auto values = __riscv_vle32_v_f32m1(source + offset, vl); - auto exponential = rvv_exp(__riscv_vfsub_vf_f32m1(values, peak, vl), vl); - sum = __riscv_vfredosum_vs_f32m1_f32m1(exponential, sum, vl); - offset += vl; - } - auto logsum = __riscv_vfadd_vv_f32m1(maximum, logSum(sum), 1); - const float normalization = __riscv_vfmv_f_s_f32m1_f32(logsum); - for (size_t offset = 0; offset < width;) { - const size_t vl = __riscv_vsetvl_e32m1(width - offset); - auto values = __riscv_vle32_v_f32m1(source + offset, vl); - auto result = - rvv_exp(__riscv_vfsub_vf_f32m1(values, normalization, vl), vl); - __riscv_vse32_v_f32m1(destination + offset, result, vl); - offset += vl; - } - } -} - -extern "C" void rvv_logsumexp_softmax_launch(float *output, const float *input, - size_t rows, size_t width, - uint32_t *flags, - uint32_t rounding) { - asm volatile("csrw fcsr, %0" ::"r"(rounding) : "memory"); - rvv_logsumexp_softmax(output, input, rows, width); - uint32_t result; - asm volatile("csrr %0, fflags" : "=r"(result)::"memory"); - *flags = result; -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/tanh.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/tanh.cpp deleted file mode 100644 index e46e5d62d..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/tanh.cpp +++ /dev/null @@ -1,54 +0,0 @@ -#include "math/math.h" -#include - -extern "C" void rvv_tanh_launch(float *output, const float *input, size_t count, - uint32_t *flags, uint32_t rounding) { - asm volatile("csrw fcsr, %0" ::"r"(rounding) : "memory"); - for (size_t offset = 0; offset < count;) { - const size_t vl = __riscv_vsetvl_e32m1(count - offset); - auto values = __riscv_vle32_v_f32m1(input + offset, vl); - auto bits = __riscv_vreinterpret_u32m1(values); - auto magnitude = __riscv_vand_vx_u32m1(bits, 0x7fffffff, vl); - auto finite = __riscv_vmsleu_vx_u32m1_b32(magnitude, 0x7f7fffff, vl); - auto absolute = - __riscv_vmerge_vvm_f32m1(__riscv_vfmv_v_f_f32m1(0, vl), - __riscv_vfabs_v_f32m1(values, vl), finite, vl); - auto saturated = __riscv_vfmin_vf_f32m1(absolute, 10, vl); - auto e = rvv_exp(__riscv_vfmul_vf_f32m1(saturated, -2, vl), vl); - auto numerator = __riscv_vfrsub_vf_f32m1(e, 1, vl); - auto denominator = __riscv_vfadd_vf_f32m1(e, 1, vl); - auto result = __riscv_vfdiv_vv_f32m1(numerator, denominator, vl); - // The odd series avoids exponential cancellation near the origin. - auto bounded = __riscv_vfmin_vf_f32m1(absolute, 0.125f, vl); - auto square = __riscv_vfmul_vv_f32m1(bounded, bounded, vl); - auto polynomial = __riscv_vfadd_vf_f32m1( - __riscv_vfmul_vf_f32m1(square, -17.0f / 315.0f, vl), 2.0f / 15.0f, vl); - polynomial = __riscv_vfadd_vf_f32m1( - __riscv_vfmul_vv_f32m1(polynomial, square, vl), -1.0f / 3.0f, vl); - polynomial = __riscv_vfmul_vv_f32m1(polynomial, square, vl); - auto series = __riscv_vfadd_vv_f32m1( - bounded, __riscv_vfmul_vv_f32m1(polynomial, bounded, vl), vl); - auto small = __riscv_vmfle_vf_f32m1_b32(absolute, 0.125f, vl); - result = __riscv_vmerge_vvm_f32m1(result, series, small, vl); - auto infinity = __riscv_vmseq_vx_u32m1_b32(magnitude, 0x7f800000, vl); - auto nan = __riscv_vmsgtu_vx_u32m1_b32(magnitude, 0x7f800000, vl); - auto quiet = - __riscv_vreinterpret_f32m1(__riscv_vor_vx_u32m1(bits, 0x00400000, vl)); - result = __riscv_vmerge_vvm_f32m1(result, __riscv_vfmv_v_f_f32m1(1, vl), - infinity, vl); - result = __riscv_vmerge_vvm_f32m1(result, quiet, nan, vl); - auto signaling = __riscv_vmand_mm_b32( - nan, - __riscv_vmseq_vx_u32m1_b32(__riscv_vand_vx_u32m1(bits, 0x00400000, vl), - 0, vl), - vl); - if (__riscv_vcpop_m_b32(signaling, vl)) - asm volatile("csrs fflags, %0" ::"r"(uint32_t(16)) : "memory"); - result = __riscv_vfsgnj_vv_f32m1(result, values, vl); - __riscv_vse32_v_f32m1(output + offset, result, vl); - offset += vl; - } - uint32_t result; - asm volatile("csrr %0, fflags" : "=r"(result)::"memory"); - *flags = result; -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/trig.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/trig.cpp deleted file mode 100644 index d4dde4e03..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/trig.cpp +++ /dev/null @@ -1,26 +0,0 @@ -#include "math/math.h" -#include - -template -static void trig(float *output, const float *input, size_t count, - uint32_t *flags, uint32_t rounding) { - asm volatile("csrw fcsr, %0" ::"r"(rounding) : "memory"); - for (size_t offset = 0; offset < count;) { - size_t vl = __riscv_vsetvl_e32m1(count - offset); - auto values = __riscv_vle32_v_f32m1(input + offset, vl); - auto result = sine ? rvv_sin(values, vl) : rvv_cos(values, vl); - __riscv_vse32_v_f32m1(output + offset, result, vl); - offset += vl; - } - uint32_t status; - asm volatile("csrr %0, fflags" : "=r"(status)::"memory"); - *flags = status; -} -extern "C" void rvv_sin_launch(float *output, const float *input, size_t count, - uint32_t *flags, uint32_t rounding) { - trig(output, input, count, flags, rounding); -} -extern "C" void rvv_cos_launch(float *output, const float *input, size_t count, - uint32_t *flags, uint32_t rounding) { - trig(output, input, count, flags, rounding); -} diff --git a/bb-tests/workloads/lib/bbsw/linux/multicore/multicore.h b/bb-tests/workloads/lib/bbsw/linux/multicore/multicore.h deleted file mode 100644 index 1ece5e551..000000000 --- a/bb-tests/workloads/lib/bbsw/linux/multicore/multicore.h +++ /dev/null @@ -1,20 +0,0 @@ -#ifndef BBSW_LINUX_MULTICORE_H -#define BBSW_LINUX_MULTICORE_H - -#include -#include -#include -#include - -static inline uint32_t bb_get_hart_id(void) { - unsigned cpu; - if (syscall(SYS_getcpu, &cpu, 0, 0) != 0) - __builtin_trap(); - return cpu; -} - -static inline core_id_t bb_get_core_id(void) { - return bb_topology_core_id(bb_get_hart_id()); -} - -#endif diff --git a/bb-tests/workloads/lib/bbsw/linux/test.c b/bb-tests/workloads/lib/bbsw/linux/test.c deleted file mode 100644 index 3272fa403..000000000 --- a/bb-tests/workloads/lib/bbsw/linux/test.c +++ /dev/null @@ -1,6 +0,0 @@ -#include -#include - -void bb_test_pass(void) {} - -void bb_test_fail(void) { exit(1); } diff --git a/bb-tests/workloads/lib/bbsw/test.h b/bb-tests/workloads/lib/bbsw/test.h deleted file mode 100644 index de53b0fa7..000000000 --- a/bb-tests/workloads/lib/bbsw/test.h +++ /dev/null @@ -1,15 +0,0 @@ -#ifndef BBSW_TEST_H -#define BBSW_TEST_H - -#ifdef __cplusplus -extern "C" { -#endif - -void bb_test_pass(void); -void bb_test_fail(void) __attribute__((noreturn)); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/bb-tests/workloads/lib/buckyball.h b/bb-tests/workloads/lib/buckyball.h index e3801c438..de78240e8 100644 --- a/bb-tests/workloads/lib/buckyball.h +++ b/bb-tests/workloads/lib/buckyball.h @@ -3,25 +3,11 @@ #include -// String macros (from xcustom.h) -// #define STR1(x) #x -// #ifndef STR -// #define STR(x) STR1(x) -// #endif - -#define CAT_(A, B) A##B -#define CAT(A, B) CAT_(A, B) - // Data type for matrix elements typedef int8_t elem_t; typedef int32_t result_t; -#define MULTICORE_INIT(hart_id) \ - __attribute__((constructor)) static void _multicore_init() { \ - multicore(hart_id); \ - } - -static inline void multicore(int target_hart_id) { +static inline void multithread(int target_hart_id) { int hart_id; asm volatile("csrr %0, mhartid" : "=r"(hart_id)); diff --git a/bb-tests/workloads/lib/kernel b/bb-tests/workloads/lib/kernel deleted file mode 160000 index 9e16330e6..000000000 --- a/bb-tests/workloads/lib/kernel +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 9e16330e6df5154e20b17d7b86891400407fcedb diff --git a/bb-tests/workloads/scripts/build.py b/bb-tests/workloads/scripts/build.py deleted file mode 100644 index fff1a58ec..000000000 --- a/bb-tests/workloads/scripts/build.py +++ /dev/null @@ -1,263 +0,0 @@ -from __future__ import annotations -import argparse -import os -import re -import shlex -import shutil -import subprocess -from pathlib import Path - - -def _repo(raw: str | Path) -> Path: - root = Path(raw).resolve() - if not root.is_dir(): - raise ValueError(f"repo does not exist: {root}") - return root - - -def _run( - cmd: list[str], - *, - cwd: Path, - prefix: str, - env: dict | None = None, - logger: object | None = None, - task_scope: str | None = None, -) -> None: - if logger is None: - result = subprocess.run(cmd, cwd=cwd, env=env) - else: - from utils.stream_run import stream_run_logger - - result = stream_run_logger( - cmd=shlex.join(cmd), - logger=logger, - cwd=str(cwd), - stdout_prefix=prefix, - stderr_prefix=prefix, - task_scope=task_scope, - env=env, - ) - if result.returncode != 0: - raise RuntimeError(f"command failed ({result.returncode}): {' '.join(cmd)}") - - -def _cmake_defs(repo: Path, chip: str) -> dict[str, str]: - path = ( - repo - / "examples" - / "chips" - / chip - / "configs" - / "generated" - / "workload" - / "cmake.defs" - ) - if not path.is_file(): - raise RuntimeError(f"missing {path}; run bbdev config --install") - values: dict[str, str] = {} - for line in path.read_text(encoding="utf-8").splitlines(): - key, sep, value = line.partition("=") - if not sep: - raise RuntimeError(f"bad cmake.defs line in {path}: {line!r}") - if key in values: - raise RuntimeError(f"duplicate cmake.defs key {key} in {path}") - values[key] = value - required = ("BUCKYBALL_WORKLOAD_CHIP", "BUCKYBALL_CHIP_PB") - missing = [k for k in required if k not in values] - if missing: - raise RuntimeError(f"{path} missing {missing}") - return values - - -def _require_riscv() -> Path: - raw = os.environ.get("RISCV", "") - if not raw: - raise RuntimeError("RISCV is unset; enter nix develop") - root = Path(raw) - if not root.is_dir(): - raise RuntimeError(f"RISCV is not a directory: {root}") - return root - - -def _workload_src(repo: Path) -> Path: - src = repo / "bb-tests" / "workloads" - cmake = src / "CMakeLists.txt" - if not cmake.is_file(): - raise RuntimeError(f"missing {cmake}") - return src - - -def _workload_build_dir(repo: Path, instance: str) -> Path: - return repo / "bb-tests" / "workloads" / "build" / instance - - -def build_workload( - repo: str | Path, - chip: str, - *, - instance: str | None = None, - chip_pb: str | Path | None = None, - ctest: bool = False, - mlirtest: bool = False, - soctest: bool = False, - stable: bool = False, - logger: object | None = None, - task_scope: str | None = None, -) -> None: - if not re.fullmatch("[A-Za-z0-9_-]+", chip): - raise ValueError(f"invalid chip: {chip}") - build_instance = instance or chip - if not re.fullmatch("[A-Za-z0-9_-]+", build_instance): - raise ValueError(f"invalid workload build instance: {build_instance}") - if (instance is None) != (chip_pb is None): - raise ValueError("workload variant requires both instance and chip_pb") - if sum((ctest, mlirtest, soctest)) > 1: - raise ValueError("--ctest, --mlirtest and --soctest are mutually exclusive") - root = _repo(repo) - defs = _cmake_defs(root, chip) - if chip_pb is not None: - selected_pb = Path(chip_pb).resolve() - if not selected_pb.is_file(): - raise RuntimeError(f"missing {selected_pb}") - defs["BUCKYBALL_WORKLOAD_CHIP"] = build_instance - defs["BUCKYBALL_WORKLOAD_SOURCE_CHIP"] = chip - defs["BUCKYBALL_CHIP_PB"] = str(selected_pb) - compiler_build = ( - root - / "stack" - / "compiler" - / "thirdparty" - / "buddy-mlir" - / "build" - / build_instance - ) - riscv = _require_riscv() - project_python = riscv / "bin" / "python3" - python = ( - str(project_python) if project_python.is_file() else shutil.which("python3") - ) - if not python: - raise RuntimeError("python3 not in PATH; enter nix develop") - linux_cc = riscv / "bin" / "riscv64-unknown-linux-gnu-gcc" - linux_cxx = riscv / "bin" / "riscv64-unknown-linux-gnu-g++" - if not linux_cc.is_file() or not linux_cxx.is_file(): - raise RuntimeError(f"missing RISC-V linux toolchain under {riscv / 'bin'}") - source_root = _workload_src(root) - build_root = _workload_build_dir(root, build_instance) - env = os.environ.copy() - env["PATH"] = f"{riscv / 'bin'}:{env.get('PATH', '')}" - env["RISCV"] = str(riscv) - env["CC"] = str(linux_cc) - env["CXX"] = str(linux_cxx) - if not (ctest or soctest): - compiler_config = {} - for line in (compiler_build / "CMakeCache.txt").read_text().splitlines(): - key, sep, value = line.partition("=") - name = key.split(":")[0] - if sep and name in ("BUDDY_EXTERNAL_DIALECTS_DIR", "BUCKYBALL_CHIP_PB"): - if name in compiler_config: - raise RuntimeError(f"duplicate {name} in compiler CMakeCache.txt") - compiler_config[name] = value - if ( - Path(compiler_config["BUDDY_EXTERNAL_DIALECTS_DIR"]).resolve() - != root / "stack" / "compiler" - ): - raise RuntimeError("compiler does not use this repository's stack/compiler") - if ( - Path(compiler_config["BUCKYBALL_CHIP_PB"]).resolve() - != Path(defs["BUCKYBALL_CHIP_PB"]).resolve() - ): - raise RuntimeError("compiler Chip.pb does not match the workload Chip.pb") - _run( - [ - "cmake", - "--build", - str(compiler_build), - "--target", - "buddy-opt", - "buddy-translate", - ], - cwd=root, - env=env, - prefix="workload compiler", - logger=logger, - task_scope=task_scope, - ) - src = source_root - build = build_root - target = ( - "sync-ctest-bin" - if ctest - else ( - "sync-mlirtest-bin" - if mlirtest - else "sync-soctest-bin" if soctest else "sync-bin" - ) - ) - build.mkdir(parents=True, exist_ok=True) - cmake_args = [ - "cmake", - "-G", - "Ninja", - "-DCMAKE_BUILD_TYPE=Release", - "-DCMAKE_C_FLAGS_RELEASE=-O2 -UNDEBUG", - "-DCMAKE_CXX_FLAGS_RELEASE=-O2 -UNDEBUG", - "-S", - str(src), - "-B", - str(build), - f"-DPython3_EXECUTABLE={python}", - f"-DCMAKE_C_COMPILER={linux_cc}", - f"-DCMAKE_CXX_COMPILER={linux_cxx}", - ] - if not (ctest or soctest): - cmake_args.append(f"-DBUCKYBALL_STABLE={'ON' if stable else 'OFF'}") - for key, value in defs.items(): - cmake_args.append(f"-D{key}={value}") - _run( - cmake_args, - cwd=root, - env=env, - prefix="workload configure", - logger=logger, - task_scope=task_scope, - ) - ninja = ["ninja", "-C", str(build), f"-j{1}"] - ninja.append(target) - _run( - ninja, - cwd=root, - env=env, - prefix="workload build", - logger=logger, - task_scope=task_scope, - ) - - -def main() -> None: - parser = argparse.ArgumentParser(description="Build chip workloads") - parser.add_argument("--repo", type=Path, required=True) - parser.add_argument("--chip", required=True) - parser.add_argument("--instance") - parser.add_argument("--chip-pb", type=Path) - scope = parser.add_mutually_exclusive_group() - scope.add_argument("--ctest", action="store_true") - scope.add_argument("--mlirtest", action="store_true") - scope.add_argument("--soctest", action="store_true") - parser.add_argument("--stable", action="store_true") - args = parser.parse_args() - build_workload( - args.repo, - args.chip, - instance=args.instance, - chip_pb=args.chip_pb, - ctest=args.ctest, - mlirtest=args.mlirtest, - soctest=args.soctest, - stable=args.stable, - ) - - -if __name__ == "__main__": - main() diff --git a/bb-tests/workloads/src/CMakeLists.txt b/bb-tests/workloads/src/CMakeLists.txt index d3009b022..18c407fa4 100644 --- a/bb-tests/workloads/src/CMakeLists.txt +++ b/bb-tests/workloads/src/CMakeLists.txt @@ -1,7 +1,12 @@ -set(CTEST_WORKLOAD_DIR ${WORKLOAD_SRC_DIR}/CTest) -set(MLIR_WORKLOAD_DIR ${WORKLOAD_SRC_DIR}/MLIRTest) -set(SOC_WORKLOAD_DIR ${WORKLOAD_SRC_DIR}/SocTest) - add_subdirectory(CTest) add_subdirectory(MLIRTest) add_subdirectory(SocTest) + +# =============================================================== +# Design-specific workloads: the workload entry of the simulation. +# It defines target design-workloads-build. +# =============================================================== +add_subdirectory( + ${DESIGN_WORKLOADS} + ${CMAKE_CURRENT_BINARY_DIR}/CTest/designs/${DESIGN_NAME} +) diff --git a/bb-tests/workloads/src/CTest/CMakeLists.txt b/bb-tests/workloads/src/CTest/CMakeLists.txt index 94a1040c8..77ea75cd6 100644 --- a/bb-tests/workloads/src/CTest/CMakeLists.txt +++ b/bb-tests/workloads/src/CTest/CMakeLists.txt @@ -1,278 +1,265 @@ # =============================================================== # Generic functions to help add CTest workloads # =============================================================== -set(CTEST_MAX_LINES 100) -set(BBHW_MEM_C ${BUCKYBALL_REPO_ROOT}/stack/runtime/src/dma.c) -set(BUCKYBALL_CTEST_C_FLAGS - -g -fno-common -O2 -static -march=rv64gc -mcmodel=medany - -fno-builtin-printf -specs=nano.specs -specs=nosys.specs -nostartfiles - -Wl,-T,${BBSIM_LD} -) -file(GLOB BUCKYBALL_BALL_WORKLOAD_DIRS CONFIGURE_DEPENDS - "${BUCKYBALL_REPO_ROOT}/examples/balls/*/workloads") -file(GLOB BUCKYBALL_BALL_ISA_DEPS CONFIGURE_DEPENDS - "${BUCKYBALL_REPO_ROOT}/examples/balls/*/workloads/isa/*") - function(buckyball_enforce_ctest_line_limit SOURCE_FILE) - if(IS_ABSOLUTE "${SOURCE_FILE}") - set(_src "${SOURCE_FILE}") - else() - set(_src "${CMAKE_CURRENT_SOURCE_DIR}/${SOURCE_FILE}") - endif() - if(NOT EXISTS "${_src}") - message(FATAL_ERROR "ctest source does not exist: ${_src}") - endif() + if(IS_ABSOLUTE "${SOURCE_FILE}") + set(_src "${SOURCE_FILE}") + else() + set(_src "${CMAKE_CURRENT_SOURCE_DIR}/${SOURCE_FILE}") + endif() + if(NOT EXISTS "${_src}") + message(FATAL_ERROR "ctest source does not exist: ${_src}") + endif() - file(READ "${_src}" _content) - if(_content STREQUAL "") - set(_n 0) - else() - string(REGEX MATCHALL "\n" _newlines "${_content}") - list(LENGTH _newlines _n) - string(REGEX MATCH "[^\n]$" _no_trailing_nl "${_content}") - if(_no_trailing_nl) - math(EXPR _n "${_n} + 1") + file(READ "${_src}" _content) + if(_content STREQUAL "") + set(_n 0) + else() + string(REGEX MATCHALL "\n" _newlines "${_content}") + list(LENGTH _newlines _n) + string(REGEX MATCH "[^\n]$" _no_trailing_nl "${_content}") + if(_no_trailing_nl) + math(EXPR _n "${_n} + 1") + endif() endif() - endif() - if(_n GREATER CTEST_MAX_LINES) - message(FATAL_ERROR - "ctest exceeds ${CTEST_MAX_LINES} lines: ${_src} (${_n} lines). " - "Split it into smaller focused ctests. " - "Do not evade this check by moving functional code into headers.") - endif() + if(_n GREATER CTEST_MAX_LINES) + message( + FATAL_ERROR + "ctest exceeds ${CTEST_MAX_LINES} lines: ${_src} (${_n} lines). " + "Split it into smaller focused ctests. " + "Do not evade this check by moving functional code into headers." + ) + endif() endfunction() function(buckyball_ctest_deps OUT_DEPS SOURCE_DIR SOURCE_FILE) - if(NOT DEFINED BUCKYBALL_CTEST_TARGET OR BUCKYBALL_CTEST_TARGET STREQUAL "") - message(FATAL_ERROR "BUCKYBALL_CTEST_TARGET is required for C tests") - endif() - list(FIND BUCKYBALL_TARGETS "${BUCKYBALL_CTEST_TARGET}" _buckyball_target_index) - if(_buckyball_target_index EQUAL -1) - message(FATAL_ERROR "unknown C test target: ${BUCKYBALL_CTEST_TARGET}") - endif() - file(GLOB BUCKYBALL_ISA_DEPS CONFIGURE_DEPENDS - ${WORKLOAD_LIB_DIR}/bbhw/isa/*) - set(DEPS - ${SOURCE_DIR}/${SOURCE_FILE} - ${WORKLOAD_LIB_DIR}/buckyball.c - ${WORKLOAD_LIB_DIR}/buckyball.h - ${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/ballISA.h - ${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/params.h - ${BUCKYBALL_ISA_DIR}/chip/topology.h - ${BBHW_MEM_C} - ${BUCKYBALL_REPO_ROOT}/stack/runtime/include/dma.h - ${BUCKYBALL_CHIP_COMMON_SOURCES} - ${BUCKYBALL_CHIP_COMMON_HEADERS} - ${BUCKYBALL_ISA_DEPS} - ${BUCKYBALL_BALL_ISA_DEPS}) - file(GLOB _LOCAL_HDRS CONFIGURE_DEPENDS ${SOURCE_DIR}/*.h) - list(APPEND DEPS ${_LOCAL_HDRS}) + if(NOT DEFINED BUCKYBALL_CTEST_TARGET OR BUCKYBALL_CTEST_TARGET STREQUAL "") + message(FATAL_ERROR "BUCKYBALL_CTEST_TARGET is required for C tests") + endif() + list( + FIND DESIGN_PROFILES + "${BUCKYBALL_CTEST_TARGET}" + _buckyball_target_index + ) + if(_buckyball_target_index EQUAL -1) + message(FATAL_ERROR "unknown C test target: ${BUCKYBALL_CTEST_TARGET}") + endif() + file( + GLOB BUCKYBALL_ISA_DEPS + CONFIGURE_DEPENDS + ${WORKLOAD_LIB_DIR}/bbhw/isa/* + ) + set(DEPS + ${SOURCE_DIR}/${SOURCE_FILE} + ${WORKLOAD_LIB_DIR}/buckyball.c + ${WORKLOAD_LIB_DIR}/buckyball.h + ${BBHW_MEM_C} + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/dma.h + ${BUCKYBALL_DESIGN_COMMON_SOURCES} + ${BUCKYBALL_DESIGN_COMMON_HEADERS} + ${BUCKYBALL_ISA_DEPS} + ${BUCKYBALL_BALL_ISA_DEPS} + ) + file(GLOB _LOCAL_HDRS CONFIGURE_DEPENDS ${SOURCE_DIR}/*.h) + list(APPEND DEPS ${_LOCAL_HDRS}) - set(${OUT_DEPS} ${DEPS} PARENT_SCOPE) + set(${OUT_DEPS} ${DEPS} PARENT_SCOPE) endfunction() function(add_buckyball_linux_ctest TEST_NAME SOURCE_DIR SOURCE_FILE) - set(EXECUTABLE "${TEST_NAME}-linux") + set(EXECUTABLE "${TEST_NAME}-linux") - add_executable(${EXECUTABLE} - ${SOURCE_DIR}/${SOURCE_FILE} - ${WORKLOAD_LIB_DIR}/buckyball.c - ${BBHW_MEM_C} - ${BUCKYBALL_CHIP_COMMON_SOURCES}) - target_include_directories(${EXECUTABLE} PRIVATE - ${WORKLOAD_LIB_DIR} - ${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET} - ${BUCKYBALL_CHIP_COMMON_INCLUDE_DIRS} - ${BUCKYBALL_BALL_WORKLOAD_DIRS} - ${SOURCE_DIR}) - set_target_properties(${EXECUTABLE} PROPERTIES LINKER_LANGUAGE C) + add_executable( + ${EXECUTABLE} + ${SOURCE_DIR}/${SOURCE_FILE} + ${WORKLOAD_LIB_DIR}/buckyball.c + ${BBHW_MEM_C} + ${BUCKYBALL_DESIGN_COMMON_SOURCES} + ) + target_include_directories( + ${EXECUTABLE} + PRIVATE + ${WORKLOAD_LIB_DIR} + ${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services + ${BUCKYBALL_DESIGN_COMMON_INCLUDE_DIRS} + ${BUCKYBALL_BALL_WORKLOAD_DIRS} + ${SOURCE_DIR} + ) + target_compile_definitions( + ${EXECUTABLE} + PRIVATE ${DESIGN_TARGET_DEFS_${BUCKYBALL_CTEST_TARGET}} + ) + set_target_properties(${EXECUTABLE} PROPERTIES LINKER_LANGUAGE C) - add_custom_target(${TEST_NAME}-linux-build - DEPENDS ${EXECUTABLE}) + add_custom_target(${TEST_NAME}-linux-build DEPENDS ${EXECUTABLE}) endfunction() function(add_buckyball_baremetal_ctest TEST_NAME SOURCE_DIR SOURCE_FILE) - set(EXECUTABLE "${TEST_NAME}-baremetal") - buckyball_ctest_deps(TEST_DEPS ${SOURCE_DIR} ${SOURCE_FILE}) + set(EXECUTABLE "${TEST_NAME}-baremetal") + buckyball_ctest_deps(TEST_DEPS ${SOURCE_DIR} ${SOURCE_FILE}) - set(BUCKYBALL_CHIP_COMMON_INCLUDE_FLAGS) - foreach(CHIP_COMMON_INCLUDE_DIR ${BUCKYBALL_CHIP_COMMON_INCLUDE_DIRS}) - list(APPEND BUCKYBALL_CHIP_COMMON_INCLUDE_FLAGS -I${CHIP_COMMON_INCLUDE_DIR}) - endforeach() - set(BALL_ISA_INCLUDE_FLAGS) - foreach(BALL_WORKLOAD_DIR ${BUCKYBALL_BALL_WORKLOAD_DIRS}) - list(APPEND BALL_ISA_INCLUDE_FLAGS -I${BALL_WORKLOAD_DIR}) - endforeach() + set(BUCKYBALL_DESIGN_COMMON_INCLUDE_FLAGS) + foreach(DESIGN_COMMON_INCLUDE_DIR ${BUCKYBALL_DESIGN_COMMON_INCLUDE_DIRS}) + list( + APPEND BUCKYBALL_DESIGN_COMMON_INCLUDE_FLAGS + -I${DESIGN_COMMON_INCLUDE_DIR} + ) + endforeach() + set(TARGET_DEFINES ${DESIGN_TARGET_DEFS_${BUCKYBALL_CTEST_TARGET}}) + list(TRANSFORM TARGET_DEFINES PREPEND "-D") + set(BALL_ISA_INCLUDE_FLAGS) + foreach(BALL_WORKLOAD_DIR ${BUCKYBALL_BALL_WORKLOAD_DIRS}) + list(APPEND BALL_ISA_INCLUDE_FLAGS -I${BALL_WORKLOAD_DIR}) + endforeach() - add_custom_command( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${EXECUTABLE} - COMMAND ${ELF_CC} ${BUCKYBALL_CTEST_C_FLAGS} - -o ${EXECUTABLE} - ${BBSW_BAREMETAL_DIR}/crt0.S - ${WORKLOAD_LIB_DIR}/buckyball.c - ${BBHW_MEM_C} - ${BUCKYBALL_CHIP_COMMON_SOURCES} - ${SOURCE_DIR}/${SOURCE_FILE} - -I${BUCKYBALL_REPO_ROOT}/stack/runtime/include -I${WORKLOAD_LIB_DIR} - -I${BUCKYBALL_ISA_DIR}/chip - -I${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET} - ${BUCKYBALL_CHIP_COMMON_INCLUDE_FLAGS} - ${BALL_ISA_INCLUDE_FLAGS} - -I${SOURCE_DIR} - DEPENDS - ${TEST_DEPS} - ${BBSW_BAREMETAL_DIR}/crt0.S - ${BBSIM_LD} - COMMENT "Building baremetal executable: ${EXECUTABLE}" - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - ) + add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${EXECUTABLE} + COMMAND + ${ELF_CC} ${BUCKYBALL_CTEST_C_FLAGS} -o ${EXECUTABLE} + ${BAREMETAL_DIR}/crt0.S ${WORKLOAD_LIB_DIR}/buckyball.c + ${BBHW_MEM_C} ${BUCKYBALL_DESIGN_COMMON_SOURCES} + ${SOURCE_DIR}/${SOURCE_FILE} + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${WORKLOAD_LIB_DIR} ${DESIGN_DEFINES} -I${RUNTIME_INCLUDE_DIR} + ${TARGET_DEFINES} + ${BUCKYBALL_DESIGN_COMMON_INCLUDE_FLAGS} ${BALL_ISA_INCLUDE_FLAGS} + -I${SOURCE_DIR} + DEPENDS ${TEST_DEPS} ${BAREMETAL_DIR}/crt0.S ${BBSIM_LD} + COMMENT "Building baremetal executable: ${EXECUTABLE}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM + ) - add_custom_target(${TEST_NAME}-baremetal-build - DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${EXECUTABLE}) + add_custom_target( + ${TEST_NAME}-baremetal-build + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${EXECUTABLE} + ) endfunction() function(add_buckyball_ctest SOURCE_FILE) - if(BUCKYBALL_WORKLOAD_CHIP STREQUAL "") - message(FATAL_ERROR - "BUCKYBALL_WORKLOAD_CHIP must be set before add_buckyball_ctest()") - endif() - get_filename_component(_stem ${SOURCE_FILE} NAME_WE) - if(NOT DEFINED BUCKYBALL_CTEST_TARGET OR BUCKYBALL_CTEST_TARGET STREQUAL "") - message(FATAL_ERROR "BUCKYBALL_CTEST_TARGET is required for ${SOURCE_FILE}") - endif() - buckyball_ctest_name(TEST_NAME ${BUCKYBALL_CTEST_TARGET} ${_stem}) - set(SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) - - buckyball_enforce_ctest_line_limit(${SOURCE_FILE}) - buckyball_ctest_deps(TEST_DEPS ${SOURCE_DIR} ${SOURCE_FILE}) + get_filename_component(_stem ${SOURCE_FILE} NAME_WE) + if(NOT DEFINED BUCKYBALL_CTEST_TARGET OR BUCKYBALL_CTEST_TARGET STREQUAL "") + message( + FATAL_ERROR + "BUCKYBALL_CTEST_TARGET is required for ${SOURCE_FILE}" + ) + endif() + set(TEST_NAME + "${DESIGN_NAME}-${BUCKYBALL_CTEST_TARGET}-ctest-${_stem}" + ) + set(SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) - add_buckyball_linux_ctest(${TEST_NAME} ${SOURCE_DIR} ${SOURCE_FILE}) - add_buckyball_baremetal_ctest(${TEST_NAME} ${SOURCE_DIR} ${SOURCE_FILE}) + buckyball_enforce_ctest_line_limit(${SOURCE_FILE}) - add_custom_target(${TEST_NAME}-ctest-build - DEPENDS - ${TEST_NAME}-linux-build - ${TEST_NAME}-baremetal-build - COMMENT "Building CTest workload ${TEST_NAME}" - ) + add_buckyball_linux_ctest(${TEST_NAME} ${SOURCE_DIR} ${SOURCE_FILE}) + add_buckyball_baremetal_ctest(${TEST_NAME} ${SOURCE_DIR} ${SOURCE_FILE}) - if(DEFINED BUCKYBALL_CTEST_GROUP_TARGET) - add_dependencies(${BUCKYBALL_CTEST_GROUP_TARGET} ${TEST_NAME}-ctest-build) - endif() -endfunction() + add_custom_target( + ${TEST_NAME}-ctest-build + DEPENDS ${TEST_NAME}-linux-build ${TEST_NAME}-baremetal-build + COMMENT "Building CTest workload ${TEST_NAME}" + ) -function(add_buckyball_ctests) - foreach(SOURCE_FILE ${ARGV}) - add_buckyball_ctest(${SOURCE_FILE}) - endforeach() + if(DEFINED BUCKYBALL_CTEST_GROUP_TARGET) + add_dependencies( + ${BUCKYBALL_CTEST_GROUP_TARGET} + ${TEST_NAME}-ctest-build + ) + endif() endfunction() -add_custom_target(core-ctests-build) +add_custom_target(component-ctests-build) -function(add_buckyball_core_workload NAME) - cmake_parse_arguments(ARG "" "" "SOURCES" ${ARGN}) - if(NOT DEFINED BUCKYBALL_CORE_WORKLOAD_TARGET OR - BUCKYBALL_CORE_WORKLOAD_TARGET STREQUAL "") - message(FATAL_ERROR "Core workload target is required for ${NAME}") - endif() - if(NOT DEFINED BUCKYBALL_CORE_WORKLOAD_PROFILE OR - BUCKYBALL_CORE_WORKLOAD_PROFILE STREQUAL "" OR NOT ARG_SOURCES) - message(FATAL_ERROR "Core workload profile and sources are required for ${NAME}") - endif() - set(BUCKYBALL_CTEST_TARGET "${BUCKYBALL_CORE_WORKLOAD_TARGET}") - buckyball_ctest_name(_test ${BUCKYBALL_CTEST_TARGET} core-${NAME}) - set(_elf "${_test}-multicore-baremetal") - set(_sources) - foreach(_source IN LISTS ARG_SOURCES) - buckyball_enforce_ctest_line_limit("${_source}") - list(APPEND _sources "${CMAKE_CURRENT_SOURCE_DIR}/${_source}") - endforeach() - set(_ball_includes) - foreach(_dir IN LISTS BUCKYBALL_BALL_WORKLOAD_DIRS) - list(APPEND _ball_includes -I${_dir}) - endforeach() - add_custom_command( - OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${_elf}" - COMMAND ${RISCV_GNU_TOOLCHAIN}/bin/riscv64-unknown-elf-g++ - ${BUCKYBALL_CTEST_C_FLAGS} - -DBB_CORE_WORKLOAD_PROFILE=${BUCKYBALL_CORE_WORKLOAD_PROFILE} - -I${BUCKYBALL_REPO_ROOT}/stack/runtime/include -I${WORKLOAD_LIB_DIR} - -I${WORKLOAD_LIB_DIR}/bbsw/baremetal/multicore - -I${BUCKYBALL_ISA_DIR}/chip - -I${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET} - ${_ball_includes} - -o ${_elf} - ${WORKLOAD_LIB_DIR}/bbsw/baremetal/start.S - ${_sources} - ${WORKLOAD_LIB_DIR}/buckyball.c - ${BBHW_MEM_C} - DEPENDS ${_sources} - ${WORKLOAD_LIB_DIR}/bbsw/baremetal/start.S - ${BUCKYBALL_ISA_DIR}/chip/topology.h - ${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/ballISA.h - ${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/params.h - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) - add_custom_target(${_test}-core-build - DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${_elf}") - add_dependencies(core-ctests-build ${_test}-core-build) +function(add_buckyball_component_workload NAME) + cmake_parse_arguments(ARG "" "" "SOURCES" ${ARGN}) + if( + NOT DEFINED BUCKYBALL_CTEST_TARGET + OR BUCKYBALL_CTEST_TARGET STREQUAL "" + OR NOT ARG_SOURCES + ) + message( + FATAL_ERROR + "Workload target and sources are required for ${NAME}" + ) + endif() + list(FIND DESIGN_PROFILES "${BUCKYBALL_CTEST_TARGET}" _profile) + set(_defines ${DESIGN_TARGET_DEFS_${BUCKYBALL_CTEST_TARGET}}) + list(TRANSFORM _defines PREPEND "-D") + set(_test + "${DESIGN_NAME}-${BUCKYBALL_CTEST_TARGET}-ctest-pipeline-${NAME}" + ) + set(_elf "${_test}-multicore-baremetal") + set(_sources) + foreach(_source IN LISTS ARG_SOURCES) + buckyball_enforce_ctest_line_limit("${_source}") + list(APPEND _sources "${CMAKE_CURRENT_SOURCE_DIR}/${_source}") + endforeach() + file(GLOB_RECURSE _headers CONFIGURE_DEPENDS "${WORKLOAD_LIB_DIR}/*.h") + set(_ball_includes) + foreach(_dir IN LISTS BUCKYBALL_BALL_WORKLOAD_DIRS) + list(APPEND _ball_includes -I${_dir}) + file(GLOB_RECURSE _ball_headers CONFIGURE_DEPENDS "${_dir}/*.h") + list(APPEND _headers ${_ball_headers}) + endforeach() + add_custom_command( + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${_elf}" + COMMAND + ${ELF_CXX} ${BUCKYBALL_CTEST_C_FLAGS} + -DBB_COMPONENT_WORKLOAD_PROFILE=${_profile} + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${WORKLOAD_LIB_DIR} -I${BAREMETAL_DIR}/multithread + ${DESIGN_DEFINES} -I${RUNTIME_INCLUDE_DIR} + ${_defines} ${_ball_includes} + -o ${_elf} ${BAREMETAL_DIR}/start.S ${_sources} + ${WORKLOAD_LIB_DIR}/buckyball.c ${BBHW_MEM_C} + DEPENDS + ${_sources} + ${_headers} + ${BAREMETAL_DIR}/start.S + ${RUNTIME_INCLUDE_DIR}/bb_cores.h + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM + ) + add_custom_target( + ${_test}-thread-build + DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${_elf}" + ) + add_dependencies(${BUCKYBALL_CTEST_GROUP_TARGET} ${_test}-thread-build) endfunction() # =============================================================== -# Common workloads for all chips -# =============================================================== -add_subdirectory(base) -add_subdirectory(tlb) -add_subdirectory(tutorial) -add_subdirectory(embench) -add_subdirectory(coremark) -buckyball_add_core_workload_subdirs(ctests) - -# =============================================================== -# Chip-specific workloads +# Common workloads for all designs # =============================================================== -set(BUCKYBALL_CHIP_WORKLOAD_DIR - ${BUCKYBALL_REPO_ROOT}/examples/chips/${BUCKYBALL_WORKLOAD_SOURCE_CHIP}/workloads) - -if(NOT EXISTS ${BUCKYBALL_CHIP_WORKLOAD_DIR}/CMakeLists.txt) - message(FATAL_ERROR - "Buckyball chip workloads do not exist: ${BUCKYBALL_CHIP_WORKLOAD_DIR}") -endif() - -add_subdirectory(${BUCKYBALL_CHIP_WORKLOAD_DIR} - ${CMAKE_CURRENT_BINARY_DIR}/chips/${BUCKYBALL_WORKLOAD_SOURCE_CHIP}) - -if(NOT TARGET chip-workloads-build) - message(FATAL_ERROR - "Buckyball chip workloads must define target chip-workloads-build: ${BUCKYBALL_WORKLOAD_CHIP}") -endif() - -add_custom_target(ctest-workloads-build - DEPENDS - base-ctests-build - tlb-ctests-build - tutorial-ctests-build - core-ctests-build - chip-ctests-build - COMMENT "Building CTest workloads" +add_custom_target( + ctest-workloads-build + DEPENDS design-ctests-build + COMMENT "Building registered CTest workloads" ) -add_custom_target(sync-ctest-bin - COMMAND ${CMAKE_COMMAND} -E make_directory ${OUTPUT_BIN_DIR}/src/CTest - COMMAND rsync -a --delete --exclude=CMakeFiles - "--exclude=*/mlir_tests/**" "--include=*/" - "--include=*-baremetal" "--include=*-linux" - "--include=*-baremetal.expect" "--exclude=*" - ${CMAKE_CURRENT_BINARY_DIR}/ ${OUTPUT_BIN_DIR}/src/CTest/ - DEPENDS ctest-workloads-build - COMMENT "Syncing CTest binaries to output directory" +add_custom_target( + sync-ctest-bin + COMMAND ${CMAKE_COMMAND} -E make_directory ${OUTPUT_BIN_DIR}/src/CTest + COMMAND + rsync -a --delete --exclude=CMakeFiles "--exclude=*/mlir_tests/**" + "--include=*/" "--include=*-baremetal" "--include=*-linux" + "--include=*-baremetal.expect" "--exclude=*" + ${CMAKE_CURRENT_BINARY_DIR}/ ${OUTPUT_BIN_DIR}/src/CTest/ + DEPENDS ctest-workloads-build + COMMENT "Syncing CTest binaries to output directory" ) -add_custom_target(sync-native-bin - COMMAND ${CMAKE_COMMAND} -E make_directory ${OUTPUT_BIN_DIR}/src/CTest - COMMAND rsync -a --delete --exclude=CMakeFiles "--exclude=mlir_tests" "--include=*/" - "--include=*-baremetal" "--include=*-baremetal.expect" - "--include=*-linux" "--include=*-run" "--exclude=*" - ${CMAKE_CURRENT_BINARY_DIR}/ ${OUTPUT_BIN_DIR}/src/CTest/ - DEPENDS ctest-workloads-build embench-all coremark-all - COMMENT "Syncing CTest and benchmark binaries") +add_custom_target( + sync-native-bin + COMMAND ${CMAKE_COMMAND} -E make_directory ${OUTPUT_BIN_DIR}/src/CTest + COMMAND + rsync -a --delete --exclude=CMakeFiles "--exclude=mlir_tests" + "--include=*/" "--include=*-baremetal" "--include=*-baremetal.expect" + "--include=*-linux" "--include=*-run" "--exclude=*" + ${CMAKE_CURRENT_BINARY_DIR}/ ${OUTPUT_BIN_DIR}/src/CTest/ + DEPENDS ctest-workloads-build + COMMENT "Syncing CTest and benchmark binaries" +) diff --git a/bb-tests/workloads/src/CTest/base/CMakeLists.txt b/bb-tests/workloads/src/CTest/base/CMakeLists.txt index f4fdbdd2b..fdcb2160c 100644 --- a/bb-tests/workloads/src/CTest/base/CMakeLists.txt +++ b/bb-tests/workloads/src/CTest/base/CMakeLists.txt @@ -1,32 +1,37 @@ add_custom_target(base-ctests-build) set(BUCKYBALL_CTEST_GROUP_TARGET base-ctests-build) -foreach(BUCKYBALL_CTEST_TARGET IN LISTS BUCKYBALL_CPU_TARGETS) - set(_params "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/params.h") - foreach(_name BANK_NUM BANK_WIDTH BANK_LINES BB_PRIVATE_VBANK_MAX) - file(STRINGS "${_params}" _line REGEX "^#define ${_name} ") - string(REGEX REPLACE ".* " "" _${_name} "${_line}") - endforeach() - if(_BB_PRIVATE_VBANK_MAX EQUAL 0) - continue() - endif() - math(EXPR _word_remainder "${_BANK_WIDTH} % 32") - if(_BANK_NUM LESS 4 OR _BB_PRIVATE_VBANK_MAX LESS 2 OR _BANK_LINES LESS 24 OR - _BANK_WIDTH LESS 64 OR _BANK_WIDTH GREATER 2048 OR NOT _word_remainder EQUAL 0) - message(FATAL_ERROR - "base DMA tests require 4 banks for ACC, vbank2, >=24 rows for 2D, and 64..2048-bit rows divisible by 32; ${BUCKYBALL_CTEST_TARGET}: banks=${_BANK_NUM}, privateMax=${_BB_PRIVATE_VBANK_MAX}, lines=${_BANK_LINES}, width=${_BANK_WIDTH}") - endif() - if(_BB_PRIVATE_VBANK_MAX GREATER_EQUAL 4) - add_buckyball_ctest(mset_transfer_test.c) - endif() - add_buckyball_ctests( - mvin_mvout_group_test.c - mvin_2d_test.c - mvin_2d_pixel_stride_test.c - mvin_mvout_acc_test.c - mvin_mvout_stride_test.c - mvin_mvout_test.c - mvin_mvout_unaligned_test.c - mvin_unaligned_test.c - ) +foreach(BUCKYBALL_CTEST_TARGET IN LISTS DESIGN_CPU_TARGETS) + set(_BANK_NUM ${DESIGN_PROFILES_${BUCKYBALL_CTEST_TARGET}_THREAD_MEM_BANK_NUM}) + set(_BANK_WIDTH ${DESIGN_PROFILES_${BUCKYBALL_CTEST_TARGET}_THREAD_MEM_BANK_WIDTH}) + set(_BANK_ENTRIES ${DESIGN_PROFILES_${BUCKYBALL_CTEST_TARGET}_THREAD_MEM_BANK_ENTRIES}) + set(_MAX_PRIVATE_VBANK ${DESIGN_PROFILES_${BUCKYBALL_CTEST_TARGET}_THREAD_FRONTEND_MAX_PRIVATE_VBANK}) + if(_MAX_PRIVATE_VBANK EQUAL 0) + continue() + endif() + math(EXPR _word_remainder "${_BANK_WIDTH} % 32") + if( + _BANK_NUM LESS 4 + OR _MAX_PRIVATE_VBANK LESS 2 + OR _BANK_ENTRIES LESS 24 + OR _BANK_WIDTH LESS 64 + OR _BANK_WIDTH GREATER 2048 + OR NOT _word_remainder EQUAL 0 + ) + message( + FATAL_ERROR + "base DMA tests require 4 banks for ACC, vbank2, >=24 rows for 2D, and 64..2048-bit rows divisible by 32; ${BUCKYBALL_CTEST_TARGET}: banks=${_BANK_NUM}, privateMax=${_MAX_PRIVATE_VBANK}, lines=${_BANK_ENTRIES}, width=${_BANK_WIDTH}" + ) + endif() + if(_MAX_PRIVATE_VBANK GREATER_EQUAL 4) + add_buckyball_ctest(mset_transfer_test.c) + endif() + add_buckyball_ctest(mvin_mvout_group_test.c) + add_buckyball_ctest(mvin_2d_test.c) + add_buckyball_ctest(mvin_2d_pixel_stride_test.c) + add_buckyball_ctest(mvin_mvout_acc_test.c) + add_buckyball_ctest(mvin_mvout_stride_test.c) + add_buckyball_ctest(mvin_mvout_test.c) + add_buckyball_ctest(mvin_mvout_unaligned_test.c) + add_buckyball_ctest(mvin_unaligned_test.c) endforeach() diff --git a/bb-tests/workloads/src/CTest/base/mset_transfer_test.c b/bb-tests/workloads/src/CTest/base/mset_transfer_test.c index 540d27834..9a23fe1d3 100644 --- a/bb-tests/workloads/src/CTest/base/mset_transfer_test.c +++ b/bb-tests/workloads/src/CTest/base/mset_transfer_test.c @@ -1,16 +1,16 @@ #include -#include #include #include #include -enum { ROW_BYTES = BANK_WIDTH / 8 }; +enum { ROW_BYTES = THREAD_MEM_BANK_WIDTH / 8 }; static uint8_t first[2 * ROW_BYTES] __attribute__((aligned(16))); static uint8_t second[4 * ROW_BYTES] __attribute__((aligned(16))); static uint8_t replacement[2 * ROW_BYTES] __attribute__((aligned(16))); static uint8_t result[6 * ROW_BYTES] __attribute__((aligned(16))); -static uint8_t zeros[BANK_NUM * ROW_BYTES] __attribute__((aligned(16))); +static uint8_t zeros[THREAD_MEM_BANK_NUM * ROW_BYTES] + __attribute__((aligned(16))); int main(int argc, char **argv) { for (unsigned i = 0; i < sizeof(first); ++i) { @@ -54,7 +54,7 @@ int main(int argc, char **argv) { } bb_mem_release(0); bb_mem_release(3); - bb_mset_clear(4, 1, BANK_NUM); + bb_mset_clear(4, 1, THREAD_MEM_BANK_NUM); bb_mvout((uintptr_t)zeros, 4, 1, 1); for (unsigned i = 0; i < sizeof(zeros); ++i) if (zeros[i]) diff --git a/bb-tests/workloads/src/CTest/base/mvin_2d_pixel_stride_test.c b/bb-tests/workloads/src/CTest/base/mvin_2d_pixel_stride_test.c index b9a45bb46..9bf01a0e1 100644 --- a/bb-tests/workloads/src/CTest/base/mvin_2d_pixel_stride_test.c +++ b/bb-tests/workloads/src/CTest/base/mvin_2d_pixel_stride_test.c @@ -1,19 +1,20 @@ #include "buckyball.h" #include #include -#include #include #include -_Static_assert(BANK_WIDTH == 128 && BANK_LINES >= 64, +_Static_assert(THREAD_MEM_BANK_WIDTH == 128 && THREAD_MEM_BANK_ENTRIES >= 64, "MVIN2D boundary test requires 128-bit rows and at least 1 KiB"); static const uint8_t input[8 * 256] __attribute__((aligned(128))) = { [0 * 256] = 11, [1 * 256] = 22, [2 * 256] = 33, [3 * 256] = 44, [4 * 256] = 55, [5 * 256] = 66, [6 * 256] = 77, [7 * 256] = 88, }; -static const uint8_t zeros[64 * BANK_WIDTH / 8] __attribute__((aligned(128))); -static uint8_t output[64 * BANK_WIDTH / 8] __attribute__((aligned(128))); +static const uint8_t zeros[64 * THREAD_MEM_BANK_WIDTH / 8] + __attribute__((aligned(128))); +static uint8_t output[64 * THREAD_MEM_BANK_WIDTH / 8] + __attribute__((aligned(128))); // MobileNet's 72-byte pixels: the last read straddles the 4 KiB page. static const uint8_t page_input[8192] __attribute__((aligned(4096))) = { @@ -32,15 +33,16 @@ int main(void) { bb_mem_alloc(bank, 1, 1); if (test == 2) bb_mvin((uintptr_t)zeros, bank, 64, 1); - bb_mvin_2d(source, bank, 1, stride, 8, first_row, 8, BANK_WIDTH / 8); + bb_mvin_2d(source, bank, 1, stride, 8, first_row, 8, + THREAD_MEM_BANK_WIDTH / 8); bb_mvout((uintptr_t)output, bank, rows, 1); for (int row = 0; row < rows; ++row) { - for (int byte = 0; byte < BANK_WIDTH / 8; ++byte) { + for (int byte = 0; byte < THREAD_MEM_BANK_WIDTH / 8; ++byte) { uint8_t expected = row >= first_row && byte == 0 ? (uint8_t)((row - first_row + 1) * 11) : 0; - uint8_t actual = output[row * (BANK_WIDTH / 8) + byte]; + uint8_t actual = output[row * (THREAD_MEM_BANK_WIDTH / 8) + byte]; if (actual != expected) { printf("mvin_2d pixel stride mismatch case=%d row=%d byte=%d " "expected=%u " diff --git a/bb-tests/workloads/src/CTest/base/mvin_2d_test.c b/bb-tests/workloads/src/CTest/base/mvin_2d_test.c index 5949fc055..eeac6de55 100644 --- a/bb-tests/workloads/src/CTest/base/mvin_2d_test.c +++ b/bb-tests/workloads/src/CTest/base/mvin_2d_test.c @@ -1,29 +1,29 @@ #include "buckyball.h" #include #include -#include #include enum { SOURCE_HEIGHT = 5, SOURCE_WIDTH = 7, - PANEL_BYTES = (BANK_WIDTH / 8), + PANEL_BYTES = (THREAD_MEM_BANK_WIDTH / 8), CHANNELS = (PANEL_BYTES + PANEL_BYTES / 2), - TILE_HEIGHT = 4, - TILE_WIDTH = 3, + CORE_HEIGHT = 4, + CORE_WIDTH = 3, START_Y = 1, START_X = 2, - TILE_ROWS = (TILE_HEIGHT * TILE_WIDTH) + CORE_ROWS = (CORE_HEIGHT * CORE_WIDTH) }; static elem_t input[SOURCE_HEIGHT * SOURCE_WIDTH * CHANNELS] __attribute__((aligned(128))); -static elem_t output[2 * TILE_ROWS * PANEL_BYTES] __attribute__((aligned(128))); +static elem_t output[2 * CORE_ROWS * PANEL_BYTES] __attribute__((aligned(128))); +static elem_t dirty[2 * CORE_ROWS * PANEL_BYTES] __attribute__((aligned(128))); static int check_output(void) { for (int panel = 0; panel < 2; ++panel) { - for (int y = 0; y < TILE_HEIGHT; ++y) { - for (int x = 0; x < TILE_WIDTH; ++x) { + for (int y = 0; y < CORE_HEIGHT; ++y) { + for (int x = 0; x < CORE_WIDTH; ++x) { for (int lane = 0; lane < PANEL_BYTES; ++lane) { int channel = panel * PANEL_BYTES + lane; elem_t expected = @@ -32,7 +32,7 @@ static int check_output(void) { CHANNELS + channel] : 0; - int row = panel * TILE_ROWS + y * TILE_WIDTH + x; + int row = panel * CORE_ROWS + y * CORE_WIDTH + x; elem_t actual = output[row * PANEL_BYTES + lane]; if (actual != expected) { printf("mvin_2d mismatch panel=%d y=%d x=%d lane=%d expected=%d " @@ -50,18 +50,22 @@ static int check_output(void) { int main(void) { for (int i = 0; i < SOURCE_HEIGHT * SOURCE_WIDTH * CHANNELS; ++i) input[i] = (elem_t)((i * 13 + 7) & 0x7f); - for (int i = 0; i < 2 * TILE_ROWS * PANEL_BYTES; ++i) + for (int i = 0; i < 2 * CORE_ROWS * PANEL_BYTES; ++i) { output[i] = (elem_t)-1; + dirty[i] = (elem_t)0x5a; + } uint32_t bank = 0; bb_mem_alloc(bank, 1, 1); + // Tail lanes must be zeroed even when the destination already contains data. + bb_mvin((uintptr_t)dirty, bank, 2 * CORE_ROWS, 1); uintptr_t source = (uintptr_t)&input[(START_Y * SOURCE_WIDTH + START_X) * CHANNELS]; - bb_mvin_2d(source, bank, TILE_HEIGHT, CHANNELS, SOURCE_WIDTH, 0, TILE_WIDTH, + bb_mvin_2d(source, bank, CORE_HEIGHT, CHANNELS, SOURCE_WIDTH, 0, CORE_WIDTH, PANEL_BYTES); - bb_mvin_2d(source + PANEL_BYTES, bank, TILE_HEIGHT, CHANNELS, SOURCE_WIDTH, - TILE_ROWS, TILE_WIDTH, CHANNELS - PANEL_BYTES); - bb_mvout((uintptr_t)output, bank, 2 * TILE_ROWS, 1); + bb_mvin_2d(source + PANEL_BYTES, bank, CORE_HEIGHT, CHANNELS, SOURCE_WIDTH, + CORE_ROWS, CORE_WIDTH, CHANNELS - PANEL_BYTES); + bb_mvout((uintptr_t)output, bank, 2 * CORE_ROWS, 1); if (!check_output()) return 1; diff --git a/bb-tests/workloads/src/CTest/base/mvin_mvout_acc_test.c b/bb-tests/workloads/src/CTest/base/mvin_mvout_acc_test.c index 39aab8ea0..08124ee3a 100644 --- a/bb-tests/workloads/src/CTest/base/mvin_mvout_acc_test.c +++ b/bb-tests/workloads/src/CTest/base/mvin_mvout_acc_test.c @@ -1,13 +1,12 @@ #include "buckyball.h" #include #include -#include #include #include #include // One ACC row spans four configured physical bank rows. -enum { ROWS = 16, COLS = (4 * BANK_WIDTH / 32) }; +enum { ROWS = 16, COLS = (4 * THREAD_MEM_BANK_WIDTH / 32) }; static result_t output_matrix[ROWS * COLS] __attribute__((aligned(64))); static result_t expected_matrix[ROWS * COLS] __attribute__((aligned(64))); diff --git a/bb-tests/workloads/src/CTest/base/mvin_mvout_group_test.c b/bb-tests/workloads/src/CTest/base/mvin_mvout_group_test.c index 3115e5861..c2b3116ec 100644 --- a/bb-tests/workloads/src/CTest/base/mvin_mvout_group_test.c +++ b/bb-tests/workloads/src/CTest/base/mvin_mvout_group_test.c @@ -1,9 +1,8 @@ #include -#include #include #include #include -enum { ROW_BYTES = BANK_WIDTH / 8 }; +enum { ROW_BYTES = THREAD_MEM_BANK_WIDTH / 8 }; static uint8_t initial[6 * ROW_BYTES] __attribute__((aligned(16))); static uint8_t replacement[3 * ROW_BYTES] __attribute__((aligned(16))); static uint8_t full[6 * ROW_BYTES] __attribute__((aligned(16))); diff --git a/bb-tests/workloads/src/CTest/base/mvin_mvout_stride_test.c b/bb-tests/workloads/src/CTest/base/mvin_mvout_stride_test.c index 13d65c592..b5e899b91 100644 --- a/bb-tests/workloads/src/CTest/base/mvin_mvout_stride_test.c +++ b/bb-tests/workloads/src/CTest/base/mvin_mvout_stride_test.c @@ -1,11 +1,15 @@ #include "buckyball.h" #include #include -#include #include #include -enum { ROW_BYTES = (BANK_WIDTH / 8), ROWS = 4, IN_STRIDE = 3, OUT_STRIDE = 5 }; +enum { + ROW_BYTES = (THREAD_MEM_BANK_WIDTH / 8), + ROWS = 4, + IN_STRIDE = 3, + OUT_STRIDE = 5 +}; static elem_t input_matrix[ROWS * IN_STRIDE * ROW_BYTES] __attribute__((aligned(128))); diff --git a/bb-tests/workloads/src/CTest/base/mvin_mvout_test.c b/bb-tests/workloads/src/CTest/base/mvin_mvout_test.c index dc0e68c55..7ddecaaf5 100644 --- a/bb-tests/workloads/src/CTest/base/mvin_mvout_test.c +++ b/bb-tests/workloads/src/CTest/base/mvin_mvout_test.c @@ -1,12 +1,11 @@ #include "buckyball.h" #include #include -#include #include #include #include -enum { ROWS = 16, COLS = (BANK_WIDTH / 8) }; +enum { ROWS = 16, COLS = (THREAD_MEM_BANK_WIDTH / 8) }; // Test matrices static elem_t input_matrix[ROWS * COLS] __attribute__((aligned(128))); diff --git a/bb-tests/workloads/src/CTest/base/mvin_mvout_unaligned_test.c b/bb-tests/workloads/src/CTest/base/mvin_mvout_unaligned_test.c index 33cbc9d2a..c6d9c762a 100644 --- a/bb-tests/workloads/src/CTest/base/mvin_mvout_unaligned_test.c +++ b/bb-tests/workloads/src/CTest/base/mvin_mvout_unaligned_test.c @@ -1,12 +1,16 @@ #include "buckyball.h" #include #include -#include #include #include #include -enum { ROW_BYTES = (BANK_WIDTH / 8), ROWS = 4, OFFSET = 4, GUARD = 0x5a }; +enum { + ROW_BYTES = (THREAD_MEM_BANK_WIDTH / 8), + ROWS = 4, + OFFSET = 4, + GUARD = 0x5a +}; static elem_t input_matrix[ROWS * ROW_BYTES] __attribute__((aligned(128))); static uint8_t output_bytes[OFFSET + ROWS * ROW_BYTES + ROW_BYTES] diff --git a/bb-tests/workloads/src/CTest/base/mvin_unaligned_test.c b/bb-tests/workloads/src/CTest/base/mvin_unaligned_test.c index 75dd88175..e4b966965 100644 --- a/bb-tests/workloads/src/CTest/base/mvin_unaligned_test.c +++ b/bb-tests/workloads/src/CTest/base/mvin_unaligned_test.c @@ -1,12 +1,16 @@ #include "buckyball.h" #include #include -#include #include #include #include -enum { ROW_BYTES = (BANK_WIDTH / 8), ROWS = 4, OFFSET = 4, GUARD = 0xa5 }; +enum { + ROW_BYTES = (THREAD_MEM_BANK_WIDTH / 8), + ROWS = 4, + OFFSET = 4, + GUARD = 0xa5 +}; static uint8_t input_bytes[OFFSET + ROWS * ROW_BYTES + ROW_BYTES] __attribute__((aligned(128))); diff --git a/bb-tests/workloads/src/CTest/coremark/CMakeLists.txt b/bb-tests/workloads/src/CTest/coremark/CMakeLists.txt index 6820d958d..663236780 100644 --- a/bb-tests/workloads/src/CTest/coremark/CMakeLists.txt +++ b/bb-tests/workloads/src/CTest/coremark/CMakeLists.txt @@ -1,65 +1,70 @@ -set(ELF_CC "riscv64-unknown-elf-gcc") - -set(CTEST_COREMARK_WORKLOAD_DIR ${CTEST_WORKLOAD_DIR}/coremark) +set(CTEST_COREMARK_WORKLOAD_DIR ${CMAKE_CURRENT_SOURCE_DIR}) set(COREMARK_PORT_DIR ${CTEST_COREMARK_WORKLOAD_DIR}/buckyball) set(COREMARK_SRC_DIR ${CTEST_COREMARK_WORKLOAD_DIR}/src) -set(BBSIM_LD ${BBSW_BAREMETAL_DIR}/bbsim.ld) -list(GET BUCKYBALL_CPU_TARGETS 0 COREMARK_CORE) +set(COREMARK_THREAD ${BUCKYBALL_CONTROL_TARGET}) set(COREMARK_SOURCES - ${COREMARK_SRC_DIR}/core_list_join.c - ${COREMARK_SRC_DIR}/core_main.c - ${COREMARK_SRC_DIR}/core_matrix.c - ${COREMARK_SRC_DIR}/core_state.c - ${COREMARK_SRC_DIR}/core_util.c - ${COREMARK_PORT_DIR}/core_portme.c - ${COREMARK_PORT_DIR}/main.c + ${COREMARK_SRC_DIR}/core_list_join.c + ${COREMARK_SRC_DIR}/core_main.c + ${COREMARK_SRC_DIR}/core_matrix.c + ${COREMARK_SRC_DIR}/core_state.c + ${COREMARK_SRC_DIR}/core_util.c + ${COREMARK_PORT_DIR}/core_portme.c + ${COREMARK_PORT_DIR}/main.c ) set(COREMARK_HEADERS - ${COREMARK_SRC_DIR}/coremark.h - ${COREMARK_PORT_DIR}/core_portme.h + ${COREMARK_SRC_DIR}/coremark.h + ${COREMARK_PORT_DIR}/core_portme.h ) set(COREMARK_C_FLAGS - -g -fno-common -O2 -static -std=gnu99 -march=rv64gc -mcmodel=medany - -fno-builtin-printf -specs=nano.specs -specs=nosys.specs -nostartfiles - -Wl,-T,${BBSIM_LD} - -I${COREMARK_SRC_DIR} - -I${COREMARK_PORT_DIR} - -I${BUCKYBALL_ISA_DIR}/${COREMARK_CORE} - -I${BUCKYBALL_REPO_ROOT}/stack/runtime/include -I${WORKLOAD_LIB_DIR} - -ffunction-sections -fdata-sections - -Wl,--gc-sections - -DITERATIONS=10 - -DBUCKYBALL_WORKLOAD=1 + -g + -fno-common + -O2 + -static + -std=gnu99 + -march=rv64gc + -mcmodel=medany + -fno-builtin-printf + -specs=nano.specs + -specs=nosys.specs + -nostartfiles + -Wl,-T,${BBSIM_LD} + -I${COREMARK_SRC_DIR} + -I${COREMARK_PORT_DIR} + -DCORE_HART_ID=${DESIGN_WORKLOAD_CONTROL_HART} + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${WORKLOAD_LIB_DIR} + -ffunction-sections + -fdata-sections + -Wl,--gc-sections + -DITERATIONS=10 + -DBUCKYBALL_WORKLOAD=1 ) -function(add_coremark_target) - buckyball_ctest_name(COREMARK_NAME ${COREMARK_CORE} benchmark-coremark) - set(EXECUTABLE "${COREMARK_NAME}-baremetal") +set(COREMARK_NAME + "${DESIGN_NAME}-${COREMARK_THREAD}-ctest-benchmark-coremark" +) +set(EXECUTABLE "${COREMARK_NAME}-baremetal") - add_custom_command( +add_custom_command( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${EXECUTABLE} - COMMAND ${ELF_CC} ${COREMARK_C_FLAGS} -o ${EXECUTABLE} - ${BBSW_BAREMETAL_DIR}/crt0.S - ${BBSW_BAREMETAL_DIR}/syscalls.c - ${BBSW_BAREMETAL_DIR}/test.c - ${COREMARK_SOURCES} - -lm - DEPENDS ${COREMARK_SOURCES} - ${COREMARK_HEADERS} - ${BBSW_BAREMETAL_DIR}/crt0.S - ${BBSW_BAREMETAL_DIR}/syscalls.c - ${BBSW_BAREMETAL_DIR}/test.c - ${BUCKYBALL_ISA_DIR}/${COREMARK_CORE}/params.h + COMMAND + ${ELF_CC} ${COREMARK_C_FLAGS} -o ${EXECUTABLE} ${BAREMETAL_DIR}/crt0.S + ${BAREMETAL_DIR}/syscalls.c ${COREMARK_SOURCES} -lm + DEPENDS + ${COREMARK_SOURCES} + ${COREMARK_HEADERS} + ${BAREMETAL_DIR}/crt0.S + ${BAREMETAL_DIR}/syscalls.c COMMENT "Building coremark: ${EXECUTABLE}" WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - ) +) - add_custom_target(coremark-all +add_custom_target( + coremark-all DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${EXECUTABLE} - COMMENT "Building coremark for Buckyball") -endfunction() - -add_coremark_target() + COMMENT "Building coremark for Buckyball" +) diff --git a/bb-tests/workloads/src/CTest/coremark/buckyball/main.c b/bb-tests/workloads/src/CTest/coremark/buckyball/main.c index 92755e170..91ab33f0b 100644 --- a/bb-tests/workloads/src/CTest/coremark/buckyball/main.c +++ b/bb-tests/workloads/src/CTest/coremark/buckyball/main.c @@ -1,16 +1,3 @@ -#include -#include - int coremark_main(void); -int main(void) { - int ret = coremark_main(); - if (ret == 0) { - bb_test_pass(); - while (1) { - } - } - - printf("CoreMark failed with code %d\n", ret); - bb_test_fail(); -} +int main(void) { return coremark_main(); } diff --git a/bb-tests/workloads/src/CTest/coremark/src/core_state.c b/bb-tests/workloads/src/CTest/coremark/src/core_state.c index 7a7a91a8b..e956cb45e 100644 --- a/bb-tests/workloads/src/CTest/coremark/src/core_state.c +++ b/bb-tests/workloads/src/CTest/coremark/src/core_state.c @@ -28,8 +28,8 @@ Topic: Description implementation is used instead, trading speed of direct coding for ease of maintenance. - Since the main goal of using a state machine in CoreMark is to excercise -the switch/if behaviour, we are using a small moore machine. + Since the main goal of using a state machine in CoreMark is to +excercise the switch/if behaviour, we are using a small moore machine. In particular, this machine tests type of string input, trying to determine whether the input is a number or something else. diff --git a/bb-tests/workloads/src/CTest/coremark/src/coremark.h b/bb-tests/workloads/src/CTest/coremark/src/coremark.h index 8f80d25bc..f014a0b46 100644 --- a/bb-tests/workloads/src/CTest/coremark/src/coremark.h +++ b/bb-tests/workloads/src/CTest/coremark/src/coremark.h @@ -151,7 +151,7 @@ typedef struct RESULTS_S { core_portable port; } core_results; -/* Multicore execution handling */ +/* Multithread execution handling */ #if (MULTITHREAD > 1) ee_u8 core_start_parallel(core_results *res); ee_u8 core_stop_parallel(core_results *res); diff --git a/bb-tests/workloads/src/CTest/embench/CMakeLists.txt b/bb-tests/workloads/src/CTest/embench/CMakeLists.txt index 93a110695..0faa0d99b 100644 --- a/bb-tests/workloads/src/CTest/embench/CMakeLists.txt +++ b/bb-tests/workloads/src/CTest/embench/CMakeLists.txt @@ -1,174 +1,205 @@ -set(ELF_CC "riscv64-unknown-elf-gcc") -set(ELF_OBJCOPY "riscv64-unknown-elf-objcopy") - # Embench: baremetal benchmarks sharing toy's linker script and startup -# code in lib/bbsw/baremetal/. -set(CTEST_EMBENCH_WORKLOAD_DIR ${CTEST_WORKLOAD_DIR}/embench) +# code in stack/os/baremetal/. +set(CTEST_EMBENCH_WORKLOAD_DIR ${CMAKE_CURRENT_SOURCE_DIR}) set(EMBENCH_SUPPORT_DIR ${CTEST_EMBENCH_WORKLOAD_DIR}/support) set(EMBENCH_SRC_DIR ${CTEST_EMBENCH_WORKLOAD_DIR}/src) -set(BBSIM_LD ${BBSW_BAREMETAL_DIR}/bbsim.ld) -list(GET BUCKYBALL_CPU_TARGETS 0 EMBENCH_CORE) +set(EMBENCH_THREAD ${BUCKYBALL_CONTROL_TARGET}) set(EMBENCH_SUPPORT_SOURCES - ${EMBENCH_SUPPORT_DIR}/main.c - ${EMBENCH_SUPPORT_DIR}/beebsc.c - ${EMBENCH_SUPPORT_DIR}/boardsupport.c - ${EMBENCH_SUPPORT_DIR}/chipsupport.c + ${EMBENCH_SUPPORT_DIR}/main.c + ${EMBENCH_SUPPORT_DIR}/beebsc.c + ${EMBENCH_SUPPORT_DIR}/boardsupport.c + ${EMBENCH_SUPPORT_DIR}/chipsupport.c ) set(EMBENCH_TOP_SUPPORT_SOURCES - ${EMBENCH_SUPPORT_DIR}/top.c - ${EMBENCH_SUPPORT_DIR}/beebsc.c - ${EMBENCH_SUPPORT_DIR}/boardsupport.c - ${EMBENCH_SUPPORT_DIR}/chipsupport.c + ${EMBENCH_SUPPORT_DIR}/top.c + ${EMBENCH_SUPPORT_DIR}/beebsc.c + ${EMBENCH_SUPPORT_DIR}/boardsupport.c + ${EMBENCH_SUPPORT_DIR}/chipsupport.c ) set(EMBENCH_C_FLAGS - -g -fno-common -O2 -static -march=rv64gc -mcmodel=medany - -fno-builtin-printf -specs=nano.specs -specs=nosys.specs -nostartfiles - -Wl,-T,${BBSIM_LD} - -I${BUCKYBALL_REPO_ROOT}/stack/runtime/include -I${WORKLOAD_LIB_DIR} - -I${BUCKYBALL_ISA_DIR}/${EMBENCH_CORE} - -ffunction-sections -fdata-sections - -Wl,--gc-sections - -I${EMBENCH_SUPPORT_DIR} + -g + -fno-common + -O2 + -static + -march=rv64gc + -mcmodel=medany + -fno-builtin-printf + -specs=nano.specs + -specs=nosys.specs + -nostartfiles + -Wl,-T,${BBSIM_LD} + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${WORKLOAD_LIB_DIR} + -DCORE_HART_ID=${DESIGN_WORKLOAD_CONTROL_HART} + -ffunction-sections + -fdata-sections + -Wl,--gc-sections + -I${EMBENCH_SUPPORT_DIR} ) set(EMBENCH_TOP_OBJ_FLAGS - -g -fno-common -O2 -march=rv64gc -mcmodel=medany - -fno-builtin-printf - -I${BUCKYBALL_REPO_ROOT}/stack/runtime/include -I${WORKLOAD_LIB_DIR} - -I${BUCKYBALL_ISA_DIR}/${EMBENCH_CORE} - -ffunction-sections -fdata-sections - -I${EMBENCH_SUPPORT_DIR} + -g + -fno-common + -O2 + -march=rv64gc + -mcmodel=medany + -fno-builtin-printf + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${WORKLOAD_LIB_DIR} + -DCORE_HART_ID=${DESIGN_WORKLOAD_CONTROL_HART} + -ffunction-sections + -fdata-sections + -I${EMBENCH_SUPPORT_DIR} ) function(add_embench_target BMARK_NAME) - file(GLOB BMARK_SOURCES "${EMBENCH_SRC_DIR}/${BMARK_NAME}/*.c") - buckyball_ctest_name(EMBENCH_NAME ${EMBENCH_CORE} - benchmark-embench_${BMARK_NAME}) - set(EXECUTABLE "${EMBENCH_NAME}-baremetal") - - add_custom_command( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${EXECUTABLE} - COMMAND ${ELF_CC} ${EMBENCH_C_FLAGS} -o ${EXECUTABLE} - ${BBSW_BAREMETAL_DIR}/crt0.S - ${BBSW_BAREMETAL_DIR}/syscalls.c - ${EMBENCH_SUPPORT_SOURCES} - ${BMARK_SOURCES} - -lm - DEPENDS ${BMARK_SOURCES} ${EMBENCH_SUPPORT_SOURCES} - ${BBSIM_LD} - ${BBSW_BAREMETAL_DIR}/crt0.S - ${BBSW_BAREMETAL_DIR}/syscalls.c - COMMENT "Building embench: ${EXECUTABLE}" - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - ) - - add_custom_target(embench_${BMARK_NAME} - DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${EXECUTABLE} - COMMENT "Building embench ${BMARK_NAME}") - - if(BMARK_NAME STREQUAL "matmult-int" OR BMARK_NAME STREQUAL "sglib-combined") - buckyball_ctest_name(CHECK_NAME ${EMBENCH_CORE} check-embench_${BMARK_NAME}) - set(CHECK_EXECUTABLE ${CHECK_NAME}-baremetal) - set(CHECK_SUPPORT_SOURCES ${EMBENCH_SUPPORT_SOURCES}) - list(REMOVE_ITEM CHECK_SUPPORT_SOURCES ${EMBENCH_SUPPORT_DIR}/main.c) + file(GLOB BMARK_SOURCES "${EMBENCH_SRC_DIR}/${BMARK_NAME}/*.c") + set(EMBENCH_NAME + "${DESIGN_NAME}-${EMBENCH_THREAD}-ctest-benchmark-embench_${BMARK_NAME}" + ) + set(EXECUTABLE "${EMBENCH_NAME}-baremetal") + add_custom_command( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${CHECK_EXECUTABLE} - COMMAND ${ELF_CC} ${EMBENCH_C_FLAGS} -o ${CHECK_EXECUTABLE} - ${BBSW_BAREMETAL_DIR}/crt0.S ${BBSW_BAREMETAL_DIR}/syscalls.c - ${EMBENCH_SUPPORT_DIR}/check.c ${CHECK_SUPPORT_SOURCES} ${BMARK_SOURCES} -lm - DEPENDS ${EMBENCH_SUPPORT_DIR}/check.c ${CHECK_SUPPORT_SOURCES} ${BMARK_SOURCES} - ${BBSW_BAREMETAL_DIR}/crt0.S ${BBSW_BAREMETAL_DIR}/syscalls.c ${BBSIM_LD} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) - add_custom_target(embench_check_${BMARK_NAME} - DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${CHECK_EXECUTABLE}) - add_dependencies(embench_${BMARK_NAME} embench_check_${BMARK_NAME}) - endif() + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${EXECUTABLE} + COMMAND + ${ELF_CC} ${EMBENCH_C_FLAGS} -o ${EXECUTABLE} + ${BAREMETAL_DIR}/crt0.S ${BAREMETAL_DIR}/syscalls.c + ${EMBENCH_SUPPORT_SOURCES} ${BMARK_SOURCES} -lm + DEPENDS + ${BMARK_SOURCES} + ${EMBENCH_SUPPORT_SOURCES} + ${BBSIM_LD} + ${BAREMETAL_DIR}/crt0.S + ${BAREMETAL_DIR}/syscalls.c + COMMENT "Building embench: ${EXECUTABLE}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + + add_custom_target( + embench_${BMARK_NAME} + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${EXECUTABLE} + COMMENT "Building embench ${BMARK_NAME}" + ) + + if( + BMARK_NAME STREQUAL "matmult-int" + OR BMARK_NAME STREQUAL "sglib-combined" + ) + set(CHECK_NAME + "${DESIGN_NAME}-${EMBENCH_THREAD}-ctest-check-embench_${BMARK_NAME}" + ) + set(CHECK_EXECUTABLE ${CHECK_NAME}-baremetal) + set(CHECK_SUPPORT_SOURCES ${EMBENCH_SUPPORT_SOURCES}) + list(REMOVE_ITEM CHECK_SUPPORT_SOURCES ${EMBENCH_SUPPORT_DIR}/main.c) + add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${CHECK_EXECUTABLE} + COMMAND + ${ELF_CC} ${EMBENCH_C_FLAGS} -o ${CHECK_EXECUTABLE} + ${BAREMETAL_DIR}/crt0.S ${BAREMETAL_DIR}/syscalls.c + ${EMBENCH_SUPPORT_DIR}/check.c ${CHECK_SUPPORT_SOURCES} + ${BMARK_SOURCES} -lm + DEPENDS + ${EMBENCH_SUPPORT_DIR}/check.c + ${CHECK_SUPPORT_SOURCES} + ${BMARK_SOURCES} + ${BAREMETAL_DIR}/crt0.S + ${BAREMETAL_DIR}/syscalls.c + ${BBSIM_LD} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + add_custom_target( + embench_check_${BMARK_NAME} + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${CHECK_EXECUTABLE} + ) + add_dependencies(embench_${BMARK_NAME} embench_check_${BMARK_NAME}) + endif() endfunction() macro(add_embench_top_object BMARK_NAME PREFIX SOURCE_FILES) - set(OBJ "${CMAKE_CURRENT_BINARY_DIR}/${PREFIX}.top.o") - - add_custom_command( - OUTPUT ${OBJ} - COMMAND ${ELF_CC} -r ${EMBENCH_TOP_OBJ_FLAGS} - -Dbenchmark=${PREFIX}_benchmark - -Dinitialise_benchmark=${PREFIX}_initialise_benchmark - -Dwarm_caches=${PREFIX}_warm_caches - -Dverify_benchmark=${PREFIX}_verify_benchmark - -o ${OBJ} - ${SOURCE_FILES} - COMMAND ${ELF_OBJCOPY} - --keep-global-symbol=${PREFIX}_benchmark - --keep-global-symbol=${PREFIX}_initialise_benchmark - --keep-global-symbol=${PREFIX}_warm_caches - --keep-global-symbol=${PREFIX}_verify_benchmark - ${OBJ} - DEPENDS ${SOURCE_FILES} - COMMENT "Building embench top object: ${BMARK_NAME}" - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - ) - - list(APPEND EMBENCH_TOP_OBJECTS ${OBJ}) + set(OBJ "${CMAKE_CURRENT_BINARY_DIR}/${PREFIX}.top.o") + + add_custom_command( + OUTPUT ${OBJ} + COMMAND + ${ELF_CC} -r ${EMBENCH_TOP_OBJ_FLAGS} + -Dbenchmark=${PREFIX}_benchmark + -Dinitialise_benchmark=${PREFIX}_initialise_benchmark + -Dwarm_caches=${PREFIX}_warm_caches + -Dverify_benchmark=${PREFIX}_verify_benchmark -o ${OBJ} + ${SOURCE_FILES} + COMMAND + ${ELF_OBJCOPY} --keep-global-symbol=${PREFIX}_benchmark + --keep-global-symbol=${PREFIX}_initialise_benchmark + --keep-global-symbol=${PREFIX}_warm_caches + --keep-global-symbol=${PREFIX}_verify_benchmark ${OBJ} + DEPENDS ${SOURCE_FILES} + COMMENT "Building embench top object: ${BMARK_NAME}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + + list(APPEND EMBENCH_TOP_OBJECTS ${OBJ}) endmacro() macro(add_embench_top_benchmark BMARK_NAME PREFIX) - file(GLOB BMARK_SOURCES "${EMBENCH_SRC_DIR}/${BMARK_NAME}/*.c") - add_embench_top_object(${BMARK_NAME} ${PREFIX} "${BMARK_SOURCES}") + file(GLOB BMARK_SOURCES "${EMBENCH_SRC_DIR}/${BMARK_NAME}/*.c") + add_embench_top_object(${BMARK_NAME} ${PREFIX} "${BMARK_SOURCES}") endmacro() function(add_embench_top_target) - set(EMBENCH_TOP_OBJECTS) - - add_embench_top_benchmark(aha-mont64 aha_mont64) - add_embench_top_benchmark(crc32 crc32) - add_embench_top_benchmark(edn edn) - add_embench_top_benchmark(huffbench huffbench) - add_embench_top_benchmark(matmult-int matmult_int) - add_embench_top_benchmark(md5sum md5sum) - add_embench_top_benchmark(minver minver) - add_embench_top_benchmark(nbody nbody) - add_embench_top_benchmark(nettle-aes nettle_aes) - add_embench_top_benchmark(nettle-sha256 nettle_sha256) - add_embench_top_benchmark(nsichneu nsichneu) - add_embench_top_benchmark(picojpeg picojpeg) - add_embench_top_benchmark(primecount primecount) - add_embench_top_benchmark(qrduino qrduino) - add_embench_top_benchmark(sglib-combined sglib_combined) - add_embench_top_benchmark(slre slre) - add_embench_top_benchmark(st st) - add_embench_top_benchmark(statemate statemate) - add_embench_top_benchmark(tarfind tarfind) - add_embench_top_benchmark(ud ud) - add_embench_top_benchmark(wikisort wikisort) - - buckyball_ctest_name(EMBENCH_TOP_NAME ${EMBENCH_CORE} benchmark-embench_top) - set(EXECUTABLE "${EMBENCH_TOP_NAME}-baremetal") - add_custom_command( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${EXECUTABLE} - COMMAND ${ELF_CC} ${EMBENCH_C_FLAGS} -o ${EXECUTABLE} - ${BBSW_BAREMETAL_DIR}/crt0.S - ${BBSW_BAREMETAL_DIR}/syscalls.c - ${EMBENCH_TOP_SUPPORT_SOURCES} - ${EMBENCH_TOP_OBJECTS} - ${BBSW_BAREMETAL_DIR}/test.c - -lm - DEPENDS ${EMBENCH_TOP_SUPPORT_SOURCES} - ${BBSIM_LD} - ${EMBENCH_TOP_OBJECTS} - ${BBSW_BAREMETAL_DIR}/crt0.S - ${BBSW_BAREMETAL_DIR}/syscalls.c - ${BBSW_BAREMETAL_DIR}/test.c - COMMENT "Building embench top: ${EXECUTABLE}" - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - ) - - add_custom_target(embench_top DEPENDS - ${CMAKE_CURRENT_BINARY_DIR}/${EXECUTABLE} - COMMENT "Building embench top workload for Buckyball" - ) + set(EMBENCH_TOP_OBJECTS) + + add_embench_top_benchmark(aha-mont64 aha_mont64) + add_embench_top_benchmark(crc32 crc32) + add_embench_top_benchmark(edn edn) + add_embench_top_benchmark(huffbench huffbench) + add_embench_top_benchmark(matmult-int matmult_int) + add_embench_top_benchmark(md5sum md5sum) + add_embench_top_benchmark(minver minver) + add_embench_top_benchmark(nbody nbody) + add_embench_top_benchmark(nettle-aes nettle_aes) + add_embench_top_benchmark(nettle-sha256 nettle_sha256) + add_embench_top_benchmark(nsichneu nsichneu) + add_embench_top_benchmark(picojpeg picojpeg) + add_embench_top_benchmark(primecount primecount) + add_embench_top_benchmark(qrduino qrduino) + add_embench_top_benchmark(sglib-combined sglib_combined) + add_embench_top_benchmark(slre slre) + add_embench_top_benchmark(st st) + add_embench_top_benchmark(statemate statemate) + add_embench_top_benchmark(tarfind tarfind) + add_embench_top_benchmark(ud ud) + add_embench_top_benchmark(wikisort wikisort) + + set(EMBENCH_TOP_NAME + "${DESIGN_NAME}-${EMBENCH_THREAD}-ctest-benchmark-embench_top" + ) + set(EXECUTABLE "${EMBENCH_TOP_NAME}-baremetal") + add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${EXECUTABLE} + COMMAND + ${ELF_CC} ${EMBENCH_C_FLAGS} -o ${EXECUTABLE} + ${BAREMETAL_DIR}/crt0.S ${BAREMETAL_DIR}/syscalls.c + ${EMBENCH_TOP_SUPPORT_SOURCES} ${EMBENCH_TOP_OBJECTS} -lm + DEPENDS + ${EMBENCH_TOP_SUPPORT_SOURCES} + ${BBSIM_LD} + ${EMBENCH_TOP_OBJECTS} + ${BAREMETAL_DIR}/crt0.S + ${BAREMETAL_DIR}/syscalls.c + COMMENT "Building embench top: ${EXECUTABLE}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + + add_custom_target( + embench_top + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${EXECUTABLE} + COMMENT "Building embench top workload for Buckyball" + ) endfunction() #------------------------------------------------------------------------------- @@ -201,14 +232,31 @@ add_embench_target(wikisort) add_embench_top_target() # Master embench build target (excludes cubic due to long double link issue) -add_custom_target(embench-all DEPENDS - embench_aha-mont64 embench_crc32 embench_edn - embench_huffbench embench_matmult-int embench_md5sum embench_minver - embench_nbody - embench_nettle-aes embench_nettle-sha256 embench_nsichneu - embench_picojpeg embench_primecount embench_qrduino embench_sglib-combined - embench_slre embench_st embench_statemate embench_tarfind embench_ud - embench_wikisort - embench_top - COMMENT "Building all embench benchmarks for Buckyball (21/22, cubic excluded)" +add_custom_target( + embench-all + DEPENDS + embench_aha-mont64 + embench_crc32 + embench_edn + embench_huffbench + embench_matmult-int + embench_md5sum + embench_minver + embench_nbody + embench_nettle-aes + embench_nettle-sha256 + embench_nsichneu + embench_picojpeg + embench_primecount + embench_qrduino + embench_sglib-combined + embench_slre + embench_st + embench_statemate + embench_tarfind + embench_ud + embench_wikisort + embench_top + COMMENT + "Building all embench benchmarks for Buckyball (21/22, cubic excluded)" ) diff --git a/bb-tests/workloads/src/CTest/embench/support/chipsupport.c b/bb-tests/workloads/src/CTest/embench/support/chipsupport.c index 5c229a8d0..e0f3de0fe 100644 --- a/bb-tests/workloads/src/CTest/embench/support/chipsupport.c +++ b/bb-tests/workloads/src/CTest/embench/support/chipsupport.c @@ -1,4 +1,4 @@ -/* Buckyball chip support implementation for Embench +/* Buckyball design support implementation for Embench Adapted for Buckyball baremetal environment. @@ -6,11 +6,11 @@ #include "chipsupport.h" -/* Initialize chip +/* Initialize design - For Buckyball, no chip-specific initialization is needed. + For Buckyball, no design-specific initialization is needed. All necessary setup is done by crt0.S/start.S. */ -void initialise_chip(void) { - /* Empty - no chip-specific initialization needed */ +void initialise_design(void) { + /* Empty - no design-specific initialization needed */ } diff --git a/bb-tests/workloads/src/CTest/embench/support/chipsupport.h b/bb-tests/workloads/src/CTest/embench/support/chipsupport.h index 8c26ec4b4..d62d69ed3 100644 --- a/bb-tests/workloads/src/CTest/embench/support/chipsupport.h +++ b/bb-tests/workloads/src/CTest/embench/support/chipsupport.h @@ -1,4 +1,4 @@ -/* Buckyball chip support header for Embench +/* Buckyball design support header for Embench Adapted for Buckyball baremetal environment. @@ -7,7 +7,7 @@ #ifndef CHIPSUPPORT_H #define CHIPSUPPORT_H -/* Chip initialization function */ -void initialise_chip(void); +/* Design initialization function */ +void initialise_design(void); #endif /* CHIPSUPPORT_H */ diff --git a/bb-tests/workloads/src/CTest/rvv/rvv_kernel.c b/bb-tests/workloads/src/CTest/rvv/rvv_kernel.c index a54b6632a..9b50e31fa 100644 --- a/bb-tests/workloads/src/CTest/rvv/rvv_kernel.c +++ b/bb-tests/workloads/src/CTest/rvv/rvv_kernel.c @@ -23,16 +23,16 @@ int main(void) { bb_mem_transfer(2, 4); bb_mem_transfer(0, 5); bb_mem_transfer(1, 5); - mvin_kernel(image + 1, sizeof(image) - 4, VIRTUAL_BANK_NUM); - run_kernel(4, VIRTUAL_BANK_NUM, 5, 0); + mvin_kernel(image + 1, sizeof(image) - 4, CORE_VIRTUAL_BANK_COUNT); + run_kernel(4, CORE_VIRTUAL_BANK_COUNT, 5, 0); call.args[0] = (uint64_t)4 << 32; call.args[1] = (uint64_t)1 << 32; bb_mvin_group((uintptr_t)&call, 5, 0, sizeof(call) / 16, 1); - run_kernel(4, VIRTUAL_BANK_NUM, 5, 0); + run_kernel(4, CORE_VIRTUAL_BANK_COUNT, 5, 0); uint32_t groups[2][64] __attribute__((aligned(16))); bb_mvout_group((uintptr_t)groups[0], 5, 1, 16, 1); bb_mvout_group((uintptr_t)groups[1], 5, 2, 16, 1); - release_kernel(VIRTUAL_BANK_NUM); + release_kernel(CORE_VIRTUAL_BANK_COUNT); bb_mem_release(4); bb_mem_release(5); for (unsigned row = 0; row < 16; ++row) diff --git a/bb-tests/workloads/src/CTest/tlb/CMakeLists.txt b/bb-tests/workloads/src/CTest/tlb/CMakeLists.txt index b6088f0aa..ead8a138b 100644 --- a/bb-tests/workloads/src/CTest/tlb/CMakeLists.txt +++ b/bb-tests/workloads/src/CTest/tlb/CMakeLists.txt @@ -1,14 +1,12 @@ add_custom_target(tlb-ctests-build) set(BUCKYBALL_CTEST_GROUP_TARGET tlb-ctests-build) -foreach(BUCKYBALL_CTEST_TARGET IN LISTS BUCKYBALL_CPU_TARGETS) - add_buckyball_ctests( - tlb_bare_mode_test.c - tlb_repeated_sfence_test.c - tlb_sfence_flush_test.c - tlb_sfence_remap_test.c - tlb_sv39_4k_alias_test.c - tlb_sv39_identity_test.c - tlb_sv39_nonidentity_test.c - ) +foreach(BUCKYBALL_CTEST_TARGET IN LISTS DESIGN_CPU_TARGETS) + add_buckyball_ctest(tlb_bare_mode_test.c) + add_buckyball_ctest(tlb_repeated_sfence_test.c) + add_buckyball_ctest(tlb_sfence_flush_test.c) + add_buckyball_ctest(tlb_sfence_remap_test.c) + add_buckyball_ctest(tlb_sv39_4k_alias_test.c) + add_buckyball_ctest(tlb_sv39_identity_test.c) + add_buckyball_ctest(tlb_sv39_nonidentity_test.c) endforeach() diff --git a/bb-tests/workloads/src/CTest/tlb/tlb_common.h b/bb-tests/workloads/src/CTest/tlb/tlb_common.h index a11b6c2c1..0ec971df8 100644 --- a/bb-tests/workloads/src/CTest/tlb/tlb_common.h +++ b/bb-tests/workloads/src/CTest/tlb/tlb_common.h @@ -28,6 +28,9 @@ #define TLB_SATP_MODE_BARE 0UL #define TLB_PMPCFG_NAPOT 0x18UL #define TLB_PMPCFG_RWX 0x07UL +#define TLB_MSTATUS_MPRV (1UL << 17) +#define TLB_MSTATUS_MPP (3UL << 11) +#define TLB_MSTATUS_MPP_S (1UL << 11) #define TLB_DIM 16 @@ -116,10 +119,15 @@ static void tlb_enable_sv39(void) { tlb_fence_rw(); asm volatile("csrw satp, %0" ::"r"(satp_val) : "memory"); asm volatile("sfence.vma" ::: "memory"); + uintptr_t status; + asm volatile("csrr %0, mstatus" : "=r"(status)); + status = (status & ~TLB_MSTATUS_MPP) | TLB_MSTATUS_MPRV | TLB_MSTATUS_MPP_S; + asm volatile("csrw mstatus, %0" ::"r"(status) : "memory"); tlb_fence_rw(); } static void tlb_disable_vm(void) { + asm volatile("csrc mstatus, %0" ::"r"(TLB_MSTATUS_MPRV) : "memory"); uint64_t satp_val = TLB_SATP_MODE_BARE << 60; tlb_fence_rw(); asm volatile("csrw satp, %0" ::"r"(satp_val) : "memory"); @@ -160,8 +168,8 @@ static int tlb_finish(const char *name, int passed) { } static void tlb_select_hart(void) { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif } diff --git a/bb-tests/workloads/src/CTest/tutorial/CMakeLists.txt b/bb-tests/workloads/src/CTest/tutorial/CMakeLists.txt index 9f2607c76..2b4259f3d 100644 --- a/bb-tests/workloads/src/CTest/tutorial/CMakeLists.txt +++ b/bb-tests/workloads/src/CTest/tutorial/CMakeLists.txt @@ -1,14 +1,18 @@ add_custom_target(tutorial-ctests-build) set(BUCKYBALL_CTEST_GROUP_TARGET tutorial-ctests-build) -# Baremetal single-core only (matches Bazel //bb-tests/.../CTest/tutorial). -foreach(BUCKYBALL_CTEST_TARGET IN LISTS BUCKYBALL_CPU_TARGETS) - buckyball_ctest_name(BUCKYBALL_TUTORIAL_TEST - ${BUCKYBALL_CTEST_TARGET} tutorial) - add_buckyball_baremetal_ctest( - ${BUCKYBALL_TUTORIAL_TEST} - ${CMAKE_CURRENT_SOURCE_DIR} - tutorial.c) - add_dependencies(tutorial-ctests-build - ${BUCKYBALL_TUTORIAL_TEST}-baremetal-build) +# Runs on the selected bare-metal hart. +foreach(BUCKYBALL_CTEST_TARGET IN LISTS DESIGN_CPU_TARGETS) + set(BUCKYBALL_TUTORIAL_TEST + "${DESIGN_NAME}-${BUCKYBALL_CTEST_TARGET}-ctest-tutorial" + ) + add_buckyball_baremetal_ctest( + ${BUCKYBALL_TUTORIAL_TEST} + ${CMAKE_CURRENT_SOURCE_DIR} + tutorial.c + ) + add_dependencies( + tutorial-ctests-build + ${BUCKYBALL_TUTORIAL_TEST}-baremetal-build + ) endforeach() diff --git a/bb-tests/workloads/src/MLIRTest/CMakeLists.txt b/bb-tests/workloads/src/MLIRTest/CMakeLists.txt index 2f4424e0f..a48fffd69 100644 --- a/bb-tests/workloads/src/MLIRTest/CMakeLists.txt +++ b/bb-tests/workloads/src/MLIRTest/CMakeLists.txt @@ -1,315 +1,244 @@ -include(${CMAKE_CURRENT_LIST_DIR}/compiler/CMakeLists.txt) -set(MLIRTEST_WORKLOAD_DIR ${CMAKE_CURRENT_SOURCE_DIR}) - -# The workload root chooses the compiler instance for this chip's core. -if(NOT DEFINED BUDDY_BINARY_DIR) - message(FATAL_ERROR - "BUDDY_BINARY_DIR must be set by the workload root.") -endif() - -if(NOT DEFINED LLVM_BUILD_DIR) - message(FATAL_ERROR "LLVM_BUILD_DIR is not set") -endif() - -if(NOT DEFINED BUDDY_OPT) - message(FATAL_ERROR "BUDDY_OPT is not set") -endif() - # Bank-level / ball-op MLIR -> baremetal + linux ELF (same pipeline as toy MLIRTest). function(buckyball_enforce_mlir_test_line_limit SOURCE_FILE) - if(NOT EXISTS "${SOURCE_FILE}") - message(FATAL_ERROR "MLIR test source does not exist: ${SOURCE_FILE}") - endif() - file(READ "${SOURCE_FILE}" _content) - string(REGEX MATCHALL "\n" _newlines "${_content}") - list(LENGTH _newlines _lines) - string(REGEX MATCH "[^\n]$" _trailing_content "${_content}") - if(_trailing_content) - math(EXPR _lines "${_lines} + 1") - endif() - if(_lines GREATER 100) - message(FATAL_ERROR - "MLIR test exceeds 100 lines: ${SOURCE_FILE} (${_lines} lines). " - "Split it into one focused test per file.") - endif() -endfunction() - -function(buckyball_require_smatmul_bank_shape TARGET) - set(_params "${BUCKYBALL_ISA_DIR}/${TARGET}/params.h") - file(STRINGS "${_params}" _bank_lines REGEX "^#define BANK_LINES ") - file(STRINGS "${_params}" _bank_width REGEX "^#define BANK_WIDTH ") - string(REGEX REPLACE ".* " "" _lines "${_bank_lines}") - string(REGEX REPLACE ".* " "" _width "${_bank_width}") - math(EXPR _line_remainder "${_lines} % 64") - if(NOT (_width EQUAL 128) OR _lines LESS 64 OR - NOT (_line_remainder EQUAL 0)) - message(FATAL_ERROR - "SMatMul BankOp test requires BANK_WIDTH=128 and BANK_LINES a multiple " - "of 64 (got width=${_width}, lines=${_lines})") - endif() -endfunction() - -function(buckyball_require_quant_i32_bank_shape TARGET) - set(_params "${BUCKYBALL_ISA_DIR}/${TARGET}/params.h") - file(STRINGS "${_params}" _bank_lines REGEX "^#define BANK_LINES ") - string(REGEX REPLACE ".* " "" _lines "${_bank_lines}") - math(EXPR _remainder "${_lines} % 4") - if(_lines LESS 4 OR NOT (_remainder EQUAL 0)) - message(FATAL_ERROR - "ToInt8 INT32 BankOp test requires BANK_LINES divisible by four " - "(got lines=${_lines})") - endif() -endfunction() - -function(buckyball_require_mxfp2int_bank_shape TARGET) - set(_params "${BUCKYBALL_ISA_DIR}/${TARGET}/params.h") - file(STRINGS "${_params}" _bank_lines REGEX "^#define BANK_LINES ") - string(REGEX REPLACE ".* " "" _lines "${_bank_lines}") - math(EXPR _remainder "${_lines} % 32") - if(_lines LESS 32 OR NOT (_remainder EQUAL 0)) - message(FATAL_ERROR - "Mxfp2Int BankOp test requires BANK_LINES divisible by 32 " - "(got lines=${_lines})") - endif() -endfunction() - -function(buckyball_require_min_bank_lines TARGET MINIMUM BALL) - set(_params "${BUCKYBALL_ISA_DIR}/${TARGET}/params.h") - file(STRINGS "${_params}" _bank_lines REGEX "^#define BANK_LINES ") - string(REGEX REPLACE ".* " "" _lines "${_bank_lines}") - if(_lines LESS MINIMUM) - message(FATAL_ERROR - "${BALL} BankOp test requires BANK_LINES >= ${MINIMUM} (got ${_lines})") - endif() + if(NOT EXISTS "${SOURCE_FILE}") + message(FATAL_ERROR "MLIR test source does not exist: ${SOURCE_FILE}") + endif() + file(READ "${SOURCE_FILE}" _content) + string(REGEX MATCHALL "\n" _newlines "${_content}") + list(LENGTH _newlines _lines) + string(REGEX MATCH "[^\n]$" _trailing_content "${_content}") + if(_trailing_content) + math(EXPR _lines "${_lines} + 1") + endif() + if(_lines GREATER 100) + message( + FATAL_ERROR + "MLIR test exceeds 100 lines: ${SOURCE_FILE} (${_lines} lines). " + "Split it into one focused test per file." + ) + endif() endfunction() function(add_buckyball_mlir_test NAME) - cmake_parse_arguments(ARG "" "TARGET;CRUNNER_SOURCE" "PASSES;LIBRARIES;BAREMETAL_LIBRARIES" ${ARGN}) - - if(NOT DEFINED BUCKYBALL_MLIR_TEST_PREFIX) - message(FATAL_ERROR "BUCKYBALL_MLIR_TEST_PREFIX is not set") - endif() - if(BUCKYBALL_WORKLOAD_CHIP STREQUAL "") - message(FATAL_ERROR "BUCKYBALL_WORKLOAD_CHIP must be set before mlir tests") - endif() - if(NOT ARG_TARGET) - message(FATAL_ERROR "${NAME}: TARGET is required") - endif() - list(FIND BUCKYBALL_TARGETS "${ARG_TARGET}" TARGET_INDEX) - if(TARGET_INDEX EQUAL -1) - message(FATAL_ERROR "${NAME}: unknown Buckyball target ${ARG_TARGET}") - endif() - buckyball_assign_physical_banks(${ARG_TARGET} ASSIGN_PHYSICAL_BANKS) - set(BB_BALL_ISA_INCLUDE_DIR "${BUCKYBALL_ISA_DIR}/${ARG_TARGET}") - set(BB_BALL_ISA_H "${BB_BALL_ISA_INCLUDE_DIR}/ballISA.h") - set(BB_PARAMS_H "${BB_BALL_ISA_INCLUDE_DIR}/params.h") - if(NOT EXISTS "${BB_BALL_ISA_H}") - message(FATAL_ERROR "${NAME}: missing target ISA header ${BB_BALL_ISA_H}") - endif() - if(NOT EXISTS "${BB_PARAMS_H}") - message(FATAL_ERROR "${NAME}: missing target params.h ${BB_PARAMS_H}") - endif() - set(MLIR_SRC ${CMAKE_CURRENT_SOURCE_DIR}/${NAME}.mlir) - set(MLIR_TEMPLATE ${CMAKE_CURRENT_SOURCE_DIR}/${NAME}.mlir.in) - if(EXISTS ${MLIR_TEMPLATE}) - file(STRINGS ${BB_PARAMS_H} _bank_lines REGEX "^#define BANK_LINES ") - file(STRINGS ${BB_PARAMS_H} _bank_width REGEX "^#define BANK_WIDTH ") - file(STRINGS ${BB_PARAMS_H} _bank_isqrt REGEX "^#define BANK_ISQRT ") - string(REGEX REPLACE ".* " "" BB_BANK_LINES "${_bank_lines}") - string(REGEX REPLACE ".* " "" BB_BANK_WIDTH_BITS "${_bank_width}") - string(REGEX REPLACE ".* " "" BB_BANK_ISQRT "${_bank_isqrt}") - math(EXPR BB_BANK_I8_LANES "${BB_BANK_WIDTH_BITS} / 8") - math(EXPR BB_BANK_I32_LANES "${BB_BANK_WIDTH_BITS} / 32") - math(EXPR BB_BANK_SQUARE "${BB_BANK_ISQRT} * ${BB_BANK_ISQRT}") - set(BB_IM2COL_SIDE ${BB_BANK_ISQRT}) - if(BB_IM2COL_SIDE GREATER 16) - set(BB_IM2COL_SIDE 16) + cmake_parse_arguments( + ARG + "" + "TARGET" + "PASSES;LIBRARIES;BAREMETAL_LIBRARIES" + ${ARGN} + ) + + if(NOT DEFINED BUCKYBALL_MLIR_TEST_PREFIX) + message(FATAL_ERROR "BUCKYBALL_MLIR_TEST_PREFIX is not set") endif() - math(EXPR BB_IM2COL_INPUT_ROWS "${BB_IM2COL_SIDE} * ${BB_IM2COL_SIDE}") - math(EXPR BB_SMM_M "${BB_BANK_LINES} / 4") - math(EXPR BB_SMM_CONFIG "${BB_SMM_M} | (16 << 12) | (16 << 24)") - # QUANT_I32_TO_I8 has a seven-bit outputHeight field, so one instruction - # can consume at most 508 packed INT32 bank rows. - math(EXPR BB_QUANT_I32_INPUT_ROWS "${BB_BANK_LINES} - (${BB_BANK_LINES} % 4)") - if(BB_QUANT_I32_INPUT_ROWS GREATER 508) - set(BB_QUANT_I32_INPUT_ROWS 508) + if(NOT ARG_TARGET) + message(FATAL_ERROR "${NAME}: TARGET is required") endif() - math(EXPR BB_QUANT_I32_OUTPUT_ROWS "${BB_QUANT_I32_INPUT_ROWS} / 4") - math(EXPR BB_MXFP_BLOCKS "${BB_BANK_LINES} / 2") - math(EXPR BB_MXFP_MMIO_ROWS "${BB_MXFP_BLOCKS} / 16") - if(BB_BANK_LINES LESS 1 OR BB_BANK_ISQRT LESS 1 OR BB_BANK_I8_LANES LESS 1 OR - BB_BANK_I32_LANES LESS 1) - message(FATAL_ERROR "${NAME}: invalid generated bank parameters") + list(FIND DESIGN_PROFILES "${ARG_TARGET}" TARGET_INDEX) + if(TARGET_INDEX EQUAL -1) + message(FATAL_ERROR "${NAME}: unknown Buckyball target ${ARG_TARGET}") endif() - set(MLIR_SRC ${CMAKE_CURRENT_BINARY_DIR}/${ARG_TARGET}-${NAME}.mlir) - configure_file(${MLIR_TEMPLATE} ${MLIR_SRC} @ONLY) - endif() - set(MAIN_SRC ${CMAKE_CURRENT_SOURCE_DIR}/${NAME}_main.cpp) - buckyball_enforce_mlir_test_line_limit(${MLIR_SRC}) - buckyball_enforce_mlir_test_line_limit(${MAIN_SRC}) - set(OBJ ${CMAKE_CURRENT_BINARY_DIR}/${ARG_TARGET}-${NAME}.o) - set(PREFIX ${BUCKYBALL_MLIR_TEST_PREFIX}) - set(TEST_CASE ${NAME}) - if(NAME MATCHES "^${PREFIX}_(.+)$") - set(TEST_CASE ${CMAKE_MATCH_1}) - endif() - buckyball_mlirtest_name(TEST_NAME ${ARG_TARGET} ${PREFIX}_${TEST_CASE}) - set(BAREMETAL_BIN ${TEST_NAME}-baremetal) - set(LINUX_BIN ${TEST_NAME}-linux) - set(BAREMETAL_TARGET ${BAREMETAL_BIN}-build) - set(LINUX_TARGET ${LINUX_BIN}-build) - set(GROUP_TARGET ${TEST_NAME}) - - set(BBSIM_LD ${BBSW_BAREMETAL_DIR}/bbsim.ld) - set(C_FLAGS -g -fno-common -O2 -static -march=rv64gc -mcmodel=medany - -fno-builtin-printf -specs=nano.specs -specs=nosys.specs -nostartfiles - -DBAREMETAL -Wl,-T,${BBSIM_LD}) - set(LINUX_CXX ${RISCV_GNU_TOOLCHAIN}/bin/riscv64-unknown-linux-gnu-g++) - set(LINUX_FLAGS -static -Wl,--no-dynamic-linker -march=rv64gc) - set(CRUNNER_UTILS_SRC ${WORKLOAD_LIB_DIR}/bbsw/CRunnerUtils/CRunnerUtils.cpp) - if(ARG_CRUNNER_SOURCE) - set(CRUNNER_UTILS_SRC "${ARG_CRUNNER_SOURCE}") - endif() - set(BBHW_MEM_C ${BUCKYBALL_REPO_ROOT}/stack/runtime/src/dma.c) - set(LLVM_MLIR_EXECUTION_ENGINE_DIR - ${BUDDY_MLIR_DIR}/llvm/mlir/include/mlir/ExecutionEngine) - - if(ARG_PASSES) - set(MLIR_PASSES) - foreach(PASS IN LISTS ARG_PASSES) - if(PASS STREQUAL "-assign-physical-banks") - list(APPEND MLIR_PASSES ${ASSIGN_PHYSICAL_BANKS}) - else() - list(APPEND MLIR_PASSES ${PASS}) - endif() - endforeach() - else() - set(MLIR_PASSES - ${ASSIGN_PHYSICAL_BANKS} - ${BUCKYBALL_LOWER_BANK_SSA_TO_INTRINSICS} - -convert-linalg-to-loops - -expand-strided-metadata - -lower-affine - -convert-scf-to-cf - -convert-cf-to-llvm - ${BUCKYBALL_LOWER_BUCKYBALL} - -convert-arith-to-llvm - -convert-math-to-llvm - -finalize-memref-to-llvm - -convert-func-to-llvm - -reconcile-unrealized-casts) - endif() - string(JOIN " " MLIR_PASS_COMMAND ${MLIR_PASSES}) - - add_custom_command( - OUTPUT ${OBJ} - COMMAND bash -o pipefail -c - "${BUDDY_OPT} ${MLIR_SRC} --target=${ARG_TARGET} ${MLIR_PASS_COMMAND} | ${BUDDY_TRANSLATE} --buddy-to-llvmir | ${BUDDY_LLC} -filetype=obj -mtriple=riscv64 -O2 -code-model=medium -mattr=${BUCKYBALL_RISCV_MATTR} -float-abi=hard -o ${OBJ}" - DEPENDS ${MLIR_SRC} ${BUDDY_OPT} - COMMENT "Building ${NAME}.o from ${PREFIX} MLIR" - VERBATIM) - - set(MLIR_LINK_LIBRARIES) - foreach(LIBRARY IN LISTS ARG_LIBRARIES) - if(NOT TARGET ${LIBRARY}) - message(FATAL_ERROR "${NAME}: unknown runtime library ${LIBRARY}") + if(NOT DESIGN_PROFILES_${ARG_TARGET}_THREAD_MEM_BANK_NUM GREATER 0) + message(FATAL_ERROR "${ARG_TARGET}: no physical banks") endif() - list(APPEND MLIR_LINK_LIBRARIES "$") - endforeach() - - set(MLIR_BAREMETAL_LIBRARIES ${MLIR_LINK_LIBRARIES}) - if(ARG_BAREMETAL_LIBRARIES) - set(MLIR_BAREMETAL_LIBRARIES) - foreach(LIBRARY IN LISTS ARG_BAREMETAL_LIBRARIES) - if(NOT TARGET ${LIBRARY}) - message(FATAL_ERROR "${NAME}: unknown bare-metal library ${LIBRARY}") - endif() - list(APPEND MLIR_BAREMETAL_LIBRARIES "$") + set(ASSIGN_PHYSICAL_BANKS -assign-physical-banks) + set(BIN "${DESIGN_COMPILER}/bin") + # buddy-opt with this target's compiler plugins. + string(JOIN " " OPT ${DESIGN_TARGET_OPT_${ARG_TARGET}}) + # The banks and Ball ISA of the target, as compiler flags. + set(TARGET_DEFINES ${DESIGN_TARGET_DEFS_${ARG_TARGET}}) + list(TRANSFORM TARGET_DEFINES PREPEND "-D") + set(MLIR_SRC ${CMAKE_CURRENT_SOURCE_DIR}/${NAME}.mlir) + set(MLIR_TEMPLATE ${CMAKE_CURRENT_SOURCE_DIR}/${NAME}.mlir.in) + if(EXISTS ${MLIR_TEMPLATE}) + set(BB_BANK_LINES ${DESIGN_PROFILES_${ARG_TARGET}_THREAD_MEM_BANK_ENTRIES}) + set(BB_BANK_WIDTH_BITS ${DESIGN_PROFILES_${ARG_TARGET}_THREAD_MEM_BANK_WIDTH}) + math(EXPR BB_BANK_I8_LANES "${BB_BANK_WIDTH_BITS} / 8") + math(EXPR BB_BANK_I32_LANES "${BB_BANK_WIDTH_BITS} / 32") + if( + BB_BANK_LINES LESS 1 + OR BB_BANK_I8_LANES LESS 1 + OR BB_BANK_I32_LANES LESS 1 + ) + message(FATAL_ERROR "${NAME}: invalid generated bank parameters") + endif() + set(MLIR_SRC ${CMAKE_CURRENT_BINARY_DIR}/${ARG_TARGET}-${NAME}.mlir) + configure_file(${MLIR_TEMPLATE} ${MLIR_SRC} @ONLY) + endif() + set(MAIN_SRC ${CMAKE_CURRENT_SOURCE_DIR}/${NAME}_main.cpp) + buckyball_enforce_mlir_test_line_limit(${MLIR_SRC}) + buckyball_enforce_mlir_test_line_limit(${MAIN_SRC}) + set(OBJ ${CMAKE_CURRENT_BINARY_DIR}/${ARG_TARGET}-${NAME}.o) + set(PREFIX ${BUCKYBALL_MLIR_TEST_PREFIX}) + set(TEST_CASE ${NAME}) + if(NAME MATCHES "^${PREFIX}_(.+)$") + set(TEST_CASE ${CMAKE_MATCH_1}) + endif() + set(TEST_NAME + "${DESIGN_NAME}-${ARG_TARGET}-mlirtest-${PREFIX}_${TEST_CASE}" + ) + set(BAREMETAL_BIN ${TEST_NAME}-baremetal) + set(LINUX_BIN ${TEST_NAME}-linux) + set(BAREMETAL_TARGET ${BAREMETAL_BIN}-build) + set(LINUX_TARGET ${LINUX_BIN}-build) + set(GROUP_TARGET ${TEST_NAME}) + + set(C_FLAGS + -g + -fno-common + -O2 + -static + -march=rv64gc + -mcmodel=medany + -fno-builtin-printf + -specs=nano.specs + -specs=nosys.specs + -nostartfiles + -DBAREMETAL + -Wl,-T,${BBSIM_LD} + ) + set(LINUX_FLAGS -static -Wl,--no-dynamic-linker -march=rv64gc) + set(CRUNNER_UTILS_SRC + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/mlir.cpp + ) + set(BBHW_MEM_C ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/dma.c) + set(LLVM_MLIR_EXECUTION_ENGINE_DIR + ${BUDDY_MLIR_DIR}/llvm/mlir/include/mlir/ExecutionEngine + ) + + if(ARG_PASSES) + set(MLIR_PASSES ${ARG_PASSES}) + else() + set(MLIR_PASSES + ${ASSIGN_PHYSICAL_BANKS} + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-arith-to-llvm + -convert-math-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts + ) + endif() + string(JOIN " " MLIR_PASS_COMMAND ${MLIR_PASSES}) + + add_custom_command( + OUTPUT ${OBJ} + COMMAND + bash -o pipefail -c + "${OPT} ${MLIR_SRC} ${MLIR_PASS_COMMAND} | ${BIN}/buddy-translate --buddy-to-llvmir | ${BIN}/buddy-llc -filetype=obj -mtriple=riscv64 -O2 -code-model=medium -mattr=${BUCKYBALL_RISCV_MATTR} -float-abi=hard -o ${OBJ}" + DEPENDS ${MLIR_SRC} ${DESIGN_TARGET_PLUGIN_${ARG_TARGET}} + COMMENT "Building ${NAME}.o from ${PREFIX} MLIR" + VERBATIM + ) + + set(MLIR_LINK_LIBRARIES) + foreach(LIBRARY IN LISTS ARG_LIBRARIES) + if(NOT TARGET ${LIBRARY}) + message(FATAL_ERROR "${NAME}: unknown runtime library ${LIBRARY}") + endif() + list(APPEND MLIR_LINK_LIBRARIES "$") endforeach() - endif() - - add_custom_command( - OUTPUT ${BAREMETAL_BIN} - COMMAND ${ELF_CC} ${C_FLAGS} - -I${LLVM_MLIR_EXECUTION_ENGINE_DIR} - -I${BUCKYBALL_REPO_ROOT}/stack/runtime/include -I${WORKLOAD_LIB_DIR} - -I${BB_BALL_ISA_INCLUDE_DIR} - -o ${CMAKE_CURRENT_BINARY_DIR}/${BAREMETAL_BIN} - ${BBSW_BAREMETAL_DIR}/crt0.S - ${BBSW_BAREMETAL_DIR}/syscalls.c - ${CRUNNER_UTILS_SRC} - ${BBHW_MEM_C} - ${MAIN_SRC} - ${OBJ} ${MLIR_BAREMETAL_LIBRARIES} - DEPENDS ${OBJ} ${ARG_BAREMETAL_LIBRARIES} ${ARG_LIBRARIES} ${MAIN_SRC} ${CRUNNER_UTILS_SRC} ${BBHW_MEM_C} - ${BB_BALL_ISA_H} ${BB_PARAMS_H} - ${BBSW_BAREMETAL_DIR}/crt0.S ${BBSW_BAREMETAL_DIR}/syscalls.c - ${BBSIM_LD} - COMMENT "Linking ${BAREMETAL_BIN}" - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - VERBATIM) - add_custom_command( - OUTPUT ${LINUX_BIN} - COMMAND ${LINUX_CXX} ${LINUX_FLAGS} - -I${LLVM_MLIR_EXECUTION_ENGINE_DIR} - -I${BUCKYBALL_REPO_ROOT}/stack/runtime/include -I${WORKLOAD_LIB_DIR} - -I${BB_BALL_ISA_INCLUDE_DIR} - -o ${CMAKE_CURRENT_BINARY_DIR}/${LINUX_BIN} - ${CRUNNER_UTILS_SRC} - -x c ${BBHW_MEM_C} -x none - ${MAIN_SRC} - ${OBJ} ${MLIR_LINK_LIBRARIES} - DEPENDS ${OBJ} ${ARG_LIBRARIES} ${MAIN_SRC} ${CRUNNER_UTILS_SRC} ${BBHW_MEM_C} ${BB_BALL_ISA_H} - ${BB_PARAMS_H} - COMMENT "Linking ${LINUX_BIN}" - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - VERBATIM) - - add_custom_target(${BAREMETAL_TARGET} - DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${BAREMETAL_BIN}) - add_custom_target(${LINUX_TARGET} - DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${LINUX_BIN}) - add_custom_target(${GROUP_TARGET} - DEPENDS ${BAREMETAL_TARGET} ${LINUX_TARGET}) + set(MLIR_BAREMETAL_LIBRARIES ${MLIR_LINK_LIBRARIES}) + if(ARG_BAREMETAL_LIBRARIES) + set(MLIR_BAREMETAL_LIBRARIES) + foreach(LIBRARY IN LISTS ARG_BAREMETAL_LIBRARIES) + if(NOT TARGET ${LIBRARY}) + message( + FATAL_ERROR + "${NAME}: unknown bare-metal library ${LIBRARY}" + ) + endif() + list(APPEND MLIR_BAREMETAL_LIBRARIES "$") + endforeach() + endif() - if(DEFINED BUCKYBALL_MLIR_GROUP_TARGET) - add_dependencies(${BUCKYBALL_MLIR_GROUP_TARGET} ${GROUP_TARGET}) - endif() + add_custom_command( + OUTPUT ${BAREMETAL_BIN} + COMMAND + ${ELF_CC} ${C_FLAGS} -I${LLVM_MLIR_EXECUTION_ENGINE_DIR} + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${WORKLOAD_LIB_DIR} ${TARGET_DEFINES} -o + ${CMAKE_CURRENT_BINARY_DIR}/${BAREMETAL_BIN} ${BAREMETAL_DIR}/crt0.S + ${BAREMETAL_DIR}/syscalls.c ${CRUNNER_UTILS_SRC} ${BBHW_MEM_C} + ${MAIN_SRC} ${OBJ} ${MLIR_BAREMETAL_LIBRARIES} + DEPENDS + ${OBJ} + ${ARG_BAREMETAL_LIBRARIES} + ${ARG_LIBRARIES} + ${MAIN_SRC} + ${CRUNNER_UTILS_SRC} + ${BBHW_MEM_C} + ${BAREMETAL_DIR}/crt0.S + ${BAREMETAL_DIR}/syscalls.c + ${BBSIM_LD} + COMMENT "Linking ${BAREMETAL_BIN}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM + ) + + add_custom_command( + OUTPUT ${LINUX_BIN} + COMMAND + ${LINUX_CXX} ${LINUX_FLAGS} -I${LLVM_MLIR_EXECUTION_ENGINE_DIR} + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${WORKLOAD_LIB_DIR} ${TARGET_DEFINES} -o + ${CMAKE_CURRENT_BINARY_DIR}/${LINUX_BIN} ${CRUNNER_UTILS_SRC} -x c + ${BBHW_MEM_C} -x none ${MAIN_SRC} ${OBJ} ${MLIR_LINK_LIBRARIES} + DEPENDS + ${OBJ} + ${ARG_LIBRARIES} + ${MAIN_SRC} + ${CRUNNER_UTILS_SRC} + ${BBHW_MEM_C} + COMMENT "Linking ${LINUX_BIN}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM + ) + + add_custom_target( + ${BAREMETAL_TARGET} + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${BAREMETAL_BIN} + ) + add_custom_target( + ${LINUX_TARGET} + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${LINUX_BIN} + ) + add_custom_target( + ${GROUP_TARGET} + DEPENDS ${BAREMETAL_TARGET} ${LINUX_TARGET} + ) + + if(DEFINED BUCKYBALL_MLIR_GROUP_TARGET) + add_dependencies(${BUCKYBALL_MLIR_GROUP_TARGET} ${GROUP_TARGET}) + endif() endfunction() -add_subdirectory(${MLIRTEST_WORKLOAD_DIR}/base - ${CMAKE_CURRENT_BINARY_DIR}/mlir_tests/base) - add_custom_target(balls-mlir-tests-build) -buckyball_add_ball_workload_subdirs(mlir_tests) - -add_custom_target(MLIRTest-all ALL DEPENDS - base-mlirtest-build - balls-mlir-tests-build -) +add_custom_target(MLIRTest-all ALL) -set(CHIP_MLIR_TEST_DIR - ${BUCKYBALL_REPO_ROOT}/examples/chips/${BUCKYBALL_WORKLOAD_SOURCE_CHIP}/workloads/mlir_tests) -add_custom_target(chip-mlirtests-build) -if(EXISTS ${CHIP_MLIR_TEST_DIR}/CMakeLists.txt) - add_subdirectory(${CHIP_MLIR_TEST_DIR} - ${CMAKE_CURRENT_BINARY_DIR}/CTest/chips/${BUCKYBALL_WORKLOAD_SOURCE_CHIP}/mlir_tests) -endif() +add_custom_target(design-mlirtests-build) -add_custom_target(mlirtest-workloads-build - DEPENDS - MLIRTest-all - chip-mlirtests-build - COMMENT "Building MLIRTest workloads" +add_custom_target( + mlirtest-workloads-build + DEPENDS MLIRTest-all design-mlirtests-build + COMMENT "Building MLIRTest workloads" ) -add_custom_target(sync-mlirtest-bin - COMMAND ${CMAKE_COMMAND} -E make_directory ${OUTPUT_BIN_DIR}/src - COMMAND rsync -a --delete --exclude=CMakeFiles "--include=*/" - "--include=/mlir_tests/***" "--include=/balls/***" - "--include=/CTest/chips/*/mlir_tests/***" "--exclude=*" - ${CMAKE_CURRENT_BINARY_DIR}/ ${OUTPUT_BIN_DIR}/src/ - DEPENDS mlirtest-workloads-build - COMMENT "Syncing MLIRTest binaries to output directory" +add_custom_target( + sync-mlirtest-bin + COMMAND ${CMAKE_COMMAND} -E make_directory ${OUTPUT_BIN_DIR}/src + COMMAND + rsync -a --delete --exclude=CMakeFiles "--include=*/" + "--include=*-mlirtest-*-baremetal" "--include=*-mlirtest-*-linux" + "--exclude=*" ${CMAKE_BINARY_DIR}/src/ ${OUTPUT_BIN_DIR}/src/ + DEPENDS mlirtest-workloads-build + COMMENT "Syncing registered MLIRTest binaries" ) diff --git a/bb-tests/workloads/src/MLIRTest/base/CMakeLists.txt b/bb-tests/workloads/src/MLIRTest/base/CMakeLists.txt index 7fb131aa1..f47c8b940 100644 --- a/bb-tests/workloads/src/MLIRTest/base/CMakeLists.txt +++ b/bb-tests/workloads/src/MLIRTest/base/CMakeLists.txt @@ -3,69 +3,13 @@ add_custom_target(base-mlirtest-build) set(BUCKYBALL_MLIR_GROUP_TARGET base-mlirtest-build) set(BUCKYBALL_MLIR_TEST_PREFIX base) -foreach(BUCKYBALL_TARGET IN LISTS BUCKYBALL_CPU_TARGETS) - if(NOT BUCKYBALL_TARGET IN_LIST BUCKYBALL_BANK_TARGETS) +foreach(BUCKYBALL_TARGET IN LISTS DESIGN_CPU_TARGETS) + if(NOT DESIGN_PROFILES_${BUCKYBALL_TARGET}_THREAD_MEM_BANK_NUM GREATER 0) continue() endif() - add_buckyball_mlir_test(base_memref_copy_dynamic - TARGET ${BUCKYBALL_TARGET} - CRUNNER_SOURCE "${BUCKYBALL_REPO_ROOT}/stack/runtime/src/mlir.cpp" - PASSES - ${BUCKYBALL_LOWER_BUCKYBALL} - -expand-strided-metadata - -convert-scf-to-cf - -convert-cf-to-llvm - -convert-arith-to-llvm - -convert-math-to-llvm - -finalize-memref-to-llvm - -convert-func-to-llvm - -reconcile-unrealized-casts) - - add_buckyball_mlir_test(base_mvin_mvout_16x16_i8 - TARGET ${BUCKYBALL_TARGET} - PASSES - ${BUCKYBALL_LOWER_BUCKYBALL} - -convert-linalg-to-loops - -expand-strided-metadata - -lower-affine - -convert-scf-to-cf - -convert-cf-to-llvm - -convert-arith-to-llvm - -convert-math-to-llvm - -finalize-memref-to-llvm - -convert-func-to-llvm - -reconcile-unrealized-casts) - - add_buckyball_mlir_test(base_mset_transfer - TARGET ${BUCKYBALL_TARGET} - PASSES - ${BUCKYBALL_LOWER_BUCKYBALL} - -convert-linalg-to-loops - -expand-strided-metadata - -lower-affine - -convert-scf-to-cf - -convert-cf-to-llvm - -convert-arith-to-llvm - -convert-math-to-llvm - -finalize-memref-to-llvm - -convert-func-to-llvm - -reconcile-unrealized-casts) - - add_buckyball_mlir_test(base_dma_group - TARGET ${BUCKYBALL_TARGET} - PASSES - ${BUCKYBALL_LOWER_BUCKYBALL} - -convert-linalg-to-loops - -expand-strided-metadata - -lower-affine - -convert-scf-to-cf - -convert-cf-to-llvm - -convert-arith-to-llvm - -convert-math-to-llvm - -finalize-memref-to-llvm - -convert-func-to-llvm - -reconcile-unrealized-casts) - - add_buckyball_mlir_test(base_bank_mvin_mvout_16x16_i8 - TARGET ${BUCKYBALL_TARGET}) + add_buckyball_mlir_test(base_memref_copy_dynamic TARGET ${BUCKYBALL_TARGET}) + add_buckyball_mlir_test(base_mvin_mvout_16x16_i8 TARGET ${BUCKYBALL_TARGET}) + add_buckyball_mlir_test(base_bank_mvin_mvout_16x16_i8 TARGET ${BUCKYBALL_TARGET}) + add_buckyball_mlir_test(base_bank_loop_carried TARGET ${BUCKYBALL_TARGET}) + add_buckyball_mlir_test(base_bank_transfer TARGET ${BUCKYBALL_TARGET}) endforeach() diff --git a/bb-tests/workloads/src/MLIRTest/base/base_bank_loop_carried.mlir b/bb-tests/workloads/src/MLIRTest/base/base_bank_loop_carried.mlir new file mode 100644 index 000000000..25c83be49 --- /dev/null +++ b/bb-tests/workloads/src/MLIRTest/base/base_bank_loop_carried.mlir @@ -0,0 +1,41 @@ +func.func private @check_result(memref<64x16xi8>) -> () + +// One bank carried through a loop: each iteration moves 16 rows through it. +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c16 = arith.constant 16 : index + %c64 = arith.constant 64 : index + %input = memref.alloc() alignment = 64 : memref<64x16xi8> + %output = memref.alloc() alignment = 64 : memref<64x16xi8> + + scf.for %i = %c0 to %c64 step %c1 { + scf.for %j = %c0 to %c16 step %c1 { + %ii = arith.index_cast %i : index to i32 + %jj = arith.index_cast %j : index to i32 + %sum = arith.addi %ii, %jj : i32 + %value = arith.trunci %sum : i32 to i8 + memref.store %value, %input[%i, %j] : memref<64x16xi8> + } + } + + %bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %last = scf.for %row = %c0 to %c64 step %c16 iter_args(%carried = %bank) -> (i64) { + %in = memref.subview %input[%row, 0] [16, 16] [1, 1] + : memref<64x16xi8> to memref<16x16xi8, strided<[16, 1], offset: ?>> + %out = memref.subview %output[%row, 0] [16, 16] [1, 1] + : memref<64x16xi8> to memref<16x16xi8, strided<[16, 1], offset: ?>> + %loaded = buckyball.bank_mvin %in, %carried <{depth = 16 : i64, stride = 1 : i64}> + : memref<16x16xi8, strided<[16, 1], offset: ?>> + %stored = buckyball.bank_mvout %out, %loaded <{depth = 16 : i64, stride = 1 : i64}> + : memref<16x16xi8, strided<[16, 1], offset: ?>> + scf.yield %stored : i64 + } + buckyball.bank_release %last + + func.call @check_result(%output) : (memref<64x16xi8>) -> () + memref.dealloc %input : memref<64x16xi8> + memref.dealloc %output : memref<64x16xi8> + return %zero_i8 : i8 +} diff --git a/bb-tests/workloads/src/MLIRTest/base/base_bank_loop_carried_main.cpp b/bb-tests/workloads/src/MLIRTest/base/base_bank_loop_carried_main.cpp new file mode 100644 index 000000000..c4f4eb33d --- /dev/null +++ b/bb-tests/workloads/src/MLIRTest/base/base_bank_loop_carried_main.cpp @@ -0,0 +1,35 @@ +#include +#include +#include + +static void fail(void) { +#ifdef BAREMETAL + volatile uint32_t *sim_exit = (volatile uint32_t *)0x60000000; + *sim_exit = 1; + while (1) { + } +#else + exit(1); +#endif +} + +#ifdef __cplusplus +extern "C" +#endif + void check_result(int8_t *allocated, int8_t *aligned, int64_t offset, + int64_t size0, int64_t size1, int64_t stride0, + int64_t stride1) { + (void)allocated; + if (size0 != 64 || size1 != 16 || stride0 != 16 || stride1 != 1) { + fail(); + } + int8_t *out = aligned + offset; + for (int i = 0; i < 64; ++i) { + for (int j = 0; j < 16; ++j) { + if (out[i * stride0 + j * stride1] != (int8_t)(i + j)) { + fail(); + } + } + } + printf("PASSED: base bank carried through a loop\n"); +} diff --git a/bb-tests/workloads/src/MLIRTest/base/base_bank_mvin_mvout_16x16_i8.mlir b/bb-tests/workloads/src/MLIRTest/base/base_bank_mvin_mvout_16x16_i8.mlir index 32a9315cc..e1ff7fb5a 100644 --- a/bb-tests/workloads/src/MLIRTest/base/base_bank_mvin_mvout_16x16_i8.mlir +++ b/bb-tests/workloads/src/MLIRTest/base/base_bank_mvin_mvout_16x16_i8.mlir @@ -6,8 +6,6 @@ func.func @main() -> i8 { %c1 = arith.constant 1 : index %n = arith.constant 16 : index %c16 = arith.constant 16 : i32 - %depth = arith.constant 16 : i64 - %stride = arith.constant 1 : i64 %input = memref.alloc() alignment = 64 : memref<16x16xi8> %output = memref.alloc() alignment = 64 : memref<16x16xi8> @@ -22,12 +20,10 @@ func.func @main() -> i8 { } } - %bank = buckyball.bank_alloc - %loaded = buckyball.bank_mvin %input %bank %depth %stride - : memref<16x16xi8> i64 i64 i64 - %stored = buckyball.bank_mvout %output %loaded %depth %stride - : memref<16x16xi8> i64 i64 i64 - buckyball.bank_release %stored : i64 + %bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %loaded = buckyball.bank_mvin %input, %bank <{depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + %stored = buckyball.bank_mvout %output, %loaded <{depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.bank_release %stored func.call @check_result(%output) : (memref<16x16xi8>) -> () memref.dealloc %input : memref<16x16xi8> diff --git a/bb-tests/workloads/src/MLIRTest/base/base_bank_transfer.mlir b/bb-tests/workloads/src/MLIRTest/base/base_bank_transfer.mlir new file mode 100644 index 000000000..4d8504dd7 --- /dev/null +++ b/bb-tests/workloads/src/MLIRTest/base/base_bank_transfer.mlir @@ -0,0 +1,50 @@ +func.func private @check_result(memref<2x48xi8>) -> () + +// The groups of a two-column bank appended to a one-column bank. +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c2 = arith.constant 2 : index + %c16 = arith.constant 16 : index + %c32 = arith.constant 32 : index + %first = memref.alloc() alignment = 64 : memref<2x16xi8> + %second = memref.alloc() alignment = 64 : memref<2x32xi8> + %result = memref.alloc() alignment = 64 : memref<2x48xi8> + + scf.for %row = %c0 to %c2 step %c1 { + scf.for %column = %c0 to %c32 step %c1 { + %r = arith.index_cast %row : index to i32 + %c = arith.index_cast %column : index to i32 + %c32_i32 = arith.constant 32 : i32 + %offset = arith.muli %r, %c32_i32 : i32 + %value = arith.addi %offset, %c : i32 + %b = arith.trunci %value : i32 to i8 + %high = arith.constant 64 : i8 + %second_value = arith.addi %b, %high : i8 + memref.store %second_value, %second[%row, %column] : memref<2x32xi8> + %in_first = arith.cmpi ult, %column, %c16 : index + scf.if %in_first { + %c16_i32 = arith.constant 16 : i32 + %first_offset = arith.muli %r, %c16_i32 : i32 + %first_value = arith.addi %first_offset, %c : i32 + %a = arith.trunci %first_value : i32 to i8 + memref.store %a, %first[%row, %column] : memref<2x16xi8> + } + } + } + + %a = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %b = buckyball.bank_alloc <{row = 1 : i64, col = 2 : i64}> + %la = buckyball.bank_mvin %first, %a <{depth = 2 : i64, stride = 1 : i64}> : memref<2x16xi8> + %lb = buckyball.bank_mvin %second, %b <{depth = 2 : i64, stride = 1 : i64}> : memref<2x32xi8> + %merged = buckyball.bank_transfer %lb, %la + %stored = buckyball.bank_mvout %result, %merged <{depth = 2 : i64, stride = 1 : i64}> : memref<2x48xi8> + buckyball.bank_release %stored + + func.call @check_result(%result) : (memref<2x48xi8>) -> () + memref.dealloc %first : memref<2x16xi8> + memref.dealloc %second : memref<2x32xi8> + memref.dealloc %result : memref<2x48xi8> + return %zero_i8 : i8 +} diff --git a/bb-tests/workloads/src/MLIRTest/base/base_bank_transfer_main.cpp b/bb-tests/workloads/src/MLIRTest/base/base_bank_transfer_main.cpp new file mode 100644 index 000000000..42f45ab36 --- /dev/null +++ b/bb-tests/workloads/src/MLIRTest/base/base_bank_transfer_main.cpp @@ -0,0 +1,37 @@ +#include +#include +#include + +static void fail(void) { +#ifdef BAREMETAL + volatile uint32_t *sim_exit = (volatile uint32_t *)0x60000000; + *sim_exit = 1; + while (1) { + } +#else + exit(1); +#endif +} + +#ifdef __cplusplus +extern "C" +#endif + void check_result(int8_t *allocated, int8_t *aligned, int64_t offset, + int64_t size0, int64_t size1, int64_t stride0, + int64_t stride1) { + (void)allocated; + if (size0 != 2 || size1 != 48 || stride0 != 48 || stride1 != 1) { + fail(); + } + int8_t *out = aligned + offset; + for (int i = 0; i < 2; ++i) { + for (int j = 0; j < 48; ++j) { + int8_t expected = + j < 16 ? (int8_t)(i * 16 + j) : (int8_t)(i * 32 + j - 16 + 64); + if (out[i * stride0 + j * stride1] != expected) { + fail(); + } + } + } + printf("PASSED: base bank_transfer appends groups\n"); +} diff --git a/bb-tests/workloads/src/MLIRTest/base/base_dma_group.mlir b/bb-tests/workloads/src/MLIRTest/base/base_dma_group.mlir deleted file mode 100644 index adad6caa1..000000000 --- a/bb-tests/workloads/src/MLIRTest/base/base_dma_group.mlir +++ /dev/null @@ -1,52 +0,0 @@ -func.func private @check_result(memref<16x48xi8>, memref<16x32xi8>) -func.func @main() -> i8 { - %zero = arith.constant 0 : i8 - %c0 = arith.constant 0 : index - %c1 = arith.constant 1 : index - %c16 = arith.constant 16 : index - %c32 = arith.constant 32 : index - %c48 = arith.constant 48 : index - %n48 = arith.constant 48 : i32 - %n32 = arith.constant 32 : i32 - %n91 = arith.constant 91 : i8 - %n7 = arith.constant 7 : i32 - %bank = arith.constant 9 : i64 - %depth = arith.constant 16 : i64 - %stride = arith.constant 1 : i64 - %stride2 = arith.constant 2 : i64 - %input = memref.alloc() alignment = 64 : memref<16x48xi8> - %patch = memref.alloc() alignment = 64 : memref<16x32xi8> - %output = memref.alloc() alignment = 64 : memref<16x48xi8> - %selected = memref.alloc() alignment = 64 : memref<16x32xi8> - scf.for %i = %c0 to %c16 step %c1 { - %ii = arith.index_cast %i : index to i32 - scf.for %j = %c0 to %c48 step %c1 { - %jj = arith.index_cast %j : index to i32 - %base = arith.muli %ii, %n48 : i32 - %value = arith.addi %base, %jj : i32 - %byte = arith.trunci %value : i32 to i8 - memref.store %byte, %input[%i, %j] : memref<16x48xi8> - } - scf.for %j = %c0 to %c32 step %c1 { - %jj = arith.index_cast %j : index to i32 - %base = arith.muli %ii, %n32 : i32 - %value = arith.addi %base, %jj : i32 - %shifted = arith.addi %value, %n7 : i32 - %byte = arith.trunci %shifted : i32 to i8 - memref.store %byte, %patch[%i, %j] : memref<16x32xi8> - memref.store %n91, %selected[%i, %j] : memref<16x32xi8> - } - } - buckyball.mset %bank : i64 - buckyball.mvin %input %bank %depth %stride : memref<16x48xi8> i64 i64 i64 - buckyball.mvin %patch %bank %depth %stride2 : memref<16x32xi8> i64 i64 i64 - buckyball.mvout %selected %bank %depth %stride2 : memref<16x32xi8> i64 i64 i64 - buckyball.mvout %output %bank %depth %stride : memref<16x48xi8> i64 i64 i64 - buckyball.mset %bank : i64 - func.call @check_result(%output, %selected) : (memref<16x48xi8>, memref<16x32xi8>) -> () - memref.dealloc %input : memref<16x48xi8> - memref.dealloc %patch : memref<16x32xi8> - memref.dealloc %output : memref<16x48xi8> - memref.dealloc %selected : memref<16x32xi8> - return %zero : i8 -} diff --git a/bb-tests/workloads/src/MLIRTest/base/base_dma_group_main.cpp b/bb-tests/workloads/src/MLIRTest/base/base_dma_group_main.cpp deleted file mode 100644 index 19ae7a0e8..000000000 --- a/bb-tests/workloads/src/MLIRTest/base/base_dma_group_main.cpp +++ /dev/null @@ -1,46 +0,0 @@ -#include -#include -#include - -static void fail(void) { -#ifdef BAREMETAL - volatile uint32_t *sim_exit = (volatile uint32_t *)0x60000000; - *sim_exit = 1; - while (1) { - } -#else - exit(1); -#endif -} - -extern "C" void check_result(int8_t *allocated, int8_t *aligned, int64_t offset, - int64_t size0, int64_t size1, int64_t stride0, - int64_t stride1, int8_t *selectedAllocated, - int8_t *selectedAligned, int64_t selectedOffset, - int64_t selectedSize0, int64_t selectedSize1, - int64_t selectedStride0, int64_t selectedStride1) { - (void)allocated; - (void)selectedAllocated; - if (size0 != 16 || size1 != 48 || stride0 != 48 || stride1 != 1 || - selectedSize0 != 16 || selectedSize1 != 32 || selectedStride0 != 32 || - selectedStride1 != 1) { - fail(); - } - int8_t *out = aligned + offset; - int8_t *selected = selectedAligned + selectedOffset; - for (int i = 0; i < 16; ++i) { - for (int j = 0; j < 48; ++j) { - int expected = j >= 16 && j < 32 ? i * 32 + j - 16 + 7 : i * 48 + j; - if (out[i * stride0 + j] != (int8_t)expected) { - fail(); - } - } - for (int j = 0; j < 32; ++j) { - int expected = j < 16 ? i * 32 + j + 7 : 91; - if (selected[i * selectedStride0 + j] != (int8_t)expected) { - fail(); - } - } - } - printf("PASSED: selected DMA preserves other groups and stride gaps\n"); -} diff --git a/bb-tests/workloads/src/MLIRTest/base/base_memref_copy_dynamic.mlir b/bb-tests/workloads/src/MLIRTest/base/base_memref_copy_dynamic.mlir new file mode 100644 index 000000000..a3703f84d --- /dev/null +++ b/bb-tests/workloads/src/MLIRTest/base/base_memref_copy_dynamic.mlir @@ -0,0 +1,15 @@ +// The base workload pipeline explicitly runs lower-buckyball before MemRefToLLVM. +// Static strides remain visible to Buckyball's own export patterns. +func.func @copy_unit(%input: memref>, + %output: memref>) + attributes {llvm.emit_c_interface} { + memref.copy %input, %output : memref> to memref> + return +} + +func.func @copy_gap(%input: memref>, + %output: memref>) + attributes {llvm.emit_c_interface} { + memref.copy %input, %output : memref> to memref> + return +} diff --git a/bb-tests/workloads/src/MLIRTest/base/base_memref_copy_dynamic.mlir.in b/bb-tests/workloads/src/MLIRTest/base/base_memref_copy_dynamic.mlir.in deleted file mode 100644 index 06cfcfb7f..000000000 --- a/bb-tests/workloads/src/MLIRTest/base/base_memref_copy_dynamic.mlir.in +++ /dev/null @@ -1,15 +0,0 @@ -// The base workload pipeline explicitly runs lower-buckyball before MemRefToLLVM. -// Target and static strides remain visible to Buckyball's own export patterns. -func.func @copy_unit(%input: memref>, - %output: memref>) - attributes {llvm.emit_c_interface, buckyball.target = "@ARG_TARGET@"} { - memref.copy %input, %output : memref> to memref> - return -} - -func.func @copy_gap(%input: memref>, - %output: memref>) - attributes {llvm.emit_c_interface, buckyball.target = "@ARG_TARGET@"} { - memref.copy %input, %output : memref> to memref> - return -} diff --git a/bb-tests/workloads/src/MLIRTest/base/base_mset_transfer.mlir b/bb-tests/workloads/src/MLIRTest/base/base_mset_transfer.mlir deleted file mode 100644 index 5b054bdbf..000000000 --- a/bb-tests/workloads/src/MLIRTest/base/base_mset_transfer.mlir +++ /dev/null @@ -1,49 +0,0 @@ -// RUN: buddy-opt %s --target=toy -report-bank-usage="bank_num=4 verbose=true" 2>&1 | FileCheck %s --check-prefix=REPORT -// REPORT: transfer b1 -> b7 groups=1 cur=2/4 -// REPORT: transfer b2 -> b7 groups=1 cur=2/4 -// REPORT: peak=2/4 alloc=2 release=1 leaked=0 - -func.func private @check_result(memref<16x32xi8>) -func.func @main() -> i8 { - %zero = arith.constant 0 : i8 - %c0 = arith.constant 0 : index - %c1 = arith.constant 1 : index - %c16 = arith.constant 16 : index - %c32 = arith.constant 32 : index - %i32_16 = arith.constant 16 : i32 - %i32_127 = arith.constant 127 : i32 - %source0 = arith.constant 1 : i64 - %source1 = arith.constant 2 : i64 - %target = arith.constant 7 : i64 - %depth = arith.constant 16 : i64 - %stride = arith.constant 1 : i64 - %input0 = memref.alloc() alignment = 64 : memref<16x16xi8> - %input1 = memref.alloc() alignment = 64 : memref<16x16xi8> - %output = memref.alloc() alignment = 64 : memref<16x32xi8> - scf.for %i = %c0 to %c16 step %c1 { - scf.for %j = %c0 to %c16 step %c1 { - %ii = arith.index_cast %i : index to i32 - %jj = arith.index_cast %j : index to i32 - %base = arith.muli %ii, %i32_16 : i32 - %value = arith.addi %base, %jj : i32 - %other = arith.subi %i32_127, %value : i32 - %byte = arith.trunci %value : i32 to i8 - %other_byte = arith.trunci %other : i32 to i8 - memref.store %byte, %input0[%i, %j] : memref<16x16xi8> - memref.store %other_byte, %input1[%i, %j] : memref<16x16xi8> - } - } - buckyball.mset %source0 : i64 - buckyball.mset %source1 : i64 - buckyball.mvin %input0 %source0 %depth %stride : memref<16x16xi8> i64 i64 i64 - buckyball.mvin %input1 %source1 %depth %stride : memref<16x16xi8> i64 i64 i64 - buckyball.mset_transfer %source0 %target : i64 i64 - buckyball.mset_transfer %source1 %target : i64 i64 - buckyball.mvout %output %target %depth %stride : memref<16x32xi8> i64 i64 i64 - buckyball.mset %target : i64 - func.call @check_result(%output) : (memref<16x32xi8>) -> () - memref.dealloc %input0 : memref<16x16xi8> - memref.dealloc %input1 : memref<16x16xi8> - memref.dealloc %output : memref<16x32xi8> - return %zero : i8 -} diff --git a/bb-tests/workloads/src/MLIRTest/base/base_mset_transfer_main.cpp b/bb-tests/workloads/src/MLIRTest/base/base_mset_transfer_main.cpp deleted file mode 100644 index f277eeca9..000000000 --- a/bb-tests/workloads/src/MLIRTest/base/base_mset_transfer_main.cpp +++ /dev/null @@ -1,37 +0,0 @@ -#include -#include -#include - -static void fail(void) { -#ifdef BAREMETAL - volatile uint32_t *sim_exit = (volatile uint32_t *)0x60000000; - *sim_exit = 1; - while (1) { - } -#else - exit(1); -#endif -} - -#ifdef __cplusplus -extern "C" -#endif - void check_result(int8_t *allocated, int8_t *aligned, int64_t offset, - int64_t size0, int64_t size1, int64_t stride0, - int64_t stride1) { - (void)allocated; - if (size0 != 16 || size1 != 32 || stride0 != 32 || stride1 != 1) { - fail(); - } - int8_t *out = aligned + offset; - for (int i = 0; i < 16; ++i) { - for (int j = 0; j < 16; ++j) { - int value = i * 16 + j; - if (out[i * stride0 + j] != (int8_t)value || - out[i * stride0 + 16 + j] != (int8_t)(127 - value)) { - fail(); - } - } - } - printf("PASSED: base mset transfer preserves group order\n"); -} diff --git a/bb-tests/workloads/src/MLIRTest/base/base_mvin_mvout_16x16_i8.mlir b/bb-tests/workloads/src/MLIRTest/base/base_mvin_mvout_16x16_i8.mlir index 7f3115c68..eff30e0ba 100644 --- a/bb-tests/workloads/src/MLIRTest/base/base_mvin_mvout_16x16_i8.mlir +++ b/bb-tests/workloads/src/MLIRTest/base/base_mvin_mvout_16x16_i8.mlir @@ -6,9 +6,6 @@ func.func @main() -> i8 { %c1 = arith.constant 1 : index %n = arith.constant 16 : index %c16 = arith.constant 16 : i32 - %bank = arith.constant 0 : i64 - %depth = arith.constant 16 : i64 - %stride = arith.constant 1 : i64 %input = memref.alloc() alignment = 64 : memref<16x16xi8> %output = memref.alloc() alignment = 64 : memref<16x16xi8> @@ -23,10 +20,10 @@ func.func @main() -> i8 { } } - buckyball.mset %bank : i64 - buckyball.mvin %input %bank %depth %stride : memref<16x16xi8> i64 i64 i64 - buckyball.mvout %output %bank %depth %stride : memref<16x16xi8> i64 i64 i64 - buckyball.mset %bank : i64 + buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %input <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.mvout %output <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> func.call @check_result(%output) : (memref<16x16xi8>) -> () memref.dealloc %input : memref<16x16xi8> diff --git a/bb-tests/workloads/src/MLIRTest/compiler/CMakeLists.txt b/bb-tests/workloads/src/MLIRTest/compiler/CMakeLists.txt deleted file mode 100644 index a969a34bf..000000000 --- a/bb-tests/workloads/src/MLIRTest/compiler/CMakeLists.txt +++ /dev/null @@ -1,24 +0,0 @@ -set(BUDDY_MLIR_DIR ${BUCKYBALL_REPO_ROOT}/stack/compiler/thirdparty/buddy-mlir) -set(BUDDY_BINARY_DIR ${BUDDY_MLIR_DIR}/build/${BUCKYBALL_WORKLOAD_CHIP}/bin) -set(BUDDY_OPT ${BUDDY_BINARY_DIR}/buddy-opt) -set(BUDDY_TRANSLATE ${BUDDY_BINARY_DIR}/buddy-translate) -set(BUDDY_LLC ${BUDDY_BINARY_DIR}/buddy-llc) -set(LLVM_BUILD_DIR ${BUDDY_MLIR_DIR}/llvm/build) - -option(BUCKYBALL_STABLE "Use stable LLVM Buckyball extensions" OFF) -if(BUCKYBALL_STABLE) - set(BUCKYBALL_RISCV_MATTR "+xbbpebble,+xbuckyball,+m,+D") - set(BUCKYBALL_LOWER_BUCKYBALL "--lower-buckyball=stable=true") - - set(BUCKYBALL_LOWER_BANK_SSA_TO_INTRINSICS - "--lower-bank-ssa-to-intrinsics=stable=true") - -else() - set(BUCKYBALL_RISCV_MATTR "+xbuckyball,+m,+D") - set(BUCKYBALL_LOWER_BUCKYBALL -lower-buckyball) - - set(BUCKYBALL_LOWER_BANK_SSA_TO_INTRINSICS - -lower-bank-ssa-to-intrinsics) - -endif() -message(STATUS "Buckyball RISC-V mattr: ${BUCKYBALL_RISCV_MATTR}") diff --git a/bb-tests/workloads/src/MLIRTest/compiler/bank_kernel_calls.mlir b/bb-tests/workloads/src/MLIRTest/compiler/bank_kernel_calls.mlir deleted file mode 100644 index 04e399b5d..000000000 --- a/bb-tests/workloads/src/MLIRTest/compiler/bank_kernel_calls.mlir +++ /dev/null @@ -1,17 +0,0 @@ -// RUN: buddy-opt %s --target=toy -assign-physical-banks | FileCheck %s -// CHECK-LABEL: func.func @group_output -// CHECK: call @mock -// CHECK: buckyball.mvout {{.*}} -// CHECK-COUNT-2: buckyball.mset {{.*}} ) { - %read = buckyball.bank_alloc - %write = buckyball.bank_alloc - %one = arith.constant 1 : i64 - %r, %w = buckyball.bank_kernel @mock %read %write (%one) : i64 - %stored = buckyball.bank_mvout %output %w %one %one : memref<1x4xf32> i64 i64 i64 - buckyball.bank_release %r : i64 - buckyball.bank_release %stored : i64 - return -} diff --git a/bb-tests/workloads/src/MLIRTest/compiler/bank_kernel_invalid.mlir b/bb-tests/workloads/src/MLIRTest/compiler/bank_kernel_invalid.mlir deleted file mode 100644 index 2ec5537e4..000000000 --- a/bb-tests/workloads/src/MLIRTest/compiler/bank_kernel_invalid.mlir +++ /dev/null @@ -1,46 +0,0 @@ -// RUN: buddy-opt %s --target=toy -assign-physical-banks -split-input-file -verify-diagnostics -module { - func.func private @wrong_type(i64, i64, i32) - func.func @mismatched() { - %read = buckyball.bank_alloc - %write = buckyball.bank_alloc - %one = arith.constant 1 : i64 - // expected-error @+1 {{bank kernel callee type does not match its operands}} - %r, %w = buckyball.bank_kernel @wrong_type %read %write (%one) : i64 - buckyball.bank_release %r : i64 - buckyball.bank_release %w : i64 - return - } -} -// ----- -func.func @consumed_source() { - %source = buckyball.bank_alloc - %write = buckyball.bank_alloc - %target = arith.constant 7 : i64 - %one = arith.constant 1 : i64 - // expected-error @+1 {{source bank handle is used after transfer or outside its block}} - %aggregate = buckyball.bank_transfer %source %target : i64 i64 - %r, %w = buckyball.bank_kernel @mock %source %write (%one) : i64 - buckyball.bank_release %aggregate : i64 - buckyball.bank_release %w : i64 - return -} -// ----- -func.func @unknown_read() { - %read = arith.constant 7 : i64 - %write = buckyball.bank_alloc - %one = arith.constant 1 : i64 - // expected-error @+1 {{bank kernel requires distinct live read/write handles}} - %r, %w = buckyball.bank_kernel @mock %read %write (%one) : i64 - buckyball.bank_release %w : i64 - return -} -// ----- -func.func @aliased_read_write() { - %read = buckyball.bank_alloc - %one = arith.constant 1 : i64 - // expected-error @+1 {{bank kernel requires distinct live read/write handles}} - %r, %w = buckyball.bank_kernel @mock %read %read (%one) : i64 - buckyball.bank_release %w : i64 - return -} diff --git a/bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_cfg_invalid.mlir b/bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_cfg_invalid.mlir deleted file mode 100644 index 3fa941841..000000000 --- a/bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_cfg_invalid.mlir +++ /dev/null @@ -1,40 +0,0 @@ -// RUN: buddy-opt %s --target=toy -assign-physical-banks -split-input-file -verify-diagnostics - -func.func @captured_allocation(%condition: i1) { - %source = buckyball.bank_alloc - %target = arith.constant 2 : i64 - scf.if %condition { - // expected-error @+1 {{bank transfer cannot consume a control-flow bank alias}} - %aggregate = buckyball.bank_transfer %source %target : i64 i64 - buckyball.bank_release %aggregate : i64 - } - return -} -// ----- -func.func @if_result_alias(%condition: i1) { - %source = buckyball.bank_alloc - %target = arith.constant 2 : i64 - %alias = scf.if %condition -> i64 { - scf.yield %source : i64 - } else { - scf.yield %source : i64 - } - // expected-error @+1 {{bank transfer cannot consume a control-flow bank alias}} - %aggregate = buckyball.bank_transfer %alias %target : i64 i64 - buckyball.bank_release %aggregate : i64 - return -} -// ----- -func.func @loop_result_alias() { - %source = buckyball.bank_alloc - %target = arith.constant 2 : i64 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %alias = scf.for %i = %zero to %one step %one iter_args(%bank = %source) -> i64 { - scf.yield %bank : i64 - } - // expected-error @+1 {{bank transfer cannot consume a control-flow bank alias}} - %aggregate = buckyball.bank_transfer %alias %target : i64 i64 - buckyball.bank_release %aggregate : i64 - return -} diff --git a/bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_invalid.mlir b/bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_invalid.mlir deleted file mode 100644 index 80f41c0c9..000000000 --- a/bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_invalid.mlir +++ /dev/null @@ -1,64 +0,0 @@ -// RUN: buddy-opt %s --target=toy -assign-physical-banks -split-input-file -verify-diagnostics - -func.func @source_use_after_transfer() { - %source = buckyball.bank_alloc - %target = arith.constant 2 : i64 - // expected-error @+1 {{source bank handle is used after transfer or outside its block}} - %aggregate = buckyball.bank_transfer %source %target : i64 i64 - buckyball.bank_release %source : i64 - buckyball.bank_release %aggregate : i64 - return -} -// ----- -func.func @self() { - %source = buckyball.bank_alloc - // expected-error @+1 {{bank transfer source and target must differ}} - %aggregate = buckyball.bank_transfer %source %source : i64 i64 - buckyball.bank_release %aggregate : i64 - return -} -// ----- -func.func @unknown() { - %source = arith.constant 7 : i64 - %target = arith.constant 2 : i64 - // expected-error @+1 {{transfer consumes an unknown virtual bank handle}} - %aggregate = buckyball.bank_transfer %source %target : i64 i64 - buckyball.bank_release %aggregate : i64 - return -} -// ----- -func.func @keep_other_owner() { - %source = buckyball.bank_alloc - %keep = buckyball.bank_alloc - %target = buckyball.bank_alloc - %aggregate = buckyball.bank_transfer %source %target : i64 i64 - buckyball.bank_release %aggregate : i64 - %all_free = buckyball.bank_alloc - // expected-error @+1 {{unavailable bank resources (request=1x1, used=16/16, private ID max=15)}} - %extra = buckyball.bank_alloc - buckyball.bank_release %extra : i64 - buckyball.bank_release %all_free : i64 - buckyball.bank_release %keep : i64 - return -} -// ----- -func.func @private_overflow() { - %source = buckyball.bank_alloc - %target = arith.constant 16 : i64 - // expected-error @+1 {{bank transfer target exceeds the private virtual ID range}} - %aggregate = buckyball.bank_transfer %source %target : i64 i64 - buckyball.bank_release %aggregate : i64 - return -} -// ----- -func.func @old_source_alias_after_id_reuse() { - %source = buckyball.bank_alloc - %target = arith.constant 2 : i64 - // expected-error @+1 {{source bank handle is used after transfer or outside its block}} - %aggregate = buckyball.bank_transfer %source %target : i64 i64 - %new_source_id = buckyball.bank_alloc - buckyball.bank_release %source : i64 - buckyball.bank_release %new_source_id : i64 - buckyball.bank_release %aggregate : i64 - return -} diff --git a/bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_ownership.mlir b/bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_ownership.mlir deleted file mode 100644 index e96c053e0..000000000 --- a/bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_ownership.mlir +++ /dev/null @@ -1,65 +0,0 @@ -// RUN: buddy-opt %s --target=toy -assign-physical-banks | FileCheck %s - -// CHECK-LABEL: func.func @release_fragmented -// CHECK: buckyball.mset {{.*}} {{.*}}col = 14 -// CHECK: %[[TARGET:.*]] = arith.constant 2 : i64 -// CHECK: buckyball.mset_transfer {{.*}} %[[TARGET]] -// CHECK: buckyball.mset {{.*}} - %aggregate = buckyball.bank_transfer %source %target : i64 i64 - buckyball.bank_release %aggregate : i64 - %all_free = buckyball.bank_alloc - buckyball.bank_release %all_free : i64 - buckyball.bank_release %keep : i64 - return -} - -// Virtual target 2 initially owns physical group 0. Later allocations own -// physical groups 1 and 2, with virtual IDs 0 and 1, preserving target 2. -// CHECK-LABEL: func.func @virtual_physical_collision -// CHECK: buckyball.mset -// CHECK: %[[START:.*]] = arith.constant 0 : i64 -// CHECK: %[[TARGET2:.*]] = arith.constant 2 : i64 -// CHECK: buckyball.mset_transfer %[[START]] %[[TARGET2]] -// CHECK: %[[FIRST:.*]] = arith.constant 0 : i64 -// CHECK: buckyball.mset %[[FIRST]] -// CHECK: %[[SECOND:.*]] = arith.constant 1 : i64 -// CHECK: buckyball.mset %[[SECOND]] -// CHECK: buckyball.mset {{.*}} -// CHECK: %[[OUTPUT:.*]] = memref.alloc() : memref<1x4x16x128xf32> -// CHECK: scf.for %[[HEAD:.*]] = -// CHECK: scf.for %[[TOKEN:.*]] = -// CHECK: %[[POSITION:.*]] = memref.load -// CHECK-NOT: vector.transfer -// CHECK: %[[SOURCE:.*]] = memref.subview %[[INPUT]][{{.*}}, %[[HEAD]], %[[TOKEN]], {{.*}}] [1, 1, 1, 128] [1, 1, 1, 1] -// CHECK: %[[DESTINATION:.*]] = memref.subview %[[OUTPUT]][{{.*}}, %[[HEAD]], %[[POSITION]], {{.*}}] [1, 1, 1, 128] [1, 1, 1, 1] -// CHECK: memref.copy %[[SOURCE]], %[[DESTINATION]] -// CHECK-NOT: vector.transfer -// CHECK: return %[[OUTPUT]] -func.func @scatter_rows(%input: memref<1x4x8x128xf32>, %positions: memref<8xindex>) -> memref<1x4x16x128xf32> { - %output = memref.alloc() : memref<1x4x16x128xf32> - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %four = arith.constant 4 : index - %eight = arith.constant 8 : index - %padding = arith.constant 0.0 : f32 - scf.for %head = %zero to %four step %one { - scf.for %token = %zero to %eight step %one { - %position = memref.load %positions[%token] : memref<8xindex> - %row = vector.transfer_read %input[%zero, %head, %token, %zero], %padding {in_bounds = [true]} : memref<1x4x8x128xf32>, vector<128xf32> - vector.transfer_write %row, %output[%zero, %head, %position, %zero] {in_bounds = [true]} : vector<128xf32>, memref<1x4x16x128xf32> - } - } - return %output : memref<1x4x16x128xf32> -} - -// CHECK-LABEL: func.func @alias_view -// CHECK-NOT: memref.copy -// CHECK: vector.transfer_read -// CHECK: vector.transfer_write -func.func @alias_view(%row: index) -> memref<4x128xf32> { - %buffer = memref.alloc() : memref<4x128xf32> - %view = memref.cast %buffer : memref<4x128xf32> to memref - %zero = arith.constant 0 : index - %padding = arith.constant 0.0 : f32 - %value = vector.transfer_read %view[%zero, %zero], %padding {in_bounds = [true]} : memref, vector<128xf32> - vector.transfer_write %value, %buffer[%row, %zero] {in_bounds = [true]} : vector<128xf32>, memref<4x128xf32> - return %buffer : memref<4x128xf32> -} - -// CHECK-LABEL: func.func @masked -// CHECK-NOT: memref.copy -// CHECK: vector.transfer_read -// CHECK: vector.transfer_write -func.func @masked(%input: memref<4x128xf32>, %mask: vector<128xi1>) -> memref<4x128xf32> { - %output = memref.alloc() : memref<4x128xf32> - %zero = arith.constant 0 : index - %padding = arith.constant 0.0 : f32 - %value = vector.transfer_read %input[%zero, %zero], %padding, %mask {in_bounds = [true]} : memref<4x128xf32>, vector<128xf32> - vector.transfer_write %value, %output[%zero, %zero], %mask {in_bounds = [true]} : vector<128xf32>, memref<4x128xf32> - return %output : memref<4x128xf32> -} - -// CHECK-LABEL: func.func @strided -// CHECK-NOT: memref.copy -// CHECK: vector.transfer_read -// CHECK: vector.transfer_write -func.func @strided(%input: memref<4x128xf32, strided<[256, 2], offset: ?>>) -> memref<4x128xf32> { - %output = memref.alloc() : memref<4x128xf32> - %zero = arith.constant 0 : index - %padding = arith.constant 0.0 : f32 - %value = vector.transfer_read %input[%zero, %zero], %padding {in_bounds = [true]} : memref<4x128xf32, strided<[256, 2], offset: ?>>, vector<128xf32> - vector.transfer_write %value, %output[%zero, %zero] {in_bounds = [true]} : vector<128xf32>, memref<4x128xf32> - return %output : memref<4x128xf32> -} - -// CHECK-LABEL: func.func @padding -// CHECK-NOT: memref.copy -// CHECK: vector.transfer_read -// CHECK: vector.transfer_write -func.func @padding(%input: memref<4x128xf32>, %offset: index) -> memref<4x128xf32> { - %output = memref.alloc() : memref<4x128xf32> - %zero = arith.constant 0 : index - %padding = arith.constant 0.0 : f32 - %value = vector.transfer_read %input[%zero, %offset], %padding {in_bounds = [false]} : memref<4x128xf32>, vector<128xf32> - vector.transfer_write %value, %output[%zero, %zero] {in_bounds = [true]} : vector<128xf32>, memref<4x128xf32> - return %output : memref<4x128xf32> -} - -// CHECK-LABEL: func.func @intervening_store -// CHECK-NOT: memref.copy -// CHECK: vector.transfer_read -// CHECK: memref.store -// CHECK: vector.transfer_write -func.func @intervening_store(%input: memref<4x128xf32>) -> memref<4x128xf32> { - %output = memref.alloc() : memref<4x128xf32> - %zero = arith.constant 0 : index - %padding = arith.constant 0.0 : f32 - %value = vector.transfer_read %input[%zero, %zero], %padding {in_bounds = [true]} : memref<4x128xf32>, vector<128xf32> - memref.store %padding, %input[%zero, %zero] : memref<4x128xf32> - vector.transfer_write %value, %output[%zero, %zero] {in_bounds = [true]} : vector<128xf32>, memref<4x128xf32> - return %output : memref<4x128xf32> -} diff --git a/bb-tests/workloads/src/MLIRTest/compiler/dma_group_invalid.mlir b/bb-tests/workloads/src/MLIRTest/compiler/dma_group_invalid.mlir deleted file mode 100644 index 928e4843a..000000000 --- a/bb-tests/workloads/src/MLIRTest/compiler/dma_group_invalid.mlir +++ /dev/null @@ -1,14 +0,0 @@ -// RUN: buddy-opt %s -split-input-file -verify-diagnostics -func.func @negative(%input: memref<16x16xi8>, %bank: i64, %depth: i64, %stride: i64) { - // expected-error @+1 {{group must be in [0, 31]}} - buckyball.mvin %input %bank %depth %stride : memref<16x16xi8> i64 i64 i64 - return -} - -// ----- - -func.func @overflow(%output: memref<16x16xi8>, %bank: i64, %depth: i64, %stride: i64) { - // expected-error @+1 {{group must be in [0, 31]}} - buckyball.mvout %output %bank %depth %stride : memref<16x16xi8> i64 i64 i64 - return -} diff --git a/bb-tests/workloads/src/MLIRTest/compiler/dma_group_lowering.mlir b/bb-tests/workloads/src/MLIRTest/compiler/dma_group_lowering.mlir deleted file mode 100644 index d9d5d094d..000000000 --- a/bb-tests/workloads/src/MLIRTest/compiler/dma_group_lowering.mlir +++ /dev/null @@ -1,28 +0,0 @@ -// RUN: buddy-opt %s --target=toy -lower-buckyball -canonicalize | FileCheck %s -// CHECK-LABEL: func.func @selected -// CHECK-DAG: %[[MODE:.*]] = arith.constant -8935140561191436288 : i64 -// CHECK: arith.ori {{.*}}, %[[MODE]] -// CHECK: "buckyball.intr.mvin" -// CHECK: llvm.call @dma_touch_mvout_group -// CHECK: arith.ori {{.*}}, %[[MODE]] -// CHECK: "buckyball.intr.mvout" -func.func @selected(%input: memref<16x32xi8>, %output: memref<16x32xi8>, %bank: i64) { - %depth = arith.constant 16 : i64 - %stride = arith.constant 2 : i64 - buckyball.mvin %input %bank %depth %stride : memref<16x32xi8> i64 i64 i64 - buckyball.mvout %output %bank %depth %stride : memref<16x32xi8> i64 i64 i64 - return -} - -// CHECK-LABEL: func.func @whole -// CHECK-NOT: dma_touch_mvout_group -// CHECK: "buckyball.intr.mvin" -// CHECK: llvm.call @dma_touch_mvout -// CHECK: "buckyball.intr.mvout" -func.func @whole(%input: memref<16x48xi8>, %output: memref<16x48xi8>, %bank: i64) { - %depth = arith.constant 16 : i64 - %stride = arith.constant 1 : i64 - buckyball.mvin %input %bank %depth %stride : memref<16x48xi8> i64 i64 i64 - buckyball.mvout %output %bank %depth %stride : memref<16x48xi8> i64 i64 i64 - return -} diff --git a/bb-tests/workloads/src/MLIRTest/compiler/fp32_packmm_mixed.mlir b/bb-tests/workloads/src/MLIRTest/compiler/fp32_packmm_mixed.mlir deleted file mode 100644 index b5e6fff29..000000000 --- a/bb-tests/workloads/src/MLIRTest/compiler/fp32_packmm_mixed.mlir +++ /dev/null @@ -1,28 +0,0 @@ -// RUN: buddy-opt %s --target=attention -lower-buckyball-to-bank-ssa | FileCheck %s --check-prefix=SSA -// RUN: buddy-opt %s --target=attention -lower-buckyball-to-bank-ssa -assign-physical-banks | FileCheck %s --check-prefix=BANK -// SSA-LABEL: func.func @packed -// SSA-COUNT-3: buckyball.bank_transfer -// SSA: buckyball.bank_kernel @rvv_packmm -// SSA: scf.for -// SSA: buckyball.bank_mvin {{.*}} -// SSA: buckyball.bank_mvin {{.*}} -// SSA: buckyball.bank_kernel @rvv_packmm -// SSA: buckyball.bank_mvout {{.*}} -// SSA: buckyball.bank_kernel @rvv_packmm -// SSA-COUNT-2: buckyball.bank_release -// SSA: return -// BANK-LABEL: func.func @packed -// BANK-COUNT-3: buckyball.mset_transfer -// BANK: call @rvv_packmm -// BANK: buckyball.mvin {{.*}} -// BANK: buckyball.mvin {{.*}} -// BANK: call @rvv_packmm -// BANK: buckyball.mvout {{.*}} -// BANK: call @rvv_packmm -// BANK-NOT: buckyball.bank_ -// BANK: return -func.func @packed(%lhs: memref<16x8xf32>, %rhs: memref<8x13xf32>, - %output: memref<16x13xf32>) { - buckyball.fp32_mem_matmul %lhs, %rhs, %output : memref<16x8xf32>, memref<8x13xf32>, memref<16x13xf32> - return -} diff --git a/bb-tests/workloads/src/MLIRTest/compiler/mset_transfer_invalid.mlir b/bb-tests/workloads/src/MLIRTest/compiler/mset_transfer_invalid.mlir deleted file mode 100644 index e880a259a..000000000 --- a/bb-tests/workloads/src/MLIRTest/compiler/mset_transfer_invalid.mlir +++ /dev/null @@ -1,23 +0,0 @@ -// RUN: buddy-opt %s -split-input-file -verify-diagnostics -func.func @negative() { - %source = arith.constant -1 : i64 - %target = arith.constant 17 : i64 - // expected-error @+1 {{bank ID must be in [0, 1023]}} - buckyball.mset_transfer %source %target : i64 i64 - return -} -// ----- -func.func @overflow() { - %source = arith.constant 1024 : i64 - %target = arith.constant 17 : i64 - // expected-error @+1 {{bank ID must be in [0, 1023]}} - buckyball.mset_transfer %source %target : i64 i64 - return -} -// ----- -func.func @self() { - %source = arith.constant 17 : i64 - // expected-error @+1 {{source and target must differ}} - buckyball.mset_transfer %source %source : i64 i64 - return -} diff --git a/bb-tests/workloads/src/MLIRTest/compiler/mset_transfer_lowering.mlir b/bb-tests/workloads/src/MLIRTest/compiler/mset_transfer_lowering.mlir deleted file mode 100644 index d4202266a..000000000 --- a/bb-tests/workloads/src/MLIRTest/compiler/mset_transfer_lowering.mlir +++ /dev/null @@ -1,23 +0,0 @@ -// RUN: buddy-opt %s --target=toy -lower-buckyball -canonicalize | FileCheck %s -// CHECK-LABEL: func.func @dynamic -// CHECK-SAME: %[[SRC:.*]]: i64, %[[DST:.*]]: i64 -// CHECK: llvm.call @dma_bank_transfer(%[[SRC]], %[[DST]]) : (i64, i64) -> () -// CHECK: arith.shli %[[DST]] -// CHECK: arith.ori %[[SRC]] -// CHECK: "buckyball.intr.mset" -func.func @dynamic(%source: i64, %target: i64) { - buckyball.mset_transfer %source %target : i64 i64 - return -} - -// CHECK-LABEL: func.func @constant -// CHECK-DAG: %[[MODE:.*]] = arith.constant 4096 : i64 -// CHECK-DAG: %[[PACKED:.*]] = arith.constant 17825793 : i64 -// CHECK: llvm.call @dma_bank_transfer -// CHECK: "buckyball.intr.mset"(%[[PACKED]], %[[MODE]]) -func.func @constant() { - %source = arith.constant 1 : i64 - %target = arith.constant 17 : i64 - buckyball.mset_transfer %source %target : i64 i64 - return -} diff --git a/bb-tests/workloads/src/MLIRTest/compiler/norm_quant_window_mixed.mlir b/bb-tests/workloads/src/MLIRTest/compiler/norm_quant_window_mixed.mlir deleted file mode 100644 index fec444072..000000000 --- a/bb-tests/workloads/src/MLIRTest/compiler/norm_quant_window_mixed.mlir +++ /dev/null @@ -1,51 +0,0 @@ -// RUN: buddy-opt %s --target=ffn -fuse-norm-quant-window | FileCheck %s --check-prefix=SSA -// RUN: buddy-opt %s --target=ffn -fuse-norm-quant-window -assign-physical-banks | FileCheck %s --check-prefix=BANK - -// SSA-LABEL: func.func @mixed -// SSA-COUNT-3: buckyball.bank_transfer -// SSA: buckyball.bank_kernel @rvv_norm_window -// SSA: scf.for -// SSA: buckyball.bank_mvin {{.*}} -// SSA: buckyball.bank_kernel @rvv_norm_window -// SSA: buckyball.bank_mvin {{.*}} -// SSA: buckyball.bank_kernel @rvv_norm_window -// SSA: buckyball.bank_mvout {{.*}} -// SSA: buckyball.bank_kernel @rvv_norm_window -// SSA-COUNT-2: buckyball.bank_release -// SSA-NOT: memref.copy -// SSA: return -// BANK-LABEL: func.func @mixed -// BANK-COUNT-3: buckyball.mset_transfer -// BANK: call @rvv_norm_window -// BANK: scf.for -// BANK: buckyball.mvin {{.*}} -// BANK: call @rvv_norm_window -// BANK: buckyball.mvin {{.*}} -// BANK: call @rvv_norm_window -// BANK: buckyball.mvout {{.*}} -// BANK: call @rvv_norm_window -// BANK-NOT: buckyball.bank_ -// BANK: return -module { - func.func private @rvv_norm(memref<*xf32>, memref<*xf32>, memref<*xf32>, i32, i32) - func.func private @mxfp8_quant(memref<*xf32>, memref<*xi8>, i64, i64, i64) - func.func @mixed(%input: memref<1x64xf32>, %gamma: memref<64xf32>, - %weights: memref<131072xi8, strided<[1], offset: ?>>) -> memref<1x20xf32> { - %norm = memref.alloc() : memref<1x64xf32> - %packed = memref.alloc() : memref<65536xi8> - %output = memref.alloc() : memref<1x20xf32> - %norm_view = memref.cast %norm : memref<1x64xf32> to memref<*xf32> - %input_view = memref.cast %input : memref<1x64xf32> to memref<*xf32> - %gamma_view = memref.cast %gamma : memref<64xf32> to memref<*xf32> - %packed_view = memref.cast %packed : memref<65536xi8> to memref<*xi8> - %mean = arith.constant 1015021568 : i32 - %epsilon = arith.constant 897988541 : i32 - %one = arith.constant 1 : i64 - %chunk = arith.constant 32 : i64 - %bytes = arith.constant 32768 : i64 - call @rvv_norm(%norm_view, %input_view, %gamma_view, %mean, %epsilon) : (memref<*xf32>, memref<*xf32>, memref<*xf32>, i32, i32) -> () - call @mxfp8_quant(%norm_view, %packed_view, %one, %chunk, %bytes) : (memref<*xf32>, memref<*xi8>, i64, i64, i64) -> () - buckyball.mxfp8_mem_matmul %packed, %weights, %output : memref<65536xi8>, memref<131072xi8, strided<[1], offset: ?>>, memref<1x20xf32> - return %output : memref<1x20xf32> - } -} diff --git a/bb-tests/workloads/src/MLIRTest/compiler/rvv_rms_suffix.mlir b/bb-tests/workloads/src/MLIRTest/compiler/rvv_rms_suffix.mlir deleted file mode 100644 index 07295d225..000000000 --- a/bb-tests/workloads/src/MLIRTest/compiler/rvv_rms_suffix.mlir +++ /dev/null @@ -1,94 +0,0 @@ -// RUN: buddy-opt %s -outline-rvv-kernels -canonicalize | FileCheck %s -// CHECK-LABEL: func.func @weighted_residual -// CHECK: call @rvv_norm( -// CHECK-NOT: math.powf -// CHECK-NOT: linalg.reduce -// CHECK: linalg.generic -// CHECK: arith.addf -// CHECK-NEXT: {{.*}} = arith.mulf -// CHECK-NEXT: linalg.yield -// CHECK-LABEL: func.func @unweighted -// CHECK: call @rvv_norm_no_weight( -// CHECK-NOT: math.powf -// CHECK-NOT: linalg.reduce -// CHECK: return -#identity = affine_map<(d0, d1) -> (d0, d1)> -#row = affine_map<(d0, d1) -> (d0, 0)> -#weight = affine_map<(d0, d1) -> (0, d1)> -module { - func.func @weighted_residual(%x: tensor<2x4xf32>, %w: tensor<4xf32>, %residual: tensor<2x4xf32>) -> tensor<2x4xf32> { - %zero = arith.constant 0.0 : f32 - %mean = arith.constant 0.25 : f32 - %eps = arith.constant 0.000001 : f32 - %power = arith.constant -0.5 : f32 - %two = arith.constant 2 : i32 - %scale = arith.constant 0.5 : f32 - %empty = tensor.empty() : tensor<2x4xf32> - %square = linalg.generic {indexing_maps=[#identity,#identity],iterator_types=["parallel","parallel"]} ins(%x:tensor<2x4xf32>) outs(%empty:tensor<2x4xf32>) { - ^bb0(%v:f32,%o:f32): - %s = math.fpowi %v, %two : f32, i32 - linalg.yield %s : f32 - } -> tensor<2x4xf32> - %rows = tensor.empty() : tensor<2xf32> - %init = linalg.fill ins(%zero:f32) outs(%rows:tensor<2xf32>) -> tensor<2xf32> - %sum = linalg.reduce ins(%square:tensor<2x4xf32>) outs(%init:tensor<2xf32>) dimensions=[1] (%v:f32,%acc:f32) { - %s = arith.addf %v, %acc : f32 - linalg.yield %s : f32 - } - %expanded = tensor.expand_shape %sum [[0,1]] output_shape [2,1] : tensor<2xf32> into tensor<2x1xf32> - %inverseEmpty = tensor.empty() : tensor<2x1xf32> - %inverse = linalg.generic {indexing_maps=[#identity,#identity],iterator_types=["parallel","parallel"]} ins(%expanded:tensor<2x1xf32>) outs(%inverseEmpty:tensor<2x1xf32>) { - ^bb0(%v:f32,%o:f32): - %a = arith.mulf %mean, %v : f32 - %b = arith.addf %a, %eps : f32 - %c = math.powf %b, %power : f32 - linalg.yield %c : f32 - } -> tensor<2x1xf32> - %weights = tensor.expand_shape %w [[0,1]] output_shape [1,4] : tensor<4xf32> into tensor<1x4xf32> - %result = linalg.generic {indexing_maps=[#identity,#row,#weight,#identity,#identity],iterator_types=["parallel","parallel"]} ins(%x,%inverse,%weights,%residual:tensor<2x4xf32>,tensor<2x1xf32>,tensor<1x4xf32>,tensor<2x4xf32>) outs(%empty:tensor<2x4xf32>) { - ^bb0(%v:f32,%inv:f32,%gamma:f32,%res:f32,%o:f32): - %a = arith.mulf %v, %inv : f32 - %b = arith.mulf %a, %gamma : f32 - %c = arith.addf %b, %res : f32 - %d = arith.mulf %c, %scale : f32 - linalg.yield %d : f32 - } -> tensor<2x4xf32> - return %result : tensor<2x4xf32> - } - func.func @unweighted(%x: tensor<2x4xf32>, %w: tensor<4xf32>, %residual: tensor<2x4xf32>) -> tensor<2x4xf32> { - %zero = arith.constant 0.0 : f32 - %mean = arith.constant 0.25 : f32 - %eps = arith.constant 0.000001 : f32 - %power = arith.constant -0.5 : f32 - %two = arith.constant 2 : i32 - %scale = arith.constant 0.5 : f32 - %empty = tensor.empty() : tensor<2x4xf32> - %square = linalg.generic {indexing_maps=[#identity,#identity],iterator_types=["parallel","parallel"]} ins(%x:tensor<2x4xf32>) outs(%empty:tensor<2x4xf32>) { - ^bb0(%v:f32,%o:f32): - %s = math.fpowi %v, %two : f32, i32 - linalg.yield %s : f32 - } -> tensor<2x4xf32> - %rows = tensor.empty() : tensor<2xf32> - %init = linalg.fill ins(%zero:f32) outs(%rows:tensor<2xf32>) -> tensor<2xf32> - %sum = linalg.reduce ins(%square:tensor<2x4xf32>) outs(%init:tensor<2xf32>) dimensions=[1] (%v:f32,%acc:f32) { - %s = arith.addf %v, %acc : f32 - linalg.yield %s : f32 - } - %expanded = tensor.expand_shape %sum [[0,1]] output_shape [2,1] : tensor<2xf32> into tensor<2x1xf32> - %inverseEmpty = tensor.empty() : tensor<2x1xf32> - %inverse = linalg.generic {indexing_maps=[#identity,#identity],iterator_types=["parallel","parallel"]} ins(%expanded:tensor<2x1xf32>) outs(%inverseEmpty:tensor<2x1xf32>) { - ^bb0(%v:f32,%o:f32): - %a = arith.mulf %mean, %v : f32 - %b = arith.addf %a, %eps : f32 - %c = math.powf %b, %power : f32 - linalg.yield %c : f32 - } -> tensor<2x1xf32> - %weights = tensor.expand_shape %w [[0,1]] output_shape [1,4] : tensor<4xf32> into tensor<1x4xf32> - %result = linalg.generic {indexing_maps=[#identity,#row,#weight,#identity,#identity],iterator_types=["parallel","parallel"]} ins(%x,%inverse,%weights,%residual:tensor<2x4xf32>,tensor<2x1xf32>,tensor<1x4xf32>,tensor<2x4xf32>) outs(%empty:tensor<2x4xf32>) { - ^bb0(%v:f32,%inv:f32,%gamma:f32,%res:f32,%o:f32): - %a = arith.mulf %v, %inv : f32 - linalg.yield %a : f32 - } -> tensor<2x4xf32> - return %result : tensor<2x4xf32> - } -} diff --git a/bb-tests/workloads/src/MLIRTest/compiler/rvv_tanh.mlir b/bb-tests/workloads/src/MLIRTest/compiler/rvv_tanh.mlir deleted file mode 100644 index d515a5618..000000000 --- a/bb-tests/workloads/src/MLIRTest/compiler/rvv_tanh.mlir +++ /dev/null @@ -1,61 +0,0 @@ -// RUN: buddy-opt %s -outline-rvv-kernels -canonicalize | FileCheck %s -// CHECK-LABEL: func.func @tanh -// CHECK: call @rvv_tanh -// CHECK-NOT: math.tanh -// CHECK: return -// CHECK-LABEL: func.func @mixed_expression -// CHECK-NOT: call @rvv_tanh -// CHECK: math.tanh -// CHECK: arith.addf -// CHECK: return -// CHECK-LABEL: func.func @sin -// CHECK: call @rvv_sin -// CHECK-NOT: math.sin -// CHECK: return -// CHECK-LABEL: func.func @cos -// CHECK: call @rvv_cos -// CHECK-NOT: math.cos -// CHECK: return -#identity = affine_map<(d0) -> (d0)> -module { - func.func @tanh(%x: tensor<17xf32>) -> tensor<17xf32> { - %empty = tensor.empty() : tensor<17xf32> - %result = linalg.generic {indexing_maps = [#identity, #identity], iterator_types = ["parallel"]} ins(%x : tensor<17xf32>) outs(%empty : tensor<17xf32>) { - ^bb0(%xval: f32, %out: f32): - %t = math.tanh %xval : f32 - linalg.yield %t : f32 - } -> tensor<17xf32> - return %result : tensor<17xf32> - } - func.func @mixed_expression(%x: tensor<17xf32>) -> tensor<17xf32> { - %empty = tensor.empty() : tensor<17xf32> - %one = arith.constant 1.0 : f32 - %result = linalg.generic {indexing_maps = [#identity, #identity], iterator_types = ["parallel"]} ins(%x : tensor<17xf32>) outs(%empty : tensor<17xf32>) { - ^bb0(%xval: f32, %out: f32): - %t = math.tanh %xval : f32 - %a = arith.addf %t, %one : f32 - linalg.yield %a : f32 - } -> tensor<17xf32> - return %result : tensor<17xf32> - } - func.func @sin(%x: tensor<17xf32>) -> tensor<17xf32> { - %empty = tensor.empty() : tensor<17xf32> - %result = linalg.generic {indexing_maps = [#identity, #identity], iterator_types = ["parallel"]} ins(%x : tensor<17xf32>) outs(%empty : tensor<17xf32>) { - ^bb0(%xval: f32, %out: f32): - %t = math.sin %xval : f32 - linalg.yield %t : f32 - } -> tensor<17xf32> - return %result : tensor<17xf32> - } - - func.func @cos(%x: tensor<17xf32>) -> tensor<17xf32> { - %empty = tensor.empty() : tensor<17xf32> - %result = linalg.generic {indexing_maps = [#identity, #identity], iterator_types = ["parallel"]} ins(%x : tensor<17xf32>) outs(%empty : tensor<17xf32>) { - ^bb0(%xval: f32, %out: f32): - %t = math.cos %xval : f32 - linalg.yield %t : f32 - } -> tensor<17xf32> - return %result : tensor<17xf32> - } - -} diff --git a/bb-tests/workloads/src/SocTest/CMakeLists.txt b/bb-tests/workloads/src/SocTest/CMakeLists.txt index cf7be0508..80dd35e77 100644 --- a/bb-tests/workloads/src/SocTest/CMakeLists.txt +++ b/bb-tests/workloads/src/SocTest/CMakeLists.txt @@ -1,62 +1,64 @@ # =============================================================== -# SoC tests: CPU-visible chip devices (CLINT, PLIC, SCU) exercised on -# the chip's boot hart, independent of any Ball ISA. +# SoC tests: CPU-visible design devices (CLINT, PLIC, SCU) exercised on +# the design's boot hart, independent of any Ball ISA. # =============================================================== set(SOCTEST_MAX_LINES 100) set(BUCKYBALL_SOCTEST_C_FLAGS - -g -fno-common -O2 -static -march=rv64gc -mcmodel=medany - -fno-builtin-printf -specs=nano.specs -specs=nosys.specs -nostartfiles - -Wl,-T,${BBSIM_LD} + -g + -fno-common + -O2 + -static + -march=rv64gc + -mcmodel=medany + -fno-builtin-printf + -specs=nano.specs + -specs=nosys.specs + -nostartfiles + -Wl,-T,${BBSIM_LD} ) -# The target whose generated profile boots hart 0 owns the SoC tests. -unset(BUCKYBALL_SOCTEST_TARGET) -foreach(_target IN LISTS BUCKYBALL_TARGETS) - file(STRINGS "${BUCKYBALL_ISA_DIR}/${_target}/params.h" _boot - REGEX "^#define BB_TEST_HART 0$") - if(_boot) - set(BUCKYBALL_SOCTEST_TARGET ${_target}) - break() - endif() -endforeach() -if(NOT BUCKYBALL_SOCTEST_TARGET) - message(FATAL_ERROR - "SoC tests need a target with BB_TEST_HART 0; targets: ${BUCKYBALL_TARGETS}") -endif() +set(BUCKYBALL_SOCTEST_TARGET ${BUCKYBALL_CONTROL_TARGET}) add_custom_target(soctests-build) function(add_buckyball_soctest SOURCE_FILE) - get_filename_component(_stem ${SOURCE_FILE} NAME_WE) - buckyball_soctest_name(TEST_NAME ${BUCKYBALL_SOCTEST_TARGET} ${_stem}) - set(SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/${SOURCE_FILE}) - file(STRINGS ${SOURCE} _lines) - list(LENGTH _lines _n) - if(_n GREATER SOCTEST_MAX_LINES) - message(FATAL_ERROR - "soctest exceeds ${SOCTEST_MAX_LINES} lines: ${SOURCE} (${_n} lines)") - endif() + get_filename_component(_stem ${SOURCE_FILE} NAME_WE) + set(TEST_NAME + "${DESIGN_NAME}-${BUCKYBALL_SOCTEST_TARGET}-soctest-${_stem}" + ) + set(SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/${SOURCE_FILE}) + file(STRINGS ${SOURCE} _lines) + list(LENGTH _lines _n) + if(_n GREATER SOCTEST_MAX_LINES) + message( + FATAL_ERROR + "soctest exceeds ${SOCTEST_MAX_LINES} lines: ${SOURCE} (${_n} lines)" + ) + endif() - set(EXECUTABLE "${TEST_NAME}-baremetal") - set(PARAMS ${BUCKYBALL_ISA_DIR}/${BUCKYBALL_SOCTEST_TARGET}/params.h) - add_custom_command( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${EXECUTABLE} - COMMAND ${ELF_CC} ${BUCKYBALL_SOCTEST_C_FLAGS} - -o ${EXECUTABLE} - ${BBSW_BAREMETAL_DIR}/crt0.S - ${BBSW_BAREMETAL_DIR}/syscalls.c - ${SOURCE} - -I${BUCKYBALL_ISA_DIR}/chip - -I${BUCKYBALL_ISA_DIR}/${BUCKYBALL_SOCTEST_TARGET} - -I${CMAKE_CURRENT_SOURCE_DIR} - DEPENDS ${SOURCE} ${PARAMS} ${BBSW_BAREMETAL_DIR}/crt0.S ${BBSIM_LD} - ${BBSW_BAREMETAL_DIR}/syscalls.c ${CMAKE_CURRENT_SOURCE_DIR}/soc.h - COMMENT "Building SoC test: ${EXECUTABLE}" - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - ) - add_custom_target(${TEST_NAME}-baremetal-build - DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${EXECUTABLE}) - add_dependencies(soctests-build ${TEST_NAME}-baremetal-build) + set(EXECUTABLE "${TEST_NAME}-baremetal") + add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${EXECUTABLE} + COMMAND + ${ELF_CC} ${BUCKYBALL_SOCTEST_C_FLAGS} -o ${EXECUTABLE} + ${BAREMETAL_DIR}/crt0.S ${BAREMETAL_DIR}/syscalls.c ${SOURCE} + -DCORE_HART_ID=${DESIGN_WORKLOAD_CONTROL_HART} ${DESIGN_DEFINES} + -I${CMAKE_CURRENT_SOURCE_DIR} + DEPENDS + ${SOURCE} + ${BAREMETAL_DIR}/crt0.S + ${BBSIM_LD} + ${BAREMETAL_DIR}/syscalls.c + ${CMAKE_CURRENT_SOURCE_DIR}/soc.h + COMMENT "Building SoC test: ${EXECUTABLE}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM + ) + add_custom_target( + ${TEST_NAME}-baremetal-build + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${EXECUTABLE} + ) + add_dependencies(soctests-build ${TEST_NAME}-baremetal-build) endfunction() add_buckyball_soctest(clint_timer_interrupt.c) @@ -68,10 +70,13 @@ add_buckyball_soctest(lrsc_word.c) add_buckyball_soctest(lrsc_translated.c) add_buckyball_soctest(plic_register_access.c) -add_custom_target(sync-soctest-bin - COMMAND ${CMAKE_COMMAND} -E make_directory ${OUTPUT_BIN_DIR}/src/SocTest - COMMAND rsync -a --delete --exclude=CMakeFiles "--include=*/" - "--include=*-baremetal" "--include=*-baremetal.expect" "--exclude=*" - ${CMAKE_CURRENT_BINARY_DIR}/ ${OUTPUT_BIN_DIR}/src/SocTest/ - DEPENDS soctests-build - COMMENT "Syncing SoC test binaries to output directory") +add_custom_target( + sync-soctest-bin + COMMAND ${CMAKE_COMMAND} -E make_directory ${OUTPUT_BIN_DIR}/src/SocTest + COMMAND + rsync -a --delete --exclude=CMakeFiles "--include=*/" + "--include=*-baremetal" "--include=*-baremetal.expect" "--exclude=*" + ${CMAKE_CURRENT_BINARY_DIR}/ ${OUTPUT_BIN_DIR}/src/SocTest/ + DEPENDS soctests-build + COMMENT "Syncing SoC test binaries to output directory" +) diff --git a/bb-tests/workloads/src/SocTest/soc.h b/bb-tests/workloads/src/SocTest/soc.h index f6bbe9e73..fe111b313 100644 --- a/bb-tests/workloads/src/SocTest/soc.h +++ b/bb-tests/workloads/src/SocTest/soc.h @@ -1,7 +1,7 @@ #pragma once #include -// Chip device map, matching framework.system.device.DeviceParams. +// Design device map, matching framework.system.device.DeviceParams. #define CLINT_BASE 0x02000000UL #define PLIC_BASE 0x0C000000UL diff --git a/bbdev b/bbdev deleted file mode 160000 index 5dcb56cbb..000000000 --- a/bbdev +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 5dcb56cbbfc33a96e70b2236d8e468a68a2502bc diff --git a/bbdev/.github/workflows/nix.yml b/bbdev/.github/workflows/nix.yml new file mode 100644 index 000000000..ea29f7976 --- /dev/null +++ b/bbdev/.github/workflows/nix.yml @@ -0,0 +1,16 @@ +name: Nix + +on: + pull_request: + push: + branches: [main] + +jobs: + build: + name: Build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/determinate-nix-action@v3 + - name: Build bbdev + run: nix develop diff --git a/bbdev/.gitignore b/bbdev/.gitignore new file mode 100644 index 000000000..2574abf3b --- /dev/null +++ b/bbdev/.gitignore @@ -0,0 +1,28 @@ +node_modules/ +python_modules/ +.venv/ +venv/ +.motia/ +.mermaid/ +dist/ +__pycache__/ +*.pyc +motia-workbench.json +types.d.ts +appendonlydir/ +dump.rdb +*.log +*.jou +dramsim3* +output/ +result/ + +.cursor/ +.claude/ + +# Temp iii config files generated by bbdev script mode +api/iii-config-*.yaml +api/data/ + +api/steps/ip/scripts/_smoke_out/ +lc_output.txt diff --git a/arch/src/main/scala/framework/rvv/LICENSE b/bbdev/LICENSE similarity index 100% rename from arch/src/main/scala/framework/rvv/LICENSE rename to bbdev/LICENSE diff --git a/bbdev/README.md b/bbdev/README.md new file mode 100644 index 000000000..5445e2108 --- /dev/null +++ b/bbdev/README.md @@ -0,0 +1,18 @@ +# bbdev +Agent-friendly developer toolchain backend + +## Build + +```bash +# Enter dev environment +nix develop + +# Install dependencies +cd api +pnpm install + +# Start dev server +pnpm dev +``` + +This Project is **Built with 💙 by [Motia](https://motia.dev)** diff --git a/bbdev/api/config.yaml b/bbdev/api/config.yaml new file mode 100644 index 000000000..6f0ed8878 --- /dev/null +++ b/bbdev/api/config.yaml @@ -0,0 +1,63 @@ +port: 49134 + +modules: + # ── Worker Module ────────────────────────────────────────────────────── + - class: modules::worker::WorkerModule + config: + port: 49134 + host: 0.0.0.0 + + # ── State Module ─────────────────────────────────────────────────────── + - class: modules::state::StateModule + config: + adapter: + class: modules::state::adapters::KvStore + config: + store_method: file_based + file_path: ./data/state_store.db + + # ── REST API Module ──────────────────────────────────────────────────── + - class: modules::api::RestApiModule + config: + port: 3111 + host: 0.0.0.0 + default_timeout: 86400000 + concurrency_request_limit: 1024 + cors: + allowed_origins: + - http://localhost:3000 + - http://localhost:5173 + allowed_methods: + - GET + - POST + - PUT + - DELETE + - OPTIONS + + # ── Queue Module ─────────────────────────────────────────────────────── + - class: modules::queue::QueueModule + config: + adapter: + class: modules::queue::BuiltinQueueAdapter + config: + max_attempts: 1 + + # ── PubSub Module ────────────────────────────────────────────────────── + - class: modules::pubsub::PubSubModule + config: + adapter: + class: modules::pubsub::LocalAdapter + + # ── Observability Module ─────────────────────────────────────────────── + - class: modules::observability::OtelModule + config: + enabled: true + exporter: memory + + # ── Exec Module (Python) ─────────────────────────────────────────────── + - class: modules::shell::ExecModule + config: + watch: + - steps/**/*.py + exec: + - PYTHONUNBUFFERED=1 PYTHONPATH=. .venv/bin/motia dev --dir steps diff --git a/bbdev/api/motia.config.ts b/bbdev/api/motia.config.ts new file mode 100644 index 000000000..8e7e1684e --- /dev/null +++ b/bbdev/api/motia.config.ts @@ -0,0 +1,9 @@ +import { defineConfig } from '@motiadev/core' +import endpointPlugin from '@motiadev/plugin-endpoint/plugin' +import logsPlugin from '@motiadev/plugin-logs/plugin' +import observabilityPlugin from '@motiadev/plugin-observability/plugin' +import statesPlugin from '@motiadev/plugin-states/plugin' + +export default defineConfig({ + plugins: [observabilityPlugin, statesPlugin, endpointPlugin, logsPlugin], +}) diff --git a/bbdev/api/package.json b/bbdev/api/package.json new file mode 100644 index 000000000..ab2d7fdb1 --- /dev/null +++ b/bbdev/api/package.json @@ -0,0 +1,29 @@ +{ + "name": "bbdev", + "version": "0.1.0", + "description": "Agent-friendly developer toolchain backend powered by Motia", + "scripts": { + "postinstall": "motia install", + "dev": "motia dev", + "dev:debug": "motia dev --debug", + "build": "motia build", + "start": "motia start", + "clean": "rm -rf dist node_modules python_modules .motia .mermaid" + }, + "keywords": [ + "motia" + ], + "dependencies": { + "motia": "0.17.14-beta.196", + "zod": "^3.24.4", + "@motiadev/core": "0.17.14-beta.196", + "@motiadev/plugin-logs": "0.17.14-beta.196", + "@motiadev/plugin-states": "0.17.14-beta.196", + "@motiadev/plugin-endpoint": "0.17.14-beta.196", + "@motiadev/plugin-observability": "0.17.14-beta.196", + "@motiadev/plugin-bullmq": "0.17.14-beta.196", + "@motiadev/plugin-streams": "0.17.14-beta.196", + "@motiadev/plugin-cron": "0.17.14-beta.196", + "@motiadev/plugin-ws": "0.17.14-beta.196" + } +} diff --git a/bbdev/api/pnpm-lock.yaml b/bbdev/api/pnpm-lock.yaml new file mode 100644 index 000000000..5bda0622c --- /dev/null +++ b/bbdev/api/pnpm-lock.yaml @@ -0,0 +1,6913 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + '@motiadev/core': + specifier: 0.17.14-beta.196 + version: 0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3) + '@motiadev/plugin-bullmq': + specifier: 0.17.14-beta.196 + version: 0.17.14-beta.196(@motiadev/adapter-bullmq-events@0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3)))(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))(@motiadev/stream-client-react@0.17.14-beta.196(react@19.2.4))(@motiadev/ui@0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))(lucide-react@0.555.0(react@19.2.4))(motia@0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)))(react@19.2.4)(zustand@5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4))) + '@motiadev/plugin-cron': + specifier: 0.17.14-beta.196 + version: 0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))(@motiadev/stream-client-react@0.17.14-beta.196(react@19.2.4))(@motiadev/ui@0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))(use-sync-external-store@1.6.0(react@19.2.4)) + '@motiadev/plugin-endpoint': + specifier: 0.17.14-beta.196 + version: 0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))(@motiadev/stream-client-react@0.17.14-beta.196(react@19.2.4))(@motiadev/ui@0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))(lucide-react@0.555.0(react@19.2.4))(monaco-editor@0.55.1)(motia@0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(zustand@5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4))) + '@motiadev/plugin-logs': + specifier: 0.17.14-beta.196 + version: 0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))(@motiadev/stream-client-react@0.17.14-beta.196(react@19.2.4))(@motiadev/ui@0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))(lucide-react@0.555.0(react@19.2.4))(motia@0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(zustand@5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4))) + '@motiadev/plugin-observability': + specifier: 0.17.14-beta.196 + version: 0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))(@motiadev/stream-client-react@0.17.14-beta.196(react@19.2.4))(@motiadev/ui@0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))(lucide-react@0.555.0(react@19.2.4))(motia@0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(zustand@5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4))) + '@motiadev/plugin-states': + specifier: 0.17.14-beta.196 + version: 0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))(@motiadev/ui@0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))(lucide-react@0.555.0(react@19.2.4))(monaco-editor@0.55.1)(motia@0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(zustand@5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4))) + '@motiadev/plugin-streams': + specifier: 0.17.14-beta.196 + version: 0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))(@motiadev/ui@0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))(lucide-react@0.555.0(react@19.2.4))(motia@0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(zustand@5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4))) + '@motiadev/plugin-ws': + specifier: 0.17.14-beta.196 + version: 0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))(@motiadev/ui@0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))(use-sync-external-store@1.6.0(react@19.2.4)) + motia: + specifier: 0.17.14-beta.196 + version: 0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)) + zod: + specifier: ^3.24.4 + version: 3.25.76 + +packages: + + '@alloc/quick-lru@5.2.0': + resolution: {integrity: sha512-UrcABB+4bUrFABwbluTIBErXwvbsU/V7TZWfmbgJfbkwiBuziS9gxdODUyuiecfdGQ85jglMW6juS3+z5TsKLw==} + engines: {node: '>=10'} + + '@amplitude/analytics-connector@1.6.4': + resolution: {integrity: sha512-SpIv0IQMNIq6SH3UqFGiaZyGSc7PBZwRdq7lvP0pBxW8i4Ny+8zwI0pV+VMfMHQwWY3wdIbWw5WQphNjpdq1/Q==} + + '@amplitude/analytics-core@2.38.0': + resolution: {integrity: sha512-cYFNox/mb8uWxdUFEMsS5Uj79BiWm3t1Dm+y5AS085tjbZ2RIxYsytb7ieeWCxd2cXEIpDPbcYhScU4Hm1Ky9g==} + + '@amplitude/analytics-node@1.5.35': + resolution: {integrity: sha512-V/W8Wy2g0hACizaYPYvkXqvItrRnb37CjSddIsCIs9GymZE2F0Vs9pWKXJjsPR0Pv2PB+qIZg4qjcXQlc+G42Q==} + + '@babel/code-frame@7.29.0': + resolution: {integrity: sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw==} + engines: {node: '>=6.9.0'} + + '@babel/compat-data@7.29.0': + resolution: {integrity: sha512-T1NCJqT/j9+cn8fvkt7jtwbLBfLC/1y1c7NtCeXFRgzGTsafi68MRv8yzkYSapBnFA6L3U2VSc02ciDzoAJhJg==} + engines: {node: '>=6.9.0'} + + '@babel/core@7.29.0': + resolution: {integrity: sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==} + engines: {node: '>=6.9.0'} + + '@babel/generator@7.29.1': + resolution: {integrity: sha512-qsaF+9Qcm2Qv8SRIMMscAvG4O3lJ0F1GuMo5HR/Bp02LopNgnZBC/EkbevHFeGs4ls/oPz9v+Bsmzbkbe+0dUw==} + engines: {node: '>=6.9.0'} + + '@babel/helper-compilation-targets@7.28.6': + resolution: {integrity: sha512-JYtls3hqi15fcx5GaSNL7SCTJ2MNmjrkHXg4FSpOA/grxK8KwyZ5bubHsCq8FXCkua6xhuaaBit+3b7+VZRfcA==} + engines: {node: '>=6.9.0'} + + '@babel/helper-globals@7.28.0': + resolution: {integrity: sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw==} + engines: {node: '>=6.9.0'} + + '@babel/helper-module-imports@7.28.6': + resolution: {integrity: sha512-l5XkZK7r7wa9LucGw9LwZyyCUscb4x37JWTPz7swwFE/0FMQAGpiWUZn8u9DzkSBWEcK25jmvubfpw2dnAMdbw==} + engines: {node: '>=6.9.0'} + + '@babel/helper-module-transforms@7.28.6': + resolution: {integrity: sha512-67oXFAYr2cDLDVGLXTEABjdBJZ6drElUSI7WKp70NrpyISso3plG9SAGEF6y7zbha/wOzUByWWTJvEDVNIUGcA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + + '@babel/helper-plugin-utils@7.28.6': + resolution: {integrity: sha512-S9gzZ/bz83GRysI7gAD4wPT/AI3uCnY+9xn+Mx/KPs2JwHJIz1W8PZkg2cqyt3RNOBM8ejcXhV6y8Og7ly/Dug==} + engines: {node: '>=6.9.0'} + + '@babel/helper-string-parser@7.27.1': + resolution: {integrity: sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==} + engines: {node: '>=6.9.0'} + + '@babel/helper-validator-identifier@7.28.5': + resolution: {integrity: sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==} + engines: {node: '>=6.9.0'} + + '@babel/helper-validator-option@7.27.1': + resolution: {integrity: sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg==} + engines: {node: '>=6.9.0'} + + '@babel/helpers@7.28.6': + resolution: {integrity: sha512-xOBvwq86HHdB7WUDTfKfT/Vuxh7gElQ+Sfti2Cy6yIWNW05P8iUslOVcZ4/sKbE+/jQaukQAdz/gf3724kYdqw==} + engines: {node: '>=6.9.0'} + + '@babel/parser@7.29.0': + resolution: {integrity: sha512-IyDgFV5GeDUVX4YdF/3CPULtVGSXXMLh1xVIgdCgxApktqnQV0r7/8Nqthg+8YLGaAtdyIlo2qIdZrbCv4+7ww==} + engines: {node: '>=6.0.0'} + hasBin: true + + '@babel/plugin-transform-react-jsx-self@7.27.1': + resolution: {integrity: sha512-6UzkCs+ejGdZ5mFFC/OCUrv028ab2fp1znZmCZjAOBKiBK2jXD1O+BPSfX8X2qjJ75fZBMSnQn3Rq2mrBJK2mw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + + '@babel/plugin-transform-react-jsx-source@7.27.1': + resolution: {integrity: sha512-zbwoTsBruTeKB9hSq73ha66iFeJHuaFkUbwvqElnygoNbj/jHRsSeokowZFN3CZ64IvEqcmmkVe89OPXc7ldAw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + + '@babel/runtime@7.28.6': + resolution: {integrity: sha512-05WQkdpL9COIMz4LjTxGpPNCdlpyimKppYNoJ5Di5EUObifl8t4tuLuUBBZEpoLYOmfvIWrsp9fCl0HoPRVTdA==} + engines: {node: '>=6.9.0'} + + '@babel/template@7.28.6': + resolution: {integrity: sha512-YA6Ma2KsCdGb+WC6UpBVFJGXL58MDA6oyONbjyF/+5sBgxY/dwkhLogbMT2GXXyU84/IhRw/2D1Os1B/giz+BQ==} + engines: {node: '>=6.9.0'} + + '@babel/traverse@7.29.0': + resolution: {integrity: sha512-4HPiQr0X7+waHfyXPZpWPfWL/J7dcN1mx9gL6WdQVMbPnF3+ZhSMs8tCxN7oHddJE9fhNE7+lxdnlyemKfJRuA==} + engines: {node: '>=6.9.0'} + + '@babel/types@7.29.0': + resolution: {integrity: sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A==} + engines: {node: '>=6.9.0'} + + '@cspotcode/source-map-support@0.8.1': + resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} + engines: {node: '>=12'} + + '@esbuild/aix-ppc64@0.25.12': + resolution: {integrity: sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/aix-ppc64@0.27.3': + resolution: {integrity: sha512-9fJMTNFTWZMh5qwrBItuziu834eOCUcEqymSH7pY+zoMVEZg3gcPuBNxH1EvfVYe9h0x/Ptw8KBzv7qxb7l8dg==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/android-arm64@0.25.12': + resolution: {integrity: sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm64@0.27.3': + resolution: {integrity: sha512-YdghPYUmj/FX2SYKJ0OZxf+iaKgMsKHVPF1MAq/P8WirnSpCStzKJFjOjzsW0QQ7oIAiccHdcqjbHmJxRb/dmg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm@0.25.12': + resolution: {integrity: sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-arm@0.27.3': + resolution: {integrity: sha512-i5D1hPY7GIQmXlXhs2w8AWHhenb00+GxjxRncS2ZM7YNVGNfaMxgzSGuO8o8SJzRc/oZwU2bcScvVERk03QhzA==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-x64@0.25.12': + resolution: {integrity: sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/android-x64@0.27.3': + resolution: {integrity: sha512-IN/0BNTkHtk8lkOM8JWAYFg4ORxBkZQf9zXiEOfERX/CzxW3Vg1ewAhU7QSWQpVIzTW+b8Xy+lGzdYXV6UZObQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/darwin-arm64@0.25.12': + resolution: {integrity: sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-arm64@0.27.3': + resolution: {integrity: sha512-Re491k7ByTVRy0t3EKWajdLIr0gz2kKKfzafkth4Q8A5n1xTHrkqZgLLjFEHVD+AXdUGgQMq+Godfq45mGpCKg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-x64@0.25.12': + resolution: {integrity: sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/darwin-x64@0.27.3': + resolution: {integrity: sha512-vHk/hA7/1AckjGzRqi6wbo+jaShzRowYip6rt6q7VYEDX4LEy1pZfDpdxCBnGtl+A5zq8iXDcyuxwtv3hNtHFg==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/freebsd-arm64@0.25.12': + resolution: {integrity: sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-arm64@0.27.3': + resolution: {integrity: sha512-ipTYM2fjt3kQAYOvo6vcxJx3nBYAzPjgTCk7QEgZG8AUO3ydUhvelmhrbOheMnGOlaSFUoHXB6un+A7q4ygY9w==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.25.12': + resolution: {integrity: sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.27.3': + resolution: {integrity: sha512-dDk0X87T7mI6U3K9VjWtHOXqwAMJBNN2r7bejDsc+j03SEjtD9HrOl8gVFByeM0aJksoUuUVU9TBaZa2rgj0oA==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/linux-arm64@0.25.12': + resolution: {integrity: sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm64@0.27.3': + resolution: {integrity: sha512-sZOuFz/xWnZ4KH3YfFrKCf1WyPZHakVzTiqji3WDc0BCl2kBwiJLCXpzLzUBLgmp4veFZdvN5ChW4Eq/8Fc2Fg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm@0.25.12': + resolution: {integrity: sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-arm@0.27.3': + resolution: {integrity: sha512-s6nPv2QkSupJwLYyfS+gwdirm0ukyTFNl3KTgZEAiJDd+iHZcbTPPcWCcRYH+WlNbwChgH2QkE9NSlNrMT8Gfw==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-ia32@0.25.12': + resolution: {integrity: sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-ia32@0.27.3': + resolution: {integrity: sha512-yGlQYjdxtLdh0a3jHjuwOrxQjOZYD/C9PfdbgJJF3TIZWnm/tMd/RcNiLngiu4iwcBAOezdnSLAwQDPqTmtTYg==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-loong64@0.25.12': + resolution: {integrity: sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-loong64@0.27.3': + resolution: {integrity: sha512-WO60Sn8ly3gtzhyjATDgieJNet/KqsDlX5nRC5Y3oTFcS1l0KWba+SEa9Ja1GfDqSF1z6hif/SkpQJbL63cgOA==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-mips64el@0.25.12': + resolution: {integrity: sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-mips64el@0.27.3': + resolution: {integrity: sha512-APsymYA6sGcZ4pD6k+UxbDjOFSvPWyZhjaiPyl/f79xKxwTnrn5QUnXR5prvetuaSMsb4jgeHewIDCIWljrSxw==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-ppc64@0.25.12': + resolution: {integrity: sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-ppc64@0.27.3': + resolution: {integrity: sha512-eizBnTeBefojtDb9nSh4vvVQ3V9Qf9Df01PfawPcRzJH4gFSgrObw+LveUyDoKU3kxi5+9RJTCWlj4FjYXVPEA==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-riscv64@0.25.12': + resolution: {integrity: sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-riscv64@0.27.3': + resolution: {integrity: sha512-3Emwh0r5wmfm3ssTWRQSyVhbOHvqegUDRd0WhmXKX2mkHJe1SFCMJhagUleMq+Uci34wLSipf8Lagt4LlpRFWQ==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-s390x@0.25.12': + resolution: {integrity: sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-s390x@0.27.3': + resolution: {integrity: sha512-pBHUx9LzXWBc7MFIEEL0yD/ZVtNgLytvx60gES28GcWMqil8ElCYR4kvbV2BDqsHOvVDRrOxGySBM9Fcv744hw==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-x64@0.25.12': + resolution: {integrity: sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/linux-x64@0.27.3': + resolution: {integrity: sha512-Czi8yzXUWIQYAtL/2y6vogER8pvcsOsk5cpwL4Gk5nJqH5UZiVByIY8Eorm5R13gq+DQKYg0+JyQoytLQas4dA==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/netbsd-arm64@0.25.12': + resolution: {integrity: sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-arm64@0.27.3': + resolution: {integrity: sha512-sDpk0RgmTCR/5HguIZa9n9u+HVKf40fbEUt+iTzSnCaGvY9kFP0YKBWZtJaraonFnqef5SlJ8/TiPAxzyS+UoA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.25.12': + resolution: {integrity: sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.27.3': + resolution: {integrity: sha512-P14lFKJl/DdaE00LItAukUdZO5iqNH7+PjoBm+fLQjtxfcfFE20Xf5CrLsmZdq5LFFZzb5JMZ9grUwvtVYzjiA==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-arm64@0.25.12': + resolution: {integrity: sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-arm64@0.27.3': + resolution: {integrity: sha512-AIcMP77AvirGbRl/UZFTq5hjXK+2wC7qFRGoHSDrZ5v5b8DK/GYpXW3CPRL53NkvDqb9D+alBiC/dV0Fb7eJcw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.25.12': + resolution: {integrity: sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.27.3': + resolution: {integrity: sha512-DnW2sRrBzA+YnE70LKqnM3P+z8vehfJWHXECbwBmH/CU51z6FiqTQTHFenPlHmo3a8UgpLyH3PT+87OViOh1AQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openharmony-arm64@0.25.12': + resolution: {integrity: sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + + '@esbuild/openharmony-arm64@0.27.3': + resolution: {integrity: sha512-NinAEgr/etERPTsZJ7aEZQvvg/A6IsZG/LgZy+81wON2huV7SrK3e63dU0XhyZP4RKGyTm7aOgmQk0bGp0fy2g==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + + '@esbuild/sunos-x64@0.25.12': + resolution: {integrity: sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/sunos-x64@0.27.3': + resolution: {integrity: sha512-PanZ+nEz+eWoBJ8/f8HKxTTD172SKwdXebZ0ndd953gt1HRBbhMsaNqjTyYLGLPdoWHy4zLU7bDVJztF5f3BHA==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/win32-arm64@0.25.12': + resolution: {integrity: sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-arm64@0.27.3': + resolution: {integrity: sha512-B2t59lWWYrbRDw/tjiWOuzSsFh1Y/E95ofKz7rIVYSQkUYBjfSgf6oeYPNWHToFRr2zx52JKApIcAS/D5TUBnA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-ia32@0.25.12': + resolution: {integrity: sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-ia32@0.27.3': + resolution: {integrity: sha512-QLKSFeXNS8+tHW7tZpMtjlNb7HKau0QDpwm49u0vUp9y1WOF+PEzkU84y9GqYaAVW8aH8f3GcBck26jh54cX4Q==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-x64@0.25.12': + resolution: {integrity: sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + + '@esbuild/win32-x64@0.27.3': + resolution: {integrity: sha512-4uJGhsxuptu3OcpVAzli+/gWusVGwZZHTlS63hh++ehExkVT8SgiEf7/uC/PclrPPkLhZqGgCTjd0VWLo6xMqA==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + + '@eslint-community/eslint-utils@4.9.1': + resolution: {integrity: sha512-phrYmNiYppR7znFEdqgfWHXR6NCkZEK7hwWDHZUjit/2/U0r6XvkDl0SYnoM51Hq7FhCGdLDT6zxCCOY1hexsQ==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + peerDependencies: + eslint: ^6.0.0 || ^7.0.0 || >=8.0.0 + + '@eslint-community/regexpp@4.12.2': + resolution: {integrity: sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==} + engines: {node: ^12.0.0 || ^14.0.0 || >=16.0.0} + + '@eslint/config-array@0.21.1': + resolution: {integrity: sha512-aw1gNayWpdI/jSYVgzN5pL0cfzU02GT3NBpeT/DXbx1/1x7ZKxFPd9bwrzygx/qiwIQiJ1sw/zD8qY/kRvlGHA==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@eslint/config-helpers@0.4.2': + resolution: {integrity: sha512-gBrxN88gOIf3R7ja5K9slwNayVcZgK6SOUORm2uBzTeIEfeVaIhOpCtTox3P6R7o2jLFwLFTLnC7kU/RGcYEgw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@eslint/core@0.17.0': + resolution: {integrity: sha512-yL/sLrpmtDaFEiUj1osRP4TI2MDz1AddJL+jZ7KSqvBuliN4xqYY54IfdN8qD8Toa6g1iloph1fxQNkjOxrrpQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@eslint/eslintrc@3.3.3': + resolution: {integrity: sha512-Kr+LPIUVKz2qkx1HAMH8q1q6azbqBAsXJUxBl/ODDuVPX45Z9DfwB8tPjTi6nNZ8BuM3nbJxC5zCAg5elnBUTQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@eslint/js@9.39.2': + resolution: {integrity: sha512-q1mjIoW1VX4IvSocvM/vbTiveKC4k9eLrajNEuSsmjymSDEbpGddtpfOoN7YGAqBK3NG+uqo8ia4PDTt8buCYA==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@eslint/object-schema@2.1.7': + resolution: {integrity: sha512-VtAOaymWVfZcmZbp6E2mympDIHvyjXs/12LqWYjVw6qjrfF+VK+fyG33kChz3nnK+SU5/NeHOqrTEHS8sXO3OA==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@eslint/plugin-kit@0.4.1': + resolution: {integrity: sha512-43/qtrDUokr7LJqoF2c3+RInu/t4zfrpYdoSDfYyhg52rwLV6TnOvdG4fXm7IkSB3wErkcmJS9iEhjVtOSEjjA==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@floating-ui/core@1.7.4': + resolution: {integrity: sha512-C3HlIdsBxszvm5McXlB8PeOEWfBhcGBTZGkGlWc2U0KFY5IwG5OQEuQ8rq52DZmcHDlPLd+YFBK+cZcytwIFWg==} + + '@floating-ui/dom@1.7.5': + resolution: {integrity: sha512-N0bD2kIPInNHUHehXhMke1rBGs1dwqvC9O9KYMyyjK7iXt7GAhnro7UlcuYcGdS/yYOlq0MAVgrow8IbWJwyqg==} + + '@floating-ui/react-dom@2.1.7': + resolution: {integrity: sha512-0tLRojf/1Go2JgEVm+3Frg9A3IW8bJgKgdO0BN5RkF//ufuz2joZM63Npau2ff3J6lUVYgDSNzNkR+aH3IVfjg==} + peerDependencies: + react: '>=16.8.0' + react-dom: '>=16.8.0' + + '@floating-ui/utils@0.2.10': + resolution: {integrity: sha512-aGTxbpbg8/b5JfU1HXSrbH3wXZuLPJcNEcZQFMxLs3oSzgtVu6nFPkbbGGUvBcUjKV2YyB9Wxxabo+HEH9tcRQ==} + + '@humanfs/core@0.19.1': + resolution: {integrity: sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==} + engines: {node: '>=18.18.0'} + + '@humanfs/node@0.16.7': + resolution: {integrity: sha512-/zUx+yOsIrG4Y43Eh2peDeKCxlRt/gET6aHfaKpuq267qXdYDFViVHfMaLyygZOnl0kGWxFIgsBy8QFuTLUXEQ==} + engines: {node: '>=18.18.0'} + + '@humanwhocodes/module-importer@1.0.1': + resolution: {integrity: sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==} + engines: {node: '>=12.22'} + + '@humanwhocodes/retry@0.4.3': + resolution: {integrity: sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==} + engines: {node: '>=18.18'} + + '@inquirer/external-editor@1.0.3': + resolution: {integrity: sha512-RWbSrDiYmO4LbejWY7ttpxczuwQyZLBUyygsA9Nsv95hpzUWwnNTVQmAq3xuh7vNwCp07UTmE5i11XAEExx4RA==} + engines: {node: '>=18'} + peerDependencies: + '@types/node': '>=18' + peerDependenciesMeta: + '@types/node': + optional: true + + '@ioredis/commands@1.5.0': + resolution: {integrity: sha512-eUgLqrMf8nJkZxT24JvVRrQya1vZkQh8BBeYNwGDqa5I0VUi8ACx7uFvAaLxintokpTenkK6DASvo/bvNbBGow==} + + '@isaacs/cliui@8.0.2': + resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} + engines: {node: '>=12'} + + '@isaacs/cliui@9.0.0': + resolution: {integrity: sha512-AokJm4tuBHillT+FpMtxQ60n8ObyXBatq7jD2/JA9dxbDDokKQm8KMht5ibGzLVU9IJDIKK4TPKgMHEYMn3lMg==} + engines: {node: '>=18'} + + '@jridgewell/gen-mapping@0.3.13': + resolution: {integrity: sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==} + + '@jridgewell/remapping@2.3.5': + resolution: {integrity: sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==} + + '@jridgewell/resolve-uri@3.1.2': + resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==} + engines: {node: '>=6.0.0'} + + '@jridgewell/sourcemap-codec@1.5.5': + resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==} + + '@jridgewell/trace-mapping@0.3.31': + resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + + '@jridgewell/trace-mapping@0.3.9': + resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==} + + '@monaco-editor/loader@1.7.0': + resolution: {integrity: sha512-gIwR1HrJrrx+vfyOhYmCZ0/JcWqG5kbfG7+d3f/C1LXk2EvzAbHSg3MQ5lO2sMlo9izoAZ04shohfKLVT6crVA==} + + '@monaco-editor/react@4.7.0': + resolution: {integrity: sha512-cyzXQCtO47ydzxpQtCGSQGOC8Gk3ZUeBXFAxD+CWXYFo5OqZyZUonFl0DwUlTyAfRHntBfw2p3w4s9R6oe1eCA==} + peerDependencies: + monaco-editor: '>= 0.25.0 < 1' + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + + '@motiadev/adapter-bullmq-events@0.17.14-beta.196': + resolution: {integrity: sha512-t9ZIRuMdbs2WiN656bXAic6KsFrT4M2vhelFEPw3RT72WHyqA2yn4PcsDtuKgX46oesEnaaCxGGty3/F0O6Cng==} + peerDependencies: + '@motiadev/core': '>=0.8.0' + + '@motiadev/adapter-redis-cron@0.17.14-beta.196': + resolution: {integrity: sha512-s+VAQ8K0/hmwWH7xSL+VoWLcJgCkDYBJd+iY5GzAjokThIzhPydrJNsSzmqfmxDNRe7cPsEmKHiF3rERUTY5MA==} + peerDependencies: + '@motiadev/core': '>=0.8.0' + + '@motiadev/adapter-redis-state@0.17.14-beta.196': + resolution: {integrity: sha512-KIceJaAiwnw7AJIvH6cQzctNuZK3rzI0qWL2gtPs3ZAB4TclUigkhIbKsyGyzWad4Vc0LF9t20lX9yrRaDVl4g==} + peerDependencies: + '@motiadev/core': '>=0.8.0' + + '@motiadev/adapter-redis-streams@0.17.14-beta.196': + resolution: {integrity: sha512-KON800F7+r+H79hIHYZAD8mqZ5WPU6HEZI8jLiuLUcZ7LCFaoA+3pbnep4SJWhDqK6AYFzgTW7/LAYd5qcaFcQ==} + peerDependencies: + '@motiadev/core': '>=0.8.0' + + '@motiadev/core@0.17.14-beta.196': + resolution: {integrity: sha512-/yFTlZWe/Sa11bWWV767RlH9dwnONx7nq8Q8bESTorzrCR2EQ6x8yt7LMrIebiTbDUOcGmS+rkw7aG37/5UXeQ==} + peerDependencies: + redis: ^5.0.0 + + '@motiadev/plugin-bullmq@0.17.14-beta.196': + resolution: {integrity: sha512-OFT4Qpiti2mvcO6WxSlK7bSoAU/6PUIhmksnT0fLnYLJuYsqSl1dYTCWxWCOJE3vQ8Ctj1TVxTFg6T7ah40oCw==} + peerDependencies: + '@motiadev/adapter-bullmq-events': 0.17.14-beta.196 + '@motiadev/core': 0.17.14-beta.196 + '@motiadev/stream-client-react': 0.17.14-beta.196 + '@motiadev/ui': 0.17.14-beta.196 + lucide-react: ^0.555.0 + motia: 0.17.14-beta.196 + react: ^19.2.3 + zustand: ^5.0.9 + + '@motiadev/plugin-cron@0.17.14-beta.196': + resolution: {integrity: sha512-RkgKDKAbeZ0GSh8qeR1InqPQpx85T+eSmdTDLgNrLUbyWN1PcMcXjjZUpmxhonaJkmUgopolrKIMjQ7Jn4Zl7w==} + peerDependencies: + '@motiadev/core': 0.17.14-beta.196 + '@motiadev/stream-client-react': 0.17.14-beta.196 + '@motiadev/ui': 0.17.14-beta.196 + + '@motiadev/plugin-endpoint@0.17.14-beta.196': + resolution: {integrity: sha512-0s6UrAAM2SwZ42Qb0SidxiybAQREB3Cz8SRe4cwA1JOFS6XwBNe+VDY9bLEGz3emZcZcKkxaZvhQXkFBUpYvDA==} + peerDependencies: + '@motiadev/core': 0.17.14-beta.196 + '@motiadev/stream-client-react': 0.17.14-beta.196 + '@motiadev/ui': 0.17.14-beta.196 + lucide-react: ^0.555.0 + motia: 0.17.14-beta.196 + react: ^19.2.3 + zustand: ^5.0.9 + + '@motiadev/plugin-logs@0.17.14-beta.196': + resolution: {integrity: sha512-jnKyzz4/EScXUoxBz89Zg38lWTl1wMRvhmjJD6DY1zwZC64KBGTo6d/bGCmax83VCAm8qZUo/SlblK6IAiUNjQ==} + peerDependencies: + '@motiadev/core': 0.17.14-beta.196 + '@motiadev/stream-client-react': 0.17.14-beta.196 + '@motiadev/ui': 0.17.14-beta.196 + lucide-react: ^0.555.0 + motia: 0.17.14-beta.196 + react: ^19.2.3 + zustand: ^5.0.9 + + '@motiadev/plugin-observability@0.17.14-beta.196': + resolution: {integrity: sha512-L1vjDB+ZOjOx2fkoBRbv+4gVSD8kAndtRfOHn+3Lv4kcq61VsbMcRWvhdkr0aVMs2x/nvE2lnM66T07/1hVH/w==} + peerDependencies: + '@motiadev/core': 0.17.14-beta.196 + '@motiadev/stream-client-react': 0.17.14-beta.196 + '@motiadev/ui': 0.17.14-beta.196 + lucide-react: ^0.555.0 + motia: 0.17.14-beta.196 + react: ^19.2.3 + zustand: ^5.0.9 + + '@motiadev/plugin-states@0.17.14-beta.196': + resolution: {integrity: sha512-Ql55ZP/vjuf/9svbNmr9SH3ADhXr/ntDKcP4IwrZq+BudlOpjPxsnRKZYStSICYUWYSijnmUTlrZxll1Bczqjg==} + peerDependencies: + '@motiadev/core': 0.17.14-beta.196 + '@motiadev/ui': 0.17.14-beta.196 + lucide-react: ^0.555.0 + motia: 0.17.14-beta.196 + react: ^19.2.3 + zustand: ^5.0.9 + + '@motiadev/plugin-streams@0.17.14-beta.196': + resolution: {integrity: sha512-pTboHvnDMxUJ3k+xUB3yPqILEhujNytw3zUvxg7+T6V1ppE4r3QyTuusbM2hlTzxI03Y/f7y5KmpRKNnWao6Ow==} + peerDependencies: + '@motiadev/core': 0.17.14-beta.196 + '@motiadev/ui': 0.17.14-beta.196 + lucide-react: ^0.555.0 + motia: 0.17.14-beta.196 + react: ^18.0.0 || ^19.0.0 + zustand: ^5.0.8 + + '@motiadev/plugin-ws@0.17.14-beta.196': + resolution: {integrity: sha512-804J5pBjJCio4S7GO85wO5CntzfH92cWc794yJgwmfx56vRSPCHwqUzNhQ+LAZI7lMsGYX0jRB4wlwzAjZXhww==} + peerDependencies: + '@motiadev/core': 0.17.14-beta.196 + '@motiadev/ui': 0.17.14-beta.196 + + '@motiadev/stream-client-browser@0.17.14-beta.196': + resolution: {integrity: sha512-X2isevw/Jg7KALDa7MjWBQqWYvgyWdIqHgL4X1k5bX+nUDgeEYXk20H0sPpTRA5z37uXwoS9jqpFIFgIzpLm9A==} + + '@motiadev/stream-client-node@0.17.14-beta.196': + resolution: {integrity: sha512-Bx00AB9i+3R+hP/RIOVrgH9A5z62cnt3tlOiva0vYXP3g4HkHM/aP9YUL1VdQMqlnovK9WPAoP37+ZNFkmhPlg==} + + '@motiadev/stream-client-react@0.17.14-beta.196': + resolution: {integrity: sha512-SJdAFRSgUE+vd9KyPYdW9isRIbya0xgV73eQ87Cbk+Hzfc3rW0jjH0YFHcFaArvBd0/sIus4j9+zp/oi3V8nxw==} + peerDependencies: + react: ^19.2.3 + + '@motiadev/stream-client@0.17.14-beta.196': + resolution: {integrity: sha512-+P/Dpl6WndLi+OgW09hRNX2fLLfReh7EY9UQvbv9AMvpU/TxN2SkuJ4EBiX0CnMFquALs1MTtbi1HrlcVvZpgA==} + + '@motiadev/ui@0.17.14-beta.196': + resolution: {integrity: sha512-C/9FD0gPtHcHRQIvsMrgKnQ3rVddD4l5Wxqu4iEPDZXh/LRSyBlHI8v3ghjD6obDXY9ujxOi4I9Z6vaIyB6hWQ==} + peerDependencies: + lucide-react: ^0.555.0 + react: ^19.2.3 + react-dom: ^19.2.3 + + '@motiadev/workbench@0.17.14-beta.196': + resolution: {integrity: sha512-gpvC4oUFyZZlpEhPkH5SSHTvqXBzTi1UwMS0lDIoSOEuNEJQQsV57ottRp7y+ziQLORpqf4IeMDsRoIn+hsz/A==} + + '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3': + resolution: {integrity: sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==} + cpu: [arm64] + os: [darwin] + + '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3': + resolution: {integrity: sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==} + cpu: [x64] + os: [darwin] + + '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.3': + resolution: {integrity: sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==} + cpu: [arm64] + os: [linux] + + '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.3': + resolution: {integrity: sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==} + cpu: [arm] + os: [linux] + + '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.3': + resolution: {integrity: sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==} + cpu: [x64] + os: [linux] + + '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.3': + resolution: {integrity: sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==} + cpu: [x64] + os: [win32] + + '@pkgjs/parseargs@0.11.0': + resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} + engines: {node: '>=14'} + + '@radix-ui/number@1.1.1': + resolution: {integrity: sha512-MkKCwxlXTgz6CFoJx3pCwn07GKp36+aZyu/u2Ln2VrA5DcdyCZkASEDBTd8x5whTQQL5CiYf4prXKLcgQdv29g==} + + '@radix-ui/primitive@1.1.3': + resolution: {integrity: sha512-JTF99U/6XIjCBo0wqkU5sK10glYe27MRRsfwoiq5zzOEZLHU3A3KCMa5X/azekYRCJ0HlwI0crAXS/5dEHTzDg==} + + '@radix-ui/react-arrow@1.1.7': + resolution: {integrity: sha512-F+M1tLhO+mlQaOWspE8Wstg+z6PwxwRd8oQ8IXceWz92kfAmalTRf0EjrouQeo7QssEPfCn05B4Ihs1K9WQ/7w==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-checkbox@1.3.3': + resolution: {integrity: sha512-wBbpv+NQftHDdG86Qc0pIyXk5IR3tM8Vd0nWLKDcX8nNn4nXFOFwsKuqw2okA/1D/mpaAkmuyndrPJTYDNZtFw==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-collection@1.1.7': + resolution: {integrity: sha512-Fh9rGN0MoI4ZFUNyfFVNU4y9LUz93u9/0K+yLgA2bwRojxM8JU1DyvvMBabnZPBgMWREAJvU2jjVzq+LrFUglw==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-compose-refs@1.1.2': + resolution: {integrity: sha512-z4eqJvfiNnFMHIIvXP3CY57y2WJs5g2v3X0zm9mEJkrkNv4rDxu+sg9Jh8EkXyeqBkB7SOcboo9dMVqhyrACIg==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-context@1.1.2': + resolution: {integrity: sha512-jCi/QKUM2r1Ju5a3J64TH2A5SpKAgh0LpknyqdQ4m6DCV0xJ2HG1xARRwNGPQfi1SLdLWZ1OJz6F4OMBBNiGJA==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-direction@1.1.1': + resolution: {integrity: sha512-1UEWRX6jnOA2y4H5WczZ44gOOjTEmlqv1uNW4GAJEO5+bauCBhv8snY65Iw5/VOS/ghKN9gr2KjnLKxrsvoMVw==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-dismissable-layer@1.1.11': + resolution: {integrity: sha512-Nqcp+t5cTB8BinFkZgXiMJniQH0PsUt2k51FUhbdfeKvc4ACcG2uQniY/8+h1Yv6Kza4Q7lD7PQV0z0oicE0Mg==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-dropdown-menu@2.1.16': + resolution: {integrity: sha512-1PLGQEynI/3OX/ftV54COn+3Sud/Mn8vALg2rWnBLnRaGtJDduNW/22XjlGgPdpcIbiQxjKtb7BkcjP00nqfJw==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-focus-guards@1.1.3': + resolution: {integrity: sha512-0rFg/Rj2Q62NCm62jZw0QX7a3sz6QCQU0LpZdNrJX8byRGaGVTqbrW9jAoIAHyMQqsNpeZ81YgSizOt5WXq0Pw==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-focus-scope@1.1.7': + resolution: {integrity: sha512-t2ODlkXBQyn7jkl6TNaw/MtVEVvIGelJDCG41Okq/KwUsJBwQ4XVZsHAVUkK4mBv3ewiAS3PGuUWuY2BoK4ZUw==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-id@1.1.1': + resolution: {integrity: sha512-kGkGegYIdQsOb4XjsfM97rXsiHaBwco+hFI66oO4s9LU+PLAC5oJ7khdOVFxkhsmlbpUqDAvXw11CluXP+jkHg==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-label@2.1.8': + resolution: {integrity: sha512-FmXs37I6hSBVDlO4y764TNz1rLgKwjJMQ0EGte6F3Cb3f4bIuHB/iLa/8I9VKkmOy+gNHq8rql3j686ACVV21A==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-menu@2.1.16': + resolution: {integrity: sha512-72F2T+PLlphrqLcAotYPp0uJMr5SjP5SL01wfEspJbru5Zs5vQaSHb4VB3ZMJPimgHHCHG7gMOeOB9H3Hdmtxg==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-popover@1.1.15': + resolution: {integrity: sha512-kr0X2+6Yy/vJzLYJUPCZEc8SfQcf+1COFoAqauJm74umQhta9M7lNJHP7QQS3vkvcGLQUbWpMzwrXYwrYztHKA==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-popper@1.2.8': + resolution: {integrity: sha512-0NJQ4LFFUuWkE7Oxf0htBKS6zLkkjBH+hM1uk7Ng705ReR8m/uelduy1DBo0PyBXPKVnBA6YBlU94MBGXrSBCw==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-portal@1.1.9': + resolution: {integrity: sha512-bpIxvq03if6UNwXZ+HTK71JLh4APvnXntDc6XOX8UVq4XQOVl7lwok0AvIl+b8zgCw3fSaVTZMpAPPagXbKmHQ==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-presence@1.1.5': + resolution: {integrity: sha512-/jfEwNDdQVBCNvjkGit4h6pMOzq8bHkopq458dPt2lMjx+eBQUohZNG9A7DtO/O5ukSbxuaNGXMjHicgwy6rQQ==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-primitive@2.1.3': + resolution: {integrity: sha512-m9gTwRkhy2lvCPe6QJp4d3G1TYEUHn/FzJUtq9MjH46an1wJU+GdoGC5VLof8RX8Ft/DlpshApkhswDLZzHIcQ==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-primitive@2.1.4': + resolution: {integrity: sha512-9hQc4+GNVtJAIEPEqlYqW5RiYdrr8ea5XQ0ZOnD6fgru+83kqT15mq2OCcbe8KnjRZl5vF3ks69AKz3kh1jrhg==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-roving-focus@1.1.11': + resolution: {integrity: sha512-7A6S9jSgm/S+7MdtNDSb+IU859vQqJ/QAtcYQcfFC6W8RS4IxIZDldLR0xqCFZ6DCyrQLjLPsxtTNch5jVA4lA==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-select@2.2.6': + resolution: {integrity: sha512-I30RydO+bnn2PQztvo25tswPH+wFBjehVGtmagkU78yMdwTwVf12wnAOF+AeP8S2N8xD+5UPbGhkUfPyvT+mwQ==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-separator@1.1.8': + resolution: {integrity: sha512-sDvqVY4itsKwwSMEe0jtKgfTh+72Sy3gPmQpjqcQneqQ4PFmr/1I0YA+2/puilhggCe2gJcx5EBAYFkWkdpa5g==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-slot@1.2.3': + resolution: {integrity: sha512-aeNmHnBxbi2St0au6VBVC7JXFlhLlOnvIIlePNniyUNAClzmtAUEY8/pBiK3iHjufOlwA+c20/8jngo7xcrg8A==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-slot@1.2.4': + resolution: {integrity: sha512-Jl+bCv8HxKnlTLVrcDE8zTMJ09R9/ukw4qBs/oZClOfoQk/cOTbDn+NceXfV7j09YPVQUryJPHurafcSg6EVKA==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-tabs@1.1.13': + resolution: {integrity: sha512-7xdcatg7/U+7+Udyoj2zodtI9H/IIopqo+YOIcZOq1nJwXWBZ9p8xiu5llXlekDbZkca79a/fozEYQXIA4sW6A==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-tooltip@1.2.8': + resolution: {integrity: sha512-tY7sVt1yL9ozIxvmbtN5qtmH2krXcBCfjEiCgKGLqunJHvgvZG2Pcl2oQ3kbcZARb1BGEHdkLzcYGO8ynVlieg==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-use-callback-ref@1.1.1': + resolution: {integrity: sha512-FkBMwD+qbGQeMu1cOHnuGB6x4yzPjho8ap5WtbEJ26umhgqVXbhekKUQO+hZEL1vU92a3wHwdp0HAcqAUF5iDg==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-use-controllable-state@1.2.2': + resolution: {integrity: sha512-BjasUjixPFdS+NKkypcyyN5Pmg83Olst0+c6vGov0diwTEo6mgdqVR6hxcEgFuh4QrAs7Rc+9KuGJ9TVCj0Zzg==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-use-effect-event@0.0.2': + resolution: {integrity: sha512-Qp8WbZOBe+blgpuUT+lw2xheLP8q0oatc9UpmiemEICxGvFLYmHm9QowVZGHtJlGbS6A6yJ3iViad/2cVjnOiA==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-use-escape-keydown@1.1.1': + resolution: {integrity: sha512-Il0+boE7w/XebUHyBjroE+DbByORGR9KKmITzbR7MyQ4akpORYP/ZmbhAr0DG7RmmBqoOnZdy2QlvajJ2QA59g==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-use-layout-effect@1.1.1': + resolution: {integrity: sha512-RbJRS4UWQFkzHTTwVymMTUv8EqYhOp8dOOviLj2ugtTiXRaRQS7GLGxZTLL1jWhMeoSCf5zmcZkqTl9IiYfXcQ==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-use-previous@1.1.1': + resolution: {integrity: sha512-2dHfToCj/pzca2Ck724OZ5L0EVrr3eHRNsG/b3xQJLA2hZpVCS99bLAX+hm1IHXDEnzU6by5z/5MIY794/a8NQ==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-use-rect@1.1.1': + resolution: {integrity: sha512-QTYuDesS0VtuHNNvMh+CjlKJ4LJickCMUAqjlE3+j8w+RlRpwyX3apEQKGFzbZGdo7XNG1tXa+bQqIE7HIXT2w==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-use-size@1.1.1': + resolution: {integrity: sha512-ewrXRDTAqAXlkl6t/fkXWNAhFX9I+CkKlw6zjEwk86RSPKwZr3xpBRso655aqYafwtnbpHLj6toFzmd6xdVptQ==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-visually-hidden@1.2.3': + resolution: {integrity: sha512-pzJq12tEaaIhqjbzpCuv/OypJY/BPavOofm+dbab+MHLajy277+1lLm6JFcGgF5eskJ6mquGirhXY2GD/8u8Ug==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/rect@1.1.1': + resolution: {integrity: sha512-HPwpGIzkl28mWyZqG52jiqDJ12waP11Pa1lGoiyUkIEuMLBP0oeK/C89esbXrxsky5we7dfd8U58nm0SgAWpVw==} + + '@redis/bloom@5.10.0': + resolution: {integrity: sha512-doIF37ob+l47n0rkpRNgU8n4iacBlKM9xLiP1LtTZTvz8TloJB8qx/MgvhMhKdYG+CvCY2aPBnN2706izFn/4A==} + engines: {node: '>= 18'} + peerDependencies: + '@redis/client': ^5.10.0 + + '@redis/client@5.10.0': + resolution: {integrity: sha512-JXmM4XCoso6C75Mr3lhKA3eNxSzkYi3nCzxDIKY+YOszYsJjuKbFgVtguVPbLMOttN4iu2fXoc2BGhdnYhIOxA==} + engines: {node: '>= 18'} + + '@redis/json@5.10.0': + resolution: {integrity: sha512-B2G8XlOmTPUuZtD44EMGbtoepQG34RCDXLZbjrtON1Djet0t5Ri7/YPXvL9aomXqP8lLTreaprtyLKF4tmXEEA==} + engines: {node: '>= 18'} + peerDependencies: + '@redis/client': ^5.10.0 + + '@redis/search@5.10.0': + resolution: {integrity: sha512-3SVcPswoSfp2HnmWbAGUzlbUPn7fOohVu2weUQ0S+EMiQi8jwjL+aN2p6V3TI65eNfVsJ8vyPvqWklm6H6esmg==} + engines: {node: '>= 18'} + peerDependencies: + '@redis/client': ^5.10.0 + + '@redis/time-series@5.10.0': + resolution: {integrity: sha512-cPkpddXH5kc/SdRhF0YG0qtjL+noqFT0AcHbQ6axhsPsO7iqPi1cjxgdkE9TNeKiBUUdCaU1DbqkR/LzbzPBhg==} + engines: {node: '>= 18'} + peerDependencies: + '@redis/client': ^5.10.0 + + '@rolldown/pluginutils@1.0.0-rc.3': + resolution: {integrity: sha512-eybk3TjzzzV97Dlj5c+XrBFW57eTNhzod66y9HrBlzJ6NsCrWCp/2kaPS3K9wJmurBC0Tdw4yPjXKZqlznim3Q==} + + '@rollup/rollup-android-arm-eabi@4.57.1': + resolution: {integrity: sha512-A6ehUVSiSaaliTxai040ZpZ2zTevHYbvu/lDoeAteHI8QnaosIzm4qwtezfRg1jOYaUmnzLX1AOD6Z+UJjtifg==} + cpu: [arm] + os: [android] + + '@rollup/rollup-android-arm64@4.57.1': + resolution: {integrity: sha512-dQaAddCY9YgkFHZcFNS/606Exo8vcLHwArFZ7vxXq4rigo2bb494/xKMMwRRQW6ug7Js6yXmBZhSBRuBvCCQ3w==} + cpu: [arm64] + os: [android] + + '@rollup/rollup-darwin-arm64@4.57.1': + resolution: {integrity: sha512-crNPrwJOrRxagUYeMn/DZwqN88SDmwaJ8Cvi/TN1HnWBU7GwknckyosC2gd0IqYRsHDEnXf328o9/HC6OkPgOg==} + cpu: [arm64] + os: [darwin] + + '@rollup/rollup-darwin-x64@4.57.1': + resolution: {integrity: sha512-Ji8g8ChVbKrhFtig5QBV7iMaJrGtpHelkB3lsaKzadFBe58gmjfGXAOfI5FV0lYMH8wiqsxKQ1C9B0YTRXVy4w==} + cpu: [x64] + os: [darwin] + + '@rollup/rollup-freebsd-arm64@4.57.1': + resolution: {integrity: sha512-R+/WwhsjmwodAcz65guCGFRkMb4gKWTcIeLy60JJQbXrJ97BOXHxnkPFrP+YwFlaS0m+uWJTstrUA9o+UchFug==} + cpu: [arm64] + os: [freebsd] + + '@rollup/rollup-freebsd-x64@4.57.1': + resolution: {integrity: sha512-IEQTCHeiTOnAUC3IDQdzRAGj3jOAYNr9kBguI7MQAAZK3caezRrg0GxAb6Hchg4lxdZEI5Oq3iov/w/hnFWY9Q==} + cpu: [x64] + os: [freebsd] + + '@rollup/rollup-linux-arm-gnueabihf@4.57.1': + resolution: {integrity: sha512-F8sWbhZ7tyuEfsmOxwc2giKDQzN3+kuBLPwwZGyVkLlKGdV1nvnNwYD0fKQ8+XS6hp9nY7B+ZeK01EBUE7aHaw==} + cpu: [arm] + os: [linux] + + '@rollup/rollup-linux-arm-musleabihf@4.57.1': + resolution: {integrity: sha512-rGfNUfn0GIeXtBP1wL5MnzSj98+PZe/AXaGBCRmT0ts80lU5CATYGxXukeTX39XBKsxzFpEeK+Mrp9faXOlmrw==} + cpu: [arm] + os: [linux] + + '@rollup/rollup-linux-arm64-gnu@4.57.1': + resolution: {integrity: sha512-MMtej3YHWeg/0klK2Qodf3yrNzz6CGjo2UntLvk2RSPlhzgLvYEB3frRvbEF2wRKh1Z2fDIg9KRPe1fawv7C+g==} + cpu: [arm64] + os: [linux] + + '@rollup/rollup-linux-arm64-musl@4.57.1': + resolution: {integrity: sha512-1a/qhaaOXhqXGpMFMET9VqwZakkljWHLmZOX48R0I/YLbhdxr1m4gtG1Hq7++VhVUmf+L3sTAf9op4JlhQ5u1Q==} + cpu: [arm64] + os: [linux] + + '@rollup/rollup-linux-loong64-gnu@4.57.1': + resolution: {integrity: sha512-QWO6RQTZ/cqYtJMtxhkRkidoNGXc7ERPbZN7dVW5SdURuLeVU7lwKMpo18XdcmpWYd0qsP1bwKPf7DNSUinhvA==} + cpu: [loong64] + os: [linux] + + '@rollup/rollup-linux-loong64-musl@4.57.1': + resolution: {integrity: sha512-xpObYIf+8gprgWaPP32xiN5RVTi/s5FCR+XMXSKmhfoJjrpRAjCuuqQXyxUa/eJTdAE6eJ+KDKaoEqjZQxh3Gw==} + cpu: [loong64] + os: [linux] + + '@rollup/rollup-linux-ppc64-gnu@4.57.1': + resolution: {integrity: sha512-4BrCgrpZo4hvzMDKRqEaW1zeecScDCR+2nZ86ATLhAoJ5FQ+lbHVD3ttKe74/c7tNT9c6F2viwB3ufwp01Oh2w==} + cpu: [ppc64] + os: [linux] + + '@rollup/rollup-linux-ppc64-musl@4.57.1': + resolution: {integrity: sha512-NOlUuzesGauESAyEYFSe3QTUguL+lvrN1HtwEEsU2rOwdUDeTMJdO5dUYl/2hKf9jWydJrO9OL/XSSf65R5+Xw==} + cpu: [ppc64] + os: [linux] + + '@rollup/rollup-linux-riscv64-gnu@4.57.1': + resolution: {integrity: sha512-ptA88htVp0AwUUqhVghwDIKlvJMD/fmL/wrQj99PRHFRAG6Z5nbWoWG4o81Nt9FT+IuqUQi+L31ZKAFeJ5Is+A==} + cpu: [riscv64] + os: [linux] + + '@rollup/rollup-linux-riscv64-musl@4.57.1': + resolution: {integrity: sha512-S51t7aMMTNdmAMPpBg7OOsTdn4tySRQvklmL3RpDRyknk87+Sp3xaumlatU+ppQ+5raY7sSTcC2beGgvhENfuw==} + cpu: [riscv64] + os: [linux] + + '@rollup/rollup-linux-s390x-gnu@4.57.1': + resolution: {integrity: sha512-Bl00OFnVFkL82FHbEqy3k5CUCKH6OEJL54KCyx2oqsmZnFTR8IoNqBF+mjQVcRCT5sB6yOvK8A37LNm/kPJiZg==} + cpu: [s390x] + os: [linux] + + '@rollup/rollup-linux-x64-gnu@4.57.1': + resolution: {integrity: sha512-ABca4ceT4N+Tv/GtotnWAeXZUZuM/9AQyCyKYyKnpk4yoA7QIAuBt6Hkgpw8kActYlew2mvckXkvx0FfoInnLg==} + cpu: [x64] + os: [linux] + + '@rollup/rollup-linux-x64-musl@4.57.1': + resolution: {integrity: sha512-HFps0JeGtuOR2convgRRkHCekD7j+gdAuXM+/i6kGzQtFhlCtQkpwtNzkNj6QhCDp7DRJ7+qC/1Vg2jt5iSOFw==} + cpu: [x64] + os: [linux] + + '@rollup/rollup-openbsd-x64@4.57.1': + resolution: {integrity: sha512-H+hXEv9gdVQuDTgnqD+SQffoWoc0Of59AStSzTEj/feWTBAnSfSD3+Dql1ZruJQxmykT/JVY0dE8Ka7z0DH1hw==} + cpu: [x64] + os: [openbsd] + + '@rollup/rollup-openharmony-arm64@4.57.1': + resolution: {integrity: sha512-4wYoDpNg6o/oPximyc/NG+mYUejZrCU2q+2w6YZqrAs2UcNUChIZXjtafAiiZSUc7On8v5NyNj34Kzj/Ltk6dQ==} + cpu: [arm64] + os: [openharmony] + + '@rollup/rollup-win32-arm64-msvc@4.57.1': + resolution: {integrity: sha512-O54mtsV/6LW3P8qdTcamQmuC990HDfR71lo44oZMZlXU4tzLrbvTii87Ni9opq60ds0YzuAlEr/GNwuNluZyMQ==} + cpu: [arm64] + os: [win32] + + '@rollup/rollup-win32-ia32-msvc@4.57.1': + resolution: {integrity: sha512-P3dLS+IerxCT/7D2q2FYcRdWRl22dNbrbBEtxdWhXrfIMPP9lQhb5h4Du04mdl5Woq05jVCDPCMF7Ub0NAjIew==} + cpu: [ia32] + os: [win32] + + '@rollup/rollup-win32-x64-gnu@4.57.1': + resolution: {integrity: sha512-VMBH2eOOaKGtIJYleXsi2B8CPVADrh+TyNxJ4mWPnKfLB/DBUmzW+5m1xUrcwWoMfSLagIRpjUFeW5CO5hyciQ==} + cpu: [x64] + os: [win32] + + '@rollup/rollup-win32-x64-msvc@4.57.1': + resolution: {integrity: sha512-mxRFDdHIWRxg3UfIIAwCm6NzvxG0jDX/wBN6KsQFTvKFqqg9vTrWUE68qEjHt19A5wwx5X5aUi2zuZT7YR0jrA==} + cpu: [x64] + os: [win32] + + '@standard-schema/spec@1.1.0': + resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} + + '@tailwindcss/node@4.1.18': + resolution: {integrity: sha512-DoR7U1P7iYhw16qJ49fgXUlry1t4CpXeErJHnQ44JgTSKMaZUdf17cfn5mHchfJ4KRBZRFA/Coo+MUF5+gOaCQ==} + + '@tailwindcss/oxide-android-arm64@4.1.18': + resolution: {integrity: sha512-dJHz7+Ugr9U/diKJA0W6N/6/cjI+ZTAoxPf9Iz9BFRF2GzEX8IvXxFIi/dZBloVJX/MZGvRuFA9rqwdiIEZQ0Q==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [android] + + '@tailwindcss/oxide-darwin-arm64@4.1.18': + resolution: {integrity: sha512-Gc2q4Qhs660bhjyBSKgq6BYvwDz4G+BuyJ5H1xfhmDR3D8HnHCmT/BSkvSL0vQLy/nkMLY20PQ2OoYMO15Jd0A==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + + '@tailwindcss/oxide-darwin-x64@4.1.18': + resolution: {integrity: sha512-FL5oxr2xQsFrc3X9o1fjHKBYBMD1QZNyc1Xzw/h5Qu4XnEBi3dZn96HcHm41c/euGV+GRiXFfh2hUCyKi/e+yw==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + + '@tailwindcss/oxide-freebsd-x64@4.1.18': + resolution: {integrity: sha512-Fj+RHgu5bDodmV1dM9yAxlfJwkkWvLiRjbhuO2LEtwtlYlBgiAT4x/j5wQr1tC3SANAgD+0YcmWVrj8R9trVMA==} + engines: {node: '>= 10'} + cpu: [x64] + os: [freebsd] + + '@tailwindcss/oxide-linux-arm-gnueabihf@4.1.18': + resolution: {integrity: sha512-Fp+Wzk/Ws4dZn+LV2Nqx3IilnhH51YZoRaYHQsVq3RQvEl+71VGKFpkfHrLM/Li+kt5c0DJe/bHXK1eHgDmdiA==} + engines: {node: '>= 10'} + cpu: [arm] + os: [linux] + + '@tailwindcss/oxide-linux-arm64-gnu@4.1.18': + resolution: {integrity: sha512-S0n3jboLysNbh55Vrt7pk9wgpyTTPD0fdQeh7wQfMqLPM/Hrxi+dVsLsPrycQjGKEQk85Kgbx+6+QnYNiHalnw==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + + '@tailwindcss/oxide-linux-arm64-musl@4.1.18': + resolution: {integrity: sha512-1px92582HkPQlaaCkdRcio71p8bc8i/ap5807tPRDK/uw953cauQBT8c5tVGkOwrHMfc2Yh6UuxaH4vtTjGvHg==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + + '@tailwindcss/oxide-linux-x64-gnu@4.1.18': + resolution: {integrity: sha512-v3gyT0ivkfBLoZGF9LyHmts0Isc8jHZyVcbzio6Wpzifg/+5ZJpDiRiUhDLkcr7f/r38SWNe7ucxmGW3j3Kb/g==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + + '@tailwindcss/oxide-linux-x64-musl@4.1.18': + resolution: {integrity: sha512-bhJ2y2OQNlcRwwgOAGMY0xTFStt4/wyU6pvI6LSuZpRgKQwxTec0/3Scu91O8ir7qCR3AuepQKLU/kX99FouqQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + + '@tailwindcss/oxide-wasm32-wasi@4.1.18': + resolution: {integrity: sha512-LffYTvPjODiP6PT16oNeUQJzNVyJl1cjIebq/rWWBF+3eDst5JGEFSc5cWxyRCJ0Mxl+KyIkqRxk1XPEs9x8TA==} + engines: {node: '>=14.0.0'} + cpu: [wasm32] + bundledDependencies: + - '@napi-rs/wasm-runtime' + - '@emnapi/core' + - '@emnapi/runtime' + - '@tybys/wasm-util' + - '@emnapi/wasi-threads' + - tslib + + '@tailwindcss/oxide-win32-arm64-msvc@4.1.18': + resolution: {integrity: sha512-HjSA7mr9HmC8fu6bdsZvZ+dhjyGCLdotjVOgLA2vEqxEBZaQo9YTX4kwgEvPCpRh8o4uWc4J/wEoFzhEmjvPbA==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [win32] + + '@tailwindcss/oxide-win32-x64-msvc@4.1.18': + resolution: {integrity: sha512-bJWbyYpUlqamC8dpR7pfjA0I7vdF6t5VpUGMWRkXVE3AXgIZjYUYAK7II1GNaxR8J1SSrSrppRar8G++JekE3Q==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + + '@tailwindcss/oxide@4.1.18': + resolution: {integrity: sha512-EgCR5tTS5bUSKQgzeMClT6iCY3ToqE1y+ZB0AKldj809QXk1Y+3jB0upOYZrn9aGIzPtUsP7sX4QQ4XtjBB95A==} + engines: {node: '>= 10'} + + '@tailwindcss/postcss@4.1.18': + resolution: {integrity: sha512-Ce0GFnzAOuPyfV5SxjXGn0CubwGcuDB0zcdaPuCSzAa/2vII24JTkH+I6jcbXLb1ctjZMZZI6OjDaLPJQL1S0g==} + + '@tanstack/query-core@5.90.20': + resolution: {integrity: sha512-OMD2HLpNouXEfZJWcKeVKUgQ5n+n3A2JFmBaScpNDUqSrQSjiveC7dKMe53uJUg1nDG16ttFPz2xfilz6i2uVg==} + + '@tanstack/react-query@5.90.21': + resolution: {integrity: sha512-0Lu6y5t+tvlTJMTO7oh5NSpJfpg/5D41LlThfepTixPYkJ0sE2Jj0m0f6yYqujBwIXlId87e234+MxG3D3g7kg==} + peerDependencies: + react: ^18 || ^19 + + '@tanstack/react-virtual@3.13.18': + resolution: {integrity: sha512-dZkhyfahpvlaV0rIKnvQiVoWPyURppl6w4m9IwMDpuIjcJ1sD9YGWrt0wISvgU7ewACXx2Ct46WPgI6qAD4v6A==} + peerDependencies: + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + + '@tanstack/virtual-core@3.13.18': + resolution: {integrity: sha512-Mx86Hqu1k39icq2Zusq+Ey2J6dDWTjDvEv43PJtRCoEYTLyfaPnxIQ6iy7YAOK0NV/qOEmZQ/uCufrppZxTgcg==} + + '@tsconfig/node10@1.0.12': + resolution: {integrity: sha512-UCYBaeFvM11aU2y3YPZ//O5Rhj+xKyzy7mvcIoAjASbigy8mHMryP5cK7dgjlz2hWxh1g5pLw084E0a/wlUSFQ==} + + '@tsconfig/node12@1.0.11': + resolution: {integrity: sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==} + + '@tsconfig/node14@1.0.3': + resolution: {integrity: sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==} + + '@tsconfig/node16@1.0.4': + resolution: {integrity: sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==} + + '@types/babel__core@7.20.5': + resolution: {integrity: sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==} + + '@types/babel__generator@7.27.0': + resolution: {integrity: sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg==} + + '@types/babel__template@7.4.4': + resolution: {integrity: sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==} + + '@types/babel__traverse@7.28.0': + resolution: {integrity: sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q==} + + '@types/d3-color@3.1.3': + resolution: {integrity: sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A==} + + '@types/d3-drag@3.0.7': + resolution: {integrity: sha512-HE3jVKlzU9AaMazNufooRJ5ZpWmLIoc90A37WU2JMmeq28w1FQqCZswHZ3xR+SuxYftzHq6WU6KJHvqxKzTxxQ==} + + '@types/d3-interpolate@3.0.4': + resolution: {integrity: sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA==} + + '@types/d3-selection@3.0.11': + resolution: {integrity: sha512-bhAXu23DJWsrI45xafYpkQ4NtcKMwWnAC/vKrd2l+nxMFuvOT3XMYTIj2opv8vq8AO5Yh7Qac/nSeP/3zjTK0w==} + + '@types/d3-transition@3.0.9': + resolution: {integrity: sha512-uZS5shfxzO3rGlu0cC3bjmMFKsXv+SmZZcgp0KD22ts4uGXp5EVYGzu/0YdwZeKmddhcAccYtREJKkPfXkZuCg==} + + '@types/d3-zoom@3.0.8': + resolution: {integrity: sha512-iqMC4/YlFCSlO8+2Ii1GGGliCAY4XdeG748w5vQUbevlbDu0zSjH/+jojorQVBK/se0j6DUFNPBGSqD3YWYnDw==} + + '@types/estree@1.0.8': + resolution: {integrity: sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==} + + '@types/hast@3.0.4': + resolution: {integrity: sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==} + + '@types/json-schema@7.0.15': + resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==} + + '@types/luxon@3.7.1': + resolution: {integrity: sha512-H3iskjFIAn5SlJU7OuxUmTEpebK6TKB8rxZShDslBMZJ5u9S//KM1sbdAisiSrqwLQncVjnpi2OK2J51h+4lsg==} + + '@types/node@25.2.3': + resolution: {integrity: sha512-m0jEgYlYz+mDJZ2+F4v8D1AyQb+QzsNqRuI7xg1VQX/KlKS0qT9r1Mo16yo5F/MtifXFgaofIFsdFMox2SxIbQ==} + + '@types/prismjs@1.26.6': + resolution: {integrity: sha512-vqlvI7qlMvcCBbVe0AKAb4f97//Hy0EBTaiW8AalRnG/xAN5zOiWWyrNqNXeq8+KAuvRewjCVY1+IPxk4RdNYw==} + + '@types/trusted-types@2.0.7': + resolution: {integrity: sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==} + + '@types/unist@2.0.11': + resolution: {integrity: sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==} + + '@types/yauzl@2.10.3': + resolution: {integrity: sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==} + + '@types/zen-observable@0.8.3': + resolution: {integrity: sha512-fbF6oTd4sGGy0xjHPKAt+eS2CrxJ3+6gQ3FGcBoIJR2TLAyCkCyI8JqZNy+FeON0AhVgNJoUumVoZQjBFUqHkw==} + + '@typescript-eslint/eslint-plugin@8.55.0': + resolution: {integrity: sha512-1y/MVSz0NglV1ijHC8OT49mPJ4qhPYjiK08YUQVbIOyu+5k862LKUHFkpKHWu//zmr7hDR2rhwUm6gnCGNmGBQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + '@typescript-eslint/parser': ^8.55.0 + eslint: ^8.57.0 || ^9.0.0 + typescript: '>=4.8.4 <6.0.0' + + '@typescript-eslint/parser@8.55.0': + resolution: {integrity: sha512-4z2nCSBfVIMnbuu8uinj+f0o4qOeggYJLbjpPHka3KH1om7e+H9yLKTYgksTaHcGco+NClhhY2vyO3HsMH1RGw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + eslint: ^8.57.0 || ^9.0.0 + typescript: '>=4.8.4 <6.0.0' + + '@typescript-eslint/project-service@8.55.0': + resolution: {integrity: sha512-zRcVVPFUYWa3kNnjaZGXSu3xkKV1zXy8M4nO/pElzQhFweb7PPtluDLQtKArEOGmjXoRjnUZ29NjOiF0eCDkcQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + typescript: '>=4.8.4 <6.0.0' + + '@typescript-eslint/scope-manager@8.55.0': + resolution: {integrity: sha512-fVu5Omrd3jeqeQLiB9f1YsuK/iHFOwb04bCtY4BSCLgjNbOD33ZdV6KyEqplHr+IlpgT0QTZ/iJ+wT7hvTx49Q==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@typescript-eslint/tsconfig-utils@8.55.0': + resolution: {integrity: sha512-1R9cXqY7RQd7WuqSN47PK9EDpgFUK3VqdmbYrvWJZYDd0cavROGn+74ktWBlmJ13NXUQKlZ/iAEQHI/V0kKe0Q==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + typescript: '>=4.8.4 <6.0.0' + + '@typescript-eslint/type-utils@8.55.0': + resolution: {integrity: sha512-x1iH2unH4qAt6I37I2CGlsNs+B9WGxurP2uyZLRz6UJoZWDBx9cJL1xVN/FiOmHEONEg6RIufdvyT0TEYIgC5g==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + eslint: ^8.57.0 || ^9.0.0 + typescript: '>=4.8.4 <6.0.0' + + '@typescript-eslint/types@8.55.0': + resolution: {integrity: sha512-ujT0Je8GI5BJWi+/mMoR0wxwVEQaxM+pi30xuMiJETlX80OPovb2p9E8ss87gnSVtYXtJoU9U1Cowcr6w2FE0w==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@typescript-eslint/typescript-estree@8.55.0': + resolution: {integrity: sha512-EwrH67bSWdx/3aRQhCoxDaHM+CrZjotc2UCCpEDVqfCE+7OjKAGWNY2HsCSTEVvWH2clYQK8pdeLp42EVs+xQw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + typescript: '>=4.8.4 <6.0.0' + + '@typescript-eslint/utils@8.55.0': + resolution: {integrity: sha512-BqZEsnPGdYpgyEIkDC1BadNY8oMwckftxBT+C8W0g1iKPdeqKZBtTfnvcq0nf60u7MkjFO8RBvpRGZBPw4L2ow==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + eslint: ^8.57.0 || ^9.0.0 + typescript: '>=4.8.4 <6.0.0' + + '@typescript-eslint/visitor-keys@8.55.0': + resolution: {integrity: sha512-AxNRwEie8Nn4eFS1FzDMJWIISMGoXMb037sgCBJ3UR6o0fQTzr2tqN9WT+DkWJPhIdQCfV7T6D387566VtnCJA==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@vitejs/plugin-react@5.1.4': + resolution: {integrity: sha512-VIcFLdRi/VYRU8OL/puL7QXMYafHmqOnwTZY50U1JPlCNj30PxCMx65c494b1K9be9hX83KVt0+gTEwTWLqToA==} + engines: {node: ^20.19.0 || >=22.12.0} + peerDependencies: + vite: ^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 + + '@xyflow/react@12.10.0': + resolution: {integrity: sha512-eOtz3whDMWrB4KWVatIBrKuxECHqip6PfA8fTpaS2RUGVpiEAe+nqDKsLqkViVWxDGreq0lWX71Xth/SPAzXiw==} + peerDependencies: + react: '>=17' + react-dom: '>=17' + + '@xyflow/system@0.0.74': + resolution: {integrity: sha512-7v7B/PkiVrkdZzSbL+inGAo6tkR/WQHHG0/jhSvLQToCsfa8YubOGmBYd1s08tpKpihdHDZFwzQZeR69QSBb4Q==} + + abort-controller@3.0.0: + resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} + engines: {node: '>=6.5'} + + accepts@1.3.8: + resolution: {integrity: sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==} + engines: {node: '>= 0.6'} + + acorn-jsx@5.3.2: + resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==} + peerDependencies: + acorn: ^6.0.0 || ^7.0.0 || ^8.0.0 + + acorn-walk@8.3.4: + resolution: {integrity: sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==} + engines: {node: '>=0.4.0'} + + acorn@8.15.0: + resolution: {integrity: sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==} + engines: {node: '>=0.4.0'} + hasBin: true + + agent-base@7.1.4: + resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} + engines: {node: '>= 14'} + + ajv@6.12.6: + resolution: {integrity: sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==} + + ajv@8.17.1: + resolution: {integrity: sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==} + + ansi-escapes@4.3.2: + resolution: {integrity: sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==} + engines: {node: '>=8'} + + ansi-regex@5.0.1: + resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} + engines: {node: '>=8'} + + ansi-regex@6.2.2: + resolution: {integrity: sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==} + engines: {node: '>=12'} + + ansi-styles@4.3.0: + resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} + engines: {node: '>=8'} + + ansi-styles@6.2.3: + resolution: {integrity: sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==} + engines: {node: '>=12'} + + antlr4ts@0.5.0-alpha.4: + resolution: {integrity: sha512-WPQDt1B74OfPv/IMS2ekXAKkTZIHl88uMetg6q3OTqgFxZ/dxDXI0EWLyZid/1Pe6hTftyg5N7gel5wNAGxXyQ==} + + archiver-utils@5.0.2: + resolution: {integrity: sha512-wuLJMmIBQYCsGZgYLTy5FIB2pF6Lfb6cXMSF8Qywwk3t20zWnAi7zLcQFdKQmIB8wyZpY5ER38x08GbwtR2cLA==} + engines: {node: '>= 14'} + + archiver@7.0.1: + resolution: {integrity: sha512-ZcbTaIqJOfCc03QwD468Unz/5Ir8ATtvAHsK+FdXbDIbGfihqh9mrvdcYunQzqn4HrvWWaFyaxJhGZagaJJpPQ==} + engines: {node: '>= 14'} + + arg@4.1.3: + resolution: {integrity: sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==} + + argparse@2.0.1: + resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} + + aria-hidden@1.2.6: + resolution: {integrity: sha512-ik3ZgC9dY/lYVVM++OISsaYDeg1tb0VtP5uL3ouh1koGOaUMDPpbFIei4JkFimWUFPn90sbMNMXQAIVOlnYKJA==} + engines: {node: '>=10'} + + array-flatten@1.1.1: + resolution: {integrity: sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==} + + astral-regex@2.0.0: + resolution: {integrity: sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==} + engines: {node: '>=8'} + + async@3.2.6: + resolution: {integrity: sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==} + + asynckit@0.4.0: + resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + + autoprefixer@10.4.24: + resolution: {integrity: sha512-uHZg7N9ULTVbutaIsDRoUkoS8/h3bdsmVJYZ5l3wv8Cp/6UIIoRDm90hZ+BwxUj/hGBEzLxdHNSKuFpn8WOyZw==} + engines: {node: ^10 || ^12 || >=14} + hasBin: true + peerDependencies: + postcss: ^8.1.0 + + axios@1.13.5: + resolution: {integrity: sha512-cz4ur7Vb0xS4/KUN0tPWe44eqxrIu31me+fbang3ijiNscE129POzipJJA6zniq2C/Z6sJCjMimjS8Lc/GAs8Q==} + + b4a@1.7.4: + resolution: {integrity: sha512-u20zJLDaSWpxaZ+zaAkEIB2dZZ1o+DF4T/MRbmsvGp9nletHOyiai19OzX1fF8xUBYsO1bPXxODvcd0978pnug==} + peerDependencies: + react-native-b4a: '*' + peerDependenciesMeta: + react-native-b4a: + optional: true + + babel-plugin-react-compiler@1.0.0: + resolution: {integrity: sha512-Ixm8tFfoKKIPYdCCKYTsqv+Fd4IJ0DQqMyEimo+pxUOMUR9cVPlwTrFt9Avu+3cb6Zp3mAzl+t1MrG2fxxKsxw==} + + balanced-match@1.0.2: + resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} + + balanced-match@4.0.2: + resolution: {integrity: sha512-x0K50QvKQ97fdEz2kPehIerj+YTeptKF9hyYkKf6egnwmMWAkADiO0QCzSp0R5xN8FTZgYaBfSaue46Ej62nMg==} + engines: {node: 20 || >=22} + + bare-events@2.8.2: + resolution: {integrity: sha512-riJjyv1/mHLIPX4RwiK+oW9/4c3TEUeORHKefKAKnZ5kyslbN+HXowtbaVEqt4IMUB7OXlfixcs6gsFeo/jhiQ==} + peerDependencies: + bare-abort-controller: '*' + peerDependenciesMeta: + bare-abort-controller: + optional: true + + base64-js@1.5.1: + resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} + + baseline-browser-mapping@2.9.19: + resolution: {integrity: sha512-ipDqC8FrAl/76p2SSWKSI+H9tFwm7vYqXQrItCuiVPt26Km0jS+NzSsBWAaBusvSbQcfJG+JitdMm+wZAgTYqg==} + hasBin: true + + bl@4.1.0: + resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} + + body-parser@1.20.4: + resolution: {integrity: sha512-ZTgYYLMOXY9qKU/57FAo8F+HA2dGX7bqGc71txDRC1rS4frdFI5R7NhluHxH6M0YItAP0sHB4uqAOcYKxO6uGA==} + engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} + + brace-expansion@1.1.12: + resolution: {integrity: sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==} + + brace-expansion@2.0.2: + resolution: {integrity: sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==} + + brace-expansion@5.0.2: + resolution: {integrity: sha512-Pdk8c9poy+YhOgVWw1JNN22/HcivgKWwpxKq04M/jTmHyCZn12WPJebZxdjSa5TmBqISrUSgNYU3eRORljfCCw==} + engines: {node: 20 || >=22} + + browserslist@4.28.1: + resolution: {integrity: sha512-ZC5Bd0LgJXgwGqUknZY/vkUQ04r8NXnJZ3yYi4vDmSiZmC/pdSN0NbNRPxZpbtO4uAfDUAFffO8IZoM3Gj8IkA==} + engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} + hasBin: true + + buffer-crc32@0.2.13: + resolution: {integrity: sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==} + + buffer-crc32@1.0.0: + resolution: {integrity: sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==} + engines: {node: '>=8.0.0'} + + buffer@5.7.1: + resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==} + + buffer@6.0.3: + resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==} + + bullmq@5.69.1: + resolution: {integrity: sha512-ct9cablOCuIHq4giGK32opyhftzeSDzdoQMLi/90xlBhtJmk0XoOO4FpU64egBD8KnWSCM/u4cjy3NA1J5SYYg==} + + bytes@3.1.2: + resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} + engines: {node: '>= 0.8'} + + call-bind-apply-helpers@1.0.2: + resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} + engines: {node: '>= 0.4'} + + call-bound@1.0.4: + resolution: {integrity: sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==} + engines: {node: '>= 0.4'} + + callsites@3.1.0: + resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==} + engines: {node: '>=6'} + + camelcase@6.3.0: + resolution: {integrity: sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==} + engines: {node: '>=10'} + + caniuse-lite@1.0.30001769: + resolution: {integrity: sha512-BCfFL1sHijQlBGWBMuJyhZUhzo7wer5sVj9hqekB/7xn0Ypy+pER/edCYQm4exbXj4WiySGp40P8UuTh6w1srg==} + + chalk@4.1.2: + resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} + engines: {node: '>=10'} + + character-entities-legacy@3.0.0: + resolution: {integrity: sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==} + + character-entities@2.0.2: + resolution: {integrity: sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==} + + character-reference-invalid@2.0.1: + resolution: {integrity: sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw==} + + chardet@2.1.1: + resolution: {integrity: sha512-PsezH1rqdV9VvyNhxxOW32/d75r01NY7TQCmOqomRo15ZSOKbpTFVsfjghxo6JloQUCGnH4k1LGu0R4yCLlWQQ==} + + chokidar@4.0.3: + resolution: {integrity: sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==} + engines: {node: '>= 14.16.0'} + + chownr@2.0.0: + resolution: {integrity: sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==} + engines: {node: '>=10'} + + class-variance-authority@0.7.1: + resolution: {integrity: sha512-Ka+9Trutv7G8M6WT6SeiRWz792K5qEqIGEGzXKhAE6xOWAY6pPH8U+9IY3oCMv6kqTmLsv7Xh/2w2RigkePMsg==} + + classcat@5.0.5: + resolution: {integrity: sha512-JhZUT7JFcQy/EzW605k/ktHtncoo9vnyW/2GspNYwFlN1C/WmjuV/xtS04e9SOkL2sTdw0VAZ2UGCcQ9lR6p6w==} + + cli-cursor@3.1.0: + resolution: {integrity: sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==} + engines: {node: '>=8'} + + cli-spinners@2.9.2: + resolution: {integrity: sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==} + engines: {node: '>=6'} + + cli-width@3.0.0: + resolution: {integrity: sha512-FxqpkPPwu1HjuN93Omfm4h8uIanXofW0RxVEW3k5RKx+mJJYSthzNhp32Kzxxy3YAEZ/Dc/EWN1vZRY0+kOhbw==} + engines: {node: '>= 10'} + + clone@1.0.4: + resolution: {integrity: sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==} + engines: {node: '>=0.8'} + + clsx@2.1.1: + resolution: {integrity: sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==} + engines: {node: '>=6'} + + cluster-key-slot@1.1.2: + resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} + engines: {node: '>=0.10.0'} + + color-convert@2.0.1: + resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} + engines: {node: '>=7.0.0'} + + color-name@1.1.4: + resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + + combined-stream@1.0.8: + resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} + engines: {node: '>= 0.8'} + + comma-separated-tokens@2.0.3: + resolution: {integrity: sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==} + + commander@13.1.0: + resolution: {integrity: sha512-/rFeCpNJQbhSZjGVwO9RFV3xPqbnERS8MmIQzCtD/zl6gpJuV/bMLuN92oG3F7d8oDEHHRrujSXNUr8fpjntKw==} + engines: {node: '>=18'} + + commondir@1.0.1: + resolution: {integrity: sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==} + + compress-commons@6.0.2: + resolution: {integrity: sha512-6FqVXeETqWPoGcfzrXb37E50NP0LXT8kAMu5ooZayhWWdgEY4lBEEcbQNXtkuKQsGduxiIcI4gOTsxTmuq/bSg==} + engines: {node: '>= 14'} + + concat-map@0.0.1: + resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} + + content-disposition@0.5.4: + resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==} + engines: {node: '>= 0.6'} + + content-type@1.0.5: + resolution: {integrity: sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==} + engines: {node: '>= 0.6'} + + convert-source-map@2.0.0: + resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} + + cookie-signature@1.0.7: + resolution: {integrity: sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==} + + cookie@0.7.2: + resolution: {integrity: sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==} + engines: {node: '>= 0.6'} + + cookie@1.1.1: + resolution: {integrity: sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==} + engines: {node: '>=18'} + + copy-to-clipboard@3.3.3: + resolution: {integrity: sha512-2KV8NhB5JqC3ky0r9PMCAZKbUHSwtEo4CwCs0KXgruG43gX5PMqDEBbVU4OUzw2MuAWUfsuFmWvEKG5QRfSnJA==} + + core-util-is@1.0.3: + resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==} + + crc-32@1.2.2: + resolution: {integrity: sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==} + engines: {node: '>=0.8'} + hasBin: true + + crc32-stream@6.0.0: + resolution: {integrity: sha512-piICUB6ei4IlTv1+653yq5+KoqfBYmj9bw6LqXoOneTMDXk5nM1qt12mFW1caG3LlJXEKW1Bp0WggEmIfQB34g==} + engines: {node: '>= 14'} + + create-require@1.1.1: + resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==} + + cron-parser@4.9.0: + resolution: {integrity: sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==} + engines: {node: '>=12.0.0'} + + cron@4.4.0: + resolution: {integrity: sha512-fkdfq+b+AHI4cKdhZlppHveI/mgz2qpiYxcm+t5E5TsxX7QrLS1VE0+7GENEk9z0EeGPcpSciGv6ez24duWhwQ==} + engines: {node: '>=18.x'} + + cross-spawn@7.0.6: + resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} + engines: {node: '>= 8'} + + d3-color@3.1.0: + resolution: {integrity: sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==} + engines: {node: '>=12'} + + d3-dispatch@3.0.1: + resolution: {integrity: sha512-rzUyPU/S7rwUflMyLc1ETDeBj0NRuHKKAcvukozwhshr6g6c5d8zh4c2gQjY2bZ0dXeGLWc1PF174P2tVvKhfg==} + engines: {node: '>=12'} + + d3-drag@3.0.0: + resolution: {integrity: sha512-pWbUJLdETVA8lQNJecMxoXfH6x+mO2UQo8rSmZ+QqxcbyA3hfeprFgIT//HW2nlHChWeIIMwS2Fq+gEARkhTkg==} + engines: {node: '>=12'} + + d3-ease@3.0.1: + resolution: {integrity: sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==} + engines: {node: '>=12'} + + d3-interpolate@3.0.1: + resolution: {integrity: sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==} + engines: {node: '>=12'} + + d3-selection@3.0.0: + resolution: {integrity: sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==} + engines: {node: '>=12'} + + d3-timer@3.0.1: + resolution: {integrity: sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==} + engines: {node: '>=12'} + + d3-transition@3.0.1: + resolution: {integrity: sha512-ApKvfjsSR6tg06xrL434C0WydLr7JewBB3V+/39RMHsaXTOG0zmt/OAXeng5M5LBm0ojmxJrpomQVZ1aPvBL4w==} + engines: {node: '>=12'} + peerDependencies: + d3-selection: 2 - 3 + + d3-zoom@3.0.0: + resolution: {integrity: sha512-b8AmV3kfQaqWAuacbPuNbL6vahnOJflOhexLzMMNLga62+/nh0JzvJ0aO/5a5MVgUFGS7Hu1P9P03o3fJkDCyw==} + engines: {node: '>=12'} + + dagre@0.8.5: + resolution: {integrity: sha512-/aTqmnRta7x7MCCpExk7HQL2O4owCT2h8NT//9I1OQ9vt29Pa0BzSAkR5lwFUcQ7491yVi/3CXU9jQ5o0Mn2Sw==} + + date-fns@4.1.0: + resolution: {integrity: sha512-Ukq0owbQXxa/U3EGtsdVBkR1w7KOQ5gIBqdH2hkvknzZPYvBxb/aa6E8L7tmjFtkwZBu3UXBbjIgPo/Ez4xaNg==} + + debug@2.6.9: + resolution: {integrity: sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + + debug@4.4.3: + resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + + decode-named-character-reference@1.3.0: + resolution: {integrity: sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q==} + + deep-is@0.1.4: + resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==} + + defaults@1.0.4: + resolution: {integrity: sha512-eFuaLoy/Rxalv2kr+lqMlUnrDWV+3j4pljOIJgLIhI058IQfWJ7vXhyEIHu+HtC738klGALYxOKDO0bQP3tg8A==} + + delayed-stream@1.0.0: + resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} + engines: {node: '>=0.4.0'} + + denque@2.1.0: + resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==} + engines: {node: '>=0.10'} + + depd@2.0.0: + resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} + engines: {node: '>= 0.8'} + + destroy@1.2.0: + resolution: {integrity: sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==} + engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} + + detect-libc@2.1.2: + resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} + engines: {node: '>=8'} + + detect-node-es@1.1.0: + resolution: {integrity: sha512-ypdmJU/TbBby2Dxibuv7ZLW3Bs1QEmM7nHjEANfohJLvE0XVujisn1qPJcZxg+qDucsr+bP6fLD1rPS3AhJ7EQ==} + + diff@4.0.4: + resolution: {integrity: sha512-X07nttJQkwkfKfvTPG/KSnE2OMdcUCao6+eXF3wmnIQRn2aPAHH3VxDbDOdegkd6JbPsXqShpvEOHfAT+nCNwQ==} + engines: {node: '>=0.3.1'} + + dompurify@3.2.7: + resolution: {integrity: sha512-WhL/YuveyGXJaerVlMYGWhvQswa7myDG17P7Vu65EWC05o8vfeNbvNf4d/BOvH99+ZW+LlQsc1GDKMa1vNK6dw==} + + dotenv@16.6.1: + resolution: {integrity: sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==} + engines: {node: '>=12'} + + dunder-proto@1.0.1: + resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==} + engines: {node: '>= 0.4'} + + eastasianwidth@0.2.0: + resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} + + ee-first@1.1.1: + resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} + + electron-to-chromium@1.5.286: + resolution: {integrity: sha512-9tfDXhJ4RKFNerfjdCcZfufu49vg620741MNs26a9+bhLThdB+plgMeou98CAaHu/WATj2iHOOHTp1hWtABj2A==} + + emoji-regex@8.0.0: + resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} + + emoji-regex@9.2.2: + resolution: {integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==} + + encodeurl@2.0.0: + resolution: {integrity: sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==} + engines: {node: '>= 0.8'} + + end-of-stream@1.4.5: + resolution: {integrity: sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==} + + enhanced-resolve@5.19.0: + resolution: {integrity: sha512-phv3E1Xl4tQOShqSte26C7Fl84EwUdZsyOuSSk9qtAGyyQs2s3jJzComh+Abf4g187lUUAvH+H26omrqia2aGg==} + engines: {node: '>=10.13.0'} + + es-define-property@1.0.1: + resolution: {integrity: sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==} + engines: {node: '>= 0.4'} + + es-errors@1.3.0: + resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==} + engines: {node: '>= 0.4'} + + es-object-atoms@1.1.1: + resolution: {integrity: sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==} + engines: {node: '>= 0.4'} + + es-set-tostringtag@2.1.0: + resolution: {integrity: sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==} + engines: {node: '>= 0.4'} + + esbuild@0.25.12: + resolution: {integrity: sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==} + engines: {node: '>=18'} + hasBin: true + + esbuild@0.27.3: + resolution: {integrity: sha512-8VwMnyGCONIs6cWue2IdpHxHnAjzxnw2Zr7MkVxB2vjmQ2ivqGFb4LEG3SMnv0Gb2F/G/2yA8zUaiL1gywDCCg==} + engines: {node: '>=18'} + hasBin: true + + escalade@3.2.0: + resolution: {integrity: sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==} + engines: {node: '>=6'} + + escape-html@1.0.3: + resolution: {integrity: sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==} + + escape-string-regexp@1.0.5: + resolution: {integrity: sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==} + engines: {node: '>=0.8.0'} + + escape-string-regexp@4.0.0: + resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} + engines: {node: '>=10'} + + eslint-scope@8.4.0: + resolution: {integrity: sha512-sNXOfKCn74rt8RICKMvJS7XKV/Xk9kA7DyJr8mJik3S7Cwgy3qlkkmyS2uQB3jiJg6VNdZd/pDBJu0nvG2NlTg==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + eslint-visitor-keys@3.4.3: + resolution: {integrity: sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + + eslint-visitor-keys@4.2.1: + resolution: {integrity: sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + eslint@9.39.2: + resolution: {integrity: sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + hasBin: true + peerDependencies: + jiti: '*' + peerDependenciesMeta: + jiti: + optional: true + + espree@10.4.0: + resolution: {integrity: sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + esquery@1.7.0: + resolution: {integrity: sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==} + engines: {node: '>=0.10'} + + esrecurse@4.3.0: + resolution: {integrity: sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==} + engines: {node: '>=4.0'} + + estraverse@5.3.0: + resolution: {integrity: sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==} + engines: {node: '>=4.0'} + + esutils@2.0.3: + resolution: {integrity: sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==} + engines: {node: '>=0.10.0'} + + etag@1.8.1: + resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==} + engines: {node: '>= 0.6'} + + event-target-shim@5.0.1: + resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} + engines: {node: '>=6'} + + events-universal@1.0.1: + resolution: {integrity: sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==} + + events@3.3.0: + resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} + engines: {node: '>=0.8.x'} + + express@4.22.1: + resolution: {integrity: sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==} + engines: {node: '>= 0.10.0'} + + extract-zip@2.0.1: + resolution: {integrity: sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==} + engines: {node: '>= 10.17.0'} + hasBin: true + + fast-deep-equal@3.1.3: + resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} + + fast-fifo@1.3.2: + resolution: {integrity: sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==} + + fast-json-stable-stringify@2.1.0: + resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==} + + fast-levenshtein@2.0.6: + resolution: {integrity: sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==} + + fast-uri@3.1.0: + resolution: {integrity: sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==} + + fault@1.0.4: + resolution: {integrity: sha512-CJ0HCB5tL5fYTEA7ToAq5+kTwd++Borf1/bifxd9iT70QcXr4MRrO3Llf8Ifs70q+SJcGHFtnIE/Nw6giCtECA==} + + fd-slicer@1.1.0: + resolution: {integrity: sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==} + + fdir@6.5.0: + resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} + engines: {node: '>=12.0.0'} + peerDependencies: + picomatch: ^3 || ^4 + peerDependenciesMeta: + picomatch: + optional: true + + figures@3.2.0: + resolution: {integrity: sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==} + engines: {node: '>=8'} + + file-entry-cache@8.0.0: + resolution: {integrity: sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==} + engines: {node: '>=16.0.0'} + + finalhandler@1.3.2: + resolution: {integrity: sha512-aA4RyPcd3badbdABGDuTXCMTtOneUCAYH/gxoYRTZlIJdF0YPWuGqiAsIrhNnnqdXGswYk6dGujem4w80UJFhg==} + engines: {node: '>= 0.8'} + + find-cache-dir@3.3.2: + resolution: {integrity: sha512-wXZV5emFEjrridIgED11OoUKLxiYjAcqot/NJdAkOhlJ+vGzwhOAfcG5OX1jP+S0PcjEn8bdMJv+g2jwQ3Onig==} + engines: {node: '>=8'} + + find-package-json@1.2.0: + resolution: {integrity: sha512-+SOGcLGYDJHtyqHd87ysBhmaeQ95oWspDKnMXBrnQ9Eq4OkLNqejgoaD8xVWu6GPa0B6roa6KinCMEMcVeqONw==} + + find-up@4.1.0: + resolution: {integrity: sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==} + engines: {node: '>=8'} + + find-up@5.0.0: + resolution: {integrity: sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==} + engines: {node: '>=10'} + + flat-cache@4.0.1: + resolution: {integrity: sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==} + engines: {node: '>=16'} + + flatted@3.3.3: + resolution: {integrity: sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==} + + follow-redirects@1.15.11: + resolution: {integrity: sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==} + engines: {node: '>=4.0'} + peerDependencies: + debug: '*' + peerDependenciesMeta: + debug: + optional: true + + foreground-child@3.3.1: + resolution: {integrity: sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==} + engines: {node: '>=14'} + + form-data@4.0.5: + resolution: {integrity: sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==} + engines: {node: '>= 6'} + + format@0.2.2: + resolution: {integrity: sha512-wzsgA6WOq+09wrU1tsJ09udeR/YZRaeArL9e1wPbFg3GG2yDnC2ldKpxs4xunpFF9DgqCqOIra3bc1HWrJ37Ww==} + engines: {node: '>=0.4.x'} + + forwarded@0.2.0: + resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} + engines: {node: '>= 0.6'} + + fraction.js@5.3.4: + resolution: {integrity: sha512-1X1NTtiJphryn/uLQz3whtY6jK3fTqoE3ohKs0tT+Ujr1W59oopxmoEh7Lu5p6vBaPbgoM0bzveAW4Qi5RyWDQ==} + + framer-motion@12.34.0: + resolution: {integrity: sha512-+/H49owhzkzQyxtn7nZeF4kdH++I2FWrESQ184Zbcw5cEqNHYkE5yxWxcTLSj5lNx3NWdbIRy5FHqUvetD8FWg==} + peerDependencies: + '@emotion/is-prop-valid': '*' + react: ^18.0.0 || ^19.0.0 + react-dom: ^18.0.0 || ^19.0.0 + peerDependenciesMeta: + '@emotion/is-prop-valid': + optional: true + react: + optional: true + react-dom: + optional: true + + fresh@0.5.2: + resolution: {integrity: sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==} + engines: {node: '>= 0.6'} + + fs-minipass@2.1.0: + resolution: {integrity: sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==} + engines: {node: '>= 8'} + + fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + + function-bind@1.1.2: + resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + + gensync@1.0.0-beta.2: + resolution: {integrity: sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==} + engines: {node: '>=6.9.0'} + + get-intrinsic@1.3.0: + resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==} + engines: {node: '>= 0.4'} + + get-nonce@1.0.1: + resolution: {integrity: sha512-FJhYRoDaiatfEkUK8HKlicmu/3SGFD51q3itKDGoSTysQJBnfOcxU5GxnhE1E6soB76MbT0MBtnKJuXyAx+96Q==} + engines: {node: '>=6'} + + get-port@5.1.1: + resolution: {integrity: sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==} + engines: {node: '>=8'} + + get-proto@1.0.1: + resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} + engines: {node: '>= 0.4'} + + get-stream@5.2.0: + resolution: {integrity: sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==} + engines: {node: '>=8'} + + get-tsconfig@4.13.6: + resolution: {integrity: sha512-shZT/QMiSHc/YBLxxOkMtgSid5HFoauqCE3/exfsEcwg1WkeqjG+V40yBbBrsD+jW2HDXcs28xOfcbm2jI8Ddw==} + + glob-parent@6.0.2: + resolution: {integrity: sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==} + engines: {node: '>=10.13.0'} + + glob@10.5.0: + resolution: {integrity: sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==} + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me + hasBin: true + + glob@11.1.0: + resolution: {integrity: sha512-vuNwKSaKiqm7g0THUBu2x7ckSs3XJLXE+2ssL7/MfTGPLLcrJQ/4Uq1CjPTtO5cCIiRxqvN6Twy1qOwhL0Xjcw==} + engines: {node: 20 || >=22} + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me + hasBin: true + + globals@14.0.0: + resolution: {integrity: sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==} + engines: {node: '>=18'} + + gopd@1.2.0: + resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} + engines: {node: '>= 0.4'} + + graceful-fs@4.2.11: + resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} + + graphlib@2.1.8: + resolution: {integrity: sha512-jcLLfkpoVGmH7/InMC/1hIvOPSUh38oJtGhvrOFGzioE1DZ+0YW16RgmOJhHiuWTvGiJQ9Z1Ik43JvkRPRvE+A==} + + has-flag@4.0.0: + resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} + engines: {node: '>=8'} + + has-symbols@1.1.0: + resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==} + engines: {node: '>= 0.4'} + + has-tostringtag@1.0.2: + resolution: {integrity: sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==} + engines: {node: '>= 0.4'} + + hasown@2.0.2: + resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} + engines: {node: '>= 0.4'} + + hast-util-parse-selector@4.0.0: + resolution: {integrity: sha512-wkQCkSYoOGCRKERFWcxMVMOcYE2K1AaNLU8DXS9arxnLOUEWbOXKXiJUNzEpqZ3JOKpnha3jkFrumEjVliDe7A==} + + hastscript@9.0.1: + resolution: {integrity: sha512-g7df9rMFX/SPi34tyGCyUBREQoKkapwdY/T04Qn9TDWfHhAYt4/I0gMVirzK5wEzeUqIjEB+LXC/ypb7Aqno5w==} + + highlight.js@10.7.3: + resolution: {integrity: sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A==} + + highlightjs-vue@1.0.0: + resolution: {integrity: sha512-PDEfEF102G23vHmPhLyPboFCD+BkMGu+GuJe2d9/eH4FsCwvgBpnc9n0pGE+ffKdph38s6foEZiEjdgHdzp+IA==} + + http-errors@2.0.1: + resolution: {integrity: sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==} + engines: {node: '>= 0.8'} + + https-proxy-agent@7.0.6: + resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} + engines: {node: '>= 14'} + + iconv-lite@0.4.24: + resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==} + engines: {node: '>=0.10.0'} + + iconv-lite@0.7.2: + resolution: {integrity: sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==} + engines: {node: '>=0.10.0'} + + ieee754@1.2.1: + resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} + + ignore@5.3.2: + resolution: {integrity: sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==} + engines: {node: '>= 4'} + + ignore@7.0.5: + resolution: {integrity: sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==} + engines: {node: '>= 4'} + + import-fresh@3.3.1: + resolution: {integrity: sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==} + engines: {node: '>=6'} + + imurmurhash@0.1.4: + resolution: {integrity: sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==} + engines: {node: '>=0.8.19'} + + inherits@2.0.4: + resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} + + inquirer@8.2.7: + resolution: {integrity: sha512-UjOaSel/iddGZJ5xP/Eixh6dY1XghiBw4XK13rCCIJcJfyhhoul/7KhLLUGtebEj6GDYM6Vnx/mVsjx2L/mFIA==} + engines: {node: '>=12.0.0'} + + ioredis@5.9.2: + resolution: {integrity: sha512-tAAg/72/VxOUW7RQSX1pIxJVucYKcjFjfvj60L57jrZpYCHC3XN0WCQ3sNYL4Gmvv+7GPvTAjc+KSdeNuE8oWQ==} + engines: {node: '>=12.22.0'} + + ioredis@5.9.3: + resolution: {integrity: sha512-VI5tMCdeoxZWU5vjHWsiE/Su76JGhBvWF1MJnV9ZtGltHk9BmD48oDq8Tj8haZ85aceXZMxLNDQZRVo5QKNgXA==} + engines: {node: '>=12.22.0'} + + ipaddr.js@1.9.1: + resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==} + engines: {node: '>= 0.10'} + + is-alphabetical@2.0.1: + resolution: {integrity: sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ==} + + is-alphanumerical@2.0.1: + resolution: {integrity: sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw==} + + is-decimal@2.0.1: + resolution: {integrity: sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A==} + + is-extglob@2.1.1: + resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==} + engines: {node: '>=0.10.0'} + + is-fullwidth-code-point@3.0.0: + resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==} + engines: {node: '>=8'} + + is-glob@4.0.3: + resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==} + engines: {node: '>=0.10.0'} + + is-hexadecimal@2.0.1: + resolution: {integrity: sha512-DgZQp241c8oO6cA1SbTEWiXeoxV42vlcJxgH+B3hi1AiqqKruZR3ZGF8In3fj4+/y/7rHvlOZLZtgJ/4ttYGZg==} + + is-interactive@1.0.0: + resolution: {integrity: sha512-2HvIEKRoqS62guEC+qBjpvRubdX910WCMuJTZ+I9yvqKU2/12eSL549HMwtabb4oupdj2sMP50k+XJfB/8JE6w==} + engines: {node: '>=8'} + + is-stream@2.0.1: + resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} + engines: {node: '>=8'} + + is-unicode-supported@0.1.0: + resolution: {integrity: sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==} + engines: {node: '>=10'} + + isarray@1.0.0: + resolution: {integrity: sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==} + + isexe@2.0.0: + resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} + + jackspeak@3.4.3: + resolution: {integrity: sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==} + + jackspeak@4.2.3: + resolution: {integrity: sha512-ykkVRwrYvFm1nb2AJfKKYPr0emF6IiXDYUaFx4Zn9ZuIH7MrzEZ3sD5RlqGXNRpHtvUHJyOnCEFxOlNDtGo7wg==} + engines: {node: 20 || >=22} + + jiti@2.6.1: + resolution: {integrity: sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ==} + hasBin: true + + js-tokens@4.0.0: + resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} + + js-yaml@4.1.1: + resolution: {integrity: sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==} + hasBin: true + + jsesc@3.1.0: + resolution: {integrity: sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==} + engines: {node: '>=6'} + hasBin: true + + json-buffer@3.0.1: + resolution: {integrity: sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==} + + json-schema-traverse@0.4.1: + resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==} + + json-schema-traverse@1.0.0: + resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} + + json-schema@0.4.0: + resolution: {integrity: sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==} + + json-stable-stringify-without-jsonify@1.0.1: + resolution: {integrity: sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==} + + json5@2.2.3: + resolution: {integrity: sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==} + engines: {node: '>=6'} + hasBin: true + + keyv@4.5.4: + resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} + + lazystream@1.0.1: + resolution: {integrity: sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==} + engines: {node: '>= 0.6.3'} + + levn@0.4.1: + resolution: {integrity: sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==} + engines: {node: '>= 0.8.0'} + + lightningcss-android-arm64@1.30.2: + resolution: {integrity: sha512-BH9sEdOCahSgmkVhBLeU7Hc9DWeZ1Eb6wNS6Da8igvUwAe0sqROHddIlvU06q3WyXVEOYDZ6ykBZQnjTbmo4+A==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [android] + + lightningcss-darwin-arm64@1.30.2: + resolution: {integrity: sha512-ylTcDJBN3Hp21TdhRT5zBOIi73P6/W0qwvlFEk22fkdXchtNTOU4Qc37SkzV+EKYxLouZ6M4LG9NfZ1qkhhBWA==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [darwin] + + lightningcss-darwin-x64@1.30.2: + resolution: {integrity: sha512-oBZgKchomuDYxr7ilwLcyms6BCyLn0z8J0+ZZmfpjwg9fRVZIR5/GMXd7r9RH94iDhld3UmSjBM6nXWM2TfZTQ==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [darwin] + + lightningcss-freebsd-x64@1.30.2: + resolution: {integrity: sha512-c2bH6xTrf4BDpK8MoGG4Bd6zAMZDAXS569UxCAGcA7IKbHNMlhGQ89eRmvpIUGfKWNVdbhSbkQaWhEoMGmGslA==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [freebsd] + + lightningcss-linux-arm-gnueabihf@1.30.2: + resolution: {integrity: sha512-eVdpxh4wYcm0PofJIZVuYuLiqBIakQ9uFZmipf6LF/HRj5Bgm0eb3qL/mr1smyXIS1twwOxNWndd8z0E374hiA==} + engines: {node: '>= 12.0.0'} + cpu: [arm] + os: [linux] + + lightningcss-linux-arm64-gnu@1.30.2: + resolution: {integrity: sha512-UK65WJAbwIJbiBFXpxrbTNArtfuznvxAJw4Q2ZGlU8kPeDIWEX1dg3rn2veBVUylA2Ezg89ktszWbaQnxD/e3A==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [linux] + + lightningcss-linux-arm64-musl@1.30.2: + resolution: {integrity: sha512-5Vh9dGeblpTxWHpOx8iauV02popZDsCYMPIgiuw97OJ5uaDsL86cnqSFs5LZkG3ghHoX5isLgWzMs+eD1YzrnA==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [linux] + + lightningcss-linux-x64-gnu@1.30.2: + resolution: {integrity: sha512-Cfd46gdmj1vQ+lR6VRTTadNHu6ALuw2pKR9lYq4FnhvgBc4zWY1EtZcAc6EffShbb1MFrIPfLDXD6Xprbnni4w==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [linux] + + lightningcss-linux-x64-musl@1.30.2: + resolution: {integrity: sha512-XJaLUUFXb6/QG2lGIW6aIk6jKdtjtcffUT0NKvIqhSBY3hh9Ch+1LCeH80dR9q9LBjG3ewbDjnumefsLsP6aiA==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [linux] + + lightningcss-win32-arm64-msvc@1.30.2: + resolution: {integrity: sha512-FZn+vaj7zLv//D/192WFFVA0RgHawIcHqLX9xuWiQt7P0PtdFEVaxgF9rjM/IRYHQXNnk61/H/gb2Ei+kUQ4xQ==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [win32] + + lightningcss-win32-x64-msvc@1.30.2: + resolution: {integrity: sha512-5g1yc73p+iAkid5phb4oVFMB45417DkRevRbt/El/gKXJk4jid+vPFF/AXbxn05Aky8PapwzZrdJShv5C0avjw==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [win32] + + lightningcss@1.30.2: + resolution: {integrity: sha512-utfs7Pr5uJyyvDETitgsaqSyjCb2qNRAtuqUeWIAKztsOYdcACf2KtARYXg2pSvhkt+9NfoaNY7fxjl6nuMjIQ==} + engines: {node: '>= 12.0.0'} + + locate-path@5.0.0: + resolution: {integrity: sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==} + engines: {node: '>=8'} + + locate-path@6.0.0: + resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} + engines: {node: '>=10'} + + lockfile@1.0.4: + resolution: {integrity: sha512-cvbTwETRfsFh4nHsL1eGWapU1XFi5Ot9E85sWAwia7Y7EgB7vfqcZhTKZ+l7hCGxSPoushMv5GKhT5PdLv03WA==} + + lodash.defaults@4.2.0: + resolution: {integrity: sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==} + + lodash.defaultsdeep@4.6.1: + resolution: {integrity: sha512-3j8wdDzYuWO3lM3Reg03MuQR957t287Rpcxp1njpEa8oDrikb+FwGdW3n+FELh/A6qib6yPit0j/pv9G/yeAqA==} + + lodash.isarguments@3.1.0: + resolution: {integrity: sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==} + + lodash.merge@4.6.2: + resolution: {integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==} + + lodash.truncate@4.4.2: + resolution: {integrity: sha512-jttmRe7bRse52OsWIMDLaXxWqRAmtIUccAQ3garviCqJjafXOfNMO0yMfNpdD6zbGaTU0P5Nz7e7gAT6cKmJRw==} + + lodash@4.17.23: + resolution: {integrity: sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==} + + log-symbols@4.1.0: + resolution: {integrity: sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==} + engines: {node: '>=10'} + + lowlight@1.20.0: + resolution: {integrity: sha512-8Ktj+prEb1RoCPkEOrPMYUN/nCggB7qAWe3a7OpMjWQkh3l2RD5wKRQ+o8Q8YuI9RG/xs95waaI/E6ym/7NsTw==} + + lru-cache@10.4.3: + resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} + + lru-cache@11.2.6: + resolution: {integrity: sha512-ESL2CrkS/2wTPfuend7Zhkzo2u0daGJ/A2VucJOgQ/C48S/zB8MMeMHSGKYpXhIjbPxfuezITkaBH1wqv00DDQ==} + engines: {node: 20 || >=22} + + lru-cache@5.1.1: + resolution: {integrity: sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==} + + lucide-react@0.555.0: + resolution: {integrity: sha512-D8FvHUGbxWBRQM90NZeIyhAvkFfsh3u9ekrMvJ30Z6gnpBHS6HC6ldLg7tL45hwiIz/u66eKDtdA23gwwGsAHA==} + peerDependencies: + react: ^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0 + + luxon@3.7.2: + resolution: {integrity: sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew==} + engines: {node: '>=12'} + + magic-string@0.30.21: + resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} + + make-dir@3.1.0: + resolution: {integrity: sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==} + engines: {node: '>=8'} + + make-error@1.3.6: + resolution: {integrity: sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==} + + marked@14.0.0: + resolution: {integrity: sha512-uIj4+faQ+MgHgwUW1l2PsPglZLOLOT1uErt06dAPtx2kjteLAkbsd/0FiYg/MGS+i7ZKLb7w2WClxHkzOOuryQ==} + engines: {node: '>= 18'} + hasBin: true + + math-intrinsics@1.1.0: + resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} + engines: {node: '>= 0.4'} + + media-typer@0.3.0: + resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==} + engines: {node: '>= 0.6'} + + merge-descriptors@1.0.3: + resolution: {integrity: sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==} + + methods@1.1.2: + resolution: {integrity: sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==} + engines: {node: '>= 0.6'} + + mime-db@1.52.0: + resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} + engines: {node: '>= 0.6'} + + mime-types@2.1.35: + resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} + engines: {node: '>= 0.6'} + + mime@1.6.0: + resolution: {integrity: sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==} + engines: {node: '>=4'} + hasBin: true + + mimic-fn@2.1.0: + resolution: {integrity: sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==} + engines: {node: '>=6'} + + minimatch@10.2.0: + resolution: {integrity: sha512-ugkC31VaVg9cF0DFVoADH12k6061zNZkZON+aX8AWsR9GhPcErkcMBceb6znR8wLERM2AkkOxy2nWRLpT9Jq5w==} + engines: {node: 20 || >=22} + + minimatch@3.1.2: + resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==} + + minimatch@5.1.6: + resolution: {integrity: sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==} + engines: {node: '>=10'} + + minimatch@9.0.5: + resolution: {integrity: sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==} + engines: {node: '>=16 || 14 >=14.17'} + + minimist@1.2.8: + resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} + + minipass@3.3.6: + resolution: {integrity: sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==} + engines: {node: '>=8'} + + minipass@5.0.0: + resolution: {integrity: sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==} + engines: {node: '>=8'} + + minipass@7.1.2: + resolution: {integrity: sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==} + engines: {node: '>=16 || 14 >=14.17'} + + minizlib@2.1.2: + resolution: {integrity: sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==} + engines: {node: '>= 8'} + + mkdirp@1.0.4: + resolution: {integrity: sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==} + engines: {node: '>=10'} + hasBin: true + + monaco-editor@0.55.1: + resolution: {integrity: sha512-jz4x+TJNFHwHtwuV9vA9rMujcZRb0CEilTEwG2rRSpe/A7Jdkuj8xPKttCgOh+v/lkHy7HsZ64oj+q3xoAFl9A==} + + motia@0.17.14-beta.196: + resolution: {integrity: sha512-/5ynL2WG8klmHt2H01SAmg3g0M0e0CQvrVQ5QxXwuTdY/gj2pY0vai8UcKsSzs394WsF9dCzNrDK6mX/ErRVTQ==} + hasBin: true + + motion-dom@12.34.0: + resolution: {integrity: sha512-Lql3NuEcScRDxTAO6GgUsRHBZOWI/3fnMlkMcH5NftzcN37zJta+bpbMAV9px4Nj057TuvRooMK7QrzMCgtz6Q==} + + motion-utils@12.29.2: + resolution: {integrity: sha512-G3kc34H2cX2gI63RqU+cZq+zWRRPSsNIOjpdl9TN4AQwC4sgwYPl/Q/Obf/d53nOm569T0fYK+tcoSV50BWx8A==} + + motion@12.34.0: + resolution: {integrity: sha512-01Sfa/zgsD/di8zA/uFW5Eb7/SPXoGyUfy+uMRMW5Spa8j0z/UbfQewAYvPMYFCXRlyD6e5aLHh76TxeeJD+RA==} + peerDependencies: + '@emotion/is-prop-valid': '*' + react: ^18.0.0 || ^19.0.0 + react-dom: ^18.0.0 || ^19.0.0 + peerDependenciesMeta: + '@emotion/is-prop-valid': + optional: true + react: + optional: true + react-dom: + optional: true + + ms@2.0.0: + resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==} + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + msgpackr-extract@3.0.3: + resolution: {integrity: sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==} + hasBin: true + + msgpackr@1.11.5: + resolution: {integrity: sha512-UjkUHN0yqp9RWKy0Lplhh+wlpdt9oQBYgULZOiFhV3VclSF1JnSQWZ5r9gORQlNYaUKQoR8itv7g7z1xDDuACA==} + + mute-stream@0.0.8: + resolution: {integrity: sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==} + + nanoid@3.3.11: + resolution: {integrity: sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + + natural-compare@1.4.0: + resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} + + negotiator@0.6.3: + resolution: {integrity: sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==} + engines: {node: '>= 0.6'} + + node-abort-controller@3.1.1: + resolution: {integrity: sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==} + + node-cron@3.0.3: + resolution: {integrity: sha512-dOal67//nohNgYWb+nWmg5dkFdIwDm8EpeGYMekPMrngV3637lqnX0lbUcCtgibHTz6SEz7DAIjKvKDFYCnO1A==} + engines: {node: '>=6.0.0'} + + node-gyp-build-optional-packages@5.2.2: + resolution: {integrity: sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==} + hasBin: true + + node-releases@2.0.27: + resolution: {integrity: sha512-nmh3lCkYZ3grZvqcCH+fjmQ7X+H0OeZgP40OierEaAptX4XofMh5kwNbWh7lBduUzCcV/8kZ+NDLCwm2iorIlA==} + + normalize-path@3.0.0: + resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==} + engines: {node: '>=0.10.0'} + + object-inspect@1.13.4: + resolution: {integrity: sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==} + engines: {node: '>= 0.4'} + + on-finished@2.4.1: + resolution: {integrity: sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==} + engines: {node: '>= 0.8'} + + once@1.4.0: + resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + + onetime@5.1.2: + resolution: {integrity: sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==} + engines: {node: '>=6'} + + optionator@0.9.4: + resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} + engines: {node: '>= 0.8.0'} + + ora@5.4.1: + resolution: {integrity: sha512-5b6Y85tPxZZ7QytO+BQzysW31HJku27cRIlkbAXaNx+BdcVi+LlRFmVXzeF6a7JCwJpyw5c4b+YSVImQIrBpuQ==} + engines: {node: '>=10'} + + p-limit@2.3.0: + resolution: {integrity: sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==} + engines: {node: '>=6'} + + p-limit@3.1.0: + resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==} + engines: {node: '>=10'} + + p-locate@4.1.0: + resolution: {integrity: sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==} + engines: {node: '>=8'} + + p-locate@5.0.0: + resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==} + engines: {node: '>=10'} + + p-try@2.2.0: + resolution: {integrity: sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==} + engines: {node: '>=6'} + + package-json-from-dist@1.0.1: + resolution: {integrity: sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==} + + parent-module@1.0.1: + resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==} + engines: {node: '>=6'} + + parse-entities@4.0.2: + resolution: {integrity: sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw==} + + parseurl@1.3.3: + resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==} + engines: {node: '>= 0.8'} + + path-exists@4.0.0: + resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} + engines: {node: '>=8'} + + path-key@3.1.1: + resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} + engines: {node: '>=8'} + + path-scurry@1.11.1: + resolution: {integrity: sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==} + engines: {node: '>=16 || 14 >=14.18'} + + path-scurry@2.0.1: + resolution: {integrity: sha512-oWyT4gICAu+kaA7QWk/jvCHWarMKNs6pXOGWKDTr7cw4IGcUbW+PeTfbaQiLGheFRpjo6O9J0PmyMfQPjH71oA==} + engines: {node: 20 || >=22} + + path-to-regexp@0.1.12: + resolution: {integrity: sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==} + + pend@1.2.0: + resolution: {integrity: sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==} + + picocolors@1.1.1: + resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} + + picomatch@4.0.3: + resolution: {integrity: sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==} + engines: {node: '>=12'} + + pkg-dir@4.2.0: + resolution: {integrity: sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==} + engines: {node: '>=8'} + + postcss-value-parser@4.2.0: + resolution: {integrity: sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==} + + postcss@8.5.6: + resolution: {integrity: sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg==} + engines: {node: ^10 || ^12 || >=14} + + prelude-ls@1.2.1: + resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==} + engines: {node: '>= 0.8.0'} + + prismjs@1.30.0: + resolution: {integrity: sha512-DEvV2ZF2r2/63V+tK8hQvrR2ZGn10srHbXviTlcv7Kpzw8jWiNTqbVgjO3IY8RxrrOUF8VPMQQFysYYYv0YZxw==} + engines: {node: '>=6'} + + process-nextick-args@2.0.1: + resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==} + + process@0.11.10: + resolution: {integrity: sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==} + engines: {node: '>= 0.6.0'} + + property-information@7.1.0: + resolution: {integrity: sha512-TwEZ+X+yCJmYfL7TPUOcvBZ4QfoT5YenQiJuX//0th53DE6w0xxLEtfK3iyryQFddXuvkIk51EEgrJQ0WJkOmQ==} + + proxy-addr@2.0.7: + resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} + engines: {node: '>= 0.10'} + + proxy-from-env@1.1.0: + resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} + + pump@3.0.3: + resolution: {integrity: sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA==} + + punycode@2.3.1: + resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} + engines: {node: '>=6'} + + python-ast@0.1.0: + resolution: {integrity: sha512-uMPE7HRMfsbHtQYPg/+EH9MJkynLfLr+0VJbeBgJHpt2wuDgf5hZrEczOIGNFKaO0W1HWiL7bSxA/EOOo70mIQ==} + + qs@6.14.2: + resolution: {integrity: sha512-V/yCWTTF7VJ9hIh18Ugr2zhJMP01MY7c5kh4J870L7imm6/DIzBsNLTXzMwUA3yZ5b/KBqLx8Kp3uRvd7xSe3Q==} + engines: {node: '>=0.6'} + + range-parser@1.2.1: + resolution: {integrity: sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==} + engines: {node: '>= 0.6'} + + raw-body@2.5.3: + resolution: {integrity: sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==} + engines: {node: '>= 0.8'} + + react-dom@19.2.4: + resolution: {integrity: sha512-AXJdLo8kgMbimY95O2aKQqsz2iWi9jMgKJhRBAxECE4IFxfcazB2LmzloIoibJI3C12IlY20+KFaLv+71bUJeQ==} + peerDependencies: + react: ^19.2.4 + + react-refresh@0.18.0: + resolution: {integrity: sha512-QgT5//D3jfjJb6Gsjxv0Slpj23ip+HtOpnNgnb2S5zU3CB26G/IDPGoy4RJB42wzFE46DRsstbW6tKHoKbhAxw==} + engines: {node: '>=0.10.0'} + + react-remove-scroll-bar@2.3.8: + resolution: {integrity: sha512-9r+yi9+mgU33AKcj6IbT9oRCO78WriSj6t/cF8DWBZJ9aOGPOTEDvdUDz1FwKim7QXWwmHqtdHnRJfhAxEG46Q==} + engines: {node: '>=10'} + peerDependencies: + '@types/react': '*' + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + peerDependenciesMeta: + '@types/react': + optional: true + + react-remove-scroll@2.7.2: + resolution: {integrity: sha512-Iqb9NjCCTt6Hf+vOdNIZGdTiH1QSqr27H/Ek9sv/a97gfueI/5h1s3yRi1nngzMUaOOToin5dI1dXKdXiF+u0Q==} + engines: {node: '>=10'} + peerDependencies: + '@types/react': '*' + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + react-resizable-panels@3.0.6: + resolution: {integrity: sha512-b3qKHQ3MLqOgSS+FRYKapNkJZf5EQzuf6+RLiq1/IlTHw99YrZ2NJZLk4hQIzTnnIkRg2LUqyVinu6YWWpUYew==} + peerDependencies: + react: ^16.14.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc + react-dom: ^16.14.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc + + react-router-dom@7.13.0: + resolution: {integrity: sha512-5CO/l5Yahi2SKC6rGZ+HDEjpjkGaG/ncEP7eWFTvFxbHP8yeeI0PxTDjimtpXYlR3b3i9/WIL4VJttPrESIf2g==} + engines: {node: '>=20.0.0'} + peerDependencies: + react: '>=18' + react-dom: '>=18' + + react-router@7.13.0: + resolution: {integrity: sha512-PZgus8ETambRT17BUm/LL8lX3Of+oiLaPuVTRH3l1eLvSPpKO3AvhAEb5N7ihAFZQrYDqkvvWfFh9p0z9VsjLw==} + engines: {node: '>=20.0.0'} + peerDependencies: + react: '>=18' + react-dom: '>=18' + peerDependenciesMeta: + react-dom: + optional: true + + react-style-singleton@2.2.3: + resolution: {integrity: sha512-b6jSvxvVnyptAiLjbkWLE/lOnR4lfTtDAl+eUC7RZy+QQWc6wRzIV2CE6xBuMmDxc2qIihtDCZD5NPOFl7fRBQ==} + engines: {node: '>=10'} + peerDependencies: + '@types/react': '*' + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + react-syntax-highlighter@16.1.0: + resolution: {integrity: sha512-E40/hBiP5rCNwkeBN1vRP+xow1X0pndinO+z3h7HLsHyjztbyjfzNWNKuAsJj+7DLam9iT4AaaOZnueCU+Nplg==} + engines: {node: '>= 16.20.2'} + peerDependencies: + react: '>= 0.14.0' + + react-use-resizable@0.2.0: + resolution: {integrity: sha512-gp1cSSNzRDuRouWTYYxKE7f8VC/4cqdGpgbN9rDNdaFSwCfze/ki+WJL0Zouab/HILDpfIGW2JqAz94VmfroLA==} + peerDependencies: + react: '>=16.8.0' + + react18-json-view@0.2.9: + resolution: {integrity: sha512-z3JQgCwZRKbmWh54U94loCU6vE0ZoDBK7C8ZpcMYQB8jYMi+mR/fcgMI9jKgATeF0I6+OAF025PD+UKkXIqueQ==} + peerDependencies: + react: '>=16.8.0' + + react@19.2.4: + resolution: {integrity: sha512-9nfp2hYpCwOjAN+8TZFGhtWEwgvWHXqESH8qT89AT/lWklpLON22Lc8pEtnpsZz7VmawabSU0gCjnj8aC0euHQ==} + engines: {node: '>=0.10.0'} + + readable-stream@2.3.8: + resolution: {integrity: sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==} + + readable-stream@3.6.2: + resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} + engines: {node: '>= 6'} + + readable-stream@4.7.0: + resolution: {integrity: sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + + readdir-glob@1.1.3: + resolution: {integrity: sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==} + + readdirp@4.1.2: + resolution: {integrity: sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==} + engines: {node: '>= 14.18.0'} + + redis-errors@1.2.0: + resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==} + engines: {node: '>=4'} + + redis-memory-server@0.14.1: + resolution: {integrity: sha512-18aoXxvUrbBSAmZ42nC4yEEBRmGA4Hew11Zk64dpgtW6du71218GvBn0h6zDKIR/gOcZhkpqYoDjJ4zIw5nsuA==} + engines: {node: '>=16'} + hasBin: true + + redis-parser@3.0.0: + resolution: {integrity: sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==} + engines: {node: '>=4'} + + redis@5.10.0: + resolution: {integrity: sha512-0/Y+7IEiTgVGPrLFKy8oAEArSyEJkU0zvgV5xyi9NzNQ+SLZmyFbUsWIbgPcd4UdUh00opXGKlXJwMmsis5Byw==} + engines: {node: '>= 18'} + + refractor@5.0.0: + resolution: {integrity: sha512-QXOrHQF5jOpjjLfiNk5GFnWhRXvxjUVnlFxkeDmewR5sXkr3iM46Zo+CnRR8B+MDVqkULW4EcLVcRBNOPXHosw==} + + require-from-string@2.0.2: + resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} + engines: {node: '>=0.10.0'} + + resolve-from@4.0.0: + resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==} + engines: {node: '>=4'} + + resolve-pkg-maps@1.0.0: + resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==} + + restore-cursor@3.1.0: + resolution: {integrity: sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==} + engines: {node: '>=8'} + + rimraf@5.0.10: + resolution: {integrity: sha512-l0OE8wL34P4nJH/H2ffoaniAokM2qSmrtXHmlpvYr5AVVX8msAyW0l8NVJFDxlSK4u3Uh/f41cQheDVdnYijwQ==} + hasBin: true + + rollup@4.57.1: + resolution: {integrity: sha512-oQL6lgK3e2QZeQ7gcgIkS2YZPg5slw37hYufJ3edKlfQSGGm8ICoxswK15ntSzF/a8+h7ekRy7k7oWc3BQ7y8A==} + engines: {node: '>=18.0.0', npm: '>=8.0.0'} + hasBin: true + + run-async@2.4.1: + resolution: {integrity: sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ==} + engines: {node: '>=0.12.0'} + + rxjs@7.8.2: + resolution: {integrity: sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==} + + safe-buffer@5.1.2: + resolution: {integrity: sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==} + + safe-buffer@5.2.1: + resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} + + safer-buffer@2.1.2: + resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} + + scheduler@0.27.0: + resolution: {integrity: sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==} + + semver@6.3.1: + resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==} + hasBin: true + + semver@7.7.4: + resolution: {integrity: sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==} + engines: {node: '>=10'} + hasBin: true + + send@0.19.2: + resolution: {integrity: sha512-VMbMxbDeehAxpOtWJXlcUS5E8iXh6QmN+BkRX1GARS3wRaXEEgzCcB10gTQazO42tpNIya8xIyNx8fll1OFPrg==} + engines: {node: '>= 0.8.0'} + + serve-static@1.16.3: + resolution: {integrity: sha512-x0RTqQel6g5SY7Lg6ZreMmsOzncHFU7nhnRWkKgWuMTu5NN0DR5oruckMqRvacAN9d5w6ARnRBXl9xhDCgfMeA==} + engines: {node: '>= 0.8.0'} + + set-cookie-parser@2.7.2: + resolution: {integrity: sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw==} + + setprototypeof@1.2.0: + resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} + + shebang-command@2.0.0: + resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} + engines: {node: '>=8'} + + shebang-regex@3.0.0: + resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} + engines: {node: '>=8'} + + side-channel-list@1.0.0: + resolution: {integrity: sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==} + engines: {node: '>= 0.4'} + + side-channel-map@1.0.1: + resolution: {integrity: sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==} + engines: {node: '>= 0.4'} + + side-channel-weakmap@1.0.2: + resolution: {integrity: sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==} + engines: {node: '>= 0.4'} + + side-channel@1.1.0: + resolution: {integrity: sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==} + engines: {node: '>= 0.4'} + + signal-exit@3.0.7: + resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==} + + signal-exit@4.1.0: + resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} + engines: {node: '>=14'} + + slice-ansi@4.0.0: + resolution: {integrity: sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==} + engines: {node: '>=10'} + + source-map-js@1.2.1: + resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} + engines: {node: '>=0.10.0'} + + space-separated-tokens@2.0.2: + resolution: {integrity: sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q==} + + standard-as-callback@2.1.0: + resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==} + + state-local@1.0.7: + resolution: {integrity: sha512-HTEHMNieakEnoe33shBYcZ7NX83ACUjCu8c40iOGEZsngj9zRnkqS9j1pqQPXwobB0ZcVTk27REb7COQ0UR59w==} + + statuses@2.0.2: + resolution: {integrity: sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==} + engines: {node: '>= 0.8'} + + streamx@2.23.0: + resolution: {integrity: sha512-kn+e44esVfn2Fa/O0CPFcex27fjIL6MkVae0Mm6q+E6f0hWv578YCERbv+4m02cjxvDsPKLnmxral/rR6lBMAg==} + + string-width@4.2.3: + resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} + engines: {node: '>=8'} + + string-width@5.1.2: + resolution: {integrity: sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==} + engines: {node: '>=12'} + + string_decoder@1.1.1: + resolution: {integrity: sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==} + + string_decoder@1.3.0: + resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} + + strip-ansi@6.0.1: + resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} + engines: {node: '>=8'} + + strip-ansi@7.1.2: + resolution: {integrity: sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA==} + engines: {node: '>=12'} + + strip-bom@3.0.0: + resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==} + engines: {node: '>=4'} + + strip-json-comments@3.1.1: + resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} + engines: {node: '>=8'} + + supports-color@7.2.0: + resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} + engines: {node: '>=8'} + + table@6.9.0: + resolution: {integrity: sha512-9kY+CygyYM6j02t5YFHbNz2FN5QmYGv9zAjVp4lCDjlCw7amdckXlEt/bjMhUIfj4ThGRE4gCUH5+yGnNuPo5A==} + engines: {node: '>=10.0.0'} + + tailwind-merge@2.6.1: + resolution: {integrity: sha512-Oo6tHdpZsGpkKG88HJ8RR1rg/RdnEkQEfMoEk2x1XRI3F1AxeU+ijRXpiVUF4UbLfcxxRGw6TbUINKYdWVsQTQ==} + + tailwind-merge@3.4.0: + resolution: {integrity: sha512-uSaO4gnW+b3Y2aWoWfFpX62vn2sR3skfhbjsEnaBI81WD1wBLlHZe5sWf0AqjksNdYTbGBEd0UasQMT3SNV15g==} + + tailwindcss@4.1.18: + resolution: {integrity: sha512-4+Z+0yiYyEtUVCScyfHCxOYP06L5Ne+JiHhY2IjR2KWMIWhJOYZKLSGZaP5HkZ8+bY0cxfzwDE5uOmzFXyIwxw==} + + tapable@2.3.0: + resolution: {integrity: sha512-g9ljZiwki/LfxmQADO3dEY1CbpmXT5Hm2fJ+QaGKwSXUylMybePR7/67YW7jOrrvjEgL1Fmz5kzyAjWVWLlucg==} + engines: {node: '>=6'} + + tar-stream@3.1.7: + resolution: {integrity: sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==} + + tar@6.2.1: + resolution: {integrity: sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==} + engines: {node: '>=10'} + deprecated: Old versions of tar are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me + + text-decoder@1.2.4: + resolution: {integrity: sha512-mzlffA3tBNhziEHPK5L5InZg1d/ElNIpJhnhbDRNUtem/edZcJ5zg5FgwKKKOyklxk+6Jt+TrSu83musmvrDlg==} + + through@2.3.8: + resolution: {integrity: sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==} + + tinyglobby@0.2.15: + resolution: {integrity: sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==} + engines: {node: '>=12.0.0'} + + tmp@0.2.5: + resolution: {integrity: sha512-voyz6MApa1rQGUxT3E+BK7/ROe8itEx7vD8/HEvt4xwXucvQ5G5oeEiHkmHZJuBO21RpOf+YYm9MOivj709jow==} + engines: {node: '>=14.14'} + + toggle-selection@1.0.6: + resolution: {integrity: sha512-BiZS+C1OS8g/q2RRbJmy59xpyghNBqrr6k5L/uKBGRsTfxmu3ffiRnd8mlGPUVayg8pvfi5urfnu8TU7DVOkLQ==} + + toidentifier@1.0.1: + resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==} + engines: {node: '>=0.6'} + + ts-api-utils@2.4.0: + resolution: {integrity: sha512-3TaVTaAv2gTiMB35i3FiGJaRfwb3Pyn/j3m/bfAvGe8FB7CF6u+LMYqYlDh7reQf7UNvoTvdfAqHGmPGOSsPmA==} + engines: {node: '>=18.12'} + peerDependencies: + typescript: '>=4.8.4' + + ts-node@10.9.2: + resolution: {integrity: sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==} + hasBin: true + peerDependencies: + '@swc/core': '>=1.2.50' + '@swc/wasm': '>=1.2.50' + '@types/node': '*' + typescript: '>=2.7' + peerDependenciesMeta: + '@swc/core': + optional: true + '@swc/wasm': + optional: true + + tsconfig-paths@4.2.0: + resolution: {integrity: sha512-NoZ4roiN7LnbKn9QqE1amc9DJfzvZXxF4xDavcOWt1BPkdx+m+0gJuPM+S0vCe7zTJMYUP0R8pO2XMr+Y8oLIg==} + engines: {node: '>=6'} + + tslib@2.8.1: + resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + + tsx@4.21.0: + resolution: {integrity: sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==} + engines: {node: '>=18.0.0'} + hasBin: true + + tw-animate-css@1.4.0: + resolution: {integrity: sha512-7bziOlRqH0hJx80h/3mbicLW7o8qLsH5+RaLR2t+OHM3D0JlWGODQKQ4cxbK7WlvmUxpcj6Kgu6EKqjrGFe3QQ==} + + type-check@0.4.0: + resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} + engines: {node: '>= 0.8.0'} + + type-fest@0.21.3: + resolution: {integrity: sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==} + engines: {node: '>=10'} + + type-is@1.6.18: + resolution: {integrity: sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==} + engines: {node: '>= 0.6'} + + typescript-eslint@8.55.0: + resolution: {integrity: sha512-HE4wj+r5lmDVS9gdaN0/+iqNvPZwGfnJ5lZuz7s5vLlg9ODw0bIiiETaios9LvFI1U94/VBXGm3CB2Y5cNFMpw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + eslint: ^8.57.0 || ^9.0.0 + typescript: '>=4.8.4 <6.0.0' + + typescript@5.9.3: + resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} + engines: {node: '>=14.17'} + hasBin: true + + undici-types@7.16.0: + resolution: {integrity: sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==} + + unpipe@1.0.0: + resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==} + engines: {node: '>= 0.8'} + + update-browserslist-db@1.2.3: + resolution: {integrity: sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==} + hasBin: true + peerDependencies: + browserslist: '>= 4.21.0' + + uri-js@4.4.1: + resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} + + use-callback-ref@1.3.3: + resolution: {integrity: sha512-jQL3lRnocaFtu3V00JToYz/4QkNWswxijDaCVNZRiRTO3HQDLsdu1ZtmIUvV4yPp+rvWm5j0y0TG/S61cuijTg==} + engines: {node: '>=10'} + peerDependencies: + '@types/react': '*' + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + use-sidecar@1.1.3: + resolution: {integrity: sha512-Fedw0aZvkhynoPYlA5WXrMCAMm+nSWdZt6lzJQ7Ok8S6Q+VsHmHpRWndVRJ8Be0ZbkfPc5LRYH+5XrzXcEeLRQ==} + engines: {node: '>=10'} + peerDependencies: + '@types/react': '*' + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + use-sync-external-store@1.6.0: + resolution: {integrity: sha512-Pp6GSwGP/NrPIrxVFAIkOQeyw8lFenOHijQWkUTrDvrF4ALqylP2C/KCkeS9dpUM3KvYRQhna5vt7IL95+ZQ9w==} + peerDependencies: + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + + util-deprecate@1.0.2: + resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + + utils-merge@1.0.1: + resolution: {integrity: sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==} + engines: {node: '>= 0.4.0'} + + uuid@11.1.0: + resolution: {integrity: sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==} + hasBin: true + + uuid@13.0.0: + resolution: {integrity: sha512-XQegIaBTVUjSHliKqcnFqYypAd4S+WCYt5NIeRs6w/UAry7z8Y9j5ZwRRL4kzq9U3sD6v+85er9FvkEaBpji2w==} + hasBin: true + + uuid@8.3.2: + resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==} + hasBin: true + + v8-compile-cache-lib@3.0.1: + resolution: {integrity: sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==} + + vary@1.1.2: + resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==} + engines: {node: '>= 0.8'} + + vite@7.3.1: + resolution: {integrity: sha512-w+N7Hifpc3gRjZ63vYBXA56dvvRlNWRczTdmCBBa+CotUzAPf5b7YMdMR/8CQoeYE5LX3W4wj6RYTgonm1b9DA==} + engines: {node: ^20.19.0 || >=22.12.0} + hasBin: true + peerDependencies: + '@types/node': ^20.19.0 || >=22.12.0 + jiti: '>=1.21.0' + less: ^4.0.0 + lightningcss: ^1.21.0 + sass: ^1.70.0 + sass-embedded: ^1.70.0 + stylus: '>=0.54.8' + sugarss: ^5.0.0 + terser: ^5.16.0 + tsx: ^4.8.1 + yaml: ^2.4.2 + peerDependenciesMeta: + '@types/node': + optional: true + jiti: + optional: true + less: + optional: true + lightningcss: + optional: true + sass: + optional: true + sass-embedded: + optional: true + stylus: + optional: true + sugarss: + optional: true + terser: + optional: true + tsx: + optional: true + yaml: + optional: true + + wcwidth@1.0.1: + resolution: {integrity: sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==} + + which@2.0.2: + resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} + engines: {node: '>= 8'} + hasBin: true + + word-wrap@1.2.5: + resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==} + engines: {node: '>=0.10.0'} + + wrap-ansi@6.2.0: + resolution: {integrity: sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==} + engines: {node: '>=8'} + + wrap-ansi@7.0.0: + resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==} + engines: {node: '>=10'} + + wrap-ansi@8.1.0: + resolution: {integrity: sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==} + engines: {node: '>=12'} + + wrappy@1.0.2: + resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + + ws@8.19.0: + resolution: {integrity: sha512-blAT2mjOEIi0ZzruJfIhb3nps74PRWTCz1IjglWEEpQl5XS/UNama6u2/rjFkDDouqr4L67ry+1aGIALViWjDg==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + + yallist@3.1.1: + resolution: {integrity: sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==} + + yallist@4.0.0: + resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} + + yauzl@2.10.0: + resolution: {integrity: sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==} + + yn@3.1.1: + resolution: {integrity: sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==} + engines: {node: '>=6'} + + yocto-queue@0.1.0: + resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} + engines: {node: '>=10'} + + zen-observable-ts@1.1.0: + resolution: {integrity: sha512-1h4zlLSqI2cRLPJUHJFL8bCWHhkpuXkF+dbGkRaWjgDIG26DmzyshUMrdV/rL3UnR+mhaX4fRq8LPouq0MYYIA==} + + zen-observable@0.8.15: + resolution: {integrity: sha512-PQ2PC7R9rslx84ndNBZB/Dkv8V8fZEpk83RLgXtYd0fwUgEjseMn1Dgajh2x6S8QbZAFa9p2qVCEuYZNgve0dQ==} + + zip-stream@6.0.1: + resolution: {integrity: sha512-zK7YHHz4ZXpW89AHXUPbQVGKI7uvkd3hzusTdotCg1UxyaVtg0zFJSTfW/Dq5f7OBBVnq6cZIaC8Ti4hb6dtCA==} + engines: {node: '>= 14'} + + zod@3.25.76: + resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} + + zod@4.3.6: + resolution: {integrity: sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==} + + zustand@4.5.7: + resolution: {integrity: sha512-CHOUy7mu3lbD6o6LJLfllpjkzhHXSBlX8B9+qPddUsIfeF5S/UZ5q0kmCsnRqT1UHFQZchNFDDzMbQsuesHWlw==} + engines: {node: '>=12.7.0'} + peerDependencies: + '@types/react': '>=16.8' + immer: '>=9.0.6' + react: '>=16.8' + peerDependenciesMeta: + '@types/react': + optional: true + immer: + optional: true + react: + optional: true + + zustand@5.0.11: + resolution: {integrity: sha512-fdZY+dk7zn/vbWNCYmzZULHRrss0jx5pPFiOuMZ/5HJN6Yv3u+1Wswy/4MpZEkEGhtNH+pwxZB8OKgUBPzYAGg==} + engines: {node: '>=12.20.0'} + peerDependencies: + '@types/react': '>=18.0.0' + immer: '>=9.0.6' + react: '>=18.0.0' + use-sync-external-store: '>=1.2.0' + peerDependenciesMeta: + '@types/react': + optional: true + immer: + optional: true + react: + optional: true + use-sync-external-store: + optional: true + +snapshots: + + '@alloc/quick-lru@5.2.0': {} + + '@amplitude/analytics-connector@1.6.4': {} + + '@amplitude/analytics-core@2.38.0': + dependencies: + '@amplitude/analytics-connector': 1.6.4 + tslib: 2.8.1 + zen-observable-ts: 1.1.0 + + '@amplitude/analytics-node@1.5.35': + dependencies: + '@amplitude/analytics-core': 2.38.0 + tslib: 2.8.1 + + '@babel/code-frame@7.29.0': + dependencies: + '@babel/helper-validator-identifier': 7.28.5 + js-tokens: 4.0.0 + picocolors: 1.1.1 + + '@babel/compat-data@7.29.0': {} + + '@babel/core@7.29.0': + dependencies: + '@babel/code-frame': 7.29.0 + '@babel/generator': 7.29.1 + '@babel/helper-compilation-targets': 7.28.6 + '@babel/helper-module-transforms': 7.28.6(@babel/core@7.29.0) + '@babel/helpers': 7.28.6 + '@babel/parser': 7.29.0 + '@babel/template': 7.28.6 + '@babel/traverse': 7.29.0 + '@babel/types': 7.29.0 + '@jridgewell/remapping': 2.3.5 + convert-source-map: 2.0.0 + debug: 4.4.3 + gensync: 1.0.0-beta.2 + json5: 2.2.3 + semver: 6.3.1 + transitivePeerDependencies: + - supports-color + + '@babel/generator@7.29.1': + dependencies: + '@babel/parser': 7.29.0 + '@babel/types': 7.29.0 + '@jridgewell/gen-mapping': 0.3.13 + '@jridgewell/trace-mapping': 0.3.31 + jsesc: 3.1.0 + + '@babel/helper-compilation-targets@7.28.6': + dependencies: + '@babel/compat-data': 7.29.0 + '@babel/helper-validator-option': 7.27.1 + browserslist: 4.28.1 + lru-cache: 5.1.1 + semver: 6.3.1 + + '@babel/helper-globals@7.28.0': {} + + '@babel/helper-module-imports@7.28.6': + dependencies: + '@babel/traverse': 7.29.0 + '@babel/types': 7.29.0 + transitivePeerDependencies: + - supports-color + + '@babel/helper-module-transforms@7.28.6(@babel/core@7.29.0)': + dependencies: + '@babel/core': 7.29.0 + '@babel/helper-module-imports': 7.28.6 + '@babel/helper-validator-identifier': 7.28.5 + '@babel/traverse': 7.29.0 + transitivePeerDependencies: + - supports-color + + '@babel/helper-plugin-utils@7.28.6': {} + + '@babel/helper-string-parser@7.27.1': {} + + '@babel/helper-validator-identifier@7.28.5': {} + + '@babel/helper-validator-option@7.27.1': {} + + '@babel/helpers@7.28.6': + dependencies: + '@babel/template': 7.28.6 + '@babel/types': 7.29.0 + + '@babel/parser@7.29.0': + dependencies: + '@babel/types': 7.29.0 + + '@babel/plugin-transform-react-jsx-self@7.27.1(@babel/core@7.29.0)': + dependencies: + '@babel/core': 7.29.0 + '@babel/helper-plugin-utils': 7.28.6 + + '@babel/plugin-transform-react-jsx-source@7.27.1(@babel/core@7.29.0)': + dependencies: + '@babel/core': 7.29.0 + '@babel/helper-plugin-utils': 7.28.6 + + '@babel/runtime@7.28.6': {} + + '@babel/template@7.28.6': + dependencies: + '@babel/code-frame': 7.29.0 + '@babel/parser': 7.29.0 + '@babel/types': 7.29.0 + + '@babel/traverse@7.29.0': + dependencies: + '@babel/code-frame': 7.29.0 + '@babel/generator': 7.29.1 + '@babel/helper-globals': 7.28.0 + '@babel/parser': 7.29.0 + '@babel/template': 7.28.6 + '@babel/types': 7.29.0 + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + + '@babel/types@7.29.0': + dependencies: + '@babel/helper-string-parser': 7.27.1 + '@babel/helper-validator-identifier': 7.28.5 + + '@cspotcode/source-map-support@0.8.1': + dependencies: + '@jridgewell/trace-mapping': 0.3.9 + + '@esbuild/aix-ppc64@0.25.12': + optional: true + + '@esbuild/aix-ppc64@0.27.3': + optional: true + + '@esbuild/android-arm64@0.25.12': + optional: true + + '@esbuild/android-arm64@0.27.3': + optional: true + + '@esbuild/android-arm@0.25.12': + optional: true + + '@esbuild/android-arm@0.27.3': + optional: true + + '@esbuild/android-x64@0.25.12': + optional: true + + '@esbuild/android-x64@0.27.3': + optional: true + + '@esbuild/darwin-arm64@0.25.12': + optional: true + + '@esbuild/darwin-arm64@0.27.3': + optional: true + + '@esbuild/darwin-x64@0.25.12': + optional: true + + '@esbuild/darwin-x64@0.27.3': + optional: true + + '@esbuild/freebsd-arm64@0.25.12': + optional: true + + '@esbuild/freebsd-arm64@0.27.3': + optional: true + + '@esbuild/freebsd-x64@0.25.12': + optional: true + + '@esbuild/freebsd-x64@0.27.3': + optional: true + + '@esbuild/linux-arm64@0.25.12': + optional: true + + '@esbuild/linux-arm64@0.27.3': + optional: true + + '@esbuild/linux-arm@0.25.12': + optional: true + + '@esbuild/linux-arm@0.27.3': + optional: true + + '@esbuild/linux-ia32@0.25.12': + optional: true + + '@esbuild/linux-ia32@0.27.3': + optional: true + + '@esbuild/linux-loong64@0.25.12': + optional: true + + '@esbuild/linux-loong64@0.27.3': + optional: true + + '@esbuild/linux-mips64el@0.25.12': + optional: true + + '@esbuild/linux-mips64el@0.27.3': + optional: true + + '@esbuild/linux-ppc64@0.25.12': + optional: true + + '@esbuild/linux-ppc64@0.27.3': + optional: true + + '@esbuild/linux-riscv64@0.25.12': + optional: true + + '@esbuild/linux-riscv64@0.27.3': + optional: true + + '@esbuild/linux-s390x@0.25.12': + optional: true + + '@esbuild/linux-s390x@0.27.3': + optional: true + + '@esbuild/linux-x64@0.25.12': + optional: true + + '@esbuild/linux-x64@0.27.3': + optional: true + + '@esbuild/netbsd-arm64@0.25.12': + optional: true + + '@esbuild/netbsd-arm64@0.27.3': + optional: true + + '@esbuild/netbsd-x64@0.25.12': + optional: true + + '@esbuild/netbsd-x64@0.27.3': + optional: true + + '@esbuild/openbsd-arm64@0.25.12': + optional: true + + '@esbuild/openbsd-arm64@0.27.3': + optional: true + + '@esbuild/openbsd-x64@0.25.12': + optional: true + + '@esbuild/openbsd-x64@0.27.3': + optional: true + + '@esbuild/openharmony-arm64@0.25.12': + optional: true + + '@esbuild/openharmony-arm64@0.27.3': + optional: true + + '@esbuild/sunos-x64@0.25.12': + optional: true + + '@esbuild/sunos-x64@0.27.3': + optional: true + + '@esbuild/win32-arm64@0.25.12': + optional: true + + '@esbuild/win32-arm64@0.27.3': + optional: true + + '@esbuild/win32-ia32@0.25.12': + optional: true + + '@esbuild/win32-ia32@0.27.3': + optional: true + + '@esbuild/win32-x64@0.25.12': + optional: true + + '@esbuild/win32-x64@0.27.3': + optional: true + + '@eslint-community/eslint-utils@4.9.1(eslint@9.39.2(jiti@2.6.1))': + dependencies: + eslint: 9.39.2(jiti@2.6.1) + eslint-visitor-keys: 3.4.3 + + '@eslint-community/regexpp@4.12.2': {} + + '@eslint/config-array@0.21.1': + dependencies: + '@eslint/object-schema': 2.1.7 + debug: 4.4.3 + minimatch: 3.1.2 + transitivePeerDependencies: + - supports-color + + '@eslint/config-helpers@0.4.2': + dependencies: + '@eslint/core': 0.17.0 + + '@eslint/core@0.17.0': + dependencies: + '@types/json-schema': 7.0.15 + + '@eslint/eslintrc@3.3.3': + dependencies: + ajv: 6.12.6 + debug: 4.4.3 + espree: 10.4.0 + globals: 14.0.0 + ignore: 5.3.2 + import-fresh: 3.3.1 + js-yaml: 4.1.1 + minimatch: 3.1.2 + strip-json-comments: 3.1.1 + transitivePeerDependencies: + - supports-color + + '@eslint/js@9.39.2': {} + + '@eslint/object-schema@2.1.7': {} + + '@eslint/plugin-kit@0.4.1': + dependencies: + '@eslint/core': 0.17.0 + levn: 0.4.1 + + '@floating-ui/core@1.7.4': + dependencies: + '@floating-ui/utils': 0.2.10 + + '@floating-ui/dom@1.7.5': + dependencies: + '@floating-ui/core': 1.7.4 + '@floating-ui/utils': 0.2.10 + + '@floating-ui/react-dom@2.1.7(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@floating-ui/dom': 1.7.5 + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@floating-ui/utils@0.2.10': {} + + '@humanfs/core@0.19.1': {} + + '@humanfs/node@0.16.7': + dependencies: + '@humanfs/core': 0.19.1 + '@humanwhocodes/retry': 0.4.3 + + '@humanwhocodes/module-importer@1.0.1': {} + + '@humanwhocodes/retry@0.4.3': {} + + '@inquirer/external-editor@1.0.3(@types/node@25.2.3)': + dependencies: + chardet: 2.1.1 + iconv-lite: 0.7.2 + optionalDependencies: + '@types/node': 25.2.3 + + '@ioredis/commands@1.5.0': {} + + '@isaacs/cliui@8.0.2': + dependencies: + string-width: 5.1.2 + string-width-cjs: string-width@4.2.3 + strip-ansi: 7.1.2 + strip-ansi-cjs: strip-ansi@6.0.1 + wrap-ansi: 8.1.0 + wrap-ansi-cjs: wrap-ansi@7.0.0 + + '@isaacs/cliui@9.0.0': {} + + '@jridgewell/gen-mapping@0.3.13': + dependencies: + '@jridgewell/sourcemap-codec': 1.5.5 + '@jridgewell/trace-mapping': 0.3.31 + + '@jridgewell/remapping@2.3.5': + dependencies: + '@jridgewell/gen-mapping': 0.3.13 + '@jridgewell/trace-mapping': 0.3.31 + + '@jridgewell/resolve-uri@3.1.2': {} + + '@jridgewell/sourcemap-codec@1.5.5': {} + + '@jridgewell/trace-mapping@0.3.31': + dependencies: + '@jridgewell/resolve-uri': 3.1.2 + '@jridgewell/sourcemap-codec': 1.5.5 + + '@jridgewell/trace-mapping@0.3.9': + dependencies: + '@jridgewell/resolve-uri': 3.1.2 + '@jridgewell/sourcemap-codec': 1.5.5 + + '@monaco-editor/loader@1.7.0': + dependencies: + state-local: 1.0.7 + + '@monaco-editor/react@4.7.0(monaco-editor@0.55.1)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@monaco-editor/loader': 1.7.0 + monaco-editor: 0.55.1 + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@motiadev/adapter-bullmq-events@0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))': + dependencies: + '@motiadev/core': 0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3) + bullmq: 5.69.1 + ioredis: 5.9.3 + uuid: 11.1.0 + transitivePeerDependencies: + - supports-color + + '@motiadev/adapter-redis-cron@0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))': + dependencies: + '@motiadev/core': 0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3) + redis: 5.10.0 + uuid: 11.1.0 + + '@motiadev/adapter-redis-state@0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))': + dependencies: + '@motiadev/core': 0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3) + redis: 5.10.0 + + '@motiadev/adapter-redis-streams@0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))': + dependencies: + '@motiadev/core': 0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3) + redis: 5.10.0 + + '@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3)': + dependencies: + '@amplitude/analytics-node': 1.5.35 + '@standard-schema/spec': 1.1.0 + ajv: 8.17.1 + body-parser: 1.20.4 + dotenv: 16.6.1 + esbuild: 0.25.12 + express: 4.22.1 + node-cron: 3.0.3 + picocolors: 1.1.1 + redis: 5.10.0 + ts-node: 10.9.2(@types/node@25.2.3)(typescript@5.9.3) + tsconfig-paths: 4.2.0 + tsx: 4.21.0 + uuid: 11.1.0 + ws: 8.19.0 + zod: 4.3.6 + transitivePeerDependencies: + - '@swc/core' + - '@swc/wasm' + - '@types/node' + - bufferutil + - supports-color + - typescript + - utf-8-validate + + '@motiadev/plugin-bullmq@0.17.14-beta.196(@motiadev/adapter-bullmq-events@0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3)))(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))(@motiadev/stream-client-react@0.17.14-beta.196(react@19.2.4))(@motiadev/ui@0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))(lucide-react@0.555.0(react@19.2.4))(motia@0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)))(react@19.2.4)(zustand@5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))': + dependencies: + '@motiadev/adapter-bullmq-events': 0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3)) + '@motiadev/core': 0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3) + '@motiadev/stream-client-react': 0.17.14-beta.196(react@19.2.4) + '@motiadev/ui': 0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + '@tanstack/react-query': 5.90.21(react@19.2.4) + bullmq: 5.69.1 + date-fns: 4.1.0 + ioredis: 5.9.3 + lucide-react: 0.555.0(react@19.2.4) + motia: 0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)) + react: 19.2.4 + react18-json-view: 0.2.9(react@19.2.4) + zustand: 5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + transitivePeerDependencies: + - supports-color + + '@motiadev/plugin-cron@0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))(@motiadev/stream-client-react@0.17.14-beta.196(react@19.2.4))(@motiadev/ui@0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))(use-sync-external-store@1.6.0(react@19.2.4))': + dependencies: + '@motiadev/core': 0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3) + '@motiadev/stream-client-react': 0.17.14-beta.196(react@19.2.4) + '@motiadev/ui': 0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + lucide-react: 0.555.0(react@19.2.4) + react: 19.2.4 + zustand: 5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + transitivePeerDependencies: + - '@types/react' + - immer + - use-sync-external-store + + '@motiadev/plugin-endpoint@0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))(@motiadev/stream-client-react@0.17.14-beta.196(react@19.2.4))(@motiadev/ui@0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))(lucide-react@0.555.0(react@19.2.4))(monaco-editor@0.55.1)(motia@0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(zustand@5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))': + dependencies: + '@monaco-editor/react': 4.7.0(monaco-editor@0.55.1)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@motiadev/core': 0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3) + '@motiadev/stream-client-react': 0.17.14-beta.196(react@19.2.4) + '@motiadev/ui': 0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + clsx: 2.1.1 + json-schema: 0.4.0 + lucide-react: 0.555.0(react@19.2.4) + motia: 0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)) + react: 19.2.4 + react-syntax-highlighter: 16.1.0(react@19.2.4) + react18-json-view: 0.2.9(react@19.2.4) + tailwind-merge: 3.4.0 + zustand: 5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + transitivePeerDependencies: + - monaco-editor + - react-dom + + '@motiadev/plugin-logs@0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))(@motiadev/stream-client-react@0.17.14-beta.196(react@19.2.4))(@motiadev/ui@0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))(lucide-react@0.555.0(react@19.2.4))(motia@0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(zustand@5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))': + dependencies: + '@motiadev/core': 0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3) + '@motiadev/stream-client-react': 0.17.14-beta.196(react@19.2.4) + '@motiadev/ui': 0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + '@tanstack/react-virtual': 3.13.18(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + lucide-react: 0.555.0(react@19.2.4) + motia: 0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)) + react: 19.2.4 + react18-json-view: 0.2.9(react@19.2.4) + zustand: 5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + transitivePeerDependencies: + - react-dom + + '@motiadev/plugin-observability@0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))(@motiadev/stream-client-react@0.17.14-beta.196(react@19.2.4))(@motiadev/ui@0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))(lucide-react@0.555.0(react@19.2.4))(motia@0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(zustand@5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))': + dependencies: + '@motiadev/core': 0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3) + '@motiadev/stream-client-react': 0.17.14-beta.196(react@19.2.4) + '@motiadev/ui': 0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + '@radix-ui/react-popover': 1.1.15(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@tanstack/react-virtual': 3.13.18(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + date-fns: 4.1.0 + lucide-react: 0.555.0(react@19.2.4) + motia: 0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)) + react: 19.2.4 + react18-json-view: 0.2.9(react@19.2.4) + zustand: 5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + transitivePeerDependencies: + - '@types/react' + - '@types/react-dom' + - react-dom + + '@motiadev/plugin-states@0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))(@motiadev/ui@0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))(lucide-react@0.555.0(react@19.2.4))(monaco-editor@0.55.1)(motia@0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(zustand@5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))': + dependencies: + '@monaco-editor/react': 4.7.0(monaco-editor@0.55.1)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@motiadev/core': 0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3) + '@motiadev/ui': 0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + lucide-react: 0.555.0(react@19.2.4) + motia: 0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)) + react: 19.2.4 + react18-json-view: 0.2.9(react@19.2.4) + zustand: 5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + transitivePeerDependencies: + - monaco-editor + - react-dom + + '@motiadev/plugin-streams@0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))(@motiadev/ui@0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))(lucide-react@0.555.0(react@19.2.4))(motia@0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(zustand@5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))': + dependencies: + '@motiadev/core': 0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3) + '@motiadev/ui': 0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + clsx: 2.1.1 + lucide-react: 0.555.0(react@19.2.4) + motia: 0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)) + motion: 12.34.0(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + react: 19.2.4 + react18-json-view: 0.2.9(react@19.2.4) + tailwind-merge: 2.6.1 + zustand: 5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + transitivePeerDependencies: + - '@emotion/is-prop-valid' + - react-dom + + '@motiadev/plugin-ws@0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))(@motiadev/ui@0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))(use-sync-external-store@1.6.0(react@19.2.4))': + dependencies: + '@motiadev/core': 0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3) + '@motiadev/ui': 0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + lucide-react: 0.555.0(react@19.2.4) + react: 19.2.4 + zustand: 5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + transitivePeerDependencies: + - '@types/react' + - immer + - use-sync-external-store + + '@motiadev/stream-client-browser@0.17.14-beta.196': + dependencies: + '@motiadev/stream-client': 0.17.14-beta.196 + uuid: 13.0.0 + + '@motiadev/stream-client-node@0.17.14-beta.196': + dependencies: + '@motiadev/stream-client': 0.17.14-beta.196 + uuid: 13.0.0 + ws: 8.19.0 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + + '@motiadev/stream-client-react@0.17.14-beta.196(react@19.2.4)': + dependencies: + '@motiadev/stream-client-browser': 0.17.14-beta.196 + react: 19.2.4 + + '@motiadev/stream-client@0.17.14-beta.196': + dependencies: + uuid: 13.0.0 + + '@motiadev/ui@0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4))': + dependencies: + '@radix-ui/react-checkbox': 1.3.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-dropdown-menu': 2.1.16(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-label': 2.1.8(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-select': 2.2.6(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-separator': 1.1.8(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-slot': 1.2.4(react@19.2.4) + '@radix-ui/react-tabs': 1.1.13(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-tooltip': 1.2.8(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + class-variance-authority: 0.7.1 + clsx: 2.1.1 + lucide-react: 0.555.0(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + react-resizable-panels: 3.0.6(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + react-use-resizable: 0.2.0(react@19.2.4) + tailwind-merge: 3.4.0 + zustand: 5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + transitivePeerDependencies: + - '@types/react' + - '@types/react-dom' + - immer + - use-sync-external-store + + '@motiadev/workbench@0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(jiti@2.6.1)(lightningcss@1.30.2)(monaco-editor@0.55.1)(redis@5.10.0)(tsx@4.21.0)(use-sync-external-store@1.6.0(react@19.2.4))': + dependencies: + '@monaco-editor/react': 4.7.0(monaco-editor@0.55.1)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@motiadev/core': 0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3) + '@motiadev/stream-client-react': 0.17.14-beta.196(react@19.2.4) + '@motiadev/ui': 0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + '@radix-ui/react-tooltip': 1.2.8(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@tailwindcss/postcss': 4.1.18 + '@vitejs/plugin-react': 5.1.4(vite@7.3.1(@types/node@25.2.3)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)) + '@xyflow/react': 12.10.0(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + autoprefixer: 10.4.24(postcss@8.5.6) + babel-plugin-react-compiler: 1.0.0 + class-variance-authority: 0.7.1 + clsx: 2.1.1 + dagre: 0.8.5 + fast-deep-equal: 3.1.3 + json-schema: 0.4.0 + lucide-react: 0.555.0(react@19.2.4) + postcss: 8.5.6 + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + react-router-dom: 7.13.0(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + react-syntax-highlighter: 16.1.0(react@19.2.4) + react18-json-view: 0.2.9(react@19.2.4) + tailwind-merge: 3.4.0 + tailwindcss: 4.1.18 + tw-animate-css: 1.4.0 + typescript: 5.9.3 + typescript-eslint: 8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3) + vite: 7.3.1(@types/node@25.2.3)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0) + zod: 4.3.6 + zustand: 5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + transitivePeerDependencies: + - '@swc/core' + - '@swc/wasm' + - '@types/node' + - '@types/react' + - '@types/react-dom' + - bufferutil + - eslint + - immer + - jiti + - less + - lightningcss + - monaco-editor + - redis + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + - tsx + - use-sync-external-store + - utf-8-validate + - yaml + + '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.3': + optional: true + + '@pkgjs/parseargs@0.11.0': + optional: true + + '@radix-ui/number@1.1.1': {} + + '@radix-ui/primitive@1.1.3': {} + + '@radix-ui/react-arrow@1.1.7(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@radix-ui/react-checkbox@1.3.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/primitive': 1.1.3 + '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) + '@radix-ui/react-context': 1.1.2(react@19.2.4) + '@radix-ui/react-presence': 1.1.5(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-use-controllable-state': 1.2.2(react@19.2.4) + '@radix-ui/react-use-previous': 1.1.1(react@19.2.4) + '@radix-ui/react-use-size': 1.1.1(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@radix-ui/react-collection@1.1.7(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) + '@radix-ui/react-context': 1.1.2(react@19.2.4) + '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-slot': 1.2.3(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@radix-ui/react-compose-refs@1.1.2(react@19.2.4)': + dependencies: + react: 19.2.4 + + '@radix-ui/react-context@1.1.2(react@19.2.4)': + dependencies: + react: 19.2.4 + + '@radix-ui/react-direction@1.1.1(react@19.2.4)': + dependencies: + react: 19.2.4 + + '@radix-ui/react-dismissable-layer@1.1.11(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/primitive': 1.1.3 + '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) + '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-use-callback-ref': 1.1.1(react@19.2.4) + '@radix-ui/react-use-escape-keydown': 1.1.1(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@radix-ui/react-dropdown-menu@2.1.16(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/primitive': 1.1.3 + '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) + '@radix-ui/react-context': 1.1.2(react@19.2.4) + '@radix-ui/react-id': 1.1.1(react@19.2.4) + '@radix-ui/react-menu': 2.1.16(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-use-controllable-state': 1.2.2(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@radix-ui/react-focus-guards@1.1.3(react@19.2.4)': + dependencies: + react: 19.2.4 + + '@radix-ui/react-focus-scope@1.1.7(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) + '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-use-callback-ref': 1.1.1(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@radix-ui/react-id@1.1.1(react@19.2.4)': + dependencies: + '@radix-ui/react-use-layout-effect': 1.1.1(react@19.2.4) + react: 19.2.4 + + '@radix-ui/react-label@2.1.8(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/react-primitive': 2.1.4(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@radix-ui/react-menu@2.1.16(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/primitive': 1.1.3 + '@radix-ui/react-collection': 1.1.7(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) + '@radix-ui/react-context': 1.1.2(react@19.2.4) + '@radix-ui/react-direction': 1.1.1(react@19.2.4) + '@radix-ui/react-dismissable-layer': 1.1.11(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-focus-guards': 1.1.3(react@19.2.4) + '@radix-ui/react-focus-scope': 1.1.7(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-id': 1.1.1(react@19.2.4) + '@radix-ui/react-popper': 1.2.8(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-portal': 1.1.9(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-presence': 1.1.5(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-roving-focus': 1.1.11(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-slot': 1.2.3(react@19.2.4) + '@radix-ui/react-use-callback-ref': 1.1.1(react@19.2.4) + aria-hidden: 1.2.6 + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + react-remove-scroll: 2.7.2(react@19.2.4) + + '@radix-ui/react-popover@1.1.15(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/primitive': 1.1.3 + '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) + '@radix-ui/react-context': 1.1.2(react@19.2.4) + '@radix-ui/react-dismissable-layer': 1.1.11(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-focus-guards': 1.1.3(react@19.2.4) + '@radix-ui/react-focus-scope': 1.1.7(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-id': 1.1.1(react@19.2.4) + '@radix-ui/react-popper': 1.2.8(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-portal': 1.1.9(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-presence': 1.1.5(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-slot': 1.2.3(react@19.2.4) + '@radix-ui/react-use-controllable-state': 1.2.2(react@19.2.4) + aria-hidden: 1.2.6 + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + react-remove-scroll: 2.7.2(react@19.2.4) + + '@radix-ui/react-popper@1.2.8(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@floating-ui/react-dom': 2.1.7(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-arrow': 1.1.7(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) + '@radix-ui/react-context': 1.1.2(react@19.2.4) + '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-use-callback-ref': 1.1.1(react@19.2.4) + '@radix-ui/react-use-layout-effect': 1.1.1(react@19.2.4) + '@radix-ui/react-use-rect': 1.1.1(react@19.2.4) + '@radix-ui/react-use-size': 1.1.1(react@19.2.4) + '@radix-ui/rect': 1.1.1 + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@radix-ui/react-portal@1.1.9(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-use-layout-effect': 1.1.1(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@radix-ui/react-presence@1.1.5(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) + '@radix-ui/react-use-layout-effect': 1.1.1(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@radix-ui/react-primitive@2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/react-slot': 1.2.3(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@radix-ui/react-primitive@2.1.4(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/react-slot': 1.2.4(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@radix-ui/react-roving-focus@1.1.11(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/primitive': 1.1.3 + '@radix-ui/react-collection': 1.1.7(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) + '@radix-ui/react-context': 1.1.2(react@19.2.4) + '@radix-ui/react-direction': 1.1.1(react@19.2.4) + '@radix-ui/react-id': 1.1.1(react@19.2.4) + '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-use-callback-ref': 1.1.1(react@19.2.4) + '@radix-ui/react-use-controllable-state': 1.2.2(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@radix-ui/react-select@2.2.6(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/number': 1.1.1 + '@radix-ui/primitive': 1.1.3 + '@radix-ui/react-collection': 1.1.7(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) + '@radix-ui/react-context': 1.1.2(react@19.2.4) + '@radix-ui/react-direction': 1.1.1(react@19.2.4) + '@radix-ui/react-dismissable-layer': 1.1.11(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-focus-guards': 1.1.3(react@19.2.4) + '@radix-ui/react-focus-scope': 1.1.7(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-id': 1.1.1(react@19.2.4) + '@radix-ui/react-popper': 1.2.8(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-portal': 1.1.9(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-slot': 1.2.3(react@19.2.4) + '@radix-ui/react-use-callback-ref': 1.1.1(react@19.2.4) + '@radix-ui/react-use-controllable-state': 1.2.2(react@19.2.4) + '@radix-ui/react-use-layout-effect': 1.1.1(react@19.2.4) + '@radix-ui/react-use-previous': 1.1.1(react@19.2.4) + '@radix-ui/react-visually-hidden': 1.2.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + aria-hidden: 1.2.6 + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + react-remove-scroll: 2.7.2(react@19.2.4) + + '@radix-ui/react-separator@1.1.8(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/react-primitive': 2.1.4(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@radix-ui/react-slot@1.2.3(react@19.2.4)': + dependencies: + '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) + react: 19.2.4 + + '@radix-ui/react-slot@1.2.4(react@19.2.4)': + dependencies: + '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) + react: 19.2.4 + + '@radix-ui/react-tabs@1.1.13(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/primitive': 1.1.3 + '@radix-ui/react-context': 1.1.2(react@19.2.4) + '@radix-ui/react-direction': 1.1.1(react@19.2.4) + '@radix-ui/react-id': 1.1.1(react@19.2.4) + '@radix-ui/react-presence': 1.1.5(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-roving-focus': 1.1.11(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-use-controllable-state': 1.2.2(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@radix-ui/react-tooltip@1.2.8(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/primitive': 1.1.3 + '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) + '@radix-ui/react-context': 1.1.2(react@19.2.4) + '@radix-ui/react-dismissable-layer': 1.1.11(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-id': 1.1.1(react@19.2.4) + '@radix-ui/react-popper': 1.2.8(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-portal': 1.1.9(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-presence': 1.1.5(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-slot': 1.2.3(react@19.2.4) + '@radix-ui/react-use-controllable-state': 1.2.2(react@19.2.4) + '@radix-ui/react-visually-hidden': 1.2.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@radix-ui/react-use-callback-ref@1.1.1(react@19.2.4)': + dependencies: + react: 19.2.4 + + '@radix-ui/react-use-controllable-state@1.2.2(react@19.2.4)': + dependencies: + '@radix-ui/react-use-effect-event': 0.0.2(react@19.2.4) + '@radix-ui/react-use-layout-effect': 1.1.1(react@19.2.4) + react: 19.2.4 + + '@radix-ui/react-use-effect-event@0.0.2(react@19.2.4)': + dependencies: + '@radix-ui/react-use-layout-effect': 1.1.1(react@19.2.4) + react: 19.2.4 + + '@radix-ui/react-use-escape-keydown@1.1.1(react@19.2.4)': + dependencies: + '@radix-ui/react-use-callback-ref': 1.1.1(react@19.2.4) + react: 19.2.4 + + '@radix-ui/react-use-layout-effect@1.1.1(react@19.2.4)': + dependencies: + react: 19.2.4 + + '@radix-ui/react-use-previous@1.1.1(react@19.2.4)': + dependencies: + react: 19.2.4 + + '@radix-ui/react-use-rect@1.1.1(react@19.2.4)': + dependencies: + '@radix-ui/rect': 1.1.1 + react: 19.2.4 + + '@radix-ui/react-use-size@1.1.1(react@19.2.4)': + dependencies: + '@radix-ui/react-use-layout-effect': 1.1.1(react@19.2.4) + react: 19.2.4 + + '@radix-ui/react-visually-hidden@1.2.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@radix-ui/rect@1.1.1': {} + + '@redis/bloom@5.10.0(@redis/client@5.10.0)': + dependencies: + '@redis/client': 5.10.0 + + '@redis/client@5.10.0': + dependencies: + cluster-key-slot: 1.1.2 + + '@redis/json@5.10.0(@redis/client@5.10.0)': + dependencies: + '@redis/client': 5.10.0 + + '@redis/search@5.10.0(@redis/client@5.10.0)': + dependencies: + '@redis/client': 5.10.0 + + '@redis/time-series@5.10.0(@redis/client@5.10.0)': + dependencies: + '@redis/client': 5.10.0 + + '@rolldown/pluginutils@1.0.0-rc.3': {} + + '@rollup/rollup-android-arm-eabi@4.57.1': + optional: true + + '@rollup/rollup-android-arm64@4.57.1': + optional: true + + '@rollup/rollup-darwin-arm64@4.57.1': + optional: true + + '@rollup/rollup-darwin-x64@4.57.1': + optional: true + + '@rollup/rollup-freebsd-arm64@4.57.1': + optional: true + + '@rollup/rollup-freebsd-x64@4.57.1': + optional: true + + '@rollup/rollup-linux-arm-gnueabihf@4.57.1': + optional: true + + '@rollup/rollup-linux-arm-musleabihf@4.57.1': + optional: true + + '@rollup/rollup-linux-arm64-gnu@4.57.1': + optional: true + + '@rollup/rollup-linux-arm64-musl@4.57.1': + optional: true + + '@rollup/rollup-linux-loong64-gnu@4.57.1': + optional: true + + '@rollup/rollup-linux-loong64-musl@4.57.1': + optional: true + + '@rollup/rollup-linux-ppc64-gnu@4.57.1': + optional: true + + '@rollup/rollup-linux-ppc64-musl@4.57.1': + optional: true + + '@rollup/rollup-linux-riscv64-gnu@4.57.1': + optional: true + + '@rollup/rollup-linux-riscv64-musl@4.57.1': + optional: true + + '@rollup/rollup-linux-s390x-gnu@4.57.1': + optional: true + + '@rollup/rollup-linux-x64-gnu@4.57.1': + optional: true + + '@rollup/rollup-linux-x64-musl@4.57.1': + optional: true + + '@rollup/rollup-openbsd-x64@4.57.1': + optional: true + + '@rollup/rollup-openharmony-arm64@4.57.1': + optional: true + + '@rollup/rollup-win32-arm64-msvc@4.57.1': + optional: true + + '@rollup/rollup-win32-ia32-msvc@4.57.1': + optional: true + + '@rollup/rollup-win32-x64-gnu@4.57.1': + optional: true + + '@rollup/rollup-win32-x64-msvc@4.57.1': + optional: true + + '@standard-schema/spec@1.1.0': {} + + '@tailwindcss/node@4.1.18': + dependencies: + '@jridgewell/remapping': 2.3.5 + enhanced-resolve: 5.19.0 + jiti: 2.6.1 + lightningcss: 1.30.2 + magic-string: 0.30.21 + source-map-js: 1.2.1 + tailwindcss: 4.1.18 + + '@tailwindcss/oxide-android-arm64@4.1.18': + optional: true + + '@tailwindcss/oxide-darwin-arm64@4.1.18': + optional: true + + '@tailwindcss/oxide-darwin-x64@4.1.18': + optional: true + + '@tailwindcss/oxide-freebsd-x64@4.1.18': + optional: true + + '@tailwindcss/oxide-linux-arm-gnueabihf@4.1.18': + optional: true + + '@tailwindcss/oxide-linux-arm64-gnu@4.1.18': + optional: true + + '@tailwindcss/oxide-linux-arm64-musl@4.1.18': + optional: true + + '@tailwindcss/oxide-linux-x64-gnu@4.1.18': + optional: true + + '@tailwindcss/oxide-linux-x64-musl@4.1.18': + optional: true + + '@tailwindcss/oxide-wasm32-wasi@4.1.18': + optional: true + + '@tailwindcss/oxide-win32-arm64-msvc@4.1.18': + optional: true + + '@tailwindcss/oxide-win32-x64-msvc@4.1.18': + optional: true + + '@tailwindcss/oxide@4.1.18': + optionalDependencies: + '@tailwindcss/oxide-android-arm64': 4.1.18 + '@tailwindcss/oxide-darwin-arm64': 4.1.18 + '@tailwindcss/oxide-darwin-x64': 4.1.18 + '@tailwindcss/oxide-freebsd-x64': 4.1.18 + '@tailwindcss/oxide-linux-arm-gnueabihf': 4.1.18 + '@tailwindcss/oxide-linux-arm64-gnu': 4.1.18 + '@tailwindcss/oxide-linux-arm64-musl': 4.1.18 + '@tailwindcss/oxide-linux-x64-gnu': 4.1.18 + '@tailwindcss/oxide-linux-x64-musl': 4.1.18 + '@tailwindcss/oxide-wasm32-wasi': 4.1.18 + '@tailwindcss/oxide-win32-arm64-msvc': 4.1.18 + '@tailwindcss/oxide-win32-x64-msvc': 4.1.18 + + '@tailwindcss/postcss@4.1.18': + dependencies: + '@alloc/quick-lru': 5.2.0 + '@tailwindcss/node': 4.1.18 + '@tailwindcss/oxide': 4.1.18 + postcss: 8.5.6 + tailwindcss: 4.1.18 + + '@tanstack/query-core@5.90.20': {} + + '@tanstack/react-query@5.90.21(react@19.2.4)': + dependencies: + '@tanstack/query-core': 5.90.20 + react: 19.2.4 + + '@tanstack/react-virtual@3.13.18(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@tanstack/virtual-core': 3.13.18 + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@tanstack/virtual-core@3.13.18': {} + + '@tsconfig/node10@1.0.12': {} + + '@tsconfig/node12@1.0.11': {} + + '@tsconfig/node14@1.0.3': {} + + '@tsconfig/node16@1.0.4': {} + + '@types/babel__core@7.20.5': + dependencies: + '@babel/parser': 7.29.0 + '@babel/types': 7.29.0 + '@types/babel__generator': 7.27.0 + '@types/babel__template': 7.4.4 + '@types/babel__traverse': 7.28.0 + + '@types/babel__generator@7.27.0': + dependencies: + '@babel/types': 7.29.0 + + '@types/babel__template@7.4.4': + dependencies: + '@babel/parser': 7.29.0 + '@babel/types': 7.29.0 + + '@types/babel__traverse@7.28.0': + dependencies: + '@babel/types': 7.29.0 + + '@types/d3-color@3.1.3': {} + + '@types/d3-drag@3.0.7': + dependencies: + '@types/d3-selection': 3.0.11 + + '@types/d3-interpolate@3.0.4': + dependencies: + '@types/d3-color': 3.1.3 + + '@types/d3-selection@3.0.11': {} + + '@types/d3-transition@3.0.9': + dependencies: + '@types/d3-selection': 3.0.11 + + '@types/d3-zoom@3.0.8': + dependencies: + '@types/d3-interpolate': 3.0.4 + '@types/d3-selection': 3.0.11 + + '@types/estree@1.0.8': {} + + '@types/hast@3.0.4': + dependencies: + '@types/unist': 2.0.11 + + '@types/json-schema@7.0.15': {} + + '@types/luxon@3.7.1': {} + + '@types/node@25.2.3': + dependencies: + undici-types: 7.16.0 + + '@types/prismjs@1.26.6': {} + + '@types/trusted-types@2.0.7': + optional: true + + '@types/unist@2.0.11': {} + + '@types/yauzl@2.10.3': + dependencies: + '@types/node': 25.2.3 + optional: true + + '@types/zen-observable@0.8.3': {} + + '@typescript-eslint/eslint-plugin@8.55.0(@typescript-eslint/parser@8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3))(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)': + dependencies: + '@eslint-community/regexpp': 4.12.2 + '@typescript-eslint/parser': 8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3) + '@typescript-eslint/scope-manager': 8.55.0 + '@typescript-eslint/type-utils': 8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3) + '@typescript-eslint/utils': 8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3) + '@typescript-eslint/visitor-keys': 8.55.0 + eslint: 9.39.2(jiti@2.6.1) + ignore: 7.0.5 + natural-compare: 1.4.0 + ts-api-utils: 2.4.0(typescript@5.9.3) + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/parser@8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)': + dependencies: + '@typescript-eslint/scope-manager': 8.55.0 + '@typescript-eslint/types': 8.55.0 + '@typescript-eslint/typescript-estree': 8.55.0(typescript@5.9.3) + '@typescript-eslint/visitor-keys': 8.55.0 + debug: 4.4.3 + eslint: 9.39.2(jiti@2.6.1) + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/project-service@8.55.0(typescript@5.9.3)': + dependencies: + '@typescript-eslint/tsconfig-utils': 8.55.0(typescript@5.9.3) + '@typescript-eslint/types': 8.55.0 + debug: 4.4.3 + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/scope-manager@8.55.0': + dependencies: + '@typescript-eslint/types': 8.55.0 + '@typescript-eslint/visitor-keys': 8.55.0 + + '@typescript-eslint/tsconfig-utils@8.55.0(typescript@5.9.3)': + dependencies: + typescript: 5.9.3 + + '@typescript-eslint/type-utils@8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)': + dependencies: + '@typescript-eslint/types': 8.55.0 + '@typescript-eslint/typescript-estree': 8.55.0(typescript@5.9.3) + '@typescript-eslint/utils': 8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3) + debug: 4.4.3 + eslint: 9.39.2(jiti@2.6.1) + ts-api-utils: 2.4.0(typescript@5.9.3) + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/types@8.55.0': {} + + '@typescript-eslint/typescript-estree@8.55.0(typescript@5.9.3)': + dependencies: + '@typescript-eslint/project-service': 8.55.0(typescript@5.9.3) + '@typescript-eslint/tsconfig-utils': 8.55.0(typescript@5.9.3) + '@typescript-eslint/types': 8.55.0 + '@typescript-eslint/visitor-keys': 8.55.0 + debug: 4.4.3 + minimatch: 9.0.5 + semver: 7.7.4 + tinyglobby: 0.2.15 + ts-api-utils: 2.4.0(typescript@5.9.3) + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/utils@8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)': + dependencies: + '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.2(jiti@2.6.1)) + '@typescript-eslint/scope-manager': 8.55.0 + '@typescript-eslint/types': 8.55.0 + '@typescript-eslint/typescript-estree': 8.55.0(typescript@5.9.3) + eslint: 9.39.2(jiti@2.6.1) + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/visitor-keys@8.55.0': + dependencies: + '@typescript-eslint/types': 8.55.0 + eslint-visitor-keys: 4.2.1 + + '@vitejs/plugin-react@5.1.4(vite@7.3.1(@types/node@25.2.3)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0))': + dependencies: + '@babel/core': 7.29.0 + '@babel/plugin-transform-react-jsx-self': 7.27.1(@babel/core@7.29.0) + '@babel/plugin-transform-react-jsx-source': 7.27.1(@babel/core@7.29.0) + '@rolldown/pluginutils': 1.0.0-rc.3 + '@types/babel__core': 7.20.5 + react-refresh: 0.18.0 + vite: 7.3.1(@types/node@25.2.3)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0) + transitivePeerDependencies: + - supports-color + + '@xyflow/react@12.10.0(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@xyflow/system': 0.0.74 + classcat: 5.0.5 + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + zustand: 4.5.7(react@19.2.4) + transitivePeerDependencies: + - '@types/react' + - immer + + '@xyflow/system@0.0.74': + dependencies: + '@types/d3-drag': 3.0.7 + '@types/d3-interpolate': 3.0.4 + '@types/d3-selection': 3.0.11 + '@types/d3-transition': 3.0.9 + '@types/d3-zoom': 3.0.8 + d3-drag: 3.0.0 + d3-interpolate: 3.0.1 + d3-selection: 3.0.0 + d3-zoom: 3.0.0 + + abort-controller@3.0.0: + dependencies: + event-target-shim: 5.0.1 + + accepts@1.3.8: + dependencies: + mime-types: 2.1.35 + negotiator: 0.6.3 + + acorn-jsx@5.3.2(acorn@8.15.0): + dependencies: + acorn: 8.15.0 + + acorn-walk@8.3.4: + dependencies: + acorn: 8.15.0 + + acorn@8.15.0: {} + + agent-base@7.1.4: {} + + ajv@6.12.6: + dependencies: + fast-deep-equal: 3.1.3 + fast-json-stable-stringify: 2.1.0 + json-schema-traverse: 0.4.1 + uri-js: 4.4.1 + + ajv@8.17.1: + dependencies: + fast-deep-equal: 3.1.3 + fast-uri: 3.1.0 + json-schema-traverse: 1.0.0 + require-from-string: 2.0.2 + + ansi-escapes@4.3.2: + dependencies: + type-fest: 0.21.3 + + ansi-regex@5.0.1: {} + + ansi-regex@6.2.2: {} + + ansi-styles@4.3.0: + dependencies: + color-convert: 2.0.1 + + ansi-styles@6.2.3: {} + + antlr4ts@0.5.0-alpha.4: {} + + archiver-utils@5.0.2: + dependencies: + glob: 10.5.0 + graceful-fs: 4.2.11 + is-stream: 2.0.1 + lazystream: 1.0.1 + lodash: 4.17.23 + normalize-path: 3.0.0 + readable-stream: 4.7.0 + + archiver@7.0.1: + dependencies: + archiver-utils: 5.0.2 + async: 3.2.6 + buffer-crc32: 1.0.0 + readable-stream: 4.7.0 + readdir-glob: 1.1.3 + tar-stream: 3.1.7 + zip-stream: 6.0.1 + transitivePeerDependencies: + - bare-abort-controller + - react-native-b4a + + arg@4.1.3: {} + + argparse@2.0.1: {} + + aria-hidden@1.2.6: + dependencies: + tslib: 2.8.1 + + array-flatten@1.1.1: {} + + astral-regex@2.0.0: {} + + async@3.2.6: {} + + asynckit@0.4.0: {} + + autoprefixer@10.4.24(postcss@8.5.6): + dependencies: + browserslist: 4.28.1 + caniuse-lite: 1.0.30001769 + fraction.js: 5.3.4 + picocolors: 1.1.1 + postcss: 8.5.6 + postcss-value-parser: 4.2.0 + + axios@1.13.5: + dependencies: + follow-redirects: 1.15.11 + form-data: 4.0.5 + proxy-from-env: 1.1.0 + transitivePeerDependencies: + - debug + + b4a@1.7.4: {} + + babel-plugin-react-compiler@1.0.0: + dependencies: + '@babel/types': 7.29.0 + + balanced-match@1.0.2: {} + + balanced-match@4.0.2: + dependencies: + jackspeak: 4.2.3 + + bare-events@2.8.2: {} + + base64-js@1.5.1: {} + + baseline-browser-mapping@2.9.19: {} + + bl@4.1.0: + dependencies: + buffer: 5.7.1 + inherits: 2.0.4 + readable-stream: 3.6.2 + + body-parser@1.20.4: + dependencies: + bytes: 3.1.2 + content-type: 1.0.5 + debug: 2.6.9 + depd: 2.0.0 + destroy: 1.2.0 + http-errors: 2.0.1 + iconv-lite: 0.4.24 + on-finished: 2.4.1 + qs: 6.14.2 + raw-body: 2.5.3 + type-is: 1.6.18 + unpipe: 1.0.0 + transitivePeerDependencies: + - supports-color + + brace-expansion@1.1.12: + dependencies: + balanced-match: 1.0.2 + concat-map: 0.0.1 + + brace-expansion@2.0.2: + dependencies: + balanced-match: 1.0.2 + + brace-expansion@5.0.2: + dependencies: + balanced-match: 4.0.2 + + browserslist@4.28.1: + dependencies: + baseline-browser-mapping: 2.9.19 + caniuse-lite: 1.0.30001769 + electron-to-chromium: 1.5.286 + node-releases: 2.0.27 + update-browserslist-db: 1.2.3(browserslist@4.28.1) + + buffer-crc32@0.2.13: {} + + buffer-crc32@1.0.0: {} + + buffer@5.7.1: + dependencies: + base64-js: 1.5.1 + ieee754: 1.2.1 + + buffer@6.0.3: + dependencies: + base64-js: 1.5.1 + ieee754: 1.2.1 + + bullmq@5.69.1: + dependencies: + cron-parser: 4.9.0 + ioredis: 5.9.2 + msgpackr: 1.11.5 + node-abort-controller: 3.1.1 + semver: 7.7.4 + tslib: 2.8.1 + uuid: 11.1.0 + transitivePeerDependencies: + - supports-color + + bytes@3.1.2: {} + + call-bind-apply-helpers@1.0.2: + dependencies: + es-errors: 1.3.0 + function-bind: 1.1.2 + + call-bound@1.0.4: + dependencies: + call-bind-apply-helpers: 1.0.2 + get-intrinsic: 1.3.0 + + callsites@3.1.0: {} + + camelcase@6.3.0: {} + + caniuse-lite@1.0.30001769: {} + + chalk@4.1.2: + dependencies: + ansi-styles: 4.3.0 + supports-color: 7.2.0 + + character-entities-legacy@3.0.0: {} + + character-entities@2.0.2: {} + + character-reference-invalid@2.0.1: {} + + chardet@2.1.1: {} + + chokidar@4.0.3: + dependencies: + readdirp: 4.1.2 + + chownr@2.0.0: {} + + class-variance-authority@0.7.1: + dependencies: + clsx: 2.1.1 + + classcat@5.0.5: {} + + cli-cursor@3.1.0: + dependencies: + restore-cursor: 3.1.0 + + cli-spinners@2.9.2: {} + + cli-width@3.0.0: {} + + clone@1.0.4: {} + + clsx@2.1.1: {} + + cluster-key-slot@1.1.2: {} + + color-convert@2.0.1: + dependencies: + color-name: 1.1.4 + + color-name@1.1.4: {} + + combined-stream@1.0.8: + dependencies: + delayed-stream: 1.0.0 + + comma-separated-tokens@2.0.3: {} + + commander@13.1.0: {} + + commondir@1.0.1: {} + + compress-commons@6.0.2: + dependencies: + crc-32: 1.2.2 + crc32-stream: 6.0.0 + is-stream: 2.0.1 + normalize-path: 3.0.0 + readable-stream: 4.7.0 + + concat-map@0.0.1: {} + + content-disposition@0.5.4: + dependencies: + safe-buffer: 5.2.1 + + content-type@1.0.5: {} + + convert-source-map@2.0.0: {} + + cookie-signature@1.0.7: {} + + cookie@0.7.2: {} + + cookie@1.1.1: {} + + copy-to-clipboard@3.3.3: + dependencies: + toggle-selection: 1.0.6 + + core-util-is@1.0.3: {} + + crc-32@1.2.2: {} + + crc32-stream@6.0.0: + dependencies: + crc-32: 1.2.2 + readable-stream: 4.7.0 + + create-require@1.1.1: {} + + cron-parser@4.9.0: + dependencies: + luxon: 3.7.2 + + cron@4.4.0: + dependencies: + '@types/luxon': 3.7.1 + luxon: 3.7.2 + + cross-spawn@7.0.6: + dependencies: + path-key: 3.1.1 + shebang-command: 2.0.0 + which: 2.0.2 + + d3-color@3.1.0: {} + + d3-dispatch@3.0.1: {} + + d3-drag@3.0.0: + dependencies: + d3-dispatch: 3.0.1 + d3-selection: 3.0.0 + + d3-ease@3.0.1: {} + + d3-interpolate@3.0.1: + dependencies: + d3-color: 3.1.0 + + d3-selection@3.0.0: {} + + d3-timer@3.0.1: {} + + d3-transition@3.0.1(d3-selection@3.0.0): + dependencies: + d3-color: 3.1.0 + d3-dispatch: 3.0.1 + d3-ease: 3.0.1 + d3-interpolate: 3.0.1 + d3-selection: 3.0.0 + d3-timer: 3.0.1 + + d3-zoom@3.0.0: + dependencies: + d3-dispatch: 3.0.1 + d3-drag: 3.0.0 + d3-interpolate: 3.0.1 + d3-selection: 3.0.0 + d3-transition: 3.0.1(d3-selection@3.0.0) + + dagre@0.8.5: + dependencies: + graphlib: 2.1.8 + lodash: 4.17.23 + + date-fns@4.1.0: {} + + debug@2.6.9: + dependencies: + ms: 2.0.0 + + debug@4.4.3: + dependencies: + ms: 2.1.3 + + decode-named-character-reference@1.3.0: + dependencies: + character-entities: 2.0.2 + + deep-is@0.1.4: {} + + defaults@1.0.4: + dependencies: + clone: 1.0.4 + + delayed-stream@1.0.0: {} + + denque@2.1.0: {} + + depd@2.0.0: {} + + destroy@1.2.0: {} + + detect-libc@2.1.2: {} + + detect-node-es@1.1.0: {} + + diff@4.0.4: {} + + dompurify@3.2.7: + optionalDependencies: + '@types/trusted-types': 2.0.7 + + dotenv@16.6.1: {} + + dunder-proto@1.0.1: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-errors: 1.3.0 + gopd: 1.2.0 + + eastasianwidth@0.2.0: {} + + ee-first@1.1.1: {} + + electron-to-chromium@1.5.286: {} + + emoji-regex@8.0.0: {} + + emoji-regex@9.2.2: {} + + encodeurl@2.0.0: {} + + end-of-stream@1.4.5: + dependencies: + once: 1.4.0 + + enhanced-resolve@5.19.0: + dependencies: + graceful-fs: 4.2.11 + tapable: 2.3.0 + + es-define-property@1.0.1: {} + + es-errors@1.3.0: {} + + es-object-atoms@1.1.1: + dependencies: + es-errors: 1.3.0 + + es-set-tostringtag@2.1.0: + dependencies: + es-errors: 1.3.0 + get-intrinsic: 1.3.0 + has-tostringtag: 1.0.2 + hasown: 2.0.2 + + esbuild@0.25.12: + optionalDependencies: + '@esbuild/aix-ppc64': 0.25.12 + '@esbuild/android-arm': 0.25.12 + '@esbuild/android-arm64': 0.25.12 + '@esbuild/android-x64': 0.25.12 + '@esbuild/darwin-arm64': 0.25.12 + '@esbuild/darwin-x64': 0.25.12 + '@esbuild/freebsd-arm64': 0.25.12 + '@esbuild/freebsd-x64': 0.25.12 + '@esbuild/linux-arm': 0.25.12 + '@esbuild/linux-arm64': 0.25.12 + '@esbuild/linux-ia32': 0.25.12 + '@esbuild/linux-loong64': 0.25.12 + '@esbuild/linux-mips64el': 0.25.12 + '@esbuild/linux-ppc64': 0.25.12 + '@esbuild/linux-riscv64': 0.25.12 + '@esbuild/linux-s390x': 0.25.12 + '@esbuild/linux-x64': 0.25.12 + '@esbuild/netbsd-arm64': 0.25.12 + '@esbuild/netbsd-x64': 0.25.12 + '@esbuild/openbsd-arm64': 0.25.12 + '@esbuild/openbsd-x64': 0.25.12 + '@esbuild/openharmony-arm64': 0.25.12 + '@esbuild/sunos-x64': 0.25.12 + '@esbuild/win32-arm64': 0.25.12 + '@esbuild/win32-ia32': 0.25.12 + '@esbuild/win32-x64': 0.25.12 + + esbuild@0.27.3: + optionalDependencies: + '@esbuild/aix-ppc64': 0.27.3 + '@esbuild/android-arm': 0.27.3 + '@esbuild/android-arm64': 0.27.3 + '@esbuild/android-x64': 0.27.3 + '@esbuild/darwin-arm64': 0.27.3 + '@esbuild/darwin-x64': 0.27.3 + '@esbuild/freebsd-arm64': 0.27.3 + '@esbuild/freebsd-x64': 0.27.3 + '@esbuild/linux-arm': 0.27.3 + '@esbuild/linux-arm64': 0.27.3 + '@esbuild/linux-ia32': 0.27.3 + '@esbuild/linux-loong64': 0.27.3 + '@esbuild/linux-mips64el': 0.27.3 + '@esbuild/linux-ppc64': 0.27.3 + '@esbuild/linux-riscv64': 0.27.3 + '@esbuild/linux-s390x': 0.27.3 + '@esbuild/linux-x64': 0.27.3 + '@esbuild/netbsd-arm64': 0.27.3 + '@esbuild/netbsd-x64': 0.27.3 + '@esbuild/openbsd-arm64': 0.27.3 + '@esbuild/openbsd-x64': 0.27.3 + '@esbuild/openharmony-arm64': 0.27.3 + '@esbuild/sunos-x64': 0.27.3 + '@esbuild/win32-arm64': 0.27.3 + '@esbuild/win32-ia32': 0.27.3 + '@esbuild/win32-x64': 0.27.3 + + escalade@3.2.0: {} + + escape-html@1.0.3: {} + + escape-string-regexp@1.0.5: {} + + escape-string-regexp@4.0.0: {} + + eslint-scope@8.4.0: + dependencies: + esrecurse: 4.3.0 + estraverse: 5.3.0 + + eslint-visitor-keys@3.4.3: {} + + eslint-visitor-keys@4.2.1: {} + + eslint@9.39.2(jiti@2.6.1): + dependencies: + '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.2(jiti@2.6.1)) + '@eslint-community/regexpp': 4.12.2 + '@eslint/config-array': 0.21.1 + '@eslint/config-helpers': 0.4.2 + '@eslint/core': 0.17.0 + '@eslint/eslintrc': 3.3.3 + '@eslint/js': 9.39.2 + '@eslint/plugin-kit': 0.4.1 + '@humanfs/node': 0.16.7 + '@humanwhocodes/module-importer': 1.0.1 + '@humanwhocodes/retry': 0.4.3 + '@types/estree': 1.0.8 + ajv: 6.12.6 + chalk: 4.1.2 + cross-spawn: 7.0.6 + debug: 4.4.3 + escape-string-regexp: 4.0.0 + eslint-scope: 8.4.0 + eslint-visitor-keys: 4.2.1 + espree: 10.4.0 + esquery: 1.7.0 + esutils: 2.0.3 + fast-deep-equal: 3.1.3 + file-entry-cache: 8.0.0 + find-up: 5.0.0 + glob-parent: 6.0.2 + ignore: 5.3.2 + imurmurhash: 0.1.4 + is-glob: 4.0.3 + json-stable-stringify-without-jsonify: 1.0.1 + lodash.merge: 4.6.2 + minimatch: 3.1.2 + natural-compare: 1.4.0 + optionator: 0.9.4 + optionalDependencies: + jiti: 2.6.1 + transitivePeerDependencies: + - supports-color + + espree@10.4.0: + dependencies: + acorn: 8.15.0 + acorn-jsx: 5.3.2(acorn@8.15.0) + eslint-visitor-keys: 4.2.1 + + esquery@1.7.0: + dependencies: + estraverse: 5.3.0 + + esrecurse@4.3.0: + dependencies: + estraverse: 5.3.0 + + estraverse@5.3.0: {} + + esutils@2.0.3: {} + + etag@1.8.1: {} + + event-target-shim@5.0.1: {} + + events-universal@1.0.1: + dependencies: + bare-events: 2.8.2 + transitivePeerDependencies: + - bare-abort-controller + + events@3.3.0: {} + + express@4.22.1: + dependencies: + accepts: 1.3.8 + array-flatten: 1.1.1 + body-parser: 1.20.4 + content-disposition: 0.5.4 + content-type: 1.0.5 + cookie: 0.7.2 + cookie-signature: 1.0.7 + debug: 2.6.9 + depd: 2.0.0 + encodeurl: 2.0.0 + escape-html: 1.0.3 + etag: 1.8.1 + finalhandler: 1.3.2 + fresh: 0.5.2 + http-errors: 2.0.1 + merge-descriptors: 1.0.3 + methods: 1.1.2 + on-finished: 2.4.1 + parseurl: 1.3.3 + path-to-regexp: 0.1.12 + proxy-addr: 2.0.7 + qs: 6.14.2 + range-parser: 1.2.1 + safe-buffer: 5.2.1 + send: 0.19.2 + serve-static: 1.16.3 + setprototypeof: 1.2.0 + statuses: 2.0.2 + type-is: 1.6.18 + utils-merge: 1.0.1 + vary: 1.1.2 + transitivePeerDependencies: + - supports-color + + extract-zip@2.0.1: + dependencies: + debug: 4.4.3 + get-stream: 5.2.0 + yauzl: 2.10.0 + optionalDependencies: + '@types/yauzl': 2.10.3 + transitivePeerDependencies: + - supports-color + + fast-deep-equal@3.1.3: {} + + fast-fifo@1.3.2: {} + + fast-json-stable-stringify@2.1.0: {} + + fast-levenshtein@2.0.6: {} + + fast-uri@3.1.0: {} + + fault@1.0.4: + dependencies: + format: 0.2.2 + + fd-slicer@1.1.0: + dependencies: + pend: 1.2.0 + + fdir@6.5.0(picomatch@4.0.3): + optionalDependencies: + picomatch: 4.0.3 + + figures@3.2.0: + dependencies: + escape-string-regexp: 1.0.5 + + file-entry-cache@8.0.0: + dependencies: + flat-cache: 4.0.1 + + finalhandler@1.3.2: + dependencies: + debug: 2.6.9 + encodeurl: 2.0.0 + escape-html: 1.0.3 + on-finished: 2.4.1 + parseurl: 1.3.3 + statuses: 2.0.2 + unpipe: 1.0.0 + transitivePeerDependencies: + - supports-color + + find-cache-dir@3.3.2: + dependencies: + commondir: 1.0.1 + make-dir: 3.1.0 + pkg-dir: 4.2.0 + + find-package-json@1.2.0: {} + + find-up@4.1.0: + dependencies: + locate-path: 5.0.0 + path-exists: 4.0.0 + + find-up@5.0.0: + dependencies: + locate-path: 6.0.0 + path-exists: 4.0.0 + + flat-cache@4.0.1: + dependencies: + flatted: 3.3.3 + keyv: 4.5.4 + + flatted@3.3.3: {} + + follow-redirects@1.15.11: {} + + foreground-child@3.3.1: + dependencies: + cross-spawn: 7.0.6 + signal-exit: 4.1.0 + + form-data@4.0.5: + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + es-set-tostringtag: 2.1.0 + hasown: 2.0.2 + mime-types: 2.1.35 + + format@0.2.2: {} + + forwarded@0.2.0: {} + + fraction.js@5.3.4: {} + + framer-motion@12.34.0(react-dom@19.2.4(react@19.2.4))(react@19.2.4): + dependencies: + motion-dom: 12.34.0 + motion-utils: 12.29.2 + tslib: 2.8.1 + optionalDependencies: + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + fresh@0.5.2: {} + + fs-minipass@2.1.0: + dependencies: + minipass: 3.3.6 + + fsevents@2.3.3: + optional: true + + function-bind@1.1.2: {} + + gensync@1.0.0-beta.2: {} + + get-intrinsic@1.3.0: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-define-property: 1.0.1 + es-errors: 1.3.0 + es-object-atoms: 1.1.1 + function-bind: 1.1.2 + get-proto: 1.0.1 + gopd: 1.2.0 + has-symbols: 1.1.0 + hasown: 2.0.2 + math-intrinsics: 1.1.0 + + get-nonce@1.0.1: {} + + get-port@5.1.1: {} + + get-proto@1.0.1: + dependencies: + dunder-proto: 1.0.1 + es-object-atoms: 1.1.1 + + get-stream@5.2.0: + dependencies: + pump: 3.0.3 + + get-tsconfig@4.13.6: + dependencies: + resolve-pkg-maps: 1.0.0 + + glob-parent@6.0.2: + dependencies: + is-glob: 4.0.3 + + glob@10.5.0: + dependencies: + foreground-child: 3.3.1 + jackspeak: 3.4.3 + minimatch: 9.0.5 + minipass: 7.1.2 + package-json-from-dist: 1.0.1 + path-scurry: 1.11.1 + + glob@11.1.0: + dependencies: + foreground-child: 3.3.1 + jackspeak: 4.2.3 + minimatch: 10.2.0 + minipass: 7.1.2 + package-json-from-dist: 1.0.1 + path-scurry: 2.0.1 + + globals@14.0.0: {} + + gopd@1.2.0: {} + + graceful-fs@4.2.11: {} + + graphlib@2.1.8: + dependencies: + lodash: 4.17.23 + + has-flag@4.0.0: {} + + has-symbols@1.1.0: {} + + has-tostringtag@1.0.2: + dependencies: + has-symbols: 1.1.0 + + hasown@2.0.2: + dependencies: + function-bind: 1.1.2 + + hast-util-parse-selector@4.0.0: + dependencies: + '@types/hast': 3.0.4 + + hastscript@9.0.1: + dependencies: + '@types/hast': 3.0.4 + comma-separated-tokens: 2.0.3 + hast-util-parse-selector: 4.0.0 + property-information: 7.1.0 + space-separated-tokens: 2.0.2 + + highlight.js@10.7.3: {} + + highlightjs-vue@1.0.0: {} + + http-errors@2.0.1: + dependencies: + depd: 2.0.0 + inherits: 2.0.4 + setprototypeof: 1.2.0 + statuses: 2.0.2 + toidentifier: 1.0.1 + + https-proxy-agent@7.0.6: + dependencies: + agent-base: 7.1.4 + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + + iconv-lite@0.4.24: + dependencies: + safer-buffer: 2.1.2 + + iconv-lite@0.7.2: + dependencies: + safer-buffer: 2.1.2 + + ieee754@1.2.1: {} + + ignore@5.3.2: {} + + ignore@7.0.5: {} + + import-fresh@3.3.1: + dependencies: + parent-module: 1.0.1 + resolve-from: 4.0.0 + + imurmurhash@0.1.4: {} + + inherits@2.0.4: {} + + inquirer@8.2.7(@types/node@25.2.3): + dependencies: + '@inquirer/external-editor': 1.0.3(@types/node@25.2.3) + ansi-escapes: 4.3.2 + chalk: 4.1.2 + cli-cursor: 3.1.0 + cli-width: 3.0.0 + figures: 3.2.0 + lodash: 4.17.23 + mute-stream: 0.0.8 + ora: 5.4.1 + run-async: 2.4.1 + rxjs: 7.8.2 + string-width: 4.2.3 + strip-ansi: 6.0.1 + through: 2.3.8 + wrap-ansi: 6.2.0 + transitivePeerDependencies: + - '@types/node' + + ioredis@5.9.2: + dependencies: + '@ioredis/commands': 1.5.0 + cluster-key-slot: 1.1.2 + debug: 4.4.3 + denque: 2.1.0 + lodash.defaults: 4.2.0 + lodash.isarguments: 3.1.0 + redis-errors: 1.2.0 + redis-parser: 3.0.0 + standard-as-callback: 2.1.0 + transitivePeerDependencies: + - supports-color + + ioredis@5.9.3: + dependencies: + '@ioredis/commands': 1.5.0 + cluster-key-slot: 1.1.2 + debug: 4.4.3 + denque: 2.1.0 + lodash.defaults: 4.2.0 + lodash.isarguments: 3.1.0 + redis-errors: 1.2.0 + redis-parser: 3.0.0 + standard-as-callback: 2.1.0 + transitivePeerDependencies: + - supports-color + + ipaddr.js@1.9.1: {} + + is-alphabetical@2.0.1: {} + + is-alphanumerical@2.0.1: + dependencies: + is-alphabetical: 2.0.1 + is-decimal: 2.0.1 + + is-decimal@2.0.1: {} + + is-extglob@2.1.1: {} + + is-fullwidth-code-point@3.0.0: {} + + is-glob@4.0.3: + dependencies: + is-extglob: 2.1.1 + + is-hexadecimal@2.0.1: {} + + is-interactive@1.0.0: {} + + is-stream@2.0.1: {} + + is-unicode-supported@0.1.0: {} + + isarray@1.0.0: {} + + isexe@2.0.0: {} + + jackspeak@3.4.3: + dependencies: + '@isaacs/cliui': 8.0.2 + optionalDependencies: + '@pkgjs/parseargs': 0.11.0 + + jackspeak@4.2.3: + dependencies: + '@isaacs/cliui': 9.0.0 + + jiti@2.6.1: {} + + js-tokens@4.0.0: {} + + js-yaml@4.1.1: + dependencies: + argparse: 2.0.1 + + jsesc@3.1.0: {} + + json-buffer@3.0.1: {} + + json-schema-traverse@0.4.1: {} + + json-schema-traverse@1.0.0: {} + + json-schema@0.4.0: {} + + json-stable-stringify-without-jsonify@1.0.1: {} + + json5@2.2.3: {} + + keyv@4.5.4: + dependencies: + json-buffer: 3.0.1 + + lazystream@1.0.1: + dependencies: + readable-stream: 2.3.8 + + levn@0.4.1: + dependencies: + prelude-ls: 1.2.1 + type-check: 0.4.0 + + lightningcss-android-arm64@1.30.2: + optional: true + + lightningcss-darwin-arm64@1.30.2: + optional: true + + lightningcss-darwin-x64@1.30.2: + optional: true + + lightningcss-freebsd-x64@1.30.2: + optional: true + + lightningcss-linux-arm-gnueabihf@1.30.2: + optional: true + + lightningcss-linux-arm64-gnu@1.30.2: + optional: true + + lightningcss-linux-arm64-musl@1.30.2: + optional: true + + lightningcss-linux-x64-gnu@1.30.2: + optional: true + + lightningcss-linux-x64-musl@1.30.2: + optional: true + + lightningcss-win32-arm64-msvc@1.30.2: + optional: true + + lightningcss-win32-x64-msvc@1.30.2: + optional: true + + lightningcss@1.30.2: + dependencies: + detect-libc: 2.1.2 + optionalDependencies: + lightningcss-android-arm64: 1.30.2 + lightningcss-darwin-arm64: 1.30.2 + lightningcss-darwin-x64: 1.30.2 + lightningcss-freebsd-x64: 1.30.2 + lightningcss-linux-arm-gnueabihf: 1.30.2 + lightningcss-linux-arm64-gnu: 1.30.2 + lightningcss-linux-arm64-musl: 1.30.2 + lightningcss-linux-x64-gnu: 1.30.2 + lightningcss-linux-x64-musl: 1.30.2 + lightningcss-win32-arm64-msvc: 1.30.2 + lightningcss-win32-x64-msvc: 1.30.2 + + locate-path@5.0.0: + dependencies: + p-locate: 4.1.0 + + locate-path@6.0.0: + dependencies: + p-locate: 5.0.0 + + lockfile@1.0.4: + dependencies: + signal-exit: 3.0.7 + + lodash.defaults@4.2.0: {} + + lodash.defaultsdeep@4.6.1: {} + + lodash.isarguments@3.1.0: {} + + lodash.merge@4.6.2: {} + + lodash.truncate@4.4.2: {} + + lodash@4.17.23: {} + + log-symbols@4.1.0: + dependencies: + chalk: 4.1.2 + is-unicode-supported: 0.1.0 + + lowlight@1.20.0: + dependencies: + fault: 1.0.4 + highlight.js: 10.7.3 + + lru-cache@10.4.3: {} + + lru-cache@11.2.6: {} + + lru-cache@5.1.1: + dependencies: + yallist: 3.1.1 + + lucide-react@0.555.0(react@19.2.4): + dependencies: + react: 19.2.4 + + luxon@3.7.2: {} + + magic-string@0.30.21: + dependencies: + '@jridgewell/sourcemap-codec': 1.5.5 + + make-dir@3.1.0: + dependencies: + semver: 6.3.1 + + make-error@1.3.6: {} + + marked@14.0.0: {} + + math-intrinsics@1.1.0: {} + + media-typer@0.3.0: {} + + merge-descriptors@1.0.3: {} + + methods@1.1.2: {} + + mime-db@1.52.0: {} + + mime-types@2.1.35: + dependencies: + mime-db: 1.52.0 + + mime@1.6.0: {} + + mimic-fn@2.1.0: {} + + minimatch@10.2.0: + dependencies: + brace-expansion: 5.0.2 + + minimatch@3.1.2: + dependencies: + brace-expansion: 1.1.12 + + minimatch@5.1.6: + dependencies: + brace-expansion: 2.0.2 + + minimatch@9.0.5: + dependencies: + brace-expansion: 2.0.2 + + minimist@1.2.8: {} + + minipass@3.3.6: + dependencies: + yallist: 4.0.0 + + minipass@5.0.0: {} + + minipass@7.1.2: {} + + minizlib@2.1.2: + dependencies: + minipass: 3.3.6 + yallist: 4.0.0 + + mkdirp@1.0.4: {} + + monaco-editor@0.55.1: + dependencies: + dompurify: 3.2.7 + marked: 14.0.0 + + motia@0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)): + dependencies: + '@amplitude/analytics-node': 1.5.35 + '@motiadev/adapter-bullmq-events': 0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3)) + '@motiadev/adapter-redis-cron': 0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3)) + '@motiadev/adapter-redis-state': 0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3)) + '@motiadev/adapter-redis-streams': 0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3)) + '@motiadev/core': 0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3) + '@motiadev/stream-client-node': 0.17.14-beta.196 + '@motiadev/workbench': 0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(jiti@2.6.1)(lightningcss@1.30.2)(monaco-editor@0.55.1)(redis@5.10.0)(tsx@4.21.0)(use-sync-external-store@1.6.0(react@19.2.4)) + antlr4ts: 0.5.0-alpha.4 + archiver: 7.0.1 + axios: 1.13.5 + chokidar: 4.0.3 + commander: 13.1.0 + cron: 4.4.0 + dotenv: 16.6.1 + esbuild: 0.25.12 + express: 4.22.1 + glob: 11.1.0 + inquirer: 8.2.7(@types/node@25.2.3) + jiti: 2.6.1 + node-cron: 3.0.3 + picocolors: 1.1.1 + python-ast: 0.1.0 + redis: 5.10.0 + redis-memory-server: 0.14.1 + table: 6.9.0 + ts-node: 10.9.2(@types/node@25.2.3)(typescript@5.9.3) + tsx: 4.21.0 + zod: 4.3.6 + transitivePeerDependencies: + - '@swc/core' + - '@swc/wasm' + - '@types/node' + - '@types/react' + - '@types/react-dom' + - bare-abort-controller + - bufferutil + - debug + - eslint + - immer + - less + - lightningcss + - monaco-editor + - react-native-b4a + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + - typescript + - use-sync-external-store + - utf-8-validate + - yaml + + motion-dom@12.34.0: + dependencies: + motion-utils: 12.29.2 + + motion-utils@12.29.2: {} + + motion@12.34.0(react-dom@19.2.4(react@19.2.4))(react@19.2.4): + dependencies: + framer-motion: 12.34.0(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + tslib: 2.8.1 + optionalDependencies: + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + ms@2.0.0: {} + + ms@2.1.3: {} + + msgpackr-extract@3.0.3: + dependencies: + node-gyp-build-optional-packages: 5.2.2 + optionalDependencies: + '@msgpackr-extract/msgpackr-extract-darwin-arm64': 3.0.3 + '@msgpackr-extract/msgpackr-extract-darwin-x64': 3.0.3 + '@msgpackr-extract/msgpackr-extract-linux-arm': 3.0.3 + '@msgpackr-extract/msgpackr-extract-linux-arm64': 3.0.3 + '@msgpackr-extract/msgpackr-extract-linux-x64': 3.0.3 + '@msgpackr-extract/msgpackr-extract-win32-x64': 3.0.3 + optional: true + + msgpackr@1.11.5: + optionalDependencies: + msgpackr-extract: 3.0.3 + + mute-stream@0.0.8: {} + + nanoid@3.3.11: {} + + natural-compare@1.4.0: {} + + negotiator@0.6.3: {} + + node-abort-controller@3.1.1: {} + + node-cron@3.0.3: + dependencies: + uuid: 8.3.2 + + node-gyp-build-optional-packages@5.2.2: + dependencies: + detect-libc: 2.1.2 + optional: true + + node-releases@2.0.27: {} + + normalize-path@3.0.0: {} + + object-inspect@1.13.4: {} + + on-finished@2.4.1: + dependencies: + ee-first: 1.1.1 + + once@1.4.0: + dependencies: + wrappy: 1.0.2 + + onetime@5.1.2: + dependencies: + mimic-fn: 2.1.0 + + optionator@0.9.4: + dependencies: + deep-is: 0.1.4 + fast-levenshtein: 2.0.6 + levn: 0.4.1 + prelude-ls: 1.2.1 + type-check: 0.4.0 + word-wrap: 1.2.5 + + ora@5.4.1: + dependencies: + bl: 4.1.0 + chalk: 4.1.2 + cli-cursor: 3.1.0 + cli-spinners: 2.9.2 + is-interactive: 1.0.0 + is-unicode-supported: 0.1.0 + log-symbols: 4.1.0 + strip-ansi: 6.0.1 + wcwidth: 1.0.1 + + p-limit@2.3.0: + dependencies: + p-try: 2.2.0 + + p-limit@3.1.0: + dependencies: + yocto-queue: 0.1.0 + + p-locate@4.1.0: + dependencies: + p-limit: 2.3.0 + + p-locate@5.0.0: + dependencies: + p-limit: 3.1.0 + + p-try@2.2.0: {} + + package-json-from-dist@1.0.1: {} + + parent-module@1.0.1: + dependencies: + callsites: 3.1.0 + + parse-entities@4.0.2: + dependencies: + '@types/unist': 2.0.11 + character-entities-legacy: 3.0.0 + character-reference-invalid: 2.0.1 + decode-named-character-reference: 1.3.0 + is-alphanumerical: 2.0.1 + is-decimal: 2.0.1 + is-hexadecimal: 2.0.1 + + parseurl@1.3.3: {} + + path-exists@4.0.0: {} + + path-key@3.1.1: {} + + path-scurry@1.11.1: + dependencies: + lru-cache: 10.4.3 + minipass: 7.1.2 + + path-scurry@2.0.1: + dependencies: + lru-cache: 11.2.6 + minipass: 7.1.2 + + path-to-regexp@0.1.12: {} + + pend@1.2.0: {} + + picocolors@1.1.1: {} + + picomatch@4.0.3: {} + + pkg-dir@4.2.0: + dependencies: + find-up: 4.1.0 + + postcss-value-parser@4.2.0: {} + + postcss@8.5.6: + dependencies: + nanoid: 3.3.11 + picocolors: 1.1.1 + source-map-js: 1.2.1 + + prelude-ls@1.2.1: {} + + prismjs@1.30.0: {} + + process-nextick-args@2.0.1: {} + + process@0.11.10: {} + + property-information@7.1.0: {} + + proxy-addr@2.0.7: + dependencies: + forwarded: 0.2.0 + ipaddr.js: 1.9.1 + + proxy-from-env@1.1.0: {} + + pump@3.0.3: + dependencies: + end-of-stream: 1.4.5 + once: 1.4.0 + + punycode@2.3.1: {} + + python-ast@0.1.0: + dependencies: + antlr4ts: 0.5.0-alpha.4 + + qs@6.14.2: + dependencies: + side-channel: 1.1.0 + + range-parser@1.2.1: {} + + raw-body@2.5.3: + dependencies: + bytes: 3.1.2 + http-errors: 2.0.1 + iconv-lite: 0.4.24 + unpipe: 1.0.0 + + react-dom@19.2.4(react@19.2.4): + dependencies: + react: 19.2.4 + scheduler: 0.27.0 + + react-refresh@0.18.0: {} + + react-remove-scroll-bar@2.3.8(react@19.2.4): + dependencies: + react: 19.2.4 + react-style-singleton: 2.2.3(react@19.2.4) + tslib: 2.8.1 + + react-remove-scroll@2.7.2(react@19.2.4): + dependencies: + react: 19.2.4 + react-remove-scroll-bar: 2.3.8(react@19.2.4) + react-style-singleton: 2.2.3(react@19.2.4) + tslib: 2.8.1 + use-callback-ref: 1.3.3(react@19.2.4) + use-sidecar: 1.1.3(react@19.2.4) + + react-resizable-panels@3.0.6(react-dom@19.2.4(react@19.2.4))(react@19.2.4): + dependencies: + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + react-router-dom@7.13.0(react-dom@19.2.4(react@19.2.4))(react@19.2.4): + dependencies: + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + react-router: 7.13.0(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + + react-router@7.13.0(react-dom@19.2.4(react@19.2.4))(react@19.2.4): + dependencies: + cookie: 1.1.1 + react: 19.2.4 + set-cookie-parser: 2.7.2 + optionalDependencies: + react-dom: 19.2.4(react@19.2.4) + + react-style-singleton@2.2.3(react@19.2.4): + dependencies: + get-nonce: 1.0.1 + react: 19.2.4 + tslib: 2.8.1 + + react-syntax-highlighter@16.1.0(react@19.2.4): + dependencies: + '@babel/runtime': 7.28.6 + highlight.js: 10.7.3 + highlightjs-vue: 1.0.0 + lowlight: 1.20.0 + prismjs: 1.30.0 + react: 19.2.4 + refractor: 5.0.0 + + react-use-resizable@0.2.0(react@19.2.4): + dependencies: + react: 19.2.4 + + react18-json-view@0.2.9(react@19.2.4): + dependencies: + copy-to-clipboard: 3.3.3 + react: 19.2.4 + + react@19.2.4: {} + + readable-stream@2.3.8: + dependencies: + core-util-is: 1.0.3 + inherits: 2.0.4 + isarray: 1.0.0 + process-nextick-args: 2.0.1 + safe-buffer: 5.1.2 + string_decoder: 1.1.1 + util-deprecate: 1.0.2 + + readable-stream@3.6.2: + dependencies: + inherits: 2.0.4 + string_decoder: 1.3.0 + util-deprecate: 1.0.2 + + readable-stream@4.7.0: + dependencies: + abort-controller: 3.0.0 + buffer: 6.0.3 + events: 3.3.0 + process: 0.11.10 + string_decoder: 1.3.0 + + readdir-glob@1.1.3: + dependencies: + minimatch: 5.1.6 + + readdirp@4.1.2: {} + + redis-errors@1.2.0: {} + + redis-memory-server@0.14.1: + dependencies: + camelcase: 6.3.0 + cross-spawn: 7.0.6 + debug: 4.4.3 + extract-zip: 2.0.1 + find-cache-dir: 3.3.2 + find-package-json: 1.2.0 + get-port: 5.1.1 + https-proxy-agent: 7.0.6 + lockfile: 1.0.4 + lodash.defaultsdeep: 4.6.1 + rimraf: 5.0.10 + semver: 7.7.4 + tar: 6.2.1 + tmp: 0.2.5 + uuid: 8.3.2 + transitivePeerDependencies: + - supports-color + + redis-parser@3.0.0: + dependencies: + redis-errors: 1.2.0 + + redis@5.10.0: + dependencies: + '@redis/bloom': 5.10.0(@redis/client@5.10.0) + '@redis/client': 5.10.0 + '@redis/json': 5.10.0(@redis/client@5.10.0) + '@redis/search': 5.10.0(@redis/client@5.10.0) + '@redis/time-series': 5.10.0(@redis/client@5.10.0) + + refractor@5.0.0: + dependencies: + '@types/hast': 3.0.4 + '@types/prismjs': 1.26.6 + hastscript: 9.0.1 + parse-entities: 4.0.2 + + require-from-string@2.0.2: {} + + resolve-from@4.0.0: {} + + resolve-pkg-maps@1.0.0: {} + + restore-cursor@3.1.0: + dependencies: + onetime: 5.1.2 + signal-exit: 3.0.7 + + rimraf@5.0.10: + dependencies: + glob: 10.5.0 + + rollup@4.57.1: + dependencies: + '@types/estree': 1.0.8 + optionalDependencies: + '@rollup/rollup-android-arm-eabi': 4.57.1 + '@rollup/rollup-android-arm64': 4.57.1 + '@rollup/rollup-darwin-arm64': 4.57.1 + '@rollup/rollup-darwin-x64': 4.57.1 + '@rollup/rollup-freebsd-arm64': 4.57.1 + '@rollup/rollup-freebsd-x64': 4.57.1 + '@rollup/rollup-linux-arm-gnueabihf': 4.57.1 + '@rollup/rollup-linux-arm-musleabihf': 4.57.1 + '@rollup/rollup-linux-arm64-gnu': 4.57.1 + '@rollup/rollup-linux-arm64-musl': 4.57.1 + '@rollup/rollup-linux-loong64-gnu': 4.57.1 + '@rollup/rollup-linux-loong64-musl': 4.57.1 + '@rollup/rollup-linux-ppc64-gnu': 4.57.1 + '@rollup/rollup-linux-ppc64-musl': 4.57.1 + '@rollup/rollup-linux-riscv64-gnu': 4.57.1 + '@rollup/rollup-linux-riscv64-musl': 4.57.1 + '@rollup/rollup-linux-s390x-gnu': 4.57.1 + '@rollup/rollup-linux-x64-gnu': 4.57.1 + '@rollup/rollup-linux-x64-musl': 4.57.1 + '@rollup/rollup-openbsd-x64': 4.57.1 + '@rollup/rollup-openharmony-arm64': 4.57.1 + '@rollup/rollup-win32-arm64-msvc': 4.57.1 + '@rollup/rollup-win32-ia32-msvc': 4.57.1 + '@rollup/rollup-win32-x64-gnu': 4.57.1 + '@rollup/rollup-win32-x64-msvc': 4.57.1 + fsevents: 2.3.3 + + run-async@2.4.1: {} + + rxjs@7.8.2: + dependencies: + tslib: 2.8.1 + + safe-buffer@5.1.2: {} + + safe-buffer@5.2.1: {} + + safer-buffer@2.1.2: {} + + scheduler@0.27.0: {} + + semver@6.3.1: {} + + semver@7.7.4: {} + + send@0.19.2: + dependencies: + debug: 2.6.9 + depd: 2.0.0 + destroy: 1.2.0 + encodeurl: 2.0.0 + escape-html: 1.0.3 + etag: 1.8.1 + fresh: 0.5.2 + http-errors: 2.0.1 + mime: 1.6.0 + ms: 2.1.3 + on-finished: 2.4.1 + range-parser: 1.2.1 + statuses: 2.0.2 + transitivePeerDependencies: + - supports-color + + serve-static@1.16.3: + dependencies: + encodeurl: 2.0.0 + escape-html: 1.0.3 + parseurl: 1.3.3 + send: 0.19.2 + transitivePeerDependencies: + - supports-color + + set-cookie-parser@2.7.2: {} + + setprototypeof@1.2.0: {} + + shebang-command@2.0.0: + dependencies: + shebang-regex: 3.0.0 + + shebang-regex@3.0.0: {} + + side-channel-list@1.0.0: + dependencies: + es-errors: 1.3.0 + object-inspect: 1.13.4 + + side-channel-map@1.0.1: + dependencies: + call-bound: 1.0.4 + es-errors: 1.3.0 + get-intrinsic: 1.3.0 + object-inspect: 1.13.4 + + side-channel-weakmap@1.0.2: + dependencies: + call-bound: 1.0.4 + es-errors: 1.3.0 + get-intrinsic: 1.3.0 + object-inspect: 1.13.4 + side-channel-map: 1.0.1 + + side-channel@1.1.0: + dependencies: + es-errors: 1.3.0 + object-inspect: 1.13.4 + side-channel-list: 1.0.0 + side-channel-map: 1.0.1 + side-channel-weakmap: 1.0.2 + + signal-exit@3.0.7: {} + + signal-exit@4.1.0: {} + + slice-ansi@4.0.0: + dependencies: + ansi-styles: 4.3.0 + astral-regex: 2.0.0 + is-fullwidth-code-point: 3.0.0 + + source-map-js@1.2.1: {} + + space-separated-tokens@2.0.2: {} + + standard-as-callback@2.1.0: {} + + state-local@1.0.7: {} + + statuses@2.0.2: {} + + streamx@2.23.0: + dependencies: + events-universal: 1.0.1 + fast-fifo: 1.3.2 + text-decoder: 1.2.4 + transitivePeerDependencies: + - bare-abort-controller + - react-native-b4a + + string-width@4.2.3: + dependencies: + emoji-regex: 8.0.0 + is-fullwidth-code-point: 3.0.0 + strip-ansi: 6.0.1 + + string-width@5.1.2: + dependencies: + eastasianwidth: 0.2.0 + emoji-regex: 9.2.2 + strip-ansi: 7.1.2 + + string_decoder@1.1.1: + dependencies: + safe-buffer: 5.1.2 + + string_decoder@1.3.0: + dependencies: + safe-buffer: 5.2.1 + + strip-ansi@6.0.1: + dependencies: + ansi-regex: 5.0.1 + + strip-ansi@7.1.2: + dependencies: + ansi-regex: 6.2.2 + + strip-bom@3.0.0: {} + + strip-json-comments@3.1.1: {} + + supports-color@7.2.0: + dependencies: + has-flag: 4.0.0 + + table@6.9.0: + dependencies: + ajv: 8.17.1 + lodash.truncate: 4.4.2 + slice-ansi: 4.0.0 + string-width: 4.2.3 + strip-ansi: 6.0.1 + + tailwind-merge@2.6.1: {} + + tailwind-merge@3.4.0: {} + + tailwindcss@4.1.18: {} + + tapable@2.3.0: {} + + tar-stream@3.1.7: + dependencies: + b4a: 1.7.4 + fast-fifo: 1.3.2 + streamx: 2.23.0 + transitivePeerDependencies: + - bare-abort-controller + - react-native-b4a + + tar@6.2.1: + dependencies: + chownr: 2.0.0 + fs-minipass: 2.1.0 + minipass: 5.0.0 + minizlib: 2.1.2 + mkdirp: 1.0.4 + yallist: 4.0.0 + + text-decoder@1.2.4: + dependencies: + b4a: 1.7.4 + transitivePeerDependencies: + - react-native-b4a + + through@2.3.8: {} + + tinyglobby@0.2.15: + dependencies: + fdir: 6.5.0(picomatch@4.0.3) + picomatch: 4.0.3 + + tmp@0.2.5: {} + + toggle-selection@1.0.6: {} + + toidentifier@1.0.1: {} + + ts-api-utils@2.4.0(typescript@5.9.3): + dependencies: + typescript: 5.9.3 + + ts-node@10.9.2(@types/node@25.2.3)(typescript@5.9.3): + dependencies: + '@cspotcode/source-map-support': 0.8.1 + '@tsconfig/node10': 1.0.12 + '@tsconfig/node12': 1.0.11 + '@tsconfig/node14': 1.0.3 + '@tsconfig/node16': 1.0.4 + '@types/node': 25.2.3 + acorn: 8.15.0 + acorn-walk: 8.3.4 + arg: 4.1.3 + create-require: 1.1.1 + diff: 4.0.4 + make-error: 1.3.6 + typescript: 5.9.3 + v8-compile-cache-lib: 3.0.1 + yn: 3.1.1 + + tsconfig-paths@4.2.0: + dependencies: + json5: 2.2.3 + minimist: 1.2.8 + strip-bom: 3.0.0 + + tslib@2.8.1: {} + + tsx@4.21.0: + dependencies: + esbuild: 0.27.3 + get-tsconfig: 4.13.6 + optionalDependencies: + fsevents: 2.3.3 + + tw-animate-css@1.4.0: {} + + type-check@0.4.0: + dependencies: + prelude-ls: 1.2.1 + + type-fest@0.21.3: {} + + type-is@1.6.18: + dependencies: + media-typer: 0.3.0 + mime-types: 2.1.35 + + typescript-eslint@8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3): + dependencies: + '@typescript-eslint/eslint-plugin': 8.55.0(@typescript-eslint/parser@8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3))(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3) + '@typescript-eslint/parser': 8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3) + '@typescript-eslint/typescript-estree': 8.55.0(typescript@5.9.3) + '@typescript-eslint/utils': 8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3) + eslint: 9.39.2(jiti@2.6.1) + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + + typescript@5.9.3: {} + + undici-types@7.16.0: {} + + unpipe@1.0.0: {} + + update-browserslist-db@1.2.3(browserslist@4.28.1): + dependencies: + browserslist: 4.28.1 + escalade: 3.2.0 + picocolors: 1.1.1 + + uri-js@4.4.1: + dependencies: + punycode: 2.3.1 + + use-callback-ref@1.3.3(react@19.2.4): + dependencies: + react: 19.2.4 + tslib: 2.8.1 + + use-sidecar@1.1.3(react@19.2.4): + dependencies: + detect-node-es: 1.1.0 + react: 19.2.4 + tslib: 2.8.1 + + use-sync-external-store@1.6.0(react@19.2.4): + dependencies: + react: 19.2.4 + + util-deprecate@1.0.2: {} + + utils-merge@1.0.1: {} + + uuid@11.1.0: {} + + uuid@13.0.0: {} + + uuid@8.3.2: {} + + v8-compile-cache-lib@3.0.1: {} + + vary@1.1.2: {} + + vite@7.3.1(@types/node@25.2.3)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0): + dependencies: + esbuild: 0.27.3 + fdir: 6.5.0(picomatch@4.0.3) + picomatch: 4.0.3 + postcss: 8.5.6 + rollup: 4.57.1 + tinyglobby: 0.2.15 + optionalDependencies: + '@types/node': 25.2.3 + fsevents: 2.3.3 + jiti: 2.6.1 + lightningcss: 1.30.2 + tsx: 4.21.0 + + wcwidth@1.0.1: + dependencies: + defaults: 1.0.4 + + which@2.0.2: + dependencies: + isexe: 2.0.0 + + word-wrap@1.2.5: {} + + wrap-ansi@6.2.0: + dependencies: + ansi-styles: 4.3.0 + string-width: 4.2.3 + strip-ansi: 6.0.1 + + wrap-ansi@7.0.0: + dependencies: + ansi-styles: 4.3.0 + string-width: 4.2.3 + strip-ansi: 6.0.1 + + wrap-ansi@8.1.0: + dependencies: + ansi-styles: 6.2.3 + string-width: 5.1.2 + strip-ansi: 7.1.2 + + wrappy@1.0.2: {} + + ws@8.19.0: {} + + yallist@3.1.1: {} + + yallist@4.0.0: {} + + yauzl@2.10.0: + dependencies: + buffer-crc32: 0.2.13 + fd-slicer: 1.1.0 + + yn@3.1.1: {} + + yocto-queue@0.1.0: {} + + zen-observable-ts@1.1.0: + dependencies: + '@types/zen-observable': 0.8.3 + zen-observable: 0.8.15 + + zen-observable@0.8.15: {} + + zip-stream@6.0.1: + dependencies: + archiver-utils: 5.0.2 + compress-commons: 6.0.2 + readable-stream: 4.7.0 + + zod@3.25.76: {} + + zod@4.3.6: {} + + zustand@4.5.7(react@19.2.4): + dependencies: + use-sync-external-store: 1.6.0(react@19.2.4) + optionalDependencies: + react: 19.2.4 + + zustand@5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)): + optionalDependencies: + react: 19.2.4 + use-sync-external-store: 1.6.0(react@19.2.4) diff --git a/bbdev/api/pnpm-workspace.yaml b/bbdev/api/pnpm-workspace.yaml new file mode 100644 index 000000000..3334c0e43 --- /dev/null +++ b/bbdev/api/pnpm-workspace.yaml @@ -0,0 +1 @@ +packages: [] diff --git a/bbdev/api/pyproject.toml b/bbdev/api/pyproject.toml new file mode 100644 index 000000000..761bcaac3 --- /dev/null +++ b/bbdev/api/pyproject.toml @@ -0,0 +1,14 @@ +[project] +name = "bbdev" +version = "0.1.0" +description = "Agent-friendly developer toolchain backend powered by Motia" +requires-python = ">=3.10" +dependencies = [ + "motia[otel]==1.0.0rc17", + "iii-sdk==0.2.0", + "pydantic>=2.0", + "PyYAML>=6.0", +] + +[project.optional-dependencies] +dev = ["pytest>=8.0.0"] diff --git a/bbdev/api/requirements.txt b/bbdev/api/requirements.txt new file mode 100644 index 000000000..7d31babef --- /dev/null +++ b/bbdev/api/requirements.txt @@ -0,0 +1,3 @@ +pydantic>=2.0.0 +requests>=2.28.0 +protobuf>=5.0.0 diff --git a/bbdev/api/services/README.md b/bbdev/api/services/README.md new file mode 100644 index 000000000..9dd3d0ba6 --- /dev/null +++ b/bbdev/api/services/README.md @@ -0,0 +1,435 @@ +# Function Calling Tool Management System + +A modular and extensible Function Calling tool management framework for AI Agent interaction with external systems. + +## 📁 Directory Structure + +``` +tools/ +├── __init__.py # Module exports +├── base.py # Tool base class and context +├── registry.py # Tool registry and manager +├── file_tools.py # File operation tools +├── presets.py # Predefined tool sets +└── README.md # This document +``` + +## 🚀 Quick Start + +### 1. Using Predefined Tool Manager + +```python +from steps.tools import create_code_agent_manager + +# Create tool manager (file_tools already registered) +manager = create_code_agent_manager() + +# Get tool definitions (send to LLM) +tools_schema = manager.get_tools_schema() + +# Call LLM +response = llm.chat( + messages=messages, + tools=tools_schema # Pass in tool definitions +) + +# Execute tool calls returned by LLM +if response.tool_calls: + for tool_call in response.tool_calls: + result = manager.execute_tool( + tool_name=tool_call.function.name, + arguments=tool_call.function.arguments, + work_dir="/path/to/project", + logger=logger + ) +``` + +### 2. Custom Tools + +```python +from steps.tools import Tool, ToolManager +import json + +class RunCommandTool(Tool): + """Command execution tool""" + + def get_name(self) -> str: + return "run_command" + + def get_description(self) -> str: + return "Execute a shell command" + + def get_parameters(self) -> dict: + return { + "type": "object", + "properties": { + "command": {"type": "string", "description": "Command to execute"} + }, + "required": ["command"] + } + + def execute(self, arguments: dict, context) -> str: + import subprocess + cmd = arguments.get("command") + + try: + result = subprocess.run( + cmd, + shell=True, + capture_output=True, + text=True, + timeout=30 + ) + + return json.dumps({ + "stdout": result.stdout, + "stderr": result.stderr, + "returncode": result.returncode + }) + except Exception as e: + return json.dumps({"error": str(e)}) + +# Register custom tool +manager = ToolManager() +manager.register_tool(RunCommandTool()) +``` + +## 📚 Core Concepts + +### Tool (Base Class) + +All tools inherit from the `Tool` base class: + +```python +class MyTool(Tool): + def get_name(self) -> str: + """Tool name (unique identifier)""" + return "my_tool" + + def get_description(self) -> str: + """Tool description (tells AI what this tool does)""" + return "My custom tool" + + def get_parameters(self) -> dict: + """Parameter definition (JSON Schema format)""" + return { + "type": "object", + "properties": { + "param1": {"type": "string", "description": "Parameter 1"}, + "param2": {"type": "integer", "description": "Parameter 2"} + }, + "required": ["param1"] + } + + def execute(self, arguments: dict, context) -> str: + """ + Execute tool logic + + Args: + arguments: Parameters passed by AI + context: ToolContext object (contains work_dir, logger, etc.) + + Returns: + Execution result (string, can be JSON) + """ + # Implement tool logic + return json.dumps({"result": "success"}) +``` + +### ToolContext (Execution Context) + +Execution environment provided to tools: + +```python +context = ToolContext( + work_dir="/path/to/project", # Working directory + logger=logger, # Logger + extra_key="extra_value" # Custom extension fields +) + +# Use in tool +class MyTool(Tool): + def execute(self, arguments, context): + context.log_info("Starting execution") + work_dir = context.work_dir + custom = context.extra.get("extra_key") + # ... +``` + +### ToolRegistry (Tool Registry) + +Manage tool registration and lookup: + +```python +from steps.tools import ToolRegistry, ReadFileTool, WriteFileTool + +registry = ToolRegistry() +registry.register(ReadFileTool()) +registry.register(WriteFileTool()) + +# Get tool +tool = registry.get("read_file") + +# List all tools +tools = registry.list_tools() # ['read_file', 'write_file'] + +# Convert to OpenAI format +schema = registry.to_openai_format() +``` + +### ToolManager (Tool Manager) + +High-level wrapper providing more convenient interface: + +```python +from steps.tools import ToolManager + +manager = ToolManager() +manager.register_tools([ReadFileTool(), WriteFileTool()]) + +# Get tool definitions +schema = manager.get_tools_schema() + +# Execute tool +result = manager.execute_tool( + tool_name="read_file", + arguments={"path": "main.py"}, + work_dir="/project" +) + +# View execution log +log = manager.get_execution_log() +``` + +## 🛠️ Built-in Tools + +### File Operation Tools + +#### read_file +Read file content + +```json +{ + "name": "read_file", + "parameters": { + "path": "relative/path/to/file.txt" + } +} +``` + +#### write_file +Write file content (automatically creates directories) + +```json +{ + "name": "write_file", + "parameters": { + "path": "output/file.txt", + "content": "file content here" + } +} +``` + +#### list_files +List files in directory + +```json +{ + "name": "list_files", + "parameters": { + "path": "src" // Optional, defaults to current directory + } +} +``` + +## 🎨 Predefined Tool Sets + +```python +from steps.tools import get_preset, list_presets + +# View available tool sets +presets = list_presets() # ['file_tools', 'code_agent'] + +# Get tool set +tools = get_preset("file_tools") # Returns [ReadFileTool, WriteFileTool, ...] + +# Create manager +from steps.tools import create_code_agent_manager +manager = create_code_agent_manager() +``` + +## 📝 Complete Examples + +### Agent Integration Example + +```python +from steps.tools import create_code_agent_manager +import httpx +import json + +async def run_agent(prompt: str, work_dir: str): + # 1. Create tool manager + manager = create_code_agent_manager() + tools_schema = manager.get_tools_schema() + + # 2. Initialize conversation + messages = [ + {"role": "system", "content": "You are a code assistant"}, + {"role": "user", "content": prompt} + ] + + # 3. AI loop + max_iterations = 10 + for i in range(max_iterations): + # Call LLM + response = await call_llm(messages, tools_schema) + assistant_msg = response["choices"][0]["message"] + messages.append(assistant_msg) + + # Check for tool calls + if not assistant_msg.get("tool_calls"): + print(f"Done! Final response: {assistant_msg['content']}") + break + + # Execute all tool calls + for tool_call in assistant_msg["tool_calls"]: + result = manager.execute_tool( + tool_name=tool_call["function"]["name"], + arguments=tool_call["function"]["arguments"], + work_dir=work_dir + ) + + # Add tool result to conversation + messages.append({ + "role": "tool", + "tool_call_id": tool_call["id"], + "content": result + }) +``` + +## 🔒 Security Features + +### Path Security + +All file operation tools include path traversal checks: + +```python +# ✅ Allowed +read_file("src/main.py") + +# ❌ Denied (path traversal) +read_file("../../../etc/passwd") +``` + +### Error Handling + +Tool execution automatically catches exceptions: + +```python +# Internal exceptions are caught and returned as error messages +result = manager.execute_tool("read_file", {"path": "nonexist.txt"}) +# Returns: {"error": "File not found: nonexist.txt"} +``` + +## 🧪 Testing + +Create test file: + +```python +# test_tools.py +from steps.tools import create_code_agent_manager +import tempfile +import os + +def test_file_tools(): + manager = create_code_agent_manager() + + with tempfile.TemporaryDirectory() as tmpdir: + # Test write_file + result = manager.execute_tool( + "write_file", + {"path": "test.txt", "content": "hello"}, + work_dir=tmpdir + ) + assert "success" in result + + # Test read_file + result = manager.execute_tool( + "read_file", + {"path": "test.txt"}, + work_dir=tmpdir + ) + assert result == "hello" + + print("✅ Test passed") + +if __name__ == "__main__": + test_file_tools() +``` + +## 📦 Extending Tools + +### Adding New Tool Categories + +Create new file `network_tools.py`: + +```python +from .base import Tool +import json +import httpx + +class HttpGetTool(Tool): + def get_name(self) -> str: + return "http_get" + + def get_description(self) -> str: + return "Make HTTP GET request" + + def get_parameters(self) -> dict: + return { + "type": "object", + "properties": { + "url": {"type": "string"} + }, + "required": ["url"] + } + + def execute(self, arguments, context): + url = arguments["url"] + response = httpx.get(url) + return json.dumps({ + "status": response.status_code, + "body": response.text[:1000] + }) +``` + +Then add to `presets.py`: + +```python +def create_network_tools(): + from .network_tools import HttpGetTool + return [HttpGetTool()] +``` + +## 🎯 Best Practices + +1. **Tool Naming**: Use clear verb+noun format (`read_file`, `list_users`) +2. **Parameter Descriptions**: Describe each parameter in detail to help AI use them correctly +3. **Error Handling**: Return JSON format error messages with `error` field +4. **Logging**: Use `context.log_info/log_error` to record key operations +5. **Security Checks**: Validate input parameters, prevent path traversal and other security issues +6. **Result Format**: Return JSON strings or plain text, maintain consistency + +## 🤝 Contributing + +Steps to add new tools: + +1. Inherit from `Tool` base class +2. Implement 4 abstract methods +3. Add to appropriate tool set in `presets.py` +4. Export in `__init__.py` +5. Update README documentation + +## 📄 License + +Same as main project diff --git a/bbdev/api/services/__init__.py b/bbdev/api/services/__init__.py new file mode 100644 index 000000000..6de3040ca --- /dev/null +++ b/bbdev/api/services/__init__.py @@ -0,0 +1,84 @@ +""" +Function Calling Tool Management Module + +This module provides a complete Function Calling tool management system, including: +- Tool base class definitions +- Tool registration and management +- Predefined tool sets +- Tool execution context + +Usage example: + +```python +from tools import create_code_agent_manager + +# Create tool manager +manager = create_code_agent_manager() + +# Get tool definitions (to send to LLM) +tools_schema = manager.get_tools_schema() + +# Execute tool call +result = manager.execute_tool( + tool_name="read_file", + arguments={"path": "main.py"}, + work_dir="/path/to/project", + logger=logger +) +``` +""" + +from .base import Tool, ToolContext +from .registry import ToolRegistry, ToolManager +from .file_tools import ( + ReadFileTool, + WriteFileTool, + ListFilesTool, + MakeDirTool, + DeleteFileTool, + GetPathTool, + GrepFilesTool, +) +from .workflow_tools import WorkflowAPITool +from .deepwiki_tools import DeepwikiAskTool, DeepwikiReadWikiTool +from .agent_tools import CallAgentTool +from .presets import ( + create_file_tools, + create_code_agent_tools, + create_default_manager, + create_code_agent_manager, + get_preset, + list_presets, +) + +__all__ = [ + # Base classes + "Tool", + "ToolContext", + # Registration and management + "ToolRegistry", + "ToolManager", + # File operation tools + "ReadFileTool", + "WriteFileTool", + "ListFilesTool", + "MakeDirTool", + "DeleteFileTool", + "GetPathTool", + "GrepFilesTool", + # Agent coordination + "CallAgentTool", + # API tools + "WorkflowAPITool", + "DeepwikiAskTool", + "DeepwikiReadWikiTool", + # Predefined tool sets + "create_file_tools", + "create_code_agent_tools", + "create_default_manager", + "create_code_agent_manager", + "get_preset", + "list_presets", +] + +__version__ = "1.0.0" diff --git a/bbdev/api/services/agent_tools.py b/bbdev/api/services/agent_tools.py new file mode 100644 index 000000000..fa0153d73 --- /dev/null +++ b/bbdev/api/services/agent_tools.py @@ -0,0 +1,123 @@ +"""Agent coordination tools""" + +import httpx +import uuid +from typing import Dict, Any +from .base import Tool + + +class CallAgentTool(Tool): + """Tool for calling other agents""" + + def get_name(self) -> str: + return "call_agent" + + def get_description(self) -> str: + return """Call another agent (spec_agent, code_agent, review_agent, verify_agent) with a task. + Use this to delegate work to specialized agents. + Returns the agent's response and generated files.""" + + def get_parameters(self) -> Dict[str, Any]: + return { + "type": "object", + "properties": { + "agent_role": { + "type": "string", + "enum": ["spec", "code", "review", "verify"], + "description": "Which agent to call (spec/code/review/verify)", + }, + "task_description": { + "type": "string", + "description": "Task description or instructions for the agent", + }, + "context_files": { + "type": "array", + "items": {"type": "string"}, + "description": "Optional: List of file paths the agent should read for context", + }, + "model": { + "type": "string", + "description": "Optional: LLM model to use (will inherit from parent if not specified)", + }, + }, + "required": ["agent_role", "task_description"], + } + + def execute(self, arguments: Dict[str, Any], context: Any) -> str: + agent_role = arguments.get("agent_role") + task_description = arguments.get("task_description") + context_files = arguments.get("context_files", []) + # Prefer model from parameters, otherwise inherit from context + model = arguments.get("model") or context.extra.get("model") + + # Generate temporary task file + import os + import tempfile + + # Create temporary task file + task_content = task_description + + # Add context file references if specified + if context_files: + task_content += "\n\n## Context Files\n" + for filepath in context_files: + task_content += f"- {filepath}\n" + + # Write to temporary file + temp_dir = os.path.join(context.work_dir, ".agent_tasks") + os.makedirs(temp_dir, exist_ok=True) + + task_id = str(uuid.uuid4())[:8] + task_file = os.path.join(temp_dir, f"task_{agent_role}_{task_id}.md") + + with open(task_file, "w", encoding="utf-8") as f: + f.write(task_content) + + context.log_info(f"Created task file: {task_file}") + context.log_info(f"Calling {agent_role}_agent with task") + + try: + url = "http://localhost:3001/agent" + + context.log_info(f"Calling workflow API at: {url}") + + payload = { + "agentRole": agent_role, + "promptPath": task_file, + "workDir": context.work_dir, + } + + # Add model to payload if specified + if model: + payload["model"] = model + context.log_info(f"Using model: {model}") + + response = httpx.post(url, json=payload, timeout=600.0) + + if response.status_code == 200: + result = response.json() + + # Clean up temporary file + try: + os.remove(task_file) + except Exception: + pass + + return str( + { + "status": "success", + "agent": agent_role, + "result": result, + "files": result.get("files", []), + } + ) + else: + return str( + { + "error": f"Agent call failed with status {response.status_code}", + "response": response.text[:500], + } + ) + + except Exception as e: + return str({"error": f"Failed to call agent: {str(e)}"}) diff --git a/bbdev/api/services/base.py b/bbdev/api/services/base.py new file mode 100644 index 000000000..7b91d9ed4 --- /dev/null +++ b/bbdev/api/services/base.py @@ -0,0 +1,114 @@ +"""Function Calling tool base classes""" + +from abc import ABC, abstractmethod +from typing import Dict, Any, Optional +import json + + +class Tool(ABC): + """Tool base class""" + + def __init__(self): + self.name = self.get_name() + self.description = self.get_description() + self.parameters = self.get_parameters() + + @abstractmethod + def get_name(self) -> str: + """Return tool name""" + pass + + @abstractmethod + def get_description(self) -> str: + """Return tool description""" + pass + + @abstractmethod + def get_parameters(self) -> Dict[str, Any]: + """Return tool parameter definition (JSON Schema)""" + pass + + @abstractmethod + def execute(self, arguments: Dict[str, Any], context: Any) -> str: + """ + Execute tool + + Args: + arguments: Tool parameters + context: Execution context (contains logger, work_dir, etc.) + + Returns: + Execution result (string format) + """ + pass + + def to_openai_format(self) -> Dict[str, Any]: + """Convert to OpenAI Function Calling format""" + return { + "type": "function", + "function": { + "name": self.name, + "description": self.description, + "parameters": self.parameters, + }, + } + + def safe_execute(self, arguments: Any, context: Any) -> str: + """ + Safely execute tool (with error handling) + + Args: + arguments: Tool parameters (can be string or dict) + context: Execution context + + Returns: + Execution result or error message + """ + try: + # Parse arguments + if isinstance(arguments, str): + args = json.loads(arguments) + else: + args = arguments + + # Execute tool + result = self.execute(args, context) + return result + + except json.JSONDecodeError as e: + error = f"Invalid JSON arguments: {str(e)}" + if hasattr(context, "logger"): + context.logger.error(f"Tool {self.name} - {error}") + return json.dumps({"error": error}) + + except Exception as e: + error = f"Tool execution failed: {str(e)}" + if hasattr(context, "logger"): + context.logger.error(f"Tool {self.name} - {error}") + return json.dumps({"error": error}) + + def __repr__(self) -> str: + return f"Tool({self.name})" + + +class ToolContext: + """Tool execution context""" + + def __init__(self, work_dir: str, logger: Any = None, **kwargs): + self.work_dir = work_dir + self.logger = logger + self.extra = kwargs + + def log_info(self, message: str): + """Log info message""" + if self.logger: + self.logger.info(message) + else: + print(f"[INFO] {message}") + + def log_error(self, message: str): + """Log error message""" + if self.logger: + self.logger.error(message) + else: + print(f"[ERROR] {message}") diff --git a/bbdev/api/services/deepwiki_tools.py b/bbdev/api/services/deepwiki_tools.py new file mode 100644 index 000000000..214cdbdaa --- /dev/null +++ b/bbdev/api/services/deepwiki_tools.py @@ -0,0 +1,327 @@ +"""Deepwiki tool wrapper - via MCP Streamable HTTP""" + +import httpx +import json +import uuid +from httpx_sse import connect_sse +from typing import Dict, Any +from .base import Tool + + +class DeepwikiAskTool(Tool): + """Deepwiki Q&A tool - via MCP""" + + def get_name(self) -> str: + return "deepwiki_ask" + + def get_description(self) -> str: + return """Ask questions about a GitHub repository using Deepwiki. + Use this to understand code, architecture, and implementation details.""" + + def get_parameters(self) -> Dict[str, Any]: + return { + "type": "object", + "properties": { + "repo": { + "type": "string", + "description": ( + "GitHub repository in format 'owner/repo' " + "(e.g., 'DangoSys/buckyball', 'ucb-bar/gemmini')" + ), + }, + "question": { + "type": "string", + "description": "Question to ask about the repository", + }, + }, + "required": ["repo", "question"], + } + + def execute(self, arguments: Dict[str, Any], context: Any) -> str: + repo = arguments.get("repo") + question = arguments.get("question") + + try: + context.log_info( + f"Asking Deepwiki via MCP HTTP: {question[:100]}... (repo: {repo})" + ) + + # MCP tool call request (JSON-RPC 2.0) + request_payload = { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": { + "name": "ask_question", + "arguments": {"repoName": repo, "question": question}, + }, + } + + # Use Streamable HTTP endpoint + mcp_url = "https://mcp.deepwiki.com/mcp" + + context.log_info(f"MCP URL: {mcp_url}") + + with httpx.Client(timeout=120.0) as client: + # Step 1: Initialize session (without sessionId) + init_payload = { + "jsonrpc": "2.0", + "id": 0, + "method": "initialize", + "params": { + "protocolVersion": "2024-11-05", + "capabilities": {}, + "clientInfo": { + "name": "buckyball-workflow", + "version": "1.0.0", + }, + }, + } + + context.log_info("Initializing MCP session (without sessionId)...") + + session_id = None + # Use POST request directly, get sessionId from response headers + init_resp = client.post( + mcp_url, + json=init_payload, + headers={ + "Content-Type": "application/json", + "Accept": "application/json, text/event-stream", + }, + ) + + # Get sessionId from response headers + session_id = init_resp.headers.get("mcp-session-id") + if not session_id: + context.log_error(f"Init response status: {init_resp.status_code}") + context.log_error( + f"Init response headers: {dict(init_resp.headers)}" + ) + return "Error: No session ID in init response headers" + + context.log_info(f"Session initialized, ID: {session_id}") + + # Step 2: Call tool (with sessionId) - use SSE streaming + context.log_info( + f"Calling tool: {json.dumps(request_payload, ensure_ascii=False)[:300]}" + ) + + # Use stream instead of connect_sse + with client.stream( + "POST", + mcp_url, + json=request_payload, + headers={ + "Content-Type": "application/json", + "Accept": "application/json, text/event-stream", + "Mcp-Session-Id": session_id, + }, + ) as tool_resp: + context.log_info(f"Tool response status: {tool_resp.status_code}") + + # Manually parse SSE stream + for line in tool_resp.iter_lines(): + if line.startswith("data: "): + # Remove "data: " prefix + data_str = line[6:] + + # Skip heartbeat + if data_str.strip() == "ping": + continue + + try: + result = json.loads(data_str) + context.log_info( + f"Tool response: {json.dumps(result, ensure_ascii=False)[:500]}" + ) + + # Handle JSON-RPC 2.0 response + if "error" in result: + error = result["error"] + error_msg = f"MCP Error: {error.get('message', 'Unknown error')}" + context.log_error(error_msg) + return error_msg + + if "result" in result: + content = result["result"].get("content", []) + if content and len(content) > 0: + answer = content[0].get("text", "") + context.log_info( + f"Deepwiki answer length: {len(answer)} chars" + ) + context.log_info( + f"Deepwiki answer preview: {answer[:200]}..." + ) + + # Limit return length + if len(answer) > 3000: + answer = answer[:3000] + "\n... (truncated)" + return answer + except json.JSONDecodeError as e: + context.log_error( + f"Failed to parse SSE data: {e}, line: {line[:100]}" + ) + continue + + return "No valid response from Deepwiki" + + except Exception as e: + error_msg = f"Error calling Deepwiki MCP: {str(e)}" + context.log_error(error_msg) + return error_msg + + +class DeepwikiReadWikiTool(Tool): + """Read Deepwiki wiki content - via MCP""" + + def get_name(self) -> str: + return "deepwiki_read_wiki" + + def get_description(self) -> str: + return """Read wiki documentation for a GitHub repository from Deepwiki. + Use this to get structured documentation about the repository.""" + + def get_parameters(self) -> Dict[str, Any]: + return { + "type": "object", + "properties": { + "repo": { + "type": "string", + "description": "GitHub repository in format 'owner/repo'", + } + }, + "required": ["repo"], + } + + def execute(self, arguments: Dict[str, Any], context: Any) -> str: + repo = arguments.get("repo") + + try: + context.log_info(f"Reading Deepwiki wiki via MCP HTTP for: {repo}") + + # MCP tool call request + request_payload = { + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": { + "name": "read_wiki_contents", + "arguments": {"repoName": repo}, + }, + } + + # Use Streamable HTTP endpoint + mcp_url = "https://mcp.deepwiki.com/mcp" + + context.log_info(f"MCP URL: {mcp_url}") + + with httpx.Client(timeout=120.0) as client: + # Step 1: Initialize session (without sessionId) + init_payload = { + "jsonrpc": "2.0", + "id": 0, + "method": "initialize", + "params": { + "protocolVersion": "2024-11-05", + "capabilities": {}, + "clientInfo": { + "name": "buckyball-workflow", + "version": "1.0.0", + }, + }, + } + + context.log_info("Initializing MCP session (without sessionId)...") + + session_id = None + # Use POST request directly, get sessionId from response headers + init_resp = client.post( + mcp_url, + json=init_payload, + headers={ + "Content-Type": "application/json", + "Accept": "application/json, text/event-stream", + }, + ) + + # Get sessionId from response headers + session_id = init_resp.headers.get("mcp-session-id") + if not session_id: + context.log_error(f"Init response status: {init_resp.status_code}") + context.log_error( + f"Init response headers: {dict(init_resp.headers)}" + ) + return "Error: No session ID in init response headers" + + context.log_info(f"Session initialized, ID: {session_id}") + + # Step 2: Call tool (with sessionId) - use SSE streaming + context.log_info( + f"Calling tool: {json.dumps(request_payload, ensure_ascii=False)}" + ) + + # Use stream instead of connect_sse + with client.stream( + "POST", + mcp_url, + json=request_payload, + headers={ + "Content-Type": "application/json", + "Accept": "application/json, text/event-stream", + "Mcp-Session-Id": session_id, + }, + ) as tool_resp: + context.log_info(f"Tool response status: {tool_resp.status_code}") + + # Manually parse SSE stream + for line in tool_resp.iter_lines(): + if line.startswith("data: "): + # Remove "data: " prefix + data_str = line[6:] + + # Skip heartbeat + if data_str.strip() == "ping": + continue + + try: + result = json.loads(data_str) + context.log_info( + f"Tool response: {json.dumps(result, ensure_ascii=False)[:500]}" + ) + + # Handle JSON-RPC 2.0 response + if "error" in result: + error = result["error"] + error_msg = f"MCP Error: {error.get('message', 'Unknown error')}" + context.log_error(error_msg) + return error_msg + + if "result" in result: + content = result["result"].get("content", []) + if content and len(content) > 0: + wiki_text = content[0].get("text", "") + context.log_info( + f"Deepwiki wiki length: {len(wiki_text)} chars" + ) + context.log_info( + f"Deepwiki wiki preview: {wiki_text[:200]}..." + ) + + if len(wiki_text) > 5000: + wiki_text = ( + wiki_text[:5000] + "\n... (truncated)" + ) + return wiki_text + except json.JSONDecodeError as e: + context.log_error( + f"Failed to parse SSE data: {e}, line: {line[:100]}" + ) + continue + + return "No valid wiki content from Deepwiki" + + except Exception as e: + error_msg = f"Error reading Deepwiki wiki via MCP: {str(e)}" + context.log_error(error_msg) + return error_msg diff --git a/bbdev/api/services/file_tools.py b/bbdev/api/services/file_tools.py new file mode 100644 index 000000000..cec4ca8c1 --- /dev/null +++ b/bbdev/api/services/file_tools.py @@ -0,0 +1,359 @@ +"""File operation related tools""" + +import os +import json +import shutil +from typing import Dict, Any +from .base import Tool + + +class MakeDirTool(Tool): + """Create directory tool""" + + def get_name(self) -> str: + return "make_dir" + + def get_description(self) -> str: + return "Create a new directory (supports creating parent directories)" + + def get_parameters(self) -> Dict[str, Any]: + return { + "type": "object", + "properties": { + "path": {"type": "string", "description": "Directory path to create"} + }, + "required": ["path"], + } + + def execute(self, arguments: Dict[str, Any], context: Any) -> str: + path = arguments.get("path") + + if not os.path.isabs(path): + path = os.path.join(context.work_dir, path) + + try: + if os.path.exists(path): + return json.dumps({"status": "exists", "path": path}) + + os.makedirs(path, exist_ok=True) + context.log_info(f"Created directory: {path}") + return json.dumps({"status": "success", "path": path}) + + except Exception as e: + return json.dumps({"error": f"Failed to create directory: {str(e)}"}) + + +class GetPathTool(Tool): + """Get path information tool""" + + def get_name(self) -> str: + return "get_path_info" + + def get_description(self) -> str: + return "Get absolute path and check if path exists" + + def get_parameters(self) -> Dict[str, Any]: + return { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Path to check (optional, defaults to work_dir)", + } + }, + } + + def execute(self, arguments: Dict[str, Any], context: Any) -> str: + path = arguments.get("path", ".") + + if not os.path.isabs(path): + path = os.path.join(context.work_dir, path) + + abs_path = os.path.abspath(path) + exists = os.path.exists(abs_path) + + info = { + "absolute_path": abs_path, + "exists": exists, + "work_dir": context.work_dir, + } + + if exists: + info["is_file"] = os.path.isfile(abs_path) + info["is_dir"] = os.path.isdir(abs_path) + + return json.dumps(info, indent=2) + + +class GrepFilesTool(Tool): + """Search file content tool""" + + def get_name(self) -> str: + return "grep_files" + + def get_description(self) -> str: + return "Search for text pattern in files" + + def get_parameters(self) -> Dict[str, Any]: + return { + "type": "object", + "properties": { + "pattern": {"type": "string", "description": "Text pattern to search"}, + "path": {"type": "string", "description": "Directory or file path"}, + "file_ext": { + "type": "string", + "description": "File extension filter (e.g., '.scala')", + }, + }, + "required": ["pattern", "path"], + } + + def execute(self, arguments: Dict[str, Any], context: Any) -> str: + pattern = arguments.get("pattern") + path = arguments.get("path") + file_ext = arguments.get("file_ext") + + if not os.path.isabs(path): + path = os.path.join(context.work_dir, path) + + try: + results = [] + files_to_search = [] + + if os.path.isfile(path): + files_to_search = [path] + else: + for root, _, files in os.walk(path): + for file in files: + if file_ext and not file.endswith(file_ext): + continue + files_to_search.append(os.path.join(root, file)) + + # Limit file count + for filepath in files_to_search[:100]: + try: + with open(filepath, "r", encoding="utf-8", errors="ignore") as f: + for line_num, line in enumerate(f, 1): + if pattern in line: + results.append( + { + "file": filepath, + "line": line_num, + "content": line.strip()[:150], + } + ) + # Limit result count + if len(results) >= 50: + break + except Exception: + continue + + if len(results) >= 50: + break + + return json.dumps({"matches": len(results), "results": results}, indent=2) + + except Exception as e: + return json.dumps({"error": f"Search failed: {str(e)}"}) + + +class DeleteFileTool(Tool): + """Delete file tool""" + + def get_name(self) -> str: + return "delete_file" + + def get_description(self) -> str: + return "Delete a file (use with caution)" + + def get_parameters(self) -> Dict[str, Any]: + return { + "type": "object", + "properties": { + "path": {"type": "string", "description": "Path to file to delete"} + }, + "required": ["path"], + } + + def execute(self, arguments: Dict[str, Any], context: Any) -> str: + path = arguments.get("path") + + if not os.path.isabs(path): + path = os.path.join(context.work_dir, path) + + try: + if not os.path.exists(path): + return json.dumps({"status": "not_found", "path": path}) + + if os.path.isfile(path): + os.remove(path) + context.log_info(f"Deleted file: {path}") + return json.dumps({"status": "success", "path": path}) + else: + return json.dumps({"error": "Path is a directory, not a file"}) + + except Exception as e: + return json.dumps({"error": f"Failed to delete: {str(e)}"}) + + +class ReadFileTool(Tool): + """Read file content""" + + def get_name(self) -> str: + return "read_file" + + def get_description(self) -> str: + return "Read the content of a file" + + def get_parameters(self) -> Dict[str, Any]: + return { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "File path relative to work directory", + } + }, + "required": ["path"], + } + + def execute(self, arguments: Dict[str, Any], context: Any) -> str: + file_path = arguments.get("path") + + if not file_path: + return json.dumps({"error": "Missing required parameter: path"}) + + full_path = os.path.join(context.work_dir, file_path) + + # Security check: prevent path traversal + abs_full = os.path.abspath(full_path) + abs_work = os.path.abspath(context.work_dir) + if not abs_full.startswith(abs_work): + return json.dumps({"error": "Access denied: path outside work directory"}) + + if not os.path.exists(full_path): + return json.dumps({"error": f"File not found: {file_path}"}) + + if not os.path.isfile(full_path): + return json.dumps({"error": f"Not a file: {file_path}"}) + + try: + with open(full_path, "r", encoding="utf-8") as f: + content = f.read() + + context.log_info(f"Tool: read_file({file_path}) - {len(content)} chars") + return content + + except UnicodeDecodeError: + return json.dumps( + {"error": "Cannot read file: not a text file or encoding issue"} + ) + + +class WriteFileTool(Tool): + """Write file content""" + + def get_name(self) -> str: + return "write_file" + + def get_description(self) -> str: + return "Write content to a file (creates directories if needed)" + + def get_parameters(self) -> Dict[str, Any]: + return { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "File path relative to work directory", + }, + "content": { + "type": "string", + "description": "Content to write to the file", + }, + }, + "required": ["path", "content"], + } + + def execute(self, arguments: Dict[str, Any], context: Any) -> str: + file_path = arguments.get("path") + content = arguments.get("content") + + if not file_path: + return json.dumps({"error": "Missing required parameter: path"}) + + if content is None: + return json.dumps({"error": "Missing required parameter: content"}) + + full_path = os.path.join(context.work_dir, file_path) + + # Security check: prevent path traversal + abs_full = os.path.abspath(full_path) + abs_work = os.path.abspath(context.work_dir) + if not abs_full.startswith(abs_work): + return json.dumps({"error": "Access denied: path outside work directory"}) + + try: + # Create directory + dir_path = os.path.dirname(full_path) + if dir_path: + os.makedirs(dir_path, exist_ok=True) + + # Write file + with open(full_path, "w", encoding="utf-8") as f: + f.write(content) + + context.log_info(f"Tool: write_file({file_path}) - {len(content)} chars") + + return json.dumps( + {"success": True, "path": file_path, "size": len(content)} + ) + + except Exception as e: + return json.dumps({"error": f"Failed to write file: {str(e)}"}) + + +class ListFilesTool(Tool): + """List files in directory""" + + def get_name(self) -> str: + return "list_files" + + def get_description(self) -> str: + return "List files in a directory" + + def get_parameters(self) -> Dict[str, Any]: + return { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Directory path relative to work directory (default: .)", + } + }, + } + + def execute(self, arguments: Dict[str, Any], context: Any) -> str: + dir_path = arguments.get("path", ".") + full_path = os.path.join(context.work_dir, dir_path) + + # Security check + abs_full = os.path.abspath(full_path) + abs_work = os.path.abspath(context.work_dir) + if not abs_full.startswith(abs_work): + return json.dumps({"error": "Access denied: path outside work directory"}) + + if not os.path.exists(full_path): + return json.dumps({"error": f"Directory not found: {dir_path}"}) + + if not os.path.isdir(full_path): + return json.dumps({"error": f"Not a directory: {dir_path}"}) + + try: + files = os.listdir(full_path) + context.log_info(f"Tool: list_files({dir_path}) - {len(files)} items") + + return json.dumps({"path": dir_path, "files": files, "count": len(files)}) + + except Exception as e: + return json.dumps({"error": f"Failed to list directory: {str(e)}"}) diff --git a/bbdev/api/services/presets.py b/bbdev/api/services/presets.py new file mode 100644 index 000000000..6ef8be893 --- /dev/null +++ b/bbdev/api/services/presets.py @@ -0,0 +1,98 @@ +"""Predefined tool sets""" + +from typing import List +from .base import Tool +from .file_tools import ( + ReadFileTool, + WriteFileTool, + ListFilesTool, + MakeDirTool, + DeleteFileTool, + GetPathTool, + GrepFilesTool, +) +from .workflow_tools import WorkflowAPITool +from .deepwiki_tools import DeepwikiAskTool, DeepwikiReadWikiTool +from .agent_tools import CallAgentTool +from .registry import ToolManager + + +def create_file_tools() -> List[Tool]: + """Create file operation tool set""" + return [ + ReadFileTool(), + WriteFileTool(), + ListFilesTool(), + MakeDirTool(), + DeleteFileTool(), + GetPathTool(), + GrepFilesTool(), + ] + + +def create_code_agent_tools() -> List[Tool]: + """Create Code Agent tool set (includes all required tools)""" + return [ + # File operations + ReadFileTool(), + WriteFileTool(), + ListFilesTool(), + MakeDirTool(), + DeleteFileTool(), + GetPathTool(), + GrepFilesTool(), + # Agent coordination + CallAgentTool(), + # Workflow API + WorkflowAPITool(), + # Deepwiki + DeepwikiAskTool(), + DeepwikiReadWikiTool(), + ] + + +def create_default_manager() -> ToolManager: + """Create default tool manager""" + manager = ToolManager() + manager.register_tools(create_file_tools()) + return manager + + +def create_code_agent_manager() -> ToolManager: + """Create Code Agent dedicated tool manager""" + manager = ToolManager() + manager.register_tools(create_code_agent_tools()) + return manager + + +def preset_configs(): + return { + "file_tools": create_file_tools, + "code_agent": create_code_agent_tools, + } + + +def get_preset(name: str) -> List[Tool]: + """ + Get predefined tool set + + Args: + name: Tool set name ("file_tools", "code_agent") + + Returns: + Tool list + + Raises: + ValueError: If tool set does not exist + """ + presets = preset_configs() + factory = presets.get(name) + if factory is None: + available = ", ".join(presets) + raise ValueError(f"Unknown preset: {name}. Available: {available}") + return factory() + + +def list_presets() -> List[str]: + """List all available predefined tool sets""" + return list(preset_configs()) diff --git a/bbdev/api/services/registry.py b/bbdev/api/services/registry.py new file mode 100644 index 000000000..139224669 --- /dev/null +++ b/bbdev/api/services/registry.py @@ -0,0 +1,131 @@ +"""Function Calling tool registry""" + +from typing import Dict, List, Any, Optional +from .base import Tool, ToolContext + + +class ToolRegistry: + """Tool registration and management""" + + def __init__(self): + self._tools: Dict[str, Tool] = {} + + def register(self, tool: Tool): + """Register a tool""" + self._tools[tool.name] = tool + + def register_all(self, tools: List[Tool]): + """Batch register tools""" + for tool in tools: + self.register(tool) + + def get(self, name: str) -> Optional[Tool]: + """Get tool""" + return self._tools.get(name) + + def list_tools(self) -> List[str]: + """List all tool names""" + return list(self._tools.keys()) + + def to_openai_format(self) -> List[Dict[str, Any]]: + """Convert to OpenAI Function Calling format""" + return [tool.to_openai_format() for tool in self._tools.values()] + + def execute(self, tool_name: str, arguments: Any, context: ToolContext) -> str: + """ + Execute tool + + Args: + tool_name: Tool name + arguments: Tool parameters + context: Execution context + + Returns: + Execution result + """ + tool = self.get(tool_name) + + if not tool: + return f'{{"error": "Unknown tool: {tool_name}"}}' + + return tool.safe_execute(arguments, context) + + def __len__(self) -> int: + return len(self._tools) + + def __repr__(self) -> str: + return f"ToolRegistry({len(self)} tools: {', '.join(self.list_tools())})" + + +class ToolManager: + """Tool manager (high-level wrapper)""" + + def __init__(self, registry: Optional[ToolRegistry] = None): + self.registry = registry or ToolRegistry() + self._execution_log: List[Dict[str, Any]] = [] + + def register_tool(self, tool: Tool): + """Register tool""" + self.registry.register(tool) + + def register_tools(self, tools: List[Tool]): + """Batch register tools""" + self.registry.register_all(tools) + + def get_tools_schema(self) -> List[Dict[str, Any]]: + """Get tool definitions (OpenAI format)""" + return self.registry.to_openai_format() + + def execute_tool( + self, + tool_name: str, + arguments: Any, + work_dir: str, + logger: Any = None, + **kwargs, + ) -> str: + """ + Execute tool call + + Args: + tool_name: Tool name + arguments: Tool parameters + work_dir: Working directory + logger: Logger + **kwargs: Other context parameters + + Returns: + Execution result + """ + # Create context + context = ToolContext(work_dir=work_dir, logger=logger, **kwargs) + + # Execute tool + result = self.registry.execute(tool_name, arguments, context) + + # Log execution + self._execution_log.append( + { + "tool": tool_name, + "arguments": arguments, + # Truncate long results + "result": result[:200] if len(result) > 200 else result, + } + ) + + return result + + def get_execution_log(self) -> List[Dict[str, Any]]: + """Get execution log""" + return self._execution_log + + def clear_log(self): + """Clear execution log""" + self._execution_log.clear() + + def get_tool_names(self) -> List[str]: + """Get all tool names""" + return self.registry.list_tools() + + def __repr__(self) -> str: + return f"ToolManager({len(self.registry)} tools registered)" diff --git a/bbdev/api/services/workflow_tools.py b/bbdev/api/services/workflow_tools.py new file mode 100644 index 000000000..0cd2b90fd --- /dev/null +++ b/bbdev/api/services/workflow_tools.py @@ -0,0 +1,59 @@ +"""Workflow internal API call tools""" + +import httpx +from typing import Dict, Any +from .base import Tool + + +class WorkflowAPITool(Tool): + """Generic tool for calling Workflow internal APIs""" + + def get_name(self) -> str: + return "call_workflow_api" + + def get_description(self) -> str: + return ( + "Call internal workflow API endpoints.\n Available endpoints:\n -" + " /verilator/verilog: Generate Verilog\n - /verilator/build: Build" + " verilator (params: jobs)\n - /verilator/sim: Run simulation (params:" + " binary, batch)\n - /workload/clean: Clean workload output directory\n " + " - /workload/build: Build workload (params: design, stable, ctest," + " mlirtest)\n - /model/build: Build model recipe (params: design, model)" + ) + + def get_parameters(self) -> Dict[str, Any]: + return { + "type": "object", + "properties": { + "endpoint": { + "type": "string", + "description": "API endpoint path (e.g., '/verilator/build')", + }, + "params": { + "type": "object", + "description": "Request parameters as JSON object", + "additionalProperties": True, + }, + }, + "required": ["endpoint"], + } + + def execute(self, arguments: Dict[str, Any], context: Any) -> str: + endpoint = arguments.get("endpoint") + params = arguments.get("params", {}) + url = f"http://localhost:3001{endpoint}" + try: + context.log_info(f"Calling workflow API: {url}") + context.log_info(f"Parameters: {params}") + response = httpx.post(url, json=params, timeout=300.0) + if response.status_code == 200: + return str(response.json()) + else: + return str( + { + "error": f"API call failed with status {response.status_code}", + "response": response.text[:500], + } + ) + except Exception as e: + return str({"error": f"Workflow API call failed: {str(e)}"}) diff --git a/bbdev/api/steps/bebop/bemu/02_sim_api.step.py b/bbdev/api/steps/bebop/bemu/02_sim_api.step.py new file mode 100644 index 000000000..02bbe2c93 --- /dev/null +++ b/bbdev/api/steps/bebop/bemu/02_sim_api.step.py @@ -0,0 +1,41 @@ +from utils.event_common import require_target +import os +import sys + +from motia import ApiRequest, ApiResponse, FlowContext, api + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) + +from utils.event_common import require_design + +config = { + "name": "bebop-bemu-sim-api", + "description": "Run bebop bemu emulator", + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/bemu/sim")], + "enqueues": ["bebop.bemu.sim"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + try: + selection = require_target(body, ("device", "core")) + require_design(body) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + if not body.get("binary"): + return ApiResponse(status=400, body={"error": "binary parameter is required"}) + + await ctx.enqueue( + { + "topic": "bebop.bemu.sim", + "data": {**body, "_trace_id": ctx.trace_id, "target": selection}, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py new file mode 100644 index 000000000..0190903bd --- /dev/null +++ b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py @@ -0,0 +1,213 @@ +""" +bebop bemu event handler + +Run a guest ELF through BEMU. +""" + +import os +import shlex +import shutil +import sys +from datetime import datetime +from pathlib import Path +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) +from utils.path import ( + bebop_target_dir, + bebop_bin_path, + get_buckyball_path, + log_dir, + design_output_root, +) +from utils.stream_run import stream_run_logger_async +from utils.search_workload import search_workload +from utils.event_common import check_result, get_origin_trace_id +from utils.process_registry import cancellation_requested +from bemu_common import bemu_binary, bemu_manifest +from steps.bebop.performance_report import performance_report +from utils.reports import source_context + +config = { + "name": "bebop-bemu-sim", + "description": "Run bebop bemu emulator", + "flows": ["bebop"], + "triggers": [queue("bebop.bemu.sim")], + "enqueues": [], +} + + +def resolve_bemu_binary( + bbdir: str, design: str, binary_name: str, selection: str +) -> str | None: + workload_root = ( + Path(design_output_root(bbdir, design, selection=selection)) / "workloads" + ) + return search_workload(str(workload_root), binary_name) + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + design = input_data["design"] + bemu_cargo_manifest = bemu_manifest(design, bbdir, selection=input_data["target"]) + binary_name = input_data["binary"] + binary_path = resolve_bemu_binary(bbdir, design, binary_name, input_data["target"]) + if binary_path is None: + ctx.logger.error(f"binary not found: {binary_name}") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "binary_not_found", "binary": binary_name}, + trace_id=origin_tid, + ) + return + ctx.logger.info(f"binary_path: {binary_path}") + binary_dir = os.path.dirname(binary_path) + timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") + run_log = log_dir( + bbdir, + design, + "verilog", + timestamp, + "bemu", + binary_name, + selection=input_data["target"], + ) + os.makedirs(run_log, exist_ok=True) + target = Path( + bebop_target_dir(bbdir, design, "bemu", selection=input_data["target"]) + ) + installed = Path( + bebop_bin_path(bbdir, design, "bemu", selection=input_data["target"]) + ) + build_args = [ + "cargo", + "build", + "--release", + "--manifest-path", + str(bemu_cargo_manifest), + "--target-dir", + str(target), + "--bin", + bemu_binary(), + "--features", + "bemu", + "--config", + 'build.rustflags=["-C", "target-cpu=native"]', + "--config", + 'profile.release.lto="thin"', + "--config", + "profile.release.codegen-units=1", + ] + build_result = await stream_run_logger_async( + cmd=shlex.join(["nix", "develop", "-c", *build_args]), + executable="/bin/bash", + logger=ctx.logger, + cwd=bbdir, + task_scope=origin_tid, + stdout_prefix="BEMU build", + stderr_prefix="BEMU build", + ) + if cancellation_requested(origin_tid): + return + if build_result.returncode: + await check_result( + ctx, + build_result.returncode, + continue_run=False, + extra_fields={"task": "build", "design": design}, + trace_id=origin_tid, + ) + return + installed.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(target / "release/bebop-bemu", installed) + run_args = [str(installed), "--elf", binary_path, "--log-dir", run_log] + # `bbdev kernel --build` lays the kernel and the model into -ddr.bin + # next to the firmware ELF. + ddr_image = Path(binary_path).with_suffix("").as_posix() + "-ddr.bin" + if os.path.isfile(ddr_image): + run_args.extend(["--ddr-image", ddr_image]) + for option in ("dtb", "initrd"): + if input_data.get(option): + run_args.extend([f"--{option}", input_data[option]]) + if input_data.get("memory-mib") is not None: + run_args.extend(["--memory-mib", str(input_data["memory-mib"])]) + if input_data.get("disasm"): + run_args.append("--disasm") + if input_data.get("tool-profile"): + run_args.append("--tool-profile") + for trace_name in ("itrace", "mtrace", "ctrace"): + if input_data.get(trace_name, False): + run_args.append(f"--{trace_name}") + inner_cmd = f"cd {shlex.quote(binary_dir)} && {shlex.join(run_args)}" + run_cmd = f"nix develop -c sh -c {shlex.quote(inner_cmd)}" + run_cmd = f"set -o pipefail; {run_cmd} 2>&1 | tee {shlex.quote(str(Path(run_log) / 'run.log'))}" + ctx.logger.info(f"Running bebop bemu: {run_cmd}") + report_context = source_context(bbdir, design, input_data["target"]) + run_result = await stream_run_logger_async( + cmd=run_cmd, + logger=ctx.logger, + cwd=bbdir, + executable="/bin/bash", + stdout_prefix="bebop bemu", + stderr_prefix="bebop bemu", + task_scope=origin_tid, + env=os.environ.copy(), + ) + if cancellation_requested(origin_tid): + return + try: + report_path = performance_report( + bbdir, + report_context, + origin_tid, + "bemu", + run_log, + (Path(run_log) / "run.log").read_text(), + run_result.returncode, + ) + except (OSError, ValueError, KeyError) as error: + await check_result( + ctx, + 1, + extra_fields={"task": "report", "error": str(error), "log_dir": run_log}, + trace_id=origin_tid, + ) + return + if run_result.returncode != 0: + await check_result( + ctx, + run_result.returncode, + continue_run=False, + extra_fields={ + "task": "bemu", + "report_path": report_path, + "binary": binary_path, + "design": design, + "log_dir": run_log, + "timestamp": timestamp, + }, + trace_id=origin_tid, + ) + return + await check_result( + ctx, + 0, + continue_run=False, + extra_fields={ + "task": "bemu", + "report_path": report_path, + "binary": binary_path, + "design": design, + "log_dir": run_log, + "timestamp": timestamp, + }, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/bebop/bemu/03_batch_api.step.py b/bbdev/api/steps/bebop/bemu/03_batch_api.step.py new file mode 100644 index 000000000..16c74f537 --- /dev/null +++ b/bbdev/api/steps/bebop/bemu/03_batch_api.step.py @@ -0,0 +1,61 @@ +from utils.event_common import require_target +import os +import sys +from motia import ApiRequest, ApiResponse, FlowContext, api + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) +from utils.event_common import require_design + +config = { + "name": "bebop-bemu-batch-api", + "description": "Run bebop bemu batch regression", + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/bemu/batch")], + "enqueues": ["bebop.bemu.batch"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + try: + selection = require_target(body, ("device",)) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + design = body.get("design") + try: + require_design({"design": design}) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) + test_type = body.get("test") + if not test_type: + return ApiResponse( + status=400, + body={ + "error": "Missing required parameter: --test must be specified (bare-tests or linux-tests)" + }, + ) + if test_type not in ("bare-tests", "linux-tests"): + return ApiResponse( + status=400, + body={ + "error": f"Invalid test type: {test_type}. Must be 'bare-tests' or 'linux-tests'" + }, + ) + await ctx.enqueue( + { + "topic": "bebop.bemu.batch", + "data": { + "design": design, + "test": test_type, + "jobs": body.get("jobs", 1), + "_trace_id": ctx.trace_id, + "target": selection, + }, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py new file mode 100644 index 000000000..459323f8c --- /dev/null +++ b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py @@ -0,0 +1,174 @@ +""" +bebop bemu batch event handler + +Runs bebop bemu batch regression: + 1. Build the selected design's BEMU wrapper + 2. Run the BEMU regression harness +""" + +from datetime import datetime +from pathlib import Path +from utils.path import design_output_root, log_dir + + +import os +import shlex +import sys +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) +bebop_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +if bebop_path not in sys.path: + sys.path.insert(0, bebop_path) +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) +from utils.path import ( + bebop_target_dir, + bebop_bin_path, + get_buckyball_path, + workloads_output_root, +) +from utils.stream_run import stream_run_logger_async +from utils.event_common import check_result, get_origin_trace_id +from bemu_common import bemu_manifest +from utils.workload_manifest import resolve_workload_toml + +config = { + "name": "bebop-bemu-batch", + "description": "Run bebop bemu batch regression", + "flows": ["bebop"], + "triggers": [queue("bebop.bemu.batch")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + design = input_data.get("design") + if not design: + ctx.logger.error("Missing required parameter: design must be specified") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "missing_design"}, + trace_id=origin_tid, + ) + return + try: + bemu_cargo_manifest = bemu_manifest( + design, bbdir, selection=input_data["target"] + ) + except ValueError as e: + ctx.logger.error(str(e)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "invalid_design", "design": design}, + trace_id=origin_tid, + ) + return + elf_root = str( + Path(design_output_root(bbdir, design, selection=input_data["target"])) + / "workloads" + ) + env = os.environ.copy() + test_type = input_data.get("test", "bare-tests") + try: + workload_toml = resolve_workload_toml( + design, "bemu", test_type, bbdir, selection=input_data["target"] + ) + except ValueError as e: + ctx.logger.error(str(e)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "error": "invalid_regression", + "test": test_type, + "design": design, + }, + trace_id=origin_tid, + ) + return + ctx.logger.info(f"Running {test_type} with workload config: {workload_toml}") + target = bebop_target_dir(bbdir, design, "bemu", selection=input_data["target"]) + build_cmd = ( + "nix develop -c cargo build --manifest-path" + f" {shlex.quote(str(bemu_cargo_manifest))} --target-dir {shlex.quote(target)}" + " --features bemu --tests --bin bebop-bemu" + ) + ctx.logger.info("Building bebop bemu (tests)...") + build_result = await stream_run_logger_async( + cmd=build_cmd, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="bebop bemu build", + stderr_prefix="bebop bemu build", + env=env, + ) + if build_result.returncode != 0: + await check_result( + ctx, + build_result.returncode, + continue_run=False, + extra_fields={"task": "build", "backend": "bemu"}, + trace_id=origin_tid, + ) + return + run_log = log_dir( + bbdir, + design, + "verilog", + datetime.now().strftime("%Y-%m-%d-%H-%M"), + "bebop-bemu", + test_type, + selection=input_data["target"], + ) + harness_args = [ + "--", + "--workload-toml", + workload_toml, + "--bb-tests-root", + elf_root, + "--log-dir", + run_log, + "--jobs", + str(input_data.get("jobs", 1)), + ] + harness = shlex.join(harness_args) + test_cmd = ( + "nix develop -c cargo test --manifest-path" + f" {shlex.quote(str(bemu_cargo_manifest))} --target-dir {shlex.quote(target)}" + f" --features bemu --test test_bemu {harness}" + ) + ctx.logger.info(f"Running bebop bemu regression: {test_cmd}") + run_result = await stream_run_logger_async( + cmd=test_cmd, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="bebop bemu batch", + stderr_prefix="bebop bemu batch", + env=env, + ) + await check_result( + ctx, + run_result.returncode, + continue_run=False, + extra_fields={ + "task": "batch", + "backend": "bemu", + "design": design, + "test_type": test_type, + "workload_toml": workload_toml, + "log_dir": run_log, + "junit": os.path.join(run_log, "junit.xml"), + }, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/bebop/bemu/04_analysis_api.step.py b/bbdev/api/steps/bebop/bemu/04_analysis_api.step.py new file mode 100644 index 000000000..15e36d83c --- /dev/null +++ b/bbdev/api/steps/bebop/bemu/04_analysis_api.step.py @@ -0,0 +1,80 @@ +from utils.event_common import require_target +import os +import sys + +from motia import ApiRequest, ApiResponse, FlowContext, api + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) + +from utils.path import get_buckyball_path, run_log_dir +from bemu_analysis import design_maps + +config = { + "name": "bebop-bemu-analysis-api", + "description": "Analyze bebop bemu itrace/mtrace", + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/bemu/analysis")], + "enqueues": ["bebop.bemu.analysis"], +} + + +def _fail(message: str, status: int = 400) -> ApiResponse: + return ApiResponse( + status=status, + body={ + "success": False, + "failure": True, + "returncode": status, + "message": message, + }, + ) + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + allowed = {"device", "core", "design", "run", "itrace", "mtrace"} + unknown = sorted(k for k in body if k not in allowed) + if unknown: + return _fail(f"unknown parameter(s): {', '.join(unknown)}") + + design = body.get("design", "") + if not design: + return _fail("design parameter is required") + try: + design_maps(get_buckyball_path(), design, selection) + except ValueError as e: + return _fail(str(e)) + + itrace = bool(body.get("itrace", False)) + mtrace = bool(body.get("mtrace", False)) + if not itrace and not mtrace: + return _fail("need --itrace and/or --mtrace") + + try: + run_log_dir(get_buckyball_path(), body.get("run")) + except ValueError as e: + return _fail(str(e)) + + await ctx.enqueue( + { + "topic": "bebop.bemu.analysis", + "data": { + "design": design, + "run": body["run"], + "itrace": itrace, + "mtrace": mtrace, + "_trace_id": ctx.trace_id, + "target": selection, + }, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/bemu/04_analysis_event.step.py b/bbdev/api/steps/bebop/bemu/04_analysis_event.step.py new file mode 100644 index 000000000..82035624a --- /dev/null +++ b/bbdev/api/steps/bebop/bemu/04_analysis_event.step.py @@ -0,0 +1,96 @@ +import os +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) + +from utils.path import get_buckyball_path, run_log_dir +from utils.event_common import check_result, get_origin_trace_id +from bemu_analysis import analysis_dir, design_maps, write_report + +config = { + "name": "bebop-bemu-analysis", + "description": "Analyze bebop bemu itrace/mtrace", + "flows": ["bebop"], + "triggers": [queue("bebop.bemu.analysis")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + design = input_data.get("design") + if not design: + ctx.logger.error("Missing required parameter: design") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "missing_design"}, + trace_id=origin_tid, + ) + return + + try: + log_path = run_log_dir(get_buckyball_path(), input_data.get("run")) + except ValueError as e: + ctx.logger.error(str(e)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "invalid_run", "message": str(e)}, + trace_id=origin_tid, + ) + return + + itrace = bool(input_data.get("itrace", False)) + mtrace = bool(input_data.get("mtrace", False)) + try: + names, matrix, depth = design_maps( + get_buckyball_path(), design, input_data["target"] + ) + text = analysis_dir( + log_path, + names=names, + matrix=matrix, + bank_depth=depth, + itrace=itrace, + mtrace=mtrace, + ) + out = write_report(log_path, text) + except ValueError as e: + ctx.logger.error(str(e)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "error": "analysis_failed", + "message": str(e), + "log_dir": str(log_path), + }, + trace_id=origin_tid, + ) + return + + for line in text.splitlines(): + ctx.logger.info(line) + await check_result( + ctx, + 0, + continue_run=False, + extra_fields={ + "task": "bemu-analysis", + "design": design, + "log_dir": str(log_path), + "analysis": str(out), + }, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/bebop/bemu/scripts/bank_replay.py b/bbdev/api/steps/bebop/bemu/scripts/bank_replay.py new file mode 100644 index 000000000..6ba6add41 --- /dev/null +++ b/bbdev/api/steps/bebop/bemu/scripts/bank_replay.py @@ -0,0 +1,402 @@ +import argparse +from array import array +from collections import deque +import csv +import heapq +import json +import math +from pathlib import Path +import sys +import time + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np + +repo = Path(__file__).resolve().parents[6] +sys.path.insert(0, str(repo)) +from stack.scripts.design import load + +sys.path.insert(0, str(Path(__file__).resolve().parents[4])) +from utils.path import get_buckyball_path, run_log_dir + + +class Replay: + def __init__(self, thread, virtual_banks, calibration): + self.capacity = thread.mem.bank.num + self.depth = thread.frontend.rob_entries + self.alias_base = virtual_banks + (2 if thread.gpdomain.enable else 0) + self.bids = { + i.funct7: ball.id + for ball in thread.balldomain.balls + for i in ball.instructions + } + self.calibration = calibration + self.now = 0 + self.frontier = deque() + self.pending = [] + self.tail = 0 + self.next_alloc = 0 + self.next_issue = 0 + self.mapping = {} + self.current_alias = list(range(self.alias_base)) + self.alias_busy = set() + self.alias_live = set() + self.alias_readers = [0] * (self.alias_base + self.depth) + self.busy_mask = 0 + self.busy_count = 0 + self.count = 0 + self.events = [ + array("Q", [0]), + array("B", [0]), + array("B", [0]), + array("B", [0]), + ] + self.commands = 0 + self.frontend_stalls = 0 + + def record(self): + values = (self.count, len(self.alias_live), self.busy_count) + assert ( + values[0] <= self.capacity + and values[1] <= self.depth + and values[2] <= values[0] + ) + if values != tuple(a[-1] for a in self.events[1:]): + self.events[0].append(self.now) + for column, value in zip(self.events[1:], values): + column.append(value) + + def enqueue(self, event): + f, a, b = event["f"], event["a"], event["b"] + mode = f >> 4 + reads = [a & 1023] if mode in (1, 3, 4) or (f == 32 and b & 4096) else [] + if mode == 4: + reads.append((a >> 10) & 1023) + write = ( + ((a & 1023) if f == 32 and not b & 4096 else (a >> 20) & 1023) + if mode in (2, 3, 4) and f != 35 + else None + ) + assert all(v < self.alias_base for v in reads) and ( + write is None or write < self.alias_base + ) + config = f == 32 + bid = self.bids[f] if f in self.bids else None + uses = set(reads) + if write is not None: + uses.add(write) + read_mask = sum(1 << v for v in set(reads)) + write_mask = 0 if write is None else 1 << write + mask = read_mask | write_mask + deps = [] + for older in self.frontier: + conflict = bool( + read_mask & older["write_mask"] or write_mask & older["mask"] + ) + config_conflict = ( + config + and older["config"] + or (config or older["config"]) + and bool(mask & older["mask"]) + ) + if config_conflict: + deps.append((older, "committed")) + elif conflict or bid is not None and bid == older["bid"]: + deps.append((older, "done")) + alias = self.alias_base + self.tail if write is not None else None + renamed_reads = [self.current_alias[v] for v in reads] + entry = { + "f": f, + "a": a, + "b": b, + "base": event["base"], + "reads": reads, + "write": write, + "writes": set() if write is None else {write}, + "uses": uses, + "config": config, + "bid": bid, + "alias": alias, + "renamed_reads": renamed_reads, + "deps": deps, + "issued": False, + "done": False, + "committed": False, + "allocated": self.now, + "physical": [], + "number": self.commands, + "mask": mask, + "write_mask": write_mask, + } + if alias is not None: + assert alias not in self.alias_live + self.current_alias[write] = alias + self.alias_live.add(alias) + self.alias_busy.add(alias) + for alias_read in renamed_reads: + self.alias_readers[alias_read] += 1 + self.frontier.append(entry) + self.tail = (self.tail + 1) % self.depth + self.next_alloc = self.now + 1 + self.commands += 1 + + def advance(self, incoming, arrival): + while True: + while self.pending and self.pending[0][0] <= self.now: + _, _, entry = heapq.heappop(self.pending) + entry["done"] = True + entry["completed"] = self.now + self.busy_mask &= ~entry["mask"] + self.busy_count -= len(entry["physical"]) + for alias_read in entry["renamed_reads"]: + self.alias_readers[alias_read] -= 1 + if entry["alias"] is not None: + self.alias_busy.remove(entry["alias"]) + committed = False + while ( + self.frontier + and self.frontier[0]["done"] + and self.frontier[0]["completed"] < self.now + ): + entry = self.frontier.popleft() + entry["committed"] = True + committed = True + if entry["alias"] is not None: + self.alias_live.remove(entry["alias"]) + if self.current_alias[entry["write"]] == entry["alias"]: + self.current_alias[entry["write"]] = entry["write"] + if entry["config"]: + v, b = entry["a"] & 1023, entry["b"] + if b & 4096: + target = (entry["a"] >> 20) & 1023 + self.mapping[target] = self.mapping.get( + target, [] + ) + self.mapping.pop(v) + else: + self.mapping.pop(v, None) + if b & 1024: + groups = (b >> 5) & 31 + groups = groups if groups else self.capacity + occupied = {p for ps in self.mapping.values() for p in ps} + available = [ + p for p in range(self.capacity) if p not in occupied + ] + assert len(available) >= groups + self.mapping[v] = available[:groups] + self.count = sum(map(len, self.mapping.values())) + if self.now >= self.next_issue: + for entry in self.frontier: + if ( + entry["issued"] + or entry["allocated"] >= self.now + or entry["mask"] & self.busy_mask + ): + continue + if any(not dep[field] for dep, field in entry["deps"]) or any( + a in self.alias_busy for a in entry["renamed_reads"] + ): + continue + entry["issued"] = True + entry["deps"] = [] + entry["physical"] = [ + p + for v in entry["uses"] + if not entry["config"] + for p in self.mapping[v] + ] + self.busy_mask |= entry["mask"] + self.busy_count += len(entry["physical"]) + f, a, b = entry["f"], entry["a"], entry["b"] + factor = self.calibration[str(f)] + base = entry["base"] + if f in (16, 33, 49, 69, 70, 72) and not ( + f in (16, 33) and b >> 63 + ): + v = ( + (a & 1023) + if f in (16, 49, 69, 70, 72) + else (a >> 20) & 1023 + ) + base *= len(self.mapping[v]) + if ( + factor.get("model") + == "effective_rows*K + bias_rows + writeback_rows + startup" + ): + rows, k = b & 4095, a >> 30 + features = ( + (7.125 if rows == 1 else rows) * k, + ((b >> 24) & 1) * rows, + ((b >> 25) & 1) * rows, + 1, + ) + duration = sum( + x * y for x, y in zip(features, factor["coefficients"]) + ) + else: + duration = base * factor["scale"] + assert duration > 0 + entry["issue_cycle"] = self.now + completion = self.now + math.ceil(duration) + heapq.heappush(self.pending, (completion, entry["number"], entry)) + self.next_issue = self.now + 1 + break + # The RTL forbids allocating on a commit cycle and reusing an alias + # still referenced by any incomplete entry. + alias = self.alias_base + self.tail + alias_referenced = ( + alias in self.alias_busy or self.alias_readers[alias] != 0 + ) + if ( + incoming is not None + and arrival <= self.now + and not committed + and len(self.frontier) < self.depth + and self.now >= self.next_alloc + and not alias_referenced + ): + self.enqueue(incoming) + self.record() + return + self.record() + if incoming is None and not self.frontier: + return + candidates = [] + if incoming is not None and arrival > self.now: + candidates.append(arrival) + if self.pending: + candidates.append(self.pending[0][0]) + if self.frontier and self.frontier[0]["done"]: + candidates.append(max(self.now + 1, self.frontier[0]["completed"] + 1)) + if any( + not e["issued"] + and (not any(not dep[field] for dep, field in e["deps"])) + and not any(a in self.alias_busy for a in e["renamed_reads"]) + and not e["mask"] & self.busy_mask + for e in self.frontier + ): + candidates.append(max(self.now + 1, self.next_issue)) + if committed: + candidates.append(self.now + 1) + if ( + incoming is not None + and len(self.frontier) < self.depth + and not alias_referenced + and self.next_alloc > self.now + ): + candidates.append(self.next_alloc) + assert candidates, "Replay deadlock" + self.now = min(t for t in candidates if t > self.now) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--design-pb", type=Path, required=True) + parser.add_argument("--thread-index", type=int, required=True) + parser.add_argument("--run", required=True) + parser.add_argument("--calibration", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args() + run_log = run_log_dir(get_buckyball_path(), args.run) + assert all( + p.is_absolute() for p in [args.design_pb, args.calibration, args.output_dir] + ) + design = load(args.design_pb) + thread = design.threads[args.thread_index] + core = next(t for t in design.device.cores if args.thread_index in t.thread_indices) + model = Replay( + thread, core.virtual_bank_count, json.loads(args.calibration.read_text()) + ) + started = time.monotonic() + first_cycle = None + with (run_log / "bdb.ndjson").open() as f: + for line in f: + e = json.loads(line) + if e["type"] != "itrace": + continue + if first_cycle is None: + first_cycle = e["issue_cycle"] + funct = int(e["funct"], 16) + if funct == 1: + model.advance(None, 0) + continue + base = e["modeled_cycles"] + if base is None: + raise ValueError("This kernel has no validated timing model") + model.advance( + { + "f": funct, + "a": int(e["rs1"], 16), + "b": int(e["rs2"], 16), + "base": base, + }, + e["issue_cycle"] - first_cycle, + ) + model.advance(None, 0) + model.events[0].append(model.now) + for a in model.events[1:]: + a.append(a[-1]) + args.output_dir.mkdir(exist_ok=True, parents=True) + values = np.array(model.events, dtype=np.uint64) + np.savez_compressed( + args.output_dir / "bank-replay-events.npz", + cycle=values[0], + allocated_physical_banks=values[1], + live_bat_aliases=values[2], + busy_physical_banks=values[3], + ) + result = { + "method": ( + "Event-driven replay of architectural-bank dependencies before the DMA issue-order " + "fix, BAT alias lifetime and MSET physical mappings. Completion latencies are RTL- " + "calibrated estimates; BEMU CPU issue times are arrival constraints. Not cycle- " + "accurate RTL or FPGA measurement." + ), + "limitations": [ + ( + "The four-slot thread admission interface is not modeled; ROB occupancy may exceed " + "what the current issue path can supply." + ), + "The per-producer DMA issue-order dependency added after the MVOUT deadlock diagnosis is not modeled.", + "Issue transport/endpoint backpressure is not modeled cycle by cycle.", + "Latency calibration extrapolates outside measured shapes.", + "Alias count is not a physical-bank count.", + ], + "log_dir": str(run_log), + "design_pb": str(args.design_pb), + "thread_index": args.thread_index, + "rob_entries": model.depth, + "physical_banks": model.capacity, + "bank_bytes": thread.mem.bank.width // 8 * thread.mem.bank.entries, + "instructions": model.commands, + "estimated_cycles": model.now, + "offline_seconds": time.monotonic() - started, + "physical_allocation_peak": int(values[1].max()), + "bat_alias_peak": int(values[2].max()), + "busy_physical_peak": int(values[3].max()), + "final_physical_banks": model.count, + "final_bat_aliases": len(model.alias_live), + } + (args.output_dir / "results.json").write_text(json.dumps(result, indent=2) + "\n") + fig, axes = plt.subplots(3, 1, figsize=(11, 7), constrained_layout=True) + for ax, y, label, capacity, color in zip( + axes, + values[1:], + [ + "Mapped physical data banks", + "Live BAT aliases", + "In-flight reserved physical banks", + ], + [model.capacity, model.depth, model.capacity], + ["#2563eb", "#d97706", "#0d9488"], + ): + ax.step(values[0] / 1e6, y, where="post", linewidth=0.65, color=color) + ax.set_ylabel(label, fontsize=9) + ax.set_ylim(0, capacity + 1) + ax.grid(alpha=0.2) + axes[-1].set_xlabel("Estimated replay cycles (million; not FPGA time)") + fig.savefig(args.output_dir / "bank-replay.png", dpi=180) + fig.savefig(args.output_dir / "bank-replay.pdf") + print(json.dumps(result, indent=2), flush=True) diff --git a/bbdev/api/steps/bebop/bemu/scripts/bank_replay_native.py b/bbdev/api/steps/bebop/bemu/scripts/bank_replay_native.py new file mode 100644 index 000000000..ca667cd57 --- /dev/null +++ b/bbdev/api/steps/bebop/bemu/scripts/bank_replay_native.py @@ -0,0 +1,459 @@ +import argparse +import json +import math +from pathlib import Path +import sys +import time + +import numpy as np +from numba import njit + +sys.path.insert(0, str(Path(__file__).resolve().parents[6])) +from stack.scripts.design import load + +sys.path.insert(0, str(Path(__file__).resolve().parents[4])) +from utils.path import get_buckyball_path, run_log_dir + + +@njit(cache=True) +def simulate( + commands, capacity, depth, alias_base, bids, scales, shape_model, coefficients +): + valid = np.zeros(depth, np.bool_) + issued = np.zeros(depth, np.bool_) + done = np.zeros(depth, np.bool_) + config = np.zeros(depth, np.bool_) + function = np.zeros(depth, np.int64) + operand1 = np.zeros(depth, np.uint64) + operand2 = np.zeros(depth, np.uint64) + base_cycles = np.zeros(depth, np.float64) + write_bank = np.full(depth, -1, np.int64) + read0 = np.full(depth, -1, np.int64) + read1 = np.full(depth, -1, np.int64) + uses = np.zeros(depth, np.uint64) + writes = np.zeros(depth, np.uint64) + dependencies = np.zeros(depth, np.uint64) + config_dependencies = np.zeros(depth, np.uint64) + allocated = np.zeros(depth, np.int64) + completion = np.zeros(depth, np.int64) + physical_count = np.zeros(depth, np.int64) + current_alias = np.arange(alias_base, dtype=np.int64) + alias_busy = np.zeros(depth, np.bool_) + alias_live = np.zeros(depth, np.bool_) + alias_readers = np.zeros(alias_base + depth, np.int64) + owners = np.full(capacity, -1, np.int64) + groups = np.zeros(alias_base, np.int64) + n = len(commands) + times = np.zeros(n * 4 + 2, np.uint64) + mapped = np.zeros(n * 4 + 2, np.uint8) + aliases = np.zeros(n * 4 + 2, np.uint8) + reserved = np.zeros(n * 4 + 2, np.uint8) + points = 1 + now = 0 + head = 0 + tail = 0 + live_entries = 0 + mapped_count = 0 + alias_count = 0 + busy_count = 0 + busy_mask = np.uint64(0) + next_alloc = 0 + next_issue = 0 + index = 0 + while True: + completed_mask = np.uint64(0) + for slot in range(depth): + if ( + valid[slot] + and issued[slot] + and not done[slot] + and completion[slot] <= now + ): + done[slot] = True + completed_mask |= np.uint64(1) << np.uint64(slot) + busy_mask &= ~uses[slot] + busy_count -= physical_count[slot] + if write_bank[slot] >= 0: + alias_busy[slot] = False + if read0[slot] >= 0: + alias_readers[read0[slot]] -= 1 + if read1[slot] >= 0: + alias_readers[read1[slot]] -= 1 + committed = False + committed_mask = np.uint64(0) + while live_entries and done[head] and completion[head] < now: + slot = head + valid[slot] = False + live_entries -= 1 + committed = True + committed_mask |= np.uint64(1) << np.uint64(slot) + if write_bank[slot] >= 0: + alias_live[slot] = False + alias_count -= 1 + if current_alias[write_bank[slot]] == alias_base + slot: + current_alias[write_bank[slot]] = write_bank[slot] + if config[slot]: + v = np.int64(operand1[slot] & np.uint64(1023)) + b = operand2[slot] + if b & np.uint64(4096): + target = np.int64( + (operand1[slot] >> np.uint64(20)) & np.uint64(1023) + ) + for p in range(capacity): + if owners[p] == v: + owners[p] = target + groups[target] += groups[v] + groups[v] = 0 + else: + mapped_count -= groups[v] + groups[v] = 0 + for p in range(capacity): + if owners[p] == v: + owners[p] = -1 + if b & np.uint64(1024): + count = np.int64((b >> np.uint64(5)) & np.uint64(31)) + if count == 0: + count = capacity + for p in range(capacity): + if owners[p] < 0 and count: + owners[p] = v + count -= 1 + groups[v] += 1 + mapped_count += 1 + assert count == 0 + head = (head + 1) % depth + for slot in range(depth): + dependencies[slot] &= ~(completed_mask | committed_mask) + config_dependencies[slot] &= ~committed_mask + if now >= next_issue: + for offset in range(depth): + slot = (head + offset) % depth + if ( + not valid[slot] + or issued[slot] + or allocated[slot] >= now + or uses[slot] & busy_mask + ): + continue + if dependencies[slot] or config_dependencies[slot]: + continue + if read0[slot] >= alias_base and alias_busy[read0[slot] - alias_base]: + continue + if read1[slot] >= alias_base and alias_busy[read1[slot] - alias_base]: + continue + issued[slot] = True + busy_mask |= uses[slot] + count = 0 + if not config[slot]: + for v in range(alias_base): + if uses[slot] & (np.uint64(1) << np.uint64(v)): + assert groups[v] > 0 + count += groups[v] + physical_count[slot] = count + busy_count += count + f = function[slot] + a = operand1[slot] + b = operand2[slot] + base = base_cycles[slot] + if f == 16 or f == 33 or f == 49 or f == 69 or f == 70 or f == 72: + if not ((f == 16 or f == 33) and b >> np.uint64(63)): + v = ( + np.int64(a & np.uint64(1023)) + if f == 16 or f == 49 or f == 69 or f == 70 or f == 72 + else np.int64((a >> np.uint64(20)) & np.uint64(1023)) + ) + base *= groups[v] + if shape_model[f]: + rows = float(b & np.uint64(4095)) + k = float(a >> np.uint64(30)) + effective = 7.125 if rows == 1 else rows + duration = coefficients[f, 0] * (effective * k) + duration += ( + coefficients[f, 1] + * float((b >> np.uint64(24)) & np.uint64(1)) + * rows + ) + duration += ( + coefficients[f, 2] + * float((b >> np.uint64(25)) & np.uint64(1)) + * rows + ) + duration += coefficients[f, 3] + else: + duration = base * scales[f] + assert duration > 0 + completion[slot] = now + np.int64(math.ceil(duration)) + next_issue = now + 1 + break + can_allocate = ( + index < n + and not committed + and live_entries < depth + and now >= next_alloc + and not alias_busy[tail] + and alias_readers[alias_base + tail] == 0 + ) + if can_allocate and np.int64(commands[index]["arrival"]) <= now: + slot = tail + f = np.int64(commands[index]["f"]) + a = commands[index]["a"] + b = commands[index]["b"] + mode = f >> 4 + r0 = ( + np.int64(a & np.uint64(1023)) + if mode == 1 + or mode == 3 + or mode == 4 + or f == 32 + and b & np.uint64(4096) + else -1 + ) + r1 = np.int64((a >> np.uint64(10)) & np.uint64(1023)) if mode == 4 else -1 + w = -1 + if (mode == 2 or mode == 3 or mode == 4) and f != 35: + w = ( + np.int64(a & np.uint64(1023)) + if f == 32 and not b & np.uint64(4096) + else np.int64((a >> np.uint64(20)) & np.uint64(1023)) + ) + assert r0 < alias_base and r1 < alias_base and w < alias_base + read_mask = np.uint64(0) + if r0 >= 0: + read_mask |= np.uint64(1) << np.uint64(r0) + if r1 >= 0: + read_mask |= np.uint64(1) << np.uint64(r1) + write_mask = np.uint64(0) if w < 0 else np.uint64(1) << np.uint64(w) + mask = read_mask | write_mask + deps = np.uint64(0) + conf_deps = np.uint64(0) + for older in range(depth): + if not valid[older]: + continue + config_conflict = ( + f == 32 + and config[older] + or (f == 32 or config[older]) + and bool(mask & uses[older]) + ) + conflict = bool(read_mask & writes[older] or write_mask & uses[older]) + same_ball = bids[f] >= 0 and bids[f] == bids[function[older]] + if config_conflict: + conf_deps |= np.uint64(1) << np.uint64(older) + elif not done[older] and (conflict or same_ball): + deps |= np.uint64(1) << np.uint64(older) + valid[slot] = True + issued[slot] = False + done[slot] = False + config[slot] = f == 32 + function[slot] = f + operand1[slot] = a + operand2[slot] = b + base_cycles[slot] = commands[index]["base"] + allocated[slot] = now + physical_count[slot] = 0 + write_bank[slot] = w + uses[slot] = mask + writes[slot] = write_mask + dependencies[slot] = deps + config_dependencies[slot] = conf_deps + read0[slot] = -1 if r0 < 0 else current_alias[r0] + read1[slot] = -1 if r1 < 0 else current_alias[r1] + if read0[slot] >= 0: + alias_readers[read0[slot]] += 1 + if read1[slot] >= 0: + alias_readers[read1[slot]] += 1 + if w >= 0: + assert not alias_live[slot] + current_alias[w] = alias_base + slot + alias_live[slot] = True + alias_busy[slot] = True + alias_count += 1 + tail = (tail + 1) % depth + live_entries += 1 + index += 1 + next_alloc = now + 1 + admitted = True + else: + admitted = False + assert ( + busy_count >= 0 + and busy_count <= mapped_count + and mapped_count <= capacity + and alias_count <= depth + ) + if ( + mapped_count != mapped[points - 1] + or alias_count != aliases[points - 1] + or busy_count != reserved[points - 1] + ): + times[points] = now + mapped[points] = mapped_count + aliases[points] = alias_count + reserved[points] = busy_count + points += 1 + if admitted: + continue + if index == n and live_entries == 0: + break + next_time = np.int64(9223372036854775807) + if index < n and np.int64(commands[index]["arrival"]) > now: + next_time = min(next_time, np.int64(commands[index]["arrival"])) + for slot in range(depth): + if not valid[slot]: + continue + if issued[slot] and not done[slot]: + next_time = min(next_time, completion[slot]) + if ( + not issued[slot] + and not dependencies[slot] + and not config_dependencies[slot] + and not uses[slot] & busy_mask + ): + hazard = ( + read0[slot] >= alias_base + and alias_busy[read0[slot] - alias_base] + or read1[slot] >= alias_base + and alias_busy[read1[slot] - alias_base] + ) + if not hazard: + next_time = min(next_time, max(now + 1, next_issue)) + if live_entries and done[head]: + next_time = min(next_time, max(now + 1, completion[head] + 1)) + if committed: + next_time = min(next_time, now + 1) + if ( + index < n + and live_entries < depth + and not alias_busy[tail] + and alias_readers[alias_base + tail] == 0 + and next_alloc > now + ): + next_time = min(next_time, next_alloc) + assert next_time > now and next_time < 9223372036854775807 + now = next_time + times[points] = now + mapped[points] = mapped_count + aliases[points] = alias_count + reserved[points] = busy_count + return ( + times[: points + 1], + mapped[: points + 1], + aliases[: points + 1], + reserved[: points + 1], + now, + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--design-pb", type=Path, required=True) + parser.add_argument("--thread-index", type=int, required=True) + parser.add_argument("--run", required=True) + parser.add_argument("--calibration", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args() + run_log = run_log_dir(get_buckyball_path(), args.run) + assert all( + p.is_absolute() for p in [args.design_pb, args.calibration, args.output_dir] + ) + design = load(args.design_pb) + thread = design.threads[args.thread_index] + core = next(t for t in design.device.cores if args.thread_index in t.thread_indices) + capacity = thread.mem.bank.num + depth = thread.frontend.rob_entries + alias_base = core.virtual_bank_count + (2 if thread.gpdomain.enable else 0) + assert capacity < 256 and depth < 64 and alias_base < 64 + calibration = json.loads(args.calibration.read_text()) + bids = np.full(128, -1, np.int64) + scales = np.zeros(128) + shape_model = np.zeros(128, np.bool_) + coefficients = np.zeros((128, 4)) + for ball in thread.balldomain.balls: + for isa in ball.instructions: + bids[isa.funct7] = ball.id + for f, row in calibration.items(): + if "model" in row: + assert ( + row["model"] + == "effective_rows*K + bias_rows + writeback_rows + startup" + ) + shape_model[int(f)] = True + coefficients[int(f)] = row["coefficients"] + else: + scales[int(f)] = row["scale"] + dtype = np.dtype( + [("f", "u1"), ("a", "u8"), ("b", "u8"), ("base", "u8"), ("arrival", "u8")] + ) + started = time.monotonic() + + def instructions(): + first = None + with (run_log / "bdb.ndjson").open() as stream: + for line in stream: + e = json.loads(line) + if e["type"] != "itrace": + continue + f = int(e["funct"], 16) + assert f != 1, "Barrier replay requires a separate drain boundary" + if first is None: + first = e["issue_cycle"] + assert e["modeled_cycles"] is not None + assert str(f) in calibration + yield ( + f, + int(e["rs1"], 16), + int(e["rs2"], 16), + e["modeled_cycles"], + e["issue_cycle"] - first, + ) + + commands = np.fromiter(instructions(), dtype=dtype) + parsed = time.monotonic() + t, m, a, b, end = simulate( + commands, capacity, depth, alias_base, bids, scales, shape_model, coefficients + ) + finished = time.monotonic() + args.output_dir.mkdir(parents=True, exist_ok=True) + np.savez_compressed( + args.output_dir / "bank-replay-events.npz", + cycle=t, + allocated_physical_banks=m, + live_bat_aliases=a, + busy_physical_banks=b, + ) + result = { + "method": ( + "Native event-driven replay of architectural-bank dependencies before the DMA issue- " + "order fix, BAT alias lifetime and MSET physical mappings. Calibrated completion " + "estimates; not cycle-accurate RTL or FPGA measurement." + ), + "limitations": [ + ( + "The four-slot thread admission interface is not modeled; ROB occupancy may exceed " + "what the current issue path can supply." + ), + "The per-producer DMA issue-order dependency added after the MVOUT deadlock diagnosis is not modeled.", + "Issue transport/endpoint backpressure is not modeled cycle by cycle.", + "Physical mapping changes are approximated at MSET commit, rather than actual backend config handshake.", + "Latency calibration extrapolates outside measured shapes.", + "Alias count is not a physical-bank count.", + ], + "log_dir": str(run_log), + "design_pb": str(args.design_pb), + "thread_index": args.thread_index, + "rob_entries": depth, + "physical_banks": capacity, + "bank_bytes": thread.mem.bank.width // 8 * thread.mem.bank.entries, + "instructions": len(commands), + "estimated_cycles": end, + "offline_seconds": finished - started, + "parse_seconds": parsed - started, + "replay_and_jit_seconds": finished - parsed, + "physical_allocation_peak": int(m.max()), + "bat_alias_peak": int(a.max()), + "busy_physical_peak": int(b.max()), + "final_physical_banks": int(m[-1]), + "final_bat_aliases": int(a[-1]), + } + (args.output_dir / "results.json").write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps(result, indent=2), flush=True) diff --git a/bbdev/api/steps/bebop/bemu/scripts/bemu_analysis.py b/bbdev/api/steps/bebop/bemu/scripts/bemu_analysis.py new file mode 100644 index 000000000..d77c783c8 --- /dev/null +++ b/bbdev/api/steps/bebop/bemu/scripts/bemu_analysis.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +"""Summarize bebop-bemu itrace/mtrace in bdb.ndjson.""" + +from __future__ import annotations + +import argparse +import json +import sys +from collections import Counter +from pathlib import Path + +if str(Path(__file__).resolve().parents[4]) not in sys.path: + sys.path.insert(0, str(Path(__file__).resolve().parents[4])) + + +def _hex_u(value: object, field: str, line_no: int) -> int: + if not isinstance(value, str) or not value.startswith("0x"): + raise ValueError(f"bdb.ndjson:{line_no}: {field} must be a 0x hex string") + return int(value, 16) + + +def design_maps( + bbdir: str, design: str, selection: str +) -> tuple[dict[int, str], set[int], int]: + system_isa = { + 0: "fence", + 1: "barrier", + 16: "mvout", + 32: "mset", + 33: "mvin", + 34: "mmio_set", + 35: "mvin_mmio", + } + from utils.path import load_design_config + + data = load_design_config(bbdir, design, selection=selection) + configurations = { + ( + p.thread.balldomain.SerializeToString(deterministic=True), + p.thread.mem.bank.entries, + ): p.thread + for p in data.profiles + } + if len(configurations) != 1: + raise ValueError( + f"design {design} requires a unique instruction map and bank depth for this analysis" + ) + config = next(iter(configurations.values())) + names: dict[int, str] = dict(system_isa) + matrix: set[int] = set() + for instance in config.balldomain.balls: + for instruction in instance.instructions: + funct, mnemonic = instruction.funct7, instruction.mnemonic + if funct in names: + raise ValueError( + f"design {design}: duplicate or reserved funct7 {funct}" + ) + names[funct] = mnemonic.lower() + if mnemonic in {"MATRIX", "MATRIX_F32", "SMATMUL_OS"}: + matrix.add(funct) + return names, matrix, config.mem.bank.entries + + +def _mnk(rs1: int, rs2: int) -> tuple[int, int, int]: + return rs2 & 0xFFF, (rs2 >> 12) & 0xFFF, rs1 >> 30 + + +def analysis_dir( + log_dir: Path, + *, + names: dict[int, str], + matrix: set[int], + bank_depth: int, + itrace: bool, + mtrace: bool, +) -> str: + if bank_depth <= 0: + raise ValueError(f"bank_depth must be a positive int, got {bank_depth}") + if not log_dir.is_dir(): + raise ValueError(f"run is not a directory: {log_dir}") + bdb = log_dir / "bdb.ndjson" + if not bdb.is_file(): + raise ValueError(f"missing bdb.ndjson: {bdb}") + + n_itrace = 0 + n_mtrace = 0 + prev_event: int | None = None + funct_n: Counter[str] = Counter() + mnk_n: Counter[tuple[int, int, int]] = Counter() + bank_n: Counter[int] = Counter() + access_n: Counter[str] = Counter() + + with bdb.open(encoding="utf-8") as source: + for line_no, raw in enumerate(source, start=1): + line = raw.strip() + if not line: + continue + try: + obj = json.loads(line) + except json.JSONDecodeError as exc: + raise ValueError(f"bdb.ndjson:{line_no}: invalid JSON") from exc + if not isinstance(obj, dict): + raise ValueError(f"bdb.ndjson:{line_no}: line must be a JSON object") + kind = obj.get("type") + if kind == "itrace": + n_itrace += 1 + event_index = obj["event_index"] + if not isinstance(event_index, int) or event_index < 0: + raise ValueError( + f"bdb.ndjson:{line_no}: event_index must be a non-negative int" + ) + if prev_event is not None and event_index <= prev_event: + raise ValueError( + f"bdb.ndjson:{line_no}: instruction event order is not increasing" + ) + funct = _hex_u(obj.get("funct"), "funct", line_no) + if funct not in names: + known = ", ".join( + f"{v}=0x{k:02x}" for k, v in sorted(names.items()) + ) + raise ValueError( + f"bdb.ndjson:{line_no}: unknown funct 0x{funct:02x}; known: {known}" + ) + name = names[funct] + funct_n[name] += 1 + prev_event = event_index + if funct in matrix: + rs1 = _hex_u(obj.get("rs1"), "rs1", line_no) + rs2 = _hex_u(obj.get("rs2"), "rs2", line_no) + mnk_n[_mnk(rs1, rs2)] += 1 + elif kind == "mtrace": + n_mtrace += 1 + bank_n[obj["vbank_id"]] += 1 + access_n[obj["event"]] += 1 + elif kind is None: + raise ValueError(f"bdb.ndjson:{line_no}: missing type") + + if itrace and n_itrace == 0: + raise ValueError(f"no itrace events in {bdb}") + if mtrace and n_mtrace == 0: + raise ValueError(f"no mtrace events in {bdb}") + + lines: list[str] = [f"log_dir: {log_dir}"] + if itrace: + lines.append("== itrace ==") + lines.append(f"events: {n_itrace}") + lines.append(f"{'funct':<16} {'n':>10} {'pct':>8}") + for name, count in funct_n.most_common(): + pct = 100.0 * count / n_itrace + lines.append(f"{name:<16} {count:>10} {pct:>7.1f}%") + lines.append(f"fence: {funct_n.get('fence', 0)}") + lines.append(f"mset: {funct_n.get('mset', 0)}") + if matrix: + lines.append("matrix (M,N,K):") + if not mnk_n: + lines.append(" (none)") + else: + for (m, n, k), count in mnk_n.most_common(): + lines.append(f" ({m},{n},{k}) n={count}") + if mtrace: + lines.append("== mtrace ==") + lines.append(f"events: {n_mtrace}") + lines.append(f"bank_depth: {bank_depth}") + lines.append(f"reads: {access_n['read']}") + lines.append(f"writes: {access_n['write']}") + lines.append("bank events:") + for bank, count in sorted(bank_n.items()): + lines.append(f" {bank}: {count}") + return "\n".join(lines) + "\n" + + +def write_report(log_dir: Path, text: str) -> Path: + out = log_dir / "analysis.txt" + out.write_text(text, encoding="utf-8") + return out + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--design", required=True) + parser.add_argument("--target", required=True) + parser.add_argument("--run", required=True) + parser.add_argument("--bbdir", type=Path) + parser.add_argument("--itrace", action="store_true") + parser.add_argument("--mtrace", action="store_true") + args = parser.parse_args(argv) + if not args.itrace and not args.mtrace: + raise SystemExit("need --itrace and/or --mtrace") + api_root = Path(__file__).resolve().parents[4] + if str(api_root) not in sys.path: + sys.path.insert(0, str(api_root)) + from utils.path import get_buckyball_path, run_log_dir + + bbdir = args.bbdir if args.bbdir is not None else Path(get_buckyball_path()) + log_dir = run_log_dir(bbdir, args.run) + names, matrix, depth = design_maps(str(bbdir), args.design, args.target) + text = analysis_dir( + log_dir, + names=names, + matrix=matrix, + bank_depth=depth, + itrace=args.itrace, + mtrace=args.mtrace, + ) + write_report(log_dir, text) + sys.stdout.write(text) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except ValueError as exc: + print(exc, file=sys.stderr) + raise SystemExit(1) diff --git a/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py b/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py new file mode 100644 index 000000000..9128a1543 --- /dev/null +++ b/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py @@ -0,0 +1,23 @@ +from pathlib import Path + +from utils.event_common import require_design +from utils.path import runner_manifest + + +def _repo(bbdir: str | None) -> Path: + from utils.path import get_buckyball_path + + return Path(bbdir or get_buckyball_path()) + + +def bemu_manifest(design: str, bbdir: str | None = None, *, selection: str) -> Path: + design = require_design({"design": design}) + root = _repo(bbdir) + path = runner_manifest(root, design, selection) + if not path.is_file(): + raise FileNotFoundError(f"missing design manifest: {path}") + return path + + +def bemu_binary() -> str: + return "bebop-bemu" diff --git a/bbdev/api/steps/bebop/p2e/00_clean_api.step.py b/bbdev/api/steps/bebop/p2e/00_clean_api.step.py new file mode 100644 index 000000000..b2bc18016 --- /dev/null +++ b/bbdev/api/steps/bebop/p2e/00_clean_api.step.py @@ -0,0 +1,38 @@ +from utils.event_common import require_target +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.event_common import require_design +from utils.path import get_buckyball_path + +config = { + "name": "bebop-p2e-clean-api", + "description": "Clean P2E build directory", + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/p2e/clean")], + "enqueues": ["bebop.p2e.clean"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = req.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) + await ctx.enqueue( + { + "topic": "bebop.p2e.clean", + "data": { + **body, + "design": design, + "task": "clean", + "_trace_id": ctx.trace_id, + "target": selection, + }, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/p2e/00_clean_event.step.py b/bbdev/api/steps/bebop/p2e/00_clean_event.step.py new file mode 100644 index 000000000..ed4bb2a0f --- /dev/null +++ b/bbdev/api/steps/bebop/p2e/00_clean_event.step.py @@ -0,0 +1,62 @@ +import os +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.event_common import require_design +from utils.path import get_buckyball_path, rtl_dir +from utils.stream_run import stream_run_logger_async +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "bebop-p2e-clean", + "description": "Clean P2E build directory", + "flows": ["bebop"], + "triggers": [queue("bebop.p2e.clean")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + try: + design = require_design(input_data) + except ValueError as error: + ctx.logger.error(str(error)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "missing_design"}, + trace_id=origin_tid, + ) + return + build_dir = rtl_dir( + bbdir, + design, + "p2e", + input_data.get("output_dir"), + selection=input_data["target"], + ) + + command = f"rm -rf {build_dir}" + result = await stream_run_logger_async( + cmd=command, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="bebop p2e clean", + stderr_prefix="bebop p2e clean", + ) + + await check_result( + ctx, + result.returncode, + continue_run=False, + extra_fields={"task": "clean", "output_dir": build_dir}, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/bebop/p2e/01_verilog_api.step.py b/bbdev/api/steps/bebop/p2e/01_verilog_api.step.py new file mode 100644 index 000000000..fb0520eed --- /dev/null +++ b/bbdev/api/steps/bebop/p2e/01_verilog_api.step.py @@ -0,0 +1,35 @@ +from utils.event_common import require_target +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.event_common import require_design +from utils.path import get_buckyball_path + +config = { + "name": "bebop-p2e-verilog-api", + "description": "Generate SystemVerilog for P2E DDR4 backdoor", + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/p2e/verilog")], + "enqueues": ["bebop.p2e.verilog"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = req.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) + data = {"design": design, "diff": bool(body.get("diff", False))} + if body.get("output-dir"): + data["output_dir"] = body["output-dir"] + await ctx.enqueue( + { + "topic": "bebop.p2e.verilog", + "data": {**data, "_trace_id": ctx.trace_id, "target": selection}, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py b/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py new file mode 100644 index 000000000..30160fbf4 --- /dev/null +++ b/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py @@ -0,0 +1,50 @@ +from utils.event_common import require_target +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.event_common import require_design +from utils.path import get_buckyball_path, rtl_dir + +config = { + "name": "bebop-p2e-buildbitstream-api", + "description": "Build Bebop P2E runtime case", + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/p2e/buildbitstream")], + "enqueues": ["bebop.p2e.buildbitstream"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + bbdir = get_buckyball_path() + body = request.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) + vsrc_dir = rtl_dir( + bbdir, + design, + "p2e", + body.get("vsrc_dir") or body.get("vsrc-dir"), + selection=selection, + ) + + output_dir = body.get("output_dir") or body.get("output-dir") + data = { + "design": design, + "diff": bool(body.get("diff", False)), + "itrace": bool(body.get("itrace", False)), + "mtrace": bool(body.get("mtrace", False)), + "vsrc_dir": vsrc_dir, + "output_dir": output_dir, + } + await ctx.enqueue( + { + "topic": "bebop.p2e.buildbitstream", + "data": {**data, "_trace_id": ctx.trace_id, "target": selection}, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py new file mode 100644 index 000000000..81fc1527d --- /dev/null +++ b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py @@ -0,0 +1,169 @@ +""" +bebop p2e buildbitstream event handler + +Builds the P2E VVAC runtime case via bebop CLI: + 1. Resolve Verilog source directory (VSRC_PATH) from config + 2. Run bebop build p2e with rtl_dir and out_dir + 3. Validate generated bitstream and runtime artifacts +""" + +from utils.path import runner_manifest + +import os +import shlex +import shutil +import sys +from datetime import datetime +from pathlib import Path + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.event_common import require_design +from utils.path import bebop_target_dir, get_buckyball_path, rtl_dir +from steps.bebop.p2e.scripts.timescale import normalize_p2e_timescale +from utils.stream_run import stream_run_logger_async +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "bebop-p2e-buildbitstream", + "description": "Build Bebop P2E runtime case", + "flows": ["bebop"], + "triggers": [queue("bebop.p2e.buildbitstream")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + bebop_dir = f"{bbdir}/bebop" + + try: + design = require_design(input_data) + except ValueError as error: + ctx.logger.error(str(error)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "missing_design"}, + trace_id=origin_tid, + ) + return + vsrc_dir = rtl_dir( + bbdir, design, "p2e", input_data.get("vsrc_dir"), selection=input_data["target"] + ) + normalize_p2e_timescale(vsrc_dir, ctx.logger) + timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") + diff = bool(input_data.get("diff", False)) + build_dir = input_data.get("output_dir") or str( + Path(bbdir) + / "examples/designs" + / design + / "output/p2e" + / f"{timestamp}{'-diff' if diff else ''}" + ) + build_path = Path(build_dir).resolve() + build_root = (Path(bbdir) / "examples/designs" / design / "output/p2e").resolve() + if build_root not in build_path.parents: + raise ValueError(f"P2E output_dir must be under {build_root}: {build_path}") + if build_path.exists(): + shutil.rmtree(build_path) + build_path.mkdir(parents=True) + + manifest = runner_manifest(bbdir, design, input_data["target"]) + features = ["p2e"] + for trace in ("itrace", "mtrace"): + if input_data.get(trace, False): + features.append(f"bebop-p2e/{trace}") + build_env = { + **os.environ, + "VSRC_PATH": vsrc_dir, + "OUT_PATH": str(build_path), + } + if diff: + features.append("bemu") + build_cmd = shlex.join( + [ + "nix", + "develop", + "--ignore-env", + "--keep-env-var", + "HOME", + "--keep-env-var", + "ALL_PROXY", + "--keep-env-var", + "VSRC_PATH", + "--keep-env-var", + "OUT_PATH", + "-c", + "env", + "-u", + "LD_LIBRARY_PATH", + "cargo", + "run", + "--release", + "--manifest-path", + str(manifest), + "--target-dir", + bebop_target_dir(bbdir, design, "p2e", selection=input_data["target"]), + "--bin", + "bebop", + "--features", + ",".join(features), + "--", + "build", + "p2e", + "--design-dir", + str(Path(bbdir) / "examples/designs" / design), + "--entry-dir", + str(Path(manifest).parent), + "--rtl-dir", + vsrc_dir, + "--out-dir", + build_dir, + *(["--diff"] if diff else []), + *(["--itrace"] if input_data.get("itrace", False) else []), + *(["--mtrace"] if input_data.get("mtrace", False) else []), + ] + ) + build_cmd = f"cd {shlex.quote(bbdir)} && {build_cmd}" + ctx.logger.info("Building bebop p2e runtime case ...") + build_result = await stream_run_logger_async( + cmd=build_cmd, + logger=ctx.logger, + cwd=bebop_dir, + stdout_prefix="bebop p2e build", + stderr_prefix="bebop p2e build", + env=build_env, + ) + + rtcfg_path = os.path.join(build_dir, "vvacDir", "runtimeDir", "rtcfg") + libvctb_path = os.path.join( + build_dir, "vvacDir", "runtimeDir", "lib", "lib_arm", "libvCtb.so" + ) + bitstream_path = os.path.join(build_dir, "fpgaCompDir", "bitstream.bit") + runtime_path = os.path.join(build_dir, "bebop-p2e") + extra_fields = { + "task": "build", + "vsrc_dir": vsrc_dir, + "build_dir": build_dir, + "rtcfg": rtcfg_path, + "libvCtb": libvctb_path, + "bitstream": bitstream_path, + "runtime": runtime_path, + "diff": diff, + "timestamp": timestamp, + } + extra_fields["runtime_ready"] = build_result.returncode == 0 + await check_result( + ctx, + build_result.returncode, + continue_run=False, + extra_fields=extra_fields, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/bebop/p2e/03_runworkload_api.step.py b/bbdev/api/steps/bebop/p2e/03_runworkload_api.step.py new file mode 100644 index 000000000..1e3bef4c4 --- /dev/null +++ b/bbdev/api/steps/bebop/p2e/03_runworkload_api.step.py @@ -0,0 +1,42 @@ +from utils.event_common import require_target + +from motia import ApiRequest, ApiResponse, FlowContext, api +from steps.bebop.p2e.scripts.runtime_case import validate_runtime_reuse + +config = { + "name": "bebop-p2e-runworkload-api", + "description": "Run workload on FPGA via bebop p2e CLI", + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/p2e/runworkload")], + "enqueues": ["bebop.p2e.runworkload"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + if not body.get("image") or not body.get("bitstream"): + return ApiResponse( + status=400, body={"error": "image and bitstream are required"} + ) + bitstream = body["bitstream"] + if body.get("reuse-runtime", False): + try: + validate_runtime_reuse( + bitstream, + bool(body.get("diff", False)), + bool(body.get("itrace", False)), + bool(body.get("mtrace", False)), + ) + except (ValueError, OSError) as error: + return ApiResponse(status=400, body={"error": str(error)}) + await ctx.enqueue( + { + "topic": "bebop.p2e.runworkload", + "data": {**body, "_trace_id": ctx.trace_id, "target": selection}, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py new file mode 100644 index 000000000..0bb0d3a85 --- /dev/null +++ b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py @@ -0,0 +1,381 @@ +""" +bebop p2e runworkload event handler + +Loads a kernel image into FPGA and runs the workload via bebop CLI: + 1. Resolve image name to .hex file under bb-tests/output/ + 2. Validate bitstream .bit file path + 3. Run bebop --image --bitstream [--multi-fpga] [--wave] [--wave-start ] +""" + +from utils.path import runner_manifest + + +import glob +import hashlib +import os +import re +import shlex +import shutil +import sys +from datetime import datetime +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) +from utils.event_common import require_design +from utils.path import bebop_target_dir, bebop_bin_path, get_buckyball_path, log_dir +from utils.stream_run import stream_run_logger_async +from utils.event_common import check_result, get_origin_trace_id +from resolve_image import resolve_image +from steps.bebop.p2e.scripts.runtime_case import trace_modes, validate_runtime_reuse +from steps.bebop.performance_report import performance_report +from utils.reports import source_context +from pathlib import Path + +config = { + "name": "bebop-p2e-runworkload", + "description": "Run workload on FPGA via bebop p2e CLI", + "flows": ["bebop"], + "triggers": [queue("bebop.p2e.runworkload")], + "enqueues": [], +} + + +def resolve_runtime_config(bitstream: str, requested_config: object) -> str: + if isinstance(requested_config, str) and requested_config: + return requested_config + build_dir = os.path.dirname(os.path.dirname(os.path.abspath(bitstream))) + case_name = os.path.basename(build_dir) + return re.sub("-\\d{4}-\\d{2}-\\d{2}-\\d{2}-\\d{2}$", "", case_name) + + +def case_uses_multi_fpga(build_dir: str) -> bool: + fpga_comp_dir = os.path.join(build_dir, "fpgaCompDir") + part_dirs = glob.glob(os.path.join(fpga_comp_dir, "part_b*_f*")) + return len([path for path in part_dirs if os.path.isdir(path)]) > 1 + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + try: + design = require_design(input_data) + except ValueError as error: + ctx.logger.error(str(error)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "missing_design"}, + trace_id=origin_tid, + ) + return + bbdir = get_buckyball_path() + bebop_dir = f"{bbdir}/bebop" + image_name = input_data.get("image", "") + bitstream = input_data.get("bitstream", "") + multi_fpga = bool(input_data.get("multi-fpga", False)) + fpga_location = ( + input_data.get("fpga-location") or input_data.get("fpga_location") or "0.A" + ) + if not isinstance(fpga_location, str) or not re.fullmatch( + "[0-3]\\.[A-E]", fpga_location + ): + ctx.logger.error(f"invalid FPGA location: {fpga_location}") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "error": "invalid_fpga_location", + "fpga_location": fpga_location, + }, + trace_id=origin_tid, + ) + return + wave = bool(input_data.get("wave", False)) + diff = bool(input_data.get("diff", False)) + if "wave_start" in input_data: + ctx.logger.error("invalid parameter: --wave_start (use --wave-start)") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "invalid_parameter", "parameter": "wave_start"}, + trace_id=origin_tid, + ) + return + wave_start_raw = input_data.get("wave-start") + wave_start = None + if wave_start_raw is not None: + try: + wave_start = int(wave_start_raw) + except (TypeError, ValueError): + ctx.logger.error(f"invalid wave_start: {wave_start_raw}") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "error": "invalid_wave_start", + "wave_start": wave_start_raw, + }, + trace_id=origin_tid, + ) + return + if wave_start < 0: + ctx.logger.error(f"wave_start must be >= 0: {wave_start}") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "invalid_wave_start", "wave_start": wave_start}, + trace_id=origin_tid, + ) + return + wave = True + try: + image_path = resolve_image(bbdir, image_name, design, input_data["target"]) + image_base = os.path.splitext(image_path)[0] + elf_path = ( + f"{image_base}.elf" if os.path.isfile(f"{image_base}.elf") else image_base + ) + except (ValueError, OSError, TypeError) as error: + ctx.logger.error(str(error)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "invalid_cold_load_inputs", "message": str(error)}, + trace_id=origin_tid, + ) + return + if not bitstream or not os.path.isfile(bitstream): + ctx.logger.error(f"bitstream .bit file not found: {bitstream}") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "bitstream_not_found", "bitstream": bitstream}, + trace_id=origin_tid, + ) + return + bitstream = os.path.abspath(bitstream) + build_dir = os.path.dirname(os.path.dirname(bitstream)) + if not os.path.isdir(build_dir): + ctx.logger.error(f"P2E build case not found for bitstream: {build_dir}") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "build_dir_not_found", "build_dir": build_dir}, + trace_id=origin_tid, + ) + return + if not multi_fpga and case_uses_multi_fpga(build_dir): + multi_fpga = True + ctx.logger.info(f"Detected multi-FPGA P2E case: {build_dir}") + timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") + run_log = log_dir( + bbdir, + design, + "p2e", + timestamp, + "p2e", + image_name, + input_data.get("vsrc_dir"), + selection=input_data["target"], + ) + os.makedirs(run_log, exist_ok=True) + runtime_lib_dir = os.path.join(build_dir, "vvacDir", "runtimeDir", "lib", "lib_arm") + rtcfg_path = os.path.join(build_dir, "vvacDir", "runtimeDir", "rtcfg") + libvctb_path = os.path.join(runtime_lib_dir, "libvCtb.so") + bebop_p2e_path = os.path.join(build_dir, "bebop-p2e") + runtime_artifacts = [rtcfg_path, libvctb_path] + if not all((os.path.isfile(path) for path in runtime_artifacts)): + missing = [path for path in runtime_artifacts if not os.path.isfile(path)] + if missing: + ctx.logger.error(f"P2E runtime artifacts missing: {missing}") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "task": "runtime", + "build_dir": build_dir, + "missing": missing, + }, + trace_id=origin_tid, + ) + return + manifest = str(runner_manifest(bbdir, design, input_data["target"])) + build_env = {**os.environ, "OUT_PATH": build_dir} + reuse_runtime = input_data.get("reuse-runtime", False) + if reuse_runtime: + validate_runtime_reuse( + bitstream, + diff, + bool(input_data.get("itrace", False)), + bool(input_data.get("mtrace", False)), + ) + ctx.logger.info( + f"Reusing selected P2E runtime without rebuilding: {bebop_p2e_path}" + ) + else: + modes = trace_modes(bitstream) + features = ["p2e"] + if modes & {"btrace", "btrace_nb_v1"}: + features.append("bemu") + for trace in ("itrace", "mtrace"): + if trace in modes: + features.append(f"bebop-p2e/{trace}") + build_cmd = shlex.join( + [ + "nix", + "develop", + "--ignore-env", + "--keep-env-var", + "HOME", + "--keep-env-var", + "ALL_PROXY", + "--keep-env-var", + "OUT_PATH", + "-c", + "cargo", + "build", + "--release", + "--manifest-path", + manifest, + "--target-dir", + bebop_target_dir(bbdir, design, "p2e", selection=input_data["target"]), + "--bin", + "bebop-p2e", + "--features", + ",".join(features), + ] + ) + ctx.logger.info("Building bebop p2e runtime for the selected case...") + build_result = await stream_run_logger_async( + cmd=build_cmd, + logger=ctx.logger, + cwd=bebop_dir, + stdout_prefix="bebop p2e runtime build", + stderr_prefix="bebop p2e runtime build", + env=build_env, + ) + if build_result.returncode != 0: + await check_result( + ctx, + build_result.returncode, + continue_run=False, + extra_fields={"task": "runtime_build", "build_dir": build_dir}, + trace_id=origin_tid, + ) + return + target_release = os.path.join( + bebop_target_dir(bbdir, design, "p2e", selection=input_data["target"]), + "release", + ) + staged_runtime = f"{bebop_p2e_path}.new" + shutil.copy2(os.path.join(target_release, "bebop-p2e"), staged_runtime) + os.replace(staged_runtime, bebop_p2e_path) + rust_library = Path(target_release) / "deps/libbebop_p2e.so" + shutil.copy2(rust_library, Path(runtime_lib_dir) / "libbebop_p2e.so") + design_libraries = Path(bbdir) / "examples/designs" / design / "output/lib" + design_libraries.mkdir(parents=True, exist_ok=True) + shutil.copy2(rust_library, design_libraries / "libbebop_p2e.so") + installed_runtime = Path( + bebop_bin_path(bbdir, design, "p2e", selection=input_data["target"]) + ) + installed_runtime.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(bebop_p2e_path, installed_runtime) + run_args = [ + "nix", + "develop", + "-c", + "env", + "-u", + "LD_LIBRARY_PATH", + bebop_p2e_path, + "--bitstream", + bitstream, + "--log-dir", + run_log, + "--fpga-location", + fpga_location, + ] + run_args += ["--image", image_path] + run_cmd = f"cd {shlex.quote(bbdir)} && {shlex.join(run_args)}" + report_context = source_context(bbdir, design, input_data["target"]) + with open(bitstream, "rb") as source: + report_context["provenance"]["bitstream_sha256"] = hashlib.file_digest( + source, "sha256" + ).hexdigest() + with open(image_path, "rb") as source: + report_context["provenance"]["executable_sha256"] = hashlib.file_digest( + source, "sha256" + ).hexdigest() + + if multi_fpga: + run_cmd += " --multi-fpga" + if wave: + run_cmd += " --wave" + if wave_start is not None: + run_cmd += f' --wave-start="{wave_start}"' + if diff: + run_cmd += f" --diff --image-elf={shlex.quote(elf_path)}" + for trace_name in ("itrace", "mtrace", "pmctrace", "ctrace", "banktrace"): + if input_data.get(trace_name, False): + run_cmd += f" --{trace_name}" + ctx.logger.info(f"Running bebop p2e runworkload: {run_cmd}") + run_env = build_env + run_result = await stream_run_logger_async( + cmd=run_cmd, + logger=ctx.logger, + cwd=bebop_dir, + stdout_prefix="bebop p2e runworkload", + stderr_prefix="bebop p2e runworkload", + env=run_env, + ) + uart_logs = sorted(Path(run_log).glob("uart_hart_*.log")) + try: + report_path = performance_report( + bbdir, + report_context, + origin_tid, + "p2e", + run_log, + "\n".join(path.read_text() for path in uart_logs), + run_result.returncode, + ) + except (OSError, ValueError, KeyError) as error: + await check_result( + ctx, + 1, + extra_fields={"task": "report", "log_dir": run_log, "error": str(error)}, + trace_id=origin_tid, + ) + return + await check_result( + ctx, + run_result.returncode, + continue_run=False, + extra_fields={ + "task": "runworkload", + "report_path": report_path, + "image": image_path, + "bitstream": bitstream, + "build_dir": build_dir, + "log_dir": run_log, + "fpga_location": fpga_location, + "bdb_trace": os.path.join(run_log, "bdb.ndjson"), + "uart_logs": [str(path) for path in uart_logs], + "bank_diff": os.path.join(run_log, "diff.ndjson") if diff else None, + "diff": diff, + "timestamp": timestamp, + }, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/bebop/p2e/04_batch_api.step.py b/bbdev/api/steps/bebop/p2e/04_batch_api.step.py new file mode 100644 index 000000000..6e84c7e0e --- /dev/null +++ b/bbdev/api/steps/bebop/p2e/04_batch_api.step.py @@ -0,0 +1,76 @@ +from utils.event_common import require_target +from motia import ApiRequest, ApiResponse, FlowContext, api + +config = { + "name": "bebop-p2e-batch-api", + "description": "Run bebop p2e batch regression", + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/p2e/batch")], + "enqueues": ["bebop.p2e.batch"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + try: + selection = require_target(body, ("device",)) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + + bitstream = body.get("bitstream", "") + if not bitstream: + return ApiResponse( + status=400, + body={ + "success": False, + "failure": True, + "returncode": 400, + "message": "--bitstream parameter is required", + }, + ) + + design = body.get("design") + if not design: + return ApiResponse( + status=400, + body={"error": "Missing required parameter: --design must be specified"}, + ) + + test_type = body.get("test") + if not test_type: + return ApiResponse( + status=400, + body={ + "error": "Missing required parameter: --test must be specified (bare-tests or linux-tests)" + }, + ) + + if test_type not in ["bare-tests", "linux-tests"]: + return ApiResponse( + status=400, + body={ + "error": f"Invalid test type: {test_type}. Must be 'bare-tests' or 'linux-tests'" + }, + ) + + fpga_location = body.get("fpga-location", "0.A") + if not isinstance(fpga_location, str) or not fpga_location: + return ApiResponse( + status=400, body={"error": "--fpga-location must be a location such as 1.A"} + ) + + diff = bool(body.get("diff", False)) + data = { + "design": design, + "bitstream": bitstream, + "test": test_type, + "diff": diff, + "fpga-location": fpga_location, + } + await ctx.enqueue( + { + "topic": "bebop.p2e.batch", + "data": {**data, "_trace_id": ctx.trace_id, "target": selection}, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/p2e/04_batch_event.step.py b/bbdev/api/steps/bebop/p2e/04_batch_event.step.py new file mode 100644 index 000000000..6a8dd470b --- /dev/null +++ b/bbdev/api/steps/bebop/p2e/04_batch_event.step.py @@ -0,0 +1,226 @@ +""" +bebop p2e batch event handler + +Runs bebop p2e batch regression (aligned with runworkload): + 1. Resolve the runtime from the bitstream case + 2. Build the test harness against that runtime + 3. Run the P2E regression harness serially on one FPGA +""" + +from utils.path import runner_manifest +from datetime import datetime +from pathlib import Path +from utils.path import design_output_root, log_dir + + +import os +import re +import shlex +import sys +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) +bebop_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +if bebop_path not in sys.path: + sys.path.insert(0, bebop_path) +from utils.path import bebop_target_dir, get_buckyball_path, workloads_output_root +from utils.stream_run import stream_run_logger_async +from utils.event_common import check_result, get_origin_trace_id +from utils.workload_manifest import resolve_workload_toml + +config = { + "name": "bebop-p2e-batch", + "description": "Run bebop p2e batch regression", + "flows": ["bebop"], + "triggers": [queue("bebop.p2e.batch")], + "enqueues": [], +} + + +def resolve_runtime_config(bitstream: str, requested_config: object) -> str: + if isinstance(requested_config, str) and requested_config: + return requested_config + build_dir = os.path.dirname(os.path.dirname(os.path.abspath(bitstream))) + case_name = os.path.basename(build_dir) + return re.sub("-\\d{4}-\\d{2}-\\d{2}-\\d{2}-\\d{2}$", "", case_name) + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + bebop_dir = f"{bbdir}/bebop" + bitstream = input_data.get("bitstream", "") + if not bitstream or not os.path.isfile(bitstream): + ctx.logger.error(f"bitstream .bit file not found: {bitstream}") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "bitstream_not_found", "bitstream": bitstream}, + trace_id=origin_tid, + ) + return + bitstream = os.path.abspath(bitstream) + build_dir = os.path.dirname(os.path.dirname(bitstream)) + if not os.path.isdir(build_dir): + ctx.logger.error(f"P2E build case not found for bitstream: {build_dir}") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "build_dir_not_found", "build_dir": build_dir}, + trace_id=origin_tid, + ) + return + design = input_data.get("design") + if not design: + ctx.logger.error("Missing required parameter: design must be specified") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "missing_design"}, + trace_id=origin_tid, + ) + return + elf_root = str( + Path(design_output_root(bbdir, design, selection=input_data["target"])) + / "workloads" + ) + test_type = input_data.get("test", "bare-tests") + diff = bool(input_data.get("diff", False)) + try: + workload_toml = resolve_workload_toml( + design, "p2e", test_type, bbdir, diff=diff, selection=input_data["target"] + ) + except ValueError as e: + ctx.logger.error(str(e)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "error": "invalid_regression", + "test": test_type, + "design": design, + }, + trace_id=origin_tid, + ) + return + ctx.logger.info( + f"Running {test_type} with workload config: {workload_toml} diff={diff}" + ) + ctx.logger.info(f"P2E case dir (from bitstream): {build_dir}") + manifest = runner_manifest(bbdir, design, input_data["target"]) + features = ["p2e"] + runtime_env = os.environ.copy() + if diff: + features.append("bemu") + feature_arg = ",".join(features) + rtcfg_path = os.path.join(build_dir, "vvacDir", "runtimeDir", "rtcfg") + libvctb_path = os.path.join( + build_dir, "vvacDir", "runtimeDir", "lib", "lib_arm", "libvCtb.so" + ) + runtime_artifacts = [rtcfg_path, libvctb_path, os.path.join(build_dir, "bebop-p2e")] + if not all((os.path.isfile(path) for path in runtime_artifacts)): + missing = [path for path in runtime_artifacts if not os.path.isfile(path)] + if missing: + ctx.logger.error(f"P2E runtime artifacts missing: {missing}") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "task": "runtime", + "build_dir": build_dir, + "missing": missing, + }, + trace_id=origin_tid, + ) + return + cargo_out = f'''--config="env.OUT_PATH='{build_dir}'"''' + build_cmd = ( + f"nix develop -c cargo {cargo_out} build --release --target-dir" + f" {shlex.quote(bebop_target_dir(bbdir, design, 'p2e', selection=input_data['target']))}" + f" --manifest-path {shlex.quote(str(manifest))} --features" + f" {shlex.quote(feature_arg)} --tests" + ) + ctx.logger.info("Building bebop p2e (tests)...") + build_result = await stream_run_logger_async( + cmd=build_cmd, + logger=ctx.logger, + cwd=bebop_dir, + stdout_prefix="bebop p2e build", + stderr_prefix="bebop p2e build", + env=runtime_env, + ) + if build_result.returncode != 0: + await check_result( + ctx, + build_result.returncode, + continue_run=False, + extra_fields={"task": "build", "backend": "p2e", "build_dir": build_dir}, + trace_id=origin_tid, + ) + return + env = runtime_env.copy() + env["OUT_PATH"] = build_dir + harness_args = [ + "--", + "--workload-toml", + workload_toml, + "--bb-tests-root", + elf_root, + "--p2e-bitstream", + bitstream, + "--p2e-fpga-location", + input_data.get("fpga-location", "0.A"), + "--log-dir", + log_dir( + bbdir, + design, + "p2e", + datetime.now().strftime("%Y-%m-%d-%H-%M"), + "bebop-p2e", + test_type, + selection=input_data["target"], + ), + ] + if diff: + harness_args.append("--diff") + harness = shlex.join(harness_args) + test_cmd = ( + f"nix develop -c cargo {cargo_out} test --release --target-dir" + f" {shlex.quote(bebop_target_dir(bbdir, design, 'p2e', selection=input_data['target']))}" + f" --manifest-path {shlex.quote(str(manifest))} --features" + f" {shlex.quote(feature_arg)} --test test_p2e {harness}" + ) + ctx.logger.info(f"Running bebop p2e regression: {test_cmd}") + run_result = await stream_run_logger_async( + cmd=test_cmd, + logger=ctx.logger, + cwd=bebop_dir, + stdout_prefix="bebop p2e batch", + stderr_prefix="bebop p2e batch", + env=env, + ) + extra_fields = { + "task": "batch", + "backend": "p2e", + "design": design, + "bitstream": bitstream, + "build_dir": build_dir, + "test_type": test_type, + "diff": diff, + "workload_toml": workload_toml, + } + + await check_result( + ctx, + run_result.returncode, + continue_run=False, + extra_fields=extra_fields, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/bebop/p2e/scripts/resolve_image.py b/bbdev/api/steps/bebop/p2e/scripts/resolve_image.py new file mode 100644 index 000000000..161e4f3b3 --- /dev/null +++ b/bbdev/api/steps/bebop/p2e/scripts/resolve_image.py @@ -0,0 +1,25 @@ +import glob +import os + +from utils.path import design_output_root + + +def resolve_image(bbdir: str, image_name: str, design: str, selection: str) -> str: + """The hex image of a workload or kernel built for the simulation entry.""" + image_name = image_name.replace(r"\_", "_") + if os.path.isabs(image_name): + raise ValueError( + f"image must be a logical name, not an absolute path: {image_name!r}" + ) + workloads = os.path.join( + design_output_root(bbdir, design, selection=selection), "workloads" + ) + matches = glob.glob( + os.path.join(workloads, "**", f"{image_name}.hex"), recursive=True + ) + if len(matches) != 1: + raise ValueError( + f"expected one image for {image_name!r} under {workloads}, " + f"found {len(matches)}" + ) + return matches[0] diff --git a/bbdev/api/steps/bebop/p2e/scripts/runtime_case.py b/bbdev/api/steps/bebop/p2e/scripts/runtime_case.py new file mode 100644 index 000000000..926184eef --- /dev/null +++ b/bbdev/api/steps/bebop/p2e/scripts/runtime_case.py @@ -0,0 +1,40 @@ +from pathlib import Path +import os + + +def trace_modes(bitstream: str) -> set[str]: + case = Path(bitstream).resolve().parent.parent + modes = set((case / "p2e_trace_mode").read_text().strip().split("+")) + known = {"none", "btrace", "btrace_nb_v1", "itrace", "mtrace"} + if not modes <= known or ("none" in modes and len(modes) != 1): + raise ValueError(f"Unknown P2E trace mode: {modes}") + return modes + + +def validate_runtime_reuse( + bitstream: str, diff: bool, itrace: bool = False, mtrace: bool = False +) -> Path: + bit = Path(bitstream).resolve() + case = bit.parent.parent + executable = case / "bebop-p2e" + files = [ + bit, + executable, + case / "vvacDir/runtimeDir/rtcfg", + case / "vvacDir/runtimeDir/lib/lib_arm/libvCtb.so", + case / "p2e_trace_mode", + ] + missing = [str(path) for path in files if not path.is_file()] + if missing: + raise ValueError(f"Reusable P2E case is incomplete: {missing}") + if not os.access(executable, os.X_OK): + raise ValueError(f"Reusable P2E runtime is not executable: {executable}") + modes = trace_modes(bitstream) + if diff and not modes & {"btrace", "btrace_nb_v1"}: + raise ValueError("Reusable P2E case does not support diff") + for trace, requested in (("itrace", itrace), ("mtrace", mtrace)): + if requested and trace not in modes: + raise ValueError( + f"Reusable P2E case does not support {trace}; rebuild with --{trace}" + ) + return executable diff --git a/bbdev/api/steps/bebop/p2e/scripts/timescale.py b/bbdev/api/steps/bebop/p2e/scripts/timescale.py new file mode 100644 index 000000000..5156875e2 --- /dev/null +++ b/bbdev/api/steps/bebop/p2e/scripts/timescale.py @@ -0,0 +1,22 @@ +import os + + +def normalize_p2e_timescale(build_dir, logger): + if not os.path.isdir(build_dir): + raise FileNotFoundError(f"P2E rtl dir not found: {build_dir}") + patched = 0 + for root, _, files in os.walk(build_dir): + for name in files: + if not name.endswith((".v", ".sv")): + continue + path = os.path.join(root, name) + with open(path) as f: + content = f.read() + if "`timescale" in content: + continue + with open(path, "w") as f: + f.write("`timescale 1ns/1ps\n") + f.write(content) + patched += 1 + logger.info(f"Normalized P2E timescale in {patched} generated Verilog files") + return patched diff --git a/bbdev/api/steps/bebop/performance_report.py b/bbdev/api/steps/bebop/performance_report.py new file mode 100644 index 000000000..f4a270343 --- /dev/null +++ b/bbdev/api/steps/bebop/performance_report.py @@ -0,0 +1,13 @@ +from pathlib import Path + +from utils.reports import new_report, finish_report + + +def performance_report(repo, context, trace_id, backend, log, stdout, returncode): + report = new_report(context, f"{backend}.execution", trace_id, backend) + report["execution"] = { + "status": "pass" if returncode == 0 else "fail", + "returncode": returncode, + "performance": "not_collected", + } + return finish_report(report, Path(log) / "report") diff --git a/bbdev/api/steps/bebop/vcs/01_verilog_api.step.py b/bbdev/api/steps/bebop/vcs/01_verilog_api.step.py new file mode 100644 index 000000000..77f7f9b23 --- /dev/null +++ b/bbdev/api/steps/bebop/vcs/01_verilog_api.step.py @@ -0,0 +1,24 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.event_common import require_design, require_target + +config = { + "name": "bebop-vcs-verilog-api", + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/vcs/verilog")], + "enqueues": ["vcs.verilog"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + await ctx.enqueue( + { + "topic": "vcs.verilog", + "data": { + "design": require_design(req.body or {}), + "target": require_target(req.body or {}), + "_trace_id": ctx.trace_id, + }, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/vcs/02_build_api.step.py b/bbdev/api/steps/bebop/vcs/02_build_api.step.py new file mode 100644 index 000000000..598e57a65 --- /dev/null +++ b/bbdev/api/steps/bebop/vcs/02_build_api.step.py @@ -0,0 +1,31 @@ +from utils.event_common import require_target +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.event_common import require_design + +config = { + "name": "bebop-vcs-build-api", + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/vcs/build")], + "enqueues": ["vcs.build"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = req.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + await ctx.enqueue( + { + "topic": "vcs.build", + "data": { + "design": require_design(body), + "jobs": body.get("jobs", 16), + "_trace_id": ctx.trace_id, + "target": selection, + }, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/vcs/03_batch_api.step.py b/bbdev/api/steps/bebop/vcs/03_batch_api.step.py new file mode 100644 index 000000000..01fc0a6ea --- /dev/null +++ b/bbdev/api/steps/bebop/vcs/03_batch_api.step.py @@ -0,0 +1,33 @@ +from utils.event_common import require_target + +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.event_common import require_design + +config = { + "name": "bebop-vcs-batch-api", + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/vcs/batch")], + "enqueues": ["bebop.vcs.batch"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = req.body or {} + try: + selection = require_target(body, ("device",)) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + await ctx.enqueue( + { + "topic": "bebop.vcs.batch", + "data": { + "design": require_design(body), + "test": body.get("test", "bare-tests"), + "jobs": body.get("jobs", 1), + "_trace_id": ctx.trace_id, + "target": selection, + }, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/vcs/03_batch_event.step.py b/bbdev/api/steps/bebop/vcs/03_batch_event.step.py new file mode 100644 index 000000000..2199e6697 --- /dev/null +++ b/bbdev/api/steps/bebop/vcs/03_batch_event.step.py @@ -0,0 +1,83 @@ +from pathlib import Path +import shlex +from datetime import datetime + +from motia import FlowContext, queue +from utils.event_common import check_result, get_origin_trace_id, require_design +from utils.path import ( + get_buckyball_path, + design_output_root, + bebop_target_dir, + runner_manifest, + log_dir, +) +from utils.workload_manifest import resolve_workload_toml +from utils.stream_run import stream_run_logger_async + +config = { + "name": "bebop-vcs-batch", + "flows": ["bebop"], + "triggers": [queue("bebop.vcs.batch")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + design = require_design(input_data) + repo = get_buckyball_path() + selection = input_data["target"] + tests = input_data.get("test", "bare-tests") + manifest = resolve_workload_toml(design, "vcs", tests, repo, selection=selection) + output = Path(design_output_root(repo, design, selection=selection)) + stamp = datetime.now().strftime("%Y-%m-%d-%H-%M") + report = ( + Path( + log_dir( + repo, design, "tapeout", stamp, "bebop-vcs", tests, selection=selection + ) + ) + / origin_tid + ) + command = [ + "cargo", + "test", + "--release", + "--manifest-path", + str(runner_manifest(repo, design, selection)), + "--target-dir", + bebop_target_dir(repo, design, "vcs", selection=selection), + "--test", + "test_vcs", + "--", + "--workload-toml", + manifest, + "--bb-tests-root", + str(output / "workloads"), + "--simv", + str(output / "vcs/simv"), + "--dram-configs", + str(Path(repo) / "result/share/dramsim3/configs"), + "--log-dir", + str(report), + "--jobs", + str(input_data.get("jobs", 1)), + ] + result = await stream_run_logger_async( + cmd=shlex.join(command), + logger=ctx.logger, + cwd=repo, + stdout_prefix="bebop vcs batch", + stderr_prefix="bebop vcs batch", + ) + await check_result( + ctx, + result.returncode, + extra_fields={ + "task": "batch", + "backend": "vcs", + "log_dir": str(report), + "junit": str(report / "junit.xml"), + }, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/bebop/vcs/04_sim_api.step.py b/bbdev/api/steps/bebop/vcs/04_sim_api.step.py new file mode 100644 index 000000000..1ef7c2b48 --- /dev/null +++ b/bbdev/api/steps/bebop/vcs/04_sim_api.step.py @@ -0,0 +1,26 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api +from utils.event_common import require_design, require_target + +config = { + "name": "bebop-vcs-sim-api", + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/vcs/sim")], + "enqueues": ["vcs.sim"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = req.body or {} + await ctx.enqueue( + { + "topic": "vcs.sim", + "data": { + "design": require_design(body), + "target": require_target(body, ("device", "core")), + "binary": body["binary"], + "wave": body.get("wave", False), + "_trace_id": ctx.trace_id, + }, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/verilator/01_clean_api.step.py b/bbdev/api/steps/bebop/verilator/01_clean_api.step.py new file mode 100644 index 000000000..8c142eb57 --- /dev/null +++ b/bbdev/api/steps/bebop/verilator/01_clean_api.step.py @@ -0,0 +1,31 @@ +from utils.event_common import require_target +from motia import ApiRequest, ApiResponse, FlowContext, api +from utils.event_common import require_design + +config = { + "name": "bebop-verilator-clean-api", + "description": "Clean verilator build directory", + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/verilator/clean")], + "enqueues": ["bebop.verilator.clean"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = req.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) + data = {"design": design} + await ctx.enqueue( + { + "topic": "bebop.verilator.clean", + "data": {**data, "_trace_id": ctx.trace_id, "target": selection}, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/verilator/01_clean_event.step.py b/bbdev/api/steps/bebop/verilator/01_clean_event.step.py new file mode 100644 index 000000000..deb179a21 --- /dev/null +++ b/bbdev/api/steps/bebop/verilator/01_clean_event.step.py @@ -0,0 +1,65 @@ +import os +import sys +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) +from utils.event_common import require_design +from utils.path import get_buckyball_path, rtl_dir +from utils.stream_run import stream_run_logger_async +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "bebop-verilator-clean", + "description": "Clean verilator build directory", + "flows": ["bebop"], + "triggers": [queue("bebop.verilator.clean"), queue("bebop.verilator.run.clean")], + "enqueues": ["bebop.verilator.verilog", "bebop.verilator.run.verilog"], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + try: + design = require_design(input_data) + except ValueError as error: + ctx.logger.error(str(error)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "missing_design"}, + trace_id=origin_tid, + ) + return + build_dir = rtl_dir( + bbdir, + design, + "bebop", + selection=input_data["target"], + ) + command = f"rm -rf {build_dir}" + result = await stream_run_logger_async( + cmd=command, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="bebop verilator clean", + stderr_prefix="bebop verilator clean", + ) + await check_result( + ctx, + result.returncode, + continue_run=input_data.get("from_run_workflow", False), + extra_fields={"task": "clean", "rtl_dir": build_dir}, + trace_id=origin_tid, + ) + if input_data.get("from_run_workflow"): + data = dict(input_data) + await ctx.enqueue( + { + "topic": "bebop.verilator.run.verilog", + "data": {**data, "task": "run", "target": input_data["target"]}, + } + ) diff --git a/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py b/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py new file mode 100644 index 000000000..79f32514c --- /dev/null +++ b/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py @@ -0,0 +1,44 @@ +from utils.event_common import require_target + +from motia import ApiRequest, ApiResponse, FlowContext, api +from utils.event_common import require_design + +config = { + "name": "bebop-verilator-verilog-api", + "description": "Generate verilog code via mill", + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/verilator/verilog")], + "enqueues": ["bebop.verilator.verilog"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + if body.get("balltype"): + return ApiResponse( + status=400, + body={"error": "ball-only verilog belongs to uvm; use bbdev uvm --verilog"}, + ) + try: + design = require_design(body) + except ValueError as e: + return ApiResponse( + status=400, + body={ + "status": "error", + "message": str(e), + "example": 'bbdev bebop verilator --verilog "--design toy"', + }, + ) + data = {"design": design, "diff": bool(body.get("diff", False))} + await ctx.enqueue( + { + "topic": "bebop.verilator.verilog", + "data": {**data, "_trace_id": ctx.trace_id, "target": selection}, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/verilator/03_build_api.step.py b/bbdev/api/steps/bebop/verilator/03_build_api.step.py new file mode 100644 index 000000000..8e8fd04f0 --- /dev/null +++ b/bbdev/api/steps/bebop/verilator/03_build_api.step.py @@ -0,0 +1,35 @@ +from utils.event_common import require_target +from motia import ApiRequest, ApiResponse, FlowContext, api +from utils.event_common import require_design + +config = { + "name": "bebop-verilator-build-api", + "description": "Build bebop verilator binary", + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/verilator/build")], + "enqueues": ["bebop.verilator.build"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) + data = { + "design": design, + "jobs": body.get("jobs", 16), + "diff": bool(body.get("diff", False)), + } + await ctx.enqueue( + { + "topic": "bebop.verilator.build", + "data": {**data, "_trace_id": ctx.trace_id, "target": selection}, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/verilator/03_build_event.step.py b/bbdev/api/steps/bebop/verilator/03_build_event.step.py new file mode 100644 index 000000000..e961afe2f --- /dev/null +++ b/bbdev/api/steps/bebop/verilator/03_build_event.step.py @@ -0,0 +1,121 @@ +""" +bebop verilator build event handler + +Builds the entry's Verilator runner; bebop-verilator finds the entry's RTL by its output layout +""" + +from utils.path import runner_manifest + + +import os +import shlex +import shutil +import sys +from pathlib import Path +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) +from utils.event_common import require_design +from utils.path import bebop_target_dir, bebop_bin_path, get_buckyball_path, rtl_dir +from utils.stream_run import stream_run_logger_async +from utils.event_common import check_result, get_origin_trace_id +from build_marker import write_build_marker + +config = { + "name": "bebop-verilator-build", + "description": "Build bebop verilator binary", + "flows": ["bebop"], + "triggers": [queue("bebop.verilator.build"), queue("bebop.verilator.run.build")], + "enqueues": ["bebop.verilator.sim", "bebop.verilator.run.sim"], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + design = require_design(input_data) + vsrc_dir = rtl_dir( + bbdir, + design, + "bebop", + selection=input_data["target"], + ) + ctx.logger.info(f"Using verilog source directory: {vsrc_dir}") + diff = bool(input_data.get("diff", False)) + manifest = runner_manifest(bbdir, design, input_data["target"]) + build_dir = str(manifest.parent) + features = ["verilator"] + if diff: + features.append("bemu") + jobs = input_data.get("jobs", 16) + cargo_build_cmd = shlex.join( + [ + "cargo", + "build", + "--release", + "--manifest-path", + str(manifest), + "--bin", + "bebop-verilator", + "--target-dir", + bebop_target_dir( + bbdir, design, "verilator", selection=input_data["target"] + ), + "--features", + ",".join(features), + "--jobs", + str(jobs), + ] + ) + build_inner = f"cd {shlex.quote(str(build_dir))} && exec {cargo_build_cmd}" + build_cmd = f"nix develop -c sh -c {shlex.quote(build_inner)}" + ctx.logger.info(f"Building bebop verilator (diff={diff}) ...") + build_result = await stream_run_logger_async( + cmd=build_cmd, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="bebop verilator build", + stderr_prefix="bebop verilator build", + ) + bebop_bin = bebop_bin_path( + bbdir, design, "verilator", selection=input_data["target"] + ) + target_dir = bebop_target_dir( + bbdir, design, "verilator", selection=input_data["target"] + ) + if build_result.returncode == 0: + Path(bebop_bin).parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(Path(target_dir) / "release/bebop-verilator", bebop_bin) + write_build_marker(target_dir, design, vsrc_dir, bebop_bin, diff=diff) + await check_result( + ctx, + build_result.returncode, + continue_run=input_data.get("from_run_workflow", False), + extra_fields={ + "task": "build", + "vsrc_dir": vsrc_dir, + "binary": bebop_bin, + "diff": diff, + "design": design, + }, + trace_id=origin_tid, + ) + if build_result.returncode != 0: + return + if input_data.get("from_run_workflow"): + await ctx.enqueue( + { + "topic": "bebop.verilator.run.sim", + "data": { + **input_data, + "design": design, + "task": "run", + "target": input_data["target"], + }, + } + ) diff --git a/bbdev/api/steps/bebop/verilator/04_sim_api.step.py b/bbdev/api/steps/bebop/verilator/04_sim_api.step.py new file mode 100644 index 000000000..495aa2314 --- /dev/null +++ b/bbdev/api/steps/bebop/verilator/04_sim_api.step.py @@ -0,0 +1,44 @@ +from utils.event_common import require_target +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.event_common import require_design + +config = { + "name": "bebop-verilator-sim-api", + "description": "Run bebop verilator simulation", + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/verilator/sim")], + "enqueues": ["bebop.verilator.sim"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + binary = body.get("binary", "") + if not binary: + return ApiResponse( + status=400, + body={ + "success": False, + "failure": True, + "returncode": 400, + "message": "binary parameter is required", + }, + ) + try: + design = require_design(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) + + data = {**body, "design": design, "_trace_id": ctx.trace_id} + await ctx.enqueue( + { + "topic": "bebop.verilator.sim", + "data": {**(data), "target": selection}, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py new file mode 100644 index 000000000..ecccbf291 --- /dev/null +++ b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py @@ -0,0 +1,276 @@ +""" +bebop verilator event handler + +Run a guest ELF through Verilator. +""" + +from pathlib import Path +from utils.path import design_output_root + + +import json +import os +import shlex +import subprocess +import sys +from datetime import datetime +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) +from utils.event_common import require_design +from utils.path import ( + bebop_target_dir, + bebop_bin_path, + get_buckyball_path, + log_dir, + rtl_dir, +) +from utils.stream_run import stream_run_logger_async +from utils.search_workload import search_workload +from utils.event_common import check_result, get_origin_trace_id +from build_marker import build_marker_path, read_build_marker + +config = { + "name": "bebop-verilator-sim", + "description": "Run bebop verilator simulation", + "flows": ["bebop"], + "triggers": [queue("bebop.verilator.sim"), queue("bebop.verilator.run.sim")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + bebop_dir = f"{bbdir}/bebop" + try: + design = require_design(input_data) + except ValueError as error: + ctx.logger.error(str(error)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "missing_design"}, + trace_id=origin_tid, + ) + return + diff = bool(input_data.get("diff", False)) + vsrc_dir = rtl_dir( + bbdir, + design, + "bebop", + selection=input_data["target"], + ) + ctx.logger.info(f"Using verilog source directory: {vsrc_dir}") + build_dir = bebop_dir + design = input_data.get("design") + if diff: + if not isinstance(design, str) or not design: + ctx.logger.error( + "Missing required parameter: --design (required with --diff)" + ) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "missing_design"}, + trace_id=origin_tid, + ) + return + bebop_bin = bebop_bin_path( + bbdir, design, "verilator", selection=input_data["target"] + ) + if not os.path.isfile(bebop_bin): + ctx.logger.error(f"bebop binary does not exist: {bebop_bin}") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "bebop_binary_not_found", "binary": bebop_bin}, + trace_id=origin_tid, + ) + return + target_dir = bebop_target_dir( + bbdir, design, "verilator", selection=input_data["target"] + ) + marker_path = build_marker_path(target_dir) + try: + marker = read_build_marker(target_dir) + except FileNotFoundError: + ctx.logger.error(f"bebop verilator build marker does not exist: {marker_path}") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "build_marker_not_found", "marker": marker_path}, + trace_id=origin_tid, + ) + return + except (OSError, json.JSONDecodeError) as e: + ctx.logger.error(f"failed to read bebop verilator build marker: {e}") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "error": "build_marker_read_failed", + "marker": marker_path, + "detail": str(e), + }, + trace_id=origin_tid, + ) + return + expect_vsrc = os.path.abspath(vsrc_dir) + expect_bin = os.path.abspath(bebop_bin) + if ( + marker.get("config") != design + or marker.get("vsrc_dir") != expect_vsrc + or marker.get("binary") != expect_bin + or (bool(marker.get("diff", False)) != diff) + ): + ctx.logger.error(f"bebop verilator build marker mismatch: {marker}") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "error": "build_marker_mismatch", + "marker": marker, + "expected": { + "vsrc_dir": expect_vsrc, + "binary": expect_bin, + "diff": diff, + }, + }, + trace_id=origin_tid, + ) + return + binary_name = input_data.get("binary", "") + try: + binary_path = search_workload( + str( + Path(design_output_root(bbdir, design, selection=input_data["target"])) + / "workloads" + ), + binary_name, + ) + except ValueError as error: + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "error": "binary_resolution_error", + "binary": binary_name, + "message": str(error), + }, + trace_id=origin_tid, + ) + return + if binary_path is None: + ctx.logger.error(f"binary not found: {binary_name}") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "binary_not_found", "binary": binary_name}, + trace_id=origin_tid, + ) + return + ctx.logger.info(f"binary_path: {binary_path}") + timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") + run_log = log_dir( + bbdir, + design, + "verilog", + timestamp, + "verilator", + binary_name, + selection=input_data["target"], + ) + os.makedirs(run_log, exist_ok=True) + wave_arg = ( + " --no-wave" + if input_data.get("no-wave", False) or input_data.get("no_wave", False) + else "" + ) + trace_args = "" + for trace_name in ("itrace", "mtrace", "pmctrace", "ctrace", "banktrace"): + if input_data.get(trace_name, False): + trace_args += f" --{trace_name}" + run_cmd = ( + f"{shlex.quote(bebop_bin)} --elf={shlex.quote(binary_path)}" + f" --log-dir={shlex.quote(run_log)}{(' --diff' if diff else '')}{wave_arg}{trace_args}" + ) + if diff: + dram_library = f"{bbdir}/result/lib/libdramsim3.so" + dram_rpath = ( + subprocess.check_output( + ["patchelf", "--print-rpath", dram_library], text=True + ) + .strip() + .split(os.pathsep) + ) + cpp_libraries = [ + os.path.join(path, "libstdc++.so.6") + for path in dram_rpath + if os.path.isfile(os.path.join(path, "libstdc++.so.6")) + ] + if len(cpp_libraries) != 1: + raise ValueError( + f"DRAMSim requires one C++ runtime in its RPATH: {dram_rpath}" + ) + preload = os.pathsep.join( + ( + path + for path in ( + dram_library, + cpp_libraries[0], + os.environ.get("LD_PRELOAD", ""), + ) + if path + ) + ) + # VVAC embeds its older C++ runtime in the executable's search path. + # DRAMSim is built by the active Nix toolchain and needs that toolchain's + # runtime. Resolve it from DRAMSim and preload only the simulator process. + run_inner = ( + f"cd {shlex.quote(build_dir)} && " + f"export LD_PRELOAD={shlex.quote(preload)} && " + f"exec {run_cmd}" + ) + run_cmd = ( + f"cd {shlex.quote(bbdir)} && nix develop -c sh -c {shlex.quote(run_inner)}" + ) + ctx.logger.info(f"Running bebop verilator (diff={diff}): {run_cmd}") + stdout_prefix = "bebop verilator difftest" if diff else "bebop verilator" + stderr_prefix = stdout_prefix + run_result = await stream_run_logger_async( + cmd=run_cmd, + logger=ctx.logger, + cwd=bebop_dir, + stdout_prefix=stdout_prefix, + stderr_prefix=stderr_prefix, + ) + await check_result( + ctx, + run_result.returncode, + continue_run=False, + extra_fields={ + "task": "verilator", + "backend": "guest-elf", + "diff": diff, + "design": design, + "binary": binary_path, + "log_dir": run_log, + "bank_diff": os.path.join(run_log, "diff.ndjson") if diff else None, + "timestamp": timestamp, + }, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/bebop/verilator/05_run_api.step.py b/bbdev/api/steps/bebop/verilator/05_run_api.step.py new file mode 100644 index 000000000..957d7d0d4 --- /dev/null +++ b/bbdev/api/steps/bebop/verilator/05_run_api.step.py @@ -0,0 +1,55 @@ +from utils.event_common import require_target +from motia import ApiRequest, ApiResponse, FlowContext, api +from utils.event_common import require_design + +config = { + "name": "bebop-verilator-run-api", + "description": "Run complete bebop verilator workflow: clean → verilog → build → sim", + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/verilator/run")], + "enqueues": ["bebop.verilator.run.clean"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) + binary = body.get("binary", "") + if not binary: + return ApiResponse( + status=400, + body={ + "success": False, + "failure": True, + "returncode": 400, + "message": "binary parameter is required", + }, + ) + diff = body.get("diff", False) + data = { + "design": design, + "binary": binary, + "itrace": body.get("itrace", False), + "mtrace": body.get("mtrace", False), + "pmctrace": body.get("pmctrace", False), + "ctrace": body.get("ctrace", False), + "banktrace": body.get("banktrace", False), + "diff": diff, + "no-wave": body.get("no-wave", body.get("no_wave", False)), + "jobs": body.get("jobs", 16), + "from_run_workflow": True, + } + await ctx.enqueue( + { + "topic": "bebop.verilator.run.clean", + "data": {**data, "_trace_id": ctx.trace_id, "target": selection}, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/verilator/06_batch_api.step.py b/bbdev/api/steps/bebop/verilator/06_batch_api.step.py new file mode 100644 index 000000000..af39185af --- /dev/null +++ b/bbdev/api/steps/bebop/verilator/06_batch_api.step.py @@ -0,0 +1,52 @@ +from utils.event_common import require_target +from motia import ApiRequest, ApiResponse, FlowContext, api +from utils.event_common import require_design + +config = { + "name": "bebop-verilator-batch-api", + "description": "Run bebop verilator batch regression", + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/verilator/batch")], + "enqueues": ["bebop.verilator.batch"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + try: + selection = require_target(body, ("device",)) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) + diff = bool(body.get("diff", False)) + test_type = body.get("test") + if not test_type: + return ApiResponse( + status=400, + body={ + "error": "Missing required parameter: --test must be specified (bare-tests or linux-tests)" + }, + ) + if test_type not in ("bare-tests", "linux-tests"): + return ApiResponse( + status=400, + body={ + "error": f"Invalid test type: {test_type}. Must be 'bare-tests' or 'linux-tests'" + }, + ) + data = { + "design": design, + "test": test_type, + "jobs": body.get("jobs", 1), + "diff": diff, + } + await ctx.enqueue( + { + "topic": "bebop.verilator.batch", + "data": {**data, "_trace_id": ctx.trace_id, "target": selection}, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py new file mode 100644 index 000000000..1e2524e70 --- /dev/null +++ b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py @@ -0,0 +1,153 @@ +""" +bebop verilator batch event handler + +Runs bebop verilator batch regression (requires prior --build). +""" + +from datetime import datetime +from pathlib import Path +from utils.path import design_output_root, log_dir +from utils.path import runner_manifest + + +import os +import shlex +import sys +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) +bebop_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +if bebop_path not in sys.path: + sys.path.insert(0, bebop_path) +from utils.event_common import require_design +from utils.path import ( + bebop_target_dir, + get_buckyball_path, + rtl_dir, +) +from utils.stream_run import stream_run_logger_async +from utils.event_common import check_result, get_origin_trace_id +from utils.workload_manifest import resolve_workload_toml + +config = { + "name": "bebop-verilator-batch", + "description": "Run bebop verilator batch regression", + "flows": ["bebop"], + "triggers": [queue("bebop.verilator.batch")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + try: + design = require_design(input_data) + except ValueError as error: + ctx.logger.error(str(error)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "missing_design"}, + trace_id=origin_tid, + ) + return + elf_root = str( + Path(design_output_root(bbdir, design, selection=input_data["target"])) + / "workloads" + ) + test_type = input_data.get("test", "bare-tests") + diff = bool(input_data.get("diff", False)) + try: + workload_toml = resolve_workload_toml( + design, + "verilator", + test_type, + bbdir, + diff=diff, + selection=input_data["target"], + ) + except ValueError as e: + ctx.logger.error(str(e)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "error": "invalid_regression", + "test": test_type, + "design": design, + }, + trace_id=origin_tid, + ) + return + ctx.logger.info( + f"Running {test_type} with workload config: {workload_toml} diff={diff}" + ) + vsrc_dir = rtl_dir( + bbdir, + design, + "bebop", + selection=input_data["target"], + ) + manifest = str(runner_manifest(bbdir, design, input_data["target"])) + features = "verilator,bemu" if diff else "verilator" + run_log = log_dir( + bbdir, + design, + "verilog", + datetime.now().strftime("%Y-%m-%d-%H-%M"), + "bebop-verilator", + test_type, + selection=input_data["target"], + ) + harness_args = [ + "--", + "--workload-toml", + workload_toml, + "--bb-tests-root", + elf_root, + "--arch-config", + design, + "--log-dir", + run_log, + "--jobs", + str(input_data.get("jobs", 1)), + ] + if diff: + harness_args.append("--diff") + harness = shlex.join(harness_args) + test_cmd = ( + "nix develop -c cargo test --release --target-dir" + f" {shlex.quote(bebop_target_dir(bbdir, design, 'verilator', selection=input_data['target']))}" + f" --manifest-path {shlex.quote(manifest)} --features {shlex.quote(features)}" + f" --test test_verilator {harness}" + ) + ctx.logger.info(f"Running bebop verilator regression: {test_cmd}") + run_result = await stream_run_logger_async( + cmd=test_cmd, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="bebop verilator batch", + stderr_prefix="bebop verilator batch", + ) + await check_result( + ctx, + run_result.returncode, + continue_run=False, + extra_fields={ + "task": "batch", + "backend": "difftest" if diff else "verilator", + "design": design, + "vsrc_dir": vsrc_dir, + "test_type": test_type, + "diff": diff, + "workload_toml": workload_toml, + "log_dir": run_log, + "junit": os.path.join(run_log, "junit.xml"), + }, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/bebop/verilator/scripts/build_marker.py b/bbdev/api/steps/bebop/verilator/scripts/build_marker.py new file mode 100644 index 000000000..852c75933 --- /dev/null +++ b/bbdev/api/steps/bebop/verilator/scripts/build_marker.py @@ -0,0 +1,34 @@ +import json +import os + + +def build_marker_path(target_dir: str) -> str: + return os.path.join(target_dir, ".bbdev-verilator-build.json") + + +def write_build_marker( + target_dir: str, + config: str, + vsrc_dir: str, + binary: str, + diff: bool = False, +): + marker = build_marker_path(target_dir) + os.makedirs(os.path.dirname(marker), exist_ok=True) + data = { + "config": config, + "vsrc_dir": os.path.abspath(vsrc_dir), + "binary": os.path.abspath(binary), + "diff": diff, + } + tmp = f"{marker}.tmp" + with open(tmp, "w") as f: + json.dump(data, f, indent=2, sort_keys=True) + f.write("\n") + os.replace(tmp, marker) + + +def read_build_marker(target_dir: str) -> dict: + marker = build_marker_path(target_dir) + with open(marker) as f: + return json.load(f) diff --git a/bbdev/api/steps/common/01_result_query_api.step.py b/bbdev/api/steps/common/01_result_query_api.step.py new file mode 100644 index 000000000..054807298 --- /dev/null +++ b/bbdev/api/steps/common/01_result_query_api.step.py @@ -0,0 +1,64 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +config = { + "name": "result-query-api", + "description": "Query task result by trace_id", + "flows": ["common"], + "triggers": [api("GET", "/result/{trace_id}")], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + """ + Query the result of a task by its trace_id. + + Returns: + 200: {"status": "success|failure|processing", "body": {...}} + 404: {"error": "No result found for trace_id"} + """ + trace_id = request.path_params.get("trace_id", "") + + if not trace_id: + return ApiResponse(status=400, body={"error": "trace_id is required"}) + + cancelled_state = await ctx.state.get(trace_id, "cancelled") + if cancelled_state: + return ApiResponse( + status=200, + body={ + "status": "cancelled", + "body": cancelled_state.get("body", cancelled_state), + }, + ) + + # Check for success state + success_state = await ctx.state.get(trace_id, "success") + if success_state: + return ApiResponse( + status=200, + body={ + "status": "success", + "body": success_state.get("body", success_state), + }, + ) + + # Check for failure state + failure_state = await ctx.state.get(trace_id, "failure") + if failure_state: + return ApiResponse( + status=200, + body={ + "status": "failure", + "body": failure_state.get("body", failure_state), + }, + ) + + # Check for processing state + processing_state = await ctx.state.get(trace_id, "processing") + if processing_state: + return ApiResponse(status=200, body={"status": "processing", "body": {}}) + + # No state found + return ApiResponse( + status=404, body={"error": f"No result found for trace_id: {trace_id}"} + ) diff --git a/bbdev/api/steps/common/02_cancel_api.step.py b/bbdev/api/steps/common/02_cancel_api.step.py new file mode 100644 index 000000000..dee1a2375 --- /dev/null +++ b/bbdev/api/steps/common/02_cancel_api.step.py @@ -0,0 +1,30 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +config = { + "name": "task-cancel-api", + "description": "Cancel a queued or running bbdev task by trace_id", + "flows": ["common"], + "triggers": [api("POST", "/task/{trace_id}/cancel")], + "enqueues": ["common.task.cancel"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + trace_id = request.path_params.get("trace_id", "") + if not trace_id: + return ApiResponse(status=400, body={"error": "trace_id is required"}) + + for key in ("success", "failure", "cancelled"): + if await ctx.state.get(trace_id, key): + return ApiResponse( + status=409, + body={"error": "task is already terminal", "trace_id": trace_id}, + ) + + await ctx.enqueue( + { + "topic": "common.task.cancel", + "data": {"_trace_id": trace_id}, + } + ) + return ApiResponse(status=202, body={"trace_id": trace_id, "status": "cancelling"}) diff --git a/bbdev/api/steps/common/02_cancel_event.step.py b/bbdev/api/steps/common/02_cancel_event.step.py new file mode 100644 index 000000000..392cd7929 --- /dev/null +++ b/bbdev/api/steps/common/02_cancel_event.step.py @@ -0,0 +1,38 @@ +import asyncio + +from motia import FlowContext, queue + +from utils.event_common import get_origin_trace_id +from utils.process_registry import cancel_process, force_cancel_process, process_pid + +config = { + "name": "task-cancel", + "description": "Terminate a task process group", + "flows": ["common"], + "triggers": [queue("common.task.cancel")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + trace_id = get_origin_trace_id(input_data, ctx) + pid = process_pid(trace_id) + found = cancel_process(trace_id) + if pid is not None: + await asyncio.sleep(5) + force_cancel_process(trace_id, pid) + await ctx.state.set( + trace_id, + "cancelled", + { + "status": 200, + "body": { + "success": False, + "failure": False, + "cancelled": True, + "processing": False, + "trace_id": trace_id, + "process_found": found, + }, + }, + ) diff --git a/bbdev/api/steps/config/.gitignore b/bbdev/api/steps/config/.gitignore new file mode 100644 index 000000000..08616723a --- /dev/null +++ b/bbdev/api/steps/config/.gitignore @@ -0,0 +1 @@ +scripts/design_pb2.py diff --git a/bbdev/api/steps/config/01_install_api.step.py b/bbdev/api/steps/config/01_install_api.step.py new file mode 100644 index 000000000..98b7ab7ff --- /dev/null +++ b/bbdev/api/steps/config/01_install_api.step.py @@ -0,0 +1,29 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +config = { + "name": "config-install-api", + "description": "install all design configs under examples/designs", + "flows": ["config"], + "triggers": [api("POST", "/config/install")], + "enqueues": ["config.install"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + unknown = set(body) - {"design"} + if unknown: + return ApiResponse( + status=400, body={"error": f"Unexpected parameters: {sorted(unknown)}"} + ) + if "design" in body and ( + not isinstance(body["design"], str) + or not body["design"].replace("_", "").replace("-", "").isalnum() + ): + return ApiResponse( + status=400, body={"error": "--design requires a design name"} + ) + await ctx.enqueue( + {"topic": "config.install", "data": {**body, "_trace_id": ctx.trace_id}} + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/config/01_install_event.step.py b/bbdev/api/steps/config/01_install_event.step.py new file mode 100644 index 000000000..277217822 --- /dev/null +++ b/bbdev/api/steps/config/01_install_event.step.py @@ -0,0 +1,150 @@ +import importlib.util +import json +import os +import shlex +import sys +import tempfile +from pathlib import Path + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +scripts_dir = os.path.join(os.path.dirname(__file__), "scripts") + +from utils.event_common import check_result, get_origin_trace_id +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger_async + +config = { + "name": "config-install", + "description": "install all design configs under examples/designs", + "flows": ["config"], + "triggers": [queue("config.install")], + "enqueues": [], +} + + +def _load_script(name: str): + path = os.path.join(scripts_dir, name) + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise ValueError(f"cannot load {path}") + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = Path(get_buckyball_path()) + designs = bbdir / "examples" / "designs" + + selected = input_data.get("design") + tops = ( + [designs / selected / "configs" / "design.toml"] + if selected + else sorted(designs.glob("*/configs/design.toml")) + ) + if any(not top.is_file() for top in tops): + raise ValueError(f"design {selected!r} has no configs/design.toml assembly") + if not tops: + raise ValueError(f"no configs/design.toml under {designs}") + + toml2json = _load_script("1_toml2json.py") + derive = _load_script("2_parameter_derivation.py") + + # step 1: expand each design, then apply the overrides of each of its simulations + json_outs: list[Path] = [] + designs_of: list[str] = [] + for top in tops: + data = toml2json.design2json(top) + for entry, name, selection in toml2json.simulations(data): + out = ( + top.parent.parent + / "output/sims" + / selection + / "generated/config/config.json" + ) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text( + json.dumps( + toml2json.simulation2json(data, entry, name, selection), + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + json_outs.append(out) + designs_of.append(top.parent.parent.name) + ctx.logger.info(f"Wrote simulation config JSON: {out}") + + # step 2: derive computed parameters from each simulation config + derived_outs: list[Path] = [] + for design, json_path in zip(designs_of, json_outs): + data = json.loads(json_path.read_text(encoding="utf-8")) + derived_path = json_path.parent / "derived.json" + derive.write_derived(data, bbdir, design, derived_path) + derived_outs.append(derived_path) + ctx.logger.info(f"Wrote derived config: {derived_path}") + + # step 3.1: generate design_pb2.py from design.proto + scripts = Path(scripts_dir) + with tempfile.TemporaryDirectory(prefix="bbdev-proto-") as generated: + schemas = [ + scripts / "proto" / "design.proto", + scripts / "proto" / "hardware.proto", + *sorted(Path(bbdir).glob("examples/prefab/balls/*/configs/*_params.proto")), + *sorted(Path(bbdir).glob("examples/prefab/cores/*/configs/*_params.proto")), + ] + includes = sorted({schema.parent for schema in schemas}) + cmd = " ".join( + [ + "protoc", + *(f"-I{shlex.quote(str(directory))}" for directory in includes), + f"--python_out={shlex.quote(generated)}", + *(shlex.quote(str(schema)) for schema in schemas), + ] + ) + ctx.logger.info(f"Running protoc: {cmd}") + result = await stream_run_logger_async( + cmd=cmd, + logger=ctx.logger, + stdout_prefix="protoc", + stderr_prefix="protoc", + task_scope=origin_tid, + ) + if result.returncode != 0: + raise RuntimeError(f"protoc failed with returncode={result.returncode}") + for source in Path(generated).glob("*_pb2.py"): + content = source.read_bytes() + destination = scripts / source.name + if not destination.is_file() or destination.read_bytes() != content: + destination.write_bytes(content) + ctx.logger.info(f"Generated design python bindings: {scripts / 'design_pb2.py'}") + + # step 3.2: one design.pb per simulation, the one input of every build system + json2proto = _load_script("3_json2proto.py") + pb_outs: list[Path] = [] + for json_path, derived_path in zip(json_outs, derived_outs): + out = json_path.parent.parent / "design.pb" + json2proto.write_design_pb(json_path, derived_path, out, bbdir) + pb_outs.append(out) + ctx.logger.info(f"Wrote simulation protobuf: {out}") + + await check_result( + ctx, + 0, + continue_run=False, + extra_fields={ + "task": "install", + "json": [str(p) for p in json_outs], + "derived": [str(p) for p in derived_outs], + "pb2": str(scripts / "design_pb2.py"), + "design_pb": [str(p) for p in pb_outs], + }, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/config/scripts/1_toml2json.py b/bbdev/api/steps/config/scripts/1_toml2json.py new file mode 100644 index 000000000..ec014c380 --- /dev/null +++ b/bbdev/api/steps/config/scripts/1_toml2json.py @@ -0,0 +1,265 @@ +#!/usr/bin/env python3 +"""Recursively expand one TOML entry and every include / *.toml reference into JSON.""" + +from __future__ import annotations + +import json +import re +import sys +import tomllib +from pathlib import Path +from typing import Any + + +def _die(msg: str) -> None: + print(f"1_toml2json: {msg}", file=sys.stderr) + raise SystemExit(1) + + +def _load(path: Path) -> dict[str, Any]: + if not path.is_file(): + _die(f"missing TOML: {path}") + with path.open("rb") as f: + data = tomllib.load(f) + if not isinstance(data, dict): + _die(f"TOML root must be a table: {path}") + return data + + +IP_DEFAULTS = { + "uncached": "arch/src/main/scala/framework/core/memory/uncached/configs/default.toml", + "fetch": "arch/src/main/scala/framework/core/execution/fetch/src/main/scala/configs/default.toml", + "fpu": "arch/src/main/scala/framework/core/execution/fpu/src/main/scala/configs/default.toml", + "controller": "arch/src/main/scala/framework/core/controller/configs/default.toml", + "axi": "arch/src/main/scala/framework/ip/axi4/configs/default.toml", + "requesters": "arch/src/main/scala/framework/ip/coherence/configs/requesters.toml", + "ddr": "arch/src/main/scala/framework/device/ddr/configs/default.toml", + "bootrom": "arch/src/main/scala/framework/device/peripherals/bootrom/configs/default.toml", + "clint": "arch/src/main/scala/framework/device/peripherals/clint/configs/default.toml", + "plic": "arch/src/main/scala/framework/device/peripherals/plic/configs/default.toml", + "scu": "arch/src/main/scala/framework/device/peripherals/scu/configs/default.toml", + "frontend": "arch/src/main/scala/framework/acc/frontend/configs/default.toml", + "memdomain": "arch/src/main/scala/framework/acc/memdomain/configs/default.toml", + "gpdomain": "arch/src/main/scala/framework/acc/gpdomain/configs/default.toml", + "transport": ( + "arch/src/main/scala/framework/acc/" + "memdomain/src/main/scala/tomem/configs/default.toml" + ), +} +CONFIG_REFERENCES = set(IP_DEFAULTS) | { + "device", + "sims", + "accelerator", + "balldomain", + "cpu", +} + + +def _reference(base: Path, value: str, slot: str | None) -> Path: + if value == "default": + if slot not in IP_DEFAULTS: + _die(f"{base}: {slot!r} requires an explicit configuration path") + return Path(__file__).resolve().parents[5] / IP_DEFAULTS[slot] + return (base / value).resolve() + + +def _merge(left: dict, right: dict, path: Path) -> dict: + result = dict(left) + for key, value in right.items(): + if key == "_file": + continue + if key not in result: + result[key] = value + elif isinstance(result[key], dict) and isinstance(value, dict): + result[key] = _merge(result[key], value, path / key) + else: + _die(f"{path}: include key collision: {key}") + return result + + +def _walk_file( + path: Path, + includes: list[str], + stack: list[str], + override: bool = False, +) -> dict[str, Any]: + path = path.resolve() + key = str(path) + if key in stack: + _die("include cycle: %s" % " -> ".join(stack + [key])) + if key not in includes: + includes.append(key) + data = _load(path) + stack.append(key) + out = _walk(data, path.parent, includes, stack, override=override) + stack.pop() + if not isinstance(out, dict): + _die(f"{path}: root must be a table") + if "_file" in out: + _die(f"{path}: reserved key '_file'") + out["_file"] = key + return out + + +def _walk( + obj: Any, + base: Path, + includes: list[str], + stack: list[str], + slot: str | None = None, + override: bool = False, +) -> Any: + if isinstance(obj, dict): + inc = obj.get("include") + if inc is not None: + if not isinstance(inc, str) or not inc: + _die(f"{base}: include must be a non-empty string") + loaded = _walk_file(_reference(base, inc, slot), includes, stack) + rest = {k: v for k, v in obj.items() if k != "include"} + walked = _walk(rest, base, includes, stack, override=override) + if not isinstance(walked, dict): + _die(f"{base}: include siblings must be a table") + if override: + # Inside a device config, include siblings replace included keys. + _override(loaded, walked, str(base / inc)) + return loaded + return _merge(loaded, walked, base / inc) + result = {} + for key, value in obj.items(): + if key == "config" and value == "default" and "ip" in obj: + repo = Path(__file__).resolve().parents[5] + catalog = _load(repo / "arch/uvm.toml") + path = repo / catalog["paths"][obj["ip"]] / "configs/uvm.toml" + result[key] = _walk_file(path, includes, stack) + elif key == "param" and value == "default": + component = obj["component"] + if not isinstance(component, str) or not component.isidentifier(): + _die(f"{base}: invalid Ball component {component!r}") + path = ( + Path(__file__).resolve().parents[5] + / "examples/prefab/balls" + / component + / "configs/default.toml" + ) + result[key] = _walk_file(path, includes, stack) + else: + # A device config holds overrides of the design config. + result[key] = _walk( + value, base, includes, stack, key, override or key == "configs" + ) + return result + if isinstance(obj, list): + return [_walk(x, base, includes, stack, slot, override) for x in obj] + if isinstance(obj, str) and obj == "default" and slot in CONFIG_REFERENCES: + return _walk_file(_reference(base, obj, slot), includes, stack) + if isinstance(obj, str) and obj.endswith(".toml"): + path = (base / obj).resolve() + if not path.is_file(): + _die(f"toml path does not exist: {path} (from {base} + {obj!r})") + return _walk_file(path, includes, stack, override) + return obj + + +def toml2json(top: Path, design: Path | None = None) -> dict[str, Any]: + path = Path(top) + if not path.is_file(): + raise FileNotFoundError(f"missing TOML: {path}") + path = path.resolve() + includes: list[str] = [] + if design is None: + data = _walk_file(path, includes, []) + else: + design = Path(design) + if not design.is_file(): + raise FileNotFoundError(f"missing TOML: {design}") + design = design.resolve() + raw = _load(path) + if not isinstance(raw.get("device"), dict): + _die(f"{path}: missing [device]") + raw["device"] = {"include": design.relative_to(path.parent).as_posix()} + includes.append(str(path)) + data = _walk(raw, path.parent, includes, [str(path)]) + if not isinstance(data, dict): + _die(f"{path}: root must be a table") + if "_file" in data: + _die(f"{path}: reserved key '_file'") + data["_file"] = str(path) + if "includes" in data: + _die(f"{path}: reserved key 'includes'") + data["includes"] = includes + return data + + +def design2json(top: Path) -> dict[str, Any]: + data = toml2json(top) + data["workloads"] = _walk_file( + Path(top).resolve().parent.parent / "workloads/configs/tests.toml", + data["includes"], + [], + ) + return data + + +def _override(config: dict, override: dict, path: str) -> None: + for key, value in override.items(): + if key == "_file": + continue + if key not in config: + _die(f"{path}: override key {key!r} is not in the design config") + if isinstance(config[key], dict) and isinstance(value, dict): + _override(config[key], value, f"{path}.{key}") + else: + config[key] = value + + +def simulations(data: dict[str, Any]) -> list[tuple[str, str | None, str]]: + """(entry, config, selection) of every simulation of the design: one per + config registered on the device entry, one per core entry.""" + design = Path(data["_file"]).parent.parent.name + prefix = "".join(part.capitalize() for part in re.split("[-_]", design)) + result = [] + for entry, registration in data["sims"]["targets"].items(): + if entry == "_file": + continue + if entry != "device": + if "configs" in registration: + _die(f"targets.{entry}: only the device entry registers configs") + result.append((entry, None, entry)) + continue + if not registration["configs"]: + _die("targets.device.configs registers no config") + for name in registration["configs"]: + if not re.fullmatch(prefix + "[A-Za-z0-9]*Config", name): + _die(f"targets.device.configs: {name!r} must be {prefix}Config") + result.append((entry, name, f"device/{name}")) + return result + + +def simulation2json( + data: dict[str, Any], entry: str, name: str | None, selection: str +) -> dict[str, Any]: + """The config of one simulation: the design config with the tables of device + config `name` replacing the same keys (arrays whole).""" + config = json.loads(json.dumps(data)) + registration = config["sims"]["targets"][entry] + if name is not None: + overrides = registration.pop("configs")[name] + _override(config, overrides, overrides["_file"]) + config["sims"]["targets"] = {entry: registration} + config["sims"]["selection"] = selection + return config + + +def main() -> None: + if len(sys.argv) != 3: + _die(f"usage: {sys.argv[0]} DESIGN.toml SELECTION") + data = design2json(Path(sys.argv[1])) + found = [s for s in simulations(data) if s[2] == sys.argv[2]] + if not found: + _die(f"{sys.argv[1]}: no simulation {sys.argv[2]!r}") + out = simulation2json(data, *found[0]) + sys.stdout.write(json.dumps(out, indent=2, sort_keys=True) + "\n") + + +if __name__ == "__main__": + main() diff --git a/bbdev/api/steps/config/scripts/2_parameter_derivation.py b/bbdev/api/steps/config/scripts/2_parameter_derivation.py new file mode 100644 index 000000000..c6dbda817 --- /dev/null +++ b/bbdev/api/steps/config/scripts/2_parameter_derivation.py @@ -0,0 +1,402 @@ +#!/usr/bin/env python3 +"""Derive computed design parameters from step1 config.json.""" + +from __future__ import annotations + +import json +import sys +from collections.abc import Iterator +from pathlib import Path +from typing import Any + +if str(Path(__file__).resolve().parent) not in sys.path: + sys.path.insert(0, str(Path(__file__).resolve().parent)) +from ball_normalize import normalize_ball_domain + + +def _die(msg: str) -> None: + raise ValueError(msg) + + +def _repo_rel(repo: Path, path: str | Path) -> str: + p = Path(path) + if not p.is_absolute(): + return p.as_posix() + return p.resolve().relative_to(repo.resolve()).as_posix() + + +def _require_id(obj: dict[str, Any], key: str, ctx: str) -> int: + val = obj.get(key) + if type(val) is not int or val < 0: + _die(f"{ctx}: missing {key} (non-negative int required)") + return val + + +def _core_threads(core: dict[str, Any]) -> list[dict[str, Any]]: + threads = core["threads"] + if not isinstance(threads, list): + _die("threads must be an explicit instance list") + ids = [_require_id(thread, "id", "thread") for thread in threads] + if len(set(ids)) != len(ids): + _die("duplicate thread ID within core") + return threads + + +def configured_cores(design: dict[str, Any]) -> list[dict[str, Any]]: + cores = design["cores"] + if not isinstance(cores, list): + _die("design cores must be an explicit instance list") + harts = [_require_id(core, "hartId", "core") for core in cores] + if len(set(harts)) != len(harts): + _die("duplicate hart ID") + for core in cores: + if "count" in core or "factory" in core: + _die( + "core instances must be declared directly; hardware is assembled by the design" + ) + return cores + + +def iter_threads(topo: dict[str, Any]) -> Iterator[dict[str, Any]]: + for core in configured_cores(topo): + yield from _core_threads(core) + + +def bank_params(thread: dict, pkg: str) -> tuple[int, int, int]: + mem = thread.get("memdomain") + if not isinstance(mem, dict): + _die(f"{pkg}: missing memdomain") + bank = mem.get("bank") + if not isinstance(bank, dict): + _die(f"{pkg}: missing [bank]") + num, width, entries = bank.get("num"), bank.get("width"), bank.get("entries") + if not isinstance(num, int) or num <= 0: + _die(f"{pkg}: bank.num must be a positive int") + if not isinstance(width, int) or width < 8 or width % 8 != 0: + _die(f"{pkg}: bank.width must be a positive multiple of 8") + if not isinstance(entries, int) or entries <= 0: + _die(f"{pkg}: bank.entries must be a positive int") + return num, width, entries + + +def _balldomain_base_dir(repo: Path, thread: dict, pkg: str) -> str: + bd = thread.get("balldomain") + if isinstance(bd, dict): + rel = bd.get("_file") + if isinstance(rel, str): + return _repo_rel(repo, Path(rel).parent) + raise ValueError(f"{pkg}: missing BallDomain configuration source") + + +def _ball_num(thread: dict) -> int: + bd = normalize_ball_domain(thread) + if not isinstance(bd, dict): + return 0 + return len(bd["balls"]) + + +def _derive_threads(repo: Path, topo: dict[str, Any]) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + for index, thread in enumerate(iter_threads(topo)): + rel = thread.get("_file") + if not isinstance(rel, str): + _die("thread config missing _file") + pkg = thread["component"] + if not isinstance(pkg, str) or not pkg.isidentifier(): + _die(f"invalid component name: {pkg!r}") + role = thread.get("name") + if role is not None and not isinstance(role, str): + _die("thread name must be a string") + thread_id = _require_id(thread, "id", f"thread index {index}") + ball_num = _ball_num(thread) + if isinstance(thread.get("memdomain"), dict): + num, width, entries = bank_params(thread, pkg) + elif ball_num > 0: + _die(f"{pkg}: buckyball thread missing memdomain") + else: + num, width, entries = 0, 0, 0 + entry: dict[str, Any] = { + "index": index, + "thread_id": thread_id, + "pkg": pkg, + "role": role or "", + "profile": thread["compiler"]["name"], + "cpu_kind": thread["cpu"]["kind"] if "cpu" in thread else "", + "config_path": _repo_rel(repo, rel), + "balldomain_base_dir": _balldomain_base_dir(repo, thread, pkg), + "bank_num": num, + "bank_width": width, + "bank_entries": entries, + "ball_num": ball_num, + } + if ball_num > 0: + bd = normalize_ball_domain(thread) + if not isinstance(bd, dict): + _die(f"{pkg}: missing balldomain") + entry["balls"] = [ + {"id": ball["id"], "component": ball["component"]} + for ball in bd["balls"] + ] + out.append(entry) + return out + + +def _validate_spm(config: dict[str, Any]) -> None: + base, size, bits = (config[key] for key in ("base", "bytes", "dataBits")) + if any(type(value) is not int for value in (base, size, bits)): + _die("SPM base/bytes/dataBits must be integers") + if bits < 64 or bits & (bits - 1): + _die("SPM dataBits must be a power of two >= 64") + beat = bits // 8 + if ( + size < 2 * beat + or size & (size - 1) + or base < 0 + or base % beat + or base + size > 1 << 64 + ): + _die( + "SPM requires an aligned base and power-of-two capacity within the 64-bit address space" + ) + + +def _derive_cores( + repo: Path, topo: dict[str, Any], threads: list[dict[str, Any]] +) -> list[dict[str, Any]]: + placements: list[dict[str, Any]] = [] + offset = 0 + for core in configured_cores(topo): + core_path = core.get("_file") + if not isinstance(core_path, str): + _die("core missing _file") + core_threads = _core_threads(core) + n = len(core_threads) + ants = [ + thread + for thread in core_threads + if "cpu" in thread and thread["cpu"]["kind"] == "ant" + ] + if ants: + shared_spm = core.get("tss") + if not isinstance(shared_spm, dict): + _die("Ant compute core requires explicit tss geometry") + _validate_spm(shared_spm) + for thread in ants: + params = thread["cpu"]["config"] + tls = params["tls"] + _validate_spm(tls) + code = params["codeBytes"] + if type(code) is not int or code < 32 or code & (code - 1): + _die("Ant codeBytes must be a power of two >= 32") + if not 1 <= params["taskBits"] <= 64: + _die("Ant taskBits must be in [1,64]") + if min(tls["base"], shared_spm["base"]) < code: + _die("Ant code overlaps TLS/TSS") + if not ( + tls["base"] + tls["bytes"] <= shared_spm["base"] + or shared_spm["base"] + shared_spm["bytes"] <= tls["base"] + ): + _die("Ant TLS/TSS windows overlap") + if tls["dataBits"] != shared_spm["dataBits"]: + _die("Ant TLS/TSS access widths must match") + shared = core.get("sharedMem") + if isinstance(shared, dict): + if shared["bankWidth"] != 128: + _die("sharedMem.bankWidth must be 128 bits") + rows = shared["bankEntries"] + if type(rows) is not int or rows < 2 or rows > 65536 or rows & (rows - 1): + _die("sharedMem.bankEntries must be a power of two in [2,65536]") + if shared["enable"] and ( + shared["entries"] <= 0 or shared["entries"] % rows + ): + _die("sharedMem.entries must be a positive multiple of bankEntries") + vbc = _require_id(shared, "virtualBankCount", "shared memory") + indices = list(range(offset, offset + n)) + placements.append( + { + "hart_id": core["hartId"], + "path": _repo_rel(repo, core_path), + "thread_indices": indices, + "threads_per_core": n, + "virtual_bank_count": vbc, + } + ) + offset += n + return placements + + +def _bemu_balls(repo: Path, profiles: list[dict[str, Any]]) -> list[dict[str, str]]: + seen: set[str] = set() + balls: list[dict[str, str]] = [] + for profile in profiles: + bd = normalize_ball_domain(profile["config"]) + if not isinstance(bd, dict): + continue + for ball in bd["balls"]: + instructions = ball["instructions"] + if any(entry["funct7"] == 0 for entry in instructions): + _die("funct7 zero is not a Buckyball instruction") + if not any( + entry["funct7"] not in {1, 16, 32, 33, 34, 35} for entry in instructions + ): + continue + ball_dir = ball["component"] + ball_class = f"examples.balls.{ball_dir}.{ball['name']}" + if ball_class in seen: + continue + seen.add(ball_class) + emu_lib = ( + repo + / "examples" + / "prefab" + / "balls" + / ball_dir + / "emu" + / "src" + / "lib.rs" + ) + if not emu_lib.is_file(): + _die(f"missing BEMU ball source for {ball_class}: {emu_lib}") + balls.append( + { + "emu_class": ball_class, + "component": ball_dir, + "emu_lib": _repo_rel(repo, emu_lib), + } + ) + return balls + + +def _ball_ctest_dirs(repo: Path, thread: dict[str, Any]) -> list[str]: + bd = normalize_ball_domain(thread) + if not isinstance(bd, dict): + _die("thread missing balldomain for ctest dirs") + dirs: list[str] = [] + for ball in bd["balls"]: + path = ( + repo + / "examples" + / "prefab" + / "balls" + / ball["component"] + / "workloads" + / "ctests" + ) + if not path.is_dir(): + _die(f"missing {path}") + dirs.append(_repo_rel(repo, path)) + return dirs + + +def compiler_profiles(device: dict[str, Any]) -> list[dict[str, Any]]: + profiles = [] + for core in configured_cores(device): + for thread in core["threads"]: + declaration = thread["compiler"] + profiles.append( + { + "name": declaration["name"], + "component": declaration["component"], + "config": thread, + } + ) + for core in configured_cores(device): + if "accelerator" in core: + declaration = core["compiler"] + profiles.append( + { + "name": declaration["name"], + "component": declaration["component"], + "config": core["accelerator"], + } + ) + return profiles + + +def _derive_targets( + repo: Path, profiles: list[dict[str, Any]] +) -> dict[str, dict[str, Any]]: + targets = {} + for profile in profiles: + name, pkg, config = profile["name"], profile["component"], profile["config"] + bank_params(config, pkg) + targets[name] = { + "ball_ctest_dirs": _ball_ctest_dirs(repo, config), + } + return targets + + +def _derive_harts( + cores: list[dict[str, Any]], threads: list[dict[str, Any]] +) -> list[dict[str, Any]]: + return [{"hart_id": core["hart_id"]} for core in cores] + + +def derive(data: dict[str, Any], repo: Path, design: str) -> dict[str, Any]: + repo = repo.resolve() + topo = data.get("device") + if not isinstance(topo, dict): + _die(f"{design}: missing device") + + root_file = data.get("_file") + if not isinstance(root_file, str): + _die(f"{design}: missing root _file") + design_file = topo.get("_file") + if not isinstance(design_file, str): + _die(f"{design}: device missing _file") + + includes_raw = data.get("includes") + if not isinstance(includes_raw, list): + _die(f"{design}: includes must be a list") + + threads = _derive_threads(repo, topo) + cores = _derive_cores(repo, topo, threads) + profiles = compiler_profiles(topo) + targets = _derive_targets(repo, profiles) + harts = _derive_harts(cores, threads) + if sorted(hart["hart_id"] for hart in harts) != list(range(len(harts))): + _die("complete design hart IDs must be contiguous from zero") + n_cores = len(configured_cores(topo)) + + return { + "design": design, + "name": Path(root_file).parent.parent.name, + "path": _repo_rel(repo, root_file), + "core_config_path": _repo_rel(repo, design_file), + "includes": [_repo_rel(repo, p) for p in includes_raw], + "n_cores": n_cores, + "targets": targets, + "harts": harts, + "threads": threads, + "cores": cores, + "bemu": { + "balls": _bemu_balls(repo, profiles), + }, + } + + +def write_derived( + data: dict[str, Any], repo: Path, design: str, out: Path +) -> dict[str, Any]: + derived = derive(data, repo, design) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text( + json.dumps(derived, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return derived + + +def main() -> None: + if len(sys.argv) not in (4, 5): + _die(f"usage: {sys.argv[0]} REPO DESIGN CONFIG.json [OUT.json]") + repo = Path(sys.argv[1]) + design = sys.argv[2] + config = Path(sys.argv[3]) + out = Path(sys.argv[4]) if len(sys.argv) == 5 else config.parent / "derived.json" + data = json.loads(config.read_text(encoding="utf-8")) + derived = write_derived(data, repo, design, out) + sys.stdout.write(json.dumps(derived, indent=2, sort_keys=True) + "\n") + + +if __name__ == "__main__": + main() diff --git a/bbdev/api/steps/config/scripts/3_json2proto.py b/bbdev/api/steps/config/scripts/3_json2proto.py new file mode 100644 index 000000000..bfce53628 --- /dev/null +++ b/bbdev/api/steps/config/scripts/3_json2proto.py @@ -0,0 +1,573 @@ +#!/usr/bin/env python3 +"""Build Design protobuf from config.json + derived.json.""" + +from __future__ import annotations + +import importlib.util +import json +import hashlib +import re +import tomllib +from google.protobuf.json_format import ParseDict +import sys +from pathlib import Path +from typing import Any + +if str(Path(__file__).resolve().parent) not in sys.path: + sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import design_pb2 as pb # noqa: E402 +from ball_normalize import normalize_ball_domain +from ball_parameters import pack_parameters + + +def _load_derive(): + path = Path(__file__).resolve().with_name("2_parameter_derivation.py") + spec = importlib.util.spec_from_file_location("parameter_derivation", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load {path}") + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +_derive = _load_derive() + + +def _rel(bbdir: Path, path: str | Path) -> str: + p = Path(path) + if p.is_absolute(): + return p.resolve().relative_to(bbdir.resolve()).as_posix() + return p.as_posix() + + +def _raw_thread_list(designs: dict[str, Any]) -> list[dict[str, Any]]: + return list(_derive.iter_threads(designs)) + + +def _fill_bank(msg: pb.BankConfig, d: dict[str, Any]) -> None: + msg.num = d["num"] + msg.width = d["width"] + msg.entries = d["entries"] + msg.mask_len = d["maskLen"] + msg.request_ports = d["requestPorts"] + msg.layout_enable = d["layoutEnable"] + + +def _fill_mem(msg: pb.MemDomainConfig, d: dict[str, Any], bbdir: Path) -> None: + msg.source_path = _rel(bbdir, d["_file"]) + _fill_bank(msg.bank, d["bank"]) + dma = d["dma"] + msg.dma.bus_width = dma["busWidth"] + mmio = d["mmio"] + msg.mmio.enable = mmio["enable"] + msg.mmio.bank_num = mmio["bankNum"] + msg.mmio.bank_entries = mmio["bankEntries"] + msg.mmio.bank_width = mmio["bankWidth"] + msg.mmio.read_width = mmio["readWidth"] + msg.addr_len = d["addrLen"] + + +def _fill_ball(msg: pb.BallDomain, d: dict[str, Any], bbdir: Path) -> None: + for ball in d["balls"]: + instance = msg.balls.add() + instance.id = ball["id"] + instance.name = ball["name"] + instance.component = ball["component"] + classes = [] + root = bbdir / "examples/prefab/balls" / ball["component"] / "arch" + for source in sorted(root.rglob("*Ball.scala")): + text = source.read_text() + names = re.findall( + r"class (\w+)\([^{}]*?\)\s+extends Module\s+with HasBlink\b", text, re.S + ) + if names: + package = re.search(r"(?m)^package ([\w.]+)", text).group(1) + classes.extend(f"{package}.{name}" for name in names) + if len(classes) != 1: + raise ValueError( + f"{ball['component']}: expected one Ball implementation, found {classes}" + ) + instance.rtl_class = classes[0] + port = ball["port"] + instance.in_bw = port["inBW"] + instance.out_bw = port["outBW"] + instance.mmio_read_bw = port["mmioReadBW"] + instance.mmio_write_bw = port["mmioWriteBW"] + pack_parameters( + instance, + {key: value for key, value in ball["param"].items() if key != "_file"}, + ) + for instruction in ball["instructions"]: + entry = instance.instructions.add() + entry.mnemonic = instruction["mnemonic"] + entry.funct7 = instruction["funct7"] + + +def _check_cpu_kind(cpu: dict[str, Any], pkg: str) -> str: + kind = cpu.get("kind") + if kind != "ant": + raise ValueError(f"{pkg}: kind must be 'ant', got {kind!r}") + if not isinstance(cpu.get("config"), dict): + raise ValueError(f"{pkg}: cpu.config must reference a TOML file") + return kind + + +def _fill_frontend(msg: pb.FrontendConfig, d: dict[str, Any], bbdir: Path) -> None: + msg.source_path = _rel(bbdir, d["_file"]) + msg.rob_entries = d["robEntries"] + msg.rs_out_of_order_response = d["rsOutOfOrderResponse"] + msg.bank_id_len = d["bankIdLen"] + msg.max_private_vbank = d["maxPrivateVbank"] + msg.shared_vbank_base = d["sharedVbankBase"] + msg.iter_len = d["iterLen"] + msg.sub_rob_enable = d["subRobEnable"] + msg.sub_rob_depth = d["subRobDepth"] + msg.bank_alias_enable = d["bankAliasEnable"] + + +def _fill_gpdomain(msg: pb.GpDomainConfig, d: dict[str, Any], bbdir: Path) -> None: + if type(d["enable"]) is not bool: + raise ValueError("gpdomain.enable must be an explicit boolean") + msg.enable = d["enable"] + for key in ("laneNumber", "vLen", "eLen", "iBufWords", "memoryPorts"): + if type(d[key]) is not int or d[key] <= 0: + raise ValueError(f"gpdomain.{key} must be a positive integer") + msg.source_path = _rel(bbdir, d["_file"]) + msg.lane_number = d["laneNumber"] + msg.v_len = d["vLen"] + msg.e_len = d["eLen"] + msg.i_buf_words = d["iBufWords"] + msg.memory_ports = d["memoryPorts"] + + +def _fill_core_params(msg: pb.CoreParamConfig, d: dict[str, Any]) -> None: + msg.thread_data_bytes = d["threadDataBytes"] + msg.x_len = d["xLen"] + msg.vaddr_bits = d["vaddrBits"] + msg.paddr_bits = d["paddrBits"] + msg.pg_idx_bits = d["pgIdxBits"] + msg.pg_levels = d["pgLevels"] + msg.n_pmps = d["nPMPs"] + + +def _fill_spm(msg: pb.SpmConfig, config: dict[str, Any]) -> None: + msg.base = config["base"] + msg.bytes = config["bytes"] + msg.data_bits = config["dataBits"] + + +def _fill_cpu(msg: pb.CpuConfig, cpu: dict[str, Any], context: str, bbdir: Path) -> str: + kind = _check_cpu_kind(cpu, context) + msg.kind = kind + msg.source_path = _rel(bbdir, cpu["_file"]) + config = cpu["config"] + msg.ant.code_bytes = config["codeBytes"] + msg.ant.task_bits = config["taskBits"] + _fill_spm(msg.ant.tls, config["tls"]) + return kind + + +def _hardware_fields(value: Any) -> Any: + if isinstance(value, dict): + return { + key: _hardware_fields(item) for key, item in value.items() if key != "_file" + } + if isinstance(value, list): + return [_hardware_fields(item) for item in value] + return value + + +def _thread_signature(thread: pb.ThreadInstance) -> int: + hardware = pb.ThreadInstance() + hardware.CopyFrom(thread) + for ball in hardware.balldomain.balls: + ball.ClearField("rtl_class") + for field in ( + "index", + "role", + "config_path", + "source_path", + "rtl_class", + "balldomain_base_dir", + "local_id", + "signature", + "transport", + "profile", + ): + hardware.ClearField(field) + for message in (hardware.cpu, hardware.mem, hardware.frontend, hardware.gpdomain): + message.ClearField("source_path") + digest = hashlib.sha256(hardware.SerializeToString(deterministic=True)).digest() + return int.from_bytes(digest[:8], "little") + + +def _fill_thread( + ci: pb.ThreadInstance, + raw: dict[str, Any], + meta: dict[str, Any], + bbdir: Path, + *, + hardware: bool, +) -> None: + ci.index = meta["index"] + ci.local_id = meta["thread_id"] + ci.role = meta["role"] + ci.pkg = meta["pkg"] + ci.profile = raw["compiler"]["name"] if "compiler" in raw else meta["role"] + ci.config_path = meta["config_path"] + source = (Path(raw["_file"]).parent / raw["source"]).resolve() + if not source.is_dir(): + raise FileNotFoundError(f"missing component source directory: {source}") + ci.source_path = _rel(bbdir, source) + if hardware: + classes = [] + for implementation in sorted((source / "arch").rglob("*.scala")): + text = implementation.read_text() + names = re.findall( + r"class (\w+)\([^{}]*?\)\s+extends Module\s+with HasTLink\b", text, re.S + ) + if names: + package = re.search(r"(?m)^package ([\w.]+)", text).group(1) + classes.extend(f"{package}.{name}" for name in names) + if len(classes) != 1: + raise ValueError( + f"{source}: expected one Thread implementation, found {classes}" + ) + ci.rtl_class = classes[0] + ci.balldomain_base_dir = meta["balldomain_base_dir"] + if "cpu" in raw: + _fill_cpu(ci.cpu, raw["cpu"], meta["pkg"], bbdir) + domain = normalize_ball_domain(raw) + if domain is not None: + _fill_ball(ci.balldomain, domain, bbdir) + if "memdomain" in raw: + _fill_mem(ci.mem, raw["memdomain"], bbdir) + if "frontend" in raw: + _fill_frontend(ci.frontend, raw["frontend"], bbdir) + if "gpdomain" in raw: + _fill_gpdomain(ci.gpdomain, raw["gpdomain"], bbdir) + if "transport" in raw: + ParseDict(_hardware_fields(raw["transport"]), ci.transport) + ci.signature = _thread_signature(ci) + + +def fill_design( + config: dict[str, Any], derived: dict[str, Any], bbdir: Path +) -> pb.Design: + """Serialize explicitly declared instances and their typed parameters into design.pb.""" + b = pb.Design() + b.name = derived["name"] + b.path = derived["path"] + b.n_cores = derived["n_cores"] + b.includes.extend(derived["includes"]) + + for key in ("ddr", "clint", "plic", "scu"): + ParseDict(_hardware_fields(config["device"][key]), getattr(b.device, key)) + bootrom = config["device"]["bootrom"] + image_path = bbdir / bootrom["image"] + image = bytearray(image_path.read_bytes()) + if len(image) < 24 or len(image) > bootrom["bytes"]: + raise ValueError( + "BootROM image must contain its configuration header and fit the ROM" + ) + if any(image[8:24]): + raise ValueError( + "BootROM image configuration header must contain zero placeholders" + ) + image[8:16] = b.n_cores.to_bytes(8, "little") + image[16:24] = bootrom["entry"].to_bytes(8, "little") + b.device.bootrom.base = bootrom["base"] + b.device.bootrom.bytes = bootrom["bytes"] + b.device.bootrom.image.extend(byte if byte < 128 else byte - 256 for byte in image) + b.includes.append(_rel(bbdir, image_path)) + + ParseDict( + {key: _hardware_fields(config["sims"][key]) for key in ("diffTest", "memory")}, + b.sims, + ) + if not b.device.IsInitialized() or not b.sims.IsInitialized(): + raise ValueError("Device and simulation parameters must be complete") + + root = Path(derived["path"]).parent.parent + ((name, registration),) = config["sims"]["targets"].items() + if not re.fullmatch(r"(core/)?[A-Za-z0-9_-]+", name) or name == "core": + raise ValueError(f"invalid simulation target: {name}") + if name.startswith("core/") and "regression" in registration: + raise ValueError(f"{name}: regression belongs to the device entry") + target = b.simulation + target.selection = config["sims"]["selection"] + target.output = str(root / "output/sims" / target.selection) + target.directory = str(root / "sims" / name) + for field in ("emu", "stack", "workloads"): + setattr(target, field, str(root / "sims" / name / field)) + target.arch.directory = str(root / "sims" / name / "arch") + target.arch.module = "simulation" + target.arch.config = str(Path(target.output) / "generated/design.pb") + target.entry_class = registration["config"] + target.top = "BBSimHarness" + target.stack_name = registration["stack_name"] + manifest = bbdir / target.emu / "Cargo.toml" + target.emu_package = tomllib.loads(manifest.read_text())["package"]["name"] + for resource in ("regression", "kernel", "tapeout"): + if resource in registration: + target.resources[resource] = str(root / registration[resource]) + for entry in ( + Path(target.arch.directory) / "build.sc", + Path(target.stack) / "CMakeLists.txt", + Path(target.workloads) / "CMakeLists.txt", + ): + if not (bbdir / entry).is_file(): + raise FileNotFoundError(bbdir / entry) + from verification import ip_registrations + + profiles = ip_registrations( + bbdir, config["sims"]["verification"]["registry"]["ips"] + ) + b.verification.ips.extend(dict.fromkeys(profile.ip for profile in profiles)) + b.verification.profiles.extend(profiles) + + for raw in _derive.configured_cores(config["device"]): + _append_core(b, raw, bbdir) + + if "sharedBankFabric" in config["device"]: + fabric = config["device"]["sharedBankFabric"] + harts = {core.hart_id for core in b.device.cores} + owner, participants = fabric["ownerHart"], fabric["participantHarts"] + if ( + type(owner) is not int + or owner not in harts + or not participants + or owner in participants + or len(set(participants)) != len(participants) + or set(participants) | {owner} != harts + or len(fabric["bankNodes"]) != len(harts) + ): + raise ValueError( + "sharedBankFabric must explicitly connect each declared hart once" + ) + ParseDict(_hardware_fields(fabric), b.device.shared_bank_fabric) + + domains = [ + core.accelerator.balldomain + for core in b.device.cores + if core.HasField("accelerator") + ] + domains.extend(thread.balldomain for thread in b.threads) + b.verification.balls.extend( + dict.fromkeys(ball.component for domain in domains for ball in domain.balls) + ) + + targets = derived["targets"] + compiled = {} + for profile in _derive.compiler_profiles(config["device"]): + name = profile["name"] + raw = profile["config"] + meta = { + "index": 0, + "thread_id": 0, + "role": name, + "pkg": profile["component"], + "config_path": _rel(bbdir, raw["_file"]), + "balldomain_base_dir": _derive._balldomain_base_dir( + bbdir, raw, profile["component"] + ), + } + instance = pb.ThreadInstance() + _fill_thread(instance, raw, meta, bbdir, hardware=False) + if name in compiled: + if compiled[name].thread.signature != instance.signature: + raise ValueError( + f"compiler {name}: instances have different parameters; declare distinct compiler names" + ) + continue + p = b.profiles.add() + compiled[name] = p + p.name = name + p.ball_ctest_dirs.extend(targets[name]["ball_ctest_dirs"]) + p.thread.CopyFrom(instance) + # Programs of the profile run on the first core that runs it, on its + # accelerator or on one of its threads. + p.core.CopyFrom( + next( + core + for core in b.device.cores + if core.profile == name + or any(b.threads[i].profile == name for i in core.thread_indices) + ) + ) + + bemu = derived["bemu"] + for ball in bemu["balls"]: + e = b.bemu.balls.add() + e.emu_class = ball["emu_class"] + e.component = ball["component"] + e.emu_lib = ball["emu_lib"] + + harts = {core.hart_id for core in b.device.cores} + tests = config["workloads"] + for key, field in (("controlHart", "control_hart"), ("reportHart", "report_hart")): + hart = tests[key] + if type(hart) is not int or hart not in harts: + raise ValueError(f"workload {key}: unknown hart {hart!r}") + setattr(b.workload, field, hart) + return b + + +def write_thread_pb(raw: dict[str, Any], out_path: Path, bbdir: Path, pkg: str) -> Path: + """One thread config, independent of any design: the compiler input.""" + rel = _rel(bbdir, raw["_file"]) + meta = { + "index": 0, + "thread_id": 0, + "role": "", + "pkg": pkg, + "config_path": rel, + "balldomain_base_dir": _derive._balldomain_base_dir(bbdir, raw, pkg), + } + thread = pb.ThreadInstance() + _fill_thread(thread, raw, meta, bbdir, hardware=True) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_bytes(thread.SerializeToString()) + return out_path + + +def _append_core(design: pb.Design, raw: dict[str, Any], bbdir: Path) -> None: + core = design.device.cores.add() + core.path = _rel(bbdir, raw["_file"]) + component = raw.get("component") + if not isinstance(component, str) or not component: + raise ValueError("Core component must be explicitly named") + core.component = component + core.profile = raw["compiler"]["name"] if "compiler" in raw else "" + core.hart_id = raw["hartId"] + core.reset_vector = raw["resetVector"] + core.mtime_address = raw["mtimeAddress"] + for key in ("memory", "fetch", "fpu", "interconnect", "c2c", "controller"): + ParseDict(_hardware_fields(raw[key]), getattr(core, key)) + if "tbus" in raw: + ParseDict(_hardware_fields(raw["tbus"]), core.tbus) + if "dmaCache" in raw: + ParseDict(_hardware_fields(raw["dmaCache"]), core.dma_cache) + if not core.IsInitialized(): + raise ValueError("Core IP configuration is incomplete") + if "implementation" in raw: + from importlib import import_module + + implementation = import_module( + f"{raw['implementation']['component']}_params_pb2" + ).Params() + ParseDict( + _hardware_fields( + {k: v for k, v in raw["implementation"].items() if k != "component"} + ), + implementation, + ) + core.implementation.Pack(implementation, deterministic=True) + if "param" in raw or "implementation" not in raw: + _fill_core_params(core.param, raw["param"]) + if "tss" in raw: + _fill_spm(core.tss, raw["tss"]) + shared = raw["sharedMem"] + core.virtual_bank_count = shared["virtualBankCount"] + core.shared_mem.enable = shared["enable"] + core.shared_mem.entries = shared["entries"] + core.shared_mem.bank_entries = shared["bankEntries"] + core.shared_mem.bank_width = shared["bankWidth"] + core.shared_mem.input_channels = shared["inputChannels"] + core.shared_mem.default_group_count = shared["defaultGroupCount"] + core.shared_mem.virtual_bank_count = shared["virtualBankCount"] + core.shared_mem.latency = shared["latency"] + if "accelerator" in raw: + acc = raw["accelerator"] + _fill_ball(core.accelerator.balldomain, acc["balldomain"], bbdir) + _fill_mem(core.accelerator.mem, acc["memdomain"], bbdir) + _fill_frontend(core.accelerator.frontend, acc["frontend"], bbdir) + _fill_gpdomain(core.accelerator.gpdomain, acc["gpdomain"], bbdir) + ParseDict(_hardware_fields(acc["transport"]), core.accelerator.transport) + threads = raw["threads"] + ids = [thread["id"] for thread in threads] + if any(type(value) is not int or value < 0 for value in ids) or len( + set(ids) + ) != len(ids): + raise ValueError( + "Core-local thread IDs must be explicit, non-negative and unique" + ) + offset = len(design.threads) + for local, thread in enumerate(threads): + index = offset + local + package = thread["component"] + if not isinstance(package, str) or not package.isidentifier(): + raise ValueError("invalid thread component") + meta = { + "index": index, + "thread_id": thread["id"], + "role": thread["name"], + "pkg": package, + "config_path": _rel(bbdir, thread["_file"]), + "balldomain_base_dir": _derive._balldomain_base_dir(bbdir, thread, package), + } + item = design.threads.add() + _fill_thread(item, thread, meta, bbdir, hardware=True) + core.thread_indices.append(index) + # The signature names the kind of core software runs on; where an instance + # sits (hart, config file, NoC nodes) is not part of it. + hardware = pb.CorePlacement() + hardware.CopyFrom(core) + for field in ("hart_id", "path", "thread_indices"): + hardware.ClearField(field) + + def clear_nodes(message): + for field, value in message.ListFields(): + if field.name == "node_id": + message.node_id = 0 + elif field.message_type is not None: + for item in [value] if hasattr(value, "ListFields") else value: + clear_nodes(item) + + clear_nodes(hardware) + payload = hardware.SerializeToString(deterministic=True) + b"".join( + design.threads[index].signature.to_bytes(8, "little") + for index in core.thread_indices + ) + core.signature = int.from_bytes(hashlib.sha256(payload).digest()[:8], "little") + + +def write_core_pb(raw: dict[str, Any], out_path: Path, bbdir: Path) -> Path: + design = pb.Design(name=f"hart-{raw['hartId']}", n_cores=1) + _append_core(design, raw, bbdir) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_bytes(design.SerializeToString(deterministic=True)) + return out_path + + +def write_design_pb( + config_path: Path, + derived_path: Path, + out_path: Path, + bbdir: Path, +) -> Path: + config = json.loads(config_path.read_text(encoding="utf-8")) + derived = json.loads(derived_path.read_text(encoding="utf-8")) + design = fill_design(config, derived, bbdir) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_bytes(design.SerializeToString()) + return out_path + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser( + description="Build a configured core protobuf using the standard TOML expansion" + ) + parser.add_argument("--core", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--repo", type=Path, required=True) + options = parser.parse_args() + source = Path(__file__).with_name("1_toml2json.py") + spec = importlib.util.spec_from_file_location("toml_expansion", source) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + output = write_core_pb(module.toml2json(options.core), options.output, options.repo) + print(output) diff --git a/bbdev/api/steps/config/scripts/ball_normalize.py b/bbdev/api/steps/config/scripts/ball_normalize.py new file mode 100644 index 000000000..872cdd4c0 --- /dev/null +++ b/bbdev/api/steps/config/scripts/ball_normalize.py @@ -0,0 +1,45 @@ +"""Validate Ball instances and their directly owned instructions.""" + +from copy import deepcopy +from typing import Any + + +def normalize_ball_domain(thread: dict[str, Any]) -> dict[str, Any] | None: + raw = thread.get("balldomain") + if raw is None: + return None + domain = deepcopy(raw) + balls = domain["balls"] + ids, names, codes = set(), set(), set() + for ball in balls: + identity, name = ball["id"], ball["name"] + if type(identity) is not int or not 0 <= identity < 32 or identity in ids: + raise ValueError(f"Invalid or duplicate Ball ID: {identity}") + if name in names: + raise ValueError(f"Duplicate Ball instance name: {name}") + ids.add(identity) + names.add(name) + if not ball["component"].isidentifier(): + raise ValueError(f"Invalid Ball component: {ball['component']}") + port = ball["port"] + if any( + type(port[key]) is not int or port[key] <= 0 for key in ("inBW", "outBW") + ): + raise ValueError( + "Ball input and output bandwidths must be positive integers" + ) + if any( + type(port[key]) is not int or port[key] < 0 + for key in ("mmioReadBW", "mmioWriteBW") + ): + raise ValueError("Ball MMIO bandwidths must be non-negative integers") + mnemonics = set() + for instruction in ball["instructions"]: + code, mnemonic = instruction["funct7"], instruction["mnemonic"] + if type(code) is not int or not 0 <= code < 128 or code in codes: + raise ValueError(f"Invalid or duplicate instruction encoding: {code}") + if mnemonic in mnemonics or code == 15 or mnemonic == "RUN_KERNEL": + raise ValueError(f"Invalid or duplicate Ball instruction: {mnemonic}") + codes.add(code) + mnemonics.add(mnemonic) + return domain diff --git a/bbdev/api/steps/config/scripts/ball_parameters.py b/bbdev/api/steps/config/scripts/ball_parameters.py new file mode 100644 index 000000000..a2d7d3f91 --- /dev/null +++ b/bbdev/api/steps/config/scripts/ball_parameters.py @@ -0,0 +1,31 @@ +"""Typed parameter messages owned by individual Ball components.""" + +from importlib import import_module +from google.protobuf.json_format import ParseDict + + +def pack_parameters(instance, values): + message = import_module(f"{instance.component}_params_pb2").Params() + ParseDict(values, message) + missing = [ + field.name + for field in message.DESCRIPTOR.fields + if not message.HasField(field.name) + ] + if missing: + raise ValueError(f"{instance.name}: missing parameters {missing}") + instance.parameters.Pack(message, deterministic=True) + + +def unpack_parameters(instance): + message = import_module(f"{instance.component}_params_pb2").Params() + if not instance.parameters.Unpack(message): + raise ValueError(f"{instance.name}: wrong parameter message type") + missing = [ + field.name + for field in message.DESCRIPTOR.fields + if not message.HasField(field.name) + ] + if missing: + raise ValueError(f"{instance.name}: missing parameters {missing}") + return message diff --git a/bbdev/api/steps/config/scripts/boom_params_pb2.py b/bbdev/api/steps/config/scripts/boom_params_pb2.py new file mode 100644 index 000000000..02f769d2a --- /dev/null +++ b/bbdev/api/steps/config/scripts/boom_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: boom_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "boom_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x11\x62oom_params.proto\x12\x19\x62uckyball.components.boom"\x19\n\x06Params\x12\x0f\n\x07profile\x18\x01 \x02(\tB\x1d\n\x19\x62uckyball.components.boomP\x01' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "boom_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\031buckyball.components.boomP\001" + _globals["_PARAMS"]._serialized_start = 48 + _globals["_PARAMS"]._serialized_end = 73 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/f32add_params_pb2.py b/bbdev/api/steps/config/scripts/f32add_params_pb2.py new file mode 100644 index 000000000..8d13feb6b --- /dev/null +++ b/bbdev/api/steps/config/scripts/f32add_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: f32add_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "f32add_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x13\x66\x33\x32\x61\x64\x64_params.proto\x12\x16\x62uckyball.balls.f32add"\x08\n\x06ParamsB\x1a\n\x16\x62uckyball.balls.f32addP\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "f32add_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\026buckyball.balls.f32addP\001" + _globals["_PARAMS"]._serialized_start = 47 + _globals["_PARAMS"]._serialized_end = 55 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/gemmini_params_pb2.py b/bbdev/api/steps/config/scripts/gemmini_params_pb2.py new file mode 100644 index 000000000..81c248f11 --- /dev/null +++ b/bbdev/api/steps/config/scripts/gemmini_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: gemmini_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "gemmini_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x14gemmini_params.proto\x12\x17\x62uckyball.balls.gemmini"\xfc\x02\n\x06Params\x12\x15\n\x08meshRows\x18\x01 \x01(\x05H\x00\x88\x01\x01\x12\x18\n\x0bmeshColumns\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x15\n\x08tileRows\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x18\n\x0btileColumns\x18\x04 \x01(\x05H\x03\x88\x01\x01\x12\x17\n\ninputWidth\x18\x05 \x01(\x05H\x04\x88\x01\x01\x12\x15\n\x08\x61\x63\x63Width\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x1f\n\x12spatialOutputWidth\x18\x07 \x01(\x05H\x06\x88\x01\x01\x12\x18\n\x0btileLatency\x18\x08 \x01(\x05H\x07\x88\x01\x01\x12\x18\n\x0boutputDelay\x18\t \x01(\x05H\x08\x88\x01\x01\x42\x0b\n\t_meshRowsB\x0e\n\x0c_meshColumnsB\x0b\n\t_tileRowsB\x0e\n\x0c_tileColumnsB\r\n\x0b_inputWidthB\x0b\n\t_accWidthB\x15\n\x13_spatialOutputWidthB\x0e\n\x0c_tileLatencyB\x0e\n\x0c_outputDelayB\x1b\n\x17\x62uckyball.balls.gemminiP\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "gemmini_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\027buckyball.balls.gemminiP\001" + _globals["_PARAMS"]._serialized_start = 50 + _globals["_PARAMS"]._serialized_end = 430 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/hardware_pb2.py b/bbdev/api/steps/config/scripts/hardware_pb2.py new file mode 100644 index 000000000..169879f0d --- /dev/null +++ b/bbdev/api/steps/config/scripts/hardware_pb2.py @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: hardware.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "hardware.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x0ehardware.proto\x12\x10\x62uckyball.config"L\n\x0b\x43hiHardware\x12\x14\n\x0cnode_id_bits\x18\x01 \x02(\r\x12\x14\n\x0c\x61\x64\x64ress_bits\x18\x02 \x02(\r\x12\x11\n\tdata_bits\x18\x03 \x02(\r"G\n\x0b\x41xiHardware\x12\x14\n\x0c\x61\x64\x64ress_bits\x18\x01 \x02(\r\x12\x11\n\tdata_bits\x18\x02 \x02(\r\x12\x0f\n\x07id_bits\x18\x03 \x02(\r"r\n\x10TrackingHardware\x12\x0f\n\x07\x65ntries\x18\x01 \x02(\r\x12\x14\n\x0c\x61\x64\x64ress_bits\x18\x02 \x02(\r\x12\x0f\n\x07id_bits\x18\x03 \x02(\r\x12\x12\n\nline_bytes\x18\x04 \x02(\r\x12\x12\n\nmax_ranges\x18\x05 \x02(\r"\x98\x01\n\x0eRegionHardware\x12\x0c\n\x04\x62\x61se\x18\x01 \x02(\x04\x12\r\n\x05\x62ytes\x18\x02 \x02(\x04\x12\x11\n\tcacheable\x18\x03 \x02(\x08\x12\x12\n\nexecutable\x18\x04 \x02(\x08\x12\x10\n\x08readable\x18\x05 \x02(\x08\x12\x10\n\x08writable\x18\x06 \x02(\x08\x12\x0e\n\x06\x61tomic\x18\x07 \x02(\x08\x12\x0e\n\x06normal\x18\x08 \x02(\x08"\x95\x01\n\rCacheHardware\x12\x14\n\x0c\x61\x64\x64ress_bits\x18\x01 \x02(\r\x12\x12\n\nline_bytes\x18\x02 \x02(\r\x12\x0c\n\x04sets\x18\x03 \x02(\r\x12\x0c\n\x04ways\x18\x04 \x02(\r\x12\x0f\n\x07id_bits\x18\x05 \x02(\r\x12\x15\n\rmetadata_bits\x18\x06 \x02(\r\x12\x16\n\x0eresponse_depth\x18\x07 \x02(\r"\x8b\x01\n\x0bRnfHardware\x12\x0f\n\x07node_id\x18\x01 \x02(\r\x12\r\n\x05lines\x18\x02 \x02(\r\x12\x0f\n\x07home_id\x18\x03 \x02(\r\x12\x12\n\nhome_count\x18\x04 \x02(\r\x12\r\n\x05\x62\x61nks\x18\x05 \x02(\r\x12\x13\n\x0bline_result\x18\x06 \x02(\x08\x12\r\n\x05probe\x18\x07 \x02(\x08J\x04\x08\x08\x10\t"\xa8\x01\n\x0cHomeHardware\x12.\n\x05\x63\x61\x63he\x18\x01 \x02(\x0b\x32\x1f.buckyball.config.CacheHardware\x12\x0e\n\x06\x61gents\x18\x02 \x02(\r\x12\x14\n\x0cmshr_entries\x18\x03 \x02(\r\x12\x0f\n\x07node_id\x18\x04 \x02(\r\x12\x31\n\nrequesters\x18\x05 \x02(\x0b\x32\x1d.buckyball.config.ChiHardware"\x8c\x01\n\x13PreparationHardware\x12*\n\x03\x62us\x18\x01 \x02(\x0b\x32\x1d.buckyball.config.ChiHardware\x12\x10\n\x08\x63ontexts\x18\x02 \x02(\r\x12\x12\n\nmax_ranges\x18\x03 \x02(\r\x12\x0f\n\x07id_bits\x18\x04 \x02(\r\x12\x12\n\nbeat_bytes\x18\x05 \x02(\r"\xe1\x01\n\x11TransportHardware\x12\x34\n\x08tracking\x18\x01 \x02(\x0b\x32".buckyball.config.TrackingHardware\x12\x37\n\x08prepared\x18\x02 \x02(\x0b\x32%.buckyball.config.PreparationHardware\x12*\n\x03\x61xi\x18\x03 \x02(\x0b\x32\x1d.buckyball.config.AxiHardware\x12\x31\n\x07regions\x18\x04 \x03(\x0b\x32 .buckyball.config.RegionHardware"\xc6\x01\n\x0b\x44\x64rHardware\x12+\n\x04line\x18\x01 \x02(\x0b\x32\x1d.buckyball.config.ChiHardware\x12\x0f\n\x07\x63lients\x18\x02 \x02(\r\x12\x18\n\x10slots_per_client\x18\x03 \x02(\r\x12\x11\n\tdata_bits\x18\x04 \x02(\r\x12\x0f\n\x07id_bits\x18\x05 \x02(\r\x12\x0f\n\x07id_base\x18\x06 \x02(\r\x12\x0c\n\x04\x62\x61se\x18\x07 \x02(\x04\x12\r\n\x05\x62ytes\x18\x08 \x02(\x04\x12\r\n\x05model\x18\t \x02(\x04"=\n\x0f\x42ootRomHardware\x12\x0c\n\x04\x62\x61se\x18\x01 \x02(\x04\x12\r\n\x05\x62ytes\x18\x02 \x02(\x04\x12\r\n\x05image\x18\x03 \x03(\x11"S\n\rClintHardware\x12\x10\n\x08hart_ids\x18\x01 \x03(\r\x12\x0c\n\x04\x62\x61se\x18\x02 \x02(\x04\x12\r\n\x05\x62ytes\x18\x03 \x02(\x04\x12\x13\n\x0btick_cycles\x18\x04 \x02(\r"e\n\x0cPlicHardware\x12\x10\n\x08hart_ids\x18\x01 \x03(\r\x12\x0c\n\x04\x62\x61se\x18\x02 \x02(\x04\x12\r\n\x05\x62ytes\x18\x03 \x02(\x04\x12\x0f\n\x07sources\x18\x04 \x02(\r\x12\x15\n\rpriority_bits\x18\x05 \x02(\r"f\n\x0bScuHardware\x12\x14\n\x0c\x62\x61se_address\x18\x01 \x02(\x04\x12\x14\n\x0cstride_bytes\x18\x02 \x02(\x04\x12\x18\n\x10total_size_bytes\x18\x03 \x02(\x04\x12\x11\n\tmax_harts\x18\x04 \x02(\r"E\n\x11SimMemoryHardware\x12\x12\n\nline_bytes\x18\x03 \x02(\r\x12\x10\n\x08\x63lock_hz\x18\x04 \x02(\x04J\x04\x08\x01\x10\x02J\x04\x08\x02\x10\x03"[\n\x0bSimHardware\x12\x11\n\tdiff_test\x18\x01 \x02(\x08\x12\x33\n\x06memory\x18\x02 \x02(\x0b\x32#.buckyball.config.SimMemoryHardwareJ\x04\x08\x03\x10\x04"\x9c\x01\n\x12MemoryAccessConfig\x12\x10\n\x08tag_bits\x18\x02 \x02(\r\x12\x1b\n\x13translation_entries\x18\x03 \x02(\r\x12\x18\n\x10result_line_bits\x18\x04 \x02(\r\x12\x31\n\x07regions\x18\x06 \x03(\x0b\x32 .buckyball.config.RegionHardwareJ\x04\x08\x01\x10\x02J\x04\x08\x05\x10\x06"r\n\x0c\x44\x43\x61\x63heConfig\x12,\n\x05\x63\x61\x63he\x18\x01 \x02(\x0b\x32\x1d.buckyball.config.RnfHardware\x12\x34\n\x08tracking\x18\x02 \x02(\x0b\x32".buckyball.config.TrackingHardware"K\n\x0eUncachedConfig\x12\r\n\x05ports\x18\x01 \x02(\r\x12*\n\x03\x61xi\x18\x02 \x02(\x0b\x32\x1d.buckyball.config.AxiHardware"\x8b\x02\n\x0cMemoryConfig\x12\x32\n\x08uncached\x18\x06 \x02(\x0b\x32 .buckyball.config.UncachedConfig\x12\x34\n\x06\x61\x63\x63\x65ss\x18\x02 \x02(\x0b\x32$.buckyball.config.MemoryAccessConfig\x12-\n\x06icache\x18\x03 \x02(\x0b\x32\x1d.buckyball.config.RnfHardware\x12.\n\x06\x64\x63\x61\x63he\x18\x04 \x02(\x0b\x32\x1e.buckyball.config.DCacheConfig\x12,\n\x04home\x18\x05 \x02(\x0b\x32\x1e.buckyball.config.HomeHardwareJ\x04\x08\x01\x10\x02"(\n\x0b\x46\x65tchConfig\x12\x19\n\x11instruction_bytes\x18\x01 \x02(\r"\x1a\n\tFpuConfig\x12\r\n\x05\x66_len\x18\x01 \x02(\r"O\n\x12InterconnectConfig\x12*\n\x03\x61xi\x18\x01 \x02(\x0b\x32\x1d.buckyball.config.AxiHardware\x12\r\n\x05slots\x18\x02 \x02(\r"\\\n\tC2CConfig\x12\x0f\n\x07\x65nabled\x18\x01 \x02(\x08\x12\x12\n\npeer_cores\x18\x02 \x02(\r\x12*\n\x03\x61xi\x18\x03 \x02(\x0b\x32\x1d.buckyball.config.AxiHardware" \n\nTBusConfig\x12\x12\n\nbank_nodes\x18\x01 \x03(\r"!\n\x10\x43ontrollerConfig\x12\r\n\x05moves\x18\x01 \x02(\x08\x42\x14\n\x10\x62uckyball.configP\x01' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "hardware_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\020buckyball.configP\001" + _globals["_CHIHARDWARE"]._serialized_start = 36 + _globals["_CHIHARDWARE"]._serialized_end = 112 + _globals["_AXIHARDWARE"]._serialized_start = 114 + _globals["_AXIHARDWARE"]._serialized_end = 185 + _globals["_TRACKINGHARDWARE"]._serialized_start = 187 + _globals["_TRACKINGHARDWARE"]._serialized_end = 301 + _globals["_REGIONHARDWARE"]._serialized_start = 304 + _globals["_REGIONHARDWARE"]._serialized_end = 456 + _globals["_CACHEHARDWARE"]._serialized_start = 459 + _globals["_CACHEHARDWARE"]._serialized_end = 608 + _globals["_RNFHARDWARE"]._serialized_start = 611 + _globals["_RNFHARDWARE"]._serialized_end = 750 + _globals["_HOMEHARDWARE"]._serialized_start = 753 + _globals["_HOMEHARDWARE"]._serialized_end = 921 + _globals["_PREPARATIONHARDWARE"]._serialized_start = 924 + _globals["_PREPARATIONHARDWARE"]._serialized_end = 1064 + _globals["_TRANSPORTHARDWARE"]._serialized_start = 1067 + _globals["_TRANSPORTHARDWARE"]._serialized_end = 1292 + _globals["_DDRHARDWARE"]._serialized_start = 1295 + _globals["_DDRHARDWARE"]._serialized_end = 1493 + _globals["_BOOTROMHARDWARE"]._serialized_start = 1495 + _globals["_BOOTROMHARDWARE"]._serialized_end = 1556 + _globals["_CLINTHARDWARE"]._serialized_start = 1558 + _globals["_CLINTHARDWARE"]._serialized_end = 1641 + _globals["_PLICHARDWARE"]._serialized_start = 1643 + _globals["_PLICHARDWARE"]._serialized_end = 1744 + _globals["_SCUHARDWARE"]._serialized_start = 1746 + _globals["_SCUHARDWARE"]._serialized_end = 1848 + _globals["_SIMMEMORYHARDWARE"]._serialized_start = 1850 + _globals["_SIMMEMORYHARDWARE"]._serialized_end = 1919 + _globals["_SIMHARDWARE"]._serialized_start = 1921 + _globals["_SIMHARDWARE"]._serialized_end = 2012 + _globals["_MEMORYACCESSCONFIG"]._serialized_start = 2015 + _globals["_MEMORYACCESSCONFIG"]._serialized_end = 2171 + _globals["_DCACHECONFIG"]._serialized_start = 2173 + _globals["_DCACHECONFIG"]._serialized_end = 2287 + _globals["_UNCACHEDCONFIG"]._serialized_start = 2289 + _globals["_UNCACHEDCONFIG"]._serialized_end = 2364 + _globals["_MEMORYCONFIG"]._serialized_start = 2367 + _globals["_MEMORYCONFIG"]._serialized_end = 2634 + _globals["_FETCHCONFIG"]._serialized_start = 2636 + _globals["_FETCHCONFIG"]._serialized_end = 2676 + _globals["_FPUCONFIG"]._serialized_start = 2678 + _globals["_FPUCONFIG"]._serialized_end = 2704 + _globals["_INTERCONNECTCONFIG"]._serialized_start = 2706 + _globals["_INTERCONNECTCONFIG"]._serialized_end = 2785 + _globals["_C2CCONFIG"]._serialized_start = 2787 + _globals["_C2CCONFIG"]._serialized_end = 2879 + _globals["_TBUSCONFIG"]._serialized_start = 2881 + _globals["_TBUSCONFIG"]._serialized_end = 2913 + _globals["_CONTROLLERCONFIG"]._serialized_start = 2915 + _globals["_CONTROLLERCONFIG"]._serialized_end = 2948 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/im2col_params_pb2.py b/bbdev/api/steps/config/scripts/im2col_params_pb2.py new file mode 100644 index 000000000..ac744c174 --- /dev/null +++ b/bbdev/api/steps/config/scripts/im2col_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: im2col_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "im2col_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x13im2col_params.proto\x12\x16\x62uckyball.balls.im2col"\x9e\x01\n\x06Params\x12\x17\n\ninputWidth\x18\x01 \x01(\x05H\x00\x88\x01\x01\x12\x14\n\x07maxIter\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x15\n\x08maxKSize\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x17\n\nmaxPadding\x18\x04 \x01(\x05H\x03\x88\x01\x01\x42\r\n\x0b_inputWidthB\n\n\x08_maxIterB\x0b\n\t_maxKSizeB\r\n\x0b_maxPaddingB\x1a\n\x16\x62uckyball.balls.im2colP\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "im2col_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\026buckyball.balls.im2colP\001" + _globals["_PARAMS"]._serialized_start = 48 + _globals["_PARAMS"]._serialized_end = 206 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/int2fp_params_pb2.py b/bbdev/api/steps/config/scripts/int2fp_params_pb2.py new file mode 100644 index 000000000..61c5f0b33 --- /dev/null +++ b/bbdev/api/steps/config/scripts/int2fp_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: int2fp_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "int2fp_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x13int2fp_params.proto\x12\x16\x62uckyball.balls.int2fp"V\n\x06Params\x12\x16\n\tinputType\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\noutputType\x18\x02 \x01(\tH\x01\x88\x01\x01\x42\x0c\n\n_inputTypeB\r\n\x0b_outputTypeB\x1a\n\x16\x62uckyball.balls.int2fpP\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "int2fp_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\026buckyball.balls.int2fpP\001" + _globals["_PARAMS"]._serialized_start = 47 + _globals["_PARAMS"]._serialized_end = 133 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/int8add_params_pb2.py b/bbdev/api/steps/config/scripts/int8add_params_pb2.py new file mode 100644 index 000000000..1ff7736a2 --- /dev/null +++ b/bbdev/api/steps/config/scripts/int8add_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: int8add_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "int8add_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x14int8add_params.proto\x12\x17\x62uckyball.balls.int8add"\x08\n\x06ParamsB\x1b\n\x17\x62uckyball.balls.int8addP\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "int8add_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\027buckyball.balls.int8addP\001" + _globals["_PARAMS"]._serialized_start = 49 + _globals["_PARAMS"]._serialized_end = 57 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/int8mul_params_pb2.py b/bbdev/api/steps/config/scripts/int8mul_params_pb2.py new file mode 100644 index 000000000..7271656e3 --- /dev/null +++ b/bbdev/api/steps/config/scripts/int8mul_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: int8mul_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "int8mul_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x14int8mul_params.proto\x12\x17\x62uckyball.balls.int8mul"\x08\n\x06ParamsB\x1b\n\x17\x62uckyball.balls.int8mulP\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "int8mul_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\027buckyball.balls.int8mulP\001" + _globals["_PARAMS"]._serialized_start = 49 + _globals["_PARAMS"]._serialized_end = 57 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/lut_params_pb2.py b/bbdev/api/steps/config/scripts/lut_params_pb2.py new file mode 100644 index 000000000..9d42d2466 --- /dev/null +++ b/bbdev/api/steps/config/scripts/lut_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: lut_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "lut_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x10lut_params.proto\x12\x13\x62uckyball.balls.lut"\x08\n\x06ParamsB\x17\n\x13\x62uckyball.balls.lutP\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "lut_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\023buckyball.balls.lutP\001" + _globals["_PARAMS"]._serialized_start = 41 + _globals["_PARAMS"]._serialized_end = 49 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/matadd_params_pb2.py b/bbdev/api/steps/config/scripts/matadd_params_pb2.py new file mode 100644 index 000000000..fe65713c8 --- /dev/null +++ b/bbdev/api/steps/config/scripts/matadd_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: matadd_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "matadd_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x13matadd_params.proto\x12\x16\x62uckyball.balls.matadd"\x08\n\x06ParamsB\x1a\n\x16\x62uckyball.balls.mataddP\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "matadd_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\026buckyball.balls.mataddP\001" + _globals["_PARAMS"]._serialized_start = 47 + _globals["_PARAMS"]._serialized_end = 55 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/maxpool_params_pb2.py b/bbdev/api/steps/config/scripts/maxpool_params_pb2.py new file mode 100644 index 000000000..68512d9ff --- /dev/null +++ b/bbdev/api/steps/config/scripts/maxpool_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: maxpool_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "maxpool_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x14maxpool_params.proto\x12\x17\x62uckyball.balls.maxpool"\x08\n\x06ParamsB\x1b\n\x17\x62uckyball.balls.maxpoolP\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "maxpool_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\027buckyball.balls.maxpoolP\001" + _globals["_PARAMS"]._serialized_start = 49 + _globals["_PARAMS"]._serialized_end = 57 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/model_pb2.py b/bbdev/api/steps/config/scripts/model_pb2.py new file mode 100644 index 000000000..105156b3a --- /dev/null +++ b/bbdev/api/steps/config/scripts/model_pb2.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: model.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "model.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x0bmodel.proto\x12\x10\x62uckyball.config"_\n\x0eModelParameter\x12\x11\n\x07integer\x18\x01 \x01(\x12H\x00\x12\x0e\n\x04real\x18\x02 \x01(\x01H\x00\x12\x0e\n\x04text\x18\x03 \x01(\tH\x00\x12\x11\n\x07\x62oolean\x18\x04 \x01(\x08H\x00\x42\x07\n\x05value"\xc4\x01\n\x0bModelConfig\x12\x0f\n\x07version\x18\x01 \x01(\r\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x41\n\nparameters\x18\x03 \x03(\x0b\x32-.buckyball.config.ModelConfig.ParametersEntry\x1aS\n\x0fParametersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12/\n\x05value\x18\x02 \x01(\x0b\x32 .buckyball.config.ModelParameter:\x02\x38\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "model_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + DESCRIPTOR._loaded_options = None + _globals["_MODELCONFIG_PARAMETERSENTRY"]._loaded_options = None + _globals["_MODELCONFIG_PARAMETERSENTRY"]._serialized_options = b"8\001" + _globals["_MODELPARAMETER"]._serialized_start = 33 + _globals["_MODELPARAMETER"]._serialized_end = 128 + _globals["_MODELCONFIG"]._serialized_start = 131 + _globals["_MODELCONFIG"]._serialized_end = 327 + _globals["_MODELCONFIG_PARAMETERSENTRY"]._serialized_start = 244 + _globals["_MODELCONFIG_PARAMETERSENTRY"]._serialized_end = 327 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/mxfp2int_params_pb2.py b/bbdev/api/steps/config/scripts/mxfp2int_params_pb2.py new file mode 100644 index 000000000..447b0de63 --- /dev/null +++ b/bbdev/api/steps/config/scripts/mxfp2int_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: mxfp2int_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "mxfp2int_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x15mxfp2int_params.proto\x12\x18\x62uckyball.balls.mxfp2int"0\n\x06Params\x12\x17\n\nmxfpFormat\x18\x01 \x01(\tH\x00\x88\x01\x01\x42\r\n\x0b_mxfpFormatB\x1c\n\x18\x62uckyball.balls.mxfp2intP\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "mxfp2int_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\030buckyball.balls.mxfp2intP\001" + _globals["_PARAMS"]._serialized_start = 51 + _globals["_PARAMS"]._serialized_end = 99 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/mxmm_params_pb2.py b/bbdev/api/steps/config/scripts/mxmm_params_pb2.py new file mode 100644 index 000000000..1883aa8c3 --- /dev/null +++ b/bbdev/api/steps/config/scripts/mxmm_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: mxmm_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "mxmm_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x11mxmm_params.proto\x12\x14\x62uckyball.balls.mxmm"P\n\x06Params\x12\x15\n\x08tileRows\x18\x01 \x01(\x05H\x00\x88\x01\x01\x12\x15\n\x08tileCols\x18\x02 \x01(\x05H\x01\x88\x01\x01\x42\x0b\n\t_tileRowsB\x0b\n\t_tileColsB\x18\n\x14\x62uckyball.balls.mxmmP\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "mxmm_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\024buckyball.balls.mxmmP\001" + _globals["_PARAMS"]._serialized_start = 43 + _globals["_PARAMS"]._serialized_end = 123 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/mxquant_params_pb2.py b/bbdev/api/steps/config/scripts/mxquant_params_pb2.py new file mode 100644 index 000000000..38024b677 --- /dev/null +++ b/bbdev/api/steps/config/scripts/mxquant_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: mxquant_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "mxquant_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x14mxquant_params.proto\x12\x17\x62uckyball.balls.mxquant"\x08\n\x06ParamsB\x1b\n\x17\x62uckyball.balls.mxquantP\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "mxquant_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\027buckyball.balls.mxquantP\001" + _globals["_PARAMS"]._serialized_start = 49 + _globals["_PARAMS"]._serialized_end = 57 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/proto/design.proto b/bbdev/api/steps/config/scripts/proto/design.proto new file mode 100644 index 000000000..f9bb17c83 --- /dev/null +++ b/bbdev/api/steps/config/scripts/proto/design.proto @@ -0,0 +1,345 @@ +// design.proto is the Design schema. +// Java: mill protoJava; Python: design_pb2.py (generated by config install step3). +syntax = "proto3"; + +package buckyball.config; + +import "google/protobuf/any.proto"; +import "google/protobuf/descriptor.proto"; +import "hardware.proto"; + +option java_package = "buckyball.config"; +option java_multiple_files = true; + +// Marks the field that names an element of a repeated message; design.cmake +// keys the element by it (stack/scripts/design.py). Unmarked: its position. +extend google.protobuf.FieldOptions { + bool key = 50000; +} + +message MillConfig { + reserved 1, 2; + map targets = 3; + map tops = 4; + map resources = 5; +} + +message BankConfig { + uint32 num = 1; + uint32 width = 2; + uint32 entries = 3; + uint32 mask_len = 4; + uint32 request_ports = 5; + optional bool layout_enable = 6; +} + +message MmioConfig { + bool enable = 1; + uint32 bank_num = 2; + uint32 bank_entries = 3; + uint32 bank_width = 4; + uint32 read_width = 5; +} + +message DmaConfig { + uint32 bus_width = 3; +} + +message MemDomainConfig { + BankConfig bank = 1; + DmaConfig dma = 2; + MmioConfig mmio = 5; + reserved 6; + string source_path = 7; + uint32 addr_len = 8; +} + +message BallInstance { + uint32 id = 1; + string name = 2; + string component = 3 [(key) = true]; + uint32 in_bw = 4; + uint32 out_bw = 5; + uint32 mmio_read_bw = 6; + uint32 mmio_write_bw = 7; + repeated BallInstruction instructions = 8; + google.protobuf.Any parameters = 9; + string rtl_class = 10; + reserved 11; +} + +message BallInstruction { + string mnemonic = 1 [(key) = true]; + uint32 funct7 = 2; +} + +message BallDomain { + repeated BallInstance balls = 1; +} + +message FrontendConfig { + uint32 rob_entries = 1; + bool rs_out_of_order_response = 2; + uint32 bank_id_len = 3; + // Virtual bank ids [0, max_private_vbank] are private. + uint32 max_private_vbank = 4; + // Virtual bank ids from shared_vbank_base up are shared. + uint32 shared_vbank_base = 5; + uint32 iter_len = 6; + bool sub_rob_enable = 7; + uint32 sub_rob_depth = 8; + string source_path = 9; + optional bool bank_alias_enable = 10; +} + +message GpDomainConfig { + uint32 lane_number = 1; + uint32 v_len = 2; + uint32 e_len = 3; + uint32 i_buf_words = 4; + uint32 memory_ports = 5; + string source_path = 6; + optional bool enable = 7; +} + +message CoreParamConfig { + uint32 thread_data_bytes = 1; + uint32 x_len = 2; + uint32 vaddr_bits = 3; + uint32 paddr_bits = 4; + uint32 pg_idx_bits = 5; + uint32 pg_levels = 6; + uint32 n_pmps = 7; +} + +message SpmConfig { + uint64 base = 1; + uint32 bytes = 2; + uint32 data_bits = 3; +} + +message AntConfig { + uint32 code_bytes = 1; + SpmConfig tls = 2; + uint32 task_bits = 3; +} + +message CpuConfig { + string kind = 1; + oneof cpu { + AntConfig ant = 5; + } + string source_path = 4; +} + +message ThreadInstance { + uint32 index = 1; + string role = 2; + string pkg = 3; + string config_path = 4; + BallDomain balldomain = 5; + MemDomainConfig mem = 6; + string balldomain_base_dir = 7; + CpuConfig cpu = 8; + FrontendConfig frontend = 9; + reserved 10; + GpDomainConfig gpdomain = 11; + reserved 12; + TransportHardware transport = 13; + optional uint32 local_id = 14; + optional uint64 signature = 15; + string profile = 16; + string source_path = 17; + string rtl_class = 18; +} + +message BemuBall { + string emu_class = 1; + string component = 2 [(key) = true]; + string emu_lib = 3; +} + +message BemuConfig { + reserved 1, 2; + repeated BemuBall balls = 3; +} + +message SharedMemConfig { + bool enable = 1; + uint32 entries = 2; + uint32 input_channels = 3; + uint32 default_group_count = 4; + uint32 virtual_bank_count = 5; + uint32 latency = 6; + uint32 bank_entries = 7; + uint32 bank_width = 8; +} + +message LocalAccelerator { + BallDomain balldomain = 1; + MemDomainConfig mem = 2; + FrontendConfig frontend = 3; + GpDomainConfig gpdomain = 4; + TransportHardware transport = 5; +} + +message CorePlacement { + string path = 1; + uint32 virtual_bank_count = 2; + repeated uint32 thread_indices = 3; + reserved 5; + SharedMemConfig shared_mem = 6; + CoreParamConfig param = 7; + reserved 8; + reserved 9; + SpmConfig tss = 10; + reserved 11; + reserved 12; + google.protobuf.Any implementation = 13; + LocalAccelerator accelerator = 14; + string profile = 16; + reserved 17; + uint32 hart_id = 18 [(key) = true]; + uint64 reset_vector = 19; + uint64 mtime_address = 20; + MemoryConfig memory = 21; + FetchConfig fetch = 22; + FpuConfig fpu = 23; + InterconnectConfig interconnect = 24; + C2CConfig c2c = 25; + TBusConfig tbus = 26; + ControllerConfig controller = 27; + RnfHardware dma_cache = 28; + string component = 29; + // Identity of the core hardware and its threads; runtime jobs select cores by it. + uint64 signature = 30; + +} + +message WorkloadConfig { + reserved 1, 2; + optional uint32 control_hart = 3; + optional uint32 report_hart = 4; +} + +// A compiler target. Its programs see `thread` and `core` by their paths +// from the profile: THREAD_MEM_BANK_WIDTH, CORE_HART_ID. +message CompilerProfile { + string name = 1 [(key) = true]; + reserved 2, 3, 4, 5, 6, 9; + repeated string ball_ctest_dirs = 7; + ThreadInstance thread = 8; + // The core its programs run on: the first core running the profile. + CorePlacement core = 10; +} + +message Design { + string name = 1; + MillConfig mill = 2; + repeated string includes = 4; + reserved 5; + uint32 n_cores = 6; + reserved 7; + repeated ThreadInstance threads = 8; + reserved 9; + BemuConfig bemu = 10; + WorkloadConfig workload = 11; + string path = 12; + repeated CompilerProfile profiles = 13; + reserved 14; + DeviceConfig device = 15; + SimHardware sims = 16; + VerificationConfig verification = 17; + reserved 18; + // The simulation this design.pb was generated for. + SimulationTarget simulation = 19; +} + +message SharedBankFabricConfig { + optional uint32 owner_hart = 1; + repeated uint32 participant_harts = 2; + repeated uint32 bank_nodes = 3; +} + +message DeviceConfig { + repeated CorePlacement cores = 1; + DdrHardware ddr = 2; + BootRomHardware bootrom = 3; + ClintHardware clint = 4; + PlicHardware plic = 5; + ScuHardware scu = 6; + SharedBankFabricConfig shared_bank_fabric = 7; +} + +message MillTarget { + string directory = 1; + string module = 2; + string config = 3; +} +message VerificationConfig { + repeated string balls = 1; + repeated string ips = 2; + reserved 3; + repeated VerificationProfile profiles = 4; +} + +message VerificationBuild { + string kind = 1; + string directory = 2; + string module = 3; + repeated string arguments = 4; +} + +message VerificationReference { + string manifest = 1; + string library = 2; +} + +message VerificationTest { + string name = 1; + string top = 2; + string test = 3; + string filelist = 4; + uint64 timeout_ns = 5; + uint32 wall_timeout_s = 6; + string kind = 7; + repeated string rtl_scopes = 8; + repeated string metrics = 9; + uint32 sink_ready = 10; + uint32 sink_stall = 11; + repeated string designs = 12; + repeated string plusargs = 13; + string assertion = 14; + repeated string functional_groups = 15; +} + +message VerificationProfile { + string ip = 1; + string profile = 2; + string config_path = 3; + string root = 4; + string parameters_json = 5; + map defines = 6; + VerificationBuild build = 7; + VerificationReference model = 8; + repeated VerificationTest tests = 9; + string resources = 10; + string scope = 11; +} + +message SimulationTarget { + string directory = 1; + MillTarget arch = 2; + string entry_class = 3; + string top = 4; + string emu = 5; + string emu_package = 6; + string stack = 7; + string stack_name = 8; + string workloads = 9; + map resources = 10; + // device/ or core/: names the run and its output root. + string selection = 11; + // output/sims/, relative to the repository. + string output = 12; +} diff --git a/bbdev/api/steps/config/scripts/proto/hardware.proto b/bbdev/api/steps/config/scripts/proto/hardware.proto new file mode 100644 index 000000000..c3f12f596 --- /dev/null +++ b/bbdev/api/steps/config/scripts/proto/hardware.proto @@ -0,0 +1,158 @@ +syntax = "proto2"; +package buckyball.config; +option java_package = "buckyball.config"; +option java_multiple_files = true; + +message ChiHardware { + required uint32 node_id_bits = 1; + required uint32 address_bits = 2; + required uint32 data_bits = 3; +} +message AxiHardware { + required uint32 address_bits = 1; + required uint32 data_bits = 2; + required uint32 id_bits = 3; +} +message TrackingHardware { + required uint32 entries = 1; + required uint32 address_bits = 2; + required uint32 id_bits = 3; + required uint32 line_bytes = 4; + required uint32 max_ranges = 5; +} +message RegionHardware { + required uint64 base = 1; + required uint64 bytes = 2; + required bool cacheable = 3; + required bool executable = 4; + required bool readable = 5; + required bool writable = 6; + required bool atomic = 7; + required bool normal = 8; +} +message CacheHardware { + required uint32 address_bits = 1; + required uint32 line_bytes = 2; + required uint32 sets = 3; + required uint32 ways = 4; + required uint32 id_bits = 5; + required uint32 metadata_bits = 6; + required uint32 response_depth = 7; +} +message RnfHardware { + required uint32 node_id = 1; + required uint32 lines = 2; + required uint32 home_id = 3; + required uint32 home_count = 4; + required uint32 banks = 5; + required bool line_result = 6; + required bool probe = 7; + reserved 8; +} +message HomeHardware { + required CacheHardware cache = 1; + required uint32 agents = 2; + required uint32 mshr_entries = 3; + required uint32 node_id = 4; + required ChiHardware requesters = 5; +} +message PreparationHardware { + required ChiHardware bus = 1; + required uint32 contexts = 2; + required uint32 max_ranges = 3; + required uint32 id_bits = 4; + required uint32 beat_bytes = 5; +} +message TransportHardware { + required TrackingHardware tracking = 1; + required PreparationHardware prepared = 2; + required AxiHardware axi = 3; + repeated RegionHardware regions = 4; +} + + +message DdrHardware { + required ChiHardware line = 1; + required uint32 clients = 2; + required uint32 slots_per_client = 3; + required uint32 data_bits = 4; + required uint32 id_bits = 5; + required uint32 id_base = 6; + // DDR address map; the top `model` bytes hold the model image, the rest is + // what the CPUs run in. + required uint64 base = 7; + required uint64 bytes = 8; + required uint64 model = 9; +} +message BootRomHardware { + required uint64 base = 1; + required uint64 bytes = 2; + repeated sint32 image = 3; +} +message ClintHardware { + repeated uint32 hart_ids = 1; + required uint64 base = 2; + required uint64 bytes = 3; + required uint32 tick_cycles = 4; +} +message PlicHardware { + repeated uint32 hart_ids = 1; + required uint64 base = 2; + required uint64 bytes = 3; + required uint32 sources = 4; + required uint32 priority_bits = 5; +} +message ScuHardware { + required uint64 base_address = 1; + required uint64 stride_bytes = 2; + required uint64 total_size_bytes = 3; + required uint32 max_harts = 4; +} +message SimMemoryHardware { + reserved 1, 2; + required uint32 line_bytes = 3; + required uint64 clock_hz = 4; +} +message SimHardware { + required bool diff_test = 1; + required SimMemoryHardware memory = 2; + reserved 3; +} + +message MemoryAccessConfig { + reserved 1; + required uint32 tag_bits = 2; + required uint32 translation_entries = 3; + required uint32 result_line_bits = 4; + reserved 5; + repeated RegionHardware regions = 6; +} +message DCacheConfig { + required RnfHardware cache = 1; + required TrackingHardware tracking = 2; +} +message UncachedConfig { + required uint32 ports = 1; + required AxiHardware axi = 2; +} +message MemoryConfig { + reserved 1; + required UncachedConfig uncached = 6; + required MemoryAccessConfig access = 2; + required RnfHardware icache = 3; + required DCacheConfig dcache = 4; + required HomeHardware home = 5; +} +message FetchConfig { required uint32 instruction_bytes = 1; } +message FpuConfig { required uint32 f_len = 1; } +message InterconnectConfig { + required AxiHardware axi = 1; + required uint32 slots = 2; +} +message C2CConfig { + required bool enabled = 1; + required uint32 peer_cores = 2; + required AxiHardware axi = 3; +} +message TBusConfig { repeated uint32 bank_nodes = 1; } +message ControllerConfig { required bool moves = 1; } diff --git a/bbdev/api/steps/config/scripts/proto/model.proto b/bbdev/api/steps/config/scripts/proto/model.proto new file mode 100644 index 000000000..5e12c9682 --- /dev/null +++ b/bbdev/api/steps/config/scripts/proto/model.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; +package buckyball.config; + +message ModelParameter { + oneof value { + sint64 integer = 1; + double real = 2; + string text = 3; + bool boolean = 4; + } +} + +message ModelConfig { + uint32 version = 1; + string name = 2; + map parameters = 3; +} diff --git a/bbdev/api/steps/config/scripts/redmule_params_pb2.py b/bbdev/api/steps/config/scripts/redmule_params_pb2.py new file mode 100644 index 000000000..9aa96f62f --- /dev/null +++ b/bbdev/api/steps/config/scripts/redmule_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: redmule_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "redmule_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x14redmule_params.proto\x12\x17\x62uckyball.balls.redmule"&\n\x06Params\x12\x12\n\x05lanes\x18\x01 \x01(\x05H\x00\x88\x01\x01\x42\x08\n\x06_lanesB\x1b\n\x17\x62uckyball.balls.redmuleP\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "redmule_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\027buckyball.balls.redmuleP\001" + _globals["_PARAMS"]._serialized_start = 49 + _globals["_PARAMS"]._serialized_end = 87 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/relu_params_pb2.py b/bbdev/api/steps/config/scripts/relu_params_pb2.py new file mode 100644 index 000000000..cb3e7123f --- /dev/null +++ b/bbdev/api/steps/config/scripts/relu_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: relu_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "relu_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x11relu_params.proto\x12\x14\x62uckyball.balls.relu"\x08\n\x06ParamsB\x18\n\x14\x62uckyball.balls.reluP\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "relu_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\024buckyball.balls.reluP\001" + _globals["_PARAMS"]._serialized_start = 43 + _globals["_PARAMS"]._serialized_end = 51 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/rocket_params_pb2.py b/bbdev/api/steps/config/scripts/rocket_params_pb2.py new file mode 100644 index 000000000..8c6cbead1 --- /dev/null +++ b/bbdev/api/steps/config/scripts/rocket_params_pb2.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: rocket_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "rocket_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x13rocket_params.proto\x12\x1b\x62uckyball.components.rocket"\xfa\x01\n\x06Params\x12\x1d\n\x15physical_address_bits\x18\x01 \x02(\r\x12\x14\n\x0chart_id_bits\x18\x02 \x02(\r\x12\x13\n\x0bpage_levels\x18\x03 \x02(\r\x12\x13\n\x0bpmp_entries\x18\x04 \x02(\r\x12\x16\n\x0evirtual_memory\x18\x05 \x02(\x08\x12\x12\n\nsupervisor\x18\x06 \x02(\x08\x12\x0c\n\x04user\x18\x07 \x02(\x08\x12\x16\n\x0e\x66loating_point\x18\x08 \x02(\x08\x12\x0f\n\x07\x61tomics\x18\t \x02(\x08\x12\x17\n\x0fmultiply_divide\x18\n \x02(\x08\x12\x15\n\rexternal_home\x18\x0b \x02(\x08\x42\x1f\n\x1b\x62uckyball.components.rocketP\x01' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "rocket_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = ( + b"\n\033buckyball.components.rocketP\001" + ) + _globals["_PARAMS"]._serialized_start = 53 + _globals["_PARAMS"]._serialized_end = 303 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/smatmul_params_pb2.py b/bbdev/api/steps/config/scripts/smatmul_params_pb2.py new file mode 100644 index 000000000..fa13c0b8f --- /dev/null +++ b/bbdev/api/steps/config/scripts/smatmul_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: smatmul_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "smatmul_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x14smatmul_params.proto\x12\x17\x62uckyball.balls.smatmul"P\n\x06Params\x12\x15\n\x08tileRows\x18\x01 \x01(\x05H\x00\x88\x01\x01\x12\x15\n\x08tileCols\x18\x02 \x01(\x05H\x01\x88\x01\x01\x42\x0b\n\t_tileRowsB\x0b\n\t_tileColsB\x1b\n\x17\x62uckyball.balls.smatmulP\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "smatmul_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\027buckyball.balls.smatmulP\001" + _globals["_PARAMS"]._serialized_start = 49 + _globals["_PARAMS"]._serialized_end = 129 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/toint8_params_pb2.py b/bbdev/api/steps/config/scripts/toint8_params_pb2.py new file mode 100644 index 000000000..ce767b8ea --- /dev/null +++ b/bbdev/api/steps/config/scripts/toint8_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: toint8_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "toint8_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x13toint8_params.proto\x12\x16\x62uckyball.balls.toint8"\x08\n\x06ParamsB\x1a\n\x16\x62uckyball.balls.toint8P\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "toint8_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\026buckyball.balls.toint8P\001" + _globals["_PARAMS"]._serialized_start = 47 + _globals["_PARAMS"]._serialized_end = 55 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/trace_params_pb2.py b/bbdev/api/steps/config/scripts/trace_params_pb2.py new file mode 100644 index 000000000..6c325cae2 --- /dev/null +++ b/bbdev/api/steps/config/scripts/trace_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: trace_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "trace_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x12trace_params.proto\x12\x15\x62uckyball.balls.trace"\x08\n\x06ParamsB\x19\n\x15\x62uckyball.balls.traceP\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "trace_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\025buckyball.balls.traceP\001" + _globals["_PARAMS"]._serialized_start = 45 + _globals["_PARAMS"]._serialized_end = 53 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/transpose_params_pb2.py b/bbdev/api/steps/config/scripts/transpose_params_pb2.py new file mode 100644 index 000000000..7171096d3 --- /dev/null +++ b/bbdev/api/steps/config/scripts/transpose_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: transpose_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "transpose_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x16transpose_params.proto\x12\x19\x62uckyball.balls.transpose"T\n\x06Params\x12\x15\n\x08InputNum\x18\x01 \x01(\x05H\x00\x88\x01\x01\x12\x17\n\ninputWidth\x18\x02 \x01(\x05H\x01\x88\x01\x01\x42\x0b\n\t_InputNumB\r\n\x0b_inputWidthB\x1d\n\x19\x62uckyball.balls.transposeP\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "transpose_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\031buckyball.balls.transposeP\001" + _globals["_PARAMS"]._serialized_start = 53 + _globals["_PARAMS"]._serialized_end = 137 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/vector_params_pb2.py b/bbdev/api/steps/config/scripts/vector_params_pb2.py new file mode 100644 index 000000000..e1e8bf1ea --- /dev/null +++ b/bbdev/api/steps/config/scripts/vector_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: vector_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "vector_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x13vector_params.proto\x12\x16\x62uckyball.balls.vector"\xf6\x01\n\x06Params\x12\x15\n\x08InputNum\x18\x01 \x01(\x05H\x00\x88\x01\x01\x12\x17\n\ninputWidth\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x11\n\x04lane\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x18\n\x0boutputWidth\x18\x04 \x01(\x05H\x03\x88\x01\x01\x12\x1a\n\rnumMulThreads\x18\x05 \x01(\x05H\x04\x88\x01\x01\x12\x1a\n\rnumCasThreads\x18\x06 \x01(\x05H\x05\x88\x01\x01\x42\x0b\n\t_InputNumB\r\n\x0b_inputWidthB\x07\n\x05_laneB\x0e\n\x0c_outputWidthB\x10\n\x0e_numMulThreadsB\x10\n\x0e_numCasThreadsB\x1a\n\x16\x62uckyball.balls.vectorP\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "vector_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\026buckyball.balls.vectorP\001" + _globals["_PARAMS"]._serialized_start = 48 + _globals["_PARAMS"]._serialized_end = 294 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/verification.py b/bbdev/api/steps/config/scripts/verification.py new file mode 100644 index 000000000..915676256 --- /dev/null +++ b/bbdev/api/steps/config/scripts/verification.py @@ -0,0 +1,99 @@ +import importlib.util +import json +import tomllib +from pathlib import Path +from google.protobuf.json_format import ParseDict +import design_pb2 + + +def ip_registrations(repo, entries): + repo = Path(repo) + catalog = tomllib.loads((repo / "arch/uvm.toml").read_text()) + profiles = [] + identities = set() + for entry in entries: + if set(entry) != {"ip", "config"}: + raise ValueError("UVM registration must contain exactly ip and config") + ip = entry["ip"] + source = Path(entry["config"]["_file"]).relative_to(repo) + identity = (ip, source.stem) + if identity in identities: + raise ValueError(f"Duplicate UVM registration: {ip}/{source.stem}") + identities.add(identity) + root = repo / catalog["paths"][ip] + suite = tomllib.loads((root / "src/main/resources/suite.toml").read_text()) + tests = suite["tests"] + names = [test["name"] for test in tests] + if not names or len(names) != len(set(names)): + raise ValueError(f"Suite must declare unique test names: {root}") + for test in tests: + if test["kind"] not in {"functional", "assertion"}: + raise ValueError(f"Invalid test kind: {test['kind']}") + if test["kind"] == "assertion" and not test["assertion"]: + raise ValueError( + f"Assertion test requires an exact assertion message: {test['name']}" + ) + if test["timeout_ns"] <= 0 or test["wall_timeout_s"] <= 0: + raise ValueError( + f"Positive simulation and wall timeouts are required: {test['name']}" + ) + if not test["rtl_scopes"] or not set(test["metrics"]) <= { + "line", + "cond", + "tgl", + }: + raise ValueError( + f"Invalid DUT coverage scope or metrics: {test['name']}" + ) + if not (root / "src/main/resources" / test["filelist"]).is_file(): + raise FileNotFoundError(test["filelist"]) + config = entry["config"] + parameters = { + key: value for key, value in config["parameters"].items() if key != "_file" + } + defines = { + key: parameters[value] + for key, value in config["defines"].items() + if key != "_file" + } + profile = design_pb2.VerificationProfile( + ip=ip, + profile=source.stem, + config_path=str(source), + root=str(root.relative_to(repo)), + parameters_json=json.dumps(parameters), + defines={key: str(value) for key, value in defines.items()}, + resources="src/main/resources", + scope="ip", + ) + build = catalog["build"][ip] + profile.build.kind = "mill" + profile.build.directory = build["directory"] + profile.build.module = build["module"] + profile.build.arguments.extend(build["arguments"]) + ParseDict(suite["model"], profile.model) + for test in tests: + ParseDict(test, profile.tests.add()) + profiles.append(profile) + return profiles + + +def default_ip_config(repo, ip): + repo = Path(repo) + catalog = tomllib.loads((repo / "arch/uvm.toml").read_text()) + root = repo / catalog["paths"][ip] + spec = importlib.util.spec_from_file_location( + "verification_toml", Path(__file__).with_name("1_toml2json.py") + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + expanded = module.toml2json(root / "configs/uvm.toml") + profiles = ip_registrations(repo, [{"ip": ip, "config": expanded}]) + + config = design_pb2.Design(name=ip) + config.verification.ips.append(ip) + config.verification.profiles.extend(profiles) + output = root / "build/verification/default/design.pb" + output.parent.mkdir(parents=True, exist_ok=True) + output.write_bytes(config.SerializeToString(deterministic=True)) + return output diff --git a/bbdev/api/steps/config/scripts/xiangshan_params_pb2.py b/bbdev/api/steps/config/scripts/xiangshan_params_pb2.py new file mode 100644 index 000000000..f83266f3f --- /dev/null +++ b/bbdev/api/steps/config/scripts/xiangshan_params_pb2.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: xiangshan_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "xiangshan_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +import hardware_pb2 as hardware__pb2 + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x16xiangshan_params.proto\x12\x1e\x62uckyball.components.xiangshan\x1a\x0ehardware.proto"\\\n\x06Params\x12\x0f\n\x07node_id\x18\x01 \x02(\r\x12\x0f\n\x07home_id\x18\x02 \x02(\r\x12\x30\n\tdma_cache\x18\x03 \x02(\x0b\x32\x1d.buckyball.config.RnfHardwareB"\n\x1e\x62uckyball.components.xiangshanP\x01' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "xiangshan_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = ( + b"\n\036buckyball.components.xiangshanP\001" + ) + _globals["_PARAMS"]._serialized_start = 74 + _globals["_PARAMS"]._serialized_end = 166 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/dc/01_area_api.step.py b/bbdev/api/steps/dc/01_area_api.step.py new file mode 100644 index 000000000..935c99cf9 --- /dev/null +++ b/bbdev/api/steps/dc/01_area_api.step.py @@ -0,0 +1,34 @@ +from utils.event_common import require_target +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.event_common import require_design + +config = { + "name": "dc-area-api", + "description": "run DC synthesis and area reports", + "flows": ["dc"], + "triggers": [api("POST", "/dc/area")], + "enqueues": ["dc.verilog"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = req.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + design = require_design(body) + await ctx.enqueue( + { + "topic": "dc.verilog", + "data": { + "design": design, + "top": body.get("top"), + "from_area_workflow": True, + "_trace_id": ctx.trace_id, + "target": selection, + }, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/dc/01_area_event.step.py b/bbdev/api/steps/dc/01_area_event.step.py new file mode 100644 index 000000000..e33a8405b --- /dev/null +++ b/bbdev/api/steps/dc/01_area_event.step.py @@ -0,0 +1,203 @@ +import json +import os +import shlex +import shutil +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) + +from utils.event_common import check_result, get_origin_trace_id +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger_async +from tapeout import get_tapeout_contract, write_run_tcl +from steps.dc.scripts.report import area_report +from utils.reports import source_context + +config = { + "name": "dc-area", + "description": "run Synopsys Design Compiler and area reports on prepared DC RTL", + "flows": ["dc"], + "triggers": [queue("dc.area")], + "enqueues": ["dc.sim"], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + source_list_path = input_data.get("source_list") + top_module = input_data.get("top") + analysis_dir = input_data.get("analysis_dir") + if not isinstance(source_list_path, str) or not os.path.isfile(source_list_path): + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "dc", "error": "missing prepared DC source list"}, + trace_id=origin_tid, + ) + return + if not isinstance(analysis_dir, str) or not analysis_dir: + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "task": "dc", + "error": "missing fixed DC analysis output directory", + }, + trace_id=origin_tid, + ) + return + if shutil.which("dc_shell") is None: + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "task": "dc", + "error": "dc_shell is not on PATH; source the DC host environment before running bbdev dc --area", + }, + trace_id=origin_tid, + ) + return + + try: + contract = get_tapeout_contract( + get_buckyball_path(), + input_data.get("design"), + top_module, + selection=input_data["target"], + ) + top_module = contract.top_module + except (OSError, ValueError) as exc: + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "dc", "error": str(exc)}, + trace_id=origin_tid, + ) + return + + output_dir = os.path.join(analysis_dir, "outputs") + report_dir = os.path.join(analysis_dir, "reports") + os.makedirs(output_dir, exist_ok=True) + os.makedirs(report_dir, exist_ok=True) + try: + gman = input_data.get("generate_manifest") + if not isinstance(gman, str) or not gman: + rman = input_data.get("replace_manifest") + if isinstance(rman, str) and os.path.isfile(rman): + with open(rman, encoding="utf-8") as handle: + gman = json.load(handle).get("generate_manifest") + if not isinstance(gman, str) or not os.path.isfile(gman): + raise ValueError("missing generate_manifest with link_dbs") + with open(gman, encoding="utf-8") as handle: + link_dbs = json.load(handle).get("link_dbs") + if not isinstance(link_dbs, list): + raise ValueError(f"generate_manifest.link_dbs must be a list: {gman}") + for path in link_dbs: + if not isinstance(path, str) or not os.path.isfile(path): + raise ValueError(f"sram db missing: {path}") + tech = { + "target_library": contract.target_library, + "synthetic_library": contract.synthetic_library, + "link_library": list(contract.link_library) + list(link_dbs), + "max_threads": contract.max_threads, + } + run_config = write_run_tcl( + os.path.join(analysis_dir, "run.tcl"), + { + "top": top_module, + "source_list": source_list_path, + "output_dir": output_dir, + "report_dir": report_dir, + "sdc": str(contract.constraints_sdc), + **tech, + }, + ) + except (OSError, ValueError, json.JSONDecodeError) as exc: + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "dc", "error": str(exc)}, + trace_id=origin_tid, + ) + return + script_path = contract.dc_script + report_context = source_context( + get_buckyball_path(), contract.design, input_data["target"] + ) + dc_log = os.path.join(analysis_dir, "dc_shell.log") + ctx.logger.info( + f"Running design-owned DC synthesis for {contract.design}, top {top_module}: {script_path}" + ) + result = await stream_run_logger_async( + cmd=( + f"set -o pipefail; dc_shell -f {shlex.quote(str(script_path))} " + f"-x {shlex.quote('set RUN_CONFIG ' + '{' + str(run_config) + '}')} " + f"2>&1 | tee {shlex.quote(dc_log)}" + ), + logger=ctx.logger, + cwd=os.path.dirname(script_path), + stdout_prefix="dc", + stderr_prefix="dc", + ) + extra_fields = { + "task": "area", + "top_module": top_module, + "dc_script": str(script_path), + "dc_output_dir": output_dir, + "dc_report_dir": report_dir, + "dc_log": dc_log, + "run_config": str(run_config), + "design": contract.design, + "tapeout_dir": str(contract.root), + "replace_manifest": input_data.get("replace_manifest"), + } + if result.returncode == 0: + try: + extra_fields["report_path"] = area_report( + report_context, origin_tid, report_dir, contract + ) + except (OSError, ValueError) as error: + await check_result( + ctx, + 1, + extra_fields={**extra_fields, "error": str(error)}, + trace_id=origin_tid, + ) + return + + await check_result( + ctx, + result.returncode, + continue_run=False, + extra_fields=extra_fields, + trace_id=origin_tid, + ) + if result.returncode == 0 and input_data.get("from_power_workflow"): + await ctx.enqueue( + { + "topic": "dc.sim", + "data": { + **input_data, + "area_dir": output_dir, + "design": contract.design, + "_trace_id": origin_tid, + "target": input_data["target"], + }, + } + ) diff --git a/bbdev/api/steps/dc/02_verilog_api.step.py b/bbdev/api/steps/dc/02_verilog_api.step.py new file mode 100644 index 000000000..0aa5a761c --- /dev/null +++ b/bbdev/api/steps/dc/02_verilog_api.step.py @@ -0,0 +1,33 @@ +from utils.event_common import require_target +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.event_common import require_design + +config = { + "name": "dc-verilog-api", + "description": "generate verilog for dc flow", + "flows": ["dc"], + "triggers": [api("POST", "/dc/verilog")], + "enqueues": ["dc.verilog"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = req.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + design = require_design(body) + await ctx.enqueue( + { + "topic": "dc.verilog", + "data": { + "design": design, + "top": body.get("top"), + "_trace_id": ctx.trace_id, + "target": selection, + }, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/dc/03_sim_event.step.py b/bbdev/api/steps/dc/03_sim_event.step.py new file mode 100644 index 000000000..19257914f --- /dev/null +++ b/bbdev/api/steps/dc/03_sim_event.step.py @@ -0,0 +1,287 @@ +from pathlib import Path +from utils.path import design_output_root +import os +import shlex +import shutil +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) + +from utils.event_common import check_result, get_origin_trace_id +from utils.path import get_buckyball_path, workload_build_dir, workloads_output_root +from utils.search_workload import search_workload +from utils.stream_run import stream_run_logger_async +from tapeout import get_tapeout_contract, resolve_power_window, write_run_env + +config = { + "name": "dc-sim", + "description": "rerun design-owned simulation and produce activity for PTPX", + "flows": ["dc"], + "triggers": [queue("dc.sim")], + "enqueues": ["pt.run"], +} + + +def _first(data: dict, *names: str): + for name in names: + value = data.get(name) + if value is not None and value != "": + return value + return None + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + design = input_data.get("design") + top_module = input_data.get("top") + analysis_dir = input_data.get("analysis_dir") + if not isinstance(analysis_dir, str) or not analysis_dir: + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "sim", "error": "missing analysis directory"}, + trace_id=origin_tid, + ) + return + + try: + contract = get_tapeout_contract( + get_buckyball_path(), + input_data.get("design"), + top_module, + selection=input_data["target"], + ) + top_module = contract.top_module + except (OSError, ValueError) as exc: + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "sim", "error": str(exc)}, + trace_id=origin_tid, + ) + return + + activity_format = str(_first(input_data, "format") or contract.power_format).lower() + if activity_format not in {"saif", "vcd", "fsdb"}: + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "task": "sim", + "error": f"unsupported activity format: {activity_format}", + }, + trace_id=origin_tid, + ) + return + + # An explicit activity file is retained as a debugging escape hatch. Normal + # power runs leave it unset and always regenerate activity below. + explicit_activity = _first(input_data, "activity") + if explicit_activity: + if not os.path.isfile(explicit_activity): + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "task": "sim", + "error": f"activity file does not exist: {explicit_activity}", + }, + trace_id=origin_tid, + ) + return + await ctx.enqueue( + { + "topic": "pt.run", + "data": { + **input_data, + "activity": explicit_activity, + "format": activity_format, + "_trace_id": origin_tid, + "target": input_data["target"], + }, + } + ) + return + + if contract.power_sim_script is None: + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "task": "sim", + "error": f"design {design} has no gate-level power simulation entry; provide an activity file", + }, + trace_id=origin_tid, + ) + return + + if shutil.which("bash") is None: + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "task": "sim", + "error": "bash is required for design power_sim.sh", + }, + trace_id=origin_tid, + ) + return + + activity_dir = os.path.join(analysis_dir, "activity") + os.makedirs(activity_dir, exist_ok=True) + workload_name = _first(input_data, "workload") or contract.power_workload + if not workload_name: + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "task": "sim", + "error": "missing workload name; pass --workload ", + }, + trace_id=origin_tid, + ) + return + if os.path.isabs(str(workload_name)): + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "task": "sim", + "error": "workload must be a built ELF name, not an absolute path", + }, + trace_id=origin_tid, + ) + return + if not design: + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "sim", "error": "missing design"}, + trace_id=origin_tid, + ) + return + bbdir = get_buckyball_path() + workload = None + for root in ( + str( + Path(design_output_root(bbdir, design, selection=input_data["target"])) + / "workloads" + ), + os.path.join( + workload_build_dir(bbdir, design, selection=input_data["target"]), "src" + ), + ): + workload = search_workload(root, str(workload_name)) + if workload: + break + if not workload: + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "task": "sim", + "error": f"workload not found for design {design}: {workload_name}", + }, + trace_id=origin_tid, + ) + return + try: + start_ns, end_ns = resolve_power_window( + contract, + _first(input_data, "start_ns", "start-ns"), + _first(input_data, "end_ns", "end-ns"), + ) + except ValueError as exc: + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "sim", "error": str(exc)}, + trace_id=origin_tid, + ) + return + netlist = os.path.join(input_data.get("area_dir") or "", f"{top_module}.v") + sdc = os.path.join(input_data.get("area_dir") or "", f"{top_module}.sdc") + activity_path = os.path.join(activity_dir, f"activity.{activity_format}") + run_env = write_run_env( + os.path.join(activity_dir, "run.env"), + { + "OUTPUT_DIR": activity_dir, + "ACTIVITY_FILE": activity_path, + "ACTIVITY_FORMAT": activity_format, + "TOP": top_module, + "CONFIG": design, + "NETLIST": netlist, + "SDC": sdc, + "WORKLOAD": workload, + "START_NS": start_ns, + "END_NS": end_ns, + "ACTIVITY_STRIP_PATH": contract.power_strip_path, + }, + ) + + ctx.logger.info( + f"Running design-owned power simulation for {contract.design}; " + f"format={activity_format}, workload={workload or ''}, " + f"window={start_ns or ''}..{end_ns or ''} ns" + ) + result = await stream_run_logger_async( + cmd=f"bash {shlex.quote(str(contract.power_sim_script))} {shlex.quote(str(run_env))}", + logger=ctx.logger, + cwd=str(contract.root), + stdout_prefix="power sim", + stderr_prefix="power sim", + ) + if result.returncode == 0 and not os.path.isfile(activity_path): + result.returncode = 1 + error = f"design power simulation completed without producing {activity_path}" + else: + error = None + await check_result( + ctx, + result.returncode, + continue_run=False, + extra_fields={ + "task": "sim", + "design": contract.design, + "power_sim_script": str(contract.power_sim_script), + "activity": activity_path, + "format": activity_format, + "workload": workload, + "start_ns": start_ns, + "end_ns": end_ns, + **({"error": error} if error else {}), + }, + trace_id=origin_tid, + ) + if result.returncode == 0: + await ctx.enqueue( + { + "topic": "pt.run", + "data": { + **input_data, + "activity": activity_path, + "format": activity_format, + "strip_path": contract.power_strip_path, + "_trace_id": origin_tid, + "target": input_data["target"], + }, + } + ) diff --git a/bbdev/api/steps/dc/04_power_api.step.py b/bbdev/api/steps/dc/04_power_api.step.py new file mode 100644 index 000000000..80be9c2e0 --- /dev/null +++ b/bbdev/api/steps/dc/04_power_api.step.py @@ -0,0 +1,35 @@ +from utils.event_common import require_target +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.event_common import require_design + +config = { + "name": "dc-power-api", + "description": "run DC synthesis and PrimeTime PX power analysis", + "flows": ["dc"], + "triggers": [api("POST", "/dc/power")], + "enqueues": ["dc.verilog"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = req.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + design = require_design(body) + await ctx.enqueue( + { + "topic": "dc.verilog", + "data": { + **body, + "design": design, + "top": body.get("top"), + "from_power_workflow": True, + "_trace_id": ctx.trace_id, + "target": selection, + }, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/dc/scripts/emit_seq_mem_models.py b/bbdev/api/steps/dc/scripts/emit_seq_mem_models.py new file mode 100644 index 000000000..9e89c31b3 --- /dev/null +++ b/bbdev/api/steps/dc/scripts/emit_seq_mem_models.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Emit behavioral models for FIRRTL externalized sequential memories. + +These models are used only by gate-level activity simulation when a design has +not yet supplied PDK SRAM macro simulation collateral. They retain the +external-memory interface from ``mems.conf`` so the DC netlist is simulated +with functional memories rather than X-producing black boxes. +""" + +from __future__ import annotations + +import argparse +import re +from pathlib import Path + + +def write_masked( + lines: list[str], *, addr: str, data: str, mask: str, width: int, gran: int +) -> None: + lines.extend( + [ + f" for (int i = 0; i < {width // gran}; i++) begin", + f" if ({mask}[i]) memory[{addr}][i*{gran} +: {gran}] <= {data}[i*{gran} +: {gran}];", + " end", + ] + ) + + +def emit(name: str, depth: int, width: int, ports: str, mask_gran: int | None) -> str: + lines = [f"module {name}("] + if ports == "mrw": + lines += [ + " input logic [$clog2(" + str(depth) + ")-1:0] RW0_addr,", + " input logic RW0_en, RW0_clk, RW0_wmode,", + f" input logic [{width - 1}:0] RW0_wdata,", + f" output logic [{width - 1}:0] RW0_rdata,", + f" input logic [{width // mask_gran - 1}:0] RW0_wmask", + ");", + f" logic [{width - 1}:0] memory [0:{depth - 1}];", + " always @(posedge RW0_clk) begin", + " if (RW0_en) begin", + " if (RW0_wmode) begin", + ] + write_masked( + lines, + addr="RW0_addr", + data="RW0_wdata", + mask="RW0_wmask", + width=width, + gran=mask_gran or width, + ) + lines += [ + " end", + " RW0_rdata <= memory[RW0_addr];", + " end", + " end", + ] + elif ports == "rw": + lines += [ + " input logic [$clog2(" + str(depth) + ")-1:0] RW0_addr,", + " input logic RW0_en, RW0_clk, RW0_wmode,", + f" input logic [{width - 1}:0] RW0_wdata,", + f" output logic [{width - 1}:0] RW0_rdata", + ");", + f" logic [{width - 1}:0] memory [0:{depth - 1}];", + " always @(posedge RW0_clk) begin", + " if (RW0_en) begin", + " if (RW0_wmode) memory[RW0_addr] <= RW0_wdata;", + " RW0_rdata <= memory[RW0_addr];", + " end", + " end", + ] + elif ports in {"mwrite,read", "write,read"}: + has_mask = ports.startswith("mwrite") + lines += [ + " input logic [$clog2(" + str(depth) + ")-1:0] R0_addr,", + " input logic R0_en, R0_clk,", + f" output logic [{width - 1}:0] R0_data,", + " input logic [$clog2(" + str(depth) + ")-1:0] W0_addr,", + " input logic W0_en, W0_clk,", + f" input logic [{width - 1}:0] W0_data" + ("," if has_mask else ""), + ] + if has_mask: + lines.append(f" input logic [{width // mask_gran - 1}:0] W0_mask") + lines += [ + ");", + f" logic [{width - 1}:0] memory [0:{depth - 1}];", + " always @(posedge R0_clk) if (R0_en) R0_data <= memory[R0_addr];", + " always @(posedge W0_clk) if (W0_en) begin", + ] + if has_mask: + write_masked( + lines, + addr="W0_addr", + data="W0_data", + mask="W0_mask", + width=width, + gran=mask_gran or width, + ) + else: + lines.append(" memory[W0_addr] <= W0_data;") + lines += [" end"] + else: + raise ValueError(f"unsupported memory port topology for {name}: {ports}") + lines += ["endmodule", ""] + return "\n".join(lines) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--mem-conf", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + models = [] + for raw in args.mem_conf.read_text(encoding="utf-8").splitlines(): + line = raw.strip() + if not line: + continue + match = re.fullmatch( + r"name\s+(?P\S+)\s+depth\s+(?P\d+)\s+width\s+(?P\d+)" + r"\s+ports\s+(?P\S+)(?:\s+mask_gran\s+(?P\d+))?", + line, + ) + if match is None: + raise ValueError(f"invalid mems.conf entry: {line}") + values = match.groupdict() + models.append( + emit( + values["name"], + int(values["depth"]), + int(values["width"]), + values["ports"], + int(values["mask"]) if values["mask"] else None, + ) + ) + if not models: + raise ValueError(f"no memories found in {args.mem_conf}") + args.output.write_text( + "`timescale 1ns/1ps\n\n" + "\n".join(models), encoding="utf-8" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bbdev/api/steps/dc/scripts/gate_dpi.cc b/bbdev/api/steps/dc/scripts/gate_dpi.cc new file mode 100644 index 000000000..0b28a9fbe --- /dev/null +++ b/bbdev/api/steps/dc/scripts/gate_dpi.cc @@ -0,0 +1,43 @@ +#include +#include + +extern "C" void dpi_bdb_set_clk(unsigned long long) {} + +extern "C" void scu_uart_write(std::uint32_t, std::uint32_t ch) { + std::fputc(static_cast(ch & 0xff), stdout); + std::fflush(stdout); +} + +// Gate power runs use the requested measurement horizon, rather than ending +// when a workload's SCU writes its completion register. +extern "C" void scu_sim_exit(std::uint32_t hart, std::uint32_t code) { + std::fprintf(stderr, "[SCU] hart %u: workload exit code %u\n", hart, code); + std::fflush(stderr); +} + +extern "C" void scu_uart_rx_sample(std::uint32_t, std::uint32_t, + std::uint32_t *valid, std::uint32_t *data) { + *valid = 0; + *data = 0; +} + +extern "C" void dpi_ant_trace(std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t) {} +extern "C" void dpi_itrace(std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t, std::uint32_t) {} +extern "C" void dpi_mtrace(std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t) {} +extern "C" void dpi_pmctrace(std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t) {} +extern "C" void dpi_mem_pmctrace(std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t) {} +extern "C" void dpi_mtrace_issue(std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t, std::uint32_t) {} diff --git a/bbdev/api/steps/dc/scripts/ptpx.tcl b/bbdev/api/steps/dc/scripts/ptpx.tcl new file mode 100755 index 000000000..4c27e50de --- /dev/null +++ b/bbdev/api/steps/dc/scripts/ptpx.tcl @@ -0,0 +1,31 @@ +# Shared PrimeTime PX implementation. Design tapeout/power.tcl files are thin +# wrappers; this reports only standard-cell dynamic power. +if {![info exists RUN_CONFIG] || $RUN_CONFIG eq ""} { error "bbdev must pass -x {set RUN_CONFIG }" } +source [file normalize $RUN_CONFIG] +set report_dir [file normalize $RUN_REPORT_DIR] +file mkdir $report_dir +set target_library [list $RUN_TARGET_LIBRARY] +set link_library [concat [list *] $target_library $RUN_SYNTHETIC_LIBRARY $RUN_LINK_LIBRARY] +read_verilog [file normalize $RUN_NETLIST] +current_design $RUN_TOP +link +read_sdc [file normalize $RUN_SDC] +set opts [list] +if {$RUN_START_NS ne "" || $RUN_END_NS ne ""} { + lappend opts -time [list "${RUN_START_NS}ns" "${RUN_END_NS}ns"] +} +set format [string tolower $RUN_ACTIVITY_FORMAT] +if {$format eq "fsdb"} { + if {$RUN_STRIP_PATH ne ""} { read_fsdb {*}$opts -strip_path $RUN_STRIP_PATH [file normalize $RUN_ACTIVITY] } else { read_fsdb {*}$opts [file normalize $RUN_ACTIVITY] } +} elseif {$format eq "vcd"} { + if {$RUN_STRIP_PATH ne ""} { read_vcd {*}$opts -strip_path $RUN_STRIP_PATH [file normalize $RUN_ACTIVITY] } else { read_vcd {*}$opts [file normalize $RUN_ACTIVITY] } +} elseif {$format eq "saif"} { + read_saif {*}$opts [file normalize $RUN_ACTIVITY] +} else { error "unsupported activity format: $format" } +set power_enable_analysis true +set power_model_preference nlpm +update_power +report_power -hierarchy -levels 3 > [file join $report_dir power_hierarchy.rpt] +report_power -verbose > [file join $report_dir power_total.rpt] +report_power -hierarchy -levels 2 -sort_by total_power > [file join $report_dir power_hierarchy_sorted.rpt] +exit diff --git a/bbdev/api/steps/dc/scripts/report.py b/bbdev/api/steps/dc/scripts/report.py new file mode 100644 index 000000000..a32ff901e --- /dev/null +++ b/bbdev/api/steps/dc/scripts/report.py @@ -0,0 +1,35 @@ +import hashlib +import re +from pathlib import Path + +from utils.reports import new_report, attachment, finish_report + + +def area_report(context, trace_id, report_dir, contract): + report_dir = Path(report_dir) + output = report_dir.parent / "report" + report = new_report(context, "dc.area", trace_id, "dc") + values = re.findall( + r"Total cell area:\s*([0-9.eE+-]+)", (report_dir / "area.rpt").read_text() + ) + if len(values) != 1: + raise ValueError("DC report must contain one total cell area") + constraints = (report_dir / "constraint.rpt").read_text() + if "Report : constraint" not in constraints: + raise ValueError("invalid DC constraint report") + report["area"] = { + "status": "pass", + "cell_um2": float(values[0]), + "scope": contract.top_module, + "conditions": ( + f"DC · {Path(contract.target_library).name} · library SHA256 " + f"{hashlib.sha256(Path(contract.target_library).read_bytes()).hexdigest()} · SDC " + f"{hashlib.sha256(contract.constraints_sdc.read_bytes()).hexdigest()}" + ), + "constraints_met": "(VIOLATED)" not in constraints, + "report_url": attachment(report, output, report_dir / "area.rpt", "area.rpt"), + "constraints_report_url": attachment( + report, output, report_dir / "constraint.rpt", "area-constraints.rpt" + ), + } + return finish_report(report, output) diff --git a/bbdev/api/steps/dc/scripts/tapeout.py b/bbdev/api/steps/dc/scripts/tapeout.py new file mode 100644 index 000000000..a7774f95b --- /dev/null +++ b/bbdev/api/steps/dc/scripts/tapeout.py @@ -0,0 +1,176 @@ +"""Resolve design-owned tapeout flow contracts.""" + +from __future__ import annotations + +import os +import re +import shlex +import tomllib +from dataclasses import dataclass +from pathlib import Path +from utils.path import design_resource_dir + + +def _tcl_word(value: str) -> str: + return "{" + str(value).replace("\\", "\\\\").replace("}", "\\}") + "}" + + +def _tcl_list(values: list[str]) -> str: + return "[list " + " ".join(_tcl_word(value) for value in values) + "]" + + +@dataclass(frozen=True) +class SramGeom: + name: str + words: int + mux: int + bits: int + bitwrite: bool = False + + +@dataclass(frozen=True) +class TapeoutContract: + design: str + root: Path + dc_script: Path + constraints_sdc: Path + power_script: Path + power_sim_script: Path | None + sram_mdf: Path + top_module: str + target_library: str + synthetic_library: list[str] + link_library: list[str] + max_threads: int + power_format: str + power_start_ns: str | None + power_end_ns: str | None + power_workload: str | None + power_strip_path: str + sram_process: str + sram_corner: str + lc_shell: Path + sram_table: dict[str, SramGeom] + + +def _p(root: Path, value: str) -> Path: + p = Path(value) + return p if p.is_absolute() else root / p + + +def resolve_power_window( + contract: TapeoutContract, start_ns: object | None, end_ns: object | None +) -> tuple[str | None, str | None]: + start = ( + str(start_ns) + if start_ns is not None and str(start_ns) != "" + else contract.power_start_ns + ) + end = ( + str(end_ns) + if end_ns is not None and str(end_ns) != "" + else contract.power_end_ns + ) + if (start is None) != (end is None): + raise ValueError("power start_ns and end_ns must be supplied together") + if start is not None: + try: + if float(start) < 0 or float(start) >= float(end): + raise ValueError + except ValueError as exc: + raise ValueError("power start_ns must be smaller than end_ns") from exc + return start, end + + +def clock_period_ns(contract: TapeoutContract) -> float: + text = contract.constraints_sdc.read_text(encoding="utf-8") + match = re.search(r"create_clock\b[^;\n]*-period\s+([0-9]+(?:\.[0-9]+)?)", text) + if match is None: + raise ValueError(f"no create_clock -period in {contract.constraints_sdc}") + period = float(match.group(1)) + if period <= 0: + raise ValueError(f"invalid create_clock -period in {contract.constraints_sdc}") + return period + + +def get_tapeout_contract( + bbdir: str | os.PathLike[str], + design: str, + top: str | None = None, + *, + selection: str, +) -> TapeoutContract: + root = design_resource_dir(bbdir, design, "tapeout", selection=selection) + with (root / "config.toml").open("rb") as f: + s = tomllib.load(f)["tapeout"] + + sram_table = {} + for row in s["sram"]: + name = row["name"] + if name in sram_table: + raise ValueError(f"duplicate tapeout.sram name {name}") + bitwrite = row["bitwrite"] if "bitwrite" in row else False + if not isinstance(bitwrite, bool): + raise ValueError( + f"tapeout.sram {name}: bitwrite must be bool, got {bitwrite!r}" + ) + sram_table[name] = SramGeom( + name, row["words"], row["mux"], row["bits"], bitwrite + ) + + pw = s.get("power_workload") + return TapeoutContract( + design=design, + root=root, + dc_script=_p(root, s["dc_script"]), + constraints_sdc=_p(root, s["constraints_sdc"]), + power_script=_p(root, s["power_script"]), + power_sim_script=( + _p(root, s["power_sim_script"]) if "power_sim_script" in s else None + ), + sram_mdf=_p(root, s["sram_mdf"]), + top_module=top or s["top"], + target_library=str(_p(root, s["target_library"])), + synthetic_library=[str(_p(root, p)) for p in s["synthetic_library"]], + link_library=[str(_p(root, p)) for p in s.get("link_library", [])], + max_threads=s["max_threads"], + power_format=s["power_format"], + power_start_ns=s.get("power_start_ns"), + power_end_ns=s.get("power_end_ns"), + power_workload=None if pw in (None, "") else str(pw), + power_strip_path=s.get("power_strip_path", ""), + sram_process=s["sram_process"], + sram_corner=s["sram_corner"], + lc_shell=_p(root, s["lc_shell"]), + sram_table=sram_table, + ) + + +def write_run_tcl(path: str | os.PathLike[str], values: dict[str, object]) -> Path: + output = Path(path) + output.parent.mkdir(parents=True, exist_ok=True) + lines = [] + for key, value in values.items(): + name = "RUN_" + key.upper() + if isinstance(value, list): + lines.append(f"set {name} {_tcl_list([str(item) for item in value])}") + elif value is None: + lines.append(f"set {name} {{}}") + elif isinstance(value, (int, float)): + lines.append(f"set {name} {value}") + else: + lines.append(f"set {name} {_tcl_word(str(value))}") + output.write_text("\n".join(lines) + "\n", encoding="utf-8") + return output + + +def write_run_env(path: str | os.PathLike[str], values: dict[str, object]) -> Path: + output = Path(path) + output.parent.mkdir(parents=True, exist_ok=True) + lines = [ + f"{key}={shlex.quote(str(value))}" + for key, value in values.items() + if value is not None + ] + output.write_text("\n".join(lines) + "\n", encoding="utf-8") + return output diff --git a/bbdev/api/steps/firesim/.gitignore b/bbdev/api/steps/firesim/.gitignore new file mode 100644 index 000000000..1e82fc7de --- /dev/null +++ b/bbdev/api/steps/firesim/.gitignore @@ -0,0 +1 @@ +*.yaml diff --git a/bbdev/api/steps/firesim/00_enumeratefpgas_api.step.py b/bbdev/api/steps/firesim/00_enumeratefpgas_api.step.py new file mode 100644 index 000000000..855525837 --- /dev/null +++ b/bbdev/api/steps/firesim/00_enumeratefpgas_api.step.py @@ -0,0 +1,31 @@ +from utils.event_common import require_target +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.event_common import require_design + +config = { + "name": "firesim-enumeratefpgas-api", + "description": "enumerate FPGAs", + "flows": ["firesim"], + "triggers": [api("POST", "/firesim/enumeratefpgas")], + "enqueues": ["firesim.enumeratefpgas"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) + await ctx.enqueue( + { + "topic": "firesim.enumeratefpgas", + "data": {"design": design, "_trace_id": ctx.trace_id, "target": selection}, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/firesim/00_enumeratefpgas_event.step.py b/bbdev/api/steps/firesim/00_enumeratefpgas_event.step.py new file mode 100644 index 000000000..734147116 --- /dev/null +++ b/bbdev/api/steps/firesim/00_enumeratefpgas_event.step.py @@ -0,0 +1,67 @@ +import os +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path, sim_name +from utils.stream_run import stream_run_logger_async +from utils.event_common import check_result, get_origin_trace_id, require_design + +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) + +from firesim_env import setup_firesim_env +from design_yaml import firesim_cmd, prepare_yamls + +config = { + "name": "firesim-enumeratefpgas", + "description": "enumerate FPGAs", + "flows": ["firesim"], + "triggers": [queue("firesim.enumeratefpgas")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + + try: + design = require_design(input_data) + target_config = sim_name( + bbdir, design, "firesim", selection=input_data["target"] + ) + yaml_dir = prepare_yamls(bbdir, design, target_config) + except ValueError as error: + ctx.logger.error(str(error)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": str(error)}, + trace_id=origin_tid, + ) + return + + env = setup_firesim_env() + ctx.logger.info(f"firesim enumeratefpgas design={design}") + result = await stream_run_logger_async( + cmd=firesim_cmd("enumeratefpgas", yaml_dir), + logger=ctx.logger, + stdout_prefix="firesim enumeratefpgas", + stderr_prefix="firesim enumeratefpgas", + env=env, + ) + + await check_result( + ctx, + result.returncode, + continue_run=False, + extra_fields={"design": design, "yaml_dir": yaml_dir}, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/firesim/01_buildbitstream_api.step.py b/bbdev/api/steps/firesim/01_buildbitstream_api.step.py new file mode 100644 index 000000000..608be6c2d --- /dev/null +++ b/bbdev/api/steps/firesim/01_buildbitstream_api.step.py @@ -0,0 +1,31 @@ +from utils.event_common import require_target +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.event_common import require_design + +config = { + "name": "firesim-buildbitstream-api", + "description": "build bitstream", + "flows": ["firesim"], + "triggers": [api("POST", "/firesim/buildbitstream")], + "enqueues": ["firesim.buildbitstream"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) + await ctx.enqueue( + { + "topic": "firesim.buildbitstream", + "data": {"design": design, "_trace_id": ctx.trace_id, "target": selection}, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/firesim/01_buildbitstream_event.step.py b/bbdev/api/steps/firesim/01_buildbitstream_event.step.py new file mode 100644 index 000000000..feed9d36d --- /dev/null +++ b/bbdev/api/steps/firesim/01_buildbitstream_event.step.py @@ -0,0 +1,87 @@ +import os +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path, sim_name +from utils.stream_run import stream_run_logger_async +from utils.event_common import check_result, get_origin_trace_id, require_design + +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) + +from firesim_env import setup_firesim_env +from design_yaml import firesim_cmd, prepare_yamls + +config = { + "name": "firesim-buildbitstream", + "description": "build bitstream", + "flows": ["firesim"], + "triggers": [queue("firesim.buildbitstream")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + + try: + design = require_design(input_data) + target_config = sim_name( + bbdir, design, "firesim", selection=input_data["target"] + ) + except ValueError as error: + ctx.logger.error(str(error)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": str(error)}, + trace_id=origin_tid, + ) + return + + yaml_dir = prepare_yamls(bbdir, design, target_config) + env = setup_firesim_env() + + au280_tcl = f"{bbdir}/thirdparty/firesim/platforms/xilinx_alveo_u280/cl_firesim/scripts/au280.tcl" + if os.path.exists(au280_tcl): + with open(au280_tcl, "r") as f: + content = f.read() + fixed_content = content.replace( + "xilinx.com:au280:part0:1.2", + "xilinx.com:au280:part0:1.0", + ) + if content != fixed_content: + with open(au280_tcl, "w") as f: + f.write(fixed_content) + ctx.logger.info("Fixed AU280 board_part version: 1.2 -> 1.0") + + ctx.logger.info( + f"firesim buildbitstream design={design} TARGET_CONFIG={target_config}" + ) + result = await stream_run_logger_async( + cmd=firesim_cmd("buildbitstream", yaml_dir), + logger=ctx.logger, + stdout_prefix="firesim buildbitstream", + stderr_prefix="firesim buildbitstream", + env=env, + ) + + await check_result( + ctx, + result.returncode, + continue_run=False, + extra_fields={ + "design": design, + "target_config": target_config, + "yaml_dir": yaml_dir, + }, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/firesim/02_infrasetup_api.step.py b/bbdev/api/steps/firesim/02_infrasetup_api.step.py new file mode 100644 index 000000000..5e8dbb957 --- /dev/null +++ b/bbdev/api/steps/firesim/02_infrasetup_api.step.py @@ -0,0 +1,31 @@ +from utils.event_common import require_target +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.event_common import require_design + +config = { + "name": "firesim-infrasetup-api", + "description": "infrasetup", + "flows": ["firesim"], + "triggers": [api("POST", "/firesim/infrasetup")], + "enqueues": ["firesim.infrasetup"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) + await ctx.enqueue( + { + "topic": "firesim.infrasetup", + "data": {"design": design, "_trace_id": ctx.trace_id, "target": selection}, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/firesim/02_infrasetup_event.step.py b/bbdev/api/steps/firesim/02_infrasetup_event.step.py new file mode 100644 index 000000000..4a4a9e600 --- /dev/null +++ b/bbdev/api/steps/firesim/02_infrasetup_event.step.py @@ -0,0 +1,78 @@ +import os +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path, sim_name +from utils.stream_run import stream_run_logger_async +from utils.event_common import check_result, get_origin_trace_id, require_design + +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) + +from firesim_env import setup_firesim_env +from design_yaml import firesim_cmd, prepare_yamls, recipe_name + +config = { + "name": "firesim-infrasetup", + "description": "infrasetup", + "flows": ["firesim"], + "triggers": [queue("firesim.infrasetup")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + + try: + design = require_design(input_data) + target_config = sim_name( + bbdir, design, "firesim", selection=input_data["target"] + ) + yaml_dir = prepare_yamls(bbdir, design, target_config) + hwdb = os.path.join(yaml_dir, "config_hwdb.yaml") + text = open(hwdb, encoding="utf-8").read() + if "bitstream_tar: null" in text or "bitstream_tar: file://" not in text: + raise ValueError( + f"no bitstream for recipe {recipe_name(design)}; " + f"run bbdev firesim --buildbitstream '--design {design}' first" + ) + except ValueError as error: + ctx.logger.error(str(error)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": str(error)}, + trace_id=origin_tid, + ) + return + + env = setup_firesim_env() + ctx.logger.info(f"firesim infrasetup design={design} TARGET_CONFIG={target_config}") + result = await stream_run_logger_async( + cmd=firesim_cmd("infrasetup", yaml_dir), + logger=ctx.logger, + stdout_prefix="firesim infrasetup", + stderr_prefix="firesim infrasetup", + env=env, + ) + + await check_result( + ctx, + result.returncode, + continue_run=False, + extra_fields={ + "design": design, + "target_config": target_config, + "yaml_dir": yaml_dir, + }, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/firesim/03_runworkload_api.step.py b/bbdev/api/steps/firesim/03_runworkload_api.step.py new file mode 100644 index 000000000..4dff37470 --- /dev/null +++ b/bbdev/api/steps/firesim/03_runworkload_api.step.py @@ -0,0 +1,31 @@ +from utils.event_common import require_target +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.event_common import require_design + +config = { + "name": "firesim-runworkload-api", + "description": "run workload", + "flows": ["firesim"], + "triggers": [api("POST", "/firesim/runworkload")], + "enqueues": ["firesim.runworkload"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) + data = {"design": design, "_trace_id": ctx.trace_id} + if body.get("jobs") is not None: + data["jobs"] = body["jobs"] + await ctx.enqueue( + {"topic": "firesim.runworkload", "data": {**(data), "target": selection}} + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/firesim/03_runworkload_event.step.py b/bbdev/api/steps/firesim/03_runworkload_event.step.py new file mode 100644 index 000000000..41b36169f --- /dev/null +++ b/bbdev/api/steps/firesim/03_runworkload_event.step.py @@ -0,0 +1,80 @@ +import os +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path, sim_name +from utils.stream_run import stream_run_logger_async +from utils.event_common import check_result, get_origin_trace_id, require_design + +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) + +from firesim_env import setup_firesim_env +from design_yaml import firesim_cmd, prepare_yamls, recipe_name + +config = { + "name": "firesim-runworkload", + "description": "run workload", + "flows": ["firesim"], + "triggers": [queue("firesim.runworkload")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + + try: + design = require_design(input_data) + target_config = sim_name( + bbdir, design, "firesim", selection=input_data["target"] + ) + yaml_dir = prepare_yamls(bbdir, design, target_config) + hwdb = os.path.join(yaml_dir, "config_hwdb.yaml") + text = open(hwdb, encoding="utf-8").read() + if "bitstream_tar: null" in text or "bitstream_tar: file://" not in text: + raise ValueError( + f"no bitstream for recipe {recipe_name(design)}; " + f"run bbdev firesim --buildbitstream '--design {design}' first" + ) + except ValueError as error: + ctx.logger.error(str(error)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": str(error)}, + trace_id=origin_tid, + ) + return + + env = setup_firesim_env() + ctx.logger.info( + f"firesim runworkload design={design} TARGET_CONFIG={target_config}" + ) + result = await stream_run_logger_async( + cmd=firesim_cmd("runworkload", yaml_dir), + logger=ctx.logger, + stdout_prefix="firesim runworkload", + stderr_prefix="firesim runworkload", + env=env, + ) + + await check_result( + ctx, + result.returncode, + continue_run=False, + extra_fields={ + "design": design, + "target_config": target_config, + "yaml_dir": yaml_dir, + }, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/firesim/README.md b/bbdev/api/steps/firesim/README.md new file mode 100644 index 000000000..29258fecb --- /dev/null +++ b/bbdev/api/steps/firesim/README.md @@ -0,0 +1,18 @@ +# FireSim Workflow + +FireSim FPGA simulation. All commands take `--design`; mill class comes from +`examples/designs//configs/design.toml` `[sims].firesim`. + +Manager YAMLs are generated per design under `scripts/yaml//`. + +## Commands + +```bash +bbdev firesim --enumeratefpgas '--design toy' +bbdev firesim --buildbitstream '--design toy' +bbdev firesim --infrasetup '--design toy' +bbdev firesim --runworkload '--design toy' +``` + +`buildbitstream` takes hours. `infrasetup` / `runworkload` require a prior +bitstream under `thirdparty/firesim/deploy/results-build` for that design recipe. diff --git a/bbdev/api/steps/firesim/scripts/build.sc b/bbdev/api/steps/firesim/scripts/build.sc new file mode 100644 index 000000000..1e818e7a9 --- /dev/null +++ b/bbdev/api/steps/firesim/scripts/build.sc @@ -0,0 +1,2 @@ +// FireSim uses the dedicated Mill graph in arch/build.sc. +// This file documents the entry point used by the FireSim step. diff --git a/bbdev/api/steps/firesim/scripts/design_yaml.py b/bbdev/api/steps/firesim/scripts/design_yaml.py new file mode 100644 index 000000000..eb4afd314 --- /dev/null +++ b/bbdev/api/steps/firesim/scripts/design_yaml.py @@ -0,0 +1,168 @@ +"""Prepare per-design FireSim manager YAMLs from design sims.firesim.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Optional + + +def recipe_name(design: str) -> str: + return f"alveo_u280_firesim_{design}_no_nic" + + +def _latest_bitstream(results: Path, recipe: str) -> Optional[Path]: + if not results.is_dir(): + return None + dirs = sorted( + (p for p in results.iterdir() if p.is_dir() and recipe in p.name), + key=lambda p: p.stat().st_mtime, + ) + for d in reversed(dirs): + matches = list(d.rglob("firesim.tar.gz")) + if matches: + return matches[0] + return None + + +def prepare_yamls(bbdir: str, design: str, target_config: str) -> str: + """Write design-scoped firesim yamls under scripts/yaml/. Return that dir.""" + if not target_config or "." not in target_config: + raise ValueError(f"invalid firesim TARGET_CONFIG: {target_config!r}") + recipe = recipe_name(design) + script_dir = Path(bbdir) / "bbdev" / "api" / "steps" / "firesim" / "scripts" + out = script_dir / "yaml" / design + out.mkdir(parents=True, exist_ok=True) + + deploy = Path(bbdir) / "thirdparty" / "firesim" / "deploy" + makefrag = script_dir / "makefrag" / "firesim" + if not makefrag.is_dir(): + raise FileNotFoundError(f"missing firesim makefrag: {makefrag}") + + build_dir = deploy / "FIRESIM_BUILD_DIR" + runs_dir = deploy / "FIRESIM_RUNS_DIR" + build_dir.mkdir(parents=True, exist_ok=True) + runs_dir.mkdir(parents=True, exist_ok=True) + + bitstream = _latest_bitstream(deploy / "results-build", recipe) + if bitstream is None: + bitstream_line = ( + " # bitstream_tar filled after buildbitstream; required for infrasetup/runworkload\n" + " bitstream_tar: null" + ) + else: + bitstream_line = f" bitstream_tar: file://{bitstream.resolve()}" + + (out / "config_build_recipes.yaml").write_text( + f"""# Auto-generated for design={design}. Do not edit by hand. +{recipe}: + PLATFORM: xilinx_alveo_u280 + TARGET_PROJECT: firesim + TARGET_PROJECT_MAKEFRAG: {makefrag} + DESIGN: FireSim + TARGET_CONFIG: {target_config} + PLATFORM_CONFIG: BaseXilinxAlveoU280Config + deploy_quintuplet: null + platform_config_args: + fpga_frequency: 30 + build_strategy: TIMING + post_build_hook: null + metasim_customruntimeconfig: null + bit_builder_recipe: bit-builder-recipes/xilinx_alveo_u280.yaml +""", + encoding="utf-8", + ) + + (out / "config_build.yaml").write_text( + f"""# Auto-generated for design={design}. Do not edit by hand. +build_farm: + base_recipe: build-farm-recipes/externally_provisioned.yaml + recipe_arg_overrides: + default_build_dir: {build_dir} + build_farm_hosts: + - localhost + +builds_to_run: + - {recipe} + +agfis_to_share: [] + +share_with_accounts: {{}} +""", + encoding="utf-8", + ) + + (out / "config_runtime.yaml").write_text( + f"""# Auto-generated for design={design}. Do not edit by hand. +run_farm: + base_recipe: run-farm-recipes/externally_provisioned.yaml + recipe_arg_overrides: + default_platform: XilinxAlveoU280InstanceDeployManager + default_simulation_dir: {runs_dir} + default_fpga_db: /opt/firesim-db.json + run_farm_hosts_to_use: + - localhost: one_fpgas_spec + +metasimulation: + metasimulation_enabled: false + metasimulation_host_simulator: verilator + metasimulation_only_plusargs: "+fesvr-step-size=128 +max-cycles=100000000" + metasimulation_only_vcs_plusargs: "+vcs+initreg+0 +vcs+initmem+0" + +target_config: + topology: no_net_config + no_net_num_nodes: 1 + link_latency: 6405 + switching_latency: 10 + net_bandwidth: 200 + profile_interval: -1 + default_hw_config: {recipe} + plusarg_passthrough: "" + +tracing: + enable: no + output_format: 0 + selector: 1 + start: 0 + end: -1 + +autocounter: + read_rate: 0 + +workload: + workload_name: interactive.json + terminate_on_completion: no + suffix_tag: null + +host_debug: + zero_out_dram: no + disable_synth_asserts: no + +synth_print: + start: 0 + end: -1 + cycle_prefix: yes +""", + encoding="utf-8", + ) + + (out / "config_hwdb.yaml").write_text( + f"""# Auto-generated for design={design}. bitstream_tar updated by buildbitstream / results-build scan. +{recipe}: +{bitstream_line} + deploy_quintuplet_override: null + custom_runtime_config: null +""", + encoding="utf-8", + ) + + return str(out) + + +def firesim_cmd(action: str, yaml_dir: str) -> str: + return ( + f"firesim {action}" + f" -a {yaml_dir}/config_hwdb.yaml" + f" -b {yaml_dir}/config_build.yaml" + f" -r {yaml_dir}/config_build_recipes.yaml" + f" -c {yaml_dir}/config_runtime.yaml" + ) diff --git a/bbdev/api/steps/firesim/scripts/firesim_env.py b/bbdev/api/steps/firesim/scripts/firesim_env.py new file mode 100644 index 000000000..03af44084 --- /dev/null +++ b/bbdev/api/steps/firesim/scripts/firesim_env.py @@ -0,0 +1,54 @@ +"""FireSim environment setup.""" + +import os +import subprocess + + +def setup_firesim_env() -> dict: + """Setup FireSim env vars + SSH agent. Returns env dict for subprocess.""" + env = os.environ.copy() + env["FIRESIM_SOURCED"] = "1" # For firesim Python CLI + env["FIRESIM_ENV_SOURCED"] = "1" # For firesim Makefile + env["FIRESIM_RUNFARM_PREFIX"] = "" + + # Create minimal env.sh for firesim if it doesn't exist + firesim_dir = os.path.abspath( + os.path.join( + os.path.dirname(__file__), "../../../../..", "thirdparty", "firesim" + ) + ) + env["PATH"] = os.path.join(firesim_dir, "deploy") + os.pathsep + env.get("PATH", "") + env_sh = os.path.join(firesim_dir, "env.sh") + if not os.path.exists(env_sh): + riscv_path = env.get( + "RISCV", + os.path.abspath( + os.path.join(os.path.dirname(__file__), "../../../../..", "result") + ), + ) + with open(env_sh, "w") as f: + f.write(f"""#!/usr/bin/env bash +# Minimal env.sh for FireSim (auto-generated by firesim_env.py) + +export FIRESIM_ENV_SOURCED=1 +export FS_DIR="{firesim_dir}" +export RISCV="{riscv_path}" +""") + + # Setup SSH agent for localhost FPGA enumeration + key = os.path.expanduser("~/.ssh/firesim.pem") + if ( + os.path.exists(key) + and "firesim.pem" + not in subprocess.run(["ssh-add", "-l"], capture_output=True, text=True).stdout + ): + if not env.get("SSH_AUTH_SOCK"): + for line in subprocess.run( + ["ssh-agent", "-s"], capture_output=True, text=True + ).stdout.splitlines(): + if "=" in line: + k, v = line.split("=", 1) + env[k] = v.split(";", 1)[0] + subprocess.run(["ssh-add", key], env=env, capture_output=True) + + return env diff --git a/bbdev/api/steps/firesim/scripts/makefrag/firesim/build.mk b/bbdev/api/steps/firesim/scripts/makefrag/firesim/build.mk new file mode 100644 index 000000000..ece8ee0e2 --- /dev/null +++ b/bbdev/api/steps/firesim/scripts/makefrag/firesim/build.mk @@ -0,0 +1,54 @@ +# See LICENSE for license details. + +CHIPYARD_STAGING_DIR := $(abspath $(makefile_dir)/../../../../../../../thirdparty/firesim-staging) + +# target scala directories to copy into midas. used by TARGET_COPY_TO_MIDAS_SCALA_DIRS +TARGET_COPY_TO_MIDAS_SCALA_DIRS := \ + $(addprefix $(chipyard_dir)/generators/firechip/,bridgeinterfaces goldengateimplementations) + +# this rule always is run, but may not update the timestamp of the targets (depending on what the Chipyard make does). +# if that is the case (Chipyard make doesn't update it's outputs), then downstream rules *should* be skipped. +# all other chipyard collateral is located in chipyard's generated sources area. +$(FIRRTL_FILE) $(ANNO_FILE) &: SHELL := /usr/bin/env bash # needed for running source in recipe +$(FIRRTL_FILE) $(ANNO_FILE) &: firesim_target_symlink_hook + @mkdir -p $(@D) + @mkdir -p $(TARGET_SBT_DIR)/target/generated-src/$(long_name) + cd $(TARGET_MILL_DIR) && \ + mill -i firesim.runMain chipyard.Generator \ + --target-dir $(TARGET_SBT_DIR)/target/generated-src/$(long_name) \ + --name $(long_name) \ + --top-module $(DESIGN_PACKAGE).$(DESIGN) \ + --legacy-configs $(TARGET_CONFIG_PACKAGE):$(TARGET_CONFIG) \ + --emit-legacy-sfc + # Link to the generated files + ln -sf $(TARGET_SBT_DIR)/target/generated-src/$(long_name)/$(long_name).sfc.fir $(FIRRTL_FILE) + ln -sf $(TARGET_SBT_DIR)/target/generated-src/$(long_name)/$(long_name).anno.json $(ANNO_FILE) + # .d needed to run metasim CI tests + ln -sf $(TARGET_SBT_DIR)/target/generated-src/$(long_name)/$(long_name).d $(GENERATED_DIR)/$(long_name).d + +####################################### +# Setup Extra Verilator Compile Flags # +####################################### + +## default flags added for cva6 +CVA6_VERILATOR_FLAGS = \ + --unroll-count 256 \ + -Werror-PINMISSING \ + -Werror-IMPLICIT \ + -Wno-fatal \ + -Wno-PINCONNECTEMPTY \ + -Wno-ASSIGNDLY \ + -Wno-DECLFILENAME \ + -Wno-UNUSED \ + -Wno-UNOPTFLAT \ + -Wno-BLKANDNBLK \ + -Wno-style \ + -Wall + +# normal flags used for midas builds (that are incompatible with cva6) +DEFAULT_MIDAS_VERILATOR_FLAGS = \ + --assert + +# AJG: this must be evaluated after verilog generation to work (hence the =) +EXTRA_VERILATOR_FLAGS = \ + $(shell if ! grep -iq "module.*cva6" $(simulator_verilog); then echo "$(DEFAULT_MIDAS_VERILATOR_FLAGS)"; else echo "$(CVA6_VERILATOR_FLAGS)"; fi) diff --git a/bbdev/api/steps/firesim/scripts/makefrag/firesim/config.mk b/bbdev/api/steps/firesim/scripts/makefrag/firesim/config.mk new file mode 100644 index 000000000..01ec696ff --- /dev/null +++ b/bbdev/api/steps/firesim/scripts/makefrag/firesim/config.mk @@ -0,0 +1,35 @@ +# Custom configuration for Buckyball FireSim builds +# This file overrides the default TARGET_CONFIG_PACKAGE to use our custom configs + +# Only used in this projects makefrags +makefile_path := $(abspath $(lastword $(MAKEFILE_LIST))) +makefile_dir := $(patsubst %/,%,$(dir $(makefile_path))) +# makefile_dir = .../bbdev/api/steps/firesim/scripts/makefrag/firesim/ +# Need to go up 7 levels to reach buckyball root +chipyard_dir := $(abspath $(makefile_dir)/../../../../../../../arch/thirdparty/chipyard) + +# These point at the main class of the target's Chisel generator +DESIGN_PACKAGE ?= firechip.design +DESIGN ?= FireSim + +# Override to use our custom config package +TARGET_CONFIG_PACKAGE ?= sims.firesim +ifndef TARGET_CONFIG +$(error TARGET_CONFIG must name an available FireSim target) +endif + +# These guide chisel elaboration of simulation components by MIDAS, +# including models and widgets. +PLATFORM_CONFIG_PACKAGE ?= firesim.firesim +PLATFORM_CONFIG ?= BaseF1Config + +# Override project for the target. +TARGET_SBT_PROJECT := buckyball + +# Point to our project directory +TARGET_SBT_DIR := $(abspath $(makefile_dir)/../../../../../../../arch) +TARGET_MILL_DIR := $(TARGET_SBT_DIR) +TARGET_SOURCE_DIRS := $(abspath $(makefile_dir)/../../../../../../../arch/src/main/scala) + +# SBT launcher +SBT ?= java -jar $(chipyard_dir)/scripts/sbt-launch.jar $(SBT_OPTS) diff --git a/bbdev/api/steps/firesim/scripts/makefrag/firesim/driver.mk b/bbdev/api/steps/firesim/scripts/makefrag/firesim/driver.mk new file mode 100644 index 000000000..a9a0d9c60 --- /dev/null +++ b/bbdev/api/steps/firesim/scripts/makefrag/firesim/driver.mk @@ -0,0 +1,70 @@ +# See LICENSE for license details. + +# DOC include start: Bridge Build System Changes +########################## +# Driver Sources & Flags # +########################## + +ifeq (,$(wildcard $(RISCV)/lib/libriscv.so)) +$(warning libriscv not found) +LRISCV= +else +LRISCV=-lriscv +endif + +firechip_lib_dir = $(chipyard_dir)/generators/firechip/design/src/main/cc +firechip_bridgestubs_lib_dir = $(chipyard_dir)/generators/firechip/bridgestubs/src/main/cc +testchipip_csrc_dir = $(chipyard_dir)/generators/testchipip/src/main/resources/testchipip/csrc + +# DRIVER_H only used to update recipe pre-reqs (ok to track more files) + +# fesvr and related srcs +DRIVER_H += \ + $(shell find $(testchipip_csrc_dir) -name "*.h") \ + $(shell find $(firechip_bridgestubs_lib_dir)/fesvr -name "*.h") +DRIVER_CC += \ + $(testchipip_csrc_dir)/cospike_impl.cc \ + $(testchipip_csrc_dir)/testchip_tsi.cc \ + $(testchipip_csrc_dir)/testchip_dtm.cc \ + $(testchipip_csrc_dir)/testchip_htif.cc \ + $(firechip_bridgestubs_lib_dir)/fesvr/firesim_tsi.cc \ + $(firechip_bridgestubs_lib_dir)/fesvr/firesim_dtm.cc \ + $(RISCV)/lib/libfesvr.a +# Disable missing override warning for testchipip. +TARGET_CXX_FLAGS += \ + -isystem $(testchipip_csrc_dir) \ + -isystem $(RISCV)/include \ + -Wno-inconsistent-missing-override +TARGET_LD_FLAGS += \ + -L$(RISCV)/lib \ + -Wl,-rpath,$(RISCV)/lib \ + $(LRISCV) + +# top-level sources +DRIVER_CC += $(addprefix $(firechip_lib_dir)/firesim/, $(addsuffix .cc, firesim_top)) +TARGET_CXX_FLAGS += -I$(firechip_bridgestubs_lib_dir)/bridge/test + +# bridge sources +DRIVER_H += $(shell find $(firechip_bridgestubs_lib_dir) -name "*.h") +DRIVER_CC += \ + $(wildcard \ + $(addprefix \ + $(firechip_bridgestubs_lib_dir)/, \ + $(addsuffix .cc,bridges/* bridges/tracerv/* bridges/cospike/*) \ + ) \ + ) +TARGET_CXX_FLAGS += \ + -I$(firechip_bridgestubs_lib_dir) \ + -I$(firechip_bridgestubs_lib_dir)/bridge \ + -I$(firechip_bridgestubs_lib_dir)/bridge/tracerv \ + -I$(firechip_bridgestubs_lib_dir)/bridge/cospike +TARGET_LD_FLAGS += \ + -l:libdwarf.so -l:libelf.so \ + -lz \ + +# other +TARGET_CXX_FLAGS += \ + -I$(GENERATED_DIR) \ + -g + +# DOC include end: Bridge Build System Changes diff --git a/bbdev/api/steps/firesim/scripts/makefrag/firesim/metasim.mk b/bbdev/api/steps/firesim/scripts/makefrag/firesim/metasim.mk new file mode 100644 index 000000000..1e7c50a35 --- /dev/null +++ b/bbdev/api/steps/firesim/scripts/makefrag/firesim/metasim.mk @@ -0,0 +1,140 @@ +# See LICENSE for license details. + +################################################################ +# SW RTL Simulation Args -- for MIDAS- & FPGA-level Simulation # +################################################################ +TIMEOUT_CYCLES = 100000000 + +NET_SLOT ?= 0 +NET_LINK_LATENCY ?= 6405 +NET_BW ?= 100 +NET_SHMEMPORTNAME ?= $(shell printf '%0100d' $(NET_SLOT)) +NET_LOOPBACK ?= +nic-loopback0 +NET_MACADDR ?= $(shell printf '00:00:00:00:00:%02x' $$(($(NET_SLOT)+2))) +nic_args = +shmemportname0=$(NET_SHMEMPORTNAME) +macaddr0=$(NET_MACADDR) \ + +niclog0=niclog$(NET_SLOT) +linklatency0=$(NET_LINK_LATENCY) \ + +netbw0=$(NET_BW) +netburst0=8 $(NET_LOOPBACK) +tracer_args = +tracefile=TRACEFILE +blkdev_args = +blkdev-in-mem0=128 +blkdev-log0=blkdev-log$(NET_SLOT) +autocounter_args = +autocounter-readrate=1000 +autocounter-filename-base=AUTOCOUNTERFILE +# Neglecting this +arg will make the simulator use the same step size as on the +# FPGA. This will make ML simulation more closely match results seen on the +# FPGA at the expense of dramatically increased target runtime +serial_args = +fesvr-step-size=128 +#serial_args = + +COMMON_SIM_ARGS := $(COMMON_SIM_ARGS) $(serial_args) $(nic_args) $(tracer_args) $(blkdev_args) $(autocounter_args) + +# Arguments used only at a particular simulation abstraction +MIDAS_LEVEL_SIM_ARGS ?= +max-cycles=$(TIMEOUT_CYCLES) +FPGA_LEVEL_SIM_ARGS ?= + +################################ +# Verilator/VCS/XSIM execution # +################################ + +verilator = $(GENERATED_DIR)/V$(DESIGN) +verilator_debug = $(GENERATED_DIR)/V$(DESIGN)-debug +verilator_args = +vcs = $(GENERATED_DIR)/$(DESIGN) +vcs_debug = $(GENERATED_DIR)/$(DESIGN)-debug +vcs_args = +vcs+initreg+0 +vcs+initmem+0 +xsim = $(GENERATED_DIR)/$(DESIGN)-$(PLATFORM) +xcelium = $(GENERATED_DIR)/X$(DESIGN) +sim_binary_basename := $(basename $(notdir $(SIM_BINARY))) + +run-verilator: $(verilator) + cd $(dir $<) && \ + $(verilator) +permissive $(verilator_args) $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) +permissive-off $(abspath $(SIM_BINARY)) +which_disasm := $(shell which spike-dasm 2> /dev/null) +ifneq ($(which_disasm),) + disasm := 3>&1 1>&2 2>&3 | $(which_disasm) $(DISASM_EXTENSION) > +endif + +# Some of the generated suites use specific plus args, that are prefixed with +# the binary name. These are captured with $($*_ARGS) +$(OUTPUT_DIR)/%.run: $(OUTPUT_DIR)/% $(EMUL) + cd $(dir $($(EMUL))) && \ + ./$(notdir $($(EMUL))) $< $($*_ARGS) $($(EMUL)_args) $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) \ + 2> /dev/null 2> $@ && [ $$PIPESTATUS -eq 0 ] + +$(OUTPUT_DIR)/%.out: $(OUTPUT_DIR)/% $(EMUL) + cd $(dir $($(EMUL))) && \ + ./$(notdir $($(EMUL))) $< $($*_ARGS) $($(EMUL)_args) $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) \ + $(disasm) $@ && [ $$PIPESTATUS -eq 0 ] + +$(OUTPUT_DIR)/%.vpd: $(OUTPUT_DIR)/% $(EMUL)-debug + cd $(dir $($(EMUL)_debug)) && \ + ./$(notdir $($(EMUL)_debug)) $< +vcdplusfile=$@ $($*_ARGS) $($(EMUL)_args) $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) \ + $(disasm) $(patsubst %.vpd,%.out,$@) && [ $$PIPESTATUS -eq 0 ] + +$(OUTPUT_DIR)/%.fsdb: $(OUTPUT_DIR)/% $(EMUL)-debug + cd $(dir $($(EMUL)_debug)) && \ + ./$(notdir $($(EMUL)_debug)) $< +fsdbfile=$@ $($*_ARGS) $($(EMUL)_args) $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) \ + $(disasm) $(patsubst %.fsdb,%.out,$@) && [ $$PIPESTATUS -eq 0 ] + +.PRECIOUS: $(OUTPUT_DIR)/%.fsdb $(OUTPUT_DIR)/%.vpd $(OUTPUT_DIR)/%.out $(OUTPUT_DIR)/%.run + +# TraceGen rules + +$(OUTPUT_DIR)/tracegen.out: $($(EMUL)) + mkdir -p $(OUTPUT_DIR) && \ + cd $(dir $($(EMUL))) && \ + ./$(notdir $($(EMUL))) $($(EMUL)_args) $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) \ + 2> /dev/null 2> $@ && [ $$PIPESTATUS -eq 0 ] + +$(OUTPUT_DIR)/tracegen.result: $(OUTPUT_DIR)/tracegen.out $(AXE) + $(chipyard_dir)/scripts/check-tracegen.sh $< > $@ + +fsim-tracegen: $(OUTPUT_DIR)/tracegen.result + +.PHONY: fsim-tracegen diff --git a/bbdev/api/steps/ip/01_generate_api.step.py b/bbdev/api/steps/ip/01_generate_api.step.py new file mode 100644 index 000000000..849cde514 --- /dev/null +++ b/bbdev/api/steps/ip/01_generate_api.step.py @@ -0,0 +1,58 @@ +from utils.event_common import require_target +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.event_common import require_design + + +def req_arg(body: dict, name: str): + return body.get(name) or body.get(name.replace("_", "-")) + + +config = { + "name": "ip-generate-api", + "description": "generate SRAM macro wrappers from elaborator mems.conf via the process sram function", + "flows": ["ip", "dc", "yosys"], + "triggers": [api("POST", "/ip/generate")], + "enqueues": ["ip.generate"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = req.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) + + data = { + "design": design, + "consumer": req_arg(body, "consumer") or "dc", + "top": req_arg(body, "top"), + } + if data["consumer"] not in ("dc", "yosys"): + return ApiResponse(status=400, body={"error": "consumer must be dc or yosys"}) + output_dir = req_arg(body, "output_dir") + if output_dir is not None: + if not isinstance(output_dir, str) or not output_dir: + return ApiResponse( + status=400, body={"error": "output_dir must be a non-empty string"} + ) + data["output_dir"] = output_dir + next_topic = req_arg(body, "next_topic") + if next_topic is not None: + if not isinstance(next_topic, str) or not next_topic: + return ApiResponse( + status=400, body={"error": "next_topic must be a non-empty string"} + ) + data["next_topic"] = next_topic + await ctx.enqueue( + { + "topic": "ip.generate", + "data": {**data, "_trace_id": ctx.trace_id, "target": selection}, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/ip/01_generate_event.step.py b/bbdev/api/steps/ip/01_generate_event.step.py new file mode 100644 index 000000000..10487f1aa --- /dev/null +++ b/bbdev/api/steps/ip/01_generate_event.step.py @@ -0,0 +1,103 @@ +import os +import sys +from pathlib import Path + +import yaml + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) + +from utils.event_common import check_result, get_origin_trace_id, require_design +from utils.path import get_buckyball_path, rtl_dir +from steps.dc.scripts.tapeout import get_tapeout_contract +from generate import generate_sram + +config = { + "name": "ip-generate", + "description": "generate SRAM macro wrappers from elaborator mems.conf via the process sram function", + "flows": ["ip", "dc", "yosys"], + "triggers": [queue("ip.generate")], + "enqueues": ["ip.replace"], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + try: + design = require_design(input_data) + consumer = input_data.get("consumer") or "dc" + if consumer not in ("dc", "yosys"): + raise ValueError("consumer must be dc or yosys") + # The synthesis top: --top, else the design's tapeout config (dc) or the yosys config. + top = input_data.get("top") or ( + get_tapeout_contract( + get_buckyball_path(), design, None, selection=input_data["target"] + ).top_module + if consumer == "dc" + else yaml.safe_load( + ( + Path(__file__).parents[1] / "yosys/scripts/yosys-config.yaml" + ).read_text() + )["top"] + ) + if not isinstance(top, str) or not top: + raise ValueError("top must be a non-empty string") + bbdir = Path(get_buckyball_path()).resolve() + build_dir = Path( + rtl_dir( + bbdir, + design, + "dc" if consumer == "dc" else "synth", + input_data.get("output_dir"), + selection=input_data["target"], + ) + ).resolve() + man = generate_sram( + bbdir=bbdir, + design=design, + selection=input_data["target"], + build_dir=build_dir, + top_module=top, + source_list=build_dir / f"{consumer}_sources.list", + ) + except Exception as exc: + _, failure_result = await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "ip.generate", "error": str(exc)}, + trace_id=origin_tid, + ) + return failure_result + + extra = { + "task": "ip.generate", + "design": man["design"], + "generate_manifest": man["generate_manifest"], + "sram_macros_v": man["sram_macros_v"], + "mem_count": man["mem_count"], + "build_dir": str(build_dir), + } + await check_result( + ctx, 0, continue_run=True, extra_fields=extra, trace_id=origin_tid + ) + await ctx.enqueue( + { + "topic": "ip.replace", + "data": { + **input_data, + "design": design, + "consumer": consumer, + "top": top, + "_trace_id": origin_tid, + "target": input_data["target"], + }, + } + ) diff --git a/bbdev/api/steps/ip/02_replace_api.step.py b/bbdev/api/steps/ip/02_replace_api.step.py new file mode 100644 index 000000000..b3196aadf --- /dev/null +++ b/bbdev/api/steps/ip/02_replace_api.step.py @@ -0,0 +1,58 @@ +from utils.event_common import require_target +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.event_common import require_design + + +def req_arg(body: dict, name: str): + return body.get(name) or body.get(name.replace("_", "-")) + + +config = { + "name": "ip-replace-api", + "description": "assemble top-scoped synthesis RTL plus generated SRAM macros", + "flows": ["ip", "dc", "yosys"], + "triggers": [api("POST", "/ip/replace")], + "enqueues": ["ip.replace"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = req.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) + + data = { + "design": design, + "consumer": req_arg(body, "consumer") or "dc", + "top": req_arg(body, "top"), + } + if data["consumer"] not in ("dc", "yosys"): + return ApiResponse(status=400, body={"error": "consumer must be dc or yosys"}) + output_dir = req_arg(body, "output_dir") + if output_dir is not None: + if not isinstance(output_dir, str) or not output_dir: + return ApiResponse( + status=400, body={"error": "output_dir must be a non-empty string"} + ) + data["output_dir"] = output_dir + next_topic = req_arg(body, "next_topic") + if next_topic is not None: + if not isinstance(next_topic, str) or not next_topic: + return ApiResponse( + status=400, body={"error": "next_topic must be a non-empty string"} + ) + data["next_topic"] = next_topic + await ctx.enqueue( + { + "topic": "ip.replace", + "data": {**data, "_trace_id": ctx.trace_id, "target": selection}, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/ip/02_replace_event.step.py b/bbdev/api/steps/ip/02_replace_event.step.py new file mode 100644 index 000000000..484b1d366 --- /dev/null +++ b/bbdev/api/steps/ip/02_replace_event.step.py @@ -0,0 +1,147 @@ +import os +import sys +from pathlib import Path + +import yaml + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) + +from utils.event_common import check_result, get_origin_trace_id, require_design +from utils.path import get_buckyball_path, rtl_dir +from steps.dc.scripts.tapeout import get_tapeout_contract +from replace import replace_sources + +config = { + "name": "ip-replace", + "description": "assemble top-scoped synthesis RTL plus generated SRAM macros", + "flows": ["ip", "dc", "yosys"], + "triggers": [queue("ip.replace")], + "enqueues": ["dc.area", "yosys.synth"], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + try: + design = require_design(input_data) + consumer = input_data.get("consumer") or "dc" + if consumer not in {"dc", "yosys"}: + raise ValueError("consumer must be dc or yosys") + # The synthesis top: --top, else the design's tapeout config (dc) or the yosys config. + top_module = input_data.get("top") or ( + get_tapeout_contract( + get_buckyball_path(), design, None, selection=input_data["target"] + ).top_module + if consumer == "dc" + else yaml.safe_load( + ( + Path(__file__).parents[1] / "yosys/scripts/yosys-config.yaml" + ).read_text() + )["top"] + ) + if not isinstance(top_module, str) or not top_module: + raise ValueError("top must be a non-empty string") + + bbdir = Path(get_buckyball_path()).resolve() + build_dir = Path( + rtl_dir( + bbdir, + design, + "dc" if consumer == "dc" else "synth", + input_data.get("output_dir"), + selection=input_data["target"], + ) + ).resolve() + source_list_path = build_dir / f"{consumer}_sources.list" + if not source_list_path.is_file(): + raise FileNotFoundError(f"missing source list: {source_list_path}") + sources = [ + line.strip() + for line in source_list_path.read_text().splitlines() + if line.strip() + ] + if not sources: + raise ValueError(f"empty source list: {source_list_path}") + + gen_dir = build_dir / "ip-generate" + sram_macros_v = gen_dir / "sram_macros.v" + generate_manifest = gen_dir / "generate_manifest.json" + if not generate_manifest.is_file(): + raise FileNotFoundError(f"missing generate_manifest: {generate_manifest}") + + result = replace_sources( + source_paths=sources, + top_module=top_module, + sram_macros_v=sram_macros_v, + output_dir=build_dir / "ip-replace", + consumer=consumer, + generate_manifest=generate_manifest, + ) + except Exception as exc: + _, failure_result = await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "ip.replace", "error": str(exc)}, + trace_id=origin_tid, + ) + return failure_result + + extra = { + "task": "ip.replace", + "consumer": consumer, + "source_list": result["source_list"], + "replace_manifest": result["replace_manifest"], + "source_count": result["source_count"], + "top_module": top_module, + "build_dir": str(build_dir), + } + if generate_manifest.is_file(): + extra["generate_manifest"] = str(generate_manifest) + + next_topic = input_data.get("next_topic") + if next_topic is not None: + if not isinstance(next_topic, str) or not next_topic: + _, failure_result = await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "task": "ip.replace", + "error": "next_topic must be a non-empty string", + }, + trace_id=origin_tid, + ) + return failure_result + await check_result( + ctx, 0, continue_run=True, extra_fields=extra, trace_id=origin_tid + ) + await ctx.enqueue( + { + "topic": next_topic, + "data": { + **input_data, + "design": design, + "consumer": consumer, + "top": top_module, + "source_list": result["source_list"], + "replace_manifest": result["replace_manifest"], + "generate_manifest": str(generate_manifest), + "_trace_id": origin_tid, + "target": input_data["target"], + }, + } + ) + return + + await check_result( + ctx, 0, continue_run=False, extra_fields=extra, trace_id=origin_tid + ) diff --git a/bbdev/api/steps/ip/scripts/generate.py b/bbdev/api/steps/ip/scripts/generate.py new file mode 100644 index 000000000..d83939046 --- /dev/null +++ b/bbdev/api/steps/ip/scripts/generate.py @@ -0,0 +1,195 @@ +from __future__ import annotations + +import json +import sys +from pathlib import Path + +if str(Path(__file__).resolve().parents[2] / "dc" / "scripts") not in sys.path: + sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "dc" / "scripts")) + +from tapeout import SramGeom, get_tapeout_contract +from replace import source_scope +from sram_compiler import generate_sram_dbs, generate_sram_macros + + +def load_mdf(mdf: Path) -> dict[str, tuple[int, int]]: + raw = json.loads(mdf.read_text()) + if not isinstance(raw, list): + raise ValueError(f"mdf must be a list: {mdf}") + index: dict[str, tuple[int, int]] = {} + for entry in raw: + if not isinstance(entry, dict): + raise ValueError(f"mdf entry must be object: {mdf}") + name = entry.get("name") + if not isinstance(name, str) or not name: + raise ValueError(f"mdf entry missing name: {mdf}") + if name in index: + raise ValueError(f"duplicate mdf name {name}: {mdf}") + try: + depth = int(entry["depth"]) + width = int(entry["width"]) + except (KeyError, TypeError, ValueError) as exc: + raise ValueError(f"mdf entry {name} missing depth/width") from exc + if depth <= 0 or width <= 0: + raise ValueError(f"mdf entry {name}: depth/width must be positive") + index[name] = (depth, width) + return index + + +def geoms_from_names( + *, + names: list[str], + sram_table: dict[str, SramGeom], + mdf: Path, +) -> list[SramGeom]: + index = load_mdf(mdf) + geoms: list[SramGeom] = [] + for name in names: + if name not in sram_table: + raise ValueError(f"sram leaf {name} missing from tapeout.sram") + if name not in index: + raise ValueError(f"sram leaf {name} missing from mdf: {mdf}") + depth, width = index[name] + geom = sram_table[name] + if depth != geom.words: + raise ValueError( + f"sram leaf {name}: mdf depth {depth} != tapeout.sram words {geom.words}" + ) + if width != geom.bits: + raise ValueError( + f"sram leaf {name}: mdf width {width} != tapeout.sram bits {geom.bits}" + ) + geoms.append(geom) + return geoms + + +def check_bitwrite_contract( + geoms: list[SramGeom], mdf: Path, sram_table: dict[str, SramGeom] +) -> None: + raw = json.loads(mdf.read_text()) + if not isinstance(raw, list): + raise ValueError(f"mdf must be a list: {mdf}") + by_name: dict[str, dict] = {} + for entry in raw: + if not isinstance(entry, dict): + raise ValueError(f"mdf entry must be object: {mdf}") + name = entry.get("name") + if not isinstance(name, str) or not name: + raise ValueError(f"mdf entry missing name: {mdf}") + if name in by_name: + raise ValueError(f"duplicate mdf name {name}: {mdf}") + by_name[name] = entry + for g in geoms: + if g.name not in by_name: + raise ValueError(f"sram leaf {g.name} missing from mdf: {mdf}") + ports = by_name[g.name].get("ports") + if not isinstance(ports, list) or len(ports) != 1: + raise ValueError(f"{g.name}: expected exactly 1 mdf port") + p = ports[0] + if not isinstance(p, dict): + raise ValueError(f"{g.name}: mdf port must be object") + mask_name = p.get("mask port name") + if g.bitwrite: + if mask_name != "BWEN": + raise ValueError( + f"{g.name}: bitwrite requires mask port BWEN, got {mask_name!r}" + ) + if p.get("mask port polarity") != "active low": + raise ValueError(f"{g.name}: BWEN polarity must be active low") + if p.get("mask granularity") != 1: + raise ValueError(f"{g.name}: BWEN mask granularity must be 1") + elif mask_name is not None: + raise ValueError(f"{g.name}: mask port {mask_name!r} but bitwrite is false") + used = {g.name for g in geoms} + for geom in sram_table.values(): + if geom.bitwrite and geom.name not in used: + raise RuntimeError( + f"bitwrite sram {geom.name} not instantiated by any memory" + ) + + +def _leaf_paths(cache_dir: Path, g: SramGeom, corner: str) -> dict[str, str]: + name = g.name + leaf = cache_dir / f"{name}-mux{g.mux}" + return { + "v": str((leaf / f"{name}.v").resolve()), + "lib": str((leaf / f"{name}_{corner}.lib").resolve()), + "db": str((leaf / f"{name}_{corner}.db").resolve()), + } + + +def generate_sram( + *, + bbdir: str | Path, + design: str, + selection: str, + build_dir: str | Path, + top_module: str, + source_list: str | Path, + out_dir: str | Path | None = None, +) -> dict: + bbdir = Path(bbdir).resolve() + build_dir = Path(build_dir).resolve() + src = build_dir / "mems.conf" + if not src.is_file(): + raise FileNotFoundError(f"missing elaborator mems.conf: {src}") + mems = src.read_text() + reachable, _ = source_scope(Path(source_list).read_text().splitlines(), top_module) + selected = "".join( + line + "\n" + for line in mems.splitlines() + if line.strip() and line.split()[1] in reachable + ) + contract = get_tapeout_contract(bbdir, design, selection=selection) + if out_dir is None: + dest = build_dir / "ip-generate" + else: + if isinstance(out_dir, str) and not out_dir.strip(): + raise ValueError("empty out_dir") + dest = Path(out_dir).resolve() + dest.mkdir(parents=True, exist_ok=True) + mems_out = dest / "mems.conf" + verilog = dest / "sram_macros.v" + mems_out.write_text(selected) + if selected: + text, names = generate_sram_macros( + process=contract.sram_process, mems=selected, mdf=contract.sram_mdf + ) + verilog.write_text(text) + geoms = geoms_from_names( + names=names, + sram_table=contract.sram_table, + mdf=contract.sram_mdf, + ) + check_bitwrite_contract(geoms, contract.sram_mdf, contract.sram_table) + else: + verilog.write_text("") + geoms = [] + ip_db = build_dir.parent / f"{build_dir.name}-ip-db" + if geoms: + db_paths, corner_tag = generate_sram_dbs( + geoms=geoms, + process=contract.sram_process, + corner=contract.sram_corner, + cache_dir=ip_db, + lc_shell=contract.lc_shell, + ) + else: + db_paths, corner_tag = [], contract.sram_corner + names = [g.name for g in geoms] + man = { + "design": design, + "sram_mdf": str(contract.sram_mdf), + "mems_conf": str(mems_out), + "sram_macros_v": str(verilog), + "sram_corner": corner_tag, + "ip_db": str(ip_db), + "leaves": names, + "leaf_paths": {g.name: _leaf_paths(ip_db, g, corner_tag) for g in geoms}, + "link_dbs": [str(p) for p in db_paths], + "leaf_count": len(names), + "mem_count": len(selected.splitlines()), + } + manifest_path = dest / "generate_manifest.json" + manifest_path.write_text(json.dumps(man, indent=2) + "\n") + return {**man, "generate_manifest": str(manifest_path)} diff --git a/bbdev/api/steps/ip/scripts/replace.py b/bbdev/api/steps/ip/scripts/replace.py new file mode 100644 index 000000000..8eb667023 --- /dev/null +++ b/bbdev/api/steps/ip/scripts/replace.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import json +import re +from pathlib import Path + + +def source_scope( + source_paths: list[str | Path], top_module: str +) -> tuple[set[str], set[Path]]: + paths = [Path(path).resolve() for path in source_paths] + modules: dict[str, Path] = {} + texts: dict[str, str] = {} + for path in paths: + text = re.sub(r"/\*.*?\*/|//[^\n]*", "", path.read_text(), flags=re.S) + matches = list( + re.finditer( + r"^\s*(?:\(\*.*?\*\)\s*)*module\s+([A-Za-z_][A-Za-z0-9_$]*)\b", + text, + re.M, + ) + ) + for match in matches: + name = match.group(1) + if name in modules: + raise RuntimeError( + f"duplicate module {name}: {modules[name]} vs {path}" + ) + modules[name] = path + end = re.search(r"\bendmodule\b", text[match.end() :]).end() + match.end() + texts[name] = text[match.end() : end] + + if top_module not in modules: + raise ValueError(f"synthesis top is not defined in source list: {top_module}") + + reachable = {top_module} + pending = [top_module] + while pending: + cur = pending.pop() + for child, _inst in re.findall( + r"\b([A-Za-z_][A-Za-z0-9_$]*)\s*(?:#\s*\(.*?\)\s*)?([A-Za-z_][A-Za-z0-9_$]*)\s*\(", + texts[cur], + re.M | re.S, + ): + # Undefined children (memory black boxes) stay in the set for mems.conf selection. + if child not in reachable: + reachable.add(child) + if child in modules: + pending.append(child) + + return reachable, {modules[name] for name in reachable if name in modules} + + +def replace_sources( + *, + source_paths: list[str], + top_module: str, + sram_macros_v: str | Path, + output_dir: str | Path, + consumer: str, + generate_manifest: str | Path | None = None, +) -> dict: + paths = [Path(path).resolve() for path in source_paths] + macros = Path(sram_macros_v).resolve() + + if not macros.is_file(): + raise FileNotFoundError(f"missing sram_macros.v: {macros}") + reachable, selected_paths = source_scope(paths, top_module) + selected = [path for path in paths if path in selected_paths] + if macros not in selected: + selected.append(macros) + + out = Path(output_dir) + out.mkdir(parents=True, exist_ok=True) + source_list_path = out / f"{consumer}_sources.list" + source_list_path.write_text("\n".join(str(path) for path in selected) + "\n") + + manifest = { + "top_module": top_module, + "consumer": consumer, + "source_count": len(selected), + "sram_macros_v": str(macros), + } + if generate_manifest is not None: + gman = Path(generate_manifest).resolve() + if not gman.is_file(): + raise FileNotFoundError(f"missing generate_manifest: {gman}") + manifest["generate_manifest"] = str(gman) + manifest_path = out / "replace_manifest.json" + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n") + + return { + "source_list": str(source_list_path), + "replace_manifest": str(manifest_path), + "source_count": len(selected), + } diff --git a/bbdev/api/steps/ip/scripts/sram_compiler.py b/bbdev/api/steps/ip/scripts/sram_compiler.py new file mode 100644 index 000000000..0b1ff1e7f --- /dev/null +++ b/bbdev/api/steps/ip/scripts/sram_compiler.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +for path in ( + Path(__file__).resolve().parents[2], + Path(__file__).resolve().parents[5] / "thirdparty" / "soc-framework" / "ip", +): + if str(path) not in sys.path: + sys.path.insert(0, str(path)) + +from smic180.compiler import generate_smic180_sram_dbs +from smic180.sram import smic180_sram + + +def generate_sram_macros( + *, process: str, mems: str, mdf: Path +) -> tuple[str, list[str]]: + if process == "smic180": + return smic180_sram(mems, mdf) + raise ValueError(f"unknown sram process {process!r}") + + +def generate_sram_dbs( + *, + geoms: list, + process: str, + corner: str, + cache_dir: Path, + lc_shell: Path, +) -> tuple[list[Path], str]: + if process == "smic180": + return generate_smic180_sram_dbs(geoms, corner, cache_dir, lc_shell) + elif process == "tsmc28": + raise ValueError("tsmc28 sram is not supported yet") + raise ValueError(f"unknown sram process {process!r}") diff --git a/bbdev/api/steps/kernel/01_build_api.step.py b/bbdev/api/steps/kernel/01_build_api.step.py new file mode 100644 index 000000000..d5e7b016b --- /dev/null +++ b/bbdev/api/steps/kernel/01_build_api.step.py @@ -0,0 +1,31 @@ +from utils.event_common import require_target +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.event_common import require_design + +config = { + "name": "kernel-build-api", + "description": "build RISC-V kernel + rootfs for Pegasus", + "flows": ["kernel"], + "triggers": [api("POST", "/kernel/build")], + "enqueues": ["kernel.build"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + try: + selection = require_target(body, ("device",)) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + require_design(body) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + await ctx.enqueue( + { + "topic": "kernel.build", + "data": {**body, "_trace_id": ctx.trace_id, "target": selection}, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/kernel/01_build_event.step.py b/bbdev/api/steps/kernel/01_build_event.step.py new file mode 100644 index 000000000..a23b2b6e4 --- /dev/null +++ b/bbdev/api/steps/kernel/01_build_event.step.py @@ -0,0 +1,139 @@ +from utils.path import design_resource_dir +import os +import shlex +import shutil +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import design_output_root, get_buckyball_path, load_design_config +from utils.stream_run import stream_run_logger_async +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "kernel-build", + "description": "build RISC-V kernel + rootfs for image via stack/os/linux", + "flows": ["kernel"], + "triggers": [queue("kernel.build")], + "enqueues": [], +} + + +def kernel_build_dir( + bbdir: str, + hart_count: int, + design: str, + selection: str, + interactive: bool, + guest_memory_mib: int, +) -> str: + suffix = "" if guest_memory_mib == 512 else f"-mem{guest_memory_mib}M" + suffix += f"-design-{design}-{selection.replace('/', '-')}" + if interactive: + suffix += "-interactive" + name = "kernel" if hart_count == 64 else f"kernel-h{hart_count}" + return os.path.join(bbdir, "bb-tests", "build", f"{name}{suffix}") + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + + kernel_src = os.path.join(bbdir, "stack", "os", "linux") + design = input_data["design"] + design_config = load_design_config(bbdir, design, selection=input_data["target"]) + hart_count = len(design_config.device.cores) + ddr = design_config.device.ddr + guest_memory_mib = (ddr.bytes - ddr.model) // (1024 * 1024) + # A DDR without a model region leaves the base empty: init runs no model. + model_base = f"{ddr.base + ddr.bytes - ddr.model:#x}" if ddr.model else "" + interactive = input_data.get("interactive", False) + output_root = design_output_root(bbdir, design, selection=input_data["target"]) + output_dir = os.path.join(output_root, "workloads/kernel") + os.makedirs(output_dir, exist_ok=True) + kernel_build = kernel_build_dir( + bbdir, hart_count, design, input_data["target"], interactive, guest_memory_mib + ) + interactive_arg = "ON" if interactive else "OFF" + programs = design_resource_dir( + bbdir, design, "kernel", selection=input_data["target"] + ) + + # cmake configure + configure_cmd = ( + f"cmake -B {kernel_build} -S {kernel_src} " + f"-DBUCKYBALL_HART_COUNT={hart_count} " + f"-DBUCKYBALL_GUEST_MEMORY_MIB={guest_memory_mib} " + f"-DBUCKYBALL_KERNEL_DESIGN={design} " + f"-DBUCKYBALL_KERNEL_OUTPUT={output_root} " + f"-DBUCKYBALL_KERNEL_INTERACTIVE={interactive_arg} " + f"-DBUCKYBALL_MODEL_BASE={model_base} " + f"-DBUCKYBALL_KERNEL_WORKLOADS={programs}" + ) + + result = await stream_run_logger_async( + cmd=configure_cmd, + logger=ctx.logger, + stdout_prefix="marshal build", + stderr_prefix="marshal build", + ) + if result.returncode != 0: + await check_result( + ctx, result.returncode, continue_run=False, trace_id=origin_tid + ) + return + + # cmake build + build_cmd = f"cmake --build {kernel_build} --target kernel-build" + result = await stream_run_logger_async( + cmd=build_cmd, + logger=ctx.logger, + stdout_prefix="marshal build", + stderr_prefix="marshal build", + ) + + if result.returncode != 0: + await check_result( + ctx, result.returncode, continue_run=False, trace_id=origin_tid + ) + return + + # The output directory belongs to one device config, so the name is fixed. + payload_name = "fw_payload-linux" + fw_payload_bin = os.path.join(output_dir, f"{payload_name}.bin") + fw_payload_elf = os.path.join(output_dir, f"{payload_name}.elf") + built_payload_elf = os.path.join( + kernel_build, + "opensbi", + "platform", + "buckyball", + "firmware", + "fw_payload.elf", + ) + + shutil.copy2(built_payload_elf, fw_payload_elf) + + # The DDR image: the kernel and, when given, the model, each at its place. + # BEMU takes the bin, P2E the hex. + image = os.path.join(output_dir, f"{payload_name}-ddr") + pack = os.path.join(kernel_build, "image-pack") + model = input_data.get("model") + pack_cmd = [pack, f"{image}.bin", f"{image}.hex", str(ddr.bytes), fw_payload_bin] + if model: + pack_cmd += [ + os.path.join( + bbdir, "stack/models/archs/buckyball", design, model, "output/model.img" + ), + str(ddr.bytes - ddr.model), + ] + result = await stream_run_logger_async( + cmd=shlex.join(pack_cmd), + logger=ctx.logger, + stdout_prefix="image pack", + stderr_prefix="image pack", + ) + await check_result(ctx, result.returncode, continue_run=False, trace_id=origin_tid) diff --git a/bbdev/api/steps/kernel/scripts/elf2hex.py b/bbdev/api/steps/kernel/scripts/elf2hex.py new file mode 100755 index 000000000..e9441e77f --- /dev/null +++ b/bbdev/api/steps/kernel/scripts/elf2hex.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""ELF -> hex converter for Buckyball P2E DDR loading. + +Output format: + @0 + XX + XX + ... +where XX is one byte per line (uppercase hex). + +The hex file represents bytes starting at DDR offset 0, which maps to +CPU address 0x80000000. All LOAD segments are concatenated by physical +address, with gaps padded by zeros. +""" + +import argparse +import subprocess +import sys +from pathlib import Path + + +def elf_to_bin(elf_path: Path, bin_path: Path, objcopy: str) -> None: + subprocess.run( + [objcopy, "-O", "binary", str(elf_path), str(bin_path)], + check=True, + ) + + +def bin_to_hex(bin_path: Path, hex_path: Path) -> int: + data = bin_path.read_bytes() + lines = ["@0"] + lines.extend(f"{b:02X}" for b in data) + hex_path.write_text("\n".join(lines) + "\n") + return len(data) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("elf", type=Path, help="Input ELF file") + parser.add_argument( + "hex", + type=Path, + nargs="?", + help="Output hex file (default: .hex)", + ) + parser.add_argument( + "--objcopy", + default="riscv64-unknown-elf-objcopy", + help="objcopy binary (default: riscv64-unknown-elf-objcopy)", + ) + args = parser.parse_args() + + if not args.elf.is_file(): + print(f"error: ELF not found: {args.elf}", file=sys.stderr) + return 1 + + hex_path = args.hex or args.elf.with_suffix(".hex") + bin_path = hex_path.with_suffix(".bin") + + try: + elf_to_bin(args.elf, bin_path, args.objcopy) + except FileNotFoundError: + print(f"error: objcopy not found: {args.objcopy}", file=sys.stderr) + print("hint: run inside `nix develop` shell", file=sys.stderr) + return 1 + except subprocess.CalledProcessError as e: + print(f"error: objcopy failed: {e}", file=sys.stderr) + return 1 + + n = bin_to_hex(bin_path, hex_path) + bin_path.unlink() + + print(f"wrote {hex_path} ({n} bytes, {n + 1} lines)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bbdev/api/steps/mill/01_verilator_verilog_event.step.py b/bbdev/api/steps/mill/01_verilator_verilog_event.step.py new file mode 100644 index 000000000..3273c059f --- /dev/null +++ b/bbdev/api/steps/mill/01_verilator_verilog_event.step.py @@ -0,0 +1,98 @@ +import os +import shutil +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.event_common import require_design +from utils.path import get_buckyball_path +from utils.path import rtl_out +from steps.mill.scripts.mill import design_cmd +from utils.event_common import check_result, get_origin_trace_id +from utils.stream_run import stream_run_logger_async + +config = { + "name": "verilator-verilog", + "description": "generate design verilog code", + "flows": ["verilator"], + "triggers": [queue("verilator.verilog")], + "enqueues": ["verilator.build"], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + arch = os.path.join(bbdir, "arch") + + try: + design = require_design(input_data) + mill_config, build_dir = rtl_out( + bbdir, + design, + "verilog", + input_data.get("output_dir"), + selection=input_data["target"], + ) + if os.path.exists(build_dir): + shutil.rmtree(build_dir) + os.makedirs(build_dir) + ctx.logger.info(f"Using mill config: {mill_config}") + ctx.logger.info(f"Using build directory: {build_dir}") + command, directory = design_cmd( + bbdir, + design, + "verilator", + build_dir, + diff=bool(input_data.get("diff", False)), + selection=input_data["target"], + ) + returncode = ( + await stream_run_logger_async( + cmd=command, + logger=ctx.logger, + cwd=directory, + stdout_prefix="verilator verilog", + stderr_prefix="verilator verilog", + ) + ).returncode + except (ValueError, RuntimeError) as error: + ctx.logger.error(str(error)) + _, failure_result = await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "task": "validation", + "error": str(error), + "example": 'bbdev verilator --verilog "--design toy"', + }, + trace_id=origin_tid, + ) + return failure_result + ctx.logger.info(f"Using design: {design}") + + await check_result( + ctx, + returncode, + continue_run=input_data.get("from_run_workflow", False) and returncode == 0, + extra_fields={"task": "verilog"}, + trace_id=origin_tid, + ) + + if input_data.get("from_run_workflow") and returncode == 0: + await ctx.enqueue( + { + "topic": "verilator.build", + "data": { + **input_data, + "output_dir": build_dir, + "task": "run", + "target": input_data["target"], + }, + } + ) diff --git a/bbdev/api/steps/mill/02_vcs_verilog_event.step.py b/bbdev/api/steps/mill/02_vcs_verilog_event.step.py new file mode 100644 index 000000000..4980838a1 --- /dev/null +++ b/bbdev/api/steps/mill/02_vcs_verilog_event.step.py @@ -0,0 +1,86 @@ +import os +import shutil +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.event_common import require_design +from utils.path import get_buckyball_path +from utils.path import rtl_out +from steps.mill.scripts.mill import elaborate_cmd +from utils.event_common import check_result, get_origin_trace_id +from utils.stream_run import stream_run_logger_async + +config = { + "name": "vcs-verilog", + "description": "generate BBSimHarness RTL for VCS", + "flows": ["vcs"], + "triggers": [queue("vcs.verilog")], + "enqueues": ["vcs.build"], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + try: + design = require_design(input_data) + except ValueError as exc: + await check_result( + ctx, + 1, + extra_fields={"task": "verilog", "error": str(exc)}, + trace_id=origin_tid, + ) + return + bbdir = get_buckyball_path() + arch = os.path.join(bbdir, "arch") + try: + mill_config, build_dir = rtl_out( + bbdir, + design, + "tapeout", + selection=input_data["target"], + ) + if os.path.exists(build_dir): + shutil.rmtree(build_dir) + os.makedirs(build_dir) + returncode = ( + await stream_run_logger_async( + cmd=elaborate_cmd(design, input_data["target"], "verilog", build_dir), + logger=ctx.logger, + cwd=arch, + stdout_prefix="vcs verilog", + stderr_prefix="vcs verilog", + ) + ).returncode + except (ValueError, RuntimeError) as exc: + await check_result( + ctx, + 1, + extra_fields={"task": "verilog", "error": str(exc)}, + trace_id=origin_tid, + ) + return + await check_result( + ctx, + returncode, + continue_run=input_data.get("from_run_workflow", False) and returncode == 0, + extra_fields={"task": "verilog", "output_dir": build_dir}, + trace_id=origin_tid, + ) + if input_data.get("from_run_workflow") and returncode == 0: + await ctx.enqueue( + { + "topic": "vcs.build", + "data": { + **input_data, + "output_dir": build_dir, + "_trace_id": origin_tid, + "target": input_data["target"], + }, + } + ) diff --git a/bbdev/api/steps/mill/03_yosys_verilog_event.step.py b/bbdev/api/steps/mill/03_yosys_verilog_event.step.py new file mode 100644 index 000000000..e0b2ac305 --- /dev/null +++ b/bbdev/api/steps/mill/03_yosys_verilog_event.step.py @@ -0,0 +1,298 @@ +import os +import shutil +import sys +import glob +import re +import yaml +from datetime import datetime + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) +step_path = os.path.dirname(__file__) +if step_path not in sys.path: + sys.path.insert(0, step_path) + +from utils.event_common import require_design +from utils.path import get_buckyball_path, log_dir +from utils.stream_run import stream_run_logger_async +from utils.path import rtl_out +from steps.mill.scripts.mill import elaborate_cmd +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "yosys verilog", + "description": "generate verilog for yosys flow", + "flows": ["yosys"], + "triggers": [queue("yosys.run"), queue("yosys.verilog")], + "enqueues": ["ip.generate"], +} + + +def load_yosys_config(): + bbdir = get_buckyball_path() + config_path = f"{bbdir}/bbdev/api/steps/yosys/scripts/yosys-config.yaml" + if os.path.exists(config_path): + with open(config_path, "r") as f: + return yaml.safe_load(f) or {} + return {} + + +def prepare_yosys_verilog(build_dir: str, yosys_log_dir: str, logger): + def _build_stub_from_header(src_path: str, content: str, force_blackbox: bool): + m = re.search( + r"module\s+([A-Za-z_][A-Za-z0-9_]*)\s*\((.*?)\)\s*;", content, re.S + ) + if m is None: + raise RuntimeError(f"invalid module header format: {src_path}") + mod_name = m.group(1) + ports_block = m.group(2).rstrip() + head = "(* blackbox *) " if force_blackbox else "" + return f"{head}module {mod_name}(\n{ports_block}\n);\nendmodule\n" + + def _collect_defined_modules(sv_files): + defined = set() + for path in sv_files: + with open(path, "r") as f: + content = f.read() + for m in re.finditer( + r"^\s*module\s+([A-Za-z_][A-Za-z0-9_]*)\b", content, re.M + ): + defined.add(m.group(1)) + return defined + + def _extract_external_modules(extern_path): + if not os.path.exists(extern_path): + return [] + names = [] + with open(extern_path, "r") as f: + for line in f: + m = re.match( + r"\s*//\s*external module\s+([A-Za-z_][A-Za-z0-9_]*)\s*$", line + ) + if m: + names.append(m.group(1)) + return names + + def _collect_ports(mod_name, sv_files): + ports = set() + inst_re = re.compile( + rf"\b{mod_name}\b\s+[A-Za-z_][A-Za-z0-9_]*\s*\((.*?)\)\s*;", re.S + ) + port_re = re.compile(r"\.([A-Za-z_][A-Za-z0-9_]*)\s*\(") + for path in sv_files: + with open(path, "r") as f: + content = f.read() + for im in inst_re.finditer(content): + block = im.group(1) + for pm in port_re.finditer(block): + ports.add(pm.group(1)) + return sorted(ports) + + def _build_blackbox(mod_name, ports): + lines = [f"(* blackbox *) module {mod_name}("] + if ports: + for idx, p in enumerate(ports): + end = "," if idx < len(ports) - 1 else "" + lines.append(f" input {p}{end}") + lines.append(");") + lines.append("endmodule") + lines.append("") + return "\n".join(lines) + + stub_dir = os.path.join(yosys_log_dir, "dpi_stubs") + os.makedirs(stub_dir, exist_ok=True) + + all_sv = glob.glob(f"{build_dir}/**/*.sv", recursive=True) + kept = [] + dpi_stubbed = [] + pattern_stubbed = [] + skipped = [] + + for path in all_sv: + with open(path, "r") as f: + content = f.read() + if 'import "DPI-C"' in content: + stub_path = os.path.join(stub_dir, f"stub_{os.path.basename(path)}") + with open(stub_path, "w") as wf: + wf.write(_build_stub_from_header(path, content, False)) + kept.append(stub_path) + dpi_stubbed.append(os.path.basename(path)) + elif "'{" in content: + stub_path = os.path.join(stub_dir, f"pattern_stub_{os.path.basename(path)}") + try: + with open(stub_path, "w") as wf: + wf.write(_build_stub_from_header(path, content, True)) + kept.append(stub_path) + pattern_stubbed.append(os.path.basename(path)) + except RuntimeError: + skipped.append(os.path.basename(path)) + else: + kept.append(path) + + extern_path = os.path.join(build_dir, "extern_modules.sv") + extern_names = _extract_external_modules(extern_path) + defined = _collect_defined_modules(kept) + ext_stubbed = [] + for mod_name in extern_names: + if mod_name in defined: + continue + ports = _collect_ports(mod_name, all_sv) + stub_path = os.path.join(stub_dir, f"ext_stub_{mod_name}.sv") + with open(stub_path, "w") as f: + f.write(_build_blackbox(mod_name, ports)) + kept.append(stub_path) + ext_stubbed.append(mod_name) + + if not kept: + raise RuntimeError("no yosys source generated") + + source_list_path = os.path.join(build_dir, "yosys_sources.list") + with open(source_list_path, "w") as f: + for path in kept: + f.write(path + "\n") + + if dpi_stubbed: + logger.info( + f"Stubbed {len(dpi_stubbed)} DPI modules:" + f" {', '.join(dpi_stubbed[:10])}{'...' if len(dpi_stubbed) > 10 else ''}" + ) + if pattern_stubbed: + logger.info( + f"Stubbed {len(pattern_stubbed)} pattern modules:" + f" {', '.join(pattern_stubbed[:10])}{'...' if len(pattern_stubbed) > 10 else ''}" + ) + if ext_stubbed: + logger.info( + f"Stubbed {len(ext_stubbed)} external modules:" + f" {', '.join(ext_stubbed[:10])}{'...' if len(ext_stubbed) > 10 else ''}" + ) + if skipped: + logger.info( + f"Skipped {len(skipped)} unsupported files:" + f" {', '.join(skipped[:10])}{'...' if len(skipped) > 10 else ''}" + ) + logger.info(f"Prepared yosys source list: {source_list_path}") + return source_list_path + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + arch = os.path.join(bbdir, "arch") + try: + design = require_design(input_data) + mill_config, build_dir = rtl_out( + bbdir, + design, + "synth", + input_data.get("output_dir"), + selection=input_data["target"], + ) + if os.path.exists(build_dir): + shutil.rmtree(build_dir) + os.makedirs(build_dir) + ctx.logger.info(f"Using mill config: {mill_config}") + ctx.logger.info(f"Using build directory: {build_dir}") + returncode = ( + await stream_run_logger_async( + cmd=elaborate_cmd( + design, + input_data["target"], + "verilog", + build_dir, + seq_mem=True, + ), + logger=ctx.logger, + cwd=arch, + stdout_prefix="yosys verilog", + stderr_prefix="yosys verilog", + ) + ).returncode + except (ValueError, RuntimeError) as error: + success_result, failure_result = await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "validation", "error": str(error)}, + trace_id=origin_tid, + ) + return failure_result + if returncode != 0: + success_result, failure_result = await check_result( + ctx, + returncode, + continue_run=False, + extra_fields={"task": "verilog"}, + trace_id=origin_tid, + ) + return failure_result + yosys_cfg = load_yosys_config() + top_module = input_data.get("top") or yosys_cfg["top"] + yosys_log_dir = log_dir( + bbdir, + design, + "synth", + datetime.now().strftime("%Y-%m-%d-%H-%M"), + "yosys", + top_module, + input_data.get("output_dir"), + selection=input_data["target"], + ) + ctx.logger.info(f"Yosys log dir: {yosys_log_dir}") + mem_conf = os.path.join(build_dir, "mems.conf") + arch_dir = f"{bbdir}/arch" + + for unwanted in ["TestHarness.sv", "TargetBall.sv"]: + topname_file = f"{arch_dir}/{unwanted}" + if os.path.exists(topname_file): + os.remove(topname_file) + + os.makedirs(yosys_log_dir, exist_ok=True) + try: + source_list_path = prepare_yosys_verilog(build_dir, yosys_log_dir, ctx.logger) + except Exception as e: + success_result, failure_result = await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "verilog", "error": str(e)}, + trace_id=origin_tid, + ) + return failure_result + + need_ip = bool(input_data.get("from_run_workflow")) + await check_result( + ctx, + 0, + continue_run=need_ip, + extra_fields={ + "task": "verilog", + "source_list": source_list_path, + "mem_conf": mem_conf, + }, + trace_id=origin_tid, + ) + if not need_ip: + return + + await ctx.enqueue( + { + "topic": "ip.generate", + "data": { + **input_data, + "design": design, + "consumer": "yosys", + "top": top_module, + "analysis_dir": yosys_log_dir, + "next_topic": "yosys.synth", + "task": "run", + "_trace_id": origin_tid, + "target": input_data["target"], + }, + } + ) + + return diff --git a/bbdev/api/steps/mill/04_dc_verilog_event.step.py b/bbdev/api/steps/mill/04_dc_verilog_event.step.py new file mode 100644 index 000000000..0e5d97bf2 --- /dev/null +++ b/bbdev/api/steps/mill/04_dc_verilog_event.step.py @@ -0,0 +1,222 @@ +import os +import shutil +import sys +import glob +import re +from datetime import datetime + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) +step_path = os.path.dirname(__file__) +if step_path not in sys.path: + sys.path.insert(0, step_path) + +from utils.event_common import check_result, get_origin_trace_id +from utils.event_common import require_design +from utils.path import get_buckyball_path, log_dir +from utils.stream_run import stream_run_logger_async + +mill_dir = os.path.join(os.path.dirname(__file__)) +mill_scripts = os.path.join(mill_dir, "scripts") +if mill_dir not in sys.path: + sys.path.insert(0, mill_dir) +if mill_scripts not in sys.path: + sys.path.insert(0, mill_scripts) +from utils.path import rtl_out +from steps.mill.scripts.mill import elaborate_cmd +from steps.dc.scripts.tapeout import get_tapeout_contract + +config = { + "name": "dc-verilog", + "description": "generate RTL and memory metadata for downstream DC/tapeout flow", + "flows": ["dc"], + "triggers": [queue("dc.verilog")], + "enqueues": ["ip.generate"], +} + + +def is_dpi_source(path: str) -> bool: + with open(path, "r", encoding="utf-8", errors="ignore") as f: + return 'import "DPI-C"' in f.read() + + +def build_stub_from_header(src_path: str) -> str: + with open(src_path, "r", encoding="utf-8", errors="ignore") as f: + content = f.read() + m = re.search( + r"module\s+([A-Za-z_][A-Za-z0-9_]*)\s*(#\s*\(.*?\)\s*)?\((.*?)\)\s*;", + content, + re.S, + ) + if m is None: + raise RuntimeError(f"invalid module header format: {src_path}") + mod_name = m.group(1) + params_block = m.group(2) or "" + ports_block = m.group(3).rstrip() + return f"(* blackbox *) module {mod_name} {params_block}(\n{ports_block}\n);\nendmodule\n" + + +def prepare_dc_verilog(build_dir: str): + vsrcs = sorted( + glob.glob(f"{build_dir}/**/*.sv", recursive=True) + + glob.glob(f"{build_dir}/**/*.v", recursive=True) + ) + stub_dir = os.path.join(build_dir, "dc_stubs") + os.makedirs(stub_dir, exist_ok=True) + skip_dirs = { + os.path.join(build_dir, name) + for name in ("ip-generate", "ip-replace", "dc_stubs") + } + kept = [] + stubbed_dpi = [] + for path in vsrcs: + if any(path == d or path.startswith(d + os.sep) for d in skip_dirs): + continue + if is_dpi_source(path): + stub_path = os.path.join(stub_dir, f"stub_{os.path.basename(path)}") + with open(stub_path, "w") as f: + f.write(build_stub_from_header(path)) + kept.append(stub_path) + stubbed_dpi.append(path) + else: + kept.append(path) + if not kept: + raise RuntimeError("no dc verilog source generated") + source_list_path = os.path.join(build_dir, "dc_sources.list") + with open(source_list_path, "w") as f: + for path in kept: + f.write(path + "\n") + return source_list_path, stubbed_dpi + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + arch = os.path.join(bbdir, "arch") + try: + design = require_design(input_data) + mill_config, build_dir = rtl_out( + bbdir, + design, + "dc", + input_data.get("output_dir"), + selection=input_data["target"], + ) + if os.path.exists(build_dir): + shutil.rmtree(build_dir) + os.makedirs(build_dir) + ctx.logger.info(f"Using mill config: {mill_config}") + ctx.logger.info(f"Using build directory: {build_dir}") + returncode = ( + await stream_run_logger_async( + cmd=elaborate_cmd( + design, + input_data["target"], + "verilog", + build_dir, + seq_mem=True, + ), + logger=ctx.logger, + cwd=arch, + stdout_prefix="dc verilog", + stderr_prefix="dc verilog", + ) + ).returncode + except (ValueError, RuntimeError) as error: + _, failure_result = await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "validation", "error": str(error)}, + trace_id=origin_tid, + ) + return failure_result + if returncode != 0: + _, failure_result = await check_result( + ctx, + returncode, + continue_run=False, + extra_fields={"task": "verilog"}, + trace_id=origin_tid, + ) + return failure_result + top_module = get_tapeout_contract( + bbdir, design, input_data.get("top"), selection=input_data["target"] + ).top_module + ctx.logger.info( + f"Using DC RTL output directory: {build_dir}, synthesis top: {top_module}" + ) + mem_conf = os.path.join(build_dir, "mems.conf") + + try: + source_list_path, stubbed_dpi = prepare_dc_verilog(build_dir) + except Exception as e: + _, failure_result = await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "verilog", "error": str(e)}, + trace_id=origin_tid, + ) + return failure_result + if stubbed_dpi: + ctx.logger.info( + f"Stubbed {len(stubbed_dpi)} DPI-C sources for DC: " + f"{', '.join(os.path.basename(path) for path in stubbed_dpi[:10])}" + f"{'...' if len(stubbed_dpi) > 10 else ''}" + ) + + need_ip = bool( + input_data.get("from_area_workflow") or input_data.get("from_power_workflow") + ) + await check_result( + ctx, + 0, + continue_run=need_ip, + extra_fields={ + "task": "verilog", + "source_list": source_list_path, + "mem_conf": mem_conf, + "top_module": top_module, + }, + trace_id=origin_tid, + ) + if not need_ip: + return + + payload = { + **input_data, + "design": design, + "consumer": "dc", + "top": top_module, + "next_topic": "dc.area", + "_trace_id": origin_tid, + } + if input_data.get("from_power_workflow"): + payload["analysis_dir"] = log_dir( + bbdir, + design, + "tapeout", + datetime.now().strftime("%Y-%m-%d-%H-%M"), + "dc", + "power", + selection=input_data["target"], + ) + else: + payload["analysis_dir"] = log_dir( + bbdir, + design, + "tapeout", + datetime.now().strftime("%Y-%m-%d-%H-%M"), + "dc", + "area", + selection=input_data["target"], + ) + await ctx.enqueue( + {"topic": "ip.generate", "data": {**(payload), "target": input_data["target"]}} + ) + + return diff --git a/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py b/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py new file mode 100644 index 000000000..c5dfa946a --- /dev/null +++ b/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py @@ -0,0 +1,151 @@ +""" +bebop verilator verilog event handler + +Generates Verilog via mill for bebop verilator +""" + +import os +import shutil +import sys +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) +from utils.event_common import require_design +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger_async +from utils.path import rtl_out +from steps.mill.scripts.mill import design_cmd +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "bebop-verilator-verilog", + "description": "Generate verilog code via mill", + "flows": ["bebop"], + "triggers": [ + queue("bebop.verilator.verilog"), + queue("bebop.verilator.run.verilog"), + ], + "enqueues": ["bebop.verilator.build", "bebop.verilator.run.build"], +} + + +def check_verilog_output(build_dir: str) -> dict: + exists = os.path.exists(build_dir) + is_dir = os.path.isdir(build_dir) + sv_count = 0 + if is_dir: + sv_count = sum( + (1 for name in os.listdir(build_dir) if name.endswith((".sv", ".v"))) + ) + return {"exists": exists, "is_dir": is_dir, "sv_count": sv_count} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + arch = os.path.join(bbdir, "arch") + design = None + try: + design = require_design(input_data) + mill_config, build_dir = rtl_out( + bbdir, + design, + "bebop", + input_data.get("output_dir"), + selection=input_data["target"], + ) + if not os.path.isabs(build_dir): + raise ValueError("RTL output directory must be absolute") + command, directory = design_cmd( + bbdir, + design, + "verilator", + build_dir, + diff=bool(input_data.get("diff", False)), + selection=input_data["target"], + ) + if os.path.exists(build_dir): + shutil.rmtree(build_dir) + os.makedirs(build_dir) + ctx.logger.info(f"Using mill target: {input_data["target"]}") + ctx.logger.info(f"Using build directory: {build_dir}") + returncode = ( + await stream_run_logger_async( + cmd=command, + logger=ctx.logger, + cwd=directory, + stdout_prefix="bebop verilator verilog", + stderr_prefix="bebop verilator verilog", + ) + ).returncode + except (ValueError, RuntimeError) as error: + ctx.logger.error(str(error)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "task": "validation", + "error": str(error), + "example": 'bbdev bebop verilator --verilog "--design toy"', + }, + trace_id=origin_tid, + ) + return + ctx.logger.info(f"Using design: {design}") + output_status = check_verilog_output(build_dir) + if returncode != 0: + await check_result( + ctx, + returncode, + continue_run=False, + extra_fields={ + "task": "verilog", + "design": design, + "output_dir": build_dir, + "output_status": output_status, + }, + trace_id=origin_tid, + ) + return + if not output_status["is_dir"] or output_status["sv_count"] == 0: + ctx.logger.error( + f"Verilog output is invalid: {build_dir} (exists={output_status['exists']}," + f" is_dir={output_status['is_dir']}, sv_count={output_status['sv_count']})" + ) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "task": "verilog", + "error": "invalid_verilog_output", + "design": design, + "output_dir": build_dir, + "output_status": output_status, + }, + trace_id=origin_tid, + ) + return + await check_result( + ctx, + returncode, + continue_run=input_data.get("from_run_workflow", False) and returncode == 0, + extra_fields={"task": "verilog", "design": design, "output_dir": build_dir}, + trace_id=origin_tid, + ) + if input_data.get("from_run_workflow") and returncode == 0: + await ctx.enqueue( + { + "topic": "bebop.verilator.run.build", + "data": { + **input_data, + "output_dir": build_dir, + "vsrc_dir": build_dir, + "task": "run", + "target": input_data["target"], + }, + } + ) diff --git a/bbdev/api/steps/mill/07_bebop_p2e_verilog_event.step.py b/bbdev/api/steps/mill/07_bebop_p2e_verilog_event.step.py new file mode 100644 index 000000000..b1df85ad1 --- /dev/null +++ b/bbdev/api/steps/mill/07_bebop_p2e_verilog_event.step.py @@ -0,0 +1,84 @@ +import os +import shutil +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.event_common import require_design +from utils.path import get_buckyball_path +from utils.path import rtl_out +from steps.bebop.p2e.scripts.timescale import normalize_p2e_timescale +from utils.stream_run import stream_run_logger_async +from steps.mill.scripts.mill import elaborate_cmd +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "bebop-p2e-verilog", + "description": "Generate P2E Verilog via mill", + "flows": ["bebop"], + "triggers": [queue("bebop.p2e.verilog")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + arch = os.path.join(bbdir, "arch") + try: + design = require_design(input_data) + mill_config, build_dir = rtl_out( + bbdir, + design, + "p2e", + input_data.get("output_dir"), + selection=input_data["target"], + ) + shutil.rmtree(build_dir, ignore_errors=True) + os.makedirs(build_dir) + ctx.logger.info(f"Using mill config: {mill_config}") + ctx.logger.info(f"Using build directory: {build_dir}") + prefix = "bebop p2e verilog" + command = elaborate_cmd(design, input_data["target"], "p2e", build_dir) + if input_data.get("diff"): + command += " --difftest" + returncode = ( + await stream_run_logger_async( + cmd=command, + logger=ctx.logger, + cwd=arch, + stdout_prefix=prefix, + stderr_prefix=prefix, + ) + ).returncode + if returncode == 0: + normalize_p2e_timescale(build_dir, ctx.logger) + except (ValueError, RuntimeError) as error: + ctx.logger.error(str(error)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "missing_design", "detail": str(error)}, + trace_id=origin_tid, + ) + return + ctx.logger.info(f"Using P2E design: {design}") + ctx.logger.info(f"Using P2E output directory: {build_dir}") + + await check_result( + ctx, + returncode, + continue_run=False, + extra_fields={ + "task": "verilog", + "design": design, + "output_dir": build_dir, + "top_module": "P2ETop", + }, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/mill/scripts/mill.py b/bbdev/api/steps/mill/scripts/mill.py new file mode 100644 index 000000000..45b8cf36f --- /dev/null +++ b/bbdev/api/steps/mill/scripts/mill.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +import os +import shlex +from utils.path import get_buckyball_path, load_design_config, simulation_target +from pathlib import Path + + +def mill_cmd( + repo: str, directory: str | Path, target: str, arguments=(), *, show: bool = False +) -> str: + repo = Path(repo).resolve() + directory = (repo / directory).resolve() + designs = repo / "examples/designs" + if directory.is_relative_to(designs): + relative = directory.relative_to(designs) + owner = designs / relative.parts[0] + if directory.is_relative_to(owner / "sims"): + work = ( + owner + / "output/sims" + / directory.relative_to(owner / "sims").parent + / "mill" + ) + else: + work = owner / "output/mill" + else: + owner = ( + directory.parent + if directory.name == "arch" and directory.is_relative_to(repo / "examples") + else directory + ) + work = owner / "output/mill" + work.mkdir(parents=True, exist_ok=True) + relative = Path(os.path.relpath(directory, work)) + imported = ".".join("^" if part == ".." else f"`{part}`" for part in relative.parts) + source = f"import $file.{imported}.{{build => source}}\n\ndef project = source\n" + launcher = work / "build.sc" + if not launcher.is_file() or launcher.read_text() != source: + launcher.write_text(source) + command = shlex.join( + [ + "flock", + "/tmp/buckyball-mill.lock", + "mill", + "--no-server", + *(["show"] if show else []), + f"project.{target}", + *arguments, + ] + ) + return f"cd {shlex.quote(str(work))} && {command}" + + +def design_cmd( + repo: str, + design: str, + product: str, + out: str, + *, + selection: str, + diff: bool = False, +) -> tuple[str, str]: + registered = simulation_target(repo, design, selection) + build = registered.arch + directory = Path(repo) / build.directory + config = Path(repo) / build.config + command = mill_cmd( + repo, + directory, + build.module + ".runMain", + [ + "sims.p2e.Elaborate" if product == "p2e" else "sims.verilator.Elaborate", + "--config", + str(config), + "--target-class", + registered.entry_class, + "--target-dir", + out, + "--disable-annotation-unknown", + "--strip-debug-info", + "-O=debug", + *(["--difftest"] if diff else []), + ], + ) + return command, str(directory) + + +def elaborate_cmd( + design: str, selection: str, product: str, out: str, *, seq_mem: bool = False +) -> str: + command, _ = design_cmd( + get_buckyball_path(), design, product, out, selection=selection + ) + if seq_mem: + command += " " + shlex.join( + [ + "-lowering-options=disallowLocalVariables", + "--repl-seq-mem", + f"--repl-seq-mem-file={os.path.join(out, 'mems.conf')}", + ] + ) + return command diff --git a/bbdev/api/steps/model/01_build_api.step.py b/bbdev/api/steps/model/01_build_api.step.py new file mode 100644 index 000000000..4040fd5b3 --- /dev/null +++ b/bbdev/api/steps/model/01_build_api.step.py @@ -0,0 +1,32 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.event_common import require_design, require_target + +config = { + "name": "model-build-api", + "description": "Build the model recipes of a design", + "flows": ["model"], + "triggers": [api("POST", "/model/build")], + "enqueues": ["model.build"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + try: + design = require_design(body) + selection = require_target(body, ("device",)) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + await ctx.enqueue( + { + "topic": "model.build", + "data": { + "design": design, + "model": body.get("model"), + "target": selection, + "_trace_id": ctx.trace_id, + }, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/model/01_build_event.step.py b/bbdev/api/steps/model/01_build_event.step.py new file mode 100644 index 000000000..eb91fd441 --- /dev/null +++ b/bbdev/api/steps/model/01_build_event.step.py @@ -0,0 +1,46 @@ +import asyncio +import os +import sys + +from motia import FlowContext, queue + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))) +from utils.event_common import check_result, get_origin_trace_id +from steps.model.scripts.build import build_model +from utils.path import get_buckyball_path + +config = { + "name": "model-build", + "description": "Build a design model recipe", + "flows": ["model"], + "triggers": [queue("model.build")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + trace_id = get_origin_trace_id(input_data, ctx) + params = {key: value for key, value in input_data.items() if key != "_trace_id"} + try: + await asyncio.to_thread( + build_model, + get_buckyball_path(), + params["design"], + params["model"], + selection=params["target"], + logger=ctx.logger, + task_scope=trace_id, + ) + except Exception as error: + ctx.logger.error(str(error)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "model_build_failed", "detail": str(error)}, + trace_id=trace_id, + ) + return + await check_result( + ctx, 0, continue_run=False, extra_fields=params, trace_id=trace_id + ) diff --git a/bbdev/api/steps/model/scripts/build.py b/bbdev/api/steps/model/scripts/build.py new file mode 100644 index 000000000..cfc82c61c --- /dev/null +++ b/bbdev/api/steps/model/scripts/build.py @@ -0,0 +1,78 @@ +import shlex +from pathlib import Path + +from utils.stream_run import stream_run_logger + + +def build_model( + repo: str, + design: str, + model: str | None = None, + *, + selection: str, + logger, + task_scope: str, +) -> None: + """Build one model recipe of `design`, or every recipe of it when `model` is None. + Models run on the whole device, so they build against a device config.""" + recipes = Path(repo) / "examples/models" / design + models = ( + [model] + if model + else sorted(path.parent.name for path in recipes.glob("*/CMakeLists.txt")) + ) + source = Path(repo) / "stack/models" + from steps.stack.scripts.build import build_design + + # Models compile with the design's compiler plugins and link its device runtime. + design_cmake = build_design( + repo, design=design, selection=selection, logger=logger, task_scope=task_scope + ) + for model in models: + build = source / "build" / design / selection / model + build_command = ["cmake", "--build", str(build), "--parallel", "1"] + build_command.extend(["--target", f"{model}-model-build"]) + # A recipe reads the prepared model while it configures, so the model + # is prepared first, by its own project: its weights and checkpoints + # are rebuilt only when their inputs change. + prepare = source / "build" / "models" / model + commands = [ + # Prepared weights can predate this build tree; use file timestamps. + [ + "cmake", + "-S", + str(source), + "-B", + str(prepare), + "-G", + "Unix Makefiles", + f"-DMODEL={model}", + ], + ["cmake", "--build", str(prepare), "--target", f"{model}-prepare"], + [ + "cmake", + "-S", + str(source), + "-B", + str(build), + "-G", + "Ninja", + "-DARCH=buckyball", + f"-DDESIGN_CMAKE={design_cmake}", + f"-DMODEL={model}", + ], + build_command, + ] + for command in commands: + result = stream_run_logger( + cmd=shlex.join(command), + cwd=repo, + logger=logger, + stdout_prefix=f"model {model}", + stderr_prefix=f"model {model}", + task_scope=task_scope, + ) + if result.returncode: + raise RuntimeError( + f"Model build failed ({result.returncode}): {shlex.join(command)}" + ) diff --git a/bbdev/api/steps/model/scripts/config.py b/bbdev/api/steps/model/scripts/config.py new file mode 100644 index 000000000..246addab1 --- /dev/null +++ b/bbdev/api/steps/model/scripts/config.py @@ -0,0 +1,95 @@ +"""Compile model-owned TOML parameters to protobuf and ordinary build arguments.""" + +import ast +import json +import math +import operator +import re +import tomllib +from pathlib import Path + +from steps.config.scripts.model_pb2 import ModelConfig + +_OPERATORS = { + ast.Add: operator.add, + ast.Sub: operator.sub, + ast.Mult: operator.mul, + ast.FloorDiv: operator.floordiv, + ast.Mod: operator.mod, +} + + +def compile_config(source: Path, destination: Path, name: str) -> None: + config = tomllib.loads(source.read_text()) + unknown = config.keys() - {"parameters", "derived"} + if unknown: + raise ValueError(f"Unknown model config sections: {sorted(unknown)}") + values = dict(config["parameters"]) + derived = config.get("derived", {}) + if values.keys() & derived.keys(): + raise ValueError("A model parameter cannot be both supplied and derived") + visiting = set() + + def resolve(key): + if key in values: + return values[key] + if key in visiting: + raise ValueError(f"Cyclic model parameter: {key}") + visiting.add(key) + value = evaluate(ast.parse(derived[key], mode="eval").body) + visiting.remove(key) + values[key] = value + return value + + def evaluate(node): + if isinstance(node, ast.Name): + return resolve(node.id) + if isinstance(node, ast.Constant) and type(node.value) in (int, float): + return node.value + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.USub): + return -evaluate(node.operand) + if isinstance(node, ast.BinOp) and type(node.op) in _OPERATORS: + left, right = evaluate(node.left), evaluate(node.right) + if type(left) not in (int, float) or type(right) not in (int, float): + raise ValueError("Derived arithmetic requires numeric parameters") + return _OPERATORS[type(node.op)](left, right) + raise ValueError( + "Model expressions support names, numbers, +, -, *, // and % only" + ) + + for key in derived: + resolve(key) + message = ModelConfig(version=1, name=name) + fields = {int: "integer", float: "real", str: "text", bool: "boolean"} + for key, value in values.items(): + if not re.fullmatch(r"[a-z][a-z0-9_]*", key): + raise ValueError(f"Invalid model parameter name: {key}") + if ( + type(value) not in fields + or isinstance(value, float) + and not math.isfinite(value) + ): + raise ValueError(f"Invalid model parameter value: {key}") + setattr(message.parameters[key], fields[type(value)], value) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(message.SerializeToString(deterministic=True)) + destination.with_suffix(".json").write_text( + json.dumps(values, indent=2, sort_keys=True) + "\n" + ) + + +def cmake_arguments(path: Path, name: str) -> list[str]: + message = ModelConfig.FromString(path.read_bytes()) + if message.version != 1 or message.name != name: + raise ValueError(f"Invalid model config version or model name: {path}") + arguments = [] + for key, parameter in sorted(message.parameters.items()): + field = parameter.WhichOneof("value") + if field is None: + raise ValueError(f"Model parameter has no value: {key}") + value = getattr(parameter, field) + if field == "boolean": + arguments.append(f"-D{key.upper()}:BOOL={'ON' if value else 'OFF'}") + else: + arguments.append(f"-D{key.upper()}:STRING={value}") + return arguments diff --git a/bbdev/api/steps/pt/01_ptpx_event.step.py b/bbdev/api/steps/pt/01_ptpx_event.step.py new file mode 100644 index 000000000..1aa1b3de2 --- /dev/null +++ b/bbdev/api/steps/pt/01_ptpx_event.step.py @@ -0,0 +1,145 @@ +import os +import shlex +import shutil +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) +dc_scripts_path = os.path.abspath( + os.path.join(os.path.dirname(__file__), "..", "dc", "scripts") +) +if dc_scripts_path not in sys.path: + sys.path.insert(0, dc_scripts_path) + +from utils.event_common import check_result, get_origin_trace_id +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger_async +from power import read_dynamic_power +from tapeout import get_tapeout_contract, resolve_power_window, write_run_tcl + +config = { + "name": "pt-ptpx", + "description": "run PrimeTime PX power analysis", + "flows": ["pt", "dc"], + "triggers": [queue("pt.run")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + top_module = input_data["top"] + activity_path = input_data.get("activity") + activity_format = input_data.get("format") + analysis_dir = input_data.get("analysis_dir") + if not isinstance(analysis_dir, str) or not analysis_dir: + error = "missing fixed DC analysis output directory" + elif not isinstance(activity_path, str) or not activity_path: + error = "missing required --activity" + elif not isinstance(activity_format, str) or activity_format not in { + "saif", + "vcd", + "fsdb", + }: + error = "missing or invalid --format; expected saif, vcd, or fsdb" + elif shutil.which("pt_shell") is None: + error = "pt_shell is not on PATH" + else: + error = None + if error: + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "pt", "error": error}, + trace_id=origin_tid, + ) + return + + try: + netlist = os.path.join(analysis_dir, "outputs", f"{top_module}.v") + sdc = os.path.join(analysis_dir, "outputs", f"{top_module}.sdc") + contract = get_tapeout_contract( + get_buckyball_path(), + input_data.get("design"), + top_module, + selection=input_data["target"], + ) + start_ns, end_ns = resolve_power_window( + contract, + input_data.get("start_ns", input_data.get("start-ns")), + input_data.get("end_ns", input_data.get("end-ns")), + ) + tech = { + "target_library": contract.target_library, + "synthetic_library": contract.synthetic_library, + "link_library": contract.link_library, + "max_threads": contract.max_threads, + } + run_config = write_run_tcl( + os.path.join(analysis_dir, "power-run.tcl"), + { + "top": top_module, + "netlist": netlist, + "sdc": sdc, + "activity": activity_path, + "activity_format": activity_format, + "report_dir": os.path.join(analysis_dir, "power-reports"), + "strip_path": input_data.get("strip_path") or "", + "start_ns": start_ns or "", + "end_ns": end_ns or "", + **tech, + }, + ) + script = contract.power_script + except (OSError, ValueError) as exc: + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "pt", "error": str(exc)}, + trace_id=origin_tid, + ) + return + + result = await stream_run_logger_async( + cmd=( + f"set -o pipefail; pt_shell -f {shlex.quote(str(script))} " + f"-x {shlex.quote('set RUN_CONFIG ' + '{' + str(run_config) + '}')} " + f"2>&1 | tee {shlex.quote(os.path.join(analysis_dir, 'pt_shell.log'))}" + ), + logger=ctx.logger, + cwd=os.path.dirname(script), + stdout_prefix="pt", + stderr_prefix="pt", + ) + report_dir = os.path.join(analysis_dir, "power-reports") + dynamic_power = read_dynamic_power(os.path.join(report_dir, "power_total.rpt")) + extra_fields = { + "task": "pt", + "top_module": top_module, + "activity": activity_path, + "format": activity_format, + "power_script": str(script), + "run_config": str(run_config), + "pt_log": os.path.join(analysis_dir, "pt_shell.log"), + "power_report_dir": report_dir, + "power_scope": "dynamic (internal + switching)", + "start_ns": start_ns, + "end_ns": end_ns, + } + if dynamic_power: + extra_fields["dynamic_power"] = dynamic_power + await check_result( + ctx, + result.returncode, + continue_run=False, + extra_fields=extra_fields, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/pt/scripts/power.py b/bbdev/api/steps/pt/scripts/power.py new file mode 100644 index 000000000..fcc3251d3 --- /dev/null +++ b/bbdev/api/steps/pt/scripts/power.py @@ -0,0 +1,36 @@ +"""Small report helpers for dynamic-power-only summaries.""" + +from __future__ import annotations + +import re +from pathlib import Path + + +def _watts(value: str) -> float: + match = re.fullmatch(r"([-+]?\d+(?:\.\d*)?(?:[eE][-+]?\d+)?)\s*([munp]?W)\b", value) + if match is None: + raise ValueError(f"unsupported power value: {value}") + scale = {"W": 1.0, "mW": 1e-3, "uW": 1e-6, "nW": 1e-9, "pW": 1e-12} + return float(match.group(1)) * scale[match.group(2)] + + +def read_dynamic_power(report: str | Path) -> dict[str, str] | None: + path = Path(report) + if not path.is_file(): + return None + values: dict[str, str] = {} + patterns = { + "internal_power": r"Cell Internal Power\s*=\s*([-+]?\d+(?:\.\d*)?(?:[eE][-+]?\d+)?\s*[munp]?W)\b", + "switching_power": r"Net Switching Power\s*=\s*([-+]?\d+(?:\.\d*)?(?:[eE][-+]?\d+)?\s*[munp]?W)\b", + "total_power": r"Total Power\s*=\s*([-+]?\d+(?:\.\d*)?(?:[eE][-+]?\d+)?\s*[munp]?W)\b", + } + text = path.read_text(encoding="utf-8", errors="ignore") + for key, pattern in patterns.items(): + match = re.search(pattern, text) + if match: + values[key] = match.group(1).strip() + if "internal_power" not in values or "switching_power" not in values: + return None + dynamic_watts = _watts(values["internal_power"]) + _watts(values["switching_power"]) + values["dynamic_power"] = f"{dynamic_watts:.12g} W" + return values diff --git a/bbdev/api/steps/stack/01_build_api.step.py b/bbdev/api/steps/stack/01_build_api.step.py new file mode 100644 index 000000000..fd02f3910 --- /dev/null +++ b/bbdev/api/steps/stack/01_build_api.step.py @@ -0,0 +1,39 @@ +from utils.event_common import require_target +import os +import re +import sys + +from motia import ApiRequest, ApiResponse, FlowContext, api + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +config = { + "name": "build-stack-api", + "description": "build one level of a design's stack", + "flows": ["stack"], + "triggers": [api("POST", "/stack/build")], + "enqueues": ["stack.build"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + if set(body) - {"design", "device", "core"}: + raise ValueError( + "stack build takes --design and --device or --core " + ) + if not isinstance(body.get("design"), str): + raise ValueError("--design requires a design name") + await ctx.enqueue( + { + "topic": "stack.build", + "data": {**body, "_trace_id": ctx.trace_id, "target": selection}, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/stack/01_build_event.step.py b/bbdev/api/steps/stack/01_build_event.step.py new file mode 100644 index 000000000..f717052c6 --- /dev/null +++ b/bbdev/api/steps/stack/01_build_event.step.py @@ -0,0 +1,54 @@ +import asyncio +import importlib.util +import os +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path + +_stack_build_path = os.path.join(os.path.dirname(__file__), "scripts", "build.py") +_spec = importlib.util.spec_from_file_location("stack_build_module", _stack_build_path) +if _spec is None or _spec.loader is None: + raise RuntimeError(f"cannot load {_stack_build_path}") +stack_build = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(stack_build) + +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "build-stack", + "description": "Build one level of a design's stack", + "flows": ["stack"], + "triggers": [queue("stack.build")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + log = {"logger": ctx.logger, "task_scope": origin_tid} + try: + await asyncio.to_thread( + stack_build.build_design, + bbdir, + design=input_data["design"], + selection=input_data["target"], + **log, + ) + except Exception as exc: + ctx.logger.error(f"Stack build failed: {exc}") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "build", "error": str(exc)}, + trace_id=origin_tid, + ) + return + await check_result(ctx, 0, continue_run=False, trace_id=origin_tid) diff --git a/bbdev/api/steps/stack/scripts/build.py b/bbdev/api/steps/stack/scripts/build.py new file mode 100644 index 000000000..a8e00560c --- /dev/null +++ b/bbdev/api/steps/stack/scripts/build.py @@ -0,0 +1,246 @@ +from __future__ import annotations + +import argparse +import fcntl +import os +import shlex +import shutil +import subprocess +import sys +from pathlib import Path + + +def run( + cmd: list[str], + *, + repo: Path, + logger: object | None = None, + task_scope: str | None = None, + output_prefix: str, +) -> None: + cmd = ["nix", "develop", str(repo), "-c", *cmd] + if logger is None: + result = subprocess.run(cmd, cwd=repo) + else: + from utils.stream_run import stream_run_logger + + result = stream_run_logger( + cmd=shlex.join(cmd), + logger=logger, + cwd=str(repo), + stdout_prefix=output_prefix, + stderr_prefix=output_prefix, + task_scope=task_scope, + ) + if result.returncode != 0: + detail = result.stderr.strip() if logger is not None else "" + if not detail and logger is not None: + lines = result.stdout.splitlines() + failed = [i for i, line in enumerate(lines) if "FAILED:" in line] + if failed: + start = failed[0] + detail = "\n".join(lines[start:]) + else: + detail = "\n".join(lines[-20:]) + if detail: + raise RuntimeError( + f"command failed ({result.returncode}): {' '.join(cmd)}\n{detail}" + ) + raise RuntimeError(f"command failed ({result.returncode}): {' '.join(cmd)}") + + +def compiler_python(repo: str | Path) -> str: + root = Path(repo).resolve() + candidates = [root / "result" / "bin" / "python3"] + path_python = shutil.which("python3") + if path_python: + candidates.append(Path(path_python)) + for candidate in candidates: + if not candidate.is_file(): + continue + probe = subprocess.run( + [str(candidate), "-c", "import nanobind"], capture_output=True + ) + if probe.returncode == 0: + return str(candidate) + checked = ", ".join(str(candidate) for candidate in candidates) + raise RuntimeError(f"python3 with nanobind not found (checked: {checked})") + + +def build_llvm( + repo: str | Path, *, logger: object | None = None, task_scope: str | None = None +) -> Path: + root = Path(repo).resolve() + buddy = root / "stack" / "thirdparty" / "buddy-mlir" + llvm_src = buddy / "llvm" / "llvm" + llvm_build = buddy / "llvm" / "build" + if not llvm_src.is_dir(): + raise RuntimeError(f"missing LLVM source: {llvm_src}") + + python = compiler_python(root) + cmake = [ + "cmake", + "-G", + "Ninja", + "-S", + str(llvm_src), + "-B", + str(llvm_build), + "-DLLVM_ENABLE_PROJECTS=mlir;clang", + "-DLLVM_ENABLE_RUNTIMES=openmp", + "-DLLVM_TARGETS_TO_BUILD=host;RISCV", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DOPENMP_ENABLE_LIBOMPTARGET=OFF", + "-DLIBOMP_LIBFLAGS=-lrt", + "-DCMAKE_BUILD_TYPE=RELEASE", + "-DMLIR_ENABLE_BINDINGS_PYTHON=ON", + f"-DPython3_EXECUTABLE={python}", + f"-DPython_EXECUTABLE={python}", + ] + llvm_build.mkdir(parents=True, exist_ok=True) + with open(llvm_build / ".bbdev.lock", "a+b") as lockf: + fcntl.flock(lockf.fileno(), fcntl.LOCK_EX) + if not (llvm_build / "build.ninja").is_file(): + run( + cmake, + repo=root, + logger=logger, + task_scope=task_scope, + output_prefix="compiler llvm configure", + ) + run( + ["ninja", "-C", str(llvm_build), "-j", str(os.cpu_count() or 1)], + repo=root, + logger=logger, + task_scope=task_scope, + output_prefix="compiler llvm build", + ) + mlir_cmake = llvm_build / "lib" / "cmake" / "mlir" + if not mlir_cmake.is_dir(): + raise RuntimeError(f"LLVM/MLIR build failed: missing {mlir_cmake}") + return llvm_build + + +def build_buddy( + repo: str | Path, *, logger: object | None = None, task_scope: str | None = None +) -> Path: + """The one buddy-mlir build every stack's compiler plugin is loaded into.""" + root = Path(repo).resolve() + buddy = root / "stack/thirdparty/buddy-mlir" + llvm = build_llvm(root, logger=logger, task_scope=task_scope) + python = compiler_python(root) + build = buddy / "build" + cmake = [ + "cmake", + "-G", + "Ninja", + "-S", + str(buddy), + "-B", + str(build), + f"-DMLIR_DIR={llvm / 'lib' / 'cmake' / 'mlir'}", + f"-DLLVM_DIR={llvm / 'lib' / 'cmake' / 'llvm'}", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DCMAKE_BUILD_TYPE=RELEASE", + "-DBUDDY_MLIR_ENABLE_PYTHON_PACKAGES=ON", + f"-DPython3_EXECUTABLE={python}", + f"-DPython_EXECUTABLE={python}", + ] + with open(buddy / ".bbdev.lock", "a+b") as lockf: + fcntl.flock(lockf.fileno(), fcntl.LOCK_EX) + run( + cmake, + repo=root, + logger=logger, + task_scope=task_scope, + output_prefix="buddy-mlir configure", + ) + run( + [ + "ninja", + "-C", + str(build), + "-j", + str(os.cpu_count() or 1), + "buddy-opt", + "buddy-translate", + "buddy-llc", + "rax-pack", + "python-package-buddy", + "BuddyMLIRPythonModules", + ], + repo=root, + logger=logger, + task_scope=task_scope, + output_prefix="buddy-mlir build", + ) + return build + + +def build_design( + repo: str | Path, + *, + design: str, + selection: str, + logger: object | None = None, + task_scope: str | None = None, +) -> Path: + """Build the software stack of one simulation of `design`, and return its + design.cmake, the input of every software build of that simulation. + + The stack/CMakeLists.txt of its entry is the top of that build: it lists the + thread, core and device stacks the entry needs and builds each of them. + """ + root = Path(repo).resolve() + sys.path.insert(0, str(root)) + from stack.scripts import design as design_config + + design_pb = ( + root + / "examples/designs" + / design + / "output/sims" + / selection + / "generated/design.pb" + ) + config = design_config.load(design_pb) + build = design_config.stack_build(config) + design_config.generate(design_pb, build / "generated") + build_buddy(root, logger=logger, task_scope=task_scope) + run( + [ + "cmake", + "-G", + "Ninja", + "-S", + config.simulation.stack, + "-B", + str(build), + f"-DDESIGN_CMAKE={build / 'generated/design.cmake'}", + ], + repo=root, + logger=logger, + task_scope=task_scope, + output_prefix=f"stack {selection} configure", + ) + run( + ["ninja", "-C", str(build), "-j", str(os.cpu_count() or 1)], + repo=root, + logger=logger, + task_scope=task_scope, + output_prefix=f"stack {selection} build", + ) + return build / "generated/design.cmake" + + +def main() -> None: + parser = argparse.ArgumentParser(description="Build the stacks of a design") + parser.add_argument("--repo", type=Path, required=True) + parser.add_argument("--design", required=True) + parser.add_argument("--target", required=True) + args = parser.parse_args() + build_design(args.repo, design=args.design, selection=args.target) + + +if __name__ == "__main__": + main() diff --git a/bbdev/api/steps/uvm/00_waive_event.step.py b/bbdev/api/steps/uvm/00_waive_event.step.py new file mode 100644 index 000000000..acd1cc751 --- /dev/null +++ b/bbdev/api/steps/uvm/00_waive_event.step.py @@ -0,0 +1,65 @@ +import asyncio +import os +import sys +from pathlib import Path + +from motia import FlowContext, queue + +step_dir = os.path.dirname(os.path.abspath(__file__)) +utils_path = os.path.abspath(os.path.join(step_dir, "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) +if step_dir not in sys.path: + sys.path.insert(0, step_dir) + +from scripts.waive import apply_waivers +from utils.event_common import check_result, get_origin_trace_id, require_design +from utils.path import get_buckyball_path, rtl_dir + +config = { + "name": "uvm-waive", + "description": "apply common Chisel coverage waivers to generated RTL", + "flows": ["uvm"], + "triggers": [queue("uvm.waive")], + "enqueues": ["uvm.build", "uvm.run"], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + try: + design = require_design(input_data) + task = input_data["uvm_task"] + if task not in ("build", "run"): + raise ValueError(f"invalid UVM task: {task}") + bbdir = get_buckyball_path() + count = await asyncio.to_thread( + apply_waivers, + Path(rtl_dir(bbdir, design, "tapeout", selection=input_data["target"])), + ) + except (KeyError, ValueError) as error: + await check_result( + ctx, + 1, + extra_fields={"task": "waive", "error": str(error)}, + trace_id=origin_tid, + ) + return + await check_result( + ctx, + 0, + continue_run=True, + extra_fields={"task": "waive", "count": count}, + trace_id=origin_tid, + ) + await ctx.enqueue( + { + "topic": f"uvm.{task}", + "data": { + **input_data, + "force_rebuild": True, + "_trace_id": origin_tid, + "target": input_data["target"], + }, + } + ) diff --git a/bbdev/api/steps/uvm/01_build_api.step.py b/bbdev/api/steps/uvm/01_build_api.step.py new file mode 100644 index 000000000..b52dda5a9 --- /dev/null +++ b/bbdev/api/steps/uvm/01_build_api.step.py @@ -0,0 +1,55 @@ +from utils.event_common import require_target +from motia import ApiRequest, ApiResponse, FlowContext, api + +config = { + "name": "uvm-build-api", + "description": "Build a Ball UVM simulation", + "flows": ["uvm"], + "triggers": [api("POST", "/uvm/build")], + "enqueues": ["uvm.build"], +} + + +def check_args(body: dict) -> str | None: + allowed = {"design", "device", "ball", "ip"} + for key in body: + if key not in allowed: + return f"Unexpected parameter: --{key}" + if body.get("design") is True or ("design" in body and not body["design"]): + return "Parameter --design requires a value" + if not body.get("design") and not isinstance(body.get("ip"), str): + return "Standalone UVM requires --ip" + if body.get("ball") is True: + return "Parameter --ball requires a value" + if body.get("ip") is True: + return "Parameter --ip requires a value" + if body.get("ball") and not body.get("design"): + return "Ball UVM requires --design and --device " + if body.get("ball") and body.get("ip"): + return "Parameters --ball and --ip are mutually exclusive" + return None + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + error = check_args(body) + if error: + return ApiResponse(status=400, body={"error": error}) + # A design is verified through one of its device configs. + selection = None + if "design" in body: + try: + selection = require_target( + {key: body[key] for key in ("design", "device") if key in body}, + ("device",), + ) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + + await ctx.enqueue( + { + "topic": "uvm.build", + "data": {**body, "_trace_id": ctx.trace_id, "target": selection}, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/uvm/01_build_event.step.py b/bbdev/api/steps/uvm/01_build_event.step.py new file mode 100644 index 000000000..e6dbf5538 --- /dev/null +++ b/bbdev/api/steps/uvm/01_build_event.step.py @@ -0,0 +1,70 @@ +import asyncio +import os +import sys + +from motia import FlowContext, queue + +step_dir = os.path.dirname(os.path.abspath(__file__)) +utils_path = os.path.abspath(os.path.join(step_dir, "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) +if step_dir not in sys.path: + sys.path.insert(0, step_dir) + +from utils.event_common import check_result, get_origin_trace_id +from utils.path import get_buckyball_path +from scripts.run import run_verification +from scripts.ball import run_design + +config = { + "name": "uvm-build", + "description": "Build a UVM simulation", + "flows": ["uvm"], + "triggers": [queue("uvm.build")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + + try: + if input_data.get("ball") is not None: + info = await asyncio.to_thread( + run_design, + bbdir, + input_data["design"], + input_data["target"], + input_data["ball"], + ctx, + False, + ) + else: + info = await asyncio.to_thread( + run_verification, + bbdir, + input_data.get("design"), + input_data["target"], + input_data.get("ip"), + ctx, + False, + ) + except Exception as e: + ctx.logger.error(str(e)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": str(e)}, + trace_id=origin_tid, + ) + return + + await check_result( + ctx, + 1 if info.get("failures") else 0, + continue_run=False, + extra_fields={"task": "build", **info}, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/uvm/02_run_api.step.py b/bbdev/api/steps/uvm/02_run_api.step.py new file mode 100644 index 000000000..0ec71ab99 --- /dev/null +++ b/bbdev/api/steps/uvm/02_run_api.step.py @@ -0,0 +1,55 @@ +from utils.event_common import require_target +from motia import ApiRequest, ApiResponse, FlowContext, api + +config = { + "name": "uvm-run-api", + "description": "Build and run a Ball UVM simulation", + "flows": ["uvm"], + "triggers": [api("POST", "/uvm/run")], + "enqueues": ["uvm.run"], +} + + +def check_args(body: dict) -> str | None: + allowed = {"design", "device", "ball", "ip"} + for key in body: + if key not in allowed: + return f"Unexpected parameter: --{key}" + if body.get("design") is True or ("design" in body and not body["design"]): + return "Parameter --design requires a value" + if not body.get("design") and not isinstance(body.get("ip"), str): + return "Standalone UVM requires --ip" + if body.get("ball") is True: + return "Parameter --ball requires a value" + if body.get("ip") is True: + return "Parameter --ip requires a value" + if body.get("ball") and not body.get("design"): + return "Ball UVM requires --design and --device " + if body.get("ball") and body.get("ip"): + return "Parameters --ball and --ip are mutually exclusive" + return None + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + error = check_args(body) + if error: + return ApiResponse(status=400, body={"error": error}) + # A design is verified through one of its device configs. + selection = None + if "design" in body: + try: + selection = require_target( + {key: body[key] for key in ("design", "device") if key in body}, + ("device",), + ) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + + await ctx.enqueue( + { + "topic": "uvm.run", + "data": {**body, "_trace_id": ctx.trace_id, "target": selection}, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/uvm/02_run_event.step.py b/bbdev/api/steps/uvm/02_run_event.step.py new file mode 100644 index 000000000..f08116123 --- /dev/null +++ b/bbdev/api/steps/uvm/02_run_event.step.py @@ -0,0 +1,70 @@ +import asyncio +import os +import sys + +from motia import FlowContext, queue + +step_dir = os.path.dirname(os.path.abspath(__file__)) +utils_path = os.path.abspath(os.path.join(step_dir, "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) +if step_dir not in sys.path: + sys.path.insert(0, step_dir) + +from utils.event_common import check_result, get_origin_trace_id +from utils.path import get_buckyball_path +from scripts.run import run_verification +from scripts.ball import run_design + +config = { + "name": "uvm-run", + "description": "Build and run a UVM simulation", + "flows": ["uvm"], + "triggers": [queue("uvm.run")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + + try: + if input_data.get("ball") is not None: + info = await asyncio.to_thread( + run_design, + bbdir, + input_data["design"], + input_data["target"], + input_data["ball"], + ctx, + True, + ) + else: + info = await asyncio.to_thread( + run_verification, + bbdir, + input_data.get("design"), + input_data["target"], + input_data.get("ip"), + ctx, + True, + ) + except Exception as e: + ctx.logger.error(str(e)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": str(e)}, + trace_id=origin_tid, + ) + return + + await check_result( + ctx, + 1 if info.get("failures") else 0, + continue_run=False, + extra_fields={"task": "run", **info}, + trace_id=origin_tid, + ) diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/README.md b/bbdev/api/steps/uvm/scripts/__init__.py similarity index 100% rename from arch/src/main/scala/framework/mem-core/chi/src/main/resources/README.md rename to bbdev/api/steps/uvm/scripts/__init__.py diff --git a/bbdev/api/steps/uvm/scripts/ball.py b/bbdev/api/steps/uvm/scripts/ball.py new file mode 100644 index 000000000..6812d500d --- /dev/null +++ b/bbdev/api/steps/uvm/scripts/ball.py @@ -0,0 +1,381 @@ +import os +import re +import shlex +import sys +import tomllib +from datetime import datetime +from pathlib import Path +from utils.path import design_arch_root, log_dir +from steps.mill.scripts.mill import mill_cmd +from utils.stream_run import stream_run_logger + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "config/scripts")) + + +def load_design(bbdir: str, design: str, selection: str): + from utils.path import load_design_config + + msg = load_design_config(bbdir, design, selection=selection) + if not msg.name or not msg.device.cores: + raise ValueError(f"empty design.pb of {design} {selection}") + return msg + + +def ball_domain(design, ball: str | None = None): + domains = [ + core.accelerator.balldomain + for core in design.device.cores + if core.HasField("accelerator") + ] + domains.extend(thread.balldomain for thread in design.threads) + domains = [domain for domain in domains if domain.balls] + if not domains: + raise ValueError("design.pb has no BallDomain instances") + if ball is None: + domain = domains[0] + if any(other.balls != domain.balls for other in domains[1:]): + raise ValueError( + "design.pb has different BallDomains; select a Ball explicitly" + ) + return domain + matches = [ + (domain, mapping) + for domain in domains + for mapping in domain.balls + if mapping.component == ball + ] + if not matches: + raise ValueError(f"ball {ball!r} not in design.pb") + domain, mapping = matches[0] + if any(other != mapping for _, other in matches[1:]): + raise ValueError(f"ball {ball!r} has different instance definitions") + return domain + + +def selected_mappings(domain, ball: str | None): + if ball is None: + return list(domain.balls) + for m in domain.balls: + if m.component == ball: + return [m] + raise ValueError(f"ball {ball!r} not in design.pb") + + +def ball_owner(design, ball): + owners = [] + for core in design.device.cores: + accelerators = [core.accelerator] if core.HasField("accelerator") else [] + accelerators.extend(design.threads[index] for index in core.thread_indices) + owners.extend( + (acc, core) + for acc in accelerators + if any(item.component == ball for item in acc.balldomain.balls) + ) + if not owners: + raise ValueError(f"ball {ball!r} not in design.pb") + # The first owner elaborates the unsuffixed Ball module that tb_top + # instantiates. Owners may differ only in bank count, which changes the + # group and column port widths of their own module copies. + accelerator, core = owners[0] + for other, owner in owners[1:]: + if ( + other.mem.bank.entries != accelerator.mem.bank.entries + or other.mem.bank.width != accelerator.mem.bank.width + or other.mem.bank.mask_len != accelerator.mem.bank.mask_len + or other.frontend.rob_entries != accelerator.frontend.rob_entries + or other.frontend.sub_rob_depth != accelerator.frontend.sub_rob_depth + or owner.shared_mem != core.shared_mem + or owner.virtual_bank_count != core.virtual_bank_count + ): + raise ValueError( + f"ball {ball!r} has different verification interfaces across instances" + ) + return accelerator, core + + +def vcs_defines(domain, mapping, thread, core, parameters): + shared_bank_num = ( + core.shared_mem.entries // core.shared_mem.bank_entries + if core.shared_mem.enable + else 0 + ) + max_group_count = max(thread.mem.bank.num, shared_bank_num) + defs = [ + f"+define+BB_IN_BW={mapping.in_bw}", + f"+define+BB_OUT_BW={mapping.out_bw}", + f"+define+BB_MMIO_READ_BW={mapping.mmio_read_bw}", + f"+define+BB_MMIO_WRITE_BW={mapping.mmio_write_bw}", + f"+define+BB_BANK_ID_W={thread.frontend.bank_id_len}", + f"+define+BB_GROUP_COUNT_W={max_group_count.bit_length()}", + f"+define+BB_GROUP_ID_W={(max_group_count - 1).bit_length()}", + f"+define+BB_BANK_ADDR_W={(thread.mem.bank.entries - 1).bit_length()}", + f"+define+BB_ROB_ID_W={(thread.frontend.rob_entries - 1).bit_length()}", + ] + for e in mapping.instructions: + defs.append(f"+define+{e.mnemonic}_FUNCT7={e.funct7}") + from ball_parameters import unpack_parameters + + typed = unpack_parameters(mapping) + for define, parameter in parameters.items(): + defs.append(f"+define+{define}={getattr(typed, parameter)}") + return defs + + +def _filelist( + verify_dir: Path, + ball_dir: str, + uvm_rel: str, + rtl_dir: Path, + sim_dir: Path, +) -> str: + src = verify_dir / "filelists" / f"{ball_dir}_ball.f" + dst = sim_dir / f"{ball_dir}_ball.f" + sim_dir.mkdir(parents=True, exist_ok=True) + text = src.read_text() + dst.write_text( + text.replace("@UVM@", uvm_rel).replace("@RTL@", str(rtl_dir.resolve())) + ) + return str(dst) + + +def check_uvm_result( + result, target: str, expected_assertion: str | None = None +) -> None: + output = result.stdout + result.stderr + if expected_assertion is not None: + assertions = re.findall(r"Assertion failed: ([^\r\n]+)", output) + failure_counts = re.findall(r"UVM_(?:ERROR|FATAL)\s*:\s*(\d+)", output) + if ( + not assertions + or set(assertions) != {expected_assertion} + or "Fatal:" not in output + or any(int(count) for count in failure_counts) + or result.returncode < 0 + ): + raise RuntimeError( + f"Expected RTL assertion did not terminate {target}: {expected_assertion}" + ) + return + if result.returncode != 0: + raise RuntimeError(f"UVM failed for {target}") + counts = dict(re.findall(r"UVM_(ERROR|FATAL)\s*:\s*(\d+)", output)) + if set(counts) != {"ERROR", "FATAL"}: + raise RuntimeError(f"UVM report summary missing for {target}") + if counts != {"ERROR": "0", "FATAL": "0"}: + raise RuntimeError( + f"UVM failed for {target}: errors={counts['ERROR']} fatals={counts['FATAL']}" + ) + + +def build_ball( + bbdir: str, design_name: str, selection: str, mill_cfg: str, domain, mapping, ctx +) -> None: + ball = mapping.component + verify_dir = Path(bbdir) / "examples" / "prefab" / "balls" / ball / "verify" + casegen = verify_dir / "casegen" / "Cargo.toml" + rtl_dir = Path(design_arch_root(bbdir, design_name, selection=selection)) / ( + f"verilog/{mill_cfg}" + ) + design = load_design(bbdir, design_name, selection) + thread, core = ball_owner(design, ball) + sim_dir = verify_dir.parent / "output" / "uvm" / design_name / selection + uvm_rel = os.path.relpath(Path(bbdir) / "verify" / "uvm", verify_dir) + flist = _filelist(verify_dir, ball, uvm_rel, rtl_dir, sim_dir) + parameters_path = verify_dir / "parameters.toml" + parameters = ( + tomllib.loads(parameters_path.read_text())["defines"] + if parameters_path.is_file() + else {} + ) + cargo = ( + f"nix develop {shlex.quote(str(Path(bbdir) / 'verify'))} --command " + f"cargo build --manifest-path {shlex.quote(str(casegen))} " + f"--target-dir {shlex.quote(str(verify_dir.parent / 'output' / 'casegen'))}" + ) + r = stream_run_logger( + cmd=cargo, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="uvm dpi", + stderr_prefix="uvm dpi", + ) + if r.returncode != 0: + raise RuntimeError(f"cargo build failed for {ball}") + crate = tomllib.loads(casegen.read_text()).get("package", {}).get("name") + if not crate: + raise ValueError(f"package.name missing in {casegen}") + simv = sim_dir / "simv" + csrc = sim_dir / "csrc" + hier = sim_dir / "cm_hier.cfg" + sim_dir.mkdir(parents=True, exist_ok=True) + hier.write_text("+tree tb_top.dut\n") + script = ( + f"cd {shlex.quote(str(verify_dir))} && " + f"rm -rf {shlex.quote(str(csrc))} {shlex.quote(str(simv))} {shlex.quote(str(simv))}.daidir && " + f"mkdir -p {shlex.quote(str(sim_dir))} {shlex.quote(str(csrc))} && " + "vcs -full64 -sverilog -timescale=1ns/1ps -debug_access+all -top tb_top " + "${=VCS_UVM_ARGS} " + + " ".join( + shlex.quote(d) + for d in vcs_defines(domain, mapping, thread, core, parameters) + ) + + f" -cm line+cond+tgl+assert -cm_hier {shlex.quote(str(hier))} " + f"-Mdir={shlex.quote(str(csrc))} -o {shlex.quote(str(simv))} " + f"-f {shlex.quote(flist)}" + ) + vcs = f"nix develop {shlex.quote(str(Path(bbdir) / 'verify'))} --command zsh -c {shlex.quote(script)}" + r = stream_run_logger( + cmd=vcs, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="uvm vcs", + stderr_prefix="uvm vcs", + ) + if r.returncode != 0: + raise RuntimeError(f"vcs failed for {ball}") + + +def run_ball( + bbdir: str, + design_name: str, + selection: str, + mill_cfg: str, + domain, + mapping, + ctx, + cov_dir: str, +) -> None: + ball = mapping.component + verify_dir = Path(bbdir) / "examples" / "prefab" / "balls" / ball / "verify" + simv = verify_dir.parent / "output" / "uvm" / design_name / selection / "simv" + build_ball(bbdir, design_name, selection, mill_cfg, domain, mapping, ctx) + crate = tomllib.loads((verify_dir / "casegen" / "Cargo.toml").read_text())[ + "package" + ]["name"] + dpi = ( + verify_dir.parent + / "output" + / "casegen" + / "debug" + / f"lib{crate.replace('-', '_')}" + ) + test = f"{ball}_ball_test" + verify_config = ( + verify_dir.parent + / "output" + / "uvm" + / design_name + / selection + / "verify_config.env" + ) + design = load_design(bbdir, design_name, selection) + thread, _ = ball_owner(design, ball) + verify_config.write_text( + f"bank_entries={thread.mem.bank.entries}\nball_id={mapping.id}\n" + ) + simv_q = shlex.quote(str(simv)) + script = ( + f"cd {shlex.quote(str(verify_dir))} && " + f'loader="$(patchelf --print-interpreter {simv_q})"; ' + f'library_path="$(patchelf --print-rpath {simv_q})"; ' + f"BB_VERIFY_CONFIG={shlex.quote(str(verify_config))} " + f'"$loader" --library-path "$library_path" {simv_q} -no_save ' + f"-sv_lib {shlex.quote(str(dpi))} " + f"+UVM_TESTNAME={shlex.quote(test)} +BID={mapping.id} " + f"-cm line+cond+tgl+assert -cm_name {shlex.quote(test)}" + ) + cmd = f"nix develop {shlex.quote(str(Path(bbdir) / 'verify'))} --command zsh -ic {shlex.quote(script)}" + r = stream_run_logger( + cmd=cmd, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="uvm run", + stderr_prefix="uvm run", + ) + check_uvm_result(r, f"{ball} test={test}") + Path(cov_dir).mkdir(parents=True, exist_ok=True) + urg = ( + f"nix develop {shlex.quote(str(Path(bbdir) / 'verify'))} --command " + f"urg -dir {shlex.quote(str(simv))}.vdb -format text -report {shlex.quote(cov_dir)}" + ) + r = stream_run_logger( + cmd=urg, + logger=ctx.logger, + cwd=str(verify_dir), + stdout_prefix="uvm urg", + stderr_prefix="uvm urg", + ) + if r.returncode != 0: + raise RuntimeError(f"urg failed for {ball}") + + +def dashboard_summary(cov_dir: str) -> dict[str, str]: + path = Path(cov_dir) / "dashboard.txt" + lines = path.read_text().splitlines() + i = 0 + while i < len(lines) and "Total Coverage Summary" not in lines[i]: + i += 1 + if i >= len(lines): + raise RuntimeError(f"Total Coverage Summary missing in {path}") + i += 1 + while i < len(lines) and not lines[i].strip().startswith("SCORE"): + i += 1 + if i + 1 >= len(lines): + raise RuntimeError(f"SCORE row missing in {path}") + keys = lines[i].split() + vals = lines[i + 1].split() + if len(vals) != len(keys): + raise RuntimeError(f"SCORE/value mismatch in {path}: {keys!r} vs {vals!r}") + return dict(zip(keys, vals)) + + +def run_design( + bbdir: str, design: str, selection: str, ball: str | None, ctx, do_run: bool +) -> dict: + msg = load_design(bbdir, design, selection) + domain = ball_domain(msg, ball) + mill_cfg = msg.simulation.top + if not mill_cfg: + raise ValueError(f"design {design!r} has no sims.verilator for UVM") + maps = selected_mappings(domain, ball) + ran = [] + covs = [] + run_root = None + if do_run: + stamp = datetime.now().strftime("%Y-%m-%d-%H-%M") + run_name = "all" if ball is None else maps[0].component + run_root = log_dir( + bbdir, design, "verilog", stamp, "uvm", run_name, selection=selection + ) + for m in maps: + if do_run: + cov = ( + os.path.join(run_root, m.component, "coverage") + if ball is None + else os.path.join(run_root, "coverage") + ) + run_ball(bbdir, design, selection, mill_cfg, domain, m, ctx, cov) + if not (Path(cov) / "dashboard.txt").is_file(): + raise RuntimeError(f"urg wrote no dashboard.txt under {cov}") + covs.append((m.component, cov)) + else: + build_ball(bbdir, design, selection, mill_cfg, domain, m, ctx) + ran.append(m.component) + info = {"design": design, "mill": mill_cfg, "balls": ran} + if do_run and ball is None: + index_dir = Path(run_root) / "coverage" + index_dir.mkdir(parents=True, exist_ok=True) + body = ["ball_dir score line cond toggle assert group result"] + for b, p in covs: + s = dashboard_summary(p) + body.append( + f"{b} {s['SCORE']} {s['LINE']} {s['COND']} {s['TOGGLE']} " + f"{s.get('ASSERT', '--')} {s.get('GROUP', '--')} pass" + ) + index = index_dir / "index.txt" + index.write_text("\n".join(body) + "\n") + info["index"] = str(index) + info["log"] = run_root + elif do_run: + info["log"] = run_root + return info diff --git a/bbdev/api/steps/uvm/scripts/build.py b/bbdev/api/steps/uvm/scripts/build.py new file mode 100644 index 000000000..b6d90b93c --- /dev/null +++ b/bbdev/api/steps/uvm/scripts/build.py @@ -0,0 +1,186 @@ +import fcntl +import json +import shlex +import shutil +from pathlib import Path + +from steps.mill.scripts.mill import mill_cmd +from utils.stream_run import stream_run_logger + + +class VerificationFailure(RuntimeError): + def __init__(self, stage, test, logfile): + self.stage, self.test, self.logfile = stage, test, str(logfile) + super().__init__(f"{stage} failed: {test}; see {logfile}") + + +def execute(repo, arguments, cwd, logfile, timeout, ctx): + logfile = Path(logfile) + logfile.parent.mkdir(parents=True, exist_ok=True) + command = [ + "nix", + "develop", + str(Path(repo) / "verify"), + "--command", + "timeout", + "--kill-after=5s", + f"{timeout}s", + *map(str, arguments), + ] + result = stream_run_logger( + command, + ctx.logger, + cwd=str(cwd), + shell=False, + timeout=timeout + 30, + stdout_prefix="verification", + stderr_prefix="verification", + ) + logfile.write_text(result.stdout + "\n" + result.stderr) + return result + + +def build(repo, design, profile, logdir, ctx): + repo = Path(repo) + root = repo / profile["root"] + scope = design if design is not None else "default" + output = root / "build/verification" / scope / profile["profile"] + output.mkdir(parents=True, exist_ok=True) + lock = (root / "build/.verification.lock").open("w") + with lock: + fcntl.flock(lock, fcntl.LOCK_EX) + model = profile["model"] + if model["manifest"]: + result = execute( + repo, + [ + "cargo", + "build", + "--manifest-path", + root / model["manifest"], + "--target-dir", + root / "build/verification/reference", + ], + root, + Path(logdir) / "reference.log", + 600, + ctx, + ) + if result.returncode: + raise VerificationFailure( + "reference", "", Path(logdir) / "reference.log" + ) + parameters = output / "parameters.json" + parameters.write_text(json.dumps(profile["parameters"])) + rtl = output / "rtl" + registration = profile["build"] + if registration["kind"] != "mill": + raise ValueError(f"Unknown emitter kind: {registration['kind']}") + replacements = { + "{repo}": str(repo), + "{design}": design, + "{config}": str(parameters), + "{rtl}": str(rtl), + "{targets}": ",".join(test["name"] for test in profile["tests"]), + } + arguments = [] + for argument in registration["arguments"]: + for token, value in replacements.items(): + if token in argument: + if value is None: + raise ValueError( + f"{profile['ip']} requires a design for {token}" + ) + argument = argument.replace(token, value) + arguments.append(argument) + if profile["parameters"] and not any( + "{config}" in value for value in registration["arguments"] + ): + raise ValueError(f"Emitter does not consume parameters: {profile['ip']}") + command = mill_cmd( + repo, + registration["directory"], + registration["module"] + ".run", + arguments, + ) + result = execute( + repo, + ["zsh", "-c", command], + repo / registration["directory"], + Path(logdir) / "emit.log", + 900, + ctx, + ) + if result.returncode: + raise VerificationFailure("emit", "", Path(logdir) / "emit.log") + if not any("{rtl}" in value for value in registration["arguments"]): + shutil.copytree( + root / "build", + rtl, + dirs_exist_ok=True, + ignore=shutil.ignore_patterns( + "verification", "uvm", ".verification.lock", ".build.lock" + ), + ) + resources = root / profile["resources"] + compiled = {} + binaries = {} + for test in profile["tests"]: + directory = output / test["name"] + directory.mkdir(parents=True, exist_ok=True) + source = resources / test["filelist"] + filelist = directory / "filelist.f" + filelist.write_text( + source.read_text() + .replace("@VERIFY@", str(repo / "verify")) + .replace("@RESOURCES@", str(source.parent)) + .replace("@RTL@", str(rtl)) + ) + signature = ( + filelist.read_text(), + test["top"], + tuple(test["metrics"]), + tuple(test["rtl_scopes"]), + ) + if signature in compiled: + binaries[test["name"]] = compiled[signature] + continue + hierarchy = directory / "coverage.cfg" + hierarchy.write_text( + "".join( + f"+tree {test['top']}.{scope}\n" for scope in test["rtl_scopes"] + ) + ) + metrics = "+".join([*test["metrics"], "assert"]) + command = [ + "uvm-compile", + "-top", + test["top"], + "-debug_access+all", + "-f", + filelist, + "-o", + directory / "simv", + "-Mdir=" + str(directory / "csrc"), + *[ + f"+define+{name}={value}" + for name, value in profile["defines"].items() + ], + ] + if test["metrics"]: + command.extend(["-cm", metrics, "-cm_hier", hierarchy]) + result = execute( + repo, + command, + directory, + Path(logdir) / test["name"] / "compile.log", + 600, + ctx, + ) + if result.returncode: + raise VerificationFailure( + "compile", test["name"], Path(logdir) / test["name"] / "compile.log" + ) + compiled[signature] = directory / "simv" + binaries[test["name"]] = directory / "simv" + return {"output": output, "binaries": binaries} diff --git a/bbdev/api/steps/uvm/scripts/config.py b/bbdev/api/steps/uvm/scripts/config.py new file mode 100644 index 000000000..20946d672 --- /dev/null +++ b/bbdev/api/steps/uvm/scripts/config.py @@ -0,0 +1,57 @@ +import json +import sys +from pathlib import Path +from google.protobuf.json_format import MessageToDict + + +def profiles(repo, design, selection, ip): + repo = Path(repo) + sys.path.insert(0, str(repo / "bbdev/api/steps/config/scripts")) + import design_pb2 + + if design is None: + from verification import default_ip_config + + path = default_ip_config(repo, ip) + else: + path = ( + repo + / "examples/designs" + / design + / "output/sims" + / selection + / "generated/design.pb" + ) + config = design_pb2.Design.FromString(path.read_bytes()) + entries = decode_profiles(config) + selected = [entry for entry in entries if ip is None or entry["ip"] == ip] + if not selected: + raise ValueError( + f"No IP verification registrations for design={design!r} ip={ip!r}" + ) + result = [] + for entry in selected: + tests = [ + test + for test in entry["tests"] + if not test["designs"] or design in test["designs"] + ] + if not tests: + raise ValueError(f"No applicable tests: {entry['ip']}/{entry['profile']}") + result.append({**entry, "tests": tests}) + return result + + +def decode_profiles(config): + entries = [] + for descriptor in config.verification.profiles: + entry = MessageToDict( + descriptor, + preserving_proto_field_name=True, + always_print_fields_with_no_presence=True, + ) + entry["parameters"] = json.loads(entry.pop("parameters_json")) + for test in entry["tests"]: + test["timeout_ns"] = int(test["timeout_ns"]) + entries.append(entry) + return entries diff --git a/bbdev/api/steps/uvm/scripts/coverage.py b/bbdev/api/steps/uvm/scripts/coverage.py new file mode 100644 index 000000000..f249e9cb5 --- /dev/null +++ b/bbdev/api/steps/uvm/scripts/coverage.py @@ -0,0 +1,75 @@ +import re +from pathlib import Path + +from .build import VerificationFailure, execute + + +def summary(directory): + lines = (Path(directory) / "dashboard.txt").read_text().splitlines() + start = lines.index("Total Coverage Summary ") + for index in range(start + 1, len(lines)): + if lines[index].strip().startswith("SCORE"): + return { + name.lower(): None if value == "--" else float(value) + for name, value in zip( + lines[index].split(), lines[index + 1].split(), strict=True + ) + } + raise ValueError(f"Missing coverage summary: {directory}") + + +def collect(repo, test, database, output, ctx): + output = Path(output) + hierarchy = output.parent / "dut.cfg" + hierarchy.write_text( + "".join(f"+tree {test['top']}.{scope}\n" for scope in test["rtl_scopes"]) + ) + commands = [ + ( + ["urg", "-dir", database, "-format", "text", "-report", output / "all"], + "all", + ), + ( + [ + "urg", + "-dir", + database, + "-format", + "text", + "-hier", + hierarchy, + "-metric", + "+".join(test["metrics"]), + "-report", + output / "dut", + ], + "dut", + ), + ] + for command, name in commands: + logfile = output.parent / f"coverage-{name}.log" + result = execute(repo, command, output.parent, logfile, 120, ctx) + if result.returncode: + raise VerificationFailure("coverage", test["name"], logfile) + values = summary(output / "dut") + for metric in test["metrics"]: + name = {"tgl": "toggle", "cond": "cond", "line": "line"}[metric] + if values[name] is None: + raise VerificationFailure( + "coverage_metric", test["name"], output / "dut/dashboard.txt" + ) + groups = {} + if test["functional_groups"]: + text = (output / "all/grpinfo.txt").read_text() + for name in test["functional_groups"]: + match = re.search( + r"Group : " + re.escape(name) + r"\s.*?\nSCORE[^\n]*\n\s*([\d.]+)", + text, + re.DOTALL, + ) + if match is None: + raise VerificationFailure( + "functional_coverage", test["name"], output / "all/groups.txt" + ) + groups[name] = float(match[1]) + return {"dut": values, "functional": groups, "report": str(output)} diff --git a/bbdev/api/steps/uvm/scripts/report.py b/bbdev/api/steps/uvm/scripts/report.py new file mode 100644 index 000000000..79183b6fd --- /dev/null +++ b/bbdev/api/steps/uvm/scripts/report.py @@ -0,0 +1,96 @@ +from pathlib import Path + +from utils.reports import new_report, attachment, finish_report +from utils.path import load_design_config +from .uvm_common import dashboard_summary, load_ip + + +def verification_targets(repo, design, selection, ball=None, ip=None): + if ip is not None: + return [ + f"ip/{ip}/{item['name']}" + for profile in load_ip(str(repo), ip, design, selection) + for item in profile["targets"] + ] + if ball is not None: + return [f"ball/{ball}"] + config = load_design_config(repo, design, selection=selection).verification + targets = [f"ball/{ball}" for ball in config.balls] + for ip in config.ips: + targets.extend( + f"ip/{ip}/{target['name']}" + for profile in load_ip(str(repo), ip, design, selection) + for target in profile["targets"] + ) + return targets + + +def coverage_report(context, trace_id, info, output, expected_targets): + report = new_report(context, "uvm.run", trace_id, "uvm") + for result in info["results"]: + if "balls" in result: + rows = [ + ( + f"ball/{name}", + Path(result["log"]) + / (f"{name}/coverage" if len(result["balls"]) > 1 else "coverage"), + ) + for name in result["balls"] + ] + else: + rows = [ + ( + f"ip/{result['ip']}/{target}", + Path(result["log"]) / target / "coverage", + ) + for target in result["targets"] + ] + for target, directory in rows: + report["verification_targets"].append(target) + if ( + result.get("expected_assertions") + and target.removeprefix(f"ip/{result['ip']}/") + in result["expected_assertions"] + ): + report["verification"].append( + { + "target": target, + "status": "pass", + "coverage": None, + "report_url": None, + } + ) + continue + values = dashboard_summary(str(directory)) + url = attachment( + report, + output, + directory / "dashboard.txt", + f"coverage-{target.replace('/', '-')}.txt", + ) + report["verification"].append( + { + "target": target, + "status": "pass", + "coverage": { + name: float(value) + for name, value in values.items() + if value != "--" + }, + "report_url": url, + } + ) + for failure in info["failures"]: + target = failure["target"] + report["verification_targets"].append(target) + report["verification"].append( + { + "target": target, + "status": "fail", + "coverage": None, + "report_url": None, + "error": failure["error"], + } + ) + report["verification_targets"] = expected_targets + return finish_report(report, output) diff --git a/bbdev/api/steps/uvm/scripts/run.py b/bbdev/api/steps/uvm/scripts/run.py new file mode 100644 index 000000000..f505dbcc4 --- /dev/null +++ b/bbdev/api/steps/uvm/scripts/run.py @@ -0,0 +1,210 @@ +import json +import re +import shutil +from datetime import datetime +from pathlib import Path + +from utils.path import log_dir +from .build import build, execute, VerificationFailure +from .config import profiles +from .coverage import collect + + +def judge(test, process, result_file): + text = process.stdout + "\n" + process.stderr + logfile = result_file.parent / "simulation.log" + if process.returncode in (124, 137): + raise VerificationFailure("wall_timeout", test["name"], logfile) + if "[PH_TIMEOUT]" in text: + raise VerificationFailure("simulation_timeout", test["name"], logfile) + severities = dict(re.findall(r"UVM_(ERROR|FATAL)\s*:\s*(\d+)", text)) + if test["kind"] == "assertion": + assertions = re.findall(r"Assertion failed: ([^\r\n]+)", text) + passed = ( + bool(assertions) + and set(assertions) == {test["assertion"]} + and "Fatal:" in text + ) + passed = passed and not any(int(value) for value in severities.values()) + if not passed: + raise VerificationFailure("expected_assertion", test["name"], logfile) + return {"status": "pass", "checks": 1} + if process.returncode: + raise VerificationFailure("simulation", test["name"], logfile) + if any(int(value) for value in severities.values()): + raise VerificationFailure("checking", test["name"], logfile) + if not result_file.is_file(): + raise VerificationFailure("result_missing", test["name"], logfile) + data = json.loads(result_file.read_text()) + required = { + "status", + "comparisons", + "checks", + "expected", + "received", + "cancelled", + "pending", + } + if set(data) != required: + raise ValueError(f"Invalid verification evidence: {result_file}") + passed = data["status"] == "pass" and data["comparisons"] + data["checks"] > 0 + passed = ( + passed + and data["pending"] == 0 + and data["expected"] == data["comparisons"] + data["cancelled"] + ) + passed = ( + passed + and data["received"] == data["comparisons"] + and severities == {"ERROR": "0", "FATAL": "0"} + ) + passed = passed and not re.search( + r"^(?:Error|Fatal):|^UVM_(?:ERROR|FATAL)\s+(?!:)", text, re.MULTILINE + ) + if not passed: + raise VerificationFailure("checking", test["name"], logfile) + return data + + +def run_profile(repo, design, profile, directory, do_run, ctx): + directory = Path(directory) + directory.mkdir(parents=True, exist_ok=True) + (directory / "configuration.json").write_text(json.dumps(profile, indent=2) + "\n") + artifacts = build(repo, design, profile, directory, ctx) + output = artifacts["output"] + records = [] + if not do_run: + return { + "ip": profile["ip"], + "profile": profile["profile"], + "status": "built", + "output": str(output), + } + for test in profile["tests"]: + record_dir = directory / test["name"] + record_dir.mkdir(parents=True, exist_ok=True) + evidence = record_dir / "evidence.json" + if evidence.exists(): + raise FileExistsError(evidence) + binary = artifacts["binaries"][test["name"]] + if test["metrics"]: + shutil.copytree(Path(str(binary) + ".vdb"), record_dir / "simulation.vdb") + arguments = [ + "uvm-simulate", + binary, + "-no_save", + f"+UVM_TESTNAME={test['test']}", + f"+RESULT={evidence}", + f"+TIMEOUT_NS={test['timeout_ns']}", + f"+SINK_READY={test['sink_ready']}", + f"+SINK_STALL={test['sink_stall']}", + *test["plusargs"], + ] + if test["metrics"]: + arguments.extend( + [ + "-cm", + "+".join([*test["metrics"], "assert"]), + "-cm_dir", + record_dir / "simulation.vdb", + ] + ) + if profile["model"]["manifest"]: + arguments.extend( + [ + "-sv_lib", + Path(repo) + / profile["root"] + / "build/verification/reference/debug" + / profile["model"]["library"], + ] + ) + process = execute( + repo, + arguments, + binary.parent, + record_dir / "simulation.log", + test["wall_timeout_s"], + ctx, + ) + verdict = judge(test, process, evidence) + record = { + "ip": profile["ip"], + "profile": profile["profile"], + "test": test["name"], + "id": f"{profile['scope']}/{profile['ip']}/{profile['profile']}/{test['name']}", + "kind": test["kind"], + "configuration": profile["config_path"], + "log": str(record_dir / "simulation.log"), + **verdict, + "coverage": None, + } + if ( + verdict["status"] == "pass" + and test["kind"] == "functional" + and test["metrics"] + ): + # Runtime databases belong to this run, never the reusable build directory. + record["coverage"] = collect( + repo, test, record_dir / "simulation.vdb", record_dir / "coverage", ctx + ) + (record_dir / "result.json").write_text(json.dumps(record, indent=2) + "\n") + records.append(record) + return { + "ip": profile["ip"], + "profile": profile["profile"], + "configuration": profile["config_path"], + "status": "pass", + "tests": records, + } + + +def run_verification(repo, design, selection, ip, ctx, do_run): + selected = profiles(repo, design, selection, ip) + return run_suite(repo, design, selected, ip, ctx, do_run) + + +def run_suite(repo, design, selected, name, ctx, do_run): + scope = design if design is not None else selected[0]["scope"] + stamp = datetime.now().strftime("%Y-%m-%d-%H-%M-%S-%f") + directory = Path( + log_dir( + repo, scope, "verilog", stamp, "uvm", name if name is not None else "suite" + ) + ) + directory.mkdir(parents=True) + records = [] + for profile in selected: + records.append( + run_profile( + repo, + design, + profile, + directory / profile["ip"] / profile["profile"], + do_run, + ctx, + ) + ) + planned = [ + f"{profile['scope']}/{profile['ip']}/{profile['profile']}/{test['name']}" + for profile in selected + for test in profile["tests"] + ] + report = { + "design": design, + "operation": "run" if do_run else "build", + "planned_tests": planned, + "status": "pass", + "profiles": records, + "log": str(directory), + } + path = directory / "result.json" + path.write_text(json.dumps(report, indent=2) + "\n") + return { + "design": design, + "success": True, + "results": records, + "failures": [], + "report_path": str(path), + "log": str(directory), + } diff --git a/bbdev/api/steps/uvm/scripts/test_report.py b/bbdev/api/steps/uvm/scripts/test_report.py new file mode 100644 index 000000000..bbc7b8864 --- /dev/null +++ b/bbdev/api/steps/uvm/scripts/test_report.py @@ -0,0 +1,28 @@ +from pathlib import Path + + +def discover_tests(bbdir: str, design: str, ball: str) -> list[Path]: + root = ( + Path(bbdir) + / "bb-tests" + / "output" + / design + / "workloads" + / "src" + / "CTest" + / "designs" + / design + / "balls" + / design + / ball + / "ctests" + ) + return sorted(root.glob("*-baremetal")) + + +def write_test_index(path: Path, rows: list[tuple[str, str, str]]) -> None: + path.write_text( + "ball_dir bemu verilator\n" + + "\n".join(f"{ball} {bemu} {verilator}" for ball, bemu, verilator in rows) + + "\n" + ) diff --git a/bbdev/api/steps/uvm/scripts/uvm_common.py b/bbdev/api/steps/uvm/scripts/uvm_common.py new file mode 100644 index 000000000..543bfed02 --- /dev/null +++ b/bbdev/api/steps/uvm/scripts/uvm_common.py @@ -0,0 +1,1004 @@ +import json +import shutil +import fcntl +import os +import re +import shlex +import sys +import tomllib +from datetime import datetime +from pathlib import Path + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import design_arch_root, get_buckyball_path, log_dir +from steps.mill.scripts.mill import mill_cmd +from utils.stream_run import stream_run_logger + +config_scripts = os.path.abspath( + os.path.join(os.path.dirname(__file__), "..", "..", "config", "scripts") +) +sys.path.insert(0, config_scripts) + + +def load_design(bbdir: str, design: str, selection: str): + from utils.path import load_design_config + + msg = load_design_config(bbdir, design, selection=selection) + if not msg.name or not msg.device.cores: + raise ValueError(f"empty design.pb of {design} {selection}") + return msg + + +def load_design_uvm(bbdir: str, design: str, selection: str) -> dict[str, list[str]]: + msg = load_design(bbdir, design, selection) + config = {"balls": list(msg.verification.balls), "ips": list(msg.verification.ips)} + if not config["balls"] and not config["ips"]: + raise ValueError(f"design {design}: protobuf has no UVM targets") + return config + + +def ball_domain(design, ball: str | None = None): + domains = [ + core.accelerator.balldomain + for core in design.device.cores + if core.HasField("accelerator") + ] + domains.extend(thread.balldomain for thread in design.threads) + domains = [domain for domain in domains if domain.balls] + if not domains: + raise ValueError("design.pb has no BallDomain instances") + if ball is None: + domain = domains[0] + if any(other.balls != domain.balls for other in domains[1:]): + raise ValueError( + "design.pb has different BallDomains; select a Ball explicitly" + ) + return domain + matches = [ + (domain, mapping) + for domain in domains + for mapping in domain.balls + if mapping.component == ball + ] + if not matches: + raise ValueError(f"ball {ball!r} not in design.pb") + domain, mapping = matches[0] + if any(other != mapping for _, other in matches[1:]): + raise ValueError(f"ball {ball!r} has different instance definitions") + return domain + + +def selected_mappings(domain, ball: str | None): + if ball is None: + return list(domain.balls) + for m in domain.balls: + if m.component == ball: + return [m] + raise ValueError(f"ball {ball!r} not in design.pb") + + +def ball_owner(design, ball): + owners = [] + for core in design.device.cores: + accelerators = [core.accelerator] if core.HasField("accelerator") else [] + accelerators.extend(design.threads[index] for index in core.thread_indices) + owners.extend( + (acc, core) + for acc in accelerators + if any(item.component == ball for item in acc.balldomain.balls) + ) + if not owners: + raise ValueError(f"ball {ball!r} not in design.pb") + accelerator, core = owners[0] + for other, owner in owners[1:]: + if ( + other.mem.bank != accelerator.mem.bank + or other.frontend.rob_entries != accelerator.frontend.rob_entries + or other.frontend.sub_rob_depth != accelerator.frontend.sub_rob_depth + or owner.shared_mem != core.shared_mem + or owner.virtual_bank_count != core.virtual_bank_count + ): + raise ValueError( + f"ball {ball!r} has different verification interfaces across instances" + ) + return accelerator, core + + +def vcs_defines(domain, mapping, thread, core, parameters): + shared_bank_num = ( + core.shared_mem.entries // core.shared_mem.bank_entries + if core.shared_mem.enable + else 0 + ) + max_group_count = max(thread.mem.bank.num, shared_bank_num) + defs = [ + f"+define+BB_IN_BW={mapping.in_bw}", + f"+define+BB_OUT_BW={mapping.out_bw}", + f"+define+BB_MMIO_READ_BW={mapping.mmio_read_bw}", + f"+define+BB_MMIO_WRITE_BW={mapping.mmio_write_bw}", + f"+define+BB_BANK_ID_W={(core.virtual_bank_count - 1).bit_length()}", + f"+define+BB_GROUP_COUNT_W={max_group_count.bit_length()}", + f"+define+BB_GROUP_ID_W={(max_group_count - 1).bit_length()}", + f"+define+BB_BANK_ADDR_W={(thread.mem.bank.entries - 1).bit_length()}", + f"+define+BB_ROB_ID_W={(thread.frontend.rob_entries - 1).bit_length()}", + f"+define+BB_SUB_ROB_ID_W={(thread.frontend.sub_rob_depth * 4 - 1).bit_length()}", + ] + for e in mapping.instructions: + defs.append(f"+define+{e.mnemonic}_FUNCT7={e.funct7}") + from ball_parameters import unpack_parameters + + typed = unpack_parameters(mapping) + for define, parameter in parameters.items(): + defs.append(f"+define+{define}={getattr(typed, parameter)}") + return defs + + +def _filelist( + verify_dir: Path, + ball_dir: str, + uvm_rel: str, + rtl_dir: Path, + sim_dir: Path, +) -> str: + src = verify_dir / "filelists" / f"{ball_dir}_ball.f" + dst = sim_dir / f"{ball_dir}_ball.f" + sim_dir.mkdir(parents=True, exist_ok=True) + text = src.read_text() + dst.write_text( + text.replace("@UVM@", uvm_rel).replace("@RTL@", str(rtl_dir.resolve())) + ) + return str(dst) + + +def load_ip( + bbdir: str, name: str, design: str | None, selection: str | None +) -> list[dict]: + from verification import default_ip_config + + if design is not None: + msg = load_design(bbdir, design, selection) + else: + import design_pb2 + + msg = design_pb2.Design.FromString(default_ip_config(bbdir, name).read_bytes()) + profiles = [ + profile + for profile in json.loads(msg.verification.ip_registry_json) + if profile["ip"] == name + ] + if not profiles: + raise ValueError(f"IP {name!r} has no UVM registrations in design {design!r}") + return [load_ip_profile(bbdir, name, design, profile) for profile in profiles] + + +def load_ip_profile(bbdir: str, name: str, design: str | None, profile: dict) -> dict: + root = Path(bbdir) / profile["definition"]["root"] + build = profile["definition"]["build"] + if set(build) - {"directory", "module", "arguments", "runtime"}: + raise ValueError(f"Invalid IP build settings for {name!r}: {build}") + directory = build.get("directory", str(root.relative_to(bbdir))) + module = build.get("module", name) + arguments = build.get("arguments", []) + if not isinstance(arguments, list) or not all( + isinstance(value, str) for value in arguments + ): + raise ValueError(f"IP build arguments must be a list of strings: {name!r}") + if ( + not isinstance(directory, str) + or not directory + or not isinstance(module, str) + or not module + ): + raise ValueError( + f"IP build directory and module must be nonempty strings: {name!r}" + ) + resources = root / "src" / "main" / "resources" + targets = [] + for filelist in sorted([*resources.glob("*.f"), *resources.glob("*/filelist.f")]): + grouped = filelist.parent != resources + organized = filelist.name == "filelist.f" + target = ( + filelist.parent.name if grouped else name if organized else filelist.stem + ) + targets.append( + { + "name": target, + "filelist": str(filelist.relative_to(resources)), + "top": f"{target}_tb", + "test": "protocol_test", + "exclusions": ( + str((filelist.parent / "coverage.el").relative_to(resources)) + if organized + else f"{target}.el" + ), + "hierarchy": ( + str((filelist.parent / "coverage.cm_hier").relative_to(resources)) + if organized + else f"{target}.cm_hier" + ), + } + ) + for target in targets: + metadata = (resources / target["filelist"]).with_suffix(".toml") + if metadata.is_file(): + settings = tomllib.loads(metadata.read_text()) + if not settings or set(settings) - { + "expected_assertion", + "designs", + "invalid_case", + "rtl_scopes", + }: + raise ValueError(f"Invalid IP test settings: {metadata}") + if "rtl_scopes" in settings: + scopes = settings["rtl_scopes"] + if ( + not isinstance(scopes, list) + or not scopes + or any( + not isinstance(scope, str) + or not re.fullmatch( + r"[A-Za-z_][A-Za-z0-9_$]*(?:\[\d+\])?(?:\.[A-Za-z_][A-Za-z0-9_$]*(?:\[\d+\])?)*", + scope, + ) + for scope in scopes + ) + ): + raise ValueError(f"Invalid RTL instance scopes: {metadata}") + target["rtl_scopes"] = scopes + if "invalid_case" in settings: + case = settings["invalid_case"] + if ( + type(case) is not int + or case < 0 + or "expected_assertion" not in settings + ): + raise ValueError( + f"invalid_case requires a nonnegative integer and expected_assertion: {metadata}" + ) + target["invalid_case"] = case + if "expected_assertion" in settings: + assertion = settings["expected_assertion"] + if not isinstance(assertion, str) or not assertion or "\n" in assertion: + raise ValueError(f"Expected one assertion message in {metadata}") + target["expected_assertion"] = assertion + if "designs" in settings: + profiles = settings["designs"] + if ( + not isinstance(profiles, list) + or not profiles + or not all( + isinstance(value, str) + and value.strip() + and value == value.strip() + for value in profiles + ) + ): + raise ValueError( + f"designs must be a nonempty list of nonempty design names: {metadata}" + ) + if design is None: + raise ValueError( + f"Design is required to select declared IP profiles: {metadata}" + ) + target["designs"] = profiles + if not targets: + raise ValueError(f"registered IP {name!r} has no filelists under {resources}") + targets = [ + target + for target in targets + if "designs" not in target or design in target["designs"] + ] + if not targets: + raise ValueError(f"IP {name!r} has no applicable targets for design {design!r}") + for target in targets: + target["target"] = target["name"] + target["name"] = f"{profile['profile']}/{target['name']}" + return { + "profile": profile["profile"], + "config_path": profile["config_path"], + "parameters": profile["parameters"], + "defines": profile["defines"], + "root": str(root.relative_to(bbdir)), + "mill_module": module, + "mill_directory": directory, + "mill_arguments": arguments, + "runtime": build.get("runtime", []), + "model": "src/csrc/Cargo.toml", + "resources": "src/main/resources", + "targets": targets, + } + + +def _ip_filelist( + resources: Path, source: str, sim_dir: Path, bbdir: str, rtl: Path +) -> Path: + source_path = resources / source + dst = sim_dir / source_path.name + sim_dir.mkdir(parents=True, exist_ok=True) + text = source_path.read_text() + dst.write_text( + text.replace("@VERIFY@", str(Path(bbdir) / "verify")) + .replace("@RESOURCES@", str(source_path.parent)) + .replace("@RTL@", str(rtl)) + ) + return dst + + +def check_uvm_result( + result, target: str, expected_assertion: str | None = None +) -> None: + output = result.stdout + result.stderr + if expected_assertion is not None: + assertions = re.findall(r"Assertion failed: ([^\r\n]+)", output) + failure_counts = re.findall(r"UVM_(?:ERROR|FATAL)\s*:\s*(\d+)", output) + if ( + not assertions + or set(assertions) != {expected_assertion} + or "Fatal:" not in output + or any(int(count) for count in failure_counts) + or result.returncode < 0 + ): + raise RuntimeError( + f"Expected RTL assertion did not terminate {target}: {expected_assertion}" + ) + return + if result.returncode != 0: + raise RuntimeError(f"UVM failed for {target}") + counts = dict(re.findall(r"UVM_(ERROR|FATAL)\s*:\s*(\d+)", output)) + if set(counts) != {"ERROR", "FATAL"}: + raise RuntimeError(f"UVM report summary missing for {target}") + if counts != {"ERROR": "0", "FATAL": "0"}: + raise RuntimeError( + f"UVM failed for {target}: errors={counts['ERROR']} fatals={counts['FATAL']}" + ) + + +def build_ip( + bbdir: str, design: str | None, selection: str | None, name: str, ctx +) -> dict: + targets = [] + for config in load_ip(bbdir, name, design, selection): + build_ip_profile(bbdir, design, name, ctx, config) + targets.extend(config["targets"]) + return {"targets": targets} + + +def build_ip_profile( + bbdir: str, design: str | None, name: str, ctx, config: dict +) -> dict: + root = Path(bbdir) / config["root"] + resources = root / config["resources"] + manifest = root / config["model"] + rtl = root / "build" + sim_root = rtl / "uvm" / (design if design is not None else "default") + rtl.mkdir(parents=True, exist_ok=True) + build_lock = (rtl / ".build.lock").open("w") + fcntl.flock(build_lock, fcntl.LOCK_EX) + + cargo = ( + f"nix develop {shlex.quote(str(Path(bbdir) / 'verify'))} --command " + f"cargo build --manifest-path {shlex.quote(str(manifest))}" + ) + result = stream_run_logger( + cmd=cargo, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="uvm dpi", + stderr_prefix="uvm dpi", + ) + if result.returncode != 0: + raise RuntimeError(f"cargo build failed for IP {name}") + + profile_root = sim_root / config["profile"] + profile_root.mkdir(parents=True, exist_ok=True) + parameter_file = profile_root / "config.json" + parameter_file.write_text(json.dumps(config["parameters"])) + profile_rtl = profile_root / "rtl" + if design is None and any( + "{design}" in argument for argument in config["mill_arguments"] + ): + raise ValueError(f"IP {name} requires a design configuration") + if config["parameters"] and not any( + "{config}" in argument for argument in config["mill_arguments"] + ): + raise ValueError( + f"IP {name} emitter does not consume its UVM parameter configuration" + ) + arguments = [ + argument.replace("{repo}", str(Path(bbdir).resolve())) + .replace("{design}", design if design is not None else "") + .replace("{config}", str(parameter_file)) + .replace("{rtl}", str(profile_rtl)) + .replace( + "{targets}", ",".join(target["target"] for target in config["targets"]) + ) + for argument in config["mill_arguments"] + ] + if config["runtime"]: + classpath = [] + for entry in [ + {"directory": config["mill_directory"], "module": config["mill_module"]}, + *config["runtime"], + ]: + result = stream_run_logger( + cmd=mill_cmd( + bbdir, + entry["directory"], + entry["module"] + ".runClasspath", + show=True, + ), + logger=ctx.logger, + cwd=str(Path(bbdir) / entry["directory"]), + stdout_prefix="uvm classpath", + stderr_prefix="uvm classpath", + ) + if result.returncode != 0: + raise RuntimeError(f"Classpath build failed for IP {name}") + classpath.extend( + value.split(":", 3)[3] for value in json.loads(result.stdout) + ) + result = stream_run_logger( + cmd=mill_cmd( + bbdir, + config["mill_directory"], + config["mill_module"] + ".mainClass", + show=True, + ), + logger=ctx.logger, + cwd=str(Path(bbdir) / config["mill_directory"]), + stdout_prefix="uvm main", + stderr_prefix="uvm main", + ) + if result.returncode != 0: + raise RuntimeError(f"Main class lookup failed for IP {name}") + emit = shlex.join( + [ + str(Path(bbdir) / "result" / "bin" / "java"), + "-cp", + ":".join(dict.fromkeys(classpath)), + json.loads(result.stdout), + *arguments, + ] + ) + else: + emit = mill_cmd( + bbdir, config["mill_directory"], config["mill_module"] + ".run", arguments + ) + result = stream_run_logger( + cmd=emit, + logger=ctx.logger, + cwd=str(Path(bbdir) / config["mill_directory"]), + stdout_prefix="uvm rtl", + stderr_prefix="uvm rtl", + ) + if result.returncode != 0: + raise RuntimeError(f"RTL generation failed for IP {name}") + + if not any("{rtl}" in argument for argument in config["mill_arguments"]): + shutil.copytree( + rtl, + profile_rtl, + dirs_exist_ok=True, + ignore=shutil.ignore_patterns("uvm", ".build.lock"), + ) + for target in config["targets"]: + sim_dir = sim_root / target["name"] + filelist = _ip_filelist( + resources, target["filelist"], sim_dir, bbdir, profile_rtl + ) + simv = sim_dir / "simv" + csrc = sim_dir / "csrc" + hier = sim_dir / "cm_hier.cfg" + declared_hier = resources / target["hierarchy"] + if declared_hier.is_file(): + scopes = [ + line.strip() + for line in declared_hier.read_text().splitlines() + if line.strip() + ] + if not scopes or any( + not re.fullmatch( + r"\+tree @TOP@\.[A-Za-z_][A-Za-z0-9_$]*(?:\[\d+\])?(?:\.[A-Za-z_][A-Za-z0-9_$]*(?:\[\d+\])?)*", + line, + ) + for line in scopes + ): + raise ValueError(f"Invalid IP coverage hierarchy: {declared_hier}") + hier.write_text( + "\n".join(line.replace("@TOP@", target["top"]) for line in scopes) + + "\n" + ) + else: + hier.write_text( + f"+tree {target['top']}.dut\n" + f"+tree {target['top']}.source_if\n" + f"+tree {target['top']}.sink_if\n" + ) + script = ( + f"cd {shlex.quote(str(sim_dir))} && " + f"rm -rf {shlex.quote(str(csrc))} {shlex.quote(str(simv))} {shlex.quote(str(simv))}.daidir && " + f"mkdir -p {shlex.quote(str(csrc))} && " + f"vcs -full64 -sverilog -timescale=1ns/1ps -debug_access+all -top {shlex.quote(target['top'])} " + "${=VCS_UVM_ARGS} " + + " ".join( + f"+define+{key}={value}" for key, value in config["defines"].items() + ) + + " " + f"-cm line+cond+tgl+assert -cm_hier {shlex.quote(str(hier))} " + f"-cm_assert_hier {shlex.quote(str(hier))} " + f"-Mdir={shlex.quote(str(csrc))} -o {shlex.quote(str(simv))} " + f"-f {shlex.quote(str(filelist))}" + ) + command = ( + f"nix develop {shlex.quote(str(Path(bbdir) / 'verify'))} --command " + f"zsh -c {shlex.quote(script)}" + ) + result = stream_run_logger( + cmd=command, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="uvm vcs", + stderr_prefix="uvm vcs", + ) + if result.returncode != 0: + raise RuntimeError(f"VCS failed for IP {name} target={target['name']}") + build_lock.close() + return config + + +def validate_ip_exclusions(exclusions: Path, exported: Path) -> None: + current = set() + for path in exported.glob("fullexclude.*"): + for block in path.read_text().split('// CHECKSUM: "')[1:]: + checksum = block.split('"', 1)[0] + instance = re.search(r"^// INSTANCE: (\S+)", block, re.MULTILINE) + if instance: + current.add((instance[1], checksum)) + checksum = None + for line in exclusions.read_text().splitlines(): + if line.startswith("CHECKSUM:"): + checksum = line.split('"')[1] + elif line.startswith("INSTANCE:"): + instance = line.split()[1] + if (instance, checksum) not in current: + raise RuntimeError( + f"coverage exclusions require review: {exclusions}: {instance}" + ) + elif line.startswith("MODULE:"): + raise ValueError(f"IP exclusions must use instance scopes: {exclusions}") + + +def report_ip_coverage( + bbdir: str, root: Path, target: dict, simv: Path, cov_dir: Path, ctx +) -> None: + cov_dir.mkdir(parents=True, exist_ok=True) + exclusions = root / "src/main/resources" / target["exclusions"] + hierarchy = simv.parent / "rtl_hier.cfg" + hierarchy.write_text( + "".join( + f"+tree {target['top']}.{instance}\n" + for instance in target.get("rtl_scopes", ["dut"]) + ) + ) + scope = f" -hier {shlex.quote(str(hierarchy))} -metric line+cond+tgl" + reports = [(cov_dir, ""), (cov_dir / "rtl_raw", scope)] + if exclusions.is_file(): + reports[0] = (cov_dir, " -dump full_exclusions") + reports.append( + ( + cov_dir / "rtl", + scope + f" -elfile {shlex.quote(str(exclusions))} -excl_strict", + ) + ) + for report, options in reports: + if report.name == "rtl": + validate_ip_exclusions(exclusions, simv.parent) + command = ( + f"nix develop {shlex.quote(str(Path(bbdir) / 'verify'))} --command " + f"urg -dir {shlex.quote(str(simv))}.vdb -format text" + f"{options} -report {shlex.quote(str(report))}" + ) + result = stream_run_logger( + cmd=command, + logger=ctx.logger, + cwd=str(simv.parent), + stdout_prefix="uvm urg", + stderr_prefix="uvm urg", + ) + if result.returncode != 0: + raise RuntimeError(f"URG failed for {target['name']}: {report}") + if report.name == "rtl" and re.search( + r"(?:Warning|Error)-\[", result.stdout + result.stderr + ): + raise RuntimeError( + f"coverage exclusions require review for {target['name']}: {report}" + ) + + +def run_ip( + bbdir: str, design: str | None, selection: str | None, name: str, ctx, cov_root: str +) -> dict: + results = [ + run_ip_profile(bbdir, design, name, ctx, cov_root, config) + for config in load_ip(bbdir, name, design, selection) + ] + return {"design": design, "results": results, "failures": []} + + +def run_ip_profile( + bbdir: str, design: str | None, name: str, ctx, cov_root: str, config: dict +) -> dict: + build_ip_profile(bbdir, design, name, ctx, config) + root = Path(bbdir) / config["root"] + manifest = root / config["model"] + crate = tomllib.loads(manifest.read_text())["package"]["name"].replace("-", "_") + model = manifest.parent / "target" / "debug" / f"lib{crate}" + sim_root = root / "build" / "uvm" / (design if design is not None else "default") + + coverage = [] + expected_assertions = [] + for target in config["targets"]: + simv = sim_root / target["name"] / "simv" + simv_q = shlex.quote(str(simv)) + cov_dir = Path(cov_root) / target["name"] / "coverage" + scenario = ( + f"+invalid_case={target['invalid_case']} " + if "invalid_case" in target + else "" + ) + script = ( + f'loader="$(patchelf --print-interpreter {simv_q})"; ' + f'library_path="$(patchelf --print-rpath {simv_q})"; ' + f'"$loader" --library-path "$library_path" {simv_q} -no_save ' + f"-sv_lib {shlex.quote(str(model))} +UVM_TESTNAME={shlex.quote(target['test'])} {scenario}" + f"-cm line+cond+tgl+assert -cm_name {shlex.quote(target['test'])}" + ) + command = ( + f"nix develop {shlex.quote(str(Path(bbdir) / 'verify'))} --command " + f"zsh -ic {shlex.quote(script)}" + ) + result = stream_run_logger( + cmd=command, + logger=ctx.logger, + cwd=str(root), + stdout_prefix="uvm run", + stderr_prefix="uvm run", + ) + expected_assertion = target.get("expected_assertion") + check_uvm_result( + result, f"IP {name} target={target['name']}", expected_assertion + ) + if expected_assertion is not None: + expected_assertions.append(target["name"]) + continue + + report_ip_coverage(bbdir, root, target, simv, cov_dir, ctx) + coverage.append((target["name"], cov_dir)) + + index_dir = Path(cov_root) / config["profile"] / "coverage" + index_dir.mkdir(parents=True, exist_ok=True) + rows = ["target score line cond toggle assert group result"] + for target, path in coverage: + summary = dashboard_summary(str(path)) + rows.append( + f"{target} {summary['SCORE']} {summary['LINE']} {summary['COND']} " + f"{summary['TOGGLE']} {summary.get('ASSERT', '--')} {summary.get('GROUP', '--')} pass" + ) + rows.extend( + f"{target} -- -- -- -- -- -- expected-assertion" + for target in expected_assertions + ) + index = index_dir / "index.txt" + index.write_text("\n".join(rows) + "\n") + return { + "design": design, + "config_path": config["config_path"], + "parameters": config["parameters"], + "ip": name, + "targets": [t["name"] for t in config["targets"]], + "expected_assertions": expected_assertions, + "index": str(index), + "rtl_raw_coverage": { + target["name"]: str( + Path(cov_root) / target["name"] / "coverage" / "rtl_raw" + ) + for target in config["targets"] + if target["name"] not in expected_assertions + }, + "rtl_coverage": { + target["name"]: str(Path(cov_root) / target["name"] / "coverage" / "rtl") + for target in config["targets"] + if target["name"] not in expected_assertions + and (root / "src/main/resources" / target["exclusions"]).is_file() + }, + "log": cov_root, + } + + +def build_ball( + bbdir: str, design_name: str, selection: str, mill_cfg: str, domain, mapping, ctx +) -> None: + ball = mapping.component + verify_dir = Path(bbdir) / "examples" / "prefab" / "balls" / ball / "verify" + casegen = verify_dir / "casegen" / "Cargo.toml" + rtl_dir = Path(design_arch_root(bbdir, design_name, selection=selection)) / ( + f"verilog/{mill_cfg}" + ) + design = load_design(bbdir, design_name, selection) + thread, core = ball_owner(design, ball) + sim_dir = verify_dir.parent / "output" / "uvm" / design_name / selection + uvm_rel = os.path.relpath(Path(bbdir) / "verify" / "uvm", verify_dir) + flist = _filelist(verify_dir, ball, uvm_rel, rtl_dir, sim_dir) + parameters_path = verify_dir / "parameters.toml" + parameters = ( + tomllib.loads(parameters_path.read_text())["defines"] + if parameters_path.is_file() + else {} + ) + cargo = ( + f"nix develop {shlex.quote(str(Path(bbdir) / 'verify'))} --command " + f"cargo build --manifest-path {shlex.quote(str(casegen))} " + f"--target-dir {shlex.quote(str(verify_dir.parent / 'output' / 'casegen'))}" + ) + r = stream_run_logger( + cmd=cargo, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="uvm dpi", + stderr_prefix="uvm dpi", + ) + if r.returncode != 0: + raise RuntimeError(f"cargo build failed for {ball}") + crate = tomllib.loads(casegen.read_text()).get("package", {}).get("name") + if not crate: + raise ValueError(f"package.name missing in {casegen}") + simv = sim_dir / "simv" + csrc = sim_dir / "csrc" + hier = sim_dir / "cm_hier.cfg" + sim_dir.mkdir(parents=True, exist_ok=True) + hier.write_text("+tree tb_top.dut\n") + script = ( + f"cd {shlex.quote(str(verify_dir))} && " + f"rm -rf {shlex.quote(str(csrc))} {shlex.quote(str(simv))} {shlex.quote(str(simv))}.daidir && " + f"mkdir -p {shlex.quote(str(sim_dir))} {shlex.quote(str(csrc))} && " + "vcs -full64 -sverilog -timescale=1ns/1ps -debug_access+all -top tb_top " + "${=VCS_UVM_ARGS} " + + " ".join( + shlex.quote(d) + for d in vcs_defines(domain, mapping, thread, core, parameters) + ) + + f" -cm line+cond+tgl+assert -cm_hier {shlex.quote(str(hier))} " + f"-Mdir={shlex.quote(str(csrc))} -o {shlex.quote(str(simv))} " + f"-f {shlex.quote(flist)}" + ) + vcs = f"nix develop {shlex.quote(str(Path(bbdir) / 'verify'))} --command zsh -c {shlex.quote(script)}" + r = stream_run_logger( + cmd=vcs, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="uvm vcs", + stderr_prefix="uvm vcs", + ) + if r.returncode != 0: + raise RuntimeError(f"vcs failed for {ball}") + + +def run_ball( + bbdir: str, + design_name: str, + selection: str, + mill_cfg: str, + domain, + mapping, + ctx, + cov_dir: str, +) -> None: + ball = mapping.component + verify_dir = Path(bbdir) / "examples" / "prefab" / "balls" / ball / "verify" + simv = verify_dir.parent / "output" / "uvm" / design_name / selection / "simv" + build_ball(bbdir, design_name, selection, mill_cfg, domain, mapping, ctx) + crate = tomllib.loads((verify_dir / "casegen" / "Cargo.toml").read_text())[ + "package" + ]["name"] + dpi = ( + verify_dir.parent + / "output" + / "casegen" + / "debug" + / f"lib{crate.replace('-', '_')}" + ) + test = f"{ball}_ball_test" + verify_config = ( + verify_dir.parent + / "output" + / "uvm" + / design_name + / selection + / "verify_config.env" + ) + design = load_design(bbdir, design_name, selection) + thread, _ = ball_owner(design, ball) + verify_config.write_text( + f"bank_entries={thread.mem.bank.entries}\nball_id={mapping.id}\n" + ) + simv_q = shlex.quote(str(simv)) + script = ( + f"cd {shlex.quote(str(verify_dir))} && " + f'loader="$(patchelf --print-interpreter {simv_q})"; ' + f'library_path="$(patchelf --print-rpath {simv_q})"; ' + f"BB_VERIFY_CONFIG={shlex.quote(str(verify_config))} " + f'"$loader" --library-path "$library_path" {simv_q} -no_save ' + f"-sv_lib {shlex.quote(str(dpi))} " + f"+UVM_TESTNAME={shlex.quote(test)} +BID={mapping.id} " + f"-cm line+cond+tgl+assert -cm_name {shlex.quote(test)}" + ) + cmd = f"nix develop {shlex.quote(str(Path(bbdir) / 'verify'))} --command zsh -ic {shlex.quote(script)}" + r = stream_run_logger( + cmd=cmd, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="uvm run", + stderr_prefix="uvm run", + ) + check_uvm_result(r, f"{ball} test={test}") + Path(cov_dir).mkdir(parents=True, exist_ok=True) + urg = ( + f"nix develop {shlex.quote(str(Path(bbdir) / 'verify'))} --command " + f"urg -dir {shlex.quote(str(simv))}.vdb -format text -report {shlex.quote(cov_dir)}" + ) + r = stream_run_logger( + cmd=urg, + logger=ctx.logger, + cwd=str(verify_dir), + stdout_prefix="uvm urg", + stderr_prefix="uvm urg", + ) + if r.returncode != 0: + raise RuntimeError(f"urg failed for {ball}") + + +def dashboard_summary(cov_dir: str) -> dict[str, str]: + path = Path(cov_dir) / "dashboard.txt" + lines = path.read_text().splitlines() + i = 0 + while i < len(lines) and "Total Coverage Summary" not in lines[i]: + i += 1 + if i >= len(lines): + raise RuntimeError(f"Total Coverage Summary missing in {path}") + i += 1 + while i < len(lines) and not lines[i].strip().startswith("SCORE"): + i += 1 + if i + 1 >= len(lines): + raise RuntimeError(f"SCORE row missing in {path}") + keys = lines[i].split() + vals = lines[i + 1].split() + if len(vals) != len(keys): + raise RuntimeError(f"SCORE/value mismatch in {path}: {keys!r} vs {vals!r}") + return dict(zip(keys, vals)) + + +def run_design( + bbdir: str, design: str, selection: str, ball: str | None, ctx, do_run: bool +) -> dict: + msg = load_design(bbdir, design, selection) + domain = ball_domain(msg, ball) + mill_cfg = msg.simulation.top + if not mill_cfg: + raise ValueError(f"design {design!r} has no sims.verilator for UVM") + maps = selected_mappings(domain, ball) + ran = [] + covs = [] + run_root = None + if do_run: + stamp = datetime.now().strftime("%Y-%m-%d-%H-%M") + run_name = "all" if ball is None else maps[0].component + run_root = log_dir( + bbdir, design, "verilog", stamp, "uvm", run_name, selection=selection + ) + for m in maps: + if do_run: + cov = ( + os.path.join(run_root, m.component, "coverage") + if ball is None + else os.path.join(run_root, "coverage") + ) + run_ball(bbdir, design, selection, mill_cfg, domain, m, ctx, cov) + if not (Path(cov) / "dashboard.txt").is_file(): + raise RuntimeError(f"urg wrote no dashboard.txt under {cov}") + covs.append((m.component, cov)) + else: + build_ball(bbdir, design, selection, mill_cfg, domain, m, ctx) + ran.append(m.component) + info = {"design": design, "mill": mill_cfg, "balls": ran} + if do_run and ball is None: + index_dir = Path(run_root) / "coverage" + index_dir.mkdir(parents=True, exist_ok=True) + body = ["ball_dir score line cond toggle assert group result"] + for b, p in covs: + s = dashboard_summary(p) + body.append( + f"{b} {s['SCORE']} {s['LINE']} {s['COND']} {s['TOGGLE']} " + f"{s.get('ASSERT', '--')} {s.get('GROUP', '--')} pass" + ) + index = index_dir / "index.txt" + index.write_text("\n".join(body) + "\n") + info["index"] = str(index) + info["log"] = run_root + elif do_run: + info["log"] = run_root + return info + + +def run_uvm( + bbdir: str, + design: str, + selection: str, + ball: str | None, + ip: str | None, + ctx, + do_run: bool, +) -> dict: + if ball is not None: + return run_design(bbdir, design, selection, ball, ctx, do_run) + if ip is not None: + if do_run: + stamp = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + run_root = log_dir( + bbdir, + design if design is not None else "ip", + "verilog", + stamp, + "uvm", + ip, + ) + return run_ip(bbdir, design, selection, ip, ctx, run_root) + config = build_ip(bbdir, design, selection, ip, ctx) + return { + "design": design, + "ip": ip, + "targets": [target["name"] for target in config["targets"]], + } + + targets = load_design_uvm(bbdir, design, selection) + results, failures = [], [] + for ball in targets["balls"]: + try: + results.append(run_design(bbdir, design, selection, ball, ctx, do_run)) + except Exception as error: + if not do_run: + raise + ctx.logger.error(f"UVM ball/{ball} failed: {error}") + failures.append({"target": f"ball/{ball}", "error": str(error)}) + for ip in targets["ips"]: + try: + if do_run: + stamp = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + run_root = log_dir( + bbdir, + design if design is not None else "ip", + "verilog", + stamp, + "uvm", + ip, + ) + results.extend( + run_ip(bbdir, design, selection, ip, ctx, run_root)["results"] + ) + else: + config = build_ip(bbdir, design, selection, ip, ctx) + results.append( + { + "design": design, + "ip": ip, + "targets": [item["name"] for item in config["targets"]], + } + ) + except Exception as error: + if not do_run: + raise + ctx.logger.error(f"UVM ip/{ip} failed: {error}") + failures.append({"target": f"ip/{ip}", "error": str(error)}) + return {"design": design, "results": results, "failures": failures} diff --git a/bbdev/api/steps/uvm/scripts/waive.py b/bbdev/api/steps/uvm/scripts/waive.py new file mode 100644 index 000000000..a1f40125e --- /dev/null +++ b/bbdev/api/steps/uvm/scripts/waive.py @@ -0,0 +1,26 @@ +import re +from pathlib import Path + + +def apply_waivers(rtl_dir: Path) -> int: + if not rtl_dir.is_dir(): + raise ValueError(f"missing generated RTL directory: {rtl_dir}") + changed = 0 + for path in sorted(rtl_dir.glob("*.sv")): + lines = path.read_text().splitlines(keepends=True) + output = [] + excluded_module = False + for line in lines: + if line.strip() == "// VCS coverage exclude_file": + # FIRRTL memory models share a file with the DUT in unsplit RTL. + output.append("//VCS coverage off\n") + excluded_module = True + changed += 1 + continue + if excluded_module and re.match(r"^\s*endmodule\b", line): + output.extend((line, "//VCS coverage on\n")) + excluded_module = False + continue + output.append(line) + path.write_text("".join(output)) + return changed diff --git a/bbdev/api/steps/vcs/01_clean_api.step.py b/bbdev/api/steps/vcs/01_clean_api.step.py new file mode 100644 index 000000000..30142467f --- /dev/null +++ b/bbdev/api/steps/vcs/01_clean_api.step.py @@ -0,0 +1,31 @@ +from utils.event_common import require_target +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.event_common import require_design + +config = { + "name": "vcs-clean-api", + "description": "clean VCS artifacts without deleting generated RTL", + "flows": ["vcs"], + "triggers": [api("POST", "/vcs/clean")], + "enqueues": ["vcs.clean"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = req.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) + await ctx.enqueue( + { + "topic": "vcs.clean", + "data": {"design": design, "_trace_id": ctx.trace_id, "target": selection}, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/vcs/01_clean_event.step.py b/bbdev/api/steps/vcs/01_clean_event.step.py new file mode 100644 index 000000000..0ff2c2676 --- /dev/null +++ b/bbdev/api/steps/vcs/01_clean_event.step.py @@ -0,0 +1,38 @@ +from utils.path import design_output_root +import os +import shutil +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.event_common import check_result, get_origin_trace_id +from utils.path import get_buckyball_path + +config = { + "name": "vcs-clean", + "description": "remove simulator-specific VCS artifacts", + "flows": ["vcs"], + "triggers": [queue("vcs.clean")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + design = input_data.get("design") + artifact_dir = os.path.join( + design_output_root(bbdir, design, selection=input_data["target"]), "vcs" + ) + if os.path.isdir(artifact_dir): + shutil.rmtree(artifact_dir) + await check_result( + ctx, + 0, + extra_fields={"task": "clean", "artifact_dir": artifact_dir}, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/vcs/02_verilog_api.step.py b/bbdev/api/steps/vcs/02_verilog_api.step.py new file mode 100644 index 000000000..07d8ddc90 --- /dev/null +++ b/bbdev/api/steps/vcs/02_verilog_api.step.py @@ -0,0 +1,28 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.event_common import require_design, require_target + +config = { + "name": "vcs-verilog-api", + "description": "generate BBSimHarness RTL for VCS", + "flows": ["vcs"], + "triggers": [api("POST", "/vcs/verilog")], + "enqueues": ["vcs.verilog"], +} + + +def _request_data(body: dict) -> dict: + design = require_design(body) + data = {"design": design, "target": require_target(body)} + return data + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + try: + data = _request_data(req.body or {}) + except ValueError as exc: + return ApiResponse(status=400, body={"error": str(exc)}) + await ctx.enqueue( + {"topic": "vcs.verilog", "data": {**data, "_trace_id": ctx.trace_id}} + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/vcs/03_build_api.step.py b/bbdev/api/steps/vcs/03_build_api.step.py new file mode 100644 index 000000000..c4f95f601 --- /dev/null +++ b/bbdev/api/steps/vcs/03_build_api.step.py @@ -0,0 +1,46 @@ +from utils.event_common import require_target +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.event_common import require_design + +config = { + "name": "vcs-build-api", + "description": "build a VCS BBSimHarness executable", + "flows": ["vcs"], + "triggers": [api("POST", "/vcs/build")], + "enqueues": ["vcs.build"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = req.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) + jobs = body.get("jobs", 16) + try: + jobs = int(jobs) + except (TypeError, ValueError): + return ApiResponse( + status=400, body={"error": "jobs must be a positive integer"} + ) + if jobs <= 0: + return ApiResponse( + status=400, body={"error": "jobs must be a positive integer"} + ) + data = { + "design": design, + "jobs": jobs, + } + await ctx.enqueue( + { + "topic": "vcs.build", + "data": {**data, "_trace_id": ctx.trace_id, "target": selection}, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/vcs/03_build_event.step.py b/bbdev/api/steps/vcs/03_build_event.step.py new file mode 100644 index 000000000..1e9cc5373 --- /dev/null +++ b/bbdev/api/steps/vcs/03_build_event.step.py @@ -0,0 +1,93 @@ +from pathlib import Path +import shlex +import shutil + +from motia import FlowContext, queue +from utils.event_common import check_result, get_origin_trace_id, require_design +from utils.path import ( + get_buckyball_path, + rtl_dir, + design_output_root, + runner_manifest, + bebop_target_dir, + bebop_bin_path, +) +from utils.stream_run import stream_run_logger_async + +config = { + "name": "vcs-build", + "description": "Build the independent Bebop VCS node and SoC simulator", + "flows": ["vcs"], + "triggers": [queue("vcs.build")], + "enqueues": ["vcs.sim"], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + design = require_design(input_data) + repo = get_buckyball_path() + selection = input_data["target"] + target = Path(bebop_target_dir(repo, design, "vcs", selection=selection)) + output = Path(design_output_root(repo, design, selection=selection)) / "vcs" + rtl = rtl_dir(repo, design, "tapeout", selection=selection) + commands = [ + [ + "cargo", + "build", + "--release", + "--manifest-path", + str(runner_manifest(repo, design, selection)), + "--target-dir", + str(target), + "--bin", + "bebop", + "--bin", + "bebop-vcs", + ], + [ + str(target / "release/bebop"), + "build", + "vcs", + "--rtl-dir", + rtl, + "--artifact-dir", + str(output), + "--jobs", + str(input_data.get("jobs", 16)), + ], + ] + for command in commands: + result = await stream_run_logger_async( + cmd=shlex.join(command), + logger=ctx.logger, + cwd=repo, + stdout_prefix="vcs build", + stderr_prefix="vcs build", + ) + if result.returncode: + await check_result( + ctx, + result.returncode, + extra_fields={"task": "build"}, + trace_id=origin_tid, + ) + return + installed = Path(bebop_bin_path(repo, design, "vcs", selection=selection)) + installed.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(target / "release/bebop-vcs", installed) + await check_result( + ctx, + 0, + continue_run=input_data.get("from_run_workflow", False), + extra_fields={ + "task": "build", + "simv": str(output / "simv"), + "compile_log": str(output / "vcs_compile.log"), + }, + trace_id=origin_tid, + ) + if input_data.get("from_run_workflow"): + await ctx.enqueue( + {"topic": "vcs.sim", "data": {**input_data, "_trace_id": origin_tid}} + ) diff --git a/bbdev/api/steps/vcs/04_sim_api.step.py b/bbdev/api/steps/vcs/04_sim_api.step.py new file mode 100644 index 000000000..771d9a1d5 --- /dev/null +++ b/bbdev/api/steps/vcs/04_sim_api.step.py @@ -0,0 +1,42 @@ +from utils.event_common import require_target +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.event_common import require_design + +config = { + "name": "vcs-sim-api", + "description": "run a built VCS BBSimHarness simulation", + "flows": ["vcs"], + "triggers": [api("POST", "/vcs/sim")], + "enqueues": ["vcs.sim"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = req.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + binary = body.get("binary") + try: + design = require_design(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) + if not isinstance(binary, str) or not binary: + return ApiResponse( + status=400, + body={"error": "Missing required parameter: --binary must be specified"}, + ) + await ctx.enqueue( + { + "topic": "vcs.sim", + "data": { + **body, + "design": design, + "_trace_id": ctx.trace_id, + "target": selection, + }, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/vcs/04_sim_event.step.py b/bbdev/api/steps/vcs/04_sim_event.step.py new file mode 100644 index 000000000..3ada2dde7 --- /dev/null +++ b/bbdev/api/steps/vcs/04_sim_event.step.py @@ -0,0 +1,118 @@ +from utils.path import design_output_root +import os +import shlex +import sys +from datetime import datetime +from pathlib import Path + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.event_common import check_result, get_origin_trace_id +from utils.event_common import require_design +from utils.path import ( + get_buckyball_path, + log_dir, + bebop_bin_path, +) +from utils.search_workload import search_workload +from utils.stream_run import stream_run_logger_async + +config = { + "name": "vcs-sim", + "description": "run a VCS BBSimHarness simulation", + "flows": ["vcs"], + "triggers": [queue("vcs.sim")], + "enqueues": [], +} + + +def _resolve_binary(bbdir: str, design: str, binary: str, selection: str) -> str | None: + if os.path.isfile(binary): + return os.path.abspath(binary) + return search_workload( + str(Path(design_output_root(bbdir, design, selection=selection)) / "workloads"), + binary, + ) + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + try: + design = require_design(input_data) + except ValueError as exc: + await check_result( + ctx, 1, extra_fields={"task": "sim", "error": str(exc)}, trace_id=origin_tid + ) + return + binary_name = input_data.get("binary") + if not isinstance(binary_name, str) or not binary_name: + await check_result( + ctx, + 1, + extra_fields={"task": "sim", "error": "missing binary"}, + trace_id=origin_tid, + ) + return + bbdir = get_buckyball_path() + binary = _resolve_binary(bbdir, design, binary_name, input_data["target"]) + if binary is None: + await check_result( + ctx, + 1, + extra_fields={"task": "sim", "error": f"binary not found: {binary_name}"}, + trace_id=origin_tid, + ) + return + artifact_dir = ( + Path(design_output_root(bbdir, design, selection=input_data["target"])) / "vcs" + ) + simv = artifact_dir / "simv" + timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") + run_log = Path( + log_dir( + bbdir, + design, + "tapeout", + timestamp, + "vcs", + binary_name, + selection=input_data["target"], + ) + ) + run_log.mkdir(parents=True) + command = [ + bebop_bin_path(bbdir, design, "vcs", selection=input_data["target"]), + "--simv", + str(simv), + "--elf", + binary, + "--dram-configs", + str(Path(bbdir) / "result/share/dramsim3/configs"), + "--log-dir", + str(run_log), + ] + wave = input_data.get("wave", False) + if wave: + command.extend(["--wave", "vpd" if wave is True else str(wave).lower()]) + result = await stream_run_logger_async( + cmd=shlex.join(command), + logger=ctx.logger, + cwd=artifact_dir, + stdout_prefix="vcs sim", + stderr_prefix="vcs sim", + ) + await check_result( + ctx, + result.returncode, + extra_fields={ + "task": "sim", + "simv": str(simv), + "binary": binary, + "log_dir": str(run_log), + }, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/vcs/05_run_api.step.py b/bbdev/api/steps/vcs/05_run_api.step.py new file mode 100644 index 000000000..40df9a4d2 --- /dev/null +++ b/bbdev/api/steps/vcs/05_run_api.step.py @@ -0,0 +1,55 @@ +from utils.event_common import require_target +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.event_common import require_design + +config = { + "name": "vcs-run-api", + "description": "generate RTL, build, and run VCS", + "flows": ["vcs"], + "triggers": [api("POST", "/vcs/run")], + "enqueues": ["vcs.verilog"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = req.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + binary = body.get("binary") + try: + design = require_design(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) + if not isinstance(binary, str) or not binary: + return ApiResponse( + status=400, + body={"error": "Missing required parameter: --binary must be specified"}, + ) + jobs = body.get("jobs", 16) + try: + jobs = int(jobs) + except (TypeError, ValueError): + return ApiResponse( + status=400, body={"error": "jobs must be a positive integer"} + ) + if jobs <= 0: + return ApiResponse( + status=400, body={"error": "jobs must be a positive integer"} + ) + data = { + "design": design, + "binary": binary, + "batch": bool(body.get("batch", False)), + "jobs": jobs, + "from_run_workflow": True, + } + await ctx.enqueue( + { + "topic": "vcs.verilog", + "data": {**data, "_trace_id": ctx.trace_id, "target": selection}, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/vcs/README.md b/bbdev/api/steps/vcs/README.md new file mode 100644 index 000000000..fb7ee5820 --- /dev/null +++ b/bbdev/api/steps/vcs/README.md @@ -0,0 +1,21 @@ +# VCS workflow + +VCS follows the same user-facing stages as Verilator while keeping its own +simulator products below `examples/designs//output/sims//vcs/`: + +`vcs.verilog` is implemented under `steps/mill/02_vcs_verilog_event.step.py` +(same mill elaboration as Verilator). Flow: + +```text +vcs.verilog → vcs.build → vcs.sim +``` + +`bbdev vcs --run` routes through all three stages. The generated RTL is the +standard `BBSimHarness` elaboration; `vcs.build` invokes the independent node under `bebop/src/nodes/vcs`, +which owns the SystemVerilog testbench, DPI shim, build and execution. The shim supplies BBSimDRAM, SCU UART, and +SCU exit callbacks without linking the Verilator-specific BBSim C++ main. + +The VCS flow is intended for RTL functional simulation. Gate-level power +simulation remains design-owned under `examples/designs//sims//tapeout/`, because +that flow must define the design's workload, reset/boot skip interval, and SDF +policy. diff --git a/bbdev/api/steps/verilator/01_clean_api.step.py b/bbdev/api/steps/verilator/01_clean_api.step.py new file mode 100644 index 000000000..4173ff4f4 --- /dev/null +++ b/bbdev/api/steps/verilator/01_clean_api.step.py @@ -0,0 +1,37 @@ +from utils.event_common import require_target +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.event_common import require_design + +config = { + "name": "verilator-clean-api", + "description": "clean build directory", + "flows": ["verilator"], + "triggers": [api("POST", "/verilator/clean")], + "enqueues": ["verilator.clean"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) + await ctx.enqueue( + { + "topic": "verilator.clean", + "data": { + **body, + "design": design, + "task": "clean", + "_trace_id": ctx.trace_id, + "target": selection, + }, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/verilator/01_clean_event.step.py b/bbdev/api/steps/verilator/01_clean_event.step.py new file mode 100644 index 000000000..d6470b264 --- /dev/null +++ b/bbdev/api/steps/verilator/01_clean_event.step.py @@ -0,0 +1,83 @@ +import os +import sys + +from motia import FlowContext, queue + +# Add the utils directory to the Python path +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.event_common import require_design +from utils.path import get_buckyball_path, rtl_dir +from utils.stream_run import stream_run_logger_async +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "verilator-clean", + "description": "clean build directory", + "flows": ["verilator"], + "triggers": [ + queue("verilator.run"), + queue("verilator.clean"), + ], + "enqueues": ["verilator.verilog"], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + try: + design = require_design(input_data) + except ValueError as error: + ctx.logger.error(str(error)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "missing_design"}, + trace_id=origin_tid, + ) + return + bbdir = get_buckyball_path() + build_dir = rtl_dir( + bbdir, + design, + "verilog", + selection=input_data["target"], + ) + # ================================================================================== + # Execute operation + # ================================================================================== + command = f"rm -rf {build_dir}" + result = await stream_run_logger_async( + cmd=command, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="verilator clean", + stderr_prefix="verilator clean", + ) + + # ================================================================================== + # Return result to API + # ================================================================================== + success_result, failure_result = await check_result( + ctx, + result.returncode, + continue_run=input_data.get("from_run_workflow", False), + extra_fields={"task": "clean"}, + trace_id=origin_tid, + ) + + # ================================================================================== + # Continue routing + # ================================================================================== + if input_data.get("from_run_workflow"): + await ctx.enqueue( + { + "topic": "verilator.verilog", + "data": {**input_data, "task": "run", "target": input_data["target"]}, + } + ) + + return diff --git a/bbdev/api/steps/verilator/02_verilog_api.step.py b/bbdev/api/steps/verilator/02_verilog_api.step.py new file mode 100644 index 000000000..1b1e90c25 --- /dev/null +++ b/bbdev/api/steps/verilator/02_verilog_api.step.py @@ -0,0 +1,41 @@ +from utils.event_common import require_target +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.event_common import require_design + +config = { + "name": "verilator-verilog-api", + "description": "generate design verilog code", + "flows": ["verilator"], + "triggers": [api("POST", "/verilator/verilog")], + "enqueues": ["verilator.verilog"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + + try: + design = require_design(body) + except ValueError as e: + return ApiResponse( + status=400, + body={ + "status": "error", + "message": str(e), + "example": 'bbdev verilator --verilog "--design toy"', + }, + ) + + data = {"design": design} + await ctx.enqueue( + { + "topic": "verilator.verilog", + "data": {**data, "_trace_id": ctx.trace_id, "target": selection}, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/verilator/03_build_api.step.py b/bbdev/api/steps/verilator/03_build_api.step.py new file mode 100644 index 000000000..da7574b8c --- /dev/null +++ b/bbdev/api/steps/verilator/03_build_api.step.py @@ -0,0 +1,35 @@ +from utils.event_common import require_target +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.event_common import require_design + +config = { + "name": "verilator-build-api", + "description": "build verilator executable", + "flows": ["verilator"], + "triggers": [api("POST", "/verilator/build")], + "enqueues": ["verilator.build"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) + data = { + "design": design, + "jobs": body.get("jobs", 16), + } + await ctx.enqueue( + { + "topic": "verilator.build", + "data": {**data, "_trace_id": ctx.trace_id, "target": selection}, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/verilator/03_build_event.step.py b/bbdev/api/steps/verilator/03_build_event.step.py new file mode 100644 index 000000000..8c3ec5f11 --- /dev/null +++ b/bbdev/api/steps/verilator/03_build_event.step.py @@ -0,0 +1,200 @@ +import os +import shutil +import subprocess +import glob +import sys + +from motia import FlowContext, queue + +# Add the utils directory to the Python path +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.event_common import require_design +from utils.path import get_buckyball_path, rtl_dir +from utils.stream_run import stream_run_logger_async +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "verilator-build", + "description": "build verilator executable", + "flows": ["verilator"], + "triggers": [queue("verilator.build")], + "enqueues": ["verilator.sim"], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + design = require_design(input_data) + bbdir = get_buckyball_path() + arch_dir = f"{bbdir}/arch" + build_dir = rtl_dir( + bbdir, + design, + "verilog", + selection=input_data["target"], + ) + coverage = input_data.get("coverage", False) + ctx.logger.info(f"Using build directory: {build_dir}") + + obj_dir = f"{build_dir}/obj_dir" + if os.path.exists(obj_dir): + shutil.rmtree(obj_dir) + os.makedirs(obj_dir) + + # ================================================================================== + # Find sources + # ================================================================================== + vsrcs = glob.glob(f"{build_dir}/**/*.v", recursive=True) + glob.glob( + f"{build_dir}/**/*.sv", recursive=True + ) + csrcs = ( + glob.glob(f"{arch_dir}/src/csrc/**/*.c", recursive=True) + + glob.glob(f"{arch_dir}/src/csrc/**/*.cc", recursive=True) + + glob.glob(f"{arch_dir}/src/csrc/**/*.cpp", recursive=True) + + glob.glob(f"{build_dir}/**/*.c", recursive=True) + + glob.glob(f"{build_dir}/**/*.cc", recursive=True) + + glob.glob(f"{build_dir}/**/*.cpp", recursive=True) + ) + + unexpected_tsi = [ + f + for f in csrcs + if os.path.basename(f) in {"testchip_tsi.cc", "testchip_htif.cc", "SimTSI.cc"} + ] + if unexpected_tsi: + raise RuntimeError( + "Unexpected TSI sources in BBSim build: " + ", ".join(unexpected_tsi) + ) + + topname = "BBSimHarness" + + # ================================================================================== + # Build flags + # ================================================================================== + result_dir = f"{bbdir}/result" + result_lib = f"{result_dir}/lib" + inc_flags = " ".join( + [ + f"-I{result_dir}/include", + f"-I{build_dir}", + f"-I{arch_dir}/src/csrc/include", + ] + ) + + # -DBBSIM: selects VBBSimHarness in bdb.h / main.cc + # BDB NDJSON trace (+trace=...) is runtime-only; bbdev sim uses +trace=all (04_sim_event.step.py). + cflags = f"{inc_flags} -DBBSIM -DTOP_NAME='\"V{topname}\"' -std=c++17" + + def _gcc_lib_dir(soname: str) -> str: + printed = subprocess.run( + ["g++", f"-print-file-name={soname}"], + capture_output=True, + text=True, + check=True, + ) + return os.path.dirname(os.path.realpath(printed.stdout.strip())) + + rpath_dirs = [] + for lib_dir in ( + result_lib, + _gcc_lib_dir("liblz4.so"), + _gcc_lib_dir("libstdc++.so"), + ): + if lib_dir not in rpath_dirs: + rpath_dirs.append(lib_dir) + rpath_flags = " ".join(f"-Wl,-rpath,{lib_dir}" for lib_dir in rpath_dirs) + + ldflags = f"-lreadline -ldramsim3 -lstdc++ -lz " f"-L{result_lib} {rpath_flags} " + + sources = " ".join(vsrcs + csrcs) + jobs = input_data.get("jobs", "") + + build_env = os.environ.copy() + if ( + subprocess.run("command -v ccache", shell=True, capture_output=True).returncode + == 0 + ): + build_env["OBJCACHE"] = "ccache" + + # Use lld for faster linking if available + use_lld = ( + subprocess.run("command -v ld.lld", shell=True, capture_output=True).returncode + == 0 + ) + if use_lld: + ldflags += " -fuse-ld=lld" + + # ================================================================================== + # Run verilator + # ================================================================================== + verilator_cmd = ( + f"verilator -MMD -cc --vpi --trace -O3 --x-assign fast --x-initial fast --assert -Wno-fatal " + f"--trace-fst --trace-threads 1 --output-split 10000 --output-split-cfuncs 100 " + f"--unroll-count 256 " + f"{'--coverage-line ' if coverage else ''}" + f"-Wno-PINCONNECTEMPTY " + f"-Wno-ASSIGNDLY " + f"-Wno-DECLFILENAME " + f"-Wno-UNUSED " + f"-Wno-UNOPTFLAT " + f"-Wno-BLKANDNBLK " + f"-Wno-style " + f"-Wall " + f"--timing -j {jobs} +incdir+{build_dir} --top {topname} {sources} " + f"-CFLAGS '{cflags}' -LDFLAGS '{ldflags}' --Mdir {obj_dir} --exe" + ) + + result = await stream_run_logger_async( + cmd=verilator_cmd, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="verilator verilation", + stderr_prefix="verilator verilation", + env=build_env, + ) + if result.returncode != 0: + await check_result( + ctx, + result.returncode, + continue_run=False, + extra_fields={"task": "build"}, + trace_id=origin_tid, + ) + return + + make_jobs = jobs if jobs else str(os.cpu_count() or 16) + result = await stream_run_logger_async( + cmd=f"make -j{make_jobs} VM_PARALLEL_BUILDS=1 -C {obj_dir} -f V{topname}.mk V{topname}", + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="verilator build", + stderr_prefix="verilator build", + env=build_env, + ) + + # ================================================================================== + # Return result to API + # ================================================================================== + await check_result( + ctx, + result.returncode, + continue_run=input_data.get("from_run_workflow", False), + extra_fields={"task": "build"}, + trace_id=origin_tid, + ) + + # ================================================================================== + # Continue routing + # ================================================================================== + if input_data.get("from_run_workflow"): + await ctx.enqueue( + { + "topic": "verilator.sim", + "data": {**input_data, "task": "run", "target": input_data["target"]}, + } + ) + + return diff --git a/bbdev/api/steps/verilator/04_sim_api.step.py b/bbdev/api/steps/verilator/04_sim_api.step.py new file mode 100644 index 000000000..d4cf19090 --- /dev/null +++ b/bbdev/api/steps/verilator/04_sim_api.step.py @@ -0,0 +1,49 @@ +from utils.event_common import require_target +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.event_common import require_design + +config = { + "name": "verilator-sim-api", + "description": "run verilator simulation", + "flows": ["verilator"], + "triggers": [api("POST", "/verilator/sim")], + "enqueues": ["verilator.sim"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + binary = body.get("binary", "") + if not binary: + return ApiResponse( + status=400, + body={ + "success": False, + "failure": True, + "returncode": 400, + "message": "binary parameter is required", + }, + ) + try: + design = require_design(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) + + await ctx.enqueue( + { + "topic": "verilator.sim", + "data": { + **body, + "design": design, + "task": "sim", + "_trace_id": ctx.trace_id, + "target": selection, + }, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/verilator/04_sim_event.step.py b/bbdev/api/steps/verilator/04_sim_event.step.py new file mode 100644 index 000000000..401dfc3e9 --- /dev/null +++ b/bbdev/api/steps/verilator/04_sim_event.step.py @@ -0,0 +1,274 @@ +from pathlib import Path +from utils.path import design_output_root +import os +import shlex +import struct +import subprocess +import sys +from datetime import datetime +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) +from utils.event_common import require_design +from utils.path import get_buckyball_path, log_dir, rtl_dir +from utils.stream_run import stream_run_logger_async +from utils.search_workload import search_workload +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "verilator-sim", + "description": "run simulation", + "flows": ["verilator"], + "triggers": [queue("verilator.sim")], + "enqueues": [], +} + + +def validate_bbsim_elf(path: str) -> tuple[bool, str]: + """Validate the ELF layout accepted by BBSimDRAM before starting Verilator.""" + try: + with open(path, "rb") as binary: + header = binary.read(64) + if len(header) != 64: + return (False, "not a complete ELF64 header") + if header[:4] != b"\x7fELF" or header[4] != 2 or header[5] != 1: + return (False, "requires a little-endian ELF64 binary") + _, _, machine, _, _, phoff, _, _, ehsize, phentsize, phnum, _, _, _ = ( + struct.unpack("<16sHHIQQQIHHHHHH", header) + ) + if machine != 243: + return (False, "requires a RISC-V ELF") + if ehsize != 64 or phentsize != 56 or phnum == 0: + return (False, "has an unsupported ELF program-header layout") + file_size = os.fstat(binary.fileno()).st_size + if phoff + phnum * phentsize > file_size: + return (False, "program headers extend past the end of the file") + dram_base = 2147483648 + dram_size = 268435456 + load_segments = 0 + for index in range(phnum): + binary.seek(phoff + index * phentsize) + program_header = binary.read(phentsize) + if len(program_header) != phentsize: + return (False, "cannot read ELF program headers") + p_type, _, p_offset, _, p_paddr, p_filesz, p_memsz, _ = struct.unpack( + " p_memsz or p_offset + p_filesz > file_size: + return (False, "has an invalid loadable segment") + if p_paddr < dram_base or p_paddr + p_memsz > dram_base + dram_size: + return ( + False, + ( + f"load segment {index} at 0x{p_paddr:x} is outside" + f" BBSimDRAM [0x{dram_base:x}, 0x{dram_base + dram_size:x})" + ), + ) + except (OSError, struct.error): + return (False, "cannot read a valid ELF64 program-header table") + if not load_segments: + return (False, "has no loadable ELF segments") + return (True, "") + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + try: + design = require_design(input_data) + except ValueError as error: + ctx.logger.error(str(error)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "missing_design"}, + trace_id=origin_tid, + ) + return + bbdir = get_buckyball_path() + build_dir = rtl_dir( + bbdir, + design, + "verilog", + selection=input_data["target"], + ) + ctx.logger.info(f"Using build directory: {build_dir}") + timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") + binary_name = input_data.get("binary", "") + if not binary_name: + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "binary_required"}, + trace_id=origin_tid, + ) + return + coverage = input_data.get("coverage", False) + workload_root = str( + Path(design_output_root(bbdir, design, selection=input_data["target"])) + / "workloads" + ) + binary_path = search_workload(workload_root, binary_name) + ctx.logger.info(f"binary_path: {binary_path}") + if binary_path is None: + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "error": "binary_not_found", + "binary": binary_name, + "search_dir": workload_root, + }, + trace_id=origin_tid, + ) + return + valid_elf, elf_error = validate_bbsim_elf(binary_path) + if not valid_elf: + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "error": "unsupported_elf", + "binary": binary_path, + "detail": ( + "direct Verilator requires a baremetal RISC-V ELF in BBSimDRAM;" + f" {elf_error}. Use bebop-verilator for Linux ELFs" + ), + }, + trace_id=origin_tid, + ) + return + topname = "BBSimHarness" + run_log = log_dir( + bbdir, + design, + "verilog", + timestamp, + "verilator", + binary_name, + selection=input_data["target"], + ) + waveform_dir = os.path.join(run_log, "waveform") + os.makedirs(run_log, exist_ok=True) + no_wave = bool(input_data.get("no-wave", input_data.get("no_wave", False))) + if not no_wave: + os.makedirs(waveform_dir, exist_ok=True) + coverage_flag = "" + if coverage: + coverage_dat_path = f"{run_log}/coverage.dat" + coverage_flag = f"+verilator+coverage+file+{coverage_dat_path}" + bin_path = f"{build_dir}/obj_dir/V{topname}" + if not os.path.isfile(bin_path): + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "simulator_not_found", "simulator": bin_path}, + trace_id=origin_tid, + ) + return + batch = input_data.get("batch", False) + trace_names = [ + name + for name in ("itrace", "mtrace", "pmctrace", "ctrace", "banktrace") + if input_data.get(name, False) + ] + trace_arg = ",".join(trace_names) if trace_names else "none" + log_path = f"{run_log}/bdb.ndjson" + stdout_path = f"{run_log}/stdout.log" + meta_path = f"{run_log}/sim_meta.txt" + fst_path = f"{waveform_dir}/waveform.fst" + result_lib = f"{bbdir}/result/lib" + + def _gcc_lib_dir(soname: str) -> str: + printed = subprocess.run( + ["g++", f"-print-file-name={soname}"], + capture_output=True, + text=True, + check=True, + ) + return os.path.dirname(os.path.realpath(printed.stdout.strip())) + + lib_dirs = [result_lib, _gcc_lib_dir("liblz4.so"), _gcc_lib_dir("libstdc++.so")] + ld_lib_path = ":".join(dict.fromkeys(lib_dirs)) + ctx.logger.info(f"LD_LIBRARY_PATH prefix: {ld_lib_path}") + dramsim_outdir = os.path.join(run_log, "dramsim") + sim_args = [ + bin_path, + "+permissive", + f"+elf={binary_path}", + f"+dramsim_outdir={dramsim_outdir}", + ] + if batch: + sim_args.append("+batch") + if coverage_flag: + sim_args.append(coverage_flag) + if no_wave: + sim_args.append("+no-wave") + else: + sim_args.append(f"+fst={fst_path}") + sim_args.extend( + [ + f"+log={log_path}", + f"+stdout={stdout_path}", + f"+trace={trace_arg}", + "+permissive-off", + ] + ) + dasm_bin = os.path.join(bbdir, "bebop", "target", "release", "bebop") + if not os.path.isfile(dasm_bin): + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "dasm_not_found", "dasm": dasm_bin}, + trace_id=origin_tid, + ) + return + dasm_cmd = shlex.join([dasm_bin, "dasm"]) + sim_cmd = ( + f"export LD_LIBRARY_PATH={shlex.quote(ld_lib_path)}:$LD_LIBRARY_PATH; export" + f" BDB_SIM_META={shlex.quote(meta_path)}; {shlex.join(sim_args)} 2>" + f" >({dasm_cmd} > {shlex.quote(os.path.join(run_log, 'disasm.log'))})" + ) + script_dir = os.path.dirname(__file__) + result = await stream_run_logger_async( + cmd=sim_cmd, + logger=ctx.logger, + cwd=script_dir, + stdout_prefix="verilator sim", + stderr_prefix="verilator sim", + executable="bash", + ) + success_result, failure_result = await check_result( + ctx, returncode=result.returncode, continue_run=True, trace_id=origin_tid + ) + if failure_result: + ctx.logger.error("sim failed", failure_result) + return + extra_fields = { + "task": "sim", + "binary": binary_path, + "log_dir": run_log, + "waveform_dir": None if no_wave else waveform_dir, + "timestamp": timestamp, + "sim_meta": meta_path, + } + if coverage: + extra_fields["coverage_dat"] = coverage_dat_path + await check_result( + ctx, + result.returncode, + continue_run=False, + extra_fields=extra_fields, + trace_id=origin_tid, + ) + return diff --git a/bbdev/api/steps/verilator/05_run_api.step.py b/bbdev/api/steps/verilator/05_run_api.step.py new file mode 100644 index 000000000..3aeb4fc70 --- /dev/null +++ b/bbdev/api/steps/verilator/05_run_api.step.py @@ -0,0 +1,90 @@ +from pathlib import Path +from utils.path import design_output_root +from utils.event_common import require_target +import os +import sys + +from motia import ApiRequest, ApiResponse, FlowContext, api + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.event_common import require_design +from utils.path import get_buckyball_path +from utils.search_workload import search_workload + +config = { + "name": "verilator-run-api", + "description": "trigger complete verilator workflow", + "flows": ["verilator"], + "triggers": [api("POST", "/verilator/run")], + "enqueues": ["verilator.run"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) + binary = body.get("binary", "") + if not binary: + return ApiResponse( + status=400, + body={ + "success": False, + "failure": True, + "returncode": 400, + "message": "binary parameter is required", + }, + ) + + search_dir = str( + Path( + design_output_root( + get_buckyball_path(), body["design"], selection=selection + ) + ) + / "workloads" + ) + if search_workload(search_dir, binary) is None: + return ApiResponse( + status=400, + body={ + "success": False, + "failure": True, + "returncode": 1, + "error": "binary_not_found", + "binary": binary, + "search_dir": search_dir, + }, + ) + + data = { + "design": design, + "binary": binary, + "jobs": body.get("jobs", "16"), + "batch": body.get("batch", False), + "coverage": body.get("coverage", False), + "no-wave": body.get("no-wave", body.get("no_wave", False)), + "itrace": body.get("itrace", False), + "mtrace": body.get("mtrace", False), + "pmctrace": body.get("pmctrace", False), + "ctrace": body.get("ctrace", False), + "banktrace": body.get("banktrace", False), + "from_run_workflow": True, + } + + await ctx.enqueue( + { + "topic": "verilator.run", + "data": {**data, "_trace_id": ctx.trace_id, "target": selection}, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/verilator/README.md b/bbdev/api/steps/verilator/README.md new file mode 100644 index 000000000..a2608df16 --- /dev/null +++ b/bbdev/api/steps/verilator/README.md @@ -0,0 +1,210 @@ +# Verilator Simulation Workflow + +Hardware simulation workflow based on Verilator in the Buckyball framework, providing a complete automation flow from RTL generation to simulation execution. Verilator is a high-performance Verilog simulator that supports fast functional verification and performance analysis. + +## II. Original API Usage Guide + +#### `run` +**Endpoint**: `POST /verilator/run` + +**Function**: Execute complete workflow. Clean build directory, generate Verilog, compile Verilator into simulation file, and run simulation directly + +**Parameters**: + +- **`jobs`** - Number of parallel compilation tasks + - Default value: `16` +- **`binary`** [Required] - Test binary file path + - Default value: `""` +- **`batch`** - Run without the interactive monitor +- **`no-wave`** - Disable FST waveform generation +- **`itrace`**, **`mtrace`**, **`pmctrace`**, **`ctrace`**, **`banktrace`** - Enable selected BDB traces + +Direct Verilator uses `BBSimHarness` and therefore requires a baremetal RISC-V +ELF linked into the simulated `0x80000000` DRAM window. Linux workload ELFs +must be run with `bebop-verilator` instead. + +**Example**: +```bash +# bbdev wrapper +bbdev verilator --run "--design toy --jobs 16 --binary toy-toy-ctest-mvin_mvout_test-baremetal --batch" + +# Raw command +curl -X POST http://localhost:5000/verilator/run -H "Content-Type: application/json" -d '{"design":"toy", "jobs":8, "binary":"toy-toy-ctest-mvin_mvout_test-baremetal", "batch":true}' +``` + + +#### `clean` + +**Endpoint**: `POST /verilator/clean` + +**Function**: Clean build folder + +**Parameters**: None + +**Example**: +```bash +curl -X POST http://localhost:5000/verilator/clean +``` + +#### `verilog` + +**Endpoint**: `POST /verilator/verilog` + +**Function**: Only generate Verilog code, without compilation and simulation + +**Parameters**: None + +**Example**: +```bash +curl -X POST http://localhost:5000/verilator/verilog -d '{"jobs": 8}' +``` + +#### `build` + +**Endpoint**: `POST /verilator/build` + +**Function**: Compile verilog source files and cpp source files into executable simulation file + +**Parameters**: + +- **`jobs`** - Number of parallel compilation tasks + - Default value: `16` + +**Example**: +```bash +curl -X POST http://localhost:5000/verilator/build -d '{"jobs": 16}' +``` + +#### `sim` + +**Endpoint**: `POST /verilator/sim` + +**Function**: Run existing simulation executable + +**Parameters**: + +- **`binary`** [Required] - Custom test binary file path +- **`batch`** - Run without the interactive monitor +- **`no-wave`** - Disable FST waveform generation +- **`itrace`**, **`mtrace`**, **`pmctrace`**, **`ctrace`**, **`banktrace`** - Enable selected BDB traces + +The binary may be a workload filename or an existing absolute path. It must be +a baremetal ELF; Linux workload ELFs require `bebop-verilator`. + +**Example**: +```bash +curl -X POST http://localhost:5000/verilator/sim \ + -H "Content-Type: application/json" \ + -d '{"binary": "/home/user/test_program.elf"}' +``` + + + + +## II. Developer Documentation + +### Directory Structure + +``` +steps/verilator/ +├── 00_start_node_noop_step.py # Workflow entry node definition +├── 00_start_node_noop_step.tsx # Frontend UI component +├── 01_run_api_step.py # Complete workflow API entry +├── 01_clean_api_step.py # Clean API endpoint +├── 01_verilog_api_step.py # Verilog generation API endpoint +├── 01_build_api_step.py # Build API endpoint +├── 01_sim_api_step.py # Simulation API endpoint +├── 02_clean_event_step.py # Clean build directory +├── 03_verilog_event_step.py # Verilog code generation +├── 04_build_event_step.py # Verilator compilation +├── 05_sim_event_step.py # Simulation execution +├── 99_complete_event_step.py # Completion handling +├── 99_error_event_step.py # Error handling +└── README.md # This document +``` + +### Workflow Steps Detailed + +#### 1. Entry Node (`00_start_node_noop_step.py`) +- **Type**: `noop` node +- **Function**: Provide UI interface entry point +- **Frontend**: "Start Build Verilator" button + +#### 2. API Endpoints +- **Complete Workflow API** (`01_run_api_step.py`): `/verilator` → `verilator.run` +- **Clean API** (`01_clean_api_step.py`): `/verilator/clean` → `verilator.clean` +- **Verilog Generation API** (`01_verilog_api_step.py`): `/verilator/verilog` → `verilator.verilog` +- **Build API** (`01_build_api_step.py`): `/verilator/build` → `verilator.build` +- **Simulation API** (`01_sim_api_step.py`): `/verilator/sim` → `verilator.sim` + +#### 3. Clean Step (`02_clean_event_step.py`) +- **Type**: `event` step +- **Subscribes**: `verilator.run`, `verilator.clean` +- **Emits**: `verilator.verilog`, `verilator.complete` +- **Function**: Delete build directory, serves workflow or standalone operation + +#### 4. Verilog Generation (`03_verilog_event_step.py`) +- **Type**: `event` step +- **Subscribes**: `verilator.verilog` +- **Emits**: `verilator.build`, `verilator.complete` +- **Function**: Use mill to generate Verilog code to build directory + +#### 5. Verilator Compilation (`04_build_event_step.py`) +- **Type**: `event` step +- **Subscribes**: `verilator.build` +- **Emits**: `verilator.sim`, `verilator.complete` +- **Function**: Compile Verilog and C++ source files into executable simulation file + +#### 6. Simulation Execution (`05_sim_event_step.py`) +- **Type**: `event` step +- **Subscribes**: `verilator.sim` +- **Emits**: `verilator.complete` +- **Function**: Run simulation, supports custom binary parameter + +#### 7. Completion Handling (`99_complete_event_step.py`) +- **Type**: `event` step +- **Subscribes**: `verilator.complete` +- **Function**: Print success message, mark workflow as complete + +#### 8. Error Handling (`99_error_event_step.py`) +- **Type**: `event` step +- **Subscribes**: `verilator.error` +- **Function**: Print error message, handle workflow exceptions + +### Workflow Diagram + +```mermaid +graph TD; + API[POST /verilator
Complete Workflow] --> RUN[verilator.run] + + CLEAN_DIRECT[verilator.clean
Single-step Clean] --> CLEAN_STEP[02_clean_event_step] + VERILOG_DIRECT[verilator.verilog
Single-step Generate] --> VERILOG_STEP[03_verilog_event_step] + BUILD_DIRECT[verilator.build
Single-step Build] --> BUILD_STEP[04_build_event_step] + SIM_DIRECT[verilator.sim
Single-step Simulation] --> SIM_STEP[05_sim_event_step] + + RUN --> CLEAN_STEP + CLEAN_STEP --> |Workflow Mode| VERILOG_STEP + CLEAN_STEP --> |Single-step Mode| COMPLETE[verilator.complete] + + VERILOG_STEP --> |Workflow Mode| BUILD_STEP + VERILOG_STEP --> |Single-step Mode| COMPLETE + + BUILD_STEP --> |Workflow Mode| SIM_STEP + BUILD_STEP --> |Single-step Mode| COMPLETE + + SIM_STEP --> COMPLETE + + COMPLETE --> COMPLETE_STEP[99_complete_event_step] + + CLEAN_STEP -.-> |Error| ERROR[verilator.error] + VERILOG_STEP -.-> |Error| ERROR + BUILD_STEP -.-> |Error| ERROR + SIM_STEP -.-> |Error| ERROR + + ERROR --> ERROR_STEP[99_error_event_step] + + classDef apiNode fill:#e1f5fe + classDef eventNode fill:#f3e5f5 + classDef stepNode fill:#e8f5e8 + classDef endNode fill:#fff3e0 +``` diff --git a/bbdev/api/steps/workload/00_clean_api.step.py b/bbdev/api/steps/workload/00_clean_api.step.py new file mode 100644 index 000000000..15c65f66a --- /dev/null +++ b/bbdev/api/steps/workload/00_clean_api.step.py @@ -0,0 +1,41 @@ +from utils.event_common import require_target +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.event_common import require_design + +config = { + "name": "workload-clean-api", + "description": "clean workload output directory for one design", + "flows": ["workload"], + "triggers": [api("POST", "/workload/clean")], + "enqueues": ["workload.clean"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + + unknown = sorted(k for k in body if k not in {"design", "device", "core"}) + if unknown: + return ApiResponse( + status=400, + body={ + "error": f"Unknown workload clean parameter(s): {', '.join(unknown)}" + }, + ) + + await ctx.enqueue( + { + "topic": "workload.clean", + "data": {**body, "_trace_id": ctx.trace_id, "target": selection}, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/workload/00_clean_event.step.py b/bbdev/api/steps/workload/00_clean_event.step.py new file mode 100644 index 000000000..bffa594e0 --- /dev/null +++ b/bbdev/api/steps/workload/00_clean_event.step.py @@ -0,0 +1,76 @@ +import os +import shutil +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.event_common import require_design +from utils.event_common import check_result, get_origin_trace_id +from utils.path import design_output_root, get_buckyball_path, workload_build_dir + +config = { + "name": "workload-clean", + "description": "clean workload output directory for one design", + "flows": ["workload"], + "triggers": [queue("workload.clean")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + allowed = {"device", "core", "target", "design", "_trace_id"} + unknown = sorted(k for k in input_data if k not in allowed) + if unknown: + ctx.logger.error(f"Unknown workload clean parameter(s): {', '.join(unknown)}") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "unknown_parameter", "parameters": unknown}, + trace_id=origin_tid, + ) + return + + try: + design = require_design(input_data) + except ValueError as error: + ctx.logger.error(str(error)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "missing_design"}, + trace_id=origin_tid, + ) + return + + bbdir = get_buckyball_path() + output_path = os.path.join( + design_output_root(bbdir, design, selection=input_data["target"]), "workloads" + ) + build_path = workload_build_dir(bbdir, design, selection=input_data["target"]) + + for path in (output_path, build_path): + if os.path.exists(path): + ctx.logger.info("Removing workload directory", {"path": path}) + shutil.rmtree(path) + else: + ctx.logger.info("Workload directory already clean", {"path": path}) + + await check_result( + ctx, + 0, + continue_run=False, + extra_fields={ + "task": "clean", + "design": design, + "output_path": output_path, + "build_path": build_path, + }, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/workload/01_buidl_api.step.py b/bbdev/api/steps/workload/01_buidl_api.step.py new file mode 100644 index 000000000..9943a918c --- /dev/null +++ b/bbdev/api/steps/workload/01_buidl_api.step.py @@ -0,0 +1,75 @@ +from utils.event_common import require_target +import os +import re +import sys +from pathlib import Path +from motia import ApiRequest, ApiResponse, FlowContext, api + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) +from utils.path import get_buckyball_path + +config = { + "name": "workload-build-api", + "description": "build workload", + "flows": ["workload"], + "triggers": [api("POST", "/workload/build")], + "enqueues": ["workload.build"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + allowed = {"device", "core", "design", "ctest", "mlirtest", "soctest"} + unknown = sorted((k for k in body if k not in allowed)) + if unknown: + return ApiResponse( + status=400, + body={ + "error": f"Unknown workload build parameter(s): {', '.join(unknown)}" + }, + ) + design = body.get("design") + if not design: + return ApiResponse( + status=400, + body={"error": "Missing required parameter: --design must be specified"}, + ) + if not isinstance(design, str) or not re.fullmatch("[A-Za-z0-9_-]+", design): + return ApiResponse(status=400, body={"error": f"Invalid design: {design}"}) + design_dir = Path(get_buckyball_path()) / "examples" / "designs" / design + if not design_dir.is_dir(): + return ApiResponse( + status=400, body={"error": f"Workload design does not exist: {design}"} + ) + ctest = body.get("ctest", False) + mlirtest = body.get("mlirtest", False) + soctest = body.get("soctest", False) + if not all(isinstance(flag, bool) for flag in (ctest, mlirtest, soctest)): + return ApiResponse( + status=400, + body={"error": "--ctest, --mlirtest and --soctest must be boolean flags"}, + ) + if sum((ctest, mlirtest, soctest)) > 1: + return ApiResponse( + status=400, + body={"error": "--ctest, --mlirtest and --soctest are mutually exclusive"}, + ) + data = { + "design": design, + "ctest": ctest, + "mlirtest": mlirtest, + "soctest": soctest, + } + await ctx.enqueue( + { + "topic": "workload.build", + "data": {**data, "_trace_id": ctx.trace_id, "target": selection}, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/workload/01_build_event.step.py b/bbdev/api/steps/workload/01_build_event.step.py new file mode 100644 index 000000000..cc8a8dac1 --- /dev/null +++ b/bbdev/api/steps/workload/01_build_event.step.py @@ -0,0 +1,176 @@ +import importlib.util +import os +import sys +import re +from pathlib import Path +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) +from utils.path import get_buckyball_path, workload_build_dir +from utils.event_common import check_result, get_origin_trace_id + +_stack_build_path = os.path.join( + os.path.dirname(__file__), "..", "stack", "scripts", "build.py" +) +_spec = importlib.util.spec_from_file_location("stack_build_module", _stack_build_path) +stack_build = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(stack_build) +config = { + "name": "workload-build", + "description": "build workload", + "flows": ["workload"], + "triggers": [queue("workload.build")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + allowed = { + "target", + "device", + "core", + "design", + "ctest", + "mlirtest", + "soctest", + "_trace_id", + } + unknown = sorted((k for k in input_data if k not in allowed)) + if unknown: + ctx.logger.error(f"Unknown workload build parameter(s): {', '.join(unknown)}") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "unknown_parameter", "parameters": unknown}, + trace_id=origin_tid, + ) + return + design = input_data.get("design") + if not design: + ctx.logger.error("Missing required parameter: design must be specified") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "missing_design"}, + trace_id=origin_tid, + ) + return + if not isinstance(design, str) or not re.fullmatch("[A-Za-z0-9_-]+", design): + ctx.logger.error(f"Invalid design: {design}") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "invalid_design", "design": design}, + trace_id=origin_tid, + ) + return + design_dir = Path(bbdir) / "examples" / "designs" / design + if not design_dir.is_dir(): + ctx.logger.error(f"Workload design does not exist: {design}") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "unknown_design", "design": design}, + trace_id=origin_tid, + ) + return + ctest = input_data.get("ctest", False) + mlirtest = input_data.get("mlirtest", False) + soctest = input_data.get("soctest", False) + if not all(isinstance(flag, bool) for flag in (ctest, mlirtest, soctest)): + ctx.logger.error("--ctest, --mlirtest and --soctest must be boolean flags") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "invalid_test_scope"}, + trace_id=origin_tid, + ) + return + if sum((ctest, mlirtest, soctest)) > 1: + ctx.logger.error("--ctest, --mlirtest and --soctest are mutually exclusive") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "conflicting_test_scope"}, + trace_id=origin_tid, + ) + return + try: + # Workloads link the device runtime; MLIR tests compile with its plugins. + design_cmake = stack_build.build_design( + bbdir, + design=design, + selection=input_data["target"], + logger=ctx.logger, + task_scope=origin_tid, + ) + build = workload_build_dir(bbdir, design, selection=input_data["target"]) + stack_build.run( + [ + "cmake", + "-G", + "Ninja", + "-S", + os.path.join(bbdir, "bb-tests", "workloads"), + "-B", + build, + f"-DDESIGN_CMAKE={design_cmake}", + ], + repo=Path(bbdir), + logger=ctx.logger, + task_scope=origin_tid, + output_prefix="workload configure", + ) + target = ( + "sync-ctest-bin" + if ctest + else ( + "sync-mlirtest-bin" + if mlirtest + else "sync-soctest-bin" if soctest else "sync-bin" + ) + ) + stack_build.run( + ["ninja", "-C", build, "-j", "1", target], + repo=Path(bbdir), + logger=ctx.logger, + task_scope=origin_tid, + output_prefix="workload build", + ) + except Exception as error: + ctx.logger.error(str(error)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "error": "workload_build_failed", + "design": design, + "detail": str(error), + }, + trace_id=origin_tid, + ) + return + await check_result( + ctx, + 0, + continue_run=False, + extra_fields={ + "design": design, + "ctest": ctest, + "mlirtest": mlirtest, + "soctest": soctest, + }, + trace_id=origin_tid, + ) + return diff --git a/bbdev/api/steps/workload/02_tohex_api.step.py b/bbdev/api/steps/workload/02_tohex_api.step.py new file mode 100644 index 000000000..18e85fc38 --- /dev/null +++ b/bbdev/api/steps/workload/02_tohex_api.step.py @@ -0,0 +1,32 @@ +from utils.event_common import require_target +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.event_common import require_design + +config = { + "name": "workload-tohex-api", + "description": "convert design workload ELF to hex", + "flows": ["workload"], + "triggers": [api("POST", "/workload/tohex")], + "enqueues": ["workload.tohex"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + + await ctx.enqueue( + { + "topic": "workload.tohex", + "data": {**body, "_trace_id": ctx.trace_id, "target": selection}, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/workload/02_tohex_event.step.py b/bbdev/api/steps/workload/02_tohex_event.step.py new file mode 100644 index 000000000..273cbace1 --- /dev/null +++ b/bbdev/api/steps/workload/02_tohex_event.step.py @@ -0,0 +1,85 @@ +import os +import subprocess +import sys +from pathlib import Path + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.event_common import require_design +from utils.event_common import check_result, get_origin_trace_id +from utils.path import get_buckyball_path, workload_tests_root +from utils.stream_run import stream_run_logger_async + +config = { + "name": "workload-tohex", + "description": "convert all -baremetal ELF files under design workload output to hex", + "flows": ["workload"], + "triggers": [queue("workload.tohex")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + try: + design = require_design(input_data) + except ValueError as error: + ctx.logger.error(str(error)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "missing_design"}, + trace_id=origin_tid, + ) + return + + script = f"{bbdir}/bbdev/api/steps/workload/scripts/elf2hex.py" + search_root = Path( + workload_tests_root(bbdir, design, selection=input_data["target"]) + ) + + ctx.logger.info("Search root", {"path": str(search_root)}) + + if not search_root.is_dir(): + ctx.logger.error( + "Workload output directory not found", + {"path": str(search_root)}, + ) + await check_result(ctx, 1, continue_run=False, trace_id=origin_tid) + return + + elf_files = sorted(p for p in search_root.rglob("*-baremetal") if p.is_file()) + + if not elf_files: + ctx.logger.warn( + "No -baremetal ELF files found", + {"search_root": str(search_root)}, + ) + await check_result(ctx, 0, continue_run=False, trace_id=origin_tid) + return + + overall_rc = 0 + for elf in elf_files: + command = f"python3 {script} {elf}" + result = await stream_run_logger_async( + cmd=command, + logger=ctx.logger, + cwd=str(elf.parent), + executable="bash", + stdout_prefix=f"to-hex {elf.name}", + stderr_prefix=f"to-hex {elf.name}", + ) + if result.returncode != 0: + overall_rc = result.returncode + ctx.logger.error( + "elf2hex failed", + {"elf": str(elf), "returncode": result.returncode}, + ) + + await check_result(ctx, overall_rc, continue_run=False, trace_id=origin_tid) diff --git a/bbdev/api/steps/workload/README.md b/bbdev/api/steps/workload/README.md new file mode 100644 index 000000000..56b92f0a1 --- /dev/null +++ b/bbdev/api/steps/workload/README.md @@ -0,0 +1,65 @@ +# Workload Workflow + +Workload build workflow in Buckyball framework, used to build test workloads and benchmark programs. + +## API Usage + +### `clean` +**Endpoint**: `POST /workload/clean` + +**Function**: Clean workload output directory for one design. + +**Parameters**: +- **`design`** - Required design name. + +**Examples**: +```bash +bbdev workload --clean "--design toy" +``` + +### `build` +**Endpoint**: `POST /workload/build` + +**Function**: Build workload + +**Parameters**: +- **`design`** - Required design name. Selects design-specific workloads. +- **`stable`** - Optional boolean flag. If set, build with stable LLVM Buckyball extensions. +- **`ctest`** - Build CTest workloads only. +- **`mlirtest`** - Build MLIRTest workloads only. +- **`soctest`** - Build SoC test workloads only. + +For design workloads under paths like `*/designs/`, only the directory selected by `design` is synced to `bb-tests/output//workloads`. + +**Examples**: +```bash +# Build all workloads +bbdev workload --build "--design toy" + +# Build only CTest workloads +bbdev workload --build "--design pebble --ctest" + +# Build only MLIRTest workloads +bbdev workload --build "--design pebble --mlirtest" + +# Build only SoC test workloads +bbdev workload --build "--design goban --soctest" +``` + +**Response**: +```json +{ + "status": 200, + "body": { + "success": true, + "processing": false, + "return_code": 0 + } +} +``` + +## Notes + +- Workload build directory is `bb-tests/workloads/build/` +- Workload source code is distributed under `bb-tests/workloads/src` and `examples/*/*/workloads` +- Workload binaries are emitted under `bb-tests/output//workloads/src` diff --git a/bbdev/api/steps/workload/scripts/elf2hex.py b/bbdev/api/steps/workload/scripts/elf2hex.py new file mode 100755 index 000000000..782f2b080 --- /dev/null +++ b/bbdev/api/steps/workload/scripts/elf2hex.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""ELF -> hex converter for Buckyball P2E DDR loading. + +Output format: + @0x0 + XX + XX + ... +where XX is one byte per line (uppercase hex). + +The hex file represents bytes starting at DDR offset 0, which maps to +CPU address 0x80000000. All LOAD segments are concatenated by physical +address, with gaps padded by zeros. +""" + +import argparse +import subprocess +import sys +from pathlib import Path + + +def elf_to_bin(elf_path: Path, bin_path: Path, objcopy: str) -> None: + subprocess.run( + [objcopy, "-O", "binary", str(elf_path), str(bin_path)], + check=True, + ) + + +def bin_to_hex(bin_path: Path, hex_path: Path) -> int: + data = bin_path.read_bytes() + # VDBG accepts hexadecimal address markers only, including address zero. + lines = ["@0x0"] + lines.extend(f"{b:02X}" for b in data) + hex_path.write_text("\n".join(lines) + "\n") + return len(data) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("elf", type=Path, help="Input ELF file") + parser.add_argument( + "hex", + type=Path, + nargs="?", + help="Output hex file (default: .hex)", + ) + parser.add_argument( + "--objcopy", + default="riscv64-unknown-elf-objcopy", + help="objcopy binary (default: riscv64-unknown-elf-objcopy)", + ) + args = parser.parse_args() + + if not args.elf.is_file(): + print(f"error: ELF not found: {args.elf}", file=sys.stderr) + return 1 + + hex_path = args.hex or args.elf.with_suffix(".hex") + bin_path = hex_path.with_suffix(".bin") + + try: + elf_to_bin(args.elf, bin_path, args.objcopy) + except FileNotFoundError: + print(f"error: objcopy not found: {args.objcopy}", file=sys.stderr) + print("hint: run inside `nix develop` shell", file=sys.stderr) + return 1 + except subprocess.CalledProcessError as e: + print(f"error: objcopy failed: {e}", file=sys.stderr) + return 1 + + n = bin_to_hex(bin_path, hex_path) + bin_path.unlink() + + print(f"wrote {hex_path} ({n} bytes, {n + 1} lines)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bbdev/api/steps/yosys/.gitignore b/bbdev/api/steps/yosys/.gitignore new file mode 100644 index 000000000..6bfe6b19e --- /dev/null +++ b/bbdev/api/steps/yosys/.gitignore @@ -0,0 +1 @@ +log diff --git a/bbdev/api/steps/yosys/01_run_api.step.py b/bbdev/api/steps/yosys/01_run_api.step.py new file mode 100644 index 000000000..0e67e3bfc --- /dev/null +++ b/bbdev/api/steps/yosys/01_run_api.step.py @@ -0,0 +1,64 @@ +from utils.event_common import require_target +import os +import sys + +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.event_common import require_design + +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) + +from yosys_log import req_arg + +config = { + "name": "yosys-run", + "description": "run yosys flow", + "flows": ["yosys"], + "triggers": [api("POST", "/yosys/run")], + "enqueues": ["yosys.run"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = req.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + unknown = body.keys() - { + "device", + "core", + "design", + "top", + "output-dir", + "output_dir", + "vcd", + } + if unknown: + return ApiResponse( + status=400, + body={"error": f"Unknown parameter(s): {', '.join(sorted(unknown))}"}, + ) + try: + design = require_design(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) + + data = { + "design": design, + "top": req_arg(body, "top"), + "vcd": req_arg(body, "vcd"), + "from_run_workflow": True, + } + output_dir = req_arg(body, "output_dir") + if output_dir: + data["output_dir"] = output_dir + await ctx.enqueue( + { + "topic": "yosys.run", + "data": {**data, "_trace_id": ctx.trace_id, "target": selection}, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/yosys/02_verilog_api.step.py b/bbdev/api/steps/yosys/02_verilog_api.step.py new file mode 100644 index 000000000..0b0164cf9 --- /dev/null +++ b/bbdev/api/steps/yosys/02_verilog_api.step.py @@ -0,0 +1,58 @@ +from utils.event_common import require_target +import os +import sys + +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.event_common import require_design + +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) + +from yosys_log import req_arg + +config = { + "name": "yosys-verilog-api", + "description": "generate verilog for yosys flow", + "flows": ["yosys"], + "triggers": [api("POST", "/yosys/verilog")], + "enqueues": ["yosys.verilog"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = req.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + unknown = body.keys() - { + "device", + "core", + "design", + "top", + "output-dir", + "output_dir", + } + if unknown: + return ApiResponse( + status=400, + body={"error": f"Unknown parameter(s): {', '.join(sorted(unknown))}"}, + ) + try: + design = require_design(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) + + data = {"design": design, "top": req_arg(body, "top")} + output_dir = req_arg(body, "output_dir") + if output_dir: + data["output_dir"] = output_dir + await ctx.enqueue( + { + "topic": "yosys.verilog", + "data": {**data, "_trace_id": ctx.trace_id, "target": selection}, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/yosys/03_synth_api.step.py b/bbdev/api/steps/yosys/03_synth_api.step.py new file mode 100644 index 000000000..066caf325 --- /dev/null +++ b/bbdev/api/steps/yosys/03_synth_api.step.py @@ -0,0 +1,67 @@ +from utils.event_common import require_target +import os +import sys + +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.event_common import require_design + +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) + +from yosys_log import req_arg + +config = { + "name": "yosys-synth-api", + "description": "run yosys synthesis for area estimation", + "flows": ["yosys"], + "triggers": [api("POST", "/yosys/synth")], + "enqueues": ["ip.generate"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = req.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + unknown = body.keys() - { + "device", + "core", + "design", + "top", + "output-dir", + "output_dir", + "vcd", + } + if unknown: + return ApiResponse( + status=400, + body={"error": f"Unknown parameter(s): {', '.join(sorted(unknown))}"}, + ) + try: + design = require_design(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) + + data = { + "design": design, + "consumer": "yosys", + "top": req_arg(body, "top"), + "next_topic": "yosys.synth", + } + output_dir = req_arg(body, "output_dir") + if output_dir: + data["output_dir"] = output_dir + vcd = req_arg(body, "vcd") + if vcd: + data["vcd"] = vcd + await ctx.enqueue( + { + "topic": "ip.generate", + "data": {**data, "_trace_id": ctx.trace_id, "target": selection}, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/yosys/03_synth_event.step.py b/bbdev/api/steps/yosys/03_synth_event.step.py new file mode 100644 index 000000000..3de7df02f --- /dev/null +++ b/bbdev/api/steps/yosys/03_synth_event.step.py @@ -0,0 +1,306 @@ +import json +import os +import re +import sys +import yaml +from datetime import datetime + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.event_common import require_design +from utils.path import rtl_dir, get_buckyball_path, log_dir +from utils.stream_run import stream_run_logger_async +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "yosys synth", + "description": "run yosys synthesis for area estimation", + "flows": ["yosys"], + "triggers": [queue("yosys.synth")], + "enqueues": [], +} + + +def dynamic_power_report( + power_report: str, activity_source: str +) -> tuple[str, float] | None: + """Return OpenSTA's internal-plus-switching power, excluding leakage.""" + match = re.search( + r"^Total\s+([0-9.eE+-]+)\s+([0-9.eE+-]+)\s+([0-9.eE+-]+)\s+([0-9.eE+-]+)", + power_report, + re.M, + ) + if match is None: + return None + internal, switching, _leakage, _total = (float(value) for value in match.groups()) + dynamic = internal + switching + return ( + ( + "OpenSTA dynamic power\n" + f" Activity source: {activity_source}\n" + f" Internal: {internal:.6e} W\n" + f" Switching: {switching:.6e} W\n" + f" Dynamic (internal + switching): {dynamic:.6e} W\n" + ), + dynamic, + ) + + +def load_yosys_config(): + config_path = os.path.join( + os.path.dirname(__file__), "scripts", "yosys-config.yaml" + ) + if os.path.exists(config_path): + with open(config_path, "r") as f: + config = yaml.safe_load(f) or {} + # Keep paths portable between the Nix shell and explicit user + # configurations while retaining normal YAML scalar values. + if isinstance(config.get("liberty"), str): + config["liberty"] = os.path.expandvars( + os.path.expanduser(config["liberty"]) + ) + if isinstance(config.get("vcd"), str) and config["vcd"]: + config["vcd"] = os.path.expandvars(os.path.expanduser(config["vcd"])) + return config + return {} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + try: + design = require_design(input_data) + build_dir = rtl_dir( + bbdir, + design, + "synth", + input_data.get("output_dir"), + selection=input_data["target"], + ) + except ValueError as exc: + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "synth", "error": str(exc)}, + trace_id=origin_tid, + ) + return + + yosys_cfg = load_yosys_config() + top_module = input_data.get("top") or yosys_cfg["top"] + liberty = yosys_cfg.get("liberty") + if isinstance(liberty, str): + liberty = os.path.expandvars(os.path.expanduser(liberty)) + + source_list_path = input_data.get("source_list") + if not isinstance(source_list_path, str) or not os.path.isfile(source_list_path): + _, failure_result = await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "task": "synth", + "error": "missing prepared yosys source list", + }, + trace_id=origin_tid, + ) + return failure_result + + with open(source_list_path, "r") as f: + vsrcs = [line.strip() for line in f.readlines() if line.strip()] + + if not vsrcs: + _, failure_result = await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "synth", "error": "empty source list"}, + trace_id=origin_tid, + ) + return failure_result + + stamp = datetime.now().strftime("%Y-%m-%d-%H-%M") + yosys_output_dir = input_data.get("analysis_dir") or log_dir( + bbdir, + design, + "synth", + stamp, + "yosys", + top_module, + input_data.get("output_dir"), + selection=input_data["target"], + ) + os.makedirs(yosys_output_dir, exist_ok=True) + ctx.logger.info(f"Yosys log dir: {yosys_output_dir}") + + read_commands = "\n".join([f"read_verilog -sv {src}" for src in vsrcs]) + # SRAM macros instantiated by sram_macros.v are cells of their own liberty. + with open(input_data["generate_manifest"], "r") as f: + leaf_paths = json.load(f)["leaf_paths"] + for leaf in leaf_paths.values(): + read_commands += f"\nread_liberty -lib {leaf['lib']}" + yosys_script = f"{yosys_output_dir}/synth_area.ys" + with open(yosys_script, "w") as f: + f.write(f"{read_commands}\n") + f.write(f"hierarchy -top {top_module}\n") + f.write("proc\n") + f.write("opt\n") + f.write(f"synth -top {top_module}\n") + + if liberty and os.path.exists(liberty): + f.write(f"dfflibmap -liberty {liberty}\n") + f.write(f"abc -liberty {liberty}\n") + f.write( + f"tee -o {yosys_output_dir}/hierarchy_report.txt stat -liberty" + f" {liberty}\n" + ) + f.write("flatten\n") + f.write("opt\n") + f.write(f"stat -liberty {liberty}\n") + f.write( + f"tee -o {yosys_output_dir}/area_report.txt stat -liberty {liberty}\n" + ) + f.write(f"write_verilog {yosys_output_dir}/synth_netlist.v\n") + else: + f.write(f"tee -o {yosys_output_dir}/hierarchy_report.txt stat\n") + f.write("flatten\n") + f.write("opt\n") + f.write("stat\n") + f.write(f"tee -o {yosys_output_dir}/area_report.txt stat\n") + + result = await stream_run_logger_async( + cmd=f"yosys -s {yosys_script}", + logger=ctx.logger, + cwd=build_dir, + stdout_prefix="yosys synth", + stderr_prefix="yosys synth", + ) + + extra = { + "task": "synth", + "output_dir": yosys_output_dir, + "replace_manifest": input_data.get("replace_manifest"), + } + netlist_file = f"{yosys_output_dir}/synth_netlist.v" + timing_report_file = f"{yosys_output_dir}/timing_report.txt" + power_report_file = f"{yosys_output_dir}/power_report.txt" + dynamic_power_report_file = f"{yosys_output_dir}/dynamic_power_report.txt" + sta_returncode = 0 + + if ( + liberty + and os.path.exists(liberty) + and os.path.exists(netlist_file) + and result.returncode == 0 + ): + clock_period = yosys_cfg.get("clock_period", 10.0) + clock_name = yosys_cfg.get("clock_name", "clock") + vcd = input_data.get("vcd") or yosys_cfg.get("vcd") + if isinstance(vcd, str): + vcd = os.path.expandvars(os.path.expanduser(vcd)) + if vcd and not os.path.isfile(vcd): + extra["error"] = f"VCD file does not exist: {vcd}" + sta_returncode = 1 + vcd = None + if sta_returncode: + await check_result( + ctx, + sta_returncode, + continue_run=False, + extra_fields=extra, + trace_id=origin_tid, + ) + return + sta_script = f"{yosys_output_dir}/sta_timing.tcl" + with open(sta_script, "w") as f: + f.write(f"read_liberty {liberty}\n") + f.write(f"read_verilog {netlist_file}\n") + f.write(f"link_design {top_module}\n") + # Prefer the configured port, then use the generated-memory clock + # naming convention. Some small modules are combinational and + # therefore have no clock at all. + f.write(f"set bb_clock_port [get_ports -quiet {clock_name}]\n") + f.write( + "if {[llength $bb_clock_port] == 0} { set bb_clock_port [get_ports" + " -quiet *clk*] }\n" + ) + f.write( + "if {[llength $bb_clock_port] > 0} { create_clock $bb_clock_port" + f" -name clk -period {clock_period} }}\n" + ) + f.write( + f"report_checks -path_delay max -format full > {timing_report_file}\n" + ) + f.write("report_checks -path_delay max -format full\n") + if vcd and os.path.exists(vcd): + f.write(f"read_vcd {vcd} -scope {top_module}\n") + extra["power_activity_source"] = vcd + else: + input_activity = yosys_cfg.get("input_activity", 0.1) + input_duty = yosys_cfg.get("input_duty", 0.5) + clock_activity = yosys_cfg.get("clock_activity", 1.0) + clock_duty = yosys_cfg.get("clock_duty", 0.5) + f.write( + f"set_power_activity -input -activity {input_activity} -duty" + f" {input_duty}\n" + ) + # OpenSTA expects the clock collection after ``-clock``. + # Applying the default rate to all clocks keeps this usable + # for generated designs whose clock port is renamed. + f.write( + "if {[llength [all_clocks]] > 0} { set_power_activity -clock" + f" [all_clocks] -activity {clock_activity} -duty {clock_duty} }}\n" + ) + extra["power_activity_source"] = "default" + f.write(f"report_power -digits 4 > {power_report_file}\n") + f.write("exit\n") + + sta_result = await stream_run_logger_async( + cmd=f"sta {sta_script}", + logger=ctx.logger, + cwd=yosys_output_dir, + stdout_prefix="opensta", + stderr_prefix="opensta", + ) + + if sta_result.returncode == 0 and os.path.exists(timing_report_file): + with open(timing_report_file, "r") as f: + extra["timing_report"] = f.read() + sta_returncode = sta_result.returncode + if sta_result.returncode == 0 and os.path.exists(power_report_file): + with open(power_report_file, "r") as f: + extra["power_report"] = f.read() + dynamic_result = dynamic_power_report( + extra["power_report"], extra.get("power_activity_source", "unknown") + ) + if dynamic_result is not None: + dynamic_report, dynamic_power = dynamic_result + with open(dynamic_power_report_file, "w") as f: + f.write(dynamic_report) + extra["dynamic_power_report"] = dynamic_report + extra["dynamic_power_w"] = dynamic_power + + report_file = f"{yosys_output_dir}/area_report.txt" + if os.path.exists(report_file): + with open(report_file, "r") as f: + extra["area_report"] = f.read() + + hierarchy_file = f"{yosys_output_dir}/hierarchy_report.txt" + if os.path.exists(hierarchy_file): + with open(hierarchy_file, "r") as f: + extra["hierarchy_report"] = f.read() + + await check_result( + ctx, + result.returncode or sta_returncode, + continue_run=False, + extra_fields=extra, + trace_id=origin_tid, + ) + + return diff --git a/bbdev/api/steps/yosys/scripts/yosys-config.yaml b/bbdev/api/steps/yosys/scripts/yosys-config.yaml new file mode 100644 index 000000000..6cafe815c --- /dev/null +++ b/bbdev/api/steps/yosys/scripts/yosys-config.yaml @@ -0,0 +1,27 @@ +# Yosys synthesis configuration +# Users can customize liberty library paths and synthesis options here + +# Liberty (.lib) file path for ASIC area estimation +# If not specified, yosys will use generic gate-level statistics. +liberty: + +# Default top module name (override with --top on CLI) +top: DeviceTop + +# OpenSTA timing analysis settings +# Clock period in nanoseconds (used for STA constraint) +# Default 10ns = 100MHz +clock_period: 10.0 + +# Clock port name in the design +clock_name: clock + +# Power estimation reports dynamic power as Internal + Switching; leakage is +# retained in OpenSTA's raw report but not included in dynamic_power_report.txt. +# Supply a workload VCD through `bbdev yosys --synth --vcd ` for an +# activity-driven result. Empty vcd uses the default toggle-rate estimate below. +vcd: +input_activity: 0.1 +input_duty: 0.5 +clock_activity: 1.0 +clock_duty: 0.5 diff --git a/bbdev/api/steps/yosys/scripts/yosys_log.py b/bbdev/api/steps/yosys/scripts/yosys_log.py new file mode 100644 index 000000000..c0bcd8565 --- /dev/null +++ b/bbdev/api/steps/yosys/scripts/yosys_log.py @@ -0,0 +1,2 @@ +def req_arg(body: dict, name: str): + return body.get(name) or body.get(name.replace("_", "-")) diff --git a/bbdev/api/tests/_api_test_helper.py b/bbdev/api/tests/_api_test_helper.py new file mode 100644 index 000000000..662c52b15 --- /dev/null +++ b/bbdev/api/tests/_api_test_helper.py @@ -0,0 +1,6 @@ +import shlex +import subprocess + + +def run_bbdev_case(command: str): + subprocess.run(shlex.split(command), check=True) diff --git a/bbdev/api/tests/fixtures/bemu-analysis/bdb.ndjson b/bbdev/api/tests/fixtures/bemu-analysis/bdb.ndjson new file mode 100644 index 000000000..5cafa8fed --- /dev/null +++ b/bbdev/api/tests/fixtures/bemu-analysis/bdb.ndjson @@ -0,0 +1,4 @@ +{"type":"itrace","clk":10,"event":"complete","funct":"0x00","pc":"0x0","rs1":"0x0","rs2":"0x0"} +{"type":"itrace","clk":20,"event":"complete","funct":"0x41","pc":"0x0","rs1":"0x0","rs2":"0x0000000100010400"} +{"type":"mtrace","clk":15,"event":"read","addr":"0x0","rows":1024,"line_bytes":16,"row_stride":16,"vbank_id":0,"pbank_id":0,"group_id":0} +{"type":"mtrace","clk":16,"event":"write","addr":"0x0","rows":1,"line_bytes":16,"row_stride":16,"vbank_id":1,"pbank_id":1,"group_id":0} diff --git a/bbdev/api/tests/test_all_api_serial.py b/bbdev/api/tests/test_all_api_serial.py new file mode 100644 index 000000000..1b59f4279 --- /dev/null +++ b/bbdev/api/tests/test_all_api_serial.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 + +import subprocess +from pathlib import Path + + +def main(): + test_dir = Path(__file__).resolve().parent + tests = [ + "test_api_stack_build.py", + "test_api_workload_clean.py", + "test_api_workload_build.py", + "test_api_verilator_clean.py", + "test_api_verilator_verilog.py", + "test_api_verilator_build.py", + "test_api_verilator_sim.py", + "test_api_verilator_run.py", + "test_api_yosys_run.py", + "test_api_yosys_verilog.py", + "test_api_yosys_synth.py", + "test_api_dc_verilog.py", + "test_api_uvm_build.py", + "test_api_uvm_run.py", + "test_api_kernel_build.py", + "test_api_firesim_enumeratefpgas.py", + "test_api_firesim_buildbitstream.py", + "test_api_firesim_infrasetup.py", + "test_api_firesim_runworkload.py", + "test_api_bebop_bemu_sim.py", + "test_api_bebop_bemu_analysis.py", + "test_api_bebop_bemu_batch.py", + "test_api_bebop_verilator_clean.py", + "test_api_bebop_verilator_verilog.py", + "test_api_bebop_verilator_build.py", + "test_api_bebop_verilator_sim.py", + "test_api_bebop_verilator_run.py", + "test_api_bebop_verilator_batch.py", + "test_api_bebop_p2e_clean.py", + "test_api_bebop_p2e_verilog.py", + "test_api_bebop_p2e_buildbitstream.py", + "test_api_bebop_p2e_runworkload.py", + ] + for name in tests: + path = test_dir / name + if not path.exists(): + raise RuntimeError(f"missing test: {name}") + print(f"RUN {name}") + ret = subprocess.run(["python", str(path)], cwd=str(test_dir)) + if ret.returncode != 0: + raise RuntimeError(f"failed: {name}") + print(f"PASS: {len(tests)} api tests done.") + + +if __name__ == "__main__": + main() diff --git a/bbdev/api/tests/test_api_bebop_bemu_analysis.py b/bbdev/api/tests/test_api_bebop_bemu_analysis.py new file mode 100644 index 000000000..d4ed24226 --- /dev/null +++ b/bbdev/api/tests/test_api_bebop_bemu_analysis.py @@ -0,0 +1,32 @@ +from datetime import datetime +from pathlib import Path +import shlex +import shutil +import sys +from uuid import uuid4 + +from _api_test_helper import run_bbdev_case + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from utils.path import get_buckyball_path, log_dir + +run_log = Path( + log_dir( + get_buckyball_path(), + "pebble", + "verilog", + datetime.now().strftime("%Y-%m-%d-%H-%M"), + "bemu", + f"analysis-test-{uuid4().hex}", + ) +) +run_log.mkdir(parents=True) +try: + shutil.copyfile( + Path(__file__).parent / "fixtures/bemu-analysis/bdb.ndjson", + run_log / "bdb.ndjson", + ) + arguments = f"--design pebble --run {run_log.name} --itrace --mtrace" + run_bbdev_case(f"bbdev bebop-bemu --analysis {shlex.quote(arguments)}") +finally: + shutil.rmtree(run_log) diff --git a/bbdev/api/tests/test_api_bebop_bemu_batch.py b/bbdev/api/tests/test_api_bebop_bemu_batch.py new file mode 100644 index 000000000..94d11e0a4 --- /dev/null +++ b/bbdev/api/tests/test_api_bebop_bemu_batch.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev bebop-bemu --batch '--design toy --test bare-tests'") diff --git a/bbdev/api/tests/test_api_bebop_bemu_sim.py b/bbdev/api/tests/test_api_bebop_bemu_sim.py new file mode 100644 index 000000000..9f36f0ce1 --- /dev/null +++ b/bbdev/api/tests/test_api_bebop_bemu_sim.py @@ -0,0 +1,6 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case( + "bbdev bebop-bemu --sim '--design toy --binary toy_relu_test-singlecore-baremetal'" +) +run_bbdev_case("bbdev bebop-bemu --sim '--design toy --binary toy_relu_test-linux'") diff --git a/bbdev/api/tests/test_api_bebop_p2e_buildbitstream.py b/bbdev/api/tests/test_api_bebop_p2e_buildbitstream.py new file mode 100644 index 000000000..e05880561 --- /dev/null +++ b/bbdev/api/tests/test_api_bebop_p2e_buildbitstream.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev bebop-p2e --buildbitstream '--design toy'") diff --git a/bbdev/api/tests/test_api_bebop_p2e_clean.py b/bbdev/api/tests/test_api_bebop_p2e_clean.py new file mode 100644 index 000000000..e1b431233 --- /dev/null +++ b/bbdev/api/tests/test_api_bebop_p2e_clean.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev bebop-p2e --clean '--design toy'") diff --git a/bbdev/api/tests/test_api_bebop_p2e_runworkload.py b/bbdev/api/tests/test_api_bebop_p2e_runworkload.py new file mode 100644 index 000000000..06a828bbb --- /dev/null +++ b/bbdev/api/tests/test_api_bebop_p2e_runworkload.py @@ -0,0 +1,7 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case( + "bbdev bebop-p2e --runworkload '--design toy --image vecunit_matmul_ones" + " --bitstream" + " /home/wanghui/Code/buckyball/bebop/build/p2e/sims.p2e.P2EToyConfig/bitstream.bit'" +) diff --git a/bbdev/api/tests/test_api_bebop_p2e_verilog.py b/bbdev/api/tests/test_api_bebop_p2e_verilog.py new file mode 100644 index 000000000..a50fa4f19 --- /dev/null +++ b/bbdev/api/tests/test_api_bebop_p2e_verilog.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev bebop-p2e --verilog '--design toy'") diff --git a/bbdev/api/tests/test_api_bebop_verilator_batch.py b/bbdev/api/tests/test_api_bebop_verilator_batch.py new file mode 100644 index 000000000..f81ba8d21 --- /dev/null +++ b/bbdev/api/tests/test_api_bebop_verilator_batch.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev bebop-verilator --batch '--design toy --test bare-tests'") diff --git a/bbdev/api/tests/test_api_bebop_verilator_build.py b/bbdev/api/tests/test_api_bebop_verilator_build.py new file mode 100644 index 000000000..73a39f88d --- /dev/null +++ b/bbdev/api/tests/test_api_bebop_verilator_build.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev bebop-verilator --build '--jobs 16 --design toy'") diff --git a/bbdev/api/tests/test_api_bebop_verilator_clean.py b/bbdev/api/tests/test_api_bebop_verilator_clean.py new file mode 100644 index 000000000..f9ce532a3 --- /dev/null +++ b/bbdev/api/tests/test_api_bebop_verilator_clean.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev bebop-verilator --clean '--design toy'") diff --git a/bbdev/api/tests/test_api_bebop_verilator_run.py b/bbdev/api/tests/test_api_bebop_verilator_run.py new file mode 100644 index 000000000..3a9bede5e --- /dev/null +++ b/bbdev/api/tests/test_api_bebop_verilator_run.py @@ -0,0 +1,5 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case( + "bbdev bebop-verilator --run '--diff --no-wave --jobs 16 --binary toy_relu_test-singlecore-baremetal --design toy'" +) diff --git a/bbdev/api/tests/test_api_bebop_verilator_sim.py b/bbdev/api/tests/test_api_bebop_verilator_sim.py new file mode 100644 index 000000000..2472e5e2e --- /dev/null +++ b/bbdev/api/tests/test_api_bebop_verilator_sim.py @@ -0,0 +1,6 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case( + "bbdev bebop-verilator --sim '--binary toy_relu_test-singlecore-baremetal --design" + " toy --itrace --mtrace --pmctrace --ctrace --banktrace'" +) diff --git a/bbdev/api/tests/test_api_bebop_verilator_verilog.py b/bbdev/api/tests/test_api_bebop_verilator_verilog.py new file mode 100644 index 000000000..8a6be92cc --- /dev/null +++ b/bbdev/api/tests/test_api_bebop_verilator_verilog.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev bebop-verilator --verilog '--design toy'") diff --git a/bbdev/api/tests/test_api_dc_verilog.py b/bbdev/api/tests/test_api_dc_verilog.py new file mode 100644 index 000000000..b8c5533d0 --- /dev/null +++ b/bbdev/api/tests/test_api_dc_verilog.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev dc --verilog '--design toy'") diff --git a/bbdev/api/tests/test_api_firesim_buildbitstream.py b/bbdev/api/tests/test_api_firesim_buildbitstream.py new file mode 100644 index 000000000..320c515bf --- /dev/null +++ b/bbdev/api/tests/test_api_firesim_buildbitstream.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev firesim --buildbitstream '--design toy'") diff --git a/bbdev/api/tests/test_api_firesim_enumeratefpgas.py b/bbdev/api/tests/test_api_firesim_enumeratefpgas.py new file mode 100644 index 000000000..20654bbe4 --- /dev/null +++ b/bbdev/api/tests/test_api_firesim_enumeratefpgas.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev firesim --enumeratefpgas '--design toy'") diff --git a/bbdev/api/tests/test_api_firesim_infrasetup.py b/bbdev/api/tests/test_api_firesim_infrasetup.py new file mode 100644 index 000000000..ac0a02ede --- /dev/null +++ b/bbdev/api/tests/test_api_firesim_infrasetup.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev firesim --infrasetup '--design toy'") diff --git a/bbdev/api/tests/test_api_firesim_runworkload.py b/bbdev/api/tests/test_api_firesim_runworkload.py new file mode 100644 index 000000000..b46c3a665 --- /dev/null +++ b/bbdev/api/tests/test_api_firesim_runworkload.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev firesim --runworkload '--design toy'") diff --git a/bbdev/api/tests/test_api_kernel_build.py b/bbdev/api/tests/test_api_kernel_build.py new file mode 100644 index 000000000..732a5e8db --- /dev/null +++ b/bbdev/api/tests/test_api_kernel_build.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev kernel --build '--design toy'") diff --git a/bbdev/api/tests/test_api_stack_build.py b/bbdev/api/tests/test_api_stack_build.py new file mode 100644 index 000000000..5fabf4627 --- /dev/null +++ b/bbdev/api/tests/test_api_stack_build.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev stack --build '--design toy'") diff --git a/bbdev/api/tests/test_api_uvm_build.py b/bbdev/api/tests/test_api_uvm_build.py new file mode 100644 index 000000000..491866b53 --- /dev/null +++ b/bbdev/api/tests/test_api_uvm_build.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev uvm --build '--design toy --ip axis'") diff --git a/bbdev/api/tests/test_api_uvm_run.py b/bbdev/api/tests/test_api_uvm_run.py new file mode 100644 index 000000000..dad9a320e --- /dev/null +++ b/bbdev/api/tests/test_api_uvm_run.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev uvm --run '--design toy --ip axis'") diff --git a/bbdev/api/tests/test_api_verilator_build.py b/bbdev/api/tests/test_api_verilator_build.py new file mode 100644 index 000000000..3440ef00c --- /dev/null +++ b/bbdev/api/tests/test_api_verilator_build.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev verilator --build '--jobs 16 --design toy'") diff --git a/bbdev/api/tests/test_api_verilator_clean.py b/bbdev/api/tests/test_api_verilator_clean.py new file mode 100644 index 000000000..ee2f8a631 --- /dev/null +++ b/bbdev/api/tests/test_api_verilator_clean.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev verilator --clean '--design toy'") diff --git a/bbdev/api/tests/test_api_verilator_run.py b/bbdev/api/tests/test_api_verilator_run.py new file mode 100644 index 000000000..821209439 --- /dev/null +++ b/bbdev/api/tests/test_api_verilator_run.py @@ -0,0 +1,5 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case( + "bbdev verilator --run '--jobs 16 --binary toy-toy-ctest-mvin_mvout_test-baremetal --design toy --batch'" +) diff --git a/bbdev/api/tests/test_api_verilator_sim.py b/bbdev/api/tests/test_api_verilator_sim.py new file mode 100644 index 000000000..6b0aea79b --- /dev/null +++ b/bbdev/api/tests/test_api_verilator_sim.py @@ -0,0 +1,5 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case( + "bbdev verilator --sim '--binary toy-toy-ctest-mvin_mvout_test-baremetal --batch --design toy'" +) diff --git a/bbdev/api/tests/test_api_verilator_verilog.py b/bbdev/api/tests/test_api_verilator_verilog.py new file mode 100644 index 000000000..1c4b29475 --- /dev/null +++ b/bbdev/api/tests/test_api_verilator_verilog.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev verilator --verilog '--design toy'") diff --git a/bbdev/api/tests/test_api_workload_build.py b/bbdev/api/tests/test_api_workload_build.py new file mode 100644 index 000000000..54c3dcc97 --- /dev/null +++ b/bbdev/api/tests/test_api_workload_build.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev workload --build '--design toy'") diff --git a/bbdev/api/tests/test_api_workload_clean.py b/bbdev/api/tests/test_api_workload_clean.py new file mode 100644 index 000000000..fc83995a8 --- /dev/null +++ b/bbdev/api/tests/test_api_workload_clean.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev workload --clean '--design toy'") diff --git a/bbdev/api/tests/test_api_workload_tohex.py b/bbdev/api/tests/test_api_workload_tohex.py new file mode 100644 index 000000000..6faf6996a --- /dev/null +++ b/bbdev/api/tests/test_api_workload_tohex.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev workload --tohex '--design toy'") diff --git a/bbdev/api/tests/test_api_yosys_run.py b/bbdev/api/tests/test_api_yosys_run.py new file mode 100644 index 000000000..bcaa238e3 --- /dev/null +++ b/bbdev/api/tests/test_api_yosys_run.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev yosys --run '--design toy'") diff --git a/bbdev/api/tests/test_api_yosys_synth.py b/bbdev/api/tests/test_api_yosys_synth.py new file mode 100644 index 000000000..032b9314e --- /dev/null +++ b/bbdev/api/tests/test_api_yosys_synth.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev yosys --synth '--design toy'") diff --git a/bbdev/api/tests/test_api_yosys_verilog.py b/bbdev/api/tests/test_api_yosys_verilog.py new file mode 100644 index 000000000..a899a059c --- /dev/null +++ b/bbdev/api/tests/test_api_yosys_verilog.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev yosys --verilog '--design toy'") diff --git a/bbdev/api/utils/__init__.py b/bbdev/api/utils/__init__.py new file mode 100644 index 000000000..6db5263f8 --- /dev/null +++ b/bbdev/api/utils/__init__.py @@ -0,0 +1,10 @@ +from .stream_run import stream_run +from .path import get_buckyball_path +from .port import find_available_port +from .search_workload import ( + search_workload, + search_workload_all, + search_workload_pattern, +) + +__all__ = ["stream_run", "get_buckyball_path", "find_available_port", "search_workload"] diff --git a/bbdev/api/utils/event_common.py b/bbdev/api/utils/event_common.py new file mode 100644 index 000000000..1f7cd1113 --- /dev/null +++ b/bbdev/api/utils/event_common.py @@ -0,0 +1,113 @@ +""" +Common utility functions for all event steps. +""" + +import re + +from utils.path import design_pb_path, get_buckyball_path +from utils.process_registry import set_current_task_scope + + +def require_design(data: dict) -> str: + design = data.get("design") + if not isinstance(design, str) or not design: + raise ValueError("Missing required parameter: --design") + extra = data.get("config") + if isinstance(extra, str) and extra and extra != "None": + raise ValueError( + "do not pass --config; mill class comes from design config sims" + ) + if not re.fullmatch(r"[A-Za-z0-9_-]+", design): + raise ValueError(f"invalid design: {design}") + return design + + +def require_target(data: dict, kinds=("device", "core")) -> str: + """The selection of a request: device/ for `--device `, a + config registered by the design's sims config, or core/.""" + selected = [kind for kind in ("device", "core", "ip", "ball") if kind in data] + if len(selected) != 1: + raise ValueError( + "Specify exactly one of --device , --core , --ip , --ball after --design" + ) + kind = selected[0] + if kind not in kinds: + raise ValueError(f"--{kind} is not supported by this operation") + value = data[kind] + if not isinstance(value, str) or not re.fullmatch(r"[A-Za-z0-9_-]+", value): + raise ValueError(f"--{kind} requires a name") + if kind != "device": + return f"{kind}/{value}" + design = require_design(data) + pb = design_pb_path(get_buckyball_path(), design, selection=f"device/{value}") + if not pb.is_file(): + raise ValueError( + f"design {design!r} registers no device config {value!r}" + " (run bbdev config --install after editing its sims config)" + ) + return f"device/{value}" + + +def get_origin_trace_id(input_data, ctx): + """Get the origin trace_id from input_data (passed by API step) or fall back to ctx.trace_id. + + iii 0.7+ assigns a new trace_id per handler invocation, so the API step's + trace_id must be forwarded explicitly via input_data["_trace_id"]. + """ + trace_id = ( + input_data["_trace_id"] + if isinstance(input_data, dict) and "_trace_id" in input_data + else ctx.trace_id + ) + set_current_task_scope(trace_id) + return trace_id + + +async def check_result( + ctx, returncode, continue_run=False, extra_fields=None, trace_id=None +): + """ + Check returncode, create appropriate result objects and set state. + + Args: + ctx: The flow context object + returncode: The return code (int) + continue_run: If True, set processing state instead of success/failure + extra_fields: Optional dictionary of extra fields to include in result body + trace_id: Optional trace_id to use as state scope (defaults to ctx.trace_id) + + Returns: + tuple: (success_result, failure_result) - one will be None based on returncode and continue_run + """ + extra_fields = extra_fields or {} + scope = trace_id or ctx.trace_id + + if returncode != 0: + failure_result = { + "status": 500, + "body": { + "success": False, + "failure": True, + "processing": False, + "returncode": returncode, + **extra_fields, + }, + } + await ctx.state.set(scope, "failure", failure_result) + return None, failure_result + elif continue_run: + await ctx.state.set(scope, "processing", {"processing": True, **extra_fields}) + return None, None + else: + success_result = { + "status": 200, + "body": { + "success": True, + "failure": False, + "processing": False, + "returncode": returncode, + **extra_fields, + }, + } + await ctx.state.set(scope, "success", success_result) + return success_result, None diff --git a/bbdev/api/utils/model.py b/bbdev/api/utils/model.py new file mode 100644 index 000000000..d19d8fbbf --- /dev/null +++ b/bbdev/api/utils/model.py @@ -0,0 +1,18 @@ +def model_layout_name(model: str) -> str: + aliases = { + "lenet": "lenet", + "mobilenet": "mobilenet-v3-small", + "resnet": "resnet18", + "yolo": "yolo26n", + "bert": "bert", + "qwen3": "qwen3-8b", + "gemma4": "gemma4-e2b-it", + "deepseekr1": "deepseek-r1-0528-qwen3-8b", + "llama2": "llama2", + "stable-diffusion": "stable-diffusion", + "whisper": "whisper", + } + try: + return aliases[model.lower()] + except KeyError as error: + raise ValueError(f"unknown model: {model}") from error diff --git a/bbdev/api/utils/path.py b/bbdev/api/utils/path.py new file mode 100644 index 000000000..118fc3d6c --- /dev/null +++ b/bbdev/api/utils/path.py @@ -0,0 +1,171 @@ +import json +import os +import re +import sys +from pathlib import Path + + +def get_buckyball_path(): + current_dir = os.path.dirname(__file__) + root = os.path.realpath( + os.path.dirname(os.path.dirname(os.path.dirname(current_dir))) + ) + home = os.path.realpath(str(Path.home())) + if not Path(root).is_relative_to(home): + raise ValueError(f"buckyball root must be under $HOME ({home}): {root}") + return root + + +def _design_name(design): + if not isinstance(design, str) or not design or design == "None": + raise ValueError("missing required parameter: design") + if not re.fullmatch("[A-Za-z0-9_.-]+", design): + raise ValueError(f"invalid design name: {design}") + return design + + +def design_output_root(bbdir, design, *, selection): + design = _design_name(design) + return os.path.join(bbdir, "examples/designs", design, "output/sims", selection) + + +def workloads_output_root(bbdir): + return os.path.join(bbdir, "bb-tests", "output") + + +def workload_build_dir(bbdir, design, *, selection): + design = _design_name(design) + return os.path.join( + design_output_root(bbdir, design, selection=selection), "workloads-build" + ) + + +def workload_tests_root(bbdir, design, *, selection): + return os.path.join( + design_output_root(bbdir, design, selection=selection), "workloads", "src" + ) + + +def bebop_target_dir(bbdir, design, backend, *, selection): + design = _design_name(design) + if backend not in {"bemu", "verilator", "p2e", "vcs"}: + raise ValueError(f"unknown Bebop backend: {backend}") + return os.path.join( + bbdir, + "examples", + "designs", + design, + "output", + "sims", + selection, + "emu", + backend, + "target", + ) + + +def bebop_bin_path(bbdir, design, backend, *, selection): + design = _design_name(design) + if backend not in {"bemu", "verilator", "p2e", "vcs"}: + raise ValueError(f"unknown Bebop backend: {backend}") + return os.path.join( + bbdir, + "examples", + "designs", + design, + "output", + "sims", + selection, + "bin", + f"bebop-{backend}", + ) + + +def design_arch_root(bbdir, design, *, selection): + return os.path.join( + bbdir, + "examples/designs", + _design_name(design), + "output/sims", + selection, + "arch", + ) + + +def design_pb_path(bbdir, design, *, selection): + """The design.pb of one simulation: the design with its config's overrides.""" + return Path(design_output_root(bbdir, design, selection=selection)) / ( + "generated/design.pb" + ) + + +def load_design_config(bbdir, design, *, selection): + sys.path.insert(0, str(Path(bbdir) / "bbdev/api/steps/config/scripts")) + import design_pb2 + + path = design_pb_path(bbdir, design, selection=selection) + return design_pb2.Design.FromString(path.read_bytes()) + + +def simulation_target(bbdir, design, selection): + return load_design_config(bbdir, design, selection=selection).simulation + + +def runner_manifest(bbdir, design, selection): + """The emu crate of the simulation's entry: its model and the runners built from it.""" + return Path(bbdir) / simulation_target(bbdir, design, selection).emu / "Cargo.toml" + + +def design_resource_dir(bbdir, design, name, *, selection): + resources = simulation_target(bbdir, design, selection).resources + if name not in resources: + raise ValueError(f"{design}/{selection} has no registered {name} directory") + return Path(bbdir) / resources[name] + + +def sim_name(bbdir, design, product, *, selection): + target = simulation_target(bbdir, design, selection) + return "P2ETop" if product == "p2e" else target.top + + +def rtl_dir(bbdir, design, product, output_dir=None, *, selection): + # Each product (verilog, tapeout, dc, p2e) elaborates its own RTL. + name = sim_name(bbdir, design, product, selection=selection) + return output_dir or os.path.join( + design_arch_root(bbdir, design, selection=selection), product, name + ) + + +def rtl_out(bbdir, design, product, output_dir=None, *, selection): + name = sim_name(bbdir, design, product, selection=selection) + return name, rtl_dir(bbdir, design, product, output_dir, selection=selection) + + +def _log_part(value, what): + if not isinstance(value, str) or not value: + raise ValueError(f"log dir missing {what}") + if os.path.basename(value) != value: + raise ValueError(f"log {what} must be a single path component: {value!r}") + if not re.fullmatch("[A-Za-z0-9_.-]+", value): + raise ValueError(f"invalid log {what}: {value!r}") + return value + + +def log_dir( + bbdir, design, product, stamp, tool, name, output_dir=None, *, selection=None +): + design = _design_name(design) + stamp = _log_part(stamp, "timestamp") + tool = _log_part(tool, "tool") + name = _log_part(os.path.basename(name), "name") + if selection is not None: + name = _log_part(selection.replace("/", "-"), "target") + "-" + name + return os.path.join(bbdir, "log", f"{stamp}-{design}-{tool}-{name}") + + +def run_log_dir(bbdir, run): + root = (Path(bbdir) / "log").resolve() + path = (root / _log_part(run, "run")).resolve() + if path.parent != root: + raise ValueError(f"run must name a directory under {root}: {run!r}") + return path diff --git a/bbdev/api/utils/port.py b/bbdev/api/utils/port.py new file mode 100644 index 000000000..c4067a612 --- /dev/null +++ b/bbdev/api/utils/port.py @@ -0,0 +1,50 @@ +import socket + + +def find_available_port(start_port: int = 5000, end_port: int = 5500) -> int: + """Find an available port in the specified range""" + for port in range(start_port, end_port + 1): + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", port)) + return port + except OSError: + continue + + raise RuntimeError(f"No available port found in range {start_port}-{end_port}") + + +def _port_order(start_port: int, end_port: int, preferred_port: int | None = None): + if preferred_port is None: + yield from range(start_port, end_port + 1) + return + + if preferred_port < start_port or preferred_port > end_port: + raise ValueError("preferred_port must be within the requested range") + + yield from range(preferred_port, end_port + 1) + yield from range(start_port, preferred_port) + + +def reserve_port( + start_port: int = 5000, + end_port: int = 5500, + preferred_port: int | None = None, +) -> tuple[int, socket.socket]: + """Reserve a port by binding and holding the socket open. + + Do not set SO_REUSEADDR: with it, two processes can both bind the same + port, then both fail when iii tries to listen. Without it, bind is exclusive. + + The caller must close the returned socket once the engine has taken over the port. + """ + for port in _port_order(start_port, end_port, preferred_port): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + sock.bind(("127.0.0.1", port)) + return port, sock + except OSError: + sock.close() + continue + + raise RuntimeError(f"No available port found in range {start_port}-{end_port}") diff --git a/bbdev/api/utils/process.py b/bbdev/api/utils/process.py new file mode 100644 index 000000000..73794ee4d --- /dev/null +++ b/bbdev/api/utils/process.py @@ -0,0 +1,227 @@ +import os +import signal +import shutil +import subprocess +import time + + +def _stat(pid: int): + try: + with open(f"/proc/{pid}/stat") as f: + return f.read().split() + except (FileNotFoundError, PermissionError): + return None + + +def _exists(pid: int) -> bool: + stat = _stat(pid) + return stat is not None and stat[2] != "Z" + + +def _cmdline(pid: int) -> list[str]: + try: + with open(f"/proc/{pid}/cmdline", "rb") as f: + raw = f.read() + except (FileNotFoundError, PermissionError): + return [] + return [part.decode(errors="replace") for part in raw.split(b"\0") if part] + + +def _cwd(pid: int) -> str | None: + try: + return os.readlink(f"/proc/{pid}/cwd") + except (FileNotFoundError, PermissionError, OSError): + return None + + +def _env(pid: int) -> dict[str, str]: + try: + with open(f"/proc/{pid}/environ", "rb") as f: + raw = f.read() + except (FileNotFoundError, PermissionError): + return {} + + env = {} + for item in raw.split(b"\0"): + if not item or b"=" not in item: + continue + key, value = item.split(b"=", 1) + env[key.decode(errors="replace")] = value.decode(errors="replace") + return env + + +def _own(pid: int) -> bool: + try: + return os.stat(f"/proc/{pid}").st_uid == os.getuid() + except (FileNotFoundError, PermissionError): + return False + + +def _children(pid: int) -> list[int]: + children = [] + try: + entries = os.listdir("/proc") + except FileNotFoundError: + return children + + for entry in entries: + if not entry.isdigit(): + continue + child = int(entry) + stat = _stat(child) + if not stat: + continue + try: + if int(stat[3]) == pid: + children.append(child) + children.extend(_children(child)) + except (IndexError, ValueError): + continue + return children + + +def _wait_gone(pids: set[int], timeout: float) -> set[int]: + deadline = time.time() + timeout + alive = set(pids) + while alive and time.time() < deadline: + alive = {pid for pid in alive if _exists(pid)} + if alive: + time.sleep(0.1) + return {pid for pid in alive if _exists(pid)} + + +def _signal(pids: set[int], sig: signal.Signals): + for pid in sorted(pids, reverse=True): + if pid == os.getpid(): + continue + try: + os.kill(pid, sig) + except (ProcessLookupError, PermissionError, OSError): + pass + + +def kill_tree(pid: int, term_timeout: float = 5.0): + targets = set(_children(pid)) + targets.add(pid) + _signal(targets, signal.SIGTERM) + alive = _wait_gone(targets, term_timeout) + if alive: + _signal(alive, signal.SIGKILL) + _wait_gone(alive, 1.0) + + +def terminate_group(proc: subprocess.Popen, term_timeout: float = 5.0): + if proc.poll() is not None: + return + kill_tree(proc.pid, term_timeout=term_timeout) + proc.wait(timeout=1) + + +def _is_workspace_server( + pid: int, workflow_dir: str, worker_url: str | None = None +) -> bool: + repo_dir = os.path.dirname(workflow_dir) + root_dir = os.path.dirname(repo_dir) + if not _own(pid): + return False + + cwd = _cwd(pid) + if cwd not in (workflow_dir, repo_dir, root_dir): + return False + + cmd = _cmdline(pid) + if not cmd: + return False + + exe = os.path.basename(cmd[0]) + is_bbdev = len(cmd) >= 2 and os.path.abspath(cmd[1]) == os.path.join( + repo_dir, "bbdev" + ) + if is_bbdev and worker_url is None: + return True + + if cwd != workflow_dir: + return False + if worker_url and _env(pid).get("III_URL") != worker_url: + return False + + text = " ".join(cmd) + return exe == "iii" or ( + "motia" in text and "dev" in cmd and "--dir" in cmd and "steps" in cmd + ) + + +def _workspace_pids(workflow_dir: str, worker_url: str | None = None) -> set[int]: + pids = set() + try: + entries = os.listdir("/proc") + except FileNotFoundError: + return pids + + for entry in entries: + if entry.isdigit() and _is_workspace_server( + int(entry), workflow_dir, worker_url + ): + pids.add(int(entry)) + return pids + + +def _port_pids( + workflow_dir: str, + port: int | None, + worker_url: str | None = None, +) -> set[int]: + if port is None: + return set() + + result = subprocess.run( + ["lsof", "-ti", f":{port}"], + capture_output=True, + text=True, + check=False, + ) + pids = set() + for line in result.stdout.splitlines(): + try: + pid = int(line) + if _is_workspace_server(pid, workflow_dir, worker_url): + pids.add(pid) + except ValueError: + pass + return pids + + +def stop_workspace_servers( + workflow_dir: str, + port: int | None = None, + worker_url: str | None = None, +) -> int: + if port is None: + pids = _workspace_pids(workflow_dir, worker_url=worker_url) + else: + pids = _port_pids(workflow_dir, port, worker_url=worker_url) + if not pids: + return 0 + + targets = set(pids) + for pid in list(pids): + targets.update(_children(pid)) + + _signal(targets, signal.SIGTERM) + alive = _wait_gone(targets, 5.0) + if alive: + _signal(alive, signal.SIGKILL) + _wait_gone(alive, 1.0) + return len(targets) + + +def clean_python_caches(workflow_dir: str): + for rel in ("steps", "utils"): + root = os.path.join(workflow_dir, rel) + if not os.path.isdir(root): + continue + for dirpath, dirnames, _ in os.walk(root): + if "__pycache__" not in dirnames: + continue + shutil.rmtree(os.path.join(dirpath, "__pycache__"), ignore_errors=True) + dirnames.remove("__pycache__") diff --git a/bbdev/api/utils/process_registry.py b/bbdev/api/utils/process_registry.py new file mode 100644 index 000000000..f15d81690 --- /dev/null +++ b/bbdev/api/utils/process_registry.py @@ -0,0 +1,67 @@ +"""In-process registry for cancellable bbdev subprocesses.""" + +import os +import signal +import threading +from contextvars import ContextVar +from typing import Dict + +from utils.process import kill_tree + +_lock = threading.Lock() +_processes: Dict[str, int] = {} +_cancelled: set[str] = set() +_current_task_scope: ContextVar[str | None] = ContextVar( + "bbdev_task_scope", default=None +) + + +def set_current_task_scope(scope: str) -> None: + _current_task_scope.set(scope) + + +def current_task_scope() -> str | None: + return _current_task_scope.get() + + +def register_process(scope: str, pid: int) -> None: + with _lock: + _processes[scope] = pid + _cancelled.discard(scope) + + +def unregister_process(scope: str) -> None: + with _lock: + _processes.pop(scope, None) + + +def cancel_process(scope: str) -> bool: + with _lock: + pid = _processes.get(scope) + _cancelled.add(scope) + if pid is None: + return False + kill_tree(pid) + return True + + +def force_cancel_process(scope: str, pid: int) -> bool: + """Escalate a still-registered task process group after its grace period.""" + with _lock: + if _processes.get(scope) != pid: + return False + try: + os.killpg(pid, signal.SIGKILL) + except ProcessLookupError: + return False + return True + + +def process_pid(scope: str) -> int | None: + with _lock: + return _processes.get(scope) + + +def cancellation_requested(scope: str) -> bool: + with _lock: + return scope in _cancelled diff --git a/bbdev/api/utils/report_schema.py b/bbdev/api/utils/report_schema.py new file mode 100644 index 000000000..2427115d4 --- /dev/null +++ b/bbdev/api/utils/report_schema.py @@ -0,0 +1,226 @@ +import math +import re +from datetime import datetime, timezone +from pathlib import Path + +STATUSES = {"pass", "fail", "timeout", "infra_error"} +CATEGORIES = { + "Conv", + "MatMul", + "Transfer", + "Quantization", + "Activation", + "Pool", + "Other", +} + + +def report_directory(run): + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", run["design"]): + raise ValueError("design must be a safe directory name") + timestamp = datetime.fromisoformat(run["timestamp"].replace("Z", "+00:00")) + if timestamp.utcoffset() is None: + raise ValueError("timestamp must include a timezone") + folder = run["id"] + return Path(run["design"]) / folder + + +def nonempty(value, label): + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{label} must be a non-empty string") + return value + + +def number(value, label, *, positive=False, integer=False): + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(value) + ): + raise ValueError(f"{label} must be a finite number") + if ( + value < 0 + or (positive and value == 0) + or (integer and (value != int(value) or value > 2**53 - 1)) + ): + raise ValueError(f"invalid {label}: {value}") + return value + + +def validate_run(run): + if run["schema_version"] != 3: + raise ValueError("invalid schema_version") + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", run["id"]): + raise ValueError("run id must be a safe file name") + stamp = datetime.fromisoformat(run["timestamp"].replace("Z", "+00:00")) + if stamp.utcoffset() is None: + raise ValueError("timestamp must include a timezone") + if not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", run["repository"]): + raise ValueError("repository must be owner/name") + if run["commit"] is not None and not re.fullmatch(r"[0-9a-f]{40}", run["commit"]): + raise ValueError("commit must be a full 40-character SHA") + for key in ("branch", "design", "backend", "config_hash", "producer"): + nonempty(run[key], key) + if run["clock_hz"] is not None: + number(run["clock_hz"], "clock_hz", positive=True) + if run["area"] is not None: + area = run["area"] + if area["status"] != "pass": + raise ValueError("area measurement requires a successful DC result") + number(area["cell_um2"], "cell_um2", positive=True) + if not isinstance(area["constraints_met"], bool): + raise ValueError("area constraints_met must be a boolean") + for key in ("scope", "conditions", "report_url", "constraints_report_url"): + nonempty(area[key], f"area {key}") + if len(set(run["verification_targets"])) != len(run["verification_targets"]): + raise ValueError("duplicate expected verification target") + for target in run["verification_targets"]: + nonempty(target, "expected verification target") + targets = set() + for result in run["verification"]: + target = nonempty(result["target"], "verification target") + if target in targets: + raise ValueError(f"duplicate verification target: {target}") + targets.add(target) + if result["status"] not in STATUSES | {"not_run"}: + raise ValueError("invalid verification status") + if result["coverage"] is not None: + if not result["coverage"]: + raise ValueError("empty coverage measurements") + for metric, value in result["coverage"].items(): + nonempty(metric, "coverage metric") + if number(value, "coverage percent") > 100: + raise ValueError("coverage must be in [0,100]") + nonempty(result["report_url"], "coverage report_url") + if targets - set(run["verification_targets"]): + raise ValueError("verification results contain undeclared targets") + names = set() + for model in run["workloads"]: + if model["kind"] not in {"model", "core", "ball"}: + raise ValueError("invalid workload kind") + if model["latency_ms"] is not None: + number(model["latency_ms"], "latency_ms", positive=True) + if model["throughput"] is not None: + number(model["throughput"]["value"], "throughput", positive=True) + nonempty(model["throughput"]["unit"], "throughput unit") + name = nonempty(model["name"], "model name") + if name in names: + raise ValueError(f"duplicate model: {name}") + names.add(name) + for key in ("workload_hash", "dataset_hash"): + nonempty(model[key], key) + if model["status"] not in STATUSES: + raise ValueError(f"unknown status: {model['status']}") + if model["accuracy"] is not None: + accuracy = number(model["accuracy"], "accuracy") + if accuracy > 1: + raise ValueError("accuracy must be in [0,1]") + nonempty(model["accuracy_metric"], "accuracy_metric") + counter = model["counter"] + if counter not in ("riscv-cycle", "rtl-cycle", "x86-tsc"): + raise ValueError(f"unknown model counter: {counter}") + count_field = "cycles" if counter != "x86-tsc" else "ticks" + other = "ticks" if counter != "x86-tsc" else "cycles" + if other in model: + raise ValueError("model must not mix cycle and tick fields") + if model["status"] == "pass": + number(model[count_field], count_field, positive=True, integer=True) + if model["trace_url"] is not None and not model["operators"]: + raise ValueError("trace attachment requires operator measurements") + if model["error"] is not None: + raise ValueError("passing model cannot contain an error") + else: + nonempty(model["error"], "model error") + if ( + model[count_field] is not None + or model["operators"] + or model["latency_ms"] is not None + or model["throughput"] is not None + ): + raise ValueError( + "failed models cannot contain performance measurements" + ) + ids = set() + for op in model["operators"]: + nonempty(op["id"], "operator id") + nonempty(op["name"], "operator name") + if op["id"] in ids: + raise ValueError("duplicate operator id") + ids.add(op["id"]) + if op["category"] not in CATEGORIES: + raise ValueError("unknown operator category") + if op["counter"] != counter or other in op: + raise ValueError("operator counter differs from its model") + unit = "cycle" if counter != "x86-tsc" else "tick" + opposite = "tick" if unit == "cycle" else "cycle" + if f"start_{opposite}" in op or f"end_{opposite}" in op: + raise ValueError("operator mixes cycle and tick intervals") + for key in (count_field, f"start_{unit}", f"end_{unit}", "level"): + number(op[key], key, integer=True) + number(op["calls"], "calls", positive=True, integer=True) + if ( + op[f"start_{unit}"] + op[count_field] > op[f"end_{unit}"] + or op[f"end_{unit}"] > model[count_field] + ): + raise ValueError("operator interval exceeds model trace span") + if run["submission_errors"] != submission_errors(run): + raise ValueError("submission status does not match the report measurements") + return run + + +def submission_errors(run): + errors = [] + if not run["workloads"] and not run["verification"] and run["area"] is None: + errors.append("没有测量或验证结果") + if run["commit"] is None and not run.get("provenance", {}).get("executable_sha256"): + errors.append("缺少源码版本或执行产物哈希") + return errors + + +def validate_submission(run): + validate_run(run) + if run["verification"] or run["verification_targets"]: + raise ValueError( + "verification results belong in test reports, not the leaderboard" + ) + if run["backend"] == "dc": + if run["area"] is None or run["workloads"]: + raise ValueError( + "DC submissions require an area measurement without model results" + ) + if run["commit"] is None: + raise ValueError("DC area report requires a source revision") + else: + if run["area"] is not None: + raise ValueError( + "area measurements must be submitted as a separate DC report" + ) + if run["backend"] != "p2e" or not run["workloads"]: + raise ValueError("leaderboard requires FPGA end-to-end model results") + provenance = run["provenance"] + for field in ("bitstream_sha256", "executable_sha256"): + if not re.fullmatch(r"[0-9a-f]{64}", provenance.get(field, "")): + raise ValueError(f"FPGA model report requires {field}") + for result in run["workloads"]: + if ( + result["kind"] != "model" + or result["benchmark"]["scope"] != "model_execution" + ): + raise ValueError( + "core, Ball and operator tests are not leaderboard model runs" + ) + if result["counter"] != "riscv-cycle": + raise ValueError( + "FPGA model performance requires the device cycle counter" + ) + number( + result["benchmark"]["samples"], + "benchmark samples", + positive=True, + integer=True, + ) + nonempty(result["report_url"], "raw end-to-end benchmark log") + missing = submission_errors(run) + if missing: + raise ValueError(f"submission blocked for {run['id']}: {'; '.join(missing)}") + return run diff --git a/bbdev/api/utils/reports.py b/bbdev/api/utils/reports.py new file mode 100644 index 000000000..221584b17 --- /dev/null +++ b/bbdev/api/utils/reports.py @@ -0,0 +1,115 @@ +import hashlib +import json +import shutil +import subprocess +from datetime import datetime, timezone +from pathlib import Path +from utils.path import _design_name, design_pb_path + +from utils.report_schema import ( + report_directory, + submission_errors, + validate_run, +) + + +def source_context(repo, design, selection): + repo = Path(repo) + design = _design_name(design) + config = design_pb_path(repo, design, selection=selection).read_bytes() + + def git(*args): + return subprocess.check_output(["git", *args], cwd=repo, text=True).strip() + + return { + "repository": "DangoSys/buckyball", + "branch": git("branch", "--show-current") or git("rev-parse", "HEAD"), + "commit": git("rev-parse", "HEAD"), + "design": design, + "config_hash": hashlib.sha256(config).hexdigest(), + "provenance": { + "source_dirty": bool( + git( + "status", + "--porcelain", + "--untracked-files=all", + "--ignore-submodules=none", + ) + ), + "workflow_url": None, + "comparable": True, + }, + } + + +def new_report(context, producer, trace_id, backend): + now = datetime.now(timezone.utc) + return { + "schema_version": 3, + "id": f"{context['design']}-{producer.replace('.', '-')}-{trace_id}", + "timestamp": now.isoformat().replace("+00:00", "Z"), + **context, + "producer": producer, + "backend": backend, + "clock_hz": None, + "area": None, + "verification_targets": [], + "verification": [], + "workloads": [], + "artifacts": {}, + } + + +def attachment(report, directory, source, name): + directory = Path(directory) + directory.mkdir(parents=True, exist_ok=True) + source = Path(source).resolve(strict=True) + target = directory / name + if target.exists(): + raise FileExistsError(target) + shutil.copyfile(source, target) + report["artifacts"][name] = hashlib.sha256(target.read_bytes()).hexdigest() + return f"data/{report_directory(report).as_posix()}/{name}" + + +def finish_report(report, directory): + report["submission_errors"] = submission_errors(report) + validate_run(report) + directory = Path(directory) + directory.mkdir(parents=True, exist_ok=True) + path = directory / "report.json" + temporary = directory / ".report.json.tmp" + temporary.write_text(json.dumps(report, indent=2) + "\n") + temporary.replace(path) + return str(path) + + +def verify_artifacts(report, directory): + prefix = f"data/{report_directory(report).as_posix()}/" + references = [] + if report["area"] is not None: + references.extend( + [report["area"]["report_url"], report["area"]["constraints_report_url"]] + ) + references.extend( + row["report_url"] + for row in report["verification"] + if row["report_url"] is not None + ) + references.extend( + model[field] + for model in report["workloads"] + for field in ("trace_url", "report_url") + if model[field] is not None + ) + for url in references: + if not url.startswith(prefix): + raise ValueError("attachment URL does not belong to this report") + name = url[len(prefix) :] + if name not in report["artifacts"]: + raise ValueError(f"unregistered report attachment: {name}") + for name, digest in report["artifacts"].items(): + if Path(name).name != name: + raise ValueError("attachment name must be a file name") + if hashlib.sha256((Path(directory) / name).read_bytes()).hexdigest() != digest: + raise ValueError(f"attachment hash mismatch: {name}") diff --git a/bbdev/api/utils/search_workload.py b/bbdev/api/utils/search_workload.py new file mode 100644 index 000000000..522b17792 --- /dev/null +++ b/bbdev/api/utils/search_workload.py @@ -0,0 +1,79 @@ +import os +from typing import Optional, List + + +def search_workload(search_dir: str, filename: str) -> Optional[str]: + """ + Recursively resolve one workload file name in the directory and its subdirectories. + + Args: + search_dir: Root directory to search + filename: Filename to search for + + Returns: + Absolute path of the file, or None if not found. + + Raises: + ValueError: the file name has more than one match. + """ + if os.path.isfile(filename): + return os.path.abspath(filename) + + matches = search_workload_all(search_dir, filename) + if len(matches) > 1: + raise ValueError( + f"workload file name {filename!r} is ambiguous under {search_dir}: " + + ", ".join(matches) + ) + return matches[0] if matches else None + + +def search_workload_all(search_dir: str, filename: str) -> List[str]: + """ + Recursively search for a specified filename in the directory and its subdirectories, returning all matches + + Args: + search_dir: Root directory to search + filename: Filename to search for + + Returns: + List of absolute paths of all found files + """ + results = [] + + if not os.path.exists(search_dir): + return results + + for root, dirs, files in os.walk(search_dir): + dirs.sort() + if filename in files: + results.append(os.path.abspath(os.path.join(root, filename))) + + return results + + +def search_workload_pattern(search_dir: str, pattern: str) -> List[str]: + """ + Recursively search for files matching a specified pattern in the directory and its subdirectories + + Args: + search_dir: Root directory to search + pattern: Filename pattern (supports wildcards * and ?) + + Returns: + List of absolute paths of all matching files + """ + import fnmatch + + results = [] + + if not os.path.exists(search_dir): + return results + + for root, dirs, files in os.walk(search_dir): + dirs.sort() + for file in files: + if fnmatch.fnmatch(file, pattern): + results.append(os.path.abspath(os.path.join(root, file))) + + return results diff --git a/bbdev/api/utils/stream_run.py b/bbdev/api/utils/stream_run.py new file mode 100644 index 000000000..13a4180df --- /dev/null +++ b/bbdev/api/utils/stream_run.py @@ -0,0 +1,245 @@ +import asyncio +import subprocess +import threading +from typing import Optional, List, Callable + +from utils.process import kill_tree +from utils.process_registry import ( + current_task_scope, + register_process, + unregister_process, +) + + +class StreamResult: + """Result object mimicking subprocess.CompletedProcess""" + + def __init__(self, returncode: int, stdout: str, stderr: str): + self.returncode = returncode + self.stdout = stdout + self.stderr = stderr + + +def stream_run( + cmd: str, + cwd: Optional[str] = None, + shell: bool = True, + executable: Optional[str] = None, + timeout: Optional[float] = None, + on_stdout: Optional[Callable[[str], None]] = None, + on_stderr: Optional[Callable[[str], None]] = None, + stdout_prefix: str = "STDOUT", + stderr_prefix: str = "STDERR", + env: Optional[dict] = None, + task_scope: Optional[str] = None, +) -> StreamResult: + """ + Execute command and stream output in real-time + + Args: + cmd: Command to execute + cwd: Working directory + shell: Whether to execute using shell + timeout: Timeout in seconds + on_stdout: Callback function for stdout lines + on_stderr: Callback function for stderr lines + stdout_prefix: Prefix for stdout output + stderr_prefix: Prefix for stderr output + + Returns: + StreamResult: Result object containing returncode, stdout, stderr + + Example: + def log_stdout(line): + logger.info(f'[STDOUT] {line}') + + def log_stderr(line): + logger.info(f'[STDERR] {line}') + + result = stream_run( + "make build", + cwd="/path/to/project", + on_stdout=log_stdout, + on_stderr=log_stderr + ) + """ + + def read_stream(stream, output_list: List[str], callback: Optional[Callable]): + """Thread function to read stream output as soon as a line/progress record is available.""" + pending = [] + + def emit(): + if not pending: + return + line = "".join(pending).rstrip() + pending.clear() + if not line: + return + output_list.append(line) + if callback: + callback(line) + + try: + while True: + char = stream.read(1) + if char == "": + emit() + break + if char in ("\n", "\r"): + emit() + else: + pending.append(char) + finally: + stream.close() + + # Start process + # Use errors='replace' to handle non-UTF-8 bytes from simulation UART output + task_scope = task_scope or current_task_scope() + process = subprocess.Popen( + cmd, + cwd=cwd, + shell=shell, + # MCP workers do not provide interactive input. Closing the child's + # stdin makes batch workloads observe EOF instead of waiting forever + # at an application prompt. + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding="utf-8", + errors="replace", # Replace invalid UTF-8 bytes with � instead of crashing + bufsize=1, + executable=executable, + env=env, + # A task owns a process group, so a generic cancel request also + # terminates shells and descendants such as nix/cargo/ninja. + start_new_session=True, + ) + if task_scope: + register_process(task_scope, process.pid) + + stdout_lines = [] + stderr_lines = [] + + # Create threads to read stdout and stderr + stdout_thread = threading.Thread( + target=read_stream, + args=(process.stdout, stdout_lines, on_stdout), + ) + stderr_thread = threading.Thread( + target=read_stream, + args=(process.stderr, stderr_lines, on_stderr), + ) + + stdout_thread.start() + stderr_thread.start() + + try: + # Wait for process to finish (with timeout) + process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + kill_tree(process.pid) + process.wait() + + # Wait for threads to finish + stdout_thread.join() + stderr_thread.join() + + if task_scope: + unregister_process(task_scope) + return StreamResult( + returncode=process.returncode, + stdout="\n".join(stdout_lines), + stderr="\n".join(stderr_lines), + ) + + +def stream_run_logger( + cmd: str, + logger, + cwd: Optional[str] = None, + shell: bool = True, + executable: Optional[str] = None, + timeout: Optional[float] = None, + stdout_prefix: str = "STDOUT", + stderr_prefix: str = "STDERR", + verbose: bool = False, + env: Optional[dict] = None, + task_scope: Optional[str] = None, +) -> StreamResult: + """ + Convenience function for streaming output using logger + + Args: + cmd: Command to execute + logger: Logger instance + cwd: Working directory + shell: Whether to execute using shell + timeout: Timeout in seconds + stdout_prefix: Prefix for stdout output + stderr_prefix: Prefix for stderr output + verbose: Whether to use verbose output mode (verbose mode uses logger with timestamp, non-verbose prints directly) + + Returns: + StreamResult: Result object containing returncode, stdout, stderr + """ + + def log_stdout(line): + if verbose: + # Verbose mode: use logger.info, includes timestamp and task ID + logger.info(f"[{stdout_prefix}] {line}") + else: + # Non-verbose mode: print directly, use green for STDOUT + print(f"\033[32m[{stdout_prefix}]\033[0m {line}", flush=True) + + def log_stderr(line): + if verbose: + # Verbose mode: use logger.info, includes timestamp and task ID + logger.info(f"[{stderr_prefix}] {line}") + else: + # Non-verbose mode: print directly, use red for STDERR + print(f"\033[31m[{stderr_prefix}]\033[0m {line}", flush=True) + + return stream_run( + cmd=cmd, + cwd=cwd, + shell=shell, + executable=executable, + timeout=timeout, + on_stdout=log_stdout, + on_stderr=log_stderr, + stdout_prefix=stdout_prefix, + stderr_prefix=stderr_prefix, + env=env, + task_scope=task_scope, + ) + + +async def stream_run_logger_async( + cmd: str, + logger, + cwd: Optional[str] = None, + shell: bool = True, + executable: Optional[str] = None, + timeout: Optional[float] = None, + stdout_prefix: str = "STDOUT", + stderr_prefix: str = "STDERR", + verbose: bool = False, + env: Optional[dict] = None, + task_scope: Optional[str] = None, +) -> StreamResult: + """Run stream_run_logger off the Motia event loop so long jobs do not stall the worker.""" + return await asyncio.to_thread( + stream_run_logger, + cmd, + logger, + cwd, + shell, + executable, + timeout, + stdout_prefix, + stderr_prefix, + verbose, + env, + task_scope, + ) diff --git a/bbdev/api/utils/workers.py b/bbdev/api/utils/workers.py new file mode 100644 index 000000000..0386687ca --- /dev/null +++ b/bbdev/api/utils/workers.py @@ -0,0 +1,170 @@ +import json +import os +import subprocess +from pathlib import Path + + +def server_ports_path(workflow_dir: str) -> str: + return os.path.join(workflow_dir, "data", "bbdev-server.json") + + +def write_server_ports( + workflow_dir: str, http_port: int, worker_port: int, bb_root: str +) -> None: + path = server_ports_path(workflow_dir) + os.makedirs(os.path.dirname(path), exist_ok=True) + tmp = f"{path}.tmp" + with open(tmp, "w", encoding="utf-8") as f: + json.dump( + { + "http_port": http_port, + "worker_port": worker_port, + "bb_root": os.path.realpath(bb_root), + }, + f, + ) + f.write("\n") + os.replace(tmp, path) + + +def read_server_ports(workflow_dir: str) -> dict: + path = server_ports_path(workflow_dir) + with open(path, encoding="utf-8") as f: + data = json.load(f) + if not isinstance(data, dict): + raise RuntimeError(f"invalid server ports file: {path}") + for key in ("http_port", "worker_port", "bb_root"): + if key not in data: + raise RuntimeError(f"server ports file missing {key}: {path}") + return data + + +def list_engine_workers(worker_port: int, timeout_s: float = 10.0) -> list: + result = subprocess.run( + [ + "iii", + "trigger", + "--function-id", + "engine::workers::list", + "--address", + "127.0.0.1", + "--port", + str(worker_port), + "--timeout-ms", + str(int(timeout_s * 1000)), + ], + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise RuntimeError( + f"engine::workers::list failed (rc={result.returncode}): " + f"{result.stderr.strip() or result.stdout.strip()}" + ) + try: + payload = json.loads(result.stdout) + except json.JSONDecodeError as e: + raise RuntimeError( + f"engine::workers::list returned non-JSON: {result.stdout[:500]}" + ) from e + workers = payload.get("workers") + if not isinstance(workers, list): + raise RuntimeError(f"engine::workers::list missing workers list: {payload}") + return workers + + +def _worker_pid(worker: dict) -> int | None: + name = worker.get("name") + if not isinstance(name, str) or ":" not in name: + return None + tail = name.rsplit(":", 1)[-1] + if not tail.isdigit(): + return None + return int(tail) + + +def _cwd(pid: int) -> str | None: + try: + return os.readlink(f"/proc/{pid}/cwd") + except (FileNotFoundError, PermissionError, OSError): + return None + + +def motia_worker_count() -> int: + raw = os.environ.get("BBDEV_MOTIA_WORKERS", "1") + try: + n = int(raw) + except ValueError as e: + raise RuntimeError(f"BBDEV_MOTIA_WORKERS must be an int, got {raw!r}") from e + if n < 1: + raise RuntimeError(f"BBDEV_MOTIA_WORKERS must be >= 1, got {n}") + return n + + +def classify_workspace_workers(worker_port: int, bb_root: str) -> tuple[list, list]: + """Return (local, foreign) Motia workers attached to this iii.""" + root = os.path.realpath(bb_root) + workers = list_engine_workers(worker_port) + foreign = [] + local = [] + for worker in workers: + funcs = worker.get("function_count") or 0 + runtime = worker.get("runtime") + if funcs <= 0 and runtime not in ("python", "motia"): + continue + pid = _worker_pid(worker) + if pid is None: + foreign.append(worker) + continue + cwd = _cwd(pid) + if cwd is None: + foreign.append(worker) + continue + cwd_real = os.path.realpath(cwd) + if Path(cwd_real).is_relative_to(root): + local.append(worker) + else: + foreign.append(worker) + return local, foreign + + +def assert_sole_workspace_workers( + worker_port: int, bb_root: str, min_local: int | None = None +) -> int: + """Refuse to run if any Motia worker outside this repo is connected to our iii. + + Shared machines often leave orphan Motia processes that reconnect to whatever + process later binds the same worker port; iii then load-balances jobs into + the foreign tree (e.g. /home/other/buckyball). + + Returns the number of local Motia workers when isolation checks pass. + """ + root = os.path.realpath(bb_root) + need = motia_worker_count() if min_local is None else min_local + if need < 1: + raise RuntimeError(f"min_local must be >= 1, got {need}") + local, foreign = classify_workspace_workers(worker_port, bb_root) + + if foreign: + detail = [] + for worker in foreign: + pid = _worker_pid(worker) + detail.append( + { + "name": worker.get("name"), + "id": worker.get("id"), + "function_count": worker.get("function_count"), + "pid_cwd": _cwd(pid) if pid is not None else None, + } + ) + raise RuntimeError( + "foreign Motia worker(s) connected to this iii engine; " + f"refusing to run (worker_port={worker_port}, bb_root={root}): {detail}" + ) + if len(local) < need: + raise RuntimeError( + f"need {need} Motia worker(s) from this workspace, found {len(local)} " + f"(worker_port={worker_port}, bb_root={root})" + ) + return len(local) diff --git a/bbdev/api/utils/workload_manifest.py b/bbdev/api/utils/workload_manifest.py new file mode 100644 index 000000000..59ca1efcf --- /dev/null +++ b/bbdev/api/utils/workload_manifest.py @@ -0,0 +1,50 @@ +import os +from pathlib import Path +from .path import get_buckyball_path, design_resource_dir + + +def validate_design(design: str | None) -> str: + if not design: + raise ValueError("missing required parameter: design") + if not isinstance(design, str): + raise ValueError("design must be a string") + if os.path.sep in design or design in {".", ".."}: + raise ValueError(f"invalid design: {design}") + return design + + +def design_regression_dir( + design: str, backend: str, bbdir: str | None = None, *, selection: str +) -> Path: + design = validate_design(design) + root = Path(bbdir or get_buckyball_path()) + return ( + design_resource_dir(root, design, "regression", selection=selection) / backend + ) + + +def resolve_workload_toml( + design: str, + backend: str, + test_type: str, + bbdir: str | None = None, + diff: bool = False, + *, + selection: str, +) -> str: + if test_type == "bare-tests": + suffix = "bare" + elif test_type == "linux-tests": + suffix = "linux" + else: + raise ValueError(f"invalid test type: {test_type}") + regression_dir = design_regression_dir(design, backend, bbdir, selection=selection) + if diff: + diff_toml = regression_dir / f"workloads-{suffix}-diff.toml" + if diff_toml.is_file(): + return str(diff_toml) + name = f"workloads-{suffix}.toml" + toml = regression_dir / name + if not toml.is_file(): + raise ValueError(f"regression workload toml does not exist: {toml}") + return str(toml) diff --git a/bbdev/bbdev b/bbdev/bbdev new file mode 100755 index 000000000..6f5031086 --- /dev/null +++ b/bbdev/bbdev @@ -0,0 +1,980 @@ +#! /usr/bin/env python3 + +import sys +import argparse +import fcntl +import subprocess +import os +import shlex +import json +import time +import tempfile +import threading +import requests +import shutil + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "api")) +from utils.port import reserve_port +from utils.process import clean_python_caches, stop_workspace_servers, terminate_group +from utils.workers import ( + assert_sole_workspace_workers, + motia_worker_count, + write_server_ports, +) + +workflow_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "api") +workspace_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +# Subcommand definitions: {name: {operation: help_text, ...}} +# Operations with string values accept sub-arguments; True means boolean-only (no sub-args). +WORKFLOW_COMMANDS = { + "verilator": { + "clean": ( + 'Clean build directory. Args: "--design (--device | --core' + ' )"' + ), + "verilog": ( + 'Generate verilog files. Args: "--design (--device | --core' + ' )"' + ), + "build": ( + 'Build simulation executable. Args: "--design (--device | --core' + ' ) [--jobs ]"' + ), + "sim": ( + "Run a built simulation on a baremetal workload of the entry. Args:" + ' "--design (--device | --core ) --binary ' + " [--batch] [--no-wave] [--coverage] [--itrace] [--mtrace] [--pmctrace]" + ' [--ctrace] [--banktrace]"' + ), + "run": ( + 'Generate RTL, build and run. Args: "--design (--device | --core' + " ) --binary [--jobs ] [--batch] [--no-wave]" + ' [--coverage] [--itrace] [--mtrace] [--pmctrace] [--ctrace] [--banktrace]"' + ), + }, + "vcs": { + "clean": ( + 'Clean the entry VCS build. Args: "--design (--device | --core' + ' )"' + ), + "verilog": ( + 'Generate the entry tapeout RTL for VCS. Args: "--design (--device ' + ' | --core )"' + ), + "build": ( + 'Build the entry VCS simv. Args: "--design (--device | --core' + ' ) [--jobs ]"' + ), + "sim": ( + 'Run a built simv on a baremetal workload of the entry. Args: "--design' + " (--device | --core ) --binary [--batch]" + ' [--wave]"' + ), + "run": ( + 'Generate RTL, build and run VCS. Args: "--design (--device |' + ' --core ) --binary [--jobs ] [--batch]"' + ), + }, + "workload": { + "clean": ( + 'Clean workload output directory. Args: "--design (--device |' + ' --core )"' + ), + "build": ( + 'Build test workloads. Args: "--design (--device | --core )' + ' [--ctest|--mlirtest|--soctest]"' + ), + "tohex": ( + 'Convert design workload ELF to hex. Args: "--design (--device |' + ' --core )"' + ), + }, + "model": { + "build": ( + "Build model recipes on the device of a design (all its models when --model" + ' is omitted). Args: "--design --device [--model ]"' + ), + }, + "kernel": { + "build": ( + "Build RISC-V kernel + rootfs carrying the Linux programs of a design." + ' Args: "--design --device [--interactive]"' + ), + }, + "firesim": { + "enumeratefpgas": ( + 'Enumerate FPGAs. Args: "--design (--device | --core )"' + ), + "buildbitstream": ( + 'Build bitstream. Args: "--design (--device | --core )"' + ), + "infrasetup": ( + 'Infrasetup. Args: "--design (--device | --core )"' + ), + "runworkload": ( + 'Run workload. Args: "--design (--device | --core )"' + ), + }, + "stack": { + "build": ( + 'Build a registered simulation stack. Args: "--design (--device |' + ' --core )"' + ), + }, + "bebop-bemu": { + "sim": ( + 'Run Bebop emulator simulation. Args: "--design (--device | --core' + " ) --binary [--disasm] [--tool-profile] [--itrace]" + ' [--mtrace] [--ctrace]"; resolves files in the selected design workload/kernel' + " outputs" + ), + "analysis": ( + 'Analyze itrace/mtrace in an existing log dir. Args: "--design ' + ' (--device | --core ) --run --itrace|--mtrace"' + ), + "batch": ( + 'Run Bebop BEMU batch regression. Args: "--design --device --test' + ' [--jobs ]"' + ), + }, + "bebop-verilator": { + "clean": ( + 'Clean the entry RTL. Args: "--design (--device | --core )"' + ), + "verilog": ( + 'Generate the entry RTL. Args: "--design (--device | --core' + ' ) [--diff]"' + ), + "build": ( + 'Build the entry Verilator runner. Args: "--design (--device |' + ' --core ) [--diff] [--jobs ]"' + ), + "sim": ( + 'Run a built runner on a workload of the entry. Args: "--design ' + " (--device | --core ) --binary [--diff] [--no-wave]" + ' [--itrace] [--mtrace] [--pmctrace] [--ctrace] [--banktrace]"' + ), + "run": ( + 'Generate RTL, build and run. Args: "--design (--device | --core' + " ) --binary [--diff] [--jobs ] [--no-wave]" + ' [--itrace] [--mtrace] [--pmctrace] [--ctrace] [--banktrace]"' + ), + "batch": ( + 'Run the entry regression. Args: "--design --device --test' + ' [--diff] [--jobs ]"' + ), + }, + "bebop-vcs": { + "sim": ( + 'Run one SoC workload through the Bebop VCS node. Args: "--design ' + ' (--device | --core ) --binary [--wave]"' + ), + "verilog": ( + 'Generate Bebop VCS RTL once. Args: "--design (--device | --core' + ' )"' + ), + "build": ( + 'Build the reusable Bebop VCS simv. Args: "--design (--device |' + ' --core ) [--jobs ]"' + ), + "batch": ( + 'Run design-owned VCS regression. Args: "--design --device --test' + ' [--jobs ]"' + ), + }, + "bebop-p2e": { + "clean": ( + 'Clean Bebop P2E build directory. Args: "--design (--device |' + ' --core ) [--output-dir ]"' + ), + "verilog": ( + 'Generate Bebop P2E SystemVerilog for DDR4 backdoor. Args: "--design' + ' (--device | --core ) [--diff] [--output-dir ]"' + ), + "buildbitstream": ( + 'Build Bebop P2E runtime case. Args: "--design (--device | --core' + " ) [--diff] [--itrace] [--mtrace] [--vsrc-dir ] [--output-dir" + ' ]"' + ), + "runworkload": ( + 'Cold-load Bebop P2E workload. Args: "--design (--device | --core' + " ) --image --bitstream [--fpga-location" + " ] [--multi-fpga] [--wave] [--wave-start ] [--diff] [--itrace]" + ' [--mtrace] [--pmctrace] [--ctrace] [--reuse-runtime]"' + ), + "batch": ( + 'Run Bebop P2E batch regression. Args: "--design --device ' + " --bitstream --test [--diff]" + ' [--fpga-location ]"' + ), + }, + "dc": { + "verilog": ( + "Generate top-scoped DC RTL under arch/build//. Args:" + ' "--design (--device | --core ) [--top ]"' + ), + "area": ( + "Generate RTL, run DC synthesis, and report area under" + ' log////area. Args: "--design ' + ' (--device | --core ) [--top ]".' + ), + "power": ( + "Generate RTL, run design-owned power simulation, then run PrimeTime PX" + ' under log////power. Args: "--design ' + " (--device | --core ) [--top ] [--workload ] [--start-ns ] [--end-ns ] [--activity --format" + ' ]".' + ), + }, + "ip": { + "generate": ( + "Generate SRAM macros from elaborator mems.conf under" + ' arch/build//. Args: "--design (--device | --core' + " ) [--consumer ] [--top ] [--output-dir ]" + ' [--next-topic ]"' + ), + "replace": ( + "Assemble top-scoped synthesis RTL plus SRAM macros under" + ' arch/build//. Args: "--design (--device | --core' + " ) [--consumer ] [--top ] [--output-dir ]" + ' [--next-topic ]"' + ), + }, + "yosys": { + "run": ( + 'Run yosys flow. Args: "[--top ] [--design (--device |' + ' --core )] [--output-dir ] [--vcd ]"' + ), + "verilog": ( + 'Generate verilog for yosys flow. Args: "[--design (--device |' + ' --core )] [--output-dir ]"' + ), + "synth": ( + 'Run yosys synthesis. Args: "[--top ] [--output-dir ]' + ' [--vcd ]"' + ), + }, + "uvm": { + "verilog": ( + 'Generate isolated Ball RTL for UVM. Args: "--balltype --output-dir' + ' "' + ), + "build": ( + 'Build a Ball or IP UVM simulation. Args: "[--design --device' + ' ] [--ball | --ip ]"' + ), + "run": ( + 'Build and run a Ball or IP UVM simulation. Args: "[--design ' + ' --device ] [--ball | --ip ]"' + ), + }, + "config": { + "install": ( + "Install design configurations: one design.pb per device config and core" + ' entry. Args: "[--design ]"' + ), + }, +} + + +def make_iii_config( + port: int, + worker_port: int, + quiet: bool = False, + exec_module: bool = True, + state_dir: str | None = None, +) -> str: + """Create a temporary config.yaml with unique HTTP and Worker ports, return the path.""" + base_config = os.path.join(workflow_dir, "config.yaml") + with open(base_config, "r") as f: + content = f.read() + # Replace top-level engine WebSocket port and WorkerModule port (both default 49134), + # then RestApiModule HTTP port (default 3111). + content = content.replace("port: 49134", f"port: {worker_port}") + content = content.replace("port: 3111", f"port: {port}", 1) + content = content.replace("host: 0.0.0.0", "host: 127.0.0.1") + if state_dir is not None: + content = content.replace( + "file_path: ./data/state_store.db", + f"file_path: {state_dir}", + ) + if quiet: + content = content.replace( + "exporter: memory", + "exporter: memory\n level: warn\n logs_console_output: false", + ) + if not exec_module: + marker = ( + "\n # ── Exec Module (Python)" + " ───────────────────────────────────────────────" + ) + if marker not in content: + raise RuntimeError("failed to remove ExecModule from iii config") + content = content.split(marker, 1)[0].rstrip() + "\n" + else: + # Configuration generation must not restart active batch workers. + content = content.replace(" watch:\n - steps/**/*.py\n", "") + motia_exec = "PYTHONUNBUFFERED=1 PYTHONPATH=. .venv/bin/motia dev --dir steps" + single = f" exec:\n - {motia_exec}\n" + if single not in content: + raise RuntimeError("failed to expand Motia worker exec list in iii config") + n = motia_worker_count() + multi = " exec:\n" + "".join(f" - {motia_exec}\n" for _ in range(n)) + content = content.replace(single, multi, 1) + tmp = tempfile.NamedTemporaryFile( + mode="w", suffix=".yaml", prefix="iii-config-", dir=workflow_dir, delete=False + ) + tmp.write(content) + tmp.close() + return tmp.name + + +def user_preferred_port(start: int, end: int) -> int: + size = end - start + 1 + offset = (os.getuid() * 37 + os.getpid()) % size + return start + offset + + +def cleanup_config(config_path: str): + """Remove temp config file, ignoring if already deleted.""" + try: + os.unlink(config_path) + except FileNotFoundError: + pass + + +def load_host_environment() -> None: + """Import the host login-shell environment used by the EDA tools.""" + result = subprocess.run( + ["zsh", "-lic", "env -0"], + check=True, + capture_output=True, + stdin=subprocess.DEVNULL, + start_new_session=True, + ) + for item in result.stdout.split(b"\0"): + if item: + key, value = item.split(b"=", 1) + os.environ[key.decode()] = value.decode() + + +def parse_args(argv: list[str]) -> argparse.Namespace: + try: + argv = _normalize_workflow_args(argv) + except ValueError as e: + print(f"bbdev: error: {e}", file=sys.stderr) + sys.exit(2) + + parser = argparse.ArgumentParser( + prog="bbdev", + description=( + "Development tool for buckyball project.\n" + "--device names a device config registered in the design's sims" + " config\n(configs/sims/default.toml [targets.device.configs]), e.g." + " --device PolyVerilatorConfig." + ), + formatter_class=argparse.RawTextHelpFormatter, + ) + + # Global parameters shared across all subcommands + common = argparse.ArgumentParser(add_help=False) + common.add_argument( + "--port", type=int, default=None, help="Port for dev server (default: auto)" + ) + common.add_argument("--server", action="store_true", help="Server mode") + + subparsers = parser.add_subparsers(dest="command", help="Available commands") + + # start / stop + subparsers.add_parser("start", parents=[common], help="Start dev server") + stop_parser = subparsers.add_parser( + "stop", parents=[common], help="Stop dev server" + ) + stop_parser.add_argument("--all", action="store_true", help="Stop all servers") + + # Build workflow subcommands from WORKFLOW_COMMANDS table + for cmd_name, operations in WORKFLOW_COMMANDS.items(): + cmd_parser = subparsers.add_parser( + cmd_name, parents=[common], help=f"{cmd_name} operations" + ) + group = cmd_parser.add_mutually_exclusive_group(required=True) + for op_name, op_help in operations.items(): + if op_help is True: + group.add_argument( + f"--{op_name}", + dest=op_name, + action="store_true", + help=f"{op_name.capitalize()}.", + ) + else: + group.add_argument( + f"--{op_name}", + dest=op_name, + type=str, + nargs="?", + const="", + metavar="ARGS", + help=op_help, + ) + + return parser.parse_args(argv) + + +def _normalize_workflow_args(argv: list[str]) -> list[str]: + """Attach operation sub-arguments to the operation flag before argparse sees them.""" + if not argv: + return argv + + command = argv[0] + if command not in WORKFLOW_COMMANDS: + return argv + + operations = WORKFLOW_COMMANDS[command] + out = [command] + i = 1 + while i < len(argv): + token = argv[i] + if token.startswith("--") and "=" in token: + op_name, _value = token[2:].split("=", 1) + if op_name in operations and operations[op_name] is not True: + raise ValueError( + f"--{op_name} sub-arguments must be passed as a separate quoted" + " string" + ) + if token.startswith("--") and "=" not in token: + op_name = token[2:] + if op_name in operations and operations[op_name] is not True: + values = [] + j = i + 1 + while j < len(argv): + next_token = argv[j] + next_key = ( + next_token[2:].split("=", 1)[0] + if next_token.startswith("--") + else "" + ) + if next_key in operations: + break + values.append(next_token) + j += 1 + if values: + if len(values) > 1 and values[0].startswith("--"): + raise ValueError( + f"{token} sub-arguments must be quoted as one string" + ) + value = values[0] if len(values) == 1 else shlex.join(values) + out.append(f"{token}={value}") + else: + out.append(token) + i = j + continue + out.append(token) + i += 1 + + return out + + +def extract_command_info(args: argparse.Namespace) -> dict: + """Extract command, operation, and parsed sub-arguments from argparse Namespace.""" + result = {"command": args.command, "operation": None, "args": {}} + + for attr_name, attr_value in vars(args).items(): + if attr_name in ("command", "port", "server"): + continue + if attr_value is None or attr_value is False: + continue + + result["operation"] = attr_name + if attr_value is True or attr_value == "": + result["args"] = {} + else: + try: + result["args"] = _parse_sub_args(attr_value) + except ValueError as e: + print(f"bbdev: error: {e}", file=sys.stderr) + sys.exit(2) + break + + return result + + +def _parse_sub_args(arg_string: str) -> dict: + """Parse a sub-argument string like '--binary /path --batch' into a dict.""" + args_dict = {} + try: + tokens = shlex.split(arg_string) + except ValueError as e: + raise ValueError(f"failed to parse sub-arguments: {e}") from e + + i = 0 + while i < len(tokens): + token = tokens[i] + if token.startswith("--"): + if "=" in token: + key, value = token[2:].split("=", 1) + if not key or value == "": + raise ValueError(f"Invalid argument: {token}") + if value in ("true", "false"): + value = value == "true" + args_dict[key] = value + i += 1 + else: + key = token[2:] + if not key: + raise ValueError(f"Invalid argument: {token}") + if i + 1 < len(tokens) and not tokens[i + 1].startswith("--"): + args_dict[key] = tokens[i + 1] + i += 2 + else: + args_dict[key] = True + i += 1 + elif token.startswith("-") and len(token) == 2: + key = token[1:] + if i + 1 < len(tokens) and not tokens[i + 1].startswith("--"): + args_dict[key] = tokens[i + 1] + i += 2 + else: + args_dict[key] = True + i += 1 + else: + raise ValueError(f"Unexpected argument: {token}") + unknown = args_dict.keys() & {"log-dir", "log_dir", "fst-dir", "fst_dir"} + if unknown: + raise ValueError(f"Unknown parameter(s): {', '.join(sorted(unknown))}") + if "arguments" in args_dict: + values = json.loads(args_dict["arguments"]) + if not isinstance(values, list) or not all( + isinstance(value, str) for value in values + ): + raise ValueError("arguments must be a JSON list of strings") + args_dict["arguments"] = values + return args_dict + + +def wait_for_http(url: str, check_fn, timeout: int, error_msg: str): + """Wait for an actual HTTP response satisfying the readiness predicate.""" + deadline = time.monotonic() + timeout + with requests.Session() as session: + session.trust_env = False + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + break + try: + response = session.get(url, timeout=min(1, remaining)) + if check_fn(response): + return True + except requests.exceptions.RequestException: + pass + remaining = deadline - time.monotonic() + if remaining > 0: + time.sleep(min(1, remaining)) + print(error_msg) + return False + + +# If you find that bbdev has failed without any logs, +# you can disable these log filters. These contain the bbdev runtime logs. +def forward_iii_output(pipe, active_event: threading.Event, quiet: bool = False): + """Forward task output to stdout. In quiet mode, suppress engine infrastructure logs.""" + for line in iter(pipe.readline, ""): + if not active_event.is_set(): + continue + if quiet and not any( + marker in line for marker in ("ERROR", "error:", "Error:") + ): + # Preserve failures even when infrastructure progress is suppressed. + if any( + module in line + for module in ( + "iii::modules::worker", + "iii::modules::config", + "iii::modules::rest_api", + "iii::modules::api", + "iii::modules::queue", + "iii::modules::state", + "iii::modules::pubsub", + "iii::modules::shell", + "iii::modules::telemetry", + "iii::builtins", + "iii::trigger", + "iii::function", + "function_name", + "service_name", + "iii::services", + "iii::workers", + "motia.runtime", + "motia.tracing", + "motia.cli", + ) + ): + continue + sys.stdout.write(line) + sys.stdout.flush() + else: + sys.stdout.write(line) + sys.stdout.flush() + pipe.close() + + +def read_state_file(path: str): + """Read JSON from an iii state file (JSON + binary padding). Returns parsed dict or None.""" + try: + with open(path, "r", errors="ignore") as f: + raw = f.read() + state_data, _ = json.JSONDecoder().raw_decode(raw) + return state_data + except (FileNotFoundError, json.JSONDecodeError, ValueError): + return None + + +def run_server_mode(args, cmd_info): + """Handle --server mode: interact with an already-running (or new) iii server.""" + command = cmd_info["command"] + + if command == "start": + print(f"Starting dev server on port {args.port}") + if not args.port: + print("Error: --server start requires --port", file=sys.stderr) + sys.exit(2) + load_host_environment() + worker_port_start, worker_port_end = 49200, 49500 + preferred_worker = user_preferred_port(worker_port_start, worker_port_end) + worker_port, worker_sock = reserve_port( + start_port=worker_port_start, + end_port=worker_port_end, + preferred_port=preferred_worker, + ) + worker_sock.close() + config_path = make_iii_config(args.port, worker_port) + worker_url = f"ws://127.0.0.1:{worker_port}" + print(f"Using worker port {worker_port} (preferred {preferred_worker})") + proc = None + try: + repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + write_server_ports(workflow_dir, args.port, worker_port, repo_root) + env = { + **os.environ, + "PYTHONDONTWRITEBYTECODE": "1", + "III_URL": worker_url, + } + proc = subprocess.Popen( + ["iii", "-c", config_path], + cwd=workflow_dir, + env=env, + start_new_session=True, + ) + need = motia_worker_count() + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + if proc.poll() is not None: + sys.exit(proc.returncode or 1) + try: + n = assert_sole_workspace_workers( + worker_port, repo_root, min_local=need + ) + break + except RuntimeError as e: + msg = str(e) + if "foreign Motia worker" in msg: + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) + time.sleep(0.5) + continue + else: + print( + f"Error: need {need} Motia worker(s) from this workspace" + " within 60s", + file=sys.stderr, + ) + sys.exit(1) + print(f"Workspace Motia isolation ok: {n} worker(s) on port {worker_port}") + rc = proc.wait() + if rc != 0: + sys.exit(rc) + finally: + if proc: + terminate_group(proc) + stop_workspace_servers(workflow_dir, worker_url=worker_url) + cleanup_config(config_path) + + elif command == "stop": + if cmd_info["operation"] == "all": + count = stop_workspace_servers(workflow_dir) + print(f"Stopped {count} process(es) for this workspace") + else: + count = stop_workspace_servers(workflow_dir, port=args.port) + if args.port: + print(f"Stopped {count} process(es) on port {args.port}") + else: + print(f"Stopped {count} process(es) for this workspace") + + elif command in WORKFLOW_COMMANDS: + api_path = f"/{command.replace('-', '/')}/{cmd_info['operation']}" + resp = requests.post( + f"http://127.0.0.1:{args.port}{api_path}", + json=cmd_info["args"], + timeout=30, + proxies={"http": None, "https": None, "all": None}, + ) + print(resp.text) + + else: + print(f"Unknown command: {command}") + print(f"Available commands: {', '.join(['start', 'stop', *WORKFLOW_COMMANDS])}") + sys.exit(1) + + +def _submit_and_poll(port: int, api_path: str, cmd_info: dict, state_dir: str): + """Submit a task to iii and poll for its result. Returns exit code (0=ok, 1=fail).""" + base_url = f"http://127.0.0.1:{port}" + print(f"Executing {cmd_info['command']} {cmd_info['operation']}...") + try: + submit_resp = requests.post( + f"{base_url}{api_path}", + json=cmd_info["args"], + timeout=30, + proxies={"http": None, "https": None, "all": None}, + ) + submit_data = submit_resp.json() + except Exception as e: + print(f"Error: Failed to submit task: {e}") + return 1 + + if submit_resp.status_code >= 400: + print(f"Error: {submit_data}") + return 1 + + trace_id = submit_data.get("trace_id", "") + if not trace_id: + print(f"Error: No trace_id in response: {submit_data}") + return 1 + + # Poll for result + state_file = os.path.join(state_dir, f"{trace_id}.bin") + poll_timeout = 24 * 60 * 60 # 24 hours max + task_result = None + legacy_processing_count = 0 + last_heartbeat = time.monotonic() + for _ in range(poll_timeout // 2): + state_data = read_state_file(state_file) + if state_data: + if "success" in state_data: + task_result = state_data["success"].get("body", state_data["success"]) + break + elif "failure" in state_data: + task_result = state_data["failure"].get("body", state_data["failure"]) + break + elif state_data.get("processing") is True: + legacy_processing_count += 1 + if legacy_processing_count >= 30: + print( + "Error: Task is stuck in legacy processing state; worker did" + " not advance the queue" + ) + return 1 + else: + legacy_processing_count = 0 + now = time.monotonic() + if now - last_heartbeat >= 600: + print(f"[bbdev] still running trace {trace_id}...", flush=True) + last_heartbeat = now + time.sleep(2) + else: + print("Error: Task polling timed out after 24 hours") + return 1 + + print(f"\nTask completed on http://127.0.0.1:{port}") + if task_result and not task_result.get("success", False): + err = task_result.get("error") or task_result + print(f"Error: Task failed: {err}") + return 1 + return 0 + + +def run_script_mode(args, cmd_info): + """Handle script mode: start iii, run one task, then shut down.""" + command = cmd_info["command"] + + if command == "start": + print(f"'{command}' does nothing in script mode") + return + if command == "stop": + count = stop_workspace_servers(workflow_dir, port=args.port) + if args.port: + print(f"Stopped {count} process(es) on port {args.port}") + else: + print(f"Stopped {count} process(es) for this workspace") + return + + if command not in WORKFLOW_COMMANDS: + print(f"Unknown command: {command}") + print(f"Available commands: {', '.join(['start', 'stop', *WORKFLOW_COMMANDS])}") + sys.exit(1) + + load_host_environment() + + api_path = f"/{command.replace('-', '/')}/{cmd_info['operation']}" + + # Serialize only engine startup. The reservation socket must be released before + # iii can bind it, so without this lock another bbdev can select the same port + # during that handoff. + server_start_lock_path = os.path.join(workflow_dir, "data", "server-start.lock") + os.makedirs(os.path.dirname(server_start_lock_path), exist_ok=True) + server_start_lock = open(server_start_lock_path, "w") + fcntl.flock(server_start_lock, fcntl.LOCK_EX) + + http_port_start, http_port_end = 5100, 5500 + worker_port_start, worker_port_end = 49200, 49500 + + # Reserve both ports atomically to prevent TOCTOU races with concurrent bbdev instances. + # Sockets stay bound until the engine has taken over (HTTP is up), then are released. + if args.port: + available_port = args.port + http_sock = None + else: + available_port, http_sock = reserve_port( + start_port=http_port_start, + end_port=http_port_end, + preferred_port=user_preferred_port(http_port_start, http_port_end), + ) + worker_port, worker_sock = reserve_port( + start_port=worker_port_start, + end_port=worker_port_end, + preferred_port=user_preferred_port(worker_port_start, worker_port_end), + ) + print(f"Starting server on port {available_port}...") + + data_dir = os.path.join(workflow_dir, "data") + os.makedirs(data_dir, exist_ok=True) + state_dir = tempfile.mkdtemp(prefix="state-", dir=data_dir) + clean_python_caches(workflow_dir) + + config_path = make_iii_config( + available_port, + worker_port, + quiet=True, + exec_module=False, + state_dir=state_dir, + ) + iii_output_active = threading.Event() + iii_output_active.set() + + # Release reserved sockets just before engine starts so it can bind the same ports. + if http_sock: + http_sock.close() + worker_sock.close() + + repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + # Set III_URL so motia SDK connects to the correct worker port + iii_env = { + **os.environ, + "III_URL": f"ws://127.0.0.1:{worker_port}", + "PYTHONUNBUFFERED": "1", + "PYTHONIOENCODING": "utf-8", + "PYTHONDONTWRITEBYTECODE": "1", + } + worker_procs = [] + proc = subprocess.Popen( + ["iii", "-c", config_path], + cwd=workflow_dir, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + env=iii_env, + start_new_session=True, + ) + + output_thread = threading.Thread( + target=forward_iii_output, + args=(proc.stdout, iii_output_active, True), + daemon=True, + ) + output_thread.start() + + try: + base_url = f"http://127.0.0.1:{available_port}" + + # Phase 1: wait for iii engine HTTP + if not wait_for_http( + f"{base_url}/", + check_fn=lambda r: True, + timeout=120, + error_msg="Server failed to start within 120 seconds", + ): + sys.exit(1) + + need = motia_worker_count() + for _ in range(need): + worker_proc = subprocess.Popen( + [".venv/bin/motia", "dev", "--dir", "steps"], + cwd=workflow_dir, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + env=iii_env, + start_new_session=True, + ) + worker_procs.append(worker_proc) + worker_output_thread = threading.Thread( + target=forward_iii_output, + args=(worker_proc.stdout, iii_output_active, True), + daemon=True, + ) + worker_output_thread.start() + + # Phase 2: wait for route registration + if not wait_for_http( + f"{base_url}{api_path}", + check_fn=lambda r: r.status_code != 404, + timeout=120, + error_msg="Routes failed to register within timeout", + ): + sys.exit(1) + time.sleep(2) + + try: + n = assert_sole_workspace_workers(worker_port, repo_root, min_local=need) + except RuntimeError as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) + + fcntl.flock(server_start_lock, fcntl.LOCK_UN) + server_start_lock.close() + server_start_lock = None + + print(f"Server is ready on port {available_port} with {n} Motia worker(s)") + rc = _submit_and_poll(available_port, api_path, cmd_info, state_dir) + if rc != 0: + sys.exit(rc) + + finally: + # Keep released ports reserved against new startup until worker cleanup ends. + if server_start_lock is None: + server_start_lock = open(server_start_lock_path, "w") + fcntl.flock(server_start_lock, fcntl.LOCK_EX) + try: + iii_output_active.clear() + time.sleep(0.5) + for worker_proc in worker_procs: + terminate_group(worker_proc) + terminate_group(proc) + stop_workspace_servers(workflow_dir, worker_url=iii_env["III_URL"]) + cleanup_config(config_path) + shutil.rmtree(state_dir) + finally: + fcntl.flock(server_start_lock, fcntl.LOCK_UN) + server_start_lock.close() + + +if __name__ == "__main__": + args = parse_args(sys.argv[1:]) + cmd_info = extract_command_info(args) + + if args.server: + run_server_mode(args, cmd_info) + else: + run_script_mode(args, cmd_info) diff --git a/bbdev/flake.lock b/bbdev/flake.lock new file mode 100644 index 000000000..a57f1a193 --- /dev/null +++ b/bbdev/flake.lock @@ -0,0 +1,61 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1785454630, + "narHash": "sha256-LQy14TZp77TwbQf40gg1V3jo8FwJG0jGDkAH+zRHqg8=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "1559d3daa3ecc813a650b79375ea61b6741b8746", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/bbdev/flake.nix b/bbdev/flake.nix new file mode 100644 index 000000000..4f3ec5e95 --- /dev/null +++ b/bbdev/flake.nix @@ -0,0 +1,46 @@ +{ + description = "bbdev - Agent-friendly developer toolchain backend powered by Motia"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = { self, nixpkgs, flake-utils }: + flake-utils.lib.eachDefaultSystem (system: + let + overlays = [ (import ./nix/overlay.nix) ]; + pkgs = import nixpkgs { inherit system overlays; }; + in + { + devShells.default = pkgs.mkShell { + buildInputs = with pkgs; [ + # Python with system-level dependencies (managed by Nix) + bbdevPythonPkgs + + # System tools + cacert + curl + git + ]; + + shellHook = '' + export PATH="$PWD:$PATH" + + # Create venv for pip-managed packages (motia, iii-sdk) + VENV_DIR="$PWD/api/.venv" + if [ ! -d "$VENV_DIR" ]; then + echo "Creating Python venv..." + python3 -m venv "$VENV_DIR" --system-site-packages + fi + "$VENV_DIR/bin/pip" install -q 'motia[otel]==1.0.0rc17' iii-sdk==0.2.0 || exit $? + export PATH="$VENV_DIR/bin:$PATH" + + echo "bbdev dev environment ready" + echo " Python: $(python3 --version)" + echo " bbdev: $(which bbdev)" + ''; + }; + } + ); +} diff --git a/bbdev/mcp/__init__.py b/bbdev/mcp/__init__.py new file mode 100644 index 000000000..6edf60f97 --- /dev/null +++ b/bbdev/mcp/__init__.py @@ -0,0 +1 @@ +"""buckyball-dev MCP package.""" diff --git a/bbdev/mcp/__main__.py b/bbdev/mcp/__main__.py new file mode 100644 index 000000000..5b5037498 --- /dev/null +++ b/bbdev/mcp/__main__.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +"""Run: python -u bbdev/mcp/__main__.py (stdio MCP).""" + +from __future__ import annotations + +import sys +from pathlib import Path + +if str(Path(__file__).resolve().parent) not in sys.path: + sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from server import main + +if __name__ == "__main__": + main() diff --git a/bbdev/mcp/common.py b/bbdev/mcp/common.py new file mode 100644 index 000000000..782017bfa --- /dev/null +++ b/bbdev/mcp/common.py @@ -0,0 +1,486 @@ +"""Shared helpers for buckyball-dev MCP (bbdev HTTP lifecycle + polling).""" + +from __future__ import annotations + +import atexit +import json +import importlib.util +import os +import shutil +import socket +import subprocess +import sys +import time +import tomllib +import urllib.error +import urllib.request +from pathlib import Path +from typing import Any, Dict, Optional + + +# bbdev/mcp -> repo root +def repo_path() -> Path: + return Path(__file__).resolve().parents[2] + + +def bbdev_path() -> Path: + return repo_path() / "bbdev" / "bbdev" + + +def api_path() -> Path: + return repo_path() / "bbdev" / "api" + + +def motia_path() -> Path: + return api_path() / ".venv" / "bin" / "motia" + + +def log_path() -> Path: + return repo_path() / "bbdev" / "server.log" + + +def state_dir() -> Path: + return api_path() / "data" / "state_store.db" + + +_proc: Optional[subprocess.Popen] = None +_port: Optional[int] = None +_log_fh = None +_submitted_trace_ids: set[str] = set() + + +def _log(msg: str) -> None: + print(f"[buckyball-dev] {msg}", file=sys.stderr, flush=True) + + +def _fmt(payload: Any) -> str: + return json.dumps(payload, ensure_ascii=False, indent=2) + + +def _err(msg: str) -> str: + return _fmt({"success": False, "failure": True, "error": msg}) + + +def _need(name: str, value: Optional[str]) -> Optional[str]: + if value is None or not str(value).strip(): + return f"missing required parameter: {name}" + return None + + +def _opt(params: Dict[str, Any], **kwargs: Any) -> Dict[str, Any]: + for k, v in kwargs.items(): + if v is None: + continue + if isinstance(v, str) and not v.strip(): + continue + params[k] = v + return params + + +def _preferred_http_port(lo: int = 5100, hi: int = 5500) -> int: + size = hi - lo + 1 + return lo + (os.getuid() * 37) % size + + +def _free_port(lo: int = 5100, hi: int = 5500) -> int: + preferred = _preferred_http_port(lo, hi) + order = list(range(preferred, hi + 1)) + list(range(lo, preferred)) + for p in order: + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", p)) + return p + except OSError: + continue + raise RuntimeError(f"No available port in {lo}-{hi}") + + +def _assert_workspace_workers() -> None: + sys.path.insert(0, str(api_path())) + from utils.workers import assert_sole_workspace_workers, read_server_ports + + ports = read_server_ports(str(api_path())) + assert_sole_workspace_workers(int(ports["worker_port"]), str(ports["bb_root"])) + + +def _http( + method: str, url: str, payload: Optional[Dict[str, Any]] = None, timeout: int = 30 +): + data = None + headers = {} + if payload is not None: + data = json.dumps(payload).encode("utf-8") + headers["Content-Type"] = "application/json" + req = urllib.request.Request(url, data=data, headers=headers, method=method) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + body = resp.read().decode("utf-8") + return resp.status, json.loads(body) if body else {} + except urllib.error.HTTPError as e: + raw = e.read().decode("utf-8", errors="replace") + try: + return e.code, json.loads(raw) if raw else {} + except json.JSONDecodeError: + return e.code, {"response_body": raw} + + +def _ready(port: int) -> bool: + try: + code, _ = _http("GET", f"http://127.0.0.1:{port}/stack/build", timeout=2) + return code != 404 + except Exception: + return False + + +def _stop() -> None: + global _proc, _port, _log_fh + if _port is not None and bbdev_path().is_file(): + subprocess.run( + [ + "nix", + "develop", + "--command", + str(bbdev_path()), + "stop", + "--server", + "--port", + str(_port), + ], + cwd=str(repo_path()), + capture_output=True, + text=True, + timeout=30, + check=False, + ) + if _proc is not None: + _proc.terminate() + try: + _proc.wait(timeout=5) + except subprocess.TimeoutExpired: + _proc.kill() + _proc = None + _port = None + if _log_fh is not None: + _log_fh.close() + _log_fh = None + + +def _ensure() -> int: + """Start `bbdev start --server` if needed. Returns HTTP port.""" + global _proc, _port, _log_fh + + if ( + _port is not None + and _proc is not None + and _proc.poll() is None + and _ready(_port) + ): + _assert_workspace_workers() + return _port + if _proc is not None: + _log(f"bbdev on port {_port} died; restarting") + _stop() + + if not bbdev_path().is_file(): + raise RuntimeError(f"missing bbdev CLI: {bbdev_path()}") + if shutil.which("nix") is None: + raise RuntimeError( + "nix not found; MCP must start bbdev through the project development environment" + ) + if not motia_path().is_file(): + raise RuntimeError( + f"missing {motia_path()}; install with: " + f"cd {api_path()} && uv venv .venv --python python3 --seed && " + "uv pip install --python .venv/bin/python -r pyproject.toml" + ) + + port = _free_port() + log_path().parent.mkdir(parents=True, exist_ok=True) + _log_fh = open(log_path(), "a", encoding="utf-8") + _proc = subprocess.Popen( + [ + "nix", + "develop", + "--command", + str(bbdev_path()), + "start", + "--server", + "--port", + str(port), + ], + cwd=str(repo_path()), + stdout=_log_fh, + stderr=_log_fh, + start_new_session=True, + ) + _port = port + _log(f"starting bbdev on port {port} (log: {log_path()})") + + for _ in range(120): + if _proc.poll() is not None: + tail = "" + try: + tail = log_path().read_text(encoding="utf-8", errors="replace")[-2000:] + except OSError: + pass + _stop() + raise RuntimeError( + f"bbdev exited early; see {log_path()}\n--- log tail ---\n{tail}" + ) + if _ready(port): + try: + _assert_workspace_workers() + except RuntimeError as e: + msg = str(e) + if ( + "need" in msg and "Motia worker" in msg + ) or "engine::workers::list failed" in msg: + time.sleep(1) + continue + _stop() + raise + _log(f"bbdev ready on port {port}") + return port + time.sleep(1) + + _stop() + raise RuntimeError( + f"bbdev failed to start on port {port} within 120s; see {log_path()}" + ) + + +def _read_state(trace_id: str) -> Optional[Dict[str, Any]]: + """Same as bbdev CLI: poll iii file state store (HTTP /result path_params are broken).""" + path = state_dir() / f"{trace_id}.bin" + if not path.is_file(): + return None + try: + raw = path.read_text(encoding="utf-8", errors="ignore") + data, _ = json.JSONDecoder().raw_decode(raw) + except (OSError, json.JSONDecodeError, ValueError): + return None + if not isinstance(data, dict): + raise RuntimeError(f"state file root must be object: {path}") + return data + + +def submit(endpoint: str, params: Dict[str, Any]) -> Dict[str, Any]: + """Submit an asynchronous bbdev task without waiting for completion.""" + port = _ensure() + _assert_workspace_workers() + base = f"http://127.0.0.1:{port}" + _log(f"POST {endpoint} params={params}") + + status, submit = _http("POST", f"{base}{endpoint}", params, timeout=30) + if status >= 400: + return { + "success": False, + "failure": True, + "status_code": status, + "error": submit, + "server_log": str(log_path()), + "port": port, + } + + trace_id = submit.get("trace_id") + if not trace_id: + return { + "success": False, + "failure": True, + "error": "no trace_id in submit response", + "response": submit, + "server_log": str(log_path()), + "port": port, + } + + _submitted_trace_ids.add(trace_id) + return { + "accepted": True, + "processing": True, + "trace_id": trace_id, + "port": port, + } + + +def task_status(trace_id: str) -> Dict[str, Any]: + """Read the terminal or in-progress state for a submitted bbdev task.""" + if not trace_id: + raise ValueError("trace_id is required") + + state = _read_state(trace_id) + if state is None: + if trace_id not in _submitted_trace_ids: + raise RuntimeError(f"unknown task trace_id: {trace_id}") + return { + "accepted": True, + "processing": True, + "queued": True, + "trace_id": trace_id, + } + if "cancelled" in state: + _submitted_trace_ids.discard(trace_id) + body = state["cancelled"].get("body", state["cancelled"]) + if not isinstance(body, dict): + raise RuntimeError(f"cancelled body must be object: {body!r}") + body.setdefault("success", False) + body.setdefault("failure", False) + body.setdefault("cancelled", True) + body.setdefault("processing", False) + body.setdefault("trace_id", trace_id) + return body + if "success" in state: + _submitted_trace_ids.discard(trace_id) + out = state["success"].get("body", state["success"]) + if not isinstance(out, dict): + raise RuntimeError(f"success body must be object: {out!r}") + out.setdefault("success", True) + out.setdefault("failure", False) + out.setdefault("processing", False) + out.setdefault("trace_id", trace_id) + return out + if "failure" in state: + _submitted_trace_ids.discard(trace_id) + body = state["failure"].get("body", state["failure"]) + return { + "success": False, + "failure": True, + "processing": False, + "trace_id": trace_id, + "body": body, + "server_log": str(log_path()), + } + if "processing" in state: + body = state["processing"] + if body is True: + out = {k: v for k, v in state.items() if k != "processing"} + elif isinstance(body, dict): + out = dict(body) + else: + raise RuntimeError(f"processing body must be object or true: {body!r}") + out.setdefault("accepted", True) + out["processing"] = True + out.setdefault("trace_id", trace_id) + return out + raise RuntimeError(f"invalid task state for trace_id {trace_id}: {state!r}") + + +def _load_toml(path: Path) -> Dict[str, Any]: + with path.open("rb") as f: + data = tomllib.load(f) + if not isinstance(data, dict): + raise ValueError(f"TOML root must be a table: {path}") + return data + + +def _balldomain_path(design: str, device: str, balldomain: Optional[str]) -> Path: + if balldomain: + raw = Path(balldomain) + if raw.is_absolute() or len(raw.parts) > 1: + return (repo_path() / raw).resolve() + derived = ( + repo_path() + / "examples" + / "designs" + / design + / "output/sims/device" + / device + / "generated/config/derived.json" + ) + cfg = json.loads(derived.read_text(encoding="utf-8")) + domains = {thread["balldomain_base_dir"] for thread in cfg["threads"]} + if len(domains) != 1: + raise ValueError( + f"design {design} has multiple BallDomains; provide an explicit TOML path" + ) + directory = repo_path() / domains.pop() + if balldomain: + return directory / ( + balldomain if balldomain.endswith(".toml") else f"{balldomain}.toml" + ) + thread_file = repo_path() / cfg["threads"][0]["config_path"] + include = _load_toml(thread_file)["balldomain"] + return (thread_file.parent / include).resolve() + + +def _validate_domain(path: Path, cfg: Dict[str, Any]) -> Dict[str, Any]: + from ball_normalize import normalize_ball_domain + + domain = normalize_ball_domain({"balldomain": cfg}) + balls = domain["balls"] + checks = { + "instance_contract": {"pass": True, "count": len(balls)}, + "every_ball_has_instructions": { + "pass": all(ball["instructions"] for ball in balls), + "empty": [ball["id"] for ball in balls if not ball["instructions"]], + }, + } + return { + "passed": all(check["pass"] for check in checks.values()), + "design_balldomain": str(path.relative_to(repo_path())), + "checks": checks, + "balls": balls, + } + + +def _validate_design( + design: str, device: str, balldomain: Optional[str] +) -> Dict[str, Any]: + scripts = repo_path() / "bbdev/api/steps/config/scripts" + sys.path.insert(0, str(scripts)) + from ball_normalize import normalize_ball_domain + + spec = importlib.util.spec_from_file_location( + "ball_validation_topology", scripts / "2_parameter_derivation.py" + ) + topology = importlib.util.module_from_spec(spec) + spec.loader.exec_module(topology) + expanded = ( + repo_path() + / "examples/designs" + / design + / "output/sims/device" + / device + / "generated/config/config.json" + ) + config = json.loads(expanded.read_text(encoding="utf-8")) + selected = ( + _balldomain_path(design, device, balldomain) if balldomain is not None else None + ) + results = [] + for index, thread in enumerate(topology.iter_threads(config["device"])): + raw = thread.get("balldomain") + if not isinstance(raw, dict) or not raw.get("_file"): + raise ValueError(f"Thread {index}: missing expanded BallDomain") + path = Path(raw["_file"]).resolve() + if selected is not None and path != selected: + continue + try: + domain = normalize_ball_domain(thread) + if domain is None: + raise ValueError("missing effective BallDomain") + result = _validate_domain(path, domain) + except ValueError as error: + result = {"passed": False, "error": str(error)} + results.append( + {"thread_index": index, "thread_config": thread["_file"], **result} + ) + if not results: + raise ValueError(f"No expanded thread matches BallDomain {selected}") + return { + "design": design, + "passed": all(result["passed"] for result in results), + "threads": results, + } + + +atexit.register(_stop) + + +# Public aliases for tool modules +log = _log +fmt = _fmt +err = _err +need = _need +opt = _opt +balldomain_path = _balldomain_path +validate_balldomains = _validate_design diff --git a/bbdev/mcp/server.py b/bbdev/mcp/server.py new file mode 100644 index 000000000..96fa4caa7 --- /dev/null +++ b/bbdev/mcp/server.py @@ -0,0 +1,21 @@ +"""FastMCP entry for buckyball-dev.""" + +from __future__ import annotations + +from mcp.server.fastmcp import FastMCP + +from tools import register_all + +mcp = FastMCP( + "buckyball-dev", + instructions=( + "`device` names a device config registered in the design's" + " configs/sims/default.toml [targets.device.configs], e.g." + " PolyVerilatorConfig; `core` names a core simulation entry." + ), +) +register_all(mcp) + + +def main() -> None: + mcp.run(transport="stdio") diff --git a/bbdev/mcp/tests/test_common.py b/bbdev/mcp/tests/test_common.py new file mode 100644 index 000000000..b19b856bd --- /dev/null +++ b/bbdev/mcp/tests/test_common.py @@ -0,0 +1,166 @@ +import sys +import unittest +from pathlib import Path +from unittest.mock import mock_open, patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import common + + +class SubmitTest(unittest.TestCase): + def setUp(self): + common._submitted_trace_ids.clear() + + @patch("common._http") + @patch("common._assert_workspace_workers") + @patch("common._ensure", return_value=5100) + def test_submit_returns_trace_without_reading_state(self, ensure, workers, http): + http.return_value = 202, {"trace_id": "trace-1"} + + result = common.submit("/workload/build", {"design": "pebble"}) + + self.assertEqual( + result, + { + "accepted": True, + "processing": True, + "trace_id": "trace-1", + "port": 5100, + }, + ) + ensure.assert_called_once_with() + workers.assert_called_once_with() + http.assert_called_once_with( + "POST", + "http://127.0.0.1:5100/workload/build", + {"design": "pebble"}, + timeout=30, + ) + + @patch("common._read_state", return_value=None) + def test_task_status_reports_queued_before_state_exists(self, read_state): + common._submitted_trace_ids.add("trace-1") + self.assertEqual( + common.task_status("trace-1"), + { + "accepted": True, + "processing": True, + "queued": True, + "trace_id": "trace-1", + }, + ) + read_state.assert_called_once_with("trace-1") + + @patch("common._read_state", return_value=None) + def test_task_status_rejects_unknown_trace(self, read_state): + with self.assertRaisesRegex(RuntimeError, "unknown task trace_id"): + common.task_status("unknown") + + +class EnsureTest(unittest.TestCase): + def setUp(self): + self.proc = common._proc + self.port = common._port + self.log_fh = common._log_fh + common._proc = None + common._port = None + common._log_fh = None + + def tearDown(self): + common._proc = self.proc + common._port = self.port + common._log_fh = self.log_fh + + def test_ensure_starts_server_in_nix_environment(self): + with ( + patch("common.shutil.which", return_value="/usr/bin/nix"), + patch("common._free_port", return_value=5199), + patch("common._ready", return_value=True), + patch("common._assert_workspace_workers"), + patch("common.open", mock_open()), + patch("common.subprocess.Popen") as popen, + ): + popen.return_value.poll.return_value = None + self.assertEqual(common._ensure(), 5199) + + popen.assert_called_once_with( + [ + "nix", + "develop", + "--command", + str(common.bbdev_path()), + "start", + "--server", + "--port", + "5199", + ], + cwd=str(common.bbdev_path().parent), + stdout=common._log_fh, + stderr=common._log_fh, + start_new_session=True, + ) + + @patch("common._read_state") + def test_task_status_reports_success(self, read_state): + read_state.return_value = { + "success": {"body": {"returncode": 0, "task": "build"}} + } + + self.assertEqual( + common.task_status("trace-1"), + { + "returncode": 0, + "task": "build", + "success": True, + "failure": False, + "processing": False, + "trace_id": "trace-1", + }, + ) + + @patch("common._read_state") + def test_task_status_reports_failure(self, read_state): + read_state.return_value = {"failure": {"body": {"returncode": 1}}} + + self.assertEqual( + common.task_status("trace-1"), + { + "success": False, + "failure": True, + "processing": False, + "trace_id": "trace-1", + "body": {"returncode": 1}, + "server_log": str(common.log_path()), + }, + ) + + @patch("common._read_state") + def test_task_status_reports_nested_processing(self, read_state): + read_state.return_value = { + "processing": { + "processing": True, + "task": "bebop.p2e.buildbitstream", + "design": "pebble", + } + } + + self.assertEqual( + common.task_status("trace-1"), + { + "processing": True, + "task": "bebop.p2e.buildbitstream", + "design": "pebble", + "accepted": True, + "trace_id": "trace-1", + }, + ) + + @patch("common._read_state", return_value={"unknown": True}) + def test_task_status_rejects_invalid_state(self, read_state): + with self.assertRaisesRegex(RuntimeError, "invalid task state"): + common.task_status("trace-1") + + +if __name__ == "__main__": + unittest.main() diff --git a/bbdev/mcp/tools/__init__.py b/bbdev/mcp/tools/__init__.py new file mode 100644 index 000000000..d6ab9f625 --- /dev/null +++ b/bbdev/mcp/tools/__init__.py @@ -0,0 +1,19 @@ +"""Discover MCP capabilities from their owning tool modules.""" + +import importlib +import pkgutil + + +def modules(): + return [ + importlib.import_module(f"{__name__}.{entry.name}") + for entry in sorted( + pkgutil.iter_modules(__path__), key=lambda entry: entry.name + ) + if not entry.name.startswith("_") + ] + + +def register_all(mcp): + for module in modules(): + module.register(mcp) diff --git a/bbdev/mcp/tools/bebop_p2e_batch.py b/bbdev/mcp/tools/bebop_p2e_batch.py new file mode 100644 index 000000000..4684a6d2a --- /dev/null +++ b/bbdev/mcp/tools/bebop_p2e_batch.py @@ -0,0 +1,37 @@ +"""MCP tool: bbdev_bebop_p2e_batch.""" + +from __future__ import annotations + + +from common import submit, err, fmt, need + + +def register(mcp): + @mcp.tool() + def bbdev_bebop_p2e_batch( + design: str, + bitstream: str, + test: str, + diff: bool = False, + device: str | None = None, + ) -> str: + """Batch bebop-p2e regression. test: bare-tests|linux-tests. POST /bebop/p2e/batch.""" + for n, v in (("design", design), ("bitstream", bitstream)): + if e := need(n, v): + return err(e) + if test not in ("bare-tests", "linux-tests"): + return err("test must be bare-tests or linux-tests") + request = { + "design": design, + "bitstream": bitstream, + "test": test, + "diff": diff, + } + if device is not None: + request["device"] = device + return fmt( + submit( + "/bebop/p2e/batch", + request, + ) + ) diff --git a/bbdev/mcp/tools/bebop_p2e_buildbitstream.py b/bbdev/mcp/tools/bebop_p2e_buildbitstream.py new file mode 100644 index 000000000..d02796f23 --- /dev/null +++ b/bbdev/mcp/tools/bebop_p2e_buildbitstream.py @@ -0,0 +1,39 @@ +"""MCP tool: bbdev_bebop_p2e_buildbitstream.""" + +from __future__ import annotations + +from typing import Optional + +from common import submit, err, fmt, need, opt + + +def register(mcp): + @mcp.tool() + def bbdev_bebop_p2e_buildbitstream( + design: str, + diff: bool = False, + itrace: bool = False, + mtrace: bool = False, + vsrc_dir: Optional[str] = None, + output_dir: Optional[str] = None, + device: str | None = None, + core: str | None = None, + ) -> str: + """Build bebop-p2e bitstream. POST /bebop/p2e/buildbitstream.""" + if e := need("design", design): + return err(e) + request = opt( + {"design": design, "diff": diff, "itrace": itrace, "mtrace": mtrace}, + vsrc_dir=vsrc_dir, + output_dir=output_dir, + ) + if device is not None: + request["device"] = device + if core is not None: + request["core"] = core + return fmt( + submit( + "/bebop/p2e/buildbitstream", + request, + ) + ) diff --git a/bbdev/mcp/tools/bebop_p2e_clean.py b/bbdev/mcp/tools/bebop_p2e_clean.py new file mode 100644 index 000000000..657a11bd3 --- /dev/null +++ b/bbdev/mcp/tools/bebop_p2e_clean.py @@ -0,0 +1,22 @@ +"""MCP tool: bbdev_bebop_p2e_clean.""" + +from __future__ import annotations + + +from common import submit, err, fmt, need, opt + + +def register(mcp): + @mcp.tool() + def bbdev_bebop_p2e_clean( + design: str, device: str | None = None, core: str | None = None + ) -> str: + """Clean bebop-p2e build. POST /bebop/p2e/clean.""" + if e := need("design", design): + return err(e) + request = {"design": design} + if device is not None: + request["device"] = device + if core is not None: + request["core"] = core + return fmt(submit("/bebop/p2e/clean", request)) diff --git a/bbdev/mcp/tools/bebop_p2e_runworkload.py b/bbdev/mcp/tools/bebop_p2e_runworkload.py new file mode 100644 index 000000000..195f6d4c7 --- /dev/null +++ b/bbdev/mcp/tools/bebop_p2e_runworkload.py @@ -0,0 +1,50 @@ +"""MCP tool: bbdev_bebop_p2e_runworkload.""" + +from __future__ import annotations + +from typing import Any, Dict + +from common import submit, err, fmt, need + + +def register(mcp): + @mcp.tool() + def bbdev_bebop_p2e_runworkload( + design: str, + image: str = "", + bitstream: str = "", + itrace: bool = False, + mtrace: bool = False, + pmctrace: bool = False, + ctrace: bool = False, + reuse_runtime: bool = False, + diff: bool = False, + device: str | None = None, + core: str | None = None, + ) -> str: + """Cold-load one hex image on P2E.""" + for n, v in (("design", design), ("image", image), ("bitstream", bitstream)): + if e := need(n, v): + return err(e) + params: Dict[str, Any] = { + "design": design, + "image": image, + "bitstream": bitstream, + } + if itrace: + params["itrace"] = True + if mtrace: + params["mtrace"] = True + if reuse_runtime: + params["reuse-runtime"] = True + if pmctrace: + params["pmctrace"] = True + if ctrace: + params["ctrace"] = True + if diff: + params["diff"] = True + if device is not None: + params["device"] = device + if core is not None: + params["core"] = core + return fmt(submit("/bebop/p2e/runworkload", params)) diff --git a/bbdev/mcp/tools/bebop_p2e_verilog.py b/bbdev/mcp/tools/bebop_p2e_verilog.py new file mode 100644 index 000000000..57ef7f892 --- /dev/null +++ b/bbdev/mcp/tools/bebop_p2e_verilog.py @@ -0,0 +1,32 @@ +"""MCP tool: bbdev_bebop_p2e_verilog.""" + +from __future__ import annotations + +from typing import Optional + +from common import submit, err, fmt, need, opt + + +def register(mcp): + @mcp.tool() + def bbdev_bebop_p2e_verilog( + design: str, + output_dir: Optional[str] = None, + diff: bool = False, + device: str | None = None, + core: str | None = None, + ) -> str: + """Generate Verilog for bebop-p2e. POST /bebop/p2e/verilog.""" + if e := need("design", design): + return err(e) + request = opt({"design": design, "diff": diff}, output_dir=output_dir) + if device is not None: + request["device"] = device + if core is not None: + request["core"] = core + return fmt( + submit( + "/bebop/p2e/verilog", + request, + ) + ) diff --git a/bbdev/mcp/tools/bebop_verilator_batch.py b/bbdev/mcp/tools/bebop_verilator_batch.py new file mode 100644 index 000000000..3f8faf4da --- /dev/null +++ b/bbdev/mcp/tools/bebop_verilator_batch.py @@ -0,0 +1,33 @@ +"""MCP tool: bbdev_bebop_verilator_batch.""" + +from __future__ import annotations +from common import submit, err, fmt, need + + +def register(mcp): + + @mcp.tool() + def bbdev_bebop_verilator_batch( + design: str, + test: str, + diff: bool = False, + device: str | None = None, + ) -> str: + """Batch bebop-verilator regression. POST /bebop/verilator/batch.""" + if e := need("design", design): + return err(e) + if test not in ("bare-tests", "linux-tests"): + return err("test must be bare-tests or linux-tests") + request = { + "design": design, + "test": test, + "diff": diff, + } + if device is not None: + request["device"] = device + return fmt( + submit( + "/bebop/verilator/batch", + request, + ) + ) diff --git a/bbdev/mcp/tools/bebop_verilator_build.py b/bbdev/mcp/tools/bebop_verilator_build.py new file mode 100644 index 000000000..209556500 --- /dev/null +++ b/bbdev/mcp/tools/bebop_verilator_build.py @@ -0,0 +1,31 @@ +"""MCP tool: bbdev_bebop_verilator_build.""" + +from __future__ import annotations + + +from common import submit, err, fmt, need, opt + + +def register(mcp): + @mcp.tool() + def bbdev_bebop_verilator_build( + design: str, + jobs: int = 16, + diff: bool = False, + device: str | None = None, + core: str | None = None, + ) -> str: + """Build bebop verilator binary. POST /bebop/verilator/build.""" + if e := need("design", design): + return err(e) + request = {"design": design, "jobs": jobs, "diff": diff} + if device is not None: + request["device"] = device + if core is not None: + request["core"] = core + return fmt( + submit( + "/bebop/verilator/build", + request, + ) + ) diff --git a/bbdev/mcp/tools/bebop_verilator_clean.py b/bbdev/mcp/tools/bebop_verilator_clean.py new file mode 100644 index 000000000..cd215bf34 --- /dev/null +++ b/bbdev/mcp/tools/bebop_verilator_clean.py @@ -0,0 +1,22 @@ +"""MCP tool: bbdev_bebop_verilator_clean.""" + +from __future__ import annotations + + +from common import submit, err, fmt, need, opt + + +def register(mcp): + @mcp.tool() + def bbdev_bebop_verilator_clean( + design: str, device: str | None = None, core: str | None = None + ) -> str: + """Clean bebop-verilator build. POST /bebop/verilator/clean.""" + if e := need("design", design): + return err(e) + request = {"design": design} + if device is not None: + request["device"] = device + if core is not None: + request["core"] = core + return fmt(submit("/bebop/verilator/clean", request)) diff --git a/bbdev/mcp/tools/bebop_verilator_run.py b/bbdev/mcp/tools/bebop_verilator_run.py new file mode 100644 index 000000000..be0f6ca51 --- /dev/null +++ b/bbdev/mcp/tools/bebop_verilator_run.py @@ -0,0 +1,49 @@ +"""MCP tool: bbdev_bebop_verilator_run.""" + +from __future__ import annotations +from common import submit, err, fmt, need + + +def register(mcp): + + @mcp.tool() + def bbdev_bebop_verilator_run( + binary: str, + design: str, + jobs: int = 16, + itrace: bool = False, + mtrace: bool = False, + pmctrace: bool = False, + ctrace: bool = False, + banktrace: bool = False, + no_wave: bool = False, + diff: bool = False, + device: str | None = None, + core: str | None = None, + ) -> str: + """Full bebop-verilator flow. POST /bebop/verilator/run.""" + for n, v in (("binary", binary), ("design", design)): + if e := need(n, v): + return err(e) + request = { + "binary": binary, + "design": design, + "jobs": jobs, + "itrace": itrace, + "mtrace": mtrace, + "pmctrace": pmctrace, + "ctrace": ctrace, + "banktrace": banktrace, + "no-wave": no_wave, + "diff": diff, + } + if device is not None: + request["device"] = device + if core is not None: + request["core"] = core + return fmt( + submit( + "/bebop/verilator/run", + request, + ) + ) diff --git a/bbdev/mcp/tools/bebop_verilator_sim.py b/bbdev/mcp/tools/bebop_verilator_sim.py new file mode 100644 index 000000000..a0307681c --- /dev/null +++ b/bbdev/mcp/tools/bebop_verilator_sim.py @@ -0,0 +1,43 @@ +"""MCP tool: bbdev_bebop_verilator_sim.""" + +from __future__ import annotations +from typing import Any, Dict +from common import submit, err, fmt, need + + +def register(mcp): + + @mcp.tool() + def bbdev_bebop_verilator_sim( + binary: str, + design: str, + itrace: bool = False, + mtrace: bool = False, + pmctrace: bool = False, + ctrace: bool = False, + banktrace: bool = False, + no_wave: bool = False, + diff: bool = False, + device: str | None = None, + core: str | None = None, + ) -> str: + """Run one workload on bebop-verilator. POST /bebop/verilator/sim.""" + for n, v in (("binary", binary), ("design", design)): + if e := need(n, v): + return err(e) + params: Dict[str, Any] = { + "binary": binary, + "design": design, + "itrace": itrace, + "mtrace": mtrace, + "pmctrace": pmctrace, + "ctrace": ctrace, + "banktrace": banktrace, + "no-wave": no_wave, + "diff": diff, + } + if device is not None: + params["device"] = device + if core is not None: + params["core"] = core + return fmt(submit("/bebop/verilator/sim", params)) diff --git a/bbdev/mcp/tools/bebop_verilator_verilog.py b/bbdev/mcp/tools/bebop_verilator_verilog.py new file mode 100644 index 000000000..abe6e52c2 --- /dev/null +++ b/bbdev/mcp/tools/bebop_verilator_verilog.py @@ -0,0 +1,25 @@ +"""MCP tool: bbdev_bebop_verilator_verilog.""" + +from __future__ import annotations + + +from common import submit, err, fmt, need, opt + + +def register(mcp): + @mcp.tool() + def bbdev_bebop_verilator_verilog( + design: str, + diff: bool = False, + device: str | None = None, + core: str | None = None, + ) -> str: + """Generate Verilog for bebop-verilator. POST /bebop/verilator/verilog.""" + if e := need("design", design): + return err(e) + request = {"design": design, "diff": diff} + if device is not None: + request["device"] = device + if core is not None: + request["core"] = core + return fmt(submit("/bebop/verilator/verilog", request)) diff --git a/bbdev/mcp/tools/bemu_analysis.py b/bbdev/mcp/tools/bemu_analysis.py new file mode 100644 index 000000000..2443319be --- /dev/null +++ b/bbdev/mcp/tools/bemu_analysis.py @@ -0,0 +1,40 @@ +"""MCP tool: bbdev_bemu_analysis.""" + +from __future__ import annotations + +from common import submit, err, fmt, need + + +def register(mcp): + @mcp.tool() + def bbdev_bemu_analysis( + design: str, + run: str, + itrace: bool = False, + mtrace: bool = False, + device: str | None = None, + core: str | None = None, + ) -> str: + """Analyze a run directory under the repository log/ directory. POST /bebop/bemu/analysis.""" + if e := need("design", design): + return err(e) + if e := need("run", run): + return err(e) + if not itrace and not mtrace: + return err("need itrace and/or mtrace") + request = { + "design": design, + "run": run, + "itrace": itrace, + "mtrace": mtrace, + } + if device is not None: + request["device"] = device + if core is not None: + request["core"] = core + return fmt( + submit( + "/bebop/bemu/analysis", + request, + ) + ) diff --git a/bbdev/mcp/tools/bemu_batch.py b/bbdev/mcp/tools/bemu_batch.py new file mode 100644 index 000000000..286cf6c36 --- /dev/null +++ b/bbdev/mcp/tools/bemu_batch.py @@ -0,0 +1,33 @@ +"""MCP tool: bbdev_bemu_batch.""" + +from __future__ import annotations +from common import submit, err, fmt, need + + +def register(mcp): + + @mcp.tool() + def bbdev_bemu_batch( + design: str, + test: str, + jobs: int = 1, + device: str | None = None, + ) -> str: + """Batch bemu regression. test: bare-tests|linux-tests. POST /bebop/bemu/batch.""" + if e := need("design", design): + return err(e) + if test not in ("bare-tests", "linux-tests"): + return err("test must be bare-tests or linux-tests") + request = { + "design": design, + "test": test, + "jobs": jobs, + } + if device is not None: + request["device"] = device + return fmt( + submit( + "/bebop/bemu/batch", + request, + ) + ) diff --git a/bbdev/mcp/tools/bemu_sim.py b/bbdev/mcp/tools/bemu_sim.py new file mode 100644 index 000000000..e8802ee4a --- /dev/null +++ b/bbdev/mcp/tools/bemu_sim.py @@ -0,0 +1,49 @@ +"""MCP tool: bbdev_bemu_sim.""" + +from __future__ import annotations +from typing import Any, Dict +from common import submit, err, fmt, need + + +def register(mcp): + + @mcp.tool() + def bbdev_bemu_sim( + design: str, + binary: str | None = None, + dtb: str | None = None, + initrd: str | None = None, + memory_mib: int | None = None, + disasm: bool = False, + tool_profile: bool = False, + itrace: bool = False, + mtrace: bool = False, + ctrace: bool = False, + device: str | None = None, + core: str | None = None, + ) -> str: + """Run a workload or kernel ELF on BEMU. POST /bebop/bemu/sim.""" + for n, v in (("design", design), ("binary", binary)): + if e := need(n, v): + return err(e) + params: Dict[str, Any] = { + "design": design, + "binary": binary, + "disasm": disasm, + "tool-profile": tool_profile, + "itrace": itrace, + "mtrace": mtrace, + "ctrace": ctrace, + } + for key, value in ( + ("dtb", dtb), + ("initrd", initrd), + ("memory-mib", memory_mib), + ): + if value is not None: + params[key] = value + if device is not None: + params["device"] = device + if core is not None: + params["core"] = core + return fmt(submit("/bebop/bemu/sim", params)) diff --git a/bbdev/mcp/tools/dc_area.py b/bbdev/mcp/tools/dc_area.py new file mode 100644 index 000000000..20ed90b5c --- /dev/null +++ b/bbdev/mcp/tools/dc_area.py @@ -0,0 +1,26 @@ +"""MCP tool: bbdev_dc_area.""" + +from __future__ import annotations + +from typing import Optional + +from common import err, fmt, opt, submit + + +def register(mcp): + @mcp.tool() + def bbdev_dc_area( + design: Optional[str] = None, + top: Optional[str] = None, + device: str | None = None, + core: str | None = None, + ) -> str: + """Run DC synthesis and area reporting. Defaults to DigitalTop.""" + if not design: + return err("design is required") + request = opt({}, design=design, top=top) + if device is not None: + request["device"] = device + if core is not None: + request["core"] = core + return fmt(submit("/dc/area", request)) diff --git a/bbdev/mcp/tools/dc_power.py b/bbdev/mcp/tools/dc_power.py new file mode 100644 index 000000000..936a15194 --- /dev/null +++ b/bbdev/mcp/tools/dc_power.py @@ -0,0 +1,53 @@ +"""MCP tool: bbdev_dc_power.""" + +from __future__ import annotations + +from typing import Optional + +from common import err, fmt, opt, submit + + +def register(mcp): + @mcp.tool() + def bbdev_dc_power( + design: Optional[str] = None, + top: Optional[str] = None, + activity: Optional[str] = None, + activity_format: Optional[str] = None, + strip_path: Optional[str] = None, + workload: Optional[str] = None, + start_ns: Optional[str] = None, + end_ns: Optional[str] = None, + device: str | None = None, + core: str | None = None, + ) -> str: + """Run DC synthesis, design-owned activity simulation, and PrimeTime PX. + + Omit activity/activity_format for the normal rerun-and-measure flow. + Supply both only as a debugging override. + """ + if not design: + return err("design is required") + if bool(activity) != bool(activity_format): + return err("activity and activity_format must be provided together") + request = opt( + {}, + design=design, + top=top, + activity=activity, + format=activity_format, + strip_path=strip_path, + workload=workload, + start_ns=start_ns, + end_ns=end_ns, + ) + if device is not None: + request["device"] = device + if core is not None: + request["core"] = core + return fmt( + submit( + "/dc/power", + request, + ) + ) diff --git a/bbdev/mcp/tools/dc_verilog.py b/bbdev/mcp/tools/dc_verilog.py new file mode 100644 index 000000000..7b0f07bd2 --- /dev/null +++ b/bbdev/mcp/tools/dc_verilog.py @@ -0,0 +1,31 @@ +"""MCP tool: bbdev_dc_verilog.""" + +from __future__ import annotations + +from typing import Optional + +from common import submit, err, fmt, opt + + +def register(mcp): + @mcp.tool() + def bbdev_dc_verilog( + design: Optional[str] = None, + top: Optional[str] = None, + device: str | None = None, + core: str | None = None, + ) -> str: + """Generate top-scoped DC RTL collateral. Defaults to DigitalTop.""" + if not design: + return err("design is required") + request = opt({}, design=design, top=top) + if device is not None: + request["device"] = device + if core is not None: + request["core"] = core + return fmt( + submit( + "/dc/verilog", + request, + ) + ) diff --git a/bbdev/mcp/tools/firesim_buildbitstream.py b/bbdev/mcp/tools/firesim_buildbitstream.py new file mode 100644 index 000000000..1d5d83ac1 --- /dev/null +++ b/bbdev/mcp/tools/firesim_buildbitstream.py @@ -0,0 +1,21 @@ +"""MCP tool: bbdev_firesim_buildbitstream.""" + +from __future__ import annotations + +from common import submit, err, fmt, need + + +def register(mcp): + @mcp.tool() + def bbdev_firesim_buildbitstream( + design: str, device: str | None = None, core: str | None = None + ) -> str: + """Build FireSim bitstream. POST /firesim/buildbitstream.""" + if e := need("design", design): + return err(e) + request = {"design": design} + if device is not None: + request["device"] = device + if core is not None: + request["core"] = core + return fmt(submit("/firesim/buildbitstream", request)) diff --git a/bbdev/mcp/tools/firesim_enumeratefpgas.py b/bbdev/mcp/tools/firesim_enumeratefpgas.py new file mode 100644 index 000000000..d809abf4d --- /dev/null +++ b/bbdev/mcp/tools/firesim_enumeratefpgas.py @@ -0,0 +1,21 @@ +"""MCP tool: bbdev_firesim_enumeratefpgas.""" + +from __future__ import annotations + +from common import submit, err, fmt, need + + +def register(mcp): + @mcp.tool() + def bbdev_firesim_enumeratefpgas( + design: str, device: str | None = None, core: str | None = None + ) -> str: + """Enumerate FireSim FPGAs. POST /firesim/enumeratefpgas.""" + if e := need("design", design): + return err(e) + request = {"design": design} + if device is not None: + request["device"] = device + if core is not None: + request["core"] = core + return fmt(submit("/firesim/enumeratefpgas", request)) diff --git a/bbdev/mcp/tools/firesim_infrasetup.py b/bbdev/mcp/tools/firesim_infrasetup.py new file mode 100644 index 000000000..e57356e32 --- /dev/null +++ b/bbdev/mcp/tools/firesim_infrasetup.py @@ -0,0 +1,21 @@ +"""MCP tool: bbdev_firesim_infrasetup.""" + +from __future__ import annotations + +from common import submit, err, fmt, need + + +def register(mcp): + @mcp.tool() + def bbdev_firesim_infrasetup( + design: str, device: str | None = None, core: str | None = None + ) -> str: + """FireSim infrasetup. POST /firesim/infrasetup.""" + if e := need("design", design): + return err(e) + request = {"design": design} + if device is not None: + request["device"] = device + if core is not None: + request["core"] = core + return fmt(submit("/firesim/infrasetup", request)) diff --git a/bbdev/mcp/tools/firesim_runworkload.py b/bbdev/mcp/tools/firesim_runworkload.py new file mode 100644 index 000000000..81a5502c9 --- /dev/null +++ b/bbdev/mcp/tools/firesim_runworkload.py @@ -0,0 +1,21 @@ +"""MCP tool: bbdev_firesim_runworkload.""" + +from __future__ import annotations + +from common import submit, err, fmt, need + + +def register(mcp): + @mcp.tool() + def bbdev_firesim_runworkload( + design: str, device: str | None = None, core: str | None = None + ) -> str: + """Run FireSim workload. POST /firesim/runworkload.""" + if e := need("design", design): + return err(e) + request = {"design": design} + if device is not None: + request["device"] = device + if core is not None: + request["core"] = core + return fmt(submit("/firesim/runworkload", request)) diff --git a/bbdev/mcp/tools/ip_generate.py b/bbdev/mcp/tools/ip_generate.py new file mode 100644 index 000000000..fcadd26ab --- /dev/null +++ b/bbdev/mcp/tools/ip_generate.py @@ -0,0 +1,40 @@ +"""MCP tool: bbdev_ip_generate.""" + +from __future__ import annotations + +from typing import Optional + +from common import err, fmt, need, opt, submit + + +def register(mcp): + @mcp.tool() + def bbdev_ip_generate( + design: str, + consumer: Optional[str] = None, + top: Optional[str] = None, + output_dir: Optional[str] = None, + next_topic: Optional[str] = None, + device: str | None = None, + core: str | None = None, + ) -> str: + """Generate SRAM macros from mems.conf. POST /ip/generate.""" + if e := need("design", design): + return err(e) + request = opt( + {"design": design}, + consumer=consumer, + top=top, + output_dir=output_dir, + next_topic=next_topic, + ) + if device is not None: + request["device"] = device + if core is not None: + request["core"] = core + return fmt( + submit( + "/ip/generate", + request, + ) + ) diff --git a/bbdev/mcp/tools/ip_replace.py b/bbdev/mcp/tools/ip_replace.py new file mode 100644 index 000000000..a6f186e2e --- /dev/null +++ b/bbdev/mcp/tools/ip_replace.py @@ -0,0 +1,40 @@ +"""MCP tool: bbdev_ip_replace.""" + +from __future__ import annotations + +from typing import Optional + +from common import err, fmt, need, opt, submit + + +def register(mcp): + @mcp.tool() + def bbdev_ip_replace( + design: str, + consumer: Optional[str] = None, + top: Optional[str] = None, + output_dir: Optional[str] = None, + next_topic: Optional[str] = None, + device: str | None = None, + core: str | None = None, + ) -> str: + """Assemble synthesis source list plus SRAM macros. POST /ip/replace.""" + if e := need("design", design): + return err(e) + request = opt( + {"design": design}, + consumer=consumer, + top=top, + output_dir=output_dir, + next_topic=next_topic, + ) + if device is not None: + request["device"] = device + if core is not None: + request["core"] = core + return fmt( + submit( + "/ip/replace", + request, + ) + ) diff --git a/bbdev/mcp/tools/kernel_build.py b/bbdev/mcp/tools/kernel_build.py new file mode 100644 index 000000000..453f2c10f --- /dev/null +++ b/bbdev/mcp/tools/kernel_build.py @@ -0,0 +1,37 @@ +"""MCP tool: bbdev_kernel_build.""" + +from __future__ import annotations + +from typing import Any, Dict + +from common import submit, fmt, need, err + + +def register(mcp): + @mcp.tool() + def bbdev_kernel_build( + design: str, + interactive: bool = False, + device: str | None = None, + model: str | None = None, + ) -> str: + """Build RISC-V kernel + rootfs. POST /kernel/build. + + With --design only: packs the programs listed in + examples/designs//sims/device/workloads/kernel/CMakeLists.txt + and runs them under Linux using the shared workload /init. + With --interactive: keep shared /init shell and do not auto-run. + With model: include the model image in the DDR image. + """ + if e := need("design", design): + return err(e) + params: Dict[str, Any] = { + "design": design, + } + if interactive: + params["interactive"] = True + if device is not None: + params["device"] = device + if model is not None: + params["model"] = model + return fmt(submit("/kernel/build", params)) diff --git a/bbdev/mcp/tools/model_build.py b/bbdev/mcp/tools/model_build.py new file mode 100644 index 000000000..f1ae18a8d --- /dev/null +++ b/bbdev/mcp/tools/model_build.py @@ -0,0 +1,16 @@ +"""MCP tool: bbdev_model_build.""" + +from common import submit, err, fmt, need + + +def register(mcp): + @mcp.tool() + def bbdev_model_build(design: str, device: str, model: str | None = None) -> str: + """Build one model recipe on a device config, or all recipes of the design + when model is omitted. POST /model/build.""" + if error := need("design", design) or need("device", device): + return err(error) + params = {"design": design, "device": device} + if model is not None: + params["model"] = model + return fmt(submit("/model/build", params)) diff --git a/bbdev/mcp/tools/stack_build.py b/bbdev/mcp/tools/stack_build.py new file mode 100644 index 000000000..9d7985f13 --- /dev/null +++ b/bbdev/mcp/tools/stack_build.py @@ -0,0 +1,20 @@ +"""MCP tool: bbdev_stack_build.""" + +from __future__ import annotations + + +from common import submit, fmt + + +def register(mcp): + @mcp.tool() + def bbdev_stack_build( + design: str, core: str | None = None, device: str | None = None + ) -> str: + """Build a registered device or core simulation stack. POST /stack/build.""" + body = {"design": design} + if core is not None: + body["core"] = core + if device is not None: + body["device"] = device + return fmt(submit("/stack/build", body)) diff --git a/bbdev/mcp/tools/task_cancel.py b/bbdev/mcp/tools/task_cancel.py new file mode 100644 index 000000000..7b12eca54 --- /dev/null +++ b/bbdev/mcp/tools/task_cancel.py @@ -0,0 +1,17 @@ +"""MCP tool: bbdev_task_cancel.""" + +from __future__ import annotations + +from common import err, fmt, need, submit + + +def register(mcp): + @mcp.tool() + def bbdev_task_cancel(trace_id: str) -> str: + """Cancel a queued or running bbdev task and terminate its process group.""" + if error := need("trace_id", trace_id): + return err(error) + try: + return fmt(submit(f"/task/{trace_id}/cancel", {})) + except Exception as ex: + return fmt({"success": False, "failure": True, "error": str(ex)}) diff --git a/bbdev/mcp/tools/task_status.py b/bbdev/mcp/tools/task_status.py new file mode 100644 index 000000000..902832afa --- /dev/null +++ b/bbdev/mcp/tools/task_status.py @@ -0,0 +1,17 @@ +"""MCP tool: bbdev_task_status.""" + +from __future__ import annotations + +from common import err, fmt, need, task_status + + +def register(mcp): + @mcp.tool() + def bbdev_task_status(trace_id: str) -> str: + """Read a bbdev task result by trace_id without blocking.""" + if error := need("trace_id", trace_id): + return err(error) + try: + return fmt(task_status(trace_id)) + except Exception as ex: + return fmt({"success": False, "failure": True, "error": str(ex)}) diff --git a/bbdev/mcp/tools/uvm_build.py b/bbdev/mcp/tools/uvm_build.py new file mode 100644 index 000000000..d7ebf8787 --- /dev/null +++ b/bbdev/mcp/tools/uvm_build.py @@ -0,0 +1,30 @@ +"""MCP tool: bbdev_uvm_build.""" + +from __future__ import annotations + +from typing import Optional + +from common import submit, err, fmt + + +def register(mcp): + @mcp.tool() + def bbdev_uvm_build( + design: Optional[str] = None, + ball: Optional[str] = None, + ip: Optional[str] = None, + device: str | None = None, + ) -> str: + """Build a Ball or IP UVM simulation. POST /uvm/build.""" + if design is None and not ip: + return err("Standalone UVM requires ip") + params = {} if design is None else {"design": design} + if ball and ip: + return err("Parameters --ball and --ip are mutually exclusive") + if ball: + params["ball"] = ball + if ip: + params["ip"] = ip + if device is not None: + params["device"] = device + return fmt(submit("/uvm/build", params)) diff --git a/bbdev/mcp/tools/uvm_run.py b/bbdev/mcp/tools/uvm_run.py new file mode 100644 index 000000000..de62f62ba --- /dev/null +++ b/bbdev/mcp/tools/uvm_run.py @@ -0,0 +1,30 @@ +"""MCP tool: bbdev_uvm_run.""" + +from __future__ import annotations + +from typing import Optional + +from common import submit, err, fmt + + +def register(mcp): + @mcp.tool() + def bbdev_uvm_run( + design: Optional[str] = None, + ball: Optional[str] = None, + ip: Optional[str] = None, + device: str | None = None, + ) -> str: + """Build and run a Ball or IP UVM simulation. POST /uvm/run.""" + if design is None and not ip: + return err("Standalone UVM requires ip") + params = {} if design is None else {"design": design} + if ball and ip: + return err("Parameters --ball and --ip are mutually exclusive") + if ball: + params["ball"] = ball + if ip: + params["ip"] = ip + if device is not None: + params["device"] = device + return fmt(submit("/uvm/run", params)) diff --git a/bbdev/mcp/tools/validate.py b/bbdev/mcp/tools/validate.py new file mode 100644 index 000000000..746cef31a --- /dev/null +++ b/bbdev/mcp/tools/validate.py @@ -0,0 +1,21 @@ +"""MCP tool: validate.""" + +from __future__ import annotations + +from typing import Optional + +from common import err, fmt, need, validate_balldomains + + +def register(mcp): + @mcp.tool() + def validate(design: str, device: str, balldomain: Optional[str] = None) -> str: + """Validate the compiler Thread's BallDomain registration for a device config of a Design.""" + if e := need("design", design) or need("device", device): + return err(e) + try: + return fmt(validate_balldomains(design, device, balldomain)) + except Exception as ex: + return fmt( + {"passed": False, "success": False, "failure": True, "error": str(ex)} + ) diff --git a/bbdev/mcp/tools/verilator_build.py b/bbdev/mcp/tools/verilator_build.py new file mode 100644 index 000000000..d9f599325 --- /dev/null +++ b/bbdev/mcp/tools/verilator_build.py @@ -0,0 +1,27 @@ +"""MCP tool: bbdev_verilator_build.""" + +from __future__ import annotations + +from common import submit, err, fmt, need + + +def register(mcp): + @mcp.tool() + def bbdev_verilator_build( + design: str, + jobs: int = 16, + coverage: bool = False, + device: str | None = None, + core: str | None = None, + ) -> str: + """Build non-bebop Verilator sim binary. POST /verilator/build.""" + if e := need("design", design): + return err(e) + params = {"design": design, "jobs": jobs} + if coverage: + params["coverage"] = True + if device is not None: + params["device"] = device + if core is not None: + params["core"] = core + return fmt(submit("/verilator/build", params)) diff --git a/bbdev/mcp/tools/verilator_clean.py b/bbdev/mcp/tools/verilator_clean.py new file mode 100644 index 000000000..a74844a5f --- /dev/null +++ b/bbdev/mcp/tools/verilator_clean.py @@ -0,0 +1,21 @@ +"""MCP tool: bbdev_verilator_clean.""" + +from __future__ import annotations + +from common import submit, err, fmt, need + + +def register(mcp): + @mcp.tool() + def bbdev_verilator_clean( + design: str, device: str | None = None, core: str | None = None + ) -> str: + """Clean non-bebop Verilator build. POST /verilator/clean.""" + if e := need("design", design): + return err(e) + request = {"design": design} + if device is not None: + request["device"] = device + if core is not None: + request["core"] = core + return fmt(submit("/verilator/clean", request)) diff --git a/bbdev/mcp/tools/verilator_run.py b/bbdev/mcp/tools/verilator_run.py new file mode 100644 index 000000000..6004a6423 --- /dev/null +++ b/bbdev/mcp/tools/verilator_run.py @@ -0,0 +1,50 @@ +"""MCP tool: bbdev_verilator_run.""" + +from __future__ import annotations + +from typing import Any, Dict, Optional + +from common import submit, err, fmt, need + + +def register(mcp): + @mcp.tool() + def bbdev_verilator_run( + binary: str, + design: str, + batch: bool = False, + coverage: bool = False, + no_wave: bool = False, + jobs: Optional[int] = None, + itrace: bool = False, + mtrace: bool = False, + pmctrace: bool = False, + ctrace: bool = False, + banktrace: bool = False, + device: str | None = None, + core: str | None = None, + ) -> str: + """Full non-bebop Verilator flow. POST /verilator/run.""" + if e := need("binary", binary): + return err(e) + if e := need("design", design): + return err(e) + params: Dict[str, Any] = { + "binary": binary, + "design": design, + "batch": batch, + "coverage": coverage, + "no-wave": no_wave, + "itrace": itrace, + "mtrace": mtrace, + "pmctrace": pmctrace, + "ctrace": ctrace, + "banktrace": banktrace, + } + if jobs is not None: + params["jobs"] = jobs + if device is not None: + params["device"] = device + if core is not None: + params["core"] = core + return fmt(submit("/verilator/run", params)) diff --git a/bbdev/mcp/tools/verilator_sim.py b/bbdev/mcp/tools/verilator_sim.py new file mode 100644 index 000000000..0a74da65c --- /dev/null +++ b/bbdev/mcp/tools/verilator_sim.py @@ -0,0 +1,47 @@ +"""MCP tool: bbdev_verilator_sim.""" + +from __future__ import annotations + +from typing import Any, Dict + +from common import submit, err, fmt, need + + +def register(mcp): + @mcp.tool() + def bbdev_verilator_sim( + binary: str, + design: str, + batch: bool = False, + coverage: bool = False, + no_wave: bool = False, + itrace: bool = False, + mtrace: bool = False, + pmctrace: bool = False, + ctrace: bool = False, + banktrace: bool = False, + device: str | None = None, + core: str | None = None, + ) -> str: + """Run one workload on non-bebop Verilator. POST /verilator/sim.""" + for n, v in (("binary", binary), ("design", design)): + if e := need(n, v): + return err(e) + params: Dict[str, Any] = { + "binary": binary, + "design": design, + "batch": batch, + "no-wave": no_wave, + "itrace": itrace, + "mtrace": mtrace, + "pmctrace": pmctrace, + "ctrace": ctrace, + "banktrace": banktrace, + } + if coverage: + params["coverage"] = True + if device is not None: + params["device"] = device + if core is not None: + params["core"] = core + return fmt(submit("/verilator/sim", params)) diff --git a/bbdev/mcp/tools/verilator_verilog.py b/bbdev/mcp/tools/verilator_verilog.py new file mode 100644 index 000000000..f4a91da8d --- /dev/null +++ b/bbdev/mcp/tools/verilator_verilog.py @@ -0,0 +1,21 @@ +"""MCP tool: bbdev_verilator_verilog.""" + +from __future__ import annotations + +from common import submit, err, fmt, need + + +def register(mcp): + @mcp.tool() + def bbdev_verilator_verilog( + design: str, device: str | None = None, core: str | None = None + ) -> str: + """Generate Verilog for non-bebop Verilator. POST /verilator/verilog.""" + if e := need("design", design): + return err(e) + request = {"design": design} + if device is not None: + request["device"] = device + if core is not None: + request["core"] = core + return fmt(submit("/verilator/verilog", request)) diff --git a/bbdev/mcp/tools/workload_build.py b/bbdev/mcp/tools/workload_build.py new file mode 100644 index 000000000..e5580c86e --- /dev/null +++ b/bbdev/mcp/tools/workload_build.py @@ -0,0 +1,32 @@ +"""MCP tool: bbdev_workload_build.""" + +from __future__ import annotations +from typing import Any, Dict +from common import submit, err, fmt, need + + +def register(mcp): + + @mcp.tool() + def bbdev_workload_build( + design: str, + ctest: bool = False, + mlirtest: bool = False, + device: str | None = None, + core: str | None = None, + ) -> str: + """Build workloads for a design. POST /workload/build.""" + if e := need("design", design): + return err(e) + params: Dict[str, Any] = {"design": design} + if ctest and mlirtest: + return err("ctest and mlirtest cannot be used together") + if ctest: + params["ctest"] = True + if mlirtest: + params["mlirtest"] = True + if device is not None: + params["device"] = device + if core is not None: + params["core"] = core + return fmt(submit("/workload/build", params)) diff --git a/bbdev/mcp/tools/workload_clean.py b/bbdev/mcp/tools/workload_clean.py new file mode 100644 index 000000000..732825afc --- /dev/null +++ b/bbdev/mcp/tools/workload_clean.py @@ -0,0 +1,19 @@ +"""MCP tool: bbdev_workload_clean.""" + +from __future__ import annotations + +from common import submit, fmt + + +def register(mcp): + @mcp.tool() + def bbdev_workload_clean( + design: str, device: str | None = None, core: str | None = None + ) -> str: + """Clean workload artifacts for one design. POST /workload/clean.""" + request = {"design": design} + if device is not None: + request["device"] = device + if core is not None: + request["core"] = core + return fmt(submit("/workload/clean", request)) diff --git a/bbdev/mcp/tools/workload_tohex.py b/bbdev/mcp/tools/workload_tohex.py new file mode 100644 index 000000000..3b663f6e3 --- /dev/null +++ b/bbdev/mcp/tools/workload_tohex.py @@ -0,0 +1,19 @@ +"""MCP tool: bbdev_workload_tohex.""" + +from __future__ import annotations + +from common import submit, fmt + + +def register(mcp): + @mcp.tool() + def bbdev_workload_tohex( + design: str, device: str | None = None, core: str | None = None + ) -> str: + """Convert design workload ELF to hex. POST /workload/tohex.""" + request = {"design": design} + if device is not None: + request["device"] = device + if core is not None: + request["core"] = core + return fmt(submit("/workload/tohex", request)) diff --git a/bbdev/mcp/tools/yosys_run.py b/bbdev/mcp/tools/yosys_run.py new file mode 100644 index 000000000..4aaf303df --- /dev/null +++ b/bbdev/mcp/tools/yosys_run.py @@ -0,0 +1,38 @@ +"""MCP tool: bbdev_yosys_run.""" + +from __future__ import annotations + +from typing import Optional + +from common import submit, err, fmt, need, opt + + +def register(mcp): + @mcp.tool() + def bbdev_yosys_run( + design: str, + top: Optional[str] = None, + output_dir: Optional[str] = None, + vcd: Optional[str] = None, + device: str | None = None, + core: str | None = None, + ) -> str: + """Full yosys flow. POST /yosys/run.""" + if e := need("design", design): + return err(e) + request = opt( + {"design": design}, + top=top, + output_dir=output_dir, + vcd=vcd, + ) + if device is not None: + request["device"] = device + if core is not None: + request["core"] = core + return fmt( + submit( + "/yosys/run", + request, + ) + ) diff --git a/bbdev/mcp/tools/yosys_synth.py b/bbdev/mcp/tools/yosys_synth.py new file mode 100644 index 000000000..634f13c56 --- /dev/null +++ b/bbdev/mcp/tools/yosys_synth.py @@ -0,0 +1,38 @@ +"""MCP tool: bbdev_yosys_synth.""" + +from __future__ import annotations + +from typing import Optional + +from common import submit, err, fmt, need, opt + + +def register(mcp): + @mcp.tool() + def bbdev_yosys_synth( + design: str, + top: Optional[str] = None, + output_dir: Optional[str] = None, + vcd: Optional[str] = None, + device: str | None = None, + core: str | None = None, + ) -> str: + """Yosys synthesis + OpenSTA. POST /yosys/synth.""" + if e := need("design", design): + return err(e) + request = opt( + {"design": design}, + top=top, + output_dir=output_dir, + vcd=vcd, + ) + if device is not None: + request["device"] = device + if core is not None: + request["core"] = core + return fmt( + submit( + "/yosys/synth", + request, + ) + ) diff --git a/bbdev/mcp/tools/yosys_verilog.py b/bbdev/mcp/tools/yosys_verilog.py new file mode 100644 index 000000000..6c2186c5d --- /dev/null +++ b/bbdev/mcp/tools/yosys_verilog.py @@ -0,0 +1,36 @@ +"""MCP tool: bbdev_yosys_verilog.""" + +from __future__ import annotations + +from typing import Optional + +from common import submit, err, fmt, need, opt + + +def register(mcp): + @mcp.tool() + def bbdev_yosys_verilog( + design: str, + top: Optional[str] = None, + output_dir: Optional[str] = None, + device: str | None = None, + core: str | None = None, + ) -> str: + """Generate Verilog for yosys flow. POST /yosys/verilog.""" + if e := need("design", design): + return err(e) + request = opt( + {"design": design}, + top=top, + output_dir=output_dir, + ) + if device is not None: + request["device"] = device + if core is not None: + request["core"] = core + return fmt( + submit( + "/yosys/verilog", + request, + ) + ) diff --git a/bbdev/nix/overlay.nix b/bbdev/nix/overlay.nix new file mode 100644 index 000000000..2619d2f42 --- /dev/null +++ b/bbdev/nix/overlay.nix @@ -0,0 +1,11 @@ +# Motia Python dependencies overlay +# Provides Python packages needed by the project +final: prev: { + bbdevPythonPkgs = final.python312.withPackages (ps: with ps; [ + pydantic + requests + gitpython + humanfriendly + doit + ]); +} diff --git a/bebop b/bebop index 9c245acd1..4b022aec6 160000 --- a/bebop +++ b/bebop @@ -1 +1 @@ -Subproject commit 9c245acd1ef8863ea10e93a496ab81d2dfd7b45b +Subproject commit 4b022aec6daac142f3ea8b076ef52e004994fa11 diff --git a/docs b/docs index 454b8f083..b3bb81823 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 454b8f083f12c5dbeb3905baab14bbe05a5b635b +Subproject commit b3bb818239b383b650f7bb6c2971290016777584 diff --git a/examples/.gitignore b/examples/.gitignore index 6a9039890..d2a87887b 100644 --- a/examples/.gitignore +++ b/examples/.gitignore @@ -3,3 +3,4 @@ build/ test-artifacts/ generated/ +output/ diff --git a/examples/balls/f32add/arch/src/main/scala/F32AddBall.scala b/examples/balls/f32add/arch/src/main/scala/F32AddBall.scala deleted file mode 100644 index fa9020471..000000000 --- a/examples/balls/f32add/arch/src/main/scala/F32AddBall.scala +++ /dev/null @@ -1,124 +0,0 @@ -package examples.balls.f32add - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import framework.balldomain.blink.{BallStatus, BlinkIO, HasBallStatus, HasBlink, SubRobRow} -import framework.balldomain.blink.mmio.{MmioRead, MmioWrite} -import framework.balldomain.rs.BallRsIssue -import framework.top.GlobalConfig - -@instantiable -class F32AddBall(val b: GlobalConfig) extends Module with HasBlink with HasBallStatus { - private val mapping = b.ballDomain.ballIdMappings.find(_.ballName == "F32AddBall") - .getOrElse(throw new IllegalArgumentException("F32AddBall not found in config")) - private val funct = b.ballDomain.ballISA.find(_.mnemonic == "F32ADD").map(_.funct7) - .getOrElse(throw new IllegalArgumentException("F32ADD not found in ballISA")) - require(mapping.inBW == 2 && mapping.outBW == 1) - require(funct == 76) - require(b.memDomain.bankWidth == 128 && b.memDomain.bankMaskLen == 16) - - @public val io = IO(new BlinkIO(b, mapping.inBW, mapping.outBW)) - def blink: BlinkIO = io - def status: BallStatus = io.status - - private val idle :: channels :: reading :: calculate :: writing :: acknowledge :: complete :: Nil = Enum(7) - private val state = RegInit(idle) - private val issue = Reg(new BallRsIssue(b)) - private val row = RegInit(0.U(16.W)) - private val requested = RegInit(VecInit(Seq.fill(2)(false.B))) - private val received = RegInit(VecInit(Seq.fill(2)(false.B))) - private val words = Reg(Vec(2, UInt(128.W))) - private val output = Reg(UInt(128.W)) - private val first = issue.cmd.rs2(5) - private val sourceGroup = issue.cmd.rs2(4, 0) - private val add = Instantiate(new F32AddRow) - add.io.source := words(0) - add.io.accumulator := Mux(first, 0.U(128.W), words(1)) - - io.cmdReq.ready := state === idle - io.cmdResp.valid := state === complete - io.cmdResp.bits.rob_id := issue.rob_id - io.cmdResp.bits.is_sub := issue.is_sub - io.cmdResp.bits.sub_rob_id := issue.sub_rob_id - io.status.idle := state === idle - io.status.running := state =/= idle - io.subRobReq.valid := false.B - io.subRobReq.bits := SubRobRow.tieOff(b) - MmioRead.tieOff(io.mmioRead) - MmioWrite.tieOff(io.mmioWrite) - - for (port <- 0 until 2) { - val read = io.bankRead(port) - val active = if (port == 0) true.B else !first - read.rob_id := issue.rob_id - read.ball_id := 0.U - read.bank_id := (if (port == 0) issue.cmd.op1_bank else issue.cmd.op2_bank) - read.group_id := (if (port == 0) sourceGroup else 0.U) - read.io.req.valid := state === reading && active && !requested(port) - read.io.req.bits.addr := row - read.io.resp.ready := state === reading && active && requested(port) && !received(port) - when(read.io.req.fire)(requested(port) := true.B) - when(read.io.resp.fire) { - words(port) := read.io.resp.bits.data - received(port) := true.B - } - } - - private val write = io.bankWrite(0) - write.rob_id := issue.rob_id - write.ball_id := 0.U - write.bank_id := issue.cmd.wr_bank - write.group_id := 0.U - write.io.req.valid := state === writing - write.io.req.bits.addr := row - write.io.req.bits.data := output - write.io.req.bits.mask := VecInit(Seq.fill(16)(true.B)) - write.io.resp.ready := state === acknowledge - - when(io.cmdReq.fire) { - val cmd = io.cmdReq.bits.cmd - assert(cmd.funct7 === funct.U, "F32ADD invalid funct7") - assert(cmd.op1_en && cmd.op2_en && cmd.wr_spad_en, "F32ADD requires two inputs and one output") - assert(cmd.rs2(63, 6) === 0.U, "F32ADD reserved rs2 bits") - assert( - cmd.op1_bank =/= cmd.op2_bank && cmd.op1_bank =/= cmd.wr_bank && cmd.op2_bank =/= cmd.wr_bank, - "F32ADD banks must differ" - ) - assert( - cmd.rs2(4, 0) < cmd.op1_col && cmd.op2_col === 1.U && cmd.wr_col === 1.U, - "F32ADD source group must exist and accumulators must have one group" - ) - for (bank <- Seq(cmd.op1_bank, cmd.op2_bank, cmd.wr_bank)) { - assert(bank < b.memDomain.virtualBankCount.U, "F32ADD invalid virtual bank") - val shared = bank >= b.frontend.shared_bank_id_base.U - assert(!shared || b.memDomain.sharedEnable.B, "F32ADD shared memory is disabled") - val depth = Mux(shared, b.memDomain.sharedBankEntries.U, b.memDomain.bankEntries.U) - assert(cmd.iter =/= 0.U && cmd.iter <= depth, "F32ADD row span exceeds bank") - } - issue := io.cmdReq.bits - row := 0.U - requested.foreach(_ := false.B) - received.foreach(_ := false.B) - state := channels - } - when(state === channels && io.channelReady)(state := reading) - when(state === reading && received(0) && (first || received(1)))(state := calculate) - when(state === calculate) { - output := add.io.sum - state := writing - } - when(write.io.req.fire)(state := acknowledge) - when(write.io.resp.fire) { - assert(write.io.resp.bits.ok, "F32ADD bank write failed") - when(row +& 1.U === issue.cmd.iter) { - state := complete - }.otherwise { - row := row + 1.U - requested.foreach(_ := false.B) - received.foreach(_ := false.B) - state := reading - } - } - when(io.cmdResp.fire)(state := idle) -} diff --git a/examples/balls/f32add/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp b/examples/balls/f32add/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp deleted file mode 100644 index 101d392d7..000000000 --- a/examples/balls/f32add/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp +++ /dev/null @@ -1,27 +0,0 @@ -#include "Buckyball/BuckyballOps.h" -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { -class BankF32AddPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(BankF32AddOp op, - PatternRewriter &rewriter) const override { - rewriter.create(op.getLoc(), op.getA(), op.getB(), op.getC(), - op.getRowsAttr(), op.getGroupAttr(), - op.getFirstAttr()); - rewriter.replaceOp(op, op.getC()); - return success(); - } -}; -} // namespace - -namespace mlir::buddy { -void populateF32AddBallAssignPhysicalBankPatterns(RewritePatternSet &patterns, - PhysicalBankState &) { - patterns.add(patterns.getContext()); -} -} // namespace mlir::buddy diff --git a/examples/balls/f32add/compiler/src/Dialect/Buckyball/F32AddBall.td b/examples/balls/f32add/compiler/src/Dialect/Buckyball/F32AddBall.td deleted file mode 100644 index bcca4d6b3..000000000 --- a/examples/balls/f32add/compiler/src/Dialect/Buckyball/F32AddBall.td +++ /dev/null @@ -1,30 +0,0 @@ -#ifndef F32ADDBALL_OPS -#define F32ADDBALL_OPS - -include "Buckyball.td" - -def F32AddOp : Buckyball_Op<"f32add", [MemoryEffects<[MemRead, MemWrite]>]> { - let summary = "Ordered RNE accumulator plus a selected FP32 source group"; - let description = [{ - Each 128-bit row contains four FP32 elements. For each element, compute - accumulator + source with RNE and canonical NaN 0x7fc00000. When first is - true, use positive zero without reading b. The three virtual banks must - differ. Only a uses the selected group; b and c each use group zero. - }]; - let arguments = (ins I64:$a, I64:$b, I64:$c, - I64Attr:$rows, I64Attr:$group, BoolAttr:$first); - let hasVerifier = 1; - let assemblyFormat = "$a `,` $b `,` $c prop-dict attr-dict `:` type($a)"; -} - -def BankF32AddOp : Buckyball_Op<"bank_f32add", [MemoryEffects<[MemRead, MemWrite]>]> { - let summary = "FP32 grouped addition with an SSA output bank handle"; - let arguments = (ins I64:$a, I64:$b, I64:$c, - I64Attr:$rows, I64Attr:$group, BoolAttr:$first); - let results = (outs I64:$cOut); - let hasVerifier = 1; - let assemblyFormat = "$a $b $c prop-dict attr-dict `:` " - "type($a) type($b) type($c)"; -} - -#endif diff --git a/examples/balls/f32add/compiler/src/Dialect/Buckyball/IR/F32Add.cpp b/examples/balls/f32add/compiler/src/Dialect/Buckyball/IR/F32Add.cpp deleted file mode 100644 index 30344af2b..000000000 --- a/examples/balls/f32add/compiler/src/Dialect/Buckyball/IR/F32Add.cpp +++ /dev/null @@ -1,30 +0,0 @@ -#include "Buckyball/BuckyballOps.h" -#include "mlir/Dialect/Arith/IR/Arith.h" - -using namespace mlir; -using namespace buddy::buckyball; - -namespace { -template LogicalResult verifyAddition(Op op) { - if (op.getRows() <= 0 || op.getRows() > ((int64_t(1) << 34) - 1)) - return op.emitOpError("rows must be in [1, 17179869183]"); - if (op.getGroup() < 0 || op.getGroup() > 31) - return op.emitOpError("group must be in [0, 31]"); - auto banks = op.getOperands(); - for (unsigned i = 0; i < banks.size(); ++i) { - auto lhs = banks[i].template getDefiningOp(); - if (lhs && (lhs.value() < 0 || lhs.value() > 1023)) - return op.emitOpError("bank ID must be in [0, 1023]"); - for (unsigned j = 0; j < i; ++j) { - auto rhs = banks[j].template getDefiningOp(); - if (banks[i] == banks[j] || (lhs && rhs && lhs.value() == rhs.value())) - return op.emitOpError( - "source, accumulator and output banks must differ"); - } - } - return success(); -} -} // namespace - -LogicalResult F32AddOp::verify() { return verifyAddition(*this); } -LogicalResult BankF32AddOp::verify() { return verifyAddition(*this); } diff --git a/examples/balls/f32add/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/balls/f32add/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 1176c2946..000000000 --- a/examples/balls/f32add/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,46 +0,0 @@ -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" - -using namespace mlir; -using namespace buddy::buckyball; -using namespace buddy::buckyball::legalize; - -namespace { -class F32AddLowering : public ConvertOpToLLVMPattern { -public: - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - LogicalResult - matchAndRewrite(F32AddOp op, OpAdaptor operands, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("F32AddBall"); - const auto &target = buckyball_target::getBuckyballTarget(); - if (target.bankWidthBits != 128 || op.getRows() > target.bankDepth) - return op.emitError( - "F32ADD rows exceed the target 128-bit bank capacity"); - Location loc = op.getLoc(); - Value rs1 = - packRs1BanksIter(rewriter, loc, operands.getA(), operands.getB(), - operands.getC(), cstI64(rewriter, loc, op.getRows())); - Value rs2 = cstI64(rewriter, loc, op.getGroup() | (op.getFirst() ? 32 : 0)); - rewriter.replaceOpWithNewOp( - op, rs1, rs2, - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("F32ADD"))); - return success(); - } -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateF32AddBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool, int64_t) { - patterns.add(converter); -} - -void configureF32AddBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool) { - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/examples/balls/f32add/configs/default.toml b/examples/balls/f32add/configs/default.toml deleted file mode 100644 index c61b39cb6..000000000 --- a/examples/balls/f32add/configs/default.toml +++ /dev/null @@ -1 +0,0 @@ -[ball] diff --git a/examples/balls/f32add/emu/src/dispatch.rs b/examples/balls/f32add/emu/src/dispatch.rs deleted file mode 100644 index 4cc8f7b9e..000000000 --- a/examples/balls/f32add/emu/src/dispatch.rs +++ /dev/null @@ -1,26 +0,0 @@ -use crate::inst::instruction::ExecContext; - -pub fn execute_known( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, - ctx: &mut ExecContext, -) -> Option { - if ball_class != "examples.balls.f32add.F32AddBall" { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("F32ADD") => Some(super::add::execute(xs1, xs2, ctx)), - _ => None, - } -} -pub fn cycles_after_issue(ball_class: &str, funct: u32, _xs1: u64, _xs2: u64) -> Option { - if ball_class != "examples.balls.f32add.F32AddBall" { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("F32ADD") => Some(1), - _ => None, - } -} diff --git a/examples/balls/f32add/emu/src/lib.rs b/examples/balls/f32add/emu/src/lib.rs deleted file mode 100644 index 0b9abb860..000000000 --- a/examples/balls/f32add/emu/src/lib.rs +++ /dev/null @@ -1,4 +0,0 @@ -mod add; -mod dispatch; - -pub use dispatch::{cycles_after_issue, execute_known}; diff --git a/examples/balls/f32add/workloads/ctests/CMakeLists.txt b/examples/balls/f32add/workloads/ctests/CMakeLists.txt deleted file mode 100644 index 7ddbd9531..000000000 --- a/examples/balls/f32add/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_buckyball_ctests(f32add_groups.c) diff --git a/examples/balls/f32add/workloads/isa/f32add.h b/examples/balls/f32add/workloads/isa/f32add.h deleted file mode 100644 index ccf3418f8..000000000 --- a/examples/balls/f32add/workloads/isa/f32add.h +++ /dev/null @@ -1,16 +0,0 @@ -#ifndef F32ADD_H -#define F32ADD_H -#include -#include - -// Four IEEE FP32 elements per 128-bit row, RNE, canonical NaN. -// rs2[4:0] selects the input group; bit 5 starts with +0 instead of reading b. -static inline void bb_f32add(uint32_t a, uint32_t b, uint32_t c, uint32_t rows, - uint32_t group, uint32_t first) { - if (a > 1023 || b > 1023 || c > 1023 || !rows || group > 31 || first > 1) - __builtin_trap(); - BUCKYBALL_INSTRUCTION_R_R(BB_BANK0(a) | BB_BANK1(b) | BB_BANK2(c) | - BB_ITER(rows), - group | (uint64_t)first << 5, BB_FUNC7(F32ADD)); -} -#endif diff --git a/examples/balls/f32add/workloads/mlir_tests/ball/f32add_groups.mlir b/examples/balls/f32add/workloads/mlir_tests/ball/f32add_groups.mlir deleted file mode 100644 index ee2f15544..000000000 --- a/examples/balls/f32add/workloads/mlir_tests/ball/f32add_groups.mlir +++ /dev/null @@ -1,17 +0,0 @@ -// RUN: buddy-opt %s --target=main -lower-buckyball -canonicalize | FileCheck %s -// CHECK-LABEL: func.func @reduce_groups -// CHECK-DAG: %[[AC:[^ ]+]] = arith.constant 2183169056 : i64 -// CHECK-DAG: %[[CA:[^ ]+]] = arith.constant 2182121504 : i64 -// CHECK-DAG: %[[FIRST:[^ ]+]] = arith.constant 32 : i64 -// CHECK: buckyball.intr.custom %[[AC]], %[[FIRST]] -// CHECK: buckyball.intr.custom %[[CA]], {{.*}} -// CHECK: buckyball.intr.custom %[[AC]], {{.*}} -func.func @reduce_groups() -> i64 attributes {llvm.emit_c_interface} { - %source = arith.constant 32 : i64 - %accumulator = arith.constant 33 : i64 - %output = arith.constant 34 : i64 - buckyball.f32add %source, %accumulator, %output : i64 - buckyball.f32add %source, %output, %accumulator : i64 - buckyball.f32add %source, %accumulator, %output : i64 - return %output : i64 -} diff --git a/examples/balls/f32add/workloads/mlir_tests/ball/f32add_groups_main.cpp b/examples/balls/f32add/workloads/mlir_tests/ball/f32add_groups_main.cpp deleted file mode 100644 index dc797d410..000000000 --- a/examples/balls/f32add/workloads/mlir_tests/ball/f32add_groups_main.cpp +++ /dev/null @@ -1,55 +0,0 @@ -#include -#include -#include -#include - -extern "C" int64_t _mlir_ciface_reduce_groups(); -static_assert(BB_SHARED_BANK_BASE == 32); - -static int checkF32Add() { - alignas(64) static const uint32_t input[3][8] = { - {0x3f800000, 0xc0200000, 0x7f800000, 0xff800000, 0, 0x80000000, - 0x7f7fffff, 1}, - {0x40000000, 0x40400000, 0xff800000, 0x7f800000, 0x80000000, 0, - 0xff7fffff, 1}, - {0xbf800000, 0x3f000000, 0x40a00000, 0xc0a00000, 0, 0x80000000, - 0x3f800000, 0x80000001}}; - alignas(64) static const uint32_t poison[8] = { - 0x7fa00001, 0x7fa00001, 0x7fa00001, 0x7fa00001, - 0x7fa00001, 0x7fa00001, 0x7fa00001, 0x7fa00001}; - alignas(64) static uint32_t output[8]; - bb_mem_alloc(32, 1, 3); - bb_mem_alloc(33, 1, 1); - bb_mem_alloc(34, 1, 1); - bb_mvin((uintptr_t)poison, 33, 2, 1); - for (unsigned group = 0; group < 3; ++group) - bb_mvin_group((uintptr_t)input[group], 32, group, 2, 1); - if (_mlir_ciface_reduce_groups() != 34) - return 1; - bb_mvout((uintptr_t)output, 34, 2, 1); - for (unsigned i = 0; i < 8; ++i) { - union { - uint32_t bits; - float value; - } expected = {0}, source; - for (unsigned group = 0; group < 3; ++group) { - source.bits = input[group][i]; - asm volatile("fadd.s %0, %1, %2, rne" - : "=f"(expected.value) - : "f"(expected.value), "f"(source.value)); - } - if (output[i] != expected.bits) - return 2; - } - bb_mem_release(32); - bb_mem_release(33); - bb_mem_release(34); - return 0; -} - -int main() { - int result = checkF32Add(); - if (!result) - puts("F32ADD Ball MLIR grouped/RNE/NaN/first/ordered sum PASS"); - return result; -} diff --git a/examples/balls/f32add/workloads/mlir_tests/ball/f32add_invalid.mlir b/examples/balls/f32add/workloads/mlir_tests/ball/f32add_invalid.mlir deleted file mode 100644 index 8b05ee7b4..000000000 --- a/examples/balls/f32add/workloads/mlir_tests/ball/f32add_invalid.mlir +++ /dev/null @@ -1,36 +0,0 @@ -// RUN: buddy-opt %s --target=main -split-input-file -verify-diagnostics -func.func @zero_rows(%a: i64, %b: i64, %c: i64) { - // expected-error @+1 {{rows must be in [1, 17179869183]}} - buckyball.f32add %a, %b, %c : i64 - return -} -// ----- -func.func @wide_group(%a: i64, %b: i64, %c: i64) { - // expected-error @+1 {{group must be in [0, 31]}} - buckyball.f32add %a, %b, %c : i64 - return -} -// ----- -func.func @wide_rows(%a: i64, %b: i64, %c: i64) { - // expected-error @+1 {{rows must be in [1, 17179869183]}} - buckyball.f32add %a, %b, %c : i64 - return -} -// ----- -func.func @equal_bank_constants() { - %a = arith.constant 32 : i64 - %b = arith.constant 33 : i64 - %c = arith.constant 32 : i64 - // expected-error @+1 {{source, accumulator and output banks must differ}} - buckyball.f32add %a, %b, %c : i64 - return -} -// ----- -func.func @wide_bank() { - %a = arith.constant 1024 : i64 - %b = arith.constant 33 : i64 - %c = arith.constant 34 : i64 - // expected-error @+1 {{bank ID must be in [0, 1023]}} - buckyball.f32add %a, %b, %c : i64 - return -} diff --git a/examples/balls/f32add/workloads/mlir_tests/bank/f32add_groups.mlir b/examples/balls/f32add/workloads/mlir_tests/bank/f32add_groups.mlir deleted file mode 100644 index f51cb20ad..000000000 --- a/examples/balls/f32add/workloads/mlir_tests/bank/f32add_groups.mlir +++ /dev/null @@ -1,21 +0,0 @@ -// RUN: buddy-opt %s --target=main -assign-physical-banks | FileCheck %s --check-prefix=BANK -// BANK-LABEL: func.func @reduce_groups -// BANK-COUNT-3: buckyball.f32add -// BANK-NOT: buckyball.bank_f32add -// RUN: buddy-opt %s --target=main -assign-physical-banks -lower-bank-ssa-to-intrinsics -lower-buckyball -canonicalize | FileCheck %s -// CHECK-LABEL: func.func @reduce_groups -// CHECK-DAG: %[[AC:[^ ]+]] = arith.constant 2183169056 : i64 -// CHECK-DAG: %[[CA:[^ ]+]] = arith.constant 2182121504 : i64 -// CHECK-DAG: %[[FIRST:[^ ]+]] = arith.constant 32 : i64 -// CHECK: buckyball.intr.custom %[[AC]], %[[FIRST]] -// CHECK: buckyball.intr.custom %[[CA]], {{.*}} -// CHECK: buckyball.intr.custom %[[AC]], {{.*}} -func.func @reduce_groups() -> i64 attributes {llvm.emit_c_interface} { - %source = arith.constant 32 : i64 - %accumulator = arith.constant 33 : i64 - %output = arith.constant 34 : i64 - %first = buckyball.bank_f32add %source %accumulator %output : i64 i64 i64 - %second = buckyball.bank_f32add %source %first %accumulator : i64 i64 i64 - %third = buckyball.bank_f32add %source %second %output : i64 i64 i64 - return %third : i64 -} diff --git a/examples/balls/f32add/workloads/mlir_tests/bank/f32add_groups_main.cpp b/examples/balls/f32add/workloads/mlir_tests/bank/f32add_groups_main.cpp deleted file mode 100644 index 43da2ae85..000000000 --- a/examples/balls/f32add/workloads/mlir_tests/bank/f32add_groups_main.cpp +++ /dev/null @@ -1,55 +0,0 @@ -#include -#include -#include -#include - -extern "C" int64_t _mlir_ciface_reduce_groups(); -static_assert(BB_SHARED_BANK_BASE == 32); - -static int checkF32Add() { - alignas(64) static const uint32_t input[3][8] = { - {0x3f800000, 0xc0200000, 0x7f800000, 0xff800000, 0, 0x80000000, - 0x7f7fffff, 1}, - {0x40000000, 0x40400000, 0xff800000, 0x7f800000, 0x80000000, 0, - 0xff7fffff, 1}, - {0xbf800000, 0x3f000000, 0x40a00000, 0xc0a00000, 0, 0x80000000, - 0x3f800000, 0x80000001}}; - alignas(64) static const uint32_t poison[8] = { - 0x7fa00001, 0x7fa00001, 0x7fa00001, 0x7fa00001, - 0x7fa00001, 0x7fa00001, 0x7fa00001, 0x7fa00001}; - alignas(64) static uint32_t output[8]; - bb_mem_alloc(32, 1, 3); - bb_mem_alloc(33, 1, 1); - bb_mem_alloc(34, 1, 1); - bb_mvin((uintptr_t)poison, 33, 2, 1); - for (unsigned group = 0; group < 3; ++group) - bb_mvin_group((uintptr_t)input[group], 32, group, 2, 1); - if (_mlir_ciface_reduce_groups() != 34) - return 1; - bb_mvout((uintptr_t)output, 34, 2, 1); - for (unsigned i = 0; i < 8; ++i) { - union { - uint32_t bits; - float value; - } expected = {0}, source; - for (unsigned group = 0; group < 3; ++group) { - source.bits = input[group][i]; - asm volatile("fadd.s %0, %1, %2, rne" - : "=f"(expected.value) - : "f"(expected.value), "f"(source.value)); - } - if (output[i] != expected.bits) - return 2; - } - bb_mem_release(32); - bb_mem_release(33); - bb_mem_release(34); - return 0; -} - -int main() { - int result = checkF32Add(); - if (!result) - puts("F32ADD Bank MLIR grouped/RNE/NaN/first/SSA output PASS"); - return result; -} diff --git a/examples/balls/f32add/workloads/mlir_tests/bank/f32add_invalid.mlir b/examples/balls/f32add/workloads/mlir_tests/bank/f32add_invalid.mlir deleted file mode 100644 index 90d633d65..000000000 --- a/examples/balls/f32add/workloads/mlir_tests/bank/f32add_invalid.mlir +++ /dev/null @@ -1,27 +0,0 @@ -// RUN: buddy-opt %s --target=main -split-input-file -verify-diagnostics -func.func @same_ssa_bank(%a: i64, %b: i64) { - // expected-error @+1 {{source, accumulator and output banks must differ}} - %out = buckyball.bank_f32add %a %b %b : i64 i64 i64 - return -} -// ----- -func.func @zero_rows(%a: i64, %b: i64, %c: i64) { - // expected-error @+1 {{rows must be in [1, 17179869183]}} - %out = buckyball.bank_f32add %a %b %c : i64 i64 i64 - return -} -// ----- -func.func @negative_group(%a: i64, %b: i64, %c: i64) { - // expected-error @+1 {{group must be in [0, 31]}} - %out = buckyball.bank_f32add %a %b %c : i64 i64 i64 - return -} -// ----- -func.func @negative_bank() { - %a = arith.constant -1 : i64 - %b = arith.constant 33 : i64 - %c = arith.constant 34 : i64 - // expected-error @+1 {{bank ID must be in [0, 1023]}} - %out = buckyball.bank_f32add %a %b %c : i64 i64 i64 - return -} diff --git a/examples/balls/gemmini/arch/src/main/scala/GemminiBall.scala b/examples/balls/gemmini/arch/src/main/scala/GemminiBall.scala deleted file mode 100644 index e76a09705..000000000 --- a/examples/balls/gemmini/arch/src/main/scala/GemminiBall.scala +++ /dev/null @@ -1,295 +0,0 @@ -package examples.balls.gemmini - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.balldomain.blink.{BallStatus, BlinkIO, HasBallStatus, HasBlink} -import framework.balldomain.blink.mmio.MmioRead -import framework.balldomain.rs.BallRsComplete -import framework.top.GlobalConfig - -@instantiable -class GemminiBall(val b: GlobalConfig) extends Module with HasBlink with HasBallStatus { - require(b.memDomain.bankEntries <= 1024, "GemminiBall rs2 row-base fields support at most 1024 bank entries") - require( - b.frontend.sub_rob_enable, - "GemminiBall requires frontend.sub_rob_enable=true because loop instructions emit SubROB rows" - ) - - val ballCommonConfig = b.ballDomain.ballIdMappings - .find(_.ballName == "GemminiBall") - .getOrElse( - throw new IllegalArgumentException("GemminiBall not found in config") - ) - - val inBW = ballCommonConfig.inBW - val outBW = ballCommonConfig.outBW - - @public - val io = IO(new BlinkIO(b, inBW, outBW)) - - def blink: BlinkIO = io - def status: BallStatus = io.status - dontTouch(io) - - // ========================================================================= - // Sub-modules - // ========================================================================= - val exCtrl: Instance[GemminiExCtrl] = Instantiate(new GemminiExCtrl(b)) - - val matmulUnroller: Instance[LoopMatmulUnroller] = Instantiate( - new LoopMatmulUnroller(b) - ) - - val convUnroller: Instance[LoopConvUnroller] = Instantiate( - new LoopConvUnroller(b) - ) - - val encoder: Instance[LoopCmdEncoder] = Instantiate(new LoopCmdEncoder(b)) - - // ========================================================================= - // Config registers for Loop modes - // ========================================================================= - val loopWsConfig = Reg(new LoopWsConfig(b)) - val loopConvConfig = Reg(new LoopConvWsConfig(b)) - - // rob_id tracking for bank metadata - val rob_id_reg = RegInit(0.U(log2Up(b.frontend.rob_entries).W)) - when(io.cmdReq.fire) { - rob_id_reg := io.cmdReq.bits.rob_id - } - - // ========================================================================= - // Instruction routing by funct7 - // ========================================================================= - val funct7 = io.cmdReq.bits.cmd.funct7 - - def ballFunct(mnemonic: String): UInt = - b.ballDomain.ballISA - .find(_.mnemonic == mnemonic) - .map(_.funct7.U(7.W)) - .getOrElse(throw new IllegalArgumentException(s"$mnemonic not found in ballISA")) - - val configFunct = ballFunct("GEMMINI_CONFIG") - val preloadFunct = ballFunct("GEMMINI_PRELOAD") - val computePreFunct = ballFunct("GEMMINI_COMPUTE_PRELOADED") - val computeAccFunct = ballFunct("GEMMINI_COMPUTE_ACCUMULATED") - val flushFunct = ballFunct("GEMMINI_FLUSH") - - val loopWsConfigFuncts = Seq( - ballFunct("GEMMINI_LOOP_WS_CONFIG_BOUNDS"), - ballFunct("GEMMINI_LOOP_WS_CONFIG_ADDR_A"), - ballFunct("GEMMINI_LOOP_WS_CONFIG_ADDR_B"), - ballFunct("GEMMINI_LOOP_WS_CONFIG_ADDR_D"), - ballFunct("GEMMINI_LOOP_WS_CONFIG_ADDR_C"), - ballFunct("GEMMINI_LOOP_WS_CONFIG_STRIDES_AB"), - ballFunct("GEMMINI_LOOP_WS_CONFIG_STRIDES_DC") - ) - - val loopWsTriggerFunct = ballFunct("GEMMINI_LOOP_WS") - - val loopConvConfigFuncts = Seq( - ballFunct("GEMMINI_LOOP_CONV_WS_CONFIG_1"), - ballFunct("GEMMINI_LOOP_CONV_WS_CONFIG_2"), - ballFunct("GEMMINI_LOOP_CONV_WS_CONFIG_3"), - ballFunct("GEMMINI_LOOP_CONV_WS_CONFIG_4"), - ballFunct("GEMMINI_LOOP_CONV_WS_CONFIG_5"), - ballFunct("GEMMINI_LOOP_CONV_WS_CONFIG_6"), - ballFunct("GEMMINI_LOOP_CONV_WS_CONFIG_7"), - ballFunct("GEMMINI_LOOP_CONV_WS_CONFIG_8"), - ballFunct("GEMMINI_LOOP_CONV_WS_CONFIG_9") - ) - - val loopConvTriggerFunct = ballFunct("GEMMINI_LOOP_CONV_WS") - - val rs2Data = io.cmdReq.bits.cmd.special - - val isConfig = funct7 === configFunct - val isPreload = funct7 === preloadFunct - val isComputePre = funct7 === computePreFunct - val isComputeAcc = funct7 === computeAccFunct - val isFlush = funct7 === flushFunct - val isExUnit = - isConfig || isPreload || isComputePre || isComputeAcc || isFlush - - val isLoopWsConfig = loopWsConfigFuncts.map(funct7 === _).reduce(_ || _) - val isLoopWsTrigger = funct7 === loopWsTriggerFunct - val isLoopConvConfig = loopConvConfigFuncts.map(funct7 === _).reduce(_ || _) - val isLoopConvTrigger = funct7 === loopConvTriggerFunct - - // ========================================================================= - // ExUnit path (non-Loop: CONFIG/PRELOAD/COMPUTE/FLUSH) - // ========================================================================= - exCtrl.exio.cmdReq.valid := io.cmdReq.valid && isExUnit - exCtrl.exio.cmdReq.bits := io.cmdReq.bits - - // ========================================================================= - // Config latch path (immediate cmdResp) - // ========================================================================= - val configRespValid = RegInit(false.B) - val configRespBits = Reg(new BallRsComplete(b)) - configRespValid := false.B // default: pulse - - when(io.cmdReq.fire && isLoopWsConfig) { - configRespValid := true.B - configRespBits.rob_id := io.cmdReq.bits.rob_id - configRespBits.is_sub := io.cmdReq.bits.is_sub - configRespBits.sub_rob_id := io.cmdReq.bits.sub_rob_id - switch(funct7) { - is(loopWsConfigFuncts(0)) { - loopWsConfig.max_k := rs2Data(15, 0) - loopWsConfig.max_j := rs2Data(31, 16) - loopWsConfig.max_i := rs2Data(47, 32) - } - is(loopWsConfigFuncts(1))(loopWsConfig.dram_addr_a := rs2Data(38, 0)) - is(loopWsConfigFuncts(2))(loopWsConfig.dram_addr_b := rs2Data(38, 0)) - is(loopWsConfigFuncts(3))(loopWsConfig.dram_addr_d := rs2Data(38, 0)) - is(loopWsConfigFuncts(4))(loopWsConfig.dram_addr_c := rs2Data(38, 0)) - is(loopWsConfigFuncts(5)) { - loopWsConfig.stride_a := rs2Data(31, 0) - loopWsConfig.stride_b := rs2Data(63, 32) - } - is(loopWsConfigFuncts(6)) { - loopWsConfig.stride_d := rs2Data(31, 0) - loopWsConfig.stride_c := rs2Data(63, 32) - } - } - } - - when(io.cmdReq.fire && isLoopConvConfig) { - configRespValid := true.B - configRespBits.rob_id := io.cmdReq.bits.rob_id - configRespBits.is_sub := io.cmdReq.bits.is_sub - configRespBits.sub_rob_id := io.cmdReq.bits.sub_rob_id - switch(funct7) { - is(loopConvConfigFuncts(0)) { - loopConvConfig.batch_size := rs2Data(15, 0) - loopConvConfig.in_dim := rs2Data(31, 16) - loopConvConfig.in_channels := rs2Data(47, 32) - } - is(loopConvConfigFuncts(1)) { - loopConvConfig.out_channels := rs2Data(15, 0) - loopConvConfig.out_dim := rs2Data(31, 16) - loopConvConfig.stride := rs2Data(39, 32) - loopConvConfig.padding := rs2Data(47, 40) - } - is(loopConvConfigFuncts(2)) { - loopConvConfig.kernel_dim := rs2Data(7, 0) - loopConvConfig.pool_size := rs2Data(15, 8) - loopConvConfig.pool_stride := rs2Data(23, 16) - loopConvConfig.pool_padding := rs2Data(31, 24) - } - is(loopConvConfigFuncts(3))(loopConvConfig.dram_addr_bias := rs2Data(38, 0)) - is(loopConvConfigFuncts(4))(loopConvConfig.dram_addr_input := rs2Data(38, 0)) - is(loopConvConfigFuncts(5))(loopConvConfig.dram_addr_weight := rs2Data(38, 0)) - is(loopConvConfigFuncts(6))(loopConvConfig.dram_addr_output := rs2Data(38, 0)) - is(loopConvConfigFuncts(7)) { - loopConvConfig.input_stride := rs2Data(31, 0) - loopConvConfig.weight_stride := rs2Data(63, 32) - } - is(loopConvConfigFuncts(8))(loopConvConfig.output_stride := rs2Data(31, 0)) - } - } - - // ========================================================================= - // Loop trigger: latch bank IDs and start unroller (no cmdResp) - // Bank values come from rs2Data in the trigger instruction, but loopWsConfig - // Reg won't update until next edge. Override start.bits combinationally. - // ========================================================================= - matmulUnroller.io.start.valid := false.B - matmulUnroller.io.start.bits := loopWsConfig - - when(io.cmdReq.fire && isLoopWsTrigger) { - loopWsConfig.bank_a := rs2Data(9, 0) - loopWsConfig.bank_b := rs2Data(19, 10) - loopWsConfig.bank_c := rs2Data(29, 20) - loopWsConfig.low_d := rs2Data(30) - matmulUnroller.io.start.valid := true.B - matmulUnroller.io.start.bits.bank_a := rs2Data(9, 0) - matmulUnroller.io.start.bits.bank_b := rs2Data(19, 10) - matmulUnroller.io.start.bits.bank_c := rs2Data(29, 20) - matmulUnroller.io.start.bits.low_d := rs2Data(30) - } - - convUnroller.io.start.valid := false.B - convUnroller.io.start.bits := loopConvConfig - - when(io.cmdReq.fire && isLoopConvTrigger) { - loopConvConfig.bank_input := rs2Data(9, 0) - loopConvConfig.bank_weight := rs2Data(19, 10) - loopConvConfig.bank_output := rs2Data(29, 20) - loopConvConfig.no_bias := rs2Data(30) - convUnroller.io.start.valid := true.B - convUnroller.io.start.bits.bank_input := rs2Data(9, 0) - convUnroller.io.start.bits.bank_weight := rs2Data(19, 10) - convUnroller.io.start.bits.bank_output := rs2Data(29, 20) - convUnroller.io.start.bits.no_bias := rs2Data(30) - } - - // ========================================================================= - // LoopUnrollers → Arbiter → LoopCmdEncoder → io.subRobReq - // ========================================================================= - val cmdArb = Module(new Arbiter(new LoopCmd(b), 2)) - cmdArb.io.in(0) <> matmulUnroller.io.cmd - cmdArb.io.in(1) <> convUnroller.io.cmd - encoder.io.cmd <> cmdArb.io.out - encoder.io.subRobRow <> io.subRobReq - encoder.io.ballId := ballCommonConfig.ballId.U - encoder.io.masterRobId := rob_id_reg - - // ========================================================================= - // cmdReq.ready: route to correct consumer - // ========================================================================= - val idleReady = - exCtrl.exio.cmdReq.ready && !matmulUnroller.io.busy && !convUnroller.io.busy - io.cmdReq.ready := Mux( - isExUnit, - exCtrl.exio.cmdReq.ready, - Mux( - isLoopWsConfig || isLoopConvConfig, - idleReady, - Mux( - isLoopWsTrigger, - !matmulUnroller.io.busy, - Mux(isLoopConvTrigger, !convUnroller.io.busy, idleReady) - ) - ) - ) - - // ========================================================================= - // cmdResp: mux between exUnit and config immediate response - // ========================================================================= - io.cmdResp <> exCtrl.exio.cmdResp - when(configRespValid) { - io.cmdResp.valid := true.B - io.cmdResp.bits := configRespBits - } - - // ========================================================================= - // Bank connections (unchanged from original) - // ========================================================================= - for (i <- 0 until inBW) { - io.bankRead(i).io.req <> exCtrl.exio.bankReadReq(i) - exCtrl.exio.bankReadResp(i) <> io.bankRead(i).io.resp - io.bankRead(i).rob_id := rob_id_reg - io.bankRead(i).ball_id := 0.U - io.bankRead(i).group_id := 0.U - } - io.bankRead(0).bank_id := exCtrl.exio.op1_bank_o - io.bankRead(0).group_id := exCtrl.exio.op1_group_o - if (inBW > 1) { - io.bankRead(1).bank_id := exCtrl.exio.op2_bank_o - } - - for (i <- 0 until outBW) { - io.bankWrite(i).io <> exCtrl.exio.bankWrite(i) - io.bankWrite(i).bank_id := exCtrl.exio.wr_bank_o - io.bankWrite(i).rob_id := rob_id_reg - io.bankWrite(i).ball_id := 0.U - io.bankWrite(i).group_id := i.U - } - - io.status <> exCtrl.exio.status - - MmioRead.tieOff(io.mmioRead) -} diff --git a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrl.scala b/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrl.scala deleted file mode 100644 index 8b8a5cc05..000000000 --- a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrl.scala +++ /dev/null @@ -1,26 +0,0 @@ -package examples.balls.gemmini - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.top.GlobalConfig - -@instantiable -class GemminiExCtrl(val b: GlobalConfig) - extends Module - with GemminiExCtrlDefs - with GemminiExCtrlDefaults - with GemminiExCtrlCmdStates - with GemminiExCtrlPreloadStates - with GemminiExCtrlComputeReadState - with GemminiExCtrlComputeFeedState - with GemminiExCtrlStoreOps - with GemminiExCtrlAccumulateState - with GemminiExCtrlFsm { - @public val exio = io - - applyDefaults() - runFsm() - - io.status.idle := state === sIdle - io.status.running := state =/= sIdle -} diff --git a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlCmdStates.scala b/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlCmdStates.scala deleted file mode 100644 index 4657da2fc..000000000 --- a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlCmdStates.scala +++ /dev/null @@ -1,105 +0,0 @@ -package examples.balls.gemmini - -import chisel3._ -import chisel3.util._ -import gemmini._ - -/** Sub-command encoding within the special field */ -object GemminiSubCmd { - val CONFIG = 0.U(4.W) - val PRELOAD = 1.U(4.W) - val COMPUTE_PRELOADED = 2.U(4.W) - val COMPUTE_ACCUMULATED = 3.U(4.W) - val FLUSH = 4.U(4.W) -} - -trait GemminiExCtrlCmdStates { this: GemminiExCtrl => - - protected def handleIdleState(): Unit = { - io.cmdReq.ready := true.B - when(io.cmdReq.fire) { - rob_id_reg := io.cmdReq.bits.rob_id - is_sub_reg := io.cmdReq.bits.is_sub - sub_rob_id_reg := io.cmdReq.bits.sub_rob_id - op1_bank := io.cmdReq.bits.cmd.op1_bank - op2_bank := io.cmdReq.bits.cmd.op2_bank - wr_bank := io.cmdReq.bits.cmd.wr_bank - op1_base := io.cmdReq.bits.cmd.special(15, 6) - op2_base := io.cmdReq.bits.cmd.special(25, 16) - wr_base := io.cmdReq.bits.cmd.special(35, 26) - total_rows := Mux( - io.cmdReq.bits.cmd.iter === 0.U, - DIM.U, - io.cmdReq.bits.cmd.iter - ) - zero_op2 := io.cmdReq.bits.cmd.special(4) - zero_op1_tail := io.cmdReq.bits.cmd.special(5) - accumulated_cmd := sub_cmd === GemminiSubCmd.COMPUTE_ACCUMULATED - reset_acc_mesh := sub_cmd === GemminiSubCmd.COMPUTE_ACCUMULATED - acc_read_group := 0.U - acc_read_pending := false.B - - when(sub_cmd === GemminiSubCmd.CONFIG) { - cfg_dataflow := io.cmdReq.bits.cmd.special(4) - cfg_a_transpose := io.cmdReq.bits.cmd.special(7) - cfg_bd_transpose := io.cmdReq.bits.cmd.special(8) - cfg_in_shift := io.cmdReq.bits.cmd.special( - log2Up(config.accWidth) + 8, - 9 - ) - io.cmdResp.valid := true.B - state := sCommit - }.elsewhen(sub_cmd === GemminiSubCmd.PRELOAD) { - read_row_cnt := 0.U - feed_row_cnt := 0.U - req_sent := false.B - xpose_ready := false.B - xpose_row_cnt := 0.U - state := sPreloadRead - }.elsewhen( - sub_cmd === GemminiSubCmd.COMPUTE_PRELOADED || sub_cmd === GemminiSubCmd.COMPUTE_ACCUMULATED - ) { - read_row_cnt := 0.U - feed_row_cnt := 0.U - outBufRows := 0.U - outBufCollected := 0.U - req_sent := false.B - xpose_ready := false.B - xpose_row_cnt := 0.U - read_done.foreach(_ := false.B) - state := Mux(sub_cmd === GemminiSubCmd.COMPUTE_ACCUMULATED, sPreloadFeed, sComputeRead) - }.elsewhen(sub_cmd === GemminiSubCmd.FLUSH) { - state := sFlush - } - } - } - - protected def handleCommitState(): Unit = { - io.cmdResp.valid := true.B - when(io.cmdResp.fire) { - state := sIdle - } - } - - protected def handleFlushState(): Unit = { - mesh.io.req.valid := true.B - mesh.io.req.bits.flush := 2.U - mesh.io.req.bits.total_rows := DIM.U - mesh.io.req.bits.tag.rob := robIdAsTag8(rob_id_reg) - - mesh.io.a.valid := true.B - mesh.io.a.bits := 0.U.asTypeOf(mesh.A_TYPE) - mesh.io.b.valid := true.B - mesh.io.b.bits := 0.U.asTypeOf(mesh.B_TYPE) - mesh.io.d.valid := true.B - mesh.io.d.bits := 0.U.asTypeOf(mesh.D_TYPE) - - when(mesh.io.req.ready) { - io.cmdResp.valid := true.B - when(io.cmdResp.fire) { - state := sIdle - } - } - } - -} diff --git a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlDefaults.scala b/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlDefaults.scala deleted file mode 100644 index 932af6393..000000000 --- a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlDefaults.scala +++ /dev/null @@ -1,43 +0,0 @@ -package examples.balls.gemmini - -import chisel3._ -import chisel3.util._ - -trait GemminiExCtrlDefaults { this: GemminiExCtrl => - - protected def applyDefaults(): Unit = { - io.cmdReq.ready := state === sIdle - - io.cmdResp.valid := false.B - io.cmdResp.bits.rob_id := rob_id_reg - io.cmdResp.bits.is_sub := is_sub_reg - io.cmdResp.bits.sub_rob_id := sub_rob_id_reg - - for (i <- 0 until inBW) { - io.bankReadReq(i).valid := false.B - io.bankReadReq(i).bits.addr := 0.U - } - - rdQueue0.io.deq.ready := false.B - rdQueue1.io.deq.ready := false.B - - mesh.io.a.valid := false.B - mesh.io.a.bits := 0.U.asTypeOf(mesh.A_TYPE) - mesh.io.b.valid := false.B - mesh.io.b.bits := 0.U.asTypeOf(mesh.B_TYPE) - mesh.io.d.valid := false.B - mesh.io.d.bits := 0.U.asTypeOf(mesh.D_TYPE) - mesh.io.req.valid := false.B - mesh.io.req.bits := 0.U.asTypeOf(mesh.io.req.bits) - // mesh.io.resp is Valid-only (no ready); MeshWithDelays assumes immediate observation when valid - - io.bankWrite.foreach { bw => - bw.req.valid := false.B - bw.req.bits.addr := 0.U - bw.req.bits.data := 0.U - bw.req.bits.mask := VecInit(Seq.fill(b.memDomain.bankMaskLen)(false.B)) - bw.resp.ready := true.B - } - } - -} diff --git a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlDefs.scala b/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlDefs.scala deleted file mode 100644 index e705d55f8..000000000 --- a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlDefs.scala +++ /dev/null @@ -1,138 +0,0 @@ -package examples.balls.gemmini - -import chisel3._ -import chisel3.util._ -import framework.balldomain.blink.BallStatus -import framework.balldomain.rs.{BallRsComplete, BallRsIssue} -import framework.memdomain.backend.banks.{SramReadReq, SramReadResp, SramWriteIO} -import examples.balls.gemmini.configs.GemminiBallParam -import gemmini._ -import gemmini.Util._ - -/** Minimal tag for MeshWithDelays that satisfies TagQueueTag */ -class SimpleTag extends Bundle with gemmini.TagQueueTag { - val rob = UInt(8.W) - override def make_this_garbage(dummy: Int = 0): Unit = - rob := 0xff.U -} - -trait GemminiExCtrlDefs { this: GemminiExCtrl => - val config = GemminiBallParam(b) - val DIM = config.blockSize - - val ballMapping = b.ballDomain.ballIdMappings - .find(_.ballName == "GemminiBall") - .getOrElse( - throw new IllegalArgumentException("GemminiBall not found in config") - ) - - val inBW = ballMapping.inBW - val outBW = ballMapping.outBW - - val inputType = SInt(config.inputWidth.W) - val accType = SInt(config.accWidth.W) - val meshOutputType = SInt(config.spatialOutputWidth.W) - - val ctrlIo = IO(new Bundle { - val cmdReq = Flipped(Decoupled(new BallRsIssue(b))) - val cmdResp = Decoupled(new BallRsComplete(b)) - val bankReadReq = Vec(inBW, Decoupled(new SramReadReq(b))) - val bankReadResp = Vec(inBW, Flipped(Decoupled(new SramReadResp(b)))) - val bankWrite = Vec(outBW, Flipped(new SramWriteIO(b))) - val op1_bank_o = Output(UInt(b.memDomain.vbankIdWidth.W)) - val op1_group_o = Output(UInt(b.memDomain.groupIdWidth.W)) - val op2_bank_o = Output(UInt(b.memDomain.vbankIdWidth.W)) - val wr_bank_o = Output(UInt(b.memDomain.vbankIdWidth.W)) - val status = new BallStatus - }) - - val io = ctrlIo - - val mesh = Module( - new MeshWithDelays( - inputType = inputType, - weightType = inputType, - outputType = meshOutputType, - accType = accType, - tagType = new SimpleTag, - df = Dataflow.BOTH, - tree_reduction = false, - tile_latency = config.tileLatency, - output_delay = config.outputDelay, - tileRows = config.tileRows, - tileColumns = config.tileColumns, - meshRows = config.meshRows, - meshColumns = config.meshColumns, - leftBanks = 1, - upBanks = 1 - ) - ) - - protected def widenMeshToAcc(src: Vec[Vec[SInt]]): Vec[Vec[SInt]] = - VecInit(src.map(col => VecInit(col.map(_.asTypeOf(accType))))) - - val cfg_dataflow = RegInit(0.U(1.W)) - val cfg_in_shift = RegInit(0.U(log2Up(config.accWidth).W)) - val cfg_a_transpose = RegInit(false.B) - val cfg_bd_transpose = RegInit(false.B) - val zero_op2 = RegInit(false.B) - val zero_op1_tail = RegInit(false.B) - val accumulated_cmd = RegInit(false.B) - val reset_acc_mesh = RegInit(false.B) - val acc_read_group = RegInit(0.U(log2Up(outBW).W)) - val acc_read_pending = RegInit(false.B) - val execution_dataflow = Mux(accumulated_cmd, Dataflow.OS.id.U, cfg_dataflow) - val execution_shift = Mux(accumulated_cmd, 0.U, cfg_in_shift) - - val sIdle :: sPreloadRead :: sPreloadFeed :: sComputeRead :: sComputeFeed :: sComputeFlush :: sFlush :: sDrain :: sReadAccum :: sStore :: sCommit :: Nil = - Enum(11) - val state = RegInit(sIdle) - - val rob_id_reg = RegInit(0.U(log2Up(b.frontend.rob_entries).W)) - val is_sub_reg = RegInit(false.B) - val sub_rob_id_reg = RegInit(0.U(log2Up(b.frontend.sub_rob_depth * 4).W)) - - /** Match SimpleTag.rob (8b); must match mesh req tag bits */ - protected def robIdAsTag8(x: UInt): UInt = { - val w = x.getWidth - if (w >= 8) x(7, 0) else Cat(0.U((8 - w).W), x) - } - - val op1_bank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - val op2_bank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - val wr_bank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - val op1_base = RegInit(0.U(10.W)) - val op2_base = RegInit(0.U(10.W)) - val wr_base = RegInit(0.U(10.W)) - io.op1_bank_o := Mux(state === sReadAccum, wr_bank, op1_bank) - io.op1_group_o := Mux(state === sReadAccum, acc_read_group, 0.U) - io.op2_bank_o := op2_bank - io.wr_bank_o := wr_bank - - val read_row_cnt = RegInit(0.U(log2Up(DIM + 1).W)) - val feed_row_cnt = RegInit(0.U(log2Up(DIM + 1).W)) - val store_row_cnt = RegInit(0.U(log2Up(DIM + 1).W)) - val total_rows = RegInit(DIM.U(log2Up(DIM + 1).W)) - val req_sent = RegInit(false.B) - val read_done = RegInit(VecInit(Seq.fill(inBW)(false.B))) - val xpose_ready = RegInit(false.B) - val xpose_row_cnt = RegInit(0.U(log2Up(DIM + 1).W)) - - val sub_cmd = io.cmdReq.bits.cmd.special(3, 0) - - val rdQueue0 = Module(new Queue(new SramReadResp(b), entries = DIM)) - val rdQueue1 = Module(new Queue(new SramReadResp(b), entries = DIM)) - rdQueue0.io.enq <> io.bankReadResp(0) - rdQueue1.io.enq <> io.bankReadResp(1) - - val outBuf = Reg( - Vec(DIM, Vec(config.meshColumns, Vec(config.tileColumns, accType))) - ) - - val outBufRows = RegInit(0.U(log2Up(DIM + 1).W)) - val outBufCollected = RegInit(0.U(log2Up(DIM + 1).W)) - val op1Buf = Reg(Vec(DIM, Vec(DIM, inputType))) - val op2Buf = Reg(Vec(DIM, Vec(DIM, inputType))) - - val port_written = RegInit(VecInit(Seq.fill(outBW)(false.B))) -} diff --git a/examples/balls/gemmini/arch/src/main/scala/LoopCmd.scala b/examples/balls/gemmini/arch/src/main/scala/LoopCmd.scala deleted file mode 100644 index 9e1285a71..000000000 --- a/examples/balls/gemmini/arch/src/main/scala/LoopCmd.scala +++ /dev/null @@ -1,77 +0,0 @@ -package examples.balls.gemmini - -import chisel3._ -import chisel3.util._ -import framework.top.GlobalConfig - -object LoopSubCmdType { - val MSET_ALLOC = 0.U(3.W) - val MSET_FREE = 1.U(3.W) - val MVIN = 2.U(3.W) - val MVOUT = 3.U(3.W) - val PRELOAD = 4.U(3.W) - val COMPUTE = 5.U(3.W) -} - -class LoopSubCmd(val b: GlobalConfig) extends Bundle { - val cmdType = UInt(3.W) - val bank_id = UInt(b.memDomain.vbankIdWidth.W) - val dram_addr = UInt(b.memDomain.memAddrLen.W) - val stride = UInt(19.W) - val iter = UInt(b.frontend.iter_len.W) - val bank_row = UInt(5.W) - val bank_col = UInt(5.W) - val op1_bank = UInt(b.memDomain.vbankIdWidth.W) - val op2_bank = UInt(b.memDomain.vbankIdWidth.W) - val wr_bank = UInt(b.memDomain.vbankIdWidth.W) - val compute_mode = UInt(2.W) - val zero_op2 = Bool() - val zero_op1_tail = Bool() -} - -class LoopCmd(val b: GlobalConfig) extends Bundle { - val slots = Vec(4, Valid(new LoopSubCmd(b))) -} - -class LoopWsConfig(val b: GlobalConfig) extends Bundle { - val max_i = UInt(16.W) - val max_j = UInt(16.W) - val max_k = UInt(16.W) - val dram_addr_a = UInt(b.memDomain.memAddrLen.W) - val dram_addr_b = UInt(b.memDomain.memAddrLen.W) - val dram_addr_d = UInt(b.memDomain.memAddrLen.W) - val dram_addr_c = UInt(b.memDomain.memAddrLen.W) - val stride_a = UInt(32.W) - val stride_b = UInt(32.W) - val stride_d = UInt(32.W) - val stride_c = UInt(32.W) - val bank_a = UInt(b.memDomain.vbankIdWidth.W) - val bank_b = UInt(b.memDomain.vbankIdWidth.W) - val bank_c = UInt(b.memDomain.vbankIdWidth.W) - val low_d = Bool() -} - -class LoopConvWsConfig(val b: GlobalConfig) extends Bundle { - val batch_size = UInt(16.W) - val in_dim = UInt(16.W) - val in_channels = UInt(16.W) - val out_channels = UInt(16.W) - val out_dim = UInt(16.W) - val stride = UInt(8.W) - val padding = UInt(8.W) - val kernel_dim = UInt(8.W) - val pool_size = UInt(8.W) - val pool_stride = UInt(8.W) - val pool_padding = UInt(8.W) - val dram_addr_bias = UInt(b.memDomain.memAddrLen.W) - val dram_addr_input = UInt(b.memDomain.memAddrLen.W) - val dram_addr_weight = UInt(b.memDomain.memAddrLen.W) - val dram_addr_output = UInt(b.memDomain.memAddrLen.W) - val input_stride = UInt(32.W) - val weight_stride = UInt(32.W) - val output_stride = UInt(32.W) - val bank_input = UInt(b.memDomain.vbankIdWidth.W) - val bank_weight = UInt(b.memDomain.vbankIdWidth.W) - val bank_output = UInt(b.memDomain.vbankIdWidth.W) - val no_bias = Bool() -} diff --git a/examples/balls/gemmini/arch/src/main/scala/LoopCmdEncoder.scala b/examples/balls/gemmini/arch/src/main/scala/LoopCmdEncoder.scala deleted file mode 100644 index 964498e4e..000000000 --- a/examples/balls/gemmini/arch/src/main/scala/LoopCmdEncoder.scala +++ /dev/null @@ -1,143 +0,0 @@ -package examples.balls.gemmini - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.top.GlobalConfig -import framework.balldomain.blink.SubRobRow -import framework.frontend.scoreboard.BankAccessInfo -import framework.frontend.decoder.DomainId - -/** Converts LoopCmd (logical) → SubRobRow (hardware PostGDCmd) for SubROB. */ -@instantiable -class LoopCmdEncoder(val b: GlobalConfig) extends Module { - val bankIdLen = b.memDomain.vbankIdWidth - - def ballFunct(mnemonic: String): UInt = - b.ballDomain.ballISA - .find(_.mnemonic == mnemonic) - .map(_.funct7.U(7.W)) - .getOrElse(throw new IllegalArgumentException(s"$mnemonic not found in ballISA")) - - val preloadFunct = ballFunct("GEMMINI_PRELOAD") - val computePreFunct = ballFunct("GEMMINI_COMPUTE_PRELOADED") - val computeAccFunct = ballFunct("GEMMINI_COMPUTE_ACCUMULATED") - - @public - val io = IO(new Bundle { - val cmd = Flipped(Decoupled(new LoopCmd(b))) - val subRobRow = Decoupled(new SubRobRow(b)) - val ballId = Input(UInt(log2Up(b.ballDomain.ballNum).W)) - val masterRobId = Input(UInt(log2Up(b.frontend.rob_entries).W)) - }) - - // Passthrough handshake - io.subRobRow.valid := io.cmd.valid - io.cmd.ready := io.subRobRow.ready - - io.subRobRow.bits.ball_id := io.ballId - io.subRobRow.bits.master_rob_id := io.masterRobId - - for (i <- 0 until 4) { - val slot = io.subRobRow.bits.slots(i) - val lsub = io.cmd.bits.slots(i) - slot.valid := lsub.valid - - // Defaults — all unused RoCCCommandBB fields zero - slot.cmd.domain_id := 0.U - slot.cmd.ball_bid := io.ballId - slot.cmd.cmd.raw_inst := 0.U - slot.cmd.cmd.pc := 0.U - slot.cmd.cmd.funct := 0.U - slot.cmd.cmd.funct3 := 0.U - slot.cmd.cmd.rs2 := 0.U - slot.cmd.cmd.rs1 := 0.U - slot.cmd.cmd.xd := false.B - slot.cmd.cmd.xs1 := false.B - slot.cmd.cmd.xs2 := false.B - slot.cmd.cmd.rd := 0.U - slot.cmd.cmd.opcode := 0.U - slot.cmd.cmd.rs1Data := 0.U - slot.cmd.cmd.rs2Data := 0.U - slot.cmd.bankAccess := BankAccessInfo.none(bankIdLen) - slot.cmd.op1_col := 0.U - slot.cmd.op2_col := 0.U - slot.cmd.wr_col := 0.U - slot.cmd.isBarrier := false.B - - when(lsub.valid) { - switch(lsub.bits.cmdType) { - is(LoopSubCmdType.MSET_ALLOC) { - slot.cmd.domain_id := DomainId.MEM - slot.cmd.cmd.funct := 0x20.U // MSET (enable=010, opcode=0) - slot.cmd.cmd.rs1Data := lsub.bits.bank_id - slot.cmd.cmd.rs2Data := lsub.bits.bank_row.asUInt | - (lsub.bits.bank_col.asUInt << 5) | - (1.U << 10) // alloc=1 - slot.cmd.bankAccess.wr_bank_valid := true.B - slot.cmd.bankAccess.wr_bank_id := lsub.bits.bank_id - } - is(LoopSubCmdType.MSET_FREE) { - slot.cmd.domain_id := DomainId.MEM - slot.cmd.cmd.funct := 0x20.U // MSET - slot.cmd.cmd.rs1Data := lsub.bits.bank_id - slot.cmd.cmd.rs2Data := 0.U // alloc=0 - slot.cmd.bankAccess.wr_bank_valid := true.B - slot.cmd.bankAccess.wr_bank_id := lsub.bits.bank_id - } - is(LoopSubCmdType.MVIN) { - slot.cmd.domain_id := DomainId.MEM - slot.cmd.cmd.funct := 0x21.U // MVIN (enable=010, opcode=1) - slot.cmd.cmd.rs1Data := (lsub.bits.bank_id << 20) | (lsub.bits.iter << 30) - slot.cmd.cmd.rs2Data := lsub.bits.dram_addr | (lsub.bits.stride << 39) - slot.cmd.bankAccess.wr_bank_valid := true.B - slot.cmd.bankAccess.wr_bank_id := lsub.bits.bank_id - } - is(LoopSubCmdType.MVOUT) { - slot.cmd.domain_id := DomainId.MEM - slot.cmd.cmd.funct := 0x10.U // MVOUT (enable=001, opcode=0) - slot.cmd.cmd.rs1Data := lsub.bits.bank_id | (lsub.bits.iter << 30) - slot.cmd.cmd.rs2Data := lsub.bits.dram_addr | (lsub.bits.stride << 39) - slot.cmd.bankAccess.rd_bank_0_valid := true.B - slot.cmd.bankAccess.rd_bank_0_id := lsub.bits.bank_id - } - is(LoopSubCmdType.PRELOAD) { - slot.cmd.domain_id := DomainId.BALL - slot.cmd.cmd.funct := preloadFunct - slot.cmd.cmd.rs1Data := lsub.bits.op1_bank | - (lsub.bits.wr_bank << 20) | - (lsub.bits.iter << 30) - slot.cmd.cmd.rs2Data := 1.U - slot.cmd.bankAccess.rd_bank_0_valid := true.B - slot.cmd.bankAccess.rd_bank_0_id := lsub.bits.op1_bank - slot.cmd.bankAccess.wr_bank_valid := true.B - slot.cmd.bankAccess.wr_bank_id := lsub.bits.wr_bank - } - is(LoopSubCmdType.COMPUTE) { - slot.cmd.domain_id := DomainId.BALL - slot.cmd.cmd.funct := Mux( - lsub.bits.compute_mode === 0.U, - computePreFunct, - computeAccFunct - ) - slot.cmd.cmd.rs1Data := lsub.bits.op1_bank | - (lsub.bits.op2_bank << 10) | - (lsub.bits.wr_bank << 20) | - (lsub.bits.iter << 30) - slot.cmd.cmd.rs2Data := Mux( - lsub.bits.compute_mode === 0.U, - 2.U, - 3.U - ) | (lsub.bits.zero_op2 << 4) | - (lsub.bits.zero_op1_tail << 5) - slot.cmd.bankAccess.rd_bank_0_valid := true.B - slot.cmd.bankAccess.rd_bank_0_id := lsub.bits.op1_bank - slot.cmd.bankAccess.rd_bank_1_valid := !lsub.bits.zero_op2 - slot.cmd.bankAccess.rd_bank_1_id := lsub.bits.op2_bank - slot.cmd.bankAccess.wr_bank_valid := true.B - slot.cmd.bankAccess.wr_bank_id := lsub.bits.wr_bank - } - } - } - } -} diff --git a/examples/balls/gemmini/arch/src/main/scala/LoopConvUnroller.scala b/examples/balls/gemmini/arch/src/main/scala/LoopConvUnroller.scala deleted file mode 100644 index 778f50306..000000000 --- a/examples/balls/gemmini/arch/src/main/scala/LoopConvUnroller.scala +++ /dev/null @@ -1,301 +0,0 @@ -package examples.balls.gemmini - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.top.GlobalConfig -import examples.balls.gemmini.configs.GemminiBallParam - -/** - * LoopConvUnroller — expands LOOP_CONV_WS into LoopCmd rows. - * - * Simplified iteration: - * batch → orow → ocol → och → krow → kcol → kch (innermost) - * - * Each (orow, ocol, och, krow, kcol, kch) tile generates: - * MVIN input + MVIN weight → PRELOAD + COMPUTE → (on last k-iter) MVOUT - * - * First implementation: no pooling, no im2col tiling, single DIM×DIM tiles. - */ -@instantiable -class LoopConvUnroller(val b: GlobalConfig) extends Module { - val config = GemminiBallParam(b) - val DIM = config.blockSize - val elemSize = config.inputWidth / 8 - val bankBytes = b.memDomain.bankWidth / 8 - - @public - val io = IO(new Bundle { - val start = Flipped(Decoupled(new LoopConvWsConfig(b))) - val cmd = Decoupled(new LoopCmd(b)) - val busy = Output(Bool()) - }) - - val addrGen: Instance[LoopConvAddrGen] = Instantiate(new LoopConvAddrGen(b)) - - // FSM states - val sIdle :: sAlloc :: sMvinInput :: sMvinWeight :: sCompute :: sMvout :: sFree :: sDone :: Nil = - Enum(8) - val state = RegInit(sIdle) - - val cfg = Reg(new LoopConvWsConfig(b)) - - // Iterator registers - val batch_reg = RegInit(0.U(16.W)) - val orow_reg = RegInit(0.U(16.W)) - val ocol_reg = RegInit(0.U(16.W)) - val och_reg = RegInit(0.U(16.W)) - val krow_reg = RegInit(0.U(16.W)) - val kcol_reg = RegInit(0.U(16.W)) - val kch_reg = RegInit(0.U(16.W)) - - // Is this the first k-iteration for this (orow, ocol, och)? - val isFirstK = krow_reg === 0.U && kcol_reg === 0.U && kch_reg === 0.U - - // Is this the last k-iteration? - val isLastK = (krow_reg === cfg.kernel_dim - 1.U) && - (kcol_reg === cfg.kernel_dim - 1.U) && - (kch_reg + DIM.U >= cfg.in_channels) // Tile boundary - - // Connect address generator - addrGen.io.cfg := cfg - addrGen.io.batch := batch_reg - addrGen.io.orow := orow_reg - addrGen.io.ocol := ocol_reg - addrGen.io.och := och_reg - addrGen.io.krow := krow_reg - addrGen.io.kcol := kcol_reg - addrGen.io.kch := kch_reg - - // Defaults - io.start.ready := state === sIdle - io.cmd.valid := false.B - io.cmd.bits := 0.U.asTypeOf(new LoopCmd(b)) - io.busy := state =/= sIdle - - def emptySlot(): Valid[LoopSubCmd] = { - val v = Wire(Valid(new LoopSubCmd(b))) - v.valid := false.B - v.bits := 0.U.asTypeOf(new LoopSubCmd(b)) - v - } - - def msetSlot( - bankId: UInt, - alloc: Boolean, - row: Int, - col: Int - ): Valid[LoopSubCmd] = { - val v = Wire(Valid(new LoopSubCmd(b))) - v.valid := true.B - v.bits := 0.U.asTypeOf(new LoopSubCmd(b)) - v.bits.cmdType := (if (alloc) LoopSubCmdType.MSET_ALLOC - else LoopSubCmdType.MSET_FREE) - v.bits.bank_id := bankId - v.bits.bank_row := row.U - v.bits.bank_col := col.U - v - } - - def mvinSlot( - bankId: UInt, - addr: UInt, - iter: UInt, - stride: UInt - ): Valid[LoopSubCmd] = { - val v = Wire(Valid(new LoopSubCmd(b))) - v.valid := true.B - v.bits := 0.U.asTypeOf(new LoopSubCmd(b)) - v.bits.cmdType := LoopSubCmdType.MVIN - v.bits.bank_id := bankId - v.bits.dram_addr := addr - v.bits.stride := stride - v.bits.iter := iter - v - } - - def outRows(): UInt = 1.U - - def inMemStride(bytes: UInt): UInt = - (bytes / bankBytes.U).pad(19)(18, 0) - - def outMemStride(bytes: UInt): UInt = - (bytes / (bankBytes * 4).U).pad(19)(18, 0) - - def weightTileStride(): UInt = - cfg.out_channels * elemSize.U - - // Advance k-iterators (kch → kcol → krow), then output iterators (och → ocol → orow → batch) - def advanceIter(): Unit = { - when(kch_reg + DIM.U < cfg.in_channels) { - kch_reg := kch_reg + DIM.U - }.elsewhen(kcol_reg + 1.U < cfg.kernel_dim) { - kch_reg := 0.U; kcol_reg := kcol_reg + 1.U - }.elsewhen(krow_reg + 1.U < cfg.kernel_dim) { - kch_reg := 0.U; kcol_reg := 0.U; krow_reg := krow_reg + 1.U - }.elsewhen(och_reg + DIM.U < cfg.out_channels) { - kch_reg := 0.U; kcol_reg := 0.U; krow_reg := 0.U; - och_reg := och_reg + DIM.U - }.elsewhen(ocol_reg + 1.U < cfg.out_dim) { - kch_reg := 0.U; kcol_reg := 0.U; krow_reg := 0.U; och_reg := 0.U - ocol_reg := ocol_reg + 1.U - }.elsewhen(orow_reg + 1.U < cfg.out_dim) { - kch_reg := 0.U; kcol_reg := 0.U; krow_reg := 0.U; och_reg := 0.U; - ocol_reg := 0.U - orow_reg := orow_reg + 1.U - }.elsewhen(batch_reg + 1.U < cfg.batch_size) { - kch_reg := 0.U; kcol_reg := 0.U; krow_reg := 0.U; och_reg := 0.U; - ocol_reg := 0.U; orow_reg := 0.U - batch_reg := batch_reg + 1.U - }.otherwise { - state := sFree // All iterations done - } - } - - // Track whether all iterations are done (used to decide next state) - val allDone = (batch_reg === cfg.batch_size - 1.U) && - (orow_reg === cfg.out_dim - 1.U) && (ocol_reg === cfg.out_dim - 1.U) && - (och_reg + DIM.U >= cfg.out_channels) && isLastK - - switch(state) { - is(sIdle) { - io.start.ready := true.B - when(io.start.fire) { - cfg := io.start.bits - batch_reg := 0.U; orow_reg := 0.U; ocol_reg := 0.U; och_reg := 0.U - krow_reg := 0.U; kcol_reg := 0.U; kch_reg := 0.U - state := sAlloc - } - } - - // [MSET_alloc(input), MSET_alloc(weight), MSET_alloc(output), --] - is(sAlloc) { - io.cmd.valid := true.B - io.cmd.bits - .slots(0) := msetSlot(cfg.bank_input, alloc = true, row = 1, col = 1) - io.cmd.bits - .slots(1) := msetSlot(cfg.bank_weight, alloc = true, row = 1, col = 1) - io.cmd.bits - .slots(2) := msetSlot(cfg.bank_output, alloc = true, row = 1, col = 4) - io.cmd.bits.slots(3) := emptySlot() - when(io.cmd.fire) { - state := sMvinInput - } - } - - // [MVIN_input, MVIN_weight, --, --] - is(sMvinInput) { - io.cmd.valid := true.B - val inputSlot = Mux( - addrGen.io.isPadding, - emptySlot(), // Skip MVIN if padding (load zeros implicitly) - mvinSlot( - cfg.bank_input, - addrGen.io.inputAddr, - outRows(), - inMemStride(cfg.input_stride) - ) - ) - io.cmd.bits.slots(0) := inputSlot - io.cmd.bits.slots(1) := mvinSlot( - cfg.bank_weight, - addrGen.io.weightAddr, - DIM.U, - inMemStride(weightTileStride()) - ) - io.cmd.bits.slots(2) := emptySlot() - io.cmd.bits.slots(3) := emptySlot() - when(io.cmd.fire) { - state := sCompute - } - } - - // [PRELOAD, COMPUTE, --, --] - is(sCompute) { - io.cmd.valid := true.B - val preSlot = Wire(Valid(new LoopSubCmd(b))) - preSlot.valid := true.B - preSlot.bits := 0.U.asTypeOf(new LoopSubCmd(b)) - preSlot.bits.cmdType := LoopSubCmdType.PRELOAD - preSlot.bits.op1_bank := cfg.bank_weight - preSlot.bits.wr_bank := cfg.bank_output - preSlot.bits.iter := DIM.U - - val compSlot = Wire(Valid(new LoopSubCmd(b))) - compSlot.valid := true.B - compSlot.bits := 0.U.asTypeOf(new LoopSubCmd(b)) - compSlot.bits.cmdType := LoopSubCmdType.COMPUTE - compSlot.bits.op1_bank := cfg.bank_input - compSlot.bits.op2_bank := cfg.bank_weight - compSlot.bits.wr_bank := cfg.bank_output - compSlot.bits.compute_mode := Mux( - isFirstK, - 0.U, - 1.U - ) // PRELOADED first, then ACCUMULATED - compSlot.bits.iter := DIM.U - compSlot.bits.zero_op2 := cfg.no_bias - compSlot.bits.zero_op1_tail := true.B - - io.cmd.bits.slots(0) := preSlot - io.cmd.bits.slots(1) := compSlot - io.cmd.bits.slots(2) := emptySlot() - io.cmd.bits.slots(3) := emptySlot() - - when(io.cmd.fire) { - when(isLastK) { - state := sMvout // Need to output results after last k iteration - }.otherwise { - advanceIter() - when(state =/= sFree)(state := sMvinInput) - } - } - } - - // [MVOUT, --, --, --] - is(sMvout) { - io.cmd.valid := true.B - val mvSlot = Wire(Valid(new LoopSubCmd(b))) - mvSlot.valid := true.B - mvSlot.bits := 0.U.asTypeOf(new LoopSubCmd(b)) - mvSlot.bits.cmdType := LoopSubCmdType.MVOUT - mvSlot.bits.bank_id := cfg.bank_output - mvSlot.bits.dram_addr := addrGen.io.outputAddr - mvSlot.bits.stride := outMemStride(cfg.output_stride) - mvSlot.bits.iter := outRows() - - io.cmd.bits.slots(0) := mvSlot - io.cmd.bits.slots(1) := emptySlot() - io.cmd.bits.slots(2) := emptySlot() - io.cmd.bits.slots(3) := emptySlot() - - when(io.cmd.fire) { - when(allDone) { - state := sFree - }.otherwise { - advanceIter() - when(state =/= sFree)(state := sMvinInput) - } - } - } - - // [MSET_free(input), MSET_free(weight), MSET_free(output), --] - is(sFree) { - io.cmd.valid := true.B - io.cmd.bits - .slots(0) := msetSlot(cfg.bank_input, alloc = false, row = 0, col = 0) - io.cmd.bits - .slots(1) := msetSlot(cfg.bank_weight, alloc = false, row = 0, col = 0) - io.cmd.bits - .slots(2) := msetSlot(cfg.bank_output, alloc = false, row = 0, col = 0) - io.cmd.bits.slots(3) := emptySlot() - when(io.cmd.fire) { - state := sDone - } - } - - is(sDone) { - state := sIdle - } - } -} diff --git a/examples/balls/gemmini/arch/src/main/scala/LoopMatmulUnroller.scala b/examples/balls/gemmini/arch/src/main/scala/LoopMatmulUnroller.scala deleted file mode 100644 index 14055ff0f..000000000 --- a/examples/balls/gemmini/arch/src/main/scala/LoopMatmulUnroller.scala +++ /dev/null @@ -1,321 +0,0 @@ -package examples.balls.gemmini - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.top.GlobalConfig -import examples.balls.gemmini.configs.GemminiBallParam - -/** - * LoopMatmulUnroller — expands LOOP_WS into a stream of LoopCmd rows. - * - * FSM: sIdle → sAllocBanks → sPrimeLoad → sMainLoop (Row1/Row2) → sDrainLast → sFreeBanks → sDone - * - * Row layout per iteration: - * Row 1: [PRELOAD(cur), COMPUTE(cur)] - * Row 2: [MVOUT(cur), MVIN_A(next), MVIN_B(next)] - * Last iteration: only Row 1, then sDrainLast handles final MVOUT. - */ -@instantiable -class LoopMatmulUnroller(val b: GlobalConfig) extends Module { - val config = GemminiBallParam(b) - val DIM = config.blockSize - val elemSize = config.inputWidth / 8 - val accBytes = config.accWidth / 8 - val bankBytes = b.memDomain.bankWidth / 8 - - @public - val io = IO(new Bundle { - val start = Flipped(Decoupled(new LoopWsConfig(b))) - val cmd = Decoupled(new LoopCmd(b)) - val busy = Output(Bool()) - }) - - // FSM states - val sIdle :: sAllocBanks :: sPrimeLoad :: sMainRow1 :: sMainRow2 :: sDrainLast :: sFreeBanks :: sDone :: Nil = - Enum(8) - val state = RegInit(sIdle) - - // Config registers (latched at start) - val cfg = Reg(new LoopWsConfig(b)) - - // Loop iterator registers - val i_reg = RegInit(0.U(16.W)) - val j_reg = RegInit(0.U(16.W)) - val k_reg = RegInit(0.U(16.W)) - - // Total iterations and current iteration index - val totalIter = cfg.max_i * cfg.max_j * cfg.max_k - val curIter = RegInit(0.U(32.W)) - - // Whether this is the last iteration - val isLastIter = curIter === (totalIter - 1.U) - - // Defaults - io.start.ready := state === sIdle - io.cmd.valid := false.B - io.cmd.bits := 0.U.asTypeOf(new LoopCmd(b)) - io.busy := state =/= sIdle - - // Helper to build an invalid LoopSubCmd slot - def emptySlot(): Valid[LoopSubCmd] = { - val v = Wire(Valid(new LoopSubCmd(b))) - v.valid := false.B - v.bits := 0.U.asTypeOf(new LoopSubCmd(b)) - v - } - - // Helper to build an MSET slot - def msetSlot( - bankId: UInt, - alloc: Boolean, - row: Int, - col: Int - ): Valid[LoopSubCmd] = { - val v = Wire(Valid(new LoopSubCmd(b))) - v.valid := true.B - v.bits := 0.U.asTypeOf(new LoopSubCmd(b)) - v.bits.cmdType := (if (alloc) LoopSubCmdType.MSET_ALLOC - else LoopSubCmdType.MSET_FREE) - v.bits.bank_id := bankId - v.bits.bank_row := row.U - v.bits.bank_col := col.U - v - } - - // Helper to build an MVIN slot - def mvinSlot( - bankId: UInt, - addr: UInt, - iter: UInt, - stride: UInt - ): Valid[LoopSubCmd] = { - val v = Wire(Valid(new LoopSubCmd(b))) - v.valid := true.B - v.bits := 0.U.asTypeOf(new LoopSubCmd(b)) - v.bits.cmdType := LoopSubCmdType.MVIN - v.bits.bank_id := bankId - v.bits.dram_addr := addr - v.bits.stride := stride - v.bits.iter := iter - v - } - - // Helper to build an MVOUT slot - def mvoutSlot( - bankId: UInt, - addr: UInt, - iter: UInt, - stride: UInt - ): Valid[LoopSubCmd] = { - val v = Wire(Valid(new LoopSubCmd(b))) - v.valid := true.B - v.bits := 0.U.asTypeOf(new LoopSubCmd(b)) - v.bits.cmdType := LoopSubCmdType.MVOUT - v.bits.bank_id := bankId - v.bits.dram_addr := addr - v.bits.stride := stride - v.bits.iter := iter - v - } - - // Address computation - // addr_a(i,k) = dram_addr_a + i * stride_a + k * DIM * elemSize - // addr_b(k,j) = dram_addr_b + k * stride_b + j * DIM * elemSize - // addr_c(i,j) = dram_addr_c + i * stride_c + j * DIM * accBytes - def addrA(i: UInt, k: UInt): UInt = - cfg.dram_addr_a + i * cfg.stride_a + k * (DIM * elemSize).U - - def addrB(k: UInt, j: UInt): UInt = - cfg.dram_addr_b + k * cfg.stride_b + j * (DIM * elemSize).U - - def addrC(i: UInt, j: UInt): UInt = - cfg.dram_addr_c + i * cfg.stride_c + j * (DIM * accBytes).U - - def inMemStride(bytes: UInt): UInt = - (bytes / bankBytes.U)(18, 0) - - def outMemStride(bytes: UInt): UInt = - (bytes / (bankBytes * 4).U)(18, 0) - - // Next iterator values (advance k, then j, then i) - val next_k = Wire(UInt(16.W)) - val next_j = Wire(UInt(16.W)) - val next_i = Wire(UInt(16.W)) - - when(k_reg + 1.U < cfg.max_k) { - next_k := k_reg + 1.U - next_j := j_reg - next_i := i_reg - }.elsewhen(j_reg + 1.U < cfg.max_j) { - next_k := 0.U - next_j := j_reg + 1.U - next_i := i_reg - }.otherwise { - next_k := 0.U - next_j := 0.U - next_i := i_reg + 1.U - } - - // Compute mode: first k iteration uses PRELOADED (0), rest use ACCUMULATED (1) - val computeMode = Mux(k_reg === 0.U, 0.U(2.W), 1.U(2.W)) - - switch(state) { - is(sIdle) { - io.start.ready := true.B - when(io.start.fire) { - cfg := io.start.bits - i_reg := 0.U - j_reg := 0.U - k_reg := 0.U - curIter := 0.U - state := sAllocBanks - } - } - - // Row 0: [MSET_alloc(A), MSET_alloc(B), MSET_alloc(C), --] - is(sAllocBanks) { - io.cmd.valid := true.B - io.cmd.bits - .slots(0) := msetSlot(cfg.bank_a, alloc = true, row = 1, col = 1) - io.cmd.bits - .slots(1) := msetSlot(cfg.bank_b, alloc = true, row = 1, col = 1) - io.cmd.bits.slots(2) := msetSlot( - cfg.bank_c, - alloc = true, - row = 1, - col = 4 - ) // accWidth = 4x inputWidth - io.cmd.bits.slots(3) := emptySlot() - when(io.cmd.fire) { - state := sPrimeLoad - } - } - - // Row 1: [MVIN_A(0), MVIN_B(0), --, --] - is(sPrimeLoad) { - io.cmd.valid := true.B - io.cmd.bits.slots(0) := mvinSlot( - cfg.bank_a, - addrA(0.U, 0.U), - DIM.U, - inMemStride(cfg.stride_a) - ) - io.cmd.bits.slots(1) := mvinSlot( - cfg.bank_b, - addrB(0.U, 0.U), - DIM.U, - inMemStride(cfg.stride_b) - ) - io.cmd.bits.slots(2) := emptySlot() - io.cmd.bits.slots(3) := emptySlot() - when(io.cmd.fire) { - state := sMainRow1 - } - } - - // Main loop Row 1: [PRELOAD(cur), COMPUTE(cur), --, --] - is(sMainRow1) { - io.cmd.valid := true.B - val preSlot = Wire(Valid(new LoopSubCmd(b))) - preSlot.valid := true.B - preSlot.bits := 0.U.asTypeOf(new LoopSubCmd(b)) - preSlot.bits.cmdType := LoopSubCmdType.PRELOAD - preSlot.bits.op1_bank := cfg.bank_a // preload A (activations) into matrix array - preSlot.bits.wr_bank := cfg.bank_c - preSlot.bits.iter := DIM.U - - val compSlot = Wire(Valid(new LoopSubCmd(b))) - compSlot.valid := true.B - compSlot.bits := 0.U.asTypeOf(new LoopSubCmd(b)) - compSlot.bits.cmdType := LoopSubCmdType.COMPUTE - compSlot.bits.op1_bank := cfg.bank_a - compSlot.bits.op2_bank := cfg.bank_b - compSlot.bits.wr_bank := cfg.bank_c - compSlot.bits.compute_mode := computeMode - compSlot.bits.iter := DIM.U - - io.cmd.bits.slots(0) := preSlot - io.cmd.bits.slots(1) := compSlot - io.cmd.bits.slots(2) := emptySlot() - io.cmd.bits.slots(3) := emptySlot() - - when(io.cmd.fire) { - when(isLastIter) { - state := sDrainLast // Skip Row 2 for last iteration - }.otherwise { - state := sMainRow2 - } - } - } - - // Main loop Row 2: [MVOUT(cur), MVIN_A(next), MVIN_B(next), --] - is(sMainRow2) { - io.cmd.valid := true.B - io.cmd.bits.slots(0) := mvoutSlot( - cfg.bank_c, - addrC(i_reg, j_reg), - DIM.U, - outMemStride(cfg.stride_c) - ) - io.cmd.bits.slots(1) := mvinSlot( - cfg.bank_a, - addrA(next_i, next_k), - DIM.U, - inMemStride(cfg.stride_a) - ) - io.cmd.bits.slots(2) := mvinSlot( - cfg.bank_b, - addrB(next_k, next_j), - DIM.U, - inMemStride(cfg.stride_b) - ) - io.cmd.bits.slots(3) := emptySlot() - - when(io.cmd.fire) { - // Advance iterators - i_reg := next_i - j_reg := next_j - k_reg := next_k - curIter := curIter + 1.U - state := sMainRow1 - } - } - - // Drain: emit final MVOUT for last iteration - is(sDrainLast) { - io.cmd.valid := true.B - io.cmd.bits.slots(0) := mvoutSlot( - cfg.bank_c, - addrC(i_reg, j_reg), - DIM.U, - outMemStride(cfg.stride_c) - ) - io.cmd.bits.slots(1) := emptySlot() - io.cmd.bits.slots(2) := emptySlot() - io.cmd.bits.slots(3) := emptySlot() - when(io.cmd.fire) { - state := sFreeBanks - } - } - - // Free: [MSET_free(A), MSET_free(B), MSET_free(C), --] - is(sFreeBanks) { - io.cmd.valid := true.B - io.cmd.bits - .slots(0) := msetSlot(cfg.bank_a, alloc = false, row = 0, col = 0) - io.cmd.bits - .slots(1) := msetSlot(cfg.bank_b, alloc = false, row = 0, col = 0) - io.cmd.bits - .slots(2) := msetSlot(cfg.bank_c, alloc = false, row = 0, col = 0) - io.cmd.bits.slots(3) := emptySlot() - when(io.cmd.fire) { - state := sDone - } - } - - is(sDone) { - state := sIdle - } - } -} diff --git a/examples/balls/gemmini/compiler/python/quant.py b/examples/balls/gemmini/compiler/python/quant.py deleted file mode 100644 index 298f42767..000000000 --- a/examples/balls/gemmini/compiler/python/quant.py +++ /dev/null @@ -1,197 +0,0 @@ -import numpy as np -import torch -from buddy.compiler.graph.operation import Op, OpType, Conv2dOp, AddMMOp, TOp, PermuteOp -from buddy.compiler.graph.type import TensorDType -from buddy.compiler.ops import tosa as ops -from buddy_mlir import ir -from buddy_mlir.dialects import tosa -from stack.compiler.quant.rax import QuantTensor, RaxQuantPackage, write_rax - - -class GemminiConvOp(Op): - def __init__(self): - super().__init__() - self._op_type = OpType.ReduceType - - -def lower(node, symbols): - input, weight, bias = [symbols[(name, 0)] for name in node.args] - shape = list(node.tensor_meta["shape"]) - linear = len(shape) == 2 - if linear: - m, k = ir.RankedTensorType(input.type).shape - input = tosa.ReshapeOp( - input, - ops._create_shape_operand([m, k, 1, 1]), - results=[ir.RankedTensorType.get([m, k, 1, 1], ir.F32Type.get())], - ).result - result_shape = [shape[0], shape[1], 1, 1] - else: - result_shape = shape - attributes = { - name: ir.IntegerAttr.get(ir.IntegerType.get_signless(64), value) - for name, value in zip(("kh", "kw", "stride", "padding"), node.geometry) - } - attributes.update( - { - name: ir.FloatAttr.get(ir.F32Type.get(), float(value)) - for name, value in zip(("inputScale", "weightScale"), node.scales) - } - ) - result = ir.Operation.create( - "buckyball.gemmini_conv", - operands=[input, weight, bias], - results=[ir.RankedTensorType.get(result_shape, ir.F32Type.get())], - attributes=attributes, - ).result - if linear: - result = tosa.ReshapeOp( - result, - ops._create_shape_operand(shape), - results=[ir.RankedTensorType.get(shape, ir.F32Type.get())], - ).result - return result - - -def apply( - graph, - params, - names, - output, - name, - calibration, - quantized, - packer, - *, - reorder, - pool_result, -): - parameters = list(graph.params) - inputs = [graph._body[i] for i in graph._inputs] - positions = {p.name: i for i, p in enumerate(parameters)} - originals = {p.name: list(p.tensor_meta["shape"]) for p in parameters} - removed, consumed = set(), set() - occurrences = {} - for node in list(graph._body): - if isinstance(node, Conv2dOp): - activation, weight, bias = node.args[:3] - if node.args[5] != [1, 1] or node.args[6] or node.args[8] != 1: - raise ValueError( - "Gemmini model convolution requires unit dilation and one group" - ) - if len(set(node.args[3])) != 1 or len(set(node.args[4])) != 1: - raise ValueError( - "Gemmini model convolution requires equal spatial strides/padding" - ) - geometry = (*originals[weight][2:], node.args[3][0], node.args[4][0]) - elif isinstance(node, AddMMOp): - bias, activation, transposed = node.args[:3] - transpose = graph.node_table[transposed] - if not ( - isinstance(transpose, TOp) - or isinstance(transpose, PermuteOp) - and list(transpose.args[1]) == [1, 0] - ): - raise ValueError( - "Gemmini linear requires a transposed weight parameter" - ) - weight = transpose._parents[0] - removed.add(transposed) - geometry = (1, 1, 1, 0) - else: - continue - index = positions[weight] - parameter_name = names[index] - codes, scale = quantized[parameter_name] - if weight not in consumed: - packed = reorder(codes) - params[index] = torch.from_numpy(packed.copy()) - parameters[index].tensor_meta.update( - shape=list(packed.shape), dtype=TensorDType.Int8 - ) - consumed.add(weight) - occurrence = occurrences.get(parameter_name, 0) - input_scale = calibration[parameter_name][occurrence][0] - occurrences[parameter_name] = occurrence + 1 - replacement = GemminiConvOp() - replacement._name = node.name - replacement._arguments = [activation, weight, bias] - replacement._parents = [activation, weight, bias] - replacement._children = list(node._children) - replacement._tensor_meta = dict(node.tensor_meta) - replacement.trace_meta = node.trace_meta - replacement.geometry = geometry - replacement.scales = (input_scale, scale) - graph._body[graph._body.index(node)] = replacement - graph.node_table[node.name] = replacement - if node.name not in parameters[index]._children: - parameters[index]._children.append(node.name) - for key in removed: - node = graph.node_table[key] - if any( - not isinstance(graph.node_table[child], GemminiConvOp) - for child in node._children - ): - raise ValueError("Weight transpose has another consumer") - graph.node_table[node._parents[0]]._children.remove(key) - graph._body.remove(node) - del graph.node_table[key] - if {names[positions[p]] for p in consumed} != set(quantized): - raise ValueError("Gemmini quantization decisions do not match graph weights") - graph._fake_params = [graph._body.index(p) for p in parameters] - graph._inputs = [graph._body.index(p) for p in inputs] - for group in graph.op_groups: - graph.op_groups[group] = [ - graph.node_table[n.name] - for n in graph.op_groups[group] - if n.name not in removed - ] - graph._ops_registry["GemminiConvOp"] = lower - graph._ops_registry["MaxPool2dOp"] = pool_result - tensors, weights, floats, scales = [], [], [], [] - wo = fo = so = 0 - for node, parameter, parameter_name in zip(parameters, params, names): - raw = parameter.detach().numpy().tobytes() - if node.name in consumed: - scale = np.asarray(quantized[parameter_name][1], dtype=np.float32).tobytes() - tensors.append( - QuantTensor( - parameter_name, - originals[node.name], - list(parameter.shape), - "i8", - [], - wo, - len(raw), - so, - len(scale), - ) - ) - weights.append(raw) - scales.append(scale) - wo += len(raw) - so += len(scale) - else: - tensors.append( - QuantTensor( - parameter_name, - originals[node.name], - originals[node.name], - "f32", - [], - fo, - len(raw), - 0, - 0, - ) - ) - floats.append(raw) - fo += len(raw) - package = RaxQuantPackage( - tensors, b"".join(weights), b"".join(floats), b"".join(scales), {} - ) - output.mkdir(parents=True, exist_ok=True) - write_rax(package, output / f"{name}.rax", packer, name) - (output / "weights.bin").write_bytes(package.weights) - (output / "params.f32").write_bytes(package.params_f32) - (output / "scales.bin").write_bytes(package.scales_f32) diff --git a/examples/balls/gemmini/compiler/runtime/conv.cpp b/examples/balls/gemmini/compiler/runtime/conv.cpp deleted file mode 100644 index a7c8cc751..000000000 --- a/examples/balls/gemmini/compiler/runtime/conv.cpp +++ /dev/null @@ -1,95 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - -extern "C" void _mlir_ciface_gemmini_conv(StridedMemRefType *input, - StridedMemRefType *weight, - StridedMemRefType *bias, - StridedMemRefType *output, - int64_t kh, int64_t kw, - int64_t stride, int64_t padding, - float inputScale, float weightScale) { - const int64_t batches = input->sizes[0], channels = input->sizes[1], - height = input->sizes[2], width = input->sizes[3]; - const int64_t oh = output->sizes[2], ow = output->sizes[3], - n = weight->sizes[1], k = channels * kh * kw, - m = batches * oh * ow; - if (k != weight->sizes[0] || n != bias->sizes[0] || - output->sizes[0] != batches || output->sizes[1] != n || - oh != (height + 2 * padding - kh) / stride + 1 || - ow != (width + 2 * padding - kw) / stride + 1 || - k > INT32_MAX / (128 * 128) || !std::isfinite(inputScale) || - inputScale <= 0 || !std::isfinite(weightScale) || weightScale <= 0) { - fputs("invalid Gemmini convolution shape/scales\n", stderr); - abort(); - } - int8_t a[256] __attribute__((aligned(64))), - b[256] __attribute__((aligned(64))); - int32_t c[256] __attribute__((aligned(64))); - bb_mem_alloc(0, 1, 1); - bb_mem_alloc(1, 1, 1); - bb_mem_alloc(2, 1, 1); - bb_mem_alloc(3, 1, 4); - bb_mset_clear(2, 1, 1); - bb_gemmini_config(1, 0, 0, 0, 0); - for (int64_t row = 0; row < m; row += 16) - for (int64_t col = 0; col < n; col += 16) { - for (int64_t inner = 0; inner < k; inner += 16) { - for (int i = 0; i < 16; ++i) - for (int j = 0; j < 16; ++j) { - int64_t index = row + i, channel = (inner + j) % channels, - ky = (inner + j) / (kw * channels), - kx = ((inner + j) / channels) % kw; - int64_t batch = index / (oh * ow), - y = (index / ow) % oh * stride + ky - padding, - x = index % ow * stride + kx - padding; - float value = 0; - if (index < m && inner + j < k && y >= 0 && y < height && x >= 0 && - x < width) - value = - input->data[input->offset + batch * input->strides[0] + - channel * input->strides[1] + - y * input->strides[2] + x * input->strides[3]]; - if (!std::isfinite(value)) { - fputs("non-finite Gemmini activation\n", stderr); - abort(); - } - a[i * 16 + j] = int8_t( - std::clamp(std::nearbyint(value / inputScale), -128.f, 127.f)); - b[i * 16 + j] = - inner + i < k && col + j < n - ? weight->data[weight->offset + - (inner + i) * weight->strides[0] + - (col + j) * weight->strides[1]] - : 0; - } - bb_mvin((uintptr_t)a, 1, 16, 1); - bb_mvin((uintptr_t)b, 0, 16, 1); - if (inner == 0) { - bb_gemmini_preload(0, 3, 16, 0, 0); - bb_gemmini_compute_preloaded(1, 2, 3, 16, 0, 0, 0); - } else - bb_gemmini_compute_accumulated(1, 0, 3, 16, 0, 0, 0); - } - bb_mvout((uintptr_t)c, 3, 16, 1); - for (int i = 0; i < 16 && row + i < m; ++i) - for (int j = 0; j < 16 && col + j < n; ++j) { - int64_t index = row + i, batch = index / (oh * ow), - y = (index / ow) % oh, x = index % ow; - output->data[output->offset + batch * output->strides[0] + - (col + j) * output->strides[1] + y * output->strides[2] + - x * output->strides[3]] = - float(c[i * 16 + j]) * (inputScale * weightScale) + - bias->data[bias->offset + (col + j) * bias->strides[0]]; - } - } - bb_mem_release(0); - bb_mem_release(1); - bb_mem_release(2); - bb_mem_release(3); -} diff --git a/examples/balls/gemmini/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp b/examples/balls/gemmini/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp deleted file mode 100644 index 4a8b83091..000000000 --- a/examples/balls/gemmini/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp +++ /dev/null @@ -1,92 +0,0 @@ -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace mlir::buddy { -void populateGemminiBallAssignPhysicalBankPatterns(RewritePatternSet &patterns, - PhysicalBankState &state); -} - -namespace { -Value shift(PatternRewriter &rewriter, Location loc, Value value, - int64_t bits) { - return rewriter.create( - loc, value, rewriter.create(loc, bits, 64)); -} - -Value rs1(PatternRewriter &rewriter, Location loc, Value a, Value b, Value c, - Value iter) { - Value ab = rewriter.create(loc, a, shift(rewriter, loc, b, 10)); - Value ci = rewriter.create(loc, shift(rewriter, loc, c, 20), - shift(rewriter, loc, iter, 30)); - return rewriter.create(loc, ab, ci); -} - -Value rs2(PatternRewriter &rewriter, Location loc, Value a, Value b, Value c, - int64_t subcommand) { - Value packed = rewriter.create(loc, subcommand, 64); - packed = - rewriter.create(loc, packed, shift(rewriter, loc, a, 6)); - packed = - rewriter.create(loc, packed, shift(rewriter, loc, b, 16)); - return rewriter.create(loc, packed, - shift(rewriter, loc, c, 26)); -} - -class BankGemminiPreloadPattern - : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(BankGemminiPreloadOp op, - PatternRewriter &rewriter) const override { - Value zero = rewriter.create(op.getLoc(), 0, 64); - rewriter.create( - op.getLoc(), rewriter.getStringAttr("GEMMINI_PRELOAD"), - rs1(rewriter, op.getLoc(), op.getInBank(), zero, op.getOutBank(), - op.getIter()), - rs2(rewriter, op.getLoc(), op.getInBase(), zero, op.getOutBase(), 1)); - rewriter.replaceOp(op, op.getOutBank()); - return success(); - } -}; - -template -class BankGemminiComputePattern : public OpRewritePattern { -public: - BankGemminiComputePattern(MLIRContext *context, StringRef mnemonic, - int64_t subcommand) - : OpRewritePattern(context), mnemonic(mnemonic), - subcommand(subcommand) {} - LogicalResult matchAndRewrite(Op op, - PatternRewriter &rewriter) const override { - rewriter.create( - op.getLoc(), rewriter.getStringAttr(mnemonic), - rs1(rewriter, op.getLoc(), op.getABank(), op.getBBank(), - op.getOutBank(), op.getIter()), - rs2(rewriter, op.getLoc(), op.getABase(), op.getBBase(), - op.getOutBase(), subcommand)); - rewriter.replaceOp(op, op.getOutBank()); - return success(); - } - -private: - StringRef mnemonic; - int64_t subcommand; -}; -} // namespace - -void mlir::buddy::populateGemminiBallAssignPhysicalBankPatterns( - RewritePatternSet &patterns, mlir::buddy::PhysicalBankState &state) { - (void)state; - patterns.add(patterns.getContext()); - patterns.add>( - patterns.getContext(), "GEMMINI_COMPUTE_PRELOADED", 2); - patterns.add>( - patterns.getContext(), "GEMMINI_COMPUTE_ACCUMULATED", 3); -} diff --git a/examples/balls/gemmini/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp b/examples/balls/gemmini/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp deleted file mode 100644 index b41ec08c8..000000000 --- a/examples/balls/gemmini/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp +++ /dev/null @@ -1,72 +0,0 @@ -#include "Buckyball/BuckyballOps.h" -#include "Target/BuckyballTargetRegistry.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/IR/PatternMatch.h" -using namespace mlir; -using namespace buddy::buckyball; -namespace { -struct ConvLowering : OpRewritePattern { - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(GemminiMemConvOp op, - PatternRewriter &b) const override { - buckyball_target::requireBuckyballBall("GemminiBall"); - auto input = cast(op.getInput().getType()), - weight = cast(op.getWeight().getType()), - bias = cast(op.getBias().getType()), - output = cast(op.getOutput().getType()); - if (input.getRank() != 4 || weight.getRank() != 2 || bias.getRank() != 1 || - output.getRank() != 4 || !input.getElementType().isF32() || - !weight.getElementType().isInteger(8) || - !bias.getElementType().isF32() || !output.getElementType().isF32()) - return op.emitError("Gemmini model ABI expects F32 NCHW, I8 matrix " - "weights and F32 bias/output"); - auto dynamic = ShapedType::kDynamic; - auto matrix = [&](int rank, Type element) { - return MemRefType::get( - SmallVector(rank, dynamic), element, - StridedLayoutAttr::get(b.getContext(), dynamic, - SmallVector(rank, dynamic))); - }; - auto a = matrix(4, b.getF32Type()), w = matrix(2, b.getI8Type()), - d = matrix(1, b.getF32Type()); - auto module = op->getParentOfType(); - auto loc = op.getLoc(); - if (!module.lookupSymbol("gemmini_conv")) { - OpBuilder::InsertionGuard guard(b); - b.setInsertionPointToStart(module.getBody()); - auto fn = b.create( - loc, "gemmini_conv", - b.getFunctionType({a, w, d, a, b.getI64Type(), b.getI64Type(), - b.getI64Type(), b.getI64Type(), b.getF32Type(), - b.getF32Type()}, - {})); - fn.setPrivate(); - fn->setAttr("llvm.emit_c_interface", b.getUnitAttr()); - } - b.create( - loc, "gemmini_conv", TypeRange{}, - ValueRange{b.create(loc, a, op.getInput()), - b.create(loc, w, op.getWeight()), - b.create(loc, d, op.getBias()), - b.create(loc, a, op.getOutput()), - b.create(loc, op.getKh(), 64), - b.create(loc, op.getKw(), 64), - b.create(loc, op.getStride(), 64), - b.create(loc, op.getPadding(), 64), - b.create(loc, b.getF32Type(), - op.getInputScaleAttr()), - b.create(loc, b.getF32Type(), - op.getWeightScaleAttr())}); - b.eraseOp(op); - return success(); - } -}; -} // namespace -namespace mlir::buddy { -void populateGemminiBallLowerBuckyballToBankSSAPatterns( - RewritePatternSet &patterns) { - patterns.add(patterns.getContext()); -} -} // namespace mlir::buddy diff --git a/examples/balls/gemmini/compiler/src/Dialect/Buckyball/GemminiBall.td b/examples/balls/gemmini/compiler/src/Dialect/Buckyball/GemminiBall.td deleted file mode 100644 index 944024194..000000000 --- a/examples/balls/gemmini/compiler/src/Dialect/Buckyball/GemminiBall.td +++ /dev/null @@ -1,66 +0,0 @@ -//===-- GemminiBall.td - Gemmini Ball operations ------------------------===// - -#ifndef GEMMINI_BALL_OPS -#define GEMMINI_BALL_OPS - -include "Buckyball.td" -include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.td" - -def GemminiInstructionOp : Buckyball_Op<"gemmini"> { - let summary = "Gemmini command encoded for the target ISA"; - let description = [{ - Emits one Gemmini command. mnemonic must name an ISA entry enabled by the - selected target, and rs1/rs2 are the exact command operands defined by - that entry. This covers GEMMINI_CONFIG, GEMMINI_PRELOAD, - GEMMINI_COMPUTE_PRELOADED, GEMMINI_COMPUTE_ACCUMULATED, GEMMINI_LOOP_WS, - and GEMMINI_LOOP_CONV_WS without hard-coding funct7 values in the IR. - }]; - let arguments = (ins StrAttr:$mnemonic, I64:$rs1, I64:$rs2); - let assemblyFormat = "$mnemonic $rs1 `,` $rs2 prop-dict attr-dict `:` type($rs1)"; -} - -def BankGemminiPreloadOp : Buckyball_Op<"bank_gemmini_preload"> { - let summary = "BankSSA Gemmini preload"; - let arguments = (ins I64:$inBank, I64:$outBank, I64:$iter, I64:$inBase, - I64:$outBase); - let results = (outs I64:$outBankOut); - let assemblyFormat = "$inBank $outBank $iter $inBase $outBase prop-dict attr-dict `:` " - "type($inBank) type($outBank) type($iter) type($inBase) type($outBase)"; -} - -def BankGemminiComputePreloadedOp : Buckyball_Op<"bank_gemmini_compute_preloaded"> { - let summary = "BankSSA Gemmini compute using preloaded state"; - let arguments = (ins I64:$aBank, I64:$bBank, I64:$outBank, I64:$iter, - I64:$aBase, I64:$bBase, I64:$outBase); - let results = (outs I64:$outBankOut); - let assemblyFormat = "$aBank $bBank $outBank $iter $aBase $bBase $outBase prop-dict attr-dict `:` " - "type($aBank) type($bBank) type($outBank) type($iter) type($aBase) type($bBase) type($outBase)"; -} - -def BankGemminiComputeAccumulatedOp : Buckyball_Op<"bank_gemmini_compute_accumulated"> { - let summary = "BankSSA Gemmini compute accumulating the output bank"; - let arguments = (ins I64:$aBank, I64:$bBank, I64:$outBank, I64:$iter, - I64:$aBase, I64:$bBase, I64:$outBase); - let results = (outs I64:$outBankOut); - let assemblyFormat = "$aBank $bBank $outBank $iter $aBase $bBase $outBase prop-dict attr-dict `:` " - "type($aBank) type($bBank) type($outBank) type($iter) type($aBase) type($bBase) type($outBase)"; -} - -def GemminiConvOp : Buckyball_Op<"gemmini_conv", [ - DeclareOpInterfaceMethods]> { - let summary = "NCHW convolution with INT8 HWIO matrix weights and scalar quantization scales"; - let arguments = (ins AnyRankedTensor:$input, AnyRankedTensor:$weight, - AnyRankedTensor:$bias, I64Attr:$kh, I64Attr:$kw, I64Attr:$stride, - I64Attr:$padding, F32Attr:$inputScale, F32Attr:$weightScale); - let results = (outs AnyRankedTensor:$output); - let assemblyFormat = "$input `,` $weight `,` $bias prop-dict attr-dict `:` type($input) `,` type($weight) `,` type($bias) `->` type($output)"; -} -def GemminiMemConvOp : Buckyball_Op<"gemmini_mem_conv"> { - let arguments = (ins AnyMemRef:$input, AnyMemRef:$weight, AnyMemRef:$bias, - AnyMemRef:$output, I64Attr:$kh, I64Attr:$kw, I64Attr:$stride, - I64Attr:$padding, F32Attr:$inputScale, F32Attr:$weightScale); - let assemblyFormat = "$input `,` $weight `,` $bias `,` $output prop-dict attr-dict `:` type($input) `,` type($weight) `,` type($bias) `,` type($output)"; -} - -#endif // GEMMINI_BALL_OPS diff --git a/examples/balls/gemmini/compiler/src/Dialect/Buckyball/IR/GemminiConv.cpp b/examples/balls/gemmini/compiler/src/Dialect/Buckyball/IR/GemminiConv.cpp deleted file mode 100644 index e91d3ef9f..000000000 --- a/examples/balls/gemmini/compiler/src/Dialect/Buckyball/IR/GemminiConv.cpp +++ /dev/null @@ -1,37 +0,0 @@ -#include "Buckyball/BuckyballOps.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -using namespace mlir; -using namespace mlir::bufferization; -using namespace buddy::buckyball; -bool GemminiConvOp::bufferizesToAllocation(Value) { return true; } -bool GemminiConvOp::bufferizesToMemoryRead(OpOperand &, const AnalysisState &) { - return true; -} -bool GemminiConvOp::bufferizesToMemoryWrite(OpOperand &, - const AnalysisState &) { - return false; -} -AliasingValueList GemminiConvOp::getAliasingValues(OpOperand &, - const AnalysisState &) { - return {}; -} -LogicalResult GemminiConvOp::bufferize(RewriterBase &b, - const BufferizationOptions &options, - BufferizationState &state) { - auto input = getBuffer(b, getInput(), options, state), - weight = getBuffer(b, getWeight(), options, state), - bias = getBuffer(b, getBias(), options, state); - if (failed(input) || failed(weight) || failed(bias)) - return failure(); - auto type = cast(getOutput().getType()); - if (!type.hasStaticShape()) - return emitError("Gemmini requires static model shapes"); - Value output = b.create( - getLoc(), MemRefType::get(type.getShape(), type.getElementType())); - b.create(getLoc(), *input, *weight, *bias, output, - getKhAttr(), getKwAttr(), getStrideAttr(), - getPaddingAttr(), getInputScaleAttr(), - getWeightScaleAttr()); - replaceOpWithBufferizedValues(b, getOperation(), output); - return success(); -} diff --git a/examples/balls/gemmini/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/balls/gemmini/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 15acde3d4..000000000 --- a/examples/balls/gemmini/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,46 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace buddy::buckyball; - -namespace { -class GemminiInstructionLowering - : public ConvertOpToLLVMPattern { -public: - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - - LogicalResult - matchAndRewrite(GemminiInstructionOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("GemminiBall"); - rewriter.replaceOpWithNewOp( - op, adaptor.getRs1(), adaptor.getRs2(), - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7(op.getMnemonic()))); - return success(); - } -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateGemminiBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth) { - (void)stable; - (void)bankDepth; - - patterns.add(converter); -} - -void configureGemminiBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable) { - (void)stable; - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/examples/balls/gemmini/configs/GemminiBallParam.scala b/examples/balls/gemmini/configs/GemminiBallParam.scala deleted file mode 100644 index e13343ecd..000000000 --- a/examples/balls/gemmini/configs/GemminiBallParam.scala +++ /dev/null @@ -1,40 +0,0 @@ -package examples.balls.gemmini.configs - -import framework.balldomain.configs.BallParamLoader -import framework.top.GlobalConfig - -case class GemminiBallParam( - meshRows: Int, - meshColumns: Int, - tileRows: Int, - tileColumns: Int, - inputWidth: Int, - accWidth: Int, - spatialOutputWidth: Int, - tileLatency: Int, - outputDelay: Int) { - - val totalRows: Int = meshRows * tileRows - val totalColumns: Int = meshColumns * tileColumns - val blockSize: Int = totalRows -} - -object GemminiBallParam { - private val ballName = "GemminiBall" - - def apply(b: GlobalConfig): GemminiBallParam = { - val tbl = BallParamLoader.ballTable(b, ballName) - GemminiBallParam( - meshRows = BallParamLoader.int(tbl, "meshRows"), - meshColumns = BallParamLoader.int(tbl, "meshColumns"), - tileRows = BallParamLoader.int(tbl, "tileRows"), - tileColumns = BallParamLoader.int(tbl, "tileColumns"), - inputWidth = BallParamLoader.int(tbl, "inputWidth"), - accWidth = BallParamLoader.int(tbl, "accWidth"), - spatialOutputWidth = BallParamLoader.int(tbl, "spatialOutputWidth"), - tileLatency = BallParamLoader.int(tbl, "tileLatency"), - outputDelay = BallParamLoader.int(tbl, "outputDelay") - ) - } - -} diff --git a/examples/balls/gemmini/configs/default.toml b/examples/balls/gemmini/configs/default.toml deleted file mode 100644 index 5958be6b8..000000000 --- a/examples/balls/gemmini/configs/default.toml +++ /dev/null @@ -1,12 +0,0 @@ -# Gemmini Ball Configuration - -[ball] -meshRows = 16 -meshColumns = 16 -tileRows = 1 -tileColumns = 1 -inputWidth = 8 -accWidth = 32 -spatialOutputWidth = 20 -tileLatency = 0 -outputDelay = 1 diff --git a/examples/balls/gemmini/emu/src/02_gemmini_config.rs b/examples/balls/gemmini/emu/src/02_gemmini_config.rs deleted file mode 100644 index 1cd2b5e05..000000000 --- a/examples/balls/gemmini/emu/src/02_gemmini_config.rs +++ /dev/null @@ -1,21 +0,0 @@ -//===- 02_gemmini_config.rs - GEMMINI_CONFIG instruction -------------------===// - -use super::gemmini_state::gemini; -use super::instruction::{BallInstruction, ExecContext}; - -pub struct GemminiConfig; - -impl BallInstruction for GemminiConfig { - fn exec(_xs1: u64, xs2: u64, _ctx: &mut ExecContext) -> u64 { - let mut g = gemini().lock().unwrap(); - g.cfg.dataflow = ((xs2 >> 4) & 1) as u8; - g.cfg.a_transpose = ((xs2 >> 7) & 1) != 0; - g.cfg.b_transpose = ((xs2 >> 8) & 1) != 0; - g.cfg.in_shift = ((xs2 >> 9) & 0xFFFFFFFF) as u32; - 0 - } - - fn latency(_xs1: u64, _xs2: u64) -> u64 { - 1 - } -} diff --git a/examples/balls/gemmini/emu/src/03_gemmini_flush.rs b/examples/balls/gemmini/emu/src/03_gemmini_flush.rs deleted file mode 100644 index 91333d08f..000000000 --- a/examples/balls/gemmini/emu/src/03_gemmini_flush.rs +++ /dev/null @@ -1,15 +0,0 @@ -//===- 03_gemmini_flush.rs - GEMMINI_FLUSH instruction ---------------------===// - -use super::instruction::{BallInstruction, ExecContext}; - -pub struct GemminiFlush; - -impl BallInstruction for GemminiFlush { - fn exec(_xs1: u64, _xs2: u64, _ctx: &mut ExecContext) -> u64 { - 0 - } - - fn latency(_xs1: u64, _xs2: u64) -> u64 { - 1 - } -} diff --git a/examples/balls/gemmini/emu/src/53_gemmini_preload.rs b/examples/balls/gemmini/emu/src/53_gemmini_preload.rs deleted file mode 100644 index 891d2d172..000000000 --- a/examples/balls/gemmini/emu/src/53_gemmini_preload.rs +++ /dev/null @@ -1,42 +0,0 @@ -use super::bank_matrix::read_i8_nn_at; -use super::decode::{pbank, rs1_b0, rs1_b2, rs1_iter}; -use super::gemmini_state::gemini; -use super::instruction::{BallInstruction, ExecContext}; - -pub struct GemminiPreload; - -impl BallInstruction for GemminiPreload { - fn exec(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { - let op1 = rs1_b0(xs1); - let wr = rs1_b2(xs1); - let n = rs1_iter(xs1) as usize; - - if !ctx.config(op1).allocated || !ctx.config(wr).allocated { - panic!("gemmini_preload: bank not allocated"); - } - if n == 0 || n > 64 { - panic!("gemmini_preload: bad iter"); - } - - let p1 = pbank(ctx, op1); - let mut gm = gemini().lock().unwrap(); - - if gm.cfg.dataflow == 1 { - let b = read_i8_nn_at(&ctx.banks, p1, ((xs2 >> 6) & 0x3ff) as usize, n); - gm.ws_b = Some(if gm.cfg.b_transpose { - (0..n).map(|i| (0..n).map(|j| b[j][i]).collect()).collect() - } else { - b - }); - } else { - // RTL OS preload only primes the mesh and injects D=0. It does - // not write the destination SRAM bank; compute produces C. - } - 0 - } - - fn latency(xs1: u64, _xs2: u64) -> u64 { - let n = rs1_iter(xs1).clamp(1, 64); - n.saturating_mul(n) - } -} diff --git a/examples/balls/gemmini/emu/src/67_gemmini_compute_accumulated.rs b/examples/balls/gemmini/emu/src/67_gemmini_compute_accumulated.rs deleted file mode 100644 index fee711ae2..000000000 --- a/examples/balls/gemmini/emu/src/67_gemmini_compute_accumulated.rs +++ /dev/null @@ -1,67 +0,0 @@ -use super::bank_matrix::{read_i32_nn_groups_at, read_i8_nn_at, write_i32_nn_groups_at}; -use super::decode::{pbank, pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; -use super::gemmini_state::{gemini, in_shift as apply_in_shift}; -use super::instruction::{BallInstruction, ExecContext}; - -pub struct GemminiComputeAccumulated; - -impl BallInstruction for GemminiComputeAccumulated { - fn exec(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { - let op_a = rs1_b0(xs1); - let op_b = rs1_b1(xs1); - let wr = rs1_b2(xs1); - let n = rs1_iter(xs1) as usize; - - if !ctx.config(op_a).allocated || !ctx.config(op_b).allocated || !ctx.config(wr).allocated { - panic!("gemmini_compute_accumulated: bank not allocated"); - } - if n == 0 || n > 64 { - panic!("gemmini_compute_accumulated: bad iter"); - } - - let pa = pbank(ctx, op_a); - let pb = pbank(ctx, op_b); - let pw: Vec<_> = (0..ctx.config(wr).cols) - .map(|group| pbank_group(ctx, wr, group)) - .collect(); - - let gm = gemini().lock().unwrap(); - let a_transpose = gm.cfg.a_transpose; - let b_transpose = gm.cfg.b_transpose; - let shift = gm.cfg.in_shift; - drop(gm); - - let zero_op1_tail = ((xs2 >> 5) & 1) != 0; - let a_base = ((xs2 >> 6) & 0x3ff) as usize; - let b_base = ((xs2 >> 16) & 0x3ff) as usize; - let out_base = ((xs2 >> 26) & 0x3ff) as usize; - let op1_valid_rows = ctx.config(op_a).valid_rows.min(n as u64) as usize; - let mut a = read_i8_nn_at(&ctx.banks, pa, a_base, n); - if zero_op1_tail { - for row in &mut a[op1_valid_rows..] { - row.fill(0); - } - } - let b = read_i8_nn_at(&ctx.banks, pb, b_base, n); - let mut c = read_i32_nn_groups_at(&ctx.banks, &pw, out_base, n); - - for i in 0..n { - for j in 0..n { - for k in 0..n { - let av = if a_transpose { a[k][i] } else { a[i][k] }; - let bv = if b_transpose { b[j][k] } else { b[k][j] }; - c[i][j] += av as i32 * bv as i32; - } - c[i][j] = apply_in_shift(c[i][j], shift); - } - } - - write_i32_nn_groups_at(&mut ctx.banks, &pw, out_base, &c, n); - 0 - } - - fn latency(xs1: u64, _xs2: u64) -> u64 { - let n = rs1_iter(xs1).clamp(1, 64); - n.saturating_mul(n).saturating_mul(n) / 4 + n.saturating_mul(n) - } -} diff --git a/examples/balls/gemmini/emu/src/80_gemmini_loop_ws.rs b/examples/balls/gemmini/emu/src/80_gemmini_loop_ws.rs deleted file mode 100644 index 6b82fa3ae..000000000 --- a/examples/balls/gemmini/emu/src/80_gemmini_loop_ws.rs +++ /dev/null @@ -1,143 +0,0 @@ -//===- 80_gemmini_loop_ws.rs - GEMMINI_LOOP_WS instruction -----------------===// - -use super::super::bank::{bank_row_bytes, MATRIX_SIZE}; -use super::gemmini_state::gemini; -use super::instruction::ExecContext; -use super::loop_micro_ops::{ - alloc, checked_stride, compute, free_after_hash, mvin, mvout, preload, -}; - -// Shared implementation -fn exec_cfg_impl(mnemonic: &str, xs2: u64) -> u64 { - let mut g = gemini().lock().unwrap(); - match mnemonic { - "GEMMINI_LOOP_WS_CONFIG_BOUNDS" => { - g.loop_ws.max_k = xs2 & 0xffff; - g.loop_ws.max_j = (xs2 >> 16) & 0xffff; - g.loop_ws.max_i = (xs2 >> 32) & 0xffff; - } - "GEMMINI_LOOP_WS_CONFIG_ADDR_A" => g.loop_ws.addr_a = xs2 & ((1u64 << 39) - 1), - "GEMMINI_LOOP_WS_CONFIG_ADDR_B" => g.loop_ws.addr_b = xs2 & ((1u64 << 39) - 1), - "GEMMINI_LOOP_WS_CONFIG_ADDR_D" => g.loop_ws.addr_d = xs2 & ((1u64 << 39) - 1), - "GEMMINI_LOOP_WS_CONFIG_ADDR_C" => g.loop_ws.addr_c = xs2 & ((1u64 << 39) - 1), - "GEMMINI_LOOP_WS_CONFIG_STRIDES_AB" => { - g.loop_ws.stride_a = xs2 & 0xffff_ffff; - g.loop_ws.stride_b = xs2 >> 32; - } - "GEMMINI_LOOP_WS_CONFIG_STRIDES_DC" => { - g.loop_ws.stride_d = xs2 & 0xffff_ffff; - g.loop_ws.stride_c = xs2 >> 32; - } - _ => panic!("gemmini_loop_ws: unknown cfg mnemonic={mnemonic}"), - } - 0 -} - -fn checked_tile_addr( - base: u64, - outer: u64, - outer_stride: u64, - inner: u64, - inner_stride: u64, -) -> u64 { - let outer_offset = outer - .checked_mul(outer_stride) - .expect("gemmini_loop_ws: outer address offset overflow"); - let inner_offset = inner - .checked_mul(inner_stride) - .expect("gemmini_loop_ws: inner address offset overflow"); - base.checked_add(outer_offset) - .and_then(|addr| addr.checked_add(inner_offset)) - .expect("gemmini_loop_ws: address overflow") -} - -fn exec_loop_impl(xs2: u64, ctx: &mut ExecContext) -> u64 { - let g = gemini().lock().unwrap(); - let lw = g.loop_ws.clone(); - let dataflow = g.cfg.dataflow; - drop(g); - - let bank_a = xs2 & 0x3ff; - let bank_b = (xs2 >> 10) & 0x3ff; - let bank_c = (xs2 >> 20) & 0x3ff; - let low_d = ((xs2 >> 30) & 1) != 0; - assert!( - bank_a != bank_b && bank_a != bank_c && bank_b != bank_c, - "gemmini_loop_ws: banks must be distinct" - ); - assert!( - lw.max_i > 0 && lw.max_j > 0 && lw.max_k > 0, - "gemmini_loop_ws: bounds must be > 0" - ); - assert_eq!( - dataflow, 0, - "gemmini_loop_ws: current RTL CISC contract supports OS dataflow only" - ); - assert!( - low_d && lw.addr_d == 0 && lw.stride_d == 0, - "gemmini_loop_ws: only zero D is supported" - ); - - let dim = MATRIX_SIZE as u64; - let bank_bytes = bank_row_bytes() as u64; - let stride_a = checked_stride(lw.stride_a, bank_bytes, "gemmini_loop_ws stride_a"); - let stride_b = checked_stride(lw.stride_b, bank_bytes, "gemmini_loop_ws stride_b"); - let stride_c = checked_stride( - lw.stride_c, - bank_bytes - .checked_mul(4) - .expect("gemmini_loop_ws: output row width overflow"), - "gemmini_loop_ws stride_c", - ); - - alloc(ctx, bank_a, 1); - alloc(ctx, bank_b, 1); - alloc(ctx, bank_c, 4); - - mvin(ctx, bank_a, lw.addr_a, dim, stride_a); - mvin(ctx, bank_b, lw.addr_b, dim, stride_b); - - for i in 0..lw.max_i { - for j in 0..lw.max_j { - for k in 0..lw.max_k { - preload(ctx, bank_a, bank_c, dim); - compute(ctx, k != 0, bank_a, bank_b, bank_c, dim, false, false); - - let addr_c = checked_tile_addr(lw.addr_c, i, lw.stride_c, j, dim * 4); - mvout(ctx, bank_c, addr_c, dim, stride_c); - - let next = if k + 1 < lw.max_k { - Some((i, j, k + 1)) - } else if j + 1 < lw.max_j { - Some((i, j + 1, 0)) - } else if i + 1 < lw.max_i { - Some((i + 1, 0, 0)) - } else { - None - }; - if let Some((next_i, next_j, next_k)) = next { - let addr_a = checked_tile_addr(lw.addr_a, next_i, lw.stride_a, next_k, dim); - let addr_b = checked_tile_addr(lw.addr_b, next_k, lw.stride_b, next_j, dim); - mvin(ctx, bank_a, addr_a, dim, stride_a); - mvin(ctx, bank_b, addr_b, dim, stride_b); - } - } - } - } - - free_after_hash(ctx, bank_a); - free_after_hash(ctx, bank_b); - free_after_hash(ctx, bank_c); - 0 -} - -pub fn execute(mnemonic: &str, xs2: u64, ctx: &mut ExecContext) -> u64 { - match mnemonic { - "GEMMINI_LOOP_WS" => exec_loop_impl(xs2, ctx), - _ => exec_cfg_impl(mnemonic, xs2), - } -} - -pub fn latency(mnemonic: &str) -> u64 { - if mnemonic == "GEMMINI_LOOP_WS" { 256 } else { 1 } -} diff --git a/examples/balls/gemmini/emu/src/dispatch.rs b/examples/balls/gemmini/emu/src/dispatch.rs deleted file mode 100644 index 13aa22526..000000000 --- a/examples/balls/gemmini/emu/src/dispatch.rs +++ /dev/null @@ -1,58 +0,0 @@ -use crate::inst::instruction::{BallInstruction, ExecContext}; - -use super::{ - f02_gemmini_config, f03_gemmini_flush, f53_gemmini_preload, f66_gemmini_compute_preloaded, - f67_gemmini_compute_accumulated, f80_gemmini_loop_ws, f96_gemmini_loop_conv_ws, -}; - -const BALL_CLASS: &str = "examples.balls.gemmini.GemminiBall"; - -pub fn execute_known( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, - ctx: &mut ExecContext, -) -> Option { - if ball_class != BALL_CLASS { - return None; - } - let mnemonic = crate::config::ball_domain::mnemonic_for_funct(funct)?; - let result = match mnemonic.as_str() { - "GEMMINI_CONFIG" => f02_gemmini_config::GemminiConfig::exec(xs1, xs2, ctx), - "GEMMINI_FLUSH" => f03_gemmini_flush::GemminiFlush::exec(xs1, xs2, ctx), - "GEMMINI_PRELOAD" => f53_gemmini_preload::GemminiPreload::exec(xs1, xs2, ctx), - "GEMMINI_COMPUTE_PRELOADED" => { - f66_gemmini_compute_preloaded::GemminiComputePreloaded::exec(xs1, xs2, ctx) - } - "GEMMINI_COMPUTE_ACCUMULATED" => { - f67_gemmini_compute_accumulated::GemminiComputeAccumulated::exec(xs1, xs2, ctx) - } - name if name.starts_with("GEMMINI_LOOP_WS_") || name == "GEMMINI_LOOP_WS" => { - f80_gemmini_loop_ws::execute(name, xs2, ctx) - } - name if name.starts_with("GEMMINI_LOOP_CONV_WS_") || name == "GEMMINI_LOOP_CONV_WS" => { - f96_gemmini_loop_conv_ws::execute(name, xs2, ctx) - } - _ => return None, - }; - Some(result) -} - -pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { - if ball_class != BALL_CLASS { - return None; - } - let mnemonic = crate::config::ball_domain::mnemonic_for_funct(funct)?; - let latency = match mnemonic.as_str() { - "GEMMINI_CONFIG" => f02_gemmini_config::GemminiConfig::latency(xs1, xs2), - "GEMMINI_FLUSH" => f03_gemmini_flush::GemminiFlush::latency(xs1, xs2), - "GEMMINI_PRELOAD" => f53_gemmini_preload::GemminiPreload::latency(xs1, xs2), - "GEMMINI_COMPUTE_PRELOADED" => f66_gemmini_compute_preloaded::GemminiComputePreloaded::latency(xs1, xs2), - "GEMMINI_COMPUTE_ACCUMULATED" => f67_gemmini_compute_accumulated::GemminiComputeAccumulated::latency(xs1, xs2), - name if name.starts_with("GEMMINI_LOOP_WS_") || name == "GEMMINI_LOOP_WS" => f80_gemmini_loop_ws::latency(name), - name if name.starts_with("GEMMINI_LOOP_CONV_WS_") || name == "GEMMINI_LOOP_CONV_WS" => f96_gemmini_loop_conv_ws::latency(name), - _ => return None, - }; - Some(latency) -} diff --git a/examples/balls/gemmini/emu/src/lib.rs b/examples/balls/gemmini/emu/src/lib.rs deleted file mode 100644 index 9ee62e0fb..000000000 --- a/examples/balls/gemmini/emu/src/lib.rs +++ /dev/null @@ -1,22 +0,0 @@ -pub(crate) use crate::inst::{bank_matrix, decode, instruction}; - -#[path = "02_gemmini_config.rs"] -mod f02_gemmini_config; -#[path = "03_gemmini_flush.rs"] -mod f03_gemmini_flush; -#[path = "53_gemmini_preload.rs"] -mod f53_gemmini_preload; -#[path = "66_gemmini_compute_preloaded.rs"] -mod f66_gemmini_compute_preloaded; -#[path = "67_gemmini_compute_accumulated.rs"] -mod f67_gemmini_compute_accumulated; -#[path = "80_gemmini_loop_ws.rs"] -mod f80_gemmini_loop_ws; -#[path = "96_gemmini_loop_conv_ws.rs"] -mod f96_gemmini_loop_conv_ws; -mod gemmini_state; -mod loop_micro_ops; - -mod dispatch; - -pub use dispatch::{cycles_after_issue, execute_known}; diff --git a/examples/balls/gemmini/verify/filelists/gemmini_ball.f b/examples/balls/gemmini/verify/filelists/gemmini_ball.f deleted file mode 100644 index c4b12100e..000000000 --- a/examples/balls/gemmini/verify/filelists/gemmini_ball.f +++ /dev/null @@ -1,29 +0,0 @@ -+incdir+@UVM@/src/ball -+incdir+src -+incdir+src/pkg -+incdir+src/tests -+incdir+@RTL@ - -@UVM@/src/ball/bb_blink_if.sv -@RTL@/PE.sv -@RTL@/AlwaysOutTransposer.sv -@RTL@/MacUnit.sv -@RTL@/PE_256.sv -@RTL@/Tile.sv -@RTL@/Mesh.sv -@RTL@/TagQueue.sv -@RTL@/ram_6x16.sv -@RTL@/Queue6_TagWithIdAndTotalRows.sv -@RTL@/MeshWithDelays.sv -@RTL@/ram_data_16x128.sv -@RTL@/Queue16_SramReadResp.sv -@RTL@/GemminiExCtrl.sv -@RTL@/LoopMatmulUnroller.sv -@RTL@/LoopConvAddrGen.sv -@RTL@/LoopConvUnroller.sv -@RTL@/LoopCmdEncoder.sv -@RTL@/Arbiter2_LoopCmd.sv -@RTL@/GemminiBall.sv - -src/pkg/gemmini_pkg.sv -src/tb_top.sv diff --git a/examples/balls/gemmini/verify/src/common/gemmini_defs.svh b/examples/balls/gemmini/verify/src/common/gemmini_defs.svh deleted file mode 100644 index c2e9a59da..000000000 --- a/examples/balls/gemmini/verify/src/common/gemmini_defs.svh +++ /dev/null @@ -1,3 +0,0 @@ -`ifndef GEMMINI_LOOP_WS_CONFIG_BOUNDS_FUNCT7 -`error "GEMMINI_LOOP_WS_CONFIG_BOUNDS_FUNCT7 must be provided by the selected Core ballISA" -`endif diff --git a/examples/balls/gemmini/verify/src/tb_top.sv b/examples/balls/gemmini/verify/src/tb_top.sv deleted file mode 100644 index be6204609..000000000 --- a/examples/balls/gemmini/verify/src/tb_top.sv +++ /dev/null @@ -1,81 +0,0 @@ -module tb_top; - import uvm_pkg::*; - import gemmini_pkg::*; - - bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); - - GemminiBall dut ( - .clock(intf.clock), - .reset(intf.reset), - .io_channelReady(1'b1), - .io_cmdReq_ready(intf.cmd_req_ready), - .io_cmdReq_valid(intf.cmd_req_valid), - .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), - .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), - .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), - .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), - .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), - .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), - .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), - .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), - .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), - .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank), - .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank), - .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank), - .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), - .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), - .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), - .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank), - .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), - .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), - .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), - .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), - .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), - .io_cmdResp_ready(intf.cmd_resp_ready), - .io_cmdResp_valid(intf.cmd_resp_valid), - .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), - .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), - .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), - .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), - .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), - .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), - .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), - .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), - .io_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), - .io_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), - .io_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), - .io_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), - .io_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), - .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), - .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), - .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), - .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), - .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), - .io_bankWrite_1_io_req_ready(intf.bank_write_req_ready[1]), - .io_bankWrite_1_io_req_valid(intf.bank_write_req_valid[1]), - .io_bankWrite_1_io_resp_ready(intf.bank_write_resp_ready[1]), - .io_bankWrite_1_io_resp_valid(intf.bank_write_resp_valid[1]), - .io_bankWrite_1_io_resp_bits_ok(intf.bank_write_resp_ok[1]), - .io_bankWrite_2_io_req_ready(intf.bank_write_req_ready[2]), - .io_bankWrite_2_io_req_valid(intf.bank_write_req_valid[2]), - .io_bankWrite_2_io_resp_ready(intf.bank_write_resp_ready[2]), - .io_bankWrite_2_io_resp_valid(intf.bank_write_resp_valid[2]), - .io_bankWrite_2_io_resp_bits_ok(intf.bank_write_resp_ok[2]), - .io_bankWrite_3_io_req_ready(intf.bank_write_req_ready[3]), - .io_bankWrite_3_io_req_valid(intf.bank_write_req_valid[3]), - .io_bankWrite_3_io_resp_ready(intf.bank_write_resp_ready[3]), - .io_bankWrite_3_io_resp_valid(intf.bank_write_resp_valid[3]), - .io_bankWrite_3_io_resp_bits_ok(intf.bank_write_resp_ok[3]), - .io_subRobReq_ready(intf.sub_rob_req_ready) - ); - - initial begin - intf.clock = 1'b0; - forever #5 intf.clock = ~intf.clock; - end - - initial begin - uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); - run_test("gemmini_ball_test"); - end -endmodule diff --git a/examples/balls/gemmini/verify/src/tests/gemmini_test.svh b/examples/balls/gemmini/verify/src/tests/gemmini_test.svh deleted file mode 100644 index f986e8e0d..000000000 --- a/examples/balls/gemmini/verify/src/tests/gemmini_test.svh +++ /dev/null @@ -1,98 +0,0 @@ -class gemmini_ball_test extends uvm_test; - `uvm_component_utils(gemmini_ball_test) - - typedef virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) vif_t; - vif_t vif; - - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(vif_t)::get(this, "", "vif", vif)) - `uvm_fatal("NOVIF", "bb_blink_if not found") - endfunction - - task run_phase(uvm_phase phase); - int cycles; - int unsigned bid; - phase.raise_objection(this); - apply_reset(); - - @(negedge vif.clock); - if (!$value$plusargs("BID=%d", bid)) `uvm_fatal("BID", "+BID is required") - vif.cmd_req_bits_cmd_bid = bid[4:0]; - vif.cmd_req_bits_cmd_funct7 = `GEMMINI_LOOP_WS_CONFIG_BOUNDS_FUNCT7; - vif.cmd_req_bits_cmd_iter = '0; - vif.cmd_req_bits_cmd_op1_en = 1'b0; - vif.cmd_req_bits_cmd_op2_en = 1'b0; - vif.cmd_req_bits_cmd_wr_spad_en = 1'b0; - vif.cmd_req_bits_cmd_op1_from_spad = 1'b0; - vif.cmd_req_bits_cmd_op2_from_spad = 1'b0; - vif.cmd_req_bits_cmd_special = 64'h0003_0002_0001; - vif.cmd_req_bits_cmd_op1_bank = '0; - vif.cmd_req_bits_cmd_op2_bank = '0; - vif.cmd_req_bits_cmd_wr_bank = '0; - vif.cmd_req_bits_cmd_op1_col = '0; - vif.cmd_req_bits_cmd_op2_col = '0; - vif.cmd_req_bits_cmd_wr_col = '0; - vif.cmd_req_bits_cmd_meta_bank = '0; - vif.cmd_req_bits_cmd_rs1 = '0; - vif.cmd_req_bits_cmd_rs2 = '0; - vif.cmd_req_bits_rob_id = 4'h9; - vif.cmd_req_bits_is_sub = 1'b1; - vif.cmd_req_bits_sub_rob_id = 8'h5a; - vif.cmd_req_valid = 1'b1; - - cycles = 0; - do begin - @(posedge vif.clock); - cycles++; - if (cycles > 20) `uvm_fatal("TIMEOUT", "Gemmini config command was not accepted") - end while (!vif.cmd_req_ready); - @(negedge vif.clock); - vif.cmd_req_valid = 1'b0; - - if (!vif.cmd_resp_valid) `uvm_fatal("RESP", "Gemmini config response was not observed") - if (vif.cmd_resp_bits_rob_id !== 4'h9 || vif.cmd_resp_bits_is_sub !== 1'b1 || - vif.cmd_resp_bits_sub_rob_id !== 8'h5a) - `uvm_fatal("RESP", $sformatf( - "metadata mismatch: rob=%0h is_sub=%0b sub_rob=%0h", - vif.cmd_resp_bits_rob_id, - vif.cmd_resp_bits_is_sub, - vif.cmd_resp_bits_sub_rob_id - )) - for (int i = 0; i < `BB_IN_BW; i++) begin - if (vif.bank_read_req_valid[i]) - `uvm_fatal("BANK", "Gemmini config command unexpectedly read a bank") - end - for (int i = 0; i < `BB_OUT_BW; i++) begin - if (vif.bank_write_req_valid[i]) - `uvm_fatal("BANK", "Gemmini config command unexpectedly wrote a bank") - end - - `uvm_info("GEMMINI", "loop-ws config command passed", UVM_LOW) - phase.drop_objection(this); - endtask - - task apply_reset(); - vif.cmd_req_valid = 1'b0; - vif.cmd_resp_ready = 1'b1; - vif.sub_rob_req_ready = 1'b1; - for (int i = 0; i < `BB_IN_BW; i++) begin - vif.bank_read_req_ready[i] = 1'b1; - vif.bank_read_resp_valid[i] = 1'b0; - vif.bank_read_resp_data[i] = '0; - end - for (int i = 0; i < `BB_OUT_BW; i++) begin - vif.bank_write_req_ready[i] = 1'b1; - vif.bank_write_resp_valid[i] = 1'b0; - vif.bank_write_resp_ok[i] = 1'b1; - end - vif.reset = 1'b1; - repeat (5) @(posedge vif.clock); - vif.reset = 1'b0; - repeat (2) @(posedge vif.clock); - endtask -endclass diff --git a/examples/balls/gemmini/workloads/ctests/CMakeLists.txt b/examples/balls/gemmini/workloads/ctests/CMakeLists.txt deleted file mode 100644 index 229898a07..000000000 --- a/examples/balls/gemmini/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,16 +0,0 @@ -add_buckyball_ctests( - gemmini_accumulated_16x16_test.c - gemmini_os_cisc_atranspose_test.c - gemmini_os_cisc_basic_test.c - gemmini_os_cisc_btranspose_test.c - gemmini_os_cisc_shift_test.c - gemmini_os_risc_abtranspose_test.c - gemmini_os_risc_atranspose_test.c - gemmini_os_risc_basic_test.c - gemmini_os_risc_btranspose_test.c - gemmini_os_risc_shift_test.c - gemmini_ws_cisc_conv_test.c - gemmini_ws_risc_basic_test.c - gemmini_ws_risc_btranspose_test.c - gemmini_ws_risc_shift_test.c -) diff --git a/examples/balls/gemmini/workloads/isa/gemmini_flush.h b/examples/balls/gemmini/workloads/isa/gemmini_flush.h deleted file mode 100644 index 2c55b62c0..000000000 --- a/examples/balls/gemmini/workloads/isa/gemmini_flush.h +++ /dev/null @@ -1,13 +0,0 @@ -#ifndef _BB_GEMMINI_FLUSH_H_ -#define _BB_GEMMINI_FLUSH_H_ - -#include -#include - -#define BB_GEMMINI_FLUSH_RS2 4ULL - -// Flush the matrix array state -#define bb_gemmini_flush() \ - BUCKYBALL_INSTRUCTION_R_R(0, BB_GEMMINI_FLUSH_RS2, BB_FUNC7(GEMMINI_FLUSH)) - -#endif // _BB_GEMMINI_FLUSH_H_ diff --git a/examples/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16.mlir b/examples/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16.mlir deleted file mode 100644 index 496db7704..000000000 --- a/examples/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16.mlir +++ /dev/null @@ -1,68 +0,0 @@ -func.func private @check_result(memref<16x16xi32>) -> () -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %one_i8 = arith.constant 1 : i8 - %initial = arith.constant 7 : i32 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %two = arith.constant 2 : index - %three = arith.constant 3 : index - %five = arith.constant 5 : index - %seven = arith.constant 7 : index - %sixteen = arith.constant 16 : index - %depth16 = arith.constant 16 : i64 - %stride = arith.constant 1 : i64 - %a = memref.alloc() : memref<16x16xi8> - %b = memref.alloc() : memref<16x16xi8> - %c = memref.alloc() : memref<16x16xi32> - linalg.fill ins(%initial : i32) outs(%c : memref<16x16xi32>) - scf.for %row = %zero to %sixteen step %one { - scf.for %col = %zero to %sixteen step %one { - %r3 = arith.muli %row, %three : index - %c5 = arith.muli %col, %five : index - %asum = arith.addi %r3, %c5 : index - %arem = arith.remui %asum, %seven : index - %aval = arith.subi %arem, %three : index - %av = arith.index_cast %aval : index to i8 - %r2 = arith.muli %row, %two : index - %c3 = arith.muli %col, %three : index - %bsum = arith.addi %r2, %c3 : index - %brem = arith.remui %bsum, %five : index - %bval = arith.subi %brem, %two : index - %bv = arith.index_cast %bval : index to i8 - memref.store %av, %a[%row, %col] : memref<16x16xi8> - memref.store %bv, %b[%row, %col] : memref<16x16xi8> - } - } - %a_bank = arith.constant 0 : i64 - %b_bank = arith.constant 1 : i64 - %c_bank = arith.constant 3 : i64 - buckyball.mset %a_bank : i64 - buckyball.mset %b_bank : i64 - buckyball.mset %c_bank : i64 - buckyball.mvin %a %a_bank %depth16 %stride : memref<16x16xi8> i64 i64 i64 - buckyball.mvin %b %b_bank %depth16 %stride : memref<16x16xi8> i64 i64 i64 - buckyball.mvin %c %c_bank %depth16 %stride : memref<16x16xi32> i64 i64 i64 - %config = arith.constant 2064 : i64 - %zero64 = arith.constant 0 : i64 - %shift10 = arith.constant 10 : i64 - %shift20 = arith.constant 20 : i64 - %shift30 = arith.constant 30 : i64 - %iter_bits = arith.shli %depth16, %shift30 : i64 - %c_bits = arith.shli %c_bank, %shift20 : i64 - %b_bits = arith.shli %b_bank, %shift10 : i64 - %compute_rs1 = arith.ori %iter_bits, %b_bits : i64 - %compute_rs1_final = arith.ori %compute_rs1, %c_bits : i64 - %compute_rs2 = arith.constant 3 : i64 - buckyball.gemmini "GEMMINI_CONFIG" %zero64, %config : i64 - buckyball.gemmini "GEMMINI_COMPUTE_ACCUMULATED" %compute_rs1_final, %compute_rs2 : i64 - buckyball.mvout %c %c_bank %depth16 %stride : memref<16x16xi32> i64 i64 i64 - func.call @check_result(%c) : (memref<16x16xi32>) -> () - buckyball.mset %a_bank : i64 - buckyball.mset %b_bank : i64 - buckyball.mset %c_bank : i64 - memref.dealloc %a : memref<16x16xi8> - memref.dealloc %b : memref<16x16xi8> - memref.dealloc %c : memref<16x16xi32> - return %zero_i8 : i8 -} diff --git a/examples/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16.mlir b/examples/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16.mlir deleted file mode 100644 index 00f54ed03..000000000 --- a/examples/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16.mlir +++ /dev/null @@ -1,77 +0,0 @@ -func.func private @check_result(memref<16x16xi32>) -> () -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %one_i8 = arith.constant 1 : i8 - %zero_i32 = arith.constant 0 : i32 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %two = arith.constant 2 : index - %three = arith.constant 3 : index - %five = arith.constant 5 : index - %seven = arith.constant 7 : index - %sixteen = arith.constant 16 : index - %depth16 = arith.constant 16 : i64 - %stride = arith.constant 1 : i64 - %a = memref.alloc() : memref<16x16xi8> - %b = memref.alloc() : memref<16x16xi8> - %d = memref.alloc() : memref<16x16xi8> - %c = memref.alloc() : memref<16x16xi32> - linalg.fill ins(%zero_i8 : i8) outs(%d : memref<16x16xi8>) - linalg.fill ins(%zero_i32 : i32) outs(%c : memref<16x16xi32>) - scf.for %row = %zero to %sixteen step %one { - scf.for %col = %zero to %sixteen step %one { - %r3 = arith.muli %row, %three : index - %c5 = arith.muli %col, %five : index - %asum = arith.addi %r3, %c5 : index - %arem = arith.remui %asum, %seven : index - %aval = arith.subi %arem, %three : index - %av = arith.index_cast %aval : index to i8 - %r2 = arith.muli %row, %two : index - %c3 = arith.muli %col, %three : index - %bsum = arith.addi %r2, %c3 : index - %brem = arith.remui %bsum, %five : index - %bval = arith.subi %brem, %two : index - %bv = arith.index_cast %bval : index to i8 - memref.store %av, %a[%row, %col] : memref<16x16xi8> - memref.store %bv, %b[%row, %col] : memref<16x16xi8> - } - } - %a_bank = arith.constant 0 : i64 - %b_bank = arith.constant 1 : i64 - %d_bank = arith.constant 2 : i64 - %c_bank = arith.constant 3 : i64 - buckyball.mset %a_bank : i64 - buckyball.mset %b_bank : i64 - buckyball.mset %d_bank : i64 - buckyball.mset %c_bank : i64 - buckyball.mvin %a %a_bank %depth16 %stride : memref<16x16xi8> i64 i64 i64 - buckyball.mvin %b %b_bank %depth16 %stride : memref<16x16xi8> i64 i64 i64 - buckyball.mvin %d %d_bank %depth16 %stride : memref<16x16xi8> i64 i64 i64 - %config = arith.constant 0 : i64 - %shift10 = arith.constant 10 : i64 - %shift20 = arith.constant 20 : i64 - %shift30 = arith.constant 30 : i64 - %iter_bits = arith.shli %depth16, %shift30 : i64 - %c_bits = arith.shli %c_bank, %shift20 : i64 - %preload_rs1_base = arith.ori %iter_bits, %d_bank : i64 - %preload_rs1 = arith.ori %preload_rs1_base, %c_bits : i64 - %b_bits = arith.shli %b_bank, %shift10 : i64 - %compute_rs1 = arith.ori %iter_bits, %b_bits : i64 - %compute_rs1_final = arith.ori %compute_rs1, %c_bits : i64 - %preload_rs2 = arith.constant 1 : i64 - %compute_rs2 = arith.constant 2 : i64 - buckyball.gemmini "GEMMINI_CONFIG" %config, %config : i64 - buckyball.gemmini "GEMMINI_PRELOAD" %preload_rs1, %preload_rs2 : i64 - buckyball.gemmini "GEMMINI_COMPUTE_PRELOADED" %compute_rs1_final, %compute_rs2 : i64 - buckyball.mvout %c %c_bank %depth16 %stride : memref<16x16xi32> i64 i64 i64 - func.call @check_result(%c) : (memref<16x16xi32>) -> () - buckyball.mset %a_bank : i64 - buckyball.mset %b_bank : i64 - buckyball.mset %d_bank : i64 - buckyball.mset %c_bank : i64 - memref.dealloc %a : memref<16x16xi8> - memref.dealloc %b : memref<16x16xi8> - memref.dealloc %d : memref<16x16xi8> - memref.dealloc %c : memref<16x16xi32> - return %zero_i8 : i8 -} diff --git a/examples/balls/gemmini/workloads/mlir_tests/bank/CMakeLists.txt b/examples/balls/gemmini/workloads/mlir_tests/bank/CMakeLists.txt deleted file mode 100644 index a0c200267..000000000 --- a/examples/balls/gemmini/workloads/mlir_tests/bank/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) - set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) -endif() -set(BUCKYBALL_MLIR_TEST_PREFIX bank) -buckyball_require_min_bank_lines(${BUCKYBALL_MLIR_ACTIVE_TARGET} 16 Gemmini) -add_buckyball_mlir_test(gemmini_compute_preloaded_bank_tiles TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) -add_buckyball_mlir_test(gemmini_compute_accumulated_bank_tiles TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles.mlir.in b/examples/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles.mlir.in deleted file mode 100644 index d5f6071b1..000000000 --- a/examples/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles.mlir.in +++ /dev/null @@ -1,75 +0,0 @@ -func.func private @check_result(memref<@BB_BANK_LINES@x16xi32>) -> () -func.func @main() -> i8 { - %z8 = arith.constant 0 : i8 - %o8 = arith.constant 1 : i8 - %z32 = arith.constant 0 : i32 - %z = arith.constant 0 : index - %one = arith.constant 1 : index - %two = arith.constant 2 : index - %three = arith.constant 3 : index - %five = arith.constant 5 : index - %seven = arith.constant 7 : index - %dim = arith.constant 16 : index - %rows = arith.constant @BB_BANK_LINES@ : index - %d16 = arith.constant 16 : i64 - %rows64 = arith.constant @BB_BANK_LINES@ : i64 - %zero64 = arith.constant 0 : i64 - %stride = arith.constant 1 : i64 - buckyball.gemmini "GEMMINI_CONFIG" %zero64, %zero64 : i64 - %a = memref.alloc() : memref<@BB_BANK_LINES@x16xi8> - %b = memref.alloc() : memref<16x16xi8> - %d = memref.alloc() : memref<16x16xi8> - %c = memref.alloc() : memref<@BB_BANK_LINES@x16xi32> - linalg.fill ins(%z8 : i8) outs(%d : memref<16x16xi8>) - linalg.fill ins(%z32 : i32) outs(%c : memref<@BB_BANK_LINES@x16xi32>) - scf.for %tile = %z to %rows step %dim { - scf.for %row = %z to %dim step %one { - %arow = arith.addi %tile, %row : index - scf.for %k = %z to %dim step %one { - %r3 = arith.muli %row, %three : index - %k5 = arith.muli %k, %five : index - %sum = arith.addi %r3, %k5 : index - %rem = arith.remui %sum, %seven : index - %value = arith.subi %rem, %three : index - %v8 = arith.index_cast %value : index to i8 - memref.store %v8, %a[%arow, %k] : memref<@BB_BANK_LINES@x16xi8> - } - } - } - scf.for %row = %z to %dim step %one { - scf.for %col = %z to %dim step %one { - %r2 = arith.muli %row, %two : index - %c3 = arith.muli %col, %three : index - %sum = arith.addi %r2, %c3 : index - %rem = arith.remui %sum, %five : index - %value = arith.subi %rem, %two : index - %v8 = arith.index_cast %value : index to i8 - memref.store %v8, %b[%row, %col] : memref<16x16xi8> - } - } - %ab = buckyball.bank_alloc - %bb = buckyball.bank_alloc - %db = buckyball.bank_alloc - %cb = buckyball.bank_alloc - %al = buckyball.bank_mvin %a %ab %rows64 %stride : memref<@BB_BANK_LINES@x16xi8> i64 i64 i64 - %bl = buckyball.bank_mvin %b %bb %d16 %stride : memref<16x16xi8> i64 i64 i64 - %dl = buckyball.bank_mvin %d %db %d16 %stride : memref<16x16xi8> i64 i64 i64 - %result = scf.for %base = %z to %rows step %dim iter_args(%c0 = %cb) -> (i64) { - %base64 = arith.index_cast %base : index to i64 - %p = buckyball.bank_gemmini_preload %dl %c0 %d16 %base64 %base64 : i64 i64 i64 i64 i64 - %first = buckyball.bank_gemmini_compute_preloaded %al %bl %p %d16 %base64 %zero64 %base64 : i64 i64 i64 i64 i64 i64 i64 - %sum = buckyball.bank_gemmini_compute_accumulated %al %bl %first %d16 %base64 %zero64 %base64 : i64 i64 i64 i64 i64 i64 i64 - scf.yield %sum : i64 - } - %stored = buckyball.bank_mvout %c %result %rows64 %stride : memref<@BB_BANK_LINES@x16xi32> i64 i64 i64 - func.call @check_result(%c) : (memref<@BB_BANK_LINES@x16xi32>) -> () - buckyball.bank_release %al : i64 - buckyball.bank_release %bl : i64 - buckyball.bank_release %dl : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %a : memref<@BB_BANK_LINES@x16xi8> - memref.dealloc %b : memref<16x16xi8> - memref.dealloc %d : memref<16x16xi8> - memref.dealloc %c : memref<@BB_BANK_LINES@x16xi32> - return %z8 : i8 -} diff --git a/examples/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles_main.cpp b/examples/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles_main.cpp deleted file mode 100644 index 35d8446b8..000000000 --- a/examples/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles_main.cpp +++ /dev/null @@ -1,24 +0,0 @@ -#include -#include -#include -#include -extern "C" void check_result(int32_t *, int32_t *data, int64_t offset, - int64_t rows, int64_t lanes, int64_t stride, - int64_t lane_stride) { - if (rows != BANK_LINES || lanes != 16 || stride != 16 || lane_stride != 1) - exit(1); - for (int row = 0; row < BANK_LINES; ++row) - for (int column = 0; column < 16; ++column) { - int32_t expected = 0; - for (int k = 0; k < 16; ++k) - expected += 2 * ((3 * (row % 16) + 5 * k) % 7 - 3) * - ((2 * k + 3 * column) % 5 - 2); - if (data[offset + row * stride + column] != expected) { - printf("gemmini accumulated mismatch row=%d column=%d expected=%d " - "actual=%d\n", - row, column, expected, data[offset + row * stride + column]); - exit(1); - } - } - printf("gemmini compute-accumulated bank tiles PASS rows=%d\n", BANK_LINES); -} diff --git a/examples/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles.mlir.in b/examples/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles.mlir.in deleted file mode 100644 index 1db91f60b..000000000 --- a/examples/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles.mlir.in +++ /dev/null @@ -1,74 +0,0 @@ -func.func private @check_result(memref<@BB_BANK_LINES@x16xi32>) -> () -func.func @main() -> i8 { - %z8 = arith.constant 0 : i8 - %o8 = arith.constant 1 : i8 - %z32 = arith.constant 0 : i32 - %z = arith.constant 0 : index - %one = arith.constant 1 : index - %two = arith.constant 2 : index - %three = arith.constant 3 : index - %five = arith.constant 5 : index - %seven = arith.constant 7 : index - %dim = arith.constant 16 : index - %rows = arith.constant @BB_BANK_LINES@ : index - %rows64 = arith.constant @BB_BANK_LINES@ : i64 - %d16 = arith.constant 16 : i64 - %zero64 = arith.constant 0 : i64 - %stride = arith.constant 1 : i64 - buckyball.gemmini "GEMMINI_CONFIG" %zero64, %zero64 : i64 - %a = memref.alloc() : memref<@BB_BANK_LINES@x16xi8> - %b = memref.alloc() : memref<16x16xi8> - %d = memref.alloc() : memref<16x16xi8> - %c = memref.alloc() : memref<@BB_BANK_LINES@x16xi32> - linalg.fill ins(%z8 : i8) outs(%d : memref<16x16xi8>) - linalg.fill ins(%z32 : i32) outs(%c : memref<@BB_BANK_LINES@x16xi32>) - scf.for %tile = %z to %rows step %dim { - scf.for %row = %z to %dim step %one { - %arow = arith.addi %tile, %row : index - scf.for %k = %z to %dim step %one { - %r3 = arith.muli %row, %three : index - %k5 = arith.muli %k, %five : index - %sum = arith.addi %r3, %k5 : index - %rem = arith.remui %sum, %seven : index - %value = arith.subi %rem, %three : index - %v8 = arith.index_cast %value : index to i8 - memref.store %v8, %a[%arow, %k] : memref<@BB_BANK_LINES@x16xi8> - } - } - } - scf.for %row = %z to %dim step %one { - scf.for %col = %z to %dim step %one { - %r2 = arith.muli %row, %two : index - %c3 = arith.muli %col, %three : index - %sum = arith.addi %r2, %c3 : index - %rem = arith.remui %sum, %five : index - %value = arith.subi %rem, %two : index - %v8 = arith.index_cast %value : index to i8 - memref.store %v8, %b[%row, %col] : memref<16x16xi8> - } - } - %ab = buckyball.bank_alloc - %bb = buckyball.bank_alloc - %db = buckyball.bank_alloc - %cb = buckyball.bank_alloc - %al = buckyball.bank_mvin %a %ab %rows64 %stride : memref<@BB_BANK_LINES@x16xi8> i64 i64 i64 - %bl = buckyball.bank_mvin %b %bb %d16 %stride : memref<16x16xi8> i64 i64 i64 - %dl = buckyball.bank_mvin %d %db %d16 %stride : memref<16x16xi8> i64 i64 i64 - %result = scf.for %base = %z to %rows step %dim iter_args(%c0 = %cb) -> (i64) { - %base64 = arith.index_cast %base : index to i64 - %p = buckyball.bank_gemmini_preload %dl %c0 %d16 %base64 %base64 : i64 i64 i64 i64 i64 - %r = buckyball.bank_gemmini_compute_preloaded %al %bl %p %d16 %base64 %zero64 %base64 : i64 i64 i64 i64 i64 i64 i64 - scf.yield %r : i64 - } - %stored = buckyball.bank_mvout %c %result %rows64 %stride : memref<@BB_BANK_LINES@x16xi32> i64 i64 i64 - func.call @check_result(%c) : (memref<@BB_BANK_LINES@x16xi32>) -> () - buckyball.bank_release %al : i64 - buckyball.bank_release %bl : i64 - buckyball.bank_release %dl : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %a : memref<@BB_BANK_LINES@x16xi8> - memref.dealloc %b : memref<16x16xi8> - memref.dealloc %d : memref<16x16xi8> - memref.dealloc %c : memref<@BB_BANK_LINES@x16xi32> - return %z8 : i8 -} diff --git a/examples/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles_main.cpp b/examples/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles_main.cpp deleted file mode 100644 index ce5a5fb20..000000000 --- a/examples/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles_main.cpp +++ /dev/null @@ -1,23 +0,0 @@ -#include -#include -#include -#include -extern "C" void check_result(int32_t *, int32_t *data, int64_t offset, - int64_t rows, int64_t lanes, int64_t stride, - int64_t lane_stride) { - if (rows != BANK_LINES || lanes != 16 || stride != 16 || lane_stride != 1) - exit(1); - for (int row = 0; row < BANK_LINES; ++row) - for (int column = 0; column < 16; ++column) { - int32_t expected = 0; - for (int k = 0; k < 16; ++k) - expected += - ((3 * (row % 16) + 5 * k) % 7 - 3) * ((2 * k + 3 * column) % 5 - 2); - if (data[offset + row * stride + column] != expected) { - printf("gemmini bank mismatch row=%d column=%d expected=%d actual=%d\n", - row, column, expected, data[offset + row * stride + column]); - exit(1); - } - } - printf("gemmini compute-preloaded bank tiles PASS rows=%d\n", BANK_LINES); -} diff --git a/examples/balls/im2col/arch/src/main/scala/Im2col.scala b/examples/balls/im2col/arch/src/main/scala/Im2col.scala deleted file mode 100644 index 6c411d08e..000000000 --- a/examples/balls/im2col/arch/src/main/scala/Im2col.scala +++ /dev/null @@ -1,223 +0,0 @@ -package examples.balls.im2col - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.balldomain.blink.{BallStatus, BankRead, BankWrite} -import examples.balls.im2col.configs.Im2colBallParam -import framework.balldomain.rs.{BallRsComplete, BallRsIssue} -import framework.top.GlobalConfig - -@instantiable -class Im2col(val b: GlobalConfig) extends Module { - private val ballCfg = Im2colBallParam(b) - private val maxIter = ballCfg.maxIter - private val maxKSize = ballCfg.maxKSize - private val maxPad = ballCfg.maxPadding - private val tile = 16 - - private val mapping = b.ballDomain.ballIdMappings - .find(_.ballName == "Im2colBall") - .getOrElse(throw new IllegalArgumentException("Im2colBall not found in config")) - - private val im2colFunct = b.ballDomain.ballISA - .find(_.mnemonic == "IM2COL") - .map(_.funct7) - .getOrElse(throw new IllegalArgumentException("IM2COL not found in ballISA")) - - require(mapping.inBW == 1, "Im2colBall requires inBW=1") - require(mapping.outBW == 1, "Im2colBall requires outBW=1") - require(b.memDomain.bankWidth == 128, "Im2colBall requires 128-bit bank rows") - require(isPow2(b.memDomain.bankEntries), "Im2colBall requires power-of-two bank depth") - require(maxIter + 2 * maxPad <= 255, "Im2colBall dimensions must fit the divider") - require((im2colFunct >> 4) == 3, "IM2COL must encode one read and one write") - - @public - val io = IO(new Bundle { - val cmdReq = Flipped(Decoupled(new BallRsIssue(b))) - val cmdResp = Decoupled(new BallRsComplete(b)) - val bankRead = Vec(mapping.inBW, Flipped(new BankRead(b))) - val bankWrite = Vec(mapping.outBW, Flipped(new BankWrite(b))) - val status = new BallStatus - }) - - private val cfg = Module(new Im2colConfigRegs(b, maxIter, maxKSize, maxPad)) - private val win = Module(new Im2colWindow(maxKSize)) - private val div = Module(new RestoringDiv(16, 8)) - private val lineBuf: Instance[LineBufferManager] = Instantiate(new LineBufferManager(b)) - private val writer: Instance[StreamWriter] = Instantiate(new StreamWriter(b)) - - private val cIdle :: cDivRow :: cDivCol :: cCheck :: cDivStart :: cLaunch :: Nil = Enum(6) - private val calc = RegInit(cIdle) - - private val running = RegInit(false.B) - private val inputReady = RegInit(false.B) - private val respPending = RegInit(false.B) - private val armed = RegInit(false.B) - private val outRowsReg = RegInit(0.U(16.W)) - private val outColsReg = RegInit(0.U(16.W)) - private val startOutRow = RegInit(0.U(16.W)) - private val startOutCol = RegInit(0.U(16.W)) - private val localWindow = RegInit(0.U(12.W)) - private val elemAge = RegInit(0.U(2.W)) - private val launch = WireDefault(false.B) - - cfg.io.cmd := io.cmdReq.bits - cfg.io.load := io.cmdReq.fire - - io.cmdReq.ready := !running && !respPending && !armed && calc === cIdle - io.cmdResp.valid := respPending - io.cmdResp.bits.rob_id := cfg.io.robId - io.cmdResp.bits.is_sub := cfg.io.isSub - io.cmdResp.bits.sub_rob_id := cfg.io.subRobId - io.status.idle := !running && !respPending && !armed && calc === cIdle - io.status.running := running || armed || calc =/= cIdle - - div.io.start := false.B - div.io.a := 0.U - div.io.b := 0.U - - when(io.cmdReq.fire) { - val command = io.cmdReq.bits.cmd - assert(command.funct7 === im2colFunct.U, "Im2colBall received an unknown funct7") - assert(command.rs1(9, 0) < b.memDomain.virtualBankCount.U, "IM2COL input bank is invalid") - assert(command.rs1(19, 10) === 0.U, "IM2COL reserves input bank 1") - assert(command.rs1(29, 20) < b.memDomain.virtualBankCount.U, "IM2COL output bank is invalid") - assert(command.rs2(63) === 0.U, "IM2COL reserves rs2[63]") - assert(command.iter(b.frontend.iter_len - 1, 16) === 0.U, "IM2COL input size exceeds 16 bits") - assert( - command.op1_col === 1.U && command.op2_col === 0.U && command.wr_col === 1.U, - "IM2COL requires one physical input bank and one physical output bank" - ) - assert(command.rs2(7, 0) > 0.U && command.rs2(7, 0) <= maxKSize.U, "IM2COL kernel must be in the configured range") - armed := true.B - } - - when(armed) { - armed := false.B - assert(!cfg.io.invalid, "IM2COL command has an illegal shape or bank assignment") - val padded = cfg.io.inputSize +& (cfg.io.padding << 1) - div.io.start := true.B - div.io.a := (padded - cfg.io.kernel - cfg.io.startRow)(15, 0) - div.io.b := cfg.io.stride - calc := cDivRow - } - - when(calc === cDivRow && div.io.done) { - assert(div.io.r === 0.U, "IM2COL output row shape must be integral") - outRowsReg := div.io.q + 1.U - val padded = cfg.io.inputSize +& (cfg.io.padding << 1) - div.io.start := true.B - div.io.a := (padded - cfg.io.kernel - cfg.io.startCol)(15, 0) - div.io.b := cfg.io.stride - calc := cDivCol - } - - when(calc === cDivCol && div.io.done) { - assert(div.io.r === 0.U, "IM2COL output column shape must be integral") - outColsReg := div.io.q + 1.U - calc := cCheck - } - - when(calc === cCheck) { - val inputValues = cfg.io.inputSize * cfg.io.inputSize - val inputRows = (inputValues +& (tile - 1).U) >> log2Ceil(tile) - val windows = outRowsReg * outColsReg - val windowEnd = cfg.io.windowStart +& cfg.io.windowCount - val kernelElems = cfg.io.kernel * cfg.io.kernel - val mTiles = (cfg.io.windowCount +& (tile - 1).U) >> log2Ceil(tile) - val kTiles = (kernelElems +& (tile - 1).U) >> log2Ceil(tile) - val outputRows = mTiles * kTiles * tile.U - - assert(cfg.io.windowStart < windows && windowEnd <= windows, "IM2COL selected window range exceeds the output shape") - assert(inputRows <= b.memDomain.bankEntries.U, "IM2COL input footprint exceeds bank depth") - assert(outputRows <= b.memDomain.bankEntries.U, "IM2COL output footprint exceeds bank depth") - - div.io.start := true.B - div.io.a := cfg.io.windowStart - div.io.b := outColsReg(7, 0) - calc := cDivStart - } - - when(calc === cDivStart && div.io.done) { - assert(div.io.q < outRowsReg && div.io.r < outColsReg, "IM2COL window start decoding exceeded the output shape") - startOutRow := div.io.q - startOutCol := div.io.r - calc := cLaunch - } - - when(calc === cLaunch) { - launch := true.B - running := true.B - inputReady := false.B - localWindow := 0.U - elemAge := 0.U - calc := cIdle - } - - when(io.cmdResp.fire) { - respPending := false.B - } - - win.io.init := launch - win.io.next := false.B - win.io.kernel := cfg.io.kernel - win.io.outputCols := outColsReg - win.io.startRow := startOutRow - win.io.startCol := startOutCol - win.io.windowCount := cfg.io.windowCount - private val canEmitElem = running && inputReady && elemAge === 3.U - win.io.elemFire := canEmitElem && writer.io.elemIn.ready - - lineBuf.io.bankRead(0) <> io.bankRead(0) - lineBuf.io.start := launch - lineBuf.io.inRows := cfg.io.inputSize - lineBuf.io.inCols := cfg.io.inputSize - lineBuf.io.rowStride := cfg.io.stride - lineBuf.io.colStride := cfg.io.stride - lineBuf.io.padding := cfg.io.padding - lineBuf.io.startRow := cfg.io.startRow - lineBuf.io.startCol := cfg.io.startCol - lineBuf.io.inputBase := cfg.io.inputBase - lineBuf.io.lane := cfg.io.lane - lineBuf.io.outRow := win.io.outRow - lineBuf.io.outCol := win.io.outCol - lineBuf.io.kRowIdx := win.io.kRowIdx - lineBuf.io.kColIdx := win.io.kColIdx - lineBuf.io.rBankId := cfg.io.rBank - lineBuf.io.robId := cfg.io.robId - - writer.io.bankWrite(0) <> io.bankWrite(0) - writer.io.init := launch - writer.io.wBankId := cfg.io.wBank - writer.io.robId := cfg.io.robId - writer.io.elemIn.valid := canEmitElem - writer.io.elemIn.bits := lineBuf.io.elemData - writer.io.elemLast := win.io.elemLast - writer.io.finalWindow := win.io.last - writer.io.kRows := cfg.io.kernel - writer.io.kCols := cfg.io.kernel - writer.io.windowIdx := localWindow - - when(launch) { - inputReady := false.B - elemAge := 0.U - }.elsewhen(running && !inputReady && lineBuf.io.loadDone) { - inputReady := true.B - elemAge := 0.U - }.elsewhen(writer.io.elemIn.fire || writer.io.windowComplete) { - elemAge := 0.U - }.elsewhen(running && inputReady && elemAge =/= 3.U) { - elemAge := elemAge + 1.U - } - - when(writer.io.opComplete) { - inputReady := false.B - running := false.B - respPending := true.B - }.elsewhen(writer.io.windowComplete) { - win.io.next := true.B - localWindow := localWindow + 1.U - } - -} diff --git a/examples/balls/im2col/arch/src/main/scala/Im2colBall.scala b/examples/balls/im2col/arch/src/main/scala/Im2colBall.scala deleted file mode 100644 index 5e71a1593..000000000 --- a/examples/balls/im2col/arch/src/main/scala/Im2colBall.scala +++ /dev/null @@ -1,57 +0,0 @@ -package examples.balls.im2col - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.balldomain.blink.{BlinkIO, HasBlink, SubRobRow} -import framework.balldomain.blink.mmio.MmioRead -import examples.balls.im2col.Im2col -import framework.top.GlobalConfig - -/** - * Im2colBall - An Im2col computation Ball that complies with the Blink protocol - */ -@instantiable -class Im2colBall(val b: GlobalConfig) extends Module with HasBlink { - - val ballCommonConfig = b.ballDomain.ballIdMappings - .find(_.ballName == "Im2colBall") - .getOrElse( - throw new IllegalArgumentException("Im2colBall not found in config") - ) - - val inBW = ballCommonConfig.inBW - val outBW = ballCommonConfig.outBW - - @public - val io = IO(new BlinkIO(b, inBW, outBW)) - - def blink: BlinkIO = io - dontTouch(io) - - // Instantiate Im2col - val im2colUnit: Instance[Im2col] = Instantiate(new Im2col(b)) - - // Connect command interface - im2colUnit.io.cmdReq <> io.cmdReq - im2colUnit.io.cmdResp <> io.cmdResp - - for (i <- 0 until inBW) { - im2colUnit.io.bankRead(i) <> io.bankRead(i) - } - - // Connect SRAM write interface - Im2col needs to write to scratchpad - for (i <- 0 until outBW) { - im2colUnit.io.bankWrite(i) <> io.bankWrite(i) - } - - // Connect Status signals - directly obtained from internal unit - io.status <> im2colUnit.io.status - - // Ball does not use SubROB: tie off subRobReq - io.subRobReq.valid := false.B - io.subRobReq.bits := SubRobRow.tieOff(b) - - // MMIO: this Ball does not consume MMIO metadata - MmioRead.tieOff(io.mmioRead) -} diff --git a/examples/balls/im2col/arch/src/main/scala/Im2colConfigRegs.scala b/examples/balls/im2col/arch/src/main/scala/Im2colConfigRegs.scala deleted file mode 100644 index 1a5e29f09..000000000 --- a/examples/balls/im2col/arch/src/main/scala/Im2colConfigRegs.scala +++ /dev/null @@ -1,95 +0,0 @@ -package examples.balls.im2col - -import chisel3._ -import chisel3.util._ -import framework.balldomain.rs.BallRsIssue -import framework.top.GlobalConfig - -class Im2colConfigRegs( - val b: GlobalConfig, - maxIter: Int, - maxKSize: Int, - maxPadding: Int) - extends Module { - - private val kW = log2Ceil(maxKSize + 1) - - val io = IO(new Bundle { - val cmd = Input(new BallRsIssue(b)) - val load = Input(Bool()) - val invalid = Output(Bool()) - val robId = Output(UInt(log2Up(b.frontend.rob_entries).W)) - val isSub = Output(Bool()) - val subRobId = Output(UInt(log2Up(b.frontend.sub_rob_depth * 4).W)) - val rBank = Output(UInt(b.memDomain.vbankIdWidth.W)) - val wBank = Output(UInt(b.memDomain.vbankIdWidth.W)) - val inputSize = Output(UInt(16.W)) - val kernel = Output(UInt(kW.W)) - val stride = Output(UInt(8.W)) - val padding = Output(UInt(8.W)) - val startRow = Output(UInt(8.W)) - val startCol = Output(UInt(8.W)) - val inputBase = Output(UInt(6.W)) - val lane = Output(UInt(4.W)) - val windowStart = Output(UInt(6.W)) - val windowCount = Output(UInt(7.W)) - }) - - private val robId = RegInit(0.U(log2Up(b.frontend.rob_entries).W)) - private val isSub = RegInit(false.B) - private val subRobId = RegInit(0.U(log2Up(b.frontend.sub_rob_depth * 4).W)) - private val rBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - private val wBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - private val inputSize = RegInit(0.U(16.W)) - private val kernel = RegInit(0.U(kW.W)) - private val stride = RegInit(0.U(8.W)) - private val padding = RegInit(0.U(8.W)) - private val startRow = RegInit(0.U(8.W)) - private val startCol = RegInit(0.U(8.W)) - private val inputBase = RegInit(0.U(6.W)) - private val lane = RegInit(0.U(4.W)) - private val windowStart = RegInit(0.U(6.W)) - private val windowCount = RegInit(0.U(7.W)) - - when(io.load) { - robId := io.cmd.rob_id - isSub := io.cmd.is_sub - subRobId := io.cmd.sub_rob_id - rBank := io.cmd.cmd.op1_bank - wBank := io.cmd.cmd.wr_bank - inputSize := io.cmd.cmd.iter - kernel := io.cmd.cmd.rs2(7, 0)(kW - 1, 0) - stride := io.cmd.cmd.rs2(15, 8) - padding := io.cmd.cmd.rs2(23, 16) - startCol := io.cmd.cmd.rs2(31, 24) - startRow := io.cmd.cmd.rs2(39, 32) - inputBase := io.cmd.cmd.rs2(45, 40) - lane := io.cmd.cmd.rs2(49, 46) - windowStart := io.cmd.cmd.rs2(55, 50) - windowCount := io.cmd.cmd.rs2(62, 56) - } - - private val padded = inputSize +& (padding << 1) - io.invalid := inputSize === 0.U || inputSize > maxIter.U || - kernel === 0.U || kernel > maxKSize.U || stride === 0.U || - padding > maxPadding.U || startRow > padding || startCol > padding || - inputBase +& inputSize * inputSize > b.memDomain.bankEntries.U || - padded < kernel + startRow || padded < kernel + startCol || - windowCount === 0.U || rBank === wBank - - io.robId := robId - io.isSub := isSub - io.subRobId := subRobId - io.rBank := rBank - io.wBank := wBank - io.inputSize := inputSize - io.kernel := kernel - io.stride := stride - io.padding := padding - io.startRow := startRow - io.startCol := startCol - io.inputBase := inputBase - io.lane := lane - io.windowStart := windowStart - io.windowCount := windowCount -} diff --git a/examples/balls/im2col/arch/src/main/scala/Im2colWindow.scala b/examples/balls/im2col/arch/src/main/scala/Im2colWindow.scala deleted file mode 100644 index 2187d4a11..000000000 --- a/examples/balls/im2col/arch/src/main/scala/Im2colWindow.scala +++ /dev/null @@ -1,68 +0,0 @@ -package examples.balls.im2col - -import chisel3._ -import chisel3.util._ - -class Im2colWindow(maxK: Int) extends Module { - private val kW = log2Ceil(maxK + 1) - - val io = IO(new Bundle { - val init = Input(Bool()) - val next = Input(Bool()) - val elemFire = Input(Bool()) - val kernel = Input(UInt(kW.W)) - val outputCols = Input(UInt(16.W)) - val startRow = Input(UInt(16.W)) - val startCol = Input(UInt(16.W)) - val windowCount = Input(UInt(12.W)) - val outRow = Output(UInt(16.W)) - val outCol = Output(UInt(16.W)) - val kRowIdx = Output(UInt(kW.W)) - val kColIdx = Output(UInt(kW.W)) - val elemLast = Output(Bool()) - val last = Output(Bool()) - }) - - private val outRowReg = RegInit(0.U(16.W)) - private val outColReg = RegInit(0.U(16.W)) - private val kRowReg = RegInit(0.U(kW.W)) - private val kColReg = RegInit(0.U(kW.W)) - private val colsReg = RegInit(0.U(16.W)) - private val remaining = RegInit(0.U(12.W)) - - private val elemLast = kRowReg === io.kernel - 1.U && kColReg === io.kernel - 1.U - - when(io.init) { - outRowReg := io.startRow - outColReg := io.startCol - kRowReg := 0.U - kColReg := 0.U - colsReg := io.outputCols - remaining := io.windowCount - }.elsewhen(io.next) { - assert(remaining > 1.U, "Im2colWindow cannot advance past the selected range") - kRowReg := 0.U - kColReg := 0.U - remaining := remaining - 1.U - when(outColReg + 1.U === colsReg) { - outRowReg := outRowReg + 1.U - outColReg := 0.U - }.otherwise { - outColReg := outColReg + 1.U - } - }.elsewhen(io.elemFire && !elemLast) { - when(kColReg === io.kernel - 1.U) { - kColReg := 0.U - kRowReg := kRowReg + 1.U - }.otherwise { - kColReg := kColReg + 1.U - } - } - - io.outRow := outRowReg - io.outCol := outColReg - io.kRowIdx := kRowReg - io.kColIdx := kColReg - io.elemLast := elemLast - io.last := remaining === 1.U -} diff --git a/examples/balls/im2col/arch/src/main/scala/LineBufferManager.scala b/examples/balls/im2col/arch/src/main/scala/LineBufferManager.scala deleted file mode 100644 index f22ef3c8c..000000000 --- a/examples/balls/im2col/arch/src/main/scala/LineBufferManager.scala +++ /dev/null @@ -1,160 +0,0 @@ -package examples.balls.im2col - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.balldomain.blink.BankRead -import examples.balls.im2col.configs.Im2colBallParam -import framework.top.GlobalConfig - -@instantiable -class LineBufferManager(val b: GlobalConfig) extends Module { - private val ballCfg = Im2colBallParam(b) - private val maxIter = ballCfg.maxIter - private val maxKSize = ballCfg.maxKSize - private val elemWidth = ballCfg.inputWidth - private val bankWidth = b.memDomain.bankWidth - private val lanesPerBeat = bankWidth / elemWidth - require(isPow2(lanesPerBeat), "LineBuffer lanesPerBeat must be power of 2") - private val maxBeats = maxIter * maxIter - require(isPow2(maxBeats) && maxBeats >= 2, "LineBuffer depth must be an even power of two") - private val halfBeats = maxBeats / 2 - // FIXME: This is a local line-buffer address, sized for maxIter^2; it must not use - // the external bank address width. Each command's external footprint is - // checked separately against bankEntries in Im2colConfigRegs. - private val memoryAddrW = log2Ceil(maxBeats) - private val kW = log2Ceil(maxKSize + 1) - private val addrW = log2Ceil(b.memDomain.bankEntries) - - private val map = b.ballDomain.ballIdMappings - .find(_.ballName == "Im2colBall") - .getOrElse( - throw new IllegalArgumentException("Im2colBall not found in config") - ) - - private val inBW = map.inBW - - @public - val io = IO(new Bundle { - val bankRead = Vec(inBW, Flipped(new BankRead(b))) - val start = Input(Bool()) - val inRows = Input(UInt(16.W)) - val inCols = Input(UInt(16.W)) - val rowStride = Input(UInt(8.W)) - val colStride = Input(UInt(8.W)) - val padding = Input(UInt(8.W)) - val startRow = Input(UInt(8.W)) - val startCol = Input(UInt(8.W)) - val inputBase = Input(UInt(6.W)) - val lane = Input(UInt(log2Ceil(lanesPerBeat).W)) - val outRow = Input(UInt(16.W)) - val outCol = Input(UInt(16.W)) - val kRowIdx = Input(UInt(kW.W)) - val kColIdx = Input(UInt(kW.W)) - val rBankId = Input(UInt(b.memDomain.vbankIdWidth.W)) - val robId = Input(UInt(log2Up(b.frontend.rob_entries).W)) - val loadDone = Output(Bool()) - val elemData = Output(UInt(elemWidth.W)) - }) - - private val bufLo = SyncReadMem(halfBeats, UInt(bankWidth.W)) - private val bufHi = SyncReadMem(halfBeats, UInt(bankWidth.W)) - private val active = RegInit(false.B) - private val pending = RegInit(false.B) - private val beat = RegInit(0.U(addrW.W)) - private val dimW = log2Ceil(maxIter + 1) - private val beatsReg = RegInit(0.U((addrW + 1).W)) - - for (i <- 0 until inBW) { - io.bankRead(i).io.req.valid := false.B - io.bankRead(i).io.req.bits.addr := 0.U - io.bankRead(i).io.resp.ready := false.B - io.bankRead(i).bank_id := io.rBankId - io.bankRead(i).rob_id := io.robId - io.bankRead(i).ball_id := 0.U - io.bankRead(i).group_id := 0.U - } - - io.bankRead(0).io.req.valid := active && !pending - io.bankRead(0).io.req.bits.addr := (io.inputBase +& beat).asTypeOf(UInt(addrW.W)) - io.bankRead(0).io.resp.ready := pending - io.loadDone := !active - - when(io.start) { - val rows = io.inRows(dimW - 1, 0) - val cols = io.inCols(dimW - 1, 0) - beatsReg := rows * cols - active := true.B - pending := false.B - beat := 0.U - }.elsewhen(io.bankRead(0).io.req.fire) { - pending := true.B - } - - when(io.bankRead(0).io.resp.fire) { - pending := false.B - when(beat +& 1.U === beatsReg) { - active := false.B - }.otherwise { - beat := beat + 1.U - } - } - - private val paddedRow = io.startRow +& (io.outRow(dimW - 1, 0) * io.rowStride) +& io.kRowIdx - private val paddedCol = io.startCol +& (io.outCol(dimW - 1, 0) * io.colStride) +& io.kColIdx - private val rowValid = - paddedRow >= io.padding && paddedRow < io.padding + io.inRows - private val colValid = - paddedCol >= io.padding && paddedCol < io.padding + io.inCols - private val inBound = rowValid && colValid - private val srcRow = Mux(inBound, paddedRow - io.padding, 0.U) - private val srcCol = Mux(inBound, paddedCol - io.padding, 0.U) - private val elemIndex = srcRow(dimW - 1, 0) * io.inCols(dimW - 1, 0) + srcCol(dimW - 1, 0) - private val beatIndex = elemIndex - private val laneIndex = io.lane - private val memoryWrite = io.bankRead(0).io.resp.fire - private val memoryRead = !active - - private val memoryAddress = Mux( - memoryWrite, - beat.pad(memoryAddrW), - beatIndex(memoryAddrW - 1, 0) - ) - - private val memoryBank = memoryAddress(memoryAddrW - 1) - private val memoryRow = memoryAddress(log2Ceil(halfBeats) - 1, 0) - private val memoryEnable = memoryWrite || memoryRead - - private val loData = bufLo.readWrite( - memoryRow, - io.bankRead(0).io.resp.bits.data.asUInt, - memoryEnable && !memoryBank, - memoryWrite - ) - - private val hiData = bufHi.readWrite( - memoryRow, - io.bankRead(0).io.resp.bits.data.asUInt, - memoryEnable && memoryBank, - memoryWrite - ) - - private val readBank = RegEnable(memoryBank, false.B, memoryRead) - private val memoryData = Mux(readBank, hiData, loData) - private val laneR = RegInit(0.U(log2Ceil(lanesPerBeat).W)) - private val boundR = RegInit(false.B) - private val beatData = RegInit(0.U(bankWidth.W)) - private val laneD = RegInit(0.U(log2Ceil(lanesPerBeat).W)) - private val boundD = RegInit(false.B) - private val elemHold = RegInit(0.U(elemWidth.W)) - laneR := laneIndex - boundR := inBound - beatData := memoryData - laneD := laneR - boundD := boundR - private val lanes = - beatData.asTypeOf(Vec(lanesPerBeat, UInt(elemWidth.W))) - elemHold := Mux(boundD, lanes(laneD), 0.U) - - io.elemData := elemHold -} diff --git a/examples/balls/im2col/arch/src/main/scala/README.md b/examples/balls/im2col/arch/src/main/scala/README.md deleted file mode 100644 index c246a8dad..000000000 --- a/examples/balls/im2col/arch/src/main/scala/README.md +++ /dev/null @@ -1,68 +0,0 @@ -# Im2col Ball - -This directory implements the Buckyball `Im2colBall` RTL for instruction -function 48. - -## ISA contract - -```c -bb_im2col(input_bank, output_bank, iter, ksize, stride, padding) -``` - -- `iter` is both the input height and input width. -- `ksize` is both the kernel height and kernel width. -- `stride` applies to output rows and columns. -- `padding` is zero-padding on every input edge. -- Input and output elements are signed or unsigned 8-bit payload bytes; Im2col - only rearranges bytes and does not multiply or accumulate them. - -The output dimension is: - -```text -outputDim = floor((iter + 2 * padding - ksize) / stride) + 1 -``` - -Windows form the rows of SMatMulBall operand A. The kernel elements form its K -dimension. Output is stored in SMatMulBall's `(M tile, K tile, tile row, lane)` -layout, with both tile dimensions fixed at 16: - -```text -M = outputDim * outputDim -K = ksize * ksize -mTiles = ceil(M / 16) -kTiles = ceil(K / 16) - -bankRow(window, kernel) = - ((window / 16) * kTiles + (kernel / 16)) * 16 + (window % 16) -lane(window, kernel) = kernel % 16 -``` - -Every bank row contains 16 int8 lanes. Unused lanes in the final K tile are -zero. Consequently each M tile occupies `kTiles * 16` bank rows; a kernel wider -than 16 elements continues in the next K tile rather than immediately after -the preceding window. - -For output coordinate `(orow, ocol)` and kernel coordinate `(krow, kcol)`, the -unpadded input coordinate is: - -```text -inputRow = orow * stride + krow - padding -inputCol = ocol * stride + kcol - padding -``` - -An out-of-range coordinate emits zero. Otherwise the source byte offset is: - -```text -inputRow * iter + inputCol -``` - -## Files - -- `Im2col.scala`: top-level command and response control. -- `Im2colConfigRegs.scala`: ISA field decoding and validation. -- `Im2colWindow.scala`: output-window and kernel traversal. -- `LineBufferManager.scala`: input preload and padded element selection. -- `StreamWriter.scala`: SMatMulBall-compatible tiled output packing and bank - writes. -- `Im2colBall.scala`: Blink-compatible ball wrapper. -- `configs/Im2colBallParam.scala`: maximum dimension and element width. diff --git a/examples/balls/im2col/arch/src/main/scala/StreamWriter.scala b/examples/balls/im2col/arch/src/main/scala/StreamWriter.scala deleted file mode 100644 index f2c698eaf..000000000 --- a/examples/balls/im2col/arch/src/main/scala/StreamWriter.scala +++ /dev/null @@ -1,229 +0,0 @@ -package examples.balls.im2col - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} - -import framework.balldomain.blink.BankWrite -import framework.top.GlobalConfig -import examples.balls.im2col.configs.Im2colBallParam - -/** - * Packs im2col K tiles into 128-bit bank rows (SMatMulBall A layout). - * - * ((w / 16) * ceil(kernelElems / 16) + kt) * 16 + (w % 16) - * - * Partial final K-tile lanes and partial final M-tile rows are zero-filled. - */ -@instantiable -class StreamWriter(val b: GlobalConfig) extends Module { - private val ballCfg = Im2colBallParam(b) - private val maxKSize = ballCfg.maxKSize - private val elemWidth = ballCfg.inputWidth - private val bankWidth = b.memDomain.bankWidth - private val bankEntries = b.memDomain.bankEntries - private val lanesPerBeat = bankWidth / elemWidth - private val laneIdxWidth = log2Ceil(lanesPerBeat) - private val kW = log2Ceil(maxKSize + 1) - private val bankAddrW = log2Ceil(bankEntries) - private val maxKTiles = - (maxKSize * maxKSize + lanesPerBeat - 1) / lanesPerBeat - private val kTilesW = log2Ceil(maxKTiles + 1) - require( - isPow2(bankEntries), - "StreamWriter requires power-of-two bank depth" - ) - - private val mapping = b.ballDomain.ballIdMappings - .find(_.ballName == "Im2colBall") - .getOrElse( - throw new IllegalArgumentException("Im2colBall not found in config") - ) - - private val outBW = mapping.outBW - - require(outBW == 1, "StreamWriter requires outBW=1") - require(bankWidth % elemWidth == 0) - require(lanesPerBeat == 16, "Im2col SMatMulBall layout requires 16 int8 lanes") - - @public - val io = IO(new Bundle { - val bankWrite = Vec(outBW, Flipped(new BankWrite(b))) - - val elemIn = Flipped(Decoupled(UInt(elemWidth.W))) - val elemLast = Input(Bool()) - val finalWindow = Input(Bool()) - val init = Input(Bool()) - val windowIdx = Input(UInt(32.W)) - val kRows = Input(UInt(kW.W)) - val kCols = Input(UInt(kW.W)) - val wBankId = Input(UInt(b.memDomain.vbankIdWidth.W)) - val robId = Input(UInt(log2Up(b.frontend.rob_entries).W)) - - val busy = Output(Bool()) - val windowComplete = Output(Bool()) - val opComplete = Output(Bool()) - }) - - private val zeros = VecInit(Seq.fill(lanesPerBeat)(0.U(elemWidth.W))) - - private val packCntReg = RegInit(0.U(log2Ceil(lanesPerBeat + 1).W)) - - private val packReg = RegInit( - VecInit(Seq.fill(lanesPerBeat)(0.U(elemWidth.W))) - ) - - private val chunkIdxReg = RegInit(0.U(kTilesW.W)) - private val wrPendingReg = RegInit(false.B) - private val issued = RegInit(false.B) - private val endWindowReg = RegInit(false.B) - private val finalWinReg = RegInit(false.B) - private val padActive = RegInit(false.B) - private val padMTile = RegInit(0.U(32.W)) - private val padKTiles = RegInit(0.U(kTilesW.W)) - private val padK = RegInit(0.U(kTilesW.W)) - private val padRow = RegInit(0.U(laneIdxWidth.W)) - private val padRow0 = RegInit(0.U(laneIdxWidth.W)) - private val wAddrReg = RegInit(0.U(bankAddrW.W)) - - private val kernelElems = io.kRows * io.kCols - private val kTilesReg = RegInit(0.U(kTilesW.W)) - - private val mTile = io.windowIdx >> laneIdxWidth - private val mRow = io.windowIdx(laneIdxWidth - 1, 0) - - private def tileOff(tile: UInt, kTiles: UInt): UInt = { - MuxLookup(kTiles, 0.U)( - Seq( - 1.U -> tile, - 2.U -> (tile << 1), - 3.U -> ((tile << 1) + tile), - 4.U -> (tile << 2) - ) - ) - } - - private val targetAddr = - ((tileOff(mTile, kTilesReg) + chunkIdxReg) << laneIdxWidth) + mRow - private val padAddr = - ((tileOff(padMTile, padKTiles) + padK) << laneIdxWidth) + padRow - private val writeFire = io.bankWrite(0).io.req.fire - private val writeResp = io.bankWrite(0).io.resp.fire - - io.busy := wrPendingReg || padActive - io.windowComplete := wrPendingReg && endWindowReg && writeResp && !finalWinReg - io.opComplete := Mux( - padActive, - wrPendingReg && writeResp && - (padK + 1.U === padKTiles) && (padRow === (lanesPerBeat - 1).U), - wrPendingReg && endWindowReg && writeResp && finalWinReg && - (mRow === (lanesPerBeat - 1).U) - ) - - for (i <- 0 until outBW) { - io.bankWrite(i).io.req.valid := false.B - io.bankWrite(i).io.req.bits.addr := 0.U - io.bankWrite(i).io.req.bits.data := 0.U - io.bankWrite(i).io.req.bits.mask := - VecInit(Seq.fill(b.memDomain.bankMaskLen)(false.B)) - io.bankWrite(i).io.resp.ready := false.B - io.bankWrite(i).bank_id := 0.U - io.bankWrite(i).rob_id := 0.U - io.bankWrite(i).ball_id := 0.U - io.bankWrite(i).group_id := 0.U - } - - io.bankWrite(0).io.req.valid := wrPendingReg && !issued - io.bankWrite(0).io.req.bits.addr := wAddrReg - io.bankWrite(0).io.req.bits.data := Cat(packReg.reverse) - io.bankWrite(0).io.req.bits.mask := - VecInit(Seq.fill(b.memDomain.bankMaskLen)(true.B)) - io.bankWrite(0).io.resp.ready := true.B - io.bankWrite(0).bank_id := io.wBankId - io.bankWrite(0).rob_id := io.robId - io.bankWrite(0).group_id := 0.U - - io.elemIn.ready := !wrPendingReg && !padActive - - when(io.init) { - packCntReg := 0.U - packReg := zeros - chunkIdxReg := 0.U - wrPendingReg := false.B - issued := false.B - endWindowReg := false.B - finalWinReg := false.B - wAddrReg := 0.U - padActive := false.B - padMTile := 0.U - padKTiles := 0.U - padK := 0.U - padRow := 0.U - padRow0 := 0.U - kTilesReg := ((kernelElems +& (lanesPerBeat - 1).U) >> laneIdxWidth) - .asTypeOf(UInt(kTilesW.W)) - assert( - ((kernelElems +& (lanesPerBeat - 1).U) >> laneIdxWidth) >= 1.U && - ((kernelElems +& (lanesPerBeat - 1).U) >> laneIdxWidth) <= 4.U, - "StreamWriter kTiles must be 1..4" - ) - }.otherwise { - when(writeFire) { - issued := true.B - } - when(writeResp) { - packCntReg := 0.U - packReg := zeros - wrPendingReg := false.B - issued := false.B - endWindowReg := false.B - when(padActive) { - when(padRow === (lanesPerBeat - 1).U) { - when(padK + 1.U === padKTiles) { - padActive := false.B - padK := 0.U - padRow := 0.U - }.otherwise { - padK := padK + 1.U - padRow := padRow0 - } - }.otherwise { - padRow := padRow + 1.U - } - }.elsewhen(endWindowReg) { - chunkIdxReg := 0.U - when(finalWinReg && (mRow =/= (lanesPerBeat - 1).U)) { - padActive := true.B - padMTile := mTile - padKTiles := kTilesReg - padK := 0.U - padRow := mRow + 1.U - padRow0 := mRow + 1.U - } - finalWinReg := false.B - }.otherwise { - chunkIdxReg := chunkIdxReg + 1.U - } - } - - when(padActive && !wrPendingReg) { - wrPendingReg := true.B - endWindowReg := false.B - finalWinReg := false.B - packReg := zeros - wAddrReg := padAddr.asTypeOf(UInt(bankAddrW.W)) - } - - when(io.elemIn.fire) { - packReg(packCntReg(laneIdxWidth - 1, 0)) := io.elemIn.bits - val nextCnt = packCntReg + 1.U - packCntReg := nextCnt - when(nextCnt === lanesPerBeat.U || io.elemLast) { - wrPendingReg := true.B - endWindowReg := io.elemLast - finalWinReg := io.elemLast && io.finalWindow - wAddrReg := targetAddr.asTypeOf(UInt(bankAddrW.W)) - } - } - } -} diff --git a/examples/balls/im2col/compiler/src/Backend/include/IntrinsicsRISCVIm2colBall.td b/examples/balls/im2col/compiler/src/Backend/include/IntrinsicsRISCVIm2colBall.td deleted file mode 100644 index 52037c087..000000000 --- a/examples/balls/im2col/compiler/src/Backend/include/IntrinsicsRISCVIm2colBall.td +++ /dev/null @@ -1,4 +0,0 @@ -//===- IntrinsicsRISCVIm2colBall.td - Im2colBall intrinsics --------------===// - -let TargetPrefix = "riscv" in -def int_riscv_bb_im2col : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; diff --git a/examples/balls/im2col/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp b/examples/balls/im2col/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp deleted file mode 100644 index 8f11d9c92..000000000 --- a/examples/balls/im2col/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp +++ /dev/null @@ -1,41 +0,0 @@ -//===- AssignPhysicalBankPatterns.cpp - Im2col bank assignment patterns ---===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace mlir::buddy { -void populateIm2colBallAssignPhysicalBankPatterns(RewritePatternSet &patterns, - PhysicalBankState &state); -} // namespace mlir::buddy - -namespace { - -class BankIm2colPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(BankIm2colOp op, - PatternRewriter &rewriter) const override { - rewriter.create(op.getLoc(), op.getInBank(), op.getOutBank(), - op.getIter(), op.getKsize(), op.getStride(), - op.getPadding(), op.getInputBase(), op.getLane(), - op.getStartRowAttr(), op.getStartColAttr(), - op.getWindowStartAttr(), op.getWindowCountAttr()); - rewriter.replaceOp(op, op.getOutBank()); - return success(); - } -}; - -} // namespace - -void mlir::buddy::populateIm2colBallAssignPhysicalBankPatterns( - RewritePatternSet &patterns, mlir::buddy::PhysicalBankState &state) { - (void)state; - patterns.add(patterns.getContext()); -} diff --git a/examples/balls/im2col/compiler/src/Dialect/Buckyball/Im2colBall.td b/examples/balls/im2col/compiler/src/Dialect/Buckyball/Im2colBall.td deleted file mode 100644 index 450922bd5..000000000 --- a/examples/balls/im2col/compiler/src/Dialect/Buckyball/Im2colBall.td +++ /dev/null @@ -1,133 +0,0 @@ -//===-- Im2colBall.td - Im2colBall operations --------------*- tablegen -*-===// -// -// Im2colBall operations for image-to-column transformation. -// -//===----------------------------------------------------------------------===// - -#ifndef IM2COLBALL_OPS -#define IM2COLBALL_OPS - -include "Buckyball.td" - -//===----------------------------------------------------------------------===// -// Im2colBall operations -//===----------------------------------------------------------------------===// - -// Memref-level im2col+gemm (like smatmul_matmul). Chip bank-ssa decides -// f32 quant / Bank* composition; this op carries no quant policy. -def Im2colMatmulOp : Buckyball_Op<"im2col_matmul"> { - let summary = "Packed-plane im2col then matrix gemm"; - let description = [{ - One cin-plane im2col, then gemm over all N columns of `filter`. - `n` is OC (padded to a multiple of 16, 16..4096). `input` is one packed - HxW plane (bank rows x 16). `filter`/`output` are KxN and MxN. - Asym pad co-design: padding=max(lo,hi), startRow/Col=padding-lo. - }]; - let arguments = (ins MemRefRankOf<[AnyType], [2]>:$input, - MemRefRankOf<[AnyType], [2]>:$filter, - MemRefRankOf<[AnyType], [2]>:$output, - I64Attr:$inSize, - I64Attr:$ksize, - I64Attr:$n, - DefaultValuedAttr:$stride, - DefaultValuedAttr:$padding, - DefaultValuedAttr:$startRow, - DefaultValuedAttr:$startCol, - DefaultValuedAttr:$dwAddr, - DefaultValuedAttr:$dwBytes, - DefaultValuedAttr:$perChannel); - let assemblyFormat = [{$input $filter $output prop-dict attr-dict `:` - type($input) type($filter) type($output)}]; -} - -// Standard conv: nCin input planes -> one im2col+gemm with K = nCin*K^2. -def Im2colFatMatmulOp : Buckyball_Op<"im2col_fat_matmul"> { - let summary = "Multi-plane im2col then fat gemm over nCin input channels"; - let description = [{ - Stack $nCin input planes as [nCin*inRows, 16] and filters as - [nCin*bRows, n]. Im2col each plane, concatenate on K, then one - (wins x nCin*K^2) @ (nCin*K^2 x n) gemm. $n is OC padded to 16. - }]; - let arguments = (ins MemRefRankOf<[AnyType], [2]>:$input, - MemRefRankOf<[AnyType], [2]>:$filter, - MemRefRankOf<[AnyType], [2]>:$output, - I64Attr:$inSize, - I64Attr:$ksize, - I64Attr:$n, - I64Attr:$nCin, - DefaultValuedAttr:$stride, - DefaultValuedAttr:$padding, - DefaultValuedAttr:$startRow, - DefaultValuedAttr:$startCol, - DefaultValuedAttr:$dwAddr, - DefaultValuedAttr:$dwBytes, - DefaultValuedAttr:$perChannel); - let assemblyFormat = [{$input $filter $output prop-dict attr-dict `:` - type($input) type($filter) type($output)}]; -} - -// Depthwise: n channels (1..16) share one fat GEMM, not per-channel GEMV. -// Input planes stacked as [n*inRows, 16]; filter is block-diag [n*bRows, 16]. -def Im2colDepthwiseMatmulOp : Buckyball_Op<"im2col_depthwise_matmul"> { - let summary = "Multi-plane im2col then fat depthwise gemm (N=lane)"; - let description = [{ - For channel tile size $n (1..16): - im2col each plane, concatenate on K, then (wins x n*K^2) @ (n*K^2 x n) - with block-diagonal weights. Lane-full GEMM; not GEMV. - }]; - let arguments = (ins MemRefRankOf<[AnyType], [2]>:$input, - MemRefRankOf<[AnyType], [2]>:$filter, - MemRefRankOf<[AnyType], [2]>:$output, - I64Attr:$inSize, - I64Attr:$ksize, - I64Attr:$n, - DefaultValuedAttr:$stride, - DefaultValuedAttr:$padding, - DefaultValuedAttr:$startRow, - DefaultValuedAttr:$startCol, - DefaultValuedAttr:$dwAddr, - DefaultValuedAttr:$dwBytes, - DefaultValuedAttr:$perChannel); - let assemblyFormat = [{$input $filter $output prop-dict attr-dict `:` - type($input) type($filter) type($output)}]; -} - -def Im2colOp : Buckyball_Op<"im2col"> { - let summary = "Image-to-column transformation for convolution"; - let description = [{ - Transforms image data into column format for efficient convolution. - Each input bank row is one spatial position with 16 INT8 channels. - inputBase selects the first spatial row and lane selects one channel. - windowStart/windowCount select a contiguous output-window range and rebase - it to row zero in the output bank. - Legal range is defined in examples/balls/im2col/configs/default.toml. - }]; - let arguments = (ins I64:$inputBankId, I64:$outputBankId, - I64:$iter, I64:$ksize, I64:$stride, I64:$padding, - I64:$inputBase, I64:$lane, - I64Attr:$startRow, I64Attr:$startCol, - I64Attr:$windowStart, I64Attr:$windowCount); - let assemblyFormat = "$inputBankId `,` $outputBankId `,` " - "$iter `,` $ksize `,` $stride `,` $padding `,` " - "$inputBase `,` $lane prop-dict attr-dict `:` " - "type($inputBankId)"; -} - -def BankIm2colOp : Buckyball_Op<"bank_im2col"> { - let summary = "Im2col using virtual bank handles"; - let arguments = (ins I64:$inBank, I64:$outBank, I64:$iter, I64:$ksize, - I64:$stride, I64:$padding, - I64:$inputBase, I64:$lane, - I64Attr:$startRow, I64Attr:$startCol, - I64Attr:$windowStart, I64Attr:$windowCount); - let results = (outs I64:$outBankOut); - let assemblyFormat = - "$inBank $outBank $iter $ksize $stride $padding $inputBase $lane " - "prop-dict attr-dict `:` type($inBank) type($outBank) type($iter) type($ksize) " - "type($stride) type($padding) type($inputBase) type($lane)"; -} - -def Im2colIntrOp : Buckyball_IntrOpBase<"im2col">, - Arguments<(ins LLVM_Type, LLVM_Type)>; - -#endif // IM2COLBALL_OPS diff --git a/examples/balls/im2col/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/balls/im2col/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index a38a3c5e5..000000000 --- a/examples/balls/im2col/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,234 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/IR/PatternMatch.h" - -#include - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace buddy::buckyball; -using namespace buddy::buckyball::legalize; - -namespace { -static LogicalResult -validateIm2colEncoding(Operation *op, Value iterValue, Value ksizeValue, - Value strideValue, Value paddingValue, - Value inputBaseValue, Value laneValue, int64_t startRow, - int64_t startCol, int64_t windowStart, - int64_t windowCount, int64_t bankDepth) { - auto getConst = [](Value value) -> std::optional { - if (auto constant = value.getDefiningOp()) - if (auto attr = dyn_cast(constant.getValue())) - return attr.getInt(); - return std::nullopt; - }; - auto iter = getConst(iterValue); - auto ksize = getConst(ksizeValue); - auto stride = getConst(strideValue); - auto padding = getConst(paddingValue); - auto inputBase = getConst(inputBaseValue); - auto lane = getConst(laneValue); - if (!iter || !ksize || !stride || !padding) - return op->emitError( - "Im2col instruction requires constant iter/ksize/stride/padding"); - if (*iter <= 0 || *ksize <= 0 || *ksize > 255 || *stride <= 0 || - *stride > 255 || *padding < 0 || *padding > 255 || startRow < 0 || - startRow > 255 || startCol < 0 || startCol > 255 || - (inputBase && (*inputBase < 0 || *inputBase >= 64)) || - (lane && (*lane < 0 || *lane >= 16))) - return op->emitError("Im2col instruction field out of range"); - int64_t inputRows = *iter * *iter; - if (inputBase && *inputBase + inputRows > bankDepth) - return op->emitError("Im2col input range exceeds the physical bank depth"); - if (startRow > *padding || startCol > *padding) - return op->emitError("Im2col startRow/startCol must not exceed padding"); - if (windowStart < 0 || windowStart >= 64 || windowCount <= 0 || - windowCount > 64) - return op->emitError( - "Im2col requires windowStart in [0, 63] and windowCount in [1, 64]"); - - int64_t padded = *iter + 2 * *padding; - if (padded < *ksize + startRow || padded < *ksize + startCol || - (padded - *ksize - startRow) % *stride || - (padded - *ksize - startCol) % *stride) - return op->emitError("Im2col shape does not produce integral windows"); - int64_t rows = (padded - *ksize - startRow) / *stride + 1; - int64_t cols = (padded - *ksize - startCol) / *stride + 1; - if (windowStart + windowCount > rows * cols) - return op->emitError("Im2col window range exceeds output shape"); - int64_t kernelTiles = (*ksize * *ksize + 15) / 16; - int64_t outputRows = (windowCount + 15) / 16 * kernelTiles * 16; - if (outputRows > bankDepth) - return op->emitError("Im2col output range exceeds the physical bank depth"); - return success(); -} - -struct Im2colLowering : public ConvertOpToLLVMPattern { - Im2colLowering(LLVMTypeConverter &converter, bool stable, int64_t bankDepth) - : ConvertOpToLLVMPattern(converter), stable(stable), - bankDepth(bankDepth) {} - - LogicalResult - matchAndRewrite(Im2colOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("Im2colBall"); - if (failed(validateIm2colEncoding( - op, op.getIter(), op.getKsize(), op.getStride(), op.getPadding(), - op.getInputBase(), op.getLane(), op.getStartRow(), op.getStartCol(), - op.getWindowStart(), op.getWindowCount(), bankDepth))) - return failure(); - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInputBankId(), - cstI64(rewriter, loc, 0), - adaptor.getOutputBankId(), adaptor.getIter()); - - Value rs2 = adaptor.getKsize(); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getStride(), - cstI64(rewriter, loc, 8))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getPadding(), - cstI64(rewriter, loc, 16))); - Value startCol = cstI64(rewriter, loc, op.getStartCol()); - Value startRow = cstI64(rewriter, loc, op.getStartRow()); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, startCol, - cstI64(rewriter, loc, 24))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, startRow, - cstI64(rewriter, loc, 32))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getInputBase(), - cstI64(rewriter, loc, 40))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getLane(), - cstI64(rewriter, loc, 46))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create( - loc, cstI64(rewriter, loc, op.getWindowStart()), - cstI64(rewriter, loc, 50))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create( - loc, cstI64(rewriter, loc, op.getWindowCount()), - cstI64(rewriter, loc, 56))); - - if (stable) { - rewriter.replaceOpWithNewOp(op, rs1, rs2); - return success(); - } - rewriter.replaceOpWithNewOp( - op, rs1, rs2, - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("IM2COL"))); - return success(); - } - -private: - bool stable = false; - int64_t bankDepth; -}; -struct BankIm2colLowering : public ConvertOpToLLVMPattern { - BankIm2colLowering(LLVMTypeConverter &converter, bool stable, - int64_t bankDepth) - : ConvertOpToLLVMPattern(converter), stable(stable), - bankDepth(bankDepth) {} - - LogicalResult - matchAndRewrite(BankIm2colOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("Im2colBall"); - if (failed(validateIm2colEncoding( - op, op.getIter(), op.getKsize(), op.getStride(), op.getPadding(), - op.getInputBase(), op.getLane(), op.getStartRow(), op.getStartCol(), - op.getWindowStart(), op.getWindowCount(), bankDepth))) - return failure(); - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInBank(), - cstI64(rewriter, loc, 0), adaptor.getOutBank(), - adaptor.getIter()); - - Value rs2 = adaptor.getKsize(); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getStride(), - cstI64(rewriter, loc, 8))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getPadding(), - cstI64(rewriter, loc, 16))); - Value startCol = cstI64(rewriter, loc, op.getStartCol()); - Value startRow = cstI64(rewriter, loc, op.getStartRow()); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, startCol, - cstI64(rewriter, loc, 24))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, startRow, - cstI64(rewriter, loc, 32))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getInputBase(), - cstI64(rewriter, loc, 40))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getLane(), - cstI64(rewriter, loc, 46))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create( - loc, cstI64(rewriter, loc, op.getWindowStart()), - cstI64(rewriter, loc, 50))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create( - loc, cstI64(rewriter, loc, op.getWindowCount()), - cstI64(rewriter, loc, 56))); - - if (stable) { - rewriter.replaceOpWithNewOp(op, rs1, rs2); - return success(); - } - rewriter.replaceOpWithNewOp( - op, rs1, rs2, - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("IM2COL"))); - return success(); - } - -private: - bool stable = false; - int64_t bankDepth; -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateIm2colBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth) { - patterns.add(converter, stable, bankDepth); - patterns.add(converter, stable, bankDepth); -} - -void configureIm2colBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable) { - if (stable) - target.addLegalOp(); - else - target.addIllegalOp(); - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/examples/balls/im2col/configs/Im2colBallParam.scala b/examples/balls/im2col/configs/Im2colBallParam.scala deleted file mode 100644 index 82b8b58ae..000000000 --- a/examples/balls/im2col/configs/Im2colBallParam.scala +++ /dev/null @@ -1,25 +0,0 @@ -package examples.balls.im2col.configs - -import framework.balldomain.configs.BallParamLoader -import framework.top.GlobalConfig - -case class Im2colBallParam( - maxIter: Int, - maxKSize: Int, - maxPadding: Int, - inputWidth: Int) - -object Im2colBallParam { - private val ballName = "Im2colBall" - - def apply(b: GlobalConfig): Im2colBallParam = { - val tbl = BallParamLoader.ballTable(b, ballName) - Im2colBallParam( - maxIter = BallParamLoader.int(tbl, "maxIter"), - maxKSize = BallParamLoader.int(tbl, "maxKSize"), - maxPadding = BallParamLoader.int(tbl, "maxPadding"), - inputWidth = BallParamLoader.int(tbl, "inputWidth") - ) - } - -} diff --git a/examples/balls/im2col/configs/default.toml b/examples/balls/im2col/configs/default.toml deleted file mode 100644 index 1316acf13..000000000 --- a/examples/balls/im2col/configs/default.toml +++ /dev/null @@ -1,29 +0,0 @@ -# Im2col Ball Configuration -# -# Legal command range (enforced fail-hard in bemu/RTL): -# iter in 1 ..= maxIter -# ksize in 1 ..= maxKSize # covers 1x1 / 3x3 / 5x5 / 7x7 -# stride >= 1 -# padding in 0 ..= maxPadding -# start_row / start_col in 0 ..= padding (asym pad: pad=max(lo,hi), start=pad-lo) -# paddedSize = iter + 2*padding >= ksize + start -# op1_bank != wr_bank -# inputBase + iter*iter <= bankEntries -# lane in 0 .. 15 -# outputDim = floor((iter + 2*padding - ksize - start) / stride) + 1 -# M = outputDim * outputDim -# K = ksize * ksize -# outputRows = ceil(M/16) * ceil(K/16) * 16 -# outputRows <= bankEntries # bankEntries from chip memdomain -# -# Partial final K-tile lanes and partial final M-tile rows are zero. -# One bank row is one spatial position with 16 INT8 channels. - -[ball] -inputWidth = 8 -# Resident Conv lowering can request a 4x4 output tile from a stride-2, -# kernel-7 producer, which needs a 13x13 local input. Keep this a power of two -# because the RTL line buffer splits maxIter^2 beats into two equal banks. -maxIter = 16 -maxKSize = 7 -maxPadding = 7 diff --git a/examples/balls/im2col/emu/src/dispatch.rs b/examples/balls/im2col/emu/src/dispatch.rs deleted file mode 100644 index 4128672a0..000000000 --- a/examples/balls/im2col/emu/src/dispatch.rs +++ /dev/null @@ -1,31 +0,0 @@ -use crate::inst::instruction::ExecContext; - -use super::im2col; - -const BALL_CLASS: &str = "examples.balls.im2col.Im2colBall"; - -pub fn execute_known( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, - ctx: &mut ExecContext, -) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("IM2COL") => Some(im2col::exec_im2col(xs1, xs2, ctx)), - Some(_) | None => None, - } -} - -pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("IM2COL") => Some(im2col::latency(xs1, xs2)), - Some(_) | None => None, - } -} diff --git a/examples/balls/im2col/emu/src/im2col.rs b/examples/balls/im2col/emu/src/im2col.rs deleted file mode 100644 index b70b5e008..000000000 --- a/examples/balls/im2col/emu/src/im2col.rs +++ /dev/null @@ -1,174 +0,0 @@ -use super::super::bank::{bank_lines, bank_row_bytes}; -use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; -use super::instruction::ExecContext; - -const TILE: usize = 16; -const BALL_CLASS: &str = "examples.balls.im2col.Im2colBall"; - -fn max_iter() -> usize { - crate::config::ball_domain::param(BALL_CLASS, "maxIter") -} - -fn max_kernel() -> usize { - crate::config::ball_domain::param(BALL_CLASS, "maxKSize") -} - -fn max_padding() -> usize { - crate::config::ball_domain::param(BALL_CLASS, "maxPadding") -} - -#[derive(Clone, Copy)] -struct Shape { - input_size: usize, - kernel: usize, - stride: usize, - padding: usize, - start_row: usize, - start_col: usize, - input_base: usize, - lane: usize, - window_start: usize, - window_count: usize, -} - -fn decode_shape(xs1: u64, xs2: u64) -> Shape { - Shape { - input_size: rs1_iter(xs1) as usize, - kernel: (xs2 & 0xff) as usize, - stride: ((xs2 >> 8) & 0xff) as usize, - padding: ((xs2 >> 16) & 0xff) as usize, - start_col: ((xs2 >> 24) & 0xff) as usize, - start_row: ((xs2 >> 32) & 0xff) as usize, - input_base: ((xs2 >> 40) & 0x3f) as usize, - lane: ((xs2 >> 46) & 0xf) as usize, - window_start: ((xs2 >> 50) & 0x3f) as usize, - window_count: ((xs2 >> 56) & 0x7f) as usize, - } -} - -fn dimensions(shape: Shape) -> (usize, usize, usize) { - if shape.input_size == 0 - || shape.input_size > max_iter() - || shape.kernel == 0 - || shape.kernel > max_kernel() - || shape.stride == 0 - || shape.padding > max_padding() - || shape.start_row > shape.padding - || shape.start_col > shape.padding - || shape.window_count == 0 - || shape.window_start >= 64 - || shape.window_count > 64 - { - panic!("im2col: illegal shape field"); - } - let padded = shape - .input_size - .checked_add(shape.padding * 2) - .unwrap_or_else(|| panic!("im2col: padded input overflow")); - if padded < shape.kernel + shape.start_row || padded < shape.kernel + shape.start_col { - panic!("im2col: kernel and start exceed padded input"); - } - let row_numerator = padded - shape.kernel - shape.start_row; - let col_numerator = padded - shape.kernel - shape.start_col; - if row_numerator % shape.stride != 0 || col_numerator % shape.stride != 0 { - panic!("im2col: output shape is not integral"); - } - let output_rows = row_numerator / shape.stride + 1; - let output_cols = col_numerator / shape.stride + 1; - let windows = output_rows - .checked_mul(output_cols) - .unwrap_or_else(|| panic!("im2col: output window count overflow")); - if shape.window_start >= windows || shape.window_start + shape.window_count > windows { - panic!("im2col: selected window range exceeds output shape"); - } - (output_rows, output_cols, windows) -} - -fn footprints(shape: Shape) -> (usize, usize) { - let input_values = shape - .input_size - .checked_mul(shape.input_size) - .unwrap_or_else(|| panic!("im2col: input footprint overflow")); - let kernel_values = shape.kernel * shape.kernel; - let input_rows = input_values; - let output_rows = shape.window_count.div_ceil(TILE) * kernel_values.div_ceil(TILE) * TILE; - if shape.input_base + input_rows > bank_lines() || output_rows > bank_lines() { - panic!("im2col: input or output footprint exceeds bank depth"); - } - (input_rows, output_rows) -} - -pub(crate) fn exec_im2col(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { - let input_bank = rs1_b0(xs1); - let output_bank = rs1_b2(xs1); - if rs1_b1(xs1) != 0 { - panic!("im2col: input bank 1 must be zero"); - } - if xs2 >> 63 != 0 { - panic!("im2col: rs2[63] must be zero"); - } - if input_bank == output_bank { - panic!("im2col: input and output banks must differ"); - } - if !ctx.config(input_bank).allocated || !ctx.config(output_bank).allocated { - panic!("im2col: bank not allocated"); - } - if ctx.config(input_bank).cols != 1 || ctx.config(output_bank).cols != 1 { - panic!("im2col: input and output banks must each have one column"); - } - - let shape = decode_shape(xs1, xs2); - let (_, output_cols, _) = dimensions(shape); - let (_, output_bank_rows) = footprints(shape); - let kernel_values = shape.kernel * shape.kernel; - let kernel_tiles = kernel_values.div_ceil(TILE); - let pi = pbank(ctx, input_bank); - let po = pbank(ctx, output_bank); - let (input, output) = ctx.banks.read_write(pi, po); - output[..output_bank_rows * bank_row_bytes()].fill(0); - - for local_window in 0..shape.window_count { - let global_window = shape.window_start + local_window; - let output_row = global_window / output_cols; - let output_col = global_window % output_cols; - for kernel_row in 0..shape.kernel { - for kernel_col in 0..shape.kernel { - let source_row = (shape.start_row + output_row * shape.stride + kernel_row) - as isize - - shape.padding as isize; - let source_col = (shape.start_col + output_col * shape.stride + kernel_col) - as isize - - shape.padding as isize; - if source_row < 0 - || source_col < 0 - || source_row >= shape.input_size as isize - || source_col >= shape.input_size as isize - { - continue; - } - let kernel_element = kernel_row * shape.kernel + kernel_col; - let bank_row = ((local_window / TILE) * kernel_tiles + kernel_element / TILE) - * TILE - + local_window % TILE; - let lane = kernel_element % TILE; - let source = source_row as usize * shape.input_size + source_col as usize; - output[bank_row * TILE + lane] = - input[(shape.input_base + source) * TILE + shape.lane]; - } - } - } - 0 -} - -pub(crate) fn latency(xs1: u64, xs2: u64) -> u64 { - if rs1_b1(xs1) != 0 { - panic!("im2col: input bank 1 must be zero"); - } - if xs2 >> 63 != 0 { - panic!("im2col: rs2[63] must be zero"); - } - let shape = decode_shape(xs1, xs2); - dimensions(shape); - footprints(shape); - (shape.window_count * shape.kernel * shape.kernel).max(16) as u64 -} diff --git a/examples/balls/im2col/emu/src/lib.rs b/examples/balls/im2col/emu/src/lib.rs deleted file mode 100644 index 88380c622..000000000 --- a/examples/balls/im2col/emu/src/lib.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub(crate) use crate::inst::{decode, instruction}; - -mod im2col; - -mod dispatch; - -pub use dispatch::{cycles_after_issue, execute_known}; diff --git a/examples/balls/im2col/verify/README.md b/examples/balls/im2col/verify/README.md deleted file mode 100644 index 273f1579c..000000000 --- a/examples/balls/im2col/verify/README.md +++ /dev/null @@ -1,102 +0,0 @@ -# Im2colBall UVM Verification - -Verifies the generated `Im2colBall` module directly using the shared Blink UVM framework. - -## Structure - -- `../../../../verify/uvm/src/ball/bb_uvm_pkg.sv`: common Blink UVM transaction items and base env -- `src/common/im2col_defs.svh`: DPI imports and constants -- `src/common/im2col_items.svh`: `im2col_cmd_item` with `load_rust_case` -- `src/seq/im2col_sequences.svh`: `im2col_basic_seq` driving one case per run -- `src/cov/im2col_cov.svh`: ksize/stride/pad/iter coverage -- `src/env/im2col_scoreboard.svh`: preloads mem model, compares writes vs DPI dst words -- `src/env/im2col_env.svh`: extends `bb_blink_env#(1,1)` with scoreboard and cov -- `src/tests/im2col_test.svh`: directed cases 0,1 then random 2..21 with fixed seed -- `src/pkg/im2col_pkg.sv`: package include entry -- `src/tb_top.sv`: instantiates `bb_blink_if#(1,1)` and `Im2colBall` -- `filelists/im2col_ball.f`: VCS/Verilator filelist - -## Test plan - -`+UVM_TESTNAME=im2col_ball_test` runs: - -- case 0: directed k3 6x6 pad0 (ctest `im2col_k3_test.c` data) -- case 1: directed k3 6x6 pad1 (ctest `im2col_k3_pad_test.c` data) -- cases 2..21: random with fixed seed `0xCAFE_BABE`, iter in {3,4,5,6,7}, - ksize in {1,3,5}, stride=1, pad in {0,1}, distinct banks 0..7 - -Each case: reset, preload src words into `mem_model`, drive one command, wait for -`scb.done()` or timeout at 100000 cycles. Scoreboard compares observed writes -(data/addr/mask/bank/rob) with DPI `im2col_case_dst_word_*`. `im2col_cov` -requires ksize/stride/pad/iter bins hit across the whole test, else -`check_phase` fatal. - -Blink command fields (not ISA rs packing): - -- `iter` = cmd.iter -- `ksize` = cmd.special[7:0] -- `stride` = cmd.special[15:8] -- `padding` = cmd.special[23:16] -- banks = op1_bank / wr_bank; funct7 is injected from the selected Core ballISA -- TB: op1_col=1, wr_col=1, rs1=0, rs2=0 - -## BID (required) - -`+BID=` is mandatory. Missing it fatals at test start. There is no default bid. - -| Config | Plusarg | -|--------|---------| -| `sims.verilator.BuckyballPebbleVerilatorConfig` | `+BID=2` | -| `sims.verilator.BuckyballToyVerilatorConfig` | `+BID=3` | - -```console -./simv +UVM_TESTNAME=im2col_ball_test +BID=2 -``` - -## Build - -Enter the shared UVM/VCS environment: - -```console -nix develop ../../../../verify -``` - -Build the DPI reference library: - -```console -cargo build --manifest-path casegen/Cargo.toml -``` - -Compile from this directory (resolve `@UVM@` / `@RTL@` like bbdev): - -```console -# Do not pass -hsopt=off: VCS W-2024.09 segfaults on Im2colBall+UVM with it. -vcs -full64 -sverilog -timescale=1ns/1ps \ - $VCS_UVM_ARGS \ - -sv_lib casegen/target/debug/libim2col_casegen \ - -f build//im2col_ball.f -``` - -Run: - -```console -./simv +UVM_TESTNAME=im2col_ball_test +BID=2 -``` - -Or via bbdev: - -```console -bbdev uvm --build '--ball=im2col --config sims.verilator.BuckyballPebbleVerilatorConfig' -bbdev uvm --run '--ball=im2col' -- '+BID=2' -``` - -## Acceptance (2026-08-04) - -- [x] Layout matches Blink common + ball-local split; filelist uses `@UVM@` / `@RTL@` -- [x] casegen pure im2col model (no ExecContext); whole-case DPI API -- [x] directed k3 pad0/pad1 scoreboard pass -- [x] deterministic random idx 2..21 pass -- [x] protocol + semantic cover enforced (`check_phase` fatal) -- [x] pebble: `+BID=2` — 22/22 -- [x] toy: `+BID=3` — 22/22 -- [x] Note: omit `-hsopt=off` for Im2col (VCS segfault); bbdev skips it for this ball diff --git a/examples/balls/im2col/verify/casegen/src/casegen.rs b/examples/balls/im2col/verify/casegen/src/casegen.rs deleted file mode 100644 index a482ed4f9..000000000 --- a/examples/balls/im2col/verify/casegen/src/casegen.rs +++ /dev/null @@ -1,312 +0,0 @@ -use crate::model; - -pub const BANK_ROW_BYTES: usize = model::BANK_ROW_BYTES; -pub const MAX_WORDS: usize = 128; - -#[repr(C)] -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct Im2colCmd { - pub bid: u32, - pub iter: u32, - pub ksize: u32, - pub stride: u32, - pub padding: u32, - pub window_count: u32, - pub op1_bank: u32, - pub wr_bank: u32, - pub op1_col: u32, - pub wr_col: u32, - pub rob_id: u32, - pub num_src_words: u32, - pub num_dst_words: u32, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct Im2colCase { - pub cmd: Im2colCmd, - pub src: Vec, - pub dst: Vec, -} - -impl Im2colCase { - pub fn src_word_lo(&self, word_index: usize) -> u64 { - word_lo(&self.src, word_index) - } - pub fn src_word_hi(&self, word_index: usize) -> u64 { - word_hi(&self.src, word_index) - } - pub fn dst_word_lo(&self, word_index: usize) -> u64 { - word_lo(&self.dst, word_index) - } - pub fn dst_word_hi(&self, word_index: usize) -> u64 { - word_hi(&self.dst, word_index) - } -} - -fn word_lo(data: &[u8], word_index: usize) -> u64 { - let off = word_index * BANK_ROW_BYTES; - if off + 8 > data.len() { - panic!("word_lo: word_index {word_index} out of range"); - } - let mut buf = [0u8; 8]; - buf.copy_from_slice(&data[off..off + 8]); - u64::from_le_bytes(buf) -} - -fn word_hi(data: &[u8], word_index: usize) -> u64 { - let off = word_index * BANK_ROW_BYTES + 8; - if off + 8 > data.len() { - panic!("word_hi: word_index {word_index} out of range"); - } - let mut buf = [0u8; 8]; - buf.copy_from_slice(&data[off..off + 8]); - u64::from_le_bytes(buf) -} - -pub fn gen_case(seed: u32, index: u32, bid: u32) -> Im2colCase { - match index { - 0 => directed_k3_pad0(bid), - 1 => directed_k3_pad1(bid), - _ => random_case(seed, index, bid), - } -} - -fn build_case( - bid: u32, - iter: usize, - ksize: usize, - stride: usize, - padding: usize, - op1_bank: u32, - wr_bank: u32, - rob_id: u32, - flat: &[u8], -) -> Im2colCase { - if op1_bank == wr_bank { - panic!("build_case: op1_bank and wr_bank must differ"); - } - if op1_bank > 7 || wr_bank > 7 { - panic!("build_case: banks must be in 0..7"); - } - if flat.len() != iter * iter { - panic!("build_case: flat len mismatch"); - } - let rows = model::output_rows(iter, ksize, stride, padding); - let windows = ((iter + 2 * padding - ksize) / stride + 1).pow(2); - if rows > model::bank_entries() { - panic!("build_case: output_rows={rows} exceeds bank depth"); - } - let dst_flat = model::im2col(flat, iter, ksize, stride, padding); - let src = model::pack_bank_words(flat); - let nsrc = model::num_words(src.len()); - let ndst = model::num_words(dst_flat.len()); - if nsrc > model::bank_entries() || ndst > model::bank_entries() || nsrc > MAX_WORDS || ndst > MAX_WORDS { - panic!("build_case: word count out of range src={nsrc} dst={ndst}"); - } - Im2colCase { - cmd: Im2colCmd { - bid, - iter: iter as u32, - ksize: ksize as u32, - stride: stride as u32, - padding: padding as u32, - window_count: windows as u32, - op1_bank, - wr_bank, - op1_col: 1, - wr_col: 1, - rob_id, - num_src_words: nsrc as u32, - num_dst_words: ndst as u32, - }, - src, - dst: dst_flat, - } -} - -fn directed_k3_pad0(bid: u32) -> Im2colCase { - let iter = 6usize; - let k = 3usize; - let flat: [i8; 36] = [ - -1, 2, 3, 4, -5, 6, 0, 8, 9, -10, 11, 12, 13, 0, -15, 16, 17, 18, 19, -20, 0, 22, 23, -24, - -25, 26, 27, 0, -29, 30, 31, 32, 33, -34, 0, 36, - ]; - let bytes: Vec = flat.iter().map(|&x| x as u8).collect(); - build_case(bid, iter, k, 1, 0, 0, 1, 3, &bytes) -} - -fn directed_k3_pad1(bid: u32) -> Im2colCase { - let iter = 6usize; - let k = 3usize; - let flat: [i8; 36] = [ - 7, -2, 3, 0, 5, -6, 1, 8, -9, 10, 11, 0, -4, 12, 13, 14, 0, 16, 17, 0, 19, -20, 21, 22, 0, - 24, 25, 26, -27, 28, 29, -30, 0, 32, 33, 34, - ]; - let bytes: Vec = flat.iter().map(|&x| x as u8).collect(); - build_case(bid, iter, k, 1, 1, 0, 1, 5, &bytes) -} - -fn shape_ok(iter: usize, ksize: usize, stride: usize, padding: usize) -> bool { - if iter == 0 || ksize == 0 || stride == 0 { - return false; - } - let padded = iter + 2 * padding; - if padded < ksize { - return false; - } - let rows = model::output_rows(iter, ksize, stride, padding); - rows <= model::bank_entries() -} - -fn random_case(seed: u32, index: u32, bid: u32) -> Im2colCase { - let mut rng = Rng::new(seed, index); - let iter_pool = [3usize, 4, 5, 6, 7]; - let ksize_pool = [1usize, 3, 5]; - let pad_pool = [0usize, 1]; - let base = (index - 2) as usize; - - for attempt in 0..10_000 { - let iter = iter_pool[(base + attempt) % iter_pool.len()]; - let ksize = ksize_pool[((base / iter_pool.len()) + attempt) % ksize_pool.len()]; - let stride = 1usize; - let padding = pad_pool[(base + attempt / 3) % pad_pool.len()]; - if !shape_ok(iter, ksize, stride, padding) { - continue; - } - let op1_bank = rng.range(0, 7); - let mut wr_bank = rng.range(0, 7); - if wr_bank == op1_bank { - wr_bank = (wr_bank + 1) & 7; - } - let mut flat = vec![0u8; iter * iter]; - for b in flat.iter_mut() { - *b = (rng.next() & 0xff) as u8; - } - return build_case( - bid, - iter, - ksize, - stride, - padding, - op1_bank, - wr_bank, - rng.range(0, 15), - &flat, - ); - } - panic!("random_case: failed to find legal shape for index={index}"); -} - -struct Rng { - state: u64, -} - -impl Rng { - fn new(seed: u32, index: u32) -> Self { - let state = (u64::from(seed) << 32) ^ u64::from(index) ^ 0x9E37_79B9_7F4A_7C15; - Self { state } - } - fn next(&mut self) -> u32 { - self.state ^= self.state >> 12; - self.state ^= self.state << 25; - self.state ^= self.state >> 27; - ((self.state.wrapping_mul(0x2545_F491_4F6C_DD1D)) >> 32) as u32 - } - fn range(&mut self, lo: u32, hi: u32) -> u32 { - lo + (self.next() % (hi - lo + 1)) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn directed_k3_pad0_matches_ctest() { - let case = gen_case(0, 0, 2); - assert_eq!(case.cmd.iter, 6); - assert_eq!(case.cmd.ksize, 3); - assert_eq!(case.cmd.stride, 1); - assert_eq!(case.cmd.padding, 0); - assert_eq!(case.cmd.op1_bank, 0); - assert_eq!(case.cmd.wr_bank, 1); - assert_eq!(case.cmd.op1_col, 1); - assert_eq!(case.cmd.wr_col, 1); - assert_eq!(case.cmd.num_src_words, 36); - assert_eq!(case.cmd.num_dst_words, 16); - assert_eq!(case.src.len(), 36 * 16); - assert_eq!(case.dst.len(), 16 * 16); - assert_eq!(case.src[0], (-1i8) as u8); - assert_eq!(case.src[35 * 16], 36); - } - - #[test] - fn directed_k3_pad1_shape() { - let case = gen_case(0, 1, 2); - assert_eq!(case.cmd.iter, 6); - assert_eq!(case.cmd.ksize, 3); - assert_eq!(case.cmd.padding, 1); - assert_eq!(case.cmd.num_dst_words, 48); - } - - #[test] - fn random_deterministic_and_legal() { - let a = gen_case(0xCAFE_BABE, 2, 2); - let b = gen_case(0xCAFE_BABE, 2, 2); - assert_eq!(a, b); - assert!([3u32, 4, 5, 6, 7].contains(&a.cmd.iter)); - assert!([1u32, 3, 5].contains(&a.cmd.ksize)); - assert_eq!(a.cmd.stride, 1); - assert!(a.cmd.padding == 0 || a.cmd.padding == 1); - assert_ne!(a.cmd.op1_bank, a.cmd.wr_bank); - assert!(a.cmd.num_dst_words <= model::bank_entries() as u32); - } - - #[test] - fn coverage_bins_hit_across_suite() { - let mut saw_iter = [false; 8]; - let mut saw_k = [false; 6]; - let mut saw_pad = [false; 2]; - for i in 0..=21 { - let c = gen_case(0xCAFE_BABE, i, 2); - saw_iter[c.cmd.iter as usize] = true; - saw_k[c.cmd.ksize as usize] = true; - saw_pad[c.cmd.padding as usize] = true; - assert_eq!(c.cmd.stride, 1); - } - for it in [3usize, 4, 5, 6, 7] { - assert!(saw_iter[it], "missing iter bin {it}"); - } - for k in [1usize, 3, 5] { - assert!(saw_k[k], "missing ksize bin {k}"); - } - assert!(saw_pad[0] && saw_pad[1], "missing pad bins"); - } - - #[test] - fn bid_required_arg() { - assert_eq!(gen_case(0, 0, 2).cmd.bid, 2); - assert_eq!(gen_case(0, 0, 3).cmd.bid, 3); - } - - #[test] - fn word_lo_hi_round_trip() { - let case = gen_case(0, 0, 2); - for i in 0..case.cmd.num_src_words as usize { - let lo = case.src_word_lo(i); - let hi = case.src_word_hi(i); - let off = i * BANK_ROW_BYTES; - let expect_lo = u64::from_le_bytes(case.src[off..off + 8].try_into().unwrap()); - let expect_hi = u64::from_le_bytes(case.src[off + 8..off + 16].try_into().unwrap()); - assert_eq!(lo, expect_lo); - assert_eq!(hi, expect_hi); - } - } - - #[test] - #[should_panic] - fn word_lo_out_of_range_panics() { - let case = gen_case(0, 0, 2); - let _ = case.src_word_lo(MAX_WORDS); - } -} diff --git a/examples/balls/im2col/verify/casegen/src/lib.rs b/examples/balls/im2col/verify/casegen/src/lib.rs deleted file mode 100644 index 492750020..000000000 --- a/examples/balls/im2col/verify/casegen/src/lib.rs +++ /dev/null @@ -1,112 +0,0 @@ -mod casegen; -mod model; - -use std::cell::RefCell; - -use casegen::{Im2colCase, Im2colCmd, MAX_WORDS}; - -thread_local! { - static CURRENT: RefCell> = RefCell::new(None); -} - -#[no_mangle] -pub extern "C" fn im2col_case_load(seed: u32, index: u32, bid: u32) -> i32 { - let case = casegen::gen_case(seed, index, bid); - let nsrc = case.cmd.num_src_words as usize; - let ndst = case.cmd.num_dst_words as usize; - if nsrc > MAX_WORDS || ndst > MAX_WORDS { - panic!("im2col_case_load: word count out of range src={nsrc} dst={ndst}"); - } - CURRENT.with(|c| *c.borrow_mut() = Some(case)); - 0 -} - -fn current_case R, R>(f: F) -> R { - CURRENT.with(|c| match c.borrow().as_ref() { - Some(case) => f(case), - None => panic!("im2col_case: no case loaded; call im2col_case_load first"), - }) -} - -#[no_mangle] -pub extern "C" fn im2col_case_cmd(out_ptr: *mut Im2colCmd) { - current_case(|case| unsafe { - if out_ptr.is_null() { - panic!("im2col_case_cmd: null out_ptr"); - } - *out_ptr = case.cmd; - }); -} - -#[no_mangle] -pub extern "C" fn im2col_case_src_word_lo(word_index: u32) -> u64 { - if word_index as usize >= MAX_WORDS { - panic!("im2col_case_src_word_lo: word_index out of range"); - } - current_case(|case| case.src_word_lo(word_index as usize)) -} - -#[no_mangle] -pub extern "C" fn im2col_case_src_word_hi(word_index: u32) -> u64 { - if word_index as usize >= MAX_WORDS { - panic!("im2col_case_src_word_hi: word_index out of range"); - } - current_case(|case| case.src_word_hi(word_index as usize)) -} - -#[no_mangle] -pub extern "C" fn im2col_case_dst_word_lo(word_index: u32) -> u64 { - if word_index as usize >= MAX_WORDS { - panic!("im2col_case_dst_word_lo: word_index out of range"); - } - current_case(|case| case.dst_word_lo(word_index as usize)) -} - -#[no_mangle] -pub extern "C" fn im2col_case_dst_word_hi(word_index: u32) -> u64 { - if word_index as usize >= MAX_WORDS { - panic!("im2col_case_dst_word_hi: word_index out of range"); - } - current_case(|case| case.dst_word_hi(word_index as usize)) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn dpi_load_then_cmd_and_words() { - assert_eq!(im2col_case_load(0x1234, 0, 2), 0); - let mut cmd = Im2colCmd { - bid: 0, - iter: 0, - ksize: 0, - stride: 0, - padding: 0, - window_count: 0, - op1_bank: 0, - wr_bank: 0, - op1_col: 0, - wr_col: 0, - rob_id: 0, - num_src_words: 0, - num_dst_words: 0, - }; - im2col_case_cmd(&mut cmd as *mut Im2colCmd); - assert_eq!(cmd.iter, 6); - assert_eq!(cmd.ksize, 3); - assert_eq!(cmd.num_src_words, 36); - assert_eq!(cmd.num_dst_words, 16); - let _lo = im2col_case_src_word_lo(0); - let _hi = im2col_case_src_word_hi(0); - let _dlo = im2col_case_dst_word_lo(0); - let _dhi = im2col_case_dst_word_hi(0); - } - - #[test] - #[should_panic(expected = "no case loaded")] - fn current_case_panics_without_load() { - CURRENT.with(|c| *c.borrow_mut() = None); - current_case(|_| ()); - } -} diff --git a/examples/balls/im2col/verify/casegen/src/model.rs b/examples/balls/im2col/verify/casegen/src/model.rs deleted file mode 100644 index 35955b18b..000000000 --- a/examples/balls/im2col/verify/casegen/src/model.rs +++ /dev/null @@ -1,103 +0,0 @@ -pub const TILE: usize = 16; -pub const BANK_ROW_BYTES: usize = TILE; - -pub fn bank_entries() -> usize { - let config = match std::env::var("BB_VERIFY_CONFIG") { - Ok(path) => path, - Err(_) if cfg!(test) => return 64, - Err(_) => panic!("BB_VERIFY_CONFIG is required"), - }; - let text = std::fs::read_to_string(config).expect("failed to read BB_VERIFY_CONFIG"); - text.lines() - .find_map(|line| line.strip_prefix("bank_entries=")) - .unwrap_or_else(|| panic!("bank_entries missing from BB_VERIFY_CONFIG")) - .parse() - .expect("invalid bank_entries in BB_VERIFY_CONFIG") -} - -pub fn output_rows(iter: usize, ksize: usize, stride: usize, padding: usize) -> usize { - if iter == 0 || ksize == 0 || stride == 0 { - panic!("im2col model: iter/ksize/stride must be >= 1"); - } - let padded = iter + 2 * padding; - if padded < ksize { - panic!("im2col model: kernel larger than padded input"); - } - let out_dim = (padded - ksize) / stride + 1; - let windows = out_dim * out_dim; - let kernel = ksize * ksize; - let m_tiles = windows.div_ceil(TILE); - let k_tiles = kernel.div_ceil(TILE); - m_tiles * k_tiles * TILE -} - -pub fn im2col(src: &[u8], iter: usize, ksize: usize, stride: usize, padding: usize) -> Vec { - if src.len() != iter * iter { - panic!( - "im2col model: src len {} != iter*iter {}", - src.len(), - iter * iter - ); - } - let padded = iter + 2 * padding; - if padded < ksize { - panic!("im2col model: kernel larger than padded input"); - } - if stride == 0 { - panic!("im2col model: stride must be >= 1"); - } - let out_dim = (padded - ksize) / stride + 1; - let windows = out_dim * out_dim; - let kernel_elems = ksize * ksize; - let m_tiles = windows.div_ceil(TILE); - let k_tiles = kernel_elems.div_ceil(TILE); - let rows = m_tiles * k_tiles * TILE; - let mut output = vec![0u8; rows * TILE]; - - let mut window = 0usize; - for orow in 0..out_dim { - for ocol in 0..out_dim { - for kr in 0..ksize { - for kc in 0..ksize { - let kernel = kr * ksize + kc; - let m_tile = window / TILE; - let m_row = window % TILE; - let k_tile = kernel / TILE; - let lane = kernel % TILE; - let bank_row = (m_tile * k_tiles + k_tile) * TILE + m_row; - let out = bank_row * TILE + lane; - if out >= output.len() { - panic!("im2col model: output range out={out}"); - } - let input_row = (orow * stride + kr) as isize - padding as isize; - let input_col = (ocol * stride + kc) as isize - padding as isize; - if input_row >= 0 - && input_row < iter as isize - && input_col >= 0 - && input_col < iter as isize - { - let src_i = input_row as usize * iter + input_col as usize; - output[out] = src[src_i]; - } - } - } - window += 1; - } - } - output -} - -pub fn pack_bank_words(flat: &[u8]) -> Vec { - let mut out = vec![0u8; flat.len() * BANK_ROW_BYTES]; - for (index, value) in flat.iter().enumerate() { - out[index * BANK_ROW_BYTES] = *value; - } - out -} - -pub fn num_words(total_bytes: usize) -> usize { - if total_bytes % BANK_ROW_BYTES != 0 { - panic!("num_words: total_bytes {total_bytes} not a multiple of bank row"); - } - total_bytes / BANK_ROW_BYTES -} diff --git a/examples/balls/im2col/verify/src/common/im2col_defs.svh b/examples/balls/im2col/verify/src/common/im2col_defs.svh deleted file mode 100644 index 3a0ab79b0..000000000 --- a/examples/balls/im2col/verify/src/common/im2col_defs.svh +++ /dev/null @@ -1,42 +0,0 @@ -typedef struct { - int unsigned bid; - int unsigned iter; - int unsigned ksize; - int unsigned stride; - int unsigned padding; - int unsigned window_count; - int unsigned op1_bank; - int unsigned wr_bank; - int unsigned op1_col; - int unsigned wr_col; - int unsigned rob_id; - int unsigned num_src_words; - int unsigned num_dst_words; -} im2col_cmd_dpi_t; - -import "DPI-C" function int im2col_case_load( - input int unsigned seed, - input int unsigned index, - input int unsigned bid -); -import "DPI-C" function void im2col_case_cmd(output im2col_cmd_dpi_t cmd); -import "DPI-C" function longint unsigned im2col_case_src_word_lo(input int unsigned word_index); -import "DPI-C" function longint unsigned im2col_case_src_word_hi(input int unsigned word_index); -import "DPI-C" function longint unsigned im2col_case_dst_word_lo(input int unsigned word_index); -import "DPI-C" function longint unsigned im2col_case_dst_word_hi(input int unsigned word_index); - -`ifndef IM2COL_FUNCT7 -`error "IM2COL_FUNCT7 must be provided by the selected Core ballISA" -`endif -localparam int IM2COL_CORE_FUNCT7 = `IM2COL_FUNCT7; -localparam int IM2COL_MAX_WORDS = 128; -localparam int IM2COL_TIMEOUT_CYCLES = 100000; -localparam int IM2COL_SEED = 32'hCAFE_BABE; - -function automatic int unsigned im2col_require_bid(); - int unsigned bid; - if (!$value$plusargs("BID=%d", bid)) begin - `uvm_fatal("BID", "missing required plusarg +BID=") - end - return bid; -endfunction diff --git a/examples/balls/im2col/verify/src/common/im2col_items.svh b/examples/balls/im2col/verify/src/common/im2col_items.svh deleted file mode 100644 index 2dd71678a..000000000 --- a/examples/balls/im2col/verify/src/common/im2col_items.svh +++ /dev/null @@ -1,117 +0,0 @@ -class im2col_cmd_item extends bb_blink_cmd_item; - `uvm_object_utils(im2col_cmd_item) - - bit [31:0] ksize; - bit [31:0] stride; - bit [31:0] padding; - bit [31:0] window_count; - bit [31:0] num_src_words; - bit [31:0] num_dst_words; - bit [127:0] src_words[IM2COL_MAX_WORDS]; - bit [127:0] dst_words[IM2COL_MAX_WORDS]; - - constraint legal_c { - funct7 == IM2COL_CORE_FUNCT7[6:0]; - op1_en == 1'b1; - op2_en == 1'b0; - wr_spad_en == 1'b1; - op1_from_spad == 1'b1; - op2_from_spad == 1'b0; - op2_bank == 5'd0; - op2_col == 5'd0; - meta_bank == 5'd0; - rs1 == 64'd0; - rs2[63] == 1'b0; - is_sub == 1'b0; - sub_rob_id == 8'h00; - } - - function new(string name = "im2col_cmd_item"); - super.new(name); - endfunction - - function void load_rust_case(int unsigned seed, int unsigned index, int unsigned bid); - im2col_cmd_dpi_t cmd; - int unsigned rc; - longint unsigned w_lo; - longint unsigned w_hi; - int i; - - rc = im2col_case_load(seed, index, bid); - if (rc != 0) begin - `uvm_fatal("CASE", $sformatf("im2col_case_load returned %0d for index %0d", rc, index)) - end - im2col_case_cmd(cmd); - - bid = cmd.bid[4:0]; - funct7 = IM2COL_CORE_FUNCT7[6:0]; - iter = cmd.iter; - ksize = cmd.ksize; - stride = cmd.stride; - padding = cmd.padding; - window_count = cmd.window_count; - op1_bank = cmd.op1_bank[4:0]; - wr_bank = cmd.wr_bank[4:0]; - op1_col = cmd.op1_col[4:0]; - wr_col = cmd.wr_col[4:0]; - rob_id = cmd.rob_id[3:0]; - num_src_words = cmd.num_src_words; - num_dst_words = cmd.num_dst_words; - - special = 64'd0; - op1_en = 1'b1; - op2_en = 1'b0; - wr_spad_en = 1'b1; - op1_from_spad = 1'b1; - op2_from_spad = 1'b0; - op2_bank = 5'd0; - op2_col = 5'd0; - meta_bank = 5'd0; - rs1 = 64'd0; - rs2 = { - 1'b0, window_count[6:0], 6'd0, 4'd0, 6'd0, 8'd0, 8'd0, padding[7:0], stride[7:0], ksize[7:0] - }; - is_sub = 1'b0; - sub_rob_id = 8'h00; - - if (num_src_words > IM2COL_MAX_WORDS || num_dst_words > IM2COL_MAX_WORDS) begin - `uvm_fatal("CASE", $sformatf("word count out of range src=%0d dst=%0d", num_src_words, - num_dst_words)) - end - if (op1_bank == wr_bank) begin - `uvm_fatal("CASE", "op1_bank and wr_bank overlap") - end - if (stride == 0) begin - `uvm_fatal("CASE", "stride must be >= 1") - end - - for (i = 0; i < num_src_words; i++) begin - w_lo = im2col_case_src_word_lo(i); - w_hi = im2col_case_src_word_hi(i); - src_words[i] = {w_hi, w_lo}; - end - for (i = 0; i < num_dst_words; i++) begin - w_lo = im2col_case_dst_word_lo(i); - w_hi = im2col_case_dst_word_hi(i); - dst_words[i] = {w_hi, w_lo}; - end - endfunction - - function void do_copy(uvm_object rhs); - im2col_cmd_item rhs_; - super.do_copy(rhs); - if (!$cast(rhs_, rhs)) begin - `uvm_fatal("COPY", "rhs is not im2col_cmd_item") - end - ksize = rhs_.ksize; - stride = rhs_.stride; - padding = rhs_.padding; - window_count = rhs_.window_count; - num_src_words = rhs_.num_src_words; - num_dst_words = rhs_.num_dst_words; - for (int i = 0; i < IM2COL_MAX_WORDS; i++) begin - src_words[i] = rhs_.src_words[i]; - dst_words[i] = rhs_.dst_words[i]; - end - endfunction -endclass diff --git a/examples/balls/im2col/verify/src/cov/im2col_cov.svh b/examples/balls/im2col/verify/src/cov/im2col_cov.svh deleted file mode 100644 index bd512cffe..000000000 --- a/examples/balls/im2col/verify/src/cov/im2col_cov.svh +++ /dev/null @@ -1,47 +0,0 @@ -class im2col_cov extends uvm_component; - `uvm_component_utils(im2col_cov) - - uvm_analysis_imp_cmd #(bb_blink_cmd_item, im2col_cov) cmd_imp; - - covergroup cmd_cg; - coverpoint cur_ksize {bins k1 = {1}; bins k3 = {3}; bins k5 = {5};} - coverpoint cur_stride {bins s1 = {1};} - coverpoint cur_pad {bins p0 = {0}; bins p1 = {1};} - coverpoint cur_iter { - bins it3 = {3}; bins it4 = {4}; bins it5 = {5}; bins it6 = {6}; bins it7 = {7}; - } - endgroup - - int unsigned cur_ksize; - int unsigned cur_stride; - int unsigned cur_pad; - int unsigned cur_iter; - int cmd_count; - - function new(string name, uvm_component parent); - super.new(name, parent); - cmd_imp = new("cmd_imp", this); - cmd_cg = new(); - endfunction - - function void write_cmd(bb_blink_cmd_item item); - cur_ksize = int'(item.rs2[7:0]); - cur_stride = int'(item.rs2[15:8]); - cur_pad = int'(item.rs2[23:16]); - cur_iter = int'(item.iter); - cmd_count++; - cmd_cg.sample(); - endfunction - - function void check_phase(uvm_phase phase); - super.check_phase(phase); - if (cmd_count == 0) `uvm_fatal("COV", "no cmd observed") - if (cmd_cg.get_coverage() != 100.0) begin - `uvm_fatal( - "COV", - $sformatf( - "im2col_cov coverage incomplete: %0.2f%% (need ksize {1,3,5}, stride {1}, pad {0,1}, iter {3,4,5,6,7})", - cmd_cg.get_coverage())) - end - endfunction -endclass diff --git a/examples/balls/im2col/verify/src/env/im2col_scoreboard.svh b/examples/balls/im2col/verify/src/env/im2col_scoreboard.svh deleted file mode 100644 index b79a5cb74..000000000 --- a/examples/balls/im2col/verify/src/env/im2col_scoreboard.svh +++ /dev/null @@ -1,170 +0,0 @@ -class im2col_scoreboard extends uvm_scoreboard; - `uvm_component_utils(im2col_scoreboard) - - uvm_analysis_imp_stim #(bb_blink_cmd_item, im2col_scoreboard) stim_imp; - uvm_analysis_imp_cmd #(bb_blink_cmd_item, im2col_scoreboard) cmd_imp; - uvm_analysis_imp_read #(bb_blink_read_item, im2col_scoreboard) read_imp; - uvm_analysis_imp_write #(bb_blink_write_item, im2col_scoreboard) write_imp; - uvm_analysis_imp_resp #(bb_blink_resp_item, im2col_scoreboard) resp_imp; - - bb_blink_mem_model #(1, 1) mem_model; - - im2col_cmd_item stim_q[$]; - bit [127:0] expected_dst[IM2COL_MAX_WORDS]; - bit seen_addr[IM2COL_MAX_WORDS]; - int unsigned expected_writes; - int unsigned cmd_count; - int unsigned read_count; - int unsigned write_count; - int unsigned resp_count; - - function new(string name, uvm_component parent); - super.new(name, parent); - stim_imp = new("stim_imp", this); - cmd_imp = new("cmd_imp", this); - read_imp = new("read_imp", this); - write_imp = new("write_imp", this); - resp_imp = new("resp_imp", this); - endfunction - - function void write_stim(bb_blink_cmd_item item); - im2col_cmd_item titem; - im2col_cmd_item clone; - int i; - - if (stim_q.size() != 0) begin - `uvm_fatal("SCB", "single outstanding command supported") - end - if (!$cast(titem, item)) begin - `uvm_fatal("SCB", "stim item is not im2col_cmd_item") - end - if (!$cast(clone, titem.clone())) begin - `uvm_fatal("SCB", "failed to clone stimulus item") - end - stim_q.push_back(clone); - - if (mem_model == null) begin - `uvm_fatal("SCB", "mem_model handle not set") - end - mem_model.clear_mem(); - for (i = 0; i < clone.num_src_words; i++) begin - mem_model.load_word(int'(clone.op1_bank), i, clone.src_words[i]); - end - mem_model.arm(); - - for (i = 0; i < clone.num_dst_words; i++) begin - expected_dst[i] = clone.dst_words[i]; - seen_addr[i] = 1'b0; - end - expected_writes = clone.num_dst_words; - endfunction - - function void write_cmd(bb_blink_cmd_item item); - im2col_cmd_item stim; - stim = current_stim("CMD"); - check_cmd(item, stim); - cmd_count++; - endfunction - - function void check_cmd(bb_blink_cmd_item got, im2col_cmd_item exp); - if (got.bid !== exp.bid) - `uvm_fatal("CMD", $sformatf("bid mismatch: got %0d exp %0d", got.bid, exp.bid)) - if (got.funct7 !== exp.funct7) - `uvm_fatal("CMD", $sformatf("funct7 mismatch: got %0d exp %0d", got.funct7, exp.funct7)) - if (got.iter !== exp.iter) - `uvm_fatal("CMD", $sformatf("iter mismatch: got %0d exp %0d", got.iter, exp.iter)) - if (got.op1_bank !== exp.op1_bank || got.wr_bank !== exp.wr_bank) - `uvm_fatal("CMD", "bank field mismatch") - if (got.op1_col !== exp.op1_col || got.wr_col !== exp.wr_col) - `uvm_fatal("CMD", "column field mismatch") - if (got.rob_id !== exp.rob_id) - `uvm_fatal("CMD", $sformatf("rob_id mismatch: got %0d exp %0d", got.rob_id, exp.rob_id)) - if (got.rs2[7:0] !== exp.ksize[7:0]) - `uvm_fatal("CMD", $sformatf("ksize mismatch: got %0d exp %0d", got.rs2[7:0], exp.ksize)) - if (got.rs2[15:8] !== exp.stride[7:0]) - `uvm_fatal("CMD", $sformatf("stride mismatch: got %0d exp %0d", got.rs2[15:8], exp.stride)) - if (got.rs2[23:16] !== exp.padding[7:0]) - `uvm_fatal("CMD", $sformatf("padding mismatch: got %0d exp %0d", got.rs2[23:16], exp.padding)) - if (got.rs1 !== 64'd0) `uvm_fatal("CMD", "rs1 must be 0") - endfunction - - function void write_read(bb_blink_read_item item); - im2col_cmd_item stim; - stim = current_stim("READ"); - if (item.bank_id !== stim.op1_bank) - `uvm_fatal("READ", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.op1_bank)) - if (item.rob_id !== stim.rob_id) - `uvm_fatal("READ", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.group_id !== '0) - `uvm_fatal("READ", $sformatf("group_id mismatch: got %0d exp 0", item.group_id)) - if (item.addr >= stim.num_src_words[9:0]) - `uvm_fatal("READ", $sformatf( - "addr out of range: got %0d max %0d", item.addr, stim.num_src_words)) - read_count++; - endfunction - - function void write_write(bb_blink_write_item item); - im2col_cmd_item stim; - int unsigned addr; - stim = current_stim("WRITE"); - if (item.bank_id !== stim.wr_bank) - `uvm_fatal("WRITE", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.wr_bank)) - if (item.rob_id !== stim.rob_id) - `uvm_fatal("WRITE", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.group_id !== '0) - `uvm_fatal("WRITE", $sformatf("group_id mismatch: got %0d exp 0", item.group_id)) - addr = int'(item.addr); - if (addr >= expected_writes) - `uvm_fatal("WRITE", $sformatf("addr out of range: got %0d max %0d", addr, expected_writes)) - if (seen_addr[addr]) `uvm_fatal("WRITE", $sformatf("duplicate write addr %0d", addr)) - if (item.mask !== 16'hFFFF) - `uvm_fatal("WRITE", $sformatf("mask mismatch: got 0x%04h", item.mask)) - if (item.data !== expected_dst[addr]) - `uvm_fatal( - "SCB", $sformatf( - "data mismatch at addr %0d: got 0x%032h exp 0x%032h", addr, item.data, expected_dst[addr] - )) - seen_addr[addr] = 1'b1; - write_count++; - endfunction - - function void write_resp(bb_blink_resp_item item); - im2col_cmd_item stim; - stim = current_stim("RESP"); - if (item.rob_id !== stim.rob_id) - `uvm_fatal("RESP", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.is_sub !== 1'b0) `uvm_fatal("RESP", "is_sub should be 0") - resp_count++; - endfunction - - function im2col_cmd_item current_stim(string tag); - if (stim_q.size() == 0) begin - `uvm_fatal("SCB", $sformatf("%s observed before stimulus", tag)) - return null; - end - return stim_q[0]; - endfunction - - function bit done(); - return cmd_count == 1 && write_count == expected_writes && resp_count == 1; - endfunction - - function void reset_counters(); - stim_q.delete(); - for (int i = 0; i < IM2COL_MAX_WORDS; i++) seen_addr[i] = 1'b0; - cmd_count = 0; - read_count = 0; - write_count = 0; - resp_count = 0; - expected_writes = 0; - endfunction - - function void check_phase(uvm_phase phase); - super.check_phase(phase); - if (!done()) begin - `uvm_fatal("SCB", $sformatf("incomplete: cmds=%0d reads=%0d writes=%0d/%0d resp=%0d", - cmd_count, read_count, write_count, expected_writes, resp_count)) - end - stim_q.delete(); - endfunction -endclass diff --git a/examples/balls/im2col/verify/src/tb_top.sv b/examples/balls/im2col/verify/src/tb_top.sv deleted file mode 100644 index fd043ab42..000000000 --- a/examples/balls/im2col/verify/src/tb_top.sv +++ /dev/null @@ -1,91 +0,0 @@ -module tb_top; - import uvm_pkg::*; - import im2col_pkg::*; - - bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); - - Im2colBall dut ( - .clock(intf.clock), - .reset(intf.reset), - .io_cmdReq_ready(intf.cmd_req_ready), - .io_cmdReq_valid(intf.cmd_req_valid), - .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), - .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), - .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), - .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), - .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), - .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), - .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), - .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), - .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), - .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank[3:0]), - .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank[3:0]), - .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank[3:0]), - .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), - .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), - .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), - .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank[3:0]), - .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), - .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), - .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), - .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), - .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), - .io_cmdResp_ready(intf.cmd_resp_ready), - .io_cmdResp_valid(intf.cmd_resp_valid), - .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), - .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), - .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), - .io_bankRead_0_bank_id(intf.bank_read_bank_id[0][3:0]), - .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), - .io_bankRead_0_group_id(intf.bank_read_group_id[0][3:0]), - .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), - .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), - .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), - .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), - .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), - .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), - .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0][3:0]), - .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), - .io_bankWrite_0_group_id(intf.bank_write_group_id[0][3:0]), - .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), - .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), - .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), - .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), - .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), - .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), - .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), - .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), - .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), - .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), - .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), - .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), - .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), - .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), - .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), - .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), - .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), - .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), - .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), - .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), - .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), - .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), - .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), - .io_subRobReq_ready(intf.sub_rob_req_ready), - .io_status_idle(), - .io_status_running(), - .io_channelReady(1'b1), - .io_bankRead_0_ball_id(), - .io_bankWrite_0_ball_id() - ); - - initial begin - intf.clock = 1'b0; - intf.reset = 1'b1; - forever #5 intf.clock = ~intf.clock; - end - - initial begin - uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); - run_test("im2col_ball_test"); - end -endmodule diff --git a/examples/balls/im2col/verify/src/tests/im2col_test.svh b/examples/balls/im2col/verify/src/tests/im2col_test.svh deleted file mode 100644 index e969a2059..000000000 --- a/examples/balls/im2col/verify/src/tests/im2col_test.svh +++ /dev/null @@ -1,76 +0,0 @@ -class im2col_ball_test extends uvm_test; - `uvm_component_utils(im2col_ball_test) - - typedef virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) vif_t; - vif_t vif; - im2col_env env; - - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(vif_t)::get(this, "", "vif", vif)) begin - `uvm_fatal("NOVIF", "bb_blink_if not found") - end - env = im2col_env::type_id::create("env", this); - endfunction - - task run_phase(uvm_phase phase); - int unsigned bid; - bid = im2col_require_bid(); - phase.raise_objection(this); - run_directed(bid); - run_random(bid); - phase.drop_objection(this); - endtask - - task run_directed(int unsigned bid); - run_case(0, "DIR_K3_PAD0", bid); - run_case(1, "DIR_K3_PAD1", bid); - endtask - - task run_random(int unsigned bid); - int i; - for (i = 2; i <= 21; i++) begin - run_case(i, $sformatf("RND_%0d", i), bid); - end - endtask - - task run_case(int unsigned idx, string label, int unsigned bid); - im2col_basic_seq seq; - int cycles; - - apply_reset(); - env.scb.reset_counters(); - - seq = im2col_basic_seq::type_id::create("seq"); - seq.case_index = idx; - seq.bid = bid; - seq.start(env.cmd_agent.seqr); - - cycles = 0; - while (!env.scb.done()) begin - @(posedge vif.clock); - cycles++; - if (cycles > IM2COL_TIMEOUT_CYCLES) begin - `uvm_fatal("TIMEOUT", - $sformatf( - "Im2colBall %s (idx=%0d) timeout: cmds=%0d reads=%0d wr=%0d/%0d resp=%0d", - label, idx, env.scb.cmd_count, env.scb.read_count, env.scb.write_count, - env.scb.expected_writes, env.scb.resp_count)) - end - end - - `uvm_info("IM2COL", $sformatf("Im2colBall %s (idx=%0d) passed in %0d cycles", label, idx, - cycles), UVM_LOW) - endtask - - task apply_reset(); - vif.reset = 1'b1; - repeat (5) @(posedge vif.clock); - vif.reset = 1'b0; - repeat (2) @(posedge vif.clock); - endtask -endclass diff --git a/examples/balls/im2col/workloads/ctests/CMakeLists.txt b/examples/balls/im2col/workloads/ctests/CMakeLists.txt deleted file mode 100644 index eca3b8d08..000000000 --- a/examples/balls/im2col/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -add_buckyball_ctests( - im2col_window_test.c - im2col_k1_test.c - im2col_k3_test.c - im2col_k3_pad_test.c - im2col_k3_asym_pad_test.c - im2col_k3_partial_test.c - im2col_k5_test.c - im2col_k7_test.c -) diff --git a/examples/balls/im2col/workloads/ctests/im2col_k1_test.c b/examples/balls/im2col/workloads/ctests/im2col_k1_test.c deleted file mode 100644 index 1a8857d3e..000000000 --- a/examples/balls/im2col/workloads/ctests/im2col_k1_test.c +++ /dev/null @@ -1,74 +0,0 @@ -#include "buckyball.h" -#include -#include -#include -#include - -enum { ITER = 7, K = 1, STRIDE = 1, PAD = 0, LANES = 16 }; -enum { - OUT_DIM = (ITER + 2 * PAD - K) / STRIDE + 1, - WINDOWS = OUT_DIM * OUT_DIM, - KERNEL = K * K, - M_TILES = (WINDOWS + LANES - 1) / LANES, - K_TILES = (KERNEL + LANES - 1) / LANES, - OUT_ROWS = M_TILES * K_TILES * LANES, -}; - -static elem_t in[ITER * ITER] __attribute__((aligned(64))) = { - 1, 2, -3, 4, 0, -6, 7, 8, -9, 0, 11, -12, 13, - 14, 0, 16, 17, -18, 19, 0, -21, 22, 23, -24, 0, 26, - -27, 28, 29, 0, 31, 32, -33, 34, 0, -36, 37, 38, -39, - 0, 41, -42, 43, 44, 0, 46, 47, -48, 49, -}; -static elem_t packed[BANK_LINES * LANES] __attribute__((aligned(64))); -static elem_t out[OUT_ROWS * LANES] __attribute__((aligned(64))); -static elem_t exp[OUT_ROWS * LANES] __attribute__((aligned(64))); - -static void build_expected(void) { - int w = 0; - clear_i8_matrix(exp, OUT_ROWS, LANES); - for (int orow = 0; orow < OUT_DIM; orow++) { - for (int ocol = 0; ocol < OUT_DIM; ocol++) { - for (int krow = 0; krow < K; krow++) { - for (int kcol = 0; kcol < K; kcol++) { - int ki = krow * K + kcol; - int bank_row = - ((w / LANES) * K_TILES + ki / LANES) * LANES + (w % LANES); - int row = orow * STRIDE + krow - PAD; - int col = ocol * STRIDE + kcol - PAD; - exp[bank_row * LANES + ki % LANES] = - (row < 0 || row >= ITER || col < 0 || col >= ITER) - ? 0 - : in[row * ITER + col]; - } - } - w++; - } - } -} - -int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); -#endif - clear_i8_matrix(out, OUT_ROWS, LANES); - build_expected(); - clear_i8_matrix(packed, BANK_LINES, LANES); - for (int i = 0; i < ITER * ITER; ++i) - packed[i * LANES] = in[i]; - - bb_mem_alloc(0, 1, 1); - bb_mem_alloc(1, 1, 1); - bb_mvin((uintptr_t)packed, 0, BANK_LINES, 1); - bb_im2col(0, 1, ITER, K, STRIDE, PAD, 0, 0, 0, 0, 0, WINDOWS); - bb_mvout((uintptr_t)out, 1, OUT_ROWS, 1); - bb_mem_release(0); - bb_mem_release(1); - - if (compare_i8_matrices(out, exp, OUT_ROWS, LANES)) { - printf("im2col k1 7x7 PASSED\n"); - return 0; - } - printf("im2col k1 7x7 FAILED\n"); - return 1; -} diff --git a/examples/balls/im2col/workloads/ctests/im2col_k3_asym_pad_test.c b/examples/balls/im2col/workloads/ctests/im2col_k3_asym_pad_test.c deleted file mode 100644 index bcfb5385d..000000000 --- a/examples/balls/im2col/workloads/ctests/im2col_k3_asym_pad_test.c +++ /dev/null @@ -1,73 +0,0 @@ -#include "buckyball.h" -#include -#include -#include -#include - -/* asym pad: padding=max(lo,hi)=2, start=padding-lo=1 (lo=1,hi=2) */ -enum { ITER = 6, K = 3, STRIDE = 1, PAD = 2, START = 1, LANES = 16 }; -enum { - OUT_DIM = (ITER + 2 * PAD - K - START) / STRIDE + 1, - WINDOWS = OUT_DIM * OUT_DIM, - KERNEL = K * K, - M_TILES = (WINDOWS + LANES - 1) / LANES, - K_TILES = (KERNEL + LANES - 1) / LANES, - OUT_ROWS = M_TILES * K_TILES * LANES, -}; - -static elem_t in[ITER * ITER] __attribute__((aligned(64))) = { - 7, -2, 3, 0, 5, -6, 1, 8, -9, 10, 11, 0, -4, 12, 13, 14, 0, 16, - 17, 0, 19, -20, 21, 22, 0, 24, 25, 26, -27, 28, 29, -30, 0, 32, 33, 34, -}; -static elem_t packed[BANK_LINES * LANES] __attribute__((aligned(64))); -static elem_t out[OUT_ROWS * LANES] __attribute__((aligned(64))); -static elem_t exp[OUT_ROWS * LANES] __attribute__((aligned(64))); - -static void build_expected(void) { - int w = 0; - clear_i8_matrix(exp, OUT_ROWS, LANES); - for (int orow = 0; orow < OUT_DIM; orow++) { - for (int ocol = 0; ocol < OUT_DIM; ocol++) { - for (int krow = 0; krow < K; krow++) { - for (int kcol = 0; kcol < K; kcol++) { - int ki = krow * K + kcol; - int bank_row = - ((w / LANES) * K_TILES + ki / LANES) * LANES + (w % LANES); - int row = START + orow * STRIDE + krow - PAD; - int col = START + ocol * STRIDE + kcol - PAD; - exp[bank_row * LANES + ki % LANES] = - (row < 0 || row >= ITER || col < 0 || col >= ITER) - ? 0 - : in[row * ITER + col]; - } - } - w++; - } - } -} - -int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); -#endif - clear_i8_matrix(out, OUT_ROWS, LANES); - build_expected(); - clear_i8_matrix(packed, BANK_LINES, LANES); - for (int i = 0; i < ITER * ITER; ++i) - packed[i * LANES] = in[i]; - - bb_mem_alloc(0, 1, 1); - bb_mem_alloc(1, 1, 1); - bb_mvin((uintptr_t)packed, 0, BANK_LINES, 1); - bb_im2col(0, 1, ITER, K, STRIDE, PAD, 0, 0, START, START, 0, WINDOWS); - bb_mvout((uintptr_t)out, 1, OUT_ROWS, 1); - bb_mem_release(0); - bb_mem_release(1); - - if (compare_i8_matrices(out, exp, OUT_ROWS, LANES)) { - printf("im2col k3 6x6 asym pad PASSED\n"); - return 0; - } - printf("im2col k3 6x6 asym pad FAILED\n"); - return 1; -} diff --git a/examples/balls/im2col/workloads/ctests/im2col_k3_pad_test.c b/examples/balls/im2col/workloads/ctests/im2col_k3_pad_test.c deleted file mode 100644 index 6552d46ce..000000000 --- a/examples/balls/im2col/workloads/ctests/im2col_k3_pad_test.c +++ /dev/null @@ -1,73 +0,0 @@ -#include "buckyball.h" -#include -#include -#include -#include - -/* padding path: out-of-bound window samples are zero */ -enum { ITER = 6, K = 3, STRIDE = 1, PAD = 1, LANES = 16 }; -enum { - OUT_DIM = (ITER + 2 * PAD - K) / STRIDE + 1, - WINDOWS = OUT_DIM * OUT_DIM, - KERNEL = K * K, - M_TILES = (WINDOWS + LANES - 1) / LANES, - K_TILES = (KERNEL + LANES - 1) / LANES, - OUT_ROWS = M_TILES * K_TILES * LANES, -}; - -static elem_t in[ITER * ITER] __attribute__((aligned(64))) = { - 7, -2, 3, 0, 5, -6, 1, 8, -9, 10, 11, 0, -4, 12, 13, 14, 0, 16, - 17, 0, 19, -20, 21, 22, 0, 24, 25, 26, -27, 28, 29, -30, 0, 32, 33, 34, -}; -static elem_t packed[BANK_LINES * LANES] __attribute__((aligned(64))); -static elem_t out[OUT_ROWS * LANES] __attribute__((aligned(64))); -static elem_t exp[OUT_ROWS * LANES] __attribute__((aligned(64))); - -static void build_expected(void) { - int w = 0; - clear_i8_matrix(exp, OUT_ROWS, LANES); - for (int orow = 0; orow < OUT_DIM; orow++) { - for (int ocol = 0; ocol < OUT_DIM; ocol++) { - for (int krow = 0; krow < K; krow++) { - for (int kcol = 0; kcol < K; kcol++) { - int ki = krow * K + kcol; - int bank_row = - ((w / LANES) * K_TILES + ki / LANES) * LANES + (w % LANES); - int row = orow * STRIDE + krow - PAD; - int col = ocol * STRIDE + kcol - PAD; - exp[bank_row * LANES + ki % LANES] = - (row < 0 || row >= ITER || col < 0 || col >= ITER) - ? 0 - : in[row * ITER + col]; - } - } - w++; - } - } -} - -int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); -#endif - clear_i8_matrix(out, OUT_ROWS, LANES); - build_expected(); - clear_i8_matrix(packed, BANK_LINES, LANES); - for (int i = 0; i < ITER * ITER; ++i) - packed[i * LANES] = in[i]; - - bb_mem_alloc(0, 1, 1); - bb_mem_alloc(1, 1, 1); - bb_mvin((uintptr_t)packed, 0, BANK_LINES, 1); - bb_im2col(0, 1, ITER, K, STRIDE, PAD, 0, 0, 0, 0, 0, WINDOWS); - bb_mvout((uintptr_t)out, 1, OUT_ROWS, 1); - bb_mem_release(0); - bb_mem_release(1); - - if (compare_i8_matrices(out, exp, OUT_ROWS, LANES)) { - printf("im2col k3 6x6 p1 PASSED\n"); - return 0; - } - printf("im2col k3 6x6 p1 FAILED\n"); - return 1; -} diff --git a/examples/balls/im2col/workloads/ctests/im2col_k3_partial_test.c b/examples/balls/im2col/workloads/ctests/im2col_k3_partial_test.c deleted file mode 100644 index 2fc546064..000000000 --- a/examples/balls/im2col/workloads/ctests/im2col_k3_partial_test.c +++ /dev/null @@ -1,73 +0,0 @@ -#include "buckyball.h" -#include -#include -#include -#include - -/* partial M tile: 5x5 k3 -> 9 windows, unused m rows must be 0 */ -enum { ITER = 5, K = 3, STRIDE = 1, PAD = 0, LANES = 16 }; -enum { - OUT_DIM = (ITER + 2 * PAD - K) / STRIDE + 1, - WINDOWS = OUT_DIM * OUT_DIM, - KERNEL = K * K, - M_TILES = (WINDOWS + LANES - 1) / LANES, - K_TILES = (KERNEL + LANES - 1) / LANES, - OUT_ROWS = M_TILES * K_TILES * LANES, -}; - -static elem_t in[ITER * ITER] __attribute__((aligned(64))) = { - -1, 2, 3, 0, 5, 6, 7, 0, 9, 10, 11, 0, -13, - 14, 15, 0, 17, 18, 19, 0, 21, 22, 23, 0, -25, -}; -static elem_t packed[BANK_LINES * LANES] __attribute__((aligned(64))); -static elem_t out[OUT_ROWS * LANES] __attribute__((aligned(64))); -static elem_t exp[OUT_ROWS * LANES] __attribute__((aligned(64))); - -static void build_expected(void) { - int w = 0; - clear_i8_matrix(exp, OUT_ROWS, LANES); - for (int orow = 0; orow < OUT_DIM; orow++) { - for (int ocol = 0; ocol < OUT_DIM; ocol++) { - for (int krow = 0; krow < K; krow++) { - for (int kcol = 0; kcol < K; kcol++) { - int ki = krow * K + kcol; - int bank_row = - ((w / LANES) * K_TILES + ki / LANES) * LANES + (w % LANES); - int row = orow * STRIDE + krow - PAD; - int col = ocol * STRIDE + kcol - PAD; - exp[bank_row * LANES + ki % LANES] = - (row < 0 || row >= ITER || col < 0 || col >= ITER) - ? 0 - : in[row * ITER + col]; - } - } - w++; - } - } -} - -int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); -#endif - clear_i8_matrix(out, OUT_ROWS, LANES); - build_expected(); - clear_i8_matrix(packed, BANK_LINES, LANES); - for (int i = 0; i < ITER * ITER; ++i) - packed[i * LANES] = in[i]; - - bb_mem_alloc(0, 1, 1); - bb_mem_alloc(1, 1, 1); - bb_mvin((uintptr_t)packed, 0, BANK_LINES, 1); - bb_im2col(0, 1, ITER, K, STRIDE, PAD, 0, 0, 0, 0, 0, WINDOWS); - bb_mvout((uintptr_t)out, 1, OUT_ROWS, 1); - bb_mem_release(0); - bb_mem_release(1); - - if (compare_i8_matrices(out, exp, OUT_ROWS, LANES)) { - printf("im2col k3 5x5 partial-M PASSED\n"); - return 0; - } - printf("im2col k3 5x5 partial-M FAILED\n"); - return 1; -} diff --git a/examples/balls/im2col/workloads/ctests/im2col_k3_test.c b/examples/balls/im2col/workloads/ctests/im2col_k3_test.c deleted file mode 100644 index 96897dc02..000000000 --- a/examples/balls/im2col/workloads/ctests/im2col_k3_test.c +++ /dev/null @@ -1,74 +0,0 @@ -#include "buckyball.h" -#include -#include -#include -#include - -/* exact M tile: 6x6 k3 -> 16 windows */ -enum { ITER = 6, K = 3, STRIDE = 1, PAD = 0, LANES = 16 }; -enum { - OUT_DIM = (ITER + 2 * PAD - K) / STRIDE + 1, - WINDOWS = OUT_DIM * OUT_DIM, - KERNEL = K * K, - M_TILES = (WINDOWS + LANES - 1) / LANES, - K_TILES = (KERNEL + LANES - 1) / LANES, - OUT_ROWS = M_TILES * K_TILES * LANES, -}; - -static elem_t in[ITER * ITER] __attribute__((aligned(64))) = { - -1, 2, 3, 4, -5, 6, 0, 8, 9, -10, 11, 12, - 13, 0, -15, 16, 17, 18, 19, -20, 0, 22, 23, -24, - -25, 26, 27, 0, -29, 30, 31, 32, 33, -34, 0, 36, -}; -static elem_t packed[BANK_LINES * LANES] __attribute__((aligned(64))); -static elem_t out[OUT_ROWS * LANES] __attribute__((aligned(64))); -static elem_t exp[OUT_ROWS * LANES] __attribute__((aligned(64))); - -static void build_expected(void) { - int w = 0; - clear_i8_matrix(exp, OUT_ROWS, LANES); - for (int orow = 0; orow < OUT_DIM; orow++) { - for (int ocol = 0; ocol < OUT_DIM; ocol++) { - for (int krow = 0; krow < K; krow++) { - for (int kcol = 0; kcol < K; kcol++) { - int ki = krow * K + kcol; - int bank_row = - ((w / LANES) * K_TILES + ki / LANES) * LANES + (w % LANES); - int row = orow * STRIDE + krow - PAD; - int col = ocol * STRIDE + kcol - PAD; - exp[bank_row * LANES + ki % LANES] = - (row < 0 || row >= ITER || col < 0 || col >= ITER) - ? 0 - : in[row * ITER + col]; - } - } - w++; - } - } -} - -int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); -#endif - clear_i8_matrix(out, OUT_ROWS, LANES); - build_expected(); - clear_i8_matrix(packed, BANK_LINES, LANES); - for (int i = 0; i < ITER * ITER; ++i) - packed[i * LANES] = in[i]; - - bb_mem_alloc(0, 1, 1); - bb_mem_alloc(1, 1, 1); - bb_mvin((uintptr_t)packed, 0, BANK_LINES, 1); - bb_im2col(0, 1, ITER, K, STRIDE, PAD, 0, 0, 0, 0, 0, WINDOWS); - bb_mvout((uintptr_t)out, 1, OUT_ROWS, 1); - bb_mem_release(0); - bb_mem_release(1); - - if (compare_i8_matrices(out, exp, OUT_ROWS, LANES)) { - printf("im2col k3 6x6 exact-M PASSED\n"); - return 0; - } - printf("im2col k3 6x6 exact-M FAILED\n"); - return 1; -} diff --git a/examples/balls/im2col/workloads/ctests/im2col_k5_test.c b/examples/balls/im2col/workloads/ctests/im2col_k5_test.c deleted file mode 100644 index 590978ade..000000000 --- a/examples/balls/im2col/workloads/ctests/im2col_k5_test.c +++ /dev/null @@ -1,73 +0,0 @@ -#include "buckyball.h" -#include -#include -#include -#include - -enum { ITER = 7, K = 5, STRIDE = 1, PAD = 0, LANES = 16 }; -enum { - OUT_DIM = (ITER + 2 * PAD - K) / STRIDE + 1, - WINDOWS = OUT_DIM * OUT_DIM, - KERNEL = K * K, - M_TILES = (WINDOWS + LANES - 1) / LANES, - K_TILES = (KERNEL + LANES - 1) / LANES, - OUT_ROWS = M_TILES * K_TILES * LANES, -}; - -static elem_t in[ITER * ITER] __attribute__((aligned(64))) = { - 2, -4, 6, 0, 8, -10, 12, -1, 3, 5, 7, 0, 11, -13, 14, 0, 16, - -18, 20, 22, 24, -15, 17, 19, 0, 21, -23, 25, 26, 28, 0, 30, -32, 34, - 36, -27, 29, 31, 33, 0, 35, -37, 38, 0, 40, -42, 44, 46, 48, -}; -static elem_t packed[BANK_LINES * LANES] __attribute__((aligned(64))); -static elem_t out[OUT_ROWS * LANES] __attribute__((aligned(64))); -static elem_t exp[OUT_ROWS * LANES] __attribute__((aligned(64))); - -static void build_expected(void) { - int w = 0; - clear_i8_matrix(exp, OUT_ROWS, LANES); - for (int orow = 0; orow < OUT_DIM; orow++) { - for (int ocol = 0; ocol < OUT_DIM; ocol++) { - for (int krow = 0; krow < K; krow++) { - for (int kcol = 0; kcol < K; kcol++) { - int ki = krow * K + kcol; - int bank_row = - ((w / LANES) * K_TILES + ki / LANES) * LANES + (w % LANES); - int row = orow * STRIDE + krow - PAD; - int col = ocol * STRIDE + kcol - PAD; - exp[bank_row * LANES + ki % LANES] = - (row < 0 || row >= ITER || col < 0 || col >= ITER) - ? 0 - : in[row * ITER + col]; - } - } - w++; - } - } -} - -int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); -#endif - clear_i8_matrix(out, OUT_ROWS, LANES); - build_expected(); - clear_i8_matrix(packed, BANK_LINES, LANES); - for (int i = 0; i < ITER * ITER; ++i) - packed[i * LANES] = in[i]; - - bb_mem_alloc(0, 1, 1); - bb_mem_alloc(1, 1, 1); - bb_mvin((uintptr_t)packed, 0, BANK_LINES, 1); - bb_im2col(0, 1, ITER, K, STRIDE, PAD, 0, 0, 0, 0, 0, WINDOWS); - bb_mvout((uintptr_t)out, 1, OUT_ROWS, 1); - bb_mem_release(0); - bb_mem_release(1); - - if (compare_i8_matrices(out, exp, OUT_ROWS, LANES)) { - printf("im2col k5 7x7 PASSED\n"); - return 0; - } - printf("im2col k5 7x7 FAILED\n"); - return 1; -} diff --git a/examples/balls/im2col/workloads/ctests/im2col_k7_test.c b/examples/balls/im2col/workloads/ctests/im2col_k7_test.c deleted file mode 100644 index 11c9c13e4..000000000 --- a/examples/balls/im2col/workloads/ctests/im2col_k7_test.c +++ /dev/null @@ -1,73 +0,0 @@ -#include "buckyball.h" -#include -#include -#include -#include - -enum { ITER = 7, K = 7, STRIDE = 1, PAD = 0, LANES = 16 }; -enum { - OUT_DIM = (ITER + 2 * PAD - K) / STRIDE + 1, - WINDOWS = OUT_DIM * OUT_DIM, - KERNEL = K * K, - M_TILES = (WINDOWS + LANES - 1) / LANES, - K_TILES = (KERNEL + LANES - 1) / LANES, - OUT_ROWS = M_TILES * K_TILES * LANES, -}; - -static elem_t in[ITER * ITER] __attribute__((aligned(64))) = { - -5, 1, 2, 3, 0, 4, 5, 6, -7, 8, 9, 10, 0, 11, 12, 13, -14, - 15, 16, 17, 0, 0, 18, 19, -20, 21, 22, 23, 24, 0, 25, 26, -27, 28, - 29, 30, 31, 0, 32, 33, -34, 35, 36, 37, 38, 0, 39, 40, -41, -}; -static elem_t packed[BANK_LINES * LANES] __attribute__((aligned(64))); -static elem_t out[OUT_ROWS * LANES] __attribute__((aligned(64))); -static elem_t exp[OUT_ROWS * LANES] __attribute__((aligned(64))); - -static void build_expected(void) { - int w = 0; - clear_i8_matrix(exp, OUT_ROWS, LANES); - for (int orow = 0; orow < OUT_DIM; orow++) { - for (int ocol = 0; ocol < OUT_DIM; ocol++) { - for (int krow = 0; krow < K; krow++) { - for (int kcol = 0; kcol < K; kcol++) { - int ki = krow * K + kcol; - int bank_row = - ((w / LANES) * K_TILES + ki / LANES) * LANES + (w % LANES); - int row = orow * STRIDE + krow - PAD; - int col = ocol * STRIDE + kcol - PAD; - exp[bank_row * LANES + ki % LANES] = - (row < 0 || row >= ITER || col < 0 || col >= ITER) - ? 0 - : in[row * ITER + col]; - } - } - w++; - } - } -} - -int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); -#endif - clear_i8_matrix(out, OUT_ROWS, LANES); - build_expected(); - clear_i8_matrix(packed, BANK_LINES, LANES); - for (int i = 0; i < ITER * ITER; ++i) - packed[i * LANES] = in[i]; - - bb_mem_alloc(0, 1, 1); - bb_mem_alloc(1, 1, 1); - bb_mvin((uintptr_t)packed, 0, BANK_LINES, 1); - bb_im2col(0, 1, ITER, K, STRIDE, PAD, 0, 0, 0, 0, 0, WINDOWS); - bb_mvout((uintptr_t)out, 1, OUT_ROWS, 1); - bb_mem_release(0); - bb_mem_release(1); - - if (compare_i8_matrices(out, exp, OUT_ROWS, LANES)) { - printf("im2col k7 7x7 PASSED\n"); - return 0; - } - printf("im2col k7 7x7 FAILED\n"); - return 1; -} diff --git a/examples/balls/im2col/workloads/ctests/im2col_window_test.c b/examples/balls/im2col/workloads/ctests/im2col_window_test.c deleted file mode 100644 index 559aeeccc..000000000 --- a/examples/balls/im2col/workloads/ctests/im2col_window_test.c +++ /dev/null @@ -1,45 +0,0 @@ -#include "buckyball.h" -#include -#include -#include -#include -#include - -enum { INPUT_BASE = 5, LANE = 7 }; - -static int8_t input[64 * 16] __attribute__((aligned(64))); -static int8_t actual[16 * 16] __attribute__((aligned(64))); - -int main(void) { - for (int i = 0; i < 64 * 16; ++i) - input[i] = -99; - for (int i = 0; i < 36; ++i) - input[(INPUT_BASE + i) * 16 + LANE] = i + 1; - - bb_mem_alloc(0, 1, 1); - bb_mem_alloc(1, 1, 1); - bb_mvin((uintptr_t)input, 0, 64, 1); - bb_im2col(0, 1, 6, 3, 1, 0, INPUT_BASE, LANE, 0, 0, 4, 8); - bb_mvout((uintptr_t)actual, 1, 16, 1); - - for (int local = 0; local < 8; ++local) { - int global = local + 4; - int output_row = global / 4; - int output_col = global % 4; - for (int kr = 0; kr < 3; ++kr) { - for (int kc = 0; kc < 3; ++kc) { - int expected = - input[(INPUT_BASE + (output_row + kr) * 6 + output_col + kc) * 16 + - LANE]; - int index = local * 16 + kr * 3 + kc; - if (actual[index] != expected) { - printf("im2col_window FAIL local=%d k=%d expected=%d actual=%d\n", - local, kr * 3 + kc, expected, actual[index]); - return 1; - } - } - } - } - printf("im2col_window PASS\n"); - return 0; -} diff --git a/examples/balls/im2col/workloads/isa/im2col.h b/examples/balls/im2col/workloads/isa/im2col.h deleted file mode 100644 index aa7834692..000000000 --- a/examples/balls/im2col/workloads/isa/im2col.h +++ /dev/null @@ -1,18 +0,0 @@ -#ifndef _BB_IM2COL_H_ -#define _BB_IM2COL_H_ - -#include -#include - -#define bb_im2col(input_bank, output_bank, input_size, kernel, stride, \ - padding, input_base, lane, start_row, start_col, \ - window_start, window_count) \ - BUCKYBALL_INSTRUCTION_R_R( \ - BB_BANK0(input_bank) | BB_BANK2(output_bank) | BB_ITER(input_size), \ - FIELD(kernel, 0, 7) | FIELD(stride, 8, 15) | FIELD(padding, 16, 23) | \ - FIELD(start_col, 24, 31) | FIELD(start_row, 32, 39) | \ - FIELD(input_base, 40, 45) | FIELD(lane, 46, 49) | \ - FIELD(window_start, 50, 55) | FIELD(window_count, 56, 62), \ - BB_FUNC7(IM2COL)) - -#endif // _BB_IM2COL_H_ diff --git a/examples/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3.mlir b/examples/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3.mlir deleted file mode 100644 index 31c376a82..000000000 --- a/examples/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3.mlir +++ /dev/null @@ -1,59 +0,0 @@ -// Already-lowered ball Op: buckyball.im2col (physical-bank form). -// Same 6x6 k3 shape / golden as bank_im2col. - -func.func private @check_result(memref<16x16xi8>) -> () - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %c0 = arith.constant 0 : index - %c1 = arith.constant 1 : index - %c5 = arith.constant 5 : index - %c6 = arith.constant 6 : index - %c7 = arith.constant 7 : index - - %input = memref.alloc() alignment = 64 : memref<64x16xi8> - %output = memref.alloc() alignment = 64 : memref<16x16xi8> - linalg.fill ins(%zero_i8 : i8) outs(%input : memref<64x16xi8>) - linalg.fill ins(%zero_i8 : i8) outs(%output : memref<16x16xi8>) - - scf.for %r = %c0 to %c6 step %c1 { - scf.for %col = %c0 to %c6 step %c1 { - %tmp = arith.muli %r, %c6 : index - %idx = arith.addi %tmp, %col : index - %bank_row = arith.addi %c5, %idx : index - %ri = arith.index_cast %r : index to i32 - %ci = arith.index_cast %col : index to i32 - %v32 = arith.addi %ri, %ci : i32 - %v8 = arith.trunci %v32 : i32 to i8 - memref.store %v8, %input[%bank_row, %c7] : memref<64x16xi8> - } - } - - %iter = arith.constant 6 : i64 - %ksize = arith.constant 3 : i64 - %stride = arith.constant 1 : i64 - %padding = arith.constant 0 : i64 - %din = arith.constant 64 : i64 - %dout = arith.constant 16 : i64 - %s = arith.constant 1 : i64 - - %in = buckyball.bank_alloc - %out = buckyball.bank_alloc - %loaded = buckyball.bank_mvin %input %in %din %s - : memref<64x16xi8> i64 i64 i64 - %inputBase = arith.constant 5 : i64 - %lane = arith.constant 7 : i64 - buckyball.im2col %loaded, %out, %iter, %ksize, %stride, %padding, - %inputBase, %lane - : i64 - %stored = buckyball.bank_mvout %output %out %dout %s - : memref<16x16xi8> i64 i64 i64 - buckyball.bank_release %loaded : i64 - buckyball.bank_release %stored : i64 - - func.call @check_result(%output) : (memref<16x16xi8>) -> () - memref.dealloc %input : memref<64x16xi8> - memref.dealloc %output : memref<16x16xi8> - return %zero_i8 : i8 -} diff --git a/examples/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3_main.cpp b/examples/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3_main.cpp deleted file mode 100644 index 3e388bacd..000000000 --- a/examples/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3_main.cpp +++ /dev/null @@ -1,71 +0,0 @@ -#include -#include -#include - -enum { ITER = 6, K = 3, STRIDE = 1, PAD = 0, LANES = 16 }; -enum { - OUT_DIM = (ITER + 2 * PAD - K) / STRIDE + 1, - WINDOWS = OUT_DIM * OUT_DIM, - KERNEL = K * K, - M_TILES = (WINDOWS + LANES - 1) / LANES, - K_TILES = (KERNEL + LANES - 1) / LANES, - OUT_ROWS = M_TILES * K_TILES * LANES, -}; - -static void fail(void) { -#ifdef BAREMETAL - volatile uint32_t *sim_exit = (volatile uint32_t *)0x60000000; - *sim_exit = 1; - while (1) { - } -#else - exit(1); -#endif -} - -static int8_t src(int row, int col) { - if (row < 0 || row >= ITER || col < 0 || col >= ITER) - return 0; - return (int8_t)(row + col); -} - -#ifdef __cplusplus -extern "C" -#endif - void check_result(int8_t *allocated, int8_t *aligned, int64_t offset, - int64_t size0, int64_t size1, int64_t stride0, - int64_t stride1) { - (void)allocated; - if (size0 != OUT_ROWS || size1 != LANES || stride0 != LANES || stride1 != 1) { - printf("FAILED: im2col ball_im2col unexpected shape " - "(size=%ldx%ld stride=%ldx%ld)\n", - (long)size0, (long)size1, (long)stride0, (long)stride1); - fail(); - } - - int8_t *out = aligned + offset; - int w = 0; - for (int orow = 0; orow < OUT_DIM; orow++) { - for (int ocol = 0; ocol < OUT_DIM; ocol++) { - for (int krow = 0; krow < K; krow++) { - for (int kcol = 0; kcol < K; kcol++) { - int ki = krow * K + kcol; - int bank_row = - ((w / LANES) * K_TILES + ki / LANES) * LANES + (w % LANES); - int8_t exp = - src(orow * STRIDE + krow - PAD, ocol * STRIDE + kcol - PAD); - int8_t got = out[bank_row * stride0 + (ki % LANES) * stride1]; - if (got != exp) { - printf("FAILED: im2col ball_im2col w=%d ki=%d " - "(expected %d, got %d)\n", - w, ki, (int)exp, (int)got); - fail(); - } - } - } - w++; - } - } - - printf("PASSED: im2col ball_im2col 6x6 k3\n"); -} diff --git a/examples/balls/im2col/workloads/mlir_tests/bank/CMakeLists.txt b/examples/balls/im2col/workloads/mlir_tests/bank/CMakeLists.txt deleted file mode 100644 index 6fca83f3c..000000000 --- a/examples/balls/im2col/workloads/mlir_tests/bank/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) - set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) -endif() -set(BUCKYBALL_MLIR_TEST_PREFIX bank) - -set(_params "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_MLIR_ACTIVE_TARGET}/params.h") -file(STRINGS "${_params}" _lines_define REGEX "^#define BANK_LINES ") -file(STRINGS "${_params}" _width_define REGEX "^#define BANK_WIDTH ") -file(STRINGS "${_params}" _sqrt_define REGEX "^#define BANK_ISQRT ") -string(REGEX REPLACE ".* " "" _lines "${_lines_define}") -string(REGEX REPLACE ".* " "" _width "${_width_define}") -string(REGEX REPLACE ".* " "" _sqrt "${_sqrt_define}") -math(EXPR _remainder "${_lines} % 64") -if(_lines LESS 64 OR NOT _remainder EQUAL 0 OR _width LESS 72) - message(FATAL_ERROR "Im2col K3 bank batches require BANK_LINES divisible by 64 and at least 9 INT8 lanes (got lines=${_lines}, width=${_width})") -endif() -add_buckyball_mlir_test(im2col_k3_padded_bank_windows TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/balls/im2col/workloads/mlir_tests/bank/im2col_k3_padded_bank_windows.mlir.in b/examples/balls/im2col/workloads/mlir_tests/bank/im2col_k3_padded_bank_windows.mlir.in deleted file mode 100644 index 7db8852e9..000000000 --- a/examples/balls/im2col/workloads/mlir_tests/bank/im2col_k3_padded_bank_windows.mlir.in +++ /dev/null @@ -1,41 +0,0 @@ -func.func private @check_result(memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %side = arith.constant 8 : index - %rows = arith.constant @BB_BANK_LINES@ : index - %windows = arith.constant 64 : index - %lane = arith.constant 7 : index - %input_depth = arith.constant 64 : i64 - %output_depth = arith.constant 64 : i64 - %side64 = arith.constant 8 : i64 - %one64 = arith.constant 1 : i64 - %three64 = arith.constant 3 : i64 - %input = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - %output = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - linalg.fill ins(%zero_i8 : i8) outs(%input : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) - scf.for %index = %zero to %rows step %one { - %value32 = arith.index_cast %index : index to i32 - %value = arith.trunci %value32 : i32 to i8 - memref.store %value, %input[%index, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - } - %input_bank = buckyball.bank_alloc - %output_bank = buckyball.bank_alloc - %zero64 = arith.constant 0 : i64 - %lane64 = arith.constant 7 : i64 - %final:2 = scf.for %base = %zero to %rows step %windows iter_args(%ib = %input_bank, %ob = %output_bank) -> (i64, i64) { - %in = memref.subview %input[%base, 0] [64, @BB_BANK_I8_LANES@] [1, 1] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> to memref<64x@BB_BANK_I8_LANES@xi8, strided<[@BB_BANK_I8_LANES@, 1], offset: ?>> - %out = memref.subview %output[%base, 0] [64, @BB_BANK_I8_LANES@] [1, 1] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> to memref<64x@BB_BANK_I8_LANES@xi8, strided<[@BB_BANK_I8_LANES@, 1], offset: ?>> - %loaded = buckyball.bank_mvin %in %ib %input_depth %one64 : memref<64x@BB_BANK_I8_LANES@xi8, strided<[@BB_BANK_I8_LANES@, 1], offset: ?>> i64 i64 i64 - %result = buckyball.bank_im2col %loaded %ob %side64 %three64 %one64 %one64 %zero64 %lane64 : i64 i64 i64 i64 i64 i64 i64 i64 - %stored = buckyball.bank_mvout %out %result %output_depth %one64 : memref<64x@BB_BANK_I8_LANES@xi8, strided<[@BB_BANK_I8_LANES@, 1], offset: ?>> i64 i64 i64 - scf.yield %loaded, %stored : i64, i64 - } - func.call @check_result(%output) : (memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () - buckyball.bank_release %final#0 : i64 - buckyball.bank_release %final#1 : i64 - memref.dealloc %input : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - memref.dealloc %output : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - return %zero_i8 : i8 -} diff --git a/examples/balls/im2col/workloads/mlir_tests/bank/im2col_k3_padded_bank_windows_main.cpp b/examples/balls/im2col/workloads/mlir_tests/bank/im2col_k3_padded_bank_windows_main.cpp deleted file mode 100644 index f0be7350b..000000000 --- a/examples/balls/im2col/workloads/mlir_tests/bank/im2col_k3_padded_bank_windows_main.cpp +++ /dev/null @@ -1,34 +0,0 @@ -#include -#include -#include -#include - -extern "C" void check_result(int8_t *, int8_t *data, int64_t offset, - int64_t rows, int64_t lanes, int64_t stride, - int64_t lane_stride) { - if (rows != BANK_LINES || lanes != BANK_WIDTH / 8 || stride != lanes || - lane_stride != 1) - exit(1); - constexpr int windows = BANK_LINES; - constexpr int side = 8; - for (int window = 0; window < windows; ++window) { - int batch = window / 64; - int y = (window % 64) / side; - int x = window % side; - for (int ky = 0; ky < 3; ++ky) - for (int kx = 0; kx < 3; ++kx) { - int source_y = y + ky - 1; - int source_x = x + kx - 1; - int expected = 0; - if (source_y >= 0 && source_y < side && source_x >= 0 && - source_x < side) - expected = batch * 64 + source_y * side + source_x; - int lane = ky * 3 + kx; - if (data[offset + window * stride + lane] != (int8_t)expected) { - printf("im2col K3 bank mismatch window=%d lane=%d\n", window, lane); - exit(1); - } - } - } - printf("im2col K3 padded bank PASS rows=%d\n", windows); -} diff --git a/examples/balls/int2fp/arch/src/main/scala/Int2Fp.scala b/examples/balls/int2fp/arch/src/main/scala/Int2Fp.scala deleted file mode 100644 index 8d4b54bd7..000000000 --- a/examples/balls/int2fp/arch/src/main/scala/Int2Fp.scala +++ /dev/null @@ -1,205 +0,0 @@ -package examples.balls.int2fp - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} - -import framework.balldomain.blink.{BallStatus, BankRead, BankWrite} -import framework.balldomain.rs.{BallRsComplete, BallRsIssue} -import framework.top.GlobalConfig - -@instantiable -class Int2Fp(val b: GlobalConfig) extends Module { - - private val mapping = b.ballDomain.ballIdMappings - .find(_.ballName == "Int2FpBall") - .getOrElse(throw new IllegalArgumentException("Int2FpBall not found in config")) - - private val funct = b.ballDomain.ballISA - .find(_.mnemonic == "INT32_TO_FP32") - .map(_.funct7) - .getOrElse(throw new IllegalArgumentException("INT32_TO_FP32 not found in ballISA")) - - require(b.memDomain.bankWidth == 128, "Int2FpBall requires 128-bit bank rows") - require(mapping.inBW == 2, "Int2FpBall requires inBW=2") - require(mapping.outBW == 1, "Int2FpBall requires outBW=1") - require((funct >> 4) == 4, "INT32_TO_FP32 must encode two reads and one write") - - @public - val io = IO(new Bundle { - val cmdReq = Flipped(Decoupled(new BallRsIssue(b))) - val cmdResp = Decoupled(new BallRsComplete(b)) - val bankRead = Vec(mapping.inBW, Flipped(new BankRead(b))) - val bankWrite = Vec(mapping.outBW, Flipped(new BankWrite(b))) - val status = new BallStatus - }) - - private val idle :: readReq :: readResp :: convert :: multiply :: writeReq :: writeResp :: complete :: Nil = - Enum(8) - private val state = RegInit(idle) - - private val robIdReg = RegInit(0.U(log2Up(b.frontend.rob_entries).W)) - private val isSubReg = RegInit(false.B) - private val subRobIdReg = RegInit(0.U(log2Up(b.frontend.sub_rob_depth * 4).W)) - private val inputBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - private val scaleBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - private val outputBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - private val iterReg = RegInit(0.U(b.frontend.iter_len.W)) - private val rowReg = RegInit(0.U(log2Ceil(b.memDomain.bankEntries).W)) - private val reluReg = RegInit(false.B) - private val inputData = Reg(UInt(128.W)) - private val scaleData = Reg(UInt(128.W)) - private val fpOutputs = Reg(Vec(4, UInt(32.W))) - private val mulStarted = RegInit(false.B) - private val laneReg = RegInit(0.U(2.W)) - private val writeData = Reg(UInt(128.W)) - private val multiplier = Module(new Fp32Mul) - - private def int32ToFp32(value: UInt): UInt = { - val sign = value(31) - val abs = Mux(sign, (~value).asUInt + 1.U, value) - val zero = abs === 0.U - val leading = 31.U - PriorityEncoder(Reverse(abs)) - val shift = Mux(leading > 23.U, leading - 23.U, 0.U) - val truncated = abs >> shift - val half = Mux(shift === 0.U, 0.U(32.W), 1.U(32.W) << (shift - 1.U)) - val remainder = Mux(shift === 0.U, 0.U(32.W), abs & ((1.U(32.W) << shift) - 1.U)) - val roundUp = remainder > half || (remainder === half && truncated(0)) - val rounded = truncated +& roundUp.asUInt - val carry = rounded(24) - val significand = Mux(leading > 23.U, rounded(22, 0), (abs << (23.U - leading))(22, 0)) - val exponent = (leading +& 127.U +& carry)(7, 0) - Mux(zero, 0.U, Cat(sign, exponent, Mux(carry, 0.U(23.W), significand))) - } - - private def positiveFinite(value: UInt): Bool = - !value(31) && value(30, 23) =/= 255.U && value(30, 0) =/= 0.U - - for (i <- 0 until mapping.inBW) { - io.bankRead(i).rob_id := robIdReg - io.bankRead(i).ball_id := 0.U - io.bankRead(i).bank_id := Mux(i.U === 0.U, inputBank, scaleBank) - io.bankRead(i).group_id := 0.U - io.bankRead(i).io.req.valid := false.B - io.bankRead(i).io.req.bits.addr := Mux(i.U === 0.U, rowReg, rowReg(1, 0)) - io.bankRead(i).io.resp.ready := false.B - } - io.bankWrite(0).rob_id := robIdReg - io.bankWrite(0).ball_id := 0.U - io.bankWrite(0).bank_id := outputBank - io.bankWrite(0).group_id := 0.U - io.bankWrite(0).io.req.valid := false.B - io.bankWrite(0).io.req.bits.addr := rowReg - io.bankWrite(0).io.req.bits.data := writeData - io.bankWrite(0).io.req.bits.mask := VecInit(Seq.fill(b.memDomain.bankMaskLen)(true.B)) - io.bankWrite(0).io.resp.ready := false.B - - private val inputValue = (inputData >> (laneReg << 5))(31, 0) - private val scaleValue = (scaleData >> (laneReg << 5))(31, 0) - multiplier.io.start := false.B - multiplier.io.a := int32ToFp32(Mux(reluReg && inputValue(31), 0.U, inputValue)) - multiplier.io.b := scaleValue - - io.cmdReq.ready := state === idle - io.cmdResp.valid := state === complete - io.cmdResp.bits.rob_id := robIdReg - io.cmdResp.bits.is_sub := isSubReg - io.cmdResp.bits.sub_rob_id := subRobIdReg - io.status.idle := state === idle - io.status.running := state =/= idle && state =/= complete - - switch(state) { - is(idle) { - when(io.cmdReq.fire) { - val cmd = io.cmdReq.bits.cmd - assert(cmd.funct7 === funct.U, "Int2FpBall received an unknown funct7") - assert(cmd.iter > 0.U && cmd.iter(1, 0) === 0.U, "INT32_TO_FP32 iter must be a positive multiple of four") - assert(cmd.iter <= b.memDomain.bankEntries.U, "INT32_TO_FP32 input exceeds bank depth") - assert(cmd.op1_bank < b.memDomain.virtualBankCount.U, "INT32_TO_FP32 input bank is invalid") - assert(cmd.op2_bank < b.memDomain.virtualBankCount.U, "INT32_TO_FP32 scale bank is invalid") - assert(cmd.wr_bank < b.memDomain.virtualBankCount.U, "INT32_TO_FP32 output bank is invalid") - assert( - cmd.op1_col === 1.U && cmd.op2_col === 1.U && cmd.wr_col === 1.U, - "INT32_TO_FP32 operands must each occupy one bank" - ) - assert( - cmd.op1_bank =/= cmd.op2_bank && cmd.op1_bank =/= cmd.wr_bank && cmd.op2_bank =/= cmd.wr_bank, - "INT32_TO_FP32 banks must be distinct" - ) - assert(cmd.rs2(63, 1) === 0.U, "INT32_TO_FP32 reserves rs2[63:1]") - - robIdReg := io.cmdReq.bits.rob_id - isSubReg := io.cmdReq.bits.is_sub - subRobIdReg := io.cmdReq.bits.sub_rob_id - inputBank := cmd.op1_bank - scaleBank := cmd.op2_bank - outputBank := cmd.wr_bank - iterReg := cmd.iter - rowReg := 0.U - reluReg := cmd.rs2(0) - state := readReq - } - } - is(readReq) { - for (i <- 0 until mapping.inBW) io.bankRead(i).io.req.valid := true.B - when(io.bankRead.map(_.io.req.fire).reduce(_ && _)) { - state := readResp - } - } - is(readResp) { - for (i <- 0 until mapping.inBW) io.bankRead(i).io.resp.ready := true.B - when(io.bankRead.map(_.io.resp.fire).reduce(_ && _)) { - inputData := io.bankRead(0).io.resp.bits.data - scaleData := io.bankRead(1).io.resp.bits.data - state := convert - } - } - is(convert) { - for (i <- 0 until 4) { - val scale = scaleData(32 * i + 31, 32 * i) - assert(positiveFinite(scale), "INT32_TO_FP32 scales must be finite and positive") - } - mulStarted := false.B - laneReg := 0.U - state := multiply - } - is(multiply) { - when(!mulStarted) { - multiplier.io.start := true.B - mulStarted := true.B - } - when(multiplier.io.done) { - fpOutputs(laneReg) := multiplier.io.result - when(laneReg === 3.U) { - writeData := Cat(multiplier.io.result, fpOutputs(2), fpOutputs(1), fpOutputs(0)) - state := writeReq - }.otherwise { - laneReg := laneReg + 1.U - mulStarted := false.B - } - } - } - is(writeReq) { - io.bankWrite(0).io.req.valid := true.B - when(io.bankWrite(0).io.req.fire) { - state := writeResp - } - } - is(writeResp) { - io.bankWrite(0).io.resp.ready := true.B - when(io.bankWrite(0).io.resp.fire) { - when(rowReg === iterReg - 1.U) { - state := complete - }.otherwise { - rowReg := rowReg + 1.U - state := readReq - } - } - } - is(complete) { - when(io.cmdResp.fire) { - state := idle - } - } - } -} diff --git a/examples/balls/int2fp/arch/src/main/scala/Int2FpBall.scala b/examples/balls/int2fp/arch/src/main/scala/Int2FpBall.scala deleted file mode 100644 index cb4d34922..000000000 --- a/examples/balls/int2fp/arch/src/main/scala/Int2FpBall.scala +++ /dev/null @@ -1,34 +0,0 @@ -package examples.balls.int2fp - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} - -import framework.balldomain.blink.{BallStatus, BlinkIO, HasBallStatus, HasBlink, SubRobRow} -import framework.balldomain.blink.mmio.{MmioRead, MmioWrite} -import framework.top.GlobalConfig - -@instantiable -class Int2FpBall(val b: GlobalConfig) extends Module with HasBlink with HasBallStatus { - - private val mapping = b.ballDomain.ballIdMappings - .find(_.ballName == "Int2FpBall") - .getOrElse(throw new IllegalArgumentException("Int2FpBall not found in config")) - - @public - val io = IO(new BlinkIO(b, mapping.inBW, mapping.outBW)) - def blink: BlinkIO = io - def status: BallStatus = io.status - dontTouch(io) - - private val unit: Instance[Int2Fp] = Instantiate(new Int2Fp(b)) - unit.io.cmdReq <> io.cmdReq - unit.io.cmdResp <> io.cmdResp - for (i <- 0 until mapping.inBW) unit.io.bankRead(i) <> io.bankRead(i) - for (i <- 0 until mapping.outBW) unit.io.bankWrite(i) <> io.bankWrite(i) - io.status <> unit.io.status - - io.subRobReq.valid := false.B - io.subRobReq.bits := SubRobRow.tieOff(b) - MmioRead.tieOff(io.mmioRead) - MmioWrite.tieOff(io.mmioWrite) -} diff --git a/examples/balls/int2fp/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp b/examples/balls/int2fp/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp deleted file mode 100644 index bdd086410..000000000 --- a/examples/balls/int2fp/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp +++ /dev/null @@ -1,39 +0,0 @@ -//===- AssignPhysicalBankPatterns.cpp - Int2Fp bank assignment patterns ---===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace mlir::buddy { -void populateInt2FpBallAssignPhysicalBankPatterns(RewritePatternSet &patterns, - PhysicalBankState &state); -} // namespace mlir::buddy - -namespace { - -class BankInt32ToFp32Pattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(BankInt32ToFp32Op op, - PatternRewriter &rewriter) const override { - rewriter.create(op.getLoc(), op.getInBank(), - op.getScaleBank(), op.getOutBank(), - op.getIter(), op.getReluAttr()); - rewriter.replaceOp(op, op.getOutBank()); - return success(); - } -}; - -} // namespace - -void mlir::buddy::populateInt2FpBallAssignPhysicalBankPatterns( - RewritePatternSet &patterns, mlir::buddy::PhysicalBankState &state) { - (void)state; - patterns.add(patterns.getContext()); -} diff --git a/examples/balls/int2fp/compiler/src/Dialect/Buckyball/Int2FpBall.td b/examples/balls/int2fp/compiler/src/Dialect/Buckyball/Int2FpBall.td deleted file mode 100644 index c5c241cd9..000000000 --- a/examples/balls/int2fp/compiler/src/Dialect/Buckyball/Int2FpBall.td +++ /dev/null @@ -1,40 +0,0 @@ -//===-- Int2FpBall.td - Int2FpBall operations --------------*- tablegen -*-===// -// -// Int2FpBall operations for Mega Kernel output conversion. -// -//===----------------------------------------------------------------------===// - -#ifndef INT2FPBALL_OPS -#define INT2FPBALL_OPS - -include "Buckyball.td" - -//===----------------------------------------------------------------------===// -// Int2FpBall operations -//===----------------------------------------------------------------------===// - -def Int32ToFp32Op : Buckyball_Op<"int32_to_fp32"> { - let summary = "Convert INT32 accumulators to FP32"; - let description = [{ - Converts four INT32 values per input row to FP32 and multiplies them by - the matching values from a 16-channel FP32 scale bank. Scale rows repeat - every four accumulator rows. - }]; - let arguments = (ins I64:$inputBankId, I64:$scaleBankId, - I64:$outputBankId, I64:$iter, BoolAttr:$relu); - let assemblyFormat = "$inputBankId `,` $scaleBankId `,` $outputBankId `,` " - "$iter prop-dict attr-dict `:` " - "type($inputBankId)"; -} - -def BankInt32ToFp32Op : Buckyball_Op<"bank_int32_to_fp32"> { - let summary = "INT32 to FP32 conversion with virtual bank handles"; - let arguments = (ins I64:$inBank, I64:$scaleBank, I64:$outBank, - I64:$iter, BoolAttr:$relu); - let results = (outs I64:$outBankOut); - let assemblyFormat = - "$inBank $scaleBank $outBank $iter prop-dict attr-dict `:` " - "type($inBank) type($scaleBank) type($outBank) type($iter)"; -} - -#endif // INT2FPBALL_OPS diff --git a/examples/balls/int2fp/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/balls/int2fp/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 98a4ca959..000000000 --- a/examples/balls/int2fp/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,46 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace buddy::buckyball; -using namespace buddy::buckyball::legalize; - -namespace { -struct Int32ToFp32Lowering : public ConvertOpToLLVMPattern { - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - - LogicalResult - matchAndRewrite(Int32ToFp32Op op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("Int2FpBall"); - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInputBankId(), - adaptor.getScaleBankId(), - adaptor.getOutputBankId(), adaptor.getIter()); - rewriter.replaceOpWithNewOp( - op, rs1, cstI64(rewriter, loc, op.getRelu() ? 1 : 0), - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("INT32_TO_FP32"))); - return success(); - } -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateInt2FpBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t) { - (void)stable; - patterns.add(converter); -} - -void configureInt2FpBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable) { - (void)stable; - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/examples/balls/int2fp/configs/Int2FpBallParam.scala b/examples/balls/int2fp/configs/Int2FpBallParam.scala deleted file mode 100644 index 2b61dd2d1..000000000 --- a/examples/balls/int2fp/configs/Int2FpBallParam.scala +++ /dev/null @@ -1,21 +0,0 @@ -package examples.balls.int2fp.configs - -import framework.balldomain.configs.BallParamLoader -import framework.top.GlobalConfig - -case class Int2FpBallParam( - inputType: String, - outputType: String) - -object Int2FpBallParam { - private val ballName = "Int2FpBall" - - def apply(b: GlobalConfig): Int2FpBallParam = { - val tbl = BallParamLoader.ballTable(b, ballName) - Int2FpBallParam( - inputType = BallParamLoader.str(tbl, "inputType"), - outputType = BallParamLoader.str(tbl, "outputType") - ) - } - -} diff --git a/examples/balls/int2fp/configs/default.toml b/examples/balls/int2fp/configs/default.toml deleted file mode 100644 index cd0c70328..000000000 --- a/examples/balls/int2fp/configs/default.toml +++ /dev/null @@ -1,5 +0,0 @@ -# Int2Fp Ball Configuration - -[ball] -inputType = "INT32" -outputType = "FP32" diff --git a/examples/balls/int2fp/emu/src/dispatch.rs b/examples/balls/int2fp/emu/src/dispatch.rs deleted file mode 100644 index 8d0b6a128..000000000 --- a/examples/balls/int2fp/emu/src/dispatch.rs +++ /dev/null @@ -1,31 +0,0 @@ -use crate::inst::instruction::ExecContext; - -use super::int2fp; - -const BALL_CLASS: &str = "examples.balls.int2fp.Int2FpBall"; - -pub fn execute_known( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, - ctx: &mut ExecContext, -) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("INT32_TO_FP32") => Some(int2fp::execute(xs1, xs2, ctx)), - Some(_) | None => None, - } -} - -pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("INT32_TO_FP32") => Some(int2fp::latency(xs1, xs2)), - Some(_) | None => None, - } -} diff --git a/examples/balls/int2fp/emu/src/int2fp.rs b/examples/balls/int2fp/emu/src/int2fp.rs deleted file mode 100644 index 35fd35d79..000000000 --- a/examples/balls/int2fp/emu/src/int2fp.rs +++ /dev/null @@ -1,64 +0,0 @@ -use super::super::bank::bank_lines; -use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; -use super::instruction::ExecContext; - -pub(crate) fn execute(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { - let input_bank = rs1_b0(xs1); - let scale_bank = rs1_b1(xs1); - let output_bank = rs1_b2(xs1); - let iter = rs1_iter(xs1) as usize; - if input_bank == scale_bank || input_bank == output_bank || scale_bank == output_bank { - panic!("int32_to_fp32: banks must be distinct"); - } - if !ctx.config(input_bank).allocated - || !ctx.config(scale_bank).allocated - || !ctx.config(output_bank).allocated - { - panic!("int32_to_fp32: bank not allocated"); - } - if ctx.config(input_bank).cols != 1 - || ctx.config(scale_bank).cols != 1 - || ctx.config(output_bank).cols != 1 - { - panic!("int32_to_fp32: operands must each occupy one bank"); - } - if iter == 0 || iter % 4 != 0 || iter > bank_lines() { - panic!("int32_to_fp32: iter must be a positive multiple of four within bank depth"); - } - if xs2 >> 1 != 0 { - panic!("int32_to_fp32: rs2[63:1] must be zero"); - } - let relu = xs2 & 1 != 0; - - let input = pbank(ctx, input_bank); - let scale = pbank(ctx, scale_bank); - let output = pbank(ctx, output_bank); - for row in 0..iter { - for lane in 0..4 { - let offset = row * 16 + lane * 4; - let scale_offset = (row % 4) * 16 + lane * 4; - let value = - i32::from_le_bytes(ctx.banks[input][offset..offset + 4].try_into().unwrap()); - let factor = f32::from_bits(u32::from_le_bytes( - ctx.banks[scale][scale_offset..scale_offset + 4] - .try_into() - .unwrap(), - )); - if !factor.is_finite() || factor <= 0.0 { - panic!("int32_to_fp32: scales must be finite and positive"); - } - let value = if relu { value.max(0) } else { value }; - let result = (value as f32) * factor; - ctx.banks[output][offset..offset + 4].copy_from_slice(&result.to_le_bytes()); - } - } - 0 -} - -pub(crate) fn latency(xs1: u64, xs2: u64) -> u64 { - let iter = rs1_iter(xs1); - if iter == 0 || iter % 4 != 0 || xs2 >> 1 != 0 { - panic!("int32_to_fp32: illegal encoding"); - } - iter -} diff --git a/examples/balls/int2fp/emu/src/lib.rs b/examples/balls/int2fp/emu/src/lib.rs deleted file mode 100644 index 3f8080cde..000000000 --- a/examples/balls/int2fp/emu/src/lib.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub(crate) use crate::inst::{decode, instruction}; - -mod int2fp; - -mod dispatch; - -pub use dispatch::{cycles_after_issue, execute_known}; diff --git a/examples/balls/int2fp/emu/src/model.rs b/examples/balls/int2fp/emu/src/model.rs deleted file mode 100644 index a58c6d01a..000000000 --- a/examples/balls/int2fp/emu/src/model.rs +++ /dev/null @@ -1,117 +0,0 @@ -pub fn int2fp_fp32_bits(value: i32, dw_bits: u32) -> u32 { - fp32_multiply(int32_to_fp32(value), dw_bits) -} - -pub fn int2fp_dequant_bits(value: i32, da_bits: u32, dw_bits: u32) -> u32 { - int2fp_fp32_bits(value, fp32_multiply(da_bits, dw_bits)) -} - -pub fn int32_to_fp32(value: i32) -> u32 { - if value == 0 { - return 0; - } - - let sign = ((value as u32) >> 31) & 1; - let abs = if value == i32::MIN { - 0x8000_0000u32 - } else { - value.unsigned_abs() - }; - - let leading_one = 31u32 - abs.leading_zeros(); - let mut exponent = leading_one + 127; - let significand = if leading_one > 23 { - let right_shift = leading_one - 23; - let abs_wide = u64::from(abs); - let truncated = abs_wide >> right_shift; - let half = 1u64 << (right_shift - 1); - let remainder = abs_wide & ((1u64 << right_shift) - 1); - let round_up = remainder > half || (remainder == half && (truncated & 1) != 0); - let rounded = truncated + u64::from(round_up); - if ((rounded >> 24) & 1) != 0 { - exponent = leading_one + 128; - (rounded >> 1) as u32 - } else { - rounded as u32 - } - } else { - abs << (23 - leading_one) - }; - - (sign << 31) | ((exponent & 0xff) << 23) | (significand & 0x7f_ffff) -} - -fn fp32_multiply(a: u32, b: u32) -> u32 { - let a_sign = (a >> 31) & 1; - let b_sign = (b >> 31) & 1; - let a_exp = (a >> 23) & 0xff; - let b_exp = (b >> 23) & 0xff; - let a_frac = a & 0x7f_ffff; - let b_frac = b & 0x7f_ffff; - let a_mant = (1u64 << 23) | u64::from(a_frac); - let b_mant = (1u64 << 23) | u64::from(b_frac); - let a_zero = a_exp == 0 && a_frac == 0; - let b_zero = b_exp == 0 && b_frac == 0; - let prod = a_mant * b_mant; - let (sig, guard, round, sticky, exp_adjust) = if ((prod >> 47) & 1) != 0 { - ( - (prod >> 24) as u32, - ((prod >> 23) & 1) != 0, - ((prod >> 22) & 1) != 0, - (prod & ((1u64 << 22) - 1)) != 0, - 1u32, - ) - } else { - ( - (prod >> 23) as u32, - ((prod >> 22) & 1) != 0, - ((prod >> 21) & 1) != 0, - (prod & ((1u64 << 21) - 1)) != 0, - 0u32, - ) - }; - let increment = guard && (round || sticky || (sig & 1) != 0); - let rounded = u64::from(sig) + u64::from(increment); - let carry = ((rounded >> 24) & 1) as u32; - let final_sig = if carry != 0 { rounded >> 1 } else { rounded } as u32; - let exp_wide = (a_exp + b_exp + exp_adjust + carry).wrapping_sub(127) & 0x3ff; - - if a_zero || b_zero { - 0 - } else if (exp_wide & 0x200) != 0 { - 0 - } else if (exp_wide & 0x100) != 0 { - ((a_sign ^ b_sign) << 31) | (0xff << 23) - } else { - ((a_sign ^ b_sign) << 31) | ((exp_wide & 0xff) << 23) | (final_sig & 0x7f_ffff) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn int_to_fp_basic() { - let scale = 0x3F80_0000; - assert_eq!(int2fp_fp32_bits(1, scale), 0x3F80_0000); - assert_eq!(int2fp_fp32_bits(-1, scale), 0xBF80_0000); - assert_eq!(int2fp_fp32_bits(0, scale), 0); - assert_eq!(int2fp_fp32_bits(i32::MIN, scale), 0xCF00_0000); - } - - #[test] - fn int8_to_fp_scale() { - let scale = 0x3E80_0000; // 0.25 - assert_eq!(int2fp_fp32_bits(-128, scale), (-32.0f32).to_bits()); - assert_eq!(int2fp_fp32_bits(127, scale), (31.75f32).to_bits()); - } - - #[test] - fn dequant_uses_both_scales() { - assert_eq!( - int2fp_dequant_bits(8, 0x3f00_0000, 0x3e80_0000), - 1.0f32.to_bits() - ); - } -} diff --git a/examples/balls/int2fp/verify/README.md b/examples/balls/int2fp/verify/README.md deleted file mode 100644 index 8e3ec940c..000000000 --- a/examples/balls/int2fp/verify/README.md +++ /dev/null @@ -1,77 +0,0 @@ -# Int2FpBall UVM Verification - -Verifies the generated `Int2FpBall` module using the shared Blink UVM framework. - -RTL contract: one mnemonic `INT32_TO_FP32`, `inBW=2` (input + scale banks), -`outBW=1`, `rs2[0]=relu`, `rs2[63:1]=0`, iter a positive multiple of 4. -Scales are bank rows, not MMIO. - -## Structure - -- `../../../../verify/uvm/src/ball/bb_uvm_pkg.sv`: common Blink UVM agents and base env -- `src/common/int2fp_defs.svh`: DPI imports, `INT32_TO_FP32_FUNCT7`, timeouts -- `src/common/int2fp_items.svh`: `int2fp_cmd_item` with `load_rust_case` -- `src/seq/int2fp_sequences.svh`: one-case sequence -- `src/cov/int2fp_cov.svh`: relu `{0,1}` and iter `{4,8,16}`, 100% or fatal -- `src/env/int2fp_scoreboard.svh`: preload src/scale banks, compare writes vs DPI dst -- `src/env/int2fp_env.svh`: extends `bb_blink_env #(BB_IN_BW, BB_OUT_BW)` -- `src/tests/int2fp_test.svh`: `int2fp_ball_test` loops cases 0..5 -- `src/pkg/int2fp_pkg.sv`: package entry -- `src/tb_top.sv`: `bb_blink_if #(BB_IN_BW, BB_OUT_BW)` + `Int2FpBall` (2 read, 1 write) -- `filelists/int2fp_ball.f`: VCS filelist (`@UVM@` / `@RTL@`) - -## Test plan - -`+UVM_TESTNAME=int2fp_ball_test` runs six directed cases: - -| Index | iter | relu | Intent | -|-------|------|------|--------| -| 0 | 4 | 0 | MLIR-scale values | -| 1 | 4 | 1 | relu clamps negatives | -| 2 | 8 | 0 | eight-row, no relu | -| 3 | 8 | 1 | eight-row, relu | -| 4 | 16 | 0 | sixteen-row, no relu | -| 5 | 16 | 1 | sixteen-row, relu | - -Each case: reset, preload 16-byte i32 input rows and f32 scale rows into -`mem_model`, drive one `INT32_TO_FP32` command, wait for `scb.done()` or -timeout at 20000 cycles. Expected dst = `f32(i32) * scale` with relu clamp -on the integer before convert. `int2fp_cov` fatals unless relu and iter bins -are 100%. - -## BID (required) - -`+BID=` is mandatory. Missing it fatals at test start. No default bid. - -| Config | Plusarg | -|--------|---------| -| `sims.verilator.BuckyballPebbleVerilatorConfig` | `+BID=4` | - -```console -./simv +UVM_TESTNAME=int2fp_ball_test +BID=4 -``` - -## Build / Run - -Enter the shared UVM/VCS environment and build the DPI reference library: - -```console -nix develop ../../../../verify -cargo build --manifest-path casegen/Cargo.toml -``` - -Human: - -```console -bbdev uvm --run '--chip pebble' -bbdev uvm --run '--chip pebble --ball int2fp' -``` - -Agent (MCP): - -```text -bbdev_uvm_run(chip="pebble", ball="int2fp") -``` - -bbdev injects `INT32_TO_FP32_FUNCT7`, `BB_IN_BW`, `BB_OUT_BW`, `+BID`, and -`+UVM_TESTNAME=int2fp_ball_test` from pebble chip.pb. diff --git a/examples/balls/int2fp/verify/casegen/src/casegen.rs b/examples/balls/int2fp/verify/casegen/src/casegen.rs deleted file mode 100644 index be3725102..000000000 --- a/examples/balls/int2fp/verify/casegen/src/casegen.rs +++ /dev/null @@ -1,346 +0,0 @@ -use crate::model::int2fp_fp32_bits; - -pub const BANK_NUM: u32 = 24; -pub const BANK_ENTRIES: u32 = 64; -pub const LANES: usize = 4; -pub const SCALE_ROWS: usize = 4; -pub const MAX_SRC: usize = 16; -pub const MAX_SCALE: usize = 4; -pub const MAX_DST: usize = 16; -pub const NUM_CASES: u32 = 6; - -#[repr(C)] -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct Int2FpCmd { - pub bid: u32, - pub iter: u32, - pub relu: u32, - pub op1_bank: u32, - pub op2_bank: u32, - pub wr_bank: u32, - pub op1_col: u32, - pub op2_col: u32, - pub wr_col: u32, - pub rob_id: u32, - pub rs1_lo: u32, - pub rs1_hi: u32, - pub rs2_lo: u32, - pub rs2_hi: u32, - pub num_src_words: u32, - pub num_scale_words: u32, - pub num_dst_words: u32, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct Int2FpCase { - pub cmd: Int2FpCmd, - pub src_words: [u128; MAX_SRC], - pub scale_words: [u128; MAX_SCALE], - pub dst_words: [u128; MAX_DST], -} - -impl Int2FpCase { - pub fn src_lo(&self, index: usize) -> u64 { - self.src_words[index] as u64 - } - - pub fn src_hi(&self, index: usize) -> u64 { - (self.src_words[index] >> 64) as u64 - } - - pub fn scale_lo(&self, index: usize) -> u64 { - self.scale_words[index] as u64 - } - - pub fn scale_hi(&self, index: usize) -> u64 { - (self.scale_words[index] >> 64) as u64 - } - - pub fn dst_lo(&self, index: usize) -> u64 { - self.dst_words[index] as u64 - } - - pub fn dst_hi(&self, index: usize) -> u64 { - (self.dst_words[index] >> 64) as u64 - } -} - -pub fn gen_case(index: u32, bid: u32) -> Int2FpCase { - if index >= NUM_CASES { - panic!("int2fp: unsupported directed case {index}"); - } - match index { - 0 => directed(bid, 2, 4, false, &MLIR_INPUT, &[0.125; 16]), - 1 => directed(bid, 3, 4, true, &RELU_INPUT, &[1.0; 16]), - 2 => directed(bid, 4, 8, false, &ITER8, &SCALE16), - 3 => directed(bid, 5, 8, true, &ITER8, &SCALE16), - 4 => directed(bid, 6, 16, false, &ITER16, &SCALE16), - 5 => directed(bid, 7, 16, true, &ITER16, &SCALE16), - _ => panic!("int2fp: unsupported directed case {index}"), - } -} - -const MLIR_INPUT: [i32; 16] = [1, 2, 3, -1, -2, 0, 4, 5, 10, -10, 7, 100, -100, 8, 16, -8]; -const RELU_INPUT: [i32; 16] = [-8, -1, 0, 1, 2, -3, 4, -5, 6, 7, -8, 9, -10, 11, 12, -13]; -const ITER8: [i32; 32] = [ - 8, -4, 16, 0, 1, -2, 3, -8, 10, -10, 7, 100, -100, 8, 16, -8, 4, -1, 0, 2, -16, 32, -32, 5, 9, - -9, 12, -12, 6, -6, 11, -11, -]; -const ITER16: [i32; 64] = [ - 1, 2, 3, -1, -2, 0, 4, 5, 10, -10, 7, 100, -100, 8, 16, -8, 8, -4, 16, 0, 1, -2, 3, -8, 12, -12, - 6, -6, 9, -9, 11, -11, 20, -20, 24, -24, 15, -15, 18, -18, 30, -30, 40, -40, 25, -25, 28, -28, - 50, -50, 60, -60, 7, -7, 13, -13, 14, -14, 17, -17, 19, -19, 21, -21, -]; -const SCALE16: [f32; 16] = [ - 0.125, 0.25, 0.5, 1.0, 0.25, 0.5, 1.0, 2.0, 0.5, 1.0, 2.0, 0.125, 1.0, 0.125, 0.25, 0.5, -]; - -fn pack_u32s(vals: &[u32]) -> Vec { - if vals.len() % LANES != 0 { - panic!("int2fp: value count {} is not a multiple of four", vals.len()); - } - vals.chunks(LANES) - .map(|chunk| { - let mut word = 0u128; - for (lane, value) in chunk.iter().enumerate() { - word |= u128::from(*value) << (lane * 32); - } - word - }) - .collect() -} - -fn copy_words(src: &[u128]) -> [u128; N] { - if src.len() > N { - panic!("int2fp: {} words exceed buffer {N}", src.len()); - } - let mut out = [0u128; N]; - out[..src.len()].copy_from_slice(src); - out -} - -fn encode_rs1(op1: u32, op2: u32, wr: u32, iter: u32) -> u64 { - u64::from(op1) | (u64::from(op2) << 10) | (u64::from(wr) << 20) | (u64::from(iter) << 30) -} - -fn split_rs(value: u64) -> (u32, u32) { - (value as u32, (value >> 32) as u32) -} - -fn positive_finite(bits: u32) -> bool { - bits >> 31 == 0 && ((bits >> 23) & 0xff) != 0xff && (bits & 0x7fff_ffff) != 0 -} - -pub(crate) fn validate( - op1_bank: u32, - op2_bank: u32, - wr_bank: u32, - op1_col: u32, - op2_col: u32, - wr_col: u32, - iter: u32, - rs1: u64, - rs2: u64, - scales: &[u128], -) { - if iter == 0 || iter % 4 != 0 { - panic!("INT32_TO_FP32 iter must be a positive multiple of four"); - } - if iter > BANK_ENTRIES { - panic!("INT32_TO_FP32 input exceeds bank depth"); - } - if op1_bank >= BANK_NUM || op2_bank >= BANK_NUM || wr_bank >= BANK_NUM { - panic!("INT32_TO_FP32 bank id is invalid"); - } - if op1_col != 1 || op2_col != 1 || wr_col != 1 { - panic!("INT32_TO_FP32 operands must each occupy one bank"); - } - if op1_bank == op2_bank || op1_bank == wr_bank || op2_bank == wr_bank { - panic!("INT32_TO_FP32 banks must be distinct"); - } - if (rs1 & 0x3ff) != u64::from(op1_bank) - || ((rs1 >> 10) & 0x3ff) != u64::from(op2_bank) - || ((rs1 >> 20) & 0x3ff) != u64::from(wr_bank) - { - panic!("int2fp: rs1 bank fields mismatch"); - } - if ((rs1 >> 30) & 0x3_ffff_ffff) != u64::from(iter) { - panic!("int2fp: rs1 iter mismatch"); - } - if rs2 >> 1 != 0 { - panic!("INT32_TO_FP32 reserves rs2[63:1]"); - } - if scales.len() != SCALE_ROWS { - panic!("INT32_TO_FP32 must emit four scale rows"); - } - for word in scales { - for lane in 0..LANES { - let bits = (*word >> (lane * 32)) as u32; - if !positive_finite(bits) { - panic!("INT32_TO_FP32 scales must be finite and positive"); - } - } - } -} - -fn dst_words(src: &[u128], scales: &[u128], relu: bool) -> Vec { - src.iter() - .enumerate() - .map(|(row, word)| { - let scale = scales[row % SCALE_ROWS]; - let mut packed = 0u128; - for lane in 0..LANES { - let raw = (*word >> (lane * 32)) as u32 as i32; - let value = if relu { raw.max(0) } else { raw }; - let bits = int2fp_fp32_bits(value, (scale >> (lane * 32)) as u32); - packed |= u128::from(bits) << (lane * 32); - } - packed - }) - .collect() -} - -fn directed(bid: u32, rob_id: u32, iter: u32, relu: bool, values: &[i32], scales: &[f32]) -> Int2FpCase { - if values.len() != iter as usize * LANES { - panic!( - "int2fp: {} values do not match iter {iter} * {LANES}", - values.len() - ); - } - if scales.len() != SCALE_ROWS * LANES { - panic!("int2fp: {} scales, expected {}", scales.len(), SCALE_ROWS * LANES); - } - let src = pack_u32s(&values.iter().map(|v| *v as u32).collect::>()); - let scale = pack_u32s(&scales.iter().map(|v| v.to_bits()).collect::>()); - let op1_bank = 0; - let op2_bank = 1; - let wr_bank = 2; - let op1_col = 1; - let op2_col = 1; - let wr_col = 1; - let rs1 = encode_rs1(op1_bank, op2_bank, wr_bank, iter); - let rs2 = u64::from(relu); - validate( - op1_bank, op2_bank, wr_bank, op1_col, op2_col, wr_col, iter, rs1, rs2, &scale, - ); - if src.len() != iter as usize { - panic!("int2fp: src rows {} != iter {iter}", src.len()); - } - let dst = dst_words(&src, &scale, relu); - if dst.len() != iter as usize { - panic!("int2fp: dst rows {} != iter {iter}", dst.len()); - } - let (rs1_lo, rs1_hi) = split_rs(rs1); - let (rs2_lo, rs2_hi) = split_rs(rs2); - Int2FpCase { - cmd: Int2FpCmd { - bid, - iter, - relu: u32::from(relu), - op1_bank, - op2_bank, - wr_bank, - op1_col, - op2_col, - wr_col, - rob_id, - rs1_lo, - rs1_hi, - rs2_lo, - rs2_hi, - num_src_words: src.len() as u32, - num_scale_words: scale.len() as u32, - num_dst_words: dst.len() as u32, - }, - src_words: copy_words(&src), - scale_words: copy_words(&scale), - dst_words: copy_words(&dst), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn cases_hit_relu_and_iter_bins() { - let mut relu = [false; 2]; - let mut iter = [false; 3]; - for index in 0..NUM_CASES { - let case = gen_case(index, 4); - assert_eq!(case.cmd.bid, 4); - assert_eq!(case.cmd.op1_col, 1); - assert_eq!(case.cmd.op2_col, 1); - assert_eq!(case.cmd.wr_col, 1); - assert_eq!(case.cmd.num_src_words, case.cmd.iter); - assert_eq!(case.cmd.num_dst_words, case.cmd.iter); - assert_eq!(case.cmd.num_scale_words, SCALE_ROWS as u32); - assert_eq!(case.cmd.rs2_lo & 1, case.cmd.relu); - assert_eq!(case.cmd.rs2_lo >> 1, 0); - assert_eq!(case.cmd.rs2_hi, 0); - relu[case.cmd.relu as usize] = true; - match case.cmd.iter { - 4 => iter[0] = true, - 8 => iter[1] = true, - 16 => iter[2] = true, - other => panic!("unexpected iter {other}"), - } - } - assert!(relu.iter().all(|hit| *hit)); - assert!(iter.iter().all(|hit| *hit)); - } - - #[test] - fn case_zero_matches_mlir_scale() { - let case = gen_case(0, 4); - assert_eq!(case.cmd.iter, 4); - assert_eq!(case.cmd.relu, 0); - assert_eq!(case.dst_words[0] as u32, 0.125f32.to_bits()); - assert_eq!((case.dst_words[0] >> 32) as u32, 0.25f32.to_bits()); - assert_eq!((case.dst_words[0] >> 64) as u32, 0.375f32.to_bits()); - assert_eq!((case.dst_words[0] >> 96) as u32, (-0.125f32).to_bits()); - } - - #[test] - fn case_one_relu_clamps_negatives() { - let case = gen_case(1, 4); - assert_eq!(case.cmd.iter, 4); - assert_eq!(case.cmd.relu, 1); - assert_eq!(case.dst_words[0] as u32, 0); - assert_eq!((case.dst_words[0] >> 32) as u32, 0); - assert_eq!((case.dst_words[0] >> 64) as u32, 0); - assert_eq!((case.dst_words[0] >> 96) as u32, 1.0f32.to_bits()); - } - - #[test] - #[should_panic(expected = "unsupported directed case")] - fn unknown_index_panics() { - let _ = gen_case(NUM_CASES, 4); - } - - #[test] - #[should_panic(expected = "positive multiple of four")] - fn illegal_iter_panics() { - validate(0, 1, 2, 1, 1, 1, 3, encode_rs1(0, 1, 2, 3), 0, &[0; 4]); - } - - #[test] - #[should_panic(expected = "banks must be distinct")] - fn overlapping_banks_panics() { - validate(0, 0, 2, 1, 1, 1, 4, encode_rs1(0, 0, 2, 4), 0, &[0; 4]); - } - - #[test] - #[should_panic(expected = "reserves rs2[63:1]")] - fn reserved_rs2_panics() { - let scale = pack_u32s(&[1.0f32.to_bits(); 16]); - validate(0, 1, 2, 1, 1, 1, 4, encode_rs1(0, 1, 2, 4), 2, &scale); - } - - #[test] - #[should_panic(expected = "finite and positive")] - fn nonpositive_scale_panics() { - let scale = pack_u32s(&[0u32; 16]); - validate(0, 1, 2, 1, 1, 1, 4, encode_rs1(0, 1, 2, 4), 0, &scale); - } -} diff --git a/examples/balls/int2fp/verify/casegen/src/lib.rs b/examples/balls/int2fp/verify/casegen/src/lib.rs deleted file mode 100644 index ad3308513..000000000 --- a/examples/balls/int2fp/verify/casegen/src/lib.rs +++ /dev/null @@ -1,158 +0,0 @@ -mod casegen; -#[path = "../../../emu/src/model.rs"] -mod model; - -use std::cell::RefCell; - -use casegen::{Int2FpCase, Int2FpCmd, MAX_DST, MAX_SCALE, MAX_SRC}; - -thread_local! { - static CURRENT: RefCell> = const { RefCell::new(None) }; -} - -#[no_mangle] -pub extern "C" fn int2fp_case_load(index: u32, bid: u32) -> i32 { - let case = casegen::gen_case(index, bid); - CURRENT.with(|current| *current.borrow_mut() = Some(case)); - 0 -} - -fn current_case R, R>(f: F) -> R { - CURRENT.with(|current| match current.borrow().as_ref() { - Some(case) => f(case), - None => panic!("int2fp_case: no case loaded; call int2fp_case_load first"), - }) -} - -#[no_mangle] -pub extern "C" fn int2fp_case_cmd(out_ptr: *mut Int2FpCmd) { - if out_ptr.is_null() { - panic!("int2fp_case_cmd: null out_ptr"); - } - current_case(|case| unsafe { - *out_ptr = case.cmd; - }); -} - -fn src_index(index: u32, limit: u32, what: &str) -> usize { - if index >= limit { - panic!("int2fp_case_{what}: index {index} out of range {limit}"); - } - index as usize -} - -#[no_mangle] -pub extern "C" fn int2fp_case_src_word_lo(word_index: u32) -> u64 { - current_case(|case| { - let i = src_index(word_index, case.cmd.num_src_words, "src_word_lo"); - if i >= MAX_SRC { - panic!("int2fp_case_src_word_lo: word_index out of range"); - } - case.src_lo(i) - }) -} - -#[no_mangle] -pub extern "C" fn int2fp_case_src_word_hi(word_index: u32) -> u64 { - current_case(|case| { - let i = src_index(word_index, case.cmd.num_src_words, "src_word_hi"); - if i >= MAX_SRC { - panic!("int2fp_case_src_word_hi: word_index out of range"); - } - case.src_hi(i) - }) -} - -#[no_mangle] -pub extern "C" fn int2fp_case_scale_word_lo(word_index: u32) -> u64 { - current_case(|case| { - let i = src_index(word_index, case.cmd.num_scale_words, "scale_word_lo"); - if i >= MAX_SCALE { - panic!("int2fp_case_scale_word_lo: word_index out of range"); - } - case.scale_lo(i) - }) -} - -#[no_mangle] -pub extern "C" fn int2fp_case_scale_word_hi(word_index: u32) -> u64 { - current_case(|case| { - let i = src_index(word_index, case.cmd.num_scale_words, "scale_word_hi"); - if i >= MAX_SCALE { - panic!("int2fp_case_scale_word_hi: word_index out of range"); - } - case.scale_hi(i) - }) -} - -#[no_mangle] -pub extern "C" fn int2fp_case_dst_word_lo(word_index: u32) -> u64 { - current_case(|case| { - let i = src_index(word_index, case.cmd.num_dst_words, "dst_word_lo"); - if i >= MAX_DST { - panic!("int2fp_case_dst_word_lo: word_index out of range"); - } - case.dst_lo(i) - }) -} - -#[no_mangle] -pub extern "C" fn int2fp_case_dst_word_hi(word_index: u32) -> u64 { - current_case(|case| { - let i = src_index(word_index, case.cmd.num_dst_words, "dst_word_hi"); - if i >= MAX_DST { - panic!("int2fp_case_dst_word_hi: word_index out of range"); - } - case.dst_hi(i) - }) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn dpi_load_then_cmd_and_words() { - assert_eq!(int2fp_case_load(0, 4), 0); - let mut cmd = Int2FpCmd { - bid: 0, - iter: 0, - relu: 0, - op1_bank: 0, - op2_bank: 0, - wr_bank: 0, - op1_col: 0, - op2_col: 0, - wr_col: 0, - rob_id: 0, - rs1_lo: 0, - rs1_hi: 0, - rs2_lo: 0, - rs2_hi: 0, - num_src_words: 0, - num_scale_words: 0, - num_dst_words: 0, - }; - int2fp_case_cmd(&mut cmd as *mut Int2FpCmd); - assert_eq!(cmd.bid, 4); - assert_eq!(cmd.iter, 4); - assert_eq!(cmd.relu, 0); - assert_eq!(cmd.op1_col, 1); - assert_eq!(cmd.op2_col, 1); - assert_eq!(cmd.wr_col, 1); - assert_eq!(cmd.num_src_words, 4); - assert_eq!(cmd.num_scale_words, 4); - assert_eq!(cmd.num_dst_words, 4); - let _lo = int2fp_case_src_word_lo(0); - let _hi = int2fp_case_src_word_hi(0); - let _slo = int2fp_case_scale_word_lo(0); - let _dlo = int2fp_case_dst_word_lo(0); - } - - #[test] - #[should_panic(expected = "no case loaded")] - fn current_case_panics_without_load() { - CURRENT.with(|c| *c.borrow_mut() = None); - current_case(|_| ()); - } -} diff --git a/examples/balls/int2fp/verify/src/common/int2fp_defs.svh b/examples/balls/int2fp/verify/src/common/int2fp_defs.svh deleted file mode 100644 index 96a52851a..000000000 --- a/examples/balls/int2fp/verify/src/common/int2fp_defs.svh +++ /dev/null @@ -1,50 +0,0 @@ -typedef struct { - int unsigned bid; - int unsigned iter; - int unsigned relu; - int unsigned op1_bank; - int unsigned op2_bank; - int unsigned wr_bank; - int unsigned op1_col; - int unsigned op2_col; - int unsigned wr_col; - int unsigned rob_id; - int unsigned rs1_lo; - int unsigned rs1_hi; - int unsigned rs2_lo; - int unsigned rs2_hi; - int unsigned num_src_words; - int unsigned num_scale_words; - int unsigned num_dst_words; -} int2fp_cmd_dpi_t; - -import "DPI-C" function int int2fp_case_load( - input int unsigned index, - input int unsigned bid -); -import "DPI-C" function void int2fp_case_cmd(output int2fp_cmd_dpi_t cmd); -import "DPI-C" function longint unsigned int2fp_case_src_word_lo(input int unsigned word_index); -import "DPI-C" function longint unsigned int2fp_case_src_word_hi(input int unsigned word_index); -import "DPI-C" function longint unsigned int2fp_case_scale_word_lo(input int unsigned word_index); -import "DPI-C" function longint unsigned int2fp_case_scale_word_hi(input int unsigned word_index); -import "DPI-C" function longint unsigned int2fp_case_dst_word_lo(input int unsigned word_index); -import "DPI-C" function longint unsigned int2fp_case_dst_word_hi(input int unsigned word_index); - -`ifndef INT32_TO_FP32_FUNCT7 -`error "INT32_TO_FP32_FUNCT7 must be provided by the selected Core ballISA" -`endif - -localparam int INT32_TO_FP32_CORE_FUNCT7 = `INT32_TO_FP32_FUNCT7; -localparam int INT2FP_MAX_SRC_WORDS = 16; -localparam int INT2FP_MAX_SCALE_WORDS = 4; -localparam int INT2FP_MAX_DST_WORDS = 16; -localparam int INT2FP_NUM_CASES = 6; -localparam int INT2FP_TIMEOUT_CYCLES = 20000; - -function automatic int unsigned int2fp_require_bid(); - int unsigned bid; - if (!$value$plusargs("BID=%d", bid)) begin - `uvm_fatal("BID", "missing required plusarg +BID=") - end - return bid; -endfunction diff --git a/examples/balls/int2fp/verify/src/common/int2fp_items.svh b/examples/balls/int2fp/verify/src/common/int2fp_items.svh deleted file mode 100644 index eafba5bce..000000000 --- a/examples/balls/int2fp/verify/src/common/int2fp_items.svh +++ /dev/null @@ -1,125 +0,0 @@ -class int2fp_cmd_item extends bb_blink_cmd_item; - `uvm_object_utils(int2fp_cmd_item) - - bit [31:0] relu; - bit [31:0] num_src_words; - bit [31:0] num_scale_words; - bit [31:0] num_dst_words; - bit [127:0] src_words[INT2FP_MAX_SRC_WORDS]; - bit [127:0] scale_words[INT2FP_MAX_SCALE_WORDS]; - bit [127:0] dst_words[INT2FP_MAX_DST_WORDS]; - - constraint legal_c { - funct7 == INT32_TO_FP32_CORE_FUNCT7[6:0]; - op1_en == 1'b1; - op2_en == 1'b1; - wr_spad_en == 1'b1; - op1_from_spad == 1'b1; - op2_from_spad == 1'b1; - op1_col == 5'd1; - op2_col == 5'd1; - wr_col == 5'd1; - meta_bank == 5'd0; - is_sub == 1'b0; - sub_rob_id == 8'h00; - special == 64'd0; - } - - function new(string name = "int2fp_cmd_item"); - super.new(name); - endfunction - - function void load_rust_case(int unsigned index, int unsigned bid); - int2fp_cmd_dpi_t cmd; - int unsigned rc; - longint unsigned w_lo; - longint unsigned w_hi; - int i; - - rc = int2fp_case_load(index, bid); - if (rc != 0) begin - `uvm_fatal("CASE", $sformatf("int2fp_case_load returned %0d for index %0d", rc, index)) - end - int2fp_case_cmd(cmd); - - this.bid = cmd.bid[4:0]; - iter = cmd.iter; - relu = cmd.relu; - op1_bank = cmd.op1_bank[4:0]; - op2_bank = cmd.op2_bank[4:0]; - wr_bank = cmd.wr_bank[4:0]; - op1_col = cmd.op1_col[4:0]; - op2_col = cmd.op2_col[4:0]; - wr_col = cmd.wr_col[4:0]; - rob_id = cmd.rob_id[3:0]; - rs1 = {cmd.rs1_hi, cmd.rs1_lo}; - rs2 = {cmd.rs2_hi, cmd.rs2_lo}; - num_src_words = cmd.num_src_words; - num_scale_words = cmd.num_scale_words; - num_dst_words = cmd.num_dst_words; - funct7 = INT32_TO_FP32_CORE_FUNCT7[6:0]; - special = 64'd0; - op1_en = 1'b1; - op2_en = 1'b1; - wr_spad_en = 1'b1; - op1_from_spad = 1'b1; - op2_from_spad = 1'b1; - meta_bank = 5'd0; - is_sub = 1'b0; - sub_rob_id = 8'h00; - - if (iter == 0 || iter[1:0] != 2'b00) - `uvm_fatal("CASE", "INT32_TO_FP32 iter must be a positive multiple of four") - if (iter != 4 && iter != 8 && iter != 16) - `uvm_fatal("CASE", $sformatf("INT32_TO_FP32 iter %0d is not in {4,8,16}", iter)) - if (op1_col != 1 || op2_col != 1 || wr_col != 1) - `uvm_fatal("CASE", "INT32_TO_FP32 operands must each occupy one bank") - if (op1_bank == op2_bank || op1_bank == wr_bank || op2_bank == wr_bank) - `uvm_fatal("CASE", "INT32_TO_FP32 banks must be distinct") - if (rs2[63:1] != 0) `uvm_fatal("CASE", "INT32_TO_FP32 reserves rs2[63:1]") - if (rs2[0] != relu[0]) `uvm_fatal("CASE", "INT32_TO_FP32 rs2[0] must match relu") - if (num_src_words != iter) - `uvm_fatal("CASE", $sformatf("num_src_words %0d != iter %0d", num_src_words, iter)) - if (num_dst_words != iter) - `uvm_fatal("CASE", $sformatf("num_dst_words %0d != iter %0d", num_dst_words, iter)) - if (num_scale_words != 4) - `uvm_fatal("CASE", $sformatf("num_scale_words %0d, expected 4", num_scale_words)) - if (num_src_words == 0 || num_src_words > INT2FP_MAX_SRC_WORDS) - `uvm_fatal("CASE", $sformatf("num_src_words out of range: %0d", num_src_words)) - if (num_dst_words == 0 || num_dst_words > INT2FP_MAX_DST_WORDS) - `uvm_fatal("CASE", $sformatf("num_dst_words out of range: %0d", num_dst_words)) - if (funct7 != INT32_TO_FP32_CORE_FUNCT7[6:0]) - `uvm_fatal("CASE", $sformatf("invalid funct7: %0d", funct7)) - - for (i = 0; i < num_src_words; i++) begin - w_lo = int2fp_case_src_word_lo(i); - w_hi = int2fp_case_src_word_hi(i); - src_words[i] = {w_hi, w_lo}; - end - for (i = 0; i < num_scale_words; i++) begin - w_lo = int2fp_case_scale_word_lo(i); - w_hi = int2fp_case_scale_word_hi(i); - scale_words[i] = {w_hi, w_lo}; - end - for (i = 0; i < num_dst_words; i++) begin - w_lo = int2fp_case_dst_word_lo(i); - w_hi = int2fp_case_dst_word_hi(i); - dst_words[i] = {w_hi, w_lo}; - end - endfunction - - function void do_copy(uvm_object rhs); - int2fp_cmd_item rhs_; - super.do_copy(rhs); - if (!$cast(rhs_, rhs)) begin - `uvm_fatal("COPY", "rhs is not int2fp_cmd_item") - end - relu = rhs_.relu; - num_src_words = rhs_.num_src_words; - num_scale_words = rhs_.num_scale_words; - num_dst_words = rhs_.num_dst_words; - for (int i = 0; i < INT2FP_MAX_SRC_WORDS; i++) src_words[i] = rhs_.src_words[i]; - for (int i = 0; i < INT2FP_MAX_SCALE_WORDS; i++) scale_words[i] = rhs_.scale_words[i]; - for (int i = 0; i < INT2FP_MAX_DST_WORDS; i++) dst_words[i] = rhs_.dst_words[i]; - endfunction -endclass diff --git a/examples/balls/int2fp/verify/src/cov/int2fp_cov.svh b/examples/balls/int2fp/verify/src/cov/int2fp_cov.svh deleted file mode 100644 index 8b0727983..000000000 --- a/examples/balls/int2fp/verify/src/cov/int2fp_cov.svh +++ /dev/null @@ -1,37 +0,0 @@ -class int2fp_cov extends uvm_component; - `uvm_component_utils(int2fp_cov) - - uvm_analysis_imp_cmd #(bb_blink_cmd_item, int2fp_cov) cmd_imp; - - bit cur_relu; - int unsigned cur_iter; - - covergroup cmd_cg; - coverpoint cur_relu {bins off = {0}; bins on = {1};} - coverpoint cur_iter {bins i4 = {4}; bins i8 = {8}; bins i16 = {16};} - endgroup - - int cmd_count; - - function new(string name, uvm_component parent); - super.new(name, parent); - cmd_imp = new("cmd_imp", this); - cmd_cg = new(); - endfunction - - function void write_cmd(bb_blink_cmd_item item); - if (item.funct7 != INT32_TO_FP32_CORE_FUNCT7[6:0]) - `uvm_fatal("COV", $sformatf("unexpected funct7 %0d", item.funct7)) - cur_relu = item.rs2[0]; - cur_iter = item.iter; - cmd_count++; - cmd_cg.sample(); - endfunction - - function void check_phase(uvm_phase phase); - super.check_phase(phase); - if (cmd_count == 0) `uvm_fatal("COV", "no cmd observed") - if (cmd_cg.get_coverage() != 100.0) - `uvm_fatal("COV", $sformatf("int2fp_cov %0.2f%%", cmd_cg.get_coverage())) - endfunction -endclass diff --git a/examples/balls/int2fp/verify/src/env/int2fp_scoreboard.svh b/examples/balls/int2fp/verify/src/env/int2fp_scoreboard.svh deleted file mode 100644 index d21c506d6..000000000 --- a/examples/balls/int2fp/verify/src/env/int2fp_scoreboard.svh +++ /dev/null @@ -1,192 +0,0 @@ -class int2fp_scoreboard extends uvm_scoreboard; - `uvm_component_utils(int2fp_scoreboard) - - uvm_analysis_imp_stim #(bb_blink_cmd_item, int2fp_scoreboard) stim_imp; - uvm_analysis_imp_cmd #(bb_blink_cmd_item, int2fp_scoreboard) cmd_imp; - uvm_analysis_imp_read #(bb_blink_read_item, int2fp_scoreboard) read_imp; - uvm_analysis_imp_write #(bb_blink_write_item, int2fp_scoreboard) write_imp; - uvm_analysis_imp_resp #(bb_blink_resp_item, int2fp_scoreboard) resp_imp; - - bb_blink_mem_model #(`BB_IN_BW, `BB_OUT_BW) mem_model; - - int2fp_cmd_item stim_q[$]; - int unsigned expected_reads[`BB_IN_BW]; - int unsigned expected_writes; - int unsigned cmd_count; - int unsigned read_count[`BB_IN_BW]; - int unsigned write_count; - int unsigned resp_count; - - function new(string name, uvm_component parent); - super.new(name, parent); - stim_imp = new("stim_imp", this); - cmd_imp = new("cmd_imp", this); - read_imp = new("read_imp", this); - write_imp = new("write_imp", this); - resp_imp = new("resp_imp", this); - endfunction - - function void write_stim(bb_blink_cmd_item item); - int2fp_cmd_item got; - int2fp_cmd_item clone; - int i; - - if (stim_q.size() != 0) begin - `uvm_fatal("SCB", "single outstanding command supported") - end - if (!$cast(got, item)) begin - `uvm_fatal("SCB", "stim item is not int2fp_cmd_item") - end - if (!$cast(clone, got.clone())) begin - `uvm_fatal("SCB", "failed to clone stimulus item") - end - stim_q.push_back(clone); - - if (mem_model == null) begin - `uvm_fatal("SCB", "mem_model handle not set") - end - mem_model.clear_mem(); - for (i = 0; i < clone.num_src_words; i++) begin - mem_model.load_word(int'(clone.op1_bank), i, clone.src_words[i]); - end - for (i = 0; i < clone.num_scale_words; i++) begin - mem_model.load_word(int'(clone.op2_bank), i, clone.scale_words[i]); - end - mem_model.arm(); - - expected_writes = clone.num_dst_words; - expected_reads[0] = clone.iter; - expected_reads[1] = clone.iter; - endfunction - - function void write_cmd(bb_blink_cmd_item item); - int2fp_cmd_item stim; - stim = current_stim("CMD"); - check_cmd(item, stim); - cmd_count++; - endfunction - - function void check_cmd(bb_blink_cmd_item got, int2fp_cmd_item exp); - if (got.bid !== exp.bid) - `uvm_fatal("CMD", $sformatf("bid mismatch: got %0d exp %0d", got.bid, exp.bid)) - if (got.funct7 !== exp.funct7) - `uvm_fatal("CMD", $sformatf("funct7 mismatch: got %0d exp %0d", got.funct7, exp.funct7)) - if (got.iter !== exp.iter) - `uvm_fatal("CMD", $sformatf("iter mismatch: got %0d exp %0d", got.iter, exp.iter)) - if (got.op1_bank !== exp.op1_bank || got.op2_bank !== exp.op2_bank || - got.wr_bank !== exp.wr_bank) - `uvm_fatal("CMD", "bank field mismatch") - if (got.op1_col !== exp.op1_col || got.op2_col !== exp.op2_col || got.wr_col !== exp.wr_col) - `uvm_fatal("CMD", "column field mismatch") - if (got.rs1 !== exp.rs1 || got.rs2 !== exp.rs2) `uvm_fatal("CMD", "rs field mismatch") - if (got.rob_id !== exp.rob_id) - `uvm_fatal("CMD", $sformatf("rob_id mismatch: got %0d exp %0d", got.rob_id, exp.rob_id)) - if (got.rs2[63:1] !== '0) `uvm_fatal("CMD", "INT32_TO_FP32 reserves rs2[63:1]") - endfunction - - function void write_read(bb_blink_read_item item); - int2fp_cmd_item stim; - int unsigned expect_bank; - int unsigned expect_addr; - - stim = current_stim("READ"); - if (item.port != 0 && item.port != 1) - `uvm_fatal("READ", $sformatf("Int2FpBall has two read ports, got port %0d", item.port)) - if (item.group_id !== 5'd0) - `uvm_fatal("READ", $sformatf("group mismatch: got %0d exp 0", item.group_id)) - if (item.rob_id !== stim.rob_id) - `uvm_fatal("READ", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - - if (item.port == 0) begin - expect_bank = stim.op1_bank; - expect_addr = read_count[0]; - end else begin - expect_bank = stim.op2_bank; - expect_addr = read_count[1] & 32'h3; - end - if (item.bank_id !== expect_bank[4:0]) - `uvm_fatal("READ", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, expect_bank)) - if (item.addr !== expect_addr[BB_BLINK_BANK_ADDR_W-1:0]) - `uvm_fatal("READ", $sformatf("addr mismatch: got %0d exp %0d", item.addr, expect_addr)) - read_count[item.port]++; - endfunction - - function void write_write(bb_blink_write_item item); - int2fp_cmd_item stim; - - stim = current_stim("WRITE"); - if (item.port != 0) - `uvm_fatal("WRITE", $sformatf("Int2FpBall has one write port, got port %0d", item.port)) - if (item.bank_id !== stim.wr_bank) - `uvm_fatal("WRITE", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.wr_bank)) - if (item.rob_id !== stim.rob_id) - `uvm_fatal("WRITE", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.group_id !== 5'd0) - `uvm_fatal("WRITE", $sformatf("group mismatch: got %0d exp 0", item.group_id)) - if (item.mask !== 16'hFFFF) - `uvm_fatal("WRITE", $sformatf("mask mismatch: got 0x%04h", item.mask)) - if (write_count >= stim.num_dst_words) - `uvm_fatal("WRITE", $sformatf("extra write %0d", write_count)) - if (item.addr !== write_count[BB_BLINK_BANK_ADDR_W-1:0]) - `uvm_fatal("WRITE", $sformatf("addr mismatch: got %0d exp %0d", item.addr, write_count)) - if (item.data !== stim.dst_words[write_count]) - `uvm_fatal("SCB", $sformatf( - "data mismatch at addr %0d: got 0x%032h exp 0x%032h", - item.addr, - item.data, - stim.dst_words[write_count] - )) - write_count++; - endfunction - - function void write_resp(bb_blink_resp_item item); - int2fp_cmd_item stim; - - stim = current_stim("RESP"); - if (item.rob_id !== stim.rob_id) - `uvm_fatal("RESP", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.is_sub !== 1'b0) `uvm_fatal("RESP", "is_sub should be 0") - if (item.sub_rob_id !== 8'h00) - `uvm_fatal("RESP", $sformatf("sub_rob_id mismatch: got 0x%0h", item.sub_rob_id)) - resp_count++; - endfunction - - function int2fp_cmd_item current_stim(string tag); - if (stim_q.size() == 0) begin - `uvm_fatal("SCB", $sformatf("%s observed before stimulus", tag)) - return null; - end - return stim_q[0]; - endfunction - - function bit done(); - return cmd_count == 1 && - read_count[0] == expected_reads[0] && - read_count[1] == expected_reads[1] && - write_count == expected_writes && - resp_count == 1; - endfunction - - function void reset_counters(); - stim_q.delete(); - cmd_count = 0; - read_count[0] = 0; - read_count[1] = 0; - write_count = 0; - resp_count = 0; - expected_reads[0] = 0; - expected_reads[1] = 0; - expected_writes = 0; - endfunction - - function void check_phase(uvm_phase phase); - super.check_phase(phase); - if (!done()) begin - `uvm_fatal("SCB", $sformatf( - "incomplete: cmds=%0d reads=%0d/%0d,%0d/%0d writes=%0d/%0d resp=%0d", - cmd_count, read_count[0], expected_reads[0], read_count[1], - expected_reads[1], write_count, expected_writes, resp_count)) - end - stim_q.delete(); - endfunction -endclass diff --git a/examples/balls/int2fp/verify/src/tb_top.sv b/examples/balls/int2fp/verify/src/tb_top.sv deleted file mode 100644 index 961f4a179..000000000 --- a/examples/balls/int2fp/verify/src/tb_top.sv +++ /dev/null @@ -1,101 +0,0 @@ -module tb_top; - import uvm_pkg::*; - import int2fp_pkg::*; - - bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); - - Int2FpBall dut ( - .clock(intf.clock), - .reset(intf.reset), - .io_status_idle(), - .io_status_running(), - .io_channelReady(1'b1), - .io_cmdReq_ready(intf.cmd_req_ready), - .io_cmdReq_valid(intf.cmd_req_valid), - .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), - .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), - .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), - .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), - .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), - .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), - .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), - .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), - .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), - .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank), - .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank), - .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank), - .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), - .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), - .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), - .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank), - .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), - .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), - .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), - .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), - .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), - .io_cmdResp_ready(intf.cmd_resp_ready), - .io_cmdResp_valid(intf.cmd_resp_valid), - .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), - .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), - .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), - .io_bankRead_0_bank_id(intf.bank_read_bank_id[0]), - .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), - .io_bankRead_0_ball_id(), - .io_bankRead_0_group_id(intf.bank_read_group_id[0]), - .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), - .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), - .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), - .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), - .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), - .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), - .io_bankRead_1_bank_id(intf.bank_read_bank_id[1]), - .io_bankRead_1_rob_id(intf.bank_read_rob_id[1]), - .io_bankRead_1_ball_id(), - .io_bankRead_1_group_id(intf.bank_read_group_id[1]), - .io_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), - .io_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), - .io_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), - .io_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), - .io_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), - .io_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), - .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), - .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), - .io_bankWrite_0_ball_id(), - .io_bankWrite_0_group_id(intf.bank_write_group_id[0]), - .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), - .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), - .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), - .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), - .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), - .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), - .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), - .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), - .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), - .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), - .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), - .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), - .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), - .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), - .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), - .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), - .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), - .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), - .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), - .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), - .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), - .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), - .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), - .io_subRobReq_ready(intf.sub_rob_req_ready) - ); - - initial begin - intf.clock = 1'b0; - intf.reset = 1'b1; - forever #5 intf.clock = ~intf.clock; - end - - initial begin - uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); - run_test("int2fp_ball_test"); - end -endmodule diff --git a/examples/balls/int2fp/workloads/ctests/CMakeLists.txt b/examples/balls/int2fp/workloads/ctests/CMakeLists.txt deleted file mode 100644 index 27a32c4b6..000000000 --- a/examples/balls/int2fp/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -add_buckyball_ctests( - int2fp_tensor_values_test.c - int2fp_tensor_groups_test.c - int2fp_tensor_rows_test.c - int2fp_channel_lanes_test.c - int2fp_channel_base_test.c - int2fp_channel_two_rows_test.c -) diff --git a/examples/balls/int2fp/workloads/isa/int2fp.h b/examples/balls/int2fp/workloads/isa/int2fp.h deleted file mode 100644 index 1a832365c..000000000 --- a/examples/balls/int2fp/workloads/isa/int2fp.h +++ /dev/null @@ -1,13 +0,0 @@ -#ifndef _BB_INT2FP_H_ -#define _BB_INT2FP_H_ - -#include -#include - -#define bb_int32_to_fp32(input_bank, scale_bank, output_bank, iter, relu) \ - BUCKYBALL_INSTRUCTION_R_R(BB_BANK0(input_bank) | BB_BANK1(scale_bank) | \ - BB_BANK2(output_bank) | BB_ITER(iter), \ - FIELD((uint64_t)(relu), 0, 0), \ - BB_FUNC7(INT32_TO_FP32)) - -#endif // _BB_INT2FP_H_ diff --git a/examples/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4.mlir b/examples/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4.mlir deleted file mode 100644 index cb6610e77..000000000 --- a/examples/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4.mlir +++ /dev/null @@ -1,34 +0,0 @@ -func.func private @check_result(memref<4x4xf32>) -> () - -memref.global "private" constant @input : memref<4x4xi32> = dense<[ - [1, 2, 3, -1], [-2, 0, 4, 5], - [10, -10, 7, 100], [-100, 8, 16, -8] -]> -memref.global "private" constant @scales : memref<4x4xf32> = - dense<1.250000e-01> - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %depth = arith.constant 4 : i64 - %stride = arith.constant 1 : i64 - %input = memref.get_global @input : memref<4x4xi32> - %scales = memref.get_global @scales : memref<4x4xf32> - %output = memref.alloc() alignment = 64 : memref<4x4xf32> - %input_bank = buckyball.bank_alloc - %scale_bank = buckyball.bank_alloc - %output_bank = buckyball.bank_alloc - %loaded_input = buckyball.bank_mvin %input %input_bank %depth %stride - : memref<4x4xi32> i64 i64 i64 - %loaded_scales = buckyball.bank_mvin %scales %scale_bank %depth %stride - : memref<4x4xf32> i64 i64 i64 - buckyball.int32_to_fp32 - %loaded_input, %loaded_scales, %output_bank, %depth : i64 - %stored = buckyball.bank_mvout %output %output_bank %depth %stride - : memref<4x4xf32> i64 i64 i64 - func.call @check_result(%output) : (memref<4x4xf32>) -> () - buckyball.bank_release %loaded_input : i64 - buckyball.bank_release %loaded_scales : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %output : memref<4x4xf32> - return %zero_i8 : i8 -} diff --git a/examples/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4.mlir b/examples/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4.mlir deleted file mode 100644 index 862113577..000000000 --- a/examples/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4.mlir +++ /dev/null @@ -1,33 +0,0 @@ -func.func private @check_result(memref<4x4xf32>) -> () - -memref.global "private" constant @input : memref<4x4xi32> = dense<[ - [-8, -1, 0, 1], [2, -3, 4, -5], - [6, 7, -8, 9], [-10, 11, 12, -13] -]> -memref.global "private" constant @scales : memref<4x4xf32> = dense<1.0> - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %depth = arith.constant 4 : i64 - %stride = arith.constant 1 : i64 - %input = memref.get_global @input : memref<4x4xi32> - %scales = memref.get_global @scales : memref<4x4xf32> - %output = memref.alloc() alignment = 64 : memref<4x4xf32> - %input_bank = buckyball.bank_alloc - %scale_bank = buckyball.bank_alloc - %output_bank = buckyball.bank_alloc - %loaded_input = buckyball.bank_mvin %input %input_bank %depth %stride - : memref<4x4xi32> i64 i64 i64 - %loaded_scales = buckyball.bank_mvin %scales %scale_bank %depth %stride - : memref<4x4xf32> i64 i64 i64 - buckyball.int32_to_fp32 - %loaded_input, %loaded_scales, %output_bank, %depth : i64 - %stored = buckyball.bank_mvout %output %output_bank %depth %stride - : memref<4x4xf32> i64 i64 i64 - func.call @check_result(%output) : (memref<4x4xf32>) -> () - buckyball.bank_release %loaded_input : i64 - buckyball.bank_release %loaded_scales : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %output : memref<4x4xf32> - return %zero_i8 : i8 -} diff --git a/examples/balls/int2fp/workloads/mlir_tests/bank/CMakeLists.txt b/examples/balls/int2fp/workloads/mlir_tests/bank/CMakeLists.txt deleted file mode 100644 index 732c284a2..000000000 --- a/examples/balls/int2fp/workloads/mlir_tests/bank/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) - set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) -endif() -set(BUCKYBALL_MLIR_TEST_PREFIX bank) - -add_buckyball_mlir_test(int2fp_unit_scale TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4.mlir b/examples/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4.mlir deleted file mode 100644 index 5a79915c1..000000000 --- a/examples/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4.mlir +++ /dev/null @@ -1,35 +0,0 @@ -func.func private @check_result(memref<4x4xf32>) -> () - -memref.global "private" constant @input : memref<4x4xi32> = dense<[ - [1, 2, 3, -1], [-2, 0, 4, 5], - [10, -10, 7, 100], [-100, 8, 16, -8] -]> -memref.global "private" constant @scales : memref<4x4xf32> = - dense<1.250000e-01> - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %depth = arith.constant 4 : i64 - %stride = arith.constant 1 : i64 - %input = memref.get_global @input : memref<4x4xi32> - %scales = memref.get_global @scales : memref<4x4xf32> - %output = memref.alloc() alignment = 64 : memref<4x4xf32> - %input_bank = buckyball.bank_alloc - %scale_bank = buckyball.bank_alloc - %output_bank = buckyball.bank_alloc - %loaded_input = buckyball.bank_mvin %input %input_bank %depth %stride - : memref<4x4xi32> i64 i64 i64 - %loaded_scales = buckyball.bank_mvin %scales %scale_bank %depth %stride - : memref<4x4xf32> i64 i64 i64 - %result = buckyball.bank_int32_to_fp32 - %loaded_input %loaded_scales %output_bank %depth - : i64 i64 i64 i64 - %stored = buckyball.bank_mvout %output %result %depth %stride - : memref<4x4xf32> i64 i64 i64 - func.call @check_result(%output) : (memref<4x4xf32>) -> () - buckyball.bank_release %loaded_input : i64 - buckyball.bank_release %loaded_scales : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %output : memref<4x4xf32> - return %zero_i8 : i8 -} diff --git a/examples/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4.mlir b/examples/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4.mlir deleted file mode 100644 index f9b8ab927..000000000 --- a/examples/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4.mlir +++ /dev/null @@ -1,34 +0,0 @@ -func.func private @check_result(memref<4x4xf32>) -> () - -memref.global "private" constant @input : memref<4x4xi32> = dense<[ - [-8, -1, 0, 1], [2, -3, 4, -5], - [6, 7, -8, 9], [-10, 11, 12, -13] -]> -memref.global "private" constant @scales : memref<4x4xf32> = dense<1.0> - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %depth = arith.constant 4 : i64 - %stride = arith.constant 1 : i64 - %input = memref.get_global @input : memref<4x4xi32> - %scales = memref.get_global @scales : memref<4x4xf32> - %output = memref.alloc() alignment = 64 : memref<4x4xf32> - %input_bank = buckyball.bank_alloc - %scale_bank = buckyball.bank_alloc - %output_bank = buckyball.bank_alloc - %loaded_input = buckyball.bank_mvin %input %input_bank %depth %stride - : memref<4x4xi32> i64 i64 i64 - %loaded_scales = buckyball.bank_mvin %scales %scale_bank %depth %stride - : memref<4x4xf32> i64 i64 i64 - %result = buckyball.bank_int32_to_fp32 - %loaded_input %loaded_scales %output_bank %depth - : i64 i64 i64 i64 - %stored = buckyball.bank_mvout %output %result %depth %stride - : memref<4x4xf32> i64 i64 i64 - func.call @check_result(%output) : (memref<4x4xf32>) -> () - buckyball.bank_release %loaded_input : i64 - buckyball.bank_release %loaded_scales : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %output : memref<4x4xf32> - return %zero_i8 : i8 -} diff --git a/examples/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale.mlir.in b/examples/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale.mlir.in deleted file mode 100644 index 19be15e57..000000000 --- a/examples/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale.mlir.in +++ /dev/null @@ -1,39 +0,0 @@ -func.func private @check_result(memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xf32>) -> () -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %rows = arith.constant @BB_BANK_LINES@ : index - %lanes = arith.constant @BB_BANK_I32_LANES@ : index - %depth = arith.constant @BB_BANK_LINES@ : i64 - %depth4 = arith.constant 4 : i64 - %stride = arith.constant 1 : i64 - %input = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> - %scale = memref.alloc() : memref<4x4xf32> - %output = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xf32> - %one_f32 = arith.constant 1.0 : f32 - linalg.fill ins(%one_f32 : f32) outs(%scale : memref<4x4xf32>) - scf.for %row = %zero to %rows step %one { - scf.for %lane = %zero to %lanes step %one { - %r = arith.index_cast %row : index to i32 - %l = arith.index_cast %lane : index to i32 - %value = arith.subi %r, %l : i32 - memref.store %value, %input[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> - } - } - %input_bank = buckyball.bank_alloc - %scale_bank = buckyball.bank_alloc - %output_bank = buckyball.bank_alloc - %loaded_input = buckyball.bank_mvin %input %input_bank %depth %stride : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> i64 i64 i64 - %loaded_scale = buckyball.bank_mvin %scale %scale_bank %depth4 %stride : memref<4x4xf32> i64 i64 i64 - %result = buckyball.bank_int32_to_fp32 %loaded_input %loaded_scale %output_bank %depth : i64 i64 i64 i64 - %stored = buckyball.bank_mvout %output %result %depth %stride : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xf32> i64 i64 i64 - func.call @check_result(%output) : (memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xf32>) -> () - buckyball.bank_release %loaded_input : i64 - buckyball.bank_release %loaded_scale : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %input : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> - memref.dealloc %scale : memref<4x4xf32> - memref.dealloc %output : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xf32> - return %zero_i8 : i8 -} diff --git a/examples/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale_main.cpp b/examples/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale_main.cpp deleted file mode 100644 index 578c4e1f5..000000000 --- a/examples/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale_main.cpp +++ /dev/null @@ -1,18 +0,0 @@ -#include -#include -#include -#include -extern "C" void check_result(float *, float *data, int64_t offset, int64_t rows, - int64_t lanes, int64_t stride, - int64_t lane_stride) { - if (rows != BANK_LINES || lanes != BANK_WIDTH / 32 || stride != lanes || - lane_stride != 1) - exit(1); - for (int row = 0; row < rows; ++row) - for (int lane = 0; lane < lanes; ++lane) - if (data[offset + row * stride + lane] != (float)(row - lane)) { - printf("int2fp bank mismatch row=%d lane=%d\n", row, lane); - exit(1); - } - printf("int2fp bank PASS rows=%d\n", BANK_LINES); -} diff --git a/examples/balls/int8add/arch/src/main/scala/Int8AddBall.scala b/examples/balls/int8add/arch/src/main/scala/Int8AddBall.scala deleted file mode 100644 index e580601ab..000000000 --- a/examples/balls/int8add/arch/src/main/scala/Int8AddBall.scala +++ /dev/null @@ -1,278 +0,0 @@ -package examples.balls.int8add - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public} -import chisel3.util._ -import hardfloat.{rawFloatFromRecFN, recFNFromFN, RecFNToIN, RoundRawFNToRecFN} -import hardfloat.consts.{round_near_even, tininess_afterRounding} -import freechips.rocketchip.tile.{MulAddRecFNPipe, PipelinedINToRecFN} - -import framework.balldomain.blink.{BallStatus, BlinkIO, HasBallStatus, HasBlink, SubRobRow} -import framework.balldomain.blink.mmio.{MmioRead, MmioWrite} -import framework.top.GlobalConfig - -@instantiable -class Int8AddBall(val b: GlobalConfig) extends Module with HasBlink with HasBallStatus { - - private val mapping = b.ballDomain.ballIdMappings - .find(_.ballName == "Int8AddBall") - .getOrElse(throw new IllegalArgumentException("Int8AddBall not found in config")) - - private val addFunct = b.ballDomain.ballISA - .find(_.mnemonic == "INT8ADD") - .map(_.funct7) - .getOrElse(throw new IllegalArgumentException("INT8ADD not found in ballISA")) - - private val reluFunct = b.ballDomain.ballISA - .find(_.mnemonic == "INT8ADD_RELU") - .map(_.funct7) - .getOrElse(throw new IllegalArgumentException("INT8ADD_RELU not found in ballISA")) - - require(mapping.inBW == 2, "Int8AddBall requires inBW=2") - require(mapping.outBW == 1, "Int8AddBall requires outBW=1") - require(b.memDomain.bankWidth == 128, "Int8AddBall requires 128-bit bank rows") - require((addFunct >> 4) == 4 && (reluFunct >> 4) == 4, "Int8AddBall instructions must encode two reads and one write") - - @public - val io = IO(new BlinkIO(b, mapping.inBW, mapping.outBW)) - def blink: BlinkIO = io - def status: BallStatus = io.status - dontTouch(io) - - private val idle :: waitForChannels :: readRequest :: readResponse :: calculate :: calculateDrain :: writeRequest :: writeResponse :: complete :: Nil = - Enum(9) - private val state = RegInit(idle) - private val robId = RegInit(0.U(log2Up(b.frontend.rob_entries).W)) - private val isSub = RegInit(false.B) - private val subRobId = RegInit(0.U(log2Up(b.frontend.sub_rob_depth * 4).W)) - private val lhsBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - private val rhsBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - private val outputBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - private val groups = RegInit(0.U(5.W)) - private val group = RegInit(0.U(5.W)) - private val iter = RegInit(0.U(b.frontend.iter_len.W)) - private val row = RegInit(0.U(log2Ceil(b.memDomain.bankEntries).W)) - private val lhsRatio = Reg(UInt(32.W)) - private val rhsRatio = Reg(UInt(32.W)) - private val relu = RegInit(false.B) - private val lhsWord = Reg(UInt(128.W)) - private val rhsWord = Reg(UInt(128.W)) - private val lhsRequested = RegInit(false.B) - private val rhsRequested = RegInit(false.B) - private val lhsReceived = RegInit(false.B) - private val rhsReceived = RegInit(false.B) - private val lane = RegInit(0.U(4.W)) - private val completed = RegInit(0.U(5.W)) - private val outputWord = Reg(Vec(16, UInt(8.W))) - - private def positiveFinite(value: UInt): Bool = - !value(31) && value(30, 23) =/= 255.U && value(30, 0) =/= 0.U - - private val lhsToFloat = Module(new PipelinedINToRecFN(8, 8, 24)) - private val rhsToFloat = Module(new PipelinedINToRecFN(8, 8, 24)) - private val lhsMul = Module(new MulAddRecFNPipe(3, 8, 24)) - private val rhsMul = Module(new MulAddRecFNPipe(3, 8, 24)) - private val add = Module(new PipelinedAddRawFN(8, 24)) - private val round = Module(new RoundRawFNToRecFN(8, 24, 0)) - private val toInt = Module(new RecFNToIN(8, 24, 8)) - private val lhsValue = (lhsWord >> (lane << 3))(7, 0) - private val rhsValue = (rhsWord >> (lane << 3))(7, 0) - private val calculateIssue = state === calculate - - private val ratioStage0 = RegEnable(lhsRatio, calculateIssue) - private val ratioStage1 = RegEnable(ratioStage0, RegNext(calculateIssue, false.B)) - private val rhsRatioStage0 = RegEnable(rhsRatio, calculateIssue) - private val rhsRatioStage1 = RegEnable(rhsRatioStage0, RegNext(calculateIssue, false.B)) - private val laneStage0 = RegEnable(lane, calculateIssue) - private val laneStage1 = RegEnable(laneStage0, RegNext(calculateIssue, false.B)) - - private val conversionValid = lhsToFloat.io.validout - private val productLane = Pipe(conversionValid, laneStage1, 3) - - private val addResult = add.io.rawOut - private val addLane = Pipe(lhsMul.io.validout, productLane.bits, 4) - private val addValid = add.io.validout - private val roundedResult = RegEnable(round.io.out, addValid) - private val roundedLane = RegEnable(addLane, addValid) - private val resultValid = RegNext(addValid, false.B) - - lhsToFloat.io.validin := calculateIssue - lhsToFloat.io.signedIn := true.B - lhsToFloat.io.in := lhsValue - lhsToFloat.io.bypassIn := lhsValue - lhsToFloat.io.roundingMode := round_near_even - lhsToFloat.io.detectTininess := tininess_afterRounding - lhsToFloat.io.typeTagIn := 0.U - lhsToFloat.io.wflagsIn := false.B - rhsToFloat.io.validin := calculateIssue - rhsToFloat.io.signedIn := true.B - rhsToFloat.io.in := rhsValue - rhsToFloat.io.bypassIn := rhsValue - rhsToFloat.io.roundingMode := round_near_even - rhsToFloat.io.detectTininess := tininess_afterRounding - rhsToFloat.io.typeTagIn := 0.U - rhsToFloat.io.wflagsIn := false.B - - lhsMul.io.validin := conversionValid - lhsMul.io.op := 0.U - lhsMul.io.a := lhsToFloat.io.out - lhsMul.io.b := recFNFromFN(8, 24, ratioStage1) - lhsMul.io.c := recFNFromFN(8, 24, 0.U) - lhsMul.io.roundingMode := round_near_even - lhsMul.io.detectTininess := tininess_afterRounding.asUInt - rhsMul.io.validin := conversionValid - rhsMul.io.op := 0.U - rhsMul.io.a := rhsToFloat.io.out - rhsMul.io.b := recFNFromFN(8, 24, rhsRatioStage1) - rhsMul.io.c := recFNFromFN(8, 24, 0.U) - rhsMul.io.roundingMode := round_near_even - rhsMul.io.detectTininess := tininess_afterRounding.asUInt - - add.io.validin := lhsMul.io.validout - add.io.subOp := false.B - add.io.a := rawFloatFromRecFN(8, 24, lhsMul.io.out) - add.io.b := rawFloatFromRecFN(8, 24, rhsMul.io.out) - add.io.roundingMode := round_near_even - round.io.invalidExc := add.io.invalidExc - round.io.infiniteExc := false.B - round.io.in := addResult - round.io.roundingMode := round_near_even - round.io.detectTininess := tininess_afterRounding - toInt.io.in := roundedResult - toInt.io.roundingMode := round_near_even - toInt.io.signedOut := true.B - private val result = Mux(relu && toInt.io.out(7), 0.U, toInt.io.out) - - io.cmdReq.ready := state === idle - io.cmdResp.valid := state === complete - io.cmdResp.bits.rob_id := robId - io.cmdResp.bits.is_sub := isSub - io.cmdResp.bits.sub_rob_id := subRobId - io.status.idle := state === idle - io.status.running := state =/= idle && state =/= complete - - for (port <- 0 until 2) { - io.bankRead(port).rob_id := robId - io.bankRead(port).ball_id := 0.U - io.bankRead(port).bank_id := Mux(port.U === 0.U, lhsBank, rhsBank) - io.bankRead(port).group_id := group - io.bankRead(port).io.req.valid := false.B - io.bankRead(port).io.req.bits.addr := row - io.bankRead(port).io.resp.ready := false.B - } - io.bankWrite(0).rob_id := robId - io.bankWrite(0).ball_id := 0.U - io.bankWrite(0).bank_id := outputBank - io.bankWrite(0).group_id := group - io.bankWrite(0).io.req.valid := false.B - io.bankWrite(0).io.req.bits.addr := row - io.bankWrite(0).io.req.bits.data := Cat(outputWord.reverse) - io.bankWrite(0).io.req.bits.mask := VecInit(Seq.fill(16)(true.B)) - io.bankWrite(0).io.resp.ready := false.B - io.subRobReq.valid := false.B - io.subRobReq.bits := SubRobRow.tieOff(b) - MmioRead.tieOff(io.mmioRead) - MmioWrite.tieOff(io.mmioWrite) - - switch(state) { - is(idle) { - when(io.cmdReq.fire) { - val cmd = io.cmdReq.bits.cmd - assert(cmd.funct7 === addFunct.U || cmd.funct7 === reluFunct.U, "Int8AddBall received an unknown funct7") - assert( - positiveFinite(cmd.rs2(31, 0)) && positiveFinite(cmd.rs2(63, 32)), - "Int8AddBall ratios must be finite and positive" - ) - assert( - cmd.op1_col =/= 0.U && cmd.op1_col === cmd.op2_col && cmd.op1_col === cmd.wr_col, - "Int8AddBall bank groups must match" - ) - assert(cmd.op1_col <= b.memDomain.bankNum.U, "Int8AddBall bank groups exceed physical banks") - assert( - cmd.op1_bank =/= cmd.op2_bank && cmd.op1_bank =/= cmd.wr_bank && cmd.op2_bank =/= cmd.wr_bank, - "Int8AddBall banks must be distinct" - ) - assert(cmd.iter =/= 0.U && cmd.iter <= b.memDomain.bankEntries.U, "Int8AddBall iter must fit one physical bank") - robId := io.cmdReq.bits.rob_id - isSub := io.cmdReq.bits.is_sub - subRobId := io.cmdReq.bits.sub_rob_id - lhsBank := cmd.op1_bank - rhsBank := cmd.op2_bank - outputBank := cmd.wr_bank - groups := cmd.op1_col - group := 0.U - iter := cmd.iter - row := 0.U - lhsRatio := cmd.rs2(31, 0) - rhsRatio := cmd.rs2(63, 32) - relu := cmd.funct7 === reluFunct.U - lhsRequested := false.B - rhsRequested := false.B - lhsReceived := false.B - rhsReceived := false.B - lane := 0.U - state := waitForChannels - } - } - is(waitForChannels) { - when(io.channelReady)(state := readRequest) - } - is(readRequest) { - io.bankRead(0).io.req.valid := !lhsRequested - io.bankRead(1).io.req.valid := !rhsRequested - when(io.bankRead(0).io.req.fire)(lhsRequested := true.B) - when(io.bankRead(1).io.req.fire)(rhsRequested := true.B) - when((lhsRequested || io.bankRead(0).io.req.fire) && (rhsRequested || io.bankRead(1).io.req.fire)) { - state := readResponse - } - } - is(readResponse) { - io.bankRead(0).io.resp.ready := !lhsReceived - io.bankRead(1).io.resp.ready := !rhsReceived - when(io.bankRead(0).io.resp.fire) { lhsWord := io.bankRead(0).io.resp.bits.data; lhsReceived := true.B } - when(io.bankRead(1).io.resp.fire) { rhsWord := io.bankRead(1).io.resp.bits.data; rhsReceived := true.B } - when((lhsReceived || io.bankRead(0).io.resp.fire) && (rhsReceived || io.bankRead(1).io.resp.fire)) { - lane := 0.U - completed := 0.U - state := calculate - } - } - is(calculate) { - when(lane === 15.U) { - state := calculateDrain - }.otherwise { - lane := lane + 1.U - } - } - is(calculateDrain) {} - is(writeRequest) { - io.bankWrite(0).io.req.valid := true.B - when(io.bankWrite(0).io.req.fire)(state := writeResponse) - } - is(writeResponse) { - io.bankWrite(0).io.resp.ready := true.B - when(io.bankWrite(0).io.resp.fire) { - lhsRequested := false.B - rhsRequested := false.B - lhsReceived := false.B - rhsReceived := false.B - when(row +& 1.U === iter) { - row := 0.U - when(group +& 1.U === groups)(state := complete) - .otherwise { group := group + 1.U; state := readRequest } - }.otherwise { row := row + 1.U; state := readRequest } - } - } - is(complete) { - when(io.cmdResp.fire)(state := idle) - } - } - - when((state === calculate || state === calculateDrain) && resultValid) { - outputWord(roundedLane.bits) := result - completed := completed + 1.U - when(completed === 15.U) { - state := writeRequest - } - } -} diff --git a/examples/balls/int8add/arch/src/main/scala/PipelinedAddRawFN.scala b/examples/balls/int8add/arch/src/main/scala/PipelinedAddRawFN.scala deleted file mode 100644 index ac30ed8df..000000000 --- a/examples/balls/int8add/arch/src/main/scala/PipelinedAddRawFN.scala +++ /dev/null @@ -1,167 +0,0 @@ -package examples.balls.int8add - -import chisel3._ -import chisel3.util._ -import hardfloat.{countLeadingZeros, isSigNaNRawFloat, lowMask, orReduceBy2, orReduceBy4, RawFloat} -import hardfloat.consts.round_min - -/** - * Throughput-oriented raw floating-point adder used by Int8AddBall. - * - * The Berkeley HardFloat AddRawFN implementation is combinational. This - * wrapper keeps the same raw-float equations but registers the alignment, - * normalization, and final select boundaries. It accepts one operand pair - * per cycle and reports the result four cycles later. - */ -class PipelinedAddRawFN(expWidth: Int, sigWidth: Int) extends Module { - - val io = IO(new Bundle { - val validin = Input(Bool()) - val subOp = Input(Bool()) - val a = Input(new RawFloat(expWidth, sigWidth)) - val b = Input(new RawFloat(expWidth, sigWidth)) - val roundingMode = Input(UInt(3.W)) - val validout = Output(Bool()) - val invalidExc = Output(Bool()) - val rawOut = Output(new RawFloat(expWidth, sigWidth + 2)) - }) - - private val alignDistWidth = log2Ceil(sigWidth) - - // Stage 0: register the raw operands and exponent comparison. - private val a0 = RegNext(io.a) - private val b0 = RegNext(io.b) - private val effSignB0 = RegNext(io.b.sign ^ io.subOp) - private val eqSigns0 = RegNext(io.a.sign === (io.b.sign ^ io.subOp)) - private val sDiffExps0 = RegNext(io.a.sExp - io.b.sExp) - private val roundingMode0 = RegNext(io.roundingMode) - - // Stage 1: prepare close-path subtraction and far-path operand selection. - private val modNatAlignDist1 = Mux(sDiffExps0 < 0.S, -sDiffExps0, sDiffExps0)(alignDistWidth - 1, 0) - - private val isMaxAlign1 = - (sDiffExps0 >> alignDistWidth) =/= 0.S && - ((sDiffExps0 >> alignDistWidth) =/= -1.S || sDiffExps0(alignDistWidth - 1, 0) === 0.U) - - private val alignDist1 = Mux(isMaxAlign1, ((BigInt(1) << alignDistWidth) - 1).U, modNatAlignDist1) - private val closeSubMags1 = !eqSigns0 && !isMaxAlign1 && (modNatAlignDist1 <= 1.U) - - private val closeAlignedSigA1 = - Mux((0.S <= sDiffExps0) && sDiffExps0(0), a0.sig << 2, 0.U) | - Mux((0.S <= sDiffExps0) && !sDiffExps0(0), a0.sig << 1, 0.U) | - Mux(sDiffExps0 < 0.S, a0.sig, 0.U) - - private val closeSSigSum1 = Wire(SInt((sigWidth + 3).W)) - closeSSigSum1 := closeAlignedSigA1.asSInt - (b0.sig << 1).asSInt - private val closeSigSum1 = Wire(UInt((sigWidth + 2).W)) - closeSigSum1 := Mux(closeSSigSum1 < 0.S, -closeSSigSum1, closeSSigSum1)(sigWidth + 1, 0) - private val closeAdjustedSigSum1 = Wire(UInt((sigWidth + 2).W)) - closeAdjustedSigSum1 := closeSigSum1 << (sigWidth & 1) - private val closeReduced2SigSum1 = orReduceBy2(closeAdjustedSigSum1) - - private val farSignOut1 = Mux(sDiffExps0 < 0.S, effSignB0, a0.sign) - private val farSigLarger1 = Wire(UInt(sigWidth.W)) - farSigLarger1 := Mux(sDiffExps0 < 0.S, b0.sig, a0.sig)(sigWidth - 1, 0) - private val farSigSmaller1 = Wire(UInt(sigWidth.W)) - farSigSmaller1 := Mux(sDiffExps0 < 0.S, a0.sig, b0.sig)(sigWidth - 1, 0) - - private val a1 = RegNext(a0) - private val b1 = RegNext(b0) - private val effSignB1 = RegNext(effSignB0) - private val eqSigns1 = RegNext(eqSigns0) - private val sDiffExps1 = RegNext(sDiffExps0) - private val roundingMode1 = RegNext(roundingMode0) - private val alignDist1r = RegInit(0.U(alignDistWidth.W)) - alignDist1r := alignDist1 - private val closeSubMags1r = RegInit(false.B) - closeSubMags1r := closeSubMags1 - private val closeSSigSum1r = RegInit(0.S((sigWidth + 3).W)) - closeSSigSum1r := closeSSigSum1 - private val closeSigSum1r = RegInit(0.U((sigWidth + 2).W)) - closeSigSum1r := closeSigSum1 - private val closeReduced2SigSum1r = RegInit(0.U(((sigWidth + 3) / 2).W)) - closeReduced2SigSum1r := closeReduced2SigSum1 - private val farSignOut1r = RegNext(farSignOut1) - private val farSigLarger1r = RegInit(0.U(sigWidth.W)) - farSigLarger1r := farSigLarger1 - private val farSigSmaller1r = RegInit(0.U(sigWidth.W)) - farSigSmaller1r := farSigSmaller1 - - // Stage 2: leading-zero normalization and far-path alignment. - private val closeNormDistReduced2_2 = countLeadingZeros(closeReduced2SigSum1r) - private val closeNearNormDist2 = (closeNormDistReduced2_2 << 1)(alignDistWidth - 1, 0) - private val closeSigOut2 = ((closeSigSum1r << closeNearNormDist2) << 1)(sigWidth + 2, 0) - private val closeTotalCancellation2 = !(closeSigOut2(sigWidth + 2, sigWidth + 1).orR) - private val closeNotTotalCancellationSignOut2 = a1.sign ^ (closeSSigSum1r < 0.S) - - private val farMainAlignedSigSmaller2 = Wire(UInt((sigWidth + 5).W)) - farMainAlignedSigSmaller2 := (farSigSmaller1r << 5) >> alignDist1r - private val farReduced4SigSmaller2 = Wire(UInt(((sigWidth + 5) / 4).W)) - farReduced4SigSmaller2 := orReduceBy4(Cat(farSigSmaller1r, 0.U(2.W))) - private val farRoundExtraMask2 = lowMask(alignDist1r(alignDistWidth - 1, 2), (sigWidth + 5) / 4, 0) - - private val farAlignedSigSmaller2 = Cat( - farMainAlignedSigSmaller2 >> 3, - farMainAlignedSigSmaller2(2, 0).orR || (farReduced4SigSmaller2 & farRoundExtraMask2).orR - ) - - private val farSubMags2 = !eqSigns1 - - private val a2 = RegNext(a1) - private val b2 = RegNext(b1) - private val effSignB2 = RegNext(effSignB1) - private val eqSigns2 = RegNext(eqSigns1) - private val sDiffExps2 = RegNext(sDiffExps1) - private val roundingMode2 = RegNext(roundingMode1) - private val closeSubMags2r = RegNext(closeSubMags1r) - private val closeNearNormDist2r = RegNext(closeNearNormDist2) - private val closeSigOut2r = RegNext(closeSigOut2) - private val closeTotalCancellation2r = RegNext(closeTotalCancellation2) - private val closeNotTotalCancellationSignOut2r = RegNext(closeNotTotalCancellationSignOut2) - private val farSignOut2r = RegNext(farSignOut1r) - private val farSigLarger2r = RegNext(farSigLarger1r) - private val farAlignedSigSmaller2r = RegNext(farAlignedSigSmaller2) - private val farSubMags2r = RegNext(farSubMags2) - - // Stage 3: significand add/subtract and special-case selection. - private val farNegAlignedSigSmaller3 = Mux( - farSubMags2r, - Cat(1.U, ~farAlignedSigSmaller2r), - farAlignedSigSmaller2r - ) - - private val farSigSum3 = (farSigLarger2r << 3) + farNegAlignedSigSmaller3 + farSubMags2r - private val farSigOut3 = Mux(farSubMags2r, farSigSum3, (farSigSum3 >> 1) | farSigSum3(0))(sigWidth + 2, 0) - - private val notSigNaNInvalidExc3 = a2.isInf && b2.isInf && !eqSigns2 - private val notNaNIsInfOut3 = a2.isInf || b2.isInf - private val addZeros3 = a2.isZero && b2.isZero - private val notNaNSpecialCase3 = notNaNIsInfOut3 || addZeros3 - private val notNaNIsZeroOut3 = addZeros3 || (!notNaNIsInfOut3 && closeSubMags2r && closeTotalCancellation2r) - - private val notNaNSignOut3 = - (eqSigns2 && a2.sign) || - (a2.isInf && a2.sign) || - (b2.isInf && effSignB2) || - (notNaNIsZeroOut3 && !eqSigns2 && roundingMode2 === round_min) || - (!notNaNSpecialCase3 && closeSubMags2r && !closeTotalCancellation2r && closeNotTotalCancellationSignOut2r) || - (!notNaNSpecialCase3 && !closeSubMags2r && farSignOut2r) - - private val commonSExp3 = - Mux(closeSubMags2r || (sDiffExps2 < 0.S), b2.sExp, a2.sExp) - - Mux(closeSubMags2r, closeNearNormDist2r, farSubMags2r).zext - - private val commonSigOut3 = Mux(closeSubMags2r, closeSigOut2r, farSigOut3) - - private val rawOutReg = RegInit(0.U.asTypeOf(new RawFloat(expWidth, sigWidth + 2))) - rawOutReg.isNaN := a2.isNaN || b2.isNaN - rawOutReg.isInf := notNaNIsInfOut3 - rawOutReg.isZero := notNaNIsZeroOut3 - rawOutReg.sExp := commonSExp3 - rawOutReg.sign := notNaNSignOut3 - rawOutReg.sig := commonSigOut3 - - io.rawOut := rawOutReg - io.invalidExc := RegNext(isSigNaNRawFloat(a2) || isSigNaNRawFloat(b2) || notSigNaNInvalidExc3) - io.validout := ShiftRegister(io.validin, 4) -} diff --git a/examples/balls/int8add/compiler/src/Backend/include/IntrinsicsRISCVInt8AddBall.td b/examples/balls/int8add/compiler/src/Backend/include/IntrinsicsRISCVInt8AddBall.td deleted file mode 100644 index eddf2840a..000000000 --- a/examples/balls/int8add/compiler/src/Backend/include/IntrinsicsRISCVInt8AddBall.td +++ /dev/null @@ -1 +0,0 @@ -// Int8AddBall uses CustomIntrOp with funct7 from the target ballISA. diff --git a/examples/balls/int8add/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp b/examples/balls/int8add/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp deleted file mode 100644 index 27db06fe8..000000000 --- a/examples/balls/int8add/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp +++ /dev/null @@ -1,35 +0,0 @@ -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace mlir::buddy { -void populateInt8AddBallAssignPhysicalBankPatterns(RewritePatternSet &patterns, - PhysicalBankState &state); -} - -namespace { -class BankInt8AddPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(BankInt8AddOp op, - PatternRewriter &rewriter) const override { - rewriter.create( - op.getLoc(), op.getLhsBank(), op.getRhsBank(), op.getOutputBank(), - op.getIter(), op.getLhsRatio(), op.getRhsRatio(), op.getReluAttr()); - rewriter.replaceOp(op, op.getOutputBank()); - return success(); - } -}; -} // namespace - -void mlir::buddy::populateInt8AddBallAssignPhysicalBankPatterns( - RewritePatternSet &patterns, PhysicalBankState &state) { - (void)state; - patterns.add(patterns.getContext()); -} diff --git a/examples/balls/int8add/compiler/src/Dialect/Buckyball/Int8AddBall.td b/examples/balls/int8add/compiler/src/Dialect/Buckyball/Int8AddBall.td deleted file mode 100644 index 253a89e99..000000000 --- a/examples/balls/int8add/compiler/src/Dialect/Buckyball/Int8AddBall.td +++ /dev/null @@ -1,26 +0,0 @@ -#ifndef INT8ADDBALL_OPS -#define INT8ADDBALL_OPS - -include "Buckyball.td" - -def Int8AddOp : Buckyball_Op<"int8add"> { - let summary = "Scaled INT8 addition on physical banks"; - let arguments = (ins I64:$lhsBankId, I64:$rhsBankId, I64:$outputBankId, - I64:$iter, F32:$lhsRatio, F32:$rhsRatio, - BoolAttr:$relu); - let assemblyFormat = "$lhsBankId `,` $rhsBankId `,` $outputBankId `,` $iter `,` " - "$lhsRatio `,` $rhsRatio prop-dict attr-dict `:` type($lhsBankId)"; -} - -def BankInt8AddOp : Buckyball_Op<"bank_int8add"> { - let summary = "Scaled INT8 addition on virtual banks"; - let arguments = (ins I64:$lhsBank, I64:$rhsBank, I64:$outputBank, - I64:$iter, F32:$lhsRatio, F32:$rhsRatio, - BoolAttr:$relu); - let results = (outs I64:$outputBankOut); - let assemblyFormat = "$lhsBank $rhsBank $outputBank $iter $lhsRatio $rhsRatio " - "prop-dict attr-dict `:` type($lhsBank) type($rhsBank) " - "type($outputBank) type($iter) type($lhsRatio)"; -} - -#endif // INT8ADDBALL_OPS diff --git a/examples/balls/int8add/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/balls/int8add/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 090933fe4..000000000 --- a/examples/balls/int8add/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,56 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace buddy::buckyball; -using namespace buddy::buckyball::legalize; - -namespace { -struct Int8AddLowering : public ConvertOpToLLVMPattern { - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - - LogicalResult - matchAndRewrite(Int8AddOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("Int8AddBall"); - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getLhsBankId(), - adaptor.getRhsBankId(), - adaptor.getOutputBankId(), adaptor.getIter()); - Value lhsBits = rewriter.create( - loc, rewriter.getI32Type(), adaptor.getLhsRatio()); - Value rhsBits = rewriter.create( - loc, rewriter.getI32Type(), adaptor.getRhsRatio()); - Value lhs64 = - rewriter.create(loc, rewriter.getI64Type(), lhsBits); - Value rhs64 = - rewriter.create(loc, rewriter.getI64Type(), rhsBits); - Value rs2 = rewriter.create( - loc, lhs64, - rewriter.create(loc, rhs64, cstI64(rewriter, loc, 32))); - rewriter.replaceOpWithNewOp( - op, rs1, rs2, - rewriter.getI32IntegerAttr(buckyball_target::getBuckyballFunct7( - op.getRelu() ? "INT8ADD_RELU" : "INT8ADD"))); - return success(); - } -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateInt8AddBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool, int64_t) { - patterns.add(converter); -} - -void configureInt8AddBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool) { - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/examples/balls/int8add/configs/default.toml b/examples/balls/int8add/configs/default.toml deleted file mode 100644 index 15ef6060d..000000000 --- a/examples/balls/int8add/configs/default.toml +++ /dev/null @@ -1,3 +0,0 @@ -# Int8AddBall has no private configuration. - -[ball] diff --git a/examples/balls/int8add/emu/src/dispatch.rs b/examples/balls/int8add/emu/src/dispatch.rs deleted file mode 100644 index ede132ed1..000000000 --- a/examples/balls/int8add/emu/src/dispatch.rs +++ /dev/null @@ -1,32 +0,0 @@ -use crate::inst::instruction::ExecContext; - -use super::int8add; - -const BALL_CLASS: &str = "examples.balls.int8add.Int8AddBall"; - -pub fn execute_known( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, - ctx: &mut ExecContext, -) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("INT8ADD") => Some(int8add::Int8Add::exec(xs1, xs2, ctx, false)), - Some("INT8ADD_RELU") => Some(int8add::Int8Add::exec(xs1, xs2, ctx, true)), - Some(_) | None => None, - } -} - -pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("INT8ADD") | Some("INT8ADD_RELU") => Some(int8add::Int8Add::latency(xs1, xs2)), - Some(_) | None => None, - } -} diff --git a/examples/balls/int8add/emu/src/lib.rs b/examples/balls/int8add/emu/src/lib.rs deleted file mode 100644 index b1ebd2d62..000000000 --- a/examples/balls/int8add/emu/src/lib.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub(crate) use crate::inst::{decode, instruction}; - -mod int8add; - -mod dispatch; - -pub use dispatch::{cycles_after_issue, execute_known}; diff --git a/examples/balls/int8add/verify/casegen/src/casegen.rs b/examples/balls/int8add/verify/casegen/src/casegen.rs deleted file mode 100644 index c5c4a0cd6..000000000 --- a/examples/balls/int8add/verify/casegen/src/casegen.rs +++ /dev/null @@ -1,339 +0,0 @@ -pub const ROW_BYTES: usize = 16; -pub const BANK_ENTRIES: u32 = 64; -pub const MAX_ROWS: usize = 64; -pub const NUM_CASES: u32 = 2; - -#[repr(C)] -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct Int8AddCmd { - pub relu: u32, - pub bid: u32, - pub iter: u32, - pub op1_bank: u32, - pub op2_bank: u32, - pub wr_bank: u32, - pub op1_col: u32, - pub op2_col: u32, - pub wr_col: u32, - pub rob_id: u32, - pub rs1_lo: u32, - pub rs1_hi: u32, - pub rs2_lo: u32, - pub rs2_hi: u32, - pub num_lhs_words: u32, - pub num_rhs_words: u32, - pub num_dst_words: u32, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct Int8AddCase { - pub cmd: Int8AddCmd, - pub lhs_words: [u128; MAX_ROWS], - pub rhs_words: [u128; MAX_ROWS], - pub dst_words: [u128; MAX_ROWS], -} - -impl Int8AddCase { - pub fn lhs_lo(&self, index: usize) -> u64 { - self.lhs_words[index] as u64 - } - - pub fn lhs_hi(&self, index: usize) -> u64 { - (self.lhs_words[index] >> 64) as u64 - } - - pub fn rhs_lo(&self, index: usize) -> u64 { - self.rhs_words[index] as u64 - } - - pub fn rhs_hi(&self, index: usize) -> u64 { - (self.rhs_words[index] >> 64) as u64 - } - - pub fn dst_lo(&self, index: usize) -> u64 { - self.dst_words[index] as u64 - } - - pub fn dst_hi(&self, index: usize) -> u64 { - (self.dst_words[index] >> 64) as u64 - } -} - -pub fn gen_case(index: u32, bid: u32) -> Int8AddCase { - if index >= NUM_CASES { - panic!("int8add: unsupported directed case {index}"); - } - match index { - 0 => directed(bid, 2, false), - 1 => directed(bid, 3, true), - _ => panic!("int8add: unsupported directed case {index}"), - } -} - -fn compute(lhs: i8, rhs: i8, lhs_ratio: f32, rhs_ratio: f32, relu: bool) -> i8 { - let value = (lhs as f32) * lhs_ratio + (rhs as f32) * rhs_ratio; - let rounded = value.round_ties_even(); - let clamped = if relu { - rounded.max(0.0) - } else { - rounded - } - .clamp(-128.0, 127.0); - clamped as i8 -} - -fn positive_finite(value: f32) -> bool { - value.is_finite() && value > 0.0 -} - -fn ctest_values(rows: usize) -> (Vec, Vec) { - let count = rows * ROW_BYTES; - let mut lhs = vec![0u8; count]; - let mut rhs = vec![0u8; count]; - for i in 0..count { - lhs[i] = ((i as i32 * 37 & 255) - 128) as i8 as u8; - rhs[i] = ((i as i32 * 19 & 255) - 128) as i8 as u8; - } - (pack_bytes(&lhs), pack_bytes(&rhs)) -} - -fn pack_bytes(bytes: &[u8]) -> Vec { - if bytes.len() % ROW_BYTES != 0 { - panic!( - "int8add: {} bytes is not a multiple of {ROW_BYTES}", - bytes.len() - ); - } - bytes - .chunks(ROW_BYTES) - .map(|chunk| { - let mut word = 0u128; - for (lane, byte) in chunk.iter().enumerate() { - word |= u128::from(*byte) << (lane * 8); - } - word - }) - .collect() -} - -fn copy_words(src: &[u128]) -> [u128; N] { - if src.len() > N { - panic!("int8add: {} words exceed buffer {N}", src.len()); - } - let mut out = [0u128; N]; - out[..src.len()].copy_from_slice(src); - out -} - -fn encode_rs1(op1: u32, op2: u32, wr: u32, iter: u32) -> u64 { - u64::from(op1) | (u64::from(op2) << 10) | (u64::from(wr) << 20) | (u64::from(iter) << 30) -} - -fn encode_rs2(lhs_ratio: f32, rhs_ratio: f32) -> u64 { - u64::from(lhs_ratio.to_bits()) | (u64::from(rhs_ratio.to_bits()) << 32) -} - -fn split_rs(value: u64) -> (u32, u32) { - (value as u32, (value >> 32) as u32) -} - -pub(crate) fn validate( - op1_bank: u32, - op2_bank: u32, - wr_bank: u32, - op1_col: u32, - op2_col: u32, - wr_col: u32, - iter: u32, - rs1: u64, - rs2: u64, - lhs_ratio: f32, - rhs_ratio: f32, -) { - if iter == 0 || iter > BANK_ENTRIES { - panic!("int8add: iter must fit in one physical bank"); - } - if op1_col != 1 || op2_col != 1 || wr_col != 1 { - panic!("int8add: bank groups must match"); - } - if op1_bank == op2_bank || op1_bank == wr_bank || op2_bank == wr_bank { - panic!("int8add: banks must be distinct"); - } - if !positive_finite(lhs_ratio) || !positive_finite(rhs_ratio) { - panic!("int8add: ratios must be finite and positive"); - } - if (rs1 & 0x3ff) != u64::from(op1_bank) - || ((rs1 >> 10) & 0x3ff) != u64::from(op2_bank) - || ((rs1 >> 20) & 0x3ff) != u64::from(wr_bank) - { - panic!("int8add: rs1 bank fields mismatch"); - } - if ((rs1 >> 30) & 0x3_ffff_ffff) != u64::from(iter) { - panic!("int8add: rs1 iter mismatch"); - } - if rs2 != encode_rs2(lhs_ratio, rhs_ratio) { - panic!("int8add: rs2 ratio fields mismatch"); - } -} - -fn dst_words( - lhs: &[u128], - rhs: &[u128], - iter: u32, - lhs_ratio: f32, - rhs_ratio: f32, - relu: bool, -) -> Vec { - let mut out = Vec::with_capacity(iter as usize); - for row in 0..iter as usize { - let mut packed = 0u128; - for lane in 0..ROW_BYTES { - let l = ((lhs[row] >> (lane * 8)) & 0xff) as u8 as i8; - let r = ((rhs[row] >> (lane * 8)) & 0xff) as u8 as i8; - let q = compute(l, r, lhs_ratio, rhs_ratio, relu); - packed |= u128::from(q as u8) << (lane * 8); - } - out.push(packed); - } - out -} - -fn directed(bid: u32, rob_id: u32, relu: bool) -> Int8AddCase { - let iter = 7u32; - let lhs_ratio = 0.5f32; - let rhs_ratio = 0.25f32; - let (lhs, rhs) = ctest_values(iter as usize); - let op1_bank = 0; - let op2_bank = 1; - let wr_bank = 2; - let op1_col = 1; - let op2_col = 1; - let wr_col = 1; - let rs1 = encode_rs1(op1_bank, op2_bank, wr_bank, iter); - let rs2 = encode_rs2(lhs_ratio, rhs_ratio); - validate( - op1_bank, - op2_bank, - wr_bank, - op1_col, - op2_col, - wr_col, - iter, - rs1, - rs2, - lhs_ratio, - rhs_ratio, - ); - if lhs.len() != iter as usize || rhs.len() != iter as usize { - panic!("int8add: operand rows != iter {iter}"); - } - let dst = dst_words(&lhs, &rhs, iter, lhs_ratio, rhs_ratio, relu); - let (rs1_lo, rs1_hi) = split_rs(rs1); - let (rs2_lo, rs2_hi) = split_rs(rs2); - Int8AddCase { - cmd: Int8AddCmd { - relu: u32::from(relu), - bid, - iter, - op1_bank, - op2_bank, - wr_bank, - op1_col, - op2_col, - wr_col, - rob_id, - rs1_lo, - rs1_hi, - rs2_lo, - rs2_hi, - num_lhs_words: lhs.len() as u32, - num_rhs_words: rhs.len() as u32, - num_dst_words: dst.len() as u32, - }, - lhs_words: copy_words(&lhs), - rhs_words: copy_words(&rhs), - dst_words: copy_words(&dst), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn ctest_expected(a: i8, b: i8, relu: bool) -> i8 { - compute(a, b, 0.5, 0.25, relu) - } - - #[test] - fn case_zero_matches_ctest() { - let case = gen_case(0, 7); - assert_eq!(case.cmd.bid, 7); - assert_eq!(case.cmd.relu, 0); - assert_eq!(case.cmd.iter, 7); - assert_eq!(case.cmd.op1_bank, 0); - assert_eq!(case.cmd.op2_bank, 1); - assert_eq!(case.cmd.wr_bank, 2); - assert_eq!(case.cmd.num_lhs_words, 7); - assert_eq!(case.cmd.num_rhs_words, 7); - assert_eq!(case.cmd.num_dst_words, 7); - let lhs0 = ((case.lhs_words[0] >> 0) & 0xff) as u8 as i8; - let rhs0 = ((case.rhs_words[0] >> 0) & 0xff) as u8 as i8; - assert_eq!(lhs0, -128); - assert_eq!(rhs0, -128); - let got0 = (case.dst_words[0] & 0xff) as u8 as i8; - assert_eq!(got0, ctest_expected(lhs0, rhs0, false)); - } - - #[test] - fn cases_hit_funct7_bins() { - let mut hit = [false; 2]; - for index in 0..NUM_CASES { - let case = gen_case(index, 7); - assert_eq!(case.cmd.bid, 7); - assert_eq!(case.cmd.op1_col, 1); - assert_eq!(case.cmd.op2_col, 1); - assert_eq!(case.cmd.wr_col, 1); - assert_eq!(case.cmd.num_lhs_words, case.cmd.iter); - assert_eq!(case.cmd.num_rhs_words, case.cmd.iter); - assert_eq!(case.cmd.num_dst_words, case.cmd.iter); - hit[case.cmd.relu as usize] = true; - } - assert!(hit.iter().all(|h| *h)); - } - - #[test] - fn relu_floors_negatives() { - let case = gen_case(1, 7); - assert_eq!(case.cmd.relu, 1); - for row in 0..case.cmd.iter as usize { - for lane in 0..ROW_BYTES { - let got = ((case.dst_words[row] >> (lane * 8)) & 0xff) as u8 as i8; - assert!(got >= 0); - } - } - } - - #[test] - #[should_panic(expected = "unsupported directed case")] - fn unknown_index_panics() { - let _ = gen_case(NUM_CASES, 7); - } - - #[test] - #[should_panic(expected = "iter must fit")] - fn illegal_iter_panics() { - validate(0, 1, 2, 1, 1, 1, 0, encode_rs1(0, 1, 2, 0), encode_rs2(0.5, 0.25), 0.5, 0.25); - } - - #[test] - #[should_panic(expected = "banks must be distinct")] - fn overlapping_banks_panics() { - validate(0, 0, 2, 1, 1, 1, 7, encode_rs1(0, 0, 2, 7), encode_rs2(0.5, 0.25), 0.5, 0.25); - } - - #[test] - #[should_panic(expected = "finite and positive")] - fn nonpositive_ratio_panics() { - validate(0, 1, 2, 1, 1, 1, 7, encode_rs1(0, 1, 2, 7), encode_rs2(0.0, 0.25), 0.0, 0.25); - } -} diff --git a/examples/balls/int8add/verify/casegen/src/lib.rs b/examples/balls/int8add/verify/casegen/src/lib.rs deleted file mode 100644 index d14ad09c7..000000000 --- a/examples/balls/int8add/verify/casegen/src/lib.rs +++ /dev/null @@ -1,117 +0,0 @@ -mod casegen; - -use std::cell::RefCell; - -use casegen::{Int8AddCase, Int8AddCmd}; - -thread_local! { - static CURRENT: RefCell> = const { RefCell::new(None) }; -} - -#[no_mangle] -pub extern "C" fn int8add_case_load(index: u32, bid: u32) -> i32 { - let case = casegen::gen_case(index, bid); - CURRENT.with(|current| *current.borrow_mut() = Some(case)); - 0 -} - -fn current_case R, R>(f: F) -> R { - CURRENT.with(|current| match current.borrow().as_ref() { - Some(case) => f(case), - None => panic!("int8add_case: no case loaded; call int8add_case_load first"), - }) -} - -#[no_mangle] -pub extern "C" fn int8add_case_cmd(out_ptr: *mut Int8AddCmd) { - if out_ptr.is_null() { - panic!("int8add_case_cmd: null out_ptr"); - } - current_case(|case| unsafe { - *out_ptr = case.cmd; - }); -} - -fn idx(index: u32, limit: u32, what: &str) -> usize { - if index >= limit { - panic!("int8add_case_{what}: index {index} out of range {limit}"); - } - index as usize -} - -#[no_mangle] -pub extern "C" fn int8add_case_lhs_word_lo(word_index: u32) -> u64 { - current_case(|case| case.lhs_lo(idx(word_index, case.cmd.num_lhs_words, "lhs_word_lo"))) -} - -#[no_mangle] -pub extern "C" fn int8add_case_lhs_word_hi(word_index: u32) -> u64 { - current_case(|case| case.lhs_hi(idx(word_index, case.cmd.num_lhs_words, "lhs_word_hi"))) -} - -#[no_mangle] -pub extern "C" fn int8add_case_rhs_word_lo(word_index: u32) -> u64 { - current_case(|case| case.rhs_lo(idx(word_index, case.cmd.num_rhs_words, "rhs_word_lo"))) -} - -#[no_mangle] -pub extern "C" fn int8add_case_rhs_word_hi(word_index: u32) -> u64 { - current_case(|case| case.rhs_hi(idx(word_index, case.cmd.num_rhs_words, "rhs_word_hi"))) -} - -#[no_mangle] -pub extern "C" fn int8add_case_dst_word_lo(word_index: u32) -> u64 { - current_case(|case| case.dst_lo(idx(word_index, case.cmd.num_dst_words, "dst_word_lo"))) -} - -#[no_mangle] -pub extern "C" fn int8add_case_dst_word_hi(word_index: u32) -> u64 { - current_case(|case| case.dst_hi(idx(word_index, case.cmd.num_dst_words, "dst_word_hi"))) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn dpi_load_then_cmd_and_words() { - assert_eq!(int8add_case_load(0, 7), 0); - let mut cmd = Int8AddCmd { - relu: 0, - bid: 0, - iter: 0, - op1_bank: 0, - op2_bank: 0, - wr_bank: 0, - op1_col: 0, - op2_col: 0, - wr_col: 0, - rob_id: 0, - rs1_lo: 0, - rs1_hi: 0, - rs2_lo: 0, - rs2_hi: 0, - num_lhs_words: 0, - num_rhs_words: 0, - num_dst_words: 0, - }; - int8add_case_cmd(&mut cmd as *mut Int8AddCmd); - assert_eq!(cmd.bid, 7); - assert_eq!(cmd.iter, 7); - assert_eq!(cmd.relu, 0); - assert_eq!(cmd.num_lhs_words, 7); - assert_eq!(cmd.num_rhs_words, 7); - assert_eq!(cmd.num_dst_words, 7); - let _ = int8add_case_lhs_word_lo(0); - let _ = int8add_case_rhs_word_lo(0); - let _ = int8add_case_dst_word_lo(0); - assert_eq!(casegen::NUM_CASES, 2); - } - - #[test] - #[should_panic(expected = "no case loaded")] - fn current_case_panics_without_load() { - CURRENT.with(|c| *c.borrow_mut() = None); - current_case(|_| ()); - } -} diff --git a/examples/balls/int8add/verify/src/common/int8add_defs.svh b/examples/balls/int8add/verify/src/common/int8add_defs.svh deleted file mode 100644 index 908c50e71..000000000 --- a/examples/balls/int8add/verify/src/common/int8add_defs.svh +++ /dev/null @@ -1,52 +0,0 @@ -typedef struct { - int unsigned relu; - int unsigned bid; - int unsigned iter; - int unsigned op1_bank; - int unsigned op2_bank; - int unsigned wr_bank; - int unsigned op1_col; - int unsigned op2_col; - int unsigned wr_col; - int unsigned rob_id; - int unsigned rs1_lo; - int unsigned rs1_hi; - int unsigned rs2_lo; - int unsigned rs2_hi; - int unsigned num_lhs_words; - int unsigned num_rhs_words; - int unsigned num_dst_words; -} int8add_cmd_dpi_t; - -import "DPI-C" function int int8add_case_load( - input int unsigned index, - input int unsigned bid -); -import "DPI-C" function void int8add_case_cmd(output int8add_cmd_dpi_t cmd); -import "DPI-C" function longint unsigned int8add_case_lhs_word_lo(input int unsigned word_index); -import "DPI-C" function longint unsigned int8add_case_lhs_word_hi(input int unsigned word_index); -import "DPI-C" function longint unsigned int8add_case_rhs_word_lo(input int unsigned word_index); -import "DPI-C" function longint unsigned int8add_case_rhs_word_hi(input int unsigned word_index); -import "DPI-C" function longint unsigned int8add_case_dst_word_lo(input int unsigned word_index); -import "DPI-C" function longint unsigned int8add_case_dst_word_hi(input int unsigned word_index); - -`ifndef INT8ADD_FUNCT7 -`error "INT8ADD_FUNCT7 must be provided by the selected Core ballISA" -`endif -`ifndef INT8ADD_RELU_FUNCT7 -`error "INT8ADD_RELU_FUNCT7 must be provided by the selected Core ballISA" -`endif - -localparam int INT8ADD_CORE_FUNCT7 = `INT8ADD_FUNCT7; -localparam int INT8ADD_RELU_CORE_FUNCT7 = `INT8ADD_RELU_FUNCT7; -localparam int INT8ADD_MAX_ROWS = 64; -localparam int INT8ADD_NUM_CASES = 2; -localparam int INT8ADD_TIMEOUT_CYCLES = 20000; - -function automatic int unsigned int8add_require_bid(); - int unsigned bid; - if (!$value$plusargs("BID=%d", bid)) begin - `uvm_fatal("BID", "missing required plusarg +BID=") - end - return bid; -endfunction diff --git a/examples/balls/int8add/verify/src/common/int8add_items.svh b/examples/balls/int8add/verify/src/common/int8add_items.svh deleted file mode 100644 index eb3b2d3a8..000000000 --- a/examples/balls/int8add/verify/src/common/int8add_items.svh +++ /dev/null @@ -1,122 +0,0 @@ -class int8add_cmd_item extends bb_blink_cmd_item; - `uvm_object_utils(int8add_cmd_item) - - bit [31:0] relu; - bit [31:0] num_lhs_words; - bit [31:0] num_rhs_words; - bit [31:0] num_dst_words; - bit [127:0] lhs_words[INT8ADD_MAX_ROWS]; - bit [127:0] rhs_words[INT8ADD_MAX_ROWS]; - bit [127:0] dst_words[INT8ADD_MAX_ROWS]; - - constraint legal_c { - funct7 inside {INT8ADD_CORE_FUNCT7[6:0], INT8ADD_RELU_CORE_FUNCT7[6:0]}; - op1_en == 1'b1; - op2_en == 1'b1; - wr_spad_en == 1'b1; - op1_from_spad == 1'b1; - op2_from_spad == 1'b1; - op1_col == 5'd1; - op2_col == 5'd1; - wr_col == 5'd1; - meta_bank == 5'd0; - is_sub == 1'b0; - sub_rob_id == 8'h00; - special == 64'd0; - } - - function new(string name = "int8add_cmd_item"); - super.new(name); - endfunction - - function void load_rust_case(int unsigned index, int unsigned bid); - int8add_cmd_dpi_t cmd; - int unsigned rc; - longint unsigned w_lo; - longint unsigned w_hi; - int i; - - rc = int8add_case_load(index, bid); - if (rc != 0) begin - `uvm_fatal("CASE", $sformatf("int8add_case_load returned %0d for index %0d", rc, index)) - end - int8add_case_cmd(cmd); - - if (cmd.relu != 0 && cmd.relu != 1) begin - `uvm_fatal("CASE", $sformatf("invalid relu flag %0d", cmd.relu)) - end - - relu = cmd.relu; - this.bid = cmd.bid[4:0]; - iter = cmd.iter; - op1_bank = cmd.op1_bank[4:0]; - op2_bank = cmd.op2_bank[4:0]; - wr_bank = cmd.wr_bank[4:0]; - op1_col = cmd.op1_col[4:0]; - op2_col = cmd.op2_col[4:0]; - wr_col = cmd.wr_col[4:0]; - rob_id = cmd.rob_id[3:0]; - rs1 = {cmd.rs1_hi, cmd.rs1_lo}; - rs2 = {cmd.rs2_hi, cmd.rs2_lo}; - num_lhs_words = cmd.num_lhs_words; - num_rhs_words = cmd.num_rhs_words; - num_dst_words = cmd.num_dst_words; - special = 64'd0; - op1_en = 1'b1; - op2_en = 1'b1; - wr_spad_en = 1'b1; - op1_from_spad = 1'b1; - op2_from_spad = 1'b1; - meta_bank = 5'd0; - is_sub = 1'b0; - sub_rob_id = 8'h00; - - if (relu == 0) funct7 = INT8ADD_CORE_FUNCT7[6:0]; - else funct7 = INT8ADD_RELU_CORE_FUNCT7[6:0]; - - if (iter == 0) `uvm_fatal("CASE", "Int8AddBall iter must be positive") - if (op1_col != 1 || op2_col != 1 || wr_col != 1) - `uvm_fatal("CASE", "Int8AddBall bank groups must match") - if (op1_bank == op2_bank || op1_bank == wr_bank || op2_bank == wr_bank) - `uvm_fatal("CASE", "Int8AddBall banks must be distinct") - if (num_lhs_words != iter) - `uvm_fatal("CASE", $sformatf("num_lhs_words %0d != iter %0d", num_lhs_words, iter)) - if (num_rhs_words != iter) - `uvm_fatal("CASE", $sformatf("num_rhs_words %0d != iter %0d", num_rhs_words, iter)) - if (num_dst_words != iter) - `uvm_fatal("CASE", $sformatf("num_dst_words %0d != iter %0d", num_dst_words, iter)) - if (funct7 != INT8ADD_CORE_FUNCT7[6:0] && funct7 != INT8ADD_RELU_CORE_FUNCT7[6:0]) - `uvm_fatal("CASE", $sformatf("invalid funct7: %0d", funct7)) - - for (i = 0; i < num_lhs_words; i++) begin - w_lo = int8add_case_lhs_word_lo(i); - w_hi = int8add_case_lhs_word_hi(i); - lhs_words[i] = {w_hi, w_lo}; - end - for (i = 0; i < num_rhs_words; i++) begin - w_lo = int8add_case_rhs_word_lo(i); - w_hi = int8add_case_rhs_word_hi(i); - rhs_words[i] = {w_hi, w_lo}; - end - for (i = 0; i < num_dst_words; i++) begin - w_lo = int8add_case_dst_word_lo(i); - w_hi = int8add_case_dst_word_hi(i); - dst_words[i] = {w_hi, w_lo}; - end - endfunction - - function void do_copy(uvm_object rhs); - int8add_cmd_item rhs_; - super.do_copy(rhs); - if (!$cast(rhs_, rhs)) begin - `uvm_fatal("COPY", "rhs is not int8add_cmd_item") - end - relu = rhs_.relu; - num_lhs_words = rhs_.num_lhs_words; - num_rhs_words = rhs_.num_rhs_words; - num_dst_words = rhs_.num_dst_words; - for (int i = 0; i < INT8ADD_MAX_ROWS; i++) lhs_words[i] = rhs_.lhs_words[i]; - for (int i = 0; i < INT8ADD_MAX_ROWS; i++) rhs_words[i] = rhs_.rhs_words[i]; - for (int i = 0; i < INT8ADD_MAX_ROWS; i++) dst_words[i] = rhs_.dst_words[i]; - endfunction -endclass diff --git a/examples/balls/int8add/verify/src/cov/int8add_cov.svh b/examples/balls/int8add/verify/src/cov/int8add_cov.svh deleted file mode 100644 index 1c756c3a5..000000000 --- a/examples/balls/int8add/verify/src/cov/int8add_cov.svh +++ /dev/null @@ -1,36 +0,0 @@ -class int8add_cov extends uvm_component; - `uvm_component_utils(int8add_cov) - - uvm_analysis_imp_cmd #(bb_blink_cmd_item, int8add_cov) cmd_imp; - - bit [6:0] cur_funct7; - - covergroup cmd_cg; - coverpoint cur_funct7 { - bins add = {INT8ADD_CORE_FUNCT7[6:0]}; bins relu = {INT8ADD_RELU_CORE_FUNCT7[6:0]}; - } - endgroup - - int cmd_count; - - function new(string name, uvm_component parent); - super.new(name, parent); - cmd_imp = new("cmd_imp", this); - cmd_cg = new(); - endfunction - - function void write_cmd(bb_blink_cmd_item item); - if (item.funct7 != INT8ADD_CORE_FUNCT7[6:0] && item.funct7 != INT8ADD_RELU_CORE_FUNCT7[6:0]) - `uvm_fatal("COV", $sformatf("unexpected funct7 %0d", item.funct7)) - cur_funct7 = item.funct7; - cmd_count++; - cmd_cg.sample(); - endfunction - - function void check_phase(uvm_phase phase); - super.check_phase(phase); - if (cmd_count == 0) `uvm_fatal("COV", "no cmd observed") - if (cmd_cg.get_coverage() != 100.0) - `uvm_fatal("COV", $sformatf("int8add_cov %0.2f%%", cmd_cg.get_coverage())) - endfunction -endclass diff --git a/examples/balls/int8add/verify/src/tb_top.sv b/examples/balls/int8add/verify/src/tb_top.sv deleted file mode 100644 index 250ef5dc8..000000000 --- a/examples/balls/int8add/verify/src/tb_top.sv +++ /dev/null @@ -1,101 +0,0 @@ -module tb_top; - import uvm_pkg::*; - import int8add_pkg::*; - - bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); - - Int8AddBall dut ( - .clock(intf.clock), - .reset(intf.reset), - .io_status_idle(), - .io_status_running(), - .io_channelReady(1'b1), - .io_cmdReq_ready(intf.cmd_req_ready), - .io_cmdReq_valid(intf.cmd_req_valid), - .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), - .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), - .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), - .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), - .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), - .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), - .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), - .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), - .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), - .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank), - .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank), - .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank), - .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), - .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), - .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), - .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank), - .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), - .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), - .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), - .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), - .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), - .io_cmdResp_ready(intf.cmd_resp_ready), - .io_cmdResp_valid(intf.cmd_resp_valid), - .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), - .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), - .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), - .io_bankRead_0_bank_id(intf.bank_read_bank_id[0]), - .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), - .io_bankRead_0_ball_id(), - .io_bankRead_0_group_id(intf.bank_read_group_id[0]), - .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), - .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), - .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), - .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), - .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), - .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), - .io_bankRead_1_bank_id(intf.bank_read_bank_id[1]), - .io_bankRead_1_rob_id(intf.bank_read_rob_id[1]), - .io_bankRead_1_ball_id(), - .io_bankRead_1_group_id(intf.bank_read_group_id[1]), - .io_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), - .io_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), - .io_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), - .io_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), - .io_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), - .io_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), - .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), - .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), - .io_bankWrite_0_ball_id(), - .io_bankWrite_0_group_id(intf.bank_write_group_id[0]), - .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), - .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), - .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), - .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), - .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), - .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), - .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), - .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), - .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), - .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), - .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), - .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), - .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), - .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), - .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), - .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), - .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), - .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), - .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), - .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), - .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), - .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), - .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), - .io_subRobReq_ready(intf.sub_rob_req_ready) - ); - - initial begin - intf.clock = 1'b0; - intf.reset = 1'b1; - forever #5 intf.clock = ~intf.clock; - end - - initial begin - uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); - run_test("int8add_ball_test"); - end -endmodule diff --git a/examples/balls/int8add/workloads/ctests/CMakeLists.txt b/examples/balls/int8add/workloads/ctests/CMakeLists.txt deleted file mode 100644 index ce68547cc..000000000 --- a/examples/balls/int8add/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_buckyball_ctests(int8add_scaled_test.c int8add_relu_test.c) diff --git a/examples/balls/int8add/workloads/mlir_tests/ball/int8add_relu.mlir b/examples/balls/int8add/workloads/mlir_tests/ball/int8add_relu.mlir deleted file mode 100644 index 57575b219..000000000 --- a/examples/balls/int8add/workloads/mlir_tests/ball/int8add_relu.mlir +++ /dev/null @@ -1,42 +0,0 @@ -func.func private @check_result(memref<4x16xi8>) -> () - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %four = arith.constant 4 : index - %sixteen = arith.constant 16 : index - %depth = arith.constant 4 : i64 - %stride = arith.constant 1 : i64 - %ratio = arith.constant 1.0 : f32 - %a = memref.alloc() : memref<4x16xi8> - %b = memref.alloc() : memref<4x16xi8> - %out = memref.alloc() : memref<4x16xi8> - scf.for %row = %zero to %four step %one { - scf.for %lane = %zero to %sixteen step %one { - %r = arith.index_cast %row : index to i32 - %l = arith.index_cast %lane : index to i32 - %a32 = arith.subi %l, %r : i32 - %b32 = arith.subi %r, %l : i32 - %a8 = arith.trunci %a32 : i32 to i8 - %b8 = arith.trunci %b32 : i32 to i8 - memref.store %a8, %a[%row, %lane] : memref<4x16xi8> - memref.store %b8, %b[%row, %lane] : memref<4x16xi8> - } - } - %a_bank = buckyball.bank_alloc - %b_bank = buckyball.bank_alloc - %out_bank = buckyball.bank_alloc - %a_loaded = buckyball.bank_mvin %a %a_bank %depth %stride : memref<4x16xi8> i64 i64 i64 - %b_loaded = buckyball.bank_mvin %b %b_bank %depth %stride : memref<4x16xi8> i64 i64 i64 - buckyball.int8add %a_loaded, %b_loaded, %out_bank, %depth, %ratio, %ratio : i64 - %stored = buckyball.bank_mvout %out %out_bank %depth %stride : memref<4x16xi8> i64 i64 i64 - func.call @check_result(%out) : (memref<4x16xi8>) -> () - buckyball.bank_release %a_loaded : i64 - buckyball.bank_release %b_loaded : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %a : memref<4x16xi8> - memref.dealloc %b : memref<4x16xi8> - memref.dealloc %out : memref<4x16xi8> - return %zero_i8 : i8 -} diff --git a/examples/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale.mlir.in b/examples/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale.mlir.in deleted file mode 100644 index 8abc7defa..000000000 --- a/examples/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale.mlir.in +++ /dev/null @@ -1,44 +0,0 @@ -func.func private @check_result(memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %mod = arith.constant 31 : index - %rows = arith.constant @BB_BANK_LINES@ : index - %lanes = arith.constant @BB_BANK_I8_LANES@ : index - %depth = arith.constant @BB_BANK_LINES@ : i64 - %stride = arith.constant 1 : i64 - %ratio = arith.constant 1.0 : f32 - %a = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - %b = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - %out = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - scf.for %row = %zero to %rows step %one { - scf.for %lane = %zero to %lanes step %one { - %row_mod = arith.remui %row, %mod : index - %r = arith.index_cast %row_mod : index to i32 - %l = arith.index_cast %lane : index to i32 - %a32 = arith.subi %l, %r : i32 - %b32 = arith.subi %r, %l : i32 - %a8 = arith.trunci %a32 : i32 to i8 - %b8 = arith.trunci %b32 : i32 to i8 - memref.store %a8, %a[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - memref.store %b8, %b[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - } - } - %a_bank = buckyball.bank_alloc - %b_bank = buckyball.bank_alloc - %out_bank = buckyball.bank_alloc - %a_loaded = buckyball.bank_mvin %a %a_bank %depth %stride : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> i64 i64 i64 - %b_loaded = buckyball.bank_mvin %b %b_bank %depth %stride : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> i64 i64 i64 - %result = buckyball.bank_int8add %a_loaded %b_loaded %out_bank %depth %ratio %ratio : i64 i64 i64 i64 f32 - %stored = buckyball.bank_mvout %out %result %depth %stride : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> i64 i64 i64 - func.call @check_result(%out) : (memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () - buckyball.bank_release %a_loaded : i64 - buckyball.bank_release %b_loaded : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %a : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - memref.dealloc %b : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - memref.dealloc %out : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - return %zero_i8 : i8 -} diff --git a/examples/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale_main.cpp b/examples/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale_main.cpp deleted file mode 100644 index cea5836c7..000000000 --- a/examples/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale_main.cpp +++ /dev/null @@ -1,17 +0,0 @@ -#include -#include -#include -#include -extern "C" void check_result(int8_t *, int8_t *data, int64_t offset, - int64_t rows, int64_t lanes, int64_t stride, - int64_t lane_stride) { - if (rows != BANK_LINES || lanes != BANK_WIDTH / 8 || stride != lanes || - lane_stride != 1) - exit(1); - for (int i = 0; i < rows * lanes; ++i) - if (data[offset + i] != 0) { - printf("int8add bank mismatch index=%d\n", i); - exit(1); - } - printf("int8add bank PASS rows=%d\n", BANK_LINES); -} diff --git a/examples/balls/int8mul/arch/src/main/scala/Int8MulBall.scala b/examples/balls/int8mul/arch/src/main/scala/Int8MulBall.scala deleted file mode 100644 index 825deca55..000000000 --- a/examples/balls/int8mul/arch/src/main/scala/Int8MulBall.scala +++ /dev/null @@ -1,238 +0,0 @@ -package examples.balls.int8mul - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public} -import chisel3.util._ -import hardfloat.{recFNFromFN, INToRecFN, RecFNToIN} -import hardfloat.consts.{round_near_even, tininess_afterRounding} -import freechips.rocketchip.tile.MulAddRecFNPipe - -import framework.balldomain.blink.{BallStatus, BlinkIO, HasBallStatus, HasBlink, SubRobRow} -import framework.balldomain.blink.mmio.{MmioRead, MmioWrite} -import framework.top.GlobalConfig - -@instantiable -class Int8MulBall(val b: GlobalConfig) extends Module with HasBlink with HasBallStatus { - - private val mapping = b.ballDomain.ballIdMappings - .find(_.ballName == "Int8MulBall") - .getOrElse(throw new IllegalArgumentException("Int8MulBall not found in config")) - - private val funct = b.ballDomain.ballISA - .find(_.mnemonic == "INT8MUL") - .map(_.funct7) - .getOrElse(throw new IllegalArgumentException("INT8MUL not found in ballISA")) - - require(mapping.inBW == 2, "Int8MulBall requires inBW=2") - require(mapping.outBW == 1, "Int8MulBall requires outBW=1") - require(b.memDomain.bankWidth == 128, "Int8MulBall requires 128-bit bank rows") - private val bankRowBits = log2Ceil(b.memDomain.bankEntries) - require(bankRowBits <= 32, "Int8MulBall gate_row must fit rs2[63:32]") - require((funct >> 4) == 4, "INT8MUL must encode two reads and one write") - - @public - val io = IO(new BlinkIO(b, mapping.inBW, mapping.outBW)) - def blink: BlinkIO = io - def status: BallStatus = io.status - dontTouch(io) - - private val idle :: waitForChannels :: gateRequest :: gateResponse :: inputRequest :: inputResponse :: calculate :: calculateDrain :: writeRequest :: writeResponse :: complete :: Nil = - Enum(11) - private val state = RegInit(idle) - private val robId = RegInit(0.U(log2Up(b.frontend.rob_entries).W)) - private val isSub = RegInit(false.B) - private val subRobId = RegInit(0.U(log2Up(b.frontend.sub_rob_depth * 4).W)) - private val gateBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - private val inputBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - private val outputBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - private val groups = RegInit(0.U(5.W)) - private val group = RegInit(0.U(5.W)) - private val iter = RegInit(0.U(b.frontend.iter_len.W)) - private val gateRow = RegInit(0.U(bankRowBits.W)) - private val inputRow = RegInit(0.U(log2Ceil(b.memDomain.bankEntries).W)) - private val ratio = Reg(UInt(32.W)) - private val gateWord = Reg(UInt(128.W)) - private val inputWord = Reg(UInt(128.W)) - private val lane = RegInit(0.U(4.W)) - private val outputWord = Reg(Vec(16, UInt(8.W))) - - private val productReg = Reg(UInt(16.W)) - private val productLaneReg = Reg(UInt(4.W)) - private val productValid = RegInit(false.B) - private val productFloatReg = Reg(UInt(33.W)) - private val floatLaneReg = Reg(UInt(4.W)) - private val floatValid = RegInit(false.B) - private val scaledReg = Reg(UInt(33.W)) - private val scaledLaneReg = Reg(UInt(4.W)) - private val scaledValid = RegInit(false.B) - - private def positiveFinite(value: UInt): Bool = - !value(31) && value(30, 23) =/= 255.U && value(30, 0) =/= 0.U - - private val gateValue = (gateWord >> (lane << 3))(7, 0).asSInt - private val inputValue = (inputWord >> (lane << 3))(7, 0).asSInt - private val integerProduct = (gateValue * inputValue).asUInt - private val productToFloat = Module(new INToRecFN(16, 8, 24)) - private val scale = Module(new MulAddRecFNPipe(3, 8, 24)) - private val toInt8 = Module(new RecFNToIN(8, 24, 8)) - - productToFloat.io.signedIn := true.B - productToFloat.io.in := productReg - productToFloat.io.roundingMode := round_near_even - productToFloat.io.detectTininess := tininess_afterRounding - scale.io.validin := floatValid - scale.io.op := 0.U - scale.io.a := productFloatReg - scale.io.b := recFNFromFN(8, 24, ratio) - scale.io.c := recFNFromFN(8, 24, 0.U) - scale.io.roundingMode := round_near_even - scale.io.detectTininess := tininess_afterRounding.asUInt - toInt8.io.in := scaledReg - toInt8.io.roundingMode := round_near_even - toInt8.io.signedOut := true.B - - io.cmdReq.ready := state === idle - io.cmdResp.valid := state === complete - io.cmdResp.bits.rob_id := robId - io.cmdResp.bits.is_sub := isSub - io.cmdResp.bits.sub_rob_id := subRobId - io.status.idle := state === idle - io.status.running := state =/= idle && state =/= complete - - for (port <- 0 until 2) { - io.bankRead(port).rob_id := robId - io.bankRead(port).ball_id := 0.U - io.bankRead(port).bank_id := Mux(port.U === 0.U, gateBank, inputBank) - io.bankRead(port).group_id := group - io.bankRead(port).io.req.valid := false.B - io.bankRead(port).io.req.bits.addr := Mux(port.U === 0.U, gateRow, inputRow) - io.bankRead(port).io.resp.ready := false.B - } - - io.bankWrite(0).rob_id := robId - io.bankWrite(0).ball_id := 0.U - io.bankWrite(0).bank_id := outputBank - io.bankWrite(0).group_id := group - io.bankWrite(0).io.req.valid := false.B - io.bankWrite(0).io.req.bits.addr := inputRow - io.bankWrite(0).io.req.bits.data := Cat(outputWord.reverse) - io.bankWrite(0).io.req.bits.mask := VecInit(Seq.fill(16)(true.B)) - io.bankWrite(0).io.resp.ready := false.B - - io.subRobReq.valid := false.B - io.subRobReq.bits := SubRobRow.tieOff(b) - MmioRead.tieOff(io.mmioRead) - MmioWrite.tieOff(io.mmioWrite) - - switch(state) { - is(idle) { - when(io.cmdReq.fire) { - val cmd = io.cmdReq.bits.cmd - assert(cmd.funct7 === funct.U, "Int8MulBall funct7 must be INT8MUL") - if (bankRowBits < 32) - assert( - cmd.rs2(63, 32 + bankRowBits) === 0.U, - s"Int8MulBall reserves rs2 bits 63:${32 + bankRowBits}" - ) - assert(positiveFinite(cmd.rs2(31, 0)), "Int8MulBall ratio must be finite and positive") - assert( - cmd.rs2(31 + bankRowBits, 32) < b.memDomain.bankEntries.U, - "Int8MulBall gate_row must fit one physical bank" - ) - assert( - cmd.op1_col =/= 0.U && cmd.op1_col === cmd.op2_col && cmd.op1_col === cmd.wr_col, - "Int8MulBall bank groups must match" - ) - assert(cmd.op1_col <= b.memDomain.bankNum.U, "Int8MulBall bank groups exceed physical banks") - assert( - cmd.op1_bank =/= cmd.op2_bank && cmd.op1_bank =/= cmd.wr_bank && cmd.op2_bank =/= cmd.wr_bank, - "Int8MulBall banks must be distinct" - ) - assert(cmd.iter =/= 0.U && cmd.iter <= b.memDomain.bankEntries.U, "Int8MulBall iter must fit one physical bank") - robId := io.cmdReq.bits.rob_id - isSub := io.cmdReq.bits.is_sub - subRobId := io.cmdReq.bits.sub_rob_id - gateBank := cmd.op1_bank - inputBank := cmd.op2_bank - outputBank := cmd.wr_bank - groups := cmd.op1_col - group := 0.U - iter := cmd.iter - gateRow := cmd.rs2(31 + bankRowBits, 32) - inputRow := 0.U - ratio := cmd.rs2(31, 0) - lane := 0.U - state := waitForChannels - } - } - is(waitForChannels) { - when(io.channelReady)(state := gateRequest) - } - is(gateRequest) { - io.bankRead(0).io.req.valid := true.B - when(io.bankRead(0).io.req.fire)(state := gateResponse) - } - is(gateResponse) { - io.bankRead(0).io.resp.ready := true.B - when(io.bankRead(0).io.resp.fire) { - gateWord := io.bankRead(0).io.resp.bits.data - state := inputRequest - } - } - is(inputRequest) { - io.bankRead(1).io.req.valid := true.B - when(io.bankRead(1).io.req.fire)(state := inputResponse) - } - is(inputResponse) { - io.bankRead(1).io.resp.ready := true.B - when(io.bankRead(1).io.resp.fire) { - inputWord := io.bankRead(1).io.resp.bits.data - lane := 0.U - state := calculate - } - } - is(calculate) { - productReg := integerProduct - productLaneReg := lane - productValid := true.B - when(lane === 15.U)(state := calculateDrain) - .otherwise(lane := lane + 1.U) - } - is(calculateDrain) { - when(scaledValid && scaledLaneReg === 15.U)(state := writeRequest) - } - is(writeRequest) { - io.bankWrite(0).io.req.valid := true.B - when(io.bankWrite(0).io.req.fire)(state := writeResponse) - } - is(writeResponse) { - io.bankWrite(0).io.resp.ready := true.B - when(io.bankWrite(0).io.resp.fire) { - when(inputRow +& 1.U === iter) { - inputRow := 0.U - when(group +& 1.U === groups)(state := complete) - .otherwise { group := group + 1.U; state := gateRequest } - }.otherwise { inputRow := inputRow + 1.U; state := inputRequest } - } - } - is(complete) { - when(io.cmdResp.fire)(state := idle) - } - } - - when(productValid) { - productFloatReg := productToFloat.io.out - floatLaneReg := productLaneReg - } - when(scale.io.validout) { - scaledReg := scale.io.out - scaledLaneReg := Pipe(floatValid, floatLaneReg, 3).bits - } - when(scaledValid) { - outputWord(scaledLaneReg) := toInt8.io.out - } - - productValid := state === calculate - floatValid := productValid - scaledValid := scale.io.validout -} diff --git a/examples/balls/int8mul/compiler/src/Backend/include/IntrinsicsRISCVInt8MulBall.td b/examples/balls/int8mul/compiler/src/Backend/include/IntrinsicsRISCVInt8MulBall.td deleted file mode 100644 index 0a743ab0f..000000000 --- a/examples/balls/int8mul/compiler/src/Backend/include/IntrinsicsRISCVInt8MulBall.td +++ /dev/null @@ -1 +0,0 @@ -// Int8MulBall uses CustomIntrOp with funct7 from the target ballISA. diff --git a/examples/balls/int8mul/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp b/examples/balls/int8mul/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp deleted file mode 100644 index da185c78b..000000000 --- a/examples/balls/int8mul/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp +++ /dev/null @@ -1,35 +0,0 @@ -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace mlir::buddy { -void populateInt8MulBallAssignPhysicalBankPatterns(RewritePatternSet &patterns, - PhysicalBankState &state); -} - -namespace { -class BankInt8MulPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(BankInt8MulOp op, - PatternRewriter &rewriter) const override { - rewriter.create(op.getLoc(), op.getGateBank(), op.getInputBank(), - op.getOutputBank(), op.getIter(), op.getRatio(), - op.getGateRow()); - rewriter.replaceOp(op, op.getOutputBank()); - return success(); - } -}; -} // namespace - -void mlir::buddy::populateInt8MulBallAssignPhysicalBankPatterns( - RewritePatternSet &patterns, PhysicalBankState &state) { - (void)state; - patterns.add(patterns.getContext()); -} diff --git a/examples/balls/int8mul/compiler/src/Dialect/Buckyball/Int8MulBall.td b/examples/balls/int8mul/compiler/src/Dialect/Buckyball/Int8MulBall.td deleted file mode 100644 index 6295351a1..000000000 --- a/examples/balls/int8mul/compiler/src/Dialect/Buckyball/Int8MulBall.td +++ /dev/null @@ -1,26 +0,0 @@ -#ifndef INT8MULBALL_OPS -#define INT8MULBALL_OPS - -include "Buckyball.td" - -def Int8MulOp : Buckyball_Op<"int8mul"> { - let summary = "Scaled INT8 per-channel broadcast multiply on physical banks"; - let arguments = (ins I64:$gateBankId, I64:$inputBankId, I64:$outputBankId, - I64:$iter, F32:$ratio, I64:$gateRow); - let assemblyFormat = "$gateBankId `,` $inputBankId `,` $outputBankId `,` " - "$iter `,` $ratio `,` $gateRow attr-dict `:` " - "type($gateBankId)"; -} - -def BankInt8MulOp : Buckyball_Op<"bank_int8mul"> { - let summary = "Scaled INT8 per-channel broadcast multiply on virtual banks"; - let arguments = (ins I64:$gateBank, I64:$inputBank, I64:$outputBank, - I64:$iter, F32:$ratio, I64:$gateRow); - let results = (outs I64:$outputBankOut); - let assemblyFormat = "$gateBank $inputBank $outputBank $iter $ratio " - "$gateRow attr-dict `:` type($gateBank) " - "type($inputBank) type($outputBank) type($iter) " - "type($ratio) type($gateRow)"; -} - -#endif // INT8MULBALL_OPS diff --git a/examples/balls/int8mul/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/balls/int8mul/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index cbcfd4377..000000000 --- a/examples/balls/int8mul/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,52 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace buddy::buckyball; -using namespace buddy::buckyball::legalize; - -namespace { -struct Int8MulLowering : public ConvertOpToLLVMPattern { - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - - LogicalResult - matchAndRewrite(Int8MulOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("Int8MulBall"); - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getGateBankId(), - adaptor.getInputBankId(), - adaptor.getOutputBankId(), adaptor.getIter()); - Value ratioBits = rewriter.create( - loc, rewriter.getI32Type(), adaptor.getRatio()); - Value ratio = - rewriter.create(loc, rewriter.getI64Type(), ratioBits); - Value gateRow = rewriter.create(loc, adaptor.getGateRow(), - cstI64(rewriter, loc, 32)); - Value rs2 = rewriter.create(loc, ratio, gateRow); - rewriter.replaceOpWithNewOp( - op, rs1, rs2, - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("INT8MUL"))); - return success(); - } -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateInt8MulBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool, int64_t) { - patterns.add(converter); -} - -void configureInt8MulBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool) { - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/examples/balls/int8mul/configs/default.toml b/examples/balls/int8mul/configs/default.toml deleted file mode 100644 index b7bdf2941..000000000 --- a/examples/balls/int8mul/configs/default.toml +++ /dev/null @@ -1,3 +0,0 @@ -# Int8MulBall has no private configuration. - -[ball] diff --git a/examples/balls/int8mul/emu/src/dispatch.rs b/examples/balls/int8mul/emu/src/dispatch.rs deleted file mode 100644 index f6d4f818c..000000000 --- a/examples/balls/int8mul/emu/src/dispatch.rs +++ /dev/null @@ -1,31 +0,0 @@ -use crate::inst::instruction::ExecContext; - -use super::int8mul; - -const BALL_CLASS: &str = "examples.balls.int8mul.Int8MulBall"; - -pub fn execute_known( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, - ctx: &mut ExecContext, -) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("INT8MUL") => Some(int8mul::Int8Mul::exec(xs1, xs2, ctx)), - Some(_) | None => None, - } -} - -pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("INT8MUL") => Some(int8mul::Int8Mul::latency(xs1, xs2)), - Some(_) | None => None, - } -} diff --git a/examples/balls/int8mul/emu/src/lib.rs b/examples/balls/int8mul/emu/src/lib.rs deleted file mode 100644 index 7a691a4aa..000000000 --- a/examples/balls/int8mul/emu/src/lib.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub(crate) use crate::inst::{decode, instruction}; - -mod int8mul; - -mod dispatch; - -pub use dispatch::{cycles_after_issue, execute_known}; diff --git a/examples/balls/int8mul/verify/casegen/src/casegen.rs b/examples/balls/int8mul/verify/casegen/src/casegen.rs deleted file mode 100644 index c42ddd88a..000000000 --- a/examples/balls/int8mul/verify/casegen/src/casegen.rs +++ /dev/null @@ -1,456 +0,0 @@ -pub const ROW_BYTES: usize = 16; -pub const MAX_ROWS: usize = 64; -pub const NUM_CASES: u32 = 3; -pub const GATE_ROW: u32 = 1; -pub const RATIO: f32 = 0.25; - -#[repr(C)] -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct Int8MulCmd { - pub bid: u32, - pub iter: u32, - pub gate_bank: u32, - pub input_bank: u32, - pub output_bank: u32, - pub op1_col: u32, - pub op2_col: u32, - pub wr_col: u32, - pub gate_row: u32, - pub bank_entries: u32, - pub rob_id: u32, - pub rs1_lo: u32, - pub rs1_hi: u32, - pub rs2_lo: u32, - pub rs2_hi: u32, - pub num_gate_words: u32, - pub num_input_words: u32, - pub num_dst_words: u32, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct Int8MulCase { - pub cmd: Int8MulCmd, - pub gate_words: [u128; MAX_ROWS], - pub input_words: [u128; MAX_ROWS], - pub dst_words: [u128; MAX_ROWS], -} - -impl Int8MulCase { - pub fn gate_lo(&self, index: usize) -> u64 { - self.gate_words[index] as u64 - } - - pub fn gate_hi(&self, index: usize) -> u64 { - (self.gate_words[index] >> 64) as u64 - } - - pub fn input_lo(&self, index: usize) -> u64 { - self.input_words[index] as u64 - } - - pub fn input_hi(&self, index: usize) -> u64 { - (self.input_words[index] >> 64) as u64 - } - - pub fn dst_lo(&self, index: usize) -> u64 { - self.dst_words[index] as u64 - } - - pub fn dst_hi(&self, index: usize) -> u64 { - (self.dst_words[index] >> 64) as u64 - } -} - -pub fn gen_case(index: u32, bid: u32, bank_entries: u32) -> Int8MulCase { - if bank_entries == 0 { - panic!("int8mul: bank entries must be positive"); - } - if index >= NUM_CASES { - panic!("int8mul: unsupported directed case {index}"); - } - match index { - 0 => directed(bid, 2, 1, GATE_ROW, bank_entries), - 1 => directed(bid, 3, 4, GATE_ROW, bank_entries), - 2 => directed(bid, 4, 1, bank_entries - 1, bank_entries), - _ => panic!("int8mul: unsupported directed case {index}"), - } -} - -fn compute(gate: i8, input: i8, ratio: f32) -> i8 { - let value = (gate as f32) * (input as f32) * ratio; - value.round_ties_even().clamp(-128.0, 127.0) as i8 -} - -fn positive_finite(value: f32) -> bool { - value.is_finite() && value > 0.0 -} - -fn ctest_gate() -> u128 { - let mut row = [0u8; ROW_BYTES]; - for lane in 0..ROW_BYTES { - row[lane] = (lane as i8 - 8) as u8; - } - pack_bytes(&row)[0] -} - -fn ctest_input(rows: usize) -> Vec { - let count = rows * ROW_BYTES; - let bytes: Vec = (0..count) - .map(|i| ((i as i32 * 37 & 255) - 128) as i8 as u8) - .collect(); - pack_bytes(&bytes) -} - -fn pack_bytes(bytes: &[u8]) -> Vec { - if bytes.len() % ROW_BYTES != 0 { - panic!( - "int8mul: {} bytes is not a multiple of {ROW_BYTES}", - bytes.len() - ); - } - bytes - .chunks(ROW_BYTES) - .map(|chunk| { - let mut word = 0u128; - for (lane, byte) in chunk.iter().enumerate() { - word |= u128::from(*byte) << (lane * 8); - } - word - }) - .collect() -} - -fn copy_words(src: &[u128]) -> [u128; N] { - if src.len() > N { - panic!("int8mul: {} words exceed buffer {N}", src.len()); - } - let mut out = [0u128; N]; - out[..src.len()].copy_from_slice(src); - out -} - -fn encode_rs1(gate: u32, input: u32, output: u32, iter: u32) -> u64 { - u64::from(gate) | (u64::from(input) << 10) | (u64::from(output) << 20) | (u64::from(iter) << 30) -} - -fn encode_rs2(ratio: f32, gate_row: u32) -> u64 { - u64::from(ratio.to_bits()) | (u64::from(gate_row) << 32) -} - -fn split_rs(value: u64) -> (u32, u32) { - (value as u32, (value >> 32) as u32) -} - -pub(crate) fn validate( - gate_bank: u32, - input_bank: u32, - output_bank: u32, - op1_col: u32, - op2_col: u32, - wr_col: u32, - iter: u32, - gate_row: u32, - rs1: u64, - rs2: u64, - ratio: f32, - bank_entries: u32, -) { - if bank_entries == 0 { - panic!("int8mul: bank entries must be positive"); - } - if iter == 0 || iter > bank_entries { - panic!("int8mul: iter must fit one physical bank"); - } - if gate_row >= bank_entries { - panic!("int8mul: gate row must fit one physical bank"); - } - if op1_col != 1 || op2_col != 1 || wr_col != 1 { - panic!("int8mul: bank groups must match"); - } - if gate_bank == input_bank || gate_bank == output_bank || input_bank == output_bank { - panic!("int8mul: banks must be distinct"); - } - if !positive_finite(ratio) { - panic!("int8mul: ratio must be finite and positive"); - } - let gate_row_bits = u32::BITS - (bank_entries - 1).leading_zeros(); - if rs2 >> (32 + gate_row_bits) != 0 { - panic!("int8mul: reserved rs2 bits must be zero"); - } - if (rs1 & 0x3ff) != u64::from(gate_bank) - || ((rs1 >> 10) & 0x3ff) != u64::from(input_bank) - || ((rs1 >> 20) & 0x3ff) != u64::from(output_bank) - { - panic!("int8mul: rs1 bank fields mismatch"); - } - if ((rs1 >> 30) & 0x3_ffff_ffff) != u64::from(iter) { - panic!("int8mul: rs1 iter mismatch"); - } - if rs2 != encode_rs2(ratio, gate_row) { - panic!("int8mul: rs2 ratio/gate_row fields mismatch"); - } -} - -fn dst_words(gate_word: u128, input: &[u128], iter: u32, ratio: f32) -> Vec { - let mut out = Vec::with_capacity(iter as usize); - for row in 0..iter as usize { - let mut packed = 0u128; - for lane in 0..ROW_BYTES { - let g = ((gate_word >> (lane * 8)) & 0xff) as u8 as i8; - let inp = ((input[row] >> (lane * 8)) & 0xff) as u8 as i8; - let q = compute(g, inp, ratio); - packed |= u128::from(q as u8) << (lane * 8); - } - out.push(packed); - } - out -} - -fn directed(bid: u32, rob_id: u32, iter: u32, gate_row: u32, bank_entries: u32) -> Int8MulCase { - let gate_bank = 0; - let input_bank = 1; - let output_bank = 2; - let op1_col = 1; - let op2_col = 1; - let wr_col = 1; - let ratio = RATIO; - let gate = ctest_gate(); - let input = ctest_input(iter as usize); - let rs1 = encode_rs1(gate_bank, input_bank, output_bank, iter); - let rs2 = encode_rs2(ratio, gate_row); - validate( - gate_bank, - input_bank, - output_bank, - op1_col, - op2_col, - wr_col, - iter, - gate_row, - rs1, - rs2, - ratio, - bank_entries, - ); - if input.len() != iter as usize { - panic!("int8mul: input rows {} != iter {iter}", input.len()); - } - let dst = dst_words(gate, &input, iter, ratio); - let (rs1_lo, rs1_hi) = split_rs(rs1); - let (rs2_lo, rs2_hi) = split_rs(rs2); - Int8MulCase { - cmd: Int8MulCmd { - bid, - iter, - gate_bank, - input_bank, - output_bank, - op1_col, - op2_col, - wr_col, - gate_row, - bank_entries, - rob_id, - rs1_lo, - rs1_hi, - rs2_lo, - rs2_hi, - num_gate_words: 1, - num_input_words: input.len() as u32, - num_dst_words: dst.len() as u32, - }, - gate_words: copy_words(&[gate]), - input_words: copy_words(&input), - dst_words: copy_words(&dst), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - const TEST_BANK_ENTRIES: u32 = 4096; - - fn ctest_expected(gate: i8, input: i8) -> i8 { - compute(gate, input, RATIO) - } - - #[test] - fn case_zero_matches_ctest() { - let case = gen_case(0, 8, TEST_BANK_ENTRIES); - assert_eq!(case.cmd.bid, 8); - assert_eq!(case.cmd.iter, 1); - assert_eq!(case.cmd.gate_bank, 0); - assert_eq!(case.cmd.input_bank, 1); - assert_eq!(case.cmd.output_bank, 2); - assert_eq!(case.cmd.gate_row, GATE_ROW); - assert_eq!(case.cmd.bank_entries, TEST_BANK_ENTRIES); - assert_eq!(case.cmd.num_gate_words, 1); - assert_eq!(case.cmd.num_input_words, 1); - assert_eq!(case.cmd.num_dst_words, 1); - let gate_lane0 = (case.gate_words[0] & 0xff) as u8 as i8; - let input0 = ((case.input_words[0] >> 0) & 0xff) as u8 as i8; - assert_eq!(gate_lane0, -8); - assert_eq!(input0, -128); - let got0 = (case.dst_words[0] & 0xff) as u8 as i8; - assert_eq!(got0, ctest_expected(gate_lane0, input0)); - } - - #[test] - fn cases_hit_iter_bins() { - let mut hit = [false; 2]; - for index in 0..NUM_CASES { - let case = gen_case(index, 8, TEST_BANK_ENTRIES); - assert_eq!(case.cmd.bid, 8); - assert_eq!(case.cmd.op1_col, 1); - assert_eq!(case.cmd.op2_col, 1); - assert_eq!(case.cmd.wr_col, 1); - assert_eq!(case.cmd.num_input_words, case.cmd.iter); - assert_eq!(case.cmd.num_dst_words, case.cmd.iter); - let rs2 = u64::from(case.cmd.rs2_lo) | (u64::from(case.cmd.rs2_hi) << 32); - assert_eq!(rs2 >> (32 + TEST_BANK_ENTRIES.ilog2()), 0); - match case.cmd.iter { - 1 => hit[0] = true, - 4 => hit[1] = true, - other => panic!("unexpected iter {other}"), - } - } - assert!(hit.iter().all(|h| *h)); - } - - #[test] - fn final_gate_row_tracks_bank_entries() { - let small = gen_case(2, 8, 64); - let non_power_of_two = gen_case(2, 8, 65); - let large = gen_case(2, 8, TEST_BANK_ENTRIES); - assert_eq!(small.cmd.gate_row, 63); - assert_eq!(non_power_of_two.cmd.gate_row, 64); - assert_eq!(large.cmd.gate_row, TEST_BANK_ENTRIES - 1); - let small_rs2 = u64::from(small.cmd.rs2_lo) | (u64::from(small.cmd.rs2_hi) << 32); - let non_power_rs2 = - u64::from(non_power_of_two.cmd.rs2_lo) | (u64::from(non_power_of_two.cmd.rs2_hi) << 32); - let large_rs2 = u64::from(large.cmd.rs2_lo) | (u64::from(large.cmd.rs2_hi) << 32); - assert_eq!(small_rs2 >> 38, 0); - assert_eq!(non_power_rs2 >> 39, 0); - assert_eq!(large_rs2 >> 44, 0); - } - - #[test] - #[should_panic(expected = "unsupported directed case")] - fn unknown_index_panics() { - let _ = gen_case(NUM_CASES, 8, TEST_BANK_ENTRIES); - } - - #[test] - #[should_panic(expected = "iter must fit")] - fn illegal_iter_panics() { - validate( - 0, - 1, - 2, - 1, - 1, - 1, - 0, - GATE_ROW, - encode_rs1(0, 1, 2, 0), - encode_rs2(RATIO, GATE_ROW), - RATIO, - TEST_BANK_ENTRIES, - ); - } - - #[test] - #[should_panic(expected = "gate row must fit")] - fn illegal_gate_row_panics() { - validate( - 0, - 1, - 2, - 1, - 1, - 1, - 1, - TEST_BANK_ENTRIES, - encode_rs1(0, 1, 2, 1), - encode_rs2(RATIO, TEST_BANK_ENTRIES), - RATIO, - TEST_BANK_ENTRIES, - ); - } - - #[test] - #[should_panic(expected = "banks must be distinct")] - fn overlapping_banks_panics() { - validate( - 0, - 0, - 2, - 1, - 1, - 1, - 1, - GATE_ROW, - encode_rs1(0, 0, 2, 1), - encode_rs2(RATIO, GATE_ROW), - RATIO, - TEST_BANK_ENTRIES, - ); - } - - #[test] - #[should_panic(expected = "finite and positive")] - fn nonpositive_ratio_panics() { - validate( - 0, - 1, - 2, - 1, - 1, - 1, - 1, - GATE_ROW, - encode_rs1(0, 1, 2, 1), - encode_rs2(0.0, GATE_ROW), - 0.0, - TEST_BANK_ENTRIES, - ); - } - - #[test] - fn gate_row_1024_uses_configured_address_bits() { - validate( - 0, - 1, - 2, - 1, - 1, - 1, - 1, - 1024, - encode_rs1(0, 1, 2, 1), - encode_rs2(RATIO, 1024), - RATIO, - TEST_BANK_ENTRIES, - ); - } - - #[test] - #[should_panic(expected = "reserved rs2 bits")] - fn reserved_rs2_panics() { - validate( - 0, - 1, - 2, - 1, - 1, - 1, - 1, - GATE_ROW, - encode_rs1(0, 1, 2, 1), - 1_u64 << (32 + TEST_BANK_ENTRIES.ilog2()), - RATIO, - TEST_BANK_ENTRIES, - ); - } -} diff --git a/examples/balls/int8mul/verify/casegen/src/lib.rs b/examples/balls/int8mul/verify/casegen/src/lib.rs deleted file mode 100644 index 9432937af..000000000 --- a/examples/balls/int8mul/verify/casegen/src/lib.rs +++ /dev/null @@ -1,130 +0,0 @@ -mod casegen; -mod config; - -use std::cell::RefCell; - -use casegen::{Int8MulCase, Int8MulCmd}; - -thread_local! { - static CURRENT: RefCell> = const { RefCell::new(None) }; -} - -#[no_mangle] -pub extern "C" fn int8mul_case_load(index: u32, bid: u32) -> i32 { - let case = casegen::gen_case(index, bid, config::bank_entries()); - CURRENT.with(|current| *current.borrow_mut() = Some(case)); - 0 -} - -fn current_case R, R>(f: F) -> R { - CURRENT.with(|current| match current.borrow().as_ref() { - Some(case) => f(case), - None => panic!("int8mul_case: no case loaded; call int8mul_case_load first"), - }) -} - -#[no_mangle] -pub extern "C" fn int8mul_case_cmd(out_ptr: *mut Int8MulCmd) { - if out_ptr.is_null() { - panic!("int8mul_case_cmd: null out_ptr"); - } - current_case(|case| unsafe { - *out_ptr = case.cmd; - }); -} - -fn idx(index: u32, limit: u32, what: &str) -> usize { - if index >= limit { - panic!("int8mul_case_{what}: index {index} out of range {limit}"); - } - index as usize -} - -#[no_mangle] -pub extern "C" fn int8mul_case_gate_word_lo(word_index: u32) -> u64 { - current_case(|case| case.gate_lo(idx(word_index, case.cmd.num_gate_words, "gate_word_lo"))) -} - -#[no_mangle] -pub extern "C" fn int8mul_case_gate_word_hi(word_index: u32) -> u64 { - current_case(|case| case.gate_hi(idx(word_index, case.cmd.num_gate_words, "gate_word_hi"))) -} - -#[no_mangle] -pub extern "C" fn int8mul_case_input_word_lo(word_index: u32) -> u64 { - current_case(|case| case.input_lo(idx(word_index, case.cmd.num_input_words, "input_word_lo"))) -} - -#[no_mangle] -pub extern "C" fn int8mul_case_input_word_hi(word_index: u32) -> u64 { - current_case(|case| case.input_hi(idx(word_index, case.cmd.num_input_words, "input_word_hi"))) -} - -#[no_mangle] -pub extern "C" fn int8mul_case_dst_word_lo(word_index: u32) -> u64 { - current_case(|case| case.dst_lo(idx(word_index, case.cmd.num_dst_words, "dst_word_lo"))) -} - -#[no_mangle] -pub extern "C" fn int8mul_case_dst_word_hi(word_index: u32) -> u64 { - current_case(|case| case.dst_hi(idx(word_index, case.cmd.num_dst_words, "dst_word_hi"))) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn configure_bank_entries(entries: u32) { - let path = std::env::temp_dir().join(format!( - "int8mul-verify-config-{}-{entries}", - std::process::id() - )); - std::fs::write(&path, format!("bank_entries={entries}\n")).unwrap(); - std::env::set_var("BB_VERIFY_CONFIG", path); - } - - #[test] - fn dpi_load_then_cmd_and_words() { - configure_bank_entries(4096); - assert_eq!(int8mul_case_load(0, 8), 0); - let mut cmd = Int8MulCmd { - bid: 0, - iter: 0, - gate_bank: 0, - input_bank: 0, - output_bank: 0, - op1_col: 0, - op2_col: 0, - wr_col: 0, - gate_row: 0, - bank_entries: 0, - rob_id: 0, - rs1_lo: 0, - rs1_hi: 0, - rs2_lo: 0, - rs2_hi: 0, - num_gate_words: 0, - num_input_words: 0, - num_dst_words: 0, - }; - int8mul_case_cmd(&mut cmd as *mut Int8MulCmd); - assert_eq!(cmd.bid, 8); - assert_eq!(cmd.iter, 1); - assert_eq!(cmd.gate_row, 1); - assert_eq!(cmd.bank_entries, 4096); - assert_eq!(cmd.num_gate_words, 1); - assert_eq!(cmd.num_input_words, 1); - assert_eq!(cmd.num_dst_words, 1); - let _ = int8mul_case_gate_word_lo(0); - let _ = int8mul_case_input_word_lo(0); - let _ = int8mul_case_dst_word_lo(0); - assert_eq!(casegen::NUM_CASES, 3); - } - - #[test] - #[should_panic(expected = "no case loaded")] - fn current_case_panics_without_load() { - CURRENT.with(|c| *c.borrow_mut() = None); - current_case(|_| ()); - } -} diff --git a/examples/balls/int8mul/verify/src/common/int8mul_defs.svh b/examples/balls/int8mul/verify/src/common/int8mul_defs.svh deleted file mode 100644 index 8911278e8..000000000 --- a/examples/balls/int8mul/verify/src/common/int8mul_defs.svh +++ /dev/null @@ -1,49 +0,0 @@ -typedef struct { - int unsigned bid; - int unsigned iter; - int unsigned gate_bank; - int unsigned input_bank; - int unsigned output_bank; - int unsigned op1_col; - int unsigned op2_col; - int unsigned wr_col; - int unsigned gate_row; - int unsigned bank_entries; - int unsigned rob_id; - int unsigned rs1_lo; - int unsigned rs1_hi; - int unsigned rs2_lo; - int unsigned rs2_hi; - int unsigned num_gate_words; - int unsigned num_input_words; - int unsigned num_dst_words; -} int8mul_cmd_dpi_t; - -import "DPI-C" function int int8mul_case_load( - input int unsigned index, - input int unsigned bid -); -import "DPI-C" function void int8mul_case_cmd(output int8mul_cmd_dpi_t cmd); -import "DPI-C" function longint unsigned int8mul_case_gate_word_lo(input int unsigned word_index); -import "DPI-C" function longint unsigned int8mul_case_gate_word_hi(input int unsigned word_index); -import "DPI-C" function longint unsigned int8mul_case_input_word_lo(input int unsigned word_index); -import "DPI-C" function longint unsigned int8mul_case_input_word_hi(input int unsigned word_index); -import "DPI-C" function longint unsigned int8mul_case_dst_word_lo(input int unsigned word_index); -import "DPI-C" function longint unsigned int8mul_case_dst_word_hi(input int unsigned word_index); - -`ifndef INT8MUL_FUNCT7 -`error "INT8MUL_FUNCT7 must be provided by the selected Core ballISA" -`endif - -localparam int INT8MUL_CORE_FUNCT7 = `INT8MUL_FUNCT7; -localparam int INT8MUL_MAX_ROWS = 64; -localparam int INT8MUL_NUM_CASES = 3; -localparam int INT8MUL_TIMEOUT_CYCLES = 20000; - -function automatic int unsigned int8mul_require_bid(); - int unsigned bid; - if (!$value$plusargs("BID=%d", bid)) begin - `uvm_fatal("BID", "missing required plusarg +BID=") - end - return bid; -endfunction diff --git a/examples/balls/int8mul/verify/src/common/int8mul_items.svh b/examples/balls/int8mul/verify/src/common/int8mul_items.svh deleted file mode 100644 index b9f3bd93f..000000000 --- a/examples/balls/int8mul/verify/src/common/int8mul_items.svh +++ /dev/null @@ -1,124 +0,0 @@ -class int8mul_cmd_item extends bb_blink_cmd_item; - `uvm_object_utils(int8mul_cmd_item) - - bit [31:0] gate_row; - bit [31:0] bank_entries; - bit [31:0] num_gate_words; - bit [31:0] num_input_words; - bit [31:0] num_dst_words; - bit [127:0] gate_words[INT8MUL_MAX_ROWS]; - bit [127:0] input_words[INT8MUL_MAX_ROWS]; - bit [127:0] dst_words[INT8MUL_MAX_ROWS]; - - constraint legal_c { - funct7 == INT8MUL_CORE_FUNCT7[6:0]; - op1_en == 1'b1; - op2_en == 1'b1; - wr_spad_en == 1'b1; - op1_from_spad == 1'b1; - op2_from_spad == 1'b1; - op1_col == 5'd1; - op2_col == 5'd1; - wr_col == 5'd1; - meta_bank == 5'd0; - is_sub == 1'b0; - sub_rob_id == 8'h00; - special == 64'd0; - } - - function new(string name = "int8mul_cmd_item"); - super.new(name); - endfunction - - function void load_rust_case(int unsigned index, int unsigned bid); - int8mul_cmd_dpi_t cmd; - int unsigned rc; - longint unsigned w_lo; - longint unsigned w_hi; - int i; - - rc = int8mul_case_load(index, bid); - if (rc != 0) begin - `uvm_fatal("CASE", $sformatf("int8mul_case_load returned %0d for index %0d", rc, index)) - end - int8mul_case_cmd(cmd); - - this.bid = cmd.bid[4:0]; - iter = cmd.iter; - op1_bank = cmd.gate_bank[4:0]; - op2_bank = cmd.input_bank[4:0]; - wr_bank = cmd.output_bank[4:0]; - op1_col = cmd.op1_col[4:0]; - op2_col = cmd.op2_col[4:0]; - wr_col = cmd.wr_col[4:0]; - gate_row = cmd.gate_row; - bank_entries = cmd.bank_entries; - rob_id = cmd.rob_id[3:0]; - rs1 = {cmd.rs1_hi, cmd.rs1_lo}; - rs2 = {cmd.rs2_hi, cmd.rs2_lo}; - num_gate_words = cmd.num_gate_words; - num_input_words = cmd.num_input_words; - num_dst_words = cmd.num_dst_words; - funct7 = INT8MUL_CORE_FUNCT7[6:0]; - special = 64'd0; - op1_en = 1'b1; - op2_en = 1'b1; - wr_spad_en = 1'b1; - op1_from_spad = 1'b1; - op2_from_spad = 1'b1; - meta_bank = 5'd0; - is_sub = 1'b0; - sub_rob_id = 8'h00; - - if (iter != 1 && iter != 4) - `uvm_fatal("CASE", $sformatf("Int8MulBall iter %0d is not in {1,4}", iter)) - if (op1_col != 1 || op2_col != 1 || wr_col != 1) - `uvm_fatal("CASE", "Int8MulBall bank groups must match") - if (op1_bank == op2_bank || op1_bank == wr_bank || op2_bank == wr_bank) - `uvm_fatal("CASE", "Int8MulBall banks must be distinct") - if (bank_entries == 0) `uvm_fatal("CASE", "Int8MulBall bank_entries must be positive") - if (gate_row >= bank_entries) - `uvm_fatal("CASE", $sformatf( - "Int8MulBall gate_row %0d exceeds bank_entries %0d", gate_row, bank_entries)) - if ((rs2 >> (32 + $clog2(bank_entries))) != 0) - `uvm_fatal("CASE", $sformatf("Int8MulBall reserved rs2 bits set: 0x%016h", rs2)) - if (num_input_words != iter) - `uvm_fatal("CASE", $sformatf("num_input_words %0d != iter %0d", num_input_words, iter)) - if (num_dst_words != iter) - `uvm_fatal("CASE", $sformatf("num_dst_words %0d != iter %0d", num_dst_words, iter)) - if (funct7 != INT8MUL_CORE_FUNCT7[6:0]) - `uvm_fatal("CASE", $sformatf("invalid funct7: %0d", funct7)) - - for (i = 0; i < num_gate_words; i++) begin - w_lo = int8mul_case_gate_word_lo(i); - w_hi = int8mul_case_gate_word_hi(i); - gate_words[i] = {w_hi, w_lo}; - end - for (i = 0; i < num_input_words; i++) begin - w_lo = int8mul_case_input_word_lo(i); - w_hi = int8mul_case_input_word_hi(i); - input_words[i] = {w_hi, w_lo}; - end - for (i = 0; i < num_dst_words; i++) begin - w_lo = int8mul_case_dst_word_lo(i); - w_hi = int8mul_case_dst_word_hi(i); - dst_words[i] = {w_hi, w_lo}; - end - endfunction - - function void do_copy(uvm_object rhs); - int8mul_cmd_item rhs_; - super.do_copy(rhs); - if (!$cast(rhs_, rhs)) begin - `uvm_fatal("COPY", "rhs is not int8mul_cmd_item") - end - gate_row = rhs_.gate_row; - bank_entries = rhs_.bank_entries; - num_gate_words = rhs_.num_gate_words; - num_input_words = rhs_.num_input_words; - num_dst_words = rhs_.num_dst_words; - for (int i = 0; i < INT8MUL_MAX_ROWS; i++) gate_words[i] = rhs_.gate_words[i]; - for (int i = 0; i < INT8MUL_MAX_ROWS; i++) input_words[i] = rhs_.input_words[i]; - for (int i = 0; i < INT8MUL_MAX_ROWS; i++) dst_words[i] = rhs_.dst_words[i]; - endfunction -endclass diff --git a/examples/balls/int8mul/verify/src/env/int8mul_scoreboard.svh b/examples/balls/int8mul/verify/src/env/int8mul_scoreboard.svh deleted file mode 100644 index dedac48d8..000000000 --- a/examples/balls/int8mul/verify/src/env/int8mul_scoreboard.svh +++ /dev/null @@ -1,193 +0,0 @@ -class int8mul_scoreboard extends uvm_scoreboard; - `uvm_component_utils(int8mul_scoreboard) - - uvm_analysis_imp_stim #(bb_blink_cmd_item, int8mul_scoreboard) stim_imp; - uvm_analysis_imp_cmd #(bb_blink_cmd_item, int8mul_scoreboard) cmd_imp; - uvm_analysis_imp_read #(bb_blink_read_item, int8mul_scoreboard) read_imp; - uvm_analysis_imp_write #(bb_blink_write_item, int8mul_scoreboard) write_imp; - uvm_analysis_imp_resp #(bb_blink_resp_item, int8mul_scoreboard) resp_imp; - - bb_blink_mem_model #(`BB_IN_BW, `BB_OUT_BW) mem_model; - - int8mul_cmd_item stim_q[$]; - int unsigned expected_reads[`BB_IN_BW]; - int unsigned expected_writes; - int unsigned cmd_count; - int unsigned read_count[`BB_IN_BW]; - int unsigned write_count; - int unsigned resp_count; - - function new(string name, uvm_component parent); - super.new(name, parent); - stim_imp = new("stim_imp", this); - cmd_imp = new("cmd_imp", this); - read_imp = new("read_imp", this); - write_imp = new("write_imp", this); - resp_imp = new("resp_imp", this); - endfunction - - function void write_stim(bb_blink_cmd_item item); - int8mul_cmd_item got; - int8mul_cmd_item clone; - int i; - - if (stim_q.size() != 0) begin - `uvm_fatal("SCB", "single outstanding command supported") - end - if (!$cast(got, item)) begin - `uvm_fatal("SCB", "stim item is not int8mul_cmd_item") - end - if (!$cast(clone, got.clone())) begin - `uvm_fatal("SCB", "failed to clone stimulus item") - end - stim_q.push_back(clone); - - if (mem_model == null) begin - `uvm_fatal("SCB", "mem_model handle not set") - end - mem_model.clear_mem(); - for (i = 0; i < clone.num_gate_words; i++) begin - mem_model.load_word(int'(clone.op1_bank), int'(clone.gate_row) + i, clone.gate_words[i]); - end - for (i = 0; i < clone.num_input_words; i++) begin - mem_model.load_word(int'(clone.op2_bank), i, clone.input_words[i]); - end - mem_model.arm(); - - expected_writes = clone.num_dst_words; - expected_reads[0] = 1; - expected_reads[1] = clone.iter; - endfunction - - function void write_cmd(bb_blink_cmd_item item); - int8mul_cmd_item stim; - stim = current_stim("CMD"); - check_cmd(item, stim); - cmd_count++; - endfunction - - function void check_cmd(bb_blink_cmd_item got, int8mul_cmd_item exp); - if (got.bid !== exp.bid) - `uvm_fatal("CMD", $sformatf("bid mismatch: got %0d exp %0d", got.bid, exp.bid)) - if (got.funct7 !== exp.funct7) - `uvm_fatal("CMD", $sformatf("funct7 mismatch: got %0d exp %0d", got.funct7, exp.funct7)) - if (got.iter !== exp.iter) - `uvm_fatal("CMD", $sformatf("iter mismatch: got %0d exp %0d", got.iter, exp.iter)) - if (got.op1_bank !== exp.op1_bank || got.op2_bank !== exp.op2_bank || - got.wr_bank !== exp.wr_bank) - `uvm_fatal("CMD", "bank field mismatch") - if (got.op1_col !== exp.op1_col || got.op2_col !== exp.op2_col || got.wr_col !== exp.wr_col) - `uvm_fatal("CMD", "column field mismatch") - if (got.rs1 !== exp.rs1 || got.rs2 !== exp.rs2) `uvm_fatal("CMD", "rs field mismatch") - if (got.rob_id !== exp.rob_id) - `uvm_fatal("CMD", $sformatf("rob_id mismatch: got %0d exp %0d", got.rob_id, exp.rob_id)) - if ((got.rs2 >> (32 + $clog2(exp.bank_entries))) != 0) - `uvm_fatal("CMD", $sformatf("reserved rs2 bits set: 0x%016h", got.rs2)) - endfunction - - function void write_read(bb_blink_read_item item); - int8mul_cmd_item stim; - int unsigned expect_bank; - int unsigned expect_addr; - - stim = current_stim("READ"); - if (item.port != 0 && item.port != 1) - `uvm_fatal("READ", $sformatf("Int8MulBall has two read ports, got port %0d", item.port)) - if (item.group_id !== 5'd0) - `uvm_fatal("READ", $sformatf("group mismatch: got %0d exp 0", item.group_id)) - if (item.rob_id !== stim.rob_id) - `uvm_fatal("READ", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - - if (item.port == 0) begin - expect_bank = stim.op1_bank; - expect_addr = stim.gate_row; - if (read_count[0] != 0) `uvm_fatal("READ", "Int8MulBall gate bank read more than once") - end else begin - expect_bank = stim.op2_bank; - expect_addr = read_count[1]; - end - if (item.bank_id !== expect_bank[4:0]) - `uvm_fatal("READ", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, expect_bank)) - if (item.addr !== expect_addr[BB_BLINK_BANK_ADDR_W-1:0]) - `uvm_fatal("READ", $sformatf("addr mismatch: got %0d exp %0d", item.addr, expect_addr)) - read_count[item.port]++; - endfunction - - function void write_write(bb_blink_write_item item); - int8mul_cmd_item stim; - - stim = current_stim("WRITE"); - if (item.port != 0) - `uvm_fatal("WRITE", $sformatf("Int8MulBall has one write port, got port %0d", item.port)) - if (item.bank_id !== stim.wr_bank) - `uvm_fatal("WRITE", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.wr_bank)) - if (item.rob_id !== stim.rob_id) - `uvm_fatal("WRITE", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.group_id !== 5'd0) - `uvm_fatal("WRITE", $sformatf("group mismatch: got %0d exp 0", item.group_id)) - if (item.mask !== 16'hFFFF) - `uvm_fatal("WRITE", $sformatf("mask mismatch: got 0x%04h", item.mask)) - if (write_count >= stim.num_dst_words) - `uvm_fatal("WRITE", $sformatf("extra write %0d", write_count)) - if (item.addr !== write_count[BB_BLINK_BANK_ADDR_W-1:0]) - `uvm_fatal("WRITE", $sformatf("addr mismatch: got %0d exp %0d", item.addr, write_count)) - if (item.data !== stim.dst_words[write_count]) - `uvm_fatal("SCB", $sformatf( - "data mismatch at addr %0d: got 0x%032h exp 0x%032h", - item.addr, - item.data, - stim.dst_words[write_count] - )) - write_count++; - endfunction - - function void write_resp(bb_blink_resp_item item); - int8mul_cmd_item stim; - - stim = current_stim("RESP"); - if (item.rob_id !== stim.rob_id) - `uvm_fatal("RESP", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.is_sub !== 1'b0) `uvm_fatal("RESP", "is_sub should be 0") - if (item.sub_rob_id !== 8'h00) - `uvm_fatal("RESP", $sformatf("sub_rob_id mismatch: got 0x%0h", item.sub_rob_id)) - resp_count++; - endfunction - - function int8mul_cmd_item current_stim(string tag); - if (stim_q.size() == 0) begin - `uvm_fatal("SCB", $sformatf("%s observed before stimulus", tag)) - end - return stim_q[0]; - endfunction - - function bit done(); - if (cmd_count != 1 || write_count != expected_writes || resp_count != 1) return 1'b0; - for (int p = 0; p < `BB_IN_BW; p++) begin - if (read_count[p] != expected_reads[p]) return 1'b0; - end - return 1'b1; - endfunction - - function void reset_counters(); - stim_q.delete(); - cmd_count = 0; - write_count = 0; - resp_count = 0; - expected_writes = 0; - for (int p = 0; p < `BB_IN_BW; p++) begin - read_count[p] = 0; - expected_reads[p] = 0; - end - endfunction - - function void check_phase(uvm_phase phase); - super.check_phase(phase); - if (!done()) begin - `uvm_fatal("SCB", $sformatf( - "incomplete: cmds=%0d reads=%0d/%0d,%0d/%0d writes=%0d/%0d resp=%0d", - cmd_count, read_count[0], expected_reads[0], read_count[1], - expected_reads[1], write_count, expected_writes, resp_count)) - end - stim_q.delete(); - endfunction -endclass diff --git a/examples/balls/int8mul/verify/src/tb_top.sv b/examples/balls/int8mul/verify/src/tb_top.sv deleted file mode 100644 index 385256808..000000000 --- a/examples/balls/int8mul/verify/src/tb_top.sv +++ /dev/null @@ -1,101 +0,0 @@ -module tb_top; - import uvm_pkg::*; - import int8mul_pkg::*; - - bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); - - Int8MulBall dut ( - .clock(intf.clock), - .reset(intf.reset), - .io_status_idle(), - .io_status_running(), - .io_channelReady(1'b1), - .io_cmdReq_ready(intf.cmd_req_ready), - .io_cmdReq_valid(intf.cmd_req_valid), - .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), - .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), - .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), - .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), - .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), - .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), - .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), - .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), - .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), - .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank), - .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank), - .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank), - .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), - .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), - .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), - .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank), - .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), - .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), - .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), - .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), - .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), - .io_cmdResp_ready(intf.cmd_resp_ready), - .io_cmdResp_valid(intf.cmd_resp_valid), - .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), - .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), - .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), - .io_bankRead_0_bank_id(intf.bank_read_bank_id[0]), - .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), - .io_bankRead_0_ball_id(), - .io_bankRead_0_group_id(intf.bank_read_group_id[0]), - .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), - .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), - .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), - .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), - .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), - .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), - .io_bankRead_1_bank_id(intf.bank_read_bank_id[1]), - .io_bankRead_1_rob_id(intf.bank_read_rob_id[1]), - .io_bankRead_1_ball_id(), - .io_bankRead_1_group_id(intf.bank_read_group_id[1]), - .io_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), - .io_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), - .io_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), - .io_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), - .io_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), - .io_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), - .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), - .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), - .io_bankWrite_0_ball_id(), - .io_bankWrite_0_group_id(intf.bank_write_group_id[0]), - .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), - .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), - .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), - .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), - .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), - .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), - .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), - .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), - .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), - .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), - .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), - .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), - .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), - .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), - .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), - .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), - .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), - .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), - .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), - .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), - .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), - .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), - .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), - .io_subRobReq_ready(intf.sub_rob_req_ready) - ); - - initial begin - intf.clock = 1'b0; - intf.reset = 1'b1; - forever #5 intf.clock = ~intf.clock; - end - - initial begin - uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); - run_test("int8mul_ball_test"); - end -endmodule diff --git a/examples/balls/int8mul/workloads/ctests/CMakeLists.txt b/examples/balls/int8mul/workloads/ctests/CMakeLists.txt deleted file mode 100644 index 627f04523..000000000 --- a/examples/balls/int8mul/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -add_buckyball_ctests( - int8mul_per_channel_gate_test.c - int8mul_large_gate_row_test.c -) diff --git a/examples/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate.mlir b/examples/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate.mlir deleted file mode 100644 index d00b72e90..000000000 --- a/examples/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate.mlir +++ /dev/null @@ -1,41 +0,0 @@ -func.func private @check_result(memref<4x16xi8>) -> () -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %one_i8 = arith.constant 1 : i8 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %four = arith.constant 4 : index - %sixteen = arith.constant 16 : index - %depth4 = arith.constant 4 : i64 - %depth1 = arith.constant 1 : i64 - %stride = arith.constant 1 : i64 - %ratio = arith.constant 1.0 : f32 - %gate_row = arith.constant 0 : i64 - %gate = memref.alloc() : memref<1x16xi8> - %input = memref.alloc() : memref<4x16xi8> - %output = memref.alloc() : memref<4x16xi8> - scf.for %lane = %zero to %sixteen step %one { - memref.store %one_i8, %gate[%zero, %lane] : memref<1x16xi8> - } - scf.for %row = %zero to %four step %one { - scf.for %lane = %zero to %sixteen step %one { - %value = arith.index_cast %lane : index to i8 - memref.store %value, %input[%row, %lane] : memref<4x16xi8> - } - } - %gate_bank = buckyball.bank_alloc - %input_bank = buckyball.bank_alloc - %output_bank = buckyball.bank_alloc - %loaded_gate = buckyball.bank_mvin %gate %gate_bank %depth1 %stride : memref<1x16xi8> i64 i64 i64 - %loaded_input = buckyball.bank_mvin %input %input_bank %depth4 %stride : memref<4x16xi8> i64 i64 i64 - buckyball.int8mul %loaded_gate, %loaded_input, %output_bank, %depth4, %ratio, %gate_row : i64 - %stored = buckyball.bank_mvout %output %output_bank %depth4 %stride : memref<4x16xi8> i64 i64 i64 - func.call @check_result(%output) : (memref<4x16xi8>) -> () - buckyball.bank_release %loaded_gate : i64 - buckyball.bank_release %loaded_input : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %gate : memref<1x16xi8> - memref.dealloc %input : memref<4x16xi8> - memref.dealloc %output : memref<4x16xi8> - return %zero_i8 : i8 -} diff --git a/examples/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate.mlir.in b/examples/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate.mlir.in deleted file mode 100644 index 8310fb5a6..000000000 --- a/examples/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate.mlir.in +++ /dev/null @@ -1,41 +0,0 @@ -func.func private @check_result(memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %one_i8 = arith.constant 1 : i8 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %rows = arith.constant @BB_BANK_LINES@ : index - %lanes = arith.constant @BB_BANK_I8_LANES@ : index - %depth = arith.constant @BB_BANK_LINES@ : i64 - %depth1 = arith.constant 1 : i64 - %stride = arith.constant 1 : i64 - %ratio = arith.constant 1.0 : f32 - %gate_row = arith.constant 0 : i64 - %gate = memref.alloc() : memref<1x@BB_BANK_I8_LANES@xi8> - %input = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - %output = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - scf.for %lane = %zero to %lanes step %one { - memref.store %one_i8, %gate[%zero, %lane] : memref<1x@BB_BANK_I8_LANES@xi8> - } - scf.for %row = %zero to %rows step %one { - scf.for %lane = %zero to %lanes step %one { - %value = arith.index_cast %lane : index to i8 - memref.store %value, %input[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - } - } - %gate_bank = buckyball.bank_alloc - %input_bank = buckyball.bank_alloc - %output_bank = buckyball.bank_alloc - %loaded_gate = buckyball.bank_mvin %gate %gate_bank %depth1 %stride : memref<1x@BB_BANK_I8_LANES@xi8> i64 i64 i64 - %loaded_input = buckyball.bank_mvin %input %input_bank %depth %stride : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> i64 i64 i64 - %result = buckyball.bank_int8mul %loaded_gate %loaded_input %output_bank %depth %ratio %gate_row : i64 i64 i64 i64 f32 i64 - %stored = buckyball.bank_mvout %output %result %depth %stride : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> i64 i64 i64 - func.call @check_result(%output) : (memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () - buckyball.bank_release %loaded_gate : i64 - buckyball.bank_release %loaded_input : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %gate : memref<1x@BB_BANK_I8_LANES@xi8> - memref.dealloc %input : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - memref.dealloc %output : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - return %zero_i8 : i8 -} diff --git a/examples/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate_main.cpp b/examples/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate_main.cpp deleted file mode 100644 index 805bb757d..000000000 --- a/examples/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate_main.cpp +++ /dev/null @@ -1,18 +0,0 @@ -#include -#include -#include -#include -extern "C" void check_result(int8_t *, int8_t *data, int64_t offset, - int64_t rows, int64_t lanes, int64_t stride, - int64_t lane_stride) { - if (rows != BANK_LINES || lanes != BANK_WIDTH / 8 || stride != lanes || - lane_stride != 1) - exit(1); - for (int row = 0; row < rows; ++row) - for (int lane = 0; lane < lanes; ++lane) - if (data[offset + row * stride + lane] != lane) { - printf("int8mul bank mismatch row=%d lane=%d\n", row, lane); - exit(1); - } - printf("int8mul bank PASS rows=%d\n", BANK_LINES); -} diff --git a/examples/balls/lut/arch/src/main/scala/LutBall.scala b/examples/balls/lut/arch/src/main/scala/LutBall.scala deleted file mode 100644 index aa7b395a4..000000000 --- a/examples/balls/lut/arch/src/main/scala/LutBall.scala +++ /dev/null @@ -1,165 +0,0 @@ -package examples.balls.lut - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public} -import chisel3.util._ -import framework.balldomain.blink.{BallStatus, BlinkIO, HasBallStatus, HasBlink, SubRobRow} -import framework.balldomain.blink.mmio.{MmioRead, MmioWrite} -import framework.top.GlobalConfig - -@instantiable -class LutBall(val b: GlobalConfig) extends Module with HasBlink with HasBallStatus { - - private val mapping = b.ballDomain.ballIdMappings - .find(_.ballName == "LutBall") - .getOrElse(throw new IllegalArgumentException("LutBall not found in config")) - - private val funct = b.ballDomain.ballISA - .find(_.mnemonic == "LUT") - .map(_.funct7) - .getOrElse(throw new IllegalArgumentException("LUT not found in ballISA")) - - require(b.memDomain.bankWidth == 128, "LutBall requires 128-bit bank rows") - require(b.memDomain.bankEntries >= 16, "LutBall requires sixteen LUT rows") - require(mapping.inBW == 2, "LutBall requires inBW=2") - require(mapping.outBW == 1, "LutBall requires outBW=1") - require((funct >> 4) == 4, "LUT must encode two reads and one write") - - @public - val io = IO(new BlinkIO(b, mapping.inBW, mapping.outBW)) - def blink: BlinkIO = io - def status: BallStatus = io.status - dontTouch(io) - - private val idle :: inputReq :: inputResp :: lutReq :: lutResp :: writeReq :: writeResp :: complete :: Nil = - Enum(8) - private val state = RegInit(idle) - private val robId = RegInit(0.U(log2Up(b.frontend.rob_entries).W)) - private val isSub = RegInit(false.B) - private val subRobId = RegInit(0.U(log2Up(b.frontend.sub_rob_depth * 4).W)) - private val inputBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - private val lutBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - private val outputBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - private val iter = RegInit(0.U(b.frontend.iter_len.W)) - private val inputRow = RegInit(0.U(log2Ceil(b.memDomain.bankEntries).W)) - private val inputData = Reg(UInt(128.W)) - private val laneTable = RegInit(false.B) - private val lookupLane = RegInit(0.U(4.W)) - private val outputWord = Reg(Vec(16, UInt(8.W))) - - io.cmdReq.ready := state === idle - io.cmdResp.valid := state === complete - io.cmdResp.bits.rob_id := robId - io.cmdResp.bits.is_sub := isSub - io.cmdResp.bits.sub_rob_id := subRobId - io.status.idle := state === idle - io.status.running := state =/= idle && state =/= complete - - for (port <- 0 until 2) { - io.bankRead(port).rob_id := robId - io.bankRead(port).ball_id := 0.U - io.bankRead(port).group_id := 0.U - io.bankRead(port).io.req.valid := false.B - io.bankRead(port).io.req.bits.addr := 0.U - io.bankRead(port).io.resp.ready := false.B - } - io.bankRead(0).bank_id := inputBank - io.bankRead(1).bank_id := lutBank - io.bankRead(1).group_id := Mux(laneTable, lookupLane >> 2, 0.U) - - io.bankWrite(0).rob_id := robId - io.bankWrite(0).ball_id := 0.U - io.bankWrite(0).bank_id := outputBank - io.bankWrite(0).group_id := 0.U - io.bankWrite(0).io.req.valid := false.B - io.bankWrite(0).io.req.bits.addr := inputRow - io.bankWrite(0).io.req.bits.data := Cat(outputWord.reverse) - io.bankWrite(0).io.req.bits.mask := VecInit(Seq.fill(b.memDomain.bankMaskLen)(true.B)) - io.bankWrite(0).io.resp.ready := false.B - - io.subRobReq.valid := false.B - io.subRobReq.bits := SubRobRow.tieOff(b) - MmioRead.tieOff(io.mmioRead) - MmioWrite.tieOff(io.mmioWrite) - - switch(state) { - is(idle) { - when(io.cmdReq.fire) { - val cmd = io.cmdReq.bits.cmd - assert(cmd.funct7 === funct.U, "LutBall funct7 must be LUT") - assert(cmd.rs2 === 0.U, "LutBall reserves rs2") - assert(cmd.iter > 0.U && cmd.iter <= b.memDomain.bankEntries.U, "LutBall iter must fit in one bank") - assert(cmd.op1_bank < b.memDomain.virtualBankCount.U, "LutBall input bank is invalid") - assert(cmd.op2_bank < b.memDomain.virtualBankCount.U, "LutBall table bank is invalid") - assert(cmd.wr_bank < b.memDomain.virtualBankCount.U, "LutBall output bank is invalid") - assert( - cmd.op1_col === 1.U && (cmd.op2_col === 1.U || cmd.op2_col === 4.U) && cmd.wr_col === 1.U, - "LutBall requires col=1 input/output and col=1 or col=4 table" - ) - assert( - cmd.op1_bank =/= cmd.op2_bank && cmd.op1_bank =/= cmd.wr_bank && - cmd.op2_bank =/= cmd.wr_bank, - "LutBall banks must be distinct" - ) - robId := io.cmdReq.bits.rob_id - isSub := io.cmdReq.bits.is_sub - subRobId := io.cmdReq.bits.sub_rob_id - inputBank := cmd.op1_bank - lutBank := cmd.op2_bank - outputBank := cmd.wr_bank - iter := cmd.iter - inputRow := 0.U - laneTable := cmd.op2_col === 4.U - lookupLane := 0.U - state := inputReq - } - } - is(inputReq) { - io.bankRead(0).io.req.valid := true.B - io.bankRead(0).io.req.bits.addr := inputRow - when(io.bankRead(0).io.req.fire)(state := inputResp) - } - is(inputResp) { - io.bankRead(0).io.resp.ready := true.B - when(io.bankRead(0).io.resp.fire) { - inputData := io.bankRead(0).io.resp.bits.data - lookupLane := 0.U - state := lutReq - } - } - is(lutReq) { - val inputByte = (inputData >> (lookupLane << 3))(7, 0) - io.bankRead(1).io.req.valid := true.B - io.bankRead(1).io.req.bits.addr := Mux( - laneTable, - Cat(lookupLane(1, 0), inputByte(7, 4)), - inputByte(7, 4) - ) - when(io.bankRead(1).io.req.fire)(state := lutResp) - } - is(lutResp) { - val inputByte = (inputData >> (lookupLane << 3))(7, 0) - io.bankRead(1).io.resp.ready := true.B - when(io.bankRead(1).io.resp.fire) { - outputWord(lookupLane) := - (io.bankRead(1).io.resp.bits.data >> (inputByte(3, 0) << 3))(7, 0) - when(lookupLane === 15.U)(state := writeReq) - .otherwise { lookupLane := lookupLane + 1.U; state := lutReq } - } - } - is(writeReq) { - io.bankWrite(0).io.req.valid := true.B - when(io.bankWrite(0).io.req.fire)(state := writeResp) - } - is(writeResp) { - io.bankWrite(0).io.resp.ready := true.B - when(io.bankWrite(0).io.resp.fire) { - when(inputRow === iter - 1.U)(state := complete) - .otherwise { inputRow := inputRow + 1.U; state := inputReq } - } - } - is(complete) { - when(io.cmdResp.fire)(state := idle) - } - } -} diff --git a/examples/balls/lut/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp b/examples/balls/lut/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp deleted file mode 100644 index 818bc823c..000000000 --- a/examples/balls/lut/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp +++ /dev/null @@ -1,54 +0,0 @@ -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace mlir::buddy; -using namespace ::buddy::buckyball; - -namespace mlir::buddy { -void populateLutBallAssignPhysicalBankPatterns(RewritePatternSet &patterns, - PhysicalBankState &state); -} // namespace mlir::buddy - -namespace { -class BankLutPattern : public OpRewritePattern { -public: - BankLutPattern(MLIRContext *context, PhysicalBankState &state) - : OpRewritePattern(context), state(state) {} - - LogicalResult matchAndRewrite(BankLutOp op, - PatternRewriter &rewriter) const override { - auto input = state.getSlot(op.getInBank()); - auto lut = state.getSlot(op.getLutBank()); - auto output = state.getSlot(op.getOutBank()); - if (!input || !lut || !output) - return failure(); - if (input->row != 1 || input->col != 1 || output->row != 1 || - output->col != 1 || lut->row != 1 || (lut->col != 1 && lut->col != 4)) - return op.emitError( - "LUT requires col=1 input/output and col=1 or col=4 table"); - auto overlaps = [](const BankSlot &lhs, const BankSlot &rhs) { - return lhs.base < rhs.base + rhs.row * rhs.col && - rhs.base < lhs.base + lhs.row * lhs.col; - }; - if (overlaps(*input, *lut) || overlaps(*input, *output) || - overlaps(*lut, *output)) - return op.emitError("LUT bank groups must not overlap"); - rewriter.create(op.getLoc(), op.getInBank(), op.getLutBank(), - op.getOutBank(), op.getIter()); - rewriter.replaceOp(op, op.getOutBank()); - return success(); - } - -private: - PhysicalBankState &state; -}; -} // namespace - -void mlir::buddy::populateLutBallAssignPhysicalBankPatterns( - RewritePatternSet &patterns, PhysicalBankState &state) { - patterns.add(patterns.getContext(), state); -} diff --git a/examples/balls/lut/compiler/src/Dialect/Buckyball/LutBall.td b/examples/balls/lut/compiler/src/Dialect/Buckyball/LutBall.td deleted file mode 100644 index 386b75259..000000000 --- a/examples/balls/lut/compiler/src/Dialect/Buckyball/LutBall.td +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef LUTBALL_OPS -#define LUTBALL_OPS - -include "Buckyball.td" - -def LutOp : Buckyball_Op<"lut"> { - let summary = "Apply a shared or 16-lane INT8 lookup table"; - let arguments = (ins I64:$inputBankId, I64:$lutBankId, - I64:$outputBankId, I64:$iter); - let assemblyFormat = "$inputBankId `,` $lutBankId `,` $outputBankId `,` " - "$iter attr-dict `:` type($inputBankId)"; -} - -def BankLutOp : Buckyball_Op<"bank_lut"> { - let summary = "Apply a shared or 16-lane INT8 lookup table with virtual banks"; - let arguments = (ins I64:$inBank, I64:$lutBank, I64:$outBank, I64:$iter); - let results = (outs I64:$outBankOut); - let assemblyFormat = "$inBank $lutBank $outBank $iter attr-dict `:` " - "type($inBank) type($lutBank) type($outBank) type($iter)"; -} - -#endif // LUTBALL_OPS diff --git a/examples/balls/lut/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/balls/lut/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index d45794738..000000000 --- a/examples/balls/lut/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,60 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace buddy::buckyball; -using namespace buddy::buckyball::legalize; - -namespace { -struct LutLowering : public ConvertOpToLLVMPattern { - LutLowering(LLVMTypeConverter &converter, int64_t bankDepth) - : ConvertOpToLLVMPattern(converter), bankDepth(bankDepth) {} - - LogicalResult - matchAndRewrite(LutOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("LutBall"); - auto iterOp = op.getIter().getDefiningOp(); - auto iterAttr = - iterOp ? dyn_cast(iterOp.getValue()) : IntegerAttr(); - if (!iterAttr || iterAttr.getInt() <= 0 || iterAttr.getInt() > bankDepth) - return op.emitError( - "LUT iter must be a positive constant within one bank"); - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInputBankId(), - adaptor.getLutBankId(), - adaptor.getOutputBankId(), adaptor.getIter()); - rewriter.replaceOpWithNewOp( - op, rs1, cstI64(rewriter, loc, 0), - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("LUT"))); - return success(); - } - -private: - int64_t bankDepth; -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateLutBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &converter, - RewritePatternSet &patterns, - bool stable, - int64_t bankDepth) { - (void)stable; - - patterns.add(converter, bankDepth); -} - -void configureLutBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable) { - (void)stable; - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/examples/balls/lut/configs/default.toml b/examples/balls/lut/configs/default.toml deleted file mode 100644 index 289d8f7e3..000000000 --- a/examples/balls/lut/configs/default.toml +++ /dev/null @@ -1,3 +0,0 @@ -# LutBall has no private configuration parameters. - -[ball] diff --git a/examples/balls/lut/emu/src/dispatch.rs b/examples/balls/lut/emu/src/dispatch.rs deleted file mode 100644 index 4f2d604b7..000000000 --- a/examples/balls/lut/emu/src/dispatch.rs +++ /dev/null @@ -1,36 +0,0 @@ -use crate::inst::instruction::{BallInstruction, ExecContext}; - -use super::lut; - -const BALL_CLASS: &str = "examples.balls.lut.LutBall"; - -pub fn execute_known( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, - ctx: &mut ExecContext, -) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("LUT") => Some(lut::Lut::exec(xs1, xs2, ctx)), - Some(_) | None => None, - } -} - -pub fn cycles_after_issue( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, -) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("LUT") => Some(lut::Lut::latency(xs1, xs2)), - Some(_) | None => None, - } -} diff --git a/examples/balls/lut/emu/src/lib.rs b/examples/balls/lut/emu/src/lib.rs deleted file mode 100644 index c774116e0..000000000 --- a/examples/balls/lut/emu/src/lib.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub(crate) use crate::inst::{decode, instruction}; - -mod lut; - -mod dispatch; - -pub use dispatch::{cycles_after_issue, execute_known}; diff --git a/examples/balls/lut/verify/casegen/src/casegen.rs b/examples/balls/lut/verify/casegen/src/casegen.rs deleted file mode 100644 index e6c9426db..000000000 --- a/examples/balls/lut/verify/casegen/src/casegen.rs +++ /dev/null @@ -1,353 +0,0 @@ -pub const BYTES_PER_ROW: usize = 16; -pub const SHARED_TABLE_BYTES: usize = 256; -pub const LANE_TABLE_BYTES: usize = 16 * SHARED_TABLE_BYTES; -pub const MAX_SRC: usize = 8; -pub const MAX_LUT: usize = LANE_TABLE_BYTES / BYTES_PER_ROW; -pub const MAX_DST: usize = 8; -pub const NUM_CASES: u32 = 4; - -#[repr(C)] -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct LutCmd { - pub bid: u32, - pub iter: u32, - pub op1_bank: u32, - pub op2_bank: u32, - pub wr_bank: u32, - pub op1_col: u32, - pub op2_col: u32, - pub wr_col: u32, - pub rob_id: u32, - pub rs1_lo: u32, - pub rs1_hi: u32, - pub rs2_lo: u32, - pub rs2_hi: u32, - pub num_src_words: u32, - pub num_lut_words: u32, - pub num_dst_words: u32, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct LutCase { - pub cmd: LutCmd, - pub src_words: [u128; MAX_SRC], - pub lut_words: [u128; MAX_LUT], - pub dst_words: [u128; MAX_DST], -} - -impl LutCase { - pub fn src_lo(&self, index: usize) -> u64 { - self.src_words[index] as u64 - } - - pub fn src_hi(&self, index: usize) -> u64 { - (self.src_words[index] >> 64) as u64 - } - - pub fn lut_lo(&self, index: usize) -> u64 { - self.lut_words[index] as u64 - } - - pub fn lut_hi(&self, index: usize) -> u64 { - (self.lut_words[index] >> 64) as u64 - } - - pub fn dst_lo(&self, index: usize) -> u64 { - self.dst_words[index] as u64 - } - - pub fn dst_hi(&self, index: usize) -> u64 { - (self.dst_words[index] >> 64) as u64 - } -} - -pub fn gen_case(index: u32, bid: u32) -> LutCase { - if index >= NUM_CASES { - panic!("lut: unsupported directed case {index}"); - } - match index { - 0 => directed(bid, 2, 4, false), - 1 => directed(bid, 3, 1, false), - 2 => directed(bid, 4, 8, false), - 3 => directed(bid, 5, 4, true), - _ => panic!("lut: unsupported directed case {index}"), - } -} - -fn shared_table() -> Vec { - let mut table = vec![0u8; SHARED_TABLE_BYTES]; - for (i, slot) in table.iter_mut().enumerate() { - *slot = (((i * 73 + 19) ^ 0xa5) & 0xff) as u8; - } - table -} - -fn lane_table() -> Vec { - let mut table = vec![0u8; LANE_TABLE_BYTES]; - for channel in 0..16 { - for input in 0..256 { - table[channel * 256 + input] = ((input as i32 + channel as i32 * 17 - 128) as i8) as u8; - } - } - table -} - -fn pack_lane_table(table: &[u8]) -> Vec { - if table.len() != LANE_TABLE_BYTES { - panic!("lut: invalid lane table size {}", table.len()); - } - let mut packed = vec![0u8; LANE_TABLE_BYTES]; - for row in 0..64 { - for group in 0..4 { - for byte in 0..16 { - packed[group * 64 * 16 + row * 16 + byte] = - table[(group * 4 + row / 16) * 256 + (row % 16) * 16 + byte]; - } - } - } - packed -} - -fn ctest_input(nbytes: usize) -> Vec { - (0..nbytes) - .map(|i| (i as i32 * 29 - 128) as i8 as u8) - .collect() -} - -fn pack_bytes(bytes: &[u8]) -> Vec { - if bytes.len() % BYTES_PER_ROW != 0 { - panic!( - "lut: {} bytes is not a multiple of {BYTES_PER_ROW}", - bytes.len() - ); - } - bytes - .chunks(BYTES_PER_ROW) - .map(|chunk| { - let mut word = 0u128; - for (lane, byte) in chunk.iter().enumerate() { - word |= u128::from(*byte) << (lane * 8); - } - word - }) - .collect() -} - -fn copy_words(src: &[u128]) -> [u128; N] { - if src.len() > N { - panic!("lut: {} words exceed buffer {N}", src.len()); - } - let mut out = [0u128; N]; - out[..src.len()].copy_from_slice(src); - out -} - -fn encode_rs1(op1: u32, op2: u32, wr: u32, iter: u32) -> u64 { - u64::from(op1) | (u64::from(op2) << 10) | (u64::from(wr) << 20) | (u64::from(iter) << 30) -} - -fn split_rs(value: u64) -> (u32, u32) { - (value as u32, (value >> 32) as u32) -} - -pub(crate) fn validate( - op1_bank: u32, - op2_bank: u32, - wr_bank: u32, - op1_col: u32, - op2_col: u32, - wr_col: u32, - iter: u32, - rs1: u64, - rs2: u64, -) { - if iter == 0 { - panic!("lut: iter must be positive"); - } - if op1_col != 1 || (op2_col != 1 && op2_col != 4) || wr_col != 1 { - panic!("lut: requires col=1 input/output and col=1 or col=4 table"); - } - if op1_bank == op2_bank || op1_bank == wr_bank || op2_bank == wr_bank { - panic!("lut: banks must be distinct"); - } - if (rs1 & 0x3ff) != u64::from(op1_bank) - || ((rs1 >> 10) & 0x3ff) != u64::from(op2_bank) - || ((rs1 >> 20) & 0x3ff) != u64::from(wr_bank) - { - panic!("lut: rs1 bank fields mismatch"); - } - if ((rs1 >> 30) & 0x3_ffff_ffff) != u64::from(iter) { - panic!("lut: rs1 iter mismatch"); - } - if rs2 != 0 { - panic!("lut: rs2 must be zero"); - } -} - -fn dst_words(table: &[u8], input: &[u8], lane_table: bool) -> Vec { - let mapped: Vec = input - .iter() - .enumerate() - .map(|(index, &byte)| { - let channel = index % 16; - table[(if lane_table { channel * 256 } else { 0 }) + byte as usize] - }) - .collect(); - pack_bytes(&mapped) -} - -fn directed(bid: u32, rob_id: u32, iter: u32, per_lane: bool) -> LutCase { - let nbytes = iter as usize * BYTES_PER_ROW; - let table = if per_lane { - lane_table() - } else { - shared_table() - }; - let input = ctest_input(nbytes); - let src = pack_bytes(&input); - let lut = if per_lane { - pack_bytes(&pack_lane_table(&table)) - } else { - pack_bytes(&table) - }; - let dst = dst_words(&table, &input, per_lane); - let op1_bank = 0; - let op2_bank = 1; - let wr_bank = 2; - let op1_col = 1; - let op2_col = if per_lane { 4 } else { 1 }; - let wr_col = 1; - let rs1 = encode_rs1(op1_bank, op2_bank, wr_bank, iter); - let rs2 = 0u64; - validate( - op1_bank, op2_bank, wr_bank, op1_col, op2_col, wr_col, iter, rs1, rs2, - ); - if src.len() != iter as usize { - panic!("lut: src rows {} != iter {iter}", src.len()); - } - let expected_lut_rows = if per_lane { MAX_LUT } else { 16 }; - if lut.len() != expected_lut_rows { - panic!("lut: table rows {} != {expected_lut_rows}", lut.len()); - } - if dst.len() != iter as usize { - panic!("lut: dst rows {} != iter {iter}", dst.len()); - } - let (rs1_lo, rs1_hi) = split_rs(rs1); - let (rs2_lo, rs2_hi) = split_rs(rs2); - LutCase { - cmd: LutCmd { - bid, - iter, - op1_bank, - op2_bank, - wr_bank, - op1_col, - op2_col, - wr_col, - rob_id, - rs1_lo, - rs1_hi, - rs2_lo, - rs2_hi, - num_src_words: src.len() as u32, - num_lut_words: lut.len() as u32, - num_dst_words: dst.len() as u32, - }, - src_words: copy_words(&src), - lut_words: copy_words(&lut), - dst_words: copy_words(&dst), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn case_zero_matches_ctest() { - let case = gen_case(0, 5); - assert_eq!(case.cmd.iter, 4); - assert_eq!(case.cmd.bid, 5); - assert_eq!(case.cmd.num_src_words, 4); - assert_eq!(case.cmd.num_lut_words, 16); - assert_eq!(case.cmd.num_dst_words, 4); - let table = shared_table(); - let input = ctest_input(64); - assert_eq!(table[0], ((19 ^ 0xa5) & 0xff) as u8); - assert_eq!(input[0], (-128i8) as u8); - let exp0 = table[input[0] as usize]; - assert_eq!(case.dst_words[0] as u8, exp0); - for i in 0..16 { - let got = ((case.src_words[0] >> (i * 8)) & 0xff) as u8; - assert_eq!(got, input[i]); - } - for i in 0..16 { - let got = ((case.lut_words[0] >> (i * 8)) & 0xff) as u8; - assert_eq!(got, table[i]); - } - } - - #[test] - fn cases_hit_iter_bins() { - let mut hit = [false; 3]; - for index in 0..NUM_CASES { - let case = gen_case(index, 5); - assert_eq!(case.cmd.bid, 5); - assert_eq!(case.cmd.op1_col, 1); - assert!(case.cmd.op2_col == 1 || case.cmd.op2_col == 4); - assert_eq!(case.cmd.wr_col, 1); - assert_eq!(case.cmd.rs2_lo, 0); - assert_eq!(case.cmd.rs2_hi, 0); - assert_eq!(case.cmd.num_src_words, case.cmd.iter); - assert_eq!(case.cmd.num_dst_words, case.cmd.iter); - assert_eq!( - case.cmd.num_lut_words, - if case.cmd.op2_col == 4 { 256 } else { 16 } - ); - match case.cmd.iter { - 1 => hit[0] = true, - 4 => hit[1] = true, - 8 => hit[2] = true, - other => panic!("unexpected iter {other}"), - } - } - assert!(hit.iter().all(|h| *h)); - } - - #[test] - fn lane_case_selects_channel_table() { - let case = gen_case(3, 5); - let input = ctest_input(64); - let table = lane_table(); - assert_eq!(case.cmd.op2_col, 4); - assert_eq!(case.cmd.num_lut_words, 256); - for lane in 0..16 { - let got = ((case.dst_words[0] >> (lane * 8)) & 0xff) as u8; - assert_eq!(got, table[lane * 256 + input[lane] as usize]); - } - } - - #[test] - #[should_panic(expected = "unsupported directed case")] - fn unknown_index_panics() { - let _ = gen_case(NUM_CASES, 5); - } - - #[test] - #[should_panic(expected = "iter must be positive")] - fn illegal_iter_panics() { - validate(0, 1, 2, 1, 1, 1, 0, encode_rs1(0, 1, 2, 0), 0); - } - - #[test] - #[should_panic(expected = "banks must be distinct")] - fn overlapping_banks_panics() { - validate(0, 0, 2, 1, 1, 1, 4, encode_rs1(0, 0, 2, 4), 0); - } - - #[test] - #[should_panic(expected = "rs2 must be zero")] - fn reserved_rs2_panics() { - validate(0, 1, 2, 1, 1, 1, 4, encode_rs1(0, 1, 2, 4), 1); - } -} diff --git a/examples/balls/lut/verify/casegen/src/lib.rs b/examples/balls/lut/verify/casegen/src/lib.rs deleted file mode 100644 index 257e15998..000000000 --- a/examples/balls/lut/verify/casegen/src/lib.rs +++ /dev/null @@ -1,115 +0,0 @@ -mod casegen; - -use std::cell::RefCell; - -use casegen::{LutCase, LutCmd}; - -thread_local! { - static CURRENT: RefCell> = const { RefCell::new(None) }; -} - -#[no_mangle] -pub extern "C" fn lut_case_load(index: u32, bid: u32) -> i32 { - let case = casegen::gen_case(index, bid); - CURRENT.with(|current| *current.borrow_mut() = Some(case)); - 0 -} - -fn current_case R, R>(f: F) -> R { - CURRENT.with(|current| match current.borrow().as_ref() { - Some(case) => f(case), - None => panic!("lut_case: no case loaded; call lut_case_load first"), - }) -} - -#[no_mangle] -pub extern "C" fn lut_case_cmd(out_ptr: *mut LutCmd) { - if out_ptr.is_null() { - panic!("lut_case_cmd: null out_ptr"); - } - current_case(|case| unsafe { - *out_ptr = case.cmd; - }); -} - -fn src_index(index: u32, limit: u32, what: &str) -> usize { - if index >= limit { - panic!("lut_case_{what}: index {index} out of range {limit}"); - } - index as usize -} - -#[no_mangle] -pub extern "C" fn lut_case_src_word_lo(word_index: u32) -> u64 { - current_case(|case| case.src_lo(src_index(word_index, case.cmd.num_src_words, "src_word_lo"))) -} - -#[no_mangle] -pub extern "C" fn lut_case_src_word_hi(word_index: u32) -> u64 { - current_case(|case| case.src_hi(src_index(word_index, case.cmd.num_src_words, "src_word_hi"))) -} - -#[no_mangle] -pub extern "C" fn lut_case_lut_word_lo(word_index: u32) -> u64 { - current_case(|case| case.lut_lo(src_index(word_index, case.cmd.num_lut_words, "lut_word_lo"))) -} - -#[no_mangle] -pub extern "C" fn lut_case_lut_word_hi(word_index: u32) -> u64 { - current_case(|case| case.lut_hi(src_index(word_index, case.cmd.num_lut_words, "lut_word_hi"))) -} - -#[no_mangle] -pub extern "C" fn lut_case_dst_word_lo(word_index: u32) -> u64 { - current_case(|case| case.dst_lo(src_index(word_index, case.cmd.num_dst_words, "dst_word_lo"))) -} - -#[no_mangle] -pub extern "C" fn lut_case_dst_word_hi(word_index: u32) -> u64 { - current_case(|case| case.dst_hi(src_index(word_index, case.cmd.num_dst_words, "dst_word_hi"))) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn dpi_load_then_cmd_and_words() { - assert_eq!(lut_case_load(0, 5), 0); - let mut cmd = LutCmd { - bid: 0, - iter: 0, - op1_bank: 0, - op2_bank: 0, - wr_bank: 0, - op1_col: 0, - op2_col: 0, - wr_col: 0, - rob_id: 0, - rs1_lo: 0, - rs1_hi: 0, - rs2_lo: 0, - rs2_hi: 0, - num_src_words: 0, - num_lut_words: 0, - num_dst_words: 0, - }; - lut_case_cmd(&mut cmd as *mut LutCmd); - assert_eq!(cmd.bid, 5); - assert_eq!(cmd.iter, 4); - assert_eq!(cmd.num_src_words, 4); - assert_eq!(cmd.num_lut_words, 16); - assert_eq!(cmd.num_dst_words, 4); - let _ = lut_case_src_word_lo(0); - let _ = lut_case_lut_word_lo(0); - let _ = lut_case_dst_word_lo(0); - assert_eq!(casegen::NUM_CASES, 4); - } - - #[test] - #[should_panic(expected = "no case loaded")] - fn current_case_panics_without_load() { - CURRENT.with(|c| *c.borrow_mut() = None); - current_case(|_| ()); - } -} diff --git a/examples/balls/lut/verify/src/common/lut_defs.svh b/examples/balls/lut/verify/src/common/lut_defs.svh deleted file mode 100644 index 90758d450..000000000 --- a/examples/balls/lut/verify/src/common/lut_defs.svh +++ /dev/null @@ -1,50 +0,0 @@ -typedef struct { - int unsigned bid; - int unsigned iter; - int unsigned op1_bank; - int unsigned op2_bank; - int unsigned wr_bank; - int unsigned op1_col; - int unsigned op2_col; - int unsigned wr_col; - int unsigned rob_id; - int unsigned rs1_lo; - int unsigned rs1_hi; - int unsigned rs2_lo; - int unsigned rs2_hi; - int unsigned num_src_words; - int unsigned num_lut_words; - int unsigned num_dst_words; -} lut_cmd_dpi_t; - -import "DPI-C" function int lut_case_load( - input int unsigned index, - input int unsigned bid -); -import "DPI-C" function void lut_case_cmd(output lut_cmd_dpi_t cmd); -import "DPI-C" function longint unsigned lut_case_src_word_lo(input int unsigned word_index); -import "DPI-C" function longint unsigned lut_case_src_word_hi(input int unsigned word_index); -import "DPI-C" function longint unsigned lut_case_lut_word_lo(input int unsigned word_index); -import "DPI-C" function longint unsigned lut_case_lut_word_hi(input int unsigned word_index); -import "DPI-C" function longint unsigned lut_case_dst_word_lo(input int unsigned word_index); -import "DPI-C" function longint unsigned lut_case_dst_word_hi(input int unsigned word_index); - -`ifndef LUT_FUNCT7 -`error "LUT_FUNCT7 must be provided by the selected Core ballISA" -`endif - -localparam int LUT_CORE_FUNCT7 = `LUT_FUNCT7; -localparam int LUT_MAX_SRC_WORDS = 8; -localparam int LUT_SHARED_TABLE_ROWS = 16; -localparam int LUT_LANE_TABLE_ROWS = 256; -localparam int LUT_MAX_DST_WORDS = 8; -localparam int LUT_NUM_CASES = 4; -localparam int LUT_TIMEOUT_CYCLES = 20000; - -function automatic int unsigned lut_require_bid(); - int unsigned bid; - if (!$value$plusargs("BID=%d", bid)) begin - `uvm_fatal("BID", "missing required plusarg +BID=") - end - return bid; -endfunction diff --git a/examples/balls/lut/verify/src/common/lut_items.svh b/examples/balls/lut/verify/src/common/lut_items.svh deleted file mode 100644 index 533ff69e9..000000000 --- a/examples/balls/lut/verify/src/common/lut_items.svh +++ /dev/null @@ -1,115 +0,0 @@ -class lut_cmd_item extends bb_blink_cmd_item; - `uvm_object_utils(lut_cmd_item) - - bit [31:0] num_src_words; - bit [31:0] num_lut_words; - bit [31:0] num_dst_words; - bit [127:0] src_words[LUT_MAX_SRC_WORDS]; - bit [127:0] lut_words[LUT_LANE_TABLE_ROWS]; - bit [127:0] dst_words[LUT_MAX_DST_WORDS]; - - constraint legal_c { - funct7 == LUT_CORE_FUNCT7[6:0]; - op1_en == 1'b1; - op2_en == 1'b1; - wr_spad_en == 1'b1; - op1_from_spad == 1'b1; - op2_from_spad == 1'b1; - op1_col == 5'd1; - op2_col inside {5'd1, 5'd4}; - wr_col == 5'd1; - meta_bank == 5'd0; - is_sub == 1'b0; - sub_rob_id == 8'h00; - special == 64'd0; - rs2 == 64'd0; - } - - function new(string name = "lut_cmd_item"); - super.new(name); - endfunction - - function void load_rust_case(int unsigned index, int unsigned bid); - lut_cmd_dpi_t cmd; - int unsigned rc; - longint unsigned w_lo; - longint unsigned w_hi; - int i; - - rc = lut_case_load(index, bid); - if (rc != 0) begin - `uvm_fatal("CASE", $sformatf("lut_case_load returned %0d for index %0d", rc, index)) - end - lut_case_cmd(cmd); - - this.bid = cmd.bid[4:0]; - iter = cmd.iter; - op1_bank = cmd.op1_bank[4:0]; - op2_bank = cmd.op2_bank[4:0]; - wr_bank = cmd.wr_bank[4:0]; - op1_col = cmd.op1_col[4:0]; - op2_col = cmd.op2_col[4:0]; - wr_col = cmd.wr_col[4:0]; - rob_id = cmd.rob_id[3:0]; - rs1 = {cmd.rs1_hi, cmd.rs1_lo}; - rs2 = {cmd.rs2_hi, cmd.rs2_lo}; - num_src_words = cmd.num_src_words; - num_lut_words = cmd.num_lut_words; - num_dst_words = cmd.num_dst_words; - funct7 = LUT_CORE_FUNCT7[6:0]; - special = 64'd0; - op1_en = 1'b1; - op2_en = 1'b1; - wr_spad_en = 1'b1; - op1_from_spad = 1'b1; - op2_from_spad = 1'b1; - meta_bank = 5'd0; - is_sub = 1'b0; - sub_rob_id = 8'h00; - - if (iter != 1 && iter != 4 && iter != 8) - `uvm_fatal("CASE", $sformatf("LUT iter %0d is not in {1,4,8}", iter)) - if (op1_col != 1 || (op2_col != 1 && op2_col != 4) || wr_col != 1) - `uvm_fatal("CASE", "LUT requires col=1 input/output and col=1 or col=4 table") - if (op1_bank == op2_bank || op1_bank == wr_bank || op2_bank == wr_bank) - `uvm_fatal("CASE", "LUT banks must be distinct") - if (rs2 != 64'd0) `uvm_fatal("CASE", "LUT rs2 must be zero") - if (num_src_words != iter) - `uvm_fatal("CASE", $sformatf("num_src_words %0d != iter %0d", num_src_words, iter)) - if (num_dst_words != iter) - `uvm_fatal("CASE", $sformatf("num_dst_words %0d != iter %0d", num_dst_words, iter)) - if (num_lut_words != (op2_col == 4 ? LUT_LANE_TABLE_ROWS : LUT_SHARED_TABLE_ROWS)) - `uvm_fatal("CASE", $sformatf( - "num_lut_words %0d does not match col %0d", num_lut_words, op2_col)) - - for (i = 0; i < num_src_words; i++) begin - w_lo = lut_case_src_word_lo(i); - w_hi = lut_case_src_word_hi(i); - src_words[i] = {w_hi, w_lo}; - end - for (i = 0; i < num_lut_words; i++) begin - w_lo = lut_case_lut_word_lo(i); - w_hi = lut_case_lut_word_hi(i); - lut_words[i] = {w_hi, w_lo}; - end - for (i = 0; i < num_dst_words; i++) begin - w_lo = lut_case_dst_word_lo(i); - w_hi = lut_case_dst_word_hi(i); - dst_words[i] = {w_hi, w_lo}; - end - endfunction - - function void do_copy(uvm_object rhs); - lut_cmd_item rhs_; - super.do_copy(rhs); - if (!$cast(rhs_, rhs)) begin - `uvm_fatal("COPY", "rhs is not lut_cmd_item") - end - num_src_words = rhs_.num_src_words; - num_lut_words = rhs_.num_lut_words; - num_dst_words = rhs_.num_dst_words; - for (int i = 0; i < LUT_MAX_SRC_WORDS; i++) src_words[i] = rhs_.src_words[i]; - for (int i = 0; i < LUT_LANE_TABLE_ROWS; i++) lut_words[i] = rhs_.lut_words[i]; - for (int i = 0; i < LUT_MAX_DST_WORDS; i++) dst_words[i] = rhs_.dst_words[i]; - endfunction -endclass diff --git a/examples/balls/lut/verify/src/tb_top.sv b/examples/balls/lut/verify/src/tb_top.sv deleted file mode 100644 index decec36b3..000000000 --- a/examples/balls/lut/verify/src/tb_top.sv +++ /dev/null @@ -1,101 +0,0 @@ -module tb_top; - import uvm_pkg::*; - import lut_pkg::*; - - bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); - - LutBall dut ( - .clock(intf.clock), - .reset(intf.reset), - .io_status_idle(), - .io_status_running(), - .io_channelReady(1'b1), - .io_cmdReq_ready(intf.cmd_req_ready), - .io_cmdReq_valid(intf.cmd_req_valid), - .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), - .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), - .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), - .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), - .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), - .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), - .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), - .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), - .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), - .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank), - .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank), - .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank), - .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), - .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), - .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), - .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank), - .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), - .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), - .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), - .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), - .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), - .io_cmdResp_ready(intf.cmd_resp_ready), - .io_cmdResp_valid(intf.cmd_resp_valid), - .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), - .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), - .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), - .io_bankRead_0_bank_id(intf.bank_read_bank_id[0]), - .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), - .io_bankRead_0_ball_id(), - .io_bankRead_0_group_id(intf.bank_read_group_id[0]), - .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), - .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), - .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), - .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), - .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), - .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), - .io_bankRead_1_bank_id(intf.bank_read_bank_id[1]), - .io_bankRead_1_rob_id(intf.bank_read_rob_id[1]), - .io_bankRead_1_ball_id(), - .io_bankRead_1_group_id(intf.bank_read_group_id[1]), - .io_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), - .io_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), - .io_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), - .io_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), - .io_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), - .io_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), - .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), - .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), - .io_bankWrite_0_ball_id(), - .io_bankWrite_0_group_id(intf.bank_write_group_id[0]), - .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), - .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), - .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), - .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), - .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), - .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), - .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), - .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), - .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), - .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), - .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), - .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), - .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), - .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), - .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), - .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), - .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), - .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), - .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), - .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), - .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), - .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), - .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), - .io_subRobReq_ready(intf.sub_rob_req_ready) - ); - - initial begin - intf.clock = 1'b0; - intf.reset = 1'b1; - forever #5 intf.clock = ~intf.clock; - end - - initial begin - uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); - run_test("lut_ball_test"); - end -endmodule diff --git a/examples/balls/lut/workloads/ctests/CMakeLists.txt b/examples/balls/lut/workloads/ctests/CMakeLists.txt deleted file mode 100644 index 9c489f467..000000000 --- a/examples/balls/lut/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_buckyball_ctests(lut_uniform_lookup_test.c lut_channel_table_lookup_test.c) diff --git a/examples/balls/lut/workloads/mlir_tests/ball/lut_identity.mlir b/examples/balls/lut/workloads/mlir_tests/ball/lut_identity.mlir deleted file mode 100644 index f5a1d4bd4..000000000 --- a/examples/balls/lut/workloads/mlir_tests/ball/lut_identity.mlir +++ /dev/null @@ -1,51 +0,0 @@ -func.func private @check_result(memref<4x16xi8>) -> () - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %four = arith.constant 4 : index - %sixteen = arith.constant 16 : index - %depth4 = arith.constant 4 : i64 - %depth16 = arith.constant 16 : i64 - %stride = arith.constant 1 : i64 - %sixteen_i32 = arith.constant 16 : i32 - %input = memref.alloc() : memref<4x16xi8> - %table = memref.alloc() : memref<16x16xi8> - %output = memref.alloc() : memref<4x16xi8> - scf.for %row = %zero to %sixteen step %one { - scf.for %lane = %zero to %sixteen step %one { - %row32 = arith.index_cast %row : index to i32 - %lane32 = arith.index_cast %lane : index to i32 - %base = arith.muli %row32, %sixteen_i32 : i32 - %index = arith.addi %base, %lane32 : i32 - %value = arith.trunci %index : i32 to i8 - memref.store %value, %table[%row, %lane] : memref<16x16xi8> - } - } - scf.for %row = %zero to %four step %one { - scf.for %lane = %zero to %sixteen step %one { - %row32 = arith.index_cast %row : index to i32 - %lane32 = arith.index_cast %lane : index to i32 - %base = arith.muli %row32, %sixteen_i32 : i32 - %index = arith.addi %base, %lane32 : i32 - %value = arith.trunci %index : i32 to i8 - memref.store %value, %input[%row, %lane] : memref<4x16xi8> - } - } - %input_bank = buckyball.bank_alloc - %table_bank = buckyball.bank_alloc - %output_bank = buckyball.bank_alloc - %loaded_input = buckyball.bank_mvin %input %input_bank %depth4 %stride : memref<4x16xi8> i64 i64 i64 - %loaded_table = buckyball.bank_mvin %table %table_bank %depth16 %stride : memref<16x16xi8> i64 i64 i64 - buckyball.lut %loaded_input, %loaded_table, %output_bank, %depth4 : i64 - %stored = buckyball.bank_mvout %output %output_bank %depth4 %stride : memref<4x16xi8> i64 i64 i64 - func.call @check_result(%output) : (memref<4x16xi8>) -> () - buckyball.bank_release %loaded_input : i64 - buckyball.bank_release %loaded_table : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %input : memref<4x16xi8> - memref.dealloc %table : memref<16x16xi8> - memref.dealloc %output : memref<4x16xi8> - return %zero_i8 : i8 -} diff --git a/examples/balls/lut/workloads/mlir_tests/bank/lut_identity_table.mlir.in b/examples/balls/lut/workloads/mlir_tests/bank/lut_identity_table.mlir.in deleted file mode 100644 index dc1c8a007..000000000 --- a/examples/balls/lut/workloads/mlir_tests/bank/lut_identity_table.mlir.in +++ /dev/null @@ -1,53 +0,0 @@ -func.func private @check_result(memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %rows = arith.constant @BB_BANK_LINES@ : index - %lanes = arith.constant @BB_BANK_I8_LANES@ : index - %sixteen = arith.constant 16 : index - %depth = arith.constant @BB_BANK_LINES@ : i64 - %table_depth = arith.constant 16 : i64 - %stride = arith.constant 1 : i64 - %lanes_i32 = arith.constant @BB_BANK_I8_LANES@ : i32 - %sixteen_i32 = arith.constant 16 : i32 - %input = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - %table = memref.alloc() : memref<16x16xi8> - %output = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - scf.for %row = %zero to %sixteen step %one { - scf.for %lane = %zero to %sixteen step %one { - %row32 = arith.index_cast %row : index to i32 - %lane32 = arith.index_cast %lane : index to i32 - %base = arith.muli %row32, %sixteen_i32 : i32 - %index = arith.addi %base, %lane32 : i32 - %value = arith.trunci %index : i32 to i8 - memref.store %value, %table[%row, %lane] : memref<16x16xi8> - } - } - scf.for %row = %zero to %rows step %one { - scf.for %lane = %zero to %lanes step %one { - %row32 = arith.index_cast %row : index to i32 - %lane32 = arith.index_cast %lane : index to i32 - %base = arith.muli %row32, %lanes_i32 : i32 - %index = arith.addi %base, %lane32 : i32 - %value = arith.trunci %index : i32 to i8 - memref.store %value, %input[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - } - } - %input_bank = buckyball.bank_alloc - %table_bank = buckyball.bank_alloc - %output_bank = buckyball.bank_alloc - %loaded_input = buckyball.bank_mvin %input %input_bank %depth %stride : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> i64 i64 i64 - %loaded_table = buckyball.bank_mvin %table %table_bank %table_depth %stride : memref<16x16xi8> i64 i64 i64 - %result = buckyball.bank_lut %loaded_input %loaded_table %output_bank %depth : i64 i64 i64 i64 - %stored = buckyball.bank_mvout %output %result %depth %stride : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> i64 i64 i64 - func.call @check_result(%output) : (memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () - buckyball.bank_release %loaded_input : i64 - buckyball.bank_release %loaded_table : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %input : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - memref.dealloc %table : memref<16x16xi8> - memref.dealloc %output : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - return %zero_i8 : i8 -} diff --git a/examples/balls/lut/workloads/mlir_tests/bank/lut_identity_table_main.cpp b/examples/balls/lut/workloads/mlir_tests/bank/lut_identity_table_main.cpp deleted file mode 100644 index b7c91f060..000000000 --- a/examples/balls/lut/workloads/mlir_tests/bank/lut_identity_table_main.cpp +++ /dev/null @@ -1,17 +0,0 @@ -#include -#include -#include -#include -extern "C" void check_result(int8_t *, int8_t *data, int64_t offset, - int64_t rows, int64_t lanes, int64_t stride, - int64_t lane_stride) { - if (rows != BANK_LINES || lanes != BANK_WIDTH / 8 || stride != lanes || - lane_stride != 1) - exit(1); - for (int i = 0; i < rows * lanes; ++i) - if (data[offset + i] != (int8_t)i) { - printf("lut bank mismatch index=%d\n", i); - exit(1); - } - printf("lut bank PASS rows=%d\n", BANK_LINES); -} diff --git a/examples/balls/matadd/arch/src/main/scala/MatAdd.scala b/examples/balls/matadd/arch/src/main/scala/MatAdd.scala deleted file mode 100644 index c6c0d678c..000000000 --- a/examples/balls/matadd/arch/src/main/scala/MatAdd.scala +++ /dev/null @@ -1,202 +0,0 @@ -package examples.balls.matadd - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public} -import chisel3.util._ -import framework.balldomain.blink.{BallStatus, BankRead, BankWrite} -import framework.balldomain.rs.{BallRsComplete, BallRsIssue} -import framework.top.GlobalConfig - -@instantiable -class MatAdd(val b: GlobalConfig) extends Module { - private val addressWidth = log2Up(b.memDomain.bankEntries) - private val countWidth = log2Up(b.memDomain.bankEntries + 1) - private val bankIdWidth = b.memDomain.vbankIdWidth - - private val mapping = b.ballDomain.ballIdMappings - .find(_.ballName == "MatAddBall") - .getOrElse(throw new IllegalArgumentException("MatAddBall not found in config")) - - private val funct = b.ballDomain.ballISA - .find(_.mnemonic == "MATADD") - .map(_.funct7) - .getOrElse(throw new IllegalArgumentException("MATADD not found in ballISA")) - - require(mapping.inBW == 2, "MatAddBall requires two SRAM read ports") - require(mapping.outBW == 1, "MatAddBall requires one SRAM write port") - require(b.memDomain.bankWidth == 128, "MatAddBall requires 128-bit SRAM rows") - require(b.memDomain.bankMaskLen == 16, "MatAddBall requires sixteen byte enables") - - @public - val io = IO(new Bundle { - val cmdReq = Flipped(Decoupled(new BallRsIssue(b))) - val cmdResp = Decoupled(new BallRsComplete(b)) - val channelReady = Input(Bool()) - val bankRead = Vec(2, Flipped(new BankRead(b))) - val bankWrite = Vec(1, Flipped(new BankWrite(b))) - val status = new BallStatus - }) - - val Seq(idle, waitForChannels, readRequest, readResponse, writeRequest, writeResponse, complete) = Enum(7) - val state = RegInit(idle) - - val robId = RegInit(0.U(log2Up(b.frontend.rob_entries).W)) - val isSub = RegInit(false.B) - val subRobId = RegInit(0.U(log2Up(b.frontend.sub_rob_depth * 4).W)) - val aBank = RegInit(0.U(bankIdWidth.W)) - val bBank = RegInit(0.U(bankIdWidth.W)) - val cBank = RegInit(0.U(bankIdWidth.W)) - val groups = RegInit(0.U(5.W)) - val group = RegInit(0.U(5.W)) - val iter = RegInit(0.U(countWidth.W)) - val line = RegInit(0.U(addressWidth.W)) - val aWord = Reg(UInt(128.W)) - val bWord = Reg(UInt(128.W)) - val aRequestSent = RegInit(false.B) - val bRequestSent = RegInit(false.B) - val aReceived = RegInit(false.B) - val bReceived = RegInit(false.B) - - val sum = Wire(Vec(4, UInt(32.W))) - for (lane <- 0 until 4) { - sum(lane) := (aWord(32 * lane + 31, 32 * lane) +& - bWord(32 * lane + 31, 32 * lane))(31, 0) - } - - io.cmdReq.ready := state === idle - io.cmdResp.valid := state === complete - io.cmdResp.bits.rob_id := robId - io.cmdResp.bits.is_sub := isSub - io.cmdResp.bits.sub_rob_id := subRobId - - for (port <- 0 until 2) { - io.bankRead(port).rob_id := robId - io.bankRead(port).ball_id := 0.U - io.bankRead(port).bank_id := (if (port == 0) aBank else bBank) - io.bankRead(port).group_id := group - io.bankRead(port).io.req.valid := false.B - io.bankRead(port).io.req.bits.addr := line - io.bankRead(port).io.resp.ready := false.B - } - - io.bankWrite(0).rob_id := robId - io.bankWrite(0).ball_id := 0.U - io.bankWrite(0).bank_id := cBank - io.bankWrite(0).group_id := group - io.bankWrite(0).io.req.valid := false.B - io.bankWrite(0).io.req.bits.addr := line - io.bankWrite(0).io.req.bits.data := Cat(sum.reverse) - io.bankWrite(0).io.req.bits.mask := VecInit(Seq.fill(16)(true.B)) - io.bankWrite(0).io.resp.ready := false.B - - switch(state) { - is(idle) { - when(io.cmdReq.fire) { - val command = io.cmdReq.bits.cmd - assert(command.funct7 === funct.U(7.W), "MatAddBall funct7 must be MATADD") - assert(command.op1_en && command.op2_en && command.wr_spad_en, "MatAddBall requires two inputs and one output") - assert(command.op1_bank < b.memDomain.virtualBankCount.U, "MatAddBall input bank 0 is invalid") - assert(command.op2_bank < b.memDomain.virtualBankCount.U, "MatAddBall input bank 1 is invalid") - assert(command.wr_bank < b.memDomain.virtualBankCount.U, "MatAddBall output bank is invalid") - assert( - command.op1_bank =/= command.op2_bank && command.op1_bank =/= command.wr_bank && - command.op2_bank =/= command.wr_bank, - "MatAddBall banks must be distinct" - ) - assert( - command.op1_col =/= 0.U && command.op1_col === command.op2_col && - command.op1_col === command.wr_col, - "MatAddBall bank groups must match" - ) - assert(command.op1_col <= b.memDomain.bankNum.U, "MatAddBall bank groups must fit in physical banks") - assert( - command.iter =/= 0.U && command.iter <= b.memDomain.bankEntries.U, - "MatAddBall iter must fit in one physical bank" - ) - assert(command.special === 0.U, "MatAddBall reserves rs2") - - robId := io.cmdReq.bits.rob_id - isSub := io.cmdReq.bits.is_sub - subRobId := io.cmdReq.bits.sub_rob_id - aBank := command.op1_bank - bBank := command.op2_bank - cBank := command.wr_bank - groups := command.op1_col - group := 0.U - iter := command.iter - line := 0.U - aRequestSent := false.B - bRequestSent := false.B - aReceived := false.B - bReceived := false.B - state := waitForChannels - } - } - - is(waitForChannels) { - when(io.channelReady)(state := readRequest) - } - - is(readRequest) { - io.bankRead(0).io.req.valid := !aRequestSent - io.bankRead(1).io.req.valid := !bRequestSent - when(io.bankRead(0).io.req.fire)(aRequestSent := true.B) - when(io.bankRead(1).io.req.fire)(bRequestSent := true.B) - when((aRequestSent || io.bankRead(0).io.req.fire) && - (bRequestSent || io.bankRead(1).io.req.fire)) { - state := readResponse - } - } - - is(readResponse) { - io.bankRead(0).io.resp.ready := !aReceived - io.bankRead(1).io.resp.ready := !bReceived - when(io.bankRead(0).io.resp.fire) { - aWord := io.bankRead(0).io.resp.bits.data - aReceived := true.B - } - when(io.bankRead(1).io.resp.fire) { - bWord := io.bankRead(1).io.resp.bits.data - bReceived := true.B - } - when((aReceived || io.bankRead(0).io.resp.fire) && - (bReceived || io.bankRead(1).io.resp.fire)) { - state := writeRequest - } - } - - is(writeRequest) { - io.bankWrite(0).io.req.valid := true.B - when(io.bankWrite(0).io.req.fire)(state := writeResponse) - } - - is(writeResponse) { - io.bankWrite(0).io.resp.ready := true.B - when(io.bankWrite(0).io.resp.fire) { - aRequestSent := false.B - bRequestSent := false.B - aReceived := false.B - bReceived := false.B - when(line +& 1.U === iter) { - line := 0.U - when(group +& 1.U === groups) { - state := complete - }.otherwise { - group := group + 1.U - state := readRequest - } - }.otherwise { - line := line + 1.U - state := readRequest - } - } - } - - is(complete) { - when(io.cmdResp.fire)(state := idle) - } - } - - io.status.idle := state === idle - io.status.running := state =/= idle -} diff --git a/examples/balls/matadd/arch/src/main/scala/MatAddBall.scala b/examples/balls/matadd/arch/src/main/scala/MatAddBall.scala deleted file mode 100644 index 4193186da..000000000 --- a/examples/balls/matadd/arch/src/main/scala/MatAddBall.scala +++ /dev/null @@ -1,34 +0,0 @@ -package examples.balls.matadd - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.balldomain.blink.{BlinkIO, HasBallStatus, HasBlink, SubRobRow} -import framework.balldomain.blink.mmio.MmioRead -import framework.top.GlobalConfig - -@instantiable -class MatAddBall(val b: GlobalConfig) extends Module with HasBlink with HasBallStatus { - - private val mapping = b.ballDomain.ballIdMappings - .find(_.ballName == "MatAddBall") - .getOrElse(throw new IllegalArgumentException("MatAddBall not found in config")) - - @public - val io = IO(new BlinkIO(b, mapping.inBW, mapping.outBW)) - - def blink: BlinkIO = io - def status = io.status - dontTouch(io) - - val unit: Instance[MatAdd] = Instantiate(new MatAdd(b)) - unit.io.cmdReq <> io.cmdReq - unit.io.cmdResp <> io.cmdResp - unit.io.channelReady := io.channelReady - unit.io.bankRead(0) <> io.bankRead(0) - unit.io.bankRead(1) <> io.bankRead(1) - unit.io.bankWrite(0) <> io.bankWrite(0) - io.status <> unit.io.status - io.subRobReq.valid := false.B - io.subRobReq.bits := SubRobRow.tieOff(b) - MmioRead.tieOff(io.mmioRead) -} diff --git a/examples/balls/matadd/compiler/src/Backend/include/IntrinsicsRISCVMatAddBall.td b/examples/balls/matadd/compiler/src/Backend/include/IntrinsicsRISCVMatAddBall.td deleted file mode 100644 index 125b1e629..000000000 --- a/examples/balls/matadd/compiler/src/Backend/include/IntrinsicsRISCVMatAddBall.td +++ /dev/null @@ -1 +0,0 @@ -// MatAddBall uses CustomIntrOp with funct7 from the target ballISA. diff --git a/examples/balls/matadd/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp b/examples/balls/matadd/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp deleted file mode 100644 index 6040e0a1f..000000000 --- a/examples/balls/matadd/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp +++ /dev/null @@ -1,34 +0,0 @@ -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace mlir::buddy { -void populateMatAddBallAssignPhysicalBankPatterns(RewritePatternSet &patterns, - PhysicalBankState &state); -} - -namespace { -class BankMatAddPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(BankMatAddOp op, - PatternRewriter &rewriter) const override { - rewriter.create(op.getLoc(), op.getABank(), op.getBBank(), - op.getCBank(), op.getIter()); - rewriter.replaceOp(op, op.getCBank()); - return success(); - } -}; -} // namespace - -void mlir::buddy::populateMatAddBallAssignPhysicalBankPatterns( - RewritePatternSet &patterns, mlir::buddy::PhysicalBankState &state) { - (void)state; - patterns.add(patterns.getContext()); -} diff --git a/examples/balls/matadd/compiler/src/Dialect/Buckyball/MatAddBall.td b/examples/balls/matadd/compiler/src/Dialect/Buckyball/MatAddBall.td deleted file mode 100644 index 62e34bd00..000000000 --- a/examples/balls/matadd/compiler/src/Dialect/Buckyball/MatAddBall.td +++ /dev/null @@ -1,21 +0,0 @@ -#ifndef MATADDBALL_OPS -#define MATADDBALL_OPS - -include "Buckyball.td" - -def MatAddOp : Buckyball_Op<"matadd"> { - let summary = "Add matching INT32 physical-bank lines"; - let arguments = (ins I64:$aBankId, I64:$bBankId, I64:$cBankId, I64:$iter); - let assemblyFormat = "$aBankId `,` $bBankId `,` $cBankId `,` $iter attr-dict `:` " - "type($aBankId)"; -} - -def BankMatAddOp : Buckyball_Op<"bank_matadd"> { - let summary = "Add matching INT32 virtual-bank lines"; - let arguments = (ins I64:$aBank, I64:$bBank, I64:$cBank, I64:$iter); - let results = (outs I64:$cBankOut); - let assemblyFormat = "$aBank $bBank $cBank $iter attr-dict `:` " - "type($aBank) type($bBank) type($cBank) type($iter)"; -} - -#endif // MATADDBALL_OPS diff --git a/examples/balls/matadd/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/balls/matadd/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 8029495ce..000000000 --- a/examples/balls/matadd/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,44 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace buddy::buckyball; -using namespace buddy::buckyball::legalize; - -namespace { -struct MatAddLowering : public ConvertOpToLLVMPattern { - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - - LogicalResult - matchAndRewrite(MatAddOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("MatAddBall"); - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getABankId(), - adaptor.getBBankId(), adaptor.getCBankId(), - adaptor.getIter()); - rewriter.replaceOpWithNewOp( - op, rs1, cstI64(rewriter, loc, 0), - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("MATADD"))); - return success(); - } -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateMatAddBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool, int64_t) { - patterns.add(converter); -} - -void configureMatAddBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool) { - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/examples/balls/matadd/configs/default.toml b/examples/balls/matadd/configs/default.toml deleted file mode 100644 index c61b39cb6..000000000 --- a/examples/balls/matadd/configs/default.toml +++ /dev/null @@ -1 +0,0 @@ -[ball] diff --git a/examples/balls/matadd/emu/src/dispatch.rs b/examples/balls/matadd/emu/src/dispatch.rs deleted file mode 100644 index 434c7cccd..000000000 --- a/examples/balls/matadd/emu/src/dispatch.rs +++ /dev/null @@ -1,31 +0,0 @@ -use crate::inst::instruction::{BallInstruction, ExecContext}; - -use super::f64_matadd; - -const BALL_CLASS: &str = "examples.balls.matadd.MatAddBall"; - -pub fn execute_known( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, - ctx: &mut ExecContext, -) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("MATADD") => Some(f64_matadd::MatAdd::exec(xs1, xs2, ctx)), - Some(_) | None => None, - } -} - -pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("MATADD") => Some(f64_matadd::MatAdd::latency(xs1, xs2)), - Some(_) | None => None, - } -} diff --git a/examples/balls/matadd/emu/src/lib.rs b/examples/balls/matadd/emu/src/lib.rs deleted file mode 100644 index d4db1c64c..000000000 --- a/examples/balls/matadd/emu/src/lib.rs +++ /dev/null @@ -1,8 +0,0 @@ -pub(crate) use crate::inst::{decode, instruction}; - -#[path = "64_matadd.rs"] -mod f64_matadd; - -mod dispatch; - -pub use dispatch::{cycles_after_issue, execute_known}; diff --git a/examples/balls/matadd/verify/src/common/matadd_defs.svh b/examples/balls/matadd/verify/src/common/matadd_defs.svh deleted file mode 100644 index 6b7f36f39..000000000 --- a/examples/balls/matadd/verify/src/common/matadd_defs.svh +++ /dev/null @@ -1,15 +0,0 @@ -`ifndef MATADD_FUNCT7 -`error "MATADD_FUNCT7 must be provided by the selected Core ballISA" -`endif - -localparam int MATADD_CORE_FUNCT7 = `MATADD_FUNCT7; -localparam int MATADD_MAX_LINES = 64; -localparam int MATADD_TIMEOUT_CYCLES = 100000; - -function automatic int unsigned matadd_require_bid(); - int unsigned bid; - if (!$value$plusargs("BID=%d", bid)) begin - `uvm_fatal("BID", "missing required plusarg +BID=") - end - return bid; -endfunction diff --git a/examples/balls/matadd/verify/src/common/matadd_items.svh b/examples/balls/matadd/verify/src/common/matadd_items.svh deleted file mode 100644 index 4ca129f43..000000000 --- a/examples/balls/matadd/verify/src/common/matadd_items.svh +++ /dev/null @@ -1,97 +0,0 @@ -class matadd_cmd_item extends bb_blink_cmd_item; - `uvm_object_utils(matadd_cmd_item) - - int unsigned lines; - int unsigned group_count; - int unsigned case_index; - - constraint legal_c { - funct7 == MATADD_CORE_FUNCT7[6:0]; - op1_en == 1'b1; - op2_en == 1'b1; - wr_spad_en == 1'b1; - op1_from_spad == 1'b1; - op2_from_spad == 1'b1; - meta_bank == 5'd0; - rs1 == 64'd0; - rs2 == 64'd0; - special == 64'd0; - is_sub == 1'b0; - sub_rob_id == 8'd0; - } - - function new(string name = "matadd_cmd_item"); - super.new(name); - endfunction - - function void load_case(int unsigned index, int unsigned ball_id); - if (index > 2) `uvm_fatal("CASE", $sformatf("unsupported case %0d", index)) - case_index = index; - this.bid = ball_id[4:0]; - funct7 = MATADD_CORE_FUNCT7[6:0]; - iter = (index == 2) ? MATADD_MAX_LINES : 16; - lines = iter; - group_count = (index == 0) ? 1 : 2; - op1_bank = 5'd0; - op2_bank = 5'd1; - wr_bank = 5'd2; - op1_col = group_count[4:0]; - op2_col = group_count[4:0]; - wr_col = group_count[4:0]; - rob_id = 4 + index; - op1_en = 1'b1; - op2_en = 1'b1; - wr_spad_en = 1'b1; - op1_from_spad = 1'b1; - op2_from_spad = 1'b1; - meta_bank = 5'd0; - rs1 = 64'd0; - rs2 = 64'd0; - special = 64'd0; - is_sub = 1'b0; - sub_rob_id = 8'd0; - endfunction - - function automatic bit [31:0] a_lane(int unsigned group, int unsigned line, int unsigned lane); - int unsigned offset; - offset = (group * lines + line) * 4 + lane; - if (case_index == 2) return 32'hffff_ff00 + offset; - return 32'h1000_0000 + offset * 32'h0001_0011; - endfunction - - function automatic bit [31:0] b_lane(int unsigned group, int unsigned line, int unsigned lane); - int unsigned offset; - offset = (group * lines + line) * 4 + lane; - if (case_index == 2) return 32'h0000_0200 + offset * 5; - return 32'he000_0000 + offset * 32'h0000_0103; - endfunction - - function automatic bit [127:0] a_word(int unsigned group, int unsigned line); - bit [127:0] word; - for (int lane = 0; lane < 4; lane++) word[lane*32+:32] = a_lane(group, line, lane); - return word; - endfunction - - function automatic bit [127:0] b_word(int unsigned group, int unsigned line); - bit [127:0] word; - for (int lane = 0; lane < 4; lane++) word[lane*32+:32] = b_lane(group, line, lane); - return word; - endfunction - - function automatic bit [127:0] sum_word(int unsigned group, int unsigned line); - bit [127:0] word; - for (int lane = 0; lane < 4; lane++) begin - word[lane*32+:32] = a_lane(group, line, lane) + b_lane(group, line, lane); - end - return word; - endfunction - - function void do_copy(uvm_object rhs); - matadd_cmd_item rhs_item; - super.do_copy(rhs); - if (!$cast(rhs_item, rhs)) `uvm_fatal("COPY", "rhs is not matadd_cmd_item") - lines = rhs_item.lines; - group_count = rhs_item.group_count; - case_index = rhs_item.case_index; - endfunction -endclass diff --git a/examples/balls/matadd/verify/src/tb_top.sv b/examples/balls/matadd/verify/src/tb_top.sv deleted file mode 100644 index 677b20a9e..000000000 --- a/examples/balls/matadd/verify/src/tb_top.sv +++ /dev/null @@ -1,96 +0,0 @@ -module tb_top; - import uvm_pkg::*; - import matadd_pkg::*; - - bb_blink_if #(2, 1) intf (); - - MatAddBall dut ( - .clock(intf.clock), - .reset(intf.reset), - .io_channelReady(1'b1), - .io_cmdReq_ready(intf.cmd_req_ready), - .io_cmdReq_valid(intf.cmd_req_valid), - .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), - .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), - .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), - .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), - .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), - .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), - .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), - .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), - .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), - .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank[3:0]), - .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank[3:0]), - .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank[3:0]), - .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), - .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), - .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), - .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank[3:0]), - .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), - .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), - .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), - .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), - .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), - .io_cmdResp_ready(intf.cmd_resp_ready), - .io_cmdResp_valid(intf.cmd_resp_valid), - .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), - .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), - .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), - .io_bankRead_0_bank_id(intf.bank_read_bank_id[0][3:0]), - .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), - .io_bankRead_0_group_id(intf.bank_read_group_id[0][3:0]), - .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), - .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), - .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), - .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), - .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), - .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), - .io_bankRead_1_bank_id(intf.bank_read_bank_id[1][3:0]), - .io_bankRead_1_rob_id(intf.bank_read_rob_id[1]), - .io_bankRead_1_group_id(intf.bank_read_group_id[1][3:0]), - .io_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), - .io_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), - .io_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), - .io_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), - .io_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), - .io_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), - .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0][3:0]), - .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), - .io_bankWrite_0_group_id(intf.bank_write_group_id[0][3:0]), - .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), - .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), - .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), - .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), - .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), - .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), - .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), - .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), - .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), - .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), - .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), - .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), - .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), - .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), - .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), - .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), - .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), - .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), - .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), - .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), - .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), - .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), - .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), - .io_subRobReq_ready(intf.sub_rob_req_ready) - ); - - initial begin - intf.clock = 1'b0; - intf.reset = 1'b1; - forever #5 intf.clock = ~intf.clock; - end - - initial begin - uvm_config_db#(virtual bb_blink_if #(2, 1))::set(null, "*", "vif", intf); - run_test("matadd_ball_test"); - end -endmodule diff --git a/examples/balls/matadd/workloads/ctests/CMakeLists.txt b/examples/balls/matadd/workloads/ctests/CMakeLists.txt deleted file mode 100644 index 2fc0cad9f..000000000 --- a/examples/balls/matadd/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_buckyball_ctests(matadd_i32_16row_test.c) diff --git a/examples/balls/matadd/workloads/mlir_tests/ball/matadd_i32.mlir b/examples/balls/matadd/workloads/mlir_tests/ball/matadd_i32.mlir deleted file mode 100644 index a3b560048..000000000 --- a/examples/balls/matadd/workloads/mlir_tests/ball/matadd_i32.mlir +++ /dev/null @@ -1,42 +0,0 @@ -func.func private @check_result(memref<16x4xi32>) -> () - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %four = arith.constant 4 : index - %sixteen = arith.constant 16 : index - %depth = arith.constant 16 : i64 - %stride = arith.constant 1 : i64 - %a = memref.alloc() : memref<16x4xi32> - %b = memref.alloc() : memref<16x4xi32> - %out = memref.alloc() : memref<16x4xi32> - scf.for %row = %zero to %sixteen step %one { - scf.for %lane = %zero to %four step %one { - %r = arith.index_cast %row : index to i32 - %l = arith.index_cast %lane : index to i32 - %x = arith.addi %r, %l : i32 - %y = arith.subi %l, %r : i32 - memref.store %x, %a[%row, %lane] : memref<16x4xi32> - memref.store %y, %b[%row, %lane] : memref<16x4xi32> - } - } - %a_bank = buckyball.bank_alloc - %b_bank = buckyball.bank_alloc - %out_bank = buckyball.bank_alloc - %a_loaded = buckyball.bank_mvin %a %a_bank %depth %stride - : memref<16x4xi32> i64 i64 i64 - %b_loaded = buckyball.bank_mvin %b %b_bank %depth %stride - : memref<16x4xi32> i64 i64 i64 - buckyball.matadd %a_loaded, %b_loaded, %out_bank, %depth : i64 - %stored = buckyball.bank_mvout %out %out_bank %depth %stride - : memref<16x4xi32> i64 i64 i64 - func.call @check_result(%out) : (memref<16x4xi32>) -> () - buckyball.bank_release %a_loaded : i64 - buckyball.bank_release %b_loaded : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %a : memref<16x4xi32> - memref.dealloc %b : memref<16x4xi32> - memref.dealloc %out : memref<16x4xi32> - return %zero_i8 : i8 -} diff --git a/examples/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise.mlir.in b/examples/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise.mlir.in deleted file mode 100644 index 0ae0c975d..000000000 --- a/examples/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise.mlir.in +++ /dev/null @@ -1,43 +0,0 @@ -func.func private @check_result(memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32>) -> () - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %rows = arith.constant @BB_BANK_LINES@ : index - %lanes = arith.constant @BB_BANK_I32_LANES@ : index - %depth = arith.constant @BB_BANK_LINES@ : i64 - %stride = arith.constant 1 : i64 - %a = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> - %b = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> - %out = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> - scf.for %row = %zero to %rows step %one { - scf.for %lane = %zero to %lanes step %one { - %r = arith.index_cast %row : index to i32 - %l = arith.index_cast %lane : index to i32 - %x = arith.addi %r, %l : i32 - %y = arith.subi %l, %r : i32 - memref.store %x, %a[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> - memref.store %y, %b[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> - } - } - %a_bank = buckyball.bank_alloc - %b_bank = buckyball.bank_alloc - %out_bank = buckyball.bank_alloc - %a_loaded = buckyball.bank_mvin %a %a_bank %depth %stride - : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> i64 i64 i64 - %b_loaded = buckyball.bank_mvin %b %b_bank %depth %stride - : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> i64 i64 i64 - %result = buckyball.bank_matadd %a_loaded %b_loaded %out_bank %depth - : i64 i64 i64 i64 - %stored = buckyball.bank_mvout %out %result %depth %stride - : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> i64 i64 i64 - func.call @check_result(%out) : (memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32>) -> () - buckyball.bank_release %a_loaded : i64 - buckyball.bank_release %b_loaded : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %a : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> - memref.dealloc %b : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> - memref.dealloc %out : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> - return %zero_i8 : i8 -} diff --git a/examples/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise_main.cpp b/examples/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise_main.cpp deleted file mode 100644 index 9676e005b..000000000 --- a/examples/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise_main.cpp +++ /dev/null @@ -1,19 +0,0 @@ -#include -#include -#include -#include - -extern "C" void check_result(int32_t *, int32_t *data, int64_t offset, - int64_t rows, int64_t lanes, int64_t stride, - int64_t lane_stride) { - if (rows != BANK_LINES || lanes != BANK_WIDTH / 32 || stride != lanes || - lane_stride != 1) - exit(1); - for (int row = 0; row < rows; ++row) - for (int lane = 0; lane < lanes; ++lane) - if (data[offset + row * stride + lane] != 2 * lane) { - printf("matadd bank mismatch row=%d lane=%d\n", row, lane); - exit(1); - } - printf("matadd bank PASS rows=%d lanes=%d\n", BANK_LINES, BANK_WIDTH / 32); -} diff --git a/examples/balls/maxpool/arch/src/main/scala/MaxPoolBall.scala b/examples/balls/maxpool/arch/src/main/scala/MaxPoolBall.scala deleted file mode 100644 index c2595c9aa..000000000 --- a/examples/balls/maxpool/arch/src/main/scala/MaxPoolBall.scala +++ /dev/null @@ -1,258 +0,0 @@ -package examples.balls.maxpool - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public} -import chisel3.util._ -import framework.balldomain.blink.{BallStatus, BlinkIO, HasBallStatus, HasBlink, SubRobRow} -import framework.balldomain.blink.mmio.{MmioRead, MmioWrite} -import framework.top.GlobalConfig - -@instantiable -class MaxPoolBall(val b: GlobalConfig) extends Module with HasBlink with HasBallStatus { - - private val mapping = b.ballDomain.ballIdMappings - .find(_.ballName == "MaxPoolBall") - .getOrElse(throw new IllegalArgumentException("MaxPoolBall not found in config")) - - private val funct = b.ballDomain.ballISA - .find(_.mnemonic == "MAXPOOL") - .map(_.funct7) - .getOrElse(throw new IllegalArgumentException("MAXPOOL not found in ballISA")) - - require(b.memDomain.bankWidth == 128, "MaxPoolBall requires 128-bit bank rows") - require(mapping.inBW == 1, "MaxPoolBall requires inBW=1") - require(mapping.outBW == 1, "MaxPoolBall requires outBW=1") - require((funct >> 4) == 3, "MAXPOOL must encode one read and one write") - - @public - val io = IO(new BlinkIO(b, mapping.inBW, mapping.outBW)) - def blink: BlinkIO = io - def status: BallStatus = io.status - dontTouch(io) - - private val idle :: scan :: readResp :: writeReq :: writeResp :: complete :: Nil = Enum(6) - private val state = RegInit(idle) - private val robId = RegInit(0.U(log2Up(b.frontend.rob_entries).W)) - private val isSub = RegInit(false.B) - private val subRobId = RegInit(0.U(log2Up(b.frontend.sub_rob_depth * 4).W)) - private val inputBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - private val outputBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - private val iter = RegInit(0.U(b.frontend.iter_len.W)) - private val inputSide = RegInit(0.U(8.W)) - private val outputSide = RegInit(0.U(8.W)) - private val kernel = RegInit(0.U(8.W)) - private val stride = RegInit(0.U(8.W)) - private val padding = RegInit(0.U(8.W)) - private val inputBase = RegInit(0.U(6.W)) - private val outputBase = RegInit(0.U(6.W)) - private val outputStride = RegInit(0.U(6.W)) - private val startRow = RegInit(0.U(4.W)) - private val startCol = RegInit(0.U(4.W)) - private val outputRow = RegInit(0.U(log2Ceil(b.memDomain.bankEntries).W)) - private val outputY = RegInit(0.U(8.W)) - private val outputX = RegInit(0.U(8.W)) - private val kernelY = RegInit(0.U(8.W)) - private val kernelX = RegInit(0.U(8.W)) - private val maximum = RegInit(VecInit(Seq.fill(16)("h80".U(8.W)))) - private val writeData = Reg(UInt(128.W)) - - private val sourceY = outputY * stride +& kernelY +& startRow - private val sourceX = outputX * stride +& kernelX +& startCol - private val sourceValid = sourceY >= padding && sourceX >= padding && - sourceY < padding +& inputSide && sourceX < padding +& inputSide - private val inputY = sourceY - padding - private val inputX = sourceX - padding - private val inputAddress = inputBase +& inputY * inputSide +& inputX - private val outputAddress = outputBase +& outputY * outputStride +& outputX - private val lastKernel = kernelY +& 1.U === kernel && kernelX +& 1.U === kernel - - io.cmdReq.ready := state === idle - io.cmdResp.valid := state === complete - io.cmdResp.bits.rob_id := robId - io.cmdResp.bits.is_sub := isSub - io.cmdResp.bits.sub_rob_id := subRobId - io.status.idle := state === idle - io.status.running := state =/= idle && state =/= complete - - io.bankRead(0).rob_id := robId - io.bankRead(0).ball_id := 0.U - io.bankRead(0).bank_id := inputBank - io.bankRead(0).group_id := 0.U - io.bankRead(0).io.req.valid := false.B - io.bankRead(0).io.req.bits.addr := inputAddress - io.bankRead(0).io.resp.ready := false.B - - io.bankWrite(0).rob_id := robId - io.bankWrite(0).ball_id := 0.U - io.bankWrite(0).bank_id := outputBank - io.bankWrite(0).group_id := 0.U - io.bankWrite(0).io.req.valid := false.B - io.bankWrite(0).io.req.bits.addr := outputAddress - io.bankWrite(0).io.req.bits.data := writeData - io.bankWrite(0).io.req.bits.mask := VecInit(Seq.fill(b.memDomain.bankMaskLen)(true.B)) - io.bankWrite(0).io.resp.ready := false.B - - io.subRobReq.valid := false.B - io.subRobReq.bits := SubRobRow.tieOff(b) - MmioRead.tieOff(io.mmioRead) - MmioWrite.tieOff(io.mmioWrite) - - switch(state) { - is(idle) { - when(io.cmdReq.fire) { - val cmd = io.cmdReq.bits.cmd - val inSide = cmd.rs2(3, 0) - val outSide = cmd.rs2(7, 4) - val kernelSize = cmd.rs2(11, 8) - val poolStride = cmd.rs2(15, 12) - val poolPadding = cmd.rs2(19, 16) - val inBase = cmd.rs2(25, 20) - val outBase = cmd.rs2(31, 26) - val outStride = cmd.rs2(37, 32) - val poolStartRow = cmd.rs2(41, 38) - val poolStartCol = cmd.rs2(45, 42) - assert(cmd.rs2(63, 46) === 0.U, "MAXPOOL reserved rs2 bits must be zero") - assert(cmd.funct7 === funct.U, "MaxPoolBall funct7 must be MAXPOOL") - assert(cmd.rs1(19, 10) === 0.U, "MAXPOOL reserves input bank 1") - assert( - cmd.rs1(9, 0) < b.memDomain.virtualBankCount.U && - cmd.rs1(29, 20) < b.memDomain.virtualBankCount.U, - "MAXPOOL bank id is invalid" - ) - assert( - cmd.op1_col === 1.U && cmd.op2_col === 0.U && cmd.wr_col === 1.U, - "MAXPOOL requires one input bank and one output bank" - ) - assert(cmd.op1_bank =/= cmd.wr_bank, "MAXPOOL input and output banks must differ") - assert( - inSide > 0.U && outSide > 0.U && kernelSize > 0.U && poolStride > 0.U, - "MAXPOOL dimensions must be positive" - ) - assert(outStride >= outSide, "MAXPOOL output stride is too small") - val lastSourceRow = poolStartRow +& (outSide - 1.U) * poolStride +& (kernelSize - 1.U) - val lastSourceCol = poolStartCol +& (outSide - 1.U) * poolStride +& (kernelSize - 1.U) - val inputEnd = poolPadding +& inSide - val readsInput = poolStartRow < inputEnd && poolStartCol < inputEnd && - lastSourceRow >= poolPadding && lastSourceCol >= poolPadding - val maxInputRow = Mux(lastSourceRow < inputEnd, lastSourceRow - poolPadding, inSide - 1.U) - val maxInputCol = Mux(lastSourceCol < inputEnd, lastSourceCol - poolPadding, inSide - 1.U) - assert( - !readsInput || inBase +& maxInputRow * inSide +& maxInputCol < b.memDomain.bankEntries.U, - "MAXPOOL input window exceeds bank depth" - ) - assert( - outSide * outSide === cmd.iter && - outBase +& (outSide - 1.U) * outStride +& outSide <= b.memDomain.bankEntries.U, - "MAXPOOL output tile exceeds bank depth" - ) - assert( - poolStartRow +& (outSide - 1.U) * poolStride +& kernelSize <= - inSide +& (poolPadding << 1) && - poolStartCol +& (outSide - 1.U) * poolStride +& kernelSize <= - inSide +& (poolPadding << 1), - "MAXPOOL pooling geometry is inconsistent" - ) - - robId := io.cmdReq.bits.rob_id - isSub := io.cmdReq.bits.is_sub - subRobId := io.cmdReq.bits.sub_rob_id - inputBank := cmd.op1_bank - outputBank := cmd.wr_bank - iter := cmd.iter - inputSide := inSide - outputSide := outSide - kernel := kernelSize - stride := poolStride - padding := poolPadding - inputBase := inBase - outputBase := outBase - outputStride := outStride - startRow := poolStartRow - startCol := poolStartCol - outputRow := 0.U - outputY := 0.U - outputX := 0.U - kernelY := 0.U - kernelX := 0.U - maximum := VecInit(Seq.fill(16)("h80".U(8.W))) - state := scan - } - } - - is(scan) { - when(sourceValid) { - io.bankRead(0).io.req.valid := true.B - when(io.bankRead(0).io.req.fire) { - state := readResp - } - }.elsewhen(lastKernel) { - writeData := Cat(maximum.reverse) - state := writeReq - }.otherwise { - when(kernelX +& 1.U === kernel) { - kernelX := 0.U - kernelY := kernelY + 1.U - }.otherwise { - kernelX := kernelX + 1.U - } - } - } - - is(readResp) { - io.bankRead(0).io.resp.ready := true.B - when(io.bankRead(0).io.resp.fire) { - val data = io.bankRead(0).io.resp.bits.data.asTypeOf(Vec(16, UInt(8.W))) - val nextMaximum = Wire(Vec(16, UInt(8.W))) - for (lane <- 0 until 16) - nextMaximum(lane) := Mux(data(lane).asSInt > maximum(lane).asSInt, data(lane), maximum(lane)) - maximum := nextMaximum - when(lastKernel) { - writeData := Cat(nextMaximum.reverse) - state := writeReq - }.otherwise { - when(kernelX +& 1.U === kernel) { - kernelX := 0.U - kernelY := kernelY + 1.U - }.otherwise { - kernelX := kernelX + 1.U - } - state := scan - } - } - } - - is(writeReq) { - io.bankWrite(0).io.req.valid := true.B - when(io.bankWrite(0).io.req.fire) { - state := writeResp - } - } - - is(writeResp) { - io.bankWrite(0).io.resp.ready := true.B - when(io.bankWrite(0).io.resp.fire) { - when(outputRow +& 1.U === iter) { - state := complete - }.otherwise { - outputRow := outputRow + 1.U - when(outputX +& 1.U === outputSide) { - outputX := 0.U - outputY := outputY + 1.U - }.otherwise { - outputX := outputX + 1.U - } - kernelY := 0.U - kernelX := 0.U - maximum := VecInit(Seq.fill(16)("h80".U(8.W))) - state := scan - } - } - } - - is(complete) { - when(io.cmdResp.fire) { - state := idle - } - } - } -} diff --git a/examples/balls/maxpool/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp b/examples/balls/maxpool/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp deleted file mode 100644 index ff9f77a4d..000000000 --- a/examples/balls/maxpool/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp +++ /dev/null @@ -1,38 +0,0 @@ -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace mlir::buddy { -void populateMaxPoolBallAssignPhysicalBankPatterns(RewritePatternSet &patterns, - PhysicalBankState &state); -} // namespace mlir::buddy - -namespace { -class BankMaxPoolPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(BankMaxPoolOp op, - PatternRewriter &rewriter) const override { - rewriter.create( - op.getLoc(), op.getInBank(), op.getOutBank(), op.getIter(), - op.getInputSideAttr(), op.getOutputSideAttr(), op.getKernelAttr(), - op.getStrideAttr(), op.getPaddingAttr(), op.getInputBase(), - op.getOutputBase(), op.getOutputStride(), op.getStartRowAttr(), - op.getStartColAttr()); - rewriter.replaceOp(op, op.getOutBank()); - return success(); - } -}; -} // namespace - -void mlir::buddy::populateMaxPoolBallAssignPhysicalBankPatterns( - RewritePatternSet &patterns, PhysicalBankState &state) { - (void)state; - patterns.add(patterns.getContext()); -} diff --git a/examples/balls/maxpool/compiler/src/Dialect/Buckyball/MaxPoolBall.td b/examples/balls/maxpool/compiler/src/Dialect/Buckyball/MaxPoolBall.td deleted file mode 100644 index 98ec6617e..000000000 --- a/examples/balls/maxpool/compiler/src/Dialect/Buckyball/MaxPoolBall.td +++ /dev/null @@ -1,44 +0,0 @@ -//===-- MaxPoolBall.td - MaxPoolBall operations ------------*- tablegen -*-===// - -#ifndef MAXPOOLBALL_OPS -#define MAXPOOLBALL_OPS - -include "Buckyball.td" - -def MaxPoolOp : Buckyball_Op<"maxpool"> { - let summary = "Signed INT8 spatial max pooling"; - let description = [{ - Each bank row contains the 16 channels of one spatial position. The input - and output tiles are square. inputBase selects the first input row. - outputBase and outputStride place the result inside a larger resident - feature-map tile. iter is the number of output spatial positions. - }]; - let arguments = (ins I64:$inputBankId, I64:$outputBankId, I64:$iter, - I64Attr:$inputSide, I64Attr:$outputSide, - I64Attr:$kernel, I64Attr:$stride, I64Attr:$padding, - I64:$inputBase, I64:$outputBase, - I64:$outputStride, I64Attr:$startRow, - I64Attr:$startCol); - let assemblyFormat = "$inputBankId `,` $outputBankId `,` $iter `,` " - "$inputBase `,` $outputBase `,` $outputStride " - "prop-dict attr-dict `:` type($inputBankId) type($outputBankId) " - "type($iter) type($inputBase) type($outputBase) " - "type($outputStride)"; -} - -def BankMaxPoolOp : Buckyball_Op<"bank_maxpool"> { - let summary = "Signed INT8 spatial max pooling with virtual banks"; - let arguments = (ins I64:$inBank, I64:$outBank, I64:$iter, - I64Attr:$inputSide, I64Attr:$outputSide, - I64Attr:$kernel, I64Attr:$stride, I64Attr:$padding, - I64:$inputBase, I64:$outputBase, - I64:$outputStride, I64Attr:$startRow, - I64Attr:$startCol); - let results = (outs I64:$outBankOut); - let assemblyFormat = "$inBank $outBank $iter $inputBase $outputBase " - "$outputStride prop-dict attr-dict `:` type($inBank) " - "type($outBank) type($iter) type($inputBase) " - "type($outputBase) type($outputStride)"; -} - -#endif // MAXPOOLBALL_OPS diff --git a/examples/balls/maxpool/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/balls/maxpool/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 332d2d7c2..000000000 --- a/examples/balls/maxpool/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,109 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace buddy::buckyball; -using namespace buddy::buckyball::legalize; - -namespace { -struct MaxPoolLowering : public ConvertOpToLLVMPattern { - MaxPoolLowering(LLVMTypeConverter &converter, int64_t bankDepth) - : ConvertOpToLLVMPattern(converter), bankDepth(bankDepth) {} - - LogicalResult - matchAndRewrite(MaxPoolOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("MaxPoolBall"); - int64_t inputSide = op.getInputSide(); - int64_t outputSide = op.getOutputSide(); - int64_t kernel = op.getKernel(); - int64_t stride = op.getStride(); - int64_t padding = op.getPadding(); - int64_t startRow = op.getStartRow(); - int64_t startCol = op.getStartCol(); - auto iterOp = op.getIter().getDefiningOp(); - auto iterAttr = - iterOp ? dyn_cast(iterOp.getValue()) : IntegerAttr(); - if (!iterAttr || inputSide <= 0 || inputSide > 15 || outputSide <= 0 || - outputSide > 15 || kernel <= 0 || kernel > 15 || stride <= 0 || - stride > 15 || padding < 0 || padding > 15 || startRow < 0 || - startRow > 15 || startCol < 0 || startCol > 15) - return op.emitError("MaxPool shape does not fit one physical bank tile"); - - int64_t lastSourceRow = startRow + (outputSide - 1) * stride + kernel - 1; - int64_t lastSourceCol = startCol + (outputSide - 1) * stride + kernel - 1; - int64_t inputEnd = padding + inputSide; - bool readsInput = startRow < inputEnd && startCol < inputEnd && - lastSourceRow >= padding && lastSourceCol >= padding; - int64_t inputFootprint = 0; - if (readsInput) { - int64_t maxInputRow = std::min(lastSourceRow, inputEnd - 1) - padding; - int64_t maxInputCol = std::min(lastSourceCol, inputEnd - 1) - padding; - inputFootprint = maxInputRow * inputSide + maxInputCol + 1; - } - if (inputFootprint > bankDepth || - outputSide * outputSide != iterAttr.getInt() || - inputSide + 2 * padding < kernel + startRow || - inputSide + 2 * padding < kernel + startCol || - startRow + (outputSide - 1) * stride + kernel > - inputSide + 2 * padding || - startCol + (outputSide - 1) * stride + kernel > inputSide + 2 * padding) - return op.emitError("MaxPool shape does not fit one physical bank tile"); - - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInputBankId(), - cstI64(rewriter, loc, 0), - adaptor.getOutputBankId(), adaptor.getIter()); - uint64_t geometry = static_cast(inputSide) | - (static_cast(outputSide) << 4) | - (static_cast(kernel) << 8) | - (static_cast(stride) << 12) | - (static_cast(padding) << 16) | - (static_cast(startRow) << 38) | - (static_cast(startCol) << 42); - Value rs2 = cstI64(rewriter, loc, geometry); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getInputBase(), - cstI64(rewriter, loc, 20))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getOutputBase(), - cstI64(rewriter, loc, 26))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getOutputStride(), - cstI64(rewriter, loc, 32))); - rewriter.replaceOpWithNewOp( - op, rs1, rs2, - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("MAXPOOL"))); - return success(); - } - -private: - int64_t bankDepth; -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateMaxPoolBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth) { - (void)stable; - - patterns.add(converter, bankDepth); -} - -void configureMaxPoolBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable) { - (void)stable; - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/examples/balls/maxpool/configs/default.toml b/examples/balls/maxpool/configs/default.toml deleted file mode 100644 index c632a36cb..000000000 --- a/examples/balls/maxpool/configs/default.toml +++ /dev/null @@ -1,3 +0,0 @@ -# MaxPoolBall has no private configuration parameters. - -[ball] diff --git a/examples/balls/maxpool/emu/src/dispatch.rs b/examples/balls/maxpool/emu/src/dispatch.rs deleted file mode 100644 index c1ec8769d..000000000 --- a/examples/balls/maxpool/emu/src/dispatch.rs +++ /dev/null @@ -1,36 +0,0 @@ -use crate::inst::instruction::{BallInstruction, ExecContext}; - -use super::maxpool; - -const BALL_CLASS: &str = "examples.balls.maxpool.MaxPoolBall"; - -pub fn execute_known( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, - ctx: &mut ExecContext, -) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("MAXPOOL") => Some(maxpool::MaxPool::exec(xs1, xs2, ctx)), - Some(_) | None => None, - } -} - -pub fn cycles_after_issue( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, -) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("MAXPOOL") => Some(maxpool::MaxPool::latency(xs1, xs2)), - Some(_) | None => None, - } -} diff --git a/examples/balls/maxpool/emu/src/lib.rs b/examples/balls/maxpool/emu/src/lib.rs deleted file mode 100644 index c1c40e7f1..000000000 --- a/examples/balls/maxpool/emu/src/lib.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub(crate) use crate::inst::{decode, instruction}; - -mod maxpool; - -mod dispatch; - -pub use dispatch::{cycles_after_issue, execute_known}; diff --git a/examples/balls/maxpool/emu/src/maxpool.rs b/examples/balls/maxpool/emu/src/maxpool.rs deleted file mode 100644 index 4f5dbf2bb..000000000 --- a/examples/balls/maxpool/emu/src/maxpool.rs +++ /dev/null @@ -1,168 +0,0 @@ -use super::super::bank::{bank_lines, bank_row_bytes}; -use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; -use super::instruction::{BallInstruction, ExecContext}; - -pub struct MaxPool; - -fn legal_geometry( - input_side: u64, - output_side: u64, - kernel: u64, - stride: u64, - padding: u64, - input_base: u64, - output_base: u64, - output_stride: u64, - start_row: u64, - start_col: u64, -) -> bool { - if input_side == 0 - || output_side == 0 - || kernel == 0 - || stride == 0 - || output_stride < output_side - { - return false; - } - - let padded_side = input_side + 2 * padding; - let last_row = start_row + (output_side - 1) * stride + kernel - 1; - let last_col = start_col + (output_side - 1) * stride + kernel - 1; - if kernel + start_row > padded_side - || kernel + start_col > padded_side - || last_row >= padded_side - || last_col >= padded_side - || output_base + (output_side - 1) * output_stride + output_side > bank_lines() as u64 - { - return false; - } - - let input_end = padding + input_side; - let reads_input = start_row < input_end - && start_col < input_end - && last_row >= padding - && last_col >= padding; - if !reads_input { - return true; - } - - let max_row = last_row.min(input_end - 1) - padding; - let max_col = last_col.min(input_end - 1) - padding; - input_base + max_row * input_side + max_col < bank_lines() as u64 -} - -impl BallInstruction for MaxPool { - fn exec(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { - let input_bank = rs1_b0(xs1); - let output_bank = rs1_b2(xs1); - let iter = rs1_iter(xs1) as usize; - let input_side = (xs2 & 0xf) as usize; - let output_side = ((xs2 >> 4) & 0xf) as usize; - let kernel = ((xs2 >> 8) & 0xf) as usize; - let stride = ((xs2 >> 12) & 0xf) as usize; - let padding = ((xs2 >> 16) & 0xf) as usize; - let input_base = ((xs2 >> 20) & 0x3f) as usize; - let output_base = ((xs2 >> 26) & 0x3f) as usize; - let output_stride = ((xs2 >> 32) & 0x3f) as usize; - let start_row = ((xs2 >> 38) & 0xf) as usize; - let start_col = ((xs2 >> 42) & 0xf) as usize; - - if rs1_b1(xs1) != 0 { - panic!("maxpool: input bank 1 must be zero"); - } - if input_bank == output_bank { - panic!("maxpool: input and output banks must differ"); - } - for bank in [input_bank, output_bank] { - if !ctx.config(bank).allocated || ctx.config(bank).cols != 1 { - panic!("maxpool: each operand must occupy one allocated bank"); - } - } - if xs2 >> 46 != 0 - || output_side * output_side != iter - || !legal_geometry( - input_side as u64, - output_side as u64, - kernel as u64, - stride as u64, - padding as u64, - input_base as u64, - output_base as u64, - output_stride as u64, - start_row as u64, - start_col as u64, - ) - { - panic!("maxpool: illegal square pooling geometry"); - } - - let pi = pbank(ctx, input_bank); - let po = pbank(ctx, output_bank); - let (input, output) = ctx.banks.read_write(pi, po); - for output_y in 0..output_side { - for output_x in 0..output_side { - let output_offset = - (output_base + output_y * output_stride + output_x) * bank_row_bytes(); - let mut maximum = [-128i8; 16]; - for kernel_y in 0..kernel { - for kernel_x in 0..kernel { - let input_y = - (output_y * stride + kernel_y + start_row) as isize - padding as isize; - let input_x = - (output_x * stride + kernel_x + start_col) as isize - padding as isize; - if input_y < 0 - || input_x < 0 - || input_y >= input_side as isize - || input_x >= input_side as isize - { - continue; - } - let input_offset = - (input_base + input_y as usize * input_side + input_x as usize) - * bank_row_bytes(); - for lane in 0..16 { - maximum[lane] = maximum[lane].max(input[input_offset + lane] as i8); - } - } - } - for lane in 0..16 { - output[output_offset + lane] = maximum[lane] as u8; - } - } - } - 0 - } - - fn latency(xs1: u64, xs2: u64) -> u64 { - let iter = rs1_iter(xs1); - let input_side = xs2 & 0xf; - let output_side = (xs2 >> 4) & 0xf; - let kernel = (xs2 >> 8) & 0xf; - let stride = (xs2 >> 12) & 0xf; - let padding = (xs2 >> 16) & 0xf; - let input_base = (xs2 >> 20) & 0x3f; - let output_base = (xs2 >> 26) & 0x3f; - let output_stride = (xs2 >> 32) & 0x3f; - let start_row = (xs2 >> 38) & 0xf; - let start_col = (xs2 >> 42) & 0xf; - if rs1_b1(xs1) != 0 - || xs2 >> 46 != 0 - || output_side * output_side != iter - || !legal_geometry( - input_side, - output_side, - kernel, - stride, - padding, - input_base, - output_base, - output_stride, - start_row, - start_col, - ) - { - panic!("maxpool: illegal encoding"); - } - iter * (kernel * kernel * 2 + 2) - } -} diff --git a/examples/balls/maxpool/verify/casegen/src/casegen.rs b/examples/balls/maxpool/verify/casegen/src/casegen.rs deleted file mode 100644 index 64b54d950..000000000 --- a/examples/balls/maxpool/verify/casegen/src/casegen.rs +++ /dev/null @@ -1,565 +0,0 @@ -use crate::model; - -pub const MAX_INPUT_WORDS: usize = 64; -pub const MAX_DST: usize = 64; -pub const NUM_CASES: u32 = 9; - -#[repr(C)] -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct MaxPoolCmd { - pub bid: u32, - pub iter: u32, - pub op1_bank: u32, - pub wr_bank: u32, - pub op1_col: u32, - pub wr_col: u32, - pub rob_id: u32, - pub rs1_lo: u32, - pub rs1_hi: u32, - pub rs2_lo: u32, - pub rs2_hi: u32, - pub input_base: u32, - pub output_base: u32, - pub output_stride: u32, - pub input_side: u32, - pub output_side: u32, - pub kernel: u32, - pub stride: u32, - pub padding: u32, - pub start_row: u32, - pub start_col: u32, - pub num_input_words: u32, - pub num_dst_words: u32, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct MaxPoolCase { - pub cmd: MaxPoolCmd, - pub input_words: [u128; MAX_INPUT_WORDS], - pub input_addr: [u32; MAX_INPUT_WORDS], - pub dst_words: [u128; MAX_DST], - pub dst_addr: [u32; MAX_DST], -} - -impl MaxPoolCase { - pub fn input_lo(&self, index: usize) -> u64 { - self.input_words[index] as u64 - } - - pub fn input_hi(&self, index: usize) -> u64 { - (self.input_words[index] >> 64) as u64 - } - - pub fn dst_lo(&self, index: usize) -> u64 { - self.dst_words[index] as u64 - } - - pub fn dst_hi(&self, index: usize) -> u64 { - (self.dst_words[index] >> 64) as u64 - } -} - -#[derive(Clone, Copy)] -struct CaseSpec { - geo: model::Geo, - op1_bank: u32, - wr_bank: u32, - rob_id: u32, - ctest_input: bool, - seed: u32, -} - -fn field(val: u32, lo: u32, hi: u32) -> u64 { - let width = hi - lo + 1; - let mask = if width == 64 { - u64::MAX - } else { - (1u64 << width) - 1 - }; - (u64::from(val) & mask) << lo -} - -fn encode_rs1(input_bank: u32, output_bank: u32, iter: u32) -> u64 { - field(input_bank, 0, 9) | field(0, 10, 19) | field(output_bank, 20, 29) | field(iter, 30, 63) -} - -fn encode_rs2(geo: &model::Geo) -> u64 { - field(geo.input_side as u32, 0, 3) - | field(geo.output_side as u32, 4, 7) - | field(geo.kernel as u32, 8, 11) - | field(geo.stride as u32, 12, 15) - | field(geo.padding as u32, 16, 19) - | field(geo.input_base as u32, 20, 25) - | field(geo.output_base as u32, 26, 31) - | field(geo.output_stride as u32, 32, 37) - | field(geo.start_row as u32, 38, 41) - | field(geo.start_col as u32, 42, 45) -} - -fn split_rs(value: u64) -> (u32, u32) { - (value as u32, (value >> 32) as u32) -} - -fn validate_cmd(op1_bank: u32, wr_bank: u32, iter: u32, rs1: u64, rs2: u64) { - if iter == 0 { - panic!("maxpool: iter must be positive"); - } - if op1_bank == wr_bank { - panic!("maxpool: input and output banks must differ"); - } - if (rs1 & 0x3ff) != u64::from(op1_bank) { - panic!("maxpool: rs1 bank0 mismatch"); - } - if ((rs1 >> 10) & 0x3ff) != 0 { - panic!("maxpool: input bank 1 must be zero"); - } - if ((rs1 >> 20) & 0x3ff) != u64::from(wr_bank) { - panic!("maxpool: rs1 bank2 mismatch"); - } - if ((rs1 >> 30) & 0x3fff_ffff) != u64::from(iter) { - panic!("maxpool: rs1 iter mismatch"); - } - if (rs2 >> 46) != 0 { - panic!("maxpool: reserved rs2 bits must be zero"); - } -} - -fn ctest_tile(geo: &model::Geo) -> Vec { - let tile = geo.input_side * geo.input_side; - let mut bytes = vec![0u8; tile * model::BANK_ROW_BYTES]; - for position in 0..tile { - for channel in 0..model::BANK_ROW_BYTES { - bytes[position * model::BANK_ROW_BYTES + channel] = - (((position * 37 + channel * 19) & 255) as i32 - 128) as u8; - } - } - bytes -} - -fn seeded_tile(geo: &model::Geo, seed: u32) -> Vec { - let tile = geo.input_side * geo.input_side; - let mut bytes = vec![0u8; tile * model::BANK_ROW_BYTES]; - for position in 0..tile { - for channel in 0..model::BANK_ROW_BYTES { - let mix = seed - .wrapping_mul(0x9E37_79B9) - .wrapping_add(position as u32 * 131) - .wrapping_add(channel as u32 * 17); - bytes[position * model::BANK_ROW_BYTES + channel] = ((mix & 255) as i32 - 128) as u8; - } - } - bytes -} - -fn copy_words(src: &[u128]) -> [u128; N] { - if src.len() > N { - panic!("maxpool: {} words exceed buffer {N}", src.len()); - } - let mut out = [0u128; N]; - out[..src.len()].copy_from_slice(src); - out -} - -fn copy_addrs(src: &[u32]) -> [u32; N] { - if src.len() > N { - panic!("maxpool: {} addrs exceed buffer {N}", src.len()); - } - let mut out = [0u32; N]; - out[..src.len()].copy_from_slice(src); - out -} - -fn build_case(spec: CaseSpec, bid: u32) -> MaxPoolCase { - model::validate(&spec.geo); - let iter = model::iter_for(&spec.geo); - let input_bytes = if spec.ctest_input { - ctest_tile(&spec.geo) - } else { - seeded_tile(&spec.geo, spec.seed) - }; - - let mut referenced = vec![false; spec.geo.input_side * spec.geo.input_side]; - for output_y in 0..spec.geo.output_side { - for output_x in 0..spec.geo.output_side { - for kernel_y in 0..spec.geo.kernel { - for kernel_x in 0..spec.geo.kernel { - let y = output_y * spec.geo.stride + kernel_y + spec.geo.start_row; - let x = output_x * spec.geo.stride + kernel_x + spec.geo.start_col; - if y >= spec.geo.padding - && x >= spec.geo.padding - && y < spec.geo.padding + spec.geo.input_side - && x < spec.geo.padding + spec.geo.input_side - { - referenced[(y - spec.geo.padding) * spec.geo.input_side - + (x - spec.geo.padding)] = true; - } - } - } - } - } - - let mut input_addrs = Vec::new(); - let mut input_words = Vec::new(); - for y in 0..spec.geo.input_side { - for x in 0..spec.geo.input_side { - if !referenced[y * spec.geo.input_side + x] { - continue; - } - let addr = (spec.geo.input_base + y * spec.geo.input_side + x) as u32; - let row_off = (y * spec.geo.input_side + x) * model::BANK_ROW_BYTES; - let mut row = [0u8; model::BANK_ROW_BYTES]; - row.copy_from_slice(&input_bytes[row_off..row_off + model::BANK_ROW_BYTES]); - input_addrs.push(addr); - input_words.push(model::pack_row(&row)); - } - } - - let gold = model::maxpool_rows(&input_bytes, &spec.geo); - let dst_addrs: Vec = gold.iter().map(|(addr, _)| *addr).collect(); - let dst_words: Vec = gold.iter().map(|(_, row)| model::pack_row(row)).collect(); - - if dst_addrs.len() as u32 != iter { - panic!("maxpool: gold rows {} != iter {iter}", dst_addrs.len()); - } - - let rs1 = encode_rs1(spec.op1_bank, spec.wr_bank, iter); - let rs2 = encode_rs2(&spec.geo); - validate_cmd(spec.op1_bank, spec.wr_bank, iter, rs1, rs2); - - let (rs1_lo, rs1_hi) = split_rs(rs1); - let (rs2_lo, rs2_hi) = split_rs(rs2); - - MaxPoolCase { - cmd: MaxPoolCmd { - bid, - iter, - op1_bank: spec.op1_bank, - wr_bank: spec.wr_bank, - op1_col: 1, - wr_col: 1, - rob_id: spec.rob_id, - rs1_lo, - rs1_hi, - rs2_lo, - rs2_hi, - input_base: spec.geo.input_base as u32, - output_base: spec.geo.output_base as u32, - output_stride: spec.geo.output_stride as u32, - input_side: spec.geo.input_side as u32, - output_side: spec.geo.output_side as u32, - kernel: spec.geo.kernel as u32, - stride: spec.geo.stride as u32, - padding: spec.geo.padding as u32, - start_row: spec.geo.start_row as u32, - start_col: spec.geo.start_col as u32, - num_input_words: input_words.len() as u32, - num_dst_words: iter, - }, - input_words: copy_words(&input_words), - input_addr: copy_addrs::(&input_addrs), - dst_words: copy_words(&dst_words), - dst_addr: copy_addrs::(&dst_addrs), - } -} - -fn spec_for(index: u32) -> CaseSpec { - match index { - 0 => CaseSpec { - geo: model::Geo { - input_side: 6, - output_side: 3, - kernel: 2, - stride: 2, - padding: 0, - input_base: 7, - output_base: 11, - output_stride: 7, - start_row: 0, - start_col: 0, - }, - op1_bank: 0, - wr_bank: 1, - rob_id: 2, - ctest_input: true, - seed: 0, - }, - 1 => CaseSpec { - geo: model::Geo { - input_side: 4, - output_side: 3, - kernel: 2, - stride: 1, - padding: 0, - input_base: 0, - output_base: 20, - output_stride: 4, - start_row: 0, - start_col: 0, - }, - op1_bank: 0, - wr_bank: 1, - rob_id: 3, - ctest_input: false, - seed: 1, - }, - 2 => CaseSpec { - geo: model::Geo { - input_side: 4, - output_side: 3, - kernel: 2, - stride: 1, - padding: 1, - input_base: 0, - output_base: 30, - output_stride: 4, - start_row: 0, - start_col: 0, - }, - op1_bank: 0, - wr_bank: 1, - rob_id: 4, - ctest_input: false, - seed: 2, - }, - 3 => CaseSpec { - geo: model::Geo { - input_side: 6, - output_side: 3, - kernel: 2, - stride: 2, - padding: 1, - input_base: 0, - output_base: 40, - output_stride: 4, - start_row: 0, - start_col: 0, - }, - op1_bank: 0, - wr_bank: 1, - rob_id: 5, - ctest_input: false, - seed: 3, - }, - 4 => CaseSpec { - geo: model::Geo { - input_side: 5, - output_side: 3, - kernel: 3, - stride: 1, - padding: 0, - input_base: 0, - output_base: 50, - output_stride: 4, - start_row: 0, - start_col: 0, - }, - op1_bank: 0, - wr_bank: 1, - rob_id: 6, - ctest_input: false, - seed: 4, - }, - 5 => CaseSpec { - geo: model::Geo { - input_side: 7, - output_side: 3, - kernel: 3, - stride: 2, - padding: 0, - input_base: 0, - output_base: 0, - output_stride: 7, - start_row: 0, - start_col: 0, - }, - op1_bank: 0, - wr_bank: 1, - rob_id: 7, - ctest_input: false, - seed: 5, - }, - 6 => CaseSpec { - geo: model::Geo { - input_side: 5, - output_side: 3, - kernel: 3, - stride: 1, - padding: 1, - input_base: 0, - output_base: 15, - output_stride: 4, - start_row: 0, - start_col: 0, - }, - op1_bank: 0, - wr_bank: 1, - rob_id: 8, - ctest_input: false, - seed: 6, - }, - 7 => CaseSpec { - geo: model::Geo { - input_side: 7, - output_side: 3, - kernel: 3, - stride: 2, - padding: 1, - input_base: 0, - output_base: 25, - output_stride: 7, - start_row: 0, - start_col: 0, - }, - op1_bank: 0, - wr_bank: 1, - rob_id: 9, - ctest_input: false, - seed: 7, - }, - 8 => CaseSpec { - geo: model::Geo { - input_side: 7, - output_side: 1, - kernel: 1, - stride: 1, - padding: 0, - input_base: 63, - output_base: 0, - output_stride: 1, - start_row: 0, - start_col: 0, - }, - op1_bank: 0, - wr_bank: 1, - rob_id: 10, - ctest_input: false, - seed: 8, - }, - _ => panic!("maxpool: unsupported directed case {index}"), - } -} - -pub fn gen_case(index: u32, bid: u32) -> MaxPoolCase { - if index >= NUM_CASES { - panic!("maxpool: unsupported directed case {index}"); - } - build_case(spec_for(index), bid) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn case_zero_matches_ctest() { - let case = gen_case(0, 6); - assert_eq!(case.cmd.bid, 6); - assert_eq!(case.cmd.iter, 9); - assert_eq!(case.cmd.input_side, 6); - assert_eq!(case.cmd.output_side, 3); - assert_eq!(case.cmd.kernel, 2); - assert_eq!(case.cmd.stride, 2); - assert_eq!(case.cmd.padding, 0); - assert_eq!(case.cmd.input_base, 7); - assert_eq!(case.cmd.output_base, 11); - assert_eq!(case.cmd.output_stride, 7); - assert_eq!(case.cmd.op1_bank, 0); - assert_eq!(case.cmd.wr_bank, 1); - assert_eq!(case.cmd.num_input_words, 36); - assert_eq!(case.cmd.num_dst_words, 9); - let rs2 = u64::from(case.cmd.rs2_lo) | (u64::from(case.cmd.rs2_hi) << 32); - assert_eq!(rs2 >> 46, 0); - let rs1 = u64::from(case.cmd.rs1_lo) | (u64::from(case.cmd.rs1_hi) << 32); - assert_eq!(rs1 & 0x3ff, 0); - assert_eq!((rs1 >> 10) & 0x3ff, 0); - assert_eq!((rs1 >> 20) & 0x3ff, 1); - assert_eq!((rs1 >> 30) & 0xffff_ffff, 9); - } - - #[test] - fn cases_hit_cover_bins() { - let mut k2 = false; - let mut k3 = false; - let mut k1 = false; - let mut s1 = false; - let mut s2 = false; - let mut p0 = false; - let mut p1 = false; - for index in 0..NUM_CASES { - let case = gen_case(index, 6); - assert_eq!(case.cmd.bid, 6); - assert_eq!(case.cmd.op1_col, 1); - assert_eq!(case.cmd.wr_col, 1); - assert_ne!(case.cmd.op1_bank, case.cmd.wr_bank); - assert_eq!(case.cmd.num_dst_words, case.cmd.iter); - assert_eq!(case.cmd.iter, case.cmd.output_side * case.cmd.output_side); - match case.cmd.kernel { - 1 => k1 = true, - 2 => k2 = true, - 3 => k3 = true, - other => panic!("unexpected kernel {other}"), - } - match case.cmd.stride { - 1 => s1 = true, - 2 => s2 = true, - other => panic!("unexpected stride {other}"), - } - match case.cmd.padding { - 0 => p0 = true, - 1 => p1 = true, - other => panic!("unexpected padding {other}"), - } - } - assert!(k1 && k2 && k3 && s1 && s2 && p0 && p1); - } - - #[test] - #[should_panic(expected = "unsupported directed case")] - fn unknown_index_panics() { - let _ = gen_case(NUM_CASES, 6); - } - - #[test] - #[should_panic(expected = "illegal square pooling geometry")] - fn illegal_geometry_panics() { - model::validate(&model::Geo { - input_side: 2, - output_side: 3, - kernel: 2, - stride: 1, - padding: 0, - input_base: 0, - output_base: 0, - output_stride: 4, - start_row: 0, - start_col: 0, - }); - } - - #[test] - fn gold_matches_manual_window() { - let geo = model::Geo { - input_side: 2, - output_side: 1, - kernel: 2, - stride: 1, - padding: 0, - input_base: 0, - output_base: 5, - output_stride: 1, - start_row: 0, - start_col: 0, - }; - let mut input = vec![0u8; 2 * 2 * model::BANK_ROW_BYTES]; - for lane in 0..model::BANK_ROW_BYTES { - input[lane] = 1; - input[model::BANK_ROW_BYTES + lane] = 3; - input[2 * model::BANK_ROW_BYTES + lane] = 5; - input[3 * model::BANK_ROW_BYTES + lane] = 2; - } - let gold = model::maxpool_rows(&input, &geo); - assert_eq!(gold.len(), 1); - assert_eq!(gold[0].0, 5); - assert_eq!(gold[0].1[0], 5); - } -} diff --git a/examples/balls/maxpool/verify/casegen/src/lib.rs b/examples/balls/maxpool/verify/casegen/src/lib.rs deleted file mode 100644 index fd6b84ef8..000000000 --- a/examples/balls/maxpool/verify/casegen/src/lib.rs +++ /dev/null @@ -1,160 +0,0 @@ -mod casegen; -mod model; - -use std::cell::RefCell; - -use casegen::{MaxPoolCase, MaxPoolCmd, MAX_DST, MAX_INPUT_WORDS}; - -thread_local! { - static CURRENT: RefCell> = const { RefCell::new(None) }; -} - -#[no_mangle] -pub extern "C" fn maxpool_case_load(index: u32, bid: u32) -> i32 { - let case = casegen::gen_case(index, bid); - CURRENT.with(|current| *current.borrow_mut() = Some(case)); - 0 -} - -fn current_case R, R>(f: F) -> R { - CURRENT.with(|current| match current.borrow().as_ref() { - Some(case) => f(case), - None => panic!("maxpool_case: no case loaded; call maxpool_case_load first"), - }) -} - -#[no_mangle] -pub extern "C" fn maxpool_case_cmd(out_ptr: *mut MaxPoolCmd) { - if out_ptr.is_null() { - panic!("maxpool_case_cmd: null out_ptr"); - } - current_case(|case| unsafe { - *out_ptr = case.cmd; - }); -} - -fn idx(index: u32, limit: u32, what: &str) -> usize { - if index >= limit { - panic!("maxpool_case_{what}: index {index} out of range {limit}"); - } - index as usize -} - -#[no_mangle] -pub extern "C" fn maxpool_case_input_word_lo(word_index: u32) -> u64 { - current_case(|case| { - let i = idx(word_index, case.cmd.num_input_words, "input_word_lo"); - if i >= MAX_INPUT_WORDS { - panic!("maxpool_case_input_word_lo: word_index out of range"); - } - case.input_lo(i) - }) -} - -#[no_mangle] -pub extern "C" fn maxpool_case_input_word_hi(word_index: u32) -> u64 { - current_case(|case| { - let i = idx(word_index, case.cmd.num_input_words, "input_word_hi"); - if i >= MAX_INPUT_WORDS { - panic!("maxpool_case_input_word_hi: word_index out of range"); - } - case.input_hi(i) - }) -} - -#[no_mangle] -pub extern "C" fn maxpool_case_input_addr(word_index: u32) -> u32 { - current_case(|case| { - let i = idx(word_index, case.cmd.num_input_words, "input_addr"); - if i >= MAX_INPUT_WORDS { - panic!("maxpool_case_input_addr: word_index out of range"); - } - case.input_addr[i] - }) -} - -#[no_mangle] -pub extern "C" fn maxpool_case_dst_word_lo(word_index: u32) -> u64 { - current_case(|case| { - let i = idx(word_index, case.cmd.num_dst_words, "dst_word_lo"); - if i >= MAX_DST { - panic!("maxpool_case_dst_word_lo: word_index out of range"); - } - case.dst_lo(i) - }) -} - -#[no_mangle] -pub extern "C" fn maxpool_case_dst_word_hi(word_index: u32) -> u64 { - current_case(|case| { - let i = idx(word_index, case.cmd.num_dst_words, "dst_word_hi"); - if i >= MAX_DST { - panic!("maxpool_case_dst_word_hi: word_index out of range"); - } - case.dst_hi(i) - }) -} - -#[no_mangle] -pub extern "C" fn maxpool_case_dst_addr(word_index: u32) -> u32 { - current_case(|case| { - let i = idx(word_index, case.cmd.num_dst_words, "dst_addr"); - if i >= MAX_DST { - panic!("maxpool_case_dst_addr: word_index out of range"); - } - case.dst_addr[i] - }) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn dpi_load_then_cmd_and_words() { - assert_eq!(maxpool_case_load(0, 6), 0); - let mut cmd = MaxPoolCmd { - bid: 0, - iter: 0, - op1_bank: 0, - wr_bank: 0, - op1_col: 0, - wr_col: 0, - rob_id: 0, - rs1_lo: 0, - rs1_hi: 0, - rs2_lo: 0, - rs2_hi: 0, - input_base: 0, - output_base: 0, - output_stride: 0, - input_side: 0, - output_side: 0, - kernel: 0, - stride: 0, - padding: 0, - start_row: 0, - start_col: 0, - num_input_words: 0, - num_dst_words: 0, - }; - maxpool_case_cmd(&mut cmd as *mut MaxPoolCmd); - assert_eq!(cmd.bid, 6); - assert_eq!(cmd.iter, 9); - assert_eq!(cmd.num_input_words, 36); - assert_eq!(cmd.num_dst_words, 9); - let _ = maxpool_case_input_word_lo(0); - let _ = maxpool_case_input_word_hi(0); - let _ = maxpool_case_input_addr(0); - let _ = maxpool_case_dst_word_lo(0); - let _ = maxpool_case_dst_addr(0); - assert_eq!(casegen::NUM_CASES, 9); - } - - #[test] - #[should_panic(expected = "no case loaded")] - fn current_case_panics_without_load() { - CURRENT.with(|c| *c.borrow_mut() = None); - current_case(|_| ()); - } -} diff --git a/examples/balls/maxpool/verify/casegen/src/model.rs b/examples/balls/maxpool/verify/casegen/src/model.rs deleted file mode 100644 index e9a5fe01b..000000000 --- a/examples/balls/maxpool/verify/casegen/src/model.rs +++ /dev/null @@ -1,104 +0,0 @@ -pub const BANK_LINES: usize = 64; -pub const BANK_ROW_BYTES: usize = 16; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct Geo { - pub input_side: usize, - pub output_side: usize, - pub kernel: usize, - pub stride: usize, - pub padding: usize, - pub input_base: usize, - pub output_base: usize, - pub output_stride: usize, - pub start_row: usize, - pub start_col: usize, -} - -pub fn validate(geo: &Geo) { - let g = geo; - let dimensions_valid = - g.input_side != 0 && g.output_side != 0 && g.kernel != 0 && g.stride != 0; - if !dimensions_valid { - panic!("maxpool: illegal square pooling geometry"); - } - let last_row = g.start_row + (g.output_side - 1) * g.stride + g.kernel - 1; - let last_col = g.start_col + (g.output_side - 1) * g.stride + g.kernel - 1; - let input_end = g.padding + g.input_side; - let reads_input = g.start_row < input_end - && g.start_col < input_end - && last_row >= g.padding - && last_col >= g.padding; - let input_fits = !reads_input - || g.input_base - + (last_row.min(input_end - 1) - g.padding) * g.input_side - + (last_col.min(input_end - 1) - g.padding) - < BANK_LINES; - if g.output_stride < g.output_side - || !input_fits - || g.output_base + (g.output_side - 1) * g.output_stride + g.output_side > BANK_LINES - || g.input_side + 2 * g.padding < g.kernel + g.start_row - || g.input_side + 2 * g.padding < g.kernel + g.start_col - || g.start_row + (g.output_side - 1) * g.stride + g.kernel > g.input_side + 2 * g.padding - || g.start_col + (g.output_side - 1) * g.stride + g.kernel > g.input_side + 2 * g.padding - { - panic!("maxpool: illegal square pooling geometry"); - } -} - -pub fn iter_for(geo: &Geo) -> u32 { - let iter = geo.output_side * geo.output_side; - if iter == 0 { - panic!("maxpool: iter must be positive"); - } - iter as u32 -} - -pub fn maxpool_rows(input: &[u8], geo: &Geo) -> Vec<(u32, [u8; BANK_ROW_BYTES])> { - validate(geo); - let tile = geo.input_side * geo.input_side; - if input.len() != tile * BANK_ROW_BYTES { - panic!("maxpool: input bytes {} != tile {} rows", input.len(), tile); - } - - let mut out = Vec::with_capacity(geo.output_side * geo.output_side); - for output_y in 0..geo.output_side { - for output_x in 0..geo.output_side { - let mut maximum = [0x80u8; BANK_ROW_BYTES]; - for kernel_y in 0..geo.kernel { - for kernel_x in 0..geo.kernel { - let input_y = output_y * geo.stride + kernel_y + geo.start_row; - let input_x = output_x * geo.stride + kernel_x + geo.start_col; - let sy = input_y as isize - geo.padding as isize; - let sx = input_x as isize - geo.padding as isize; - if sy < 0 - || sx < 0 - || sy >= geo.input_side as isize - || sx >= geo.input_side as isize - { - continue; - } - let row = (sy as usize) * geo.input_side + sx as usize; - let off = row * BANK_ROW_BYTES; - for lane in 0..BANK_ROW_BYTES { - let v = input[off + lane] as i8; - if v > maximum[lane] as i8 { - maximum[lane] = v as u8; - } - } - } - } - let addr = (geo.output_base + output_y * geo.output_stride + output_x) as u32; - out.push((addr, maximum)); - } - } - out -} - -pub fn pack_row(bytes: &[u8; BANK_ROW_BYTES]) -> u128 { - let mut word = 0u128; - for (lane, byte) in bytes.iter().enumerate() { - word |= u128::from(*byte) << (lane * 8); - } - word -} diff --git a/examples/balls/maxpool/verify/src/common/maxpool_defs.svh b/examples/balls/maxpool/verify/src/common/maxpool_defs.svh deleted file mode 100644 index a08b4a7a9..000000000 --- a/examples/balls/maxpool/verify/src/common/maxpool_defs.svh +++ /dev/null @@ -1,55 +0,0 @@ -typedef struct { - int unsigned bid; - int unsigned iter; - int unsigned op1_bank; - int unsigned wr_bank; - int unsigned op1_col; - int unsigned wr_col; - int unsigned rob_id; - int unsigned rs1_lo; - int unsigned rs1_hi; - int unsigned rs2_lo; - int unsigned rs2_hi; - int unsigned input_base; - int unsigned output_base; - int unsigned output_stride; - int unsigned input_side; - int unsigned output_side; - int unsigned kernel; - int unsigned stride; - int unsigned padding; - int unsigned start_row; - int unsigned start_col; - int unsigned num_input_words; - int unsigned num_dst_words; -} maxpool_cmd_dpi_t; - -import "DPI-C" function int maxpool_case_load( - input int unsigned index, - input int unsigned bid -); -import "DPI-C" function void maxpool_case_cmd(output maxpool_cmd_dpi_t cmd); -import "DPI-C" function longint unsigned maxpool_case_input_word_lo(input int unsigned word_index); -import "DPI-C" function longint unsigned maxpool_case_input_word_hi(input int unsigned word_index); -import "DPI-C" function int unsigned maxpool_case_input_addr(input int unsigned word_index); -import "DPI-C" function longint unsigned maxpool_case_dst_word_lo(input int unsigned word_index); -import "DPI-C" function longint unsigned maxpool_case_dst_word_hi(input int unsigned word_index); -import "DPI-C" function int unsigned maxpool_case_dst_addr(input int unsigned word_index); - -`ifndef MAXPOOL_FUNCT7 -`error "MAXPOOL_FUNCT7 must be provided by the selected Core ballISA" -`endif - -localparam int MAXPOOL_CORE_FUNCT7 = `MAXPOOL_FUNCT7; -localparam int MAXPOOL_MAX_INPUT_WORDS = 64; -localparam int MAXPOOL_MAX_DST_WORDS = 64; -localparam int MAXPOOL_NUM_CASES = 8; -localparam int MAXPOOL_TIMEOUT_CYCLES = 20000; - -function automatic int unsigned maxpool_require_bid(); - int unsigned bid; - if (!$value$plusargs("BID=%d", bid)) begin - `uvm_fatal("BID", "missing required plusarg +BID=") - end - return bid; -endfunction diff --git a/examples/balls/maxpool/verify/src/common/maxpool_items.svh b/examples/balls/maxpool/verify/src/common/maxpool_items.svh deleted file mode 100644 index 1722971e5..000000000 --- a/examples/balls/maxpool/verify/src/common/maxpool_items.svh +++ /dev/null @@ -1,145 +0,0 @@ -class maxpool_cmd_item extends bb_blink_cmd_item; - `uvm_object_utils(maxpool_cmd_item) - - bit [31:0] input_base; - bit [31:0] output_base; - bit [31:0] output_stride; - bit [31:0] input_side; - bit [31:0] output_side; - bit [31:0] kernel; - bit [31:0] stride; - bit [31:0] padding; - bit [31:0] start_row; - bit [31:0] start_col; - bit [31:0] num_input_words; - bit [31:0] num_dst_words; - bit [127:0] input_words[MAXPOOL_MAX_INPUT_WORDS]; - bit [31:0] input_addr[MAXPOOL_MAX_INPUT_WORDS]; - bit [127:0] dst_words[MAXPOOL_MAX_DST_WORDS]; - bit [31:0] dst_addr[MAXPOOL_MAX_DST_WORDS]; - - constraint legal_c { - funct7 == MAXPOOL_CORE_FUNCT7[6:0]; - op1_en == 1'b1; - op2_en == 1'b0; - wr_spad_en == 1'b1; - op1_from_spad == 1'b1; - op2_from_spad == 1'b0; - op1_col == 5'd1; - op2_bank == 5'd0; - op2_col == 5'd0; - wr_col == 5'd1; - meta_bank == 5'd0; - is_sub == 1'b0; - sub_rob_id == 8'h00; - special == 64'd0; - } - - function new(string name = "maxpool_cmd_item"); - super.new(name); - endfunction - - function void load_rust_case(int unsigned index, int unsigned bid); - maxpool_cmd_dpi_t cmd; - int unsigned rc; - longint unsigned w_lo; - longint unsigned w_hi; - int i; - - rc = maxpool_case_load(index, bid); - if (rc != 0) begin - `uvm_fatal("CASE", $sformatf("maxpool_case_load returned %0d for index %0d", rc, index)) - end - maxpool_case_cmd(cmd); - - this.bid = cmd.bid[4:0]; - iter = cmd.iter; - op1_bank = cmd.op1_bank[4:0]; - wr_bank = cmd.wr_bank[4:0]; - op1_col = cmd.op1_col[4:0]; - wr_col = cmd.wr_col[4:0]; - rob_id = cmd.rob_id[3:0]; - rs1 = {cmd.rs1_hi, cmd.rs1_lo}; - rs2 = {cmd.rs2_hi, cmd.rs2_lo}; - input_base = cmd.input_base; - output_base = cmd.output_base; - output_stride = cmd.output_stride; - input_side = cmd.input_side; - output_side = cmd.output_side; - kernel = cmd.kernel; - stride = cmd.stride; - padding = cmd.padding; - start_row = cmd.start_row; - start_col = cmd.start_col; - num_input_words = cmd.num_input_words; - num_dst_words = cmd.num_dst_words; - funct7 = MAXPOOL_CORE_FUNCT7[6:0]; - op1_en = 1'b1; - op2_en = 1'b0; - wr_spad_en = 1'b1; - op1_from_spad = 1'b1; - op2_from_spad = 1'b0; - op2_bank = 5'd0; - op2_col = 5'd0; - meta_bank = 5'd0; - special = 64'd0; - is_sub = 1'b0; - sub_rob_id = 8'h00; - - if (rs1[19:10] != 0) `uvm_fatal("CASE", "MAXPOOL reserves input bank 1") - if (op1_col != 1 || wr_col != 1) - `uvm_fatal("CASE", "MAXPOOL requires one input bank and one output bank") - if (op1_bank == wr_bank) `uvm_fatal("CASE", "MAXPOOL input and output banks must differ") - if (rs2[63:46] != 0) `uvm_fatal("CASE", "MAXPOOL reserved rs2 bits must be zero") - if (num_dst_words != iter) - `uvm_fatal("CASE", $sformatf("num_dst_words %0d != iter %0d", num_dst_words, iter)) - if (iter != output_side * output_side) - `uvm_fatal("CASE", $sformatf("iter %0d != output_side^2 %0d", iter, output_side * output_side - )) - if (num_input_words == 0 || num_input_words > MAXPOOL_MAX_INPUT_WORDS) - `uvm_fatal("CASE", $sformatf("num_input_words out of range: %0d", num_input_words)) - if (num_dst_words == 0 || num_dst_words > MAXPOOL_MAX_DST_WORDS) - `uvm_fatal("CASE", $sformatf("num_dst_words out of range: %0d", num_dst_words)) - - for (i = 0; i < num_input_words; i++) begin - w_lo = maxpool_case_input_word_lo(i); - w_hi = maxpool_case_input_word_hi(i); - input_words[i] = {w_hi, w_lo}; - input_addr[i] = maxpool_case_input_addr(i); - end - for (i = 0; i < num_dst_words; i++) begin - w_lo = maxpool_case_dst_word_lo(i); - w_hi = maxpool_case_dst_word_hi(i); - dst_words[i] = {w_hi, w_lo}; - dst_addr[i] = maxpool_case_dst_addr(i); - end - endfunction - - function void do_copy(uvm_object rhs); - maxpool_cmd_item rhs_; - super.do_copy(rhs); - if (!$cast(rhs_, rhs)) begin - `uvm_fatal("COPY", "rhs is not maxpool_cmd_item") - end - input_base = rhs_.input_base; - output_base = rhs_.output_base; - output_stride = rhs_.output_stride; - input_side = rhs_.input_side; - output_side = rhs_.output_side; - kernel = rhs_.kernel; - stride = rhs_.stride; - padding = rhs_.padding; - start_row = rhs_.start_row; - start_col = rhs_.start_col; - num_input_words = rhs_.num_input_words; - num_dst_words = rhs_.num_dst_words; - for (int i = 0; i < MAXPOOL_MAX_INPUT_WORDS; i++) begin - input_words[i] = rhs_.input_words[i]; - input_addr[i] = rhs_.input_addr[i]; - end - for (int i = 0; i < MAXPOOL_MAX_DST_WORDS; i++) begin - dst_words[i] = rhs_.dst_words[i]; - dst_addr[i] = rhs_.dst_addr[i]; - end - endfunction -endclass diff --git a/examples/balls/maxpool/verify/src/tb_top.sv b/examples/balls/maxpool/verify/src/tb_top.sv deleted file mode 100644 index 72ea12549..000000000 --- a/examples/balls/maxpool/verify/src/tb_top.sv +++ /dev/null @@ -1,86 +0,0 @@ -module tb_top; - import uvm_pkg::*; - import maxpool_pkg::*; - - bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); - - MaxPoolBall dut ( - .clock(intf.clock), - .reset(intf.reset), - .io_cmdReq_ready(intf.cmd_req_ready), - .io_cmdReq_valid(intf.cmd_req_valid), - .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), - .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), - .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), - .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), - .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), - .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), - .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), - .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), - .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), - .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank), - .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank), - .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank), - .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), - .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), - .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), - .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank), - .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), - .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), - .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), - .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), - .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), - .io_cmdResp_ready(intf.cmd_resp_ready), - .io_cmdResp_valid(intf.cmd_resp_valid), - .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), - .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), - .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), - .io_bankRead_0_bank_id(intf.bank_read_bank_id[0]), - .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), - .io_bankRead_0_group_id(intf.bank_read_group_id[0]), - .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), - .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), - .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), - .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), - .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), - .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), - .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), - .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), - .io_bankWrite_0_group_id(intf.bank_write_group_id[0]), - .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), - .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), - .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), - .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), - .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), - .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), - .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), - .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), - .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), - .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), - .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), - .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), - .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), - .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), - .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), - .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), - .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), - .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), - .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), - .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), - .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), - .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), - .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), - .io_subRobReq_ready(intf.sub_rob_req_ready) - ); - - initial begin - intf.clock = 1'b0; - intf.reset = 1'b1; - forever #5 intf.clock = ~intf.clock; - end - - initial begin - uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); - run_test("maxpool_ball_test"); - end -endmodule diff --git a/examples/balls/maxpool/workloads/ctests/CMakeLists.txt b/examples/balls/maxpool/workloads/ctests/CMakeLists.txt deleted file mode 100644 index b86255542..000000000 --- a/examples/balls/maxpool/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_buckyball_ctests(maxpool_window_test.c maxpool_offset_test.c) diff --git a/examples/balls/maxpool/workloads/isa/maxpool.h b/examples/balls/maxpool/workloads/isa/maxpool.h deleted file mode 100644 index a9f4477eb..000000000 --- a/examples/balls/maxpool/workloads/isa/maxpool.h +++ /dev/null @@ -1,20 +0,0 @@ -#ifndef _BB_MAXPOOL_H_ -#define _BB_MAXPOOL_H_ - -#include -#include - -#define bb_maxpool(input_bank, output_bank, input_side, output_side, kernel, \ - stride, padding, input_base, output_base, output_stride, \ - start_row, start_col) \ - BUCKYBALL_INSTRUCTION_R_R( \ - BB_BANK0(input_bank) | BB_BANK2(output_bank) | \ - BB_ITER((output_side) * (output_side)), \ - FIELD(input_side, 0, 3) | FIELD(output_side, 4, 7) | \ - FIELD(kernel, 8, 11) | FIELD(stride, 12, 15) | \ - FIELD(padding, 16, 19) | FIELD(input_base, 20, 25) | \ - FIELD(output_base, 26, 31) | FIELD(output_stride, 32, 37) | \ - FIELD(start_row, 38, 41) | FIELD(start_col, 42, 45), \ - BB_FUNC7(MAXPOOL)) - -#endif // _BB_MAXPOOL_H_ diff --git a/examples/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2.mlir b/examples/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2.mlir deleted file mode 100644 index b010bbb0a..000000000 --- a/examples/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2.mlir +++ /dev/null @@ -1,32 +0,0 @@ -func.func private @check_result(memref<9x16xi8>) -> () -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %thirty_six = arith.constant 36 : index - %sixteen = arith.constant 16 : index - %depth36 = arith.constant 36 : i64 - %depth9 = arith.constant 9 : i64 - %zero64 = arith.constant 0 : i64 - %three64 = arith.constant 3 : i64 - %stride = arith.constant 1 : i64 - %input = memref.alloc() : memref<36x16xi8> - %output = memref.alloc() : memref<9x16xi8> - scf.for %row = %zero to %thirty_six step %one { - %value = arith.index_cast %row : index to i8 - scf.for %lane = %zero to %sixteen step %one { - memref.store %value, %input[%row, %lane] : memref<36x16xi8> - } - } - %in_bank = buckyball.bank_alloc - %out_bank = buckyball.bank_alloc - %loaded = buckyball.bank_mvin %input %in_bank %depth36 %stride : memref<36x16xi8> i64 i64 i64 - buckyball.maxpool %loaded, %out_bank, %depth9, %zero64, %zero64, %three64 : i64 i64 i64 i64 i64 i64 - %stored = buckyball.bank_mvout %output %out_bank %depth9 %stride : memref<9x16xi8> i64 i64 i64 - func.call @check_result(%output) : (memref<9x16xi8>) -> () - buckyball.bank_release %loaded : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %input : memref<36x16xi8> - memref.dealloc %output : memref<9x16xi8> - return %zero_i8 : i8 -} diff --git a/examples/balls/maxpool/workloads/mlir_tests/bank/CMakeLists.txt b/examples/balls/maxpool/workloads/mlir_tests/bank/CMakeLists.txt deleted file mode 100644 index 77ba0e692..000000000 --- a/examples/balls/maxpool/workloads/mlir_tests/bank/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) - set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) -endif() -set(BUCKYBALL_MLIR_TEST_PREFIX bank) -set(_params "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_MLIR_ACTIVE_TARGET}/params.h") -file(STRINGS "${_params}" _lines_define REGEX "^#define BANK_LINES ") -file(STRINGS "${_params}" _sqrt_define REGEX "^#define BANK_ISQRT ") -string(REGEX REPLACE ".* " "" _lines "${_lines_define}") -string(REGEX REPLACE ".* " "" _sqrt "${_sqrt_define}") -math(EXPR _remainder "${_lines} % 64") -if(_lines LESS 64 OR NOT _remainder EQUAL 0) - message(FATAL_ERROR - "MaxPool K3 bank batches require BANK_LINES divisible by 64 (got ${_lines})") -endif() -add_buckyball_mlir_test(maxpool_k3_padded_spatial TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial.mlir.in b/examples/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial.mlir.in deleted file mode 100644 index f7ce8a64d..000000000 --- a/examples/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial.mlir.in +++ /dev/null @@ -1,41 +0,0 @@ -func.func private @check_result(memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %rows = arith.constant @BB_BANK_LINES@ : index - %batch_rows = arith.constant 64 : index - %lanes = arith.constant @BB_BANK_I8_LANES@ : index - %mod = arith.constant 31 : index - %bias = arith.constant 15 : index - %depth = arith.constant 64 : i64 - %zero64 = arith.constant 0 : i64 - %side64 = arith.constant 8 : i64 - %stride = arith.constant 1 : i64 - %input = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - %output = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - scf.for %row = %zero to %rows step %one { - %wrapped = arith.remui %row, %mod : index - %signed = arith.subi %wrapped, %bias : index - %value = arith.index_cast %signed : index to i8 - scf.for %lane = %zero to %lanes step %one { - memref.store %value, %input[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - memref.store %zero_i8, %output[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - } - } - %in_bank = buckyball.bank_alloc - %out_bank = buckyball.bank_alloc - %final:2 = scf.for %base = %zero to %rows step %batch_rows iter_args(%ib = %in_bank, %ob = %out_bank) -> (i64, i64) { - %in = memref.subview %input[%base, 0] [64, @BB_BANK_I8_LANES@] [1, 1] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> to memref<64x@BB_BANK_I8_LANES@xi8, strided<[@BB_BANK_I8_LANES@, 1], offset: ?>> - %out = memref.subview %output[%base, 0] [64, @BB_BANK_I8_LANES@] [1, 1] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> to memref<64x@BB_BANK_I8_LANES@xi8, strided<[@BB_BANK_I8_LANES@, 1], offset: ?>> - %loaded_input = buckyball.bank_mvin %in %ib %depth %stride : memref<64x@BB_BANK_I8_LANES@xi8, strided<[@BB_BANK_I8_LANES@, 1], offset: ?>> i64 i64 i64 - %loaded_output = buckyball.bank_mvin %out %ob %depth %stride : memref<64x@BB_BANK_I8_LANES@xi8, strided<[@BB_BANK_I8_LANES@, 1], offset: ?>> i64 i64 i64 - %result = buckyball.bank_maxpool %loaded_input %loaded_output %depth %zero64 %zero64 %side64 : i64 i64 i64 i64 i64 i64 - %stored = buckyball.bank_mvout %out %result %depth %stride : memref<64x@BB_BANK_I8_LANES@xi8, strided<[@BB_BANK_I8_LANES@, 1], offset: ?>> i64 i64 i64 - scf.yield %loaded_input, %stored : i64, i64 - } - func.call @check_result(%output) : (memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () - buckyball.bank_release %final#0 : i64 - buckyball.bank_release %final#1 : i64 - return %zero_i8 : i8 -} diff --git a/examples/balls/mxfp2int/arch/src/main/scala/Mxfp2Int.scala b/examples/balls/mxfp2int/arch/src/main/scala/Mxfp2Int.scala deleted file mode 100644 index 42dca6482..000000000 --- a/examples/balls/mxfp2int/arch/src/main/scala/Mxfp2Int.scala +++ /dev/null @@ -1,292 +0,0 @@ -package examples.balls.mxfp2int - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} - -import framework.balldomain.rs.{BallRsComplete, BallRsIssue} -import framework.balldomain.blink.{BallStatus, BankRead, BankWrite} -import framework.balldomain.blink.mmio.MmioRead -import framework.top.GlobalConfig -import examples.balls.mxfp2int.configs.Mxfp2IntBallParam - -/** - * PipelinedMxfp2Int - MXFP to INT8 dequantizer with MMIO per-block scale. - * - * Supports MXFP4, MXFP6, MXFP8 formats (32 elements per block). - * - * Input format (main bank, per block): - * MXFP4: 32 × 4-bit = 128 bits = 1 word - * MXFP6: 32 × 6-bit = 192 bits, padded to 2 words (256 bits, high 64 bits = 0) - * MXFP8: 32 × 8-bit = 256 bits = 2 words - * - * Scale (MMIO bank): per-block E8M0 (8-bit biased exponent, bias=127). - * Read MMIO byte at rel_addr = block_idx with meta_bank = wr_bank. - * - * FP4/6/8 decoding (v1, simplified): signed_val = raw - (1 << (bits-1)) - * FP4: raw - 8 (range -8..+7) - * FP6: raw - 32 (range -32..+31) - * FP8: raw - 128 (range -128..+127) - * - * Output (per block, 2 SRAM words = 32 bytes): - * 32 × INT8 = saturate(signed_val << (scale - 127)). - * Lower 16 INT8 to row (2 * block_idx), upper 16 to row (2 * block_idx + 1). - * - * iter encodes the number of MXFP blocks. - */ -@instantiable -class PipelinedMxfp2Int(val b: GlobalConfig) extends Module { - val ballConfig = Mxfp2IntBallParam(b) - val decoder = MxfpDecoder.fromFormat(ballConfig.mxfpFormat) - val elemBits = decoder.elemBits - - val bankWidth = b.memDomain.bankWidth - require( - bankWidth == 128, - s"Mxfp2IntBall requires bankWidth = 128, got $bankWidth" - ) - - val wordsPerBlock = ballConfig.mxfpFormat match { - case "MXFP4" => 1 - case "MXFP6" => - 2 // padded: 192 bits data + 64 bits padding = 256 bits = 2 words - case "MXFP8" => 2 - case _ => - throw new IllegalArgumentException( - s"Unsupported mxfpFormat: ${ballConfig.mxfpFormat}" - ) - } - - val FP_PER_BLOCK = 32 - val INT8_PER_WORD = bankWidth / 8 // 16 - - val ballMapping = b.ballDomain.ballIdMappings - .find(_.ballName == "Mxfp2IntBall") - .getOrElse( - throw new IllegalArgumentException("Mxfp2IntBall not found in config") - ) - - val inBW = ballMapping.inBW - val outBW = ballMapping.outBW - - @public - val io = IO(new Bundle { - val cmdReq = Flipped(Decoupled(new BallRsIssue(b))) - val cmdResp = Decoupled(new BallRsComplete(b)) - val bankRead = Vec(inBW, Flipped(new BankRead(b))) - val bankWrite = Vec(outBW, Flipped(new BankWrite(b))) - val status = new BallStatus - val mmioRead = Flipped(new MmioRead(b)) - }) - - // ROB bookkeeping - val rob_id_reg = RegInit(0.U(log2Up(b.frontend.rob_entries).W)) - val is_sub_reg = RegInit(false.B) - val sub_rob_id_reg = RegInit(0.U(log2Up(b.frontend.sub_rob_depth * 4).W)) - - when(io.cmdReq.fire) { - rob_id_reg := io.cmdReq.bits.rob_id - is_sub_reg := io.cmdReq.bits.is_sub - sub_rob_id_reg := io.cmdReq.bits.sub_rob_id - } - - // FSM - val idle :: sReadReq :: sReadResp :: sMmioReq :: sMmioResp :: sWriteLoReq :: sWriteLoResp :: sWriteHiReq :: sWriteHiResp :: sComplete :: Nil = - Enum(10) - val state = RegInit(idle) - - // Per-instruction registers - val raddr_reg = RegInit(0.U(b.frontend.iter_len.W)) - val waddr_reg = RegInit(0.U(b.frontend.iter_len.W)) - val rbank_reg = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - val wbank_reg = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - val iter_reg = RegInit(0.U(b.frontend.iter_len.W)) - val block_idx_reg = RegInit(0.U(b.frontend.iter_len.W)) - val word_idx_reg = RegInit(0.U(2.W)) // 0..wordsPerBlock-1 - - val inputWordsReg = RegInit( - VecInit(Seq.fill(2)(0.U(bankWidth.W))) - ) // max 2 words for MXFP6/8 - val scaleReg = RegInit(0.U(8.W)) - - // Extract 32 FP elements from inputWordsReg based on format - val fpElements = VecInit(Seq.tabulate(FP_PER_BLOCK) { i => - val bitOffset = i * elemBits - val wordIdx = bitOffset / bankWidth - val bitInWord = bitOffset % bankWidth - if (wordIdx < wordsPerBlock && bitInWord + elemBits <= bankWidth) { - inputWordsReg(wordIdx)(bitInWord + elemBits - 1, bitInWord) - } else if (wordIdx < wordsPerBlock - 1 && bitInWord + elemBits > bankWidth) { - // Cross-word boundary (only for MXFP6 if packed, but we use padded so this won't happen) - val lowBits = bankWidth - bitInWord - val highBits = elemBits - lowBits - Cat( - inputWordsReg(wordIdx + 1)(highBits - 1, 0), - inputWordsReg(wordIdx)(bankWidth - 1, bitInWord) - ) - } else { - 0.U(elemBits.W) - } - }) - - // Combinationally compute all 32 INT8 values from the latched elements + scale - val int8Vec = VecInit( - fpElements.map(raw => decoder.dequant(raw, scaleReg).asUInt) - ) - - val outWordLo = Cat((0 until INT8_PER_WORD).map(i => int8Vec(i)).reverse) - - val outWordHi = Cat( - (INT8_PER_WORD until FP_PER_BLOCK).map(i => int8Vec(i)).reverse - ) - - // === Default IO === - for (i <- 0 until inBW) { - io.bankRead(i).io.req.valid := false.B - io.bankRead(i).io.req.bits.addr := 0.U - io.bankRead(i).io.resp.ready := false.B - io.bankRead(i).rob_id := rob_id_reg - io.bankRead(i).ball_id := 0.U - io.bankRead(i).bank_id := rbank_reg - io.bankRead(i).group_id := 0.U - } - for (i <- 0 until outBW) { - io.bankWrite(i).io.req.valid := false.B - io.bankWrite(i).io.req.bits.addr := 0.U - io.bankWrite(i).io.req.bits.data := 0.U - io.bankWrite(i).io.req.bits.mask := VecInit( - Seq.fill(b.memDomain.bankMaskLen)(0.U(1.W)) - ) - io.bankWrite(i).io.resp.ready := false.B - io.bankWrite(i).rob_id := rob_id_reg - io.bankWrite(i).ball_id := 0.U - io.bankWrite(i).bank_id := wbank_reg - io.bankWrite(i).group_id := 0.U - } - - io.mmioRead.req.valid := false.B - io.mmioRead.req.bits.addr := 0.U - io.mmioRead.resp.ready := false.B - io.mmioRead.rob_id := rob_id_reg - io.mmioRead.ball_id := 0.U - io.cmdReq.ready := state === idle - io.cmdResp.valid := false.B - io.cmdResp.bits.rob_id := rob_id_reg - io.cmdResp.bits.is_sub := is_sub_reg - io.cmdResp.bits.sub_rob_id := sub_rob_id_reg - - // === FSM === - switch(state) { - is(idle) { - when(io.cmdReq.fire) { - rbank_reg := io.cmdReq.bits.cmd.op1_bank - wbank_reg := io.cmdReq.bits.cmd.wr_bank - iter_reg := io.cmdReq.bits.cmd.iter - block_idx_reg := 0.U - word_idx_reg := 0.U - raddr_reg := 0.U - waddr_reg := 0.U - when(io.cmdReq.bits.cmd.iter === 0.U) { - state := sComplete - }.otherwise { - state := sReadReq - } - } - } - - is(sReadReq) { - io.bankRead(0).io.req.valid := true.B - io.bankRead(0) - .io - .req - .bits - .addr := raddr_reg + block_idx_reg * wordsPerBlock.U + word_idx_reg - when(io.bankRead(0).io.req.fire) { - state := sReadResp - } - } - - is(sReadResp) { - io.bankRead(0).io.resp.ready := true.B - when(io.bankRead(0).io.resp.fire) { - inputWordsReg(word_idx_reg) := io.bankRead(0).io.resp.bits.data - when(word_idx_reg === (wordsPerBlock - 1).U) { - word_idx_reg := 0.U - state := sMmioReq - }.otherwise { - word_idx_reg := word_idx_reg + 1.U - state := sReadReq - } - } - } - - is(sMmioReq) { - io.mmioRead.req.valid := true.B - io.mmioRead.req.bits.addr := block_idx_reg - when(io.mmioRead.req.fire) { - state := sMmioResp - } - } - - is(sMmioResp) { - io.mmioRead.resp.ready := true.B - when(io.mmioRead.resp.fire) { - scaleReg := io.mmioRead.resp.bits.data(7, 0) - state := sWriteLoReq - } - } - - is(sWriteLoReq) { - io.bankWrite(0).io.req.valid := true.B - io.bankWrite(0).io.req.bits.addr := waddr_reg + (block_idx_reg << 1) - io.bankWrite(0).io.req.bits.data := outWordLo - io.bankWrite(0).io.req.bits.mask := VecInit( - Seq.fill(b.memDomain.bankMaskLen)(1.U(1.W)) - ) - when(io.bankWrite(0).io.req.fire) { - state := sWriteLoResp - } - } - - is(sWriteLoResp) { - io.bankWrite(0).io.resp.ready := true.B - when(io.bankWrite(0).io.resp.fire) { - state := sWriteHiReq - } - } - - is(sWriteHiReq) { - io.bankWrite(0).io.req.valid := true.B - io.bankWrite(0).io.req.bits.addr := waddr_reg + (block_idx_reg << 1) + 1.U - io.bankWrite(0).io.req.bits.data := outWordHi - io.bankWrite(0).io.req.bits.mask := VecInit( - Seq.fill(b.memDomain.bankMaskLen)(1.U(1.W)) - ) - when(io.bankWrite(0).io.req.fire) { - state := sWriteHiResp - } - } - - is(sWriteHiResp) { - io.bankWrite(0).io.resp.ready := true.B - when(io.bankWrite(0).io.resp.fire) { - when(block_idx_reg === iter_reg - 1.U) { - state := sComplete - }.otherwise { - block_idx_reg := block_idx_reg + 1.U - state := sReadReq - } - } - } - - is(sComplete) { - io.cmdResp.valid := true.B - when(io.cmdResp.fire) { - state := idle - } - } - } - - io.status.idle := state === idle - io.status.running := state =/= idle -} diff --git a/examples/balls/mxfp2int/arch/src/main/scala/Mxfp2IntBall.scala b/examples/balls/mxfp2int/arch/src/main/scala/Mxfp2IntBall.scala deleted file mode 100644 index 637ca9c66..000000000 --- a/examples/balls/mxfp2int/arch/src/main/scala/Mxfp2IntBall.scala +++ /dev/null @@ -1,57 +0,0 @@ -package examples.balls.mxfp2int - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} - -import framework.balldomain.blink.{BlinkIO, HasBlink, SubRobRow} -import framework.top.GlobalConfig - -/** - * Mxfp2IntBall - MXFP4 to INT8 dequantizer. - * - * Wraps PipelinedMxfp2Int and connects to the Blink protocol, - * including the MMIO read channel for per-block E8M0 scales. - */ -@instantiable -class Mxfp2IntBall(val b: GlobalConfig) extends Module with HasBlink { - - val ballCommonConfig = b.ballDomain.ballIdMappings - .find(_.ballName == "Mxfp2IntBall") - .getOrElse( - throw new IllegalArgumentException("Mxfp2IntBall not found in config") - ) - - val inBW = ballCommonConfig.inBW - val outBW = ballCommonConfig.outBW - val mmioReadBW = ballCommonConfig.mmioReadBW - require(mmioReadBW == 1, "Mxfp2IntBall requires exactly one MMIO read line") - - @public - val io = IO(new BlinkIO(b, inBW, outBW, mmioReadBW)) - - def blink: BlinkIO = io - dontTouch(io) - - val mxfp2intUnit: Instance[PipelinedMxfp2Int] = Instantiate( - new PipelinedMxfp2Int(b) - ) - - mxfp2intUnit.io.cmdReq <> io.cmdReq - mxfp2intUnit.io.cmdResp <> io.cmdResp - - for (i <- 0 until inBW) { - mxfp2intUnit.io.bankRead(i) <> io.bankRead(i) - } - - for (i <- 0 until outBW) { - mxfp2intUnit.io.bankWrite(i) <> io.bankWrite(i) - } - - io.status <> mxfp2intUnit.io.status - - io.subRobReq.valid := false.B - io.subRobReq.bits := SubRobRow.tieOff(b) - - io.mmioRead(0) <> mxfp2intUnit.io.mmioRead -} diff --git a/examples/balls/mxfp2int/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp b/examples/balls/mxfp2int/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp deleted file mode 100644 index 4a5c3f6d8..000000000 --- a/examples/balls/mxfp2int/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp +++ /dev/null @@ -1,34 +0,0 @@ -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace mlir::buddy { -void populateMxfp2IntBallAssignPhysicalBankPatterns(RewritePatternSet &patterns, - PhysicalBankState &state); -} // namespace mlir::buddy - -namespace { -class BankMxfp2IntPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(BankMxfp2IntOp op, - PatternRewriter &rewriter) const override { - rewriter.create(op.getLoc(), op.getInBank(), op.getOutBank(), - op.getIter(), op.getSpecial()); - rewriter.replaceOp(op, op.getOutBank()); - return success(); - } -}; -} // namespace - -void mlir::buddy::populateMxfp2IntBallAssignPhysicalBankPatterns( - RewritePatternSet &patterns, mlir::buddy::PhysicalBankState &state) { - (void)state; - patterns.add(patterns.getContext()); -} diff --git a/examples/balls/mxfp2int/compiler/src/Dialect/Buckyball/Mxfp2IntBall.td b/examples/balls/mxfp2int/compiler/src/Dialect/Buckyball/Mxfp2IntBall.td deleted file mode 100644 index 75e4b813c..000000000 --- a/examples/balls/mxfp2int/compiler/src/Dialect/Buckyball/Mxfp2IntBall.td +++ /dev/null @@ -1,26 +0,0 @@ -//===-- Mxfp2IntBall.td - MXFP to INT8 dequantization operations --------===// - -#ifndef MXFP2INT_BALL_OPS -#define MXFP2INT_BALL_OPS - -include "Buckyball.td" - -def Mxfp2IntOp : Buckyball_Op<"mxfp2int"> { - let summary = "Dequantize MXFP blocks to INT8"; - let description = [{ - Reads MXFP blocks from inputBankId, writes two INT8 rows per block to - outputBankId, and uses special as rs2 for format and rounding controls. - }]; - let arguments = (ins I64:$inputBankId, I64:$outputBankId, I64:$iter, - I64:$special); - let assemblyFormat = "$inputBankId `,` $outputBankId `,` $iter `,` $special attr-dict `:` type($inputBankId)"; -} - -def BankMxfp2IntOp : Buckyball_Op<"bank_mxfp2int"> { - let summary = "MXFP to INT8 dequantization using bank-SSA handles"; - let arguments = (ins I64:$inBank, I64:$outBank, I64:$iter, I64:$special); - let results = (outs I64:$outBankOut); - let assemblyFormat = "$inBank $outBank $iter $special attr-dict `:` type($inBank) type($outBank) type($iter) type($special)"; -} - -#endif // MXFP2INT_BALL_OPS diff --git a/examples/balls/mxfp2int/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/balls/mxfp2int/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 86ea32318..000000000 --- a/examples/balls/mxfp2int/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,50 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace buddy::buckyball; -using namespace buddy::buckyball::legalize; - -namespace { -class Mxfp2IntLowering : public ConvertOpToLLVMPattern { -public: - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - - LogicalResult - matchAndRewrite(Mxfp2IntOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("Mxfp2IntBall"); - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInputBankId(), - cstI64(rewriter, loc, 0), - adaptor.getOutputBankId(), adaptor.getIter()); - rewriter.replaceOpWithNewOp( - op, rs1, adaptor.getSpecial(), - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("MXFP2INT"))); - return success(); - } -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateMxfp2IntBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth) { - (void)stable; - (void)bankDepth; - - patterns.add(converter); -} - -void configureMxfp2IntBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable) { - (void)stable; - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/examples/balls/mxfp2int/configs/Mxfp2IntBallParam.scala b/examples/balls/mxfp2int/configs/Mxfp2IntBallParam.scala deleted file mode 100644 index b8d2a71e7..000000000 --- a/examples/balls/mxfp2int/configs/Mxfp2IntBallParam.scala +++ /dev/null @@ -1,17 +0,0 @@ -package examples.balls.mxfp2int.configs - -import framework.balldomain.configs.BallParamLoader -import framework.top.GlobalConfig - -case class Mxfp2IntBallParam( - mxfpFormat: String) - -object Mxfp2IntBallParam { - private val ballName = "Mxfp2IntBall" - - def apply(b: GlobalConfig): Mxfp2IntBallParam = { - val tbl = BallParamLoader.ballTable(b, ballName) - Mxfp2IntBallParam(mxfpFormat = BallParamLoader.str(tbl, "mxfpFormat")) - } - -} diff --git a/examples/balls/mxfp2int/configs/default.toml b/examples/balls/mxfp2int/configs/default.toml deleted file mode 100644 index 63342a459..000000000 --- a/examples/balls/mxfp2int/configs/default.toml +++ /dev/null @@ -1,4 +0,0 @@ -# Mxfp2Int Ball Configuration - -[ball] -mxfpFormat = "MXFP4" diff --git a/examples/balls/mxfp2int/emu/src/55_mxfp2int.rs b/examples/balls/mxfp2int/emu/src/55_mxfp2int.rs deleted file mode 100644 index d7e5ecfa4..000000000 --- a/examples/balls/mxfp2int/emu/src/55_mxfp2int.rs +++ /dev/null @@ -1,103 +0,0 @@ -//===- 55_mxfp2int.rs - MXFP2INT instruction (MXFP to INT8) ----------------===// -// -// Dequantizes MXFP4/6/8 data to INT8 using per-block E8M0 scales from MMIO. -// -// rs1[9:0]: op1_bank (BANK0, input MXFP data) -// rs1[29:20]: wr_bank (BANK2, output INT8 data) -// rs1[63:30]: iter (number of MXFP blocks to process) -// -// For MXFP4: each block = 32 x 4-bit FP = 16 bytes input → 32 bytes INT8 output -// Scale: per-block E8M0 (8-bit) from MMIO, indexed by block_idx. -// -//===-----------------------------------------------------------------===// - -use super::super::bank::{bank_width, mmio_read_byte}; -use super::decode::{rs1_b0, rs1_b2, rs1_iter}; -use super::instruction::{BallInstruction, ExecContext}; - -pub struct Mxfp2Int; - -impl BallInstruction for Mxfp2Int { - fn exec(xs1: u64, _xs2: u64, ctx: &mut ExecContext) -> u64 { - let in_bank = rs1_b0(xs1) as u32; - let out_bank = rs1_b2(xs1) as u32; - let iter = rs1_iter(xs1) as usize; // number of MXFP blocks - - // Resolve physical banks - let in_pbank = ctx - .bank_map - .resolve(in_bank) - .unwrap_or_else(|| panic!("mxfp2int: input bank {} not allocated", in_bank)); - let out_pbank = ctx - .bank_map - .resolve(out_bank) - .unwrap_or_else(|| panic!("mxfp2int: output bank {} not allocated", out_bank)); - - const FP4_PER_BLOCK: usize = 32; - const BYTES_PER_OUTPUT_BLOCK: usize = 32; // 32 x INT8 - - for block_idx in 0..iter { - // Read E8M0 scale from the globally encoded MMIO address space. - let scale_e8m0 = mmio_read_byte(ctx.mmio_banks, block_idx); - - // Read input MXFP4 block (16 bytes = 1 bank row) - let in_row_addr = block_idx; - let row_bytes = bank_width() / 8; - let in_row_bytes = - &ctx.banks[in_pbank][in_row_addr * row_bytes..(in_row_addr + 1) * row_bytes]; - - // Dequantize 32 FP4 elements to INT8 - let mut out_bytes = [0i8; BYTES_PER_OUTPUT_BLOCK]; - for (elem_idx, out_byte) in out_bytes.iter_mut().enumerate().take(FP4_PER_BLOCK) { - let byte_idx = elem_idx / 2; - let nibble_shift = (elem_idx % 2) * 4; - let fp4_raw = (in_row_bytes[byte_idx] >> nibble_shift) & 0x0F; - - // FP4 decode: signed_val = raw - 8 (range -8..+7) - let signed_val = (fp4_raw as i32) - 8; - - // Scale: shift = scale_e8m0 - 127 - let shift = (scale_e8m0 as i32) - 127; - - // Apply scale and saturate to INT8 - let scaled = if shift >= 0 { - signed_val - .checked_shl(shift as u32) - .unwrap_or(if signed_val >= 0 { i32::MAX } else { i32::MIN }) - } else { - let abs_shift = (-shift) as u32; - if abs_shift >= 32 { - 0 // Shift too large, result is 0 - } else { - signed_val >> abs_shift - } - }; - let saturated = scaled.clamp(-128, 127) as i8; - - *out_byte = saturated; - } - - // Write output INT8 data (32 bytes = 2 bank rows) - let out_row_base = block_idx * 2; - for row_offset in 0..2 { - let out_row_addr = out_row_base + row_offset; - let out_row_start = out_row_addr * row_bytes; - let src_start = row_offset * row_bytes; - let src_end = src_start + row_bytes; - - // Copy 16 bytes (1 row) from out_bytes to output bank - for (i, &byte) in out_bytes[src_start..src_end].iter().enumerate() { - ctx.banks[out_pbank][out_row_start + i] = byte as u8; - } - } - } - - 0 - } - - fn latency(_xs1: u64, _xs2: u64) -> u64 { - let iter = rs1_iter(_xs1); - // Approximate: each block needs 1 MMIO read + 1 input read + 2 output writes - iter * 4 - } -} diff --git a/examples/balls/mxfp2int/emu/src/dispatch.rs b/examples/balls/mxfp2int/emu/src/dispatch.rs deleted file mode 100644 index a38a68451..000000000 --- a/examples/balls/mxfp2int/emu/src/dispatch.rs +++ /dev/null @@ -1,31 +0,0 @@ -use crate::inst::instruction::{BallInstruction, ExecContext}; - -use super::f55_mxfp2int; - -const BALL_CLASS: &str = "examples.balls.mxfp2int.Mxfp2IntBall"; - -pub fn execute_known( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, - ctx: &mut ExecContext, -) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("MXFP2INT") => Some(f55_mxfp2int::Mxfp2Int::exec(xs1, xs2, ctx)), - Some(_) | None => None, - } -} - -pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("MXFP2INT") => Some(f55_mxfp2int::Mxfp2Int::latency(xs1, xs2)), - Some(_) | None => None, - } -} diff --git a/examples/balls/mxfp2int/emu/src/lib.rs b/examples/balls/mxfp2int/emu/src/lib.rs deleted file mode 100644 index b35b3155f..000000000 --- a/examples/balls/mxfp2int/emu/src/lib.rs +++ /dev/null @@ -1,8 +0,0 @@ -pub(crate) use crate::inst::{decode, instruction}; - -#[path = "55_mxfp2int.rs"] -mod f55_mxfp2int; - -mod dispatch; - -pub use dispatch::{cycles_after_issue, execute_known}; diff --git a/examples/balls/mxfp2int/workloads/ctests/CMakeLists.txt b/examples/balls/mxfp2int/workloads/ctests/CMakeLists.txt deleted file mode 100644 index 635814a36..000000000 --- a/examples/balls/mxfp2int/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -add_buckyball_ctests( - mxfp4_e8m0_two_block_test.c -) diff --git a/examples/balls/mxfp2int/workloads/ctests/mxfp4_e8m0_two_block_test.c b/examples/balls/mxfp2int/workloads/ctests/mxfp4_e8m0_two_block_test.c deleted file mode 100644 index c94f9199c..000000000 --- a/examples/balls/mxfp2int/workloads/ctests/mxfp4_e8m0_two_block_test.c +++ /dev/null @@ -1,85 +0,0 @@ -#include "buckyball.h" -#include -#include -#include -#include -#include -#include -#include - -#define NUM_BLOCKS 2 -#define FP4_PER_BLOCK 32 -#define BYTES_PER_BLOCK 16 -#define E8M0_BIAS 127 - -static uint8_t input_mxfp4[NUM_BLOCKS * BYTES_PER_BLOCK] - __attribute__((aligned(64))); -static uint8_t scales_e8m0[16] __attribute__((aligned(16))); -static int8_t output_int8[NUM_BLOCKS * FP4_PER_BLOCK] - __attribute__((aligned(64))); - -static uint8_t gen_fp4(int block, int elem) { - return (uint8_t)((elem + block) & 0x0F); -} - -static int8_t golden(int block, int elem) { - int val = (int)gen_fp4(block, elem) - 8; - int prod = val << block; /* scale = bias + block */ - if (prod > 127) - prod = 127; - if (prod < -128) - prod = -128; - return (int8_t)prod; -} - -int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); -#endif - - for (int b = 0; b < NUM_BLOCKS; b++) { - for (int e = 0; e < FP4_PER_BLOCK; e += 2) { - uint8_t lo = gen_fp4(b, e); - uint8_t hi = gen_fp4(b, e + 1); - input_mxfp4[b * BYTES_PER_BLOCK + e / 2] = - (uint8_t)((hi << 4) | (lo & 0x0F)); - } - scales_e8m0[b] = (uint8_t)(E8M0_BIAS + b); - } - - mmio_allocator_t mmio_alloc; - mmio_allocator_init(&mmio_alloc); - uint16_t mmio_addr = mmio_allocator_alloc(&mmio_alloc, 1); - if (mmio_addr == (uint16_t)-1) { - printf("FAIL: MMIO alloc\n"); - return 1; - } - - const uint32_t in_bank = 0; - const uint32_t out_bank = 1; - bb_mem_alloc(in_bank, 1, 1); - bb_mem_alloc(out_bank, 1, 1); - // Publish scales and packed input once before the NPU command chain. - bb_mvin_mmio((uintptr_t)scales_e8m0, mmio_addr, 1, NUM_BLOCKS); - bb_mvin((uintptr_t)input_mxfp4, in_bank, NUM_BLOCKS, 1); - bb_mxfp2int(in_bank, out_bank, NUM_BLOCKS); - bb_mvout((uintptr_t)output_int8, out_bank, NUM_BLOCKS * 2, 1); - - int passed = 1; - for (int b = 0; b < NUM_BLOCKS; b++) { - for (int e = 0; e < FP4_PER_BLOCK; e++) { - int8_t got = output_int8[b * FP4_PER_BLOCK + e]; - int8_t exp = golden(b, e); - if (got != exp) { - printf("MISMATCH b=%d e=%d got=%d exp=%d\n", b, e, (int)got, (int)exp); - passed = 0; - } - } - } - - bb_mem_release(in_bank); - bb_mem_release(out_bank); - mmio_allocator_free(&mmio_alloc, mmio_addr, 1); - printf("mxfp4 e8m0 two-block %s\n", passed ? "PASSED" : "FAILED"); - return passed ? 0 : 1; -} diff --git a/examples/balls/mxfp2int/workloads/isa/mxfp2int.h b/examples/balls/mxfp2int/workloads/isa/mxfp2int.h deleted file mode 100644 index 51d40cf2a..000000000 --- a/examples/balls/mxfp2int/workloads/isa/mxfp2int.h +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef _BB_MXFP2INT_H_ -#define _BB_MXFP2INT_H_ - -#include -#include - -// Basic version: -// rs1 = bank0(read) | bank2(write) | iter -// rs2 = 0 -#define bb_mxfp2int(bank_id, wr_bank_id, iter) \ - BUCKYBALL_INSTRUCTION_R_R( \ - (BB_BANK0(bank_id) | BB_BANK2(wr_bank_id) | BB_ITER(iter)), 0, \ - BB_FUNC7(MXFP2INT)) - -// Extended version: -// rs2 carries user-defined special field. -// Useful later for format select / rounding mode / debug flags. -#define bb_mxfp2int_ex(bank_id, wr_bank_id, iter, special) \ - BUCKYBALL_INSTRUCTION_R_R( \ - (BB_BANK0(bank_id) | BB_BANK2(wr_bank_id) | BB_ITER(iter)), (special), \ - BB_FUNC7(MXFP2INT)) - -#endif // _BB_MXFP2INT_H_ diff --git a/examples/balls/mxfp2int/workloads/mlir_tests/ball/CMakeLists.txt b/examples/balls/mxfp2int/workloads/mlir_tests/ball/CMakeLists.txt deleted file mode 100644 index 313d9257a..000000000 --- a/examples/balls/mxfp2int/workloads/mlir_tests/ball/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) - set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) -endif() -set(BUCKYBALL_MLIR_TEST_PREFIX ball) -add_buckyball_mlir_test(mxfp4_unit_scale_2blocks TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/balls/mxfp2int/workloads/mlir_tests/ball/mxfp4_unit_scale_2blocks.mlir b/examples/balls/mxfp2int/workloads/mlir_tests/ball/mxfp4_unit_scale_2blocks.mlir deleted file mode 100644 index 5a6e31194..000000000 --- a/examples/balls/mxfp2int/workloads/mlir_tests/ball/mxfp4_unit_scale_2blocks.mlir +++ /dev/null @@ -1,35 +0,0 @@ -func.func private @check_result(memref<4x16xi8>) -> () -func.func @main() -> i8 { - %z8 = arith.constant 0 : i8 - %raw = arith.constant -103 : i8 - %scale = arith.constant 127 : i8 - %z = arith.constant 0 : index - %one = arith.constant 1 : index - %two = arith.constant 2 : i64 - %one64 = arith.constant 1 : i64 - %four = arith.constant 4 : i64 - %sixteen = arith.constant 16 : i64 - %zero64 = arith.constant 0 : i64 - %stride = arith.constant 1 : i64 - %input = memref.alloc() : memref<2x16xi8> - %scales = memref.alloc() : memref<1x16xi8> - %output = memref.alloc() : memref<4x16xi8> - linalg.fill ins(%raw : i8) outs(%input : memref<2x16xi8>) - linalg.fill ins(%scale : i8) outs(%scales : memref<1x16xi8>) - linalg.fill ins(%z8 : i8) outs(%output : memref<4x16xi8>) - %in = arith.constant 0 : i64 - %out = arith.constant 1 : i64 - buckyball.mset %in : i64 - buckyball.mset %out : i64 - buckyball.mvin %input %in %two %stride : memref<2x16xi8> i64 i64 i64 - buckyball.mvin_mmio %scales %zero64 %one64 %sixteen : memref<1x16xi8> i64 i64 i64 - buckyball.mxfp2int %in, %out, %two, %zero64 : i64 - buckyball.mvout %output %out %four %stride : memref<4x16xi8> i64 i64 i64 - func.call @check_result(%output) : (memref<4x16xi8>) -> () - buckyball.mset %in : i64 - buckyball.mset %out : i64 - memref.dealloc %input : memref<2x16xi8> - memref.dealloc %scales : memref<1x16xi8> - memref.dealloc %output : memref<4x16xi8> - return %z8 : i8 -} diff --git a/examples/balls/mxfp2int/workloads/mlir_tests/ball/mxfp4_unit_scale_2blocks_main.cpp b/examples/balls/mxfp2int/workloads/mlir_tests/ball/mxfp4_unit_scale_2blocks_main.cpp deleted file mode 100644 index ea93e39af..000000000 --- a/examples/balls/mxfp2int/workloads/mlir_tests/ball/mxfp4_unit_scale_2blocks_main.cpp +++ /dev/null @@ -1,15 +0,0 @@ -#include -#include -#include -extern "C" void check_result(int8_t *, int8_t *data, int64_t offset, - int64_t rows, int64_t lanes, int64_t stride, - int64_t lane_stride) { - if (rows != 4 || lanes != 16 || stride != 16 || lane_stride != 1) - exit(1); - for (int i = 0; i < 64; ++i) - if (data[offset + i] != 1) { - printf("mxfp4 unit-scale mismatch index=%d\n", i); - exit(1); - } - printf("mxfp4 unit-scale 2block PASS\n"); -} diff --git a/examples/balls/mxfp2int/workloads/mlir_tests/bank/CMakeLists.txt b/examples/balls/mxfp2int/workloads/mlir_tests/bank/CMakeLists.txt deleted file mode 100644 index e44f42b29..000000000 --- a/examples/balls/mxfp2int/workloads/mlir_tests/bank/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) - set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) -endif() -set(BUCKYBALL_MLIR_TEST_PREFIX bank) -buckyball_require_mxfp2int_bank_shape(${BUCKYBALL_MLIR_ACTIVE_TARGET}) -add_buckyball_mlir_test(mxfp4_unit_scale_bank TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/balls/mxfp2int/workloads/mlir_tests/bank/mxfp4_unit_scale_bank.mlir.in b/examples/balls/mxfp2int/workloads/mlir_tests/bank/mxfp4_unit_scale_bank.mlir.in deleted file mode 100644 index bc92a2351..000000000 --- a/examples/balls/mxfp2int/workloads/mlir_tests/bank/mxfp4_unit_scale_bank.mlir.in +++ /dev/null @@ -1,37 +0,0 @@ -func.func private @check_result(memref<@BB_BANK_LINES@x16xi8>) -> () -func.func @main() -> i8 { - %z8 = arith.constant 0 : i8 - %raw = arith.constant -103 : i8 - %scale = arith.constant 127 : i8 - %z = arith.constant 0 : index - %one = arith.constant 1 : index - %inputDepth = arith.constant @BB_MXFP_BLOCKS@ : i64 - %outputDepth = arith.constant @BB_BANK_LINES@ : i64 - %scaleDepth = arith.constant @BB_MXFP_MMIO_ROWS@ : i64 - %columns = arith.constant 16 : i64 - %zero64 = arith.constant 0 : i64 - %stride = arith.constant 1 : i64 - %input = memref.alloc() : memref<@BB_MXFP_BLOCKS@x16xi8> - %scales = memref.alloc() : memref<@BB_MXFP_MMIO_ROWS@x16xi8> - %output = memref.alloc() : memref<@BB_BANK_LINES@x16xi8> - linalg.fill ins(%raw : i8) outs(%input : memref<@BB_MXFP_BLOCKS@x16xi8>) - linalg.fill ins(%scale : i8) outs(%scales : memref<@BB_MXFP_MMIO_ROWS@x16xi8>) - linalg.fill ins(%z8 : i8) outs(%output : memref<@BB_BANK_LINES@x16xi8>) - %inBank = buckyball.bank_alloc - %outBank = buckyball.bank_alloc - buckyball.mvin_mmio %scales %zero64 %scaleDepth %columns - : memref<@BB_MXFP_MMIO_ROWS@x16xi8> i64 i64 i64 - %loaded = buckyball.bank_mvin %input %inBank %inputDepth %stride - : memref<@BB_MXFP_BLOCKS@x16xi8> i64 i64 i64 - %result = buckyball.bank_mxfp2int %loaded %outBank %inputDepth %zero64 - : i64 i64 i64 i64 - %stored = buckyball.bank_mvout %output %result %outputDepth %stride - : memref<@BB_BANK_LINES@x16xi8> i64 i64 i64 - func.call @check_result(%output) : (memref<@BB_BANK_LINES@x16xi8>) -> () - buckyball.bank_release %loaded : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %input : memref<@BB_MXFP_BLOCKS@x16xi8> - memref.dealloc %scales : memref<@BB_MXFP_MMIO_ROWS@x16xi8> - memref.dealloc %output : memref<@BB_BANK_LINES@x16xi8> - return %z8 : i8 -} diff --git a/examples/balls/mxfp2int/workloads/mlir_tests/bank/mxfp4_unit_scale_bank_main.cpp b/examples/balls/mxfp2int/workloads/mlir_tests/bank/mxfp4_unit_scale_bank_main.cpp deleted file mode 100644 index 014f1d9fc..000000000 --- a/examples/balls/mxfp2int/workloads/mlir_tests/bank/mxfp4_unit_scale_bank_main.cpp +++ /dev/null @@ -1,16 +0,0 @@ -#include -#include -#include -#include -extern "C" void check_result(int8_t *, int8_t *data, int64_t offset, - int64_t rows, int64_t lanes, int64_t stride, - int64_t lane_stride) { - if (rows != BANK_LINES || lanes != 16 || stride != 16 || lane_stride != 1) - exit(1); - for (int i = 0; i < rows * lanes; ++i) - if (data[offset + i] != 1) { - printf("mxfp4 unit-scale mismatch index=%d\n", i); - exit(1); - } - printf("mxfp4 unit-scale bank PASS rows=%d\n", BANK_LINES); -} diff --git a/examples/balls/mxmm/arch/src/main/scala/Array.scala b/examples/balls/mxmm/arch/src/main/scala/Array.scala deleted file mode 100644 index d539c6847..000000000 --- a/examples/balls/mxmm/arch/src/main/scala/Array.scala +++ /dev/null @@ -1,51 +0,0 @@ -package examples.balls.mxmm - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import examples.balls.mxmm.configs.MxmmBallParam -import hardfloat._ - -@instantiable -class Array(p: MxmmBallParam) extends Module { - - @public - val io = IO(new Bundle { - val valid = Input(Bool()) - val context = Input(UInt(log2Ceil(p.contexts).W)) - val separate = Input(Bool()) - val vector = Input(Bool()) - val a = Input(Vec(p.tileRows, UInt(32.W))) - val b = Input(Vec(p.tileCols, UInt(32.W))) - val load = Input(Bool()) - val row = Input(UInt(4.W)) - val rowData = Input(UInt(512.W)) - val rowOut = Output(UInt(512.W)) - val completed = Output(Bool()) - }) - - val accumulators = Reg(Vec(p.contexts, Vec(p.tileRows, Vec(p.tileCols, UInt(33.W))))) - when(io.load) { - for (col <- 0 until p.tileCols) { - accumulators(io.context)(io.row)(col) := recFNFromFN(8, 24, io.rowData(32 * col + 31, 32 * col)) - } - } - io.rowOut := Cat((0 until p.tileCols).reverse.map(col => fNFromRecFN(8, 24, accumulators(io.context)(io.row)(col)))) - val fusedTag = Pipe(io.valid && !io.separate, io.context, p.arithmeticLatency) - val separateTag = Pipe(io.valid && io.separate, io.context, 2 * p.arithmeticLatency) - io.completed := fusedTag.valid || separateTag.valid - for { - row <- 0 until p.tileRows - col <- 0 until p.tileCols - } { - val pe = Instantiate(new PE(p.arithmeticLatency)) - pe.io.valid := io.valid && (if (row == 0) true.B else !io.vector) - pe.io.separate := io.separate - pe.io.a := io.a(row) - pe.io.b := io.b(col) - pe.io.c := accumulators(io.context)(row)(col) - when(pe.io.outValid) { - accumulators(Mux(separateTag.valid, separateTag.bits, fusedTag.bits))(row)(col) := pe.io.out - } - } -} diff --git a/examples/balls/mxmm/arch/src/main/scala/Emit.scala b/examples/balls/mxmm/arch/src/main/scala/Emit.scala deleted file mode 100644 index be7271ad2..000000000 --- a/examples/balls/mxmm/arch/src/main/scala/Emit.scala +++ /dev/null @@ -1,36 +0,0 @@ -package examples.balls.mxmm - -import framework.top.GlobalConfig -import framework.balldomain.configs.{BallDomainParam, BallISAEntry, BallIdMapping} - -object Emit extends App { - require(args.length == 2, "Emit requires OUTPUT_DIRECTORY BALL_CONFIG") - val defaults = GlobalConfig() - - val b = defaults.copy( - memDomain = defaults.memDomain.copy( - bankNum = 8, - bankWidth = 128, - bankEntries = 1024, - bankMaskLen = 16, - virtualBankCount = 8, - sharedBankNum = 8, - mmioBankNum = 8, - mmioBankEntries = 1024, - mmioBankWidth = 128, - mmioReadWidth = 16 - ), - frontend = defaults.frontend.copy(rob_entries = 16, sub_rob_depth = 8, iter_len = 34), - ballDomain = BallDomainParam( - 1, - Seq(BallIdMapping(0, "MxmmBall", "examples.balls.mxmm.MxmmBall", Some(args(1)), 2, 1)), - Seq(BallISAEntry("MXMM_MXFP8", 71, 0), BallISAEntry("MXMM_FMA32", 72, 0), BallISAEntry("MXMM_F32", 73, 0)) - ) - ) - - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new MxmmBall(b), - args = _root_.scala.Array("--target-dir", args(0), "--split-verilog"), - firtoolOpts = _root_.scala.Array("--disable-annotation-unknown", "--strip-debug-info") - ) -} diff --git a/examples/balls/mxmm/arch/src/main/scala/InputWindow.scala b/examples/balls/mxmm/arch/src/main/scala/InputWindow.scala deleted file mode 100644 index bc509cec8..000000000 --- a/examples/balls/mxmm/arch/src/main/scala/InputWindow.scala +++ /dev/null @@ -1,82 +0,0 @@ -package examples.balls.mxmm - -import chisel3._ -import chisel3.util._ -import framework.memdomain.backend.banks.{SramReadReq, SramReadResp} -import framework.top.GlobalConfig - -class InputWindow(b: GlobalConfig) extends Module { - private val addressWidth = log2Ceil(b.memDomain.bankEntries) - - val io = IO(new Bundle { - val start = Input(Bool()) - val enable = Input(Bool()) - val rows = Input(UInt(12.W)) - val count = Input(UInt(34.W)) - val fullK = Input(UInt(16.W)) - val startK = Input(UInt(16.W)) - val request = Decoupled(new SramReadReq(b)) - val response = Flipped(Decoupled(new SramReadResp(b))) - val done = Output(Bool()) - val write = Output(Bool()) - val scaleWrite = Output(Bool()) - val lane = Output(UInt(4.W)) - val address = Output(UInt(addressWidth.W)) - val word = Output(UInt(128.W)) - val scale = Output(UInt(8.W)) - }) - - private val request :: response :: scales :: done :: Nil = Enum(4) - private val state = RegInit(done) - private val rows = Reg(UInt(12.W)) - private val count = Reg(UInt(34.W)) - private val fullK = Reg(UInt(16.W)) - private val startK = Reg(UInt(16.W)) - private val row = Reg(UInt(12.W)) - private val offset = Reg(UInt(addressWidth.W)) - private val scaleMode = Reg(Bool()) - private val word = Reg(UInt(128.W)) - private val scaleAddress = rows * fullK + row * (fullK >> 5) + (startK >> 5) + offset - private val byte = scaleAddress(3, 0) - io.request.valid := io.enable && state === request - io.request.bits.addr := Mux(scaleMode, scaleAddress >> 4, row * (fullK >> 4) + (startK >> 4) + offset) - io.response.ready := io.enable && state === response - io.done := state === done - io.write := io.response.fire && !scaleMode - io.scaleWrite := io.enable && state === scales - io.lane := row(3, 0) - io.address := (row >> 4) * Mux(scaleMode, count >> 5, count >> 4) + offset - io.word := io.response.bits.data - io.scale := (word >> (byte << 3))(7, 0) - when(io.start) { - rows := io.rows - count := io.count - fullK := io.fullK - startK := io.startK - row := 0.U - offset := 0.U - scaleMode := false.B - state := request - } - when(io.request.fire)(state := response) - when(io.response.fire) { - when(scaleMode) { word := io.response.bits.data; state := scales }.otherwise { - when(offset + 1.U === (count >> 4)) { - offset := 0.U - when(row + 1.U === rows) { row := 0.U; scaleMode := true.B } - .otherwise(row := row + 1.U) - }.otherwise(offset := offset + 1.U) - state := request - } - } - when(io.enable && state === scales) { - when(offset + 1.U === (count >> 5)) { - offset := 0.U - when(row + 1.U === rows)(state := done) - .otherwise { row := row + 1.U; state := request } - }.otherwise { - offset := offset + 1.U - when(byte === 15.U)(state := request) - } - } -} diff --git a/examples/balls/mxmm/arch/src/main/scala/MxmmBall.scala b/examples/balls/mxmm/arch/src/main/scala/MxmmBall.scala deleted file mode 100644 index 847866b8b..000000000 --- a/examples/balls/mxmm/arch/src/main/scala/MxmmBall.scala +++ /dev/null @@ -1,26 +0,0 @@ -package examples.balls.mxmm - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import framework.balldomain.blink.{BallStatus, BlinkIO, HasBallStatus, HasBlink, SubRobRow} -import framework.top.GlobalConfig -import framework.balldomain.blink.mmio.{MmioRead, MmioWrite} - -@instantiable -class MxmmBall(val b: GlobalConfig) extends Module with HasBlink with HasBallStatus { - @public - val io = IO(new BlinkIO(b, 2, 1)) - def blink: BlinkIO = io - def status: BallStatus = io.status - val unit = Instantiate(new Unit(b)) - unit.io.cmdReq <> io.cmdReq - unit.io.cmdResp <> io.cmdResp - unit.io.channelReady := io.channelReady - unit.io.bankRead <> io.bankRead - unit.io.bankWrite <> io.bankWrite - io.status <> unit.io.status - io.subRobReq.valid := false.B - io.subRobReq.bits := SubRobRow.tieOff(b) - MmioRead.tieOff(io.mmioRead) - MmioWrite.tieOff(io.mmioWrite) -} diff --git a/examples/balls/mxmm/arch/src/main/scala/PE.scala b/examples/balls/mxmm/arch/src/main/scala/PE.scala deleted file mode 100644 index ca89bc1c6..000000000 --- a/examples/balls/mxmm/arch/src/main/scala/PE.scala +++ /dev/null @@ -1,48 +0,0 @@ -package examples.balls.mxmm - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import freechips.rocketchip.tile.MulAddRecFNPipe -import hardfloat._ - -@instantiable -class PE(latency: Int) extends Module { - require(latency == 3) - - @public - val io = IO(new Bundle { - val valid = Input(Bool()) - val separate = Input(Bool()) - val a = Input(UInt(32.W)) - val b = Input(UInt(32.W)) - val c = Input(UInt(33.W)) - val out = Output(UInt(33.W)) - val outValid = Output(Bool()) - }) - - val multiply = Module(new MulAddRecFNPipe(latency, 8, 24)) - multiply.io.validin := io.valid - multiply.io.op := 0.U - multiply.io.a := recFNFromFN(8, 24, io.a) - multiply.io.b := recFNFromFN(8, 24, io.b) - multiply.io.c := Mux(io.separate, Cat(io.a(31) ^ io.b(31), 0.U(32.W)), io.c) - multiply.io.roundingMode := consts.round_near_even - multiply.io.detectTininess := consts.tininess_afterRounding.asUInt - - val productMode = Pipe(io.valid, io.separate, latency) - val previous = Pipe(io.valid && io.separate, io.c, latency) - val add = Instantiate(new AddPipe) - add.io.valid := multiply.io.validout && productMode.bits - add.io.a := multiply.io.out - add.io.b := previous.bits - when(add.io.valid) { - assert(fNFromRecFN(8, 24, multiply.io.out)(30, 23) =/= 255.U, "MXMM_F32 product must remain finite") - } - val fusedValid = multiply.io.validout && !productMode.bits - io.outValid := fusedValid || add.io.outValid - io.out := Mux(add.io.outValid, add.io.out, multiply.io.out) - when(io.outValid) { - assert(fNFromRecFN(8, 24, io.out)(30, 23) =/= 255.U, "MATMUL accumulation must remain finite") - } -} diff --git a/examples/balls/mxmm/arch/src/main/scala/Unit.scala b/examples/balls/mxmm/arch/src/main/scala/Unit.scala deleted file mode 100644 index 1174774ee..000000000 --- a/examples/balls/mxmm/arch/src/main/scala/Unit.scala +++ /dev/null @@ -1,377 +0,0 @@ -package examples.balls.mxmm - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import framework.balldomain.rs.{BallRsComplete, BallRsIssue} -import framework.balldomain.blink.{BallStatus, BankRead, BankWrite} -import framework.top.GlobalConfig -import examples.balls.mxmm.configs.MxmmBallParam - -@instantiable -class Unit(b: GlobalConfig) extends Module { - private val p = MxmmBallParam(b) - private val entries = b.memDomain.bankEntries - private val addressWidth = log2Ceil(entries) - require(b.memDomain.bankWidth == 128 && b.memDomain.bankMaskLen == 16) - require(entries >= 64 && entries % 32 == 0) - private val mapping = b.ballDomain.ballIdMappings.find(_.ballName == "MxmmBall").get - require(mapping.inBW == 2 && mapping.outBW == 1) - - @public - val io = IO(new Bundle { - val cmdReq = Flipped(Decoupled(new BallRsIssue(b))) - val cmdResp = Decoupled(new BallRsComplete(b)) - val bankRead = Vec(2, Flipped(new BankRead(b))) - val bankWrite = Vec(1, Flipped(new BankWrite(b))) - val channelReady = Input(Bool()) - val status = new BallStatus - }) - - private val Seq( - idle, - waitChannels, - loadInputs, - initRequest, - initResponse, - compute, - drain, - store, - outputRequest, - outputResponse, - outputWrite, - outputAck, - complete - ) = Enum(13) - - private val state = RegInit(idle) - private val command = Reg(new BallRsIssue(b)) - private val m = Reg(UInt(12.W)) - private val n = Reg(UInt(12.W)) - private val k = Reg(UInt(34.W)) - private val mx = Reg(Bool()) - private val windowed = Reg(Bool()) - private val separate = Reg(Bool()) - private val first = Reg(Bool()) - private val last = Reg(Bool()) - private val base = Reg(UInt(6.W)) - private val chain = RegInit(false.B) - private val chainM = Reg(UInt(12.W)) - private val chainN = Reg(UInt(12.W)) - private val chainFormat = Reg(UInt(7.W)) - private val chainBank = Reg(UInt(b.memDomain.vbankIdWidth.W)) - private val chainBase = Reg(UInt(6.W)) - - private val nGroups = n >> 4 - private val panelHeight = Mux(m === 1.U, 1.U, 16.U) - private val totalPanels = Mux(m === 1.U, 1.U, m >> 4) * nGroups - private val batch = Reg(UInt(addressWidth.W)) - private val count = Mux(totalPanels - batch > p.contexts.U, p.contexts.U, totalPanels - batch) - private val minimumPeriod = Mux(separate, (2 * p.arithmeticLatency + 1).U, (p.arithmeticLatency + 1).U) - private val period = Mux(count > minimumPeriod, count, minimumPeriod) - private val context = Reg(UInt(log2Ceil(p.contexts).W)) - private val row = Reg(UInt(4.W)) - private val reduction = Reg(UInt(34.W)) - private val completed = Reg(UInt(48.W)) - private val panel = batch + context - private val rowGroup = panel / nGroups - private val columnGroup = panel % nGroups - private val accAddress = rowGroup * 16.U * nGroups + row * nGroups + columnGroup - private val accumulator = SyncReadMem(entries / 4, UInt(512.W)) - private val savedRow = Reg(UInt(512.W)) - private val outputLine = Reg(UInt(addressWidth.W)) - private val outputQuarter = Reg(UInt(2.W)) - - private val panels = Instantiate(new Panels(entries)) - private val array = Instantiate(new Array(p)) - private val readValid = state === compute && context < count - panels.io.read := readValid - panels.io.rowGroup := rowGroup - panels.io.columnGroup := columnGroup - panels.io.k := reduction - panels.io.reduction := k - panels.io.mxfp8 := mx - panels.io.vector := m === 1.U - array.io.valid := panels.io.valid - array.io.a := panels.io.a - array.io.b := panels.io.b - array.io.separate := separate - array.io.vector := m === 1.U - val computeContext = RegNext(context) - array.io.context := Mux(state === compute || state === drain, computeContext, context) - array.io.row := row - array.io.load := false.B - array.io.rowData := 0.U - - private val accReadEnable = state === outputRequest || (state === initRequest && !first && row < panelHeight) - private val accWriteEnable = state === store && row < panelHeight - - private val accRead = accumulator.readWrite( - Mux(state === outputRequest, outputLine, accAddress), - array.io.rowOut, - accReadEnable || accWriteEnable, - accWriteEnable - ) - - assert(!(accReadEnable && accWriteEnable)) - - private val inputState = Reg(Vec(2, UInt(2.W))) - private val inputWord = Reg(Vec(2, UInt(128.W))) - private val inputIndex = Reg(Vec(2, UInt(addressWidth.W))) - private val inputRow = Reg(Vec(2, UInt(12.W))) - private val rowOffset = Reg(Vec(2, UInt(addressWidth.W))) - private val scaleMode = Reg(Vec(2, Bool())) - private val scaleByte = Reg(Vec(2, UInt(4.W))) - private val scaleCount = Reg(Vec(2, UInt((addressWidth + 4).W))) - private val rowWords = Mux(mx, k >> 4, k >> 2) - private val window = Module(new InputWindow(b)) - window.io.start := io.cmdReq.fire && io.cmdReq.bits.cmd.funct7 === 75.U - window.io.enable := state === loadInputs && windowed - window.io.rows := io.cmdReq.bits.cmd.rs2(11, 0) - window.io.count := io.cmdReq.bits.cmd.rs1(63, 30) - window.io.fullK := io.cmdReq.bits.cmd.rs2(47, 32) - window.io.startK := io.cmdReq.bits.cmd.rs2(63, 48) - window.io.request.ready := io.bankRead(0).io.req.ready && windowed - window.io.response.valid := io.bankRead(0).io.resp.valid && windowed - window.io.response.bits := io.bankRead(0).io.resp.bits - - io.cmdReq.ready := state === idle - io.cmdResp.valid := state === complete - io.cmdResp.bits.rob_id := command.rob_id - io.cmdResp.bits.is_sub := command.is_sub - io.cmdResp.bits.sub_rob_id := command.sub_rob_id - io.status.idle := state === idle - io.status.running := state =/= idle && state =/= complete - - for (operand <- 0 until 2) { - val rows = if (operand == 0) m else n - val codeWords = rows * rowWords - val scales = rows * (k >> 5) - val port = io.bankRead(operand) - port.rob_id := command.rob_id - port.ball_id := 0.U - port.group_id := 0.U - port.bank_id := (if (operand == 0) command.cmd.op1_bank else command.cmd.op2_bank) - port.io.req.valid := state === loadInputs && inputState(operand) === 0.U - port.io.req.bits.addr := inputIndex(operand) - port.io.resp.ready := state === loadInputs && inputState(operand) === 1.U - panels.io.write(operand) := port.io.resp.fire && !scaleMode(operand) - panels.io.scaleWrite(operand) := state === loadInputs && inputState(operand) === 2.U - panels.io.lane(operand) := inputRow(operand)(3, 0) - panels.io.address(operand) := (inputRow(operand) >> 4) * Mux(scaleMode(operand), k >> 5, rowWords) + rowOffset( - operand - ) - panels.io.word(operand) := port.io.resp.bits.data - panels.io.scale(operand) := (inputWord(operand) >> (scaleByte(operand) << 3))(7, 0) - val legacy = if (operand == 0) !windowed else true.B - when(port.io.req.fire && legacy)(inputState(operand) := 1.U) - when(port.io.resp.fire && legacy) { - when(scaleMode(operand)) { - inputWord(operand) := port.io.resp.bits.data - scaleByte(operand) := 0.U - inputState(operand) := 2.U - }.otherwise { - inputIndex(operand) := inputIndex(operand) + 1.U - when(rowOffset(operand) + 1.U === rowWords) { - rowOffset(operand) := 0.U - inputRow(operand) := inputRow(operand) + 1.U - }.otherwise(rowOffset(operand) := rowOffset(operand) + 1.U) - when(inputIndex(operand) +& 1.U === codeWords) { - when(mx) { - scaleMode(operand) := true.B - inputRow(operand) := 0.U - rowOffset(operand) := 0.U - inputState(operand) := 0.U - }.otherwise(inputState(operand) := 3.U) - }.otherwise(inputState(operand) := 0.U) - } - } - when(state === loadInputs && inputState(operand) === 2.U && legacy) { - scaleCount(operand) := scaleCount(operand) + 1.U - scaleByte(operand) := scaleByte(operand) + 1.U - when(rowOffset(operand) + 1.U === (k >> 5)) { - rowOffset(operand) := 0.U - inputRow(operand) := inputRow(operand) + 1.U - }.otherwise(rowOffset(operand) := rowOffset(operand) + 1.U) - when(scaleCount(operand) + 1.U === scales) { - inputState(operand) := 3.U - }.elsewhen(scaleByte(operand) === 15.U) { - inputIndex(operand) := inputIndex(operand) + 1.U - inputState(operand) := 0.U - } - } - } - - when(windowed) { - io.bankRead(0).io.req.valid := window.io.request.valid - io.bankRead(0).io.req.bits.addr := window.io.request.bits.addr - io.bankRead(0).io.resp.ready := window.io.response.ready - panels.io.write(0) := window.io.write - panels.io.scaleWrite(0) := window.io.scaleWrite - panels.io.lane(0) := window.io.lane - panels.io.address(0) := window.io.address - panels.io.word(0) := window.io.word - panels.io.scale(0) := window.io.scale - } - - private val writePort = io.bankWrite(0) - writePort.rob_id := command.rob_id - writePort.ball_id := 0.U - writePort.group_id := 0.U - writePort.bank_id := command.cmd.wr_bank - writePort.io.req.valid := state === outputWrite - writePort.io.req.bits.addr := base + (outputLine << 2) + outputQuarter - writePort.io.req.bits.data := (savedRow >> (outputQuarter << 7))(127, 0) - writePort.io.req.bits.mask := VecInit(Seq.fill(16)(true.B)) - writePort.io.resp.ready := state === outputAck - - when(io.cmdReq.fire) { - val cmd = io.cmdReq.bits.cmd - val rows = cmd.rs2(11, 0) - val cols = cmd.rs2(23, 12) - val inputK = cmd.rs1(63, 30) - val isWindow = cmd.funct7 === 75.U - val isMx = cmd.funct7 === 71.U || isWindow - assert(isMx || cmd.funct7 === 72.U || cmd.funct7 === 73.U, "MATMUL format is invalid") - assert(rows === 1.U || (rows =/= 0.U && rows(3, 0) === 0.U), "MATMUL M must be one or a positive multiple of 16") - assert(cols =/= 0.U && cols(3, 0) === 0.U, "MATMUL N must be a positive multiple of 16") - assert(inputK =/= 0.U && Mux(isMx, inputK(4, 0) === 0.U, inputK(1, 0) === 0.U), "MATMUL K alignment is invalid") - when(isWindow) { - val full = cmd.rs2(47, 32) - val start = cmd.rs2(63, 48) - assert(full =/= 0.U && full(4, 0) === 0.U && start(4, 0) === 0.U, "MATMUL A window alignment is invalid") - assert(start +& inputK <= full, "MATMUL A window exceeds fullK") - }.otherwise(assert(cmd.rs2(63, 32) === 0.U, "MATMUL reserves rs2[63:32]")) - assert( - cmd.op1_bank =/= cmd.op2_bank && cmd.op1_bank =/= cmd.wr_bank && cmd.op2_bank =/= cmd.wr_bank, - "MATMUL requires distinct banks" - ) - assert( - cmd.op1_col === 1.U && cmd.op2_col === 1.U && cmd.wr_col === 1.U, - "MATMUL requires one physical bank per operand" - ) - assert(cmd.rs1(9, 0) < b.memDomain.virtualBankCount.U && cmd.rs1(19, 10) < b.memDomain.virtualBankCount.U && cmd.rs1( - 29, - 20 - ) < b.memDomain.virtualBankCount.U) - val aElements = rows * Mux(isWindow, cmd.rs2(47, 32), inputK) - val bElements = cols * inputK - assert(Mux(isMx, aElements + (aElements >> 5), aElements << 2) <= (entries * 16).U, "MATMUL A exceeds bank capacity") - assert(Mux(isMx, bElements + (bElements >> 5), bElements << 2) <= (entries * 16).U, "MATMUL B exceeds bank capacity") - assert((cmd.rs2(31, 26) << 4) +& ((rows * cols) << 2) <= (entries * 16).U, "MATMUL C exceeds bank capacity") - when(cmd.rs2(24)) { - assert(!chain, "MATMUL first cannot replace a live chain") - chain := true.B - chainM := rows - chainN := cols - chainFormat := cmd.funct7 - chainBank := cmd.wr_bank - chainBase := cmd.rs2(31, 26) - }.otherwise { - assert( - chain && chainM === rows && chainN === cols && chainFormat === cmd.funct7 && chainBank === cmd.wr_bank && chainBase === cmd.rs2( - 31, - 26 - ), - "MATMUL continuation changed its chain" - ) - } - command := io.cmdReq.bits - m := rows - n := cols - k := inputK - mx := isMx - windowed := isWindow - separate := cmd.funct7 === 73.U - first := cmd.rs2(24) - last := cmd.rs2(25) - base := cmd.rs2(31, 26) - batch := 0.U - context := 0.U - row := 0.U - reduction := 0.U - completed := 0.U - for (operand <- 0 until 2) { - inputState(operand) := 0.U - inputIndex(operand) := 0.U - inputRow(operand) := 0.U - rowOffset(operand) := 0.U - scaleMode(operand) := false.B - scaleCount(operand) := 0.U - } - state := waitChannels - } - - switch(state) { - is(waitChannels)(when(io.channelReady)(state := loadInputs)) - is(loadInputs)( - when(Mux(windowed, window.io.done, inputState(0) === 3.U) && inputState(1) === 3.U)(state := initRequest) - ) - is(initRequest) { - when(first || row >= panelHeight) { - array.io.load := true.B - when(row === 15.U) { - row := 0.U - when(context + 1.U === count) { context := 0.U; state := compute } - .otherwise(context := context + 1.U) - }.otherwise(row := row + 1.U) - }.otherwise(state := initResponse) - } - is(initResponse) { - array.io.load := true.B - array.io.rowData := accRead - when(row === 15.U) { - row := 0.U - when(context + 1.U === count) { context := 0.U; state := compute } - .otherwise { context := context + 1.U; state := initRequest } - }.otherwise { row := row + 1.U; state := initRequest } - } - is(compute) { - when(context + 1.U === period) { - context := 0.U - when(reduction + 1.U === k)(state := drain) - .otherwise(reduction := reduction + 1.U) - }.otherwise(context := context + 1.U) - } - is(store) { - when(row === 15.U) { - row := 0.U - when(context + 1.U === count) { - context := 0.U - when(batch + count === totalPanels) { - when(last) { outputLine := 0.U; outputQuarter := 0.U; state := outputRequest } - .otherwise(state := complete) - }.otherwise { - batch := batch + count - reduction := 0.U - completed := 0.U - state := initRequest - } - }.otherwise(context := context + 1.U) - }.otherwise(row := row + 1.U) - } - is(outputRequest)(state := outputResponse) - is(outputResponse) { savedRow := accRead; state := outputWrite } - is(outputWrite)(when(writePort.io.req.fire)(state := outputAck)) - is(outputAck) { - when(writePort.io.resp.fire) { - assert(writePort.io.resp.bits.ok, "MATMUL bank write failed") - when(outputQuarter === 3.U) { - outputQuarter := 0.U - when(outputLine + 1.U === m * nGroups) { chain := false.B; state := complete } - .otherwise { outputLine := outputLine + 1.U; state := outputRequest } - }.otherwise { outputQuarter := outputQuarter + 1.U; state := outputWrite } - } - } - is(complete)(when(io.cmdResp.fire)(state := idle)) - } - when(array.io.completed) { - assert(state === compute || state === drain) - completed := completed + 1.U - when(completed + 1.U === count * k) { - context := 0.U - row := 0.U - state := store - } - } -} diff --git a/examples/balls/mxmm/compiler/python/fp32.py b/examples/balls/mxmm/compiler/python/fp32.py deleted file mode 100644 index b6b200ca9..000000000 --- a/examples/balls/mxmm/compiler/python/fp32.py +++ /dev/null @@ -1,66 +0,0 @@ -import array - -from buddy_mlir import ir - - -def lower(node, symbols): - lhs, rhs = [symbols[(name, 0)] for name in node.args] - result = ir.RankedTensorType.get(node.tensor_meta["shape"], ir.F32Type.get()) - attributes = {"fused": ir.BoolAttr.get(node.fused)} - owner = rhs.owner - if isinstance(owner, ir.OpView): - owner = owner.operation - reshape = None - if isinstance(owner, ir.Operation) and owner.name == "tosa.reshape": - original = ir.RankedTensorType(owner.operands[0].type) - current = ir.RankedTensorType(rhs.type) - if ( - original.rank == 4 - and original.shape[0] == 1 - and list(current.shape) == list(original.shape[1:]) - ): - reshape = current - owner = owner.operands[0].owner - if isinstance(owner, ir.OpView): - owner = owner.operation - if isinstance(owner, ir.Operation) and owner.name == "tosa.transpose": - rank = ir.RankedTensorType(owner.operands[0].type).rank - permutation = list(ir.DenseI32ArrayAttr(owner.attributes["perms"])) - expected = list(range(rank)) - expected[-2:] = [rank - 1, rank - 2] - if permutation == expected: - rhs = owner.operands[0] - if reshape is not None: - shape = list(reshape.shape) - shape[-2:] = [shape[-1], shape[-2]] - tensor = ir.RankedTensorType.get(shape, ir.F32Type.get()) - shape_type = ir.Type.parse(f"!tosa.shape<{len(shape)}>") - dimensions = ir.Operation.create( - "tosa.const_shape", - results=[shape_type], - attributes={ - "values": ir.DenseElementsAttr.get( - array.array("q", shape), - type=ir.IndexType.get(), - shape=[len(shape)], - ) - }, - ).result - rhs = ir.Operation.create( - "tosa.reshape", operands=[rhs, dimensions], results=[tensor] - ).result - attributes["rhs_transposed"] = ir.UnitAttr.get() - return ir.Operation.create( - "buckyball.fp32_matmul", - operands=[lhs, rhs], - results=[result], - attributes=attributes, - ).result - - -def apply(graph, *, fused): - for node in graph._body: - if node.__class__.__name__ in ("MatmulOp", "BatchMatmulOp"): - node.fused = fused - graph._ops_registry["MatmulOp"] = lower - graph._ops_registry["BatchMatmulOp"] = lower diff --git a/examples/balls/mxmm/compiler/python/grouped_attention.py b/examples/balls/mxmm/compiler/python/grouped_attention.py deleted file mode 100644 index bd4a38a27..000000000 --- a/examples/balls/mxmm/compiler/python/grouped_attention.py +++ /dev/null @@ -1,107 +0,0 @@ -import array -import math - -from buddy_mlir import ir - -from . import fp32 - - -def reshape(value, shape): - shape_type = ir.Type.parse(f"!tosa.shape<{len(shape)}>") - dimensions = ir.Operation.create( - "tosa.const_shape", - results=[shape_type], - attributes={ - "values": ir.DenseElementsAttr.get( - array.array("q", shape), type=ir.IndexType.get(), shape=[len(shape)] - ) - }, - ).result - result_type = ir.RankedTensorType.get(shape, ir.F32Type.get()) - return ir.Operation.create( - "tosa.reshape", operands=[value, dimensions], results=[result_type] - ).result - - -def lower(node, symbols): - result = fp32.lower(node, symbols) - native = result.owner - if isinstance(native, ir.OpView): - native = native.operation - lhs, rhs = native.operands - repeated = rhs - while True: - owner = repeated.owner - if isinstance(owner, ir.OpView): - owner = owner.operation - if not isinstance(owner, ir.Operation) or owner.name != "tosa.reshape": - break - repeated = owner.operands[0] - # Other matmuls keep their native lowering. This pattern is a five-axis KV repeat. - repeated_type = ir.RankedTensorType(repeated.type) - if ( - not isinstance(owner, ir.Operation) - or owner.name != "tosa.add" - or repeated_type.rank != 5 - ): - return result - source, zeros = owner.operands - zero_op = zeros.owner - if isinstance(zero_op, ir.OpView): - zero_op = zero_op.operation - if not isinstance(zero_op, ir.Operation) or zero_op.name != "tosa.const": - raise ValueError("grouped attention requires an explicit +0 KV broadcast") - zero_values = ir.DenseElementsAttr(zero_op.attributes["values"]) - if not zero_values.is_splat: - raise ValueError("grouped attention KV broadcast must contain only +0") - zero = ir.FloatAttr(zero_values.get_splat_value()).value - if zero != 0.0 or math.copysign(1.0, zero) != 1.0: - raise ValueError("grouped attention KV broadcast must contain only +0") - if ir.BoolAttr(native.attributes["fused"]).value: - raise ValueError("grouped attention requires unfused FP32 accumulation") - if any(size <= 0 for size in repeated_type.shape): - raise ValueError("grouped attention requires positive static dimensions") - batch, kv_heads, groups, length, dimension = repeated_type.shape - source_shape = list(ir.RankedTensorType(source.type).shape) - left_shape = list(ir.RankedTensorType(lhs.type).shape) - right_shape = list(ir.RankedTensorType(rhs.type).shape) - transposed = "rhs_transposed" in native.attributes - reduction, columns = (dimension, length) if transposed else (length, dimension) - if ( - source_shape != [batch, kv_heads, 1, length, dimension] - or len(left_shape) != 3 - or left_shape[0] != batch * kv_heads * groups - or left_shape[2] != reduction - or right_shape != [batch * kv_heads * groups, length, dimension] - ): - raise ValueError( - "grouped attention requires contiguous head-major Q/P and KV shapes" - ) - rows = left_shape[1] - if rows <= 0: - raise ValueError("grouped attention requires a positive static row count") - grouped_left = reshape(lhs, [batch * kv_heads, groups * rows, reduction]) - grouped_right = reshape(source, [batch * kv_heads, length, dimension]) - attributes = {"fused": ir.BoolAttr.get(False)} - if transposed: - attributes["rhs_transposed"] = ir.UnitAttr.get() - grouped_type = ir.RankedTensorType.get( - [batch * kv_heads, groups * rows, columns], ir.F32Type.get() - ) - grouped = ir.Operation.create( - "buckyball.fp32_matmul", - operands=[grouped_left, grouped_right], - results=[grouped_type], - attributes=attributes, - ).result - shape = list(ir.RankedTensorType(result.type).shape) - native.erase() - # With finite operands, RNE and initial +0, removing the repeat's +0 changes - # only signed-zero products, which cannot change an unfused dot's result. - return reshape(grouped, shape) - - -def apply(graph): - fp32.apply(graph, fused=False) - graph._ops_registry["MatmulOp"] = lower - graph._ops_registry["BatchMatmulOp"] = lower diff --git a/examples/balls/mxmm/compiler/python/layout.py b/examples/balls/mxmm/compiler/python/layout.py deleted file mode 100644 index f506ac778..000000000 --- a/examples/balls/mxmm/compiler/python/layout.py +++ /dev/null @@ -1,98 +0,0 @@ -import math - -import torch - - -def plan(rows, columns, reduction, bank_bytes, *, mxfp8): - alignment = 32 if mxfp8 else 4 - if min(rows, columns, reduction, bank_bytes) <= 0 or reduction % alignment: - raise ValueError("matmul shape violates the format alignment") - row_limit = 1 if rows == 1 else (rows + 15) // 16 * 16 - column_limit = (columns + 15) // 16 * 16 - candidates = [] - for m in ([1] if rows == 1 else range(16, row_limit + 1, 16)): - for n in range(16, column_limit + 1, 16): - if m * n * 4 > bank_bytes: - continue - factor = 33 / 32 if mxfp8 else 4 - k = min(reduction, int(bank_bytes / (max(m, n) * factor))) - k = k // alignment * alignment - if not k: - continue - panels = ( - math.ceil(rows / m) * math.ceil(columns / n) * math.ceil(reduction / k) - ) - quant_panels = ( - math.ceil(rows / m) * math.ceil(reduction / k) if mxfp8 else 0 - ) - candidates.append(((panels + quant_panels, panels, -k, -m, -n), (m, n, k))) - if not candidates: - raise ValueError("matmul cannot fit an aligned block in its banks") - m, n, k = min(candidates)[1] - return { - "tile_m": m, - "tile_n": n, - "tile_k": k, - "bank_bytes": bank_bytes, - "panel_stride": bank_bytes, - } - - -def pack(codes, scales, layout): - if ( - not isinstance(codes, torch.Tensor) - or not isinstance(scales, torch.Tensor) - or codes.dtype != torch.int8 - or scales.dtype != torch.int8 - ): - raise ValueError("MXFP8 packing requires INT8 Tensor codes and scales") - if codes.ndim != 2 or scales.device != codes.device: - raise ValueError("MXFP8 packing requires a matrix and matching devices") - rows, width = codes.shape - n, k = layout["tile_n"], layout["tile_k"] - stride = layout["panel_stride"] - if width % 32 or scales.shape != (rows, width // 32): - raise ValueError("MXFP8 codes and scales disagree") - panels = [] - for begin_row in range(0, rows, n): - for begin_k in range(0, width, k): - count = min(k, width - begin_k) - plane = torch.zeros((n, count), dtype=torch.int8, device=codes.device) - scale = torch.full( - (n, count // 32), 127, dtype=torch.int8, device=codes.device - ) - valid = min(n, rows - begin_row) - plane[:valid] = codes[ - begin_row : begin_row + valid, begin_k : begin_k + count - ] - scale[:valid] = scales[ - begin_row : begin_row + valid, begin_k // 32 : (begin_k + count) // 32 - ] - payload = torch.cat((plane.flatten(), scale.flatten())) - if payload.numel() > stride: - raise ValueError("MXFP8 panel exceeds its bank window") - panels.append( - torch.nn.functional.pad(payload, (0, stride - payload.numel())) - ) - return torch.cat(panels) - - -def bank_bytes(compiler_build, target): - import re - from pathlib import Path - - text = ( - Path(compiler_build) / "external_dialects/BuckyballTargetRegistry.inc" - ).read_text() - configs = { - name: int(width) * int(depth) // 8 - for name, width, depth in re.findall( - r'\{"([^"]+)", "[^"]+", \d+, (\d+), (\d+), \d+, (?:true|false), llvm::ArrayRef', - text, - ) - } - if target not in configs: - raise ValueError( - f"Target {target} has no registered bank geometry in {compiler_build}" - ) - return configs[target] diff --git a/examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp b/examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp deleted file mode 100644 index 02ac7257f..000000000 --- a/examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp +++ /dev/null @@ -1,58 +0,0 @@ -#include "Buckyball/BuckyballOps.h" -#include "Conversion/LowerBuckyball/LowerBuckyball.h" -#include "mlir/IR/PatternMatch.h" -using namespace mlir; -using namespace ::buddy::buckyball; -namespace { -class BankMXFP8Pattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(BankMXFP8Op op, - PatternRewriter &rewriter) const override { - rewriter.create(op.getLoc(), op.getOp1Bank(), op.getOp2Bank(), - op.getWrBank(), op.getConfig(), op.getFirst(), - op.getLast(), op.getOutputBase()); - rewriter.replaceOp(op, op.getWrBank()); - return success(); - } -}; - -class BankMXFP8WindowPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(BankMXFP8WindowOp op, - PatternRewriter &rewriter) const override { - rewriter.create( - op.getLoc(), op.getOp1Bank(), op.getOp2Bank(), op.getWrBank(), - op.getRows(), op.getCols(), op.getCount(), op.getFullK(), - op.getStartK(), op.getFirst(), op.getLast(), op.getOutputBase()); - rewriter.replaceOp(op, op.getWrBank()); - return success(); - } -}; - -class BankFP32Pattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(BankFP32Op op, - PatternRewriter &rewriter) const override { - rewriter.create(op.getLoc(), op.getOp1Bank(), op.getOp2Bank(), - op.getWrBank(), op.getConfig(), op.getFirst(), - op.getLast(), op.getOutputBase(), - op.getFusedAttr()); - rewriter.replaceOp(op, op.getWrBank()); - return success(); - } -}; - -} // namespace -namespace mlir::buddy { -void populateMxmmBallAssignPhysicalBankPatterns(RewritePatternSet &patterns, - PhysicalBankState &) { - patterns.add( - patterns.getContext()); -} - -} // namespace mlir::buddy diff --git a/examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/FP32Patterns.inc b/examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/FP32Patterns.inc deleted file mode 100644 index b77d73a1d..000000000 --- a/examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/FP32Patterns.inc +++ /dev/null @@ -1,337 +0,0 @@ -class FP32ToBanks : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(FP32MemMatmulOp op, - PatternRewriter &b) const override { - auto a = cast(op.getLhs().getType()), - w = cast(op.getRhs().getType()), - out = cast(op.getOutput().getType()); - int rank = a.getRank(); - if ((rank != 2 && rank != 3) || w.getRank() != rank || - out.getRank() != rank || !a.hasStaticShape() || !w.hasStaticShape() || - !out.hasStaticShape() || !a.getElementType().isF32() || - !w.getElementType().isF32() || !out.getElementType().isF32()) - return op.emitError( - "FP32 matmul requires static rank-2 or rank-3 FP32 operands"); - int64_t M = a.getShape()[rank - 2], K = a.getShape()[rank - 1], - N = w.getShape()[rank - 1], - batches = rank == 3 ? a.getShape()[0] : 1; - if (M <= 0 || N <= 0 || K <= 0 || K % 4 || w.getShape()[rank - 2] != K || - out.getShape()[rank - 2] != M || out.getShape()[rank - 1] != N || - (rank == 3 && - (w.getShape()[0] != batches || out.getShape()[0] != batches))) - return op.emitError( - "FP32 matmul shape mismatch or K not divisible by four"); - const auto &target = buckyball_target::getBuckyballTarget(); - if (target.bankWidthBits != 128) - return op.emitError("FP32 matmul requires 128-bit bank rows"); - int64_t bytes = target.bankDepth * 16, rows = 0, columns = 0, chunk = 0; - std::tuple best{INT64_MAX, 0, 0, 0}; - for (int64_t m = M == 1 ? 1 : 16; m <= (M == 1 ? 1 : (M + 15) / 16 * 16); - m += 16) - for (int64_t n = 16; n <= (N + 15) / 16 * 16; n += 16) { - if (m * n * 4 > bytes) - continue; - int64_t k = std::min(K, bytes / (4 * std::max(m, n))) / 4 * 4; - if (!k) - continue; - auto score = std::tuple{((M + m - 1) / m) * ((N + n - 1) / n) * - ((K + k - 1) / k), - -k, -m, -n}; - if (score < best) { - best = score; - rows = m; - columns = n; - chunk = k; - } - } - if (!rows) - return op.emitError("FP32 matmul cannot fit a bank block"); - SmallVector rhsStrides; - int64_t rhsOffset; - bool bankPack = - target.bankNum >= 5 && target.rvvEnabled && - succeeded(w.getStridesAndOffset(rhsStrides, rhsOffset)) && - rhsStrides[rank - 2] == N && rhsStrides.back() == 1 && - K * N * 4 <= bytes && chunk == K && rows * columns * 4 >= 64; - Location loc = op.getLoc(); - auto idx = [&](int64_t v) -> Value { - return b.create(loc, v); - }; - Value zero = idx(0), one = idx(1), four = idx(4); - Value ab = allocBank(b, loc, 1, 1), wb = allocBank(b, loc, 1, 1), - cb = allocBank(b, loc, 1, 1); - auto packType = MemRefType::get({bytes / 16, 4}, b.getF32Type()); - Value ap = b.create(loc, packType), - bp = b.create(loc, packType), - cp = b.create(loc, packType); - for (Value buffer : {ap, bp, cp}) - buffer.getDefiningOp()->setAttr("alignment", b.getI64IntegerAttr(bytes)); - SmallVector banks{ab, wb, cb}; - if (bankPack) { - Value input = allocBank(b, loc, 1, 1); - Value descriptor = allocBank(b, loc, 1, 1); - Value read = b.create(loc, b.getI64Type(), ab, input); - Value write = b.create(loc, b.getI64Type(), wb, descriptor); - write = b.create(loc, b.getI64Type(), cb, write); - auto loaded = b.create( - loc, b.getI64Type(), b.getI64Type(), read, write, - FlatSymbolRefAttr::get(b.getContext(), "rvv_packmm"), ValueRange{ - createI64Const(b, loc, 0), createI64Const(b, loc, K), - createI64Const(b, loc, N), createI64Const(b, loc, 0), - createI64Const(b, loc, 0), createI64Const(b, loc, K), - createI64Const(b, loc, columns), createI64Const(b, loc, rows), - createI64Const(b, loc, 1), createI64Const(b, loc, 1), - createI64Const(b, loc, op.getFusedAttr().getValue())}); - banks = {loaded.getReadOut(), loaded.getWriteOut()}; - } - auto batchLoop = b.create(loc, zero, idx(batches), one, banks); - b.setInsertionPointToStart(batchLoop.getBody()); - Value batch = batchLoop.getInductionVar(); - SmallVector batchBanks(batchLoop.getRegionIterArgs()); - if (bankPack) { - SmallVector offsets(rank, b.getIndexAttr(0)); - SmallVector sizes; - for (int64_t dim : w.getShape()) - sizes.push_back(b.getIndexAttr(dim)); - SmallVector strides(rank, b.getIndexAttr(1)); - if (rank == 3) { - offsets[0] = batch; - sizes[0] = b.getIndexAttr(1); - } - auto rowType = memref::SubViewOp::inferRankReducedResultType( - {K, N}, w, offsets, sizes, strides); - Value source = b.create(loc, rowType, op.getRhs(), - offsets, sizes, strides); - Value flat = b.create( - loc, source, ArrayRef{{0, 1}}); - Value packed = b.create( - loc, ArrayRef{K * N / 4, 4}, flat, - ArrayRef{{0, 1}}); - batchBanks[0] = b.create( - loc, b.getI64Type(), packed, batchBanks[0], - createI64Const(b, loc, K * N / 4), createI64Const(b, loc, 1), - b.getI64IntegerAttr(0)); - } - auto rowLoop = - b.create(loc, zero, idx(M), idx(rows), batchBanks); - b.setInsertionPointToStart(rowLoop.getBody()); - Value row = rowLoop.getInductionVar(); - auto colLoop = b.create(loc, zero, idx(N), idx(columns), - rowLoop.getRegionIterArgs()); - b.setInsertionPointToStart(colLoop.getBody()); - Value column = colLoop.getInductionVar(); - ab = colLoop.getRegionIterArgs()[0]; - wb = colLoop.getRegionIterArgs()[1]; - cb = bankPack ? wb : colLoop.getRegionIterArgs()[2]; - for (int64_t begin = 0; begin < K; begin += chunk) { - int64_t count = std::min(chunk, K - begin); - for (int operand = 0; operand < (bankPack ? 1 : 2); ++operand) { - Value source = operand ? op.getRhs() : op.getLhs(); - Value packed = operand ? bp : ap; - auto sourceType = cast(source.getType()); - int axis = operand ? rank - 2 : rank - 1; - SmallVector strides; - int64_t offset; - if (failed(sourceType.getStridesAndOffset(strides, offset))) - return op.emitError("FP32 matmul requires strided operands"); - int64_t blockRows = operand ? columns : rows; - int64_t logicalRows = operand ? N : M; - Value base = operand ? column : row; - Value validRows = b.create( - loc, idx(blockRows), - b.create(loc, idx(logicalRows), base)); - auto flatType = MemRefType::get({bytes / 4}, b.getF32Type()); - Value flatBuffer = b.create( - loc, flatType, packed, ArrayRef{{0, 1}}); - auto rLoop = b.create(loc, zero, validRows, one); - b.setInsertionPointToStart(rLoop.getBody()); - Value r = rLoop.getInductionVar(); - Value global = b.create(loc, base, r); - Value flat = b.create(loc, r, idx(count)); - if (strides[axis] == 1) { - SmallVector sourceOffsets(rank, b.getIndexAttr(0)); - SmallVector sourceSizes(rank, b.getIndexAttr(1)); - SmallVector sourceStrides(rank, b.getIndexAttr(1)); - if (rank == 3) - sourceOffsets[0] = batch; - sourceOffsets[rank - 2] = operand - ? OpFoldResult(b.getIndexAttr(begin)) - : OpFoldResult(global); - sourceOffsets[rank - 1] = operand - ? OpFoldResult(global) - : OpFoldResult(b.getIndexAttr(begin)); - sourceSizes[axis] = b.getIndexAttr(count); - auto sourceRowType = memref::SubViewOp::inferRankReducedResultType( - {count}, sourceType, sourceOffsets, sourceSizes, sourceStrides); - Value sourceRow = b.create( - loc, sourceRowType, source, sourceOffsets, sourceSizes, - sourceStrides); - SmallVector targetOffsets{flat}; - SmallVector targetSizes{b.getIndexAttr(count)}; - SmallVector targetStrides{b.getIndexAttr(1)}; - auto targetRowType = memref::SubViewOp::inferRankReducedResultType( - {count}, flatType, targetOffsets, targetSizes, targetStrides); - Value targetRow = b.create( - loc, targetRowType, flatBuffer, targetOffsets, targetSizes, - targetStrides); - b.create(loc, sourceRow, targetRow); - } else { - auto kLoop = b.create(loc, zero, idx(count), one); - b.setInsertionPointToStart(kLoop.getBody()); - Value k = kLoop.getInductionVar(); - Value kk = b.create(loc, k, idx(begin)); - SmallVector indices; - if (rank == 3) - indices.push_back(batch); - indices.push_back(operand ? kk : global); - indices.push_back(operand ? global : kk); - Value value = b.create(loc, source, indices); - Value destination = b.create(loc, flat, k); - b.create(loc, value, flatBuffer, - ValueRange{destination}); - } - b.setInsertionPointAfter(rLoop); - if (logicalRows % blockRows) { - Value start = b.create(loc, validRows, idx(count * 4)); - Value pointer = - b.create(loc, packed); - Value address = b.create(loc, pointer, start); - Value integer = - b.create(loc, b.getI64Type(), address); - Value destination = b.create( - loc, LLVM::LLVMPointerType::get(b.getContext()), integer); - Value length = - b.create(loc, idx(blockRows * count * 4), start); - Value byteLength = - b.create(loc, b.getI64Type(), length); - Value byteZero = b.create(loc, 0, 8); - b.create(loc, destination, byteZero, byteLength, - false); - } - } - if (bankPack) { - ab = b.create( - loc, b.getI64Type(), ap, ab, - createI64Const(b, loc, rows * count / 4), - createI64Const(b, loc, 1), b.getI64IntegerAttr(1)); - auto packed = b.create( - loc, b.getI64Type(), b.getI64Type(), ab, wb, - FlatSymbolRefAttr::get(b.getContext(), "rvv_packmm"), ValueRange{ - createI64Const(b, loc, 1), createI64Const(b, loc, K), - createI64Const(b, loc, N), createI64Const(b, loc, begin), - b.create(loc, b.getI64Type(), column), - createI64Const(b, loc, count), createI64Const(b, loc, columns), - createI64Const(b, loc, rows), createI64Const(b, loc, begin == 0), - createI64Const(b, loc, begin + count == K), - createI64Const(b, loc, op.getFusedAttr().getValue())}); - ab = packed.getReadOut(); - wb = cb = packed.getWriteOut(); - } else { - ab = mvinBank(b, loc, ap, ab, rows * count / 4); - wb = mvinBank(b, loc, bp, wb, columns * count / 4); - cb = b.create(loc, b.getI64Type(), ab, wb, cb, - createI64Const(b, loc, - packBits(rows, 0, 11) | - packBits(columns, 12, 23) | - packBits(count, 24, 35)), - createI1Const(b, loc, begin == 0), - createI1Const(b, loc, begin + count == K), - createI64Const(b, loc, 0), op.getFusedAttr()) - .getWrBankOut(); - } - } - if (bankPack) - cb = b.create( - loc, b.getI64Type(), cp, cb, - createI64Const(b, loc, rows * columns / 4), - createI64Const(b, loc, 1), b.getI64IntegerAttr(2)); - else - cb = mvoutBank(b, loc, cp, cb, rows * columns / 4); - SmallVector outputStrides; - int64_t outputOffset; - if (failed(out.getStridesAndOffset(outputStrides, outputOffset))) - return op.emitError("FP32 matmul requires a strided output"); - Value validColumns = b.create( - loc, idx(columns), b.create(loc, idx(N), column)); - auto rr = b.create( - loc, zero, - b.create(loc, idx(rows), - b.create(loc, idx(M), row)), - one); - b.setInsertionPointToStart(rr.getBody()); - Value r = rr.getInductionVar(); - if (outputStrides.back() == 1) { - auto flatType = MemRefType::get({bytes / 4}, b.getF32Type()); - Value flat = b.create( - loc, flatType, cp, ArrayRef{{0, 1}}); - SmallVector offsets{ - b.create(loc, r, idx(columns)).getResult()}; - SmallVector sizes{validColumns}, strides{b.getIndexAttr(1)}; - auto sourceType = memref::SubViewOp::inferRankReducedResultType( - {ShapedType::kDynamic}, flatType, offsets, sizes, strides); - Value source = b.create(loc, sourceType, flat, offsets, - sizes, strides); - SmallVector outOffsets(rank, b.getIndexAttr(0)); - SmallVector outSizes(rank, b.getIndexAttr(1)); - SmallVector outSteps(rank, b.getIndexAttr(1)); - if (rank == 3) - outOffsets[0] = batch; - outOffsets[rank - 2] = b.create(loc, row, r).getResult(); - outOffsets[rank - 1] = column; - outSizes.back() = validColumns; - auto rowType = memref::SubViewOp::inferRankReducedResultType( - {ShapedType::kDynamic}, out, outOffsets, outSizes, outSteps); - Value destination = b.create( - loc, rowType, op.getOutput(), outOffsets, outSizes, outSteps); - b.create(loc, source, destination); - } else { - auto cc = b.create(loc, zero, validColumns, one); - b.setInsertionPointToStart(cc.getBody()); - Value c = cc.getInductionVar(); - Value flat = b.create( - loc, b.create(loc, r, idx(columns)), c); - Value result = b.create( - loc, cp, - ValueRange{b.create(loc, flat, four), - b.create(loc, flat, four)}); - SmallVector outputIndices; - if (rank == 3) - outputIndices.push_back(batch); - outputIndices.push_back(b.create(loc, row, r)); - outputIndices.push_back(b.create(loc, column, c)); - b.create(loc, result, op.getOutput(), outputIndices); - } - b.setInsertionPointAfter(rr); - SmallVector nextBanks{ab, wb, cb}; - if (bankPack) - nextBanks = {ab, cb}; - b.create(loc, nextBanks); - b.setInsertionPointAfter(colLoop); - b.create(loc, colLoop.getResults()); - b.setInsertionPointAfter(rowLoop); - b.create(loc, rowLoop.getResults()); - b.setInsertionPointAfter(batchLoop); - if (bankPack) { - auto finished = b.create( - loc, b.getI64Type(), b.getI64Type(), batchLoop.getResult(0), - batchLoop.getResult(1), - FlatSymbolRefAttr::get(b.getContext(), "rvv_packmm"), ValueRange{ - createI64Const(b, loc, 2), createI64Const(b, loc, K), - createI64Const(b, loc, N), createI64Const(b, loc, 0), - createI64Const(b, loc, 0), createI64Const(b, loc, K), - createI64Const(b, loc, columns), createI64Const(b, loc, rows), - createI64Const(b, loc, 1), createI64Const(b, loc, 1), - createI64Const(b, loc, op.getFusedAttr().getValue())}); - releaseBank(b, loc, finished.getReadOut()); - releaseBank(b, loc, finished.getWriteOut()); - } else { - for (Value bank : batchLoop.getResults()) - releaseBank(b, loc, bank); - } - for (Value buffer : {ap, bp, cp}) - b.create(loc, buffer); - b.eraseOp(op); - return success(); - } -}; diff --git a/examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp b/examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp deleted file mode 100644 index 9a2219a17..000000000 --- a/examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp +++ /dev/null @@ -1,28 +0,0 @@ -#include "Buckyball/BuckyballOps.h" -#include "Conversion/LowerBuckyball/LowerBuckyball.h" -#include "Target/BuckyballTargetRegistry.h" -#include "Utils/BankUtils.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include -#include -#include -using namespace mlir; -using namespace ::buddy::buckyball; -namespace { -#include "FP32Patterns.inc" -#include "MXFP8Patterns.inc" -#include "SharedPanelPatterns.inc" -#include "SharedWindowPatterns.inc" -} // namespace -namespace mlir::buddy { -void populateMxmmBallLowerBuckyballToBankSSAPatterns( - RewritePatternSet &patterns) { - patterns.add(patterns.getContext()); -} - -} // namespace mlir::buddy diff --git a/examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/MXFP8Patterns.inc b/examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/MXFP8Patterns.inc deleted file mode 100644 index b886f9b11..000000000 --- a/examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/MXFP8Patterns.inc +++ /dev/null @@ -1,222 +0,0 @@ -class MXFP8ToBanks : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(MXFP8MemMatmulOp op, - PatternRewriter &b) const override { - auto aTy = cast(op.getInput().getType()); - auto wTy = cast(op.getWeight().getType()); - auto oTy = cast(op.getOutput().getType()); - if (!aTy.hasStaticShape() || !wTy.hasStaticShape() || - !oTy.hasStaticShape() || aTy.getRank() != 1 || wTy.getRank() != 1 || - oTy.getRank() != 2 || !aTy.getElementType().isInteger(8) || - !wTy.getElementType().isInteger(8) || !oTy.getElementType().isF32()) - return op.emitError( - "MXFP8 requires static packed activation/weight bytes " - "and an FP32 output matrix"); - if (aTy.getStridesAndOffset().first[0] != 1 || - wTy.getStridesAndOffset().first[0] != 1) - return op.emitError("MXFP8 packed tensors must have unit byte stride"); - int64_t M = oTy.getShape()[0], K = op.getReductionK(), - N = oTy.getShape()[1]; - int64_t rows = M == 1 ? 1 : op.getTileM(), columns = op.getTileN(); - int64_t chunk = op.getTileK(), activationChunk = op.getActivationTileK(); - bool windowInput = activationChunk != chunk; - if (rows <= 0 || (rows != 1 && rows % 16) || columns <= 0 || columns % 16 || - chunk <= 0 || chunk % 32 || chunk >= 4096) - return op.emitError( - "MXFP8 tile_k must be a positive multiple of 32 below 4096"); - int64_t chunks = (K + chunk - 1) / chunk; - int64_t stride = op.getBankBytes(); - if (windowInput && (activationChunk != K || K > 65535 || - (rows * K * 33 / 32 + 15) / 16 * 16 > stride)) - return op.emitError("MXFP8 activation window must hold the complete row"); - int64_t activationChunks = windowInput ? 1 : chunks; - if (M <= 0 || K <= 0 || N <= 0 || K % 32 || - aTy.getShape()[0] != - ((M + rows - 1) / rows) * activationChunks * stride || - wTy.getShape()[0] != ((N + columns - 1) / columns) * chunks * stride) - return op.emitError("MXFP8 matrix or packed weight shape mismatch"); - const auto &target = buckyball_target::getBuckyballTarget(); - if (target.bankWidthBits != 128 || target.bankDepth * 16 != stride || - (rows != 1 && (rows <= 0 || rows % 16)) || columns <= 0 || - columns % 16 || (rows * chunk * 33 / 32 + 15) / 16 * 16 > stride || - columns * chunk * 33 / 32 > stride || rows * columns * 4 > stride) - return op.emitError( - "MXFP8 packed K block exceeds the 128-bit bank capacity"); - Location loc = op.getLoc(); - Value zero = b.create(loc, 0), - one = b.create(loc, 1); - Value packed = op.getInput(); - auto dynamic = ShapedType::kDynamic; - auto bankView = [&](Value source, Value offset, int64_t size) -> Value { - Value slice = b.create( - loc, source, SmallVector{offset}, - SmallVector{b.getIndexAttr(stride)}, - SmallVector{b.getIndexAttr(1)}); - auto ty = MemRefType::get( - {stride / 16, 16}, b.getI8Type(), - StridedLayoutAttr::get(b.getContext(), dynamic, {16, 1})); - return b.create( - loc, ty, slice, ArrayRef{{0, 1}}); - }; - Value aBank = allocBank(b, loc, 1, 1), wBank = allocBank(b, loc, 1, 1), - cBank = allocBank(b, loc, 1, 1); - bool residentInput = activationChunks + 2 <= target.bankNum; - SmallVector banks{aBank}; - if (residentInput) - for (int64_t part = 1; part < activationChunks; ++part) - banks.push_back(allocBank(b, loc, 1, 1)); - banks.push_back(wBank); - banks.push_back(cBank); - auto outPackTy = MemRefType::get({stride / 16, 4}, b.getF32Type()); - Value outPack = b.create(loc, outPackTy); - outPack.getDefiningOp()->setAttr("alignment", b.getI64IntegerAttr(stride)); - Value limit = b.create(loc, N), - step = b.create(loc, columns); - Value rowLimit = b.create(loc, M), - rowStep = b.create(loc, rows); - auto rowBlocks = b.create(loc, zero, rowLimit, rowStep, banks); - b.setInsertionPointToStart(rowBlocks.getBody()); - Value m = rowBlocks.getInductionVar(); - SmallVector inputBanks(rowBlocks.getRegionIterArgs().drop_back(2)); - aBank = inputBanks.front(); - wBank = rowBlocks.getRegionIterArgs()[banks.size() - 2]; - cBank = rowBlocks.getRegionIterArgs()[banks.size() - 1]; - Value rowPanel = b.create(loc, m, rowStep); - if (residentInput) { - for (int64_t part = 0; part < activationChunks; ++part) { - Value offset = b.create( - loc, - b.create( - loc, - b.create( - loc, rowPanel, - b.create(loc, activationChunks)), - b.create(loc, part)), - b.create(loc, stride)); - int64_t count = windowInput ? K : std::min(chunk, K - part * chunk); - int64_t aBytes = (rows * count * 33 / 32 + 15) / 16 * 16; - inputBanks[part] = mvinBank(b, loc, bankView(packed, offset, stride), - inputBanks[part], aBytes / 16); - } - aBank = inputBanks.front(); - } - auto panels = b.create(loc, zero, limit, step, - ValueRange{aBank, wBank, cBank}); - b.setInsertionPointToStart(panels.getBody()); - Value n = panels.getInductionVar(); - aBank = panels.getRegionIterArgs()[0]; - wBank = panels.getRegionIterArgs()[1]; - cBank = panels.getRegionIterArgs()[2]; - Value state = cBank; - for (int64_t part = 0; part < chunks; ++part) { - int64_t count = std::min(chunk, K - part * chunk); - int64_t aBytes = (rows * count * 33 / 32 + 15) / 16 * 16, - bBytes = columns * count * 33 / 32; - Value aOffset = b.create( - loc, - b.create( - loc, - b.create( - loc, rowPanel, b.create(loc, chunks)), - b.create(loc, part)), - b.create(loc, stride)); - Value panel = b.create(loc, n, step); - Value wOffset = b.create( - loc, - b.create( - loc, - b.create( - loc, panel, b.create(loc, chunks)), - b.create(loc, part)), - b.create(loc, stride)); - if (!residentInput) - aBank = mvinBank(b, loc, bankView(packed, aOffset, aBytes), aBank, - aBytes / 16); - wBank = mvinBank(b, loc, bankView(op.getWeight(), wOffset, bBytes), wBank, - bBytes / 16); - if (windowInput) - state = b.create( - loc, b.getI64Type(), aBank, wBank, state, - createI64Const(b, loc, rows), createI64Const(b, loc, columns), - createI64Const(b, loc, count), createI64Const(b, loc, K), - createI64Const(b, loc, part * chunk), - createI1Const(b, loc, part == 0), - createI1Const(b, loc, part == chunks - 1), - createI64Const(b, loc, 0)); - else - state = b.create( - loc, b.getI64Type(), residentInput ? inputBanks[part] : aBank, - wBank, state, - createI64Const(b, loc, matrixRs2(rows, columns, count)), - createI1Const(b, loc, part == 0), - createI1Const(b, loc, part == chunks - 1), - createI64Const(b, loc, 0)); - } - cBank = mvoutBank(b, loc, outPack, state, rows * columns / 4); - Value end = b.create( - loc, step, b.create(loc, limit, n)); - SmallVector outputStrides; - int64_t outputOffset; - if (failed(oTy.getStridesAndOffset(outputStrides, outputOffset))) - return op.emitError("MXFP8 matmul requires a strided output"); - auto rr = b.create( - loc, zero, - b.create(loc, rowStep, - b.create(loc, rowLimit, m)), - one); - b.setInsertionPointToStart(rr.getBody()); - Value r = rr.getInductionVar(); - if (outputStrides.back() == 1) { - auto flatType = MemRefType::get({stride / 4}, b.getF32Type()); - Value flat = b.create( - loc, flatType, outPack, ArrayRef{{0, 1}}); - SmallVector offsets{ - b.create( - loc, r, b.create(loc, columns)) - .getResult()}; - SmallVector sizes{end}, steps{b.getIndexAttr(1)}; - auto sourceType = memref::SubViewOp::inferRankReducedResultType( - {ShapedType::kDynamic}, flatType, offsets, sizes, steps); - Value source = b.create(loc, sourceType, flat, offsets, - sizes, steps); - SmallVector outOffsets{ - b.create(loc, m, r).getResult(), n}; - SmallVector outSizes{b.getIndexAttr(1), end}; - SmallVector outSteps(2, b.getIndexAttr(1)); - auto rowType = memref::SubViewOp::inferRankReducedResultType( - {ShapedType::kDynamic}, oTy, outOffsets, outSizes, outSteps); - Value destination = b.create( - loc, rowType, op.getOutput(), outOffsets, outSizes, outSteps); - b.create(loc, source, destination); - } else { - auto cc = b.create(loc, zero, end, one); - b.setInsertionPointToStart(cc.getBody()); - Value c = cc.getInductionVar(), - four = b.create(loc, 4); - Value pr = b.create( - loc, - b.create( - loc, r, b.create(loc, columns / 4)), - b.create(loc, c, four)); - Value pc = b.create(loc, c, four); - Value value = b.create(loc, outPack, ValueRange{pr, pc}); - b.create(loc, value, op.getOutput(), - ValueRange{b.create(loc, r, m), - b.create(loc, n, c)}); - } - b.setInsertionPointAfter(rr); - b.create(loc, ValueRange{aBank, wBank, cBank}); - b.setInsertionPointAfter(panels); - inputBanks[0] = panels.getResult(0); - inputBanks.push_back(panels.getResult(1)); - inputBanks.push_back(panels.getResult(2)); - b.create(loc, inputBanks); - b.setInsertionPointAfter(rowBlocks); - for (Value bank : rowBlocks.getResults()) - releaseBank(b, loc, bank); - b.create(loc, outPack); - b.eraseOp(op); - return success(); - } -}; diff --git a/examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/SharedPanelPatterns.inc b/examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/SharedPanelPatterns.inc deleted file mode 100644 index 4233049f5..000000000 --- a/examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/SharedPanelPatterns.inc +++ /dev/null @@ -1,63 +0,0 @@ -class MXFP8SharedPanelToBanks : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(MXFP8SharedPanelOp op, - PatternRewriter &b) const override { - auto a = cast(op.getInput().getType()); - auto w = cast(op.getWeight().getType()); - int64_t rows = op.getRows(), columns = op.getColumns(); - int64_t k = op.getReductionK(), chunk = op.getTileK(); - int64_t bytes = op.getBankBytes(); - int64_t sharedBytes = op.getSharedBytesAttr().getInt(); - const auto &target = buckyball_target::getBuckyballTarget(); - if ((rows != 1 && (rows <= 0 || rows % 16)) || columns <= 0 || - columns % 16 || rows > 4095 || columns > 4095 || k <= 0 || k % 32 || - chunk <= 0 || chunk % 32 || chunk >= 4096 || - target.bankWidthBits != 128 || target.bankDepth * 16 != bytes || - sharedBytes <= 0 || sharedBytes % 16 || - rows * columns * 4 > sharedBytes || - (rows * chunk * 33 / 32 + 15) / 16 * 16 > bytes || - columns * chunk * 33 / 32 > bytes) - return op.emitError( - "shared MXFP8 panel exceeds the original bank layout"); - int64_t chunks = (k + chunk - 1) / chunk; - for (MemRefType type : {a, w}) - if (!type.hasStaticShape() || type.getRank() != 1 || - !type.getElementType().isInteger(8) || - type.getShape()[0] != chunks * bytes || - type.getStridesAndOffset().first[0] != 1) - return op.emitError( - "shared MXFP8 panel requires one packed row/N panel"); - Location loc = op.getLoc(); - Value ab = allocBank(b, loc, 1, 1), wb = allocBank(b, loc, 1, 1); - Value output = op.getBank(); - for (int64_t part = 0; part < chunks; ++part) { - int64_t count = std::min(chunk, k - part * chunk); - auto view = [&](Value packed) -> Value { - Value slice = b.create( - loc, packed, - SmallVector{b.getIndexAttr(part * bytes)}, - SmallVector{b.getIndexAttr(bytes)}, - SmallVector{b.getIndexAttr(1)}); - auto type = *memref::ExpandShapeOp::computeExpandedType( - cast(slice.getType()), {bytes / 16, 16}, - ArrayRef{{0, 1}}); - return b.create( - loc, type, slice, ArrayRef{{0, 1}}); - }; - ab = mvinBank(b, loc, view(op.getInput()), ab, - (rows * count * 33 / 32 + 15) / 16); - wb = mvinBank(b, loc, view(op.getWeight()), wb, - columns * count * 33 / 32 / 16); - output = b.create( - loc, b.getI64Type(), ab, wb, output, - createI64Const(b, loc, matrixRs2(rows, columns, count)), - createI1Const(b, loc, part == 0), - createI1Const(b, loc, part == chunks - 1), createI64Const(b, loc, 0)); - } - releaseBank(b, loc, ab); - releaseBank(b, loc, wb); - b.replaceOp(op, output); - return success(); - } -}; diff --git a/examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/SharedWindowPatterns.inc b/examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/SharedWindowPatterns.inc deleted file mode 100644 index 792866dbb..000000000 --- a/examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/SharedWindowPatterns.inc +++ /dev/null @@ -1,62 +0,0 @@ -class MXFP8SharedWindowToBanks : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(MXFP8SharedWindowOp op, - PatternRewriter &b) const override { - auto a = cast(op.getInput().getType()); - auto w = cast(op.getWeight().getType()); - int64_t rows = op.getRows(), columns = op.getColumns(); - int64_t k = op.getReductionK(), chunk = op.getTileK(); - int64_t bytes = op.getBankBytes(); - int64_t sharedBytes = op.getSharedBytesAttr().getInt(); - const auto &target = buckyball_target::getBuckyballTarget(); - if ((rows != 1 && (rows <= 0 || rows % 16)) || rows > 4095 || - columns <= 0 || columns > 4095 || columns % 16 || k <= 0 || k > 65535 || - k % 32 || chunk <= 0 || chunk >= 4096 || chunk % 32 || - target.bankWidthBits != 128 || target.bankDepth * 16 != bytes || - sharedBytes <= 0 || sharedBytes % 16 || - rows * columns * 4 > sharedBytes || - (rows * k * 33 / 32 + 15) / 16 * 16 > bytes || - columns * chunk * 33 / 32 > bytes) - return op.emitError( - "shared MXFP8 window exceeds its activation or output bank"); - int64_t chunks = (k + chunk - 1) / chunk; - if (!a.hasStaticShape() || !w.hasStaticShape() || - a.getShape()[0] != bytes || w.getShape()[0] != chunks * bytes || - a.getStridesAndOffset().first[0] != 1 || - w.getStridesAndOffset().first[0] != 1) - return op.emitError( - "shared MXFP8 window requires a complete activation and one N panel"); - Location loc = op.getLoc(); - auto view = [&](Value source, int64_t offset) -> Value { - Value slice = b.create( - loc, source, SmallVector{b.getIndexAttr(offset)}, - SmallVector{b.getIndexAttr(bytes)}, - SmallVector{b.getIndexAttr(1)}); - auto type = *memref::ExpandShapeOp::computeExpandedType( - cast(slice.getType()), {bytes / 16, 16}, - ArrayRef{{0, 1}}); - return b.create( - loc, type, slice, ArrayRef{{0, 1}}); - }; - Value ab = allocBank(b, loc, 1, 1), wb = allocBank(b, loc, 1, 1); - ab = mvinBank(b, loc, view(op.getInput(), 0), ab, - (rows * k * 33 / 32 + 15) / 16); - Value output = op.getBank(); - for (int64_t part = 0; part < chunks; ++part) { - int64_t count = std::min(chunk, k - part * chunk); - wb = mvinBank(b, loc, view(op.getWeight(), part * bytes), wb, - columns * count * 33 / 32 / 16); - output = b.create( - loc, b.getI64Type(), ab, wb, output, createI64Const(b, loc, rows), - createI64Const(b, loc, columns), createI64Const(b, loc, count), - createI64Const(b, loc, k), createI64Const(b, loc, part * chunk), - createI1Const(b, loc, part == 0), - createI1Const(b, loc, part == chunks - 1), createI64Const(b, loc, 0)); - } - releaseBank(b, loc, ab); - releaseBank(b, loc, wb); - b.replaceOp(op, output); - return success(); - } -}; diff --git a/examples/balls/mxmm/compiler/src/Dialect/Buckyball/IR/FP32.cpp b/examples/balls/mxmm/compiler/src/Dialect/Buckyball/IR/FP32.cpp deleted file mode 100644 index fc62bde67..000000000 --- a/examples/balls/mxmm/compiler/src/Dialect/Buckyball/IR/FP32.cpp +++ /dev/null @@ -1,46 +0,0 @@ -#include "Buckyball/BuckyballOps.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -using namespace mlir; -using namespace mlir::bufferization; -using namespace ::buddy::buckyball; - -bool FP32MatmulOp::bufferizesToAllocation(Value) { return true; } -bool FP32MatmulOp::bufferizesToMemoryRead(OpOperand &, const AnalysisState &) { - return true; -} -bool FP32MatmulOp::bufferizesToMemoryWrite(OpOperand &, const AnalysisState &) { - return false; -} -AliasingValueList FP32MatmulOp::getAliasingValues(OpOperand &, - const AnalysisState &) { - return {}; -} -LogicalResult FP32MatmulOp::bufferize(RewriterBase &b, - const BufferizationOptions &options, - BufferizationState &state) { - auto a = getBuffer(b, getLhs(), options, state); - auto w = getBuffer(b, getRhs(), options, state); - if (failed(a) || failed(w)) - return failure(); - auto type = cast(getOutput().getType()); - if (!type.hasStaticShape()) - return emitError("requires static FP32 output shape"); - Value out = b.create( - getLoc(), MemRefType::get(type.getShape(), type.getElementType())); - Value rhs = *w; - if (getRhsTransposedAttr()) { - auto source = cast(rhs.getType()); - int rank = source.getRank(); - if (rank != 2 && rank != 3) - return emitError("transposed FP32 RHS requires rank 2 or 3"); - SmallVector permutation; - for (int index = 0; index < rank; ++index) - permutation.push_back(index); - std::swap(permutation[rank - 2], permutation[rank - 1]); - auto map = AffineMap::getPermutationMap(permutation, getContext()); - rhs = b.create(getLoc(), rhs, AffineMapAttr::get(map)); - } - b.create(getLoc(), *a, rhs, out, getFusedAttr()); - replaceOpWithBufferizedValues(b, getOperation(), out); - return success(); -} diff --git a/examples/balls/mxmm/compiler/src/Dialect/Buckyball/IR/MXFP8.cpp b/examples/balls/mxmm/compiler/src/Dialect/Buckyball/IR/MXFP8.cpp deleted file mode 100644 index 77fb3d642..000000000 --- a/examples/balls/mxmm/compiler/src/Dialect/Buckyball/IR/MXFP8.cpp +++ /dev/null @@ -1,157 +0,0 @@ -#include "Buckyball/BuckyballOps.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "llvm/Support/MathExtras.h" -using namespace mlir; -using namespace mlir::bufferization; -using namespace ::buddy::buckyball; - -namespace { -FailureOr packedSize(int64_t m, int64_t k, int64_t rows, int64_t chunk, - int64_t stride) { - if (m <= 0 || k <= 0 || k % 32 || rows <= 0 || (rows != 1 && rows % 16) || - chunk <= 0 || chunk % 32 || stride <= 0 || stride > 65536 || - !llvm::isPowerOf2_64(stride) || rows > stride || - chunk > stride * 32 / (rows * 33)) - return failure(); - int64_t panels, bytes; - if (llvm::MulOverflow((m - 1) / rows + 1, (k - 1) / chunk + 1, panels) || - llvm::MulOverflow(panels, stride, bytes)) - return failure(); - return bytes; -} -} // namespace - -LogicalResult MXFP8QuantOp::verify() { - auto input = cast(getInput().getType()); - auto output = cast(getOutput().getType()); - if (!input.hasStaticShape() || input.getRank() != 2 || - !input.getElementType().isF32() || !output.hasStaticShape() || - output.getRank() != 1 || !output.getElementType().isInteger(8)) - return emitOpError("requires a static FP32 matrix and packed byte tensor"); - auto bytes = packedSize(input.getShape()[0], input.getShape()[1], - getTileRows(), getTileK(), getBankBytes()); - if (failed(bytes) || output.getShape()[0] != *bytes) - return emitOpError("activation panel layout or packed byte shape mismatch"); - return success(); -} - -bool MXFP8QuantOp::bufferizesToAllocation(Value) { return true; } -bool MXFP8QuantOp::bufferizesToMemoryRead(OpOperand &, const AnalysisState &) { - return true; -} -bool MXFP8QuantOp::bufferizesToMemoryWrite(OpOperand &, const AnalysisState &) { - return false; -} -AliasingValueList MXFP8QuantOp::getAliasingValues(OpOperand &, - const AnalysisState &) { - return {}; -} -LogicalResult MXFP8QuantOp::bufferize(RewriterBase &b, - const BufferizationOptions &options, - BufferizationState &state) { - auto input = getBuffer(b, getInput(), options, state); - if (failed(input)) - return failure(); - auto type = cast(getOutput().getType()); - auto output = b.create( - getLoc(), MemRefType::get(type.getShape(), type.getElementType())); - output->setAttr("alignment", b.getI64IntegerAttr(getBankBytes())); - auto dynamic = ShapedType::kDynamic; - auto inputType = MemRefType::get( - {dynamic, dynamic}, b.getF32Type(), - StridedLayoutAttr::get(getContext(), dynamic, {dynamic, dynamic})); - auto bytesType = MemRefType::get({dynamic}, b.getI8Type()); - auto module = getOperation()->getParentOfType(); - if (!module.lookupSymbol("mxfp8_quant")) { - OpBuilder::InsertionGuard guard(b); - b.setInsertionPointToStart(module.getBody()); - auto function = b.create( - getLoc(), "mxfp8_quant", - b.getFunctionType({inputType, bytesType, b.getI64Type(), b.getI64Type(), - b.getI64Type()}, - {})); - function.setPrivate(); - function->setAttr("llvm.emit_c_interface", b.getUnitAttr()); - } - SmallVector arguments{ - b.create(getLoc(), inputType, *input), - b.create(getLoc(), bytesType, output)}; - for (int64_t value : {getTileRows(), getTileK(), getBankBytes()}) - arguments.push_back(b.create(getLoc(), value, 64)); - b.create(getLoc(), "mxfp8_quant", TypeRange{}, arguments); - replaceOpWithBufferizedValues(b, getOperation(), - ValueRange{output.getResult()}); - return success(); -} - -LogicalResult MXFP8MatmulOp::verify() { - auto input = cast(getInput().getType()); - auto weight = cast(getWeight().getType()); - auto output = cast(getOutput().getType()); - if (!input.hasStaticShape() || input.getRank() != 1 || - !input.getElementType().isInteger(8) || !weight.hasStaticShape() || - weight.getRank() != 1 || !weight.getElementType().isInteger(8) || - !output.hasStaticShape() || output.getRank() != 2 || - !output.getElementType().isF32()) - return emitOpError( - "requires packed activation/weight bytes and a static FP32 matrix"); - int64_t m = output.getShape()[0], n = output.getShape()[1], - k = getReductionK(); - if (getTileK() >= 4096) - return emitOpError("matrix panel K must be less than 4096"); - int64_t rows = m == 1 ? 1 : getTileM(), columns = getTileN(); - int64_t activationK = getActivationTileK(); - if (activationK != getTileK() && - (activationK != k || k > 65535 || rows * k * 33 / 32 > getBankBytes())) - return emitOpError("activation tile K must match the weight panel or fit a " - "complete row window"); - auto activationBytes = packedSize(m, k, rows, activationK, getBankBytes()); - auto weightBytes = packedSize(n, k, columns, getTileK(), getBankBytes()); - if (columns <= 0 || columns % 16 || failed(activationBytes) || - failed(weightBytes) || input.getShape()[0] != *activationBytes || - weight.getShape()[0] != *weightBytes) - return emitOpError("logical K, panel layout or packed byte shape mismatch"); - if (auto quant = getInput().getDefiningOp()) { - auto source = cast(quant.getInput().getType()); - if (source.getRank() != 2 || source.getShape()[0] != m || - source.getShape()[1] != k) - return emitOpError("activation quantization matrix shape mismatch"); - if (quant.getTileRows() != rows || quant.getTileK() != activationK || - quant.getBankBytes() != getBankBytes()) - return emitOpError("activation quantization panel layout mismatch"); - } - return success(); -} - -bool MXFP8MatmulOp::bufferizesToAllocation(Value) { return true; } -bool MXFP8MatmulOp::bufferizesToMemoryRead(OpOperand &, const AnalysisState &) { - return true; -} -bool MXFP8MatmulOp::bufferizesToMemoryWrite(OpOperand &, - const AnalysisState &) { - return false; -} -AliasingValueList MXFP8MatmulOp::getAliasingValues(OpOperand &, - const AnalysisState &) { - return {}; -} -LogicalResult MXFP8MatmulOp::bufferize(RewriterBase &b, - const BufferizationOptions &options, - BufferizationState &state) { - auto input = getBuffer(b, getInput(), options, state); - auto weight = getBuffer(b, getWeight(), options, state); - if (failed(input) || failed(weight)) - return failure(); - auto type = cast(getOutput().getType()); - auto output = b.create( - getLoc(), MemRefType::get(type.getShape(), type.getElementType())); - b.create(getLoc(), *input, *weight, output, - getReductionKAttr(), getTileKAttr(), - getTileMAttr(), getTileNAttr(), getBankBytesAttr(), - getActivationTileKAttr()); - replaceOpWithBufferizedValues(b, getOperation(), - ValueRange{output.getResult()}); - return success(); -} diff --git a/examples/balls/mxmm/compiler/src/Dialect/Buckyball/MxmmBall.td b/examples/balls/mxmm/compiler/src/Dialect/Buckyball/MxmmBall.td deleted file mode 100644 index a7576cbfa..000000000 --- a/examples/balls/mxmm/compiler/src/Dialect/Buckyball/MxmmBall.td +++ /dev/null @@ -1,145 +0,0 @@ -#ifndef MATMULBALL_OPS -#define MATMULBALL_OPS - -include "Buckyball.td" -include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.td" - -def MXFP8QuantOp : Buckyball_Op<"mxfp8_quant", [NoMemoryEffect, - DeclareOpInterfaceMethods]> { - let summary = "Integer RNE activation quantization into aligned MXFP8 panels"; - let description = [{ - The input is an immutable finite FP32 matrix. Each panel contains all - row-major E4M3 codes followed by row-major block32 E8M0 scales. The final - K panel uses its actual remaining width. Unused bank-window padding is - undefined and must not be consumed as codes or scales. Quantization does - not read frm or modify fflags. This operation is not speculatable. - }]; - let arguments = (ins AnyRankedTensor:$input, I64Attr:$tile_rows, - I64Attr:$tile_k, I64Attr:$bank_bytes); - let results = (outs AnyRankedTensor:$output); - let hasVerifier = 1; - let assemblyFormat = "$input prop-dict attr-dict `:` type($input) `->` type($output)"; -} - -def MXFP8MatmulOp : Buckyball_Op<"mxfp8_matmul", [ - DeclareOpInterfaceMethods]> { - let arguments = (ins AnyRankedTensor:$input, AnyRankedTensor:$weight, I64Attr:$reduction_k, I64Attr:$tile_k, I64Attr:$tile_m, I64Attr:$tile_n, I64Attr:$bank_bytes, I64Attr:$activation_tile_k); - let results = (outs AnyRankedTensor:$output); - let hasVerifier = 1; - let assemblyFormat = "$input `,` $weight prop-dict attr-dict `:` type($input) `,` type($weight) `->` type($output)"; -} -def MXFP8MemMatmulOp : Buckyball_Op<"mxfp8_mem_matmul"> { - let arguments = (ins Arg]>:$input, - Arg]>:$weight, - Arg]>:$output, I64Attr:$reduction_k, I64Attr:$tile_k, I64Attr:$tile_m, I64Attr:$tile_n, I64Attr:$bank_bytes, I64Attr:$activation_tile_k); - let assemblyFormat = "$input `,` $weight `,` $output prop-dict attr-dict `:` type($input) `,` type($weight) `,` type($output)"; -} -def MXFP8SharedPanelOp : Buckyball_Op<"mxfp8_shared_panel", [MemoryEffects<[MemRead, MemWrite]>]> { - let summary = "Compute one original MXFP8 output panel into a live shared bank"; - let arguments = (ins MemRefRankOf<[I8], [1]>:$input, - MemRefRankOf<[I8], [1]>:$weight, I64:$bank, - I64Attr:$rows, I64Attr:$columns, I64Attr:$reduction_k, - I64Attr:$tile_k, I64Attr:$bank_bytes, I64Attr:$shared_bytes); - let results = (outs I64:$bank_out); - let assemblyFormat = "$input `,` $weight `,` $bank prop-dict attr-dict `:` " - "type($input) `,` type($weight) `,` type($bank)"; -} -def MXFP8SharedWindowOp : Buckyball_Op<"mxfp8_shared_window", [MemoryEffects<[MemRead, MemWrite]>]> { - let summary = "Compute a shared output panel from one complete MXFP8 activation bank"; - let arguments = (ins MemRefRankOf<[I8], [1]>:$input, - MemRefRankOf<[I8], [1]>:$weight, I64:$bank, - I64Attr:$rows, I64Attr:$columns, I64Attr:$reduction_k, - I64Attr:$tile_k, I64Attr:$bank_bytes, I64Attr:$shared_bytes); - let results = (outs I64:$bank_out); - let assemblyFormat = "$input `,` $weight `,` $bank prop-dict attr-dict `:` " - "type($input) `,` type($weight) `,` type($bank)"; -} -def MXFP8Op : Buckyball_Op<"mxfp8"> { - let summary = "MxmmBall compute"; - let description = [{ - ISA: MXMM_MXFP8. `config` contains rows, cols, and k. - }]; - let arguments = (ins I64:$op1BankId, I64:$op2BankId, - I64:$resultBankId, I64:$config, - I1:$first, I1:$last, I64:$outputBase); - let assemblyFormat = "$op1BankId `,` $op2BankId `,` $resultBankId `,` " - "$config `,` $first `,` $last `,` $outputBase " - "prop-dict attr-dict `:` " - "type($op1BankId)"; -} - -def BankMXFP8Op : Buckyball_Op<"bank_mxfp8"> { - let summary = "MxmmBall compute with virtual bank handles"; - let arguments = (ins I64:$op1Bank, I64:$op2Bank, I64:$wrBank, I64:$config, - I1:$first, I1:$last, I64:$outputBase); - let results = (outs I64:$wrBankOut); - let assemblyFormat = - "$op1Bank $op2Bank $wrBank $config $first $last $outputBase " - "prop-dict attr-dict `:` " - "type($op1Bank) type($op2Bank) type($wrBank) type($config)"; -} - -def MXFP8WindowOp : Buckyball_Op<"mxfp8_window"> { - let summary = "MXFP8 multiplication from a complete activation row window"; - let arguments = (ins I64:$op1BankId, I64:$op2BankId, I64:$resultBankId, - I64:$rows, I64:$cols, I64:$count, I64:$fullK, I64:$startK, - I1:$first, I1:$last, I64:$outputBase); - let assemblyFormat = "$op1BankId `,` $op2BankId `,` $resultBankId `,` " - "$rows `,` $cols `,` $count `,` $fullK `,` $startK `,` " - "$first `,` $last `,` $outputBase prop-dict attr-dict `:` type($op1BankId)"; -} - -def BankMXFP8WindowOp : Buckyball_Op<"bank_mxfp8_window", [MemoryEffects<[MemRead, MemWrite]>]> { - let summary = "MXFP8 activation row window with virtual bank handles"; - let arguments = (ins I64:$op1Bank, I64:$op2Bank, I64:$wrBank, - I64:$rows, I64:$cols, I64:$count, I64:$fullK, I64:$startK, - I1:$first, I1:$last, I64:$outputBase); - let results = (outs I64:$wrBankOut); - let assemblyFormat = "$op1Bank $op2Bank $wrBank $rows $cols $count " - "$fullK $startK $first $last $outputBase prop-dict attr-dict `:` type($op1Bank)"; -} - -def FP32MatmulOp : Buckyball_Op<"fp32_matmul", [ - DeclareOpInterfaceMethods]> { - let arguments = (ins AnyRankedTensor:$lhs, AnyRankedTensor:$rhs, - OptionalAttr:$rhs_transposed, BoolAttr:$fused); - let results = (outs AnyRankedTensor:$output); - let assemblyFormat = "$lhs `,` $rhs prop-dict attr-dict `:` type($lhs) `,` type($rhs) `->` type($output)"; -} -def FP32MemMatmulOp : Buckyball_Op<"fp32_mem_matmul"> { - let arguments = (ins Arg]>:$lhs, - Arg]>:$rhs, - Arg]>:$output, BoolAttr:$fused); - let assemblyFormat = "$lhs `,` $rhs `,` $output prop-dict attr-dict `:` type($lhs) `,` type($rhs) `,` type($output)"; -} -def FP32Op : Buckyball_Op<"matmul_f32"> { - let summary = "MxmmBall compute"; - let description = [{ - ISA: MXMM_FMA32 / MXMM_F32. `config` contains rows, cols, and k. - }]; - let arguments = (ins I64:$op1BankId, I64:$op2BankId, - I64:$resultBankId, I64:$config, - I1:$first, I1:$last, I64:$outputBase, BoolAttr:$fused); - let assemblyFormat = "$op1BankId `,` $op2BankId `,` $resultBankId `,` " - "$config `,` $first `,` $last `,` $outputBase " - "prop-dict attr-dict `:` " - "type($op1BankId)"; -} - -def BankFP32Op : Buckyball_Op<"bank_matmul_f32"> { - let summary = "MxmmBall compute with virtual bank handles"; - let arguments = (ins I64:$op1Bank, I64:$op2Bank, I64:$wrBank, I64:$config, - I1:$first, I1:$last, I64:$outputBase, BoolAttr:$fused); - let results = (outs I64:$wrBankOut); - let assemblyFormat = - "$op1Bank $op2Bank $wrBank $config $first $last $outputBase " - "prop-dict attr-dict `:` " - "type($op1Bank) type($op2Bank) type($wrBank) type($config)"; -} - - - -#endif diff --git a/examples/balls/mxmm/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/balls/mxmm/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index d9d7968ae..000000000 --- a/examples/balls/mxmm/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,157 +0,0 @@ -#include -//===- LegalizeForLLVMExport.cpp - MxmmBall LLVM lowering --------------===// - -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" -#include "Utils/BankUtils.h" - -#include "llvm/Support/ErrorHandling.h" - -using namespace mlir; -using namespace ::buddy::buckyball; -using namespace ::buddy::buckyball::legalize; - -namespace { -uint64_t matrixCfg(uint64_t rows, uint64_t cols, bool first, bool last) { - return fieldBits(rows, 0, 11) | fieldBits(cols, 12, 23) | - (uint64_t(first) << 24) | (uint64_t(last) << 25); -} -struct MXFP8Lowering : public ConvertOpToLLVMPattern { - MXFP8Lowering(LLVMTypeConverter &converter, bool, int64_t bankDepth) - : ConvertOpToLLVMPattern(converter), bankDepth(bankDepth) {} - - LogicalResult - matchAndRewrite(MXFP8Op op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("MxmmBall"); - Location loc = op.getLoc(); - auto config = op.getConfig().getDefiningOp(); - if (!config) - return op.emitError("mxmm config must be a constant"); - auto configAttr = dyn_cast(config.getValue()); - if (!configAttr) - return op.emitError("mxmm config must be an integer constant"); - uint64_t cfg = configAttr.getValue().getZExtValue(); - uint64_t rows = cfg & 0xfff; - uint64_t cols = (cfg >> 12) & 0xfff; - uint64_t k = (cfg >> 24) & 0xfff; - if (cfg >> 36 || (rows != 1 && (rows == 0 || rows % 16)) || cols == 0 || - cols % 16 || k == 0 || k % 32 || - (rows * k * 33 / 32 + 15) / 16 > static_cast(bankDepth) || - cols * k * 33 / 32 / 16 > static_cast(bankDepth) || - rows * cols / 4 > static_cast(bankDepth)) - return op.emitError("Mxmm matrix shape or C footprint is invalid"); - auto base = adaptor.getOutputBase().getDefiningOp(); - auto baseAttr = - base ? dyn_cast(base.getValue()) : IntegerAttr(); - if (!baseAttr || baseAttr.getInt() < 0 || baseAttr.getInt() > 63 || - baseAttr.getInt() + rows * cols / 4 > bankDepth) - return op.emitError("Mxmm outputBase must be constant and fit C in bank"); - Value rs1 = packRs1BanksIter( - rewriter, loc, adaptor.getOp1BankId(), adaptor.getOp2BankId(), - adaptor.getResultBankId(), cstI64(rewriter, loc, k)); - Value rs2 = cstI64(rewriter, loc, matrixCfg(rows, cols, false, false)); - Value first = rewriter.create(loc, rewriter.getI64Type(), - adaptor.getFirst()); - Value last = rewriter.create(loc, rewriter.getI64Type(), - adaptor.getLast()); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, first, cstI64(rewriter, loc, 24))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, last, cstI64(rewriter, loc, 25))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getOutputBase(), - cstI64(rewriter, loc, 26))); - rewriter.replaceOpWithNewOp( - op, rs1, rs2, - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("MXMM_MXFP8"))); - return success(); - } - -private: - int64_t bankDepth; -}; - -struct FP32Lowering : public ConvertOpToLLVMPattern { - FP32Lowering(LLVMTypeConverter &converter, bool, int64_t depth) - : ConvertOpToLLVMPattern(converter), depth(depth) {} - - LogicalResult matchAndRewrite(FP32Op op, OpAdaptor adaptor, - ConversionPatternRewriter &b) const override { - buckyball_target::requireBuckyballBall("MxmmBall"); - auto constant = op.getConfig().getDefiningOp(); - auto shape = - constant ? dyn_cast(constant.getValue()) : IntegerAttr(); - if (!shape) - return op.emitError("FP32 matmul requires a constant shape"); - uint64_t cfg = shape.getValue().getZExtValue(); - uint64_t rows = cfg & 0xfff, cols = (cfg >> 12) & 0xfff, - k = (cfg >> 24) & 0xfff; - if (cfg >> 36 || (rows != 1 && (rows == 0 || rows % 16)) || cols == 0 || - cols % 16 || !k || k % 4 || rows * k / 4 > uint64_t(depth) || - cols * k / 4 > uint64_t(depth) || - buckyball_target::getBuckyballTarget().bankWidthBits != 128) - return op.emitError( - "FP32 matmul shape does not fit the configured banks"); - auto baseOp = op.getOutputBase().getDefiningOp(); - auto base = - baseOp ? dyn_cast(baseOp.getValue()) : IntegerAttr(); - if (!base || base.getInt() < 0 || base.getInt() > 63 || - base.getInt() + rows * cols / 4 > uint64_t(depth)) - return op.emitError("FP32 output base does not fit the configured bank"); - Location loc = op.getLoc(); - Value rs1 = - packRs1BanksIter(b, loc, adaptor.getOp1BankId(), adaptor.getOp2BankId(), - adaptor.getResultBankId(), cstI64(b, loc, k)); - Value rs2 = cstI64(b, loc, matrixCfg(rows, cols, false, false)); - Value first = - b.create(loc, b.getI64Type(), adaptor.getFirst()); - Value last = - b.create(loc, b.getI64Type(), adaptor.getLast()); - rs2 = b.create( - loc, rs2, b.create(loc, first, cstI64(b, loc, 24))); - rs2 = b.create( - loc, rs2, b.create(loc, last, cstI64(b, loc, 25))); - rs2 = b.create( - loc, rs2, - b.create(loc, adaptor.getOutputBase(), - cstI64(b, loc, 26))); - b.replaceOpWithNewOp( - op, rs1, rs2, - b.getI32IntegerAttr(buckyball_target::getBuckyballFunct7( - op.getFused() ? "MXMM_FMA32" : "MXMM_F32"))); - return success(); - } - int64_t depth; -}; - -#include "WindowPatterns.inc" - -} // namespace -namespace mlir::buddy::buckyball { -void populateMxmmBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &converter, - RewritePatternSet &patterns, - bool stable, int64_t depth) { - patterns.add(converter, - stable, depth); -} -void configureMxmmBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool) { - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/examples/balls/mxmm/compiler/src/Dialect/Buckyball/Transforms/WindowPatterns.inc b/examples/balls/mxmm/compiler/src/Dialect/Buckyball/Transforms/WindowPatterns.inc deleted file mode 100644 index 5fbd8d375..000000000 --- a/examples/balls/mxmm/compiler/src/Dialect/Buckyball/Transforms/WindowPatterns.inc +++ /dev/null @@ -1,76 +0,0 @@ -struct MXFP8WindowLowering : public ConvertOpToLLVMPattern { - MXFP8WindowLowering(LLVMTypeConverter &converter, bool, int64_t bankDepth) - : ConvertOpToLLVMPattern(converter), depth(bankDepth) {} - - LogicalResult matchAndRewrite(MXFP8WindowOp op, OpAdaptor a, - ConversionPatternRewriter &b) const override { - buckyball_target::requireBuckyballBall("MxmmBall"); - if (buckyball_target::getBuckyballTarget().bankWidthBits != 128) - return op.emitError("mxmm window: requires 16-byte bank rows"); - auto integer = [](Value value) -> std::optional { - if (auto constant = value.getDefiningOp()) - return cast(constant.getValue()).getInt(); - return std::nullopt; - }; - auto rows = integer(op.getRows()), cols = integer(op.getCols()); - auto count = integer(op.getCount()), full = integer(op.getFullK()); - auto start = integer(op.getStartK()), base = integer(op.getOutputBase()); - if (rows && (*rows > 4095 || (*rows != 1 && (*rows <= 0 || *rows % 16)))) - return op.emitError("mxmm window: rows must be 1 or a positive multiple " - "of 16 within 12 bits"); - if (cols && (*cols <= 0 || *cols > 4095 || *cols % 16)) - return op.emitError( - "mxmm window: cols must be a positive multiple of 16 within 12 bits"); - if (count && - (*count <= 0 || uint64_t(*count) >= (uint64_t(1) << 34) || *count % 32)) - return op.emitError("mxmm window: count must be a positive multiple of " - "32 within 34 bits"); - if (full && (*full <= 0 || *full > 65535 || *full % 32)) - return op.emitError("mxmm window: fullK must be a positive multiple of " - "32 within 16 bits"); - if (start && (*start < 0 || *start > 65535 || *start % 32)) - return op.emitError("mxmm window: startK must be a nonnegative multiple " - "of 32 within 16 bits"); - if (base && (*base < 0 || *base > 63)) - return op.emitError("mxmm window: outputBase must fit 6 bits"); - if (full && start && count && - uint64_t(*start) + uint64_t(*count) > uint64_t(*full)) - return op.emitError( - "mxmm window: reduction exceeds complete activation row"); - if (rows && full && - uint64_t(*rows) * uint64_t(*full) * 33 / 32 > uint64_t(depth) * 16) - return op.emitError( - "mxmm window: complete activation exceeds bank capacity"); - if (cols && count && - uint64_t(*cols) * uint64_t(*count) * 33 / 32 > uint64_t(depth) * 16) - return op.emitError("mxmm window: weight exceeds bank capacity"); - if (rows && cols && base && - uint64_t(*base) * 16 + uint64_t(*rows) * uint64_t(*cols) * 4 > - uint64_t(depth) * 16) - return op.emitError("mxmm window: output exceeds bank capacity"); - - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(b, loc, a.getOp1BankId(), a.getOp2BankId(), - a.getResultBankId(), a.getCount()); - Value rs2 = b.create(loc, a.getRows(), cstI64(b, loc, 4095)); - SmallVector fields{a.getCols(), a.getFirst(), a.getLast(), - a.getOutputBase(), a.getFullK(), a.getStartK()}; - const int64_t shifts[] = {12, 24, 25, 26, 32, 48}; - const int64_t masks[] = {4095, 1, 1, 63, 65535, 65535}; - for (size_t index = 0; index < fields.size(); ++index) { - Value value = fields[index]; - if (value.getType().isInteger(1)) - value = b.create(loc, b.getI64Type(), value); - value = b.create(loc, value, cstI64(b, loc, masks[index])); - rs2 = b.create( - loc, rs2, - b.create(loc, value, cstI64(b, loc, shifts[index]))); - } - b.replaceOpWithNewOp( - op, rs1, rs2, - b.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("MXMM_MXFP8_WINDOW"))); - return success(); - } - int64_t depth; -}; diff --git a/examples/balls/mxmm/compiler/src/Transforms/NormQuantWindow.cpp b/examples/balls/mxmm/compiler/src/Transforms/NormQuantWindow.cpp deleted file mode 100644 index a4eade8e7..000000000 --- a/examples/balls/mxmm/compiler/src/Transforms/NormQuantWindow.cpp +++ /dev/null @@ -1,344 +0,0 @@ -#include "Buckyball/BuckyballDialect.h" -#include "Buckyball/BuckyballOps.h" -#include "Target/BuckyballTargetRegistry.h" -#include "Utils/BankUtils.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/Interfaces/SideEffectInterfaces.h" -#include "mlir/Pass/Pass.h" -#include "mlir/Transforms/GreedyPatternRewriteDriver.h" -#include "llvm/ADT/SmallPtrSet.h" -#include - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { -Value origin(Value value) { - while (Operation *op = value.getDefiningOp()) { - if (!isa( - op)) - break; - value = op->getOperand(0); - } - return value; -} - -bool privateUses(Value value, ArrayRef allowed, - SmallVectorImpl &views) { - llvm::SmallPtrSet seen; - std::function visit = [&](Value current) { - for (Operation *user : current.getUsers()) { - if (llvm::is_contained(allowed, user)) - continue; - if (!isa( - user)) - return false; - if (seen.insert(user).second) { - if (!visit(user->getResult(0))) - return false; - views.push_back(user); - } - } - return true; - }; - return visit(value); -} - -SmallVector matrix(PatternRewriter &b, MXFP8MemMatmulOp op, Value r, - Value w, Value mean, Value epsilon) { - Location loc = op.getLoc(); - int64_t width = op.getReductionK(), chunk = op.getTileK(); - int64_t columns = op.getTileN(), bytes = op.getBankBytes(); - auto outputType = cast(op.getOutput().getType()); - int64_t outputWidth = outputType.getDimSize(1); - int64_t chunks = (width + chunk - 1) / chunk; - Value zero = b.create(loc, 0); - Value one = b.create(loc, 1); - Value four = b.create(loc, 4); - Value limit = b.create(loc, outputWidth); - Value step = b.create(loc, columns); - auto loop = b.create(loc, zero, limit, step, ValueRange{r, w}); - b.setInsertionPointToStart(loop.getBody()); - r = loop.getRegionIterArgs()[0]; - w = loop.getRegionIterArgs()[1]; - Value n = loop.getInductionVar(); - Value panel = b.create(loc, n, step); - for (int64_t part = 0; part < chunks; ++part) { - int64_t count = std::min(chunk, width - part * chunk); - Value offset = b.create( - loc, - b.create( - loc, - b.create( - loc, panel, b.create(loc, chunks)), - b.create(loc, part)), - b.create(loc, bytes)); - auto slice = b.create( - loc, op.getWeight(), SmallVector{offset}, - SmallVector{b.getIndexAttr(bytes)}, - SmallVector{b.getIndexAttr(1)}); - Value view = b.create( - loc, ArrayRef{bytes / 16, 16}, slice, - ArrayRef{{0, 1}}); - r = b.create( - loc, b.getI64Type(), view, r, - createI64Const(b, loc, columns * count * 33 / 32 / 16), - createI64Const(b, loc, 1), b.getI64IntegerAttr(0)); - auto window = b.create( - loc, b.getI64Type(), b.getI64Type(), r, w, - FlatSymbolRefAttr::get(b.getContext(), "rvv_norm_window"), - ValueRange{createI64Const(b, loc, 1), createI64Const(b, loc, width), - createI64Const(b, loc, columns), - createI64Const(b, loc, count), createI64Const(b, loc, width), - createI64Const(b, loc, part * chunk), - createI64Const(b, loc, part == 0), - createI64Const(b, loc, part == chunks - 1), - createI64Const(b, loc, bytes), mean, epsilon}); - r = window.getReadOut(); - w = window.getWriteOut(); - } - Value end = b.create(loc, step, - b.create(loc, limit, n)); - Value rows = b.create(loc, end, four); - SmallVector offsets{b.getIndexAttr(0), n}; - SmallVector sizes{b.getIndexAttr(1), end}; - SmallVector strides(2, b.getIndexAttr(1)); - auto sliceType = memref::SubViewOp::inferRankReducedResultType( - {ShapedType::kDynamic}, outputType, offsets, sizes, strides); - Value destination = b.create( - loc, sliceType, op.getOutput(), offsets, sizes, strides); - Value dmaView = b.create( - loc, ArrayRef{ShapedType::kDynamic, 4}, destination, - ArrayRef{{0, 1}}, - SmallVector{rows, b.getIndexAttr(4)}); - w = b.create( - loc, b.getI64Type(), dmaView, w, - b.create(loc, b.getI64Type(), rows), - createI64Const(b, loc, 1), b.getI64IntegerAttr(1)); - b.create(loc, ValueRange{r, w}); - b.setInsertionPointAfter(loop); - b.eraseOp(op); - return {loop.getResult(0), loop.getResult(1)}; -} - -class FuseNormQuantWindow : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(func::CallOp quant, - PatternRewriter &b) const override { - if (quant.getCallee() != "mxfp8_quant" || quant.getNumOperands() != 5) - return failure(); - Value normBuffer = origin(quant.getOperand(0)); - Value packedBuffer = origin(quant.getOperand(1)); - auto normType = dyn_cast(normBuffer.getType()); - auto packedType = dyn_cast(packedBuffer.getType()); - if (!normBuffer.getDefiningOp() || - !packedBuffer.getDefiningOp() || !normType || - !packedType || !normType.hasStaticShape() || normType.getRank() < 1 || - !packedType.hasStaticShape() || !normType.getElementType().isF32() || - !packedType.getElementType().isInteger(8)) - return failure(); - func::CallOp norm; - for (auto call : quant->getBlock()->getOps()) - if (call.getCallee() == "rvv_norm" && call.getNumOperands() == 5 && - origin(call.getOperand(0)) == normBuffer && - call->isBeforeInBlock(quant)) { - if (norm) - return failure(); - norm = call; - } - if (!norm) - return failure(); - SmallVector matrices; - for (auto op : quant->getBlock()->getOps()) - if (origin(op.getInput()) == packedBuffer) - matrices.push_back(op); - if (matrices.empty() || matrices.size() > 2) - return failure(); - if (matrices.size() == 2 && !matrices[0]->isBeforeInBlock(matrices[1])) - std::swap(matrices[0], matrices[1]); - const auto &target = buckyball_target::getBuckyballTarget(); - if (!target.rvvEnabled || target.bankWidthBits != 128 || - target.bankNum < 5 || - !llvm::is_contained(target.balls, "MxquantBall") || - !llvm::any_of( - target.isa, - [](const auto &entry) { - return entry.mnemonic == "MXMM_MXFP8_WINDOW"; - })) - return failure(); - auto module = quant->getParentOfType(); - auto normFunction = module.lookupSymbol("rvv_norm"); - auto quantFunction = module.lookupSymbol("mxfp8_quant"); - if (!normFunction || !normFunction.isExternal() || !quantFunction || - !quantFunction.isExternal()) - return failure(); - int64_t width = matrices[0].getReductionK(), chunk = matrices[0].getTileK(); - int64_t activationChunk = matrices[0].getActivationTileK(); - int64_t bytes = target.bankDepth * 16; - if (chunk <= 0 || chunk % 32 || chunk >= 4096 || width <= 0 || width % 32 || - (activationChunk != chunk && activationChunk != width)) - return failure(); - int64_t chunks = (width + chunk - 1) / chunk; - if (width <= 0 || width % 32 || width * 4 > bytes || - width + width / 32 > bytes || normType.getNumElements() != width || - normType.getDimSize(normType.getRank() - 1) != width || - packedType.getRank() != 1 || - packedType.getNumElements() != - ((width + activationChunk - 1) / activationChunk) * bytes) - return failure(); - const int64_t quantParameters[] = {1, activationChunk, bytes}; - for (size_t index = 0; index < 3; ++index) { - auto constant = - quant.getOperand(index + 2).getDefiningOp(); - if (!constant || cast(constant.getValue()).getInt() != - quantParameters[index]) - return failure(); - } - for (Operation *cursor = norm.getOperation(); - cursor != matrices.back().getOperation()->getNextNode(); - cursor = cursor->getNextNode()) { - if (cursor == norm || cursor == quant || cursor == matrices[0] || - cursor == matrices.back()) - continue; - if (isa(cursor)) - continue; - if (isa(cursor) || cursor->getNumRegions() || - !isMemoryEffectFree(cursor)) - return failure(); - } - for (auto op : matrices) { - auto output = cast(op.getOutput().getType()); - auto weight = cast(op.getWeight().getType()); - int64_t columns = op.getTileN(); - if (!output.hasStaticShape() || output.getRank() != 2 || - output.getDimSize(0) != 1 || output.getDimSize(1) % 4 || - output.getStridesAndOffset().first.back() != 1 || - !op.getOutput().getDefiningOp() || columns <= 0 || - columns % 16 || op.getReductionK() != width || - op.getTileK() != chunk || - op.getActivationTileK() != matrices[0].getActivationTileK() || - op.getBankBytes() != bytes || columns * chunk * 33 / 32 > bytes || - columns * 4 > bytes || weight.getRank() != 1 || - weight.getStridesAndOffset().first[0] != 1 || - weight.getDimSize(0) != - ((output.getDimSize(1) + columns - 1) / columns) * chunks * - bytes || - !quant->isBeforeInBlock(op)) - return failure(); - } - SmallVector views; - SmallVector packedUsers{quant.getOperation()}; - for (auto matrix : matrices) - packedUsers.push_back(matrix.getOperation()); - if (!privateUses(normBuffer, {norm.getOperation(), quant.getOperation()}, - views) || - !privateUses(packedBuffer, packedUsers, views)) - return failure(); - Value input = norm.getOperand(1), gamma = norm.getOperand(2); - for (Value *value : {&input, &gamma}) - while (auto cast = value->getDefiningOp()) - *value = cast.getSource(); - auto inputType = dyn_cast(input.getType()); - if (!inputType || inputType.getRank() < 1 || !inputType.hasStaticShape() || - inputType.getDimSize(inputType.getRank() - 1) != width || - origin(input) == normBuffer || origin(gamma) == normBuffer || - origin(input) == packedBuffer || origin(gamma) == packedBuffer) - return failure(); - for (Value value : {input, gamma}) { - auto type = dyn_cast(value.getType()); - if (!type || !type.hasStaticShape() || type.getNumElements() != width || - !type.getElementType().isF32()) - return failure(); - auto strides = type.getStridesAndOffset().first; - int64_t stride = 1; - for (int64_t axis = type.getRank() - 1; axis >= 0; --axis) { - if (type.getDimSize(axis) != 1 && strides[axis] != stride) - return failure(); - stride *= type.getDimSize(axis); - } - } - Location loc = norm.getLoc(); - b.setInsertionPoint(norm); - Value d = allocBank(b, loc, 1, 1), n = allocBank(b, loc, 1, 1), - x = allocBank(b, loc, 1, 1), g = allocBank(b, loc, 1, 1), - a = allocBank(b, loc, 1, 1); - auto rowView = [&](Value value) { - auto type = cast(value.getType()); - ReassociationIndices axes; - for (int64_t axis = 0; axis < type.getRank(); ++axis) - axes.push_back(axis); - if (type.getRank() != 1) - value = b.create( - loc, value, ArrayRef{axes}); - return Value(b.create( - loc, ArrayRef{width / 4, 4}, value, - ArrayRef{{0, 1}})); - }; - x = mvinBank(b, loc, rowView(input), x, width / 4); - g = mvinBank(b, loc, rowView(gamma), g, width / 4); - Value r = b.create(loc, b.getI64Type(), g, x); - Value w = b.create(loc, b.getI64Type(), n, d); - w = b.create(loc, b.getI64Type(), a, w); - auto initialized = b.create( - loc, b.getI64Type(), b.getI64Type(), r, w, - FlatSymbolRefAttr::get(b.getContext(), "rvv_norm_window"), - ValueRange{createI64Const(b, loc, 0), createI64Const(b, loc, width), - createI64Const(b, loc, 0), createI64Const(b, loc, 0), - createI64Const(b, loc, width), createI64Const(b, loc, 0), - createI64Const(b, loc, 0), createI64Const(b, loc, 0), - createI64Const(b, loc, bytes), norm.getOperand(3), - norm.getOperand(4)}); - Value mean = norm.getOperand(3), epsilon = norm.getOperand(4); - SmallVector state{initialized.getReadOut(), - initialized.getWriteOut()}; - b.eraseOp(norm); - b.eraseOp(quant); - for (auto op : matrices) { - b.setInsertionPoint(op); - state = matrix(b, op, state[0], state[1], mean, epsilon); - } - auto finished = b.create( - loc, b.getI64Type(), b.getI64Type(), state[0], state[1], - FlatSymbolRefAttr::get(b.getContext(), "rvv_norm_window"), - ValueRange{createI64Const(b, loc, 2), createI64Const(b, loc, width), - createI64Const(b, loc, 0), createI64Const(b, loc, 0), - createI64Const(b, loc, width), createI64Const(b, loc, 0), - createI64Const(b, loc, 0), createI64Const(b, loc, 0), - createI64Const(b, loc, bytes), mean, epsilon}); - releaseBank(b, loc, finished.getReadOut()); - releaseBank(b, loc, finished.getWriteOut()); - for (Operation *view : views) - b.eraseOp(view); - b.eraseOp(normBuffer.getDefiningOp()); - b.eraseOp(packedBuffer.getDefiningOp()); - return success(); - } -}; - -class NormQuantWindowPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(NormQuantWindowPass) - StringRef getArgument() const final { return "fuse-norm-quant-window"; } - StringRef getDescription() const final { - return "Keep full-row Norm and quantized activation in banks across paired " - "matrix consumers."; - } - void getDependentDialects(DialectRegistry ®istry) const override { - registry.insert(); - } - void runOnOperation() override { - RewritePatternSet patterns(&getContext()); - patterns.add(&getContext()); - if (failed(applyPatternsGreedily(getOperation(), std::move(patterns)))) - signalPassFailure(); - } -}; -} // namespace -namespace mlir::buddy { -void registerNormQuantWindowPass() { PassRegistration(); } -} // namespace mlir::buddy diff --git a/examples/balls/mxmm/compiler/src/Transforms/Passes.cpp b/examples/balls/mxmm/compiler/src/Transforms/Passes.cpp deleted file mode 100644 index 33e983145..000000000 --- a/examples/balls/mxmm/compiler/src/Transforms/Passes.cpp +++ /dev/null @@ -1,4 +0,0 @@ -namespace mlir::buddy { -void registerNormQuantWindowPass(); -void registerMxmmBallPasses() { registerNormQuantWindowPass(); } -} // namespace mlir::buddy diff --git a/examples/balls/mxmm/configs/MxmmBallParam.scala b/examples/balls/mxmm/configs/MxmmBallParam.scala deleted file mode 100644 index c28e1fdce..000000000 --- a/examples/balls/mxmm/configs/MxmmBallParam.scala +++ /dev/null @@ -1,19 +0,0 @@ -package examples.balls.mxmm.configs - -import framework.balldomain.configs.BallParamLoader -import framework.top.GlobalConfig - -case class MxmmBallParam(tileRows: Int, tileCols: Int) { - require(tileRows == 16 && tileCols == 16) - val arithmeticLatency = 3 - val contexts = arithmeticLatency * 2 + 1 -} - -object MxmmBallParam { - - def apply(b: GlobalConfig): MxmmBallParam = { - val table = BallParamLoader.ballTable(b, "MxmmBall") - MxmmBallParam(BallParamLoader.int(table, "tileRows"), BallParamLoader.int(table, "tileCols")) - } - -} diff --git a/examples/balls/mxmm/configs/default.toml b/examples/balls/mxmm/configs/default.toml deleted file mode 100644 index d71354112..000000000 --- a/examples/balls/mxmm/configs/default.toml +++ /dev/null @@ -1,3 +0,0 @@ -[ball] -tileRows = 16 -tileCols = 16 diff --git a/examples/balls/mxmm/emu/src/accumulator.rs b/examples/balls/mxmm/emu/src/accumulator.rs deleted file mode 100644 index cc496f008..000000000 --- a/examples/balls/mxmm/emu/src/accumulator.rs +++ /dev/null @@ -1,22 +0,0 @@ -use std::{cell::RefCell, collections::HashMap}; - -#[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) enum Format { - Mxfp8, - Mxfp8Window, - Fma32, - F32, -} - -pub(crate) struct Chain { - pub(crate) format: Format, - pub(crate) rows: usize, - pub(crate) cols: usize, - pub(crate) bank: u64, - pub(crate) base: usize, - pub(crate) values: Vec, -} - -thread_local! { - pub(crate) static CHAINS: RefCell> = RefCell::new(HashMap::new()); -} diff --git a/examples/balls/mxmm/emu/src/dispatch.rs b/examples/balls/mxmm/emu/src/dispatch.rs deleted file mode 100644 index 361cc955d..000000000 --- a/examples/balls/mxmm/emu/src/dispatch.rs +++ /dev/null @@ -1,33 +0,0 @@ -use super::{fp32, mxfp8}; -use crate::inst::instruction::ExecContext; - -const BALL_CLASS: &str = "examples.balls.mxmm.MxmmBall"; - -pub fn execute_known( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, - ctx: &mut ExecContext, -) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("MXMM_MXFP8") => Some(mxfp8::execute(xs1, xs2, false, ctx)), - Some("MXMM_MXFP8_WINDOW") => Some(mxfp8::execute(xs1, xs2, true, ctx)), - Some("MXMM_FMA32") => Some(fp32::execute(xs1, xs2, true, ctx)), - Some("MXMM_F32") => Some(fp32::execute(xs1, xs2, false, ctx)), - _ => None, - } -} - -pub fn cycles_after_issue(ball_class: &str, funct: u32, _xs1: u64, _xs2: u64) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("MXMM_MXFP8" | "MXMM_MXFP8_WINDOW" | "MXMM_FMA32" | "MXMM_F32") => Some(1), - _ => None, - } -} diff --git a/examples/balls/mxmm/emu/src/fp32.rs b/examples/balls/mxmm/emu/src/fp32.rs deleted file mode 100644 index bd2588c86..000000000 --- a/examples/balls/mxmm/emu/src/fp32.rs +++ /dev/null @@ -1,121 +0,0 @@ -use super::super::bank::{bank_lines, bank_row_bytes}; -use super::accumulator::{Chain, Format, CHAINS}; -use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; -use super::instruction::ExecContext; - -pub(crate) fn execute(xs1: u64, xs2: u64, fused: bool, ctx: &mut ExecContext) -> u64 { - let format = if fused { Format::Fma32 } else { Format::F32 }; - let rows = (xs2 & 0xfff) as usize; - let cols = ((xs2 >> 12) & 0xfff) as usize; - let k = rs1_iter(xs1) as usize; - let first = (xs2 >> 24) & 1 != 0; - let last = (xs2 >> 25) & 1 != 0; - let base = ((xs2 >> 26) & 63) as usize; - let a = rs1_b0(xs1); - let b = rs1_b1(xs1); - let c = rs1_b2(xs1); - assert!( - xs2 >> 32 == 0 && (rows == 1 || (rows > 0 && rows % 16 == 0)) && cols > 0 && cols % 16 == 0 && k > 0 && k % 4 == 0, - "f32 matmul: requires M=1/multiple of 16, N multiple of 16, positive K multiple of four and zero reserved bits" - ); - assert!(a != b && b != c && a != c, "f32 matmul: banks must differ"); - for bank in [a, b, c] { - assert!( - ctx.config(bank).allocated && ctx.config(bank).cols == 1, - "f32 matmul: banks must be allocated with one column" - ); - } - let stride = bank_row_bytes(); - assert!( - stride == 16 - && rows * k * 4 <= bank_lines() * stride - && cols * k * 4 <= bank_lines() * stride - && base * stride + rows * cols * 4 <= bank_lines() * stride, - "f32 matmul: bank footprint exceeds depth" - ); - let pa = pbank(ctx, a); - let pb = pbank(ctx, b); - let av = &ctx.banks[pa][..rows * k * 4]; - let bv = &ctx.banks[pb][..cols * k * 4]; - assert!( - av.chunks_exact(4) - .chain(bv.chunks_exact(4)) - .all(|x| f32::from_le_bytes(x.try_into().unwrap()).is_finite()), - "f32 matmul: non-finite operand" - ); - let mut chain = CHAINS.with(|chains| { - let mut chains = chains.borrow_mut(); - if first { - assert!( - !chains.contains_key(&ctx.hart_id), - "f32 matmul: first while chain is live" - ); - Chain { - format, - rows, - cols, - bank: c, - base, - values: vec![0f32; rows * cols], - } - } else { - chains - .remove(&ctx.hart_id) - .expect("f32 matmul: continuation without first") - } - }); - assert!( - chain.format == format - && chain.rows == rows - && chain.cols == cols - && chain.bank == c - && chain.base == base, - "f32 matmul: continuation changed destination" - ); - let weights: Vec<&[u8]> = (0..cols) - .map(|col| &bv[col * k * 4..(col + 1) * k * 4]) - .collect(); - for row in 0..rows { - if av[row * k * 4..(row + 1) * k * 4] - .chunks_exact(4) - .all(|x| u32::from_le_bytes(x.try_into().unwrap()) & 0x7fffffff == 0) - && chain.values[row * cols..(row + 1) * cols] - .iter() - .all(|x| x.to_bits() == 0) - { - continue; - } - let av = &av[row * k * 4..(row + 1) * k * 4]; - let dst = &mut chain.values[row * cols..(row + 1) * cols]; - let mut sums = dst.to_vec(); - for inner in 0..k { - let a = f32::from_le_bytes(av[inner * 4..inner * 4 + 4].try_into().unwrap()); - for col in 0..cols { - let b = - f32::from_le_bytes(weights[col][inner * 4..inner * 4 + 4].try_into().unwrap()); - sums[col] = if fused { - a.mul_add(b, sums[col]) - } else { - (a * b) + sums[col] - }; - } - } - assert!( - sums.iter().all(|x| x.is_finite()), - "f32 matmul: accumulator overflow" - ); - dst.copy_from_slice(&sums); - } - if last { - let pc = pbank(ctx, c); - for (index, value) in chain.values.iter().enumerate() { - let offset = base * stride + index * 4; - ctx.banks[pc][offset..offset + 4].copy_from_slice(&value.to_le_bytes()); - } - } else { - CHAINS.with(|chains| { - chains.borrow_mut().insert(ctx.hart_id, chain); - }); - } - 0 -} diff --git a/examples/balls/mxmm/emu/src/lib.rs b/examples/balls/mxmm/emu/src/lib.rs deleted file mode 100644 index e9d96b625..000000000 --- a/examples/balls/mxmm/emu/src/lib.rs +++ /dev/null @@ -1,8 +0,0 @@ -pub(crate) use crate::inst::{decode, instruction}; - -mod accumulator; -mod fp32; -mod mxfp8; -mod dispatch; - -pub use dispatch::{cycles_after_issue, execute_known}; diff --git a/examples/balls/mxmm/emu/src/mxfp8.rs b/examples/balls/mxmm/emu/src/mxfp8.rs deleted file mode 100644 index c5e5c75fa..000000000 --- a/examples/balls/mxmm/emu/src/mxfp8.rs +++ /dev/null @@ -1,176 +0,0 @@ -use super::super::bank::{bank_lines, bank_row_bytes}; -use super::accumulator::{Chain, Format, CHAINS}; -use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; -use super::instruction::ExecContext; - -fn value(code: u8, factor: f32) -> f32 { - let exponent = (code >> 3) & 15; - let mantissa = (code & 7) as u32; - let normal = ((exponent as u32 + 120) << 23) | (mantissa << 20); - let subnormal = (mantissa as f32 / 512.0).to_bits(); - let element = f32::from_bits(if exponent == 0 { subnormal } else { normal }); - let result = f32::from_bits((element * factor).to_bits() | ((code as u32 & 128) << 24)); - result -} - -pub(crate) fn execute(xs1: u64, xs2: u64, window: bool, ctx: &mut ExecContext) -> u64 { - let rows = (xs2 & 0xfff) as usize; - let cols = ((xs2 >> 12) & 0xfff) as usize; - let k = rs1_iter(xs1) as usize; - let first = (xs2 >> 24) & 1 != 0; - let last = (xs2 >> 25) & 1 != 0; - let base = ((xs2 >> 26) & 63) as usize; - let a = rs1_b0(xs1); - let b = rs1_b1(xs1); - let c = rs1_b2(xs1); - let full_k = if window { - ((xs2 >> 32) & 0xffff) as usize - } else { - k - }; - let start_k = if window { (xs2 >> 48) as usize } else { 0 }; - let format = if window { - Format::Mxfp8Window - } else { - Format::Mxfp8 - }; - assert!( - (window || xs2 >> 32 == 0) && (rows == 1 || (rows > 0 && rows % 16 == 0)) && cols > 0 && cols % 16 == 0 && k > 0 && k % 32 == 0, - "mxfp8: requires M=1/multiple of 16, N multiple of 16, K multiple of 32 and zero reserved bits" - ); - assert!( - full_k > 0 && full_k % 32 == 0 && start_k % 32 == 0 && start_k + k <= full_k, - "mxfp8: invalid activation window" - ); - assert!(a != b && b != c && a != c, "mxfp8: banks must differ"); - for bank in [a, b, c] { - assert!( - ctx.config(bank).allocated && ctx.config(bank).cols == 1, - "mxfp8: banks must be allocated with one column" - ); - } - let stride = bank_row_bytes(); - assert!(stride == 16, "mxfp8: requires 16-byte bank rows"); - assert!( - rows * full_k + rows * full_k / 32 <= bank_lines() * stride - && cols * k + cols * k / 32 <= bank_lines() * stride - && base * stride + rows * cols * 4 <= bank_lines() * stride, - "mxfp8: bank footprint exceeds depth" - ); - let pa = pbank(ctx, a); - let pb = pbank(ctx, b); - let a_bytes = &ctx.banks[pa][..rows * full_k + rows * full_k / 32]; - let b_bytes = &ctx.banks[pb][..cols * k + cols * k / 32]; - assert!( - b_bytes[..cols * k] - .iter() - .fold(true, |valid, code| valid & (code & 0x7f != 0x7f)), - "mxfp8: NaN encoding" - ); - let mut av = vec![0f32; rows * k]; - let mut bv = vec![0f32; cols * k]; - for (block, dst) in av.chunks_exact_mut(32).enumerate() { - let row = block / (k / 32); - let inner = block % (k / 32); - let code_offset = row * full_k + start_k + inner * 32; - let scale = a_bytes[rows * full_k + row * (full_k / 32) + start_k / 32 + inner]; - assert!(scale != 255, "mxfp8: NaN encoding"); - let factor = if scale == 0 { - f32::from_bits(1 << 22) - } else { - f32::from_bits((scale as u32) << 23) - }; - for (index, decoded) in dst.iter_mut().enumerate() { - let code = a_bytes[code_offset + index]; - assert!(code & 0x7f != 0x7f, "mxfp8: NaN encoding"); - *decoded = value(code, factor); - } - } - for block in 0..k / 32 { - for col in 0..cols { - let scale = b_bytes[cols * k + col * (k / 32) + block]; - assert!(scale != 255, "mxfp8: NaN encoding"); - let factor = if scale == 0 { - f32::from_bits(1 << 22) - } else { - f32::from_bits((scale as u32) << 23) - }; - let codes = &b_bytes[col * k + block * 32..col * k + (block + 1) * 32]; - let mut decoded = [0f32; 32]; - for (decoded, code) in decoded.iter_mut().zip(codes.iter()) { - *decoded = value(*code, factor); - } - for (index, decoded) in decoded.iter().enumerate() { - bv[(block * 32 + index) * cols + col] = *decoded; - } - } - } - assert!( - av.iter() - .chain(bv.iter()) - .fold(true, |valid, value| valid & value.is_finite()), - "mxfp8: decoded value overflows FP32" - ); - let mut chain = CHAINS.with(|chains| { - let mut chains = chains.borrow_mut(); - if first { - assert!( - !chains.contains_key(&ctx.hart_id), - "mxfp8: first while chain is live" - ); - Chain { - format, - rows, - cols, - bank: c, - base, - values: vec![0f32; rows * cols], - } - } else { - chains - .remove(&ctx.hart_id) - .expect("mxfp8: continuation without first") - } - }); - assert!( - chain.format == format - && chain.rows == rows - && chain.cols == cols - && chain.bank == c - && chain.base == base, - "mxfp8: continuation changed destination" - ); - for row in 0..rows { - if av[row * k..(row + 1) * k].iter().all(|value| *value == 0.0) - && chain.values[row * cols..(row + 1) * cols] - .iter() - .all(|value| value.to_bits() == 0) - { - continue; - } - let av = &av[row * k..(row + 1) * k]; - let dst = &mut chain.values[row * cols..(row + 1) * cols]; - let mut sums = dst.to_vec(); - for inner in 0..k { - let a = av[inner]; - let weights = &bv[inner * cols..(inner + 1) * cols]; - for col in 0..cols { - sums[col] = a.mul_add(weights[col], sums[col]); - } - } - assert!(sums.iter().all(|v| v.is_finite()), "accumulator overflow"); - dst.copy_from_slice(&sums); - } - if last { - let pc = pbank(ctx, c); - for (i, value) in chain.values.iter().enumerate() { - ctx.banks[pc][base * stride + i * 4..base * stride + i * 4 + 4] - .copy_from_slice(&value.to_le_bytes()); - } - } else { - CHAINS.with(|chains| { - chains.borrow_mut().insert(ctx.hart_id, chain); - }); - } - 0 -} diff --git a/examples/balls/mxmm/workloads/ctests/CMakeLists.txt b/examples/balls/mxmm/workloads/ctests/CMakeLists.txt deleted file mode 100644 index 7fdb334ba..000000000 --- a/examples/balls/mxmm/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -add_buckyball_ctests(mxmm_fp32_test.c mxmm_mxfp8_test.c mxmm_bank_capacity_test.c) - -file(READ "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/params.h" MXMM_PARAMS) -string(REGEX MATCH "#define BANK_LINES ([0-9]+)" _ "${MXMM_PARAMS}") -set(MXMM_BANK_LINES ${CMAKE_MATCH_1}) -string(REGEX MATCH "#define BANK_WIDTH ([0-9]+)" _ "${MXMM_PARAMS}") -math(EXPR MXMM_BANK_BYTES "${MXMM_BANK_LINES} * ${CMAKE_MATCH_1} / 8") -if(MXMM_BANK_BYTES LESS 15840) - message(FATAL_ERROR "Mxmm A-window row test requires 15840-byte banks; actual ${MXMM_BANK_BYTES} bytes") -endif() -add_buckyball_ctests(mxmm_window_test.c mxmm_window_rows_test.c mxmm_window_chain_test.c) diff --git a/examples/balls/mxmm/workloads/kernels/CMakeLists.txt b/examples/balls/mxmm/workloads/kernels/CMakeLists.txt deleted file mode 100644 index c8a67eea4..000000000 --- a/examples/balls/mxmm/workloads/kernels/CMakeLists.txt +++ /dev/null @@ -1,26 +0,0 @@ -get_filename_component(REPO_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../../../../.." ABSOLUTE) -set(RVV_SOURCE "${REPO_ROOT}/bb-tests/workloads/lib/bbsw/kernels/rvv") -set(IMAGES "${CMAKE_CURRENT_BINARY_DIR}/norm_window_image.h" - "${CMAKE_CURRENT_BINARY_DIR}/packmm_image.h") -add_custom_command(OUTPUT ${IMAGES} - COMMAND "${PYTHON}" "${CMAKE_CURRENT_SOURCE_DIR}/build_image.py" - --output "${CMAKE_CURRENT_BINARY_DIR}" --isa-dir "${RUNTIME_PARAMS}" - DEPENDS build_image.py norm_window.cpp window.h packmm.cpp packmm.h - "${RUNTIME_PARAMS}/ballISA.h" "${RVV_SOURCE}/build_images.py" - "${RVV_SOURCE}/norm.cpp" "${RVV_SOURCE}/pack.cpp" "${RVV_SOURCE}/kernels.h" "${RVV_SOURCE}/kernel.ld" - BYPRODUCTS norm_window.o norm.o norm_window.elf packmm.o pack.o packmm.elf - VERBATIM) -add_custom_target(mxmm-images DEPENDS ${IMAGES}) -add_library(mxmm_kernels STATIC dispatch.cpp pack_dispatch.cpp ${IMAGES}) -target_link_libraries(mxmm_kernels PRIVATE runtime) -target_compile_options(mxmm_kernels PRIVATE -O2) -target_compile_features(mxmm_kernels PRIVATE cxx_std_20) -target_include_directories(mxmm_kernels PRIVATE "${CMAKE_CURRENT_BINARY_DIR}" - "${RUNTIME_PARAMS}" "${REPO_ROOT}/bb-tests/workloads/lib") - -add_library(mxmm_kernels_ant STATIC EXCLUDE_FROM_ALL dispatch.cpp pack_dispatch.cpp ${IMAGES}) -target_link_libraries(mxmm_kernels_ant PRIVATE runtime_ant_tasks) -target_compile_options(mxmm_kernels_ant PRIVATE -O2) -target_compile_features(mxmm_kernels_ant PRIVATE cxx_std_20) -target_include_directories(mxmm_kernels_ant PRIVATE "${CMAKE_CURRENT_BINARY_DIR}" - "${RUNTIME_PARAMS}" "${REPO_ROOT}/bb-tests/workloads/lib") diff --git a/examples/balls/mxmm/workloads/kernels/build_image.py b/examples/balls/mxmm/workloads/kernels/build_image.py deleted file mode 100644 index 907420906..000000000 --- a/examples/balls/mxmm/workloads/kernels/build_image.py +++ /dev/null @@ -1,93 +0,0 @@ -import argparse -from pathlib import Path -import struct -import subprocess -import sys - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--isa-dir", type=Path, required=True) - args = parser.parse_args() - source = Path(__file__).resolve().parent - root = source.parents[4] - support = root / "bb-tests/workloads/lib/bbsw/kernels/rvv" - sys.path.insert(0, str(support)) - from build_images import array, read_segments - - tools = root / "result/bin" - args.output.mkdir(parents=True, exist_ok=True) - objects = [] - for name, path in ( - ("norm_window", source / "norm_window.cpp"), - ("norm", support / "norm.cpp"), - ("packmm", source / "packmm.cpp"), - ("pack", support / "pack.cpp"), - ): - obj = args.output / f"{name}.o" - subprocess.run( - [ - str(tools / "clang++"), - "--target=riscv64", - "-march=rv64imf_zve32f_zvl128b", - "-mabi=lp64f", - "-mcmodel=medany", - "-ffreestanding", - "-O2", - "-ffp-contract=off", - "-ffunction-sections", - "-fdata-sections", - f"-I{root / 'bb-tests/workloads/lib'}", - f"-I{args.isa_dir}", - "-c", - str(path), - "-o", - str(obj), - ], - check=True, - ) - objects.append(obj) - for name in ("norm_window", "packmm"): - elf = args.output / f"{name}.elf" - subprocess.run( - [ - str(tools / "riscv64-unknown-elf-ld"), - "-m", - "elf64lriscv", - "--gc-sections", - "-e", - name, - "-T", - str(support / "kernel.ld"), - *map(str, objects), - "-o", - str(elf), - ], - check=True, - ) - text, data, entry = read_segments(elf) - image = ( - struct.pack( - "<6I", - 0x31564B52, - len(text), - entry, - 0x40000000, - len(data), - 0, - ) - + text - + data - ) - (args.output / f"{name}_image.h").write_text( - "#pragma once\n#include \n" - + array(f"{name}_image", image) - + f"constexpr uint32_t {name}_entry = {entry};\n" - + f"constexpr uint32_t {name}_text = {len(text)};\n" - ) - print(f"{name}: image={len(image)} text={len(text)} data={len(data)}") - - -if __name__ == "__main__": - main() diff --git a/examples/balls/mxmm/workloads/kernels/dispatch.cpp b/examples/balls/mxmm/workloads/kernels/dispatch.cpp deleted file mode 100644 index b8d0f8406..000000000 --- a/examples/balls/mxmm/workloads/kernels/dispatch.cpp +++ /dev/null @@ -1,59 +0,0 @@ -#include "norm_window_image.h" -#include "window.h" -#include -#include -#include -#include -#include - -extern "C" void -_mlir_ciface_rvv_norm_window(uint64_t readBank, uint64_t writeBank, - uint64_t phase, uint64_t width, uint64_t columns, - uint64_t count, uint64_t full, uint64_t start, - uint64_t first, uint64_t last, uint64_t bankBytes, - uint32_t meanBits, uint32_t epsilonBits) { - constexpr unsigned programBank = VIRTUAL_BANK_NUM; - if (phase == 2) { - release_kernel(programBank); - return; - } - if (phase > 1 || !width || width > bankBytes / sizeof(float)) { - fputs("MXMM norm/window requires a complete row within its bank\n", stderr); - abort(); - } - if (phase == 0) - mvin_kernel(norm_window_image, sizeof(norm_window_image), programBank); - uint32_t rounding; - asm volatile("csrr %0, frm" : "=r"(rounding) : : "memory"); - struct alignas(16) Packet { - kernel_launch launch; - Window parameters; - uint32_t padding; - } packet{{norm_window_entry, - norm_window_text, - 0x40002000, - {(uint64_t(2) << 32) + sizeof(kernel_launch), - (uint64_t(2) << 32) + offsetof(kernel_launch, reserved)}, - 0}, - {uint32_t(phase), uint32_t(width), meanBits, epsilonBits, - uint32_t(columns), uint32_t(count), uint32_t(full), uint32_t(start), - uint32_t(first), uint32_t(last), rounding << 5}, - 0}; - bb_mvin_group((uintptr_t)&packet, writeBank, 0, sizeof(packet) / 16, 1); - // Finish descriptor DMA before this stack packet can be reused. - run_kernel(readBank, programBank, writeBank, 0); - if (phase == 1) - return; - alignas(16) kernel_launch completed; - bb_mvout_group((uintptr_t)&completed, writeBank, 0, sizeof(completed) / 16, - 1); - if (std::memcmp(&packet.launch, &completed, - offsetof(kernel_launch, reserved)) || - (completed.reserved & ~UINT64_C(31))) { - fputs( - "MXMM norm/window returned an invalid descriptor or exception flags\n", - stderr); - abort(); - } - asm volatile("csrs fflags, %0" : : "r"(completed.reserved) : "memory"); -} diff --git a/examples/balls/mxmm/workloads/kernels/norm_window.cpp b/examples/balls/mxmm/workloads/kernels/norm_window.cpp deleted file mode 100644 index f759124a3..000000000 --- a/examples/balls/mxmm/workloads/kernels/norm_window.cpp +++ /dev/null @@ -1,37 +0,0 @@ -#include -#include -#include -#include - -#include "window.h" - -extern "C" void norm_window(const Window *parameters, uint32_t *flags) { - asm volatile("csrw fcsr, %0" ::"r"(parameters->rounding) : "memory"); - if (parameters->phase == 0) { - rvv_norm((float *)(uint64_t(3) << 32), (const float *)0, - (const float *)(uint64_t(1) << 32), parameters->width, - parameters->mean, parameters->epsilon); - uint64_t banks = - UINT64_C(3) | (UINT64_C(4) << 20) | (uint64_t(parameters->width) << 30); - uint64_t zero = 0; - asm volatile(".insn r 0x7b, 3, %c2, x0, %0, %1" ::"r"(banks), "r"(zero), - "i"(BB_FUNC7_MXQUANT) - : "memory"); - } else if (parameters->phase == 1) { - uint64_t banks = - UINT64_C(4) | (UINT64_C(3) << 20) | (uint64_t(parameters->count) << 30); - uint64_t config = UINT64_C(1) | (uint64_t(parameters->columns) << 12) | - (uint64_t(parameters->first) << 24) | - (uint64_t(parameters->last) << 25) | - (uint64_t(parameters->full) << 32) | - (uint64_t(parameters->start) << 48); - asm volatile(".insn r 0x7b, 3, %c2, x0, %0, %1" ::"r"(banks), "r"(config), - "i"(BB_FUNC7_MXMM_MXFP8_WINDOW) - : "memory"); - } else { - asm volatile("ebreak"); - } - uint64_t result; - asm volatile("csrr %0, fflags" : "=r"(result)::"memory"); - *flags = uint32_t(result); -} diff --git a/examples/balls/mxmm/workloads/kernels/packmm.cpp b/examples/balls/mxmm/workloads/kernels/packmm.cpp deleted file mode 100644 index d04526669..000000000 --- a/examples/balls/mxmm/workloads/kernels/packmm.cpp +++ /dev/null @@ -1,32 +0,0 @@ -#include "packmm.h" -#include -#include - -extern "C" void rvv_pack(uint32_t *, const uint32_t *, size_t, size_t, size_t, - size_t, size_t, size_t); - -extern "C" void packmm(const PackMM *parameters, uint32_t *flags) { - asm volatile("csrw fcsr, %0" : : "r"(parameters->rounding) : "memory"); - rvv_pack((uint32_t *)(uint64_t(3) << 32), (const uint32_t *)0, - parameters->fullK, parameters->fullN, parameters->startK, - parameters->startN, parameters->countK, parameters->columns); - const uint64_t banks = UINT64_C(1) | (UINT64_C(3) << 10) | - (UINT64_C(4) << 20) | - (uint64_t(parameters->countK) << 30); - const uint64_t config = - uint64_t(parameters->rows) | (uint64_t(parameters->columns) << 12) | - (uint64_t(parameters->first) << 24) | (uint64_t(parameters->last) << 25); - if (parameters->fused) - asm volatile(".insn r 0x7b, 3, %c2, x0, %0, %1" - : - : "r"(banks), "r"(config), "i"(BB_FUNC7_MXMM_FMA32) - : "memory"); - else - asm volatile(".insn r 0x7b, 3, %c2, x0, %0, %1" - : - : "r"(banks), "r"(config), "i"(BB_FUNC7_MXMM_F32) - : "memory"); - uint64_t result; - asm volatile("csrr %0, fflags" : "=r"(result) : : "memory"); - *flags = uint32_t(result); -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/CMakeLists.txt b/examples/balls/mxmm/workloads/mlir_tests/CMakeLists.txt deleted file mode 100644 index 1987af32c..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -add_subdirectory(ball) -add_subdirectory(bank) -add_subdirectory(tensor) diff --git a/examples/balls/mxmm/workloads/mlir_tests/ball/CMakeLists.txt b/examples/balls/mxmm/workloads/mlir_tests/ball/CMakeLists.txt deleted file mode 100644 index 099ee8c64..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/ball/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) - set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) -endif() -set(BUCKYBALL_MLIR_TEST_PREFIX ball) -foreach(MODE mxfp8 fma32 f32) - add_buckyball_mlir_test(mxmm_${MODE} TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) - buckyball_mlirtest_name(TEST_NAME ${BUCKYBALL_MLIR_ACTIVE_TARGET} ball_mxmm_${MODE}) - foreach(KIND baremetal linux) - add_custom_command(OUTPUT ${TEST_NAME}-${KIND} APPEND - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/../mxmm_native_test.h - ${CMAKE_CURRENT_SOURCE_DIR}/../mxmm_native_steps.h) - endforeach() -endforeach() -foreach(CASE mxmm_window mxmm_window_rows mxmm_window_chain) - add_buckyball_mlir_test(${CASE} TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) - buckyball_mlirtest_name(TEST_NAME ${BUCKYBALL_MLIR_ACTIVE_TARGET} ball_${CASE}) - foreach(KIND baremetal linux) - add_custom_command(OUTPUT ${TEST_NAME}-${KIND} APPEND - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/../../isa/mxmm.h) - endforeach() -endforeach() diff --git a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_f32.mlir b/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_f32.mlir deleted file mode 100644 index c7a5324b0..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_f32.mlir +++ /dev/null @@ -1,90 +0,0 @@ -func.func @step_m1_n16_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807361 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m1_n16_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807361 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m1_n16_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807361 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m1_n48_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938433 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m1_n48_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938433 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m1_n48_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938433 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m16_n16_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807376 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m16_n16_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807376 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m16_n16_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807376 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m16_n48_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938448 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m16_n48_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938448 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m16_n48_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938448 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m32_n16_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807392 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m32_n16_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807392 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m32_n16_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807392 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m32_n48_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938464 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m32_n48_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938464 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m32_n48_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938464 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_f32_main.cpp b/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_f32_main.cpp deleted file mode 100644 index e9dbeb578..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_f32_main.cpp +++ /dev/null @@ -1,3 +0,0 @@ -#include "../mxmm_native_test.h" - -int main() { return run_mxmm_native_test(); } diff --git a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_fma32.mlir b/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_fma32.mlir deleted file mode 100644 index af6b58299..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_fma32.mlir +++ /dev/null @@ -1,90 +0,0 @@ -func.func @step_m1_n16_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807361 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m1_n16_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807361 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m1_n16_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807361 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m1_n48_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938433 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m1_n48_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938433 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m1_n48_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938433 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m16_n16_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807376 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m16_n16_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807376 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m16_n16_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807376 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m16_n48_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938448 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m16_n48_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938448 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m16_n48_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938448 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m32_n16_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807392 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m32_n16_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807392 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m32_n16_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807392 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m32_n48_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938464 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m32_n48_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938464 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m32_n48_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938464 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_fma32_main.cpp b/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_fma32_main.cpp deleted file mode 100644 index 539a04035..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_fma32_main.cpp +++ /dev/null @@ -1,3 +0,0 @@ -#include "../mxmm_native_test.h" - -int main() { return run_mxmm_native_test(); } diff --git a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_mxfp8.mlir b/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_mxfp8.mlir deleted file mode 100644 index 9632d76de..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_mxfp8.mlir +++ /dev/null @@ -1,90 +0,0 @@ -func.func @step_m1_n16_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295032833 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - buckyball.mxfp8 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m1_n16_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295032833 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.mxfp8 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m1_n16_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295032833 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.mxfp8 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m1_n48_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295163905 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - buckyball.mxfp8 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m1_n48_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295163905 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.mxfp8 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m1_n48_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295163905 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.mxfp8 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m16_n16_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295032848 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - buckyball.mxfp8 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m16_n16_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295032848 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.mxfp8 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m16_n16_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295032848 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.mxfp8 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m16_n48_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295163920 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - buckyball.mxfp8 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m16_n48_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295163920 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.mxfp8 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m16_n48_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295163920 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.mxfp8 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m32_n16_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295032864 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - buckyball.mxfp8 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m32_n16_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295032864 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.mxfp8 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m32_n16_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295032864 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.mxfp8 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m32_n48_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295163936 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - buckyball.mxfp8 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m32_n48_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295163936 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.mxfp8 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m32_n48_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295163936 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.mxfp8 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_mxfp8_main.cpp b/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_mxfp8_main.cpp deleted file mode 100644 index deba14a6b..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_mxfp8_main.cpp +++ /dev/null @@ -1,3 +0,0 @@ -#include "../mxmm_native_test.h" - -int main() { return run_mxmm_native_test(); } diff --git a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_window.mlir b/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_window.mlir deleted file mode 100644 index 8d8dd6f12..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_window.mlir +++ /dev/null @@ -1,26 +0,0 @@ -// RUN: buddy-opt %s --target=toy -assign-physical-banks -lower-bank-ssa-to-intrinsics -lower-buckyball -canonicalize | FileCheck %s -// CHECK-LABEL: func.func @window -// CHECK: buckyball.intr.custom {{.*}} -// CHECK-LABEL: func.func @window_small -// CHECK-DAG: %[[RS1:[^ ]+]] = arith.constant 34367084547 : i64 -// CHECK-DAG: %[[RS2:[^ ]+]] = arith.constant 9007474250153985 : i64 -// CHECK: buckyball.intr.custom %[[RS1]], %[[RS2]] -func.func @window(%a: i64, %b: i64, %c: i64, %rows: i64, %cols: i64, %count: i64, %full: i64, %start: i64, %first: i1, %last: i1, %base: i64) attributes {llvm.emit_c_interface} { - buckyball.mxfp8_window %a, %b, %c, %rows, %cols, %count, %full, %start, %first, %last, %base : i64 - return -} -func.func @window_small() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 - %b = arith.constant 6 : i64 - %c = arith.constant 7 : i64 - %rows = arith.constant 1 : i64 - %cols = arith.constant 16 : i64 - %count = arith.constant 32 : i64 - %full = arith.constant 64 : i64 - %start = arith.constant 32 : i64 - %first = arith.constant true - %last = arith.constant true - %base = arith.constant 1 : i64 - buckyball.mxfp8_window %a, %b, %c, %rows, %cols, %count, %full, %start, %first, %last, %base : i64 - return -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_chain.mlir b/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_chain.mlir deleted file mode 100644 index d1047488a..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_chain.mlir +++ /dev/null @@ -1,7 +0,0 @@ -// RUN: buddy-opt %s --target=toy -assign-physical-banks -lower-bank-ssa-to-intrinsics -lower-buckyball -canonicalize | FileCheck %s -// CHECK-LABEL: func.func @window -// CHECK: buckyball.intr.custom {{.*}} -func.func @window(%a: i64, %b: i64, %c: i64, %rows: i64, %cols: i64, %count: i64, %full: i64, %start: i64, %first: i1, %last: i1, %base: i64) attributes {llvm.emit_c_interface} { - buckyball.mxfp8_window %a, %b, %c, %rows, %cols, %count, %full, %start, %first, %last, %base : i64 - return -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_chain_main.cpp b/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_chain_main.cpp deleted file mode 100644 index bb3fc7a68..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_chain_main.cpp +++ /dev/null @@ -1,70 +0,0 @@ -#include "../../isa/mxmm.h" -#include "buckyball.h" -#include -#include -#include -#include -extern "C" void _mlir_ciface_window(uint64_t a, uint64_t b, uint64_t c, - uint64_t m, uint64_t n, uint64_t count, - uint64_t full, uint64_t start, bool first, - bool last, uint64_t base); - -enum { BYTES = BANK_LINES * BANK_WIDTH / 8 }; -static uint8_t a[2][BYTES] __attribute__((aligned(64))); -static uint8_t packed[BYTES] __attribute__((aligned(64))); -static uint8_t w[BYTES] __attribute__((aligned(64))); -static uint8_t actual[BYTES] __attribute__((aligned(64))); -static uint8_t expected[BYTES] __attribute__((aligned(64))); -int main(void) { - memset(a, 255, sizeof(a)); - memset(a[0], 0x38, 32); - a[0][64] = 127; - memset(a[1] + 32, 0x40, 32); - a[1][97] = 127; - for (int col = 0; col < 16; ++col) { - memset(w + col * 32, col & 1 ? 0xb8 : 0x38, 32); - w[512 + col] = 127; - } - memset(actual, 0x5a, BYTES); - memset(expected, 0x5a, BYTES); - for (int bank = 3; bank <= 8; ++bank) - bb_mem_alloc(bank, 1, 1); - bb_mvin((uintptr_t)a[0], 3, BANK_LINES, 1); - bb_mvin((uintptr_t)a[1], 4, BANK_LINES, 1); - bb_mvin((uintptr_t)w, 6, BANK_LINES, 1); - bb_mvin((uintptr_t)expected, 7, BANK_LINES, 1); - bb_mvin((uintptr_t)actual, 8, BANK_LINES, 1); - for (int mode = 0; mode < 2; ++mode) - for (int part = 0; part < 2; ++part) { - if (mode == 0) { - memcpy(packed, a[part] + part * 32, 32); - packed[32] = 127; - bb_mvin((uintptr_t)packed, 5, 3, 1); - bb_mxmm_mxfp8(5, 6, 7, 1, 16, 32, part == 0, part == 1, 1); - bb_mvout((uintptr_t)expected, 7, BANK_LINES, 1); - } else { - _mlir_ciface_window(3 + part, 6, 8, 1, 16, 32, part ? 96 : 64, - part * 32, part == 0, part == 1, 1); - bb_mvout((uintptr_t)actual, 8, BANK_LINES, 1); - } - if (part == 0) - for (int i = 0; i < BYTES; ++i) - if ((mode ? actual : expected)[i] != 0x5a) - return 1; - } - for (int col = 0; col < 16; ++col) { - uint32_t bits; - memcpy(&bits, expected + 16 + col * 4, 4); - if (bits != (col & 1 ? 0xc2c00000u : 0x42c00000u)) - return 4; - } - if (memcmp(actual, expected, BYTES)) - return 2; - for (int i = 0; i < BYTES; ++i) - if ((i < 16 || i >= 80) && actual[i] != 0x5a) - return 3; - for (int bank = 3; bank <= 8; ++bank) - bb_mem_release(bank); - puts("mxmm A-window continuation bank/layout/poison/guard MLIR ball PASS"); - return 0; -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_main.cpp b/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_main.cpp deleted file mode 100644 index f93584db2..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_main.cpp +++ /dev/null @@ -1,88 +0,0 @@ -#include "../../isa/mxmm.h" -#include "buckyball.h" -#include -#include -#include -#include -extern "C" void _mlir_ciface_window(uint64_t a, uint64_t b, uint64_t c, - uint64_t m, uint64_t n, uint64_t count, - uint64_t full, uint64_t start, bool first, - bool last, uint64_t base); -extern "C" void _mlir_ciface_window_small(); - -enum { BYTES = BANK_LINES * BANK_WIDTH / 8 }; -static uint8_t a[BYTES] __attribute__((aligned(64))); -static uint8_t packed[BYTES] __attribute__((aligned(64))); -static uint8_t w[BYTES] __attribute__((aligned(64))); -static uint8_t actual[BYTES] __attribute__((aligned(64))); -static uint8_t expected[BYTES] __attribute__((aligned(64))); - -int main(void) { - const int shapes[][4] = {{1, 16, 64, 32}}; - const uint8_t codes[] = {0, 0x80, 1, 0x81, 0x38, - 0xb8, 0x37, 0xb7, 0x7e, 0xfe}; - const uint8_t scales[] = {0, 119, 127, 128}; - for (int test = 0; test < 1; ++test) { - int m = shapes[test][0], n = shapes[test][1]; - int full = shapes[test][2], start = shapes[test][3]; - memset(a, 0x5a, sizeof(a)); - for (int row = 0; row < m; ++row) { - for (int k = 0; k < full; ++k) - a[row * full + k] = k < start ? 0x7f : codes[(k + row) % 10]; - for (int block = 0; block < full / 32; ++block) - a[m * full + row * (full / 32) + block] = - block < start / 32 ? 255 : scales[(block + row) % 4]; - } - memset(actual, 0x5a, sizeof(actual)); - memset(expected, 0x5a, sizeof(expected)); - for (int bank = 3; bank <= 7; ++bank) - bb_mem_alloc(bank, 1, 1); - bb_mvin((uintptr_t)a, 3, BANK_LINES, 1); - bb_mvin((uintptr_t)expected, 5, BANK_LINES, 1); - bb_mvin((uintptr_t)actual, 7, BANK_LINES, 1); - /* Complete old-71 chain before new-75 chain: one chain slot per hart. */ - for (int mode = 0; mode < 2; ++mode) { - for (int k = start; k < full; k += 480) { - int count = full - k < 480 ? full - k : 480; - memset(w, 0x5a, sizeof(w)); - for (int col = 0; col < n; ++col) { - for (int i = 0; i < count; ++i) - w[col * count + i] = codes[(k + i + col + 4) % 10]; - for (int block = 0; block < count / 32; ++block) - w[n * count + col * (count / 32) + block] = - scales[(k / 32 + block + col) % 4]; - } - if (mode == 0) { - memset(packed, 0x5a, sizeof(packed)); - for (int row = 0; row < m; ++row) { - memcpy(packed + row * count, a + row * full + k, count); - memcpy(packed + m * count + row * (count / 32), - a + m * full + row * (full / 32) + k / 32, count / 32); - } - } - bb_mvin((uintptr_t)w, 6, BANK_LINES, 1); - if (mode == 0) { - bb_mvin((uintptr_t)packed, 4, BANK_LINES, 1); - bb_mxmm_mxfp8(4, 6, 5, m, n, count, k == start, k + count == full, 1); - bb_mvout((uintptr_t)expected, 5, BANK_LINES, 1); - } else { - _mlir_ciface_window_small(); - bb_mvout((uintptr_t)actual, 7, BANK_LINES, 1); - } - if (k + count != full) - for (int i = 0; i < BYTES; ++i) - if ((mode ? actual : expected)[i] != 0x5a) - return 1; - } - } - if (memcmp(actual, expected, BYTES)) - return 2; - for (int i = 0; i < BYTES; ++i) - if ((i < 16 || i >= 16 + m * n * 4) && actual[i] != 0x5a) - return 3; - for (int bank = 3; bank <= 7; ++bank) - bb_mem_release(bank); - } - puts("mxmm A-window small old71/75/guard MLIR ball PASS"); - return 0; -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_rows.mlir b/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_rows.mlir deleted file mode 100644 index d1047488a..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_rows.mlir +++ /dev/null @@ -1,7 +0,0 @@ -// RUN: buddy-opt %s --target=toy -assign-physical-banks -lower-bank-ssa-to-intrinsics -lower-buckyball -canonicalize | FileCheck %s -// CHECK-LABEL: func.func @window -// CHECK: buckyball.intr.custom {{.*}} -func.func @window(%a: i64, %b: i64, %c: i64, %rows: i64, %cols: i64, %count: i64, %full: i64, %start: i64, %first: i1, %last: i1, %base: i64) attributes {llvm.emit_c_interface} { - buckyball.mxfp8_window %a, %b, %c, %rows, %cols, %count, %full, %start, %first, %last, %base : i64 - return -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_rows_main.cpp b/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_rows_main.cpp deleted file mode 100644 index e8197c986..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_rows_main.cpp +++ /dev/null @@ -1,88 +0,0 @@ -#include "../../isa/mxmm.h" -#include "buckyball.h" -#include -#include -#include -#include -extern "C" void _mlir_ciface_window(uint64_t a, uint64_t b, uint64_t c, - uint64_t m, uint64_t n, uint64_t count, - uint64_t full, uint64_t start, bool first, - bool last, uint64_t base); - -enum { BYTES = BANK_LINES * BANK_WIDTH / 8 }; -static uint8_t a[BYTES] __attribute__((aligned(64))); -static uint8_t packed[BYTES] __attribute__((aligned(64))); -static uint8_t w[BYTES] __attribute__((aligned(64))); -static uint8_t actual[BYTES] __attribute__((aligned(64))); -static uint8_t expected[BYTES] __attribute__((aligned(64))); - -int main(void) { - const int shapes[][4] = {{16, 32, 960, 480}, {1, 32, 4096, 0}}; - const uint8_t codes[] = {0, 0x80, 1, 0x81, 0x38, - 0xb8, 0x37, 0xb7, 0x7e, 0xfe}; - const uint8_t scales[] = {0, 119, 127, 128}; - for (int test = 0; test < 2; ++test) { - int m = shapes[test][0], n = shapes[test][1]; - int full = shapes[test][2], start = shapes[test][3]; - memset(a, 0x5a, sizeof(a)); - for (int row = 0; row < m; ++row) { - for (int k = 0; k < full; ++k) - a[row * full + k] = k < start ? 0x7f : codes[(k + row) % 10]; - for (int block = 0; block < full / 32; ++block) - a[m * full + row * (full / 32) + block] = - block < start / 32 ? 255 : scales[(block + row) % 4]; - } - memset(actual, 0x5a, sizeof(actual)); - memset(expected, 0x5a, sizeof(expected)); - for (int bank = 3; bank <= 7; ++bank) - bb_mem_alloc(bank, 1, 1); - bb_mvin((uintptr_t)a, 3, BANK_LINES, 1); - bb_mvin((uintptr_t)expected, 5, BANK_LINES, 1); - bb_mvin((uintptr_t)actual, 7, BANK_LINES, 1); - /* Complete old-71 chain before new-75 chain: one chain slot per hart. */ - for (int mode = 0; mode < 2; ++mode) { - for (int k = start; k < full; k += 480) { - int count = full - k < 480 ? full - k : 480; - memset(w, 0x5a, sizeof(w)); - for (int col = 0; col < n; ++col) { - for (int i = 0; i < count; ++i) - w[col * count + i] = codes[(k + i + col + 4) % 10]; - for (int block = 0; block < count / 32; ++block) - w[n * count + col * (count / 32) + block] = - scales[(k / 32 + block + col) % 4]; - } - if (mode == 0) { - memset(packed, 0x5a, sizeof(packed)); - for (int row = 0; row < m; ++row) { - memcpy(packed + row * count, a + row * full + k, count); - memcpy(packed + m * count + row * (count / 32), - a + m * full + row * (full / 32) + k / 32, count / 32); - } - } - bb_mvin((uintptr_t)w, 6, BANK_LINES, 1); - if (mode == 0) { - bb_mvin((uintptr_t)packed, 4, BANK_LINES, 1); - bb_mxmm_mxfp8(4, 6, 5, m, n, count, k == start, k + count == full, 1); - bb_mvout((uintptr_t)expected, 5, BANK_LINES, 1); - } else { - _mlir_ciface_window(3, 6, 7, m, n, count, full, k, k == start, - k + count == full, 1); - bb_mvout((uintptr_t)actual, 7, BANK_LINES, 1); - } - if (k + count != full) - for (int i = 0; i < BYTES; ++i) - if ((mode ? actual : expected)[i] != 0x5a) - return 1; - } - } - if (memcmp(actual, expected, BYTES)) - return 2; - for (int i = 0; i < BYTES; ++i) - if ((i < 16 || i >= 16 + m * n * 4) && actual[i] != 0x5a) - return 3; - for (int bank = 3; bank <= 7; ++bank) - bb_mem_release(bank); - } - puts("mxmm A-window row-gap/4096-chain/scale-byte/guard MLIR ball PASS"); - return 0; -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/bank/CMakeLists.txt b/examples/balls/mxmm/workloads/mlir_tests/bank/CMakeLists.txt deleted file mode 100644 index 8932020ab..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/bank/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) - set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) -endif() -set(BUCKYBALL_MLIR_TEST_PREFIX bank) -foreach(MODE mxfp8 fma32 f32) - add_buckyball_mlir_test(mxmm_${MODE} TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) - buckyball_mlirtest_name(TEST_NAME ${BUCKYBALL_MLIR_ACTIVE_TARGET} bank_mxmm_${MODE}) - foreach(KIND baremetal linux) - add_custom_command(OUTPUT ${TEST_NAME}-${KIND} APPEND - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/../mxmm_native_test.h - ${CMAKE_CURRENT_SOURCE_DIR}/../mxmm_native_steps.h) - endforeach() -endforeach() -foreach(CASE mxmm_window mxmm_window_rows mxmm_window_chain) - add_buckyball_mlir_test(${CASE} TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) - buckyball_mlirtest_name(TEST_NAME ${BUCKYBALL_MLIR_ACTIVE_TARGET} bank_${CASE}) - foreach(KIND baremetal linux) - add_custom_command(OUTPUT ${TEST_NAME}-${KIND} APPEND - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/../../isa/mxmm.h) - endforeach() -endforeach() diff --git a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_f32.mlir b/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_f32.mlir deleted file mode 100644 index b328dd843..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_f32.mlir +++ /dev/null @@ -1,90 +0,0 @@ -func.func @step_m1_n16_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807361 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m1_n16_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807361 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m1_n16_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807361 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m1_n48_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938433 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m1_n48_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938433 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m1_n48_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938433 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m16_n16_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807376 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m16_n16_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807376 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m16_n16_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807376 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m16_n48_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938448 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m16_n48_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938448 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m16_n48_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938448 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m32_n16_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807392 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m32_n16_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807392 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m32_n16_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807392 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m32_n48_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938464 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m32_n48_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938464 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m32_n48_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938464 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_f32_main.cpp b/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_f32_main.cpp deleted file mode 100644 index e9dbeb578..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_f32_main.cpp +++ /dev/null @@ -1,3 +0,0 @@ -#include "../mxmm_native_test.h" - -int main() { return run_mxmm_native_test(); } diff --git a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_fma32.mlir b/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_fma32.mlir deleted file mode 100644 index 004ec5046..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_fma32.mlir +++ /dev/null @@ -1,90 +0,0 @@ -func.func @step_m1_n16_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807361 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m1_n16_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807361 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m1_n16_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807361 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m1_n48_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938433 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m1_n48_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938433 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m1_n48_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938433 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m16_n16_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807376 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m16_n16_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807376 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m16_n16_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807376 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m16_n48_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938448 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m16_n48_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938448 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m16_n48_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938448 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m32_n16_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807392 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m32_n16_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807392 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m32_n16_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807392 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m32_n48_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938464 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m32_n48_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938464 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m32_n48_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938464 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_fma32_main.cpp b/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_fma32_main.cpp deleted file mode 100644 index 539a04035..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_fma32_main.cpp +++ /dev/null @@ -1,3 +0,0 @@ -#include "../mxmm_native_test.h" - -int main() { return run_mxmm_native_test(); } diff --git a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_mxfp8.mlir b/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_mxfp8.mlir deleted file mode 100644 index 36c3a3500..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_mxfp8.mlir +++ /dev/null @@ -1,90 +0,0 @@ -func.func @step_m1_n16_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295032833 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m1_n16_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295032833 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m1_n16_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295032833 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m1_n48_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295163905 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m1_n48_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295163905 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m1_n48_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295163905 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m16_n16_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295032848 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m16_n16_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295032848 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m16_n16_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295032848 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m16_n48_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295163920 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m16_n48_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295163920 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m16_n48_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295163920 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m32_n16_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295032864 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m32_n16_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295032864 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m32_n16_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295032864 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m32_n48_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295163936 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m32_n48_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295163936 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m32_n48_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295163936 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_mxfp8_main.cpp b/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_mxfp8_main.cpp deleted file mode 100644 index deba14a6b..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_mxfp8_main.cpp +++ /dev/null @@ -1,3 +0,0 @@ -#include "../mxmm_native_test.h" - -int main() { return run_mxmm_native_test(); } diff --git a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_window.mlir b/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_window.mlir deleted file mode 100644 index baf9ba037..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_window.mlir +++ /dev/null @@ -1,28 +0,0 @@ -// RUN: buddy-opt %s --target=toy -assign-physical-banks | FileCheck %s --check-prefix=BANK -// BANK: buckyball.mxfp8_window -// RUN: buddy-opt %s --target=toy -assign-physical-banks -lower-bank-ssa-to-intrinsics -lower-buckyball -canonicalize | FileCheck %s -// CHECK-LABEL: func.func @window -// CHECK: buckyball.intr.custom {{.*}} -// CHECK-LABEL: func.func @window_small -// CHECK-DAG: %[[RS1:[^ ]+]] = arith.constant 34367084547 : i64 -// CHECK-DAG: %[[RS2:[^ ]+]] = arith.constant 9007474250153985 : i64 -// CHECK: buckyball.intr.custom %[[RS1]], %[[RS2]] -func.func @window(%a: i64, %b: i64, %c: i64, %rows: i64, %cols: i64, %count: i64, %full: i64, %start: i64, %first: i1, %last: i1, %base: i64) attributes {llvm.emit_c_interface} { - %state = buckyball.bank_mxfp8_window %a %b %c %rows %cols %count %full %start %first %last %base : i64 - return -} -func.func @window_small() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 - %b = arith.constant 6 : i64 - %c = arith.constant 7 : i64 - %rows = arith.constant 1 : i64 - %cols = arith.constant 16 : i64 - %count = arith.constant 32 : i64 - %full = arith.constant 64 : i64 - %start = arith.constant 32 : i64 - %first = arith.constant true - %last = arith.constant true - %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8_window %a %b %c %rows %cols %count %full %start %first %last %base : i64 - return -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_chain.mlir b/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_chain.mlir deleted file mode 100644 index 6008bc01a..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_chain.mlir +++ /dev/null @@ -1,9 +0,0 @@ -// RUN: buddy-opt %s --target=toy -assign-physical-banks | FileCheck %s --check-prefix=BANK -// BANK: buckyball.mxfp8_window -// RUN: buddy-opt %s --target=toy -assign-physical-banks -lower-bank-ssa-to-intrinsics -lower-buckyball -canonicalize | FileCheck %s -// CHECK-LABEL: func.func @window -// CHECK: buckyball.intr.custom {{.*}} -func.func @window(%a: i64, %b: i64, %c: i64, %rows: i64, %cols: i64, %count: i64, %full: i64, %start: i64, %first: i1, %last: i1, %base: i64) attributes {llvm.emit_c_interface} { - %state = buckyball.bank_mxfp8_window %a %b %c %rows %cols %count %full %start %first %last %base : i64 - return -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_chain_main.cpp b/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_chain_main.cpp deleted file mode 100644 index c0d4394a4..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_chain_main.cpp +++ /dev/null @@ -1,70 +0,0 @@ -#include "../../isa/mxmm.h" -#include "buckyball.h" -#include -#include -#include -#include -extern "C" void _mlir_ciface_window(uint64_t a, uint64_t b, uint64_t c, - uint64_t m, uint64_t n, uint64_t count, - uint64_t full, uint64_t start, bool first, - bool last, uint64_t base); - -enum { BYTES = BANK_LINES * BANK_WIDTH / 8 }; -static uint8_t a[2][BYTES] __attribute__((aligned(64))); -static uint8_t packed[BYTES] __attribute__((aligned(64))); -static uint8_t w[BYTES] __attribute__((aligned(64))); -static uint8_t actual[BYTES] __attribute__((aligned(64))); -static uint8_t expected[BYTES] __attribute__((aligned(64))); -int main(void) { - memset(a, 255, sizeof(a)); - memset(a[0], 0x38, 32); - a[0][64] = 127; - memset(a[1] + 32, 0x40, 32); - a[1][97] = 127; - for (int col = 0; col < 16; ++col) { - memset(w + col * 32, col & 1 ? 0xb8 : 0x38, 32); - w[512 + col] = 127; - } - memset(actual, 0x5a, BYTES); - memset(expected, 0x5a, BYTES); - for (int bank = 3; bank <= 8; ++bank) - bb_mem_alloc(bank, 1, 1); - bb_mvin((uintptr_t)a[0], 3, BANK_LINES, 1); - bb_mvin((uintptr_t)a[1], 4, BANK_LINES, 1); - bb_mvin((uintptr_t)w, 6, BANK_LINES, 1); - bb_mvin((uintptr_t)expected, 7, BANK_LINES, 1); - bb_mvin((uintptr_t)actual, 8, BANK_LINES, 1); - for (int mode = 0; mode < 2; ++mode) - for (int part = 0; part < 2; ++part) { - if (mode == 0) { - memcpy(packed, a[part] + part * 32, 32); - packed[32] = 127; - bb_mvin((uintptr_t)packed, 5, 3, 1); - bb_mxmm_mxfp8(5, 6, 7, 1, 16, 32, part == 0, part == 1, 1); - bb_mvout((uintptr_t)expected, 7, BANK_LINES, 1); - } else { - _mlir_ciface_window(3 + part, 6, 8, 1, 16, 32, part ? 96 : 64, - part * 32, part == 0, part == 1, 1); - bb_mvout((uintptr_t)actual, 8, BANK_LINES, 1); - } - if (part == 0) - for (int i = 0; i < BYTES; ++i) - if ((mode ? actual : expected)[i] != 0x5a) - return 1; - } - for (int col = 0; col < 16; ++col) { - uint32_t bits; - memcpy(&bits, expected + 16 + col * 4, 4); - if (bits != (col & 1 ? 0xc2c00000u : 0x42c00000u)) - return 4; - } - if (memcmp(actual, expected, BYTES)) - return 2; - for (int i = 0; i < BYTES; ++i) - if ((i < 16 || i >= 80) && actual[i] != 0x5a) - return 3; - for (int bank = 3; bank <= 8; ++bank) - bb_mem_release(bank); - puts("mxmm A-window continuation bank/layout/poison/guard MLIR bank PASS"); - return 0; -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_main.cpp b/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_main.cpp deleted file mode 100644 index 1ac1d78df..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_main.cpp +++ /dev/null @@ -1,88 +0,0 @@ -#include "../../isa/mxmm.h" -#include "buckyball.h" -#include -#include -#include -#include -extern "C" void _mlir_ciface_window(uint64_t a, uint64_t b, uint64_t c, - uint64_t m, uint64_t n, uint64_t count, - uint64_t full, uint64_t start, bool first, - bool last, uint64_t base); -extern "C" void _mlir_ciface_window_small(); - -enum { BYTES = BANK_LINES * BANK_WIDTH / 8 }; -static uint8_t a[BYTES] __attribute__((aligned(64))); -static uint8_t packed[BYTES] __attribute__((aligned(64))); -static uint8_t w[BYTES] __attribute__((aligned(64))); -static uint8_t actual[BYTES] __attribute__((aligned(64))); -static uint8_t expected[BYTES] __attribute__((aligned(64))); - -int main(void) { - const int shapes[][4] = {{1, 16, 64, 32}}; - const uint8_t codes[] = {0, 0x80, 1, 0x81, 0x38, - 0xb8, 0x37, 0xb7, 0x7e, 0xfe}; - const uint8_t scales[] = {0, 119, 127, 128}; - for (int test = 0; test < 1; ++test) { - int m = shapes[test][0], n = shapes[test][1]; - int full = shapes[test][2], start = shapes[test][3]; - memset(a, 0x5a, sizeof(a)); - for (int row = 0; row < m; ++row) { - for (int k = 0; k < full; ++k) - a[row * full + k] = k < start ? 0x7f : codes[(k + row) % 10]; - for (int block = 0; block < full / 32; ++block) - a[m * full + row * (full / 32) + block] = - block < start / 32 ? 255 : scales[(block + row) % 4]; - } - memset(actual, 0x5a, sizeof(actual)); - memset(expected, 0x5a, sizeof(expected)); - for (int bank = 3; bank <= 7; ++bank) - bb_mem_alloc(bank, 1, 1); - bb_mvin((uintptr_t)a, 3, BANK_LINES, 1); - bb_mvin((uintptr_t)expected, 5, BANK_LINES, 1); - bb_mvin((uintptr_t)actual, 7, BANK_LINES, 1); - /* Complete old-71 chain before new-75 chain: one chain slot per hart. */ - for (int mode = 0; mode < 2; ++mode) { - for (int k = start; k < full; k += 480) { - int count = full - k < 480 ? full - k : 480; - memset(w, 0x5a, sizeof(w)); - for (int col = 0; col < n; ++col) { - for (int i = 0; i < count; ++i) - w[col * count + i] = codes[(k + i + col + 4) % 10]; - for (int block = 0; block < count / 32; ++block) - w[n * count + col * (count / 32) + block] = - scales[(k / 32 + block + col) % 4]; - } - if (mode == 0) { - memset(packed, 0x5a, sizeof(packed)); - for (int row = 0; row < m; ++row) { - memcpy(packed + row * count, a + row * full + k, count); - memcpy(packed + m * count + row * (count / 32), - a + m * full + row * (full / 32) + k / 32, count / 32); - } - } - bb_mvin((uintptr_t)w, 6, BANK_LINES, 1); - if (mode == 0) { - bb_mvin((uintptr_t)packed, 4, BANK_LINES, 1); - bb_mxmm_mxfp8(4, 6, 5, m, n, count, k == start, k + count == full, 1); - bb_mvout((uintptr_t)expected, 5, BANK_LINES, 1); - } else { - _mlir_ciface_window_small(); - bb_mvout((uintptr_t)actual, 7, BANK_LINES, 1); - } - if (k + count != full) - for (int i = 0; i < BYTES; ++i) - if ((mode ? actual : expected)[i] != 0x5a) - return 1; - } - } - if (memcmp(actual, expected, BYTES)) - return 2; - for (int i = 0; i < BYTES; ++i) - if ((i < 16 || i >= 16 + m * n * 4) && actual[i] != 0x5a) - return 3; - for (int bank = 3; bank <= 7; ++bank) - bb_mem_release(bank); - } - puts("mxmm A-window small old71/75/guard MLIR bank PASS"); - return 0; -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_rows.mlir b/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_rows.mlir deleted file mode 100644 index 6008bc01a..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_rows.mlir +++ /dev/null @@ -1,9 +0,0 @@ -// RUN: buddy-opt %s --target=toy -assign-physical-banks | FileCheck %s --check-prefix=BANK -// BANK: buckyball.mxfp8_window -// RUN: buddy-opt %s --target=toy -assign-physical-banks -lower-bank-ssa-to-intrinsics -lower-buckyball -canonicalize | FileCheck %s -// CHECK-LABEL: func.func @window -// CHECK: buckyball.intr.custom {{.*}} -func.func @window(%a: i64, %b: i64, %c: i64, %rows: i64, %cols: i64, %count: i64, %full: i64, %start: i64, %first: i1, %last: i1, %base: i64) attributes {llvm.emit_c_interface} { - %state = buckyball.bank_mxfp8_window %a %b %c %rows %cols %count %full %start %first %last %base : i64 - return -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_rows_main.cpp b/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_rows_main.cpp deleted file mode 100644 index b35b1028f..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_rows_main.cpp +++ /dev/null @@ -1,88 +0,0 @@ -#include "../../isa/mxmm.h" -#include "buckyball.h" -#include -#include -#include -#include -extern "C" void _mlir_ciface_window(uint64_t a, uint64_t b, uint64_t c, - uint64_t m, uint64_t n, uint64_t count, - uint64_t full, uint64_t start, bool first, - bool last, uint64_t base); - -enum { BYTES = BANK_LINES * BANK_WIDTH / 8 }; -static uint8_t a[BYTES] __attribute__((aligned(64))); -static uint8_t packed[BYTES] __attribute__((aligned(64))); -static uint8_t w[BYTES] __attribute__((aligned(64))); -static uint8_t actual[BYTES] __attribute__((aligned(64))); -static uint8_t expected[BYTES] __attribute__((aligned(64))); - -int main(void) { - const int shapes[][4] = {{16, 32, 960, 480}, {1, 32, 4096, 0}}; - const uint8_t codes[] = {0, 0x80, 1, 0x81, 0x38, - 0xb8, 0x37, 0xb7, 0x7e, 0xfe}; - const uint8_t scales[] = {0, 119, 127, 128}; - for (int test = 0; test < 2; ++test) { - int m = shapes[test][0], n = shapes[test][1]; - int full = shapes[test][2], start = shapes[test][3]; - memset(a, 0x5a, sizeof(a)); - for (int row = 0; row < m; ++row) { - for (int k = 0; k < full; ++k) - a[row * full + k] = k < start ? 0x7f : codes[(k + row) % 10]; - for (int block = 0; block < full / 32; ++block) - a[m * full + row * (full / 32) + block] = - block < start / 32 ? 255 : scales[(block + row) % 4]; - } - memset(actual, 0x5a, sizeof(actual)); - memset(expected, 0x5a, sizeof(expected)); - for (int bank = 3; bank <= 7; ++bank) - bb_mem_alloc(bank, 1, 1); - bb_mvin((uintptr_t)a, 3, BANK_LINES, 1); - bb_mvin((uintptr_t)expected, 5, BANK_LINES, 1); - bb_mvin((uintptr_t)actual, 7, BANK_LINES, 1); - /* Complete old-71 chain before new-75 chain: one chain slot per hart. */ - for (int mode = 0; mode < 2; ++mode) { - for (int k = start; k < full; k += 480) { - int count = full - k < 480 ? full - k : 480; - memset(w, 0x5a, sizeof(w)); - for (int col = 0; col < n; ++col) { - for (int i = 0; i < count; ++i) - w[col * count + i] = codes[(k + i + col + 4) % 10]; - for (int block = 0; block < count / 32; ++block) - w[n * count + col * (count / 32) + block] = - scales[(k / 32 + block + col) % 4]; - } - if (mode == 0) { - memset(packed, 0x5a, sizeof(packed)); - for (int row = 0; row < m; ++row) { - memcpy(packed + row * count, a + row * full + k, count); - memcpy(packed + m * count + row * (count / 32), - a + m * full + row * (full / 32) + k / 32, count / 32); - } - } - bb_mvin((uintptr_t)w, 6, BANK_LINES, 1); - if (mode == 0) { - bb_mvin((uintptr_t)packed, 4, BANK_LINES, 1); - bb_mxmm_mxfp8(4, 6, 5, m, n, count, k == start, k + count == full, 1); - bb_mvout((uintptr_t)expected, 5, BANK_LINES, 1); - } else { - _mlir_ciface_window(3, 6, 7, m, n, count, full, k, k == start, - k + count == full, 1); - bb_mvout((uintptr_t)actual, 7, BANK_LINES, 1); - } - if (k + count != full) - for (int i = 0; i < BYTES; ++i) - if ((mode ? actual : expected)[i] != 0x5a) - return 1; - } - } - if (memcmp(actual, expected, BYTES)) - return 2; - for (int i = 0; i < BYTES; ++i) - if ((i < 16 || i >= 16 + m * n * 4) && actual[i] != 0x5a) - return 3; - for (int bank = 3; bank <= 7; ++bank) - bb_mem_release(bank); - } - puts("mxmm A-window row-gap/4096-chain/scale-byte/guard MLIR bank PASS"); - return 0; -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/mxmm_native_steps.h b/examples/balls/mxmm/workloads/mlir_tests/mxmm_native_steps.h deleted file mode 100644 index 0d48c8f97..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/mxmm_native_steps.h +++ /dev/null @@ -1,33 +0,0 @@ -#pragma once - -extern "C" void _mlir_ciface_step_m1_n16_first(); -extern "C" void _mlir_ciface_step_m1_n16_last(); -extern "C" void _mlir_ciface_step_m1_n16_single(); -extern "C" void _mlir_ciface_step_m1_n48_first(); -extern "C" void _mlir_ciface_step_m1_n48_last(); -extern "C" void _mlir_ciface_step_m1_n48_single(); -extern "C" void _mlir_ciface_step_m16_n16_first(); -extern "C" void _mlir_ciface_step_m16_n16_last(); -extern "C" void _mlir_ciface_step_m16_n16_single(); -extern "C" void _mlir_ciface_step_m16_n48_first(); -extern "C" void _mlir_ciface_step_m16_n48_last(); -extern "C" void _mlir_ciface_step_m16_n48_single(); -extern "C" void _mlir_ciface_step_m32_n16_first(); -extern "C" void _mlir_ciface_step_m32_n16_last(); -extern "C" void _mlir_ciface_step_m32_n16_single(); -extern "C" void _mlir_ciface_step_m32_n48_first(); -extern "C" void _mlir_ciface_step_m32_n48_last(); -extern "C" void _mlir_ciface_step_m32_n48_single(); -static void (*steps[3][2][3])() = { - {{_mlir_ciface_step_m1_n16_first, _mlir_ciface_step_m1_n16_last, - _mlir_ciface_step_m1_n16_single}, - {_mlir_ciface_step_m1_n48_first, _mlir_ciface_step_m1_n48_last, - _mlir_ciface_step_m1_n48_single}}, - {{_mlir_ciface_step_m16_n16_first, _mlir_ciface_step_m16_n16_last, - _mlir_ciface_step_m16_n16_single}, - {_mlir_ciface_step_m16_n48_first, _mlir_ciface_step_m16_n48_last, - _mlir_ciface_step_m16_n48_single}}, - {{_mlir_ciface_step_m32_n16_first, _mlir_ciface_step_m32_n16_last, - _mlir_ciface_step_m32_n16_single}, - {_mlir_ciface_step_m32_n48_first, _mlir_ciface_step_m32_n48_last, - _mlir_ciface_step_m32_n48_single}}}; diff --git a/examples/balls/mxmm/workloads/mlir_tests/mxmm_native_test.h b/examples/balls/mxmm/workloads/mlir_tests/mxmm_native_test.h deleted file mode 100644 index 7d047db9d..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/mxmm_native_test.h +++ /dev/null @@ -1,89 +0,0 @@ -#include "mxmm_native_steps.h" -#include -#include -#include -#include -#include -#include - -enum class MxmmFormat { Mxfp8, Fma32, F32 }; - -template static int run_mxmm_native_test() { - constexpr int K = Format == MxmmFormat::Mxfp8 ? 256 : 64; - constexpr int bytes = BANK_LINES * BANK_WIDTH / 8; - alignas(bytes) uint8_t a[bytes]{}, b[bytes]{}; - alignas(bytes) uint32_t result[bytes / 4]; - for (int mindex = 0; mindex < 3; mindex++) - for (int nindex = 0; nindex < 2; nindex++) { - int m = mindex == 0 ? 1 : mindex * 16, n = nindex == 0 ? 16 : 48; - memset(a, 0, sizeof(a)); - memset(b, 0, sizeof(b)); - if (Format == MxmmFormat::Mxfp8) { - memset(a, 0x38, m * K); - memset(a + m * K, 127, m * K / 32); - memset(b, 0x40, n * K); - memset(b + n * K, 127, n * K / 32); - } else { - auto *x = (uint32_t *)a; - auto *y = (uint32_t *)b; - for (int r = 0; r < m; r++) { - x[r * K] = 0xbf800000; - x[r * K + 1] = 0x3f800001; - } - for (int c = 0; c < n; c++) { - y[c * K] = 0x3f800000; - y[c * K + 1] = 0x3f7ffffe; - } - } - for (auto &v : result) - v = 0x5a5a5a5a; - for (int bank = 3; bank <= 5; bank++) - bb_mem_alloc(bank, 1, 1); - int ar = Format == MxmmFormat::Mxfp8 ? (m * K * 33 / 32 + 15) / 16 - : m * K / 4, - br = Format == MxmmFormat::Mxfp8 ? n * K * 33 / 32 / 16 : n * K / 4; - bb_mvin((uintptr_t)a, 3, ar, 1); - bb_mvin((uintptr_t)b, 4, br, 1); - bb_mvin((uintptr_t)result, 5, 1 + m * n / 4, 1); - steps[mindex][nindex][0](); - bb_mvout((uintptr_t)result, 5, 1 + m * n / 4, 1); - for (int i = 0; i < 4 + m * n; i++) - if (result[i] != 0x5a5a5a5a) - return 1; - if (Format == MxmmFormat::Mxfp8) { - memset(a, 0xb8, m * K); - memset(b, 0x38, n * K); - } else - memset(a, 0, m * K * 4); - bb_mvin((uintptr_t)a, 3, ar, 1); - bb_mvin((uintptr_t)b, 4, br, 1); - steps[mindex][nindex][1](); - bb_mvout((uintptr_t)result, 5, 1 + m * n / 4, 1); - uint32_t expected = Format == MxmmFormat::Mxfp8 ? 0x43800000 - : Format == MxmmFormat::Fma32 ? 0xa8800000 - : 0; - for (int i = 0; i < 4; i++) - if (result[i] != 0x5a5a5a5a) - return 2; - for (int i = 4; i < 4 + m * n; i++) - if (result[i] != expected) - return 3; - if (Format == MxmmFormat::Mxfp8) { - memset(a, 0, sizeof(a)); - memset(b, 0, sizeof(b)); - memset(a, 0x81, m * K); - memset(b, 1, n * K); - bb_mvin((uintptr_t)a, 3, ar, 1); - bb_mvin((uintptr_t)b, 4, br, 1); - steps[mindex][nindex][2](); - bb_mvout((uintptr_t)result, 5, 1 + m * n / 4, 1); - for (int i = 4; i < 4 + m * n; i++) - if (result[i] != 0x80000000) - return 4; - } - for (int bank = 3; bank <= 5; bank++) - bb_mem_release(bank); - } - puts("Mxmm MLIR mode/shape/chain PASS"); - return 0; -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/tensor/CMakeLists.txt b/examples/balls/mxmm/workloads/mlir_tests/tensor/CMakeLists.txt deleted file mode 100644 index efaf53511..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/tensor/CMakeLists.txt +++ /dev/null @@ -1,93 +0,0 @@ -if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) - set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) -endif() -set(BUCKYBALL_MLIR_TEST_PREFIX tensor) - -set(PYTHON ${Python3_EXECUTABLE}) -set(RUNTIME_PARAMS ${BUCKYBALL_ISA_DIR}/${BUCKYBALL_MLIR_ACTIVE_TARGET}) -if(NOT TARGET runtime) - add_subdirectory(${BUCKYBALL_REPO_ROOT}/stack/runtime ${CMAKE_CURRENT_BINARY_DIR}/runtime) -endif() -if(NOT TARGET rvv_kernels) - set(RUNTIME_PARAMS ${BUCKYBALL_ISA_DIR}/${BUCKYBALL_MLIR_ACTIVE_TARGET}) - add_subdirectory(${BUCKYBALL_REPO_ROOT}/bb-tests/workloads/lib/bbsw/kernels/rvv - ${CMAKE_CURRENT_BINARY_DIR}/rvv) -endif() -if(NOT TARGET mxfp8_pack) - add_subdirectory(${BUCKYBALL_REPO_ROOT}/bb-tests/workloads/lib/bbsw/kernels/mxfp8 - ${CMAKE_CURRENT_BINARY_DIR}/mxfp8) -endif() -if(NOT TARGET mxmm_kernels) - add_subdirectory(${BUCKYBALL_REPO_ROOT}/examples/balls/mxmm/workloads/kernels - ${CMAKE_CURRENT_BINARY_DIR}/mxmm) -endif() -get_target_property(MXMM_IMAGE_DIR mxmm_kernels BINARY_DIR) -set(MXMM_KERNEL_SOURCE ${BUCKYBALL_REPO_ROOT}/examples/balls/mxmm/workloads/kernels) -file(STRINGS ${RUNTIME_PARAMS}/params.h ROWS_DEFINE REGEX "^#define BANK_LINES ") -file(STRINGS ${RUNTIME_PARAMS}/params.h WIDTH_DEFINE REGEX "^#define BANK_WIDTH ") -string(REGEX REPLACE ".* " "" BANK_ROWS "${ROWS_DEFINE}") -string(REGEX REPLACE ".* " "" ROW_BITS "${WIDTH_DEFINE}") -math(EXPR PACKED_BYTES "${BANK_ROWS} * ${ROW_BITS} / 8") -math(EXPR DOUBLE_PACKED_BYTES "${PACKED_BYTES} * 2") -math(EXPR HALF_PACKED_BYTES "${PACKED_BYTES} / 2") -set(BARE_RVV_ARCHIVE ${CMAKE_CURRENT_BINARY_DIR}/libquant_rvv_baremetal.a) -set(BARE_RVV_TARGET mxmm_quant_rvv_baremetal_${BUCKYBALL_MLIR_ACTIVE_TARGET}) -set(RVV_SOURCE ${BUCKYBALL_REPO_ROOT}/bb-tests/workloads/lib/bbsw/kernels/rvv) -get_target_property(RVV_IMAGE_DIR rvv_images BINARY_DIR) -set(BARE_RVV_FLAGS -O2 -march=rv64gc -mcmodel=medany -std=c++20 - -I${RUNTIME_PARAMS} -I${RVV_IMAGE_DIR} -I${MXMM_IMAGE_DIR} - -I${BUCKYBALL_REPO_ROOT}/stack/runtime/include - -I${BUCKYBALL_REPO_ROOT}/bb-tests/workloads/lib - -I${BUCKYBALL_REPO_ROOT}/stack/compiler/thirdparty/buddy-mlir/llvm/mlir/include/mlir/ExecutionEngine) -add_custom_command(OUTPUT ${BARE_RVV_ARCHIVE} - COMMAND ${ELF_CC} ${BARE_RVV_FLAGS} -c ${RVV_SOURCE}/quant_dispatch.cpp -o quant_dispatch_bare.o - COMMAND ${ELF_CC} ${BARE_RVV_FLAGS} -c ${RVV_SOURCE}/../mxfp8/pack.cpp -o quant_pack_bare.o - COMMAND ${ELF_CC} ${BARE_RVV_FLAGS} -c ${RVV_IMAGE_DIR}/images.cpp -o quant_images_bare.o - COMMAND ${ELF_CC} ${BARE_RVV_FLAGS} -c ${MXMM_KERNEL_SOURCE}/pack_dispatch.cpp -o packmm_dispatch_bare.o - COMMAND ${ELF_CC} ${BARE_RVV_FLAGS} -c ${MXMM_KERNEL_SOURCE}/dispatch.cpp -o norm_window_dispatch_bare.o - COMMAND ${CMAKE_AR} rcs ${BARE_RVV_ARCHIVE} quant_dispatch_bare.o quant_pack_bare.o quant_images_bare.o packmm_dispatch_bare.o norm_window_dispatch_bare.o - DEPENDS rvv-images mxmm-images ${MXMM_KERNEL_SOURCE}/pack_dispatch.cpp - ${MXMM_KERNEL_SOURCE}/dispatch.cpp ${MXMM_KERNEL_SOURCE}/window.h - ${MXMM_KERNEL_SOURCE}/packmm.h ${RVV_SOURCE}/quant_dispatch.cpp ${RVV_SOURCE}/../mxfp8/pack.cpp - ${RVV_SOURCE}/../mxfp8/quant.h - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} VERBATIM) -add_custom_target(${BARE_RVV_TARGET}-build DEPENDS ${BARE_RVV_ARCHIVE}) -add_library(${BARE_RVV_TARGET} STATIC IMPORTED) -set_target_properties(${BARE_RVV_TARGET} PROPERTIES IMPORTED_LOCATION ${BARE_RVV_ARCHIVE}) -add_dependencies(${BARE_RVV_TARGET} ${BARE_RVV_TARGET}-build) -add_buckyball_mlir_test(mxmm_fp32 TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET} PASSES - -one-shot-bufferize=bufferize-function-boundaries - -lower-buckyball-to-bank-ssa -assign-physical-banks - -llvm-request-c-wrappers -lower-bank-ssa-to-intrinsics - -expand-strided-metadata -convert-linalg-to-loops -lower-buckyball - -lower-affine -convert-scf-to-cf -convert-cf-to-llvm -convert-arith-to-llvm - -convert-func-to-llvm -finalize-memref-to-llvm -reconcile-unrealized-casts - LIBRARIES rvv_kernels rvv_images mxmm_kernels BAREMETAL_LIBRARIES ${BARE_RVV_TARGET}) - -set(QUANT_PASSES - -canonicalize -cse -one-shot-bufferize=bufferize-function-boundaries - -buffer-deallocation-pipeline=private-function-dynamic-ownership=false - -lower-buckyball-to-bank-ssa -assign-physical-banks - -llvm-request-c-wrappers -lower-bank-ssa-to-intrinsics - -expand-strided-metadata -convert-linalg-to-loops -lower-buckyball - -lower-affine -convert-scf-to-cf -convert-cf-to-llvm -convert-arith-to-llvm - -convert-func-to-llvm -finalize-memref-to-llvm -reconcile-unrealized-casts) -foreach(BEHAVIOR shared layout input branch) - add_buckyball_mlir_test(mxmm_quant_${BEHAVIOR} - TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET} PASSES ${QUANT_PASSES} - LIBRARIES mxfp8_pack rvv_quant_backend rvv_images BAREMETAL_LIBRARIES ${BARE_RVV_TARGET}) - set(SOURCE ${CMAKE_CURRENT_BINARY_DIR}/${BUCKYBALL_MLIR_ACTIVE_TARGET}-mxmm_quant_${BEHAVIOR}.mlir) - set(CHECKED ${CMAKE_CURRENT_BINARY_DIR}/mxmm_quant_${BEHAVIOR}-checked.mlir) - add_custom_command(OUTPUT ${CHECKED} - COMMAND ${BUDDY_OPT} ${SOURCE} --target=${BUCKYBALL_MLIR_ACTIVE_TARGET} - -canonicalize -cse -one-shot-bufferize=bufferize-function-boundaries - -buffer-deallocation-pipeline=private-function-dynamic-ownership=false -o ${CHECKED} - COMMAND ${BUCKYBALL_REPO_ROOT}/stack/compiler/thirdparty/buddy-mlir/llvm/build/bin/FileCheck - ${SOURCE} --input-file=${CHECKED} - DEPENDS ${SOURCE} ${BUDDY_OPT} VERBATIM) - buckyball_mlirtest_name(TEST ${BUCKYBALL_MLIR_ACTIVE_TARGET} tensor_mxmm_quant_${BEHAVIOR}) - add_custom_target(${TEST}-ir-check DEPENDS ${CHECKED}) - add_dependencies(${TEST}-linux-build ${TEST}-ir-check) - add_dependencies(${TEST}-baremetal-build ${TEST}-ir-check) - add_dependencies(${BUCKYBALL_MLIR_GROUP_TARGET} ${TEST}-ir-check) -endforeach() diff --git a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_fp32.mlir b/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_fp32.mlir deleted file mode 100644 index 679e25cfa..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_fp32.mlir +++ /dev/null @@ -1,8 +0,0 @@ -func.func @tensor_unfused(%a: tensor<8x64xf32>, %b: tensor<64x48xf32>) -> tensor<8x48xf32> { - %0 = buckyball.fp32_matmul %a, %b : tensor<8x64xf32>, tensor<64x48xf32> -> tensor<8x48xf32> - return %0 : tensor<8x48xf32> -} -func.func @tensor_fma(%a: tensor<8x64xf32>, %b: tensor<64x48xf32>) -> tensor<8x48xf32> { - %0 = buckyball.fp32_matmul %a, %b : tensor<8x64xf32>, tensor<64x48xf32> -> tensor<8x48xf32> - return %0 : tensor<8x48xf32> -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_fp32_main.cpp b/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_fp32_main.cpp deleted file mode 100644 index 0f2963c30..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_fp32_main.cpp +++ /dev/null @@ -1,39 +0,0 @@ -#include -#include -#include -#include -#include -extern "C" void _mlir_ciface_tensor_fma(StridedMemRefType *, - StridedMemRefType *, - StridedMemRefType *); -extern "C" void _mlir_ciface_tensor_unfused(StridedMemRefType *, - StridedMemRefType *, - StridedMemRefType *); -int main() { - float a[8 * 64]{}, b[64 * 48]{}; - for (int r = 0; r < 8; r++) { - uint32_t x = 0xbf800000, y = 0x3f800001; - memcpy(a + r * 64, &x, 4); - memcpy(a + r * 64 + 1, &y, 4); - } - for (int c = 0; c < 48; c++) { - uint32_t x = 0x3f800000, y = 0x3f7ffffe; - memcpy(b + c, &x, 4); - memcpy(b + 48 + c, &y, 4); - } - StridedMemRefType lhs{a, a, 0, {8, 64}, {64, 1}}, - rhs{b, b, 0, {64, 48}, {48, 1}}, result; - for (int fused = 0; fused < 2; fused++) { - (fused ? _mlir_ciface_tensor_fma : _mlir_ciface_tensor_unfused)(&result, - &lhs, &rhs); - for (int i = 0; i < 384; i++) { - uint32_t bits; - memcpy(&bits, result.data + i, 4); - if (bits != (fused ? 0xa8800000u : 0)) - return 1; - } - free(result.basePtr); - } - puts("Mxmm FP32 tensor M8/N48 fused/unfused PASS"); - return 0; -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_branch.mlir.in b/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_branch.mlir.in deleted file mode 100644 index 2e2836d33..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_branch.mlir.in +++ /dev/null @@ -1,26 +0,0 @@ -// CHECK-LABEL: func.func @quant_branch -// CHECK: %[[PACK:.*]] = memref.alloc() alignment -// CHECK: call @mxfp8_quant -// CHECK: scf.if -// CHECK-NOT: memref.dealloc %[[PACK]] -// CHECK: buckyball.mxfp8_mem_matmul %[[PACK]], -// CHECK-NOT: memref.dealloc %[[PACK]] -// CHECK: } else { -// CHECK-NOT: memref.dealloc %[[PACK]] -// CHECK: buckyball.mxfp8_mem_matmul %[[PACK]], -// CHECK: buckyball.mxfp8_mem_matmul %[[PACK]], -// CHECK: memref.dealloc %[[PACK]] -// CHECK-NOT: memref.dealloc %[[PACK]] -// CHECK: return -func.func @quant_branch(%x: tensor<16x64xf32>,%w0: tensor<@PACKED_BYTES@xi8> {bufferization.buffer_layout=strided<[1],offset:?>},%w1: tensor<@PACKED_BYTES@xi8> {bufferization.buffer_layout=strided<[1],offset:?>},%condition:i1) -> (tensor<16x16xf32>,tensor<16x16xf32>) { - %q = buckyball.mxfp8_quant %x : tensor<16x64xf32> -> tensor<@PACKED_BYTES@xi8> - %first = scf.if %condition -> (tensor<16x16xf32>) { - %a = buckyball.mxfp8_matmul %q,%w0 : tensor<@PACKED_BYTES@xi8>,tensor<@PACKED_BYTES@xi8> -> tensor<16x16xf32> - scf.yield %a : tensor<16x16xf32> - } else { - %a = buckyball.mxfp8_matmul %q,%w1 : tensor<@PACKED_BYTES@xi8>,tensor<@PACKED_BYTES@xi8> -> tensor<16x16xf32> - scf.yield %a : tensor<16x16xf32> - } - %last = buckyball.mxfp8_matmul %q,%w1 : tensor<@PACKED_BYTES@xi8>,tensor<@PACKED_BYTES@xi8> -> tensor<16x16xf32> - return %first,%last : tensor<16x16xf32>,tensor<16x16xf32> -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_branch_main.cpp b/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_branch_main.cpp deleted file mode 100644 index 083c943b4..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_branch_main.cpp +++ /dev/null @@ -1,45 +0,0 @@ -#include -#include -#include -#include -#include -#include -struct Results { - StridedMemRefType first, last; -}; -extern "C" void _mlir_ciface_quant_branch(Results *, - StridedMemRefType *, - StridedMemRefType *, - StridedMemRefType *, bool); -int main() { - enum { Bytes = BANK_LINES * (BANK_WIDTH / 8) }; - alignas(64) float input[16 * 64]; - for (auto &value : input) - value = 1.0f; - alignas(64) int8_t a[Bytes]{}, b[Bytes]{}; - memset(a, 0x38, 16 * 64); - memset(b, 0x40, 16 * 64); - memset(a + 16 * 64, 127, 16 * 2); - memset(b + 16 * 64, 127, 16 * 2); - StridedMemRefType x{input, input, 0, {16, 64}, {64, 1}}; - StridedMemRefType wa{a, a, 0, {Bytes}, {1}}, - wb{b, b, 0, {Bytes}, {1}}; - for (bool condition : {false, true}) { - Results result; - _mlir_ciface_quant_branch(&result, &x, &wa, &wb, condition); - for (int row = 0; row < 16; row++) - for (int col = 0; col < 16; col++) { - if (result.first - .data[result.first.offset + row * result.first.strides[0] + - col * result.first.strides[1]] != - (condition ? 64.0f : 128.0f) || - result.last.data[result.last.offset + row * result.last.strides[0] + - col * result.last.strides[1]] != 128.0f) - return 1; - } - free(result.first.basePtr); - free(result.last.basePtr); - } - puts("Mxmm shared packed lifetime across both branch paths PASS"); - return 0; -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_input.mlir.in b/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_input.mlir.in deleted file mode 100644 index 652306906..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_input.mlir.in +++ /dev/null @@ -1,9 +0,0 @@ -// CHECK-LABEL: func.func @quant_input -// CHECK-COUNT-2: call @mxfp8_quant -// CHECK-NOT: call @mxfp8_quant -// CHECK: return -func.func @quant_input(%a: tensor<16x64xf32>,%b: tensor<16x64xf32>) -> (tensor<@PACKED_BYTES@xi8>,tensor<@PACKED_BYTES@xi8>) { - %q0 = buckyball.mxfp8_quant %a : tensor<16x64xf32> -> tensor<@PACKED_BYTES@xi8> - %q1 = buckyball.mxfp8_quant %b : tensor<16x64xf32> -> tensor<@PACKED_BYTES@xi8> - return %q0,%q1 : tensor<@PACKED_BYTES@xi8>,tensor<@PACKED_BYTES@xi8> -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_input_main.cpp b/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_input_main.cpp deleted file mode 100644 index 93f1e85db..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_input_main.cpp +++ /dev/null @@ -1,38 +0,0 @@ -#include -#include -#include -#include -#include -#include -struct Results { - StridedMemRefType a, b; -}; -extern "C" void _mlir_ciface_quant_input(Results *, - StridedMemRefType *, - StridedMemRefType *); -int main() { - alignas(64) float a[16 * 64], b[16 * 64]; - for (auto &value : a) - value = 1.0f; - for (auto &value : b) - value = 2.0f; - StridedMemRefType x{a, a, 0, {16, 64}, {64, 1}}, - y{b, b, 0, {16, 64}, {64, 1}}; - Results result; - _mlir_ciface_quant_input(&result, &x, &y); - StridedMemRefType *buffers[] = {&result.a, &result.b}; - for (int which = 0; which < 2; which++) { - auto &buffer = *buffers[which]; - for (int i = 0; i < 16 * 64; i++) - if (uint8_t(buffer.data[buffer.offset + i * buffer.strides[0]]) != 0x78) - return 1; - for (int i = 0; i < 16 * 2; i++) - if (uint8_t( - buffer.data[buffer.offset + (16 * 64 + i) * buffer.strides[0]]) != - 119 + which) - return 2; - free(buffer.basePtr); - } - puts("Mxmm distinct activation SSA inputs PASS"); - return 0; -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_layout.mlir.in b/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_layout.mlir.in deleted file mode 100644 index 5a4f32900..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_layout.mlir.in +++ /dev/null @@ -1,11 +0,0 @@ -// CHECK-LABEL: func.func @quant_layout -// CHECK-COUNT-4: call @mxfp8_quant -// CHECK-NOT: call @mxfp8_quant -// CHECK: return -func.func @quant_layout(%x: tensor<16x64xf32>) -> (tensor<@PACKED_BYTES@xi8>,tensor<@PACKED_BYTES@xi8>,tensor<@DOUBLE_PACKED_BYTES@xi8>,tensor<@PACKED_BYTES@xi8>) { - %q0 = buckyball.mxfp8_quant %x : tensor<16x64xf32> -> tensor<@PACKED_BYTES@xi8> - %q1 = buckyball.mxfp8_quant %x : tensor<16x64xf32> -> tensor<@PACKED_BYTES@xi8> - %q2 = buckyball.mxfp8_quant %x : tensor<16x64xf32> -> tensor<@DOUBLE_PACKED_BYTES@xi8> - %q3 = buckyball.mxfp8_quant %x : tensor<16x64xf32> -> tensor<@PACKED_BYTES@xi8> - return %q0,%q1,%q2,%q3 : tensor<@PACKED_BYTES@xi8>,tensor<@PACKED_BYTES@xi8>,tensor<@DOUBLE_PACKED_BYTES@xi8>,tensor<@PACKED_BYTES@xi8> -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_layout_main.cpp b/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_layout_main.cpp deleted file mode 100644 index 4afb041c2..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_layout_main.cpp +++ /dev/null @@ -1,45 +0,0 @@ -#include -#include -#include -#include -#include -#include -struct Results { - StridedMemRefType row16, row32, k32, stride; -}; -extern "C" void _mlir_ciface_quant_layout(Results *, - StridedMemRefType *); -int main() { - alignas(64) float input[16 * 64]; - for (auto &value : input) - value = 1.0f; - StridedMemRefType x{input, input, 0, {16, 64}, {64, 1}}; - Results result; - _mlir_ciface_quant_layout(&result, &x); - enum { Bytes = BANK_LINES * (BANK_WIDTH / 8) }; - StridedMemRefType *buffers[] = {&result.row16, &result.row32, - &result.k32, &result.stride}; - int rows[] = {16, 32, 16, 16}, ks[] = {64, 64, 32, 32}, - strides[] = {Bytes, Bytes, Bytes, Bytes / 2}; - for (int which = 0; which < 4; which++) { - auto &buffer = *buffers[which]; - for (int panel = 0; panel < 64 / ks[which]; panel++) - for (int row = 0; row < rows[which]; row++) { - int base = panel * strides[which], width = ks[which]; - for (int k = 0; k < width; k++) - if (uint8_t(buffer.data[buffer.offset + (base + row * width + k) * - buffer.strides[0]]) != - (row < 16 ? 0x78 : 0)) - return 1; - for (int k = 0; k < width / 32; k++) - if (uint8_t(buffer.data[buffer.offset + (base + rows[which] * width + - row * (width / 32) + k) * - buffer.strides[0]]) != - (row < 16 ? 119 : 127)) - return 2; - } - free(buffer.basePtr); - } - puts("Mxmm distinct rows/K/stride layouts: defined payload PASS"); - return 0; -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_shared.mlir.in b/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_shared.mlir.in deleted file mode 100644 index 75cd802ee..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_shared.mlir.in +++ /dev/null @@ -1,16 +0,0 @@ -// CHECK-LABEL: func.func @quant_shared -// CHECK: %[[PACK:.*]] = memref.alloc() alignment -// CHECK: call @mxfp8_quant -// CHECK-NOT: call @mxfp8_quant -// CHECK: buckyball.mxfp8_mem_matmul %[[PACK]], -// CHECK: buckyball.mxfp8_mem_matmul %[[PACK]], -// CHECK: memref.dealloc %[[PACK]] -// CHECK-NOT: memref.dealloc %[[PACK]] -// CHECK: return -func.func @quant_shared(%x: tensor<16x64xf32>, %gate: tensor<@PACKED_BYTES@xi8> {bufferization.buffer_layout=strided<[1],offset:?>}, %up: tensor<@PACKED_BYTES@xi8> {bufferization.buffer_layout=strided<[1],offset:?>}) -> (tensor<16x16xf32>,tensor<16x16xf32>) { - %q0 = buckyball.mxfp8_quant %x : tensor<16x64xf32> -> tensor<@PACKED_BYTES@xi8> - %a = buckyball.mxfp8_matmul %q0,%gate : tensor<@PACKED_BYTES@xi8>,tensor<@PACKED_BYTES@xi8> -> tensor<16x16xf32> - %q1 = buckyball.mxfp8_quant %x : tensor<16x64xf32> -> tensor<@PACKED_BYTES@xi8> - %b = buckyball.mxfp8_matmul %q1,%up : tensor<@PACKED_BYTES@xi8>,tensor<@PACKED_BYTES@xi8> -> tensor<16x16xf32> - return %a,%b : tensor<16x16xf32>,tensor<16x16xf32> -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_shared_main.cpp b/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_shared_main.cpp deleted file mode 100644 index 823137b3c..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_shared_main.cpp +++ /dev/null @@ -1,41 +0,0 @@ -#include -#include -#include -#include -#include -#include -struct Results { - StridedMemRefType gate, up; -}; -extern "C" void _mlir_ciface_quant_shared(Results *, - StridedMemRefType *, - StridedMemRefType *, - StridedMemRefType *); -int main() { - enum { Bytes = BANK_LINES * (BANK_WIDTH / 8) }; - alignas(64) float input[16 * 64]; - alignas(64) int8_t gate[Bytes]{}, up[Bytes]{}; - for (auto &value : input) - value = 1.0f; - memset(gate, 0x38, 16 * 64); - memset(up, 0x40, 16 * 64); - memset(gate + 16 * 64, 127, 16 * 2); - memset(up + 16 * 64, 127, 16 * 2); - StridedMemRefType x{input, input, 0, {16, 64}, {64, 1}}; - StridedMemRefType a{gate, gate, 0, {Bytes}, {1}}, - b{up, up, 0, {Bytes}, {1}}; - Results result; - _mlir_ciface_quant_shared(&result, &x, &a, &b); - for (int row = 0; row < 16; row++) - for (int col = 0; col < 16; col++) { - if (result.gate.data[result.gate.offset + row * result.gate.strides[0] + - col * result.gate.strides[1]] != 64.0f || - result.up.data[result.up.offset + row * result.up.strides[0] + - col * result.up.strides[1]] != 128.0f) - return 1; - } - free(result.gate.basePtr); - free(result.up.basePtr); - puts("Mxmm shared activation: two matrix consumers PASS"); - return 0; -} diff --git a/examples/balls/mxquant/arch/src/main/scala/MxquantBall.scala b/examples/balls/mxquant/arch/src/main/scala/MxquantBall.scala deleted file mode 100644 index a5b180db5..000000000 --- a/examples/balls/mxquant/arch/src/main/scala/MxquantBall.scala +++ /dev/null @@ -1,158 +0,0 @@ -package examples.balls.mxquant - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.balldomain.blink.{BallStatus, BlinkIO, HasBallStatus, HasBlink, SubRobRow} -import framework.balldomain.blink.mmio.{MmioRead, MmioWrite} -import framework.top.GlobalConfig - -@instantiable -class MxquantBall(val b: GlobalConfig) extends Module with HasBlink with HasBallStatus { - private val mapping = b.ballDomain.ballIdMappings.find(_.ballName == "MxquantBall").get - private val funct = b.ballDomain.ballISA.find(_.mnemonic == "MXQUANT").get.funct7 - require( - b.memDomain.bankWidth == 128 && b.memDomain.bankMaskLen == 16, - "MxquantBall requires 128-bit bank rows and byte write masks" - ) - require(b.memDomain.bankEntries >= 8, "MxquantBall requires eight input rows") - require(mapping.inBW == 1 && mapping.outBW == 1, "MxquantBall requires one read and one write") - require((funct >> 4) == 3, "MxquantBall funct7 must encode one read and one write") - - @public - val io = IO(new BlinkIO(b, 1, 1)) - def blink: BlinkIO = io - def status: BallStatus = io.status - - private val idle :: readReq :: readResp :: exponent :: encode :: codeReq :: codeResp :: scaleReq :: scaleResp :: complete :: Nil = - Enum(10) - private val state = RegInit(idle) - private val robId = Reg(UInt(log2Up(b.frontend.rob_entries).W)) - private val subId = Reg(UInt(log2Up(b.frontend.sub_rob_depth * 4).W)) - private val isSub = Reg(Bool()) - private val source = Reg(UInt(b.memDomain.vbankIdWidth.W)) - private val destination = Reg(UInt(b.memDomain.vbankIdWidth.W)) - private val count = Reg(UInt(b.frontend.iter_len.W)) - private val group = RegInit(0.U(b.frontend.iter_len.W)) - private val readRow = RegInit(0.U(3.W)) - private val lane = RegInit(0.U(5.W)) - private val codeRow = RegInit(0.U(1.W)) - private val maximum = RegInit(0.U(31.W)) - private val blockExponent = Reg(SInt(10.W)) - private val scale = Reg(UInt(8.W)) - private val values = Reg(Vec(32, UInt(32.W))) - private val codes = Reg(Vec(32, UInt(8.W))) - private val math = Module(new Encode) - math.io.input := values(lane) - math.io.blockExponent := blockExponent - - private val active = !reset.asBool - io.cmdReq.ready := state === idle && active - io.cmdResp.valid := state === complete && active - io.cmdResp.bits.rob_id := robId - io.cmdResp.bits.is_sub := isSub - io.cmdResp.bits.sub_rob_id := subId - io.status.idle := state === idle - io.status.running := state =/= idle && state =/= complete - io.subRobReq.valid := false.B - io.subRobReq.bits := SubRobRow.tieOff(b) - MmioRead.tieOff(io.mmioRead) - MmioWrite.tieOff(io.mmioWrite) - - val read = io.bankRead(0) - read.rob_id := robId - read.ball_id := 0.U - read.bank_id := source - read.group_id := 0.U - read.io.req.valid := state === readReq && active - read.io.req.bits.addr := (group << 3) + readRow - read.io.resp.ready := state === readResp && active - - val write = io.bankWrite(0) - write.rob_id := robId - write.ball_id := 0.U - write.bank_id := destination - write.group_id := 0.U - write.io.req.valid := (state === codeReq || state === scaleReq) && active - write.io.req.bits.addr := Mux(state === scaleReq, (count >> 4) + (group >> 4), (group << 1) + codeRow) - val lowCodes = Cat((15 to 0 by -1).map(codes(_))) - val highCodes = Cat((31 to 16 by -1).map(codes(_))) - write.io.req.bits.data := Mux( - state === scaleReq, - (scale.pad(128) << (group(3, 0) << 3))(127, 0), - Mux(codeRow === 0.U, lowCodes, highCodes) - ) - write.io.req.bits.mask := VecInit((0 until 16).map(i => state =/= scaleReq || group(3, 0) === i.U)) - write.io.resp.ready := (state === codeResp || state === scaleResp) && active - - switch(state) { - is(idle) { - when(io.cmdReq.fire) { - val cmd = io.cmdReq.bits.cmd - assert(cmd.funct7 === funct.U, "MxquantBall unknown funct7") - assert(cmd.iter > 0.U && cmd.iter(4, 0) === 0.U, "MxquantBall count must be positive and divisible by 32") - assert(cmd.iter <= (b.memDomain.bankEntries * 4).U, "MxquantBall input exceeds capacity") - assert(cmd.iter +& (cmd.iter >> 5) <= (b.memDomain.bankEntries * 16).U, "MxquantBall output exceeds capacity") - assert(cmd.rs1(19, 10) === 0.U && cmd.rs2 === 0.U, "MxquantBall reserved fields must be zero") - assert( - cmd.rs1(9, 0) < b.memDomain.virtualBankCount.U && cmd.rs1(29, 20) < b.memDomain.virtualBankCount.U, - "MxquantBall invalid bank" - ) - assert(cmd.op1_col === 1.U && cmd.wr_col === 1.U, "MxquantBall operands must occupy one bank") - assert(cmd.op1_bank =/= cmd.wr_bank, "MxquantBall banks must differ") - robId := io.cmdReq.bits.rob_id - subId := io.cmdReq.bits.sub_rob_id - isSub := io.cmdReq.bits.is_sub - source := cmd.op1_bank - destination := cmd.wr_bank - count := cmd.iter - group := 0.U - readRow := 0.U - maximum := 0.U - state := readReq - } - } - is(readReq)(when(read.io.req.fire)(state := readResp)) - is(readResp) { - when(read.io.resp.fire) { - val words = (0 until 4).map(i => read.io.resp.bits.data(32 * i + 31, 32 * i)) - for (i <- 0 until 4) { - assert(words(i)(30, 23) =/= 255.U, "MxquantBall input must be finite") - values(Cat(readRow, i.U(2.W))) := words(i) - } - maximum := (Seq(maximum) ++ words.map(_(30, 0))).reduce((a, c) => Mux(a > c, a, c)) - when(readRow === 7.U)(state := exponent).otherwise { readRow := readRow + 1.U; state := readReq } - } - } - is(exponent) { - val raw = maximum(30, 23).zext - 135.S(10.W) - val selected = Mux(maximum === 0.U, 0.S(10.W), Mux(raw < -127.S, -127.S(10.W), raw)) - blockExponent := selected - scale := (selected + 127.S).asUInt - lane := 0.U - state := encode - } - is(encode) { - codes(lane) := math.io.code - when(lane === 31.U) { codeRow := 0.U; state := codeReq }.otherwise(lane := lane + 1.U) - } - is(codeReq)(when(write.io.req.fire)(state := codeResp)) - is(codeResp) { - when(write.io.resp.fire) { - when(codeRow === 0.U) { codeRow := 1.U; state := codeReq }.otherwise(state := scaleReq) - } - } - is(scaleReq)(when(write.io.req.fire)(state := scaleResp)) - is(scaleResp) { - when(write.io.resp.fire) { - when(group === (count >> 5) - 1.U)(state := complete).otherwise { - group := group + 1.U - readRow := 0.U - maximum := 0.U - state := readReq - } - } - } - is(complete)(when(io.cmdResp.fire)(state := idle)) - } -} diff --git a/examples/balls/mxquant/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp b/examples/balls/mxquant/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp deleted file mode 100644 index 90aae51ad..000000000 --- a/examples/balls/mxquant/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp +++ /dev/null @@ -1,30 +0,0 @@ -#include "Buckyball/BuckyballOps.h" -#include "Conversion/LowerBuckyball/LowerBuckyball.h" -#include "mlir/IR/PatternMatch.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace mlir::buddy { -void populateMxquantBallAssignPhysicalBankPatterns(RewritePatternSet &, - PhysicalBankState &); -} - -namespace { -class BankMxquantPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(BankMxquantOp op, - PatternRewriter &rewriter) const override { - rewriter.create(op.getLoc(), op.getInBank(), op.getOutBank(), - op.getIter()); - rewriter.replaceOp(op, op.getOutBank()); - return success(); - } -}; -} // namespace - -void mlir::buddy::populateMxquantBallAssignPhysicalBankPatterns( - RewritePatternSet &patterns, PhysicalBankState &) { - patterns.add(patterns.getContext()); -} diff --git a/examples/balls/mxquant/compiler/src/Dialect/Buckyball/MxquantBall.td b/examples/balls/mxquant/compiler/src/Dialect/Buckyball/MxquantBall.td deleted file mode 100644 index 6e9cff07b..000000000 --- a/examples/balls/mxquant/compiler/src/Dialect/Buckyball/MxquantBall.td +++ /dev/null @@ -1,21 +0,0 @@ -#ifndef MXQUANTBALL_OPS -#define MXQUANTBALL_OPS - -include "Buckyball.td" - -def MxquantOp : Buckyball_Op<"mxquant"> { - let summary = "Finite FP32 to block32 MXFP8 quantization"; - let arguments = (ins I64:$inputBankId, I64:$outputBankId, I64:$iter); - let assemblyFormat = "$inputBankId `,` $outputBankId `,` $iter " - "prop-dict attr-dict `:` type($inputBankId)"; -} - -def BankMxquantOp : Buckyball_Op<"bank_mxquant", [MemoryEffects<[MemRead, MemWrite]>]> { - let summary = "FP32 to block32 MXFP8 quantization with virtual banks"; - let arguments = (ins I64:$inBank, I64:$outBank, I64:$iter); - let results = (outs I64:$outBankOut); - let assemblyFormat = "$inBank $outBank $iter prop-dict attr-dict `:` " - "type($inBank) type($outBank) type($iter)"; -} - -#endif diff --git a/examples/balls/mxquant/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/balls/mxquant/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 473dea024..000000000 --- a/examples/balls/mxquant/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,56 +0,0 @@ -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/Dialect/Arith/IR/Arith.h" - -using namespace mlir; -using namespace buddy::buckyball; -using namespace buddy::buckyball::legalize; - -namespace { -struct MxquantLowering : public ConvertOpToLLVMPattern { - MxquantLowering(LLVMTypeConverter &converter, int64_t bankDepth) - : ConvertOpToLLVMPattern(converter), bankDepth(bankDepth) {} - - LogicalResult - matchAndRewrite(MxquantOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("MxquantBall"); - if (buckyball_target::getBuckyballTarget().bankWidthBits != 128) - return op.emitError("mxquant: requires 16-byte bank rows"); - if (auto constant = op.getIter().getDefiningOp()) { - int64_t count = cast(constant.getValue()).getInt(); - if (count <= 0 || count % 32) - return op.emitError("mxquant: count must be a positive multiple of 32"); - if (count > bankDepth * 4 || count + count / 32 > bankDepth * 16) - return op.emitError("mxquant: bank footprint exceeds capacity"); - } - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInputBankId(), - cstI64(rewriter, loc, 0), - adaptor.getOutputBankId(), adaptor.getIter()); - rewriter.replaceOpWithNewOp( - op, rs1, cstI64(rewriter, loc, 0), - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("MXQUANT"))); - return success(); - } - -private: - int64_t bankDepth; -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateMxquantBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool, - int64_t bankDepth) { - patterns.add(converter, bankDepth); -} -void configureMxquantBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool) { - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/examples/balls/mxquant/configs/default.toml b/examples/balls/mxquant/configs/default.toml deleted file mode 100644 index c61b39cb6..000000000 --- a/examples/balls/mxquant/configs/default.toml +++ /dev/null @@ -1 +0,0 @@ -[ball] diff --git a/examples/balls/mxquant/emu/src/dispatch.rs b/examples/balls/mxquant/emu/src/dispatch.rs deleted file mode 100644 index afa7b38aa..000000000 --- a/examples/balls/mxquant/emu/src/dispatch.rs +++ /dev/null @@ -1,30 +0,0 @@ -use super::quant; -use crate::inst::instruction::ExecContext; - -const BALL_CLASS: &str = "examples.balls.mxquant.MxquantBall"; - -pub fn execute_known( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, - ctx: &mut ExecContext, -) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("MXQUANT") => Some(quant::execute(xs1, xs2, ctx)), - _ => None, - } -} - -pub fn cycles_after_issue(ball_class: &str, funct: u32, _xs1: u64, _xs2: u64) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("MXQUANT") => Some(1), - _ => None, - } -} diff --git a/examples/balls/mxquant/emu/src/lib.rs b/examples/balls/mxquant/emu/src/lib.rs deleted file mode 100644 index 647db3f0c..000000000 --- a/examples/balls/mxquant/emu/src/lib.rs +++ /dev/null @@ -1,6 +0,0 @@ -pub(crate) use crate::inst::{decode, instruction}; - -mod dispatch; -mod quant; - -pub use dispatch::{cycles_after_issue, execute_known}; diff --git a/examples/balls/mxquant/emu/src/quant.rs b/examples/balls/mxquant/emu/src/quant.rs deleted file mode 100644 index a657a420b..000000000 --- a/examples/balls/mxquant/emu/src/quant.rs +++ /dev/null @@ -1,91 +0,0 @@ -use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; -use super::instruction::ExecContext; -use crate::bank::{bank_lines, bank_row_bytes}; - -fn rounded_shift(value: u32, shift: u32) -> u32 { - if shift > 31 { - return 0; - } - let half = 1 << (shift - 1); - (value + half - 1 + ((value >> shift) & 1)) >> shift -} - -fn encode(bits: u32, block_exponent: i32) -> u8 { - let sign = ((bits >> 24) & 128) as u8; - if bits & 0x7fffffff == 0 { - return sign; - } - let mut exponent = ((bits >> 23) & 255) as i32; - let mut fraction = bits & 0x7fffff; - if exponent == 0 { - let shift = fraction.leading_zeros() - 8; - fraction = (fraction << shift) & 0x7fffff; - exponent = 1 - shift as i32; - } - exponent -= block_exponent + 120; - let code = if exponent <= 0 { - rounded_shift(fraction | 0x800000, (21 - exponent) as u32) - } else { - exponent as u32 * 8 + rounded_shift(fraction, 20) - }; - sign | code.min(126) as u8 -} - -pub(crate) fn execute(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { - let input = rs1_b0(xs1); - let output = rs1_b2(xs1); - let count = rs1_iter(xs1) as usize; - assert!( - xs2 == 0 && rs1_b1(xs1) == 0, - "mxquant: reserved fields must be zero" - ); - assert!( - count > 0 && count % 32 == 0, - "mxquant: count must be a positive multiple of 32" - ); - assert!( - input != output, - "mxquant: input and output banks must differ" - ); - for bank in [input, output] { - assert!( - ctx.config(bank).allocated && ctx.config(bank).cols == 1, - "mxquant: banks must be allocated with one column" - ); - } - let bytes = bank_lines() * bank_row_bytes(); - assert!( - bank_row_bytes() == 16 && count <= bytes / 4 && count + count / 32 <= bytes, - "mxquant: bank footprint exceeds capacity" - ); - let source = &ctx.banks[pbank(ctx, input)][..count * 4]; - assert!( - source - .chunks_exact(4) - .all( - |element| u32::from_le_bytes(element.try_into().unwrap()) & 0x7fffffff < 0x7f800000 - ), - "mxquant: non-finite activation" - ); - let mut packed = vec![0u8; count + count / 32]; - for (block, values) in source.chunks_exact(128).enumerate() { - let maximum = values - .chunks_exact(4) - .map(|element| u32::from_le_bytes(element.try_into().unwrap()) & 0x7fffffff) - .max() - .unwrap(); - let exponent = if maximum == 0 { - 0 - } else { - ((maximum >> 23) as i32 - 135).max(-127) - }; - packed[count + block] = (exponent + 127) as u8; - for (index, element) in values.chunks_exact(4).enumerate() { - packed[block * 32 + index] = - encode(u32::from_le_bytes(element.try_into().unwrap()), exponent); - } - } - let destination = pbank(ctx, output); - ctx.banks[destination][..packed.len()].copy_from_slice(&packed); - 0 -} diff --git a/examples/balls/mxquant/workloads/ctests/CMakeLists.txt b/examples/balls/mxquant/workloads/ctests/CMakeLists.txt deleted file mode 100644 index b4967e1cd..000000000 --- a/examples/balls/mxquant/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -file(READ "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/params.h" MXQUANT_PARAMS) -string(REGEX MATCH "#define BANK_LINES ([0-9]+)" _ "${MXQUANT_PARAMS}") -set(MXQUANT_BANK_LINES ${CMAKE_MATCH_1}) -string(REGEX MATCH "#define BANK_WIDTH ([0-9]+)" _ "${MXQUANT_PARAMS}") -math(EXPR MXQUANT_BANK_BYTES "${MXQUANT_BANK_LINES} * ${CMAKE_MATCH_1} / 8") -math(EXPR MXQUANT_BANK_TAIL "${MXQUANT_BANK_BYTES} % 128") -if(MXQUANT_BANK_BYTES LESS 3840 OR MXQUANT_BANK_TAIL) - message(FATAL_ERROR - "Mxquant CTests require banks >= 3840 bytes and a 32-element FP32 capacity; actual ${MXQUANT_BANK_BYTES} bytes") -endif() -add_buckyball_ctests(mxquant_values_test.c mxquant_capacity_test.c) -buckyball_ctest_name(TEST_NAME ${BUCKYBALL_CTEST_TARGET} mxquant_values_test) -add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${TEST_NAME}-baremetal APPEND - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/../reference.h) diff --git a/examples/balls/mxquant/workloads/mlir_tests/CMakeLists.txt b/examples/balls/mxquant/workloads/mlir_tests/CMakeLists.txt deleted file mode 100644 index 9d0ed5a15..000000000 --- a/examples/balls/mxquant/workloads/mlir_tests/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) - set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) -endif() -add_subdirectory(ball) -add_subdirectory(bank) diff --git a/examples/balls/mxquant/workloads/mlir_tests/ball/CMakeLists.txt b/examples/balls/mxquant/workloads/mlir_tests/ball/CMakeLists.txt deleted file mode 100644 index d94a22636..000000000 --- a/examples/balls/mxquant/workloads/mlir_tests/ball/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -set(BUCKYBALL_MLIR_TEST_PREFIX ball) -add_buckyball_mlir_test(mxquant TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) -buckyball_mlirtest_name(TEST_NAME ${BUCKYBALL_MLIR_ACTIVE_TARGET} ball_mxquant) -foreach(KIND baremetal linux) - add_custom_command(OUTPUT ${TEST_NAME}-${KIND} APPEND - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/../../reference.h) -endforeach() diff --git a/examples/balls/mxquant/workloads/mlir_tests/ball/mxquant.mlir b/examples/balls/mxquant/workloads/mlir_tests/ball/mxquant.mlir deleted file mode 100644 index 6b9a62331..000000000 --- a/examples/balls/mxquant/workloads/mlir_tests/ball/mxquant.mlir +++ /dev/null @@ -1,20 +0,0 @@ -// RUN: buddy-opt %s --target=toy -assign-physical-banks -lower-bank-ssa-to-intrinsics -lower-buckyball -canonicalize | FileCheck %s -// CHECK-LABEL: func.func @quant_block_32 -// CHECK: %[[ZERO:[^ ]+]] = arith.constant 0 : i64 -// CHECK: %[[PACKED:[^ ]+]] = arith.constant 34363932675 : i64 -// CHECK: buckyball.intr.custom %[[PACKED]], %[[ZERO]] -// CHECK-LABEL: func.func @quant_dynamic -// CHECK: buckyball.intr.custom {{.*}} -func.func @quant_block_32() attributes {llvm.emit_c_interface} { - %input = arith.constant 3 : i64 - %output = arith.constant 4 : i64 - %count = arith.constant 32 : i64 - buckyball.mxquant %input, %output, %count : i64 - return -} -func.func @quant_dynamic(%count: i64) attributes {llvm.emit_c_interface} { - %input = arith.constant 3 : i64 - %output = arith.constant 4 : i64 - buckyball.mxquant %input, %output, %count : i64 - return -} diff --git a/examples/balls/mxquant/workloads/mlir_tests/ball/mxquant_main.cpp b/examples/balls/mxquant/workloads/mlir_tests/ball/mxquant_main.cpp deleted file mode 100644 index d11774f36..000000000 --- a/examples/balls/mxquant/workloads/mlir_tests/ball/mxquant_main.cpp +++ /dev/null @@ -1,86 +0,0 @@ -#include "buckyball.h" -#include -#include -#include -#include -#include -extern "C" void _mlir_ciface_quant_block_32(); -extern "C" void _mlir_ciface_quant_dynamic(int64_t count); -#include "../../reference.h" -static uint32_t input[32] __attribute__((aligned(64))); -static uint8_t actual[64] __attribute__((aligned(64))); -static uint8_t expected[64]; -int main(void) { - const uint32_t edges[] = {0, 0x80000000, 1, 0x80000001, - 0x007fffff, 0x00800000, 0x3f880000, 0x3f980000, - 0x43e00000, 0x7f7fffff, 0xff7fffff}; - const uint32_t subnormals[] = {0, 1, 0x1000, 0x2000, - 0x3000, 0x7fffff, 0x80000001, 0x807fffff}; - uint32_t random = 0x6d2b79f5; - for (int test = 0; test < 68; ++test) { - for (int i = 0; i < 32; ++i) { - random ^= random << 13; - random ^= random >> 17; - random ^= random << 5; - input[i] = - test == 0 ? (i & 1 ? 0x80000000u : 0) - : test == 1 ? edges[i % 11] - : test == 2 ? (i ? (i & 1 ? 0x3f880000u : 0x3f980000u) : 0x43e00000u) - : test == 3 ? subnormals[i % 8] - : ((random & 0x807fffffu) | - ((((test - 4) * 4 + (random >> 23 & 3)) % 255) << 23)); - } - memset(actual, 0x5a, sizeof(actual)); - memset(expected, 0x5a, sizeof(expected)); - mxquant_reference32(input, expected); - if (test == 0 && - (expected[0] != 0 || expected[1] != 128 || expected[32] != 127)) - return 1; - if (test == 1 && - (expected[9] != 126 || expected[10] != 254 || expected[32] != 246)) - return 2; - if (test == 2 && - (expected[0] != 126 || expected[1] != 56 || expected[2] != 58)) - return 2; - uint32_t state = 0x1fu | (test % 5) << 5, after; - asm volatile("csrw fcsr, %0" : : "r"(state)); - bb_mem_alloc(3, 1, 1); - bb_mem_alloc(4, 1, 1); - bb_mvin((uintptr_t)input, 3, 8, 1); - bb_mvin((uintptr_t)actual, 4, 4, 1); - _mlir_ciface_quant_block_32(); - bb_mvout((uintptr_t)actual, 4, 4, 1); - asm volatile("csrr %0, fcsr" : "=r"(after)); - if (after != state || memcmp(actual, expected, sizeof(actual))) - return 3; - bb_mem_release(3); - bb_mem_release(4); - } - enum { BANK_BYTES = BANK_LINES * BANK_WIDTH / 8, MAX_COUNT = BANK_BYTES / 4 }; - static uint32_t capacity_input[MAX_COUNT] __attribute__((aligned(64))); - static uint8_t capacity_actual[BANK_BYTES] __attribute__((aligned(64))); - const int counts[] = {32, 480, 960, MAX_COUNT}; - for (int test = 0; test < 4; ++test) { - int count = counts[test]; - for (int i = 0; i < MAX_COUNT; ++i) - capacity_input[i] = (i & 1) ? 0xbf800000u : 0x3f800000u; - memset(capacity_actual, 0x5a, sizeof(capacity_actual)); - bb_mem_alloc(3, 1, 1); - bb_mem_alloc(4, 1, 1); - bb_mvin((uintptr_t)capacity_input, 3, BANK_LINES, 1); - bb_mvin((uintptr_t)capacity_actual, 4, BANK_LINES, 1); - _mlir_ciface_quant_dynamic(count); - bb_mvout((uintptr_t)capacity_actual, 4, BANK_LINES, 1); - for (int i = 0; i < BANK_BYTES; ++i) { - uint8_t expected = i < count ? ((i & 1) ? 0xf8 : 0x78) - : i < count + count / 32 ? 119 - : 0x5a; - if (capacity_actual[i] != expected) - return 1; - } - bb_mem_release(3); - bb_mem_release(4); - } - puts("mxquant ball MLIR finite/RNE/signed-zero/capacity/guard PASS"); - return 0; -} diff --git a/examples/balls/mxquant/workloads/mlir_tests/bank/CMakeLists.txt b/examples/balls/mxquant/workloads/mlir_tests/bank/CMakeLists.txt deleted file mode 100644 index b486e6c26..000000000 --- a/examples/balls/mxquant/workloads/mlir_tests/bank/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -set(BUCKYBALL_MLIR_TEST_PREFIX bank) -add_buckyball_mlir_test(mxquant TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) -buckyball_mlirtest_name(TEST_NAME ${BUCKYBALL_MLIR_ACTIVE_TARGET} bank_mxquant) -foreach(KIND baremetal linux) - add_custom_command(OUTPUT ${TEST_NAME}-${KIND} APPEND - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/../../reference.h) -endforeach() diff --git a/examples/balls/mxquant/workloads/mlir_tests/bank/mxquant.mlir b/examples/balls/mxquant/workloads/mlir_tests/bank/mxquant.mlir deleted file mode 100644 index da2203912..000000000 --- a/examples/balls/mxquant/workloads/mlir_tests/bank/mxquant.mlir +++ /dev/null @@ -1,25 +0,0 @@ -// RUN: buddy-opt %s --target=toy -assign-physical-banks | FileCheck %s --check-prefix=BANK -// BANK-LABEL: func.func @quant_block_32 -// BANK: buckyball.mxquant -// BANK-LABEL: func.func @quant_dynamic -// BANK: buckyball.mxquant -// RUN: buddy-opt %s --target=toy -assign-physical-banks -lower-bank-ssa-to-intrinsics -lower-buckyball -canonicalize | FileCheck %s -// CHECK-LABEL: func.func @quant_block_32 -// CHECK: %[[ZERO:[^ ]+]] = arith.constant 0 : i64 -// CHECK: %[[PACKED:[^ ]+]] = arith.constant 34363932675 : i64 -// CHECK: buckyball.intr.custom %[[PACKED]], %[[ZERO]] -// CHECK-LABEL: func.func @quant_dynamic -// CHECK: buckyball.intr.custom {{.*}} -func.func @quant_block_32() attributes {llvm.emit_c_interface} { - %input = arith.constant 3 : i64 - %output = arith.constant 4 : i64 - %count = arith.constant 32 : i64 - %result = buckyball.bank_mxquant %input %output %count : i64 i64 i64 - return -} -func.func @quant_dynamic(%count: i64) attributes {llvm.emit_c_interface} { - %input = arith.constant 3 : i64 - %output = arith.constant 4 : i64 - %result = buckyball.bank_mxquant %input %output %count : i64 i64 i64 - return -} diff --git a/examples/balls/mxquant/workloads/mlir_tests/bank/mxquant_main.cpp b/examples/balls/mxquant/workloads/mlir_tests/bank/mxquant_main.cpp deleted file mode 100644 index 8f98f53be..000000000 --- a/examples/balls/mxquant/workloads/mlir_tests/bank/mxquant_main.cpp +++ /dev/null @@ -1,86 +0,0 @@ -#include "buckyball.h" -#include -#include -#include -#include -#include -extern "C" void _mlir_ciface_quant_block_32(); -extern "C" void _mlir_ciface_quant_dynamic(int64_t count); -#include "../../reference.h" -static uint32_t input[32] __attribute__((aligned(64))); -static uint8_t actual[64] __attribute__((aligned(64))); -static uint8_t expected[64]; -int main(void) { - const uint32_t edges[] = {0, 0x80000000, 1, 0x80000001, - 0x007fffff, 0x00800000, 0x3f880000, 0x3f980000, - 0x43e00000, 0x7f7fffff, 0xff7fffff}; - const uint32_t subnormals[] = {0, 1, 0x1000, 0x2000, - 0x3000, 0x7fffff, 0x80000001, 0x807fffff}; - uint32_t random = 0x6d2b79f5; - for (int test = 0; test < 68; ++test) { - for (int i = 0; i < 32; ++i) { - random ^= random << 13; - random ^= random >> 17; - random ^= random << 5; - input[i] = - test == 0 ? (i & 1 ? 0x80000000u : 0) - : test == 1 ? edges[i % 11] - : test == 2 ? (i ? (i & 1 ? 0x3f880000u : 0x3f980000u) : 0x43e00000u) - : test == 3 ? subnormals[i % 8] - : ((random & 0x807fffffu) | - ((((test - 4) * 4 + (random >> 23 & 3)) % 255) << 23)); - } - memset(actual, 0x5a, sizeof(actual)); - memset(expected, 0x5a, sizeof(expected)); - mxquant_reference32(input, expected); - if (test == 0 && - (expected[0] != 0 || expected[1] != 128 || expected[32] != 127)) - return 1; - if (test == 1 && - (expected[9] != 126 || expected[10] != 254 || expected[32] != 246)) - return 2; - if (test == 2 && - (expected[0] != 126 || expected[1] != 56 || expected[2] != 58)) - return 2; - uint32_t state = 0x1fu | (test % 5) << 5, after; - asm volatile("csrw fcsr, %0" : : "r"(state)); - bb_mem_alloc(3, 1, 1); - bb_mem_alloc(4, 1, 1); - bb_mvin((uintptr_t)input, 3, 8, 1); - bb_mvin((uintptr_t)actual, 4, 4, 1); - _mlir_ciface_quant_block_32(); - bb_mvout((uintptr_t)actual, 4, 4, 1); - asm volatile("csrr %0, fcsr" : "=r"(after)); - if (after != state || memcmp(actual, expected, sizeof(actual))) - return 3; - bb_mem_release(3); - bb_mem_release(4); - } - enum { BANK_BYTES = BANK_LINES * BANK_WIDTH / 8, MAX_COUNT = BANK_BYTES / 4 }; - static uint32_t capacity_input[MAX_COUNT] __attribute__((aligned(64))); - static uint8_t capacity_actual[BANK_BYTES] __attribute__((aligned(64))); - const int counts[] = {32, 480, 960, MAX_COUNT}; - for (int test = 0; test < 4; ++test) { - int count = counts[test]; - for (int i = 0; i < MAX_COUNT; ++i) - capacity_input[i] = (i & 1) ? 0xbf800000u : 0x3f800000u; - memset(capacity_actual, 0x5a, sizeof(capacity_actual)); - bb_mem_alloc(3, 1, 1); - bb_mem_alloc(4, 1, 1); - bb_mvin((uintptr_t)capacity_input, 3, BANK_LINES, 1); - bb_mvin((uintptr_t)capacity_actual, 4, BANK_LINES, 1); - _mlir_ciface_quant_dynamic(count); - bb_mvout((uintptr_t)capacity_actual, 4, BANK_LINES, 1); - for (int i = 0; i < BANK_BYTES; ++i) { - uint8_t expected = i < count ? ((i & 1) ? 0xf8 : 0x78) - : i < count + count / 32 ? 119 - : 0x5a; - if (capacity_actual[i] != expected) - return 1; - } - bb_mem_release(3); - bb_mem_release(4); - } - puts("mxquant bank MLIR finite/RNE/signed-zero/capacity/guard PASS"); - return 0; -} diff --git a/examples/balls/relu/README.md b/examples/balls/relu/README.md deleted file mode 100644 index f4b81faa7..000000000 --- a/examples/balls/relu/README.md +++ /dev/null @@ -1,26 +0,0 @@ -# ReluBall - -`ReluBall` computes signed INT32 `max(x, 0)` in place in one physical pbank. -It has no internal SRAM: a 128-bit register holds one pbank line between its -read and write requests. - -## Command - -`RELU(vbank, group, iter, stride)` selects physical pbank `group` in virtual -bank `vbank`. `group` is a pbank number, never a data-layout term. - -`iter` is the number of 128-bit physical lines processed in each segment. -`stride` is the distance between segments. Both are positive; `iter` is a -multiple of 16, `iter <= stride`, and `stride` divides the pbank depth. The -command processes `iter` lines at every `stride` boundary until it reaches the -end of the selected pbank. - -For every line the Ball performs one single-port sequence: - -```text -pbank read -> 4 signed int32 ReLUs -> pbank write -``` - -The selected group must be allocated. There is no output vbank, no fallback -layout, and no tail path. Tile lowering pads logical matrices to 16-aligned -shapes; compiler packing fills the final pbank chunk before issuing ReLU. diff --git a/examples/balls/relu/arch/src/main/scala/Relu.scala b/examples/balls/relu/arch/src/main/scala/Relu.scala deleted file mode 100644 index 4cfbf250c..000000000 --- a/examples/balls/relu/arch/src/main/scala/Relu.scala +++ /dev/null @@ -1,164 +0,0 @@ -package examples.balls.relu - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public} -import chisel3.util._ -import framework.balldomain.blink.{BallStatus, BankRead, BankWrite} -import framework.balldomain.rs.{BallRsComplete, BallRsIssue} -import framework.top.GlobalConfig - -@instantiable -class Relu(val b: GlobalConfig) extends Module { - private val addressWidth = log2Up(b.memDomain.bankEntries) - private val countWidth = log2Up(b.memDomain.bankEntries + 1) - private val bankIdWidth = b.memDomain.vbankIdWidth - private val groupWidth = log2Up(b.memDomain.bankNum) - - private val mapping = b.ballDomain.ballIdMappings - .find(_.ballName == "ReluBall") - .getOrElse(throw new IllegalArgumentException("ReluBall not found in config")) - - private val funct = b.ballDomain.ballISA - .find(_.mnemonic == "RELU") - .map(_.funct7) - .getOrElse(throw new IllegalArgumentException("RELU not found in ballISA")) - - require(mapping.inBW == 1, "ReluBall requires one SRAM read port") - require(mapping.outBW == 1, "ReluBall requires one SRAM write port") - require(b.memDomain.bankWidth == 128, "ReluBall requires 128-bit SRAM rows") - require(b.memDomain.bankMaskLen == 16, "ReluBall requires sixteen byte enables") - - @public - val io = IO(new Bundle { - val cmdReq = Flipped(Decoupled(new BallRsIssue(b))) - val cmdResp = Decoupled(new BallRsComplete(b)) - val channelReady = Input(Bool()) - val bankRead = Vec(1, Flipped(new BankRead(b))) - val bankWrite = Vec(1, Flipped(new BankWrite(b))) - val status = new BallStatus - }) - - val Seq(idle, waitForChannels, readRequest, readResponse, writeRequest, writeResponse, complete) = Enum(7) - val state = RegInit(idle) - - val robId = RegInit(0.U(log2Up(b.frontend.rob_entries).W)) - val isSub = RegInit(false.B) - val subRobId = RegInit(0.U(log2Up(b.frontend.sub_rob_depth * 4).W)) - val bank = RegInit(0.U(bankIdWidth.W)) - val group = RegInit(0.U(groupWidth.W)) - val iter = RegInit(0.U(countWidth.W)) - val stride = RegInit(0.U(countWidth.W)) - val segment = RegInit(0.U(addressWidth.W)) - val line = RegInit(0.U(addressWidth.W)) - val word = Reg(UInt(128.W)) - - val relu = Wire(Vec(4, UInt(32.W))) - for (lane <- 0 until 4) { - val value = word(32 * lane + 31, 32 * lane).asSInt - relu(lane) := Mux(value < 0.S, 0.U, value.asUInt) - } - val address = segment + line - - io.cmdReq.ready := state === idle - io.cmdResp.valid := state === complete - io.cmdResp.bits.rob_id := robId - io.cmdResp.bits.is_sub := isSub - io.cmdResp.bits.sub_rob_id := subRobId - - io.bankRead(0).rob_id := robId - io.bankRead(0).ball_id := 0.U - io.bankRead(0).bank_id := bank - io.bankRead(0).group_id := group - io.bankRead(0).io.req.valid := false.B - io.bankRead(0).io.req.bits.addr := address - io.bankRead(0).io.resp.ready := false.B - - io.bankWrite(0).rob_id := robId - io.bankWrite(0).ball_id := 0.U - io.bankWrite(0).bank_id := bank - io.bankWrite(0).group_id := group - io.bankWrite(0).io.req.valid := false.B - io.bankWrite(0).io.req.bits.addr := address - io.bankWrite(0).io.req.bits.data := Cat(relu.reverse) - io.bankWrite(0).io.req.bits.mask := VecInit(Seq.fill(16)(true.B)) - io.bankWrite(0).io.resp.ready := false.B - - switch(state) { - is(idle) { - when(io.cmdReq.fire) { - val command = io.cmdReq.bits.cmd - assert(command.funct7 === funct.U(7.W), "ReluBall funct7 must be RELU") - assert(command.wr_bank === 0.U, "ReluBall bank2 must be zero") - assert(command.op1_col =/= 0.U && command.op2_bank < command.op1_col, "ReluBall group is not allocated in bank") - assert(command.iter =/= 0.U && command.iter(3, 0) === 0.U, "ReluBall iter must be a positive multiple of 16") - assert( - command.special =/= 0.U && command.special <= b.memDomain.bankEntries.U, - "ReluBall stride must fit in one physical bank" - ) - assert( - b.memDomain.bankEntries.U % command.special === 0.U, - "ReluBall stride must divide the physical bank depth" - ) - assert(command.iter <= command.special, "ReluBall iter must not exceed stride") - - robId := io.cmdReq.bits.rob_id - isSub := io.cmdReq.bits.is_sub - subRobId := io.cmdReq.bits.sub_rob_id - bank := command.op1_bank - group := command.op2_bank - iter := command.iter - stride := command.special - segment := 0.U - line := 0.U - state := waitForChannels - } - } - - is(waitForChannels) { - when(io.channelReady)(state := readRequest) - } - - is(readRequest) { - io.bankRead(0).io.req.valid := true.B - when(io.bankRead(0).io.req.fire)(state := readResponse) - } - - is(readResponse) { - io.bankRead(0).io.resp.ready := true.B - when(io.bankRead(0).io.resp.fire) { - word := io.bankRead(0).io.resp.bits.data - state := writeRequest - } - } - - is(writeRequest) { - io.bankWrite(0).io.req.valid := true.B - when(io.bankWrite(0).io.req.fire)(state := writeResponse) - } - - is(writeResponse) { - io.bankWrite(0).io.resp.ready := true.B - when(io.bankWrite(0).io.resp.fire) { - when(line +& 1.U === iter) { - when(segment +& stride === b.memDomain.bankEntries.U) { - state := complete - }.otherwise { - segment := segment + stride - line := 0.U - state := readRequest - } - }.otherwise { - line := line + 1.U - state := readRequest - } - } - } - - is(complete) { - when(io.cmdResp.fire)(state := idle) - } - } - - io.status.idle := state === idle - io.status.running := state =/= idle -} diff --git a/examples/balls/relu/arch/src/main/scala/ReluBall.scala b/examples/balls/relu/arch/src/main/scala/ReluBall.scala deleted file mode 100644 index 91024b258..000000000 --- a/examples/balls/relu/arch/src/main/scala/ReluBall.scala +++ /dev/null @@ -1,33 +0,0 @@ -package examples.balls.relu - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.balldomain.blink.{BlinkIO, HasBallStatus, HasBlink, SubRobRow} -import framework.balldomain.blink.mmio.MmioRead -import framework.top.GlobalConfig - -@instantiable -class ReluBall(val b: GlobalConfig) extends Module with HasBlink with HasBallStatus { - - private val mapping = b.ballDomain.ballIdMappings - .find(_.ballName == "ReluBall") - .getOrElse(throw new IllegalArgumentException("ReluBall not found in config")) - - @public - val io = IO(new BlinkIO(b, mapping.inBW, mapping.outBW)) - - def blink: BlinkIO = io - def status = io.status - dontTouch(io) - - val unit: Instance[Relu] = Instantiate(new Relu(b)) - unit.io.cmdReq <> io.cmdReq - unit.io.cmdResp <> io.cmdResp - unit.io.channelReady := io.channelReady - unit.io.bankRead(0) <> io.bankRead(0) - unit.io.bankWrite(0) <> io.bankWrite(0) - io.status <> unit.io.status - io.subRobReq.valid := false.B - io.subRobReq.bits := SubRobRow.tieOff(b) - MmioRead.tieOff(io.mmioRead) -} diff --git a/examples/balls/relu/compiler/src/Backend/include/IntrinsicsRISCVReluBall.td b/examples/balls/relu/compiler/src/Backend/include/IntrinsicsRISCVReluBall.td deleted file mode 100644 index 803eccdf1..000000000 --- a/examples/balls/relu/compiler/src/Backend/include/IntrinsicsRISCVReluBall.td +++ /dev/null @@ -1,4 +0,0 @@ -//===- IntrinsicsRISCVReluBall.td - ReluBall intrinsics ------------------===// - -let TargetPrefix = "riscv" in -def int_riscv_bb_relu : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; diff --git a/examples/balls/relu/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp b/examples/balls/relu/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp deleted file mode 100644 index 8ecc276ff..000000000 --- a/examples/balls/relu/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp +++ /dev/null @@ -1,34 +0,0 @@ -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace mlir::buddy { -void populateReluBallAssignPhysicalBankPatterns(RewritePatternSet &patterns, - PhysicalBankState &state); -} - -namespace { -class BankReluPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(BankReluOp op, - PatternRewriter &rewriter) const override { - rewriter.create(op.getLoc(), op.getBank(), op.getGroup(), - op.getIter(), op.getStride()); - rewriter.replaceOp(op, op.getBank()); - return success(); - } -}; -} // namespace - -void mlir::buddy::populateReluBallAssignPhysicalBankPatterns( - RewritePatternSet &patterns, mlir::buddy::PhysicalBankState &state) { - (void)state; - patterns.add(patterns.getContext()); -} diff --git a/examples/balls/relu/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp b/examples/balls/relu/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp deleted file mode 100644 index c88dde897..000000000 --- a/examples/balls/relu/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp +++ /dev/null @@ -1,112 +0,0 @@ -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Target/BuckyballTargetRegistry.h" -#include "Utils/BankUtils.h" - -#include - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { - -constexpr int64_t kValuesPerLine = 4; - -class ReluMatrixToBankSSAPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(ReluMatrixOp op, - PatternRewriter &b) const override { - auto inputType = dyn_cast(op.getInput().getType()); - auto outputType = dyn_cast(op.getOutput().getType()); - if (!inputType || !outputType || !inputType.hasStaticShape() || - inputType != outputType || !inputType.getElementType().isInteger(32)) - return op.emitError("requires matching static memref"); - - int64_t rows = inputType.getShape()[0]; - int64_t columns = inputType.getShape()[1]; - if (rows <= 0 || columns <= 0 || rows % 16 || columns % 16) - return op.emitError("requires positive 16-aligned dimensions"); - - const auto &target = buckyball_target::getBuckyballTarget(); - if (target.bankWidthBits != 128 || target.bankDepth <= 0 || - target.bankDepth % 16) - return op.emitError( - "ReluBall requires a 128-bit bank with 16-aligned depth"); - - Location loc = op.getLoc(); - int64_t lines = target.bankDepth; - int64_t values = lines * kValuesPerLine; - int64_t total = rows * columns; - Value bank = allocBank(b, loc, 1, 1); - Value zero = b.create(loc, 0); - Value one = b.create(loc, 1); - Value four = b.create(loc, kValuesPerLine); - Value columnsValue = b.create(loc, columns); - auto packType = MemRefType::get({lines, kValuesPerLine}, b.getI32Type()); - - for (int64_t base = 0; base < total; base += values) { - int64_t count = std::min(values, total - base); - Value inputPack = b.create(loc, packType); - Value outputPack = b.create(loc, packType); - Value zeroValue = b.create(loc, 0, 32); - b.create(loc, zeroValue, inputPack); - - Value countValue = b.create(loc, count); - Value baseValue = b.create(loc, base); - auto copyIn = b.create(loc, zero, countValue, one); - b.setInsertionPointToStart(copyIn.getBody()); - Value index = copyIn.getInductionVar(); - Value source = b.create(loc, baseValue, index); - Value row = b.create(loc, source, columnsValue); - Value column = b.create(loc, source, columnsValue); - Value line = b.create(loc, index, four); - Value lane = b.create(loc, index, four); - Value value = - b.create(loc, op.getInput(), ValueRange{row, column}); - b.create(loc, value, inputPack, ValueRange{line, lane}); - b.setInsertionPointAfter(copyIn); - - Value loaded = mvinBank(b, loc, inputPack, bank, lines); - Value result = b.create( - loc, bank.getType(), loaded, createI64Const(b, loc, 0), - createI64Const(b, loc, lines), createI64Const(b, loc, lines)); - bank = mvoutBank(b, loc, outputPack, result, lines); - - auto copyOut = b.create(loc, zero, countValue, one); - b.setInsertionPointToStart(copyOut.getBody()); - index = copyOut.getInductionVar(); - source = b.create(loc, baseValue, index); - row = b.create(loc, source, columnsValue); - column = b.create(loc, source, columnsValue); - line = b.create(loc, index, four); - lane = b.create(loc, index, four); - value = b.create(loc, outputPack, ValueRange{line, lane}); - b.create(loc, value, op.getOutput(), - ValueRange{row, column}); - b.setInsertionPointAfter(copyOut); - b.create(loc, inputPack); - b.create(loc, outputPack); - } - - releaseBank(b, loc, bank); - b.eraseOp(op); - return success(); - } -}; - -} // namespace - -void mlir::buddy::populateReluBallLowerBuckyballToBankSSAPatterns( - RewritePatternSet &patterns) { - patterns.add(patterns.getContext()); -} diff --git a/examples/balls/relu/compiler/src/Conversion/LowerTileToBuckyball/TileReluPatterns.cpp b/examples/balls/relu/compiler/src/Conversion/LowerTileToBuckyball/TileReluPatterns.cpp deleted file mode 100644 index 9c49d9ec5..000000000 --- a/examples/balls/relu/compiler/src/Conversion/LowerTileToBuckyball/TileReluPatterns.cpp +++ /dev/null @@ -1,91 +0,0 @@ -#include "Buckyball/BuckyballOps.h" -#include "Tile/TileOps.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/PatternMatch.h" - -using namespace mlir; -using namespace ::buddy::buckyball; -namespace tile = ::buddy::tile; - -namespace { -class TileReluLowering : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(tile::TileReluOp op, - PatternRewriter &b) const override { - Value input = op.getInput(); - Value output = op.getOutput(); - auto inputType = dyn_cast(input.getType()); - auto outputType = dyn_cast(output.getType()); - if (!inputType || !outputType || !inputType.hasStaticShape() || - inputType != outputType || inputType.getRank() != 2 || - !inputType.getElementType().isInteger(32)) - return op.emitError("requires matching static memref"); - - int64_t rows = inputType.getShape()[0]; - int64_t columns = inputType.getShape()[1]; - if (rows <= 0 || columns <= 0) - return op.emitError("requires positive dimensions"); - - int64_t paddedRows = llvm::alignTo(rows, 16); - int64_t paddedColumns = llvm::alignTo(columns, 16); - Location loc = op.getLoc(); - if (paddedRows == rows && paddedColumns == columns) { - b.create(loc, input, output); - b.eraseOp(op); - return success(); - } - - auto paddedType = - MemRefType::get({paddedRows, paddedColumns}, b.getI32Type()); - Value paddedInput = b.create(loc, paddedType); - Value paddedOutput = b.create(loc, paddedType); - Value zeroValue = b.create(loc, 0, 32); - b.create(loc, zeroValue, paddedInput); - - Value zero = b.create(loc, 0); - Value one = b.create(loc, 1); - Value rowLimit = b.create(loc, rows); - Value columnLimit = b.create(loc, columns); - auto copyInputRows = b.create(loc, zero, rowLimit, one); - b.setInsertionPointToStart(copyInputRows.getBody()); - Value row = copyInputRows.getInductionVar(); - auto copyInputColumns = b.create(loc, zero, columnLimit, one); - b.setInsertionPointToStart(copyInputColumns.getBody()); - Value column = copyInputColumns.getInductionVar(); - Value value = b.create(loc, input, ValueRange{row, column}); - b.create(loc, value, paddedInput, ValueRange{row, column}); - b.setInsertionPointAfter(copyInputRows); - - b.create(loc, paddedInput, paddedOutput); - - auto copyOutputRows = b.create(loc, zero, rowLimit, one); - b.setInsertionPointToStart(copyOutputRows.getBody()); - row = copyOutputRows.getInductionVar(); - auto copyOutputColumns = b.create(loc, zero, columnLimit, one); - b.setInsertionPointToStart(copyOutputColumns.getBody()); - column = copyOutputColumns.getInductionVar(); - value = - b.create(loc, paddedOutput, ValueRange{row, column}); - b.create(loc, value, output, ValueRange{row, column}); - b.setInsertionPointAfter(copyOutputRows); - - b.create(loc, paddedInput); - b.create(loc, paddedOutput); - b.eraseOp(op); - return success(); - } -}; -} // namespace - -namespace mlir::buddy { -void populateReluBallTileLoweringPatterns(RewritePatternSet &patterns, int64_t, - int64_t, int64_t) { - patterns.add(patterns.getContext()); -} -} // namespace mlir::buddy diff --git a/examples/balls/relu/compiler/src/Dialect/Buckyball/ReluBall.td b/examples/balls/relu/compiler/src/Dialect/Buckyball/ReluBall.td deleted file mode 100644 index 70fafee38..000000000 --- a/examples/balls/relu/compiler/src/Dialect/Buckyball/ReluBall.td +++ /dev/null @@ -1,31 +0,0 @@ -//===-- ReluBall.td - ReluBall operations ------------------*- tablegen -*-===// - -#ifndef RELUBALL_OPS -#define RELUBALL_OPS - -include "Buckyball.td" - -def ReluMatrixOp : Buckyball_Op<"relu_matrix"> { - let summary = "Logical INT32 ReLU matrix tile"; - let arguments = (ins MemRefRankOf<[AnyType], [2]>:$input, - MemRefRankOf<[AnyType], [2]>:$output); - let assemblyFormat = [{$input $output attr-dict `:` - type($input) type($output)}]; -} - -def ReluOp : Buckyball_Op<"relu"> { - let summary = "In-place INT32 ReLU on one physical bank"; - let arguments = (ins I64:$bankId, I64:$group, I64:$iter, I64:$stride); - let assemblyFormat = "$bankId `,` $group `,` $iter `,` $stride attr-dict `:` " - "type($bankId)"; -} - -def BankReluOp : Buckyball_Op<"bank_relu"> { - let summary = "In-place INT32 ReLU with a virtual bank handle"; - let arguments = (ins I64:$bank, I64:$group, I64:$iter, I64:$stride); - let results = (outs I64:$bankOut); - let assemblyFormat = "$bank $group $iter $stride attr-dict `:` " - "type($bank) type($group) type($iter) type($stride)"; -} - -#endif // RELUBALL_OPS diff --git a/examples/balls/relu/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/balls/relu/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 36dec7d68..000000000 --- a/examples/balls/relu/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,45 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace buddy::buckyball; -using namespace buddy::buckyball::legalize; - -namespace { -struct ReluLowering : public ConvertOpToLLVMPattern { - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - - LogicalResult - matchAndRewrite(ReluOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("ReluBall"); - Location loc = op.getLoc(); - Value rs1 = - packRs1BanksIter(rewriter, loc, adaptor.getBankId(), adaptor.getGroup(), - cstI64(rewriter, loc, 0), adaptor.getIter()); - rewriter.replaceOpWithNewOp( - op, rs1, adaptor.getStride(), - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("RELU"))); - return success(); - } -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateReluBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &converter, - RewritePatternSet &patterns, - bool, int64_t) { - patterns.add(converter); -} - -void configureReluBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool) { - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/examples/balls/relu/configs/default.toml b/examples/balls/relu/configs/default.toml deleted file mode 100644 index 80321fad9..000000000 --- a/examples/balls/relu/configs/default.toml +++ /dev/null @@ -1,2 +0,0 @@ -# ReluBall has no implementation-specific parameters. -[ball] diff --git a/examples/balls/relu/emu/src/dispatch.rs b/examples/balls/relu/emu/src/dispatch.rs deleted file mode 100644 index aca809580..000000000 --- a/examples/balls/relu/emu/src/dispatch.rs +++ /dev/null @@ -1,31 +0,0 @@ -use crate::inst::instruction::{BallInstruction, ExecContext}; - -use super::f50_relu; - -const BALL_CLASS: &str = "examples.balls.relu.ReluBall"; - -pub fn execute_known( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, - ctx: &mut ExecContext, -) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("RELU") => Some(f50_relu::Relu::exec(xs1, xs2, ctx)), - Some(_) | None => None, - } -} - -pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("RELU") => Some(f50_relu::Relu::latency(xs1, xs2)), - Some(_) | None => None, - } -} diff --git a/examples/balls/relu/emu/src/lib.rs b/examples/balls/relu/emu/src/lib.rs deleted file mode 100644 index c691f0d9e..000000000 --- a/examples/balls/relu/emu/src/lib.rs +++ /dev/null @@ -1,8 +0,0 @@ -pub(crate) use crate::inst::{decode, instruction}; - -#[path = "50_relu.rs"] -mod f50_relu; - -mod dispatch; - -pub use dispatch::{cycles_after_issue, execute_known}; diff --git a/examples/balls/relu/workloads/ctests/CMakeLists.txt b/examples/balls/relu/workloads/ctests/CMakeLists.txt deleted file mode 100644 index cfdbcc1df..000000000 --- a/examples/balls/relu/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -add_buckyball_ctests( - relu_i32_16x16_test.c -) diff --git a/examples/balls/relu/workloads/isa/relu.h b/examples/balls/relu/workloads/isa/relu.h deleted file mode 100644 index 7f6589b67..000000000 --- a/examples/balls/relu/workloads/isa/relu.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef _BB_RELU_H_ -#define _BB_RELU_H_ - -#include -#include - -#define bb_relu(bank_id, group, iter, stride) \ - BUCKYBALL_INSTRUCTION_R_R( \ - (BB_BANK0(bank_id) | BB_BANK1(group) | BB_ITER(iter)), (stride), \ - BB_FUNC7(RELU)) - -#endif // _BB_RELU_H_ diff --git a/examples/balls/relu/workloads/mlir_tests/ball/relu_i32_16row.mlir b/examples/balls/relu/workloads/mlir_tests/ball/relu_i32_16row.mlir deleted file mode 100644 index 496be2993..000000000 --- a/examples/balls/relu/workloads/mlir_tests/ball/relu_i32_16row.mlir +++ /dev/null @@ -1,34 +0,0 @@ -func.func private @check_result(memref<16x4xi32>) -> () -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %four = arith.constant 4 : index - %sixteen = arith.constant 16 : index - %four_i32 = arith.constant 4 : i32 - %sixteen_i32 = arith.constant 16 : i32 - %iter = arith.constant 16 : i64 - %group = arith.constant 0 : i64 - %stride = arith.constant 1 : i64 - %input = memref.alloc() : memref<16x4xi32> - %output = memref.alloc() : memref<16x4xi32> - scf.for %row = %zero to %sixteen step %one { - scf.for %lane = %zero to %four step %one { - %r = arith.index_cast %row : index to i32 - %l = arith.index_cast %lane : index to i32 - %base = arith.muli %r, %four_i32 : i32 - %value = arith.addi %base, %l : i32 - %signed = arith.subi %value, %sixteen_i32 : i32 - memref.store %signed, %input[%row, %lane] : memref<16x4xi32> - } - } - %bank = buckyball.bank_alloc - %loaded = buckyball.bank_mvin %input %bank %iter %stride : memref<16x4xi32> i64 i64 i64 - buckyball.relu %loaded, %group, %iter, %iter : i64 - %stored = buckyball.bank_mvout %output %loaded %iter %stride : memref<16x4xi32> i64 i64 i64 - func.call @check_result(%output) : (memref<16x4xi32>) -> () - buckyball.bank_release %stored : i64 - memref.dealloc %input : memref<16x4xi32> - memref.dealloc %output : memref<16x4xi32> - return %zero_i8 : i8 -} diff --git a/examples/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign.mlir.in b/examples/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign.mlir.in deleted file mode 100644 index f6de71d79..000000000 --- a/examples/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign.mlir.in +++ /dev/null @@ -1,43 +0,0 @@ -func.func private @check_result(memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32>) -> () - -func.func @main() -> i8 { - %zeroI8 = arith.constant 0 : i8 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %group = arith.constant 0 : i64 - %lines = arith.constant @BB_BANK_LINES@ : i64 - %stride = arith.constant 1 : i64 - %limit = arith.constant @BB_BANK_LINES@ : index - %lanes = arith.constant @BB_BANK_I32_LANES@ : index - %lane_count = arith.constant @BB_BANK_I32_LANES@ : i32 - %seventeen = arith.constant 17 : i32 - %bias = arith.constant 8 : i32 - - %input = memref.alloc() alignment = 64 : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> - %output = memref.alloc() alignment = 64 : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> - - scf.for %line = %zero to %limit step %one { - scf.for %lane = %zero to %lanes step %one { - %line_i32 = arith.index_cast %line : index to i32 - %lane_i32 = arith.index_cast %lane : index to i32 - %base = arith.muli %line_i32, %lane_count : i32 - %index = arith.addi %base, %lane_i32 : i32 - %remainder = arith.remui %index, %seventeen : i32 - %signed = arith.subi %remainder, %bias : i32 - memref.store %signed, %input[%line, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> - } - } - - %bank = buckyball.bank_alloc - %loaded = buckyball.bank_mvin %input %bank %lines %stride - : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> i64 i64 i64 - %result = buckyball.bank_relu %loaded %group %lines %lines : i64 i64 i64 i64 - %stored = buckyball.bank_mvout %output %result %lines %stride - : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> i64 i64 i64 - buckyball.bank_release %stored : i64 - - func.call @check_result(%output) : (memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32>) -> () - memref.dealloc %input : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> - memref.dealloc %output : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> - return %zeroI8 : i8 -} diff --git a/examples/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign_main.cpp b/examples/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign_main.cpp deleted file mode 100644 index 02bea257c..000000000 --- a/examples/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign_main.cpp +++ /dev/null @@ -1,45 +0,0 @@ -#include -#include -#include -#include - -static void fail(void) { -#ifdef BAREMETAL - volatile uint32_t *sim_exit = (volatile uint32_t *)0x60000000; - *sim_exit = 1; - while (1) { - } -#else - exit(1); -#endif -} - -#ifdef __cplusplus -extern "C" -#endif - void check_result(int32_t *allocated, int32_t *aligned, int64_t offset, - int64_t size0, int64_t size1, int64_t stride0, - int64_t stride1) { - (void)allocated; - if (size0 != BANK_LINES || size1 != BANK_WIDTH / 32 || - stride0 != BANK_WIDTH / 32 || stride1 != 1) { - printf("FAILED: relu bank shape %dx%d stride %dx%d\n", (int)size0, - (int)size1, (int)stride0, (int)stride1); - fail(); - } - - int32_t *output = aligned + offset; - for (int line = 0; line < BANK_LINES; ++line) { - for (int lane = 0; lane < BANK_WIDTH / 32; ++lane) { - int32_t input = (line * (BANK_WIDTH / 32) + lane) % 17 - 8; - int32_t expected = input < 0 ? 0 : input; - int32_t actual = output[line * stride0 + lane * stride1]; - if (actual != expected) { - printf("FAILED: relu bank[%d][%d] expected %d got %d\n", line, lane, - expected, actual); - fail(); - } - } - } - printf("relu i32 mixed-sign PASS rows=%d\n", BANK_LINES); -} diff --git a/examples/balls/smatmul/README.md b/examples/balls/smatmul/README.md deleted file mode 100644 index 482c99155..000000000 --- a/examples/balls/smatmul/README.md +++ /dev/null @@ -1,31 +0,0 @@ -# SMatMulBall - -`SMatMulBall` computes signed INT8 matrix products with signed INT32 output. -The array is 16x16, SRAM rows are 128 bits, and every SRAM is single-port. - -## Commands - -Every command uses the framework-defined `rs1` layout: bank0 in bits 9:0, -bank1 in bits 19:10, bank2 in bits 29:20, and `iter` in bits 63:30. There are -no bank-relative base fields; every operand starts at row zero. - -`SMATMUL_BIAS` reads four rows from bank0 and retains one 16-element INT32 -bias vector. bank1, bank2, and `rs2` are zero, and `iter` is four. - -`SMATMUL_OS` uses bank0 for A, bank1 for B, and bank2 for C. `iter` is K. -`rs2[11:0]` is M, `rs2[23:12]` is N, and bits 24 and 25 are the first/last -accumulation-block flags. The remaining `rs2` bits are zero. M and K are -positive multiples of 16 and N is exactly 16. - -## Layout - -Every result row has four 128-bit words. With `outBW=1`, logical output row -`r` uses four consecutive bank rows: - -```text -line r*4 + word -``` - -The Ball keeps partial sums internally between commands with `last=0`; only -the command with `last=1` writes C. A later accumulation block may select new -A and B banks, but each still starts at bank row zero. diff --git a/examples/balls/smatmul/arch/src/main/scala/Array.scala b/examples/balls/smatmul/arch/src/main/scala/Array.scala deleted file mode 100644 index 77d458342..000000000 --- a/examples/balls/smatmul/arch/src/main/scala/Array.scala +++ /dev/null @@ -1,83 +0,0 @@ -package examples.balls.smatmul - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import examples.balls.smatmul.configs.SMatMulBallParam -import framework.top.GlobalConfig - -@instantiable -class Array(val b: GlobalConfig) extends Module { - private val param = SMatMulBallParam(b) - private val rows = param.tileRows - private val cols = param.tileCols - - @public - val io = IO(new Bundle { - val start = Input(Bool()) - val aRows = Input(Vec(rows, UInt(128.W))) - val bRows = Input(Vec(cols, UInt(128.W))) - val done = Output(Bool()) - val result = Output(Vec(rows, UInt(512.W))) - }) - - val running = RegInit(false.B) - val cycle = RegInit(0.U(6.W)) - val pe = Seq.tabulate(rows, cols)((_, _) => Instantiate(new PE)) - val aShift = Reg(Vec(rows, UInt(128.W))) - // The PE product register adds one cycle between the systolic operands and - // the accumulator. Keep the final drain cycle explicit so the last tile - // product is included before done is asserted. - val lastCyc = (rows + cols + 15).U - - when(io.start) { - assert(!running, "Array: start while array is running") - running := true.B - cycle := 0.U - aShift := io.aRows - }.elsewhen(running) { - when(cycle === lastCyc) { - running := false.B - }.otherwise { - cycle := cycle + 1.U - } - for (row <- 0 until rows) { - when(cycle >= row.U && cycle < (row + cols).U) { - aShift(row) := aShift(row) >> 8 - } - } - } - - for (row <- 0 until rows) { - val aWord = aShift(row)(7, 0).asSInt - val aValid = running && cycle >= row.U && cycle < (row + cols).U - - for (column <- 0 until cols) { - pe(row)(column).io.clear := io.start - if (column == 0) { - pe(row)(column).io.aIn := aWord - pe(row)(column).io.aInValid := aValid - } else { - pe(row)(column).io.aIn := pe(row)(column - 1).io.aOut - pe(row)(column).io.aInValid := pe(row)(column - 1).io.aOutValid - } - - val bCycle = cycle - column.U - val bRow = bCycle(3, 0) - val bWord = io.bRows(bRow)(8 * column + 7, 8 * column).asSInt - val bValid = running && cycle >= column.U && cycle < (column + cols).U - if (row == 0) { - pe(row)(column).io.bIn := bWord - pe(row)(column).io.bInValid := bValid - } else { - pe(row)(column).io.bIn := pe(row - 1)(column).io.bOut - pe(row)(column).io.bInValid := pe(row - 1)(column).io.bOutValid - } - } - } - - for (row <- 0 until rows) { - io.result(row) := Cat((0 until cols).reverse.map(column => pe(row)(column).io.sum.asUInt)) - } - io.done := running && cycle === lastCyc -} diff --git a/examples/balls/smatmul/arch/src/main/scala/PE.scala b/examples/balls/smatmul/arch/src/main/scala/PE.scala deleted file mode 100644 index b01cae5b5..000000000 --- a/examples/balls/smatmul/arch/src/main/scala/PE.scala +++ /dev/null @@ -1,60 +0,0 @@ -package examples.balls.smatmul - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public} - -@instantiable -class PE extends Module { - - // One tile accumulates sixteen signed 8x8 products. Twenty bits cover the - // full range [-262144, 258064]; the unit-level accumulator remains 32-bit - // for reduction across tiles. - private val tileSumWidth = 20 - - @public - val io = IO(new Bundle { - val aIn = Input(SInt(8.W)) - val aInValid = Input(Bool()) - val bIn = Input(SInt(8.W)) - val bInValid = Input(Bool()) - val clear = Input(Bool()) - val aOut = Output(SInt(8.W)) - val aOutValid = Output(Bool()) - val bOut = Output(SInt(8.W)) - val bOutValid = Output(Bool()) - val sum = Output(SInt(32.W)) - }) - - val accumulator = RegInit(0.S(tileSumWidth.W)) - val aPipe = RegInit(0.S(8.W)) - val aPipeValid = RegInit(false.B) - val bPipe = RegInit(0.S(8.W)) - val bPipeValid = RegInit(false.B) - val product = RegInit(0.S(16.W)) - val productValid = RegInit(false.B) - - when(io.clear) { - accumulator := 0.S - aPipeValid := false.B - bPipeValid := false.B - productValid := false.B - }.elsewhen(productValid) { - accumulator := accumulator + product - } - when(!io.clear) { - product := aPipe * bPipe - productValid := aPipeValid && bPipeValid - } - when(!io.clear) { - aPipe := io.aIn - aPipeValid := io.aInValid - bPipe := io.bIn - bPipeValid := io.bInValid - } - - io.aOut := aPipe - io.aOutValid := aPipeValid - io.bOut := bPipe - io.bOutValid := bPipeValid - io.sum := accumulator.pad(32) -} diff --git a/examples/balls/smatmul/arch/src/main/scala/SMatMulBall.scala b/examples/balls/smatmul/arch/src/main/scala/SMatMulBall.scala deleted file mode 100644 index d4b4970e5..000000000 --- a/examples/balls/smatmul/arch/src/main/scala/SMatMulBall.scala +++ /dev/null @@ -1,50 +0,0 @@ -package examples.balls.smatmul - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.balldomain.blink.{BallStatus, BlinkIO, HasBallStatus, HasBlink, SubRobRow} -import framework.balldomain.blink.mmio.MmioRead -import framework.top.GlobalConfig - -@instantiable -class SMatMulBall(val b: GlobalConfig) extends Module with HasBlink with HasBallStatus { - - val ballCommonConfig = b.ballDomain.ballIdMappings - .find(_.ballName == "SMatMulBall") - .getOrElse( - throw new IllegalArgumentException("SMatMulBall not found in config") - ) - - val inBW = ballCommonConfig.inBW - val outBW = ballCommonConfig.outBW - - @public - val io = IO(new BlinkIO(b, inBW, outBW)) - - def blink: BlinkIO = io - def status: BallStatus = io.status - dontTouch(io) - - val smatmulArrayUnit: Instance[SMatMulUnit] = Instantiate(new SMatMulUnit(b)) - - smatmulArrayUnit.io.cmdReq <> io.cmdReq - smatmulArrayUnit.io.cmdResp <> io.cmdResp - smatmulArrayUnit.io.channelReady := io.channelReady - - for (i <- 0 until inBW) { - smatmulArrayUnit.io.bankRead(i) <> io.bankRead(i) - } - - for (i <- 0 until outBW) { - smatmulArrayUnit.io.bankWrite(i) <> io.bankWrite(i) - } - - io.status <> smatmulArrayUnit.io.status - - io.subRobReq.valid := false.B - io.subRobReq.bits := SubRobRow.tieOff(b) - - // MMIO: this Ball does not consume MMIO metadata - MmioRead.tieOff(io.mmioRead) -} diff --git a/examples/balls/smatmul/arch/src/main/scala/SMatMulUnit.scala b/examples/balls/smatmul/arch/src/main/scala/SMatMulUnit.scala deleted file mode 100644 index 8304e912f..000000000 --- a/examples/balls/smatmul/arch/src/main/scala/SMatMulUnit.scala +++ /dev/null @@ -1,431 +0,0 @@ -package examples.balls.smatmul - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.balldomain.rs.{BallRsComplete, BallRsIssue} -import framework.balldomain.blink.{BallStatus, BankRead, BankWrite} -import framework.top.GlobalConfig -import examples.balls.smatmul.configs.SMatMulBallParam - -@instantiable -class SMatMulUnit(val b: GlobalConfig) extends Module { - private val param = SMatMulBallParam(b) - private val rows = param.tileRows - private val cols = param.tileCols - private val resultWords = 4 - private val addressWidth = log2Ceil(b.memDomain.bankEntries) - private val bankIdWidth = b.memDomain.vbankIdWidth - private val maxOutputRows = b.memDomain.bankEntries / resultWords - private val accDepth = maxOutputRows.max(32) - private val accAddrWidth = log2Ceil(accDepth) - - private val mapping = b.ballDomain.ballIdMappings - .find(_.ballName == "SMatMulBall") - .getOrElse(throw new IllegalArgumentException("SMatMulBall not found in config")) - - private val osFunct = b.ballDomain.ballISA - .find(_.mnemonic == "SMATMUL_OS") - .map(_.funct7) - .getOrElse(throw new IllegalArgumentException("SMATMUL_OS not found in ballISA")) - - private val biasFunct = b.ballDomain.ballISA - .find(_.mnemonic == "SMATMUL_BIAS") - .map(_.funct7) - .getOrElse(throw new IllegalArgumentException("SMATMUL_BIAS not found in ballISA")) - - require(mapping.inBW == 2, "SMatMulBall requires inBW=2") - require(mapping.outBW == 1, "SMatMulBall requires outBW=1") - require(b.memDomain.bankWidth == 128, "SMatMulBall requires 128-bit bank rows") - require(b.memDomain.bankMaskLen == 16, "SMatMulBall requires sixteen byte enables") - require(b.memDomain.bankEntries % resultWords == 0, "SMatMulBall bank depth must be divisible by four") - require(maxOutputRows >= rows, "SMatMulBall accumulator must hold at least one tile") - require((osFunct >> 4) == 4, "SMATMUL_OS must encode two reads and one write") - require((biasFunct >> 4) == 1, "SMATMUL_BIAS must encode one read") - - @public - val io = IO(new Bundle { - val cmdReq = Flipped(Decoupled(new BallRsIssue(b))) - val cmdResp = Decoupled(new BallRsComplete(b)) - val bankRead = Vec(mapping.inBW, Flipped(new BankRead(b))) - val bankWrite = Vec(mapping.outBW, Flipped(new BankWrite(b))) - val channelReady = Input(Bool()) - val status = new BallStatus - }) - - private val Seq( - idle, - waitForChannels, - biasReadReq, - biasReadResp, - initAccumulator, - loadTile, - runArray, - readAccumulator, - writeAccumulator, - readResult, - holdResult, - writeResult, - waitForCWrite, - complete - ) = Enum(14) - - private val state = RegInit(idle) - - private val robId = RegInit(0.U(log2Up(b.frontend.rob_entries).W)) - private val isSub = RegInit(false.B) - private val subRobId = RegInit(0.U(log2Up(b.frontend.sub_rob_depth * 4).W)) - - private val biasCommand = RegInit(false.B) - private val firstBlock = RegInit(false.B) - private val lastBlock = RegInit(false.B) - private val vectorMode = RegInit(false.B) - private val biasValid = RegInit(false.B) - private val chainLive = RegInit(false.B) - private val biasBank = RegInit(0.U(bankIdWidth.W)) - private val biasBase = RegInit(0.U(log2Ceil(b.memDomain.bankEntries).W)) - private val biasRow = RegInit(0.U(2.W)) - private val biasWords = Reg(Vec(resultWords, UInt(128.W))) - - private val aBank = RegInit(0.U(bankIdWidth.W)) - private val bBank = RegInit(0.U(bankIdWidth.W)) - private val cBank = RegInit(0.U(bankIdWidth.W)) - private val cBase = RegInit(0.U(addressWidth.W)) - private val chainRows = RegInit(0.U(12.W)) - private val chainCBank = RegInit(0.U(bankIdWidth.W)) - private val chainCBase = RegInit(0.U(addressWidth.W)) - - private val outputTileCount = RegInit(0.U(8.W)) - private val reductionTileCount = RegInit(0.U(8.W)) - private val outputTile = RegInit(0.U(8.W)) - private val reductionTile = RegInit(0.U(8.W)) - private val accumulatorRow = RegInit(0.U(4.W)) - private val resultRow = RegInit(0.U(4.W)) - private val outputWord = RegInit(0.U(2.W)) - private val aRowsRequested = RegInit(0.U(5.W)) - private val aRowsStored = RegInit(0.U(5.W)) - private val bRowsRequested = RegInit(0.U(5.W)) - private val bRowsStored = RegInit(0.U(5.W)) - private val cRowData = Reg(UInt(512.W)) - - private val aRows = Reg(Vec(rows, UInt(128.W))) - private val bRows = Reg(Vec(cols, UInt(128.W))) - private val accumulator = SyncReadMem(accDepth, UInt(512.W)) - private val array: Instance[Array] = Instantiate(new Array(b)) - - private val computeGlobalRow = outputTile * rows.U + accumulatorRow - private val resultGlobalRow = outputTile * rows.U + resultRow - private val accumulatorRead = state === readAccumulator || state === readResult - private val accumulatorWrite = state === initAccumulator || state === writeAccumulator - - private val accumulatorAddress = Mux( - state === readResult, - resultGlobalRow(accAddrWidth - 1, 0), - computeGlobalRow(accAddrWidth - 1, 0) - ) - - private val biasData = Cat(biasWords.reverse) - private val accumulatorWriteData = Wire(UInt(512.W)) - - private val accumulatorData = accumulator.readWrite( - accumulatorAddress, - accumulatorWriteData, - accumulatorRead || accumulatorWrite, - accumulatorWrite - ) - - private val accumulatedResult = Wire(Vec(cols, UInt(32.W))) - for (column <- 0 until cols) { - val oldValue = accumulatorData(32 * column + 31, 32 * column).asSInt - val newValue = array.io.result(accumulatorRow)(32 * column + 31, 32 * column).asSInt - accumulatedResult(column) := (oldValue + newValue).asUInt - } - accumulatorWriteData := Mux(state === initAccumulator, biasData, Cat(accumulatedResult.reverse)) - assert(!(accumulatorRead && accumulatorWrite), "SMatMulBall accumulator SRAM is single-port") - - private val aTileLine = Mux( - vectorMode, - reductionTile, - (outputTile * reductionTileCount + reductionTile) * rows.U - ) - - private val bTileLine = reductionTile * cols.U - private val cLine = cBase +& (resultGlobalRow << 2) +& outputWord - - for (port <- 0 until mapping.inBW) { - io.bankRead(port).rob_id := robId - io.bankRead(port).ball_id := 0.U - io.bankRead(port).bank_id := Mux(port.U === 0.U, aBank, bBank) - io.bankRead(port).group_id := 0.U - io.bankRead(port).io.req.valid := false.B - io.bankRead(port).io.req.bits.addr := 0.U - io.bankRead(port).io.resp.ready := false.B - } - io.bankRead(0).bank_id := Mux(state === biasReadReq || state === biasReadResp, biasBank, aBank) - io.bankRead(0).io.req.valid := Mux( - state === biasReadReq, - true.B, - state === loadTile && aRowsRequested < Mux(vectorMode, 1.U, rows.U) - ) - io.bankRead(0).io.req.bits.addr := Mux( - state === biasReadReq, - (biasBase +& biasRow)(addressWidth - 1, 0), - (aTileLine + aRowsRequested)(addressWidth - 1, 0) - ) - io.bankRead(0).io.resp.ready := state === biasReadResp || (state === loadTile && aRowsStored < rows.U) - io.bankRead(1).io.req.valid := state === loadTile && bRowsRequested < cols.U - io.bankRead(1).io.req.bits.addr := (bTileLine + bRowsRequested)(addressWidth - 1, 0) - io.bankRead(1).io.resp.ready := state === loadTile && bRowsStored < cols.U - - private val cWords = cRowData.asTypeOf(Vec(resultWords, UInt(128.W))) - io.bankWrite(0).rob_id := robId - io.bankWrite(0).ball_id := 0.U - io.bankWrite(0).bank_id := cBank - io.bankWrite(0).group_id := 0.U - io.bankWrite(0).io.req.valid := state === writeResult - io.bankWrite(0).io.req.bits.addr := cLine(addressWidth - 1, 0) - io.bankWrite(0).io.req.bits.data := cWords(outputWord) - io.bankWrite(0).io.req.bits.mask := VecInit(Seq.fill(16)(true.B)) - io.bankWrite(0).io.resp.ready := state === waitForCWrite - - array.io.start := state === loadTile && - aRowsStored === Mux(vectorMode, 1.U, rows.U) && bRowsStored === cols.U - array.io.aRows := aRows - array.io.bRows := bRows - - io.cmdReq.ready := state === idle - io.cmdResp.valid := state === complete - io.cmdResp.bits.rob_id := robId - io.cmdResp.bits.is_sub := isSub - io.cmdResp.bits.sub_rob_id := subRobId - io.status.idle := state === idle - io.status.running := state =/= idle && state =/= complete - - when(io.cmdReq.fire) { - val command = io.cmdReq.bits.cmd - val isBias = command.funct7 === biasFunct.U - val isMatmul = command.funct7 === osFunct.U - val rows = command.rs2(11, 0) - val columns = command.rs2(23, 12) - val reduction = command.iter - val first = command.rs2(24) - val last = command.rs2(25) - - assert(isBias || isMatmul, "SMatMulBall received an unknown funct7") - assert(command.rs1(9, 0) < b.memDomain.virtualBankCount.U, "SMatMulBall input bank 0 is invalid") - - robId := io.cmdReq.bits.rob_id - isSub := io.cmdReq.bits.is_sub - subRobId := io.cmdReq.bits.sub_rob_id - biasCommand := isBias - - when(isBias) { - assert(command.rs1(29, 10) === 0.U, "SMATMUL_BIAS reserves bank1 and bank2") - assert(command.iter === 4.U, "SMATMUL_BIAS iter must be four rows") - assert(command.rs2(63, 6) === 0.U, "SMATMUL_BIAS reserves rs2[63:6]") - assert( - command.rs2(5, 0) +& 4.U <= b.memDomain.bankEntries.U, - "SMATMUL_BIAS inputBase plus four rows must fit one bank" - ) - assert( - command.op1_col === 1.U && command.op2_col === 0.U && command.wr_col === 0.U, - "SMATMUL_BIAS requires exactly one input bank" - ) - assert(!chainLive, "SMATMUL_BIAS cannot replace bias during an accumulation chain") - biasBank := command.op1_bank - biasBase := command.rs2(5, 0) - biasRow := 0.U - }.otherwise { - assert(command.rs1(19, 10) < b.memDomain.virtualBankCount.U, "SMATMUL_OS input bank 1 is invalid") - assert(command.rs1(29, 20) < b.memDomain.virtualBankCount.U, "SMATMUL_OS output bank is invalid") - assert( - command.op1_bank =/= command.op2_bank && command.op1_bank =/= command.wr_bank && - command.op2_bank =/= command.wr_bank, - "SMATMUL_OS requires distinct A, B, and C banks" - ) - assert( - command.op1_col === 1.U && command.op2_col === 1.U && command.wr_col === 1.U, - "SMATMUL_OS requires one physical bank per operand" - ) - assert( - rows === 1.U || (rows =/= 0.U && rows(2, 0) === 0.U), - "SMATMUL_OS M must be one or a positive multiple of 8" - ) - assert(columns === cols.U, "SMATMUL_OS N must be 16") - assert(reduction =/= 0.U && reduction(3, 0) === 0.U, "SMATMUL_OS K must be a positive multiple of 16") - assert(command.rs2(63, 32) === 0.U, "SMATMUL_OS reserves rs2[63:32]") - assert( - Mux(rows === 1.U, reduction >> 4, (rows * reduction) >> 4) <= b.memDomain.bankEntries.U, - "SMATMUL_OS A footprint exceeds bank depth" - ) - assert(reduction <= b.memDomain.bankEntries.U, "SMATMUL_OS B footprint exceeds bank depth") - assert( - command.rs2(31, 26) +& rows * resultWords.U <= b.memDomain.bankEntries.U, - "SMATMUL_OS C footprint exceeds bank depth" - ) - when(first) { - assert(!chainLive, "SMATMUL_OS first block cannot start while a chain is live") - assert(biasValid, "SMATMUL_OS first block requires a bias preload") - chainLive := true.B - chainRows := rows - chainCBank := command.wr_bank - chainCBase := command.rs2(31, 26) - }.otherwise { - assert(chainLive, "SMATMUL_OS continuation requires a live chain") - assert( - rows === chainRows && command.wr_bank === chainCBank && - command.rs2(31, 26) === chainCBase, - "SMATMUL_OS continuation changed M or C destination/base" - ) - } - - firstBlock := first - lastBlock := last - vectorMode := rows === 1.U - aBank := command.op1_bank - bBank := command.op2_bank - cBank := command.wr_bank - cBase := command.rs2(31, 26) - outputTileCount := Mux(rows === 1.U, 1.U, rows / param.tileRows.U) - reductionTileCount := reduction >> 4 - outputTile := 0.U - reductionTile := 0.U - accumulatorRow := 0.U - resultRow := 0.U - outputWord := 0.U - aRowsRequested := 0.U - aRowsStored := 0.U - bRowsRequested := 0.U - bRowsStored := 0.U - for (row <- 1 until 8) { - aRows(row) := 0.U - } - } - state := waitForChannels - } - - when(state === waitForChannels && io.channelReady) { - state := Mux(biasCommand, biasReadReq, Mux(firstBlock, initAccumulator, loadTile)) - } - - when(state === biasReadReq && io.bankRead(0).io.req.fire) { - state := biasReadResp - } - when(state === biasReadResp && io.bankRead(0).io.resp.fire) { - biasWords(biasRow) := io.bankRead(0).io.resp.bits.data - when(biasRow === (resultWords - 1).U) { - biasValid := true.B - state := complete - }.otherwise { - biasRow := biasRow + 1.U - state := biasReadReq - } - } - - when(state === initAccumulator) { - when(accumulatorRow === Mux(vectorMode, 0.U, (rows - 1).U)) { - accumulatorRow := 0.U - when(outputTile + 1.U === outputTileCount) { - outputTile := 0.U - reductionTile := 0.U - aRowsRequested := 0.U - aRowsStored := 0.U - bRowsRequested := 0.U - bRowsStored := 0.U - state := loadTile - }.otherwise { - outputTile := outputTile + 1.U - } - }.otherwise { - accumulatorRow := accumulatorRow + 1.U - } - } - - when(state === loadTile) { - when(io.bankRead(0).io.req.fire)(aRowsRequested := aRowsRequested + 1.U) - when(io.bankRead(1).io.req.fire)(bRowsRequested := bRowsRequested + 1.U) - when(io.bankRead(0).io.resp.fire) { - aRows(aRowsStored(3, 0)) := io.bankRead(0).io.resp.bits.data - aRowsStored := aRowsStored + 1.U - } - when(io.bankRead(1).io.resp.fire) { - bRows(bRowsStored(3, 0)) := io.bankRead(1).io.resp.bits.data - bRowsStored := bRowsStored + 1.U - } - when(aRowsStored === Mux(vectorMode, 1.U, rows.U) && bRowsStored === cols.U) { - state := runArray - } - } - - when(state === runArray && array.io.done) { - accumulatorRow := 0.U - state := readAccumulator - } - when(state === readAccumulator) { - state := writeAccumulator - } - when(state === writeAccumulator) { - when(accumulatorRow === Mux(vectorMode, 0.U, (rows - 1).U)) { - accumulatorRow := 0.U - when(reductionTile + 1.U < reductionTileCount) { - reductionTile := reductionTile + 1.U - aRowsRequested := 0.U - aRowsStored := 0.U - bRowsRequested := 0.U - bRowsStored := 0.U - state := loadTile - }.elsewhen(outputTile + 1.U < outputTileCount) { - outputTile := outputTile + 1.U - reductionTile := 0.U - aRowsRequested := 0.U - aRowsStored := 0.U - bRowsRequested := 0.U - bRowsStored := 0.U - state := loadTile - }.otherwise { - when(lastBlock) { - outputTile := 0.U - resultRow := 0.U - state := readResult - }.otherwise { - state := complete - } - } - }.otherwise { - accumulatorRow := accumulatorRow + 1.U - state := readAccumulator - } - } - - when(state === readResult) { - outputWord := 0.U - state := holdResult - } - when(state === holdResult) { - cRowData := accumulatorData - state := writeResult - } - when(state === writeResult && io.bankWrite(0).io.req.fire) { - state := waitForCWrite - } - when(state === waitForCWrite && io.bankWrite(0).io.resp.fire) { - when(outputWord =/= (resultWords - 1).U) { - outputWord := outputWord + 1.U - state := writeResult - }.elsewhen(resultRow =/= Mux(vectorMode, 0.U, (rows - 1).U)) { - resultRow := resultRow + 1.U - state := readResult - }.elsewhen(outputTile + 1.U < outputTileCount) { - outputTile := outputTile + 1.U - resultRow := 0.U - state := readResult - }.otherwise { - chainLive := false.B - state := complete - } - } - - when(state === complete && io.cmdResp.fire) { - state := idle - } -} diff --git a/examples/balls/smatmul/compiler/runtime/fp32.cpp b/examples/balls/smatmul/compiler/runtime/fp32.cpp deleted file mode 100644 index 73c7c7768..000000000 --- a/examples/balls/smatmul/compiler/runtime/fp32.cpp +++ /dev/null @@ -1,99 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -extern "C" void _mlir_ciface_fp32_matmul(UnrankedMemRefType *lhs, - UnrankedMemRefType *rhs, - UnrankedMemRefType *result) { - DynamicMemRefType a(*lhs), b(*rhs), c(*result); - const int rank = a.rank; - if ((rank != 2 && rank != 3) || b.rank != rank || c.rank != rank) { - fputs("FP32 matmul requires matrices or matched batches\n", stderr); - abort(); - } - const int64_t batches = rank == 3 ? a.sizes[0] : 1; - const int64_t m = a.sizes[rank - 2], k = a.sizes[rank - 1], - n = b.sizes[rank - 1]; - if (batches <= 0 || m <= 0 || k <= 0 || n <= 0 || k != b.sizes[rank - 2] || - c.sizes[rank - 2] != m || c.sizes[rank - 1] != n || - (rank == 3 && (b.sizes[0] != batches || c.sizes[0] != batches))) { - fputs("FP32 matmul shape mismatch\n", stderr); - abort(); - } - constexpr int block = BANK_LINES * (BANK_WIDTH / 8) / (16 * sizeof(float)); - static_assert(block > 0 && block % 4 == 0 && block < 4096 && - BANK_WIDTH == 128); - alignas(64) float ap[16 * block], bp[16 * block], cp[256]; - for (int bank = 0; bank < 3; ++bank) - bb_mem_alloc(bank, 1, 1); - for (int64_t batch = 0; batch < batches; ++batch) { - const float *av = - a.data + a.offset + (rank == 3 ? batch * a.strides[0] : 0); - const float *bv = - b.data + b.offset + (rank == 3 ? batch * b.strides[0] : 0); - float *cv = c.data + c.offset + (rank == 3 ? batch * c.strides[0] : 0); - for (int64_t row = 0; row < m; row += 16) - for (int64_t column = 0; column < n; column += 16) { - int rows = m - row == 1 ? 1 : 16; - for (int64_t inner = 0; inner < k; inner += block) { - int count = int((std::min(block, k - inner) + 3) / 4 * 4); - const float *as = - av + row * a.strides[rank - 2] + inner * a.strides[rank - 1]; - const float *bs = - bv + inner * b.strides[rank - 2] + column * b.strides[rank - 1]; - bool direct_a = row + rows <= m && inner + count <= k && - a.strides[rank - 1] == 1 && - (rows == 1 || a.strides[rank - 2] == count); - bool direct_b = column + 16 <= n && inner + count <= k && - b.strides[rank - 2] == 1 && - b.strides[rank - 1] == count; - if (!direct_a) - for (int i = 0; i < rows; ++i) - if (row + i < m && a.strides[rank - 1] == 1) { - int valid = int(std::min(count, k - inner)); - std::memcpy(ap + i * count, as + i * a.strides[rank - 2], - valid * sizeof(float)); - std::fill_n(ap + i * count + valid, count - valid, 0.0f); - } else - for (int j = 0; j < count; ++j) - ap[i * count + j] = row + i < m && inner + j < k - ? as[i * a.strides[rank - 2] + - j * a.strides[rank - 1]] - : 0; - if (!direct_b) - for (int i = 0; i < 16; ++i) - if (column + i < n && b.strides[rank - 2] == 1) { - int valid = int(std::min(count, k - inner)); - std::memcpy(bp + i * count, bs + i * b.strides[rank - 1], - valid * sizeof(float)); - std::fill_n(bp + i * count + valid, count - valid, 0.0f); - } else - for (int j = 0; j < count; ++j) - bp[i * count + j] = column + i < n && inner + j < k - ? bs[j * b.strides[rank - 2] + - i * b.strides[rank - 1]] - : 0; - bb_mvin((uintptr_t)(direct_a ? as : ap), 0, rows * count / 4, 1); - bb_mvin((uintptr_t)(direct_b ? bs : bp), 1, 16 * count / 4, 1); - bb_smatmul_f32(0, 1, 2, rows, 16, count, inner == 0, - inner + count >= k, 0); - } - bb_mvout((uintptr_t)cp, 2, rows * 4, 1); - for (int i = 0; i < rows && row + i < m; ++i) - if (c.strides[rank - 1] == 1) - std::memcpy(cv + (row + i) * c.strides[rank - 2] + column, - cp + i * 16, - std::min(16, n - column) * sizeof(float)); - else - for (int j = 0; j < 16 && column + j < n; ++j) - cv[(row + i) * c.strides[rank - 2] + - (column + j) * c.strides[rank - 1]] = cp[i * 16 + j]; - } - } - for (int bank = 0; bank < 3; ++bank) - bb_mem_release(bank); -} diff --git a/examples/balls/smatmul/compiler/src/Backend/include/IntrinsicsRISCVSMatMulBall.td b/examples/balls/smatmul/compiler/src/Backend/include/IntrinsicsRISCVSMatMulBall.td deleted file mode 100644 index 6b92a9730..000000000 --- a/examples/balls/smatmul/compiler/src/Backend/include/IntrinsicsRISCVSMatMulBall.td +++ /dev/null @@ -1,7 +0,0 @@ -//===- IntrinsicsRISCVSMatMulBall.td - SMatMulBall intrinsics --------------===// -// -// Stable LLVM intrinsic for SMatMulBall is not wired yet. Pebble uses -// CustomIntrOp(funct7 from Core ballISA SMATMUL). Keep this file as a placeholder for a future -// int_riscv_bb_smatmul addition to the RISC-V backend. -// -//===----------------------------------------------------------------------===// diff --git a/examples/balls/smatmul/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp b/examples/balls/smatmul/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp deleted file mode 100644 index 62dcf68a9..000000000 --- a/examples/balls/smatmul/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp +++ /dev/null @@ -1,53 +0,0 @@ -//===- AssignPhysicalBankPatterns.cpp - Matrix bank assignment patterns ---===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace mlir::buddy { -void populateSMatMulBallAssignPhysicalBankPatterns(RewritePatternSet &patterns, - PhysicalBankState &state); -} // namespace mlir::buddy - -namespace { - -class BankSMatMulPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(BankSMatMulOp op, - PatternRewriter &rewriter) const override { - rewriter.create(op.getLoc(), op.getOp1Bank(), op.getOp2Bank(), - op.getWrBank(), op.getConfig(), op.getFirst(), - op.getLast(), op.getOutputBase()); - rewriter.replaceOp(op, op.getWrBank()); - return success(); - } -}; - -class BankSMatMulBiasPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(BankSMatMulBiasOp op, - PatternRewriter &rewriter) const override { - rewriter.create(op.getLoc(), op.getBiasBank(), - op.getInputBase()); - rewriter.replaceOp(op, op.getBiasBank()); - return success(); - } -}; - -} // namespace - -void mlir::buddy::populateSMatMulBallAssignPhysicalBankPatterns( - RewritePatternSet &patterns, mlir::buddy::PhysicalBankState &state) { - (void)state; - patterns.add( - patterns.getContext()); -} diff --git a/examples/balls/smatmul/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp b/examples/balls/smatmul/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp deleted file mode 100644 index dfeeed1e5..000000000 --- a/examples/balls/smatmul/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp +++ /dev/null @@ -1,357 +0,0 @@ -//===- LowerBuckyballToBankSSAPatterns.cpp - Mega MatMul to banks --------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Target/BuckyballTargetRegistry.h" -#include "Utils/BankUtils.h" - -#include - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { - -constexpr int64_t kTile = 16; -constexpr int64_t kInt32RowsPerTile = 64; - -class MegaMatmulToBankSSAPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(MegaMatmulOp op, - PatternRewriter &b) const override { - Location loc = op.getLoc(); - auto aTy = dyn_cast(op.getInput().getType()); - auto wTy = dyn_cast(op.getWeight().getType()); - auto biasTy = dyn_cast(op.getBias().getType()); - auto scaleTy = dyn_cast(op.getScale().getType()); - auto lutTy = dyn_cast(op.getLut().getType()); - auto outTy = dyn_cast(op.getOutput().getType()); - auto kernel = op->getParentOfType(); - if (!kernel) - return op.emitError( - "Mega MatMul must be nested in buckyball.mega_kernel"); - bool finalOutput = op.getOutput() == kernel.getOutput(); - if (!aTy || !wTy || !biasTy || !scaleTy || !lutTy || !outTy || - !aTy.hasStaticShape() || !wTy.hasStaticShape() || - !biasTy.hasStaticShape() || !scaleTy.hasStaticShape() || - !lutTy.hasStaticShape() || !outTy.hasStaticShape()) - return op.emitError("Mega MatMul requires static memrefs"); - if (!aTy.getElementType().isInteger(8) || - !wTy.getElementType().isInteger(8) || - !biasTy.getElementType().isInteger(32) || - !scaleTy.getElementType().isF32() || - !lutTy.getElementType().isInteger(8) || lutTy.getRank() != 1 || - lutTy.getShape()[0] != (op.getActivation() == 2 ? 256 : 1) || - op.getActivation() < 0 || op.getActivation() > 2 || - (finalOutput && op.getActivation() == 2) || - (finalOutput ? !outTy.getElementType().isF32() - : !outTy.getElementType().isInteger(8))) - return op.emitError( - "Mega MatMul input/weight must be INT8, bias INT32, scale FP32, " - "and only the last stage may output FP32"); - - int64_t M = aTy.getShape()[0]; - int64_t K = aTy.getShape()[1]; - int64_t N = wTy.getShape()[1]; - if (M <= 0 || K <= 0 || N <= 0 || wTy.getShape()[0] != K || - biasTy.getShape()[0] != N || scaleTy.getShape()[0] != N || - outTy.getShape()[0] != M || outTy.getShape()[1] != N) - return op.emitError("Mega MatMul shape mismatch"); - const auto &target = buckyball_target::getBuckyballTarget(); - if (target.bankWidthBits != 128 || target.bankDepth < kInt32RowsPerTile || - target.bankDepth % kInt32RowsPerTile) - return op.emitError( - "Mega MatMul requires 128-bit banks whose depth is a positive " - "multiple of 64 rows"); - if (buckyball_target::getBuckyballBallMapping("SMatMulBall").outBW != 1) - return op.emitError("Mega MatMul requires SMatMulBall outBW=1"); - - const int64_t hardwareRows = - buckyball_target::getBuckyballBallParam("SMatMulBall", "tileRows"); - if (hardwareRows <= 0 || kTile % hardwareRows) - return op.emitError( - "SMatMul tileRows must divide the 16-row software tile"); - const int64_t tileRows = M == 1 ? 1 : kTile; - const int64_t resultRows = tileRows * 4; - int64_t paddedM = (M + tileRows - 1) / tileRows * tileRows; - int64_t paddedK = (K + kTile - 1) / kTile * kTile; - int64_t kChunk = target.bankDepth / kTile * kTile; - if (kChunk <= 0) - return op.emitError("bank depth cannot hold one K tile"); - // outputBase is a six-bit row index; one 16x16 result uses 64 rows. - constexpr int64_t tilesPerResultBank = 1; - - Value zero = b.create(loc, 0); - Value one = b.create(loc, 1); - Value tileSize = b.create(loc, kTile); - Value columns = b.create(loc, N); - Value zeroI8 = - b.create(loc, b.getI8Type(), b.getI8IntegerAttr(0)); - auto matrixRowLoop = b.create( - loc, zero, b.create(loc, paddedM), - b.create(loc, tileRows)); - b.setInsertionPointToStart(matrixRowLoop.getBody()); - Value m0 = matrixRowLoop.getInductionVar(); - Value validRows = b.create( - loc, b.create(loc, tileRows), - b.create(loc, b.create(loc, M), - m0)); - SmallVector inputPacks; - { - for (int64_t k0 = 0; k0 < paddedK; k0 += kChunk) { - int64_t thisK = std::min(kChunk, paddedK - k0); - int64_t validK = std::min(thisK, K - k0); - auto aPackTy = - MemRefType::get({thisK / kTile * tileRows, kTile}, b.getI8Type()); - Value aPack = b.create(loc, aPackTy); - inputPacks.push_back(aPack); - b.create(loc, zeroI8, aPack); - - Value zero = b.create(loc, 0); - Value one = b.create(loc, 1); - Value rowEnd = validRows; - Value kTileEnd = - b.create(loc, (validK + kTile - 1) / kTile); - auto ktLoop = b.create(loc, zero, kTileEnd, one); - b.setInsertionPointToStart(ktLoop.getBody()); - Value kt = ktLoop.getInductionVar(); - auto rowLoop = b.create(loc, zero, rowEnd, one); - b.setInsertionPointToStart(rowLoop.getBody()); - Value row = rowLoop.getInductionVar(); - Value remaining = b.create( - loc, b.create(loc, validK), - b.create( - loc, kt, b.create(loc, kTile))); - Value columnEnd = b.create( - loc, remaining, b.create(loc, kTile)); - auto columnLoop = b.create(loc, zero, columnEnd, one); - b.setInsertionPointToStart(columnLoop.getBody()); - Value column = columnLoop.getInductionVar(); - Value sourceRow = b.create(loc, m0, row); - Value sourceColumn = b.create( - loc, b.create(loc, k0), - b.create( - loc, - b.create( - loc, kt, b.create(loc, kTile)), - column)); - Value hardwareRowsValue = b.create( - loc, std::min(hardwareRows, tileRows)); - Value rowTile = b.create(loc, row, hardwareRowsValue); - Value rowInTile = b.create(loc, row, hardwareRowsValue); - Value tileIndex = b.create( - loc, - b.create( - loc, rowTile, - b.create(loc, thisK / kTile)), - kt); - Value packedRow = b.create( - loc, b.create(loc, tileIndex, hardwareRowsValue), - rowInTile); - Value value = b.create( - loc, op.getInput(), ValueRange{sourceRow, sourceColumn}); - b.create(loc, value, aPack, - ValueRange{packedRow, column}); - b.setInsertionPointAfter(ktLoop); - } - } - auto panelLoop = b.create(loc, zero, columns, tileSize); - b.setInsertionPointToStart(panelLoop.getBody()); - { - Value n0 = panelLoop.getInductionVar(); - Value thisN = b.create( - loc, tileSize, b.create(loc, columns, n0)); - Value aBank = allocBank(b, loc, 1, 1); - Value wBank = allocBank(b, loc, 1, 1); - auto biasPackTy = MemRefType::get({4, 4}, b.getI32Type()); - auto scalePackTy = MemRefType::get({4, 4}, b.getF32Type()); - Value biasPack = b.create(loc, biasPackTy); - Value scalePack = b.create(loc, scalePackTy); - Value zeroI32 = b.create(loc, b.getI32Type(), - b.getI32IntegerAttr(0)); - Value oneF32 = b.create(loc, b.getF32Type(), - b.getF32FloatAttr(1.0)); - b.create(loc, zeroI32, biasPack); - b.create(loc, oneF32, scalePack); - auto channelLoop = b.create(loc, zero, thisN, one); - b.setInsertionPointToStart(channelLoop.getBody()); - { - Value i = channelLoop.getInductionVar(); - Value four = b.create(loc, 4); - Value source = b.create(loc, n0, i); - Value row = b.create(loc, i, four); - Value column = b.create(loc, i, four); - Value bias = b.create(loc, op.getBias(), source); - Value scale = b.create(loc, op.getScale(), source); - b.create(loc, bias, biasPack, ValueRange{row, column}); - b.create(loc, scale, scalePack, - ValueRange{row, column}); - } - b.setInsertionPointAfter(channelLoop); - - Value biasBank = allocBank(b, loc, 1, 1); - Value biasLoaded = mvinBank(b, loc, biasPack, biasBank, 4); - Value biasState = b.create( - loc, biasLoaded.getType(), biasLoaded, createI64Const(b, loc, 0)); - Value scaleBank = allocBank(b, loc, 1, 1); - Value scaleLoaded = mvinBank(b, loc, scalePack, scaleBank, 4); - - { - int64_t tileCount = tilesPerResultBank; - Value resultBank = allocBank(b, loc, 1, 1); - Value resultState = resultBank; - SmallVector hostPacks; - - for (int64_t tile = 0; tile < tileCount; ++tile) { - for (int64_t k0 = 0; k0 < paddedK; k0 += kChunk) { - int64_t thisK = std::min(kChunk, paddedK - k0); - int64_t validK = std::min(thisK, std::max(0, K - k0)); - Value aPack = inputPacks[k0 / kChunk]; - Value aLoaded = - mvinBank(b, loc, aPack, aBank, thisK / kTile * tileRows); - Value wLoaded; - auto [weightStrides, weightOffset] = wTy.getStridesAndOffset(); - if (validK == thisK && N % kTile == 0 && weightStrides[1] == 1 && - weightStrides[0] > 0 && weightStrides[0] % kTile == 0) { - Value panel = b.create( - loc, op.getWeight(), - SmallVector{b.getIndexAttr(k0), n0}, - SmallVector{b.getIndexAttr(thisK), - b.getIndexAttr(kTile)}, - SmallVector{b.getIndexAttr(1), - b.getIndexAttr(1)}); - wLoaded = mvinBank(b, loc, panel, wBank, thisK, - weightStrides[0] / kTile); - } else { - auto wPackTy = MemRefType::get({thisK, kTile}, b.getI8Type()); - Value wPack = b.create(loc, wPackTy); - hostPacks.push_back(wPack); - b.create(loc, zeroI8, wPack); - Value validKEnd = b.create(loc, validK); - Value validNEnd = thisN; - auto weightRowLoop = - b.create(loc, zero, validKEnd, one); - b.setInsertionPointToStart(weightRowLoop.getBody()); - Value weightRow = weightRowLoop.getInductionVar(); - auto weightColumnLoop = - b.create(loc, zero, validNEnd, one); - b.setInsertionPointToStart(weightColumnLoop.getBody()); - Value weightColumn = weightColumnLoop.getInductionVar(); - Value sourceWeightRow = b.create( - loc, b.create(loc, k0), weightRow); - Value sourceWeightColumn = - b.create(loc, n0, weightColumn); - Value weightValue = b.create( - loc, op.getWeight(), - ValueRange{sourceWeightRow, sourceWeightColumn}); - b.create(loc, weightValue, wPack, - ValueRange{weightRow, weightColumn}); - b.setInsertionPointAfter(weightRowLoop); - - wLoaded = mvinBank(b, loc, wPack, wBank, thisK); - } - uint64_t cfg = matrixRs2(tileRows, kTile, thisK); - auto smatmul = b.create( - loc, resultState.getType(), aLoaded, wLoaded, resultState, - createI64Const(b, loc, static_cast(cfg)), - createI1Const(b, loc, k0 == 0), - createI1Const(b, loc, k0 + thisK == paddedK), - createI64Const(b, loc, tile * resultRows)); - resultState = smatmul.getWrBankOut(); - aBank = aLoaded; - wBank = wLoaded; - } - } - - Value outputBank = allocBank(b, loc, 1, 1); - Value converted = - finalOutput - ? b.create( - loc, outputBank.getType(), resultState, scaleLoaded, - outputBank, - createI64Const(b, loc, tileCount * resultRows), - b.getBoolAttr(op.getActivation() == 1)) - .getOutBankOut() - : b.create( - loc, outputBank.getType(), resultState, scaleLoaded, - outputBank, - createI64Const(b, loc, tileCount * resultRows), - createI64Const(b, loc, 0), createI64Const(b, loc, 0), - b.getI64IntegerAttr(tileCount * tileRows), - b.getI64IntegerAttr(1), - b.getI64IntegerAttr(tileCount * tileRows), - b.getBoolAttr(op.getActivation() == 1)) - .getOutBankOut(); - releaseBank(b, loc, resultState); - - Type outputElementType = - finalOutput ? Type(b.getF32Type()) : Type(b.getI8Type()); - auto packedOutTy = - MemRefType::get({tileCount * tileRows, kTile}, outputElementType); - Value packedOut = b.create(loc, packedOutTy); - Value stored = mvoutBank(b, loc, packedOut, converted, - finalOutput ? tileCount * resultRows - : tileCount * tileRows); - for (Value pack : hostPacks) - b.create(loc, pack); - - Value zero = b.create(loc, 0); - Value one = b.create(loc, 1); - Value rowEnd = validRows; - Value columnEnd = thisN; - auto rowLoop = b.create(loc, zero, rowEnd, one); - b.setInsertionPointToStart(rowLoop.getBody()); - Value row = rowLoop.getInductionVar(); - auto columnLoop = b.create(loc, zero, columnEnd, one); - b.setInsertionPointToStart(columnLoop.getBody()); - Value column = columnLoop.getInductionVar(); - Value value = - b.create(loc, packedOut, ValueRange{row, column}); - Value outputRow = b.create(loc, m0, row); - Value outputColumn = b.create(loc, n0, column); - b.create(loc, value, op.getOutput(), - ValueRange{outputRow, outputColumn}); - b.setInsertionPointAfter(rowLoop); - releaseBank(b, loc, stored); - b.create(loc, packedOut); - } - - releaseBank(b, loc, biasState); - releaseBank(b, loc, scaleLoaded); - b.create(loc, biasPack); - b.create(loc, scalePack); - releaseBank(b, loc, aBank); - releaseBank(b, loc, wBank); - } - b.setInsertionPointAfter(panelLoop); - for (Value pack : inputPacks) - b.create(loc, pack); - b.setInsertionPointAfter(matrixRowLoop); - b.eraseOp(op); - return success(); - } -}; - -} // namespace - -void mlir::buddy::populateSMatMulBallLowerBuckyballToBankSSAPatterns( - RewritePatternSet &patterns) { - populateMatmulRegionToBankSSAPatterns(patterns, false, 0, -1); -} - -LogicalResult mlir::buddy::lowerMatmulToBanks(MegaMatmulOp op, - PatternRewriter &rewriter) { - return MegaMatmulToBankSSAPattern(op.getContext()) - .matchAndRewrite(op, rewriter); -} diff --git a/examples/balls/smatmul/compiler/src/Conversion/LowerBuckyball/MegaKernelToBankSSAPatterns.cpp b/examples/balls/smatmul/compiler/src/Conversion/LowerBuckyball/MegaKernelToBankSSAPatterns.cpp deleted file mode 100644 index 91c01b4bc..000000000 --- a/examples/balls/smatmul/compiler/src/Conversion/LowerBuckyball/MegaKernelToBankSSAPatterns.cpp +++ /dev/null @@ -1,452 +0,0 @@ -//===- MegaKernelToBankSSAPatterns.cpp - MegaKernel to bank SSA ----------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Target/BuckyballTargetRegistry.h" -#include "Trace/TraceOps.h" -#include "Utils/BankUtils.h" -#include "llvm/Support/MathExtras.h" - -#include -#include - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { - -constexpr int64_t kTile = 16; -constexpr int64_t kInt32Rows = 64; - -class MatmulMegaKernelToBankSSAPattern : public OpRewritePattern { -public: - MatmulMegaKernelToBankSSAPattern(MLIRContext *context, bool traceMegaStages, - int64_t traceMegaStageStart, - int64_t traceMegaStageLimit) - : OpRewritePattern(context), - traceMegaStages(traceMegaStages), - traceMegaStageStart(traceMegaStageStart), - traceMegaStageLimit(traceMegaStageLimit) {} - - LogicalResult matchAndRewrite(MegaKernelOp kernel, - PatternRewriter &b) const override { - if (kernel.getBody().empty()) - return kernel.emitError("MegaKernel region must contain one block"); - - Block &body = kernel.getBody().front(); - if (body.without_terminator().empty()) - return kernel.emitError( - "MegaKernel region must contain at least one stage"); - if (!isa(body.front())) - return failure(); - - SmallVector stages; - for (Operation &op : body.without_terminator()) { - auto matmul = dyn_cast(op); - if (!matmul) - return kernel.emitError( - "MatMul MegaKernel cannot contain non-MatMul stages"); - if (matmul.getActivation() == 2) - buckyball_target::requireBuckyballBall("LutBall"); - stages.push_back(matmul); - } - - const auto &target = buckyball_target::getBuckyballTarget(); - if (target.bankWidthBits != 128 || target.bankDepth < kInt32Rows || - !llvm::isPowerOf2_64(target.bankDepth)) - return kernel.emitError( - "MatMul MegaKernel requires 128-bit power-of-two-depth banks with " - "at least 64 rows"); - if (buckyball_target::getBuckyballBallMapping("SMatMulBall").outBW != 1) - return kernel.emitError("MatMul MegaKernel requires SMatMulBall outBW=1"); - - int64_t rows = - cast(stages.front().getInput().getType()).getShape()[0]; - if (stages.size() == 1) { - if (failed(mlir::buddy::lowerMatmulToBanks(stages.front(), b))) - return failure(); - b.eraseOp(body.getTerminator()); - b.inlineBlockBefore(&body, kernel); - b.eraseOp(kernel); - return success(); - } - Value expectedInput = kernel.getInput(); - for (auto [index, stage] : llvm::enumerate(stages)) { - bool last = index + 1 == stages.size(); - auto inputTy = cast(stage.getInput().getType()); - auto weightTy = cast(stage.getWeight().getType()); - auto biasTy = cast(stage.getBias().getType()); - auto scaleTy = cast(stage.getScale().getType()); - auto lutTy = cast(stage.getLut().getType()); - auto outputTy = cast(stage.getOutput().getType()); - if (stage.getInput() != expectedInput || - (last ? stage.getOutput() != kernel.getOutput() - : stage.getOutput() == kernel.getOutput())) - return stage.emitError("MatMul stage breaks the MegaKernel data chain"); - if (!inputTy.hasStaticShape() || !weightTy.hasStaticShape() || - !biasTy.hasStaticShape() || !scaleTy.hasStaticShape() || - !lutTy.hasStaticShape() || !outputTy.hasStaticShape()) - return stage.emitError("MatMul MegaKernel requires static shapes"); - int64_t m = inputTy.getShape()[0]; - int64_t k = inputTy.getShape()[1]; - int64_t n = weightTy.getShape()[1]; - if (m <= 0 || m != rows || k <= 0 || n <= 0 || - weightTy.getShape()[0] != k || outputTy.getShape()[0] != m || - outputTy.getShape()[1] != n || biasTy.getShape()[0] != n || - scaleTy.getShape()[0] != n || lutTy.getRank() != 1 || - !lutTy.getElementType().isInteger(8) || - lutTy.getShape()[0] != (stage.getActivation() == 2 ? 256 : 1) || - stage.getActivation() < 0 || stage.getActivation() > 2 || - (last && stage.getActivation() == 2) || - (last ? !outputTy.getElementType().isF32() - : !outputTy.getElementType().isInteger(8))) - return stage.emitError("MatMul MegaKernel requires matching " - "INT8/FP32 stage shapes"); - int64_t inputBanks = index == 0 ? 0 : (k + 4 * kTile - 1) / (4 * kTile); - int64_t outputBanks = (n + 4 * kTile - 1) / (4 * kTile); - int64_t requiredBanks = inputBanks + (last ? 5 : outputBanks + 6); - if (requiredBanks > target.bankNum) - return stage.emitError("MatMul MegaKernel exceeds bank capacity"); - expectedInput = stage.getOutput(); - } - - Location loc = kernel.getLoc(); - b.setInsertionPoint(kernel); - SmallVector hostPacks; - SmallVector activationBanks; - SmallVector activationK; - SmallVector initialPacks; - SmallVector initialK; - - auto firstTy = cast(stages.front().getInput().getType()); - int64_t firstK = firstTy.getShape()[1]; - Value zeroI8 = - b.create(loc, b.getI8Type(), b.getI8IntegerAttr(0)); - Value zero = b.create(loc, 0); - Value one = b.create(loc, 1); - Value sixteen = b.create(loc, kTile); - auto rowLoop = b.create( - loc, zero, b.create(loc, rows), one); - b.setInsertionPointToStart(rowLoop.getBody()); - Value inputRow = rowLoop.getInductionVar(); - SmallVector> tracedOutputs; - int64_t paddedFirstK = (firstK + kTile - 1) / kTile * kTile; - for (int64_t k0 = 0; k0 < paddedFirstK; k0 += target.bankDepth) { - int64_t thisK = std::min(target.bankDepth, paddedFirstK - k0); - int64_t validK = std::min(thisK, firstK - k0); - Value inputPack = b.create( - loc, MemRefType::get({thisK / kTile, kTile}, b.getI8Type())); - hostPacks.push_back(inputPack); - b.create(loc, zeroI8, inputPack); - auto inputLoop = b.create( - loc, zero, b.create(loc, validK), one); - b.setInsertionPointToStart(inputLoop.getBody()); - Value localK = inputLoop.getInductionVar(); - Value sourceK = b.create( - loc, b.create(loc, k0), localK); - Value value = b.create(loc, stages.front().getInput(), - ValueRange{inputRow, sourceK}); - Value tile = b.create(loc, localK, sixteen); - Value packedColumn = b.create(loc, localK, sixteen); - b.create(loc, value, inputPack, - ValueRange{tile, packedColumn}); - b.setInsertionPointAfter(inputLoop); - initialPacks.push_back(inputPack); - initialK.push_back(thisK); - } - - for (auto [stageIndex, stage] : llvm::enumerate(stages)) { - bool last = stageIndex + 1 == stages.size(); - int64_t logicalK = - cast(stage.getInput().getType()).getShape()[1]; - int64_t n = cast(stage.getWeight().getType()).getShape()[1]; - SmallVector outputs; - SmallVector outputK; - SmallVector finalPacks; - Value packedOutput; - int64_t packedRows = 0; - Value lutBank; - if (stage.getActivation() == 2) { - Value lutPack = b.create( - loc, MemRefType::get({kTile, kTile}, b.getI8Type())); - hostPacks.push_back(lutPack); - auto lutLoop = b.create( - loc, zero, b.create(loc, 256), one); - b.setInsertionPointToStart(lutLoop.getBody()); - Value index = lutLoop.getInductionVar(); - Value row = b.create(loc, index, sixteen); - Value column = b.create(loc, index, sixteen); - Value value = b.create(loc, stage.getLut(), index); - b.create(loc, value, lutPack, ValueRange{row, column}); - b.setInsertionPointAfter(lutLoop); - lutBank = allocBank(b, loc, 1, 1); - lutBank = mvinBank(b, loc, lutPack, lutBank, 16); - } - - for (int64_t n0 = 0; n0 < n; n0 += kTile) { - int64_t validN = std::min(kTile, n - n0); - if (!last && n0 % (4 * kTile) == 0) { - packedOutput = allocBank(b, loc, 1, 1); - packedRows = 0; - } - Value biasPack = b.create( - loc, MemRefType::get({4, 4}, b.getI32Type())); - Value scalePack = b.create( - loc, MemRefType::get({4, 4}, b.getF32Type())); - hostPacks.push_back(biasPack); - hostPacks.push_back(scalePack); - Value zeroI32 = b.create(loc, b.getI32Type(), - b.getI32IntegerAttr(0)); - Value oneF32 = b.create(loc, b.getF32Type(), - b.getF32FloatAttr(1.0)); - b.create(loc, zeroI32, biasPack); - b.create(loc, oneF32, scalePack); - auto channelLoop = b.create( - loc, zero, b.create(loc, validN), one); - b.setInsertionPointToStart(channelLoop.getBody()); - Value channel = channelLoop.getInductionVar(); - Value sourceChannel = b.create( - loc, b.create(loc, n0), channel); - Value group = b.create( - loc, channel, b.create(loc, 4)); - Value lane = b.create( - loc, channel, b.create(loc, 4)); - Value bias = - b.create(loc, stage.getBias(), sourceChannel); - Value scale = - b.create(loc, stage.getScale(), sourceChannel); - b.create(loc, bias, biasPack, ValueRange{group, lane}); - b.create(loc, scale, scalePack, - ValueRange{group, lane}); - b.setInsertionPointAfter(channelLoop); - - Value biasBank = allocBank(b, loc, 1, 1); - Value biasLoaded = mvinBank(b, loc, biasPack, biasBank, 4); - Value biasState = b.create( - loc, biasLoaded.getType(), biasLoaded, createI64Const(b, loc, 0)); - Value scaleBank = allocBank(b, loc, 1, 1); - Value scaleLoaded = mvinBank(b, loc, scalePack, scaleBank, 4); - Value weightBank = allocBank(b, loc, 1, 1); - Value resultState = allocBank(b, loc, 1, 1); - - int64_t kBase = 0; - size_t chunkCount = - stageIndex == 0 ? initialPacks.size() : activationBanks.size(); - for (size_t chunk = 0; chunk < chunkCount; ++chunk) { - int64_t thisK = - stageIndex == 0 ? initialK[chunk] : activationK[chunk]; - int64_t validK = std::min(thisK, logicalK - kBase); - Value activation; - if (stageIndex == 0) { - Value inputBank = allocBank(b, loc, 1, 1); - activation = - mvinBank(b, loc, initialPacks[chunk], inputBank, thisK / kTile); - } else { - activation = activationBanks[chunk]; - } - auto weightTy = cast(stage.getWeight().getType()); - auto [strides, offset] = weightTy.getStridesAndOffset(); - if (validK == thisK && validN == kTile && strides[1] == 1 && - strides[0] > 0 && strides[0] % kTile == 0) { - Value panel = b.create( - loc, stage.getWeight(), - SmallVector{b.getIndexAttr(kBase), - b.getIndexAttr(n0)}, - SmallVector{b.getIndexAttr(thisK), - b.getIndexAttr(kTile)}, - SmallVector{b.getIndexAttr(1), - b.getIndexAttr(1)}); - weightBank = - mvinBank(b, loc, panel, weightBank, thisK, strides[0] / kTile); - } else { - Value weightPack = b.create( - loc, MemRefType::get({thisK, kTile}, b.getI8Type())); - hostPacks.push_back(weightPack); - b.create(loc, zeroI8, weightPack); - auto kLoop = b.create( - loc, zero, b.create(loc, validK), one); - b.setInsertionPointToStart(kLoop.getBody()); - Value localK = kLoop.getInductionVar(); - auto nLoop = b.create( - loc, zero, b.create(loc, validN), one); - b.setInsertionPointToStart(nLoop.getBody()); - Value localN = nLoop.getInductionVar(); - Value sourceK = b.create( - loc, b.create(loc, kBase), localK); - Value sourceN = b.create( - loc, b.create(loc, n0), localN); - Value weight = b.create( - loc, stage.getWeight(), ValueRange{sourceK, sourceN}); - b.create(loc, weight, weightPack, - ValueRange{localK, localN}); - b.setInsertionPointAfter(kLoop); - - weightBank = mvinBank(b, loc, weightPack, weightBank, thisK); - } - auto smatmul = b.create( - loc, resultState.getType(), activation, weightBank, resultState, - createI64ConstU(b, loc, matrixRs2(1, kTile, thisK)), - createI1Const(b, loc, chunk == 0), - createI1Const(b, loc, chunk + 1 == chunkCount), - createI64Const(b, loc, 0)); - resultState = smatmul.getWrBankOut(); - if (stageIndex == 0) - releaseBank(b, loc, activation); - kBase += thisK; - } - - if (last) { - Value outputBank = allocBank(b, loc, 1, 1); - Value converted = b.create( - loc, outputBank.getType(), resultState, scaleLoaded, outputBank, - createI64Const(b, loc, 4), - b.getBoolAttr(stage.getActivation() == 1)); - releaseBank(b, loc, resultState); - Value packed = b.create( - loc, MemRefType::get({1, kTile}, b.getF32Type())); - hostPacks.push_back(packed); - Value stored = mvoutBank(b, loc, packed, converted, 4); - releaseBank(b, loc, stored); - finalPacks.push_back(packed); - } else { - Value converted = b.create( - loc, packedOutput.getType(), resultState, scaleLoaded, - packedOutput, createI64Const(b, loc, 4), - createI64Const(b, loc, packedRows), createI64Const(b, loc, 0), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(1), - b.getBoolAttr(stage.getActivation() == 1)); - releaseBank(b, loc, resultState); - packedOutput = converted; - ++packedRows; - if (packedRows == 4 || n0 + kTile >= n) { - if (lutBank) { - Value lutOutput = allocBank(b, loc, 1, 1); - OperationState lut(loc, "buckyball.bank_lut"); - lut.addOperands({packedOutput, lutBank, lutOutput, - createI64Const(b, loc, packedRows)}); - lut.addTypes(lutOutput.getType()); - Value transformed = b.create(lut)->getResult(0); - releaseBank(b, loc, packedOutput); - packedOutput = transformed; - } - outputs.push_back(packedOutput); - outputK.push_back(packedRows * kTile); - packedOutput = {}; - } - } - releaseBank(b, loc, biasState); - releaseBank(b, loc, scaleLoaded); - releaseBank(b, loc, weightBank); - } - if (lutBank) - releaseBank(b, loc, lutBank); - - bool traceStage = - traceMegaStages && !last && - static_cast(stageIndex) >= traceMegaStageStart && - (traceMegaStageLimit < 0 || - static_cast(stageIndex) < traceMegaStageLimit); - if (traceStage) { - int64_t channelBase = 0; - for (auto [bankIndex, output] : llvm::enumerate(outputs)) { - int64_t rows = outputK[bankIndex] / kTile; - Value pack = b.create( - loc, MemRefType::get({rows, kTile}, b.getI8Type())); - hostPacks.push_back(pack); - outputs[bankIndex] = mvoutBank(b, loc, pack, output, rows); - int64_t validChannels = std::min(outputK[bankIndex], n - channelBase); - auto channelLoop = b.create( - loc, zero, b.create(loc, validChannels), - one); - b.setInsertionPointToStart(channelLoop.getBody()); - Value channel = channelLoop.getInductionVar(); - Value row = b.create(loc, channel, sixteen); - Value column = b.create(loc, channel, sixteen); - Value value = - b.create(loc, pack, ValueRange{row, column}); - Value outputChannel = b.create( - loc, b.create(loc, channelBase), channel); - b.create(loc, value, stage.getOutput(), - ValueRange{inputRow, outputChannel}); - b.setInsertionPointAfter(channelLoop); - channelBase += outputK[bankIndex]; - } - tracedOutputs.push_back( - {stage.getOutput(), b.getI64IntegerAttr(1000 + stageIndex), false}); - } - - for (Value activation : activationBanks) - releaseBank(b, loc, activation); - activationBanks = outputs; - activationK = outputK; - - if (last) { - for (auto [panel, packed] : llvm::enumerate(finalPacks)) { - int64_t n0 = panel * kTile; - int64_t validN = std::min(kTile, n - n0); - auto outputLoop = b.create( - loc, zero, b.create(loc, validN), one); - b.setInsertionPointToStart(outputLoop.getBody()); - Value localN = outputLoop.getInductionVar(); - Value output = - b.create(loc, packed, ValueRange{zero, localN}); - Value outputN = b.create( - loc, b.create(loc, n0), localN); - b.create(loc, output, kernel.getOutput(), - ValueRange{inputRow, outputN}); - b.setInsertionPointAfter(outputLoop); - } - if (traceMegaStages && - traceMegaStageStart <= static_cast(stageIndex) && - (traceMegaStageLimit < 0 || - static_cast(stageIndex) < traceMegaStageLimit)) { - tracedOutputs.push_back( - {stage.getOutput(), b.getI64IntegerAttr(1001), true}); - } - } - } - - for (Value bank : activationBanks) - releaseBank(b, loc, bank); - for (Value pack : hostPacks) - b.create(loc, pack); - b.setInsertionPointAfter(rowLoop); - for (auto [output, id, last] : tracedOutputs) { - auto trace = b.create<::buddy::trace::EndOp>( - loc, output.getType(), output, id, - b.getStringAttr(last ? "classifier-output" : "mega-matmul-stage")); - trace->setAttr("id_path", b.getArrayAttr({id})); - if (!last) - trace->setAttr("buckyball.stage_trace", b.getUnitAttr()); - } - b.eraseOp(kernel); - return success(); - } - -private: - bool traceMegaStages; - int64_t traceMegaStageStart; - int64_t traceMegaStageLimit; -}; - -} // namespace - -namespace mlir::buddy { -void populateMatmulRegionToBankSSAPatterns(RewritePatternSet &patterns, - bool traceMegaStages, - int64_t traceMegaStageStart, - int64_t traceMegaStageLimit) { - patterns.add( - patterns.getContext(), traceMegaStages, traceMegaStageStart, - traceMegaStageLimit); -} -} // namespace mlir::buddy diff --git a/examples/balls/smatmul/compiler/src/Conversion/LowerTileToBuckyball/TileMatMulPatterns.cpp b/examples/balls/smatmul/compiler/src/Conversion/LowerTileToBuckyball/TileMatMulPatterns.cpp deleted file mode 100644 index abb98389d..000000000 --- a/examples/balls/smatmul/compiler/src/Conversion/LowerTileToBuckyball/TileMatMulPatterns.cpp +++ /dev/null @@ -1,324 +0,0 @@ -//===- TileMatMulPatterns.cpp - tile.matmul -> buckyball.smatmul_matmul -//----===// -// -// Non-quantized matmul is split into bank-sized panels. Quantized matmul -// retains its full N dimension so bank lowering can quantize one activation -// chunk and hand it to every output panel. -// -//===----------------------------------------------------------------------===// - -#include "Conversion/LowerTileToBuckyball/LowerTileToBuckyball.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Tile/TileOps.h" - -#include - -using namespace mlir; -using namespace ::buddy::buckyball; -using namespace ::buddy::tile; -using mlir::buddy::aMvinDepthLines; -using mlir::buddy::bMvinDepthLines; -using mlir::buddy::ceilDiv; -using mlir::buddy::cMvoutDepthLines; -using mlir::buddy::kBankLane; - -namespace { - -class TileMatMulLowering : public OpRewritePattern { -public: - explicit TileMatMulLowering(MLIRContext *context, int64_t /*bankWidthBytes*/, - int64_t bankDepth, int64_t /*bankNum*/) - : OpRewritePattern(context), bankDepth(bankDepth) {} - - LogicalResult matchAndRewrite(TileMatMulOp tileMatMulOp, - PatternRewriter &rewriter) const override { - Location loc = tileMatMulOp.getLoc(); - - Value aMemArray = tileMatMulOp.getAMemArray(); - Value bMemArray = tileMatMulOp.getBMemArray(); - Value cMemArray = tileMatMulOp.getCMemArray(); - - auto aType = cast(aMemArray.getType()); - auto bType = cast(bMemArray.getType()); - auto cType = cast(cMemArray.getType()); - - auto aShape = aType.getShape(); - auto bShape = bType.getShape(); - auto cShape = cType.getShape(); - size_t M = aShape[aShape.size() - 2]; - size_t K = aShape[aShape.size() - 1]; - size_t N = bShape[bShape.size() - 1]; - - if (bShape[bShape.size() - 2] != (int64_t)K || - cShape[cShape.size() - 2] != (int64_t)M || - cShape[cShape.size() - 1] != (int64_t)N) - return tileMatMulOp.emitError("matmul input/output shapes mismatch"); - - const bool isQuantized = aType.getElementType().isF32() && - bType.getElementType().isInteger(8) && - cType.getElementType().isF32(); - const bool isWideFloat = - (aType.getElementType().isF32() || aType.getElementType().isBF16()) && - bType.getElementType() == aType.getElementType() && - cType.getElementType() == aType.getElementType(); - const bool isInt8 = aType.getElementType().isInteger(8) && - bType.getElementType().isInteger(8) && - cType.getElementType().isInteger(32); - if (!isQuantized && !isWideFloat && !isInt8) - return tileMatMulOp.emitError( - "requires matching floating-point types, FP32 x INT8 -> FP32, " - "or INT8 x INT8 -> INT32"); - auto dwAddrAttr = tileMatMulOp->getAttrOfType("dw_addr"); - auto dwBytesAttr = tileMatMulOp->getAttrOfType("dw_bytes"); - auto perChannelAttr = tileMatMulOp->getAttrOfType("per_channel"); - if (isQuantized && (!dwAddrAttr || !dwBytesAttr || !perChannelAttr || - dwAddrAttr.getInt() < 16)) - return tileMatMulOp.emitError( - "quantized matmul requires RAX Dw metadata"); - - constexpr size_t mTileSize = 8; - size_t M_pad = ceilDiv(M, mTileSize) * mTileSize; - size_t K_pad = ceilDiv(K, kBankLane) * kBankLane; - size_t N_pad = ceilDiv(N, kBankLane) * kBankLane; - bool needPadding = (M_pad != M) || (K_pad != K) || (N_pad != N); - - Value aMemArrayPadded = aMemArray; - Value bMemArrayPadded = bMemArray; - Value cMemArrayPadded = cMemArray; - - if (needPadding) { - auto aPadType = MemRefType::get({(int64_t)M_pad, (int64_t)K_pad}, - aType.getElementType()); - auto bPadType = MemRefType::get({(int64_t)K_pad, (int64_t)N_pad}, - bType.getElementType()); - auto cPadType = MemRefType::get({(int64_t)M_pad, (int64_t)N_pad}, - cType.getElementType()); - - aMemArrayPadded = rewriter.create(loc, aPadType); - bMemArrayPadded = rewriter.create(loc, bPadType); - cMemArrayPadded = rewriter.create(loc, cPadType); - - Value aZero = rewriter.create( - loc, aType.getElementType(), - rewriter.getZeroAttr(aType.getElementType())); - Value bZero = rewriter.create( - loc, bType.getElementType(), - rewriter.getZeroAttr(bType.getElementType())); - Value cZero = rewriter.create( - loc, cType.getElementType(), - rewriter.getZeroAttr(cType.getElementType())); - rewriter.create(loc, aZero, aMemArrayPadded); - rewriter.create(loc, bZero, bMemArrayPadded); - rewriter.create(loc, cZero, cMemArrayPadded); - - Value aView = rewriter.create( - loc, aMemArrayPadded, - SmallVector{rewriter.getIndexAttr(0), - rewriter.getIndexAttr(0)}, - SmallVector{rewriter.getIndexAttr(M), - rewriter.getIndexAttr(K)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - rewriter.create(loc, aMemArray, aView); - - Value bView = rewriter.create( - loc, bMemArrayPadded, - SmallVector{rewriter.getIndexAttr(0), - rewriter.getIndexAttr(0)}, - SmallVector{rewriter.getIndexAttr(K), - rewriter.getIndexAttr(N)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - rewriter.create(loc, bMemArray, bView); - } - - size_t M_tiling = needPadding ? M_pad : M; - size_t K_tiling = needPadding ? K_pad : K; - size_t N_tiling = needPadding ? N_pad : N; - - if (bankDepth <= 0 || (size_t)bankDepth % kBankLane != 0) - return tileMatMulOp.emitError( - "bankDepth must be a positive multiple of bank lane"); - if (M_tiling % mTileSize || K_tiling % kBankLane || N_tiling % kBankLane) - return tileMatMulOp.emitError( - "M must be a multiple of 8 and K/N must be multiples of bank lane"); - if (isQuantized) { - const int64_t dwRequired = perChannelAttr.getValue() ? N_tiling * 4 : 4; - if (dwBytesAttr.getInt() < dwRequired) - return tileMatMulOp.emitError("Dw scale image does not cover padded N"); - } - - // Bank unit on M (depth) and N (lane). Quantized matmul carries one - // logical K into SMatMul so FP2INT computes one activation scale. - const size_t nTileSize = kBankLane; - const size_t kTileSize = - (isWideFloat || isQuantized) ? K_tiling : kBankLane; - if (M_tiling % mTileSize != 0) - return tileMatMulOp.emitError("M does not split into bank-depth units"); - - if (!isWideFloat && !isQuantized) { - if (aMvinDepthLines(mTileSize, kTileSize) > (size_t)bankDepth || - bMvinDepthLines(kTileSize, nTileSize) > (size_t)bankDepth || - cMvoutDepthLines(mTileSize, nTileSize) > (size_t)bankDepth) - return tileMatMulOp.emitError( - "bank-unit matmul exceeds 8-bit mvin or 32-bit mvout depth"); - } - - const size_t kTileNum = ceilDiv(K_tiling, kTileSize); - - OpBuilder::InsertionGuard guard(rewriter); - Value zeroIdx = rewriter.create(loc, 0); - Value mStepVal = rewriter.create(loc, mTileSize); - Value kStepVal = rewriter.create(loc, kTileSize); - Value mUpperVal = rewriter.create(loc, M_tiling); - Value kUpperVal = rewriter.create(loc, K_tiling); - - auto mLoop = rewriter.create(loc, zeroIdx, mUpperVal, mStepVal); - rewriter.setInsertionPointToStart(mLoop.getBody()); - Value mIv = mLoop.getInductionVar(); - - if (isQuantized) { - Value aTile = rewriter.create( - loc, aMemArrayPadded, - SmallVector{mIv, rewriter.getIndexAttr(0)}, - SmallVector{rewriter.getIndexAttr(mTileSize), - rewriter.getIndexAttr(K_tiling)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - Value bTile = rewriter.create( - loc, bMemArrayPadded, - SmallVector{rewriter.getIndexAttr(0), - rewriter.getIndexAttr(0)}, - SmallVector{rewriter.getIndexAttr(K_tiling), - rewriter.getIndexAttr(N_tiling)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - Value cTile = rewriter.create( - loc, cMemArrayPadded, - SmallVector{mIv, rewriter.getIndexAttr(0)}, - SmallVector{rewriter.getIndexAttr(mTileSize), - rewriter.getIndexAttr(N_tiling)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - Value zero = rewriter.create( - loc, cType.getElementType(), - rewriter.getZeroAttr(cType.getElementType())); - rewriter.create(loc, zero, cTile); - auto smatmul = rewriter.create(loc, aTile, bTile, cTile); - smatmul->setAttr("dwAddr", dwAddrAttr); - smatmul->setAttr("dwBytes", dwBytesAttr); - smatmul->setAttr("perChannel", perChannelAttr); - } else { - // N is static. Non-quantized matmul remains one physical N panel per op. - for (size_t n0 = 0; n0 < N_tiling; n0 += nTileSize) { - Value cTile = rewriter.create( - loc, cMemArrayPadded, - SmallVector{mIv, rewriter.getIndexAttr(n0)}, - SmallVector{rewriter.getIndexAttr(mTileSize), - rewriter.getIndexAttr(nTileSize)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - - auto cElemType = cType.getElementType(); - Value zero = rewriter.create( - loc, cElemType, rewriter.getZeroAttr(cElemType)); - rewriter.create(loc, zero, cTile); - - if (kTileNum == 1) { - Value aTile = rewriter.create( - loc, aMemArrayPadded, - SmallVector{mIv, rewriter.getIndexAttr(0)}, - SmallVector{rewriter.getIndexAttr(mTileSize), - rewriter.getIndexAttr(kTileSize)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - Value bTile = rewriter.create( - loc, bMemArrayPadded, - SmallVector{rewriter.getIndexAttr(0), - rewriter.getIndexAttr(n0)}, - SmallVector{rewriter.getIndexAttr(kTileSize), - rewriter.getIndexAttr(nTileSize)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - auto smatmul = - rewriter.create(loc, aTile, bTile, cTile); - } else { - auto kLoop = - rewriter.create(loc, zeroIdx, kUpperVal, kStepVal); - rewriter.setInsertionPointToStart(kLoop.getBody()); - Value kIv = kLoop.getInductionVar(); - Value aTile = rewriter.create( - loc, aMemArrayPadded, SmallVector{mIv, kIv}, - SmallVector{rewriter.getIndexAttr(mTileSize), - rewriter.getIndexAttr(kTileSize)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - Value bTile = rewriter.create( - loc, bMemArrayPadded, - SmallVector{kIv, rewriter.getIndexAttr(n0)}, - SmallVector{rewriter.getIndexAttr(kTileSize), - rewriter.getIndexAttr(nTileSize)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - if (isWideFloat) - return tileMatMulOp.emitError( - "FP32 matmul must pass full K in one smatmul_matmul"); - // 8-bit legalize overwrites C; accumulate via host partial. - auto partialTy = MemRefType::get( - {(int64_t)mTileSize, (int64_t)nTileSize}, cElemType); - Value partial = rewriter.create(loc, partialTy); - rewriter.create(loc, zero, partial); - rewriter.create(loc, aTile, bTile, partial); - rewriter.create( - loc, ValueRange{cTile, partial}, ValueRange{cTile}, - linalg::ElementwiseKind::add); - rewriter.create(loc, partial); - rewriter.setInsertionPointAfter(kLoop); - } - } - } - - rewriter.setInsertionPointAfter(mLoop); - - if (needPadding) { - Value cView = rewriter.create( - loc, cMemArrayPadded, - SmallVector{rewriter.getIndexAttr(0), - rewriter.getIndexAttr(0)}, - SmallVector{rewriter.getIndexAttr(M), - rewriter.getIndexAttr(N)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - rewriter.create(loc, cView, cMemArray); - rewriter.create(loc, aMemArrayPadded); - rewriter.create(loc, bMemArrayPadded); - rewriter.create(loc, cMemArrayPadded); - } - - rewriter.eraseOp(tileMatMulOp); - return success(); - } - -private: - int64_t bankDepth; -}; - -} // namespace - -namespace mlir::buddy { -void populateSMatMulBallTileLoweringPatterns(RewritePatternSet &patterns, - int64_t bankWidthBytes, - int64_t bankDepth, - int64_t bankNum) { - patterns.add(patterns.getContext(), bankWidthBytes, - bankDepth, bankNum); -} -} // namespace mlir::buddy diff --git a/examples/balls/smatmul/compiler/src/Dialect/Buckyball/SMatMulBall.td b/examples/balls/smatmul/compiler/src/Dialect/Buckyball/SMatMulBall.td deleted file mode 100644 index 0d51988e1..000000000 --- a/examples/balls/smatmul/compiler/src/Dialect/Buckyball/SMatMulBall.td +++ /dev/null @@ -1,71 +0,0 @@ -//===-- SMatMulBall.td - SMatMulBall ops -------*- tablegen -*-===// - -#ifndef SMATMULBALL_OPS -#define SMATMULBALL_OPS - -include "Buckyball.td" -include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.td" - -def SMatMulMatmulOp : Buckyball_Op<"smatmul_matmul"> { - let summary = "Logical matmul lowered to SMatMulBall"; - let description = [{ - Performs C[M,N] = A[M,K] x B[K,N] with int8 inputs / int32 output. - Memrefs must already match the SMatMulBall bank tile layout. Logical N - may exceed 16; bank lowering emits one 16-lane C panel at a time. - }]; - let arguments = (ins MemRefRankOf<[AnyType], [2]>:$aMemArray, - MemRefRankOf<[AnyType], [2]>:$bMemArray, - MemRefRankOf<[AnyType], [2]>:$cMemArray, - DefaultValuedAttr:$dwAddr, - DefaultValuedAttr:$dwBytes, - DefaultValuedAttr:$perChannel); - let assemblyFormat = [{$aMemArray $bMemArray $cMemArray prop-dict attr-dict `:` - type($aMemArray) type($bMemArray) type($cMemArray)}]; -} - -def SMatMulOp : Buckyball_Op<"smatmul"> { - let summary = "SMatMulBall compute"; - let description = [{ - ISA: SMATMUL_OS. `config` contains rows, cols, and k. - }]; - let arguments = (ins I64:$op1BankId, I64:$op2BankId, - I64:$resultBankId, I64:$config, - I1:$first, I1:$last, I64:$outputBase); - let assemblyFormat = "$op1BankId `,` $op2BankId `,` $resultBankId `,` " - "$config `,` $first `,` $last `,` $outputBase " - "attr-dict `:` " - "type($op1BankId)"; -} - -def SMatMulBiasOp : Buckyball_Op<"smatmul_bias"> { - let summary = "Preload one 16-channel INT32 bias vector"; - let description = [{ - Reads four 128-bit SRAM rows containing 16 INT32 bias values. SMatMulBall - uses the retained vector to initialize every new 16x16 accumulator tile. - }]; - let arguments = (ins I64:$biasBankId, I64:$inputBase); - let assemblyFormat = "$biasBankId `,` $inputBase attr-dict `:` " - "type($biasBankId)"; -} - -def BankSMatMulOp : Buckyball_Op<"bank_smatmul"> { - let summary = "SMatMulBall compute with virtual bank handles"; - let arguments = (ins I64:$op1Bank, I64:$op2Bank, I64:$wrBank, I64:$config, - I1:$first, I1:$last, I64:$outputBase); - let results = (outs I64:$wrBankOut); - let assemblyFormat = - "$op1Bank $op2Bank $wrBank $config $first $last $outputBase " - "attr-dict `:` " - "type($op1Bank) type($op2Bank) type($wrBank) type($config)"; -} - -def BankSMatMulBiasOp : Buckyball_Op<"bank_smatmul_bias"> { - let summary = "SMatMul bias preload with a virtual bank handle"; - let arguments = (ins I64:$biasBank, I64:$inputBase); - let results = (outs I64:$biasBankOut); - let assemblyFormat = "$biasBank $inputBase attr-dict `:` " - "type($biasBank) type($inputBase)"; -} - - -#endif // SMATMULBALL_OPS diff --git a/examples/balls/smatmul/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/balls/smatmul/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index a7df304f8..000000000 --- a/examples/balls/smatmul/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,263 +0,0 @@ -//===- LegalizeForLLVMExport.cpp - SMatMulBall LLVM lowering --------------===// - -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" -#include "Utils/BankUtils.h" - -#include "llvm/Support/ErrorHandling.h" - -using namespace mlir; -using namespace buddy::buckyball; -using namespace buddy::buckyball::legalize; - -namespace { - -uint64_t matrixCfg(uint64_t rows, uint64_t cols, bool first = true, - bool last = true) { - if (rows == 0 || rows > 0xfff || (rows != 1 && rows % 8) || cols != 16) - llvm::report_fatal_error( - "matrix cfg: rows must be 1 or a multiple of 8 and cols must be 16"); - return fieldBits(rows, 0, 11) | fieldBits(cols, 12, 23) | - (uint64_t(first) << 24) | (uint64_t(last) << 25); -} - -struct SMatMulMatmulLowering : public ConvertOpToLLVMPattern { - SMatMulMatmulLowering(LLVMTypeConverter &converter, bool /*stable*/) - : ConvertOpToLLVMPattern(converter) {} - - LogicalResult - matchAndRewrite(SMatMulMatmulOp op, OpAdaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("SMatMulBall"); - Location loc = op.getLoc(); - Value aMem = op.getAMemArray(); - Value bMem = op.getBMemArray(); - Value cMem = op.getCMemArray(); - - auto aTy = dyn_cast(aMem.getType()); - auto bTy = dyn_cast(bMem.getType()); - auto cTy = dyn_cast(cMem.getType()); - if (!aTy || !bTy || !cTy || !aTy.hasStaticShape() || - !bTy.hasStaticShape() || !cTy.hasStaticShape()) - return rewriter.notifyMatchFailure( - op, "buckyball.smatmul_matmul requires static memref shapes"); - - uint64_t m = aTy.getShape()[0]; - uint64_t k = aTy.getShape()[1]; - uint64_t kb = bTy.getShape()[0]; - uint64_t n = bTy.getShape()[1]; - if (k != kb || cTy.getShape()[0] != (int64_t)m || - cTy.getShape()[1] != (int64_t)n) - return rewriter.notifyMatchFailure(op, "matmul shapes mismatch"); - const int64_t bankDepth = buckyball_target::getBuckyballTarget().bankDepth; - if (bankDepth <= 0) - return op.emitError("smatmul lowering requires target bankDepth > 0"); - // A occupies m lines, B occupies k lines, and packed int32 C occupies 2m. - if (m == 0 || n != 16 || k != 16 || m % 8 != 0 || - m > static_cast(bankDepth / 2) || - k > static_cast(bankDepth)) - return rewriter.notifyMatchFailure( - op, "SMatMul requires K/N exactly 16 and A/B/C to fit their banks"); - if (!aTy.getElementType().isInteger(8) || - !bTy.getElementType().isInteger(8) || - !cTy.getElementType().isInteger(32)) - return rewriter.notifyMatchFailure( - op, "SMatMul Ball supports only 8-bit A/B with 32-bit C"); - - const uint64_t aBank = 0; - const uint64_t bBank = 1; - const uint64_t cBank = 2; - uint64_t depthA = m; // K <= 16 => one bank row per M row - uint64_t depthB = k; - uint64_t depthC = 2 * m; - - emitMset(rewriter, loc, aBank, 1, 1, 1); - emitMset(rewriter, loc, bBank, 1, 1, 1); - emitMset(rewriter, loc, cBank, 1, 2, 1); - - Value aPtr = extractPtr(rewriter, loc, aMem); - Value bPtr = extractPtr(rewriter, loc, bMem); - auto packedType = MemRefType::get({static_cast(depthC), 8}, - cTy.getElementType()); - Value packed = rewriter.create(loc, packedType); - Value packedPtr = extractPtr(rewriter, loc, packed); - - Value rs1A = packRs1BankIter(rewriter, loc, cstI64(rewriter, loc, aBank), - cstI64(rewriter, loc, depthA)); - Value rs2A = - packRs2MemStride(rewriter, loc, aPtr, cstI64(rewriter, loc, 1)); - - rewriter.create(loc, rs1A, rs2A); - - Value rs1B = packRs1BankIter(rewriter, loc, cstI64(rewriter, loc, bBank), - cstI64(rewriter, loc, depthB)); - Value rs2B = - packRs2MemStride(rewriter, loc, bPtr, cstI64(rewriter, loc, 1)); - - rewriter.create(loc, rs1B, rs2B); - - rewriter.create( - loc, - packRs1BanksIter(rewriter, loc, cstI64(rewriter, loc, aBank), - cstI64(rewriter, loc, bBank), - cstI64(rewriter, loc, cBank), - cstI64(rewriter, loc, k)), - cstI64(rewriter, loc, matrixCfg(m, n)), - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("SMATMUL_OS"))); - - Value rs1C = packRs1BankIter(rewriter, loc, cstI64(rewriter, loc, cBank), - cstI64(rewriter, loc, depthC)); - Value rs2C = - packRs2MemStride(rewriter, loc, packedPtr, cstI64(rewriter, loc, 1)); - - rewriter.create(loc, rs1C, rs2C); - - Value indexZero = rewriter.create(loc, 0); - Value indexOne = rewriter.create(loc, 1); - Value indexTwo = rewriter.create(loc, 2); - Value indexEight = rewriter.create(loc, 8); - Value indexM = rewriter.create(loc, m); - Value indexN = rewriter.create(loc, n); - auto rowLoop = - rewriter.create(loc, indexZero, indexM, indexOne); - rewriter.setInsertionPointToStart(rowLoop.getBody()); - Value row = rowLoop.getInductionVar(); - auto columnLoop = - rewriter.create(loc, indexZero, indexN, indexOne); - rewriter.setInsertionPointToStart(columnLoop.getBody()); - Value column = columnLoop.getInductionVar(); - Value packedRow = rewriter.create( - loc, rewriter.create(loc, row, indexTwo), - rewriter.create(loc, column, indexEight)); - Value packedColumn = - rewriter.create(loc, column, indexEight); - Value value = rewriter.create( - loc, packed, ValueRange{packedRow, packedColumn}); - rewriter.create(loc, value, cMem, ValueRange{row, column}); - rewriter.setInsertionPointAfter(rowLoop); - rewriter.create(loc, packed); - - emitMset(rewriter, loc, aBank, 0, 0, 0); - emitMset(rewriter, loc, bBank, 0, 0, 0); - emitMset(rewriter, loc, cBank, 0, 0, 0); - - rewriter.eraseOp(op); - return success(); - } -}; - -struct SMatMulLowering : public ConvertOpToLLVMPattern { - SMatMulLowering(LLVMTypeConverter &converter, bool, int64_t bankDepth) - : ConvertOpToLLVMPattern(converter), bankDepth(bankDepth) {} - - LogicalResult - matchAndRewrite(SMatMulOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("SMatMulBall"); - Location loc = op.getLoc(); - auto config = op.getConfig().getDefiningOp(); - if (!config) - return op.emitError("smatmul config must be a constant"); - auto configAttr = dyn_cast(config.getValue()); - if (!configAttr) - return op.emitError("smatmul config must be an integer constant"); - uint64_t cfg = configAttr.getValue().getZExtValue(); - uint64_t rows = cfg & 0xfff; - uint64_t cols = (cfg >> 12) & 0xfff; - uint64_t k = (cfg >> 24) & 0xfff; - if (rows == 0 || (rows != 1 && rows % 8) || cols != 16 || k == 0 || - k % 16 || rows * 4 > static_cast(bankDepth)) - return op.emitError("SMatMul matrix shape or C footprint is invalid"); - auto base = adaptor.getOutputBase().getDefiningOp(); - auto baseAttr = - base ? dyn_cast(base.getValue()) : IntegerAttr(); - if (!baseAttr || baseAttr.getInt() < 0 || baseAttr.getInt() > 63 || - baseAttr.getInt() + rows * 4 > bankDepth) - return op.emitError( - "SMatMul outputBase must be constant and fit C in bank"); - Value rs1 = packRs1BanksIter( - rewriter, loc, adaptor.getOp1BankId(), adaptor.getOp2BankId(), - adaptor.getResultBankId(), cstI64(rewriter, loc, k)); - Value rs2 = cstI64(rewriter, loc, matrixCfg(rows, cols, false, false)); - Value first = rewriter.create(loc, rewriter.getI64Type(), - adaptor.getFirst()); - Value last = rewriter.create(loc, rewriter.getI64Type(), - adaptor.getLast()); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, first, cstI64(rewriter, loc, 24))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, last, cstI64(rewriter, loc, 25))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getOutputBase(), - cstI64(rewriter, loc, 26))); - rewriter.replaceOpWithNewOp( - op, rs1, rs2, - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("SMATMUL_OS"))); - return success(); - } - -private: - int64_t bankDepth; -}; - -struct SMatMulBiasLowering : public ConvertOpToLLVMPattern { - SMatMulBiasLowering(LLVMTypeConverter &converter, bool) - : ConvertOpToLLVMPattern(converter) {} - - LogicalResult - matchAndRewrite(SMatMulBiasOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("SMatMulBall"); - Location loc = op.getLoc(); - auto base = adaptor.getInputBase().getDefiningOp(); - auto baseAttr = - base ? dyn_cast(base.getValue()) : IntegerAttr(); - if (!baseAttr || baseAttr.getInt() < 0 || - baseAttr.getInt() + 4 > - static_cast( - buckyball_target::getBuckyballTarget().bankDepth)) - return op.emitError( - "SMatMul bias inputBase must be constant and fit in bank"); - Value rs1 = packRs1BankIter(rewriter, loc, adaptor.getBiasBankId(), - cstI64(rewriter, loc, 4)); - rewriter.replaceOpWithNewOp( - op, rs1, adaptor.getInputBase(), - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("SMATMUL_BIAS"))); - return success(); - } -}; - -} // namespace - -namespace mlir::buddy::buckyball { -void populateSMatMulBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth) { - patterns.add(converter, stable); - patterns.add(converter, stable, bankDepth); - patterns.add(converter, stable); -} - -void configureSMatMulBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool /*stable*/) { - target.addIllegalOp(); - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/examples/balls/smatmul/configs/SMatMulBallParam.scala b/examples/balls/smatmul/configs/SMatMulBallParam.scala deleted file mode 100644 index c103f17ae..000000000 --- a/examples/balls/smatmul/configs/SMatMulBallParam.scala +++ /dev/null @@ -1,21 +0,0 @@ -package examples.balls.smatmul.configs - -import framework.balldomain.configs.BallParamLoader -import framework.top.GlobalConfig - -case class SMatMulBallParam( - tileRows: Int, - tileCols: Int) - -object SMatMulBallParam { - private val ballName = "SMatMulBall" - - def apply(b: GlobalConfig): SMatMulBallParam = { - val tbl = BallParamLoader.ballTable(b, ballName) - SMatMulBallParam( - tileRows = BallParamLoader.int(tbl, "tileRows"), - tileCols = BallParamLoader.int(tbl, "tileCols") - ) - } - -} diff --git a/examples/balls/smatmul/configs/default.toml b/examples/balls/smatmul/configs/default.toml deleted file mode 100644 index 8aa081cae..000000000 --- a/examples/balls/smatmul/configs/default.toml +++ /dev/null @@ -1,5 +0,0 @@ -# SMatMul Ball Configuration - -[ball] -tileRows = 16 -tileCols = 16 diff --git a/examples/balls/smatmul/configs/lane16.toml b/examples/balls/smatmul/configs/lane16.toml deleted file mode 100644 index 8e0df9360..000000000 --- a/examples/balls/smatmul/configs/lane16.toml +++ /dev/null @@ -1,5 +0,0 @@ -# SMatMul Ball Configuration - -[ball] -tileRows = 8 -tileCols = 16 diff --git a/examples/balls/smatmul/emu/src/dispatch.rs b/examples/balls/smatmul/emu/src/dispatch.rs deleted file mode 100644 index 6c85e5dec..000000000 --- a/examples/balls/smatmul/emu/src/dispatch.rs +++ /dev/null @@ -1,38 +0,0 @@ -use crate::inst::decode; -use crate::inst::instruction::ExecContext; - -use super::{fp32, mxfp8, smatmul}; - -const BALL_CLASS: &str = "examples.balls.smatmul.SMatMulBall"; - -pub fn execute_known( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, - ctx: &mut ExecContext, -) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("SMATMUL_OS") => Some(smatmul::exec_smatmul(xs1, xs2, ctx)), - Some("SMATMUL_MXFP8") => Some(mxfp8::execute(xs1, xs2, ctx)), - Some("SMATMUL_F32") => Some(fp32::execute(xs1, xs2, ctx)), - Some("SMATMUL_BIAS") => Some(smatmul::exec_bias(xs1, xs2, ctx)), - _ => None, - } -} - -pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("SMATMUL_OS") => Some(smatmul::latency(xs1, xs2)), - Some("SMATMUL_MXFP8") => Some(smatmul::latency(xs1, xs2)), - Some("SMATMUL_F32") => Some(decode::rs1_iter(xs1) as u64 + (xs2 & 0xfff) + ((xs2 >> 12) & 0xfff)), - Some("SMATMUL_BIAS") => Some(smatmul::bias_latency(xs1, xs2)), - _ => None, - } -} diff --git a/examples/balls/smatmul/emu/src/fp32.rs b/examples/balls/smatmul/emu/src/fp32.rs deleted file mode 100644 index 02ba5e3aa..000000000 --- a/examples/balls/smatmul/emu/src/fp32.rs +++ /dev/null @@ -1,119 +0,0 @@ -use super::super::bank::{bank_lines, bank_row_bytes}; -use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; -use super::instruction::ExecContext; -use std::{cell::RefCell, collections::HashMap}; - -struct Chain { - rows: usize, - bank: u64, - base: usize, - values: Vec, -} - -thread_local! { - static CHAINS: RefCell> = RefCell::new(HashMap::new()); -} - -pub(crate) fn execute(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { - let rows = (xs2 & 0xfff) as usize; - let cols = ((xs2 >> 12) & 0xfff) as usize; - let k = rs1_iter(xs1) as usize; - let first = (xs2 >> 24) & 1 != 0; - let last = (xs2 >> 25) & 1 != 0; - let base = ((xs2 >> 26) & 63) as usize; - let a = rs1_b0(xs1); - let b = rs1_b1(xs1); - let c = rs1_b2(xs1); - assert!( - xs2 >> 32 == 0 && (rows == 1 || rows == 16) && cols == 16 && k > 0 && k % 4 == 0, - "f32 matmul: requires M=1/16, N=16, positive K multiple of four and zero reserved bits" - ); - assert!(a != b && b != c && a != c, "f32 matmul: banks must differ"); - for bank in [a, b, c] { - assert!( - ctx.config(bank).allocated && ctx.config(bank).cols == 1, - "f32 matmul: banks must be allocated with one column" - ); - } - let stride = bank_row_bytes(); - assert!( - stride == 16 - && rows * k * 4 <= bank_lines() * stride - && cols * k * 4 <= bank_lines() * stride - && base * stride + rows * cols * 4 <= bank_lines() * stride, - "f32 matmul: bank footprint exceeds depth" - ); - let pa = pbank(ctx, a); - let pb = pbank(ctx, b); - let av = &ctx.banks[pa][..rows * k * 4]; - let bv = &ctx.banks[pb][..cols * k * 4]; - assert!( - av.chunks_exact(4) - .chain(bv.chunks_exact(4)) - .all(|x| f32::from_le_bytes(x.try_into().unwrap()).is_finite()), - "f32 matmul: non-finite operand" - ); - let mut chain = CHAINS.with(|chains| { - let mut chains = chains.borrow_mut(); - if first { - assert!( - !chains.contains_key(&ctx.hart_id), - "f32 matmul: first while chain is live" - ); - Chain { - rows, - bank: c, - base, - values: vec![0f32; rows * cols], - } - } else { - chains - .remove(&ctx.hart_id) - .expect("f32 matmul: continuation without first") - } - }); - assert!( - chain.rows == rows && chain.bank == c && chain.base == base, - "f32 matmul: continuation changed destination" - ); - let weights: [&[u8]; 16] = std::array::from_fn(|col| &bv[col * k * 4..(col + 1) * k * 4]); - for row in 0..rows { - if av[row * k * 4..(row + 1) * k * 4] - .chunks_exact(4) - .all(|x| u32::from_le_bytes(x.try_into().unwrap()) & 0x7fffffff == 0) - && chain.values[row * cols..(row + 1) * cols] - .iter() - .all(|x| x.to_bits() == 0) - { - continue; - } - let av = &av[row * k * 4..(row + 1) * k * 4]; - let dst = &mut chain.values[row * cols..(row + 1) * cols]; - let mut sums: [f32; 16] = dst.try_into().unwrap(); - for inner in 0..k { - let a = f32::from_le_bytes(av[inner * 4..inner * 4 + 4].try_into().unwrap()); - for col in 0..16 { - let b = - f32::from_le_bytes(weights[col][inner * 4..inner * 4 + 4].try_into().unwrap()); - sums[col] = a.mul_add(b, sums[col]); - } - } - assert!( - sums.iter().all(|x| x.is_finite()), - "f32 matmul: accumulator overflow" - ); - dst.copy_from_slice(&sums); - } - if last { - let pc = pbank(ctx, c); - for (index, value) in chain.values.iter().enumerate() { - let offset = base * stride + index * 4; - ctx.banks[pc][offset..offset + 4].copy_from_slice(&value.to_le_bytes()); - } - } else { - CHAINS.with(|chains| { - chains.borrow_mut().insert(ctx.hart_id, chain); - }); - } - 0 -} diff --git a/examples/balls/smatmul/emu/src/lib.rs b/examples/balls/smatmul/emu/src/lib.rs deleted file mode 100644 index 864e215a8..000000000 --- a/examples/balls/smatmul/emu/src/lib.rs +++ /dev/null @@ -1,9 +0,0 @@ -pub(crate) use crate::inst::{decode, instruction}; - -mod fp32; -mod mxfp8; -mod smatmul; - -mod dispatch; - -pub use dispatch::{cycles_after_issue, execute_known}; diff --git a/examples/balls/smatmul/emu/src/mxfp8.rs b/examples/balls/smatmul/emu/src/mxfp8.rs deleted file mode 100644 index fc1a388a5..000000000 --- a/examples/balls/smatmul/emu/src/mxfp8.rs +++ /dev/null @@ -1,165 +0,0 @@ -use super::super::bank::{bank_lines, bank_row_bytes}; -use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; -use super::instruction::ExecContext; -use std::cell::RefCell; -use std::collections::HashMap; - -struct Chain { - rows: usize, - cols: usize, - bank: u64, - base: usize, - values: Vec, -} - -thread_local! { - static CHAINS: RefCell> = RefCell::new(HashMap::new()); -} - -fn value(code: u8, factor: f32) -> f32 { - let exponent = (code >> 3) & 15; - let mantissa = (code & 7) as u32; - let normal = ((exponent as u32 + 120) << 23) | (mantissa << 20); - let subnormal = (mantissa as f32 / 512.0).to_bits(); - let element = f32::from_bits(if exponent == 0 { subnormal } else { normal }); - let result = f32::from_bits((element * factor).to_bits() | ((code as u32 & 128) << 24)); - result -} - -pub(crate) fn execute(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { - let rows = (xs2 & 0xfff) as usize; - let cols = ((xs2 >> 12) & 0xfff) as usize; - let k = rs1_iter(xs1) as usize; - let first = (xs2 >> 24) & 1 != 0; - let last = (xs2 >> 25) & 1 != 0; - let base = ((xs2 >> 26) & 63) as usize; - let a = rs1_b0(xs1); - let b = rs1_b1(xs1); - let c = rs1_b2(xs1); - assert!( - xs2 >> 32 == 0 && (rows == 1 || rows == 16) && cols == 16 && k > 0 && k % 32 == 0, - "mxfp8: requires M=1/16, N=16, K multiple of 32 and zero reserved bits" - ); - assert!(a != b && b != c && a != c, "mxfp8: banks must differ"); - for bank in [a, b, c] { - assert!( - ctx.config(bank).allocated && ctx.config(bank).cols == 1, - "mxfp8: banks must be allocated with one column" - ); - } - let stride = bank_row_bytes(); - assert!(stride == 16, "mxfp8: requires 16-byte bank rows"); - assert!( - rows * k + rows * k / 32 <= bank_lines() * stride - && cols * k + cols * k / 32 <= bank_lines() * stride - && base * stride + rows * cols * 4 <= bank_lines() * stride, - "mxfp8: bank footprint exceeds depth" - ); - let pa = pbank(ctx, a); - let pb = pbank(ctx, b); - let a_bytes = &ctx.banks[pa][..rows * k + rows * k / 32]; - let b_bytes = &ctx.banks[pb][..cols * k + cols * k / 32]; - assert!( - a_bytes[..rows * k] - .iter() - .chain(b_bytes[..cols * k].iter()) - .fold(true, |valid, code| valid & (code & 0x7f != 0x7f)), - "mxfp8: NaN encoding" - ); - let mut av = vec![0f32; rows * k]; - let mut bv = vec![0f32; cols * k]; - for (block, dst) in av.chunks_exact_mut(32).enumerate() { - let scale = a_bytes[rows * k + block]; - assert!(scale != 255, "mxfp8: NaN encoding"); - let factor = if scale == 0 { - f32::from_bits(1 << 22) - } else { - f32::from_bits((scale as u32) << 23) - }; - for (index, decoded) in dst.iter_mut().enumerate() { - *decoded = value(a_bytes[block * 32 + index], factor); - } - } - for col in 0..cols { - for block in 0..k / 32 { - let scale = b_bytes[cols * k + col * (k / 32) + block]; - assert!(scale != 255, "mxfp8: NaN encoding"); - let factor = if scale == 0 { - f32::from_bits(1 << 22) - } else { - f32::from_bits((scale as u32) << 23) - }; - let codes = &b_bytes[col * k + block * 32..col * k + (block + 1) * 32]; - let mut decoded = [0f32; 32]; - for (decoded, code) in decoded.iter_mut().zip(codes.iter()) { - *decoded = value(*code, factor); - } - for (index, decoded) in decoded.iter().enumerate() { - bv[(block * 32 + index) * cols + col] = *decoded; - } - } - } - assert!( - av.iter() - .chain(bv.iter()) - .fold(true, |valid, value| valid & value.is_finite()), - "mxfp8: decoded value overflows FP32" - ); - let mut chain = CHAINS.with(|chains| { - let mut chains = chains.borrow_mut(); - if first { - assert!( - !chains.contains_key(&ctx.hart_id), - "mxfp8: first while chain is live" - ); - Chain { - rows, - cols, - bank: c, - base, - values: vec![0f32; rows * cols], - } - } else { - chains - .remove(&ctx.hart_id) - .expect("mxfp8: continuation without first") - } - }); - assert!( - chain.rows == rows && chain.cols == cols && chain.bank == c && chain.base == base, - "mxfp8: continuation changed destination" - ); - for row in 0..rows { - if av[row * k..(row + 1) * k].iter().all(|value| *value == 0.0) - && chain.values[row * cols..(row + 1) * cols] - .iter() - .all(|value| value.to_bits() == 0) - { - continue; - } - let av = &av[row * k..(row + 1) * k]; - let dst = &mut chain.values[row * cols..(row + 1) * cols]; - let mut sums: [f32; 16] = dst.try_into().unwrap(); - for inner in 0..k { - let a = av[inner]; - let weights = &bv[inner * 16..(inner + 1) * 16]; - for col in 0..16 { - sums[col] = a.mul_add(weights[col], sums[col]); - } - } - assert!(sums.iter().all(|v| v.is_finite()), "accumulator overflow"); - dst.copy_from_slice(&sums); - } - if last { - let pc = pbank(ctx, c); - for (i, value) in chain.values.iter().enumerate() { - ctx.banks[pc][base * stride + i * 4..base * stride + i * 4 + 4] - .copy_from_slice(&value.to_le_bytes()); - } - } else { - CHAINS.with(|chains| { - chains.borrow_mut().insert(ctx.hart_id, chain); - }); - } - 0 -} diff --git a/examples/balls/smatmul/emu/src/smatmul.rs b/examples/balls/smatmul/emu/src/smatmul.rs deleted file mode 100644 index e4152858b..000000000 --- a/examples/balls/smatmul/emu/src/smatmul.rs +++ /dev/null @@ -1,223 +0,0 @@ -use super::super::bank::{bank_lines, bank_row_bytes}; -use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; -use super::instruction::ExecContext; -use std::cell::RefCell; -use std::collections::HashMap; - -const BALL_CLASS: &str = "examples.balls.smatmul.SMatMulBall"; - -fn tile_rows() -> usize { crate::config::ball_domain::param(BALL_CLASS, "tileRows") } -fn tile_cols() -> usize { crate::config::ball_domain::param(BALL_CLASS, "tileCols") } - -#[derive(Default)] -struct State { - bias: Option>, - chain: Option, -} - -struct Chain { - rows: usize, - cols: usize, - output_bank: u64, - output_base: usize, - accumulators: Vec, -} - -thread_local! { - static STATES: RefCell> = RefCell::new(HashMap::new()); -} - -fn read_i32(bank: &[u8], row: usize, lane: usize) -> i32 { - let offset = row * bank_row_bytes() + lane * 4; - i32::from_le_bytes(bank[offset..offset + 4].try_into().unwrap()) -} - -pub(crate) fn exec_bias(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { - let bank = rs1_b0(xs1); - let input_base = (xs2 & 0x3f) as usize; - if rs1_b1(xs1) != 0 || rs1_b2(xs1) != 0 || rs1_iter(xs1) != 4 || xs2 >> 6 != 0 { - panic!("smatmul_bias: bank1/bank2/rs2[63:6] must be zero and iter must be four"); - } - if !ctx.config(bank).allocated || ctx.config(bank).cols != 1 { - panic!("smatmul_bias: bias bank must be one allocated column"); - } - if input_base + 4 > bank_lines() { - panic!("smatmul_bias: inputBase plus four rows exceeds bank depth"); - } - let physical = pbank(ctx, bank); - let mut bias = vec![0i32; tile_cols()]; - for group in 0..4 { - for lane in 0..4 { - bias[group * 4 + lane] = read_i32(&ctx.banks[physical], input_base + group, lane); - } - } - STATES.with(|states| { - let mut states = states.borrow_mut(); - let state = states.entry(ctx.hart_id).or_default(); - if state.chain.is_some() { - panic!("smatmul_bias: cannot replace bias during an accumulation chain"); - } - state.bias = Some(bias); - }); - 0 -} - -pub(crate) fn exec_smatmul(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { - let m_tile = tile_rows(); - let k_tile = tile_cols(); - let n_tile = tile_cols(); - let a_bank = rs1_b0(xs1); - let b_bank = rs1_b1(xs1); - let c_bank = rs1_b2(xs1); - let rows = (xs2 & 0xfff) as usize; - let cols = ((xs2 >> 12) & 0xfff) as usize; - let k = rs1_iter(xs1) as usize; - let first = (xs2 >> 24) & 1 != 0; - let last = (xs2 >> 25) & 1 != 0; - let output_base = ((xs2 >> 26) & 0x3f) as usize; - if xs2 >> 32 != 0 { - panic!("smatmul: rs2[63:32] must be zero"); - } - if (rows != 1 && (rows == 0 || rows % m_tile != 0)) || cols != n_tile || k == 0 || k % k_tile != 0 { - panic!("smatmul: M must be one or a positive multiple of 8, K must be a positive multiple of 16, and N must be 16"); - } - if a_bank == b_bank || a_bank == c_bank || b_bank == c_bank { - panic!("smatmul: A, B, and C banks must differ"); - } - if !ctx.config(a_bank).allocated - || !ctx.config(b_bank).allocated - || !ctx.config(c_bank).allocated - { - panic!("smatmul: bank not allocated"); - } - if ctx.config(a_bank).cols != 1 || ctx.config(b_bank).cols != 1 || ctx.config(c_bank).cols != 1 - { - panic!("smatmul: A, B, and C banks must each have one column"); - } - - let a_rows = if rows == 1 { k / k_tile } else { rows * k / k_tile }; - let b_rows = k; - let c_rows = rows * 4; - if a_rows > bank_lines() || b_rows > bank_lines() || output_base + c_rows > bank_lines() { - panic!("smatmul: bank footprint exceeds bank depth"); - } - - let mut chain = STATES.with(|states| { - let mut states = states.borrow_mut(); - let state = states.entry(ctx.hart_id).or_default(); - if first { - if state.chain.is_some() { - panic!("smatmul: first block issued while another chain is live"); - } - let bias = state - .bias - .as_ref() - .unwrap_or_else(|| panic!("smatmul: bias must be preloaded before first block")); - let mut accumulators = vec![0i32; rows * cols]; - for row in 0..rows { - accumulators[row * cols..(row + 1) * cols].copy_from_slice(bias); - } - Chain { - rows, - cols, - output_bank: c_bank, - output_base, - accumulators, - } - } else { - state - .chain - .take() - .unwrap_or_else(|| panic!("smatmul: continuation has no live chain")) - } - }); - if chain.rows != rows - || chain.cols != cols - || chain.output_bank != c_bank - || chain.output_base != output_base - { - panic!("smatmul: continuation changed output shape or destination"); - } - - let pa = pbank(ctx, a_bank); - let pb = pbank(ctx, b_bank); - let k_tiles = k / k_tile; - let row_bytes = bank_row_bytes(); - - // Keep the hardware-visible operation unchanged, but traverse the data in - // the layout used by the banks. The old col->k loop repeatedly computed - // addresses and reread the same B row for every output lane. This order - // reads one A byte and one complete B row, then updates all 16 lanes. - // Wrapping arithmetic is intentionally retained to match the accumulator - // semantics of the RTL and the previous emulator implementation. - for inner in 0..k { - let b_base = ((inner / k_tile) * k_tile + inner % k_tile) * row_bytes; - let b_row = &ctx.banks[pb][b_base..b_base + cols]; - let mut b_values = vec![0i32; n_tile]; - for col in 0..cols { - b_values[col] = b_row[col] as i8 as i32; - } - for row in 0..rows { - let a_row = if rows == 1 { - inner / k_tile - } else { - ((row / m_tile) * k_tiles + inner / k_tile) * m_tile + row % m_tile - }; - let a = ctx.banks[pa][a_row * row_bytes + inner % k_tile] as i8 as i32; - let acc_base = row * cols; - let acc_row = &mut chain.accumulators[acc_base..acc_base + cols]; - for col in 0..cols { - acc_row[col] = acc_row[col].wrapping_add(a.wrapping_mul(b_values[col])); - } - } - } - - if last { - let pc = pbank(ctx, c_bank); - for row in 0..rows { - for col in 0..cols { - let offset = (output_base + row * 4 + col / 4) * bank_row_bytes() + col % 4 * 4; - ctx.banks[pc][offset..offset + 4] - .copy_from_slice(&chain.accumulators[row * cols + col].to_le_bytes()); - } - } - } else { - STATES.with(|states| { - states.borrow_mut().entry(ctx.hart_id).or_default().chain = Some(chain); - }); - } - 0 -} - -pub(crate) fn bias_latency(xs1: u64, xs2: u64) -> u64 { - let input_base = xs2 & 0x3f; - if rs1_b1(xs1) != 0 - || rs1_b2(xs1) != 0 - || rs1_iter(xs1) != 4 - || xs2 >> 6 != 0 - || input_base + 4 > bank_lines() as u64 - { - panic!("smatmul_bias: illegal inputBase or reserved fields"); - } - 4 -} - -pub(crate) fn latency(xs1: u64, xs2: u64) -> u64 { - let m_tile = tile_rows(); - let k_tile = tile_cols(); - let n_tile = tile_cols(); - let rows = xs2 & 0xfff; - let cols = (xs2 >> 12) & 0xfff; - let k = rs1_iter(xs1); - let output_base = (xs2 >> 26) & 0x3f; - if xs2 >> 32 != 0 - || (rows != 1 && (rows == 0 || rows % m_tile as u64 != 0)) - || cols != n_tile as u64 - || k == 0 - || k % k_tile as u64 != 0 - || output_base + rows * 4 > bank_lines() as u64 - { - panic!("smatmul: illegal matrix encoding"); - } - rows * cols * k / m_tile as u64 + if (xs2 >> 25) & 1 != 0 { rows * 4 } else { 0 } -} diff --git a/examples/balls/smatmul/verify/README.md b/examples/balls/smatmul/verify/README.md deleted file mode 100644 index 0dea2ea0e..000000000 --- a/examples/balls/smatmul/verify/README.md +++ /dev/null @@ -1,81 +0,0 @@ -# SMatMulBall UVM Verification - -Verifies the generated `SMatMulBall` module using the shared Blink UVM framework. - -## Structure - -- `../../../../verify/uvm/src/ball/bb_uvm_pkg.sv`: common Blink UVM agents and base env -- `src/common/smatmul_defs.svh`: DPI imports, `matrix_require_bid`, timeouts -- `src/common/smatmul_items.svh`: `smatmul_cmd_item` with `load_rust_case` -- `src/seq/smatmul_sequences.svh`: one-case sequence -- `src/cov/smatmul_cov.svh`: M/N/K shape bins `{1,2,4,8,16}` -- `src/env/smatmul_scoreboard.svh`: preload A/B, compare writes vs DPI expected list -- `src/env/smatmul_env.svh`: extends `bb_blink_env#(2,4)` -- `src/tests/smatmul_test.svh`: directed 0..3, random 4..23 -- `src/pkg/smatmul_pkg.sv`: package entry -- `src/tb_top.sv`: `bb_blink_if#(2,2)` + `SMatMulBall` (2 read, 2 write ports) -- `filelists/smatmul_ball.f`: VCS filelist - -## Test plan - -`+UVM_TESTNAME=smatmul_ball_test` runs: - -- case 0: 4x4x4 OS (banks 0,1,2) -- case 1: 5x7x3 OS tail -- case 2: 16x16x16 OS -- case 3: 32x16x16 WS -- cases 4..23: random M in {1,2,4,8,16,32}, N/K in {1,2,4,8,16}, distinct banks 0..7 - -Each case: reset, preload A/B into mem model, drive one command, wait for -`scb.done()` or timeout at 100000 cycles. Scoreboard compares writes -(group/addr/data/mask) in DPI emission order. - -## BID (required) - -`+BID=` is mandatory. Missing it fatals at test start. No default bid. - -| Config | Plusarg | -|--------|---------| -| `sims.verilator.BuckyballPebbleVerilatorConfig` | `+BID=1` | -| `sims.verilator.BuckyballToyVerilatorConfig` | `+BID=4` | - -```console -./simv +UVM_TESTNAME=smatmul_ball_test +BID=4 -``` - -## Build - -```console -nix develop ../../../../verify -cargo build --manifest-path casegen/Cargo.toml -``` - -Compile from this directory: - -```console -vcs -full64 -sverilog -timescale=1ns/1ps \ - $VCS_UVM_ARGS \ - -sv_lib casegen/target/debug/libmatrix_casegen \ - -f filelists/smatmul_ball.f -``` - -Run: - -```console -./simv +UVM_TESTNAME=smatmul_ball_test +BID=4 -``` - -Or via bbdev: - -```console -bbdev uvm --build '--ball=matrix' -bbdev uvm --run '--ball=matrix' -- '+BID=4' -``` - -## Acceptance - -- [x] Layout matches Blink common + ball-local split; filelist uses `@UVM@` / `@RTL@` -- [x] casegen whole-case DPI API; `cargo test` in casegen -- [x] VCS build with `smatmul_ball.f` (no segfault on Instantiate DUT) -- [x] `+BID=` required; pebble `+BID=1` / toy `+BID=4` -- [ ] 23/23 functional pass diff --git a/examples/balls/smatmul/verify/casegen/src/casegen.rs b/examples/balls/smatmul/verify/casegen/src/casegen.rs deleted file mode 100644 index d776fb6f1..000000000 --- a/examples/balls/smatmul/verify/casegen/src/casegen.rs +++ /dev/null @@ -1,156 +0,0 @@ -use crate::model; - -pub const BIAS_BANK: u32 = 0; -pub const C_BANK: u32 = 5; - -#[repr(C)] -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub struct MatrixCmd { - pub kind: u32, - pub bid: u32, - pub rob_id: u32, - pub op1_words: u32, - pub op2_words: u32, - pub rs1_lo: u32, - pub rs1_hi: u32, - pub rs2_lo: u32, - pub rs2_hi: u32, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct MatrixCase { - pub commands: Vec, - pub bias: Vec, - pub a: Vec>, - pub b: Vec>, - pub writes: Vec, -} - -pub fn gen_case(seed: u32, index: u32, bid: u32, tile_rows: usize) -> MatrixCase { - match index { - 0 => build_case(seed, bid, 16, 1, tile_rows), - 1 => build_case(seed, bid, 16, 2, tile_rows), - 2 => build_case(seed ^ index, bid, 16, 1, tile_rows), - _ => { - let mut rng = Rng::new(seed, index); - let blocks = 1 + (rng.next() % 2) as usize; - build_case(rng.next(), bid, 16, blocks, tile_rows) - } - } -} - -fn build_case(seed: u32, bid: u32, rows: usize, blocks: usize, tile_rows: usize) -> MatrixCase { - assert!(matches!(rows, 16 | 32 | 64)); - assert!(matches!(blocks, 1 | 2)); - assert!(rows * model::RESULT_WORDS <= model::BANK_DEPTH); - let mut rng = Rng::new(seed, rows as u32 ^ blocks as u32); - let bias_values: Vec = (0..model::TILE).map(|_| rng.value() as i32).collect(); - let mut bias = Vec::with_capacity(4 * model::ROW_BYTES); - for chunk in bias_values.chunks(4) { - for value in chunk { - bias.extend_from_slice(&value.to_le_bytes()); - } - } - - let mut a = Vec::with_capacity(blocks); - let mut b = Vec::with_capacity(blocks); - let mut result = vec![bias_values.clone(); rows]; - let mut commands = Vec::with_capacity(blocks + 1); - commands.push(MatrixCmd { - kind: 0, - bid, - rob_id: 0, - op1_words: 4, - rs1_lo: model::encode_rs1(BIAS_BANK, 0, 0, 4) as u32, - rs1_hi: (model::encode_rs1(BIAS_BANK, 0, 0, 4) >> 32) as u32, - ..MatrixCmd::default() - }); - - for block in 0..blocks { - let a_bank = 1 + 2 * block as u32; - let b_bank = 2 + 2 * block as u32; - let a_values: Vec = (0..rows * model::TILE).map(|_| rng.value()).collect(); - let b_values: Vec = (0..model::TILE * model::TILE) - .map(|_| rng.value()) - .collect(); - let a_packed = model::pack_a(&a_values, rows); - let b_packed = model::pack_b(&b_values); - a.push(a_packed); - b.push(b_packed); - for row in 0..rows { - for column in 0..model::TILE { - for k in 0..model::TILE { - result[row][column] = result[row][column].wrapping_add( - a_values[row * model::TILE + k] as i32 - * b_values[k * model::TILE + column] as i32, - ); - } - } - } - let rs1 = model::encode_rs1(a_bank, b_bank, C_BANK, model::TILE as u64); - let rs2 = model::encode_rs2(rows, block == 0, block + 1 == blocks); - commands.push(MatrixCmd { - kind: 1, - bid, - rob_id: (block + 1) as u32, - op1_words: rows as u32, - op2_words: (model::TILE * rows.div_ceil(tile_rows)) as u32, - rs1_lo: rs1 as u32, - rs1_hi: (rs1 >> 32) as u32, - rs2_lo: rs2 as u32, - rs2_hi: (rs2 >> 32) as u32, - }); - } - - MatrixCase { - commands, - bias, - a, - b, - writes: model::emit_writes(&result), - } -} - -struct Rng { - state: u64, -} - -impl Rng { - fn new(seed: u32, salt: u32) -> Self { - Self { - state: u64::from(seed) << 32 | u64::from(salt), - } - } - fn next(&mut self) -> u32 { - self.state ^= self.state << 13; - self.state ^= self.state >> 7; - self.state ^= self.state << 17; - self.state as u32 - } - fn value(&mut self) -> i8 { - (self.next() % 17) as i8 - 8 - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn single_block_fills_one_result_bank() { - let case = gen_case(1, 2, 1, 16); - assert_eq!(case.commands.len(), 2); - assert_eq!(case.writes.len(), 64); - assert_eq!(case.writes.last().unwrap().addr, 63); - } - - #[test] - fn two_blocks_encode_one_live_chain() { - let case = gen_case(1, 1, 1, 16); - assert_eq!(case.commands.len(), 3); - assert_eq!((case.commands[1].rs2_lo >> 24) & 3, 1); - assert_eq!((case.commands[2].rs2_lo >> 24) & 3, 2); - assert_eq!(case.commands[1].rs1_lo, 1 | (2 << 10) | (5 << 20)); - assert_eq!(case.commands[2].rs1_lo, 3 | (4 << 10) | (5 << 20)); - } -} diff --git a/examples/balls/smatmul/verify/casegen/src/lib.rs b/examples/balls/smatmul/verify/casegen/src/lib.rs deleted file mode 100644 index db9ec2e44..000000000 --- a/examples/balls/smatmul/verify/casegen/src/lib.rs +++ /dev/null @@ -1,114 +0,0 @@ -mod casegen; -mod model; - -use std::cell::RefCell; - -use casegen::{MatrixCase, MatrixCmd}; - -thread_local! { - static CURRENT: RefCell> = const { RefCell::new(None) }; -} - -#[no_mangle] -pub extern "C" fn smatmul_case_load(seed: u32, index: u32, bid: u32, tile_rows: u32) { - CURRENT.with(|current| { - *current.borrow_mut() = Some(casegen::gen_case(seed, index, bid, tile_rows as usize)) - }); -} - -fn current_case R, R>(f: F) -> R { - CURRENT.with(|current| f(current.borrow().as_ref().expect("smatmul case not loaded"))) -} - -#[no_mangle] -pub extern "C" fn smatmul_case_num_commands() -> u32 { - current_case(|case| case.commands.len() as u32) -} - -#[no_mangle] -pub extern "C" fn smatmul_case_cmd(index: u32, out: *mut MatrixCmd) { - assert!(!out.is_null(), "smatmul_case_cmd: null output"); - current_case(|case| unsafe { - *out = *case - .commands - .get(index as usize) - .expect("command index out of range"); - }); -} - -#[no_mangle] -pub extern "C" fn smatmul_case_bias_words() -> u32 { - 4 -} - -#[no_mangle] -pub extern "C" fn smatmul_case_a_words(block: u32) -> u32 { - current_case(|case| (case.a[block as usize].len() / model::ROW_BYTES) as u32) -} - -#[no_mangle] -pub extern "C" fn smatmul_case_b_words(block: u32) -> u32 { - current_case(|case| (case.b[block as usize].len() / model::ROW_BYTES) as u32) -} - -fn word_halves(data: &[u8], index: u32) -> (u64, u64) { - let word = model::word(data, index as usize); - ( - u64::from_le_bytes(word[0..8].try_into().unwrap()), - u64::from_le_bytes(word[8..16].try_into().unwrap()), - ) -} - -#[no_mangle] -pub extern "C" fn smatmul_case_bias_word_lo(index: u32) -> u64 { - current_case(|case| word_halves(&case.bias, index).0) -} - -#[no_mangle] -pub extern "C" fn smatmul_case_bias_word_hi(index: u32) -> u64 { - current_case(|case| word_halves(&case.bias, index).1) -} - -#[no_mangle] -pub extern "C" fn smatmul_case_a_word_lo(block: u32, index: u32) -> u64 { - current_case(|case| word_halves(&case.a[block as usize], index).0) -} - -#[no_mangle] -pub extern "C" fn smatmul_case_a_word_hi(block: u32, index: u32) -> u64 { - current_case(|case| word_halves(&case.a[block as usize], index).1) -} - -#[no_mangle] -pub extern "C" fn smatmul_case_b_word_lo(block: u32, index: u32) -> u64 { - current_case(|case| word_halves(&case.b[block as usize], index).0) -} - -#[no_mangle] -pub extern "C" fn smatmul_case_b_word_hi(block: u32, index: u32) -> u64 { - current_case(|case| word_halves(&case.b[block as usize], index).1) -} - -#[no_mangle] -pub extern "C" fn smatmul_case_num_writes() -> u32 { - current_case(|case| case.writes.len() as u32) -} - -#[no_mangle] -pub extern "C" fn smatmul_case_write_addr(index: u32) -> u32 { - current_case(|case| case.writes[index as usize].addr) -} - -#[no_mangle] -pub extern "C" fn smatmul_case_write_data_lo(index: u32) -> u64 { - current_case(|case| { - u64::from_le_bytes(case.writes[index as usize].data[0..8].try_into().unwrap()) - }) -} - -#[no_mangle] -pub extern "C" fn smatmul_case_write_data_hi(index: u32) -> u64 { - current_case(|case| { - u64::from_le_bytes(case.writes[index as usize].data[8..16].try_into().unwrap()) - }) -} diff --git a/examples/balls/smatmul/verify/casegen/src/model.rs b/examples/balls/smatmul/verify/casegen/src/model.rs deleted file mode 100644 index 48b7e9118..000000000 --- a/examples/balls/smatmul/verify/casegen/src/model.rs +++ /dev/null @@ -1,58 +0,0 @@ -pub const TILE: usize = 16; -pub const ROW_BYTES: usize = 16; -pub const RESULT_WORDS: usize = 4; -pub const BANK_DEPTH: usize = 64; - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct WriteExp { - pub addr: u32, - pub data: [u8; ROW_BYTES], -} - -pub fn pack_a(src: &[i8], rows: usize) -> Vec { - assert_eq!(src.len(), rows * TILE); - src.iter().map(|value| *value as u8).collect() -} - -pub fn pack_b(src: &[i8]) -> Vec { - assert_eq!(src.len(), TILE * TILE); - src.iter().map(|value| *value as u8).collect() -} - -pub fn emit_writes(c: &[Vec]) -> Vec { - let mut writes = Vec::with_capacity(c.len() * RESULT_WORDS); - for (row, values) in c.iter().enumerate() { - assert_eq!(values.len(), TILE); - for word in 0..RESULT_WORDS { - let mut data = [0; ROW_BYTES]; - for lane in 0..4 { - let value = values[word * 4 + lane]; - data[lane * 4..lane * 4 + 4].copy_from_slice(&value.to_le_bytes()); - } - writes.push(WriteExp { - addr: (row * RESULT_WORDS + word) as u32, - data, - }); - } - } - writes -} - -pub fn word(data: &[u8], index: usize) -> [u8; ROW_BYTES] { - data[index * ROW_BYTES..(index + 1) * ROW_BYTES] - .try_into() - .unwrap() -} - -pub fn encode_rs1(op1: u32, op2: u32, wr: u32, iter: u64) -> u64 { - assert!(op1 < 1024 && op2 < 1024 && wr < 1024 && iter < (1_u64 << 34)); - u64::from(op1) | (u64::from(op2) << 10) | (u64::from(wr) << 20) | (iter << 30) -} - -pub fn encode_rs2(rows: usize, first: bool, last: bool) -> u64 { - assert!(matches!(rows, 16 | 32 | 64)); - rows as u64 - | ((TILE as u64) << 12) - | ((first as u64) << 24) - | ((last as u64) << 25) -} diff --git a/examples/balls/smatmul/verify/src/common/smatmul_items.svh b/examples/balls/smatmul/verify/src/common/smatmul_items.svh deleted file mode 100644 index 6dc1f82f3..000000000 --- a/examples/balls/smatmul/verify/src/common/smatmul_items.svh +++ /dev/null @@ -1,78 +0,0 @@ -class smatmul_cmd_item extends bb_blink_cmd_item; - `uvm_object_utils(smatmul_cmd_item) - - bit [31:0] kind; - bit [31:0] command_index; - bit [31:0] command_count; - bit [31:0] op1_words; - bit [31:0] op2_words; - - function new(string name = "smatmul_cmd_item"); - super.new(name); - endfunction - - function void load_rust_command(int unsigned index); - matrix_cmd_dpi_t cmd; - smatmul_case_cmd(index, cmd); - if (cmd.kind > 1) begin - `uvm_fatal("CASE", $sformatf("invalid command kind %0d", cmd.kind)) - end - - kind = cmd.kind; - command_index = index; - command_count = smatmul_case_num_commands(); - bid = cmd.bid[4:0]; - rob_id = cmd.rob_id[3:0]; - op1_words = cmd.op1_words; - op2_words = cmd.op2_words; - rs1 = {cmd.rs1_hi, cmd.rs1_lo}; - rs2 = {cmd.rs2_hi, cmd.rs2_lo}; - op1_bank = rs1[4:0]; - op2_bank = rs1[14:10]; - wr_bank = rs1[24:20]; - iter = rs1[63:30]; - special = 64'd0; - meta_bank = 5'd0; - is_sub = 1'b0; - sub_rob_id = 8'd0; - - if (kind == 0) begin - funct7 = SMATMUL_BIAS_CORE_FUNCT7[6:0]; - op1_en = 1'b1; - op2_en = 1'b0; - wr_spad_en = 1'b0; - op1_from_spad = 1'b1; - op2_from_spad = 1'b0; - op1_col = 5'd1; - op2_col = 5'd0; - wr_col = 5'd0; - if (rs1[29:10] != 0 || iter != 4 || rs2 != 0 || op1_words != 4) - `uvm_fatal("CASE", "invalid bias command") - end else begin - funct7 = SMATMUL_OS_CORE_FUNCT7[6:0]; - op1_en = 1'b1; - op2_en = 1'b1; - wr_spad_en = 1'b1; - op1_from_spad = 1'b1; - op2_from_spad = 1'b1; - op1_col = 5'd1; - op2_col = 5'd1; - wr_col = 5'd1; - if (op1_bank == op2_bank || op1_bank == wr_bank || op2_bank == wr_bank) - `uvm_fatal("CASE", "SMATMUL_OS banks must be distinct") - if (rs2[63:26] != 0 || rs2[23:12] != 16 || iter != 16) - `uvm_fatal("CASE", "invalid SMATMUL_OS shape or reserved bits") - end - endfunction - - function void do_copy(uvm_object rhs); - smatmul_cmd_item rhs_; - super.do_copy(rhs); - if (!$cast(rhs_, rhs)) `uvm_fatal("COPY", "rhs is not smatmul_cmd_item") - kind = rhs_.kind; - command_index = rhs_.command_index; - command_count = rhs_.command_count; - op1_words = rhs_.op1_words; - op2_words = rhs_.op2_words; - endfunction -endclass diff --git a/examples/balls/smatmul/verify/src/tb_top.sv b/examples/balls/smatmul/verify/src/tb_top.sv deleted file mode 100644 index a69d8e3f6..000000000 --- a/examples/balls/smatmul/verify/src/tb_top.sv +++ /dev/null @@ -1,101 +0,0 @@ -module tb_top; - import uvm_pkg::*; - import smatmul_pkg::*; - - bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); - - SMatMulBall dut ( - .clock(intf.clock), - .reset(intf.reset), - .io_status_idle(), - .io_status_running(), - .io_channelReady(1'b1), - .io_cmdReq_ready(intf.cmd_req_ready), - .io_cmdReq_valid(intf.cmd_req_valid), - .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), - .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), - .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), - .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), - .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), - .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), - .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), - .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), - .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), - .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank), - .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank), - .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank), - .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), - .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), - .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), - .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank), - .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), - .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), - .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), - .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), - .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), - .io_cmdResp_ready(intf.cmd_resp_ready), - .io_cmdResp_valid(intf.cmd_resp_valid), - .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), - .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), - .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), - .io_bankRead_0_bank_id(intf.bank_read_bank_id[0]), - .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), - .io_bankRead_0_ball_id(), - .io_bankRead_0_group_id(intf.bank_read_group_id[0]), - .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), - .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), - .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), - .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), - .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), - .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), - .io_bankRead_1_bank_id(intf.bank_read_bank_id[1]), - .io_bankRead_1_rob_id(intf.bank_read_rob_id[1]), - .io_bankRead_1_ball_id(), - .io_bankRead_1_group_id(intf.bank_read_group_id[1]), - .io_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), - .io_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), - .io_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), - .io_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), - .io_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), - .io_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), - .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), - .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), - .io_bankWrite_0_ball_id(), - .io_bankWrite_0_group_id(intf.bank_write_group_id[0]), - .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), - .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), - .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), - .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), - .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), - .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), - .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), - .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), - .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), - .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), - .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), - .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), - .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), - .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), - .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), - .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), - .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), - .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), - .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), - .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), - .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), - .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), - .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), - .io_subRobReq_ready(intf.sub_rob_req_ready) - ); - - initial begin - intf.clock = 1'b0; - intf.reset = 1'b1; - forever #5 intf.clock = ~intf.clock; - end - - initial begin - uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); - run_test("smatmul_ball_test"); - end -endmodule diff --git a/examples/balls/smatmul/workloads/ctests/CMakeLists.txt b/examples/balls/smatmul/workloads/ctests/CMakeLists.txt deleted file mode 100644 index 8c9dad8d2..000000000 --- a/examples/balls/smatmul/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -add_buckyball_ctests( - smatmul_bias_accumulate_test.c - smatmul_1x16x64_test.c - smatmul_8x16_test.c - smatmul_16x16_test.c - smatmul_16x16x32_test.c - smatmul_nondefault_banks_test.c -) -file(STRINGS "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/ballISA.h" F32_ISA - REGEX "^#define BB_FUNC7_SMATMUL_F32 ") -if(F32_ISA) - add_buckyball_ctest(smatmul_f32_test.c) -endif() diff --git a/examples/balls/smatmul/workloads/isa/smatmul.h b/examples/balls/smatmul/workloads/isa/smatmul.h deleted file mode 100644 index 67b2334d7..000000000 --- a/examples/balls/smatmul/workloads/isa/smatmul.h +++ /dev/null @@ -1,38 +0,0 @@ -#ifndef _BB_SMATMUL_H_ -#define _BB_SMATMUL_H_ - -#include -#include -#define BB_SMATMUL_CFG(rows, cols, first, last, output_base) \ - (FIELD((rows), 0, 11) | FIELD((cols), 12, 23) | FIELD((first), 24, 24) | \ - FIELD((last), 25, 25) | FIELD((output_base), 26, 31)) - -#define bb_smatmul_bias(bias_bank, input_base) \ - BUCKYBALL_INSTRUCTION_R_R(BB_BANK0(bias_bank) | BB_ITER(4), \ - FIELD((input_base), 0, 5), BB_FUNC7(SMATMUL_BIAS)) - -#define bb_smatmul_os(a_bank, b_bank, c_bank, rows, cols, k, first, last, \ - output_base) \ - BUCKYBALL_INSTRUCTION_R_R( \ - BB_BANK0(a_bank) | BB_BANK1(b_bank) | BB_BANK2(c_bank) | BB_ITER(k), \ - BB_SMATMUL_CFG(rows, cols, first, last, output_base), \ - BB_FUNC7(SMATMUL_OS)) - -// E4M3 codes in row-major A[M,K] and B[N,K], followed by E8M0[M,K/32] -// and E8M0[N,K/32]. C is row-major FP32. K is a multiple of 32. -#define bb_smatmul_mxfp8(a_bank, b_bank, c_bank, rows, cols, k, first, last, \ - output_base) \ - BUCKYBALL_INSTRUCTION_R_R( \ - BB_BANK0(a_bank) | BB_BANK1(b_bank) | BB_BANK2(c_bank) | BB_ITER(k), \ - BB_SMATMUL_CFG(rows, cols, first, last, output_base), \ - BB_FUNC7(SMATMUL_MXFP8)) - -// Row-major FP32 A[M,K], B[N,K], C[M,N]. K is a multiple of four. -#define bb_smatmul_f32(a_bank, b_bank, c_bank, rows, cols, k, first, last, \ - output_base) \ - BUCKYBALL_INSTRUCTION_R_R( \ - BB_BANK0(a_bank) | BB_BANK1(b_bank) | BB_BANK2(c_bank) | BB_ITER(k), \ - BB_SMATMUL_CFG(rows, cols, first, last, output_base), \ - BB_FUNC7(SMATMUL_F32)) - -#endif // _BB_SMATMUL_H_ diff --git a/examples/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate.mlir b/examples/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate.mlir deleted file mode 100644 index 520c8c27d..000000000 --- a/examples/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate.mlir +++ /dev/null @@ -1,86 +0,0 @@ -func.func private @check_result(memref<64x4xi32>) -> () - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero_i32 = arith.constant 0 : i32 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %four = arith.constant 4 : index - %eight = arith.constant 8 : index - %sixteen = arith.constant 16 : index - %depth4 = arith.constant 4 : i64 - %depth16 = arith.constant 16 : i64 - %depth64 = arith.constant 64 : i64 - %stride = arith.constant 1 : i64 - %config = arith.constant 268501008 : i64 - %true = arith.constant true - %false = arith.constant false - %zero64 = arith.constant 0 : i64 - - %input0 = memref.alloc() alignment = 64 : memref<16x16xi8> - %input1 = memref.alloc() alignment = 64 : memref<16x16xi8> - %weight0 = memref.alloc() alignment = 64 : memref<16x16xi8> - %weight1 = memref.alloc() alignment = 64 : memref<16x16xi8> - %bias = memref.alloc() alignment = 64 : memref<4x4xi32> - %output = memref.alloc() alignment = 64 : memref<64x4xi32> - linalg.fill ins(%zero_i32 : i32) outs(%output : memref<64x4xi32>) - - scf.for %row = %zero to %sixteen step %one { - scf.for %column = %zero to %sixteen step %one { - %row_i32 = arith.index_cast %row : index to i32 - %column_i32 = arith.index_cast %column : index to i32 - %sum = arith.addi %row_i32, %column_i32 : i32 - %difference = arith.subi %row_i32, %column_i32 : i32 - %twice_row = arith.addi %row_i32, %row_i32 : i32 - %second = arith.subi %twice_row, %column_i32 : i32 - %sum_i8 = arith.trunci %sum : i32 to i8 - %difference_i8 = arith.trunci %difference : i32 to i8 - %second_i8 = arith.trunci %second : i32 to i8 - memref.store %difference_i8, %input0[%row, %column] : memref<16x16xi8> - memref.store %sum_i8, %weight0[%row, %column] : memref<16x16xi8> - memref.store %sum_i8, %input1[%row, %column] : memref<16x16xi8> - memref.store %second_i8, %weight1[%row, %column] : memref<16x16xi8> - } - %bias_value = arith.subi %row, %eight : index - %bias_i32 = arith.index_cast %bias_value : index to i32 - %bias_row = arith.divui %row, %four : index - %bias_lane = arith.remui %row, %four : index - memref.store %bias_i32, %bias[%bias_row, %bias_lane] : memref<4x4xi32> - } - - %bias_bank = buckyball.bank_alloc - %a0_bank = buckyball.bank_alloc - %b0_bank = buckyball.bank_alloc - %a1_bank = buckyball.bank_alloc - %b1_bank = buckyball.bank_alloc - %result_bank = buckyball.bank_alloc - %loaded_bias = buckyball.bank_mvin %bias %bias_bank %depth4 %stride - : memref<4x4xi32> i64 i64 i64 - %loaded_a0 = buckyball.bank_mvin %input0 %a0_bank %depth16 %stride - : memref<16x16xi8> i64 i64 i64 - %loaded_b0 = buckyball.bank_mvin %weight0 %b0_bank %depth16 %stride - : memref<16x16xi8> i64 i64 i64 - %loaded_a1 = buckyball.bank_mvin %input1 %a1_bank %depth16 %stride - : memref<16x16xi8> i64 i64 i64 - %loaded_b1 = buckyball.bank_mvin %weight1 %b1_bank %depth16 %stride - : memref<16x16xi8> i64 i64 i64 - buckyball.smatmul_bias %loaded_bias, %zero64 : i64 - buckyball.smatmul %loaded_a0, %loaded_b0, %result_bank, %config, %true, %false, %zero64 : i64 - buckyball.smatmul %loaded_a1, %loaded_b1, %result_bank, %config, %false, %true, %zero64 : i64 - %stored = buckyball.bank_mvout %output %result_bank %depth64 %stride - : memref<64x4xi32> i64 i64 i64 - func.call @check_result(%output) : (memref<64x4xi32>) -> () - buckyball.bank_release %loaded_bias : i64 - buckyball.bank_release %loaded_a0 : i64 - buckyball.bank_release %loaded_b0 : i64 - buckyball.bank_release %loaded_a1 : i64 - buckyball.bank_release %loaded_b1 : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %input0 : memref<16x16xi8> - memref.dealloc %input1 : memref<16x16xi8> - memref.dealloc %weight0 : memref<16x16xi8> - memref.dealloc %weight1 : memref<16x16xi8> - memref.dealloc %bias : memref<4x4xi32> - memref.dealloc %output : memref<64x4xi32> - return %zero_i8 : i8 -} diff --git a/examples/balls/smatmul/workloads/mlir_tests/bank/CMakeLists.txt b/examples/balls/smatmul/workloads/mlir_tests/bank/CMakeLists.txt deleted file mode 100644 index 51bcb8fda..000000000 --- a/examples/balls/smatmul/workloads/mlir_tests/bank/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) - set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) -endif() -set(BUCKYBALL_MLIR_TEST_PREFIX bank) - -buckyball_require_smatmul_bank_shape(${BUCKYBALL_MLIR_ACTIVE_TARGET}) -add_buckyball_mlir_test(smatmul_bias_two_terms TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms.mlir.in b/examples/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms.mlir.in deleted file mode 100644 index 8cdd1861c..000000000 --- a/examples/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms.mlir.in +++ /dev/null @@ -1,80 +0,0 @@ -func.func private @check_result(memref<@BB_BANK_LINES@x4xi32>) -> () -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero_i32 = arith.constant 0 : i32 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %four = arith.constant 4 : index - %eight = arith.constant 8 : index - %sixteen = arith.constant 16 : index - %m = arith.constant @BB_SMM_M@ : index - %depth4 = arith.constant 4 : i64 - %depth16 = arith.constant 16 : i64 - %depth_out = arith.constant @BB_BANK_LINES@ : i64 - %m_i64 = arith.constant @BB_SMM_M@ : i64 - %stride = arith.constant 1 : i64 - %config = arith.constant @BB_SMM_CONFIG@ : i64 - %true = arith.constant true - %false = arith.constant false - %zero64 = arith.constant 0 : i64 - %a = memref.alloc() : memref<@BB_SMM_M@x16xi8> - %b0 = memref.alloc() : memref<16x16xi8> - %b1 = memref.alloc() : memref<16x16xi8> - %bias = memref.alloc() : memref<4x4xi32> - %output = memref.alloc() : memref<@BB_BANK_LINES@x4xi32> - linalg.fill ins(%zero_i32 : i32) outs(%output : memref<@BB_BANK_LINES@x4xi32>) - scf.for %row = %zero to %m step %one { - %slot = arith.remui %row, %sixteen : index - scf.for %col = %zero to %sixteen step %one { - %v = arith.subi %slot, %col : index - %v8 = arith.index_cast %v : index to i8 - memref.store %v8, %a[%row, %col] : memref<@BB_SMM_M@x16xi8> - } - } - scf.for %row = %zero to %sixteen step %one { - %bias_value = arith.subi %row, %eight : index - %bias_i32 = arith.index_cast %bias_value : index to i32 - %bias_row = arith.divui %row, %four : index - %bias_lane = arith.remui %row, %four : index - memref.store %bias_i32, %bias[%bias_row, %bias_lane] : memref<4x4xi32> - scf.for %col = %zero to %sixteen step %one { - %r32 = arith.index_cast %row : index to i32 - %c32 = arith.index_cast %col : index to i32 - %plus32 = arith.addi %r32, %c32 : i32 - %twice32 = arith.addi %r32, %r32 : i32 - %minus32 = arith.subi %twice32, %c32 : i32 - %plus = arith.trunci %plus32 : i32 to i8 - %minus = arith.trunci %minus32 : i32 to i8 - memref.store %plus, %b0[%row, %col] : memref<16x16xi8> - memref.store %minus, %b1[%row, %col] : memref<16x16xi8> - } - } - %bias_bank = buckyball.bank_alloc - %a0_bank = buckyball.bank_alloc - %b0_bank = buckyball.bank_alloc - %a1_bank = buckyball.bank_alloc - %b1_bank = buckyball.bank_alloc - %result_bank = buckyball.bank_alloc - %loaded_bias = buckyball.bank_mvin %bias %bias_bank %depth4 %stride : memref<4x4xi32> i64 i64 i64 - %loaded_a0 = buckyball.bank_mvin %a %a0_bank %m_i64 %stride : memref<@BB_SMM_M@x16xi8> i64 i64 i64 - %loaded_b0 = buckyball.bank_mvin %b0 %b0_bank %depth16 %stride : memref<16x16xi8> i64 i64 i64 - %loaded_a1 = buckyball.bank_mvin %a %a1_bank %m_i64 %stride : memref<@BB_SMM_M@x16xi8> i64 i64 i64 - %loaded_b1 = buckyball.bank_mvin %b1 %b1_bank %depth16 %stride : memref<16x16xi8> i64 i64 i64 - %bias_state = buckyball.bank_smatmul_bias %loaded_bias %zero64 : i64 i64 - %partial = buckyball.bank_smatmul %loaded_a0 %loaded_b0 %result_bank %config %true %false %zero64 : i64 i64 i64 i64 - %result = buckyball.bank_smatmul %loaded_a1 %loaded_b1 %partial %config %false %true %zero64 : i64 i64 i64 i64 - %stored = buckyball.bank_mvout %output %result %depth_out %stride : memref<@BB_BANK_LINES@x4xi32> i64 i64 i64 - func.call @check_result(%output) : (memref<@BB_BANK_LINES@x4xi32>) -> () - buckyball.bank_release %bias_state : i64 - buckyball.bank_release %loaded_a0 : i64 - buckyball.bank_release %loaded_b0 : i64 - buckyball.bank_release %loaded_a1 : i64 - buckyball.bank_release %loaded_b1 : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %a : memref<@BB_SMM_M@x16xi8> - memref.dealloc %b0 : memref<16x16xi8> - memref.dealloc %b1 : memref<16x16xi8> - memref.dealloc %bias : memref<4x4xi32> - memref.dealloc %output : memref<@BB_BANK_LINES@x4xi32> - return %zero_i8 : i8 -} diff --git a/examples/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms_main.cpp b/examples/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms_main.cpp deleted file mode 100644 index 5a3eff2ea..000000000 --- a/examples/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms_main.cpp +++ /dev/null @@ -1,23 +0,0 @@ -#include -#include -#include -#include -extern "C" void check_result(int32_t *, int32_t *data, int64_t offset, - int64_t rows, int64_t lanes, int64_t stride, - int64_t lane_stride) { - if (rows != BANK_LINES || lanes != 4 || stride != 4 || lane_stride != 1) - exit(1); - for (int matrix_row = 0; matrix_row < BANK_LINES / 4; ++matrix_row) - for (int column = 0; column < 16; ++column) { - int bank_row = matrix_row * 4 + column / 4; - int bank_lane = column % 4; - int32_t expected = column - 8; - for (int k = 0; k < 16; ++k) - expected += (matrix_row % 16 - k) * ((k + column) + (2 * k - column)); - if (data[offset + bank_row * stride + bank_lane] != expected) { - printf("smatmul bank mismatch row=%d column=%d\n", matrix_row, column); - exit(1); - } - } - printf("smatmul bank PASS rows=%d\n", BANK_LINES); -} diff --git a/examples/balls/toint8/arch/src/main/scala/ToInt8Ball.scala b/examples/balls/toint8/arch/src/main/scala/ToInt8Ball.scala deleted file mode 100644 index 0f32431a5..000000000 --- a/examples/balls/toint8/arch/src/main/scala/ToInt8Ball.scala +++ /dev/null @@ -1,325 +0,0 @@ -package examples.balls.toint8 - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import hardfloat.{recFNFromFN, INToRecFN, RecFNToIN} -import hardfloat.consts.{round_near_even, tininess_afterRounding} -import freechips.rocketchip.tile.MulAddRecFNPipe - -import framework.balldomain.blink.{BallStatus, BlinkIO, HasBallStatus, HasBlink, SubRobRow} -import framework.balldomain.blink.mmio.{MmioRead, MmioWrite} -import framework.top.GlobalConfig - -@instantiable -class ToInt8Ball(val b: GlobalConfig) extends Module with HasBlink with HasBallStatus { - - private val mapping = b.ballDomain.ballIdMappings - .find(_.ballName == "ToInt8Ball") - .getOrElse(throw new IllegalArgumentException("ToInt8Ball not found in config")) - - private val f32Funct = b.ballDomain.ballISA - .find(_.mnemonic == "QUANT_F32_TO_I8") - .map(_.funct7) - .getOrElse(throw new IllegalArgumentException("QUANT_F32_TO_I8 not found in ballISA")) - - private val i32Funct = b.ballDomain.ballISA - .find(_.mnemonic == "QUANT_I32_TO_I8") - .map(_.funct7) - .getOrElse(throw new IllegalArgumentException("QUANT_I32_TO_I8 not found in ballISA")) - - require(b.memDomain.bankWidth == 128, "ToInt8Ball requires 128-bit bank rows") - require(b.memDomain.bankEntries >= 4, "ToInt8Ball requires at least four bank rows") - require(mapping.inBW == 1, "ToInt8Ball requires inBW=1") - require(mapping.outBW == 1, "ToInt8Ball requires outBW=1") - require((f32Funct >> 4) == 3, "QUANT_F32_TO_I8 must encode one read and one write") - require((i32Funct >> 4) == 4, "QUANT_I32_TO_I8 must encode two reads and one write") - - @public - val io = IO(new BlinkIO(b, mapping.inBW, mapping.outBW)) - def blink: BlinkIO = io - def status: BallStatus = io.status - dontTouch(io) - - private val idle :: scaleReq :: scaleResp :: inputReq :: inputResp :: pack :: packDrain :: writeReq :: writeResp :: complete :: Nil = - Enum(10) - private val state = RegInit(idle) - - private val robIdReg = RegInit(0.U(log2Up(b.frontend.rob_entries).W)) - private val isSubReg = RegInit(false.B) - private val subRobIdReg = RegInit(0.U(log2Up(b.frontend.sub_rob_depth * 4).W)) - private val inputBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - private val scaleBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - private val outputBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - private val iterReg = RegInit(0.U(b.frontend.iter_len.W)) - private val inputBaseReg = RegInit(0.U(log2Ceil(b.memDomain.bankEntries).W)) - // Relative row within the command. inputBaseReg is the absolute bank row. - private val relativeRow = RegInit(0.U(log2Ceil(b.memDomain.bankEntries).W)) - private val outputRow = RegInit(0.U(log2Ceil(b.memDomain.bankEntries).W)) - private val outputWidth = RegInit(0.U(7.W)) - private val outputStride = RegInit(0.U(7.W)) - private val outputColumn = RegInit(0.U(7.W)) - private val scaleRow = RegInit(0.U(2.W)) - private val i32Mode = RegInit(false.B) - private val reluReg = RegInit(false.B) - private val tensorScale = RegInit(0.U(32.W)) - private val scales = Reg(Vec(16, UInt(32.W))) - private val inputData = Reg(UInt(128.W)) - private val outputData = RegInit(0.U(128.W)) - private val writeData = Reg(UInt(128.W)) - private val lane = RegInit(0.U(2.W)) - private val outputBytes = Reg(Vec(4, UInt(8.W))) - - private def positiveFinite(value: UInt): Bool = - !value(31) && value(30, 23) =/= 255.U && value(30, 0) =/= 0.U - - private val intToFloat = Module(new INToRecFN(32, 8, 24)) - // The unpipelined recoded-float multiplier was the critical Buckyball path - // (about 8.5 ns before clock uncertainty). FMA mode with c=0 preserves the - // multiplication result while splitting the datapath into three stages. - private val multiply = Module(new MulAddRecFNPipe(3, 8, 24)) - private val floatToInt = Module(new RecFNToIN(8, 24, 8)) - private val input = (inputData >> (lane << 5))(31, 0) - private val scale = Mux(i32Mode, scales(Cat(relativeRow(1, 0), lane)), tensorScale) - private val packIssue = state === pack - - private val inputStage0 = RegEnable(input, packIssue) - private val scaleStage0 = RegEnable(scale, packIssue) - private val laneStage0 = RegEnable(lane, packIssue) - private val convertValid = RegNext(packIssue, false.B) - - private val multiplyInput = RegEnable( - Mux(i32Mode, intToFloat.io.out, recFNFromFN(8, 24, inputStage0)), - convertValid - ) - - private val multiplyScale = RegEnable(recFNFromFN(8, 24, scaleStage0), convertValid) - private val laneStage1 = RegEnable(laneStage0, convertValid) - private val multiplyValid = RegNext(convertValid, false.B) - - private val multiplyResult = RegEnable(multiply.io.out, multiply.io.validout) - private val multiplyLane = Pipe(multiplyValid, laneStage1, 3) - private val laneStage2 = RegEnable(multiplyLane.bits, multiply.io.validout) - private val toIntValid = RegNext(multiply.io.validout, false.B) - - private val quantizedResult = RegEnable( - Mux(reluReg && floatToInt.io.out(7), 0.U, floatToInt.io.out), - toIntValid - ) - - private val quantizedLane = RegEnable(laneStage2, toIntValid) - private val quantizedValid = RegNext(toIntValid, false.B) - - intToFloat.io.signedIn := true.B - intToFloat.io.in := inputStage0 - intToFloat.io.roundingMode := round_near_even - intToFloat.io.detectTininess := tininess_afterRounding - - multiply.io.a := multiplyInput - multiply.io.b := multiplyScale - multiply.io.c := recFNFromFN(8, 24, 0.U) - multiply.io.op := 0.U - multiply.io.validin := multiplyValid - multiply.io.roundingMode := round_near_even - multiply.io.detectTininess := tininess_afterRounding.asUInt - - floatToInt.io.in := multiplyResult - floatToInt.io.roundingMode := round_near_even - floatToInt.io.signedOut := true.B - - io.cmdReq.ready := state === idle - io.cmdResp.valid := state === complete - io.cmdResp.bits.rob_id := robIdReg - io.cmdResp.bits.is_sub := isSubReg - io.cmdResp.bits.sub_rob_id := subRobIdReg - io.status.idle := state === idle - io.status.running := state =/= idle && state =/= complete - - io.bankRead(0).rob_id := robIdReg - io.bankRead(0).ball_id := 0.U - io.bankRead(0).bank_id := Mux(state === scaleReq || state === scaleResp, scaleBank, inputBank) - io.bankRead(0).group_id := 0.U - io.bankRead(0).io.req.valid := false.B - io.bankRead(0).io.req.bits.addr := 0.U - io.bankRead(0).io.resp.ready := false.B - - io.bankWrite(0).rob_id := robIdReg - io.bankWrite(0).ball_id := 0.U - io.bankWrite(0).bank_id := outputBank - io.bankWrite(0).group_id := 0.U - io.bankWrite(0).io.req.valid := false.B - io.bankWrite(0).io.req.bits.addr := outputRow - io.bankWrite(0).io.req.bits.data := writeData - io.bankWrite(0).io.req.bits.mask := VecInit(Seq.fill(b.memDomain.bankMaskLen)(true.B)) - io.bankWrite(0).io.resp.ready := false.B - - io.subRobReq.valid := false.B - io.subRobReq.bits := SubRobRow.tieOff(b) - MmioRead.tieOff(io.mmioRead) - MmioWrite.tieOff(io.mmioWrite) - - switch(state) { - is(idle) { - when(io.cmdReq.fire) { - val cmd = io.cmdReq.bits.cmd - val isF32 = cmd.funct7 === f32Funct.U - val isI32 = cmd.funct7 === i32Funct.U - - assert(isF32 || isI32, "ToInt8Ball received an unknown funct7") - assert(cmd.iter > 0.U && cmd.iter(1, 0) === 0.U, "ToInt8Ball iter must be a positive multiple of four") - assert(cmd.iter <= b.memDomain.bankEntries.U, "ToInt8Ball input exceeds bank depth") - assert(cmd.rs1(9, 0) < b.memDomain.virtualBankCount.U, "ToInt8Ball input bank is invalid") - assert(cmd.rs1(29, 20) < b.memDomain.virtualBankCount.U, "ToInt8Ball output bank is invalid") - assert(cmd.op1_col === 1.U && cmd.wr_col === 1.U, "ToInt8Ball input and output must each occupy one bank") - assert(cmd.op1_bank =/= cmd.wr_bank, "ToInt8Ball input and output banks must differ") - when(isF32) { - assert(cmd.rs1(19, 10) === 0.U, "QUANT_F32_TO_I8 reserves input bank 1") - assert(cmd.rs2(63, 32) === 0.U, "QUANT_F32_TO_I8 reserves rs2[63:32]") - assert(positiveFinite(cmd.rs2(31, 0)), "QUANT_F32_TO_I8 scale must be finite and positive") - }.otherwise { - assert(cmd.rs1(19, 10) < b.memDomain.virtualBankCount.U, "QUANT_I32_TO_I8 scale bank is invalid") - assert(cmd.op2_col === 1.U, "QUANT_I32_TO_I8 scale must occupy one bank") - assert(cmd.op1_bank =/= cmd.op2_bank, "QUANT_I32_TO_I8 input and scale banks must differ") - assert(cmd.op2_bank =/= cmd.wr_bank, "QUANT_I32_TO_I8 scale and output banks must differ") - val outputBase = cmd.rs2(7, 1) - val width = cmd.rs2(14, 8) - val height = cmd.rs2(21, 15) - val stride = cmd.rs2(28, 22) - val outputRows = cmd.iter >> 2 - val outputEnd = outputBase +& ((height - 1.U) * stride) +& width - val inputBase = cmd.rs2(34, 29) - assert(cmd.rs2(63, 35) === 0.U, "QUANT_I32_TO_I8 reserves rs2[63:35]") - assert(width > 0.U && height > 0.U && stride >= width, "QUANT_I32_TO_I8 output geometry is invalid") - assert(width * height === outputRows, "QUANT_I32_TO_I8 iter does not match output geometry") - assert(outputEnd <= b.memDomain.bankEntries.U, "QUANT_I32_TO_I8 output exceeds bank depth") - assert(inputBase +& cmd.iter <= b.memDomain.bankEntries.U, "QUANT_I32_TO_I8 input exceeds bank depth") - } - - robIdReg := io.cmdReq.bits.rob_id - isSubReg := io.cmdReq.bits.is_sub - subRobIdReg := io.cmdReq.bits.sub_rob_id - inputBank := cmd.op1_bank - scaleBank := cmd.op2_bank - outputBank := cmd.wr_bank - iterReg := cmd.iter - inputBaseReg := Mux(isI32, cmd.rs2(34, 29), 0.U) - relativeRow := 0.U - outputRow := Mux(isI32, cmd.rs2(7, 1), 0.U) - outputWidth := Mux(isI32, cmd.rs2(14, 8), 0.U) - outputStride := Mux(isI32, cmd.rs2(28, 22), 0.U) - outputColumn := 0.U - scaleRow := 0.U - i32Mode := isI32 - reluReg := isI32 && cmd.rs2(0) - tensorScale := cmd.rs2(31, 0) - outputData := 0.U - lane := 0.U - state := Mux(isI32, scaleReq, inputReq) - } - } - - is(scaleReq) { - io.bankRead(0).io.req.valid := true.B - io.bankRead(0).io.req.bits.addr := scaleRow - when(io.bankRead(0).io.req.fire) { - state := scaleResp - } - } - - is(scaleResp) { - io.bankRead(0).io.resp.ready := true.B - when(io.bankRead(0).io.resp.fire) { - for (lane <- 0 until 4) { - val value = io.bankRead(0).io.resp.bits.data(32 * lane + 31, 32 * lane) - assert(positiveFinite(value), "QUANT_I32_TO_I8 scales must be finite and positive") - scales(Cat(scaleRow, lane.U(2.W))) := value - } - when(scaleRow === 3.U) { - state := inputReq - }.otherwise { - scaleRow := scaleRow + 1.U - state := scaleReq - } - } - } - - is(inputReq) { - io.bankRead(0).io.req.valid := true.B - io.bankRead(0).io.req.bits.addr := inputBaseReg +& relativeRow - when(io.bankRead(0).io.req.fire) { - state := inputResp - } - } - - is(inputResp) { - io.bankRead(0).io.resp.ready := true.B - when(io.bankRead(0).io.resp.fire) { - inputData := io.bankRead(0).io.resp.bits.data - lane := 0.U - state := pack - } - } - - is(pack) { - when(!i32Mode) { - assert(input(30, 23) =/= 255.U, "QUANT_F32_TO_I8 input must be finite") - } - when(lane === 3.U) { - state := packDrain - }.otherwise { - lane := lane + 1.U - } - } - - is(packDrain) {} - - is(writeReq) { - io.bankWrite(0).io.req.valid := true.B - when(io.bankWrite(0).io.req.fire) { - state := writeResp - } - } - - is(writeResp) { - io.bankWrite(0).io.resp.ready := true.B - when(io.bankWrite(0).io.resp.fire) { - when(relativeRow === iterReg - 1.U) { - state := complete - }.otherwise { - relativeRow := relativeRow + 1.U - when(outputColumn === outputWidth - 1.U) { - outputRow := outputRow + outputStride - outputWidth + 1.U - outputColumn := 0.U - }.otherwise { - outputRow := outputRow + 1.U - outputColumn := outputColumn + 1.U - } - state := inputReq - } - } - } - - is(complete) { - when(io.cmdResp.fire) { - state := idle - } - } - } - - when((state === pack || state === packDrain) && quantizedValid) { - outputBytes(quantizedLane) := quantizedResult - when(quantizedLane === 3.U) { - val bytes = Cat(quantizedResult, outputBytes(2), outputBytes(1), outputBytes(0)) - val nextWord = outputData | (bytes << (relativeRow(1, 0) << 5)) - when(relativeRow(1, 0) === 3.U) { - writeData := nextWord - outputData := 0.U - state := writeReq - }.otherwise { - outputData := nextWord - relativeRow := relativeRow + 1.U - state := inputReq - } - } - } -} diff --git a/examples/balls/toint8/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp b/examples/balls/toint8/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp deleted file mode 100644 index 35ff7f000..000000000 --- a/examples/balls/toint8/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp +++ /dev/null @@ -1,56 +0,0 @@ -//===- AssignPhysicalBankPatterns.cpp - ToInt8 bank assignment patterns ---===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace mlir::buddy { -void populateToInt8BallAssignPhysicalBankPatterns(RewritePatternSet &patterns, - PhysicalBankState &state); -} // namespace mlir::buddy - -namespace { - -class BankQuantF32ToI8Pattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(BankQuantF32ToI8Op op, - PatternRewriter &rewriter) const override { - rewriter.create(op.getLoc(), op.getInBank(), - op.getOutBank(), op.getIter(), - op.getScale()); - rewriter.replaceOp(op, op.getOutBank()); - return success(); - } -}; - -class BankQuantI32ToI8Pattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(BankQuantI32ToI8Op op, - PatternRewriter &rewriter) const override { - rewriter.create( - op.getLoc(), op.getInBank(), op.getScaleBank(), op.getOutBank(), - op.getIter(), op.getOutputBase(), op.getInputBase(), - op.getOutputWidthAttr(), op.getOutputHeightAttr(), - op.getOutputStrideAttr(), op.getReluAttr()); - rewriter.replaceOp(op, op.getOutBank()); - return success(); - } -}; - -} // namespace - -void mlir::buddy::populateToInt8BallAssignPhysicalBankPatterns( - RewritePatternSet &patterns, mlir::buddy::PhysicalBankState &state) { - (void)state; - patterns.add( - patterns.getContext()); -} diff --git a/examples/balls/toint8/compiler/src/Dialect/Buckyball/ToInt8Ball.td b/examples/balls/toint8/compiler/src/Dialect/Buckyball/ToInt8Ball.td deleted file mode 100644 index 0cc1ea566..000000000 --- a/examples/balls/toint8/compiler/src/Dialect/Buckyball/ToInt8Ball.td +++ /dev/null @@ -1,59 +0,0 @@ -//===-- ToInt8Ball.td - ToInt8Ball operations ----------------*- tablegen -*-===// -// -// ToInt8Ball operations for quantization. -// -//===----------------------------------------------------------------------===// - -#ifndef TOINT8BALL_OPS -#define TOINT8BALL_OPS - -include "Buckyball.td" - -def QuantF32ToI8Op : Buckyball_Op<"quant_f32_to_i8"> { - let summary = "Per-tensor FP32 to INT8 quantization"; - let arguments = (ins I64:$inputBankId, I64:$outputBankId, - I64:$iter, F32:$scale); - let assemblyFormat = "$inputBankId `,` $outputBankId `,` $iter `,` $scale " - "prop-dict attr-dict `:` type($inputBankId)"; -} - -def BankQuantF32ToI8Op : Buckyball_Op<"bank_quant_f32_to_i8"> { - let summary = "Per-tensor FP32 to INT8 quantization with virtual banks"; - let arguments = (ins I64:$inBank, I64:$outBank, I64:$iter, F32:$scale); - let results = (outs I64:$outBankOut); - let assemblyFormat = "$inBank $outBank $iter $scale prop-dict attr-dict `:` " - "type($inBank) type($outBank) type($iter) type($scale)"; -} - -def QuantI32ToI8Op : Buckyball_Op<"quant_i32_to_i8"> { - let summary = "Per-channel INT32 to INT8 requantization"; - let description = [{ - The output is a dense input stream placed in a strided 2-D bank tile. - iter/4 must equal outputWidth*outputHeight. - }]; - let arguments = (ins I64:$inputBankId, I64:$scaleBankId, - I64:$outputBankId, I64:$iter, I64:$outputBase, - I64:$inputBase, - I64Attr:$outputWidth, I64Attr:$outputHeight, - I64Attr:$outputStride, - BoolAttr:$relu); - let assemblyFormat = "$inputBankId `,` $scaleBankId `,` $outputBankId `,` " - "$iter `,` $outputBase `,` $inputBase prop-dict attr-dict `:` " - "type($inputBankId) type($outputBase) type($inputBase)"; -} - -def BankQuantI32ToI8Op : Buckyball_Op<"bank_quant_i32_to_i8"> { - let summary = "Per-channel INT32 to INT8 requantization with virtual banks"; - let arguments = (ins I64:$inBank, I64:$scaleBank, I64:$outBank, - I64:$iter, I64:$outputBase, - I64:$inputBase, - I64Attr:$outputWidth, I64Attr:$outputHeight, - I64Attr:$outputStride, BoolAttr:$relu); - let results = (outs I64:$outBankOut); - let assemblyFormat = "$inBank $scaleBank $outBank $iter $outputBase $inputBase " - "prop-dict attr-dict `:` type($inBank) type($scaleBank) " - "type($outBank) type($iter) type($outputBase) " - "type($inputBase)"; -} - -#endif // TOINT8BALL_OPS diff --git a/examples/balls/toint8/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/balls/toint8/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index a83e6e77f..000000000 --- a/examples/balls/toint8/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,122 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace buddy::buckyball; -using namespace buddy::buckyball::legalize; - -namespace { -struct QuantF32ToI8Lowering : public ConvertOpToLLVMPattern { - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - - LogicalResult - matchAndRewrite(QuantF32ToI8Op op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("ToInt8Ball"); - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInputBankId(), - cstI64(rewriter, loc, 0), - adaptor.getOutputBankId(), adaptor.getIter()); - Value scaleBits = rewriter.create( - loc, rewriter.getI32Type(), adaptor.getScale()); - Value rs2 = - rewriter.create(loc, rewriter.getI64Type(), scaleBits); - rewriter.replaceOpWithNewOp( - op, rs1, rs2, - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("QUANT_F32_TO_I8"))); - return success(); - } -}; - -struct QuantI32ToI8Lowering : public ConvertOpToLLVMPattern { - QuantI32ToI8Lowering(LLVMTypeConverter &converter, int64_t bankDepth) - : ConvertOpToLLVMPattern(converter), - bankDepth(bankDepth) {} - - LogicalResult - matchAndRewrite(QuantI32ToI8Op op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("ToInt8Ball"); - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInputBankId(), - adaptor.getScaleBankId(), - adaptor.getOutputBankId(), adaptor.getIter()); - auto iterOp = op.getIter().getDefiningOp(); - auto iterAttr = - iterOp ? dyn_cast(iterOp.getValue()) : IntegerAttr(); - if (!iterAttr || iterAttr.getInt() <= 0) - return op.emitError("INT32 to INT8 iter must be a positive constant"); - if (iterAttr.getInt() % 4 != 0 || iterAttr.getInt() > bankDepth) - return op.emitError( - "INT32 to INT8 iter must be a multiple of four within bank depth"); - int64_t outputRows = iterAttr.getInt() / 4; - int64_t outputWidth = op.getOutputWidthAttr().getInt(); - int64_t outputHeight = op.getOutputHeightAttr().getInt(); - int64_t outputStride = op.getOutputStrideAttr().getInt(); - auto outputBaseOp = - adaptor.getOutputBase().getDefiningOp(); - auto inputBaseOp = - adaptor.getInputBase().getDefiningOp(); - auto outputBaseAttr = outputBaseOp - ? dyn_cast(outputBaseOp.getValue()) - : IntegerAttr(); - auto inputBaseAttr = inputBaseOp - ? dyn_cast(inputBaseOp.getValue()) - : IntegerAttr(); - if (outputWidth <= 0 || outputHeight <= 0 || outputStride < outputWidth || - outputWidth * outputHeight != outputRows || outputWidth >= 128 || - outputHeight >= 128 || outputStride >= 128 || !outputBaseAttr || - !inputBaseAttr || outputBaseAttr.getInt() < 0 || - inputBaseAttr.getInt() < 0 || - outputBaseAttr.getInt() + (outputHeight - 1) * outputStride + - outputWidth > - bankDepth || - inputBaseAttr.getInt() + iterAttr.getInt() > bankDepth) - return op.emitError( - "INT32 to INT8 output tile is invalid for the physical bank"); - Value rs2 = rewriter.create(loc, adaptor.getOutputBase(), - cstI64(rewriter, loc, 1)); - rs2 = rewriter.create( - loc, rs2, - cstI64(rewriter, loc, - (outputWidth << 8) | (outputHeight << 15) | - (outputStride << 22) | (op.getRelu() ? 1 : 0))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getInputBase(), - cstI64(rewriter, loc, 29))); - rewriter.replaceOpWithNewOp( - op, rs1, rs2, - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("QUANT_I32_TO_I8"))); - return success(); - } - -private: - int64_t bankDepth; -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateToInt8BallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth) { - (void)stable; - patterns.add(converter); - patterns.add(converter, bankDepth); -} - -void configureToInt8BallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable) { - (void)stable; - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/examples/balls/toint8/configs/default.toml b/examples/balls/toint8/configs/default.toml deleted file mode 100644 index 83a5aded0..000000000 --- a/examples/balls/toint8/configs/default.toml +++ /dev/null @@ -1,3 +0,0 @@ -# ToInt8Ball has no private configuration parameters. - -[ball] diff --git a/examples/balls/toint8/emu/src/dispatch.rs b/examples/balls/toint8/emu/src/dispatch.rs deleted file mode 100644 index c85c832ff..000000000 --- a/examples/balls/toint8/emu/src/dispatch.rs +++ /dev/null @@ -1,33 +0,0 @@ -use crate::inst::instruction::ExecContext; - -use super::quant; - -const BALL_CLASS: &str = "examples.balls.toint8.ToInt8Ball"; - -pub fn execute_known( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, - ctx: &mut ExecContext, -) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("QUANT_F32_TO_I8") => Some(quant::exec_f32_to_i8(xs1, xs2, ctx)), - Some("QUANT_I32_TO_I8") => Some(quant::exec_i32_to_i8(xs1, xs2, ctx)), - Some(_) | None => None, - } -} - -pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("QUANT_F32_TO_I8") => Some(quant::f32_to_i8_latency(xs1, xs2)), - Some("QUANT_I32_TO_I8") => Some(quant::i32_to_i8_latency(xs1, xs2)), - Some(_) | None => None, - } -} diff --git a/examples/balls/toint8/emu/src/lib.rs b/examples/balls/toint8/emu/src/lib.rs deleted file mode 100644 index 253402123..000000000 --- a/examples/balls/toint8/emu/src/lib.rs +++ /dev/null @@ -1,12 +0,0 @@ -pub(crate) use crate::inst::decode; -pub(crate) use crate::inst::instruction; - -mod model; -mod quant; - -#[cfg(test)] -mod tests; - -mod dispatch; - -pub use dispatch::{cycles_after_issue, execute_known}; diff --git a/examples/balls/toint8/emu/src/quant.rs b/examples/balls/toint8/emu/src/quant.rs deleted file mode 100644 index 314adec29..000000000 --- a/examples/balls/toint8/emu/src/quant.rs +++ /dev/null @@ -1,166 +0,0 @@ -use super::super::bank::bank_lines; -use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; -use super::instruction::ExecContext; - -pub(crate) fn exec_f32_to_i8(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { - let src = rs1_b0(xs1); - let unused = rs1_b1(xs1); - let dst = rs1_b2(xs1); - let iter = rs1_iter(xs1) as usize; - if unused != 0 { - panic!("quant_f32_to_i8: input bank 1 must be zero"); - } - if src == dst { - panic!("quant_f32_to_i8: input and output banks must differ"); - } - if !ctx.config(src).allocated || !ctx.config(dst).allocated { - panic!("quant_f32_to_i8: bank not allocated"); - } - if ctx.config(src).cols != 1 || ctx.config(dst).cols != 1 { - panic!("quant_f32_to_i8: banks must have one column"); - } - if iter == 0 || iter % 4 != 0 || iter > bank_lines() { - panic!("quant_f32_to_i8: iter must be a positive multiple of four within bank depth"); - } - if xs2 >> 32 != 0 { - panic!("quant_f32_to_i8: rs2[63:32] must be zero"); - } - let scale = f32::from_bits(xs2 as u32); - if !scale.is_finite() || scale <= 0.0 { - panic!("quant_f32_to_i8: scale must be finite and positive"); - } - - let ps = pbank(ctx, src); - let pd = pbank(ctx, dst); - let (input, output) = ctx.banks.read_write(ps, pd); - for output_row in 0..iter / 4 { - let mut packed = [0u8; 16]; - for group in 0..4 { - let input_row = output_row * 4 + group; - let base = input_row * 16; - for lane in 0..4 { - let offset = base + lane * 4; - let bits = u32::from_le_bytes(input[offset..offset + 4].try_into().unwrap()); - packed[group * 4 + lane] = - super::model::quantize_f32(f32::from_bits(bits), scale, false) as u8; - } - } - let base = output_row * 16; - output[base..base + 16].copy_from_slice(&packed); - } - 0 -} - -pub(crate) fn exec_i32_to_i8(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { - let src = rs1_b0(xs1); - let scale_bank = rs1_b1(xs1); - let dst = rs1_b2(xs1); - let iter = rs1_iter(xs1) as usize; - if src == scale_bank || src == dst || scale_bank == dst { - panic!("quant_i32_to_i8: input, scale, and output banks must differ"); - } - if !ctx.config(src).allocated || !ctx.config(scale_bank).allocated || !ctx.config(dst).allocated - { - panic!("quant_i32_to_i8: bank not allocated"); - } - if ctx.config(src).cols != 1 || ctx.config(scale_bank).cols != 1 || ctx.config(dst).cols != 1 { - panic!("quant_i32_to_i8: banks must have one column"); - } - if iter == 0 || iter % 4 != 0 || iter > bank_lines() { - panic!("quant_i32_to_i8: iter must be a positive multiple of four within bank depth"); - } - if xs2 >> 35 != 0 { - panic!("quant_i32_to_i8: rs2[63:35] must be zero"); - } - let input_base = ((xs2 >> 29) & 0x3f) as usize; - let output_base = ((xs2 >> 1) & 0x7f) as usize; - let output_width = ((xs2 >> 8) & 0x7f) as usize; - let output_height = ((xs2 >> 15) & 0x7f) as usize; - let output_stride = ((xs2 >> 22) & 0x7f) as usize; - let output_rows = iter / 4; - if output_width == 0 - || output_height == 0 - || output_stride < output_width - || output_width.checked_mul(output_height) != Some(output_rows) - || output_base - .checked_add((output_height - 1) * output_stride + output_width) - .is_none_or(|end| end > bank_lines()) - || input_base + iter > bank_lines() - { - panic!("quant_i32_to_i8: invalid output tile"); - } - let relu = xs2 & 1 != 0; - - let pscale = pbank(ctx, scale_bank); - let mut scales = [0.0f32; 16]; - for group in 0..4 { - let base = group * 16; - for lane in 0..4 { - let offset = base + lane * 4; - scales[group * 4 + lane] = f32::from_bits(u32::from_le_bytes( - ctx.banks[pscale][offset..offset + 4].try_into().unwrap(), - )); - if !scales[group * 4 + lane].is_finite() || scales[group * 4 + lane] <= 0.0 { - panic!("quant_i32_to_i8: scales must be finite and positive"); - } - } - } - - let ps = pbank(ctx, src); - let pd = pbank(ctx, dst); - let (input, output) = ctx.banks.read_write(ps, pd); - for output_row in 0..iter / 4 { - let mut packed = [0u8; 16]; - for group in 0..4 { - let input_row = input_base + output_row * 4 + group; - let base = input_row * 16; - for lane in 0..4 { - let offset = base + lane * 4; - let value = i32::from_le_bytes(input[offset..offset + 4].try_into().unwrap()); - let channel = group * 4 + lane; - packed[channel] = super::model::quantize_i32(value, scales[channel], relu) as u8; - } - } - let row = - output_base + (output_row / output_width) * output_stride + output_row % output_width; - let base = row * 16; - output[base..base + 16].copy_from_slice(&packed); - } - 0 -} - -pub(crate) fn f32_to_i8_latency(xs1: u64, xs2: u64) -> u64 { - let iter = rs1_iter(xs1); - if iter == 0 || iter % 4 != 0 || xs2 >> 32 != 0 { - panic!("quant_f32_to_i8: illegal encoding"); - } - let scale = f32::from_bits(xs2 as u32); - if !scale.is_finite() || scale <= 0.0 { - panic!("quant_f32_to_i8: scale must be finite and positive"); - } - iter -} - -pub(crate) fn i32_to_i8_latency(xs1: u64, xs2: u64) -> u64 { - let iter = rs1_iter(xs1); - let input_base = (xs2 >> 29) & 0x3f; - if xs2 >> 35 != 0 { - panic!("quant_i32_to_i8: illegal encoding"); - } - let output_base = (xs2 >> 1) & 0x7f; - let output_width = (xs2 >> 8) & 0x7f; - let output_height = (xs2 >> 15) & 0x7f; - let output_stride = (xs2 >> 22) & 0x7f; - if iter == 0 - || iter % 4 != 0 - || output_width == 0 - || output_height == 0 - || output_stride < output_width - || output_width * output_height != iter / 4 - || output_base + (output_height - 1) * output_stride + output_width > bank_lines() as u64 - || input_base + iter > bank_lines() as u64 - { - panic!("quant_i32_to_i8: illegal encoding"); - } - iter + 4 -} diff --git a/examples/balls/toint8/emu/src/tests/activation_scale.rs b/examples/balls/toint8/emu/src/tests/activation_scale.rs deleted file mode 100644 index 60af3f030..000000000 --- a/examples/balls/toint8/emu/src/tests/activation_scale.rs +++ /dev/null @@ -1,11 +0,0 @@ -use super::super::model::{toint8_da_from_max_abs_bits, fp32_divide}; - -#[test] -fn computes_activation_scale_with_fp32_division() { - assert_eq!( - fp32_divide(2.0f32.to_bits(), 127.0f32.to_bits()), - 0x3c81_0204 - ); - assert_eq!(toint8_da_from_max_abs_bits(0), 1.0f32.to_bits()); - assert_eq!(toint8_da_from_max_abs_bits(63.5f32.to_bits()), 0.5f32.to_bits()); -} diff --git a/examples/balls/toint8/emu/src/tests/int32_rounding.rs b/examples/balls/toint8/emu/src/tests/int32_rounding.rs deleted file mode 100644 index c08859048..000000000 --- a/examples/balls/toint8/emu/src/tests/int32_rounding.rs +++ /dev/null @@ -1,21 +0,0 @@ -use super::super::model::toint8_i32_bits; - -#[test] -fn rounds_to_nearest_even() { - let scale = 0x3f80_0000; - let cases = [ - (0x3f80_0000, 1), - (0x4000_0000, 2), - (0xbf80_0000, -1), - (0x3f00_0000, 0), - (0xbf00_0000, 0), - (0x3fc0_0000, 2), - (0xbfc0_0000, -2), - (0x4020_0000, 2), - (0xc020_0000, -2), - ]; - - for (input, expected) in cases { - assert_eq!(toint8_i32_bits(input, scale), expected); - } -} diff --git a/examples/balls/toint8/emu/src/tests/int8_saturation.rs b/examples/balls/toint8/emu/src/tests/int8_saturation.rs deleted file mode 100644 index 47ac8554b..000000000 --- a/examples/balls/toint8/emu/src/tests/int8_saturation.rs +++ /dev/null @@ -1,9 +0,0 @@ -use super::super::model::toint8_i8_bits; - -#[test] -fn saturates_beyond_int8_range() { - let scale = 0x3f80_0000; - - assert_eq!(toint8_i8_bits(0x4300_0000, scale), 127); - assert_eq!(toint8_i8_bits(0xc300_0000, scale), -128); -} diff --git a/examples/balls/toint8/emu/src/tests/mod.rs b/examples/balls/toint8/emu/src/tests/mod.rs deleted file mode 100644 index 80ad99feb..000000000 --- a/examples/balls/toint8/emu/src/tests/mod.rs +++ /dev/null @@ -1 +0,0 @@ -mod quant; diff --git a/examples/balls/toint8/emu/src/tests/quant.rs b/examples/balls/toint8/emu/src/tests/quant.rs deleted file mode 100644 index da07539ed..000000000 --- a/examples/balls/toint8/emu/src/tests/quant.rs +++ /dev/null @@ -1,10 +0,0 @@ -use super::super::model::{quantize_f32, quantize_i32}; - -#[test] -fn rounds_ties_to_even_and_saturates() { - assert_eq!(quantize_f32(2.5, 1.0, false), 2); - assert_eq!(quantize_f32(3.5, 1.0, false), 4); - assert_eq!(quantize_i32(1000, 1.0, false), 127); - assert_eq!(quantize_i32(-1000, 1.0, false), -128); - assert_eq!(quantize_i32(-7, 1.0, true), 0); -} diff --git a/examples/balls/toint8/emu/src/tests/workload_vectors.rs b/examples/balls/toint8/emu/src/tests/workload_vectors.rs deleted file mode 100644 index dca0d4869..000000000 --- a/examples/balls/toint8/emu/src/tests/workload_vectors.rs +++ /dev/null @@ -1,19 +0,0 @@ -use super::super::model::toint8_i8_bits; - -#[test] -fn quantizes_workload_vectors() { - let scale = 2.0f32.to_bits(); - let input = [ - 0.125f32, -0.125, 0.25, -0.25, 0.75, -0.75, 1.25, -1.25, 1.75, -1.75, 63.25, 63.75, - -63.75, -64.75, 0.0, -0.0, 2.25, -2.25, 2.75, -2.75, 3.25, -3.25, 3.75, -3.75, 10.125, - -10.125, 20.25, -20.25, 0.375, -0.375, 64.25, -65.25, - ]; - let expected = [ - 0i8, 0, 0, 0, 2, -2, 2, -2, 4, -4, 126, 127, -128, -128, 0, 0, 4, -4, 6, -6, 6, -6, 8, - -8, 20, -20, 40, -40, 1, -1, 127, -128, - ]; - - for (value, expected) in input.into_iter().zip(expected) { - assert_eq!(toint8_i8_bits(value.to_bits(), scale), expected, "input={value}"); - } -} diff --git a/examples/balls/toint8/verify/README.md b/examples/balls/toint8/verify/README.md deleted file mode 100644 index 122824def..000000000 --- a/examples/balls/toint8/verify/README.md +++ /dev/null @@ -1,93 +0,0 @@ -# ToInt8Ball UVM Verification - -Verifies the generated `ToInt8Ball` module using the shared Blink UVM framework. - -## Structure - -- `../../../../verify/uvm/src/ball/bb_uvm_pkg.sv`: common Blink UVM agents and base env -- `src/common/toint8_defs.svh`: DPI imports, `toint8_require_bid`, timeouts -- `src/common/toint8_items.svh`: `toint8_cmd_item` with `load_rust_case` (Da address in `special[12:0]`) -- `src/seq/toint8_sequences.svh`: one-case sequence -- `src/cov/toint8_cov.svh`: online INT8 layout and iter 1 coverage -- `src/env/toint8_scoreboard.svh`: preload mem model (group-aware for i8), compare writes vs DPI ref -- `src/env/toint8_env.svh`: extends `bb_blink_env#(1,1)` -- `src/tests/toint8_*_test.svh`: one directed case per UVM test -- `src/pkg/toint8_pkg.sv`: package entry -- `src/tb_top.sv`: `bb_blink_if#(1,1)` + `ToInt8Ball` -- `filelists/toint8_ball.f`: VCS filelist - -## Test plan - -The test names are: - -- `toint8_signed_test`: signed values across all four source groups -- `toint8_zero_test`: all-zero activation (`Da=1.0`) -- `toint8_rounding_test`: ties-to-even quantization boundaries -- `toint8_rows_test`: two-row scan and packed output traversal -- `toint8_scale_rows_test`: maximum value is in the second row - -Each case: reset, preload src words into `mem_model` (INT8 uses `group_id` 0..3), -drive one command, wait for `scb.done()` or timeout at 400 cycles. Scoreboard -compares observed writes with DPI `toint8_ref_i8`. `toint8_cov` requires the online -INT8 layout and one-row iteration bin, else `check_phase` fatal. - -## BID (required) - -`+BID=` is mandatory. Missing it fatals at test start. No default bid. - -| Config | Plusarg | -|--------|---------| -| `sims.verilator.BuckyballPebbleVerilatorConfig` | `+BID=3` | -| `sims.verilator.BuckyballToyVerilatorConfig` (full.toml) | `+BID=5` | - -```console -./simv +UVM_TESTNAME=toint8_scale_rows_test +BID=3 -``` - -## Build - -Enter the shared UVM/VCS environment: - -```console -nix develop ../../../../verify -``` - -Build the DPI reference library: - -```console -cargo build --manifest-path casegen/Cargo.toml -``` - -Compile from this directory: - -```console -vcs -full64 -sverilog -timescale=1ns/1ps \ - $VCS_UVM_ARGS \ - -sv_lib casegen/target/debug/libtoint8_casegen \ - -f filelists/toint8_ball.f -``` - -Run: - -```console -./simv +UVM_TESTNAME=toint8_scale_rows_test +BID=3 -``` - -Or via bbdev: - -```console -bbdev uvm --build '--ball=toint8' \ - --config sims.verilator.BuckyballPebbleVerilatorConfig \ - --core-config examples/cores/pebble/configs/default.toml -bbdev uvm --run '--ball=toint8 --plusargs +BID=3' -``` - -## Acceptance (2026-08-04) - -- [x] Layout matches Blink common + ball-local split; filelist uses `@UVM@` / `@RTL@` -- [x] casegen reuses toint8 model; whole-case DPI API with injected bid -- [x] directed online INT8 scoreboard pass -- [x] signed, zero, rounding, and row-traversal directed cases pass -- [x] protocol + semantic cover enforced (`check_phase` fatal) -- [x] pebble: `+BID=3` config `sims.verilator.BuckyballPebbleVerilatorConfig` -- [ ] toy full: `+BID=5` config `sims.verilator.BuckyballToyVerilatorConfig` diff --git a/examples/balls/toint8/verify/casegen/src/casegen.rs b/examples/balls/toint8/verify/casegen/src/casegen.rs deleted file mode 100644 index 0491fc909..000000000 --- a/examples/balls/toint8/verify/casegen/src/casegen.rs +++ /dev/null @@ -1,593 +0,0 @@ -use crate::model::{quantize_f32, quantize_i32}; - -pub const KIND_F32: u32 = 0; -pub const KIND_I32: u32 = 1; -pub const BANK_NUM: u32 = 24; -pub const BANK_ENTRIES: u32 = 64; -pub const SCALE_ROWS: usize = 4; -pub const LANES: usize = 4; -pub const MAX_SRC: usize = 16; -pub const MAX_SCALE: usize = 4; -pub const MAX_DST: usize = 4; - -#[repr(C)] -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct ToInt8Cmd { - pub kind: u32, - pub bid: u32, - pub iter: u32, - pub op1_bank: u32, - pub op2_bank: u32, - pub wr_bank: u32, - pub op1_col: u32, - pub op2_col: u32, - pub wr_col: u32, - pub rob_id: u32, - pub rs1_lo: u32, - pub rs1_hi: u32, - pub rs2_lo: u32, - pub rs2_hi: u32, - pub input_base: u32, - pub num_src_words: u32, - pub num_scale_words: u32, - pub num_dst_words: u32, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ToInt8Case { - pub cmd: ToInt8Cmd, - pub src_words: [u128; MAX_SRC], - pub scale_words: [u128; MAX_SCALE], - pub dst_words: [u128; MAX_DST], - pub dst_addr: [u32; MAX_DST], -} - -impl ToInt8Case { - pub fn src_lo(&self, index: usize) -> u64 { - self.src_words[index] as u64 - } - - pub fn src_hi(&self, index: usize) -> u64 { - (self.src_words[index] >> 64) as u64 - } - - pub fn scale_lo(&self, index: usize) -> u64 { - self.scale_words[index] as u64 - } - - pub fn scale_hi(&self, index: usize) -> u64 { - (self.scale_words[index] >> 64) as u64 - } - - pub fn dst_lo(&self, index: usize) -> u64 { - self.dst_words[index] as u64 - } - - pub fn dst_hi(&self, index: usize) -> u64 { - (self.dst_words[index] >> 64) as u64 - } -} - -pub fn gen_case(index: u32, bid: u32) -> ToInt8Case { - match index { - 0 => f32_signed(bid), - 1 => f32_zero(bid), - 2 => f32_rounding(bid), - 3 => f32_rows(bid), - 4 => i32_scale_rows(bid), - 5 => i32_relu_tile(bid), - 6 => i32_signed(bid), - 7 => i32_offset(bid), - _ => panic!("toint8: unsupported directed case {index}"), - } -} - -fn pack_u32s(vals: &[u32]) -> Vec { - if vals.len() % LANES != 0 { - panic!("toint8: value count {} is not a multiple of four", vals.len()); - } - vals.chunks(LANES) - .map(|chunk| { - let mut word = 0u128; - for (lane, value) in chunk.iter().enumerate() { - word |= u128::from(*value) << (lane * 32); - } - word - }) - .collect() -} - -fn copy_words(src: &[u128]) -> [u128; N] { - if src.len() > N { - panic!("toint8: {} words exceed buffer {N}", src.len()); - } - let mut out = [0u128; N]; - out[..src.len()].copy_from_slice(src); - out -} - -fn copy_addrs(src: &[u32]) -> [u32; MAX_DST] { - if src.len() > MAX_DST { - panic!("toint8: {} dst rows exceed buffer", src.len()); - } - let mut out = [0u32; MAX_DST]; - out[..src.len()].copy_from_slice(src); - out -} - -fn encode_rs1(op1: u32, op2: u32, wr: u32, iter: u32) -> u64 { - u64::from(op1) | (u64::from(op2) << 10) | (u64::from(wr) << 20) | (u64::from(iter) << 30) -} - -fn encode_i32_rs2( - relu: bool, - out_base: u32, - width: u32, - height: u32, - stride: u32, - in_base: u32, -) -> u64 { - u64::from(u32::from(relu)) - | (u64::from(out_base) << 1) - | (u64::from(width) << 8) - | (u64::from(height) << 15) - | (u64::from(stride) << 22) - | (u64::from(in_base) << 29) -} - -fn positive_finite(bits: u32) -> bool { - let value = f32::from_bits(bits); - value.is_finite() && value > 0.0 -} - -fn split_rs(value: u64) -> (u32, u32) { - (value as u32, (value >> 32) as u32) -} - -pub(crate) fn validate( - kind: u32, - op1_bank: u32, - op2_bank: u32, - wr_bank: u32, - op1_col: u32, - op2_col: u32, - wr_col: u32, - iter: u32, - rs1: u64, - rs2: u64, -) { - if kind != KIND_F32 && kind != KIND_I32 { - panic!("toint8: unknown kind {kind}"); - } - if iter == 0 || iter % 4 != 0 { - panic!("ToInt8Ball iter must be a positive multiple of four"); - } - if iter > BANK_ENTRIES { - panic!("ToInt8Ball input exceeds bank depth"); - } - if op1_bank >= BANK_NUM { - panic!("ToInt8Ball input bank is invalid"); - } - if wr_bank >= BANK_NUM { - panic!("ToInt8Ball output bank is invalid"); - } - if op1_col != 1 || wr_col != 1 { - panic!("ToInt8Ball input and output must each occupy one bank"); - } - if op1_bank == wr_bank { - panic!("ToInt8Ball input and output banks must differ"); - } - if (rs1 & 0x3ff) != u64::from(op1_bank) || ((rs1 >> 20) & 0x3ff) != u64::from(wr_bank) { - panic!("toint8: rs1 bank fields mismatch"); - } - if ((rs1 >> 30) & 0x3_ffff_ffff) != u64::from(iter) { - panic!("toint8: rs1 iter mismatch"); - } - if kind == KIND_F32 { - if ((rs1 >> 10) & 0x3ff) != 0 || op2_bank != 0 { - panic!("QUANT_F32_TO_I8 reserves input bank 1"); - } - if rs2 >> 32 != 0 { - panic!("QUANT_F32_TO_I8 reserves rs2[63:32]"); - } - if !positive_finite(rs2 as u32) { - panic!("QUANT_F32_TO_I8 scale must be finite and positive"); - } - } else { - if op2_bank >= BANK_NUM { - panic!("QUANT_I32_TO_I8 scale bank is invalid"); - } - if ((rs1 >> 10) & 0x3ff) != u64::from(op2_bank) { - panic!("toint8: rs1 scale bank mismatch"); - } - if op2_col != 1 { - panic!("QUANT_I32_TO_I8 scale must occupy one bank"); - } - if op1_bank == op2_bank { - panic!("QUANT_I32_TO_I8 input and scale banks must differ"); - } - if op2_bank == wr_bank { - panic!("QUANT_I32_TO_I8 scale and output banks must differ"); - } - if rs2 >> 35 != 0 { - panic!("QUANT_I32_TO_I8 reserves rs2[63:35]"); - } - let out_base = ((rs2 >> 1) & 0x7f) as u32; - let width = ((rs2 >> 8) & 0x7f) as u32; - let height = ((rs2 >> 15) & 0x7f) as u32; - let stride = ((rs2 >> 22) & 0x7f) as u32; - let in_base = ((rs2 >> 29) & 0x3f) as u32; - let output_rows = iter / 4; - if width == 0 || height == 0 || stride < width { - panic!("QUANT_I32_TO_I8 output geometry is invalid"); - } - if width.checked_mul(height) != Some(output_rows) { - panic!("QUANT_I32_TO_I8 iter does not match output geometry"); - } - let output_end = out_base - .checked_add((height - 1).checked_mul(stride).expect("stride overflow")) - .and_then(|value| value.checked_add(width)); - match output_end { - Some(end) if end <= BANK_ENTRIES => {} - _ => panic!("QUANT_I32_TO_I8 output exceeds bank depth"), - } - if in_base.checked_add(iter).is_none_or(|end| end > BANK_ENTRIES) { - panic!("QUANT_I32_TO_I8 input exceeds bank depth"); - } - } -} - -fn f32_dst(src: &[u128], scale: f32) -> Vec { - if src.len() % 4 != 0 { - panic!("QUANT_F32_TO_I8 src rows must be a multiple of four"); - } - src.chunks(4) - .map(|group| { - let mut packed = 0u128; - for (g, word) in group.iter().enumerate() { - for lane in 0..LANES { - let bits = (*word >> (lane * 32)) as u32; - if (bits >> 23) & 0xff == 0xff { - panic!("QUANT_F32_TO_I8 input must be finite"); - } - let q = quantize_f32(f32::from_bits(bits), scale, false) as u8; - packed |= u128::from(q) << ((g * LANES + lane) * 8); - } - } - packed - }) - .collect() -} - -fn i32_dst(src: &[u128], scales: &[u128], relu: bool) -> Vec { - if src.len() % 4 != 0 { - panic!("QUANT_I32_TO_I8 src rows must be a multiple of four"); - } - if scales.len() != SCALE_ROWS { - panic!("QUANT_I32_TO_I8 requires four scale rows"); - } - let mut scale_vals = [0.0f32; 16]; - for (row, word) in scales.iter().enumerate() { - for lane in 0..LANES { - let bits = (*word >> (lane * 32)) as u32; - if !positive_finite(bits) { - panic!("QUANT_I32_TO_I8 scales must be finite and positive"); - } - scale_vals[row * LANES + lane] = f32::from_bits(bits); - } - } - src.chunks(4) - .map(|group| { - let mut packed = 0u128; - for (g, word) in group.iter().enumerate() { - for lane in 0..LANES { - let bits = (*word >> (lane * 32)) as u32; - let channel = g * LANES + lane; - let q = quantize_i32(bits as i32, scale_vals[channel], relu) as u8; - packed |= u128::from(q) << (channel * 8); - } - } - packed - }) - .collect() -} - -fn i32_addrs(iter: u32, out_base: u32, width: u32, stride: u32) -> Vec { - let output_rows = iter / 4; - (0..output_rows) - .map(|row| out_base + (row / width) * stride + row % width) - .collect() -} - -fn finish( - kind: u32, - bid: u32, - iter: u32, - op1_bank: u32, - op2_bank: u32, - wr_bank: u32, - op2_col: u32, - rob_id: u32, - rs2: u64, - input_base: u32, - src: Vec, - scale: Vec, - dst: Vec, - dst_addr: Vec, -) -> ToInt8Case { - let op1_col = 1; - let wr_col = 1; - let rs1 = encode_rs1(op1_bank, op2_bank, wr_bank, iter); - validate( - kind, op1_bank, op2_bank, wr_bank, op1_col, op2_col, wr_col, iter, rs1, rs2, - ); - if src.len() != iter as usize { - panic!("toint8: src rows {} != iter {iter}", src.len()); - } - if dst.len() != (iter / 4) as usize || dst_addr.len() != dst.len() { - panic!("toint8: dst rows do not match iter/4"); - } - if kind == KIND_F32 { - if !scale.is_empty() { - panic!("QUANT_F32_TO_I8 must not emit scale rows"); - } - } else if scale.len() != SCALE_ROWS { - panic!("QUANT_I32_TO_I8 must emit four scale rows"); - } - let (rs1_lo, rs1_hi) = split_rs(rs1); - let (rs2_lo, rs2_hi) = split_rs(rs2); - ToInt8Case { - cmd: ToInt8Cmd { - kind, - bid, - iter, - op1_bank, - op2_bank, - wr_bank, - op1_col, - op2_col, - wr_col, - rob_id, - rs1_lo, - rs1_hi, - rs2_lo, - rs2_hi, - input_base, - num_src_words: src.len() as u32, - num_scale_words: scale.len() as u32, - num_dst_words: dst.len() as u32, - }, - src_words: copy_words(&src), - scale_words: copy_words(&scale), - dst_words: copy_words(&dst), - dst_addr: copy_addrs(&dst_addr), - } -} - -fn f32_case(bid: u32, rob_id: u32, scale: f32, values: &[f32]) -> ToInt8Case { - let bits: Vec = values.iter().map(|v| v.to_bits()).collect(); - let src = pack_u32s(&bits); - let iter = src.len() as u32; - let dst = f32_dst(&src, scale); - let dst_addr: Vec = (0..dst.len() as u32).collect(); - finish( - KIND_F32, - bid, - iter, - 0, - 0, - 1, - 0, - rob_id, - u64::from(scale.to_bits()), - 0, - src, - Vec::new(), - dst, - dst_addr, - ) -} - -fn i32_case( - bid: u32, - rob_id: u32, - op1_bank: u32, - op2_bank: u32, - wr_bank: u32, - relu: bool, - out_base: u32, - width: u32, - height: u32, - stride: u32, - in_base: u32, - values: &[i32], - scales: &[f32], -) -> ToInt8Case { - let src = pack_u32s(&values.iter().map(|v| *v as u32).collect::>()); - let scale = pack_u32s(&scales.iter().map(|v| v.to_bits()).collect::>()); - let iter = src.len() as u32; - let rs2 = encode_i32_rs2(relu, out_base, width, height, stride, in_base); - let dst = i32_dst(&src, &scale, relu); - let dst_addr = i32_addrs(iter, out_base, width, stride); - finish( - KIND_I32, - bid, - iter, - op1_bank, - op2_bank, - wr_bank, - 1, - rob_id, - rs2, - in_base, - src, - scale, - dst, - dst_addr, - ) -} - -fn f32_signed(bid: u32) -> ToInt8Case { - f32_case( - bid, - 2, - 0.5, - &[ - -300.0, -255.0, -5.0, -3.0, -1.0, 0.0, 1.0, 3.0, 5.0, 254.0, 255.0, 300.0, 2.0, 6.0, - 10.0, 14.0, - ], - ) -} - -fn f32_zero(bid: u32) -> ToInt8Case { - f32_case(bid, 3, 1.0, &[0.0; 16]) -} - -fn f32_rounding(bid: u32) -> ToInt8Case { - f32_case( - bid, - 4, - 1.0, - &[ - 2.5, -2.5, 3.5, -3.5, 0.5, -0.5, 1.5, -1.5, 63.5, -63.5, 127.5, -128.5, 0.0, 1.0, 2.0, - 3.0, - ], - ) -} - -fn f32_rows(bid: u32) -> ToInt8Case { - f32_case( - bid, - 5, - 2.0, - &[ - 0.125, -0.125, 0.25, -0.25, 0.75, -0.75, 1.25, -1.25, 1.75, -1.75, 63.25, 63.75, - -63.75, -64.75, 0.0, -0.0, 2.25, -2.25, 2.75, -2.75, 3.25, -3.25, 3.75, -3.75, 10.125, - -10.125, 20.25, -20.25, 0.375, -0.375, 64.25, -65.25, - ], - ) -} - -fn i32_scale_rows(bid: u32) -> ToInt8Case { - let mut values = [0i32; 16]; - let mut scales = [0.0f32; 16]; - for i in 0..16 { - values[i] = i as i32 - 8; - scales[i] = 0.5 + (i as f32) * 0.25; - } - i32_case(bid, 6, 0, 1, 2, false, 0, 1, 1, 1, 0, &values, &scales) -} - -fn i32_relu_tile(bid: u32) -> ToInt8Case { - let mut values = [0i32; 64]; - for position in 0..4 { - for channel in 0..16 { - values[position * 16 + channel] = position as i32 * 20 + channel as i32 - 8; - } - } - i32_case(bid, 7, 0, 1, 2, true, 3, 2, 2, 4, 0, &values, &[1.0; 16]) -} - -fn i32_signed(bid: u32) -> ToInt8Case { - let values: [i32; 16] = [ - -300, -128, -5, -1, 0, 1, 5, 127, 128, 200, -64, 64, -7, 7, 3, -3, - ]; - i32_case(bid, 8, 0, 1, 2, false, 0, 1, 1, 1, 0, &values, &[1.0; 16]) -} - -fn i32_offset(bid: u32) -> ToInt8Case { - let mut values = [0i32; 32]; - for i in 0..32 { - values[i] = i as i32 - 16; - } - i32_case(bid, 9, 0, 1, 2, true, 1, 2, 1, 2, 4, &values, &[0.5; 16]) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn cases_cover_both_quant_kinds() { - let f32 = gen_case(0, 3); - let i32 = gen_case(5, 3); - assert_eq!(f32.cmd.kind, KIND_F32); - assert_eq!(i32.cmd.kind, KIND_I32); - assert_eq!(f32.cmd.iter % 4, 0); - assert_eq!(f32.cmd.op1_col, 1); - assert_eq!(f32.cmd.wr_col, 1); - assert_eq!(i32.cmd.op2_col, 1); - assert_eq!(i32.cmd.rs2_lo & 1, 1); - assert_eq!(i32.cmd.num_scale_words, 4); - assert_eq!(f32.cmd.num_scale_words, 0); - assert_eq!(f32.cmd.bid, 3); - } - - #[test] - fn former_indices_are_legal() { - for index in 0..8 { - let case = gen_case(index, 3); - assert!(case.cmd.kind == KIND_F32 || case.cmd.kind == KIND_I32); - assert_eq!(case.cmd.num_src_words, case.cmd.iter); - assert_eq!(case.cmd.num_dst_words, case.cmd.iter / 4); - } - } - - #[test] - fn f32_ctest_vector_saturates() { - let case = gen_case(0, 3); - let packed = case.dst_words[0]; - let bytes: [i8; 16] = [ - -128, -128, -2, -2, 0, 0, 0, 2, 2, 127, 127, 127, 1, 3, 5, 7, - ]; - for (i, expected) in bytes.iter().enumerate() { - assert_eq!(((packed >> (i * 8)) as u8) as i8, *expected); - } - } - - #[test] - #[should_panic(expected = "unsupported directed case")] - fn unknown_index_panics() { - let _ = gen_case(8, 3); - } - - #[test] - #[should_panic(expected = "positive multiple of four")] - fn illegal_iter_panics() { - validate(KIND_F32, 0, 0, 1, 1, 0, 1, 3, encode_rs1(0, 0, 1, 3), u64::from(1.0f32.to_bits())); - } - - #[test] - #[should_panic(expected = "reserves rs2[63:32]")] - fn f32_high_rs2_panics() { - validate( - KIND_F32, - 0, - 0, - 1, - 1, - 0, - 1, - 4, - encode_rs1(0, 0, 1, 4), - u64::from(1.0f32.to_bits()) | (1u64 << 32), - ); - } - - #[test] - #[should_panic(expected = "output geometry is invalid")] - fn i32_zero_width_panics() { - validate( - KIND_I32, - 0, - 1, - 2, - 1, - 1, - 1, - 4, - encode_rs1(0, 1, 2, 4), - encode_i32_rs2(false, 0, 0, 1, 1, 0), - ); - } -} diff --git a/examples/balls/toint8/verify/casegen/src/lib.rs b/examples/balls/toint8/verify/casegen/src/lib.rs deleted file mode 100644 index ead22d824..000000000 --- a/examples/balls/toint8/verify/casegen/src/lib.rs +++ /dev/null @@ -1,165 +0,0 @@ -mod casegen; -#[path = "../../../emu/src/model.rs"] -mod model; - -use std::cell::RefCell; - -use casegen::{ToInt8Case, ToInt8Cmd, MAX_DST, MAX_SCALE, MAX_SRC}; - -thread_local! { - static CURRENT: RefCell> = const { RefCell::new(None) }; -} - -#[no_mangle] -pub extern "C" fn toint8_case_load(index: u32, bid: u32) -> i32 { - let case = casegen::gen_case(index, bid); - CURRENT.with(|current| *current.borrow_mut() = Some(case)); - 0 -} - -fn current_case R, R>(f: F) -> R { - CURRENT.with(|current| match current.borrow().as_ref() { - Some(case) => f(case), - None => panic!("toint8_case: no case loaded; call toint8_case_load first"), - }) -} - -#[no_mangle] -pub extern "C" fn toint8_case_cmd(out_ptr: *mut ToInt8Cmd) { - if out_ptr.is_null() { - panic!("toint8_case_cmd: null out_ptr"); - } - current_case(|case| unsafe { - *out_ptr = case.cmd; - }); -} - -fn src_index(index: u32, limit: u32, what: &str) -> usize { - if index >= limit { - panic!("toint8_case_{what}: index {index} out of range {limit}"); - } - index as usize -} - -#[no_mangle] -pub extern "C" fn toint8_case_src_word_lo(word_index: u32) -> u64 { - current_case(|case| { - let i = src_index(word_index, case.cmd.num_src_words, "src_word_lo"); - if i >= MAX_SRC { - panic!("toint8_case_src_word_lo: word_index out of range"); - } - case.src_lo(i) - }) -} - -#[no_mangle] -pub extern "C" fn toint8_case_src_word_hi(word_index: u32) -> u64 { - current_case(|case| { - let i = src_index(word_index, case.cmd.num_src_words, "src_word_hi"); - if i >= MAX_SRC { - panic!("toint8_case_src_word_hi: word_index out of range"); - } - case.src_hi(i) - }) -} - -#[no_mangle] -pub extern "C" fn toint8_case_scale_word_lo(word_index: u32) -> u64 { - current_case(|case| { - let i = src_index(word_index, case.cmd.num_scale_words, "scale_word_lo"); - if i >= MAX_SCALE { - panic!("toint8_case_scale_word_lo: word_index out of range"); - } - case.scale_lo(i) - }) -} - -#[no_mangle] -pub extern "C" fn toint8_case_scale_word_hi(word_index: u32) -> u64 { - current_case(|case| { - let i = src_index(word_index, case.cmd.num_scale_words, "scale_word_hi"); - if i >= MAX_SCALE { - panic!("toint8_case_scale_word_hi: word_index out of range"); - } - case.scale_hi(i) - }) -} - -#[no_mangle] -pub extern "C" fn toint8_case_dst_word_lo(word_index: u32) -> u64 { - current_case(|case| { - let i = src_index(word_index, case.cmd.num_dst_words, "dst_word_lo"); - if i >= MAX_DST { - panic!("toint8_case_dst_word_lo: word_index out of range"); - } - case.dst_lo(i) - }) -} - -#[no_mangle] -pub extern "C" fn toint8_case_dst_word_hi(word_index: u32) -> u64 { - current_case(|case| { - let i = src_index(word_index, case.cmd.num_dst_words, "dst_word_hi"); - if i >= MAX_DST { - panic!("toint8_case_dst_word_hi: word_index out of range"); - } - case.dst_hi(i) - }) -} - -#[no_mangle] -pub extern "C" fn toint8_case_dst_addr(word_index: u32) -> u32 { - current_case(|case| { - let i = src_index(word_index, case.cmd.num_dst_words, "dst_addr"); - if i >= MAX_DST { - panic!("toint8_case_dst_addr: word_index out of range"); - } - case.dst_addr[i] - }) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn dpi_load_then_cmd_and_words() { - assert_eq!(toint8_case_load(0, 3), 0); - let mut cmd = ToInt8Cmd { - kind: 99, - bid: 0, - iter: 0, - op1_bank: 0, - op2_bank: 0, - wr_bank: 0, - op1_col: 0, - op2_col: 0, - wr_col: 0, - rob_id: 0, - rs1_lo: 0, - rs1_hi: 0, - rs2_lo: 0, - rs2_hi: 0, - input_base: 0, - num_src_words: 0, - num_scale_words: 0, - num_dst_words: 0, - }; - toint8_case_cmd(&mut cmd as *mut ToInt8Cmd); - assert_eq!(cmd.bid, 3); - assert_eq!(cmd.kind, casegen::KIND_F32); - assert_eq!(cmd.iter, 4); - assert_eq!(cmd.op1_col, 1); - assert_eq!(cmd.num_src_words, 4); - let _lo = toint8_case_src_word_lo(0); - let _hi = toint8_case_src_word_hi(0); - let _addr = toint8_case_dst_addr(0); - } - - #[test] - #[should_panic(expected = "no case loaded")] - fn current_case_panics_without_load() { - CURRENT.with(|c| *c.borrow_mut() = None); - current_case(|_| ()); - } -} diff --git a/examples/balls/toint8/verify/src/common/toint8_defs.svh b/examples/balls/toint8/verify/src/common/toint8_defs.svh deleted file mode 100644 index fc68e0d95..000000000 --- a/examples/balls/toint8/verify/src/common/toint8_defs.svh +++ /dev/null @@ -1,58 +0,0 @@ -typedef struct { - int unsigned kind; - int unsigned bid; - int unsigned iter; - int unsigned op1_bank; - int unsigned op2_bank; - int unsigned wr_bank; - int unsigned op1_col; - int unsigned op2_col; - int unsigned wr_col; - int unsigned rob_id; - int unsigned rs1_lo; - int unsigned rs1_hi; - int unsigned rs2_lo; - int unsigned rs2_hi; - int unsigned input_base; - int unsigned num_src_words; - int unsigned num_scale_words; - int unsigned num_dst_words; -} toint8_cmd_dpi_t; - -import "DPI-C" function int toint8_case_load( - input int unsigned index, - input int unsigned bid -); -import "DPI-C" function void toint8_case_cmd(output toint8_cmd_dpi_t cmd); -import "DPI-C" function longint unsigned toint8_case_src_word_lo(input int unsigned word_index); -import "DPI-C" function longint unsigned toint8_case_src_word_hi(input int unsigned word_index); -import "DPI-C" function longint unsigned toint8_case_scale_word_lo(input int unsigned word_index); -import "DPI-C" function longint unsigned toint8_case_scale_word_hi(input int unsigned word_index); -import "DPI-C" function longint unsigned toint8_case_dst_word_lo(input int unsigned word_index); -import "DPI-C" function longint unsigned toint8_case_dst_word_hi(input int unsigned word_index); -import "DPI-C" function int unsigned toint8_case_dst_addr(input int unsigned word_index); - -`ifndef QUANT_F32_TO_I8_FUNCT7 -`error "QUANT_F32_TO_I8_FUNCT7 must be provided by the selected Core ballISA" -`endif -`ifndef QUANT_I32_TO_I8_FUNCT7 -`error "QUANT_I32_TO_I8_FUNCT7 must be provided by the selected Core ballISA" -`endif - -localparam int QUANT_F32_TO_I8_CORE_FUNCT7 = `QUANT_F32_TO_I8_FUNCT7; -localparam int QUANT_I32_TO_I8_CORE_FUNCT7 = `QUANT_I32_TO_I8_FUNCT7; -localparam int TOINT8_KIND_F32 = 0; -localparam int TOINT8_KIND_I32 = 1; -localparam int TOINT8_MAX_SRC_WORDS = 16; -localparam int TOINT8_MAX_SCALE_WORDS = 4; -localparam int TOINT8_MAX_DST_WORDS = 4; -localparam int TOINT8_NUM_CASES = 8; -localparam int TOINT8_TIMEOUT_CYCLES = 4000; - -function automatic int unsigned toint8_require_bid(); - int unsigned bid; - if (!$value$plusargs("BID=%d", bid)) begin - `uvm_fatal("BID", "missing required plusarg +BID=") - end - return bid; -endfunction diff --git a/examples/balls/toint8/verify/src/common/toint8_items.svh b/examples/balls/toint8/verify/src/common/toint8_items.svh deleted file mode 100644 index d345b8e2f..000000000 --- a/examples/balls/toint8/verify/src/common/toint8_items.svh +++ /dev/null @@ -1,145 +0,0 @@ -class toint8_cmd_item extends bb_blink_cmd_item; - `uvm_object_utils(toint8_cmd_item) - - bit [31:0] kind; - bit [31:0] input_base; - bit [31:0] num_src_words; - bit [31:0] num_scale_words; - bit [31:0] num_dst_words; - bit [127:0] src_words[TOINT8_MAX_SRC_WORDS]; - bit [127:0] scale_words[TOINT8_MAX_SCALE_WORDS]; - bit [127:0] dst_words[TOINT8_MAX_DST_WORDS]; - bit [31:0] dst_addr[TOINT8_MAX_DST_WORDS]; - - constraint legal_c { - funct7 inside {QUANT_F32_TO_I8_CORE_FUNCT7[6:0], QUANT_I32_TO_I8_CORE_FUNCT7[6:0]}; - op1_en == 1'b1; - wr_spad_en == 1'b1; - op1_from_spad == 1'b1; - op1_col == 5'd1; - wr_col == 5'd1; - meta_bank == 5'd0; - is_sub == 1'b0; - sub_rob_id == 8'h00; - special == 64'd0; - } - - function new(string name = "toint8_cmd_item"); - super.new(name); - endfunction - - function void load_rust_case(int unsigned index, int unsigned bid); - toint8_cmd_dpi_t cmd; - int unsigned rc; - longint unsigned w_lo; - longint unsigned w_hi; - int i; - - rc = toint8_case_load(index, bid); - if (rc != 0) begin - `uvm_fatal("CASE", $sformatf("toint8_case_load returned %0d for index %0d", rc, index)) - end - toint8_case_cmd(cmd); - - if (cmd.kind != TOINT8_KIND_F32 && cmd.kind != TOINT8_KIND_I32) begin - `uvm_fatal("CASE", $sformatf("unknown toint8 kind %0d", cmd.kind)) - end - - kind = cmd.kind; - this.bid = cmd.bid[4:0]; - iter = cmd.iter; - op1_bank = cmd.op1_bank[4:0]; - op2_bank = cmd.op2_bank[4:0]; - wr_bank = cmd.wr_bank[4:0]; - op1_col = cmd.op1_col[4:0]; - op2_col = cmd.op2_col[4:0]; - wr_col = cmd.wr_col[4:0]; - rob_id = cmd.rob_id[3:0]; - rs1 = {cmd.rs1_hi, cmd.rs1_lo}; - rs2 = {cmd.rs2_hi, cmd.rs2_lo}; - input_base = cmd.input_base; - num_src_words = cmd.num_src_words; - num_scale_words = cmd.num_scale_words; - num_dst_words = cmd.num_dst_words; - special = 64'd0; - op1_en = 1'b1; - wr_spad_en = 1'b1; - op1_from_spad = 1'b1; - meta_bank = 5'd0; - is_sub = 1'b0; - sub_rob_id = 8'h00; - - if (kind == TOINT8_KIND_F32) begin - funct7 = QUANT_F32_TO_I8_CORE_FUNCT7[6:0]; - op2_en = 1'b0; - op2_from_spad = 1'b0; - if (op2_bank != 0 || op2_col != 0 || rs1[19:10] != 0) - `uvm_fatal("CASE", "QUANT_F32_TO_I8 reserves input bank 1") - if (rs2[63:32] != 0) `uvm_fatal("CASE", "QUANT_F32_TO_I8 reserves rs2[63:32]") - if (num_scale_words != 0) `uvm_fatal("CASE", "QUANT_F32_TO_I8 must not emit scale rows") - if (input_base != 0) `uvm_fatal("CASE", "QUANT_F32_TO_I8 input_base must be 0") - end else begin - funct7 = QUANT_I32_TO_I8_CORE_FUNCT7[6:0]; - op2_en = 1'b1; - op2_from_spad = 1'b1; - if (op2_col != 1) `uvm_fatal("CASE", "QUANT_I32_TO_I8 scale must occupy one bank") - if (op1_bank == op2_bank) - `uvm_fatal("CASE", "QUANT_I32_TO_I8 input and scale banks must differ") - if (op2_bank == wr_bank) - `uvm_fatal("CASE", "QUANT_I32_TO_I8 scale and output banks must differ") - if (rs2[63:35] != 0) `uvm_fatal("CASE", "QUANT_I32_TO_I8 reserves rs2[63:35]") - if (num_scale_words != 4) `uvm_fatal("CASE", "QUANT_I32_TO_I8 must emit four scale rows") - end - - if (iter == 0 || iter[1:0] != 2'b00) - `uvm_fatal("CASE", "ToInt8Ball iter must be a positive multiple of four") - if (op1_col != 1 || wr_col != 1) - `uvm_fatal("CASE", "ToInt8Ball input and output must each occupy one bank") - if (op1_bank == wr_bank) `uvm_fatal("CASE", "ToInt8Ball input and output banks must differ") - if (num_src_words != iter) - `uvm_fatal("CASE", $sformatf("num_src_words %0d != iter %0d", num_src_words, iter)) - if (num_dst_words != (iter >> 2)) `uvm_fatal("CASE", "num_dst_words does not match iter/4") - if (num_src_words == 0 || num_src_words > TOINT8_MAX_SRC_WORDS) - `uvm_fatal("CASE", $sformatf("num_src_words out of range: %0d", num_src_words)) - if (num_dst_words == 0 || num_dst_words > TOINT8_MAX_DST_WORDS) - `uvm_fatal("CASE", $sformatf("num_dst_words out of range: %0d", num_dst_words)) - if (funct7 != QUANT_F32_TO_I8_CORE_FUNCT7[6:0] && funct7 != QUANT_I32_TO_I8_CORE_FUNCT7[6:0]) - `uvm_fatal("CASE", $sformatf("invalid funct7: %0d", funct7)) - - for (i = 0; i < num_src_words; i++) begin - w_lo = toint8_case_src_word_lo(i); - w_hi = toint8_case_src_word_hi(i); - src_words[i] = {w_hi, w_lo}; - end - for (i = 0; i < num_scale_words; i++) begin - w_lo = toint8_case_scale_word_lo(i); - w_hi = toint8_case_scale_word_hi(i); - scale_words[i] = {w_hi, w_lo}; - end - for (i = 0; i < num_dst_words; i++) begin - w_lo = toint8_case_dst_word_lo(i); - w_hi = toint8_case_dst_word_hi(i); - dst_words[i] = {w_hi, w_lo}; - dst_addr[i] = toint8_case_dst_addr(i); - end - endfunction - - function void do_copy(uvm_object rhs); - toint8_cmd_item rhs_; - super.do_copy(rhs); - if (!$cast(rhs_, rhs)) begin - `uvm_fatal("COPY", "rhs is not toint8_cmd_item") - end - kind = rhs_.kind; - input_base = rhs_.input_base; - num_src_words = rhs_.num_src_words; - num_scale_words = rhs_.num_scale_words; - num_dst_words = rhs_.num_dst_words; - for (int i = 0; i < TOINT8_MAX_SRC_WORDS; i++) src_words[i] = rhs_.src_words[i]; - for (int i = 0; i < TOINT8_MAX_SCALE_WORDS; i++) scale_words[i] = rhs_.scale_words[i]; - for (int i = 0; i < TOINT8_MAX_DST_WORDS; i++) begin - dst_words[i] = rhs_.dst_words[i]; - dst_addr[i] = rhs_.dst_addr[i]; - end - endfunction -endclass diff --git a/examples/balls/toint8/verify/src/cov/toint8_cov.svh b/examples/balls/toint8/verify/src/cov/toint8_cov.svh deleted file mode 100644 index f18079d10..000000000 --- a/examples/balls/toint8/verify/src/cov/toint8_cov.svh +++ /dev/null @@ -1,34 +0,0 @@ -class toint8_cov extends uvm_component; - `uvm_component_utils(toint8_cov) - - uvm_analysis_imp_cmd #(bb_blink_cmd_item, toint8_cov) cmd_imp; - - bit [6:0] cur_funct7; - - covergroup cmd_cg; - coverpoint cur_funct7 { - bins f32 = {QUANT_F32_TO_I8_CORE_FUNCT7[6:0]}; bins i32 = {QUANT_I32_TO_I8_CORE_FUNCT7[6:0]}; - } - endgroup - - int cmd_count; - - function new(string name, uvm_component parent); - super.new(name, parent); - cmd_imp = new("cmd_imp", this); - cmd_cg = new(); - endfunction - - function void write_cmd(bb_blink_cmd_item item); - cur_funct7 = item.funct7; - cmd_count++; - cmd_cg.sample(); - endfunction - - function void check_phase(uvm_phase phase); - super.check_phase(phase); - if (cmd_count == 0) `uvm_fatal("COV", "no cmd observed") - if (cmd_cg.get_coverage() != 100.0) - `uvm_fatal("COV", $sformatf("toint8_cov %0.2f%%", cmd_cg.get_coverage())) - endfunction -endclass diff --git a/examples/balls/toint8/verify/src/env/toint8_scoreboard.svh b/examples/balls/toint8/verify/src/env/toint8_scoreboard.svh deleted file mode 100644 index cc0de35bd..000000000 --- a/examples/balls/toint8/verify/src/env/toint8_scoreboard.svh +++ /dev/null @@ -1,189 +0,0 @@ -class toint8_scoreboard extends uvm_scoreboard; - `uvm_component_utils(toint8_scoreboard) - - uvm_analysis_imp_stim #(bb_blink_cmd_item, toint8_scoreboard) stim_imp; - uvm_analysis_imp_cmd #(bb_blink_cmd_item, toint8_scoreboard) cmd_imp; - uvm_analysis_imp_read #(bb_blink_read_item, toint8_scoreboard) read_imp; - uvm_analysis_imp_write #(bb_blink_write_item, toint8_scoreboard) write_imp; - uvm_analysis_imp_resp #(bb_blink_resp_item, toint8_scoreboard) resp_imp; - - bb_blink_mem_model #(`BB_IN_BW, `BB_OUT_BW) mem_model; - - toint8_cmd_item stim_q[$]; - int unsigned expected_reads; - int unsigned expected_writes; - int unsigned cmd_count; - int unsigned read_count; - int unsigned write_count; - int unsigned resp_count; - - function new(string name, uvm_component parent); - super.new(name, parent); - stim_imp = new("stim_imp", this); - cmd_imp = new("cmd_imp", this); - read_imp = new("read_imp", this); - write_imp = new("write_imp", this); - resp_imp = new("resp_imp", this); - endfunction - - function void write_stim(bb_blink_cmd_item item); - toint8_cmd_item got; - toint8_cmd_item clone; - int i; - - if (stim_q.size() != 0) begin - `uvm_fatal("SCB", "single outstanding command supported") - end - if (!$cast(got, item)) begin - `uvm_fatal("SCB", "stim item is not toint8_cmd_item") - end - if (!$cast(clone, got.clone())) begin - `uvm_fatal("SCB", "failed to clone stimulus item") - end - stim_q.push_back(clone); - - if (mem_model == null) begin - `uvm_fatal("SCB", "mem_model handle not set") - end - mem_model.clear_mem(); - for (i = 0; i < clone.num_src_words; i++) begin - mem_model.load_word(int'(clone.op1_bank), int'(clone.input_base) + i, clone.src_words[i]); - end - for (i = 0; i < clone.num_scale_words; i++) begin - mem_model.load_word(int'(clone.op2_bank), i, clone.scale_words[i]); - end - mem_model.arm(); - - expected_writes = clone.num_dst_words; - expected_reads = clone.num_scale_words + clone.num_src_words; - endfunction - - function void write_cmd(bb_blink_cmd_item item); - toint8_cmd_item stim; - stim = current_stim("CMD"); - check_cmd(item, stim); - cmd_count++; - endfunction - - function void check_cmd(bb_blink_cmd_item got, toint8_cmd_item exp); - if (got.bid !== exp.bid) - `uvm_fatal("CMD", $sformatf("bid mismatch: got %0d exp %0d", got.bid, exp.bid)) - if (got.funct7 !== exp.funct7) - `uvm_fatal("CMD", $sformatf("funct7 mismatch: got %0d exp %0d", got.funct7, exp.funct7)) - if (got.iter !== exp.iter) - `uvm_fatal("CMD", $sformatf("iter mismatch: got %0d exp %0d", got.iter, exp.iter)) - if (got.op1_bank !== exp.op1_bank || got.op2_bank !== exp.op2_bank || - got.wr_bank !== exp.wr_bank) - `uvm_fatal("CMD", "bank field mismatch") - if (got.op1_col !== exp.op1_col || got.op2_col !== exp.op2_col || got.wr_col !== exp.wr_col) - `uvm_fatal("CMD", "column field mismatch") - if (got.rs1 !== exp.rs1 || got.rs2 !== exp.rs2) `uvm_fatal("CMD", "rs field mismatch") - if (got.rob_id !== exp.rob_id) - `uvm_fatal("CMD", $sformatf("rob_id mismatch: got %0d exp %0d", got.rob_id, exp.rob_id)) - endfunction - - function void write_read(bb_blink_read_item item); - toint8_cmd_item stim; - int unsigned scale_reads; - int unsigned expect_bank; - int unsigned expect_addr; - - stim = current_stim("READ"); - if (item.port != 0) - `uvm_fatal("READ", $sformatf("ToInt8Ball has one read port, got port %0d", item.port)) - if (item.group_id !== 5'd0) - `uvm_fatal("READ", $sformatf("group mismatch: got %0d exp 0", item.group_id)) - if (item.rob_id !== stim.rob_id) - `uvm_fatal("READ", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - - scale_reads = stim.num_scale_words; - if (read_count < scale_reads) begin - expect_bank = stim.op2_bank; - expect_addr = read_count; - end else begin - expect_bank = stim.op1_bank; - expect_addr = stim.input_base + (read_count - scale_reads); - end - if (item.bank_id !== expect_bank[4:0]) - `uvm_fatal("READ", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, expect_bank)) - if (item.addr !== expect_addr[BB_BLINK_BANK_ADDR_W-1:0]) - `uvm_fatal("READ", $sformatf("addr mismatch: got %0d exp %0d", item.addr, expect_addr)) - read_count++; - endfunction - - function void write_write(bb_blink_write_item item); - toint8_cmd_item stim; - - stim = current_stim("WRITE"); - if (item.port != 0) - `uvm_fatal("WRITE", $sformatf("ToInt8Ball has one write port, got port %0d", item.port)) - if (item.bank_id !== stim.wr_bank) - `uvm_fatal("WRITE", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.wr_bank)) - if (item.rob_id !== stim.rob_id) - `uvm_fatal("WRITE", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.group_id !== 5'd0) - `uvm_fatal("WRITE", $sformatf("group mismatch: got %0d exp 0", item.group_id)) - if (item.mask !== 16'hFFFF) - `uvm_fatal("WRITE", $sformatf("mask mismatch: got 0x%04h", item.mask)) - if (write_count >= stim.num_dst_words) - `uvm_fatal("WRITE", $sformatf("extra write %0d", write_count)) - if (item.addr !== stim.dst_addr[write_count][BB_BLINK_BANK_ADDR_W-1:0]) - `uvm_fatal("WRITE", $sformatf( - "addr mismatch: got %0d exp %0d", item.addr, stim.dst_addr[write_count])) - if (item.data !== stim.dst_words[write_count]) - `uvm_fatal("SCB", $sformatf( - "data mismatch at addr %0d: got 0x%032h exp 0x%032h", - item.addr, - item.data, - stim.dst_words[write_count] - )) - write_count++; - endfunction - - function void write_resp(bb_blink_resp_item item); - toint8_cmd_item stim; - - stim = current_stim("RESP"); - if (item.rob_id !== stim.rob_id) - `uvm_fatal("RESP", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.is_sub !== 1'b0) `uvm_fatal("RESP", "is_sub should be 0") - if (item.sub_rob_id !== 8'h00) - `uvm_fatal("RESP", $sformatf("sub_rob_id mismatch: got 0x%0h", item.sub_rob_id)) - resp_count++; - endfunction - - function toint8_cmd_item current_stim(string tag); - if (stim_q.size() == 0) begin - `uvm_fatal("SCB", $sformatf("%s observed before stimulus", tag)) - return null; - end - return stim_q[0]; - endfunction - - function bit done(); - return cmd_count == 1 && - read_count == expected_reads && - write_count == expected_writes && - resp_count == 1; - endfunction - - function void reset_counters(); - stim_q.delete(); - cmd_count = 0; - read_count = 0; - write_count = 0; - resp_count = 0; - expected_reads = 0; - expected_writes = 0; - endfunction - - function void check_phase(uvm_phase phase); - super.check_phase(phase); - if (!done()) begin - `uvm_fatal("SCB", $sformatf("incomplete: cmds=%0d reads=%0d/%0d writes=%0d/%0d resp=%0d", - cmd_count, read_count, expected_reads, write_count, - expected_writes, resp_count)) - end - stim_q.delete(); - endfunction -endclass diff --git a/examples/balls/toint8/verify/src/tb_top.sv b/examples/balls/toint8/verify/src/tb_top.sv deleted file mode 100644 index 402dae56a..000000000 --- a/examples/balls/toint8/verify/src/tb_top.sv +++ /dev/null @@ -1,90 +0,0 @@ -module tb_top; - import uvm_pkg::*; - import toint8_pkg::*; - - bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); - - ToInt8Ball dut ( - .clock(intf.clock), - .reset(intf.reset), - .io_cmdReq_ready(intf.cmd_req_ready), - .io_cmdReq_valid(intf.cmd_req_valid), - .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), - .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), - .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), - .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), - .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), - .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), - .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), - .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), - .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), - .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank), - .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank), - .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank), - .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), - .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), - .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), - .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank), - .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), - .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), - .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), - .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), - .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), - .io_cmdResp_ready(intf.cmd_resp_ready), - .io_cmdResp_valid(intf.cmd_resp_valid), - .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), - .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), - .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), - .io_bankRead_0_bank_id(intf.bank_read_bank_id[0]), - .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), - .io_bankRead_0_group_id(intf.bank_read_group_id[0]), - .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), - .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), - .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), - .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), - .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), - .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), - .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), - .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), - .io_bankWrite_0_group_id(intf.bank_write_group_id[0]), - .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), - .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), - .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), - .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), - .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), - .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), - .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), - .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), - .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), - .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), - .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), - .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), - .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), - .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), - .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), - .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), - .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), - .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), - .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), - .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), - .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), - .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), - .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), - .io_subRobReq_ready(intf.sub_rob_req_ready), - .io_status_idle(), - .io_status_running(), - .io_channelReady(1'b1), - .io_bankRead_0_ball_id(), - .io_bankWrite_0_ball_id() - ); - - initial begin - intf.clock = 1'b0; - forever #5 intf.clock = ~intf.clock; - end - - initial begin - uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); - run_test("toint8_ball_test"); - end -endmodule diff --git a/examples/balls/toint8/workloads/ctests/CMakeLists.txt b/examples/balls/toint8/workloads/ctests/CMakeLists.txt deleted file mode 100644 index f9035c051..000000000 --- a/examples/balls/toint8/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -add_buckyball_ctests( - quant_f32_to_i8_test.c - quant_i32_to_i8_test.c - toint8_i8_rounding_test.c - toint8_i8_rows_test.c - toint8_i8_scale_rows_test.c - toint8_i8_signed_test.c -) diff --git a/examples/balls/toint8/workloads/isa/quant.h b/examples/balls/toint8/workloads/isa/quant.h deleted file mode 100644 index e2ba22677..000000000 --- a/examples/balls/toint8/workloads/isa/quant.h +++ /dev/null @@ -1,32 +0,0 @@ -#ifndef _BB_QUANT_H_ -#define _BB_QUANT_H_ - -#include -#include - -static inline void bb_quant_f32_to_i8(uint64_t input_bank, uint64_t output_bank, - uint64_t iter, float scale) { - union { - float f; - uint32_t u; - } bits = {.f = scale}; - BUCKYBALL_INSTRUCTION_R_R(BB_BANK0(input_bank) | BB_BANK2(output_bank) | - BB_ITER(iter), - bits.u, BB_FUNC7(QUANT_F32_TO_I8)); -} - -static inline void bb_quant_i32_to_i8(uint64_t input_bank, uint64_t scale_bank, - uint64_t output_bank, uint64_t iter, - uint64_t input_base, uint64_t output_base, - uint64_t output_width, - uint64_t output_height, - uint64_t output_stride, int relu) { - BUCKYBALL_INSTRUCTION_R_R(BB_BANK0(input_bank) | BB_BANK1(scale_bank) | - BB_BANK2(output_bank) | BB_ITER(iter), - (output_base << 1) | (output_width << 8) | - (output_height << 15) | (output_stride << 22) | - (input_base << 29) | (uint64_t)(relu != 0), - BB_FUNC7(QUANT_I32_TO_I8)); -} - -#endif // _BB_QUANT_H_ diff --git a/examples/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8.mlir b/examples/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8.mlir deleted file mode 100644 index 1142af638..000000000 --- a/examples/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8.mlir +++ /dev/null @@ -1,30 +0,0 @@ -func.func private @check_result(memref<1x16xi8>) -> () - -memref.global "private" constant @input : memref<4x4xf32> = dense<[ - [-3.000000e+02, -2.550000e+02, -5.000000e+00, -3.000000e+00], - [-1.000000e+00, 0.000000e+00, 1.000000e+00, 3.000000e+00], - [5.000000e+00, 2.540000e+02, 2.550000e+02, 3.000000e+02], - [2.000000e+00, 6.000000e+00, 1.000000e+01, 1.400000e+01] -]> - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %depth4 = arith.constant 4 : i64 - %depth1 = arith.constant 1 : i64 - %stride = arith.constant 1 : i64 - %scale = arith.constant 5.000000e-01 : f32 - %input = memref.get_global @input : memref<4x4xf32> - %output = memref.alloc() alignment = 64 : memref<1x16xi8> - %input_bank = buckyball.bank_alloc - %output_bank = buckyball.bank_alloc - %loaded = buckyball.bank_mvin %input %input_bank %depth4 %stride - : memref<4x4xf32> i64 i64 i64 - buckyball.quant_f32_to_i8 %loaded, %output_bank, %depth4, %scale : i64 - %stored = buckyball.bank_mvout %output %output_bank %depth1 %stride - : memref<1x16xi8> i64 i64 i64 - func.call @check_result(%output) : (memref<1x16xi8>) -> () - buckyball.bank_release %loaded : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %output : memref<1x16xi8> - return %zero_i8 : i8 -} diff --git a/examples/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8.mlir b/examples/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8.mlir deleted file mode 100644 index 7b65d1f11..000000000 --- a/examples/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8.mlir +++ /dev/null @@ -1,62 +0,0 @@ -func.func private @check_result(memref<12x16xi8>) -> () - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %one_f32 = arith.constant 1.0 : f32 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %four = arith.constant 4 : index - %sixteen = arith.constant 16 : index - %twenty = arith.constant 20 : index - %eight = arith.constant 8 : index - %depth16 = arith.constant 16 : i64 - %depth4 = arith.constant 4 : i64 - %depth12 = arith.constant 12 : i64 - %output_base = arith.constant 3 : i64 - %zero64 = arith.constant 0 : i64 - %stride = arith.constant 1 : i64 - %input = memref.alloc() alignment = 64 : memref<16x4xi32> - %scales = memref.alloc() alignment = 64 : memref<4x4xf32> - %output_seed = memref.alloc() alignment = 64 : memref<12x16xi8> - %output = memref.alloc() alignment = 64 : memref<12x16xi8> - linalg.fill ins(%one_f32 : f32) outs(%scales : memref<4x4xf32>) - linalg.fill ins(%zero_i8 : i8) outs(%output_seed : memref<12x16xi8>) - linalg.fill ins(%zero_i8 : i8) outs(%output : memref<12x16xi8>) - scf.for %position = %zero to %four step %one { - scf.for %channel = %zero to %sixteen step %one { - %position_base = arith.muli %position, %twenty : index - %with_channel = arith.addi %position_base, %channel : index - %value = arith.subi %with_channel, %eight : index - %value_i32 = arith.index_cast %value : index to i32 - %channel_row = arith.divui %channel, %four : index - %row_base = arith.muli %position, %four : index - %row = arith.addi %row_base, %channel_row : index - %lane = arith.remui %channel, %four : index - memref.store %value_i32, %input[%row, %lane] : memref<16x4xi32> - } - } - %input_bank = buckyball.bank_alloc - %scale_bank = buckyball.bank_alloc - %output_bank = buckyball.bank_alloc - %loaded_input = buckyball.bank_mvin %input %input_bank %depth16 %stride - : memref<16x4xi32> i64 i64 i64 - %loaded_scales = buckyball.bank_mvin %scales %scale_bank %depth4 %stride - : memref<4x4xf32> i64 i64 i64 - %loaded_output = buckyball.bank_mvin %output_seed %output_bank %depth12 %stride - : memref<12x16xi8> i64 i64 i64 - buckyball.quant_i32_to_i8 - %loaded_input, %loaded_scales, %loaded_output, %depth16, %output_base, %zero64 - : i64 i64 i64 - %stored = buckyball.bank_mvout %output %loaded_output %depth12 %stride - : memref<12x16xi8> i64 i64 i64 - func.call @check_result(%output) : (memref<12x16xi8>) -> () - buckyball.bank_release %loaded_input : i64 - buckyball.bank_release %loaded_scales : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %input : memref<16x4xi32> - memref.dealloc %scales : memref<4x4xf32> - memref.dealloc %output_seed : memref<12x16xi8> - memref.dealloc %output : memref<12x16xi8> - return %zero_i8 : i8 -} diff --git a/examples/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8_main.cpp b/examples/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8_main.cpp deleted file mode 100644 index 85eb15724..000000000 --- a/examples/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8_main.cpp +++ /dev/null @@ -1,42 +0,0 @@ -#include -#include -#include - -static void fail(void) { -#ifdef BAREMETAL - *(volatile uint32_t *)0x60000000 = 1; - while (1) { - } -#else - exit(1); -#endif -} - -extern "C" void check_result(int8_t *allocated, int8_t *aligned, int64_t offset, - int64_t rows, int64_t columns, int64_t row_stride, - int64_t column_stride) { - (void)allocated; - if (rows != 12 || columns != 16 || row_stride != 16 || column_stride != 1) - fail(); - int8_t *output = aligned + offset; - for (int row = 0; row < 12; ++row) { - for (int channel = 0; channel < 16; ++channel) { - int position = row == 3 ? 0 - : row == 4 ? 1 - : row == 7 ? 2 - : row == 8 ? 3 - : -1; - int expected = position < 0 ? 0 : position * 20 + channel - 8; - if (expected < 0) - expected = 0; - int actual = output[row * row_stride + channel * column_stride]; - if (actual != expected) { - printf( - "FAILED: quant_i32_to_i8 row=%d channel=%d expected=%d actual=%d\n", - row, channel, expected, actual); - fail(); - } - } - } - printf("PASSED: quant_i32_to_i8 relu and strided output\n"); -} diff --git a/examples/balls/toint8/workloads/mlir_tests/bank/CMakeLists.txt b/examples/balls/toint8/workloads/mlir_tests/bank/CMakeLists.txt deleted file mode 100644 index 653a5c340..000000000 --- a/examples/balls/toint8/workloads/mlir_tests/bank/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) - set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) -endif() -set(BUCKYBALL_MLIR_TEST_PREFIX bank) - -buckyball_require_quant_i32_bank_shape(${BUCKYBALL_MLIR_ACTIVE_TARGET}) -add_buckyball_mlir_test(quant_i32_pack16 TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) -add_buckyball_mlir_test(quant_f32_pack16 TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16.mlir.in b/examples/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16.mlir.in deleted file mode 100644 index 118f0d8c6..000000000 --- a/examples/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16.mlir.in +++ /dev/null @@ -1,39 +0,0 @@ -func.func private @check_result(memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8>) -> () -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %four = arith.constant 4 : index - %sixteen = arith.constant 16 : index - %rows = arith.constant @BB_QUANT_I32_OUTPUT_ROWS@ : index - %depth = arith.constant @BB_BANK_LINES@ : i64 - %out_depth = arith.constant @BB_QUANT_I32_OUTPUT_ROWS@ : i64 - %stride = arith.constant 1 : i64 - %scale = arith.constant 1.0 : f32 - %input = memref.alloc() : memref<@BB_BANK_LINES@x4xf32> - %output = memref.alloc() : memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8> - scf.for %position = %zero to %rows step %one { - scf.for %channel = %zero to %sixteen step %one { - %base = arith.muli %position, %sixteen : index - %linear = arith.addi %base, %channel : index - %value_i32 = arith.index_cast %linear : index to i32 - %value = arith.sitofp %value_i32 : i32 to f32 - %row_base = arith.muli %position, %four : index - %channel_row = arith.divui %channel, %four : index - %row = arith.addi %row_base, %channel_row : index - %lane = arith.remui %channel, %four : index - memref.store %value, %input[%row, %lane] : memref<@BB_BANK_LINES@x4xf32> - } - } - %input_bank = buckyball.bank_alloc - %output_bank = buckyball.bank_alloc - %loaded_input = buckyball.bank_mvin %input %input_bank %depth %stride : memref<@BB_BANK_LINES@x4xf32> i64 i64 i64 - %result = buckyball.bank_quant_f32_to_i8 %loaded_input %output_bank %depth %scale : i64 i64 i64 f32 - %stored = buckyball.bank_mvout %output %result %out_depth %stride : memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8> i64 i64 i64 - func.call @check_result(%output) : (memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8>) -> () - buckyball.bank_release %loaded_input : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %input : memref<@BB_BANK_LINES@x4xf32> - memref.dealloc %output : memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8> - return %zero_i8 : i8 -} diff --git a/examples/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16.mlir.in b/examples/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16.mlir.in deleted file mode 100644 index cffa6f5c8..000000000 --- a/examples/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16.mlir.in +++ /dev/null @@ -1,50 +0,0 @@ -func.func private @check_result(memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8>) -> () -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %one_f32 = arith.constant 1.0 : f32 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %four = arith.constant 4 : index - %sixteen = arith.constant 16 : index - %rows = arith.constant @BB_QUANT_I32_OUTPUT_ROWS@ : index - %depth = arith.constant @BB_QUANT_I32_INPUT_ROWS@ : i64 - %scale_depth = arith.constant 4 : i64 - %out_depth = arith.constant @BB_QUANT_I32_OUTPUT_ROWS@ : i64 - %zero64 = arith.constant 0 : i64 - %stride = arith.constant 1 : i64 - %input = memref.alloc() : memref<@BB_QUANT_I32_INPUT_ROWS@x4xi32> - %scale = memref.alloc() : memref<4x4xf32> - %seed = memref.alloc() : memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8> - %output = memref.alloc() : memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8> - linalg.fill ins(%one_f32 : f32) outs(%scale : memref<4x4xf32>) - linalg.fill ins(%zero_i8 : i8) outs(%seed : memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8>) - scf.for %position = %zero to %rows step %one { - scf.for %channel = %zero to %sixteen step %one { - %base = arith.muli %position, %sixteen : index - %linear = arith.addi %base, %channel : index - %value = arith.index_cast %linear : index to i32 - %row_base = arith.muli %position, %four : index - %channel_row = arith.divui %channel, %four : index - %row = arith.addi %row_base, %channel_row : index - %lane = arith.remui %channel, %four : index - memref.store %value, %input[%row, %lane] : memref<@BB_QUANT_I32_INPUT_ROWS@x4xi32> - } - } - %input_bank = buckyball.bank_alloc - %scale_bank = buckyball.bank_alloc - %output_bank = buckyball.bank_alloc - %loaded_input = buckyball.bank_mvin %input %input_bank %depth %stride : memref<@BB_QUANT_I32_INPUT_ROWS@x4xi32> i64 i64 i64 - %loaded_scale = buckyball.bank_mvin %scale %scale_bank %scale_depth %stride : memref<4x4xf32> i64 i64 i64 - %loaded_output = buckyball.bank_mvin %seed %output_bank %out_depth %stride : memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8> i64 i64 i64 - %result = buckyball.bank_quant_i32_to_i8 %loaded_input %loaded_scale %loaded_output %depth %zero64 %zero64 : i64 i64 i64 i64 i64 i64 - %stored = buckyball.bank_mvout %output %result %out_depth %stride : memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8> i64 i64 i64 - func.call @check_result(%output) : (memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8>) -> () - buckyball.bank_release %loaded_input : i64 - buckyball.bank_release %loaded_scale : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %input : memref<@BB_QUANT_I32_INPUT_ROWS@x4xi32> - memref.dealloc %scale : memref<4x4xf32> - memref.dealloc %seed : memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8> - memref.dealloc %output : memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8> - return %zero_i8 : i8 -} diff --git a/examples/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16_main.cpp b/examples/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16_main.cpp deleted file mode 100644 index 6d965d0e7..000000000 --- a/examples/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16_main.cpp +++ /dev/null @@ -1,23 +0,0 @@ -#include -#include -#include -#include -extern "C" void check_result(int8_t *, int8_t *data, int64_t offset, - int64_t rows, int64_t lanes, int64_t stride, - int64_t lane_stride) { - const int expected_rows = (BANK_LINES < 508 ? BANK_LINES : 508) / 4; - if (rows != expected_rows || lanes != BANK_WIDTH / 8 || stride != lanes || - lane_stride != 1) - exit(1); - for (int row = 0; row < rows; ++row) - for (int lane = 0; lane < lanes; ++lane) { - int expected = row * lanes + lane; - if (expected > 127) - expected = 127; - if (data[offset + row * stride + lane] != expected) { - printf("quant i32 bank mismatch row=%d lane=%d\n", row, lane); - exit(1); - } - } - printf("quant i32 bank PASS rows=%d\n", BANK_LINES); -} diff --git a/examples/balls/trace/arch/src/main/scala/README.md b/examples/balls/trace/arch/src/main/scala/README.md deleted file mode 100644 index af1974e8e..000000000 --- a/examples/balls/trace/arch/src/main/scala/README.md +++ /dev/null @@ -1,127 +0,0 @@ -# TraceBall — debug/trace Ball - -## Overview - -TraceBall is a non-computing Ball that exposes runtime debug features through the Buckyball instruction channel. It has two main roles: - -1. **Cycle counter** — set/release multiple independent cycle counters via instructions to measure elapsed cycles over arbitrary code regions -2. **Bank backdoor** — inject data into SRAM banks or read bank data out via DPI-C - -All DPI-C hooks live inside TraceBall only and do not affect other modules. - ---- - -## Instruction encoding - -TraceBall uses two Core-owned `ballISA` mnemonics. The selected Core assigns -their `funct7` values at compile time. - -### Instruction 1: `bdb_counter` - -Cycle counter control. **Does not touch SRAM, needs no bank ports, completes in 1 cycle.** - -rs1 layout: -- rs1 = don’t care (unused; no need to set BB_RD0/BB_WR) - -rs2 layout (64-bit): -``` -rs2[3:0] = subcmd subcommand (0=START, 1=STOP, 2=READ) -rs2[7:4] = ctr_id counter id (0–15, up to 16 independent counters) -rs2[63:8] = payload -``` - -Subcommands: - -| subcmd | name | behavior | payload meaning | -|--------|------|----------|-----------------| -| 0 | `CTR_START` | Start counter `ctr_id`, record current cycle as start | payload = tag (user tag, echoed in trace) | -| 1 | `CTR_STOP` | Stop `ctr_id`, emit elapsed cycles to DPI-C trace, then free the counter | payload = ignored | -| 2 | `CTR_READ` | Read current value of `ctr_id` (does not stop), emit to DPI-C trace | payload = ignored | - -DPI-C trace format (written to bdb.log): -``` -[CTRACE] CTR_START ctr=0 tag=0xDEAD cycle=10042 -[CTRACE] CTR_STOP ctr=0 tag=0xDEAD elapsed=387 cycle=10429 -[CTRACE] CTR_READ ctr=0 current=200 cycle=10242 -``` - -### Instruction 2: `bdb_backdoor` - -SRAM backdoor read/write; **all parameters (bank_id, row, data) come from DPI-C**. **Requires bank ports (inBW=1, outBW=1).** - -rs1 layout: -``` -rs1[45] = BB_RD0 read: get address from DPI-C, read SRAM, send data to DPI-C -rs1[47] = BB_WR write: get address+data from DPI-C, write SRAM -rs1[63:48] = iter repeat count (0 = once, >0 = loop `iter` times) -``` - -rs2 layout: -- rs2 = don’t care (unused) - -Modes: - -| rs1 flag | behavior | DPI-C interaction | -|----------|----------|-------------------| -| BB_RD0 | read | RTL calls `dpi_backdoor_get_read_addr()` for (bank_id, row), reads SRAM, `dpi_backdoor_put_read_data()` for data | -| BB_WR | write | RTL calls `dpi_backdoor_get_write_req()` for (bank_id, row, data), writes SRAM | - -DPI-C trace format: -``` -[BANK-TRACE] BACKDOOR_READ bank=2 row=5 data=0x00010002000300040005000600070008 -[BANK-TRACE] BACKDOOR_WRITE bank=3 row=10 data=0xDEADBEEFDEADBEEFDEADBEEFDEADBEEF -``` - - -## Examples - -### Measure kernel cycles - -```c -// One matmul region -bdb_counter_start(0, 0xA001); // counter 0, tag=matmul -bb_vecmat16(A, B, C, 16); -bdb_counter_stop(0); // prints elapsed - -// Nested regions -bdb_counter_start(0, 0xB001); // outer: whole conv - bdb_counter_start(1, 0xB002); // inner: im2col - bb_im2col(...); - bdb_counter_stop(1); - - bdb_counter_start(2, 0xB003); // inner: matmul - bb_vecmat16(...); - bdb_counter_stop(2); -bdb_counter_stop(0); // outer end -``` - -bdb.log sample: -``` -[CTRACE] CTR_START ctr=0 tag=0xB001 cycle=0 -[CTRACE] CTR_START ctr=1 tag=0xB002 cycle=0 -[CTRACE] CTR_STOP ctr=1 tag=0xB002 elapsed=150 cycle=150 -[CTRACE] CTR_START ctr=2 tag=0xB003 cycle=0 -[CTRACE] CTR_STOP ctr=2 tag=0xB003 elapsed=300 cycle=300 -[CTRACE] CTR_STOP ctr=0 tag=0xB001 elapsed=456 cycle=456 -``` - -### Backdoor test data into SRAM - -TraceBall has a private bank that is not visible to normal configuration. - -```c -// No DMA: inject test data into bank 0 via DPI-C -// (C++ first injects into TraceBall’s internal bank via DPI-C) -bb_alloc(0, 1, 1); -bdb_backdoor_mvin(16); // inject 16 rows into private bank -bdb_backdoor_write(0, 16); // move 16 rows from private bank to bank 0 - -bb_transpose(0, 1, 16); -bdb_backdoor_peek(0, 15); // inspect last row - -bdb_backdoor_read(1, 16); // dump 16 rows from bank 1 to trace -``` - - - -All RTL changes stay inside TraceBall; do not modify external banks or shared infrastructure there. diff --git a/examples/balls/trace/arch/src/main/scala/Trace.scala b/examples/balls/trace/arch/src/main/scala/Trace.scala deleted file mode 100644 index 823615095..000000000 --- a/examples/balls/trace/arch/src/main/scala/Trace.scala +++ /dev/null @@ -1,200 +0,0 @@ -package examples.balls.trace - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.balldomain.rs.{BallRsComplete, BallRsIssue} -import framework.balldomain.blink.{BallStatus, BankRead, BankWrite} -import framework.top.GlobalConfig - -/** - * Trace — TraceBall inner processing unit. - * - * Handles cycle counter management (START/STOP/READ). - */ -@instantiable -class Trace(val b: GlobalConfig) extends Module { - - val ballMapping = b.ballDomain.ballIdMappings - .find(_.ballName == "TraceBall") - .getOrElse( - throw new IllegalArgumentException("TraceBall not found in config") - ) - - val inBW = ballMapping.inBW - val outBW = ballMapping.outBW - - val bankWidth = b.memDomain.bankWidth - - @public - val io = IO(new Bundle { - val cmdReq = Flipped(Decoupled(new BallRsIssue(b))) - val cmdResp = Decoupled(new BallRsComplete(b)) - val bankRead = Vec(inBW, Flipped(new BankRead(b))) - val bankWrite = Vec(outBW, Flipped(new BankWrite(b))) - val status = new BallStatus - }) - - // ============================================================ - // Constants - // ============================================================ - val CTR_START = 0.U(4.W) - val CTR_STOP = 1.U(4.W) - val CTR_READ = 2.U(4.W) - - val NUM_COUNTERS = 16 - - // ============================================================ - // State machine - // ============================================================ - val idle :: sCounter :: complete :: Nil = Enum(3) - val state = RegInit(idle) - - // ============================================================ - // Registers - // ============================================================ - val rob_id_reg = RegInit(0.U(log2Up(b.frontend.rob_entries).W)) - val is_sub_reg = RegInit(false.B) - val sub_rob_id_reg = RegInit(0.U(log2Up(b.frontend.sub_rob_depth * 4).W)) - - // Counter-specific registers - val subcmd_reg = RegInit(0.U(4.W)) - val ctr_id_reg = RegInit(0.U(4.W)) - val payload_reg = RegInit(0.U(56.W)) - - // Cycle counters: 16 independent counters - val cycleCounter = RegInit(0.U(64.W)) - cycleCounter := cycleCounter + 1.U - - val ctrStartCycle = RegInit(VecInit(Seq.fill(NUM_COUNTERS)(0.U(64.W)))) - val ctrTag = RegInit(VecInit(Seq.fill(NUM_COUNTERS)(0.U(56.W)))) - val ctrActive = RegInit(VecInit(Seq.fill(NUM_COUNTERS)(false.B))) - - // ============================================================ - // DPI-C modules - // ============================================================ - val ctraceDpi = Module(new CTraceDPI) - ctraceDpi.io.clock := clock - ctraceDpi.io.reset := reset.asBool - ctraceDpi.io.subcmd := 0.U - ctraceDpi.io.ctr_id := 0.U - ctraceDpi.io.tag := 0.U - ctraceDpi.io.elapsed := 0.U - ctraceDpi.io.cycle := cycleCounter - ctraceDpi.io.enable := false.B - - // ============================================================ - // External bank port defaults - // ============================================================ - for (i <- 0 until inBW) { - io.bankRead(i).rob_id := rob_id_reg - io.bankRead(i).ball_id := 0.U - io.bankRead(i).bank_id := 0.U - io.bankRead(i).group_id := 0.U - io.bankRead(i).io.req.valid := false.B - io.bankRead(i).io.req.bits.addr := 0.U - io.bankRead(i).io.resp.ready := false.B - } - - for (i <- 0 until outBW) { - io.bankWrite(i).rob_id := rob_id_reg - io.bankWrite(i).ball_id := 0.U - io.bankWrite(i).bank_id := 0.U - io.bankWrite(i).group_id := 0.U - io.bankWrite(i).io.req.valid := false.B - io.bankWrite(i).io.req.bits.addr := 0.U - io.bankWrite(i).io.req.bits.data := 0.U - io.bankWrite(i).io.req.bits.mask := VecInit( - Seq.fill(b.memDomain.bankMaskLen)(0.U(1.W)) - ) - io.bankWrite(i).io.resp.ready := false.B - } - - // ============================================================ - // Command interface - // ============================================================ - io.cmdReq.ready := state === idle - io.cmdResp.valid := false.B - io.cmdResp.bits.rob_id := rob_id_reg - io.cmdResp.bits.is_sub := is_sub_reg - io.cmdResp.bits.sub_rob_id := sub_rob_id_reg - - // ============================================================ - // State machine - // ============================================================ - switch(state) { - is(idle) { - when(io.cmdReq.fire) { - rob_id_reg := io.cmdReq.bits.rob_id - is_sub_reg := io.cmdReq.bits.is_sub - sub_rob_id_reg := io.cmdReq.bits.sub_rob_id - - val cmd = io.cmdReq.bits.cmd - val rs2 = cmd.rs2 - - // Counter fields from rs2 - subcmd_reg := rs2(3, 0) - ctr_id_reg := rs2(7, 4) - payload_reg := rs2(63, 8) - - state := sCounter - } - } - - // ---------------------------------------------------------- - // Counter instruction: execute and complete in 1 cycle - // ---------------------------------------------------------- - is(sCounter) { - val cid = ctr_id_reg - - ctraceDpi.io.subcmd := subcmd_reg - ctraceDpi.io.ctr_id := cid - - switch(subcmd_reg) { - is(CTR_START) { - ctrStartCycle(cid) := cycleCounter - ctrTag(cid) := payload_reg - ctrActive(cid) := true.B - - ctraceDpi.io.tag := payload_reg - ctraceDpi.io.elapsed := 0.U - ctraceDpi.io.enable := true.B - } - is(CTR_STOP) { - val elapsed = cycleCounter - ctrStartCycle(cid) - ctrActive(cid) := false.B - - ctraceDpi.io.tag := ctrTag(cid) - ctraceDpi.io.elapsed := elapsed - ctraceDpi.io.enable := true.B - } - is(CTR_READ) { - val current = cycleCounter - ctrStartCycle(cid) - - ctraceDpi.io.tag := ctrTag(cid) - ctraceDpi.io.elapsed := current - ctraceDpi.io.enable := true.B - } - } - - state := complete - } - - // ---------------------------------------------------------- - // Complete: fire cmdResp - // ---------------------------------------------------------- - is(complete) { - io.cmdResp.valid := true.B - io.cmdResp.bits.rob_id := rob_id_reg - when(io.cmdResp.fire) { - state := idle - } - } - } - - // ============================================================ - // Status - // ============================================================ - io.status.idle := state === idle - io.status.running := state =/= idle && state =/= complete -} diff --git a/examples/balls/trace/arch/src/main/scala/TraceBall.scala b/examples/balls/trace/arch/src/main/scala/TraceBall.scala deleted file mode 100644 index 53c48cf51..000000000 --- a/examples/balls/trace/arch/src/main/scala/TraceBall.scala +++ /dev/null @@ -1,53 +0,0 @@ -package examples.balls.trace - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.balldomain.blink.{BlinkIO, HasBlink, SubRobRow} -import framework.balldomain.blink.mmio.MmioRead -import framework.top.GlobalConfig - -/** - * TraceBall — Debug trace Ball providing cycle counters. - * - * The Core ballISA assigns the funct7 encoding for bdb_counter. - */ -@instantiable -class TraceBall(val b: GlobalConfig) extends Module with HasBlink { - - val ballCommonConfig = b.ballDomain.ballIdMappings - .find(_.ballName == "TraceBall") - .getOrElse( - throw new IllegalArgumentException("TraceBall not found in config") - ) - - val inBW = ballCommonConfig.inBW - val outBW = ballCommonConfig.outBW - - @public - val io = IO(new BlinkIO(b, inBW, outBW)) - - def blink: BlinkIO = io - dontTouch(io) - - val traceUnit: Instance[Trace] = Instantiate(new Trace(b)) - - traceUnit.io.cmdReq <> io.cmdReq - traceUnit.io.cmdResp <> io.cmdResp - - for (i <- 0 until inBW) { - traceUnit.io.bankRead(i) <> io.bankRead(i) - } - - for (i <- 0 until outBW) { - traceUnit.io.bankWrite(i) <> io.bankWrite(i) - } - - io.status <> traceUnit.io.status - - io.subRobReq.valid := false.B - io.subRobReq.bits := SubRobRow.tieOff(b) - - // MMIO: this Ball does not consume MMIO metadata - MmioRead.tieOff(io.mmioRead) -} diff --git a/examples/balls/trace/compiler/src/Dialect/Buckyball/TraceBall.td b/examples/balls/trace/compiler/src/Dialect/Buckyball/TraceBall.td deleted file mode 100644 index 398293258..000000000 --- a/examples/balls/trace/compiler/src/Dialect/Buckyball/TraceBall.td +++ /dev/null @@ -1,18 +0,0 @@ -//===-- TraceBall.td - Trace Ball operations ----------------------------===// - -#ifndef TRACE_BALL_OPS -#define TRACE_BALL_OPS - -include "Buckyball.td" - -def BdbCounterOp : Buckyball_Op<"bdb_counter"> { - let summary = "Start, stop, or read a Buckyball debug counter"; - let description = [{ - Encodes BDB_COUNTER rs2 as subcommand[3:0], counterId[7:4], and - payload[63:8]. rs1 is always zero. - }]; - let arguments = (ins I64:$subcommand, I64:$counterId, I64:$payload); - let assemblyFormat = "$subcommand `,` $counterId `,` $payload attr-dict `:` type($subcommand)"; -} - -#endif // TRACE_BALL_OPS diff --git a/examples/balls/trace/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/balls/trace/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 475ecd5ab..000000000 --- a/examples/balls/trace/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,55 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace buddy::buckyball; -using namespace buddy::buckyball::legalize; - -namespace { -class BdbCounterLowering : public ConvertOpToLLVMPattern { -public: - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - - LogicalResult - matchAndRewrite(BdbCounterOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("TraceBall"); - Location loc = op.getLoc(); - Value counter = rewriter.create(loc, adaptor.getCounterId(), - cstI64(rewriter, loc, 4)); - Value payload = rewriter.create(loc, adaptor.getPayload(), - cstI64(rewriter, loc, 8)); - Value rs2 = - rewriter.create(loc, adaptor.getSubcommand(), counter); - rs2 = rewriter.create(loc, rs2, payload); - rewriter.replaceOpWithNewOp( - op, cstI64(rewriter, loc, 0), rs2, - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("BDB_COUNTER"))); - return success(); - } -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateTraceBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth) { - (void)stable; - (void)bankDepth; - - patterns.add(converter); -} - -void configureTraceBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable) { - (void)stable; - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/examples/balls/trace/configs/default.toml b/examples/balls/trace/configs/default.toml deleted file mode 100644 index 6f2c952cd..000000000 --- a/examples/balls/trace/configs/default.toml +++ /dev/null @@ -1,3 +0,0 @@ -# Trace Ball Configuration - -[ball] diff --git a/examples/balls/trace/emu/src/04_bdb_counter.rs b/examples/balls/trace/emu/src/04_bdb_counter.rs deleted file mode 100644 index 4fd1f9437..000000000 --- a/examples/balls/trace/emu/src/04_bdb_counter.rs +++ /dev/null @@ -1,15 +0,0 @@ -//===- 04_bdb_counter.rs - BDB_COUNTER instruction -------------------------===// - -use super::instruction::{BallInstruction, ExecContext}; - -pub struct BdbCounter; - -impl BallInstruction for BdbCounter { - fn exec(_xs1: u64, _xs2: u64, _ctx: &mut ExecContext) -> u64 { - 0 - } - - fn latency(_xs1: u64, _xs2: u64) -> u64 { - 1 - } -} diff --git a/examples/balls/trace/emu/src/dispatch.rs b/examples/balls/trace/emu/src/dispatch.rs deleted file mode 100644 index a50ec9e07..000000000 --- a/examples/balls/trace/emu/src/dispatch.rs +++ /dev/null @@ -1,31 +0,0 @@ -use crate::inst::instruction::{BallInstruction, ExecContext}; - -use super::f04_bdb_counter; - -const BALL_CLASS: &str = "examples.balls.trace.TraceBall"; - -pub fn execute_known( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, - ctx: &mut ExecContext, -) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("BDB_COUNTER") => Some(f04_bdb_counter::BdbCounter::exec(xs1, xs2, ctx)), - Some(_) | None => None, - } -} - -pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("BDB_COUNTER") => Some(f04_bdb_counter::BdbCounter::latency(xs1, xs2)), - Some(_) | None => None, - } -} diff --git a/examples/balls/trace/emu/src/lib.rs b/examples/balls/trace/emu/src/lib.rs deleted file mode 100644 index fe1e94f18..000000000 --- a/examples/balls/trace/emu/src/lib.rs +++ /dev/null @@ -1,8 +0,0 @@ -pub(crate) use crate::inst::instruction; - -#[path = "04_bdb_counter.rs"] -mod f04_bdb_counter; - -mod dispatch; - -pub use dispatch::{cycles_after_issue, execute_known}; diff --git a/examples/balls/trace/workloads/ctests/CMakeLists.txt b/examples/balls/trace/workloads/ctests/CMakeLists.txt deleted file mode 100644 index fbfa8b478..000000000 --- a/examples/balls/trace/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -add_buckyball_ctests( - bdb_counter_start_stop_test.c - bdb_counter_read_active_test.c - bdb_counter_nested_pair_test.c - bdb_counter_multi_id_test.c -) diff --git a/examples/balls/trace/workloads/mlir_tests/ball/bdb_counter_start_read_stop.mlir b/examples/balls/trace/workloads/mlir_tests/ball/bdb_counter_start_read_stop.mlir deleted file mode 100644 index 923de4009..000000000 --- a/examples/balls/trace/workloads/mlir_tests/ball/bdb_counter_start_read_stop.mlir +++ /dev/null @@ -1,12 +0,0 @@ -func.func @main() -> i8 { - %zero8 = arith.constant 0 : i8 - %start = arith.constant 0 : i64 - %stop = arith.constant 1 : i64 - %read = arith.constant 2 : i64 - %counter = arith.constant 3 : i64 - %tag = arith.constant 48879 : i64 - buckyball.bdb_counter %start, %counter, %tag : i64 - buckyball.bdb_counter %read, %counter, %start : i64 - buckyball.bdb_counter %stop, %counter, %start : i64 - return %zero8 : i8 -} diff --git a/examples/balls/transpose/arch/src/main/scala/README.md b/examples/balls/transpose/arch/src/main/scala/README.md deleted file mode 100644 index 3c01b53d7..000000000 --- a/examples/balls/transpose/arch/src/main/scala/README.md +++ /dev/null @@ -1,115 +0,0 @@ -# Matrix Transpose Accelerator - -## Overview - -This directory implements Buckyball's matrix transpose accelerator for matrix transpose operations. Located at `arch/src/main/scala/prototype/transpose`, it serves as a matrix transpose accelerator supporting pipelined transpose operations. - -Core components: -- **Transpose.scala**: Pipelined transposer implementation - -## Code Structure - -``` -transpose/ -└── Transpose.scala - Pipelined transposer -``` - -### Module Responsibilities - -**Transpose.scala** (Transpose implementation layer) -- Implements PipelinedTransposer module -- Manages matrix data read, transpose, and write-back -- Provides Ball domain command interface - -## Module Description - -### Transpose.scala - -**Main functionality**: Implements pipelined matrix transpose operation - -**State machine definition**: -```scala -val idle :: sRead :: sWrite :: complete :: Nil = Enum(4) -val state = RegInit(idle) -``` - -**Storage structure**: -```scala -// Matrix storage register (veclane x veclane) -val regArray = Reg(Vec(b.veclane, Vec(b.veclane, UInt(b.inputType.getWidth.W)))) -``` - -**Counter management**: -```scala -val readCounter = RegInit(0.U(log2Ceil(b.veclane + 1).W)) -val respCounter = RegInit(0.U(log2Ceil(b.veclane + 1).W)) -val writeCounter = RegInit(0.U(log2Ceil(b.veclane + 1).W)) -``` - -**Instruction registers**: -```scala -val robid_reg = RegInit(0.U(10.W)) // ROB ID -val waddr_reg = RegInit(0.U(10.W)) // Write address -val wbank_reg = RegInit(0.U(log2Up(b.sp_banks).W)) // Write bank -val raddr_reg = RegInit(0.U(10.W)) // Read address -val rbank_reg = RegInit(0.U(log2Up(b.sp_banks).W)) // Read bank -val iter_reg = RegInit(0.U(10.W)) // Iteration count -``` - -**Interface definition**: -```scala -val io = IO(new Bundle { - val cmdReq = Flipped(Decoupled(new BallRsIssue)) - val cmdResp = Decoupled(new BallRsComplete) - val sramRead = Vec(b.sp_banks, Flipped(new SramReadIO(...))) - val sramWrite = Vec(b.sp_banks, Flipped(new SramWriteIO(...))) -}) -``` - -**Processing flow**: -1. **idle**: Wait for command, parse transpose parameters -2. **sRead**: Read matrix data row by row into register array -3. **sWrite**: Write transposed data column by column -4. **complete**: Send completion signal - -**Transpose algorithm**: -- Uses veclane×veclane register array to store matrix -- Reads row-wise, writes column-wise to implement transpose -- Supports block-wise transpose for matrices of arbitrary size - -## Usage - -### Implementation Details - -**State machine**: -```scala -val idle :: sRead :: sWrite :: complete :: Nil = Enum(4) -``` -- `idle`: Wait for instruction -- `sRead`: Read matrix data -- `sWrite`: Write transpose result -- `complete`: Complete and respond - -**Register array**: -```scala -val regArray = Reg(Vec(b.veclane, Vec(b.veclane, UInt(b.inputType.getWidth.W)))) -``` -Uses veclane×veclane register array to cache matrix data. - -**Transpose operation**: -- Read phase: Read data row by row into `regArray(row)(col)` -- Write phase: Read `regArray(i)(col)` column by column to form new rows for writing - -### Configuration Parameters - -**Matrix size**: Determined by b.veclane parameter -**Data width**: Determined by b.inputType.getWidth -**Bank configuration**: Supports multi-bank SRAM access - -### Notes - -1. **Matrix size limitation**: Maximum support for veclane×veclane matrices -2. **Memory bandwidth**: Transpose operation has high memory bandwidth requirements -3. **Register overhead**: Requires veclane² registers to store matrix -4. **Address calculation**: Transposed address calculation needs to be handled correctly -5. **Pipeline control**: Read/write counters need to be synchronized correctly diff --git a/examples/balls/transpose/arch/src/main/scala/Transpose.scala b/examples/balls/transpose/arch/src/main/scala/Transpose.scala deleted file mode 100644 index b4a6230dc..000000000 --- a/examples/balls/transpose/arch/src/main/scala/Transpose.scala +++ /dev/null @@ -1,260 +0,0 @@ -package examples.balls.transpose - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} - -import framework.balldomain.rs.{BallRsComplete, BallRsIssue} -import framework.balldomain.blink.{BallStatus, BankRead, BankWrite} -import framework.top.GlobalConfig -import examples.balls.transpose.configs.TransposeBallParam - -@instantiable -class Transpose(val b: GlobalConfig) extends Module { - val ballConfig = TransposeBallParam(b) - val bankWidth = b.memDomain.bankWidth - val rowBytes = bankWidth / 8 - - val ballMapping = b.ballDomain.ballIdMappings - .find(_.ballName == "TransposeBall") - .getOrElse( - throw new IllegalArgumentException("TransposeBall not found in config") - ) - - val inBW = ballMapping.inBW - val outBW = ballMapping.outBW - require(inBW == outBW, "TransposeBall requires inBW == outBW") - require(inBW == 1, "TransposeBall gather/scatter path requires inBW == 1") - require(bankWidth % 8 == 0, "bankWidth must be byte-aligned") - require( - ballConfig.InputNum * ballConfig.inputWidth == bankWidth, - "TransposeBall InputNum*inputWidth must equal bankWidth" - ) - - @public - val io = IO(new Bundle { - val cmdReq = Flipped(Decoupled(new BallRsIssue(b))) - val cmdResp = Decoupled(new BallRsComplete(b)) - val bankRead = Vec(inBW, Flipped(new BankRead(b))) - val bankWrite = Vec(outBW, Flipped(new BankWrite(b))) - val status = new BallStatus - }) - - val rob_id_reg = RegInit(0.U(log2Up(b.frontend.rob_entries).W)) - val is_sub_reg = RegInit(false.B) - val sub_rob_id_reg = RegInit(0.U(log2Up(b.frontend.sub_rob_depth * 4).W)) - - val rbank_reg = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - val wbank_reg = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - val ncol_reg = RegInit(0.U(log2Up(b.memDomain.bankNum + 1).W)) - val iter_reg = RegInit(0.U(b.frontend.iter_len.W)) - val elem_reg = RegInit(0.U(8.W)) - - val idle :: sRead :: sWrite :: complete :: Nil = Enum(4) - val state = RegInit(idle) - - // Walk destination dense index 0 .. iter*W-1, filling write beats. - val pending = RegInit(false.B) - val wrBytes = Reg(Vec(rowBytes, UInt(8.W))) - val wrMask = RegInit(VecInit(Seq.fill(b.memDomain.bankMaskLen)(0.U(1.W)))) - val wrGroup = RegInit(0.U(log2Up(b.memDomain.bankNum + 1).W)) - val wrAddr = RegInit(0.U(b.frontend.iter_len.W)) - val wrValid = RegInit(false.B) - val beatFill = RegInit(0.U(8.W)) - - val cached = RegInit(false.B) - val cacheAddr = RegInit(0.U(b.frontend.iter_len.W)) - val cacheGroup = RegInit(0.U(log2Up(b.memDomain.bankNum + 1).W)) - val cacheData = RegInit(0.U(bankWidth.W)) - - val srcR = RegInit(0.U(b.frontend.iter_len.W)) - val srcC = RegInit(0.U(32.W)) - val dstVirtRow = RegInit(0.U(b.frontend.iter_len.W)) - val dstVirtCol = RegInit(0.U(32.W)) - val epgReg = RegInit(0.U(8.W)) - val wElemsReg = RegInit(0.U(32.W)) - val elemBytesReg = RegInit(0.U(8.W)) - val lastBeat = RegInit(false.B) - - for (i <- 0 until inBW) { - io.bankRead(i).rob_id := rob_id_reg - io.bankRead(i).ball_id := 0.U - io.bankRead(i).bank_id := rbank_reg - io.bankRead(i).group_id := 0.U - io.bankRead(i).io.req.valid := false.B - io.bankRead(i).io.req.bits.addr := 0.U - io.bankRead(i).io.resp.ready := false.B - } - for (i <- 0 until outBW) { - io.bankWrite(i).rob_id := rob_id_reg - io.bankWrite(i).ball_id := 0.U - io.bankWrite(i).bank_id := wbank_reg - io.bankWrite(i).group_id := 0.U - io.bankWrite(i).io.req.valid := false.B - io.bankWrite(i).io.req.bits.addr := 0.U - io.bankWrite(i).io.req.bits.data := 0.U - io.bankWrite(i).io.req.bits.mask := VecInit( - Seq.fill(b.memDomain.bankMaskLen)(0.U(1.W)) - ) - io.bankWrite(i).io.resp.ready := (state =/= idle) - } - - io.cmdReq.ready := (state === idle) - io.cmdResp.valid := false.B - io.cmdResp.bits.rob_id := rob_id_reg - io.cmdResp.bits.is_sub := is_sub_reg - io.cmdResp.bits.sub_rob_id := sub_rob_id_reg - - require(isPow2(rowBytes), "Transpose rowBytes must be power of 2") - - val laneBitsI8 = log2Ceil(rowBytes) - val laneBitsI32 = log2Ceil(rowBytes / 4) - - val isI8 = elemBytesReg === 1.U - val srcLane = Mux(isI8, srcC(laneBitsI8 - 1, 0), srcC(laneBitsI32 - 1, 0)) - val srcGroup = Mux(isI8, srcC >> laneBitsI8.U, srcC >> laneBitsI32.U) - .asTypeOf(UInt(log2Up(b.memDomain.bankNum + 1).W)) - val srcAddr = srcR - - val dstGroup = Mux(isI8, dstVirtCol >> laneBitsI8.U, dstVirtCol >> laneBitsI32.U) - .asTypeOf(UInt(log2Up(b.memDomain.bankNum + 1).W)) - val bytes = cacheData.asTypeOf(Vec(rowBytes, UInt(8.W))) - val words = cacheData.asTypeOf(Vec(rowBytes / 4, UInt(32.W))) - - val needRead = !cached || cacheAddr =/= srcAddr || cacheGroup =/= srcGroup - - switch(state) { - is(idle) { - when(io.cmdReq.fire) { - val cmd = io.cmdReq.bits.cmd - rob_id_reg := io.cmdReq.bits.rob_id - is_sub_reg := io.cmdReq.bits.is_sub - sub_rob_id_reg := io.cmdReq.bits.sub_rob_id - rbank_reg := cmd.op1_bank - wbank_reg := cmd.wr_bank - ncol_reg := cmd.op1_col - iter_reg := cmd.iter - elem_reg := cmd.rs2(7, 0) - pending := false.B - wrValid := false.B - beatFill := 0.U - cached := false.B - srcR := 0.U - srcC := 0.U - dstVirtRow := 0.U - dstVirtCol := 0.U - lastBeat := false.B - val i8 = cmd.rs2(7, 0) === 8.U - elemBytesReg := Mux(i8, 1.U, 4.U) - epgReg := Mux(i8, rowBytes.U, (rowBytes / 4).U) - wElemsReg := Mux(i8, cmd.op1_col << laneBitsI8.U, cmd.op1_col << laneBitsI32.U) - assert(cmd.iter > 0.U, "Transpose iter must be > 0") - assert(cmd.op1_bank < b.memDomain.virtualBankCount.U, "Transpose input bank is invalid") - assert(cmd.wr_bank < b.memDomain.virtualBankCount.U, "Transpose output bank is invalid") - assert(cmd.op1_bank =/= cmd.wr_bank, "Transpose op1 and wr must differ") - assert( - cmd.op1_col === cmd.wr_col && cmd.op1_col =/= 0.U, - "Transpose cols mismatch" - ) - assert(cmd.rs2(63, 8) === 0.U, "Transpose rs2[63:8] must be 0") - assert( - cmd.rs2(7, 0) === 8.U || cmd.rs2(7, 0) === 32.U, - "Transpose elem_bits" - ) - assert( - bankWidth.U % cmd.rs2(7, 0) === 0.U, - "Transpose bankWidth not divisible by elem_bits" - ) - state := sRead - } - } - - is(sRead) { - io.bankRead(0).group_id := srcGroup - io.bankRead(0).io.resp.ready := pending - io.bankRead(0).io.req.valid := needRead && !pending && !wrValid - io.bankRead(0).io.req.bits.addr := srcAddr - - when(io.bankRead(0).io.req.fire) { - pending := true.B - } - when(io.bankRead(0).io.resp.fire) { - cacheData := io.bankRead(0).io.resp.bits.data - cacheAddr := srcAddr - cacheGroup := srcGroup - cached := true.B - pending := false.B - } - - when(cached && !needRead && !wrValid) { - when(beatFill === 0.U) { - wrGroup := dstGroup - wrAddr := dstVirtRow - wrBytes := VecInit(Seq.fill(rowBytes)(0.U(8.W))) - } - when(isI8) { - wrBytes(beatFill) := bytes(srcLane) - }.otherwise { - val w = words(srcLane) - wrBytes(beatFill << 2) := w(7, 0) - wrBytes((beatFill << 2) + 1.U) := w(15, 8) - wrBytes((beatFill << 2) + 2.U) := w(23, 16) - wrBytes((beatFill << 2) + 3.U) := w(31, 24) - } - beatFill := beatFill + 1.U - - val lastElem = (srcR === iter_reg - 1.U) && (srcC === wElemsReg - 1.U) - when(srcR === iter_reg - 1.U) { - srcR := 0.U - srcC := srcC + 1.U - }.otherwise { - srcR := srcR + 1.U - } - when(dstVirtCol === wElemsReg - 1.U) { - dstVirtCol := 0.U - dstVirtRow := dstVirtRow + 1.U - }.otherwise { - dstVirtCol := dstVirtCol + 1.U - } - - val beatDone = (beatFill + 1.U === epgReg) || lastElem - when(beatDone) { - wrMask.foreach(_ := 1.U) - wrValid := true.B - lastBeat := lastElem - state := sWrite - } - } - } - - is(sWrite) { - io.bankWrite(0).group_id := wrGroup - io.bankWrite(0).io.req.valid := wrValid - io.bankWrite(0).io.req.bits.addr := wrAddr - io.bankWrite(0).io.req.bits.data := wrBytes.asUInt - io.bankWrite(0).io.req.bits.mask := wrMask - io.bankWrite(0).io.resp.ready := true.B - when(io.bankWrite(0).io.req.fire) { - wrValid := false.B - beatFill := 0.U - } - when(io.bankWrite(0).io.resp.fire) { - when(lastBeat) { - state := complete - }.otherwise { - state := sRead - } - } - } - - is(complete) { - io.cmdResp.valid := true.B - when(io.cmdResp.fire) { - state := idle - } - } - } - - io.status.idle := (state === idle) - io.status.running := (state =/= idle) -} diff --git a/examples/balls/transpose/arch/src/main/scala/TransposeBall.scala b/examples/balls/transpose/arch/src/main/scala/TransposeBall.scala deleted file mode 100644 index 109968b94..000000000 --- a/examples/balls/transpose/arch/src/main/scala/TransposeBall.scala +++ /dev/null @@ -1,49 +0,0 @@ -package examples.balls.transpose - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.balldomain.blink.{BlinkIO, HasBlink, SubRobRow} -import framework.balldomain.blink.mmio.MmioRead -import examples.balls.transpose.Transpose -import framework.top.GlobalConfig - -@instantiable -class TransposeBall(val b: GlobalConfig) extends Module with HasBlink { - - val ballCommonConfig = b.ballDomain.ballIdMappings - .find(_.ballName == "TransposeBall") - .getOrElse( - throw new IllegalArgumentException("TransposeBall not found in config") - ) - - val inBW = ballCommonConfig.inBW - val outBW = ballCommonConfig.outBW - - @public - val io = IO(new BlinkIO(b, inBW, outBW)) - - def blink: BlinkIO = io - dontTouch(io) - - val transposeUnit: Instance[Transpose] = Instantiate(new Transpose(b)) - - transposeUnit.io.cmdReq <> io.cmdReq - transposeUnit.io.cmdResp <> io.cmdResp - - for (i <- 0 until inBW) { - transposeUnit.io.bankRead(i) <> io.bankRead(i) - } - - for (i <- 0 until outBW) { - transposeUnit.io.bankWrite(i) <> io.bankWrite(i) - } - - io.status <> transposeUnit.io.status - - io.subRobReq.valid := false.B - io.subRobReq.bits := SubRobRow.tieOff(b) - - // MMIO: this Ball does not consume MMIO metadata - MmioRead.tieOff(io.mmioRead) -} diff --git a/examples/balls/transpose/compiler/src/Backend/include/IntrinsicsRISCVTransposeBall.td b/examples/balls/transpose/compiler/src/Backend/include/IntrinsicsRISCVTransposeBall.td deleted file mode 100644 index db632b990..000000000 --- a/examples/balls/transpose/compiler/src/Backend/include/IntrinsicsRISCVTransposeBall.td +++ /dev/null @@ -1,4 +0,0 @@ -//===- IntrinsicsRISCVTransposeBall.td - TransposeBall intrinsics --------===// - -let TargetPrefix = "riscv" in -def int_riscv_bb_transpose : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; diff --git a/examples/balls/transpose/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp b/examples/balls/transpose/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp deleted file mode 100644 index 20819a12c..000000000 --- a/examples/balls/transpose/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp +++ /dev/null @@ -1,38 +0,0 @@ -//===- AssignPhysicalBankPatterns.cpp - Transpose bank assignment patterns ===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace mlir::buddy { -void populateTransposeBallAssignPhysicalBankPatterns( - RewritePatternSet &patterns, PhysicalBankState &state); -} // namespace mlir::buddy - -namespace { - -class BankTransposePattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(BankTransposeOp op, - PatternRewriter &rewriter) const override { - rewriter.create(op.getLoc(), op.getInBank(), op.getOutBank(), - op.getIter(), op.getElemBits()); - rewriter.replaceOp(op, op.getOutBank()); - return success(); - } -}; - -} // namespace - -void mlir::buddy::populateTransposeBallAssignPhysicalBankPatterns( - RewritePatternSet &patterns, mlir::buddy::PhysicalBankState &state) { - (void)state; - patterns.add(patterns.getContext()); -} diff --git a/examples/balls/transpose/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp b/examples/balls/transpose/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp deleted file mode 100644 index 880276a9a..000000000 --- a/examples/balls/transpose/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp +++ /dev/null @@ -1,52 +0,0 @@ -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/Utils/StructuredOpsUtils.h" -#include "mlir/IR/AffineMap.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace mlir::buddy { -void populateTransposeBallLowerBuckyballToBankSSAPatterns( - RewritePatternSet &patterns); -} // namespace mlir::buddy - -namespace { -class MemTransposeToLinalgPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(MemTransposeOp op, - PatternRewriter &rewriter) const override { - auto input = dyn_cast(op.getInput().getType()); - auto output = dyn_cast(op.getOutput().getType()); - if (!input || !output || !input.hasStaticShape() || - !output.hasStaticShape() || input.getRank() != 2 || - output.getRank() != 2 || output.getShape()[0] != input.getShape()[1] || - output.getShape()[1] != input.getShape()[0]) - return op.emitError("requires matching static rank-2 transpose memrefs"); - MLIRContext *context = rewriter.getContext(); - AffineMap identity = AffineMap::getMultiDimIdentityMap(2, context); - AffineMap transpose = AffineMap::get( - 2, 0, {rewriter.getAffineDimExpr(1), rewriter.getAffineDimExpr(0)}, - context); - SmallVector iterators(2, - utils::IteratorType::parallel); - rewriter.create( - op.getLoc(), TypeRange{}, ValueRange{op.getInput()}, - ValueRange{op.getOutput()}, ArrayRef{identity, transpose}, - iterators, [](OpBuilder &builder, Location location, ValueRange args) { - builder.create(location, args[0]); - }); - rewriter.eraseOp(op); - return success(); - } -}; -} // namespace - -void mlir::buddy::populateTransposeBallLowerBuckyballToBankSSAPatterns( - RewritePatternSet &patterns) { - patterns.add(patterns.getContext()); -} diff --git a/examples/balls/transpose/compiler/src/Conversion/LowerTileToBuckyball/TileTransposePatterns.cpp b/examples/balls/transpose/compiler/src/Conversion/LowerTileToBuckyball/TileTransposePatterns.cpp deleted file mode 100644 index 8b0a81c2a..000000000 --- a/examples/balls/transpose/compiler/src/Conversion/LowerTileToBuckyball/TileTransposePatterns.cpp +++ /dev/null @@ -1,50 +0,0 @@ -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Tile/TileOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; -namespace tile = ::buddy::tile; - -namespace mlir::buddy { -void populateTransposeBallTileLoweringPatterns(RewritePatternSet &patterns, - int64_t bankWidthBytes, - int64_t bankDepth, - int64_t bankNum); -} // namespace mlir::buddy - -namespace { -class TileTransposeLowering : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(tile::TileTransposeOp op, - PatternRewriter &rewriter) const override { - auto inputType = dyn_cast(op.getAMemArray().getType()); - auto outputType = dyn_cast(op.getBMemArray().getType()); - if (!inputType || !outputType || !inputType.hasStaticShape() || - !outputType.hasStaticShape() || inputType.getRank() != 2 || - outputType.getRank() != 2) - return op.emitError("requires static rank-2 input and output memrefs"); - if (outputType.getShape()[0] != inputType.getShape()[1] || - outputType.getShape()[1] != inputType.getShape()[0] || - inputType.getElementType() != outputType.getElementType()) - return op.emitError("requires a matching transposed output memref"); - rewriter.create(op.getLoc(), op.getAMemArray(), - op.getBMemArray()); - rewriter.eraseOp(op); - return success(); - } -}; -} // namespace - -void mlir::buddy::populateTransposeBallTileLoweringPatterns( - RewritePatternSet &patterns, int64_t bankWidthBytes, int64_t bankDepth, - int64_t bankNum) { - (void)bankWidthBytes; - (void)bankDepth; - (void)bankNum; - patterns.add(patterns.getContext()); -} diff --git a/examples/balls/transpose/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/balls/transpose/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 1d6dfa1b3..000000000 --- a/examples/balls/transpose/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,115 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace buddy::buckyball; -using namespace buddy::buckyball::legalize; - -namespace { -static LogicalResult validateTranspose(Operation *op, Value elemBits, - Value iter, int64_t bankDepth) { - auto constant = elemBits.getDefiningOp(); - auto value = - constant ? dyn_cast(constant.getValue()) : IntegerAttr(); - if (!value || (value.getInt() != 8 && value.getInt() != 32)) - return op->emitError("Transpose elemBits must be constant 8 or 32"); - auto iterOp = iter.getDefiningOp(); - auto iterAttr = - iterOp ? dyn_cast(iterOp.getValue()) : IntegerAttr(); - if (!iterAttr || iterAttr.getInt() <= 0 || iterAttr.getInt() > bankDepth) - return op->emitError( - "Transpose iter must be a positive constant within the bank depth"); - return success(); -} - -struct TransposeLowering : public ConvertOpToLLVMPattern { - TransposeLowering(LLVMTypeConverter &converter, bool stable, - int64_t bankDepth) - : ConvertOpToLLVMPattern(converter), stable(stable), - bankDepth(bankDepth) {} - - LogicalResult - matchAndRewrite(TransposeOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("TransposeBall"); - Location loc = op.getLoc(); - if (failed( - validateTranspose(op, op.getElemBits(), op.getIter(), bankDepth))) - return failure(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInputBankId(), - cstI64(rewriter, loc, 0), - adaptor.getOutputBankId(), adaptor.getIter()); - if (stable) { - rewriter.replaceOpWithNewOp(op, rs1, - adaptor.getElemBits()); - return success(); - } - rewriter.replaceOpWithNewOp( - op, rs1, adaptor.getElemBits(), - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("TRANSPOSE"))); - return success(); - } - -private: - bool stable = false; - int64_t bankDepth; -}; -struct BankTransposeLowering : public ConvertOpToLLVMPattern { - BankTransposeLowering(LLVMTypeConverter &converter, bool stable, - int64_t bankDepth) - : ConvertOpToLLVMPattern(converter), stable(stable), - bankDepth(bankDepth) {} - - LogicalResult - matchAndRewrite(BankTransposeOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("TransposeBall"); - Location loc = op.getLoc(); - if (failed( - validateTranspose(op, op.getElemBits(), op.getIter(), bankDepth))) - return failure(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInBank(), - cstI64(rewriter, loc, 0), adaptor.getOutBank(), - adaptor.getIter()); - if (stable) { - rewriter.replaceOpWithNewOp(op, rs1, - adaptor.getElemBits()); - return success(); - } - rewriter.replaceOpWithNewOp( - op, rs1, adaptor.getElemBits(), - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("TRANSPOSE"))); - return success(); - } - -private: - bool stable = false; - int64_t bankDepth; -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateTransposeBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth) { - patterns.add(converter, stable, bankDepth); - patterns.add(converter, stable, bankDepth); -} - -void configureTransposeBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable) { - if (stable) - target.addLegalOp(); - else - target.addIllegalOp(); - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/examples/balls/transpose/compiler/src/Dialect/Buckyball/TransposeBall.td b/examples/balls/transpose/compiler/src/Dialect/Buckyball/TransposeBall.td deleted file mode 100644 index 0cf5f4898..000000000 --- a/examples/balls/transpose/compiler/src/Dialect/Buckyball/TransposeBall.td +++ /dev/null @@ -1,52 +0,0 @@ -//===-- TransposeBall.td - TransposeBall operations --------*- tablegen -*-===// -// -// TransposeBall operations for matrix transpose. -// -//===----------------------------------------------------------------------===// - -#ifndef TRANSPOSEBALL_OPS -#define TRANSPOSEBALL_OPS - -include "Buckyball.td" - -// Memref-level transpose (like smatmul_matmul). Chip bank-ssa decides -// pad / tile / Bank* composition; this op carries no bank policy. -def MemTransposeOp : Buckyball_Op<"mem_transpose"> { - let summary = "Logical matrix transpose on memrefs"; - let description = [{ - Transposes input[R,C] into output[C,R]. Element type and layout - constraints are enforced by the chip bank-ssa lowering. - }]; - let arguments = (ins MemRefRankOf<[AnyType], [2]>:$input, - MemRefRankOf<[AnyType], [2]>:$output); - let assemblyFormat = [{$input $output prop-dict attr-dict `:` - type($input) type($output)}]; -} - -def TransposeOp : Buckyball_Op<"transpose"> { - let summary = "Matrix transpose operation"; - let description = [{ - Transposes iter x W to W x iter in scratchpad banks. - ISA: bb_transpose(op1_bank, wr_bank, iter, elem_bits); funct7 - from Core ballISA. - W = (bankWidth * cols) / elem_bits. - }]; - let arguments = (ins I64:$inputBankId, I64:$outputBankId, - I64:$iter, I64:$elemBits); - let assemblyFormat = "$inputBankId `,` $outputBankId `,` " - "$iter `,` $elemBits prop-dict attr-dict `:` " - "type($inputBankId)"; -} - -def BankTransposeOp : Buckyball_Op<"bank_transpose"> { - let summary = "Transpose using virtual bank handles"; - let arguments = (ins I64:$inBank, I64:$outBank, I64:$iter, I64:$elemBits); - let results = (outs I64:$outBankOut); - let assemblyFormat = - "$inBank $outBank $iter $elemBits prop-dict attr-dict `:` type($inBank) type($outBank) type($iter) type($elemBits)"; -} - -def TransposeIntrOp : Buckyball_IntrOpBase<"transpose">, - Arguments<(ins LLVM_Type, LLVM_Type)>; - -#endif // TRANSPOSEBALL_OPS diff --git a/examples/balls/transpose/configs/TransposeBallParam.scala b/examples/balls/transpose/configs/TransposeBallParam.scala deleted file mode 100644 index 1bb46a731..000000000 --- a/examples/balls/transpose/configs/TransposeBallParam.scala +++ /dev/null @@ -1,21 +0,0 @@ -package examples.balls.transpose.configs - -import framework.balldomain.configs.BallParamLoader -import framework.top.GlobalConfig - -case class TransposeBallParam( - InputNum: Int, - inputWidth: Int) - -object TransposeBallParam { - private val ballName = "TransposeBall" - - def apply(b: GlobalConfig): TransposeBallParam = { - val tbl = BallParamLoader.ballTable(b, ballName) - TransposeBallParam( - InputNum = BallParamLoader.int(tbl, "InputNum"), - inputWidth = BallParamLoader.int(tbl, "inputWidth") - ) - } - -} diff --git a/examples/balls/transpose/configs/default.toml b/examples/balls/transpose/configs/default.toml deleted file mode 100644 index 5c957d025..000000000 --- a/examples/balls/transpose/configs/default.toml +++ /dev/null @@ -1,5 +0,0 @@ -# Transpose Ball Configuration - -[ball] -InputNum = 16 -inputWidth = 8 diff --git a/examples/balls/transpose/emu/src/dispatch.rs b/examples/balls/transpose/emu/src/dispatch.rs deleted file mode 100644 index 208951e45..000000000 --- a/examples/balls/transpose/emu/src/dispatch.rs +++ /dev/null @@ -1,31 +0,0 @@ -use crate::inst::instruction::ExecContext; - -use super::transpose; - -const BALL_CLASS: &str = "examples.balls.transpose.TransposeBall"; - -pub fn execute_known( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, - ctx: &mut ExecContext, -) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("TRANSPOSE") => Some(transpose::exec_transpose(xs1, xs2, ctx)), - Some(_) | None => None, - } -} - -pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("TRANSPOSE") => Some(transpose::latency(xs1, xs2)), - Some(_) | None => None, - } -} diff --git a/examples/balls/transpose/emu/src/lib.rs b/examples/balls/transpose/emu/src/lib.rs deleted file mode 100644 index 38f0fa485..000000000 --- a/examples/balls/transpose/emu/src/lib.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub(crate) use crate::inst::{decode, instruction}; - -mod transpose; - -mod dispatch; - -pub use dispatch::{cycles_after_issue, execute_known}; diff --git a/examples/balls/transpose/verify/README.md b/examples/balls/transpose/verify/README.md deleted file mode 100644 index abd6c90c9..000000000 --- a/examples/balls/transpose/verify/README.md +++ /dev/null @@ -1,91 +0,0 @@ -# TransposeBall UVM Verification - -Verifies the generated `TransposeBall` module directly using the shared Blink UVM framework. - -## Structure - -- `../../../../verify/uvm/src/ball/bb_uvm_pkg.sv`: common Blink UVM transaction items and base env -- `src/common/transpose_defs.svh`: DPI imports and constants -- `src/common/transpose_items.svh`: `transpose_cmd_item` with `load_rust_case` -- `src/seq/transpose_sequences.svh`: `transpose_basic_seq` driving one case per run -- `src/cov/transpose_cov.svh`: `elem_bits` {8,32} and `iter` {1,2,4,8,16} coverage -- `src/env/transpose_scoreboard.svh`: preloads mem model, compares writes vs DPI dst words -- `src/env/transpose_env.svh`: extends `bb_blink_env#(1,1)` with scoreboard and cov -- `src/tests/transpose_test.svh`: directed cases 0,1 then random 2..21 with fixed seed -- `src/pkg/transpose_pkg.sv`: package include entry -- `src/tb_top.sv`: instantiates `bb_blink_if#(1,1)` and `TransposeBall` -- `filelists/transpose_ball.f`: VCS/Verilator filelist - -## Test plan - -`+UVM_TESTNAME=transpose_ball_test` runs: - -- case 0: directed i8, iter=16 -- case 1: directed i32, iter=8 -- cases 2..21: random with fixed seed `0xCAFE_BABE`, elem_bits in {8,32}, iter in {1,2,4,8,16} - -Each case: reset, preload src words into `mem_model`, drive one command, wait for -`scb.done()` or timeout at 2000 cycles. Scoreboard compares observed writes -(data/addr/mask/bank/rob) with DPI `transpose_case_dst_word_*`. `transpose_cov` -requires both `elem_bits` bins and all `iter` bins hit across the whole test, else -`check_phase` fatal. - -## BID (required) - -`+BID=` is mandatory. Missing it fatals at test start. There is no default bid. - -| Config | Plusarg | -|--------|---------| -| `sims.verilator.BuckyballToyVerilatorConfig` | `+BID=2` | -| `sims.verilator.BuckyballPebbleVerilatorConfig` | `+BID=0` | - -```console -./simv +UVM_TESTNAME=transpose_ball_test +BID=2 -``` - -## Build - -Enter the shared UVM/VCS environment: - -```console -nix develop ../../../../verify -``` - -Build the DPI reference library: - -```console -cargo build --manifest-path casegen/Cargo.toml -``` - -Compile from this directory: - -```console -vcs -full64 -sverilog -timescale=1ns/1ps \ - $VCS_UVM_ARGS \ - -sv_lib casegen/target/debug/libtranspose_casegen \ - -f filelists/transpose_ball.f -``` - -Run: - -```console -./simv +UVM_TESTNAME=transpose_ball_test +BID=2 -``` - -Or via bbdev: - -```console -bbdev uvm --build '--ball=transpose' -bbdev uvm --run '--ball=transpose' -- '+BID=2' -``` - - -## Acceptance (2026-08-03) - -- [x] Layout matches Blink common + ball-local split; filelist uses `@UVM@` / `@RTL@` -- [x] casegen reuses transpose model; whole-case DPI API -- [x] directed i8/i32 scoreboard pass -- [x] deterministic random idx 2..21 pass -- [x] protocol + semantic cover enforced (`check_phase` fatal) -- [x] toy: `+BID=2` config `sims.verilator.BuckyballToyVerilatorConfig` — 22/22 -- [x] pebble: `+BID=0` config `sims.verilator.BuckyballPebbleVerilatorConfig` — 22/22 diff --git a/examples/balls/transpose/verify/casegen/src/casegen.rs b/examples/balls/transpose/verify/casegen/src/casegen.rs deleted file mode 100644 index 38018e500..000000000 --- a/examples/balls/transpose/verify/casegen/src/casegen.rs +++ /dev/null @@ -1,303 +0,0 @@ -use crate::model; - -pub const BANK_ROW_BYTES: usize = 16; -pub const MAX_ITER: usize = 16; -pub const MAX_WORDS: usize = MAX_ITER; - -#[repr(C)] -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct TransposeCmd { - pub bid: u32, - pub iter: u32, - pub op1_bank: u32, - pub wr_bank: u32, - pub op1_col: u32, - pub wr_col: u32, - pub rob_id: u32, - pub elem_bits: u32, - pub num_src_words: u32, - pub num_dst_words: u32, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct TransposeCase { - pub cmd: TransposeCmd, - pub src: Vec, - pub dst: Vec, -} - -impl TransposeCase { - pub fn src_word_lo(&self, word_index: usize) -> u64 { - word_lo(&self.src, word_index) - } - pub fn src_word_hi(&self, word_index: usize) -> u64 { - word_hi(&self.src, word_index) - } - pub fn dst_word_lo(&self, word_index: usize) -> u64 { - word_lo(&self.dst, word_index) - } - pub fn dst_word_hi(&self, word_index: usize) -> u64 { - word_hi(&self.dst, word_index) - } -} - -fn word_lo(data: &[u8], word_index: usize) -> u64 { - let off = word_index * BANK_ROW_BYTES; - let mut buf = [0u8; 8]; - buf.copy_from_slice(&data[off..off + 8]); - u64::from_le_bytes(buf) -} - -fn word_hi(data: &[u8], word_index: usize) -> u64 { - let off = word_index * BANK_ROW_BYTES + 8; - let mut buf = [0u8; 8]; - buf.copy_from_slice(&data[off..off + 8]); - u64::from_le_bytes(buf) -} - -pub fn gen_case(seed: u32, index: u32, bid: u32) -> TransposeCase { - match index { - 0 => directed_i8(bid), - 1 => directed_i32(bid), - _ => random_case(seed, index, bid), - } -} - -fn directed_i8(bid: u32) -> TransposeCase { - let iter = 16usize; - let elem_bytes = 1usize; - let cols = 1usize; - let w = cols * (BANK_ROW_BYTES / elem_bytes); - let total = iter * w; - let src: Vec = (0..total).map(|i| i as u8).collect(); - let dst = model::transpose_bytes(&src, iter, w, elem_bytes); - let nwords = model::num_words(total * elem_bytes) as u32; - TransposeCase { - cmd: TransposeCmd { - bid, - iter: iter as u32, - op1_bank: 0, - wr_bank: 1, - op1_col: cols as u32, - wr_col: cols as u32, - rob_id: 3, - elem_bits: (elem_bytes * 8) as u32, - num_src_words: nwords, - num_dst_words: nwords, - }, - src, - dst, - } -} - -fn directed_i32(bid: u32) -> TransposeCase { - let iter = 8usize; - let elem_bytes = 4usize; - let cols = 1usize; - let w = cols * (BANK_ROW_BYTES / elem_bytes); - let total = iter * w; - let src: Vec = (0..total) - .map(|i| i as u32) - .flat_map(|v| v.to_le_bytes()) - .collect(); - let dst = model::transpose_bytes(&src, iter, w, elem_bytes); - let nwords = model::num_words(total * elem_bytes) as u32; - TransposeCase { - cmd: TransposeCmd { - bid, - iter: iter as u32, - op1_bank: 0, - wr_bank: 1, - op1_col: cols as u32, - wr_col: cols as u32, - rob_id: 5, - elem_bits: (elem_bytes * 8) as u32, - num_src_words: nwords, - num_dst_words: nwords, - }, - src, - dst, - } -} - -fn random_case(seed: u32, index: u32, bid: u32) -> TransposeCase { - let mut rng = Rng::new(seed, index); - let elem_bits = if rng.next() & 1 == 0 { 8u32 } else { 32u32 }; - let elem_bytes = (elem_bits / 8) as usize; - let iter_pool = [1u32, 2, 4, 8, 16]; - let iter = iter_pool[(rng.next() as usize) % iter_pool.len()] as usize; - let cols = 1usize; - let w = cols * (BANK_ROW_BYTES / elem_bytes); - - // Keep banks in 0..7 so the same cases work on configs with bankNum=8 or 16. - let op1_bank = rng.range(0, 7); - let mut wr_bank = rng.range(0, 7); - if wr_bank == op1_bank { - wr_bank = (wr_bank + 1) & 7; - } - - let total = iter * w; - let mut src = vec![0u8; total * elem_bytes]; - for b in src.iter_mut() { - *b = (rng.next() & 0xFF) as u8; - } - let dst = model::transpose_bytes(&src, iter, w, elem_bytes); - let nwords = model::num_words(total * elem_bytes) as u32; - - TransposeCase { - cmd: TransposeCmd { - bid, - iter: iter as u32, - op1_bank, - wr_bank, - op1_col: cols as u32, - wr_col: cols as u32, - rob_id: rng.range(0, 15), - elem_bits, - num_src_words: nwords, - num_dst_words: nwords, - }, - src, - dst, - } -} - -struct Rng { - state: u64, -} - -impl Rng { - fn new(seed: u32, index: u32) -> Self { - let state = (u64::from(seed) << 32) ^ u64::from(index) ^ 0x9E37_79B9_7F4A_7C15; - Self { state } - } - fn next(&mut self) -> u32 { - self.state ^= self.state >> 12; - self.state ^= self.state << 25; - self.state ^= self.state >> 27; - ((self.state.wrapping_mul(0x2545_F491_4F6C_DD1D)) >> 32) as u32 - } - fn range(&mut self, lo: u32, hi: u32) -> u32 { - lo + (self.next() % (hi - lo + 1)) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn model_transpose_i8_roundtrip() { - let iter = 16; - let w = 16; - let src: Vec = (0..iter * w).map(|i| i as u8).collect(); - let dst = model::transpose_bytes(&src, iter, w, 1); - for r in 0..iter { - for c in 0..w { - assert_eq!(dst[c * iter + r], src[r * w + c]); - } - } - } - - #[test] - fn model_transpose_i32_roundtrip() { - let iter = 8; - let w = 4; - let src: Vec = (0..iter * w) - .map(|i| i as u32) - .flat_map(|v| v.to_le_bytes()) - .collect(); - let dst = model::transpose_bytes(&src, iter, w, 4); - for r in 0..iter { - for c in 0..w { - let s = (r * w + c) * 4; - let d = (c * iter + r) * 4; - assert_eq!(&dst[d..d + 4], &src[s..s + 4]); - } - } - } - - #[test] - fn directed_i8_matches_ctest_shape() { - let case = gen_case(0x1234, 0, 0); - assert_eq!(case.cmd.bid, 0); - assert_eq!(case.cmd.iter, 16); - assert_eq!(case.cmd.elem_bits, 8); - assert_eq!(case.cmd.op1_bank, 0); - assert_eq!(case.cmd.wr_bank, 1); - assert_eq!(case.cmd.op1_col, 1); - assert_eq!(case.cmd.wr_col, 1); - assert_eq!(case.cmd.num_src_words, 16); - assert_eq!(case.cmd.num_dst_words, 16); - assert_eq!(case.src.len(), 256); - assert_eq!(case.dst.len(), 256); - assert_eq!(case.src[0], 0); - assert_eq!(case.src[255], 255); - for r in 0..16 { - for c in 0..16 { - assert_eq!(case.dst[c * 16 + r], case.src[r * 16 + c]); - } - } - } - - #[test] - fn directed_i32_matches_ctest_shape() { - let case = gen_case(0, 1, 0); - assert_eq!(case.cmd.iter, 8); - assert_eq!(case.cmd.elem_bits, 32); - assert_eq!(case.cmd.op1_bank, 0); - assert_eq!(case.cmd.wr_bank, 1); - assert_eq!(case.cmd.op1_col, 1); - assert_eq!(case.cmd.wr_col, 1); - assert_eq!(case.cmd.num_src_words, 8); - assert_eq!(case.cmd.num_dst_words, 8); - assert_eq!(case.src.len(), 128); - assert_eq!(case.dst.len(), 128); - } - - #[test] - fn random_cases_deterministic_and_legal() { - let a = gen_case(0xCAFE_BABE, 2, 0); - let b = gen_case(0xCAFE_BABE, 2, 0); - assert_eq!(a, b); - assert!(a.cmd.elem_bits == 8 || a.cmd.elem_bits == 32); - assert!([1u32, 2, 4, 8, 16].contains(&a.cmd.iter)); - assert_eq!(a.cmd.op1_col, 1); - assert_eq!(a.cmd.wr_col, 1); - assert_ne!(a.cmd.op1_bank, a.cmd.wr_bank); - assert!(a.cmd.op1_bank < 32); - assert!(a.cmd.wr_bank < 32); - assert!(a.cmd.rob_id < 16); - assert_eq!(a.cmd.num_src_words, a.cmd.num_dst_words); - assert_eq!(a.src.len(), a.dst.len()); - } - - #[test] - fn bid_is_required_arg() { - let case = gen_case(0, 0, 2); - assert_eq!(case.cmd.bid, 2); - let case = gen_case(0, 0, 5); - assert_eq!(case.cmd.bid, 5); - } - - #[test] - fn word_lo_hi_round_trip() { - let case = gen_case(0x1234, 0, 0); - for i in 0..case.cmd.num_src_words as usize { - let lo = case.src_word_lo(i); - let hi = case.src_word_hi(i); - let off = i * BANK_ROW_BYTES; - let expect_lo = u64::from_le_bytes(case.src[off..off + 8].try_into().unwrap()); - let expect_hi = u64::from_le_bytes(case.src[off + 8..off + 16].try_into().unwrap()); - assert_eq!(lo, expect_lo); - assert_eq!(hi, expect_hi); - } - } - - #[test] - #[should_panic] - fn word_lo_out_of_range_panics() { - let case = gen_case(0x1234, 0, 0); - let _ = case.src_word_lo(MAX_WORDS); - } -} diff --git a/examples/balls/transpose/verify/casegen/src/lib.rs b/examples/balls/transpose/verify/casegen/src/lib.rs deleted file mode 100644 index 7f09fb927..000000000 --- a/examples/balls/transpose/verify/casegen/src/lib.rs +++ /dev/null @@ -1,107 +0,0 @@ -mod casegen; -mod model; - -use std::cell::RefCell; - -use casegen::{TransposeCase, TransposeCmd, MAX_WORDS}; - -thread_local! { - static CURRENT: RefCell> = RefCell::new(None); -} - -#[no_mangle] -pub extern "C" fn transpose_case_load(seed: u32, index: u32, bid: u32) -> i32 { - let case = casegen::gen_case(seed, index, bid); - let nsrc = case.cmd.num_src_words as usize; - let ndst = case.cmd.num_dst_words as usize; - if nsrc > MAX_WORDS || ndst > MAX_WORDS { - panic!("transpose_case_load: word count out of range src={nsrc} dst={ndst}"); - } - CURRENT.with(|c| *c.borrow_mut() = Some(case)); - 0 -} - -fn current_case R, R>(f: F) -> R { - CURRENT.with(|c| match c.borrow().as_ref() { - Some(case) => f(case), - None => panic!("transpose_case: no case loaded; call transpose_case_load first"), - }) -} - -#[no_mangle] -pub extern "C" fn transpose_case_cmd(out_ptr: *mut TransposeCmd) { - current_case(|case| unsafe { - if out_ptr.is_null() { - panic!("transpose_case_cmd: null out_ptr"); - } - *out_ptr = case.cmd; - }); -} - -#[no_mangle] -pub extern "C" fn transpose_case_src_word_lo(word_index: u32) -> u64 { - if word_index as usize >= MAX_WORDS { - panic!("transpose_case_src_word_lo: word_index out of range"); - } - current_case(|case| case.src_word_lo(word_index as usize)) -} - -#[no_mangle] -pub extern "C" fn transpose_case_src_word_hi(word_index: u32) -> u64 { - if word_index as usize >= MAX_WORDS { - panic!("transpose_case_src_word_hi: word_index out of range"); - } - current_case(|case| case.src_word_hi(word_index as usize)) -} - -#[no_mangle] -pub extern "C" fn transpose_case_dst_word_lo(word_index: u32) -> u64 { - if word_index as usize >= MAX_WORDS { - panic!("transpose_case_dst_word_lo: word_index out of range"); - } - current_case(|case| case.dst_word_lo(word_index as usize)) -} - -#[no_mangle] -pub extern "C" fn transpose_case_dst_word_hi(word_index: u32) -> u64 { - if word_index as usize >= MAX_WORDS { - panic!("transpose_case_dst_word_hi: word_index out of range"); - } - current_case(|case| case.dst_word_hi(word_index as usize)) -} - -#[cfg(test)] -mod tests { - use super::*; - #[test] - fn dpi_load_then_cmd_and_words() { - assert_eq!(transpose_case_load(0x1234, 0, 0), 0); - let mut cmd = TransposeCmd { - bid: 0, - iter: 0, - op1_bank: 0, - wr_bank: 0, - op1_col: 0, - wr_col: 0, - rob_id: 0, - elem_bits: 0, - num_src_words: 0, - num_dst_words: 0, - }; - transpose_case_cmd(&mut cmd as *mut TransposeCmd); - assert_eq!(cmd.iter, 16); - assert_eq!(cmd.elem_bits, 8); - assert_eq!(cmd.num_src_words, 16); - let _lo = transpose_case_src_word_lo(0); - let _hi = transpose_case_src_word_hi(0); - let _dlo = transpose_case_dst_word_lo(0); - let _dhi = transpose_case_dst_word_hi(0); - } - - #[test] - #[should_panic(expected = "no case loaded")] - fn current_case_panics_without_load() { - CURRENT.with(|c| *c.borrow_mut() = None); - current_case(|_| ()); - } -} diff --git a/examples/balls/transpose/verify/src/common/transpose_defs.svh b/examples/balls/transpose/verify/src/common/transpose_defs.svh deleted file mode 100644 index 5b2313d78..000000000 --- a/examples/balls/transpose/verify/src/common/transpose_defs.svh +++ /dev/null @@ -1,39 +0,0 @@ -typedef struct { - int unsigned bid; - int unsigned iter; - int unsigned op1_bank; - int unsigned wr_bank; - int unsigned op1_col; - int unsigned wr_col; - int unsigned rob_id; - int unsigned elem_bits; - int unsigned num_src_words; - int unsigned num_dst_words; -} transpose_cmd_dpi_t; - -import "DPI-C" function int transpose_case_load( - input int unsigned seed, - input int unsigned index, - input int unsigned bid -); -import "DPI-C" function void transpose_case_cmd(output transpose_cmd_dpi_t cmd); -import "DPI-C" function longint unsigned transpose_case_src_word_lo(input int unsigned word_index); -import "DPI-C" function longint unsigned transpose_case_src_word_hi(input int unsigned word_index); -import "DPI-C" function longint unsigned transpose_case_dst_word_lo(input int unsigned word_index); -import "DPI-C" function longint unsigned transpose_case_dst_word_hi(input int unsigned word_index); - -`ifndef TRANSPOSE_FUNCT7 -`error "TRANSPOSE_FUNCT7 must be provided by the selected Core ballISA" -`endif -localparam int TRANSPOSE_CORE_FUNCT7 = `TRANSPOSE_FUNCT7; -localparam int TRANSPOSE_MAX_WORDS = 16; -localparam int TRANSPOSE_TIMEOUT_CYCLES = 2000; -localparam int TRANSPOSE_SEED = 32'hCAFE_BABE; - -function automatic int unsigned transpose_require_bid(); - int unsigned bid; - if (!$value$plusargs("BID=%d", bid)) begin - `uvm_fatal("BID", "missing required plusarg +BID=") - end - return bid; -endfunction diff --git a/examples/balls/transpose/verify/src/common/transpose_items.svh b/examples/balls/transpose/verify/src/common/transpose_items.svh deleted file mode 100644 index 19f26044e..000000000 --- a/examples/balls/transpose/verify/src/common/transpose_items.svh +++ /dev/null @@ -1,110 +0,0 @@ -class transpose_cmd_item extends bb_blink_cmd_item; - `uvm_object_utils(transpose_cmd_item) - - bit [31:0] elem_bits; - bit [31:0] num_src_words; - bit [31:0] num_dst_words; - bit [127:0] src_words[TRANSPOSE_MAX_WORDS]; - bit [127:0] dst_words[TRANSPOSE_MAX_WORDS]; - - constraint legal_c { - funct7 == TRANSPOSE_CORE_FUNCT7[6:0]; - op1_en == 1'b1; - op2_en == 1'b0; - wr_spad_en == 1'b1; - op1_from_spad == 1'b1; - op2_from_spad == 1'b0; - op2_bank == 5'd0; - op2_col == 5'd0; - meta_bank == 5'd0; - rs1 == 64'd0; - is_sub == 1'b0; - sub_rob_id == 8'h00; - special == 64'd0; - } - - function new(string name = "transpose_cmd_item"); - super.new(name); - endfunction - - function void load_rust_case(int unsigned seed, int unsigned index, int unsigned bid); - transpose_cmd_dpi_t cmd; - int unsigned rc; - longint unsigned w_lo; - longint unsigned w_hi; - int i; - - rc = transpose_case_load(seed, index, bid); - if (rc != 0) begin - `uvm_fatal("CASE", $sformatf("transpose_case_load returned %0d for index %0d", rc, index)) - end - transpose_case_cmd(cmd); - - bid = cmd.bid[4:0]; - funct7 = TRANSPOSE_CORE_FUNCT7[6:0]; - iter = cmd.iter; - op1_bank = cmd.op1_bank[4:0]; - wr_bank = cmd.wr_bank[4:0]; - op1_col = cmd.op1_col[4:0]; - wr_col = cmd.wr_col[4:0]; - rob_id = cmd.rob_id[3:0]; - elem_bits = cmd.elem_bits; - num_src_words = cmd.num_src_words; - num_dst_words = cmd.num_dst_words; - - rs2 = {56'd0, elem_bits[7:0]}; - op1_en = 1'b1; - op2_en = 1'b0; - wr_spad_en = 1'b1; - op1_from_spad = 1'b1; - op2_from_spad = 1'b0; - op2_bank = 5'd0; - op2_col = 5'd0; - meta_bank = 5'd0; - rs1 = 64'd0; - special = 64'd0; - is_sub = 1'b0; - sub_rob_id = 8'h00; - - if (num_src_words > TRANSPOSE_MAX_WORDS || num_dst_words > TRANSPOSE_MAX_WORDS) begin - `uvm_fatal("CASE", $sformatf("word count out of range src=%0d dst=%0d", num_src_words, - num_dst_words)) - end - if (num_src_words != num_dst_words) begin - `uvm_fatal("CASE", $sformatf("src/dst word count mismatch src=%0d dst=%0d", num_src_words, - num_dst_words)) - end - if (!(elem_bits == 32'd8 || elem_bits == 32'd32)) begin - `uvm_fatal("CASE", $sformatf("unsupported elem_bits=%0d", elem_bits)) - end - if (op1_bank == wr_bank) begin - `uvm_fatal("CASE", "op1_bank and wr_bank overlap") - end - - for (i = 0; i < num_src_words; i++) begin - w_lo = transpose_case_src_word_lo(i); - w_hi = transpose_case_src_word_hi(i); - src_words[i] = {w_hi, w_lo}; - end - for (i = 0; i < num_dst_words; i++) begin - w_lo = transpose_case_dst_word_lo(i); - w_hi = transpose_case_dst_word_hi(i); - dst_words[i] = {w_hi, w_lo}; - end - endfunction - - function void do_copy(uvm_object rhs); - transpose_cmd_item rhs_; - super.do_copy(rhs); - if (!$cast(rhs_, rhs)) begin - `uvm_fatal("COPY", "rhs is not transpose_cmd_item") - end - elem_bits = rhs_.elem_bits; - num_src_words = rhs_.num_src_words; - num_dst_words = rhs_.num_dst_words; - for (int i = 0; i < TRANSPOSE_MAX_WORDS; i++) begin - src_words[i] = rhs_.src_words[i]; - dst_words[i] = rhs_.dst_words[i]; - end - endfunction -endclass diff --git a/examples/balls/transpose/verify/src/tb_top.sv b/examples/balls/transpose/verify/src/tb_top.sv deleted file mode 100644 index 14deae43f..000000000 --- a/examples/balls/transpose/verify/src/tb_top.sv +++ /dev/null @@ -1,90 +0,0 @@ -module tb_top; - import uvm_pkg::*; - import transpose_pkg::*; - - bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); - - TransposeBall dut ( - .clock(intf.clock), - .reset(intf.reset), - .io_cmdReq_ready(intf.cmd_req_ready), - .io_cmdReq_valid(intf.cmd_req_valid), - .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), - .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), - .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), - .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), - .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), - .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), - .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), - .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), - .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), - .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank), - .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank), - .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank), - .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), - .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), - .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), - .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank), - .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), - .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), - .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), - .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), - .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), - .io_cmdResp_ready(intf.cmd_resp_ready), - .io_cmdResp_valid(intf.cmd_resp_valid), - .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), - .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), - .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), - .io_bankRead_0_bank_id(intf.bank_read_bank_id[0]), - .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), - .io_bankRead_0_group_id(intf.bank_read_group_id[0]), - .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), - .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), - .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), - .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), - .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), - .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), - .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), - .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), - .io_bankWrite_0_group_id(intf.bank_write_group_id[0]), - .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), - .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), - .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), - .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), - .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), - .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), - .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), - .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), - .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), - .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), - .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), - .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), - .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), - .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), - .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), - .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), - .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), - .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), - .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), - .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), - .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), - .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), - .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), - .io_subRobReq_ready(intf.sub_rob_req_ready), - .io_status_idle(), - .io_status_running(), - .io_channelReady(1'b1), - .io_bankRead_0_ball_id(), - .io_bankWrite_0_ball_id() - ); - - initial begin - intf.clock = 1'b0; - forever #5 intf.clock = ~intf.clock; - end - - initial begin - uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); - run_test("transpose_ball_test"); - end -endmodule diff --git a/examples/balls/transpose/workloads/ctests/CMakeLists.txt b/examples/balls/transpose/workloads/ctests/CMakeLists.txt deleted file mode 100644 index 9212a8e5b..000000000 --- a/examples/balls/transpose/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -add_buckyball_ctests( - transpose_i8_16x16_test.c - transpose_i8_16x32_test.c - transpose_i32_8x16_test.c -) diff --git a/examples/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8.mlir b/examples/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8.mlir deleted file mode 100644 index a03588615..000000000 --- a/examples/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8.mlir +++ /dev/null @@ -1,44 +0,0 @@ -// Already-lowered ball Op: buckyball.transpose (physical-bank form). - -func.func private @check_result(memref<16x16xi8>) -> () - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %c0 = arith.constant 0 : index - %c1 = arith.constant 1 : index - %n = arith.constant 16 : index - %c16_i32 = arith.constant 16 : i32 - %depth = arith.constant 16 : i64 - %stride = arith.constant 1 : i64 - %iter = arith.constant 16 : i64 - %elem_bits = arith.constant 8 : i64 - - %input = memref.alloc() alignment = 64 : memref<16x16xi8> - %output = memref.alloc() alignment = 64 : memref<16x16xi8> - - scf.for %i = %c0 to %n step %c1 { - scf.for %j = %c0 to %n step %c1 { - %ii = arith.index_cast %i : index to i32 - %jj = arith.index_cast %j : index to i32 - %mul = arith.muli %ii, %c16_i32 : i32 - %val = arith.addi %mul, %jj : i32 - %v8 = arith.trunci %val : i32 to i8 - memref.store %v8, %input[%i, %j] : memref<16x16xi8> - } - } - - %bin = buckyball.bank_alloc - %bout = buckyball.bank_alloc - %loaded = buckyball.bank_mvin %input %bin %depth %stride - : memref<16x16xi8> i64 i64 i64 - buckyball.transpose %loaded, %bout, %iter, %elem_bits : i64 - %stored = buckyball.bank_mvout %output %bout %depth %stride - : memref<16x16xi8> i64 i64 i64 - buckyball.bank_release %loaded : i64 - buckyball.bank_release %stored : i64 - - func.call @check_result(%output) : (memref<16x16xi8>) -> () - memref.dealloc %input : memref<16x16xi8> - memref.dealloc %output : memref<16x16xi8> - return %zero_i8 : i8 -} diff --git a/examples/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix.mlir.in b/examples/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix.mlir.in deleted file mode 100644 index d7c57a39d..000000000 --- a/examples/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix.mlir.in +++ /dev/null @@ -1,35 +0,0 @@ -func.func private @check_result(memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %rows = arith.constant @BB_BANK_LINES@ : index - %lanes = arith.constant @BB_BANK_I8_LANES@ : index - %depth = arith.constant @BB_BANK_LINES@ : i64 - %stride = arith.constant 1 : i64 - %elem_bits = arith.constant 8 : i64 - %lanes_i32 = arith.constant @BB_BANK_I8_LANES@ : i32 - %input = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - %output = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - scf.for %row = %zero to %rows step %one { - scf.for %lane = %zero to %lanes step %one { - %r = arith.index_cast %row : index to i32 - %l = arith.index_cast %lane : index to i32 - %base = arith.muli %r, %lanes_i32 : i32 - %index = arith.addi %base, %l : i32 - %value = arith.trunci %index : i32 to i8 - memref.store %value, %input[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - } - } - %input_bank = buckyball.bank_alloc - %output_bank = buckyball.bank_alloc - %loaded = buckyball.bank_mvin %input %input_bank %depth %stride : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> i64 i64 i64 - %result = buckyball.bank_transpose %loaded %output_bank %depth %elem_bits : i64 i64 i64 i64 - %stored = buckyball.bank_mvout %output %result %depth %stride : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> i64 i64 i64 - func.call @check_result(%output) : (memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () - buckyball.bank_release %loaded : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %input : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - memref.dealloc %output : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - return %zero_i8 : i8 -} diff --git a/examples/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix_main.cpp b/examples/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix_main.cpp deleted file mode 100644 index ef75bf830..000000000 --- a/examples/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix_main.cpp +++ /dev/null @@ -1,23 +0,0 @@ -#include -#include -#include -#include -extern "C" void check_result(int8_t *, int8_t *data, int64_t offset, - int64_t rows, int64_t lanes, int64_t stride, - int64_t lane_stride) { - if (rows != BANK_LINES || lanes != BANK_WIDTH / 8 || stride != lanes || - lane_stride != 1) - exit(1); - for (int row = 0; row < rows; ++row) - for (int lane = 0; lane < lanes; ++lane) { - int flat = row * lanes + lane; - int column = flat / BANK_LINES; - int source_row = flat % BANK_LINES; - int8_t expected = (int8_t)(source_row * lanes + column); - if (data[offset + flat] != expected) { - printf("transpose bank mismatch row=%d lane=%d\n", row, lane); - exit(1); - } - } - printf("transpose bank PASS rows=%d\n", BANK_LINES); -} diff --git a/examples/balls/vector/arch/src/main/scala/VecBall.scala b/examples/balls/vector/arch/src/main/scala/VecBall.scala deleted file mode 100644 index 7cf6bc243..000000000 --- a/examples/balls/vector/arch/src/main/scala/VecBall.scala +++ /dev/null @@ -1,53 +0,0 @@ -package examples.balls.vector - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.balldomain.blink.{BallStatus, BlinkIO, HasBallStatus, HasBlink, SubRobRow} -import framework.balldomain.blink.mmio.MmioRead -import examples.balls.vector.VecUnit -import framework.top.GlobalConfig - -/** - * VecBall - */ -@instantiable -class VecBall(val b: GlobalConfig) extends Module with HasBlink with HasBallStatus { - - val ballCommonConfig = b.ballDomain.ballIdMappings - .find(_.ballName == "VecBall") - .getOrElse( - throw new IllegalArgumentException("VecBall not found in config") - ) - - val inBW = ballCommonConfig.inBW - val outBW = ballCommonConfig.outBW - - @public - val io = IO(new BlinkIO(b, inBW, outBW)) - - def blink: BlinkIO = io - def status: BallStatus = io.status - dontTouch(io) - - val vecUnit: Instance[VecUnit] = Instantiate(new VecUnit(b)) - - vecUnit.io.cmdReq <> io.cmdReq - vecUnit.io.cmdResp <> io.cmdResp - - for (i <- 0 until inBW) { - vecUnit.io.bankRead(i) <> io.bankRead(i) - } - - for (i <- 0 until outBW) { - vecUnit.io.bankWrite(i) <> io.bankWrite(i) - } - - io.status <> vecUnit.io.status - - io.subRobReq.valid := false.B - io.subRobReq.bits := SubRobRow.tieOff(b) - - // MMIO: this Ball does not consume MMIO metadata - MmioRead.tieOff(io.mmioRead) -} diff --git a/examples/balls/vector/arch/src/main/scala/VecCtrlUnit.scala b/examples/balls/vector/arch/src/main/scala/VecCtrlUnit.scala deleted file mode 100644 index e43433b51..000000000 --- a/examples/balls/vector/arch/src/main/scala/VecCtrlUnit.scala +++ /dev/null @@ -1,131 +0,0 @@ -package examples.balls.vector - -import chisel3._ -import chisel3.util._ -import chisel3.stage._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.balldomain.rs.{BallRsComplete, BallRsIssue} -import framework.top.GlobalConfig - -@instantiable -class VecCtrlUnit(val b: GlobalConfig) extends Module { - - @public - val io = IO(new Bundle { - val cmdReq = Flipped(Decoupled(new BallRsIssue(b))) - val cmdResp_o = Decoupled(new BallRsComplete(b)) - - val ctrl_ld_o = Decoupled(new ctrl_ld_req(b)) - val ctrl_st_o = Decoupled(new ctrl_st_req(b)) - val ctrl_ex_o = Decoupled(new ctrl_ex_req(b)) - - val cmdResp_i = Flipped(Valid(new Bundle { - val commit = Bool() - })) // from store unit - }) - - val rob_id_reg = RegInit(0.U(log2Up(b.frontend.rob_entries).W)) - val is_sub_reg = RegInit(false.B) - val sub_rob_id_reg = RegInit(0.U(log2Up(b.frontend.sub_rob_depth * 4).W)) - val iter = RegInit(0.U(b.frontend.iter_len.W)) - val op1_bank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - - val op1_bank_addr = RegInit( - 0.U(12.W) - ) // New ISA: always 0, but keep for compatibility - val op2_bank_addr = RegInit( - 0.U(12.W) - ) // New ISA: always 0, but keep for compatibility - val op2_bank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - val wr_bank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - - val wr_bank_addr = RegInit( - 0.U(12.W) - ) // New ISA: always 0, but keep for compatibility - val has_send = RegInit(false.B) - val mode = RegInit(0.U(1.W)) - - val idle :: busy :: Nil = Enum(2) - val state = RegInit(idle) - -// ----------------------------------------------------------------------------- -// Set registers when EX instruction arrives -// ----------------------------------------------------------------------------- - - when(io.cmdReq.fire) { - iter := io.cmdReq.bits.cmd.iter - rob_id_reg := io.cmdReq.bits.rob_id - is_sub_reg := io.cmdReq.bits.is_sub - sub_rob_id_reg := io.cmdReq.bits.sub_rob_id - op1_bank := io.cmdReq.bits.cmd.op1_bank - op1_bank_addr := 0.U // New ISA: all operations start from row 0 - op2_bank := io.cmdReq.bits.cmd.op2_bank - op2_bank_addr := 0.U // New ISA: all operations start from row 0 - wr_bank := io.cmdReq.bits.cmd.wr_bank - wr_bank_addr := 0.U // New ISA: all operations start from row 0 - mode := io.cmdReq.bits.cmd.special(0) - - state := busy - } - -// ----------------------------------------------------------------------------- -// Send control signals to VecUnit's load/store/ex units -// ----------------------------------------------------------------------------- - - when(state === busy && !has_send) { - io.ctrl_ld_o.valid := true.B - io.ctrl_ld_o.bits.op1_bank := op1_bank - io.ctrl_ld_o.bits.op1_bank_addr := op1_bank_addr - io.ctrl_ld_o.bits.op2_bank := op2_bank - io.ctrl_ld_o.bits.op2_bank_addr := op2_bank_addr - io.ctrl_ld_o.bits.iter := iter - io.ctrl_ld_o.bits.mode := mode - - io.ctrl_ex_o.valid := true.B - io.ctrl_ex_o.bits.iter := iter - - io.ctrl_st_o.valid := true.B - io.ctrl_st_o.bits.wr_bank := wr_bank - io.ctrl_st_o.bits.wr_bank_addr := wr_bank_addr - io.ctrl_st_o.bits.iter := iter - - has_send := true.B - }.otherwise { - io.ctrl_ld_o.valid := false.B - io.ctrl_ld_o.bits.op1_bank := 0.U - io.ctrl_ld_o.bits.op1_bank_addr := 0.U - io.ctrl_ld_o.bits.op2_bank := 0.U - io.ctrl_ld_o.bits.op2_bank_addr := 0.U - io.ctrl_ld_o.bits.iter := 0.U - io.ctrl_ld_o.bits.mode := 0.U - - io.ctrl_ex_o.valid := false.B - io.ctrl_ex_o.bits.iter := 0.U - - io.ctrl_st_o.valid := false.B - io.ctrl_st_o.bits.wr_bank := 0.U - io.ctrl_st_o.bits.wr_bank_addr := 0.U - io.ctrl_st_o.bits.iter := 0.U - } - -// ----------------------------------------------------------------------------- -// Wait for VecUnit's final write-back to complete -// ----------------------------------------------------------------------------- - - when(io.cmdResp_i.valid) { - io.cmdResp_o.valid := true.B - io.cmdResp_o.bits.rob_id := rob_id_reg - io.cmdResp_o.bits.is_sub := is_sub_reg - io.cmdResp_o.bits.sub_rob_id := sub_rob_id_reg - state := idle - has_send := false.B - }.otherwise { - io.cmdResp_o.valid := false.B - io.cmdResp_o.bits.rob_id := 0.U - io.cmdResp_o.bits.is_sub := false.B - io.cmdResp_o.bits.sub_rob_id := 0.U - } - - io.cmdReq.ready := state === idle - -} diff --git a/examples/balls/vector/arch/src/main/scala/VecLoadUnit.scala b/examples/balls/vector/arch/src/main/scala/VecLoadUnit.scala deleted file mode 100644 index c343e9b89..000000000 --- a/examples/balls/vector/arch/src/main/scala/VecLoadUnit.scala +++ /dev/null @@ -1,178 +0,0 @@ -package examples.balls.vector - -import chisel3._ -import chisel3.util._ -import chisel3.stage._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.memdomain.backend.banks.{SramReadReq, SramReadResp} -import framework.top.GlobalConfig -import examples.balls.vector.configs.VectorBallParam - -class ctrl_ld_req(b: GlobalConfig) extends Bundle { - val op1_bank = UInt(b.memDomain.vbankIdWidth.W) - val op1_bank_addr = UInt(log2Up(b.memDomain.bankEntries).W) - val op2_bank = UInt(b.memDomain.vbankIdWidth.W) - val op2_bank_addr = UInt(log2Up(b.memDomain.bankEntries).W) - val iter = UInt(b.frontend.iter_len.W) - val mode = UInt(1.W) -} - -@instantiable -class VecLoadUnit(val b: GlobalConfig) extends Module { - val config = VectorBallParam(b) - val InputNum = config.lane - val inputWidth = config.inputWidth - val bankWidth = b.memDomain.bankWidth - val rob_id_width = log2Up(b.frontend.rob_entries) - - // Get bandwidth from config (use first VecBall mapping) - val ballMapping = b.ballDomain.ballIdMappings - .find(_.ballName == "VecBall") - .getOrElse( - throw new IllegalArgumentException("VecBall not found in config") - ) - - val inBW = ballMapping.inBW - - @public - val io = IO(new Bundle { - val bankReadReq = Vec(inBW, Decoupled(new SramReadReq(b))) - val bankReadResp = Vec(inBW, Flipped(Decoupled(new SramReadResp(b)))) - val ctrl_ld_i = Flipped(Decoupled(new ctrl_ld_req(b))) - val ld_ex_o = Decoupled(new ld_ex_req(b)) - val op1_bank_o = Output(UInt(b.memDomain.vbankIdWidth.W)) - val op2_bank_o = Output(UInt(b.memDomain.vbankIdWidth.W)) - }) - - val op1_bank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - val op2_bank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - val op1_addr = RegInit(0.U(log2Up(b.memDomain.bankEntries).W)) - val op2_addr = RegInit(0.U(log2Up(b.memDomain.bankEntries).W)) - val iter = RegInit(0.U(b.frontend.iter_len.W)) - val op1_iter_counter = RegInit(0.U(b.frontend.iter_len.W)) - val op2_iter_counter = RegInit(0.U(b.frontend.iter_len.W)) - val idle :: busy :: Nil = Enum(2) - val state = RegInit(idle) - val ld_ex_op1_reg = Reg(Vec(InputNum, UInt(inputWidth.W))) - val ld_ex_op2_reg = Reg(Vec(InputNum, UInt(inputWidth.W))) - val ld_ex_iter_reg = RegInit(0.U(b.frontend.iter_len.W)) - val wait1_reg = RegInit(false.B) - val wait2_reg = RegInit(false.B) - val wait1_cnt = RegInit(0.U(7.W)) - val wait2_cnt = RegInit(0.U(7.W)) - - val bankRespQueue0 = Module(new Queue(new SramReadResp(b), entries = 8)) - val bankRespQueue1 = Module(new Queue(new SramReadResp(b), entries = 8)) - - for (i <- 0 until inBW) { - io.bankReadReq(i).valid := false.B - io.bankReadReq(i).bits.addr := 0.U - } - - io.op1_bank_o := op1_bank - io.op2_bank_o := op2_bank - io.ctrl_ld_i.ready := state === idle - - bankRespQueue0.io.enq <> io.bankReadResp(0) - bankRespQueue1.io.enq <> io.bankReadResp(1) - -// ----------------------------------------------------------------------------- -// Set registers when Ctrl instruction arrives -// ----------------------------------------------------------------------------- - when(io.ctrl_ld_i.fire) { - op1_bank := io.ctrl_ld_i.bits.op1_bank - op2_bank := io.ctrl_ld_i.bits.op2_bank - op1_addr := io.ctrl_ld_i.bits.op1_bank_addr - op2_addr := io.ctrl_ld_i.bits.op2_bank_addr - iter := io.ctrl_ld_i.bits.iter - op1_iter_counter := 0.U - op2_iter_counter := 0.U - state := busy - assert(io.ctrl_ld_i.bits.iter > 0.U, "iter should be greater than 0") - } - -// ----------------------------------------------------------------------------- -// Send SRAM read request -// ----------------------------------------------------------------------------- - //wait1_reg := Mux(io.run , 0.U, wait1_reg) - - //wait2_reg := Mux(io.run , 0.U, wait2_reg) - - when(state === busy && io.ld_ex_o.ready && !wait1_reg) { - io.bankReadReq(0).valid := op1_iter_counter < iter - io.bankReadReq(0).bits.addr := op1_addr + op1_iter_counter - op1_iter_counter := Mux( - io.bankReadReq(0).ready, - op1_iter_counter + 1.U, - op1_iter_counter - ) - wait1_reg := Mux((op1_iter_counter + 1.U) % 16.U === 0.U, 1.U, 0.U) - when(io.bankReadReq(0).fire) {} - } - - when(state === busy && io.ld_ex_o.ready && !wait2_reg) { - io.bankReadReq(1).valid := op2_iter_counter < iter - io.bankReadReq(1).bits.addr := op2_addr + op2_iter_counter - op2_iter_counter := Mux( - io.bankReadReq(1).ready, - op2_iter_counter + 1.U, - op2_iter_counter - ) - wait2_reg := Mux((op2_iter_counter + 1.U) % 16.U === 0.U, 1.U, 0.U) - when(io.bankReadReq(1).fire) {} - } - - when(wait1_reg) { - wait1_cnt := wait1_cnt + 1.U - when(wait1_cnt === 32.U) { - wait1_reg := false.B - wait1_cnt := 0.U - } - } - - when(wait2_reg) { - wait2_cnt := wait2_cnt + 1.U - when(wait2_cnt === 32.U) { - wait2_reg := false.B - wait2_cnt := 0.U - } - } - -// ----------------------------------------------------------------------------- -// SRAM returns data and passes to EX unit -// ----------------------------------------------------------------------------- - val both_valid = bankRespQueue0.io.deq.valid && bankRespQueue1.io.deq.valid - - io.ld_ex_o.valid := both_valid - when(both_valid) { - io.ld_ex_o.bits.op1 := bankRespQueue0.io.deq.bits.data - .asTypeOf(Vec(InputNum, UInt(inputWidth.W))) - io.ld_ex_o.bits.op2 := bankRespQueue1.io.deq.bits.data - .asTypeOf(Vec(InputNum, UInt(inputWidth.W))) - io.ld_ex_o.bits.iter := ld_ex_iter_reg - }.otherwise { - io.ld_ex_o.bits.iter := 0.U - io.ld_ex_o.bits.op1 := VecInit(Seq.fill(InputNum)(0.U(inputWidth.W))) - io.ld_ex_o.bits.op2 := VecInit(Seq.fill(InputNum)(0.U(inputWidth.W))) - } - - // Only dequeue and advance iter counter on successful handshake - bankRespQueue0.io.deq.ready := io.ld_ex_o.fire - bankRespQueue1.io.deq.ready := io.ld_ex_o.fire - - when(io.ld_ex_o.fire) { - ld_ex_iter_reg := ld_ex_iter_reg + 1.U - } - -// ----------------------------------------------------------------------------- -// Reset op1_iter_counter and return to idle state -// ----------------------------------------------------------------------------- - - when(state === busy && ld_ex_iter_reg === iter) { - state := idle - op1_iter_counter := 0.U - op2_iter_counter := 0.U - ld_ex_iter_reg := 0.U - } - -} diff --git a/examples/balls/vector/arch/src/main/scala/VecMat16.scala b/examples/balls/vector/arch/src/main/scala/VecMat16.scala deleted file mode 100644 index 75c28c12e..000000000 --- a/examples/balls/vector/arch/src/main/scala/VecMat16.scala +++ /dev/null @@ -1,190 +0,0 @@ -package examples.balls.vector - -import chisel3._ -import chisel3.util._ -import framework.balldomain.blink.{BallStatus, BankRead, BankWrite} -import framework.balldomain.rs.{BallRsComplete, BallRsIssue} -import framework.top.GlobalConfig - -class VecMat16( - val b: GlobalConfig, - lane: Int, - inBW: Int, - outBW: Int, - inW: Int, - accW: Int) - extends Module { - require(lane == 16, "VecMat16 requires lane=16") - require(inW == 8, "VecMat16 requires int8 inputs") - require(accW == 32, "VecMat16 requires int32 accumulators") - require(inBW >= 2, "VecMat16 requires two read ports") - require(outBW >= 4, "VecMat16 requires four write ports") - - val io = IO(new Bundle { - val cmdReq = Flipped(Decoupled(new BallRsIssue(b))) - val cmdResp = Decoupled(new BallRsComplete(b)) - val bankRead = Vec(inBW, Flipped(new BankRead(b))) - val bankWrite = Vec(outBW, Flipped(new BankWrite(b))) - val status = new BallStatus - }) - - val sIdle :: sLoadAcc :: sReadReq :: sReadResp :: sWrite :: sDone :: Nil = - Enum(6) - val state = RegInit(sIdle) - - val robIdReg = RegInit(0.U(log2Up(b.frontend.rob_entries).W)) - val isSubReg = RegInit(false.B) - val subRobIdReg = RegInit(0.U(log2Up(b.frontend.sub_rob_depth * 4).W)) - val op1Bank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - val op2Bank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - val wrBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - val iterReg = RegInit(0.U(b.frontend.iter_len.W)) - val readRow = RegInit(0.U(b.frontend.iter_len.W)) - val aReqDone = RegInit(false.B) - val bReqDone = RegInit(false.B) - val aRespDone = RegInit(false.B) - val bRespDone = RegInit(false.B) - val aWord = RegInit(0.U(b.memDomain.bankWidth.W)) - val bWord = RegInit(0.U(b.memDomain.bankWidth.W)) - val acc = Reg(Vec(lane, Vec(lane, UInt(accW.W)))) - - val accLoad = Module(new VecMat16AccLoad(b, lane, accW)) - val store = Module(new VecMat16Store(b, lane, accW)) - accLoad.io.start := false.B - accLoad.io.wrBank := wrBank - accLoad.io.robId := robIdReg - store.io.start := false.B - store.io.wrBank := wrBank - store.io.robId := robIdReg - store.io.acc := acc - for (p <- 0 until 2) { - accLoad.io.bankRead(p).io.req.ready := io.bankRead(p).io.req.ready - accLoad.io.bankRead(p).io.resp.valid := io.bankRead(p).io.resp.valid - accLoad.io.bankRead(p).io.resp.bits := io.bankRead(p).io.resp.bits - } - for (g <- 0 until 4) { - store.io.bankWrite(g).io.req.ready := io.bankWrite(g).io.req.ready - store.io.bankWrite(g).io.resp.valid := io.bankWrite(g).io.resp.valid - store.io.bankWrite(g).io.resp.bits := io.bankWrite(g).io.resp.bits - } - - io.cmdReq.ready := state === sIdle - io.cmdResp.valid := state === sDone - io.cmdResp.bits.rob_id := robIdReg - io.cmdResp.bits.is_sub := isSubReg - io.cmdResp.bits.sub_rob_id := subRobIdReg - io.status.idle := state === sIdle - io.status.running := state =/= sIdle && state =/= sDone - VecMat16Tie.offRead(io.bankRead, robIdReg) - VecMat16Tie.offWrite(io.bankWrite, robIdReg, wrBank, b) - - switch(state) { - is(sIdle) { - when(io.cmdReq.fire) { - val cmd = io.cmdReq.bits.cmd - robIdReg := io.cmdReq.bits.rob_id - isSubReg := io.cmdReq.bits.is_sub - subRobIdReg := io.cmdReq.bits.sub_rob_id - op1Bank := cmd.op1_bank - op2Bank := cmd.op2_bank - wrBank := cmd.wr_bank - iterReg := cmd.iter - readRow := 0.U - aReqDone := false.B - bReqDone := false.B - aRespDone := false.B - bRespDone := false.B - assert(cmd.iter > 0.U, "VecMat16 iter must be > 0") - assert( - cmd.op1_col === 1.U && cmd.op2_col === 1.U && cmd.wr_col === 4.U, - "VecMat16 unsupported bank layout" - ) - accLoad.io.start := true.B - state := sLoadAcc - } - } - - is(sLoadAcc) { - for (p <- 0 until 2) { - VecMat16Tie.takeRead(io.bankRead(p), accLoad.io.bankRead(p)) - } - when(accLoad.io.done) { - acc := accLoad.io.acc - aReqDone := false.B - bReqDone := false.B - state := sReadReq - } - } - - is(sReadReq) { - io.bankRead(0).bank_id := op1Bank - io.bankRead(0).io.req.valid := !aReqDone - io.bankRead(0).io.req.bits.addr := readRow - io.bankRead(1).bank_id := op2Bank - io.bankRead(1).io.req.valid := !bReqDone - io.bankRead(1).io.req.bits.addr := readRow - when(io.bankRead(0).io.req.fire)(aReqDone := true.B) - when(io.bankRead(1).io.req.fire)(bReqDone := true.B) - when( - (aReqDone || io - .bankRead(0) - .io - .req - .fire) && (bReqDone || io.bankRead(1).io.req.fire) - ) { - aRespDone := false.B - bRespDone := false.B - state := sReadResp - } - } - - is(sReadResp) { - io.bankRead(0).bank_id := op1Bank - io.bankRead(1).bank_id := op2Bank - io.bankRead(0).io.resp.ready := !aRespDone - io.bankRead(1).io.resp.ready := !bRespDone - val aFire = io.bankRead(0).io.resp.fire - val bFire = io.bankRead(1).io.resp.fire - val nextA = Mux(aFire, io.bankRead(0).io.resp.bits.data, aWord) - val nextB = Mux(bFire, io.bankRead(1).io.resp.bits.data, bWord) - val haveA = aRespDone || aFire - val haveB = bRespDone || bFire - when(aFire)(aWord := io.bankRead(0).io.resp.bits.data) - when(bFire)(bWord := io.bankRead(1).io.resp.bits.data) - when(haveA)(aRespDone := true.B) - when(haveB)(bRespDone := true.B) - when(haveA && haveB) { - val aElems = nextA.asTypeOf(Vec(lane, UInt(inW.W))) - val bElems = nextB.asTypeOf(Vec(lane, UInt(inW.W))) - for { - i <- 0 until lane - j <- 0 until lane - } { - acc(i)(j) := (acc(i)(j).asSInt + aElems(i).asSInt * bElems( - j - ).asSInt).asUInt - } - when(readRow === iterReg - 1.U) { - store.io.start := true.B - state := sWrite - }.otherwise { - readRow := readRow + 1.U - aReqDone := false.B - bReqDone := false.B - state := sReadReq - } - } - } - - is(sWrite) { - for (g <- 0 until 4) { - VecMat16Tie.takeWrite(io.bankWrite(g), store.io.bankWrite(g)) - } - when(store.io.done)(state := sDone) - } - - is(sDone) { - when(io.cmdResp.fire)(state := sIdle) - } - } -} diff --git a/examples/balls/vector/arch/src/main/scala/VecMat16AccLoad.scala b/examples/balls/vector/arch/src/main/scala/VecMat16AccLoad.scala deleted file mode 100644 index 16c8224d9..000000000 --- a/examples/balls/vector/arch/src/main/scala/VecMat16AccLoad.scala +++ /dev/null @@ -1,120 +0,0 @@ -package examples.balls.vector - -import chisel3._ -import chisel3.util._ -import framework.balldomain.blink.BankRead -import framework.top.GlobalConfig - -class VecMat16AccLoad(val b: GlobalConfig, lane: Int, accWidth: Int) extends Module { - - val io = IO(new Bundle { - val start = Input(Bool()) - val wrBank = Input(UInt(b.memDomain.vbankIdWidth.W)) - val robId = Input(UInt(log2Up(b.frontend.rob_entries).W)) - val bankRead = Vec(2, Flipped(new BankRead(b))) - val acc = Output(Vec(lane, Vec(lane, UInt(accWidth.W)))) - val done = Output(Bool()) - val busy = Output(Bool()) - }) - - val sIdle :: sReq :: sResp :: sFinish :: Nil = Enum(4) - val state = RegInit(sIdle) - - val loadRow = RegInit(0.U(log2Ceil(lane + 1).W)) - val loadPhase = RegInit(0.U(1.W)) - val reqDone = RegInit(VecInit(Seq.fill(2)(false.B))) - val respDone = RegInit(VecInit(Seq.fill(2)(false.B))) - val acc = Reg(Vec(lane, Vec(lane, UInt(accWidth.W)))) - - def loadGroup(port: Int): UInt = Cat(loadPhase, port.U(1.W)) - def clr2() = VecInit(Seq.fill(2)(false.B)) - - io.acc := acc - io.done := false.B - io.busy := state =/= sIdle - - for (i <- 0 until 2) { - io.bankRead(i).rob_id := io.robId - io.bankRead(i).ball_id := 0.U - io.bankRead(i).bank_id := 0.U - io.bankRead(i).group_id := 0.U - io.bankRead(i).io.req.valid := false.B - io.bankRead(i).io.req.bits.addr := 0.U - io.bankRead(i).io.resp.ready := false.B - } - - switch(state) { - is(sIdle) { - when(io.start) { - loadRow := 0.U - loadPhase := 0.U - reqDone := clr2() - respDone := clr2() - state := sReq - } - } - - is(sReq) { - for (p <- 0 until 2) { - io.bankRead(p).bank_id := io.wrBank - io.bankRead(p).group_id := loadGroup(p) - io.bankRead(p).io.req.valid := !reqDone(p) - io.bankRead(p).io.req.bits.addr := loadRow - when(io.bankRead(p).io.req.fire) { - reqDone(p) := true.B - } - } - when( - (0 until 2) - .map(p => reqDone(p) || io.bankRead(p).io.req.fire) - .reduce(_ && _) - ) { - respDone := clr2() - state := sResp - } - } - - is(sResp) { - for (p <- 0 until 2) { - io.bankRead(p).bank_id := io.wrBank - io.bankRead(p).group_id := loadGroup(p) - io.bankRead(p).io.resp.ready := !respDone(p) - when(io.bankRead(p).io.resp.fire) { - val g = loadGroup(p) - val elems = - io.bankRead(p).io.resp.bits.data.asTypeOf(Vec(4, UInt(accWidth.W))) - val row = loadRow(log2Ceil(lane) - 1, 0) - for (i <- 0 until 4) { - acc(row)(g * 4.U + i.U) := elems(i) - } - respDone(p) := true.B - } - } - when( - (0 until 2) - .map(p => respDone(p) || io.bankRead(p).io.resp.fire) - .reduce(_ && _) - ) { - when(loadPhase === 0.U) { - loadPhase := 1.U - reqDone := clr2() - respDone := clr2() - state := sReq - }.elsewhen(loadRow === (lane - 1).U) { - state := sFinish - }.otherwise { - loadRow := loadRow + 1.U - loadPhase := 0.U - reqDone := clr2() - respDone := clr2() - state := sReq - } - } - } - - is(sFinish) { - io.done := true.B - state := sIdle - } - } -} diff --git a/examples/balls/vector/arch/src/main/scala/VecMat16Store.scala b/examples/balls/vector/arch/src/main/scala/VecMat16Store.scala deleted file mode 100644 index 40f4e3614..000000000 --- a/examples/balls/vector/arch/src/main/scala/VecMat16Store.scala +++ /dev/null @@ -1,112 +0,0 @@ -package examples.balls.vector - -import chisel3._ -import chisel3.util._ -import framework.balldomain.blink.BankWrite -import framework.top.GlobalConfig - -class VecMat16Store(val b: GlobalConfig, lane: Int, accWidth: Int) extends Module { - - val io = IO(new Bundle { - val start = Input(Bool()) - val wrBank = Input(UInt(b.memDomain.vbankIdWidth.W)) - val robId = Input(UInt(log2Up(b.frontend.rob_entries).W)) - val acc = Input(Vec(lane, Vec(lane, UInt(accWidth.W)))) - val bankWrite = Vec(4, Flipped(new BankWrite(b))) - val done = Output(Bool()) - val busy = Output(Bool()) - }) - - val sIdle :: sReq :: sResp :: Nil = Enum(3) - val state = RegInit(sIdle) - - val writeRow = RegInit(0.U(log2Ceil(lane + 1).W)) - val writeIssued = RegInit(VecInit(Seq.fill(4)(false.B))) - val writeAcked = RegInit(VecInit(Seq.fill(4)(false.B))) - - def writeData(group: Int): UInt = { - val row = writeRow(log2Ceil(lane) - 1, 0) - val elems = (0 until 4).map(i => io.acc(row)(group * 4 + i)) - Cat(elems.reverse) - } - - def clr4() = VecInit(Seq.fill(4)(false.B)) - - io.done := false.B - io.busy := state =/= sIdle - - for (i <- 0 until 4) { - io.bankWrite(i).rob_id := io.robId - io.bankWrite(i).ball_id := 0.U - io.bankWrite(i).bank_id := io.wrBank - io.bankWrite(i).group_id := i.U - io.bankWrite(i).io.req.valid := false.B - io.bankWrite(i).io.req.bits.addr := 0.U - io.bankWrite(i).io.req.bits.data := 0.U - io.bankWrite(i).io.req.bits.mask := VecInit( - Seq.fill(b.memDomain.bankMaskLen)(false.B) - ) - io.bankWrite(i).io.resp.ready := false.B - } - - switch(state) { - is(sIdle) { - when(io.start) { - writeRow := 0.U - writeIssued := clr4() - writeAcked := clr4() - state := sReq - } - } - - is(sReq) { - for (g <- 0 until 4) { - io.bankWrite(g).bank_id := io.wrBank - io.bankWrite(g).group_id := g.U - io.bankWrite(g).io.req.valid := !writeIssued(g) - io.bankWrite(g).io.req.bits.addr := writeRow - io.bankWrite(g).io.req.bits.data := writeData(g) - io.bankWrite(g).io.req.bits.mask := VecInit( - Seq.fill(b.memDomain.bankMaskLen)(true.B) - ) - when(io.bankWrite(g).io.req.fire) { - writeIssued(g) := true.B - } - } - when( - (0 until 4) - .map(g => writeIssued(g) || io.bankWrite(g).io.req.fire) - .reduce(_ && _) - ) { - writeAcked := clr4() - state := sResp - } - } - - is(sResp) { - for (g <- 0 until 4) { - io.bankWrite(g).bank_id := io.wrBank - io.bankWrite(g).group_id := g.U - io.bankWrite(g).io.resp.ready := !writeAcked(g) - when(io.bankWrite(g).io.resp.fire) { - writeAcked(g) := true.B - } - } - when( - (0 until 4) - .map(g => writeAcked(g) || io.bankWrite(g).io.resp.fire) - .reduce(_ && _) - ) { - when(writeRow === (lane - 1).U) { - io.done := true.B - state := sIdle - }.otherwise { - writeRow := writeRow + 1.U - writeIssued := clr4() - writeAcked := clr4() - state := sReq - } - } - } - } -} diff --git a/examples/balls/vector/arch/src/main/scala/VecStoreUnit.scala b/examples/balls/vector/arch/src/main/scala/VecStoreUnit.scala deleted file mode 100644 index b38b1733d..000000000 --- a/examples/balls/vector/arch/src/main/scala/VecStoreUnit.scala +++ /dev/null @@ -1,147 +0,0 @@ -package examples.balls.vector - -import chisel3._ -import chisel3.util._ -import chisel3.stage._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.memdomain.backend.banks.SramWriteIO -import framework.top.GlobalConfig -import examples.balls.vector.configs.VectorBallParam - -class ctrl_st_req(b: GlobalConfig) extends Bundle { - val wr_bank = UInt(b.memDomain.vbankIdWidth.W) - val wr_bank_addr = UInt(log2Up(b.memDomain.bankEntries).W) - val iter = UInt(b.frontend.iter_len.W) -} - -class ex_st_req(b: GlobalConfig) extends Bundle { - val config = VectorBallParam(b) - val InputNum = config.lane - val accWidth = config.outputWidth - val rst = Vec(InputNum, UInt(accWidth.W)) - val iter = UInt(b.frontend.iter_len.W) -} - -class BankWriteEntry(b: GlobalConfig) extends Bundle { - val addr = UInt(log2Ceil(b.memDomain.bankEntries).W) - val data = UInt(b.memDomain.bankWidth.W) - val mask = Vec(b.memDomain.bankMaskLen, Bool()) -} - -@instantiable -class VecStoreUnit(val b: GlobalConfig) extends Module { - val config = VectorBallParam(b) - val InputNum = config.lane - val accWidth = config.outputWidth - - // Get bandwidth from config (use first VecBall mapping) - val ballMapping = b.ballDomain.ballIdMappings - .find(_.ballName == "VecBall") - .getOrElse( - throw new IllegalArgumentException("VecBall not found in config") - ) - - val outBW = ballMapping.outBW - - @public - val io = IO(new Bundle { - val ctrl_st_i = Flipped(Decoupled(new ctrl_st_req(b))) - val ex_st_i = Flipped(Decoupled(new ex_st_req(b))) - val bankWrite = Vec(outBW, Flipped(new SramWriteIO(b))) - val wr_bank_o = Output(UInt(b.memDomain.vbankIdWidth.W)) - val cmdResp_o = Valid(new Bundle { val commit = Bool() }) - }) - - val wr_bank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - val wr_bank_addr = RegInit(0.U(log2Up(b.memDomain.bankEntries).W)) - val iter = RegInit(0.U(b.frontend.iter_len.W)) - val iter_counter = RegInit(0.U(b.frontend.iter_len.W)) - val idle :: busy :: Nil = Enum(2) - val state = RegInit(idle) - - val writeQueues = VecInit( - Seq.fill(outBW)(Module(new Queue(new BankWriteEntry(b), 16)).io) - ) - -// ----------------------------------------------------------------------------- -// Set registers when Ctrl instruction arrives -// ----------------------------------------------------------------------------- - io.ctrl_st_i.ready := state === idle - - when(io.ctrl_st_i.fire) { - wr_bank := io.ctrl_st_i.bits.wr_bank - wr_bank_addr := io.ctrl_st_i.bits.wr_bank_addr - iter := (io.ctrl_st_i.bits.iter + 15.U) & (~15.U(b.frontend.iter_len.W)) - iter_counter := 0.U - state := busy - } - -// ----------------------------------------------------------------------------- -// Accept computation results from EX unit and push to write queues -// ----------------------------------------------------------------------------- - io.ex_st_i.ready := state === busy && writeQueues.forall(_.enq.ready) - - for (i <- 0 until outBW) { - writeQueues(i).enq.valid := false.B - writeQueues(i).enq.bits := DontCare - } - - when(io.ex_st_i.fire) { - for (i <- 0 until outBW) { - val elementsPerChannel = InputNum / outBW - val startIdx = i * elementsPerChannel - val endIdx = startIdx + elementsPerChannel - 1 - - val entry = Wire(new BankWriteEntry(b)) - entry.addr := wr_bank_addr + iter_counter(log2Ceil(InputNum) - 1, 0) - entry.data := Cat(io.ex_st_i.bits.rst.slice(startIdx, endIdx + 1).reverse) - entry.mask := VecInit(Seq.fill(b.memDomain.bankMaskLen)(true.B)) - - writeQueues(i).enq.valid := true.B - writeQueues(i).enq.bits := entry - } - iter_counter := iter_counter + 1.U - } - -// ----------------------------------------------------------------------------- -// Drain write queues to bankWrite interface -// ----------------------------------------------------------------------------- - io.bankWrite.foreach { acc => - acc.req.valid := false.B - acc.req.bits.addr := 0.U - acc.req.bits.data := Cat(Seq.fill(accWidth / 8)(0.U(8.W))) - acc.req.bits.mask := VecInit(Seq.fill(b.memDomain.bankMaskLen)(false.B)) - acc.resp.ready := false.B - } - - for (i <- 0 until outBW) { - writeQueues(i).deq.ready := false.B - - when(writeQueues(i).deq.valid) { - io.bankWrite(i).req.valid := true.B - io.bankWrite(i).req.bits.addr := writeQueues(i).deq.bits.addr - io.bankWrite(i).req.bits.data := writeQueues(i).deq.bits.data - io.bankWrite(i).req.bits.mask := writeQueues(i).deq.bits.mask - writeQueues(i).deq.ready := io.bankWrite(i).req.ready - } - } - - // Output wr_bank for bank_id setting - io.wr_bank_o := wr_bank - -// ----------------------------------------------------------------------------- -// Reset iter counter, commit cmdResp, return to idle state -// ----------------------------------------------------------------------------- - val allQueuesEmpty = writeQueues.forall(q => !q.deq.valid) - val allDataEnqueued = state === busy && iter_counter >= iter - - when(allDataEnqueued && allQueuesEmpty) { - state := idle - io.cmdResp_o.valid := true.B - io.cmdResp_o.bits.commit := true.B - }.otherwise { - io.cmdResp_o.valid := false.B - io.cmdResp_o.bits.commit := false.B - } - -} diff --git a/examples/balls/vector/arch/src/main/scala/VecUnit.scala b/examples/balls/vector/arch/src/main/scala/VecUnit.scala deleted file mode 100644 index ebdb188ff..000000000 --- a/examples/balls/vector/arch/src/main/scala/VecUnit.scala +++ /dev/null @@ -1,49 +0,0 @@ -package examples.balls.vector - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} - -import framework.balldomain.blink.{BallStatus, BankRead, BankWrite} -import examples.balls.vector.configs.VectorBallParam -import framework.balldomain.rs.{BallRsComplete, BallRsIssue} -import framework.top.GlobalConfig - -@instantiable -class VecUnit(val b: GlobalConfig) extends Module { - val cfg = VectorBallParam(b) - val lane = cfg.lane - val inW = cfg.inputWidth - val accW = cfg.outputWidth - - require(lane == 16, "VecUnit vecmat16 requires lane=16") - require(inW == 8, "VecUnit vecmat16 requires int8 inputs") - require(accW == 32, "VecUnit vecmat16 requires int32 accumulators") - - val map = b.ballDomain.ballIdMappings - .find(_.ballName == "VecBall") - .getOrElse( - throw new IllegalArgumentException("VecBall not found in config") - ) - - val inBW = map.inBW - val outBW = map.outBW - require(inBW >= 2, "VecUnit vecmat16 requires two read ports") - require(outBW >= 4, "VecUnit vecmat16 requires four write ports") - - @public - val io = IO(new Bundle { - val cmdReq = Flipped(Decoupled(new BallRsIssue(b))) - val cmdResp = Decoupled(new BallRsComplete(b)) - val bankRead = Vec(inBW, Flipped(new BankRead(b))) - val bankWrite = Vec(outBW, Flipped(new BankWrite(b))) - val status = new BallStatus - }) - - val core = Module(new VecMat16(b, lane, inBW, outBW, inW, accW)) - core.io.cmdReq <> io.cmdReq - core.io.cmdResp <> io.cmdResp - for (i <- 0 until inBW) { core.io.bankRead(i) <> io.bankRead(i) } - for (i <- 0 until outBW) { core.io.bankWrite(i) <> io.bankWrite(i) } - io.status <> core.io.status -} diff --git a/examples/balls/vector/arch/src/main/scala/bond/vvv.scala b/examples/balls/vector/arch/src/main/scala/bond/vvv.scala deleted file mode 100644 index d63bb25ff..000000000 --- a/examples/balls/vector/arch/src/main/scala/bond/vvv.scala +++ /dev/null @@ -1,33 +0,0 @@ -//===----------------------------------------------------------------------===// -// VVV Bond: -// Input: Vec, Vec -// Output: Vec -//===----------------------------------------------------------------------===// - -package examples.balls.vector.bond - -import chisel3._ -import chisel3.util._ -import examples.balls.vector.configs.VectorBallParam -import chisel3.experimental.hierarchy.{instantiable, public} - -@instantiable -class VVV( - val config: VectorBallParam, - val inputWidth: Int, - val outputWidth: Int) - extends Bundle { - val lane = config.lane - - @public - val in = Flipped(Decoupled(new Bundle { - val in1 = Vec(lane, UInt(inputWidth.W)) - val in2 = Vec(lane, UInt(inputWidth.W)) - })) - - @public - val out = Decoupled(new Bundle { - val out = Vec(lane, UInt(outputWidth.W)) - }) - -} diff --git a/examples/balls/vector/arch/src/main/scala/thread/BaseThread.scala b/examples/balls/vector/arch/src/main/scala/thread/BaseThread.scala deleted file mode 100644 index f417786a1..000000000 --- a/examples/balls/vector/arch/src/main/scala/thread/BaseThread.scala +++ /dev/null @@ -1,13 +0,0 @@ -//===- BaseThread.scala - Level 1: Thread ---===// -package examples.balls.vector.thread - -import chisel3._ -import examples.balls.vector.configs.VectorBallParam - -//===----------------------------------------------------------------------===// -// BaseThread base class -//===----------------------------------------------------------------------===// -class BaseThread(val config: VectorBallParam, val opType: String) extends Module { - val io = IO(new Bundle {}) - val lane = config.lane -} diff --git a/examples/balls/vector/arch/src/main/scala/thread/CasThread.scala b/examples/balls/vector/arch/src/main/scala/thread/CasThread.scala deleted file mode 100644 index d2c4a6926..000000000 --- a/examples/balls/vector/arch/src/main/scala/thread/CasThread.scala +++ /dev/null @@ -1,19 +0,0 @@ -package examples.balls.vector.thread - -import chisel3._ -import chisel3.util._ -import examples.balls.vector.configs.VectorBallParam -import examples.balls.vector.bond.VVV -import examples.balls.vector.op.CascadeOp - -class CasThread(config: VectorBallParam) extends BaseThread(config, "cascade") { - val cascadeOp = Module(new CascadeOp(this.config)) - - val vvvBond = IO( - new VVV(this.config, this.config.outputWidth, this.config.outputWidth) - ) - - // Connect CascadeOp and VVVBond - cascadeOp.io.in <> vvvBond.in - cascadeOp.io.out <> vvvBond.out -} diff --git a/examples/balls/vector/arch/src/main/scala/thread/MulThread.scala b/examples/balls/vector/arch/src/main/scala/thread/MulThread.scala deleted file mode 100644 index 3ef6dd26f..000000000 --- a/examples/balls/vector/arch/src/main/scala/thread/MulThread.scala +++ /dev/null @@ -1,19 +0,0 @@ -package examples.balls.vector.thread - -import chisel3._ -import chisel3.util._ -import examples.balls.vector.configs.VectorBallParam -import examples.balls.vector.bond.VVV -import examples.balls.vector.op.MulOp - -class MulThread(config: VectorBallParam) extends BaseThread(config, "mul") { - val mulOp = Module(new MulOp(this.config)) - - val vvvBond = IO( - new VVV(this.config, this.config.inputWidth, this.config.outputWidth) - ) - - // Connect MulOp and VVVBond - mulOp.io.in <> vvvBond.in - mulOp.io.out <> vvvBond.out -} diff --git a/examples/balls/vector/compiler/src/Backend/include/IntrinsicsRISCVVecBall.td b/examples/balls/vector/compiler/src/Backend/include/IntrinsicsRISCVVecBall.td deleted file mode 100644 index 5d4fe7537..000000000 --- a/examples/balls/vector/compiler/src/Backend/include/IntrinsicsRISCVVecBall.td +++ /dev/null @@ -1,4 +0,0 @@ -//===- IntrinsicsRISCVVecBall.td - VecBall intrinsics --------------------===// - -let TargetPrefix = "riscv" in -def int_riscv_bb_vecmat16 : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; diff --git a/examples/balls/vector/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp b/examples/balls/vector/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp deleted file mode 100644 index 42b048874..000000000 --- a/examples/balls/vector/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp +++ /dev/null @@ -1,38 +0,0 @@ -//===- AssignPhysicalBankPatterns.cpp - Vector bank assignment patterns ---===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace mlir::buddy { -void populateVecBallAssignPhysicalBankPatterns(RewritePatternSet &patterns, - PhysicalBankState &state); -} // namespace mlir::buddy - -namespace { - -class BankVecMat16Pattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(BankVecMat16Op op, - PatternRewriter &rewriter) const override { - rewriter.create(op.getLoc(), op.getOp1Bank(), op.getOp2Bank(), - op.getWrBank(), op.getIter(), op.getMode()); - rewriter.replaceOp(op, op.getWrBank()); - return success(); - } -}; - -} // namespace - -void mlir::buddy::populateVecBallAssignPhysicalBankPatterns( - RewritePatternSet &patterns, mlir::buddy::PhysicalBankState &state) { - (void)state; - patterns.add(patterns.getContext()); -} diff --git a/examples/balls/vector/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/balls/vector/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 7930ab3bd..000000000 --- a/examples/balls/vector/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,47 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace buddy::buckyball; -using namespace buddy::buckyball::legalize; - -namespace { -struct VecMat16Lowering : public ConvertOpToLLVMPattern { - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - - LogicalResult - matchAndRewrite(VecMat16Op op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("VecBall"); - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getOp1BankId(), - adaptor.getOp2BankId(), adaptor.getWrBankId(), - adaptor.getIter()); - rewriter.replaceOpWithNewOp( - op, rs1, adaptor.getMode(), - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("VECMAT16"))); - return success(); - } -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateVecBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &converter, - RewritePatternSet &patterns, - bool stable, int64_t) { - (void)stable; - patterns.add(converter); -} - -void configureVecBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable) { - (void)stable; - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/examples/balls/vector/compiler/src/Dialect/Buckyball/VecBall.td b/examples/balls/vector/compiler/src/Dialect/Buckyball/VecBall.td deleted file mode 100644 index b3fdec455..000000000 --- a/examples/balls/vector/compiler/src/Dialect/Buckyball/VecBall.td +++ /dev/null @@ -1,39 +0,0 @@ -//===-- VecBall.td - VecBall operations --------------------*- tablegen -*-===// -// -// VecBall operations for matrix multiplication using warp-16 matrix array. -// -//===----------------------------------------------------------------------===// - -#ifndef VECBALL_OPS -#define VECBALL_OPS - -include "Buckyball.td" - -//===----------------------------------------------------------------------===// -// VecBall operations -//===----------------------------------------------------------------------===// - -def VecMat16Op : Buckyball_Op<"vecmat16"> { - let summary = "Warp-16 matrix multiplication"; - let description = [{ - Performs matrix multiplication using 16x16 matrix array. - Corresponds to bb_vecmat16 instruction (funct7 from Core ballISA). - Belongs to VecBall (bid=0). - }]; - let arguments = (ins I64:$op1BankId, I64:$op2BankId, - I64:$wrBankId, I64:$iter, I64:$mode); - let assemblyFormat = "$op1BankId `,` $op2BankId `,` $wrBankId `,` " - "$iter `,` $mode attr-dict `:` " - "type($op1BankId)"; -} - -def BankVecMat16Op : Buckyball_Op<"bank_vecmat16"> { - let summary = "Warp-16 matmul using virtual bank handles"; - let arguments = (ins I64:$op1Bank, I64:$op2Bank, I64:$wrBank, - I64:$iter, I64:$mode); - let results = (outs I64:$wrBankOut); - let assemblyFormat = - "$op1Bank $op2Bank $wrBank $iter $mode attr-dict `:` type($op1Bank) type($op2Bank) type($wrBank) type($iter) type($mode)"; -} - -#endif // VECBALL_OPS diff --git a/examples/balls/vector/configs/VectorBallParam.scala b/examples/balls/vector/configs/VectorBallParam.scala deleted file mode 100644 index 0eaf9d325..000000000 --- a/examples/balls/vector/configs/VectorBallParam.scala +++ /dev/null @@ -1,32 +0,0 @@ -package examples.balls.vector.configs - -import framework.balldomain.configs.BallParamLoader -import framework.top.GlobalConfig -import upickle.default._ - -case class VectorBallParam( - InputNum: Int, - inputWidth: Int, - lane: Int, - outputWidth: Int, - numMulThreads: Int, - numCasThreads: Int) - -object VectorBallParam { - implicit val rw: ReadWriter[VectorBallParam] = macroRW - - private val ballName = "VecBall" - - def apply(b: GlobalConfig): VectorBallParam = { - val tbl = BallParamLoader.ballTable(b, ballName) - VectorBallParam( - InputNum = BallParamLoader.int(tbl, "InputNum"), - inputWidth = BallParamLoader.int(tbl, "inputWidth"), - lane = BallParamLoader.int(tbl, "lane"), - outputWidth = BallParamLoader.int(tbl, "outputWidth"), - numMulThreads = BallParamLoader.int(tbl, "numMulThreads"), - numCasThreads = BallParamLoader.int(tbl, "numCasThreads") - ) - } - -} diff --git a/examples/balls/vector/configs/default.toml b/examples/balls/vector/configs/default.toml deleted file mode 100644 index dd0404912..000000000 --- a/examples/balls/vector/configs/default.toml +++ /dev/null @@ -1,9 +0,0 @@ -# Vector Ball Configuration - -[ball] -InputNum = 16 -inputWidth = 8 -lane = 16 -outputWidth = 32 -numMulThreads = 16 -numCasThreads = 16 diff --git a/examples/balls/vector/emu/src/64_vecmat16.rs b/examples/balls/vector/emu/src/64_vecmat16.rs deleted file mode 100644 index a8bbda23d..000000000 --- a/examples/balls/vector/emu/src/64_vecmat16.rs +++ /dev/null @@ -1,63 +0,0 @@ -//===- 64_vecmat16.rs - MUL_WARP16 instruction ---------------------------===// - -use super::bank_matrix::{read_i32_nn_groups, read_i8_k_rows, write_i32_nn_groups}; -use super::decode::{pbank, pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; -use super::instruction::{BallInstruction, ExecContext}; - -const WARP_M: usize = 16; -const WARP_N: usize = 16; - -pub struct VecMat16; - -impl BallInstruction for VecMat16 { - fn exec(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { - let op1 = rs1_b0(xs1); - let op2 = rs1_b1(xs1); - let wr = rs1_b2(xs1); - let iter = rs1_iter(xs1); - let _ = xs2; - - let c1 = ctx.config(op1).cols; - let c2 = ctx.config(op2).cols; - let cw = ctx.config(wr).cols; - if c1 != 1 || c2 != 1 || cw != 4 { - panic!("vecmat16: unsupported bank layout op1_cols={c1} op2_cols={c2} wr_cols={cw}"); - } - - let p1 = pbank(ctx, op1); - let p2 = pbank(ctx, op2); - let pw: Vec<_> = (0..cw).map(|group| pbank_group(ctx, wr, group)).collect(); - let kin = iter as usize; - - if kin == 0 { - panic!("vecmat16: iter must be > 0"); - } - - let need = kin * WARP_N; - if need > ctx.banks[p1].len() || need > ctx.banks[p2].len() { - panic!("vecmat16: iter too large for bank"); - } - - let a_t = read_i8_k_rows(&ctx.banks, p1, kin, WARP_N); - let b = read_i8_k_rows(&ctx.banks, p2, kin, WARP_N); - let mut c = read_i32_nn_groups(&ctx.banks, &pw, 16); - - for i in 0..WARP_M { - for j in 0..WARP_N { - let mut acc = c[i][j]; - for t in 0..kin { - acc = acc.wrapping_add((a_t[t][i] as i32).wrapping_mul(b[t][j] as i32)); - } - c[i][j] = acc; - } - } - - write_i32_nn_groups(&mut ctx.banks, &pw, &c, 16); - 0 - } - - fn latency(xs1: u64, _xs2: u64) -> u64 { - let kin = rs1_iter(xs1).max(1); - kin.saturating_mul(16) - } -} diff --git a/examples/balls/vector/emu/src/dispatch.rs b/examples/balls/vector/emu/src/dispatch.rs deleted file mode 100644 index 4b4eb6896..000000000 --- a/examples/balls/vector/emu/src/dispatch.rs +++ /dev/null @@ -1,31 +0,0 @@ -use crate::inst::instruction::{BallInstruction, ExecContext}; - -use super::f64_vecmat16; - -const BALL_CLASS: &str = "examples.balls.vector.VecBall"; - -pub fn execute_known( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, - ctx: &mut ExecContext, -) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("VECMAT16") => Some(f64_vecmat16::VecMat16::exec(xs1, xs2, ctx)), - Some(_) | None => None, - } -} - -pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("VECMAT16") => Some(f64_vecmat16::VecMat16::latency(xs1, xs2)), - Some(_) | None => None, - } -} diff --git a/examples/balls/vector/emu/src/lib.rs b/examples/balls/vector/emu/src/lib.rs deleted file mode 100644 index cd0540104..000000000 --- a/examples/balls/vector/emu/src/lib.rs +++ /dev/null @@ -1,8 +0,0 @@ -pub(crate) use crate::inst::{bank_matrix, decode, instruction}; - -#[path = "64_vecmat16.rs"] -mod f64_vecmat16; - -mod dispatch; - -pub use dispatch::{cycles_after_issue, execute_known}; diff --git a/examples/balls/vector/workloads/ctests/CMakeLists.txt b/examples/balls/vector/workloads/ctests/CMakeLists.txt deleted file mode 100644 index f1659a019..000000000 --- a/examples/balls/vector/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -add_buckyball_ctests( - vecmat16_atranspose_16x16_test.c -) diff --git a/examples/balls/vector/workloads/isa/vecmat16.h b/examples/balls/vector/workloads/isa/vecmat16.h deleted file mode 100644 index 8c802d967..000000000 --- a/examples/balls/vector/workloads/isa/vecmat16.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef _BB_VECMAT16_H_ -#define _BB_VECMAT16_H_ - -#include -#include - -#define bb_vecmat16(op1_bank_id, op2_bank_id, wr_bank_id, iter, mode) \ - BUCKYBALL_INSTRUCTION_R_R((BB_BANK0(op1_bank_id) | BB_BANK1(op2_bank_id) | \ - BB_BANK2(wr_bank_id) | BB_ITER(iter)), \ - (FIELD(mode, 0, 63)), BB_FUNC7(VECMAT16)) - -#endif // _BB_VECMAT16_H_ diff --git a/examples/balls/vector/workloads/mlir_tests/CMakeLists.txt b/examples/balls/vector/workloads/mlir_tests/CMakeLists.txt deleted file mode 100644 index 548af8da4..000000000 --- a/examples/balls/vector/workloads/mlir_tests/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -set(_params "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_MLIR_ACTIVE_TARGET}/params.h") -file(STRINGS "${_params}" _num_define REGEX "^#define BANK_NUM ") -file(STRINGS "${_params}" _lines_define REGEX "^#define BANK_LINES ") -file(STRINGS "${_params}" _width_define REGEX "^#define BANK_WIDTH ") -string(REGEX REPLACE ".* " "" _num "${_num_define}") -string(REGEX REPLACE ".* " "" _lines "${_lines_define}") -string(REGEX REPLACE ".* " "" _width "${_width_define}") -if(NOT _width EQUAL 128 OR _lines LESS 16 OR _num LESS 6) - message(FATAL_ERROR - "VecBall MLIR tests require BANK_WIDTH=128, BANK_LINES>=16, and BANK_NUM>=6 " - "(got width=${_width}, lines=${_lines}, num=${_num})") -endif() - -add_subdirectory(ball) -add_subdirectory(bank) diff --git a/examples/balls/vector/workloads/mlir_tests/ball/CMakeLists.txt b/examples/balls/vector/workloads/mlir_tests/ball/CMakeLists.txt deleted file mode 100644 index c7d6655fa..000000000 --- a/examples/balls/vector/workloads/mlir_tests/ball/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) - set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) -endif() -set(BUCKYBALL_MLIR_TEST_PREFIX ball) -add_buckyball_mlir_test(vecmat16_atranspose_16x16 - TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/balls/vector/workloads/mlir_tests/ball/vecmat16_atranspose_16x16.mlir b/examples/balls/vector/workloads/mlir_tests/ball/vecmat16_atranspose_16x16.mlir deleted file mode 100644 index c966caf81..000000000 --- a/examples/balls/vector/workloads/mlir_tests/ball/vecmat16_atranspose_16x16.mlir +++ /dev/null @@ -1,57 +0,0 @@ -func.func private @check_result(memref<16x16xi32>) -> () -func.func @main() -> i8 { - %z8 = arith.constant 0 : i8 - %z32 = arith.constant 0 : i32 - %z = arith.constant 0 : index - %one = arith.constant 1 : index - %two = arith.constant 2 : index - %three = arith.constant 3 : index - %five = arith.constant 5 : index - %seven = arith.constant 7 : index - %dim = arith.constant 16 : index - %depth = arith.constant 16 : i64 - %stride = arith.constant 1 : i64 - %mode = arith.constant 0 : i64 - %at = memref.alloc() alignment = 64 : memref<16x16xi8> - %b = memref.alloc() alignment = 64 : memref<16x16xi8> - %c = memref.alloc() alignment = 64 : memref<16x16xi32> - linalg.fill ins(%z32 : i32) outs(%c : memref<16x16xi32>) - scf.for %row = %z to %dim step %one { - scf.for %col = %z to %dim step %one { - %r3 = arith.muli %row, %three : index - %c5 = arith.muli %col, %five : index - %asum = arith.addi %r3, %c5 : index - %arem = arith.remui %asum, %seven : index - %aval = arith.subi %arem, %three : index - %atv = arith.index_cast %aval : index to i8 - %r2 = arith.muli %row, %two : index - %c3 = arith.muli %col, %three : index - %bsum = arith.addi %r2, %c3 : index - %brem = arith.remui %bsum, %five : index - %bval = arith.subi %brem, %two : index - %bv = arith.index_cast %bval : index to i8 - memref.store %atv, %at[%row, %col] : memref<16x16xi8> - memref.store %bv, %b[%row, %col] : memref<16x16xi8> - } - } - %ab = buckyball.bank_alloc - %bb = buckyball.bank_alloc - %cb = buckyball.bank_alloc - %al = buckyball.bank_mvin %at %ab %depth %stride - : memref<16x16xi8> i64 i64 i64 - %bl = buckyball.bank_mvin %b %bb %depth %stride - : memref<16x16xi8> i64 i64 i64 - %cl = buckyball.bank_mvin %c %cb %depth %stride - : memref<16x16xi32> i64 i64 i64 - buckyball.vecmat16 %al, %bl, %cl, %depth, %mode : i64 - %stored = buckyball.bank_mvout %c %cl %depth %stride - : memref<16x16xi32> i64 i64 i64 - func.call @check_result(%c) : (memref<16x16xi32>) -> () - buckyball.bank_release %al : i64 - buckyball.bank_release %bl : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %at : memref<16x16xi8> - memref.dealloc %b : memref<16x16xi8> - memref.dealloc %c : memref<16x16xi32> - return %z8 : i8 -} diff --git a/examples/balls/vector/workloads/mlir_tests/ball/vecmat16_atranspose_16x16_main.cpp b/examples/balls/vector/workloads/mlir_tests/ball/vecmat16_atranspose_16x16_main.cpp deleted file mode 100644 index 5c5c1e9cb..000000000 --- a/examples/balls/vector/workloads/mlir_tests/ball/vecmat16_atranspose_16x16_main.cpp +++ /dev/null @@ -1,22 +0,0 @@ -#include -#include -#include - -extern "C" void check_result(int32_t *, int32_t *data, int64_t offset, - int64_t rows, int64_t lanes, int64_t stride, - int64_t lane_stride) { - if (rows != 16 || lanes != 16 || stride != 16 || lane_stride != 1) - exit(1); - for (int row = 0; row < 16; ++row) - for (int column = 0; column < 16; ++column) { - int32_t expected = 0; - for (int k = 0; k < 16; ++k) - expected += - ((3 * k + 5 * row) % 7 - 3) * ((2 * k + 3 * column) % 5 - 2); - if (data[offset + row * stride + column] != expected) { - printf("vecmat16 A-transpose mismatch row=%d column=%d\n", row, column); - exit(1); - } - } - printf("vecmat16 A-transpose 16x16 PASS\n"); -} diff --git a/examples/balls/vector/workloads/mlir_tests/bank/CMakeLists.txt b/examples/balls/vector/workloads/mlir_tests/bank/CMakeLists.txt deleted file mode 100644 index 441f2843b..000000000 --- a/examples/balls/vector/workloads/mlir_tests/bank/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) - set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) -endif() -set(BUCKYBALL_MLIR_TEST_PREFIX bank) -add_buckyball_mlir_test(vecmat16_atranspose_bank_depth - TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/balls/vector/workloads/mlir_tests/bank/vecmat16_atranspose_bank_depth.mlir.in b/examples/balls/vector/workloads/mlir_tests/bank/vecmat16_atranspose_bank_depth.mlir.in deleted file mode 100644 index d9f0c4e50..000000000 --- a/examples/balls/vector/workloads/mlir_tests/bank/vecmat16_atranspose_bank_depth.mlir.in +++ /dev/null @@ -1,62 +0,0 @@ -func.func private @check_result(memref<16x16xi32>) -> () -func.func @main() -> i8 { - %z8 = arith.constant 0 : i8 - %z32 = arith.constant 0 : i32 - %z = arith.constant 0 : index - %one = arith.constant 1 : index - %two = arith.constant 2 : index - %three = arith.constant 3 : index - %five = arith.constant 5 : index - %seven = arith.constant 7 : index - %dim = arith.constant 16 : index - %rows = arith.constant @BB_BANK_LINES@ : index - %depth = arith.constant @BB_BANK_LINES@ : i64 - %out_depth = arith.constant 16 : i64 - %stride = arith.constant 1 : i64 - %mode = arith.constant 0 : i64 - %at = memref.alloc() alignment = 64 : memref<@BB_BANK_LINES@x16xi8> - %b = memref.alloc() alignment = 64 : memref<@BB_BANK_LINES@x16xi8> - %c = memref.alloc() alignment = 64 : memref<16x16xi32> - linalg.fill ins(%z32 : i32) outs(%c : memref<16x16xi32>) - scf.for %k = %z to %rows step %one { - scf.for %lane = %z to %dim step %one { - %k3 = arith.muli %k, %three : index - %l5 = arith.muli %lane, %five : index - %asum = arith.addi %k3, %l5 : index - %arem = arith.remui %asum, %seven : index - %aval = arith.subi %arem, %three : index - %atv = arith.index_cast %aval : index to i8 - %k2 = arith.muli %k, %two : index - %l3 = arith.muli %lane, %three : index - %bsum = arith.addi %k2, %l3 : index - %brem = arith.remui %bsum, %five : index - %bval = arith.subi %brem, %two : index - %bv = arith.index_cast %bval : index to i8 - memref.store %atv, %at[%k, %lane] - : memref<@BB_BANK_LINES@x16xi8> - memref.store %bv, %b[%k, %lane] - : memref<@BB_BANK_LINES@x16xi8> - } - } - %ab = buckyball.bank_alloc - %bb = buckyball.bank_alloc - %cb = buckyball.bank_alloc - %al = buckyball.bank_mvin %at %ab %depth %stride - : memref<@BB_BANK_LINES@x16xi8> i64 i64 i64 - %bl = buckyball.bank_mvin %b %bb %depth %stride - : memref<@BB_BANK_LINES@x16xi8> i64 i64 i64 - %cl = buckyball.bank_mvin %c %cb %out_depth %stride - : memref<16x16xi32> i64 i64 i64 - %result = buckyball.bank_vecmat16 %al %bl %cl %depth %mode - : i64 i64 i64 i64 i64 - %stored = buckyball.bank_mvout %c %result %out_depth %stride - : memref<16x16xi32> i64 i64 i64 - func.call @check_result(%c) : (memref<16x16xi32>) -> () - buckyball.bank_release %al : i64 - buckyball.bank_release %bl : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %at : memref<@BB_BANK_LINES@x16xi8> - memref.dealloc %b : memref<@BB_BANK_LINES@x16xi8> - memref.dealloc %c : memref<16x16xi32> - return %z8 : i8 -} diff --git a/examples/balls/vector/workloads/mlir_tests/bank/vecmat16_atranspose_bank_depth_main.cpp b/examples/balls/vector/workloads/mlir_tests/bank/vecmat16_atranspose_bank_depth_main.cpp deleted file mode 100644 index c987cedfb..000000000 --- a/examples/balls/vector/workloads/mlir_tests/bank/vecmat16_atranspose_bank_depth_main.cpp +++ /dev/null @@ -1,23 +0,0 @@ -#include -#include -#include -#include - -extern "C" void check_result(int32_t *, int32_t *data, int64_t offset, - int64_t rows, int64_t lanes, int64_t stride, - int64_t lane_stride) { - if (rows != 16 || lanes != 16 || stride != 16 || lane_stride != 1) - exit(1); - for (int row = 0; row < 16; ++row) - for (int column = 0; column < 16; ++column) { - int32_t expected = 0; - for (int k = 0; k < BANK_LINES; ++k) - expected += - ((3 * k + 5 * row) % 7 - 3) * ((2 * k + 3 * column) % 5 - 2); - if (data[offset + row * stride + column] != expected) { - printf("vecmat16 bank-depth mismatch row=%d column=%d\n", row, column); - exit(1); - } - } - printf("vecmat16 A-transpose bank-depth PASS K=%d\n", BANK_LINES); -} diff --git a/examples/build.sc b/examples/build.sc new file mode 100644 index 000000000..519b74f3e --- /dev/null +++ b/examples/build.sc @@ -0,0 +1,41 @@ +import mill._ +import mill.scalalib._ +import $file.^.arch.src.main.scala.framework.{build => framework_build} +import $file.^.arch.thirdparty.{build => thirdparty_build} +import $file.prefab.balls.{build => ball_components} +import $file.prefab.threads.{build => thread_components} + +import $file.^.arch.src.main.scala.framework.acc.balldomain.src.main.scala.bbus.{build => bbus_build} +import $file.^.arch.src.main.scala.framework.core.tbus.{build => tbus_build} + +val examplesRoot: os.Path = millSourcePath + +object library extends SbtModule { + override def scalaVersion = "2.13.16" + + override def moduleDeps = Seq( + framework_build.library, + thirdparty_build.hardfloat, + thirdparty_build.midas_target_utils, + thirdparty_build.gemmini + ) + + override def ivyDeps = Agg( + ivy"org.chipsalliance::chisel:6.7.0", + ivy"com.lihaoyi::upickle:3.3.1", + ivy"com.lihaoyi::mainargs:0.5.0", + ivy"org.json4s::json4s-jackson:4.0.5", + ivy"org.scala-graph::graph-core:1.13.5" + ) + + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") + + override def generatedSources = T { + bbus_build.registry(ball_components.sources(), examplesRoot / "prefab" / "balls", T.dest) ++ + tbus_build.registry(thread_components.sources(), examplesRoot / "prefab" / "threads", T.dest) + } + + override def sources = + T.sources(ball_components.sources() ++ thread_components.sources()) +} diff --git a/examples/chips/goban/arch/src/main/scala/sims/p2e/TargetConfigs.scala b/examples/chips/goban/arch/src/main/scala/sims/p2e/TargetConfigs.scala deleted file mode 100644 index 29e647378..000000000 --- a/examples/chips/goban/arch/src/main/scala/sims/p2e/TargetConfigs.scala +++ /dev/null @@ -1,10 +0,0 @@ -package sims.p2e - -import chisel3.experimental.hierarchy.{Instance, Instantiate} -import framework.system.{System, SystemParams} - -/** Goban on the P2E board: Linux scheduler core 0 and four compute cores. */ -class GobanP2ETarget extends P2ETarget("../examples/chips/goban/configs/generated/chip.pb") { - override def instantiate(p: SystemParams): Instance[System] = - Instantiate(new framework.system.Device(p.toGlobalConfig)) -} diff --git a/examples/chips/goban/arch/src/main/scala/sims/verilator/TargetConfigs.scala b/examples/chips/goban/arch/src/main/scala/sims/verilator/TargetConfigs.scala deleted file mode 100644 index 5d1d7a79a..000000000 --- a/examples/chips/goban/arch/src/main/scala/sims/verilator/TargetConfigs.scala +++ /dev/null @@ -1,12 +0,0 @@ -package sims.verilator - -import chisel3.experimental.hierarchy.{Instance, Instantiate} -import framework.system.{System, SystemParams} - -import sims.soc.SystemTarget - -/** Goban on the explicit System: Linux scheduler core 0 and four compute cores. */ -class GobanVerilatorTarget extends SystemTarget("../examples/chips/goban/configs/generated/chip.pb") { - override def instantiate(p: SystemParams): Instance[System] = - Instantiate(new framework.system.Device(p.toGlobalConfig)) -} diff --git a/examples/chips/goban/configs/chip.toml b/examples/chips/goban/configs/chip.toml deleted file mode 100644 index 30313c6a3..000000000 --- a/examples/chips/goban/configs/chip.toml +++ /dev/null @@ -1,10 +0,0 @@ -[designs] -include = "designs/goban.toml" - -[sims] -verilator = "sims.verilator.GobanVerilatorTarget" -p2e = "sims.p2e.GobanP2ETarget" - -[uvm] -balls = ["smatmul", "toint8", "int2fp", "transpose"] -ips = [] diff --git a/examples/chips/goban/configs/designs/goban.toml b/examples/chips/goban/configs/designs/goban.toml deleted file mode 100644 index aaa885c8e..000000000 --- a/examples/chips/goban/configs/designs/goban.toml +++ /dev/null @@ -1,4 +0,0 @@ -# Goban: a main tile only. - -[main_tile] -include = "tiles/default.toml" diff --git a/examples/chips/goban/configs/designs/tiles/default.toml b/examples/chips/goban/configs/designs/tiles/default.toml deleted file mode 100644 index 08bfefebf..000000000 --- a/examples/chips/goban/configs/designs/tiles/default.toml +++ /dev/null @@ -1,46 +0,0 @@ -# Goban main tile: five Linux-visible harts, with Buckyball on harts 1 through 4. - -coreDataBytes = 64 -xLen = 64 -vaddrBits = 39 -paddrBits = 56 -pgIdxBits = 12 -pgLevels = 3 -nPMPs = 8 - -memBallChannelNum = 6 - -[sharedMem] -bankWidth = 128 -bankEntries = 1024 -enable = true -virtualBankCount = 64 -entries = 16384 -inputChannels = 28 -defaultGroupCount = 1 -memCycles = 10 - -[[cores]] -core_id = 0 -name = "scheduler" -include = "../../../../../cores/rocket/configs/default.toml" - -[[cores]] -core_id = 1 -name = "goban" -include = "../../../../../cores/goban/configs/default.toml" - -[[cores]] -core_id = 2 -name = "goban" -include = "../../../../../cores/goban/configs/default.toml" - -[[cores]] -core_id = 3 -name = "goban" -include = "../../../../../cores/goban/configs/default.toml" - -[[cores]] -core_id = 4 -name = "goban" -include = "../../../../../cores/goban/configs/default.toml" diff --git a/examples/chips/goban/kernel/workloads.toml b/examples/chips/goban/kernel/workloads.toml deleted file mode 100644 index 54401fcc3..000000000 --- a/examples/chips/goban/kernel/workloads.toml +++ /dev/null @@ -1,11 +0,0 @@ -[workloads] -tests = [ - "goban-goban-ctest-rvv_kernel-linux", - "goban-goban-ctest-rvv_silu-linux", - "goban-goban-ctest-rvv_swiglu-linux", - "goban-goban-ctest-rvv_snake-linux", - "goban-goban-ctest-rvv_mxfp8_quant-linux", - "goban-goban-ctest-rvv_matmul-linux", - "goban-goban-ctest-mvin_mvout_test-linux", - "goban-goban-ctest-mvin_mvout_stride_test-linux", -] diff --git a/examples/chips/goban/regression/batch/bemu/workloads-bare.toml b/examples/chips/goban/regression/batch/bemu/workloads-bare.toml deleted file mode 100644 index aef752407..000000000 --- a/examples/chips/goban/regression/batch/bemu/workloads-bare.toml +++ /dev/null @@ -1,16 +0,0 @@ -[workloads] -tests = [ - "goban-goban-ctest-rvv_kernel-baremetal", - "goban-goban-ctest-rvv_silu-baremetal", - "goban-goban-ctest-rvv_swiglu-baremetal", - "goban-goban-ctest-rvv_snake-baremetal", - "goban-goban-ctest-rvv_mxfp8_quant-baremetal", - "goban-goban-ctest-rvv_matmul-baremetal", - "goban-goban-ctest-mvin_mvout_test-baremetal", - "goban-goban-ctest-mvin_mvout_stride_test-baremetal", - "goban-goban-ctest-chip-mvover_rows-multicore-baremetal", - "goban-goban-ctest-core-transpose_smatmul_quant_pipeline-multicore-baremetal", - "goban-scheduler-soctest-clint_timer_interrupt-baremetal", - "goban-scheduler-soctest-clint_software_interrupt-baremetal", - "goban-scheduler-soctest-plic_register_access-baremetal", -] diff --git a/examples/chips/goban/regression/batch/bemu/workloads-linux.toml b/examples/chips/goban/regression/batch/bemu/workloads-linux.toml deleted file mode 100644 index d949632c3..000000000 --- a/examples/chips/goban/regression/batch/bemu/workloads-linux.toml +++ /dev/null @@ -1,4 +0,0 @@ -[workloads] -tests = [ - "fw_payload-h5-goban-linux.elf", -] diff --git a/examples/chips/goban/regression/batch/p2e/workloads-linux.toml b/examples/chips/goban/regression/batch/p2e/workloads-linux.toml deleted file mode 100644 index f63c2dc2f..000000000 --- a/examples/chips/goban/regression/batch/p2e/workloads-linux.toml +++ /dev/null @@ -1,4 +0,0 @@ -[workloads] -tests = [ - "fw_payload-h5-goban-linux.hex", -] diff --git a/examples/chips/goban/regression/batch/verilator/workloads-bare.toml b/examples/chips/goban/regression/batch/verilator/workloads-bare.toml deleted file mode 100644 index aef752407..000000000 --- a/examples/chips/goban/regression/batch/verilator/workloads-bare.toml +++ /dev/null @@ -1,16 +0,0 @@ -[workloads] -tests = [ - "goban-goban-ctest-rvv_kernel-baremetal", - "goban-goban-ctest-rvv_silu-baremetal", - "goban-goban-ctest-rvv_swiglu-baremetal", - "goban-goban-ctest-rvv_snake-baremetal", - "goban-goban-ctest-rvv_mxfp8_quant-baremetal", - "goban-goban-ctest-rvv_matmul-baremetal", - "goban-goban-ctest-mvin_mvout_test-baremetal", - "goban-goban-ctest-mvin_mvout_stride_test-baremetal", - "goban-goban-ctest-chip-mvover_rows-multicore-baremetal", - "goban-goban-ctest-core-transpose_smatmul_quant_pipeline-multicore-baremetal", - "goban-scheduler-soctest-clint_timer_interrupt-baremetal", - "goban-scheduler-soctest-clint_software_interrupt-baremetal", - "goban-scheduler-soctest-plic_register_access-baremetal", -] diff --git a/examples/chips/goban/workloads/CMakeLists.txt b/examples/chips/goban/workloads/CMakeLists.txt deleted file mode 100644 index 2b27cb711..000000000 --- a/examples/chips/goban/workloads/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -add_custom_target(chip-ctests-build) - -add_subdirectory(ctests) - -set(BUCKYBALL_CTEST_TARGET goban) -set(BUCKYBALL_CTEST_GROUP_TARGET chip-ctests-build) -add_subdirectory( - ${BUCKYBALL_REPO_ROOT}/examples/balls/smatmul/workloads/ctests - ${CMAKE_CURRENT_BINARY_DIR}/smatmul-ctests) - -add_custom_target(chip-workloads-build ALL - DEPENDS - chip-ctests-build - mlirtest-workloads-build - soctests-build - COMMENT "Building goban workloads" - VERBATIM) diff --git a/examples/chips/goban/workloads/ctests/CMakeLists.txt b/examples/chips/goban/workloads/ctests/CMakeLists.txt deleted file mode 100644 index 0a030fec3..000000000 --- a/examples/chips/goban/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -buckyball_ctest_name(MVOVER_NAME goban chip-mvover_rows) -set(MVOVER_ELF ${MVOVER_NAME}-multicore-baremetal) -add_custom_command( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${MVOVER_ELF} - COMMAND ${RISCV_GNU_TOOLCHAIN}/bin/riscv64-unknown-elf-g++ - -O2 -static -march=rv64gc -mcmodel=medany -fno-builtin-printf - -specs=nano.specs -specs=nosys.specs -nostartfiles -Wl,-T,${BBSIM_LD} - -I${WORKLOAD_LIB_DIR} -I${BUCKYBALL_REPO_ROOT}/stack/runtime/include - -I${WORKLOAD_LIB_DIR}/bbsw/baremetal/multicore - -I${BUCKYBALL_ISA_DIR}/chip -I${BUCKYBALL_ISA_DIR}/goban - -o ${MVOVER_ELF} ${WORKLOAD_LIB_DIR}/bbsw/baremetal/start.S - ${CMAKE_CURRENT_SOURCE_DIR}/mvover_rows.c - ${WORKLOAD_LIB_DIR}/buckyball.c ${BBHW_MEM_C} - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/mvover_rows.c - ${WORKLOAD_LIB_DIR}/bbsw/baremetal/start.S ${BBHW_MEM_C} - ${WORKLOAD_LIB_DIR}/bbhw/isa/13_mvover.c - ${BUCKYBALL_ISA_DIR}/goban/params.h ${BUCKYBALL_ISA_DIR}/chip/topology.h - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) -add_custom_target(${MVOVER_NAME}-ctest-build DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${MVOVER_ELF}) -add_dependencies(chip-ctests-build ${MVOVER_NAME}-ctest-build) diff --git a/examples/chips/goban/workloads/ctests/mvover_rows.c b/examples/chips/goban/workloads/ctests/mvover_rows.c deleted file mode 100644 index e0362484c..000000000 --- a/examples/chips/goban/workloads/ctests/mvover_rows.c +++ /dev/null @@ -1,60 +0,0 @@ -#include -#include -#include -#include -#include - -#define ROWS 8 -#define ROW_BYTES (BANK_WIDTH / 8) -#if BANK_LINES < ROWS -#error "mvover_rows requires eight rows per bank" -#endif -static uint8_t input[ROWS * ROW_BYTES] __attribute__((aligned(64))); -static uint8_t output[ROWS * ROW_BYTES] __attribute__((aligned(64))); -#if BB_SHARED_PHYSICAL_BANK_NUM < 1 || BB_SHARED_BANK_BASE >= VIRTUAL_BANK_NUM -#error "mvover_rows requires a shared bank for issuer/owner trace coverage" -#endif -static uint8_t shared_output[ROWS * ROW_BYTES] __attribute__((aligned(64))); -static unsigned source_ready, target_ready, moved, stored; -int main(void) { - core_id_t id = bb_get_core_id(); - if (id.tile != 0) - for (;;) - asm volatile("wfi"); - if (id.core == 1) { - for (unsigned i = 0; i < sizeof(input); ++i) - input[i] = (uint8_t)(i * 17 + 3); - bb_mem_alloc(1, 1, 1); - bb_mem_alloc(BB_SHARED_BANK_BASE, 1, 1); - bb_mvin((uintptr_t)input, 1, ROWS, 1); - // Hart1 issues the write; the shared storage belongs to this Tile's Core0. - bb_mvin((uintptr_t)input, BB_SHARED_BANK_BASE, ROWS, 1); - bb_mvout((uintptr_t)shared_output, BB_SHARED_BANK_BASE, ROWS, 1); - for (unsigned i = 0; i < sizeof(input); ++i) - if (shared_output[i] != input[i]) - return 1; - bb_mem_release(BB_SHARED_BANK_BASE); - __atomic_store_n(&source_ready, 1, __ATOMIC_RELEASE); - } else if (id.core == 2) { - bb_mem_alloc(8, 1, 1); - __atomic_store_n(&target_ready, 1, __ATOMIC_RELEASE); - while (!__atomic_load_n(&moved, __ATOMIC_ACQUIRE)) { - } - bb_mvout((uintptr_t)output, 8, ROWS, 1); - __atomic_store_n(&stored, 1, __ATOMIC_RELEASE); - } else if (id.core == 0) { - while (!__atomic_load_n(&source_ready, __ATOMIC_ACQUIRE) || - !__atomic_load_n(&target_ready, __ATOMIC_ACQUIRE)) { - } - bb_mvover(0, 1, 0, 1, 8, 0, ROWS); - __atomic_store_n(&moved, 1, __ATOMIC_RELEASE); - while (!__atomic_load_n(&stored, __ATOMIC_ACQUIRE)) { - } - for (unsigned i = 0; i < sizeof(input); ++i) - if (input[i] != output[i]) - return 1; - return 0; - } - for (;;) - asm volatile("wfi"); -} diff --git a/examples/chips/multi-rocket/arch/src/main/scala/sims/verilator/MultiRocketVerilatorTarget.scala b/examples/chips/multi-rocket/arch/src/main/scala/sims/verilator/MultiRocketVerilatorTarget.scala deleted file mode 100644 index 84a0fe21e..000000000 --- a/examples/chips/multi-rocket/arch/src/main/scala/sims/verilator/MultiRocketVerilatorTarget.scala +++ /dev/null @@ -1,11 +0,0 @@ -package sims.verilator - -import chisel3.experimental.hierarchy.{Instance, Instantiate} -import framework.system.{System, SystemParams} - -import sims.soc.SystemTarget - -class MultiRocketVerilatorTarget extends SystemTarget("../examples/chips/multi-rocket/configs/generated/chip.pb") { - override def instantiate(p: SystemParams): Instance[System] = - Instantiate(new framework.system.Device(p.toGlobalConfig)) -} diff --git a/examples/chips/multi-rocket/configs/chip.toml b/examples/chips/multi-rocket/configs/chip.toml deleted file mode 100644 index bbade4444..000000000 --- a/examples/chips/multi-rocket/configs/chip.toml +++ /dev/null @@ -1,9 +0,0 @@ -[designs] -include = "designs/multi-rocket.toml" - -[sims] -verilator = "sims.verilator.MultiRocketVerilatorTarget" - -[uvm] -balls = [] -ips = ["axis"] diff --git a/examples/chips/multi-rocket/configs/designs/multi-rocket.toml b/examples/chips/multi-rocket/configs/designs/multi-rocket.toml deleted file mode 100644 index 2eb85b68b..000000000 --- a/examples/chips/multi-rocket/configs/designs/multi-rocket.toml +++ /dev/null @@ -1,4 +0,0 @@ -# Multi-Rocket: a main tile only. - -[main_tile] -include = "tiles/32core.toml" diff --git a/examples/chips/multi-rocket/configs/designs/tiles/32core.toml b/examples/chips/multi-rocket/configs/designs/tiles/32core.toml deleted file mode 100644 index c55ecc9a4..000000000 --- a/examples/chips/multi-rocket/configs/designs/tiles/32core.toml +++ /dev/null @@ -1,23 +0,0 @@ -# multi-rocket Tile (32 Rocket-only cores) - -coreDataBytes = 64 -xLen = 64 -vaddrBits = 39 -paddrBits = 56 -pgIdxBits = 12 -pgLevels = 3 -nPMPs = 8 - -[sharedMem] -bankWidth = 128 -bankEntries = 16 -enable = false -entries = 16384 -inputChannels = 32 -defaultGroupCount = 1 -memCycles = 10 - -[coreTemplate] -core_ids = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31] -include = "../../../../../cores/rocket/configs/default.toml" -count = 32 diff --git a/examples/chips/multi-rocket/kernel/workloads.toml b/examples/chips/multi-rocket/kernel/workloads.toml deleted file mode 100644 index 12a9b1c7c..000000000 --- a/examples/chips/multi-rocket/kernel/workloads.toml +++ /dev/null @@ -1,4 +0,0 @@ -[workloads] -tests = [ - "multi-rocket-rocket-ctest-cpu_arithmetic-linux", -] diff --git a/examples/chips/multi-rocket/regression/batch/bemu/workloads-bare.toml b/examples/chips/multi-rocket/regression/batch/bemu/workloads-bare.toml deleted file mode 100644 index a11146aed..000000000 --- a/examples/chips/multi-rocket/regression/batch/bemu/workloads-bare.toml +++ /dev/null @@ -1,24 +0,0 @@ -[workloads] -tests = [ - "multi-rocket-rocket-ctest-check-embench_sglib-combined-baremetal", - "multi-rocket-rocket-ctest-check-embench_matmult-int-baremetal", - "multi-rocket-rocket-ctest-cpu_arithmetic-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_aha-mont64-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_crc32-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_edn-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_huffbench-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_matmult-int-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_minver-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_nbody-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_nettle-aes-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_nettle-sha256-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_nsichneu-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_picojpeg-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_qrduino-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_sglib-combined-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_slre-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_st-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_statemate-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_ud-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_wikisort-baremetal", -] diff --git a/examples/chips/multi-rocket/regression/batch/bemu/workloads-linux.toml b/examples/chips/multi-rocket/regression/batch/bemu/workloads-linux.toml deleted file mode 100644 index 42419fdd0..000000000 --- a/examples/chips/multi-rocket/regression/batch/bemu/workloads-linux.toml +++ /dev/null @@ -1,4 +0,0 @@ -[workloads] -tests = [ - "fw_payload-h32-multi-rocket-linux.elf", -] diff --git a/examples/chips/multi-rocket/regression/batch/p2e/workloads-bare.toml b/examples/chips/multi-rocket/regression/batch/p2e/workloads-bare.toml deleted file mode 100644 index eb3873881..000000000 --- a/examples/chips/multi-rocket/regression/batch/p2e/workloads-bare.toml +++ /dev/null @@ -1,21 +0,0 @@ -[workloads] -tests = [ - "multi-rocket-rocket-ctest-benchmark-embench_aha-mont64-baremetal.hex", - "multi-rocket-rocket-ctest-benchmark-embench_crc32-baremetal.hex", - "multi-rocket-rocket-ctest-benchmark-embench_edn-baremetal.hex", - "multi-rocket-rocket-ctest-benchmark-embench_huffbench-baremetal.hex", - "multi-rocket-rocket-ctest-benchmark-embench_matmult-int-baremetal.hex", - "multi-rocket-rocket-ctest-benchmark-embench_minver-baremetal.hex", - "multi-rocket-rocket-ctest-benchmark-embench_nbody-baremetal.hex", - "multi-rocket-rocket-ctest-benchmark-embench_nettle-aes-baremetal.hex", - "multi-rocket-rocket-ctest-benchmark-embench_nettle-sha256-baremetal.hex", - "multi-rocket-rocket-ctest-benchmark-embench_nsichneu-baremetal.hex", - "multi-rocket-rocket-ctest-benchmark-embench_picojpeg-baremetal.hex", - "multi-rocket-rocket-ctest-benchmark-embench_qrduino-baremetal.hex", - "multi-rocket-rocket-ctest-benchmark-embench_sglib-combined-baremetal.hex", - "multi-rocket-rocket-ctest-benchmark-embench_slre-baremetal.hex", - "multi-rocket-rocket-ctest-benchmark-embench_st-baremetal.hex", - "multi-rocket-rocket-ctest-benchmark-embench_statemate-baremetal.hex", - "multi-rocket-rocket-ctest-benchmark-embench_ud-baremetal.hex", - "multi-rocket-rocket-ctest-benchmark-embench_wikisort-baremetal.hex", -] diff --git a/examples/chips/multi-rocket/regression/batch/p2e/workloads-linux.toml b/examples/chips/multi-rocket/regression/batch/p2e/workloads-linux.toml deleted file mode 100644 index 4da6227ba..000000000 --- a/examples/chips/multi-rocket/regression/batch/p2e/workloads-linux.toml +++ /dev/null @@ -1,4 +0,0 @@ -[workloads] -tests = [ - "fw_payload-h32-multi-rocket-linux.hex", -] diff --git a/examples/chips/multi-rocket/regression/batch/verilator/workloads-bare.toml b/examples/chips/multi-rocket/regression/batch/verilator/workloads-bare.toml deleted file mode 100644 index 8dd5e23e6..000000000 --- a/examples/chips/multi-rocket/regression/batch/verilator/workloads-bare.toml +++ /dev/null @@ -1,18 +0,0 @@ -[workloads] -tests = [ - "multi-rocket-rocket-ctest-cpu_arithmetic-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_aha-mont64-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_crc32-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_edn-baremetal", - "multi-rocket-rocket-ctest-check-embench_matmult-int-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_minver-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_nbody-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_nettle-aes-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_nettle-sha256-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_nsichneu-baremetal", - "multi-rocket-rocket-ctest-check-embench_sglib-combined-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_slre-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_st-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_statemate-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_ud-baremetal", -] diff --git a/examples/chips/multi-rocket/workloads/CMakeLists.txt b/examples/chips/multi-rocket/workloads/CMakeLists.txt deleted file mode 100644 index d8327a6da..000000000 --- a/examples/chips/multi-rocket/workloads/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -add_custom_target(chip-ctests-build) -set(BUCKYBALL_CTEST_GROUP_TARGET chip-ctests-build) -set(BUCKYBALL_CTEST_TARGET rocket) -add_buckyball_ctest(cpu_arithmetic.c) - -add_custom_target(chip-workloads-build ALL - DEPENDS - chip-ctests-build - mlirtest-workloads-build - soctests-build - COMMENT "Building multi-rocket workloads" - VERBATIM) diff --git a/examples/chips/omni/arch/src/main/scala/omni_tile/OmniTile.scala b/examples/chips/omni/arch/src/main/scala/omni_tile/OmniTile.scala deleted file mode 100644 index 56a3a530b..000000000 --- a/examples/chips/omni/arch/src/main/scala/omni_tile/OmniTile.scala +++ /dev/null @@ -1,21 +0,0 @@ -package examples.chips.omni.omni_tile - -import chisel3.experimental.hierarchy.{instantiable, Instance, Instantiate} -import framework.top.GlobalConfig -import framework.system.GlobalConfigOps._ -import framework.system.core.CoreFactory -import framework.system.tile.{ComputeTilePlatform, TileConnection, TileEndpoint, TileFactory} - -@instantiable -class OmniTile(b: GlobalConfig) extends TileEndpoint(b.tileLinkParams, false) { - val p = b.tileParams - val platform = Instantiate(new ComputeTilePlatform(p)) - val controllerIndex = p.controller.get - val controller = CoreFactory.rocket(b.forCore(controllerIndex)) - val workers = p.cores.indices.filterNot(_ == controllerIndex).map(i => CoreFactory.ant(b.forCore(i))) - TileConnection.compute(platform, tlink, controller.clink, workers.map(_.clink)) -} - -object OmniTile extends TileFactory { - def instantiate(b: GlobalConfig): Instance[TileEndpoint] = Instantiate(new OmniTile(b)) -} diff --git a/examples/chips/omni/arch/src/main/scala/poly_tile/PolyTile.scala b/examples/chips/omni/arch/src/main/scala/poly_tile/PolyTile.scala deleted file mode 100644 index aff20b7e2..000000000 --- a/examples/chips/omni/arch/src/main/scala/poly_tile/PolyTile.scala +++ /dev/null @@ -1,21 +0,0 @@ -package examples.chips.omni.poly_tile - -import chisel3.experimental.hierarchy.{instantiable, Instance, Instantiate} -import framework.top.GlobalConfig -import framework.system.GlobalConfigOps._ -import framework.system.core.CoreFactory -import framework.system.tile.{ComputeTilePlatform, TileConnection, TileEndpoint, TileFactory} - -@instantiable -class PolyTile(b: GlobalConfig) extends TileEndpoint(b.tileLinkParams, false) { - val p = b.tileParams - val platform = Instantiate(new ComputeTilePlatform(p)) - val controllerIndex = p.controller.get - val controller = CoreFactory.rocket(b.forCore(controllerIndex)) - val workers = p.cores.indices.filterNot(_ == controllerIndex).map(i => CoreFactory.ant(b.forCore(i))) - TileConnection.compute(platform, tlink, controller.clink, workers.map(_.clink)) -} - -object PolyTile extends TileFactory { - def instantiate(b: GlobalConfig): Instance[TileEndpoint] = Instantiate(new PolyTile(b)) -} diff --git a/examples/chips/omni/arch/src/main/scala/sims/OmniVerilatorTarget.scala b/examples/chips/omni/arch/src/main/scala/sims/OmniVerilatorTarget.scala deleted file mode 100644 index 8a17738ca..000000000 --- a/examples/chips/omni/arch/src/main/scala/sims/OmniVerilatorTarget.scala +++ /dev/null @@ -1,9 +0,0 @@ -package sims.verilator - -import chisel3.experimental.hierarchy.{Instance, Instantiate} -import framework.system.{Device, System, SystemParams} -import sims.soc.SystemTarget - -class OmniVerilatorTarget extends SystemTarget("../examples/chips/omni/configs/generated/chip.pb") { - override def instantiate(p: SystemParams): Instance[System] = Instantiate(new Device(p.toGlobalConfig)) -} diff --git a/examples/chips/omni/configs/chip.toml b/examples/chips/omni/configs/chip.toml deleted file mode 100644 index ca92f2852..000000000 --- a/examples/chips/omni/configs/chip.toml +++ /dev/null @@ -1,9 +0,0 @@ -[designs] -include = "designs/omni.toml" - -[sims] -verilator = "sims.verilator.OmniVerilatorTarget" - -[uvm] -balls = ["smatmul", "toint8", "int2fp", "transpose"] -ips = [] diff --git a/examples/chips/omni/configs/designs/omni.toml b/examples/chips/omni/configs/designs/omni.toml deleted file mode 100644 index a57ce721c..000000000 --- a/examples/chips/omni/configs/designs/omni.toml +++ /dev/null @@ -1,11 +0,0 @@ -# Fixed main position, four Poly tiles and one Goban-operator Omni tile. -[main_tile] -include = "tiles/main.toml" - -[[tiles]] -include = "tiles/poly.toml" -count = 4 - -[[tiles]] -include = "tiles/omni.toml" -count = 1 diff --git a/examples/chips/omni/configs/designs/tiles/main.toml b/examples/chips/omni/configs/designs/tiles/main.toml deleted file mode 100644 index ba059a857..000000000 --- a/examples/chips/omni/configs/designs/tiles/main.toml +++ /dev/null @@ -1,23 +0,0 @@ -# Omni main tile: one Linux core. Add cores, Buckyball and sharedMem here as the -# main tile grows (for example cross-tile all-reduce). - -coreDataBytes = 64 -xLen = 64 -vaddrBits = 39 -paddrBits = 56 -pgIdxBits = 12 -pgLevels = 3 -nPMPs = 8 - -[sharedMem] -bankWidth = 128 -bankEntries = 16 -enable = false -entries = 16384 -inputChannels = 1 -defaultGroupCount = 1 - -[[cores]] -core_id = 0 -name = "main" -include = "../../../../../cores/rocket/configs/default.toml" diff --git a/examples/chips/omni/configs/designs/tiles/omni.toml b/examples/chips/omni/configs/designs/tiles/omni.toml deleted file mode 100644 index ca7c1c0e8..000000000 --- a/examples/chips/omni/configs/designs/tiles/omni.toml +++ /dev/null @@ -1,53 +0,0 @@ -controllerCore = 0 -factory = "examples.chips.omni.omni_tile.OmniTile" -# One system controller and four tile-local Ant contexts with Goban operators. - -coreDataBytes = 64 -xLen = 64 -vaddrBits = 39 -paddrBits = 56 -pgIdxBits = 12 -pgLevels = 3 -nPMPs = 8 - -memBallChannelNum = 6 - -[tss] -base = 0x20000 -bytes = 16384 -dataBits = 128 - -[sharedMem] -bankWidth = 128 -bankEntries = 1024 -enable = true -virtualBankCount = 64 -entries = 16384 -inputChannels = 28 -defaultGroupCount = 1 -memCycles = 10 - -[[cores]] -core_id = 0 -name = "controller" -include = "../../../../../cores/rocket/configs/default.toml" - -[[cores]] -core_id = 1 -name = "goban" -include = "../../../../../cores/goban/configs/ant.toml" - -[[cores]] -core_id = 2 -name = "goban" -include = "../../../../../cores/goban/configs/ant.toml" - -[[cores]] -core_id = 3 -name = "goban" -include = "../../../../../cores/goban/configs/ant.toml" - -[[cores]] -core_id = 4 -name = "goban" -include = "../../../../../cores/goban/configs/ant.toml" diff --git a/examples/chips/omni/configs/designs/tiles/poly.toml b/examples/chips/omni/configs/designs/tiles/poly.toml deleted file mode 100644 index 08c25906d..000000000 --- a/examples/chips/omni/configs/designs/tiles/poly.toml +++ /dev/null @@ -1,83 +0,0 @@ -controllerCore = 0 -factory = "examples.chips.omni.poly_tile.PolyTile" -# Omni Poly tile: controller core 0, two Attention cores and eight FFN cores. - -coreDataBytes = 64 -xLen = 64 -vaddrBits = 39 -paddrBits = 56 -pgIdxBits = 12 -pgLevels = 3 -nPMPs = 8 - -memBallChannelNum = 11 - -[tss] -base = 0x20000 -bytes = 16384 -dataBits = 128 - -[sharedMem] -bankWidth = 128 -bankEntries = 1024 -enable = true -virtualBankCount = 40 -entries = 16384 -inputChannels = 10 -defaultGroupCount = 1 -memCycles = 10 - -[[cores]] -core_id = 0 -name = "controller" -include = "../../../../../cores/rocket/configs/default.toml" - -[[cores]] -core_id = 1 -name = "attention" -include = "../../../../../cores/attention/configs/default.toml" - -[[cores]] -core_id = 2 -name = "attention" -include = "../../../../../cores/attention/configs/default.toml" - -[[cores]] -core_id = 3 -name = "ffn" -include = "../../../../../cores/ffn/configs/default.toml" - -[[cores]] -core_id = 4 -name = "ffn" -include = "../../../../../cores/ffn/configs/default.toml" - -[[cores]] -core_id = 5 -name = "ffn" -include = "../../../../../cores/ffn/configs/default.toml" - -[[cores]] -core_id = 6 -name = "ffn" -include = "../../../../../cores/ffn/configs/default.toml" - -[[cores]] -core_id = 7 -name = "ffn" -include = "../../../../../cores/ffn/configs/default.toml" - -[[cores]] -core_id = 8 -name = "ffn" -include = "../../../../../cores/ffn/configs/default.toml" - -[[cores]] -core_id = 9 -name = "ffn" -include = "../../../../../cores/ffn/configs/default.toml" - -[[cores]] -core_id = 10 -name = "ffn" -include = "../../../../../cores/ffn/configs/default.toml" diff --git a/examples/chips/omni/configs/system.toml b/examples/chips/omni/configs/system.toml deleted file mode 100644 index ec9dc02e4..000000000 --- a/examples/chips/omni/configs/system.toml +++ /dev/null @@ -1,44 +0,0 @@ -model = "Qwen/Qwen3-Omni-30B-A3B-Instruct" -revision = "26291f793822fb6be9555850f06dfe95f2d7e695" - -[thinker] -tensor_tiles = [0, 1, 2, 3] - -[media] -vision_tile = 0 -audio_tile = 1 -talker_tile = 2 -code2wav_tile = 3 - -[vm] -chip = 4 -enabled = true - -[[chips]] -id = 0 -role = "thinker" -layers = [0, 12] -memory_mib = 16384 - -[[chips]] -id = 1 -role = "thinker" -layers = [12, 24] -memory_mib = 16384 - -[[chips]] -id = 2 -role = "thinker" -layers = [24, 36] -memory_mib = 16384 - -[[chips]] -id = 3 -role = "thinker" -layers = [36, 48] -memory_mib = 16384 - -[[chips]] -id = 4 -role = "media" -memory_mib = 16384 diff --git a/examples/chips/omni/emu/src/main.rs b/examples/chips/omni/emu/src/main.rs deleted file mode 100644 index 13cc7de63..000000000 --- a/examples/chips/omni/emu/src/main.rs +++ /dev/null @@ -1,41 +0,0 @@ -use bebop_bemu::root::{system, tile::run::Args}; -use clap::{CommandFactory, FromArgMatches, Parser}; - -#[derive(Parser)] -struct SystemArgs { - #[command(flatten)] - core: Args, - #[arg(long, requires = "pk")] - host_io: bool, -} - -fn main() { - let config: toml::Value = - toml::from_str(include_str!("../../configs/system.toml")).expect("Omni system config"); - let chips = config["chips"].as_array().expect("Omni chips"); - let memory = chips[0]["memory_mib"].as_integer().expect("DDR capacity"); - for (id, chip) in chips.iter().enumerate() { - assert_eq!(chip["id"].as_integer(), Some(id as i64)); - assert_eq!(chip["memory_mib"].as_integer(), Some(memory)); - } - let memory = Box::leak(memory.to_string().into_boxed_str()); - let matches = SystemArgs::command() - .mut_arg("memory_mib", |arg| arg.default_value(&*memory)) - .get_matches(); - let args = SystemArgs::from_arg_matches(&matches).expect("Omni arguments"); - let vm_chip = if config["vm"]["enabled"].as_bool().expect("VM enabled") { - Some( - config["vm"]["chip"] - .as_integer() - .expect("VM chip") - .try_into() - .expect("VM chip range"), - ) - } else { - None - }; - if let Err(error) = system::run(args.core, chips.len(), vm_chip, args.host_io) { - eprintln!("error: {error}"); - std::process::exit(1); - } -} diff --git a/examples/chips/omni/kernel/workloads.toml b/examples/chips/omni/kernel/workloads.toml deleted file mode 100644 index 394c9a2a7..000000000 --- a/examples/chips/omni/kernel/workloads.toml +++ /dev/null @@ -1,8 +0,0 @@ -[workloads] -tests = [ - "omni-attention-ctest-chip-mailbox-linux", - "omni-attention-ctest-chip-process-linux", - "omni-ffn-mlirtest-chip-moe-linux", - "omni-ffn-mlirtest-chip-moe-batch-linux", - "omni-attention-mlirtest-chip-attention-linux", -] diff --git a/examples/chips/omni/regression/batch/bemu/workloads-bare.toml b/examples/chips/omni/regression/batch/bemu/workloads-bare.toml deleted file mode 100644 index 433443b17..000000000 --- a/examples/chips/omni/regression/batch/bemu/workloads-bare.toml +++ /dev/null @@ -1,6 +0,0 @@ -[workloads] -tests = [ - "omni-attention-ctest-chip-interconnect-baremetal", - "omni-attention-ctest-chip-reservation-baremetal", - "omni-attention-ctest-chip-backpressure-baremetal", -] diff --git a/examples/chips/omni/regression/batch/bemu/workloads-linux.toml b/examples/chips/omni/regression/batch/bemu/workloads-linux.toml deleted file mode 100644 index 18b570832..000000000 --- a/examples/chips/omni/regression/batch/bemu/workloads-linux.toml +++ /dev/null @@ -1,4 +0,0 @@ -[workloads] -tests = [ - "fw_payload-h5-omni-linux.elf", -] diff --git a/examples/chips/omni/regression/batch/p2e/workloads-linux.toml b/examples/chips/omni/regression/batch/p2e/workloads-linux.toml deleted file mode 100644 index d0e89cac8..000000000 --- a/examples/chips/omni/regression/batch/p2e/workloads-linux.toml +++ /dev/null @@ -1,4 +0,0 @@ -[workloads] -tests = [ - "fw_payload-h5-omni-linux.hex", -] diff --git a/examples/chips/omni/tools/weights.py b/examples/chips/omni/tools/weights.py deleted file mode 100644 index 8941cb669..000000000 --- a/examples/chips/omni/tools/weights.py +++ /dev/null @@ -1,200 +0,0 @@ -import argparse -import dataclasses -import json -import tomllib -from collections import defaultdict -from pathlib import Path - -from huggingface_hub import HfApi - -ROOT = Path(__file__).resolve().parents[1] - - -def plan(metadata, config): - chips = config["chips"] - stages = [chip for chip in chips if chip["role"] == "thinker"] - media = [chip for chip in chips if chip["role"] == "media"] - tiles = config["thinker"]["tensor_tiles"] - if not stages or len(media) != 1 or tiles != [0, 1, 2, 3]: - raise ValueError("Qwen3 Omni placement requires TP4 tiles and one media chip") - names = [ - name for info in metadata["files_metadata"].values() for name in info["tensors"] - ] - layers = ( - max( - int(name.split(".")[3]) - for name in names - if name.startswith("thinker.model.layers.") - ) - + 1 - ) - covered = [layer for chip in stages for layer in range(*chip["layers"])] - if covered != list(range(layers)): - raise ValueError("Thinker chips must cover every layer exactly once in order") - totals = [ - { - "chip": c["id"], - "capacity_bytes": c["memory_mib"] << 20, - "checkpoint_bytes": 0, - "mxfp8_k128_estimate_bytes": 0, - "mxfp8_k512_estimate_bytes": 0, - } - for c in chips - ] - components = defaultdict(int) - tensors = [] - for filename, info in sorted(metadata["files_metadata"].items()): - for name, tensor in sorted(info["tensors"].items()): - shape = tensor["shape"] - original_bytes = tensor["data_offsets"][1] - tensor["data_offsets"][0] - component = ".".join(name.split(".")[:2]) - components[component] += original_bytes - entry = { - "name": name, - "file": filename, - "dtype": tensor["dtype"], - "shape": shape, - "source_bytes": original_bytes, - "placements": [], - } - if name.startswith(("thinker.model.", "thinker.lm_head.")): - if name.startswith("thinker.model.layers."): - layer = int(name.split(".")[3]) - chip = next( - chip - for chip in stages - if chip["layers"][0] <= layer < chip["layers"][1] - ) - elif name == "thinker.model.embed_tokens.weight": - chip = stages[0] - elif name in ("thinker.model.norm.weight", "thinker.lm_head.weight"): - chip = stages[-1] - else: - raise ValueError(f"No layer placement for {name}") - selected = [(chip, tile, rank) for rank, tile in enumerate(tiles)] - if name.endswith( - ( - "q_proj.weight", - "k_proj.weight", - "v_proj.weight", - "gate_proj.weight", - "up_proj.weight", - "embed_tokens.weight", - "lm_head.weight", - ) - ): - axis = 0 - elif name.endswith(("o_proj.weight", "down_proj.weight")): - axis = 1 - elif len(shape) == 1 or name.endswith("mlp.gate.weight"): - axis = None - else: - raise ValueError(f"No partition rule for {name}: {shape}") - elif name.startswith( - ("thinker.audio_tower.", "thinker.visual.", "talker.", "code2wav.") - ): - if name.startswith("thinker.audio_tower."): - tile = config["media"]["audio_tile"] - elif name.startswith("thinker.visual."): - tile = config["media"]["vision_tile"] - elif name.startswith("talker."): - tile = config["media"]["talker_tile"] - else: - tile = config["media"]["code2wav_tile"] - selected, axis = [(media[0], tile, 0)], None - else: - raise ValueError(f"Unknown Qwen3 Omni component: {name}") - for chip, tile, rank in selected: - shard_shape = list(shape) - start = 0 - if axis is not None: - if shape[axis] % len(tiles): - raise ValueError(f"Cannot split {name} along axis {axis}") - shard_shape[axis] //= len(tiles) - start = rank * shard_shape[axis] - size = original_bytes if axis is None else original_bytes // len(tiles) - amounts = { - "checkpoint_bytes": size, - "mxfp8_k128_estimate_bytes": size, - "mxfp8_k512_estimate_bytes": size, - } - # Estimate only Thinker projection/expert matrices. Encoders, routing, - # embeddings, Talker and waveform decode retain checkpoint precision. - quantized = axis is not None and not name.endswith( - ("embed_tokens.weight", "lm_head.weight") - ) - if quantized: - if len(shard_shape) != 2: - raise ValueError(f"Expected linear matrix: {name}") - rows, width = shard_shape - for tile_k in (128, 512): - elements = ((rows + 15) // 16 * 16) * ( - (width + tile_k - 1) // tile_k * tile_k - ) - amounts[f"mxfp8_k{tile_k}_estimate_bytes"] = ( - elements + elements // 32 - ) - totals[chip["id"]]["checkpoint_bytes"] += size - for tile_k in (128, 512): - key = f"mxfp8_k{tile_k}_estimate_bytes" - totals[chip["id"]][key] += amounts[key] - entry["placements"].append( - { - "chip": chip["id"], - "tile": tile, - "axis": axis, - "start": start, - "shape": shard_shape, - **amounts, - } - ) - tensors.append(entry) - for chip in totals: - for kind in ("checkpoint", "mxfp8_k128_estimate", "mxfp8_k512_estimate"): - chip[f"{kind}_remaining_bytes"] = ( - chip["capacity_bytes"] - chip[f"{kind}_bytes"] - ) - return { - "model": config["model"], - "revision": config["revision"], - "source_bytes": sum(components.values()), - "components": dict(components), - "chips": totals, - "tensors": tensors, - } - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--metadata", type=Path) - parser.add_argument("--output", type=Path, required=True) - args = parser.parse_args() - with (ROOT / "configs/system.toml").open("rb") as source: - config = tomllib.load(source) - if args.metadata: - data = json.loads(args.metadata.read_text()) - if data["repo"] != config["model"] or data["revision"] != config["revision"]: - raise ValueError("Metadata does not match the pinned checkpoint") - metadata = data["metadata"] - else: - metadata = dataclasses.asdict( - HfApi().get_safetensors_metadata( - config["model"], revision=config["revision"] - ) - ) - result = plan(metadata, config) - args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_text(json.dumps(result, indent=2) + "\n") - print( - f"Checkpoint tensors: {len(result['tensors'])}, bytes: {result['source_bytes']:,}" - ) - for chip in result["chips"]: - print( - f"chip {chip['chip']}: checkpoint {chip['checkpoint_bytes'] / 2**30:.3f} GiB; " - f"MXFP8 K128 estimate {chip['mxfp8_k128_estimate_bytes'] / 2**30:.3f} GiB; " - f"K512 estimate {chip['mxfp8_k512_estimate_bytes'] / 2**30:.3f} GiB" - ) - - -if __name__ == "__main__": - main() diff --git a/examples/chips/omni/workloads/CMakeLists.txt b/examples/chips/omni/workloads/CMakeLists.txt deleted file mode 100644 index d61bb11d1..000000000 --- a/examples/chips/omni/workloads/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -add_subdirectory(ctests) -add_custom_target(chip-ctests-build DEPENDS omni-interconnect-build) -add_custom_target(chip-workloads-build ALL - DEPENDS - chip-ctests-build - mlirtest-workloads-build - soctests-build -) diff --git a/examples/chips/omni/workloads/ctests/CMakeLists.txt b/examples/chips/omni/workloads/ctests/CMakeLists.txt deleted file mode 100644 index 279ac2917..000000000 --- a/examples/chips/omni/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,52 +0,0 @@ -set(STARTUP ${WORKLOAD_LIB_DIR}/bbsw/baremetal/crt0.S) -set(SYSCALLS ${WORKLOAD_LIB_DIR}/bbsw/baremetal/syscalls.c) -set(LINKER_SCRIPT ${WORKLOAD_LIB_DIR}/bbsw/baremetal/bbsim.ld) -add_custom_target(omni-interconnect-build) -set(BUCKYBALL_CTEST_TARGET attention) -set(BUCKYBALL_CTEST_GROUP_TARGET omni-interconnect-build) -foreach(TEST interconnect reservation backpressure bounds peripherals) - buckyball_ctest_name(NAME attention chip-${TEST}) - set(BINARY ${NAME}-baremetal) - add_custom_command(OUTPUT ${BINARY} - COMMAND ${ELF_CC} -std=gnu11 -O2 -static -march=rv64gc -mcmodel=medany - -specs=nano.specs -specs=nosys.specs -nostartfiles - -Wl,-T,${LINKER_SCRIPT} - -I${BUCKYBALL_REPO_ROOT}/stack/runtime/include - -I${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET} - ${STARTUP} ${CMAKE_CURRENT_SOURCE_DIR}/${TEST}.c ${SYSCALLS} -o ${BINARY} - DEPENDS ${TEST}.c ${STARTUP} ${SYSCALLS} ${LINKER_SCRIPT} - ${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/params.h - ${BUCKYBALL_REPO_ROOT}/stack/runtime/include/interconnect.h - VERBATIM) - add_custom_target(${NAME}-build DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${BINARY}) - add_dependencies(omni-interconnect-build ${NAME}-build) -endforeach() - -buckyball_ctest_name(NAME attention chip-process) -set(BINARY ${NAME}-linux) -add_custom_command(OUTPUT ${BINARY} - COMMAND ${LINUX_CC} -std=gnu11 -O2 -static -march=rv64gc - ${CMAKE_CURRENT_SOURCE_DIR}/process.c -o ${BINARY} - DEPENDS process.c - VERBATIM) -add_custom_target(${NAME}-build DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${BINARY}) -add_dependencies(omni-interconnect-build ${NAME}-build) - -buckyball_ctest_name(NAME attention chip-host-io) -set(BINARY ${NAME}-linux) -add_custom_command(OUTPUT ${BINARY} - COMMAND ${LINUX_CC} -std=gnu11 -O2 -static -march=rv64gc - ${CMAKE_CURRENT_SOURCE_DIR}/host_io.c -o ${BINARY} - DEPENDS host_io.c VERBATIM) -add_custom_target(${NAME}-build DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${BINARY}) -add_dependencies(omni-interconnect-build ${NAME}-build) - -buckyball_ctest_name(NAME attention chip-mailbox) -set(BINARY ${NAME}-linux) -add_custom_command(OUTPUT ${BINARY} - COMMAND ${LINUX_CC} -std=gnu11 -O2 -static -march=rv64gc - -I${BUCKYBALL_REPO_ROOT}/stack/runtime/include - ${CMAKE_CURRENT_SOURCE_DIR}/mailbox.c -o ${BINARY} - DEPENDS mailbox.c ${BUCKYBALL_REPO_ROOT}/stack/runtime/include/interconnect.h VERBATIM) -add_custom_target(${NAME}-build DEPENDS ${BINARY}) -add_dependencies(omni-interconnect-build ${NAME}-build) diff --git a/examples/chips/omni/workloads/ctests/backpressure.c b/examples/chips/omni/workloads/ctests/backpressure.c deleted file mode 100644 index 95c0a8149..000000000 --- a/examples/chips/omni/workloads/ctests/backpressure.c +++ /dev/null @@ -1,37 +0,0 @@ -#include -#include - -int main(void) { - uint64_t chip = link_read(LINK_CHIP_ID); - uint64_t memory = link_read(LINK_MEMORY_BYTES); - volatile uint64_t *source = (volatile uint64_t *)(0x80000000ULL + memory / 2); - volatile uint64_t *target = - (volatile uint64_t *)(0x80000000ULL + memory * 3 / 4); - uint64_t capacity = link_read(LINK_EVENT_CAPACITY); - if (chip > 1) - return 0; - if (chip == 0) { - for (uint64_t index = 0; index <= capacity; ++index) { - *source = 100 + index; - if (chip_send(1, (uintptr_t)source, (uintptr_t)(target + index), 8, - index)) - return 1; - } - } else { - // Fill the completion queue before allowing any consumption. - while (link_read(LINK_EVENT_COUNT) < capacity) { - } - if (link_read(LINK_EVENT_COUNT) != capacity) - return 2; - for (uint64_t index = 0; index <= capacity; ++index) { - while (!link_read(LINK_EVENT_COUNT)) { - } - if (link_read(LINK_EVENT_SOURCE) != 0 || - link_read(LINK_EVENT_TAG) != index || target[index] != 100 + index) - return 3; - link_write(LINK_EVENT_ACK, 1); - } - puts("completion queue backpressure and ordered delivery PASSED"); - } - return 0; -} diff --git a/examples/chips/omni/workloads/ctests/bounds.c b/examples/chips/omni/workloads/ctests/bounds.c deleted file mode 100644 index cde80d5a3..000000000 --- a/examples/chips/omni/workloads/ctests/bounds.c +++ /dev/null @@ -1,10 +0,0 @@ -#include - -int main(void) { - if (link_read(LINK_CHIP_ID) != 0) - return 0; - uint64_t bytes = link_read(LINK_MEMORY_BYTES); - // This descriptor must fail rather than writing past chip 1's DDR. - chip_send(1, 0x80000000ULL, 0x80000000ULL + bytes - 8, 16, 99); - return 1; -} diff --git a/examples/chips/omni/workloads/ctests/host_io.c b/examples/chips/omni/workloads/ctests/host_io.c deleted file mode 100644 index ded6f6349..000000000 --- a/examples/chips/omni/workloads/ctests/host_io.c +++ /dev/null @@ -1,28 +0,0 @@ -#include -#include - -int main(void) { - for (;;) { - uint64_t input; - size_t offset = 0; - while (offset < sizeof(input)) { - ssize_t count = read(0, (char *)&input + offset, sizeof(input) - offset); - if (count == 0) - return offset == 0 ? 0 : 1; - if (count < 0) - return 2; - offset += count; - } - if (input == UINT64_MAX) - return 11; - uint64_t output = input * input + 7; - offset = 0; - while (offset < sizeof(output)) { - ssize_t count = - write(1, (char *)&output + offset, sizeof(output) - offset); - if (count <= 0) - return 3; - offset += count; - } - } -} diff --git a/examples/chips/omni/workloads/ctests/interconnect.c b/examples/chips/omni/workloads/ctests/interconnect.c deleted file mode 100644 index 233c2aa43..000000000 --- a/examples/chips/omni/workloads/ctests/interconnect.c +++ /dev/null @@ -1,81 +0,0 @@ -#include -#include -#include - -#define CHECK(condition) \ - do { \ - if (!(condition)) { \ - fprintf(stderr, "chip %u line %d FAILED\n", (unsigned)chip, __LINE__); \ - return 1; \ - } \ - } while (0) - -int main(void) { - uint64_t chip = link_read(LINK_CHIP_ID); - uint64_t count = link_read(LINK_CHIP_COUNT); - CHECK(count > 1 && count <= 32); - const uint64_t memory = link_read(LINK_MEMORY_BYTES); - CHECK(memory > (4ULL << 30)); - // Same physical addresses in distinct DDRs, both above a 32-bit address - // range. - volatile uint64_t *source = (volatile uint64_t *)(0x80000000ULL + memory / 2); - volatile uint64_t *result = - (volatile uint64_t *)(0x80000000ULL + memory * 3 / 4); - for (unsigned i = 0; i < 32; ++i) - source[i] = 1000 * chip + i; - CHECK(chip_send((chip + 1) % count, (uintptr_t)source, (uintptr_t)result, 256, - 10 + chip) == 0); - // Thinker ranks publish disjoint partials into the last chip's DDR. - if (chip < count - 1) { - while (!link_read(LINK_EVENT_COUNT)) { - } - uint64_t previous = (chip + count - 1) % count; - CHECK(link_read(LINK_EVENT_SOURCE) == previous); - CHECK(link_read(LINK_EVENT_TAG) == 10 + previous); - CHECK(link_read(LINK_EVENT_BYTES) == 256); - __asm__ volatile("fence rw,rw" ::: "memory"); - for (unsigned i = 0; i < 32; ++i) - CHECK(result[i] == 1000 * previous + i); - link_write(LINK_EVENT_ACK, 1); - source[0] = chip + 1; - CHECK(chip_send(count - 1, (uintptr_t)source, - (uintptr_t)(result + 64 + chip), 8, 20 + chip) == 0); - while (!link_read(LINK_EVENT_COUNT)) { - } - CHECK(link_read(LINK_EVENT_SOURCE) == count - 1); - CHECK(link_read(LINK_EVENT_TAG) == 30); - CHECK(result[128] == count * (count - 1) / 2); - link_write(LINK_EVENT_ACK, 1); - } else { - unsigned received = 0; - unsigned ring_seen = 0; - for (unsigned i = 0; i < count; ++i) { - while (!link_read(LINK_EVENT_COUNT)) { - } - uint64_t peer = link_read(LINK_EVENT_SOURCE); - if (peer == count - 2 && link_read(LINK_EVENT_TAG) == 10 + count - 2) { - CHECK(peer == count - 2 && !ring_seen); - for (unsigned j = 0; j < 32; ++j) - CHECK(result[j] == 1000 * (count - 2) + j); - ring_seen = 1; - link_write(LINK_EVENT_ACK, 1); - continue; - } - CHECK(peer < count - 1 && !(received & (1u << peer))); - CHECK(link_read(LINK_EVENT_TAG) == 20 + peer); - received |= 1u << peer; - link_write(LINK_EVENT_ACK, 1); - } - CHECK(ring_seen && received == (1u << (count - 1)) - 1); - source[0] = 0; - for (unsigned peer = 0; peer < count - 1; ++peer) - source[0] += result[64 + peer]; - CHECK(source[0] == count * (count - 1) / 2); - for (unsigned peer = 0; peer < count - 1; ++peer) - CHECK(chip_send(peer, (uintptr_t)source, (uintptr_t)(result + 128), 8, - 30) == 0); - } - printf("chip %u: isolated DDR, ring DMA, gather and broadcast PASSED\n", - (unsigned)chip); - return 0; -} diff --git a/examples/chips/omni/workloads/ctests/mailbox.c b/examples/chips/omni/workloads/ctests/mailbox.c deleted file mode 100644 index bfc0b3487..000000000 --- a/examples/chips/omni/workloads/ctests/mailbox.c +++ /dev/null @@ -1,34 +0,0 @@ -#include -#include -#include -#include - -int main(void) { - _Atomic uint64_t *owner = (_Atomic uint64_t *)CHIP_LINK_BUFFER; - uint64_t zero = 0; - if (!atomic_compare_exchange_strong(owner, &zero, 1)) - return 0; - uint64_t chip = link_read(LINK_CHIP_ID), count = link_read(LINK_CHIP_COUNT); - uint64_t physical = link_read(LINK_BUFFER_ADDRESS); - if (count < 2 || link_read(LINK_BUFFER_BYTES) < 12288) - return 1; - volatile uint64_t *source = (volatile uint64_t *)(CHIP_LINK_BUFFER + 4096); - volatile uint64_t *destination = - (volatile uint64_t *)(CHIP_LINK_BUFFER + 8192); - for (unsigned i = 0; i < 32; ++i) - source[i] = chip * 100 + i; - if (chip_send((chip + 1) % count, physical + 4096, physical + 8192, 256, 42)) - return 2; - while (!link_read(LINK_EVENT_COUNT)) { - } - if (link_read(LINK_EVENT_SOURCE) != (chip + count - 1) % count || - link_read(LINK_EVENT_TAG) != 42 || link_read(LINK_EVENT_BYTES) != 256) - return 3; - __asm__ volatile("fence rw,rw" ::: "memory"); - for (unsigned i = 0; i < 32; ++i) - if (destination[i] != ((chip + count - 1) % count) * 100 + i) - return 4; - link_write(LINK_EVENT_ACK, 1); - printf("chip %lu: PK DDR INTERCONNECT PASSED\n", chip); - return 0; -} diff --git a/examples/chips/omni/workloads/ctests/peripherals.c b/examples/chips/omni/workloads/ctests/peripherals.c deleted file mode 100644 index 491d78a6a..000000000 --- a/examples/chips/omni/workloads/ctests/peripherals.c +++ /dev/null @@ -1,32 +0,0 @@ -#include -#include -#include - -int main(void) { - if (link_read(LINK_CHIP_ID) != 4) - return 0; - volatile uint8_t *uart = (volatile uint8_t *)0x60020000ULL; - puts("OMNI VM READY"); - while (!(uart[5] & 1)) { - } - if (uart[4] != 'g') - return 1; - volatile uint32_t *vga = (volatile uint32_t *)0x11000000ULL; - volatile uint32_t *pixels = (volatile uint32_t *)0x11001000ULL; - unsigned width = vga[0], height = vga[1]; - for (unsigned y = 0; y < height; ++y) - for (unsigned x = 0; x < width; ++x) - pixels[y * width + x] = - ((x * 255 / width) << 16) | ((y * 255 / height) << 8) | 0x40; - vga[3] = 1; - volatile uint32_t *speaker = (volatile uint32_t *)0x10002000ULL; - volatile int16_t *sample = (volatile int16_t *)0x10002004ULL; - for (unsigned i = 0; i < 256; ++i) - *sample = ((int)(i % 64) - 32) * 256; - speaker[3] = 1; - while (speaker[0]) { - } - speaker[3] = 0; - puts("OMNI VM TEXT FRAME AUDIO PASSED"); - return 0; -} diff --git a/examples/chips/omni/workloads/ctests/process.c b/examples/chips/omni/workloads/ctests/process.c deleted file mode 100644 index a43f22d01..000000000 --- a/examples/chips/omni/workloads/ctests/process.c +++ /dev/null @@ -1,39 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - -int main(void) { - if (write(1, (void *)0x80000000ULL, 1) != -1 || errno != EFAULT) - return 7; - uint32_t seed; - if (getrandom(&seed, sizeof(seed), 0) != sizeof(seed)) - return 1; - for (unsigned iteration = 0; iteration < 32; ++iteration) { - size_t padding = (1 + ((seed >> (iteration % 24)) & 255)) * 4096; - unsigned char *allocation = malloc(padding + 8192); - if (!allocation) - return 2; - unsigned char *data = allocation + padding; - for (unsigned i = 0; i < 8192; ++i) - data[i] = (seed + iteration + i) & 255; - int fd = open("process.bin", O_CREAT | O_TRUNC | O_RDWR, 0600); - if (fd < 0 || write(fd, data, 8192) != 8192) - return 3; - memset(data, 0, 8192); - if (lseek(fd, 0, SEEK_SET) != 0 || read(fd, data, 8192) != 8192) - return 4; - for (unsigned i = 0; i < 8192; ++i) - if (data[i] != ((seed + iteration + i) & 255)) - return 5; - if (close(fd)) - return 6; - free(allocation); - } - puts("OMNI PROCESS MEMORY AND FILE ISOLATION PASSED"); - return 0; -} diff --git a/examples/chips/omni/workloads/ctests/reservation.c b/examples/chips/omni/workloads/ctests/reservation.c deleted file mode 100644 index c5880b467..000000000 --- a/examples/chips/omni/workloads/ctests/reservation.c +++ /dev/null @@ -1,47 +0,0 @@ -#include -#include - -int main(void) { - uint64_t chip = link_read(LINK_CHIP_ID); - const uint64_t memory = link_read(LINK_MEMORY_BYTES); - volatile uint64_t *source = (volatile uint64_t *)(0x80000000ULL + memory / 2); - volatile uint64_t *target = - (volatile uint64_t *)(0x80000000ULL + memory * 3 / 4); - if (chip > 1) - return 0; - if (chip == 0) { - uint64_t value, status; - *target = 10; - __asm__ volatile("lr.d %0, (%1)" : "=r"(value) : "r"(target) : "memory"); - __asm__ volatile("sc.d %0, %2, (%1)" - : "=r"(status) - : "r"(target), "r"(11ULL) - : "memory"); - if (value != 10 || status != 0 || *target != 11) - return 1; - __asm__ volatile("lr.d %0, (%1)" : "=r"(value) : "r"(target) : "memory"); - *source = 1; - if (chip_send(1, (uintptr_t)source, (uintptr_t)target, 8, 1)) - return 2; - while (!link_read(LINK_EVENT_COUNT)) { - } - if (link_read(LINK_EVENT_SOURCE) != 1) - return 3; - __asm__ volatile("sc.d %0, %2, (%1)" - : "=r"(status) - : "r"(target), "r"(99ULL) - : "memory"); - if (status == 0 || *target != 20) - return 4; - link_write(LINK_EVENT_ACK, 1); - puts("cross-chip DMA invalidates LR/SC reservation PASSED"); - } else { - while (!link_read(LINK_EVENT_COUNT)) { - } - link_write(LINK_EVENT_ACK, 1); - *source = 20; - if (chip_send(0, (uintptr_t)source, (uintptr_t)target, 8, 2)) - return 5; - } - return 0; -} diff --git a/examples/chips/omni/workloads/mlir_tests/CMakeLists.txt b/examples/chips/omni/workloads/mlir_tests/CMakeLists.txt deleted file mode 100644 index e52326652..000000000 --- a/examples/chips/omni/workloads/mlir_tests/CMakeLists.txt +++ /dev/null @@ -1,109 +0,0 @@ -set(REPO ${BUCKYBALL_REPO_ROOT}) -set(BUDDY ${REPO}/stack/compiler/thirdparty/buddy-mlir) -set(COMPILER ${BUDDY}/build/omni) -set(MODEL_DIR ${REPO}/stack/models/archs/buckyball/omni/qwen3-omni-30b-a3b-instruct) -set(DATA ${CMAKE_CURRENT_BINARY_DIR}/moe) -set(ATTENTION_DATA ${CMAKE_CURRENT_BINARY_DIR}/attention) -set(PYTHON ${REPO}/result/bin/python3) -set(MODEL_CONFIG ${REPO}/stack/models/models/qwen3-omni-30b-a3b-instruct/configs/model.toml) -set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${MODEL_CONFIG}) -set(CHECKPOINT_FILE ${CMAKE_CURRENT_BINARY_DIR}/checkpoint.txt) -add_custom_command(OUTPUT ${CHECKPOINT_FILE} - COMMAND ${PYTHON} -c - "import sys, tomllib; from pathlib import Path; from huggingface_hub import snapshot_download; p=tomllib.loads(Path(sys.argv[1]).read_text())['parameters']; Path(sys.argv[2]).write_text(snapshot_download(p['checkpoint'], revision=p['revision'], local_files_only=True))" - ${MODEL_CONFIG} ${CHECKPOINT_FILE} - DEPENDS ${MODEL_CONFIG} - WORKING_DIRECTORY ${REPO} VERBATIM) -file(GLOB_RECURSE MODEL_SOURCES CONFIGURE_DEPENDS ${MODEL_DIR}/trace/*.py ${MODEL_DIR}/permute/*.py ${MODEL_DIR}/quant/*.py) -list(APPEND MODEL_SOURCES ${BUDDY}/frontend/Python/ops/tosa.py - ${COMPILER}/python_packages/buddy/compiler/ops/tosa.py) -list(APPEND MODEL_SOURCES ${REPO}/examples/balls/mxmm/compiler/python/fp32.py) -add_custom_command(OUTPUT ${DATA}/router.mlir ${DATA}/router-forward.mlir - ${DATA}/expert_prefill.mlir ${DATA}/expert_prefill-forward.mlir - ${DATA}/expert.mlir ${DATA}/expert-forward.mlir ${DATA}/moe-parameters.h - COMMAND bash -o pipefail -c - "'${PYTHON}' -m stack.models.archs.buckyball.omni.qwen3-omni-30b-a3b-instruct.trace.export --checkpoint \"$(cat '${CHECKPOINT_FILE}')\" --compiler-build '${COMPILER}' --output '${DATA}'" - DEPENDS ${CHECKPOINT_FILE} ${MODEL_SOURCES} ${REPO}/stack/models/models/qwen3-omni-30b-a3b-instruct/moe.py - WORKING_DIRECTORY ${REPO} VERBATIM) -add_custom_command(OUTPUT ${ATTENTION_DATA}/prefill.mlir ${ATTENTION_DATA}/prefill-forward.mlir - ${ATTENTION_DATA}/decode.mlir ${ATTENTION_DATA}/decode-forward.mlir - ${ATTENTION_DATA}/attention-parameters.h - COMMAND bash -o pipefail -c - "'${PYTHON}' -m stack.models.archs.buckyball.omni.qwen3-omni-30b-a3b-instruct.trace.attention --checkpoint \"$(cat '${CHECKPOINT_FILE}')\" --compiler-build '${COMPILER}' --output '${ATTENTION_DATA}'" - DEPENDS ${CHECKPOINT_FILE} ${MODEL_SOURCES} ${REPO}/stack/models/models/qwen3-omni-30b-a3b-instruct/attention.py - WORKING_DIRECTORY ${REPO} VERBATIM) -set(FORWARD_PASSES - -buffer-deallocation-simplification -convert-linalg-to-loops -eliminate-empty-tensors - -llvm-request-c-wrappers -convert-math-to-llvm -convert-math-to-libm -lower-affine - -convert-scf-to-cf -convert-cf-to-llvm -convert-arith-to-llvm -expand-strided-metadata - -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts - -canonicalize -cse -lower-tile-runtime) -set(KERNEL_PASSES - -eliminate-empty-tensors -convert-elementwise-to-linalg -convert-tensor-to-linalg - -canonicalize -cse -one-shot-bufferize=bufferize-function-boundaries - -buffer-deallocation-pipeline=private-function-dynamic-ownership=false -convert-linalg-to-tile=quantized-only - -convert-tile-to-buckyball -lower-buckyball-to-bank-ssa -assign-physical-banks - -llvm-request-c-wrappers -lower-bank-ssa-to-intrinsics -expand-strided-metadata - -convert-linalg-to-loops -lower-buckyball -convert-vector-to-scf=target-rank=0 - -lower-affine -canonicalize -cse -convert-scf-to-cf -convert-cf-to-llvm - -convert-vector-to-llvm -buffer-deallocation-simplification -bufferization-lower-deallocations - -convert-math-to-llvm -convert-math-to-libm -convert-arith-to-llvm -convert-func-to-llvm - -finalize-memref-to-llvm -reconcile-unrealized-casts -canonicalize -cse -lower-tile-runtime) -foreach(STAGE router expert expert_prefill prefill decode) - set(TARGET attention) - if(STAGE MATCHES "^expert") - set(TARGET ffn) - endif() - if(STAGE STREQUAL "router" OR STAGE MATCHES "^expert") - set(STAGE_DATA ${DATA}) - set(GROUP MOE) - else() - set(STAGE_DATA ${ATTENTION_DATA}) - set(GROUP ATTENTION) - endif() - foreach(KIND kernel forward) - if(KIND STREQUAL "kernel") - set(NAME ${STAGE}) - string(JOIN " " PASSES ${KERNEL_PASSES}) - set(LOWER "'${COMPILER}/bin/buddy-opt' '${STAGE_DATA}/${NAME}.mlir' -pass-pipeline 'builtin.module(func.func(tosa-to-linalg-named,tosa-to-linalg,tosa-to-tensor,tosa-to-arith))' | '${COMPILER}/bin/buddy-opt' --target=${TARGET} ${PASSES}") - else() - set(NAME ${STAGE}-forward) - string(JOIN " " PASSES ${FORWARD_PASSES}) - set(LOWER "'${COMPILER}/bin/buddy-opt' '${STAGE_DATA}/${NAME}.mlir' --target=${TARGET} ${PASSES}") - endif() - add_custom_command(OUTPUT ${NAME}.o - COMMAND bash -o pipefail -c "${LOWER} | '${COMPILER}/bin/buddy-translate' --buddy-to-llvmir | '${BUDDY}/llvm/build/bin/opt' -passes='default' | '${COMPILER}/bin/buddy-llc' -filetype=obj -mtriple=riscv64 -O2 -mattr=+xbuckyball,+m,+D -float-abi=hard -o '${NAME}.o'" - DEPENDS ${STAGE_DATA}/${NAME}.mlir ${COMPILER}/bin/buddy-opt VERBATIM) - if(STAGE STREQUAL "expert_prefill") - list(APPEND BATCH_OBJECTS ${NAME}.o) - else() - list(APPEND ${GROUP}_OBJECTS ${NAME}.o) - endif() - endforeach() -endforeach() -set(RUNTIME_PARAMS ${BUCKYBALL_ISA_DIR}/attention) -# Ball tensor tests may already provide the shared runtime and RVV kernels. -if(NOT TARGET runtime) - add_subdirectory(${REPO}/stack/runtime runtime) -endif() -if(NOT TARGET rvv_kernels) - add_subdirectory("${REPO}/bb-tests/workloads/lib/bbsw/kernels/rvv" rvv) -endif() -if(NOT TARGET mxfp8_pack) - add_subdirectory("${REPO}/bb-tests/workloads/lib/bbsw/kernels/mxfp8" mxfp8) -endif() -buckyball_mlirtest_name(NAME ffn chip-moe) -add_executable(${NAME}-linux moe_main.cpp ${MOE_OBJECTS} ${DATA}/moe-parameters.h) -target_include_directories(${NAME}-linux PRIVATE ${DATA} ${BUDDY}/frontend/Interfaces) -target_link_libraries(${NAME}-linux PRIVATE runtime rvv_kernels mxfp8_pack rvv_quant_backend -static) -add_dependencies(chip-mlirtests-build ${NAME}-linux) -buckyball_mlirtest_name(NAME ffn chip-moe-batch) -add_executable(${NAME}-linux moe_batch_main.cpp ${BATCH_OBJECTS} expert.o expert-forward.o ${DATA}/moe-parameters.h) -target_include_directories(${NAME}-linux PRIVATE ${DATA} ${BUDDY}/frontend/Interfaces) -target_link_libraries(${NAME}-linux PRIVATE runtime rvv_kernels mxfp8_pack rvv_quant_backend -static) -add_dependencies(chip-mlirtests-build ${NAME}-linux) -buckyball_mlirtest_name(NAME attention chip-attention) -add_executable(${NAME}-linux attention_main.cpp ${ATTENTION_OBJECTS} ${ATTENTION_DATA}/attention-parameters.h) -target_include_directories(${NAME}-linux PRIVATE ${ATTENTION_DATA} ${BUDDY}/frontend/Interfaces) -target_link_libraries(${NAME}-linux PRIVATE runtime rvv_kernels mxfp8_pack rvv_quant_backend -static) -add_dependencies(chip-mlirtests-build ${NAME}-linux) diff --git a/examples/chips/omni/workloads/mlir_tests/attention_main.cpp b/examples/chips/omni/workloads/mlir_tests/attention_main.cpp deleted file mode 100644 index d5d06ec77..000000000 --- a/examples/chips/omni/workloads/mlir_tests/attention_main.cpp +++ /dev/null @@ -1,107 +0,0 @@ -#include "attention-parameters.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using Hidden = MemRef; -using Cache = MemRef; -using Slots = MemRef; -using Positions = MemRef; -using Floats = MemRef; -using Bytes = MemRef; -struct Result { - Hidden hidden; - Cache keys, values; -}; -extern "C" void _mlir_ciface_forward_prefill(Result *, Floats *, Bytes *, - Hidden *, Cache *, Cache *, - Slots *, Positions *); -extern "C" void _mlir_ciface_forward_decode(Result *, Floats *, Bytes *, - Hidden *, Cache *, Cache *, Slots *, - Positions *); - -template -void load(const std::string &path, MemRef &tensor) { - std::ifstream stream; - stream.exceptions(std::ios::failbit | std::ios::badbit); - stream.open(path, std::ios::binary); - stream.read(reinterpret_cast(tensor.getData()), - tensor.getSize() * sizeof(T)); -} - -template -void compare(const std::string &path, MemRef &actual) { - auto name = std::filesystem::path(path); - std::ofstream saved("actual-" + name.parent_path().filename().string() + "-" + - name.filename().string(), - std::ios::binary); - saved.exceptions(std::ios::failbit | std::ios::badbit); - saved.write(reinterpret_cast(actual.getData()), - actual.getSize() * sizeof(float)); - saved.close(); - std::vector shape(actual.getSizes(), actual.getSizes() + Rank); - MemRef expected(shape); - load(path, expected); - for (size_t i = 0; i < actual.getSize(); ++i) - if (!std::isfinite(actual[i]) || std::abs(actual[i] - expected[i]) > - 2e-5f + 2e-4f * std::abs(expected[i])) - throw std::runtime_error("attention mismatch: " + path + - " index=" + std::to_string(i) + - " actual=" + std::to_string(actual[i]) + - " expected=" + std::to_string(expected[i])); -} - -int main(int argc, char **argv) { - const auto fixtures = - std::filesystem::path(argv[0]).parent_path() / "attention"; - runtime_init(1024 * 1024); - void *workspace = aligned_alloc(64, 64 * 1024 * 1024); - if (!workspace) - throw std::bad_alloc(); - workspace_init(workspace, 64 * 1024 * 1024); - const std::vector cacheShape{1, kvHeads, capacity, headSize}; - Cache keys(cacheShape, 0.0f), values(cacheShape, 0.0f); - for (bool prefill : {true, false}) { - workspace_begin(workspace, 64 * 1024 * 1024); - size_t count = prefill ? 3 : 1; - std::string path = (fixtures / (prefill ? "prefill" : "decode")).string(); - Hidden hidden({1, count, hiddenSize}); - Slots slots({count}); - Positions positions({3, count}); - Floats floats({prefill ? prefillFloats : decodeFloats}); - Bytes bytes({prefill ? prefillBytes : decodeBytes}); - load(path + "/hidden.bin", hidden); - load(path + "/cache_positions.bin", slots); - load(path + "/positions.bin", positions); - load(path + "/params.f32", floats); - load(path + "/weights.bin", bytes); - if (!prefill) { - compare((fixtures / "prefill/expected-keys.bin").string(), keys); - compare((fixtures / "prefill/expected-values.bin").string(), values); - } - Result result{Hidden({1, count, hiddenSize}, false, 0), - Cache(cacheShape, false, 0), Cache(cacheShape, false, 0)}; - auto run = - prefill ? _mlir_ciface_forward_prefill : _mlir_ciface_forward_decode; - run(&result, &floats, &bytes, &hidden, &keys, &values, &slots, &positions); - compare(path + "/expected-keys.bin", result.keys); - compare(path + "/expected-values.bin", result.values); - compare(path + "/expected-hidden.bin", result.hidden); - std::copy_n(result.keys.getData(), keys.getSize(), keys.getData()); - std::copy_n(result.values.getData(), values.getSize(), values.getData()); - workspace_free(result.hidden.release()); - workspace_free(result.keys.release()); - workspace_free(result.values.release()); - std::cout << "OMNI MXFP8 ATTENTION " << (prefill ? "PREFILL" : "DECODE") - << " PASSED\n"; - } - free(workspace); -} diff --git a/examples/chips/omni/workloads/mlir_tests/moe_batch_main.cpp b/examples/chips/omni/workloads/mlir_tests/moe_batch_main.cpp deleted file mode 100644 index 4dac2e898..000000000 --- a/examples/chips/omni/workloads/mlir_tests/moe_batch_main.cpp +++ /dev/null @@ -1,61 +0,0 @@ -#include "moe-parameters.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using Matrix = MemRef; -using Bytes = MemRef; -extern "C" void _mlir_ciface_forward_expert(Matrix *, Bytes *, Matrix *); -extern "C" void _mlir_ciface_forward_expert_prefill(Matrix *, Bytes *, - Matrix *); - -template -void load(const std::filesystem::path &path, MemRef &value) { - std::ifstream stream; - stream.exceptions(std::ios::failbit | std::ios::badbit); - stream.open(path, std::ios::binary); - stream.read(reinterpret_cast(value.getData()), - value.getSize() * sizeof(T)); -} - -int main(int argc, char **argv) { - auto path = std::filesystem::path(argv[0]).parent_path() / "moe"; - runtime_init(1024 * 1024); - void *workspace = aligned_alloc(64, 64 * 1024 * 1024); - if (!workspace) - throw std::bad_alloc(); - workspace_init(workspace, 64 * 1024 * 1024); - workspace_begin(workspace, 64 * 1024 * 1024); - Matrix input({batchSize, hiddenSize}), expected({batchSize, hiddenSize}); - Bytes weights({expertBytes}); - load(path / "batch-input.f32", input); - load(path / "expert/weights.bin", weights); - load(path / "expert_prefill/expected.f32", expected); - Matrix batch({batchSize, hiddenSize}, false, 0); - _mlir_ciface_forward_expert_prefill(&batch, &weights, &input); - Matrix row({1, hiddenSize}); - for (size_t token = 0; token < batchSize; ++token) { - std::copy_n(input.getData() + token * hiddenSize, hiddenSize, - row.getData()); - Matrix single({1, hiddenSize}, false, 0); - _mlir_ciface_forward_expert(&single, &weights, &row); - for (size_t i = 0; i < hiddenSize; ++i) { - size_t index = token * hiddenSize + i; - if (!std::isfinite(batch[index]) || batch[index] != single[i] || - std::abs(batch[index] - expected[index]) > - 1e-5f + 1e-4f * std::abs(expected[index])) - throw std::runtime_error( - "MXFP8 batch differs from scalar or reference"); - } - workspace_free(single.release()); - } - workspace_free(batch.release()); - free(workspace); - std::cout << "OMNI MXFP8 EXPERT BATCH PASSED\n"; -} diff --git a/examples/chips/omni/workloads/mlir_tests/moe_main.cpp b/examples/chips/omni/workloads/mlir_tests/moe_main.cpp deleted file mode 100644 index 138254807..000000000 --- a/examples/chips/omni/workloads/mlir_tests/moe_main.cpp +++ /dev/null @@ -1,83 +0,0 @@ -#include "moe-parameters.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using Hidden = MemRef; -using Indices = MemRef; -using Floats = MemRef; -using Bytes = MemRef; -struct RouterResult { - Hidden normalized; - Indices indices; - Hidden scores; -}; -extern "C" void _mlir_ciface_forward_router(RouterResult *, Floats *, Hidden *); -extern "C" void _mlir_ciface_forward_expert(Hidden *, Bytes *, Hidden *); - -template -void load(const std::string &path, MemRef &tensor) { - std::ifstream stream; - stream.exceptions(std::ios::failbit | std::ios::badbit); - stream.open(path, std::ios::binary); - stream.read(reinterpret_cast(tensor.getData()), - tensor.getSize() * sizeof(T)); -} - -int main(int argc, char **argv) { - const std::string path = - (std::filesystem::path(argv[0]).parent_path() / "moe").string(); - runtime_init(1024 * 1024); - void *workspace = aligned_alloc(64, 64 * 1024 * 1024); - if (!workspace) - throw std::bad_alloc(); - workspace_init(workspace, 64 * 1024 * 1024); - workspace_begin(workspace, 64 * 1024 * 1024); - Hidden input({1, hiddenSize}), expected({1, hiddenSize}), scores({1, topK}); - Indices indices({1, topK}); - Floats floats({routerFloats}); - Bytes bytes({expertBytes}); - load(path + "/input.f32", input); - load(path + "/router/params.f32", floats); - load(path + "/expert/weights.bin", bytes); - RouterResult routing{Hidden({1, hiddenSize}, false, 0), - Indices({1, topK}, false, 0), - Hidden({1, topK}, false, 0)}; - _mlir_ciface_forward_router(&routing, &floats, &input); - load(path + "/router/expected.f32", expected); - load(path + "/router/indices.i64", indices); - load(path + "/router/scores.f32", scores); - for (size_t i = 0; i < hiddenSize; ++i) - if (!std::isfinite(routing.normalized[i]) || - std::abs(routing.normalized[i] - expected[i]) > 1e-5f) - throw std::runtime_error("router normalization mismatch"); - for (size_t i = 0; i < topK; ++i) - if (routing.indices[i] != indices[i] || !std::isfinite(routing.scores[i]) || - std::abs(routing.scores[i] - scores[i]) > 1e-5f) - throw std::runtime_error("router top-k mismatch"); - Hidden result({1, hiddenSize}, false, 0); - _mlir_ciface_forward_expert(&result, &bytes, &input); - load(path + "/expert/expected.f32", expected); - float maximum = 0; - for (size_t i = 0; i < hiddenSize; ++i) { - float error = std::abs(result[i] - expected[i]); - if (!std::isfinite(result[i]) || - error > 1e-5f + 1e-4f * std::abs(expected[i])) - throw std::runtime_error("MXFP8 expert mismatch"); - maximum = std::max(maximum, error); - } - std::cout << "OMNI ROUTER AND MXFP8 EXPERT PASSED max_error=" << maximum - << '\n'; - workspace_free(routing.normalized.release()); - workspace_free(routing.indices.release()); - workspace_free(routing.scores.release()); - workspace_free(result.release()); - free(workspace); -} diff --git a/examples/chips/pebble/arch/src/main/scala/sims/p2e/TargetConfigs.scala b/examples/chips/pebble/arch/src/main/scala/sims/p2e/TargetConfigs.scala deleted file mode 100644 index 9981eb52d..000000000 --- a/examples/chips/pebble/arch/src/main/scala/sims/p2e/TargetConfigs.scala +++ /dev/null @@ -1,10 +0,0 @@ -package sims.p2e - -import chisel3.experimental.hierarchy.{Instance, Instantiate} -import framework.system.{System, SystemParams} - -/** Pebble on the P2E board: one compute core with private banks over the DDR4 macro. */ -class PebbleP2ETarget extends P2ETarget("../examples/chips/pebble/configs/generated/chip.pb") { - override def instantiate(p: SystemParams): Instance[System] = - Instantiate(new framework.system.Device(p.toGlobalConfig)) -} diff --git a/examples/chips/pebble/arch/src/main/scala/sims/verilator/TargetConfigs.scala b/examples/chips/pebble/arch/src/main/scala/sims/verilator/TargetConfigs.scala deleted file mode 100644 index 9a09e967a..000000000 --- a/examples/chips/pebble/arch/src/main/scala/sims/verilator/TargetConfigs.scala +++ /dev/null @@ -1,12 +0,0 @@ -package sims.verilator - -import chisel3.experimental.hierarchy.{Instance, Instantiate} -import framework.system.{System, SystemParams} - -import sims.soc.SystemTarget - -/** Pebble on the explicit System: one compute core with private banks and no task controller. */ -class PebbleVerilatorTarget extends SystemTarget("../examples/chips/pebble/configs/generated/chip.pb") { - override def instantiate(p: SystemParams): Instance[System] = - Instantiate(new framework.system.Device(p.toGlobalConfig)) -} diff --git a/examples/chips/pebble/configs/chip.toml b/examples/chips/pebble/configs/chip.toml deleted file mode 100644 index 268aa3106..000000000 --- a/examples/chips/pebble/configs/chip.toml +++ /dev/null @@ -1,10 +0,0 @@ -[designs] -include = "designs/pebble.toml" - -[sims] -verilator = "sims.verilator.PebbleVerilatorTarget" -p2e = "sims.p2e.PebbleP2ETarget" - -[uvm] -balls = ["transpose", "smatmul", "im2col", "toint8", "int2fp", "lut", "maxpool", "int8add", "int8mul", "matadd"] -ips = [] diff --git a/examples/chips/pebble/configs/designs/pebble.toml b/examples/chips/pebble/configs/designs/pebble.toml deleted file mode 100644 index fb742d1fe..000000000 --- a/examples/chips/pebble/configs/designs/pebble.toml +++ /dev/null @@ -1,4 +0,0 @@ -# Pebble: a main tile only. - -[main_tile] -include = "tiles/default.toml" diff --git a/examples/chips/pebble/configs/designs/tiles/default.toml b/examples/chips/pebble/configs/designs/tiles/default.toml deleted file mode 100644 index d1ef6a2e3..000000000 --- a/examples/chips/pebble/configs/designs/tiles/default.toml +++ /dev/null @@ -1,23 +0,0 @@ -# Pebble tile configuration - -coreDataBytes = 64 -xLen = 64 -vaddrBits = 39 -paddrBits = 56 -pgIdxBits = 12 -pgLevels = 3 -nPMPs = 8 - -memBallChannelNum = 6 - -[sharedMem] -bankWidth = 128 -bankEntries = 512 -enable = false -entries = 16384 -inputChannels = 7 -defaultGroupCount = 1 - -[[cores]] -core_id = 0 -include = "../../../../../cores/pebble/configs/default.toml" diff --git a/examples/chips/pebble/kernel/workloads.toml b/examples/chips/pebble/kernel/workloads.toml deleted file mode 100644 index aeb45f9a9..000000000 --- a/examples/chips/pebble/kernel/workloads.toml +++ /dev/null @@ -1,36 +0,0 @@ -[workloads] -tests = [ - "pebble-pebble-mlirtest-tile_mega_conv2d_k5_tail-linux", - "pebble-pebble-mlirtest-tile_mega_matmul_m17_k65_n17-linux", - "pebble-pebble-ctest-mvin_2d_test-linux", - "pebble-pebble-ctest-mvin_2d_pixel_stride_test-linux", - "pebble-pebble-ctest-mvin_mvout_acc_test-linux", - "pebble-pebble-ctest-mvin_mvout_test-linux", - "pebble-pebble-ctest-mvin_mvout_stride_test-linux", - "pebble-pebble-ctest-mvin_mvout_unaligned_test-linux", - "pebble-pebble-ctest-mvin_unaligned_test-linux", - "pebble-pebble-mlirtest-linalg_mega_conv2d_k3_c16-linux", - "pebble-pebble-mlirtest-tile_mega_conv2d_s2_depthwise_s2-linux", - "pebble-pebble-mlirtest-tile_mega_global_avg_conv_exact-linux", - "pebble-pebble-mlirtest-tile_mega_global_avg_conv2_exact-linux", - "pebble-pebble-mlirtest-tile_mega_global_avg_7x7_c288_exact-linux", - "pebble-pebble-mlirtest-tile_mega_se_gate_conv-linux", - "pebble-pebble-ctest-mega_conv_pipeline_test-linux", - "pebble-pebble-ctest-residual_add_lut_test-linux", - "pebble-pebble-ctest-maxpool_im2col_test-linux", - "pebble-pebble-ctest-se_gate_lut_test-linux", - "pebble-pebble-ctest-int8mul_large_gate_row_test-linux", - "pebble-pebble-ctest-dual_smatmul_matadd_test-linux", - "pebble-pebble-ctest-dual_conv_add_int2fp_test-linux", - "pebble-pebble-mlirtest-bank_matadd_i32_elementwise-linux", - "pebble-pebble-mlirtest-bank_int8add_identity_scale-linux", - "pebble-pebble-mlirtest-bank_lut_identity_table-linux", - "pebble-pebble-mlirtest-bank_int8mul_unit_gate-linux", - "pebble-pebble-mlirtest-bank_maxpool_k3_padded_spatial-linux", - "pebble-pebble-mlirtest-bank_smatmul_bias_two_terms-linux", - "pebble-pebble-mlirtest-bank_int2fp_unit_scale-linux", - "pebble-pebble-mlirtest-bank_quant_i32_pack16-linux", - "pebble-pebble-mlirtest-bank_quant_f32_pack16-linux", - "pebble-pebble-mlirtest-bank_im2col_k3_padded_bank_windows-linux", - "pebble-pebble-mlirtest-bank_transpose_i8_packed_matrix-linux", -] diff --git a/examples/chips/pebble/regression/batch/bemu/workloads-bare.toml b/examples/chips/pebble/regression/batch/bemu/workloads-bare.toml deleted file mode 100644 index 8e1016606..000000000 --- a/examples/chips/pebble/regression/batch/bemu/workloads-bare.toml +++ /dev/null @@ -1,91 +0,0 @@ -[workloads] -tests = [ - "pebble-pebble-ctest-mvin_2d_test-baremetal", - "pebble-pebble-ctest-mvin_2d_pixel_stride_test-baremetal", - "pebble-pebble-ctest-mvin_mvout_acc_test-baremetal", - "pebble-pebble-ctest-mvin_mvout_test-baremetal", - "pebble-pebble-ctest-mvin_mvout_stride_test-baremetal", - "pebble-pebble-ctest-mvin_mvout_unaligned_test-baremetal", - "pebble-pebble-ctest-mvin_unaligned_test-baremetal", - "pebble-pebble-mlirtest-linalg_mega_conv2d_k3_c16-baremetal", - "pebble-pebble-mlirtest-tile_mega_conv2d_s2_depthwise_s2-baremetal", - "pebble-pebble-mlirtest-tile_mega_global_avg_conv_exact-baremetal", - "pebble-pebble-mlirtest-tile_mega_global_avg_conv2_exact-baremetal", - "pebble-pebble-mlirtest-tile_mega_global_avg_7x7_c288_exact-baremetal", - "pebble-pebble-mlirtest-tile_mega_se_gate_conv-baremetal", - "pebble-pebble-ctest-mega_conv_pipeline_test-baremetal", - "pebble-pebble-ctest-residual_add_lut_test-baremetal", - "pebble-pebble-ctest-maxpool_im2col_test-baremetal", - "pebble-pebble-ctest-se_gate_lut_test-baremetal", - "pebble-pebble-ctest-int8mul_large_gate_row_test-baremetal", - "pebble-pebble-ctest-dual_smatmul_matadd_test-baremetal", - "pebble-pebble-ctest-dual_conv_add_int2fp_test-baremetal", - "pebble-pebble-ctest-quant_smatmul_int2fp_test-baremetal", - "pebble-pebble-ctest-smatmul_mvout_bank_reuse_test-baremetal", - "pebble-pebble-mlirtest-bank_matadd_i32_elementwise-baremetal", - "pebble-pebble-mlirtest-bank_int8add_identity_scale-baremetal", - "pebble-pebble-mlirtest-bank_lut_identity_table-baremetal", - "pebble-pebble-mlirtest-bank_int8mul_unit_gate-baremetal", - "pebble-pebble-mlirtest-bank_maxpool_k3_padded_spatial-baremetal", - "pebble-pebble-mlirtest-bank_smatmul_bias_two_terms-baremetal", - "pebble-pebble-mlirtest-bank_int2fp_unit_scale-baremetal", - "pebble-pebble-mlirtest-bank_quant_i32_pack16-baremetal", - "pebble-pebble-mlirtest-bank_quant_f32_pack16-baremetal", - "pebble-pebble-mlirtest-bank_im2col_k3_padded_bank_windows-baremetal", - "pebble-pebble-mlirtest-bank_transpose_i8_packed_matrix-baremetal", - "pebble-pebble-ctest-tlb_bare_mode_test-baremetal", - "pebble-pebble-ctest-tlb_repeated_sfence_test-baremetal", - "pebble-pebble-ctest-tlb_sfence_flush_test-baremetal", - "pebble-pebble-ctest-tlb_sfence_remap_test-baremetal", - "pebble-pebble-ctest-tlb_sv39_4k_alias_test-baremetal", - "pebble-pebble-ctest-tlb_sv39_identity_test-baremetal", - "pebble-pebble-ctest-tlb_sv39_nonidentity_test-baremetal", - "pebble-pebble-ctest-transpose_i8_16x16_test-baremetal", - "pebble-pebble-ctest-transpose_i8_16x32_test-baremetal", - "pebble-pebble-ctest-transpose_i32_8x16_test-baremetal", - "pebble-pebble-mlirtest-ball_transpose_16x16_i8-baremetal", - "pebble-pebble-ctest-smatmul_bias_accumulate_test-baremetal", - "pebble-pebble-ctest-smatmul_1x16x64_test-baremetal", - "pebble-pebble-ctest-smatmul_16x16_test-baremetal", - "pebble-pebble-ctest-smatmul_16x16x32_test-baremetal", - "pebble-pebble-ctest-smatmul_nondefault_banks_test-baremetal", - "pebble-pebble-mlirtest-ball_smatmul_bias_accumulate-baremetal", - "pebble-pebble-ctest-quant_f32_to_i8_test-baremetal", - "pebble-pebble-ctest-quant_i32_to_i8_test-baremetal", - "pebble-pebble-ctest-toint8_i8_rounding_test-baremetal", - "pebble-pebble-ctest-toint8_i8_rows_test-baremetal", - "pebble-pebble-ctest-toint8_i8_scale_rows_test-baremetal", - "pebble-pebble-ctest-toint8_i8_signed_test-baremetal", - "pebble-pebble-mlirtest-ball_quant_f32_to_i8-baremetal", - "pebble-pebble-mlirtest-ball_quant_i32_to_i8-baremetal", - "pebble-pebble-ctest-im2col_window_test-baremetal", - "pebble-pebble-ctest-im2col_k1_test-baremetal", - "pebble-pebble-ctest-im2col_k3_test-baremetal", - "pebble-pebble-ctest-im2col_k3_pad_test-baremetal", - "pebble-pebble-ctest-im2col_k3_asym_pad_test-baremetal", - "pebble-pebble-ctest-im2col_k3_partial_test-baremetal", - "pebble-pebble-ctest-im2col_k5_test-baremetal", - "pebble-pebble-ctest-im2col_k7_test-baremetal", - "pebble-pebble-mlirtest-ball_im2col_6x6_k3-baremetal", - "pebble-pebble-ctest-int2fp_tensor_values_test-baremetal", - "pebble-pebble-ctest-int2fp_tensor_groups_test-baremetal", - "pebble-pebble-ctest-int2fp_tensor_rows_test-baremetal", - "pebble-pebble-ctest-int2fp_channel_lanes_test-baremetal", - "pebble-pebble-ctest-int2fp_channel_base_test-baremetal", - "pebble-pebble-ctest-int2fp_channel_two_rows_test-baremetal", - "pebble-pebble-mlirtest-ball_int2fp_4x4-baremetal", - "pebble-pebble-mlirtest-ball_int2fp_relu_4x4-baremetal", - "pebble-pebble-ctest-lut_uniform_lookup_test-baremetal", - "pebble-pebble-ctest-lut_channel_table_lookup_test-baremetal", - "pebble-pebble-mlirtest-ball_lut_identity-baremetal", - "pebble-pebble-ctest-maxpool_window_test-baremetal", - "pebble-pebble-ctest-maxpool_offset_test-baremetal", - "pebble-pebble-mlirtest-ball_maxpool_6x6_s2-baremetal", - "pebble-pebble-ctest-int8add_scaled_test-baremetal", - "pebble-pebble-ctest-int8add_relu_test-baremetal", - "pebble-pebble-mlirtest-ball_int8add_relu-baremetal", - "pebble-pebble-ctest-int8mul_per_channel_gate_test-baremetal", - "pebble-pebble-mlirtest-ball_int8mul_gate-baremetal", - "pebble-pebble-ctest-matadd_i32_16row_test-baremetal", - "pebble-pebble-mlirtest-ball_matadd_i32-baremetal", -] diff --git a/examples/chips/pebble/regression/batch/bemu/workloads-linux.toml b/examples/chips/pebble/regression/batch/bemu/workloads-linux.toml deleted file mode 100644 index ba17ce202..000000000 --- a/examples/chips/pebble/regression/batch/bemu/workloads-linux.toml +++ /dev/null @@ -1,34 +0,0 @@ -[workloads] -tests = [ - "pebble-pebble-ctest-mvin_2d_test-linux", - "pebble-pebble-ctest-mvin_2d_pixel_stride_test-linux", - "pebble-pebble-ctest-mvin_mvout_acc_test-linux", - "pebble-pebble-ctest-mvin_mvout_test-linux", - "pebble-pebble-ctest-mvin_mvout_stride_test-linux", - "pebble-pebble-ctest-mvin_mvout_unaligned_test-linux", - "pebble-pebble-ctest-mvin_unaligned_test-linux", - "pebble-pebble-mlirtest-linalg_mega_conv2d_k3_c16-linux", - "pebble-pebble-mlirtest-tile_mega_conv2d_s2_depthwise_s2-linux", - "pebble-pebble-mlirtest-tile_mega_global_avg_conv_exact-linux", - "pebble-pebble-mlirtest-tile_mega_global_avg_conv2_exact-linux", - "pebble-pebble-mlirtest-tile_mega_global_avg_7x7_c288_exact-linux", - "pebble-pebble-mlirtest-tile_mega_se_gate_conv-linux", - "pebble-pebble-ctest-mega_conv_pipeline_test-linux", - "pebble-pebble-ctest-residual_add_lut_test-linux", - "pebble-pebble-ctest-maxpool_im2col_test-linux", - "pebble-pebble-ctest-se_gate_lut_test-linux", - "pebble-pebble-ctest-int8mul_large_gate_row_test-linux", - "pebble-pebble-ctest-dual_smatmul_matadd_test-linux", - "pebble-pebble-ctest-dual_conv_add_int2fp_test-linux", - "pebble-pebble-mlirtest-bank_matadd_i32_elementwise-linux", - "pebble-pebble-mlirtest-bank_int8add_identity_scale-linux", - "pebble-pebble-mlirtest-bank_lut_identity_table-linux", - "pebble-pebble-mlirtest-bank_int8mul_unit_gate-linux", - "pebble-pebble-mlirtest-bank_maxpool_k3_padded_spatial-linux", - "pebble-pebble-mlirtest-bank_smatmul_bias_two_terms-linux", - "pebble-pebble-mlirtest-bank_int2fp_unit_scale-linux", - "pebble-pebble-mlirtest-bank_quant_i32_pack16-linux", - "pebble-pebble-mlirtest-bank_quant_f32_pack16-linux", - "pebble-pebble-mlirtest-bank_im2col_k3_padded_bank_windows-linux", - "pebble-pebble-mlirtest-bank_transpose_i8_packed_matrix-linux", -] diff --git a/examples/chips/pebble/regression/batch/p2e/workloads-bare.toml b/examples/chips/pebble/regression/batch/p2e/workloads-bare.toml deleted file mode 100644 index 9dabc3631..000000000 --- a/examples/chips/pebble/regression/batch/p2e/workloads-bare.toml +++ /dev/null @@ -1,25 +0,0 @@ -[workloads] -tests = [ - "pebble-pebble-ctest-mvin_2d_pixel_stride_test-baremetal.hex", - "pebble-pebble-ctest-mega_conv_pipeline_test-baremetal.hex", - "pebble-pebble-mlirtest-tile_mega_conv2d_s2_depthwise_s2-baremetal.hex", - "pebble-pebble-mlirtest-tile_mega_se_gate_conv-baremetal.hex", - "pebble-pebble-mlirtest-tile_mega_global_avg_conv_exact-baremetal.hex", - "pebble-pebble-mlirtest-tile_mega_global_avg_conv2_exact-baremetal.hex", - "pebble-pebble-mlirtest-tile_mega_global_avg_7x7_c288_exact-baremetal.hex", - "pebble-pebble-ctest-residual_add_lut_test-baremetal.hex", - "pebble-pebble-ctest-maxpool_im2col_test-baremetal.hex", - "pebble-pebble-ctest-se_gate_lut_test-baremetal.hex", - "pebble-pebble-ctest-dual_conv_add_int2fp_test-baremetal.hex", - "pebble-pebble-mlirtest-bank_matadd_i32_elementwise-baremetal.hex", - "pebble-pebble-mlirtest-bank_int8add_identity_scale-baremetal.hex", - "pebble-pebble-mlirtest-bank_lut_identity_table-baremetal.hex", - "pebble-pebble-mlirtest-bank_int8mul_unit_gate-baremetal.hex", - "pebble-pebble-mlirtest-bank_maxpool_k3_padded_spatial-baremetal.hex", - "pebble-pebble-mlirtest-bank_smatmul_bias_two_terms-baremetal.hex", - "pebble-pebble-mlirtest-bank_int2fp_unit_scale-baremetal.hex", - "pebble-pebble-mlirtest-bank_quant_i32_pack16-baremetal.hex", - "pebble-pebble-mlirtest-bank_quant_f32_pack16-baremetal.hex", - "pebble-pebble-mlirtest-bank_im2col_k3_padded_bank_windows-baremetal.hex", - "pebble-pebble-mlirtest-bank_transpose_i8_packed_matrix-baremetal.hex", -] diff --git a/examples/chips/pebble/regression/batch/verilator/workloads-bare.toml b/examples/chips/pebble/regression/batch/verilator/workloads-bare.toml deleted file mode 100644 index 50f5fc9ab..000000000 --- a/examples/chips/pebble/regression/batch/verilator/workloads-bare.toml +++ /dev/null @@ -1,80 +0,0 @@ -[workloads] -tests = [ - "pebble-pebble-ctest-mvin_2d_test-baremetal", - "pebble-pebble-ctest-mvin_2d_pixel_stride_test-baremetal", - "pebble-pebble-ctest-mvin_mvout_acc_test-baremetal", - "pebble-pebble-ctest-mvin_mvout_test-baremetal", - "pebble-pebble-ctest-mvin_mvout_stride_test-baremetal", - "pebble-pebble-ctest-mvin_mvout_unaligned_test-baremetal", - "pebble-pebble-ctest-mvin_unaligned_test-baremetal", - "pebble-pebble-ctest-tlb_bare_mode_test-baremetal", - "pebble-pebble-ctest-tlb_repeated_sfence_test-baremetal", - "pebble-pebble-ctest-tlb_sfence_flush_test-baremetal", - "pebble-pebble-ctest-tlb_sfence_remap_test-baremetal", - "pebble-pebble-ctest-tlb_sv39_4k_alias_test-baremetal", - "pebble-pebble-ctest-tlb_sv39_identity_test-baremetal", - "pebble-pebble-ctest-tlb_sv39_nonidentity_test-baremetal", - "pebble-pebble-mlirtest-linalg_mega_conv2d_k3_c16-baremetal", - "pebble-pebble-mlirtest-tile_mega_conv2d_s2_depthwise_s2-baremetal", - "pebble-pebble-ctest-transpose_i8_16x16_test-baremetal", - "pebble-pebble-ctest-transpose_i8_16x32_test-baremetal", - "pebble-pebble-ctest-transpose_i32_8x16_test-baremetal", - "pebble-pebble-mlirtest-ball_transpose_16x16_i8-baremetal", - "pebble-pebble-ctest-smatmul_bias_accumulate_test-baremetal", - "pebble-pebble-ctest-smatmul_1x16x64_test-baremetal", - "pebble-pebble-ctest-smatmul_16x16_test-baremetal", - "pebble-pebble-ctest-smatmul_16x16x32_test-baremetal", - "pebble-pebble-ctest-smatmul_nondefault_banks_test-baremetal", - "pebble-pebble-ctest-smatmul_mvout_bank_reuse_test-baremetal", - "pebble-pebble-mlirtest-ball_smatmul_bias_accumulate-baremetal", - "pebble-pebble-ctest-quant_f32_to_i8_test-baremetal", - "pebble-pebble-ctest-quant_i32_to_i8_test-baremetal", - "pebble-pebble-ctest-toint8_i8_rounding_test-baremetal", - "pebble-pebble-ctest-toint8_i8_rows_test-baremetal", - "pebble-pebble-ctest-toint8_i8_scale_rows_test-baremetal", - "pebble-pebble-ctest-toint8_i8_signed_test-baremetal", - "pebble-pebble-mlirtest-ball_quant_f32_to_i8-baremetal", - "pebble-pebble-mlirtest-ball_quant_i32_to_i8-baremetal", - "pebble-pebble-ctest-im2col_window_test-baremetal", - "pebble-pebble-ctest-im2col_k1_test-baremetal", - "pebble-pebble-ctest-im2col_k3_test-baremetal", - "pebble-pebble-ctest-im2col_k3_pad_test-baremetal", - "pebble-pebble-ctest-im2col_k3_asym_pad_test-baremetal", - "pebble-pebble-ctest-im2col_k3_partial_test-baremetal", - "pebble-pebble-ctest-im2col_k5_test-baremetal", - "pebble-pebble-ctest-im2col_k7_test-baremetal", - "pebble-pebble-mlirtest-ball_im2col_6x6_k3-baremetal", - "pebble-pebble-ctest-int2fp_tensor_values_test-baremetal", - "pebble-pebble-ctest-int2fp_tensor_groups_test-baremetal", - "pebble-pebble-ctest-int2fp_tensor_rows_test-baremetal", - "pebble-pebble-ctest-int2fp_channel_lanes_test-baremetal", - "pebble-pebble-ctest-int2fp_channel_base_test-baremetal", - "pebble-pebble-ctest-int2fp_channel_two_rows_test-baremetal", - "pebble-pebble-mlirtest-ball_int2fp_4x4-baremetal", - "pebble-pebble-mlirtest-ball_int2fp_relu_4x4-baremetal", - "pebble-pebble-ctest-lut_uniform_lookup_test-baremetal", - "pebble-pebble-ctest-lut_channel_table_lookup_test-baremetal", - "pebble-pebble-mlirtest-ball_lut_identity-baremetal", - "pebble-pebble-ctest-maxpool_window_test-baremetal", - "pebble-pebble-ctest-maxpool_offset_test-baremetal", - "pebble-pebble-mlirtest-ball_maxpool_6x6_s2-baremetal", - "pebble-pebble-ctest-int8add_scaled_test-baremetal", - "pebble-pebble-ctest-int8add_relu_test-baremetal", - "pebble-pebble-mlirtest-ball_int8add_relu-baremetal", - "pebble-pebble-ctest-int8mul_per_channel_gate_test-baremetal", - "pebble-pebble-ctest-int8mul_large_gate_row_test-baremetal", - "pebble-pebble-mlirtest-ball_int8mul_gate-baremetal", - "pebble-pebble-ctest-matadd_i32_16row_test-baremetal", - "pebble-pebble-mlirtest-ball_matadd_i32-baremetal", - "pebble-pebble-ctest-mega_conv_pipeline_test-baremetal", - "pebble-pebble-mlirtest-tile_mega_se_gate_conv-baremetal", - "pebble-pebble-mlirtest-tile_mega_global_avg_conv_exact-baremetal", - "pebble-pebble-mlirtest-tile_mega_global_avg_conv2_exact-baremetal", - "pebble-pebble-mlirtest-tile_mega_global_avg_7x7_c288_exact-baremetal", - "pebble-pebble-ctest-residual_add_lut_test-baremetal", - "pebble-pebble-ctest-maxpool_im2col_test-baremetal", - "pebble-pebble-ctest-se_gate_lut_test-baremetal", - "pebble-pebble-ctest-dual_smatmul_matadd_test-baremetal", - "pebble-pebble-ctest-dual_conv_add_int2fp_test-baremetal", - "pebble-pebble-ctest-quant_smatmul_int2fp_test-baremetal", -] diff --git a/examples/chips/pebble/regression/batch/verilator/workloads-linux.toml b/examples/chips/pebble/regression/batch/verilator/workloads-linux.toml deleted file mode 100644 index 1d5d6ac34..000000000 --- a/examples/chips/pebble/regression/batch/verilator/workloads-linux.toml +++ /dev/null @@ -1,69 +0,0 @@ -[workloads] -tests = [ - "pebble-pebble-ctest-mvin_2d_test-linux", - "pebble-pebble-ctest-mvin_2d_pixel_stride_test-linux", - "pebble-pebble-ctest-mvin_mvout_acc_test-linux", - "pebble-pebble-ctest-mvin_mvout_test-linux", - "pebble-pebble-ctest-mvin_mvout_stride_test-linux", - "pebble-pebble-ctest-mvin_mvout_unaligned_test-linux", - "pebble-pebble-ctest-mvin_unaligned_test-linux", - "pebble-pebble-ctest-transpose_i8_16x16_test-linux", - "pebble-pebble-ctest-transpose_i8_16x32_test-linux", - "pebble-pebble-ctest-transpose_i32_8x16_test-linux", - "pebble-pebble-mlirtest-ball_transpose_16x16_i8-linux", - "pebble-pebble-ctest-smatmul_bias_accumulate_test-linux", - "pebble-pebble-ctest-smatmul_1x16x64_test-linux", - "pebble-pebble-ctest-smatmul_16x16_test-linux", - "pebble-pebble-ctest-smatmul_16x16x32_test-linux", - "pebble-pebble-ctest-smatmul_nondefault_banks_test-linux", - "pebble-pebble-mlirtest-ball_smatmul_bias_accumulate-linux", - "pebble-pebble-ctest-quant_f32_to_i8_test-linux", - "pebble-pebble-ctest-quant_i32_to_i8_test-linux", - "pebble-pebble-ctest-toint8_i8_rounding_test-linux", - "pebble-pebble-ctest-toint8_i8_rows_test-linux", - "pebble-pebble-ctest-toint8_i8_scale_rows_test-linux", - "pebble-pebble-ctest-toint8_i8_signed_test-linux", - "pebble-pebble-mlirtest-ball_quant_f32_to_i8-linux", - "pebble-pebble-mlirtest-ball_quant_i32_to_i8-linux", - "pebble-pebble-ctest-im2col_window_test-linux", - "pebble-pebble-ctest-im2col_k1_test-linux", - "pebble-pebble-ctest-im2col_k3_test-linux", - "pebble-pebble-ctest-im2col_k3_pad_test-linux", - "pebble-pebble-ctest-im2col_k3_asym_pad_test-linux", - "pebble-pebble-ctest-im2col_k3_partial_test-linux", - "pebble-pebble-ctest-im2col_k5_test-linux", - "pebble-pebble-ctest-im2col_k7_test-linux", - "pebble-pebble-mlirtest-ball_im2col_6x6_k3-linux", - "pebble-pebble-ctest-int2fp_tensor_values_test-linux", - "pebble-pebble-ctest-int2fp_tensor_groups_test-linux", - "pebble-pebble-ctest-int2fp_tensor_rows_test-linux", - "pebble-pebble-ctest-int2fp_channel_lanes_test-linux", - "pebble-pebble-ctest-int2fp_channel_base_test-linux", - "pebble-pebble-ctest-int2fp_channel_two_rows_test-linux", - "pebble-pebble-mlirtest-ball_int2fp_4x4-linux", - "pebble-pebble-mlirtest-ball_int2fp_relu_4x4-linux", - "pebble-pebble-ctest-lut_uniform_lookup_test-linux", - "pebble-pebble-ctest-lut_channel_table_lookup_test-linux", - "pebble-pebble-mlirtest-ball_lut_identity-linux", - "pebble-pebble-ctest-maxpool_window_test-linux", - "pebble-pebble-ctest-maxpool_offset_test-linux", - "pebble-pebble-mlirtest-ball_maxpool_6x6_s2-linux", - "pebble-pebble-ctest-int8add_scaled_test-linux", - "pebble-pebble-ctest-int8add_relu_test-linux", - "pebble-pebble-mlirtest-ball_int8add_relu-linux", - "pebble-pebble-ctest-int8mul_per_channel_gate_test-linux", - "pebble-pebble-mlirtest-ball_int8mul_gate-linux", - "pebble-pebble-ctest-matadd_i32_16row_test-linux", - "pebble-pebble-mlirtest-ball_matadd_i32-linux", - "pebble-pebble-ctest-mega_conv_pipeline_test-linux", - "pebble-pebble-mlirtest-tile_mega_conv2d_s2_depthwise_s2-linux", - "pebble-pebble-mlirtest-tile_mega_se_gate_conv-linux", - "pebble-pebble-mlirtest-tile_mega_global_avg_conv_exact-linux", - "pebble-pebble-mlirtest-tile_mega_global_avg_conv2_exact-linux", - "pebble-pebble-mlirtest-tile_mega_global_avg_7x7_c288_exact-linux", - "pebble-pebble-ctest-residual_add_lut_test-linux", - "pebble-pebble-ctest-maxpool_im2col_test-linux", - "pebble-pebble-ctest-se_gate_lut_test-linux", - "pebble-pebble-ctest-dual_smatmul_matadd_test-linux", - "pebble-pebble-ctest-dual_conv_add_int2fp_test-linux", -] diff --git a/examples/chips/pebble/regression/soc/README.md b/examples/chips/pebble/regression/soc/README.md deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/chips/pebble/tapeout/README.md b/examples/chips/pebble/tapeout/README.md deleted file mode 100644 index 2b9f1a5b8..000000000 --- a/examples/chips/pebble/tapeout/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# Pebble tapeout contract - -Layout: -- `config.toml` — chip contract (scripts, libs, power window) -- `area/` — DC (`dc.tcl`, `constraints.sdc`) -- `power/` — PTPX + gate sim (`power.tcl`, `power_sim.sh`) -- `ip/` — SRAM MDF - -`bbdev dc` resolves paths from `config.toml` and injects `RUN_*`. Timing truth is -`area/constraints.sdc`. Pebble main clock is 100 MHz (`10 ns`). diff --git a/examples/chips/pebble/tapeout/area/constraints.sdc b/examples/chips/pebble/tapeout/area/constraints.sdc deleted file mode 100644 index a1877a3a3..000000000 --- a/examples/chips/pebble/tapeout/area/constraints.sdc +++ /dev/null @@ -1,93 +0,0 @@ -set core_clock [get_ports auto_chipyard_prcictrl_domain_reset_setter_clock_in_member_allClocks_uncore_clock] -set debug_clock [get_ports debug_clock] -set jtag_clock [get_ports debug_systemjtag_jtag_TCK] -set serial_tl_clock [get_ports serial_tl_0_clock_in] -set jtag_inputs [get_ports {debug_systemjtag_jtag_TMS debug_systemjtag_jtag_TDI}] -set jtag_outputs [get_ports debug_systemjtag_jtag_TDO_data] -set serial_tl_inputs [get_ports {serial_tl_0_in_valid serial_tl_0_in_bits_phit* serial_tl_0_out_ready}] -set serial_tl_outputs [get_ports {serial_tl_0_in_ready serial_tl_0_out_valid serial_tl_0_out_bits_phit*}] -set core_inputs [get_ports { - debug_dmactiveAck - custom_boot - i2c_0_scl_in - i2c_0_sda_in - uart_0_rxd - gpio_0_pins_0_i_ival - gpio_0_pins_1_i_ival - gpio_0_pins_2_i_ival - gpio_0_pins_3_i_ival - gpio_0_pins_4_i_ival - gpio_0_pins_5_i_ival - gpio_0_pins_6_i_ival - gpio_0_pins_7_i_ival - spi_0_dq_0_i - spi_0_dq_1_i - spi_0_dq_2_i - spi_0_dq_3_i -}] -set core_outputs [get_ports { - i2c_0_scl_oe - i2c_0_sda_oe - uart_0_txd - gpio_0_pins_0_o_oval - gpio_0_pins_0_o_oe - gpio_0_pins_0_o_ie - gpio_0_pins_1_o_oval - gpio_0_pins_1_o_oe - gpio_0_pins_1_o_ie - gpio_0_pins_2_o_oval - gpio_0_pins_2_o_oe - gpio_0_pins_2_o_ie - gpio_0_pins_3_o_oval - gpio_0_pins_3_o_oe - gpio_0_pins_3_o_ie - gpio_0_pins_4_o_oval - gpio_0_pins_4_o_oe - gpio_0_pins_4_o_ie - gpio_0_pins_5_o_oval - gpio_0_pins_5_o_oe - gpio_0_pins_5_o_ie - gpio_0_pins_6_o_oval - gpio_0_pins_6_o_oe - gpio_0_pins_6_o_ie - gpio_0_pins_7_o_oval - gpio_0_pins_7_o_oe - gpio_0_pins_7_o_ie - spi_0_sck - spi_0_dq_0_o - spi_0_dq_0_ie - spi_0_dq_0_oe - spi_0_dq_1_o - spi_0_dq_1_ie - spi_0_dq_1_oe - spi_0_dq_2_o - spi_0_dq_2_ie - spi_0_dq_2_oe - spi_0_dq_3_o - spi_0_dq_3_ie - spi_0_dq_3_oe - spi_0_cs_0 -}] - -create_clock -name core -period 10.0 $core_clock -create_clock -name debug -period 10.0 $debug_clock -create_clock -name jtag -period 100.0 $jtag_clock -create_clock -name serial_tl -period 10.0 $serial_tl_clock -set_clock_groups -asynchronous \ - -group [get_clocks core] \ - -group [get_clocks debug] \ - -group [get_clocks jtag] \ - -group [get_clocks serial_tl] - -set_clock_uncertainty -setup 3.0 [get_clocks {core debug serial_tl}] -set_clock_uncertainty -setup 10.0 [get_clocks jtag] -set_clock_transition 1.0 [get_clocks {core debug serial_tl}] -set_clock_transition 10.0 [get_clocks jtag] - -set_input_delay 3.0 -clock core $core_inputs -set_output_delay 3.0 -clock core $core_outputs - -set_input_delay 20.0 -clock_fall -clock jtag $jtag_inputs -set_output_delay 20.0 -clock jtag $jtag_outputs -set_input_delay 3.0 -clock serial_tl $serial_tl_inputs -set_output_delay 3.0 -clock serial_tl $serial_tl_outputs diff --git a/examples/chips/pebble/tapeout/area/dc.tcl b/examples/chips/pebble/tapeout/area/dc.tcl deleted file mode 100755 index c4fbc56ac..000000000 --- a/examples/chips/pebble/tapeout/area/dc.tcl +++ /dev/null @@ -1,65 +0,0 @@ -# Chip-owned DC flow. Timing comes from constraints.sdc (RUN_SDC). -# bbdev injects RUN_* via -x {set RUN_CONFIG }. -if {![info exists RUN_CONFIG] || $RUN_CONFIG eq ""} { - error "bbdev must pass -x {set RUN_CONFIG }" -} -source [file normalize $RUN_CONFIG] - -proc bbdev_read_filelist {path} { - set fh [open $path r] - set files [list] - while {[gets $fh line] >= 0} { - set line [string trim $line] - if {$line eq "" || [string match "#*" $line]} { continue } - lappend files [file normalize $line] - } - close $fh - if {[llength $files] == 0} { error "empty DC source list: $path" } - return $files -} - -set top $RUN_TOP -set output_dir [file normalize $RUN_OUTPUT_DIR] -set report_dir [file normalize $RUN_REPORT_DIR] -file mkdir $output_dir $report_dir [file join $report_dir work] -set target_library [list $RUN_TARGET_LIBRARY] -set synthetic_library $RUN_SYNTHETIC_LIBRARY -set link_library [concat [list *] $target_library $synthetic_library $RUN_LINK_LIBRARY] -set_host_options -max_cores $RUN_MAX_CORES -define_design_lib WORK -path [file join $report_dir work] -set search_path [list .] -set_app_var verilogout_no_tri true -set_app_var verilogout_equation false -set_app_var compile_seqmap_propagate_constants false -set_app_var hdlin_ff_always_sync_set_reset true -analyze -format sverilog -define {SYNTHESIS DC_SYN} [bbdev_read_filelist $RUN_SOURCE_LIST] -elaborate $top -current_design $top -link -set sram_cells [get_cells -hierarchical -quiet -filter {ref_name =~ "chipyard_sram_*"}] -if {[sizeof_collection $sram_cells] == 0} { - error "no chipyard_sram_* cells after link; SRAM .db link_library missing" -} -set_dont_touch $sram_cells -set sram_designs [get_designs -quiet chipyard_sram_*] -if {[sizeof_collection $sram_designs] > 0} { - set_dont_touch $sram_designs -} -read_sdc [file normalize $RUN_SDC] -if {[sizeof_collection [all_clocks]] == 0} { - error "SDC created no clocks" -} -set_load 2.0 [all_outputs] -set_app_var compile_seqmap_honor_sync_set_reset true -compile_ultra -area_high_effort_script -no_autoungroup -no_boundary_optimization -set_fix_multiple_port_nets -all -buffer_constants -change_names -hierarchy -rules verilog -write -format ddc -hierarchy -output [file join $output_dir ${top}.ddc] -write -format verilog -hierarchy -output [file join $output_dir ${top}.v] -write_sdc [file join $output_dir ${top}.sdc] -report_constraint -all_violators > [file join $report_dir constraint.rpt] -report_timing -delay max -max_paths 50 > [file join $report_dir timing_max.rpt] -report_timing -delay min -max_paths 50 > [file join $report_dir timing_min.rpt] -report_area -hierarchy > [file join $report_dir area.rpt] -report_reference > [file join $report_dir reference.rpt] -exit diff --git a/examples/chips/pebble/tapeout/config.toml b/examples/chips/pebble/tapeout/config.toml deleted file mode 100644 index a1bb97799..000000000 --- a/examples/chips/pebble/tapeout/config.toml +++ /dev/null @@ -1,117 +0,0 @@ -[tapeout] -top = "DigitalTop" -sram_process = "smic180" -sram_mdf = "ip/smic180_sram_library.mdf.json" -constraints_sdc = "area/constraints.sdc" -target_library = "/home/bb-runner/Code/eda/D_libraries/SMIC180/SCC018UG_UHD_RVT_V0.4a/SCC018UG_UHD_RVT_V0p4a/liberty/1.8v/scc018ug_uhd_rvt_ss_v1p62_125c_ccs.db" -synthetic_library = [ - "/home/bb-runner/Code/eda/A_packages/tools/syn/R-2020.09-SP5/libraries/syn/dw_foundation.sldb", - "/home/bb-runner/Code/eda/A_packages/tools/syn/R-2020.09-SP5/libraries/syn/standard.sldb", -] -link_library = [] -max_cores = 16 -power_format = "vcd" -power_start_ns = "1000000" -power_end_ns = "5000000" -dc_script = "area/dc.tcl" -power_script = "power/power.tcl" -power_sim_script = "power/power_sim.sh" -power_strip_path = "GatePowerHarness.dut" - -sram_corner = "ss" -lc_shell = "/home/bb-runner/Code/eda/A_packages/tools/lc/R-2020.09-SP5/bin/lc_shell" - -[[tapeout.sram]] -name = "chipyard_sram_32x128" -words = 32 -mux = 4 -bits = 128 - -[[tapeout.sram]] -name = "chipyard_sram_32x22" -words = 32 -mux = 4 -bits = 22 - -[[tapeout.sram]] -name = "chipyard_sram_64x21" -words = 64 -mux = 8 -bits = 21 - -[[tapeout.sram]] -name = "chipyard_sram_64x22" -words = 64 -mux = 8 -bits = 22 - -[[tapeout.sram]] -name = "chipyard_sram_512x8" -words = 512 -mux = 8 -bits = 8 - -[[tapeout.sram]] -name = "chipyard_sram_512x32" -words = 512 -mux = 8 -bits = 32 - -[[tapeout.sram]] -name = "chipyard_sram_512x64" -words = 512 -mux = 8 -bits = 64 - -[[tapeout.sram]] -name = "chipyard_sram_1024x8" -words = 1024 -mux = 8 -bits = 8 - -[[tapeout.sram]] -name = "chipyard_sram_32x21" -words = 32 -mux = 4 -bits = 21 - -[[tapeout.sram]] -name = "chipyard_sram_32x23" -words = 32 -mux = 4 -bits = 23 - -[[tapeout.sram]] -name = "chipyard_sram_64x8" -words = 64 -mux = 8 -bits = 8 - -[[tapeout.sram]] -name = "chipyard_sram_64x128" -words = 64 -# SMIC180 has only six metal layers, so this macro cannot be treated as a -# routable-over cell. mux=4 changes only the internal column organization: -# it keeps the 64x128 1RW + BWEN interface but shrinks the generated macro -# width from about 2013 um (mux=8) to about 1045 um, avoiding long routing walls. -mux = 4 -bits = 128 -bitwrite = true - -[[tapeout.sram]] -name = "chipyard_sram_128x8" -words = 128 -mux = 8 -bits = 8 - -[[tapeout.sram]] -name = "chipyard_sram_256x32" -words = 256 -mux = 8 -bits = 32 - -[[tapeout.sram]] -name = "chipyard_sram_256x64" -words = 256 -mux = 8 -bits = 64 diff --git a/examples/chips/pebble/workloads/CMakeLists.txt b/examples/chips/pebble/workloads/CMakeLists.txt deleted file mode 100644 index c743dec9c..000000000 --- a/examples/chips/pebble/workloads/CMakeLists.txt +++ /dev/null @@ -1,25 +0,0 @@ -get_filename_component(BUCKYBALL_REPO_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../../.. ABSOLUTE) - -set(BUCKYBALL_CHIP_COMMON_SOURCES - ${CMAKE_CURRENT_SOURCE_DIR}/common/pebble.c) -set(BUCKYBALL_CHIP_COMMON_HEADERS - ${CMAKE_CURRENT_SOURCE_DIR}/common/pebble.h) -set(BUCKYBALL_CHIP_COMMON_INCLUDE_DIRS - ${CMAKE_CURRENT_SOURCE_DIR}/common) -set(BUCKYBALL_CTEST_TARGET pebble) - -add_custom_target(pebble-ctests-build) -set(BUCKYBALL_CTEST_GROUP_TARGET pebble-ctests-build) - -add_subdirectory(ctests) -buckyball_add_ball_workload_subdirs(ctests) - -add_custom_target(chip-ctests-build DEPENDS pebble-ctests-build) - -add_custom_target(chip-workloads-build ALL - DEPENDS - chip-ctests-build - mlirtest-workloads-build - soctests-build - COMMENT "Building pebble workloads" - VERBATIM) diff --git a/examples/chips/pebble/workloads/ctests/CMakeLists.txt b/examples/chips/pebble/workloads/ctests/CMakeLists.txt deleted file mode 100644 index 575003eea..000000000 --- a/examples/chips/pebble/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -add_buckyball_ctests( - mega_conv_pipeline_test.c - mega_conv_k5_pipeline_test.c - residual_add_lut_test.c - maxpool_im2col_test.c - se_gate_lut_test.c - dual_smatmul_matadd_test.c - dual_conv_add_int2fp_test.c - quant_smatmul_int2fp_test.c - smatmul_mvout_bank_reuse_test.c - mvin_bank78_stress_test.c - im2col_mvin_smatmul_hazard_test.c -) diff --git a/examples/chips/pebble/workloads/ctests/dual_conv_add_int2fp_test.c b/examples/chips/pebble/workloads/ctests/dual_conv_add_int2fp_test.c deleted file mode 100644 index 199edd879..000000000 --- a/examples/chips/pebble/workloads/ctests/dual_conv_add_int2fp_test.c +++ /dev/null @@ -1,80 +0,0 @@ -#include "buckyball.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include - -enum { - TILE = 16, - IN = 4, - K = 3, - OUT = 2, - WINDOWS = 4, - PACKED_K = 16, - RESULT_ROWS = TILE * 2 -}; -static int8_t images[2][IN * IN * TILE] __attribute__((aligned(64))); -static int8_t weights[2][PACKED_K * TILE] __attribute__((aligned(64))); -static float scale[TILE] __attribute__((aligned(64))); -static int32_t bias[TILE] __attribute__((aligned(64))); -static float output[TILE * TILE] __attribute__((aligned(64))); - -static int8_t image_value(int branch, int row, int column) { - return (int8_t)(branch * 7 + row * IN + column - 8); -} - -static int32_t expected(int row, int column) { - int32_t sum = 0; - for (int branch = 0; branch < 2; ++branch) - for (int ky = 0; ky < K; ++ky) - for (int kx = 0; kx < K; ++kx) - sum += image_value(branch, row / OUT + ky, row % OUT + kx) * - weights[branch][(ky * K + kx) * TILE + column]; - return sum; -} - -static void run_branch(int branch, int out_bank) { - bb_mvin((uintptr_t)images[branch], 0, IN * IN, 1); - bb_im2col(0, 1, IN, K, 1, 0, 0, 0, 0, 0, 0, WINDOWS); - bb_mvin((uintptr_t)weights[branch], 0, PACKED_K, 1); - bb_smatmul_os(1, 0, out_bank, TILE, TILE, PACKED_K, 1, 1, 0); -} - -int main(void) { - for (int branch = 0; branch < 2; ++branch) { - for (int row = 0; row < IN; ++row) - for (int column = 0; column < IN; ++column) - images[branch][(row * IN + column) * TILE] = - image_value(branch, row, column); - for (int i = 0; i < PACKED_K * TILE; ++i) - weights[branch][i] = (int8_t)((i / TILE * 3 + i % TILE) % 41 - 20); - } - for (int lane = 0; lane < TILE; ++lane) - scale[lane] = 0.05f; - - for (int bank = 0; bank < 8; ++bank) - bb_mem_alloc(bank, 1, 1); - bb_mvin((uintptr_t)scale, 3, 4, 1); - bb_mvin((uintptr_t)bias, 7, 4, 1); - bb_smatmul_bias(7, 0); - run_branch(0, 2); - run_branch(1, 4); - bb_matadd(2, 4, 6, RESULT_ROWS); - bb_int32_to_fp32(6, 3, 5, RESULT_ROWS, 0); - bb_mvout((uintptr_t)output, 5, RESULT_ROWS, 1); - - for (int row = 0; row < WINDOWS; ++row) - for (int column = 0; column < TILE; ++column) - if (fabsf(output[row * TILE + column] - - expected(row, column) * scale[column]) > 1e-3f) { - printf("dual conv add mismatch row=%d column=%d\n", row, column); - return 1; - } - printf("dual conv add int2fp PASS\n"); - return 0; -} diff --git a/examples/chips/pebble/workloads/ctests/mega_conv_k5_pipeline_test.c b/examples/chips/pebble/workloads/ctests/mega_conv_k5_pipeline_test.c deleted file mode 100644 index 400d2a7dd..000000000 --- a/examples/chips/pebble/workloads/ctests/mega_conv_k5_pipeline_test.c +++ /dev/null @@ -1,70 +0,0 @@ -#include "buckyball.h" -#include -#include -#include -#include -#include -#include -#include -#include - -enum { INPUT_SIDE = 8, OUTPUT_SIDE = 4, KERNEL = 5, TILE = 16, PADDED_K = 32 }; -static int8_t input[INPUT_SIDE * INPUT_SIDE * TILE] - __attribute__((aligned(64))); -static int8_t weight[PADDED_K * TILE] __attribute__((aligned(64))); -static int32_t bias[TILE] __attribute__((aligned(64))); -static float scale[TILE] __attribute__((aligned(64))); -static int8_t output[OUTPUT_SIDE * OUTPUT_SIDE * TILE] - __attribute__((aligned(64))); - -int main(void) { - for (int pixel = 0; pixel < INPUT_SIDE * INPUT_SIDE; ++pixel) - input[pixel * TILE] = pixel % 5 - 2; - for (int k = 0; k < KERNEL * KERNEL; ++k) - for (int channel = 0; channel < TILE; ++channel) - weight[k * TILE + channel] = (k + channel) % 3 - 1; - for (int channel = 0; channel < TILE; ++channel) - scale[channel] = 1.0f; - - for (int bank = 0; bank < 8; ++bank) - bb_mem_alloc(bank, 1, 1); - bb_mvin((uintptr_t)input, 0, INPUT_SIDE * INPUT_SIDE, 1); - bb_mvin((uintptr_t)weight, 2, PADDED_K, 1); - bb_mvin((uintptr_t)bias, 3, 4, 1); - bb_mvin((uintptr_t)scale, 5, 4, 1); - bb_im2col(0, 1, INPUT_SIDE, KERNEL, 1, 0, 0, 0, 0, 0, 0, - OUTPUT_SIDE * OUTPUT_SIDE); - for (int row_tile = 0; row_tile < 2; ++row_tile) - for (int k_tile = 0; k_tile < 2; ++k_tile) - for (int row = 0; row < 8; ++row) { - int input_row = k_tile * 16 + row_tile * 8 + row; - int output_row = (row_tile * 2 + k_tile) * 8 + row; - bb_maxpool(1, 7, 1, 1, 1, 1, 0, input_row, output_row, 1, 0, 0); - } - bb_smatmul_bias(3, 0); - bb_smatmul_os(7, 2, 4, OUTPUT_SIDE * OUTPUT_SIDE, TILE, PADDED_K, 1, 1, 0); - bb_quant_i32_to_i8(4, 5, 6, OUTPUT_SIDE * OUTPUT_SIDE * 4, 0, 0, OUTPUT_SIDE, - OUTPUT_SIDE, OUTPUT_SIDE, 0); - bb_mvout((uintptr_t)output, 6, OUTPUT_SIDE * OUTPUT_SIDE, 1); - - for (int y = 0; y < OUTPUT_SIDE; ++y) - for (int x = 0; x < OUTPUT_SIDE; ++x) - for (int channel = 0; channel < TILE; ++channel) { - int expected = 0; - for (int ky = 0; ky < KERNEL; ++ky) - for (int kx = 0; kx < KERNEL; ++kx) { - int pixel = (y + ky) * INPUT_SIDE + x + kx; - int k = ky * KERNEL + kx; - expected += input[pixel * TILE] * weight[k * TILE + channel]; - } - int index = (y * OUTPUT_SIDE + x) * TILE + channel; - if (output[index] != expected) { - printf( - "mega_conv_k5 FAIL y=%d x=%d channel=%d expected=%d actual=%d\n", - y, x, channel, expected, output[index]); - return 1; - } - } - printf("mega_conv_k5 PASS\n"); - return 0; -} diff --git a/examples/chips/pebble/workloads/mlir_tests/CMakeLists.txt b/examples/chips/pebble/workloads/mlir_tests/CMakeLists.txt deleted file mode 100644 index ffedb96f5..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -add_custom_target(pebble-mlir-tests-build) - -add_subdirectory(bank) -add_subdirectory(tile) -add_subdirectory(linalg) - -add_dependencies(chip-mlirtests-build pebble-mlir-tests-build) diff --git a/examples/chips/pebble/workloads/mlir_tests/bank/quant_im2col_k1_single_pixel.mlir b/examples/chips/pebble/workloads/mlir_tests/bank/quant_im2col_k1_single_pixel.mlir deleted file mode 100644 index 53bc9a0d4..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/bank/quant_im2col_k1_single_pixel.mlir +++ /dev/null @@ -1,46 +0,0 @@ -// Chip integration: f32-to-i8 quantization -> im2col k=1 -> mvout. - -func.func private @check_result(memref<16x16xi8>) -> () - -memref.global "private" constant @fp_in : memref<1x16xf32> = - dense<[[1.0, 2.0, 3.0, 4.0, 0.0, 0.0, 0.0, 0.0, - 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]]> - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %depth_in = arith.constant 1 : i64 - %depth_out = arith.constant 16 : i64 - %stride = arith.constant 1 : i64 - %quant_iter = arith.constant 4 : i64 - %input_size = arith.constant 1 : i64 - %scale = arith.constant 1.0 : f32 - %da_addr = arith.constant 0 : i64 - %ksize = arith.constant 1 : i64 - %pad = arith.constant 0 : i64 - - %input = memref.get_global @fp_in : memref<1x16xf32> - %output = memref.alloc() alignment = 64 : memref<16x16xi8> - - %bin = buckyball.bank_alloc - %bq = buckyball.bank_alloc - %bout = buckyball.bank_alloc - %loaded = buckyball.bank_mvin %input %bin %depth_in %stride - : memref<1x16xf32> i64 i64 i64 - %q = buckyball.bank_quant_f32_to_i8 %loaded %bq %quant_iter %scale - : i64 i64 i64 f32 - %base = arith.constant 0 : i64 - %lane = arith.constant 0 : i64 - %im = buckyball.bank_im2col %q %bout %input_size %ksize %stride %pad %base - %lane - : i64 i64 i64 i64 i64 i64 i64 i64 - %stored = buckyball.bank_mvout %output %im %depth_out %stride - : memref<16x16xi8> i64 i64 i64 - buckyball.bank_release %loaded : i64 - buckyball.bank_release %q : i64 - buckyball.bank_release %stored : i64 - - func.call @check_result(%output) : (memref<16x16xi8>) -> () - memref.dealloc %output : memref<16x16xi8> - return %zero_i8 : i8 -} diff --git a/examples/chips/pebble/workloads/mlir_tests/bank/quant_im2col_k1_single_pixel_main.cpp b/examples/chips/pebble/workloads/mlir_tests/bank/quant_im2col_k1_single_pixel_main.cpp deleted file mode 100644 index 6d3a50db5..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/bank/quant_im2col_k1_single_pixel_main.cpp +++ /dev/null @@ -1,38 +0,0 @@ -#include -#include -#include - -static void fail(void) { -#ifdef BAREMETAL - volatile uint32_t *sim_exit = (volatile uint32_t *)0x60000000; - *sim_exit = 1; - while (1) { - } -#else - exit(1); -#endif -} - -#ifdef __cplusplus -extern "C" -#endif - void check_result(int8_t *allocated, int8_t *aligned, int64_t offset, - int64_t size0, int64_t size1, int64_t stride0, - int64_t stride1) { - (void)allocated; - if (size0 != 16 || size1 != 16 || stride0 != 16 || stride1 != 1) { - printf( - "FAILED: quant_im2col_k1_single_pixel shape %ldx%ld stride %ldx%ld\n", - (long)size0, (long)size1, (long)stride0, (long)stride1); - fail(); - } - - int8_t *out = aligned + offset; - int32_t got = out[0 * stride0 + 0 * stride1]; - if (got != 1) { - printf("FAILED: quant_im2col_k1_single_pixel out[0][0] exp=1 got=%d\n", - (int)got); - fail(); - } - printf("PASSED: quant_im2col_k1_single_pixel\n"); -} diff --git a/examples/chips/pebble/workloads/mlir_tests/linalg/CMakeLists.txt b/examples/chips/pebble/workloads/mlir_tests/linalg/CMakeLists.txt deleted file mode 100644 index 73893c006..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/linalg/CMakeLists.txt +++ /dev/null @@ -1,33 +0,0 @@ -set(BUCKYBALL_MLIR_GROUP_TARGET pebble-mlir-tests-build) -set(BUCKYBALL_MLIR_TEST_PREFIX linalg) - -set(PEBBLE_LINALG_FUNCTIONAL_PASSES - -convert-linalg-to-tile - -convert-tile-to-buckyball - -lower-buckyball-to-bank-ssa - -assign-physical-banks - ${BUCKYBALL_LOWER_BANK_SSA_TO_INTRINSICS} - -convert-linalg-to-loops - -expand-strided-metadata - -lower-affine - -convert-scf-to-cf - -convert-cf-to-llvm - ${BUCKYBALL_LOWER_BUCKYBALL} - -convert-arith-to-llvm - -convert-math-to-llvm - -finalize-memref-to-llvm - -convert-func-to-llvm - -reconcile-unrealized-casts) - -foreach(test - linalg_relu_513x17_i32 - linalg_matmul_16x16_i8 - linalg_transpose_16x16_i8 - linalg_transpose_detect_1x84x100_f32 - linalg_transpose_nchw_nhwc_1x3x8x8_f32 - linalg_transpose_oihw_ohwi_4x8x3x3_f32 - linalg_mega_conv2d_k3_c16) - add_buckyball_mlir_test(${test} - TARGET pebble - PASSES ${PEBBLE_LINALG_FUNCTIONAL_PASSES}) -endforeach() diff --git a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_matmul_16x16_i8.mlir b/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_matmul_16x16_i8.mlir deleted file mode 100644 index e92cd41c6..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_matmul_16x16_i8.mlir +++ /dev/null @@ -1,44 +0,0 @@ -func.func private @check_result(memref<16x16xi32>) -> () - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero_i32 = arith.constant 0 : i32 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %two = arith.constant 2 : index - %three = arith.constant 3 : index - %five = arith.constant 5 : index - %seven = arith.constant 7 : index - %dim = arith.constant 16 : index - %a = memref.alloc() alignment = 64 : memref<16x16xi8> - %b = memref.alloc() alignment = 64 : memref<16x16xi8> - %c = memref.alloc() alignment = 64 : memref<16x16xi32> - - scf.for %row = %zero to %dim step %one { - scf.for %column = %zero to %dim step %one { - %r3 = arith.muli %row, %three : index - %c5 = arith.muli %column, %five : index - %asum = arith.addi %r3, %c5 : index - %arem = arith.remui %asum, %seven : index - %aval = arith.subi %arem, %three : index - %av = arith.index_cast %aval : index to i8 - %r2 = arith.muli %row, %two : index - %c3 = arith.muli %column, %three : index - %bsum = arith.addi %r2, %c3 : index - %brem = arith.remui %bsum, %five : index - %bval = arith.subi %brem, %two : index - %bv = arith.index_cast %bval : index to i8 - memref.store %av, %a[%row, %column] : memref<16x16xi8> - memref.store %bv, %b[%row, %column] : memref<16x16xi8> - } - } - linalg.fill ins(%zero_i32 : i32) outs(%c : memref<16x16xi32>) - linalg.matmul ins(%a, %b : memref<16x16xi8>, memref<16x16xi8>) - outs(%c : memref<16x16xi32>) - - func.call @check_result(%c) : (memref<16x16xi32>) -> () - memref.dealloc %a : memref<16x16xi8> - memref.dealloc %b : memref<16x16xi8> - memref.dealloc %c : memref<16x16xi32> - return %zero_i8 : i8 -} diff --git a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_mega_conv2d_k3_c16.mlir b/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_mega_conv2d_k3_c16.mlir deleted file mode 100644 index 0e5d203e7..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_mega_conv2d_k3_c16.mlir +++ /dev/null @@ -1,84 +0,0 @@ -#input = affine_map<(n, h, w, c) -> (n, h, w, c)> -#weight = affine_map<(n, h, w, c) -> (n, c, 0, 0)> -#channel = affine_map<(n, h, w, c) -> (c)> -#scalar = affine_map<(n, h, w, c) -> (0)> -#output = affine_map<(n, h, w, c) -> (n, c, 0, 0)> - -func.func private @check_result(memref<1x16x1x1xf32>) -> () - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero_i32 = arith.constant 0 : i32 - %zero_f32 = arith.constant 0.0 : f32 - %one_i8 = arith.constant 1 : i8 - %one_f32 = arith.constant 1.0 : f32 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %two = arith.constant 2 : index - %three = arith.constant 3 : index - %five = arith.constant 5 : index - %eight = arith.constant 8 : index - %nine = arith.constant 9 : index - %sixteen = arith.constant 16 : index - %input = memref.alloc() : memref<1x3x3x16xi8> - %weight = memref.alloc() : memref<1x16x16x16xi8> - %bias = memref.alloc() : memref<16xi32> - %scale = memref.alloc() : memref<16xf32> - %lut = memref.alloc() : memref<1xi8> - %output = memref.alloc() : memref<1x16x1x1xf32> - linalg.fill ins(%zero_i8 : i8) outs(%input : memref<1x3x3x16xi8>) - linalg.fill ins(%zero_i8 : i8) outs(%weight : memref<1x16x16x16xi8>) - linalg.fill ins(%one_f32 : f32) outs(%scale : memref<16xf32>) - linalg.fill ins(%zero_i8 : i8) outs(%lut : memref<1xi8>) - linalg.fill ins(%zero_f32 : f32) outs(%output : memref<1x16x1x1xf32>) - scf.for %position = %zero to %nine step %one { - %p2 = arith.muli %position, %two : index - %h = arith.divui %position, %three : index - %w = arith.remui %position, %three : index - scf.for %channel = %zero to %sixteen step %one { - %c3 = arith.muli %channel, %three : index - %isum = arith.addi %p2, %c3 : index - %irem = arith.remui %isum, %five : index - %ival = arith.subi %irem, %two : index - %iv = arith.index_cast %ival : index to i8 - memref.store %iv, %input[%zero, %h, %w, %channel] : memref<1x3x3x16xi8> - scf.for %out = %zero to %sixteen step %one { - %wsum0 = arith.addi %c3, %p2 : index - %wsum = arith.addi %wsum0, %out : index - %wrem = arith.remui %wsum, %five : index - %wval = arith.subi %wrem, %two : index - %wv = arith.index_cast %wval : index to i8 - memref.store %wv, %weight[%zero, %channel, %position, %out] : memref<1x16x16x16xi8> - } - } - } - scf.for %out = %zero to %sixteen step %one { - %bval = arith.subi %out, %eight : index - %bv = arith.index_cast %bval : index to i32 - memref.store %bv, %bias[%out] : memref<16xi32> - } - linalg.generic { - indexing_maps = [#input, #weight, #channel, #channel, #scalar, #output], - iterator_types = ["parallel", "reduction", "reduction", "parallel"]} - ins(%input, %weight, %bias, %scale, %lut - : memref<1x3x3x16xi8>, memref<1x16x16x16xi8>, memref<16xi32>, - memref<16xf32>, memref<1xi8>) - outs(%output : memref<1x16x1x1xf32>) - attrs = {activation = 0 : i64, buckyball.mega_conv2d = true, - buckyball.mega_kernel = true, final_output = true, - kernel = 3 : i64, mega_kernel_id = "conv_k3_c16", - mega_kernel_size = 1 : i64, mega_kernel_stage = 0 : i64, - output_scale = 1.0 : f32, pad_high = 0 : i64, - pad_low = 0 : i64, stride = 1 : i64} { - ^bb0(%in: i8, %w: i8, %b: i32, %s: f32, %l: i8, %out: f32): - linalg.yield %out : f32 - } - func.call @check_result(%output) : (memref<1x16x1x1xf32>) -> () - memref.dealloc %input : memref<1x3x3x16xi8> - memref.dealloc %weight : memref<1x16x16x16xi8> - memref.dealloc %bias : memref<16xi32> - memref.dealloc %scale : memref<16xf32> - memref.dealloc %lut : memref<1xi8> - memref.dealloc %output : memref<1x16x1x1xf32> - return %zero_i8 : i8 -} diff --git a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_16x16_i8.mlir b/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_16x16_i8.mlir deleted file mode 100644 index 14c31826a..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_16x16_i8.mlir +++ /dev/null @@ -1,28 +0,0 @@ -func.func private @check_result(memref<16x16xi8>) -> () - -func.func @main() -> i8 { - %zero = arith.constant 0 : i8 - %c0 = arith.constant 0 : index - %c1 = arith.constant 1 : index - %c3 = arith.constant 3 : i8 - %c5 = arith.constant 5 : i8 - %c16 = arith.constant 16 : index - %input = memref.alloc() alignment = 64 : memref<16x16xi8> - %output = memref.alloc() alignment = 64 : memref<16x16xi8> - scf.for %i = %c0 to %c16 step %c1 { - scf.for %j = %c0 to %c16 step %c1 { - %ii = arith.index_cast %i : index to i8 - %jj = arith.index_cast %j : index to i8 - %iv = arith.muli %ii, %c3 : i8 - %jv = arith.muli %jj, %c5 : i8 - %v = arith.addi %iv, %jv : i8 - memref.store %v, %input[%i, %j] : memref<16x16xi8> - } - } - linalg.transpose ins(%input : memref<16x16xi8>) - outs(%output : memref<16x16xi8>) permutation = [1, 0] - func.call @check_result(%output) : (memref<16x16xi8>) -> () - memref.dealloc %input : memref<16x16xi8> - memref.dealloc %output : memref<16x16xi8> - return %zero : i8 -} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/CMakeLists.txt b/examples/chips/pebble/workloads/mlir_tests/tile/CMakeLists.txt deleted file mode 100644 index 1a3c8e258..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/tile/CMakeLists.txt +++ /dev/null @@ -1,75 +0,0 @@ -set(BUCKYBALL_MLIR_GROUP_TARGET pebble-mlir-tests-build) -set(BUCKYBALL_MLIR_TEST_PREFIX tile) - -set(PEBBLE_TILE_FUNCTIONAL_PASSES - -convert-tile-to-buckyball - -lower-buckyball-to-bank-ssa - -assign-physical-banks - ${BUCKYBALL_LOWER_BANK_SSA_TO_INTRINSICS} - -convert-linalg-to-loops - -expand-strided-metadata - -lower-affine - -convert-scf-to-cf - -convert-cf-to-llvm - ${BUCKYBALL_LOWER_BUCKYBALL} - -convert-arith-to-llvm - -convert-math-to-llvm - -finalize-memref-to-llvm - -convert-func-to-llvm - -reconcile-unrealized-casts) - -add_buckyball_mlir_test(mega_matmul_m17_k65_n17 - TARGET pebble PASSES ${PEBBLE_TILE_FUNCTIONAL_PASSES}) - -add_buckyball_mlir_test(mega_conv2d - TARGET pebble - PASSES ${PEBBLE_TILE_FUNCTIONAL_PASSES}) - -add_buckyball_mlir_test(mega_conv2d_k5_tail - TARGET pebble - PASSES ${PEBBLE_TILE_FUNCTIONAL_PASSES}) - -add_buckyball_mlir_test(mega_conv2d_s2_depthwise_s2 - TARGET pebble - PASSES ${PEBBLE_TILE_FUNCTIONAL_PASSES}) - -file(READ ${CMAKE_CURRENT_SOURCE_DIR}/data/mega_global_avg_conv_exact_input.i8 - PEBBLE_GAP_CONV_INPUT_HEX HEX) -file(READ ${CMAKE_CURRENT_SOURCE_DIR}/data/mega_global_avg_7x7_c288_input.i8 - PEBBLE_GAP_C288_INPUT_HEX HEX) -file(READ ${CMAKE_CURRENT_SOURCE_DIR}/data/mega_global_avg_conv_exact_weight.i8 - PEBBLE_GAP_CONV_WEIGHT_HEX HEX) -file(READ ${CMAKE_CURRENT_SOURCE_DIR}/data/mega_global_avg_conv2_exact_weight2.i8 - PEBBLE_GAP_CONV_WEIGHT2_HEX HEX) -file(READ ${CMAKE_CURRENT_SOURCE_DIR}/data/mega_global_avg_conv2_exact_weight3.i8 - PEBBLE_GAP_CONV_WEIGHT3_HEX HEX) -add_buckyball_mlir_test(mega_global_avg_conv_exact - TARGET pebble - PASSES ${PEBBLE_TILE_FUNCTIONAL_PASSES}) -add_buckyball_mlir_test(mega_global_avg_7x7_c288_exact - TARGET pebble - PASSES ${PEBBLE_TILE_FUNCTIONAL_PASSES}) -add_buckyball_mlir_test(mega_global_avg_conv2_exact - TARGET pebble - PASSES ${PEBBLE_TILE_FUNCTIONAL_PASSES}) -unset(PEBBLE_GAP_CONV_INPUT_HEX) -unset(PEBBLE_GAP_C288_INPUT_HEX) -unset(PEBBLE_GAP_CONV_WEIGHT_HEX) -unset(PEBBLE_GAP_CONV_WEIGHT2_HEX) -unset(PEBBLE_GAP_CONV_WEIGHT3_HEX) - -add_buckyball_mlir_test(mega_se_gate_conv - TARGET pebble - PASSES ${PEBBLE_TILE_FUNCTIONAL_PASSES}) - -add_buckyball_mlir_test(mega_matmul - TARGET pebble - PASSES ${PEBBLE_TILE_FUNCTIONAL_PASSES}) - -foreach(test - tile_matmul_16x16_i8 - tile_transpose_16x16_i8) - add_buckyball_mlir_test(${test} - TARGET pebble - PASSES ${PEBBLE_TILE_FUNCTIONAL_PASSES}) -endforeach() diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/mega_conv2d.mlir b/examples/chips/pebble/workloads/mlir_tests/tile/mega_conv2d.mlir deleted file mode 100644 index 8560e7d7b..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/tile/mega_conv2d.mlir +++ /dev/null @@ -1,100 +0,0 @@ -func.func private @check_result(memref<1x16x2x2xi8>) -> () - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero_i32 = arith.constant 0 : i32 - %one_f32 = arith.constant 1.0 : f32 - %c0 = arith.constant 0 : index - %c1 = arith.constant 1 : index - %c2 = arith.constant 2 : index - %c3 = arith.constant 3 : index - %c4 = arith.constant 4 : index - %c5 = arith.constant 5 : index - %c7 = arith.constant 7 : index - %c16 = arith.constant 16 : index - %input = memref.alloc() alignment = 64 : memref<1x4x4x2xi8> - %conv_weight = memref.alloc() alignment = 64 : memref<1x2x16x16xi8> - %conv_bias = memref.alloc() alignment = 64 : memref<16xi32> - %conv_scale = memref.alloc() alignment = 64 : memref<16xf32> - %lut = memref.alloc() alignment = 64 : memref<1xi8> - %intermediate = memref.alloc() alignment = 64 : memref<1x4x4x16xi8> - %depthwise_weight = memref.alloc() alignment = 64 : memref<3x3x16x1xi8> - %depthwise_bias = memref.alloc() alignment = 64 : memref<16xi32> - %depthwise_scale = memref.alloc() alignment = 64 : memref<16xf32> - %depthwise_output = memref.alloc() alignment = 64 : memref<1x4x4x16xi8> - %output = memref.alloc() alignment = 64 : memref<1x16x2x2xi8> - scf.for %h = %c0 to %c4 step %c1 { - scf.for %w = %c0 to %c4 step %c1 { - scf.for %ic = %c0 to %c2 step %c1 { - %h3 = arith.muli %h, %c3 : index - %w2 = arith.muli %w, %c2 : index - %sum0 = arith.addi %h3, %w2 : index - %sum = arith.addi %sum0, %ic : index - %rem = arith.remui %sum, %c7 : index - %signed = arith.subi %rem, %c3 : index - %value = arith.index_cast %signed : index to i8 - memref.store %value, %input[%c0, %h, %w, %ic] : memref<1x4x4x2xi8> - } - } - } - scf.for %ic = %c0 to %c2 step %c1 { - scf.for %p = %c0 to %c16 step %c1 { - scf.for %oc = %c0 to %c16 step %c1 { - %oc3 = arith.muli %oc, %c3 : index - %sum0 = arith.addi %p, %ic : index - %sum = arith.addi %sum0, %oc3 : index - %rem = arith.remui %sum, %c5 : index - %signed = arith.subi %rem, %c2 : index - %value = arith.index_cast %signed : index to i8 - memref.store %value, %conv_weight[%c0, %ic, %p, %oc] : memref<1x2x16x16xi8> - } - } - } - scf.for %oc = %c0 to %c16 step %c1 { - %signed = arith.subi %oc, %c4 : index - %value = arith.index_cast %signed : index to i32 - memref.store %value, %conv_bias[%oc] : memref<16xi32> - } - linalg.fill ins(%one_f32 : f32) outs(%conv_scale : memref<16xf32>) - linalg.fill ins(%zero_i8 : i8) outs(%lut : memref<1xi8>) - linalg.fill ins(%zero_i8 : i8) outs(%intermediate : memref<1x4x4x16xi8>) - scf.for %kh = %c0 to %c3 step %c1 { - scf.for %kw = %c0 to %c3 step %c1 { - scf.for %oc = %c0 to %c16 step %c1 { - %kw2 = arith.muli %kw, %c2 : index - %sum0 = arith.addi %kh, %kw2 : index - %sum = arith.addi %sum0, %oc : index - %rem = arith.remui %sum, %c5 : index - %signed = arith.subi %rem, %c2 : index - %value = arith.index_cast %signed : index to i8 - memref.store %value, %depthwise_weight[%kh, %kw, %oc, %c0] : memref<3x3x16x1xi8> - } - } - } - linalg.fill ins(%zero_i32 : i32) outs(%depthwise_bias : memref<16xi32>) - linalg.fill ins(%one_f32 : f32) outs(%depthwise_scale : memref<16xf32>) - linalg.fill ins(%zero_i8 : i8) outs(%depthwise_output : memref<1x4x4x16xi8>) - linalg.fill ins(%zero_i8 : i8) outs(%output : memref<1x16x2x2xi8>) - tile.mega_kernel %input %output - : memref<1x4x4x2xi8> memref<1x16x2x2xi8> { - tile.mega_conv2d %input %conv_weight %conv_bias %conv_scale %lut - %intermediate - : memref<1x4x4x2xi8> memref<1x2x16x16xi8> memref<16xi32> - memref<16xf32> memref<1xi8> memref<1x4x4x16xi8> - tile.mega_conv2d_depthwise %intermediate %depthwise_weight - %depthwise_bias %depthwise_scale %lut %depthwise_output - - : memref<1x4x4x16xi8> memref<3x3x16x1xi8> memref<16xi32> - memref<16xf32> memref<1xi8> memref<1x4x4x16xi8> - tile.mega_max_pool2d %depthwise_output %output - - : memref<1x4x4x16xi8> memref<1x16x2x2xi8> - } - func.call @check_result(%output) : (memref<1x16x2x2xi8>) -> () - return %zero_i8 : i8 -} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/mega_conv2d_k5_tail.mlir b/examples/chips/pebble/workloads/mlir_tests/tile/mega_conv2d_k5_tail.mlir deleted file mode 100644 index ea42e3f63..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/tile/mega_conv2d_k5_tail.mlir +++ /dev/null @@ -1,62 +0,0 @@ -func.func private @check_result(memref<1x16x5x5xf32>) -func.func @main() -> i8 { - %c0 = arith.constant 0 : index - %c1 = arith.constant 1 : index - %c2 = arith.constant 2 : index - %c3 = arith.constant 3 : index - %c5 = arith.constant 5 : index - %c7 = arith.constant 7 : index - %c16 = arith.constant 16 : index - %c25 = arith.constant 25 : index - %zi8 = arith.constant 0 : i8 - %zi32 = arith.constant 0 : i32 - %zf32 = arith.constant 0.0 : f32 - %one = arith.constant 1.0 : f32 - %input = memref.alloc() alignment = 64 : memref<1x5x5x2xi8> - %weight = memref.alloc() alignment = 64 : memref<1x2x32x16xi8> - %bias = memref.alloc() alignment = 64 : memref<16xi32> - %scale = memref.alloc() alignment = 64 : memref<16xf32> - %lut = memref.alloc() alignment = 64 : memref<1xi8> - %output = memref.alloc() alignment = 64 : memref<1x16x5x5xf32> - linalg.fill ins(%zi8 : i8) outs(%weight : memref<1x2x32x16xi8>) - linalg.fill ins(%zi32 : i32) outs(%bias : memref<16xi32>) - linalg.fill ins(%one : f32) outs(%scale : memref<16xf32>) - linalg.fill ins(%zi8 : i8) outs(%lut : memref<1xi8>) - linalg.fill ins(%zf32 : f32) outs(%output : memref<1x16x5x5xf32>) - scf.for %y = %c0 to %c5 step %c1 { - scf.for %x = %c0 to %c5 step %c1 { - scf.for %c = %c0 to %c2 step %c1 { - %y3 = arith.muli %y, %c3 : index - %x2 = arith.muli %x, %c2 : index - %s0 = arith.addi %y3, %x2 : index - %s = arith.addi %s0, %c : index - %r = arith.remui %s, %c7 : index - %v = arith.subi %r, %c3 : index - %i = arith.index_cast %v : index to i8 - memref.store %i, %input[%c0, %y, %x, %c] : memref<1x5x5x2xi8> - } - } - } - scf.for %c = %c0 to %c2 step %c1 { - scf.for %k = %c0 to %c25 step %c1 { - scf.for %o = %c0 to %c16 step %c1 { - %s0 = arith.addi %k, %c : index - %s = arith.addi %s0, %o : index - %r = arith.remui %s, %c3 : index - %v = arith.subi %r, %c1 : index - %i = arith.index_cast %v : index to i8 - memref.store %i, %weight[%c0, %c, %k, %o] : memref<1x2x32x16xi8> - } - } - } - tile.mega_kernel %input %output : memref<1x5x5x2xi8> memref<1x16x5x5xf32> { - tile.mega_conv2d %input %weight %bias %scale %lut %output - - : memref<1x5x5x2xi8> memref<1x2x32x16xi8> memref<16xi32> - memref<16xf32> memref<1xi8> memref<1x16x5x5xf32> - } - func.call @check_result(%output) : (memref<1x16x5x5xf32>) -> () - %ok = arith.constant 0 : i8 - return %ok : i8 -} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/mega_conv2d_s2_depthwise_s2.mlir b/examples/chips/pebble/workloads/mlir_tests/tile/mega_conv2d_s2_depthwise_s2.mlir deleted file mode 100644 index f86aa1d80..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/tile/mega_conv2d_s2_depthwise_s2.mlir +++ /dev/null @@ -1,91 +0,0 @@ -func.func private @check_result(memref<1x4x4x16xi8>) -> () - -func.func @main() -> i8 { - %i8_0 = arith.constant 0 : i8 - %i32_0 = arith.constant 0 : i32 - %f32_1 = arith.constant 1.0 : f32 - %c0 = arith.constant 0 : index - %c1 = arith.constant 1 : index - %c2 = arith.constant 2 : index - %c3 = arith.constant 3 : index - %c4 = arith.constant 4 : index - %c5 = arith.constant 5 : index - %c7 = arith.constant 7 : index - %c9 = arith.constant 9 : index - %c16 = arith.constant 16 : index - %input = memref.alloc() alignment = 64 : memref<1x16x16x3xi8> - %weight0 = memref.alloc() alignment = 64 : memref<1x3x16x16xi8> - %bias0 = memref.alloc() alignment = 64 : memref<16xi32> - %scale0 = memref.alloc() alignment = 64 : memref<16xf32> - %lut = memref.alloc() alignment = 64 : memref<1xi8> - %middle = memref.alloc() alignment = 64 : memref<1x8x8x16xi8> - %weight1 = memref.alloc() alignment = 64 : memref<3x3x16x1xi8> - %bias1 = memref.alloc() alignment = 64 : memref<16xi32> - %scale1 = memref.alloc() alignment = 64 : memref<16xf32> - %output = memref.alloc() alignment = 64 : memref<1x4x4x16xi8> - scf.for %y = %c0 to %c16 step %c1 { - scf.for %x = %c0 to %c16 step %c1 { - scf.for %ic = %c0 to %c3 step %c1 { - %a = arith.muli %y, %c3 : index - %b = arith.muli %x, %c2 : index - %d = arith.addi %a, %b : index - %e = arith.addi %d, %ic : index - %r = arith.remui %e, %c7 : index - %s = arith.subi %r, %c3 : index - %v = arith.index_cast %s : index to i8 - memref.store %v, %input[%c0, %y, %x, %ic] : memref<1x16x16x3xi8> - } - } - } - scf.for %ic = %c0 to %c3 step %c1 { - scf.for %p = %c0 to %c16 step %c1 { - scf.for %oc = %c0 to %c16 step %c1 { - %oc3 = arith.muli %oc, %c3 : index - %a = arith.addi %p, %ic : index - %b = arith.addi %a, %oc3 : index - %r = arith.remui %b, %c5 : index - %s = arith.subi %r, %c2 : index - %v = arith.index_cast %s : index to i8 - memref.store %v, %weight0[%c0, %ic, %p, %oc] : memref<1x3x16x16xi8> - } - } - } - scf.for %oc = %c0 to %c16 step %c1 { - %s = arith.subi %oc, %c4 : index - %v = arith.index_cast %s : index to i32 - memref.store %v, %bias0[%oc] : memref<16xi32> - } - linalg.fill ins(%f32_1 : f32) outs(%scale0 : memref<16xf32>) - linalg.fill ins(%i8_0 : i8) outs(%lut : memref<1xi8>) - linalg.fill ins(%i8_0 : i8) outs(%middle : memref<1x8x8x16xi8>) - scf.for %ky = %c0 to %c3 step %c1 { - scf.for %kx = %c0 to %c3 step %c1 { - scf.for %c = %c0 to %c16 step %c1 { - %kx2 = arith.muli %kx, %c2 : index - %a = arith.addi %ky, %kx2 : index - %b = arith.addi %a, %c : index - %r = arith.remui %b, %c5 : index - %s = arith.subi %r, %c2 : index - %v = arith.index_cast %s : index to i8 - memref.store %v, %weight1[%ky, %kx, %c, %c0] : memref<3x3x16x1xi8> - } - } - } - linalg.fill ins(%i32_0 : i32) outs(%bias1 : memref<16xi32>) - linalg.fill ins(%f32_1 : f32) outs(%scale1 : memref<16xf32>) - linalg.fill ins(%i8_0 : i8) outs(%output : memref<1x4x4x16xi8>) - tile.mega_kernel %input %output : memref<1x16x16x3xi8> memref<1x4x4x16xi8> { - tile.mega_conv2d %input %weight0 %bias0 %scale0 %lut %middle - - : memref<1x16x16x3xi8> memref<1x3x16x16xi8> memref<16xi32> - memref<16xf32> memref<1xi8> memref<1x8x8x16xi8> - tile.mega_conv2d_depthwise %middle %weight1 %bias1 %scale1 %lut %output - - : memref<1x8x8x16xi8> memref<3x3x16x1xi8> memref<16xi32> - memref<16xf32> memref<1xi8> memref<1x4x4x16xi8> - } - func.call @check_result(%output) : (memref<1x4x4x16xi8>) -> () - return %i8_0 : i8 -} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/mega_global_avg_7x7_c288_exact.mlir.in b/examples/chips/pebble/workloads/mlir_tests/tile/mega_global_avg_7x7_c288_exact.mlir.in deleted file mode 100644 index f8336126d..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/tile/mega_global_avg_7x7_c288_exact.mlir.in +++ /dev/null @@ -1,16 +0,0 @@ -memref.global "private" constant @input : memref<1x7x7x288xi8> = dense<"0x@PEBBLE_GAP_C288_INPUT_HEX@"> alignment = 64 - -func.func private @poison_reused_banks() -> () -func.func private @check_result(memref<1x1x1x288xi8>) -> () - -func.func @main() -> i8 { - %zero = arith.constant 0 : i8 - %input = memref.get_global @input : memref<1x7x7x288xi8> - %output = memref.alloc() alignment = 64 : memref<1x1x1x288xi8> - func.call @poison_reused_banks() : () -> () - tile.mega_kernel %input %output : memref<1x7x7x288xi8> memref<1x1x1x288xi8> { - tile.mega_global_avg_pool %input %output : memref<1x7x7x288xi8> memref<1x1x1x288xi8> - } - func.call @check_result(%output) : (memref<1x1x1x288xi8>) -> () - return %zero : i8 -} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/mega_global_avg_conv2_exact.mlir.in b/examples/chips/pebble/workloads/mlir_tests/tile/mega_global_avg_conv2_exact.mlir.in deleted file mode 100644 index eac3dd086..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/tile/mega_global_avg_conv2_exact.mlir.in +++ /dev/null @@ -1,58 +0,0 @@ -memref.global "private" constant @input : memref<1x14x14x96xi8> = dense<"0x@PEBBLE_GAP_CONV_INPUT_HEX@"> alignment = 64 -memref.global "private" constant @weight1raw : memref<49168xi8> = dense<"0x00000000000000000000000000000000@PEBBLE_GAP_CONV_WEIGHT_HEX@"> alignment = 64 -memref.global "private" constant @bias1 : memref<24xi32> = dense<[-1929, -3345, -826, -4425, -1950, -1378, -2544, 3397, -134, -2621, 385, -1868, -3754, -948, 275, -53, -1955, -3197, -1065, -298, 536, -2267, -3803, -48]> alignment = 64 -memref.global "private" constant @scale1 : memref<24xf32> = dense<[9.64902255E-11, 1.20455174E-10, 9.11374864E-11, 6.07460609E-11, 6.96993296E-11, 2.21286572E-10, 8.11237535E-11, 1.33313652E-10, 1.26243266E-10, 1.44089102E-10, 0.0341239199, 6.56752361E-11, 6.62821395E-11, 1.91158489E-9, 1.69707901E-10, 4.13074019E-10, 5.95885979E-11, 8.92385332E-11, 6.12655274E-11, 2.72214418E-10, 2.77576795E-10, 2.37361242E-10, 6.75774228E-11, 1.48867613E-10]> alignment = 64 -memref.global "private" constant @weight2raw : memref<36880xi8> = dense<"0x00000000000000000000000000000000@PEBBLE_GAP_CONV_WEIGHT2_HEX@"> alignment = 64 -memref.global "private" constant @bias2 : memref<96xi32> = dense<[7051, 3006, 26558, 11845, -12167, 14432, -194, 9249, -17237, 22963, 7410, 91984, 105591, 6346, 90458, 43212, 45789, 33855, 29981, 10245, 4370, -34, -712, 15905, 22004, 549465, -5252, -3669, 7224, 18985, 8518, 8198, 16536, 51867, 4243, 6598, 9711, 18037, 21720, 397303, -253, 10751, -10281, -10424, 1839, 25010, 6995, -15222, 1786, -3524, -15704, 13370, 8813, 27391, 8927, -12218, -4217, -7494, 4135, 12616, 9593, 1431, -5377, 9947, 370, 11072, 5546, 22002, 10029, -3861, 904, 95829, 4717, -14504, 52996, 33835, 12322, 3329, 21275, -872, 1465, 17484, 6331, 63424, -29681, 34866, 3915, -2807, 71222, 5504, -13650, 15420, 7960, -9852, -13797, 21711]> alignment = 64 -memref.global "private" constant @scale2 : memref<96xf32> = dense<[0.00181691104, 4.059140E-3, 0.00131794217, 0.00212815683, 0.00430351309, 0.00403421139, 5.176660E-3, 0.00188985514, 0.00558878435, 0.00152020913, 0.00459996238, 5.82394132E-4, 4.24394966E-4, 0.0019368825, 7.07225234E-4, 6.37308462E-4, 0.00137251348, 7.15450151E-4, 0.00140084175, 0.00317073381, 5.073430E-3, 0.00307706394, 0.00386786251, 0.00195057772, 0.00210299343, 1.08822038E-4, 0.00415666727, 0.00333561096, 0.00485164393, 7.86353368E-4, 0.00419554161, 0.00339963497, 0.00123977242, 0.00106979779, 0.00394831039, 0.00133779203, 0.00282861362, 0.00155300961, 0.00196025497, 5.82583889E-5, 0.0032499244, 0.00156923081, 0.00266220793, 0.00191042456, 0.00408306951, 9.02582774E-4, 0.00326385885, 0.00143681327, 0.00142906886, 0.0026714697, 0.00458595809, 0.00430518808, 2.272860E-3, 0.00113706058, 0.00272057392, 0.00554801896, 6.058620E-3, 0.00159710483, 0.00501794415, 0.00278559304, 0.00328800408, 0.00337952748, 0.00245677959, 0.00205389271, 0.00226871902, 0.00263371225, 9.1695087E-4, 0.00129066338, 0.00205622637, 2.126570E-3, 9.56189353E-4, 2.12495346E-4, 9.6136285E-4, 0.00436848169, 9.72485286E-4, 4.94753069E-4, 0.00348583842, 0.00738966605, 0.00289806188, 0.00333968201, 0.00305957114, 0.00181978801, 0.00184966961, 3.52834439E-4, 0.00117929338, 9.00093349E-4, 0.00424369285, 0.00269078952, 5.541500E-4, 0.0025951541, 0.00531184906, 0.00391697325, 0.00378050678, 0.00192236435, 0.00131825986, 0.00273139146]> alignment = 64 -memref.global "private" constant @lut : memref<256xi8> = dense<"0x191A1A1A1A1A1A1B1B1B1B1B1B1C1C1C1C1C1C1D1D1D1D1D1D1E1E1E1E1E1E1F1F1F1F1F1F2020202020202121212121212222222222222323232323232424242424242525252525252626262626262727272727272828282828282929292929292A2A2A2A2A2A2B2B2B2B2B2B2C2C2C2C2C2C2D2D2D2D2D2D2E2E2E2E2E2E2F0404040505050505050606060606060707070707070808080808080909090909090A0A0A0A0A0A0B0B0B0B0B0B0C0C0C0C0C0C0D0D0D0D0D0D0E0E0E0E0E0E0F0F0F0F0F0F1010101010101111111111111212121212121313131313131414141414141515151515151616161616161717171717171818181818181919191919"> alignment = 64 -memref.global "private" constant @weight3raw : memref<73744xi8> = dense<"0x00000000000000000000000000000000@PEBBLE_GAP_CONV_WEIGHT3_HEX@"> alignment = 64 -memref.global "private" constant @bias3 : memref<40xi32> = dense<[104, -13887, -3685, -19928, 5678, 8790, -6325, -4745, 15957, 3843, 4642, 11420, -2754, -19949, 17269, -673, 16732, -31364, 19420, 3384, -1286, -5401, 3250, -9546, 3517, 6568, 9818, -3768, -8763, 19005, 2689, -2236, -3490, 6369, -18573, -726, -16056, 10604, -4229, -7803]> alignment = 64 -memref.global "private" constant @scale3 : memref<40xf32> = dense<[0.007250275, 0.0052957274, 0.00507997582, 5.148440E-3, 0.00615183357, 0.00551314512, 0.00447647972, 0.00624849601, 4.808580E-3, 0.00621820614, 0.00767033594, 0.00502041401, 0.00771069526, 0.00713740429, 5.340260E-3, 0.00554999569, 0.00578600029, 0.00608736509, 0.00647644047, 0.00674922066, 0.00569664268, 0.00644684816, 0.00639614975, 0.00778135518, 0.00677069137, 0.00366282626, 0.00660506124, 0.00643686252, 0.00655329926, 0.00661863247, 0.00624936819, 0.0104040103, 0.00504696835, 7.654780E-3, 0.00546469772, 5.341790E-3, 0.0050635729, 0.00579059403, 0.0084069958, 0.0039168736]> alignment = 64 -memref.global "private" constant @none : memref<1xi8> = dense<0> alignment = 64 -func.func private @check_result(memref<1x14x14x40xi8>) -> () func.func private @poison_reused_bank() -> () - -func.func @main() -> i8 { - %zero = arith.constant 0 : i8 - %c0 = arith.constant 0 : index %c1 = arith.constant 1 : index - %c14 = arith.constant 14 : index - %c96 = arith.constant 96 : index - %input = memref.get_global @input : memref<1x14x14x96xi8> - %input_heap = memref.alloc() alignment = 64 : memref<1x14x14x96xi8> - scf.for %y = %c0 to %c14 step %c1 { - scf.for %x = %c0 to %c14 step %c1 { - scf.for %c = %c0 to %c96 step %c1 { - %v = memref.load %input[%c0, %y, %x, %c] : memref<1x14x14x96xi8> - memref.store %v, %input_heap[%c0, %y, %x, %c] : memref<1x14x14x96xi8> - } - } - } - %weight1raw = memref.get_global @weight1raw : memref<49168xi8> - %weight1 = memref.reinterpret_cast %weight1raw to offset: [16], sizes: [2, 96, 16, 16], strides: [24576, 256, 16, 1] : memref<49168xi8> to memref<2x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>> - %bias1 = memref.get_global @bias1 : memref<24xi32> - %scale1 = memref.get_global @scale1 : memref<24xf32> - %weight2raw = memref.get_global @weight2raw : memref<36880xi8> - %weight2 = memref.reinterpret_cast %weight2raw to offset: [16], sizes: [6, 24, 16, 16], strides: [6144, 256, 16, 1] : memref<36880xi8> to memref<6x24x16x16xi8, strided<[6144, 256, 16, 1], offset: 16>> - %bias2 = memref.get_global @bias2 : memref<96xi32> - %scale2 = memref.get_global @scale2 : memref<96xf32> - %lut = memref.get_global @lut : memref<256xi8> - %weight3raw = memref.get_global @weight3raw : memref<73744xi8> - %weight3 = memref.reinterpret_cast %weight3raw to offset: [16], sizes: [3, 96, 16, 16], strides: [24576, 256, 16, 1] : memref<73744xi8> to memref<3x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>> - %bias3 = memref.get_global @bias3 : memref<40xi32> %scale3 = memref.get_global @scale3 : memref<40xf32> - %none = memref.get_global @none : memref<1xi8> - func.call @poison_reused_bank() : () -> () - %gap = memref.alloc() alignment = 64 : memref<1x1x1x96xi8> - %middle = memref.alloc() alignment = 64 : memref<1x1x1x24xi8> - %gate = memref.alloc() alignment = 64 : memref<1x1x1x96xi8> - %mul = memref.alloc() alignment = 64 : memref<1x14x14x96xi8> - %output = memref.alloc() alignment = 64 : memref<1x14x14x40xi8> - tile.mega_kernel %input_heap %output : memref<1x14x14x96xi8> memref<1x14x14x40xi8> { - tile.mega_global_avg_pool %input_heap %gap : memref<1x14x14x96xi8> memref<1x1x1x96xi8> - tile.mega_conv2d %gap %weight1 %bias1 %scale1 %none %middle : memref<1x1x1x96xi8> memref<2x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>> memref<24xi32> memref<24xf32> memref<1xi8> memref<1x1x1x24xi8> - tile.mega_conv2d %middle %weight2 %bias2 %scale2 %lut %gate : memref<1x1x1x24xi8> memref<6x24x16x16xi8, strided<[6144, 256, 16, 1], offset: 16>> memref<96xi32> memref<96xf32> memref<256xi8> memref<1x1x1x96xi8> - tile.mega_int8_mul %gate %input_heap %mul : memref<1x1x1x96xi8> memref<1x14x14x96xi8> memref<1x14x14x96xi8> - tile.mega_conv2d %mul %weight3 %bias3 %scale3 %none %output : memref<1x14x14x96xi8> memref<3x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>> memref<40xi32> memref<40xf32> memref<1xi8> memref<1x14x14x40xi8> - } - func.call @check_result(%output) : (memref<1x14x14x40xi8>) -> () - return %zero : i8 -} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/mega_global_avg_conv_exact.mlir.in b/examples/chips/pebble/workloads/mlir_tests/tile/mega_global_avg_conv_exact.mlir.in deleted file mode 100644 index 7a2d32dbb..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/tile/mega_global_avg_conv_exact.mlir.in +++ /dev/null @@ -1,24 +0,0 @@ -memref.global "private" constant @input : memref<1x14x14x96xi8> = dense<"0x@PEBBLE_GAP_CONV_INPUT_HEX@"> alignment = 64 -memref.global "private" constant @weight : memref<2x96x16x16xi8> = dense<"0x@PEBBLE_GAP_CONV_WEIGHT_HEX@"> alignment = 64 -memref.global "private" constant @bias : memref<24xi32> = dense<[-1929, -3345, -826, -4425, -1950, -1378, -2544, 3397, -134, -2621, 385, -1868, -3754, -948, 275, -53, -1955, -3197, -1065, -298, 536, -2267, -3803, -48]> alignment = 64 -memref.global "private" constant @scale : memref<24xf32> = dense<[9.64902255E-11, 1.20455174E-10, 9.11374864E-11, 6.07460609E-11, 6.96993296E-11, 2.21286572E-10, 8.11237535E-11, 1.33313652E-10, 1.26243266E-10, 1.44089102E-10, 0.0341239199, 6.56752361E-11, 6.62821395E-11, 1.91158489E-9, 1.69707901E-10, 4.13074019E-10, 5.95885979E-11, 8.92385332E-11, 6.12655274E-11, 2.72214418E-10, 2.77576795E-10, 2.37361242E-10, 6.75774228E-11, 1.48867613E-10]> alignment = 64 -memref.global "private" constant @none : memref<1xi8> = dense<0> alignment = 64 - -func.func private @check_result(memref<1x1x1x24xi8>) -> () - -func.func @main() -> i8 { - %zero = arith.constant 0 : i8 - %input = memref.get_global @input : memref<1x14x14x96xi8> - %weight = memref.get_global @weight : memref<2x96x16x16xi8> - %bias = memref.get_global @bias : memref<24xi32> - %scale = memref.get_global @scale : memref<24xf32> - %none = memref.get_global @none : memref<1xi8> - %gap = memref.alloc() alignment = 64 : memref<1x1x1x96xi8> - %output = memref.alloc() alignment = 64 : memref<1x1x1x24xi8> - tile.mega_kernel %input %output : memref<1x14x14x96xi8> memref<1x1x1x24xi8> { - tile.mega_global_avg_pool %input %gap : memref<1x14x14x96xi8> memref<1x1x1x96xi8> - tile.mega_conv2d %gap %weight %bias %scale %none %output : memref<1x1x1x96xi8> memref<2x96x16x16xi8> memref<24xi32> memref<24xf32> memref<1xi8> memref<1x1x1x24xi8> - } - func.call @check_result(%output) : (memref<1x1x1x24xi8>) -> () - return %zero : i8 -} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/mega_matmul.mlir b/examples/chips/pebble/workloads/mlir_tests/tile/mega_matmul.mlir deleted file mode 100644 index af0225966..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/tile/mega_matmul.mlir +++ /dev/null @@ -1,57 +0,0 @@ -func.func private @check_result(memref<1x16xf32>) -> () - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero_i32 = arith.constant 0 : i32 - %one_f32 = arith.constant 1.0 : f32 - %zero_f32 = arith.constant 0.0 : f32 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %two = arith.constant 2 : index - %three = arith.constant 3 : index - %five = arith.constant 5 : index - %seven = arith.constant 7 : index - %eight = arith.constant 8 : index - %dim = arith.constant 16 : index - %input = memref.alloc() alignment = 64 : memref<1x16xi8> - %weight = memref.alloc() alignment = 64 : memref<16x16xi8> - %bias = memref.alloc() alignment = 64 : memref<16xi32> - %scale = memref.alloc() alignment = 64 : memref<16xf32> - %lut = memref.alloc() alignment = 64 : memref<1xi8> - %output = memref.alloc() alignment = 64 : memref<1x16xf32> - scf.for %k = %zero to %dim step %one { - %irem = arith.remui %k, %seven : index - %ival = arith.subi %irem, %three : index - %iv = arith.index_cast %ival : index to i8 - memref.store %iv, %input[%zero, %k] : memref<1x16xi8> - %biasv = arith.subi %k, %eight : index - %bias32 = arith.index_cast %biasv : index to i32 - memref.store %bias32, %bias[%k] : memref<16xi32> - scf.for %column = %zero to %dim step %one { - %k2 = arith.muli %k, %two : index - %c3 = arith.muli %column, %three : index - %sum = arith.addi %k2, %c3 : index - %rem = arith.remui %sum, %five : index - %value = arith.subi %rem, %two : index - %wv = arith.index_cast %value : index to i8 - memref.store %wv, %weight[%k, %column] : memref<16x16xi8> - } - } - linalg.fill ins(%one_f32 : f32) outs(%scale : memref<16xf32>) - linalg.fill ins(%zero_i8 : i8) outs(%lut : memref<1xi8>) - linalg.fill ins(%zero_f32 : f32) outs(%output : memref<1x16xf32>) - tile.mega_kernel %input %output : memref<1x16xi8> memref<1x16xf32> { - tile.mega_matmul %input %weight %bias %scale %lut %output - - : memref<1x16xi8> memref<16x16xi8> memref<16xi32> - memref<16xf32> memref<1xi8> memref<1x16xf32> - } - func.call @check_result(%output) : (memref<1x16xf32>) -> () - memref.dealloc %input : memref<1x16xi8> - memref.dealloc %weight : memref<16x16xi8> - memref.dealloc %bias : memref<16xi32> - memref.dealloc %scale : memref<16xf32> - memref.dealloc %lut : memref<1xi8> - memref.dealloc %output : memref<1x16xf32> - return %zero_i8 : i8 -} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/mega_matmul_m17_k65_n17.mlir b/examples/chips/pebble/workloads/mlir_tests/tile/mega_matmul_m17_k65_n17.mlir deleted file mode 100644 index 778dbd28d..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/tile/mega_matmul_m17_k65_n17.mlir +++ /dev/null @@ -1,61 +0,0 @@ -func.func private @check_result(memref<17x17xf32>) -> () -func.func @main() -> i8 { - %z8 = arith.constant 0 : i8 - %zf = arith.constant 0.0 : f32 - %half = arith.constant 0.5 : f32 - %z = arith.constant 0 : index - %one = arith.constant 1 : index - %two = arith.constant 2 : index - %three = arith.constant 3 : index - %five = arith.constant 5 : index - %seven = arith.constant 7 : index - %eight = arith.constant 8 : index - %dim = arith.constant 17 : index - %inner = arith.constant 65 : index - %input = memref.alloc() alignment = 64 : memref<17x65xi8> - %weight = memref.alloc() alignment = 64 : memref<65x17xi8> - %bias = memref.alloc() alignment = 64 : memref<17xi32> - %scale = memref.alloc() alignment = 64 : memref<17xf32> - %lut = memref.alloc() alignment = 64 : memref<1xi8> - %output = memref.alloc() alignment = 64 : memref<17x17xf32> - scf.for %r = %z to %dim step %one { - scf.for %k = %z to %inner step %one { - %sum = arith.addi %r, %k : index - %rem = arith.remui %sum, %seven : index - %v = arith.subi %rem, %three : index - %i8 = arith.index_cast %v : index to i8 - memref.store %i8, %input[%r, %k] : memref<17x65xi8> - } - %b = arith.subi %r, %eight : index - %b32 = arith.index_cast %b : index to i32 - memref.store %b32, %bias[%r] : memref<17xi32> - } - scf.for %k = %z to %inner step %one { - scf.for %c = %z to %dim step %one { - %k2 = arith.muli %k, %two : index - %c3 = arith.muli %c, %three : index - %sum = arith.addi %k2, %c3 : index - %rem = arith.remui %sum, %five : index - %v = arith.subi %rem, %two : index - %i8 = arith.index_cast %v : index to i8 - memref.store %i8, %weight[%k, %c] : memref<65x17xi8> - } - } - linalg.fill ins(%half : f32) outs(%scale : memref<17xf32>) - linalg.fill ins(%z8 : i8) outs(%lut : memref<1xi8>) - linalg.fill ins(%zf : f32) outs(%output : memref<17x17xf32>) - tile.mega_kernel %input %output : memref<17x65xi8> memref<17x17xf32> { - tile.mega_matmul %input %weight %bias %scale %lut %output - - : memref<17x65xi8> memref<65x17xi8> memref<17xi32> - memref<17xf32> memref<1xi8> memref<17x17xf32> - } - func.call @check_result(%output) : (memref<17x17xf32>) -> () - memref.dealloc %input : memref<17x65xi8> - memref.dealloc %weight : memref<65x17xi8> - memref.dealloc %bias : memref<17xi32> - memref.dealloc %scale : memref<17xf32> - memref.dealloc %lut : memref<1xi8> - memref.dealloc %output : memref<17x17xf32> - return %z8 : i8 -} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/mega_se_gate_conv.mlir b/examples/chips/pebble/workloads/mlir_tests/tile/mega_se_gate_conv.mlir deleted file mode 100644 index efde11309..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/tile/mega_se_gate_conv.mlir +++ /dev/null @@ -1,75 +0,0 @@ -func.func private @check_result(memref<1x6x6x40xi8>) -> () - -func.func @main() -> i8 { - %i8_0 = arith.constant 0 : i8 - %i8_1 = arith.constant 1 : i8 - %i32_0 = arith.constant 0 : i32 - %i32_1 = arith.constant 1 : i32 - %f32_001 = arith.constant 0.01 : f32 - %f32_01 = arith.constant 0.1 : f32 - %c0 = arith.constant 0 : index - %c1 = arith.constant 1 : index - %c2 = arith.constant 2 : index - %c5 = arith.constant 5 : index - %c6 = arith.constant 6 : index - %c96 = arith.constant 96 : index - %c256 = arith.constant 256 : index - %input = memref.alloc() alignment = 64 : memref<1x6x6x96xi8> - %gap = memref.alloc() alignment = 64 : memref<1x1x1x96xi8> - %w1raw = memref.alloc() alignment = 64 : memref<49168xi8> - %w1 = memref.reinterpret_cast %w1raw to offset: [16], sizes: [2, 96, 16, 16], strides: [24576, 256, 16, 1] : memref<49168xi8> to memref<2x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>> - %b1 = memref.alloc() alignment = 64 : memref<24xi32> - %s1 = memref.alloc() alignment = 64 : memref<24xf32> - %h1 = memref.alloc() alignment = 64 : memref<1x1x1x24xi8> - %w2raw = memref.alloc() alignment = 64 : memref<36880xi8> - %w2 = memref.reinterpret_cast %w2raw to offset: [16], sizes: [6, 24, 16, 16], strides: [6144, 256, 16, 1] : memref<36880xi8> to memref<6x24x16x16xi8, strided<[6144, 256, 16, 1], offset: 16>> - %b2 = memref.alloc() alignment = 64 : memref<96xi32> - %s2 = memref.alloc() alignment = 64 : memref<96xf32> - %lut = memref.alloc() alignment = 64 : memref<256xi8> - %gate = memref.alloc() alignment = 64 : memref<1x1x1x96xi8> - %mul = memref.alloc() alignment = 64 : memref<1x6x6x96xi8> - %w3raw = memref.alloc() alignment = 64 : memref<73744xi8> - %w3 = memref.reinterpret_cast %w3raw to offset: [16], sizes: [3, 96, 16, 16], strides: [24576, 256, 16, 1] : memref<73744xi8> to memref<3x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>> - %b3 = memref.alloc() alignment = 64 : memref<40xi32> - %s3 = memref.alloc() alignment = 64 : memref<40xf32> - %none = memref.alloc() alignment = 64 : memref<1xi8> - %output = memref.alloc() alignment = 64 : memref<1x6x6x40xi8> - scf.for %y = %c0 to %c6 step %c1 { - scf.for %x = %c0 to %c6 step %c1 { - scf.for %c = %c0 to %c96 step %c1 { - %a = arith.addi %y, %x : index - %d = arith.addi %a, %c : index - %r = arith.remui %d, %c5 : index - %v = arith.addi %r, %c1 : index - %q = arith.index_cast %v : index to i8 - memref.store %q, %input[%c0, %y, %x, %c] : memref<1x6x6x96xi8> - } - } - } - linalg.fill ins(%i8_1 : i8) outs(%w1 : memref<2x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>>) - linalg.fill ins(%i8_1 : i8) outs(%w2 : memref<6x24x16x16xi8, strided<[6144, 256, 16, 1], offset: 16>>) - linalg.fill ins(%i8_1 : i8) outs(%w3 : memref<3x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>>) - linalg.fill ins(%i32_1 : i32) outs(%b1 : memref<24xi32>) - linalg.fill ins(%i32_0 : i32) outs(%b2 : memref<96xi32>) - linalg.fill ins(%i32_0 : i32) outs(%b3 : memref<40xi32>) - linalg.fill ins(%f32_001 : f32) outs(%s1 : memref<24xf32>) - linalg.fill ins(%f32_01 : f32) outs(%s2 : memref<96xf32>) - linalg.fill ins(%f32_001 : f32) outs(%s3 : memref<40xf32>) - linalg.fill ins(%i8_0 : i8) outs(%none : memref<1xi8>) - scf.for %i = %c0 to %c256 step %c1 { - %v = arith.index_cast %i : index to i8 - memref.store %v, %lut[%i] : memref<256xi8> - } - linalg.fill ins(%i8_0 : i8) outs(%output : memref<1x6x6x40xi8>) - scf.for %iteration = %c0 to %c2 step %c1 { - tile.mega_kernel %input %output : memref<1x6x6x96xi8> memref<1x6x6x40xi8> { - tile.mega_global_avg_pool %input %gap : memref<1x6x6x96xi8> memref<1x1x1x96xi8> - tile.mega_conv2d %gap %w1 %b1 %s1 %none %h1 : memref<1x1x1x96xi8> memref<2x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>> memref<24xi32> memref<24xf32> memref<1xi8> memref<1x1x1x24xi8> - tile.mega_conv2d %h1 %w2 %b2 %s2 %lut %gate : memref<1x1x1x24xi8> memref<6x24x16x16xi8, strided<[6144, 256, 16, 1], offset: 16>> memref<96xi32> memref<96xf32> memref<256xi8> memref<1x1x1x96xi8> - tile.mega_int8_mul %gate %input %mul : memref<1x1x1x96xi8> memref<1x6x6x96xi8> memref<1x6x6x96xi8> - tile.mega_conv2d %mul %w3 %b3 %s3 %none %output : memref<1x6x6x96xi8> memref<3x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>> memref<40xi32> memref<40xf32> memref<1xi8> memref<1x6x6x40xi8> - } - } - func.call @check_result(%output) : (memref<1x6x6x40xi8>) -> () - return %i8_0 : i8 -} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/tile_matmul_16x16_i8.mlir b/examples/chips/pebble/workloads/mlir_tests/tile/tile_matmul_16x16_i8.mlir deleted file mode 100644 index 484ee1bdd..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/tile/tile_matmul_16x16_i8.mlir +++ /dev/null @@ -1,41 +0,0 @@ -func.func private @check_result(memref<16x16xi32>) -> () - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero_i32 = arith.constant 0 : i32 - %c0 = arith.constant 0 : index - %c1 = arith.constant 1 : index - %c2 = arith.constant 2 : index - %c3 = arith.constant 3 : index - %c5 = arith.constant 5 : index - %c7 = arith.constant 7 : index - %c16 = arith.constant 16 : index - %a = memref.alloc() alignment = 64 : memref<16x16xi8> - %b = memref.alloc() alignment = 64 : memref<16x16xi8> - %c = memref.alloc() alignment = 64 : memref<16x16xi32> - scf.for %i = %c0 to %c16 step %c1 { - scf.for %j = %c0 to %c16 step %c1 { - %i3 = arith.muli %i, %c3 : index - %j5 = arith.muli %j, %c5 : index - %as = arith.addi %i3, %j5 : index - %ar = arith.remui %as, %c7 : index - %av0 = arith.subi %ar, %c3 : index - %av = arith.index_cast %av0 : index to i8 - %i2 = arith.muli %i, %c2 : index - %j3 = arith.muli %j, %c3 : index - %bs = arith.addi %i2, %j3 : index - %br = arith.remui %bs, %c5 : index - %bv0 = arith.subi %br, %c2 : index - %bv = arith.index_cast %bv0 : index to i8 - memref.store %av, %a[%i, %j] : memref<16x16xi8> - memref.store %bv, %b[%i, %j] : memref<16x16xi8> - } - } - linalg.fill ins(%zero_i32 : i32) outs(%c : memref<16x16xi32>) - tile.tile_matmul %a %b %c : memref<16x16xi8> memref<16x16xi8> memref<16x16xi32> - func.call @check_result(%c) : (memref<16x16xi32>) -> () - memref.dealloc %a : memref<16x16xi8> - memref.dealloc %b : memref<16x16xi8> - memref.dealloc %c : memref<16x16xi32> - return %zero_i8 : i8 -} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/tile_matmul_16x16_i8_main.cpp b/examples/chips/pebble/workloads/mlir_tests/tile/tile_matmul_16x16_i8_main.cpp deleted file mode 100644 index 04b00f2a5..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/tile/tile_matmul_16x16_i8_main.cpp +++ /dev/null @@ -1,29 +0,0 @@ -#include -#include -#include - -static void fail(void) { -#ifdef BAREMETAL - *(volatile uint32_t *)0x60000000 = 1; - while (1) { - } -#else - exit(1); -#endif -} - -extern "C" void check_result(int32_t *, int32_t *out, int64_t offset, - int64_t rows, int64_t columns, int64_t row_stride, - int64_t column_stride) { - if (rows != 16 || columns != 16 || row_stride != 16 || column_stride != 1) - fail(); - for (int i = 0; i < 16; ++i) - for (int j = 0; j < 16; ++j) { - int expected = 0; - for (int k = 0; k < 16; ++k) - expected += ((3 * i + 5 * k) % 7 - 3) * ((2 * k + 3 * j) % 5 - 2); - if (out[offset + i * row_stride + j] != expected) - fail(); - } - printf("PASSED: tile.tile_matmul 16x16 ordered operands\n"); -} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/tile_transpose_16x16_i8.mlir b/examples/chips/pebble/workloads/mlir_tests/tile/tile_transpose_16x16_i8.mlir deleted file mode 100644 index 45febad14..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/tile/tile_transpose_16x16_i8.mlir +++ /dev/null @@ -1,27 +0,0 @@ -func.func private @check_result(memref<16x16xi8>) -> () - -func.func @main() -> i8 { - %zero = arith.constant 0 : i8 - %c0 = arith.constant 0 : index - %c1 = arith.constant 1 : index - %c3 = arith.constant 3 : i8 - %c5 = arith.constant 5 : i8 - %c16 = arith.constant 16 : index - %input = memref.alloc() alignment = 64 : memref<16x16xi8> - %output = memref.alloc() alignment = 64 : memref<16x16xi8> - scf.for %i = %c0 to %c16 step %c1 { - scf.for %j = %c0 to %c16 step %c1 { - %ii = arith.index_cast %i : index to i8 - %jj = arith.index_cast %j : index to i8 - %iv = arith.muli %ii, %c3 : i8 - %jv = arith.muli %jj, %c5 : i8 - %v = arith.addi %iv, %jv : i8 - memref.store %v, %input[%i, %j] : memref<16x16xi8> - } - } - tile.tile_transpose %input %output : memref<16x16xi8> memref<16x16xi8> - func.call @check_result(%output) : (memref<16x16xi8>) -> () - memref.dealloc %input : memref<16x16xi8> - memref.dealloc %output : memref<16x16xi8> - return %zero : i8 -} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/tile_transpose_16x16_i8_main.cpp b/examples/chips/pebble/workloads/mlir_tests/tile/tile_transpose_16x16_i8_main.cpp deleted file mode 100644 index c7d0cfa45..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/tile/tile_transpose_16x16_i8_main.cpp +++ /dev/null @@ -1,25 +0,0 @@ -#include -#include -#include - -static void fail(void) { -#ifdef BAREMETAL - *(volatile uint32_t *)0x60000000 = 1; - while (1) { - } -#else - exit(1); -#endif -} - -extern "C" void check_result(int8_t *, int8_t *out, int64_t offset, - int64_t rows, int64_t columns, int64_t row_stride, - int64_t column_stride) { - if (rows != 16 || columns != 16 || row_stride != 16 || column_stride != 1) - fail(); - for (int i = 0; i < 16; ++i) - for (int j = 0; j < 16; ++j) - if (out[offset + i * row_stride + j] != (int8_t)(3 * j + 5 * i)) - fail(); - printf("PASSED: tile.tile_transpose 16x16 ordered input\n"); -} diff --git a/examples/chips/poly/arch/src/main/scala/compute_tile/ComputeTile.scala b/examples/chips/poly/arch/src/main/scala/compute_tile/ComputeTile.scala deleted file mode 100644 index eb6de3909..000000000 --- a/examples/chips/poly/arch/src/main/scala/compute_tile/ComputeTile.scala +++ /dev/null @@ -1,21 +0,0 @@ -package examples.chips.poly.compute_tile - -import chisel3.experimental.hierarchy.{instantiable, Instance, Instantiate} -import framework.top.GlobalConfig -import framework.system.GlobalConfigOps._ -import framework.system.core.CoreFactory -import framework.system.tile.{ComputeTilePlatform, TileConnection, TileEndpoint, TileFactory} - -@instantiable -class ComputeTile(b: GlobalConfig) extends TileEndpoint(b.tileLinkParams, false) { - val p = b.tileParams - val platform = Instantiate(new ComputeTilePlatform(p)) - val controllerIndex = p.controller.get - val controller = CoreFactory.rocket(b.forCore(controllerIndex)) - val workers = p.cores.indices.filterNot(_ == controllerIndex).map(i => CoreFactory.ant(b.forCore(i))) - TileConnection.compute(platform, tlink, controller.clink, workers.map(_.clink)) -} - -object ComputeTile extends TileFactory { - def instantiate(b: GlobalConfig): Instance[TileEndpoint] = Instantiate(new ComputeTile(b)) -} diff --git a/examples/chips/poly/arch/src/main/scala/sims/p2e/TargetConfigs.scala b/examples/chips/poly/arch/src/main/scala/sims/p2e/TargetConfigs.scala deleted file mode 100644 index 283159298..000000000 --- a/examples/chips/poly/arch/src/main/scala/sims/p2e/TargetConfigs.scala +++ /dev/null @@ -1,10 +0,0 @@ -package sims.p2e - -import chisel3.experimental.hierarchy.{Instance, Instantiate} -import framework.system.{System, SystemParams} - -/** Poly on the P2E board: a CPU main tile and eight homogeneous compute tiles. */ -class PolyP2ETarget extends P2ETarget("../examples/chips/poly/configs/generated/chip.pb") { - override def instantiate(p: SystemParams): Instance[System] = - Instantiate(new framework.system.Device(p.toGlobalConfig)) -} diff --git a/examples/chips/poly/arch/src/main/scala/sims/verilator/TargetConfigs.scala b/examples/chips/poly/arch/src/main/scala/sims/verilator/TargetConfigs.scala deleted file mode 100644 index fadcf25bd..000000000 --- a/examples/chips/poly/arch/src/main/scala/sims/verilator/TargetConfigs.scala +++ /dev/null @@ -1,13 +0,0 @@ -package sims.verilator - -import chisel3.experimental.hierarchy.{Instance, Instantiate} -import framework.system.{System, SystemParams} - -import sims.soc.SystemTarget - -/** Poly on the explicit System: a CPU main tile and two homogeneous compute tiles. */ -class PolyVerilatorTarget extends SystemTarget("../examples/chips/poly/configs/generated/chip.pb") { - override def instantiate(p: SystemParams): Instance[System] = - Instantiate(new framework.system.Device(p.toGlobalConfig)) - override val dramBytes: BigInt = BigInt(16) << 30 -} diff --git a/examples/chips/poly/configs/chip.toml b/examples/chips/poly/configs/chip.toml deleted file mode 100644 index 9fa5b292a..000000000 --- a/examples/chips/poly/configs/chip.toml +++ /dev/null @@ -1,10 +0,0 @@ -[designs] -include = "designs/poly.toml" - -[sims] -verilator = "sims.verilator.PolyVerilatorTarget" -p2e = "sims.p2e.PolyP2ETarget" - -[uvm] -balls = ["smatmul", "toint8", "int2fp", "transpose"] -ips = [] diff --git a/examples/chips/poly/configs/designs/poly.toml b/examples/chips/poly/configs/designs/poly.toml deleted file mode 100644 index 8422b53f2..000000000 --- a/examples/chips/poly/configs/designs/poly.toml +++ /dev/null @@ -1,9 +0,0 @@ -# Poly: a CPU main tile and 8 homogeneous compute tiles, each -# 1 controller + 2 Attention + 3 FFN cores. - -[main_tile] -include = "tiles/main.toml" - -[[tiles]] -include = "tiles/compute.toml" -count = 8 diff --git a/examples/chips/poly/configs/designs/tiles/compute.toml b/examples/chips/poly/configs/designs/tiles/compute.toml deleted file mode 100644 index fd02dead1..000000000 --- a/examples/chips/poly/configs/designs/tiles/compute.toml +++ /dev/null @@ -1,58 +0,0 @@ -controllerCore = 0 -factory = "examples.chips.poly.compute_tile.ComputeTile" -# Poly compute tile: controller core 0, two Attention cores and three FFN cores. - -coreDataBytes = 64 -xLen = 64 -vaddrBits = 39 -paddrBits = 56 -pgIdxBits = 12 -pgLevels = 3 -nPMPs = 8 - -memBallChannelNum = 6 - -[tss] -base = 0x20000 -bytes = 16384 -dataBits = 128 - -[sharedMem] -bankWidth = 128 -bankEntries = 1024 -enable = true -virtualBankCount = 40 -entries = 16384 -inputChannels = 5 -defaultGroupCount = 1 -memCycles = 10 - -[[cores]] -core_id = 0 -name = "controller" -include = "../../../../../cores/rocket/configs/default.toml" - -[[cores]] -core_id = 1 -name = "attention" -include = "../../../../../cores/attention/configs/default.toml" - -[[cores]] -core_id = 2 -name = "attention" -include = "../../../../../cores/attention/configs/default.toml" - -[[cores]] -core_id = 3 -name = "ffn" -include = "../../../../../cores/ffn/configs/default.toml" - -[[cores]] -core_id = 4 -name = "ffn" -include = "../../../../../cores/ffn/configs/default.toml" - -[[cores]] -core_id = 5 -name = "ffn" -include = "../../../../../cores/ffn/configs/default.toml" diff --git a/examples/chips/poly/configs/designs/tiles/main.toml b/examples/chips/poly/configs/designs/tiles/main.toml deleted file mode 100644 index b2fd30e3a..000000000 --- a/examples/chips/poly/configs/designs/tiles/main.toml +++ /dev/null @@ -1,24 +0,0 @@ -# Poly main tile: one Linux core and shared storage for cross-tile reduction. - -coreDataBytes = 64 -xLen = 64 -vaddrBits = 39 -paddrBits = 56 -pgIdxBits = 12 -pgLevels = 3 -nPMPs = 8 -memBallChannelNum = 3 - -[sharedMem] -bankWidth = 128 -bankEntries = 1024 -enable = true -entries = 16384 -inputChannels = 3 -defaultGroupCount = 1 -virtualBankCount = 40 - -[[cores]] -core_id = 0 -name = "main" -include = "../../../../../cores/reduce/configs/default.toml" diff --git a/examples/chips/poly/kernel/workloads.toml b/examples/chips/poly/kernel/workloads.toml deleted file mode 100644 index 04d6b9c79..000000000 --- a/examples/chips/poly/kernel/workloads.toml +++ /dev/null @@ -1,10 +0,0 @@ -[workloads] -tests = [ - "poly-main-ctest-chip-controllers-linux", - "poly-main-ctest-f32add_groups-linux", - "poly-main-ctest-chip-ant_mxmm-linux", - "poly-main-ctest-chip-ant_shared-linux", - "poly-main-ctest-chip-ant_bank-linux", - "poly-main-ctest-chip-ant_parallel-linux", - "poly-main-ctest-chip-ant_stream-linux", -] diff --git a/examples/chips/poly/regression/batch/bemu/workloads-bare.toml b/examples/chips/poly/regression/batch/bemu/workloads-bare.toml deleted file mode 100644 index 70c1c5bce..000000000 --- a/examples/chips/poly/regression/batch/bemu/workloads-bare.toml +++ /dev/null @@ -1,11 +0,0 @@ -[workloads] -tests = [ - "poly-main-ctest-f32add_groups-baremetal", - "poly-main-ctest-tlink_lease-baremetal", - "poly-main-ctest-chip-controllers-multicore-baremetal", - "poly-main-ctest-chip-ant_mxmm-multicore-baremetal", - "poly-main-ctest-chip-ant_shared-multicore-baremetal", - "poly-main-ctest-chip-ant_bank-multicore-baremetal", - "poly-main-ctest-chip-ant_bank_rows-multicore-baremetal", - "poly-main-ctest-chip-ant_parallel-multicore-baremetal", -] diff --git a/examples/chips/poly/regression/batch/bemu/workloads-linux.toml b/examples/chips/poly/regression/batch/bemu/workloads-linux.toml deleted file mode 100644 index c6e1afe47..000000000 --- a/examples/chips/poly/regression/batch/bemu/workloads-linux.toml +++ /dev/null @@ -1,4 +0,0 @@ -[workloads] -tests = [ - "fw_payload-h9-poly-linux.elf", -] diff --git a/examples/chips/poly/regression/batch/p2e/workloads-linux.toml b/examples/chips/poly/regression/batch/p2e/workloads-linux.toml deleted file mode 100644 index 959714e33..000000000 --- a/examples/chips/poly/regression/batch/p2e/workloads-linux.toml +++ /dev/null @@ -1,4 +0,0 @@ -[workloads] -tests = [ - "fw_payload-h3-poly-linux.hex", -] diff --git a/examples/chips/poly/regression/batch/verilator/workloads-bare.toml b/examples/chips/poly/regression/batch/verilator/workloads-bare.toml deleted file mode 100644 index 38f366460..000000000 --- a/examples/chips/poly/regression/batch/verilator/workloads-bare.toml +++ /dev/null @@ -1,11 +0,0 @@ -[workloads] -tests = [ - "poly-main-ctest-f32add_groups-baremetal", - "poly-main-ctest-tlink_lease-baremetal", - "poly-main-ctest-chip-controllers-multicore-baremetal", - "poly-main-ctest-chip-ant_mesh_move-multicore-baremetal", - "poly-main-ctest-chip-ant_mxmm-multicore-baremetal", - "poly-main-ctest-chip-ant_shared-multicore-baremetal", - "poly-main-ctest-chip-ant_bank_rows-multicore-baremetal", - "poly-main-ctest-chip-ant_parallel-multicore-baremetal", -] diff --git a/examples/chips/poly/workloads/CMakeLists.txt b/examples/chips/poly/workloads/CMakeLists.txt deleted file mode 100644 index 8abc07554..000000000 --- a/examples/chips/poly/workloads/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -add_custom_target(poly-ctests-build) -set(BUCKYBALL_CTEST_GROUP_TARGET poly-ctests-build) -set(BUCKYBALL_CTEST_TARGET attention) -add_subdirectory(ctests) - - -add_custom_target(chip-ctests-build DEPENDS poly-ctests-build) - -add_custom_target(chip-workloads-build ALL - DEPENDS - chip-ctests-build - mlirtest-workloads-build - soctests-build - COMMENT "Building poly workloads" - VERBATIM) diff --git a/examples/chips/poly/workloads/ctests/CMakeLists.txt b/examples/chips/poly/workloads/ctests/CMakeLists.txt deleted file mode 100644 index c1eed72e1..000000000 --- a/examples/chips/poly/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,151 +0,0 @@ -# Cross-tile Linux control cores share one coherent LLC. -buckyball_ctest_name(POLY_CONTROL_NAME main chip-controllers) -set(POLY_CONTROL_ELF ${POLY_CONTROL_NAME}-multicore-baremetal) -buckyball_enforce_ctest_line_limit("${CMAKE_CURRENT_SOURCE_DIR}/controllers.c") -add_custom_command( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${POLY_CONTROL_ELF} - COMMAND ${ELF_CC} -O2 -g -march=rv64gc -mcmodel=medany -msmall-data-limit=0 - -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax -Wl,-T,${BBSIM_LD} - -I${BUCKYBALL_ISA_DIR}/chip -I${WORKLOAD_LIB_DIR}/bbsw/baremetal - -o ${POLY_CONTROL_ELF} - ${CMAKE_CURRENT_SOURCE_DIR}/controllers_start.S - ${CMAKE_CURRENT_SOURCE_DIR}/controllers.c -lgcc - DEPENDS controllers_start.S controllers.c ${WORKLOAD_LIB_DIR}/bbsw/baremetal/start.S - ${BBSIM_LD} - ${BUCKYBALL_ISA_DIR}/chip/topology.h - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) -add_custom_target(${POLY_CONTROL_NAME}-ctest-build - DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${POLY_CONTROL_ELF}) -add_dependencies(poly-ctests-build ${POLY_CONTROL_NAME}-ctest-build) - -set(BUCKYBALL_CTEST_TARGET main) -add_buckyball_linux_ctest(${POLY_CONTROL_NAME} ${CMAKE_CURRENT_SOURCE_DIR} controllers.c) -target_include_directories(${POLY_CONTROL_NAME}-linux PRIVATE ${BUCKYBALL_ISA_DIR}/chip) -target_link_libraries(${POLY_CONTROL_NAME}-linux PRIVATE pthread) -add_dependencies(poly-ctests-build ${POLY_CONTROL_NAME}-linux-build) - -foreach(POLY_ANT_CASE mxmm shared bank bank_rows parallel mesh_move highpa tlink tlink_bank tlink_reduce coherence) -set(POLY_ANT_CORE attention) -if(POLY_ANT_CASE STREQUAL "bank" OR POLY_ANT_CASE STREQUAL "bank_rows") - set(POLY_ANT_CORE ffn) -endif() -buckyball_ctest_name(POLY_ANT_NAME ${POLY_ANT_CORE} chip-ant_${POLY_ANT_CASE}) -foreach(_name BANK_WIDTH BANK_LINES BB_PRIVATE_VBANK_MAX) - file(STRINGS "${BUCKYBALL_ISA_DIR}/${POLY_ANT_CORE}/params.h" _line REGEX "^#define ${_name} ") - string(REGEX REPLACE ".* " "" _${_name} "${_line}") -endforeach() -if(NOT _BANK_WIDTH EQUAL 128 OR _BANK_LINES LESS 128 OR _BB_PRIVATE_VBANK_MAX LESS 5) - message(FATAL_ERROR - "Ant Mxmm needs 128-bit rows, >=128 rows/bank and private banks through 5; actual width=${_BANK_WIDTH}, rows=${_BANK_LINES}, privateMax=${_BB_PRIVATE_VBANK_MAX}") -endif() -set(POLY_ANT_IMAGE ${CMAKE_CURRENT_BINARY_DIR}/${POLY_ANT_NAME}-ant) -set(ANT_RUNTIME_DIR ${BUCKYBALL_REPO_ROOT}/stack/runtime/src/ant) -buckyball_enforce_ctest_line_limit("${CMAKE_CURRENT_SOURCE_DIR}/ant_${POLY_ANT_CASE}.c") -add_custom_command( - OUTPUT ${POLY_ANT_IMAGE}.elf ${POLY_ANT_IMAGE}.bin - COMMAND ${ELF_CC} -O2 -g -march=rv64im -mabi=lp64 -mcmodel=medany - -ffreestanding -fno-builtin -fno-pic -fno-stack-protector - -fno-asynchronous-unwind-tables -msmall-data-limit=0 -mno-relax - -nostdlib -nostartfiles -Wl,--no-relax -Wl,-T,${ANT_RUNTIME_DIR}/code.ld - -I${WORKLOAD_LIB_DIR} -I${BUCKYBALL_ISA_DIR}/${POLY_ANT_CORE} - -I${BUCKYBALL_REPO_ROOT}/examples/balls/mxmm/workloads - -I${BUCKYBALL_REPO_ROOT}/stack/runtime/include - ${ANT_RUNTIME_DIR}/start.S ${CMAKE_CURRENT_SOURCE_DIR}/ant_${POLY_ANT_CASE}.c - -o ${POLY_ANT_IMAGE}.elf - COMMAND ${RISCV_GNU_TOOLCHAIN}/bin/riscv64-unknown-elf-objcopy - -O binary --only-section=.text ${POLY_ANT_IMAGE}.elf ${POLY_ANT_IMAGE}.bin - DEPENDS ant_${POLY_ANT_CASE}.c ant_${POLY_ANT_CASE}.h ${ANT_RUNTIME_DIR}/start.S ${ANT_RUNTIME_DIR}/code.ld - ${BUCKYBALL_ISA_DIR}/${POLY_ANT_CORE}/params.h ${BUCKYBALL_ISA_DIR}/${POLY_ANT_CORE}/ballISA.h - ${BUCKYBALL_BALL_ISA_DEPS}) -add_custom_target(${POLY_ANT_NAME}-ant-build - DEPENDS ${POLY_ANT_IMAGE}.elf ${POLY_ANT_IMAGE}.bin) -add_dependencies(poly-ctests-build ${POLY_ANT_NAME}-ant-build) - -set(POLY_ANT_CONTROL_SOURCES) -if(POLY_ANT_CASE STREQUAL "bank" OR POLY_ANT_CASE STREQUAL "bank_rows" OR POLY_ANT_CASE STREQUAL "parallel") - list(APPEND POLY_ANT_CONTROL_SOURCES - ${CMAKE_CURRENT_SOURCE_DIR}/ant_${POLY_ANT_CASE}_exercise.c) -elseif(POLY_ANT_CASE STREQUAL "tlink_reduce") - list(APPEND POLY_ANT_CONTROL_SOURCES - ${CMAKE_CURRENT_SOURCE_DIR}/ant_tlink_reduce_control.c) -endif() -foreach(SOURCE IN LISTS POLY_ANT_CONTROL_SOURCES) - buckyball_enforce_ctest_line_limit("${SOURCE}") -endforeach() - -set(POLY_ANT_EMBED ${CMAKE_CURRENT_BINARY_DIR}/ant_${POLY_ANT_CASE}_image.S) -enable_language(ASM) -file(GENERATE OUTPUT ${POLY_ANT_EMBED} CONTENT -".section .rodata.ant,\"a\",@progbits\n.balign 8\n.global ant_image, ant_image_end\nant_image:\n.incbin \"${POLY_ANT_IMAGE}.bin\"\nant_image_end:\n.section .note.GNU-stack,\"\",@progbits\n") -set_source_files_properties(${POLY_ANT_EMBED} PROPERTIES - OBJECT_DEPENDS "${POLY_ANT_IMAGE}.bin") -set(POLY_ANT_MAIN ant_${POLY_ANT_CASE}_main.c) -if(POLY_ANT_CASE STREQUAL "mxmm") - set(POLY_ANT_MAIN ant_mxmm_main.cpp) -endif() -buckyball_enforce_ctest_line_limit("${CMAKE_CURRENT_SOURCE_DIR}/${POLY_ANT_MAIN}") -buckyball_ctest_name(POLY_ANT_HOST_NAME main chip-ant_${POLY_ANT_CASE}) -set(BUCKYBALL_CTEST_TARGET ${POLY_ANT_CORE}) -if(NOT POLY_ANT_CASE STREQUAL "highpa" AND NOT POLY_ANT_CASE MATCHES "^tlink" AND - NOT POLY_ANT_CASE STREQUAL "coherence" AND NOT POLY_ANT_CASE STREQUAL "mesh_move") -add_buckyball_linux_ctest(${POLY_ANT_HOST_NAME} ${CMAKE_CURRENT_SOURCE_DIR} ${POLY_ANT_MAIN}) -target_sources(${POLY_ANT_HOST_NAME}-linux PRIVATE ${POLY_ANT_EMBED} ${POLY_ANT_CONTROL_SOURCES}) -target_include_directories(${POLY_ANT_HOST_NAME}-linux PRIVATE ${BUCKYBALL_ISA_DIR}/chip) -target_link_libraries(${POLY_ANT_HOST_NAME}-linux PRIVATE runtime_ant pthread) -add_dependencies(${POLY_ANT_HOST_NAME}-linux ${POLY_ANT_NAME}-ant-build) -add_dependencies(poly-ctests-build ${POLY_ANT_HOST_NAME}-linux-build) - -endif() - -set(POLY_ANT_HOST_CORE ${POLY_ANT_CORE}) -set(POLY_ANT_CONTROL_OBJECTS) -if(POLY_ANT_CASE STREQUAL "tlink_reduce") - set(POLY_ANT_HOST_CORE main) - set(POLY_ANT_CONTROL_OBJECTS ${CMAKE_CURRENT_BINARY_DIR}/ant_tlink_reduce_control.o) - add_custom_command(OUTPUT ${POLY_ANT_CONTROL_OBJECTS} - COMMAND ${ELF_CC} -O2 -g -march=rv64gc -mcmodel=medany -msmall-data-limit=0 -fno-builtin - -I${BUCKYBALL_ISA_DIR}/${POLY_ANT_CORE} - -I${BUCKYBALL_REPO_ROOT}/stack/runtime/include - -c ${POLY_ANT_CONTROL_SOURCES} -o ${POLY_ANT_CONTROL_OBJECTS} - DEPENDS ${POLY_ANT_CONTROL_SOURCES} ant_tlink_reduce.h - ${BUCKYBALL_ISA_DIR}/${POLY_ANT_CORE}/params.h) - set(POLY_ANT_CONTROL_SOURCES) -endif() - -set(POLY_ANT_HOST_ELF ${POLY_ANT_HOST_NAME}-multicore-baremetal) -add_custom_command( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${POLY_ANT_HOST_ELF} - COMMAND ${ELF_CC} -O2 -g -march=rv64gc -mcmodel=medany -msmall-data-limit=0 - -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax -Wl,-T,${BBSIM_LD} - -I${BUCKYBALL_ISA_DIR}/chip -I${BUCKYBALL_ISA_DIR}/${POLY_ANT_HOST_CORE} - -I${WORKLOAD_LIB_DIR} - -I${BUCKYBALL_REPO_ROOT}/examples/balls/f32add/workloads - -I${BUCKYBALL_REPO_ROOT}/stack/runtime/include -I${WORKLOAD_LIB_DIR}/bbsw/baremetal - ${CMAKE_CURRENT_SOURCE_DIR}/controllers_start.S - ${CMAKE_CURRENT_SOURCE_DIR}/${POLY_ANT_MAIN} ${POLY_ANT_CONTROL_SOURCES} ${POLY_ANT_CONTROL_OBJECTS} ${POLY_ANT_EMBED} - ${BUCKYBALL_REPO_ROOT}/stack/runtime/src/ant.c - ${BUCKYBALL_REPO_ROOT}/stack/runtime/src/tlink.c -lgcc -o ${POLY_ANT_HOST_ELF} - DEPENDS ${POLY_ANT_MAIN} ant_${POLY_ANT_CASE}.h controllers_start.S ${POLY_ANT_EMBED} - ${BBSW_BAREMETAL_DIR}/start.S ${BBSIM_LD} - ${POLY_ANT_CONTROL_SOURCES} ${POLY_ANT_CONTROL_OBJECTS} - ${BUCKYBALL_ISA_DIR}/${POLY_ANT_HOST_CORE}/params.h - ${BUCKYBALL_ISA_DIR}/${POLY_ANT_HOST_CORE}/ballISA.h - ${POLY_ANT_IMAGE}.bin ${BUCKYBALL_REPO_ROOT}/stack/runtime/src/ant.c - ${BUCKYBALL_REPO_ROOT}/stack/runtime/include/ant.h - ${BUCKYBALL_REPO_ROOT}/stack/runtime/src/tlink.c ${BUCKYBALL_REPO_ROOT}/stack/runtime/include/tlink.h - ${BUCKYBALL_ISA_DIR}/${POLY_ANT_CORE}/params.h ${BUCKYBALL_ISA_DIR}/chip/topology.h - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) -add_custom_target(${POLY_ANT_HOST_NAME}-ctest-build - DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${POLY_ANT_HOST_ELF}) -add_dependencies(poly-ctests-build ${POLY_ANT_HOST_NAME}-ctest-build) -endforeach() - -buckyball_ctest_name(POLY_STREAM_NAME main chip-ant_stream) -buckyball_enforce_ctest_line_limit("${CMAKE_CURRENT_SOURCE_DIR}/ant_stream_main.c") -add_executable(${POLY_STREAM_NAME}-linux ant_stream_main.c) -target_include_directories(${POLY_STREAM_NAME}-linux PRIVATE - ${BUCKYBALL_ISA_DIR}/chip ${BUCKYBALL_ISA_DIR}/attention ${WORKLOAD_LIB_DIR} - ${BUCKYBALL_REPO_ROOT}/examples/balls/mxmm/workloads) -target_link_libraries(${POLY_STREAM_NAME}-linux PRIVATE runtime_ant_tasks) -target_include_directories(runtime_ant_tasks PRIVATE ${BUCKYBALL_ISA_DIR}/attention) -add_dependencies(poly-ctests-build ${POLY_STREAM_NAME}-linux) diff --git a/examples/chips/poly/workloads/ctests/ant_bank.c b/examples/chips/poly/workloads/ctests/ant_bank.c deleted file mode 100644 index 31a9026c2..000000000 --- a/examples/chips/poly/workloads/ctests/ant_bank.c +++ /dev/null @@ -1,18 +0,0 @@ -#include "ant_bank.h" -#include - -uint64_t ant_main(const struct ant_bank_args *args) { - bb_mset(3, 1, 1, 1); - bb_mvin(args->input, 3, BANK_LINES, 1); - bb_mvout(args->private_out, 3, BANK_LINES, 1); - bb_mset(3, 0, 0, 0); - bb_mset(BB_SHARED_BANK_BASE, 1, 1, 1); - bb_mvin(args->input, BB_SHARED_BANK_BASE, args->shared_rows, 1); - bb_mvout(args->shared_out, BB_SHARED_BANK_BASE, args->shared_rows, 1); - bb_mset(BB_SHARED_BANK_BASE, 0, 0, 0); - bb_mset(3, 1, 1, 1); - bb_mvin_2d(args->input, 3, 2, 16, 2, 0, 2, 8); - bb_mvout(args->tail_out, 3, 4, 1); - bb_mset(3, 0, 0, 0); - return 0; -} diff --git a/examples/chips/poly/workloads/ctests/ant_bank_exercise.c b/examples/chips/poly/workloads/ctests/ant_bank_exercise.c deleted file mode 100644 index 1c3a098a7..000000000 --- a/examples/chips/poly/workloads/ctests/ant_bank_exercise.c +++ /dev/null @@ -1,60 +0,0 @@ -#include "ant_bank.h" -#include -#include -#include -extern const unsigned char ant_image[], ant_image_end[]; -enum { - WORDS = BANK_LINES * BANK_WIDTH / 64, - SHARED_WORDS = BB_SHARED_BANK_LINES * BANK_WIDTH / 64 -}; -static uint64_t input[BB_COMPUTE_TILES][WORDS] - __attribute__((section(".data"), aligned(4096))) = {0}; -static uint64_t private_out[BB_COMPUTE_TILES][WORDS] - __attribute__((section(".data"), aligned(4096))) = {0}; -static uint64_t shared_out[BB_COMPUTE_TILES][SHARED_WORDS] - __attribute__((section(".data"), aligned(4096))) = {0}; -static uint64_t tail_out[BB_COMPUTE_TILES][8] - __attribute__((section(".data"), aligned(4096))) = {0}; -unsigned bank_done[BB_COMPUTE_TILES]; -void bank_exercise(unsigned tile) { - const unsigned context = BB_ANT_CONTEXT; - for (unsigned i = 0; i < WORDS; ++i) - input[tile][i] = (uint64_t)(tile + 1) << 32 | (i + 1); - for (unsigned i = 0; i < WORDS; ++i) - private_out[tile][i] = 0; - for (unsigned i = 0; i < SHARED_WORDS; ++i) - shared_out[tile][i] = 0; - for (unsigned i = 0; i < 8; ++i) - tail_out[tile][i] = ~0ULL; - size_t bytes = ant_image_end - ant_image; - if (bytes > ant_query(context, ANT_CODE_BYTES) || - ant_query(context, ANT_SIGNATURE) != CORE_SIGNATURE) - __builtin_trap(); - struct ant_bank_args args = {(uintptr_t)input[tile], - (uintptr_t)private_out[tile], - (uintptr_t)shared_out[tile], - (uintptr_t)tail_out[tile], BB_SHARED_BANK_LINES}; - ant_write(context, ANT_CODE, 0, ant_image, bytes); - ant_write(context, ANT_TLS, 0, &args, sizeof(args)); - uint64_t base = ant_query(context, ANT_TLS_BASE); - struct ant_task task = { - tile + 1, 0, bytes, base, base + ant_query(context, ANT_TLS_BYTES), - CORE_SIGNATURE}; - ant_acquire(); - ant_start(context, &task); - int cancelled; - if (ant_wait(context, task.id, &cancelled) || cancelled) - __builtin_trap(); - ant_release(); - for (unsigned i = 0; i < WORDS; ++i) - if (private_out[tile][i] != (((uint64_t)(tile + 1) << 32) | (i + 1))) - __builtin_trap(); - for (unsigned i = 0; i < SHARED_WORDS; ++i) - if (shared_out[tile][i] != (((uint64_t)(tile + 1) << 32) | (i + 1))) - __builtin_trap(); - for (unsigned i = 0; i < 8; ++i) - if (tail_out[tile][i] != - (i % 2 ? 0 : (((uint64_t)(tile + 1) << 32) | (i + 1)))) - __builtin_trap(); - __atomic_store_n(&bank_done[tile], 1, __ATOMIC_RELEASE); -} diff --git a/examples/chips/poly/workloads/ctests/ant_bank_main.c b/examples/chips/poly/workloads/ctests/ant_bank_main.c deleted file mode 100644 index 87507e4f9..000000000 --- a/examples/chips/poly/workloads/ctests/ant_bank_main.c +++ /dev/null @@ -1,49 +0,0 @@ -#define _GNU_SOURCE -#include "ant_bank.h" -#include -#include -#include -#ifdef __linux__ -#include -#include -#include -#include -#include -#endif -#ifdef __linux__ -static void *worker(void *argument) { - uintptr_t tile = (uintptr_t)argument; - cpu_set_t mask; - CPU_ZERO(&mask); - CPU_SET(BB_MAIN_CORES + tile, &mask); - if (pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask)) - abort(); - bank_exercise(tile); - return NULL; -} -#endif -int main(void) { -#ifdef __linux__ - if (mlockall(MCL_CURRENT | MCL_FUTURE)) - abort(); - pthread_t threads[BB_COMPUTE_TILES]; - for (uintptr_t tile = 0; tile < BB_COMPUTE_TILES; ++tile) - if (pthread_create(&threads[tile], NULL, worker, (void *)tile)) - abort(); - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - if (pthread_join(threads[tile], NULL)) - abort(); - printf("Ant bank boundary PASS tiles=%u private=%u shared=%u\n", - (unsigned)BB_COMPUTE_TILES, BANK_LINES, BB_SHARED_BANK_LINES); -#else - uintptr_t hart; - __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); - if (hart >= BB_MAIN_CORES) - bank_exercise(hart - BB_MAIN_CORES); - else - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - while (!__atomic_load_n(&bank_done[tile], __ATOMIC_ACQUIRE)) { - } -#endif - return 0; -} diff --git a/examples/chips/poly/workloads/ctests/ant_bank_rows.c b/examples/chips/poly/workloads/ctests/ant_bank_rows.c deleted file mode 100644 index 17d8d191a..000000000 --- a/examples/chips/poly/workloads/ctests/ant_bank_rows.c +++ /dev/null @@ -1,18 +0,0 @@ -#include "ant_bank_rows.h" -#include - -uint64_t ant_main(const struct ant_bank_rows_args *args) { - bb_mset(3, 1, 1, 1); - bb_mvin(args->input, 3, ANT_BANK_ROWS, 1); - bb_mvout(args->private_out, 3, ANT_BANK_ROWS, 1); - bb_mset(3, 0, 0, 0); - bb_mset(BB_SHARED_BANK_BASE, 1, 1, 1); - bb_mvin(args->input, BB_SHARED_BANK_BASE, args->shared_rows, 1); - bb_mvout(args->shared_out, BB_SHARED_BANK_BASE, args->shared_rows, 1); - bb_mset(BB_SHARED_BANK_BASE, 0, 0, 0); - bb_mset(3, 1, 1, 1); - bb_mvin_2d(args->input, 3, 2, 16, 2, 0, 2, 8); - bb_mvout(args->tail_out, 3, 4, 1); - bb_mset(3, 0, 0, 0); - return 0; -} diff --git a/examples/chips/poly/workloads/ctests/ant_bank_rows_exercise.c b/examples/chips/poly/workloads/ctests/ant_bank_rows_exercise.c deleted file mode 100644 index bd6eaa0ee..000000000 --- a/examples/chips/poly/workloads/ctests/ant_bank_rows_exercise.c +++ /dev/null @@ -1,59 +0,0 @@ -#include "ant_bank_rows.h" -#include -#include -#include -extern const unsigned char ant_image[], ant_image_end[]; -enum { - WORDS = ANT_BANK_ROWS * BANK_WIDTH / 64, - SHARED_WORDS = ANT_BANK_ROWS * BANK_WIDTH / 64 -}; -static uint64_t input[BB_COMPUTE_TILES][WORDS] - __attribute__((section(".data"), aligned(4096))) = {0}; -static uint64_t private_out[BB_COMPUTE_TILES][WORDS] - __attribute__((section(".data"), aligned(4096))) = {0}; -static uint64_t shared_out[BB_COMPUTE_TILES][SHARED_WORDS] - __attribute__((section(".data"), aligned(4096))) = {0}; -static uint64_t tail_out[BB_COMPUTE_TILES][8] - __attribute__((section(".data"), aligned(4096))) = {0}; -unsigned bank_rows_done[BB_COMPUTE_TILES]; -void bank_rows_exercise(unsigned tile) { - const unsigned context = BB_ANT_CONTEXT; - for (unsigned i = 0; i < WORDS; ++i) - input[tile][i] = (uint64_t)(tile + 1) << 32 | (i + 1); - for (unsigned i = 0; i < WORDS; ++i) - private_out[tile][i] = 0; - for (unsigned i = 0; i < SHARED_WORDS; ++i) - shared_out[tile][i] = 0; - for (unsigned i = 0; i < 8; ++i) - tail_out[tile][i] = ~0ULL; - size_t bytes = ant_image_end - ant_image; - if (bytes > ant_query(context, ANT_CODE_BYTES) || - ant_query(context, ANT_SIGNATURE) != CORE_SIGNATURE) - __builtin_trap(); - struct ant_bank_rows_args args = { - (uintptr_t)input[tile], (uintptr_t)private_out[tile], - (uintptr_t)shared_out[tile], (uintptr_t)tail_out[tile], ANT_BANK_ROWS}; - ant_write(context, ANT_CODE, 0, ant_image, bytes); - ant_write(context, ANT_TLS, 0, &args, sizeof(args)); - uint64_t base = ant_query(context, ANT_TLS_BASE); - struct ant_task task = { - tile + 1, 0, bytes, base, base + ant_query(context, ANT_TLS_BYTES), - CORE_SIGNATURE}; - ant_acquire(); - ant_start(context, &task); - int cancelled; - if (ant_wait(context, task.id, &cancelled) || cancelled) - __builtin_trap(); - ant_release(); - for (unsigned i = 0; i < WORDS; ++i) - if (private_out[tile][i] != (((uint64_t)(tile + 1) << 32) | (i + 1))) - __builtin_trap(); - for (unsigned i = 0; i < SHARED_WORDS; ++i) - if (shared_out[tile][i] != (((uint64_t)(tile + 1) << 32) | (i + 1))) - __builtin_trap(); - for (unsigned i = 0; i < 8; ++i) - if (tail_out[tile][i] != - (i % 2 ? 0 : (((uint64_t)(tile + 1) << 32) | (i + 1)))) - __builtin_trap(); - __atomic_store_n(&bank_rows_done[tile], 1, __ATOMIC_RELEASE); -} diff --git a/examples/chips/poly/workloads/ctests/ant_bank_rows_main.c b/examples/chips/poly/workloads/ctests/ant_bank_rows_main.c deleted file mode 100644 index 800d52376..000000000 --- a/examples/chips/poly/workloads/ctests/ant_bank_rows_main.c +++ /dev/null @@ -1,49 +0,0 @@ -#define _GNU_SOURCE -#include "ant_bank_rows.h" -#include -#include -#include -#ifdef __linux__ -#include -#include -#include -#include -#include -#endif -#ifdef __linux__ -static void *worker(void *argument) { - uintptr_t tile = (uintptr_t)argument; - cpu_set_t mask; - CPU_ZERO(&mask); - CPU_SET(BB_MAIN_CORES + tile, &mask); - if (pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask)) - abort(); - bank_rows_exercise(tile); - return NULL; -} -#endif -int main(void) { -#ifdef __linux__ - if (mlockall(MCL_CURRENT | MCL_FUTURE)) - abort(); - pthread_t threads[BB_COMPUTE_TILES]; - for (uintptr_t tile = 0; tile < BB_COMPUTE_TILES; ++tile) - if (pthread_create(&threads[tile], NULL, worker, (void *)tile)) - abort(); - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - if (pthread_join(threads[tile], NULL)) - abort(); - printf("Ant bank boundary PASS tiles=%u private=%u shared=%u\n", - (unsigned)BB_COMPUTE_TILES, BANK_LINES, BB_SHARED_BANK_LINES); -#else - uintptr_t hart; - __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); - if (hart >= BB_MAIN_CORES) - bank_rows_exercise(hart - BB_MAIN_CORES); - else - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - while (!__atomic_load_n(&bank_rows_done[tile], __ATOMIC_ACQUIRE)) { - } -#endif - return 0; -} diff --git a/examples/chips/poly/workloads/ctests/ant_coherence_main.c b/examples/chips/poly/workloads/ctests/ant_coherence_main.c deleted file mode 100644 index 10bb5933a..000000000 --- a/examples/chips/poly/workloads/ctests/ant_coherence_main.c +++ /dev/null @@ -1,89 +0,0 @@ -#include "ant_coherence.h" -#include -#include -#include -_Static_assert(BB_MAIN_CORES == 1 && BB_COMPUTE_TILES >= 2 && - !(BB_COMPUTE_TILES & (BB_COMPUTE_TILES - 1)), - "coherence gate requires a power-of-two compute tile count"); -extern const unsigned char ant_image[], ant_image_end[]; -// Separate 64-byte lines/sets for data and synchronization; avoid main's -// top-of-page stack sets. -enum { - INPUT = 8, - OUTPUT = INPUT + BB_COMPUTE_TILES, - READY = OUTPUT + BB_COMPUTE_TILES, - DONE = READY + BB_COMPUTE_TILES, - ROUND = DONE + BB_COMPUTE_TILES -}; -static volatile uint64_t lines[ROUND + 1][8] - __attribute__((section(".data"), aligned(4096))) = {0}; -static uint64_t pattern(unsigned round, unsigned tile, unsigned word) { - return ((uint64_t)round << 48) | ((uint64_t)(tile + 1) << 32) | (word + 17); -} -static void task(unsigned tile, unsigned phase, unsigned id) { - const unsigned context = BB_ANT_CONTEXT; - struct ant_coherence_args args = {phase, (uintptr_t)lines[INPUT + tile], - (uintptr_t)lines[OUTPUT + tile]}; - ant_write(context, ANT_TLS, 0, &args, sizeof(args)); - uint64_t base = ant_query(context, ANT_TLS_BASE); - struct ant_task command = {id, - 0, - ant_image_end - ant_image, - base, - base + ant_query(context, ANT_TLS_BYTES), - CORE_SIGNATURE}; - ant_acquire(); - ant_start(context, &command); - int cancelled; - if (ant_wait(context, id, &cancelled) || cancelled) - __builtin_trap(); - ant_release(); -} -static void controller(unsigned tile) { - size_t bytes = ant_image_end - ant_image; - if (bytes > ant_query(BB_ANT_CONTEXT, ANT_CODE_BYTES) || - ant_query(BB_ANT_CONTEXT, ANT_SIGNATURE) != CORE_SIGNATURE) - __builtin_trap(); - ant_write(BB_ANT_CONTEXT, ANT_CODE, 0, ant_image, bytes); - task(tile, 0, 1); - __atomic_store_n(&lines[READY + tile][0], 1, __ATOMIC_RELEASE); - for (unsigned round = 1; round <= 2; ++round) { - while (__atomic_load_n(&lines[ROUND][0], __ATOMIC_ACQUIRE) != round) { - } - task(tile, 1, round + 1); - __atomic_store_n(&lines[DONE + tile][0], round, __ATOMIC_RELEASE); - } - task(tile, 2, 4); - __atomic_store_n(&lines[DONE + tile][0], 3, __ATOMIC_RELEASE); -} -int main(void) { - uintptr_t hart; - __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); - if (hart) - controller(hart - 1); - else { - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - while (!__atomic_load_n(&lines[READY + tile][0], __ATOMIC_ACQUIRE)) { - } - for (unsigned round = 1; round <= 2; ++round) { - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - for (unsigned word = 0; word < 2; ++word) { - lines[INPUT + tile][word] = pattern(round, tile, word); - lines[OUTPUT + tile][word] = ~pattern(round, tile, word); - } - __atomic_store_n(&lines[ROUND][0], round, __ATOMIC_RELEASE); - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - while (__atomic_load_n(&lines[DONE + tile][0], __ATOMIC_ACQUIRE) < - round) { - } - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - for (unsigned word = 0; word < 2; ++word) - if (lines[OUTPUT + tile][word] != pattern(round, tile, word)) - __builtin_trap(); - } - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - while (__atomic_load_n(&lines[DONE + tile][0], __ATOMIC_ACQUIRE) != 3) { - } - } - return 0; -} diff --git a/examples/chips/poly/workloads/ctests/ant_highpa_main.c b/examples/chips/poly/workloads/ctests/ant_highpa_main.c deleted file mode 100644 index 78b736a8c..000000000 --- a/examples/chips/poly/workloads/ctests/ant_highpa_main.c +++ /dev/null @@ -1,64 +0,0 @@ -#include "ant_highpa.h" -#include -#include -#include -extern const unsigned char ant_image[], ant_image_end[]; -static unsigned done[BB_COMPUTE_TILES]; -static void exercise(unsigned tile) { - const unsigned context = BB_ANT_CONTEXT; - uintptr_t high = 0x180200000ULL + tile * 0x20000ULL; - volatile uint64_t *input = (void *)high, *output = (void *)(high + 0x1000); - volatile uint64_t *low = (void *)(high & 0xffffffffULL); - volatile uint64_t *sentinel = (void *)((high + 0x1000) & 0xffffffffULL); - const uint64_t marker = 0xdeadface12345678ULL; - uint64_t first = 0x1234567800000000ULL | ((uint64_t)tile << 16); - uint64_t second = 0xfedcba9800000000ULL | ((uint64_t)tile << 16); - for (unsigned i = 0; i < 4; ++i) { - low[i] = second + i; - sentinel[i] = marker; - input[i] = first + i; - output[i] = 0; - } - for (unsigned i = 0; i < 4; ++i) - if (input[i] != first + i || low[i] != second + i || sentinel[i] != marker) - __builtin_trap(); - size_t bytes = ant_image_end - ant_image; - if (bytes > ant_query(context, ANT_CODE_BYTES) || - ant_query(context, ANT_SIGNATURE) != CORE_SIGNATURE) - __builtin_trap(); - ant_write(context, ANT_CODE, 0, ant_image, bytes); - uint64_t base = ant_query(context, ANT_TLS_BASE); - for (unsigned phase = 0; phase < 2; ++phase) { - struct ant_highpa_args args = {phase ? (uintptr_t)low : high, - high + 0x1000}; - ant_write(context, ANT_TLS, 0, &args, sizeof(args)); - struct ant_task task = {tile * 2 + phase + 1, - 0, - bytes, - base, - base + ant_query(context, ANT_TLS_BYTES), - CORE_SIGNATURE}; - ant_acquire(); - ant_start(context, &task); - int cancelled; - if (ant_wait(context, task.id, &cancelled) || cancelled) - __builtin_trap(); - ant_release(); - for (unsigned i = 0; i < 4; ++i) - if (output[i] != (phase ? second : first) + i || sentinel[i] != marker || - input[i] != first + i || low[i] != second + i) - __builtin_trap(); - } - __atomic_store_n(&done[tile], 1, __ATOMIC_RELEASE); -} -int main(void) { - uintptr_t hart; - __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); - if (hart >= BB_MAIN_CORES) - exercise(hart - BB_MAIN_CORES); - else - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - while (!__atomic_load_n(&done[tile], __ATOMIC_ACQUIRE)) { - } - return 0; -} diff --git a/examples/chips/poly/workloads/ctests/ant_mesh_move_main.c b/examples/chips/poly/workloads/ctests/ant_mesh_move_main.c deleted file mode 100644 index 221653a3e..000000000 --- a/examples/chips/poly/workloads/ctests/ant_mesh_move_main.c +++ /dev/null @@ -1,89 +0,0 @@ -#define _GNU_SOURCE -#include "ant_mesh_move.h" -#include -#include -#include - -extern const unsigned char ant_image[], ant_image_end[]; -enum { ROW_WORDS = BANK_WIDTH / 64, ROWS = 2 }; -static uint64_t input[BB_COMPUTE_TILES][ROWS * ROW_WORDS] - __attribute__((section(".data"), aligned(4096))); -static uint64_t output[BB_COMPUTE_TILES][ROWS * ROW_WORDS] - __attribute__((section(".data"), aligned(4096))); -static unsigned done[BB_COMPUTE_TILES]; - -static void run(unsigned context, unsigned id, - struct ant_mesh_move_args *args) { - uint64_t base = ant_query(context, ANT_TLS_BASE); - ant_write(context, ANT_TLS, 0, args, sizeof(*args)); - struct ant_task task = {id, - 0, - ant_image_end - ant_image, - base, - base + ant_query(context, ANT_TLS_BYTES), - CORE_SIGNATURE}; - ant_start(context, &task); - int cancelled; - if (ant_wait(context, id, &cancelled) || cancelled) - __builtin_trap(); -} - -static struct ant_task prepare(unsigned context, unsigned id, - struct ant_mesh_move_args *args) { - uint64_t base = ant_query(context, ANT_TLS_BASE); - ant_write(context, ANT_TLS, 0, args, sizeof(*args)); - return (struct ant_task){id, - 0, - ant_image_end - ant_image, - base, - base + ant_query(context, ANT_TLS_BYTES), - CORE_SIGNATURE}; -} - -static void exercise(unsigned tile) { - size_t bytes = ant_image_end - ant_image; - for (unsigned context = 0; context < 2; ++context) { - if (bytes > ant_query(context, ANT_CODE_BYTES) || - ant_query(context, ANT_SIGNATURE) != CORE_SIGNATURE) - __builtin_trap(); - ant_write(context, ANT_CODE, 0, ant_image, bytes); - } - for (unsigned i = 0; i < ROWS * ROW_WORDS; ++i) { - input[tile][i] = ((uint64_t)(tile + 1) << 56) | (0x100 + i); - output[tile][i] = 0; - } - uint64_t zero[2] = {0}; - ant_write(0, ANT_TSS, 0, zero, sizeof(zero)); - struct ant_mesh_move_args args = {(uintptr_t)input[tile], - (uintptr_t)output[tile], - ant_query(0, ANT_TSS_BASE), MESH_FILL}; - ant_acquire(); - run(0, tile * 3 + 1, &args); - args.role = MESH_MOVE; - struct ant_task move = prepare(0, tile * 3 + 2, &args); - args.role = MESH_CONSUME; - struct ant_task consume = prepare(1, tile * 3 + 3, &args); - ant_start(1, &consume); - ant_start(0, &move); - int cancelled; - if (ant_wait(0, move.id, &cancelled) || cancelled || - ant_wait(1, consume.id, &cancelled) || cancelled) - __builtin_trap(); - ant_release(); - for (unsigned i = 0; i < ROWS * ROW_WORDS; ++i) - if (output[tile][i] != input[tile][i]) - __builtin_trap(); - __atomic_store_n(&done[tile], 1, __ATOMIC_RELEASE); -} - -int main(void) { - uintptr_t hart; - __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); - if (hart >= BB_MAIN_CORES) - exercise(hart - BB_MAIN_CORES); - else - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - while (!__atomic_load_n(&done[tile], __ATOMIC_ACQUIRE)) { - } - return 0; -} diff --git a/examples/chips/poly/workloads/ctests/ant_mxmm.c b/examples/chips/poly/workloads/ctests/ant_mxmm.c deleted file mode 100644 index 78bf63180..000000000 --- a/examples/chips/poly/workloads/ctests/ant_mxmm.c +++ /dev/null @@ -1,15 +0,0 @@ -#include "ant_mxmm.h" -#include -#include - -uint64_t ant_main(const struct ant_mxmm_args *args) { - for (unsigned bank = 3; bank <= 5; ++bank) - bb_mset(bank, 1, 1, 1); - bb_mvin(args->a, 3, ANT_M * ANT_K * 4 / (BANK_WIDTH / 8), 1); - bb_mvin(args->b, 4, ANT_N * ANT_K * 4 / (BANK_WIDTH / 8), 1); - bb_mxmm_f32(3, 4, 5, ANT_M, ANT_N, ANT_K, 1, 1, 0); - bb_mvout(args->output, 5, ANT_M * ANT_N * 4 / (BANK_WIDTH / 8), 1); - for (unsigned bank = 3; bank <= 5; ++bank) - bb_mset(bank, 0, 0, 0); - return 0; -} diff --git a/examples/chips/poly/workloads/ctests/ant_mxmm_main.cpp b/examples/chips/poly/workloads/ctests/ant_mxmm_main.cpp deleted file mode 100644 index 79fa6aa54..000000000 --- a/examples/chips/poly/workloads/ctests/ant_mxmm_main.cpp +++ /dev/null @@ -1,96 +0,0 @@ -#define _GNU_SOURCE -#include "ant_mxmm.h" -#include -#include -#include -#ifdef __linux__ -#include -#include -#include -#include -#include -#endif -extern const unsigned char ant_image[], ant_image_end[]; -struct matrices { - float a[BB_COMPUTE_TILES][ANT_M * ANT_K]; - float b[BB_COMPUTE_TILES][ANT_N * ANT_K]; - float out[BB_COMPUTE_TILES][ANT_M * ANT_N]; -}; -static constexpr matrices prepare() { - matrices data{}; - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) { - for (unsigned row = 0; row < ANT_M; ++row) - for (unsigned k = 0; k < ANT_K; ++k) - data.a[tile][row * ANT_K + k] = row + tile + 1; - for (unsigned col = 0; col < ANT_N; ++col) - for (unsigned k = 0; k < ANT_K; ++k) - data.b[tile][col * ANT_K + k] = col + 1; - for (unsigned i = 0; i < ANT_M * ANT_N; ++i) - data.out[tile][i] = -1; - } - return data; -} -alignas(64) static matrices data = prepare(); -static unsigned done[BB_COMPUTE_TILES]; -static void exercise(unsigned tile) { - size_t bytes = ant_image_end - ant_image; - if (bytes > ant_query(0, ANT_CODE_BYTES) || - ant_query(0, ANT_SIGNATURE) != CORE_SIGNATURE) - __builtin_trap(); - struct ant_mxmm_args args = {(uintptr_t)data.a[tile], (uintptr_t)data.b[tile], - (uintptr_t)data.out[tile]}; - ant_write(0, ANT_CODE, 0, ant_image, bytes); - ant_write(0, ANT_TLS, 0, &args, sizeof(args)); - uint64_t base = ant_query(0, ANT_TLS_BASE), - capacity = ant_query(0, ANT_TLS_BYTES); - struct ant_task task = {tile + 1, 0, bytes, base, base + capacity, - CORE_SIGNATURE}; - ant_acquire(); - ant_start(0, &task); - int cancelled; - if (ant_wait(0, task.id, &cancelled) || cancelled) - __builtin_trap(); - ant_release(); - for (unsigned row = 0; row < ANT_M; ++row) - for (unsigned col = 0; col < ANT_N; ++col) - if (data.out[tile][row * ANT_N + col] != - (float)(ANT_K * (row + tile + 1) * (col + 1))) - __builtin_trap(); - __atomic_store_n(&done[tile], 1, __ATOMIC_RELEASE); -} -#ifdef __linux__ -static void *worker(void *argument) { - uintptr_t tile = (uintptr_t)argument; - cpu_set_t mask; - CPU_ZERO(&mask); - CPU_SET(BB_MAIN_CORES + tile, &mask); - if (pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask)) - abort(); - exercise(tile); - return NULL; -} -#endif -int main(void) { -#ifdef __linux__ - if (mlockall(MCL_CURRENT | MCL_FUTURE)) - abort(); - pthread_t threads[BB_COMPUTE_TILES]; - for (uintptr_t tile = 0; tile < BB_COMPUTE_TILES; ++tile) - if (pthread_create(&threads[tile], NULL, worker, (void *)tile)) - abort(); - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - if (pthread_join(threads[tile], NULL)) - abort(); - printf("Ant Mxmm PASS tiles=%u\n", (unsigned)BB_COMPUTE_TILES); -#else - uintptr_t hart; - __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); - if (hart >= BB_MAIN_CORES) - exercise(hart - BB_MAIN_CORES); - else - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - while (!__atomic_load_n(&done[tile], __ATOMIC_ACQUIRE)) { - } -#endif - return 0; -} diff --git a/examples/chips/poly/workloads/ctests/ant_parallel_exercise.c b/examples/chips/poly/workloads/ctests/ant_parallel_exercise.c deleted file mode 100644 index 0b693c103..000000000 --- a/examples/chips/poly/workloads/ctests/ant_parallel_exercise.c +++ /dev/null @@ -1,68 +0,0 @@ -#include "ant_parallel.h" -#include -#include -#include -extern const unsigned char ant_image[], ant_image_end[]; -static uint64_t input[BB_COMPUTE_TILES][2][512] - __attribute__((aligned(4096), section(".noinit"))); -static uint64_t output[BB_COMPUTE_TILES][2][512] - __attribute__((aligned(4096), section(".noinit"))); -unsigned parallel_done[BB_COMPUTE_TILES]; -static unsigned reached[3] __attribute__((aligned(64))); -static void barrier(unsigned phase) { - __atomic_fetch_add(&reached[phase], 1, __ATOMIC_ACQ_REL); - while (__atomic_load_n(&reached[phase], __ATOMIC_ACQUIRE) != - BB_COMPUTE_TILES) { - } -} -void parallel_exercise(unsigned tile) { - size_t bytes = ant_image_end - ant_image; - uint64_t shared = ant_query(0, ANT_TSS_BASE), zero = 0; - struct ant_task tasks[2]; - for (unsigned context = 0; context < 2; ++context) { - for (unsigned word = 0; word < 2; ++word) { - input[tile][context][word] = - ((uint64_t)(tile + 1) << 32) | (context * 2 + word + 1); - output[tile][context][word] = ~0ULL; - } - uint64_t base = ant_query(context, ANT_TLS_BASE); - if (bytes > ant_query(context, ANT_CODE_BYTES) || - ant_query(context, ANT_SIGNATURE) != CORE_SIGNATURE) - __builtin_trap(); - struct ant_parallel_args args = {shared, context, - (uintptr_t)input[tile][context], - (uintptr_t)output[tile][context]}; - ant_write(context, ANT_CODE, 0, ant_image, bytes); - ant_write(context, ANT_TLS, 0, &args, sizeof(args)); - tasks[context] = (struct ant_task){ - context + 1, 0, bytes, base, base + ant_query(context, ANT_TLS_BYTES), - CORE_SIGNATURE}; - } - ant_write(0, ANT_TSS, 0, &zero, sizeof(zero)); - barrier(0); - unsigned peer = (tile + 1) % BB_COMPUTE_TILES; - for (unsigned context = 0; context < 2; ++context) - for (unsigned word = 0; word < 2; ++word) - if (*(volatile uint64_t *)&output[peer][context][word] != ~0ULL) - __builtin_trap(); - barrier(1); - ant_acquire(); - ant_start(0, &tasks[0]); - ant_start(1, &tasks[1]); - for (unsigned context = 0; context < 2; ++context) { - int cancelled; - if (ant_wait(context, tasks[context].id, &cancelled) || cancelled) - __builtin_trap(); - for (unsigned word = 0; word < 2; ++word) - if (output[tile][context][word] != input[tile][context][word]) - __builtin_trap(); - } - ant_release(); - barrier(2); - for (unsigned context = 0; context < 2; ++context) - for (unsigned word = 0; word < 2; ++word) - if (*(volatile uint64_t *)&output[peer][context][word] != - input[peer][context][word]) - __builtin_trap(); - __atomic_store_n(¶llel_done[tile], 1, __ATOMIC_RELEASE); -} diff --git a/examples/chips/poly/workloads/ctests/ant_parallel_main.c b/examples/chips/poly/workloads/ctests/ant_parallel_main.c deleted file mode 100644 index 7df2096b9..000000000 --- a/examples/chips/poly/workloads/ctests/ant_parallel_main.c +++ /dev/null @@ -1,49 +0,0 @@ -#define _GNU_SOURCE -#include "ant_parallel.h" -#include -#include -#include -#ifdef __linux__ -#include -#include -#include -#include -#include -#endif -#ifdef __linux__ -static void *worker(void *argument) { - uintptr_t tile = (uintptr_t)argument; - cpu_set_t mask; - CPU_ZERO(&mask); - CPU_SET(BB_MAIN_CORES + tile, &mask); - if (pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask)) - abort(); - parallel_exercise(tile); - return NULL; -} -#endif -int main(void) { -#ifdef __linux__ - if (mlockall(MCL_CURRENT | MCL_FUTURE)) - abort(); - pthread_t threads[BB_COMPUTE_TILES]; - for (uintptr_t tile = 0; tile < BB_COMPUTE_TILES; ++tile) - if (pthread_create(&threads[tile], NULL, worker, (void *)tile)) - abort(); - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - if (pthread_join(threads[tile], NULL)) - abort(); - printf("Ant concurrent DMA/cache sharing PASS tiles=%u contexts=2\n", - (unsigned)BB_COMPUTE_TILES); -#else - uintptr_t hart; - __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); - if (hart >= BB_MAIN_CORES) - parallel_exercise(hart - BB_MAIN_CORES); - else - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - while (!__atomic_load_n(¶llel_done[tile], __ATOMIC_ACQUIRE)) { - } -#endif - return 0; -} diff --git a/examples/chips/poly/workloads/ctests/ant_shared.h b/examples/chips/poly/workloads/ctests/ant_shared.h deleted file mode 100644 index 2b08f1465..000000000 --- a/examples/chips/poly/workloads/ctests/ant_shared.h +++ /dev/null @@ -1,6 +0,0 @@ -#pragma once -#include - -struct ant_shared_args { - uint64_t shared, producer, value; -}; diff --git a/examples/chips/poly/workloads/ctests/ant_shared_main.c b/examples/chips/poly/workloads/ctests/ant_shared_main.c deleted file mode 100644 index 8c72e74fe..000000000 --- a/examples/chips/poly/workloads/ctests/ant_shared_main.c +++ /dev/null @@ -1,95 +0,0 @@ -#define _GNU_SOURCE -#include "ant_shared.h" -#include -#include -#include -#ifdef __linux__ -#include -#include -#include -#include -#endif -extern const unsigned char ant_image[], ant_image_end[]; -static unsigned done[BB_COMPUTE_TILES]; -static void exercise(unsigned tile) { - size_t bytes = ant_image_end - ant_image; - uint64_t value = 100 + tile, shared = ant_query(0, ANT_TSS_BASE), - zero[2] = {0}; - struct ant_task tasks[2]; - for (unsigned context = 0; context < 2; ++context) { - uint64_t base = ant_query(context, ANT_TLS_BASE); - if (bytes > ant_query(context, ANT_CODE_BYTES) || - ant_query(context, ANT_SIGNATURE) != CORE_SIGNATURE) - __builtin_trap(); - tasks[context] = (struct ant_task){ - context + 1, 0, bytes, base, base + ant_query(context, ANT_TLS_BYTES), - CORE_SIGNATURE}; - struct ant_shared_args args = {shared, context == 0, context ? 0 : value}; - ant_write(context, ANT_CODE, 0, ant_image, bytes); - ant_write(context, ANT_TLS, 0, &args, sizeof(args)); - } - ant_write(0, ANT_TSS, 0, zero, sizeof(zero)); - ant_acquire(); - ant_start(1, &tasks[1]); - ant_start(0, &tasks[0]); - for (unsigned context = 0; context < 2; ++context) { - int cancelled; - if (ant_wait(context, tasks[context].id, &cancelled) != value || cancelled) - __builtin_trap(); - if (ant_read(context, ANT_TLS, 16) != value) - __builtin_trap(); - } - ant_release(); - ant_write(0, ANT_TSS, 0, zero, sizeof(zero)); - ant_acquire(); - ant_start(1, &tasks[1]); - ant_cancel(1); - int cancelled; - ant_wait(1, tasks[1].id, &cancelled); - if (!cancelled) - __builtin_trap(); - ant_release(); - uint64_t ready[2] = {value + 7, 1}; - ant_write(0, ANT_TSS, 0, ready, sizeof(ready)); - ++tasks[1].id; - ant_acquire(); - ant_start(1, &tasks[1]); - if (ant_wait(1, tasks[1].id, &cancelled) != value + 7 || cancelled) - __builtin_trap(); - ant_release(); - __atomic_store_n(&done[tile], 1, __ATOMIC_RELEASE); -} -#ifdef __linux__ -static void *worker(void *argument) { - uintptr_t tile = (uintptr_t)argument; - cpu_set_t mask; - CPU_ZERO(&mask); - CPU_SET(BB_MAIN_CORES + tile, &mask); - if (pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask)) - abort(); - exercise(tile); - return NULL; -} -#endif -int main(void) { -#ifdef __linux__ - pthread_t threads[BB_COMPUTE_TILES]; - for (uintptr_t tile = 0; tile < BB_COMPUTE_TILES; ++tile) - if (pthread_create(&threads[tile], NULL, worker, (void *)tile)) - abort(); - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - if (pthread_join(threads[tile], NULL)) - abort(); - printf("Ant shared/cancel PASS tiles=%u\n", (unsigned)BB_COMPUTE_TILES); -#else - uintptr_t hart; - __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); - if (hart >= BB_MAIN_CORES) - exercise(hart - BB_MAIN_CORES); - else - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - while (!__atomic_load_n(&done[tile], __ATOMIC_ACQUIRE)) { - } -#endif - return 0; -} diff --git a/examples/chips/poly/workloads/ctests/ant_stream_main.c b/examples/chips/poly/workloads/ctests/ant_stream_main.c deleted file mode 100644 index 2c122871b..000000000 --- a/examples/chips/poly/workloads/ctests/ant_stream_main.c +++ /dev/null @@ -1,57 +0,0 @@ -#define _GNU_SOURCE -#include -#include -#include -#include -#include -#include -#include -#include -#include -enum { M = 16, N = 16, K = 32 }; -static float a[M * K] __attribute__((aligned(4096))); -static float b[N * K] __attribute__((aligned(4096))); -static float output[M * N] __attribute__((aligned(4096))); -static void job(void *argument) { - (void)argument; - for (unsigned bank = 3; bank <= 5; ++bank) - bb_mem_alloc(bank, 1, 1); - bb_mvin((uintptr_t)a, 3, sizeof(a) / 16, 1); - // A DMA call has completed before the CPU is allowed to reuse its source. - for (unsigned i = 0; i < M * K; ++i) - a[i] = -1; - bb_mvin((uintptr_t)b, 4, sizeof(b) / 16, 1); - bb_mxmm_f32(3, 4, 5, M, N, K, 1, 1, 0); - bb_mvout((uintptr_t)output, 5, sizeof(output) / 16, 1); - for (unsigned row = 0; row < M; ++row) - for (unsigned col = 0; col < N; ++col) - if (output[row * N + col] != (float)(K * (row + 1) * (col + 1))) - abort(); - for (unsigned bank = 3; bank <= 5; ++bank) - bb_mem_release(bank); -} -int main(void) { - cpu_set_t affinity; - CPU_ZERO(&affinity); - CPU_SET(BB_MAIN_CORES, &affinity); - if (sched_setaffinity(0, sizeof(affinity), &affinity)) - abort(); - runtime_init(1024 * 1024); - void *workspace = aligned_alloc(64, 1024 * 1024); - if (!workspace) - abort(); - workspace_init(workspace, 1024 * 1024); - workspace_begin(workspace, 1024 * 1024); - for (unsigned row = 0; row < M; ++row) - for (unsigned k = 0; k < K; ++k) - a[row * K + k] = row + 1; - for (unsigned col = 0; col < N; ++col) - for (unsigned k = 0; k < K; ++k) - b[col * K + k] = col + 1; - for (unsigned i = 0; i < M * N; ++i) - output[i] = -1; - task_run(CORE_SIGNATURE, job, NULL); - puts("Ant stream DMA lifetime/Mxmm PASS"); - free(workspace); - return 0; -} diff --git a/examples/chips/poly/workloads/ctests/ant_tlink.c b/examples/chips/poly/workloads/ctests/ant_tlink.c deleted file mode 100644 index a019d6f95..000000000 --- a/examples/chips/poly/workloads/ctests/ant_tlink.c +++ /dev/null @@ -1,16 +0,0 @@ -#include "ant_tlink.h" -#include -uint64_t ant_main(const struct ant_tlink_args *args) { - if (args->phase == 0) { - bb_mset(BB_SHARED_BANK_BASE, 1, 1, 1); - bb_mset(BB_SHARED_BANK_BASE + 1, 1, 1, 1); - bb_mvin(args->input, BB_SHARED_BANK_BASE, TLINK_TEST_ROWS, 1); - } else if (args->phase == 1) { - bb_mvout(args->output, BB_SHARED_BANK_BASE + 1, TLINK_TEST_ROWS, 1); - } else if (args->phase == 2) { - bb_mset(BB_SHARED_BANK_BASE, 0, 0, 0); - bb_mset(BB_SHARED_BANK_BASE + 1, 0, 0, 0); - } else - __builtin_trap(); - return 0; -} diff --git a/examples/chips/poly/workloads/ctests/ant_tlink.h b/examples/chips/poly/workloads/ctests/ant_tlink.h deleted file mode 100644 index 8f01c1347..000000000 --- a/examples/chips/poly/workloads/ctests/ant_tlink.h +++ /dev/null @@ -1,6 +0,0 @@ -#pragma once -#include -enum { TLINK_TEST_BYTES = 4160, TLINK_TEST_ROWS = TLINK_TEST_BYTES / 16 }; -struct ant_tlink_args { - uint64_t phase, input, output; -}; diff --git a/examples/chips/poly/workloads/ctests/ant_tlink_bank.c b/examples/chips/poly/workloads/ctests/ant_tlink_bank.c deleted file mode 100644 index 9776e457d..000000000 --- a/examples/chips/poly/workloads/ctests/ant_tlink_bank.c +++ /dev/null @@ -1,16 +0,0 @@ -#include "ant_tlink_bank.h" -#include -uint64_t ant_main(const struct ant_tlink_bank_args *args) { - if (args->phase == 0) { - bb_mset(BB_SHARED_BANK_BASE, 1, 1, 1); - bb_mset(BB_SHARED_BANK_BASE + 1, 1, 1, 1); - bb_mvin(args->input, BB_SHARED_BANK_BASE, TLINK_CROSS_ROWS, 1); - } else if (args->phase == 1) { - bb_mvout(args->output, BB_SHARED_BANK_BASE, TLINK_CROSS_ROWS, 1); - } else if (args->phase == 2) { - bb_mset(BB_SHARED_BANK_BASE, 0, 0, 0); - bb_mset(BB_SHARED_BANK_BASE + 1, 0, 0, 0); - } else - __builtin_trap(); - return 0; -} diff --git a/examples/chips/poly/workloads/ctests/ant_tlink_bank.h b/examples/chips/poly/workloads/ctests/ant_tlink_bank.h deleted file mode 100644 index 13b587e46..000000000 --- a/examples/chips/poly/workloads/ctests/ant_tlink_bank.h +++ /dev/null @@ -1,6 +0,0 @@ -#pragma once -#include -enum { TLINK_CROSS_BYTES = 192, TLINK_CROSS_ROWS = TLINK_CROSS_BYTES / 16 }; -struct ant_tlink_bank_args { - uint64_t phase, input, output; -}; diff --git a/examples/chips/poly/workloads/ctests/ant_tlink_bank_main.c b/examples/chips/poly/workloads/ctests/ant_tlink_bank_main.c deleted file mode 100644 index 58ae1fb9e..000000000 --- a/examples/chips/poly/workloads/ctests/ant_tlink_bank_main.c +++ /dev/null @@ -1,92 +0,0 @@ -#include "ant_tlink_bank.h" -#include -#include -#include -#include -_Static_assert(BB_MAIN_CORES == 1 && BB_COMPUTE_TILES >= 2 && - !(BB_COMPUTE_TILES & (BB_COMPUTE_TILES - 1)), - "TLink gate requires a power-of-two compute tile count"); -extern const unsigned char ant_image[], ant_image_end[]; -static uint64_t input[BB_COMPUTE_TILES][TLINK_CROSS_BYTES / 8] - __attribute__((aligned(64))); -static uint64_t output[BB_COMPUTE_TILES][TLINK_CROSS_BYTES / 8] - __attribute__((aligned(64))); -static unsigned ready[BB_COMPUTE_TILES], done[BB_COMPUTE_TILES]; -static unsigned arrived[2]; -static void barrier(unsigned phase) { - __atomic_fetch_add(&arrived[phase], 1, __ATOMIC_ACQ_REL); - while (__atomic_load_n(&arrived[phase], __ATOMIC_ACQUIRE) != - BB_COMPUTE_TILES) { - } -} -static unsigned pattern(unsigned tile, unsigned i) { - return (i * 17 + tile * 23 + 3) & 255; -} -static void phase(unsigned tile, unsigned operation) { - const unsigned context = BB_ANT_CONTEXT; - size_t bytes = ant_image_end - ant_image; - struct ant_tlink_bank_args args = {operation, (uintptr_t)input[tile], - (uintptr_t)output[tile]}; - ant_write(context, ANT_TLS, 0, &args, sizeof(args)); - uint64_t base = ant_query(context, ANT_TLS_BASE); - struct ant_task task = {tile * 3 + operation + 1, - 0, - bytes, - base, - base + ant_query(context, ANT_TLS_BYTES), - CORE_SIGNATURE}; - ant_acquire(); - ant_start(context, &task); - int cancelled; - if (ant_wait(context, task.id, &cancelled) || cancelled) - __builtin_trap(); - ant_release(); -} -static void exercise(unsigned tile) { - uint64_t bank_bytes = tlink_query(TLINK_BANK_BYTES); - if (tlink_query(TLINK_TILE_ID) != tile + 1 || - tlink_query(TLINK_TILE_COUNT) != BB_MAIN_CORES + BB_COMPUTE_TILES || - bank_bytes < TLINK_CROSS_BYTES || - tlink_query(TLINK_SHARED_BYTES) < bank_bytes * 2) - __builtin_trap(); - size_t bytes = ant_image_end - ant_image; - if (bytes > ant_query(BB_ANT_CONTEXT, ANT_CODE_BYTES) || - ant_query(BB_ANT_CONTEXT, ANT_SIGNATURE) != CORE_SIGNATURE) - __builtin_trap(); - ant_write(BB_ANT_CONTEXT, ANT_CODE, 0, ant_image, bytes); - for (unsigned i = 0; i < TLINK_CROSS_BYTES / 8; ++i) { - uint64_t value = 0; - for (unsigned j = 0; j < 8; ++j) - value |= (uint64_t)pattern(tile, i * 8 + j) << (8 * j); - input[tile][i] = value; - } - phase(tile, 0); - barrier(0); - unsigned next = (tile + 1) % BB_COMPUTE_TILES; - unsigned previous = (tile + BB_COMPUTE_TILES - 1) % BB_COMPUTE_TILES; - tlink_transfer(0, next + 1, bank_bytes - 16, 37); - barrier(1); - tlink_transfer(bank_bytes - 16, previous + 1, 128, 37); - __atomic_store_n(&ready[tile], 1, __ATOMIC_RELEASE); - while (!__atomic_load_n(&ready[next], __ATOMIC_ACQUIRE)) { - } - phase(tile, 1); - for (unsigned i = 0; i < TLINK_CROSS_BYTES; ++i) { - unsigned expected = pattern(tile, i >= 128 && i < 165 ? i - 128 : i); - if (((unsigned char *)output[tile])[i] != expected) - __builtin_trap(); - } - phase(tile, 2); - __atomic_store_n(&done[tile], 1, __ATOMIC_RELEASE); -} -int main(void) { - uintptr_t hart; - __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); - if (hart) - exercise(hart - 1); - else - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - while (!__atomic_load_n(&done[tile], __ATOMIC_ACQUIRE)) { - } - return 0; -} diff --git a/examples/chips/poly/workloads/ctests/ant_tlink_main.c b/examples/chips/poly/workloads/ctests/ant_tlink_main.c deleted file mode 100644 index f09c51ee0..000000000 --- a/examples/chips/poly/workloads/ctests/ant_tlink_main.c +++ /dev/null @@ -1,88 +0,0 @@ -#include "ant_tlink.h" -#include -#include -#include -#include -_Static_assert(BB_MAIN_CORES == 1 && BB_COMPUTE_TILES >= 2 && - !(BB_COMPUTE_TILES & (BB_COMPUTE_TILES - 1)), - "TLink gate requires a power-of-two compute tile count"); -extern const unsigned char ant_image[], ant_image_end[]; -static uint64_t input[BB_COMPUTE_TILES][TLINK_TEST_BYTES / 8] - __attribute__((aligned(64))); -static uint64_t output[BB_COMPUTE_TILES][TLINK_TEST_BYTES / 8] - __attribute__((aligned(64))); -static unsigned done[BB_COMPUTE_TILES]; -static unsigned arrived[2]; -static void barrier(unsigned phase) { - __atomic_fetch_add(&arrived[phase], 1, __ATOMIC_ACQ_REL); - while (__atomic_load_n(&arrived[phase], __ATOMIC_ACQUIRE) != - BB_COMPUTE_TILES) { - } -} -static unsigned pattern(unsigned tile, unsigned i) { - return (i * 17 + tile * 23 + 3) & 255; -} -static void phase(unsigned tile, unsigned operation) { - const unsigned context = BB_ANT_CONTEXT; - size_t bytes = ant_image_end - ant_image; - struct ant_tlink_args args = {operation, (uintptr_t)input[tile], - (uintptr_t)output[tile]}; - ant_write(context, ANT_TLS, 0, &args, sizeof(args)); - uint64_t base = ant_query(context, ANT_TLS_BASE); - struct ant_task task = {tile * 3 + operation + 1, - 0, - bytes, - base, - base + ant_query(context, ANT_TLS_BYTES), - CORE_SIGNATURE}; - ant_acquire(); - ant_start(context, &task); - int cancelled; - if (ant_wait(context, task.id, &cancelled) || cancelled) - __builtin_trap(); - ant_release(); -} -static void exercise(unsigned tile) { - uint64_t bank_bytes = tlink_query(TLINK_BANK_BYTES); - if (tlink_query(TLINK_TILE_ID) != tile + 1 || - tlink_query(TLINK_TILE_COUNT) != BB_MAIN_CORES + BB_COMPUTE_TILES || - tlink_query(TLINK_BANK_BYTES) < TLINK_TEST_BYTES || - tlink_query(TLINK_SHARED_BYTES) < bank_bytes * 2) - __builtin_trap(); - size_t bytes = ant_image_end - ant_image; - if (bytes > ant_query(BB_ANT_CONTEXT, ANT_CODE_BYTES) || - ant_query(BB_ANT_CONTEXT, ANT_SIGNATURE) != CORE_SIGNATURE) - __builtin_trap(); - ant_write(BB_ANT_CONTEXT, ANT_CODE, 0, ant_image, bytes); - for (unsigned i = 0; i < TLINK_TEST_BYTES / 8; ++i) { - uint64_t value = 0; - for (unsigned j = 0; j < 8; ++j) - value |= (uint64_t)pattern(tile, i * 8 + j) << (8 * j); - input[tile][i] = value; - } - phase(tile, 0); - barrier(0); - unsigned next = (tile + 1) % BB_COMPUTE_TILES; - unsigned previous = (tile + BB_COMPUTE_TILES - 1) % BB_COMPUTE_TILES; - tlink_transfer(0, next + 1, bank_bytes + 16, 4101); - barrier(1); - phase(tile, 1); - for (unsigned i = 0; i < TLINK_TEST_BYTES; ++i) { - unsigned expected = i >= 16 && i < 4117 ? pattern(previous, i - 16) : 0; - if (((unsigned char *)output[tile])[i] != expected) - __builtin_trap(); - } - phase(tile, 2); - __atomic_store_n(&done[tile], 1, __ATOMIC_RELEASE); -} -int main(void) { - uintptr_t hart; - __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); - if (hart) - exercise(hart - 1); - else - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - while (!__atomic_load_n(&done[tile], __ATOMIC_ACQUIRE)) { - } - return 0; -} diff --git a/examples/chips/poly/workloads/ctests/ant_tlink_reduce.c b/examples/chips/poly/workloads/ctests/ant_tlink_reduce.c deleted file mode 100644 index 4528083d2..000000000 --- a/examples/chips/poly/workloads/ctests/ant_tlink_reduce.c +++ /dev/null @@ -1,21 +0,0 @@ -#include "ant_tlink_reduce.h" -#include -#include -uint64_t ant_main(const struct ant_tlink_reduce_args *args) { - if (args->phase == 0) { - bb_mset(BB_SHARED_BANK_BASE, 1, 1, 1); - bb_mset(0, 1, 1, 1); - bb_mset(1, 1, 1, 1); - bb_mvin(args->input, 0, 4, 1); - bb_mvin(args->input + 16 * sizeof(float), 1, 64, 1); - bb_mxmm_fma32(0, 1, BB_SHARED_BANK_BASE, 1, 16, 16, 1, 1, 0); - bb_mset(0, 0, 0, 0); - bb_mset(1, 0, 0, 0); - } else if (args->phase == 1) { - bb_mvout(args->output, BB_SHARED_BANK_BASE, 2, 1); - } else if (args->phase == 2) { - bb_mset(BB_SHARED_BANK_BASE, 0, 0, 0); - } else - __builtin_trap(); - return 0; -} diff --git a/examples/chips/poly/workloads/ctests/ant_tlink_reduce.h b/examples/chips/poly/workloads/ctests/ant_tlink_reduce.h deleted file mode 100644 index 33fd9f396..000000000 --- a/examples/chips/poly/workloads/ctests/ant_tlink_reduce.h +++ /dev/null @@ -1,8 +0,0 @@ -#pragma once -#include -struct ant_tlink_reduce_args { - uint64_t phase, input, output; -}; -void ant_tlink_reduce_phase(unsigned tile, - const struct ant_tlink_reduce_args *args); -void ant_tlink_reduce_prepare(unsigned tile, float *input, float *output); diff --git a/examples/chips/poly/workloads/ctests/ant_tlink_reduce_control.c b/examples/chips/poly/workloads/ctests/ant_tlink_reduce_control.c deleted file mode 100644 index 401a90079..000000000 --- a/examples/chips/poly/workloads/ctests/ant_tlink_reduce_control.c +++ /dev/null @@ -1,37 +0,0 @@ -#include "ant_tlink_reduce.h" -#include -#include -extern const unsigned char ant_image[], ant_image_end[]; -void ant_tlink_reduce_phase(unsigned tile, - const struct ant_tlink_reduce_args *args) { - unsigned context = BB_ANT_CONTEXT; - size_t bytes = ant_image_end - ant_image; - ant_write(context, ANT_TLS, 0, args, sizeof(*args)); - uint64_t base = ant_query(context, ANT_TLS_BASE); - struct ant_task task = {tile * 3 + args->phase + 1, - 0, - bytes, - base, - base + ant_query(context, ANT_TLS_BYTES), - CORE_SIGNATURE}; - ant_acquire(); - ant_start(context, &task); - int cancelled; - if (ant_wait(context, task.id, &cancelled) || cancelled) - __builtin_trap(); - ant_release(); -} -void ant_tlink_reduce_prepare(unsigned tile, float *input, float *output) { - size_t bytes = ant_image_end - ant_image; - if (bytes > ant_query(BB_ANT_CONTEXT, ANT_CODE_BYTES) || - ant_query(BB_ANT_CONTEXT, ANT_SIGNATURE) != CORE_SIGNATURE) - __builtin_trap(); - ant_write(BB_ANT_CONTEXT, ANT_CODE, 0, ant_image, bytes); - for (unsigned i = 0; i < 272; ++i) - input[i] = 0.0f; - input[0] = tile % 2 ? -(float)(tile + 1) * 0.5f : (float)(tile + 1) * 1.25f; - for (unsigned column = 0; column < 16; ++column) - input[16 + column * 16] = (float)(column + 1); - for (unsigned i = 0; i < 8; ++i) - output[i] = -77.0f; -} diff --git a/examples/chips/poly/workloads/ctests/ant_tlink_reduce_main.c b/examples/chips/poly/workloads/ctests/ant_tlink_reduce_main.c deleted file mode 100644 index 6acebe80d..000000000 --- a/examples/chips/poly/workloads/ctests/ant_tlink_reduce_main.c +++ /dev/null @@ -1,97 +0,0 @@ -#include "ant_tlink_reduce.h" -#include -#include -#include -#include -#include -#include -_Static_assert(BB_MAIN_CORES == 1 && BB_COMPUTE_TILES >= 2 && - (BB_COMPUTE_TILES & (BB_COMPUTE_TILES - 1)) == 0, - "Reduction needs main and a power-of-two compute tile count"); -static float input[BB_COMPUTE_TILES][272] __attribute__((aligned(64))); -static float output[BB_COMPUTE_TILES][16] __attribute__((aligned(64))); -static unsigned ready[BB_COMPUTE_TILES], broadcast, done[BB_COMPUTE_TILES]; -enum { WINDOW = 4 }; -static uint64_t shared_base[BB_COMPUTE_TILES], main_base[WINDOW]; -static unsigned main_ready, batch; -static float partial(unsigned tile, unsigned i) { - return (float)(i + 1) * - (tile % 2 ? -(float)(tile + 1) * 0.5f : (float)(tile + 1) * 1.25f); -} -static void controller(unsigned tile) { - ant_tlink_reduce_prepare(tile, input[tile], output[tile]); - struct ant_tlink_reduce_args args = {0, (uintptr_t)input[tile], - (uintptr_t)output[tile]}; - ant_tlink_reduce_phase(tile, &args); - shared_base[tile] = tlink_shared_export(0, BB_SHARED_BANK_BASE, 0); - while (!__atomic_load_n(&main_ready, __ATOMIC_ACQUIRE) || - __atomic_load_n(&batch, __ATOMIC_ACQUIRE) != tile / WINDOW) { - } - tlink_transfer(shared_base[tile], 0, main_base[tile % WINDOW], 16); - __atomic_store_n(&ready[tile], 1, __ATOMIC_RELEASE); - while (!__atomic_load_n(&broadcast, __ATOMIC_ACQUIRE)) { - } - args.phase = 1; - ant_tlink_reduce_phase(tile, &args); - for (unsigned i = 0; i < 4; ++i) { - float expected = 0.0f; - for (unsigned rank = 0; rank < BB_COMPUTE_TILES; ++rank) - expected += partial(rank, i); - if (output[tile][i] != partial(tile, i) || output[tile][i + 4] != expected) - __builtin_trap(); - } - tlink_shared_release(0, BB_SHARED_BANK_BASE, 0); - args.phase = 2; - ant_tlink_reduce_phase(tile, &args); - __atomic_store_n(&done[tile], 1, __ATOMIC_RELEASE); -} -static void reduce(void) { - unsigned groups = BB_COMPUTE_TILES < WINDOW ? BB_COMPUTE_TILES : WINDOW; - unsigned source = 32, accumulator = 33, target = 34; - bb_mem_alloc(source, 1, groups); - bb_mem_alloc(accumulator, 1, 1); - bb_mem_alloc(target, 1, 1); - for (unsigned group = 0; group < groups; ++group) - main_base[group] = tlink_shared_export(0, source, group); - __atomic_store_n(&main_ready, 1, __ATOMIC_RELEASE); - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) { - while (!__atomic_load_n(&ready[tile], __ATOMIC_ACQUIRE)) { - } - bb_f32add(source, accumulator, target, 1, tile % WINDOW, tile == 0); - unsigned previous = accumulator; - accumulator = target; - target = previous; - if (tile % WINDOW == WINDOW - 1) { - __atomic_store_n(&batch, tile / WINDOW + 1, __ATOMIC_RELEASE); - } - } - uint64_t result = tlink_shared_export(0, accumulator, 0); - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - tlink_transfer(result, tile + 1, shared_base[tile] + 16, 16); - __atomic_store_n(&broadcast, 1, __ATOMIC_RELEASE); - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - while (!__atomic_load_n(&done[tile], __ATOMIC_ACQUIRE)) { - } - tlink_shared_release(0, accumulator, 0); - for (unsigned group = 0; group < groups; ++group) - tlink_shared_release(0, source, group); - bb_mem_release(source); - bb_mem_release(accumulator); - bb_mem_release(target); -} - -int main(void) { - uintptr_t hart; - __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); - if (hart > BB_COMPUTE_TILES || tlink_query(TLINK_TILE_ID) != hart || - tlink_query(TLINK_TILE_COUNT) != BB_COMPUTE_TILES + 1 || - tlink_query(TLINK_BANK_BYTES) < 48 || - tlink_query(TLINK_SHARED_BYTES) < - tlink_query(TLINK_BANK_BYTES) + BB_COMPUTE_TILES * 16) - __builtin_trap(); - if (hart) - controller(hart - 1); - else - reduce(); - return 0; -} diff --git a/examples/chips/poly/workloads/ctests/controllers_start.S b/examples/chips/poly/workloads/ctests/controllers_start.S deleted file mode 100644 index dfa26d2d3..000000000 --- a/examples/chips/poly/workloads/ctests/controllers_start.S +++ /dev/null @@ -1,15 +0,0 @@ -#include -/* Only Linux control cores participate; data-plane workers remain parked. */ -.section .text.init,"ax",@progbits -.globl _start -_start: - csrr t0, mhartid - li t1, BB_MAIN_CORES + BB_COMPUTE_TILES - bltu t0, t1, controllers_start -1: - wfi - j 1b -#undef BB_REPORT_HART -#define BB_REPORT_HART 0 -#define _start controllers_start -#include "start.S" diff --git a/examples/chips/poly/workloads/ctests/mesh_shared_mem_test.c b/examples/chips/poly/workloads/ctests/mesh_shared_mem_test.c deleted file mode 100644 index 04cd3f76a..000000000 --- a/examples/chips/poly/workloads/ctests/mesh_shared_mem_test.c +++ /dev/null @@ -1,50 +0,0 @@ -#include -#include -#include - -#include - -#if BANK_WIDTH != 128 -#error "mesh_shared_mem_test requires 128-bit Bank rows" -#endif -#if BB_SHARED_PHYSICAL_BANK_NUM < 1 -#error "mesh_shared_mem_test requires at least one shared Bank" -#endif - -#if BB_VIRTUAL_BANK_NUM <= BB_SHARED_BANK_BASE -#error "mesh_shared_mem_test requires shared virtual Bank IDs" -#endif -#if BB_VIRTUAL_BANK_NUM - BB_SHARED_BANK_BASE > BB_SHARED_PHYSICAL_BANK_NUM -#error "mesh_shared_mem_test requires a physical Bank per shared virtual Bank" -#endif - -enum { - SHARED_TEST_BANKS = (BB_VIRTUAL_BANK_NUM - BB_SHARED_BANK_BASE), - ROW_BYTES = (BANK_WIDTH / 8) -}; - -static uint8_t source[SHARED_TEST_BANKS][ROW_BYTES] - __attribute__((aligned(64))); -static uint8_t result[SHARED_TEST_BANKS][ROW_BYTES] - __attribute__((aligned(64))); - -/* crt0 boots only the target's test hart. */ -int main(void) { - for (int bank = 0; bank < SHARED_TEST_BANKS; ++bank) { - bb_mem_alloc(BB_SHARED_BANK_BASE + bank, 1, 1); - for (int byte = 0; byte < ROW_BYTES; ++byte) - source[bank][byte] = (uint8_t)(bank * 17 + byte); - bb_mvin((uintptr_t)source[bank], BB_SHARED_BANK_BASE + bank, 1, 1); - } - - for (int bank = 0; bank < SHARED_TEST_BANKS; ++bank) - bb_mvout((uintptr_t)result[bank], BB_SHARED_BANK_BASE + bank, 1, 1); - - for (int bank = 0; bank < SHARED_TEST_BANKS; ++bank) { - for (int byte = 0; byte < ROW_BYTES; ++byte) - if (result[bank][byte] != source[bank][byte]) - return 1; - bb_mem_release(BB_SHARED_BANK_BASE + bank); - } - return 0; -} diff --git a/examples/chips/toy/arch/src/main/scala/sims/verilator/TargetConfigs.scala b/examples/chips/toy/arch/src/main/scala/sims/verilator/TargetConfigs.scala deleted file mode 100644 index f4f709879..000000000 --- a/examples/chips/toy/arch/src/main/scala/sims/verilator/TargetConfigs.scala +++ /dev/null @@ -1,11 +0,0 @@ -package sims.verilator - -import chisel3.experimental.hierarchy.{Instance, Instantiate} -import framework.system.{System, SystemParams} - -import sims.soc.SystemTarget - -class ToyVerilatorTarget extends SystemTarget("../examples/chips/toy/configs/generated/chip.pb") { - override def instantiate(p: SystemParams): Instance[System] = - Instantiate(new framework.system.Device(p.toGlobalConfig)) -} diff --git a/examples/chips/toy/configs/chip.toml b/examples/chips/toy/configs/chip.toml deleted file mode 100644 index 078e3f13d..000000000 --- a/examples/chips/toy/configs/chip.toml +++ /dev/null @@ -1,9 +0,0 @@ -[designs] -include = "designs/toy.toml" - -[sims] -verilator = "sims.verilator.ToyVerilatorTarget" - -[uvm] -balls = ["gemmini"] -ips = ["axis", "memdomain"] diff --git a/examples/chips/toy/configs/designs/tiles/default.toml b/examples/chips/toy/configs/designs/tiles/default.toml deleted file mode 100644 index d6b3c3dd6..000000000 --- a/examples/chips/toy/configs/designs/tiles/default.toml +++ /dev/null @@ -1,23 +0,0 @@ -# Toy Tile Configuration (1 core) - -coreDataBytes = 64 -xLen = 64 -vaddrBits = 39 -paddrBits = 56 -pgIdxBits = 12 -pgLevels = 3 -nPMPs = 8 - -memBallChannelNum = 6 - -[sharedMem] -bankWidth = 128 -bankEntries = 1024 -enable = false -entries = 16384 -inputChannels = 7 -defaultGroupCount = 1 - -[[cores]] -core_id = 0 -include = "../../../../../cores/toy/configs/default.toml" diff --git a/examples/chips/toy/configs/designs/toy.toml b/examples/chips/toy/configs/designs/toy.toml deleted file mode 100644 index e8636c8ee..000000000 --- a/examples/chips/toy/configs/designs/toy.toml +++ /dev/null @@ -1,4 +0,0 @@ -# Toy: a main tile only. - -[main_tile] -include = "tiles/default.toml" diff --git a/examples/chips/toy/kernel/workloads.toml b/examples/chips/toy/kernel/workloads.toml deleted file mode 100644 index f93f96786..000000000 --- a/examples/chips/toy/kernel/workloads.toml +++ /dev/null @@ -1,44 +0,0 @@ -[workloads] -tests = [ - "toy-toy-mlirtest-base_dma_group-linux", - "toy-toy-mlirtest-base_mset_transfer-linux", - "toy-toy-ctest-mvin_mvout_group_test-linux", - "toy-toy-ctest-mset_transfer_test-linux", - "toy-toy-ctest-rvv_ball-linux", - "toy-toy-ctest-rvv_kernel-linux", - "toy-toy-ctest-gemmini_accumulated_16x16_test-linux", - "toy-toy-mlirtest-ball_gemmini_accumulated_command_16x16-linux", - "toy-toy-ctest-alexnet_gemmini_round_test-linux", - "toy-toy-ctest-alexnet_gemmini_tail_test-linux", - "toy-toy-ctest-alexnet_gemmini_stride_test-linux", - "toy-toy-ctest-mvin_2d_test-linux", - "toy-toy-ctest-mvin_mvout_acc_test-linux", - "toy-toy-ctest-mvin_mvout_test-linux", - "toy-toy-ctest-mvin_mvout_stride_test-linux", - "toy-toy-ctest-mvin_mvout_unaligned_test-linux", - "toy-toy-ctest-mvin_unaligned_test-linux", - "toy-toy-ctest-bdb_counter_start_stop_test-linux", - "toy-toy-ctest-bdb_counter_read_active_test-linux", - "toy-toy-ctest-bdb_counter_nested_pair_test-linux", - "toy-toy-ctest-bdb_counter_multi_id_test-linux", - "toy-toy-mlirtest-ball_bdb_counter_start_read_stop-linux", - "toy-toy-ctest-gemmini_os_cisc_basic_test-linux", - "toy-toy-ctest-gemmini_os_cisc_atranspose_test-linux", - "toy-toy-ctest-gemmini_os_cisc_btranspose_test-linux", - "toy-toy-ctest-gemmini_os_cisc_shift_test-linux", - "toy-toy-ctest-gemmini_os_risc_atranspose_test-linux", - "toy-toy-ctest-gemmini_os_risc_btranspose_test-linux", - "toy-toy-ctest-gemmini_os_risc_abtranspose_test-linux", - "toy-toy-ctest-gemmini_os_risc_shift_test-linux", - "toy-toy-ctest-gemmini_ws_cisc_conv_test-linux", - "toy-toy-ctest-gemmini_ws_risc_basic_test-linux", - "toy-toy-ctest-mmio_minimal_test-linux", - "toy-toy-ctest-mxfp4_e8m0_two_block_test-linux", - "toy-toy-ctest-relu_i32_16x16_test-linux", - "toy-toy-mlirtest-ball_relu_i32_16row-linux", - "toy-toy-mlirtest-bank_relu_i32_mixed_sign-linux", - "toy-toy-mlirtest-bank_mxfp4_unit_scale_bank-linux", - "toy-toy-mlirtest-ball_mxfp4_unit_scale_2blocks-linux", - "toy-toy-mlirtest-bank_gemmini_compute_preloaded_bank_tiles-linux", - "toy-toy-mlirtest-bank_gemmini_compute_accumulated_bank_tiles-linux", -] diff --git a/examples/chips/toy/regression/batch/bemu/workloads-bare.toml b/examples/chips/toy/regression/batch/bemu/workloads-bare.toml deleted file mode 100644 index b2880a260..000000000 --- a/examples/chips/toy/regression/batch/bemu/workloads-bare.toml +++ /dev/null @@ -1,53 +0,0 @@ -[workloads] -tests = [ - "toy-toy-mlirtest-base_dma_group-baremetal", - "toy-toy-mlirtest-base_mset_transfer-baremetal", - "toy-toy-ctest-mvin_mvout_group_test-baremetal", - "toy-toy-ctest-mset_transfer_test-baremetal", - "toy-toy-ctest-rvv_ball-baremetal", - "toy-toy-ctest-rvv_kernel-baremetal", - "toy-toy-ctest-mvin_2d_pixel_stride_test-baremetal", - "toy-toy-ctest-gemmini_accumulated_16x16_test-baremetal", - "toy-toy-mlirtest-ball_gemmini_accumulated_command_16x16-baremetal", - "toy-toy-ctest-alexnet_gemmini_round_test-baremetal", - "toy-toy-ctest-alexnet_gemmini_tail_test-baremetal", - "toy-toy-ctest-alexnet_gemmini_stride_test-baremetal", - "toy-toy-ctest-mvin_2d_test-baremetal", - "toy-toy-ctest-mvin_mvout_acc_test-baremetal", - "toy-toy-ctest-mvin_mvout_test-baremetal", - "toy-toy-ctest-mvin_mvout_stride_test-baremetal", - "toy-toy-ctest-mvin_mvout_unaligned_test-baremetal", - "toy-toy-ctest-mvin_unaligned_test-baremetal", - "toy-toy-ctest-bdb_counter_start_stop_test-baremetal", - "toy-toy-ctest-bdb_counter_read_active_test-baremetal", - "toy-toy-ctest-bdb_counter_nested_pair_test-baremetal", - "toy-toy-ctest-bdb_counter_multi_id_test-baremetal", - "toy-toy-mlirtest-ball_bdb_counter_start_read_stop-baremetal", - "toy-toy-ctest-gemmini_os_cisc_basic_test-baremetal", - "toy-toy-ctest-gemmini_os_cisc_atranspose_test-baremetal", - "toy-toy-ctest-gemmini_os_cisc_btranspose_test-baremetal", - "toy-toy-ctest-gemmini_os_cisc_shift_test-baremetal", - "toy-toy-ctest-gemmini_os_risc_atranspose_test-baremetal", - "toy-toy-ctest-gemmini_os_risc_btranspose_test-baremetal", - "toy-toy-ctest-gemmini_os_risc_abtranspose_test-baremetal", - "toy-toy-ctest-gemmini_os_risc_shift_test-baremetal", - "toy-toy-ctest-gemmini_ws_cisc_conv_test-baremetal", - "toy-toy-ctest-gemmini_ws_risc_basic_test-baremetal", - "toy-toy-ctest-mmio_minimal_test-baremetal", - "toy-toy-ctest-mxfp4_e8m0_two_block_test-baremetal", - "toy-toy-ctest-relu_i32_16x16_test-baremetal", - "toy-toy-mlirtest-ball_relu_i32_16row-baremetal", - "toy-toy-mlirtest-bank_relu_i32_mixed_sign-baremetal", - "toy-toy-mlirtest-bank_mxfp4_unit_scale_bank-baremetal", - "toy-toy-mlirtest-ball_mxfp4_unit_scale_2blocks-baremetal", - "toy-toy-mlirtest-bank_gemmini_compute_preloaded_bank_tiles-baremetal", - "toy-toy-mlirtest-bank_gemmini_compute_accumulated_bank_tiles-baremetal", - "toy-toy-mlirtest-ball_gemmini_preloaded_command_16x16-baremetal", - "toy-toy-ctest-tlb_bare_mode_test-baremetal", - "toy-toy-ctest-tlb_sv39_identity_test-baremetal", - "toy-toy-ctest-tlb_sfence_flush_test-baremetal", - "toy-toy-ctest-tlb_repeated_sfence_test-baremetal", - "toy-toy-ctest-tlb_sv39_nonidentity_test-baremetal", - "toy-toy-ctest-tlb_sv39_4k_alias_test-baremetal", - "toy-toy-ctest-tlb_sfence_remap_test-baremetal", -] diff --git a/examples/chips/toy/regression/batch/bemu/workloads-linux.toml b/examples/chips/toy/regression/batch/bemu/workloads-linux.toml deleted file mode 100644 index 6bb6a056a..000000000 --- a/examples/chips/toy/regression/batch/bemu/workloads-linux.toml +++ /dev/null @@ -1,4 +0,0 @@ -[workloads] -tests = [ - "fw_payload-h1-toy-linux.elf", -] diff --git a/examples/chips/toy/regression/batch/p2e/workloads-bare.toml b/examples/chips/toy/regression/batch/p2e/workloads-bare.toml deleted file mode 100644 index 59d07b1d1..000000000 --- a/examples/chips/toy/regression/batch/p2e/workloads-bare.toml +++ /dev/null @@ -1,26 +0,0 @@ -[workloads] -tests = [ - "toy-toy-ctest-gemmini_accumulated_16x16_test-baremetal.hex", - "toy-toy-mlirtest-ball_gemmini_accumulated_command_16x16-baremetal.hex", - "toy-toy-ctest-alexnet_gemmini_tail_test-baremetal.hex", - "toy-toy-ctest-alexnet_gemmini_stride_test-baremetal.hex", - "toy-toy-ctest-alexnet_gemmini_round_test-baremetal.hex", - "toy-toy-ctest-mvin_mvout_test-baremetal.hex", - "toy-toy-ctest-mvin_mvout_stride_test-baremetal.hex", - "toy-toy-ctest-bdb_counter_start_stop_test-baremetal.hex", - "toy-toy-ctest-bdb_counter_read_active_test-baremetal.hex", - "toy-toy-ctest-bdb_counter_nested_pair_test-baremetal.hex", - "toy-toy-ctest-bdb_counter_multi_id_test-baremetal.hex", - "toy-toy-ctest-gemmini_os_cisc_basic_test-baremetal.hex", - "toy-toy-ctest-gemmini_os_cisc_atranspose_test-baremetal.hex", - "toy-toy-ctest-gemmini_os_cisc_btranspose_test-baremetal.hex", - "toy-toy-ctest-gemmini_os_cisc_shift_test-baremetal.hex", - "toy-toy-ctest-gemmini_os_risc_atranspose_test-baremetal.hex", - "toy-toy-ctest-gemmini_os_risc_btranspose_test-baremetal.hex", - "toy-toy-ctest-gemmini_os_risc_abtranspose_test-baremetal.hex", - "toy-toy-ctest-gemmini_os_risc_shift_test-baremetal.hex", - "toy-toy-ctest-gemmini_ws_cisc_conv_test-baremetal.hex", - "toy-toy-ctest-gemmini_ws_risc_basic_test-baremetal.hex", - "toy-toy-ctest-mmio_minimal_test-baremetal.hex", - "toy-toy-ctest-mxfp4_e8m0_two_block_test-baremetal.hex", -] diff --git a/examples/chips/toy/regression/batch/p2e/workloads-linux.toml b/examples/chips/toy/regression/batch/p2e/workloads-linux.toml deleted file mode 100644 index 1a0a1bbd3..000000000 --- a/examples/chips/toy/regression/batch/p2e/workloads-linux.toml +++ /dev/null @@ -1,4 +0,0 @@ -[workloads] -tests = [ - "fw_payload-h1-toy-linux.hex", -] diff --git a/examples/chips/toy/regression/batch/verilator/workloads-bare.toml b/examples/chips/toy/regression/batch/verilator/workloads-bare.toml deleted file mode 100644 index d302c4692..000000000 --- a/examples/chips/toy/regression/batch/verilator/workloads-bare.toml +++ /dev/null @@ -1,39 +0,0 @@ -[workloads] -tests = [ - "toy-toy-ctest-rvv_kernel-baremetal", - "toy-toy-ctest-mvin_2d_pixel_stride_test-baremetal", - "toy-toy-ctest-gemmini_accumulated_16x16_test-baremetal", - "toy-toy-mlirtest-ball_gemmini_accumulated_command_16x16-baremetal", - "toy-toy-ctest-alexnet_gemmini_round_test-baremetal", - "toy-toy-ctest-alexnet_gemmini_tail_test-baremetal", - "toy-toy-ctest-alexnet_gemmini_stride_test-baremetal", - "toy-toy-ctest-mvin_2d_test-baremetal", - "toy-toy-ctest-mvin_mvout_acc_test-baremetal", - "toy-toy-ctest-mvin_mvout_test-baremetal", - "toy-toy-ctest-mvin_mvout_unaligned_test-baremetal", - "toy-toy-ctest-mvin_unaligned_test-baremetal", - "toy-toy-ctest-bdb_counter_start_stop_test-baremetal", - "toy-toy-ctest-bdb_counter_read_active_test-baremetal", - "toy-toy-ctest-bdb_counter_nested_pair_test-baremetal", - "toy-toy-ctest-bdb_counter_multi_id_test-baremetal", - "toy-toy-mlirtest-ball_bdb_counter_start_read_stop-baremetal", - "toy-toy-ctest-gemmini_os_cisc_basic_test-baremetal", - "toy-toy-ctest-gemmini_os_cisc_atranspose_test-baremetal", - "toy-toy-ctest-gemmini_os_cisc_btranspose_test-baremetal", - "toy-toy-ctest-gemmini_os_cisc_shift_test-baremetal", - "toy-toy-ctest-gemmini_ws_cisc_conv_test-baremetal", - "toy-toy-ctest-mmio_minimal_test-baremetal", - "toy-toy-ctest-mxfp4_e8m0_two_block_test-baremetal", - "toy-toy-ctest-relu_i32_16x16_test-baremetal", - "toy-toy-mlirtest-ball_relu_i32_16row-baremetal", - "toy-toy-mlirtest-ball_mxfp4_unit_scale_2blocks-baremetal", - "toy-toy-mlirtest-ball_gemmini_preloaded_command_16x16-baremetal", - "toy-toy-ctest-tlb_bare_mode_test-baremetal", - "toy-toy-ctest-tlb_sv39_identity_test-baremetal", - "toy-toy-ctest-tlb_sfence_flush_test-baremetal", - "toy-toy-ctest-tlb_repeated_sfence_test-baremetal", - "toy-toy-ctest-tlb_sv39_nonidentity_test-baremetal", - "toy-toy-ctest-tlb_sv39_4k_alias_test-baremetal", - "toy-toy-ctest-tlb_sfence_remap_test-baremetal", - "toy-toy-ctest-mvin_mvout_stride_test-baremetal", -] diff --git a/examples/chips/toy/regression/batch/verilator/workloads-linux.toml b/examples/chips/toy/regression/batch/verilator/workloads-linux.toml deleted file mode 100644 index 6ae9a89f8..000000000 --- a/examples/chips/toy/regression/batch/verilator/workloads-linux.toml +++ /dev/null @@ -1,35 +0,0 @@ -[workloads] -tests = [ - "toy-toy-ctest-gemmini_accumulated_16x16_test-linux", - "toy-toy-mlirtest-ball_gemmini_accumulated_command_16x16-linux", - "toy-toy-ctest-alexnet_gemmini_round_test-linux", - "toy-toy-ctest-alexnet_gemmini_tail_test-linux", - "toy-toy-ctest-alexnet_gemmini_stride_test-linux", - "toy-toy-ctest-mvin_2d_test-linux", - "toy-toy-ctest-mvin_mvout_acc_test-linux", - "toy-toy-ctest-mvin_mvout_test-linux", - "toy-toy-ctest-mvin_mvout_stride_test-linux", - "toy-toy-ctest-mvin_mvout_unaligned_test-linux", - "toy-toy-ctest-mvin_unaligned_test-linux", - "toy-toy-ctest-bdb_counter_start_stop_test-linux", - "toy-toy-ctest-bdb_counter_read_active_test-linux", - "toy-toy-ctest-bdb_counter_nested_pair_test-linux", - "toy-toy-ctest-bdb_counter_multi_id_test-linux", - "toy-toy-mlirtest-ball_bdb_counter_start_read_stop-linux", - "toy-toy-ctest-gemmini_os_cisc_basic_test-linux", - "toy-toy-ctest-gemmini_os_cisc_atranspose_test-linux", - "toy-toy-ctest-gemmini_os_cisc_btranspose_test-linux", - "toy-toy-ctest-gemmini_os_cisc_shift_test-linux", - "toy-toy-ctest-gemmini_os_risc_atranspose_test-linux", - "toy-toy-ctest-gemmini_os_risc_btranspose_test-linux", - "toy-toy-ctest-gemmini_os_risc_abtranspose_test-linux", - "toy-toy-ctest-gemmini_os_risc_shift_test-linux", - "toy-toy-ctest-gemmini_ws_cisc_conv_test-linux", - "toy-toy-ctest-gemmini_ws_risc_basic_test-linux", - "toy-toy-ctest-mmio_minimal_test-linux", - "toy-toy-ctest-mxfp4_e8m0_two_block_test-linux", - "toy-toy-ctest-relu_i32_16x16_test-linux", - "toy-toy-mlirtest-ball_relu_i32_16row-linux", - "toy-toy-mlirtest-ball_mxfp4_unit_scale_2blocks-linux", - "toy-toy-mlirtest-ball_gemmini_preloaded_command_16x16-linux", -] diff --git a/examples/chips/toy/regression/soc/README.md b/examples/chips/toy/regression/soc/README.md deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/chips/toy/workloads/CMakeLists.txt b/examples/chips/toy/workloads/CMakeLists.txt deleted file mode 100644 index 644aa54ae..000000000 --- a/examples/chips/toy/workloads/CMakeLists.txt +++ /dev/null @@ -1,18 +0,0 @@ -get_filename_component(BUCKYBALL_REPO_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../../.. ABSOLUTE) - -add_custom_target(toy-ctests-build) -set(BUCKYBALL_CTEST_GROUP_TARGET toy-ctests-build) -set(BUCKYBALL_CTEST_TARGET toy) - -add_subdirectory(ctests) -buckyball_add_ball_workload_subdirs(ctests) - -add_custom_target(chip-ctests-build DEPENDS toy-ctests-build) - -add_custom_target(chip-workloads-build ALL - DEPENDS - chip-ctests-build - mlirtest-workloads-build - soctests-build - COMMENT "Building toy workloads" - VERBATIM) diff --git a/examples/chips/toy/workloads/ctests/CMakeLists.txt b/examples/chips/toy/workloads/ctests/CMakeLists.txt deleted file mode 100644 index 8f2e8db5c..000000000 --- a/examples/chips/toy/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -add_buckyball_ctests( - mmio_minimal_test.c - scu_uart_rx_echo_test.c - alexnet_gemmini_tail_test.c - alexnet_gemmini_stride_test.c - alexnet_gemmini_round_test.c -) - -foreach(_id alexnet_gemmini_tail_test alexnet_gemmini_stride_test alexnet_gemmini_round_test) - buckyball_ctest_name(_test ${BUCKYBALL_CTEST_TARGET} ${_id}) - add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${_test}-baremetal APPEND - DEPENDS ${BUCKYBALL_REPO_ROOT}/examples/models/toy/alexnet/execute/conv.c - ${BUCKYBALL_REPO_ROOT}/examples/models/toy/alexnet/execute/conv.h) -endforeach() diff --git a/examples/chips/toy/workloads/ctests/alexnet_gemmini_round_test.c b/examples/chips/toy/workloads/ctests/alexnet_gemmini_round_test.c deleted file mode 100644 index 449dfc85d..000000000 --- a/examples/chips/toy/workloads/ctests/alexnet_gemmini_round_test.c +++ /dev/null @@ -1,22 +0,0 @@ -#include "../../../../models/toy/alexnet/execute/conv.c" -#include "buckyball.h" - -static float input[6] __attribute__((aligned(64))) = {0.125f, 0.375f, -0.125f, - -0.375f, 64.f, -64.f}; -static int8_t weights[6] __attribute__((aligned(64))) = {1, 2, 3, 4, 5, 6}; -static float bias[1], output[1]; - -int main(void) { - AlexnetF32_4 in = {input, input, 0, {1, 6, 1, 1}, {6, 1, 1, 1}}; - AlexnetI8_2 w = {weights, weights, 0, {6, 1}, {1, 1}}; - AlexnetF32_1 b = {bias, bias, 0, {1}, {1}}; - AlexnetF32_4 out = {output, output, 0, {1, 1, 1, 1}, {1, 1, 1, 1}}; - _mlir_ciface_gemmini_conv(&in, &w, &b, &out, 1, 1, 1, 0, 0.25f, 0.5f); - // RNE codes are {0, 2, 0, -2, 127, -128}; accumulation is -137. - if (output[0] != -17.125f) { - printf("AlexNet Gemmini rounding FAILED: %f\n", output[0]); - return 1; - } - printf("AlexNet Gemmini rounding PASSED\n"); - return 0; -} diff --git a/examples/chips/toy/workloads/ctests/alexnet_gemmini_stride_test.c b/examples/chips/toy/workloads/ctests/alexnet_gemmini_stride_test.c deleted file mode 100644 index 57d83b07f..000000000 --- a/examples/chips/toy/workloads/ctests/alexnet_gemmini_stride_test.c +++ /dev/null @@ -1,37 +0,0 @@ -#include "../../../../models/toy/alexnet/execute/conv.c" -#include "buckyball.h" - -static float input[4 * 4 * 4] __attribute__((aligned(64))); -static int8_t weights[16 * 32] __attribute__((aligned(64))); -static float bias[32], output[32 * 3 * 3]; - -int main(void) { - for (int i = 0; i < 64; ++i) - input[i] = (float)(i % 9 - 4) / 4; - for (int i = 0; i < 512; ++i) - weights[i] = i % 5 - 2; - for (int i = 0; i < 32; ++i) - bias[i] = (float)i / 2; - AlexnetF32_4 in = {input, input, 0, {1, 4, 4, 4}, {64, 16, 4, 1}}; - AlexnetI8_2 w = {weights, weights, 0, {16, 32}, {32, 1}}; - AlexnetF32_1 b = {bias, bias, 0, {32}, {1}}; - AlexnetF32_4 out = {output, output, 0, {1, 32, 3, 3}, {288, 9, 3, 1}}; - _mlir_ciface_gemmini_conv(&in, &w, &b, &out, 2, 2, 1, 0, 0.25f, 0.125f); - for (int n = 0; n < 32; ++n) - for (int y = 0; y < 3; ++y) - for (int x = 0; x < 3; ++x) { - int sum = 0; - for (int ky = 0; ky < 2; ++ky) - for (int kx = 0; kx < 2; ++kx) - for (int c = 0; c < 4; ++c) { - int code = (c * 16 + (y + ky) * 4 + x + kx) % 9 - 4; - sum += code * weights[((ky * 2 + kx) * 4 + c) * 32 + n]; - } - if (output[n * 9 + y * 3 + x] != (float)sum * 0.03125f + bias[n]) { - printf("AlexNet Gemmini stride FAILED n=%d y=%d x=%d\n", n, y, x); - return 1; - } - } - printf("AlexNet Gemmini stride PASSED\n"); - return 0; -} diff --git a/examples/chips/toy/workloads/ctests/alexnet_gemmini_tail_test.c b/examples/chips/toy/workloads/ctests/alexnet_gemmini_tail_test.c deleted file mode 100644 index a7561055b..000000000 --- a/examples/chips/toy/workloads/ctests/alexnet_gemmini_tail_test.c +++ /dev/null @@ -1,41 +0,0 @@ -#include "../../../../models/toy/alexnet/execute/conv.c" -#include "buckyball.h" - -static float input[3 * 5 * 5] __attribute__((aligned(64))); -static int8_t weights[27 * 5] __attribute__((aligned(64))); -static float bias[5], output[5 * 3 * 3]; - -int main(void) { - for (int i = 0; i < 75; ++i) - input[i] = (float)(i % 9 - 4) / 4; - for (int i = 0; i < 135; ++i) - weights[i] = i % 5 - 2; - for (int i = 0; i < 5; ++i) - bias[i] = (float)i / 2; - AlexnetF32_4 in = {input, input, 0, {1, 3, 5, 5}, {75, 25, 5, 1}}; - AlexnetI8_2 w = {weights, weights, 0, {27, 5}, {5, 1}}; - AlexnetF32_1 b = {bias, bias, 0, {5}, {1}}; - AlexnetF32_4 out = {output, output, 0, {1, 5, 3, 3}, {45, 9, 3, 1}}; - _mlir_ciface_gemmini_conv(&in, &w, &b, &out, 3, 3, 1, 0, 0.25f, 0.125f); - for (int n = 0; n < 5; ++n) - for (int y = 0; y < 3; ++y) - for (int x = 0; x < 3; ++x) { - int sum = 0; - for (int ky = 0; ky < 3; ++ky) - for (int kx = 0; kx < 3; ++kx) - for (int c = 0; c < 3; ++c) { - int code = (c * 25 + (y + ky) * 5 + x + kx) % 9 - 4; - sum += code * weights[((ky * 3 + kx) * 3 + c) * 5 + n]; - } - float expected = (float)sum * 0.03125f + bias[n]; - if (output[n * 9 + y * 3 + x] != expected) { - printf("AlexNet Gemmini tail FAILED n=%d y=%d x=%d got32=%ld " - "expected32=%ld\n", - n, y, x, (long)(output[n * 9 + y * 3 + x] * 32), - (long)(expected * 32)); - return 1; - } - } - printf("AlexNet Gemmini tail PASSED\n"); - return 0; -} diff --git a/examples/chips/toy/workloads/mlir_tests/CMakeLists.txt b/examples/chips/toy/workloads/mlir_tests/CMakeLists.txt deleted file mode 100644 index 478a635ed..000000000 --- a/examples/chips/toy/workloads/mlir_tests/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -add_custom_target(toy-mlir-tests-build) - -add_subdirectory(linalg) - -add_dependencies(toy-mlir-tests-build - linalg-mlir-tests-build) - -add_dependencies(chip-mlirtests-build toy-mlir-tests-build) diff --git a/examples/chips/toy/workloads/mlir_tests/linalg/CMakeLists.txt b/examples/chips/toy/workloads/mlir_tests/linalg/CMakeLists.txt deleted file mode 100644 index dafa9d014..000000000 --- a/examples/chips/toy/workloads/mlir_tests/linalg/CMakeLists.txt +++ /dev/null @@ -1,28 +0,0 @@ -set(BUCKYBALL_MLIR_GROUP_TARGET linalg-mlir-tests-build) -set(BUCKYBALL_MLIR_TEST_PREFIX linalg) - -add_custom_target(linalg-mlir-tests-build) - -set(TOY_LINALG_FUNCTIONAL_PASSES - -convert-linalg-to-tile - -convert-tile-to-buckyball - -lower-buckyball-to-bank-ssa - -assign-physical-banks - ${BUCKYBALL_LOWER_BANK_SSA_TO_INTRINSICS} - -convert-linalg-to-loops - -expand-strided-metadata - -lower-affine - -convert-scf-to-cf - -convert-cf-to-llvm - ${BUCKYBALL_LOWER_BUCKYBALL} - -convert-arith-to-llvm - -convert-math-to-llvm - -finalize-memref-to-llvm - -convert-func-to-llvm - -reconcile-unrealized-casts) - -foreach(test conv2d_nhwc_hwcf conv2d_nhwc_fhwc conv2d_nchw_fchw) - add_buckyball_mlir_test(${test} - TARGET toy - PASSES ${TOY_LINALG_FUNCTIONAL_PASSES}) -endforeach() diff --git a/examples/chips/toy/workloads/mlir_tests/linalg/conv2d_nchw_fchw.mlir b/examples/chips/toy/workloads/mlir_tests/linalg/conv2d_nchw_fchw.mlir deleted file mode 100644 index a4d3b200d..000000000 --- a/examples/chips/toy/workloads/mlir_tests/linalg/conv2d_nchw_fchw.mlir +++ /dev/null @@ -1,28 +0,0 @@ -func.func private @check_result(memref<1x6x4x1xf32>) -> () - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero_f32 = arith.constant 0.0 : f32 - %one_f32 = arith.constant 1.0 : f32 - - %input = memref.alloc() : memref<1x8x4x1xf32> - %filter = memref.alloc() : memref<6x8x1x1xf32> - %output = memref.alloc() : memref<1x6x4x1xf32> - - linalg.fill ins(%one_f32 : f32) outs(%input : memref<1x8x4x1xf32>) - linalg.fill ins(%one_f32 : f32) outs(%filter : memref<6x8x1x1xf32>) - linalg.fill ins(%zero_f32 : f32) outs(%output : memref<1x6x4x1xf32>) - - linalg.conv_2d_nchw_fchw - {dilations = dense<1> : tensor<2xi64>, strides = dense<1> : tensor<2xi64>} - ins(%input, %filter : memref<1x8x4x1xf32>, memref<6x8x1x1xf32>) - outs(%output : memref<1x6x4x1xf32>) - - func.call @check_result(%output) : (memref<1x6x4x1xf32>) -> () - - memref.dealloc %input : memref<1x8x4x1xf32> - memref.dealloc %filter : memref<6x8x1x1xf32> - memref.dealloc %output : memref<1x6x4x1xf32> - - return %zero_i8 : i8 -} diff --git a/examples/chips/toy/workloads/mlir_tests/linalg/conv2d_nchw_fchw_main.cpp b/examples/chips/toy/workloads/mlir_tests/linalg/conv2d_nchw_fchw_main.cpp deleted file mode 100644 index 9490f9692..000000000 --- a/examples/chips/toy/workloads/mlir_tests/linalg/conv2d_nchw_fchw_main.cpp +++ /dev/null @@ -1,53 +0,0 @@ -#include -#include -#include - -static void fail(void) { -#ifdef BAREMETAL - *(volatile uint32_t *)0x60000000 = 1; - while (1) { - } -#else - exit(1); -#endif -} - -#ifdef __cplusplus -extern "C" -#endif - void check_result(int32_t *allocated, int32_t *aligned, int64_t offset, - int64_t size0, int64_t size1, int64_t size2, - int64_t size3, int64_t stride0, int64_t stride1, - int64_t stride2, int64_t stride3) { - (void)allocated; - - if (size0 != 1 || size1 != 6 || size2 != 4 || size3 != 1 || stride0 != 24 || - stride1 != 4 || stride2 != 1 || stride3 != 1) { - printf("FAILED: linalg conv2d NCHW unexpected memref shape " - "(size=%dx%dx%dx%d stride=%dx%dx%dx%d)\n", - (int)size0, (int)size1, (int)size2, (int)size3, (int)stride0, - (int)stride1, (int)stride2, (int)stride3); - fail(); - } - - int32_t *output = aligned + offset; - const int32_t expected = 0x41000000; - for (int i = 0; i < 1; ++i) { - for (int c = 0; c < 6; ++c) { - for (int h = 0; h < 4; ++h) { - for (int w = 0; w < 1; ++w) { - int64_t idx = i * stride0 + c * stride1 + h * stride2 + w * stride3; - int32_t got = output[idx]; - if (got != expected) { - printf("FAILED: linalg conv2d NCHW output[%d][%d][%d][%d] " - "(expected 0x%08x, got 0x%08x)\n", - i, c, h, w, expected, got); - fail(); - } - } - } - } - } - - printf("PASSED: linalg conv2d NCHW FCHW\n"); -} diff --git a/examples/chips/toy/workloads/mlir_tests/linalg/conv2d_nhwc_fhwc.mlir b/examples/chips/toy/workloads/mlir_tests/linalg/conv2d_nhwc_fhwc.mlir deleted file mode 100644 index 1c8c39948..000000000 --- a/examples/chips/toy/workloads/mlir_tests/linalg/conv2d_nhwc_fhwc.mlir +++ /dev/null @@ -1,28 +0,0 @@ -func.func private @check_result(memref<1x4x1x6xf32>) -> () - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero_f32 = arith.constant 0.0 : f32 - %one_f32 = arith.constant 1.0 : f32 - - %input = memref.alloc() : memref<1x4x1x8xf32> - %filter = memref.alloc() : memref<6x1x1x8xf32> - %output = memref.alloc() : memref<1x4x1x6xf32> - - linalg.fill ins(%one_f32 : f32) outs(%input : memref<1x4x1x8xf32>) - linalg.fill ins(%one_f32 : f32) outs(%filter : memref<6x1x1x8xf32>) - linalg.fill ins(%zero_f32 : f32) outs(%output : memref<1x4x1x6xf32>) - - linalg.conv_2d_nhwc_fhwc - {dilations = dense<1> : tensor<2xi64>, strides = dense<1> : tensor<2xi64>} - ins(%input, %filter : memref<1x4x1x8xf32>, memref<6x1x1x8xf32>) - outs(%output : memref<1x4x1x6xf32>) - - func.call @check_result(%output) : (memref<1x4x1x6xf32>) -> () - - memref.dealloc %input : memref<1x4x1x8xf32> - memref.dealloc %filter : memref<6x1x1x8xf32> - memref.dealloc %output : memref<1x4x1x6xf32> - - return %zero_i8 : i8 -} diff --git a/examples/chips/toy/workloads/mlir_tests/linalg/conv2d_nhwc_fhwc_main.cpp b/examples/chips/toy/workloads/mlir_tests/linalg/conv2d_nhwc_fhwc_main.cpp deleted file mode 100644 index dedcd26e1..000000000 --- a/examples/chips/toy/workloads/mlir_tests/linalg/conv2d_nhwc_fhwc_main.cpp +++ /dev/null @@ -1,53 +0,0 @@ -#include -#include -#include - -static void fail(void) { -#ifdef BAREMETAL - *(volatile uint32_t *)0x60000000 = 1; - while (1) { - } -#else - exit(1); -#endif -} - -#ifdef __cplusplus -extern "C" -#endif - void check_result(int32_t *allocated, int32_t *aligned, int64_t offset, - int64_t size0, int64_t size1, int64_t size2, - int64_t size3, int64_t stride0, int64_t stride1, - int64_t stride2, int64_t stride3) { - (void)allocated; - - if (size0 != 1 || size1 != 4 || size2 != 1 || size3 != 6 || stride0 != 24 || - stride1 != 6 || stride2 != 6 || stride3 != 1) { - printf("FAILED: linalg conv2d NHWC unexpected memref shape " - "(size=%dx%dx%dx%d stride=%dx%dx%dx%d)\n", - (int)size0, (int)size1, (int)size2, (int)size3, (int)stride0, - (int)stride1, (int)stride2, (int)stride3); - fail(); - } - - int32_t *output = aligned + offset; - const int32_t expected = 0x41000000; - for (int i = 0; i < 1; ++i) { - for (int h = 0; h < 4; ++h) { - for (int w = 0; w < 1; ++w) { - for (int c = 0; c < 6; ++c) { - int64_t idx = i * stride0 + h * stride1 + w * stride2 + c * stride3; - int32_t got = output[idx]; - if (got != expected) { - printf("FAILED: linalg conv2d NHWC output[%d][%d][%d][%d] " - "(expected 0x%08x, got 0x%08x)\n", - i, h, w, c, expected, got); - fail(); - } - } - } - } - } - - printf("PASSED: linalg conv2d NHWC FHWC\n"); -} diff --git a/examples/chips/toy/workloads/mlir_tests/linalg/conv2d_nhwc_hwcf.mlir b/examples/chips/toy/workloads/mlir_tests/linalg/conv2d_nhwc_hwcf.mlir deleted file mode 100644 index cca9db045..000000000 --- a/examples/chips/toy/workloads/mlir_tests/linalg/conv2d_nhwc_hwcf.mlir +++ /dev/null @@ -1,28 +0,0 @@ -func.func private @check_result(memref<1x4x1x6xf32>) -> () - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero_f32 = arith.constant 0.0 : f32 - %one_f32 = arith.constant 1.0 : f32 - - %input = memref.alloc() : memref<1x4x1x8xf32> - %filter = memref.alloc() : memref<1x1x8x6xf32> - %output = memref.alloc() : memref<1x4x1x6xf32> - - linalg.fill ins(%one_f32 : f32) outs(%input : memref<1x4x1x8xf32>) - linalg.fill ins(%one_f32 : f32) outs(%filter : memref<1x1x8x6xf32>) - linalg.fill ins(%zero_f32 : f32) outs(%output : memref<1x4x1x6xf32>) - - linalg.conv_2d_nhwc_hwcf - {dilations = dense<1> : tensor<2xi64>, strides = dense<1> : tensor<2xi64>} - ins(%input, %filter : memref<1x4x1x8xf32>, memref<1x1x8x6xf32>) - outs(%output : memref<1x4x1x6xf32>) - - func.call @check_result(%output) : (memref<1x4x1x6xf32>) -> () - - memref.dealloc %input : memref<1x4x1x8xf32> - memref.dealloc %filter : memref<1x1x8x6xf32> - memref.dealloc %output : memref<1x4x1x6xf32> - - return %zero_i8 : i8 -} diff --git a/examples/chips/toy/workloads/mlir_tests/linalg/conv2d_nhwc_hwcf_main.cpp b/examples/chips/toy/workloads/mlir_tests/linalg/conv2d_nhwc_hwcf_main.cpp deleted file mode 100644 index 2cdd0f448..000000000 --- a/examples/chips/toy/workloads/mlir_tests/linalg/conv2d_nhwc_hwcf_main.cpp +++ /dev/null @@ -1,53 +0,0 @@ -#include -#include -#include - -static void fail(void) { -#ifdef BAREMETAL - *(volatile uint32_t *)0x60000000 = 1; - while (1) { - } -#else - exit(1); -#endif -} - -#ifdef __cplusplus -extern "C" -#endif - void check_result(int32_t *allocated, int32_t *aligned, int64_t offset, - int64_t size0, int64_t size1, int64_t size2, - int64_t size3, int64_t stride0, int64_t stride1, - int64_t stride2, int64_t stride3) { - (void)allocated; - - if (size0 != 1 || size1 != 4 || size2 != 1 || size3 != 6 || stride0 != 24 || - stride1 != 6 || stride2 != 6 || stride3 != 1) { - printf("FAILED: linalg conv2d NHWC unexpected memref shape " - "(size=%dx%dx%dx%d stride=%dx%dx%dx%d)\n", - (int)size0, (int)size1, (int)size2, (int)size3, (int)stride0, - (int)stride1, (int)stride2, (int)stride3); - fail(); - } - - int32_t *output = aligned + offset; - const int32_t expected = 0x41000000; - for (int i = 0; i < 1; ++i) { - for (int h = 0; h < 4; ++h) { - for (int w = 0; w < 1; ++w) { - for (int c = 0; c < 6; ++c) { - int64_t idx = i * stride0 + h * stride1 + w * stride2 + c * stride3; - int32_t got = output[idx]; - if (got != expected) { - printf("FAILED: linalg conv2d NHWC output[%d][%d][%d][%d] " - "(expected 0x%08x, got 0x%08x)\n", - i, h, w, c, expected, got); - fail(); - } - } - } - } - } - - printf("PASSED: linalg conv2d NHWC HWCF\n"); -} diff --git a/examples/cores/README.md b/examples/cores/README.md deleted file mode 100644 index ffe742d6a..000000000 --- a/examples/cores/README.md +++ /dev/null @@ -1,24 +0,0 @@ -# Cores - -Each directory in this tree describes one concrete single-core design. A Core -owns its CPU/frontend parameters, BallDomain, MemDomain, and the -compiler package for its ISA. It does not own a tile topology, device model, -kernel, or regression suite. - -Each implemented Core has an `arch/` top implementing `HasCLink` and explicitly -instantiating its CPU and compute IPs. The framework connects CLink to shared -memory and control interfaces; concrete designs do not inherit another design. - -Every Core has a `configs/` tree and `isa/` (`ballISA.h` from balldomain). -Buckyball Cores also have `compiler/` (buddy-mlir dialect plugin; CMake is the -C++ recipe, not the product entry) and non-empty `ballISA`. Rocket-only -cores keep `configs/default.toml` without a `balldomain` key so tiles stay -Rocket-only; empty `configs/balldomains/` + `isa/` still exist for Bazel / -`compilerCore` consumers. BEMU lives in bebop; chip-level tile runners are -`examples/chips//emu/src/main.rs`. Ball functional models are -`examples/balls//emu/`. - -Chip topology files instantiate a concrete Core TOML through relative -`include` paths; that include is the config entry (not a fixed -`configs/default.toml`). Chips own tile-shared memory, runtime, and multi-core -behavior. diff --git a/examples/cores/ant/arch/src/main/scala/AntCore.scala b/examples/cores/ant/arch/src/main/scala/AntCore.scala deleted file mode 100644 index f66c99ac6..000000000 --- a/examples/cores/ant/arch/src/main/scala/AntCore.scala +++ /dev/null @@ -1,19 +0,0 @@ -package examples.cores.ant - -import framework.top.GlobalConfig -import chisel3.experimental.hierarchy.{instantiable, Instance, Instantiate} -import framework.ant.LocalExecution -import framework.system.core.{AntEndpoint, CoreConnection} -import framework.system.core.accelerator.{AntAdmission, BuckyballAccelerator} - -@instantiable -class AntCore(config: GlobalConfig) extends AntEndpoint(config) { - val execution = Instantiate(new LocalExecution(local)) - val accelerator = Instantiate(new BuckyballAccelerator(p.buckyball)) - val issuer = Instantiate(new AntAdmission(local, tracking, cpuParams.core.nPMPs, bus)) - CoreConnection.ant(clink, execution, accelerator, issuer, signature) -} - -object AntCore extends framework.system.core.AntCoreFactory { - def instantiate(b: GlobalConfig): Instance[AntEndpoint] = Instantiate(new AntCore(b)) -} diff --git a/examples/cores/ant/configs/cpu/ant.toml b/examples/cores/ant/configs/cpu/ant.toml deleted file mode 100644 index f658a9bc7..000000000 --- a/examples/cores/ant/configs/cpu/ant.toml +++ /dev/null @@ -1,2 +0,0 @@ -kind = "ant" -config = "default.toml" diff --git a/examples/cores/ant/configs/cpu/default.toml b/examples/cores/ant/configs/cpu/default.toml deleted file mode 100644 index 19fe2e82e..000000000 --- a/examples/cores/ant/configs/cpu/default.toml +++ /dev/null @@ -1,7 +0,0 @@ -codeBytes = 16384 -taskBits = 32 - -[tls] -base = 0x10000 -bytes = 16384 -dataBits = 128 diff --git a/examples/cores/ant/configs/default.toml b/examples/cores/ant/configs/default.toml deleted file mode 100644 index 9eab27336..000000000 --- a/examples/cores/ant/configs/default.toml +++ /dev/null @@ -1,2 +0,0 @@ -factory = "examples.cores.ant.AntCore" -cpu = "cpu/ant.toml" diff --git a/examples/cores/attention/arch/src/main/scala/AttentionCore.scala b/examples/cores/attention/arch/src/main/scala/AttentionCore.scala deleted file mode 100644 index 8afd2d445..000000000 --- a/examples/cores/attention/arch/src/main/scala/AttentionCore.scala +++ /dev/null @@ -1,19 +0,0 @@ -package examples.cores.attention - -import framework.top.GlobalConfig -import chisel3.experimental.hierarchy.{instantiable, Instance, Instantiate} -import framework.ant.LocalExecution -import framework.system.core.{AntEndpoint, CoreConnection} -import framework.system.core.accelerator.{AntAdmission, BuckyballAccelerator} - -@instantiable -class AttentionCore(config: GlobalConfig) extends AntEndpoint(config) { - val execution = Instantiate(new LocalExecution(local)) - val accelerator = Instantiate(new BuckyballAccelerator(p.buckyball)) - val issuer = Instantiate(new AntAdmission(local, tracking, cpuParams.core.nPMPs, bus)) - CoreConnection.ant(clink, execution, accelerator, issuer, signature) -} - -object AttentionCore extends framework.system.core.AntCoreFactory { - def instantiate(b: GlobalConfig): Instance[AntEndpoint] = Instantiate(new AttentionCore(b)) -} diff --git a/examples/cores/attention/compiler/CMakeLists.txt b/examples/cores/attention/compiler/CMakeLists.txt deleted file mode 100644 index 6b8bd19d0..000000000 --- a/examples/cores/attention/compiler/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -if(NOT COMMAND add_mlir_library OR NOT COMMAND add_mlir_dialect) - message(FATAL_ERROR "MLIR CMake helpers must be loaded. Build as part of buddy-mlir.") -endif() - -set(BB_CORE_NAME attention) -set(BB_CORE_TARGET_PREFIX "") - -set(BB_CORE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/..) -get_filename_component(_BB_REPO_ROOT "${BB_CORE_DIR}/../../.." ABSOLUTE) -include_directories(${_BB_REPO_ROOT}/examples/balls) - -add_subdirectory(src) diff --git a/examples/cores/attention/compiler/src/CMakeLists.txt b/examples/cores/attention/compiler/src/CMakeLists.txt deleted file mode 100644 index 081d3ff4a..000000000 --- a/examples/cores/attention/compiler/src/CMakeLists.txt +++ /dev/null @@ -1,51 +0,0 @@ -if(NOT COMMAND add_mlir_library OR NOT COMMAND add_mlir_dialect) - message(FATAL_ERROR "MLIR CMake helpers must be loaded. Build as part of buddy-mlir.") -endif() - -message(STATUS "Building Attention compiler Core package") -message(STATUS " Source: ${CMAKE_CURRENT_SOURCE_DIR}") -message(STATUS " Binary: ${CMAKE_CURRENT_BINARY_DIR}") - -get_filename_component(BB_REPO_ROOT ${CMAKE_CURRENT_LIST_DIR}/../../../../.. ABSOLUTE) -set(BB_COMPILER_COMMON_DIR ${BB_REPO_ROOT}/stack/compiler/src) - -if(NOT EXISTS ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball/Buckyball.td) - message(FATAL_ERROR "Missing Buckyball compiler common dir: ${BB_COMPILER_COMMON_DIR}") -endif() - -include_directories(${BB_COMPILER_COMMON_DIR}) -include_directories(${BB_COMPILER_COMMON_DIR}/Dialect) -include_directories(${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}) -include_directories(${CMAKE_CURRENT_BINARY_DIR}) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/Dialect) -include_directories(${CMAKE_CURRENT_BINARY_DIR}/Dialect) -include_directories(BEFORE ${CMAKE_CURRENT_BINARY_DIR}/Dialect/Buckyball) - -set(ATTENTION_BALL_COMPILER_DIALECT_DIRS - ${BB_REPO_ROOT}/examples/balls/toint8/compiler/src/Dialect/Buckyball - ${BB_REPO_ROOT}/examples/balls/int2fp/compiler/src/Dialect/Buckyball - ${BB_REPO_ROOT}/examples/balls/transpose/compiler/src/Dialect/Buckyball - ${BB_REPO_ROOT}/examples/balls/smatmul/compiler/src/Dialect/Buckyball -) - -foreach(ATTENTION_BALL_COMPILER_DIALECT_DIR ${ATTENTION_BALL_COMPILER_DIALECT_DIRS}) - if(NOT EXISTS ${ATTENTION_BALL_COMPILER_DIALECT_DIR}) - message(FATAL_ERROR - "Missing attention ball compiler dialect dir: ${ATTENTION_BALL_COMPILER_DIALECT_DIR}") - endif() - include_directories(${ATTENTION_BALL_COMPILER_DIALECT_DIR}) -endforeach() - -add_subdirectory(Dialect) -add_subdirectory(Conversion) -add_subdirectory(Target) - -add_custom_target(buckyball-compiler-${BB_CORE_NAME} - DEPENDS - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballTransforms - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballToLLVMIRTranslation - ${BB_CORE_TARGET_PREFIX}LowerBuckyballPass - ${BB_CORE_TARGET_PREFIX}LowerTileToBuckyballPass -) diff --git a/examples/cores/attention/compiler/src/Conversion/CMakeLists.txt b/examples/cores/attention/compiler/src/Conversion/CMakeLists.txt deleted file mode 100644 index 9ca69b1a2..000000000 --- a/examples/cores/attention/compiler/src/Conversion/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -add_subdirectory(LowerBuckyball) -add_subdirectory(LowerTileToBuckyball) diff --git a/examples/cores/attention/compiler/src/Conversion/LowerBuckyball/CMakeLists.txt b/examples/cores/attention/compiler/src/Conversion/LowerBuckyball/CMakeLists.txt deleted file mode 100644 index eb782edc6..000000000 --- a/examples/cores/attention/compiler/src/Conversion/LowerBuckyball/CMakeLists.txt +++ /dev/null @@ -1,16 +0,0 @@ -add_mlir_library(${BB_CORE_TARGET_PREFIX}LowerBuckyballPass - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/BaseAssignPhysicalBankPatterns.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/PhysicalBankState.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/ReportBankUsagePass.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/AssignPhysicalBanksPass.cpp - ${BB_assign_SOURCES} - ${BB_bank_ssa_SOURCES} - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/QuantizeTensorToBankSSAPatterns.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/RuntimeLowering.cpp - LowerBuckyballPass.cpp - LowerBuckyballToBankSSAPass.cpp - - LINK_LIBS PUBLIC - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballTransforms - MLIRLinalgDialect -) diff --git a/examples/cores/attention/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp b/examples/cores/attention/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp deleted file mode 100644 index c4989db92..000000000 --- a/examples/cores/attention/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp +++ /dev/null @@ -1,127 +0,0 @@ -//===- LowerBuckyballPass.cpp - Attention Buckyball lowering pass -//-------------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/TypeConverter.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/Pass/Pass.h" - -#include "Buckyball/BuckyballDialect.h" -#include "Buckyball/BuckyballOps.h" -#include "Buckyball/Transform.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { - -class LowerBuckyballToLLVMPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerBuckyballToLLVMPass) - LowerBuckyballToLLVMPass() = default; - LowerBuckyballToLLVMPass(const LowerBuckyballToLLVMPass &) {} - - StringRef getArgument() const final { return "lower-buckyball"; } - StringRef getDescription() const final { - return "Lower Attention Buckyball dialect ops."; - } - - Option stable{*this, "stable", - llvm::cl::desc("Use stable LLVM Buckyball intrinsics."), - llvm::cl::init(false)}; - - void getDependentDialects(DialectRegistry ®istry) const override { - registry - .insert(); - } - - void runOnOperation() override { - const auto &targetConfig = buckyball_target::getBuckyballTarget(); - MLIRContext *context = &getContext(); - ModuleOp module = getOperation(); - LLVMTypeConverter converter(context); - RewritePatternSet patterns(context); - LLVMConversionTarget target(*context); - - configureBuckyballLegalizeForExportTarget(target, stable); - target.addLegalDialect(); - populateBuckyballLegalizeForLLVMExportPatterns( - converter, patterns, targetConfig.bankWidthBits / 8, - targetConfig.bankDepth, targetConfig.bankNum, - /*includeFuncOperandForwarding=*/false, stable); - - ConversionConfig config; - config.allowPatternRollback = false; - if (failed(applyPartialConversion(module, target, std::move(patterns), - config))) - signalPassFailure(); - } -}; - -class LowerBankSSAToIntrinsicsPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerBankSSAToIntrinsicsPass) - LowerBankSSAToIntrinsicsPass() = default; - LowerBankSSAToIntrinsicsPass(const LowerBankSSAToIntrinsicsPass &) {} - - StringRef getArgument() const final { return "lower-bank-ssa-to-intrinsics"; } - StringRef getDescription() const final { - return "Lower Attention bank-SSA and Buckyball ops to intrinsic ops."; - } - - Option stable{*this, "stable", - llvm::cl::desc("Use stable LLVM Buckyball intrinsics."), - llvm::cl::init(false)}; - - void getDependentDialects(DialectRegistry ®istry) const override { - registry - .insert(); - } - - void runOnOperation() override { - const auto &targetConfig = buckyball_target::getBuckyballTarget(); - MLIRContext *context = &getContext(); - ModuleOp module = getOperation(); - LLVMTypeConverter converter(context); - RewritePatternSet patterns(context); - LLVMConversionTarget target(*context); - - configureBuckyballLegalizeForExportTarget(target, stable); - target.addLegalDialect(); - populateBuckyballLegalizeForLLVMExportPatterns( - converter, patterns, targetConfig.bankWidthBits / 8, - targetConfig.bankDepth, targetConfig.bankNum, - /*includeFuncOperandForwarding=*/false, stable); - - ConversionConfig config; - config.allowPatternRollback = false; - if (failed(applyPartialConversion(module, target, std::move(patterns), - config))) - signalPassFailure(); - } -}; - -} // namespace - -void mlir::buddy::registerLowerBuckyballPass() { - registerTileRuntimePass(); - PassRegistration(); -} - -void mlir::buddy::registerLowerBankSSAToIntrinsicsPass() { - PassRegistration(); -} diff --git a/examples/cores/attention/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp b/examples/cores/attention/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp deleted file mode 100644 index 9450ff9b5..000000000 --- a/examples/cores/attention/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp +++ /dev/null @@ -1,108 +0,0 @@ -//===- LowerBuckyballToBankSSAPass.cpp - Buckyball bank-SSA lowering -//----------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" -#include "Target/BuckyballTargetRegistry.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/Dialect/Utils/StructuredOpsUtils.h" -#include "mlir/IR/AffineMap.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/PatternMatch.h" -#include "mlir/Pass/Pass.h" -#include "mlir/Transforms/GreedyPatternRewriteDriver.h" - -#include "Buckyball/BuckyballDialect.h" -#include "Buckyball/BuckyballOps.h" -#include "Trace/TraceDialect.h" - -using namespace mlir; - -namespace mlir::buddy { -#define BUCKYBALL_BANK_SSA_HOOK(BALL) \ - void populate##BALL##LowerBuckyballToBankSSAPatterns(RewritePatternSet &); -#include "BuckyballBallLoweringHooks.inc" -#undef BUCKYBALL_BANK_SSA_HOOK -} // namespace mlir::buddy - -namespace { - -class MemTransposeToLinalgPattern - : public OpRewritePattern<::buddy::buckyball::MemTransposeOp> { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(::buddy::buckyball::MemTransposeOp op, - PatternRewriter &rewriter) const override { - auto input = dyn_cast(op.getInput().getType()); - auto output = dyn_cast(op.getOutput().getType()); - if (!input || !output || !input.hasStaticShape() || - !output.hasStaticShape() || input.getRank() != 2 || - output.getRank() != 2 || output.getShape()[0] != input.getShape()[1] || - output.getShape()[1] != input.getShape()[0]) - return op.emitError("requires matching static rank-2 transpose memrefs"); - - MLIRContext *context = rewriter.getContext(); - AffineMap identity = AffineMap::getMultiDimIdentityMap(2, context); - AffineMap transpose = AffineMap::get( - 2, 0, {rewriter.getAffineDimExpr(1), rewriter.getAffineDimExpr(0)}, - context); - SmallVector iterators(2, - utils::IteratorType::parallel); - rewriter.create( - op.getLoc(), TypeRange{}, ValueRange{op.getInput()}, - ValueRange{op.getOutput()}, ArrayRef{identity, transpose}, - iterators, [](OpBuilder &builder, Location location, ValueRange args) { - builder.create(location, args[0]); - }); - rewriter.eraseOp(op); - return success(); - } -}; - -} // namespace - -namespace { - -class LowerBuckyballToBankSSAPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerBuckyballToBankSSAPass) - - StringRef getArgument() const final { return "lower-buckyball-to-bank-ssa"; } - StringRef getDescription() const final { - return "Lower Buckyball Buckyball ops to explicit bank-SSA ops."; - } - - void getDependentDialects(DialectRegistry ®istry) const override { - registry.insert<::buddy::trace::BuddyTraceDialect, arith::ArithDialect, - linalg::LinalgDialect, memref::MemRefDialect, - scf::SCFDialect, ::buddy::buckyball::BuckyballDialect>(); - } - - void runOnOperation() override { - RewritePatternSet patterns(&getContext()); - for (llvm::StringRef ball : buckyball_target::getBuckyballTarget().balls) { -#define BUCKYBALL_BANK_SSA_HOOK(BALL) \ - if (ball == #BALL) \ - mlir::buddy::populate##BALL##LowerBuckyballToBankSSAPatterns(patterns); -#include "BuckyballBallLoweringHooks.inc" -#undef BUCKYBALL_BANK_SSA_HOOK - } - mlir::buddy::populateQuantizeTensorToBankSSAPatterns(patterns); - patterns.add(&getContext()); - if (failed(applyPatternsGreedily(getOperation(), std::move(patterns)))) - signalPassFailure(); - } -}; - -} // namespace - -void mlir::buddy::registerLowerBuckyballToBankSSAPass() { - PassRegistration(); -} diff --git a/examples/cores/attention/compiler/src/Conversion/LowerTileToBuckyball/CMakeLists.txt b/examples/cores/attention/compiler/src/Conversion/LowerTileToBuckyball/CMakeLists.txt deleted file mode 100644 index c12d60c23..000000000 --- a/examples/cores/attention/compiler/src/Conversion/LowerTileToBuckyball/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -add_mlir_library(${BB_CORE_TARGET_PREFIX}LowerTileToBuckyballPass - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerTileToBuckyball/QuantizedKernelPatterns.cpp - LowerTileToBuckyball.cpp - ${BB_REPO_ROOT}/examples/balls/smatmul/compiler/src/Conversion/LowerTileToBuckyball/TileMatMulPatterns.cpp - - LINK_LIBS PUBLIC - BuddyTile - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball - MLIRLinalgDialect -) diff --git a/examples/cores/attention/compiler/src/Conversion/LowerTileToBuckyball/LowerTileToBuckyball.cpp b/examples/cores/attention/compiler/src/Conversion/LowerTileToBuckyball/LowerTileToBuckyball.cpp deleted file mode 100644 index cca771535..000000000 --- a/examples/cores/attention/compiler/src/Conversion/LowerTileToBuckyball/LowerTileToBuckyball.cpp +++ /dev/null @@ -1,104 +0,0 @@ -#include "Target/BuckyballTargetRegistry.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinOps.h" -#include "mlir/Pass/Pass.h" -#include "mlir/Transforms/DialectConversion.h" - -#include "Buckyball/BuckyballDialect.h" -#include "Buckyball/BuckyballOps.h" -#include "Conversion/LowerTileToBuckyball/LowerTileToBuckyball.h" -#include "Tile/TileDialect.h" -#include "Tile/TileOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; -namespace tile = ::buddy::tile; - -namespace mlir::buddy { -#define BUCKYBALL_TILE_HOOK(BALL) \ - void populate##BALL##TileLoweringPatterns(RewritePatternSet &, int64_t, \ - int64_t, int64_t); -#include "BuckyballBallLoweringHooks.inc" -#undef BUCKYBALL_TILE_HOOK -} // namespace mlir::buddy - -namespace { -class TileTransposeLowering : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(tile::TileTransposeOp op, - PatternRewriter &rewriter) const override { - auto inputType = dyn_cast(op.getAMemArray().getType()); - auto outputType = dyn_cast(op.getBMemArray().getType()); - if (!inputType || !outputType || !inputType.hasStaticShape() || - !outputType.hasStaticShape() || inputType.getRank() != 2 || - outputType.getRank() != 2) - return op.emitError("requires static rank-2 input and output memrefs"); - if (outputType.getShape()[0] != inputType.getShape()[1] || - outputType.getShape()[1] != inputType.getShape()[0] || - inputType.getElementType() != outputType.getElementType()) - return op.emitError("requires a matching transposed output memref"); - - rewriter.create(op.getLoc(), op.getAMemArray(), - op.getBMemArray()); - rewriter.eraseOp(op); - return success(); - } -}; - -class LowerTileToBuckyballPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerTileToBuckyballPass) - - StringRef getArgument() const final { return "convert-tile-to-buckyball"; } - StringRef getDescription() const final { - return "Convert Tile operations for the selected Buckyball target."; - } - - void getDependentDialects(DialectRegistry ®istry) const override { - registry - .insert(); - } - - void runOnOperation() override { - const auto &targetConfig = buckyball_target::getBuckyballTarget(); - MLIRContext *context = &getContext(); - ConversionTarget target(*context); - target.addLegalDialect<::buddy::buckyball::BuckyballDialect, - memref::MemRefDialect, arith::ArithDialect, - scf::SCFDialect, func::FuncDialect, - linalg::LinalgDialect>(); - target.addIllegalOp(); - - RewritePatternSet patterns(context); - mlir::buddy::populateQuantizedKernelTileLoweringPatterns(patterns); - patterns.add(context); - for (llvm::StringRef ball : targetConfig.balls) { -#define BUCKYBALL_TILE_HOOK(BALL) \ - if (ball == #BALL) \ - mlir::buddy::populate##BALL##TileLoweringPatterns( \ - patterns, targetConfig.bankWidthBits / 8, targetConfig.bankDepth, \ - targetConfig.bankNum); -#include "BuckyballBallLoweringHooks.inc" -#undef BUCKYBALL_TILE_HOOK - } - if (failed(applyPartialConversion(getOperation(), target, - std::move(patterns)))) - signalPassFailure(); - } -}; -} // namespace - -void mlir::buddy::registerLowerTileToBuckyballPass() { - PassRegistration(); -} diff --git a/examples/cores/attention/compiler/src/Dialect/Buckyball/AttentionBuckyball.td b/examples/cores/attention/compiler/src/Dialect/Buckyball/AttentionBuckyball.td deleted file mode 100644 index e9290ae5c..000000000 --- a/examples/cores/attention/compiler/src/Dialect/Buckyball/AttentionBuckyball.td +++ /dev/null @@ -1,26 +0,0 @@ -//===-- AttentionBuckyball.td - Attention Buckyball dialect ops -*- tablegen -*-===// -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//===----------------------------------------------------------------------===// - -#ifndef ATTENTION_BUCKYBALL_OPS -#define ATTENTION_BUCKYBALL_OPS - -include "Buckyball.td" -include "ToInt8Ball.td" -include "Int2FpBall.td" -include "TransposeBall.td" -include "MxmmBall.td" - -#endif // ATTENTION_BUCKYBALL_OPS diff --git a/examples/cores/attention/compiler/src/Dialect/Buckyball/CMakeLists.txt b/examples/cores/attention/compiler/src/Dialect/Buckyball/CMakeLists.txt deleted file mode 100644 index 598c4434a..000000000 --- a/examples/cores/attention/compiler/src/Dialect/Buckyball/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -set(LLVM_TARGET_DEFINITIONS ${BB_BUCKYBALL_DIALECT_TD}) -mlir_tablegen(Buckyball.h.inc -gen-op-decls) -mlir_tablegen(Buckyball.cpp.inc -gen-op-defs) -mlir_tablegen(BuckyballTypes.h.inc -gen-typedef-decls -typedefs-dialect=buckyball) -mlir_tablegen(BuckyballTypes.cpp.inc -gen-typedef-defs -typedefs-dialect=buckyball) -mlir_tablegen(BuckyballDialect.h.inc -gen-dialect-decls -dialect=buckyball) -mlir_tablegen(BuckyballDialect.cpp.inc -gen-dialect-defs -dialect=buckyball) -add_public_tablegen_target(${BB_CORE_TARGET_PREFIX}MLIRBuckyballIncGen) -add_mlir_doc(AttentionBuckyball ${BB_CORE_TARGET_PREFIX}Buckyball Dialects/ - -gen-dialect-doc) - -set(LLVM_TARGET_DEFINITIONS ${BB_BUCKYBALL_DIALECT_TD}) -mlir_tablegen(BuckyballConversions.inc -gen-llvmir-conversions) -add_public_tablegen_target(${BB_CORE_TARGET_PREFIX}BuddyBuckyballConversionsIncGen) - -add_subdirectory(IR) -add_subdirectory(Transforms) diff --git a/examples/cores/attention/compiler/src/Dialect/Buckyball/IR/CMakeLists.txt b/examples/cores/attention/compiler/src/Dialect/Buckyball/IR/CMakeLists.txt deleted file mode 100644 index 42c7f8807..000000000 --- a/examples/cores/attention/compiler/src/Dialect/Buckyball/IR/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -add_mlir_dialect_library(${BB_CORE_TARGET_PREFIX}BuddyBuckyball - ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball/IR/BuckyballDialect.cpp - ${BB_TARGET_REGISTRY_SOURCE} - ${BB_BALL_IR_SOURCES} - - LINK_LIBS PUBLIC - MLIRIR - MLIRLLVMDialect - MLIRBufferizationDialect - MLIRMemRefDialect -) - -target_include_directories(${BB_CORE_TARGET_PREFIX}BuddyBuckyball PUBLIC - ${BB_COMPILER_COMMON_DIR} - ${BB_COMPILER_COMMON_DIR}/Dialect - ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball - ${CMAKE_CURRENT_BINARY_DIR}/.. - ${CMAKE_CURRENT_BINARY_DIR}/../../.. -) diff --git a/examples/cores/attention/compiler/src/Dialect/Buckyball/Transforms/CMakeLists.txt b/examples/cores/attention/compiler/src/Dialect/Buckyball/Transforms/CMakeLists.txt deleted file mode 100644 index 0d2cecf2e..000000000 --- a/examples/cores/attention/compiler/src/Dialect/Buckyball/Transforms/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -add_mlir_dialect_library(${BB_CORE_TARGET_PREFIX}BuddyBuckyballTransforms - ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.cpp - ${BB_REPO_ROOT}/examples/balls/toint8/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - ${BB_REPO_ROOT}/examples/balls/int2fp/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - ${BB_REPO_ROOT}/examples/balls/transpose/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - ${BB_REPO_ROOT}/examples/balls/smatmul/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - LegalizeForLLVMExport.cpp - - LINK_LIBS PUBLIC - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball - MLIRIR - MLIRLLVMCommonConversion - MLIRLLVMDialect -) diff --git a/examples/cores/attention/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/cores/attention/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 0b1a5b224..000000000 --- a/examples/cores/attention/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,45 +0,0 @@ -#include "Buckyball/Transform.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace buddy::buckyball::legalize; - -namespace mlir::buddy::buckyball { -#define BUCKYBALL_LEGALIZE_HOOK(BALL) \ - void populate##BALL##LegalizeForLLVMExportPatterns( \ - LLVMTypeConverter &, RewritePatternSet &, bool, int64_t); \ - void configure##BALL##LegalizeForExportTarget(LLVMConversionTarget &, bool); -#include "BuckyballBallLoweringHooks.inc" -#undef BUCKYBALL_LEGALIZE_HOOK -} // namespace mlir::buddy::buckyball - -void mlir::populateBuckyballLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, - int64_t bankWidthBytes, int64_t bankDepth, int64_t bankNum, - bool includeFuncOperandForwarding, bool stable) { - populateBaseLegalizeForLLVMExportPatterns(converter, patterns, - includeFuncOperandForwarding); - for (llvm::StringRef ball : buckyball_target::getBuckyballTarget().balls) { -#define BUCKYBALL_LEGALIZE_HOOK(BALL) \ - if (ball == #BALL) \ - buddy::buckyball::populate##BALL##LegalizeForLLVMExportPatterns( \ - converter, patterns, stable, bankDepth); -#include "BuckyballBallLoweringHooks.inc" -#undef BUCKYBALL_LEGALIZE_HOOK - } - (void)bankWidthBytes; - (void)bankNum; -} - -void mlir::configureBuckyballLegalizeForExportTarget( - LLVMConversionTarget &target, bool stable) { - configureBaseLegalizeForExportTarget(target); - for (llvm::StringRef ball : buckyball_target::getBuckyballTarget().balls) { -#define BUCKYBALL_LEGALIZE_HOOK(BALL) \ - if (ball == #BALL) \ - buddy::buckyball::configure##BALL##LegalizeForExportTarget(target, stable); -#include "BuckyballBallLoweringHooks.inc" -#undef BUCKYBALL_LEGALIZE_HOOK - } -} diff --git a/examples/cores/attention/compiler/src/Dialect/CMakeLists.txt b/examples/cores/attention/compiler/src/Dialect/CMakeLists.txt deleted file mode 100644 index ec181148e..000000000 --- a/examples/cores/attention/compiler/src/Dialect/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory(Buckyball) diff --git a/examples/cores/attention/compiler/src/Target/CMakeLists.txt b/examples/cores/attention/compiler/src/Target/CMakeLists.txt deleted file mode 100644 index 39d31dc9b..000000000 --- a/examples/cores/attention/compiler/src/Target/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory(LLVMIR) diff --git a/examples/cores/attention/compiler/src/Target/LLVMIR/CMakeLists.txt b/examples/cores/attention/compiler/src/Target/LLVMIR/CMakeLists.txt deleted file mode 100644 index 0ca0f41c5..000000000 --- a/examples/cores/attention/compiler/src/Target/LLVMIR/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory(Dialect) diff --git a/examples/cores/attention/compiler/src/Target/LLVMIR/Dialect/Buckyball/CMakeLists.txt b/examples/cores/attention/compiler/src/Target/LLVMIR/Dialect/Buckyball/CMakeLists.txt deleted file mode 100644 index 06b99171b..000000000 --- a/examples/cores/attention/compiler/src/Target/LLVMIR/Dialect/Buckyball/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -add_mlir_translation_library(${BB_CORE_TARGET_PREFIX}BuddyBuckyballToLLVMIRTranslation - ${BB_COMPILER_COMMON_DIR}/Target/LLVMIR/Dialect/Buckyball/BuckyballToLLVMIRTranslation.cpp - - DEPENDS - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballConversionsIncGen - intrinsics_gen - - LINK_COMPONENTS - Core - - LINK_LIBS PUBLIC - MLIRIR - MLIRSupport - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball -) diff --git a/examples/cores/attention/compiler/src/Target/LLVMIR/Dialect/CMakeLists.txt b/examples/cores/attention/compiler/src/Target/LLVMIR/Dialect/CMakeLists.txt deleted file mode 100644 index ec181148e..000000000 --- a/examples/cores/attention/compiler/src/Target/LLVMIR/Dialect/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory(Buckyball) diff --git a/examples/cores/attention/configs/balldomains/default.toml b/examples/cores/attention/configs/balldomains/default.toml deleted file mode 100644 index f9796a850..000000000 --- a/examples/cores/attention/configs/balldomains/default.toml +++ /dev/null @@ -1,23 +0,0 @@ -# Attention Ball Domain Configuration - -ballNum = 5 - -ballIdMappings = [ - { ballId = 0, ballName = "MxmmBall", ballClass = "examples.balls.mxmm.MxmmBall", config = "../../../../balls/mxmm/configs/default.toml", inBW = 2, outBW = 1 }, - { ballId = 1, ballName = "ToInt8Ball", ballClass = "examples.balls.toint8.ToInt8Ball", config = "../../../../balls/toint8/configs/default.toml", inBW = 1, outBW = 1 }, - { ballId = 2, ballName = "Int2FpBall", ballClass = "examples.balls.int2fp.Int2FpBall", config = "../../../../balls/int2fp/configs/default.toml", inBW = 2, outBW = 1 }, - { ballId = 3, ballName = "TransposeBall", ballClass = "examples.balls.transpose.TransposeBall", config = "../../../../balls/transpose/configs/default.toml", inBW = 1, outBW = 1 }, - { ballId = 4, ballName = "MxquantBall", ballClass = "examples.balls.mxquant.MxquantBall", config = "../../../../balls/mxquant/configs/default.toml", inBW = 1, outBW = 1 } -] - -ballISA = [ - { mnemonic = "MXMM_MXFP8", funct7 = 71, bid = 0 }, - { mnemonic = "MXMM_MXFP8_WINDOW", funct7 = 75, bid = 0 }, - { mnemonic = "MXMM_FMA32", funct7 = 72, bid = 0 }, - { mnemonic = "MXMM_F32", funct7 = 73, bid = 0 }, - { mnemonic = "QUANT_F32_TO_I8", funct7 = 51, bid = 1 }, - { mnemonic = "QUANT_I32_TO_I8", funct7 = 67, bid = 1 }, - { mnemonic = "INT32_TO_FP32", funct7 = 68, bid = 2 }, - { mnemonic = "TRANSPOSE", funct7 = 49, bid = 3 }, - { mnemonic = "MXQUANT", funct7 = 54, bid = 4 } -] diff --git a/examples/cores/attention/configs/cpu/default.toml b/examples/cores/attention/configs/cpu/default.toml deleted file mode 100644 index 9cf6bf076..000000000 --- a/examples/cores/attention/configs/cpu/default.toml +++ /dev/null @@ -1,2 +0,0 @@ -kind = "ant" -config = "../../../ant/configs/cpu/default.toml" diff --git a/examples/cores/attention/configs/default.toml b/examples/cores/attention/configs/default.toml deleted file mode 100644 index 582c4a554..000000000 --- a/examples/cores/attention/configs/default.toml +++ /dev/null @@ -1,8 +0,0 @@ -factory = "examples.cores.attention.AttentionCore" -# Attention Core Configuration (default) - -balldomain = "balldomains/default.toml" -memdomain = "memdomains/default.toml" -frontend = "frontends/default.toml" -rvv = "rvv/default.toml" -cpu = "cpu/default.toml" diff --git a/examples/cores/attention/configs/frontends/default.toml b/examples/cores/attention/configs/frontends/default.toml deleted file mode 100644 index 418994d6b..000000000 --- a/examples/cores/attention/configs/frontends/default.toml +++ /dev/null @@ -1,8 +0,0 @@ -robEntries = 16 -rsOutOfOrderResponse = true -bankIdLen = 10 -vbankIdUpperBound = 31 -sharedBankIdBase = 32 -iterLen = 34 -subRobEnable = false -subRobDepth = 64 diff --git a/examples/cores/attention/configs/memdomains/default.toml b/examples/cores/attention/configs/memdomains/default.toml deleted file mode 100644 index d128b5e50..000000000 --- a/examples/cores/attention/configs/memdomains/default.toml +++ /dev/null @@ -1,32 +0,0 @@ -# Attention Core Memory Domain Configuration (default) -# Private (per-core) memory parameters. Shared memory lives in the tile TOML. - -[bank] -num = 20 -width = 128 -entries = 1024 -maskLen = 16 -channel = 7 - -[dma] -nXacts = 8 -burstMaxBytes = 1024 -busWidth = 128 -maxInFlightMemReqs = 16 - -[tlb] -size = 4 - -[tma] -readChannel = 4 -writeChannel = 4 - -[mmio] -enable = false -bankNum = 16 -bankEntries = 64 -bankWidth = 128 -readWidth = 8 - -[mem] -addrLen = 39 diff --git a/examples/cores/boom/arch/src/main/scala/configs/BoomCpuParam.scala b/examples/cores/boom/arch/src/main/scala/configs/BoomCpuParam.scala deleted file mode 100644 index 32fd57a0f..000000000 --- a/examples/cores/boom/arch/src/main/scala/configs/BoomCpuParam.scala +++ /dev/null @@ -1,32 +0,0 @@ -package framework.system.core.boom.configs - -import upickle.default._ - -case class BoomDCacheParam( - nSets: Int = 64, - nWays: Int = 4, - nMSHRs: Int = 2) - -object BoomDCacheParam { - implicit val rw: ReadWriter[BoomDCacheParam] = macroRW -} - -case class BoomICacheParam( - nSets: Int = 64, - nWays: Int = 4) - -object BoomICacheParam { - implicit val rw: ReadWriter[BoomICacheParam] = macroRW -} - -/** BOOM configuration from the chip PB. The explicit Tile does not instantiate BOOM yet. */ -case class BoomCpuParam( - fetchWidth: Int = 4, - decodeWidth: Int = 1, - numRobEntries: Int = 32, - dcache: BoomDCacheParam = BoomDCacheParam(), - icache: BoomICacheParam = BoomICacheParam()) - -object BoomCpuParam { - implicit val rw: ReadWriter[BoomCpuParam] = macroRW -} diff --git a/examples/cores/boom/configs/balldomains/default.toml b/examples/cores/boom/configs/balldomains/default.toml deleted file mode 100644 index a2733b93c..000000000 --- a/examples/cores/boom/configs/balldomains/default.toml +++ /dev/null @@ -1,9 +0,0 @@ -# Boom-only core: no Buckyball balls. - -ballNum = 0 - -ballIdMappings = [ -] - -ballISA = [ -] diff --git a/examples/cores/boom/configs/cpu/boom/default.toml b/examples/cores/boom/configs/cpu/boom/default.toml deleted file mode 100644 index 51c752071..000000000 --- a/examples/cores/boom/configs/cpu/boom/default.toml +++ /dev/null @@ -1,12 +0,0 @@ -fetchWidth = 4 -decodeWidth = 1 -numRobEntries = 32 - -[dcache] -nSets = 64 -nWays = 4 -nMSHRs = 2 - -[icache] -nSets = 64 -nWays = 4 diff --git a/examples/cores/boom/configs/cpu/default.toml b/examples/cores/boom/configs/cpu/default.toml deleted file mode 100644 index 40b4f000e..000000000 --- a/examples/cores/boom/configs/cpu/default.toml +++ /dev/null @@ -1,2 +0,0 @@ -kind = "boom" -config = "./boom/default.toml" diff --git a/examples/cores/boom/configs/default.toml b/examples/cores/boom/configs/default.toml deleted file mode 100644 index c691e6a4a..000000000 --- a/examples/cores/boom/configs/default.toml +++ /dev/null @@ -1,5 +0,0 @@ -# Boom-only core configuration - -memdomain = "memdomains/default.toml" -balldomain = "balldomains/default.toml" -cpu = "cpu/default.toml" diff --git a/examples/cores/boom/configs/memdomains/default.toml b/examples/cores/boom/configs/memdomains/default.toml deleted file mode 100644 index 67f5570ea..000000000 --- a/examples/cores/boom/configs/memdomains/default.toml +++ /dev/null @@ -1,31 +0,0 @@ -# Boom-only stub memdomain for bank_params / chip select pairing. - -[bank] -num = 16 -width = 128 -entries = 16 -maskLen = 16 -channel = 1 - -[dma] -nXacts = 1 -burstMaxBytes = 1024 -busWidth = 128 -maxInFlightMemReqs = 16 - -[tlb] -size = 1 - -[tma] -readChannel = 1 -writeChannel = 1 - -[mmio] -enable = false -bankNum = 1 -bankEntries = 1 -bankWidth = 128 -readWidth = 8 - -[mem] -addrLen = 39 diff --git a/examples/cores/ffn/arch/src/main/scala/FfnCore.scala b/examples/cores/ffn/arch/src/main/scala/FfnCore.scala deleted file mode 100644 index b2e9273e7..000000000 --- a/examples/cores/ffn/arch/src/main/scala/FfnCore.scala +++ /dev/null @@ -1,19 +0,0 @@ -package examples.cores.ffn - -import framework.top.GlobalConfig -import chisel3.experimental.hierarchy.{instantiable, Instance, Instantiate} -import framework.ant.LocalExecution -import framework.system.core.{AntEndpoint, CoreConnection} -import framework.system.core.accelerator.{AntAdmission, BuckyballAccelerator} - -@instantiable -class FfnCore(config: GlobalConfig) extends AntEndpoint(config) { - val execution = Instantiate(new LocalExecution(local)) - val accelerator = Instantiate(new BuckyballAccelerator(p.buckyball)) - val issuer = Instantiate(new AntAdmission(local, tracking, cpuParams.core.nPMPs, bus)) - CoreConnection.ant(clink, execution, accelerator, issuer, signature) -} - -object FfnCore extends framework.system.core.AntCoreFactory { - def instantiate(b: GlobalConfig): Instance[AntEndpoint] = Instantiate(new FfnCore(b)) -} diff --git a/examples/cores/ffn/compiler/CMakeLists.txt b/examples/cores/ffn/compiler/CMakeLists.txt deleted file mode 100644 index 4f4cd14c7..000000000 --- a/examples/cores/ffn/compiler/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -if(NOT COMMAND add_mlir_library OR NOT COMMAND add_mlir_dialect) - message(FATAL_ERROR "MLIR CMake helpers must be loaded. Build as part of buddy-mlir.") -endif() - -set(BB_CORE_NAME ffn) -set(BB_CORE_TARGET_PREFIX "") - -set(BB_CORE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/..) -get_filename_component(_BB_REPO_ROOT "${BB_CORE_DIR}/../../.." ABSOLUTE) -include_directories(${_BB_REPO_ROOT}/examples/balls) - -add_subdirectory(src) diff --git a/examples/cores/ffn/compiler/src/CMakeLists.txt b/examples/cores/ffn/compiler/src/CMakeLists.txt deleted file mode 100644 index 49dfefaae..000000000 --- a/examples/cores/ffn/compiler/src/CMakeLists.txt +++ /dev/null @@ -1,51 +0,0 @@ -if(NOT COMMAND add_mlir_library OR NOT COMMAND add_mlir_dialect) - message(FATAL_ERROR "MLIR CMake helpers must be loaded. Build as part of buddy-mlir.") -endif() - -message(STATUS "Building FFN compiler Core package") -message(STATUS " Source: ${CMAKE_CURRENT_SOURCE_DIR}") -message(STATUS " Binary: ${CMAKE_CURRENT_BINARY_DIR}") - -get_filename_component(BB_REPO_ROOT ${CMAKE_CURRENT_LIST_DIR}/../../../../.. ABSOLUTE) -set(BB_COMPILER_COMMON_DIR ${BB_REPO_ROOT}/stack/compiler/src) - -if(NOT EXISTS ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball/Buckyball.td) - message(FATAL_ERROR "Missing Buckyball compiler common dir: ${BB_COMPILER_COMMON_DIR}") -endif() - -include_directories(${BB_COMPILER_COMMON_DIR}) -include_directories(${BB_COMPILER_COMMON_DIR}/Dialect) -include_directories(${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}) -include_directories(${CMAKE_CURRENT_BINARY_DIR}) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/Dialect) -include_directories(${CMAKE_CURRENT_BINARY_DIR}/Dialect) -include_directories(BEFORE ${CMAKE_CURRENT_BINARY_DIR}/Dialect/Buckyball) - -set(FFN_BALL_COMPILER_DIALECT_DIRS - ${BB_REPO_ROOT}/examples/balls/toint8/compiler/src/Dialect/Buckyball - ${BB_REPO_ROOT}/examples/balls/int2fp/compiler/src/Dialect/Buckyball - ${BB_REPO_ROOT}/examples/balls/transpose/compiler/src/Dialect/Buckyball - ${BB_REPO_ROOT}/examples/balls/smatmul/compiler/src/Dialect/Buckyball -) - -foreach(FFN_BALL_COMPILER_DIALECT_DIR ${FFN_BALL_COMPILER_DIALECT_DIRS}) - if(NOT EXISTS ${FFN_BALL_COMPILER_DIALECT_DIR}) - message(FATAL_ERROR - "Missing ffn ball compiler dialect dir: ${FFN_BALL_COMPILER_DIALECT_DIR}") - endif() - include_directories(${FFN_BALL_COMPILER_DIALECT_DIR}) -endforeach() - -add_subdirectory(Dialect) -add_subdirectory(Conversion) -add_subdirectory(Target) - -add_custom_target(buckyball-compiler-${BB_CORE_NAME} - DEPENDS - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballTransforms - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballToLLVMIRTranslation - ${BB_CORE_TARGET_PREFIX}LowerBuckyballPass - ${BB_CORE_TARGET_PREFIX}LowerTileToBuckyballPass -) diff --git a/examples/cores/ffn/compiler/src/Conversion/CMakeLists.txt b/examples/cores/ffn/compiler/src/Conversion/CMakeLists.txt deleted file mode 100644 index 9ca69b1a2..000000000 --- a/examples/cores/ffn/compiler/src/Conversion/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -add_subdirectory(LowerBuckyball) -add_subdirectory(LowerTileToBuckyball) diff --git a/examples/cores/ffn/compiler/src/Conversion/LowerBuckyball/CMakeLists.txt b/examples/cores/ffn/compiler/src/Conversion/LowerBuckyball/CMakeLists.txt deleted file mode 100644 index eb782edc6..000000000 --- a/examples/cores/ffn/compiler/src/Conversion/LowerBuckyball/CMakeLists.txt +++ /dev/null @@ -1,16 +0,0 @@ -add_mlir_library(${BB_CORE_TARGET_PREFIX}LowerBuckyballPass - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/BaseAssignPhysicalBankPatterns.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/PhysicalBankState.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/ReportBankUsagePass.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/AssignPhysicalBanksPass.cpp - ${BB_assign_SOURCES} - ${BB_bank_ssa_SOURCES} - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/QuantizeTensorToBankSSAPatterns.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/RuntimeLowering.cpp - LowerBuckyballPass.cpp - LowerBuckyballToBankSSAPass.cpp - - LINK_LIBS PUBLIC - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballTransforms - MLIRLinalgDialect -) diff --git a/examples/cores/ffn/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp b/examples/cores/ffn/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp deleted file mode 100644 index 0a3f1e218..000000000 --- a/examples/cores/ffn/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp +++ /dev/null @@ -1,126 +0,0 @@ -//===- LowerBuckyballPass.cpp - FFN Buckyball lowering pass -------------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/TypeConverter.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/Pass/Pass.h" - -#include "Buckyball/BuckyballDialect.h" -#include "Buckyball/BuckyballOps.h" -#include "Buckyball/Transform.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { - -class LowerBuckyballToLLVMPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerBuckyballToLLVMPass) - LowerBuckyballToLLVMPass() = default; - LowerBuckyballToLLVMPass(const LowerBuckyballToLLVMPass &) {} - - StringRef getArgument() const final { return "lower-buckyball"; } - StringRef getDescription() const final { - return "Lower FFN Buckyball dialect ops."; - } - - Option stable{*this, "stable", - llvm::cl::desc("Use stable LLVM Buckyball intrinsics."), - llvm::cl::init(false)}; - - void getDependentDialects(DialectRegistry ®istry) const override { - registry - .insert(); - } - - void runOnOperation() override { - const auto &targetConfig = buckyball_target::getBuckyballTarget(); - MLIRContext *context = &getContext(); - ModuleOp module = getOperation(); - LLVMTypeConverter converter(context); - RewritePatternSet patterns(context); - LLVMConversionTarget target(*context); - - configureBuckyballLegalizeForExportTarget(target, stable); - target.addLegalDialect(); - populateBuckyballLegalizeForLLVMExportPatterns( - converter, patterns, targetConfig.bankWidthBits / 8, - targetConfig.bankDepth, targetConfig.bankNum, - /*includeFuncOperandForwarding=*/false, stable); - - ConversionConfig config; - config.allowPatternRollback = false; - if (failed(applyPartialConversion(module, target, std::move(patterns), - config))) - signalPassFailure(); - } -}; - -class LowerBankSSAToIntrinsicsPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerBankSSAToIntrinsicsPass) - LowerBankSSAToIntrinsicsPass() = default; - LowerBankSSAToIntrinsicsPass(const LowerBankSSAToIntrinsicsPass &) {} - - StringRef getArgument() const final { return "lower-bank-ssa-to-intrinsics"; } - StringRef getDescription() const final { - return "Lower FFN bank-SSA and Buckyball ops to intrinsic ops."; - } - - Option stable{*this, "stable", - llvm::cl::desc("Use stable LLVM Buckyball intrinsics."), - llvm::cl::init(false)}; - - void getDependentDialects(DialectRegistry ®istry) const override { - registry - .insert(); - } - - void runOnOperation() override { - const auto &targetConfig = buckyball_target::getBuckyballTarget(); - MLIRContext *context = &getContext(); - ModuleOp module = getOperation(); - LLVMTypeConverter converter(context); - RewritePatternSet patterns(context); - LLVMConversionTarget target(*context); - - configureBuckyballLegalizeForExportTarget(target, stable); - target.addLegalDialect(); - populateBuckyballLegalizeForLLVMExportPatterns( - converter, patterns, targetConfig.bankWidthBits / 8, - targetConfig.bankDepth, targetConfig.bankNum, - /*includeFuncOperandForwarding=*/false, stable); - - ConversionConfig config; - config.allowPatternRollback = false; - if (failed(applyPartialConversion(module, target, std::move(patterns), - config))) - signalPassFailure(); - } -}; - -} // namespace - -void mlir::buddy::registerLowerBuckyballPass() { - registerTileRuntimePass(); - PassRegistration(); -} - -void mlir::buddy::registerLowerBankSSAToIntrinsicsPass() { - PassRegistration(); -} diff --git a/examples/cores/ffn/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp b/examples/cores/ffn/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp deleted file mode 100644 index 9450ff9b5..000000000 --- a/examples/cores/ffn/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp +++ /dev/null @@ -1,108 +0,0 @@ -//===- LowerBuckyballToBankSSAPass.cpp - Buckyball bank-SSA lowering -//----------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" -#include "Target/BuckyballTargetRegistry.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/Dialect/Utils/StructuredOpsUtils.h" -#include "mlir/IR/AffineMap.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/PatternMatch.h" -#include "mlir/Pass/Pass.h" -#include "mlir/Transforms/GreedyPatternRewriteDriver.h" - -#include "Buckyball/BuckyballDialect.h" -#include "Buckyball/BuckyballOps.h" -#include "Trace/TraceDialect.h" - -using namespace mlir; - -namespace mlir::buddy { -#define BUCKYBALL_BANK_SSA_HOOK(BALL) \ - void populate##BALL##LowerBuckyballToBankSSAPatterns(RewritePatternSet &); -#include "BuckyballBallLoweringHooks.inc" -#undef BUCKYBALL_BANK_SSA_HOOK -} // namespace mlir::buddy - -namespace { - -class MemTransposeToLinalgPattern - : public OpRewritePattern<::buddy::buckyball::MemTransposeOp> { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(::buddy::buckyball::MemTransposeOp op, - PatternRewriter &rewriter) const override { - auto input = dyn_cast(op.getInput().getType()); - auto output = dyn_cast(op.getOutput().getType()); - if (!input || !output || !input.hasStaticShape() || - !output.hasStaticShape() || input.getRank() != 2 || - output.getRank() != 2 || output.getShape()[0] != input.getShape()[1] || - output.getShape()[1] != input.getShape()[0]) - return op.emitError("requires matching static rank-2 transpose memrefs"); - - MLIRContext *context = rewriter.getContext(); - AffineMap identity = AffineMap::getMultiDimIdentityMap(2, context); - AffineMap transpose = AffineMap::get( - 2, 0, {rewriter.getAffineDimExpr(1), rewriter.getAffineDimExpr(0)}, - context); - SmallVector iterators(2, - utils::IteratorType::parallel); - rewriter.create( - op.getLoc(), TypeRange{}, ValueRange{op.getInput()}, - ValueRange{op.getOutput()}, ArrayRef{identity, transpose}, - iterators, [](OpBuilder &builder, Location location, ValueRange args) { - builder.create(location, args[0]); - }); - rewriter.eraseOp(op); - return success(); - } -}; - -} // namespace - -namespace { - -class LowerBuckyballToBankSSAPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerBuckyballToBankSSAPass) - - StringRef getArgument() const final { return "lower-buckyball-to-bank-ssa"; } - StringRef getDescription() const final { - return "Lower Buckyball Buckyball ops to explicit bank-SSA ops."; - } - - void getDependentDialects(DialectRegistry ®istry) const override { - registry.insert<::buddy::trace::BuddyTraceDialect, arith::ArithDialect, - linalg::LinalgDialect, memref::MemRefDialect, - scf::SCFDialect, ::buddy::buckyball::BuckyballDialect>(); - } - - void runOnOperation() override { - RewritePatternSet patterns(&getContext()); - for (llvm::StringRef ball : buckyball_target::getBuckyballTarget().balls) { -#define BUCKYBALL_BANK_SSA_HOOK(BALL) \ - if (ball == #BALL) \ - mlir::buddy::populate##BALL##LowerBuckyballToBankSSAPatterns(patterns); -#include "BuckyballBallLoweringHooks.inc" -#undef BUCKYBALL_BANK_SSA_HOOK - } - mlir::buddy::populateQuantizeTensorToBankSSAPatterns(patterns); - patterns.add(&getContext()); - if (failed(applyPatternsGreedily(getOperation(), std::move(patterns)))) - signalPassFailure(); - } -}; - -} // namespace - -void mlir::buddy::registerLowerBuckyballToBankSSAPass() { - PassRegistration(); -} diff --git a/examples/cores/ffn/compiler/src/Conversion/LowerTileToBuckyball/CMakeLists.txt b/examples/cores/ffn/compiler/src/Conversion/LowerTileToBuckyball/CMakeLists.txt deleted file mode 100644 index c12d60c23..000000000 --- a/examples/cores/ffn/compiler/src/Conversion/LowerTileToBuckyball/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -add_mlir_library(${BB_CORE_TARGET_PREFIX}LowerTileToBuckyballPass - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerTileToBuckyball/QuantizedKernelPatterns.cpp - LowerTileToBuckyball.cpp - ${BB_REPO_ROOT}/examples/balls/smatmul/compiler/src/Conversion/LowerTileToBuckyball/TileMatMulPatterns.cpp - - LINK_LIBS PUBLIC - BuddyTile - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball - MLIRLinalgDialect -) diff --git a/examples/cores/ffn/compiler/src/Conversion/LowerTileToBuckyball/LowerTileToBuckyball.cpp b/examples/cores/ffn/compiler/src/Conversion/LowerTileToBuckyball/LowerTileToBuckyball.cpp deleted file mode 100644 index cca771535..000000000 --- a/examples/cores/ffn/compiler/src/Conversion/LowerTileToBuckyball/LowerTileToBuckyball.cpp +++ /dev/null @@ -1,104 +0,0 @@ -#include "Target/BuckyballTargetRegistry.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinOps.h" -#include "mlir/Pass/Pass.h" -#include "mlir/Transforms/DialectConversion.h" - -#include "Buckyball/BuckyballDialect.h" -#include "Buckyball/BuckyballOps.h" -#include "Conversion/LowerTileToBuckyball/LowerTileToBuckyball.h" -#include "Tile/TileDialect.h" -#include "Tile/TileOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; -namespace tile = ::buddy::tile; - -namespace mlir::buddy { -#define BUCKYBALL_TILE_HOOK(BALL) \ - void populate##BALL##TileLoweringPatterns(RewritePatternSet &, int64_t, \ - int64_t, int64_t); -#include "BuckyballBallLoweringHooks.inc" -#undef BUCKYBALL_TILE_HOOK -} // namespace mlir::buddy - -namespace { -class TileTransposeLowering : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(tile::TileTransposeOp op, - PatternRewriter &rewriter) const override { - auto inputType = dyn_cast(op.getAMemArray().getType()); - auto outputType = dyn_cast(op.getBMemArray().getType()); - if (!inputType || !outputType || !inputType.hasStaticShape() || - !outputType.hasStaticShape() || inputType.getRank() != 2 || - outputType.getRank() != 2) - return op.emitError("requires static rank-2 input and output memrefs"); - if (outputType.getShape()[0] != inputType.getShape()[1] || - outputType.getShape()[1] != inputType.getShape()[0] || - inputType.getElementType() != outputType.getElementType()) - return op.emitError("requires a matching transposed output memref"); - - rewriter.create(op.getLoc(), op.getAMemArray(), - op.getBMemArray()); - rewriter.eraseOp(op); - return success(); - } -}; - -class LowerTileToBuckyballPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerTileToBuckyballPass) - - StringRef getArgument() const final { return "convert-tile-to-buckyball"; } - StringRef getDescription() const final { - return "Convert Tile operations for the selected Buckyball target."; - } - - void getDependentDialects(DialectRegistry ®istry) const override { - registry - .insert(); - } - - void runOnOperation() override { - const auto &targetConfig = buckyball_target::getBuckyballTarget(); - MLIRContext *context = &getContext(); - ConversionTarget target(*context); - target.addLegalDialect<::buddy::buckyball::BuckyballDialect, - memref::MemRefDialect, arith::ArithDialect, - scf::SCFDialect, func::FuncDialect, - linalg::LinalgDialect>(); - target.addIllegalOp(); - - RewritePatternSet patterns(context); - mlir::buddy::populateQuantizedKernelTileLoweringPatterns(patterns); - patterns.add(context); - for (llvm::StringRef ball : targetConfig.balls) { -#define BUCKYBALL_TILE_HOOK(BALL) \ - if (ball == #BALL) \ - mlir::buddy::populate##BALL##TileLoweringPatterns( \ - patterns, targetConfig.bankWidthBits / 8, targetConfig.bankDepth, \ - targetConfig.bankNum); -#include "BuckyballBallLoweringHooks.inc" -#undef BUCKYBALL_TILE_HOOK - } - if (failed(applyPartialConversion(getOperation(), target, - std::move(patterns)))) - signalPassFailure(); - } -}; -} // namespace - -void mlir::buddy::registerLowerTileToBuckyballPass() { - PassRegistration(); -} diff --git a/examples/cores/ffn/compiler/src/Dialect/Buckyball/CMakeLists.txt b/examples/cores/ffn/compiler/src/Dialect/Buckyball/CMakeLists.txt deleted file mode 100644 index 50c0c0f72..000000000 --- a/examples/cores/ffn/compiler/src/Dialect/Buckyball/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -set(LLVM_TARGET_DEFINITIONS ${BB_BUCKYBALL_DIALECT_TD}) -mlir_tablegen(Buckyball.h.inc -gen-op-decls) -mlir_tablegen(Buckyball.cpp.inc -gen-op-defs) -mlir_tablegen(BuckyballTypes.h.inc -gen-typedef-decls -typedefs-dialect=buckyball) -mlir_tablegen(BuckyballTypes.cpp.inc -gen-typedef-defs -typedefs-dialect=buckyball) -mlir_tablegen(BuckyballDialect.h.inc -gen-dialect-decls -dialect=buckyball) -mlir_tablegen(BuckyballDialect.cpp.inc -gen-dialect-defs -dialect=buckyball) -add_public_tablegen_target(${BB_CORE_TARGET_PREFIX}MLIRBuckyballIncGen) -add_mlir_doc(FFNBuckyball ${BB_CORE_TARGET_PREFIX}Buckyball Dialects/ - -gen-dialect-doc) - -set(LLVM_TARGET_DEFINITIONS ${BB_BUCKYBALL_DIALECT_TD}) -mlir_tablegen(BuckyballConversions.inc -gen-llvmir-conversions) -add_public_tablegen_target(${BB_CORE_TARGET_PREFIX}BuddyBuckyballConversionsIncGen) - -add_subdirectory(IR) -add_subdirectory(Transforms) diff --git a/examples/cores/ffn/compiler/src/Dialect/Buckyball/FFNBuckyball.td b/examples/cores/ffn/compiler/src/Dialect/Buckyball/FFNBuckyball.td deleted file mode 100644 index a11c7d6e0..000000000 --- a/examples/cores/ffn/compiler/src/Dialect/Buckyball/FFNBuckyball.td +++ /dev/null @@ -1,26 +0,0 @@ -//===-- FFNBuckyball.td - FFN Buckyball dialect ops -*- tablegen -*-===// -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//===----------------------------------------------------------------------===// - -#ifndef FFN_BUCKYBALL_OPS -#define FFN_BUCKYBALL_OPS - -include "Buckyball.td" -include "ToInt8Ball.td" -include "Int2FpBall.td" -include "TransposeBall.td" -include "MxmmBall.td" - -#endif // FFN_BUCKYBALL_OPS diff --git a/examples/cores/ffn/compiler/src/Dialect/Buckyball/IR/CMakeLists.txt b/examples/cores/ffn/compiler/src/Dialect/Buckyball/IR/CMakeLists.txt deleted file mode 100644 index 42c7f8807..000000000 --- a/examples/cores/ffn/compiler/src/Dialect/Buckyball/IR/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -add_mlir_dialect_library(${BB_CORE_TARGET_PREFIX}BuddyBuckyball - ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball/IR/BuckyballDialect.cpp - ${BB_TARGET_REGISTRY_SOURCE} - ${BB_BALL_IR_SOURCES} - - LINK_LIBS PUBLIC - MLIRIR - MLIRLLVMDialect - MLIRBufferizationDialect - MLIRMemRefDialect -) - -target_include_directories(${BB_CORE_TARGET_PREFIX}BuddyBuckyball PUBLIC - ${BB_COMPILER_COMMON_DIR} - ${BB_COMPILER_COMMON_DIR}/Dialect - ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball - ${CMAKE_CURRENT_BINARY_DIR}/.. - ${CMAKE_CURRENT_BINARY_DIR}/../../.. -) diff --git a/examples/cores/ffn/compiler/src/Dialect/Buckyball/Transforms/CMakeLists.txt b/examples/cores/ffn/compiler/src/Dialect/Buckyball/Transforms/CMakeLists.txt deleted file mode 100644 index 0d2cecf2e..000000000 --- a/examples/cores/ffn/compiler/src/Dialect/Buckyball/Transforms/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -add_mlir_dialect_library(${BB_CORE_TARGET_PREFIX}BuddyBuckyballTransforms - ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.cpp - ${BB_REPO_ROOT}/examples/balls/toint8/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - ${BB_REPO_ROOT}/examples/balls/int2fp/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - ${BB_REPO_ROOT}/examples/balls/transpose/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - ${BB_REPO_ROOT}/examples/balls/smatmul/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - LegalizeForLLVMExport.cpp - - LINK_LIBS PUBLIC - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball - MLIRIR - MLIRLLVMCommonConversion - MLIRLLVMDialect -) diff --git a/examples/cores/ffn/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/cores/ffn/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 0b1a5b224..000000000 --- a/examples/cores/ffn/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,45 +0,0 @@ -#include "Buckyball/Transform.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace buddy::buckyball::legalize; - -namespace mlir::buddy::buckyball { -#define BUCKYBALL_LEGALIZE_HOOK(BALL) \ - void populate##BALL##LegalizeForLLVMExportPatterns( \ - LLVMTypeConverter &, RewritePatternSet &, bool, int64_t); \ - void configure##BALL##LegalizeForExportTarget(LLVMConversionTarget &, bool); -#include "BuckyballBallLoweringHooks.inc" -#undef BUCKYBALL_LEGALIZE_HOOK -} // namespace mlir::buddy::buckyball - -void mlir::populateBuckyballLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, - int64_t bankWidthBytes, int64_t bankDepth, int64_t bankNum, - bool includeFuncOperandForwarding, bool stable) { - populateBaseLegalizeForLLVMExportPatterns(converter, patterns, - includeFuncOperandForwarding); - for (llvm::StringRef ball : buckyball_target::getBuckyballTarget().balls) { -#define BUCKYBALL_LEGALIZE_HOOK(BALL) \ - if (ball == #BALL) \ - buddy::buckyball::populate##BALL##LegalizeForLLVMExportPatterns( \ - converter, patterns, stable, bankDepth); -#include "BuckyballBallLoweringHooks.inc" -#undef BUCKYBALL_LEGALIZE_HOOK - } - (void)bankWidthBytes; - (void)bankNum; -} - -void mlir::configureBuckyballLegalizeForExportTarget( - LLVMConversionTarget &target, bool stable) { - configureBaseLegalizeForExportTarget(target); - for (llvm::StringRef ball : buckyball_target::getBuckyballTarget().balls) { -#define BUCKYBALL_LEGALIZE_HOOK(BALL) \ - if (ball == #BALL) \ - buddy::buckyball::configure##BALL##LegalizeForExportTarget(target, stable); -#include "BuckyballBallLoweringHooks.inc" -#undef BUCKYBALL_LEGALIZE_HOOK - } -} diff --git a/examples/cores/ffn/compiler/src/Dialect/CMakeLists.txt b/examples/cores/ffn/compiler/src/Dialect/CMakeLists.txt deleted file mode 100644 index ec181148e..000000000 --- a/examples/cores/ffn/compiler/src/Dialect/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory(Buckyball) diff --git a/examples/cores/ffn/compiler/src/Target/CMakeLists.txt b/examples/cores/ffn/compiler/src/Target/CMakeLists.txt deleted file mode 100644 index 39d31dc9b..000000000 --- a/examples/cores/ffn/compiler/src/Target/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory(LLVMIR) diff --git a/examples/cores/ffn/compiler/src/Target/LLVMIR/CMakeLists.txt b/examples/cores/ffn/compiler/src/Target/LLVMIR/CMakeLists.txt deleted file mode 100644 index 0ca0f41c5..000000000 --- a/examples/cores/ffn/compiler/src/Target/LLVMIR/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory(Dialect) diff --git a/examples/cores/ffn/compiler/src/Target/LLVMIR/Dialect/Buckyball/CMakeLists.txt b/examples/cores/ffn/compiler/src/Target/LLVMIR/Dialect/Buckyball/CMakeLists.txt deleted file mode 100644 index 06b99171b..000000000 --- a/examples/cores/ffn/compiler/src/Target/LLVMIR/Dialect/Buckyball/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -add_mlir_translation_library(${BB_CORE_TARGET_PREFIX}BuddyBuckyballToLLVMIRTranslation - ${BB_COMPILER_COMMON_DIR}/Target/LLVMIR/Dialect/Buckyball/BuckyballToLLVMIRTranslation.cpp - - DEPENDS - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballConversionsIncGen - intrinsics_gen - - LINK_COMPONENTS - Core - - LINK_LIBS PUBLIC - MLIRIR - MLIRSupport - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball -) diff --git a/examples/cores/ffn/compiler/src/Target/LLVMIR/Dialect/CMakeLists.txt b/examples/cores/ffn/compiler/src/Target/LLVMIR/Dialect/CMakeLists.txt deleted file mode 100644 index ec181148e..000000000 --- a/examples/cores/ffn/compiler/src/Target/LLVMIR/Dialect/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory(Buckyball) diff --git a/examples/cores/ffn/configs/balldomains/default.toml b/examples/cores/ffn/configs/balldomains/default.toml deleted file mode 100644 index b64240389..000000000 --- a/examples/cores/ffn/configs/balldomains/default.toml +++ /dev/null @@ -1,23 +0,0 @@ -# FFN Ball Domain Configuration - -ballNum = 5 - -ballIdMappings = [ - { ballId = 0, ballName = "MxmmBall", ballClass = "examples.balls.mxmm.MxmmBall", config = "../../../../balls/mxmm/configs/default.toml", inBW = 2, outBW = 1 }, - { ballId = 1, ballName = "ToInt8Ball", ballClass = "examples.balls.toint8.ToInt8Ball", config = "../../../../balls/toint8/configs/default.toml", inBW = 1, outBW = 1 }, - { ballId = 2, ballName = "Int2FpBall", ballClass = "examples.balls.int2fp.Int2FpBall", config = "../../../../balls/int2fp/configs/default.toml", inBW = 2, outBW = 1 }, - { ballId = 3, ballName = "TransposeBall", ballClass = "examples.balls.transpose.TransposeBall", config = "../../../../balls/transpose/configs/default.toml", inBW = 1, outBW = 1 }, - { ballId = 4, ballName = "MxquantBall", ballClass = "examples.balls.mxquant.MxquantBall", config = "../../../../balls/mxquant/configs/default.toml", inBW = 1, outBW = 1 } -] - -ballISA = [ - { mnemonic = "MXMM_MXFP8", funct7 = 71, bid = 0 }, - { mnemonic = "MXMM_MXFP8_WINDOW", funct7 = 75, bid = 0 }, - { mnemonic = "MXMM_FMA32", funct7 = 72, bid = 0 }, - { mnemonic = "MXMM_F32", funct7 = 73, bid = 0 }, - { mnemonic = "QUANT_F32_TO_I8", funct7 = 51, bid = 1 }, - { mnemonic = "QUANT_I32_TO_I8", funct7 = 67, bid = 1 }, - { mnemonic = "INT32_TO_FP32", funct7 = 68, bid = 2 }, - { mnemonic = "TRANSPOSE", funct7 = 49, bid = 3 }, - { mnemonic = "MXQUANT", funct7 = 54, bid = 4 } -] diff --git a/examples/cores/ffn/configs/cpu/default.toml b/examples/cores/ffn/configs/cpu/default.toml deleted file mode 100644 index 9cf6bf076..000000000 --- a/examples/cores/ffn/configs/cpu/default.toml +++ /dev/null @@ -1,2 +0,0 @@ -kind = "ant" -config = "../../../ant/configs/cpu/default.toml" diff --git a/examples/cores/ffn/configs/default.toml b/examples/cores/ffn/configs/default.toml deleted file mode 100644 index 60331493c..000000000 --- a/examples/cores/ffn/configs/default.toml +++ /dev/null @@ -1,8 +0,0 @@ -factory = "examples.cores.ffn.FfnCore" -# FFN Core Configuration (default) - -balldomain = "balldomains/default.toml" -memdomain = "memdomains/default.toml" -frontend = "frontends/default.toml" -rvv = "rvv/default.toml" -cpu = "cpu/default.toml" diff --git a/examples/cores/ffn/configs/frontends/default.toml b/examples/cores/ffn/configs/frontends/default.toml deleted file mode 100644 index 418994d6b..000000000 --- a/examples/cores/ffn/configs/frontends/default.toml +++ /dev/null @@ -1,8 +0,0 @@ -robEntries = 16 -rsOutOfOrderResponse = true -bankIdLen = 10 -vbankIdUpperBound = 31 -sharedBankIdBase = 32 -iterLen = 34 -subRobEnable = false -subRobDepth = 64 diff --git a/examples/cores/ffn/configs/memdomains/default.toml b/examples/cores/ffn/configs/memdomains/default.toml deleted file mode 100644 index 1b39694e8..000000000 --- a/examples/cores/ffn/configs/memdomains/default.toml +++ /dev/null @@ -1,32 +0,0 @@ -# FFN Core Memory Domain Configuration (default) -# Private (per-core) memory parameters. Shared memory lives in the tile TOML. - -[bank] -num = 32 -width = 128 -entries = 2048 -maskLen = 16 -channel = 7 - -[dma] -nXacts = 8 -burstMaxBytes = 1024 -busWidth = 128 -maxInFlightMemReqs = 16 - -[tlb] -size = 4 - -[tma] -readChannel = 4 -writeChannel = 4 - -[mmio] -enable = false -bankNum = 16 -bankEntries = 64 -bankWidth = 128 -readWidth = 8 - -[mem] -addrLen = 39 diff --git a/examples/cores/goban/arch/src/main/scala/AntGobanCore.scala b/examples/cores/goban/arch/src/main/scala/AntGobanCore.scala deleted file mode 100644 index 6d521027f..000000000 --- a/examples/cores/goban/arch/src/main/scala/AntGobanCore.scala +++ /dev/null @@ -1,19 +0,0 @@ -package examples.cores.goban - -import framework.top.GlobalConfig -import chisel3.experimental.hierarchy.{instantiable, Instance, Instantiate} -import framework.ant.LocalExecution -import framework.system.core.{AntEndpoint, CoreConnection} -import framework.system.core.accelerator.{AntAdmission, BuckyballAccelerator} - -@instantiable -class AntGobanCore(config: GlobalConfig) extends AntEndpoint(config) { - val execution = Instantiate(new LocalExecution(local)) - val accelerator = Instantiate(new BuckyballAccelerator(p.buckyball)) - val issuer = Instantiate(new AntAdmission(local, tracking, cpuParams.core.nPMPs, bus)) - CoreConnection.ant(clink, execution, accelerator, issuer, signature) -} - -object AntGobanCore extends framework.system.core.AntCoreFactory { - def instantiate(b: GlobalConfig): Instance[AntEndpoint] = Instantiate(new AntGobanCore(b)) -} diff --git a/examples/cores/goban/arch/src/main/scala/GobanCore.scala b/examples/cores/goban/arch/src/main/scala/GobanCore.scala deleted file mode 100644 index b2de78793..000000000 --- a/examples/cores/goban/arch/src/main/scala/GobanCore.scala +++ /dev/null @@ -1,20 +0,0 @@ -package examples.cores.goban - -import framework.top.GlobalConfig -import chisel3.experimental.hierarchy.{instantiable, Instance, Instantiate} -import framework.system.core.{CoreConnection, RocketEndpoint} -import hier.core.rocket.Core -import framework.system.core.accelerator.BuckyballAccelerator - -@instantiable -class GobanCore(b: GlobalConfig) extends RocketEndpoint(b) { - require(p.buckyball.nonEmpty) - val cpu = Instantiate(new Core(p.data, p.instruction, p.regions, Some(p.commands))(p.cpu)) - clink.cpu <> cpu.clink - val accelerator = Instantiate(new BuckyballAccelerator(p.buckyball.get)) - CoreConnection.accelerator(clink.accelerator.get, accelerator) -} - -object GobanCore extends framework.system.core.RocketCoreFactory { - def instantiate(b: GlobalConfig): Instance[RocketEndpoint] = Instantiate(new GobanCore(b)) -} diff --git a/examples/cores/goban/compiler/CMakeLists.txt b/examples/cores/goban/compiler/CMakeLists.txt deleted file mode 100644 index cf5f19d4c..000000000 --- a/examples/cores/goban/compiler/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -if(NOT COMMAND add_mlir_library OR NOT COMMAND add_mlir_dialect) - message(FATAL_ERROR "MLIR CMake helpers must be loaded. Build as part of buddy-mlir.") -endif() - -set(BB_CORE_NAME goban) -set(BB_CORE_TARGET_PREFIX "") - -set(BB_CORE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/..) -get_filename_component(_BB_REPO_ROOT "${BB_CORE_DIR}/../../.." ABSOLUTE) -include_directories(${_BB_REPO_ROOT}/examples/balls) - -add_subdirectory(src) diff --git a/examples/cores/goban/compiler/src/CMakeLists.txt b/examples/cores/goban/compiler/src/CMakeLists.txt deleted file mode 100644 index 561efedb1..000000000 --- a/examples/cores/goban/compiler/src/CMakeLists.txt +++ /dev/null @@ -1,51 +0,0 @@ -if(NOT COMMAND add_mlir_library OR NOT COMMAND add_mlir_dialect) - message(FATAL_ERROR "MLIR CMake helpers must be loaded. Build as part of buddy-mlir.") -endif() - -message(STATUS "Building Goban compiler Core package") -message(STATUS " Source: ${CMAKE_CURRENT_SOURCE_DIR}") -message(STATUS " Binary: ${CMAKE_CURRENT_BINARY_DIR}") - -get_filename_component(BB_REPO_ROOT ${CMAKE_CURRENT_LIST_DIR}/../../../../.. ABSOLUTE) -set(BB_COMPILER_COMMON_DIR ${BB_REPO_ROOT}/stack/compiler/src) - -if(NOT EXISTS ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball/Buckyball.td) - message(FATAL_ERROR "Missing Buckyball compiler common dir: ${BB_COMPILER_COMMON_DIR}") -endif() - -include_directories(${BB_COMPILER_COMMON_DIR}) -include_directories(${BB_COMPILER_COMMON_DIR}/Dialect) -include_directories(${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}) -include_directories(${CMAKE_CURRENT_BINARY_DIR}) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/Dialect) -include_directories(${CMAKE_CURRENT_BINARY_DIR}/Dialect) -include_directories(BEFORE ${CMAKE_CURRENT_BINARY_DIR}/Dialect/Buckyball) - -set(GOBAN_BALL_COMPILER_DIALECT_DIRS - ${BB_REPO_ROOT}/examples/balls/toint8/compiler/src/Dialect/Buckyball - ${BB_REPO_ROOT}/examples/balls/int2fp/compiler/src/Dialect/Buckyball - ${BB_REPO_ROOT}/examples/balls/transpose/compiler/src/Dialect/Buckyball - ${BB_REPO_ROOT}/examples/balls/smatmul/compiler/src/Dialect/Buckyball -) - -foreach(GOBAN_BALL_COMPILER_DIALECT_DIR ${GOBAN_BALL_COMPILER_DIALECT_DIRS}) - if(NOT EXISTS ${GOBAN_BALL_COMPILER_DIALECT_DIR}) - message(FATAL_ERROR - "Missing goban ball compiler dialect dir: ${GOBAN_BALL_COMPILER_DIALECT_DIR}") - endif() - include_directories(${GOBAN_BALL_COMPILER_DIALECT_DIR}) -endforeach() - -add_subdirectory(Dialect) -add_subdirectory(Conversion) -add_subdirectory(Target) - -add_custom_target(buckyball-compiler-${BB_CORE_NAME} - DEPENDS - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballTransforms - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballToLLVMIRTranslation - ${BB_CORE_TARGET_PREFIX}LowerBuckyballPass - ${BB_CORE_TARGET_PREFIX}LowerTileToBuckyballPass -) diff --git a/examples/cores/goban/compiler/src/Conversion/CMakeLists.txt b/examples/cores/goban/compiler/src/Conversion/CMakeLists.txt deleted file mode 100644 index 9ca69b1a2..000000000 --- a/examples/cores/goban/compiler/src/Conversion/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -add_subdirectory(LowerBuckyball) -add_subdirectory(LowerTileToBuckyball) diff --git a/examples/cores/goban/compiler/src/Conversion/LowerBuckyball/CMakeLists.txt b/examples/cores/goban/compiler/src/Conversion/LowerBuckyball/CMakeLists.txt deleted file mode 100644 index 12199e0c5..000000000 --- a/examples/cores/goban/compiler/src/Conversion/LowerBuckyball/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -add_mlir_library(${BB_CORE_TARGET_PREFIX}LowerBuckyballPass - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/BaseAssignPhysicalBankPatterns.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/PhysicalBankState.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/ReportBankUsagePass.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/AssignPhysicalBanksPass.cpp - ${BB_assign_SOURCES} - ${BB_REPO_ROOT}/examples/balls/smatmul/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp - ${BB_REPO_ROOT}/examples/balls/smatmul/compiler/src/Conversion/LowerBuckyball/MegaKernelToBankSSAPatterns.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/QuantizeTensorToBankSSAPatterns.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/RuntimeLowering.cpp - LowerBuckyballPass.cpp - LowerBuckyballToBankSSAPass.cpp - - LINK_LIBS PUBLIC - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballTransforms - MLIRLinalgDialect -) diff --git a/examples/cores/goban/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp b/examples/cores/goban/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp deleted file mode 100644 index eeb8ecc1a..000000000 --- a/examples/cores/goban/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp +++ /dev/null @@ -1,126 +0,0 @@ -//===- LowerBuckyballPass.cpp - Goban Buckyball lowering pass -------------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/TypeConverter.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/Pass/Pass.h" - -#include "Buckyball/BuckyballDialect.h" -#include "Buckyball/BuckyballOps.h" -#include "Buckyball/Transform.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { - -class LowerBuckyballToLLVMPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerBuckyballToLLVMPass) - LowerBuckyballToLLVMPass() = default; - LowerBuckyballToLLVMPass(const LowerBuckyballToLLVMPass &) {} - - StringRef getArgument() const final { return "lower-buckyball"; } - StringRef getDescription() const final { - return "Lower Goban Buckyball dialect ops."; - } - - Option stable{*this, "stable", - llvm::cl::desc("Use stable LLVM Buckyball intrinsics."), - llvm::cl::init(false)}; - - void getDependentDialects(DialectRegistry ®istry) const override { - registry - .insert(); - } - - void runOnOperation() override { - const auto &targetConfig = buckyball_target::getBuckyballTarget(); - MLIRContext *context = &getContext(); - ModuleOp module = getOperation(); - LLVMTypeConverter converter(context); - RewritePatternSet patterns(context); - LLVMConversionTarget target(*context); - - configureBuckyballLegalizeForExportTarget(target, stable); - target.addLegalDialect(); - populateBuckyballLegalizeForLLVMExportPatterns( - converter, patterns, targetConfig.bankWidthBits / 8, - targetConfig.bankDepth, targetConfig.bankNum, - /*includeFuncOperandForwarding=*/false, stable); - - ConversionConfig config; - config.allowPatternRollback = false; - if (failed(applyPartialConversion(module, target, std::move(patterns), - config))) - signalPassFailure(); - } -}; - -class LowerBankSSAToIntrinsicsPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerBankSSAToIntrinsicsPass) - LowerBankSSAToIntrinsicsPass() = default; - LowerBankSSAToIntrinsicsPass(const LowerBankSSAToIntrinsicsPass &) {} - - StringRef getArgument() const final { return "lower-bank-ssa-to-intrinsics"; } - StringRef getDescription() const final { - return "Lower Goban bank-SSA and Buckyball ops to intrinsic ops."; - } - - Option stable{*this, "stable", - llvm::cl::desc("Use stable LLVM Buckyball intrinsics."), - llvm::cl::init(false)}; - - void getDependentDialects(DialectRegistry ®istry) const override { - registry - .insert(); - } - - void runOnOperation() override { - const auto &targetConfig = buckyball_target::getBuckyballTarget(); - MLIRContext *context = &getContext(); - ModuleOp module = getOperation(); - LLVMTypeConverter converter(context); - RewritePatternSet patterns(context); - LLVMConversionTarget target(*context); - - configureBuckyballLegalizeForExportTarget(target, stable); - target.addLegalDialect(); - populateBuckyballLegalizeForLLVMExportPatterns( - converter, patterns, targetConfig.bankWidthBits / 8, - targetConfig.bankDepth, targetConfig.bankNum, - /*includeFuncOperandForwarding=*/false, stable); - - ConversionConfig config; - config.allowPatternRollback = false; - if (failed(applyPartialConversion(module, target, std::move(patterns), - config))) - signalPassFailure(); - } -}; - -} // namespace - -void mlir::buddy::registerLowerBuckyballPass() { - registerTileRuntimePass(); - PassRegistration(); -} - -void mlir::buddy::registerLowerBankSSAToIntrinsicsPass() { - PassRegistration(); -} diff --git a/examples/cores/goban/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp b/examples/cores/goban/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp deleted file mode 100644 index ed67fb60b..000000000 --- a/examples/cores/goban/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp +++ /dev/null @@ -1,93 +0,0 @@ -//===- LowerBuckyballToBankSSAPass.cpp - Goban bank-SSA lowering ----------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/Dialect/Utils/StructuredOpsUtils.h" -#include "mlir/IR/AffineMap.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/PatternMatch.h" -#include "mlir/Pass/Pass.h" -#include "mlir/Transforms/GreedyPatternRewriteDriver.h" - -#include "Buckyball/BuckyballDialect.h" -#include "Buckyball/BuckyballOps.h" -#include "Trace/TraceDialect.h" - -using namespace mlir; - -namespace { - -class MemTransposeToLinalgPattern - : public OpRewritePattern<::buddy::buckyball::MemTransposeOp> { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(::buddy::buckyball::MemTransposeOp op, - PatternRewriter &rewriter) const override { - auto input = dyn_cast(op.getInput().getType()); - auto output = dyn_cast(op.getOutput().getType()); - if (!input || !output || !input.hasStaticShape() || - !output.hasStaticShape() || input.getRank() != 2 || - output.getRank() != 2 || output.getShape()[0] != input.getShape()[1] || - output.getShape()[1] != input.getShape()[0]) - return op.emitError("requires matching static rank-2 transpose memrefs"); - - MLIRContext *context = rewriter.getContext(); - AffineMap identity = AffineMap::getMultiDimIdentityMap(2, context); - AffineMap transpose = AffineMap::get( - 2, 0, {rewriter.getAffineDimExpr(1), rewriter.getAffineDimExpr(0)}, - context); - SmallVector iterators(2, - utils::IteratorType::parallel); - rewriter.create( - op.getLoc(), TypeRange{}, ValueRange{op.getInput()}, - ValueRange{op.getOutput()}, ArrayRef{identity, transpose}, - iterators, [](OpBuilder &builder, Location location, ValueRange args) { - builder.create(location, args[0]); - }); - rewriter.eraseOp(op); - return success(); - } -}; - -} // namespace - -namespace { - -class LowerBuckyballToBankSSAPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerBuckyballToBankSSAPass) - - StringRef getArgument() const final { return "lower-buckyball-to-bank-ssa"; } - StringRef getDescription() const final { - return "Lower Goban Buckyball ops to explicit bank-SSA ops."; - } - - void getDependentDialects(DialectRegistry ®istry) const override { - registry.insert<::buddy::trace::BuddyTraceDialect, arith::ArithDialect, - linalg::LinalgDialect, memref::MemRefDialect, - scf::SCFDialect, ::buddy::buckyball::BuckyballDialect>(); - } - - void runOnOperation() override { - RewritePatternSet patterns(&getContext()); - mlir::buddy::populateMatmulRegionToBankSSAPatterns(patterns, false, 0, -1); - mlir::buddy::populateQuantizeTensorToBankSSAPatterns(patterns); - patterns.add(&getContext()); - if (failed(applyPatternsGreedily(getOperation(), std::move(patterns)))) - signalPassFailure(); - } -}; - -} // namespace - -void mlir::buddy::registerLowerBuckyballToBankSSAPass() { - PassRegistration(); -} diff --git a/examples/cores/goban/compiler/src/Conversion/LowerTileToBuckyball/CMakeLists.txt b/examples/cores/goban/compiler/src/Conversion/LowerTileToBuckyball/CMakeLists.txt deleted file mode 100644 index c12d60c23..000000000 --- a/examples/cores/goban/compiler/src/Conversion/LowerTileToBuckyball/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -add_mlir_library(${BB_CORE_TARGET_PREFIX}LowerTileToBuckyballPass - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerTileToBuckyball/QuantizedKernelPatterns.cpp - LowerTileToBuckyball.cpp - ${BB_REPO_ROOT}/examples/balls/smatmul/compiler/src/Conversion/LowerTileToBuckyball/TileMatMulPatterns.cpp - - LINK_LIBS PUBLIC - BuddyTile - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball - MLIRLinalgDialect -) diff --git a/examples/cores/goban/compiler/src/Conversion/LowerTileToBuckyball/LowerTileToBuckyball.cpp b/examples/cores/goban/compiler/src/Conversion/LowerTileToBuckyball/LowerTileToBuckyball.cpp deleted file mode 100644 index f17989c35..000000000 --- a/examples/cores/goban/compiler/src/Conversion/LowerTileToBuckyball/LowerTileToBuckyball.cpp +++ /dev/null @@ -1,115 +0,0 @@ -//===- LowerTileToBuckyball.cpp - Goban tile pass registration ------------===// -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//===----------------------------------------------------------------------===// - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinOps.h" -#include "mlir/Pass/Pass.h" -#include "mlir/Transforms/DialectConversion.h" - -#include "Buckyball/BuckyballDialect.h" -#include "Buckyball/BuckyballOps.h" -#include "Conversion/LowerTileToBuckyball/LowerTileToBuckyball.h" -#include "Target/BuckyballTargetRegistry.h" -#include "Tile/TileDialect.h" -#include "Tile/TileOps.h" -#include "Tile/Transform.h" - -using namespace mlir; -using namespace ::buddy::buckyball; -namespace tile = ::buddy::tile; - -namespace mlir::buddy { -void populateSMatMulBallTileLoweringPatterns(RewritePatternSet &patterns, - int64_t bankWidthBytes, - int64_t bankDepth, - int64_t bankNum); -} // namespace mlir::buddy - -namespace { -class TileTransposeLowering : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(tile::TileTransposeOp op, - PatternRewriter &rewriter) const override { - auto inputType = dyn_cast(op.getAMemArray().getType()); - auto outputType = dyn_cast(op.getBMemArray().getType()); - if (!inputType || !outputType || !inputType.hasStaticShape() || - !outputType.hasStaticShape() || inputType.getRank() != 2 || - outputType.getRank() != 2) - return op.emitError("requires static rank-2 input and output memrefs"); - if (outputType.getShape()[0] != inputType.getShape()[1] || - outputType.getShape()[1] != inputType.getShape()[0] || - inputType.getElementType() != outputType.getElementType()) - return op.emitError("requires a matching transposed output memref"); - - rewriter.create(op.getLoc(), op.getAMemArray(), - op.getBMemArray()); - rewriter.eraseOp(op); - return success(); - } -}; - -class LowerTileToBuckyballPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerTileToBuckyballPass) - StringRef getArgument() const final { return "convert-tile-to-buckyball"; } - StringRef getDescription() const final { - return "Convert Goban Tile operations to Buckyball operations"; - } - LowerTileToBuckyballPass() = default; - LowerTileToBuckyballPass(const LowerTileToBuckyballPass &) {} - - void getDependentDialects(DialectRegistry ®istry) const override { - registry - .insert(); - } - - void runOnOperation() override { - const auto &targetConfig = buckyball_target::getBuckyballTarget(); - MLIRContext *context = &getContext(); - ConversionTarget target(*context); - target.addLegalDialect<::buddy::buckyball::BuckyballDialect, - memref::MemRefDialect, arith::ArithDialect, - scf::SCFDialect, func::FuncDialect, - linalg::LinalgDialect>(); - target.addIllegalOp(); - target.addIllegalOp(); - - RewritePatternSet patterns(context); - mlir::buddy::populateSMatMulBallTileLoweringPatterns( - patterns, targetConfig.bankWidthBits / 8, targetConfig.bankDepth, - targetConfig.bankNum); - mlir::buddy::populateQuantizedKernelTileLoweringPatterns(patterns); - patterns.add(context); - if (failed(applyPartialConversion(getOperation(), target, - std::move(patterns)))) - signalPassFailure(); - } -}; -} // namespace - -void mlir::buddy::registerLowerTileToBuckyballPass() { - PassRegistration(); -} diff --git a/examples/cores/goban/compiler/src/Dialect/Buckyball/CMakeLists.txt b/examples/cores/goban/compiler/src/Dialect/Buckyball/CMakeLists.txt deleted file mode 100644 index 004ad362c..000000000 --- a/examples/cores/goban/compiler/src/Dialect/Buckyball/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -set(LLVM_TARGET_DEFINITIONS ${BB_BUCKYBALL_DIALECT_TD}) -mlir_tablegen(Buckyball.h.inc -gen-op-decls) -mlir_tablegen(Buckyball.cpp.inc -gen-op-defs) -mlir_tablegen(BuckyballTypes.h.inc -gen-typedef-decls -typedefs-dialect=buckyball) -mlir_tablegen(BuckyballTypes.cpp.inc -gen-typedef-defs -typedefs-dialect=buckyball) -mlir_tablegen(BuckyballDialect.h.inc -gen-dialect-decls -dialect=buckyball) -mlir_tablegen(BuckyballDialect.cpp.inc -gen-dialect-defs -dialect=buckyball) -add_public_tablegen_target(${BB_CORE_TARGET_PREFIX}MLIRBuckyballIncGen) -add_mlir_doc(GobanBuckyball ${BB_CORE_TARGET_PREFIX}Buckyball Dialects/ - -gen-dialect-doc) - -set(LLVM_TARGET_DEFINITIONS ${BB_BUCKYBALL_DIALECT_TD}) -mlir_tablegen(BuckyballConversions.inc -gen-llvmir-conversions) -add_public_tablegen_target(${BB_CORE_TARGET_PREFIX}BuddyBuckyballConversionsIncGen) - -add_subdirectory(IR) -add_subdirectory(Transforms) diff --git a/examples/cores/goban/compiler/src/Dialect/Buckyball/GobanBuckyball.td b/examples/cores/goban/compiler/src/Dialect/Buckyball/GobanBuckyball.td deleted file mode 100644 index bc56af64c..000000000 --- a/examples/cores/goban/compiler/src/Dialect/Buckyball/GobanBuckyball.td +++ /dev/null @@ -1,26 +0,0 @@ -//===-- GobanBuckyball.td - Goban Buckyball dialect ops -*- tablegen -*-===// -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//===----------------------------------------------------------------------===// - -#ifndef GOBAN_BUCKYBALL_OPS -#define GOBAN_BUCKYBALL_OPS - -include "Buckyball.td" -include "ToInt8Ball.td" -include "Int2FpBall.td" -include "TransposeBall.td" -include "SMatMulBall.td" - -#endif // GOBAN_BUCKYBALL_OPS diff --git a/examples/cores/goban/compiler/src/Dialect/Buckyball/IR/CMakeLists.txt b/examples/cores/goban/compiler/src/Dialect/Buckyball/IR/CMakeLists.txt deleted file mode 100644 index 42c7f8807..000000000 --- a/examples/cores/goban/compiler/src/Dialect/Buckyball/IR/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -add_mlir_dialect_library(${BB_CORE_TARGET_PREFIX}BuddyBuckyball - ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball/IR/BuckyballDialect.cpp - ${BB_TARGET_REGISTRY_SOURCE} - ${BB_BALL_IR_SOURCES} - - LINK_LIBS PUBLIC - MLIRIR - MLIRLLVMDialect - MLIRBufferizationDialect - MLIRMemRefDialect -) - -target_include_directories(${BB_CORE_TARGET_PREFIX}BuddyBuckyball PUBLIC - ${BB_COMPILER_COMMON_DIR} - ${BB_COMPILER_COMMON_DIR}/Dialect - ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball - ${CMAKE_CURRENT_BINARY_DIR}/.. - ${CMAKE_CURRENT_BINARY_DIR}/../../.. -) diff --git a/examples/cores/goban/compiler/src/Dialect/Buckyball/Transforms/CMakeLists.txt b/examples/cores/goban/compiler/src/Dialect/Buckyball/Transforms/CMakeLists.txt deleted file mode 100644 index 0d2cecf2e..000000000 --- a/examples/cores/goban/compiler/src/Dialect/Buckyball/Transforms/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -add_mlir_dialect_library(${BB_CORE_TARGET_PREFIX}BuddyBuckyballTransforms - ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.cpp - ${BB_REPO_ROOT}/examples/balls/toint8/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - ${BB_REPO_ROOT}/examples/balls/int2fp/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - ${BB_REPO_ROOT}/examples/balls/transpose/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - ${BB_REPO_ROOT}/examples/balls/smatmul/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - LegalizeForLLVMExport.cpp - - LINK_LIBS PUBLIC - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball - MLIRIR - MLIRLLVMCommonConversion - MLIRLLVMDialect -) diff --git a/examples/cores/goban/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/cores/goban/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 603fd0520..000000000 --- a/examples/cores/goban/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,73 +0,0 @@ -//===- LegalizeForLLVMExport.cpp - Goban Buckyball LLVM lowering ----------===// -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//===----------------------------------------------------------------------===// - -#include "Buckyball/Transform.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" - -using namespace mlir; -using namespace buddy::buckyball::legalize; - -namespace mlir::buddy::buckyball { -void populateTransposeBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, - RewritePatternSet &, - bool, int64_t); -void configureTransposeBallLegalizeForExportTarget(LLVMConversionTarget &, - bool); -void populateSMatMulBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, - RewritePatternSet &, bool, - int64_t); -void configureSMatMulBallLegalizeForExportTarget(LLVMConversionTarget &, bool); -void populateToInt8BallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, - RewritePatternSet &, bool, - int64_t); -void configureToInt8BallLegalizeForExportTarget(LLVMConversionTarget &, bool); -void populateInt2FpBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, - RewritePatternSet &, bool, - int64_t); -void configureInt2FpBallLegalizeForExportTarget(LLVMConversionTarget &, bool); -} // namespace mlir::buddy::buckyball - -void mlir::populateBuckyballLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, - int64_t bankWidthBytes, int64_t bankDepth, int64_t bankNum, - bool includeFuncOperandForwarding, bool stable) { - (void)bankWidthBytes; - (void)bankNum; - - populateBaseLegalizeForLLVMExportPatterns(converter, patterns, - includeFuncOperandForwarding); - mlir::buddy::buckyball::populateToInt8BallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::populateInt2FpBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::populateTransposeBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::populateSMatMulBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); -} - -void mlir::configureBuckyballLegalizeForExportTarget( - LLVMConversionTarget &target, bool stable) { - configureBaseLegalizeForExportTarget(target); - mlir::buddy::buckyball::configureToInt8BallLegalizeForExportTarget(target, - stable); - mlir::buddy::buckyball::configureInt2FpBallLegalizeForExportTarget(target, - stable); - mlir::buddy::buckyball::configureTransposeBallLegalizeForExportTarget(target, - stable); - mlir::buddy::buckyball::configureSMatMulBallLegalizeForExportTarget(target, - stable); -} diff --git a/examples/cores/goban/compiler/src/Dialect/CMakeLists.txt b/examples/cores/goban/compiler/src/Dialect/CMakeLists.txt deleted file mode 100644 index ec181148e..000000000 --- a/examples/cores/goban/compiler/src/Dialect/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory(Buckyball) diff --git a/examples/cores/goban/compiler/src/Target/CMakeLists.txt b/examples/cores/goban/compiler/src/Target/CMakeLists.txt deleted file mode 100644 index 39d31dc9b..000000000 --- a/examples/cores/goban/compiler/src/Target/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory(LLVMIR) diff --git a/examples/cores/goban/compiler/src/Target/LLVMIR/CMakeLists.txt b/examples/cores/goban/compiler/src/Target/LLVMIR/CMakeLists.txt deleted file mode 100644 index 0ca0f41c5..000000000 --- a/examples/cores/goban/compiler/src/Target/LLVMIR/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory(Dialect) diff --git a/examples/cores/goban/compiler/src/Target/LLVMIR/Dialect/Buckyball/CMakeLists.txt b/examples/cores/goban/compiler/src/Target/LLVMIR/Dialect/Buckyball/CMakeLists.txt deleted file mode 100644 index 06b99171b..000000000 --- a/examples/cores/goban/compiler/src/Target/LLVMIR/Dialect/Buckyball/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -add_mlir_translation_library(${BB_CORE_TARGET_PREFIX}BuddyBuckyballToLLVMIRTranslation - ${BB_COMPILER_COMMON_DIR}/Target/LLVMIR/Dialect/Buckyball/BuckyballToLLVMIRTranslation.cpp - - DEPENDS - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballConversionsIncGen - intrinsics_gen - - LINK_COMPONENTS - Core - - LINK_LIBS PUBLIC - MLIRIR - MLIRSupport - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball -) diff --git a/examples/cores/goban/compiler/src/Target/LLVMIR/Dialect/CMakeLists.txt b/examples/cores/goban/compiler/src/Target/LLVMIR/Dialect/CMakeLists.txt deleted file mode 100644 index ec181148e..000000000 --- a/examples/cores/goban/compiler/src/Target/LLVMIR/Dialect/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory(Buckyball) diff --git a/examples/cores/goban/configs/ant.toml b/examples/cores/goban/configs/ant.toml deleted file mode 100644 index 93cfc7804..000000000 --- a/examples/cores/goban/configs/ant.toml +++ /dev/null @@ -1,6 +0,0 @@ -factory = "examples.cores.goban.AntGobanCore" -balldomain = "balldomains/default.toml" -memdomain = "memdomains/default.toml" -frontend = "frontends/default.toml" -rvv = "rvv/default.toml" -cpu = "cpu/ant.toml" diff --git a/examples/cores/goban/configs/balldomains/default.toml b/examples/cores/goban/configs/balldomains/default.toml deleted file mode 100644 index fb41a02ab..000000000 --- a/examples/cores/goban/configs/balldomains/default.toml +++ /dev/null @@ -1,19 +0,0 @@ -# Goban Ball Domain Configuration - -ballNum = 4 - -ballIdMappings = [ - { ballId = 0, ballName = "SMatMulBall", ballClass = "examples.balls.smatmul.SMatMulBall", config = "../../../../balls/smatmul/configs/default.toml", inBW = 2, outBW = 1 }, - { ballId = 1, ballName = "ToInt8Ball", ballClass = "examples.balls.toint8.ToInt8Ball", config = "../../../../balls/toint8/configs/default.toml", inBW = 1, outBW = 1 }, - { ballId = 2, ballName = "Int2FpBall", ballClass = "examples.balls.int2fp.Int2FpBall", config = "../../../../balls/int2fp/configs/default.toml", inBW = 2, outBW = 1 }, - { ballId = 3, ballName = "TransposeBall", ballClass = "examples.balls.transpose.TransposeBall", config = "../../../../balls/transpose/configs/default.toml", inBW = 1, outBW = 1 } -] - -ballISA = [ - { mnemonic = "SMATMUL_OS", funct7 = 65, bid = 0 }, - { mnemonic = "SMATMUL_BIAS", funct7 = 17, bid = 0 }, - { mnemonic = "QUANT_F32_TO_I8", funct7 = 51, bid = 1 }, - { mnemonic = "QUANT_I32_TO_I8", funct7 = 67, bid = 1 }, - { mnemonic = "INT32_TO_FP32", funct7 = 68, bid = 2 }, - { mnemonic = "TRANSPOSE", funct7 = 49, bid = 3 } -] diff --git a/examples/cores/goban/configs/cpu/ant.toml b/examples/cores/goban/configs/cpu/ant.toml deleted file mode 100644 index 9cf6bf076..000000000 --- a/examples/cores/goban/configs/cpu/ant.toml +++ /dev/null @@ -1,2 +0,0 @@ -kind = "ant" -config = "../../../ant/configs/cpu/default.toml" diff --git a/examples/cores/goban/configs/cpu/default.toml b/examples/cores/goban/configs/cpu/default.toml deleted file mode 100644 index 1bf2f46e8..000000000 --- a/examples/cores/goban/configs/cpu/default.toml +++ /dev/null @@ -1,2 +0,0 @@ -kind = "rocket" -config = "../../../rocket/configs/cpu/rocket/default.toml" diff --git a/examples/cores/goban/configs/default.toml b/examples/cores/goban/configs/default.toml deleted file mode 100644 index 672adbd91..000000000 --- a/examples/cores/goban/configs/default.toml +++ /dev/null @@ -1,8 +0,0 @@ -factory = "examples.cores.goban.GobanCore" -# Goban Core Configuration (default) - -balldomain = "balldomains/default.toml" -memdomain = "memdomains/default.toml" -frontend = "frontends/default.toml" -rvv = "rvv/default.toml" -cpu = "cpu/default.toml" diff --git a/examples/cores/goban/configs/frontends/default.toml b/examples/cores/goban/configs/frontends/default.toml deleted file mode 100644 index 41cd684cc..000000000 --- a/examples/cores/goban/configs/frontends/default.toml +++ /dev/null @@ -1,8 +0,0 @@ -robEntries = 16 -rsOutOfOrderResponse = true -bankIdLen = 10 -vbankIdUpperBound = 15 -sharedBankIdBase = 32 -iterLen = 34 -subRobEnable = false -subRobDepth = 64 diff --git a/examples/cores/goban/configs/memdomains/default.toml b/examples/cores/goban/configs/memdomains/default.toml deleted file mode 100644 index ef9a70dbc..000000000 --- a/examples/cores/goban/configs/memdomains/default.toml +++ /dev/null @@ -1,32 +0,0 @@ -# Goban Core Memory Domain Configuration (default) -# Private (per-core) memory parameters. Shared memory lives in the tile TOML. - -[bank] -num = 20 -width = 128 -entries = 1024 -maskLen = 16 -channel = 7 - -[dma] -nXacts = 8 -burstMaxBytes = 1024 -busWidth = 128 -maxInFlightMemReqs = 16 - -[tlb] -size = 4 - -[tma] -readChannel = 4 -writeChannel = 4 - -[mmio] -enable = false -bankNum = 16 -bankEntries = 64 -bankWidth = 128 -readWidth = 8 - -[mem] -addrLen = 39 diff --git a/examples/cores/goban/workloads/ctests/CMakeLists.txt b/examples/cores/goban/workloads/ctests/CMakeLists.txt deleted file mode 100644 index 9d2130123..000000000 --- a/examples/cores/goban/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -add_subdirectory(transpose_smatmul_quant_pipeline) -set(BUCKYBALL_CTEST_TARGET ${BUCKYBALL_CORE_WORKLOAD_TARGET}) -file(RELATIVE_PATH RVV_KERNEL_SOURCE ${CMAKE_CURRENT_SOURCE_DIR} - ${WORKLOAD_SRC_DIR}/CTest/rvv/rvv_kernel.c) -add_buckyball_ctest(${RVV_KERNEL_SOURCE}) -buckyball_ctest_name(RVV_KERNEL_TEST ${BUCKYBALL_CTEST_TARGET} rvv_kernel) -add_dependencies(core-ctests-build ${RVV_KERNEL_TEST}-ctest-build) -add_subdirectory(rvv) diff --git a/examples/cores/goban/workloads/ctests/rvv/CMakeLists.txt b/examples/cores/goban/workloads/ctests/rvv/CMakeLists.txt deleted file mode 100644 index dd57c1e63..000000000 --- a/examples/cores/goban/workloads/ctests/rvv/CMakeLists.txt +++ /dev/null @@ -1,67 +0,0 @@ -set(KERNEL_DIR "${WORKLOAD_LIB_DIR}/bbsw/kernels/rvv") -set(IMAGE_SOURCE "${CMAKE_CURRENT_BINARY_DIR}/images.cpp") -set(IMAGE_HEADER "${CMAKE_CURRENT_BINARY_DIR}/images.h") -file(READ "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/params.h" PARAMS) -foreach(PARAM BANK_NUM BANK_WIDTH BANK_LINES) - string(REGEX MATCH "#define ${PARAM} ([0-9]+)" MATCH "${PARAMS}") - set(${PARAM} "${CMAKE_MATCH_1}") -endforeach() -if(BANK_NUM LESS 4 OR NOT BANK_WIDTH EQUAL 128 OR BANK_LINES LESS 32) - message(FATAL_ERROR - "RVV kernel CTests require 4 banks, 128-bit rows and 512 bytes per bank; " - "${BUCKYBALL_CTEST_TARGET} has ${BANK_NUM} banks, ${BANK_WIDTH}-bit rows, ${BANK_LINES} lines") -endif() -add_custom_command( - OUTPUT "${IMAGE_SOURCE}" "${IMAGE_HEADER}" - COMMAND "${Python3_EXECUTABLE}" "${KERNEL_DIR}/build_images.py" - --output "${CMAKE_CURRENT_BINARY_DIR}" - DEPENDS "${KERNEL_DIR}/build_images.py" "${KERNEL_DIR}/kernel.ld" - "${KERNEL_DIR}/kernels.h" "${KERNEL_DIR}/silu.cpp" - "${KERNEL_DIR}/snake.cpp" "${KERNEL_DIR}/quant.cpp" - "${KERNEL_DIR}/matmul.cpp" - "${KERNEL_DIR}/math/exp.cpp" "${KERNEL_DIR}/math/sin.cpp" - "${KERNEL_DIR}/math/math.h" "${KERNEL_DIR}/math/constants.h" - VERBATIM) -buckyball_ctest_name(IMAGE_TARGET ${BUCKYBALL_CTEST_TARGET} rvv_images) -add_custom_target(${IMAGE_TARGET} DEPENDS "${IMAGE_SOURCE}" "${IMAGE_HEADER}") - -foreach(KERNEL silu swiglu snake mxfp8_quant matmul) - buckyball_enforce_ctest_line_limit("${KERNEL}.cpp") - buckyball_ctest_name(NAME ${BUCKYBALL_CTEST_TARGET} rvv_${KERNEL}) - add_buckyball_linux_ctest(${NAME} ${CMAKE_CURRENT_SOURCE_DIR} ${KERNEL}.cpp) - if(KERNEL STREQUAL "matmul") - set(DISPATCH_SOURCE "${KERNEL_DIR}/matmul_dispatch.cpp") - elseif(KERNEL STREQUAL "mxfp8_quant") - set(DISPATCH_SOURCE "${KERNEL_DIR}/quant_dispatch.cpp") - else() - set(DISPATCH_SOURCE "${KERNEL_DIR}/dispatch.cpp") - endif() - target_sources(${NAME}-linux PRIVATE "${IMAGE_SOURCE}" "${DISPATCH_SOURCE}") - target_include_directories(${NAME}-linux PRIVATE "${CMAKE_CURRENT_BINARY_DIR}" - "${BUCKYBALL_REPO_ROOT}/stack/compiler/thirdparty/buddy-mlir/llvm/mlir/include/mlir/ExecutionEngine") - target_compile_options(${NAME}-linux PRIVATE -O2) - set_target_properties(${NAME}-linux PROPERTIES LINKER_LANGUAGE CXX) - target_link_libraries(${NAME}-linux PRIVATE m) - add_dependencies(${NAME}-linux ${IMAGE_TARGET}) - buckyball_ctest_deps(DEPS ${CMAKE_CURRENT_SOURCE_DIR} ${KERNEL}.cpp) - add_custom_command( - OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${NAME}-baremetal" - COMMAND ${RISCV_GNU_TOOLCHAIN}/bin/riscv64-unknown-elf-g++ - ${BUCKYBALL_CTEST_C_FLAGS} - -I${BUCKYBALL_REPO_ROOT}/stack/runtime/include -I${WORKLOAD_LIB_DIR} - -I${BUCKYBALL_ISA_DIR}/chip -I${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET} - -I${CMAKE_CURRENT_BINARY_DIR} - -I${BUCKYBALL_REPO_ROOT}/stack/compiler/thirdparty/buddy-mlir/llvm/mlir/include/mlir/ExecutionEngine - -o "${NAME}-baremetal" ${BBSW_BAREMETAL_DIR}/crt0.S - -x c ${WORKLOAD_LIB_DIR}/buckyball.c ${BBHW_MEM_C} - ${BUCKYBALL_CHIP_COMMON_SOURCES} - -x c++ "${CMAKE_CURRENT_SOURCE_DIR}/${KERNEL}.cpp" "${IMAGE_SOURCE}" "${DISPATCH_SOURCE}" -lm - DEPENDS ${DEPS} "${IMAGE_SOURCE}" "${IMAGE_HEADER}" "${DISPATCH_SOURCE}" - ${BBSW_BAREMETAL_DIR}/crt0.S - WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" - VERBATIM) - add_custom_target(${NAME}-ctest-build - DEPENDS ${NAME}-linux-build "${CMAKE_CURRENT_BINARY_DIR}/${NAME}-baremetal") - add_dependencies(${NAME}-ctest-build ${IMAGE_TARGET}) - add_dependencies(core-ctests-build ${NAME}-ctest-build) -endforeach() diff --git a/examples/cores/goban/workloads/ctests/transpose_smatmul_quant_pipeline/CMakeLists.txt b/examples/cores/goban/workloads/ctests/transpose_smatmul_quant_pipeline/CMakeLists.txt deleted file mode 100644 index 9676371fa..000000000 --- a/examples/cores/goban/workloads/ctests/transpose_smatmul_quant_pipeline/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -add_buckyball_core_workload(transpose_smatmul_quant_pipeline SOURCES - transpose_smatmul_quant_pipeline-main.cpp - goban.c) diff --git a/examples/cores/goban/workloads/ctests/transpose_smatmul_quant_pipeline/goban.c b/examples/cores/goban/workloads/ctests/transpose_smatmul_quant_pipeline/goban.c deleted file mode 100644 index fc4bfe290..000000000 --- a/examples/cores/goban/workloads/ctests/transpose_smatmul_quant_pipeline/goban.c +++ /dev/null @@ -1,53 +0,0 @@ -#include "buckyball.h" -#include -#include -#include -#include -#include -#include -#include - -#if BANK_WIDTH != 128 || BANK_LINES < 64 -#error "Goban core pipeline requires 128-bit banks with at least 64 rows" -#endif - -enum { DIM = 16 }; -static int8_t input[BB_CORES_PER_TILE][DIM * DIM] __attribute__((aligned(64))); -static int8_t weight[BB_CORES_PER_TILE][DIM * DIM] __attribute__((aligned(64))); -static int32_t bias[BB_CORES_PER_TILE][DIM] __attribute__((aligned(64))); -static float scale[BB_CORES_PER_TILE][DIM] __attribute__((aligned(64))); -static int8_t output[BB_CORES_PER_TILE][DIM * DIM] __attribute__((aligned(64))); - -#ifdef __cplusplus -extern "C" -#endif - int goban_core(core_id_t id) { - int slot = id.core; - for (int row = 0; row < DIM; ++row) - for (int col = 0; col < DIM; ++col) { - input[slot][row * DIM + col] = (row * 3 + col * 5) % 9 - 4; - weight[slot][row * DIM + col] = (row + 2 * col) % 5 - 2; - } - for (int col = 0; col < DIM; ++col) - scale[slot][col] = 1.0f; - for (int bank = 0; bank < 10; ++bank) - bb_mem_alloc(bank, 1, 1); - bb_mvin((uintptr_t)input[slot], 0, DIM, 1); - bb_mvin((uintptr_t)weight[slot], 2, DIM, 1); - bb_mvin((uintptr_t)bias[slot], 3, 4, 1); - bb_mvin((uintptr_t)scale[slot], 8, 4, 1); - bb_transpose(0, 1, DIM, 8); - bb_smatmul_bias(3, 0); - bb_smatmul_os(1, 2, 4, DIM, DIM, DIM, 1, 1, 0); - bb_quant_i32_to_i8(4, 8, 9, DIM * 4, 0, 0, 4, 4, 4, 0); - bb_mvout((uintptr_t)output[slot], 9, DIM, 1); - for (int row = 0; row < DIM; ++row) - for (int col = 0; col < DIM; ++col) { - int expected = 0; - for (int k = 0; k < DIM; ++k) - expected += input[slot][k * DIM + row] * weight[slot][k * DIM + col]; - if (output[slot][row * DIM + col] != expected) - return 1; - } - return 0; -} diff --git a/examples/cores/goban/workloads/ctests/transpose_smatmul_quant_pipeline/transpose_smatmul_quant_pipeline-main.cpp b/examples/cores/goban/workloads/ctests/transpose_smatmul_quant_pipeline/transpose_smatmul_quant_pipeline-main.cpp deleted file mode 100644 index 0f33dbc42..000000000 --- a/examples/cores/goban/workloads/ctests/transpose_smatmul_quant_pipeline/transpose_smatmul_quant_pipeline-main.cpp +++ /dev/null @@ -1,23 +0,0 @@ -#include - -extern "C" int goban_core(core_id_t); -static volatile uint32_t completed; -static volatile uint32_t failed; - -int main() { - core_id_t id = bb_get_core_id(); - if (id.tile == BB_WORK_TILE && - bb_topology_core_profile(id) == BB_CORE_WORKLOAD_PROFILE) { - __atomic_fetch_or(&failed, goban_core(id) != 0, __ATOMIC_RELEASE); - __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); - } - if (id.tile == BB_WORK_TILE && id.core == 0) { - uint32_t expected = - bb_topology_profile_cores_per_tile(BB_CORE_WORKLOAD_PROFILE); - while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) - asm volatile("nop"); - return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; - } - for (;;) - asm volatile("wfi"); -} diff --git a/examples/cores/pebble/arch/src/main/scala/PebbleCore.scala b/examples/cores/pebble/arch/src/main/scala/PebbleCore.scala deleted file mode 100644 index 27685dd23..000000000 --- a/examples/cores/pebble/arch/src/main/scala/PebbleCore.scala +++ /dev/null @@ -1,20 +0,0 @@ -package examples.cores.pebble - -import framework.top.GlobalConfig -import chisel3.experimental.hierarchy.{instantiable, Instance, Instantiate} -import framework.system.core.{CoreConnection, RocketEndpoint} -import hier.core.rocket.Core -import framework.system.core.accelerator.BuckyballAccelerator - -@instantiable -class PebbleCore(b: GlobalConfig) extends RocketEndpoint(b) { - require(p.buckyball.nonEmpty) - val cpu = Instantiate(new Core(p.data, p.instruction, p.regions, Some(p.commands))(p.cpu)) - clink.cpu <> cpu.clink - val accelerator = Instantiate(new BuckyballAccelerator(p.buckyball.get)) - CoreConnection.accelerator(clink.accelerator.get, accelerator) -} - -object PebbleCore extends framework.system.core.RocketCoreFactory { - def instantiate(b: GlobalConfig): Instance[RocketEndpoint] = Instantiate(new PebbleCore(b)) -} diff --git a/examples/cores/pebble/compiler/CMakeLists.txt b/examples/cores/pebble/compiler/CMakeLists.txt deleted file mode 100644 index 2551caa37..000000000 --- a/examples/cores/pebble/compiler/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -if(NOT COMMAND add_mlir_library OR NOT COMMAND add_mlir_dialect) - message(FATAL_ERROR "MLIR CMake helpers must be loaded. Build as part of buddy-mlir.") -endif() - -set(BB_CORE_NAME pebble) -set(BB_CORE_TARGET_PREFIX "") - -set(BB_CORE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/..) -get_filename_component(_BB_REPO_ROOT "${BB_CORE_DIR}/../../.." ABSOLUTE) -include_directories(${_BB_REPO_ROOT}/examples/balls) - -add_subdirectory(src) diff --git a/examples/cores/pebble/compiler/src/CMakeLists.txt b/examples/cores/pebble/compiler/src/CMakeLists.txt deleted file mode 100644 index 7dcd81bc3..000000000 --- a/examples/cores/pebble/compiler/src/CMakeLists.txt +++ /dev/null @@ -1,70 +0,0 @@ -if(NOT COMMAND add_mlir_library OR NOT COMMAND add_mlir_dialect) - message(FATAL_ERROR "MLIR CMake helpers must be loaded. Build as part of buddy-mlir.") -endif() - -message(STATUS "Building Pebble compiler Core package") -message(STATUS " Source: ${CMAKE_CURRENT_SOURCE_DIR}") -message(STATUS " Binary: ${CMAKE_CURRENT_BINARY_DIR}") - -get_filename_component(BB_REPO_ROOT ${CMAKE_CURRENT_LIST_DIR}/../../../../.. ABSOLUTE) -set(BB_COMPILER_COMMON_DIR ${BB_REPO_ROOT}/stack/compiler/src) - -if(NOT EXISTS ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball/Buckyball.td) - message(FATAL_ERROR "Missing Buckyball compiler common dir: ${BB_COMPILER_COMMON_DIR}") -endif() - -include_directories(${BB_COMPILER_COMMON_DIR}) -include_directories(${BB_COMPILER_COMMON_DIR}/Dialect) -include_directories(${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}) -include_directories(${CMAKE_CURRENT_BINARY_DIR}) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/Dialect) -include_directories(${CMAKE_CURRENT_BINARY_DIR}/Dialect) -include_directories(BEFORE ${CMAKE_CURRENT_BINARY_DIR}/Dialect/Buckyball) - -set(PEBBLE_BALL_COMPILER_DIALECT_DIRS - ${BB_REPO_ROOT}/examples/balls/transpose/compiler/src/Dialect/Buckyball - ${BB_REPO_ROOT}/examples/balls/smatmul/compiler/src/Dialect/Buckyball - ${BB_REPO_ROOT}/examples/balls/im2col/compiler/src/Dialect/Buckyball - ${BB_REPO_ROOT}/examples/balls/toint8/compiler/src/Dialect/Buckyball - ${BB_REPO_ROOT}/examples/balls/int2fp/compiler/src/Dialect/Buckyball - ${BB_REPO_ROOT}/examples/balls/lut/compiler/src/Dialect/Buckyball - ${BB_REPO_ROOT}/examples/balls/maxpool/compiler/src/Dialect/Buckyball - ${BB_REPO_ROOT}/examples/balls/int8add/compiler/src/Dialect/Buckyball - ${BB_REPO_ROOT}/examples/balls/int8mul/compiler/src/Dialect/Buckyball -) - -set(BB_CORE_BACKEND_INTRINSICS_DIR - ${CMAKE_CURRENT_SOURCE_DIR}/Backend/include - CACHE INTERNAL "Buckyball chip backend intrinsic directory") - -set(BB_BALL_BACKEND_INTRINSICS_DIRS) - -foreach(PEBBLE_BALL_COMPILER_DIALECT_DIR ${PEBBLE_BALL_COMPILER_DIALECT_DIRS}) - if(NOT EXISTS ${PEBBLE_BALL_COMPILER_DIALECT_DIR}) - message(FATAL_ERROR - "Missing pebble ball compiler dialect dir: ${PEBBLE_BALL_COMPILER_DIALECT_DIR}") - endif() - include_directories(${PEBBLE_BALL_COMPILER_DIALECT_DIR}) - get_filename_component(PEBBLE_BALL_COMPILER_DIR - ${PEBBLE_BALL_COMPILER_DIALECT_DIR}/../.. - ABSOLUTE) - list(APPEND BB_BALL_BACKEND_INTRINSICS_DIRS - ${PEBBLE_BALL_COMPILER_DIR}/Backend/include) -endforeach() - -set(BB_BALL_BACKEND_INTRINSICS_DIRS ${BB_BALL_BACKEND_INTRINSICS_DIRS} - CACHE INTERNAL "Buckyball ball backend intrinsic directories") - -add_subdirectory(Dialect) -add_subdirectory(Conversion) -add_subdirectory(Target) - -add_custom_target(buckyball-compiler-${BB_CORE_NAME} - DEPENDS - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballTransforms - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballToLLVMIRTranslation - ${BB_CORE_TARGET_PREFIX}LowerBuckyballPass - ${BB_CORE_TARGET_PREFIX}LowerTileToBuckyballPass -) diff --git a/examples/cores/pebble/compiler/src/Conversion/CMakeLists.txt b/examples/cores/pebble/compiler/src/Conversion/CMakeLists.txt deleted file mode 100644 index 9ca69b1a2..000000000 --- a/examples/cores/pebble/compiler/src/Conversion/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -add_subdirectory(LowerBuckyball) -add_subdirectory(LowerTileToBuckyball) diff --git a/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/CMakeLists.txt b/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/CMakeLists.txt deleted file mode 100644 index 9cafa2834..000000000 --- a/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/CMakeLists.txt +++ /dev/null @@ -1,27 +0,0 @@ -add_mlir_library(${BB_CORE_TARGET_PREFIX}LowerBuckyballPass - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/BaseAssignPhysicalBankPatterns.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/PhysicalBankState.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/ReportBankUsagePass.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/AssignPhysicalBanksPass.cpp - ${BB_assign_SOURCES} - ${BB_REPO_ROOT}/examples/balls/smatmul/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp - CoreBankSSALowering.cpp - LowerBuckyballPass.cpp - LowerBuckyballToBankSSAPass.cpp - ${BB_REPO_ROOT}/examples/balls/smatmul/compiler/src/Conversion/LowerBuckyball/MegaKernelToBankSSAPatterns.cpp - ResidentConvRegionToBankSSAPatterns.cpp - MegaConv2dToBankSSAPatterns.cpp - MemTransposeToBankSSAPatterns.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/QuantizeTensorToBankSSAPatterns.cpp - - LINK_LIBS PUBLIC - MLIRControlFlowDialect - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballTransforms - ${BB_CORE_TARGET_PREFIX}LowerTileToBuckyballPass -) - -# CMake's archive rule appends members, so a chip switch can retain stale -# objects from the previously configured LowerBuckyballPass target. -add_custom_command(TARGET ${BB_CORE_TARGET_PREFIX}LowerBuckyballPass PRE_LINK - COMMAND ${CMAKE_COMMAND} -E rm -f - $) diff --git a/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/CoreBankSSALowering.cpp b/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/CoreBankSSALowering.cpp deleted file mode 100644 index 1039379f3..000000000 --- a/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/CoreBankSSALowering.cpp +++ /dev/null @@ -1,19 +0,0 @@ -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -using namespace mlir; - -namespace mlir::buddy { -void populatePebbleMegaConv2dToBankSSAPatterns(RewritePatternSet &patterns); -void populatePebbleMemTransposeToBankSSAPatterns(RewritePatternSet &patterns); - -void populatePebbleCoreBankSSALoweringPatterns(RewritePatternSet &patterns, - bool traceMegaStages, - int64_t traceMegaStageStart, - int64_t traceMegaStageLimit) { - populateMatmulRegionToBankSSAPatterns( - patterns, traceMegaStages, traceMegaStageStart, traceMegaStageLimit); - populatePebbleMegaConv2dToBankSSAPatterns(patterns); - populatePebbleMemTransposeToBankSSAPatterns(patterns); - populateQuantizeTensorToBankSSAPatterns(patterns); -} -} // namespace mlir::buddy diff --git a/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp b/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp deleted file mode 100644 index 0e9bdc690..000000000 --- a/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp +++ /dev/null @@ -1,127 +0,0 @@ -//===- LowerBuckyballPass.cpp - Pebble Buckyball lowering pass -//-------------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/TypeConverter.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/Pass/Pass.h" - -#include "Buckyball/BuckyballDialect.h" -#include "Buckyball/BuckyballOps.h" -#include "Buckyball/Transform.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace mlir::buddy {} - -namespace { - -class LowerBuckyballToLLVMPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerBuckyballToLLVMPass) - LowerBuckyballToLLVMPass() = default; - LowerBuckyballToLLVMPass(const LowerBuckyballToLLVMPass &) {} - - StringRef getArgument() const final { return "lower-buckyball"; } - StringRef getDescription() const final { - return "Lower Pebble Buckyball dialect ops."; - } - - Option stable{*this, "stable", - llvm::cl::desc("Use stable LLVM Buckyball intrinsics."), - llvm::cl::init(false)}; - void getDependentDialects(DialectRegistry ®istry) const override { - registry - .insert(); - } - - void runOnOperation() override { - const auto &targetConfig = buckyball_target::getBuckyballTarget(); - MLIRContext *context = &getContext(); - ModuleOp module = getOperation(); - LLVMTypeConverter converter(context); - RewritePatternSet patterns(context); - LLVMConversionTarget target(*context); - - configureBuckyballLegalizeForExportTarget(target, stable); - target.addLegalDialect(); - populateBuckyballLegalizeForLLVMExportPatterns( - converter, patterns, targetConfig.bankWidthBits / 8, - targetConfig.bankDepth, targetConfig.bankNum, - /*includeFuncOperandForwarding=*/false, stable); - - ConversionConfig config; - config.allowPatternRollback = false; - if (failed(applyPartialConversion(module, target, std::move(patterns), - config))) - signalPassFailure(); - } -}; - -class LowerBankSSAToIntrinsicsPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerBankSSAToIntrinsicsPass) - LowerBankSSAToIntrinsicsPass() = default; - LowerBankSSAToIntrinsicsPass(const LowerBankSSAToIntrinsicsPass &) {} - - StringRef getArgument() const final { return "lower-bank-ssa-to-intrinsics"; } - StringRef getDescription() const final { - return "Lower Pebble bank-SSA and Buckyball ops to intrinsic ops."; - } - - Option stable{*this, "stable", - llvm::cl::desc("Use stable LLVM Buckyball intrinsics."), - llvm::cl::init(false)}; - - void getDependentDialects(DialectRegistry ®istry) const override { - registry - .insert(); - } - - void runOnOperation() override { - const auto &targetConfig = buckyball_target::getBuckyballTarget(); - MLIRContext *context = &getContext(); - ModuleOp module = getOperation(); - LLVMTypeConverter converter(context); - RewritePatternSet patterns(context); - LLVMConversionTarget target(*context); - - configureBuckyballLegalizeForExportTarget(target, stable); - target.addLegalDialect(); - populateBuckyballLegalizeForLLVMExportPatterns( - converter, patterns, targetConfig.bankWidthBits / 8, - targetConfig.bankDepth, targetConfig.bankNum, - /*includeFuncOperandForwarding=*/false, stable); - - ConversionConfig config; - config.allowPatternRollback = false; - if (failed(applyPartialConversion(module, target, std::move(patterns), - config))) - signalPassFailure(); - } -}; - -} // namespace - -void mlir::buddy::registerLowerBuckyballPass() { - PassRegistration(); -} - -void mlir::buddy::registerLowerBankSSAToIntrinsicsPass() { - PassRegistration(); -} diff --git a/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp b/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp deleted file mode 100644 index 8123157c3..000000000 --- a/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp +++ /dev/null @@ -1,125 +0,0 @@ -//===- LowerBuckyballToBankSSAPass.cpp - Pebble bank-SSA lowering -//----------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/Pass/Pass.h" -#include "mlir/Transforms/GreedyPatternRewriteDriver.h" - -#include "Buckyball/BuckyballDialect.h" -#include "Buckyball/BuckyballOps.h" -#include "Trace/TraceDialect.h" - -using namespace mlir; - -namespace mlir::buddy { -void populatePebbleCoreBankSSALoweringPatterns(RewritePatternSet &patterns, - bool traceMegaStages, - int64_t traceMegaStageStart, - int64_t traceMegaStageLimit); -void populatePebbleResidentConvRegionToBankSSAPatterns( - RewritePatternSet &patterns, bool traceMegaStages, - int64_t traceMegaStageStart, int64_t traceMegaStageLimit, - int64_t traceMegaRegion, bool traceMegaReloadStages, - bool traceMegaInputBeforeRegion); -} // namespace mlir::buddy - -namespace { - -class LowerBuckyballToBankSSAPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerBuckyballToBankSSAPass) - LowerBuckyballToBankSSAPass() = default; - LowerBuckyballToBankSSAPass(const LowerBuckyballToBankSSAPass &) {} - - StringRef getArgument() const final { return "lower-buckyball-to-bank-ssa"; } - StringRef getDescription() const final { - return "Lower Pebble Buckyball ops to explicit bank-SSA ops."; - } - - Option traceMegaStages{ - *this, "trace-mega-stages", - llvm::cl::desc("Materialize and trace leading MegaKernel stages"), - llvm::cl::init(false)}; - Option traceMegaStageLimit{ - *this, "trace-mega-stage-limit", - llvm::cl::desc("Trace only the first N MegaKernel stages (-1 means all)"), - llvm::cl::init(-1)}; - Option traceMegaStageStart{ - *this, "trace-mega-stage-start", - llvm::cl::desc("Do not dump MegaKernel stages before this index"), - llvm::cl::init(0)}; - Option traceMegaRegion{ - *this, "trace-mega-region", - llvm::cl::desc( - "Trace only this resident MegaKernel region ID (-1 means all)"), - llvm::cl::init(-1)}; - Option traceMegaReloadStages{ - *this, "trace-mega-reload-stages", - llvm::cl::desc("Reload traced stages for downstream consumers"), - llvm::cl::init(true)}; - Option traceMegaInputBeforeRegion{ - *this, "trace-mega-input-before-region", - llvm::cl::desc( - "Trace the input tensor immediately before the selected region"), - llvm::cl::init(false)}; - - void getDependentDialects(DialectRegistry ®istry) const override { - registry.insert(); - } - - void runOnOperation() override { - RewritePatternSet residentPatterns(&getContext()); - mlir::buddy::populatePebbleResidentConvRegionToBankSSAPatterns( - residentPatterns, traceMegaStages, traceMegaStageStart, - traceMegaStageLimit, traceMegaRegion, traceMegaReloadStages, - traceMegaInputBeforeRegion); - if (failed(applyPatternsGreedily(getOperation(), - std::move(residentPatterns)))) { - signalPassFailure(); - return; - } - RewritePatternSet patterns(&getContext()); - mlir::buddy::populatePebbleCoreBankSSALoweringPatterns( - patterns, traceMegaStages && traceMegaRegion < 0, traceMegaStageStart, - traceMegaStageLimit); - if (failed(applyPatternsGreedily(getOperation(), std::move(patterns)))) { - signalPassFailure(); - return; - } - bool illegalMega = false; - getOperation().walk([&](Operation *op) { - if (isa<::buddy::buckyball::MegaKernelOp>(op)) { - op->emitError( - "MegaKernel region-wide bank SSA lowering is not implemented"); - illegalMega = true; - } else if (isa<::buddy::buckyball::MegaMatmulOp, - ::buddy::buckyball::MegaConv2dOp, - ::buddy::buckyball::MegaConv2dDepthwiseOp>(op) && - !op->getParentOfType<::buddy::buckyball::MegaKernelOp>()) { - op->emitError( - "MegaKernel stage is only legal inside buckyball.mega_kernel"); - illegalMega = true; - } - }); - if (illegalMega) - signalPassFailure(); - } -}; - -} // namespace - -void mlir::buddy::registerLowerBuckyballToBankSSAPass() { - PassRegistration(); -} diff --git a/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/MegaConv2dToBankSSAPatterns.cpp b/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/MegaConv2dToBankSSAPatterns.cpp deleted file mode 100644 index b34cf74b3..000000000 --- a/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/MegaConv2dToBankSSAPatterns.cpp +++ /dev/null @@ -1,466 +0,0 @@ -//===- MegaConv2dToBankSSAPatterns.cpp - Mega Conv2D to banks -----------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Target/BuckyballTargetRegistry.h" -#include "Utils/BankUtils.h" - -#include - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { - -constexpr int64_t kTile = 16; - -template -class MegaConv2dToBankSSAPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(MegaOp op, PatternRewriter &b) const override { - Location loc = op.getLoc(); - auto inputTy = dyn_cast(op.getInput().getType()); - auto weightTy = dyn_cast(op.getWeight().getType()); - auto biasTy = dyn_cast(op.getBias().getType()); - auto scaleTy = dyn_cast(op.getScale().getType()); - auto lutTy = dyn_cast(op.getLut().getType()); - auto outputTy = dyn_cast(op.getOutput().getType()); - if (op->template getParentOfType()) - return failure(); - bool finalOutput = true; - if (!inputTy || !weightTy || !biasTy || !scaleTy || !lutTy || !outputTy || - !inputTy.hasStaticShape() || !weightTy.hasStaticShape() || - !biasTy.hasStaticShape() || !scaleTy.hasStaticShape() || - !lutTy.hasStaticShape() || !outputTy.hasStaticShape()) - return op.emitError("Mega Conv2D requires static memrefs"); - if (!inputTy.getElementType().isInteger(8) || - !weightTy.getElementType().isInteger(8) || - !biasTy.getElementType().isInteger(32) || - !scaleTy.getElementType().isF32() || - !lutTy.getElementType().isInteger(8) || lutTy.getRank() != 1 || - lutTy.getShape()[0] != 1 || op.getActivation() < 0 || - op.getActivation() > 1 || - (finalOutput ? !outputTy.getElementType().isF32() - : !outputTy.getElementType().isInteger(8))) - return op.emitError( - "Mega Conv2D input/weight must be INT8, bias INT32, scale FP32, " - "and only the last stage may output FP32"); - if (op.getPadLow() != op.getPadHigh()) - return op.emitError("Mega Conv2D requires symmetric padding"); - - auto inputShape = inputTy.getShape(); - auto weightShape = weightTy.getShape(); - auto outputShape = outputTy.getShape(); - int64_t batch = inputShape[0]; - int64_t inputSize = inputShape[1]; - int64_t cin = inputShape[3]; - int64_t kernelSize = weightShape[0]; - int64_t cout = kDepthwise ? cin : weightShape[3]; - int64_t outputSize = finalOutput ? outputShape[2] : outputShape[1]; - int64_t stride = op.getStride(); - int64_t padding = op.getPadLow(); - bool weightShapeMatches = - weightShape[1] == kernelSize && weightShape[2] == cin && - (kDepthwise ? weightShape[3] == 1 : weightShape[3] == cout); - if (batch <= 0 || inputSize <= 0 || cin <= 0 || kernelSize <= 0 || - cout <= 0 || stride <= 0 || padding < 0 || inputShape[2] != inputSize || - !weightShapeMatches || outputShape[0] != batch || - (finalOutput ? outputShape[3] != outputSize - : outputShape[2] != outputSize) || - (finalOutput ? outputShape[1] != cout : outputShape[3] != cout) || - biasTy.getShape()[0] != cout || scaleTy.getShape()[0] != cout) - return op.emitError("Mega Conv2D NHWC/HWCF shape mismatch"); - if (kernelSize > 7) - return op.emitError("Mega Conv2D kernel size exceeds Im2colBall limit 7"); - int64_t numerator = inputSize + 2 * padding - kernelSize; - if (numerator < 0 || outputSize != numerator / stride + 1) - return op.emitError("Mega Conv2D output shape mismatch"); - const auto &target = buckyball_target::getBuckyballTarget(); - if (target.bankWidthBits != 128 || target.bankDepth < 4 || - target.bankDepth % 4) - return op.emitError( - "Mega Conv2D requires 128-bit banks with depth divisible by 4"); - if (buckyball_target::getBuckyballBallMapping("SMatMulBall").outBW != 1) - return op.emitError("Mega Conv2D requires SMatMulBall outBW=1"); - int64_t kernelRows = (kernelSize * kernelSize + kTile - 1) / kTile * kTile; - int64_t windowCapacity = - std::min(target.bankDepth / 4, target.bankDepth / kernelRows * kTile); - windowCapacity = windowCapacity / kTile * kTile; - if (windowCapacity <= 0) - return op.emitError("Mega Conv2D kernel tile exceeds one bank"); - int64_t blockSide = 1; - while ((blockSide + 1) * (blockSide + 1) <= windowCapacity) - ++blockSide; - blockSide = std::min(blockSide, outputSize); - while (blockSide > 0 && (outputSize % blockSide != 0 || - ((blockSide - 1) * stride + kernelSize) * - ((blockSide - 1) * stride + kernelSize) > - target.bankDepth)) - --blockSide; - if (blockSide == 0) - return op.emitError("Mega Conv2D input tile exceeds one bank"); - int64_t blockInputSize = (blockSide - 1) * stride + kernelSize; - int64_t inputRows = blockInputSize * blockInputSize; - - Value zero = b.create(loc, 0); - Value one = b.create(loc, 1); - Value sixteen = b.create(loc, kTile); - Value outputSizeValue = b.create(loc, outputSize); - Value blockSideValue = b.create(loc, blockSide); - Value blockInputSizeValue = - b.create(loc, blockInputSize); - Value inputSizeValue = b.create(loc, inputSize); - Value coutValue = b.create(loc, cout); - Value paddedCoutValue = b.create( - loc, (cout + kTile - 1) / kTile * kTile); - Value strideValue = b.create(loc, stride); - Value paddingValue = b.create(loc, padding); - Value zeroI8 = - b.create(loc, b.getI8Type(), b.getI8IntegerAttr(0)); - - auto outputChannelTileLoop = - b.create(loc, zero, paddedCoutValue, sixteen); - b.setInsertionPointToStart(outputChannelTileLoop.getBody()); - Value n0 = outputChannelTileLoop.getInductionVar(); - auto biasPackTy = MemRefType::get({4, 4}, b.getI32Type()); - auto scalePackTy = MemRefType::get({4, 4}, b.getF32Type()); - Value biasPack = b.create(loc, biasPackTy); - Value scalePack = b.create(loc, scalePackTy); - Value zeroI32 = b.create(loc, b.getI32Type(), - b.getI32IntegerAttr(0)); - Value oneF32 = b.create(loc, b.getF32Type(), - b.getF32FloatAttr(1.0)); - b.create(loc, zeroI32, biasPack); - b.create(loc, oneF32, scalePack); - for (int64_t i = 0; i < kTile; ++i) { - Value source = b.create( - loc, n0, b.create(loc, i)); - Value row = b.create(loc, i / 4); - Value column = b.create(loc, i % 4); - Value validChannel = b.create( - loc, arith::CmpIPredicate::slt, source, coutValue); - auto storeChannel = b.create(loc, validChannel, false); - b.setInsertionPointToStart(&storeChannel.getThenRegion().front()); - Value bias = b.create(loc, op.getBias(), source); - Value scale = b.create(loc, op.getScale(), source); - b.create(loc, bias, biasPack, ValueRange{row, column}); - b.create(loc, scale, scalePack, ValueRange{row, column}); - b.setInsertionPointAfter(storeChannel); - } - - Value biasBank = allocBank(b, loc, 1, 1); - Value biasLoaded = mvinBank(b, loc, biasPack, biasBank, 4); - Value biasState = b.create( - loc, biasLoaded.getType(), biasLoaded, createI64Const(b, loc, 0)); - Value scaleBank = allocBank(b, loc, 1, 1); - Value scaleLoaded = mvinBank(b, loc, scalePack, scaleBank, 4); - - for (int64_t batchIndex = 0; batchIndex < batch; ++batchIndex) { - auto outputYLoop = - b.create(loc, zero, outputSizeValue, blockSideValue); - b.setInsertionPointToStart(outputYLoop.getBody()); - Value outputY0 = outputYLoop.getInductionVar(); - auto outputXLoop = - b.create(loc, zero, outputSizeValue, blockSideValue); - b.setInsertionPointToStart(outputXLoop.getBody()); - Value outputX0 = outputXLoop.getInductionVar(); - int64_t blockWindows = blockSide * blockSide; - int64_t paddedM = (blockWindows + kTile - 1) / kTile * kTile; - Value inputBank = allocBank(b, loc, 1, 1); - Value patchBank = allocBank(b, loc, 1, 1); - Value weightBank = allocBank(b, loc, 1, 1); - Value resultBank = allocBank(b, loc, 1, 1); - Value resultState = resultBank; - - int64_t accumulationBlocks = kDepthwise ? kTile : cin; - Value accumulationEnd = - b.create(loc, accumulationBlocks); - auto inputPacksTy = MemRefType::get( - {accumulationBlocks, inputRows, kTile}, b.getI8Type()); - auto weightPacksTy = MemRefType::get( - {accumulationBlocks, kernelRows, kTile}, b.getI8Type()); - Value inputPacks = b.create(loc, inputPacksTy); - Value weightPacks = b.create(loc, weightPacksTy); - b.create(loc, zeroI8, inputPacks); - b.create(loc, zeroI8, weightPacks); - - Value globalInputY0 = b.create( - loc, b.create(loc, outputY0, strideValue), - paddingValue); - Value globalInputX0 = b.create( - loc, b.create(loc, outputX0, strideValue), - paddingValue); - Value inputBatchValue = b.create(loc, batchIndex); - auto packChannelLoop = - b.create(loc, zero, accumulationEnd, one); - b.setInsertionPointToStart(packChannelLoop.getBody()); - Value block = packChannelLoop.getInductionVar(); - Value channelValue = - kDepthwise ? Value(b.create(loc, n0, block)) : block; - - auto inputRowLoop = - b.create(loc, zero, blockInputSizeValue, one); - b.setInsertionPointToStart(inputRowLoop.getBody()); - Value inputRow = inputRowLoop.getInductionVar(); - auto inputColumnLoop = - b.create(loc, zero, blockInputSizeValue, one); - b.setInsertionPointToStart(inputColumnLoop.getBody()); - Value inputColumn = inputColumnLoop.getInductionVar(); - Value globalInputY = - b.create(loc, globalInputY0, inputRow); - Value globalInputX = - b.create(loc, globalInputX0, inputColumn); - Value inBounds = b.create( - loc, - b.create( - loc, - b.create(loc, arith::CmpIPredicate::sge, - globalInputY, zero), - b.create(loc, arith::CmpIPredicate::slt, - globalInputY, inputSizeValue)), - b.create( - loc, - b.create(loc, arith::CmpIPredicate::sge, - globalInputX, zero), - b.create(loc, arith::CmpIPredicate::slt, - globalInputX, inputSizeValue))); - if constexpr (kDepthwise) - inBounds = b.create( - loc, inBounds, - b.create(loc, arith::CmpIPredicate::slt, - channelValue, coutValue)); - auto validInput = b.create(loc, inBounds, false); - b.setInsertionPointToStart(&validInput.getThenRegion().front()); - Value linear = b.create( - loc, b.create(loc, inputRow, blockInputSizeValue), - inputColumn); - Value inputValue = - b.create(loc, op.getInput(), - ValueRange{inputBatchValue, globalInputY, - globalInputX, channelValue}); - b.create(loc, inputValue, inputPacks, - ValueRange{block, linear, zero}); - b.setInsertionPointAfter(inputRowLoop); - - Value kernelEnd = b.create(loc, kernelSize); - auto kernelRowLoop = b.create(loc, zero, kernelEnd, one); - b.setInsertionPointToStart(kernelRowLoop.getBody()); - Value kernelRow = kernelRowLoop.getInductionVar(); - auto kernelColumnLoop = b.create(loc, zero, kernelEnd, one); - b.setInsertionPointToStart(kernelColumnLoop.getBody()); - Value kernelColumn = kernelColumnLoop.getInductionVar(); - Value weightRow = b.create( - loc, b.create(loc, kernelRow, kernelEnd), - kernelColumn); - if constexpr (kDepthwise) { - Value multiplier = b.create(loc, 0); - Value validChannel = b.create( - loc, arith::CmpIPredicate::slt, channelValue, coutValue); - auto storeWeight = b.create(loc, validChannel, false); - b.setInsertionPointToStart(&storeWeight.getThenRegion().front()); - Value weightValue = b.create( - loc, op.getWeight(), - ValueRange{kernelRow, kernelColumn, channelValue, multiplier}); - b.create(loc, weightValue, weightPacks, - ValueRange{block, weightRow, block}); - b.setInsertionPointAfter(storeWeight); - } else { - auto outputChannelLoop = b.create(loc, zero, sixteen, one); - b.setInsertionPointToStart(outputChannelLoop.getBody()); - Value outputChannel = outputChannelLoop.getInductionVar(); - Value sourceChannel = b.create(loc, n0, outputChannel); - Value validChannel = b.create( - loc, arith::CmpIPredicate::slt, sourceChannel, coutValue); - auto storeWeight = b.create(loc, validChannel, false); - b.setInsertionPointToStart(&storeWeight.getThenRegion().front()); - Value weightValue = b.create( - loc, op.getWeight(), - ValueRange{kernelRow, kernelColumn, channelValue, sourceChannel}); - b.create(loc, weightValue, weightPacks, - ValueRange{block, weightRow, outputChannel}); - b.setInsertionPointAfter(storeWeight); - } - b.setInsertionPointAfter(kernelRowLoop); - b.setInsertionPointAfter(packChannelLoop); - - auto emitAccumulation = [&](Value channel, Value inputState, - Value patchState, Value weightState, - Value accumulatorState, bool first, - bool last) -> SmallVector { - SmallVector inputOffsets = {channel, b.getIndexAttr(0), - b.getIndexAttr(0)}; - SmallVector inputSizes = {b.getIndexAttr(1), - b.getIndexAttr(inputRows), - b.getIndexAttr(kTile)}; - SmallVector strides(3, b.getIndexAttr(1)); - Value inputSlice = b.create( - loc, inputPacks, inputOffsets, inputSizes, strides); - Value inputPack = b.create( - loc, inputSlice, SmallVector{{0, 1}, {2}}); - SmallVector weightOffsets = {channel, b.getIndexAttr(0), - b.getIndexAttr(0)}; - SmallVector weightSizes = {b.getIndexAttr(1), - b.getIndexAttr(kernelRows), - b.getIndexAttr(kTile)}; - Value weightSlice = b.create( - loc, weightPacks, weightOffsets, weightSizes, strides); - Value weightPack = b.create( - loc, weightSlice, SmallVector{{0, 1}, {2}}); - - Value inputLoaded = mvinBank(b, loc, inputPack, inputState, inputRows); - Value patchLoaded = b.create( - loc, patchState.getType(), inputLoaded, patchState, - createI64Const(b, loc, blockInputSize), - createI64Const(b, loc, kernelSize), createI64Const(b, loc, stride), - createI64Const(b, loc, 0), createI64Const(b, loc, 0), - createI64Const(b, loc, 0), b.getI64IntegerAttr(0), - b.getI64IntegerAttr(0), b.getI64IntegerAttr(0), - b.getI64IntegerAttr(blockWindows)); - Value weightLoaded = - mvinBank(b, loc, weightPack, weightState, kernelRows); - Value accumulated = - b.create( - loc, accumulatorState.getType(), patchLoaded, weightLoaded, - accumulatorState, - createI64ConstU(b, loc, matrixRs2(paddedM, kTile, kernelRows)), - createI1Const(b, loc, first), createI1Const(b, loc, last), - createI64Const(b, loc, 0)) - .getWrBankOut(); - return {inputLoaded, patchLoaded, weightLoaded, accumulated}; - }; - - SmallVector states = - emitAccumulation(zero, inputBank, patchBank, weightBank, resultState, - true, accumulationBlocks == 1); - if (accumulationBlocks > 2) { - Value lastChannel = - b.create(loc, accumulationBlocks - 1); - auto middleLoop = b.create(loc, one, lastChannel, one, - ValueRange(states)); - b.setInsertionPointToStart(middleLoop.getBody()); - ValueRange iterStates = middleLoop.getRegionIterArgs(); - SmallVector next = emitAccumulation( - middleLoop.getInductionVar(), iterStates[0], iterStates[1], - iterStates[2], iterStates[3], false, false); - b.create(loc, next); - b.setInsertionPointAfter(middleLoop); - states.assign(middleLoop.getResults().begin(), - middleLoop.getResults().end()); - } - if (accumulationBlocks > 1) { - Value lastChannel = - b.create(loc, accumulationBlocks - 1); - states = emitAccumulation(lastChannel, states[0], states[1], states[2], - states[3], false, true); - } - inputBank = states[0]; - patchBank = states[1]; - weightBank = states[2]; - resultState = states[3]; - - Value outputBank = allocBank(b, loc, 1, 1); - Value converted = - finalOutput - ? b.create( - loc, outputBank.getType(), resultState, scaleLoaded, - outputBank, createI64Const(b, loc, paddedM * 4), - b.getBoolAttr(op.getActivation() == 1)) - .getOutBankOut() - : b.create( - loc, outputBank.getType(), resultState, scaleLoaded, - outputBank, createI64Const(b, loc, paddedM * 4), - createI64Const(b, loc, 0), createI64Const(b, loc, 0), - b.getI64IntegerAttr(paddedM), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(paddedM), - b.getBoolAttr(op.getActivation() == 1)) - .getOutBankOut(); - releaseBank(b, loc, resultState); - - Type outputElementType = - finalOutput ? Type(b.getF32Type()) : Type(b.getI8Type()); - auto packedOutputTy = - MemRefType::get({paddedM, kTile}, outputElementType); - Value packedOutput = b.create(loc, packedOutputTy); - Value stored = mvoutBank(b, loc, packedOutput, converted, - finalOutput ? paddedM * 4 : paddedM); - b.create(loc, inputPacks); - b.create(loc, weightPacks); - - Value outputChannelOffset = n0; - Value outputBatchValue = - b.create(loc, batchIndex); - auto outputRowLoop = b.create(loc, zero, blockSideValue, one); - b.setInsertionPointToStart(outputRowLoop.getBody()); - Value outputRow = outputRowLoop.getInductionVar(); - auto outputColumnLoop = - b.create(loc, zero, blockSideValue, one); - b.setInsertionPointToStart(outputColumnLoop.getBody()); - Value outputColumn = outputColumnLoop.getInductionVar(); - Value outputPackedRow = b.create( - loc, b.create(loc, outputRow, blockSideValue), - outputColumn); - Value outputChannelEnd = sixteen; - auto outputChannelLoop = - b.create(loc, zero, outputChannelEnd, one); - b.setInsertionPointToStart(outputChannelLoop.getBody()); - Value outputChannel = outputChannelLoop.getInductionVar(); - Value globalChannel = - b.create(loc, outputChannelOffset, outputChannel); - Value outputValue = b.create( - loc, packedOutput, ValueRange{outputPackedRow, outputChannel}); - Value globalY = b.create(loc, outputY0, outputRow); - Value globalX = b.create(loc, outputX0, outputColumn); - Value validChannel = b.create( - loc, arith::CmpIPredicate::slt, globalChannel, coutValue); - auto storeOutput = b.create(loc, validChannel, false); - b.setInsertionPointToStart(&storeOutput.getThenRegion().front()); - if (finalOutput) - b.create( - loc, outputValue, op.getOutput(), - ValueRange{outputBatchValue, globalChannel, globalY, globalX}); - else - b.create( - loc, outputValue, op.getOutput(), - ValueRange{outputBatchValue, globalY, globalX, globalChannel}); - b.setInsertionPointAfter(storeOutput); - b.setInsertionPointAfter(outputRowLoop); - - releaseBank(b, loc, stored); - b.create(loc, packedOutput); - releaseBank(b, loc, inputBank); - releaseBank(b, loc, patchBank); - releaseBank(b, loc, weightBank); - b.setInsertionPointAfter(outputXLoop); - b.setInsertionPointAfter(outputYLoop); - } - - releaseBank(b, loc, biasState); - releaseBank(b, loc, scaleLoaded); - b.create(loc, biasPack); - b.create(loc, scalePack); - b.setInsertionPointAfter(outputChannelTileLoop); - - b.eraseOp(op); - return success(); - } -}; - -} // namespace - -namespace mlir::buddy { -void populatePebbleMegaConv2dToBankSSAPatterns(RewritePatternSet &patterns) { - patterns.add, - MegaConv2dToBankSSAPattern>( - patterns.getContext()); -} -} // namespace mlir::buddy diff --git a/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/MemTransposeToBankSSAPatterns.cpp b/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/MemTransposeToBankSSAPatterns.cpp deleted file mode 100644 index 62f88c5d7..000000000 --- a/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/MemTransposeToBankSSAPatterns.cpp +++ /dev/null @@ -1,279 +0,0 @@ -//===- MemTransposeToBankSSAPatterns.cpp - mem_transpose -> Bank* ---------===// -// -// Bank-unit transpose (same model as transpose ctests): -// pad to bank-row width -> contiguous mvin(rows,stride=1) -// -> transpose(rows) -> mvout(rows,stride=1) -// Dest bank holds transposed data as stacked bank rows of width -// W = groups * (bankWidthBytes / elemBytes). -// That "16" for i8 is elems-per-bank-row from bank width, not a 16x16 tile. -// -// Packing: iter is always the mvin row count. For fat [R,C] with R<=maxW, -// pack long dim C as iter (up to bankDepth) so we do not emit iter=R tiny ops. -// -//===----------------------------------------------------------------------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Target/BuckyballTargetRegistry.h" -#include "Utils/BankUtils.h" - -#include - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { - -constexpr int64_t kBankWidthBytes = 16; -constexpr int64_t kMaxGroups = 4; - -static int64_t alignUp(int64_t x, int64_t a) { - if (a <= 0) - return x; - return (x + a - 1) / a * a; -} - -static size_t elemsPerBankRow(Type elemType) { - unsigned bitWidth = elemType.getIntOrFloatBitWidth(); - if (bitWidth == 0 || bitWidth % 8 != 0) - return 0; - return kBankWidthBytes / (bitWidth / 8); -} - -static Value zeroConst(OpBuilder &b, Location loc, Type elemTy) { - if (auto it = dyn_cast(elemTy)) - return b.create(loc, b.getIntegerAttr(it, 0)); - if (isa(elemTy)) - return b.create(loc, b.getF32FloatAttr(0.0f)); - return {}; -} - -static void emitBankTranspose(OpBuilder &b, Location loc, Value inContig, - Value outContig, int64_t rows, int64_t groups, - int64_t elemBits) { - // CPU packed inContig before mvin; CPU reads outContig after mvout. - Value src = allocBank(b, loc, 1, groups); - Value dst = allocBank(b, loc, 1, groups); - Value loaded = mvinBank(b, loc, inContig, src, rows, /*stride=*/1); - Value transposed = b.create( - loc, dst.getType(), loaded, dst, createI64Const(b, loc, rows), - createI64Const(b, loc, elemBits)); - mvoutBank(b, loc, outContig, transposed, rows, /*stride=*/1); - releaseBank(b, loc, loaded); - releaseBank(b, loc, transposed); -} - -// dst[i,j] = src[j, c0+i]; dst:[h,rows] src:[rows,cols] -static void gatherCols(OpBuilder &b, Location loc, Value src, Value dst, - int64_t rows, Value c0, int64_t h) { - Value zero = b.create(loc, 0); - Value one = b.create(loc, 1); - Value hUb = b.create(loc, h); - Value rUb = b.create(loc, rows); - auto iLoop = b.create(loc, zero, hUb, one); - { - OpBuilder::InsertionGuard g(b); - b.setInsertionPointToStart(iLoop.getBody()); - Value i = iLoop.getInductionVar(); - Value col = b.create(loc, c0, i); - auto jLoop = b.create(loc, zero, rUb, one); - b.setInsertionPointToStart(jLoop.getBody()); - Value j = jLoop.getInductionVar(); - Value v = b.create(loc, src, ValueRange{j, col}); - b.create(loc, v, dst, ValueRange{i, j}); - } -} - -// dst[c0+i, j] = src[j, i]; src:[rows,h] dst:[cols,rows] -static void scatterCols(OpBuilder &b, Location loc, Value src, Value dst, - int64_t rows, Value c0, int64_t h) { - Value zero = b.create(loc, 0); - Value one = b.create(loc, 1); - Value hUb = b.create(loc, h); - Value rUb = b.create(loc, rows); - auto iLoop = b.create(loc, zero, hUb, one); - { - OpBuilder::InsertionGuard g(b); - b.setInsertionPointToStart(iLoop.getBody()); - Value i = iLoop.getInductionVar(); - Value row = b.create(loc, c0, i); - auto jLoop = b.create(loc, zero, rUb, one); - b.setInsertionPointToStart(jLoop.getBody()); - Value j = jLoop.getInductionVar(); - Value v = b.create(loc, src, ValueRange{j, i}); - b.create(loc, v, dst, ValueRange{row, j}); - } -} - -class MemTransposeToBankSSAPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(MemTransposeOp op, - PatternRewriter &rewriter) const override { - Location loc = op.getLoc(); - const int64_t bankDepth = buckyball_target::getBuckyballTarget().bankDepth; - Value input = op.getInput(); - Value output = op.getOutput(); - auto inputType = dyn_cast(input.getType()); - auto outputType = dyn_cast(output.getType()); - if (!inputType || !outputType || !inputType.hasStaticShape() || - !outputType.hasStaticShape()) - return op.emitError("requires static input and output memrefs"); - - int64_t rows = inputType.getShape()[0]; - int64_t cols = inputType.getShape()[1]; - if (rows <= 0 || cols <= 0) - return op.emitError("transpose requires positive rows/cols"); - if (outputType.getShape()[0] != cols || outputType.getShape()[1] != rows) - return op.emitError("output shape must transpose the input shape"); - - Type elemTy = inputType.getElementType(); - int64_t elemBits = elemTy.getIntOrFloatBitWidth(); - if (elemBits != 8 && elemBits != 32) - return op.emitError("transpose only supports 8-bit or 32-bit elements"); - - size_t epr = elemsPerBankRow(elemTy); - if (epr == 0) - return op.emitError("unsupported transpose element type"); - - Value zero = zeroConst(rewriter, loc, elemTy); - if (!zero) - return op.emitError("unsupported transpose element type for pad"); - - int64_t eprI = (int64_t)epr; - int64_t rowsPad = alignUp(rows, eprI); - int64_t colsPad = alignUp(cols, eprI); - int64_t maxW = eprI * kMaxGroups; - - // A fixed full-depth tile lets the long dimension remain an scf loop. - // The extra tail is zero-padded and discarded by outView below. - bool fat = rowsPad <= maxW && colsPad > rowsPad; - bool chunkedFat = fat && colsPad > bankDepth; - int64_t colsStorage = chunkedFat ? alignUp(colsPad, bankDepth) : colsPad; - auto inPadTy = MemRefType::get({rowsPad, colsStorage}, elemTy); - auto outPadTy = MemRefType::get({colsStorage, rowsPad}, elemTy); - Value inPad = rewriter.create(loc, inPadTy); - Value outPad = rewriter.create(loc, outPadTy); - rewriter.create(loc, ValueRange{zero}, ValueRange{inPad}); - rewriter.create(loc, ValueRange{zero}, ValueRange{outPad}); - - Value inView = rewriter.create( - loc, inPad, - SmallVector{rewriter.getIndexAttr(0), - rewriter.getIndexAttr(0)}, - SmallVector{rewriter.getIndexAttr(rows), - rewriter.getIndexAttr(cols)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - rewriter.create(loc, input, inView); - - // Fat [R,C] with R fitting in bank width: pack C as iter (long side). - if (fat) { - if (rowsPad % eprI != 0) - return op.emitError("fat transpose rowsPad not bank-row aligned"); - int64_t groups = rowsPad / eprI; - if (chunkedFat) { - for (int64_t c0i = 0; c0i < colsStorage; c0i += bankDepth) { - int64_t h = std::min(bankDepth, colsStorage - c0i); - Value c0 = rewriter.create(loc, c0i); - auto inContigTy = MemRefType::get({h, rowsPad}, elemTy); - auto outContigTy = MemRefType::get({rowsPad, h}, elemTy); - Value inContig = rewriter.create(loc, inContigTy); - Value outContig = rewriter.create(loc, outContigTy); - gatherCols(rewriter, loc, inPad, inContig, rowsPad, c0, h); - emitBankTranspose(rewriter, loc, inContig, outContig, h, groups, - elemBits); - scatterCols(rewriter, loc, outContig, outPad, rowsPad, c0, h); - rewriter.create(loc, inContig); - rewriter.create(loc, outContig); - } - } else { - Value c0 = rewriter.create(loc, 0); - auto inContigTy = MemRefType::get({colsPad, rowsPad}, elemTy); - auto outContigTy = MemRefType::get({rowsPad, colsPad}, elemTy); - Value inContig = rewriter.create(loc, inContigTy); - Value outContig = rewriter.create(loc, outContigTy); - gatherCols(rewriter, loc, inPad, inContig, rowsPad, c0, colsPad); - emitBankTranspose(rewriter, loc, inContig, outContig, colsPad, groups, - elemBits); - scatterCols(rewriter, loc, outContig, outPad, rowsPad, c0, colsPad); - rewriter.create(loc, inContig); - rewriter.create(loc, outContig); - } - } else { - for (int64_t r0 = 0; r0 < rowsPad;) { - int64_t h = std::min(bankDepth, rowsPad - r0); - for (int64_t c0 = 0; c0 < colsPad;) { - int64_t w = std::min(maxW, colsPad - c0); - if (w % eprI != 0) - return op.emitError("transpose tile width not bank-row aligned"); - int64_t groups = w / eprI; - - Value inSub = rewriter.create( - loc, inPad, - SmallVector{rewriter.getIndexAttr(r0), - rewriter.getIndexAttr(c0)}, - SmallVector{rewriter.getIndexAttr(h), - rewriter.getIndexAttr(w)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - auto inContigTy = MemRefType::get({h, w}, elemTy); - Value inContig = rewriter.create(loc, inContigTy); - rewriter.create(loc, inSub, inContig); - - auto outContigTy = MemRefType::get({w, h}, elemTy); - Value outContig = rewriter.create(loc, outContigTy); - emitBankTranspose(rewriter, loc, inContig, outContig, h, groups, - elemBits); - - Value outSub = rewriter.create( - loc, outPad, - SmallVector{rewriter.getIndexAttr(c0), - rewriter.getIndexAttr(r0)}, - SmallVector{rewriter.getIndexAttr(w), - rewriter.getIndexAttr(h)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - rewriter.create(loc, outContig, outSub); - rewriter.create(loc, inContig); - rewriter.create(loc, outContig); - c0 += w; - } - r0 += h; - } - } - - Value outView = rewriter.create( - loc, outPad, - SmallVector{rewriter.getIndexAttr(0), - rewriter.getIndexAttr(0)}, - SmallVector{rewriter.getIndexAttr(cols), - rewriter.getIndexAttr(rows)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - rewriter.create(loc, outView, output); - rewriter.create(loc, inPad); - rewriter.create(loc, outPad); - rewriter.eraseOp(op); - return success(); - } -}; - -} // namespace - -namespace mlir::buddy { - -void populatePebbleMemTransposeToBankSSAPatterns(RewritePatternSet &patterns) { - patterns.add(patterns.getContext()); -} - -} // namespace mlir::buddy diff --git a/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/ResidentConvRegionToBankSSAPatterns.cpp b/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/ResidentConvRegionToBankSSAPatterns.cpp deleted file mode 100644 index 6eb52e8cf..000000000 --- a/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/ResidentConvRegionToBankSSAPatterns.cpp +++ /dev/null @@ -1,3290 +0,0 @@ -//===- ResidentConvRegionToBankSSAPatterns.cpp ---------------------------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/Matchers.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Target/BuckyballTargetRegistry.h" -#include "Trace/TraceOps.h" -#include "Utils/BankUtils.h" - -#include "llvm/ADT/DenseSet.h" -#include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/ScopeExit.h" - -#include -#include -#include - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { - -constexpr int64_t kTile = 16; -constexpr int64_t kMvin2dAddressableRows = 64; - -class ResidentConvRegionPattern : public OpRewritePattern { -public: - ResidentConvRegionPattern(MLIRContext *context, bool traceMegaStages, - int64_t traceMegaStageStart, - int64_t traceMegaStageLimit, - int64_t traceMegaRegion, bool traceMegaReloadStages, - bool traceMegaInputBeforeRegion) - : OpRewritePattern(context, 4), - traceMegaStages(traceMegaStages), - traceMegaStageStart(traceMegaStageStart), - traceMegaStageLimit(traceMegaStageLimit), - traceMegaRegion(traceMegaRegion), - traceMegaReloadStages(traceMegaReloadStages), - traceMegaInputBeforeRegion(traceMegaInputBeforeRegion) {} - - LogicalResult matchAndRewrite(MegaKernelOp kernel, - PatternRewriter &b) const override { - if (kernel.getBody().empty()) - return kernel.emitError("MegaKernel region must contain one block"); - Block &body = kernel.getBody().front(); - if (body.without_terminator().empty() || isa(body.front())) - return failure(); - - struct Stage { - Operation *op; - Value input; - Value rhs; - ValueRange inputs; - Value output; - Value weight; - Value weightChannelStride; - Value bias; - Value scale; - Value lut; - int64_t inputHeight; - int64_t inputWidth; - int64_t inputChannels; - int64_t outputHeight; - int64_t outputWidth; - int64_t outputChannels; - int64_t kernel; - int64_t stride; - int64_t padding; - int64_t activation; - float lhsScale; - float rhsScale; - float outputScale; - bool pool; - bool add; - bool multiply; - bool average; - bool depthwise; - bool channelSlice; - bool channelConcat; - bool resizeNearest; - int64_t channelOffset; - ArrayRef channelSegments; - int64_t scaleH; - int64_t scaleW; - bool finalOutput; - int64_t lutEntries; - SmallVector lutOutputScales; - }; - - SmallVector stages; - DenseMap producer; - for (Operation &operation : body.without_terminator()) { - Stage stage{}; - stage.op = &operation; - if (auto conv = dyn_cast(operation)) { - auto input = dyn_cast(conv.getInput().getType()); - auto weight = dyn_cast(conv.getWeight().getType()); - auto bias = dyn_cast(conv.getBias().getType()); - auto scale = dyn_cast(conv.getScale().getType()); - auto lut = dyn_cast(conv.getLut().getType()); - auto output = dyn_cast(conv.getOutput().getType()); - // A one-stage MegaKernel can still produce an INT8 value for the - // next kernel. Region/value identity alone does not mean this is - // the FP32 boundary; the output element type is the contract. - bool finalOutput = output && output.getElementType().isF32(); - if (!input || !weight || !bias || !scale || !lut || !output || - !input.hasStaticShape() || !weight.hasStaticShape() || - !bias.hasStaticShape() || !scale.hasStaticShape() || - !lut.hasStaticShape() || !output.hasStaticShape() || - input.getRank() != 4 || weight.getRank() != 4 || - lut.getRank() != 1 || !lut.getElementType().isInteger(8) || - output.getRank() != 4 || input.getShape()[0] != 1 || - output.getShape()[0] != 1 || !input.getElementType().isInteger(8) || - !weight.getElementType().isInteger(8) || - !bias.getElementType().isInteger(32) || - !scale.getElementType().isF32() || - (finalOutput ? !output.getElementType().isF32() - : !output.getElementType().isInteger(8)) || - conv.getActivation() < 0 || conv.getActivation() > 2 || - conv.getStride() <= 0 || conv.getPadLow() < 0 || - conv.getPadLow() != conv.getPadHigh()) - return conv.emitError("unsupported Conv stage in resident region"); - auto in = input.getShape(); - auto weights = weight.getShape(); - auto out = output.getShape(); - int64_t kernelSize = conv.getKernel(); - int64_t outputChannels = bias.getShape()[0]; - int64_t outputHeight = finalOutput ? out[2] : out[1]; - int64_t outputWidth = finalOutput ? out[3] : out[2]; - int64_t paddedKernel = - ((kernelSize * kernelSize + kTile - 1) / kTile) * kTile; - if (kernelSize <= 0 || kernelSize > 7 || - weights != ArrayRef({(outputChannels + kTile - 1) / kTile, - in[3], paddedKernel, kTile}) || - bias.getShape() != ArrayRef({outputChannels}) || - scale.getShape() != ArrayRef({outputChannels}) || - (conv.getActivation() == 2 - ? (lut.getShape()[0] != 256 && lut.getShape()[0] != 4096) - : lut.getShape()[0] != 1) || - (lut.getShape()[0] == 4096 && - (conv.getActivation() != 2 || outputChannels != 16)) || - (in[1] + 2 * conv.getPadLow() - kernelSize) / conv.getStride() + - 1 != - outputHeight || - (in[2] + 2 * conv.getPadLow() - kernelSize) / conv.getStride() + - 1 != - outputWidth || - (finalOutput ? (out[1] != outputChannels) - : (out[3] != outputChannels))) - return conv.emitError("Conv shape is inconsistent"); - stage.input = conv.getInput(); - stage.output = conv.getOutput(); - stage.weight = conv.getWeight(); - stage.bias = conv.getBias(); - stage.scale = conv.getScale(); - stage.lut = conv.getLut(); - stage.inputHeight = in[1]; - stage.inputWidth = in[2]; - stage.inputChannels = in[3]; - stage.outputHeight = outputHeight; - stage.outputWidth = outputWidth; - stage.outputChannels = outputChannels; - stage.kernel = kernelSize; - stage.stride = conv.getStride(); - stage.padding = conv.getPadLow(); - stage.activation = conv.getActivation(); - stage.outputScale = conv.getOutputScale().convertToFloat(); - stage.finalOutput = finalOutput; - stage.lutEntries = lut.getShape()[0]; - if (auto scales = operation.getAttrOfType( - "lane_output_scales")) - stage.lutOutputScales.assign(scales.asArrayRef().begin(), - scales.asArrayRef().end()); - if (!stage.lutOutputScales.empty() && - static_cast(stage.lutOutputScales.size()) != - outputChannels) - return conv.emitError( - "lane_output_scales count does not match output channels"); - } else if (auto conv = dyn_cast(operation)) { - auto input = dyn_cast(conv.getInput().getType()); - auto weight = dyn_cast(conv.getWeight().getType()); - auto bias = dyn_cast(conv.getBias().getType()); - auto scale = dyn_cast(conv.getScale().getType()); - auto lut = dyn_cast(conv.getLut().getType()); - auto output = dyn_cast(conv.getOutput().getType()); - bool finalOutput = output && output.getElementType().isF32(); - if (!input || !weight || !bias || !scale || !lut || !output || - !input.hasStaticShape() || !weight.hasStaticShape() || - !bias.hasStaticShape() || !scale.hasStaticShape() || - !lut.hasStaticShape() || !output.hasStaticShape() || - input.getRank() != 4 || weight.getRank() != 4 || - lut.getRank() != 1 || !lut.getElementType().isInteger(8) || - output.getRank() != 4 || input.getShape()[0] != 1 || - output.getShape()[0] != 1 || !input.getElementType().isInteger(8) || - !weight.getElementType().isInteger(8) || - !bias.getElementType().isInteger(32) || - !scale.getElementType().isF32() || - (finalOutput ? !output.getElementType().isF32() - : !output.getElementType().isInteger(8)) || - conv.getActivation() < 0 || conv.getActivation() > 2 || - conv.getStride() <= 0 || conv.getPadLow() < 0 || - conv.getPadLow() != conv.getPadHigh()) - return conv.emitError( - "unsupported Depthwise Conv stage in resident region"); - auto in = input.getShape(); - auto weights = weight.getShape(); - auto out = output.getShape(); - int64_t kernelSize = conv.getKernel(); - int64_t outputChannels = bias.getShape()[0]; - int64_t outputHeight = finalOutput ? out[2] : out[1]; - int64_t outputWidth = finalOutput ? out[3] : out[2]; - if (kernelSize <= 0 || kernelSize > 7 || - weights != ArrayRef({kernelSize, kernelSize, in[3], 1}) || - bias.getShape() != ArrayRef({outputChannels}) || - scale.getShape() != ArrayRef({outputChannels}) || - (conv.getActivation() == 2 - ? (lut.getShape()[0] != 256 && lut.getShape()[0] != 4096) - : lut.getShape()[0] != 1) || - lut.getShape()[0] == 4096 || in[3] != outputChannels || - (in[1] + 2 * conv.getPadLow() - kernelSize) / conv.getStride() + - 1 != - outputHeight || - (in[2] + 2 * conv.getPadLow() - kernelSize) / conv.getStride() + - 1 != - outputWidth || - (finalOutput ? (out[1] != outputChannels) - : (out[3] != outputChannels))) - return conv.emitError("Depthwise Conv shape is inconsistent"); - stage.input = conv.getInput(); - stage.output = conv.getOutput(); - stage.weight = conv.getWeight(); - stage.bias = conv.getBias(); - stage.scale = conv.getScale(); - stage.lut = conv.getLut(); - stage.inputHeight = in[1]; - stage.inputWidth = in[2]; - stage.inputChannels = in[3]; - stage.outputHeight = outputHeight; - stage.outputWidth = outputWidth; - stage.outputChannels = outputChannels; - stage.kernel = kernelSize; - stage.stride = conv.getStride(); - stage.padding = conv.getPadLow(); - stage.activation = conv.getActivation(); - stage.outputScale = conv.getOutputScale().convertToFloat(); - stage.depthwise = true; - stage.finalOutput = finalOutput; - stage.lutEntries = lut.getShape()[0]; - if (auto scales = operation.getAttrOfType( - "lane_output_scales")) - stage.lutOutputScales.assign(scales.asArrayRef().begin(), - scales.asArrayRef().end()); - if (!stage.lutOutputScales.empty() && - static_cast(stage.lutOutputScales.size()) != - outputChannels) - return conv.emitError( - "lane_output_scales count does not match output channels"); - } else if (auto pool = dyn_cast(operation)) { - auto input = dyn_cast(pool.getInput().getType()); - auto output = dyn_cast(pool.getOutput().getType()); - bool finalOutput = pool.getFinalOutput(); - if (!input || !output || !input.hasStaticShape() || - !output.hasStaticShape() || input.getRank() != 4 || - output.getRank() != 4 || input.getShape()[0] != 1 || - output.getShape()[0] != 1 || !input.getElementType().isInteger(8) || - !output.getElementType().isInteger(8) || pool.getKernel() <= 0 || - pool.getKernel() > 8 || pool.getStride() <= 0 || - pool.getPadding() < 0) - return pool.emitError("unsupported MaxPool stage in resident region"); - auto in = input.getShape(); - auto out = output.getShape(); - int64_t outputHeight = finalOutput ? out[2] : out[1]; - int64_t outputWidth = finalOutput ? out[3] : out[2]; - int64_t outputChannels = finalOutput ? out[1] : out[3]; - if (in[3] != outputChannels || - (in[1] + 2 * pool.getPadding() - pool.getKernel()) / - pool.getStride() + - 1 != - outputHeight || - (in[2] + 2 * pool.getPadding() - pool.getKernel()) / - pool.getStride() + - 1 != - outputWidth) - return pool.emitError("MaxPool shape is inconsistent"); - stage.input = pool.getInput(); - stage.output = pool.getOutput(); - stage.inputHeight = in[1]; - stage.inputWidth = in[2]; - stage.inputChannels = in[3]; - stage.outputHeight = outputHeight; - stage.outputWidth = outputWidth; - stage.outputChannels = outputChannels; - stage.kernel = pool.getKernel(); - stage.stride = pool.getStride(); - stage.padding = pool.getPadding(); - stage.pool = true; - stage.finalOutput = finalOutput; - } else if (auto add = dyn_cast(operation)) { - auto lhs = dyn_cast(add.getLhs().getType()); - auto rhs = dyn_cast(add.getRhs().getType()); - auto output = dyn_cast(add.getOutput().getType()); - float lhsScale = add.getLhsScale().convertToFloat(); - float rhsScale = add.getRhsScale().convertToFloat(); - float outputScale = add.getOutputScale().convertToFloat(); - if (!lhs || !rhs || !output || !lhs.hasStaticShape() || - !rhs.hasStaticShape() || !output.hasStaticShape() || - lhs.getRank() != 4 || rhs != lhs || output != lhs || - lhs.getShape()[0] != 1 || !lhs.getElementType().isInteger(8) || - add.getActivation() < 0 || add.getActivation() > 1 || - !std::isfinite(lhsScale) || !std::isfinite(rhsScale) || - !std::isfinite(outputScale) || lhsScale <= 0.0f || - rhsScale <= 0.0f || outputScale <= 0.0f) - return add.emitError("unsupported INT8 Add stage in resident region"); - auto shape = lhs.getShape(); - stage.input = add.getLhs(); - stage.rhs = add.getRhs(); - stage.output = add.getOutput(); - stage.inputHeight = stage.outputHeight = shape[1]; - stage.inputWidth = stage.outputWidth = shape[2]; - stage.inputChannels = stage.outputChannels = shape[3]; - stage.kernel = stage.stride = 1; - stage.activation = add.getActivation(); - stage.lhsScale = lhsScale; - stage.rhsScale = rhsScale; - stage.outputScale = outputScale; - stage.add = true; - } else if (auto multiply = dyn_cast(operation)) { - auto gate = dyn_cast(multiply.getLhs().getType()); - auto input = dyn_cast(multiply.getRhs().getType()); - auto output = dyn_cast(multiply.getOutput().getType()); - float gateScale = multiply.getLhsScale().convertToFloat(); - float inputScale = multiply.getRhsScale().convertToFloat(); - float outputScale = multiply.getOutputScale().convertToFloat(); - if (!gate || !input || !output || !gate.hasStaticShape() || - !input.hasStaticShape() || !output.hasStaticShape() || - gate.getRank() != 4 || input.getRank() != 4 || output != input || - gate.getShape()[0] != 1 || gate.getShape()[1] != 1 || - gate.getShape()[2] != 1 || input.getShape()[0] != 1 || - gate.getShape()[3] != input.getShape()[3] || - input.getShape()[3] <= 0 || - input.getShape()[3] > - buckyball_target::getBuckyballTarget().bankDepth * kTile || - !gate.getElementType().isInteger(8) || - !input.getElementType().isInteger(8) || - multiply.getActivation() != 0 || !std::isfinite(gateScale) || - !std::isfinite(inputScale) || !std::isfinite(outputScale) || - gateScale <= 0.0f || inputScale <= 0.0f || outputScale <= 0.0f) - return multiply.emitError( - "INT8 Mul requires [1,1,1,C] gate and [1,H,W,C] input, " - "1 <= C <= bankDepth*16, and activation=0"); - auto shape = input.getShape(); - stage.input = multiply.getRhs(); - stage.rhs = multiply.getLhs(); - stage.output = multiply.getOutput(); - stage.inputHeight = stage.outputHeight = shape[1]; - stage.inputWidth = stage.outputWidth = shape[2]; - stage.inputChannels = stage.outputChannels = shape[3]; - stage.kernel = stage.stride = 1; - stage.lhsScale = gateScale; - stage.rhsScale = inputScale; - stage.outputScale = outputScale; - stage.multiply = true; - } else if (auto average = dyn_cast(operation)) { - auto input = dyn_cast(average.getInput().getType()); - auto output = dyn_cast(average.getOutput().getType()); - float inputScale = average.getInputScale().convertToFloat(); - float outputScale = average.getOutputScale().convertToFloat(); - if (!input || !output || !input.hasStaticShape() || - !output.hasStaticShape() || input.getRank() != 4 || - output.getRank() != 4 || input.getShape()[0] != 1 || - input.getShape()[3] <= 0 || - input.getShape()[3] > - buckyball_target::getBuckyballTarget().bankDepth * kTile || - output.getShape() != - ArrayRef({1, 1, 1, input.getShape()[3]}) || - !input.getElementType().isInteger(8) || - !output.getElementType().isInteger(8) || - !std::isfinite(inputScale) || !std::isfinite(outputScale) || - inputScale <= 0.0f || outputScale <= 0.0f) - return average.emitError( - "unsupported GlobalAvgPool stage in resident region"); - auto shape = input.getShape(); - stage.input = average.getInput(); - stage.output = average.getOutput(); - stage.inputHeight = shape[1]; - stage.inputWidth = shape[2]; - stage.inputChannels = shape[3]; - stage.outputHeight = stage.outputWidth = 1; - stage.outputChannels = shape[3]; - stage.kernel = stage.stride = 1; - stage.lhsScale = inputScale; - stage.outputScale = outputScale; - stage.average = true; - } else if (auto slice = dyn_cast(operation)) { - auto input = dyn_cast(slice.getInput().getType()); - auto output = dyn_cast(slice.getOutput().getType()); - if (!input || !output || !input.hasStaticShape() || - !output.hasStaticShape() || input.getRank() != 4 || - output.getRank() != 4 || input.getShape()[0] != 1 || - output.getShape()[0] != 1 || !input.getElementType().isInteger(8) || - !output.getElementType().isInteger(8) || slice.getOffset() < 0 || - slice.getOffset() % kTile != 0 || - output.getShape()[3] % kTile != 0 || - input.getShape()[1] != output.getShape()[1] || - input.getShape()[2] != output.getShape()[2] || - slice.getOffset() + output.getShape()[3] > input.getShape()[3]) - return slice.emitError( - "unsupported channel slice in resident region"); - stage.input = slice.getInput(); - stage.output = slice.getOutput(); - stage.inputHeight = stage.outputHeight = input.getShape()[1]; - stage.inputWidth = stage.outputWidth = input.getShape()[2]; - stage.inputChannels = input.getShape()[3]; - stage.outputChannels = output.getShape()[3]; - stage.kernel = stage.stride = 1; - stage.channelSlice = true; - stage.channelOffset = slice.getOffset(); - } else if (auto concat = dyn_cast(operation)) { - auto output = dyn_cast(concat.getOutput().getType()); - ArrayRef segments = concat.getSegments(); - if (!output || !output.hasStaticShape() || output.getRank() != 4 || - output.getShape()[0] != 1 || - !output.getElementType().isInteger(8) || - concat.getInputs().empty() || - segments.size() != concat.getInputs().size()) - return concat.emitError( - "unsupported channel concat in resident region"); - int64_t channels = 0; - for (auto [inputValue, segment] : - llvm::zip(concat.getInputs(), segments)) { - auto input = dyn_cast(inputValue.getType()); - if (!input || !input.hasStaticShape() || input.getRank() != 4 || - input.getShape()[0] != 1 || - !input.getElementType().isInteger(8) || segment <= 0 || - segment % kTile != 0 || - input.getShape()[1] != output.getShape()[1] || - input.getShape()[2] != output.getShape()[2] || - input.getShape()[3] != segment) - return concat.emitError("channel concat shape is inconsistent"); - channels += segment; - } - if (channels != output.getShape()[3]) - return concat.emitError("channel concat channels are inconsistent"); - stage.inputs = concat.getInputs(); - stage.output = concat.getOutput(); - stage.outputHeight = output.getShape()[1]; - stage.outputWidth = output.getShape()[2]; - stage.outputChannels = output.getShape()[3]; - stage.kernel = stage.stride = 1; - stage.channelConcat = true; - stage.channelSegments = segments; - } else if (auto resize = dyn_cast(operation)) { - auto input = dyn_cast(resize.getInput().getType()); - auto output = dyn_cast(resize.getOutput().getType()); - if (!input || !output || !input.hasStaticShape() || - !output.hasStaticShape() || input.getRank() != 4 || - output.getRank() != 4 || input.getShape()[0] != 1 || - output.getShape()[0] != 1 || !input.getElementType().isInteger(8) || - !output.getElementType().isInteger(8) || resize.getScaleH() <= 0 || - resize.getScaleW() <= 0 || - input.getShape()[3] != output.getShape()[3] || - output.getShape()[1] != input.getShape()[1] * resize.getScaleH() || - output.getShape()[2] != input.getShape()[2] * resize.getScaleW()) - return resize.emitError( - "unsupported resize-nearest in resident region"); - stage.input = resize.getInput(); - stage.output = resize.getOutput(); - stage.inputHeight = input.getShape()[1]; - stage.inputWidth = input.getShape()[2]; - stage.inputChannels = input.getShape()[3]; - stage.outputHeight = output.getShape()[1]; - stage.outputWidth = output.getShape()[2]; - stage.outputChannels = output.getShape()[3]; - stage.kernel = stage.stride = 1; - stage.resizeNearest = true; - stage.scaleH = resize.getScaleH(); - stage.scaleW = resize.getScaleW(); - } else { - return operation.emitError( - "resident Conv region supports Conv, Depthwise Conv, MaxPool, " - "INT8 Add, INT8 Mul, GlobalAvgPool, channel slice, channel concat, " - "and nearest resize stages"); - } - - // A MegaKernel may have multiple external activation inputs, e.g. the - // projection branch of a residual block. Such values are captured by - // the region directly; only values produced inside this region enter - // `producer`. - if (producer.contains(stage.output)) - return operation.emitError("region tensor has multiple producers"); - producer[stage.output] = stages.size(); - stages.push_back(stage); - } - if (stages.back().output != kernel.getOutput()) - return kernel.emitError("final stage must produce MegaKernel output"); - for (auto [index, stage] : llvm::enumerate(stages)) { - if (stage.lutEntries != 4096) - continue; - int64_t consumers = 0; - for (const Stage &candidate : stages) - consumers += - candidate.input == stage.output || candidate.rhs == stage.output; - if (index != 0 || stages.size() < 2 || !stages[1].depthwise || - stages[1].input != stage.output || consumers != 1) - return stage.op->emitError( - "4096-entry lane LUT is only legal on stage 0 before one " - "16-channel depthwise Conv"); - } - - const auto &target = buckyball_target::getBuckyballTarget(); - if (target.bankWidthBits != 128 || target.bankDepth < 4 || - target.bankDepth % 4 != 0 || target.bankNum <= 0) - return kernel.emitError( - "resident Conv region requires 128-bit rows, bankDepth divisible " - "by 4, and a positive bank count"); - if (buckyball_target::getBuckyballBallMapping("SMatMulBall").outBW != 1) - return kernel.emitError("resident Conv region requires SMatMul outBW=1"); - const int64_t smatmulRows = - buckyball_target::getBuckyballBallParam("SMatMulBall", "tileRows"); - const int64_t smatmulCols = - buckyball_target::getBuckyballBallParam("SMatMulBall", "tileCols"); - if (smatmulRows <= 0 || kTile % smatmulRows != 0 || smatmulCols != kTile) - return kernel.emitError( - "resident Conv region requires SMatMul tileRows to divide 16 and " - "tileCols to equal 16"); - - Location loc = kernel.getLoc(); - const int64_t traceRegionId = - traceMegaStages ? nextTraceRegionId++ : int64_t{0}; - const bool traceThisRegion = - traceMegaStages && - (traceMegaRegion < 0 || traceRegionId == traceMegaRegion); - b.setInsertionPoint(kernel); - if (traceThisRegion && traceMegaInputBeforeRegion) { - auto id = b.getI64IntegerAttr(4095); - auto trace = b.create<::buddy::trace::EndOp>( - loc, kernel.getInput().getType(), kernel.getInput(), id, - b.getStringAttr("mega-input")); - trace->setAttr("id_path", - b.getArrayAttr({b.getI64IntegerAttr(traceRegionId), id})); - trace->setAttr("buckyball.stage_trace", b.getUnitAttr()); - } - Value zeroI8 = - b.create(loc, b.getI8Type(), b.getI8IntegerAttr(0)); - Value minI8 = b.create(loc, b.getI8Type(), - b.getI8IntegerAttr(-128)); - Value zeroI32 = b.create(loc, b.getI32Type(), - b.getI32IntegerAttr(0)); - Value oneF32 = b.create(loc, b.getF32Type(), - b.getF32FloatAttr(1.0)); - Value zero = b.create(loc, 0); - Value one = b.create(loc, 1); - SmallVector hostPacks; - Value zeroPack = b.create( - loc, MemRefType::get({target.bankDepth, kTile}, b.getI8Type())); - Value minPack = b.create( - loc, MemRefType::get({target.bankDepth, kTile}, b.getI8Type())); - hostPacks.push_back(zeroPack); - hostPacks.push_back(minPack); - b.create(loc, zeroI8, zeroPack); - b.create(loc, minI8, minPack); - - DenseMap packedBiases; - DenseMap packedScales; - DenseMap packedDepthwiseWeights; - DenseMap packedLuts; - DenseMap packedLutOutputScales; - for (Stage &stage : stages) { - if (stage.pool || stage.add || stage.multiply || stage.average || - stage.channelSlice || stage.channelConcat || stage.resizeNearest) - continue; - if (!stage.depthwise && stage.kernel == 1 && stage.stride == 1 && - stage.padding == 0 && stage.inputChannels % kTile == 0) { - auto metadata = - b.create(loc, stage.weight); - Value channelStride = metadata.getStrides()[1]; - Value contiguousLanes = b.create( - loc, arith::CmpIPredicate::eq, metadata.getStrides()[3], one); - Value strideLowerBound = b.create( - loc, arith::CmpIPredicate::sge, channelStride, - b.create(loc, kTile)); - Value strideUpperBound = b.create( - loc, arith::CmpIPredicate::sle, channelStride, - b.create(loc, 1016)); - Value strideAligned = b.create( - loc, arith::CmpIPredicate::eq, - b.create(loc, channelStride, - b.create(loc, 8)), - zero); - b.create( - loc, - b.create( - loc, - b.create(loc, contiguousLanes, strideLowerBound), - b.create(loc, strideUpperBound, strideAligned)), - "1x1 Conv weight requires contiguous lanes and a channel stride " - "between 16 and 1016 bytes divisible by 8"); - stage.weightChannelStride = - b.create(loc, b.getI64Type(), channelStride); - } - int64_t outputPanels = (stage.outputChannels + kTile - 1) / kTile; - Value biasPack = b.create( - loc, MemRefType::get({outputPanels, 4, 4}, b.getI32Type())); - Value scalePack = b.create( - loc, MemRefType::get({outputPanels, 4, 4}, b.getF32Type())); - b.create(loc, zeroI32, biasPack); - b.create(loc, oneF32, scalePack); - hostPacks.append({biasPack, scalePack}); - packedBiases[stage.op] = biasPack; - packedScales[stage.op] = scalePack; - if (stage.depthwise) { - int64_t paddedK = - (stage.kernel * stage.kernel + kTile - 1) / kTile * kTile; - Value weightPack = b.create( - loc, MemRefType::get({outputPanels, kTile, paddedK, kTile}, - b.getI8Type())); - b.create(loc, zeroI8, weightPack); - hostPacks.push_back(weightPack); - packedDepthwiseWeights[stage.op] = weightPack; - } - if (stage.finalOutput && stage.activation == 2) { - Value outputScalePack = b.create( - loc, MemRefType::get({outputPanels, 4, 4}, b.getF32Type())); - b.create(loc, oneF32, outputScalePack); - hostPacks.push_back(outputScalePack); - packedLutOutputScales[stage.op] = outputScalePack; - for (int64_t panel = 0; panel < outputPanels; ++panel) { - for (int64_t lane = 0; lane < kTile; ++lane) { - int64_t channel = panel * kTile + lane; - if (channel >= stage.outputChannels) - continue; - float outputScale = stage.lutOutputScales.empty() - ? stage.outputScale - : stage.lutOutputScales[channel]; - if (!std::isfinite(outputScale) || outputScale <= 0.0f) - return stage.op->emitError( - "LUT output scales must be finite and positive"); - b.create( - loc, - b.create(loc, b.getF32Type(), - b.getF32FloatAttr(outputScale)), - outputScalePack, - ValueRange{b.create(loc, panel), - b.create(loc, lane / 4), - b.create(loc, lane % 4)}); - } - } - } - auto outputPanelLoop = b.create( - loc, zero, b.create(loc, outputPanels), one); - b.setInsertionPointToStart(outputPanelLoop.getBody()); - Value outputPanel = outputPanelLoop.getInductionVar(); - Value channelBase = b.create( - loc, outputPanel, b.create(loc, kTile)); - auto outputLaneLoop = b.create( - loc, zero, b.create(loc, kTile), one); - b.setInsertionPointToStart(outputLaneLoop.getBody()); - Value outputLane = outputLaneLoop.getInductionVar(); - Value outputChannel = - b.create(loc, channelBase, outputLane); - auto validChannel = b.create( - loc, - b.create( - loc, arith::CmpIPredicate::slt, outputChannel, - b.create(loc, stage.outputChannels)), - false); - b.setInsertionPointToStart(&validChannel.getThenRegion().front()); - Value group = b.create( - loc, outputLane, b.create(loc, 4)); - Value groupLane = b.create( - loc, outputLane, b.create(loc, 4)); - b.create( - loc, b.create(loc, stage.bias, outputChannel), - biasPack, ValueRange{outputPanel, group, groupLane}); - b.create( - loc, b.create(loc, stage.scale, outputChannel), - scalePack, ValueRange{outputPanel, group, groupLane}); - if (stage.depthwise) { - for (int64_t ky = 0; ky < stage.kernel; ++ky) { - for (int64_t kx = 0; kx < stage.kernel; ++kx) { - Value weight = b.create( - loc, stage.weight, - ValueRange{b.create(loc, ky), - b.create(loc, kx), - outputChannel, zero}); - b.create( - loc, weight, packedDepthwiseWeights.lookup(stage.op), - ValueRange{outputPanel, outputLane, - b.create( - loc, ky * stage.kernel + kx), - outputLane}); - } - } - } - b.setInsertionPointAfter(validChannel); - b.setInsertionPointAfter(outputPanelLoop); - - if (stage.activation == 2) { - Value lutPack; - if (stage.lutEntries == 4096) { - constexpr int64_t laneLutRows = 4096 / (4 * kTile); - lutPack = b.create( - loc, MemRefType::get({laneLutRows, 4 * kTile}, b.getI8Type())); - } else { - lutPack = b.create( - loc, MemRefType::get({kTile, kTile}, b.getI8Type())); - } - hostPacks.push_back(lutPack); - packedLuts[stage.op] = lutPack; - auto lutLoop = b.create( - loc, zero, b.create(loc, stage.lutEntries), - one); - b.setInsertionPointToStart(lutLoop.getBody()); - Value index = lutLoop.getInductionVar(); - Value lutValue = b.create(loc, stage.lut, index); - if (stage.lutEntries == 4096) { - constexpr int64_t laneLutRows = 4096 / (4 * kTile); - Value group = b.create( - loc, index, - b.create(loc, laneLutRows * kTile)); - Value withinGroup = b.create( - loc, index, - b.create(loc, laneLutRows * kTile)); - b.create( - loc, lutValue, lutPack, - ValueRange{ - b.create( - loc, withinGroup, - b.create(loc, kTile)), - b.create( - loc, - b.create( - loc, group, - b.create(loc, kTile)), - b.create( - loc, withinGroup, - b.create(loc, kTile)))}); - } else { - b.create( - loc, lutValue, lutPack, - ValueRange{ - b.create( - loc, index, b.create(loc, kTile)), - b.create( - loc, index, - b.create(loc, kTile))}); - } - b.setInsertionPointAfter(lutLoop); - } - } - - // One resident zero tile is reused for all invalid-edge masking. - Value zeroBank = allocBank(b, loc, 1, 1); - zeroBank = mvinBank(b, loc, zeroPack, zeroBank, target.bankDepth); - - DenseSet materialized; - // Debug-only stages that were materialized for a stage hash. Downstream - // stages must consume the checked host tensor instead of recursively - // recomputing it, otherwise a passing hash does not isolate a boundary. - DenseSet traceMaterialized; - - struct TileBanks { - SmallVector banks; - int64_t panelRows; - int64_t panelsPerBank; - int64_t panelCount; - }; - - struct CachedTile { - int64_t stage = -1; - TileBanks tile{}; - int64_t firstPanel = 0; - Value y; - Value x; - int64_t width = 0; - }; - DenseMap gateCaches; - CachedTile residualCache; - - auto tilePanelsPerBank = [&](int64_t panelRows) { - if (panelRows <= 0 || panelRows > target.bankDepth) - return int64_t{0}; - if (panelRows > kMvin2dAddressableRows) - return int64_t{1}; - return std::min(target.bankDepth, kMvin2dAddressableRows) / - panelRows; - }; - - auto allocateTile = [&](int64_t panelCount, int64_t panelRows, Value fill) { - TileBanks tile{{}, panelRows, tilePanelsPerBank(panelRows), panelCount}; - if (tile.panelsPerBank == 0) { - kernel.emitError("resident tile does not fit one bank"); - return tile; - } - int64_t bankCount = - (panelCount + tile.panelsPerBank - 1) / tile.panelsPerBank; - for (int64_t index = 0; index < bankCount; ++index) { - int64_t panelsInBank = std::min( - tile.panelsPerBank, panelCount - index * tile.panelsPerBank); - Value bank = allocBank(b, loc, 1, 1); - tile.banks.push_back(mvinBank(b, loc, - fill == minI8 ? minPack : zeroPack, bank, - panelsInBank * panelRows)); - } - return tile; - }; - - auto releaseTile = [&](TileBanks &tile) { - for (Value bank : tile.banks) - releaseBank(b, loc, bank); - tile.banks.clear(); - }; - - auto loadInt8Tile = - [&](Value input, int64_t inputHeight, int64_t inputWidth, - int64_t inputChannels, Value y0, Value x0, Value firstPanel, - int64_t panelCount, int64_t height, int64_t width, - int64_t storagePanelRows = -1) -> FailureOr { - const int64_t panelRows = - storagePanelRows < 0 ? height * width : storagePanelRows; - TileBanks tile = allocateTile(panelCount, panelRows, zeroI8); - if (tile.banks.empty()) - return failure(); - Value yBegin = b.create( - loc, zero, b.create(loc, zero, y0)); - Value xBegin = b.create( - loc, zero, b.create(loc, zero, x0)); - Value yEnd = b.create( - loc, b.create(loc, height), - b.create( - loc, b.create(loc, inputHeight), y0)); - Value xEnd = b.create( - loc, b.create(loc, width), - b.create( - loc, b.create(loc, inputWidth), x0)); - auto inputType = cast(input.getType()); - SmallVector inputStrides; - int64_t inputOffset; - bool directMvin2d = - inputType.getElementType().isInteger(8) && - succeeded(inputType.getStridesAndOffset(inputStrides, inputOffset)) && - inputStrides[3] == 1 && inputStrides[2] == inputChannels && - inputStrides[1] == inputWidth * inputChannels && - inputChannels % 8 == 0 && inputChannels <= 1016 && - inputWidth <= 1023 && - std::min(panelCount, tile.panelsPerBank) * tile.panelRows <= - kMvin2dAddressableRows; - if (directMvin2d) { - Value eight = b.create(loc, 8); - Value pixelBytes = createI64Const(b, loc, inputChannels); - Value sourceWidth = createI64Const(b, loc, inputWidth); - for (int64_t localPanel = 0; localPanel < panelCount; ++localPanel) { - int64_t bankIndex = localPanel / tile.panelsPerBank; - int64_t slot = localPanel % tile.panelsPerBank; - Value channel = b.create( - loc, - b.create( - loc, firstPanel, - b.create(loc, localPanel)), - b.create(loc, kTile)); - Value validBytes = b.create( - loc, b.create(loc, kTile), - b.create( - loc, b.create(loc, inputChannels), - channel)); - Value validBytesI64 = - b.create(loc, b.getI64Type(), validBytes); - auto yLoop = b.create(loc, yBegin, yEnd, one, - ValueRange{tile.banks[bankIndex]}); - b.setInsertionPointToStart(yLoop.getBody()); - Value localY = yLoop.getInductionVar(); - auto xLoop = b.create(loc, xBegin, xEnd, eight, - yLoop.getRegionIterArgs()); - b.setInsertionPointToStart(xLoop.getBody()); - Value localX = xLoop.getInductionVar(); - Value globalY = b.create(loc, y0, localY); - Value globalX = b.create(loc, x0, localX); - Value copyWidth = b.create( - loc, eight, b.create(loc, xEnd, localX)); - Value slice = b.create( - loc, input, - SmallVector{b.getIndexAttr(0), globalY, globalX, - channel}, - SmallVector{b.getIndexAttr(1), b.getIndexAttr(1), - copyWidth, validBytes}, - SmallVector(4, b.getIndexAttr(1))); - Value row = b.create( - loc, b.create(loc, slot * tile.panelRows), - b.create( - loc, - b.create( - loc, localY, - b.create(loc, width)), - localX)); - Value loaded = b.create( - loc, b.getI64Type(), slice, xLoop.getRegionIterArgs().front(), - createI64Const(b, loc, 1), pixelBytes, sourceWidth, - b.create(loc, b.getI64Type(), row), - b.create(loc, b.getI64Type(), copyWidth), - validBytesI64); - b.create(loc, loaded); - b.setInsertionPointAfter(xLoop); - b.create(loc, xLoop.getResults()); - b.setInsertionPointAfter(yLoop); - tile.banks[bankIndex] = yLoop.getResult(0); - } - return tile; - } - for (size_t bankIndex = 0; bankIndex < tile.banks.size(); ++bankIndex) { - int64_t panelBegin = bankIndex * tile.panelsPerBank; - int64_t panelEnd = - std::min(panelCount, panelBegin + tile.panelsPerBank); - int64_t packedRows = (panelEnd - panelBegin) * tile.panelRows; - Value pack = b.create( - loc, MemRefType::get({packedRows, kTile}, b.getI8Type())); - b.create(loc, zeroI8, pack); - for (int64_t localPanel = panelBegin; localPanel < panelEnd; - ++localPanel) { - auto yLoop = b.create(loc, yBegin, yEnd, one); - b.setInsertionPointToStart(yLoop.getBody()); - Value localY = yLoop.getInductionVar(); - auto xLoop = b.create(loc, xBegin, xEnd, one); - b.setInsertionPointToStart(xLoop.getBody()); - Value localX = xLoop.getInductionVar(); - Value globalY = b.create(loc, y0, localY); - Value globalX = b.create(loc, x0, localX); - auto laneLoop = b.create( - loc, zero, b.create(loc, kTile), one); - b.setInsertionPointToStart(laneLoop.getBody()); - Value lane = laneLoop.getInductionVar(); - Value channel = b.create( - loc, - b.create( - loc, - b.create( - loc, firstPanel, - b.create(loc, localPanel)), - b.create(loc, kTile)), - lane); - int64_t slot = localPanel % tile.panelsPerBank; - Value row = b.create( - loc, b.create(loc, slot * tile.panelRows), - b.create( - loc, - b.create( - loc, localY, - b.create(loc, width)), - localX)); - auto channelValid = b.create( - loc, - b.create( - loc, arith::CmpIPredicate::slt, channel, - b.create(loc, inputChannels)), - false); - b.setInsertionPointToStart(&channelValid.getThenRegion().front()); - Value value = b.create( - loc, input, ValueRange{zero, globalY, globalX, channel}); - b.create(loc, value, pack, ValueRange{row, lane}); - b.setInsertionPointAfter(channelValid); - b.setInsertionPointAfter(laneLoop); - b.setInsertionPointAfter(xLoop); - b.setInsertionPointAfter(yLoop); - } - tile.banks[bankIndex] = - mvinBank(b, loc, pack, tile.banks[bankIndex], packedRows); - b.create(loc, pack); - } - return tile; - }; - - auto copyTile = [&](TileBanks &source, TileBanks &destination, - int64_t panelCount, int64_t height, int64_t width, - int64_t destinationBase, - int64_t destinationStride) -> LogicalResult { - if (source.panelCount != panelCount || source.panelRows != height * width) - return failure(); - SmallVector destinationStates(destination.banks.begin(), - destination.banks.end()); - for (int64_t localPanel = 0; localPanel < panelCount; ++localPanel) { - int64_t sourceBank = localPanel / source.panelsPerBank; - int64_t sourceSlot = localPanel % source.panelsPerBank; - int64_t destinationBank = localPanel / destination.panelsPerBank; - int64_t destinationSlot = localPanel % destination.panelsPerBank; - destinationStates[destinationBank] = - b.create( - loc, destinationStates[destinationBank].getType(), - source.banks[sourceBank], destinationStates[destinationBank], - createI64Const(b, loc, height * width), - b.getI64IntegerAttr(height), b.getI64IntegerAttr(width), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(0), - createI64Const(b, loc, sourceSlot * source.panelRows), - createI64Const(b, loc, - destinationSlot * destination.panelRows + - destinationBase), - createI64Const(b, loc, destinationStride), - b.getI64IntegerAttr(0), b.getI64IntegerAttr(0)) - .getOutBankOut(); - } - destination.banks.assign(destinationStates.begin(), - destinationStates.end()); - return success(); - }; - - auto repackSMatMulInput = [&](Value source, int64_t rows, - int64_t reduction) { - if (rows == 1 || smatmulRows == kTile || reduction == kTile) - return source; - Value packed = allocBank(b, loc, 1, 1); - int64_t reductionTiles = reduction / kTile; - for (int64_t row = 0; row < rows; row += smatmulRows) { - for (int64_t reductionTile = 0; reductionTile < reductionTiles; - ++reductionTile) { - int64_t sourceBase = reductionTile * kTile + row; - int64_t outputBase = - (row / smatmulRows * reductionTiles + reductionTile) * - smatmulRows; - // Copy contiguous groups rather than issuing one command per row. - // MAXPOOL with kernel=stride=1 is an identity copy of bank rows. - int64_t copySide = smatmulRows >= 4 ? 2 : 1; - int64_t copyRows = copySide * copySide; - for (int64_t offset = 0; offset < smatmulRows; offset += copyRows) { - packed = b.create( - loc, packed.getType(), source, packed, - createI64Const(b, loc, copyRows), - b.getI64IntegerAttr(copySide), - b.getI64IntegerAttr(copySide), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(0), - createI64Const(b, loc, sourceBase + offset), - createI64Const(b, loc, outputBase + offset), - createI64Const(b, loc, copySide), - b.getI64IntegerAttr(0), b.getI64IntegerAttr(0)) - .getOutBankOut(); - } - } - } - return packed; - }; - - auto copyFp32Tile = [&](Value source, Value destination, Value outputBase, - int64_t side, int64_t outputStride) -> Value { - // FP32 has four bank rows per spatial position. A contiguous tile can - // be copied as one square of raw rows, independent of its tensor shape. - if (outputStride == side * 4) { - int64_t rawSide = side * 2; - return b - .create( - loc, destination.getType(), source, destination, - createI64Const(b, loc, rawSide * rawSide), - b.getI64IntegerAttr(rawSide), b.getI64IntegerAttr(rawSide), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(0), createI64Const(b, loc, 0), outputBase, - createI64Const(b, loc, rawSide), b.getI64IntegerAttr(0), - b.getI64IntegerAttr(0)) - .getOutBankOut(); - } - for (int64_t y = 0; y < side; ++y) { - for (int64_t x = 0; x < side; ++x) { - Value base = b.create( - loc, outputBase, - createI64Const(b, loc, y * outputStride + x * 4)); - destination = b.create( - loc, destination.getType(), source, destination, - createI64Const(b, loc, 4), b.getI64IntegerAttr(2), - b.getI64IntegerAttr(2), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(0), - createI64Const(b, loc, (y * side + x) * 4), base, - createI64Const(b, loc, 2), b.getI64IntegerAttr(0), - b.getI64IntegerAttr(0)) - .getOutBankOut(); - } - } - return destination; - }; - - std::function - emitInto; - emitInto = [&](int64_t stageIndex, Value y0, Value x0, int64_t height, - int64_t width, Value firstPanel, int64_t panelCount, - TileBanks &destination, int64_t destinationBase, - int64_t destinationStride) -> LogicalResult { - // Keep nested stage emission before this anchor and resume after it. - Operation *insertionAnchor = - b.create(loc, 0).getOperation(); - b.setInsertionPoint(insertionAnchor); - llvm::scope_exit eraseInsertionAnchor([&]() { - b.setInsertionPointAfter(insertionAnchor); - b.eraseOp(insertionAnchor); - }); - Stage &stage = stages[stageIndex]; - const bool fp32Output = stage.finalOutput && !stage.pool; - const bool externalInput = !producer.contains(stage.input); - int64_t totalPanels = (stage.outputChannels + kTile - 1) / kTile; - if (height <= 0 || width <= 0 || panelCount <= 0 || - panelCount > totalPanels || destination.panelCount != panelCount || - destinationBase < 0 || destinationStride < width || - destinationBase + (height - 1) * destinationStride + width > - destination.panelRows) - return stage.op->emitError("invalid resident tile request"); - - if (traceMegaReloadStages && traceMaterialized.contains(stageIndex)) { - FailureOr loaded = - loadInt8Tile(stage.output, stage.outputHeight, stage.outputWidth, - stage.outputChannels, y0, x0, firstPanel, panelCount, - height, width); - if (failed(loaded)) - return stage.op->emitError("failed to reload traced resident stage"); - LogicalResult copied = - copyTile(*loaded, destination, panelCount, height, width, - destinationBase, destinationStride); - releaseTile(*loaded); - return copied; - } - - if (materialized.contains(stageIndex)) { - auto inputType = cast(stage.output.getType()); - SmallVector inputStrides; - int64_t inputOffset; - bool directMvin2d = - inputType.getElementType().isInteger(8) && - succeeded( - inputType.getStridesAndOffset(inputStrides, inputOffset)) && - inputStrides[3] == 1 && inputStrides[2] == stage.outputChannels && - inputStrides[1] == stage.outputWidth * stage.outputChannels && - stage.outputChannels % 8 == 0 && stage.outputChannels <= 1016 && - stage.outputWidth <= 1023 && - std::min(panelCount, destination.panelsPerBank) * - destination.panelRows <= - kMvin2dAddressableRows; - if (directMvin2d) { - Value yBegin = b.create(loc, y0, zero); - Value yEnd = b.create( - loc, - b.create( - loc, y0, b.create(loc, height)), - b.create(loc, stage.outputHeight)); - Value xBegin = b.create(loc, x0, zero); - Value xEnd = b.create( - loc, - b.create( - loc, x0, b.create(loc, width)), - b.create(loc, stage.outputWidth)); - Value eight = b.create(loc, 8); - Value pixelBytes = createI64Const(b, loc, stage.outputChannels); - Value sourceWidth = createI64Const(b, loc, stage.outputWidth); - for (int64_t localPanel = 0; localPanel < panelCount; ++localPanel) { - int64_t bankIndex = localPanel / destination.panelsPerBank; - int64_t bankSlot = localPanel % destination.panelsPerBank; - Value channel = b.create( - loc, - b.create( - loc, firstPanel, - b.create(loc, localPanel)), - b.create(loc, kTile)); - Value validBytes = b.create( - loc, b.create(loc, kTile), - b.create( - loc, - b.create(loc, stage.outputChannels), - channel)); - Value validBytesI64 = - b.create(loc, b.getI64Type(), validBytes); - auto yLoop = - b.create(loc, yBegin, yEnd, one, - ValueRange{destination.banks[bankIndex]}); - b.setInsertionPointToStart(yLoop.getBody()); - Value globalY = yLoop.getInductionVar(); - auto xLoop = b.create(loc, xBegin, xEnd, eight, - yLoop.getRegionIterArgs()); - b.setInsertionPointToStart(xLoop.getBody()); - Value globalX = xLoop.getInductionVar(); - Value copyWidth = b.create( - loc, eight, b.create(loc, xEnd, globalX)); - Value slice = b.create( - loc, stage.output, - SmallVector{b.getIndexAttr(0), globalY, globalX, - channel}, - SmallVector{b.getIndexAttr(1), b.getIndexAttr(1), - copyWidth, validBytes}, - SmallVector(4, b.getIndexAttr(1))); - Value destinationRow = b.create( - loc, - b.create( - loc, bankSlot * destination.panelRows + destinationBase), - b.create( - loc, - b.create( - loc, b.create(loc, globalY, y0), - b.create(loc, - destinationStride)), - b.create(loc, globalX, x0))); - Value loaded = b.create( - loc, b.getI64Type(), slice, xLoop.getRegionIterArgs().front(), - createI64Const(b, loc, 1), pixelBytes, sourceWidth, - b.create(loc, b.getI64Type(), - destinationRow), - b.create(loc, b.getI64Type(), copyWidth), - validBytesI64); - b.create(loc, loaded); - b.setInsertionPointAfter(xLoop); - b.create(loc, xLoop.getResults()); - b.setInsertionPointAfter(yLoop); - destination.banks[bankIndex] = yLoop.getResult(0); - } - return success(); - } - SmallVector packs; - for (Value bank : destination.banks) { - Value pack = b.create( - loc, MemRefType::get({target.bankDepth, kTile}, b.getI8Type())); - mvoutBank(b, loc, pack, bank, target.bankDepth); - packs.push_back(pack); - } - for (int64_t localPanel = 0; localPanel < panelCount; ++localPanel) { - int64_t bankIndex = localPanel / destination.panelsPerBank; - int64_t bankSlot = localPanel % destination.panelsPerBank; - auto yLoop = b.create( - loc, zero, b.create(loc, height), one); - b.setInsertionPointToStart(yLoop.getBody()); - Value localY = yLoop.getInductionVar(); - auto xLoop = b.create( - loc, zero, b.create(loc, width), one); - b.setInsertionPointToStart(xLoop.getBody()); - Value localX = xLoop.getInductionVar(); - Value globalY = b.create(loc, y0, localY); - Value globalX = b.create(loc, x0, localX); - Value yValid = b.create( - loc, - b.create(loc, arith::CmpIPredicate::sge, globalY, - zero), - b.create( - loc, arith::CmpIPredicate::slt, globalY, - b.create(loc, stage.outputHeight))); - Value xValid = b.create( - loc, - b.create(loc, arith::CmpIPredicate::sge, globalX, - zero), - b.create( - loc, arith::CmpIPredicate::slt, globalX, - b.create(loc, stage.outputWidth))); - auto valid = b.create( - loc, b.create(loc, yValid, xValid), false); - b.setInsertionPointToStart(&valid.getThenRegion().front()); - auto laneLoop = b.create( - loc, zero, b.create(loc, kTile), one); - b.setInsertionPointToStart(laneLoop.getBody()); - Value lane = laneLoop.getInductionVar(); - Value panel = b.create( - loc, firstPanel, - b.create(loc, localPanel)); - Value channel = b.create( - loc, - b.create( - loc, panel, b.create(loc, kTile)), - lane); - auto channelValid = b.create( - loc, - b.create( - loc, arith::CmpIPredicate::slt, channel, - b.create(loc, stage.outputChannels)), - false); - b.setInsertionPointToStart(&channelValid.getThenRegion().front()); - Value value = b.create( - loc, stage.output, ValueRange{zero, globalY, globalX, channel}); - Value row = b.create( - loc, - b.create( - loc, bankSlot * destination.panelRows + destinationBase), - b.create( - loc, - b.create( - loc, localY, - b.create(loc, destinationStride)), - localX)); - b.create(loc, value, packs[bankIndex], - ValueRange{row, lane}); - b.setInsertionPointAfter(channelValid); - b.setInsertionPointAfter(laneLoop); - b.setInsertionPointAfter(valid); - b.setInsertionPointAfter(yLoop); - } - for (size_t index = 0; index < destination.banks.size(); ++index) { - mvinBank(b, loc, packs[index], destination.banks[index], - target.bankDepth); - b.create(loc, packs[index]); - } - return success(); - } - - if (stage.channelSlice) { - IntegerAttr::ValueType requestedFirstPanel; - if (!matchPattern(firstPanel, m_ConstantInt(&requestedFirstPanel))) - return stage.op->emitError( - "resident channel slice panel offset must be constant"); - Value sourceFirstPanel = b.create( - loc, - requestedFirstPanel.getSExtValue() + stage.channelOffset / kTile); - TileBanks source = allocateTile(panelCount, height * width, zeroI8); - if (source.banks.empty()) - return failure(); - if (producer.contains(stage.input)) { - if (failed(emitInto(producer.lookup(stage.input), y0, x0, height, - width, sourceFirstPanel, panelCount, source, 0, - width))) - return failure(); - } else { - FailureOr loaded = - loadInt8Tile(stage.input, stage.inputHeight, stage.inputWidth, - stage.inputChannels, y0, x0, sourceFirstPanel, - panelCount, height, width); - if (failed(loaded)) - return stage.op->emitError( - "failed to load INT8 channel slice input"); - releaseTile(source); - source = std::move(*loaded); - } - LogicalResult copied = - copyTile(source, destination, panelCount, height, width, - destinationBase, destinationStride); - releaseTile(source); - return copied; - } - - if (stage.channelConcat) { - IntegerAttr::ValueType requestedFirstPanel; - if (!matchPattern(firstPanel, m_ConstantInt(&requestedFirstPanel))) - return stage.op->emitError( - "resident channel concat panel offset must be constant"); - int64_t requestedPanel = requestedFirstPanel.getSExtValue(); - for (int64_t localPanel = 0; localPanel < panelCount; ++localPanel) { - int64_t outputPanel = requestedPanel + localPanel; - int64_t sourcePanelBase = 0; - int64_t sourceIndex = -1; - for (auto [index, segment] : llvm::enumerate(stage.channelSegments)) { - int64_t panels = segment / kTile; - if (outputPanel >= sourcePanelBase && - outputPanel < sourcePanelBase + panels) { - sourceIndex = index; - break; - } - sourcePanelBase += panels; - } - if (sourceIndex < 0) - return stage.op->emitError( - "resident channel concat source is missing"); - Value sourceFirstPanel = b.create( - loc, outputPanel - sourcePanelBase); - TileBanks source = allocateTile(1, height * width, zeroI8); - if (source.banks.empty()) - return failure(); - Value sourceValue = stage.inputs[sourceIndex]; - if (producer.contains(sourceValue)) { - if (failed(emitInto(producer.lookup(sourceValue), y0, x0, height, - width, sourceFirstPanel, 1, source, 0, width))) - return failure(); - } else { - auto sourceType = cast(sourceValue.getType()); - FailureOr loaded = - loadInt8Tile(sourceValue, sourceType.getShape()[1], - sourceType.getShape()[2], sourceType.getShape()[3], - y0, x0, sourceFirstPanel, 1, height, width); - if (failed(loaded)) - return stage.op->emitError( - "failed to load INT8 channel concat input"); - releaseTile(source); - source = std::move(*loaded); - } - int64_t destinationBank = localPanel / destination.panelsPerBank; - int64_t destinationSlot = localPanel % destination.panelsPerBank; - destination.banks[destinationBank] = - b.create( - loc, destination.banks[destinationBank].getType(), - source.banks.front(), destination.banks[destinationBank], - createI64Const(b, loc, height * width), - b.getI64IntegerAttr(height), b.getI64IntegerAttr(width), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(0), createI64Const(b, loc, 0), - createI64Const(b, loc, - destinationSlot * destination.panelRows + - destinationBase), - createI64Const(b, loc, destinationStride), - b.getI64IntegerAttr(0), b.getI64IntegerAttr(0)) - .getOutBankOut(); - releaseTile(source); - } - return success(); - } - - if (stage.resizeNearest) { - for (int64_t localY = 0; localY < height; ++localY) { - Value sourceY = b.create( - loc, - b.create( - loc, y0, b.create(loc, localY)), - b.create(loc, stage.scaleH)); - for (int64_t localX = 0; localX < width; ++localX) { - Value sourceX = b.create( - loc, - b.create( - loc, x0, b.create(loc, localX)), - b.create(loc, stage.scaleW)); - if (producer.contains(stage.input)) { - if (failed(emitInto( - producer.lookup(stage.input), sourceY, sourceX, 1, 1, - firstPanel, panelCount, destination, - destinationBase + localY * destinationStride + localX, - destinationStride))) - return failure(); - } else { - TileBanks source = allocateTile(panelCount, 1, zeroI8); - if (source.banks.empty()) - return failure(); - FailureOr loaded = - loadInt8Tile(stage.input, stage.inputHeight, stage.inputWidth, - stage.inputChannels, sourceY, sourceX, - firstPanel, panelCount, 1, 1); - if (failed(loaded)) - return stage.op->emitError( - "failed to load INT8 resize-nearest input"); - releaseTile(source); - source = std::move(*loaded); - LogicalResult copied = copyTile( - source, destination, panelCount, 1, 1, - destinationBase + localY * destinationStride + localX, - destinationStride); - releaseTile(source); - if (failed(copied)) - return failure(); - } - } - } - return success(); - } - - CachedTile *cache = nullptr; - if (residualCache.stage == stageIndex) - cache = &residualCache; - if (cache) { - IntegerAttr::ValueType requestedFirstPanel; - if (!matchPattern(firstPanel, m_ConstantInt(&requestedFirstPanel))) - cache = nullptr; - else if (requestedFirstPanel.getSExtValue() < cache->firstPanel || - requestedFirstPanel.getSExtValue() + panelCount > - cache->firstPanel + cache->tile.panelCount) - cache = nullptr; - } - if (cache) { - IntegerAttr::ValueType requestedFirstPanel; - if (!matchPattern(firstPanel, m_ConstantInt(&requestedFirstPanel))) - return stage.op->emitError( - "cached tile panel offset must be constant"); - SmallVector destinationStates(destination.banks.begin(), - destination.banks.end()); - for (int64_t localPanel = 0; localPanel < panelCount; ++localPanel) { - int64_t sourcePanel = requestedFirstPanel.getSExtValue() - - cache->firstPanel + localPanel; - int64_t sourceBank = sourcePanel / cache->tile.panelsPerBank; - int64_t sourceSlot = sourcePanel % cache->tile.panelsPerBank; - int64_t destinationBank = localPanel / destination.panelsPerBank; - int64_t destinationSlot = localPanel % destination.panelsPerBank; - Value sourceBase = b.create( - loc, b.getI64Type(), - b.create( - loc, - b.create( - loc, sourceSlot * cache->tile.panelRows), - b.create( - loc, - b.create( - loc, b.create(loc, y0, cache->y), - b.create(loc, cache->width)), - b.create(loc, x0, cache->x)))); - destinationStates[destinationBank] = - b.create( - loc, destinationStates[destinationBank].getType(), - cache->tile.banks[sourceBank], - destinationStates[destinationBank], - createI64Const(b, loc, height * width), - b.getI64IntegerAttr(cache->width), - b.getI64IntegerAttr(width), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(0), sourceBase, - createI64Const(b, loc, - destinationSlot * destination.panelRows + - destinationBase), - createI64Const(b, loc, destinationStride), - b.getI64IntegerAttr(0), b.getI64IntegerAttr(0)) - .getOutBankOut(); - } - destination.banks.assign(destinationStates.begin(), - destinationStates.end()); - return success(); - } - - auto maskInvalidOutput = [&]() { - for (size_t bankIndex = 0; bankIndex < destination.banks.size(); - ++bankIndex) { - int64_t panelBegin = bankIndex * destination.panelsPerBank; - int64_t panelEnd = std::min( - panelCount, panelBegin + destination.panelsPerBank); - auto panelLoop = b.create( - loc, b.create(loc, panelBegin), - b.create(loc, panelEnd), one, - ValueRange{destination.banks[bankIndex]}); - b.setInsertionPointToStart(panelLoop.getBody()); - Value localPanel = panelLoop.getInductionVar(); - Value panelState = panelLoop.getRegionIterArgs().front(); - auto yLoop = b.create( - loc, zero, b.create(loc, height), one, - ValueRange{panelState}); - b.setInsertionPointToStart(yLoop.getBody()); - Value localY = yLoop.getInductionVar(); - Value yState = yLoop.getRegionIterArgs().front(); - auto xLoop = b.create( - loc, zero, b.create(loc, width), one, - ValueRange{yState}); - b.setInsertionPointToStart(xLoop.getBody()); - Value localX = xLoop.getInductionVar(); - Value xState = xLoop.getRegionIterArgs().front(); - Value globalY = b.create(loc, y0, localY); - Value globalX = b.create(loc, x0, localX); - Value yInvalid = b.create( - loc, - b.create(loc, arith::CmpIPredicate::slt, globalY, - zero), - b.create( - loc, arith::CmpIPredicate::sge, globalY, - b.create(loc, stage.outputHeight))); - Value xInvalid = b.create( - loc, - b.create(loc, arith::CmpIPredicate::slt, globalX, - zero), - b.create( - loc, arith::CmpIPredicate::sge, globalX, - b.create(loc, stage.outputWidth))); - auto invalid = b.create( - loc, b.create(loc, yInvalid, xInvalid), false); - b.setInsertionPointToStart(&invalid.getThenRegion().front()); - Value bankSlot = b.create( - loc, localPanel, - b.create(loc, panelBegin)); - Value outputBase = b.create( - loc, b.getI64Type(), - b.create( - loc, - b.create( - loc, - b.create(loc, bankSlot, - b.create( - loc, destination.panelRows)), - b.create(loc, destinationBase)), - b.create( - loc, - b.create(loc, localY, - b.create( - loc, destinationStride)), - b.create( - loc, localX, - b.create( - loc, - fp32Output && stage.activation != 2 ? 4 : 1))))); - b.create( - loc, xState.getType(), zeroBank, xState, - createI64Const(b, loc, - fp32Output && stage.activation != 2 ? 4 : 1), - b.getI64IntegerAttr(fp32Output && stage.activation != 2 ? 2 : 1), - b.getI64IntegerAttr(fp32Output && stage.activation != 2 ? 2 : 1), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(0), createI64Const(b, loc, 0), outputBase, - createI64Const(b, loc, - fp32Output && stage.activation != 2 ? 2 : 1), - b.getI64IntegerAttr(0), b.getI64IntegerAttr(0)); - b.setInsertionPointAfter(invalid); - b.create(loc, xState); - b.setInsertionPointAfter(xLoop); - b.create(loc, xLoop.getResult(0)); - b.setInsertionPointAfter(yLoop); - b.create(loc, yLoop.getResult(0)); - b.setInsertionPointAfter(panelLoop); - destination.banks[bankIndex] = panelLoop.getResult(0); - } - }; - - if (stage.add) { - Value lhsRatio = b.create( - loc, b.getF32Type(), - b.getF32FloatAttr(stage.lhsScale / stage.outputScale)); - Value rhsRatio = b.create( - loc, b.getF32Type(), - b.getF32FloatAttr(stage.rhsScale / stage.outputScale)); - - bool mainBranchExternal = !producer.contains(stage.input); - if (!mainBranchExternal) { - for (int64_t cursor = producer.lookup(stage.input); - stages[cursor].input != kernel.getInput();) { - if (!producer.contains(stages[cursor].input)) { - mainBranchExternal = true; - break; - } - int64_t next = producer.lookup(stages[cursor].input); - if (next >= cursor) - return stage.op->emitError("INT8 Add branch is not acyclic"); - cursor = next; - } - } - const bool usePanelAdd = - mainBranchExternal || !producer.contains(stage.rhs); - if (usePanelAdd) { - IntegerAttr::ValueType requestedFirstPanel; - if (!matchPattern(firstPanel, m_ConstantInt(&requestedFirstPanel))) - return stage.op->emitError( - "external INT8 Add panel offset must be constant"); - int64_t panelRows = height * width; - for (int64_t panelBegin = 0; panelBegin < panelCount;) { - int64_t destinationBank = panelBegin / destination.panelsPerBank; - int64_t firstDestinationSlot = - panelBegin % destination.panelsPerBank; - int64_t chunkCount = std::min( - {panelCount - panelBegin, tilePanelsPerBank(panelRows), - destination.panelsPerBank - firstDestinationSlot}); - Value panel = b.create( - loc, requestedFirstPanel.getSExtValue() + panelBegin); - TileBanks lhs = allocateTile(chunkCount, panelRows, zeroI8); - TileBanks rhs = allocateTile(chunkCount, panelRows, zeroI8); - if (lhs.banks.size() != 1 || rhs.banks.size() != 1) - return failure(); - if (producer.contains(stage.input)) { - if (failed(emitInto(producer.lookup(stage.input), y0, x0, height, - width, panel, chunkCount, lhs, 0, width))) - return failure(); - } else { - auto input = cast(stage.input.getType()); - FailureOr loaded = - loadInt8Tile(stage.input, input.getShape()[1], - input.getShape()[2], input.getShape()[3], y0, x0, - panel, chunkCount, height, width); - if (failed(loaded)) - return stage.op->emitError("failed to load INT8 Add lhs"); - releaseTile(lhs); - lhs = std::move(*loaded); - } - if (producer.contains(stage.rhs)) { - if (failed(emitInto(producer.lookup(stage.rhs), y0, x0, height, - width, panel, chunkCount, rhs, 0, width))) - return failure(); - } else { - auto input = cast(stage.rhs.getType()); - FailureOr loaded = - loadInt8Tile(stage.rhs, input.getShape()[1], - input.getShape()[2], input.getShape()[3], y0, x0, - panel, chunkCount, height, width); - if (failed(loaded)) - return stage.op->emitError("failed to load INT8 Add rhs"); - releaseTile(rhs); - rhs = std::move(*loaded); - } - TileBanks sum = allocateTile(chunkCount, panelRows, zeroI8); - if (sum.banks.size() != 1) - return failure(); - sum.banks.front() = b.create( - loc, sum.banks.front().getType(), lhs.banks.front(), - rhs.banks.front(), sum.banks.front(), - createI64Const(b, loc, target.bankDepth), lhsRatio, rhsRatio, - b.getBoolAttr(stage.activation == 1)); - for (int64_t localPanel = 0; localPanel < chunkCount; - ++localPanel) { - int64_t destinationSlot = firstDestinationSlot + localPanel; - destination.banks[destinationBank] = - b.create( - loc, destination.banks[destinationBank].getType(), - sum.banks.front(), destination.banks[destinationBank], - createI64Const(b, loc, height * width), - b.getI64IntegerAttr(height), b.getI64IntegerAttr(width), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(0), - createI64Const(b, loc, localPanel * panelRows), - createI64Const(b, loc, - destinationSlot * destination.panelRows + - destinationBase), - createI64Const(b, loc, destinationStride), - b.getI64IntegerAttr(0), b.getI64IntegerAttr(0)) - .getOutBankOut(); - } - releaseTile(lhs); - releaseTile(rhs); - releaseTile(sum); - panelBegin += chunkCount; - } - return success(); - } - - int64_t branchStage = producer.lookup(stage.input); - DenseSet branchAncestors; - for (int64_t cursor = branchStage;;) { - branchAncestors.insert(cursor); - Value input = stages[cursor].input; - if (input == kernel.getInput()) - break; - if (!producer.contains(input)) - return stage.op->emitError( - "INT8 Add main branch has no region producer"); - int64_t next = producer.lookup(input); - if (next >= cursor) - return stage.op->emitError("INT8 Add branch is not acyclic"); - cursor = next; - } - int64_t residualStage = producer.lookup(stage.rhs); - bool residualHasCommonProducer = true; - while (!branchAncestors.contains(residualStage)) { - Value input = stages[residualStage].input; - if (!producer.contains(input)) { - residualHasCommonProducer = false; - break; - } - int64_t next = producer.lookup(input); - if (next >= residualStage) - return stage.op->emitError("INT8 Add branch is not acyclic"); - residualStage = next; - } - Value residualY = y0; - Value residualX = x0; - int64_t residualHeight = height; - int64_t residualWidth = width; - for (int64_t cursor = branchStage; - residualHasCommonProducer && cursor != residualStage;) { - Stage &branch = stages[cursor]; - if (branch.add || branch.average || - branch.input == kernel.getInput() || - !producer.contains(branch.input)) - return stage.op->emitError( - "INT8 Add main branch must be a linear Conv chain from rhs"); - residualY = b.create( - loc, - b.create( - loc, residualY, - b.create(loc, branch.stride)), - b.create(loc, branch.padding)); - residualX = b.create( - loc, - b.create( - loc, residualX, - b.create(loc, branch.stride)), - b.create(loc, branch.padding)); - residualHeight = (residualHeight - 1) * branch.stride + branch.kernel; - residualWidth = (residualWidth - 1) * branch.stride + branch.kernel; - int64_t next = producer.lookup(branch.input); - if (next >= cursor) - return stage.op->emitError("INT8 Add branch is not acyclic"); - cursor = next; - } - int64_t residualPanelRows = residualHeight * residualWidth; - int64_t residualPanels = - (stages[residualStage].outputChannels + kTile - 1) / kTile; - int64_t residualPanelsPerBank = tilePanelsPerBank(residualPanelRows); - if (residualPanelsPerBank <= 0) - return stage.op->emitError( - "INT8 Add residual tile does not fit one bank"); - int64_t chunkPanels = std::min( - residualPanels, - std::min(destination.panelsPerBank, residualPanelsPerBank)); - if (chunkPanels <= 0) - return stage.op->emitError("INT8 Add panel chunk is empty"); - - for (int64_t panelBegin = 0; panelBegin < panelCount; - panelBegin += chunkPanels) { - int64_t chunkCount = - std::min(chunkPanels, panelCount - panelBegin); - Value chunkFirstPanel = firstPanel; - if (panelBegin != 0) { - IntegerAttr::ValueType firstPanelValue; - if (!matchPattern(firstPanel, m_ConstantInt(&firstPanelValue))) - return stage.op->emitError( - "INT8 Add panel offset must be constant"); - chunkFirstPanel = b.create( - loc, firstPanelValue.getSExtValue() + panelBegin); - } - - TileBanks lhs = - allocateTile(chunkCount, destination.panelRows, zeroI8); - TileBanks rhs = - allocateTile(chunkCount, destination.panelRows, zeroI8); - if (lhs.banks.size() != 1 || rhs.banks.size() != 1) - return stage.op->emitError( - "INT8 Add branch panel chunk must fit one bank"); - if (failed(emitInto(producer.lookup(stage.input), y0, x0, height, - width, chunkFirstPanel, chunkCount, lhs, 0, - destinationStride))) - return failure(); - - TileBanks residual = - allocateTile(chunkCount, residualPanelRows, zeroI8); - if (residual.banks.empty() || - failed(emitInto(residualStage, residualY, residualX, - residualHeight, residualWidth, chunkFirstPanel, - chunkCount, residual, 0, residualWidth))) - return failure(); - residualCache.stage = residualStage; - residualCache.tile = residual; - IntegerAttr::ValueType cachedFirstPanel; - if (!matchPattern(chunkFirstPanel, m_ConstantInt(&cachedFirstPanel))) - return stage.op->emitError( - "INT8 Add residual panel offset must be constant"); - residualCache.firstPanel = cachedFirstPanel.getSExtValue(); - residualCache.y = residualY; - residualCache.x = residualX; - residualCache.width = residualWidth; - if (failed(emitInto(producer.lookup(stage.rhs), y0, x0, height, width, - chunkFirstPanel, chunkCount, rhs, 0, - destinationStride))) - return failure(); - residualCache.stage = -1; - residualCache.tile.banks.clear(); - releaseTile(residual); - - TileBanks sum = - allocateTile(chunkCount, destination.panelRows, zeroI8); - if (sum.banks.size() != 1) - return stage.op->emitError( - "INT8 Add output panel chunk must fit one bank"); - b.create( - loc, sum.banks.front().getType(), lhs.banks.front(), - rhs.banks.front(), sum.banks.front(), - createI64Const(b, loc, target.bankDepth), lhsRatio, rhsRatio, - b.getBoolAttr(stage.activation == 1)); - int64_t destinationBank = panelBegin / destination.panelsPerBank; - Value destinationState = destination.banks[destinationBank]; - // BankInt8Add packs all panels in the chunk into one bank. Copy - // each panel's spatial rows into its destination slot explicitly. - for (int64_t localPanel = 0; localPanel < chunkCount; ++localPanel) { - int64_t destinationSlot = - (panelBegin + localPanel) % destination.panelsPerBank; - destinationState = - b.create( - loc, destinationState.getType(), sum.banks.front(), - destinationState, createI64Const(b, loc, height * width), - b.getI64IntegerAttr(height), b.getI64IntegerAttr(width), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(0), - createI64Const(b, loc, localPanel * destination.panelRows), - createI64Const(b, loc, - destinationSlot * destination.panelRows + - destinationBase), - createI64Const(b, loc, destinationStride), - b.getI64IntegerAttr(0), b.getI64IntegerAttr(0)) - .getOutBankOut(); - } - destination.banks[destinationBank] = destinationState; - releaseTile(lhs); - releaseTile(rhs); - releaseTile(sum); - } - maskInvalidOutput(); - return success(); - } - - if (stage.multiply) { - if (height * width > target.bankDepth) - return stage.op->emitError("INT8 Mul tile exceeds one bank"); - Value ratio = b.create( - loc, b.getF32Type(), - b.getF32FloatAttr(stage.lhsScale * stage.rhsScale / - stage.outputScale)); - int64_t gateStage = producer.lookup(stage.rhs); - auto gateIt = gateCaches.find(gateStage); - if (gateIt == gateCaches.end()) { - InFlightDiagnostic diagnostic = - stage.op->emitError("INT8 Mul gate was not precomputed"); - diagnostic << " (multiplyStage=" << stageIndex - << ", gateStage=" << gateStage << ", cachedStages="; - for (const auto &entry : gateCaches) - diagnostic << entry.first << " "; - diagnostic << ")"; - return failure(); - } - TileBanks &gate = gateIt->second; - if (gate.panelCount < panelCount || gate.banks.size() != 1) - return stage.op->emitError("INT8 Mul gate cache layout mismatch"); - - for (size_t destinationBank = 0; - destinationBank < destination.banks.size(); ++destinationBank) { - int64_t panelBegin = destinationBank * destination.panelsPerBank; - int64_t panelEnd = std::min( - panelCount, panelBegin + destination.panelsPerBank); - auto panelLoop = b.create( - loc, b.create(loc, panelBegin), - b.create(loc, panelEnd), one, - ValueRange{destination.banks[destinationBank]}); - b.setInsertionPointToStart(panelLoop.getBody()); - Value localPanel = panelLoop.getInductionVar(); - Value destinationState = panelLoop.getRegionIterArgs().front(); - Value panel = b.create(loc, firstPanel, localPanel); - TileBanks input; - if (producer.contains(stage.input)) { - input = allocateTile(1, height * width, zeroI8); - if (failed(emitInto(producer.lookup(stage.input), y0, x0, height, - width, panel, 1, input, 0, width))) - return failure(); - } else { - FailureOr loaded = loadInt8Tile( - stage.input, stage.inputHeight, stage.inputWidth, - stage.inputChannels, y0, x0, panel, 1, height, width); - if (failed(loaded)) - return stage.op->emitError("failed to load INT8 Mul input"); - input = std::move(*loaded); - } - Value multiplied = allocBank(b, loc, 1, 1); - multiplied = - b.create( - loc, multiplied.getType(), gate.banks.front(), - input.banks.front(), multiplied, - createI64Const(b, loc, height * width), ratio, - b.create(loc, b.getI64Type(), panel)) - .getOutputBankOut(); - - Value destinationSlot = b.create( - loc, localPanel, - b.create(loc, panelBegin)); - Value outputBase = b.create( - loc, b.getI64Type(), - b.create( - loc, - b.create(loc, destinationSlot, - b.create( - loc, destination.panelRows)), - b.create(loc, destinationBase))); - Value destinationNext = - b.create( - loc, destinationState.getType(), multiplied, - destinationState, createI64Const(b, loc, height * width), - b.getI64IntegerAttr(width), b.getI64IntegerAttr(width), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(0), createI64Const(b, loc, 0), - outputBase, createI64Const(b, loc, destinationStride), - b.getI64IntegerAttr(0), b.getI64IntegerAttr(0)) - .getOutBankOut(); - releaseTile(input); - releaseBank(b, loc, multiplied); - b.create(loc, destinationNext); - b.setInsertionPointAfter(panelLoop); - destination.banks[destinationBank] = panelLoop.getResult(0); - } - maskInvalidOutput(); - return success(); - } - - if (stage.average) { - int64_t inputRows = stage.inputHeight * stage.inputWidth; - int64_t paddedInputRows = ((inputRows + kTile - 1) / kTile) * kTile; - bool fitsOneBank = - paddedInputRows <= target.bankDepth && paddedInputRows <= 0xfff; - if (!fitsOneBank && !externalInput && - !materialized.contains(producer.lookup(stage.input))) - return stage.op->emitError( - "large GlobalAvgPool requires a materialized input"); - int64_t sumK = fitsOneBank ? paddedInputRows : kTile; - Value oneI8 = b.create(loc, b.getI8Type(), - b.getI8IntegerAttr(1)); - Value ratio = b.create( - loc, b.getF32Type(), - b.getF32FloatAttr(stage.lhsScale / - (inputRows * stage.outputScale))); - Value onesPack = b.create( - loc, MemRefType::get({sumK / kTile, kTile}, b.getI8Type())); - Value biasPack = b.create( - loc, MemRefType::get({4, 4}, b.getI32Type())); - Value scalePack = b.create( - loc, MemRefType::get({4, 4}, b.getF32Type())); - b.create(loc, oneI8, onesPack); - b.create(loc, zeroI32, biasPack); - b.create(loc, ratio, scalePack); - - auto panelLoop = b.create( - loc, zero, b.create(loc, panelCount), one, - ValueRange{destination.banks.front()}); - b.setInsertionPointToStart(panelLoop.getBody()); - Value localPanel = panelLoop.getInductionVar(); - Value destinationState = panelLoop.getRegionIterArgs().front(); - Value panel = b.create(loc, firstPanel, localPanel); - - TileBanks fullSource; - if (fitsOneBank) { - fullSource = allocateTile(1, sumK, zeroI8); - if (externalInput) { - FailureOr loaded = - loadInt8Tile(stage.input, stage.inputHeight, stage.inputWidth, - stage.inputChannels, zero, zero, panel, 1, - stage.inputHeight, stage.inputWidth, sumK); - if (failed(loaded)) - return stage.op->emitError( - "failed to load INT8 GlobalAvgPool input"); - releaseTile(fullSource); - fullSource = std::move(*loaded); - } else if (failed(emitInto(producer.lookup(stage.input), zero, zero, - stage.inputHeight, stage.inputWidth, panel, - 1, fullSource, 0, stage.inputWidth))) { - return failure(); - } - } - - Value onesBank = allocBank(b, loc, 1, 1); - Value onesLoaded = mvinBank(b, loc, onesPack, onesBank, sumK / kTile); - Value biasBank = allocBank(b, loc, 1, 1); - Value biasLoaded = mvinBank(b, loc, biasPack, biasBank, 4); - Value biasState = b.create( - loc, biasLoaded.getType(), biasLoaded, createI64Const(b, loc, 0)); - releaseBank(b, loc, biasState); - Value scaleBank = allocBank(b, loc, 1, 1); - Value scaleLoaded = mvinBank(b, loc, scalePack, scaleBank, 4); - Value result = allocBank(b, loc, 1, 1); - - Value resultState; - if (fitsOneBank) { - resultState = - b.create( - loc, result.getType(), onesLoaded, fullSource.banks.front(), - result, createI64ConstU(b, loc, matrixRs2(1, kTile, sumK)), - createI1Const(b, loc, true), createI1Const(b, loc, true), - createI64Const(b, loc, 0)) - .getWrBankOut(); - releaseTile(fullSource); - } else { - Value four = b.create(loc, 4); - auto yLoop = b.create( - loc, zero, - b.create(loc, stage.inputHeight), four, - ValueRange{result}); - b.setInsertionPointToStart(yLoop.getBody()); - Value y = yLoop.getInductionVar(); - Value yState = yLoop.getRegionIterArgs().front(); - auto xLoop = b.create( - loc, zero, - b.create(loc, stage.inputWidth), four, - ValueRange{yState}); - b.setInsertionPointToStart(xLoop.getBody()); - Value x = xLoop.getInductionVar(); - Value xState = xLoop.getRegionIterArgs().front(); - TileBanks source = allocateTile(1, kTile, zeroI8); - if (externalInput) { - FailureOr loaded = - loadInt8Tile(stage.input, stage.inputHeight, stage.inputWidth, - stage.inputChannels, y, x, panel, 1, 4, 4); - if (failed(loaded)) - return stage.op->emitError( - "failed to load INT8 GlobalAvgPool input"); - releaseTile(source); - source = std::move(*loaded); - } else if (failed(emitInto(producer.lookup(stage.input), y, x, 4, 4, - panel, 1, source, 0, 4))) { - return failure(); - } - Value first = b.create( - loc, - b.create(loc, arith::CmpIPredicate::eq, y, zero), - b.create(loc, arith::CmpIPredicate::eq, x, zero)); - Value last = b.create( - loc, - b.create( - loc, arith::CmpIPredicate::sge, - b.create(loc, y, four), - b.create(loc, stage.inputHeight)), - b.create( - loc, arith::CmpIPredicate::sge, - b.create(loc, x, four), - b.create(loc, stage.inputWidth))); - Value resultNext = - b.create( - loc, xState.getType(), onesLoaded, source.banks.front(), - xState, createI64ConstU(b, loc, matrixRs2(1, kTile, kTile)), - first, last, createI64Const(b, loc, 0)) - .getWrBankOut(); - releaseTile(source); - b.create(loc, resultNext); - b.setInsertionPointAfter(xLoop); - b.create(loc, xLoop.getResult(0)); - b.setInsertionPointAfter(yLoop); - resultState = yLoop.getResult(0); - } - releaseBank(b, loc, onesLoaded); - - Value quantized = allocBank(b, loc, 1, 1); - quantized = b.create( - loc, quantized.getType(), resultState, scaleLoaded, - quantized, createI64Const(b, loc, 4), - createI64Const(b, loc, 0), createI64Const(b, loc, 0), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(1), b.getBoolAttr(false)) - .getOutBankOut(); - releaseBank(b, loc, resultState); - releaseBank(b, loc, scaleLoaded); - Value outputBase = - b.create(loc, b.getI64Type(), localPanel); - Value destinationNext = - b.create( - loc, destinationState.getType(), quantized, destinationState, - createI64Const(b, loc, 1), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(0), - createI64Const(b, loc, 0), outputBase, - createI64Const(b, loc, 1), b.getI64IntegerAttr(0), - b.getI64IntegerAttr(0)) - .getOutBankOut(); - releaseBank(b, loc, quantized); - b.create(loc, destinationNext); - b.setInsertionPointAfter(panelLoop); - destination.banks.front() = panelLoop.getResult(0); - b.create(loc, onesPack); - b.create(loc, biasPack); - b.create(loc, scalePack); - return success(); - } - - bool streamMaterializedInput = - !stage.pool && !stage.depthwise && !externalInput && - materialized.contains(producer.lookup(stage.input)); - int64_t maxSide = std::min({4, height, width}); - int64_t lastInputBanks = -1; - int64_t lastPanelsPerBank = -1; - int64_t lastInputSide = -1; - while (maxSide > 0) { - int64_t inputSide = (maxSide - 1) * stage.stride + stage.kernel; - int64_t inputPanelRows = inputSide * inputSide; - // A recursively produced tile is assembled from subtiles using - // six-bit row bases. DMA-loaded tiles and long MatMul K blocks may - // use the full SRAM depth, but recursive scratch tiles must fit 64. - bool recursiveInput = - !externalInput && - !materialized.contains(producer.lookup(stage.input)) && - !traceMaterialized.contains(producer.lookup(stage.input)); - if (recursiveInput && inputPanelRows > kMvin2dAddressableRows) { - --maxSide; - continue; - } - int64_t inputPanels = stage.depthwise - ? panelCount - : (stage.inputChannels + kTile - 1) / kTile; - int64_t panelsPerBank = tilePanelsPerBank(inputPanelRows); - lastInputSide = inputSide; - lastPanelsPerBank = panelsPerBank; - int64_t inputBanks = - panelsPerBank == 0 ? target.bankNum + 1 - : externalInput ? (inputPanels + panelsPerBank - 1) / panelsPerBank - : streamMaterializedInput ? 1 - : stage.depthwise - ? 1 - : (inputPanels + panelsPerBank - 1) / panelsPerBank; - lastInputBanks = inputBanks; - int64_t reservedBanks = externalInput ? 6 : 11; - if (stage.activation == 2) - reservedBanks += 2; - if (inputBanks + static_cast(destination.banks.size()) + - reservedBanks <= - target.bankNum) - break; - --maxSide; - } - if (maxSide == 0) { - InFlightDiagnostic diagnostic = stage.op->emitError( - "resident stage tile exceeds physical bank capacity"); - diagnostic << " (request=" << height << "x" << width - << ", panels=" << panelCount - << ", destinationBanks=" << destination.banks.size() - << ", bankDepth=" << target.bankDepth - << ", bankNum=" << target.bankNum - << ", inputSide=" << lastInputSide - << ", inputBanks=" << lastInputBanks - << ", panelsPerBank=" << lastPanelsPerBank << ")"; - return failure(); - } - int64_t side = std::min({maxSide, height, width}); - if (height != side || width != side) { - if (failed(emitInto(stageIndex, y0, x0, side, side, firstPanel, - panelCount, destination, destinationBase, - destinationStride))) - return failure(); - Value nextX = b.create( - loc, x0, b.create(loc, side)); - if (width > side && - failed(emitInto( - stageIndex, y0, nextX, side, width - side, firstPanel, - panelCount, destination, - destinationBase + - side * (fp32Output && stage.activation != 2 ? 4 : 1), - destinationStride))) - return failure(); - Value nextY = b.create( - loc, y0, b.create(loc, side)); - if (height > side && - failed(emitInto(stageIndex, nextY, x0, height - side, width, - firstPanel, panelCount, destination, - destinationBase + side * destinationStride, - destinationStride))) - return failure(); - return success(); - } - - int64_t inputSide = (side - 1) * stage.stride + stage.kernel; - Value sourceY = b.create( - loc, - b.create( - loc, y0, b.create(loc, stage.stride)), - b.create(loc, stage.padding)); - Value sourceX = b.create( - loc, - b.create( - loc, x0, b.create(loc, stage.stride)), - b.create(loc, stage.padding)); - int64_t inputPanelCount = (stage.pool || stage.depthwise) - ? panelCount - : (stage.inputChannels + kTile - 1) / kTile; - TileBanks source; - if (externalInput && !stage.pool) { - source.panelRows = inputSide * inputSide; - // Keep the external input layout simple: one channel panel per bank. - // This also makes the bank lifetime explicit when Cin is large. - source.panelsPerBank = tilePanelsPerBank(source.panelRows); - source.panelCount = inputPanelCount; - if (source.panelRows <= 0 || source.panelRows > target.bankDepth) - return stage.op->emitError( - "resident region input panel does not fit one bank"); - } else if (stage.pool) { - // Pooling preserves channels. Stream input panels one bank at a time - // instead of pinning the whole channel tile (e.g. 16 banks for 256 C). - int64_t sourcePanelsPerBank = tilePanelsPerBank(inputSide * inputSide); - if (sourcePanelsPerBank <= 0) - return stage.op->emitError( - "resident pool input panel does not fit bank"); - for (int64_t sourcePanelBegin = 0; sourcePanelBegin < inputPanelCount; - sourcePanelBegin += sourcePanelsPerBank) { - int64_t sourcePanelEnd = std::min( - inputPanelCount, sourcePanelBegin + sourcePanelsPerBank); - TileBanks sourceChunk = allocateTile( - sourcePanelEnd - sourcePanelBegin, inputSide * inputSide, minI8); - if (sourceChunk.banks.size() != 1) - return stage.op->emitError( - "resident pool input chunk must fit one bank"); - if (externalInput) { - Value pack = b.create( - loc, MemRefType::get({target.bankDepth, kTile}, b.getI8Type())); - b.create(loc, minI8, pack); - auto yLoop = b.create( - loc, zero, b.create(loc, inputSide), - one); - b.setInsertionPointToStart(yLoop.getBody()); - Value localY = yLoop.getInductionVar(); - auto xLoop = b.create( - loc, zero, b.create(loc, inputSide), - one); - b.setInsertionPointToStart(xLoop.getBody()); - Value globalY = b.create(loc, sourceY, localY); - Value globalX = - b.create(loc, sourceX, xLoop.getInductionVar()); - Value yValid = b.create( - loc, - b.create(loc, arith::CmpIPredicate::sge, globalY, - zero), - b.create( - loc, arith::CmpIPredicate::slt, globalY, - b.create(loc, stage.inputHeight))); - Value xValid = b.create( - loc, - b.create(loc, arith::CmpIPredicate::sge, globalX, - zero), - b.create( - loc, arith::CmpIPredicate::slt, globalX, - b.create(loc, stage.inputWidth))); - auto valid = b.create( - loc, b.create(loc, yValid, xValid), false); - b.setInsertionPointToStart(&valid.getThenRegion().front()); - auto laneLoop = b.create( - loc, - b.create(loc, sourcePanelBegin * kTile), - b.create(loc, sourcePanelEnd * kTile), - one); - b.setInsertionPointToStart(laneLoop.getBody()); - Value channel = laneLoop.getInductionVar(); - Value inputChannel = b.create( - loc, channel, - b.create( - loc, firstPanel, - b.create(loc, kTile))); - auto channelValid = b.create( - loc, - b.create( - loc, arith::CmpIPredicate::slt, inputChannel, - b.create(loc, stage.inputChannels)), - false); - b.setInsertionPointToStart(&channelValid.getThenRegion().front()); - Value value = b.create( - loc, stage.input, - ValueRange{zero, globalY, globalX, inputChannel}); - Value row = b.create( - loc, - b.create( - loc, - b.create( - loc, - b.create( - loc, channel, - b.create( - loc, sourcePanelBegin * kTile)), - b.create(loc, kTile)), - b.create(loc, - inputSide * inputSide)), - b.create( - loc, - b.create( - loc, localY, - b.create(loc, inputSide)), - xLoop.getInductionVar())); - b.create( - loc, value, pack, - ValueRange{row, - b.create( - loc, channel, - b.create(loc, kTile))}); - b.setInsertionPointAfter(channelValid); - b.setInsertionPointAfter(laneLoop); - b.setInsertionPointAfter(valid); - b.setInsertionPointAfter(xLoop); - b.setInsertionPointAfter(yLoop); - sourceChunk.banks.front() = mvinBank( - b, loc, pack, sourceChunk.banks.front(), target.bankDepth); - b.create(loc, pack); - } else if (failed(emitInto(producer.lookup(stage.input), sourceY, - sourceX, inputSide, inputSide, - b.create( - loc, firstPanel, - b.create( - loc, sourcePanelBegin)), - sourcePanelEnd - sourcePanelBegin, - sourceChunk, 0, inputSide))) - return failure(); - for (int64_t localPanel = sourcePanelBegin; - localPanel < sourcePanelEnd; ++localPanel) { - int64_t sourceSlot = localPanel - sourcePanelBegin; - int64_t destinationBank = localPanel / destination.panelsPerBank; - int64_t destinationSlot = localPanel % destination.panelsPerBank; - destination.banks[destinationBank] = - b.create( - loc, destination.banks[destinationBank].getType(), - sourceChunk.banks.front(), - destination.banks[destinationBank], - createI64Const(b, loc, side * side), - b.getI64IntegerAttr(inputSide), b.getI64IntegerAttr(side), - b.getI64IntegerAttr(stage.kernel), - b.getI64IntegerAttr(stage.stride), b.getI64IntegerAttr(0), - createI64Const(b, loc, sourceSlot * sourceChunk.panelRows), - createI64Const(b, loc, - destinationSlot * destination.panelRows + - destinationBase), - createI64Const(b, loc, destinationStride), - b.getI64IntegerAttr(0), b.getI64IntegerAttr(0)) - .getOutBankOut(); - } - releaseTile(sourceChunk); - } - maskInvalidOutput(); - return success(); - } else if (!externalInput && !stage.depthwise && - !streamMaterializedInput) { - source = allocateTile(inputPanelCount, inputSide * inputSide, zeroI8); - if (failed(emitInto(producer.lookup(stage.input), sourceY, sourceX, - inputSide, inputSide, zero, inputPanelCount, source, - 0, inputSide))) - return failure(); - } - - if (stage.depthwise) { - int64_t paddedK = - (stage.kernel * stage.kernel + kTile - 1) / kTile * kTile; - Value lutLoaded; - if (stage.activation == 2) { - Value lutBank = allocBank(b, loc, 1, 1); - lutLoaded = - mvinBank(b, loc, packedLuts.lookup(stage.op), lutBank, kTile); - } - SmallVector destinationStates(destination.banks.begin(), - destination.banks.end()); - for (int64_t localPanel = 0; localPanel < panelCount; ++localPanel) { - int64_t destinationBank = localPanel / destination.panelsPerBank; - int64_t destinationSlot = localPanel % destination.panelsPerBank; - Value outputPanel = b.create( - loc, firstPanel, - b.create(loc, localPanel)); - TileBanks panelSource; - if (externalInput) { - auto loaded = - loadInt8Tile(stage.input, stage.inputHeight, stage.inputWidth, - stage.inputChannels, sourceY, sourceX, outputPanel, - 1, inputSide, inputSide); - if (failed(loaded)) - return stage.op->emitError("failed to load INT8 Depthwise input"); - panelSource = std::move(*loaded); - } else { - panelSource = allocateTile(1, inputSide * inputSide, zeroI8); - if (panelSource.banks.size() != 1) - return stage.op->emitError( - "Depthwise Conv input panel must fit one bank"); - if (failed(emitInto(producer.lookup(stage.input), sourceY, sourceX, - inputSide, inputSide, outputPanel, 1, - panelSource, 0, inputSide))) - return failure(); - } - SmallVector parameterOffsets = { - outputPanel, b.getIndexAttr(0), b.getIndexAttr(0)}; - SmallVector parameterSizes = { - b.getIndexAttr(1), b.getIndexAttr(4), b.getIndexAttr(4)}; - SmallVector parameterStrides(3, b.getIndexAttr(1)); - Value biasSlice = b.create( - loc, packedBiases.lookup(stage.op), parameterOffsets, - parameterSizes, parameterStrides); - Value biasPack = b.create( - loc, biasSlice, SmallVector{{0, 1}, {2}}); - Value scaleSlice = b.create( - loc, packedScales.lookup(stage.op), parameterOffsets, - parameterSizes, parameterStrides); - Value scalePack = b.create( - loc, scaleSlice, SmallVector{{0, 1}, {2}}); - - Value biasBank = allocBank(b, loc, 1, 1); - Value biasLoaded = mvinBank(b, loc, biasPack, biasBank, 4); - Value biasState = b.create( - loc, biasLoaded.getType(), biasLoaded, createI64Const(b, loc, 0)); - releaseBank(b, loc, biasState); - Value scaleBank = allocBank(b, loc, 1, 1); - Value scaleLoaded = mvinBank(b, loc, scalePack, scaleBank, 4); - Value patchState = allocBank(b, loc, 1, 1); - Value weightState = allocBank(b, loc, 1, 1); - Value resultState = allocBank(b, loc, 1, 1); - auto laneLoop = b.create( - loc, zero, b.create(loc, kTile), one, - ValueRange{patchState, weightState, resultState}); - b.setInsertionPointToStart(laneLoop.getBody()); - Value lane = laneLoop.getInductionVar(); - ValueRange laneStates = laneLoop.getRegionIterArgs(); - Value laneI64 = - b.create(loc, b.getI64Type(), lane); - Value patchNext = - b.create( - loc, laneStates[0].getType(), panelSource.banks.front(), - laneStates[0], createI64Const(b, loc, inputSide), - createI64Const(b, loc, stage.kernel), - createI64Const(b, loc, stage.stride), - createI64Const(b, loc, 0), createI64Const(b, loc, 0), - laneI64, b.getI64IntegerAttr(0), b.getI64IntegerAttr(0), - b.getI64IntegerAttr(0), b.getI64IntegerAttr(side * side)) - .getOutBankOut(); - - Value weightSlice = b.create( - loc, packedDepthwiseWeights.lookup(stage.op), - SmallVector{outputPanel, lane, b.getIndexAttr(0), - b.getIndexAttr(0)}, - SmallVector{b.getIndexAttr(1), b.getIndexAttr(1), - b.getIndexAttr(paddedK), - b.getIndexAttr(kTile)}, - SmallVector(4, b.getIndexAttr(1))); - Value weightPack = b.create( - loc, weightSlice, - SmallVector{{0, 1, 2}, {3}}); - Value weightNext = - mvinBank(b, loc, weightPack, laneStates[1], paddedK); - Value first = b.create(loc, arith::CmpIPredicate::eq, - lane, zero); - Value last = b.create( - loc, arith::CmpIPredicate::eq, lane, - b.create(loc, kTile - 1)); - Value smatmulInput = repackSMatMulInput(patchNext, kTile, paddedK); - Value resultNext = - b.create( - loc, laneStates[2].getType(), smatmulInput, weightNext, - laneStates[2], - createI64ConstU(b, loc, matrixRs2(kTile, kTile, paddedK)), - first, last, createI64Const(b, loc, 0)) - .getWrBankOut(); - if (smatmulInput != patchNext) - releaseBank(b, loc, smatmulInput); - b.create(loc, - ValueRange{patchNext, weightNext, resultNext}); - b.setInsertionPointAfter(laneLoop); - patchState = laneLoop.getResult(0); - weightState = laneLoop.getResult(1); - resultState = laneLoop.getResult(2); - releaseBank(b, loc, patchState); - releaseBank(b, loc, weightState); - - Value outputBank = allocBank(b, loc, 1, 1); - Value outputState; - if (fp32Output && stage.activation == 2) { - Value quantizedState = - b.create( - loc, outputBank.getType(), resultState, scaleLoaded, - outputBank, createI64Const(b, loc, side * side * 4), - createI64Const(b, loc, 0), createI64Const(b, loc, 0), - b.getI64IntegerAttr(side), b.getI64IntegerAttr(side), - b.getI64IntegerAttr(side), b.getBoolAttr(false)) - .getOutBankOut(); - releaseBank(b, loc, resultState); - releaseBank(b, loc, scaleLoaded); - Value lutOutput = allocBank(b, loc, 1, 1); - Value transformed = b.create( - loc, lutOutput.getType(), quantizedState, lutLoaded, lutOutput, - createI64Const(b, loc, side * side)); - releaseBank(b, loc, quantizedState); - Value destinationState = destinationStates[destinationBank]; - int64_t pixels = side * side; - for (int64_t pixel = 0; pixel < pixels; ++pixel) { - Value outputBase = - createI64Const(b, loc, - destinationSlot * destination.panelRows + - destinationBase + pixel); - destinationState = - b.create( - loc, destinationState.getType(), transformed, - destinationState, createI64Const(b, loc, 1), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(0), createI64Const(b, loc, pixel), - outputBase, createI64Const(b, loc, 1), - b.getI64IntegerAttr(0), b.getI64IntegerAttr(0)) - .getOutBankOut(); - } - destinationStates[destinationBank] = destinationState; - releaseBank(b, loc, transformed); - } else if (fp32Output) { - outputState = - b.create( - loc, outputBank.getType(), resultState, scaleLoaded, - outputBank, createI64Const(b, loc, side * side * 4), - b.getBoolAttr(stage.activation == 1)) - .getOutBankOut(); - releaseBank(b, loc, resultState); - releaseBank(b, loc, scaleLoaded); - Value destinationState = destinationStates[destinationBank]; - Value outputBase = createI64Const( - b, loc, - destinationSlot * destination.panelRows + destinationBase); - destinationState = - copyFp32Tile(outputState, destinationState, outputBase, side, - destinationStride); - destinationStates[destinationBank] = destinationState; - releaseBank(b, loc, outputState); - } else { - Value quantizedState = - b.create( - loc, outputBank.getType(), resultState, scaleLoaded, - outputBank, createI64Const(b, loc, side * side * 4), - createI64Const(b, loc, 0), createI64Const(b, loc, 0), - b.getI64IntegerAttr(side), b.getI64IntegerAttr(side), - b.getI64IntegerAttr(side), - b.getBoolAttr(stage.activation == 1)) - .getOutBankOut(); - releaseBank(b, loc, resultState); - releaseBank(b, loc, scaleLoaded); - Value transformed; - outputState = quantizedState; - if (stage.activation == 2) { - Value lutOutput = allocBank(b, loc, 1, 1); - transformed = b.create( - loc, lutOutput.getType(), quantizedState, lutLoaded, - lutOutput, createI64Const(b, loc, side * side)); - outputState = transformed; - } - Value outputBase = createI64Const( - b, loc, - destinationSlot * destination.panelRows + destinationBase); - destinationStates[destinationBank] = - b.create( - loc, destinationStates[destinationBank].getType(), - outputState, destinationStates[destinationBank], - createI64Const(b, loc, side * side), - b.getI64IntegerAttr(side), b.getI64IntegerAttr(side), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(0), createI64Const(b, loc, 0), - outputBase, createI64Const(b, loc, destinationStride), - b.getI64IntegerAttr(0), b.getI64IntegerAttr(0)) - .getOutBankOut(); - releaseBank(b, loc, quantizedState); - if (transformed) - releaseBank(b, loc, transformed); - } - releaseTile(panelSource); - } - if (lutLoaded) - releaseBank(b, loc, lutLoaded); - maskInvalidOutput(); - return success(); - } - - int64_t kernelElements = stage.kernel * stage.kernel; - int64_t paddedK = (kernelElements + kTile - 1) / kTile * kTile; - Value lutLoaded; - if (stage.activation == 2) { - int64_t lutRows = - stage.lutEntries == 4096 ? stage.lutEntries / (4 * kTile) : kTile; - Value lutBank = allocBank(b, loc, 1, stage.lutEntries == 4096 ? 4 : 1); - lutLoaded = - mvinBank(b, loc, packedLuts.lookup(stage.op), lutBank, lutRows); - } - for (size_t destinationBank = 0; - destinationBank < destination.banks.size(); ++destinationBank) { - int64_t panelBegin = destinationBank * destination.panelsPerBank; - int64_t panelEnd = std::min( - panelCount, panelBegin + destination.panelsPerBank); - auto outputPanelLoop = b.create( - loc, b.create(loc, panelBegin), - b.create(loc, panelEnd), one, - ValueRange{destination.banks[destinationBank]}); - b.setInsertionPointToStart(outputPanelLoop.getBody()); - Value localPanel = outputPanelLoop.getInductionVar(); - Value destinationState = outputPanelLoop.getRegionIterArgs().front(); - Value outputPanel = - b.create(loc, firstPanel, localPanel); - SmallVector parameterOffsets = { - outputPanel, b.getIndexAttr(0), b.getIndexAttr(0)}; - SmallVector parameterSizes = { - b.getIndexAttr(1), b.getIndexAttr(4), b.getIndexAttr(4)}; - SmallVector parameterStrides(3, b.getIndexAttr(1)); - Value biasSlice = b.create( - loc, packedBiases.lookup(stage.op), parameterOffsets, - parameterSizes, parameterStrides); - Value biasPack = b.create( - loc, biasSlice, SmallVector{{0, 1}, {2}}); - Value scaleSlice = b.create( - loc, packedScales.lookup(stage.op), parameterOffsets, - parameterSizes, parameterStrides); - Value scalePack = b.create( - loc, scaleSlice, SmallVector{{0, 1}, {2}}); - - SmallVector states; - auto ensureStates = [&]() { - if (states.empty()) { - states = {allocBank(b, loc, 1, 1), allocBank(b, loc, 1, 1), - allocBank(b, loc, 1, 1)}; - if (stage.weightChannelStride) - states[0] = mvinBank(b, loc, zeroPack, states[0], kTile); - } - }; - auto accumulateSource = [&](Value sourceBank, Value sourcePanelBegin, - Value sourcePanelEnd, - int64_t sourcePanelRows) { - Value tileSize = b.create(loc, kTile); - Value channelBegin = - b.create(loc, sourcePanelBegin, tileSize); - Value channelEnd = b.create( - loc, b.create(loc, stage.inputChannels), - b.create(loc, sourcePanelEnd, tileSize)); - int64_t channelStep = stage.weightChannelStride ? kTile : 1; - auto channelLoop = - b.create(loc, channelBegin, channelEnd, - stage.weightChannelStride ? tileSize : one, - ValueRange{states[0], states[1], states[2]}); - b.setInsertionPointToStart(channelLoop.getBody()); - Value inputChannel = channelLoop.getInductionVar(); - ValueRange iterStates = channelLoop.getRegionIterArgs(); - Value sourceSlot = b.create( - loc, - b.create( - loc, inputChannel, - b.create(loc, kTile)), - sourcePanelBegin); - Value inputBase = b.create( - loc, b.getI64Type(), - b.create( - loc, sourceSlot, - b.create(loc, sourcePanelRows))); - Value patchNext; - Value weightNext; - if (stage.weightChannelStride) { - patchNext = b.create( - loc, iterStates[0].getType(), sourceBank, iterStates[0], - createI64Const(b, loc, side * side), b.getI64IntegerAttr(side), - b.getI64IntegerAttr(side), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(0), inputBase, - createI64Const(b, loc, 0), createI64Const(b, loc, side), - b.getI64IntegerAttr(0), b.getI64IntegerAttr(0)); - weightNext = iterStates[1]; - for (int64_t half = 0; half < 2; ++half) { - Value channel = b.create( - loc, inputChannel, - b.create(loc, half * 8)); - Value weightSlice = b.create( - loc, stage.weight, - SmallVector{outputPanel, channel, - b.getIndexAttr(0), - b.getIndexAttr(0)}, - SmallVector{ - b.getIndexAttr(1), b.getIndexAttr(8), b.getIndexAttr(1), - b.getIndexAttr(kTile)}, - SmallVector(4, b.getIndexAttr(1))); - weightNext = b.create( - loc, weightNext.getType(), weightSlice, weightNext, - createI64Const(b, loc, 1), stage.weightChannelStride, - createI64Const(b, loc, 8), createI64Const(b, loc, half * 8), - createI64Const(b, loc, 8), createI64Const(b, loc, kTile)); - } - } else { - Value inputLane = b.create( - loc, b.getI64Type(), - b.create( - loc, inputChannel, - b.create(loc, kTile))); - patchNext = - b.create( - loc, iterStates[0].getType(), sourceBank, iterStates[0], - createI64Const(b, loc, inputSide), - createI64Const(b, loc, stage.kernel), - createI64Const(b, loc, stage.stride), - createI64Const(b, loc, 0), inputBase, inputLane, - b.getI64IntegerAttr(0), b.getI64IntegerAttr(0), - b.getI64IntegerAttr(0), b.getI64IntegerAttr(side * side)) - .getOutBankOut(); - SmallVector weightOffsets = { - outputPanel, inputChannel, b.getIndexAttr(0), - b.getIndexAttr(0)}; - SmallVector weightSizes = { - b.getIndexAttr(1), b.getIndexAttr(1), b.getIndexAttr(paddedK), - b.getIndexAttr(kTile)}; - SmallVector weightStrides(4, b.getIndexAttr(1)); - Value weightSlice = b.create( - loc, stage.weight, weightOffsets, weightSizes, weightStrides); - Value weightPack = b.create( - loc, weightSlice, - SmallVector{{0, 1, 2}, {3}}); - weightNext = mvinBank(b, loc, weightPack, iterStates[1], paddedK); - } - Value first = b.create(loc, arith::CmpIPredicate::eq, - inputChannel, zero); - Value last = b.create( - loc, arith::CmpIPredicate::eq, inputChannel, - b.create(loc, stage.inputChannels - - channelStep)); - int64_t smatmulM = side == 1 && paddedK == kTile ? 1 : kTile; - Value smatmulInput = repackSMatMulInput(patchNext, smatmulM, paddedK); - Value resultNext = - b.create( - loc, iterStates[2].getType(), smatmulInput, weightNext, - iterStates[2], - createI64ConstU(b, loc, matrixRs2(smatmulM, kTile, paddedK)), - first, last, createI64Const(b, loc, 0)) - .getWrBankOut(); - if (smatmulInput != patchNext) - releaseBank(b, loc, smatmulInput); - b.create(loc, - ValueRange{patchNext, weightNext, resultNext}); - b.setInsertionPointAfter(channelLoop); - states.assign(channelLoop.getResults().begin(), - channelLoop.getResults().end()); - }; - Value biasBank = allocBank(b, loc, 1, 1); - Value biasLoaded = mvinBank(b, loc, biasPack, biasBank, 4); - Value biasState = b.create( - loc, biasLoaded.getType(), biasLoaded, createI64Const(b, loc, 0)); - releaseBank(b, loc, biasState); - - if (externalInput) { - ensureStates(); - auto inputPanelLoop = b.create( - loc, zero, b.create(loc, inputPanelCount), - one, ValueRange{states[0], states[1], states[2]}); - b.setInsertionPointToStart(inputPanelLoop.getBody()); - Value panel = inputPanelLoop.getInductionVar(); - states.assign(inputPanelLoop.getRegionIterArgs().begin(), - inputPanelLoop.getRegionIterArgs().end()); - FailureOr loaded = - loadInt8Tile(stage.input, stage.inputHeight, stage.inputWidth, - stage.inputChannels, sourceY, sourceX, panel, 1, - inputSide, inputSide); - if (failed(loaded)) - return stage.op->emitError("failed to load external Conv input"); - accumulateSource(loaded->banks.front(), panel, - b.create(loc, panel, one), - loaded->panelRows); - releaseTile(*loaded); - b.create(loc, - ValueRange{states[0], states[1], states[2]}); - b.setInsertionPointAfter(inputPanelLoop); - states.assign(inputPanelLoop.getResults().begin(), - inputPanelLoop.getResults().end()); - } else if (streamMaterializedInput) { - ensureStates(); - int64_t streamedPanelsPerBank = - tilePanelsPerBank(inputSide * inputSide); - if (streamedPanelsPerBank <= 0) - return stage.op->emitError( - "streamed Conv input panel does not fit one bank"); - for (int64_t inputPanel = 0; inputPanel < inputPanelCount; - inputPanel += streamedPanelsPerBank) { - int64_t panelCountInBank = std::min( - streamedPanelsPerBank, inputPanelCount - inputPanel); - TileBanks streamedSource = - allocateTile(panelCountInBank, inputSide * inputSide, zeroI8); - if (streamedSource.banks.size() != 1) - return stage.op->emitError( - "streamed Conv input panel group must fit one bank"); - if (failed( - emitInto(producer.lookup(stage.input), sourceY, sourceX, - inputSide, inputSide, - b.create(loc, inputPanel), - panelCountInBank, streamedSource, 0, inputSide))) - return failure(); - accumulateSource(streamedSource.banks.front(), - b.create(loc, inputPanel), - b.create( - loc, inputPanel + panelCountInBank), - streamedSource.panelRows); - releaseTile(streamedSource); - } - } else { - ensureStates(); - for (size_t sourceBank = 0; sourceBank < source.banks.size(); - ++sourceBank) { - int64_t sourcePanelBegin = sourceBank * source.panelsPerBank; - int64_t sourcePanelEnd = std::min( - inputPanelCount, sourcePanelBegin + source.panelsPerBank); - accumulateSource( - source.banks[sourceBank], - b.create(loc, sourcePanelBegin), - b.create(loc, sourcePanelEnd), - source.panelRows); - } - } - releaseBank(b, loc, states[0]); - releaseBank(b, loc, states[1]); - Value scaleBank = allocBank(b, loc, 1, 1); - Value scaleLoaded = mvinBank(b, loc, scalePack, scaleBank, 4); - Value destinationSlot = b.create( - loc, localPanel, b.create(loc, panelBegin)); - Value destinationOffset = b.create( - loc, destinationSlot, - b.create(loc, destination.panelRows)); - Value outputBase = b.create( - loc, b.getI64Type(), - b.create( - loc, destinationOffset, - b.create(loc, destinationBase))); - Value outputBank = allocBank(b, loc, 1, 1); - Value outputState; - Value destinationNext = destinationState; - if (fp32Output && stage.activation == 2) { - Value quantizedState = - b.create( - loc, outputBank.getType(), states[2], scaleLoaded, - outputBank, createI64Const(b, loc, side * side * 4), - createI64Const(b, loc, 0), createI64Const(b, loc, 0), - b.getI64IntegerAttr(side), b.getI64IntegerAttr(side), - b.getI64IntegerAttr(side), b.getBoolAttr(false)) - .getOutBankOut(); - releaseBank(b, loc, states[2]); - releaseBank(b, loc, scaleLoaded); - Value lutOutput = allocBank(b, loc, 1, 1); - Value transformed = b.create( - loc, lutOutput.getType(), quantizedState, lutLoaded, lutOutput, - createI64Const(b, loc, side * side)); - releaseBank(b, loc, quantizedState); - int64_t pixels = side * side; - for (int64_t pixel = 0; pixel < pixels; ++pixel) { - Value sourceBase = createI64Const(b, loc, pixel); - Value outputBase = b.create( - loc, - b.create( - loc, b.getI64Type(), - b.create( - loc, - b.create( - loc, destinationSlot, - b.create( - loc, destination.panelRows)), - b.create(loc, - destinationBase))), - createI64Const(b, loc, pixel)); - destinationNext = - b.create( - loc, destinationNext.getType(), transformed, - destinationNext, createI64Const(b, loc, 1), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(0), sourceBase, outputBase, - createI64Const(b, loc, 1), b.getI64IntegerAttr(0), - b.getI64IntegerAttr(0)) - .getOutBankOut(); - } - releaseBank(b, loc, transformed); - } else if (fp32Output) { - outputState = - b.create( - loc, outputBank.getType(), states[2], scaleLoaded, - outputBank, createI64Const(b, loc, side * side * 4), - b.getBoolAttr(stage.activation == 1)) - .getOutBankOut(); - releaseBank(b, loc, states[2]); - releaseBank(b, loc, scaleLoaded); - Value outputBase = b.create( - loc, b.getI64Type(), - b.create( - loc, - b.create(loc, destinationSlot, - b.create( - loc, destination.panelRows)), - b.create(loc, destinationBase))); - destinationNext = copyFp32Tile(outputState, destinationNext, - outputBase, side, destinationStride); - releaseBank(b, loc, outputState); - } else { - Value quantizedState = - b.create( - loc, outputBank.getType(), states[2], scaleLoaded, - outputBank, createI64Const(b, loc, side * side * 4), - createI64Const(b, loc, 0), createI64Const(b, loc, 0), - b.getI64IntegerAttr(side), b.getI64IntegerAttr(side), - b.getI64IntegerAttr(side), - b.getBoolAttr(stage.activation == 1)) - .getOutBankOut(); - releaseBank(b, loc, states[2]); - releaseBank(b, loc, scaleLoaded); - Value transformed; - outputState = quantizedState; - if (stage.activation == 2) { - Value lutOutput = allocBank(b, loc, 1, 1); - transformed = b.create( - loc, lutOutput.getType(), quantizedState, lutLoaded, lutOutput, - createI64Const(b, loc, side * side)); - outputState = transformed; - } - destinationNext = - b.create( - loc, destinationState.getType(), outputState, - destinationState, createI64Const(b, loc, side * side), - b.getI64IntegerAttr(side), b.getI64IntegerAttr(side), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(0), createI64Const(b, loc, 0), - outputBase, createI64Const(b, loc, destinationStride), - b.getI64IntegerAttr(0), b.getI64IntegerAttr(0)) - .getOutBankOut(); - releaseBank(b, loc, quantizedState); - if (transformed) - releaseBank(b, loc, transformed); - } - b.create(loc, destinationNext); - b.setInsertionPointAfter(outputPanelLoop); - destination.banks[destinationBank] = outputPanelLoop.getResult(0); - } - if (lutLoaded) - releaseBank(b, loc, lutLoaded); - releaseTile(source); - maskInvalidOutput(); - return success(); - }; - - auto materializeStage = [&](int64_t stageIndex) -> LogicalResult { - Stage &stage = stages[stageIndex]; - const bool fp32Output = stage.finalOutput && !stage.pool; - // LUT results stay INT8 in banks; host stores apply the FP32 scale. - const int64_t outputStorageFactor = - fp32Output && stage.activation != 2 ? 4 : 1; - int64_t side = - std::min((stage.add || stage.average) ? 1 : 4, - std::min(stage.outputHeight, stage.outputWidth)); - while (side * side * outputStorageFactor > - std::min(target.bankDepth, kMvin2dAddressableRows)) - --side; - int64_t panelCount = (stage.outputChannels + kTile - 1) / kTile; - auto yLoop = b.create( - loc, zero, b.create(loc, stage.outputHeight), - b.create(loc, side)); - b.setInsertionPointToStart(yLoop.getBody()); - Value y = yLoop.getInductionVar(); - auto xLoop = b.create( - loc, zero, b.create(loc, stage.outputWidth), - b.create(loc, side)); - b.setInsertionPointToStart(xLoop.getBody()); - Value x = xLoop.getInductionVar(); - auto emitOutput = [&](Value firstPanel, - int64_t requestedPanels) -> LogicalResult { - const bool fp32LutOutput = fp32Output && stage.activation == 2; - if (fp32LutOutput && !packedLutOutputScales.lookup(stage.op)) - return stage.op->emitError( - "missing LUT output scales for FP32 final output"); - TileBanks output = allocateTile( - requestedPanels, side * side * outputStorageFactor, zeroI8); - if (output.banks.size() != 1) { - return failure(); - } - if (failed(emitInto(stageIndex, y, x, side, side, firstPanel, - requestedPanels, output, 0, - side * outputStorageFactor))) { - return failure(); - } - Type outputElementType = - fp32LutOutput - ? Type(b.getI8Type()) - : (fp32Output ? Type(b.getF32Type()) : Type(b.getI8Type())); - int64_t usedBankRows = - requestedPanels * side * side * outputStorageFactor; - int64_t packedRows = - fp32LutOutput ? usedBankRows : usedBankRows / outputStorageFactor; - Value pack = b.create( - loc, MemRefType::get({packedRows, kTile}, outputElementType)); - mvoutBank(b, loc, pack, output.banks.front(), usedBankRows); - - auto panelLoop = b.create( - loc, zero, b.create(loc, requestedPanels), - one); - b.setInsertionPointToStart(panelLoop.getBody()); - Value panelInRequest = panelLoop.getInductionVar(); - Value panel = b.create(loc, firstPanel, panelInRequest); - auto localYLoop = b.create( - loc, zero, b.create(loc, side), one); - b.setInsertionPointToStart(localYLoop.getBody()); - Value localY = localYLoop.getInductionVar(); - auto localXLoop = b.create( - loc, zero, b.create(loc, side), one); - b.setInsertionPointToStart(localXLoop.getBody()); - Value localX = localXLoop.getInductionVar(); - Value globalY = b.create(loc, y, localY); - Value globalX = b.create(loc, x, localX); - Value yValid = b.create( - loc, arith::CmpIPredicate::slt, globalY, - b.create(loc, stage.outputHeight)); - Value xValid = b.create( - loc, arith::CmpIPredicate::slt, globalX, - b.create(loc, stage.outputWidth)); - auto valid = b.create( - loc, b.create(loc, yValid, xValid), false); - b.setInsertionPointToStart(&valid.getThenRegion().front()); - auto laneLoop = b.create( - loc, zero, b.create(loc, kTile), one); - b.setInsertionPointToStart(laneLoop.getBody()); - Value lane = laneLoop.getInductionVar(); - Value channel = b.create( - loc, - b.create( - loc, panel, b.create(loc, kTile)), - lane); - auto channelValid = b.create( - loc, - b.create( - loc, arith::CmpIPredicate::slt, channel, - b.create(loc, stage.outputChannels)), - false); - b.setInsertionPointToStart(&channelValid.getThenRegion().front()); - Value row = b.create( - loc, - b.create( - loc, panelInRequest, - b.create(loc, side * side)), - b.create( - loc, - b.create( - loc, localY, b.create(loc, side)), - localX)); - Value packedLane = lane; - if (fp32Output && !fp32LutOutput) { - Value group = b.create( - loc, lane, b.create(loc, 4)); - Value laneInGroup = b.create( - loc, lane, b.create(loc, 4)); - packedLane = b.create( - loc, - b.create(loc, group, - b.create(loc, 4)), - laneInGroup); - } - Value value = - b.create(loc, pack, ValueRange{row, packedLane}); - if (fp32LutOutput) { - Value valueF32 = - b.create(loc, b.getF32Type(), value); - Value outputScale = b.create( - loc, packedLutOutputScales.lookup(stage.op), - ValueRange{ - panel, - b.create( - loc, lane, b.create(loc, 4)), - b.create( - loc, lane, b.create(loc, 4))}); - value = b.create(loc, valueF32, outputScale); - } - if (stage.finalOutput) - b.create( - loc, value, stage.output, - ValueRange{zero, channel, globalY, globalX}); - else - b.create( - loc, value, stage.output, - ValueRange{zero, globalY, globalX, channel}); - b.setInsertionPointAfter(channelValid); - b.setInsertionPointAfter(laneLoop); - b.setInsertionPointAfter(valid); - b.setInsertionPointAfter(localYLoop); - b.setInsertionPointAfter(panelLoop); - b.create(loc, pack); - releaseTile(output); - return success(); - }; - - int64_t panelsPerBank = - tilePanelsPerBank(side * side * outputStorageFactor); - if (panelsPerBank <= 0) - return stage.op->emitError("resident output tile does not fit bank"); - if (stage.depthwise) { - auto depthwisePanelLoop = b.create( - loc, zero, b.create(loc, panelCount), one); - b.setInsertionPointToStart(depthwisePanelLoop.getBody()); - if (failed(emitOutput(depthwisePanelLoop.getInductionVar(), 1))) - return failure(); - b.setInsertionPointAfter(depthwisePanelLoop); - } else { - for (int64_t panelBegin = 0; panelBegin < panelCount; - panelBegin += panelsPerBank) { - int64_t chunkCount = - std::min(panelCount - panelBegin, panelsPerBank); - if (failed( - emitOutput(b.create(loc, panelBegin), - chunkCount))) - return failure(); - } - } - b.setInsertionPointAfter(xLoop); - b.setInsertionPointAfter(yLoop); - // A zero limit is the debug mode for the normal resident path: do not - // force any stage to materialize, but trace every stage that the normal - // scheduler materializes at a pool/add/final boundary. - if (traceThisRegion && stageIndex >= traceMegaStageStart && - (traceMegaStageLimit == 0 || traceMegaStageLimit < 0 || - stageIndex < traceMegaStageLimit || - stageIndex == static_cast(stages.size()) - 1)) { - auto id = b.getI64IntegerAttr(stageIndex); - auto trace = b.create<::buddy::trace::EndOp>( - loc, stage.output.getType(), stage.output, id, - b.getStringAttr("mega-stage")); - trace->setAttr( - "id_path", - b.getArrayAttr({b.getI64IntegerAttr(traceRegionId), id})); - trace->setAttr("buckyball.stage_trace", b.getUnitAttr()); - } - materialized.insert(stageIndex); - return success(); - }; - - DenseMap consumers; - for (const Stage &stage : stages) { - if (stage.channelConcat) { - for (Value input : stage.inputs) - ++consumers[input]; - } else { - ++consumers[stage.input]; - if (stage.add || stage.multiply) - ++consumers[stage.rhs]; - } - } - - int64_t traceLimit = 0; - if (traceThisRegion) { - if (traceMegaStageStart < 0 || - traceMegaStageStart > static_cast(stages.size())) - return kernel.emitError("trace-mega-stage-start is out of range"); - traceLimit = traceMegaStageLimit < 0 - ? static_cast(stages.size()) - : std::min(traceMegaStageLimit, - static_cast(stages.size())); - if (traceLimit < 0) - return kernel.emitError("trace-mega-stage-limit must be non-negative"); - for (int64_t stageIndex = 0; stageIndex < traceLimit; ++stageIndex) { - Stage &stage = stages[stageIndex]; - if (stage.multiply) { - int64_t gateStage = producer.lookup(stage.rhs); - int64_t gatePanels = - (stages[gateStage].outputChannels + kTile - 1) / kTile; - TileBanks gate = allocateTile(gatePanels, 1, zeroI8); - if (gate.banks.size() != 1 || - failed(emitInto(gateStage, zero, zero, 1, 1, zero, gatePanels, - gate, 0, 1))) - return stage.op->emitError( - "INT8 Mul gate must fit one complete bank"); - gateCaches.try_emplace(gateStage, std::move(gate)); - } - if (failed(materializeStage(stageIndex))) - return failure(); - traceMaterialized.insert(stageIndex); - if (stage.multiply) { - for (auto &entry : gateCaches) - releaseTile(entry.second); - gateCaches.clear(); - } - } - } - for (auto [stageIndex, stage] : llvm::enumerate(stages)) { - if (static_cast(stageIndex) < traceLimit) - continue; - if (stage.multiply) { - if (producer.contains(stage.input)) { - int64_t inputStage = producer.lookup(stage.input); - if (!materialized.contains(inputStage) && - failed(materializeStage(inputStage))) - return failure(); - } - if (!producer.contains(stage.rhs)) - return stage.op->emitError( - "INT8 Mul gate must have a region producer"); - int64_t gateStage = producer.lookup(stage.rhs); - if (!gateCaches.contains(gateStage)) { - int64_t gatePanels = - (stages[gateStage].outputChannels + kTile - 1) / kTile; - TileBanks gate = allocateTile(gatePanels, 1, zeroI8); - if (gate.banks.size() != 1 || - failed(emitInto(gateStage, zero, zero, 1, 1, zero, gatePanels, - gate, 0, 1))) - return stage.op->emitError( - "INT8 Mul gate must fit one complete bank"); - gateCaches.try_emplace(gateStage, std::move(gate)); - } - } - // Spatial kernels reuse overlapping producer windows; resize repeats - // pixels. Shared DAG results also need one evaluation before consumers - // request tiles. - if ((stage.resizeNearest || stage.kernel > 1 || - consumers.lookup(stage.output) > 1) && - producer.contains(stage.input)) { - int64_t inputStage = producer.lookup(stage.input); - if (!materialized.contains(inputStage) && - failed(materializeStage(inputStage))) - return failure(); - } - if ((stage.resizeNearest || consumers.lookup(stage.output) > 1) && - !materialized.contains(stageIndex) && - failed(materializeStage(stageIndex))) - return failure(); - if (stage.add && producer.contains(stage.input)) { - int64_t inputStage = producer.lookup(stage.input); - if (!materialized.contains(inputStage) && - failed(materializeStage(inputStage))) - return failure(); - for (auto &entry : gateCaches) - releaseTile(entry.second); - gateCaches.clear(); - } - if ((stage.pool || stage.add) && !materialized.contains(stageIndex)) { - if (failed(materializeStage(stageIndex))) - return failure(); - for (auto &entry : gateCaches) - releaseTile(entry.second); - gateCaches.clear(); - } - } - - int64_t finalStage = stages.size() - 1; - if (!materialized.contains(finalStage) && - failed(materializeStage(finalStage))) - return failure(); - for (auto &entry : gateCaches) - releaseTile(entry.second); - releaseBank(b, loc, zeroBank); - for (Value pack : hostPacks) - b.create(loc, pack); - b.eraseOp(kernel); - return success(); - } - -private: - bool traceMegaStages; - int64_t traceMegaStageStart; - int64_t traceMegaStageLimit; - int64_t traceMegaRegion; - bool traceMegaReloadStages; - bool traceMegaInputBeforeRegion; - mutable int64_t nextTraceRegionId = 0; -}; - -} // namespace - -namespace mlir::buddy { -void populatePebbleResidentConvRegionToBankSSAPatterns( - RewritePatternSet &patterns, bool traceMegaStages, - int64_t traceMegaStageStart, int64_t traceMegaStageLimit, - int64_t traceMegaRegion, bool traceMegaReloadStages, - bool traceMegaInputBeforeRegion) { - patterns.add( - patterns.getContext(), traceMegaStages, traceMegaStageStart, - traceMegaStageLimit, traceMegaRegion, traceMegaReloadStages, - traceMegaInputBeforeRegion); -} -} // namespace mlir::buddy diff --git a/examples/cores/pebble/compiler/src/Conversion/LowerTileToBuckyball/CMakeLists.txt b/examples/cores/pebble/compiler/src/Conversion/LowerTileToBuckyball/CMakeLists.txt deleted file mode 100644 index 93d970513..000000000 --- a/examples/cores/pebble/compiler/src/Conversion/LowerTileToBuckyball/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -add_mlir_library(${BB_CORE_TARGET_PREFIX}LowerTileToBuckyballPass - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerTileToBuckyball/QuantizedKernelPatterns.cpp - CoreTileLowering.cpp - ${BB_REPO_ROOT}/examples/balls/transpose/compiler/src/Conversion/LowerTileToBuckyball/TileTransposePatterns.cpp - - LINK_LIBS PUBLIC - BuddyTile - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball - MLIRLinalgDialect -) diff --git a/examples/cores/pebble/compiler/src/Conversion/LowerTileToBuckyball/CoreTileLowering.cpp b/examples/cores/pebble/compiler/src/Conversion/LowerTileToBuckyball/CoreTileLowering.cpp deleted file mode 100644 index f428d492c..000000000 --- a/examples/cores/pebble/compiler/src/Conversion/LowerTileToBuckyball/CoreTileLowering.cpp +++ /dev/null @@ -1,88 +0,0 @@ -//===- CoreTileLowering.cpp - Pebble Tile to Buckyball lowering ----------===// - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinOps.h" -#include "mlir/Pass/Pass.h" -#include "mlir/Transforms/DialectConversion.h" - -#include "Buckyball/BuckyballDialect.h" -#include "Buckyball/BuckyballOps.h" -#include "Conversion/LowerTileToBuckyball/LowerTileToBuckyball.h" -#include "Target/BuckyballTargetRegistry.h" -#include "Tile/TileDialect.h" -#include "Tile/TileOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; -namespace tile = ::buddy::tile; - -namespace mlir::buddy { -void populateTransposeBallTileLoweringPatterns(RewritePatternSet &patterns, - int64_t bankWidthBytes, - int64_t bankDepth, - int64_t bankNum); -} // namespace mlir::buddy - -namespace { - -class CpuMatmulLowering : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(tile::TileMatMulOp op, - PatternRewriter &rewriter) const override { - rewriter.create( - op.getLoc(), ValueRange{op.getAMemArray(), op.getBMemArray()}, - ValueRange{op.getCMemArray()}); - rewriter.eraseOp(op); - return success(); - } -}; - -class LowerTileToBuckyballPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerTileToBuckyballPass) - - StringRef getArgument() const final { return "convert-tile-to-buckyball"; } - StringRef getDescription() const final { - return "Convert explicit Pebble Tile kernels to Buckyball"; - } - - void getDependentDialects(DialectRegistry ®istry) const override { - registry.insert(); - } - - void runOnOperation() override { - const auto &targetConfig = buckyball_target::getBuckyballTarget(); - ConversionTarget target(getContext()); - target.addLegalDialect(); - target.addIllegalDialect(); - - RewritePatternSet patterns(&getContext()); - mlir::buddy::populateTransposeBallTileLoweringPatterns( - patterns, targetConfig.bankWidthBits / 8, targetConfig.bankDepth, - targetConfig.bankNum); - mlir::buddy::populateQuantizedKernelTileLoweringPatterns(patterns); - patterns.add(&getContext()); - if (failed(applyPartialConversion(getOperation(), target, - std::move(patterns)))) - signalPassFailure(); - } -}; - -} // namespace - -namespace mlir::buddy { -void registerLowerTileToBuckyballPass() { - PassRegistration(); -} -} // namespace mlir::buddy diff --git a/examples/cores/pebble/compiler/src/Dialect/Buckyball/CMakeLists.txt b/examples/cores/pebble/compiler/src/Dialect/Buckyball/CMakeLists.txt deleted file mode 100644 index 867a85b69..000000000 --- a/examples/cores/pebble/compiler/src/Dialect/Buckyball/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -set(LLVM_TARGET_DEFINITIONS ${BB_BUCKYBALL_DIALECT_TD}) -mlir_tablegen(Buckyball.h.inc -gen-op-decls) -mlir_tablegen(Buckyball.cpp.inc -gen-op-defs) -mlir_tablegen(BuckyballTypes.h.inc -gen-typedef-decls -typedefs-dialect=buckyball) -mlir_tablegen(BuckyballTypes.cpp.inc -gen-typedef-defs -typedefs-dialect=buckyball) -mlir_tablegen(BuckyballDialect.h.inc -gen-dialect-decls -dialect=buckyball) -mlir_tablegen(BuckyballDialect.cpp.inc -gen-dialect-defs -dialect=buckyball) -add_public_tablegen_target(${BB_CORE_TARGET_PREFIX}MLIRBuckyballIncGen) -add_mlir_doc(PebbleBuckyball ${BB_CORE_TARGET_PREFIX}Buckyball Dialects/ - -gen-dialect-doc) - -set(LLVM_TARGET_DEFINITIONS ${BB_BUCKYBALL_DIALECT_TD}) -mlir_tablegen(BuckyballConversions.inc -gen-llvmir-conversions) -add_public_tablegen_target(${BB_CORE_TARGET_PREFIX}BuddyBuckyballConversionsIncGen) - -add_subdirectory(IR) -add_subdirectory(Transforms) diff --git a/examples/cores/pebble/compiler/src/Dialect/Buckyball/IR/CMakeLists.txt b/examples/cores/pebble/compiler/src/Dialect/Buckyball/IR/CMakeLists.txt deleted file mode 100644 index 42c7f8807..000000000 --- a/examples/cores/pebble/compiler/src/Dialect/Buckyball/IR/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -add_mlir_dialect_library(${BB_CORE_TARGET_PREFIX}BuddyBuckyball - ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball/IR/BuckyballDialect.cpp - ${BB_TARGET_REGISTRY_SOURCE} - ${BB_BALL_IR_SOURCES} - - LINK_LIBS PUBLIC - MLIRIR - MLIRLLVMDialect - MLIRBufferizationDialect - MLIRMemRefDialect -) - -target_include_directories(${BB_CORE_TARGET_PREFIX}BuddyBuckyball PUBLIC - ${BB_COMPILER_COMMON_DIR} - ${BB_COMPILER_COMMON_DIR}/Dialect - ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball - ${CMAKE_CURRENT_BINARY_DIR}/.. - ${CMAKE_CURRENT_BINARY_DIR}/../../.. -) diff --git a/examples/cores/pebble/compiler/src/Dialect/Buckyball/PebbleBuckyball.td b/examples/cores/pebble/compiler/src/Dialect/Buckyball/PebbleBuckyball.td deleted file mode 100644 index 73a59c06a..000000000 --- a/examples/cores/pebble/compiler/src/Dialect/Buckyball/PebbleBuckyball.td +++ /dev/null @@ -1,31 +0,0 @@ -//===-- PebbleBuckyball.td - Pebble Buckyball dialect ops -*- tablegen -*-===// -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//===----------------------------------------------------------------------===// - -#ifndef PEBBLE_BUCKYBALL_OPS -#define PEBBLE_BUCKYBALL_OPS - -include "Buckyball.td" -include "TransposeBall.td" -include "SMatMulBall.td" -include "Im2colBall.td" -include "ToInt8Ball.td" -include "Int2FpBall.td" -include "LutBall.td" -include "MaxPoolBall.td" -include "Int8AddBall.td" -include "Int8MulBall.td" - -#endif // PEBBLE_BUCKYBALL_OPS diff --git a/examples/cores/pebble/compiler/src/Dialect/Buckyball/Transforms/CMakeLists.txt b/examples/cores/pebble/compiler/src/Dialect/Buckyball/Transforms/CMakeLists.txt deleted file mode 100644 index 747f807e1..000000000 --- a/examples/cores/pebble/compiler/src/Dialect/Buckyball/Transforms/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -add_mlir_dialect_library(${BB_CORE_TARGET_PREFIX}BuddyBuckyballTransforms - ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.cpp - ${BB_REPO_ROOT}/examples/balls/transpose/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - ${BB_REPO_ROOT}/examples/balls/smatmul/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - ${BB_REPO_ROOT}/examples/balls/im2col/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - ${BB_REPO_ROOT}/examples/balls/toint8/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - ${BB_REPO_ROOT}/examples/balls/int2fp/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - ${BB_REPO_ROOT}/examples/balls/lut/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - ${BB_REPO_ROOT}/examples/balls/maxpool/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - ${BB_REPO_ROOT}/examples/balls/int8add/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - ${BB_REPO_ROOT}/examples/balls/int8mul/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - ${BB_REPO_ROOT}/examples/balls/matadd/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - LegalizeForLLVMExport.cpp - - LINK_LIBS PUBLIC - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball - MLIRIR - MLIRLLVMCommonConversion - MLIRLLVMDialect -) diff --git a/examples/cores/pebble/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/cores/pebble/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index dba15d1b9..000000000 --- a/examples/cores/pebble/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,131 +0,0 @@ -//===- LegalizeForLLVMExport.cpp - Pebble Buckyball LLVM lowering ---------===// -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//===----------------------------------------------------------------------===// - -#include "Buckyball/Transform.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" - -using namespace mlir; -using namespace buddy::buckyball::legalize; - -namespace mlir::buddy::buckyball { -void populateTransposeBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth); -void configureTransposeBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable); -void populateSMatMulBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth); -void configureSMatMulBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable); -void populateIm2colBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth); -void configureIm2colBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable); -void populateToInt8BallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth); -void configureToInt8BallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable); -void populateInt2FpBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth); -void configureInt2FpBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable); -void populateLutBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &converter, - RewritePatternSet &patterns, - bool stable, - int64_t bankDepth); -void configureLutBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable); -void populateMaxPoolBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth); -void configureMaxPoolBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable); -void populateInt8AddBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth); -void configureInt8AddBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable); -void populateInt8MulBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth); -void configureInt8MulBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable); -void populateMatAddBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth); -void configureMatAddBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable); -} // namespace mlir::buddy::buckyball - -void mlir::populateBuckyballLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, - int64_t bankWidthBytes, int64_t bankDepth, int64_t bankNum, - bool includeFuncOperandForwarding, bool stable) { - (void)bankWidthBytes; - (void)bankNum; - - populateBaseLegalizeForLLVMExportPatterns(converter, patterns, - includeFuncOperandForwarding); - mlir::buddy::buckyball::populateSMatMulBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::populateTransposeBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::populateIm2colBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::populateToInt8BallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::populateInt2FpBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::populateLutBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::populateMaxPoolBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::populateInt8AddBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::populateInt8MulBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::populateMatAddBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); -} - -void mlir::configureBuckyballLegalizeForExportTarget( - LLVMConversionTarget &target, bool stable) { - configureBaseLegalizeForExportTarget(target); - mlir::buddy::buckyball::configureSMatMulBallLegalizeForExportTarget(target, - stable); - mlir::buddy::buckyball::configureTransposeBallLegalizeForExportTarget(target, - stable); - mlir::buddy::buckyball::configureIm2colBallLegalizeForExportTarget(target, - stable); - mlir::buddy::buckyball::configureToInt8BallLegalizeForExportTarget(target, - stable); - mlir::buddy::buckyball::configureInt2FpBallLegalizeForExportTarget(target, - stable); - mlir::buddy::buckyball::configureLutBallLegalizeForExportTarget(target, - stable); - mlir::buddy::buckyball::configureMaxPoolBallLegalizeForExportTarget(target, - stable); - mlir::buddy::buckyball::configureInt8AddBallLegalizeForExportTarget(target, - stable); - mlir::buddy::buckyball::configureInt8MulBallLegalizeForExportTarget(target, - stable); - mlir::buddy::buckyball::configureMatAddBallLegalizeForExportTarget(target, - stable); -} diff --git a/examples/cores/pebble/compiler/src/Dialect/CMakeLists.txt b/examples/cores/pebble/compiler/src/Dialect/CMakeLists.txt deleted file mode 100644 index ec181148e..000000000 --- a/examples/cores/pebble/compiler/src/Dialect/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory(Buckyball) diff --git a/examples/cores/pebble/compiler/src/Target/CMakeLists.txt b/examples/cores/pebble/compiler/src/Target/CMakeLists.txt deleted file mode 100644 index 39d31dc9b..000000000 --- a/examples/cores/pebble/compiler/src/Target/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory(LLVMIR) diff --git a/examples/cores/pebble/compiler/src/Target/LLVMIR/CMakeLists.txt b/examples/cores/pebble/compiler/src/Target/LLVMIR/CMakeLists.txt deleted file mode 100644 index 0ca0f41c5..000000000 --- a/examples/cores/pebble/compiler/src/Target/LLVMIR/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory(Dialect) diff --git a/examples/cores/pebble/compiler/src/Target/LLVMIR/Dialect/Buckyball/CMakeLists.txt b/examples/cores/pebble/compiler/src/Target/LLVMIR/Dialect/Buckyball/CMakeLists.txt deleted file mode 100644 index 06b99171b..000000000 --- a/examples/cores/pebble/compiler/src/Target/LLVMIR/Dialect/Buckyball/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -add_mlir_translation_library(${BB_CORE_TARGET_PREFIX}BuddyBuckyballToLLVMIRTranslation - ${BB_COMPILER_COMMON_DIR}/Target/LLVMIR/Dialect/Buckyball/BuckyballToLLVMIRTranslation.cpp - - DEPENDS - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballConversionsIncGen - intrinsics_gen - - LINK_COMPONENTS - Core - - LINK_LIBS PUBLIC - MLIRIR - MLIRSupport - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball -) diff --git a/examples/cores/pebble/compiler/src/Target/LLVMIR/Dialect/CMakeLists.txt b/examples/cores/pebble/compiler/src/Target/LLVMIR/Dialect/CMakeLists.txt deleted file mode 100644 index ec181148e..000000000 --- a/examples/cores/pebble/compiler/src/Target/LLVMIR/Dialect/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory(Buckyball) diff --git a/examples/cores/pebble/configs/balldomains/default.toml b/examples/cores/pebble/configs/balldomains/default.toml deleted file mode 100644 index 9d2878f21..000000000 --- a/examples/cores/pebble/configs/balldomains/default.toml +++ /dev/null @@ -1,32 +0,0 @@ -# Pebble ball domain configuration - -ballNum = 10 - -ballIdMappings = [ - { ballId = 0, ballName = "TransposeBall", ballClass = "examples.balls.transpose.TransposeBall", config = "../../../../balls/transpose/configs/default.toml", inBW = 1, outBW = 1 }, - { ballId = 1, ballName = "SMatMulBall", ballClass = "examples.balls.smatmul.SMatMulBall", config = "balls/smatmul/8x16.toml", inBW = 2, outBW = 1 }, - { ballId = 2, ballName = "Im2colBall", ballClass = "examples.balls.im2col.Im2colBall", config = "../../../../balls/im2col/configs/default.toml", inBW = 1, outBW = 1 }, - { ballId = 3, ballName = "ToInt8Ball", ballClass = "examples.balls.toint8.ToInt8Ball", config = "../../../../balls/toint8/configs/default.toml", inBW = 1, outBW = 1 }, - { ballId = 4, ballName = "Int2FpBall", ballClass = "examples.balls.int2fp.Int2FpBall", config = "../../../../balls/int2fp/configs/default.toml", inBW = 2, outBW = 1 }, - { ballId = 5, ballName = "LutBall", ballClass = "examples.balls.lut.LutBall", config = "../../../../balls/lut/configs/default.toml", inBW = 2, outBW = 1 }, - { ballId = 6, ballName = "MaxPoolBall", ballClass = "examples.balls.maxpool.MaxPoolBall", config = "../../../../balls/maxpool/configs/default.toml", inBW = 1, outBW = 1 }, - { ballId = 7, ballName = "Int8AddBall", ballClass = "examples.balls.int8add.Int8AddBall", config = "../../../../balls/int8add/configs/default.toml", inBW = 2, outBW = 1 }, - { ballId = 8, ballName = "Int8MulBall", ballClass = "examples.balls.int8mul.Int8MulBall", config = "../../../../balls/int8mul/configs/default.toml", inBW = 2, outBW = 1 }, - { ballId = 9, ballName = "MatAddBall", ballClass = "examples.balls.matadd.MatAddBall", config = "../../../../balls/matadd/configs/default.toml", inBW = 2, outBW = 1 } -] - -ballISA = [ - { mnemonic = "TRANSPOSE", funct7 = 49, bid = 0 }, - { mnemonic = "SMATMUL_OS", funct7 = 65, bid = 1 }, - { mnemonic = "SMATMUL_BIAS", funct7 = 17, bid = 1 }, - { mnemonic = "IM2COL", funct7 = 48, bid = 2 }, - { mnemonic = "QUANT_F32_TO_I8", funct7 = 51, bid = 3 }, - { mnemonic = "QUANT_I32_TO_I8", funct7 = 67, bid = 3 }, - { mnemonic = "INT32_TO_FP32", funct7 = 68, bid = 4 }, - { mnemonic = "LUT", funct7 = 66, bid = 5 }, - { mnemonic = "MAXPOOL", funct7 = 50, bid = 6 }, - { mnemonic = "INT8ADD", funct7 = 69, bid = 7 }, - { mnemonic = "INT8ADD_RELU", funct7 = 70, bid = 7 }, - { mnemonic = "INT8MUL", funct7 = 71, bid = 8 }, - { mnemonic = "MATADD", funct7 = 72, bid = 9 } -] diff --git a/examples/cores/pebble/configs/cpu/default.toml b/examples/cores/pebble/configs/cpu/default.toml deleted file mode 100644 index e651581c6..000000000 --- a/examples/cores/pebble/configs/cpu/default.toml +++ /dev/null @@ -1,2 +0,0 @@ -kind = "rocket" -config = "./rocket/default.toml" diff --git a/examples/cores/pebble/configs/cpu/rocket/default.toml b/examples/cores/pebble/configs/cpu/rocket/default.toml deleted file mode 100644 index c896503f3..000000000 --- a/examples/cores/pebble/configs/cpu/rocket/default.toml +++ /dev/null @@ -1,76 +0,0 @@ -useUser = true -useSupervisor = true -useHypervisor = false -useDebug = false -useAtomics = true -useAtomicsOnlyForIO = false -useCompressed = true -useRVE = false -useConditionalZero = false -nLocalInterrupts = 0 -useNMI = false -nBreakpoints = 1 -useBPWatch = false -mcontextWidth = 0 -scontextWidth = 0 -nPerfCounters = 0 -haveBasicCounters = true -misaWritable = true -mtvecInitEnable = true -mtvecInit = 0 -mtvecWritable = true -fastLoadWord = true -fastLoadByte = false -branchPredictionModeCSR = false -clockGate = false -mvendorid = 0 -mimpid = 538447876 -haveCease = false -haveSimTimeout = false -useVM = true -useZba = true -useZbb = true -useZbs = true -haveCFlush = false - -[mulDiv] -divUnroll = 1 -divEarlyOutGranularity = 1 -enable = true -mulUnroll = 8 -mulEarlyOut = true -divEarlyOut = true - -[fpu] -divSqrt = true -sfmaLatency = 5 -dfmaLatency = 6 -fpmuLatency = 2 -ifpuLatency = 4 -enable = true -minFLen = 16 -fLen = 64 - -[dcache] -clockGate = false -nSets = 32 -nWays = 4 -nMSHRs = 0 - -[icache] -prefetch = false -nSets = 64 -nWays = 8 - -[btb] -nMatchBits = 14 -nPages = 6 -updatesOutOfOrder = false -bhtEnable = true -bhtEntries = 512 -bhtCounterLength = 1 -bhtHistoryLength = 8 -bhtHistoryBits = 3 -enable = true -nEntries = 28 -nRAS = 6 diff --git a/examples/cores/pebble/configs/default.toml b/examples/cores/pebble/configs/default.toml deleted file mode 100644 index 0e77b20e9..000000000 --- a/examples/cores/pebble/configs/default.toml +++ /dev/null @@ -1,8 +0,0 @@ -factory = "examples.cores.pebble.PebbleCore" -# Pebble Core Configuration (default) - -balldomain = "balldomains/default.toml" -memdomain = "memdomains/default.toml" -frontend = "frontends/default.toml" -rvv = "rvv/default.toml" -cpu = "cpu/default.toml" diff --git a/examples/cores/pebble/configs/frontends/default.toml b/examples/cores/pebble/configs/frontends/default.toml deleted file mode 100644 index 11926b62b..000000000 --- a/examples/cores/pebble/configs/frontends/default.toml +++ /dev/null @@ -1,8 +0,0 @@ -robEntries = 8 -rsOutOfOrderResponse = true -bankIdLen = 10 -vbankIdUpperBound = 23 -sharedBankIdBase = 32 -iterLen = 34 -subRobEnable = false -subRobDepth = 64 diff --git a/examples/cores/pebble/configs/memdomains/default.toml b/examples/cores/pebble/configs/memdomains/default.toml deleted file mode 100644 index 97ed89d7b..000000000 --- a/examples/cores/pebble/configs/memdomains/default.toml +++ /dev/null @@ -1,31 +0,0 @@ -# Pebble core memory domain configuration - -[bank] -num = 24 -width = 128 -entries = 512 -maskLen = 16 -channel = 13 - -[dma] -nXacts = 8 -burstMaxBytes = 1024 -busWidth = 128 -maxInFlightMemReqs = 16 - -[tlb] -size = 4 - -[tma] -readChannel = 4 -writeChannel = 4 - -[mmio] -enable = false -bankNum = 5 -bankEntries = 1024 -bankWidth = 8 -readWidth = 8 - -[mem] -addrLen = 39 diff --git a/examples/cores/pebble/workloads/ctests/CMakeLists.txt b/examples/cores/pebble/workloads/ctests/CMakeLists.txt deleted file mode 100644 index cb6a9e67b..000000000 --- a/examples/cores/pebble/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory(im2col_smatmul_quant_pipeline) diff --git a/examples/cores/pebble/workloads/ctests/im2col_smatmul_quant_pipeline/CMakeLists.txt b/examples/cores/pebble/workloads/ctests/im2col_smatmul_quant_pipeline/CMakeLists.txt deleted file mode 100644 index bec14fba6..000000000 --- a/examples/cores/pebble/workloads/ctests/im2col_smatmul_quant_pipeline/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -add_buckyball_core_workload(im2col_smatmul_quant_pipeline SOURCES - im2col_smatmul_quant_pipeline-main.cpp - pebble.c) diff --git a/examples/cores/pebble/workloads/ctests/im2col_smatmul_quant_pipeline/im2col_smatmul_quant_pipeline-main.cpp b/examples/cores/pebble/workloads/ctests/im2col_smatmul_quant_pipeline/im2col_smatmul_quant_pipeline-main.cpp deleted file mode 100644 index 20d2dab05..000000000 --- a/examples/cores/pebble/workloads/ctests/im2col_smatmul_quant_pipeline/im2col_smatmul_quant_pipeline-main.cpp +++ /dev/null @@ -1,23 +0,0 @@ -#include - -extern "C" int pebble_core(core_id_t); -static volatile uint32_t completed; -static volatile uint32_t failed; - -int main() { - core_id_t id = bb_get_core_id(); - if (id.tile == BB_WORK_TILE && - bb_topology_core_profile(id) == BB_CORE_WORKLOAD_PROFILE) { - __atomic_fetch_or(&failed, pebble_core(id) != 0, __ATOMIC_RELEASE); - __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); - } - if (id.tile == BB_WORK_TILE && id.core == 0) { - uint32_t expected = - bb_topology_profile_cores_per_tile(BB_CORE_WORKLOAD_PROFILE); - while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) - asm volatile("nop"); - return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; - } - for (;;) - asm volatile("wfi"); -} diff --git a/examples/cores/pebble/workloads/ctests/im2col_smatmul_quant_pipeline/pebble.c b/examples/cores/pebble/workloads/ctests/im2col_smatmul_quant_pipeline/pebble.c deleted file mode 100644 index 58b4da4de..000000000 --- a/examples/cores/pebble/workloads/ctests/im2col_smatmul_quant_pipeline/pebble.c +++ /dev/null @@ -1,63 +0,0 @@ -#include "buckyball.h" -#include -#include -#include -#include -#include -#include -#include - -#if BANK_WIDTH != 128 || BANK_LINES < 64 -#error "Pebble core pipeline requires 128-bit banks with at least 64 rows" -#endif - -enum { LANES = BANK_WIDTH / 8, INPUT_ROWS = 36, WINDOWS = 16 }; -static int8_t input[BB_CORES_PER_TILE][INPUT_ROWS * LANES] - __attribute__((aligned(64))); -static int8_t weight[BB_CORES_PER_TILE][LANES * LANES] - __attribute__((aligned(64))); -static int32_t bias[BB_CORES_PER_TILE][LANES] __attribute__((aligned(64))); -static float scale[BB_CORES_PER_TILE][LANES] __attribute__((aligned(64))); -static int8_t output[BB_CORES_PER_TILE][WINDOWS * LANES] - __attribute__((aligned(64))); - -#ifdef __cplusplus -extern "C" -#endif - int pebble_core(core_id_t id) { - int core = id.core; - for (int row = 0; row < INPUT_ROWS; ++row) - input[core][row * LANES] = (3 * row + 1) % 7 - 3; - for (int channel = 0; channel < LANES; ++channel) { - bias[core][channel] = channel - 8; - scale[core][channel] = 1.0f; - for (int k = 0; k < LANES; ++k) - weight[core][k * LANES + channel] = - k < 9 ? (2 * k + 3 * channel) % 5 - 2 : 0; - } - for (int bank = 0; bank < 7; ++bank) - bb_mem_alloc(bank, 1, 1); - bb_mvin((uintptr_t)input[core], 0, INPUT_ROWS, 1); - bb_mvin((uintptr_t)weight[core], 2, LANES, 1); - bb_mvin((uintptr_t)bias[core], 3, LANES * 32 / BANK_WIDTH, 1); - bb_mvin((uintptr_t)scale[core], 5, LANES * 32 / BANK_WIDTH, 1); - bb_im2col(0, 1, 6, 3, 1, 0, 0, 0, 0, 0, 0, WINDOWS); - bb_smatmul_bias(3, 0); - bb_smatmul_os(1, 2, 4, WINDOWS, LANES, LANES, 1, 1, 0); - bb_quant_i32_to_i8(4, 5, 6, WINDOWS * 4, 0, 0, 4, 4, 4, 0); - bb_mvout((uintptr_t)output[core], 6, WINDOWS, 1); - for (int window = 0; window < WINDOWS; ++window) - for (int channel = 0; channel < LANES; ++channel) { - int expected = bias[core][channel]; - int y = window / 4, x = window % 4; - for (int ky = 0; ky < 3; ++ky) - for (int kx = 0; kx < 3; ++kx) { - int k = ky * 3 + kx, row = (y + ky) * 6 + x + kx; - expected += - input[core][row * LANES] * weight[core][k * LANES + channel]; - } - if (output[core][window * LANES + channel] != expected) - return 1; - } - return 0; -} diff --git a/examples/cores/reduce/arch/src/main/scala/ReduceCore.scala b/examples/cores/reduce/arch/src/main/scala/ReduceCore.scala deleted file mode 100644 index bf14a7750..000000000 --- a/examples/cores/reduce/arch/src/main/scala/ReduceCore.scala +++ /dev/null @@ -1,20 +0,0 @@ -package examples.cores.reduce - -import framework.top.GlobalConfig -import chisel3.experimental.hierarchy.{instantiable, Instance, Instantiate} -import framework.system.core.{CoreConnection, RocketEndpoint} -import hier.core.rocket.Core -import framework.system.core.accelerator.BuckyballAccelerator - -@instantiable -class ReduceCore(b: GlobalConfig) extends RocketEndpoint(b) { - require(p.buckyball.nonEmpty) - val cpu = Instantiate(new Core(p.data, p.instruction, p.regions, Some(p.commands))(p.cpu)) - clink.cpu <> cpu.clink - val accelerator = Instantiate(new BuckyballAccelerator(p.buckyball.get)) - CoreConnection.accelerator(clink.accelerator.get, accelerator) -} - -object ReduceCore extends framework.system.core.RocketCoreFactory { - def instantiate(b: GlobalConfig): Instance[RocketEndpoint] = Instantiate(new ReduceCore(b)) -} diff --git a/examples/cores/reduce/compiler/CMakeLists.txt b/examples/cores/reduce/compiler/CMakeLists.txt deleted file mode 100644 index 678fa8bd2..000000000 --- a/examples/cores/reduce/compiler/CMakeLists.txt +++ /dev/null @@ -1,18 +0,0 @@ -if(NOT COMMAND add_mlir_library OR NOT COMMAND add_mlir_dialect) - message(FATAL_ERROR "MLIR CMake helpers must be loaded. Build as part of buddy-mlir.") -endif() - -set(BB_CORE_NAME reduce) -set(BB_CORE_TARGET_PREFIX "") -get_filename_component(_BB_REPO_ROOT "${CMAKE_CURRENT_LIST_DIR}/../../../.." ABSOLUTE) -set(_BB_F32ADD_DIALECT_DIR - "${_BB_REPO_ROOT}/examples/balls/f32add/compiler/src/Dialect/Buckyball") -set(_BB_F32ADD_LEGALIZE "${_BB_F32ADD_DIALECT_DIR}/Transforms/LegalizeForLLVMExport.cpp") -if(NOT _BB_F32ADD_LEGALIZE IN_LIST BB_legalize_SOURCES) - message(FATAL_ERROR "Reduce compiler requires the configured F32Add Ball lowering") -endif() -include_directories("${_BB_F32ADD_DIALECT_DIR}") - -# The base builds the generated dialect and selected Ball IR/assign/export hooks. -add_subdirectory("${_BB_REPO_ROOT}/examples/cores/toy/compiler/src" - "${CMAKE_CURRENT_BINARY_DIR}/base") diff --git a/examples/cores/reduce/configs/balldomains/default.toml b/examples/cores/reduce/configs/balldomains/default.toml deleted file mode 100644 index 810a63d1b..000000000 --- a/examples/cores/reduce/configs/balldomains/default.toml +++ /dev/null @@ -1,9 +0,0 @@ -ballNum = 1 - -ballIdMappings = [ - { ballId = 0, ballName = "F32AddBall", ballClass = "examples.balls.f32add.F32AddBall", config = "../../../../balls/f32add/configs/default.toml", inBW = 2, outBW = 1 } -] - -ballISA = [ - { mnemonic = "F32ADD", funct7 = 76, bid = 0 } -] diff --git a/examples/cores/reduce/configs/cpu/default.toml b/examples/cores/reduce/configs/cpu/default.toml deleted file mode 100644 index 1bf2f46e8..000000000 --- a/examples/cores/reduce/configs/cpu/default.toml +++ /dev/null @@ -1,2 +0,0 @@ -kind = "rocket" -config = "../../../rocket/configs/cpu/rocket/default.toml" diff --git a/examples/cores/reduce/configs/default.toml b/examples/cores/reduce/configs/default.toml deleted file mode 100644 index ec4b3d5d3..000000000 --- a/examples/cores/reduce/configs/default.toml +++ /dev/null @@ -1,6 +0,0 @@ -factory = "examples.cores.reduce.ReduceCore" -balldomain = "balldomains/default.toml" -memdomain = "memdomains/default.toml" -frontend = "frontends/default.toml" -rvv = "rvv/default.toml" -cpu = "cpu/default.toml" diff --git a/examples/cores/reduce/configs/frontends/default.toml b/examples/cores/reduce/configs/frontends/default.toml deleted file mode 100644 index 41cd684cc..000000000 --- a/examples/cores/reduce/configs/frontends/default.toml +++ /dev/null @@ -1,8 +0,0 @@ -robEntries = 16 -rsOutOfOrderResponse = true -bankIdLen = 10 -vbankIdUpperBound = 15 -sharedBankIdBase = 32 -iterLen = 34 -subRobEnable = false -subRobDepth = 64 diff --git a/examples/cores/reduce/configs/memdomains/default.toml b/examples/cores/reduce/configs/memdomains/default.toml deleted file mode 100644 index 37adae0ba..000000000 --- a/examples/cores/reduce/configs/memdomains/default.toml +++ /dev/null @@ -1,29 +0,0 @@ -[bank] -num = 4 -width = 128 -entries = 1024 -maskLen = 16 -channel = 3 - -[dma] -nXacts = 1 -burstMaxBytes = 1024 -busWidth = 128 -maxInFlightMemReqs = 16 - -[tlb] -size = 1 - -[tma] -readChannel = 2 -writeChannel = 1 - -[mmio] -enable = false -bankNum = 1 -bankEntries = 1 -bankWidth = 128 -readWidth = 8 - -[mem] -addrLen = 39 diff --git a/examples/cores/reduce/configs/rvv/default.toml b/examples/cores/reduce/configs/rvv/default.toml deleted file mode 100644 index e006b2055..000000000 --- a/examples/cores/reduce/configs/rvv/default.toml +++ /dev/null @@ -1,6 +0,0 @@ -enable = false -laneNumber = 4 -vLen = 1024 -eLen = 32 -iBufWords = 1024 -memoryPorts = 4 diff --git a/examples/cores/reduce/workloads/ctests/CMakeLists.txt b/examples/cores/reduce/workloads/ctests/CMakeLists.txt deleted file mode 100644 index abfd63910..000000000 --- a/examples/cores/reduce/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -set(BUCKYBALL_CTEST_TARGET ${BUCKYBALL_CORE_WORKLOAD_TARGET}) -set(BUCKYBALL_CTEST_GROUP_TARGET core-ctests-build) -add_subdirectory( - ${BUCKYBALL_REPO_ROOT}/examples/balls/f32add/workloads/ctests - f32add) - -list(APPEND BUCKYBALL_CHIP_COMMON_SOURCES - ${BUCKYBALL_REPO_ROOT}/stack/runtime/src/tlink.c) -add_buckyball_ctest(tlink_lease.c) diff --git a/examples/cores/reduce/workloads/ctests/tlink_lease.c b/examples/cores/reduce/workloads/ctests/tlink_lease.c deleted file mode 100644 index 529230188..000000000 --- a/examples/cores/reduce/workloads/ctests/tlink_lease.c +++ /dev/null @@ -1,36 +0,0 @@ -#include -#include -#include -#include -#include - -static const uint32_t input[2][4] __attribute__((aligned(64))) = { - {1, 2, 3, 4}, {0x12345678, 0xabcdef01, 7, 8}}; -static volatile uint32_t output[8] __attribute__((aligned(64))); - -int main(void) { - unsigned bank = BB_SHARED_BANK_BASE; - bb_mem_alloc(bank, 1, 2); - for (unsigned i = 0; i < 8; ++i) - output[i] = 0xaaaaaaaa; - for (unsigned group = 0; group < 2; ++group) - bb_mvin_group((uintptr_t)input[group], bank, group, 1, 1); - uint64_t address = tlink_shared_export(0, bank, 1); - if (tlink_shared_export(0, bank, 1) != address) - __builtin_trap(); - tlink_shared_release(0, bank, 1); - if (tlink_read64(address) != UINT64_C(0xabcdef0112345678)) - __builtin_trap(); - tlink_write64(address + 8, UINT64_C(0x76543210deadbeef)); - bb_mvout((uintptr_t)output, bank, 1, 1); - for (unsigned i = 0; i < 4; ++i) - if (output[i] != input[0][i]) - __builtin_trap(); - if (output[4] != input[1][0] || output[5] != input[1][1] || - output[6] != 0xdeadbeef || output[7] != 0x76543210) - __builtin_trap(); - tlink_shared_release(0, bank, 1); - bb_mem_release(bank); - puts("Main TLink lease and NPU shared storage PASS"); - return 0; -} diff --git a/examples/cores/rocket/arch/src/main/scala/RocketCore.scala b/examples/cores/rocket/arch/src/main/scala/RocketCore.scala deleted file mode 100644 index 57563e285..000000000 --- a/examples/cores/rocket/arch/src/main/scala/RocketCore.scala +++ /dev/null @@ -1,17 +0,0 @@ -package examples.cores.rocket - -import framework.top.GlobalConfig -import chisel3.experimental.hierarchy.{instantiable, Instance, Instantiate} -import framework.system.core.{CoreConnection, RocketEndpoint} -import hier.core.rocket.Core - -@instantiable -class RocketCore(b: GlobalConfig) extends RocketEndpoint(b) { - require(p.buckyball.isEmpty) - val cpu = Instantiate(new Core(p.data, p.instruction, p.regions, Some(p.commands))(p.cpu)) - clink.cpu <> cpu.clink -} - -object RocketCore extends framework.system.core.RocketCoreFactory { - def instantiate(b: GlobalConfig): Instance[RocketEndpoint] = Instantiate(new RocketCore(b)) -} diff --git a/examples/cores/rocket/arch/src/main/scala/cpu/AdmissionBridge.scala b/examples/cores/rocket/arch/src/main/scala/cpu/AdmissionBridge.scala deleted file mode 100644 index f0b4d7007..000000000 --- a/examples/cores/rocket/arch/src/main/scala/cpu/AdmissionBridge.scala +++ /dev/null @@ -1,164 +0,0 @@ -package hier.core.rocket - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.system.core.rocket.{CpuParams, HasCpuParameters} -import framework.system.core.rocket.RoCCCommandBB -import memcore.memory.interlock.{Params => TrackingParams} - -class RobFault(robEntries: Int) extends Bundle { - val rob_id = UInt(log2Ceil(robEntries).W) - val error = UInt(4.W) - val address = UInt(64.W) -} - -class AdmissionRetirement(tracking: TrackingParams, pmps: Int)(implicit val cpuParams: CpuParams) - extends Bundle - with HasCpuParameters { - val snapshot = new CommandSnapshot(tracking, pmps) - val error = UInt(4.W) - val address = UInt(64.W) -} - -/** Associates accepted CPU snapshots with actual external-command ROB allocations. */ -@instantiable -class AdmissionBridge( - robEntries: Int, - tracking: TrackingParams, - pmps: Int, - nonRobFuncts: Seq[Int] -)( - implicit val cpuParams: CpuParams) - extends Module - with HasCpuParameters { - require(robEntries >= 2 && tracking.entries == 4) - val robBits = log2Ceil(robEntries) - - @public - val io = IO(new Bundle { - val command = Flipped(Decoupled(new CommandSnapshot(tracking, pmps))) - val npuCommand = Decoupled(new RoCCCommandBB) - // bits is the current tail preview even when valid is low; valid means exact alloc.fire. - val allocation = Flipped(Valid(UInt(robBits.W))) - val retired = Input(UInt(robEntries.W)) - val fault = Flipped(Valid(new RobFault(robEntries))) - val unboundFault = Valid(new RobFault(robEntries)) - val faultTag = Valid(UInt(tracking.idBits.W)) - - val lookup = Vec( - 3, - new Bundle { - val robId = Input(UInt(robBits.W)) - val snapshot = Output(Valid(new CommandSnapshot(tracking, pmps))) - } - ) - - val lookupTag = new Bundle { - val tag = Input(UInt(tracking.idBits.W)) - val snapshot = Output(Valid(new CommandSnapshot(tracking, pmps))) - } - - // A ROB retirement notice alone does not establish DMA/cache completion. - val retirement = Decoupled(new AdmissionRetirement(tracking, pmps)) - }) - - val live = RegInit(VecInit(Seq.fill(tracking.entries)(false.B))) - val pending = RegInit(VecInit(Seq.fill(tracking.entries)(false.B))) - val hasRob = RegInit(VecInit(Seq.fill(tracking.entries)(false.B))) - val robIds = Reg(Vec(tracking.entries, UInt(robBits.W))) - val snapshots = Reg(Vec(tracking.entries, new CommandSnapshot(tracking, pmps))) - val errors = RegInit(VecInit(Seq.fill(tracking.entries)(0.U(4.W)))) - val addresses = RegInit(VecInit(Seq.fill(tracking.entries)(0.U(64.W)))) - val free = VecInit(live.map(!_)).asUInt - val slot = PriorityEncoder(free) - val nonRob = nonRobFuncts.map(f => io.command.bits.instruction.funct === f.U(7.W)).reduce(_ || _) - val duplicateTag = - (0 until tracking.entries).map(i => live(i) && snapshots(i).tag === io.command.bits.tag).reduce(_ || _) - val duplicateRob = - (0 until tracking.entries).map(i => live(i) && hasRob(i) && robIds(i) === io.allocation.bits).reduce(_ || _) - val eligible = free.orR && !duplicateTag && (nonRob || !duplicateRob) && !reset.asBool - io.npuCommand.valid := io.command.valid && eligible - io.npuCommand.bits := io.command.bits.instruction - io.command.ready := io.npuCommand.ready && eligible - - when(io.npuCommand.fire) { - assert(io.allocation.valid === !nonRob, "AdmissionBridge command/ROB allocation mismatch") - when(!nonRob)(assert(io.allocation.bits < robEntries.U, "AdmissionBridge ROB allocation out of range")) - live(slot) := true.B - pending(slot) := nonRob - hasRob(slot) := !nonRob - robIds(slot) := io.allocation.bits - snapshots(slot) := io.command.bits - errors(slot) := 0.U - addresses(slot) := 0.U - } - // Boot is reset-only in the actual Frontend: its allocations precede all external contexts. - when(io.allocation.valid && !io.npuCommand.fire) { - assert(!live.asUInt.orR, "AdmissionBridge boot allocation overlaps a live CPU context") - } - val heldCommand = RegNext(io.npuCommand.valid && !io.npuCommand.ready, false.B) - val previousCommand = RegEnable(io.npuCommand.bits.asUInt, io.npuCommand.valid) - when(heldCommand && !reset.asBool) { - assert( - io.npuCommand.valid && io.npuCommand.bits.asUInt === previousCommand, - "AdmissionBridge stalled NPU command changed" - ) - } - // Boot-only allocations have no external command fire and create no CPU context. - for (i <- 0 until tracking.entries) { - when(live(i) && hasRob(i) && io.retired(robIds(i)))(pending(i) := true.B) - } - for (port <- io.lookup) { - val hits = VecInit((0 until tracking.entries).map(i => - live(i) && hasRob(i) && !pending(i) && - !io.retired(robIds(i)) && robIds(i) === port.robId - )) - port.snapshot.valid := hits.asUInt.orR && !reset.asBool - port.snapshot.bits := Mux1H(hits, snapshots) - assert(PopCount(hits) <= 1.U, "AdmissionBridge duplicate live ROB binding") - } - - val tagHits = VecInit((0 until tracking.entries).map(i => - live(i) && hasRob(i) && !pending(i) && !io.retired(robIds(i)) && snapshots(i).tag === io.lookupTag.tag - )) - - io.lookupTag.snapshot.valid := tagHits.asUInt.orR && !reset.asBool - io.lookupTag.snapshot.bits := Mux1H(tagHits, snapshots) - assert(PopCount(tagHits) <= 1.U, "AdmissionBridge duplicate live admission tag") - - val noticeValid = RegInit(false.B) - val noticeSlot = RegInit(0.U(log2Ceil(tracking.entries).W)) - when(!noticeValid && pending.asUInt.orR) { - noticeValid := true.B - noticeSlot := PriorityEncoder(pending.asUInt) - } - io.retirement.valid := noticeValid && !reset.asBool - val faultHits = - VecInit((0 until tracking.entries).map(i => live(i) && hasRob(i) && robIds(i) === io.fault.bits.rob_id)) - io.unboundFault.valid := io.fault.valid && !faultHits.asUInt.orR && !reset.asBool - io.unboundFault.bits := io.fault.bits - io.faultTag.valid := io.fault.valid && faultHits.asUInt.orR && !reset.asBool - io.faultTag.bits := Mux1H(faultHits, snapshots.map(_.tag)) - when(io.fault.valid && io.fault.bits.error =/= 0.U) { - for (i <- 0 until tracking.entries) { - when(faultHits(i)) { - val alreadyOffered = noticeValid && noticeSlot === i.U - assert(!alreadyOffered, "AdmissionBridge fault arrived after retirement offer") - when(!alreadyOffered && errors(i) === 0.U) { - errors(i) := io.fault.bits.error - addresses(i) := io.fault.bits.address - } - } - } - } - io.retirement.bits.snapshot := snapshots(noticeSlot) - io.retirement.bits.error := errors(noticeSlot) - io.retirement.bits.address := addresses(noticeSlot) - when(io.retirement.fire) { - live(noticeSlot) := false.B - pending(noticeSlot) := false.B - hasRob(noticeSlot) := false.B - noticeValid := false.B - } -} diff --git a/examples/cores/rocket/arch/src/main/scala/cpu/Cache.scala b/examples/cores/rocket/arch/src/main/scala/cpu/Cache.scala deleted file mode 100644 index c394f6d7a..000000000 --- a/examples/cores/rocket/arch/src/main/scala/cpu/Cache.scala +++ /dev/null @@ -1,75 +0,0 @@ -package hier.core.rocket - -import memcore.memory.queue.Queue - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import memcore.bus.chi.{RequestFlit, RequesterPort} -import memcore.bus.chi.rnf.{BankedChiCache, CacheAccess, CacheProbe, CacheResult, RnfParams} -import memcore.memory.interlock.{Acknowledgement, Maintenance => Range, Params} - -/** A core's own L1 and range maintenance client share one explicit CHI requester. */ -@instantiable -class Cache(config: RnfParams, tracking: Params) extends Module { - override def desiredName: String = "CoreCache" - val c = config.chi - - @public - val io = IO(new Bundle { - val access = Flipped(Decoupled(new CacheAccess(c))) - val result = Decoupled(new CacheResult) - val probe = Option.when(config.probe)(new CacheProbe(c)) - val maintenance = Flipped(Decoupled(new Range(tracking))) - val maintained = Decoupled(new Acknowledgement(tracking)) - // Includes previously accepted virtual accesses, page walks and fetches outside this L1. - val olderRequestsDrained = Input(Bool()) - val chi = new RequesterPort(c) - val outstanding = Output(UInt(32.W)) - }) - - val cache: Instance[BankedChiCache] = Instantiate(new BankedChiCache(config)) - val maintenance: Instance[Maintenance] = Instantiate(new Maintenance(config, tracking)) - val drained = io.olderRequestsDrained && cache.io.outstanding === 0.U - maintenance.io.request.valid := io.maintenance.valid && drained - maintenance.io.request.bits := io.maintenance.bits - io.maintenance.ready := maintenance.io.request.ready && drained - // Existing owners may keep progressing until drained. At admission, a new CPU - // request cannot race the CMO; subsequent conflicts belong to the external interlock. - val admitMaintenance = io.maintenance.valid && io.maintenance.ready - cache.io.access.valid := io.access.valid && !admitMaintenance - cache.io.access.bits := io.access.bits - io.access.ready := cache.io.access.ready && !admitMaintenance - io.result <> cache.io.result - io.probe.foreach(_ <> cache.io.probe.get) - maintenance.io.drained := true.B - io.maintained <> maintenance.io.response - io.outstanding := cache.io.outstanding - - val requests = Module(new RRArbiter(new RequestFlit(c), 2) { - - override lazy val lastGrant = { - val previous = RegInit(0.U(1.W)) - when(io.out.fire)(previous := io.chosen) - previous - } - - }) - - requests.io.in(0) <> cache.io.chi.req - requests.io.in(1) <> maintenance.io.req - val requestQueue = Module(new Queue(new RequestFlit(c), 2)) - requestQueue.io.enq <> requests.io.out - io.chi.req <> requestQueue.io.deq - io.chi.txRsp <> cache.io.chi.txRsp - io.chi.txDat <> cache.io.chi.txDat - cache.io.chi.rxDat <> io.chi.rxDat - cache.io.chi.snp <> io.chi.snp - - val maintenanceResponse = io.chi.rxRsp.bits.txnId === config.banks.U - maintenance.io.rsp.valid := io.chi.rxRsp.valid && maintenanceResponse - maintenance.io.rsp.bits := io.chi.rxRsp.bits - cache.io.chi.rxRsp.valid := io.chi.rxRsp.valid && !maintenanceResponse - cache.io.chi.rxRsp.bits := io.chi.rxRsp.bits - io.chi.rxRsp.ready := Mux(maintenanceResponse, maintenance.io.rsp.ready, cache.io.chi.rxRsp.ready) -} diff --git a/examples/cores/rocket/arch/src/main/scala/cpu/Lsu.scala b/examples/cores/rocket/arch/src/main/scala/cpu/Lsu.scala deleted file mode 100644 index c846b5207..000000000 --- a/examples/cores/rocket/arch/src/main/scala/cpu/Lsu.scala +++ /dev/null @@ -1,252 +0,0 @@ -package hier.core.rocket - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.system.core.rocket.{CpuParams, HasCpuParameters} -import freechips.rocketchip.rocket._ -import freechips.rocketchip.rocket.constants.MemoryOpConstants -import memcore.bus.chi.rnf.CacheAtomic -import memcore.memory.cpu.{CpuMemParams, VirtualMemoryRequest, VirtualMemoryResponse} -import memcore.memory.fetch.Context - -/** A load or store offered to the L1 fast path by virtual address; the Core translates it through the data TLB. */ -class LsuProbeRequest extends Bundle { - val addr = UInt(64.W) - val size = UInt(2.W) - val write = Bool() - val data = UInt(64.W) - val privilege = UInt(2.W) - val satp = UInt(64.W) - val sum = Bool() - val mxr = Bool() -} - -class LsuProbeResult extends Bundle { - val hit = Bool() - val value = UInt(64.W) -} - -class LsuProbe extends Bundle { - val req = Decoupled(new LsuProbeRequest) - val complete = Flipped(Decoupled(new LsuProbeResult)) - val cancel = Output(Bool()) - val retire = Output(Bool()) - val active = Output(Bool()) - val cancelled = Input(Bool()) -} - -/** - * A probe hit completes in s2. Otherwise: fixed S1/S2 cancellation windows, then execute once - * and return only on the instruction's replay. - */ -@instantiable -class Lsu(cp: CpuMemParams)(implicit val cpuParams: CpuParams) - extends Module - with HasCpuParameters - with MemoryOpConstants { - - @public - val io = IO(new Bundle { - val cpu = Flipped(new HellaCacheIO) - val pc = Input(UInt(vaddrBitsExtended.W)) - val pmp = Input(Vec(nPMPs, new PMP)) - val context = Input(new Context) - val maintenance = Decoupled(new SFenceReq) - val maintained = Flipped(Decoupled(Bool())) - val capturedPmp = Output(Vec(nPMPs, new PMP)) - val request = Decoupled(new VirtualMemoryRequest(cp)) - val response = Flipped(Decoupled(new VirtualMemoryResponse(cp))) - val probe = new LsuProbe - val idle = Output(Bool()) - val cancelled = Output(Bool()) - }) - - require(!usingHypervisor && cp.tagBits <= coreParams.dcacheReqTagBits) - val idle :: s1 :: s2 :: issue :: receive :: completed :: replay1 :: replay2 :: Nil = Enum(8) - val state = RegInit(idle) - val command = Reg(new HellaCacheReq) - val instructionPc = Reg(UInt(vaddrBitsExtended.W)) - val data = Reg(UInt(64.W)) - val pmps = Reg(Vec(nPMPs, new PMP)) - val context = Reg(new Context) - val maintaining = RegInit(false.B) - val probeAccepted = RegInit(false.B) - io.capturedPmp := pmps - val result = Reg(new VirtualMemoryResponse(cp)) - val matches = io.cpu.req.bits.tag === command.tag && io.cpu.req.bits.addr === command.addr && - io.cpu.req.bits.cmd === command.cmd && io.cpu.req.bits.size === command.size && io.pc === instructionPc - io.idle := state === idle - io.cancelled := (state === s1 || state === replay1) && io.cpu.s1_kill || - (state === s2 || state === replay2) && io.cpu.s2_kill - io.cpu.req.ready := state === idle || (state === completed && (!io.cpu.req.valid || matches)) - when(io.cpu.req.fire) { - when(state === idle) { - assert(!io.cpu.req.bits.dv && io.cpu.req.bits.dprv =/= 2.U, "LSU requires nonvirtual U, S or M privilege") - if (io.cpu.req.bits.tag.getWidth > cp.tagBits) { - assert( - !io.cpu.req.bits.tag(io.cpu.req.bits.tag.getWidth - 1, cp.tagBits).orR, - "LSU arbiter tag bits must be zero" - ) - } - command := io.cpu.req.bits - instructionPc := io.pc - pmps := io.pmp - context := io.context - state := s1 - }.otherwise { - assert(matches, "LSU replay changed instruction identity") - state := replay1 - } - } - when(state === s1) { - when(io.cpu.s1_kill)(state := idle) - .otherwise { data := io.cpu.s1_data.data; state := s2 } - } - - val writes = isWrite(command.cmd) - val reads = isRead(command.cmd) - val address = if (usingVM) command.addr.asSInt.pad(64).asUInt else command.addr.pad(64) - val sfence = command.cmd === M_SFENCE - val misaligned = (address & ((1.U(64.W) << command.size) - 1.U)).orR - - val probeEligible = !misaligned && (command.cmd === M_XRD || command.cmd === M_XWR) - io.probe.active := (state === s1 || state === s2) && probeEligible - io.probe.req.valid := state === s1 && !io.cpu.s1_kill && probeEligible - io.probe.req.bits.addr := address - io.probe.req.bits.size := command.size - io.probe.req.bits.write := command.cmd === M_XWR - io.probe.req.bits.data := io.cpu.s1_data.data - io.probe.req.bits.privilege := command.dprv - io.probe.req.bits.satp := context.satp - io.probe.req.bits.sum := context.sum - io.probe.req.bits.mxr := context.mxr - io.probe.cancel := state === s2 && io.cpu.s2_kill - io.probe.retire := state === s2 && probeAccepted && !io.cpu.s2_kill - io.probe.complete.ready := state === s2 && probeAccepted && !io.cpu.s2_kill - when(io.probe.req.fire)(probeAccepted := true.B) - when(state === s2)(probeAccepted := false.B) - when(state === s2 && probeAccepted && !io.cpu.s2_kill) { - assert(io.probe.complete.valid || io.probe.cancelled, "Accepted LSU probe did not complete in s2") - } - val probed = io.probe.complete.fire && io.probe.complete.bits.hit && !io.probe.cancelled - val probedRaw = io.probe.complete.bits.value >> (address(2, 0) << 3) - - val probedLoad = MuxLookup(command.size, probedRaw)(Seq( - 0.U -> Cat(Fill(56, command.signed && probedRaw(7)), probedRaw(7, 0)), - 1.U -> Cat(Fill(48, command.signed && probedRaw(15)), probedRaw(15, 0)), - 2.U -> Cat(Fill(32, command.signed && probedRaw(31)), probedRaw(31, 0)) - )) - - when(state === s2) { - when(io.cpu.s2_kill)(state := idle).elsewhen(probed)(state := idle).otherwise { - when(misaligned && !sfence) { - result := 0.U.asTypeOf(result) - result.tag := command.tag(cp.tagBits - 1, 0) - result.misaligned := misaligned - result.accessFault := false.B - state := completed - }.otherwise(state := issue) - } - } - - val atom = MuxLookup(command.cmd, CacheAtomic.None.U)(Seq( - M_XA_SWAP -> CacheAtomic.Swap.U, - M_XA_ADD -> CacheAtomic.Add.U, - M_XA_XOR -> CacheAtomic.Xor.U, - M_XA_AND -> CacheAtomic.And.U, - M_XA_OR -> CacheAtomic.Or.U, - M_XA_MIN -> CacheAtomic.Min.U, - M_XA_MAX -> CacheAtomic.Max.U, - M_XA_MINU -> CacheAtomic.MinU.U, - M_XA_MAXU -> CacheAtomic.MaxU.U, - M_XLR -> CacheAtomic.LR.U, - M_XSC -> CacheAtomic.SC.U - )) - - io.maintenance.valid := state === issue && sfence - io.maintenance.bits := 0.U.asTypeOf(io.maintenance.bits) - io.maintenance.bits.addr := command.addr - io.maintenance.bits.asid := data - io.maintenance.bits.rs1 := command.size(0) - io.maintenance.bits.rs2 := command.size(1) - io.maintained.ready := state === receive && maintaining - when(io.maintenance.fire) { maintaining := true.B; state := receive } - when(io.maintained.fire) { - assert(io.maintained.bits, "LSU maintenance must acknowledge completion") - result := 0.U.asTypeOf(result) - result.tag := command.tag(cp.tagBits - 1, 0) - maintaining := false.B - state := completed - } - io.request.valid := state === issue && !sfence - io.request.bits := 0.U.asTypeOf(io.request.bits) - io.request.bits.vaddr := address - io.request.bits.tag := command.tag(cp.tagBits - 1, 0) - io.request.bits.size := command.size - io.request.bits.write := command.cmd === M_XWR - io.request.bits.execute := false.B - io.request.bits.signed := command.signed - io.request.bits.data := data - io.request.bits.atomic := atom - io.request.bits.privilege := command.dprv - io.request.bits.sum := context.sum - io.request.bits.mxr := context.mxr - io.request.bits.satpMode := context.satp(63, 60) - io.request.bits.rootPpn := context.satp(43, 0) - when(io.request.fire) { - assert( - command.cmd === M_XRD || command.cmd === M_XWR || command.cmd === M_XLR || - command.cmd === M_XSC || isAMO(command.cmd), - "LSU unsupported memory operation" - ) - state := receive - } - io.response.ready := state === receive && !maintaining - when(io.response.fire) { - assert(io.response.bits.tag === command.tag(cp.tagBits - 1, 0), "LSU virtual response tag mismatch") - result := io.response.bits - state := completed - } - when(state === replay1)(state := Mux(io.cpu.s1_kill, completed, replay2)) - when(state === replay2)(state := Mux(io.cpu.s2_kill, completed, idle)) - - val returns = state === replay2 && !io.cpu.s2_kill - val exception = result.misaligned || result.pageFault || result.accessFault - io.cpu.s2_nack := state === s2 && !io.cpu.s2_kill && !probed - io.cpu.s2_nack_cause_raw := false.B - io.cpu.s2_uncached := false.B - io.cpu.s2_paddr := command.addr(paddrBits - 1, 0) - io.cpu.s2_gpa := 0.U - io.cpu.s2_gpa_is_pte := false.B - io.cpu.s2_xcpt := 0.U.asTypeOf(io.cpu.s2_xcpt) - io.cpu.s2_xcpt.ma.ld := returns && !writes && result.misaligned - io.cpu.s2_xcpt.ma.st := returns && writes && result.misaligned - io.cpu.s2_xcpt.pf.ld := returns && !writes && result.pageFault - io.cpu.s2_xcpt.pf.st := returns && writes && result.pageFault - io.cpu.s2_xcpt.ae.ld := returns && !writes && result.accessFault - io.cpu.s2_xcpt.ae.st := returns && writes && result.accessFault - io.cpu.resp.valid := returns || probed - io.cpu.resp.bits := 0.U.asTypeOf(io.cpu.resp.bits) - io.cpu.resp.bits.tag := command.tag - io.cpu.resp.bits.cmd := command.cmd - io.cpu.resp.bits.size := command.size - io.cpu.resp.bits.addr := command.addr - io.cpu.resp.bits.signed := command.signed - io.cpu.resp.bits.dprv := command.dprv - io.cpu.resp.bits.has_data := reads && !command.no_resp && (probed || !exception) - io.cpu.resp.bits.data := Mux(probed, probedLoad, result.data) - io.cpu.resp.bits.data_word_bypass := Mux(probed, probedLoad, result.data) - io.cpu.resp.bits.data_raw := Mux(probed, probedLoad, result.data) - io.cpu.resp.bits.store_data := data - io.cpu.replay_next := false.B - // A buffered completion awaits instruction replay, not a memory transaction; release ordered decoding. - val memoryPending = state === s1 || state === s2 || state === issue || state === receive - io.cpu.ordered := !memoryPending - io.cpu.store_pending := memoryPending && writes - io.cpu.clock_enabled := true.B - io.cpu.perf := 0.U.asTypeOf(io.cpu.perf) - io.cpu.perf.grant := returns || probed - io.cpu.perf.blocked := !io.cpu.req.ready - io.cpu.uncached_resp.foreach { port => port.valid := false.B; port.bits := 0.U.asTypeOf(port.bits) } -} diff --git a/examples/cores/rocket/arch/src/main/scala/cpu/PermissionCheck.scala b/examples/cores/rocket/arch/src/main/scala/cpu/PermissionCheck.scala deleted file mode 100644 index 407193b33..000000000 --- a/examples/cores/rocket/arch/src/main/scala/cpu/PermissionCheck.scala +++ /dev/null @@ -1,86 +0,0 @@ -package hier.core.rocket - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.system.core.rocket.{CpuParams, HasCpuParameters} -import freechips.rocketchip.rocket.{PMP, PMPChecker} -import memcore.memory.cpu.PhysicalRegion -import memcore.memory.preflight.{Authorization, Permission, Params => PreparationParams} - -/** Checks the complete transport range against the submitting command's frozen PMP image. */ -@instantiable -class PermissionCheck(config: PreparationParams, regions: Seq[PhysicalRegion])(implicit val cpuParams: CpuParams) - extends Module - with HasCpuParameters { - val physicalBits = math.min(config.bus.addressBits, paddrBits) - require(regions.nonEmpty) - regions.foreach(r => require(r.base >= 0 && r.bytes > 0 && r.base + r.bytes <= (BigInt(1) << physicalBits))) - - @public - val io = IO(new Bundle { - val request = Flipped(Decoupled(new Authorization(config))) - // Selected from the captured admission context using request.bits.id, never the live CPU CSRs. - val contextValid = Input(Bool()) - val contextId = Input(UInt(config.idBits.W)) - val pmp = Input(Vec(nPMPs, new PMP)) - val response = Decoupled(new Permission(config)) - }) - - val idle :: checking :: reporting :: Nil = Enum(3) - val state = RegInit(idle) - val request = Reg(new Authorization(config)) - val capturedPmp = Reg(Vec(nPMPs, new PMP)) - val address = Reg(UInt(config.bus.addressBits.W)) - val remaining = Reg(UInt(13.W)) - val allowed = Reg(Bool()) - io.request.ready := state === idle && !reset.asBool - io.response.valid := state === reporting && !reset.asBool - io.response.bits.id := request.id - io.response.bits.allow := allowed - - when(io.request.fire) { - val r = io.request.bits - val end = r.pa +& r.bytes - val shape = io.contextValid && io.contextId === r.id && r.bytes =/= 0.U && r.bytes <= 4096.U && - end <= (BigInt(1) << physicalBits).U && - (r.privilege === 0.U || r.privilege === 1.U || r.privilege === 3.U) && - (!r.isPte || (!r.write && r.bytes === 8.U && r.privilege === 1.U)) - val regionAllowed = regions.map { region => - val permission = Mux(r.write, region.writable.B, region.readable.B) - region.normal.B && r.pa >= region.base.U && end <= (region.base + region.bytes).U && permission - }.reduce(_ || _) - request := r - capturedPmp := io.pmp - address := r.pa - remaining := r.bytes - allowed := shape && regionAllowed - state := Mux(shape && regionAllowed, checking, reporting) - } - - // A page segment may cover different PMP entries. Check each actual transport - // transaction (at most one 64-byte line), never a widened page-sized access. - val size = (1 to 6).foldLeft(0.U(3.W)) { (selected, logBytes) => - Mux(remaining >= (1 << logBytes).U && address(logBytes - 1, 0) === 0.U, logBytes.U, selected) - } - - val bytes = (1.U(13.W) << size)(12, 0) - val checker = Module(new PMPChecker(6)) - checker.io.pmp := capturedPmp - checker.io.prv := request.privilege - checker.io.addr := address.pad(paddrBits)(paddrBits - 1, 0) - checker.io.size := size - val permitted = Mux(request.write, checker.io.w, checker.io.r) - when(state === checking) { - when(!permitted) { - allowed := false.B - state := reporting - }.elsewhen(remaining === bytes) { - state := reporting - }.otherwise { - address := address + bytes - remaining := remaining - bytes - } - } - when(io.response.fire)(state := idle) -} diff --git a/examples/cores/rocket/arch/src/main/scala/cpu/Preparation.scala b/examples/cores/rocket/arch/src/main/scala/cpu/Preparation.scala deleted file mode 100644 index dc57d1106..000000000 --- a/examples/cores/rocket/arch/src/main/scala/cpu/Preparation.scala +++ /dev/null @@ -1,77 +0,0 @@ -package hier.core.rocket - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.system.core.rocket.{CpuParams, HasCpuParameters} -import freechips.rocketchip.rocket.PMP -import memcore.bus.chi.rnf.{CacheAccess, CacheResult} -import memcore.memory.cpu.PhysicalRegion -import memcore.memory.preflight.{ - Command, - PreparedSegment, - Preflight, - PreparedMap, - MapQuery, - MapResult, - MapReady, - MapTag, - Params => PreparationParams -} - -/** Uses the admission snapshot and retains translations until DMA and cache completion. */ -@instantiable -class Preparation(config: PreparationParams, regions: Seq[PhysicalRegion])(implicit val cpuParams: CpuParams) - extends Module - with HasCpuParameters { - - @public - val io = IO(new Bundle { - val command = Flipped(Decoupled(new Command(config))) - // AdmissionBridge owns the frozen context. Query it by tag instead of duplicating its table. - val contextTag = Output(UInt(config.idBits.W)) - val contextValid = Input(Bool()) - val contextId = Input(UInt(config.idBits.W)) - val contextPmp = Input(Vec(nPMPs, new PMP)) - // Every successful segment is delivered both to the retained map and the physical interlock. - // A terminal error goes to the caller too, which must cancel its unsealed reservation. - val ranges = Decoupled(new PreparedSegment(config)) - val ready = Decoupled(new MapReady(config)) - val release = Flipped(Decoupled(new MapTag(config))) - val queries = Input(Vec(2, new MapQuery(config))) - val results = Output(Vec(2, new MapResult(config))) - val pteRequest = Decoupled(new CacheAccess(config.bus)) - val pteResponse = Flipped(Decoupled(new CacheResult)) - }) - - val preflight: Instance[Preflight] = Instantiate(new Preflight(config)) - val maps: Instance[PreparedMap] = Instantiate(new PreparedMap(config)) - val permission: Instance[PermissionCheck] = Instantiate(new PermissionCheck(config, regions)) - preflight.io.command.valid := io.command.valid && maps.io.reserve.ready && !reset.asBool - preflight.io.command.bits := io.command.bits - maps.io.reserve.valid := io.command.valid && preflight.io.command.ready && !reset.asBool - maps.io.reserve.bits.id := io.command.bits.id - io.command.ready := preflight.io.command.ready && maps.io.reserve.ready && !reset.asBool - assert(preflight.io.command.fire === io.command.fire, "Preparation command reservation is not atomic") - assert(maps.io.reserve.fire === io.command.fire, "Preparation map reservation is not atomic") - - val authorization = preflight.io.authorization - io.contextTag := authorization.bits.id - permission.io.request <> authorization - permission.io.contextValid := io.contextValid - permission.io.contextId := io.contextId - permission.io.pmp := io.contextPmp - preflight.io.permission <> permission.io.response - io.pteRequest <> preflight.io.pteRequest - preflight.io.pteResponse <> io.pteResponse - - maps.io.prepared.valid := preflight.io.prepared.valid && io.ranges.ready - maps.io.prepared.bits := preflight.io.prepared.bits - io.ranges.valid := preflight.io.prepared.valid && maps.io.prepared.ready - io.ranges.bits := preflight.io.prepared.bits - preflight.io.prepared.ready := maps.io.prepared.ready && io.ranges.ready - io.ready <> maps.io.ready - maps.io.queries := io.queries - io.results := maps.io.results - maps.io.release <> io.release -} diff --git a/examples/cores/rocket/arch/src/main/scala/cpu/Rocket.scala b/examples/cores/rocket/arch/src/main/scala/cpu/Rocket.scala deleted file mode 100644 index 8afa51712..000000000 --- a/examples/cores/rocket/arch/src/main/scala/cpu/Rocket.scala +++ /dev/null @@ -1,655 +0,0 @@ -package hier.core.rocket - -import chisel3._ -import chisel3.util._ -import framework.system.core.clink.{CLink, HasCLink} -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.system.core.rocket.{CpuParams, HasCpuParameters} -import freechips.rocketchip.tile.{CustomCSR, FPU, FPUCoreIO, TraceBundle} -import freechips.rocketchip.rocket.{CSRs, PMPChecker, PRV} -import framework.system.core.rocket.{RoCCCommandBB, RoCCResponseBB, RocketBB} -import memcore.memory.fetch.{Fetch, Params => FetchParams} -import memcore.memory.cpu.{CpuMemParams, PhysicalRegion, VirtualMemory} -import memcore.bus.chi.RequesterPort -import memcore.bus.chi.rnf.{BankedChiCache, CacheAccess, CacheAtomic, CacheResult, RnfParams} -import memcore.memory.interlock.{ - CpuQuery, - Dispatch, - Tag, - Acknowledgement, - Maintenance => MaintenanceRange, - Params => TrackingParams -} - -case class Commands(compute: Boolean, scheduler: Boolean, tracking: TrackingParams = TrackingParams()) { - require(compute || scheduler) -} - -class CommandSnapshot(tracking: TrackingParams, pmps: Int)(implicit val cpuParams: CpuParams) - extends Bundle - with HasCpuParameters { - val tag = UInt(tracking.idBits.W) - val instruction = new RoCCCommandBB - val satp = UInt(64.W) - val effectivePrivilege = UInt(2.W) - val sum = Bool() - val mxr = Bool() - val pmp = Vec(pmps, new freechips.rocketchip.rocket.PMP) -} - -class AdmissionPorts(tracking: TrackingParams, pmps: Int, bus: memcore.bus.chi.Params)(implicit val cpuParams: CpuParams) - extends Bundle - with HasCpuParameters { - val reserve = Decoupled(new Dispatch(tracking)) - val command = Decoupled(new CommandSnapshot(tracking, pmps)) - val complete = Flipped(Decoupled(new Tag(tracking))) - val cancelled = Flipped(Decoupled(new Tag(tracking))) - val response = Flipped(Decoupled(new RoCCResponseBB)) - val interrupt = Input(Bool()) - val maintenance = Flipped(Decoupled(new MaintenanceRange(tracking))) - val maintained = Decoupled(new Acknowledgement(tracking)) - val cpuQuery = Output(new CpuQuery(tracking)) - val cpuAllow = Input(Bool()) - val cpuProbeAllow = Input(Bool()) - val pteRequest = Flipped(Decoupled(new CacheAccess(bus))) - val pteResponse = Decoupled(new CacheResult) - // Ownership registration only; TaskController must also join actual accelerator/DMA idle. - val outstanding = Output(UInt(log2Ceil(tracking.entries + 1).W)) -} - -class CpuCLink(config: RnfParams, commands: Option[Commands])(implicit val cpuParams: CpuParams) - extends CLink - with HasCpuParameters { - val cp = CpuMemParams(config.chi, tagBits = 6) - val resetVector = Input(UInt(64.W)) - val time = Input(UInt(64.W)) - val timerInterrupt = Input(Bool()) - val softwareInterrupt = Input(Bool()) - val externalInterrupt = Input(Bool()) - val supervisorExternalInterrupt = Input(Bool()) - val hartId = Input(UInt(hartIdLen.W)) - val chi = new RequesterPort(config.chi) - val instructionChi = new RequesterPort(config.chi) - val uncachedRequest = Decoupled(new memcore.memory.cpu.UncachedRequest(cp)) - val uncachedResponse = Flipped(Decoupled(new memcore.memory.cpu.UncachedResponse(cp))) - val trace = Output(new TraceBundle) - val retired = Output(Bool()) - val retiredPc = Output(UInt(64.W)) - val trapped = Output(Bool()) - val trapCause = Output(UInt(64.W)) - val trapValue = Output(UInt(64.W)) - val trapPc = Output(UInt(64.W)) - val cancelledData = Output(Bool()) - val admission = commands.map(mode => new AdmissionPorts(mode.tracking, nPMPs, config.chi)) -} - -/** One integer Rocket, own instruction/data memory, and explicit physical permission checks. */ -@instantiable -class Core( - config: RnfParams, - instruction: RnfParams, - regions: Seq[PhysicalRegion], - commands: Option[Commands] = None -)( - implicit val cpuParams: CpuParams) - extends Module - with HasCpuParameters - with HasCLink { - override def desiredName: String = "RocketCore" - require(!usingHypervisor && usingCompressed && fetchBytes == 4) - require(!usingFPU || fLen <= 64, "Core memory data path supports floating-point formats up to 64 bits") - require(instruction.chi == config.chi && instruction.nodeId != config.nodeId) - val cp = CpuMemParams(config.chi, tagBits = 6) - - @public val clink = IO(new CpuCLink(config, commands)) - - val cpu: Instance[RocketBB] = - Instantiate(new RocketBB( - Seq(CustomCSR(CSRs.time, (BigInt(1) << 64) - 1, Some(BigInt(0)))), - commands.exists(_.compute), - commands.exists(_.scheduler), - false, - false, - false - )) - - val fetch: Instance[Fetch] = Instantiate(new Fetch(FetchParams())) - val lsu: Instance[Lsu] = Instantiate(new Lsu(cp)) - val system: Instance[VirtualMemory] = Instantiate(new VirtualMemory(cp, regions, 0, 16)) - // Instruction words have their own translation, PMP check and coherent L1, apart from the data broker. - val instructionSystem: Instance[VirtualMemory] = - Instantiate(new VirtualMemory(cp, regions, 512, 16)) - val instructionCache: Instance[BankedChiCache] = Instantiate(new BankedChiCache(instruction.copy(lineResult = true))) - - val cacheTracking = commands match { - case Some(mode) => mode.tracking - case None => TrackingParams(addressBits = config.chi.addressBits) - } - - val cache: Instance[Cache] = Instantiate(new Cache(config.copy(probe = true), cacheTracking)) - val maintenanceActive = RegInit(false.B) - val maintenanceBlock = WireDefault(false.B) - val cacheOffer = RegInit(false.B) - val cacheOwner = RegInit(false.B) - val offerExternal = RegInit(false.B) - val ownerExternal = RegInit(false.B) - val offerPacket = Reg(new CacheAccess(config.chi)) - val preferExternal = RegInit(false.B) - val uncachedOffer = RegInit(false.B) - val uncachedOwner = RegInit(false.B) - val uncachedPacket = Reg(new memcore.memory.cpu.UncachedRequest(cp)) - val pteDenied = RegInit(false.B) - commands match { - case Some(_) => - val port = clink.admission.get - cache.io.maintenance <> port.maintenance - port.maintained <> cache.io.maintained - maintenanceBlock := port.maintenance.valid || maintenanceActive - when(port.maintenance.fire)(maintenanceActive := true.B) - when(port.maintained.fire)(maintenanceActive := false.B) - case None => - cache.io.maintenance.valid := false.B - cache.io.maintenance.bits := 0.U.asTypeOf(cache.io.maintenance.bits) - cache.io.maintained.ready := true.B - } - cache.io.access <> system.io.cacheRequest - system.io.cacheResponse <> cache.io.result - clink.chi <> cache.io.chi - system.io.active := true.B - // SFENCE.VMA invalidates the cached data and instruction translations. - system.io.flushTranslation := lsu.io.maintenance.fire - instructionSystem.io.flushTranslation := lsu.io.maintenance.fire - clink.uncachedRequest <> system.io.uncachedRequest - system.io.uncachedResponse <> clink.uncachedResponse - clink.trace := cpu.io.trace - clink.retired := cpu.io.trace.insns(0).valid && !cpu.io.trace.insns(0).exception - clink.retiredPc := cpu.io.trace.insns(0).iaddr - clink.trapped := cpu.io.trace.insns(0).valid && cpu.io.trace.insns(0).exception - clink.trapCause := cpu.io.trace.insns(0).cause - clink.trapValue := cpu.io.trace.insns(0).tval - clink.trapPc := cpu.io.trace.insns(0).iaddr - clink.cancelledData := lsu.io.cancelled - assert(clink.resetVector(63, paddrBits) === 0.U, "Core reset vector exceeds physical width") - cpu.io.hartid := clink.hartId - cpu.io.reset_vector := clink.resetVector(paddrBits - 1, 0) - cpu.io.interrupts := 0.U.asTypeOf(cpu.io.interrupts) - cpu.io.interrupts.mtip := clink.timerInterrupt - cpu.io.interrupts.msip := clink.softwareInterrupt - cpu.io.interrupts.meip := clink.externalInterrupt - cpu.io.interrupts.seip.foreach(_ := clink.supervisorExternalInterrupt) - cpu.io.traceStall := false.B - // TIME samples the chip's CLINT clock; CSRFile enforces its read-only address and counter permissions. - cpu.io.rocc.csrs(0).stall := false.B - cpu.io.rocc.csrs(0).set := true.B - cpu.io.rocc.csrs(0).sdata := clink.time - cpu.io.ptw.perf := 0.U.asTypeOf(cpu.io.ptw.perf) - cpu.io.ptw.clock_enabled := true.B - cpu.io.ptw.customCSRs.csrs.foreach { csr => - csr.stall := false.B - csr.set := false.B - csr.sdata := 0.U - } - val fpu = coreParams.fpu.map(cfg => Module(new FPU(cfg))) - fpu match { - case Some(unit) => - cpu.io.fpu :<>= unit.io.waiveAs[FPUCoreIO](_.cp_req, _.cp_resp) - unit.io.cp_req.valid := false.B - unit.io.cp_req.bits := DontCare - unit.io.cp_resp.ready := false.B - case None => - cpu.io.fpu.store_data := 0.U - cpu.io.fpu.toint_data := 0.U - cpu.io.fpu.fcsr_flags.valid := false.B - cpu.io.fpu.fcsr_flags.bits := 0.U - cpu.io.fpu.fcsr_rdy := true.B - cpu.io.fpu.nack_mem := false.B - cpu.io.fpu.illegal_rm := false.B - cpu.io.fpu.dec := 0.U.asTypeOf(cpu.io.fpu.dec) - cpu.io.fpu.sboard_set := false.B - cpu.io.fpu.sboard_clr := false.B - cpu.io.fpu.sboard_clra := 0.U - } - cpu.io.rocc.cmd.ready := true.B - cpu.io.rocc.resp.valid := false.B - cpu.io.rocc.resp.bits := 0.U.asTypeOf(cpu.io.rocc.resp.bits) - cpu.io.rocc.busy := false.B - cpu.io.rocc.interrupt := false.B - cpu.io.rocc.mem := DontCare - - lsu.io.cpu <> cpu.io.dmem - if (commands.nonEmpty) { - // Existing LSU owners, including the replay that retires them, must keep progressing. - val blockFresh = (cpu.io.roccOlderPending || maintenanceBlock) && lsu.io.idle - lsu.io.cpu.req.valid := cpu.io.dmem.req.valid && !blockFresh - cpu.io.dmem.req.ready := lsu.io.cpu.req.ready && !blockFresh - } - lsu.io.pc := cpu.io.dmemPc - lsu.io.pmp := cpu.io.ptw.pmp - val satp = - if (usingVM) Cat(cpu.io.ptw.ptbr.mode, cpu.io.ptw.ptbr.asid.pad(16), cpu.io.ptw.ptbr.ppn.pad(44)) else 0.U(64.W) - lsu.io.context.privilege := cpu.io.ptw.status.dprv - lsu.io.context.satp := satp - lsu.io.context.sum := cpu.io.ptw.status.sum - lsu.io.context.mxr := cpu.io.ptw.status.mxr - fetch.io.resetVector := clink.resetVector - fetch.io.context.privilege := cpu.io.ptw.status.prv - val instructionTranslation = usingVM.B && cpu.io.ptw.status.prv =/= PRV.M.U && satp(63, 60) =/= 0.U - fetch.io.context.satp := Mux(instructionTranslation, satp, 0.U) - fetch.io.context.sum := instructionTranslation && cpu.io.ptw.status.sum - fetch.io.context.mxr := instructionTranslation && cpu.io.ptw.status.mxr - - // Rocket kills its IBuf on the correction cycle. Fetch receives that captured - // correction one cycle later, breaking the BTB valid/redirect feedback path. - val redirectedPc = - if (usingVM) cpu.io.imem.req.bits.pc.asSInt.pad(64).asUInt - else cpu.io.imem.req.bits.pc.pad(64) - - fetch.io.redirect.valid := RegNext(cpu.io.imem.req.valid, false.B) - fetch.io.redirect.bits := RegEnable(redirectedPc, cpu.io.imem.req.valid) - fetch.io.flush := RegNext(cpu.io.imem.flush_icache, false.B) - cpu.io.imem.resp.valid := fetch.io.packet.valid - fetch.io.packet.ready := cpu.io.imem.resp.ready - cpu.io.imem.resp.bits := 0.U.asTypeOf(cpu.io.imem.resp.bits) - cpu.io.imem.resp.bits.pc := fetch.io.packet.bits.pc - cpu.io.imem.resp.bits.data := fetch.io.packet.bits.data - cpu.io.imem.resp.bits.mask := fetch.io.packet.bits.mask - cpu.io.imem.resp.bits.xcpt.pf.inst := fetch.io.packet.bits.pageFault - cpu.io.imem.resp.bits.xcpt.ae.inst := fetch.io.packet.bits.accessFault - cpu.io.imem.npc := fetch.io.npc - cpu.io.imem.clock_enabled := true.B - cpu.io.imem.perf := 0.U.asTypeOf(cpu.io.imem.perf) - cpu.io.imem.gpa.valid := false.B - cpu.io.imem.gpa.bits := 0.U - cpu.io.imem.gpa_is_pte := false.B - - // Fetch prepares its context one edge before its word offer becomes visible. - val effectivePmp = WireDefault(cpu.io.ptw.pmp) - for (i <- 0 until nPMPs) { - val active = cpu.io.ptw.pmp(i).cfg.a =/= 0.U - val torLowerBound = if (i + 1 < nPMPs) cpu.io.ptw.pmp(i + 1).tor else false.B - effectivePmp(i).cfg.res := 0.U - when(!active) { - effectivePmp(i).cfg.r := false.B - effectivePmp(i).cfg.w := false.B - effectivePmp(i).cfg.x := false.B - effectivePmp(i).mask := 0.U - } - when(!active && !torLowerBound)(effectivePmp(i).addr := 0.U) - } - val previousPmp = RegNext(effectivePmp) - fetch.io.invalidate := RegNext(cpu.io.imem.sfence.valid, false.B) || (effectivePmp.asUInt =/= previousPmp.asUInt) - val instructionLineExecutable = RegInit(false.B) - val fetchOffer = RegNext(fetch.io.request.valid, false.B) - val fetchPmp = Reg(Vec(nPMPs, new freechips.rocketchip.rocket.PMP)) - when(fetch.io.request.valid && !fetchOffer)(fetchPmp := previousPmp) - val wordPmp = Mux(fetchOffer, fetchPmp, previousPmp) - - instructionSystem.io.active := true.B - instructionSystem.io.request.valid := fetch.io.request.valid - instructionSystem.io.request.bits := 0.U.asTypeOf(instructionSystem.io.request.bits) - instructionSystem.io.request.bits.vaddr := fetch.io.request.bits.addr - instructionSystem.io.request.bits.size := 3.U - instructionSystem.io.request.bits.execute := true.B - instructionSystem.io.request.bits.privilege := fetch.io.request.bits.context.privilege - instructionSystem.io.request.bits.sum := fetch.io.request.bits.context.sum - instructionSystem.io.request.bits.mxr := fetch.io.request.bits.context.mxr - instructionSystem.io.request.bits.satpMode := fetch.io.request.bits.context.satp(63, 60) - instructionSystem.io.request.bits.rootPpn := fetch.io.request.bits.context.satp(43, 0) - fetch.io.request.ready := instructionSystem.io.request.ready - fetch.io.response.valid := instructionSystem.io.response.valid - fetch.io.response.bits.data := instructionSystem.io.response.bits.data - fetch.io.response.bits.line := instructionSystem.io.response.bits.line - fetch.io.response.bits.lineExecutable := instructionLineExecutable - fetch.io.response.bits.pageFault := instructionSystem.io.response.bits.pageFault - fetch.io.response.bits.accessFault := instructionSystem.io.response.bits.accessFault - instructionSystem.io.response.ready := fetch.io.response.ready - when(instructionSystem.io.response.valid) { - assert(!instructionSystem.io.response.bits.misaligned, "Aligned instruction word returned misalignment") - } - instructionCache.io.access <> instructionSystem.io.cacheRequest - instructionSystem.io.cacheResponse <> instructionCache.io.result - clink.instructionChi <> instructionCache.io.chi - - val instructionPmp = Reg(Vec(nPMPs, new freechips.rocketchip.rocket.PMP)) - val instructionAuthorization = Module(new PMPChecker(3)) - val instructionLineAuthorization = Module(new PMPChecker(6)) - instructionLineAuthorization.io.pmp := instructionPmp - instructionLineAuthorization.io.prv := instructionSystem.io.authorizationRequest.bits.privilege - val instructionPhysicalLine = (instructionSystem.io.authorizationRequest.bits.paddr >> 6) << 6 - instructionLineAuthorization.io.addr := instructionPhysicalLine(paddrBits - 1, 0) - instructionLineAuthorization.io.size := 6.U - - val instructionLineRegion = regions.filter(r => r.cacheable && r.normal && r.executable).map(r => - instructionPhysicalLine.pad(65) >= r.base.U(65.W) && - instructionPhysicalLine.pad(65) + 63.U < (r.base + r.bytes).U(65.W) - ).foldLeft(false.B)(_ || _) - - val instructionPending = RegInit(false.B) - val instructionAllowed = Reg(Bool()) - when(instructionSystem.io.request.fire)(instructionPmp := wordPmp) - instructionAuthorization.io.pmp := instructionPmp - instructionAuthorization.io.prv := instructionSystem.io.authorizationRequest.bits.privilege - instructionAuthorization.io.addr := instructionSystem.io.authorizationRequest.bits.paddr(paddrBits - 1, 0) - instructionAuthorization.io.size := instructionSystem.io.authorizationRequest.bits.size - instructionSystem.io.authorizationRequest.ready := !instructionPending - instructionSystem.io.authorizationResponse.valid := instructionPending - instructionSystem.io.authorizationResponse.bits := instructionAllowed - when(instructionSystem.io.authorizationRequest.fire) { - val r = instructionSystem.io.authorizationRequest.bits - instructionAllowed := !r.paddr(63, paddrBits).orR && - (!r.read || instructionAuthorization.io.r) && (!r.execute || instructionAuthorization.io.x) - instructionLineExecutable := !r.paddr( - 63, - paddrBits - ).orR && instructionLineAuthorization.io.x && instructionLineRegion - instructionPending := true.B - } - when(instructionSystem.io.authorizationResponse.fire)(instructionPending := false.B) - - // Instructions are fetched only through the cacheable L1; an uncached executable word faults. - val instructionUncached = RegInit(false.B) - val instructionUncachedTag = Reg(UInt(cp.tagBits.W)) - instructionSystem.io.uncachedRequest.ready := !instructionUncached - instructionSystem.io.uncachedResponse.valid := instructionUncached - instructionSystem.io.uncachedResponse.bits.tag := instructionUncachedTag - instructionSystem.io.uncachedResponse.bits.data := 0.U - instructionSystem.io.uncachedResponse.bits.error := true.B - when(instructionSystem.io.uncachedRequest.fire) { - instructionUncached := true.B - instructionUncachedTag := instructionSystem.io.uncachedRequest.bits.tag - } - when(instructionSystem.io.uncachedResponse.fire)(instructionUncached := false.B) - - val idle :: data :: barrier :: translationBarrier :: Nil = Enum(4) - val state = RegInit(idle) - cpu.io.interruptReady := state === idle && lsu.io.idle && !cpu.io.dmem.req.valid - val authorizationPmp = Reg(Vec(nPMPs, new freechips.rocketchip.rocket.PMP)) - val authorization = Module(new PMPChecker(3)) - authorization.io.pmp := authorizationPmp - authorization.io.prv := system.io.authorizationRequest.bits.privilege - authorization.io.addr := system.io.authorizationRequest.bits.paddr(paddrBits - 1, 0) - authorization.io.size := system.io.authorizationRequest.bits.size - val authorizationPending = RegInit(false.B) - val authorizationAllowed = Reg(Bool()) - system.io.authorizationRequest.ready := !authorizationPending - system.io.authorizationResponse.valid := authorizationPending - system.io.authorizationResponse.bits := authorizationAllowed - when(system.io.authorizationRequest.fire) { - val r = system.io.authorizationRequest.bits - authorizationAllowed := !r.paddr(63, paddrBits).orR && - (!r.read || authorization.io.r) && (!r.write || authorization.io.w) && - (!r.execute || authorization.io.x) - authorizationPending := true.B - } - when(system.io.authorizationResponse.fire)(authorizationPending := false.B) - - // Data fast path: the LSU's plain access completes in s2 when a data-TLB translation (or - // identity), permission, region, NPU interlock and an L1 hit all agree while the slow broker - // path is fully idle. A TLB miss or fault leaves the access to the broker, which walks. - val probe = lsu.io.probe - system.io.lookup.config.mode := probe.req.bits.satp(63, 60) - system.io.lookup.config.rootPpn := probe.req.bits.satp(43, 0) - system.io.lookup.req.vaddr := probe.req.bits.addr - system.io.lookup.req.write := probe.req.bits.write - system.io.lookup.req.execute := false.B - system.io.lookup.req.privilege := probe.req.bits.privilege - system.io.lookup.req.sum := probe.req.bits.sum - system.io.lookup.req.mxr := probe.req.bits.mxr - instructionSystem.io.lookup.config := 0.U.asTypeOf(instructionSystem.io.lookup.config) - instructionSystem.io.lookup.req := 0.U.asTypeOf(instructionSystem.io.lookup.req) - val probeAddr = system.io.lookup.paddr.pad(64) - val probeBytes = (1.U(4.W) << probe.req.bits.size)(3, 0) - val probeLast = probeAddr.pad(65) + probeBytes - 1.U - - def probeRegion( - write: Boolean - ): Bool = regions.filter(r => r.cacheable && r.normal && (if (write) r.writable else r.readable)).map(r => - probeAddr.pad(65) >= r.base.U(65.W) && probeLast < (r.base + r.bytes).U(65.W) - ) - .foldLeft(false.B)(_ || _) - - val probePmp = Module(new PMPChecker(3)) - probePmp.io.pmp := lsu.io.capturedPmp - probePmp.io.prv := probe.req.bits.privilege - probePmp.io.addr := probeAddr(paddrBits - 1, 0) - probePmp.io.size := probe.req.bits.size - val probePermitted = system.io.lookup.hit && !probeAddr(63, paddrBits).orR && - Mux(probe.req.bits.write, probePmp.io.w && probeRegion(true), probePmp.io.r && probeRegion(false)) - val slowIdle = state === idle && !authorizationPending && !cacheOffer && !cacheOwner && - !uncachedOffer && !uncachedOwner && !pteDenied && !maintenanceBlock - val probeQuery = probe.active && probePermitted && slowIdle - val probeAllow = WireDefault(true.B) - val l1Probe = cache.io.probe.get - val probeAllowed = probePermitted && slowIdle && probeAllow - l1Probe.req.valid := probe.req.valid && probeAllowed - l1Probe.req.bits.addr := probeAddr(config.chi.addressBits - 1, 0) - l1Probe.req.bits.write := probe.req.bits.write - l1Probe.req.bits.data := probe.req.bits.data << (probeAddr(2, 0) << 3) - l1Probe.req.bits.mask := (MuxLookup(probe.req.bits.size, 255.U(8.W))(Seq( - 0.U -> 1.U(8.W), - 1.U -> 3.U(8.W), - 2.U -> 15.U(8.W) - )) << - probeAddr(2, 0))(7, 0) - probe.req.ready := l1Probe.req.ready && probeAllowed - l1Probe.cancel := probe.cancel || (probe.active && !probeAllowed) - l1Probe.retire := probe.retire && probeAllowed && !probe.cancel - probe.cancelled := l1Probe.cancel - probe.complete.valid := l1Probe.complete.valid && probeAllowed && !probe.cancel - probe.complete.bits.hit := l1Probe.complete.bits.hit - probe.complete.bits.value := l1Probe.complete.bits.value - l1Probe.complete.ready := probe.complete.ready && probeAllowed && !probe.cancel - // Broker state covers the full accepted VM/PTE/data/fetch request through its response. - // The wrapper adds L1 outstanding==0; CMO activity/ACK must not gate its own drain. - val originalIdleDrain = state === idle && lsu.io.idle - val blockedPhysicalQuery = WireDefault(false.B) - cache.io.olderRequestsDrained := (originalIdleDrain || blockedPhysicalQuery) && !probe.active && - !authorizationPending && !cacheOffer && !cacheOwner && !uncachedOffer && !uncachedOwner && !pteDenied - commands.foreach { mode => - val port = clink.admission.get - val tracking = mode.tracking - require(tracking.addressBits == config.chi.addressBits) - require(tracking.idBits >= log2Ceil(tracking.entries)) - val live = RegInit(VecInit(Seq.fill(tracking.entries)(false.B))) - val sent = RegInit(VecInit(Seq.fill(tracking.entries)(false.B))) - val pending = RegInit(false.B) - val snapshot = Reg(new CommandSnapshot(tracking, nPMPs)) - val free = VecInit(live.map(!_)).asUInt - val selected = PriorityEncoder(free) - val olderDrained = state === idle && lsu.io.idle && !authorizationPending && - cache.io.outstanding === 0.U && !cacheOffer && !cacheOwner && !uncachedOffer && !uncachedOwner && !pteDenied - val capacity = free.orR && !pending && olderDrained && !maintenanceBlock - // Interlock ready is independent of valid. Pulse reserve only for an actual atomic WB acceptance; - // Rocket's pre-acceptance WB replay may withdraw its offer and must not create an orphan reservation. - cpu.io.rocc.cmd.ready := capacity && port.reserve.ready && - (!cpu.io.rocc.cmd.valid || cpu.io.roccCommitEligible) - port.reserve.valid := cpu.io.rocc.cmd.valid && cpu.io.roccCommitEligible && capacity && port.reserve.ready - port.reserve.bits.id := selected - assert(cpu.io.rocc.cmd.fire === port.reserve.fire, "Admission reservation must match Rocket command acceptance") - port.command.valid := pending - port.command.bits := snapshot - when(port.reserve.fire) { - snapshot.tag := selected - snapshot.instruction := cpu.io.rocc.cmd.bits - snapshot.satp := satp - // Like BEMU, an M-mode command translates through an enabled satp as S-mode. - snapshot.effectivePrivilege := Mux( - cpu.io.ptw.status.dprv === PRV.M.U && satp(63, 60) =/= 0.U, - PRV.S.U, - cpu.io.ptw.status.dprv - ) - snapshot.sum := cpu.io.ptw.status.sum - snapshot.mxr := cpu.io.ptw.status.mxr - snapshot.pmp := cpu.io.ptw.pmp - live(selected) := true.B - sent(selected) := false.B - pending := true.B - } - val tagIndexBits = log2Ceil(tracking.entries) - def index(tag: UInt): UInt = tag(tagIndexBits - 1, 0) - when(port.command.fire) { pending := false.B; sent(index(snapshot.tag)) := true.B } - val completeFound = port.complete.bits.tag < tracking.entries.U && live(index(port.complete.bits.tag)) && sent( - index(port.complete.bits.tag) - ) - val cancelledFound = port.cancelled.bits.tag < tracking.entries.U && live(index(port.cancelled.bits.tag)) && sent( - index(port.cancelled.bits.tag) - ) - port.complete.ready := completeFound - port.cancelled.ready := cancelledFound && !(port.complete.valid && port.complete.bits.tag === port.cancelled.bits.tag) - when(port.complete.valid)(assert(completeFound, "Admission completion must identify a delivered live command")) - when(port.cancelled.valid) { - assert(cancelledFound, "Admission cancellation must identify a delivered live command") - assert( - !port.complete.valid || port.complete.bits.tag =/= port.cancelled.bits.tag, - "Admission cannot complete and cancel the same command" - ) - } - when(port.complete.fire) { - live(index(port.complete.bits.tag)) := false.B; sent(index(port.complete.bits.tag)) := false.B - } - when(port.cancelled.fire) { - live(index(port.cancelled.bits.tag)) := false.B; sent(index(port.cancelled.bits.tag)) := false.B - } - port.outstanding := PopCount(live) - cpu.io.rocc.resp <> port.response - cpu.io.rocc.interrupt := port.interrupt - // Older accepted broker/PTW/LSU owners continue. Fresh EX memory and fresh word offers - // are stopped above/below until the MEM/WB command has reserved its identity. - val access = system.io.cacheRequest.bits - val uncached = system.io.uncachedRequest - port.cpuQuery.valid := (system.io.cacheRequest.valid && access.atomic =/= CacheAtomic.Fence.U) || - uncached.valid || probeQuery - port.cpuQuery.paddr := Mux( - probeQuery, - probeAddr(config.chi.addressBits - 1, 0), - Mux(uncached.valid, uncached.bits.addr(config.chi.addressBits - 1, 0), access.addr) - ) - port.cpuQuery.sizeLog2 := Mux( - probeQuery, - probe.req.bits.size, - Mux(uncached.valid, uncached.bits.size, Mux(access.atomicWord, 2.U, 3.U)) - ) - port.cpuQuery.write := Mux( - probeQuery, - probe.req.bits.write, - Mux( - uncached.valid, - uncached.bits.write, - access.write || access.atomic === CacheAtomic.SC.U || - (access.atomic >= CacheAtomic.Swap.U && access.atomic <= CacheAtomic.MaxU.U) - ) - ) - probeAllow := port.cpuProbeAllow - port.cpuQuery.olderDispatchPending := false.B - // Reservation itself drained all old owners; an unissued blocked physical - // query is younger and must not hold up the reservation's maintenance. - blockedPhysicalQuery := (system.io.cacheRequest.valid || uncached.valid) && !port.cpuAllow - val legalPte = !port.pteRequest.bits.write && port.pteRequest.bits.atomic === CacheAtomic.None.U && - !port.pteRequest.bits.atomicWord && port.pteRequest.bits.addr(2, 0) === 0.U && - port.pteRequest.bits.mask === 0.U - when(port.pteRequest.valid) { - assert(legalPte, "Core external PTE request must be an aligned read with the Walker mask") - } - val systemEligible = system.io.cacheRequest.valid && port.cpuAllow && - (!maintenanceBlock || state === data) - // Preparation already authorized this PA using its retained command context. - // Region classification additionally forbids device reads, including PTE reads. - val pteAddr = port.pteRequest.bits.addr.pad(65) - val pteCached = regions.filter(r => r.normal && r.readable && r.cacheable).map(r => - pteAddr >= r.base.U(65.W) && pteAddr + 8.U <= (r.base + r.bytes).U(65.W) - ).foldLeft(false.B)(_ || _) - val pteBusy = (cacheOffer && offerExternal) || (cacheOwner && ownerExternal) || pteDenied - val pteAvailable = port.pteRequest.valid && legalPte && !maintenanceBlock && !pteBusy - val pteEligible = pteAvailable && pteCached - val selectPte = pteEligible && (!systemEligible || preferExternal) - when(!cacheOffer && !cacheOwner && (systemEligible || pteEligible)) { - offerPacket := Mux(selectPte, port.pteRequest.bits, access) - offerExternal := selectPte - cacheOffer := true.B - } - cache.io.access.valid := cacheOffer - cache.io.access.bits := offerPacket - system.io.cacheRequest.ready := cacheOffer && !offerExternal && cache.io.access.ready - port.pteRequest.ready := cacheOffer && offerExternal && cache.io.access.ready - when(cache.io.access.fire) { - cacheOffer := false.B - cacheOwner := true.B - ownerExternal := offerExternal - preferExternal := !offerExternal - } - system.io.cacheResponse.valid := cacheOwner && !ownerExternal && cache.io.result.valid - system.io.cacheResponse.bits := cache.io.result.bits - port.pteResponse.valid := cacheOwner && ownerExternal && cache.io.result.valid - port.pteResponse.bits := cache.io.result.bits - cache.io.result.ready := cacheOwner && Mux(ownerExternal, port.pteResponse.ready, system.io.cacheResponse.ready) - when(cache.io.result.fire)(cacheOwner := false.B) - val uncachedCpuEligible = uncached.valid && port.cpuAllow && - (!maintenanceBlock || state === data) - when(!uncachedOffer && !uncachedOwner && uncachedCpuEligible) { - uncachedPacket := uncached.bits - uncachedOffer := true.B - } - clink.uncachedRequest.valid := uncachedOffer - clink.uncachedRequest.bits := uncachedPacket - uncached.ready := uncachedOffer && clink.uncachedRequest.ready - when(clink.uncachedRequest.fire) { - uncachedOffer := false.B - uncachedOwner := true.B - } - system.io.uncachedResponse.valid := uncachedOwner && clink.uncachedResponse.valid - system.io.uncachedResponse.bits := clink.uncachedResponse.bits - clink.uncachedResponse.ready := uncachedOwner && system.io.uncachedResponse.ready - when(clink.uncachedResponse.valid) { - assert(uncachedOwner, "Core uncached response requires an accepted owner") - assert(clink.uncachedResponse.bits.tag === uncachedPacket.tag, "Core uncached response tag must match its owner") - } - when(clink.uncachedResponse.fire)(uncachedOwner := false.B) - when(pteAvailable && !pteCached) { - port.pteRequest.ready := true.B - when(port.pteRequest.fire)(pteDenied := true.B) - } - when(pteDenied) { - port.pteResponse.valid := true.B - port.pteResponse.bits.data := 0.U - port.pteResponse.bits.error := true.B - when(port.pteResponse.fire)(pteDenied := false.B) - } - } - - lsu.io.maintenance.ready := false.B - lsu.io.maintained.valid := state === translationBarrier - lsu.io.maintained.bits := true.B - lsu.io.request.ready := false.B - lsu.io.response.valid := false.B - lsu.io.response.bits := system.io.response.bits - system.io.request.valid := false.B - system.io.request.bits := lsu.io.request.bits - system.io.response.ready := false.B - fetch.io.maintenance.ready := false.B - fetch.io.maintained.valid := state === barrier - fetch.io.maintained.bits := true.B - when(state === idle) { - when(lsu.io.maintenance.valid && cache.io.outstanding === 0.U) { - // No translation cache exists: draining the prior broker transaction is the SFENCE boundary. - lsu.io.maintenance.ready := true.B - when(lsu.io.maintenance.fire)(state := translationBarrier) - }.elsewhen(lsu.io.request.valid) { - system.io.request.valid := true.B - lsu.io.request.ready := system.io.request.ready - when(system.io.request.fire) { - authorizationPmp := lsu.io.capturedPmp - state := data - } - }.elsewhen(fetch.io.maintenance.valid && lsu.io.idle && cache.io.outstanding === 0.U) { - // Drained stores own their lines in the data L1; the coherent instruction L1 is snooped on those writes. - fetch.io.maintenance.ready := true.B - when(fetch.io.maintenance.fire)(state := barrier) - } - } - when(state === data) { - lsu.io.response.valid := system.io.response.valid - system.io.response.ready := lsu.io.response.ready - when(system.io.response.fire)(state := idle) - } - when(fetch.io.maintained.fire || lsu.io.maintained.fire)(state := idle) -} diff --git a/examples/cores/rocket/compiler/CMakeLists.txt b/examples/cores/rocket/compiler/CMakeLists.txt deleted file mode 100644 index 95412c193..000000000 --- a/examples/cores/rocket/compiler/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -if(NOT COMMAND add_mlir_library OR NOT COMMAND add_mlir_dialect) - message(FATAL_ERROR "MLIR CMake helpers must be loaded. Build as part of buddy-mlir.") -endif() - -# Rocket has no Ball lowering policy. Reuse the Ball-free compiler base rather -# than importing another Core's dialect or lowering implementation. -set(BB_CORE_NAME rocket) -set(BB_CORE_TARGET_PREFIX "") - -get_filename_component(_BB_REPO_ROOT "${CMAKE_CURRENT_LIST_DIR}/../../../.." ABSOLUTE) -add_subdirectory("${_BB_REPO_ROOT}/examples/cores/toy/compiler/src" - "${CMAKE_CURRENT_BINARY_DIR}/base") diff --git a/examples/cores/rocket/configs/balldomains/default.toml b/examples/cores/rocket/configs/balldomains/default.toml deleted file mode 100644 index a23e23157..000000000 --- a/examples/cores/rocket/configs/balldomains/default.toml +++ /dev/null @@ -1,9 +0,0 @@ -# Rocket-only core: no Buckyball balls. - -ballNum = 0 - -ballIdMappings = [ -] - -ballISA = [ -] diff --git a/examples/cores/rocket/configs/cpu/default.toml b/examples/cores/rocket/configs/cpu/default.toml deleted file mode 100644 index e651581c6..000000000 --- a/examples/cores/rocket/configs/cpu/default.toml +++ /dev/null @@ -1,2 +0,0 @@ -kind = "rocket" -config = "./rocket/default.toml" diff --git a/examples/cores/rocket/configs/cpu/rocket/default.toml b/examples/cores/rocket/configs/cpu/rocket/default.toml deleted file mode 100644 index f423a2a88..000000000 --- a/examples/cores/rocket/configs/cpu/rocket/default.toml +++ /dev/null @@ -1,76 +0,0 @@ -useUser = true -useSupervisor = true -useHypervisor = false -useDebug = false -useAtomics = true -useAtomicsOnlyForIO = false -useCompressed = true -useRVE = false -useConditionalZero = false -nLocalInterrupts = 0 -useNMI = false -nBreakpoints = 1 -useBPWatch = false -mcontextWidth = 0 -scontextWidth = 0 -nPerfCounters = 0 -haveBasicCounters = true -misaWritable = true -mtvecInitEnable = true -mtvecInit = 0 -mtvecWritable = true -fastLoadWord = true -fastLoadByte = false -branchPredictionModeCSR = false -clockGate = false -mvendorid = 0 -mimpid = 538447876 -haveCease = false -haveSimTimeout = false -useVM = true -useZba = true -useZbb = true -useZbs = true -haveCFlush = false - -[mulDiv] -divUnroll = 1 -divEarlyOutGranularity = 1 -enable = true -mulUnroll = 8 -mulEarlyOut = true -divEarlyOut = true - -[fpu] -divSqrt = true -sfmaLatency = 5 -dfmaLatency = 6 -fpmuLatency = 2 -ifpuLatency = 4 -enable = true -minFLen = 16 -fLen = 64 - -[dcache] -clockGate = false -nSets = 32 -nWays = 4 -nMSHRs = 4 - -[icache] -prefetch = false -nSets = 64 -nWays = 8 - -[btb] -nMatchBits = 14 -nPages = 6 -updatesOutOfOrder = false -bhtEnable = true -bhtEntries = 512 -bhtCounterLength = 1 -bhtHistoryLength = 8 -bhtHistoryBits = 3 -enable = true -nEntries = 28 -nRAS = 6 diff --git a/examples/cores/rocket/configs/default.toml b/examples/cores/rocket/configs/default.toml deleted file mode 100644 index 9f121207b..000000000 --- a/examples/cores/rocket/configs/default.toml +++ /dev/null @@ -1,6 +0,0 @@ -factory = "examples.cores.rocket.RocketCore" -# Rocket-only core configuration - -memdomain = "memdomains/default.toml" -balldomain = "balldomains/default.toml" -cpu = "cpu/default.toml" diff --git a/examples/cores/rocket/configs/memdomains/default.toml b/examples/cores/rocket/configs/memdomains/default.toml deleted file mode 100644 index 717803c90..000000000 --- a/examples/cores/rocket/configs/memdomains/default.toml +++ /dev/null @@ -1,32 +0,0 @@ -# Rocket-only stub memdomain for bank_params / chip select pairing. -# Not referenced by rocket configs/default.toml (tiles stay rocket-only). - -[bank] -num = 16 -width = 128 -entries = 16 -maskLen = 16 -channel = 1 - -[dma] -nXacts = 1 -burstMaxBytes = 1024 -busWidth = 128 -maxInFlightMemReqs = 16 - -[tlb] -size = 1 - -[tma] -readChannel = 1 -writeChannel = 1 - -[mmio] -enable = false -bankNum = 1 -bankEntries = 1 -bankWidth = 128 -readWidth = 8 - -[mem] -addrLen = 39 diff --git a/examples/cores/toy/arch/src/main/scala/ToyCore.scala b/examples/cores/toy/arch/src/main/scala/ToyCore.scala deleted file mode 100644 index a2c91ea31..000000000 --- a/examples/cores/toy/arch/src/main/scala/ToyCore.scala +++ /dev/null @@ -1,20 +0,0 @@ -package examples.cores.toy - -import framework.top.GlobalConfig -import chisel3.experimental.hierarchy.{instantiable, Instance, Instantiate} -import framework.system.core.{CoreConnection, RocketEndpoint} -import hier.core.rocket.Core -import framework.system.core.accelerator.BuckyballAccelerator - -@instantiable -class ToyCore(b: GlobalConfig) extends RocketEndpoint(b) { - require(p.buckyball.nonEmpty) - val cpu = Instantiate(new Core(p.data, p.instruction, p.regions, Some(p.commands))(p.cpu)) - clink.cpu <> cpu.clink - val accelerator = Instantiate(new BuckyballAccelerator(p.buckyball.get)) - CoreConnection.accelerator(clink.accelerator.get, accelerator) -} - -object ToyCore extends framework.system.core.RocketCoreFactory { - def instantiate(b: GlobalConfig): Instance[RocketEndpoint] = Instantiate(new ToyCore(b)) -} diff --git a/examples/cores/toy/compiler/CMakeLists.txt b/examples/cores/toy/compiler/CMakeLists.txt deleted file mode 100644 index 1893a2fc2..000000000 --- a/examples/cores/toy/compiler/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -if(NOT COMMAND add_mlir_library OR NOT COMMAND add_mlir_dialect) - message(FATAL_ERROR "MLIR CMake helpers must be loaded. Build as part of buddy-mlir.") -endif() - -set(BB_CORE_NAME toy) -set(BB_CORE_TARGET_PREFIX "") - -set(BB_CORE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/..) -get_filename_component(_BB_REPO_ROOT "${BB_CORE_DIR}/../../.." ABSOLUTE) -include_directories(${_BB_REPO_ROOT}/examples/balls) - -add_subdirectory(src) diff --git a/examples/cores/toy/compiler/include/Conversion/LowerBuckyball/Patterns/ToyLowerBuckyballPatterns.h b/examples/cores/toy/compiler/include/Conversion/LowerBuckyball/Patterns/ToyLowerBuckyballPatterns.h deleted file mode 100644 index c07fc3ade..000000000 --- a/examples/cores/toy/compiler/include/Conversion/LowerBuckyball/Patterns/ToyLowerBuckyballPatterns.h +++ /dev/null @@ -1,22 +0,0 @@ -//===- ToyLowerBuckyballPatterns.h - Toy lowering pattern hooks -*- C++ -*-===// - -#ifndef TOY_CONVERSION_LOWER_BUCKYBALL_PATTERNS_H -#define TOY_CONVERSION_LOWER_BUCKYBALL_PATTERNS_H - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" - -namespace mlir { -namespace buddy { - -void populateToyAssignPhysicalBankPatterns(RewritePatternSet &patterns, - PhysicalBankState &state); - -void populateToyLowerBuckyballToLLVMPatterns(RewritePatternSet &patterns); -void configureToyLowerBuckyballToLLVMTarget(LLVMConversionTarget &target); -void populateToyLowerBuckyballToBankSSAPatterns(RewritePatternSet &patterns); - -} // namespace buddy -} // namespace mlir - -#endif // TOY_CONVERSION_LOWER_BUCKYBALL_PATTERNS_H diff --git a/examples/cores/toy/compiler/src/Backend/include/IntrinsicsRISCVBuckyball.td b/examples/cores/toy/compiler/src/Backend/include/IntrinsicsRISCVBuckyball.td deleted file mode 100644 index dd695fb46..000000000 --- a/examples/cores/toy/compiler/src/Backend/include/IntrinsicsRISCVBuckyball.td +++ /dev/null @@ -1,3 +0,0 @@ -//===- IntrinsicsRISCVBuckyball.td - Buckyball intrinsics ----------------===// - -include "IntrinsicsRISCVBuckyballBase.td" diff --git a/examples/cores/toy/compiler/src/Backend/include/IntrinsicsRISCVBuckyballBase.td b/examples/cores/toy/compiler/src/Backend/include/IntrinsicsRISCVBuckyballBase.td deleted file mode 100644 index a8dc27e66..000000000 --- a/examples/cores/toy/compiler/src/Backend/include/IntrinsicsRISCVBuckyballBase.td +++ /dev/null @@ -1,10 +0,0 @@ -//===- IntrinsicsRISCVBuckyballBase.td - Base intrinsics -----------------===// - -let TargetPrefix = "riscv" in -def int_riscv_bb_mvin : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; - -let TargetPrefix = "riscv" in -def int_riscv_bb_mvout : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; - -let TargetPrefix = "riscv" in -def int_riscv_bb_mset : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; diff --git a/examples/cores/toy/compiler/src/CMakeLists.txt b/examples/cores/toy/compiler/src/CMakeLists.txt deleted file mode 100644 index 522ca6dd0..000000000 --- a/examples/cores/toy/compiler/src/CMakeLists.txt +++ /dev/null @@ -1,61 +0,0 @@ -if(NOT COMMAND add_mlir_library OR NOT COMMAND add_mlir_dialect) - message(FATAL_ERROR "MLIR CMake helpers must be loaded. Build as part of buddy-mlir.") -endif() - -message(STATUS "Building Toy compiler Core package") -message(STATUS " Source: ${CMAKE_CURRENT_SOURCE_DIR}") -message(STATUS " Binary: ${CMAKE_CURRENT_BINARY_DIR}") - -get_filename_component(BB_REPO_ROOT ${CMAKE_CURRENT_LIST_DIR}/../../../../.. ABSOLUTE) -set(BB_COMPILER_COMMON_DIR ${BB_REPO_ROOT}/stack/compiler/src) - -if(NOT EXISTS ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball/BuckyballOps.h) - message(FATAL_ERROR "Missing Buckyball compiler common dir: ${BB_COMPILER_COMMON_DIR}") -endif() - -include_directories(${BB_COMPILER_COMMON_DIR}) -include_directories(${BB_COMPILER_COMMON_DIR}/Dialect) -include_directories(${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball) -include_directories(${CMAKE_CURRENT_LIST_DIR}/../include) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}) -include_directories(${CMAKE_CURRENT_BINARY_DIR}) - -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/Dialect) -include_directories(${CMAKE_CURRENT_BINARY_DIR}/Dialect) -include_directories(BEFORE ${CMAKE_CURRENT_BINARY_DIR}/Dialect/Buckyball) - -set(TOY_BALL_COMPILER_DIALECT_DIRS) - -set(BB_CORE_BACKEND_INTRINSICS_DIR - ${CMAKE_CURRENT_SOURCE_DIR}/Backend/include - CACHE INTERNAL "Buckyball chip backend intrinsic directory") - -set(BB_BALL_BACKEND_INTRINSICS_DIRS) - -foreach(TOY_BALL_COMPILER_DIALECT_DIR ${TOY_BALL_COMPILER_DIALECT_DIRS}) - if(NOT EXISTS ${TOY_BALL_COMPILER_DIALECT_DIR}) - message(FATAL_ERROR "Missing toy ball compiler dialect dir: ${TOY_BALL_COMPILER_DIALECT_DIR}") - endif() - include_directories(${TOY_BALL_COMPILER_DIALECT_DIR}) - get_filename_component(TOY_BALL_COMPILER_DIR - ${TOY_BALL_COMPILER_DIALECT_DIR}/../.. - ABSOLUTE) - list(APPEND BB_BALL_BACKEND_INTRINSICS_DIRS - ${TOY_BALL_COMPILER_DIR}/Backend/include) -endforeach() - -set(BB_BALL_BACKEND_INTRINSICS_DIRS ${BB_BALL_BACKEND_INTRINSICS_DIRS} - CACHE INTERNAL "Buckyball ball backend intrinsic directories") - -add_subdirectory(Dialect) -add_subdirectory(Conversion) -add_subdirectory(Target) - -add_custom_target(buckyball-compiler-${BB_CORE_NAME} - DEPENDS - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballTransforms - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballToLLVMIRTranslation - ${BB_CORE_TARGET_PREFIX}LowerBuckyballPass - ${BB_CORE_TARGET_PREFIX}LowerTileToBuckyballPass -) diff --git a/examples/cores/toy/compiler/src/Conversion/CMakeLists.txt b/examples/cores/toy/compiler/src/Conversion/CMakeLists.txt deleted file mode 100644 index 5dd235cfa..000000000 --- a/examples/cores/toy/compiler/src/Conversion/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -# Buckyball Conversion Passes - -add_subdirectory(LowerBuckyball) -add_subdirectory(LowerTileToBuckyball) diff --git a/examples/cores/toy/compiler/src/Conversion/LowerBuckyball/CMakeLists.txt b/examples/cores/toy/compiler/src/Conversion/LowerBuckyball/CMakeLists.txt deleted file mode 100644 index 7bc594c92..000000000 --- a/examples/cores/toy/compiler/src/Conversion/LowerBuckyball/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -add_mlir_library(${BB_CORE_TARGET_PREFIX}LowerBuckyballPass - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/BaseAssignPhysicalBankPatterns.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/PhysicalBankState.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/ReportBankUsagePass.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/AssignPhysicalBanksPass.cpp - ${BB_assign_SOURCES} - ${BB_bank_ssa_SOURCES} - LowerBuckyballPass.cpp - LowerBuckyballToBankSSAPass.cpp - Patterns/ToyAssignPhysicalBankPatterns.cpp - Patterns/ToyLowerBuckyballToBankSSAPatterns.cpp - Patterns/ToyLowerBuckyballToLLVMPatterns.cpp - - LINK_LIBS PUBLIC - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballTransforms - ) - -add_custom_command(TARGET ${BB_CORE_TARGET_PREFIX}LowerBuckyballPass PRE_LINK - COMMAND ${CMAKE_COMMAND} -E rm -f - $) diff --git a/examples/cores/toy/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp b/examples/cores/toy/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp deleted file mode 100644 index a8ab8d2cc..000000000 --- a/examples/cores/toy/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp +++ /dev/null @@ -1,128 +0,0 @@ -//===- LowerBuckyballPass.cpp - Toy Buckyball lowering pass ---------------===// - -#include "Conversion/LowerBuckyball/Patterns/ToyLowerBuckyballPatterns.h" - -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/TypeConverter.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/Pass/Pass.h" - -#include "Buckyball/BuckyballDialect.h" -#include "Buckyball/BuckyballOps.h" -#include "Buckyball/Transform.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { - -class LowerBuckyballToLLVMPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerBuckyballToLLVMPass) - LowerBuckyballToLLVMPass() = default; - LowerBuckyballToLLVMPass(const LowerBuckyballToLLVMPass &) {} - - StringRef getArgument() const final { return "lower-buckyball"; } - StringRef getDescription() const final { - return "Lower Toy Buckyball dialect ops."; - } - - Option stable{*this, "stable", - llvm::cl::desc("Use stable LLVM Buckyball intrinsics."), - llvm::cl::init(false)}; - - void getDependentDialects(DialectRegistry ®istry) const override { - registry - .insert(); - } - - void runOnOperation() override { - const auto &targetConfig = buckyball_target::getBuckyballTarget(); - MLIRContext *context = &getContext(); - ModuleOp module = getOperation(); - LLVMTypeConverter converter(context); - RewritePatternSet patterns(context); - LLVMConversionTarget target(*context); - - configureBuckyballLegalizeForExportTarget(target, stable); - mlir::buddy::configureToyLowerBuckyballToLLVMTarget(target); - populateBuckyballLegalizeForLLVMExportPatterns( - converter, patterns, targetConfig.bankWidthBits / 8, - targetConfig.bankDepth, targetConfig.bankNum, - /*includeFuncOperandForwarding=*/false, stable); - mlir::buddy::populateToyLowerBuckyballToLLVMPatterns(patterns); - - target.addLegalDialect(); - - ConversionConfig config; - config.allowPatternRollback = false; - if (failed(applyPartialConversion(module, target, std::move(patterns), - config))) - signalPassFailure(); - } -}; - -class LowerBankSSAToIntrinsicsPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerBankSSAToIntrinsicsPass) - LowerBankSSAToIntrinsicsPass() = default; - LowerBankSSAToIntrinsicsPass(const LowerBankSSAToIntrinsicsPass &) {} - - StringRef getArgument() const final { return "lower-bank-ssa-to-intrinsics"; } - StringRef getDescription() const final { - return "Lower Toy bank-SSA and Buckyball ops to intrinsic ops."; - } - - Option stable{*this, "stable", - llvm::cl::desc("Use stable LLVM Buckyball intrinsics."), - llvm::cl::init(false)}; - - void getDependentDialects(DialectRegistry ®istry) const override { - registry - .insert(); - } - - void runOnOperation() override { - const auto &targetConfig = buckyball_target::getBuckyballTarget(); - MLIRContext *context = &getContext(); - ModuleOp module = getOperation(); - LLVMTypeConverter converter(context); - RewritePatternSet patterns(context); - LLVMConversionTarget target(*context); - - configureBuckyballLegalizeForExportTarget(target, stable); - target.addLegalDialect(); - populateBuckyballLegalizeForLLVMExportPatterns( - converter, patterns, targetConfig.bankWidthBits / 8, - targetConfig.bankDepth, targetConfig.bankNum, - /*includeFuncOperandForwarding=*/false, stable); - - ConversionConfig config; - config.allowPatternRollback = false; - if (failed(applyPartialConversion(module, target, std::move(patterns), - config))) - signalPassFailure(); - } -}; - -} // namespace - -void mlir::buddy::registerLowerBuckyballPass() { - PassRegistration(); -} - -void mlir::buddy::registerLowerBankSSAToIntrinsicsPass() { - PassRegistration(); -} diff --git a/examples/cores/toy/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp b/examples/cores/toy/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp deleted file mode 100644 index 9f5a82cbb..000000000 --- a/examples/cores/toy/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp +++ /dev/null @@ -1,48 +0,0 @@ -//===- LowerBuckyballToBankSSAPass.cpp - Toy bank-SSA lowering ------------===// - -#include "Conversion/LowerBuckyball/Patterns/ToyLowerBuckyballPatterns.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/Pass/Pass.h" -#include "mlir/Transforms/GreedyPatternRewriteDriver.h" - -#include "Buckyball/BuckyballDialect.h" - -using namespace mlir; - -namespace { - -class LowerBuckyballToBankSSAPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerBuckyballToBankSSAPass) - - StringRef getArgument() const final { return "lower-buckyball-to-bank-ssa"; } - StringRef getDescription() const final { - return "Lower Toy Buckyball ops to explicit bank-SSA ops."; - } - - void getDependentDialects(DialectRegistry ®istry) const override { - registry - .insert(); - } - - void runOnOperation() override { - RewritePatternSet patterns(&getContext()); - mlir::buddy::populateToyLowerBuckyballToBankSSAPatterns(patterns); - if (failed(applyPatternsGreedily(getOperation(), std::move(patterns)))) - signalPassFailure(); - } -}; - -} // namespace - -void mlir::buddy::registerLowerBuckyballToBankSSAPass() { - PassRegistration(); -} diff --git a/examples/cores/toy/compiler/src/Conversion/LowerBuckyball/Patterns/ToyAssignPhysicalBankPatterns.cpp b/examples/cores/toy/compiler/src/Conversion/LowerBuckyball/Patterns/ToyAssignPhysicalBankPatterns.cpp deleted file mode 100644 index 13bc120b1..000000000 --- a/examples/cores/toy/compiler/src/Conversion/LowerBuckyball/Patterns/ToyAssignPhysicalBankPatterns.cpp +++ /dev/null @@ -1,11 +0,0 @@ -//===- ToyAssignPhysicalBankPatterns.cpp - Toy bank assignment hooks ------===// - -#include "Conversion/LowerBuckyball/Patterns/ToyLowerBuckyballPatterns.h" - -using namespace mlir; - -void mlir::buddy::populateToyAssignPhysicalBankPatterns( - RewritePatternSet &patterns, mlir::buddy::PhysicalBankState &state) { - (void)patterns; - (void)state; -} diff --git a/examples/cores/toy/compiler/src/Conversion/LowerBuckyball/Patterns/ToyLowerBuckyballToBankSSAPatterns.cpp b/examples/cores/toy/compiler/src/Conversion/LowerBuckyball/Patterns/ToyLowerBuckyballToBankSSAPatterns.cpp deleted file mode 100644 index 8c4f9835c..000000000 --- a/examples/cores/toy/compiler/src/Conversion/LowerBuckyball/Patterns/ToyLowerBuckyballToBankSSAPatterns.cpp +++ /dev/null @@ -1,24 +0,0 @@ -//===- ToyLowerBuckyballToBankSSAPatterns.cpp - Toy bank SSA hooks --------===// - -#include "Conversion/LowerBuckyball/Patterns/ToyLowerBuckyballPatterns.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; - -namespace mlir::buddy { -#define BUCKYBALL_BANK_SSA_HOOK(BALL) \ - void populate##BALL##LowerBuckyballToBankSSAPatterns(RewritePatternSet &); -#include "BuckyballBallLoweringHooks.inc" -#undef BUCKYBALL_BANK_SSA_HOOK -} // namespace mlir::buddy - -void mlir::buddy::populateToyLowerBuckyballToBankSSAPatterns( - RewritePatternSet &patterns) { - for (llvm::StringRef ball : buckyball_target::getBuckyballTarget().balls) { -#define BUCKYBALL_BANK_SSA_HOOK(BALL) \ - if (ball == #BALL) \ - populate##BALL##LowerBuckyballToBankSSAPatterns(patterns); -#include "BuckyballBallLoweringHooks.inc" -#undef BUCKYBALL_BANK_SSA_HOOK - } -} diff --git a/examples/cores/toy/compiler/src/Conversion/LowerBuckyball/Patterns/ToyLowerBuckyballToLLVMPatterns.cpp b/examples/cores/toy/compiler/src/Conversion/LowerBuckyball/Patterns/ToyLowerBuckyballToLLVMPatterns.cpp deleted file mode 100644 index f995205de..000000000 --- a/examples/cores/toy/compiler/src/Conversion/LowerBuckyball/Patterns/ToyLowerBuckyballToLLVMPatterns.cpp +++ /dev/null @@ -1,172 +0,0 @@ -//===- ToyLowerBuckyballToLLVMPatterns.cpp - Toy LLVM patterns ------------===// - -#include "Conversion/LowerBuckyball/Patterns/ToyLowerBuckyballPatterns.h" - -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinTypes.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { - -static LLVM::LLVMFunctionType getPrintfType(MLIRContext *context) { - auto i32 = IntegerType::get(context, 32); - auto ptr = LLVM::LLVMPointerType::get(context); - return LLVM::LLVMFunctionType::get(i32, ptr, true); -} - -static FlatSymbolRefAttr getOrInsertPrintf(PatternRewriter &rewriter, - ModuleOp module) { - auto *context = module.getContext(); - if (module.lookupSymbol("printf")) - return SymbolRefAttr::get(context, "printf"); - - PatternRewriter::InsertionGuard insertGuard(rewriter); - rewriter.setInsertionPointToStart(module.getBody()); - rewriter.create(module.getLoc(), "printf", - getPrintfType(context)); - return SymbolRefAttr::get(context, "printf"); -} - -static Value getOrCreateGlobalString(Location loc, OpBuilder &builder, - StringRef name, StringRef value, - ModuleOp module) { - LLVM::GlobalOp global; - if (!(global = module.lookupSymbol(name))) { - OpBuilder::InsertionGuard insertGuard(builder); - builder.setInsertionPointToStart(module.getBody()); - auto type = LLVM::LLVMArrayType::get( - IntegerType::get(builder.getContext(), 8), value.size()); - global = - builder.create(loc, type, true, LLVM::Linkage::Internal, - name, builder.getStringAttr(value), 0); - } - - Value globalPtr = builder.create(loc, global); - Value zero = builder.create(loc, builder.getI64Type(), - builder.getI64IntegerAttr(0)); - return builder.create( - loc, LLVM::LLVMPointerType::get(builder.getContext()), global.getType(), - globalPtr, ArrayRef({zero, zero})); -} - -class BBPrintMemRefOpLowering : public ConversionPattern { -public: - explicit BBPrintMemRefOpLowering(MLIRContext *context) - : ConversionPattern(PrintMemRefOp::getOperationName(), 1, context) {} - - LogicalResult - matchAndRewrite(Operation *op, ArrayRef operands, - ConversionPatternRewriter &rewriter) const override { - auto memRefType = cast(*op->operand_type_begin()); - auto shape = memRefType.getShape(); - Type elemTy = memRefType.getElementType(); - auto loc = op->getLoc(); - auto module = op->getParentOfType(); - auto printfRef = getOrInsertPrintf(rewriter, module); - - Value fmt; - if (elemTy == rewriter.getF32Type() || elemTy == rewriter.getF64Type()) { - fmt = getOrCreateGlobalString(loc, rewriter, "frmt_spec", - StringRef("%f \0", 4), module); - } else if (elemTy == rewriter.getI8Type() || - elemTy == rewriter.getI32Type()) { - fmt = getOrCreateGlobalString(loc, rewriter, "frmt_spec", - StringRef("%d \0", 4), module); - } else { - op->emitError("print_memref supports only i8, i32, f32 and f64"); - return failure(); - } - - Value nl = getOrCreateGlobalString(loc, rewriter, "nl", - StringRef("\n\0", 2), module); - SmallVector ivs; - for (unsigned i = 0, e = shape.size(); i != e; ++i) { - auto lb = rewriter.create(loc, 0); - auto ub = rewriter.create(loc, shape[i]); - auto step = rewriter.create(loc, 1); - auto loop = rewriter.create(loc, lb, ub, step); - for (Operation &nested : llvm::make_early_inc_range(*loop.getBody())) - rewriter.eraseOp(&nested); - ivs.push_back(loop.getInductionVar()); - - rewriter.setInsertionPointToEnd(loop.getBody()); - if (i != e - 1) - rewriter.create(loc, getPrintfType(rewriter.getContext()), - printfRef, nl); - rewriter.create(loc); - rewriter.setInsertionPointToStart(loop.getBody()); - } - - auto printOp = cast(op); - Value elem = rewriter.create(loc, printOp.getInput(), ivs); - if (elem.getType() == rewriter.getF32Type()) - elem = rewriter.create(loc, rewriter.getF64Type(), elem); - else if (elem.getType() == rewriter.getI8Type()) - elem = rewriter.create(loc, rewriter.getI32Type(), elem); - - rewriter.create(loc, getPrintfType(rewriter.getContext()), - printfRef, ArrayRef({fmt, elem})); - rewriter.eraseOp(op); - return success(); - } -}; - -class BBPrintScalarOpLowering : public ConversionPattern { -public: - explicit BBPrintScalarOpLowering(MLIRContext *context) - : ConversionPattern(PrintScalarOp::getOperationName(), 1, context) {} - - LogicalResult - matchAndRewrite(Operation *op, ArrayRef operands, - ConversionPatternRewriter &rewriter) const override { - auto loc = op->getLoc(); - auto module = op->getParentOfType(); - auto printfRef = getOrInsertPrintf(rewriter, module); - Type elemTy = op->getOperand(0).getType(); - - Value fmt; - if (elemTy == rewriter.getF32Type() || elemTy == rewriter.getF64Type()) { - fmt = getOrCreateGlobalString(loc, rewriter, "scalar_fmt", - StringRef("%f\n\0", 5), module); - } else if (elemTy == rewriter.getI8Type() || - elemTy == rewriter.getI32Type()) { - fmt = getOrCreateGlobalString(loc, rewriter, "scalar_fmt", - StringRef("%d\n\0", 5), module); - } else { - op->emitError("print_scalar supports only i8, i32, f32 and f64"); - return failure(); - } - - Value value = op->getOperand(0); - if (elemTy == rewriter.getF32Type()) - value = rewriter.create(loc, rewriter.getF64Type(), value); - else if (elemTy == rewriter.getI8Type()) - value = rewriter.create(loc, rewriter.getI32Type(), value); - - rewriter.create(loc, getPrintfType(rewriter.getContext()), - printfRef, ArrayRef({fmt, value})); - rewriter.eraseOp(op); - return success(); - } -}; - -} // namespace - -void mlir::buddy::populateToyLowerBuckyballToLLVMPatterns( - RewritePatternSet &patterns) { - patterns.add( - patterns.getContext()); -} - -void mlir::buddy::configureToyLowerBuckyballToLLVMTarget( - LLVMConversionTarget &target) { - target.addIllegalOp(); -} diff --git a/examples/cores/toy/compiler/src/Conversion/LowerTileToBuckyball/CMakeLists.txt b/examples/cores/toy/compiler/src/Conversion/LowerTileToBuckyball/CMakeLists.txt deleted file mode 100644 index 82b282ef1..000000000 --- a/examples/cores/toy/compiler/src/Conversion/LowerTileToBuckyball/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -add_mlir_library(${BB_CORE_TARGET_PREFIX}LowerTileToBuckyballPass - LowerTileToBuckyball.cpp - - LINK_LIBS PUBLIC - BuddyTile - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball - MLIRLinalgDialect -) diff --git a/examples/cores/toy/compiler/src/Conversion/LowerTileToBuckyball/LowerTileToBuckyball.cpp b/examples/cores/toy/compiler/src/Conversion/LowerTileToBuckyball/LowerTileToBuckyball.cpp deleted file mode 100644 index 6914fbdb4..000000000 --- a/examples/cores/toy/compiler/src/Conversion/LowerTileToBuckyball/LowerTileToBuckyball.cpp +++ /dev/null @@ -1,226 +0,0 @@ -//===- LowerTileToBuckyball.cpp - Toy tile->buckyball pass ---------------===// -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//===----------------------------------------------------------------------===// - -#include "Conversion/LowerTileToBuckyball/LowerTileToBuckyball.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinOps.h" -#include "mlir/IR/PatternMatch.h" -#include "mlir/Pass/Pass.h" -#include "mlir/Transforms/DialectConversion.h" - -#include "Buckyball/BuckyballDialect.h" -#include "Target/BuckyballTargetRegistry.h" -#include "Tile/TileDialect.h" -#include "Tile/TileOps.h" -#include "Tile/Transform.h" - -using namespace mlir; -namespace tile = ::buddy::tile; -using mlir::buddy::kDefaultBankWidthBytes; - -namespace { - -static Value cstF32(OpBuilder &b, Location loc, float v) { - return b.create(loc, b.getF32Type(), b.getF32FloatAttr(v)); -} - -class TileConv2dLowering : public OpRewritePattern { -public: - explicit TileConv2dLowering(MLIRContext *context, int64_t bankWidthBytes, - int64_t bankDepth, int64_t /*bankNum*/) - : OpRewritePattern(context), bankWidthBytes(bankWidthBytes), - bankDepth(bankDepth) {} - - LogicalResult matchAndRewrite(tile::TileConv2dOp op, - PatternRewriter &rewriter) const override { - Location loc = op.getLoc(); - - Value input = op.getInput(); - Value filter = op.getFilter(); - Value output = op.getOutput(); - - auto inType = cast(input.getType()); - auto filterType = cast(filter.getType()); - auto outType = cast(output.getType()); - - auto inShape = inType.getShape(); - auto fShape = filterType.getShape(); - auto outShape = outType.getShape(); - - int64_t N = inShape[0], H = inShape[1], W = inShape[2], C = inShape[3]; - int64_t KH = fShape[0], KW = fShape[1], OC = fShape[3]; - int64_t OH = outShape[1], OW = outShape[2]; - - if (!inType.getElementType().isF32() || - !filterType.getElementType().isF32()) - return op.emitError("tile_conv2d lowering currently expects f32"); - if (N <= 0 || H <= 0 || W <= 0 || C <= 0 || KH <= 0 || KW <= 0 || OC <= 0 || - OH <= 0 || OW <= 0) - return op.emitError("tile_conv2d requires positive static shapes"); - Value zero = rewriter.create(loc, 0); - Value one = rewriter.create(loc, 1); - Value nUpper = rewriter.create(loc, N); - Value ohUpper = rewriter.create(loc, OH); - Value owUpper = rewriter.create(loc, OW); - Value ocUpper = rewriter.create(loc, OC); - Value khUpper = rewriter.create(loc, KH); - Value kwUpper = rewriter.create(loc, KW); - Value cUpper = rewriter.create(loc, C); - Value zeroF32 = cstF32(rewriter, loc, 0.0f); - - auto nLoop = rewriter.create(loc, zero, nUpper, one); - { - OpBuilder::InsertionGuard nGuard(rewriter); - rewriter.setInsertionPointToStart(nLoop.getBody()); - Value nIv = nLoop.getInductionVar(); - - auto ohLoop = rewriter.create(loc, zero, ohUpper, one); - { - OpBuilder::InsertionGuard ohGuard(rewriter); - rewriter.setInsertionPointToStart(ohLoop.getBody()); - Value ohIv = ohLoop.getInductionVar(); - - auto owLoop = rewriter.create(loc, zero, owUpper, one); - { - OpBuilder::InsertionGuard owGuard(rewriter); - rewriter.setInsertionPointToStart(owLoop.getBody()); - Value owIv = owLoop.getInductionVar(); - - auto ocLoop = rewriter.create(loc, zero, ocUpper, one); - { - OpBuilder::InsertionGuard ocGuard(rewriter); - rewriter.setInsertionPointToStart(ocLoop.getBody()); - Value ocIv = ocLoop.getInductionVar(); - - Value init = zeroF32; - auto khLoop = - rewriter.create(loc, zero, khUpper, one, init); - { - OpBuilder::InsertionGuard khGuard(rewriter); - rewriter.setInsertionPointToStart(khLoop.getBody()); - Value khIv = khLoop.getInductionVar(); - Value khAcc = khLoop.getRegionIterArgs().front(); - - auto kwLoop = - rewriter.create(loc, zero, kwUpper, one, khAcc); - { - OpBuilder::InsertionGuard kwGuard(rewriter); - rewriter.setInsertionPointToStart(kwLoop.getBody()); - Value kwIv = kwLoop.getInductionVar(); - Value kwAcc = kwLoop.getRegionIterArgs().front(); - - auto cLoop = - rewriter.create(loc, zero, cUpper, one, kwAcc); - { - OpBuilder::InsertionGuard cGuard(rewriter); - rewriter.setInsertionPointToStart(cLoop.getBody()); - Value cIv = cLoop.getInductionVar(); - Value cAcc = cLoop.getRegionIterArgs().front(); - Value inH = rewriter.create(loc, ohIv, khIv); - Value inW = rewriter.create(loc, owIv, kwIv); - Value inValue = rewriter.create( - loc, input, ValueRange{nIv, inH, inW, cIv}); - Value filterValue = rewriter.create( - loc, filter, ValueRange{khIv, kwIv, cIv, ocIv}); - Value product = - rewriter.create(loc, inValue, filterValue); - Value sum = - rewriter.create(loc, cAcc, product); - rewriter.create(loc, sum); - } - rewriter.setInsertionPointAfter(cLoop); - rewriter.create(loc, cLoop.getResult(0)); - } - rewriter.setInsertionPointAfter(kwLoop); - rewriter.create(loc, kwLoop.getResult(0)); - } - rewriter.setInsertionPointAfter(khLoop); - rewriter.create(loc, khLoop.getResult(0), output, - ValueRange{nIv, ohIv, owIv, ocIv}); - } - } - } - } - - rewriter.eraseOp(op); - return success(); - } - -private: - int64_t bankWidthBytes, bankDepth; -}; - -} // namespace - -void mlir::populateLowerTileToBuckyballConversionPatterns( - RewritePatternSet &patterns, int64_t bankWidthBytes, int64_t bankDepth, - int64_t bankNum) { - patterns.add(patterns.getContext(), bankWidthBytes, - bankDepth, bankNum); -} - -namespace { - -class LowerTileToBuckyballPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerTileToBuckyballPass) - StringRef getArgument() const final { return "convert-tile-to-buckyball"; } - StringRef getDescription() const final { - return "Convert Tile dialect to Buckyball dialect"; - } - LowerTileToBuckyballPass() = default; - LowerTileToBuckyballPass(const LowerTileToBuckyballPass &) {} - - void getDependentDialects(DialectRegistry ®istry) const override { - registry.insert<::buddy::tile::TileDialect, - ::buddy::buckyball::BuckyballDialect, func::FuncDialect, - memref::MemRefDialect, arith::ArithDialect, scf::SCFDialect, - linalg::LinalgDialect>(); - } - - void runOnOperation() override { - const auto &targetConfig = buckyball_target::getBuckyballTarget(); - MLIRContext *context = &getContext(); - ModuleOp module = getOperation(); - - ConversionTarget target(*context); - target.addLegalDialect<::buddy::buckyball::BuckyballDialect, - memref::MemRefDialect, arith::ArithDialect, - scf::SCFDialect, func::FuncDialect, - linalg::LinalgDialect>(); - target.addIllegalDialect<::buddy::tile::TileDialect>(); - - RewritePatternSet patterns(context); - populateLowerTileToBuckyballConversionPatterns( - patterns, targetConfig.bankWidthBits / 8, targetConfig.bankDepth, - targetConfig.bankNum); - - if (failed(applyPartialConversion(module, target, std::move(patterns)))) - signalPassFailure(); - } -}; - -} // namespace - -void mlir::buddy::registerLowerTileToBuckyballPass() { - PassRegistration(); -} diff --git a/examples/cores/toy/compiler/src/Dialect/Buckyball/CMakeLists.txt b/examples/cores/toy/compiler/src/Dialect/Buckyball/CMakeLists.txt deleted file mode 100644 index aa8da8d59..000000000 --- a/examples/cores/toy/compiler/src/Dialect/Buckyball/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -# Generate dialect and ops from the Toy dialect assembly. -set(LLVM_TARGET_DEFINITIONS ${BB_BUCKYBALL_DIALECT_TD}) -mlir_tablegen(Buckyball.h.inc -gen-op-decls) -mlir_tablegen(Buckyball.cpp.inc -gen-op-defs) -mlir_tablegen(BuckyballTypes.h.inc -gen-typedef-decls -typedefs-dialect=buckyball) -mlir_tablegen(BuckyballTypes.cpp.inc -gen-typedef-defs -typedefs-dialect=buckyball) -mlir_tablegen(BuckyballDialect.h.inc -gen-dialect-decls -dialect=buckyball) -mlir_tablegen(BuckyballDialect.cpp.inc -gen-dialect-defs -dialect=buckyball) -add_public_tablegen_target(${BB_CORE_TARGET_PREFIX}MLIRBuckyballIncGen) -add_mlir_doc(ToyBuckyball ${BB_CORE_TARGET_PREFIX}Buckyball Dialects/ - -gen-dialect-doc) - -# Generate LLVM IR conversions -set(LLVM_TARGET_DEFINITIONS ${BB_BUCKYBALL_DIALECT_TD}) -mlir_tablegen(BuckyballConversions.inc -gen-llvmir-conversions) -add_public_tablegen_target(${BB_CORE_TARGET_PREFIX}BuddyBuckyballConversionsIncGen) - -# Add subdirectories -add_subdirectory(IR) -add_subdirectory(Transforms) diff --git a/examples/cores/toy/compiler/src/Dialect/Buckyball/IR/CMakeLists.txt b/examples/cores/toy/compiler/src/Dialect/Buckyball/IR/CMakeLists.txt deleted file mode 100644 index 42c7f8807..000000000 --- a/examples/cores/toy/compiler/src/Dialect/Buckyball/IR/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -add_mlir_dialect_library(${BB_CORE_TARGET_PREFIX}BuddyBuckyball - ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball/IR/BuckyballDialect.cpp - ${BB_TARGET_REGISTRY_SOURCE} - ${BB_BALL_IR_SOURCES} - - LINK_LIBS PUBLIC - MLIRIR - MLIRLLVMDialect - MLIRBufferizationDialect - MLIRMemRefDialect -) - -target_include_directories(${BB_CORE_TARGET_PREFIX}BuddyBuckyball PUBLIC - ${BB_COMPILER_COMMON_DIR} - ${BB_COMPILER_COMMON_DIR}/Dialect - ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball - ${CMAKE_CURRENT_BINARY_DIR}/.. - ${CMAKE_CURRENT_BINARY_DIR}/../../.. -) diff --git a/examples/cores/toy/compiler/src/Dialect/Buckyball/ToyBuckyball.td b/examples/cores/toy/compiler/src/Dialect/Buckyball/ToyBuckyball.td deleted file mode 100644 index 303415d74..000000000 --- a/examples/cores/toy/compiler/src/Dialect/Buckyball/ToyBuckyball.td +++ /dev/null @@ -1,27 +0,0 @@ -//===-- ToyBuckyball.td - Toy Buckyball dialect operations -*- tablegen -*-===// -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//===----------------------------------------------------------------------===// -// -// Toy mounts leftover balls: Gemmini, Trace, Mxfp2Int. -// Gemmini/Trace/Mxfp2Int lower outside this dialect (Gemmini dialect / custom). -// -//===----------------------------------------------------------------------===// - -#ifndef TOY_BUCKYBALL_OPS -#define TOY_BUCKYBALL_OPS - -include "Buckyball.td" - -#endif // TOY_BUCKYBALL_OPS diff --git a/examples/cores/toy/compiler/src/Dialect/Buckyball/Transforms/CMakeLists.txt b/examples/cores/toy/compiler/src/Dialect/Buckyball/Transforms/CMakeLists.txt deleted file mode 100644 index adb89436c..000000000 --- a/examples/cores/toy/compiler/src/Dialect/Buckyball/Transforms/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -add_mlir_dialect_library(${BB_CORE_TARGET_PREFIX}BuddyBuckyballTransforms - ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.cpp - ${BB_legalize_SOURCES} - LegalizeForLLVMExport.cpp - - LINK_LIBS PUBLIC - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball - MLIRIR - MLIRLLVMCommonConversion - MLIRLLVMDialect - ) diff --git a/examples/cores/toy/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/cores/toy/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 7af964ed7..000000000 --- a/examples/cores/toy/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,51 +0,0 @@ -//===- LegalizeForLLVMExport.cpp - Toy Buckyball LLVM lowering ------------===// - -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/Transform.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace buddy::buckyball::legalize; - -namespace mlir::buddy::buckyball { -#define BUCKYBALL_LEGALIZE_HOOK(BALL) \ - void populate##BALL##LegalizeForLLVMExportPatterns( \ - LLVMTypeConverter &, RewritePatternSet &, bool, int64_t); \ - void configure##BALL##LegalizeForExportTarget(LLVMConversionTarget &, bool); -#include "BuckyballBallLoweringHooks.inc" -#undef BUCKYBALL_LEGALIZE_HOOK -} // namespace mlir::buddy::buckyball - -void mlir::populateBuckyballLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, - int64_t bankWidthBytes, int64_t bankDepth, int64_t bankNum, - bool includeFuncOperandForwarding, bool stable) { - populateBaseLegalizeForLLVMExportPatterns(converter, patterns, - includeFuncOperandForwarding); - for (llvm::StringRef ball : buckyball_target::getBuckyballTarget().balls) { -#define BUCKYBALL_LEGALIZE_HOOK(BALL) \ - if (ball == #BALL) \ - buddy::buckyball::populate##BALL##LegalizeForLLVMExportPatterns( \ - converter, patterns, stable, bankDepth); -#include "BuckyballBallLoweringHooks.inc" -#undef BUCKYBALL_LEGALIZE_HOOK - } - (void)bankWidthBytes; - (void)bankNum; -} - -void mlir::configureBuckyballLegalizeForExportTarget( - LLVMConversionTarget &target, bool stable) { - configureBaseLegalizeForExportTarget(target); - for (llvm::StringRef ball : buckyball_target::getBuckyballTarget().balls) { -#define BUCKYBALL_LEGALIZE_HOOK(BALL) \ - if (ball == #BALL) \ - buddy::buckyball::configure##BALL##LegalizeForExportTarget(target, stable); -#include "BuckyballBallLoweringHooks.inc" -#undef BUCKYBALL_LEGALIZE_HOOK - } -} diff --git a/examples/cores/toy/compiler/src/Dialect/CMakeLists.txt b/examples/cores/toy/compiler/src/Dialect/CMakeLists.txt deleted file mode 100644 index 33d2424f0..000000000 --- a/examples/cores/toy/compiler/src/Dialect/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -# Buckyball Dialects - -add_subdirectory(Buckyball) -# add_subdirectory(vecball) -# add_subdirectory(matrixball) diff --git a/examples/cores/toy/compiler/src/Target/CMakeLists.txt b/examples/cores/toy/compiler/src/Target/CMakeLists.txt deleted file mode 100644 index cf3f2b4c1..000000000 --- a/examples/cores/toy/compiler/src/Target/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -# Buckyball Target Support - -add_subdirectory(LLVMIR) diff --git a/examples/cores/toy/compiler/src/Target/LLVMIR/CMakeLists.txt b/examples/cores/toy/compiler/src/Target/LLVMIR/CMakeLists.txt deleted file mode 100644 index 486c68391..000000000 --- a/examples/cores/toy/compiler/src/Target/LLVMIR/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -# Buckyball LLVM IR Translation - -add_subdirectory(Dialect) diff --git a/examples/cores/toy/compiler/src/Target/LLVMIR/Dialect/Buckyball/CMakeLists.txt b/examples/cores/toy/compiler/src/Target/LLVMIR/Dialect/Buckyball/CMakeLists.txt deleted file mode 100644 index 4afc6820b..000000000 --- a/examples/cores/toy/compiler/src/Target/LLVMIR/Dialect/Buckyball/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -add_mlir_translation_library(${BB_CORE_TARGET_PREFIX}BuddyBuckyballToLLVMIRTranslation - ${BB_COMPILER_COMMON_DIR}/Target/LLVMIR/Dialect/Buckyball/BuckyballToLLVMIRTranslation.cpp - - DEPENDS - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballConversionsIncGen - intrinsics_gen - - LINK_COMPONENTS - Core - - LINK_LIBS PUBLIC - MLIRIR - MLIRSupport - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball - ) diff --git a/examples/cores/toy/compiler/src/Target/LLVMIR/Dialect/CMakeLists.txt b/examples/cores/toy/compiler/src/Target/LLVMIR/Dialect/CMakeLists.txt deleted file mode 100644 index 890a6514d..000000000 --- a/examples/cores/toy/compiler/src/Target/LLVMIR/Dialect/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -# Buckyball Dialect LLVM IR Translation - -add_subdirectory(Buckyball) diff --git a/examples/cores/toy/configs/balldomains/balls/smatmul/lane16.toml b/examples/cores/toy/configs/balldomains/balls/smatmul/lane16.toml deleted file mode 100644 index c0a935cc7..000000000 --- a/examples/cores/toy/configs/balldomains/balls/smatmul/lane16.toml +++ /dev/null @@ -1,8 +0,0 @@ -[ball] -InputNum = 16 -inputWidth = 8 -lane = 16 -wsReuseTiles = 2 -outputWidth = 32 -numMulThreads = 16 -numCasThreads = 16 diff --git a/examples/cores/toy/configs/balldomains/default.toml b/examples/cores/toy/configs/balldomains/default.toml deleted file mode 100644 index 791c5bd2f..000000000 --- a/examples/cores/toy/configs/balldomains/default.toml +++ /dev/null @@ -1,39 +0,0 @@ -# Toy: virtual single-core. Mounts leftover balls not used by pebble/poly/goban. - -ballNum = 4 - -ballIdMappings = [ - { ballId = 0, ballName = "GemminiBall", ballClass = "examples.balls.gemmini.GemminiBall", config = "../../../../balls/gemmini/configs/default.toml", inBW = 2, outBW = 4 }, - { ballId = 1, ballName = "TraceBall", ballClass = "examples.balls.trace.TraceBall", config = "../../../../balls/trace/configs/default.toml", inBW = 1, outBW = 1 }, - { ballId = 2, ballName = "Mxfp2IntBall", ballClass = "examples.balls.mxfp2int.Mxfp2IntBall", config = "../../../../balls/mxfp2int/configs/default.toml", inBW = 1, outBW = 1, mmioReadBW = 1 }, - { ballId = 3, ballName = "ReluBall", ballClass = "examples.balls.relu.ReluBall", config = "../../../../balls/relu/configs/default.toml", inBW = 1, outBW = 1 }, -] - -ballISA = [ - { mnemonic = "GEMMINI_CONFIG", funct7 = 2, bid = 0 }, - { mnemonic = "GEMMINI_FLUSH", funct7 = 3, bid = 0 }, - { mnemonic = "GEMMINI_PRELOAD", funct7 = 53, bid = 0 }, - { mnemonic = "GEMMINI_COMPUTE_PRELOADED", funct7 = 66, bid = 0 }, - { mnemonic = "GEMMINI_COMPUTE_ACCUMULATED", funct7 = 67, bid = 0 }, - { mnemonic = "GEMMINI_LOOP_WS_CONFIG_BOUNDS", funct7 = 80, bid = 0 }, - { mnemonic = "GEMMINI_LOOP_WS_CONFIG_ADDR_A", funct7 = 81, bid = 0 }, - { mnemonic = "GEMMINI_LOOP_WS_CONFIG_ADDR_B", funct7 = 82, bid = 0 }, - { mnemonic = "GEMMINI_LOOP_WS_CONFIG_ADDR_D", funct7 = 83, bid = 0 }, - { mnemonic = "GEMMINI_LOOP_WS_CONFIG_ADDR_C", funct7 = 84, bid = 0 }, - { mnemonic = "GEMMINI_LOOP_WS_CONFIG_STRIDES_AB", funct7 = 85, bid = 0 }, - { mnemonic = "GEMMINI_LOOP_WS_CONFIG_STRIDES_DC", funct7 = 86, bid = 0 }, - { mnemonic = "GEMMINI_LOOP_WS", funct7 = 87, bid = 0 }, - { mnemonic = "GEMMINI_LOOP_CONV_WS_CONFIG_1", funct7 = 96, bid = 0 }, - { mnemonic = "GEMMINI_LOOP_CONV_WS_CONFIG_2", funct7 = 97, bid = 0 }, - { mnemonic = "GEMMINI_LOOP_CONV_WS_CONFIG_3", funct7 = 98, bid = 0 }, - { mnemonic = "GEMMINI_LOOP_CONV_WS_CONFIG_4", funct7 = 99, bid = 0 }, - { mnemonic = "GEMMINI_LOOP_CONV_WS_CONFIG_5", funct7 = 100, bid = 0 }, - { mnemonic = "GEMMINI_LOOP_CONV_WS_CONFIG_6", funct7 = 101, bid = 0 }, - { mnemonic = "GEMMINI_LOOP_CONV_WS_CONFIG_7", funct7 = 102, bid = 0 }, - { mnemonic = "GEMMINI_LOOP_CONV_WS_CONFIG_8", funct7 = 103, bid = 0 }, - { mnemonic = "GEMMINI_LOOP_CONV_WS_CONFIG_9", funct7 = 104, bid = 0 }, - { mnemonic = "GEMMINI_LOOP_CONV_WS", funct7 = 105, bid = 0 }, - { mnemonic = "BDB_COUNTER", funct7 = 4, bid = 1 }, - { mnemonic = "MXFP2INT", funct7 = 55, bid = 2 }, - { mnemonic = "RELU", funct7 = 50, bid = 3 }, -] diff --git a/examples/cores/toy/configs/cpu/default.toml b/examples/cores/toy/configs/cpu/default.toml deleted file mode 100644 index 1bf2f46e8..000000000 --- a/examples/cores/toy/configs/cpu/default.toml +++ /dev/null @@ -1,2 +0,0 @@ -kind = "rocket" -config = "../../../rocket/configs/cpu/rocket/default.toml" diff --git a/examples/cores/toy/configs/default.toml b/examples/cores/toy/configs/default.toml deleted file mode 100644 index e40977c2a..000000000 --- a/examples/cores/toy/configs/default.toml +++ /dev/null @@ -1,8 +0,0 @@ -factory = "examples.cores.toy.ToyCore" -# Toy Core Configuration (default) - -balldomain = "balldomains/default.toml" -memdomain = "memdomains/default.toml" -frontend = "frontends/default.toml" -rvv = "rvv/default.toml" -cpu = "cpu/default.toml" diff --git a/examples/cores/toy/configs/demo.toml b/examples/cores/toy/configs/demo.toml deleted file mode 100644 index 49a49854a..000000000 --- a/examples/cores/toy/configs/demo.toml +++ /dev/null @@ -1,8 +0,0 @@ -factory = "examples.cores.toy.ToyCore" -# Toy Core Configuration (demo frontend) - -balldomain = "balldomains/default.toml" -memdomain = "memdomains/default.toml" -frontend = "frontends/demo.toml" -rvv = "rvv/default.toml" -cpu = "cpu/default.toml" diff --git a/examples/cores/toy/configs/frontends/default.toml b/examples/cores/toy/configs/frontends/default.toml deleted file mode 100644 index b17bc5c5d..000000000 --- a/examples/cores/toy/configs/frontends/default.toml +++ /dev/null @@ -1,8 +0,0 @@ -robEntries = 16 -rsOutOfOrderResponse = true -bankIdLen = 10 -vbankIdUpperBound = 15 -sharedBankIdBase = 32 -iterLen = 34 -subRobEnable = true -subRobDepth = 64 diff --git a/examples/cores/toy/configs/frontends/demo.toml b/examples/cores/toy/configs/frontends/demo.toml deleted file mode 100644 index 41cd684cc..000000000 --- a/examples/cores/toy/configs/frontends/demo.toml +++ /dev/null @@ -1,8 +0,0 @@ -robEntries = 16 -rsOutOfOrderResponse = true -bankIdLen = 10 -vbankIdUpperBound = 15 -sharedBankIdBase = 32 -iterLen = 34 -subRobEnable = false -subRobDepth = 64 diff --git a/examples/cores/toy/configs/memdomains/default.toml b/examples/cores/toy/configs/memdomains/default.toml deleted file mode 100644 index 5ee4d654e..000000000 --- a/examples/cores/toy/configs/memdomains/default.toml +++ /dev/null @@ -1,32 +0,0 @@ -# Toy Core Memory Domain Configuration (default) -# Private (per-core) memory parameters. Shared memory lives in the tile TOML. - -[bank] -num = 16 -width = 128 -entries = 1024 -maskLen = 16 -channel = 7 - -[dma] -nXacts = 8 -burstMaxBytes = 1024 -busWidth = 128 -maxInFlightMemReqs = 16 - -[tlb] -size = 4 - -[tma] -readChannel = 4 -writeChannel = 4 - -[mmio] -enable = true -bankNum = 16 -bankEntries = 64 -bankWidth = 8 -readWidth = 8 - -[mem] -addrLen = 39 diff --git a/examples/cores/toy/workloads/ctests/CMakeLists.txt b/examples/cores/toy/workloads/ctests/CMakeLists.txt deleted file mode 100644 index 931fea644..000000000 --- a/examples/cores/toy/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -add_subdirectory(gemmini_relu_trace_pipeline) -set(BUCKYBALL_CTEST_TARGET ${BUCKYBALL_CORE_WORKLOAD_TARGET}) -file(RELATIVE_PATH RVV_KERNEL_SOURCE ${CMAKE_CURRENT_SOURCE_DIR} - ${WORKLOAD_SRC_DIR}/CTest/rvv/rvv_kernel.c) -add_buckyball_ctest(${RVV_KERNEL_SOURCE}) -buckyball_ctest_name(RVV_KERNEL_TEST ${BUCKYBALL_CTEST_TARGET} rvv_kernel) -add_dependencies(core-ctests-build ${RVV_KERNEL_TEST}-ctest-build) - -add_buckyball_ctest(rvv_ball.c) -buckyball_ctest_name(RVV_BALL_TEST ${BUCKYBALL_CTEST_TARGET} rvv_ball) -add_dependencies(core-ctests-build ${RVV_BALL_TEST}-ctest-build) diff --git a/examples/cores/toy/workloads/ctests/gemmini_relu_trace_pipeline/CMakeLists.txt b/examples/cores/toy/workloads/ctests/gemmini_relu_trace_pipeline/CMakeLists.txt deleted file mode 100644 index 2da7121a8..000000000 --- a/examples/cores/toy/workloads/ctests/gemmini_relu_trace_pipeline/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -add_buckyball_core_workload(gemmini_relu_trace_pipeline SOURCES - gemmini_relu_trace_pipeline-main.cpp - toy.c) diff --git a/examples/cores/toy/workloads/ctests/gemmini_relu_trace_pipeline/gemmini_relu_trace_pipeline-main.cpp b/examples/cores/toy/workloads/ctests/gemmini_relu_trace_pipeline/gemmini_relu_trace_pipeline-main.cpp deleted file mode 100644 index 217817cb9..000000000 --- a/examples/cores/toy/workloads/ctests/gemmini_relu_trace_pipeline/gemmini_relu_trace_pipeline-main.cpp +++ /dev/null @@ -1,23 +0,0 @@ -#include - -extern "C" int toy_core(core_id_t); -static volatile uint32_t completed; -static volatile uint32_t failed; - -int main() { - core_id_t id = bb_get_core_id(); - if (id.tile == BB_WORK_TILE && - bb_topology_core_profile(id) == BB_CORE_WORKLOAD_PROFILE) { - __atomic_fetch_or(&failed, toy_core(id) != 0, __ATOMIC_RELEASE); - __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); - } - if (id.tile == BB_WORK_TILE && id.core == 0) { - uint32_t expected = - bb_topology_profile_cores_per_tile(BB_CORE_WORKLOAD_PROFILE); - while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) - asm volatile("nop"); - return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; - } - for (;;) - asm volatile("wfi"); -} diff --git a/examples/cores/toy/workloads/ctests/rvv_ball.c b/examples/cores/toy/workloads/ctests/rvv_ball.c deleted file mode 100644 index a340ed361..000000000 --- a/examples/cores/toy/workloads/ctests/rvv_ball.c +++ /dev/null @@ -1,45 +0,0 @@ -#include -#include -#include -#include - -static const uint32_t image[] __attribute__((aligned(16))) = { - 0x31564b52, 12, 0, 0x40000000, 0, 0, 0x64b5307b, 0x64d6307b, 0x00008067}; -static int32_t input[128] __attribute__((aligned(16))); -static int32_t output[128] __attribute__((aligned(16))); - -int main(void) { - for (unsigned i = 0; i < 128; ++i) - input[i] = ((int)i - 64) * 7; - bb_mem_alloc(0, 1, 1); - bb_mem_alloc(2, 1, 2); - struct kernel_launch call - __attribute__((aligned(16))) = {0, - 12, - 0x40002000, - {BB_BANK0(2) | BB_ITER(16), BANK_LINES, - BB_BANK0(3) | BB_ITER(16), BANK_LINES}, - 0}; - bb_mvin((uintptr_t)input, 2, 16, 1); - bb_mvin((uintptr_t)&call, 0, sizeof(call) / 16, 1); - bb_mem_alloc(4, 1, 1); - bb_mem_transfer(0, 3); - bb_mem_transfer(2, 3); - mvin_kernel(image, sizeof(image), VIRTUAL_BANK_NUM); - run_kernel(4, VIRTUAL_BANK_NUM, 3, 0); - int32_t groups[2][64] __attribute__((aligned(16))); - bb_mvout_group((uintptr_t)groups[0], 3, 1, 16, 1); - bb_mvout_group((uintptr_t)groups[1], 3, 2, 16, 1); - release_kernel(VIRTUAL_BANK_NUM); - for (unsigned row = 0; row < 16; ++row) - for (unsigned group = 0; group < 2; ++group) - for (unsigned lane = 0; lane < 4; ++lane) - output[row * 8 + group * 4 + lane] = groups[group][row * 4 + lane]; - for (unsigned i = 0; i < 128; ++i) - if (output[i] != (input[i] > 0 ? input[i] : 0)) - return 1; - bb_mem_release(3); - bb_mem_release(4); - puts("RVV BALL PASSED"); - return 0; -} diff --git a/examples/designs/goban/arch/build.sc b/examples/designs/goban/arch/build.sc new file mode 100644 index 000000000..170086f40 --- /dev/null +++ b/examples/designs/goban/arch/build.sc @@ -0,0 +1,17 @@ +import mill._ +import mill.scalalib._ +import $file.^.^.^.^.arch.src.main.scala.framework.device.cbus.{build => cbus_build} +import $file.^.^.^.{build => examples} +import $file.^.^.^.prefab.cores.rocket.arch.{build => rocket} + +val designRoot: os.Path = millSourcePath / os.up + +object design extends SbtModule { + override def generatedSources = T(cbus_build.registry(sources() ++ rocket.library.sources(), T.dest)) + override def scalaVersion = "2.13.16" + override def moduleDeps = Seq(examples.library, rocket.library) + override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") + override def sources = T.sources(designRoot / "arch", designRoot / "configs") +} diff --git a/examples/designs/goban/arch/device/DeviceTop.scala b/examples/designs/goban/arch/device/DeviceTop.scala new file mode 100644 index 000000000..07854da55 --- /dev/null +++ b/examples/designs/goban/arch/device/DeviceTop.scala @@ -0,0 +1,77 @@ +package examples.designs.goban + +import framework.lib.arbiter.{RRArbiter} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, Instance, Instantiate} +import framework.device.configs.DeviceParam +import framework.device.cbus.CBus +import framework.device.cbus.generated.CoreComponents +import buckyball.config.Design +import _root_.configs.GlobalConfig +import framework.core.clink.CLinkIO +import framework.core.memory.home.Home +import framework.device.ddr.Bridge +import framework.device.peripherals.Controller +import framework.device.peripherals.bootrom.BootRomLines +import framework.device.dlink.BaseDevice +import framework.ip.bus.axi4 +import framework.ip.bus.chi.snf.LineResponse +import framework.lib.queue.{Queue => ResponseQueue} + +/** One Rocket hart schedules local MatMul threads through its core. */ +@instantiable +class DeviceTop(b: GlobalConfig, design: Design) extends BaseDevice(b.device.ddr.axi) { + val device = b.device + + val Seq(corep) = device.cores + val sharedHome = corep.memory.home + require(sharedHome.memory.agents == 3) + val bus = Instantiate(new CBus(b.device, CBus.generators(design, b.device, b.sim, CoreComponents.factories))) + + val home: Instance[Home] = Instantiate(new Home(sharedHome)) + home.io.active := true.B + home.io.blockRequesterRsp := false.B + home.io.requesters <> bus.coherent + + val peripherals = Instantiate(new Controller(device, device.ddr.axi, corep.memory.memoryAccess.memory)) + + val peripheralBus = Instantiate(new axi4.Decoder( + Seq( + (device.clint.base, device.clint.bytes), + (device.plic.base, device.plic.bytes), + (device.scu.baseAddress, device.scu.totalSizeBytes) + ), + device.ddr.axi + )) + + peripheralBus.io.in <> bus.uncached + peripherals.io.axi <> peripheralBus.io.matched + bus.external <> peripheralBus.io.other + peripherals.io.hardwareInterrupt.sources := 0.U + bus.interrupts := peripherals.io.hardwareInterrupt.harts + + val rom = Instantiate(new BootRomLines(device.bootrom, sharedHome.memory.chi)) + val bridge = Instantiate(new Bridge(device.ddr)) + require(device.ddr.clients == 1) + val request = home.io.memoryReq + val inRom = request.bits.addr >= device.bootrom.base.U && + request.bits.addr < (device.bootrom.base + device.bootrom.bytes).U + rom.io.request.valid := request.valid && inRom + rom.io.request.bits := request.bits + bridge.io.request(0).valid := request.valid && !inRom + bridge.io.request(0).bits := request.bits + request.ready := Mux(inRom, rom.io.request.ready, bridge.io.request(0).ready) + val responses = Instantiate(new RRArbiter(new LineResponse(sharedHome.memory.chi), 2)) + responses.io.in(0) <> rom.io.response + responses.io.in(1) <> bridge.io.response(0) + val response = Instantiate(new ResponseQueue(new LineResponse(sharedHome.memory.chi), 1, false, false)) + response.io.enq <> responses.io.out + home.io.memoryResp <> response.io.deq + + val memoryBus = Instantiate(new axi4.Interconnect(axi4.InterconnectParam(device.ddr.axi, 2, None))) + memoryBus.io.in(0) <> bus.dlink.axi + memoryBus.io.in(1) <> bridge.io.axi + dlink.axi <> memoryBus.io.out +} diff --git a/examples/designs/goban/arch/device/core/GobanCore.scala b/examples/designs/goban/arch/device/core/GobanCore.scala new file mode 100644 index 000000000..a9161b978 --- /dev/null +++ b/examples/designs/goban/arch/device/core/GobanCore.scala @@ -0,0 +1,88 @@ +package examples.designs.goban + +import framework.core.tbus.generated.ThreadComponents +import sims.configs.SimParam +import framework.device.cbus.CoreFactory +import buckyball.config.{CorePlacement, Design} + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import examples.core.rocket.configs.{Params => RocketParam} +import framework.core.configs.CoreParam +import framework.core.controller.Controller +import framework.core.tbus.TBus +import framework.core.tss.Store +import framework.core.shm.SHM +import framework.core.clink.{CLinkIO, HasCLink} +import framework.ip.bus.chi.rnf.AxiCache + +@instantiable +class GobanCore( + corep: CoreParam, + rocketp: RocketParam, + simp: SimParam, + traceIds: Seq[Int]) + extends Module + with HasCLink { + val diffTest = simp.diffTest + require(corep.clink.harts == 1 && !corep.clink.backing && !corep.clink.shared) + require(corep.clink.requesters == 3 && corep.threads.nonEmpty) + + @public + val clink = IO(new CLinkIO(corep.clink)) + + val loom: Instance[Loom] = Instantiate(new Loom(corep, rocketp)) + val controller: Instance[Controller] = Instantiate(new Controller(corep.controller)) + + val tbus: Instance[TBus] = + Instantiate(new TBus(corep, simp, traceIds, TBus.generators(corep, simp, traceIds, ThreadComponents.factories))) + + val shm: Instance[SHM] = Instantiate(new SHM(corep.sharedFrontend, corep.sharedMemory, corep.tbus.get)) + val tss: Instance[Store] = Instantiate(new Store(corep.tss.get)) + val dma: Instance[AxiCache] = Instantiate(new AxiCache(corep.interconnect.bus, corep.dmaCache.get)) + + loom.io.interrupts := clink.interrupts(0) + controller.io.execution <> loom.io.commands + loom.io.drained := controller.io.drained + loom.io.memoryAllowed := controller.io.memoryAllowed + tbus.io.control.get <> controller.io.threads.get + tbus.io.memory.get <> loom.io.memory + tbus.io.move <> controller.io.move + shm.io <> tbus.io.shm.get + tss.io.clients <> tbus.io.tss.get + tss.io.host <> controller.io.tss.get.host + tss.io.inUse := controller.io.tss.get.inUse + tss.io.cancel := controller.io.tss.get.cancel + controller.io.tss.get.busy := tss.io.busy + controller.io.tss.get.hostBusy := tss.io.hostBusy + + clink.memory.uncached <> loom.io.uncached + clink.memory.coherent.take(2).zip(loom.io.coherent).foreach { case (port, cache) => port <> cache } + dma.io.axi <> tbus.io.ddr + dma.io.axi.ar.bits.cache := 15.U + dma.io.axi.aw.bits.cache := 15.U + clink.memory.coherent(2) <> dma.io.chi + clink.memory.ddr.aw.valid := false.B + clink.memory.ddr.aw.bits := 0.U.asTypeOf(clink.memory.ddr.aw.bits) + clink.memory.ddr.w.valid := false.B + clink.memory.ddr.w.bits := 0.U.asTypeOf(clink.memory.ddr.w.bits) + clink.memory.ddr.ar.valid := false.B + clink.memory.ddr.ar.bits := 0.U.asTypeOf(clink.memory.ddr.ar.bits) + clink.memory.ddr.b.ready := false.B + clink.memory.ddr.r.ready := false.B +} + +object Goban extends CoreFactory { + val component = "goban" + def parameters(design: Design, source: CorePlacement): CoreParam = + CoreParam.fromProto(design, source, externalHome = !RocketParam.fromProto(source).clink.backing) + + def instantiate( + source: CorePlacement, + corep: CoreParam, + simp: SimParam, + traceIds: Seq[Int] + ): CLinkIO = + Instantiate(new GobanCore(corep, RocketParam.fromProto(source), simp, traceIds)).clink + +} diff --git a/examples/designs/goban/arch/device/core/loom.scala b/examples/designs/goban/arch/device/core/loom.scala new file mode 100644 index 000000000..934ba4939 --- /dev/null +++ b/examples/designs/goban/arch/device/core/loom.scala @@ -0,0 +1,60 @@ +package examples.designs.goban + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import examples.core.rocket.Execution +import examples.core.rocket.configs.{Params => RocketParam} +import framework.core.configs.CoreParam +import framework.core.memory.{Memory, MemoryPort} +import framework.core.controller.admission.Commands +import framework.core.clink.CoreInterrupts +import framework.ip.bus.{axi4, chi} + +/** Outer Loom execution core, implemented by Rocket; runs the engine that schedules task threads. */ +@instantiable +class Loom(corep: CoreParam, rocketp: RocketParam) extends Module { + require(!corep.clink.backing) + + @public + val io = IO(new Bundle { + val interrupts = Input(new CoreInterrupts) + val commands = new Commands(corep.memory.dcache.tracking, corep.integer.xLen, corep.privileged.permission) + val drained = Input(Bool()) + val memoryAllowed = Input(Bool()) + val memory = + Vec(corep.threads.size, Flipped(new MemoryPort(corep.memory.dcache.tracking, corep.memory.memoryAccess.walker))) + val coherent = Vec(2, new chi.RequesterPort(corep.memory.home.memory.chi)) + val uncached = new axi4.Port(corep.memory.uncached.axi) + }) + + val execution: Instance[Execution] = Instantiate(new Execution(rocketp, commandsEnabled = true)) + val memory: Instance[Memory] = Instantiate(new Memory(corep.memory, externalHome = true)) + execution.io.interrupts := io.interrupts + memory.io.fetch <> execution.io.fetch + memory.io.data <> execution.io.data + memory.io.timeRequest <> execution.io.timeRequest + execution.io.timeResponse <> memory.io.timeResponse + memory.io.commandAllowed := io.memoryAllowed + memory.io.threads.get <> io.memory + io.coherent <> memory.io.coherent + io.uncached <> memory.io.uncached + + val rocc = execution.io.rocc + val command = io.commands.command + val pending = RegInit(false.B) + command.valid := rocc.cmd.valid && !pending + rocc.cmd.ready := command.ready && !pending + command.bits.tag := 0.U + command.bits.instruction := rocc.cmd.bits.instruction + command.bits.satp := rocc.cmd.bits.satp + command.bits.effectivePrivilege := rocc.cmd.bits.effectivePrivilege + command.bits.sum := rocc.cmd.bits.sum + command.bits.mxr := rocc.cmd.bits.mxr + command.bits.pmp := rocc.cmd.bits.pmp + when(command.fire)(pending := true.B) + io.commands.complete.ready := true.B + when(io.commands.complete.fire)(pending := false.B) + rocc.resp <> io.commands.response + rocc.busy := pending || !io.drained + rocc.interrupt := false.B +} diff --git a/examples/designs/goban/configs/design.toml b/examples/designs/goban/configs/design.toml new file mode 100644 index 000000000..a55a275cc --- /dev/null +++ b/examples/designs/goban/configs/design.toml @@ -0,0 +1,4 @@ +[device] +include = "device/default.toml" +[sims] +include = "sims/default.toml" diff --git a/examples/designs/goban/configs/device/clint.toml b/examples/designs/goban/configs/device/clint.toml new file mode 100644 index 000000000..417a2788c --- /dev/null +++ b/examples/designs/goban/configs/device/clint.toml @@ -0,0 +1,4 @@ +base = 0x2000000 +bytes = 0x10000 +tickCycles = 1000 +hartIds = [0] diff --git a/examples/designs/goban/configs/device/core/c2c/goban.toml b/examples/designs/goban/configs/device/core/c2c/goban.toml new file mode 100644 index 000000000..1284ec5d8 --- /dev/null +++ b/examples/designs/goban/configs/device/core/c2c/goban.toml @@ -0,0 +1,3 @@ +enabled = false +peerCores = 1 +axi = "default" diff --git a/examples/designs/goban/configs/device/core/controller/goban.toml b/examples/designs/goban/configs/device/core/controller/goban.toml new file mode 100644 index 000000000..87f38f33c --- /dev/null +++ b/examples/designs/goban/configs/device/core/controller/goban.toml @@ -0,0 +1 @@ +moves = true diff --git a/examples/designs/goban/configs/device/core/default.toml b/examples/designs/goban/configs/device/core/default.toml new file mode 100644 index 000000000..3494faaff --- /dev/null +++ b/examples/designs/goban/configs/device/core/default.toml @@ -0,0 +1,60 @@ +component = "goban" +implementation = "rocket.toml" +resetVector = 0x10000 +mtimeAddress = 0x200bff8 +memory = "memory/goban.toml" +dmaCache = "memory/dma-cache/shared.toml" +tbus = "tbus/default.toml" +fetch = "default" +fpu = "default" +interconnect = "interconnect/goban.toml" +c2c = "c2c/goban.toml" +controller = "controller/goban.toml" + +[param] +threadDataBytes = 64 +xLen = 64 +vaddrBits = 39 +paddrBits = 36 +pgIdxBits = 12 +pgLevels = 3 +nPMPs = 16 + +[sharedMem] +enable = true +entries = 16384 +inputChannels = 52 +defaultGroupCount = 1 +virtualBankCount = 64 +latency = 10 +bankEntries = 1024 +bankWidth = 128 + +[tss] +base = 0x20000 +bytes = 0x4000 +dataBits = 128 + +[[threads]] +id = 0 +name = "matmul0" +component = "matmul" +include = "../../../../../prefab/threads/matmul/configs/default.toml" + +[[threads]] +id = 1 +name = "matmul1" +component = "matmul" +include = "../../../../../prefab/threads/matmul/configs/default.toml" + +[[threads]] +id = 2 +name = "matmul2" +component = "matmul" +include = "../../../../../prefab/threads/matmul/configs/default.toml" + +[[threads]] +id = 3 +name = "matmul3" +component = "matmul" +include = "../../../../../prefab/threads/matmul/configs/default.toml" diff --git a/examples/designs/goban/configs/device/core/interconnect/goban.toml b/examples/designs/goban/configs/device/core/interconnect/goban.toml new file mode 100644 index 000000000..13d04a5a6 --- /dev/null +++ b/examples/designs/goban/configs/device/core/interconnect/goban.toml @@ -0,0 +1,2 @@ +axi = "default" +slots = 16 diff --git a/examples/designs/goban/configs/device/core/memory/access/goban.toml b/examples/designs/goban/configs/device/core/memory/access/goban.toml new file mode 100644 index 000000000..2e90444bc --- /dev/null +++ b/examples/designs/goban/configs/device/core/memory/access/goban.toml @@ -0,0 +1,52 @@ +translationEntries = 16 +tagBits = 6 +resultLineBits = 0 +[[regions]] +base = 0x80000000 +bytes = 0x20000000 +cacheable = true +executable = true +readable = true +writable = true +atomic = true +normal = true + +[[regions]] +base = 0x10000 +bytes = 0x10000 +cacheable = true +executable = true +readable = true +writable = false +atomic = false +normal = true + +[[regions]] +base = 0x2000000 +bytes = 0x10000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0xc000000 +bytes = 0x4000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0x60000000 +bytes = 0x10000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false diff --git a/examples/designs/goban/configs/device/core/memory/dcache/goban.toml b/examples/designs/goban/configs/device/core/memory/dcache/goban.toml new file mode 100644 index 000000000..386bc4427 --- /dev/null +++ b/examples/designs/goban/configs/device/core/memory/dcache/goban.toml @@ -0,0 +1,7 @@ +cache = "shared.toml" +[tracking] +entries = 8 +addressBits = 44 +idBits = 8 +lineBytes = 64 +maxRanges = 8 diff --git a/examples/designs/goban/configs/device/core/memory/dcache/shared.toml b/examples/designs/goban/configs/device/core/memory/dcache/shared.toml new file mode 100644 index 000000000..ba8310878 --- /dev/null +++ b/examples/designs/goban/configs/device/core/memory/dcache/shared.toml @@ -0,0 +1,7 @@ +nodeId = 2 +lines = 64 +homeId = 32 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/goban/configs/device/core/memory/dma-cache/shared.toml b/examples/designs/goban/configs/device/core/memory/dma-cache/shared.toml new file mode 100644 index 000000000..9a9a702c6 --- /dev/null +++ b/examples/designs/goban/configs/device/core/memory/dma-cache/shared.toml @@ -0,0 +1,7 @@ +nodeId = 3 +lines = 64 +homeId = 32 +homeCount = 1 +banks = 2 +lineResult = true +probe = false diff --git a/examples/designs/goban/configs/device/core/memory/goban.toml b/examples/designs/goban/configs/device/core/memory/goban.toml new file mode 100644 index 000000000..844bd730f --- /dev/null +++ b/examples/designs/goban/configs/device/core/memory/goban.toml @@ -0,0 +1,5 @@ +uncached = "default" +icache = "icache/shared.toml" +access = "access/goban.toml" +dcache = "dcache/goban.toml" +home = "home/goban.toml" diff --git a/examples/designs/goban/configs/device/core/memory/home/goban.toml b/examples/designs/goban/configs/device/core/memory/home/goban.toml new file mode 100644 index 000000000..329d50021 --- /dev/null +++ b/examples/designs/goban/configs/device/core/memory/home/goban.toml @@ -0,0 +1,13 @@ +agents = 3 +mshrEntries = 8 +nodeId = 32 +requesters = "default" + +[cache] +addressBits = 44 +lineBytes = 64 +sets = 64 +ways = 4 +idBits = 8 +metadataBits = 4 +responseDepth = 2 diff --git a/examples/designs/goban/configs/device/core/memory/icache/shared.toml b/examples/designs/goban/configs/device/core/memory/icache/shared.toml new file mode 100644 index 000000000..0796abb45 --- /dev/null +++ b/examples/designs/goban/configs/device/core/memory/icache/shared.toml @@ -0,0 +1,7 @@ +nodeId = 1 +lines = 64 +homeId = 32 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/goban/configs/device/core/rocket.toml b/examples/designs/goban/configs/device/core/rocket.toml new file mode 100644 index 000000000..4d6fb0712 --- /dev/null +++ b/examples/designs/goban/configs/device/core/rocket.toml @@ -0,0 +1,12 @@ +component = "rocket" +physical_address_bits = 36 +hart_id_bits = 3 +page_levels = 3 +pmp_entries = 16 +virtual_memory = true +supervisor = true +user = true +floating_point = true +atomics = true +multiply_divide = true +external_home = true diff --git a/examples/designs/goban/configs/device/core/tbus/default.toml b/examples/designs/goban/configs/device/core/tbus/default.toml new file mode 100644 index 000000000..9665aee43 --- /dev/null +++ b/examples/designs/goban/configs/device/core/tbus/default.toml @@ -0,0 +1 @@ +bankNodes = [0, 1, 2, 3] diff --git a/examples/designs/goban/configs/device/ddr.toml b/examples/designs/goban/configs/device/ddr.toml new file mode 100644 index 000000000..31815e9e1 --- /dev/null +++ b/examples/designs/goban/configs/device/ddr.toml @@ -0,0 +1,13 @@ +clients = 1 +slotsPerClient = 16 +dataBits = 128 +idBits = 4 +idBase = 0 +base = 0x80000000 +bytes = 0x20000000 +model = 0 + +[line] +nodeIdBits = 7 +addressBits = 44 +dataBits = 256 diff --git a/examples/designs/goban/configs/device/default.toml b/examples/designs/goban/configs/device/default.toml new file mode 100644 index 000000000..402388b16 --- /dev/null +++ b/examples/designs/goban/configs/device/default.toml @@ -0,0 +1,9 @@ +ddr = "ddr.toml" +bootrom = "default" +clint = "clint.toml" +plic = "plic.toml" +scu = "default" + +[[cores]] +include = "core/default.toml" +hartId = 0 diff --git a/examples/designs/goban/configs/device/plic.toml b/examples/designs/goban/configs/device/plic.toml new file mode 100644 index 000000000..dea72f94e --- /dev/null +++ b/examples/designs/goban/configs/device/plic.toml @@ -0,0 +1,5 @@ +base = 0xc000000 +bytes = 0x4000000 +sources = 1 +priorityBits = 3 +hartIds = [0] diff --git a/examples/designs/goban/configs/sims/default.toml b/examples/designs/goban/configs/sims/default.toml new file mode 100644 index 000000000..38d332fa4 --- /dev/null +++ b/examples/designs/goban/configs/sims/default.toml @@ -0,0 +1,17 @@ +diffTest = false + +[memory] +lineBytes = 64 +clockHz = 1000000000 + +[targets.device] +config = "examples.designs.goban.sims.device.DeviceConfig" +stack_name = "goban" +regression = "sims/device/regression" +kernel = "sims/device/workloads/kernel" + +[targets.device.configs] +GobanVerilatorConfig = "verilator.toml" + +[verification] +registry = "../../sims/ip/uvm/registry.toml" diff --git a/examples/designs/goban/configs/sims/verilator.toml b/examples/designs/goban/configs/sims/verilator.toml new file mode 100644 index 000000000..0a960c260 --- /dev/null +++ b/examples/designs/goban/configs/sims/verilator.toml @@ -0,0 +1 @@ +# GobanVerilatorConfig: the goban design as configured, with no overrides. diff --git a/examples/designs/goban/emu/Cargo.lock b/examples/designs/goban/emu/Cargo.lock new file mode 100644 index 000000000..adfacbb8c --- /dev/null +++ b/examples/designs/goban/emu/Cargo.lock @@ -0,0 +1,821 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "bebop-bank-hash" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "snafu", +] + +[[package]] +name = "bebop-bemu" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", + "bebop-bemu-profile", + "bebop-dasm", + "bebop-dtb", + "bebop-elf", + "bebop-keyboard", + "bebop-memory", + "bebop-microphone", + "bebop-rtc", + "bebop-speaker", + "bebop-syscall", + "bebop-uart", + "bebop-vga", + "bebop-vm", + "clap", + "linkme", + "prost", + "prost-build", + "prost-types", + "rustc_apfloat", + "rvsim", + "snafu", + "toml", +] + +[[package]] +name = "bebop-bemu-profile" +version = "0.1.0" + +[[package]] +name = "bebop-dasm" +version = "0.1.0" + +[[package]] +name = "bebop-design-goban" +version = "0.1.0" +dependencies = [ + "bebop-bemu", +] + +[[package]] +name = "bebop-dtb" +version = "0.1.0" + +[[package]] +name = "bebop-elf" +version = "0.1.0" +dependencies = [ + "bebop-memory", +] + +[[package]] +name = "bebop-keyboard" +version = "0.1.0" + +[[package]] +name = "bebop-memory" +version = "0.1.0" +dependencies = [ + "log", + "serde", + "serde_json", + "sha2", +] + +[[package]] +name = "bebop-microphone" +version = "0.1.0" + +[[package]] +name = "bebop-rtc" +version = "0.1.0" + +[[package]] +name = "bebop-speaker" +version = "0.1.0" + +[[package]] +name = "bebop-syscall" +version = "0.1.0" +dependencies = [ + "bebop-memory", + "once_cell", +] + +[[package]] +name = "bebop-uart" +version = "0.1.0" + +[[package]] +name = "bebop-vga" +version = "0.1.0" + +[[package]] +name = "bebop-vm" +version = "0.1.0" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck", + "itertools", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.119", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustc_apfloat" +version = "0.2.3+llvm-462a31f5a5ab" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "486c2179b4796f65bfe2ee33679acf0927ac83ecf583ad6c91c3b4570911b9ad" +dependencies = [ + "bitflags", + "smallvec", +] + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rvsim" +version = "0.1.0" +dependencies = [ + "rustc_apfloat", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "smallvec" +version = "1.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" + +[[package]] +name = "snafu" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/examples/designs/goban/emu/Cargo.toml b/examples/designs/goban/emu/Cargo.toml new file mode 100644 index 000000000..dbd933223 --- /dev/null +++ b/examples/designs/goban/emu/Cargo.toml @@ -0,0 +1,9 @@ +[workspace] + +[package] +name = "bebop-design-goban" +version = "0.1.0" +edition = "2021" + +[dependencies] +bebop-bemu = { path = "../../../../bebop/src/nodes/bemu" } diff --git a/examples/designs/goban/emu/src/device/core/mod.rs b/examples/designs/goban/emu/src/device/core/mod.rs new file mode 100644 index 000000000..004047758 --- /dev/null +++ b/examples/designs/goban/emu/src/device/core/mod.rs @@ -0,0 +1,37 @@ +use bebop_bemu::{ + ball::linkme, CoreFactory, CoreModel, CoreParams, Cpu, DeviceModel, FromString, ModelOptions, NoRocc, ThreadGroup, + Whatever, CORES, +}; + +/// One RV64GC hart driving its matmul hardware threads (arch: GobanCore, behind TBus). +pub struct GobanCore { + cpu: Cpu, + threads: ThreadGroup, +} + +impl CoreModel for GobanCore { + fn cpu(&self) -> &Cpu { + &self.cpu + } + fn cpu_mut(&mut self) -> &mut Cpu { + &mut self.cpu + } + fn step(&mut self, count: u64) -> Result<(), Whatever> { + self.threads.check().map_err(Whatever::without_source)?; + self.cpu.step(count, &mut NoRocc)?; + if self.cpu.finished() { + self.threads.shutdown().map_err(Whatever::without_source)?; + } + Ok(()) + } +} + +fn instantiate(device: &DeviceModel, params: &CoreParams, options: &ModelOptions) -> Result, String> { + let cpu = Cpu::new(device, params, options).map_err(|error| error.to_string())?; + let threads = ThreadGroup::new(cpu.state(), options)?; + Ok(Box::new(GobanCore { cpu, threads })) +} + +#[linkme::distributed_slice(CORES)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static GOBAN: CoreFactory = CoreFactory { component: "goban", instantiate }; diff --git a/examples/designs/goban/emu/src/device/mod.rs b/examples/designs/goban/emu/src/device/mod.rs new file mode 100644 index 000000000..db0b63ed2 --- /dev/null +++ b/examples/designs/goban/emu/src/device/mod.rs @@ -0,0 +1,2 @@ +//! The goban device: its own Core model; the cores are instantiated by component on the CBus. +pub mod core; diff --git a/examples/designs/goban/emu/src/lib.rs b/examples/designs/goban/emu/src/lib.rs new file mode 100644 index 000000000..545892414 --- /dev/null +++ b/examples/designs/goban/emu/src/lib.rs @@ -0,0 +1 @@ +pub mod device; diff --git a/examples/designs/goban/sims/device/arch/DeviceConfig.scala b/examples/designs/goban/sims/device/arch/DeviceConfig.scala new file mode 100644 index 000000000..25ae9a744 --- /dev/null +++ b/examples/designs/goban/sims/device/arch/DeviceConfig.scala @@ -0,0 +1,18 @@ +package examples.designs.goban.sims.device + +import buckyball.config.Design +import chisel3.experimental.hierarchy.{Instance, Instantiate} +import examples.designs.goban.DeviceTop +import framework.device.cbus.CBus +import framework.device.cbus.generated.CoreComponents +import framework.device.dlink.BaseDevice +import _root_.configs.GlobalConfig +import _root_.sims.soc.SystemTarget +import java.nio.file.{Files, Paths} + +class DeviceConfig(design: Design) + extends SystemTarget( + GlobalConfig.fromProto(design, CBus.parameters(design, CoreComponents.factories)) + ) { + def instantiate(b: GlobalConfig): Instance[BaseDevice] = Instantiate(new DeviceTop(b, design)) +} diff --git a/examples/designs/goban/sims/device/arch/build.sc b/examples/designs/goban/sims/device/arch/build.sc new file mode 100644 index 000000000..5b94111b8 --- /dev/null +++ b/examples/designs/goban/sims/device/arch/build.sc @@ -0,0 +1,17 @@ +import mill._ +import mill.scalalib._ +import $file.^.^.^.arch.{build => design_build} +import $file.^.^.^.^.^.^.arch.src.main.scala.sims.{build => sims_build} + +val entryRoot: os.Path = millSourcePath + +object simulation extends SbtModule { + override def millSourcePath = entryRoot + override def scalaVersion = "2.13.16" + override def moduleDeps = Seq(design_build.design, sims_build.library) + override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") + override def sources = T.sources(entryRoot / "DeviceConfig.scala") + override def mainClass = Some("sims.verilator.Elaborate") +} diff --git a/examples/designs/goban/sims/device/emu/Cargo.lock b/examples/designs/goban/sims/device/emu/Cargo.lock new file mode 100644 index 000000000..1cb8200cc --- /dev/null +++ b/examples/designs/goban/sims/device/emu/Cargo.lock @@ -0,0 +1,2008 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "assert_cmd" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aa3a22042e45de04255c7bf3626e239f450200fd0493c1e382263544b20aea6" +dependencies = [ + "anstyle", + "bstr", + "libc", + "predicates", + "predicates-core", + "predicates-tree", + "wait-timeout", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bebop" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", + "bebop-bemu", + "bebop-bemu-profile", + "bebop-dasm", + "bebop-fd-redirect", + "bebop-p2e", + "bebop-report", + "bebop-rtl-trace", + "bebop-terminal", + "bebop-trace-perfetto", + "bebop-uart", + "bebop-vcs", + "bebop-verilator", + "bebop-vm", + "camino", + "clap", + "duct", + "env_logger", + "libc", + "log", + "nix", + "serde", + "serde_json", + "snafu", + "toml", +] + +[[package]] +name = "bebop-bank-hash" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "snafu", +] + +[[package]] +name = "bebop-bemu" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", + "bebop-bemu-profile", + "bebop-dasm", + "bebop-dtb", + "bebop-elf", + "bebop-keyboard", + "bebop-memory", + "bebop-microphone", + "bebop-rtc", + "bebop-speaker", + "bebop-syscall", + "bebop-uart", + "bebop-vga", + "clap", + "linkme", + "prost", + "prost-build", + "prost-types", + "rustc_apfloat", + "snafu", + "toml", +] + +[[package]] +name = "bebop-bemu-profile" +version = "0.1.0" + +[[package]] +name = "bebop-dasm" +version = "0.1.0" + +[[package]] +name = "bebop-design-goban" +version = "0.1.0" +dependencies = [ + "bebop-bemu", +] + +[[package]] +name = "bebop-dtb" +version = "0.1.0" + +[[package]] +name = "bebop-elf" +version = "0.1.0" +dependencies = [ + "bebop-memory", +] + +[[package]] +name = "bebop-fd-redirect" +version = "0.1.0" +dependencies = [ + "libc", +] + +[[package]] +name = "bebop-keyboard" +version = "0.1.0" + +[[package]] +name = "bebop-memory" +version = "0.1.0" + +[[package]] +name = "bebop-microphone" +version = "0.1.0" + +[[package]] +name = "bebop-p2e" +version = "0.1.0" +dependencies = [ + "bebop-memory", + "bebop-rtl-trace", + "bebop-uart", + "duct", + "libc", + "log", + "serde", + "serde_json", +] + +[[package]] +name = "bebop-report" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "bebop-rtc" +version = "0.1.0" + +[[package]] +name = "bebop-rtl-trace" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", +] + +[[package]] +name = "bebop-sim-build" +version = "0.1.0" +dependencies = [ + "prost", + "prost-build", + "prost-types", +] + +[[package]] +name = "bebop-sim-goban-device" +version = "0.1.0" +dependencies = [ + "assert_cmd", + "bebop", + "bebop-bemu", + "bebop-design-goban", + "bebop-report", + "bebop-sim-build", + "bebop-verilator", + "chrono", + "clap", + "libtest-mimic", + "predicates", + "serde", + "toml", + "walkdir", +] + +[[package]] +name = "bebop-speaker" +version = "0.1.0" + +[[package]] +name = "bebop-syscall" +version = "0.1.0" +dependencies = [ + "bebop-memory", + "once_cell", +] + +[[package]] +name = "bebop-terminal" +version = "0.1.0" +dependencies = [ + "crossterm", + "ratatui", +] + +[[package]] +name = "bebop-trace-perfetto" +version = "0.1.0" +dependencies = [ + "clap", + "serde_json", +] + +[[package]] +name = "bebop-uart" +version = "0.1.0" + +[[package]] +name = "bebop-vcs" +version = "0.1.0" +dependencies = [ + "snafu", +] + +[[package]] +name = "bebop-verilator" +version = "0.1.0" +dependencies = [ + "bebop-dasm", + "bebop-fd-redirect", + "bebop-rtl-trace", + "bebop-uart", + "libc", + "snafu", +] + +[[package]] +name = "bebop-vga" +version = "0.1.0" + +[[package]] +name = "bebop-vm" +version = "0.1.0" +dependencies = [ + "clap", + "font8x8", + "sdl2", + "vt100", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "bstr" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" +dependencies = [ + "memchr", + "regex-automata", + "serde_core", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "camino" +version = "1.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbad30e4b4c14a39e3cc8aed085a12a327257c316619c93581e017bc52be591" + +[[package]] +name = "cassowary" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" + +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + +[[package]] +name = "cc" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "compact_str" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fd622ebbb56a5b2ccb651b32b911cdeb2a9b4b11776b2473bf26a26a286244e" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "static_assertions", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crossterm" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" +dependencies = [ + "bitflags 2.13.2", + "crossterm_winapi", + "mio", + "parking_lot", + "rustix 0.38.44", + "signal-hook 0.3.18", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + +[[package]] +name = "darling" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 3.0.9", +] + +[[package]] +name = "darling_macro" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" +dependencies = [ + "darling_core", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "defmt" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a0795ac740cb54d7b3775c9c17345f6aaecf884c01733b39d6017738e0e3e1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "840bb056bd99c495c9188198a258ab787c79f306d864db3e6e8451442d6d8366" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "defmt-parser" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71958f69fb2de856c5dd51ac9377e986ca1aa5919f931bf5395a807277f889a0" +dependencies = [ + "thiserror", +] + +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + +[[package]] +name = "duct" +version = "0.13.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4ab5718d1224b63252cd0c6f74f6480f9ffeb117438a2e0f5cf6d9a4798929c" +dependencies = [ + "libc", + "once_cell", + "os_pipe", + "shared_child", +] + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "env_filter" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "escape8259" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f0d310082fd5de539d4f41601182241147498444a7695c520288a0cdbead6" + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "float-cmp" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09cf3155332e944990140d967ff5eceb70df778b34f77d8075db46e4704e6d8" +dependencies = [ + "num-traits", +] + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "font8x8" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875488b8711a968268c7cf5d139578713097ca4635a76044e8fe8eedf831d07e" + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "indoc" +version = "2.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a37b2691796cffeb8a8cd305ac66e65841559f147f4e63231d0eafa4db5384d1" +dependencies = [ + "rustversion", +] + +[[package]] +name = "instability" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3b5acc1e2fd9375041a388da33d1eb8aed5f7a8c0dd3543e3ea2805adfbe20" +dependencies = [ + "darling", + "indoc", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jiff" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b005715dcbeb0089a3c0dab99f2ff1cc3b2525323552703d648585d342a383" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", +] + +[[package]] +name = "jiff-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e52fe76043ccecc9005d2305ebaadf7d7fc0cc89ca6baa10a94d6bc68c7128c" +dependencies = [ + "defmt", + "log", +] + +[[package]] +name = "jiff-static" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cc9817253cf7c7ee4684451bd327e88d6f3658014e54a29198625590650695c" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "js-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "libtest-mimic" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14e6ba06f0ade6e504aff834d7c34298e5155c6baca353cc6a4aaff2f9fd7f33" +dependencies = [ + "anstream", + "anstyle", + "clap", + "escape8259", +] + +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.5", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "os_pipe" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "portable-atomic-util" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "predicates" +version = "3.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ada8f2932f28a27ee7b70dd6c1c39ea0675c55a36879ab92f3a715eaa1e63cfe" +dependencies = [ + "anstyle", + "difflib", + "float-cmp", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cad38746f3166b4031b1a0d39ad9f954dd291e7854fcc0eed52ee41a0b50d144" + +[[package]] +name = "predicates-tree" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0de1b847b39c8131db0467e9df1ff60e6d0562ab8e9a16e568ad0fdb372e2f2" +dependencies = [ + "predicates-core", + "termtree", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck", + "itertools 0.14.0", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.119", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "ratatui" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabd94c2f37801c20583fc49dd5cd6b0ba68c716787c2dd6ed18571e1e63117b" +dependencies = [ + "bitflags 2.13.2", + "cassowary", + "compact_str", + "crossterm", + "indoc", + "instability", + "itertools 0.13.0", + "lru", + "paste", + "strum", + "unicode-segmentation", + "unicode-truncate", + "unicode-width 0.2.0", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.2", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustc_apfloat" +version = "0.2.3+llvm-462a31f5a5ab" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "486c2179b4796f65bfe2ee33679acf0927ac83ecf583ad6c91c3b4570911b9ad" +dependencies = [ + "bitflags 2.13.2", + "smallvec", +] + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys 0.12.1", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sdl2" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d42407afc6a8ab67e36f92e80b8ba34cbdc55aaeed05249efe9a2e8d0e9feef" +dependencies = [ + "bitflags 1.3.2", + "lazy_static", + "libc", + "sdl2-sys", +] + +[[package]] +name = "sdl2-sys" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ff61407fc75d4b0bbc93dc7e4d6c196439965fbef8e4a4f003a36095823eac0" +dependencies = [ + "cfg-if", + "libc", + "pkg-config", + "version-compare", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "shared_child" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "607549934f6cc26b89cfecfdc46fa90f1e5d1536a68349b0c3a4f9d1c0d37959" +dependencies = [ + "libc", + "sigchld", + "windows-sys 0.61.2", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "sigchld" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24f2b37f04360cd465089b87a9c3869c08220a2f3458463f0adf8badf5e77f2c" +dependencies = [ + "libc", + "os_pipe", + "signal-hook 0.4.5", +] + +[[package]] +name = "signal-hook" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b29ffa27d3d59a049bb328258de100f8012848d1b08d444e08bbc02ac468435e" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-mio" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b75a19a7a740b25bc7944bdee6172368f988763b744e3d4dfe753f6b4ece40cc" +dependencies = [ + "libc", + "mio", + "signal-hook 0.3.18", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8" + +[[package]] +name = "snafu" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.119", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix 1.1.5", + "windows-sys 0.61.2", +] + +[[package]] +name = "termtree" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-truncate" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3644627a5af5fa321c95b9b235a72fd24cd29c648c2c379431e6628655627bf" +dependencies = [ + "itertools 0.13.0", + "unicode-segmentation", + "unicode-width 0.1.14", +] + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "version-compare" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "579a42fc0b8e0c63b76519a339be31bed574929511fa53c1a3acae26eb258f29" + +[[package]] +name = "vt100" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84cd863bf0db7e392ba3bd04994be3473491b31e66340672af5d11943c6274de" +dependencies = [ + "itoa", + "log", + "unicode-width 0.1.14", + "vte", +] + +[[package]] +name = "vte" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5022b5fbf9407086c180e9557be968742d839e68346af7792b8592489732197" +dependencies = [ + "arrayvec", + "utf8parse", + "vte_generate_state_changes", +] + +[[package]] +name = "vte_generate_state_changes" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e369bee1b05d510a7b4ed645f5faa90619e05437111783ea5848f28d97d3c2e" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.9", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/examples/designs/goban/sims/device/emu/Cargo.toml b/examples/designs/goban/sims/device/emu/Cargo.toml new file mode 100644 index 000000000..08691a836 --- /dev/null +++ b/examples/designs/goban/sims/device/emu/Cargo.toml @@ -0,0 +1,83 @@ +[workspace] + +[package] +name = "bebop-sim-goban-device" +version = "0.1.0" +edition = "2021" +autobins = false +autotests = false + +# The model of this simulation entry; the runners below are built from it. +[lib] +name = "simulation" + +[[bin]] +name = "bebop" +path = "../../../../../../bebop/src/main.rs" + +[[bin]] +name = "bebop-bemu" +path = "../../../../../../bebop/src/bin/bebop-bemu.rs" +required-features = ["bemu"] + +[[bin]] +name = "bebop-verilator" +path = "../../../../../../bebop/src/bin/bebop-verilator.rs" +required-features = ["verilator"] + +[[bin]] +name = "bebop-p2e" +path = "../../../../../../bebop/src/bin/bebop-p2e.rs" +required-features = ["p2e"] + +[[test]] +name = "test_bemu" +path = "../../../../../../bebop/tests/test_bemu.rs" +harness = false +required-features = ["bemu"] + +[[test]] +name = "test_verilator" +path = "../../../../../../bebop/tests/test_verilator.rs" +harness = false +required-features = ["verilator"] + +[[test]] +name = "test_p2e" +path = "../../../../../../bebop/tests/test_p2e.rs" +harness = false +required-features = ["p2e"] + +[[bin]] +name = "bebop-vcs" +path = "../../../../../../bebop/src/bin/bebop-vcs.rs" + +[[test]] +name = "test_vcs" +path = "../../../../../../bebop/tests/test_vcs.rs" +harness = false + +[features] +bemu = ["bebop/bemu"] +verilator = ["bebop/verilator", "dep:bebop-verilator"] +p2e = ["bebop/p2e"] + +[dependencies] +bebop = { path = "../../../../../../bebop" } +bebop-verilator = { path = "../../../../../../bebop/src/nodes/verilator", optional = true } +bebop-bemu = { path = "../../../../../../bebop/src/nodes/bemu" } +bebop-design-goban = { path = "../../../emu" } +clap = { version = "4", features = ["derive"] } + +[build-dependencies] +bebop-sim-build = { path = "../../../../../../bebop/src/nodes/lib/sim-build" } + +[dev-dependencies] +bebop-report = { path = "../../../../../../bebop/src/nodes/lib/report" } +libtest-mimic = "0.8" +assert_cmd = "2" +walkdir = "2" +predicates = "3" +serde = { version = "1", features = ["derive"] } +toml = "0.8" +chrono = { version = "0.4", default-features = false, features = ["clock"] } diff --git a/examples/designs/goban/sims/device/emu/build.rs b/examples/designs/goban/sims/device/emu/build.rs new file mode 100644 index 000000000..beb62e98c --- /dev/null +++ b/examples/designs/goban/sims/device/emu/build.rs @@ -0,0 +1,4 @@ +fn main() { + bebop_sim_build::balls(); + bebop_sim_build::runner_links(); +} diff --git a/examples/designs/goban/sims/device/emu/src/lib.rs b/examples/designs/goban/sims/device/emu/src/lib.rs new file mode 100644 index 000000000..a24b7a8c9 --- /dev/null +++ b/examples/designs/goban/sims/device/emu/src/lib.rs @@ -0,0 +1,11 @@ +//! The goban device simulation. +//! Its Core models register in `bebop_bemu::CORES`; the Balls its design.pb +//! names are compiled here and register in `bebop_bemu::BALLS`. +extern crate bebop_design_goban; + +include!(concat!(env!("OUT_DIR"), "/balls.rs")); + +pub const SIMULATION: bebop_bemu::Simulation = bebop_bemu::Simulation { + design: include_bytes!(concat!(env!("OUT_DIR"), "/design.pb")), + instantiate: bebop_bemu::device::cbus::instantiate, +}; diff --git a/examples/designs/goban/sims/device/regression/bemu/workloads-bare.toml b/examples/designs/goban/sims/device/regression/bemu/workloads-bare.toml new file mode 100644 index 000000000..7c481bca7 --- /dev/null +++ b/examples/designs/goban/sims/device/regression/bemu/workloads-bare.toml @@ -0,0 +1,7 @@ +[workloads] +tests = [ + "goban-matmul-ctest-design-matmul_threads-baremetal", + "goban-core0-soctest-clint_timer_interrupt-baremetal", + "goban-core0-soctest-clint_software_interrupt-baremetal", + "goban-core0-soctest-plic_register_access-baremetal", +] diff --git a/examples/designs/goban/sims/device/regression/bemu/workloads-linux.toml b/examples/designs/goban/sims/device/regression/bemu/workloads-linux.toml new file mode 100644 index 000000000..41a9cd13b --- /dev/null +++ b/examples/designs/goban/sims/device/regression/bemu/workloads-linux.toml @@ -0,0 +1,4 @@ +[workloads] +tests = [ + "fw_payload-linux.elf", +] diff --git a/examples/designs/goban/sims/device/regression/p2e/workloads-linux.toml b/examples/designs/goban/sims/device/regression/p2e/workloads-linux.toml new file mode 100644 index 000000000..01747a69b --- /dev/null +++ b/examples/designs/goban/sims/device/regression/p2e/workloads-linux.toml @@ -0,0 +1,4 @@ +[workloads] +tests = [ + "fw_payload-linux.hex", +] diff --git a/examples/designs/goban/sims/device/regression/verilator/workloads-bare.toml b/examples/designs/goban/sims/device/regression/verilator/workloads-bare.toml new file mode 100644 index 000000000..7c481bca7 --- /dev/null +++ b/examples/designs/goban/sims/device/regression/verilator/workloads-bare.toml @@ -0,0 +1,7 @@ +[workloads] +tests = [ + "goban-matmul-ctest-design-matmul_threads-baremetal", + "goban-core0-soctest-clint_timer_interrupt-baremetal", + "goban-core0-soctest-clint_software_interrupt-baremetal", + "goban-core0-soctest-plic_register_access-baremetal", +] diff --git a/examples/designs/goban/sims/device/stack/CMakeLists.txt b/examples/designs/goban/sims/device/stack/CMakeLists.txt new file mode 100644 index 000000000..b91640c91 --- /dev/null +++ b/examples/designs/goban/sims/device/stack/CMakeLists.txt @@ -0,0 +1,48 @@ +# ===----------------------------------------------------------------------=== +# The software stack of the goban device simulation: every level the device +# needs, built bottom up. +# +# bbdev stack --build "--design goban --device " +# +# configures this with DESIGN_CMAKE, generated from design.pb. Each level is a +# CMake build of its own: cores and threads build buddy-opt plugins for the +# host, the device builds RISC-V libraries. +# thread/matmul libmatmul-compiler.so, program.bin, loader.o, libkernels*.a +# core/goban libengine-compiler.so, execute.o +# device runtime, runtime-baremetal, runtime-linux, engine; device.cmake +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(goban-device-stack NONE) +include(ExternalProject) +include("${DESIGN_CMAKE}") + +# The matmul hardware thread. +ExternalProject_Add(thread-matmul + SOURCE_DIR "${DESIGN_THREAD_STACK_matmul}" + BINARY_DIR "${DESIGN_THREAD_BUILD_matmul}" + CMAKE_GENERATOR Ninja + CMAKE_ARGS "-DDESIGN_CMAKE=${DESIGN_CMAKE}" -DTHREAD=matmul + BUILD_ALWAYS ON + INSTALL_COMMAND "") + +# Goban's CPU core, which runs the matmul thread. +ExternalProject_Add(core-goban + SOURCE_DIR "${DESIGN_CORE_STACK_goban}" + BINARY_DIR "${DESIGN_CORE_BUILD_goban}" + CMAKE_GENERATOR Ninja + CMAKE_ARGS "-DDESIGN_CMAKE=${DESIGN_CMAKE}" -DCORE=goban + DEPENDS thread-matmul + BUILD_ALWAYS ON + INSTALL_COMMAND "") + +# The device: the runtime that schedules jobs onto the cores. +ExternalProject_Add(device + SOURCE_DIR "${DESIGN_DEVICE_STACK}" + BINARY_DIR "${DESIGN_DEVICE_BUILD}" + CMAKE_GENERATOR Ninja + CMAKE_ARGS "-DDESIGN_CMAKE=${DESIGN_CMAKE}" + -DCMAKE_C_COMPILER=riscv64-unknown-linux-gnu-gcc + -DCMAKE_CXX_COMPILER=riscv64-unknown-linux-gnu-g++ + DEPENDS thread-matmul core-goban + BUILD_ALWAYS ON + INSTALL_COMMAND "") diff --git a/examples/designs/goban/sims/device/workloads/CMakeLists.txt b/examples/designs/goban/sims/device/workloads/CMakeLists.txt new file mode 100644 index 000000000..1deeb66ff --- /dev/null +++ b/examples/designs/goban/sims/device/workloads/CMakeLists.txt @@ -0,0 +1,75 @@ +include("${DESIGN_DEVICE_BUILD}/device.cmake") + +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/base" + "${CMAKE_BINARY_DIR}/src/CTest/base" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/tlb" + "${CMAKE_BINARY_DIR}/src/CTest/tlb" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/tutorial" + "${CMAKE_BINARY_DIR}/src/CTest/tutorial" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/embench" + "${CMAKE_BINARY_DIR}/src/CTest/embench" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/coremark" + "${CMAKE_BINARY_DIR}/src/CTest/coremark" +) +foreach(BUCKYBALL_COMPONENT_WORKLOAD_TARGET IN LISTS DESIGN_CPU_TARGETS) + set(COMPONENT_WORKLOADS + "${DESIGN_TARGET_WORKLOADS_${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}}" + ) + if( + COMPONENT_WORKLOADS + STREQUAL + "${BUCKYBALL_REPO_ROOT}/examples/designs/${DESIGN_NAME}/workloads" + ) + continue() + endif() + set(BUCKYBALL_CTEST_TARGET "${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}") + set(BUCKYBALL_CTEST_GROUP_TARGET component-ctests-build) + if(IS_DIRECTORY "${COMPONENT_WORKLOADS}/ctests") + add_subdirectory( + "${COMPONENT_WORKLOADS}/ctests" + "${CMAKE_CURRENT_BINARY_DIR}/prefab/${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}/ctests" + ) + endif() +endforeach() +add_dependencies( + ctest-workloads-build + base-ctests-build + tlb-ctests-build + tutorial-ctests-build + component-ctests-build +) +add_dependencies(sync-native-bin embench-all coremark-all) + +add_subdirectory( + "${WORKLOAD_SRC_DIR}/MLIRTest/base" + "${CMAKE_BINARY_DIR}/src/MLIRTest/mlir_tests/base" +) +foreach(BUCKYBALL_TARGET IN LISTS DESIGN_CPU_TARGETS) + set(BUCKYBALL_MLIR_ACTIVE_TARGET "${BUCKYBALL_TARGET}") + foreach(BUCKYBALL_BALL_DIR IN LISTS DESIGN_PROFILES_${BUCKYBALL_TARGET}_THREAD_BALLDOMAIN_BALLS) + set(BALL_MLIR_TESTS + "${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/${BUCKYBALL_BALL_DIR}/workloads/mlir_tests" + ) + if(IS_DIRECTORY "${BALL_MLIR_TESTS}") + add_subdirectory( + "${BALL_MLIR_TESTS}" + "${CMAKE_CURRENT_BINARY_DIR}/balls/${BUCKYBALL_TARGET}/${BUCKYBALL_BALL_DIR}" + ) + endif() + endforeach() +endforeach() +add_dependencies(MLIRTest-all base-mlirtest-build balls-mlir-tests-build) + +add_subdirectory( + "${BUCKYBALL_REPO_ROOT}/examples/designs/goban/workloads" + design +) diff --git a/examples/designs/goban/sims/device/workloads/kernel/CMakeLists.txt b/examples/designs/goban/sims/device/workloads/kernel/CMakeLists.txt new file mode 100644 index 000000000..aa658be56 --- /dev/null +++ b/examples/designs/goban/sims/device/workloads/kernel/CMakeLists.txt @@ -0,0 +1,5 @@ +# Linux programs installed into the goban rootfs, each with the hart it runs on. +# Goban's matmul threads are OS-invisible: their tests are engine tests, not +# Linux programs, so Linux here only boots. +set(BUCKYBALL_KERNEL_PROGRAMS + PARENT_SCOPE) diff --git a/examples/designs/goban/sims/ip/uvm/registry.toml b/examples/designs/goban/sims/ip/uvm/registry.toml new file mode 100644 index 000000000..8e8131449 --- /dev/null +++ b/examples/designs/goban/sims/ip/uvm/registry.toml @@ -0,0 +1 @@ +ips = [] diff --git a/examples/designs/goban/stack/device/CMakeLists.txt b/examples/designs/goban/stack/device/CMakeLists.txt new file mode 100644 index 000000000..cf3a4abb5 --- /dev/null +++ b/examples/designs/goban/stack/device/CMakeLists.txt @@ -0,0 +1,37 @@ +# ===----------------------------------------------------------------------=== +# The goban device: its cores, their hardware threads and the runtime that +# schedules jobs onto the cores. +# +# bbdev stack --build "--design goban --device " +# +# builds every thread and core stack of the device first, then configures this +# with DESIGN_CMAKE, generated from design.pb, and the RISC-V Linux toolchain. +# Products, exported in device.cmake for the programs that run on the device: +# runtime, runtime-baremetal, runtime-linux job scheduling over the cores +# engine hardware threads and instruction streams +# bindings how each core runs jobs and each +# hardware thread is loaded +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(goban-device LANGUAGES C CXX) +include("${DESIGN_CMAKE}") +set(CMAKE_POSITION_INDEPENDENT_CODE ON) + +# How each core runs jobs, and how each hardware thread is loaded. Nothing +# references these bindings, so they are linked as objects rather than taken +# from an archive; as link options they come before the libraries they call. +add_library(bindings INTERFACE) +foreach(CORE IN LISTS DESIGN_CORES) + target_link_options(bindings INTERFACE "${DESIGN_CORE_BUILD_${CORE}}/execute.o") +endforeach() +foreach(THREAD IN LISTS DESIGN_THREAD_STACKS) + target_link_options(bindings INTERFACE "${DESIGN_THREAD_BUILD_${THREAD}}/loader.o") +endforeach() +# DMA helpers are sized by the target with the fewest bank lines. +list(GET DESIGN_RUNTIME_TARGETS 0 PARAMS_TARGET) +set(RUNTIME_DEFS ${DESIGN_TARGET_DEFS_${PARAMS_TARGET}}) + +add_subdirectory("${DESIGN_REPO}/stack/engine" engine) +add_subdirectory("${DESIGN_REPO}/stack/runtime" runtime) +export(TARGETS bindings runtime runtime-baremetal runtime-linux engine + runtime-shared runtime-linux-shared engine-shared FILE device.cmake) diff --git a/examples/designs/goban/stack/device/core/CMakeLists.txt b/examples/designs/goban/stack/device/core/CMakeLists.txt new file mode 100644 index 000000000..0a1dc949c --- /dev/null +++ b/examples/designs/goban/stack/device/core/CMakeLists.txt @@ -0,0 +1,32 @@ +# ===----------------------------------------------------------------------=== +# Goban's own core: a CPU that runs the matmul hardware thread. +# +# Built by the stack top of each simulation entry after the core's threads, +# configured with DESIGN_CMAKE and CORE. Products: +# execute.o how the core runs jobs (through the engine) +# libengine-compiler.so buddy-opt pass plugin of the framework engine +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(goban-core-stack LANGUAGES C CXX) +include("${DESIGN_CMAKE}") + +set(RUNTIME "${DESIGN_REPO}/stack/runtime") +set(ENGINE "${DESIGN_REPO}/stack/engine") +find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) + +# How the core runs jobs; instruction streams use the framework engine. +add_custom_command( + OUTPUT "${CMAKE_BINARY_DIR}/execute.o" + COMMAND "${LINUX_CC}" -O2 -fPIC -ffreestanding -fno-builtin -msmall-data-limit=0 + "-DCORE_SIGNATURE=${DESIGN_DEVICE_CORES_${DESIGN_CORE_HART_${CORE}}_SIGNATURE}ULL" "-I${RUNTIME}/scheduler" + "-I${ENGINE}/include" -c "${RUNTIME}/execute/engine.c" -o "${CMAKE_BINARY_DIR}/execute.o" + DEPENDS "${RUNTIME}/execute/engine.c" "${RUNTIME}/scheduler/runtime.h" + "${ENGINE}/include/engine.h" + VERBATIM) +add_custom_target(execute ALL DEPENDS "${CMAKE_BINARY_DIR}/execute.o") + +# The engine's compiler plugin, for the threads this core runs. +add_subdirectory("${ENGINE}/compiler" engine-compiler) +add_library(engine-compiler-plugin MODULE $) +set_target_properties(engine-compiler-plugin PROPERTIES OUTPUT_NAME engine-compiler + LINKER_LANGUAGE CXX LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") diff --git a/examples/designs/goban/workloads/CMakeLists.txt b/examples/designs/goban/workloads/CMakeLists.txt new file mode 100644 index 000000000..6f05e9c42 --- /dev/null +++ b/examples/designs/goban/workloads/CMakeLists.txt @@ -0,0 +1,11 @@ +add_custom_target(design-ctests-build) + +add_subdirectory(device) + +add_custom_target(design-workloads-build ALL + DEPENDS + design-ctests-build + mlirtest-workloads-build + soctests-build + COMMENT "Building goban workloads" + VERBATIM) diff --git a/examples/designs/goban/workloads/configs/tests.toml b/examples/designs/goban/workloads/configs/tests.toml new file mode 100644 index 000000000..c1c7ceb33 --- /dev/null +++ b/examples/designs/goban/workloads/configs/tests.toml @@ -0,0 +1,2 @@ +controlHart = 0 +reportHart = 0 diff --git a/examples/designs/goban/workloads/device/CMakeLists.txt b/examples/designs/goban/workloads/device/CMakeLists.txt new file mode 100644 index 000000000..ad6d4787c --- /dev/null +++ b/examples/designs/goban/workloads/device/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(core) diff --git a/examples/designs/goban/workloads/device/core/CMakeLists.txt b/examples/designs/goban/workloads/device/core/CMakeLists.txt new file mode 100644 index 000000000..bd9914576 --- /dev/null +++ b/examples/designs/goban/workloads/device/core/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(engine) diff --git a/examples/designs/goban/workloads/device/core/engine/CMakeLists.txt b/examples/designs/goban/workloads/device/core/engine/CMakeLists.txt new file mode 100644 index 000000000..8613c8300 --- /dev/null +++ b/examples/designs/goban/workloads/device/core/engine/CMakeLists.txt @@ -0,0 +1,40 @@ +set(ENGINE_DIR ${BUCKYBALL_REPO_ROOT}/stack/engine) +set(MATMUL_TASK_DIR + ${BUCKYBALL_REPO_ROOT}/examples/prefab/threads/matmul/stack +) +set(MATMUL_NAME + "${DESIGN_NAME}-matmul-ctest-design-matmul_threads" +) +set(MATMUL_ELF ${MATMUL_NAME}-baremetal) +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${MATMUL_ELF} + COMMAND + ${ELF_CC} -O2 -march=rv64gc -mcmodel=medany -msmall-data-limit=0 + -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax -Wl,-T,${BBSIM_LD} + ${DESIGN_DEFINES} -I${RUNTIME_INCLUDE_DIR} + -I${ENGINE_DIR}/include -I${MATMUL_TASK_DIR} + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services -I${BAREMETAL_DIR} + ${BAREMETAL_DIR}/start.S ${CMAKE_CURRENT_SOURCE_DIR}/matmul_threads.c + $ $ + # How the device's cores run jobs and its hardware threads are loaded. + $ $ + -lgcc -o ${MATMUL_ELF} + DEPENDS + matmul_threads.c + ${MATMUL_TASK_DIR}/matmul.h + runtime-baremetal + runtime + engine + ${ENGINE_DIR}/include/thread.h + ${ENGINE_DIR}/include/engine.h + ${BBSIM_LD} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMAND_EXPAND_LISTS + VERBATIM +) +add_custom_target( + ${MATMUL_NAME}-ctest-build + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${MATMUL_ELF} +) +add_dependencies(design-ctests-build ${MATMUL_NAME}-ctest-build) diff --git a/examples/designs/goban/workloads/device/core/engine/matmul_threads.c b/examples/designs/goban/workloads/device/core/engine/matmul_threads.c new file mode 100644 index 000000000..148586ee8 --- /dev/null +++ b/examples/designs/goban/workloads/device/core/engine/matmul_threads.c @@ -0,0 +1,61 @@ +#include +#include +#include +#include +#include + +static int8_t a[4][256] __attribute__((aligned(64))); +static int8_t b[4][256] __attribute__((aligned(64))); +static int32_t bias[4][16] __attribute__((aligned(64))); +static int32_t output[4][256] __attribute__((aligned(64))); +static struct matmul_task args[4]; +static struct engine_job jobs[4]; +static struct engine_result results[4]; + +static int compute(void *context, void *argument) { + (void)argument; + struct engine *engine = context; + if (engine->thread_count != 4) + return 1; + for (unsigned tid = 0; tid < 4; ++tid) { + for (unsigned i = 0; i < 256; ++i) { + a[tid][i] = (i * 3 + tid) % 13 - 6; + b[tid][i] = (i * 5 + tid) % 11 - 5; + } + for (unsigned i = 0; i < 16; ++i) + bias[tid][i] = i - 8 + tid; + args[tid] = + (struct matmul_task){(uintptr_t)a[tid], (uintptr_t)b[tid], + (uintptr_t)bias[tid], (uintptr_t)output[tid]}; + jobs[tid] = (struct engine_job){ + .thread = tid, + .task = {.id = tid + 1, .signature = engine->slots[tid].signature}, + .opaque = &args[tid]}; + } + engine_run(engine, jobs, 4, results); + for (unsigned tid = 0; tid < 4; ++tid) { + if (results[tid].state != ENGINE_SUCCEEDED || results[tid].value) + return 2; + for (unsigned row = 0; row < 16; ++row) + for (unsigned col = 0; col < 16; ++col) { + int32_t expected = bias[tid][col]; + for (unsigned k = 0; k < 16; ++k) + expected += a[tid][row * 16 + k] * b[tid][k * 16 + col]; + if (output[tid][row * 16 + col] != expected) + return 3; + } + } + return 0; +} + +int main(void) { + struct runtime_core core = + runtime_core(0, bb_cores[bb_hart_slot(0)].signature, 0); + struct runtime_job job = { + .signature = core.signature, .core = 0, .entry = compute}; + struct runtime_result result; + struct runtime runtime; + runtime_init(&runtime, &core, 1, &job, 1, &result); + runtime_baremetal_run(&runtime, 0); + return result.state == RUNTIME_SUCCEEDED ? result.status : 4; +} diff --git a/examples/designs/goban/workloads/device/core/engine/mvover_rows.c b/examples/designs/goban/workloads/device/core/engine/mvover_rows.c new file mode 100644 index 000000000..8be6c3185 --- /dev/null +++ b/examples/designs/goban/workloads/device/core/engine/mvover_rows.c @@ -0,0 +1,57 @@ +#include +#include +#include + +#define ROWS 8 +#define ROW_BYTES (THREAD_MEM_BANK_WIDTH / 8) +#if THREAD_MEM_BANK_ENTRIES < ROWS +#error "mvover_rows requires eight rows per bank" +#endif +static uint8_t input[ROWS * ROW_BYTES] __attribute__((aligned(64))); +static uint8_t output[ROWS * ROW_BYTES] __attribute__((aligned(64))); +#if !CORE_SHARED_MEM_ENABLE || \ + THREAD_FRONTEND_SHARED_VBANK_BASE >= CORE_VIRTUAL_BANK_COUNT +#error "mvover_rows requires a shared bank for issuer/owner trace coverage" +#endif +static uint8_t shared_output[ROWS * ROW_BYTES] __attribute__((aligned(64))); +static unsigned source_ready, target_ready, moved, stored; +int main(void) { + const uint32_t core = bb_get_hart_id(); + if (core == 1) { + for (unsigned i = 0; i < sizeof(input); ++i) + input[i] = (uint8_t)(i * 17 + 3); + bb_mem_alloc(1, 1, 1); + bb_mem_alloc(THREAD_FRONTEND_SHARED_VBANK_BASE, 1, 1); + bb_mvin((uintptr_t)input, 1, ROWS, 1); + // Hart 1 issues the write; the shared bank pool is owned by hart 0. + bb_mvin((uintptr_t)input, THREAD_FRONTEND_SHARED_VBANK_BASE, ROWS, 1); + bb_mvout((uintptr_t)shared_output, THREAD_FRONTEND_SHARED_VBANK_BASE, ROWS, + 1); + for (unsigned i = 0; i < sizeof(input); ++i) + if (shared_output[i] != input[i]) + return 1; + bb_mem_release(THREAD_FRONTEND_SHARED_VBANK_BASE); + __atomic_store_n(&source_ready, 1, __ATOMIC_RELEASE); + } else if (core == 2) { + bb_mem_alloc(8, 1, 1); + __atomic_store_n(&target_ready, 1, __ATOMIC_RELEASE); + while (!__atomic_load_n(&moved, __ATOMIC_ACQUIRE)) { + } + bb_mvout((uintptr_t)output, 8, ROWS, 1); + __atomic_store_n(&stored, 1, __ATOMIC_RELEASE); + } else if (core == 0) { + while (!__atomic_load_n(&source_ready, __ATOMIC_ACQUIRE) || + !__atomic_load_n(&target_ready, __ATOMIC_ACQUIRE)) { + } + bb_mvover(0, 1, 0, 1, 8, 0, ROWS); + __atomic_store_n(&moved, 1, __ATOMIC_RELEASE); + while (!__atomic_load_n(&stored, __ATOMIC_ACQUIRE)) { + } + for (unsigned i = 0; i < sizeof(input); ++i) + if (input[i] != output[i]) + return 1; + return 0; + } + for (;;) + asm volatile("wfi"); +} diff --git a/examples/designs/multi-rocket/arch/build.sc b/examples/designs/multi-rocket/arch/build.sc new file mode 100644 index 000000000..170086f40 --- /dev/null +++ b/examples/designs/multi-rocket/arch/build.sc @@ -0,0 +1,17 @@ +import mill._ +import mill.scalalib._ +import $file.^.^.^.^.arch.src.main.scala.framework.device.cbus.{build => cbus_build} +import $file.^.^.^.{build => examples} +import $file.^.^.^.prefab.cores.rocket.arch.{build => rocket} + +val designRoot: os.Path = millSourcePath / os.up + +object design extends SbtModule { + override def generatedSources = T(cbus_build.registry(sources() ++ rocket.library.sources(), T.dest)) + override def scalaVersion = "2.13.16" + override def moduleDeps = Seq(examples.library, rocket.library) + override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") + override def sources = T.sources(designRoot / "arch", designRoot / "configs") +} diff --git a/examples/designs/multi-rocket/arch/device/DeviceTop.scala b/examples/designs/multi-rocket/arch/device/DeviceTop.scala new file mode 100644 index 000000000..0c67d0282 --- /dev/null +++ b/examples/designs/multi-rocket/arch/device/DeviceTop.scala @@ -0,0 +1,80 @@ +package examples.designs.multirocket + +import framework.lib.arbiter.{RRArbiter} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, Instance, Instantiate} +import framework.device.configs.DeviceParam +import framework.device.cbus.CBus +import framework.device.cbus.generated.CoreComponents +import buckyball.config.Design +import _root_.configs.GlobalConfig +import framework.core.clink.CLinkIO +import framework.core.memory.home.Home +import framework.device.ddr.Bridge +import framework.device.peripherals.Controller +import framework.device.peripherals.bootrom.BootRomLines +import framework.device.dlink.BaseDevice +import framework.ip.bus.axi4 +import framework.ip.bus.chi.snf.LineResponse +import framework.lib.queue.{Queue => ResponseQueue} + +/** Configured Rocket instances share one coherent Home and one device memory map. */ +@instantiable +class DeviceTop(b: GlobalConfig, design: Design) extends BaseDevice(b.device.ddr.axi) { + val device = b.device + val parameters = device.cores + + require(parameters.forall(p => !p.clink.backing && p.clink.requesters == 2 && p.acc.isEmpty)) + val sharedHome = parameters(0).memory.home + require(parameters.forall(_.memory.home == sharedHome)) + require(sharedHome.memory.agents == parameters.size * 2) + + val bus = Instantiate(new CBus(b.device, CBus.generators(design, b.device, b.sim, CoreComponents.factories))) + + val home: Instance[Home] = Instantiate(new Home(sharedHome)) + home.io.active := true.B + home.io.blockRequesterRsp := false.B + home.io.requesters <> bus.coherent + + val peripherals = Instantiate(new Controller(device, device.ddr.axi, parameters(0).memory.memoryAccess.memory)) + + val peripheralBus = Instantiate(new axi4.Decoder( + Seq( + (device.clint.base, device.clint.bytes), + (device.plic.base, device.plic.bytes), + (device.scu.baseAddress, device.scu.totalSizeBytes) + ), + device.ddr.axi + )) + + peripheralBus.io.in <> bus.uncached + peripherals.io.axi <> peripheralBus.io.matched + bus.external <> peripheralBus.io.other + peripherals.io.hardwareInterrupt.sources := 0.U + bus.interrupts := peripherals.io.hardwareInterrupt.harts + + val rom = Instantiate(new BootRomLines(device.bootrom, sharedHome.memory.chi)) + val bridge = Instantiate(new Bridge(device.ddr)) + require(device.ddr.clients == 1) + val request = home.io.memoryReq + val inRom = request.bits.addr >= device.bootrom.base.U && + request.bits.addr < (device.bootrom.base + device.bootrom.bytes).U + rom.io.request.valid := request.valid && inRom + rom.io.request.bits := request.bits + bridge.io.request(0).valid := request.valid && !inRom + bridge.io.request(0).bits := request.bits + request.ready := Mux(inRom, rom.io.request.ready, bridge.io.request(0).ready) + val responses = Instantiate(new RRArbiter(new LineResponse(sharedHome.memory.chi), 2)) + responses.io.in(0) <> rom.io.response + responses.io.in(1) <> bridge.io.response(0) + val response = Instantiate(new ResponseQueue(new LineResponse(sharedHome.memory.chi), 1, false, false)) + response.io.enq <> responses.io.out + home.io.memoryResp <> response.io.deq + + val memoryBus = Instantiate(new axi4.Interconnect(axi4.InterconnectParam(device.ddr.axi, 2, None))) + memoryBus.io.in(0) <> bus.dlink.axi + memoryBus.io.in(1) <> bridge.io.axi + dlink.axi <> memoryBus.io.out +} diff --git a/examples/designs/multi-rocket/configs/design.toml b/examples/designs/multi-rocket/configs/design.toml new file mode 100644 index 000000000..a55a275cc --- /dev/null +++ b/examples/designs/multi-rocket/configs/design.toml @@ -0,0 +1,4 @@ +[device] +include = "device/default.toml" +[sims] +include = "sims/default.toml" diff --git a/examples/designs/multi-rocket/configs/device/clint.toml b/examples/designs/multi-rocket/configs/device/clint.toml new file mode 100644 index 000000000..9d0ad86a0 --- /dev/null +++ b/examples/designs/multi-rocket/configs/device/clint.toml @@ -0,0 +1,4 @@ +base = 0x2000000 +bytes = 0x10000 +tickCycles = 1000 +hartIds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31] diff --git a/examples/designs/multi-rocket/configs/device/core/c2c/core.toml b/examples/designs/multi-rocket/configs/device/core/c2c/core.toml new file mode 100644 index 000000000..ebf8171e4 --- /dev/null +++ b/examples/designs/multi-rocket/configs/device/core/c2c/core.toml @@ -0,0 +1,3 @@ +enabled = false +peerCores = 32 +axi = "default" diff --git a/examples/designs/multi-rocket/configs/device/core/default.toml b/examples/designs/multi-rocket/configs/device/core/default.toml new file mode 100644 index 000000000..af52db82d --- /dev/null +++ b/examples/designs/multi-rocket/configs/device/core/default.toml @@ -0,0 +1,30 @@ +component = "rocket" +implementation = "rocket.toml" +threads = [] +resetVector = 0x10000 +mtimeAddress = 0x200bff8 +memory = "memory/core.toml" +fetch = "default" +fpu = "default" +interconnect = "interconnect/core.toml" +c2c = "c2c/core.toml" +controller = "default" + +[param] +threadDataBytes = 64 +xLen = 64 +vaddrBits = 39 +paddrBits = 36 +pgIdxBits = 12 +pgLevels = 3 +nPMPs = 16 + +[sharedMem] +enable = false +entries = 16384 +inputChannels = 1 +defaultGroupCount = 1 +virtualBankCount = 16 +latency = 10 +bankEntries = 1024 +bankWidth = 128 diff --git a/examples/designs/multi-rocket/configs/device/core/interconnect/core.toml b/examples/designs/multi-rocket/configs/device/core/interconnect/core.toml new file mode 100644 index 000000000..13d04a5a6 --- /dev/null +++ b/examples/designs/multi-rocket/configs/device/core/interconnect/core.toml @@ -0,0 +1,2 @@ +axi = "default" +slots = 16 diff --git a/examples/designs/multi-rocket/configs/device/core/memory/access/core.toml b/examples/designs/multi-rocket/configs/device/core/memory/access/core.toml new file mode 100644 index 000000000..2e90444bc --- /dev/null +++ b/examples/designs/multi-rocket/configs/device/core/memory/access/core.toml @@ -0,0 +1,52 @@ +translationEntries = 16 +tagBits = 6 +resultLineBits = 0 +[[regions]] +base = 0x80000000 +bytes = 0x20000000 +cacheable = true +executable = true +readable = true +writable = true +atomic = true +normal = true + +[[regions]] +base = 0x10000 +bytes = 0x10000 +cacheable = true +executable = true +readable = true +writable = false +atomic = false +normal = true + +[[regions]] +base = 0x2000000 +bytes = 0x10000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0xc000000 +bytes = 0x4000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0x60000000 +bytes = 0x10000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false diff --git a/examples/designs/multi-rocket/configs/device/core/memory/core.toml b/examples/designs/multi-rocket/configs/device/core/memory/core.toml new file mode 100644 index 000000000..985e1cf13 --- /dev/null +++ b/examples/designs/multi-rocket/configs/device/core/memory/core.toml @@ -0,0 +1,4 @@ +uncached = "default" +access = "access/core.toml" +dcache = "dcache/core.toml" +home = "home/core.toml" diff --git a/examples/designs/multi-rocket/configs/device/core/memory/dcache/core.toml b/examples/designs/multi-rocket/configs/device/core/memory/dcache/core.toml new file mode 100644 index 000000000..88b00a13d --- /dev/null +++ b/examples/designs/multi-rocket/configs/device/core/memory/dcache/core.toml @@ -0,0 +1,6 @@ +[tracking] +entries = 8 +addressBits = 44 +idBits = 8 +lineBytes = 64 +maxRanges = 8 diff --git a/examples/designs/multi-rocket/configs/device/core/memory/dcache/shared.toml b/examples/designs/multi-rocket/configs/device/core/memory/dcache/shared.toml new file mode 100644 index 000000000..91cce2568 --- /dev/null +++ b/examples/designs/multi-rocket/configs/device/core/memory/dcache/shared.toml @@ -0,0 +1,6 @@ +lines = 64 +homeId = 96 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/multi-rocket/configs/device/core/memory/home/core.toml b/examples/designs/multi-rocket/configs/device/core/memory/home/core.toml new file mode 100644 index 000000000..8a021b8fb --- /dev/null +++ b/examples/designs/multi-rocket/configs/device/core/memory/home/core.toml @@ -0,0 +1,13 @@ +agents = 64 +mshrEntries = 8 +nodeId = 96 +requesters = "default" + +[cache] +addressBits = 44 +lineBytes = 64 +sets = 64 +ways = 4 +idBits = 8 +metadataBits = 4 +responseDepth = 2 diff --git a/examples/designs/multi-rocket/configs/device/core/memory/icache/shared.toml b/examples/designs/multi-rocket/configs/device/core/memory/icache/shared.toml new file mode 100644 index 000000000..91cce2568 --- /dev/null +++ b/examples/designs/multi-rocket/configs/device/core/memory/icache/shared.toml @@ -0,0 +1,6 @@ +lines = 64 +homeId = 96 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/multi-rocket/configs/device/core/rocket.toml b/examples/designs/multi-rocket/configs/device/core/rocket.toml new file mode 100644 index 000000000..b94846c8d --- /dev/null +++ b/examples/designs/multi-rocket/configs/device/core/rocket.toml @@ -0,0 +1,12 @@ +component = "rocket" +physical_address_bits = 36 +hart_id_bits = 5 +page_levels = 3 +pmp_entries = 16 +virtual_memory = true +supervisor = true +user = true +floating_point = true +atomics = true +multiply_divide = true +external_home = true diff --git a/examples/designs/multi-rocket/configs/device/ddr.toml b/examples/designs/multi-rocket/configs/device/ddr.toml new file mode 100644 index 000000000..31815e9e1 --- /dev/null +++ b/examples/designs/multi-rocket/configs/device/ddr.toml @@ -0,0 +1,13 @@ +clients = 1 +slotsPerClient = 16 +dataBits = 128 +idBits = 4 +idBase = 0 +base = 0x80000000 +bytes = 0x20000000 +model = 0 + +[line] +nodeIdBits = 7 +addressBits = 44 +dataBits = 256 diff --git a/examples/designs/multi-rocket/configs/device/default.toml b/examples/designs/multi-rocket/configs/device/default.toml new file mode 100644 index 000000000..de4966bb9 --- /dev/null +++ b/examples/designs/multi-rocket/configs/device/default.toml @@ -0,0 +1,517 @@ +ddr = "ddr.toml" +bootrom = "default" +clint = "clint.toml" +plic = "plic.toml" +scu = "default" + +[[cores]] +include = "core/default.toml" +hartId = 0 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 1 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 2 + +[[cores]] +include = "core/default.toml" +hartId = 1 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 3 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 4 + +[[cores]] +include = "core/default.toml" +hartId = 2 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 5 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 6 + +[[cores]] +include = "core/default.toml" +hartId = 3 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 7 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 8 + +[[cores]] +include = "core/default.toml" +hartId = 4 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 9 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 10 + +[[cores]] +include = "core/default.toml" +hartId = 5 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 11 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 12 + +[[cores]] +include = "core/default.toml" +hartId = 6 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 13 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 14 + +[[cores]] +include = "core/default.toml" +hartId = 7 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 15 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 16 + +[[cores]] +include = "core/default.toml" +hartId = 8 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 17 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 18 + +[[cores]] +include = "core/default.toml" +hartId = 9 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 19 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 20 + +[[cores]] +include = "core/default.toml" +hartId = 10 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 21 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 22 + +[[cores]] +include = "core/default.toml" +hartId = 11 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 23 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 24 + +[[cores]] +include = "core/default.toml" +hartId = 12 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 25 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 26 + +[[cores]] +include = "core/default.toml" +hartId = 13 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 27 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 28 + +[[cores]] +include = "core/default.toml" +hartId = 14 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 29 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 30 + +[[cores]] +include = "core/default.toml" +hartId = 15 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 31 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 32 + +[[cores]] +include = "core/default.toml" +hartId = 16 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 33 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 34 + +[[cores]] +include = "core/default.toml" +hartId = 17 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 35 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 36 + +[[cores]] +include = "core/default.toml" +hartId = 18 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 37 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 38 + +[[cores]] +include = "core/default.toml" +hartId = 19 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 39 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 40 + +[[cores]] +include = "core/default.toml" +hartId = 20 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 41 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 42 + +[[cores]] +include = "core/default.toml" +hartId = 21 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 43 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 44 + +[[cores]] +include = "core/default.toml" +hartId = 22 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 45 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 46 + +[[cores]] +include = "core/default.toml" +hartId = 23 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 47 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 48 + +[[cores]] +include = "core/default.toml" +hartId = 24 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 49 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 50 + +[[cores]] +include = "core/default.toml" +hartId = 25 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 51 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 52 + +[[cores]] +include = "core/default.toml" +hartId = 26 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 53 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 54 + +[[cores]] +include = "core/default.toml" +hartId = 27 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 55 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 56 + +[[cores]] +include = "core/default.toml" +hartId = 28 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 57 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 58 + +[[cores]] +include = "core/default.toml" +hartId = 29 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 59 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 60 + +[[cores]] +include = "core/default.toml" +hartId = 30 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 61 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 62 + +[[cores]] +include = "core/default.toml" +hartId = 31 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 63 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 64 diff --git a/examples/designs/multi-rocket/configs/device/plic.toml b/examples/designs/multi-rocket/configs/device/plic.toml new file mode 100644 index 000000000..3ccf21de4 --- /dev/null +++ b/examples/designs/multi-rocket/configs/device/plic.toml @@ -0,0 +1,5 @@ +base = 0xc000000 +bytes = 0x4000000 +sources = 1 +priorityBits = 3 +hartIds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31] diff --git a/examples/designs/multi-rocket/configs/sims/default.toml b/examples/designs/multi-rocket/configs/sims/default.toml new file mode 100644 index 000000000..0cf750aff --- /dev/null +++ b/examples/designs/multi-rocket/configs/sims/default.toml @@ -0,0 +1,17 @@ +diffTest = false + +[memory] +lineBytes = 64 +clockHz = 1000000000 + +[targets.device] +config = "examples.designs.multirocket.sims.device.DeviceConfig" +stack_name = "multi-rocket" +regression = "sims/device/regression" +kernel = "sims/device/workloads/kernel" + +[targets.device.configs] +MultiRocketVerilatorConfig = "verilator.toml" + +[verification] +registry = "../../sims/ip/uvm/registry.toml" diff --git a/examples/designs/multi-rocket/configs/sims/verilator.toml b/examples/designs/multi-rocket/configs/sims/verilator.toml new file mode 100644 index 000000000..b1c32c64e --- /dev/null +++ b/examples/designs/multi-rocket/configs/sims/verilator.toml @@ -0,0 +1 @@ +# MultiRocketVerilatorConfig: the multi-rocket design as configured, with no overrides. diff --git a/examples/designs/multi-rocket/sims/device/arch/DeviceConfig.scala b/examples/designs/multi-rocket/sims/device/arch/DeviceConfig.scala new file mode 100644 index 000000000..9a9ef4f76 --- /dev/null +++ b/examples/designs/multi-rocket/sims/device/arch/DeviceConfig.scala @@ -0,0 +1,18 @@ +package examples.designs.multirocket.sims.device + +import buckyball.config.Design +import chisel3.experimental.hierarchy.{Instance, Instantiate} +import examples.designs.multirocket.DeviceTop +import framework.device.cbus.CBus +import framework.device.cbus.generated.CoreComponents +import framework.device.dlink.BaseDevice +import _root_.configs.GlobalConfig +import _root_.sims.soc.SystemTarget +import java.nio.file.{Files, Paths} + +class DeviceConfig(design: Design) + extends SystemTarget( + GlobalConfig.fromProto(design, CBus.parameters(design, CoreComponents.factories)) + ) { + def instantiate(b: GlobalConfig): Instance[BaseDevice] = Instantiate(new DeviceTop(b, design)) +} diff --git a/examples/designs/multi-rocket/sims/device/arch/build.sc b/examples/designs/multi-rocket/sims/device/arch/build.sc new file mode 100644 index 000000000..5b94111b8 --- /dev/null +++ b/examples/designs/multi-rocket/sims/device/arch/build.sc @@ -0,0 +1,17 @@ +import mill._ +import mill.scalalib._ +import $file.^.^.^.arch.{build => design_build} +import $file.^.^.^.^.^.^.arch.src.main.scala.sims.{build => sims_build} + +val entryRoot: os.Path = millSourcePath + +object simulation extends SbtModule { + override def millSourcePath = entryRoot + override def scalaVersion = "2.13.16" + override def moduleDeps = Seq(design_build.design, sims_build.library) + override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") + override def sources = T.sources(entryRoot / "DeviceConfig.scala") + override def mainClass = Some("sims.verilator.Elaborate") +} diff --git a/examples/designs/multi-rocket/sims/device/emu/Cargo.lock b/examples/designs/multi-rocket/sims/device/emu/Cargo.lock new file mode 100644 index 000000000..e2f70bc76 --- /dev/null +++ b/examples/designs/multi-rocket/sims/device/emu/Cargo.lock @@ -0,0 +1,1971 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "assert_cmd" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aa3a22042e45de04255c7bf3626e239f450200fd0493c1e382263544b20aea6" +dependencies = [ + "anstyle", + "bstr", + "libc", + "predicates", + "predicates-core", + "predicates-tree", + "wait-timeout", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bebop" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", + "bebop-bemu", + "bebop-bemu-profile", + "bebop-dasm", + "bebop-fd-redirect", + "bebop-p2e", + "bebop-report", + "bebop-rtl-trace", + "bebop-terminal", + "bebop-trace-perfetto", + "bebop-uart", + "bebop-vcs", + "bebop-verilator", + "bebop-vm", + "camino", + "clap", + "duct", + "env_logger", + "libc", + "log", + "nix", + "serde", + "serde_json", + "snafu", + "toml", +] + +[[package]] +name = "bebop-bank-hash" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "snafu", +] + +[[package]] +name = "bebop-bemu" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", + "bebop-bemu-profile", + "bebop-dasm", + "bebop-dtb", + "bebop-elf", + "bebop-keyboard", + "bebop-memory", + "bebop-microphone", + "bebop-rtc", + "bebop-speaker", + "bebop-syscall", + "bebop-uart", + "bebop-vga", + "clap", + "linkme", + "prost", + "prost-build", + "prost-types", + "rustc_apfloat", + "snafu", + "toml", +] + +[[package]] +name = "bebop-bemu-profile" +version = "0.1.0" + +[[package]] +name = "bebop-core-rocket" +version = "0.1.0" +dependencies = [ + "bebop-bemu", +] + +[[package]] +name = "bebop-dasm" +version = "0.1.0" + +[[package]] +name = "bebop-dtb" +version = "0.1.0" + +[[package]] +name = "bebop-elf" +version = "0.1.0" +dependencies = [ + "bebop-memory", +] + +[[package]] +name = "bebop-fd-redirect" +version = "0.1.0" +dependencies = [ + "libc", +] + +[[package]] +name = "bebop-keyboard" +version = "0.1.0" + +[[package]] +name = "bebop-memory" +version = "0.1.0" + +[[package]] +name = "bebop-microphone" +version = "0.1.0" + +[[package]] +name = "bebop-p2e" +version = "0.1.0" +dependencies = [ + "bebop-memory", + "bebop-rtl-trace", + "bebop-uart", + "duct", + "libc", + "log", + "serde", + "serde_json", +] + +[[package]] +name = "bebop-report" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "bebop-rtc" +version = "0.1.0" + +[[package]] +name = "bebop-rtl-trace" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", +] + +[[package]] +name = "bebop-sim-build" +version = "0.1.0" +dependencies = [ + "prost", + "prost-build", + "prost-types", +] + +[[package]] +name = "bebop-sim-multi-rocket-device" +version = "0.1.0" +dependencies = [ + "assert_cmd", + "bebop", + "bebop-bemu", + "bebop-core-rocket", + "bebop-report", + "bebop-sim-build", + "bebop-verilator", + "chrono", + "clap", + "libtest-mimic", + "predicates", + "serde", + "toml", + "walkdir", +] + +[[package]] +name = "bebop-speaker" +version = "0.1.0" + +[[package]] +name = "bebop-syscall" +version = "0.1.0" +dependencies = [ + "bebop-memory", + "once_cell", +] + +[[package]] +name = "bebop-terminal" +version = "0.1.0" +dependencies = [ + "crossterm", + "ratatui", +] + +[[package]] +name = "bebop-trace-perfetto" +version = "0.1.0" +dependencies = [ + "clap", + "serde_json", +] + +[[package]] +name = "bebop-uart" +version = "0.1.0" + +[[package]] +name = "bebop-vcs" +version = "0.1.0" +dependencies = [ + "snafu", +] + +[[package]] +name = "bebop-verilator" +version = "0.1.0" +dependencies = [ + "bebop-dasm", + "bebop-fd-redirect", + "bebop-rtl-trace", + "bebop-uart", + "libc", + "snafu", +] + +[[package]] +name = "bebop-vga" +version = "0.1.0" + +[[package]] +name = "bebop-vm" +version = "0.1.0" +dependencies = [ + "clap", + "font8x8", + "sdl2", + "vt100", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "bstr" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" +dependencies = [ + "memchr", + "regex-automata", + "serde_core", +] + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "camino" +version = "1.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbad30e4b4c14a39e3cc8aed085a12a327257c316619c93581e017bc52be591" + +[[package]] +name = "cassowary" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" + +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + +[[package]] +name = "cc" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f74872d07caf508b30a21f6836e7d7016a2eaf7d9ff4f48deaa58cd8a0407630" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "compact_str" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fd622ebbb56a5b2ccb651b32b911cdeb2a9b4b11776b2473bf26a26a286244e" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "static_assertions", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crossterm" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" +dependencies = [ + "bitflags 2.13.2", + "crossterm_winapi", + "mio", + "parking_lot", + "rustix 0.38.44", + "signal-hook 0.3.18", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + +[[package]] +name = "darling" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 3.0.9", +] + +[[package]] +name = "darling_macro" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" +dependencies = [ + "darling_core", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "defmt" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a0795ac740cb54d7b3775c9c17345f6aaecf884c01733b39d6017738e0e3e1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "840bb056bd99c495c9188198a258ab787c79f306d864db3e6e8451442d6d8366" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "defmt-parser" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71958f69fb2de856c5dd51ac9377e986ca1aa5919f931bf5395a807277f889a0" +dependencies = [ + "thiserror", +] + +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + +[[package]] +name = "duct" +version = "0.13.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4ab5718d1224b63252cd0c6f74f6480f9ffeb117438a2e0f5cf6d9a4798929c" +dependencies = [ + "libc", + "once_cell", + "os_pipe", + "shared_child", +] + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "env_filter" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "escape8259" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f0d310082fd5de539d4f41601182241147498444a7695c520288a0cdbead6" + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "float-cmp" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09cf3155332e944990140d967ff5eceb70df778b34f77d8075db46e4704e6d8" +dependencies = [ + "num-traits", +] + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "font8x8" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875488b8711a968268c7cf5d139578713097ca4635a76044e8fe8eedf831d07e" + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "indoc" +version = "2.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a37b2691796cffeb8a8cd305ac66e65841559f147f4e63231d0eafa4db5384d1" +dependencies = [ + "rustversion", +] + +[[package]] +name = "instability" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3b5acc1e2fd9375041a388da33d1eb8aed5f7a8c0dd3543e3ea2805adfbe20" +dependencies = [ + "darling", + "indoc", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jiff" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b005715dcbeb0089a3c0dab99f2ff1cc3b2525323552703d648585d342a383" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", +] + +[[package]] +name = "jiff-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e52fe76043ccecc9005d2305ebaadf7d7fc0cc89ca6baa10a94d6bc68c7128c" +dependencies = [ + "defmt", + "log", +] + +[[package]] +name = "jiff-static" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cc9817253cf7c7ee4684451bd327e88d6f3658014e54a29198625590650695c" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "js-sys" +version = "0.3.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "libtest-mimic" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14e6ba06f0ade6e504aff834d7c34298e5155c6baca353cc6a4aaff2f9fd7f33" +dependencies = [ + "anstream", + "anstyle", + "clap", + "escape8259", +] + +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.5", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "os_pipe" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-atomic-util" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "091397be61a01d4be58e7841595bd4bfedb15f1cd54977d79b8271e94ed799a3" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "predicates" +version = "3.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ada8f2932f28a27ee7b70dd6c1c39ea0675c55a36879ab92f3a715eaa1e63cfe" +dependencies = [ + "anstyle", + "difflib", + "float-cmp", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cad38746f3166b4031b1a0d39ad9f954dd291e7854fcc0eed52ee41a0b50d144" + +[[package]] +name = "predicates-tree" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0de1b847b39c8131db0467e9df1ff60e6d0562ab8e9a16e568ad0fdb372e2f2" +dependencies = [ + "predicates-core", + "termtree", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck", + "itertools 0.14.0", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.119", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "ratatui" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabd94c2f37801c20583fc49dd5cd6b0ba68c716787c2dd6ed18571e1e63117b" +dependencies = [ + "bitflags 2.13.2", + "cassowary", + "compact_str", + "crossterm", + "indoc", + "instability", + "itertools 0.13.0", + "lru", + "paste", + "strum", + "unicode-segmentation", + "unicode-truncate", + "unicode-width 0.2.0", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.2", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustc_apfloat" +version = "0.2.3+llvm-462a31f5a5ab" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "486c2179b4796f65bfe2ee33679acf0927ac83ecf583ad6c91c3b4570911b9ad" +dependencies = [ + "bitflags 2.13.2", + "smallvec", +] + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys 0.12.1", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sdl2" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d42407afc6a8ab67e36f92e80b8ba34cbdc55aaeed05249efe9a2e8d0e9feef" +dependencies = [ + "bitflags 1.3.2", + "lazy_static", + "libc", + "sdl2-sys", +] + +[[package]] +name = "sdl2-sys" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ff61407fc75d4b0bbc93dc7e4d6c196439965fbef8e4a4f003a36095823eac0" +dependencies = [ + "cfg-if", + "libc", + "pkg-config", + "version-compare", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "shared_child" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "607549934f6cc26b89cfecfdc46fa90f1e5d1536a68349b0c3a4f9d1c0d37959" +dependencies = [ + "libc", + "sigchld", + "windows-sys 0.61.2", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "sigchld" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24f2b37f04360cd465089b87a9c3869c08220a2f3458463f0adf8badf5e77f2c" +dependencies = [ + "libc", + "os_pipe", + "signal-hook 0.4.5", +] + +[[package]] +name = "signal-hook" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b29ffa27d3d59a049bb328258de100f8012848d1b08d444e08bbc02ac468435e" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-mio" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b75a19a7a740b25bc7944bdee6172368f988763b744e3d4dfe753f6b4ece40cc" +dependencies = [ + "libc", + "mio", + "signal-hook 0.3.18", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "smallvec" +version = "1.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8" + +[[package]] +name = "snafu" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.119", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix 1.1.5", + "windows-sys 0.61.2", +] + +[[package]] +name = "termtree" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-truncate" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3644627a5af5fa321c95b9b235a72fd24cd29c648c2c379431e6628655627bf" +dependencies = [ + "itertools 0.13.0", + "unicode-segmentation", + "unicode-width 0.1.14", +] + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "version-compare" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "579a42fc0b8e0c63b76519a339be31bed574929511fa53c1a3acae26eb258f29" + +[[package]] +name = "vt100" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84cd863bf0db7e392ba3bd04994be3473491b31e66340672af5d11943c6274de" +dependencies = [ + "itoa", + "log", + "unicode-width 0.1.14", + "vte", +] + +[[package]] +name = "vte" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5022b5fbf9407086c180e9557be968742d839e68346af7792b8592489732197" +dependencies = [ + "arrayvec", + "utf8parse", + "vte_generate_state_changes", +] + +[[package]] +name = "vte_generate_state_changes" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e369bee1b05d510a7b4ed645f5faa90619e05437111783ea5848f28d97d3c2e" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/examples/designs/multi-rocket/sims/device/emu/Cargo.toml b/examples/designs/multi-rocket/sims/device/emu/Cargo.toml new file mode 100644 index 000000000..38491051e --- /dev/null +++ b/examples/designs/multi-rocket/sims/device/emu/Cargo.toml @@ -0,0 +1,83 @@ +[workspace] + +[package] +name = "bebop-sim-multi-rocket-device" +version = "0.1.0" +edition = "2021" +autobins = false +autotests = false + +# The model of this simulation entry; the runners below are built from it. +[lib] +name = "simulation" + +[[bin]] +name = "bebop" +path = "../../../../../../bebop/src/main.rs" + +[[bin]] +name = "bebop-bemu" +path = "../../../../../../bebop/src/bin/bebop-bemu.rs" +required-features = ["bemu"] + +[[bin]] +name = "bebop-verilator" +path = "../../../../../../bebop/src/bin/bebop-verilator.rs" +required-features = ["verilator"] + +[[bin]] +name = "bebop-p2e" +path = "../../../../../../bebop/src/bin/bebop-p2e.rs" +required-features = ["p2e"] + +[[test]] +name = "test_bemu" +path = "../../../../../../bebop/tests/test_bemu.rs" +harness = false +required-features = ["bemu"] + +[[test]] +name = "test_verilator" +path = "../../../../../../bebop/tests/test_verilator.rs" +harness = false +required-features = ["verilator"] + +[[test]] +name = "test_p2e" +path = "../../../../../../bebop/tests/test_p2e.rs" +harness = false +required-features = ["p2e"] + +[[bin]] +name = "bebop-vcs" +path = "../../../../../../bebop/src/bin/bebop-vcs.rs" + +[[test]] +name = "test_vcs" +path = "../../../../../../bebop/tests/test_vcs.rs" +harness = false + +[features] +bemu = ["bebop/bemu"] +verilator = ["bebop/verilator", "dep:bebop-verilator"] +p2e = ["bebop/p2e"] + +[dependencies] +bebop = { path = "../../../../../../bebop" } +bebop-verilator = { path = "../../../../../../bebop/src/nodes/verilator", optional = true } +bebop-bemu = { path = "../../../../../../bebop/src/nodes/bemu" } +bebop-core-rocket = { path = "../../../../../prefab/cores/rocket/emu" } +clap = { version = "4", features = ["derive"] } + +[build-dependencies] +bebop-sim-build = { path = "../../../../../../bebop/src/nodes/lib/sim-build" } + +[dev-dependencies] +bebop-report = { path = "../../../../../../bebop/src/nodes/lib/report" } +libtest-mimic = "0.8" +assert_cmd = "2" +walkdir = "2" +predicates = "3" +serde = { version = "1", features = ["derive"] } +toml = "0.8" +chrono = { version = "0.4", default-features = false, features = ["clock"] } diff --git a/examples/designs/multi-rocket/sims/device/emu/build.rs b/examples/designs/multi-rocket/sims/device/emu/build.rs new file mode 100644 index 000000000..beb62e98c --- /dev/null +++ b/examples/designs/multi-rocket/sims/device/emu/build.rs @@ -0,0 +1,4 @@ +fn main() { + bebop_sim_build::balls(); + bebop_sim_build::runner_links(); +} diff --git a/examples/designs/multi-rocket/sims/device/emu/src/lib.rs b/examples/designs/multi-rocket/sims/device/emu/src/lib.rs new file mode 100644 index 000000000..b43b00662 --- /dev/null +++ b/examples/designs/multi-rocket/sims/device/emu/src/lib.rs @@ -0,0 +1,11 @@ +//! The multi-rocket device simulation. +//! Its Core models register in `bebop_bemu::CORES`; the Balls its design.pb +//! names are compiled here and register in `bebop_bemu::BALLS`. +extern crate bebop_core_rocket; + +include!(concat!(env!("OUT_DIR"), "/balls.rs")); + +pub const SIMULATION: bebop_bemu::Simulation = bebop_bemu::Simulation { + design: include_bytes!(concat!(env!("OUT_DIR"), "/design.pb")), + instantiate: bebop_bemu::device::cbus::instantiate, +}; diff --git a/examples/designs/multi-rocket/sims/device/regression/bemu/workloads-bare.toml b/examples/designs/multi-rocket/sims/device/regression/bemu/workloads-bare.toml new file mode 100644 index 000000000..e70ffa6d5 --- /dev/null +++ b/examples/designs/multi-rocket/sims/device/regression/bemu/workloads-bare.toml @@ -0,0 +1,25 @@ +[workloads] +tests = [ + "multi-rocket-core0-ctest-design-coherent_exchange-multicore-baremetal", + "multi-rocket-core0-ctest-check-embench_sglib-combined-baremetal", + "multi-rocket-core0-ctest-check-embench_matmult-int-baremetal", + "multi-rocket-core0-ctest-cpu_arithmetic-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_aha-mont64-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_crc32-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_edn-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_huffbench-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_matmult-int-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_minver-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_nbody-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_nettle-aes-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_nettle-sha256-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_nsichneu-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_picojpeg-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_qrduino-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_sglib-combined-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_slre-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_st-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_statemate-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_ud-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_wikisort-baremetal", +] diff --git a/examples/designs/multi-rocket/sims/device/regression/bemu/workloads-linux.toml b/examples/designs/multi-rocket/sims/device/regression/bemu/workloads-linux.toml new file mode 100644 index 000000000..41a9cd13b --- /dev/null +++ b/examples/designs/multi-rocket/sims/device/regression/bemu/workloads-linux.toml @@ -0,0 +1,4 @@ +[workloads] +tests = [ + "fw_payload-linux.elf", +] diff --git a/examples/designs/multi-rocket/sims/device/regression/p2e/workloads-bare.toml b/examples/designs/multi-rocket/sims/device/regression/p2e/workloads-bare.toml new file mode 100644 index 000000000..73064b9f6 --- /dev/null +++ b/examples/designs/multi-rocket/sims/device/regression/p2e/workloads-bare.toml @@ -0,0 +1,21 @@ +[workloads] +tests = [ + "multi-rocket-core0-ctest-benchmark-embench_aha-mont64-baremetal.hex", + "multi-rocket-core0-ctest-benchmark-embench_crc32-baremetal.hex", + "multi-rocket-core0-ctest-benchmark-embench_edn-baremetal.hex", + "multi-rocket-core0-ctest-benchmark-embench_huffbench-baremetal.hex", + "multi-rocket-core0-ctest-benchmark-embench_matmult-int-baremetal.hex", + "multi-rocket-core0-ctest-benchmark-embench_minver-baremetal.hex", + "multi-rocket-core0-ctest-benchmark-embench_nbody-baremetal.hex", + "multi-rocket-core0-ctest-benchmark-embench_nettle-aes-baremetal.hex", + "multi-rocket-core0-ctest-benchmark-embench_nettle-sha256-baremetal.hex", + "multi-rocket-core0-ctest-benchmark-embench_nsichneu-baremetal.hex", + "multi-rocket-core0-ctest-benchmark-embench_picojpeg-baremetal.hex", + "multi-rocket-core0-ctest-benchmark-embench_qrduino-baremetal.hex", + "multi-rocket-core0-ctest-benchmark-embench_sglib-combined-baremetal.hex", + "multi-rocket-core0-ctest-benchmark-embench_slre-baremetal.hex", + "multi-rocket-core0-ctest-benchmark-embench_st-baremetal.hex", + "multi-rocket-core0-ctest-benchmark-embench_statemate-baremetal.hex", + "multi-rocket-core0-ctest-benchmark-embench_ud-baremetal.hex", + "multi-rocket-core0-ctest-benchmark-embench_wikisort-baremetal.hex", +] diff --git a/examples/designs/multi-rocket/sims/device/regression/p2e/workloads-linux.toml b/examples/designs/multi-rocket/sims/device/regression/p2e/workloads-linux.toml new file mode 100644 index 000000000..01747a69b --- /dev/null +++ b/examples/designs/multi-rocket/sims/device/regression/p2e/workloads-linux.toml @@ -0,0 +1,4 @@ +[workloads] +tests = [ + "fw_payload-linux.hex", +] diff --git a/examples/designs/multi-rocket/sims/device/regression/verilator/workloads-bare.toml b/examples/designs/multi-rocket/sims/device/regression/verilator/workloads-bare.toml new file mode 100644 index 000000000..024512457 --- /dev/null +++ b/examples/designs/multi-rocket/sims/device/regression/verilator/workloads-bare.toml @@ -0,0 +1,19 @@ +[workloads] +tests = [ + "multi-rocket-core0-ctest-design-coherent_exchange-multicore-baremetal", + "multi-rocket-core0-ctest-cpu_arithmetic-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_aha-mont64-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_crc32-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_edn-baremetal", + "multi-rocket-core0-ctest-check-embench_matmult-int-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_minver-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_nbody-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_nettle-aes-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_nettle-sha256-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_nsichneu-baremetal", + "multi-rocket-core0-ctest-check-embench_sglib-combined-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_slre-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_st-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_statemate-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_ud-baremetal", +] diff --git a/examples/designs/multi-rocket/sims/device/stack/CMakeLists.txt b/examples/designs/multi-rocket/sims/device/stack/CMakeLists.txt new file mode 100644 index 000000000..da9385d20 --- /dev/null +++ b/examples/designs/multi-rocket/sims/device/stack/CMakeLists.txt @@ -0,0 +1,37 @@ +# ===----------------------------------------------------------------------=== +# The software stack of the multi-rocket device simulation: every level the device +# needs, built bottom up. +# +# bbdev stack --build "--design multi-rocket --device " +# +# configures this with DESIGN_CMAKE, generated from design.pb. Each level is a +# CMake build of its own: cores and threads build buddy-opt plugins for the +# host, the device builds RISC-V libraries. +# core/rocket execute.o +# device runtime, runtime-baremetal, runtime-linux, engine; device.cmake +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(multi-rocket-device-stack NONE) +include(ExternalProject) +include("${DESIGN_CMAKE}") + +# The Rocket core. +ExternalProject_Add(core-rocket + SOURCE_DIR "${DESIGN_CORE_STACK_rocket}" + BINARY_DIR "${DESIGN_CORE_BUILD_rocket}" + CMAKE_GENERATOR Ninja + CMAKE_ARGS "-DDESIGN_CMAKE=${DESIGN_CMAKE}" -DCORE=rocket + BUILD_ALWAYS ON + INSTALL_COMMAND "") + +# The device: the runtime that schedules jobs onto the cores. +ExternalProject_Add(device + SOURCE_DIR "${DESIGN_DEVICE_STACK}" + BINARY_DIR "${DESIGN_DEVICE_BUILD}" + CMAKE_GENERATOR Ninja + CMAKE_ARGS "-DDESIGN_CMAKE=${DESIGN_CMAKE}" + -DCMAKE_C_COMPILER=riscv64-unknown-linux-gnu-gcc + -DCMAKE_CXX_COMPILER=riscv64-unknown-linux-gnu-g++ + DEPENDS core-rocket + BUILD_ALWAYS ON + INSTALL_COMMAND "") diff --git a/examples/designs/multi-rocket/sims/device/workloads/CMakeLists.txt b/examples/designs/multi-rocket/sims/device/workloads/CMakeLists.txt new file mode 100644 index 000000000..d606b9597 --- /dev/null +++ b/examples/designs/multi-rocket/sims/device/workloads/CMakeLists.txt @@ -0,0 +1,75 @@ +include("${DESIGN_DEVICE_BUILD}/device.cmake") + +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/base" + "${CMAKE_BINARY_DIR}/src/CTest/base" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/tlb" + "${CMAKE_BINARY_DIR}/src/CTest/tlb" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/tutorial" + "${CMAKE_BINARY_DIR}/src/CTest/tutorial" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/embench" + "${CMAKE_BINARY_DIR}/src/CTest/embench" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/coremark" + "${CMAKE_BINARY_DIR}/src/CTest/coremark" +) +foreach(BUCKYBALL_COMPONENT_WORKLOAD_TARGET IN LISTS DESIGN_CPU_TARGETS) + set(COMPONENT_WORKLOADS + "${DESIGN_TARGET_WORKLOADS_${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}}" + ) + if( + COMPONENT_WORKLOADS + STREQUAL + "${BUCKYBALL_REPO_ROOT}/examples/designs/${DESIGN_NAME}/workloads" + ) + continue() + endif() + set(BUCKYBALL_CTEST_TARGET "${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}") + set(BUCKYBALL_CTEST_GROUP_TARGET component-ctests-build) + if(IS_DIRECTORY "${COMPONENT_WORKLOADS}/ctests") + add_subdirectory( + "${COMPONENT_WORKLOADS}/ctests" + "${CMAKE_CURRENT_BINARY_DIR}/prefab/${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}/ctests" + ) + endif() +endforeach() +add_dependencies( + ctest-workloads-build + base-ctests-build + tlb-ctests-build + tutorial-ctests-build + component-ctests-build +) +add_dependencies(sync-native-bin embench-all coremark-all) + +add_subdirectory( + "${WORKLOAD_SRC_DIR}/MLIRTest/base" + "${CMAKE_BINARY_DIR}/src/MLIRTest/mlir_tests/base" +) +foreach(BUCKYBALL_TARGET IN LISTS DESIGN_CPU_TARGETS) + set(BUCKYBALL_MLIR_ACTIVE_TARGET "${BUCKYBALL_TARGET}") + foreach(BUCKYBALL_BALL_DIR IN LISTS DESIGN_PROFILES_${BUCKYBALL_TARGET}_THREAD_BALLDOMAIN_BALLS) + set(BALL_MLIR_TESTS + "${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/${BUCKYBALL_BALL_DIR}/workloads/mlir_tests" + ) + if(IS_DIRECTORY "${BALL_MLIR_TESTS}") + add_subdirectory( + "${BALL_MLIR_TESTS}" + "${CMAKE_CURRENT_BINARY_DIR}/balls/${BUCKYBALL_TARGET}/${BUCKYBALL_BALL_DIR}" + ) + endif() + endforeach() +endforeach() +add_dependencies(MLIRTest-all base-mlirtest-build balls-mlir-tests-build) + +add_subdirectory( + "${BUCKYBALL_REPO_ROOT}/examples/designs/multi-rocket/workloads" + design +) diff --git a/examples/designs/multi-rocket/sims/device/workloads/kernel/CMakeLists.txt b/examples/designs/multi-rocket/sims/device/workloads/kernel/CMakeLists.txt new file mode 100644 index 000000000..8bb5ab79d --- /dev/null +++ b/examples/designs/multi-rocket/sims/device/workloads/kernel/CMakeLists.txt @@ -0,0 +1,4 @@ +# Linux programs installed into the multi-rocket rootfs, each with the hart it runs on. +set(BUCKYBALL_KERNEL_PROGRAMS + 0 multi-rocket-core0-ctest-cpu_arithmetic-linux + PARENT_SCOPE) diff --git a/examples/designs/multi-rocket/sims/ip/uvm/registry.toml b/examples/designs/multi-rocket/sims/ip/uvm/registry.toml new file mode 100644 index 000000000..8e8131449 --- /dev/null +++ b/examples/designs/multi-rocket/sims/ip/uvm/registry.toml @@ -0,0 +1 @@ +ips = [] diff --git a/examples/designs/multi-rocket/stack/device/CMakeLists.txt b/examples/designs/multi-rocket/stack/device/CMakeLists.txt new file mode 100644 index 000000000..bf1118f54 --- /dev/null +++ b/examples/designs/multi-rocket/stack/device/CMakeLists.txt @@ -0,0 +1,32 @@ +# ===----------------------------------------------------------------------=== +# The multi-rocket device: Rocket cores without Balls or hardware threads, and +# the runtime that schedules jobs onto them. +# +# bbdev stack --build "--design multi-rocket --device " +# +# builds every core stack of the device first, then configures this with +# DESIGN_CMAKE, generated from design.pb, and the RISC-V Linux toolchain. +# Products, exported in device.cmake for the programs that run on the device: +# runtime, runtime-baremetal, runtime-linux job scheduling over the cores +# engine instruction streams (no threads here) +# bindings how each core runs jobs +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(multi-rocket-device LANGUAGES C CXX) +include("${DESIGN_CMAKE}") +set(CMAKE_POSITION_INDEPENDENT_CODE ON) + +# How each core runs jobs. Nothing references these bindings, so they are +# linked as objects rather than taken from an archive; as link options they +# come before the libraries they call. +add_library(bindings INTERFACE) +foreach(CORE IN LISTS DESIGN_CORES) + target_link_options(bindings INTERFACE "${DESIGN_CORE_BUILD_${CORE}}/execute.o") +endforeach() +# The device has no banks, so no DMA helpers. +set(RUNTIME_DEFS) + +add_subdirectory("${DESIGN_REPO}/stack/engine" engine) +add_subdirectory("${DESIGN_REPO}/stack/runtime" runtime) +export(TARGETS bindings runtime runtime-baremetal runtime-linux engine + runtime-shared runtime-linux-shared engine-shared FILE device.cmake) diff --git a/examples/designs/multi-rocket/workloads/CMakeLists.txt b/examples/designs/multi-rocket/workloads/CMakeLists.txt new file mode 100644 index 000000000..af0c9633a --- /dev/null +++ b/examples/designs/multi-rocket/workloads/CMakeLists.txt @@ -0,0 +1,5 @@ +add_custom_target(design-ctests-build) +add_subdirectory(device) + +add_custom_target(design-workloads-build ALL + DEPENDS design-ctests-build mlirtest-workloads-build soctests-build) diff --git a/examples/designs/multi-rocket/workloads/configs/tests.toml b/examples/designs/multi-rocket/workloads/configs/tests.toml new file mode 100644 index 000000000..c1c7ceb33 --- /dev/null +++ b/examples/designs/multi-rocket/workloads/configs/tests.toml @@ -0,0 +1,2 @@ +controlHart = 0 +reportHart = 0 diff --git a/examples/designs/multi-rocket/workloads/device/CMakeLists.txt b/examples/designs/multi-rocket/workloads/device/CMakeLists.txt new file mode 100644 index 000000000..679d879e4 --- /dev/null +++ b/examples/designs/multi-rocket/workloads/device/CMakeLists.txt @@ -0,0 +1,2 @@ +add_subdirectory(soc) +add_subdirectory(core) diff --git a/examples/designs/multi-rocket/workloads/device/core/CMakeLists.txt b/examples/designs/multi-rocket/workloads/device/core/CMakeLists.txt new file mode 100644 index 000000000..e90e2fc5a --- /dev/null +++ b/examples/designs/multi-rocket/workloads/device/core/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(ctests) diff --git a/examples/designs/multi-rocket/workloads/device/core/ctests/CMakeLists.txt b/examples/designs/multi-rocket/workloads/device/core/ctests/CMakeLists.txt new file mode 100644 index 000000000..34e4852d3 --- /dev/null +++ b/examples/designs/multi-rocket/workloads/device/core/ctests/CMakeLists.txt @@ -0,0 +1,20 @@ +set(CPU_ARITHMETIC + "${DESIGN_NAME}-${BUCKYBALL_CONTROL_TARGET}-ctest-cpu_arithmetic" +) +set(CPU_ELF ${CMAKE_CURRENT_BINARY_DIR}/${CPU_ARITHMETIC}-baremetal) +add_custom_command( + OUTPUT ${CPU_ELF} + COMMAND + ${ELF_CC} ${BUCKYBALL_CTEST_C_FLAGS} -DCORE_HART_ID=${DESIGN_WORKLOAD_CONTROL_HART} + ${BAREMETAL_DIR}/crt0.S ${BAREMETAL_DIR}/syscalls.c + ${CMAKE_CURRENT_SOURCE_DIR}/cpu_arithmetic.c -o ${CPU_ELF} + DEPENDS + cpu_arithmetic.c + ${BAREMETAL_DIR}/crt0.S + ${BAREMETAL_DIR}/syscalls.c + ${BBSIM_LD} +) +add_custom_target(${CPU_ARITHMETIC}-baremetal-build DEPENDS ${CPU_ELF}) +add_dependencies(design-ctests-build ${CPU_ARITHMETIC}-baremetal-build) +add_executable(${CPU_ARITHMETIC}-linux cpu_arithmetic.c) +add_dependencies(design-ctests-build ${CPU_ARITHMETIC}-linux) diff --git a/examples/chips/multi-rocket/workloads/cpu_arithmetic.c b/examples/designs/multi-rocket/workloads/device/core/ctests/cpu_arithmetic.c similarity index 100% rename from examples/chips/multi-rocket/workloads/cpu_arithmetic.c rename to examples/designs/multi-rocket/workloads/device/core/ctests/cpu_arithmetic.c diff --git a/examples/designs/multi-rocket/workloads/device/soc/CMakeLists.txt b/examples/designs/multi-rocket/workloads/device/soc/CMakeLists.txt new file mode 100644 index 000000000..9d50fc377 --- /dev/null +++ b/examples/designs/multi-rocket/workloads/device/soc/CMakeLists.txt @@ -0,0 +1,13 @@ +set(COHERENT_ELF ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-${BUCKYBALL_CONTROL_TARGET}-ctest-design-coherent_exchange-multicore-baremetal) +add_custom_command(OUTPUT ${COHERENT_ELF} + COMMAND ${ELF_CC} -O2 -march=rv64gc -mcmodel=medany -msmall-data-limit=0 + -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax -Wl,-T,${BBSIM_LD} + ${DESIGN_DEFINES} -I${RUNTIME_INCLUDE_DIR} + ${BAREMETAL_DIR}/start.S ${CMAKE_CURRENT_SOURCE_DIR}/coherent_exchange.c + -lgcc -o ${COHERENT_ELF} + DEPENDS coherent_exchange.c ${BAREMETAL_DIR}/start.S ${BBSIM_LD} + ${RUNTIME_INCLUDE_DIR}/bb_cores.h + VERBATIM) +add_custom_target(coherent-exchange-build DEPENDS ${COHERENT_ELF}) +add_dependencies(design-ctests-build coherent-exchange-build) +add_dependencies(soctests-build coherent-exchange-build) diff --git a/examples/designs/multi-rocket/workloads/device/soc/coherent_exchange.c b/examples/designs/multi-rocket/workloads/device/soc/coherent_exchange.c new file mode 100644 index 000000000..a1a106c6e --- /dev/null +++ b/examples/designs/multi-rocket/workloads/device/soc/coherent_exchange.c @@ -0,0 +1,31 @@ +#include +#include + +static uint64_t payload[BB_NUM_CORES][8] __attribute__((aligned(64))); +static unsigned arrived[2] __attribute__((aligned(64))); +static unsigned consumed[2] __attribute__((aligned(64))); +static unsigned failed; + +int main(void) { + uintptr_t hart; + asm volatile("csrr %0, mhartid" : "=r"(hart)); + unsigned slot = bb_hart_slot(hart); + for (unsigned round = 0; round < 2; ++round) { + uint64_t value = 0x5a000000ULL + round * 0x10000 + slot * 9; + payload[slot][0] = value; + payload[slot][7] = ~value; + __atomic_fetch_add(&arrived[round], 1, __ATOMIC_ACQ_REL); + while (__atomic_load_n(&arrived[round], __ATOMIC_ACQUIRE) != BB_NUM_CORES) { + } + for (unsigned peer = 0; peer < BB_NUM_CORES; ++peer) { + uint64_t expected = 0x5a000000ULL + round * 0x10000 + peer * 9; + if (payload[peer][0] != expected || payload[peer][7] != ~expected) + __atomic_fetch_or(&failed, 1, __ATOMIC_RELAXED); + } + __atomic_fetch_add(&consumed[round], 1, __ATOMIC_ACQ_REL); + while (__atomic_load_n(&consumed[round], __ATOMIC_ACQUIRE) != + BB_NUM_CORES) { + } + } + return __atomic_load_n(&failed, __ATOMIC_ACQUIRE); +} diff --git a/examples/designs/pebble/arch/build.sc b/examples/designs/pebble/arch/build.sc new file mode 100644 index 000000000..9c69932e4 --- /dev/null +++ b/examples/designs/pebble/arch/build.sc @@ -0,0 +1,18 @@ +import mill._ +import mill.scalalib._ +import $file.^.^.^.^.arch.src.main.scala.framework.device.cbus.{build => cbus_build} +import $file.^.^.^.{build => examples} + +val designRoot: os.Path = millSourcePath / os.up + +object pebble extends SbtModule { + override def generatedSources = T(cbus_build.registry(sources(), T.dest)) + override def millSourcePath = designRoot + override def forkWorkingDir = T(designRoot) + override def scalaVersion = "2.13.16" + override def moduleDeps = Seq(examples.library) + override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") + override def sources = T.sources(designRoot / "arch") +} diff --git a/examples/designs/pebble/arch/device/DeviceTop.scala b/examples/designs/pebble/arch/device/DeviceTop.scala new file mode 100644 index 000000000..25400c6ae --- /dev/null +++ b/examples/designs/pebble/arch/device/DeviceTop.scala @@ -0,0 +1,73 @@ +package examples.designs.pebble + +import framework.lib.arbiter.{RRArbiter} + +import _root_.sims.configs.SimParam +import chisel3._ +import chisel3.util._ +import framework.device.peripherals.bootrom.BootRomLines +import framework.device.peripherals.Controller +import framework.ip.bus.axi4.Decoder +import framework.ip.bus.chi.snf.LineResponse +import framework.lib.queue.{Queue => ResponseQueue} +import chisel3.experimental.hierarchy.{instantiable, Instance, Instantiate} +import framework.device.configs.DeviceParam +import framework.device.cbus.CBus +import framework.device.cbus.generated.CoreComponents +import buckyball.config.Design +import _root_.configs.GlobalConfig +import framework.core.clink.CLinkIO +import framework.device.dlink.BaseDevice + +@instantiable +class DeviceTop(b: GlobalConfig, design: Design) extends BaseDevice(b.device.ddr.axi) { + val devicep = b.device + + val Seq(corep) = devicep.cores + require(corep.mtimeAddress == devicep.clint.base + 0xbff8, "Core mtime address must match CLINT") + + val bus = Instantiate(new CBus(b.device, CBus.generators(design, b.device, b.sim, CoreComponents.factories))) + dlink <> bus.dlink + + val peripherals = + Instantiate(new Controller(devicep, devicep.ddr.axi, corep.memory.memoryAccess.memory)) + + val peripheralBus = Instantiate(new Decoder( + Seq( + (devicep.clint.base, devicep.clint.bytes), + (devicep.plic.base, devicep.plic.bytes), + (devicep.scu.baseAddress, devicep.scu.totalSizeBytes) + ), + devicep.ddr.axi + )) + + peripheralBus.io.in <> bus.uncached + peripherals.io.axi <> peripheralBus.io.matched + bus.external <> peripheralBus.io.other + peripherals.io.hardwareInterrupt.sources := 0.U + bus.interrupts := peripherals.io.hardwareInterrupt.harts + + val romParams = devicep.bootrom + val rom = Instantiate(new BootRomLines(devicep.bootrom, corep.memory.home.memory.chi)) + + val backing = bus.backing(0) + val ddr = bus.memory(0) + val inRom = backing.req.bits.addr >= romParams.base.U && + backing.req.bits.addr < (romParams.base + romParams.bytes).U + rom.io.request.valid := backing.req.valid && inRom + rom.io.request.bits := backing.req.bits + ddr.req.valid := backing.req.valid && !inRom + ddr.req.bits := backing.req.bits + backing.req.ready := Mux(inRom, rom.io.request.ready, ddr.req.ready) + + val responses = Instantiate(new RRArbiter(new LineResponse(corep.memory.home.memory.chi), 2)) + responses.io.in(0) <> rom.io.response + responses.io.in(1) <> ddr.resp + + // Buffer the selected answer so downstream backpressure cannot change its source. + val response = Instantiate(new ResponseQueue(new LineResponse(corep.memory.home.memory.chi), 1, false, false)) + + response.io.enq <> responses.io.out + backing.resp <> response.io.deq + +} diff --git a/examples/designs/pebble/arch/device/core/PebbleCore.scala b/examples/designs/pebble/arch/device/core/PebbleCore.scala new file mode 100644 index 000000000..329a3841d --- /dev/null +++ b/examples/designs/pebble/arch/device/core/PebbleCore.scala @@ -0,0 +1,46 @@ +package examples.designs.pebble + +import sims.configs.SimParam +import framework.device.cbus.CoreFactory +import buckyball.config.{CorePlacement, Design} + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.core.configs.CoreParam +import framework.core.clink.{CLinkIO, HasCLink} + +/** One RV64GC hart and its local NPU, explicitly wired behind CLink. */ +@instantiable +class PebbleCore(corep: CoreParam, simp: SimParam) extends Module with HasCLink { + val diffTest = simp.diffTest + require(corep.threads.isEmpty && corep.acc.nonEmpty, "PebbleCore connects a local ACC without TBus") + require(corep.clink.harts == 1 && corep.clink.backing) + + @public + val clink = IO(new CLinkIO(corep.clink)) + + val loom: Instance[Loom] = Instantiate(new Loom(corep)) + val acc: Instance[Acc] = Instantiate(new Acc(corep, corep.acc.get, simp)) + + loom.io.rocc <> acc.io.rocc + loom.io.memory <> acc.io.memory + loom.io.interrupts := clink.interrupts(0) + clink.memory.uncached <> loom.io.uncached + clink.memory.backing.get <> loom.io.backing + clink.memory.ddr <> acc.io.ddr +} + +object Pebble extends CoreFactory { + val component = "pebble" + def parameters(design: Design, source: CorePlacement): CoreParam = + CoreParam.fromProto(design, source, externalHome = false) + + def instantiate( + source: CorePlacement, + corep: CoreParam, + simp: SimParam, + traceIds: Seq[Int] + ): CLinkIO = + Instantiate(new PebbleCore(corep, simp)).clink + +} diff --git a/examples/designs/pebble/arch/device/core/acc.scala b/examples/designs/pebble/arch/device/core/acc.scala new file mode 100644 index 000000000..28f64b8ac --- /dev/null +++ b/examples/designs/pebble/arch/device/core/acc.scala @@ -0,0 +1,113 @@ +package examples.designs.pebble + +import framework.acc.balldomain.bbus.generated.BallComponents +import sims.configs.SimParam +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.core.configs.CoreParam +import framework.core.memory.MemoryPort +import framework.acc.frontend.RoCCIO +import framework.acc.configs.AccParam +import framework.acc.frontend.Frontend +import framework.acc.memdomain.MemDomain +import framework.acc.debug.Debug +import framework.ip.bus.axi4 +import framework.acc.balldomain.BallDomain +import framework.acc.balldomain.bbus.BBus + +/** Pebble accelerator domains, concrete Balls and their local debug wiring. */ +@instantiable +class Acc(corep: CoreParam, accp: AccParam, simp: SimParam) extends Module { + val diffTest = simp.diffTest + require(!accp.memDomain.sharedEnable && !accp.gpDomain.enable) + + @public + val io = IO(new Bundle { + val rocc = new RoCCIO(corep.integer.xLen, corep.privileged.permission) + val memory = new MemoryPort(corep.memory.dcache.tracking, corep.memory.memoryAccess.walker) + val ddr = new axi4.Port(accp.transport.axi) + }) + + val frontend: Instance[Frontend] = Instantiate(new Frontend( + accp.frontend, + accp.memDomain, + accp.gpDomain, + accp.ballDomain, + corep.privileged.permission, + simp + )) + + val memDomain: Instance[MemDomain] = Instantiate(new MemDomain( + accp.frontend, + accp.memDomain, + accp.gpDomain, + accp.transport, + corep.privileged.permission, + corep.memory.dcache.tracking, + corep.memory.memoryAccess.walker, + simp + )) + + val ballDomain: Instance[BallDomain] = Instantiate(new BallDomain( + accp.frontend, + accp.memDomain, + accp.gpDomain, + accp.ballDomain, + corep.privileged.permission, + BallComponents.factories + )) + + frontend.io.rocc <> io.rocc + io.memory <> memDomain.io.memory + + // Local NPU domains are visible here; + // BBUS owns concrete Ball instances. + val debug: Instance[Debug] = Instantiate(new Debug( + accp.frontend, + accp.memDomain, + accp.gpDomain, + accp.ballDomain, + simp, + traceId = corep.hartId, + sharedOwnerId = corep.hartId + )) + + debug.io.ant.valid := false.B + debug.io.ant.bits := 0.U.asTypeOf(debug.io.ant.bits) + debug.io.acc.frontend <> frontend.io.debug + debug.io.acc.trace.memory := memDomain.io.traceMemory + debug.io.acc.trace.ballStarted := ballDomain.io.traceStarted + debug.io.acc.trace.ballCompleted := ballDomain.io.traceCompleted + debug.io.acc.banks.zip(memDomain.io.bank_hashes).foreach { case (sink, source) => sink := source } + + ballDomain.io.issue <> frontend.io.ballRequest + frontend.io.ballCompletion <> ballDomain.io.complete + memDomain.io.ball <> ballDomain.io.memory + memDomain.io.issue <> frontend.io.memoryRequest + frontend.io.memoryCompletion <> memDomain.io.complete + memDomain.io.banks <> ballDomain.io.banks + memDomain.io.ballLayoutRequest <> ballDomain.io.layoutRequest + ballDomain.io.layoutResponse <> memDomain.io.ballLayoutResponse + memDomain.io.mapping <> frontend.io.mappingUpdate + frontend.io.barrier.release := frontend.io.barrier.arrive + io.ddr <> memDomain.io.ddr + + // Pebble has no shared-memory connection; + // its complete data path is local. + for (port <- memDomain.io.shared.bank.requests) { + port.read.req.ready := false.B + port.read.resp.valid := false.B + port.read.resp.bits := 0.U.asTypeOf(port.read.resp.bits) + port.write.req.ready := false.B + port.write.resp.valid := false.B + port.write.resp.bits := 0.U.asTypeOf(port.write.resp.bits) + } + memDomain.io.shared.bank.config.ready := false.B + memDomain.io.shared.bank.mapping.ready := false.B + memDomain.io.shared.bank.local.request.valid := false.B + memDomain.io.shared.bank.local.request.bits := 0.U.asTypeOf(memDomain.io.shared.bank.local.request.bits) + memDomain.io.shared.bank.local.response.ready := false.B + memDomain.io.shared.move.command.ready := false.B + memDomain.io.shared.move.completion.valid := false.B + memDomain.io.shared.move.completion.bits := false.B +} diff --git a/examples/designs/pebble/arch/device/core/loom.scala b/examples/designs/pebble/arch/device/core/loom.scala new file mode 100644 index 000000000..277912502 --- /dev/null +++ b/examples/designs/pebble/arch/device/core/loom.scala @@ -0,0 +1,51 @@ +package examples.designs.pebble + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.core.configs.CoreParam +import framework.core.execution.Execution +import framework.core.memory.{MemoryPort, MemorySystem} +import framework.acc.frontend.RoCCIO +import framework.core.clink.CoreInterrupts +import framework.ip.bus.axi4 +import framework.ip.bus.chi.snf.{LineRequest, LineResponse} + +/** Outer Loom execution core and private memory hierarchy; directly controls the local ACC. */ +@instantiable +class Loom(corep: CoreParam) extends Module { + + @public + val io = IO(new Bundle { + val interrupts = Input(new CoreInterrupts) + val rocc = Flipped(new RoCCIO(corep.integer.xLen, corep.privileged.permission)) + val memory = Flipped(new MemoryPort(corep.memory.dcache.tracking, corep.memory.memoryAccess.walker)) + val uncached = new axi4.Port(corep.memory.uncached.axi) + + val backing = new Bundle { + val request = Decoupled(new LineRequest(corep.memory.home.memory.chi)) + val response = Flipped(Decoupled(new LineResponse(corep.memory.home.memory.chi))) + } + + }) + + val execution: Instance[Execution] = Instantiate(new Execution( + corep.execution(directAccelerator = true), + localRocc = true + )) + + val memory: Instance[MemorySystem] = Instantiate(new MemorySystem(corep.memory, externalHome = false)) + + execution.io.interrupts := io.interrupts + execution.io.rocc.get <> io.rocc + execution.io.workDrained := !io.rocc.busy + memory.io.fetch <> execution.io.fetch + memory.io.memory <> execution.io.data + memory.io.timeRequest <> execution.io.timeRequest + execution.io.timeResponse <> memory.io.timeResponse + memory.io.commandAllowed := !io.rocc.busy + memory.io.services <> io.memory + io.uncached <> memory.io.uncached + io.backing <> memory.io.backing.get + +} diff --git a/examples/designs/pebble/configs/design.toml b/examples/designs/pebble/configs/design.toml new file mode 100644 index 000000000..1d45ec7e4 --- /dev/null +++ b/examples/designs/pebble/configs/design.toml @@ -0,0 +1,5 @@ +[device] +include = "device/default.toml" + +[sims] +include = "sims/default.toml" diff --git a/examples/designs/pebble/configs/device/core/acc/ablation-bat0-blt0.toml b/examples/designs/pebble/configs/device/core/acc/ablation-bat0-blt0.toml new file mode 100644 index 000000000..269b93083 --- /dev/null +++ b/examples/designs/pebble/configs/device/core/acc/ablation-bat0-blt0.toml @@ -0,0 +1,8 @@ +source = "../../../.." +# Pebble accelerator profile + +balldomain = "balldomain/default.toml" +memdomain = "memdomain/blt-off.toml" +frontend = "frontend/bat-off-rob24.toml" +gpdomain = "default" +transport = "default" diff --git a/examples/designs/pebble/configs/device/core/acc/ablation-bat0-blt1.toml b/examples/designs/pebble/configs/device/core/acc/ablation-bat0-blt1.toml new file mode 100644 index 000000000..6f14a5454 --- /dev/null +++ b/examples/designs/pebble/configs/device/core/acc/ablation-bat0-blt1.toml @@ -0,0 +1,8 @@ +source = "../../../.." +# Pebble accelerator profile + +balldomain = "balldomain/default.toml" +memdomain = "default" +frontend = "frontend/bat-off-rob24.toml" +gpdomain = "default" +transport = "default" diff --git a/examples/designs/pebble/configs/device/core/acc/ablation-bat1-blt0.toml b/examples/designs/pebble/configs/device/core/acc/ablation-bat1-blt0.toml new file mode 100644 index 000000000..c746a0d40 --- /dev/null +++ b/examples/designs/pebble/configs/device/core/acc/ablation-bat1-blt0.toml @@ -0,0 +1,8 @@ +source = "../../../.." +# Pebble accelerator profile + +balldomain = "balldomain/default.toml" +memdomain = "memdomain/blt-off.toml" +frontend = "frontend/rob24.toml" +gpdomain = "default" +transport = "default" diff --git a/examples/cores/pebble/configs/balldomains/balls/smatmul/8x16.toml b/examples/designs/pebble/configs/device/core/acc/balldomain/balls/smatmul/8x16.toml similarity index 90% rename from examples/cores/pebble/configs/balldomains/balls/smatmul/8x16.toml rename to examples/designs/pebble/configs/device/core/acc/balldomain/balls/smatmul/8x16.toml index 93333ca8e..fa9a5342c 100644 --- a/examples/cores/pebble/configs/balldomains/balls/smatmul/8x16.toml +++ b/examples/designs/pebble/configs/device/core/acc/balldomain/balls/smatmul/8x16.toml @@ -1,5 +1,4 @@ # Pebble SMatMul Ball Configuration -[ball] tileRows = 8 tileCols = 16 diff --git a/examples/designs/pebble/configs/device/core/acc/balldomain/default.toml b/examples/designs/pebble/configs/device/core/acc/balldomain/default.toml new file mode 100644 index 000000000..f68c803b7 --- /dev/null +++ b/examples/designs/pebble/configs/device/core/acc/balldomain/default.toml @@ -0,0 +1,102 @@ +[[balls]] +id = 0 +name = "TransposeBall" +component = "transpose" +param = "default" +port = { inBW = 1, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "TRANSPOSE", funct7 = 49 }, +] + +[[balls]] +id = 1 +name = "SMatMulBall" +component = "smatmul" +param = "balls/smatmul/8x16.toml" +port = { inBW = 2, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "SMATMUL_OS", funct7 = 65 }, + { mnemonic = "SMATMUL_BIAS", funct7 = 17 }, +] + +[[balls]] +id = 2 +name = "Im2colBall" +component = "im2col" +param = "default" +port = { inBW = 1, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "IM2COL", funct7 = 48 }, +] + +[[balls]] +id = 3 +name = "ToInt8Ball" +component = "toint8" +param = "default" +port = { inBW = 1, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "QUANT_F32_TO_I8", funct7 = 51 }, + { mnemonic = "QUANT_I32_TO_I8", funct7 = 67 }, +] + +[[balls]] +id = 4 +name = "Int2FpBall" +component = "int2fp" +param = "default" +port = { inBW = 2, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "INT32_TO_FP32", funct7 = 68 }, +] + +[[balls]] +id = 5 +name = "LutBall" +component = "lut" +param = "default" +port = { inBW = 2, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "LUT", funct7 = 66 }, +] + +[[balls]] +id = 6 +name = "MaxPoolBall" +component = "maxpool" +param = "default" +port = { inBW = 1, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "MAXPOOL", funct7 = 50 }, +] + +[[balls]] +id = 7 +name = "Int8AddBall" +component = "int8add" +param = "default" +port = { inBW = 2, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "INT8ADD", funct7 = 69 }, + { mnemonic = "INT8ADD_RELU", funct7 = 70 }, +] + +[[balls]] +id = 8 +name = "Int8MulBall" +component = "int8mul" +param = "default" +port = { inBW = 2, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "INT8MUL", funct7 = 71 }, +] + +[[balls]] +id = 9 +name = "MatAddBall" +component = "matadd" +param = "default" +port = { inBW = 2, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "MATADD", funct7 = 72 }, +] diff --git a/examples/designs/pebble/configs/device/core/acc/default.toml b/examples/designs/pebble/configs/device/core/acc/default.toml new file mode 100644 index 000000000..24ccccdcc --- /dev/null +++ b/examples/designs/pebble/configs/device/core/acc/default.toml @@ -0,0 +1,8 @@ +source = "../../../.." +# Pebble accelerator profile + +balldomain = "balldomain/default.toml" +memdomain = "default" +frontend = "default" +gpdomain = "default" +transport = "default" diff --git a/examples/designs/pebble/configs/device/core/acc/frontend/bat-off-rob24.toml b/examples/designs/pebble/configs/device/core/acc/frontend/bat-off-rob24.toml new file mode 100644 index 000000000..e14dee6d0 --- /dev/null +++ b/examples/designs/pebble/configs/device/core/acc/frontend/bat-off-rob24.toml @@ -0,0 +1,9 @@ +bankAliasEnable = false +robEntries = 24 +rsOutOfOrderResponse = true +bankIdLen = 10 +maxPrivateVbank = 23 +sharedVbankBase = 32 +iterLen = 34 +subRobEnable = false +subRobDepth = 64 diff --git a/examples/designs/pebble/configs/device/core/acc/frontend/rob24.toml b/examples/designs/pebble/configs/device/core/acc/frontend/rob24.toml new file mode 100644 index 000000000..ca2defa54 --- /dev/null +++ b/examples/designs/pebble/configs/device/core/acc/frontend/rob24.toml @@ -0,0 +1,9 @@ +bankAliasEnable = true +robEntries = 24 +rsOutOfOrderResponse = true +bankIdLen = 10 +maxPrivateVbank = 23 +sharedVbankBase = 32 +iterLen = 34 +subRobEnable = false +subRobDepth = 64 diff --git a/examples/designs/pebble/configs/device/core/acc/memdomain/blt-off.toml b/examples/designs/pebble/configs/device/core/acc/memdomain/blt-off.toml new file mode 100644 index 000000000..faade6197 --- /dev/null +++ b/examples/designs/pebble/configs/device/core/acc/memdomain/blt-off.toml @@ -0,0 +1,21 @@ +# Pebble thread memory domain configuration + +addrLen = 39 + +[bank] +layoutEnable = false +num = 24 +width = 128 +entries = 512 +maskLen = 16 +requestPorts = 13 + +[dma] +busWidth = 128 + +[mmio] +enable = false +bankNum = 5 +bankEntries = 1024 +bankWidth = 8 +readWidth = 8 diff --git a/examples/designs/pebble/configs/device/core/acc/rob24.toml b/examples/designs/pebble/configs/device/core/acc/rob24.toml new file mode 100644 index 000000000..183297aff --- /dev/null +++ b/examples/designs/pebble/configs/device/core/acc/rob24.toml @@ -0,0 +1,8 @@ +source = "../../../.." +# Pebble accelerator profile + +balldomain = "balldomain/default.toml" +memdomain = "default" +frontend = "frontend/rob24.toml" +gpdomain = "default" +transport = "default" diff --git a/examples/designs/pebble/configs/device/core/c2c/pebble.toml b/examples/designs/pebble/configs/device/core/c2c/pebble.toml new file mode 100644 index 000000000..00bd5f8b4 --- /dev/null +++ b/examples/designs/pebble/configs/device/core/c2c/pebble.toml @@ -0,0 +1,3 @@ +enabled = false +peerCores = 9 +axi = "default" diff --git a/examples/designs/pebble/configs/device/core/default.toml b/examples/designs/pebble/configs/device/core/default.toml new file mode 100644 index 000000000..c07169d55 --- /dev/null +++ b/examples/designs/pebble/configs/device/core/default.toml @@ -0,0 +1,35 @@ +component = "pebble" +accelerator = "acc/default.toml" +threads = [] +hartId = 0 +resetVector = 0x10000 +mtimeAddress = 0x200bff8 +memory = "memory/pebble.toml" +fetch = "default" +fpu = "default" +interconnect = "interconnect/pebble.toml" +c2c = "c2c/pebble.toml" +controller = "default" + +[param] +threadDataBytes = 64 +xLen = 64 +vaddrBits = 39 +paddrBits = 36 +pgIdxBits = 12 +pgLevels = 3 +nPMPs = 16 + +[sharedMem] +enable = false +bankWidth = 128 +bankEntries = 1024 +virtualBankCount = 24 +entries = 16384 +inputChannels = 1 +defaultGroupCount = 1 +latency = 10 + +[compiler] +name = "pebble" +component = "pebble" diff --git a/examples/designs/pebble/configs/device/core/interconnect/pebble.toml b/examples/designs/pebble/configs/device/core/interconnect/pebble.toml new file mode 100644 index 000000000..13d04a5a6 --- /dev/null +++ b/examples/designs/pebble/configs/device/core/interconnect/pebble.toml @@ -0,0 +1,2 @@ +axi = "default" +slots = 16 diff --git a/examples/designs/pebble/configs/device/core/memory/access/pebble.toml b/examples/designs/pebble/configs/device/core/memory/access/pebble.toml new file mode 100644 index 000000000..2e90444bc --- /dev/null +++ b/examples/designs/pebble/configs/device/core/memory/access/pebble.toml @@ -0,0 +1,52 @@ +translationEntries = 16 +tagBits = 6 +resultLineBits = 0 +[[regions]] +base = 0x80000000 +bytes = 0x20000000 +cacheable = true +executable = true +readable = true +writable = true +atomic = true +normal = true + +[[regions]] +base = 0x10000 +bytes = 0x10000 +cacheable = true +executable = true +readable = true +writable = false +atomic = false +normal = true + +[[regions]] +base = 0x2000000 +bytes = 0x10000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0xc000000 +bytes = 0x4000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0x60000000 +bytes = 0x10000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false diff --git a/examples/designs/pebble/configs/device/core/memory/dcache/pebble.toml b/examples/designs/pebble/configs/device/core/memory/dcache/pebble.toml new file mode 100644 index 000000000..4fb4beb6f --- /dev/null +++ b/examples/designs/pebble/configs/device/core/memory/dcache/pebble.toml @@ -0,0 +1,15 @@ +[tracking] +entries = 8 +addressBits = 44 +idBits = 8 +lineBytes = 64 +maxRanges = 8 + +[cache] +nodeId = 2 +lines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/pebble/configs/device/core/memory/home/pebble.toml b/examples/designs/pebble/configs/device/core/memory/home/pebble.toml new file mode 100644 index 000000000..aa39cd52e --- /dev/null +++ b/examples/designs/pebble/configs/device/core/memory/home/pebble.toml @@ -0,0 +1,13 @@ +agents = 2 +mshrEntries = 8 +nodeId = 64 +requesters = "default" + +[cache] +addressBits = 44 +lineBytes = 64 +sets = 64 +ways = 4 +idBits = 8 +metadataBits = 4 +responseDepth = 2 diff --git a/examples/designs/pebble/configs/device/core/memory/icache/pebble.toml b/examples/designs/pebble/configs/device/core/memory/icache/pebble.toml new file mode 100644 index 000000000..3cc7df83a --- /dev/null +++ b/examples/designs/pebble/configs/device/core/memory/icache/pebble.toml @@ -0,0 +1,7 @@ +nodeId = 1 +lines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/pebble/configs/device/core/memory/pebble.toml b/examples/designs/pebble/configs/device/core/memory/pebble.toml new file mode 100644 index 000000000..fb90c6584 --- /dev/null +++ b/examples/designs/pebble/configs/device/core/memory/pebble.toml @@ -0,0 +1,5 @@ +uncached = "default" +access = "access/pebble.toml" +icache = "icache/pebble.toml" +dcache = "dcache/pebble.toml" +home = "home/pebble.toml" diff --git a/examples/designs/pebble/configs/device/ddr/pebble.toml b/examples/designs/pebble/configs/device/ddr/pebble.toml new file mode 100644 index 000000000..31815e9e1 --- /dev/null +++ b/examples/designs/pebble/configs/device/ddr/pebble.toml @@ -0,0 +1,13 @@ +clients = 1 +slotsPerClient = 16 +dataBits = 128 +idBits = 4 +idBase = 0 +base = 0x80000000 +bytes = 0x20000000 +model = 0 + +[line] +nodeIdBits = 7 +addressBits = 44 +dataBits = 256 diff --git a/examples/designs/pebble/configs/device/default.toml b/examples/designs/pebble/configs/device/default.toml new file mode 100644 index 000000000..b83f276f2 --- /dev/null +++ b/examples/designs/pebble/configs/device/default.toml @@ -0,0 +1,8 @@ +ddr = "ddr/pebble.toml" +bootrom = "default" +clint = "default" +plic = "default" +scu = "default" + +[[cores]] +include = "core/default.toml" diff --git a/examples/designs/pebble/configs/sims/default.toml b/examples/designs/pebble/configs/sims/default.toml new file mode 100644 index 000000000..b4ae21f73 --- /dev/null +++ b/examples/designs/pebble/configs/sims/default.toml @@ -0,0 +1,18 @@ +diffTest = false + +[memory] +lineBytes = 64 +clockHz = 1000000000 + +[targets.device] +config = "examples.designs.pebble.sims.device.DeviceConfig" +stack_name = "pebble" +regression = "sims/device/regression" +kernel = "sims/device/workloads/kernel" +tapeout = "sims/device/tapeout" + +[targets.device.configs] +PebbleVerilatorConfig = "verilator.toml" + +[verification] +registry = "../../sims/ip/uvm/registry.toml" diff --git a/examples/designs/pebble/configs/sims/verilator.toml b/examples/designs/pebble/configs/sims/verilator.toml new file mode 100644 index 000000000..bc1d59673 --- /dev/null +++ b/examples/designs/pebble/configs/sims/verilator.toml @@ -0,0 +1 @@ +# PebbleVerilatorConfig: the pebble design as configured, with no overrides. diff --git a/examples/designs/pebble/emu/Cargo.lock b/examples/designs/pebble/emu/Cargo.lock new file mode 100644 index 000000000..171dff7cd --- /dev/null +++ b/examples/designs/pebble/emu/Cargo.lock @@ -0,0 +1,821 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "bebop-bank-hash" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "snafu", +] + +[[package]] +name = "bebop-bemu" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", + "bebop-bemu-profile", + "bebop-dasm", + "bebop-dtb", + "bebop-elf", + "bebop-keyboard", + "bebop-memory", + "bebop-microphone", + "bebop-rtc", + "bebop-speaker", + "bebop-syscall", + "bebop-uart", + "bebop-vga", + "bebop-vm", + "clap", + "linkme", + "prost", + "prost-build", + "prost-types", + "rustc_apfloat", + "rvsim", + "snafu", + "toml", +] + +[[package]] +name = "bebop-bemu-profile" +version = "0.1.0" + +[[package]] +name = "bebop-dasm" +version = "0.1.0" + +[[package]] +name = "bebop-design-pebble" +version = "0.1.0" +dependencies = [ + "bebop-bemu", +] + +[[package]] +name = "bebop-dtb" +version = "0.1.0" + +[[package]] +name = "bebop-elf" +version = "0.1.0" +dependencies = [ + "bebop-memory", +] + +[[package]] +name = "bebop-keyboard" +version = "0.1.0" + +[[package]] +name = "bebop-memory" +version = "0.1.0" +dependencies = [ + "log", + "serde", + "serde_json", + "sha2", +] + +[[package]] +name = "bebop-microphone" +version = "0.1.0" + +[[package]] +name = "bebop-rtc" +version = "0.1.0" + +[[package]] +name = "bebop-speaker" +version = "0.1.0" + +[[package]] +name = "bebop-syscall" +version = "0.1.0" +dependencies = [ + "bebop-memory", + "once_cell", +] + +[[package]] +name = "bebop-uart" +version = "0.1.0" + +[[package]] +name = "bebop-vga" +version = "0.1.0" + +[[package]] +name = "bebop-vm" +version = "0.1.0" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck", + "itertools", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.119", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustc_apfloat" +version = "0.2.3+llvm-462a31f5a5ab" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "486c2179b4796f65bfe2ee33679acf0927ac83ecf583ad6c91c3b4570911b9ad" +dependencies = [ + "bitflags", + "smallvec", +] + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rvsim" +version = "0.1.0" +dependencies = [ + "rustc_apfloat", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "smallvec" +version = "1.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" + +[[package]] +name = "snafu" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/examples/designs/pebble/emu/Cargo.toml b/examples/designs/pebble/emu/Cargo.toml new file mode 100644 index 000000000..3d0a9c02a --- /dev/null +++ b/examples/designs/pebble/emu/Cargo.toml @@ -0,0 +1,9 @@ +[workspace] + +[package] +name = "bebop-design-pebble" +version = "0.1.0" +edition = "2021" + +[dependencies] +bebop-bemu = { path = "../../../../bebop/src/nodes/bemu" } diff --git a/examples/designs/pebble/emu/src/device/core/mod.rs b/examples/designs/pebble/emu/src/device/core/mod.rs new file mode 100644 index 000000000..fc633526e --- /dev/null +++ b/examples/designs/pebble/emu/src/device/core/mod.rs @@ -0,0 +1,39 @@ +use bebop_bemu::{ + ball::linkme, Accelerator, BemuProfileReport, CoreFactory, CoreModel, CoreParams, Cpu, DeviceModel, ModelOptions, + Whatever, CORES, +}; +use std::time::Duration; + +/// One RV64GC hart with its local NPU on the RoCC port (arch: PebbleCore, Loom and Acc). +pub struct PebbleCore { + cpu: Cpu, + acc: Accelerator, +} + +impl CoreModel for PebbleCore { + fn cpu(&self) -> &Cpu { + &self.cpu + } + fn cpu_mut(&mut self) -> &mut Cpu { + &mut self.cpu + } + fn step(&mut self, count: u64) -> Result<(), Whatever> { + self.cpu.step(count, &mut self.acc) + } + fn take_barrier(&mut self) -> bool { + self.acc.take_barrier() + } + fn profile_report(&self, total: Duration) -> Option { + self.acc.profile_report(total, &self.cpu) + } +} + +fn instantiate(device: &DeviceModel, params: &CoreParams, options: &ModelOptions) -> Result, String> { + let cpu = Cpu::new(device, params, options).map_err(|error| error.to_string())?; + let acc = Accelerator::attached(&cpu, options)?; + Ok(Box::new(PebbleCore { cpu, acc })) +} + +#[linkme::distributed_slice(CORES)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static PEBBLE: CoreFactory = CoreFactory { component: "pebble", instantiate }; diff --git a/examples/designs/pebble/emu/src/device/mod.rs b/examples/designs/pebble/emu/src/device/mod.rs new file mode 100644 index 000000000..aa4942065 --- /dev/null +++ b/examples/designs/pebble/emu/src/device/mod.rs @@ -0,0 +1,2 @@ +//! The pebble device: its own Core model; the cores are instantiated by component on the CBus. +pub mod core; diff --git a/examples/designs/pebble/emu/src/lib.rs b/examples/designs/pebble/emu/src/lib.rs new file mode 100644 index 000000000..545892414 --- /dev/null +++ b/examples/designs/pebble/emu/src/lib.rs @@ -0,0 +1 @@ +pub mod device; diff --git a/examples/designs/pebble/sims/device/arch/DeviceConfig.scala b/examples/designs/pebble/sims/device/arch/DeviceConfig.scala new file mode 100644 index 000000000..10af06ff8 --- /dev/null +++ b/examples/designs/pebble/sims/device/arch/DeviceConfig.scala @@ -0,0 +1,18 @@ +package examples.designs.pebble.sims.device + +import buckyball.config.Design +import chisel3.experimental.hierarchy.{Instance, Instantiate} +import examples.designs.pebble.DeviceTop +import framework.device.cbus.CBus +import framework.device.cbus.generated.CoreComponents +import framework.device.dlink.BaseDevice +import _root_.configs.GlobalConfig +import _root_.sims.soc.SystemTarget +import java.nio.file.{Files, Paths} + +class DeviceConfig(design: Design) + extends SystemTarget( + GlobalConfig.fromProto(design, CBus.parameters(design, CoreComponents.factories)) + ) { + def instantiate(b: GlobalConfig): Instance[BaseDevice] = Instantiate(new DeviceTop(b, design)) +} diff --git a/examples/designs/pebble/sims/device/arch/build.sc b/examples/designs/pebble/sims/device/arch/build.sc new file mode 100644 index 000000000..d777182d1 --- /dev/null +++ b/examples/designs/pebble/sims/device/arch/build.sc @@ -0,0 +1,17 @@ +import mill._ +import mill.scalalib._ +import $file.^.^.^.arch.{build => design_build} +import $file.^.^.^.^.^.^.arch.src.main.scala.sims.{build => sims_build} + +val entryRoot: os.Path = millSourcePath + +object simulation extends SbtModule { + override def millSourcePath = entryRoot + override def scalaVersion = "2.13.16" + override def moduleDeps = Seq(design_build.pebble, sims_build.library) + override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") + override def sources = T.sources(entryRoot / "DeviceConfig.scala") + override def mainClass = Some("sims.verilator.Elaborate") +} diff --git a/examples/designs/pebble/sims/device/emu/Cargo.lock b/examples/designs/pebble/sims/device/emu/Cargo.lock new file mode 100644 index 000000000..a86e1261b --- /dev/null +++ b/examples/designs/pebble/sims/device/emu/Cargo.lock @@ -0,0 +1,2008 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "assert_cmd" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aa3a22042e45de04255c7bf3626e239f450200fd0493c1e382263544b20aea6" +dependencies = [ + "anstyle", + "bstr", + "libc", + "predicates", + "predicates-core", + "predicates-tree", + "wait-timeout", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bebop" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", + "bebop-bemu", + "bebop-bemu-profile", + "bebop-dasm", + "bebop-fd-redirect", + "bebop-p2e", + "bebop-report", + "bebop-rtl-trace", + "bebop-terminal", + "bebop-trace-perfetto", + "bebop-uart", + "bebop-vcs", + "bebop-verilator", + "bebop-vm", + "camino", + "clap", + "duct", + "env_logger", + "libc", + "log", + "nix", + "serde", + "serde_json", + "snafu", + "toml", +] + +[[package]] +name = "bebop-bank-hash" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "snafu", +] + +[[package]] +name = "bebop-bemu" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", + "bebop-bemu-profile", + "bebop-dasm", + "bebop-dtb", + "bebop-elf", + "bebop-keyboard", + "bebop-memory", + "bebop-microphone", + "bebop-rtc", + "bebop-speaker", + "bebop-syscall", + "bebop-uart", + "bebop-vga", + "clap", + "linkme", + "prost", + "prost-build", + "prost-types", + "rustc_apfloat", + "snafu", + "toml", +] + +[[package]] +name = "bebop-bemu-profile" +version = "0.1.0" + +[[package]] +name = "bebop-dasm" +version = "0.1.0" + +[[package]] +name = "bebop-design-pebble" +version = "0.1.0" +dependencies = [ + "bebop-bemu", +] + +[[package]] +name = "bebop-dtb" +version = "0.1.0" + +[[package]] +name = "bebop-elf" +version = "0.1.0" +dependencies = [ + "bebop-memory", +] + +[[package]] +name = "bebop-fd-redirect" +version = "0.1.0" +dependencies = [ + "libc", +] + +[[package]] +name = "bebop-keyboard" +version = "0.1.0" + +[[package]] +name = "bebop-memory" +version = "0.1.0" + +[[package]] +name = "bebop-microphone" +version = "0.1.0" + +[[package]] +name = "bebop-p2e" +version = "0.1.0" +dependencies = [ + "bebop-memory", + "bebop-rtl-trace", + "bebop-uart", + "duct", + "libc", + "log", + "serde", + "serde_json", +] + +[[package]] +name = "bebop-report" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "bebop-rtc" +version = "0.1.0" + +[[package]] +name = "bebop-rtl-trace" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", +] + +[[package]] +name = "bebop-sim-build" +version = "0.1.0" +dependencies = [ + "prost", + "prost-build", + "prost-types", +] + +[[package]] +name = "bebop-sim-pebble-device" +version = "0.1.0" +dependencies = [ + "assert_cmd", + "bebop", + "bebop-bemu", + "bebop-design-pebble", + "bebop-report", + "bebop-sim-build", + "bebop-verilator", + "chrono", + "clap", + "libtest-mimic", + "predicates", + "serde", + "toml", + "walkdir", +] + +[[package]] +name = "bebop-speaker" +version = "0.1.0" + +[[package]] +name = "bebop-syscall" +version = "0.1.0" +dependencies = [ + "bebop-memory", + "once_cell", +] + +[[package]] +name = "bebop-terminal" +version = "0.1.0" +dependencies = [ + "crossterm", + "ratatui", +] + +[[package]] +name = "bebop-trace-perfetto" +version = "0.1.0" +dependencies = [ + "clap", + "serde_json", +] + +[[package]] +name = "bebop-uart" +version = "0.1.0" + +[[package]] +name = "bebop-vcs" +version = "0.1.0" +dependencies = [ + "snafu", +] + +[[package]] +name = "bebop-verilator" +version = "0.1.0" +dependencies = [ + "bebop-dasm", + "bebop-fd-redirect", + "bebop-rtl-trace", + "bebop-uart", + "libc", + "snafu", +] + +[[package]] +name = "bebop-vga" +version = "0.1.0" + +[[package]] +name = "bebop-vm" +version = "0.1.0" +dependencies = [ + "clap", + "font8x8", + "sdl2", + "vt100", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "bstr" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" +dependencies = [ + "memchr", + "regex-automata", + "serde_core", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "camino" +version = "1.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbad30e4b4c14a39e3cc8aed085a12a327257c316619c93581e017bc52be591" + +[[package]] +name = "cassowary" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" + +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + +[[package]] +name = "cc" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "compact_str" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fd622ebbb56a5b2ccb651b32b911cdeb2a9b4b11776b2473bf26a26a286244e" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "static_assertions", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crossterm" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" +dependencies = [ + "bitflags 2.13.2", + "crossterm_winapi", + "mio", + "parking_lot", + "rustix 0.38.44", + "signal-hook 0.3.18", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + +[[package]] +name = "darling" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 3.0.9", +] + +[[package]] +name = "darling_macro" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" +dependencies = [ + "darling_core", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "defmt" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a0795ac740cb54d7b3775c9c17345f6aaecf884c01733b39d6017738e0e3e1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "840bb056bd99c495c9188198a258ab787c79f306d864db3e6e8451442d6d8366" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "defmt-parser" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71958f69fb2de856c5dd51ac9377e986ca1aa5919f931bf5395a807277f889a0" +dependencies = [ + "thiserror", +] + +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + +[[package]] +name = "duct" +version = "0.13.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4ab5718d1224b63252cd0c6f74f6480f9ffeb117438a2e0f5cf6d9a4798929c" +dependencies = [ + "libc", + "once_cell", + "os_pipe", + "shared_child", +] + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "env_filter" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "escape8259" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f0d310082fd5de539d4f41601182241147498444a7695c520288a0cdbead6" + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "float-cmp" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09cf3155332e944990140d967ff5eceb70df778b34f77d8075db46e4704e6d8" +dependencies = [ + "num-traits", +] + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "font8x8" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875488b8711a968268c7cf5d139578713097ca4635a76044e8fe8eedf831d07e" + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "indoc" +version = "2.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a37b2691796cffeb8a8cd305ac66e65841559f147f4e63231d0eafa4db5384d1" +dependencies = [ + "rustversion", +] + +[[package]] +name = "instability" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3b5acc1e2fd9375041a388da33d1eb8aed5f7a8c0dd3543e3ea2805adfbe20" +dependencies = [ + "darling", + "indoc", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jiff" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b005715dcbeb0089a3c0dab99f2ff1cc3b2525323552703d648585d342a383" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", +] + +[[package]] +name = "jiff-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e52fe76043ccecc9005d2305ebaadf7d7fc0cc89ca6baa10a94d6bc68c7128c" +dependencies = [ + "defmt", + "log", +] + +[[package]] +name = "jiff-static" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cc9817253cf7c7ee4684451bd327e88d6f3658014e54a29198625590650695c" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "js-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "libtest-mimic" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14e6ba06f0ade6e504aff834d7c34298e5155c6baca353cc6a4aaff2f9fd7f33" +dependencies = [ + "anstream", + "anstyle", + "clap", + "escape8259", +] + +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.5", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "os_pipe" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "portable-atomic-util" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "predicates" +version = "3.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ada8f2932f28a27ee7b70dd6c1c39ea0675c55a36879ab92f3a715eaa1e63cfe" +dependencies = [ + "anstyle", + "difflib", + "float-cmp", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cad38746f3166b4031b1a0d39ad9f954dd291e7854fcc0eed52ee41a0b50d144" + +[[package]] +name = "predicates-tree" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0de1b847b39c8131db0467e9df1ff60e6d0562ab8e9a16e568ad0fdb372e2f2" +dependencies = [ + "predicates-core", + "termtree", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck", + "itertools 0.14.0", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.119", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "ratatui" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabd94c2f37801c20583fc49dd5cd6b0ba68c716787c2dd6ed18571e1e63117b" +dependencies = [ + "bitflags 2.13.2", + "cassowary", + "compact_str", + "crossterm", + "indoc", + "instability", + "itertools 0.13.0", + "lru", + "paste", + "strum", + "unicode-segmentation", + "unicode-truncate", + "unicode-width 0.2.0", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.2", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustc_apfloat" +version = "0.2.3+llvm-462a31f5a5ab" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "486c2179b4796f65bfe2ee33679acf0927ac83ecf583ad6c91c3b4570911b9ad" +dependencies = [ + "bitflags 2.13.2", + "smallvec", +] + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys 0.12.1", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sdl2" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d42407afc6a8ab67e36f92e80b8ba34cbdc55aaeed05249efe9a2e8d0e9feef" +dependencies = [ + "bitflags 1.3.2", + "lazy_static", + "libc", + "sdl2-sys", +] + +[[package]] +name = "sdl2-sys" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ff61407fc75d4b0bbc93dc7e4d6c196439965fbef8e4a4f003a36095823eac0" +dependencies = [ + "cfg-if", + "libc", + "pkg-config", + "version-compare", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "shared_child" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "607549934f6cc26b89cfecfdc46fa90f1e5d1536a68349b0c3a4f9d1c0d37959" +dependencies = [ + "libc", + "sigchld", + "windows-sys 0.61.2", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "sigchld" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24f2b37f04360cd465089b87a9c3869c08220a2f3458463f0adf8badf5e77f2c" +dependencies = [ + "libc", + "os_pipe", + "signal-hook 0.4.5", +] + +[[package]] +name = "signal-hook" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b29ffa27d3d59a049bb328258de100f8012848d1b08d444e08bbc02ac468435e" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-mio" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b75a19a7a740b25bc7944bdee6172368f988763b744e3d4dfe753f6b4ece40cc" +dependencies = [ + "libc", + "mio", + "signal-hook 0.3.18", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8" + +[[package]] +name = "snafu" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.119", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix 1.1.5", + "windows-sys 0.61.2", +] + +[[package]] +name = "termtree" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-truncate" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3644627a5af5fa321c95b9b235a72fd24cd29c648c2c379431e6628655627bf" +dependencies = [ + "itertools 0.13.0", + "unicode-segmentation", + "unicode-width 0.1.14", +] + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "version-compare" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "579a42fc0b8e0c63b76519a339be31bed574929511fa53c1a3acae26eb258f29" + +[[package]] +name = "vt100" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84cd863bf0db7e392ba3bd04994be3473491b31e66340672af5d11943c6274de" +dependencies = [ + "itoa", + "log", + "unicode-width 0.1.14", + "vte", +] + +[[package]] +name = "vte" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5022b5fbf9407086c180e9557be968742d839e68346af7792b8592489732197" +dependencies = [ + "arrayvec", + "utf8parse", + "vte_generate_state_changes", +] + +[[package]] +name = "vte_generate_state_changes" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e369bee1b05d510a7b4ed645f5faa90619e05437111783ea5848f28d97d3c2e" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.9", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/examples/designs/pebble/sims/device/emu/Cargo.toml b/examples/designs/pebble/sims/device/emu/Cargo.toml new file mode 100644 index 000000000..f07ef4f5f --- /dev/null +++ b/examples/designs/pebble/sims/device/emu/Cargo.toml @@ -0,0 +1,83 @@ +[workspace] + +[package] +name = "bebop-sim-pebble-device" +version = "0.1.0" +edition = "2021" +autobins = false +autotests = false + +# The model of this simulation entry; the runners below are built from it. +[lib] +name = "simulation" + +[[bin]] +name = "bebop" +path = "../../../../../../bebop/src/main.rs" + +[[bin]] +name = "bebop-bemu" +path = "../../../../../../bebop/src/bin/bebop-bemu.rs" +required-features = ["bemu"] + +[[bin]] +name = "bebop-verilator" +path = "../../../../../../bebop/src/bin/bebop-verilator.rs" +required-features = ["verilator"] + +[[bin]] +name = "bebop-p2e" +path = "../../../../../../bebop/src/bin/bebop-p2e.rs" +required-features = ["p2e"] + +[[test]] +name = "test_bemu" +path = "../../../../../../bebop/tests/test_bemu.rs" +harness = false +required-features = ["bemu"] + +[[test]] +name = "test_verilator" +path = "../../../../../../bebop/tests/test_verilator.rs" +harness = false +required-features = ["verilator"] + +[[test]] +name = "test_p2e" +path = "../../../../../../bebop/tests/test_p2e.rs" +harness = false +required-features = ["p2e"] + +[[bin]] +name = "bebop-vcs" +path = "../../../../../../bebop/src/bin/bebop-vcs.rs" + +[[test]] +name = "test_vcs" +path = "../../../../../../bebop/tests/test_vcs.rs" +harness = false + +[features] +bemu = ["bebop/bemu"] +verilator = ["bebop/verilator", "dep:bebop-verilator"] +p2e = ["bebop/p2e"] + +[dependencies] +bebop = { path = "../../../../../../bebop" } +bebop-verilator = { path = "../../../../../../bebop/src/nodes/verilator", optional = true } +bebop-bemu = { path = "../../../../../../bebop/src/nodes/bemu" } +bebop-design-pebble = { path = "../../../emu" } +clap = { version = "4", features = ["derive"] } + +[build-dependencies] +bebop-sim-build = { path = "../../../../../../bebop/src/nodes/lib/sim-build" } + +[dev-dependencies] +bebop-report = { path = "../../../../../../bebop/src/nodes/lib/report" } +libtest-mimic = "0.8" +assert_cmd = "2" +walkdir = "2" +predicates = "3" +serde = { version = "1", features = ["derive"] } +toml = "0.8" +chrono = { version = "0.4", default-features = false, features = ["clock"] } diff --git a/examples/designs/pebble/sims/device/emu/build.rs b/examples/designs/pebble/sims/device/emu/build.rs new file mode 100644 index 000000000..beb62e98c --- /dev/null +++ b/examples/designs/pebble/sims/device/emu/build.rs @@ -0,0 +1,4 @@ +fn main() { + bebop_sim_build::balls(); + bebop_sim_build::runner_links(); +} diff --git a/examples/designs/pebble/sims/device/emu/src/lib.rs b/examples/designs/pebble/sims/device/emu/src/lib.rs new file mode 100644 index 000000000..b92afd6f1 --- /dev/null +++ b/examples/designs/pebble/sims/device/emu/src/lib.rs @@ -0,0 +1,11 @@ +//! The pebble device simulation. +//! Its Core models register in `bebop_bemu::CORES`; the Balls its design.pb +//! names are compiled here and register in `bebop_bemu::BALLS`. +extern crate bebop_design_pebble; + +include!(concat!(env!("OUT_DIR"), "/balls.rs")); + +pub const SIMULATION: bebop_bemu::Simulation = bebop_bemu::Simulation { + design: include_bytes!(concat!(env!("OUT_DIR"), "/design.pb")), + instantiate: bebop_bemu::device::cbus::instantiate, +}; diff --git a/examples/designs/pebble/sims/device/regression/bemu/workloads-bare.toml b/examples/designs/pebble/sims/device/regression/bemu/workloads-bare.toml new file mode 100644 index 000000000..5a46c4f3e --- /dev/null +++ b/examples/designs/pebble/sims/device/regression/bemu/workloads-bare.toml @@ -0,0 +1,122 @@ +[workloads] +tests = [ + "pebble-pebble-mlirtest-core_mega_matmul_m17_k65_n17-baremetal", + "pebble-pebble-mlirtest-core_mega_matmul-baremetal", + "pebble-pebble-mlirtest-core_mega_conv2d_k5_tail-baremetal", + "pebble-pebble-mlirtest-core_mega_conv2d-baremetal", + "pebble-pebble-mlirtest-core_mega_conv2d_resident-baremetal", + "pebble-pebble-mlirtest-core_mega_conv2d_flatten_linear-baremetal", + "pebble-pebble-mlirtest-core_lookup_channel_i8-baremetal", + "pebble-pebble-mlirtest-core_generic_matmul_i8-baremetal", + "pebble-pebble-mlirtest-core_linear_int8-baremetal", + "pebble-pebble-mlirtest-core_linear_chain_i8-baremetal", + "pebble-pebble-ctest-bank_capacity_rename_test-baremetal", + "pebble-pebble-ctest-bank_transfer_recycle_test-baremetal", + "pebble-pebble-ctest-mset_transfer_test-baremetal", + "pebble-pebble-ctest-bank_recycle_test-baremetal", + "pebble-pebble-ctest-mvin_2d_test-baremetal", + "pebble-pebble-ctest-mvin_2d_pixel_stride_test-baremetal", + "pebble-pebble-ctest-mvin_mvout_acc_test-baremetal", + "pebble-pebble-ctest-mvin_mvout_test-baremetal", + "pebble-pebble-ctest-mvin_mvout_stride_test-baremetal", + "pebble-pebble-ctest-mvin_mvout_unaligned_test-baremetal", + "pebble-pebble-ctest-mvin_unaligned_test-baremetal", + "pebble-pebble-mlirtest-linalg_mega_conv2d_k3_c16-baremetal", + "pebble-pebble-mlirtest-core_mega_conv2d_s2_depthwise_s2-baremetal", + "pebble-pebble-mlirtest-core_mega_global_avg_conv_exact-baremetal", + "pebble-pebble-mlirtest-core_mega_global_avg_conv2_exact-baremetal", + "pebble-pebble-mlirtest-core_mega_global_avg_7x7_c288_exact-baremetal", + "pebble-pebble-mlirtest-core_mega_se_gate_conv-baremetal", + "pebble-pebble-ctest-mega_conv_pipeline_test-baremetal", + "pebble-pebble-ctest-residual_add_lut_test-baremetal", + "pebble-pebble-ctest-maxpool_im2col_test-baremetal", + "pebble-pebble-ctest-se_gate_lut_test-baremetal", + "pebble-pebble-ctest-int8mul_large_gate_row_test-baremetal", + "pebble-pebble-ctest-dual_smatmul_matadd_test-baremetal", + "pebble-pebble-ctest-dual_conv_add_int2fp_test-baremetal", + "pebble-pebble-ctest-quant_smatmul_int2fp_test-baremetal", + "pebble-pebble-ctest-smatmul_mvout_bank_reuse_test-baremetal", + "pebble-pebble-mlirtest-bank_matadd_i32_elementwise-baremetal", + "pebble-pebble-mlirtest-bank_int8add_identity_scale-baremetal", + "pebble-pebble-mlirtest-bank_lut_identity_table-baremetal", + "pebble-pebble-mlirtest-bank_int8mul_unit_gate-baremetal", + "pebble-pebble-mlirtest-bank_maxpool_k3_padded_spatial-baremetal", + "pebble-pebble-mlirtest-bank_smatmul_bias_two_terms-baremetal", + "pebble-pebble-mlirtest-bank_int2fp_unit_scale-baremetal", + "pebble-pebble-mlirtest-bank_quant_i32_pack16-baremetal", + "pebble-pebble-mlirtest-bank_quant_f32_pack16-baremetal", + "pebble-pebble-mlirtest-bank_im2col_k3_two_blocks-baremetal", + "pebble-pebble-mlirtest-bank_transpose_i8_packed_matrix-baremetal", + "pebble-pebble-ctest-tlb_bare_mode_test-baremetal", + "pebble-pebble-ctest-tlb_repeated_sfence_test-baremetal", + "pebble-pebble-ctest-tlb_sfence_flush_test-baremetal", + "pebble-pebble-ctest-tlb_sfence_remap_test-baremetal", + "pebble-pebble-ctest-tlb_sv39_4k_alias_test-baremetal", + "pebble-pebble-ctest-tlb_sv39_identity_test-baremetal", + "pebble-pebble-ctest-tlb_sv39_nonidentity_test-baremetal", + "pebble-pebble-ctest-transpose_i8_16x16_test-baremetal", + "pebble-pebble-ctest-transpose_i8_16x32_test-baremetal", + "pebble-pebble-ctest-transpose_i32_8x16_test-baremetal", + "pebble-pebble-mlirtest-ball_transpose_16x16_i8-baremetal", + "pebble-pebble-ctest-smatmul_bias_accumulate_test-baremetal", + "pebble-pebble-ctest-smatmul_1x16x64_test-baremetal", + "pebble-pebble-ctest-smatmul_16x16_test-baremetal", + "pebble-pebble-ctest-smatmul_16x16x32_test-baremetal", + "pebble-pebble-ctest-smatmul_nondefault_banks_test-baremetal", + "pebble-pebble-ctest-smatmul_k_groups_test-baremetal", + "pebble-pebble-ctest-smatmul_c_groups_test-baremetal", + "pebble-pebble-mlirtest-ball_smatmul_bias_accumulate-baremetal", + "pebble-pebble-ctest-quant_f32_to_i8_test-baremetal", + "pebble-pebble-ctest-quant_i32_to_i8_test-baremetal", + "pebble-pebble-ctest-toint8_i8_rounding_test-baremetal", + "pebble-pebble-ctest-toint8_i8_rows_test-baremetal", + "pebble-pebble-ctest-toint8_i8_scale_rows_test-baremetal", + "pebble-pebble-ctest-toint8_i8_signed_test-baremetal", + "pebble-pebble-ctest-quant_i32_groups_test-baremetal", + "pebble-pebble-ctest-quant_f32_groups_test-baremetal", + "pebble-pebble-mlirtest-ball_quant_f32_to_i8-baremetal", + "pebble-pebble-mlirtest-ball_quant_i32_to_i8-baremetal", + "pebble-pebble-ctest-im2col_window_test-baremetal", + "pebble-pebble-ctest-im2col_k1_test-baremetal", + "pebble-pebble-ctest-im2col_k3_test-baremetal", + "pebble-pebble-ctest-im2col_k3_pad_test-baremetal", + "pebble-pebble-ctest-im2col_k3_asym_pad_test-baremetal", + "pebble-pebble-ctest-im2col_k3_partial_test-baremetal", + "pebble-pebble-ctest-im2col_k5_test-baremetal", + "pebble-pebble-ctest-im2col_k7_test-baremetal", + "pebble-pebble-mlirtest-ball_im2col_6x6_k3-baremetal", + "pebble-pebble-ctest-int2fp_tensor_values_test-baremetal", + "pebble-pebble-ctest-int2fp_tensor_groups_test-baremetal", + "pebble-pebble-ctest-int2fp_tensor_rows_test-baremetal", + "pebble-pebble-ctest-int2fp_channel_lanes_test-baremetal", + "pebble-pebble-ctest-int2fp_channel_base_test-baremetal", + "pebble-pebble-ctest-int2fp_channel_two_rows_test-baremetal", + "pebble-pebble-ctest-int2fp_groups_test-baremetal", + "pebble-pebble-mlirtest-ball_int2fp_4x4-baremetal", + "pebble-pebble-mlirtest-ball_int2fp_relu_4x4-baremetal", + "pebble-pebble-ctest-lut_uniform_lookup_test-baremetal", + "pebble-pebble-ctest-lut_channel_table_lookup_test-baremetal", + "pebble-pebble-mlirtest-ball_lut_identity-baremetal", + "pebble-pebble-ctest-maxpool_window_test-baremetal", + "pebble-pebble-ctest-maxpool_offset_test-baremetal", + "pebble-pebble-ctest-maxpool_groups_test-baremetal", + "pebble-pebble-mlirtest-ball_maxpool_6x6_s2-baremetal", + "pebble-pebble-ctest-int8add_scaled_test-baremetal", + "pebble-pebble-ctest-int8add_relu_test-baremetal", + "pebble-pebble-mlirtest-ball_int8add_relu-baremetal", + "pebble-pebble-ctest-int8mul_per_channel_gate_test-baremetal", + "pebble-pebble-mlirtest-ball_int8mul_gate-baremetal", + "pebble-pebble-ctest-matadd_i32_16row_test-baremetal", + "pebble-pebble-mlirtest-ball_matadd_i32-baremetal", + "pebble-pebble-ctest-pipeline-bank_dma_reuse-multicore-baremetal", + "pebble-pebble-ctest-pipeline-bank_rename_overlap-multicore-baremetal", + "pebble-pebble-ctest-pipeline-maxpool_partial_update-multicore-baremetal", + "pebble-pebble-ctest-pipeline-bank_dma_pingpong-multicore-baremetal", + "pebble-pebble-ctest-pipeline-read_bank_shared-multicore-baremetal", + "pebble-pebble-ctest-pipeline-read_bank_split-multicore-baremetal", + "pebble-pebble-ctest-pipeline-same_ball_independent-multicore-baremetal", + "pebble-pebble-ctest-pipeline-rob_window_probe-multicore-baremetal", + "pebble-pebble-ctest-pipeline-write_bank_versions-multicore-baremetal", + "pebble-pebble-mlirtest-base_bank_loop_carried-baremetal", + "pebble-pebble-mlirtest-base_bank_transfer-baremetal", +] diff --git a/examples/designs/pebble/sims/device/regression/bemu/workloads-linux.toml b/examples/designs/pebble/sims/device/regression/bemu/workloads-linux.toml new file mode 100644 index 000000000..41a9cd13b --- /dev/null +++ b/examples/designs/pebble/sims/device/regression/bemu/workloads-linux.toml @@ -0,0 +1,4 @@ +[workloads] +tests = [ + "fw_payload-linux.elf", +] diff --git a/examples/designs/pebble/sims/device/regression/p2e/workloads-bare.toml b/examples/designs/pebble/sims/device/regression/p2e/workloads-bare.toml new file mode 100644 index 000000000..80d331ecf --- /dev/null +++ b/examples/designs/pebble/sims/device/regression/p2e/workloads-bare.toml @@ -0,0 +1,33 @@ +[workloads] +tests = [ + "pebble-pebble-ctest-mvin_2d_pixel_stride_test-baremetal.hex", + "pebble-pebble-ctest-mega_conv_pipeline_test-baremetal.hex", + "pebble-pebble-mlirtest-core_mega_conv2d_s2_depthwise_s2-baremetal.hex", + "pebble-pebble-mlirtest-core_mega_se_gate_conv-baremetal.hex", + "pebble-pebble-mlirtest-core_mega_global_avg_conv_exact-baremetal.hex", + "pebble-pebble-mlirtest-core_mega_global_avg_conv2_exact-baremetal.hex", + "pebble-pebble-mlirtest-core_mega_global_avg_7x7_c288_exact-baremetal.hex", + "pebble-pebble-ctest-residual_add_lut_test-baremetal.hex", + "pebble-pebble-ctest-maxpool_im2col_test-baremetal.hex", + "pebble-pebble-ctest-se_gate_lut_test-baremetal.hex", + "pebble-pebble-ctest-dual_conv_add_int2fp_test-baremetal.hex", + "pebble-pebble-mlirtest-bank_matadd_i32_elementwise-baremetal.hex", + "pebble-pebble-mlirtest-bank_int8add_identity_scale-baremetal.hex", + "pebble-pebble-mlirtest-bank_lut_identity_table-baremetal.hex", + "pebble-pebble-mlirtest-bank_int8mul_unit_gate-baremetal.hex", + "pebble-pebble-mlirtest-bank_maxpool_k3_padded_spatial-baremetal.hex", + "pebble-pebble-mlirtest-bank_smatmul_bias_two_terms-baremetal.hex", + "pebble-pebble-mlirtest-bank_int2fp_unit_scale-baremetal.hex", + "pebble-pebble-mlirtest-bank_quant_i32_pack16-baremetal.hex", + "pebble-pebble-mlirtest-bank_quant_f32_pack16-baremetal.hex", + "pebble-pebble-mlirtest-bank_im2col_k3_two_blocks-baremetal.hex", + "pebble-pebble-mlirtest-bank_transpose_i8_packed_matrix-baremetal.hex", + "pebble-pebble-ctest-pipeline-bank_dma_reuse-multicore-baremetal.hex", + "pebble-pebble-ctest-pipeline-maxpool_partial_update-multicore-baremetal.hex", + "pebble-pebble-ctest-pipeline-bank_dma_pingpong-multicore-baremetal.hex", + "pebble-pebble-ctest-pipeline-read_bank_shared-multicore-baremetal.hex", + "pebble-pebble-ctest-pipeline-read_bank_split-multicore-baremetal.hex", + "pebble-pebble-ctest-pipeline-same_ball_independent-multicore-baremetal.hex", + "pebble-pebble-ctest-pipeline-rob_window_probe-multicore-baremetal.hex", + "pebble-pebble-ctest-pipeline-write_bank_versions-multicore-baremetal.hex", +] diff --git a/examples/chips/pebble/regression/batch/p2e/workloads-linux.toml b/examples/designs/pebble/sims/device/regression/p2e/workloads-linux.toml similarity index 100% rename from examples/chips/pebble/regression/batch/p2e/workloads-linux.toml rename to examples/designs/pebble/sims/device/regression/p2e/workloads-linux.toml diff --git a/examples/designs/pebble/sims/device/regression/vcs/workloads-bare.toml b/examples/designs/pebble/sims/device/regression/vcs/workloads-bare.toml new file mode 100644 index 000000000..5a932f0b2 --- /dev/null +++ b/examples/designs/pebble/sims/device/regression/vcs/workloads-bare.toml @@ -0,0 +1,11 @@ +[workloads] +tests = [ + "pebble-core0-soctest-clint_timer_interrupt-baremetal", + "pebble-core0-soctest-clint_software_interrupt-baremetal", + "pebble-core0-soctest-clint_supervisor_timer-baremetal", + "pebble-core0-soctest-time_csr_clock-baremetal", + "pebble-core0-soctest-time_csr_permissions-baremetal", + "pebble-core0-soctest-lrsc_word-baremetal", + "pebble-core0-soctest-lrsc_translated-baremetal", + "pebble-core0-soctest-plic_register_access-baremetal", +] diff --git a/examples/designs/pebble/sims/device/regression/verilator/workloads-bare.toml b/examples/designs/pebble/sims/device/regression/verilator/workloads-bare.toml new file mode 100644 index 000000000..a33ba804f --- /dev/null +++ b/examples/designs/pebble/sims/device/regression/verilator/workloads-bare.toml @@ -0,0 +1,106 @@ +[workloads] +tests = [ + "pebble-pebble-mlirtest-core_mega_matmul_m17_k65_n17-baremetal", + "pebble-pebble-mlirtest-core_mega_matmul-baremetal", + "pebble-pebble-mlirtest-core_mega_conv2d_k5_tail-baremetal", + "pebble-pebble-mlirtest-core_mega_conv2d-baremetal", + "pebble-pebble-mlirtest-core_lookup_channel_i8-baremetal", + "pebble-pebble-mlirtest-core_generic_matmul_i8-baremetal", + "pebble-pebble-mlirtest-core_linear_int8-baremetal", + "pebble-pebble-mlirtest-core_linear_chain_i8-baremetal", + "pebble-pebble-ctest-bank_capacity_rename_test-baremetal", + "pebble-pebble-ctest-bank_transfer_recycle_test-baremetal", + "pebble-pebble-ctest-mset_transfer_test-baremetal", + "pebble-pebble-ctest-bank_recycle_test-baremetal", + "pebble-pebble-ctest-mvin_2d_test-baremetal", + "pebble-pebble-ctest-mvin_2d_pixel_stride_test-baremetal", + "pebble-pebble-ctest-mvin_mvout_acc_test-baremetal", + "pebble-pebble-ctest-mvin_mvout_test-baremetal", + "pebble-pebble-ctest-mvin_mvout_stride_test-baremetal", + "pebble-pebble-ctest-mvin_mvout_unaligned_test-baremetal", + "pebble-pebble-ctest-mvin_unaligned_test-baremetal", + "pebble-pebble-ctest-tlb_bare_mode_test-baremetal", + "pebble-pebble-ctest-tlb_repeated_sfence_test-baremetal", + "pebble-pebble-ctest-tlb_sfence_flush_test-baremetal", + "pebble-pebble-ctest-tlb_sfence_remap_test-baremetal", + "pebble-pebble-ctest-tlb_sv39_4k_alias_test-baremetal", + "pebble-pebble-ctest-tlb_sv39_identity_test-baremetal", + "pebble-pebble-ctest-tlb_sv39_nonidentity_test-baremetal", + "pebble-pebble-mlirtest-linalg_mega_conv2d_k3_c16-baremetal", + "pebble-pebble-mlirtest-core_mega_conv2d_s2_depthwise_s2-baremetal", + "pebble-pebble-ctest-transpose_i8_16x16_test-baremetal", + "pebble-pebble-ctest-transpose_i8_16x32_test-baremetal", + "pebble-pebble-ctest-transpose_i32_8x16_test-baremetal", + "pebble-pebble-mlirtest-ball_transpose_16x16_i8-baremetal", + "pebble-pebble-ctest-smatmul_bias_accumulate_test-baremetal", + "pebble-pebble-ctest-smatmul_1x16x64_test-baremetal", + "pebble-pebble-ctest-smatmul_16x16_test-baremetal", + "pebble-pebble-ctest-smatmul_16x16x32_test-baremetal", + "pebble-pebble-ctest-smatmul_nondefault_banks_test-baremetal", + "pebble-pebble-ctest-smatmul_k_groups_test-baremetal", + "pebble-pebble-ctest-smatmul_mvout_bank_reuse_test-baremetal", + "pebble-pebble-mlirtest-ball_smatmul_bias_accumulate-baremetal", + "pebble-pebble-ctest-quant_f32_to_i8_test-baremetal", + "pebble-pebble-ctest-quant_i32_to_i8_test-baremetal", + "pebble-pebble-ctest-toint8_i8_rounding_test-baremetal", + "pebble-pebble-ctest-toint8_i8_rows_test-baremetal", + "pebble-pebble-ctest-toint8_i8_scale_rows_test-baremetal", + "pebble-pebble-ctest-toint8_i8_signed_test-baremetal", + "pebble-pebble-ctest-quant_i32_groups_test-baremetal", + "pebble-pebble-ctest-quant_f32_groups_test-baremetal", + "pebble-pebble-mlirtest-ball_quant_f32_to_i8-baremetal", + "pebble-pebble-mlirtest-ball_quant_i32_to_i8-baremetal", + "pebble-pebble-ctest-im2col_window_test-baremetal", + "pebble-pebble-ctest-im2col_k1_test-baremetal", + "pebble-pebble-ctest-im2col_k3_test-baremetal", + "pebble-pebble-ctest-im2col_k3_pad_test-baremetal", + "pebble-pebble-ctest-im2col_k3_asym_pad_test-baremetal", + "pebble-pebble-ctest-im2col_k3_partial_test-baremetal", + "pebble-pebble-ctest-im2col_k5_test-baremetal", + "pebble-pebble-ctest-im2col_k7_test-baremetal", + "pebble-pebble-mlirtest-ball_im2col_6x6_k3-baremetal", + "pebble-pebble-ctest-int2fp_tensor_values_test-baremetal", + "pebble-pebble-ctest-int2fp_tensor_groups_test-baremetal", + "pebble-pebble-ctest-int2fp_tensor_rows_test-baremetal", + "pebble-pebble-ctest-int2fp_channel_lanes_test-baremetal", + "pebble-pebble-ctest-int2fp_channel_base_test-baremetal", + "pebble-pebble-ctest-int2fp_channel_two_rows_test-baremetal", + "pebble-pebble-ctest-int2fp_groups_test-baremetal", + "pebble-pebble-mlirtest-ball_int2fp_4x4-baremetal", + "pebble-pebble-mlirtest-ball_int2fp_relu_4x4-baremetal", + "pebble-pebble-ctest-lut_uniform_lookup_test-baremetal", + "pebble-pebble-ctest-lut_channel_table_lookup_test-baremetal", + "pebble-pebble-mlirtest-ball_lut_identity-baremetal", + "pebble-pebble-ctest-maxpool_window_test-baremetal", + "pebble-pebble-ctest-maxpool_offset_test-baremetal", + "pebble-pebble-ctest-maxpool_groups_test-baremetal", + "pebble-pebble-mlirtest-ball_maxpool_6x6_s2-baremetal", + "pebble-pebble-ctest-int8add_scaled_test-baremetal", + "pebble-pebble-ctest-int8add_relu_test-baremetal", + "pebble-pebble-mlirtest-ball_int8add_relu-baremetal", + "pebble-pebble-ctest-int8mul_per_channel_gate_test-baremetal", + "pebble-pebble-ctest-int8mul_large_gate_row_test-baremetal", + "pebble-pebble-mlirtest-ball_int8mul_gate-baremetal", + "pebble-pebble-ctest-matadd_i32_16row_test-baremetal", + "pebble-pebble-mlirtest-ball_matadd_i32-baremetal", + "pebble-pebble-ctest-mega_conv_pipeline_test-baremetal", + "pebble-pebble-mlirtest-core_mega_se_gate_conv-baremetal", + "pebble-pebble-mlirtest-core_mega_global_avg_conv_exact-baremetal", + "pebble-pebble-mlirtest-core_mega_global_avg_conv2_exact-baremetal", + "pebble-pebble-mlirtest-core_mega_global_avg_7x7_c288_exact-baremetal", + "pebble-pebble-ctest-residual_add_lut_test-baremetal", + "pebble-pebble-ctest-maxpool_im2col_test-baremetal", + "pebble-pebble-ctest-se_gate_lut_test-baremetal", + "pebble-pebble-ctest-dual_smatmul_matadd_test-baremetal", + "pebble-pebble-ctest-dual_conv_add_int2fp_test-baremetal", + "pebble-pebble-ctest-quant_smatmul_int2fp_test-baremetal", + "pebble-pebble-ctest-pipeline-bank_dma_reuse-multicore-baremetal", + "pebble-pebble-ctest-pipeline-bank_rename_overlap-multicore-baremetal", + "pebble-pebble-ctest-pipeline-maxpool_partial_update-multicore-baremetal", + "pebble-pebble-ctest-pipeline-bank_dma_pingpong-multicore-baremetal", + "pebble-pebble-ctest-pipeline-read_bank_shared-multicore-baremetal", + "pebble-pebble-ctest-pipeline-read_bank_split-multicore-baremetal", + "pebble-pebble-ctest-pipeline-same_ball_independent-multicore-baremetal", + "pebble-pebble-ctest-pipeline-rob_window_probe-multicore-baremetal", + "pebble-pebble-ctest-pipeline-write_bank_versions-multicore-baremetal", +] diff --git a/examples/designs/pebble/sims/device/stack/CMakeLists.txt b/examples/designs/pebble/sims/device/stack/CMakeLists.txt new file mode 100644 index 000000000..4323021ce --- /dev/null +++ b/examples/designs/pebble/sims/device/stack/CMakeLists.txt @@ -0,0 +1,37 @@ +# ===----------------------------------------------------------------------=== +# The software stack of the pebble device simulation: every level the device +# needs, built bottom up. +# +# bbdev stack --build "--design pebble --device " +# +# configures this with DESIGN_CMAKE, generated from design.pb. Each level is a +# CMake build of its own: cores and threads build buddy-opt plugins for the +# host, the device builds RISC-V libraries. +# core/pebble libpebble-compiler.so, execute.o +# device runtime, runtime-baremetal, runtime-linux, engine; device.cmake +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(pebble-device-stack NONE) +include(ExternalProject) +include("${DESIGN_CMAKE}") + +# Pebble's CPU core with its Ball accelerator. +ExternalProject_Add(core-pebble + SOURCE_DIR "${DESIGN_CORE_STACK_pebble}" + BINARY_DIR "${DESIGN_CORE_BUILD_pebble}" + CMAKE_GENERATOR Ninja + CMAKE_ARGS "-DDESIGN_CMAKE=${DESIGN_CMAKE}" -DCORE=pebble + BUILD_ALWAYS ON + INSTALL_COMMAND "") + +# The device: the runtime that schedules jobs onto the cores. +ExternalProject_Add(device + SOURCE_DIR "${DESIGN_DEVICE_STACK}" + BINARY_DIR "${DESIGN_DEVICE_BUILD}" + CMAKE_GENERATOR Ninja + CMAKE_ARGS "-DDESIGN_CMAKE=${DESIGN_CMAKE}" + -DCMAKE_C_COMPILER=riscv64-unknown-linux-gnu-gcc + -DCMAKE_CXX_COMPILER=riscv64-unknown-linux-gnu-g++ + DEPENDS core-pebble + BUILD_ALWAYS ON + INSTALL_COMMAND "") diff --git a/examples/designs/pebble/sims/device/tapeout/README.md b/examples/designs/pebble/sims/device/tapeout/README.md new file mode 100644 index 000000000..ee8f027b3 --- /dev/null +++ b/examples/designs/pebble/sims/device/tapeout/README.md @@ -0,0 +1,10 @@ +# Pebble tapeout contract + +Layout: +- `config.toml` — design contract (scripts, libs, power window) +- `area/` — DC (`dc.tcl`, `constraints.sdc`) +- `power/` — PTPX + gate sim (`power.tcl`, `power_sim.sh`) +- `ip/` — SRAM MDF + +`bbdev dc` resolves paths from `config.toml` and injects `RUN_*`. Timing truth is +`area/constraints.sdc`. Pebble main clock is 100 MHz (`10 ns`). diff --git a/examples/chips/pebble/tapeout/area/.gitignore b/examples/designs/pebble/sims/device/tapeout/area/.gitignore similarity index 100% rename from examples/chips/pebble/tapeout/area/.gitignore rename to examples/designs/pebble/sims/device/tapeout/area/.gitignore diff --git a/examples/designs/pebble/sims/device/tapeout/area/constraints.sdc b/examples/designs/pebble/sims/device/tapeout/area/constraints.sdc new file mode 100644 index 000000000..7af7d73af --- /dev/null +++ b/examples/designs/pebble/sims/device/tapeout/area/constraints.sdc @@ -0,0 +1,9 @@ +# DeviceTop has one clock; every other port is synchronous to it. +set core_clock [get_ports clock] +create_clock -name core -period 10.0 $core_clock + +set_clock_uncertainty -setup 3.0 [get_clocks core] +set_clock_transition 1.0 [get_clocks core] + +set_input_delay 3.0 -clock core [remove_from_collection [all_inputs] $core_clock] +set_output_delay 3.0 -clock core [all_outputs] diff --git a/examples/designs/pebble/sims/device/tapeout/area/dc.tcl b/examples/designs/pebble/sims/device/tapeout/area/dc.tcl new file mode 100755 index 000000000..3d4646417 --- /dev/null +++ b/examples/designs/pebble/sims/device/tapeout/area/dc.tcl @@ -0,0 +1,65 @@ +# Design-owned DC flow. Timing comes from constraints.sdc (RUN_SDC). +# bbdev injects RUN_* via -x {set RUN_CONFIG }. +if {![info exists RUN_CONFIG] || $RUN_CONFIG eq ""} { + error "bbdev must pass -x {set RUN_CONFIG }" +} +source [file normalize $RUN_CONFIG] + +proc bbdev_read_filelist {path} { + set fh [open $path r] + set files [list] + while {[gets $fh line] >= 0} { + set line [string trim $line] + if {$line eq "" || [string match "#*" $line]} { continue } + lappend files [file normalize $line] + } + close $fh + if {[llength $files] == 0} { error "empty DC source list: $path" } + return $files +} + +set top $RUN_TOP +set output_dir [file normalize $RUN_OUTPUT_DIR] +set report_dir [file normalize $RUN_REPORT_DIR] +file mkdir $output_dir $report_dir [file join $report_dir work] +set target_library [list $RUN_TARGET_LIBRARY] +set synthetic_library $RUN_SYNTHETIC_LIBRARY +set link_library [concat [list *] $target_library $synthetic_library $RUN_LINK_LIBRARY] +set_host_options -max_cores $RUN_MAX_THREADS +define_design_lib WORK -path [file join $report_dir work] +set search_path [list .] +set_app_var verilogout_no_tri true +set_app_var verilogout_equation false +set_app_var compile_seqmap_propagate_constants false +set_app_var hdlin_ff_always_sync_set_reset true +analyze -format sverilog -define {SYNTHESIS DC_SYN} [bbdev_read_filelist $RUN_SOURCE_LIST] +elaborate $top +current_design $top +link +set sram_cells [get_cells -hierarchical -quiet -filter {ref_name =~ "chipyard_sram_*"}] +if {[sizeof_collection $sram_cells] == 0} { + error "no chipyard_sram_* cells after link; SRAM .db link_library missing" +} +set_dont_touch $sram_cells +set sram_designs [get_designs -quiet chipyard_sram_*] +if {[sizeof_collection $sram_designs] > 0} { + set_dont_touch $sram_designs +} +read_sdc [file normalize $RUN_SDC] +if {[sizeof_collection [all_clocks]] == 0} { + error "SDC created no clocks" +} +set_load 2.0 [all_outputs] +set_app_var compile_seqmap_honor_sync_set_reset true +compile_ultra -area_high_effort_script -no_autoungroup -no_boundary_optimization +set_fix_multiple_port_nets -all -buffer_constants +change_names -hierarchy -rules verilog +write -format ddc -hierarchy -output [file join $output_dir ${top}.ddc] +write -format verilog -hierarchy -output [file join $output_dir ${top}.v] +write_sdc [file join $output_dir ${top}.sdc] +report_constraint -all_violators > [file join $report_dir constraint.rpt] +report_timing -delay max -max_paths 50 > [file join $report_dir timing_max.rpt] +report_timing -delay min -max_paths 50 > [file join $report_dir timing_min.rpt] +report_area -hierarchy > [file join $report_dir area.rpt] +report_reference > [file join $report_dir reference.rpt] +exit diff --git a/examples/designs/pebble/sims/device/tapeout/config.toml b/examples/designs/pebble/sims/device/tapeout/config.toml new file mode 100644 index 000000000..5ec421892 --- /dev/null +++ b/examples/designs/pebble/sims/device/tapeout/config.toml @@ -0,0 +1,117 @@ +[tapeout] +top = "DeviceTop" +sram_process = "smic180" +sram_mdf = "ip/smic180_sram_library.mdf.json" +constraints_sdc = "area/constraints.sdc" +target_library = "/home/bb-runner/Code/eda/D_libraries/SMIC180/SCC018UG_UHD_RVT_V0.4a/SCC018UG_UHD_RVT_V0p4a/liberty/1.8v/scc018ug_uhd_rvt_ss_v1p62_125c_ccs.db" +synthetic_library = [ + "/home/bb-runner/Code/eda/A_packages/tools/syn/R-2020.09-SP5/libraries/syn/dw_foundation.sldb", + "/home/bb-runner/Code/eda/A_packages/tools/syn/R-2020.09-SP5/libraries/syn/standard.sldb", +] +link_library = [] +max_threads = 16 +power_format = "vcd" +power_start_ns = "1000000" +power_end_ns = "5000000" +dc_script = "area/dc.tcl" +power_script = "power/power.tcl" +power_sim_script = "power/power_sim.sh" +power_strip_path = "GatePowerHarness.dut" + +sram_corner = "ss" +lc_shell = "/home/bb-runner/Code/eda/A_packages/tools/lc/R-2020.09-SP5/bin/lc_shell" + +[[tapeout.sram]] +name = "chipyard_sram_32x128" +words = 32 +mux = 4 +bits = 128 + +[[tapeout.sram]] +name = "chipyard_sram_32x22" +words = 32 +mux = 4 +bits = 22 + +[[tapeout.sram]] +name = "chipyard_sram_64x21" +words = 64 +mux = 8 +bits = 21 + +[[tapeout.sram]] +name = "chipyard_sram_64x22" +words = 64 +mux = 8 +bits = 22 + +[[tapeout.sram]] +name = "chipyard_sram_512x8" +words = 512 +mux = 8 +bits = 8 + +[[tapeout.sram]] +name = "chipyard_sram_512x32" +words = 512 +mux = 8 +bits = 32 + +[[tapeout.sram]] +name = "chipyard_sram_512x64" +words = 512 +mux = 8 +bits = 64 + +[[tapeout.sram]] +name = "chipyard_sram_1024x8" +words = 1024 +mux = 8 +bits = 8 + +[[tapeout.sram]] +name = "chipyard_sram_32x21" +words = 32 +mux = 4 +bits = 21 + +[[tapeout.sram]] +name = "chipyard_sram_32x23" +words = 32 +mux = 4 +bits = 23 + +[[tapeout.sram]] +name = "chipyard_sram_64x8" +words = 64 +mux = 8 +bits = 8 + +[[tapeout.sram]] +name = "chipyard_sram_64x128" +words = 64 +# SMIC180 has only six metal layers, so this macro cannot be treated as a +# routable-over cell. mux=4 changes only the internal column organization: +# it keeps the 64x128 1RW + BWEN interface but shrinks the generated macro +# width from about 2013 um (mux=8) to about 1045 um, avoiding long routing walls. +mux = 4 +bits = 128 +bitwrite = true + +[[tapeout.sram]] +name = "chipyard_sram_128x8" +words = 128 +mux = 8 +bits = 8 + +[[tapeout.sram]] +name = "chipyard_sram_256x32" +words = 256 +mux = 8 +bits = 32 + +[[tapeout.sram]] +name = "chipyard_sram_256x64" +words = 256 +mux = 8 +bits = 64 diff --git a/examples/chips/pebble/tapeout/ip/smic180_sram_library.mdf.json b/examples/designs/pebble/sims/device/tapeout/ip/smic180_sram_library.mdf.json similarity index 100% rename from examples/chips/pebble/tapeout/ip/smic180_sram_library.mdf.json rename to examples/designs/pebble/sims/device/tapeout/ip/smic180_sram_library.mdf.json diff --git a/examples/chips/pebble/tapeout/ip/tsmc28_sram_library.mdf.json b/examples/designs/pebble/sims/device/tapeout/ip/tsmc28_sram_library.mdf.json similarity index 100% rename from examples/chips/pebble/tapeout/ip/tsmc28_sram_library.mdf.json rename to examples/designs/pebble/sims/device/tapeout/ip/tsmc28_sram_library.mdf.json diff --git a/examples/chips/pebble/tapeout/power/power.tcl b/examples/designs/pebble/sims/device/tapeout/power/power.tcl similarity index 100% rename from examples/chips/pebble/tapeout/power/power.tcl rename to examples/designs/pebble/sims/device/tapeout/power/power.tcl diff --git a/examples/chips/pebble/tapeout/power/power_sim.sh b/examples/designs/pebble/sims/device/tapeout/power/power_sim.sh similarity index 77% rename from examples/chips/pebble/tapeout/power/power_sim.sh rename to examples/designs/pebble/sims/device/tapeout/power/power_sim.sh index 3f3969706..01192a1aa 100755 --- a/examples/chips/pebble/tapeout/power/power_sim.sh +++ b/examples/designs/pebble/sims/device/tapeout/power/power_sim.sh @@ -13,7 +13,7 @@ if [[ "$ACTIVITY_FORMAT" != "vcd" ]]; then echo "Pebble gate power simulation currently emits VCD, got $ACTIVITY_FORMAT" >&2 exit 2 fi -ROOT=$(cd "$(dirname "$0")/../../../../.." && pwd) +ROOT=$(git -C "$(dirname "$0")" rev-parse --show-toplevel) RTL_DIR="$ROOT/arch/build/${CONFIG}" MEM_CONF="$RTL_DIR/mems.conf" BUILD="$OUTPUT_DIR/gate-vcs" @@ -55,7 +55,7 @@ module GatePowerHarness; wire [31:0] axi_aw_addr, axi_ar_addr; wire [7:0] axi_aw_len, axi_ar_len, axi_w_strb; wire [2:0] axi_aw_size, axi_aw_prot, axi_ar_size, axi_ar_prot; wire [3:0] axi_aw_cache, axi_aw_qos, axi_ar_cache, axi_ar_qos; wire [63:0] axi_w_data, axi_r_data; - BBSimDRAM #(.ADDR_BITS(32), .CHIP_ID(0), .CLOCK_HZ(100000000), .DATA_BITS(64), .ID_BITS(4), .LINE_SIZE(64), .MEM_BASE(40'd2147483648), .MEM_SIZE(268435456)) mem(.clock(clock), .reset(reset), .axi_aw_ready(axi_aw_ready), .axi_aw_valid(axi_aw_valid), .axi_aw_bits_id(axi_aw_id), .axi_aw_bits_addr(axi_aw_addr), .axi_aw_bits_len(axi_aw_len), .axi_aw_bits_size(axi_aw_size), .axi_aw_bits_burst(axi_aw_burst), .axi_aw_bits_lock(axi_aw_lock), .axi_aw_bits_cache(axi_aw_cache), .axi_aw_bits_prot(axi_aw_prot), .axi_aw_bits_qos(axi_aw_qos), .axi_w_ready(axi_w_ready), .axi_w_valid(axi_w_valid), .axi_w_bits_data(axi_w_data), .axi_w_bits_strb(axi_w_strb), .axi_w_bits_last(axi_w_last), .axi_b_ready(axi_b_ready), .axi_b_valid(axi_b_valid), .axi_b_bits_id(axi_b_id), .axi_b_bits_resp(axi_b_resp), .axi_ar_ready(axi_ar_ready), .axi_ar_valid(axi_ar_valid), .axi_ar_bits_id(axi_ar_id), .axi_ar_bits_addr(axi_ar_addr), .axi_ar_bits_len(axi_ar_len), .axi_ar_bits_size(axi_ar_size), .axi_ar_bits_burst(axi_ar_burst), .axi_ar_bits_lock(axi_ar_lock), .axi_ar_bits_cache(axi_ar_cache), .axi_ar_bits_prot(axi_ar_prot), .axi_ar_bits_qos(axi_ar_qos), .axi_r_ready(axi_r_ready), .axi_r_valid(axi_r_valid), .axi_r_bits_id(axi_r_id), .axi_r_bits_data(axi_r_data), .axi_r_bits_resp(axi_r_resp), .axi_r_bits_last(axi_r_last)); + BBSimDRAM #(.ADDR_BITS(32), .DEVICE_ID(0), .CLOCK_HZ(100000000), .DATA_BITS(64), .ID_BITS(4), .LINE_SIZE(64), .MEM_BASE(40'd2147483648), .MEM_SIZE(268435456)) mem(.clock(clock), .reset(reset), .axi_aw_ready(axi_aw_ready), .axi_aw_valid(axi_aw_valid), .axi_aw_bits_id(axi_aw_id), .axi_aw_bits_addr(axi_aw_addr), .axi_aw_bits_len(axi_aw_len), .axi_aw_bits_size(axi_aw_size), .axi_aw_bits_burst(axi_aw_burst), .axi_aw_bits_lock(axi_aw_lock), .axi_aw_bits_cache(axi_aw_cache), .axi_aw_bits_prot(axi_aw_prot), .axi_aw_bits_qos(axi_aw_qos), .axi_w_ready(axi_w_ready), .axi_w_valid(axi_w_valid), .axi_w_bits_data(axi_w_data), .axi_w_bits_strb(axi_w_strb), .axi_w_bits_last(axi_w_last), .axi_b_ready(axi_b_ready), .axi_b_valid(axi_b_valid), .axi_b_bits_id(axi_b_id), .axi_b_bits_resp(axi_b_resp), .axi_ar_ready(axi_ar_ready), .axi_ar_valid(axi_ar_valid), .axi_ar_bits_id(axi_ar_id), .axi_ar_bits_addr(axi_ar_addr), .axi_ar_bits_len(axi_ar_len), .axi_ar_bits_size(axi_ar_size), .axi_ar_bits_burst(axi_ar_burst), .axi_ar_bits_lock(axi_ar_lock), .axi_ar_bits_cache(axi_ar_cache), .axi_ar_bits_prot(axi_ar_prot), .axi_ar_bits_qos(axi_ar_qos), .axi_r_ready(axi_r_ready), .axi_r_valid(axi_r_valid), .axi_r_bits_id(axi_r_id), .axi_r_bits_data(axi_r_data), .axi_r_bits_resp(axi_r_resp), .axi_r_bits_last(axi_r_last)); always #5 clock = ~clock; initial begin longint timeout_ns; diff --git a/examples/designs/pebble/sims/device/workloads/CMakeLists.txt b/examples/designs/pebble/sims/device/workloads/CMakeLists.txt new file mode 100644 index 000000000..19ba06a88 --- /dev/null +++ b/examples/designs/pebble/sims/device/workloads/CMakeLists.txt @@ -0,0 +1,75 @@ +include("${DESIGN_DEVICE_BUILD}/device.cmake") + +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/base" + "${CMAKE_BINARY_DIR}/src/CTest/base" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/tlb" + "${CMAKE_BINARY_DIR}/src/CTest/tlb" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/tutorial" + "${CMAKE_BINARY_DIR}/src/CTest/tutorial" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/embench" + "${CMAKE_BINARY_DIR}/src/CTest/embench" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/coremark" + "${CMAKE_BINARY_DIR}/src/CTest/coremark" +) +foreach(BUCKYBALL_COMPONENT_WORKLOAD_TARGET IN LISTS DESIGN_CPU_TARGETS) + set(COMPONENT_WORKLOADS + "${DESIGN_TARGET_WORKLOADS_${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}}" + ) + if( + COMPONENT_WORKLOADS + STREQUAL + "${BUCKYBALL_REPO_ROOT}/examples/designs/${DESIGN_NAME}/workloads" + ) + continue() + endif() + set(BUCKYBALL_CTEST_TARGET "${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}") + set(BUCKYBALL_CTEST_GROUP_TARGET component-ctests-build) + if(IS_DIRECTORY "${COMPONENT_WORKLOADS}/ctests") + add_subdirectory( + "${COMPONENT_WORKLOADS}/ctests" + "${CMAKE_CURRENT_BINARY_DIR}/prefab/${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}/ctests" + ) + endif() +endforeach() +add_dependencies( + ctest-workloads-build + base-ctests-build + tlb-ctests-build + tutorial-ctests-build + component-ctests-build +) +add_dependencies(sync-native-bin embench-all coremark-all) + +add_subdirectory( + "${WORKLOAD_SRC_DIR}/MLIRTest/base" + "${CMAKE_BINARY_DIR}/src/MLIRTest/mlir_tests/base" +) +foreach(BUCKYBALL_TARGET IN LISTS DESIGN_CPU_TARGETS) + set(BUCKYBALL_MLIR_ACTIVE_TARGET "${BUCKYBALL_TARGET}") + foreach(BUCKYBALL_BALL_DIR IN LISTS DESIGN_PROFILES_${BUCKYBALL_TARGET}_THREAD_BALLDOMAIN_BALLS) + set(BALL_MLIR_TESTS + "${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/${BUCKYBALL_BALL_DIR}/workloads/mlir_tests" + ) + if(IS_DIRECTORY "${BALL_MLIR_TESTS}") + add_subdirectory( + "${BALL_MLIR_TESTS}" + "${CMAKE_CURRENT_BINARY_DIR}/balls/${BUCKYBALL_TARGET}/${BUCKYBALL_BALL_DIR}" + ) + endif() + endforeach() +endforeach() +add_dependencies(MLIRTest-all base-mlirtest-build balls-mlir-tests-build) + +add_subdirectory( + "${BUCKYBALL_REPO_ROOT}/examples/designs/pebble/workloads" + design +) diff --git a/examples/designs/pebble/sims/device/workloads/kernel/CMakeLists.txt b/examples/designs/pebble/sims/device/workloads/kernel/CMakeLists.txt new file mode 100644 index 000000000..13c2e977f --- /dev/null +++ b/examples/designs/pebble/sims/device/workloads/kernel/CMakeLists.txt @@ -0,0 +1,44 @@ +# Linux programs installed into the pebble rootfs, each with the hart it runs on. +set(BUCKYBALL_KERNEL_PROGRAMS + 0 pebble-pebble-mlirtest-core_mega_matmul-linux + 0 pebble-pebble-mlirtest-core_mega_conv2d-linux + 0 pebble-pebble-mlirtest-core_mega_conv2d_resident-linux + 0 pebble-pebble-mlirtest-core_mega_conv2d_flatten_linear-linux + 0 pebble-pebble-mlirtest-core_lookup_channel_i8-linux + 0 pebble-pebble-mlirtest-core_generic_matmul_i8-linux + 0 pebble-pebble-mlirtest-core_linear_int8-linux + 0 pebble-pebble-mlirtest-core_linear_chain_i8-linux + 0 pebble-pebble-mlirtest-core_mega_conv2d_k5_tail-linux + 0 pebble-pebble-mlirtest-core_mega_matmul_m17_k65_n17-linux + 0 pebble-pebble-ctest-mvin_2d_test-linux + 0 pebble-pebble-ctest-mvin_2d_pixel_stride_test-linux + 0 pebble-pebble-ctest-mvin_mvout_acc_test-linux + 0 pebble-pebble-ctest-mvin_mvout_test-linux + 0 pebble-pebble-ctest-mvin_mvout_stride_test-linux + 0 pebble-pebble-ctest-mvin_mvout_unaligned_test-linux + 0 pebble-pebble-ctest-mvin_unaligned_test-linux + 0 pebble-pebble-mlirtest-linalg_mega_conv2d_k3_c16-linux + 0 pebble-pebble-mlirtest-core_mega_conv2d_s2_depthwise_s2-linux + 0 pebble-pebble-mlirtest-core_mega_global_avg_conv_exact-linux + 0 pebble-pebble-mlirtest-core_mega_global_avg_conv2_exact-linux + 0 pebble-pebble-mlirtest-core_mega_global_avg_7x7_c288_exact-linux + 0 pebble-pebble-mlirtest-core_mega_se_gate_conv-linux + 0 pebble-pebble-ctest-mega_conv_pipeline_test-linux + 0 pebble-pebble-ctest-residual_add_lut_test-linux + 0 pebble-pebble-ctest-maxpool_im2col_test-linux + 0 pebble-pebble-ctest-se_gate_lut_test-linux + 0 pebble-pebble-ctest-int8mul_large_gate_row_test-linux + 0 pebble-pebble-ctest-dual_smatmul_matadd_test-linux + 0 pebble-pebble-ctest-dual_conv_add_int2fp_test-linux + 0 pebble-pebble-mlirtest-bank_matadd_i32_elementwise-linux + 0 pebble-pebble-mlirtest-bank_int8add_identity_scale-linux + 0 pebble-pebble-mlirtest-bank_lut_identity_table-linux + 0 pebble-pebble-mlirtest-bank_int8mul_unit_gate-linux + 0 pebble-pebble-mlirtest-bank_maxpool_k3_padded_spatial-linux + 0 pebble-pebble-mlirtest-bank_smatmul_bias_two_terms-linux + 0 pebble-pebble-mlirtest-bank_int2fp_unit_scale-linux + 0 pebble-pebble-mlirtest-bank_quant_i32_pack16-linux + 0 pebble-pebble-mlirtest-bank_quant_f32_pack16-linux + 0 pebble-pebble-mlirtest-bank_im2col_k3_two_blocks-linux + 0 pebble-pebble-mlirtest-bank_transpose_i8_packed_matrix-linux + PARENT_SCOPE) diff --git a/examples/designs/pebble/sims/ip/uvm/registry.toml b/examples/designs/pebble/sims/ip/uvm/registry.toml new file mode 100644 index 000000000..8e8131449 --- /dev/null +++ b/examples/designs/pebble/sims/ip/uvm/registry.toml @@ -0,0 +1 @@ +ips = [] diff --git a/examples/designs/pebble/stack/device/CMakeLists.txt b/examples/designs/pebble/stack/device/CMakeLists.txt new file mode 100644 index 000000000..995c43d41 --- /dev/null +++ b/examples/designs/pebble/stack/device/CMakeLists.txt @@ -0,0 +1,37 @@ +# ===----------------------------------------------------------------------=== +# The pebble device: its cores, their hardware threads and the runtime that +# schedules jobs onto the cores. +# +# bbdev stack --build "--design pebble --device " +# +# builds every thread and core stack of the device first, then configures this +# with DESIGN_CMAKE, generated from design.pb, and the RISC-V Linux toolchain. +# Products, exported in device.cmake for the programs that run on the device: +# runtime, runtime-baremetal, runtime-linux job scheduling over the cores +# engine hardware threads and instruction streams +# bindings how each core runs jobs and each +# hardware thread is loaded +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(pebble-device LANGUAGES C CXX) +include("${DESIGN_CMAKE}") +set(CMAKE_POSITION_INDEPENDENT_CODE ON) + +# How each core runs jobs, and how each hardware thread is loaded. Nothing +# references these bindings, so they are linked as objects rather than taken +# from an archive; as link options they come before the libraries they call. +add_library(bindings INTERFACE) +foreach(CORE IN LISTS DESIGN_CORES) + target_link_options(bindings INTERFACE "${DESIGN_CORE_BUILD_${CORE}}/execute.o") +endforeach() +foreach(THREAD IN LISTS DESIGN_THREAD_STACKS) + target_link_options(bindings INTERFACE "${DESIGN_THREAD_BUILD_${THREAD}}/loader.o") +endforeach() +# DMA helpers are sized by the target with the fewest bank lines. +list(GET DESIGN_RUNTIME_TARGETS 0 PARAMS_TARGET) +set(RUNTIME_DEFS ${DESIGN_TARGET_DEFS_${PARAMS_TARGET}}) + +add_subdirectory("${DESIGN_REPO}/stack/engine" engine) +add_subdirectory("${DESIGN_REPO}/stack/runtime" runtime) +export(TARGETS bindings runtime runtime-baremetal runtime-linux engine + runtime-shared runtime-linux-shared engine-shared FILE device.cmake) diff --git a/examples/designs/pebble/stack/device/core/CMakeLists.txt b/examples/designs/pebble/stack/device/core/CMakeLists.txt new file mode 100644 index 000000000..48995af5d --- /dev/null +++ b/examples/designs/pebble/stack/device/core/CMakeLists.txt @@ -0,0 +1,61 @@ +# ===----------------------------------------------------------------------=== +# Pebble's own core: a CPU with its Ball accelerator. +# +# Built by the stack top of each simulation entry, configured with DESIGN_CMAKE +# and CORE. Products: +# execute.o how the core runs jobs (direct execution) +# libpebble-compiler.so buddy-opt plugin: the core's fusion +# (form-custom-ops) and lower-buckyball around the +# framework bank layer +# pebble-compiler-check FileCheck tests of that plugin +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(pebble-core-stack LANGUAGES C CXX) +include("${DESIGN_CMAKE}") + +set(RUNTIME "${DESIGN_REPO}/stack/runtime") +set(TARGET ${DESIGN_DEVICE_CORES_${DESIGN_CORE_HART_${CORE}}_PROFILE}) +find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) + +# How the core runs jobs. +add_custom_command( + OUTPUT "${CMAKE_BINARY_DIR}/execute.o" + COMMAND "${LINUX_CC}" -O2 -fPIC -ffreestanding -fno-builtin -msmall-data-limit=0 + "-DCORE_SIGNATURE=${DESIGN_DEVICE_CORES_${DESIGN_CORE_HART_${CORE}}_SIGNATURE}ULL" "-I${RUNTIME}/scheduler" + -c "${RUNTIME}/execute/direct.c" -o "${CMAKE_BINARY_DIR}/execute.o" + DEPENDS "${RUNTIME}/execute/direct.c" "${RUNTIME}/scheduler/runtime.h" + VERBATIM) +add_custom_target(execute ALL DEPENDS "${CMAKE_BINARY_DIR}/execute.o") + +# The core's compiler plugin. The core owns computation recognition and its +# conversion to bank dataflow. +set(BUCKYBALL_BALLS ${DESIGN_TARGET_BALL_STACKS_${TARGET}}) +file(GLOB_RECURSE BUCKYBALL_SOURCES CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp") +add_subdirectory("${DESIGN_REPO}/stack/compiler/midend" compiler) +add_library(${TARGET}-compiler MODULE) +target_link_libraries(${TARGET}-compiler PRIVATE buckyball-compiler) +set_target_properties(${TARGET}-compiler PROPERTIES + LINKER_LANGUAGE CXX LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") + +set(CHECK "${DESIGN_MLIR}/../../../bin/FileCheck") +set(OPT "${DESIGN_COMPILER}/bin/buddy-opt --load-dialect-plugin=$ --load-pass-plugin=$ --buckyball-design-pb=${DESIGN_PB} --buckyball-target=${TARGET}") +set(TESTS "${CMAKE_CURRENT_SOURCE_DIR}/tests") +set(CORE_TESTS "${CMAKE_CURRENT_SOURCE_DIR}/../../../workloads/device/core/acc/mlir_tests/core") +add_custom_target(${TARGET}-compiler-check + COMMAND bash -o pipefail -c "${OPT} ${TESTS}/form-custom-ops.mlir -allow-unregistered-dialect -form-custom-ops | ${CHECK} ${TESTS}/form-custom-ops.mlir --implicit-check-not=buckyball." + COMMAND bash -o pipefail -c "${OPT} ${TESTS}/form-int32-bias.mlir -allow-unregistered-dialect -form-custom-ops | ${CHECK} ${TESTS}/form-int32-bias.mlir --implicit-check-not=buckyball." + COMMAND bash -o pipefail -c "${OPT} ${TESTS}/lower-frontend-types.mlir -allow-unregistered-dialect -lower-frontend -split-input-file -verify-diagnostics | ${CHECK} ${TESTS}/lower-frontend-types.mlir --check-prefix=VALID --implicit-check-not=frontend." + COMMAND bash -o pipefail -c "${OPT} ${TESTS}/lower-custom-ops.mlir -convert-linalg-to-buckyball | ${CHECK} ${TESTS}/lower-custom-ops.mlir --implicit-check-not=custom.op" + COMMAND bash -o pipefail -c "${OPT} ${TESTS}/unknown-custom-op.mlir -convert-linalg-to-buckyball -verify-diagnostics -o /dev/null" + COMMAND bash -o pipefail -c "${OPT} ${CORE_TESTS}/mega_conv2d.mlir -allow-unregistered-dialect -form-custom-ops | ${CHECK} ${CORE_TESTS}/mega_conv2d.mlir --check-prefix=FORM" + COMMAND bash -o pipefail -c "${OPT} ${CORE_TESTS}/mega_conv2d.mlir -allow-unregistered-dialect -form-custom-ops -convert-linalg-to-buckyball | ${CHECK} ${CORE_TESTS}/mega_conv2d.mlir --check-prefix=BANK" + COMMAND bash -o pipefail -c "${OPT} ${CORE_TESTS}/linear_chain_i8.mlir -allow-unregistered-dialect -form-custom-ops | ${CHECK} ${CORE_TESTS}/linear_chain_i8.mlir --check-prefix=FORM" + COMMAND bash -o pipefail -c "${OPT} ${CORE_TESTS}/linear_chain_i8.mlir -allow-unregistered-dialect -form-custom-ops -convert-linalg-to-buckyball | ${CHECK} ${CORE_TESTS}/linear_chain_i8.mlir --check-prefix=BANK" + COMMAND bash -o pipefail -c "${OPT} ${CORE_TESTS}/mega_conv2d_resident.mlir -allow-unregistered-dialect -form-custom-ops | ${CHECK} ${CORE_TESTS}/mega_conv2d_resident.mlir --check-prefix=FORM" + COMMAND bash -o pipefail -c "${OPT} ${CORE_TESTS}/mega_conv2d_resident.mlir -allow-unregistered-dialect -form-custom-ops -convert-linalg-to-buckyball | ${CHECK} ${CORE_TESTS}/mega_conv2d_resident.mlir --check-prefix=BANK" + COMMAND bash -o pipefail -c "${OPT} ${CORE_TESTS}/mega_conv2d_flatten_linear.mlir -allow-unregistered-dialect -form-custom-ops | ${CHECK} ${CORE_TESTS}/mega_conv2d_flatten_linear.mlir --check-prefix=FORM" + COMMAND bash -o pipefail -c "${OPT} ${CORE_TESTS}/mega_conv2d_flatten_linear.mlir -allow-unregistered-dialect -form-custom-ops -convert-linalg-to-buckyball | ${CHECK} ${CORE_TESTS}/mega_conv2d_flatten_linear.mlir --check-prefix=BANK" + COMMAND bash -o pipefail -c "${OPT} ${TESTS}/form-depthwise.mlir -allow-unregistered-dialect -form-custom-ops | ${CHECK} ${TESTS}/form-depthwise.mlir --implicit-check-not=linalg.depthwise" + COMMAND bash -o pipefail -c "${OPT} ${TESTS}/lower-bank-updates.mlir -lower-buckyball | ${CHECK} ${TESTS}/lower-bank-updates.mlir" + DEPENDS ${TARGET}-compiler + VERBATIM) diff --git a/examples/designs/pebble/stack/device/core/src/ConvertLinalgToBuckyball.cpp b/examples/designs/pebble/stack/device/core/src/ConvertLinalgToBuckyball.cpp new file mode 100644 index 000000000..3cc742876 --- /dev/null +++ b/examples/designs/pebble/stack/device/core/src/ConvertLinalgToBuckyball.cpp @@ -0,0 +1,197 @@ +//===- ConvertLinalgToBuckyball.cpp - Pebble core lowering ----------------===// +// +// Lowers the custom computations to bank dataflow: the core tiles them at +// bank granularity and connects the banks; every Ball computation is one +// virtual-bank operation of the Ball's dialect, its contract. +// +//===----------------------------------------------------------------------===// + +#include "Buckyball/BuckyballDialect.h" +#include "Buckyball/BuckyballOps.h" +#include "Custom/CustomOps.h" +#include "Lowering.h" +#include "Target/BuckyballTargetRegistry.h" +#include "im2colOps.h" +#include "int2fpOps.h" +#include "int8mulOps.h" +#include "lutOps.h" +#include "mataddOps.h" +#include "maxpoolOps.h" +#include "smatmulOps.h" +#include "toint8Ops.h" +#include "transposeOps.h" + +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Bufferization/IR/Bufferization.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/Dialect/SCF/IR/SCF.h" +#include "mlir/Dialect/Tensor/IR/Tensor.h" +#include "mlir/Pass/Pass.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +// The recipe supplies contiguous tensors; offsets may point into packed +// weights. +Value pebble::tensorBuffer(OpBuilder &builder, Location loc, Value tensor) { + auto type = cast(tensor.getType()); + SmallVector strides(type.getRank()); + int64_t stride = 1; + for (int64_t dim = type.getRank() - 1; dim >= 0; --dim) { + strides[dim] = stride; + stride *= type.getDimSize(dim); + } + auto layout = StridedLayoutAttr::get(builder.getContext(), + ShapedType::kDynamic, strides); + auto memory = MemRefType::get(type.getShape(), type.getElementType(), layout); + return bufferization::ToBufferOp::create(builder, loc, memory, tensor, true); +} + +SmallVector pebble::repeat( + OpBuilder &builder, Location loc, int64_t begin, int64_t end, int64_t step, + bool unroll, ValueRange carried, + function_ref(OpBuilder &, Value, ValueRange)> body) { + SmallVector values(carried); + if (unroll || end - begin <= step) { + for (int64_t index = begin; index < end; index += step) + values = body( + builder, arith::ConstantIndexOp::create(builder, loc, index), values); + return values; + } + auto loop = scf::ForOp::create( + builder, loc, arith::ConstantIndexOp::create(builder, loc, begin), + arith::ConstantIndexOp::create(builder, loc, end), + arith::ConstantIndexOp::create(builder, loc, step), carried, + [&](OpBuilder &nested, Location loc, Value index, ValueRange args) { + scf::YieldOp::create(nested, loc, body(nested, index, args)); + }); + return loop.getResults(); +} + +int64_t pebble::matmulRows() { + return buckyball_target::getBuckyballTarget().bankDepth / 4; +} + +int64_t pebble::tileRows() { + return buckyball_target::getBuckyballBallParam("SMATMUL_OS", "tileRows"); +} + +int64_t pebble::im2colGroups(int64_t kernel, int64_t blocks, int64_t windows) { + int64_t depth = buckyball_target::getBuckyballTarget().bankDepth; + int64_t rows = (windows + tileRows() - 1) / tileRows() * tileRows(); + int64_t tiles = kernel * kernel * blocks; + // A group holds rows x 16 tile lines of A and 16 lines per tile of B. + int64_t perGroup = std::min(depth / rows, depth / 16); + for (int64_t groups = 1; groups <= 9; ++groups) + if (tiles % groups == 0 && tiles / groups <= perGroup) + return groups; + return 0; +} + +pebble::ConvTiling pebble::convTiling(int64_t height, int64_t width, + int64_t pool, int64_t stride, + int64_t kernel, int64_t blocks, + bool depthwise, bool floating) { + int64_t depth = buckyball_target::getBuckyballTarget().bankDepth; + int64_t maxSide = + buckyball_target::getBuckyballBallParam("IM2COL", "maxIter"); + int64_t largest = std::max(height, width) * pool; + for (int64_t side = largest; side >= pool; side -= 1) { + if (side % pool || + (side * side + tileRows() - 1) / tileRows() * tileRows() > matmulRows()) + continue; + int64_t inputSide = (side - 1) * stride + kernel; + if (inputSide > maxSide || + inputSide * inputSide * (floating ? 4 : 1) > depth) + continue; + int64_t limit = depthwise ? 1 : std::min(blocks, 8); + for (int64_t chunk = limit; chunk >= 1; --chunk) + if (im2colGroups(kernel, chunk, side * side)) + return {side, inputSide, chunk}; + } + return {0, 0, 0}; +} + +void pebble::observeResident(Operation *op, Value bank) { + for (Operation *user : llvm::make_early_inc_range(op->getUsers())) { + if (user->getName().getStringRef() != "buddy_trace.end") + continue; + OpBuilder builder(user); + Operation *trace = builder.clone(*user); + trace->setOperand(0, bank); + trace->getResult(0).setType(bank.getType()); + user->erase(); + } +} + +namespace { +class ConvertLinalgToBuckyballPass + : public PassWrapper> { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(ConvertLinalgToBuckyballPass) + StringRef getArgument() const final { return "convert-linalg-to-buckyball"; } + StringRef getDescription() const final { + return "Lower custom tensor computations to the core bank dataflow"; + } + void getDependentDialects(DialectRegistry ®istry) const override { + registry.insert(); + } + void runOnOperation() override { + DenseMap residentVectors; + SmallVector consumed; + if (failed( + pebble::lowerSpatialOps(getOperation(), residentVectors, consumed))) + return signalPassFailure(); + SmallVector<::buddy::custom::CustomOp> computations; + getOperation().walk([&](::buddy::custom::CustomOp op) { + if (!llvm::is_contained(consumed, op.getOperation())) + computations.push_back(op); + }); + for (auto op : computations) { + StringRef kind = op.getKind(); + Value result, resident; + LogicalResult status = failure(); + if (kind == "global_avg_i8") + status = pebble::lowerGlobalAverage(op, result); + else if (kind == "lookup_i8" || kind == "gate_mul_i8") + status = pebble::lowerChannelOp(op, result); + else if (kind == "transpose_i8") + status = pebble::lowerTranspose(op, result); + else if (kind == "linear_i8" || kind == "linear_i8_requant" || + kind == "matmul_i8") + status = pebble::lowerLinear( + op, residentVectors.lookup(op.getInputs()[0]), result, resident); + else + op.emitOpError("has no bank lowering on this core for ") << kind; + if (failed(status)) + return signalPassFailure(); + if (resident) { + residentVectors[op.getResult(0)] = resident; + pebble::observeResident(op, resident); + consumed.push_back(op); + continue; + } + op.getResult(0).replaceAllUsesWith(result); + op.erase(); + } + for (Operation *op : llvm::reverse(consumed)) + op->erase(); + } +}; + +PassRegistration registration; +} // namespace diff --git a/examples/designs/pebble/stack/device/core/src/FormCustomOps.cpp b/examples/designs/pebble/stack/device/core/src/FormCustomOps.cpp new file mode 100644 index 000000000..f5b7fa500 --- /dev/null +++ b/examples/designs/pebble/stack/device/core/src/FormCustomOps.cpp @@ -0,0 +1,311 @@ +//===- FormCustomOps.cpp - Recognize and fuse core computations -----------===// + +#include "Custom/CustomOps.h" +#include "Lowering.h" +#include "Target/BuckyballTargetRegistry.h" +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/Linalg/Transforms/Transforms.h" +#include "mlir/IR/Dominance.h" +#include "mlir/IR/Matchers.h" +#include "mlir/IR/PatternMatch.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" +#include "mlir/Pass/Pass.h" + +using namespace mlir; +using ::buddy::custom::CustomOp; + +namespace { +Operation *frontend(Value value, StringRef name) { + Operation *op = value.getDefiningOp(); + return op && op->getName().getStringRef() == name ? op : nullptr; +} + +} // namespace + +bool pebble::elementwise(linalg::GenericOp op, StringRef name) { + if (!op || !op.hasPureTensorSemantics() || + !linalg::isaElemwiseSingleBinaryOpInterface(op) || + op.getBody()->front().getName().getStringRef() != name) + return false; + auto result = cast(op.getResult(0).getType()); + auto maps = op.getIndexingMapsArray(); + if (!result.hasStaticShape() || !maps.back().isIdentity()) + return false; + for (auto [index, input] : llvm::enumerate(op.getInputs())) { + auto type = cast(input.getType()); + if (!type.hasStaticShape() || type.getRank() != result.getRank()) + return false; + SmallVector dimensions; + for (int64_t dim = 0; dim < type.getRank(); ++dim) + dimensions.push_back(type.getDimSize(dim) == 1 && + result.getDimSize(dim) != 1 + ? getAffineConstantExpr(0, op.getContext()) + : getAffineDimExpr(dim, op.getContext())); + if (maps[index] != + AffineMap::get(type.getRank(), 0, dimensions, op.getContext())) + return false; + } + return true; +} + +namespace { +class FormCustomOpsPass + : public PassWrapper> { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(FormCustomOpsPass) + StringRef getArgument() const final { return "form-custom-ops"; } + StringRef getDescription() const final { + return "Recognize core computations and fuse them into custom tensor ops"; + } + void getDependentDialects(DialectRegistry ®istry) const override { + registry.insert<::buddy::custom::CustomDialect, linalg::LinalgDialect>(); + } + void runOnOperation() override { + // MLIR checks both indexing maps and scalar computations, including casts. + SmallVector generics; + getOperation().walk([&](linalg::GenericOp op) { generics.push_back(op); }); + IRRewriter rewriter(&getContext()); + for (auto op : generics) { + if (op.hasPureTensorSemantics() && + (linalg::isaContractionOpInterface(op) || + linalg::isaTransposeOpInterface(op))) { + rewriter.setInsertionPoint(op); + (void)linalg::specializeGenericOp(rewriter, op); + } + } + + SmallVector matmuls; + getOperation().walk([&](linalg::MatmulOp op) { matmuls.push_back(op); }); + DominanceInfo dominance(getOperation()); + for (auto op : matmuls) { + if (!op.hasPureTensorSemantics() || op.hasUserDefinedMaps() || + op.getCast() != linalg::TypeFn::cast_signed) + continue; + rewriter.setInsertionPoint(op); + auto a = cast(op.getInputs()[0].getType()); + auto b = cast(op.getInputs()[1].getType()); + auto result = cast(op.getResult(0).getType()); + if (a.getElementType().isInteger(8) && b.getElementType().isInteger(8) && + result.getElementType().isInteger(32)) { + // matmul_i8(A, B, C) = C + A @ B, with signed INT8 operands. + auto custom = CustomOp::create( + rewriter, op.getLoc(), TypeRange{result}, "matmul_i8", + ValueRange{op.getInputs()[0], op.getInputs()[1], + op.getOutputs()[0]}); + rewriter.replaceOp(op, custom.getOutputs()); + continue; + } + + Operation *lhs = frontend(op.getInputs()[0], "frontend.int8_to_fp32"); + Operation *rhs = frontend(op.getInputs()[1], "frontend.int8_to_fp32"); + if (!lhs || !rhs || !matchPattern(op.getOutputs()[0], m_AnyZeroFloat())) + continue; + Operation *quantize = + frontend(lhs->getOperand(0), "frontend.fp32_to_int8"); + if (!quantize) + continue; + // Match bias dequantized with exactly input_step * weight_step. + Value biasedResult = op.getResult(0); + Operation *trace = nullptr; + if (biasedResult.hasOneUse()) { + Operation *user = *biasedResult.getUsers().begin(); + auto tensorTrace = user->getAttrOfType("tensor"); + // Cycle markers follow the logical Linear. Tensor traces keep their + // observed boundary and therefore prevent folding bias across it. + if (user->getName().getStringRef() == "buddy_trace.end" && + tensorTrace && !tensorTrace.getValue()) { + trace = user; + biasedResult = user->getResult(0); + } + } + Value bias; + linalg::GenericOp biasAdd, biasScale; + Operation *biasDequant = nullptr; + if (biasedResult.hasOneUse()) { + auto add = + dyn_cast(*biasedResult.getUsers().begin()); + if (pebble::elementwise(add, "arith.addf") && + llvm::is_contained(add.getInputs(), biasedResult)) { + Value other = + add.getInputs()[add.getInputs()[0] == biasedResult ? 1 : 0]; + auto dequant = frontend(other, "frontend.int32_to_fp32"); + if (dequant) { + auto scale = + dequant->getOperand(1).getDefiningOp(); + auto biasType = + cast(dequant->getOperand(0).getType()); + if (pebble::elementwise(scale, "arith.mulf") && + biasType.getElementType().isInteger(32) && + biasType.getShape() == result.getShape() && + ((scale.getInputs()[0] == lhs->getOperand(1) && + scale.getInputs()[1] == rhs->getOperand(1)) || + (scale.getInputs()[1] == lhs->getOperand(1) && + scale.getInputs()[0] == rhs->getOperand(1))) && + dominance.properlyDominates(dequant->getOperand(0), op)) { + bias = dequant->getOperand(0); + biasAdd = add; + biasScale = scale; + biasDequant = dequant; + } + } + } + } + if (!bias) { + auto biasType = + RankedTensorType::get(result.getShape(), rewriter.getI32Type()); + bias = arith::ConstantOp::create(rewriter, op.getLoc(), biasType, + rewriter.getZeroAttr(biasType)); + } + // linear_i8(x, W, bias, multiplier, input_step, weight_step) accumulates + // signed INT8 products on INT32 bias, then restores FP32 values. + auto custom = CustomOp::create( + rewriter, op.getLoc(), TypeRange{result}, "linear_i8", + ValueRange{quantize->getOperand(0), rhs->getOperand(0), bias, + quantize->getOperand(1), lhs->getOperand(1), + rhs->getOperand(1)}); + rewriter.replaceOp(op, custom.getOutputs()); + if (biasAdd) { + rewriter.replaceOp(biasAdd, + trace ? trace->getResult(0) : custom.getResult(0)); + if (biasDequant->use_empty()) + rewriter.eraseOp(biasDequant); + if (biasScale->use_empty()) + rewriter.eraseOp(biasScale); + } + if (rhs != lhs && rhs->use_empty()) + rewriter.eraseOp(rhs); + if (lhs->use_empty()) + rewriter.eraseOp(lhs); + if (quantize->use_empty()) + rewriter.eraseOp(quantize); + } + + SmallVector transposes; + getOperation().walk( + [&](linalg::TransposeOp op) { transposes.push_back(op); }); + const auto &target = buckyball_target::getBuckyballTarget(); + for (auto op : transposes) { + // TRANSPOSE holds the whole matrix in two groups of bank columns of at + // most 31 banks each. + auto type = dyn_cast(op.getInput().getType()); + if (!type || type.getRank() != 2 || !type.hasStaticShape() || + !type.getElementType().isInteger(8) || + op.getPermutation() != ArrayRef({1, 0}) || + type.getDimSize(0) > target.bankDepth || type.getDimSize(1) % 16 || + type.getDimSize(1) / 16 > 31 || + 2 * type.getDimSize(1) / 16 > target.bankNum) + continue; + rewriter.setInsertionPoint(op); + auto custom = + CustomOp::create(rewriter, op.getLoc(), op->getResultTypes(), + "transpose_i8", ValueRange{op.getInput()}); + rewriter.replaceOp(op, custom.getOutputs()); + } + pebble::formSpatialOps(getOperation()); + + // A one-row Linear whose single consumer chain [relu] ends in another + // one-row Linear requantizes for it. Its INT8 row stays in banks, one + // group per sixteen columns, when both Linears fit the banks with it. + SmallVector linears; + getOperation().walk([&](CustomOp op) { + if (op.getKind() == "linear_i8") + linears.push_back(op); + }); + auto oneRow = [](CustomOp op) { + return cast(op.getResult(0).getType()).getDimSize(0) == + 1; + }; + for (CustomOp op : linears) { + if (!oneRow(op)) + continue; + Value value = op.getResult(0); + SmallVector links, traces; + bool relu = false; + while (value.hasOneUse()) { + Operation *user = *value.getUsers().begin(); + if (user->getBlock() != op->getBlock()) + break; + if (user->getName().getStringRef() == "buddy_trace.end") { + if (cast(user->getAttr("tensor")).getValue()) + break; + traces.push_back(user); + } else { + auto maximum = dyn_cast(user); + if (relu || !pebble::elementwise(maximum, "arith.maximumf") || + !llvm::is_contained(maximum.getInputs(), value) || + !matchPattern( + maximum.getInputs()[maximum.getInputs()[0] == value ? 1 : 0], + m_AnyZeroFloat())) + break; + relu = true; + } + links.push_back(user); + value = user->getResult(0); + } + if (!value.hasOneUse()) + continue; + auto next = dyn_cast(*value.getUsers().begin()); + if (!next || next.getKind() != "linear_i8" || !oneRow(next) || + next.getInputs()[0] != value || + !dominance.properlyDominates(next.getInputs()[3], op)) + continue; + bool barrier = false; + for (Operation *between = op->getNextNode(); between != next; + between = between->getNextNode()) + if (!isMemoryEffectFree(between) && + between->getName().getDialectNamespace() != "buddy_trace") { + barrier = true; + break; + } + if (barrier) + continue; + // Banks of the producer: its A (and FP32 source) and B, or the resident + // row and B, the row it builds and four more; the consumer reads the + // row with as many groups of B. + auto input = op.getInputs()[0]; + auto producer = input.getDefiningOp(); + bool residentInput = producer && (producer->hasAttr("custom.resident") || + producer.getKind() == "flatten_nchw"); + auto inputType = cast(input.getType()); + int64_t k = inputType.getDimSize(1); + int64_t n = cast(value.getType()).getDimSize(1); + int64_t rowGroups = (n + 15) / 16; + int64_t inputBanks; + if (residentInput) { + inputBanks = producer.getKind() == "flatten_nchw" ? 2 : 2 * (k / 16); + } else { + bool floating = inputType.getElementType().isF32(); + inputBanks = + (floating ? 3 : 2) * pebble::linearSplit(k, 1, floating).groups; + } + bool resident = n % 16 == 0 && + inputBanks + rowGroups + 4 <= target.bankNum - 1 && + 2 * rowGroups + 4 <= target.bankNum - 1; + rewriter.setInsertionPoint(op); + SmallVector inputs(op.getInputs()); + inputs.push_back(next.getInputs()[3]); + auto fused = CustomOp::create( + rewriter, op.getLoc(), + TypeRange{RankedTensorType::get({1, n}, rewriter.getI8Type())}, + "linear_i8", inputs); + fused->setAttr("relu", rewriter.getBoolAttr(relu)); + if (resident) + fused->setAttr("custom.resident", rewriter.getUnitAttr()); + rewriter.setInsertionPointAfter(fused); + for (Operation *trace : traces) { + Operation *moved = rewriter.clone(*trace); + moved->setOperand(0, fused.getResult(0)); + moved->getResult(0).setType(fused.getResult(0).getType()); + } + next->setOperand(0, fused.getResult(0)); + for (Operation *link : llvm::reverse(links)) + rewriter.eraseOp(link); + rewriter.eraseOp(op); + } + } +}; + +PassRegistration registration; +} // namespace diff --git a/examples/designs/pebble/stack/device/core/src/FormSpatialOps.cpp b/examples/designs/pebble/stack/device/core/src/FormSpatialOps.cpp new file mode 100644 index 000000000..f3d3befd2 --- /dev/null +++ b/examples/designs/pebble/stack/device/core/src/FormSpatialOps.cpp @@ -0,0 +1,376 @@ +//===- FormSpatialOps.cpp - Recognize quantized convolution chains --------===// +#include "Custom/CustomOps.h" +#include "Lowering.h" +#include "Target/BuckyballTargetRegistry.h" +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/Tensor/IR/Tensor.h" +#include "mlir/IR/Dominance.h" +#include "mlir/IR/Matchers.h" +#include "mlir/IR/PatternMatch.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" +#include + +using namespace mlir; +using ::buddy::custom::CustomOp; + +namespace { +Operation *frontend(Value value, StringRef name) { + Operation *op = value.getDefiningOp(); + return op && op->getName().getStringRef() == name ? op : nullptr; +} +} // namespace + +void pebble::formSpatialOps(func::FuncOp function) { + IRRewriter rewriter(function.getContext()); + SmallVector convolutions; + function.walk([&](Operation *op) { + if (isa(op)) + convolutions.push_back(op); + }); + for (Operation *op : convolutions) { + if (!cast(op).hasPureTensorSemantics()) + continue; + bool depthwise = isa(op); + auto transpose = op->getOperand(1).getDefiningOp(); + if (!transpose || transpose.getPermutation() != + (depthwise ? ArrayRef({2, 3, 0, 1}) + : ArrayRef({0, 2, 3, 1}))) + continue; + Value input = op->getOperand(0); + auto pad = input.getDefiningOp(); + SmallVector padding{0, 0, 0, 0}; + if (pad) { + auto low = pad.getStaticLow(), high = pad.getStaticHigh(); + Value padValue = pad.getConstantPaddingValue(); + if (llvm::is_contained(low, ShapedType::kDynamic) || + llvm::is_contained(high, ShapedType::kDynamic) || low[0] || low[3] || + high[0] || high[3] || + (!padValue || !matchPattern(padValue, m_AnyZeroFloat()))) + continue; + padding = {low[1], high[1], low[2], high[2]}; + input = pad.getSource(); + } + auto lhs = frontend(input, "frontend.int8_to_fp32"); + auto rhs = frontend(transpose.getInput(), "frontend.int8_to_fp32"); + if (!lhs || !rhs) + continue; + auto quant = frontend(lhs->getOperand(0), "frontend.fp32_to_int8"); + // IM2COL takes square kernels up to its size with unit dilation, and the + // array tiles a single image with one stride. + auto weightType = cast(rhs->getOperand(0).getType()); + auto inputType = cast(input.getType()); + auto outputType = cast(op->getResult(0).getType()); + auto strides = + llvm::to_vector(cast(op->getAttr("strides")) + .getValues()); + auto dilations = + llvm::to_vector(cast(op->getAttr("dilations")) + .getValues()); + if (!weightType.hasStaticShape() || !inputType.hasStaticShape() || + !outputType.hasStaticShape() || inputType.getDimSize(0) != 1 || + weightType.getDimSize(2) != weightType.getDimSize(3) || + weightType.getDimSize(2) > + buckyball_target::getBuckyballBallParam("IM2COL", "maxKSize") || + strides[0] != strides[1] || dilations != SmallVector{1, 1} || + !pebble::convTiling(outputType.getDimSize(1), outputType.getDimSize(2), + 1, strides[0], weightType.getDimSize(2), + (inputType.getDimSize(3) + 15) / 16, depthwise, + true) + .side) + continue; + Operation *end = op; + Value biasValue; + Operation *initial = op->getOperand(2).getDefiningOp(); + tensor::CollapseShapeOp collapsed; + if (depthwise) { + auto type = cast(op->getResult(0).getType()); + Value zero = op->getOperand(2); + if (auto fill = zero.getDefiningOp()) + zero = fill.getInputs()[0]; + if (type.getDimSize(4) != 1 || !matchPattern(zero, m_AnyZeroFloat()) || + !op->getResult(0).hasOneUse()) + continue; + collapsed = dyn_cast( + *op->getResult(0).getUsers().begin()); + if (!collapsed || !collapsed.getResult().hasOneUse() || + collapsed.getReassociationIndices() != + SmallVector{{0}, {1}, {2}, {3, 4}}) + continue; + auto add = dyn_cast( + *collapsed.getResult().getUsers().begin()); + if (!add || !linalg::isaElemwiseSingleBinaryOpInterface(add) || + !isa(add.getBody()->front())) + continue; + int biasIndex = add.getInputs()[0] == collapsed.getResult() ? 1 : 0; + auto maps = add.getIndexingMapsArray(); + if (maps[biasIndex] != + AffineMap::get(4, 0, rewriter.getAffineDimExpr(3)) || + !maps[1 - biasIndex].isIdentity() || !maps[2].isIdentity()) + continue; + biasValue = add.getInputs()[biasIndex]; + end = add; + } else { + auto broadcast = dyn_cast_or_null(initial); + if (!broadcast || broadcast.getInputs().size() != 1 || + broadcast.getBody()->getOperations().size() != 1) + continue; + auto maps = broadcast.getIndexingMapsArray(); + if (maps[0] != AffineMap::get(4, 0, rewriter.getAffineDimExpr(3)) || + !maps[1].isIdentity() || + broadcast.getBody()->getTerminator()->getOperand(0) != + broadcast.getBody()->getArgument(0)) + continue; + biasValue = broadcast.getInputs()[0]; + } + auto bias = frontend(biasValue, "frontend.int32_to_fp32"); + if (!quant || !bias) + continue; + auto reshape = bias->getOperand(1).getDefiningOp(); + auto scale = reshape ? reshape.getSrc().getDefiningOp() + : linalg::GenericOp(); + if (!scale || !pebble::elementwise(scale, "arith.mulf") || + !llvm::is_contained(scale.getInputs(), lhs->getOperand(1)) || + !llvm::is_contained(scale.getInputs(), rhs->getOperand(1))) + continue; + rewriter.setInsertionPoint(end); + auto custom = + CustomOp::create(rewriter, end->getLoc(), end->getResultTypes(), + depthwise ? "depthwise_conv2d_i8" : "conv2d_i8", + ValueRange{quant->getOperand(0), rhs->getOperand(0), + bias->getOperand(0), quant->getOperand(1), + lhs->getOperand(1), rhs->getOperand(1)}); + custom->setAttr("strides", rewriter.getDenseI64ArrayAttr(strides)); + custom->setAttr("dilations", rewriter.getDenseI64ArrayAttr(dilations)); + custom->setAttr("padding", rewriter.getDenseI64ArrayAttr(padding)); + rewriter.replaceOp(end, custom.getOutputs()); + if (depthwise) { + rewriter.eraseOp(collapsed); + rewriter.eraseOp(op); + } + if (pad && pad->use_empty()) + rewriter.eraseOp(pad); + for (Operation *dead : + {initial, bias, reshape.getOperation(), scale.getOperation(), + transpose.getOperation(), rhs, lhs, quant}) + if (dead->use_empty()) + rewriter.eraseOp(dead); + } + + // A convolution fuses its single-consumer chain [relu] [maxpool] [flatten] + // ending in a quantization: the convolution requantizes for the consumer + // and produces the INT8 tensor the consumer reads. + SmallVector roots; + function.walk([&](CustomOp op) { + if (op.getKind() == "conv2d_i8" || op.getKind() == "depthwise_conv2d_i8") + roots.push_back(op); + }); + DominanceInfo dominance(function); + const auto &target = buckyball_target::getBuckyballTarget(); + for (auto root : roots) { + Value value = root.getResult(0); + auto convType = cast(value.getType()); + // The chain in program order; the cycle markers among it. + SmallVector links, traces; + bool relu = false, flatten = false; + int64_t pool = 1; + while (value.hasOneUse()) { + Operation *user = *value.getUsers().begin(); + if (user->getBlock() != root->getBlock()) + break; + if (user->getName().getStringRef() == "buddy_trace.end") { + if (cast(user->getAttr("tensor")).getValue()) + break; + traces.push_back(user); + } else if (auto maximum = dyn_cast(user)) { + if (relu || pool > 1 || flatten || + !linalg::isaElemwiseSingleBinaryOpInterface(maximum) || + !isa(maximum.getBody()->front()) || + !llvm::all_of(maximum.getIndexingMapsArray(), + [](AffineMap map) { return map.isIdentity(); }) || + !matchPattern( + maximum.getInputs()[maximum.getInputs()[0] == value ? 1 : 0], + m_AnyZeroFloat())) + break; + relu = true; + } else if (auto pooling = dyn_cast(user)) { + // MAXPOOL pools square windows that tile the convolution output. + auto filter = + cast(pooling.getInputs()[1].getType()).getShape(); + if (pool > 1 || flatten || + !llvm::all_of(pooling.getDilations().getValues(), + [](int64_t v) { return v == 1; }) || + filter[0] != filter[1] || + !llvm::all_of(pooling.getStrides().getValues(), + [&](int64_t v) { return v == filter[0]; }) || + convType.getDimSize(1) % filter[0] || + convType.getDimSize(2) % filter[0]) + break; + Value initial = pooling.getOutputs()[0]; + if (auto fill = initial.getDefiningOp()) + initial = fill.getInputs()[0]; + APFloat minimum(0.0f); + bool negativeLimit = + matchPattern(initial, m_ConstantFloat(&minimum)) && + minimum.convertToFloat() <= -std::numeric_limits::max(); + if (!negativeLimit && + !(relu && matchPattern(initial, m_AnyZeroFloat()))) + break; + pool = filter[0]; + } else if (auto transpose = dyn_cast(user)) { + if (flatten || + transpose.getPermutation() != ArrayRef({0, 3, 1, 2}) || + !transpose->getResult(0).hasOneUse()) + break; + auto reshape = dyn_cast( + *transpose->getResult(0).getUsers().begin()); + if (!reshape || reshape.getReassociationIndices() != + SmallVector{{0}, {1, 2, 3}}) + break; + flatten = true; + links.push_back(transpose); + user = reshape; + } else { + break; + } + links.push_back(user); + value = user->getResult(0); + } + if (!value.hasOneUse()) + continue; + Operation *consumer = *value.getUsers().begin(); + auto next = dyn_cast(consumer); + bool terminal = + consumer->getName().getStringRef() == "frontend.fp32_to_int8"; + if (terminal ? flatten + : (!next || next.getInputs()[0] != value || + (flatten ? next.getKind() != "linear_i8" + : next.getKind() != "conv2d_i8" && + next.getKind() != "depthwise_conv2d_i8"))) + continue; + Value multiplier = terminal ? consumer->getOperand(1) : next.getInputs()[3]; + if (!dominance.properlyDominates(multiplier, root)) + continue; + bool barrier = false; + for (Operation *between = root->getNextNode(); between != consumer; + between = between->getNextNode()) + if (!isMemoryEffectFree(between) && + between->getName().getDialectNamespace() != "buddy_trace") { + barrier = true; + break; + } + auto strides = cast(root->getAttr("strides")); + auto weightType = cast(root.getInputs()[1].getType()); + bool depthwise = root.getKind() == "depthwise_conv2d_i8"; + int64_t kernel = weightType.getDimSize(2); + int64_t blocksIn = + (cast(root.getInputs()[0].getType()).getDimSize(3) + + 15) / + 16; + int64_t height = convType.getDimSize(1), width = convType.getDimSize(2), + channels = convType.getDimSize(3); + pebble::ConvTiling tiling = + pebble::convTiling(height / pool, width / pool, pool, strides[0], + kernel, blocksIn, depthwise, + cast(root.getInputs()[0].getType()) + .getElementType() + .isF32()); + auto producer = root.getInputs()[0].getDefiningOp(); + bool residentInput = producer && producer->hasAttr("custom.resident"); + if (barrier || (!residentInput && !tiling.side)) + continue; + + // The INT8 result stays in banks for a consumer that reads it whole: one + // tile covers the convolution, and both stages fit the banks with one + // group per channel block. + int64_t blocksOut = (channels + 15) / 16; + int64_t side = height / pool; + bool resident = false; + if (!depthwise && height == width && + (residentInput || tiling.side == height)) { + int64_t groups = pebble::im2colGroups( + kernel, residentInput ? blocksIn : tiling.chunk, height * width); + bool fits = (residentInput ? blocksIn : 0) + blocksOut + 2 * groups + 4 <= + target.bankNum - 1; + if (flatten) { + auto linearWeight = + cast(next.getInputs()[1].getType()); + resident = fits && side * side == 16 && channels == 16 && + linearWeight.getDimSize(0) <= target.bankDepth; + } else if (!terminal && next.getKind() == "conv2d_i8") { + auto nextWeight = cast(next.getInputs()[1].getType()); + auto nextPadding = cast(next->getAttr("padding")); + int64_t nextStride = + cast(next->getAttr("strides"))[0]; + int64_t nextKernel = nextWeight.getDimSize(2); + int64_t span = side + 2 * nextPadding[0] - nextKernel; + int64_t outputs = span / nextStride + 1; + int64_t nextGroups = + pebble::im2colGroups(nextKernel, blocksOut, outputs * outputs); + int64_t nextBlocks = (nextWeight.getDimSize(0) + 15) / 16; + resident = + fits && llvm::all_equal(nextPadding.asArrayRef()) && + nextPadding[0] <= buckyball_target::getBuckyballBallParam( + "IM2COL", "maxPadding") && + side <= + buckyball_target::getBuckyballBallParam("IM2COL", "maxIter") && + span >= 0 && span % nextStride == 0 && + (outputs * outputs + pebble::tileRows() - 1) / pebble::tileRows() * + pebble::tileRows() <= + pebble::matmulRows() && + nextGroups && + blocksOut + nextBlocks + 2 * nextGroups + 4 <= target.bankNum - 1; + } + } + + rewriter.setInsertionPoint(root); + SmallVector inputs(root.getInputs()); + inputs.push_back(multiplier); + auto fused = CustomOp::create( + rewriter, root.getLoc(), + TypeRange{RankedTensorType::get({1, side, width / pool, channels}, + rewriter.getI8Type())}, + root.getKind(), inputs); + fused->setAttrs(root->getAttrs()); + fused->setAttr("relu", rewriter.getBoolAttr(relu)); + fused->setAttr("pool", rewriter.getI64IntegerAttr(pool)); + if (resident) + fused->setAttr("custom.resident", rewriter.getUnitAttr()); + Value codes = fused.getResult(0); + // Cycle markers now observe the fused result. + rewriter.setInsertionPointAfter(fused); + for (Operation *trace : traces) { + Operation *moved = rewriter.clone(*trace); + moved->setOperand(0, codes); + moved->getResult(0).setType(codes.getType()); + } + rewriter.setInsertionPoint(consumer); + if (terminal) { + rewriter.replaceOp(consumer, codes); + } else if (flatten && resident) { + next->setOperand( + 0, CustomOp::create(rewriter, root.getLoc(), + TypeRange{cast(value.getType()) + .clone(rewriter.getI8Type())}, + "flatten_nchw", ValueRange{codes}) + .getResult(0)); + } else if (flatten) { + auto nchw = RankedTensorType::get({1, channels, side, width / pool}, + rewriter.getI8Type()); + Value empty = tensor::EmptyOp::create( + rewriter, root.getLoc(), nchw.getShape(), nchw.getElementType()); + Value transposed = + linalg::TransposeOp::create(rewriter, root.getLoc(), codes, empty, + ArrayRef{0, 3, 1, 2}) + .getResult()[0]; + next->setOperand(0, tensor::CollapseShapeOp::create( + rewriter, root.getLoc(), transposed, + ArrayRef{{0}, {1, 2, 3}})); + } else { + next->setOperand(0, codes); + } + for (Operation *link : llvm::reverse(links)) + rewriter.eraseOp(link); + rewriter.eraseOp(root); + } +} diff --git a/examples/designs/pebble/stack/device/core/src/LowerChannelOps.cpp b/examples/designs/pebble/stack/device/core/src/LowerChannelOps.cpp new file mode 100644 index 000000000..d844ec5a7 --- /dev/null +++ b/examples/designs/pebble/stack/device/core/src/LowerChannelOps.cpp @@ -0,0 +1,207 @@ +//===- LowerChannelOps.cpp - Per-channel INT8 operations and transpose ---===// +#include "Buckyball/BuckyballOps.h" +#include "Lowering.h" +#include "Target/BuckyballTargetRegistry.h" +#include "int8mulOps.h" +#include "lutOps.h" +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Bufferization/IR/Bufferization.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/Dialect/SCF/IR/SCF.h" +#include "transposeOps.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +LogicalResult pebble::lowerChannelOp(::buddy::custom::CustomOp op, + Value &result) { + bool lookup = op.getKind() == "lookup_i8"; + auto type = cast(op.getInputs()[lookup ? 0 : 1].getType()); + auto otherType = + cast(op.getInputs()[lookup ? 1 : 0].getType()); + if (!type.hasStaticShape() || type.getRank() != 2 || + !type.getElementType().isInteger(8) || + !otherType.getElementType().isInteger(8) || + op.getResult(0).getType() != type || + (lookup ? (otherType.getRank() != 1 || (otherType.getDimSize(0) != 256 && + otherType.getDimSize(0) != 4096)) + : otherType.getShape() != + ArrayRef({1, type.getDimSize(1)}))) + return op.emitOpError( + "requires an INT8 matrix and an INT8 table or per-channel gate"); + int64_t rows = type.getDimSize(0), channels = type.getDimSize(1); + int64_t capacity = buckyball_target::getBuckyballTarget().bankDepth; + OpBuilder b(op); + Location loc = op.getLoc(); + auto index = [&](OpBuilder &b, int64_t n) -> Value { + return arith::ConstantIndexOp::create(b, loc, n); + }; + Value zero = index(b, 0), one = index(b, 1); + Type i64 = b.getI64Type(); + Value input = tensorBuffer(b, loc, op.getInputs()[lookup ? 0 : 1]); + Value other = tensorBuffer(b, loc, op.getInputs()[lookup ? 1 : 0]); + Value output = memref::AllocOp::create( + b, loc, MemRefType::get(type.getShape(), b.getI8Type())); + Value izero = arith::ConstantIntOp::create(b, loc, 0, 8); + Value tableBank, ratio; + if (lookup) { + int64_t columns = otherType.getDimSize(0) == 256 ? 1 : 4; + if (columns == 4) { + Value packed = memref::AllocOp::create( + b, loc, MemRefType::get({64, 64}, b.getI8Type())); + auto rows = scf::ForOp::create(b, loc, zero, index(b, 64), one); + { + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToStart(rows.getBody()); + Value row = rows.getInductionVar(); + auto cols = scf::ForOp::create(b, loc, zero, index(b, 64), one); + b.setInsertionPointToStart(cols.getBody()); + Value col = cols.getInductionVar(); + Value channel = arith::AddIOp::create( + b, loc, + arith::MulIOp::create( + b, loc, arith::DivUIOp::create(b, loc, col, index(b, 16)), + index(b, 4)), + arith::DivUIOp::create(b, loc, row, index(b, 16))); + Value entry = arith::AddIOp::create( + b, loc, + arith::MulIOp::create( + b, loc, arith::RemUIOp::create(b, loc, row, index(b, 16)), + index(b, 16)), + arith::RemUIOp::create(b, loc, col, index(b, 16))); + Value offset = arith::AddIOp::create( + b, loc, arith::MulIOp::create(b, loc, channel, index(b, 256)), + entry); + Value value = memref::LoadOp::create(b, loc, other, offset); + memref::StoreOp::create(b, loc, value, packed, ValueRange{row, col}); + } + other = packed; + } + tableBank = BankAllocOp::create(b, loc, i64, 1, columns); + tableBank = BankMvinOp::create(b, loc, i64, other, tableBank, + otherType.getDimSize(0) / columns / 16, 1); + } else { + ratio = arith::ConstantOp::create(b, loc, + cast(op->getAttr("ratio"))); + } + pebble::repeat( + b, loc, 0, channels, 16, false, {}, + [&](OpBuilder &b, Value c, ValueRange) { + Value lanesEnd = arith::MinSIOp::create( + b, loc, index(b, 16), + arith::SubIOp::create(b, loc, index(b, channels), c)); + Value gateBank; + if (!lookup) { + Value gate = memref::AllocOp::create( + b, loc, MemRefType::get({16}, b.getI8Type())); + linalg::FillOp::create(b, loc, izero, gate); + auto lanes = scf::ForOp::create(b, loc, zero, lanesEnd, one); + { + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToStart(lanes.getBody()); + Value lane = lanes.getInductionVar(); + Value value = memref::LoadOp::create( + b, loc, other, + ValueRange{zero, arith::AddIOp::create(b, loc, lane, c)}); + memref::StoreOp::create(b, loc, value, gate, lane); + } + gateBank = BankAllocOp::create(b, loc, i64, 1, 1); + gateBank = BankMvinOp::create(b, loc, i64, gate, gateBank, 1, 1); + } + // Full chunks of `capacity` rows share one loop; the remaining rows + // follow. + auto chunk = [&](OpBuilder &b, Value start, int64_t count) { + Value pack = memref::AllocOp::create( + b, loc, MemRefType::get({count, 16}, b.getI8Type())); + linalg::FillOp::create(b, loc, izero, pack); + auto spatial = scf::ForOp::create(b, loc, zero, index(b, count), one); + { + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToStart(spatial.getBody()); + Value r = spatial.getInductionVar(); + auto lanes = scf::ForOp::create(b, loc, zero, lanesEnd, one); + b.setInsertionPointToStart(lanes.getBody()); + Value lane = lanes.getInductionVar(); + Value value = memref::LoadOp::create( + b, loc, input, + ValueRange{arith::AddIOp::create(b, loc, r, start), + arith::AddIOp::create(b, loc, lane, c)}); + memref::StoreOp::create(b, loc, value, pack, ValueRange{r, lane}); + } + Value source = BankAllocOp::create(b, loc, i64, 1, 1); + source = BankMvinOp::create(b, loc, i64, pack, source, count, 1); + Value destination = BankAllocOp::create(b, loc, i64, 1, 1); + destination = + lookup ? Value(::buddy::lut_ball::BankLookupI8Op::create( + b, loc, i64, source, tableBank, destination, count)) + : Value(::buddy::int8mul_ball::BankGateMulI8Op::create( + b, loc, i64, gateBank, source, destination, ratio, + count)); + BankMvoutOp::create(b, loc, i64, pack, destination, count, 1); + BankReleaseOp::create(b, loc, source); + BankReleaseOp::create(b, loc, destination); + auto outputRows = + scf::ForOp::create(b, loc, zero, index(b, count), one); + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToStart(outputRows.getBody()); + Value r = outputRows.getInductionVar(); + auto lanes = scf::ForOp::create(b, loc, zero, lanesEnd, one); + b.setInsertionPointToStart(lanes.getBody()); + Value lane = lanes.getInductionVar(); + Value value = + memref::LoadOp::create(b, loc, pack, ValueRange{r, lane}); + memref::StoreOp::create( + b, loc, value, output, + ValueRange{arith::AddIOp::create(b, loc, r, start), + arith::AddIOp::create(b, loc, lane, c)}); + }; + int64_t full = rows / capacity * capacity; + pebble::repeat(b, loc, 0, full, capacity, false, {}, + [&](OpBuilder &b, Value start, ValueRange) { + chunk(b, start, capacity); + return SmallVector{}; + }); + if (rows > full) + chunk(b, index(b, full), rows - full); + if (!lookup) + BankReleaseOp::create(b, loc, gateBank); + return SmallVector{}; + }); + if (lookup) + BankReleaseOp::create(b, loc, tableBank); + result = bufferization::ToTensorOp::create(b, loc, type, output, true, true); + return success(); +} + +LogicalResult pebble::lowerTranspose(::buddy::custom::CustomOp op, + Value &result) { + auto type = cast(op.getInputs()[0].getType()); + auto outputType = cast(op.getResult(0).getType()); + if (!type.hasStaticShape() || type.getRank() != 2 || + !type.getElementType().isInteger(8) || + outputType.getShape() != + ArrayRef({type.getDimSize(1), type.getDimSize(0)})) + return op.emitOpError("transpose_i8 requires a static INT8 matrix"); + OpBuilder b(op); + Location loc = op.getLoc(); + Value input = tensorBuffer(b, loc, op.getInputs()[0]); + Value output = memref::AllocOp::create( + b, loc, MemRefType::get(outputType.getShape(), b.getI8Type())); + Value source = + BankAllocOp::create(b, loc, b.getI64Type(), 1, type.getDimSize(1) / 16); + Value destination = + BankAllocOp::create(b, loc, b.getI64Type(), 1, type.getDimSize(1) / 16); + source = BankMvinOp::create(b, loc, b.getI64Type(), input, source, + type.getDimSize(0), 1); + destination = ::buddy::transpose_ball::BankTransposeI8Op::create( + b, loc, b.getI64Type(), source, destination, type.getDimSize(0), + type.getDimSize(1)); + BankMvoutOp::create(b, loc, b.getI64Type(), output, destination, + type.getDimSize(0), 1); + BankReleaseOp::create(b, loc, source); + BankReleaseOp::create(b, loc, destination); + result = + bufferization::ToTensorOp::create(b, loc, outputType, output, true, true); + return success(); +} diff --git a/examples/designs/pebble/stack/device/core/src/LowerLinear.cpp b/examples/designs/pebble/stack/device/core/src/LowerLinear.cpp new file mode 100644 index 000000000..72080628a --- /dev/null +++ b/examples/designs/pebble/stack/device/core/src/LowerLinear.cpp @@ -0,0 +1,498 @@ +//===- LowerLinear.cpp - Matrix products in bank tiles --------------------===// +// +// C = bias + A @ W over sixteen output columns per matmul_i8. M goes in tiles +// of the matmul rows; K goes in calls of up to six groups of A and B, each +// group one reduction block. The host packs every operand in the bank layout +// of matmul_i8, so a whole-bank DMA fills all groups of a bank at once. +// +//===----------------------------------------------------------------------===// +#include "Buckyball/BuckyballOps.h" +#include "Custom/CustomOps.h" +#include "Lowering.h" +#include "Target/BuckyballTargetRegistry.h" +#include "int2fpOps.h" +#include "mataddOps.h" +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Bufferization/IR/Bufferization.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/Dialect/SCF/IR/SCF.h" +#include "mlir/Dialect/Tensor/IR/Tensor.h" +#include "smatmulOps.h" +#include "toint8Ops.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +pebble::LinearSplit pebble::linearSplit(int64_t k, int64_t rows, + bool floating) { + // The most groups of A (and of its FP32 source) and B in one matmul call; + // the accumulator, a partial, their sum and the scales take four banks + // more. + constexpr int64_t callGroups = 6; + int64_t depth = buckyball_target::getBuckyballTarget().bankDepth; + // A group of A holds rows x k / 16 lines (four times that in FP32), B k. + int64_t maxK = std::min(depth, depth * (floating ? 4 : 16) / rows) / 16 * 16; + int64_t paddedK = (k + 15) / 16 * 16; + int64_t blocks = (paddedK + maxK - 1) / maxK; + int64_t calls = (blocks + callGroups - 1) / callGroups; + int64_t groups = (blocks + calls - 1) / calls; + int64_t blockK = + ((paddedK + calls * groups - 1) / (calls * groups) + 15) / 16 * 16; + return {calls, groups, blockK}; +} + +namespace { + +// output (M x N) = bias + input (M x K) @ weight (K x N), converted with the +// per-column `scales` and `relu` to the element type of `output`: INT32 as +// is, FP32 by int32_to_fp32, INT8 by int32_to_int8. An FP32 input is +// quantized with `multiplier`. A resident input bank of one input row holds +// K in groups of `residentBlock` columns and replaces the input of a single +// call. `bias` has one row, or one row per row of the input. Without +// `output`, the INT8 row stays in the returned bank, one group per sixteen +// columns. +Value linearTiles(OpBuilder &builder, Location loc, Value input, Value resident, + int64_t residentBlock, Value multiplier, Value weight, + Value bias, Value scales, bool relu, Value output) { + auto weightType = cast(weight.getType()); + int64_t k = weightType.getDimSize(0), n = weightType.getDimSize(1); + int64_t m = output ? cast(output.getType()).getDimSize(0) : 1; + bool floating = multiplier != nullptr; + bool perRow = m > 1 && cast(bias.getType()).getDimSize(0) == m; + Type outElement = output ? cast(output.getType()).getElementType() + : builder.getI8Type(); + int64_t rows = m == 1 ? 1 : pebble::tileRows(); + int64_t paddedK = (k + 15) / 16 * 16; + pebble::LinearSplit split = pebble::linearSplit(k, rows, floating); + int64_t calls = split.calls, groups = split.groups, blockK = split.blockK; + if (resident) { + calls = 1; + blockK = residentBlock; + groups = paddedK / residentBlock; + } + int64_t aLines = blockK / 16 * rows * (floating ? 4 : 1); + int64_t aWidth = groups * (floating ? 4 : 16); + int64_t slices = (n + 15) / 16; + Type i64 = builder.getI64Type(); + auto index = [&](OpBuilder &b, int64_t value) -> Value { + return arith::ConstantIndexOp::create(b, loc, value); + }; + auto extent = [&](OpBuilder &b, int64_t size, int64_t limit, + Value start) -> Value { + return arith::MinSIOp::create( + b, loc, index(b, size), + arith::SubIOp::create(b, loc, index(b, limit), start)); + }; + // Nested host loops over [0, ends) with the given body. + auto loops = [&](OpBuilder &b, ArrayRef ends, + function_ref body) { + OpBuilder::InsertionGuard guard(b); + SmallVector indices; + for (Value end : ends) { + auto loop = scf::ForOp::create(b, loc, index(b, 0), end, index(b, 1)); + b.setInsertionPointToStart(loop.getBody()); + indices.push_back(loop.getInductionVar()); + } + body(b, indices); + }; + // The reduction block of column `kk`: call, group and column in the block. + auto block = [&](OpBuilder &b, Value kk) { + Value inBlock = arith::RemUIOp::create(b, loc, kk, index(b, blockK)); + Value blockIndex = arith::DivUIOp::create(b, loc, kk, index(b, blockK)); + Value call = arith::DivUIOp::create(b, loc, blockIndex, index(b, groups)); + Value group = arith::RemUIOp::create(b, loc, blockIndex, index(b, groups)); + return std::make_tuple(call, group, inBlock); + }; + + // B of every output slice and call, packed once: line l of group g holds + // row g * blockK + l of the block. + Value weightPack = memref::AllocOp::create( + builder, loc, + MemRefType::get({slices, calls, blockK, groups * 16}, + builder.getI8Type())); + linalg::FillOp::create( + builder, loc, + ValueRange{arith::ConstantIntOp::create(builder, loc, 0, 8)}, weightPack); + loops(builder, {index(builder, k), index(builder, n)}, + [&](OpBuilder &b, ValueRange at) { + auto [call, group, inBlock] = block(b, at[0]); + Value value = memref::LoadOp::create(b, loc, weight, at); + Value slice = arith::DivUIOp::create(b, loc, at[1], index(b, 16)); + Value lane = arith::AddIOp::create( + b, loc, arith::MulIOp::create(b, loc, group, index(b, 16)), + arith::RemUIOp::create(b, loc, at[1], index(b, 16))); + memref::StoreOp::create(b, loc, value, weightPack, + ValueRange{slice, call, inBlock, lane}); + }); + Value inputPack; + if (!resident) + inputPack = memref::AllocOp::create( + builder, loc, + MemRefType::get({calls, aLines, aWidth}, + floating ? Type(builder.getF32Type()) + : builder.getI8Type())); + Value biasPack = memref::AllocOp::create( + builder, loc, MemRefType::get({16}, builder.getI32Type())); + Value zeroPack = memref::AllocOp::create( + builder, loc, MemRefType::get({16}, builder.getI32Type())); + Value tilePack; + if (perRow) + tilePack = memref::AllocOp::create( + builder, loc, MemRefType::get({rows, 16}, builder.getI32Type())); + Value scalePack; + if (scales) + scalePack = memref::AllocOp::create( + builder, loc, MemRefType::get({16}, builder.getF32Type())); + Value outPack = memref::AllocOp::create( + builder, loc, MemRefType::get({rows, 16}, outElement)); + Value izero = arith::ConstantIntOp::create(builder, loc, 0, 32); + linalg::FillOp::create(builder, loc, izero, zeroPack); + + // A of reduction call `call`, quantized on the core from FP32. + auto loadA = [&](OpBuilder &b, Value call) -> Value { + Value slice = memref::SubViewOp::create( + b, loc, inputPack, + ArrayRef{call, b.getIndexAttr(0), b.getIndexAttr(0)}, + ArrayRef{b.getIndexAttr(1), b.getIndexAttr(aLines), + b.getIndexAttr(aWidth)}, + SmallVector(3, b.getIndexAttr(1))); + Value a = BankAllocOp::create(b, loc, i64, 1, groups); + if (!floating) + return BankMvinOp::create(b, loc, i64, slice, a, aLines, 1); + Value values = BankAllocOp::create(b, loc, i64, 1, groups); + values = BankMvinOp::create(b, loc, i64, slice, values, aLines, 1); + a = ::buddy::toint8_ball::BankFp32ToInt8Op::create(b, loc, i64, values, a, + multiplier, aLines); + BankReleaseOp::create(b, loc, values); + return a; + }; + // Reduction call `call` of output slice `slice` into `destination`, read + // as its bias. `a` is the A of the call, or null to load it. + auto product = [&](OpBuilder &b, Value slice, Value call, Value a, + Value destination) -> Value { + Value loaded = a ? a : loadA(b, call); + Value w = BankAllocOp::create(b, loc, i64, 1, groups); + w = BankMvinOp::create( + b, loc, i64, + memref::SubViewOp::create( + b, loc, weightPack, + ArrayRef{slice, call, b.getIndexAttr(0), + b.getIndexAttr(0)}, + ArrayRef{b.getIndexAttr(1), b.getIndexAttr(1), + b.getIndexAttr(blockK), + b.getIndexAttr(groups * 16)}, + SmallVector(4, b.getIndexAttr(1))), + w, blockK, 1); + Value result = ::buddy::smatmul_ball::BankMatmulI8Op::create( + b, loc, i64, loaded, w, destination, rows, blockK); + if (!a) + BankReleaseOp::create(b, loc, loaded); + BankReleaseOp::create(b, loc, w); + return result; + }; + // sum = acc + partial; add_i32 writes a third bank. + auto add = [&](OpBuilder &b, Value acc, Value partial) -> Value { + Value sum = BankAllocOp::create(b, loc, i64, 1, 1); + sum = ::buddy::matadd_ball::BankAddI32Op::create(b, loc, i64, acc, partial, + sum, rows * 4); + BankReleaseOp::create(b, loc, acc); + BankReleaseOp::create(b, loc, partial); + return sum; + }; + + Value vector; + pebble::repeat( + builder, loc, 0, m, rows, false, {}, + [&](OpBuilder &b, Value row, ValueRange) -> SmallVector { + Value valid = extent(b, rows, m, row); + if (!resident) { + linalg::FillOp::create( + b, loc, + ValueRange{ + floating + ? arith::ConstantOp::create(b, loc, b.getF32FloatAttr(0)) + .getResult() + : arith::ConstantIntOp::create(b, loc, 0, 8).getResult()}, + inputPack); + loops(b, {valid, index(b, k)}, [&](OpBuilder &b, ValueRange at) { + auto [call, group, inBlock] = block(b, at[1]); + Value value = memref::LoadOp::create( + b, loc, input, + ValueRange{arith::AddIOp::create(b, loc, at[0], row), at[1]}); + // Slice t = inBlock / 16 of M row r is line t * rows + r. + Value line = arith::AddIOp::create( + b, loc, + arith::MulIOp::create( + b, loc, + arith::DivUIOp::create(b, loc, inBlock, index(b, 16)), + index(b, rows)), + at[0]); + Value lane = arith::RemUIOp::create(b, loc, inBlock, index(b, 16)); + if (floating) { + line = arith::AddIOp::create( + b, loc, arith::MulIOp::create(b, loc, line, index(b, 4)), + arith::DivUIOp::create(b, loc, lane, index(b, 4))); + lane = arith::RemUIOp::create(b, loc, lane, index(b, 4)); + } + lane = arith::AddIOp::create( + b, loc, + arith::MulIOp::create(b, loc, group, + index(b, floating ? 4 : 16)), + lane); + memref::StoreOp::create(b, loc, value, inputPack, + ValueRange{call, line, lane}); + }); + } + // One call keeps A in its bank for every output slice. + Value a = resident; + if (!resident && calls == 1) + a = loadA(b, index(b, 0)); + auto outputSlice = [&](OpBuilder &b, + Value column) -> SmallVector { + Value columns = extent(b, 16, n, column); + Value slice = arith::DivUIOp::create(b, loc, column, index(b, 16)); + linalg::FillOp::create(b, loc, izero, biasPack); + if (!perRow) + loops(b, {columns}, [&](OpBuilder &b, ValueRange at) { + Value value = memref::LoadOp::create( + b, loc, bias, + ValueRange{index(b, 0), + arith::AddIOp::create(b, loc, at[0], column)}); + memref::StoreOp::create(b, loc, value, biasPack, at[0]); + }); + Value scaleBank; + if (scales) { + linalg::FillOp::create(b, loc, + ValueRange{arith::ConstantOp::create( + b, loc, b.getF32FloatAttr(1))}, + scalePack); + loops(b, {columns}, [&](OpBuilder &b, ValueRange at) { + Value value = memref::LoadOp::create( + b, loc, scales, + ValueRange{index(b, 0), + arith::AddIOp::create(b, loc, at[0], column)}); + memref::StoreOp::create(b, loc, value, scalePack, at[0]); + }); + scaleBank = BankAllocOp::create(b, loc, i64, 1, 1); + scaleBank = + BankMvinOp::create(b, loc, i64, scalePack, scaleBank, 4, 1); + } + Value acc = BankAllocOp::create(b, loc, i64, 1, 1); + acc = BankMvinOp::create(b, loc, i64, biasPack, acc, 4, 1); + acc = product(b, slice, index(b, 0), a, acc); + if (rows == 1) { + // One row accumulates in place: the old C is the bias. + acc = + pebble::repeat(b, loc, 1, calls, 1, false, {acc}, + [&](OpBuilder &b, Value call, ValueRange args) { + return SmallVector{ + product(b, slice, call, Value(), args[0])}; + })[0]; + } else { + for (int64_t call = 1; call < calls; ++call) { + Value partial = BankAllocOp::create(b, loc, i64, 1, 1); + partial = + BankMvinOp::create(b, loc, i64, zeroPack, partial, 4, 1); + acc = add(b, acc, + product(b, slice, index(b, call), Value(), partial)); + } + } + if (perRow) { + linalg::FillOp::create(b, loc, izero, tilePack); + loops(b, {valid, columns}, [&](OpBuilder &b, ValueRange at) { + Value value = memref::LoadOp::create( + b, loc, bias, + ValueRange{arith::AddIOp::create(b, loc, at[0], row), + arith::AddIOp::create(b, loc, at[1], column)}); + memref::StoreOp::create(b, loc, value, tilePack, at); + }); + Value initial = BankAllocOp::create(b, loc, i64, 1, 1); + initial = + BankMvinOp::create(b, loc, i64, tilePack, initial, rows * 4, 1); + acc = add(b, acc, initial); + } + Value restored = acc; + if (scales) { + restored = BankAllocOp::create(b, loc, i64, 1, 1); + restored = + outElement.isF32() + ? Value(::buddy::int2fp_ball::BankInt32ToFp32Op::create( + b, loc, i64, acc, scaleBank, restored, rows, relu)) + : Value(::buddy::toint8_ball::BankInt32ToInt8Op::create( + b, loc, i64, acc, scaleBank, restored, rows, relu)); + BankReleaseOp::create(b, loc, acc); + BankReleaseOp::create(b, loc, scaleBank); + } + if (!output) + return {restored}; + restored = + BankMvoutOp::create(b, loc, i64, outPack, restored, + outElement.isInteger(8) ? rows : rows * 4, 1); + BankReleaseOp::create(b, loc, restored); + loops(b, {valid, columns}, [&](OpBuilder &b, ValueRange at) { + Value value = memref::LoadOp::create(b, loc, outPack, at); + memref::StoreOp::create( + b, loc, value, output, + ValueRange{arith::AddIOp::create(b, loc, at[0], row), + arith::AddIOp::create(b, loc, at[1], column)}); + }); + return {}; + }; + if (output) { + pebble::repeat(b, loc, 0, n, 16, false, {}, + [&](OpBuilder &b, Value column, ValueRange) { + return outputSlice(b, column); + }); + } else { + // Each output slice appends its group to the resident row. + for (int64_t column = 0; column < n; column += 16) { + Value slice = outputSlice(b, index(b, column))[0]; + vector = vector ? BankTransferOp::create(b, loc, i64, slice, vector) + .getResult() + : slice; + } + } + if (a) + BankReleaseOp::create(b, loc, a); + return {}; + }); + return vector; +} +} // namespace + +LogicalResult pebble::lowerLinear(::buddy::custom::CustomOp op, + Value residentInput, Value &result, + Value &resident) { + StringRef kind = op.getKind(); + bool integer = kind == "matmul_i8", requant = kind == "linear_i8_requant"; + // A fused linear_i8 requantizes with the multiplier of its consumer. + ValueRange args = op.getInputs(); + bool fused = kind == "linear_i8" && args.size() == 7; + if ((!fused && args.size() != (integer ? 3 + : requant ? 4 + : 6)) || + op.getNumResults() != 1) + return op.emitOpError("has the wrong number of inputs or results for ") + << kind; + auto a = cast(args[0].getType()); + auto w = cast(args[1].getType()); + auto biasType = cast(args[2].getType()); + auto resultType = cast(op.getResult(0).getType()); + int64_t m = resultType.getDimSize(0), n = w.getDimSize(1); + bool codes = integer || requant || fused; + if (!a.hasStaticShape() || !w.hasStaticShape() || a.getRank() != 2 || + w.getRank() != 2 || a.getDimSize(1) != w.getDimSize(0) || + a.getDimSize(0) != m || !w.getElementType().isInteger(8) || + !biasType.getElementType().isInteger(32) || + biasType.getShape() != ArrayRef({integer ? m : 1, n}) || + resultType.getShape() != ArrayRef({m, n}) || + !(integer ? resultType.getElementType().isInteger(32) + : codes ? resultType.getElementType().isInteger(8) + : resultType.getElementType().isF32()) || + ((integer || requant) && !a.getElementType().isInteger(8))) + return op.emitOpError("requires matching MxK/KxN tensors and INT32 bias " + "for ") + << kind; + OpBuilder b(op); + Location loc = op.getLoc(); + Value zero = arith::ConstantIndexOp::create(b, loc, 0); + Value input, multiplier, scales; + if (!residentInput) + input = tensorBuffer(b, loc, args[0]); + if (!integer && !requant && a.getElementType().isF32()) + multiplier = + tensor::ExtractOp::create(b, loc, args[3], ValueRange{zero, zero}); + if (requant) { + scales = tensorBuffer(b, loc, args[3]); + } else if (!integer) { + // input_step * weight_step per column restores FP32 values; a fused + // Linear scales them to the codes of its consumer. + Value step = + tensor::ExtractOp::create(b, loc, args[4], ValueRange{zero, zero}); + if (fused) + step = arith::MulFOp::create( + b, loc, step, + tensor::ExtractOp::create(b, loc, args[6], ValueRange{zero, zero})); + Value weightSteps = tensorBuffer(b, loc, args[5]); + scales = memref::AllocOp::create(b, loc, + MemRefType::get({1, n}, b.getF32Type())); + auto loop = scf::ForOp::create(b, loc, zero, + arith::ConstantIndexOp::create(b, loc, n), + arith::ConstantIndexOp::create(b, loc, 1)); + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToStart(loop.getBody()); + Value c = loop.getInductionVar(); + Value product = arith::MulFOp::create( + b, loc, step, + memref::LoadOp::create(b, loc, weightSteps, ValueRange{zero, c})); + memref::StoreOp::create(b, loc, product, scales, ValueRange{zero, c}); + } + // A flattened map is one block of K; a resident Linear row has a group per + // sixteen columns. + auto producer = args[0].getDefiningOp<::buddy::custom::CustomOp>(); + int64_t residentBlock = + producer && producer.getKind() == "flatten_nchw" ? a.getDimSize(1) : 16; + bool relu = + (requant || fused) && cast(op->getAttr("relu")).getValue(); + Value output; + if (!op->hasAttr("custom.resident")) + output = memref::AllocOp::create( + b, loc, MemRefType::get({m, n}, resultType.getElementType())); + resident = linearTiles(b, loc, input, residentInput, residentBlock, + multiplier, tensorBuffer(b, loc, args[1]), + tensorBuffer(b, loc, args[2]), scales, relu, output); + if (output) + result = bufferization::ToTensorOp::create(b, loc, resultType, output, true, + true); + return success(); +} + +// The mean of every channel: ones (1 x pixels) @ input (pixels x channels), +// requantized with input_scale / (pixels * output_scale). +LogicalResult pebble::lowerGlobalAverage(::buddy::custom::CustomOp op, + Value &result) { + auto type = cast(op.getInputs()[0].getType()); + auto resultType = cast(op.getResult(0).getType()); + if (!type.hasStaticShape() || type.getRank() != 4 || + type.getDimSize(0) != 1 || !type.getElementType().isInteger(8) || + !resultType.getElementType().isInteger(8) || + resultType.getShape() != ArrayRef({1, 1, 1, type.getDimSize(3)})) + return op.emitOpError( + "global_avg_i8 requires NHWC INT8 input and 1x1x1xC INT8 output"); + int64_t pixels = type.getDimSize(1) * type.getDimSize(2), + channels = type.getDimSize(3); + float ratio = + cast(op->getAttr("input_scale")).getValueAsDouble() / + (pixels * + cast(op->getAttr("output_scale")).getValueAsDouble()); + OpBuilder b(op); + Location loc = op.getLoc(); + Value ones = memref::AllocOp::create( + b, loc, MemRefType::get({1, pixels}, b.getI8Type())); + linalg::FillOp::create( + b, loc, ValueRange{arith::ConstantIntOp::create(b, loc, 1, 8)}, ones); + Value bias = memref::AllocOp::create( + b, loc, MemRefType::get({1, channels}, b.getI32Type())); + linalg::FillOp::create( + b, loc, ValueRange{arith::ConstantIntOp::create(b, loc, 0, 32)}, bias); + Value scales = memref::AllocOp::create( + b, loc, MemRefType::get({1, channels}, b.getF32Type())); + linalg::FillOp::create( + b, loc, + ValueRange{arith::ConstantOp::create(b, loc, b.getF32FloatAttr(ratio))}, + scales); + Value weight = memref::CollapseShapeOp::create( + b, loc, tensorBuffer(b, loc, op.getInputs()[0]), + ArrayRef{{0, 1, 2}, {3}}); + Value output = memref::AllocOp::create( + b, loc, MemRefType::get({1, channels}, b.getI8Type())); + linearTiles(b, loc, ones, Value(), 0, Value(), weight, bias, scales, false, + output); + Value flat = bufferization::ToTensorOp::create( + b, loc, RankedTensorType::get({1, channels}, b.getI8Type()), output, true, + true); + result = tensor::ExpandShapeOp::create( + b, loc, resultType, flat, ArrayRef{{0, 1, 2}, {3}}); + return success(); +} diff --git a/examples/designs/pebble/stack/device/core/src/LowerSpatialOps.cpp b/examples/designs/pebble/stack/device/core/src/LowerSpatialOps.cpp new file mode 100644 index 000000000..82e8fb6f7 --- /dev/null +++ b/examples/designs/pebble/stack/device/core/src/LowerSpatialOps.cpp @@ -0,0 +1,521 @@ +//===- LowerSpatialOps.cpp - Convolutions in bank tiles -------------------===// +// +// A convolution computes sixteen output channels per matmul_i8 over a tile of +// output pixels. The input tile, sixteen channels per group, goes through +// im2col_i8 into the A operand; the host packs the weights in the same K +// order into B. Input channels beyond one call form chunks whose partial +// products add up with add_i32. A fused convolution requantizes for its +// consumer and pools; when the consumer reads the whole result, the result +// stays in banks, one group per channel block. +// +//===----------------------------------------------------------------------===// +#include "Buckyball/BuckyballOps.h" +#include "Custom/CustomOps.h" +#include "Lowering.h" +#include "Target/BuckyballTargetRegistry.h" +#include "im2colOps.h" +#include "int2fpOps.h" +#include "mataddOps.h" +#include "maxpoolOps.h" +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Bufferization/IR/Bufferization.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/Dialect/SCF/IR/SCF.h" +#include "mlir/Dialect/Tensor/IR/Tensor.h" +#include "smatmulOps.h" +#include "toint8Ops.h" +#include "transposeOps.h" + +using namespace mlir; +using namespace ::buddy::buckyball; +using Computation = ::buddy::custom::CustomOp; + +namespace { +// Lowers `op`. A resident input is a bank with one group per input channel +// block holding the whole input map. With custom.resident the INT8 result is +// returned in `image`, one group per output channel block; otherwise in +// `tensorOutput`. +LogicalResult convolution(Computation op, Value residentInput, Value &image, + Value &tensorOutput) { + bool depthwise = op.getKind() == "depthwise_conv2d_i8"; + bool fused = op.getInputs().size() == 7; + bool resident = op->hasAttr("custom.resident"); + auto args = op.getInputs(); + auto inputType = cast(args[0].getType()); + auto weightType = cast(args[1].getType()); + auto outputType = cast(op.getResult(0).getType()); + auto strides = cast(op->getAttr("strides")).asArrayRef(); + auto padding = cast(op->getAttr("padding")).asArrayRef(); + if (args.size() < 6 || !inputType.hasStaticShape() || + inputType.getRank() != 4 || !weightType.hasStaticShape() || + weightType.getRank() != 4 || inputType.getDimSize(0) != 1 || + !weightType.getElementType().isInteger(8) || + !cast(args[2].getType()).getElementType().isInteger(32)) + return op.emitOpError("requires static NHWC/FCHW convolution with INT32 " + "bias"); + bool floating = inputType.getElementType().isF32(); + int64_t pool = fused ? cast(op->getAttr("pool")).getInt() : 1; + bool relu = fused && cast(op->getAttr("relu")).getValue(); + int64_t cin = inputType.getDimSize(3), cout = weightType.getDimSize(0); + int64_t kernel = weightType.getDimSize(2), stride = strides[0]; + int64_t height = outputType.getDimSize(1) * pool, + width = outputType.getDimSize(2) * pool; + int64_t blocksIn = (cin + 15) / 16; + // The tile, the input tile around it and the channel blocks per chunk. A + // resident input is one tile of the whole map that im2col pads itself. + int64_t side, inputSide, chunk, im2colPadding; + if (residentInput) { + side = height; + inputSide = inputType.getDimSize(1); + chunk = blocksIn; + im2colPadding = padding[0]; + } else { + pebble::ConvTiling tiling = + pebble::convTiling(height / pool, width / pool, pool, stride, kernel, + blocksIn, depthwise, floating); + side = tiling.side; + inputSide = tiling.inputSide; + chunk = tiling.chunk; + im2colPadding = 0; + } + int64_t rows = (side * side + pebble::tileRows() - 1) / pebble::tileRows() * + pebble::tileRows(); + int64_t pooled = side / pool; + // Chunks of input channel blocks: [start, start + blocks). Depthwise reads + // the block of its output channels. + SmallVector> chunks; + for (int64_t start = 0; start < (depthwise ? 1 : blocksIn); start += chunk) + chunks.push_back({start, std::min(chunk, blocksIn - start)}); + + OpBuilder builder(op); + Location loc = op.getLoc(); + Type i64 = builder.getI64Type(); + auto index = [&](OpBuilder &b, int64_t n) -> Value { + return arith::ConstantIndexOp::create(b, loc, n); + }; + auto extent = [&](OpBuilder &b, int64_t size, int64_t limit, + Value start) -> Value { + return arith::MinSIOp::create( + b, loc, index(b, size), + arith::SubIOp::create(b, loc, index(b, limit), start)); + }; + auto loops = [&](OpBuilder &b, ArrayRef ends, + function_ref body) { + OpBuilder::InsertionGuard guard(b); + SmallVector indices; + for (Value end : ends) { + auto loop = scf::ForOp::create(b, loc, index(b, 0), end, index(b, 1)); + b.setInsertionPointToStart(loop.getBody()); + indices.push_back(loop.getInductionVar()); + } + body(b, indices); + }; + Value zero = index(builder, 0); + Value input; + if (!residentInput) + input = pebble::tensorBuffer(builder, loc, args[0]); + Value weight = pebble::tensorBuffer(builder, loc, args[1]); + Value bias = pebble::tensorBuffer(builder, loc, args[2]); + Value weightSteps = pebble::tensorBuffer(builder, loc, args[5]); + Value step = tensor::ExtractOp::create(builder, loc, args[4], + ValueRange{zero, zero, zero, zero}); + Value inputMultiplier, outputMultiplier; + if (floating) + inputMultiplier = tensor::ExtractOp::create( + builder, loc, args[3], ValueRange{zero, zero, zero, zero}); + if (fused) { + SmallVector indices( + cast(args[6].getType()).getRank(), zero); + outputMultiplier = + tensor::ExtractOp::create(builder, loc, args[6], indices); + } + Value output; + if (!resident) + output = memref::AllocOp::create( + builder, loc, + MemRefType::get(outputType.getShape(), outputType.getElementType())); + // Host packs: per chunk the weights of one output block in the bank layout + // of B, the input tile with sixteen channels per group, the bias row, the + // scales and one tile of results. + SmallVector groups; + SmallVector weightPacks, inputPacks; + for (auto [start, blocks] : chunks) { + int64_t outputGroups = pebble::im2colGroups(kernel, blocks, side * side); + int64_t tiles = kernel * kernel * blocks / outputGroups; + groups.push_back(outputGroups); + weightPacks.push_back(memref::AllocOp::create( + builder, loc, + MemRefType::get({16 * tiles, outputGroups * 16}, builder.getI8Type()))); + if (!residentInput) + inputPacks.push_back(memref::AllocOp::create( + builder, loc, + floating ? MemRefType::get({inputSide * inputSide * 4, blocks * 4}, + builder.getF32Type()) + : MemRefType::get({inputSide * inputSide, blocks * 16}, + builder.getI8Type()))); + } + Value biasPack = memref::AllocOp::create( + builder, loc, MemRefType::get({16}, builder.getI32Type())); + Value zeroPack = memref::AllocOp::create( + builder, loc, MemRefType::get({16}, builder.getI32Type())); + Value scalePack = memref::AllocOp::create( + builder, loc, MemRefType::get({16}, builder.getF32Type())); + Value outPack; + if (!resident) + outPack = memref::AllocOp::create( + builder, loc, + MemRefType::get({rows, 16}, fused ? Type(builder.getI8Type()) + : builder.getF32Type())); + Value izero = arith::ConstantIntOp::create(builder, loc, 0, 32); + Value byteZero = arith::ConstantIntOp::create(builder, loc, 0, 8); + linalg::FillOp::create(builder, loc, izero, zeroPack); + + // One output block of sixteen channels at `oc`; returns the INT8 result + // bank of a resident convolution. + auto outputBlock = [&](OpBuilder &b, Value oc) -> Value { + Value lanes = extent(b, 16, cout, oc); + // B: line l of group j is K tile j * tiles + l / 16, input channel l % 16 + // of the tile; tile t is kernel position t / blocks, block t % blocks. + for (auto [index_, chunkRange] : llvm::enumerate(chunks)) { + int64_t start = chunkRange.first, blocks = chunkRange.second; + int64_t tiles = kernel * kernel * blocks / groups[index_]; + Value pack = weightPacks[index_]; + linalg::FillOp::create(b, loc, byteZero, pack); + loops( + b, {lanes, index(b, 16 * tiles * groups[index_])}, + [&](OpBuilder &b, ValueRange at) { + Value lane = at[0], row = at[1]; + Value tile = arith::DivUIOp::create(b, loc, row, index(b, 16)); + Value channel = arith::RemUIOp::create(b, loc, row, index(b, 16)); + Value position = + arith::DivUIOp::create(b, loc, tile, index(b, blocks)); + Value blockIndex = + arith::RemUIOp::create(b, loc, tile, index(b, blocks)); + Value ky = + arith::DivUIOp::create(b, loc, position, index(b, kernel)); + Value kx = + arith::RemUIOp::create(b, loc, position, index(b, kernel)); + Value o = arith::AddIOp::create(b, loc, oc, lane); + Value inputChannel; + if (depthwise) + inputChannel = arith::AddIOp::create(b, loc, oc, channel); + else + inputChannel = arith::AddIOp::create( + b, loc, + arith::MulIOp::create(b, loc, + arith::AddIOp::create(b, loc, + blockIndex, + index(b, start)), + index(b, 16)), + channel); + Value present = + depthwise + ? arith::CmpIOp::create(b, loc, arith::CmpIPredicate::eq, + channel, lane) + .getResult() + : arith::CmpIOp::create(b, loc, arith::CmpIPredicate::ult, + inputChannel, index(b, cin)) + .getResult(); + auto valid = scf::IfOp::create(b, loc, present, false); + b.setInsertionPointToStart(&valid.getThenRegion().front()); + Value value = memref::LoadOp::create( + b, loc, weight, + ValueRange{o, depthwise ? index(b, 0) : inputChannel, ky, kx}); + Value line = + arith::RemUIOp::create(b, loc, row, index(b, 16 * tiles)); + Value group = + arith::DivUIOp::create(b, loc, row, index(b, 16 * tiles)); + memref::StoreOp::create( + b, loc, value, pack, + ValueRange{line, arith::AddIOp::create( + b, loc, + arith::MulIOp::create(b, loc, group, + index(b, 16)), + lane)}); + }); + } + linalg::FillOp::create(b, loc, izero, biasPack); + linalg::FillOp::create( + b, loc, + ValueRange{arith::ConstantOp::create(b, loc, b.getF32FloatAttr(1))}, + scalePack); + loops(b, {lanes}, [&](OpBuilder &b, ValueRange at) { + Value o = arith::AddIOp::create(b, loc, oc, at[0]); + memref::StoreOp::create(b, loc, memref::LoadOp::create(b, loc, bias, o), + biasPack, at[0]); + Value scale = arith::MulFOp::create( + b, loc, step, + memref::LoadOp::create( + b, loc, weightSteps, + ValueRange{o, index(b, 0), index(b, 0), index(b, 0)})); + if (fused) + scale = arith::MulFOp::create(b, loc, scale, outputMultiplier); + memref::StoreOp::create(b, loc, scale, scalePack, at[0]); + }); + Value scales = BankAllocOp::create(b, loc, i64, 1, 1); + scales = BankMvinOp::create(b, loc, i64, scalePack, scales, 4, 1); + Value result; + pebble::repeat( + b, loc, 0, height, side, false, {}, + [&](OpBuilder &b, Value y, ValueRange) -> SmallVector { + pebble::repeat( + b, loc, 0, width, side, false, {}, + [&](OpBuilder &b, Value x, ValueRange) -> SmallVector { + Value acc; + for (auto [index_, chunkRange] : llvm::enumerate(chunks)) { + int64_t start = chunkRange.first, blocks = chunkRange.second; + int64_t outputGroups = groups[index_]; + int64_t tiles = kernel * kernel * blocks / outputGroups; + Value source = residentInput; + if (!residentInput) { + // The input tile of this chunk; the halo outside the + // input stays zero. + Value pack = inputPacks[index_]; + Value first = + depthwise + ? arith::DivUIOp::create(b, loc, oc, index(b, 16)) + .getResult() + : index(b, start); + linalg::FillOp::create( + b, loc, + ValueRange{floating ? arith::ConstantOp::create( + b, loc, b.getF32FloatAttr(0)) + .getResult() + : byteZero}, + pack); + loops( + b, + {index(b, inputSide), index(b, inputSide), + index(b, blocks * 16)}, + [&](OpBuilder &b, ValueRange at) { + Value iy = arith::SubIOp::create( + b, loc, + arith::AddIOp::create( + b, loc, + arith::MulIOp::create(b, loc, y, + index(b, stride)), + at[0]), + index(b, padding[0])); + Value ix = arith::SubIOp::create( + b, loc, + arith::AddIOp::create( + b, loc, + arith::MulIOp::create(b, loc, x, + index(b, stride)), + at[1]), + index(b, padding[2])); + Value channel = arith::AddIOp::create( + b, loc, + arith::MulIOp::create(b, loc, first, + index(b, 16)), + at[2]); + auto inRange = [&](Value value, int64_t limit) { + return arith::AndIOp::create( + b, loc, + arith::CmpIOp::create(b, loc, + arith::CmpIPredicate::sge, + value, index(b, 0)), + arith::CmpIOp::create(b, loc, + arith::CmpIPredicate::slt, + value, index(b, limit))); + }; + Value inside = arith::AndIOp::create( + b, loc, + arith::AndIOp::create( + b, loc, inRange(iy, inputType.getDimSize(1)), + inRange(ix, inputType.getDimSize(2))), + inRange(channel, cin)); + auto valid = scf::IfOp::create(b, loc, inside, false); + b.setInsertionPointToStart( + &valid.getThenRegion().front()); + Value value = memref::LoadOp::create( + b, loc, input, + ValueRange{index(b, 0), iy, ix, channel}); + Value pixel = arith::AddIOp::create( + b, loc, + arith::MulIOp::create(b, loc, at[0], + index(b, inputSide)), + at[1]); + // FP32: line 4 * pixel + c / 4 of group c / 16. + Value line = pixel, lane = at[2]; + if (floating) { + Value block = arith::DivUIOp::create(b, loc, at[2], + index(b, 16)); + Value inBlock = arith::RemUIOp::create( + b, loc, at[2], index(b, 16)); + line = arith::AddIOp::create( + b, loc, + arith::MulIOp::create(b, loc, pixel, + index(b, 4)), + arith::DivUIOp::create(b, loc, inBlock, + index(b, 4))); + lane = arith::AddIOp::create( + b, loc, + arith::MulIOp::create(b, loc, block, + index(b, 4)), + arith::RemUIOp::create(b, loc, inBlock, + index(b, 4))); + } + memref::StoreOp::create(b, loc, value, pack, + ValueRange{line, lane}); + }); + int64_t pixels = inputSide * inputSide; + source = BankAllocOp::create(b, loc, i64, 1, blocks); + if (floating) { + Value values = + BankAllocOp::create(b, loc, i64, 1, blocks); + values = BankMvinOp::create(b, loc, i64, pack, values, + pixels * 4, 1); + source = ::buddy::toint8_ball::BankFp32ToInt8Op::create( + b, loc, i64, values, source, inputMultiplier, + pixels * 4); + BankReleaseOp::create(b, loc, values); + } else { + source = BankMvinOp::create(b, loc, i64, pack, source, + pixels, 1); + } + } + Value a = BankAllocOp::create(b, loc, i64, 1, outputGroups); + a = ::buddy::im2col_ball::BankIm2colI8Op::create( + b, loc, i64, source, a, inputSide, kernel, stride, + im2colPadding, pebble::tileRows()); + if (!residentInput) + BankReleaseOp::create(b, loc, source); + Value w = BankAllocOp::create(b, loc, i64, 1, outputGroups); + w = BankMvinOp::create(b, loc, i64, weightPacks[index_], w, + 16 * tiles, 1); + // The first chunk starts from the bias, the others from + // zero and add to the accumulator. + Value partial = BankAllocOp::create(b, loc, i64, 1, 1); + partial = BankMvinOp::create( + b, loc, i64, acc ? zeroPack : biasPack, partial, 4, 1); + partial = ::buddy::smatmul_ball::BankMatmulI8Op::create( + b, loc, i64, a, w, partial, rows, 16 * tiles); + BankReleaseOp::create(b, loc, a); + BankReleaseOp::create(b, loc, w); + if (!acc) { + acc = partial; + continue; + } + Value sum = BankAllocOp::create(b, loc, i64, 1, 1); + sum = ::buddy::matadd_ball::BankAddI32Op::create( + b, loc, i64, acc, partial, sum, rows * 4); + BankReleaseOp::create(b, loc, acc); + BankReleaseOp::create(b, loc, partial); + acc = sum; + } + Value converted = BankAllocOp::create(b, loc, i64, 1, 1); + converted = + fused + ? Value(::buddy::toint8_ball::BankInt32ToInt8Op::create( + b, loc, i64, acc, scales, converted, rows, relu)) + : Value(::buddy::int2fp_ball::BankInt32ToFp32Op::create( + b, loc, i64, acc, scales, converted, rows, relu)); + BankReleaseOp::create(b, loc, acc); + if (pool > 1) { + Value pooledBank = BankAllocOp::create(b, loc, i64, 1, 1); + pooledBank = ::buddy::maxpool_ball::BankMaxpoolI8Op::create( + b, loc, i64, converted, pooledBank, side, pool, pool, 0); + BankReleaseOp::create(b, loc, converted); + converted = pooledBank; + } + if (resident) { + result = converted; + return {}; + } + converted = BankMvoutOp::create( + b, loc, i64, outPack, converted, + fused ? (pool > 1 ? pooled * pooled : rows) : rows * 4, 1); + BankReleaseOp::create(b, loc, converted); + // The pixels of the tile inside the output. + Value py = arith::DivUIOp::create(b, loc, y, index(b, pool)); + Value px = arith::DivUIOp::create(b, loc, x, index(b, pool)); + loops( + b, + {extent(b, pooled, height / pool, py), + extent(b, pooled, width / pool, px), lanes}, + [&](OpBuilder &b, ValueRange at) { + Value pixel = arith::AddIOp::create( + b, loc, + arith::MulIOp::create(b, loc, at[0], + index(b, pooled)), + at[1]); + Value value = memref::LoadOp::create( + b, loc, outPack, ValueRange{pixel, at[2]}); + memref::StoreOp::create( + b, loc, value, output, + ValueRange{index(b, 0), + arith::AddIOp::create(b, loc, at[0], py), + arith::AddIOp::create(b, loc, at[1], px), + arith::AddIOp::create(b, loc, at[2], oc)}); + }); + return {}; + }); + return {}; + }); + BankReleaseOp::create(b, loc, scales); + return result; + }; + + if (resident) { + // Output blocks append their groups to one bank. + for (int64_t oc = 0; oc < cout; oc += 16) { + Value block = outputBlock(builder, index(builder, oc)); + image = image ? BankTransferOp::create(builder, loc, i64, block, image) + .getResult() + : block; + } + } else { + pebble::repeat(builder, loc, 0, cout, 16, false, {}, + [&](OpBuilder &b, Value oc, ValueRange) { + outputBlock(b, oc); + return SmallVector{}; + }); + tensorOutput = bufferization::ToTensorOp::create(builder, loc, outputType, + output, true, true); + } + if (residentInput) + BankReleaseOp::create(builder, loc, residentInput); + return success(); +} +} // namespace + +LogicalResult pebble::lowerSpatialOps(func::FuncOp function, + DenseMap &residentVectors, + SmallVector &consumed) { + SmallVector ops; + function.walk([&](Computation op) { + if (op.getKind() == "conv2d_i8" || op.getKind() == "depthwise_conv2d_i8" || + op.getKind() == "flatten_nchw") + ops.push_back(op); + }); + DenseMap images; + for (auto op : ops) { + Value input = images.lookup(op.getInputs()[0]); + if (op.getKind() == "flatten_nchw") { + // A 4x4 map of sixteen channels in channel-major order: the transposed + // 16x16 block, one K block of sixteen rows for a one-row matmul. + OpBuilder b(op); + Value vector = BankAllocOp::create(b, op.getLoc(), b.getI64Type(), 1, 1); + vector = ::buddy::transpose_ball::BankTransposeI8Op::create( + b, op.getLoc(), b.getI64Type(), input, vector, 16, 16); + BankReleaseOp::create(b, op.getLoc(), input); + residentVectors[op.getResult(0)] = vector; + consumed.push_back(op); + continue; + } + Value image, tensor; + if (failed(convolution(op, input, image, tensor))) + return failure(); + if (image) { + images[op.getResult(0)] = image; + consumed.push_back(op); + pebble::observeResident(op, image); + } else { + op.getResult(0).replaceAllUsesWith(tensor); + op.erase(); + } + } + return success(); +} diff --git a/examples/designs/pebble/stack/device/core/src/Lowering.h b/examples/designs/pebble/stack/device/core/src/Lowering.h new file mode 100644 index 000000000..ebdb49a11 --- /dev/null +++ b/examples/designs/pebble/stack/device/core/src/Lowering.h @@ -0,0 +1,81 @@ +//===- Lowering.h - Custom computations in the Pebble midend -------------===// +#ifndef PEBBLE_LOWERING_H +#define PEBBLE_LOWERING_H + +#include "Custom/CustomOps.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/IR/Builders.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLFunctionalExtras.h" + +namespace mlir::linalg { +class GenericOp; +} +namespace pebble { +bool elementwise(mlir::linalg::GenericOp op, llvm::StringRef name); +mlir::Value tensorBuffer(mlir::OpBuilder &builder, mlir::Location loc, + mlir::Value tensor); + +// Emits `body` for index = begin, begin + step, ... < end and threads the +// carried banks through it. The iterations share one scf.for unless `unroll` +// is set: then each iteration gets a constant index, for bodies whose index +// selects a bank. +llvm::SmallVector +repeat(mlir::OpBuilder &builder, mlir::Location loc, int64_t begin, int64_t end, + int64_t step, bool unroll, mlir::ValueRange carried, + llvm::function_ref( + mlir::OpBuilder &, mlir::Value, mlir::ValueRange)> + body); + +// The bank capabilities the core tiles for: a matmul_i8 takes at most +// matmulRows() rows, im2col_i8 output feeds it in M tiles of tileRows(). +int64_t matmulRows(); +int64_t tileRows(); + +// Moves the cycle markers of `op`, whose result stays in `bank`, to the bank. +void observeResident(mlir::Operation *op, mlir::Value bank); + +// The groups of the im2col_i8 output (and of the matching weights) for +// `windows` windows of a `kernel` x `kernel` convolution over `blocks` +// channel blocks: the fewest that divide the K tiles and fit a group each. +// Zero when they exceed the banks a convolution may use. +int64_t im2colGroups(int64_t kernel, int64_t blocks, int64_t windows); + +// A convolution computes `side` x `side` output pixels (a multiple of the +// pool window) per matmul from an input tile of `inputSide` pixels per side, +// and reads the input channels in chunks of `chunk` channel blocks. An FP32 +// input tile takes four lines per pixel. `side` is zero when no tile fits. +struct ConvTiling { + int64_t side, inputSide, chunk; +}; +ConvTiling convTiling(int64_t height, int64_t width, int64_t pool, + int64_t stride, int64_t kernel, int64_t blocks, + bool depthwise, bool floating); + +// A Linear reduces K in `calls` matmul calls of `groups` groups of A and B, +// `blockK` columns each, over `rows` rows per tile. +struct LinearSplit { + int64_t calls, groups, blockK; +}; +LinearSplit linearSplit(int64_t k, int64_t rows, bool floating); + +// The result of a Linear with custom.resident stays in `resident`, one +// group per sixteen columns; the others give `result`. +mlir::LogicalResult lowerLinear(::buddy::custom::CustomOp op, + mlir::Value residentInput, mlir::Value &result, + mlir::Value &resident); +mlir::LogicalResult lowerGlobalAverage(::buddy::custom::CustomOp op, + mlir::Value &result); +mlir::LogicalResult lowerChannelOp(::buddy::custom::CustomOp op, + mlir::Value &result); +mlir::LogicalResult lowerTranspose(::buddy::custom::CustomOp op, + mlir::Value &result); +void formSpatialOps(mlir::func::FuncOp function); +// Lowers the convolutions; the ones whose results stay in banks remain in +// `consumed` until their consumers are lowered. +mlir::LogicalResult +lowerSpatialOps(mlir::func::FuncOp function, + llvm::DenseMap &residentVectors, + llvm::SmallVector &consumed); +} // namespace pebble +#endif diff --git a/examples/designs/pebble/stack/device/core/tests/form-custom-ops.mlir b/examples/designs/pebble/stack/device/core/tests/form-custom-ops.mlir new file mode 100644 index 000000000..54dd3fe48 --- /dev/null +++ b/examples/designs/pebble/stack/device/core/tests/form-custom-ops.mlir @@ -0,0 +1,83 @@ +// CHECK-LABEL: func.func @named +// CHECK-NOT: linalg.matmul +// CHECK: custom.op "matmul_i8"(%arg0, %arg1, %arg2) +// CHECK: return +func.func @named(%a: tensor<1x32xi8>, %b: tensor<32x32xi8>, %initial: tensor<1x32xi32>) -> tensor<1x32xi32> { + %product = linalg.matmul ins(%a, %b: tensor<1x32xi8>, tensor<32x32xi8>) outs(%initial: tensor<1x32xi32>) -> tensor<1x32xi32> + return %product : tensor<1x32xi32> +} + +// CHECK-LABEL: func.func @generic +// CHECK-NOT: linalg.generic +// CHECK: custom.op "matmul_i8"(%arg0, %arg1, %arg2) +// CHECK: return +func.func @generic(%a: tensor<1x32xi8>, %b: tensor<32x32xi8>, %initial: tensor<1x32xi32>) -> tensor<1x32xi32> { + %product = linalg.generic { + indexing_maps = [affine_map<(m,n,k)->(m,k)>, affine_map<(m,n,k)->(k,n)>, affine_map<(m,n,k)->(m,n)>], + iterator_types = ["parallel", "parallel", "reduction"] + } ins(%a, %b: tensor<1x32xi8>, tensor<32x32xi8>) outs(%initial: tensor<1x32xi32>) { + ^bb0(%lhs: i8, %rhs: i8, %acc: i32): + %x = arith.extsi %lhs : i8 to i32 + %y = arith.extsi %rhs : i8 to i32 + %p = arith.muli %x, %y : i32 + %s = arith.addi %acc, %p : i32 + linalg.yield %s : i32 + } -> tensor<1x32xi32> + return %product : tensor<1x32xi32> +} + +// CHECK-LABEL: func.func @linear +// CHECK: %[[Q:.*]] = "frontend.fp32_to_int8" +// CHECK: %[[D:.*]] = "frontend.int8_to_fp32"(%[[Q]], +// CHECK-NOT: linalg.generic +// CHECK: %[[R:.*]] = custom.op "linear_i8"(%arg0, %arg1, %{{.*}}, %arg2, %arg3, %arg4) +// CHECK: return %[[R]], %[[D]] +func.func @linear(%a: tensor<1x32xf32>, %w: tensor<32x32xi8>, %multiplier: tensor<1x1xf32>, %input_step: tensor<1x1xf32>, %weight_step: tensor<1x32xf32>) -> (tensor<1x32xf32>, tensor<1x32xf32>) { + %zero = arith.constant dense<0.0> : tensor<1x32xf32> + %q = "frontend.fp32_to_int8"(%a, %multiplier) : (tensor<1x32xf32>, tensor<1x1xf32>) -> tensor<1x32xi8> + %x = "frontend.int8_to_fp32"(%q, %input_step) : (tensor<1x32xi8>, tensor<1x1xf32>) -> tensor<1x32xf32> + %y = "frontend.int8_to_fp32"(%w, %weight_step) : (tensor<32x32xi8>, tensor<1x32xf32>) -> tensor<32x32xf32> + %product = linalg.generic { + indexing_maps = [affine_map<(m,n,k)->(m,k)>, affine_map<(m,n,k)->(k,n)>, affine_map<(m,n,k)->(m,n)>], + iterator_types = ["parallel", "parallel", "reduction"] + } ins(%x, %y: tensor<1x32xf32>, tensor<32x32xf32>) outs(%zero: tensor<1x32xf32>) { + ^bb0(%lhs: f32, %rhs: f32, %acc: f32): + %p = arith.mulf %lhs, %rhs : f32 + %s = arith.addf %acc, %p : f32 + linalg.yield %s : f32 + } -> tensor<1x32xf32> + return %product, %x : tensor<1x32xf32>, tensor<1x32xf32> +} + +// CHECK-LABEL: func.func @different_body +// CHECK-NOT: custom.op +// CHECK: linalg.generic +// CHECK: arith.subi +// CHECK: return +func.func @different_body(%a: tensor<1x32xi8>, %b: tensor<32x32xi8>, %initial: tensor<1x32xi32>) -> tensor<1x32xi32> { + %product = linalg.generic { + indexing_maps = [affine_map<(m,n,k)->(m,k)>, affine_map<(m,n,k)->(k,n)>, affine_map<(m,n,k)->(m,n)>], + iterator_types = ["parallel", "parallel", "reduction"] + } ins(%a, %b: tensor<1x32xi8>, tensor<32x32xi8>) outs(%initial: tensor<1x32xi32>) { + ^bb0(%lhs: i8, %rhs: i8, %acc: i32): + %x = arith.extsi %lhs : i8 to i32 + %y = arith.extsi %rhs : i8 to i32 + %p = arith.muli %x, %y : i32 + %s = arith.subi %acc, %p : i32 + linalg.yield %s : i32 + } -> tensor<1x32xi32> + return %product : tensor<1x32xi32> +} + +// CHECK-LABEL: func.func @shared_producer +// CHECK: %[[CODES:.*]] = "frontend.fp32_to_int8" +// CHECK-NOT: "frontend.int8_to_fp32" +// CHECK: custom.op "linear_i8"(%arg0, %[[CODES]], %{{.*}}, %arg1, %arg2, %arg2) +// CHECK: return +func.func @shared_producer(%a: tensor<1x1xf32>, %multiplier: tensor<1x1xf32>, %step: tensor<1x1xf32>) -> tensor<1x1xf32> { + %zero = arith.constant dense<0.0> : tensor<1x1xf32> + %q = "frontend.fp32_to_int8"(%a, %multiplier) : (tensor<1x1xf32>, tensor<1x1xf32>) -> tensor<1x1xi8> + %x = "frontend.int8_to_fp32"(%q, %step) : (tensor<1x1xi8>, tensor<1x1xf32>) -> tensor<1x1xf32> + %product = linalg.matmul ins(%x, %x: tensor<1x1xf32>, tensor<1x1xf32>) outs(%zero: tensor<1x1xf32>) -> tensor<1x1xf32> + return %product : tensor<1x1xf32> +} diff --git a/examples/designs/pebble/stack/device/core/tests/form-depthwise.mlir b/examples/designs/pebble/stack/device/core/tests/form-depthwise.mlir new file mode 100644 index 000000000..2dc476017 --- /dev/null +++ b/examples/designs/pebble/stack/device/core/tests/form-depthwise.mlir @@ -0,0 +1,33 @@ +// CHECK-LABEL: func.func @depthwise +// CHECK: custom.op "depthwise_conv2d_i8" +// CHECK-SAME: padding = array +// CHECK: return +func.func @depthwise(%x: tensor<1x4x4x16xf32>, %w: tensor<16x1x3x3xi8>, %bias_codes: tensor<16xi32>, %input_step: tensor<1x1x1x1xf32>, %multiplier: tensor<1x1x1x1xf32>, %weight_step: tensor<16x1x1x1xf32>) -> tensor<1x4x4x16xf32> { + %q = "frontend.fp32_to_int8"(%x,%multiplier) : (tensor<1x4x4x16xf32>,tensor<1x1x1x1xf32>) -> tensor<1x4x4x16xi8> + %values = "frontend.int8_to_fp32"(%q,%input_step) : (tensor<1x4x4x16xi8>,tensor<1x1x1x1xf32>) -> tensor<1x4x4x16xf32> + %zero = arith.constant 0.0 : f32 + %padded = tensor.pad %values low[0,1,1,0] high[0,1,1,0] { + ^bb0(%n: index, %h: index, %wi: index, %c: index): + tensor.yield %zero : f32 + } : tensor<1x4x4x16xf32> to tensor<1x6x6x16xf32> + %wf = "frontend.int8_to_fp32"(%w,%weight_step) : (tensor<16x1x3x3xi8>,tensor<16x1x1x1xf32>) -> tensor<16x1x3x3xf32> + %we = tensor.empty() : tensor<3x3x16x1xf32> + %weights = linalg.transpose ins(%wf: tensor<16x1x3x3xf32>) outs(%we: tensor<3x3x16x1xf32>) permutation = [2,3,0,1] + %scale = linalg.generic {indexing_maps = [affine_map<(f,c,h,w)->(0,c,h,w)>,affine_map<(f,c,h,w)->(f,c,h,w)>,affine_map<(f,c,h,w)->(f,c,h,w)>], iterator_types = ["parallel","parallel","parallel","parallel"]} ins(%input_step,%weight_step: tensor<1x1x1x1xf32>,tensor<16x1x1x1xf32>) outs(%weight_step: tensor<16x1x1x1xf32>) { + ^bb0(%a: f32,%b: f32,%o: f32): + %r = arith.mulf %a,%b : f32 + linalg.yield %r : f32 + } -> tensor<16x1x1x1xf32> + %bs = tensor.collapse_shape %scale [[0,1,2,3]] : tensor<16x1x1x1xf32> into tensor<16xf32> + %bias = "frontend.int32_to_fp32"(%bias_codes,%bs) : (tensor<16xi32>,tensor<16xf32>) -> tensor<16xf32> + %initial = arith.constant dense<0.0> : tensor<1x4x4x16x1xf32> + %convolution = linalg.depthwise_conv_2d_nhwc_hwcm {strides = dense<1> : tensor<2xi64>, dilations = dense<1> : tensor<2xi64>} ins(%padded,%weights: tensor<1x6x6x16xf32>,tensor<3x3x16x1xf32>) outs(%initial: tensor<1x4x4x16x1xf32>) -> tensor<1x4x4x16x1xf32> + %collapsed = tensor.collapse_shape %convolution [[0],[1],[2],[3,4]] : tensor<1x4x4x16x1xf32> into tensor<1x4x4x16xf32> + %empty = tensor.empty() : tensor<1x4x4x16xf32> + %result = linalg.generic {indexing_maps = [affine_map<(n,h,w,c)->(c)>,affine_map<(n,h,w,c)->(n,h,w,c)>,affine_map<(n,h,w,c)->(n,h,w,c)>], iterator_types = ["parallel","parallel","parallel","parallel"]} ins(%bias,%collapsed: tensor<16xf32>,tensor<1x4x4x16xf32>) outs(%empty: tensor<1x4x4x16xf32>) { + ^bb0(%a: f32,%b: f32,%o: f32): + %r = arith.addf %a,%b : f32 + linalg.yield %r : f32 + } -> tensor<1x4x4x16xf32> + return %result : tensor<1x4x4x16xf32> +} diff --git a/examples/designs/pebble/stack/device/core/tests/form-int32-bias.mlir b/examples/designs/pebble/stack/device/core/tests/form-int32-bias.mlir new file mode 100644 index 000000000..b3148f674 --- /dev/null +++ b/examples/designs/pebble/stack/device/core/tests/form-int32-bias.mlir @@ -0,0 +1,61 @@ +// CHECK-LABEL: func.func @linear +// CHECK: buddy_trace.start +// CHECK: %[[R:.*]] = custom.op "linear_i8"(%arg0, %arg1, %arg2, %arg3, %arg4, %arg5) +// CHECK: %[[T:.*]] = buddy_trace.end %[[R]] +// CHECK-NOT: arith.addf +// CHECK-NOT: frontend.int32_to_fp32 +// CHECK: return %[[T]] +func.func @linear(%a: tensor<1x32xf32>, %w: tensor<32x32xi8>, %bias: tensor<1x32xi32>, %multiplier: tensor<1x1xf32>, %input_step: tensor<1x1xf32>, %weight_step: tensor<1x32xf32>) -> tensor<1x32xf32> { + %zero = arith.constant dense<0.0> : tensor<1x32xf32> + %q = "frontend.fp32_to_int8"(%a, %multiplier) : (tensor<1x32xf32>, tensor<1x1xf32>) -> tensor<1x32xi8> + %x = "frontend.int8_to_fp32"(%q, %input_step) : (tensor<1x32xi8>, tensor<1x1xf32>) -> tensor<1x32xf32> + %y = "frontend.int8_to_fp32"(%w, %weight_step) : (tensor<32x32xi8>, tensor<1x32xf32>) -> tensor<32x32xf32> + buddy_trace.start {cycle = true, id = 0 : i64, id_path = [0], tag = "linear", tensor = false} + %product = linalg.matmul ins(%x, %y: tensor<1x32xf32>, tensor<32x32xf32>) outs(%zero: tensor<1x32xf32>) -> tensor<1x32xf32> + %traced = buddy_trace.end %product {cycle = true, id = 0 : i64, id_path = [0], tag = "linear", tensor = false} : tensor<1x32xf32> + %empty = tensor.empty() : tensor<1x32xf32> + %scale = linalg.generic { + indexing_maps = [affine_map<(m,n)->(m,0)>, affine_map<(m,n)->(m,n)>, affine_map<(m,n)->(m,n)>], + iterator_types = ["parallel", "parallel"] + } ins(%input_step, %weight_step: tensor<1x1xf32>, tensor<1x32xf32>) outs(%empty: tensor<1x32xf32>) { + ^bb0(%lhs: f32, %rhs: f32, %unused: f32): + %s = arith.mulf %lhs, %rhs : f32 + linalg.yield %s : f32 + } -> tensor<1x32xf32> + %bias_f32 = "frontend.int32_to_fp32"(%bias, %scale) : (tensor<1x32xi32>, tensor<1x32xf32>) -> tensor<1x32xf32> + %result = linalg.generic { + indexing_maps = [affine_map<(m,n)->(m,n)>, affine_map<(m,n)->(m,n)>, affine_map<(m,n)->(m,n)>], + iterator_types = ["parallel", "parallel"] + } ins(%traced, %bias_f32: tensor<1x32xf32>, tensor<1x32xf32>) outs(%empty: tensor<1x32xf32>) { + ^bb0(%lhs: f32, %rhs: f32, %unused: f32): + %s = arith.addf %lhs, %rhs : f32 + linalg.yield %s : f32 + } -> tensor<1x32xf32> + return %result : tensor<1x32xf32> +} + +// A bias in a different scale domain cannot initialize this accumulator. +// CHECK-LABEL: func.func @unrelated_scale +// CHECK: %[[ZERO:.*]] = arith.constant dense<0> : tensor<1x32xi32> +// CHECK: custom.op "linear_i8"(%arg0, %arg1, %[[ZERO]], %arg3, %arg4, %arg5) +// CHECK: "frontend.int32_to_fp32"(%arg2, %arg6) +// CHECK: arith.addf +// CHECK: return +func.func @unrelated_scale(%a: tensor<1x32xf32>, %w: tensor<32x32xi8>, %bias: tensor<1x32xi32>, %multiplier: tensor<1x1xf32>, %input_step: tensor<1x1xf32>, %weight_step: tensor<1x32xf32>, %bias_step: tensor<1x32xf32>) -> tensor<1x32xf32> { + %zero = arith.constant dense<0.0> : tensor<1x32xf32> + %q = "frontend.fp32_to_int8"(%a, %multiplier) : (tensor<1x32xf32>, tensor<1x1xf32>) -> tensor<1x32xi8> + %x = "frontend.int8_to_fp32"(%q, %input_step) : (tensor<1x32xi8>, tensor<1x1xf32>) -> tensor<1x32xf32> + %y = "frontend.int8_to_fp32"(%w, %weight_step) : (tensor<32x32xi8>, tensor<1x32xf32>) -> tensor<32x32xf32> + %product = linalg.matmul ins(%x, %y: tensor<1x32xf32>, tensor<32x32xf32>) outs(%zero: tensor<1x32xf32>) -> tensor<1x32xf32> + %bias_f32 = "frontend.int32_to_fp32"(%bias, %bias_step) : (tensor<1x32xi32>, tensor<1x32xf32>) -> tensor<1x32xf32> + %empty = tensor.empty() : tensor<1x32xf32> + %result = linalg.generic { + indexing_maps = [affine_map<(m,n)->(m,n)>, affine_map<(m,n)->(m,n)>, affine_map<(m,n)->(m,n)>], + iterator_types = ["parallel", "parallel"] + } ins(%product, %bias_f32: tensor<1x32xf32>, tensor<1x32xf32>) outs(%empty: tensor<1x32xf32>) { + ^bb0(%lhs: f32, %rhs: f32, %unused: f32): + %s = arith.addf %lhs, %rhs : f32 + linalg.yield %s : f32 + } -> tensor<1x32xf32> + return %result : tensor<1x32xf32> +} diff --git a/examples/designs/pebble/stack/device/core/tests/lower-bank-updates.mlir b/examples/designs/pebble/stack/device/core/tests/lower-bank-updates.mlir new file mode 100644 index 000000000..f97c826a3 --- /dev/null +++ b/examples/designs/pebble/stack/device/core/tests/lower-bank-updates.mlir @@ -0,0 +1,7 @@ +// CHECK-LABEL: func.func @define_quantized_rows +// CHECK: arith.constant 4298115073 : i64 +// CHECK: return +func.func @define_quantized_rows() { + toint8_ball.int32_to_int8 <{values = 1 : i64, scales = 2 : i64, output = 3 : i64, rows = 1 : i64}> + return +} diff --git a/examples/designs/pebble/stack/device/core/tests/lower-custom-ops.mlir b/examples/designs/pebble/stack/device/core/tests/lower-custom-ops.mlir new file mode 100644 index 000000000..db9942734 --- /dev/null +++ b/examples/designs/pebble/stack/device/core/tests/lower-custom-ops.mlir @@ -0,0 +1,25 @@ +// CHECK-LABEL: func.func @named +// CHECK-NOT: fp32_to_int8 +// CHECK: buckyball.bank_mvin +// CHECK: scf.for {{.*}} to %c32{{.*}} step %c16 +// CHECK: smatmul_ball.bank_matmul_i8 {{.*}}rows = 1, k = 32> +// CHECK: buckyball.bank_mvout +// CHECK-NOT: _ball.bank_ +// CHECK: return +func.func @named(%a: tensor<1x32xi8>, %b: tensor<32x32xi8>, %initial: tensor<1x32xi32>) -> tensor<1x32xi32> { + %product = custom.op "matmul_i8"(%a, %b, %initial) : (tensor<1x32xi8>, tensor<32x32xi8>, tensor<1x32xi32>) -> tensor<1x32xi32> + return %product : tensor<1x32xi32> +} + +// CHECK-LABEL: func.func @linear +// CHECK: toint8_ball.bank_fp32_to_int8 +// CHECK: scf.for {{.*}} to %c32{{.*}} step %c16 +// CHECK: smatmul_ball.bank_matmul_i8 +// CHECK: int2fp_ball.bank_int32_to_fp32 +// CHECK: buckyball.bank_mvout +// CHECK-NOT: _ball.bank_ +// CHECK: return +func.func @linear(%a: tensor<1x32xf32>, %w: tensor<32x32xi8>, %bias: tensor<1x32xi32>, %multiplier: tensor<1x1xf32>, %input_step: tensor<1x1xf32>, %weight_step: tensor<1x32xf32>) -> tensor<1x32xf32> { + %product = custom.op "linear_i8"(%a, %w, %bias, %multiplier, %input_step, %weight_step) : (tensor<1x32xf32>, tensor<32x32xi8>, tensor<1x32xi32>, tensor<1x1xf32>, tensor<1x1xf32>, tensor<1x32xf32>) -> tensor<1x32xf32> + return %product : tensor<1x32xf32> +} diff --git a/examples/designs/pebble/stack/device/core/tests/lower-frontend-types.mlir b/examples/designs/pebble/stack/device/core/tests/lower-frontend-types.mlir new file mode 100644 index 000000000..b6ada3b79 --- /dev/null +++ b/examples/designs/pebble/stack/device/core/tests/lower-frontend-types.mlir @@ -0,0 +1,27 @@ +func.func @i32_is_not_i8(%codes: tensor<1x16xi32>, %step: tensor<1x16xf32>) -> tensor<1x16xf32> { + // expected-error @+1 {{requires an INT8 input}} + %value = "frontend.int8_to_fp32"(%codes, %step) : (tensor<1x16xi32>, tensor<1x16xf32>) -> tensor<1x16xf32> + return %value : tensor<1x16xf32> +} + +// ----- + +func.func @i8_is_not_i32(%codes: tensor<1x16xi8>, %step: tensor<1x16xf32>) -> tensor<1x16xf32> { + // expected-error @+1 {{requires an INT32 input}} + %value = "frontend.int32_to_fp32"(%codes, %step) : (tensor<1x16xi8>, tensor<1x16xf32>) -> tensor<1x16xf32> + return %value : tensor<1x16xf32> +} + +// ----- + +// VALID-LABEL: func.func @valid_formats +// VALID: arith.sitofp {{.*}} : i8 to f32 +// VALID: arith.mulf +// VALID: arith.sitofp {{.*}} : i32 to f32 +// VALID: arith.mulf +// VALID: return +func.func @valid_formats(%i8_codes: tensor<1x16xi8>, %i32_codes: tensor<1x16xi32>, %step: tensor<1x16xf32>) -> (tensor<1x16xf32>, tensor<1x16xf32>) { + %first = "frontend.int8_to_fp32"(%i8_codes, %step) : (tensor<1x16xi8>, tensor<1x16xf32>) -> tensor<1x16xf32> + %second = "frontend.int32_to_fp32"(%i32_codes, %step) : (tensor<1x16xi32>, tensor<1x16xf32>) -> tensor<1x16xf32> + return %first, %second : tensor<1x16xf32>, tensor<1x16xf32> +} diff --git a/examples/designs/pebble/stack/device/core/tests/unknown-custom-op.mlir b/examples/designs/pebble/stack/device/core/tests/unknown-custom-op.mlir new file mode 100644 index 000000000..34ab31381 --- /dev/null +++ b/examples/designs/pebble/stack/device/core/tests/unknown-custom-op.mlir @@ -0,0 +1,5 @@ +func.func @unknown(%input: tensor<1x32xf32>) -> tensor<1x32xf32> { + // expected-error @+1 {{has no bank lowering on this core for unknown}} + %result = custom.op "unknown"(%input) : (tensor<1x32xf32>) -> tensor<1x32xf32> + return %result : tensor<1x32xf32> +} diff --git a/examples/designs/pebble/workloads/CMakeLists.txt b/examples/designs/pebble/workloads/CMakeLists.txt new file mode 100644 index 000000000..1d19718bb --- /dev/null +++ b/examples/designs/pebble/workloads/CMakeLists.txt @@ -0,0 +1,24 @@ +get_filename_component(BUCKYBALL_REPO_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../../.. ABSOLUTE) + +set(BUCKYBALL_DESIGN_COMMON_SOURCES + ${CMAKE_CURRENT_SOURCE_DIR}/common/pebble.c) +set(BUCKYBALL_DESIGN_COMMON_HEADERS + ${CMAKE_CURRENT_SOURCE_DIR}/common/pebble.h) +set(BUCKYBALL_DESIGN_COMMON_INCLUDE_DIRS + ${CMAKE_CURRENT_SOURCE_DIR}/common) +set(BUCKYBALL_CTEST_TARGET pebble) + +add_custom_target(pebble-ctests-build) +set(BUCKYBALL_CTEST_GROUP_TARGET pebble-ctests-build) + +add_subdirectory(device) + +add_custom_target(design-ctests-build DEPENDS pebble-ctests-build) + +add_custom_target(design-workloads-build ALL + DEPENDS + design-ctests-build + mlirtest-workloads-build + soctests-build + COMMENT "Building pebble workloads" + VERBATIM) diff --git a/examples/chips/pebble/workloads/common/pebble.c b/examples/designs/pebble/workloads/common/pebble.c similarity index 100% rename from examples/chips/pebble/workloads/common/pebble.c rename to examples/designs/pebble/workloads/common/pebble.c diff --git a/examples/chips/pebble/workloads/common/pebble.h b/examples/designs/pebble/workloads/common/pebble.h similarity index 94% rename from examples/chips/pebble/workloads/common/pebble.h rename to examples/designs/pebble/workloads/common/pebble.h index fec3265d8..2f021f2b4 100644 --- a/examples/chips/pebble/workloads/common/pebble.h +++ b/examples/designs/pebble/workloads/common/pebble.h @@ -2,12 +2,11 @@ #define PEBBLE_H #include "buckyball.h" -#include #include enum { - PEBBLE_INT8_LANES = BANK_WIDTH / 8, - PEBBLE_ACC_LANES = BANK_WIDTH / 8, + PEBBLE_INT8_LANES = THREAD_MEM_BANK_WIDTH / 8, + PEBBLE_ACC_LANES = THREAD_MEM_BANK_WIDTH / 8, }; /* diff --git a/examples/designs/pebble/workloads/configs/tests.toml b/examples/designs/pebble/workloads/configs/tests.toml new file mode 100644 index 000000000..c1c7ceb33 --- /dev/null +++ b/examples/designs/pebble/workloads/configs/tests.toml @@ -0,0 +1,2 @@ +controlHart = 0 +reportHart = 0 diff --git a/examples/designs/pebble/workloads/device/CMakeLists.txt b/examples/designs/pebble/workloads/device/CMakeLists.txt new file mode 100644 index 000000000..ad6d4787c --- /dev/null +++ b/examples/designs/pebble/workloads/device/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(core) diff --git a/examples/designs/pebble/workloads/device/core/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/CMakeLists.txt new file mode 100644 index 000000000..2a6ad3b6a --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(acc) diff --git a/examples/designs/pebble/workloads/device/core/acc/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/CMakeLists.txt new file mode 100644 index 000000000..d17debe0e --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/CMakeLists.txt @@ -0,0 +1,10 @@ +add_subdirectory(ctests) +foreach(BUCKYBALL_TARGET IN LISTS DESIGN_CPU_TARGETS) + set(BUCKYBALL_CTEST_TARGET "${BUCKYBALL_TARGET}") + foreach(BUCKYBALL_BALL_DIR IN LISTS DESIGN_PROFILES_${BUCKYBALL_TARGET}_THREAD_BALLDOMAIN_BALLS) + add_subdirectory( + "${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/${BUCKYBALL_BALL_DIR}/workloads/ctests" + "${CMAKE_CURRENT_BINARY_DIR}/balls/${BUCKYBALL_TARGET}/${BUCKYBALL_BALL_DIR}/ctests") + endforeach() +endforeach() +add_subdirectory(mlir_tests) diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/ctests/CMakeLists.txt new file mode 100644 index 000000000..6fe03733c --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/CMakeLists.txt @@ -0,0 +1,28 @@ +add_buckyball_ctest(bank_capacity_rename_test.c) +add_buckyball_ctest(bank_recycle_test.c) +add_buckyball_ctest(bank_transfer_recycle_test.c) +add_buckyball_ctest(mega_conv_pipeline_test.c) +add_buckyball_ctest(mega_conv_k5_pipeline_test.c) +add_buckyball_ctest(residual_add_lut_test.c) +add_buckyball_ctest(maxpool_im2col_test.c) +add_buckyball_ctest(se_gate_lut_test.c) +add_buckyball_ctest(dual_smatmul_matadd_test.c) +add_buckyball_ctest(dual_conv_add_int2fp_test.c) +add_buckyball_ctest(quant_smatmul_int2fp_test.c) +add_buckyball_ctest(smatmul_mvout_bank_reuse_test.c) +add_buckyball_ctest(mvin_bank78_stress_test.c) +add_buckyball_ctest(im2col_mvin_smatmul_hazard_test.c) + +add_subdirectory(im2col_smatmul_quant_pipeline) +add_subdirectory(maxpool_partial_update) +add_subdirectory(bank_dma_reuse) +add_subdirectory(bank_rename_overlap) +add_subdirectory(bank_dma_pingpong) +add_subdirectory(read_bank_shared) +add_subdirectory(read_bank_split) +add_subdirectory(same_ball_independent) +add_subdirectory(rob_window_probe) +add_subdirectory(write_bank_versions) +if(DESIGN_PROFILES_${BUCKYBALL_CTEST_TARGET}_CORE_SHARED_MEM_ENABLE) + add_subdirectory(shared_bank_versions) +endif() diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_capacity_rename_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_capacity_rename_test.c new file mode 100644 index 000000000..2119c1e1a --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_capacity_rename_test.c @@ -0,0 +1,37 @@ +#include +#include +#include + +_Static_assert(CORE_VIRTUAL_BANK_COUNT >= THREAD_MEM_BANK_NUM, + "Test requires one virtual ID per physical bank"); +static uint8_t input[THREAD_MEM_BANK_NUM][THREAD_MEM_BANK_WIDTH / 8] + __attribute__((aligned(64))); +static uint8_t output[THREAD_MEM_BANK_NUM][THREAD_MEM_BANK_WIDTH / 8] + __attribute__((aligned(64))); + +int main(void) { + for (unsigned bank = 0; bank < THREAD_MEM_BANK_NUM; ++bank) + bb_mem_alloc(bank, 1, 1); + for (unsigned round = 0; round < 3; ++round) { + for (unsigned bank = 0; bank < THREAD_MEM_BANK_NUM; ++bank) + for (unsigned byte = 0; byte < THREAD_MEM_BANK_WIDTH / 8; ++byte) { + input[bank][byte] = bank * 7 + byte + round * 31; + output[bank][byte] = 0; + } + for (unsigned bank = 0; bank < THREAD_MEM_BANK_NUM; ++bank) + bb_mvin((uintptr_t)input[bank], bank, 1, 1); + for (unsigned bank = 0; bank < THREAD_MEM_BANK_NUM; ++bank) + bb_mvout((uintptr_t)output[bank], bank, 1, 1); + for (unsigned bank = 0; bank < THREAD_MEM_BANK_NUM; ++bank) + for (unsigned byte = 0; byte < THREAD_MEM_BANK_WIDTH / 8; ++byte) + if (output[bank][byte] != input[bank][byte]) { + printf("capacity rename mismatch round=%u bank=%u byte=%u\n", round, + bank, byte); + return 1; + } + } + for (unsigned bank = 0; bank < THREAD_MEM_BANK_NUM; ++bank) + bb_mem_release(bank); + puts("Bank capacity rename PASS"); + return 0; +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_pingpong/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_pingpong/CMakeLists.txt new file mode 100644 index 000000000..f789e46c0 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_pingpong/CMakeLists.txt @@ -0,0 +1,3 @@ +add_buckyball_component_workload(bank_dma_pingpong SOURCES + bank_dma_pingpong-main.cpp + pebble.c) diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_pingpong/bank_dma_pingpong-main.cpp b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_pingpong/bank_dma_pingpong-main.cpp new file mode 100644 index 000000000..c4736d492 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_pingpong/bank_dma_pingpong-main.cpp @@ -0,0 +1,22 @@ +#include +extern "C" int pebble_pipeline(uint32_t instance); +static volatile uint32_t completed, failed; +int main() { + const uint32_t hart = bb_get_hart_id(); + const uint32_t expected = + bb_profile_core_count(BB_COMPONENT_WORKLOAD_PROFILE); + if (!expected) + return 1; + if (bb_hart_profile(hart) == BB_COMPONENT_WORKLOAD_PROFILE) { + __atomic_fetch_or(&failed, pebble_pipeline(bb_hart_slot(hart)) != 0, + __ATOMIC_RELEASE); + __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); + } + if (hart == WORKLOAD_REPORT_HART) { + while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) + asm volatile("nop"); + return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; + } + for (;;) + asm volatile("wfi"); +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_pingpong/pebble.c b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_pingpong/pebble.c new file mode 100644 index 000000000..a8579daeb --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_pingpong/pebble.c @@ -0,0 +1,55 @@ +#include "buckyball.h" +#include +#include +#include +#include +#include +#include +#include + +enum { M = 8, N = 16, K = 64, LANES = THREAD_MEM_BANK_WIDTH / 8 }; +static int8_t input[BB_NUM_CORES][M * K] __attribute__((aligned(64))); +static int8_t first[BB_NUM_CORES][K * N] __attribute__((aligned(64))); +static int8_t second[BB_NUM_CORES][K * N] __attribute__((aligned(64))); +static int32_t bias[BB_NUM_CORES][N] __attribute__((aligned(64))); +static int32_t out0[BB_NUM_CORES][M * N] __attribute__((aligned(64))); +static int32_t out1[BB_NUM_CORES][M * N] __attribute__((aligned(64))); +static volatile int32_t warmup[BB_NUM_CORES][N] __attribute__((aligned(64))); +extern "C" int pebble_pipeline(uint32_t instance) { + int c = instance; + for (int i = 0; i < M * K; ++i) + input[c][i] = 1; + for (int i = 0; i < K * N; ++i) { + first[c][i] = 2; + second[c][i] = 3; + } + for (int bank = 0; bank < 6; ++bank) + bb_mem_alloc(bank, 1, 1); + bb_mvin((uintptr_t)input[c], 0, M * K / LANES, 1); + bb_mvin((uintptr_t)first[c], 1, K * N / LANES, 1); + bb_mvin((uintptr_t)bias[c], 2, N * 4 / LANES, 1); + bb_smatmul_bias(2, 0); + bb_smatmul_os(0, 1, 3, 1, N, N, 1, 1, 0); + bb_mvout((uintptr_t)warmup[c], 3, N * 4 / LANES, 1); + if (warmup[c][0] != 2 * N) + return 1; + uint64_t start, end; + asm volatile("rdcycle %0" : "=r"(start)); + bb_smatmul_os(0, 1, 3, M, N, K, 1, 1, 0); + bb_mvin((uintptr_t)second[c], 5, K * N / LANES, 1); + bb_smatmul_os(0, 5, 4, M, N, K, 1, 1, 0); + bb_mvout((uintptr_t)out0[c], 3, M * N * 4 / LANES, 1); + bb_mvout((uintptr_t)out1[c], 4, M * N * 4 / LANES, 1); + for (int i = 0; i < M * N; ++i) + if (out0[c][i] != 2 * K || out1[c][i] != 3 * K) { + printf("bank_dma_pingpong FAIL index=%d actual=%d,%d\n", i, out0[c][i], + out1[c][i]); + return 1; + } + asm volatile("rdcycle %0" : "=r"(end)); + for (int bank = 0; bank < 6; ++bank) + bb_mem_release(bank); + printf("bank_dma_pingpong PASS cycles=%llu\n", + (unsigned long long)(end - start)); + return 0; +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_reuse/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_reuse/CMakeLists.txt new file mode 100644 index 000000000..c7565bbcf --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_reuse/CMakeLists.txt @@ -0,0 +1,3 @@ +add_buckyball_component_workload(bank_dma_reuse SOURCES + bank_dma_reuse-main.cpp + pebble.c) diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_reuse/bank_dma_reuse-main.cpp b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_reuse/bank_dma_reuse-main.cpp new file mode 100644 index 000000000..c4736d492 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_reuse/bank_dma_reuse-main.cpp @@ -0,0 +1,22 @@ +#include +extern "C" int pebble_pipeline(uint32_t instance); +static volatile uint32_t completed, failed; +int main() { + const uint32_t hart = bb_get_hart_id(); + const uint32_t expected = + bb_profile_core_count(BB_COMPONENT_WORKLOAD_PROFILE); + if (!expected) + return 1; + if (bb_hart_profile(hart) == BB_COMPONENT_WORKLOAD_PROFILE) { + __atomic_fetch_or(&failed, pebble_pipeline(bb_hart_slot(hart)) != 0, + __ATOMIC_RELEASE); + __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); + } + if (hart == WORKLOAD_REPORT_HART) { + while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) + asm volatile("nop"); + return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; + } + for (;;) + asm volatile("wfi"); +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_reuse/pebble.c b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_reuse/pebble.c new file mode 100644 index 000000000..222236cd0 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_reuse/pebble.c @@ -0,0 +1,62 @@ +#include "buckyball.h" +#include +#include +#include +#include +#include +#include +#include + +enum { M = 8, N = 16, K = 64, LANES = THREAD_MEM_BANK_WIDTH / 8 }; +static int8_t input[BB_NUM_CORES][M * K] __attribute__((aligned(64))); +static int8_t first[BB_NUM_CORES][K * N] __attribute__((aligned(64))); +static int8_t second[BB_NUM_CORES][K * N] __attribute__((aligned(64))); +static int32_t bias[BB_NUM_CORES][N] __attribute__((aligned(64))); +static int32_t out0[BB_NUM_CORES][M * N] __attribute__((aligned(64))); +static int32_t out1[BB_NUM_CORES][M * N] __attribute__((aligned(64))); +static volatile int32_t warmup[BB_NUM_CORES][N] __attribute__((aligned(64))); +static volatile int8_t moved[BB_NUM_CORES][K * N] __attribute__((aligned(64))); +extern "C" int pebble_pipeline(uint32_t instance) { + int c = instance; + for (int i = 0; i < M * K; ++i) + input[c][i] = 1; + for (int i = 0; i < K * N; ++i) { + first[c][i] = 2; + second[c][i] = 3; + } + for (int bank = 0; bank < 6; ++bank) + bb_mem_alloc(bank, 1, 1); + bb_mvin((uintptr_t)input[c], 0, M * K / LANES, 1); + bb_mvin((uintptr_t)first[c], 1, K * N / LANES, 1); + bb_mvin((uintptr_t)bias[c], 2, N * 4 / LANES, 1); + bb_smatmul_bias(2, 0); + bb_smatmul_os(0, 1, 3, 1, N, N, 1, 1, 0); + bb_mvout((uintptr_t)warmup[c], 3, N * 4 / LANES, 1); + if (warmup[c][0] != 2 * N) + return 1; + uint64_t start, end; + asm volatile("rdcycle %0" : "=r"(start)); + bb_smatmul_os(0, 1, 3, M, N, K, 1, 1, 0); + bb_mvin((uintptr_t)second[c], 1, K * N / LANES, 1); + bb_smatmul_os(0, 1, 4, M, N, K, 1, 1, 0); + bb_mvout((uintptr_t)out0[c], 3, M * N * 4 / LANES, 1); + bb_mvout((uintptr_t)out1[c], 4, M * N * 4 / LANES, 1); + for (int i = 0; i < M * N; ++i) + if (out0[c][i] != 2 * K || out1[c][i] != 3 * K) { + printf("bank_dma_reuse FAIL index=%d actual=%d,%d\n", i, out0[c][i], + out1[c][i]); + return 1; + } + asm volatile("rdcycle %0" : "=r"(end)); + bb_mem_transfer(1, 6); + bb_mvout((uintptr_t)moved[c], 6, K * N / LANES, 1); + for (int i = 0; i < K * N; ++i) + if (moved[c][i] != 3) + return 1; + for (int bank = 0; bank < 7; ++bank) + if (bank != 1) + bb_mem_release(bank); + printf("bank_dma_reuse PASS cycles=%llu\n", + (unsigned long long)(end - start)); + return 0; +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_recycle_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_recycle_test.c new file mode 100644 index 000000000..4b8ab0b40 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_recycle_test.c @@ -0,0 +1,24 @@ +#include +#include +#include + +static uint64_t input[2] __attribute__((aligned(64))); +static uint64_t output[2] __attribute__((aligned(64))); + +int main(void) { + for (unsigned iteration = 0; iteration < 128; ++iteration) { + input[0] = 0x1234567800000000ULL + iteration; + input[1] = ~input[0]; + output[0] = output[1] = 0; + bb_mem_alloc(1, 1, 1); + bb_mvin((uintptr_t)input, 1, 1, 1); + bb_mvout((uintptr_t)output, 1, 1, 1); + if (output[0] != input[0] || output[1] != input[1]) { + printf("bank recycle mismatch iteration=%u\n", iteration); + return 1; + } + bb_mem_release(1); + } + puts("Bank recycle PASS"); + return 0; +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_rename_overlap/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_rename_overlap/CMakeLists.txt new file mode 100644 index 000000000..a38b3c993 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_rename_overlap/CMakeLists.txt @@ -0,0 +1,3 @@ +add_buckyball_component_workload(bank_rename_overlap SOURCES + bank_rename_overlap-main.cpp + pebble.c) diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_rename_overlap/bank_rename_overlap-main.cpp b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_rename_overlap/bank_rename_overlap-main.cpp new file mode 100644 index 000000000..c4736d492 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_rename_overlap/bank_rename_overlap-main.cpp @@ -0,0 +1,22 @@ +#include +extern "C" int pebble_pipeline(uint32_t instance); +static volatile uint32_t completed, failed; +int main() { + const uint32_t hart = bb_get_hart_id(); + const uint32_t expected = + bb_profile_core_count(BB_COMPONENT_WORKLOAD_PROFILE); + if (!expected) + return 1; + if (bb_hart_profile(hart) == BB_COMPONENT_WORKLOAD_PROFILE) { + __atomic_fetch_or(&failed, pebble_pipeline(bb_hart_slot(hart)) != 0, + __ATOMIC_RELEASE); + __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); + } + if (hart == WORKLOAD_REPORT_HART) { + while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) + asm volatile("nop"); + return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; + } + for (;;) + asm volatile("wfi"); +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_rename_overlap/pebble.c b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_rename_overlap/pebble.c new file mode 100644 index 000000000..a4660ee94 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_rename_overlap/pebble.c @@ -0,0 +1,54 @@ +#include "buckyball.h" +#include +#include +#include +#include +#include +#include + +enum { M = 8, N = 16, K = 64, LANES = THREAD_MEM_BANK_WIDTH / 8 }; +static int8_t input[BB_NUM_CORES][M * K] __attribute__((aligned(64))); +static int8_t first[BB_NUM_CORES][K * N] __attribute__((aligned(64))); +static int8_t second[BB_NUM_CORES][K * N] __attribute__((aligned(64))); +static int32_t bias[BB_NUM_CORES][N] __attribute__((aligned(64))); +static int32_t out0[BB_NUM_CORES][M * N] __attribute__((aligned(64))); +static int32_t out1[BB_NUM_CORES][M * N] __attribute__((aligned(64))); +static volatile int32_t warmup[BB_NUM_CORES][N] __attribute__((aligned(64))); +extern "C" int pebble_pipeline(uint32_t instance) { + int c = instance; + for (int i = 0; i < M * K; ++i) + input[c][i] = 1; + for (int i = 0; i < K * N; ++i) { + first[c][i] = 2; + second[c][i] = 3; + } + for (int bank = 0; bank < 5; ++bank) + bb_mem_alloc(bank, 1, 1); + bb_mvin((uintptr_t)input[c], 0, M * K / LANES, 1); + bb_mvin((uintptr_t)first[c], 1, K * N / LANES, 1); + bb_mvin((uintptr_t)bias[c], 2, N * 4 / LANES, 1); + bb_smatmul_bias(2, 0); + bb_smatmul_os(0, 1, 3, 1, N, N, 1, 1, 0); + bb_mvout((uintptr_t)warmup[c], 3, N * 4 / LANES, 1); + if (warmup[c][0] != 2 * N) + return 1; + uint64_t start, end; + asm volatile("rdcycle %0" : "=r"(start)); + bb_smatmul_os(0, 1, 3, M, N, K, 1, 1, 0); + bb_mvin((uintptr_t)second[c], 1, K * N / LANES, 1); + bb_smatmul_os(0, 1, 4, M, N, K, 1, 1, 0); + bb_mvout((uintptr_t)out0[c], 3, M * N * 4 / LANES, 1); + bb_mvout((uintptr_t)out1[c], 4, M * N * 4 / LANES, 1); + for (int i = 0; i < M * N; ++i) + if (out0[c][i] != 2 * K || out1[c][i] != 3 * K) { + printf("bank_rename_overlap FAIL index=%d actual=%d,%d\n", i, out0[c][i], + out1[c][i]); + return 1; + } + asm volatile("rdcycle %0" : "=r"(end)); + for (int bank = 0; bank < 5; ++bank) + bb_mem_release(bank); + printf("bank_rename_overlap PASS cycles=%llu\n", + (unsigned long long)(end - start)); + return 0; +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_transfer_recycle_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_transfer_recycle_test.c new file mode 100644 index 000000000..be5fb47ec --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_transfer_recycle_test.c @@ -0,0 +1,30 @@ +#include +#include +#include + +static uint64_t source[2] __attribute__((aligned(64))); +static uint64_t target[2] __attribute__((aligned(64))); +static uint64_t output[4] __attribute__((aligned(64))); + +int main(void) { + for (unsigned iteration = 0; iteration < 128; ++iteration) { + source[0] = 0x1234000000000000ULL + iteration; + source[1] = ~source[0]; + target[0] = 0x5678000000000000ULL + iteration; + target[1] = ~target[0]; + bb_mem_alloc(0, 1, 1); + bb_mem_alloc(1, 1, 1); + bb_mvin((uintptr_t)source, 0, 1, 1); + bb_mvin((uintptr_t)target, 1, 1, 1); + bb_mem_transfer(0, 1); + bb_mvout((uintptr_t)output, 1, 1, 1); + if (output[0] != target[0] || output[1] != target[1] || + output[2] != source[0] || output[3] != source[1]) { + printf("bank transfer recycle mismatch iteration=%u\n", iteration); + return 1; + } + bb_mem_release(1); + } + puts("Bank transfer recycle PASS"); + return 0; +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/dual_conv_add_int2fp_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/dual_conv_add_int2fp_test.c new file mode 100644 index 000000000..1c70757cf --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/dual_conv_add_int2fp_test.c @@ -0,0 +1,85 @@ +#include "buckyball.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum { + CORE = 16, + IN = 4, + K = 3, + OUT = 2, + WINDOWS = 4, + M = 8, + // K is (kernel row, kernel column, channel); only channel 0 is nonzero. + PACKED_K = K * K * CORE, + RESULT_ROWS = CORE * 2 +}; +static int8_t images[2][IN * IN * CORE] __attribute__((aligned(64))); +static int8_t weights[2][PACKED_K * CORE] __attribute__((aligned(64))); +static float scale[CORE] __attribute__((aligned(64))); +static int32_t bias[CORE] __attribute__((aligned(64))); +static float output[CORE * CORE] __attribute__((aligned(64))); + +static int8_t image_value(int branch, int row, int column) { + return (int8_t)(branch * 7 + row * IN + column - 8); +} + +static int32_t expected(int row, int column) { + int32_t sum = 0; + for (int branch = 0; branch < 2; ++branch) + for (int ky = 0; ky < K; ++ky) + for (int kx = 0; kx < K; ++kx) + sum += image_value(branch, row / OUT + ky, row % OUT + kx) * + weights[branch][(ky * K + kx) * CORE * CORE + column]; + return sum; +} + +static void run_branch(int branch, int out_bank) { + bb_mvin((uintptr_t)images[branch], 0, IN * IN, 1); + // One output group holds all nine K tiles in SMatMul's 8-row M tiles. + bb_im2col(0, 1, IN, K, 1, 0, 0, 0, 0, 0, WINDOWS, M); + bb_mvin((uintptr_t)weights[branch], 0, PACKED_K, 1); + bb_smatmul_os(1, 0, out_bank, M, CORE, PACKED_K, 1, 1, 0); +} + +int main(void) { + for (int branch = 0; branch < 2; ++branch) { + for (int row = 0; row < IN; ++row) + for (int column = 0; column < IN; ++column) + images[branch][(row * IN + column) * CORE] = + image_value(branch, row, column); + for (int tap = 0; tap < K * K; ++tap) + for (int column = 0; column < CORE; ++column) + weights[branch][tap * CORE * CORE + column] = + (int8_t)((tap * 3 + column) % 41 - 20); + } + for (int lane = 0; lane < CORE; ++lane) + scale[lane] = 0.05f; + + for (int bank = 0; bank < 8; ++bank) + bb_mem_alloc(bank, 1, 1); + bb_mvin((uintptr_t)scale, 3, 4, 1); + bb_mvin((uintptr_t)bias, 7, 4, 1); + bb_smatmul_bias(7, 0); + run_branch(0, 2); + run_branch(1, 4); + bb_matadd(2, 4, 6, RESULT_ROWS); + bb_int32_to_fp32(6, 3, 5, RESULT_ROWS, 0); + bb_mvout((uintptr_t)output, 5, RESULT_ROWS, 1); + + for (int row = 0; row < WINDOWS; ++row) + for (int column = 0; column < CORE; ++column) + if (fabsf(output[row * CORE + column] - + expected(row, column) * scale[column]) > 1e-3f) { + printf("dual conv add mismatch row=%d column=%d\n", row, column); + return 1; + } + printf("dual conv add int2fp PASS\n"); + return 0; +} diff --git a/examples/chips/pebble/workloads/ctests/dual_smatmul_matadd_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/dual_smatmul_matadd_test.c similarity index 100% rename from examples/chips/pebble/workloads/ctests/dual_smatmul_matadd_test.c rename to examples/designs/pebble/workloads/device/core/acc/ctests/dual_smatmul_matadd_test.c diff --git a/examples/chips/pebble/workloads/ctests/im2col_mvin_smatmul_hazard_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_mvin_smatmul_hazard_test.c similarity index 91% rename from examples/chips/pebble/workloads/ctests/im2col_mvin_smatmul_hazard_test.c rename to examples/designs/pebble/workloads/device/core/acc/ctests/im2col_mvin_smatmul_hazard_test.c index 24963d993..0e8be177f 100644 --- a/examples/chips/pebble/workloads/ctests/im2col_mvin_smatmul_hazard_test.c +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_mvin_smatmul_hazard_test.c @@ -6,7 +6,7 @@ #include #include -static int8_t input[4096 * 16] __attribute__((aligned(128))); +static int8_t input[THREAD_MEM_BANK_ENTRIES * 16] __attribute__((aligned(128))); static int8_t pixel_input[28 * 24] __attribute__((aligned(128))); static int8_t weights[16 * 16] __attribute__((aligned(128))); static const int32_t bias[16] __attribute__((aligned(128))); @@ -46,12 +46,12 @@ int main(void) { bb_mvin((uintptr_t)bias, 4, 4, 1); bb_smatmul_bias(4, 0); bb_mvin((uintptr_t)input, 8, 18, 1); - bb_mvin((uintptr_t)input, 8, 4096, 1); + bb_mvin((uintptr_t)input, 8, THREAD_MEM_BANK_ENTRIES, 1); bb_mvin_2d((uintptr_t)pixel_input, 8, 1, 24, 28, 6, 3, 16); bb_mvin_2d((uintptr_t)(pixel_input + 16), 8, 1, 24, 28, 15, 3, 8); for (int iteration = 0; iteration < iterations; ++iteration) for (int lane = 0; lane < 16; ++lane) { - bb_im2col(8, 5, 3, 1, 1, 0, 0, lane, 0, 0, 0, 9); + bb_im2col(8, 5, 3, 1, 1, 0, 0, 0, 0, 0, 9, 8); bb_mvin((uintptr_t)weights, 6, 16, 1); bb_smatmul_os(5, 6, 7, 16, 16, 16, lane == 0, lane == 15, 0); } diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_smatmul_quant_pipeline/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_smatmul_quant_pipeline/CMakeLists.txt new file mode 100644 index 000000000..ae4e87813 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_smatmul_quant_pipeline/CMakeLists.txt @@ -0,0 +1,3 @@ +add_buckyball_component_workload(im2col_smatmul_quant_pipeline SOURCES + im2col_smatmul_quant_pipeline-main.cpp + pebble.c) diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_smatmul_quant_pipeline/im2col_smatmul_quant_pipeline-main.cpp b/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_smatmul_quant_pipeline/im2col_smatmul_quant_pipeline-main.cpp new file mode 100644 index 000000000..c4736d492 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_smatmul_quant_pipeline/im2col_smatmul_quant_pipeline-main.cpp @@ -0,0 +1,22 @@ +#include +extern "C" int pebble_pipeline(uint32_t instance); +static volatile uint32_t completed, failed; +int main() { + const uint32_t hart = bb_get_hart_id(); + const uint32_t expected = + bb_profile_core_count(BB_COMPONENT_WORKLOAD_PROFILE); + if (!expected) + return 1; + if (bb_hart_profile(hart) == BB_COMPONENT_WORKLOAD_PROFILE) { + __atomic_fetch_or(&failed, pebble_pipeline(bb_hart_slot(hart)) != 0, + __ATOMIC_RELEASE); + __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); + } + if (hart == WORKLOAD_REPORT_HART) { + while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) + asm volatile("nop"); + return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; + } + for (;;) + asm volatile("wfi"); +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_smatmul_quant_pipeline/pebble.c b/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_smatmul_quant_pipeline/pebble.c new file mode 100644 index 000000000..9b0dbbb98 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_smatmul_quant_pipeline/pebble.c @@ -0,0 +1,68 @@ +#include "buckyball.h" +#include +#include +#include +#include +#include +#include +#include + +#if THREAD_MEM_BANK_WIDTH != 128 || THREAD_MEM_BANK_ENTRIES < 64 +#error "Pebble thread pipeline requires 128-bit banks with at least 64 rows" +#endif + +// K is (kernel row, kernel column, channel); only channel 0 is nonzero. +enum { + LANES = THREAD_MEM_BANK_WIDTH / 8, + INPUT_ROWS = 36, + WINDOWS = 16, + K = 9 * LANES +}; +static int8_t input[BB_NUM_CORES][INPUT_ROWS * LANES] + __attribute__((aligned(64))); +static int8_t weight[BB_NUM_CORES][K * LANES] __attribute__((aligned(64))); +static int32_t bias[BB_NUM_CORES][LANES] __attribute__((aligned(64))); +static float scale[BB_NUM_CORES][LANES] __attribute__((aligned(64))); +static int8_t output[BB_NUM_CORES][WINDOWS * LANES] + __attribute__((aligned(64))); + +#ifdef __cplusplus +extern "C" +#endif + int pebble_pipeline(uint32_t instance) { + int thread = instance; + for (int row = 0; row < INPUT_ROWS; ++row) + input[thread][row * LANES] = (3 * row + 1) % 7 - 3; + for (int channel = 0; channel < LANES; ++channel) { + bias[thread][channel] = channel - 8; + scale[thread][channel] = 1.0f; + for (int k = 0; k < 9; ++k) + weight[thread][k * LANES * LANES + channel] = + (2 * k + 3 * channel) % 5 - 2; + } + for (int bank = 0; bank < 7; ++bank) + bb_mem_alloc(bank, 1, 1); + bb_mvin((uintptr_t)input[thread], 0, INPUT_ROWS, 1); + bb_mvin((uintptr_t)weight[thread], 2, K, 1); + bb_mvin((uintptr_t)bias[thread], 3, LANES * 32 / THREAD_MEM_BANK_WIDTH, 1); + bb_mvin((uintptr_t)scale[thread], 5, LANES * 32 / THREAD_MEM_BANK_WIDTH, 1); + bb_im2col(0, 1, 6, 3, 1, 0, 0, 0, 0, 0, WINDOWS, 8); + bb_smatmul_bias(3, 0); + bb_smatmul_os(1, 2, 4, WINDOWS, LANES, K, 1, 1, 0); + bb_quant_i32_to_i8(4, 5, 6, WINDOWS * 4, 0, 0, 4, 4, 4, 0); + bb_mvout((uintptr_t)output[thread], 6, WINDOWS, 1); + for (int window = 0; window < WINDOWS; ++window) + for (int channel = 0; channel < LANES; ++channel) { + int expected = bias[thread][channel]; + int y = window / 4, x = window % 4; + for (int ky = 0; ky < 3; ++ky) + for (int kx = 0; kx < 3; ++kx) { + int k = ky * 3 + kx, row = (y + ky) * 6 + x + kx; + expected += input[thread][row * LANES] * + weight[thread][k * LANES * LANES + channel]; + } + if (output[thread][window * LANES + channel] != expected) + return 1; + } + return 0; +} diff --git a/examples/chips/pebble/workloads/ctests/maxpool_im2col_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_im2col_test.c similarity index 88% rename from examples/chips/pebble/workloads/ctests/maxpool_im2col_test.c rename to examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_im2col_test.c index 8cc97b799..ec7c22817 100644 --- a/examples/chips/pebble/workloads/ctests/maxpool_im2col_test.c +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_im2col_test.c @@ -6,7 +6,7 @@ #include #include -enum { LANES = BANK_WIDTH / 8, INPUT_ROWS = 36, OUTPUT_ROWS = 9 }; +enum { LANES = THREAD_MEM_BANK_WIDTH / 8, INPUT_ROWS = 36, OUTPUT_ROWS = 9 }; static int8_t input[INPUT_ROWS * LANES] __attribute__((aligned(64))); static int8_t output[OUTPUT_ROWS * LANES] __attribute__((aligned(64))); @@ -19,7 +19,7 @@ int main(void) { bb_mem_alloc(2, 1, 1); bb_mvin((uintptr_t)input, 0, INPUT_ROWS, 1); bb_maxpool(0, 1, 6, 3, 2, 2, 0, 0, 0, 3, 0, 0); - bb_im2col(1, 2, 3, 1, 1, 0, 0, 0, 0, 0, 0, OUTPUT_ROWS); + bb_im2col(1, 2, 3, 1, 1, 0, 0, 0, 0, 0, OUTPUT_ROWS, 8); bb_mvout((uintptr_t)output, 2, OUTPUT_ROWS, 1); for (int y = 0; y < 3; ++y) for (int x = 0; x < 3; ++x) { diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_partial_update/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_partial_update/CMakeLists.txt new file mode 100644 index 000000000..5374aea3c --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_partial_update/CMakeLists.txt @@ -0,0 +1,3 @@ +add_buckyball_component_workload(maxpool_partial_update SOURCES + maxpool_partial_update-main.cpp + pebble.c) diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_partial_update/maxpool_partial_update-main.cpp b/examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_partial_update/maxpool_partial_update-main.cpp new file mode 100644 index 000000000..c4736d492 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_partial_update/maxpool_partial_update-main.cpp @@ -0,0 +1,22 @@ +#include +extern "C" int pebble_pipeline(uint32_t instance); +static volatile uint32_t completed, failed; +int main() { + const uint32_t hart = bb_get_hart_id(); + const uint32_t expected = + bb_profile_core_count(BB_COMPONENT_WORKLOAD_PROFILE); + if (!expected) + return 1; + if (bb_hart_profile(hart) == BB_COMPONENT_WORKLOAD_PROFILE) { + __atomic_fetch_or(&failed, pebble_pipeline(bb_hart_slot(hart)) != 0, + __ATOMIC_RELEASE); + __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); + } + if (hart == WORKLOAD_REPORT_HART) { + while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) + asm volatile("nop"); + return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; + } + for (;;) + asm volatile("wfi"); +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_partial_update/pebble.c b/examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_partial_update/pebble.c new file mode 100644 index 000000000..7900bd9f1 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_partial_update/pebble.c @@ -0,0 +1,40 @@ +#include "buckyball.h" +#include +#include +#include +#include +#include + +enum { ROWS = 16, LANES = THREAD_MEM_BANK_WIDTH / 8 }; +static int8_t input[BB_NUM_CORES][2 * LANES] __attribute__((aligned(64))); +static int8_t initial[BB_NUM_CORES][ROWS * LANES] __attribute__((aligned(64))); +static int8_t output[BB_NUM_CORES][ROWS * LANES] __attribute__((aligned(64))); + +extern "C" int pebble_pipeline(uint32_t instance) { + int c = instance; + for (int i = 0; i < 2 * LANES; ++i) + input[c][i] = i - LANES; + for (int i = 0; i < ROWS * LANES; ++i) + initial[c][i] = -37; + bb_mem_alloc(0, 1, 1); + bb_mem_alloc(1, 1, 1); + bb_mvin((uintptr_t)input[c], 0, 2, 1); + bb_mvin((uintptr_t)initial[c], 1, ROWS, 1); + bb_maxpool(0, 1, 2, 1, 1, 1, 0, 0, 5, 1, 0, 0); + bb_maxpool(0, 1, 2, 1, 1, 1, 0, 1, 11, 1, 0, 0); + bb_mvout((uintptr_t)output[c], 1, ROWS, 1); + for (int row = 0; row < ROWS; ++row) + for (int lane = 0; lane < LANES; ++lane) { + int8_t expected = row == 5 ? input[c][lane] + : row == 11 ? input[c][LANES + lane] + : -37; + if (output[c][row * LANES + lane] != expected) { + printf("maxpool_partial_update FAIL row=%d lane=%d\n", row, lane); + return 1; + } + } + bb_mem_release(0); + bb_mem_release(1); + printf("maxpool_partial_update PASS\n"); + return 0; +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/mega_conv_k5_pipeline_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/mega_conv_k5_pipeline_test.c new file mode 100644 index 000000000..bb1525e79 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/mega_conv_k5_pipeline_test.c @@ -0,0 +1,70 @@ +#include "buckyball.h" +#include +#include +#include +#include +#include +#include +#include + +// K is (kernel row, kernel column, channel); only channel 0 is nonzero. +enum { + INPUT_SIDE = 8, + OUTPUT_SIDE = 4, + KERNEL = 5, + CORE = 16, + PADDED_K = KERNEL * KERNEL * CORE +}; +static int8_t input[INPUT_SIDE * INPUT_SIDE * CORE] + __attribute__((aligned(64))); +static int8_t weight[PADDED_K * CORE] __attribute__((aligned(64))); +static int32_t bias[CORE] __attribute__((aligned(64))); +static float scale[CORE] __attribute__((aligned(64))); +static int8_t output[OUTPUT_SIDE * OUTPUT_SIDE * CORE] + __attribute__((aligned(64))); + +int main(void) { + for (int pixel = 0; pixel < INPUT_SIDE * INPUT_SIDE; ++pixel) + input[pixel * CORE] = pixel % 5 - 2; + for (int k = 0; k < KERNEL * KERNEL; ++k) + for (int channel = 0; channel < CORE; ++channel) + weight[k * CORE * CORE + channel] = (k + channel) % 3 - 1; + for (int channel = 0; channel < CORE; ++channel) + scale[channel] = 1.0f; + + for (int bank = 0; bank < 8; ++bank) + bb_mem_alloc(bank, 1, 1); + bb_mvin((uintptr_t)input, 0, INPUT_SIDE * INPUT_SIDE, 1); + bb_mvin((uintptr_t)weight, 2, PADDED_K, 1); + bb_mvin((uintptr_t)bias, 3, 4, 1); + bb_mvin((uintptr_t)scale, 5, 4, 1); + // Im2col writes SMatMul's 8-row M tiles directly. + bb_im2col(0, 1, INPUT_SIDE, KERNEL, 1, 0, 0, 0, 0, 0, + OUTPUT_SIDE * OUTPUT_SIDE, 8); + bb_smatmul_bias(3, 0); + bb_smatmul_os(1, 2, 4, OUTPUT_SIDE * OUTPUT_SIDE, CORE, PADDED_K, 1, 1, 0); + bb_quant_i32_to_i8(4, 5, 6, OUTPUT_SIDE * OUTPUT_SIDE * 4, 0, 0, OUTPUT_SIDE, + OUTPUT_SIDE, OUTPUT_SIDE, 0); + bb_mvout((uintptr_t)output, 6, OUTPUT_SIDE * OUTPUT_SIDE, 1); + + for (int y = 0; y < OUTPUT_SIDE; ++y) + for (int x = 0; x < OUTPUT_SIDE; ++x) + for (int channel = 0; channel < CORE; ++channel) { + int expected = 0; + for (int ky = 0; ky < KERNEL; ++ky) + for (int kx = 0; kx < KERNEL; ++kx) { + int pixel = (y + ky) * INPUT_SIDE + x + kx; + int k = ky * KERNEL + kx; + expected += input[pixel * CORE] * weight[k * CORE * CORE + channel]; + } + int index = (y * OUTPUT_SIDE + x) * CORE + channel; + if (output[index] != expected) { + printf( + "mega_conv_k5 FAIL y=%d x=%d channel=%d expected=%d actual=%d\n", + y, x, channel, expected, output[index]); + return 1; + } + } + printf("mega_conv_k5 PASS\n"); + return 0; +} diff --git a/examples/chips/pebble/workloads/ctests/mega_conv_pipeline_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/mega_conv_pipeline_test.c similarity index 78% rename from examples/chips/pebble/workloads/ctests/mega_conv_pipeline_test.c rename to examples/designs/pebble/workloads/device/core/acc/ctests/mega_conv_pipeline_test.c index b56958308..bbd9c900f 100644 --- a/examples/chips/pebble/workloads/ctests/mega_conv_pipeline_test.c +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/mega_conv_pipeline_test.c @@ -8,7 +8,8 @@ #include static int8_t input[36 * 16] __attribute__((aligned(64))); -static int8_t weight[16 * 16] __attribute__((aligned(64))); +// K is (kernel row, kernel column, channel); only channel 0 is nonzero. +static int8_t weight[9 * 16 * 16] __attribute__((aligned(64))); static int32_t bias[16] __attribute__((aligned(64))); static float scale[16] __attribute__((aligned(64))); static int8_t output[16 * 16] __attribute__((aligned(64))); @@ -22,18 +23,19 @@ int main(void) { bias[row] = row - 8; scale[row] = 1.0f; for (int col = 0; col < 16; ++col) - weight[row * 16 + col] = row < 9 ? (2 * row + 3 * col) % 5 - 2 : 0; + if (row < 9) + weight[row * 16 * 16 + col] = (2 * row + 3 * col) % 5 - 2; } for (int bank = 0; bank < 7; ++bank) bb_mem_alloc(bank, 1, 1); bb_mvin((uintptr_t)input, 0, 36, 1); - bb_mvin((uintptr_t)weight, 2, 16, 1); + bb_mvin((uintptr_t)weight, 2, 9 * 16, 1); bb_mvin((uintptr_t)bias, 3, 4, 1); bb_mvin((uintptr_t)scale, 5, 4, 1); - bb_im2col(0, 1, 6, 3, 1, 0, 0, 0, 0, 0, 0, 16); + bb_im2col(0, 1, 6, 3, 1, 0, 0, 0, 0, 0, 16, 8); bb_smatmul_bias(3, 0); - bb_smatmul_os(1, 2, 4, 16, 16, 16, 1, 1, 0); + bb_smatmul_os(1, 2, 4, 16, 16, 9 * 16, 1, 1, 0); bb_quant_i32_to_i8(4, 5, 6, 64, 0, 0, 4, 4, 4, 0); bb_mvout((uintptr_t)output, 6, 16, 1); @@ -43,7 +45,7 @@ int main(void) { for (int ky = 0; ky < 3; ++ky) for (int kx = 0; kx < 3; ++kx) { int k = ky * 3 + kx, row = (y + ky) * 6 + x + kx; - expected += input[row * 16] * weight[k * 16 + channel]; + expected += input[row * 16] * weight[k * 16 * 16 + channel]; } int index = window * 16 + channel; if (output[index] != expected) { diff --git a/examples/chips/pebble/workloads/ctests/mvin_bank78_stress_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/mvin_bank78_stress_test.c similarity index 100% rename from examples/chips/pebble/workloads/ctests/mvin_bank78_stress_test.c rename to examples/designs/pebble/workloads/device/core/acc/ctests/mvin_bank78_stress_test.c diff --git a/examples/chips/pebble/workloads/ctests/quant_smatmul_int2fp_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/quant_smatmul_int2fp_test.c similarity index 94% rename from examples/chips/pebble/workloads/ctests/quant_smatmul_int2fp_test.c rename to examples/designs/pebble/workloads/device/core/acc/ctests/quant_smatmul_int2fp_test.c index 63f4d4247..0a57e0359 100644 --- a/examples/chips/pebble/workloads/ctests/quant_smatmul_int2fp_test.c +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/quant_smatmul_int2fp_test.c @@ -11,8 +11,8 @@ enum { M = 16, N = 16, K = 16, - F32_ROWS = M * N * 32 / BANK_WIDTH, - CHANNEL_ROWS = N * 32 / BANK_WIDTH + F32_ROWS = M * N * 32 / THREAD_MEM_BANK_WIDTH, + CHANNEL_ROWS = N * 32 / THREAD_MEM_BANK_WIDTH }; static float input[M * K] __attribute__((aligned(64))); static int8_t weight[K * N] __attribute__((aligned(64))); diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_shared/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_shared/CMakeLists.txt new file mode 100644 index 000000000..04f041e79 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_shared/CMakeLists.txt @@ -0,0 +1,3 @@ +add_buckyball_component_workload(read_bank_shared SOURCES + read_bank_shared-main.cpp + pebble.c) diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_shared/pebble.c b/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_shared/pebble.c new file mode 100644 index 000000000..c473ff832 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_shared/pebble.c @@ -0,0 +1,52 @@ +#include "buckyball.h" +#include +#include +#include +#include +#include +#include +#include + +enum { M = 8, N = 16, K = 16, LANES = THREAD_MEM_BANK_WIDTH / 8 }; +static int8_t input[BB_NUM_CORES][K * N] __attribute__((aligned(64))); +static int8_t weight[BB_NUM_CORES][K * N] __attribute__((aligned(64))); +static int32_t bias[BB_NUM_CORES][N] __attribute__((aligned(64))); +static int8_t transposed[BB_NUM_CORES][K * N] __attribute__((aligned(64))); +static int32_t product[BB_NUM_CORES][M * N] __attribute__((aligned(64))); +static volatile int32_t warmup[BB_NUM_CORES][N] __attribute__((aligned(64))); +extern "C" int pebble_pipeline(uint32_t instance) { + int c = instance; + for (int i = 0; i < K * N; ++i) { + input[c][i] = 1; + weight[c][i] = 2; + } + for (int bank = 0; bank < 6; ++bank) + bb_mem_alloc(bank, 1, 1); + bb_mvin((uintptr_t)input[c], 0, K * N / LANES, 1); + bb_mvin((uintptr_t)input[c], 4, K * N / LANES, 1); + bb_mvin((uintptr_t)weight[c], 2, K * N / LANES, 1); + bb_mvin((uintptr_t)bias[c], 5, N * 4 / LANES, 1); + bb_smatmul_bias(5, 0); + bb_smatmul_os(0, 2, 3, 1, N, N, 1, 1, 0); + bb_mvout((uintptr_t)warmup[c], 3, N * 4 / LANES, 1); + if (warmup[c][0] != 2 * N) + return 1; + uint64_t start, end; + asm volatile("rdcycle %0" : "=r"(start)); + bb_transpose(0, 1, K * N / LANES, 8); + bb_smatmul_os(0, 2, 3, M, N, K, 1, 1, 0); + bb_mvout((uintptr_t)transposed[c], 1, K * N / LANES, 1); + bb_mvout((uintptr_t)product[c], 3, M * N * 4 / LANES, 1); + for (int i = 0; i < K * N; ++i) + if (transposed[c][i] != 1) + return 1; + for (int i = 0; i < M * N; ++i) + if (product[c][i] != 2 * K) + return 1; + asm volatile("rdcycle %0" : "=r"(end)); + for (int bank = 0; bank < 6; ++bank) + bb_mem_release(bank); + printf("read_bank_shared PASS cycles=%llu\n", + (unsigned long long)(end - start)); + return 0; +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_shared/read_bank_shared-main.cpp b/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_shared/read_bank_shared-main.cpp new file mode 100644 index 000000000..c4736d492 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_shared/read_bank_shared-main.cpp @@ -0,0 +1,22 @@ +#include +extern "C" int pebble_pipeline(uint32_t instance); +static volatile uint32_t completed, failed; +int main() { + const uint32_t hart = bb_get_hart_id(); + const uint32_t expected = + bb_profile_core_count(BB_COMPONENT_WORKLOAD_PROFILE); + if (!expected) + return 1; + if (bb_hart_profile(hart) == BB_COMPONENT_WORKLOAD_PROFILE) { + __atomic_fetch_or(&failed, pebble_pipeline(bb_hart_slot(hart)) != 0, + __ATOMIC_RELEASE); + __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); + } + if (hart == WORKLOAD_REPORT_HART) { + while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) + asm volatile("nop"); + return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; + } + for (;;) + asm volatile("wfi"); +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_split/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_split/CMakeLists.txt new file mode 100644 index 000000000..25c5eb017 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_split/CMakeLists.txt @@ -0,0 +1,3 @@ +add_buckyball_component_workload(read_bank_split SOURCES + read_bank_split-main.cpp + pebble.c) diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_split/pebble.c b/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_split/pebble.c new file mode 100644 index 000000000..1ee586914 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_split/pebble.c @@ -0,0 +1,52 @@ +#include "buckyball.h" +#include +#include +#include +#include +#include +#include +#include + +enum { M = 8, N = 16, K = 16, LANES = THREAD_MEM_BANK_WIDTH / 8 }; +static int8_t input[BB_NUM_CORES][K * N] __attribute__((aligned(64))); +static int8_t weight[BB_NUM_CORES][K * N] __attribute__((aligned(64))); +static int32_t bias[BB_NUM_CORES][N] __attribute__((aligned(64))); +static int8_t transposed[BB_NUM_CORES][K * N] __attribute__((aligned(64))); +static int32_t product[BB_NUM_CORES][M * N] __attribute__((aligned(64))); +static volatile int32_t warmup[BB_NUM_CORES][N] __attribute__((aligned(64))); +extern "C" int pebble_pipeline(uint32_t instance) { + int c = instance; + for (int i = 0; i < K * N; ++i) { + input[c][i] = 1; + weight[c][i] = 2; + } + for (int bank = 0; bank < 6; ++bank) + bb_mem_alloc(bank, 1, 1); + bb_mvin((uintptr_t)input[c], 0, K * N / LANES, 1); + bb_mvin((uintptr_t)input[c], 4, K * N / LANES, 1); + bb_mvin((uintptr_t)weight[c], 2, K * N / LANES, 1); + bb_mvin((uintptr_t)bias[c], 5, N * 4 / LANES, 1); + bb_smatmul_bias(5, 0); + bb_smatmul_os(0, 2, 3, 1, N, N, 1, 1, 0); + bb_mvout((uintptr_t)warmup[c], 3, N * 4 / LANES, 1); + if (warmup[c][0] != 2 * N) + return 1; + uint64_t start, end; + asm volatile("rdcycle %0" : "=r"(start)); + bb_transpose(0, 1, K * N / LANES, 8); + bb_smatmul_os(4, 2, 3, M, N, K, 1, 1, 0); + bb_mvout((uintptr_t)transposed[c], 1, K * N / LANES, 1); + bb_mvout((uintptr_t)product[c], 3, M * N * 4 / LANES, 1); + for (int i = 0; i < K * N; ++i) + if (transposed[c][i] != 1) + return 1; + for (int i = 0; i < M * N; ++i) + if (product[c][i] != 2 * K) + return 1; + asm volatile("rdcycle %0" : "=r"(end)); + for (int bank = 0; bank < 6; ++bank) + bb_mem_release(bank); + printf("read_bank_split PASS cycles=%llu\n", + (unsigned long long)(end - start)); + return 0; +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_split/read_bank_split-main.cpp b/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_split/read_bank_split-main.cpp new file mode 100644 index 000000000..c4736d492 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_split/read_bank_split-main.cpp @@ -0,0 +1,22 @@ +#include +extern "C" int pebble_pipeline(uint32_t instance); +static volatile uint32_t completed, failed; +int main() { + const uint32_t hart = bb_get_hart_id(); + const uint32_t expected = + bb_profile_core_count(BB_COMPONENT_WORKLOAD_PROFILE); + if (!expected) + return 1; + if (bb_hart_profile(hart) == BB_COMPONENT_WORKLOAD_PROFILE) { + __atomic_fetch_or(&failed, pebble_pipeline(bb_hart_slot(hart)) != 0, + __ATOMIC_RELEASE); + __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); + } + if (hart == WORKLOAD_REPORT_HART) { + while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) + asm volatile("nop"); + return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; + } + for (;;) + asm volatile("wfi"); +} diff --git a/examples/chips/pebble/workloads/ctests/residual_add_lut_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/residual_add_lut_test.c similarity index 93% rename from examples/chips/pebble/workloads/ctests/residual_add_lut_test.c rename to examples/designs/pebble/workloads/device/core/acc/ctests/residual_add_lut_test.c index 0f4ad64a3..ce7e822a7 100644 --- a/examples/chips/pebble/workloads/ctests/residual_add_lut_test.c +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/residual_add_lut_test.c @@ -6,7 +6,7 @@ #include #include -enum { ROWS = 4, LANES = BANK_WIDTH / 8, VALUES = ROWS * LANES }; +enum { ROWS = 4, LANES = THREAD_MEM_BANK_WIDTH / 8, VALUES = ROWS * LANES }; static int8_t lhs[VALUES] __attribute__((aligned(64))); static int8_t rhs[VALUES] __attribute__((aligned(64))); static int8_t table[256] __attribute__((aligned(64))); diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/rob_window_probe/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/ctests/rob_window_probe/CMakeLists.txt new file mode 100644 index 000000000..401f7629c --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/rob_window_probe/CMakeLists.txt @@ -0,0 +1,3 @@ +add_buckyball_component_workload(rob_window_probe SOURCES + rob_window_probe-main.cpp + pebble.c) diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/rob_window_probe/pebble.c b/examples/designs/pebble/workloads/device/core/acc/ctests/rob_window_probe/pebble.c new file mode 100644 index 000000000..01d3b09f9 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/rob_window_probe/pebble.c @@ -0,0 +1,57 @@ +#include "buckyball.h" +#include +#include +#include +#include +#include +#include +#include + +enum { M = 8, N = 16, K = 64, LANES = THREAD_MEM_BANK_WIDTH / 8, STORES = 16 }; +static int8_t input[BB_NUM_CORES][M * K] __attribute__((aligned(64))); +static int8_t weight[BB_NUM_CORES][K * N] __attribute__((aligned(64))); +static int8_t core[BB_NUM_CORES][N * N] __attribute__((aligned(64))); +static int32_t bias[BB_NUM_CORES][N] __attribute__((aligned(64))); +static int32_t output[BB_NUM_CORES][STORES][M * N] __attribute__((aligned(64))); +static int8_t transposed[BB_NUM_CORES][N * N] __attribute__((aligned(64))); +static volatile int32_t warmup[BB_NUM_CORES][N] __attribute__((aligned(64))); +extern "C" int pebble_pipeline(uint32_t instance) { + int c = instance; + for (int i = 0; i < M * K; ++i) + input[c][i] = 1; + for (int i = 0; i < K * N; ++i) + weight[c][i] = 2; + for (int i = 0; i < N * N; ++i) + core[c][i] = 1; + for (int bank = 0; bank < 6; ++bank) + bb_mem_alloc(bank, 1, 1); + bb_mvin((uintptr_t)input[c], 0, M * K / LANES, 1); + bb_mvin((uintptr_t)weight[c], 1, K * N / LANES, 1); + bb_mvin((uintptr_t)core[c], 3, N * N / LANES, 1); + bb_mvin((uintptr_t)bias[c], 5, N * 4 / LANES, 1); + bb_smatmul_bias(5, 0); + bb_smatmul_os(0, 1, 2, 1, N, N, 1, 1, 0); + bb_mvout((uintptr_t)warmup[c], 2, N * 4 / LANES, 1); + if (warmup[c][0] != 2 * N) + return 1; + uint64_t start, end; + asm volatile("rdcycle %0" : "=r"(start)); + bb_smatmul_os(0, 1, 2, M, N, K, 1, 1, 0); + for (int i = 0; i < STORES; ++i) + bb_mvout((uintptr_t)output[c][i], 2, M * N * 4 / LANES, 1); + bb_transpose(3, 4, N * N / LANES, 8); + bb_mvout((uintptr_t)transposed[c], 4, N * N / LANES, 1); + for (int s = 0; s < STORES; ++s) + for (int i = 0; i < M * N; ++i) + if (output[c][s][i] != 2 * K) + return 1; + for (int i = 0; i < N * N; ++i) + if (transposed[c][i] != 1) + return 1; + asm volatile("rdcycle %0" : "=r"(end)); + for (int bank = 0; bank < 6; ++bank) + bb_mem_release(bank); + printf("rob_window_probe PASS cycles=%llu\n", + (unsigned long long)(end - start)); + return 0; +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/rob_window_probe/rob_window_probe-main.cpp b/examples/designs/pebble/workloads/device/core/acc/ctests/rob_window_probe/rob_window_probe-main.cpp new file mode 100644 index 000000000..c4736d492 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/rob_window_probe/rob_window_probe-main.cpp @@ -0,0 +1,22 @@ +#include +extern "C" int pebble_pipeline(uint32_t instance); +static volatile uint32_t completed, failed; +int main() { + const uint32_t hart = bb_get_hart_id(); + const uint32_t expected = + bb_profile_core_count(BB_COMPONENT_WORKLOAD_PROFILE); + if (!expected) + return 1; + if (bb_hart_profile(hart) == BB_COMPONENT_WORKLOAD_PROFILE) { + __atomic_fetch_or(&failed, pebble_pipeline(bb_hart_slot(hart)) != 0, + __ATOMIC_RELEASE); + __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); + } + if (hart == WORKLOAD_REPORT_HART) { + while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) + asm volatile("nop"); + return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; + } + for (;;) + asm volatile("wfi"); +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/runtime_job_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/runtime_job_test.c new file mode 100644 index 000000000..11237d7a7 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/runtime_job_test.c @@ -0,0 +1,52 @@ +// Two runtime jobs on the core under test: the first moves data through a +// bank of its accelerator, the second depends on it and checks the data. +#include +#include +#include +#include +#include + +static uint64_t input[2] __attribute__((aligned(64))); +static uint64_t output[2] __attribute__((aligned(64))); + +static int move(void *core, void *argument) { + (void)core; + (void)argument; + bb_mem_alloc(1, 1, 1); + bb_mvin((uintptr_t)input, 1, 1, 1); + bb_mvout((uintptr_t)output, 1, 1, 1); + bb_mem_release(1); + return 0; +} + +static int check(void *core, void *argument) { + (void)core; + (void)argument; + return output[0] != input[0] || output[1] != input[1]; +} + +int main(void) { + input[0] = 0x0123456789abcdefULL; + input[1] = ~input[0]; + const uint64_t signature = bb_cores[0].signature; + const struct runtime_core core = + runtime_core(bb_cores[0].hart, signature, NULL); + static const size_t after_move[] = {0}; + const struct runtime_job jobs[] = { + {signature, RUNTIME_ANY, move, NULL, NULL, 0}, + {signature, RUNTIME_ANY, check, NULL, after_move, 1}, + }; + struct runtime_result results[2]; + struct runtime runtime; + runtime_init(&runtime, &core, 1, jobs, 2, results); + runtime_baremetal_run(&runtime, 0); + for (unsigned job = 0; job < 2; ++job) { + if (results[job].state != RUNTIME_SUCCEEDED || results[job].status) { + printf("runtime job %u state=%d status=%d\n", job, results[job].state, + results[job].status); + return 1; + } + } + puts("Runtime job PASS"); + return 0; +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/same_ball_independent/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/ctests/same_ball_independent/CMakeLists.txt new file mode 100644 index 000000000..3adff7725 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/same_ball_independent/CMakeLists.txt @@ -0,0 +1,3 @@ +add_buckyball_component_workload(same_ball_independent SOURCES + same_ball_independent-main.cpp + pebble.c) diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/same_ball_independent/pebble.c b/examples/designs/pebble/workloads/device/core/acc/ctests/same_ball_independent/pebble.c new file mode 100644 index 000000000..60c357fed --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/same_ball_independent/pebble.c @@ -0,0 +1,56 @@ +#include "buckyball.h" +#include +#include +#include +#include +#include +#include +#include + +enum { M = 8, N = 16, K = 64, LANES = THREAD_MEM_BANK_WIDTH / 8 }; +static int8_t a0[BB_NUM_CORES][M * K] __attribute__((aligned(64))); +static int8_t b0[BB_NUM_CORES][K * N] __attribute__((aligned(64))); +static int8_t a1[BB_NUM_CORES][M * K] __attribute__((aligned(64))); +static int8_t b1[BB_NUM_CORES][K * N] __attribute__((aligned(64))); +static int32_t bias[BB_NUM_CORES][N] __attribute__((aligned(64))); +static int32_t out0[BB_NUM_CORES][M * N] __attribute__((aligned(64))); +static int32_t out1[BB_NUM_CORES][M * N] __attribute__((aligned(64))); +static volatile int32_t warmup[BB_NUM_CORES][N] __attribute__((aligned(64))); +extern "C" int pebble_pipeline(uint32_t instance) { + int c = instance; + for (int i = 0; i < M * K; ++i) { + a0[c][i] = 1; + a1[c][i] = 3; + } + for (int i = 0; i < K * N; ++i) { + b0[c][i] = 2; + b1[c][i] = 4; + } + for (int bank = 0; bank < 7; ++bank) + bb_mem_alloc(bank, 1, 1); + bb_mvin((uintptr_t)a0[c], 0, M * K / LANES, 1); + bb_mvin((uintptr_t)b0[c], 1, K * N / LANES, 1); + bb_mvin((uintptr_t)a1[c], 3, M * K / LANES, 1); + bb_mvin((uintptr_t)b1[c], 4, K * N / LANES, 1); + bb_mvin((uintptr_t)bias[c], 6, N * 4 / LANES, 1); + bb_smatmul_bias(6, 0); + bb_smatmul_os(0, 1, 2, 1, N, N, 1, 1, 0); + bb_mvout((uintptr_t)warmup[c], 2, N * 4 / LANES, 1); + if (warmup[c][0] != 2 * N) + return 1; + uint64_t start, end; + asm volatile("rdcycle %0" : "=r"(start)); + bb_smatmul_os(0, 1, 2, M, N, K, 1, 1, 0); + bb_smatmul_os(3, 4, 5, M, N, K, 1, 1, 0); + bb_mvout((uintptr_t)out0[c], 2, M * N * 4 / LANES, 1); + bb_mvout((uintptr_t)out1[c], 5, M * N * 4 / LANES, 1); + for (int i = 0; i < M * N; ++i) + if (out0[c][i] != 2 * K || out1[c][i] != 12 * K) + return 1; + asm volatile("rdcycle %0" : "=r"(end)); + for (int bank = 0; bank < 7; ++bank) + bb_mem_release(bank); + printf("same_ball_independent PASS cycles=%llu\n", + (unsigned long long)(end - start)); + return 0; +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/same_ball_independent/same_ball_independent-main.cpp b/examples/designs/pebble/workloads/device/core/acc/ctests/same_ball_independent/same_ball_independent-main.cpp new file mode 100644 index 000000000..c4736d492 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/same_ball_independent/same_ball_independent-main.cpp @@ -0,0 +1,22 @@ +#include +extern "C" int pebble_pipeline(uint32_t instance); +static volatile uint32_t completed, failed; +int main() { + const uint32_t hart = bb_get_hart_id(); + const uint32_t expected = + bb_profile_core_count(BB_COMPONENT_WORKLOAD_PROFILE); + if (!expected) + return 1; + if (bb_hart_profile(hart) == BB_COMPONENT_WORKLOAD_PROFILE) { + __atomic_fetch_or(&failed, pebble_pipeline(bb_hart_slot(hart)) != 0, + __ATOMIC_RELEASE); + __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); + } + if (hart == WORKLOAD_REPORT_HART) { + while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) + asm volatile("nop"); + return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; + } + for (;;) + asm volatile("wfi"); +} diff --git a/examples/chips/pebble/workloads/ctests/se_gate_lut_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/se_gate_lut_test.c similarity index 93% rename from examples/chips/pebble/workloads/ctests/se_gate_lut_test.c rename to examples/designs/pebble/workloads/device/core/acc/ctests/se_gate_lut_test.c index 71295e5c3..c0a351f3b 100644 --- a/examples/chips/pebble/workloads/ctests/se_gate_lut_test.c +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/se_gate_lut_test.c @@ -6,7 +6,7 @@ #include #include -enum { ROWS = 4, LANES = BANK_WIDTH / 8, VALUES = ROWS * LANES }; +enum { ROWS = 4, LANES = THREAD_MEM_BANK_WIDTH / 8, VALUES = ROWS * LANES }; static int8_t gate[LANES] __attribute__((aligned(64))); static int8_t input[VALUES] __attribute__((aligned(64))); static int8_t table[256] __attribute__((aligned(64))); diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/shared_bank_versions/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/ctests/shared_bank_versions/CMakeLists.txt new file mode 100644 index 000000000..bdcbda565 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/shared_bank_versions/CMakeLists.txt @@ -0,0 +1,3 @@ +add_buckyball_component_workload(shared_bank_versions SOURCES + shared_bank_versions-main.cpp + pebble.c) diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/shared_bank_versions/pebble.c b/examples/designs/pebble/workloads/device/core/acc/ctests/shared_bank_versions/pebble.c new file mode 100644 index 000000000..492779b2a --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/shared_bank_versions/pebble.c @@ -0,0 +1,47 @@ +#include "buckyball.h" +#include +#include +#include +#include +#include +#include +#include + +enum { + ROWS = 16, + NEW_ROWS = 8, + BYTES = ROWS * THREAD_MEM_BANK_WIDTH / 8, + BASE = THREAD_FRONTEND_SHARED_VBANK_BASE +}; +static int8_t input[BB_NUM_CORES][BYTES] __attribute__((aligned(64))); +static int8_t left[BB_NUM_CORES][BYTES] __attribute__((aligned(64))); +static int8_t right[BB_NUM_CORES][BYTES] __attribute__((aligned(64))); +static volatile int8_t warmup[BB_NUM_CORES][BYTES] __attribute__((aligned(64))); +static int8_t output[BB_NUM_CORES][BYTES] __attribute__((aligned(64))); + +extern "C" int pebble_pipeline(uint32_t instance) { + int c = instance; + for (int i = 0; i < BYTES; ++i) { + input[c][i] = 1; + left[c][i] = 3; + right[c][i] = 4; + } + for (int bank = BASE; bank < BASE + 5; ++bank) + bb_mem_alloc(bank, 1, 1); + bb_mvin((uintptr_t)input[c], BASE, ROWS, 1); + bb_mvin((uintptr_t)left[c], BASE + 3, ROWS, 1); + bb_mvin((uintptr_t)right[c], BASE + 4, ROWS, 1); + bb_mvout((uintptr_t)warmup[c], BASE + 4, ROWS, 1); + if (warmup[c][0] != 4) + return 1; + bb_transpose(BASE, BASE + 2, ROWS, 8); + bb_int8add(BASE + 3, BASE + 4, BASE + 2, NEW_ROWS, 1.0f, 1.0f); + bb_mvout((uintptr_t)output[c], BASE + 2, NEW_ROWS, 1); + for (int i = 0; i < NEW_ROWS * THREAD_MEM_BANK_WIDTH / 8; ++i) + if (output[c][i] != 7) + return 1; + for (int bank = BASE; bank < BASE + 5; ++bank) + bb_mem_release(bank); + printf("shared_bank_versions PASS\n"); + return 0; +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/shared_bank_versions/shared_bank_versions-main.cpp b/examples/designs/pebble/workloads/device/core/acc/ctests/shared_bank_versions/shared_bank_versions-main.cpp new file mode 100644 index 000000000..c4736d492 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/shared_bank_versions/shared_bank_versions-main.cpp @@ -0,0 +1,22 @@ +#include +extern "C" int pebble_pipeline(uint32_t instance); +static volatile uint32_t completed, failed; +int main() { + const uint32_t hart = bb_get_hart_id(); + const uint32_t expected = + bb_profile_core_count(BB_COMPONENT_WORKLOAD_PROFILE); + if (!expected) + return 1; + if (bb_hart_profile(hart) == BB_COMPONENT_WORKLOAD_PROFILE) { + __atomic_fetch_or(&failed, pebble_pipeline(bb_hart_slot(hart)) != 0, + __ATOMIC_RELEASE); + __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); + } + if (hart == WORKLOAD_REPORT_HART) { + while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) + asm volatile("nop"); + return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; + } + for (;;) + asm volatile("wfi"); +} diff --git a/examples/chips/pebble/workloads/ctests/smatmul_mvout_bank_reuse_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/smatmul_mvout_bank_reuse_test.c similarity index 94% rename from examples/chips/pebble/workloads/ctests/smatmul_mvout_bank_reuse_test.c rename to examples/designs/pebble/workloads/device/core/acc/ctests/smatmul_mvout_bank_reuse_test.c index 9af9e9ad4..377c23e3f 100644 --- a/examples/chips/pebble/workloads/ctests/smatmul_mvout_bank_reuse_test.c +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/smatmul_mvout_bank_reuse_test.c @@ -7,8 +7,8 @@ enum { DIM = 16, - OUTPUT_ROWS = DIM * DIM * 32 / BANK_WIDTH, - BIAS_ROWS = DIM * 32 / BANK_WIDTH, + OUTPUT_ROWS = DIM * DIM * 32 / THREAD_MEM_BANK_WIDTH, + BIAS_ROWS = DIM * 32 / THREAD_MEM_BANK_WIDTH, ITERATIONS = 24 }; static int8_t input[DIM * DIM] __attribute__((aligned(64))); diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/write_bank_versions/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/ctests/write_bank_versions/CMakeLists.txt new file mode 100644 index 000000000..add409216 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/write_bank_versions/CMakeLists.txt @@ -0,0 +1,3 @@ +add_buckyball_component_workload(write_bank_versions SOURCES + write_bank_versions-main.cpp + pebble.c) diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/write_bank_versions/pebble.c b/examples/designs/pebble/workloads/device/core/acc/ctests/write_bank_versions/pebble.c new file mode 100644 index 000000000..11f3418db --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/write_bank_versions/pebble.c @@ -0,0 +1,42 @@ +#include "buckyball.h" +#include +#include +#include +#include +#include +#include +#include + +enum { ROWS = 16, NEW_ROWS = 8, BYTES = ROWS * THREAD_MEM_BANK_WIDTH / 8 }; +static int8_t input[BB_NUM_CORES][BYTES] __attribute__((aligned(64))); +static int8_t left[BB_NUM_CORES][BYTES] __attribute__((aligned(64))); +static int8_t right[BB_NUM_CORES][BYTES] __attribute__((aligned(64))); +static volatile int8_t warmup[BB_NUM_CORES][BYTES] __attribute__((aligned(64))); +static int8_t output[BB_NUM_CORES][BYTES] __attribute__((aligned(64))); + +extern "C" int pebble_pipeline(uint32_t instance) { + int c = instance; + for (int i = 0; i < BYTES; ++i) { + input[c][i] = 1; + left[c][i] = 3; + right[c][i] = 4; + } + for (int bank = 0; bank < 5; ++bank) + bb_mem_alloc(bank, 1, 1); + bb_mvin((uintptr_t)input[c], 0, ROWS, 1); + bb_mvin((uintptr_t)left[c], 3, ROWS, 1); + bb_mvin((uintptr_t)right[c], 4, ROWS, 1); + bb_mvout((uintptr_t)warmup[c], 4, ROWS, 1); + if (warmup[c][0] != 4) + return 1; + bb_transpose(0, 2, ROWS, 8); + bb_int8add(3, 4, 2, NEW_ROWS, 1.0f, 1.0f); + bb_mvout((uintptr_t)output[c], 2, NEW_ROWS, 1); + for (int i = 0; i < NEW_ROWS * THREAD_MEM_BANK_WIDTH / 8; ++i) + if (output[c][i] != 7) + return 1; + for (int bank = 0; bank < 5; ++bank) + bb_mem_release(bank); + printf("write_bank_versions PASS\n"); + return 0; +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/write_bank_versions/write_bank_versions-main.cpp b/examples/designs/pebble/workloads/device/core/acc/ctests/write_bank_versions/write_bank_versions-main.cpp new file mode 100644 index 000000000..c4736d492 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/write_bank_versions/write_bank_versions-main.cpp @@ -0,0 +1,22 @@ +#include +extern "C" int pebble_pipeline(uint32_t instance); +static volatile uint32_t completed, failed; +int main() { + const uint32_t hart = bb_get_hart_id(); + const uint32_t expected = + bb_profile_core_count(BB_COMPONENT_WORKLOAD_PROFILE); + if (!expected) + return 1; + if (bb_hart_profile(hart) == BB_COMPONENT_WORKLOAD_PROFILE) { + __atomic_fetch_or(&failed, pebble_pipeline(bb_hart_slot(hart)) != 0, + __ATOMIC_RELEASE); + __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); + } + if (hart == WORKLOAD_REPORT_HART) { + while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) + asm volatile("nop"); + return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; + } + for (;;) + asm volatile("wfi"); +} diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/CMakeLists.txt new file mode 100644 index 000000000..d00d54687 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/CMakeLists.txt @@ -0,0 +1,7 @@ +add_custom_target(pebble-mlir-tests-build) + +add_subdirectory(bank) +add_subdirectory(core) +add_subdirectory(linalg) + +add_dependencies(design-mlirtests-build pebble-mlir-tests-build) diff --git a/examples/chips/pebble/workloads/mlir_tests/bank/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/bank/CMakeLists.txt similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/bank/CMakeLists.txt rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/bank/CMakeLists.txt diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/bank/quant_im2col_k1_single_pixel.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/bank/quant_im2col_k1_single_pixel.mlir new file mode 100644 index 000000000..58d793943 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/bank/quant_im2col_k1_single_pixel.mlir @@ -0,0 +1,30 @@ +// Design integration: f32-to-i8 quantization -> im2col k=1 -> mvout. + +func.func private @check_result(memref<8x16xi8>) -> () + +memref.global "private" constant @fp_in : memref<4x4xf32> = + dense<[[1.0, 2.0, 3.0, 4.0], [0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0]]> + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %scale = arith.constant 1.0 : f32 + + %input = memref.get_global @fp_in : memref<4x4xf32> + %output = memref.alloc() alignment = 64 : memref<8x16xi8> + + %bin = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %bq = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %bout = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %loaded = buckyball.bank_mvin %input, %bin <{depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> + %q = toint8_ball.bank_fp32_to_int8 %loaded -> %bq multiplier(%scale) <{rows = 4 : i64}> + %im = im2col_ball.bank_im2col_i8 %q -> %bout <{input_side = 1 : i64, kernel = 1 : i64, stride = 1 : i64, padding = 0 : i64, tile_rows = 8 : i64}> + %stored = buckyball.bank_mvout %output, %im <{depth = 8 : i64, stride = 1 : i64}> : memref<8x16xi8> + buckyball.bank_release %loaded + buckyball.bank_release %q + buckyball.bank_release %stored + + func.call @check_result(%output) : (memref<8x16xi8>) -> () + memref.dealloc %output : memref<8x16xi8> + return %zero_i8 : i8 +} diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/bank/quant_im2col_k1_single_pixel_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/bank/quant_im2col_k1_single_pixel_main.cpp new file mode 100644 index 000000000..9dc2953be --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/bank/quant_im2col_k1_single_pixel_main.cpp @@ -0,0 +1,42 @@ +#include +#include +#include + +static void fail(void) { +#ifdef BAREMETAL + volatile uint32_t *sim_exit = (volatile uint32_t *)0x60000000; + *sim_exit = 1; + while (1) { + } +#else + exit(1); +#endif +} + +#ifdef __cplusplus +extern "C" +#endif + void check_result(int8_t *allocated, int8_t *aligned, int64_t offset, + int64_t size0, int64_t size1, int64_t stride0, + int64_t stride1) { + (void)allocated; + if (size0 != 8 || size1 != 16 || stride0 != 16 || stride1 != 1) { + printf( + "FAILED: quant_im2col_k1_single_pixel shape %ldx%ld stride %ldx%ld\n", + (long)size0, (long)size1, (long)stride0, (long)stride1); + fail(); + } + + int8_t *out = aligned + offset; + // The one window is line 0: the sixteen channels of the pixel. + for (int channel = 0; channel < 16; ++channel) { + int expected = channel < 4 ? channel + 1 : 0; + int got = out[channel * stride1]; + if (got != expected) { + printf("FAILED: quant_im2col_k1_single_pixel channel=%d exp=%d got=%d\n", + channel, expected, got); + fail(); + } + } + printf("PASSED: quant_im2col_k1_single_pixel\n"); +} diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/CMakeLists.txt new file mode 100644 index 000000000..78a4ba6ef --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/CMakeLists.txt @@ -0,0 +1,471 @@ +set(BUCKYBALL_MLIR_GROUP_TARGET pebble-mlir-tests-build) +set(BUCKYBALL_MLIR_TEST_PREFIX core) + +add_buckyball_mlir_test(lookup_channel_i8 + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) +add_buckyball_mlir_test(linear_chain_i8 + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) +add_buckyball_mlir_test(linear_int8 + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) +add_buckyball_mlir_test(generic_matmul_i8 + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) + +add_buckyball_mlir_test(mega_matmul_m17_k65_n17 + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) + +add_buckyball_mlir_test(mega_conv2d + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) + +add_buckyball_mlir_test(mega_conv2d_resident + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) + +add_buckyball_mlir_test(mega_conv2d_flatten_linear + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) + +add_buckyball_mlir_test(mega_conv2d_k5_tail + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) + +add_buckyball_mlir_test(mega_conv2d_s2_depthwise_s2 + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) + +file(READ ${CMAKE_CURRENT_SOURCE_DIR}/data/mega_global_avg_conv_exact_input.i8 + PEBBLE_GAP_CONV_INPUT_HEX HEX) +file(READ ${CMAKE_CURRENT_SOURCE_DIR}/data/mega_global_avg_7x7_c288_input.i8 + PEBBLE_GAP_C288_INPUT_HEX HEX) +file(READ ${CMAKE_CURRENT_SOURCE_DIR}/data/mega_global_avg_conv_exact_weight.i8 + PEBBLE_GAP_CONV_WEIGHT_HEX HEX) +file(READ ${CMAKE_CURRENT_SOURCE_DIR}/data/mega_global_avg_conv2_exact_weight2.i8 + PEBBLE_GAP_CONV_WEIGHT2_HEX HEX) +file(READ ${CMAKE_CURRENT_SOURCE_DIR}/data/mega_global_avg_conv2_exact_weight3.i8 + PEBBLE_GAP_CONV_WEIGHT3_HEX HEX) +add_buckyball_mlir_test(mega_global_avg_conv_exact + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) +add_buckyball_mlir_test(mega_global_avg_7x7_c288_exact + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) +add_buckyball_mlir_test(mega_global_avg_conv2_exact + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) +unset(PEBBLE_GAP_CONV_INPUT_HEX) +unset(PEBBLE_GAP_C288_INPUT_HEX) +unset(PEBBLE_GAP_CONV_WEIGHT_HEX) +unset(PEBBLE_GAP_CONV_WEIGHT2_HEX) +unset(PEBBLE_GAP_CONV_WEIGHT3_HEX) + +add_buckyball_mlir_test(mega_se_gate_conv + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) + +add_buckyball_mlir_test(mega_matmul + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) + +add_buckyball_mlir_test(core_matmul_16x16_i8 + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) + +add_buckyball_mlir_test(core_transpose_16x16_i8 + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/core_matmul_16x16_i8.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/core_matmul_16x16_i8.mlir new file mode 100644 index 000000000..17b262bdd --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/core_matmul_16x16_i8.mlir @@ -0,0 +1,40 @@ +func.func private @check_result(memref<16x16xi32>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero_i32 = arith.constant 0 : i32 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c2 = arith.constant 2 : index + %c3 = arith.constant 3 : index + %c5 = arith.constant 5 : index + %c7 = arith.constant 7 : index + %c16 = arith.constant 16 : index + %a = memref.alloc() alignment = 64 : memref<16x16xi8> + %b = memref.alloc() alignment = 64 : memref<16x16xi8> + scf.for %i = %c0 to %c16 step %c1 { + scf.for %j = %c0 to %c16 step %c1 { + %i3 = arith.muli %i, %c3 : index + %j5 = arith.muli %j, %c5 : index + %as = arith.addi %i3, %j5 : index + %ar = arith.remui %as, %c7 : index + %av0 = arith.subi %ar, %c3 : index + %av = arith.index_cast %av0 : index to i8 + %i2 = arith.muli %i, %c2 : index + %j3 = arith.muli %j, %c3 : index + %bs = arith.addi %i2, %j3 : index + %br = arith.remui %bs, %c5 : index + %bv0 = arith.subi %br, %c2 : index + %bv = arith.index_cast %bv0 : index to i8 + memref.store %av, %a[%i, %j] : memref<16x16xi8> + memref.store %bv, %b[%i, %j] : memref<16x16xi8> + } + } + %at = bufferization.to_tensor %a restrict : memref<16x16xi8> to tensor<16x16xi8> + %bt = bufferization.to_tensor %b restrict : memref<16x16xi8> to tensor<16x16xi8> + %init = arith.constant dense<0> : tensor<16x16xi32> + %result = linalg.matmul ins(%at, %bt: tensor<16x16xi8>, tensor<16x16xi8>) outs(%init: tensor<16x16xi32>) -> tensor<16x16xi32> + %memory = bufferization.to_buffer %result : tensor<16x16xi32> to memref<16x16xi32> + func.call @check_result(%memory) : (memref<16x16xi32>) -> () + return %zero_i8 : i8 +} diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/core_matmul_16x16_i8_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/core_matmul_16x16_i8_main.cpp new file mode 100644 index 000000000..156db9a22 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/core_matmul_16x16_i8_main.cpp @@ -0,0 +1,29 @@ +#include +#include +#include + +static void fail(void) { +#ifdef BAREMETAL + *(volatile uint32_t *)0x60000000 = 1; + while (1) { + } +#else + exit(1); +#endif +} + +extern "C" void check_result(int32_t *, int32_t *out, int64_t offset, + int64_t rows, int64_t columns, int64_t row_stride, + int64_t column_stride) { + if (rows != 16 || columns != 16 || row_stride != 16 || column_stride != 1) + fail(); + for (int i = 0; i < 16; ++i) + for (int j = 0; j < 16; ++j) { + int expected = 0; + for (int k = 0; k < 16; ++k) + expected += ((3 * i + 5 * k) % 7 - 3) * ((2 * k + 3 * j) % 5 - 2); + if (out[offset + i * row_stride + j] != expected) + fail(); + } + printf("PASSED: core.core_matmul 16x16 ordered operands\n"); +} diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/core_transpose_16x16_i8.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/core_transpose_16x16_i8.mlir new file mode 100644 index 000000000..622974fa5 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/core_transpose_16x16_i8.mlir @@ -0,0 +1,27 @@ +func.func private @check_result(memref<16x16xi8>) -> () + +func.func @main() -> i8 { + %zero = arith.constant 0 : i8 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c3 = arith.constant 3 : i8 + %c5 = arith.constant 5 : i8 + %c16 = arith.constant 16 : index + %input = memref.alloc() alignment = 64 : memref<16x16xi8> + scf.for %i = %c0 to %c16 step %c1 { + scf.for %j = %c0 to %c16 step %c1 { + %ii = arith.index_cast %i : index to i8 + %jj = arith.index_cast %j : index to i8 + %iv = arith.muli %ii, %c3 : i8 + %jv = arith.muli %jj, %c5 : i8 + %v = arith.addi %iv, %jv : i8 + memref.store %v, %input[%i, %j] : memref<16x16xi8> + } + } + %it = bufferization.to_tensor %input restrict : memref<16x16xi8> to tensor<16x16xi8> + %empty = tensor.empty() : tensor<16x16xi8> + %result = linalg.transpose ins(%it: tensor<16x16xi8>) outs(%empty: tensor<16x16xi8>) permutation = [1,0] + %memory = bufferization.to_buffer %result : tensor<16x16xi8> to memref<16x16xi8> + func.call @check_result(%memory) : (memref<16x16xi8>) -> () + return %zero : i8 +} diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/core_transpose_16x16_i8_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/core_transpose_16x16_i8_main.cpp new file mode 100644 index 000000000..84b02b78b --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/core_transpose_16x16_i8_main.cpp @@ -0,0 +1,25 @@ +#include +#include +#include + +static void fail(void) { +#ifdef BAREMETAL + *(volatile uint32_t *)0x60000000 = 1; + while (1) { + } +#else + exit(1); +#endif +} + +extern "C" void check_result(int8_t *, int8_t *out, int64_t offset, + int64_t rows, int64_t columns, int64_t row_stride, + int64_t column_stride) { + if (rows != 16 || columns != 16 || row_stride != 16 || column_stride != 1) + fail(); + for (int i = 0; i < 16; ++i) + for (int j = 0; j < 16; ++j) + if (out[offset + i * row_stride + j] != (int8_t)(3 * j + 5 * i)) + fail(); + printf("PASSED: core.core_transpose 16x16 ordered input\n"); +} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/data/mega_global_avg_7x7_c288_input.i8 b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/data/mega_global_avg_7x7_c288_input.i8 similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/tile/data/mega_global_avg_7x7_c288_input.i8 rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/data/mega_global_avg_7x7_c288_input.i8 diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/data/mega_global_avg_conv2_exact_weight2.i8 b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/data/mega_global_avg_conv2_exact_weight2.i8 similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/tile/data/mega_global_avg_conv2_exact_weight2.i8 rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/data/mega_global_avg_conv2_exact_weight2.i8 diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/data/mega_global_avg_conv2_exact_weight3.i8 b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/data/mega_global_avg_conv2_exact_weight3.i8 similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/tile/data/mega_global_avg_conv2_exact_weight3.i8 rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/data/mega_global_avg_conv2_exact_weight3.i8 diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/data/mega_global_avg_conv_exact_input.i8 b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/data/mega_global_avg_conv_exact_input.i8 similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/tile/data/mega_global_avg_conv_exact_input.i8 rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/data/mega_global_avg_conv_exact_input.i8 diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/data/mega_global_avg_conv_exact_weight.i8 b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/data/mega_global_avg_conv_exact_weight.i8 similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/tile/data/mega_global_avg_conv_exact_weight.i8 rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/data/mega_global_avg_conv_exact_weight.i8 diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/generic_matmul_i8.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/generic_matmul_i8.mlir new file mode 100644 index 000000000..88373edac --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/generic_matmul_i8.mlir @@ -0,0 +1,52 @@ +func.func private @check_result(memref<1x32xi32>) + +func.func @main() -> i8 { + %exit = arith.constant 0 : i8 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c3 = arith.constant 3 : index + %c4 = arith.constant 4 : index + %c5 = arith.constant 5 : index + %c7 = arith.constant 7 : index + %c9 = arith.constant 9 : index + %c16 = arith.constant 16 : index + %c32 = arith.constant 32 : index + %a = memref.alloc() alignment = 64 : memref<1x32xi8> + %b = memref.alloc() alignment = 64 : memref<32x32xi8> + %c = memref.alloc() alignment = 64 : memref<1x32xi32> + scf.for %k = %c0 to %c32 step %c1 { + %rem = arith.remui %k, %c7 : index + %value = arith.subi %rem, %c3 : index + %a8 = arith.index_cast %value : index to i8 + memref.store %a8, %a[%c0, %k] : memref<1x32xi8> + %initial = arith.subi %k, %c16 : index + %c32value = arith.index_cast %initial : index to i32 + memref.store %c32value, %c[%c0, %k] : memref<1x32xi32> + scf.for %column = %c0 to %c32 step %c1 { + %k3 = arith.muli %k, %c3 : index + %column5 = arith.muli %column, %c5 : index + %sum = arith.addi %k3, %column5 : index + %rem9 = arith.remui %sum, %c9 : index + %signed = arith.subi %rem9, %c4 : index + %b8 = arith.index_cast %signed : index to i8 + memref.store %b8, %b[%k, %column] : memref<32x32xi8> + } + } + %lhs = bufferization.to_tensor %a restrict : memref<1x32xi8> to tensor<1x32xi8> + %rhs = bufferization.to_tensor %b restrict : memref<32x32xi8> to tensor<32x32xi8> + %initial = bufferization.to_tensor %c restrict : memref<1x32xi32> to tensor<1x32xi32> + %product = linalg.generic { + indexing_maps = [affine_map<(m,n,k)->(m,k)>, affine_map<(m,n,k)->(k,n)>, affine_map<(m,n,k)->(m,n)>], + iterator_types = ["parallel", "parallel", "reduction"] + } ins(%lhs, %rhs: tensor<1x32xi8>, tensor<32x32xi8>) outs(%initial: tensor<1x32xi32>) { + ^bb0(%x: i8, %y: i8, %acc: i32): + %x32 = arith.extsi %x : i8 to i32 + %y32 = arith.extsi %y : i8 to i32 + %mul = arith.muli %x32, %y32 : i32 + %sum = arith.addi %acc, %mul : i32 + linalg.yield %sum : i32 + } -> tensor<1x32xi32> + %output = bufferization.to_buffer %product : tensor<1x32xi32> to memref<1x32xi32> + func.call @check_result(%output) : (memref<1x32xi32>) -> () + return %exit : i8 +} diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/generic_matmul_i8_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/generic_matmul_i8_main.cpp new file mode 100644 index 000000000..07b79adfd --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/generic_matmul_i8_main.cpp @@ -0,0 +1,33 @@ +#include +#include +#include + +static void fail(void) { +#ifdef BAREMETAL + *(volatile uint32_t *)0x60000000 = 1; + while (1) { + } +#else + exit(1); +#endif +} + +extern "C" void check_result(int32_t *, int32_t *output, int64_t offset, + int64_t rows, int64_t columns, int64_t row_stride, + int64_t column_stride) { + if (rows != 1 || columns != 32 || row_stride != 32 || column_stride != 1) + fail(); + for (int column = 0; column < 32; ++column) { + int32_t expected = column - 16; + for (int k = 0; k < 32; ++k) + expected += (k % 7 - 3) * ((3 * k + 5 * column) % 9 - 4); + int32_t actual = output[offset + column * column_stride]; + if (actual != expected) { + printf("FAILED: generic matmul column=%d expected=%d actual=%d\n", column, + expected, actual); + fail(); + } + } + printf("PASSED: generic matmul INT8 signed inputs and nonzero INT32 " + "accumulator\n"); +} diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/linear_chain_i8.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/linear_chain_i8.mlir new file mode 100644 index 000000000..21033212f --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/linear_chain_i8.mlir @@ -0,0 +1,98 @@ +// FORM: custom.op "linear_i8"{{.*}}custom.resident, relu = true{{.*}}-> tensor<1x32xi8> +// BANK: toint8_ball.bank_int32_to_int8 +// BANK: buckyball.bank_transfer +// BANK-NOT: buckyball.bank_mvout +// BANK: smatmul_ball.bank_matmul_i8 {{.*}}k = 16> +func.func private @check_result(memref<1x16xf32>) + +func.func @main() -> i8 { + %exit = arith.constant 0 : i8 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c3 = arith.constant 3 : index + %c4 = arith.constant 4 : index + %c5 = arith.constant 5 : index + %c7 = arith.constant 7 : index + %c8 = arith.constant 8 : index + %c9 = arith.constant 9 : index + %c16 = arith.constant 16 : index + %c32 = arith.constant 32 : index + %c99 = arith.constant 99 : i32 + %unit1 = arith.constant 0.015625 : f32 + %unit2 = arith.constant 0.0078125 : f32 + %multiplier = arith.constant dense<0.5> : tensor<1x1xf32> + %input_step = arith.constant dense<2.0> : tensor<1x1xf32> + %zero = arith.constant dense<0.0> : tensor<1x32xf32> + %x = memref.alloc() alignment = 64 : memref<1x32xf32> + %w1 = memref.alloc() alignment = 64 : memref<32x32xi8> + %b1 = memref.alloc() alignment = 64 : memref<1x32xi32> + %s1 = memref.alloc() alignment = 64 : memref<1x32xf32> + %w2 = memref.alloc() alignment = 64 : memref<32x16xi8> + %b2 = memref.alloc() alignment = 64 : memref<1x16xi32> + %s2 = memref.alloc() alignment = 64 : memref<1x16xf32> + scf.for %k = %c0 to %c32 step %c1 { + %rem = arith.remui %k, %c7 : index + %shift = arith.subi %rem, %c3 : index + %integer = arith.index_cast %shift : index to i32 + %scaled = arith.muli %integer, %c99 : i32 + %value = arith.sitofp %scaled : i32 to f32 + memref.store %value, %x[%c0, %k] : memref<1x32xf32> + %bias = arith.subi %k, %c16 : index + %bias32 = arith.index_cast %bias : index to i32 + memref.store %bias32, %b1[%c0, %k] : memref<1x32xi32> + %number = arith.addi %k, %c1 : index + %number32 = arith.index_cast %number : index to i32 + %numberf = arith.sitofp %number32 : i32 to f32 + %scale = arith.mulf %numberf, %unit1 : f32 + memref.store %scale, %s1[%c0, %k] : memref<1x32xf32> + scf.for %column = %c0 to %c32 step %c1 { + %k3 = arith.muli %k, %c3 : index + %column5 = arith.muli %column, %c5 : index + %sum = arith.addi %k3, %column5 : index + %mod = arith.remui %sum, %c9 : index + %signed = arith.subi %mod, %c4 : index + %weight = arith.index_cast %signed : index to i8 + memref.store %weight, %w1[%k, %column] : memref<32x32xi8> + } + scf.for %column = %c0 to %c16 step %c1 { + %k5 = arith.muli %k, %c5 : index + %column3 = arith.muli %column, %c3 : index + %sum = arith.addi %k5, %column3 : index + %mod = arith.remui %sum, %c7 : index + %signed = arith.subi %mod, %c3 : index + %weight = arith.index_cast %signed : index to i8 + memref.store %weight, %w2[%k, %column] : memref<32x16xi8> + } + } + scf.for %column = %c0 to %c16 step %c1 { + %bias = arith.subi %column, %c8 : index + %bias32 = arith.index_cast %bias : index to i32 + memref.store %bias32, %b2[%c0, %column] : memref<1x16xi32> + %number = arith.addi %column, %c1 : index + %number32 = arith.index_cast %number : index to i32 + %numberf = arith.sitofp %number32 : i32 to f32 + %scale = arith.mulf %numberf, %unit2 : f32 + memref.store %scale, %s2[%c0, %column] : memref<1x16xf32> + } + %input = bufferization.to_tensor %x restrict : memref<1x32xf32> to tensor<1x32xf32> + %weight1 = bufferization.to_tensor %w1 restrict : memref<32x32xi8> to tensor<32x32xi8> + %bias1 = bufferization.to_tensor %b1 restrict : memref<1x32xi32> to tensor<1x32xi32> + %step1 = bufferization.to_tensor %s1 restrict : memref<1x32xf32> to tensor<1x32xf32> + %weight2 = bufferization.to_tensor %w2 restrict : memref<32x16xi8> to tensor<32x16xi8> + %bias2 = bufferization.to_tensor %b2 restrict : memref<1x16xi32> to tensor<1x16xi32> + %step2 = bufferization.to_tensor %s2 restrict : memref<1x16xf32> to tensor<1x16xf32> + %first = custom.op "linear_i8"(%input, %weight1, %bias1, %multiplier, %input_step, %step1) : (tensor<1x32xf32>, tensor<32x32xi8>, tensor<1x32xi32>, tensor<1x1xf32>, tensor<1x1xf32>, tensor<1x32xf32>) -> tensor<1x32xf32> + %empty = tensor.empty() : tensor<1x32xf32> + %relu = linalg.generic { + indexing_maps = [affine_map<(m,n)->(m,n)>, affine_map<(m,n)->(m,n)>, affine_map<(m,n)->(m,n)>], + iterator_types = ["parallel", "parallel"] + } ins(%first, %zero: tensor<1x32xf32>, tensor<1x32xf32>) outs(%empty: tensor<1x32xf32>) { + ^bb0(%value: f32, %zero_value: f32, %unused: f32): + %positive = arith.maximumf %value, %zero_value : f32 + linalg.yield %positive : f32 + } -> tensor<1x32xf32> + %second = custom.op "linear_i8"(%relu, %weight2, %bias2, %multiplier, %input_step, %step2) : (tensor<1x32xf32>, tensor<32x16xi8>, tensor<1x16xi32>, tensor<1x1xf32>, tensor<1x1xf32>, tensor<1x16xf32>) -> tensor<1x16xf32> + %output = bufferization.to_buffer %second : tensor<1x16xf32> to memref<1x16xf32> + func.call @check_result(%output) : (memref<1x16xf32>) -> () + return %exit : i8 +} diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/linear_chain_i8_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/linear_chain_i8_main.cpp new file mode 100644 index 000000000..76c3315d9 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/linear_chain_i8_main.cpp @@ -0,0 +1,60 @@ +#include +#include +#include + +static void fail(void) { +#ifdef BAREMETAL + *(volatile uint32_t *)0x60000000 = 1; + while (1) { + } +#else + exit(1); +#endif +} + +static int8_t quantize(float value) { + int rounded = (int)value; + float fraction = value - rounded; + if (fraction > 0.5f || (fraction == 0.5f && rounded % 2 != 0)) + ++rounded; + if (fraction < -0.5f || (fraction == -0.5f && rounded % 2 != 0)) + --rounded; + if (rounded < -128) + rounded = -128; + if (rounded > 127) + rounded = 127; + return (int8_t)rounded; +} + +extern "C" void check_result(float *, float *output, int64_t offset, + int64_t rows, int64_t columns, int64_t row_stride, + int64_t column_stride) { + if (rows != 1 || columns != 16 || row_stride != 16 || column_stride != 1) + fail(); + int8_t input[32], intermediate[32]; + for (int k = 0; k < 32; ++k) + input[k] = quantize((float)((k % 7 - 3) * 99) * 0.5f); + for (int column = 0; column < 32; ++column) { + int32_t accumulator = column - 16; + for (int k = 0; k < 32; ++k) + accumulator += input[k] * ((3 * k + 5 * column) % 9 - 4); + float scale = 2.0f * ((column + 1) / 64.0f) * 0.5f; + int8_t requantized = quantize((float)accumulator * scale); + intermediate[column] = requantized < 0 ? 0 : requantized; + } + for (int column = 0; column < 16; ++column) { + int32_t accumulator = column - 8; + for (int k = 0; k < 32; ++k) + accumulator += intermediate[k] * ((5 * k + 3 * column) % 7 - 3); + float expected = (float)accumulator * (2.0f * ((column + 1) / 128.0f)); + float actual = output[offset + column * column_stride]; + // Binary fraction scales keep this small integer reference exact. + if (actual != expected) { + printf("FAILED: linear_chain_i8 column=%d expected=%f actual=%f\n", + column, expected, actual); + fail(); + } + } + printf( + "PASSED: linear_chain_i8 two Linear layers with ReLU and INT8 rescale\n"); +} diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/linear_int8.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/linear_int8.mlir new file mode 100644 index 000000000..34107df71 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/linear_int8.mlir @@ -0,0 +1,73 @@ +func.func private @check_result(memref<1x32xf32>) + +func.func @main() -> i8 { + %exit = arith.constant 0 : i8 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c3 = arith.constant 3 : index + %c6 = arith.constant 6 : index + %c7 = arith.constant 7 : index + %c13 = arith.constant 13 : index + %c16 = arith.constant 16 : index + %c32 = arith.constant 32 : index + %scale_unit = arith.constant 0.0078125 : f32 + %input = arith.constant dense<[[ + -300.0, -255.0, -5.0, -3.0, -1.0, 0.0, 1.0, 3.0, + 5.0, 254.0, 255.0, 300.0, 2.0, 6.0, 10.0, 14.0, + -257.0, -253.0, -7.0, -9.0, -11.0, -13.0, -15.0, 7.0, + 9.0, 11.0, 13.0, 15.0, 253.0, 257.0, -254.0, -256.0 + ]]> : tensor<1x32xf32> + %multiplier = arith.constant dense<0.5> : tensor<1x1xf32> + %input_step = arith.constant dense<2.0> : tensor<1x1xf32> + %zero = arith.constant dense<0.0> : tensor<1x32xf32> + %weights = memref.alloc() alignment = 64 : memref<32x32xi8> + %steps = memref.alloc() alignment = 64 : memref<1x32xf32> + %bias = memref.alloc() alignment = 64 : memref<1x32xi32> + scf.for %column = %c0 to %c32 step %c1 { + %number = arith.addi %column, %c1 : index + %number_i32 = arith.index_cast %number : index to i32 + %number_f32 = arith.sitofp %number_i32 : i32 to f32 + %step = arith.mulf %number_f32, %scale_unit : f32 + memref.store %step, %steps[%c0, %column] : memref<1x32xf32> + %shifted = arith.subi %column, %c16 : index + %shifted_i32 = arith.index_cast %shifted : index to i32 + memref.store %shifted_i32, %bias[%c0, %column] : memref<1x32xi32> + scf.for %k = %c0 to %c32 step %c1 { + %k7 = arith.muli %k, %c7 : index + %column3 = arith.muli %column, %c3 : index + %sum = arith.addi %k7, %column3 : index + %mod = arith.remui %sum, %c13 : index + %signed = arith.subi %mod, %c6 : index + %weight = arith.index_cast %signed : index to i8 + memref.store %weight, %weights[%k, %column] : memref<32x32xi8> + } + } + %weight_codes = bufferization.to_tensor %weights restrict : memref<32x32xi8> to tensor<32x32xi8> + %weight_step = bufferization.to_tensor %steps restrict : memref<1x32xf32> to tensor<1x32xf32> + %bias_codes = bufferization.to_tensor %bias restrict : memref<1x32xi32> to tensor<1x32xi32> + %codes = "frontend.fp32_to_int8"(%input, %multiplier) : (tensor<1x32xf32>, tensor<1x1xf32>) -> tensor<1x32xi8> + %activation = "frontend.int8_to_fp32"(%codes, %input_step) : (tensor<1x32xi8>, tensor<1x1xf32>) -> tensor<1x32xf32> + %weight_values = "frontend.int8_to_fp32"(%weight_codes, %weight_step) : (tensor<32x32xi8>, tensor<1x32xf32>) -> tensor<32x32xf32> + %product = linalg.matmul ins(%activation, %weight_values : tensor<1x32xf32>, tensor<32x32xf32>) outs(%zero : tensor<1x32xf32>) -> tensor<1x32xf32> + %empty = tensor.empty() : tensor<1x32xf32> + %bias_step = linalg.generic { + indexing_maps = [affine_map<(m,n)->(m,0)>, affine_map<(m,n)->(m,n)>, affine_map<(m,n)->(m,n)>], + iterator_types = ["parallel", "parallel"] + } ins(%input_step, %weight_step: tensor<1x1xf32>, tensor<1x32xf32>) outs(%empty: tensor<1x32xf32>) { + ^bb0(%lhs: f32, %rhs: f32, %unused: f32): + %scale = arith.mulf %lhs, %rhs : f32 + linalg.yield %scale : f32 + } -> tensor<1x32xf32> + %bias_tensor = "frontend.int32_to_fp32"(%bias_codes, %bias_step) : (tensor<1x32xi32>, tensor<1x32xf32>) -> tensor<1x32xf32> + %result = linalg.generic { + indexing_maps = [affine_map<(m, n) -> (m, n)>, affine_map<(m, n) -> (m, n)>, affine_map<(m, n) -> (m, n)>], + iterator_types = ["parallel", "parallel"] + } ins(%product, %bias_tensor : tensor<1x32xf32>, tensor<1x32xf32>) outs(%empty : tensor<1x32xf32>) { + ^bb0(%value: f32, %addend: f32, %unused: f32): + %added = arith.addf %value, %addend : f32 + linalg.yield %added : f32 + } -> tensor<1x32xf32> + %output = bufferization.to_buffer %result : tensor<1x32xf32> to memref<1x32xf32> + func.call @check_result(%output) : (memref<1x32xf32>) -> () + return %exit : i8 +} diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/linear_int8_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/linear_int8_main.cpp new file mode 100644 index 000000000..d7f23ac32 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/linear_int8_main.cpp @@ -0,0 +1,58 @@ +#include +#include +#include + +static void fail(void) { +#ifdef BAREMETAL + *(volatile uint32_t *)0x60000000 = 1; + while (1) { + } +#else + exit(1); +#endif +} + +extern "C" void check_result(float *, float *output, int64_t offset, + int64_t rows, int64_t columns, int64_t row_stride, + int64_t column_stride) { + if (rows != 1 || columns != 32 || row_stride != 32 || column_stride != 1) + fail(); + const float input[32] = { + -300, -255, -5, -3, -1, 0, 1, 3, 5, 254, 255, + 300, 2, 6, 10, 14, -257, -253, -7, -9, -11, -13, + -15, 7, 9, 11, 13, 15, 253, 257, -254, -256, + }; + int8_t codes[32]; + for (int k = 0; k < 32; ++k) { + float scaled = input[k] * 0.5f; + int rounded = (int)scaled; + float fraction = scaled - rounded; + if (fraction > 0.5f || (fraction == 0.5f && rounded % 2 != 0)) + ++rounded; + if (fraction < -0.5f || (fraction == -0.5f && rounded % 2 != 0)) + --rounded; + if (rounded < -128) + rounded = -128; + if (rounded > 127) + rounded = 127; + codes[k] = (int8_t)rounded; + } + for (int column = 0; column < 32; ++column) { + int32_t accumulator = column - 16; + for (int k = 0; k < 32; ++k) { + int8_t weight = (7 * k + 3 * column) % 13 - 6; + accumulator += codes[k] * weight; + } + float step = 2.0f * ((column + 1) / 128.0f); + float expected = (float)accumulator * step; + float actual = output[offset + column * column_stride]; + // The scales are binary fractions; these results are exact. + if (actual != expected) { + printf("FAILED: linear_int8 column=%d expected=%f actual=%f\n", column, + expected, actual); + fail(); + } + } + printf("PASSED: linear_int8 1x32x32 rounding, saturation, channel scales, " + "bias\n"); +} diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/lookup_channel_i8.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/lookup_channel_i8.mlir new file mode 100644 index 000000000..e5235e3b7 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/lookup_channel_i8.mlir @@ -0,0 +1,40 @@ +func.func private @check_result(memref<4x16xi8>) +func.func @main() -> i8 { + %zero = arith.constant 0 : i8 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c4 = arith.constant 4 : index + %c16 = arith.constant 16 : index + %c17 = arith.constant 17 : index + %c29 = arith.constant 29 : index + %c128 = arith.constant 128 : index + %c256 = arith.constant 256 : index + %c4096 = arith.constant 4096 : index + %input = memref.alloc() : memref<4x16xi8> + %table = memref.alloc() : memref<4096xi8> + scf.for %row = %c0 to %c4 step %c1 { + scf.for %lane = %c0 to %c16 step %c1 { + %base = arith.muli %row, %c16 : index + %offset = arith.addi %base, %lane : index + %scaled = arith.muli %offset, %c29 : index + %shifted = arith.subi %scaled, %c128 : index + %value = arith.index_cast %shifted : index to i8 + memref.store %value, %input[%row, %lane] : memref<4x16xi8> + } + } + scf.for %offset = %c0 to %c4096 step %c1 { + %lane = arith.divui %offset, %c256 : index + %entry = arith.remui %offset, %c256 : index + %scaled = arith.muli %lane, %c17 : index + %sum = arith.addi %entry, %scaled : index + %shifted = arith.subi %sum, %c128 : index + %value = arith.index_cast %shifted : index to i8 + memref.store %value, %table[%offset] : memref<4096xi8> + } + %x = bufferization.to_tensor %input restrict : memref<4x16xi8> to tensor<4x16xi8> + %lut = bufferization.to_tensor %table restrict : memref<4096xi8> to tensor<4096xi8> + %result = custom.op "lookup_i8"(%x, %lut) : (tensor<4x16xi8>, tensor<4096xi8>) -> tensor<4x16xi8> + %memory = bufferization.to_buffer %result : tensor<4x16xi8> to memref<4x16xi8> + func.call @check_result(%memory) : (memref<4x16xi8>) -> () + return %zero : i8 +} diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/lookup_channel_i8_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/lookup_channel_i8_main.cpp new file mode 100644 index 000000000..75cc70b73 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/lookup_channel_i8_main.cpp @@ -0,0 +1,20 @@ +#include +#include +#include + +extern "C" void check_result(int8_t *, int8_t *output, int64_t offset, + int64_t rows, int64_t columns, int64_t rs, + int64_t cs) { + if (rows != 4 || columns != 16) + abort(); + for (int row = 0; row < 4; ++row) + for (int lane = 0; lane < 16; ++lane) { + uint8_t input = (uint8_t)((row * 16 + lane) * 29 - 128); + int8_t expected = (int8_t)(input + lane * 17 - 128); + if (output[offset + row * rs + lane * cs] != expected) { + printf("FAILED: channel LUT row=%d lane=%d\n", row, lane); + abort(); + } + } + printf("PASSED: sixteen independent INT8 lookup tables\n"); +} diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d.mlir new file mode 100644 index 000000000..bd62e2fbb --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d.mlir @@ -0,0 +1,100 @@ +// FORM: %[[FIRST:.*]] = custom.op "conv2d_i8"{{.*}}pool = 1 : i64, relu = true{{.*}}-> tensor<1x4x4x16xi8> +// FORM: custom.op "depthwise_conv2d_i8"(%[[FIRST]],{{.*}}pool = 2 : i64, relu = false{{.*}}-> tensor<1x2x2x16xi8> +// FORM-NOT: frontend.fp32_to_int8 +// BANK: toint8_ball.bank_int32_to_int8 {{.*}}relu> +// BANK: im2col_ball.bank_im2col_i8 +// BANK: maxpool_ball.bank_maxpool_i8 +// BANK: return +func.func private @check_result(memref<1x16x2x2xi8>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero_i32 = arith.constant 0 : i32 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c2 = arith.constant 2 : index + %c3 = arith.constant 3 : index + %c4 = arith.constant 4 : index + %c5 = arith.constant 5 : index + %c7 = arith.constant 7 : index + %c9 = arith.constant 9 : index + %c16 = arith.constant 16 : index + %input = memref.alloc() alignment = 64 : memref<1x4x4x2xi8> + %conv_weight = memref.alloc() alignment = 64 : memref<16x2x3x3xi8> + %conv_bias = memref.alloc() alignment = 64 : memref<16xi32> + %depthwise_weight = memref.alloc() alignment = 64 : memref<16x1x3x3xi8> + %depthwise_bias = memref.alloc() alignment = 64 : memref<16xi32> + scf.for %h = %c0 to %c4 step %c1 { + scf.for %w = %c0 to %c4 step %c1 { + scf.for %ic = %c0 to %c2 step %c1 { + %h3 = arith.muli %h, %c3 : index + %w2 = arith.muli %w, %c2 : index + %sum0 = arith.addi %h3, %w2 : index + %sum = arith.addi %sum0, %ic : index + %rem = arith.remui %sum, %c7 : index + %signed = arith.subi %rem, %c3 : index + %value = arith.index_cast %signed : index to i8 + memref.store %value, %input[%c0, %h, %w, %ic] : memref<1x4x4x2xi8> + } + } + } + scf.for %ic = %c0 to %c2 step %c1 { + scf.for %p = %c0 to %c9 step %c1 { + scf.for %oc = %c0 to %c16 step %c1 { + %oc3 = arith.muli %oc, %c3 : index + %sum0 = arith.addi %p, %ic : index + %sum = arith.addi %sum0, %oc3 : index + %rem = arith.remui %sum, %c5 : index + %signed = arith.subi %rem, %c2 : index + %value = arith.index_cast %signed : index to i8 + %ky_index = arith.divui %p, %c3 : index + %kx_index = arith.remui %p, %c3 : index + memref.store %value, %conv_weight[%oc, %ic, %ky_index, %kx_index] : memref<16x2x3x3xi8> + } + } + } + scf.for %oc = %c0 to %c16 step %c1 { + %signed = arith.subi %oc, %c4 : index + %value = arith.index_cast %signed : index to i32 + memref.store %value, %conv_bias[%oc] : memref<16xi32> + } + scf.for %kh = %c0 to %c3 step %c1 { + scf.for %kw = %c0 to %c3 step %c1 { + scf.for %oc = %c0 to %c16 step %c1 { + %kw2 = arith.muli %kw, %c2 : index + %sum0 = arith.addi %kh, %kw2 : index + %sum = arith.addi %sum0, %oc : index + %rem = arith.remui %sum, %c5 : index + %signed = arith.subi %rem, %c2 : index + %value = arith.index_cast %signed : index to i8 + memref.store %value, %depthwise_weight[%oc, %c0, %kh, %kw] : memref<16x1x3x3xi8> + } + } + } + linalg.fill ins(%zero_i32 : i32) outs(%depthwise_bias : memref<16xi32>) + %unit = arith.constant dense<1.0> : tensor<1x1x1x1xf32> + %weight_step = arith.constant dense<1.0> : tensor<16x1x1x1xf32> + %it = bufferization.to_tensor %input restrict : memref<1x4x4x2xi8> to tensor<1x4x4x2xi8> + %input_float = "frontend.int8_to_fp32"(%it, %unit) : (tensor<1x4x4x2xi8>, tensor<1x1x1x1xf32>) -> tensor<1x4x4x2xf32> + %wt = bufferization.to_tensor %conv_weight restrict : memref<16x2x3x3xi8> to tensor<16x2x3x3xi8> + %bt = bufferization.to_tensor %conv_bias restrict : memref<16xi32> to tensor<16xi32> + %dt = bufferization.to_tensor %depthwise_weight restrict : memref<16x1x3x3xi8> to tensor<16x1x3x3xi8> + %dbt = bufferization.to_tensor %depthwise_bias restrict : memref<16xi32> to tensor<16xi32> + %first = custom.op "conv2d_i8"(%input_float, %wt, %bt, %unit, %unit, %weight_step) {strides = array, dilations = array, padding = array} : (tensor<1x4x4x2xf32>, tensor<16x2x3x3xi8>, tensor<16xi32>, tensor<1x1x1x1xf32>, tensor<1x1x1x1xf32>, tensor<16x1x1x1xf32>) -> tensor<1x4x4x16xf32> + %zero = arith.constant dense<0.0> : tensor<1x4x4x16xf32> + %relu = linalg.generic {indexing_maps = [affine_map<(n,h,w,c)->(n,h,w,c)>, affine_map<(n,h,w,c)->(n,h,w,c)>, affine_map<(n,h,w,c)->(n,h,w,c)>], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%first, %zero: tensor<1x4x4x16xf32>, tensor<1x4x4x16xf32>) outs(%zero: tensor<1x4x4x16xf32>) { + ^bb0(%a: f32, %b: f32, %out: f32): + %r = arith.maximumf %a, %b : f32 + linalg.yield %r : f32 + } -> tensor<1x4x4x16xf32> + %second = custom.op "depthwise_conv2d_i8"(%relu, %dt, %dbt, %unit, %unit, %weight_step) {strides = array, dilations = array, padding = array} : (tensor<1x4x4x16xf32>, tensor<16x1x3x3xi8>, tensor<16xi32>, tensor<1x1x1x1xf32>, tensor<1x1x1x1xf32>, tensor<16x1x1x1xf32>) -> tensor<1x4x4x16xf32> + %filter = tensor.empty() : tensor<2x2xf32> + %minimum = arith.constant dense<-3.40282347E+38> : tensor<1x2x2x16xf32> + %pooled = linalg.pooling_nhwc_max {strides = dense<2> : vector<2xi64>, dilations = dense<1> : vector<2xi64>} ins(%second, %filter: tensor<1x4x4x16xf32>, tensor<2x2xf32>) outs(%minimum: tensor<1x2x2x16xf32>) -> tensor<1x2x2x16xf32> + %q = "frontend.fp32_to_int8"(%pooled, %unit) : (tensor<1x2x2x16xf32>, tensor<1x1x1x1xf32>) -> tensor<1x2x2x16xi8> + %empty = tensor.empty() : tensor<1x16x2x2xi8> + %nchw = linalg.transpose ins(%q: tensor<1x2x2x16xi8>) outs(%empty: tensor<1x16x2x2xi8>) permutation = [0, 3, 1, 2] + %memory = bufferization.to_buffer %nchw : tensor<1x16x2x2xi8> to memref<1x16x2x2xi8> + func.call @check_result(%memory) : (memref<1x16x2x2xi8>) -> () + return %zero_i8 : i8 +} diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_flatten_linear.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_flatten_linear.mlir new file mode 100644 index 000000000..87be183f6 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_flatten_linear.mlir @@ -0,0 +1,93 @@ +// The pooled map stays in its bank and feeds the Linear transposed. +// FORM: custom.op "conv2d_i8"{{.*}}custom.resident{{.*}}pool = 2 : i64, relu = true +// FORM: custom.op "flatten_nchw" +// BANK: maxpool_ball.bank_maxpool_i8 +// BANK-NOT: buckyball.bank_mvout +// BANK: transpose_ball.bank_transpose_i8 +// BANK: smatmul_ball.bank_matmul_i8 +func.func private @check_result(memref<1x16xf32>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c2 = arith.constant 2 : index + %c3 = arith.constant 3 : index + %c5 = arith.constant 5 : index + %c7 = arith.constant 7 : index + %c8 = arith.constant 8 : index + %c9 = arith.constant 9 : index + %c16 = arith.constant 16 : index + %c256 = arith.constant 256 : index + %input = memref.alloc() alignment = 64 : memref<1x8x8x2xi8> + %conv_weight = memref.alloc() alignment = 64 : memref<16x2x3x3xi8> + %conv_bias = memref.alloc() alignment = 64 : memref<16xi32> + %linear_weight = memref.alloc() alignment = 64 : memref<256x16xi8> + %linear_bias = memref.alloc() alignment = 64 : memref<1x16xi32> + scf.for %h = %c0 to %c8 step %c1 { + scf.for %w = %c0 to %c8 step %c1 { + scf.for %ic = %c0 to %c2 step %c1 { + %h3 = arith.muli %h, %c3 : index + %w2 = arith.muli %w, %c2 : index + %sum0 = arith.addi %h3, %w2 : index + %sum = arith.addi %sum0, %ic : index + %rem = arith.remui %sum, %c7 : index + %signed = arith.subi %rem, %c3 : index + %value = arith.index_cast %signed : index to i8 + memref.store %value, %input[%c0, %h, %w, %ic] : memref<1x8x8x2xi8> + } + } + } + scf.for %oc = %c0 to %c16 step %c1 { + scf.for %ic = %c0 to %c2 step %c1 { + scf.for %p = %c0 to %c9 step %c1 { + %oc3 = arith.muli %oc, %c3 : index + %sum0 = arith.addi %p, %ic : index + %sum = arith.addi %sum0, %oc3 : index + %rem = arith.remui %sum, %c5 : index + %signed = arith.subi %rem, %c2 : index + %value = arith.index_cast %signed : index to i8 + %ky = arith.divui %p, %c3 : index + %kx = arith.remui %p, %c3 : index + memref.store %value, %conv_weight[%oc, %ic, %ky, %kx] : memref<16x2x3x3xi8> + } + } + %bias = arith.index_cast %oc : index to i32 + memref.store %bias, %conv_bias[%oc] : memref<16xi32> + memref.store %bias, %linear_bias[%c0, %oc] : memref<1x16xi32> + scf.for %k = %c0 to %c256 step %c1 { + %sum = arith.addi %k, %oc : index + %rem = arith.remui %sum, %c5 : index + %signed = arith.subi %rem, %c2 : index + %value = arith.index_cast %signed : index to i8 + memref.store %value, %linear_weight[%k, %oc] : memref<256x16xi8> + } + } + %unit = arith.constant dense<1.0> : tensor<1x1x1x1xf32> + %step = arith.constant dense<1.0> : tensor<1x1xf32> + %conv_step = arith.constant dense<1.0> : tensor<16x1x1x1xf32> + %linear_step = arith.constant dense<1.0> : tensor<1x16xf32> + %it = bufferization.to_tensor %input restrict : memref<1x8x8x2xi8> to tensor<1x8x8x2xi8> + %x = "frontend.int8_to_fp32"(%it, %unit) : (tensor<1x8x8x2xi8>, tensor<1x1x1x1xf32>) -> tensor<1x8x8x2xf32> + %wt = bufferization.to_tensor %conv_weight restrict : memref<16x2x3x3xi8> to tensor<16x2x3x3xi8> + %bt = bufferization.to_tensor %conv_bias restrict : memref<16xi32> to tensor<16xi32> + %lw = bufferization.to_tensor %linear_weight restrict : memref<256x16xi8> to tensor<256x16xi8> + %lb = bufferization.to_tensor %linear_bias restrict : memref<1x16xi32> to tensor<1x16xi32> + %conv = custom.op "conv2d_i8"(%x, %wt, %bt, %unit, %unit, %conv_step) {strides = array, dilations = array, padding = array} : (tensor<1x8x8x2xf32>, tensor<16x2x3x3xi8>, tensor<16xi32>, tensor<1x1x1x1xf32>, tensor<1x1x1x1xf32>, tensor<16x1x1x1xf32>) -> tensor<1x8x8x16xf32> + %zero = arith.constant dense<0.0> : tensor<1x8x8x16xf32> + %relu = linalg.generic {indexing_maps = [affine_map<(n,h,w,c)->(n,h,w,c)>, affine_map<(n,h,w,c)->(n,h,w,c)>, affine_map<(n,h,w,c)->(n,h,w,c)>], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%conv, %zero: tensor<1x8x8x16xf32>, tensor<1x8x8x16xf32>) outs(%zero: tensor<1x8x8x16xf32>) { + ^bb0(%a: f32, %b: f32, %out: f32): + %r = arith.maximumf %a, %b : f32 + linalg.yield %r : f32 + } -> tensor<1x8x8x16xf32> + %filter = tensor.empty() : tensor<2x2xf32> + %minimum = arith.constant dense<-3.40282347E+38> : tensor<1x4x4x16xf32> + %pooled = linalg.pooling_nhwc_max {strides = dense<2> : vector<2xi64>, dilations = dense<1> : vector<2xi64>} ins(%relu, %filter: tensor<1x8x8x16xf32>, tensor<2x2xf32>) outs(%minimum: tensor<1x4x4x16xf32>) -> tensor<1x4x4x16xf32> + %empty = tensor.empty() : tensor<1x16x4x4xf32> + %nchw = linalg.transpose ins(%pooled: tensor<1x4x4x16xf32>) outs(%empty: tensor<1x16x4x4xf32>) permutation = [0, 3, 1, 2] + %flat = tensor.collapse_shape %nchw [[0], [1, 2, 3]] : tensor<1x16x4x4xf32> into tensor<1x256xf32> + %result = custom.op "linear_i8"(%flat, %lw, %lb, %step, %step, %linear_step) : (tensor<1x256xf32>, tensor<256x16xi8>, tensor<1x16xi32>, tensor<1x1xf32>, tensor<1x1xf32>, tensor<1x16xf32>) -> tensor<1x16xf32> + %memory = bufferization.to_buffer %result : tensor<1x16xf32> to memref<1x16xf32> + func.call @check_result(%memory) : (memref<1x16xf32>) -> () + return %zero_i8 : i8 +} diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_flatten_linear_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_flatten_linear_main.cpp new file mode 100644 index 000000000..3770cfc76 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_flatten_linear_main.cpp @@ -0,0 +1,64 @@ +#include +#include +#include + +static void fail(void) { +#ifdef BAREMETAL + *(volatile uint32_t *)0x60000000 = 1; + while (1) { + } +#else + exit(1); +#endif +} + +static int clamp8(int value) { + return value < -128 ? -128 : (value > 127 ? 127 : value); +} + +extern "C" void check_result(float *allocated, float *aligned, int64_t offset, + int64_t rows, int64_t columns, int64_t row_stride, + int64_t column_stride) { + (void)allocated; + if (rows != 1 || columns != 16 || row_stride != 16 || column_stride != 1) + fail(); + int conv[8][8][16]; + for (int y = 0; y < 8; ++y) + for (int x = 0; x < 8; ++x) + for (int c = 0; c < 16; ++c) { + int sum = c; + for (int ky = 0; ky < 3; ++ky) + for (int kx = 0; kx < 3; ++kx) + for (int ic = 0; ic < 2; ++ic) { + int iy = y + ky - 1, ix = x + kx - 1; + if (iy >= 0 && iy < 8 && ix >= 0 && ix < 8) + sum += ((3 * iy + 2 * ix + ic) % 7 - 3) * + ((3 * ky + kx + ic + 3 * c) % 5 - 2); + } + conv[y][x][c] = sum < 0 ? 0 : clamp8(sum); + } + // The pooled map in channel-major order. + int flat[256]; + for (int c = 0; c < 16; ++c) + for (int y = 0; y < 4; ++y) + for (int x = 0; x < 4; ++x) { + int maximum = -128; + for (int dy = 0; dy < 2; ++dy) + for (int dx = 0; dx < 2; ++dx) + if (conv[2 * y + dy][2 * x + dx][c] > maximum) + maximum = conv[2 * y + dy][2 * x + dx][c]; + flat[c * 16 + y * 4 + x] = maximum; + } + float *output = aligned + offset; + for (int n = 0; n < 16; ++n) { + int sum = n; + for (int k = 0; k < 256; ++k) + sum += flat[k] * ((k + n) % 5 - 2); + if (output[n * column_stride] != (float)sum) { + printf("FAILED: mega_conv2d_flatten_linear n=%d exp=%d got=%f\n", n, sum, + output[n * column_stride]); + fail(); + } + } + printf("PASSED: mega_conv2d_flatten_linear resident map to Linear\n"); +} diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_k5_tail.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_k5_tail.mlir new file mode 100644 index 000000000..0fc47a463 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_k5_tail.mlir @@ -0,0 +1,59 @@ +func.func private @check_result(memref<1x16x5x5xf32>) +func.func @main() -> i8 { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c2 = arith.constant 2 : index + %c3 = arith.constant 3 : index + %c5 = arith.constant 5 : index + %c7 = arith.constant 7 : index + %c16 = arith.constant 16 : index + %c25 = arith.constant 25 : index + %zi8 = arith.constant 0 : i8 + %zi32 = arith.constant 0 : i32 + %input = memref.alloc() alignment = 64 : memref<1x5x5x2xi8> + %weight = memref.alloc() alignment = 64 : memref<16x2x5x5xi8> + %bias = memref.alloc() alignment = 64 : memref<16xi32> + linalg.fill ins(%zi8 : i8) outs(%weight : memref<16x2x5x5xi8>) + linalg.fill ins(%zi32 : i32) outs(%bias : memref<16xi32>) + scf.for %y = %c0 to %c5 step %c1 { + scf.for %x = %c0 to %c5 step %c1 { + scf.for %c = %c0 to %c2 step %c1 { + %y3 = arith.muli %y, %c3 : index + %x2 = arith.muli %x, %c2 : index + %s0 = arith.addi %y3, %x2 : index + %s = arith.addi %s0, %c : index + %r = arith.remui %s, %c7 : index + %v = arith.subi %r, %c3 : index + %i = arith.index_cast %v : index to i8 + memref.store %i, %input[%c0, %y, %x, %c] : memref<1x5x5x2xi8> + } + } + } + scf.for %c = %c0 to %c2 step %c1 { + scf.for %k = %c0 to %c25 step %c1 { + scf.for %o = %c0 to %c16 step %c1 { + %s0 = arith.addi %k, %c : index + %s = arith.addi %s0, %o : index + %r = arith.remui %s, %c3 : index + %v = arith.subi %r, %c1 : index + %i = arith.index_cast %v : index to i8 + %ky = arith.divui %k, %c5 : index + %kx = arith.remui %k, %c5 : index + memref.store %i, %weight[%o, %c, %ky, %kx] : memref<16x2x5x5xi8> + } + } + } + %input_tensor = bufferization.to_tensor %input restrict : memref<1x5x5x2xi8> to tensor<1x5x5x2xi8> + %input_step = arith.constant dense<1.0> : tensor<1x1x1x1xf32> + %input_float = "frontend.int8_to_fp32"(%input_tensor, %input_step) : (tensor<1x5x5x2xi8>, tensor<1x1x1x1xf32>) -> tensor<1x5x5x2xf32> + %weight_tensor = bufferization.to_tensor %weight restrict : memref<16x2x5x5xi8> to tensor<16x2x5x5xi8> + %bias_tensor = bufferization.to_tensor %bias restrict : memref<16xi32> to tensor<16xi32> + %weight_step = arith.constant dense<1.0> : tensor<16x1x1x1xf32> + %result = custom.op "conv2d_i8"(%input_float, %weight_tensor, %bias_tensor, %input_step, %input_step, %weight_step) {strides = array, dilations = array, padding = array} : (tensor<1x5x5x2xf32>, tensor<16x2x5x5xi8>, tensor<16xi32>, tensor<1x1x1x1xf32>, tensor<1x1x1x1xf32>, tensor<16x1x1x1xf32>) -> tensor<1x5x5x16xf32> + %empty = tensor.empty() : tensor<1x16x5x5xf32> + %nchw = linalg.transpose ins(%result: tensor<1x5x5x16xf32>) outs(%empty: tensor<1x16x5x5xf32>) permutation = [0, 3, 1, 2] + %memory = bufferization.to_buffer %nchw : tensor<1x16x5x5xf32> to memref<1x16x5x5xf32> + func.call @check_result(%memory) : (memref<1x16x5x5xf32>) -> () + %ok = arith.constant 0 : i8 + return %ok : i8 +} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/mega_conv2d_k5_tail_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_k5_tail_main.cpp similarity index 95% rename from examples/chips/pebble/workloads/mlir_tests/tile/mega_conv2d_k5_tail_main.cpp rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_k5_tail_main.cpp index 39bfae0a9..0aee33303 100644 --- a/examples/chips/pebble/workloads/mlir_tests/tile/mega_conv2d_k5_tail_main.cpp +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_k5_tail_main.cpp @@ -41,5 +41,5 @@ extern "C" void check_result(float *, float *data, int64_t offset, int64_t n, abort(); } } - printf("PASS k5 Conv: 4x4 tiles, tail, 8x16 packing and FP32 row copies\n"); + printf("PASS k5 Conv: 4x4 cores, tail, 8x16 packing and FP32 row copies\n"); } diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/mega_conv2d_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_main.cpp similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/tile/mega_conv2d_main.cpp rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_main.cpp diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_resident.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_resident.mlir new file mode 100644 index 000000000..9ed4ec0cd --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_resident.mlir @@ -0,0 +1,100 @@ +// FORM: %[[FIRST:.*]] = custom.op "conv2d_i8"{{.*}}custom.resident{{.*}}relu = true{{.*}}-> tensor<1x4x4x16xi8> +// BANK: %[[CODES:.*]] = toint8_ball.bank_int32_to_int8 {{.*}}relu> +// BANK-NOT: buckyball.bank_mvout +// BANK: im2col_ball.bank_im2col_i8 %[[CODES]] ->{{.*}}padding = 1, +// BANK: maxpool_ball.bank_maxpool_i8 +// BANK: return +func.func private @check_result(memref<1x16x2x2xi8>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero_i32 = arith.constant 0 : i32 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c2 = arith.constant 2 : index + %c3 = arith.constant 3 : index + %c4 = arith.constant 4 : index + %c5 = arith.constant 5 : index + %c7 = arith.constant 7 : index + %c9 = arith.constant 9 : index + %c16 = arith.constant 16 : index + %input = memref.alloc() alignment = 64 : memref<1x4x4x2xi8> + %conv_weight = memref.alloc() alignment = 64 : memref<16x2x3x3xi8> + %conv_bias = memref.alloc() alignment = 64 : memref<16xi32> + %second_weight = memref.alloc() alignment = 64 : memref<16x16x3x3xi8> + %second_bias = memref.alloc() alignment = 64 : memref<16xi32> + scf.for %h = %c0 to %c4 step %c1 { + scf.for %w = %c0 to %c4 step %c1 { + scf.for %ic = %c0 to %c2 step %c1 { + %h3 = arith.muli %h, %c3 : index + %w2 = arith.muli %w, %c2 : index + %sum0 = arith.addi %h3, %w2 : index + %sum = arith.addi %sum0, %ic : index + %rem = arith.remui %sum, %c7 : index + %signed = arith.subi %rem, %c3 : index + %value = arith.index_cast %signed : index to i8 + memref.store %value, %input[%c0, %h, %w, %ic] : memref<1x4x4x2xi8> + } + } + } + scf.for %ic = %c0 to %c2 step %c1 { + scf.for %p = %c0 to %c9 step %c1 { + scf.for %oc = %c0 to %c16 step %c1 { + %oc3 = arith.muli %oc, %c3 : index + %sum0 = arith.addi %p, %ic : index + %sum = arith.addi %sum0, %oc3 : index + %rem = arith.remui %sum, %c5 : index + %signed = arith.subi %rem, %c2 : index + %value = arith.index_cast %signed : index to i8 + %ky_index = arith.divui %p, %c3 : index + %kx_index = arith.remui %p, %c3 : index + memref.store %value, %conv_weight[%oc, %ic, %ky_index, %kx_index] : memref<16x2x3x3xi8> + } + } + } + scf.for %oc = %c0 to %c16 step %c1 { + %signed = arith.subi %oc, %c4 : index + %value = arith.index_cast %signed : index to i32 + memref.store %value, %conv_bias[%oc] : memref<16xi32> + } + scf.for %p = %c0 to %c9 step %c1 { + scf.for %ic = %c0 to %c16 step %c1 { + scf.for %oc = %c0 to %c16 step %c1 { + %sum0 = arith.addi %p, %ic : index + %sum = arith.addi %sum0, %oc : index + %rem = arith.remui %sum, %c5 : index + %signed = arith.subi %rem, %c2 : index + %value = arith.index_cast %signed : index to i8 + %kh = arith.divui %p, %c3 : index + %kw = arith.remui %p, %c3 : index + memref.store %value, %second_weight[%oc, %ic, %kh, %kw] : memref<16x16x3x3xi8> + } + } + } + linalg.fill ins(%zero_i32 : i32) outs(%second_bias : memref<16xi32>) + %unit = arith.constant dense<1.0> : tensor<1x1x1x1xf32> + %weight_step = arith.constant dense<1.0> : tensor<16x1x1x1xf32> + %it = bufferization.to_tensor %input restrict : memref<1x4x4x2xi8> to tensor<1x4x4x2xi8> + %input_float = "frontend.int8_to_fp32"(%it, %unit) : (tensor<1x4x4x2xi8>, tensor<1x1x1x1xf32>) -> tensor<1x4x4x2xf32> + %wt = bufferization.to_tensor %conv_weight restrict : memref<16x2x3x3xi8> to tensor<16x2x3x3xi8> + %bt = bufferization.to_tensor %conv_bias restrict : memref<16xi32> to tensor<16xi32> + %dt = bufferization.to_tensor %second_weight restrict : memref<16x16x3x3xi8> to tensor<16x16x3x3xi8> + %dbt = bufferization.to_tensor %second_bias restrict : memref<16xi32> to tensor<16xi32> + %first = custom.op "conv2d_i8"(%input_float, %wt, %bt, %unit, %unit, %weight_step) {strides = array, dilations = array, padding = array} : (tensor<1x4x4x2xf32>, tensor<16x2x3x3xi8>, tensor<16xi32>, tensor<1x1x1x1xf32>, tensor<1x1x1x1xf32>, tensor<16x1x1x1xf32>) -> tensor<1x4x4x16xf32> + %zero = arith.constant dense<0.0> : tensor<1x4x4x16xf32> + %relu = linalg.generic {indexing_maps = [affine_map<(n,h,w,c)->(n,h,w,c)>, affine_map<(n,h,w,c)->(n,h,w,c)>, affine_map<(n,h,w,c)->(n,h,w,c)>], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%first, %zero: tensor<1x4x4x16xf32>, tensor<1x4x4x16xf32>) outs(%zero: tensor<1x4x4x16xf32>) { + ^bb0(%a: f32, %b: f32, %out: f32): + %r = arith.maximumf %a, %b : f32 + linalg.yield %r : f32 + } -> tensor<1x4x4x16xf32> + %second = custom.op "conv2d_i8"(%relu, %dt, %dbt, %unit, %unit, %weight_step) {strides = array, dilations = array, padding = array} : (tensor<1x4x4x16xf32>, tensor<16x16x3x3xi8>, tensor<16xi32>, tensor<1x1x1x1xf32>, tensor<1x1x1x1xf32>, tensor<16x1x1x1xf32>) -> tensor<1x4x4x16xf32> + %filter = tensor.empty() : tensor<2x2xf32> + %minimum = arith.constant dense<-3.40282347E+38> : tensor<1x2x2x16xf32> + %pooled = linalg.pooling_nhwc_max {strides = dense<2> : vector<2xi64>, dilations = dense<1> : vector<2xi64>} ins(%second, %filter: tensor<1x4x4x16xf32>, tensor<2x2xf32>) outs(%minimum: tensor<1x2x2x16xf32>) -> tensor<1x2x2x16xf32> + %q = "frontend.fp32_to_int8"(%pooled, %unit) : (tensor<1x2x2x16xf32>, tensor<1x1x1x1xf32>) -> tensor<1x2x2x16xi8> + %empty = tensor.empty() : tensor<1x16x2x2xi8> + %nchw = linalg.transpose ins(%q: tensor<1x2x2x16xi8>) outs(%empty: tensor<1x16x2x2xi8>) permutation = [0, 3, 1, 2] + %memory = bufferization.to_buffer %nchw : tensor<1x16x2x2xi8> to memref<1x16x2x2xi8> + func.call @check_result(%memory) : (memref<1x16x2x2xi8>) -> () + return %zero_i8 : i8 +} diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_resident_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_resident_main.cpp new file mode 100644 index 000000000..2010702f4 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_resident_main.cpp @@ -0,0 +1,73 @@ +#include +#include +#include + +static void fail(void) { +#ifdef BAREMETAL + *(volatile uint32_t *)0x60000000 = 1; + while (1) { + } +#else + exit(1); +#endif +} + +extern "C" void check_result(int8_t *allocated, int8_t *aligned, int64_t offset, + int64_t n, int64_t channels, int64_t height, + int64_t width, int64_t n_stride, + int64_t channel_stride, int64_t height_stride, + int64_t width_stride) { + (void)allocated; + if (n != 1 || channels != 16 || height != 2 || width != 2 || n_stride != 64 || + channel_stride != 4 || height_stride != 2 || width_stride != 1) + fail(); + int8_t *output = aligned + offset; + int first[4][4][16]; + int second[4][4][16]; + for (int y = 0; y < 4; ++y) + for (int x = 0; x < 4; ++x) + for (int c = 0; c < 16; ++c) { + int sum = c - 4; + for (int ky = 0; ky < 3; ++ky) + for (int kx = 0; kx < 3; ++kx) + for (int ic = 0; ic < 2; ++ic) { + int iy = y + ky - 1, ix = x + kx - 1; + if (iy >= 0 && iy < 4 && ix >= 0 && ix < 4) + sum += ((3 * iy + 2 * ix + ic) % 7 - 3) * + ((3 * ky + kx + ic + 3 * c) % 5 - 2); + } + first[y][x][c] = sum < 0 ? 0 : (sum > 127 ? 127 : sum); + } + for (int y = 0; y < 4; ++y) + for (int x = 0; x < 4; ++x) + for (int c = 0; c < 16; ++c) { + int sum = 0; + for (int ky = 0; ky < 3; ++ky) + for (int kx = 0; kx < 3; ++kx) + for (int ic = 0; ic < 16; ++ic) { + int iy = y + ky - 1, ix = x + kx - 1; + if (iy >= 0 && iy < 4 && ix >= 0 && ix < 4) + sum += first[iy][ix][ic] * ((3 * ky + kx + ic + c) % 5 - 2); + } + second[y][x][c] = sum < -128 ? -128 : (sum > 127 ? 127 : sum); + } + for (int channel = 0; channel < 16; ++channel) { + for (int y = 0; y < 2; ++y) { + for (int x = 0; x < 2; ++x) { + int expected = -128; + for (int dy = 0; dy < 2; ++dy) + for (int dx = 0; dx < 2; ++dx) + if (second[2 * y + dy][2 * x + dx][channel] > expected) + expected = second[2 * y + dy][2 * x + dx][channel]; + int actual = output[channel * channel_stride + y * height_stride + x]; + if (actual != expected) { + printf("FAILED: mega_conv2d_resident c=%d y=%d x=%d exp=%d got=%d\n", + channel, y, x, expected, actual); + fail(); + } + } + } + } + printf("PASSED: mega_conv2d_resident Conv-to-Conv-to-MaxPool with a resident " + "map\n"); +} diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_s2_depthwise_s2.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_s2_depthwise_s2.mlir new file mode 100644 index 000000000..136bf9e86 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_s2_depthwise_s2.mlir @@ -0,0 +1,88 @@ +func.func private @check_result(memref<1x4x4x16xi8>) -> () + +func.func @main() -> i8 { + %i8_0 = arith.constant 0 : i8 + %i32_0 = arith.constant 0 : i32 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c2 = arith.constant 2 : index + %c3 = arith.constant 3 : index + %c4 = arith.constant 4 : index + %c5 = arith.constant 5 : index + %c7 = arith.constant 7 : index + %c9 = arith.constant 9 : index + %c16 = arith.constant 16 : index + %input = memref.alloc() alignment = 64 : memref<1x16x16x3xi8> + %weight0 = memref.alloc() alignment = 64 : memref<16x3x3x3xi8> + %bias0 = memref.alloc() alignment = 64 : memref<16xi32> + %weight1 = memref.alloc() alignment = 64 : memref<16x1x3x3xi8> + %bias1 = memref.alloc() alignment = 64 : memref<16xi32> + scf.for %y = %c0 to %c16 step %c1 { + scf.for %x = %c0 to %c16 step %c1 { + scf.for %ic = %c0 to %c3 step %c1 { + %a = arith.muli %y, %c3 : index + %b = arith.muli %x, %c2 : index + %d = arith.addi %a, %b : index + %e = arith.addi %d, %ic : index + %r = arith.remui %e, %c7 : index + %s = arith.subi %r, %c3 : index + %v = arith.index_cast %s : index to i8 + memref.store %v, %input[%c0, %y, %x, %ic] : memref<1x16x16x3xi8> + } + } + } + scf.for %ic = %c0 to %c3 step %c1 { + scf.for %p = %c0 to %c9 step %c1 { + scf.for %oc = %c0 to %c16 step %c1 { + %oc3 = arith.muli %oc, %c3 : index + %a = arith.addi %p, %ic : index + %b = arith.addi %a, %oc3 : index + %r = arith.remui %b, %c5 : index + %s = arith.subi %r, %c2 : index + %v = arith.index_cast %s : index to i8 + %ky_index = arith.divui %p, %c3 : index + %kx_index = arith.remui %p, %c3 : index + memref.store %v, %weight0[%oc, %ic, %ky_index, %kx_index] : memref<16x3x3x3xi8> + } + } + } + scf.for %oc = %c0 to %c16 step %c1 { + %s = arith.subi %oc, %c4 : index + %v = arith.index_cast %s : index to i32 + memref.store %v, %bias0[%oc] : memref<16xi32> + } + scf.for %ky = %c0 to %c3 step %c1 { + scf.for %kx = %c0 to %c3 step %c1 { + scf.for %c = %c0 to %c16 step %c1 { + %kx2 = arith.muli %kx, %c2 : index + %a = arith.addi %ky, %kx2 : index + %b = arith.addi %a, %c : index + %r = arith.remui %b, %c5 : index + %s = arith.subi %r, %c2 : index + %v = arith.index_cast %s : index to i8 + memref.store %v, %weight1[%c, %c0, %ky, %kx] : memref<16x1x3x3xi8> + } + } + } + linalg.fill ins(%i32_0 : i32) outs(%bias1 : memref<16xi32>) + %unit = arith.constant dense<1.0> : tensor<1x1x1x1xf32> + %weight_step = arith.constant dense<1.0> : tensor<16x1x1x1xf32> + %it = bufferization.to_tensor %input restrict : memref<1x16x16x3xi8> to tensor<1x16x16x3xi8> + %input_float = "frontend.int8_to_fp32"(%it, %unit) : (tensor<1x16x16x3xi8>, tensor<1x1x1x1xf32>) -> tensor<1x16x16x3xf32> + %wt = bufferization.to_tensor %weight0 restrict : memref<16x3x3x3xi8> to tensor<16x3x3x3xi8> + %bt = bufferization.to_tensor %bias0 restrict : memref<16xi32> to tensor<16xi32> + %dt = bufferization.to_tensor %weight1 restrict : memref<16x1x3x3xi8> to tensor<16x1x3x3xi8> + %dbt = bufferization.to_tensor %bias1 restrict : memref<16xi32> to tensor<16xi32> + %first = custom.op "conv2d_i8"(%input_float, %wt, %bt, %unit, %unit, %weight_step) {strides = array, dilations = array, padding = array} : (tensor<1x16x16x3xf32>, tensor<16x3x3x3xi8>, tensor<16xi32>, tensor<1x1x1x1xf32>, tensor<1x1x1x1xf32>, tensor<16x1x1x1xf32>) -> tensor<1x8x8x16xf32> + %zero = arith.constant dense<0.0> : tensor<1x8x8x16xf32> + %relu = linalg.generic {indexing_maps = [affine_map<(n,h,w,c)->(n,h,w,c)>, affine_map<(n,h,w,c)->(n,h,w,c)>, affine_map<(n,h,w,c)->(n,h,w,c)>], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%first, %zero: tensor<1x8x8x16xf32>, tensor<1x8x8x16xf32>) outs(%zero: tensor<1x8x8x16xf32>) { + ^bb0(%a: f32, %b: f32, %out: f32): + %r = arith.maximumf %a, %b : f32 + linalg.yield %r : f32 + } -> tensor<1x8x8x16xf32> + %second = custom.op "depthwise_conv2d_i8"(%relu, %dt, %dbt, %unit, %unit, %weight_step) {strides = array, dilations = array, padding = array} : (tensor<1x8x8x16xf32>, tensor<16x1x3x3xi8>, tensor<16xi32>, tensor<1x1x1x1xf32>, tensor<1x1x1x1xf32>, tensor<16x1x1x1xf32>) -> tensor<1x4x4x16xf32> + %q = "frontend.fp32_to_int8"(%second, %unit) : (tensor<1x4x4x16xf32>, tensor<1x1x1x1xf32>) -> tensor<1x4x4x16xi8> + %memory = bufferization.to_buffer %q : tensor<1x4x4x16xi8> to memref<1x4x4x16xi8> + func.call @check_result(%memory) : (memref<1x4x4x16xi8>) -> () + return %i8_0 : i8 +} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/mega_conv2d_s2_depthwise_s2_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_s2_depthwise_s2_main.cpp similarity index 93% rename from examples/chips/pebble/workloads/mlir_tests/tile/mega_conv2d_s2_depthwise_s2_main.cpp rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_s2_depthwise_s2_main.cpp index 8f9bf9202..b3609defb 100644 --- a/examples/chips/pebble/workloads/mlir_tests/tile/mega_conv2d_s2_depthwise_s2_main.cpp +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_s2_depthwise_s2_main.cpp @@ -65,18 +65,18 @@ extern "C" void check_result(int8_t *allocated, int8_t *aligned, int64_t offset, ++mismatch_count; } } - for (int tile_y = 0; tile_y < 2; ++tile_y) { - for (int tile_x = 0; tile_x < 2; ++tile_x) { + for (int core_y = 0; core_y < 2; ++core_y) { + for (int core_x = 0; core_x < 2; ++core_x) { uint32_t hash = 2166136261U; for (int dy = 0; dy < 2; ++dy) for (int dx = 0; dx < 2; ++dx) for (int c = 0; c < 16; ++c) { - int y = 2 * tile_y + dy, x = 2 * tile_x + dx; + int y = 2 * core_y + dy, x = 2 * core_x + dx; hash ^= (uint8_t)output[y * height_stride + x * width_stride + c]; hash *= 16777619U; } - printf("tile[%d,%d]=%08x%s", tile_y, tile_x, hash, - tile_x == 1 ? "\n" : " "); + printf("core[%d,%d]=%08x%s", core_y, core_x, hash, + core_x == 1 ? "\n" : " "); } } if (mismatch_count != 0) { diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_7x7_c288_exact.mlir.in b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_7x7_c288_exact.mlir.in new file mode 100644 index 000000000..344c5bf6f --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_7x7_c288_exact.mlir.in @@ -0,0 +1,16 @@ +memref.global "private" constant @input : memref<1x7x7x288xi8> = dense<"0x@PEBBLE_GAP_C288_INPUT_HEX@"> alignment = 64 + +func.func private @poison_reused_banks() -> () +func.func private @check_result(memref<1x1x1x288xi8>) -> () + +func.func @main() -> i8 { + %zero = arith.constant 0 : i8 + %input = memref.get_global @input : memref<1x7x7x288xi8> + func.call @poison_reused_banks() : () -> () + %source_tensor = bufferization.to_tensor %input restrict : memref<1x7x7x288xi8> to tensor<1x7x7x288xi8> + %source_matrix = tensor.collapse_shape %source_tensor [[0, 1, 2], [3]] : tensor<1x7x7x288xi8> into tensor<49x288xi8> + %average = custom.op "global_avg_i8"(%source_tensor) {input_scale = 0.0578813739 : f32, output_scale = 0.0106952852 : f32} : (tensor<1x7x7x288xi8>) -> tensor<1x1x1x288xi8> + %memory = bufferization.to_buffer %average : tensor<1x1x1x288xi8> to memref<1x1x1x288xi8> + func.call @check_result(%memory) : (memref<1x1x1x288xi8>) -> () + return %zero : i8 +} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/mega_global_avg_7x7_c288_exact_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_7x7_c288_exact_main.cpp similarity index 91% rename from examples/chips/pebble/workloads/mlir_tests/tile/mega_global_avg_7x7_c288_exact_main.cpp rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_7x7_c288_exact_main.cpp index 0c2d34139..7d64ed1be 100644 --- a/examples/chips/pebble/workloads/mlir_tests/tile/mega_global_avg_7x7_c288_exact_main.cpp +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_7x7_c288_exact_main.cpp @@ -5,7 +5,6 @@ #include "buckyball.h" #include #include -#include static void fail(void) { #ifdef BAREMETAL @@ -18,12 +17,12 @@ static void fail(void) { } enum { - PHYSICAL_BANKS = BANK_NUM, - BANK_ROWS = BANK_LINES, - ROW_BYTES = BANK_WIDTH / 8 + PHYSICAL_BANKS = THREAD_MEM_BANK_NUM, + BANK_ROWS = THREAD_MEM_BANK_ENTRIES, + ROW_BYTES = THREAD_MEM_BANK_WIDTH / 8 }; static_assert( - VIRTUAL_BANK_NUM >= PHYSICAL_BANKS, + CORE_VIRTUAL_BANK_COUNT >= PHYSICAL_BANKS, "bank reuse regression requires every physical bank to be addressable"); static elem_t stale_bank[BANK_ROWS * ROW_BYTES] __attribute__((aligned(64))); diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_conv2_exact.mlir.in b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_conv2_exact.mlir.in new file mode 100644 index 000000000..3e6462970 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_conv2_exact.mlir.in @@ -0,0 +1,84 @@ +memref.global "private" constant @input : memref<1x14x14x96xi8> = dense<"0x@PEBBLE_GAP_CONV_INPUT_HEX@"> alignment = 64 +memref.global "private" constant @weight1raw : memref<49168xi8> = dense<"0x00000000000000000000000000000000@PEBBLE_GAP_CONV_WEIGHT_HEX@"> alignment = 64 +memref.global "private" constant @bias1 : memref<24xi32> = dense<[-1929, -3345, -826, -4425, -1950, -1378, -2544, 3397, -134, -2621, 385, -1868, -3754, -948, 275, -53, -1955, -3197, -1065, -298, 536, -2267, -3803, -48]> alignment = 64 +memref.global "private" constant @scale1 : memref<24xf32> = dense<[9.64902255E-11, 1.20455174E-10, 9.11374864E-11, 6.07460609E-11, 6.96993296E-11, 2.21286572E-10, 8.11237535E-11, 1.33313652E-10, 1.26243266E-10, 1.44089102E-10, 0.0341239199, 6.56752361E-11, 6.62821395E-11, 1.91158489E-9, 1.69707901E-10, 4.13074019E-10, 5.95885979E-11, 8.92385332E-11, 6.12655274E-11, 2.72214418E-10, 2.77576795E-10, 2.37361242E-10, 6.75774228E-11, 1.48867613E-10]> alignment = 64 +memref.global "private" constant @weight2raw : memref<36880xi8> = dense<"0x00000000000000000000000000000000@PEBBLE_GAP_CONV_WEIGHT2_HEX@"> alignment = 64 +memref.global "private" constant @bias2 : memref<96xi32> = dense<[7051, 3006, 26558, 11845, -12167, 14432, -194, 9249, -17237, 22963, 7410, 91984, 105591, 6346, 90458, 43212, 45789, 33855, 29981, 10245, 4370, -34, -712, 15905, 22004, 549465, -5252, -3669, 7224, 18985, 8518, 8198, 16536, 51867, 4243, 6598, 9711, 18037, 21720, 397303, -253, 10751, -10281, -10424, 1839, 25010, 6995, -15222, 1786, -3524, -15704, 13370, 8813, 27391, 8927, -12218, -4217, -7494, 4135, 12616, 9593, 1431, -5377, 9947, 370, 11072, 5546, 22002, 10029, -3861, 904, 95829, 4717, -14504, 52996, 33835, 12322, 3329, 21275, -872, 1465, 17484, 6331, 63424, -29681, 34866, 3915, -2807, 71222, 5504, -13650, 15420, 7960, -9852, -13797, 21711]> alignment = 64 +memref.global "private" constant @scale2 : memref<96xf32> = dense<[0.00181691104, 4.059140E-3, 0.00131794217, 0.00212815683, 0.00430351309, 0.00403421139, 5.176660E-3, 0.00188985514, 0.00558878435, 0.00152020913, 0.00459996238, 5.82394132E-4, 4.24394966E-4, 0.0019368825, 7.07225234E-4, 6.37308462E-4, 0.00137251348, 7.15450151E-4, 0.00140084175, 0.00317073381, 5.073430E-3, 0.00307706394, 0.00386786251, 0.00195057772, 0.00210299343, 1.08822038E-4, 0.00415666727, 0.00333561096, 0.00485164393, 7.86353368E-4, 0.00419554161, 0.00339963497, 0.00123977242, 0.00106979779, 0.00394831039, 0.00133779203, 0.00282861362, 0.00155300961, 0.00196025497, 5.82583889E-5, 0.0032499244, 0.00156923081, 0.00266220793, 0.00191042456, 0.00408306951, 9.02582774E-4, 0.00326385885, 0.00143681327, 0.00142906886, 0.0026714697, 0.00458595809, 0.00430518808, 2.272860E-3, 0.00113706058, 0.00272057392, 0.00554801896, 6.058620E-3, 0.00159710483, 0.00501794415, 0.00278559304, 0.00328800408, 0.00337952748, 0.00245677959, 0.00205389271, 0.00226871902, 0.00263371225, 9.1695087E-4, 0.00129066338, 0.00205622637, 2.126570E-3, 9.56189353E-4, 2.12495346E-4, 9.6136285E-4, 0.00436848169, 9.72485286E-4, 4.94753069E-4, 0.00348583842, 0.00738966605, 0.00289806188, 0.00333968201, 0.00305957114, 0.00181978801, 0.00184966961, 3.52834439E-4, 0.00117929338, 9.00093349E-4, 0.00424369285, 0.00269078952, 5.541500E-4, 0.0025951541, 0.00531184906, 0.00391697325, 0.00378050678, 0.00192236435, 0.00131825986, 0.00273139146]> alignment = 64 +memref.global "private" constant @lut : memref<256xi8> = dense<"0x191A1A1A1A1A1A1B1B1B1B1B1B1C1C1C1C1C1C1D1D1D1D1D1D1E1E1E1E1E1E1F1F1F1F1F1F2020202020202121212121212222222222222323232323232424242424242525252525252626262626262727272727272828282828282929292929292A2A2A2A2A2A2B2B2B2B2B2B2C2C2C2C2C2C2D2D2D2D2D2D2E2E2E2E2E2E2F0404040505050505050606060606060707070707070808080808080909090909090A0A0A0A0A0A0B0B0B0B0B0B0C0C0C0C0C0C0D0D0D0D0D0D0E0E0E0E0E0E0F0F0F0F0F0F1010101010101111111111111212121212121313131313131414141414141515151515151616161616161717171717171818181818181919191919"> alignment = 64 +memref.global "private" constant @weight3raw : memref<73744xi8> = dense<"0x00000000000000000000000000000000@PEBBLE_GAP_CONV_WEIGHT3_HEX@"> alignment = 64 +memref.global "private" constant @bias3 : memref<40xi32> = dense<[104, -13887, -3685, -19928, 5678, 8790, -6325, -4745, 15957, 3843, 4642, 11420, -2754, -19949, 17269, -673, 16732, -31364, 19420, 3384, -1286, -5401, 3250, -9546, 3517, 6568, 9818, -3768, -8763, 19005, 2689, -2236, -3490, 6369, -18573, -726, -16056, 10604, -4229, -7803]> alignment = 64 +memref.global "private" constant @scale3 : memref<40xf32> = dense<[0.007250275, 0.0052957274, 0.00507997582, 5.148440E-3, 0.00615183357, 0.00551314512, 0.00447647972, 0.00624849601, 4.808580E-3, 0.00621820614, 0.00767033594, 0.00502041401, 0.00771069526, 0.00713740429, 5.340260E-3, 0.00554999569, 0.00578600029, 0.00608736509, 0.00647644047, 0.00674922066, 0.00569664268, 0.00644684816, 0.00639614975, 0.00778135518, 0.00677069137, 0.00366282626, 0.00660506124, 0.00643686252, 0.00655329926, 0.00661863247, 0.00624936819, 0.0104040103, 0.00504696835, 7.654780E-3, 0.00546469772, 5.341790E-3, 0.0050635729, 0.00579059403, 0.0084069958, 0.0039168736]> alignment = 64 +memref.global "private" constant @none : memref<1xi8> = dense<0> alignment = 64 +func.func private @check_result(memref<1x14x14x40xi8>) -> () func.func private @poison_reused_bank() -> () + +func.func @main() -> i8 { + %zero = arith.constant 0 : i8 + %c0 = arith.constant 0 : index %c1 = arith.constant 1 : index + %c14 = arith.constant 14 : index + %c96 = arith.constant 96 : index + %input = memref.get_global @input : memref<1x14x14x96xi8> + %input_heap = memref.alloc() alignment = 64 : memref<1x14x14x96xi8> + scf.for %y = %c0 to %c14 step %c1 { + scf.for %x = %c0 to %c14 step %c1 { + scf.for %c = %c0 to %c96 step %c1 { + %v = memref.load %input[%c0, %y, %x, %c] : memref<1x14x14x96xi8> + memref.store %v, %input_heap[%c0, %y, %x, %c] : memref<1x14x14x96xi8> + } + } + } + %weight1raw = memref.get_global @weight1raw : memref<49168xi8> + %weight1 = memref.reinterpret_cast %weight1raw to offset: [16], sizes: [2, 96, 16, 16], strides: [24576, 256, 16, 1] : memref<49168xi8> to memref<2x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>> + %bias1 = memref.get_global @bias1 : memref<24xi32> + %scale1 = memref.get_global @scale1 : memref<24xf32> + %weight2raw = memref.get_global @weight2raw : memref<36880xi8> + %weight2 = memref.reinterpret_cast %weight2raw to offset: [16], sizes: [6, 24, 16, 16], strides: [6144, 256, 16, 1] : memref<36880xi8> to memref<6x24x16x16xi8, strided<[6144, 256, 16, 1], offset: 16>> + %bias2 = memref.get_global @bias2 : memref<96xi32> + %scale2 = memref.get_global @scale2 : memref<96xf32> + %lut = memref.get_global @lut : memref<256xi8> + %weight3raw = memref.get_global @weight3raw : memref<73744xi8> + %weight3 = memref.reinterpret_cast %weight3raw to offset: [16], sizes: [3, 96, 16, 16], strides: [24576, 256, 16, 1] : memref<73744xi8> to memref<3x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>> + %bias3 = memref.get_global @bias3 : memref<40xi32> %scale3 = memref.get_global @scale3 : memref<40xf32> + func.call @poison_reused_bank() : () -> () + %source_tensor = bufferization.to_tensor %input_heap restrict : memref<1x14x14x96xi8> to tensor<1x14x14x96xi8> + %source_matrix = tensor.collapse_shape %source_tensor [[0, 1, 2], [3]] : tensor<1x14x14x96xi8> into tensor<196x96xi8> + %w1_raw = bufferization.to_tensor %weight1 restrict : memref<2x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>> to tensor<2x96x16x16xi8> + %w1_empty = tensor.empty() : tensor<96x24xi8> + %weights1 = linalg.generic {indexing_maps = [affine_map<(k,n)->(n floordiv 16,k,0,n mod 16)>, affine_map<(k,n)->(k,n)>], iterator_types = ["parallel", "parallel"]} ins(%w1_raw: tensor<2x96x16x16xi8>) outs(%w1_empty: tensor<96x24xi8>) { + ^bb0(%v: i8, %o: i8): linalg.yield %v : i8 + } -> tensor<96x24xi8> + %b1_raw = bufferization.to_tensor %bias1 restrict : memref<24xi32> to tensor<24xi32> + %bias_tensor1 = tensor.expand_shape %b1_raw [[0,1]] output_shape [1,24] : tensor<24xi32> into tensor<1x24xi32> + %s1_raw = bufferization.to_tensor %scale1 restrict : memref<24xf32> to tensor<24xf32> + %scale_tensor1 = tensor.expand_shape %s1_raw [[0,1]] output_shape [1,24] : tensor<24xf32> into tensor<1x24xf32> + %w2_raw = bufferization.to_tensor %weight2 restrict : memref<6x24x16x16xi8, strided<[6144, 256, 16, 1], offset: 16>> to tensor<6x24x16x16xi8> + %w2_empty = tensor.empty() : tensor<24x96xi8> + %weights2 = linalg.generic {indexing_maps = [affine_map<(k,n)->(n floordiv 16,k,0,n mod 16)>, affine_map<(k,n)->(k,n)>], iterator_types = ["parallel", "parallel"]} ins(%w2_raw: tensor<6x24x16x16xi8>) outs(%w2_empty: tensor<24x96xi8>) { + ^bb0(%v: i8, %o: i8): linalg.yield %v : i8 + } -> tensor<24x96xi8> + %b2_raw = bufferization.to_tensor %bias2 restrict : memref<96xi32> to tensor<96xi32> + %bias_tensor2 = tensor.expand_shape %b2_raw [[0,1]] output_shape [1,96] : tensor<96xi32> into tensor<1x96xi32> + %s2_raw = bufferization.to_tensor %scale2 restrict : memref<96xf32> to tensor<96xf32> + %scale_tensor2 = tensor.expand_shape %s2_raw [[0,1]] output_shape [1,96] : tensor<96xf32> into tensor<1x96xf32> + %w3_raw = bufferization.to_tensor %weight3 restrict : memref<3x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>> to tensor<3x96x16x16xi8> + %w3_empty = tensor.empty() : tensor<96x40xi8> + %weights3 = linalg.generic {indexing_maps = [affine_map<(k,n)->(n floordiv 16,k,0,n mod 16)>, affine_map<(k,n)->(k,n)>], iterator_types = ["parallel", "parallel"]} ins(%w3_raw: tensor<3x96x16x16xi8>) outs(%w3_empty: tensor<96x40xi8>) { + ^bb0(%v: i8, %o: i8): linalg.yield %v : i8 + } -> tensor<96x40xi8> + %b3_raw = bufferization.to_tensor %bias3 restrict : memref<40xi32> to tensor<40xi32> + %bias_tensor3 = tensor.expand_shape %b3_raw [[0,1]] output_shape [1,40] : tensor<40xi32> into tensor<1x40xi32> + %s3_raw = bufferization.to_tensor %scale3 restrict : memref<40xf32> to tensor<40xf32> + %scale_tensor3 = tensor.expand_shape %s3_raw [[0,1]] output_shape [1,40] : tensor<40xf32> into tensor<1x40xf32> + %table = bufferization.to_tensor %lut restrict : memref<256xi8> to tensor<256xi8> + %average = custom.op "global_avg_i8"(%source_tensor) {input_scale = 0.043821115 : f32, output_scale = 0.023039883 : f32} : (tensor<1x14x14x96xi8>) -> tensor<1x1x1x96xi8> + %a1 = tensor.collapse_shape %average [[0,1,2],[3]] : tensor<1x1x1x96xi8> into tensor<1x96xi8> + %v1 = custom.op "linear_i8_requant"(%a1, %weights1, %bias_tensor1, %scale_tensor1) {relu = true} : (tensor<1x96xi8>, tensor<96x24xi8>, tensor<1x24xi32>, tensor<1x24xf32>) -> tensor<1x24xi8> + %v2 = custom.op "linear_i8_requant"(%v1, %weights2, %bias_tensor2, %scale_tensor2) {relu = false} : (tensor<1x24xi8>, tensor<24x96xi8>, tensor<1x96xi32>, tensor<1x96xf32>) -> tensor<1x96xi8> + %looked_up = custom.op "lookup_i8"(%v2, %table) : (tensor<1x96xi8>, tensor<256xi8>) -> tensor<1x96xi8> + %multiplied = custom.op "gate_mul_i8"(%looked_up, %source_matrix) {ratio = 3.404043242e-02 : f32} : (tensor<1x96xi8>, tensor<196x96xi8>) -> tensor<196x96xi8> + %v3 = custom.op "linear_i8_requant"(%multiplied, %weights3, %bias_tensor3, %scale_tensor3) {relu = false} : (tensor<196x96xi8>, tensor<96x40xi8>, tensor<1x40xi32>, tensor<1x40xf32>) -> tensor<196x40xi8> + %final = tensor.expand_shape %v3 [[0,1,2],[3]] output_shape [1,14,14,40] : tensor<196x40xi8> into tensor<1x14x14x40xi8> + %memory = bufferization.to_buffer %final : tensor<1x14x14x40xi8> to memref<1x14x14x40xi8> + func.call @check_result(%memory) : (memref<1x14x14x40xi8>) -> () + return %zero : i8 +} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/mega_global_avg_conv2_exact_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_conv2_exact_main.cpp similarity index 90% rename from examples/chips/pebble/workloads/mlir_tests/tile/mega_global_avg_conv2_exact_main.cpp rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_conv2_exact_main.cpp index ee1e6ccdf..89d8ec202 100644 --- a/examples/chips/pebble/workloads/mlir_tests/tile/mega_global_avg_conv2_exact_main.cpp +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_conv2_exact_main.cpp @@ -5,7 +5,6 @@ #include "buckyball.h" #include #include -#include static void fail(void) { #ifdef BAREMETAL @@ -18,12 +17,12 @@ static void fail(void) { } enum { - PHYSICAL_BANKS = BANK_NUM, - BANK_ROWS = BANK_LINES, - ROW_BYTES = BANK_WIDTH / 8 + PHYSICAL_BANKS = THREAD_MEM_BANK_NUM, + BANK_ROWS = THREAD_MEM_BANK_ENTRIES, + ROW_BYTES = THREAD_MEM_BANK_WIDTH / 8 }; static_assert( - PHYSICAL_BANKS > 5 && VIRTUAL_BANK_NUM >= PHYSICAL_BANKS, + PHYSICAL_BANKS > 5 && CORE_VIRTUAL_BANK_COUNT >= PHYSICAL_BANKS, "bank reuse regression requires physical bank 5 to be addressable"); static elem_t stale_bank[BANK_ROWS * ROW_BYTES] __attribute__((aligned(64))); diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_conv_exact.mlir.in b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_conv_exact.mlir.in new file mode 100644 index 000000000..1640fc8bb --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_conv_exact.mlir.in @@ -0,0 +1,33 @@ +memref.global "private" constant @input : memref<1x14x14x96xi8> = dense<"0x@PEBBLE_GAP_CONV_INPUT_HEX@"> alignment = 64 +memref.global "private" constant @weight : memref<2x96x16x16xi8> = dense<"0x@PEBBLE_GAP_CONV_WEIGHT_HEX@"> alignment = 64 +memref.global "private" constant @bias : memref<24xi32> = dense<[-1929, -3345, -826, -4425, -1950, -1378, -2544, 3397, -134, -2621, 385, -1868, -3754, -948, 275, -53, -1955, -3197, -1065, -298, 536, -2267, -3803, -48]> alignment = 64 +memref.global "private" constant @scale : memref<24xf32> = dense<[9.64902255E-11, 1.20455174E-10, 9.11374864E-11, 6.07460609E-11, 6.96993296E-11, 2.21286572E-10, 8.11237535E-11, 1.33313652E-10, 1.26243266E-10, 1.44089102E-10, 0.0341239199, 6.56752361E-11, 6.62821395E-11, 1.91158489E-9, 1.69707901E-10, 4.13074019E-10, 5.95885979E-11, 8.92385332E-11, 6.12655274E-11, 2.72214418E-10, 2.77576795E-10, 2.37361242E-10, 6.75774228E-11, 1.48867613E-10]> alignment = 64 +memref.global "private" constant @none : memref<1xi8> = dense<0> alignment = 64 + +func.func private @check_result(memref<1x1x1x24xi8>) -> () + +func.func @main() -> i8 { + %zero = arith.constant 0 : i8 + %input = memref.get_global @input : memref<1x14x14x96xi8> + %weight = memref.get_global @weight : memref<2x96x16x16xi8> + %bias = memref.get_global @bias : memref<24xi32> + %scale = memref.get_global @scale : memref<24xf32> + %source_tensor = bufferization.to_tensor %input restrict : memref<1x14x14x96xi8> to tensor<1x14x14x96xi8> + %source_matrix = tensor.collapse_shape %source_tensor [[0, 1, 2], [3]] : tensor<1x14x14x96xi8> into tensor<196x96xi8> + %w1_raw = bufferization.to_tensor %weight restrict : memref<2x96x16x16xi8> to tensor<2x96x16x16xi8> + %w1_empty = tensor.empty() : tensor<96x24xi8> + %weights1 = linalg.generic {indexing_maps = [affine_map<(k,n)->(n floordiv 16,k,0,n mod 16)>, affine_map<(k,n)->(k,n)>], iterator_types = ["parallel", "parallel"]} ins(%w1_raw: tensor<2x96x16x16xi8>) outs(%w1_empty: tensor<96x24xi8>) { + ^bb0(%v: i8, %o: i8): linalg.yield %v : i8 + } -> tensor<96x24xi8> + %b1_raw = bufferization.to_tensor %bias restrict : memref<24xi32> to tensor<24xi32> + %bias_tensor1 = tensor.expand_shape %b1_raw [[0,1]] output_shape [1,24] : tensor<24xi32> into tensor<1x24xi32> + %s1_raw = bufferization.to_tensor %scale restrict : memref<24xf32> to tensor<24xf32> + %scale_tensor1 = tensor.expand_shape %s1_raw [[0,1]] output_shape [1,24] : tensor<24xf32> into tensor<1x24xf32> + %average = custom.op "global_avg_i8"(%source_tensor) {input_scale = 0.043821115 : f32, output_scale = 0.023039883 : f32} : (tensor<1x14x14x96xi8>) -> tensor<1x1x1x96xi8> + %a1 = tensor.collapse_shape %average [[0,1,2],[3]] : tensor<1x1x1x96xi8> into tensor<1x96xi8> + %v1 = custom.op "linear_i8_requant"(%a1, %weights1, %bias_tensor1, %scale_tensor1) {relu = true} : (tensor<1x96xi8>, tensor<96x24xi8>, tensor<1x24xi32>, tensor<1x24xf32>) -> tensor<1x24xi8> + %final = tensor.expand_shape %v1 [[0,1,2],[3]] output_shape [1,1,1,24] : tensor<1x24xi8> into tensor<1x1x1x24xi8> + %memory = bufferization.to_buffer %final : tensor<1x1x1x24xi8> to memref<1x1x1x24xi8> + func.call @check_result(%memory) : (memref<1x1x1x24xi8>) -> () + return %zero : i8 +} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/mega_global_avg_conv_exact_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_conv_exact_main.cpp similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/tile/mega_global_avg_conv_exact_main.cpp rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_conv_exact_main.cpp diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_matmul.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_matmul.mlir new file mode 100644 index 000000000..e03d1eb8f --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_matmul.mlir @@ -0,0 +1,45 @@ +func.func private @check_result(memref<1x16xf32>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %two = arith.constant 2 : index + %three = arith.constant 3 : index + %five = arith.constant 5 : index + %seven = arith.constant 7 : index + %eight = arith.constant 8 : index + %dim = arith.constant 16 : index + %input = memref.alloc() alignment = 64 : memref<1x16xi8> + %weight = memref.alloc() alignment = 64 : memref<16x16xi8> + %bias = memref.alloc() alignment = 64 : memref<16xi32> + scf.for %k = %zero to %dim step %one { + %irem = arith.remui %k, %seven : index + %ival = arith.subi %irem, %three : index + %iv = arith.index_cast %ival : index to i8 + memref.store %iv, %input[%zero, %k] : memref<1x16xi8> + %biasv = arith.subi %k, %eight : index + %bias32 = arith.index_cast %biasv : index to i32 + memref.store %bias32, %bias[%k] : memref<16xi32> + scf.for %column = %zero to %dim step %one { + %k2 = arith.muli %k, %two : index + %c3 = arith.muli %column, %three : index + %sum = arith.addi %k2, %c3 : index + %rem = arith.remui %sum, %five : index + %value = arith.subi %rem, %two : index + %wv = arith.index_cast %value : index to i8 + memref.store %wv, %weight[%k, %column] : memref<16x16xi8> + } + } + %it = bufferization.to_tensor %input restrict : memref<1x16xi8> to tensor<1x16xi8> + %wt = bufferization.to_tensor %weight restrict : memref<16x16xi8> to tensor<16x16xi8> + %bt = bufferization.to_tensor %bias restrict : memref<16xi32> to tensor<16xi32> + %bias_tensor = tensor.expand_shape %bt [[0, 1]] output_shape [1, 16] : tensor<16xi32> into tensor<1x16xi32> + %unit = arith.constant dense<1.0> : tensor<1x1xf32> + %weight_step = arith.constant dense<1.0> : tensor<1x16xf32> + %input_float = "frontend.int8_to_fp32"(%it, %unit) : (tensor<1x16xi8>, tensor<1x1xf32>) -> tensor<1x16xf32> + %result = custom.op "linear_i8"(%input_float, %wt, %bias_tensor, %unit, %unit, %weight_step) : (tensor<1x16xf32>, tensor<16x16xi8>, tensor<1x16xi32>, tensor<1x1xf32>, tensor<1x1xf32>, tensor<1x16xf32>) -> tensor<1x16xf32> + %memory = bufferization.to_buffer %result : tensor<1x16xf32> to memref<1x16xf32> + func.call @check_result(%memory) : (memref<1x16xf32>) -> () + return %zero_i8 : i8 +} diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_matmul_m17_k65_n17.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_matmul_m17_k65_n17.mlir new file mode 100644 index 000000000..aa5b49851 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_matmul_m17_k65_n17.mlir @@ -0,0 +1,50 @@ +func.func private @check_result(memref<17x17xf32>) -> () +func.func @main() -> i8 { + %z8 = arith.constant 0 : i8 + %z = arith.constant 0 : index + %one = arith.constant 1 : index + %two = arith.constant 2 : index + %three = arith.constant 3 : index + %five = arith.constant 5 : index + %seven = arith.constant 7 : index + %eight = arith.constant 8 : index + %dim = arith.constant 17 : index + %inner = arith.constant 65 : index + %input = memref.alloc() alignment = 64 : memref<17x65xi8> + %weight = memref.alloc() alignment = 64 : memref<65x17xi8> + %bias = memref.alloc() alignment = 64 : memref<17xi32> + scf.for %r = %z to %dim step %one { + scf.for %k = %z to %inner step %one { + %sum = arith.addi %r, %k : index + %rem = arith.remui %sum, %seven : index + %v = arith.subi %rem, %three : index + %i8 = arith.index_cast %v : index to i8 + memref.store %i8, %input[%r, %k] : memref<17x65xi8> + } + %b = arith.subi %r, %eight : index + %b32 = arith.index_cast %b : index to i32 + memref.store %b32, %bias[%r] : memref<17xi32> + } + scf.for %k = %z to %inner step %one { + scf.for %c = %z to %dim step %one { + %k2 = arith.muli %k, %two : index + %c3 = arith.muli %c, %three : index + %sum = arith.addi %k2, %c3 : index + %rem = arith.remui %sum, %five : index + %v = arith.subi %rem, %two : index + %i8 = arith.index_cast %v : index to i8 + memref.store %i8, %weight[%k, %c] : memref<65x17xi8> + } + } + %it = bufferization.to_tensor %input restrict : memref<17x65xi8> to tensor<17x65xi8> + %wt = bufferization.to_tensor %weight restrict : memref<65x17xi8> to tensor<65x17xi8> + %bt = bufferization.to_tensor %bias restrict : memref<17xi32> to tensor<17xi32> + %bias_tensor = tensor.expand_shape %bt [[0, 1]] output_shape [1, 17] : tensor<17xi32> into tensor<1x17xi32> + %unit = arith.constant dense<1.0> : tensor<1x1xf32> + %weight_step = arith.constant dense<0.5> : tensor<1x17xf32> + %input_float = "frontend.int8_to_fp32"(%it, %unit) : (tensor<17x65xi8>, tensor<1x1xf32>) -> tensor<17x65xf32> + %result = custom.op "linear_i8"(%input_float, %wt, %bias_tensor, %unit, %unit, %weight_step) : (tensor<17x65xf32>, tensor<65x17xi8>, tensor<1x17xi32>, tensor<1x1xf32>, tensor<1x1xf32>, tensor<1x17xf32>) -> tensor<17x17xf32> + %memory = bufferization.to_buffer %result : tensor<17x17xf32> to memref<17x17xf32> + func.call @check_result(%memory) : (memref<17x17xf32>) -> () + return %z8 : i8 +} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/mega_matmul_m17_k65_n17_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_matmul_m17_k65_n17_main.cpp similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/tile/mega_matmul_m17_k65_n17_main.cpp rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_matmul_m17_k65_n17_main.cpp diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/mega_matmul_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_matmul_main.cpp similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/tile/mega_matmul_main.cpp rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_matmul_main.cpp diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_se_gate_conv.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_se_gate_conv.mlir new file mode 100644 index 000000000..37cce5e11 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_se_gate_conv.mlir @@ -0,0 +1,99 @@ +func.func private @check_result(memref<1x6x6x40xi8>) -> () + +func.func @main() -> i8 { + %i8_0 = arith.constant 0 : i8 + %i8_1 = arith.constant 1 : i8 + %i32_0 = arith.constant 0 : i32 + %i32_1 = arith.constant 1 : i32 + %f32_001 = arith.constant 0.01 : f32 + %f32_01 = arith.constant 0.1 : f32 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c2 = arith.constant 2 : index + %c5 = arith.constant 5 : index + %c6 = arith.constant 6 : index + %c96 = arith.constant 96 : index + %c256 = arith.constant 256 : index + %input = memref.alloc() alignment = 64 : memref<1x6x6x96xi8> + %w1raw = memref.alloc() alignment = 64 : memref<49168xi8> + %w1 = memref.reinterpret_cast %w1raw to offset: [16], sizes: [2, 96, 16, 16], strides: [24576, 256, 16, 1] : memref<49168xi8> to memref<2x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>> + %b1 = memref.alloc() alignment = 64 : memref<24xi32> + %s1 = memref.alloc() alignment = 64 : memref<24xf32> + %w2raw = memref.alloc() alignment = 64 : memref<36880xi8> + %w2 = memref.reinterpret_cast %w2raw to offset: [16], sizes: [6, 24, 16, 16], strides: [6144, 256, 16, 1] : memref<36880xi8> to memref<6x24x16x16xi8, strided<[6144, 256, 16, 1], offset: 16>> + %b2 = memref.alloc() alignment = 64 : memref<96xi32> + %s2 = memref.alloc() alignment = 64 : memref<96xf32> + %lut = memref.alloc() alignment = 64 : memref<256xi8> + %w3raw = memref.alloc() alignment = 64 : memref<73744xi8> + %w3 = memref.reinterpret_cast %w3raw to offset: [16], sizes: [3, 96, 16, 16], strides: [24576, 256, 16, 1] : memref<73744xi8> to memref<3x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>> + %b3 = memref.alloc() alignment = 64 : memref<40xi32> + %s3 = memref.alloc() alignment = 64 : memref<40xf32> + scf.for %y = %c0 to %c6 step %c1 { + scf.for %x = %c0 to %c6 step %c1 { + scf.for %c = %c0 to %c96 step %c1 { + %a = arith.addi %y, %x : index + %d = arith.addi %a, %c : index + %r = arith.remui %d, %c5 : index + %v = arith.addi %r, %c1 : index + %q = arith.index_cast %v : index to i8 + memref.store %q, %input[%c0, %y, %x, %c] : memref<1x6x6x96xi8> + } + } + } + linalg.fill ins(%i8_1 : i8) outs(%w1 : memref<2x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>>) + linalg.fill ins(%i8_1 : i8) outs(%w2 : memref<6x24x16x16xi8, strided<[6144, 256, 16, 1], offset: 16>>) + linalg.fill ins(%i8_1 : i8) outs(%w3 : memref<3x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>>) + linalg.fill ins(%i32_1 : i32) outs(%b1 : memref<24xi32>) + linalg.fill ins(%i32_0 : i32) outs(%b2 : memref<96xi32>) + linalg.fill ins(%i32_0 : i32) outs(%b3 : memref<40xi32>) + linalg.fill ins(%f32_001 : f32) outs(%s1 : memref<24xf32>) + linalg.fill ins(%f32_01 : f32) outs(%s2 : memref<96xf32>) + linalg.fill ins(%f32_001 : f32) outs(%s3 : memref<40xf32>) + scf.for %i = %c0 to %c256 step %c1 { + %v = arith.index_cast %i : index to i8 + memref.store %v, %lut[%i] : memref<256xi8> + } + %source_tensor = bufferization.to_tensor %input restrict : memref<1x6x6x96xi8> to tensor<1x6x6x96xi8> + %source_matrix = tensor.collapse_shape %source_tensor [[0, 1, 2], [3]] : tensor<1x6x6x96xi8> into tensor<36x96xi8> + %w1_raw = bufferization.to_tensor %w1 restrict : memref<2x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>> to tensor<2x96x16x16xi8> + %w1_empty = tensor.empty() : tensor<96x24xi8> + %weights1 = linalg.generic {indexing_maps = [affine_map<(k,n)->(n floordiv 16,k,0,n mod 16)>, affine_map<(k,n)->(k,n)>], iterator_types = ["parallel", "parallel"]} ins(%w1_raw: tensor<2x96x16x16xi8>) outs(%w1_empty: tensor<96x24xi8>) { + ^bb0(%v: i8, %o: i8): linalg.yield %v : i8 + } -> tensor<96x24xi8> + %b1_raw = bufferization.to_tensor %b1 restrict : memref<24xi32> to tensor<24xi32> + %bias_tensor1 = tensor.expand_shape %b1_raw [[0,1]] output_shape [1,24] : tensor<24xi32> into tensor<1x24xi32> + %s1_raw = bufferization.to_tensor %s1 restrict : memref<24xf32> to tensor<24xf32> + %scale_tensor1 = tensor.expand_shape %s1_raw [[0,1]] output_shape [1,24] : tensor<24xf32> into tensor<1x24xf32> + %w2_raw = bufferization.to_tensor %w2 restrict : memref<6x24x16x16xi8, strided<[6144, 256, 16, 1], offset: 16>> to tensor<6x24x16x16xi8> + %w2_empty = tensor.empty() : tensor<24x96xi8> + %weights2 = linalg.generic {indexing_maps = [affine_map<(k,n)->(n floordiv 16,k,0,n mod 16)>, affine_map<(k,n)->(k,n)>], iterator_types = ["parallel", "parallel"]} ins(%w2_raw: tensor<6x24x16x16xi8>) outs(%w2_empty: tensor<24x96xi8>) { + ^bb0(%v: i8, %o: i8): linalg.yield %v : i8 + } -> tensor<24x96xi8> + %b2_raw = bufferization.to_tensor %b2 restrict : memref<96xi32> to tensor<96xi32> + %bias_tensor2 = tensor.expand_shape %b2_raw [[0,1]] output_shape [1,96] : tensor<96xi32> into tensor<1x96xi32> + %s2_raw = bufferization.to_tensor %s2 restrict : memref<96xf32> to tensor<96xf32> + %scale_tensor2 = tensor.expand_shape %s2_raw [[0,1]] output_shape [1,96] : tensor<96xf32> into tensor<1x96xf32> + %w3_raw = bufferization.to_tensor %w3 restrict : memref<3x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>> to tensor<3x96x16x16xi8> + %w3_empty = tensor.empty() : tensor<96x40xi8> + %weights3 = linalg.generic {indexing_maps = [affine_map<(k,n)->(n floordiv 16,k,0,n mod 16)>, affine_map<(k,n)->(k,n)>], iterator_types = ["parallel", "parallel"]} ins(%w3_raw: tensor<3x96x16x16xi8>) outs(%w3_empty: tensor<96x40xi8>) { + ^bb0(%v: i8, %o: i8): linalg.yield %v : i8 + } -> tensor<96x40xi8> + %b3_raw = bufferization.to_tensor %b3 restrict : memref<40xi32> to tensor<40xi32> + %bias_tensor3 = tensor.expand_shape %b3_raw [[0,1]] output_shape [1,40] : tensor<40xi32> into tensor<1x40xi32> + %s3_raw = bufferization.to_tensor %s3 restrict : memref<40xf32> to tensor<40xf32> + %scale_tensor3 = tensor.expand_shape %s3_raw [[0,1]] output_shape [1,40] : tensor<40xf32> into tensor<1x40xf32> + %table = bufferization.to_tensor %lut restrict : memref<256xi8> to tensor<256xi8> + scf.for %iteration = %c0 to %c2 step %c1 { + %average = custom.op "global_avg_i8"(%source_tensor) {input_scale = 1.0 : f32, output_scale = 1.0 : f32} : (tensor<1x6x6x96xi8>) -> tensor<1x1x1x96xi8> + %a1 = tensor.collapse_shape %average [[0,1,2],[3]] : tensor<1x1x1x96xi8> into tensor<1x96xi8> + %v1 = custom.op "linear_i8_requant"(%a1, %weights1, %bias_tensor1, %scale_tensor1) {relu = true} : (tensor<1x96xi8>, tensor<96x24xi8>, tensor<1x24xi32>, tensor<1x24xf32>) -> tensor<1x24xi8> + %v2 = custom.op "linear_i8_requant"(%v1, %weights2, %bias_tensor2, %scale_tensor2) {relu = false} : (tensor<1x24xi8>, tensor<24x96xi8>, tensor<1x96xi32>, tensor<1x96xf32>) -> tensor<1x96xi8> + %looked_up = custom.op "lookup_i8"(%v2, %table) : (tensor<1x96xi8>, tensor<256xi8>) -> tensor<1x96xi8> + %multiplied = custom.op "gate_mul_i8"(%looked_up, %source_matrix) {ratio = 1.000000015e-01 : f32} : (tensor<1x96xi8>, tensor<36x96xi8>) -> tensor<36x96xi8> + %v3 = custom.op "linear_i8_requant"(%multiplied, %weights3, %bias_tensor3, %scale_tensor3) {relu = false} : (tensor<36x96xi8>, tensor<96x40xi8>, tensor<1x40xi32>, tensor<1x40xf32>) -> tensor<36x40xi8> + %final = tensor.expand_shape %v3 [[0,1,2],[3]] output_shape [1,6,6,40] : tensor<36x40xi8> into tensor<1x6x6x40xi8> + %memory = bufferization.to_buffer %final : tensor<1x6x6x40xi8> to memref<1x6x6x40xi8> + func.call @check_result(%memory) : (memref<1x6x6x40xi8>) -> () + } + return %i8_0 : i8 +} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/mega_se_gate_conv_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_se_gate_conv_main.cpp similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/tile/mega_se_gate_conv_main.cpp rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_se_gate_conv_main.cpp diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/CMakeLists.txt new file mode 100644 index 000000000..fde4acf2e --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/CMakeLists.txt @@ -0,0 +1,191 @@ +set(BUCKYBALL_MLIR_GROUP_TARGET pebble-mlir-tests-build) +set(BUCKYBALL_MLIR_TEST_PREFIX linalg) + +add_buckyball_mlir_test(linalg_relu_513x17_i32 + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) + +add_buckyball_mlir_test(linalg_matmul_16x16_i8 + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) + +add_buckyball_mlir_test(linalg_transpose_16x16_i8 + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) + +add_buckyball_mlir_test(linalg_transpose_detect_1x84x100_f32 + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) + +add_buckyball_mlir_test(linalg_transpose_nchw_nhwc_1x3x8x8_f32 + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) + +add_buckyball_mlir_test(linalg_transpose_oihw_ohwi_4x8x3x3_f32 + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) + +add_buckyball_mlir_test(linalg_mega_conv2d_k3_c16 + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_matmul_16x16_i8.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_matmul_16x16_i8.mlir new file mode 100644 index 000000000..4e8544c9a --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_matmul_16x16_i8.mlir @@ -0,0 +1,41 @@ +func.func private @check_result(memref<16x16xi32>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero_i32 = arith.constant 0 : i32 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %two = arith.constant 2 : index + %three = arith.constant 3 : index + %five = arith.constant 5 : index + %seven = arith.constant 7 : index + %dim = arith.constant 16 : index + %a = memref.alloc() alignment = 64 : memref<16x16xi8> + %b = memref.alloc() alignment = 64 : memref<16x16xi8> + + scf.for %row = %zero to %dim step %one { + scf.for %column = %zero to %dim step %one { + %r3 = arith.muli %row, %three : index + %c5 = arith.muli %column, %five : index + %asum = arith.addi %r3, %c5 : index + %arem = arith.remui %asum, %seven : index + %aval = arith.subi %arem, %three : index + %av = arith.index_cast %aval : index to i8 + %r2 = arith.muli %row, %two : index + %c3 = arith.muli %column, %three : index + %bsum = arith.addi %r2, %c3 : index + %brem = arith.remui %bsum, %five : index + %bval = arith.subi %brem, %two : index + %bv = arith.index_cast %bval : index to i8 + memref.store %av, %a[%row, %column] : memref<16x16xi8> + memref.store %bv, %b[%row, %column] : memref<16x16xi8> + } + } + %at = bufferization.to_tensor %a restrict : memref<16x16xi8> to tensor<16x16xi8> + %bt = bufferization.to_tensor %b restrict : memref<16x16xi8> to tensor<16x16xi8> + %init = arith.constant dense<0> : tensor<16x16xi32> + %result = linalg.matmul ins(%at, %bt: tensor<16x16xi8>, tensor<16x16xi8>) outs(%init: tensor<16x16xi32>) -> tensor<16x16xi32> + %memory = bufferization.to_buffer %result : tensor<16x16xi32> to memref<16x16xi32> + func.call @check_result(%memory) : (memref<16x16xi32>) -> () + return %zero_i8 : i8 +} diff --git a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_matmul_16x16_i8_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_matmul_16x16_i8_main.cpp similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/linalg/linalg_matmul_16x16_i8_main.cpp rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_matmul_16x16_i8_main.cpp diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_mega_conv2d_k3_c16.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_mega_conv2d_k3_c16.mlir new file mode 100644 index 000000000..9bc197835 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_mega_conv2d_k3_c16.mlir @@ -0,0 +1,76 @@ +func.func private @check_result(memref<1x16x1x1xf32>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero_i32 = arith.constant 0 : i32 + %zero_f32 = arith.constant 0.0 : f32 + %one_i8 = arith.constant 1 : i8 + %one_f32 = arith.constant 1.0 : f32 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %two = arith.constant 2 : index + %three = arith.constant 3 : index + %five = arith.constant 5 : index + %eight = arith.constant 8 : index + %nine = arith.constant 9 : index + %sixteen = arith.constant 16 : index + %input = memref.alloc() : memref<1x3x3x16xi8> + %weight = memref.alloc() : memref<16x16x3x3xi8> + %bias = memref.alloc() : memref<16xi32> + linalg.fill ins(%zero_i8 : i8) outs(%input : memref<1x3x3x16xi8>) + linalg.fill ins(%zero_i8 : i8) outs(%weight : memref<16x16x3x3xi8>) + scf.for %position = %zero to %nine step %one { + %p2 = arith.muli %position, %two : index + %h = arith.divui %position, %three : index + %w = arith.remui %position, %three : index + scf.for %channel = %zero to %sixteen step %one { + %c3 = arith.muli %channel, %three : index + %isum = arith.addi %p2, %c3 : index + %irem = arith.remui %isum, %five : index + %ival = arith.subi %irem, %two : index + %iv = arith.index_cast %ival : index to i8 + memref.store %iv, %input[%zero, %h, %w, %channel] : memref<1x3x3x16xi8> + scf.for %out = %zero to %sixteen step %one { + %wsum0 = arith.addi %c3, %p2 : index + %wsum = arith.addi %wsum0, %out : index + %wrem = arith.remui %wsum, %five : index + %wval = arith.subi %wrem, %two : index + %wv = arith.index_cast %wval : index to i8 + memref.store %wv, %weight[%out, %channel, %h, %w] : memref<16x16x3x3xi8> + } + } + } + scf.for %out = %zero to %sixteen step %one { + %bval = arith.subi %out, %eight : index + %bv = arith.index_cast %bval : index to i32 + memref.store %bv, %bias[%out] : memref<16xi32> + } + %unit = arith.constant dense<1.0> : tensor<1x1x1x1xf32> + %weight_step = arith.constant dense<1.0> : tensor<16x1x1x1xf32> + %it = bufferization.to_tensor %input restrict : memref<1x3x3x16xi8> to tensor<1x3x3x16xi8> + %wt = bufferization.to_tensor %weight restrict : memref<16x16x3x3xi8> to tensor<16x16x3x3xi8> + %bt = bufferization.to_tensor %bias restrict : memref<16xi32> to tensor<16xi32> + %input_float = "frontend.int8_to_fp32"(%it, %unit) : (tensor<1x3x3x16xi8>, tensor<1x1x1x1xf32>) -> tensor<1x3x3x16xf32> + %q = "frontend.fp32_to_int8"(%input_float, %unit) : (tensor<1x3x3x16xf32>, tensor<1x1x1x1xf32>) -> tensor<1x3x3x16xi8> + %x = "frontend.int8_to_fp32"(%q, %unit) : (tensor<1x3x3x16xi8>, tensor<1x1x1x1xf32>) -> tensor<1x3x3x16xf32> + %wf = "frontend.int8_to_fp32"(%wt, %weight_step) : (tensor<16x16x3x3xi8>, tensor<16x1x1x1xf32>) -> tensor<16x16x3x3xf32> + %we = tensor.empty() : tensor<16x3x3x16xf32> + %weights = linalg.transpose ins(%wf: tensor<16x16x3x3xf32>) outs(%we: tensor<16x3x3x16xf32>) permutation = [0,2,3,1] + %scale = linalg.generic {indexing_maps = [affine_map<(f,c,h,w)->(0,c,h,w)>, affine_map<(f,c,h,w)->(f,c,h,w)>, affine_map<(f,c,h,w)->(f,c,h,w)>], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%unit, %weight_step: tensor<1x1x1x1xf32>, tensor<16x1x1x1xf32>) outs(%weight_step: tensor<16x1x1x1xf32>) { + ^bb0(%a: f32, %b: f32, %o: f32): + %r = arith.mulf %a, %b : f32 + linalg.yield %r : f32 + } -> tensor<16x1x1x1xf32> + %bs = tensor.collapse_shape %scale [[0,1,2,3]] : tensor<16x1x1x1xf32> into tensor<16xf32> + %bf = "frontend.int32_to_fp32"(%bt, %bs) : (tensor<16xi32>, tensor<16xf32>) -> tensor<16xf32> + %empty = tensor.empty() : tensor<1x1x1x16xf32> + %initial = linalg.generic {indexing_maps = [affine_map<(n,h,w,c)->(c)>, affine_map<(n,h,w,c)->(n,h,w,c)>], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%bf: tensor<16xf32>) outs(%empty: tensor<1x1x1x16xf32>) { + ^bb0(%a: f32, %o: f32): linalg.yield %a : f32 + } -> tensor<1x1x1x16xf32> + %result = linalg.conv_2d_nhwc_fhwc {strides = dense<1> : tensor<2xi64>, dilations = dense<1> : tensor<2xi64>} ins(%x, %weights: tensor<1x3x3x16xf32>, tensor<16x3x3x16xf32>) outs(%initial: tensor<1x1x1x16xf32>) -> tensor<1x1x1x16xf32> + %oe = tensor.empty() : tensor<1x16x1x1xf32> + %nchw = linalg.transpose ins(%result: tensor<1x1x1x16xf32>) outs(%oe: tensor<1x16x1x1xf32>) permutation = [0,3,1,2] + %memory = bufferization.to_buffer %nchw : tensor<1x16x1x1xf32> to memref<1x16x1x1xf32> + func.call @check_result(%memory) : (memref<1x16x1x1xf32>) -> () + return %zero_i8 : i8 +} diff --git a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_mega_conv2d_k3_c16_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_mega_conv2d_k3_c16_main.cpp similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/linalg/linalg_mega_conv2d_k3_c16_main.cpp rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_mega_conv2d_k3_c16_main.cpp diff --git a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_relu_513x17_i32.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_relu_513x17_i32.mlir similarity index 94% rename from examples/chips/pebble/workloads/mlir_tests/linalg/linalg_relu_513x17_i32.mlir rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_relu_513x17_i32.mlir index 6480fca6d..bc06544a2 100644 --- a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_relu_513x17_i32.mlir +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_relu_513x17_i32.mlir @@ -38,7 +38,5 @@ func.func @main() -> i8 { } func.call @check_result(%output) : (memref<513x17xi32>) -> () - memref.dealloc %input : memref<513x17xi32> - memref.dealloc %output : memref<513x17xi32> return %zeroI8 : i8 } diff --git a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_relu_513x17_i32_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_relu_513x17_i32_main.cpp similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/linalg/linalg_relu_513x17_i32_main.cpp rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_relu_513x17_i32_main.cpp diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_16x16_i8.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_16x16_i8.mlir new file mode 100644 index 000000000..622974fa5 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_16x16_i8.mlir @@ -0,0 +1,27 @@ +func.func private @check_result(memref<16x16xi8>) -> () + +func.func @main() -> i8 { + %zero = arith.constant 0 : i8 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c3 = arith.constant 3 : i8 + %c5 = arith.constant 5 : i8 + %c16 = arith.constant 16 : index + %input = memref.alloc() alignment = 64 : memref<16x16xi8> + scf.for %i = %c0 to %c16 step %c1 { + scf.for %j = %c0 to %c16 step %c1 { + %ii = arith.index_cast %i : index to i8 + %jj = arith.index_cast %j : index to i8 + %iv = arith.muli %ii, %c3 : i8 + %jv = arith.muli %jj, %c5 : i8 + %v = arith.addi %iv, %jv : i8 + memref.store %v, %input[%i, %j] : memref<16x16xi8> + } + } + %it = bufferization.to_tensor %input restrict : memref<16x16xi8> to tensor<16x16xi8> + %empty = tensor.empty() : tensor<16x16xi8> + %result = linalg.transpose ins(%it: tensor<16x16xi8>) outs(%empty: tensor<16x16xi8>) permutation = [1,0] + %memory = bufferization.to_buffer %result : tensor<16x16xi8> to memref<16x16xi8> + func.call @check_result(%memory) : (memref<16x16xi8>) -> () + return %zero : i8 +} diff --git a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_16x16_i8_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_16x16_i8_main.cpp similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_16x16_i8_main.cpp rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_16x16_i8_main.cpp diff --git a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_detect_1x84x100_f32.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_detect_1x84x100_f32.mlir similarity index 91% rename from examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_detect_1x84x100_f32.mlir rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_detect_1x84x100_f32.mlir index 54fb4af68..3008f81eb 100644 --- a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_detect_1x84x100_f32.mlir +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_detect_1x84x100_f32.mlir @@ -24,7 +24,5 @@ func.func @main() -> i8 { linalg.transpose ins(%in : memref<1x84x100xf32>) outs(%out : memref<1x100x84xf32>) permutation = [0, 2, 1] func.call @check_result(%out) : (memref<1x100x84xf32>) -> () - memref.dealloc %in : memref<1x84x100xf32> - memref.dealloc %out : memref<1x100x84xf32> return %z : i8 } diff --git a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_detect_1x84x100_f32_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_detect_1x84x100_f32_main.cpp similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_detect_1x84x100_f32_main.cpp rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_detect_1x84x100_f32_main.cpp diff --git a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_nchw_nhwc_1x3x8x8_f32.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_nchw_nhwc_1x3x8x8_f32.mlir similarity index 92% rename from examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_nchw_nhwc_1x3x8x8_f32.mlir rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_nchw_nhwc_1x3x8x8_f32.mlir index 434d754d4..0a467830c 100644 --- a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_nchw_nhwc_1x3x8x8_f32.mlir +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_nchw_nhwc_1x3x8x8_f32.mlir @@ -28,7 +28,5 @@ func.func @main() -> i8 { linalg.transpose ins(%in : memref<1x3x8x8xf32>) outs(%out : memref<1x8x8x3xf32>) permutation = [0, 2, 3, 1] func.call @check_result(%out) : (memref<1x8x8x3xf32>) -> () - memref.dealloc %in : memref<1x3x8x8xf32> - memref.dealloc %out : memref<1x8x8x3xf32> return %z : i8 } diff --git a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_nchw_nhwc_1x3x8x8_f32_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_nchw_nhwc_1x3x8x8_f32_main.cpp similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_nchw_nhwc_1x3x8x8_f32_main.cpp rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_nchw_nhwc_1x3x8x8_f32_main.cpp diff --git a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_oihw_ohwi_4x8x3x3_f32.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_oihw_ohwi_4x8x3x3_f32.mlir similarity index 94% rename from examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_oihw_ohwi_4x8x3x3_f32.mlir rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_oihw_ohwi_4x8x3x3_f32.mlir index c9799b70d..1c5ceb748 100644 --- a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_oihw_ohwi_4x8x3x3_f32.mlir +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_oihw_ohwi_4x8x3x3_f32.mlir @@ -35,7 +35,5 @@ func.func @main() -> i8 { linalg.transpose ins(%in : memref<4x8x3x3xf32>) outs(%out : memref<4x3x3x8xf32>) permutation = [0, 2, 3, 1] func.call @check_result(%out) : (memref<4x3x3x8xf32>) -> () - memref.dealloc %in : memref<4x8x3x3xf32> - memref.dealloc %out : memref<4x3x3x8xf32> return %z : i8 } diff --git a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_oihw_ohwi_4x8x3x3_f32_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_oihw_ohwi_4x8x3x3_f32_main.cpp similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_oihw_ohwi_4x8x3x3_f32_main.cpp rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_oihw_ohwi_4x8x3x3_f32_main.cpp diff --git a/examples/designs/poly/arch/build.sc b/examples/designs/poly/arch/build.sc new file mode 100644 index 000000000..d983891ce --- /dev/null +++ b/examples/designs/poly/arch/build.sc @@ -0,0 +1,19 @@ +import mill._ +import mill.scalalib._ +import $file.^.^.^.^.arch.src.main.scala.framework.device.cbus.{build => cbus_build} +import $file.^.^.^.{build => examples} +import $file.^.^.^.prefab.cores.reduce.arch.{build => reduce} + +val designRoot: os.Path = millSourcePath / os.up + +object design extends SbtModule { + override def generatedSources = T(cbus_build.registry(sources() ++ reduce.library.sources(), T.dest)) + override def millSourcePath = designRoot + override def forkWorkingDir = T(designRoot) + override def scalaVersion = "2.13.16" + override def moduleDeps = Seq(examples.library, reduce.library) + override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") + override def sources = T.sources(designRoot / "arch", designRoot / "configs") +} diff --git a/examples/designs/poly/arch/device/DeviceTop.scala b/examples/designs/poly/arch/device/DeviceTop.scala new file mode 100644 index 000000000..8f9bb9765 --- /dev/null +++ b/examples/designs/poly/arch/device/DeviceTop.scala @@ -0,0 +1,90 @@ +package examples.designs.poly + +import framework.lib.arbiter.{RRArbiter} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, Instance, Instantiate} +import framework.core.configs.CoreParam +import framework.device.configs.DeviceParam +import framework.device.cbus.CBus +import framework.device.cbus.generated.CoreComponents +import buckyball.config.Design +import _root_.configs.GlobalConfig +import framework.core.clink.CLinkIO +import framework.core.memory.home.Home +import framework.device.ddr.Bridge +import framework.device.peripherals.Controller +import framework.device.peripherals.bootrom.BootRomLines +import framework.device.dlink.BaseDevice +import framework.ip.bus.axi4 +import framework.ip.bus.chi.snf.LineResponse +import framework.lib.queue.{Queue => ResponseQueue} + +/** Configured cores share the device coherence fabric. */ +@instantiable +class DeviceTop(b: GlobalConfig, design: Design) extends BaseDevice(b.device.ddr.axi) { + val device = b.device + val cores = device.cores + require(cores.forall(p => !p.clink.backing && p.clink.requesters == 3)) + val sharedHome = cores.head.memory.home + require( + cores.forall(_.memory.memoryAccess.memory.chi == sharedHome.memory.chi), + "Core coherent channels must match the shared Home" + ) + require(cores.forall(_.memory.home == sharedHome) && sharedHome.memory.agents == cores.size * 3) + + val bus: Instance[CBus] = + Instantiate(new CBus(b.device, CBus.generators(design, b.device, b.sim, CoreComponents.factories))) + val home: Instance[Home] = Instantiate(new Home(sharedHome)) + val peripherals: Instance[Controller] = + Instantiate(new Controller(device, device.ddr.axi, cores.head.memory.memoryAccess.memory)) + + val peripheralBus: Instance[axi4.Decoder] = Instantiate(new axi4.Decoder( + Seq( + (device.clint.base, device.clint.bytes), + (device.plic.base, device.plic.bytes), + (device.scu.baseAddress, device.scu.totalSizeBytes) + ), + device.ddr.axi + )) + + val rom: Instance[BootRomLines] = Instantiate(new BootRomLines(device.bootrom, sharedHome.memory.chi)) + val bridge: Instance[Bridge] = Instantiate(new Bridge(device.ddr)) + val responses: Instance[RRArbiter[LineResponse]] = + Instantiate(new RRArbiter(new LineResponse(sharedHome.memory.chi), 2)) + val response: Instance[ResponseQueue[LineResponse]] = + Instantiate(new ResponseQueue(new LineResponse(sharedHome.memory.chi), 1, false, false)) + val memoryBus: Instance[axi4.Interconnect] = + Instantiate(new axi4.Interconnect(axi4.InterconnectParam(device.ddr.axi, 2, None))) + + home.io.active := true.B + home.io.blockRequesterRsp := false.B + home.io.requesters <> bus.coherent + + peripheralBus.io.in <> bus.uncached + peripherals.io.axi <> peripheralBus.io.matched + bus.external <> peripheralBus.io.other + peripherals.io.hardwareInterrupt.sources := 0.U + bus.interrupts := peripherals.io.hardwareInterrupt.harts + + require(device.ddr.clients == 1) + val request = home.io.memoryReq + val inRom = request.bits.addr >= device.bootrom.base.U && + request.bits.addr < (device.bootrom.base + device.bootrom.bytes).U + rom.io.request.valid := request.valid && inRom + rom.io.request.bits := request.bits + bridge.io.request(0).valid := request.valid && !inRom + bridge.io.request(0).bits := request.bits + request.ready := Mux(inRom, rom.io.request.ready, bridge.io.request(0).ready) + + responses.io.in(0) <> rom.io.response + responses.io.in(1) <> bridge.io.response(0) + + response.io.enq <> responses.io.out + home.io.memoryResp <> response.io.deq + + memoryBus.io.in(0) <> bus.dlink.axi + memoryBus.io.in(1) <> bridge.io.axi + dlink.axi <> memoryBus.io.out +} diff --git a/examples/designs/poly/arch/device/core/PolyCore.scala b/examples/designs/poly/arch/device/core/PolyCore.scala new file mode 100644 index 000000000..27e1c6f24 --- /dev/null +++ b/examples/designs/poly/arch/device/core/PolyCore.scala @@ -0,0 +1,154 @@ +package examples.designs.poly + +import framework.core.tbus.generated.ThreadComponents +import sims.configs.SimParam +import framework.device.cbus.CoreFactory +import buckyball.config.{CorePlacement, Design} + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.core.configs.CoreParam +import framework.core.tbus.TBus +import framework.ip.bus.chi.rnf.AxiCache +import framework.core.execution.Execution +import framework.core.memory.Memory +import framework.core.tss.Store +import framework.core.shm.SHM +import framework.core.controller.Controller +import framework.core.clink.{CLinkIO, HasCLink} + +@instantiable +class PolyCore(cp: CoreParam, simp: SimParam, threadTraceIds: Seq[Int]) extends Module with HasCLink { + + val diffTest = simp.diffTest + require( + cp.clink.harts == 1 && + ((cp.clink.backing && cp.clink.requesters == 0) || + (!cp.clink.backing && cp.clink.requesters == 2 + cp.dmaCache.size)), + "PolyCore requires one hart and a matching cache interface" + ) + if (!cp.clink.backing) { + require( + cp.dmaCache.nonEmpty == cp.threads.nonEmpty, + "Thread DMA requires its own coherent cache requester" + ) + } + if (cp.threads.nonEmpty) { + if (cp.clink.shared) + require(cp.clink.c2c.contains(cp.c2c.axi), "C2C protocol must match its CLink channel") + val tracking = cp.memory.dcache.tracking + cp.threads.foreach { thread => + val transport = thread.acc.get.transport + require(transport.axi == cp.interconnect.bus, "Thread DMA protocol must match the core DDR interconnect") + require( + transport.prepared.bus.addressBits == cp.memory.memoryAccess.memory.chi.addressBits, + "Thread page-table address width must match the core memory service" + ) + require( + ( + transport.interlock.addressBits, + transport.interlock.idBits, + transport.interlock.lineBytes, + transport.interlock.maxRanges + ) == + (tracking.addressBits, tracking.idBits, tracking.lineBytes, tracking.maxRanges), + "Thread cache-maintenance interface must match the core memory service" + ) + } + } + + @public + val clink = IO(new CLinkIO(cp.clink)) + + val execution: Instance[Execution] = + Instantiate(new Execution(cp.execution(directAccelerator = false), localRocc = false)) + val controller: Instance[Controller] = Instantiate(new Controller(cp.controller)) + val memory: Instance[Memory] = + Instantiate(new Memory(cp.memory, externalHome = !cp.clink.backing)) + + execution.io.interrupts := clink.interrupts(0) + clink.memory.backing.zip(memory.io.backing).foreach { case (port, backing) => port <> backing } + clink.memory.coherent.take(2).zip(memory.io.coherent).foreach { case (port, cache) => port <> cache } + clink.memory.uncached <> memory.io.uncached + execution.io.workDrained := controller.io.drained + controller.io.execution <> execution.io.controller.get + memory.io.fetch <> execution.io.fetch + memory.io.data <> execution.io.data + memory.io.timeRequest <> execution.io.timeRequest + execution.io.timeResponse <> memory.io.timeResponse + memory.io.commandAllowed := controller.io.memoryAllowed + + if (cp.threads.nonEmpty) { + val tbus: Instance[TBus] = + Instantiate(new TBus( + cp, + simp, + threadTraceIds, + TBus.generators(cp, simp, threadTraceIds, ThreadComponents.factories) + )) + tbus.io.shm.foreach { port => + val shm: Instance[SHM] = Instantiate(new SHM(cp.sharedFrontend, cp.sharedMemory, cp.tbus.get)) + shm.io <> port + } + tbus.io.tss.foreach { port => + val tss: Instance[Store] = Instantiate(new Store(cp.tss.get)) + tss.io.clients <> port + tss.io.host <> controller.io.tss.get.host + tss.io.inUse := controller.io.tss.get.inUse + tss.io.cancel := controller.io.tss.get.cancel + controller.io.tss.get.busy := tss.io.busy + controller.io.tss.get.hostBusy := tss.io.hostBusy + } + if (cp.clink.backing) { + require(cp.interconnect.bus == cp.clink.ddr, "Thread DDR interconnect must match CLink") + clink.memory.ddr <> tbus.io.ddr + } else { + val dma = Instantiate(new AxiCache(cp.interconnect.bus, cp.dmaCache.get)) + dma.io.axi <> tbus.io.ddr + dma.io.axi.ar.bits.cache := 15.U + dma.io.axi.aw.bits.cache := 15.U + clink.memory.coherent(2) <> dma.io.chi + clink.memory.ddr.aw.valid := false.B + clink.memory.ddr.aw.bits := 0.U.asTypeOf(clink.memory.ddr.aw.bits) + clink.memory.ddr.w.valid := false.B + clink.memory.ddr.w.bits := 0.U.asTypeOf(clink.memory.ddr.w.bits) + clink.memory.ddr.ar.valid := false.B + clink.memory.ddr.ar.bits := 0.U.asTypeOf(clink.memory.ddr.ar.bits) + clink.memory.ddr.b.ready := false.B + clink.memory.ddr.r.ready := false.B + } + clink.c2c.zip(tbus.io.c2c).foreach { case (link, port) => link <> port } + tbus.io.control.zip(controller.io.threads).foreach { case (port, control) => port <> control } + tbus.io.remote.zip(controller.io.remote).foreach { case (port, remote) => port <> remote } + tbus.io.move <> controller.io.move + tbus.io.memory.zip(memory.io.threads).foreach { case (port, service) => port <> service } + } else { + require(!cp.clink.shared, "A core without a shared bank network must not advertise a C2C endpoint") + clink.memory.ddr.aw.valid := false.B + clink.memory.ddr.aw.bits := 0.U.asTypeOf(clink.memory.ddr.aw.bits) + clink.memory.ddr.w.valid := false.B + clink.memory.ddr.w.bits := 0.U.asTypeOf(clink.memory.ddr.w.bits) + clink.memory.ddr.ar.valid := false.B + clink.memory.ddr.ar.bits := 0.U.asTypeOf(clink.memory.ddr.ar.bits) + clink.memory.ddr.b.ready := false.B + clink.memory.ddr.r.ready := false.B + controller.io.move.command.ready := false.B + controller.io.move.completion.valid := false.B + controller.io.move.completion.bits := false.B + } +} + +object Poly extends CoreFactory { + val component = "poly" + def parameters(design: Design, source: CorePlacement): CoreParam = + CoreParam.fromProto(design, source, externalHome = true) + + def instantiate( + source: CorePlacement, + corep: CoreParam, + simp: SimParam, + traceIds: Seq[Int] + ): CLinkIO = + Instantiate(new PolyCore(corep, simp, traceIds)).clink + +} diff --git a/examples/designs/poly/configs/design.toml b/examples/designs/poly/configs/design.toml new file mode 100644 index 000000000..1d45ec7e4 --- /dev/null +++ b/examples/designs/poly/configs/design.toml @@ -0,0 +1,5 @@ +[device] +include = "device/default.toml" + +[sims] +include = "sims/default.toml" diff --git a/examples/designs/poly/configs/device/clint.toml b/examples/designs/poly/configs/device/clint.toml new file mode 100644 index 000000000..c94fc18d8 --- /dev/null +++ b/examples/designs/poly/configs/device/clint.toml @@ -0,0 +1,4 @@ +base = 0x2000000 +bytes = 0x10000 +tickCycles = 1000 +hartIds = [0, 1, 2, 3, 4, 5, 6, 7, 8] diff --git a/examples/designs/poly/configs/device/cores/core0/acc/balldomain/default.toml b/examples/designs/poly/configs/device/cores/core0/acc/balldomain/default.toml new file mode 100644 index 000000000..ab2226811 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core0/acc/balldomain/default.toml @@ -0,0 +1,9 @@ +[[balls]] +id = 0 +name = "F32AddBall" +component = "f32add" +param = "default" +port = { inBW = 2, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "F32ADD", funct7 = 76 }, +] diff --git a/examples/designs/poly/configs/device/cores/core0/acc/default.toml b/examples/designs/poly/configs/device/cores/core0/acc/default.toml new file mode 100644 index 000000000..0099d6438 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core0/acc/default.toml @@ -0,0 +1,6 @@ +source = "../../../../.." +balldomain = "balldomain/default.toml" +memdomain = "memdomain/poly.toml" +frontend = "frontend/poly.toml" +gpdomain = "default" +transport = "../../core1/transport.toml" diff --git a/examples/designs/poly/configs/device/cores/core0/acc/frontend/poly.toml b/examples/designs/poly/configs/device/cores/core0/acc/frontend/poly.toml new file mode 100644 index 000000000..5464f2d95 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core0/acc/frontend/poly.toml @@ -0,0 +1,9 @@ +bankAliasEnable = true +robEntries = 16 +rsOutOfOrderResponse = true +bankIdLen = 10 +maxPrivateVbank = 15 +sharedVbankBase = 32 +iterLen = 34 +subRobEnable = false +subRobDepth = 64 diff --git a/examples/designs/poly/configs/device/cores/core0/acc/memdomain/poly.toml b/examples/designs/poly/configs/device/cores/core0/acc/memdomain/poly.toml new file mode 100644 index 000000000..63a9fc482 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core0/acc/memdomain/poly.toml @@ -0,0 +1,19 @@ +addrLen = 39 + +[bank] +layoutEnable = true +num = 4 +width = 128 +entries = 1024 +maskLen = 16 +requestPorts = 3 + +[dma] +busWidth = 128 + +[mmio] +enable = false +bankNum = 1 +bankEntries = 1 +bankWidth = 128 +readWidth = 8 diff --git a/examples/designs/poly/configs/device/cores/core0/c2c/core.toml b/examples/designs/poly/configs/device/cores/core0/c2c/core.toml new file mode 100644 index 000000000..d34ac9d9b --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core0/c2c/core.toml @@ -0,0 +1,3 @@ +enabled = true +peerCores = 9 +axi = "default" diff --git a/examples/designs/poly/configs/device/cores/core0/controller/core.toml b/examples/designs/poly/configs/device/cores/core0/controller/core.toml new file mode 100644 index 000000000..87f38f33c --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core0/controller/core.toml @@ -0,0 +1 @@ +moves = true diff --git a/examples/designs/poly/configs/device/cores/core0/default.toml b/examples/designs/poly/configs/device/cores/core0/default.toml new file mode 100644 index 000000000..e6731ebbc --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core0/default.toml @@ -0,0 +1,37 @@ +component = "reduce" +accelerator = "acc/default.toml" +threads = [] +hartId = 0 +resetVector = 0x10000 +mtimeAddress = 0x200bff8 +dmaCache = "dmaCache/core.toml" +memory = "memory/core.toml" +fetch = "default" +fpu = "default" +interconnect = "interconnect/core.toml" +c2c = "c2c/core.toml" +tbus = "tbus/core.toml" +controller = "controller/core.toml" + +[param] +threadDataBytes = 64 +xLen = 64 +vaddrBits = 39 +paddrBits = 36 +pgIdxBits = 12 +pgLevels = 3 +nPMPs = 16 + +[sharedMem] +bankWidth = 128 +bankEntries = 1024 +enable = true +virtualBankCount = 40 +entries = 16384 +inputChannels = 5 +defaultGroupCount = 1 +latency = 10 + +[compiler] +name = "main" +component = "reduce" diff --git a/examples/designs/poly/configs/device/cores/core0/dmaCache/core.toml b/examples/designs/poly/configs/device/cores/core0/dmaCache/core.toml new file mode 100644 index 000000000..859ae7561 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core0/dmaCache/core.toml @@ -0,0 +1,7 @@ +nodeId = 3 +lines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = true +probe = false diff --git a/examples/designs/poly/configs/device/cores/core0/interconnect/core.toml b/examples/designs/poly/configs/device/cores/core0/interconnect/core.toml new file mode 100644 index 000000000..13d04a5a6 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core0/interconnect/core.toml @@ -0,0 +1,2 @@ +axi = "default" +slots = 16 diff --git a/examples/designs/poly/configs/device/cores/core0/memory/access/core.toml b/examples/designs/poly/configs/device/cores/core0/memory/access/core.toml new file mode 100644 index 000000000..abd42a128 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core0/memory/access/core.toml @@ -0,0 +1,52 @@ +translationEntries = 16 +tagBits = 6 +resultLineBits = 0 +[[regions]] +base = 0x80000000 +bytes = 0x400000000 +cacheable = true +executable = true +readable = true +writable = true +atomic = true +normal = true + +[[regions]] +base = 0x10000 +bytes = 0x10000 +cacheable = true +executable = true +readable = true +writable = false +atomic = false +normal = true + +[[regions]] +base = 0x2000000 +bytes = 0x10000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0xc000000 +bytes = 0x4000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0x60000000 +bytes = 0x10000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false diff --git a/examples/designs/poly/configs/device/cores/core0/memory/core.toml b/examples/designs/poly/configs/device/cores/core0/memory/core.toml new file mode 100644 index 000000000..3caf53579 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core0/memory/core.toml @@ -0,0 +1,5 @@ +uncached = "default" +access = "access/core.toml" +icache = "icache/core.toml" +dcache = "dcache/core.toml" +home = "home/core.toml" diff --git a/examples/designs/poly/configs/device/cores/core0/memory/dcache/core.toml b/examples/designs/poly/configs/device/cores/core0/memory/dcache/core.toml new file mode 100644 index 000000000..4fb4beb6f --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core0/memory/dcache/core.toml @@ -0,0 +1,15 @@ +[tracking] +entries = 8 +addressBits = 44 +idBits = 8 +lineBytes = 64 +maxRanges = 8 + +[cache] +nodeId = 2 +lines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/poly/configs/device/cores/core0/memory/home/core.toml b/examples/designs/poly/configs/device/cores/core0/memory/home/core.toml new file mode 100644 index 000000000..0b7f0ff39 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core0/memory/home/core.toml @@ -0,0 +1,13 @@ +agents = 27 +mshrEntries = 8 +nodeId = 64 +requesters = "default" + +[cache] +addressBits = 44 +lineBytes = 64 +sets = 64 +ways = 4 +idBits = 8 +metadataBits = 4 +responseDepth = 2 diff --git a/examples/designs/poly/configs/device/cores/core0/memory/icache/core.toml b/examples/designs/poly/configs/device/cores/core0/memory/icache/core.toml new file mode 100644 index 000000000..3cc7df83a --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core0/memory/icache/core.toml @@ -0,0 +1,7 @@ +nodeId = 1 +lines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/poly/configs/device/cores/core0/tbus/core.toml b/examples/designs/poly/configs/device/cores/core0/tbus/core.toml new file mode 100644 index 000000000..36bdb8cfc --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core0/tbus/core.toml @@ -0,0 +1 @@ +bankNodes = [0] diff --git a/examples/designs/poly/configs/device/cores/core1/c2c/core.toml b/examples/designs/poly/configs/device/cores/core1/c2c/core.toml new file mode 100644 index 000000000..d34ac9d9b --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core1/c2c/core.toml @@ -0,0 +1,3 @@ +enabled = true +peerCores = 9 +axi = "default" diff --git a/examples/designs/poly/configs/device/cores/core1/controller/core.toml b/examples/designs/poly/configs/device/cores/core1/controller/core.toml new file mode 100644 index 000000000..87f38f33c --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core1/controller/core.toml @@ -0,0 +1 @@ +moves = true diff --git a/examples/designs/poly/configs/device/cores/core1/default.toml b/examples/designs/poly/configs/device/cores/core1/default.toml new file mode 100644 index 000000000..011971006 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core1/default.toml @@ -0,0 +1,71 @@ +component = "poly" +hartId = 1 +resetVector = 0x10000 +mtimeAddress = 0x200bff8 +dmaCache = "dmaCache/core.toml" +memory = "memory/core.toml" +fetch = "default" +fpu = "default" +interconnect = "interconnect/core.toml" +c2c = "c2c/core.toml" +tbus = "tbus/core.toml" +controller = "controller/core.toml" + +[param] +threadDataBytes = 64 +xLen = 64 +vaddrBits = 39 +paddrBits = 36 +pgIdxBits = 12 +pgLevels = 3 +nPMPs = 16 + +[tss] +base = 0x20000 +bytes = 0x4000 +dataBits = 128 + +[sharedMem] +bankWidth = 128 +bankEntries = 1024 +enable = true +virtualBankCount = 40 +entries = 16384 +inputChannels = 65 +defaultGroupCount = 1 +latency = 10 + +[[threads]] +id = 0 +name = "attention0" +component = "attention" +transport = "transport.toml" +include = "../../../../../../prefab/threads/attention/configs/default.toml" + +[[threads]] +id = 1 +name = "attention1" +component = "attention" +transport = "transport.toml" +include = "../../../../../../prefab/threads/attention/configs/default.toml" + +[[threads]] +id = 2 +name = "ffn0" +component = "ffn" +transport = "transport.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" + +[[threads]] +id = 3 +name = "ffn1" +component = "ffn" +transport = "transport.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" + +[[threads]] +id = 4 +name = "ffn2" +component = "ffn" +transport = "transport.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" diff --git a/examples/designs/poly/configs/device/cores/core1/dmaCache/core.toml b/examples/designs/poly/configs/device/cores/core1/dmaCache/core.toml new file mode 100644 index 000000000..9b6a1a19a --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core1/dmaCache/core.toml @@ -0,0 +1,7 @@ +nodeId = 6 +lines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = true +probe = false diff --git a/examples/designs/poly/configs/device/cores/core1/interconnect/core.toml b/examples/designs/poly/configs/device/cores/core1/interconnect/core.toml new file mode 100644 index 000000000..13d04a5a6 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core1/interconnect/core.toml @@ -0,0 +1,2 @@ +axi = "default" +slots = 16 diff --git a/examples/designs/poly/configs/device/cores/core1/memory/access/core.toml b/examples/designs/poly/configs/device/cores/core1/memory/access/core.toml new file mode 100644 index 000000000..abd42a128 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core1/memory/access/core.toml @@ -0,0 +1,52 @@ +translationEntries = 16 +tagBits = 6 +resultLineBits = 0 +[[regions]] +base = 0x80000000 +bytes = 0x400000000 +cacheable = true +executable = true +readable = true +writable = true +atomic = true +normal = true + +[[regions]] +base = 0x10000 +bytes = 0x10000 +cacheable = true +executable = true +readable = true +writable = false +atomic = false +normal = true + +[[regions]] +base = 0x2000000 +bytes = 0x10000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0xc000000 +bytes = 0x4000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0x60000000 +bytes = 0x10000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false diff --git a/examples/designs/poly/configs/device/cores/core1/memory/core.toml b/examples/designs/poly/configs/device/cores/core1/memory/core.toml new file mode 100644 index 000000000..3caf53579 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core1/memory/core.toml @@ -0,0 +1,5 @@ +uncached = "default" +access = "access/core.toml" +icache = "icache/core.toml" +dcache = "dcache/core.toml" +home = "home/core.toml" diff --git a/examples/designs/poly/configs/device/cores/core1/memory/dcache/core.toml b/examples/designs/poly/configs/device/cores/core1/memory/dcache/core.toml new file mode 100644 index 000000000..2d34f225a --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core1/memory/dcache/core.toml @@ -0,0 +1,15 @@ +[tracking] +entries = 8 +addressBits = 44 +idBits = 8 +lineBytes = 64 +maxRanges = 8 + +[cache] +nodeId = 5 +lines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/poly/configs/device/cores/core1/memory/home/core.toml b/examples/designs/poly/configs/device/cores/core1/memory/home/core.toml new file mode 100644 index 000000000..0b7f0ff39 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core1/memory/home/core.toml @@ -0,0 +1,13 @@ +agents = 27 +mshrEntries = 8 +nodeId = 64 +requesters = "default" + +[cache] +addressBits = 44 +lineBytes = 64 +sets = 64 +ways = 4 +idBits = 8 +metadataBits = 4 +responseDepth = 2 diff --git a/examples/designs/poly/configs/device/cores/core1/memory/icache/core.toml b/examples/designs/poly/configs/device/cores/core1/memory/icache/core.toml new file mode 100644 index 000000000..d35c0e05b --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core1/memory/icache/core.toml @@ -0,0 +1,7 @@ +nodeId = 4 +lines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/poly/configs/device/cores/core1/tbus/core.toml b/examples/designs/poly/configs/device/cores/core1/tbus/core.toml new file mode 100644 index 000000000..81037e7d9 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core1/tbus/core.toml @@ -0,0 +1 @@ +bankNodes = [0, 1, 2, 3, 4] diff --git a/examples/designs/poly/configs/device/cores/core1/transport.toml b/examples/designs/poly/configs/device/cores/core1/transport.toml new file mode 100644 index 000000000..cb0b0cd17 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core1/transport.toml @@ -0,0 +1,82 @@ +# Per-thread DMA transport values follow the existing Pebble configuration. + +[tracking] +entries = 24 +addressBits = 44 +idBits = 8 +lineBytes = 64 +maxRanges = 8 + + +[prepared] +contexts = 24 +maxRanges = 8 +idBits = 8 +beatBytes = 16 + +[prepared.bus] +nodeIdBits = 7 +addressBits = 44 +dataBits = 256 + + + +[axi] +addressBits = 44 +dataBits = 128 +idBits = 4 + + +[[regions]] +base = 0x80000000 +bytes = 0x400000000 +cacheable = true +executable = true +readable = true +writable = true +atomic = true +normal = true + + +[[regions]] +base = 0x10000 +bytes = 0x10000 +cacheable = true +executable = true +readable = true +writable = false +atomic = false +normal = true + + +[[regions]] +base = 0x2000000 +bytes = 0x10000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + + +[[regions]] +base = 0xc000000 +bytes = 0x4000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + + +[[regions]] +base = 0x60000000 +bytes = 0x10000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false diff --git a/examples/designs/poly/configs/device/cores/core2/c2c/core.toml b/examples/designs/poly/configs/device/cores/core2/c2c/core.toml new file mode 100644 index 000000000..d34ac9d9b --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core2/c2c/core.toml @@ -0,0 +1,3 @@ +enabled = true +peerCores = 9 +axi = "default" diff --git a/examples/designs/poly/configs/device/cores/core2/controller/core.toml b/examples/designs/poly/configs/device/cores/core2/controller/core.toml new file mode 100644 index 000000000..87f38f33c --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core2/controller/core.toml @@ -0,0 +1 @@ +moves = true diff --git a/examples/designs/poly/configs/device/cores/core2/default.toml b/examples/designs/poly/configs/device/cores/core2/default.toml new file mode 100644 index 000000000..3a3ced771 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core2/default.toml @@ -0,0 +1,71 @@ +component = "poly" +hartId = 2 +resetVector = 0x10000 +mtimeAddress = 0x200bff8 +dmaCache = "dmaCache/core.toml" +memory = "memory/core.toml" +fetch = "default" +fpu = "default" +interconnect = "interconnect/core.toml" +c2c = "c2c/core.toml" +tbus = "tbus/core.toml" +controller = "controller/core.toml" + +[param] +threadDataBytes = 64 +xLen = 64 +vaddrBits = 39 +paddrBits = 36 +pgIdxBits = 12 +pgLevels = 3 +nPMPs = 16 + +[tss] +base = 0x20000 +bytes = 0x4000 +dataBits = 128 + +[sharedMem] +bankWidth = 128 +bankEntries = 1024 +enable = true +virtualBankCount = 40 +entries = 16384 +inputChannels = 65 +defaultGroupCount = 1 +latency = 10 + +[[threads]] +id = 0 +name = "attention0" +component = "attention" +transport = "../core1/transport.toml" +include = "../../../../../../prefab/threads/attention/configs/default.toml" + +[[threads]] +id = 1 +name = "attention1" +component = "attention" +transport = "../core1/transport.toml" +include = "../../../../../../prefab/threads/attention/configs/default.toml" + +[[threads]] +id = 2 +name = "ffn0" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" + +[[threads]] +id = 3 +name = "ffn1" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" + +[[threads]] +id = 4 +name = "ffn2" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" diff --git a/examples/designs/poly/configs/device/cores/core2/dmaCache/core.toml b/examples/designs/poly/configs/device/cores/core2/dmaCache/core.toml new file mode 100644 index 000000000..61d2b253d --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core2/dmaCache/core.toml @@ -0,0 +1,7 @@ +nodeId = 9 +lines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = true +probe = false diff --git a/examples/designs/poly/configs/device/cores/core2/interconnect/core.toml b/examples/designs/poly/configs/device/cores/core2/interconnect/core.toml new file mode 100644 index 000000000..13d04a5a6 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core2/interconnect/core.toml @@ -0,0 +1,2 @@ +axi = "default" +slots = 16 diff --git a/examples/designs/poly/configs/device/cores/core2/memory/access/core.toml b/examples/designs/poly/configs/device/cores/core2/memory/access/core.toml new file mode 100644 index 000000000..abd42a128 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core2/memory/access/core.toml @@ -0,0 +1,52 @@ +translationEntries = 16 +tagBits = 6 +resultLineBits = 0 +[[regions]] +base = 0x80000000 +bytes = 0x400000000 +cacheable = true +executable = true +readable = true +writable = true +atomic = true +normal = true + +[[regions]] +base = 0x10000 +bytes = 0x10000 +cacheable = true +executable = true +readable = true +writable = false +atomic = false +normal = true + +[[regions]] +base = 0x2000000 +bytes = 0x10000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0xc000000 +bytes = 0x4000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0x60000000 +bytes = 0x10000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false diff --git a/examples/designs/poly/configs/device/cores/core2/memory/core.toml b/examples/designs/poly/configs/device/cores/core2/memory/core.toml new file mode 100644 index 000000000..3caf53579 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core2/memory/core.toml @@ -0,0 +1,5 @@ +uncached = "default" +access = "access/core.toml" +icache = "icache/core.toml" +dcache = "dcache/core.toml" +home = "home/core.toml" diff --git a/examples/designs/poly/configs/device/cores/core2/memory/dcache/core.toml b/examples/designs/poly/configs/device/cores/core2/memory/dcache/core.toml new file mode 100644 index 000000000..e16221d82 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core2/memory/dcache/core.toml @@ -0,0 +1,15 @@ +[tracking] +entries = 8 +addressBits = 44 +idBits = 8 +lineBytes = 64 +maxRanges = 8 + +[cache] +nodeId = 8 +lines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/poly/configs/device/cores/core2/memory/home/core.toml b/examples/designs/poly/configs/device/cores/core2/memory/home/core.toml new file mode 100644 index 000000000..0b7f0ff39 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core2/memory/home/core.toml @@ -0,0 +1,13 @@ +agents = 27 +mshrEntries = 8 +nodeId = 64 +requesters = "default" + +[cache] +addressBits = 44 +lineBytes = 64 +sets = 64 +ways = 4 +idBits = 8 +metadataBits = 4 +responseDepth = 2 diff --git a/examples/designs/poly/configs/device/cores/core2/memory/icache/core.toml b/examples/designs/poly/configs/device/cores/core2/memory/icache/core.toml new file mode 100644 index 000000000..804e53c8c --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core2/memory/icache/core.toml @@ -0,0 +1,7 @@ +nodeId = 7 +lines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/poly/configs/device/cores/core2/tbus/core.toml b/examples/designs/poly/configs/device/cores/core2/tbus/core.toml new file mode 100644 index 000000000..81037e7d9 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core2/tbus/core.toml @@ -0,0 +1 @@ +bankNodes = [0, 1, 2, 3, 4] diff --git a/examples/designs/poly/configs/device/cores/core3/c2c/core.toml b/examples/designs/poly/configs/device/cores/core3/c2c/core.toml new file mode 100644 index 000000000..d34ac9d9b --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core3/c2c/core.toml @@ -0,0 +1,3 @@ +enabled = true +peerCores = 9 +axi = "default" diff --git a/examples/designs/poly/configs/device/cores/core3/controller/core.toml b/examples/designs/poly/configs/device/cores/core3/controller/core.toml new file mode 100644 index 000000000..87f38f33c --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core3/controller/core.toml @@ -0,0 +1 @@ +moves = true diff --git a/examples/designs/poly/configs/device/cores/core3/default.toml b/examples/designs/poly/configs/device/cores/core3/default.toml new file mode 100644 index 000000000..d960c27a9 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core3/default.toml @@ -0,0 +1,71 @@ +component = "poly" +hartId = 3 +resetVector = 0x10000 +mtimeAddress = 0x200bff8 +dmaCache = "dmaCache/core.toml" +memory = "memory/core.toml" +fetch = "default" +fpu = "default" +interconnect = "interconnect/core.toml" +c2c = "c2c/core.toml" +tbus = "tbus/core.toml" +controller = "controller/core.toml" + +[param] +threadDataBytes = 64 +xLen = 64 +vaddrBits = 39 +paddrBits = 36 +pgIdxBits = 12 +pgLevels = 3 +nPMPs = 16 + +[tss] +base = 0x20000 +bytes = 0x4000 +dataBits = 128 + +[sharedMem] +bankWidth = 128 +bankEntries = 1024 +enable = true +virtualBankCount = 40 +entries = 16384 +inputChannels = 65 +defaultGroupCount = 1 +latency = 10 + +[[threads]] +id = 0 +name = "attention0" +component = "attention" +transport = "../core1/transport.toml" +include = "../../../../../../prefab/threads/attention/configs/default.toml" + +[[threads]] +id = 1 +name = "attention1" +component = "attention" +transport = "../core1/transport.toml" +include = "../../../../../../prefab/threads/attention/configs/default.toml" + +[[threads]] +id = 2 +name = "ffn0" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" + +[[threads]] +id = 3 +name = "ffn1" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" + +[[threads]] +id = 4 +name = "ffn2" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" diff --git a/examples/designs/poly/configs/device/cores/core3/dmaCache/core.toml b/examples/designs/poly/configs/device/cores/core3/dmaCache/core.toml new file mode 100644 index 000000000..d005f7cfe --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core3/dmaCache/core.toml @@ -0,0 +1,7 @@ +nodeId = 12 +lines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = true +probe = false diff --git a/examples/designs/poly/configs/device/cores/core3/interconnect/core.toml b/examples/designs/poly/configs/device/cores/core3/interconnect/core.toml new file mode 100644 index 000000000..13d04a5a6 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core3/interconnect/core.toml @@ -0,0 +1,2 @@ +axi = "default" +slots = 16 diff --git a/examples/designs/poly/configs/device/cores/core3/memory/access/core.toml b/examples/designs/poly/configs/device/cores/core3/memory/access/core.toml new file mode 100644 index 000000000..abd42a128 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core3/memory/access/core.toml @@ -0,0 +1,52 @@ +translationEntries = 16 +tagBits = 6 +resultLineBits = 0 +[[regions]] +base = 0x80000000 +bytes = 0x400000000 +cacheable = true +executable = true +readable = true +writable = true +atomic = true +normal = true + +[[regions]] +base = 0x10000 +bytes = 0x10000 +cacheable = true +executable = true +readable = true +writable = false +atomic = false +normal = true + +[[regions]] +base = 0x2000000 +bytes = 0x10000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0xc000000 +bytes = 0x4000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0x60000000 +bytes = 0x10000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false diff --git a/examples/designs/poly/configs/device/cores/core3/memory/core.toml b/examples/designs/poly/configs/device/cores/core3/memory/core.toml new file mode 100644 index 000000000..3caf53579 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core3/memory/core.toml @@ -0,0 +1,5 @@ +uncached = "default" +access = "access/core.toml" +icache = "icache/core.toml" +dcache = "dcache/core.toml" +home = "home/core.toml" diff --git a/examples/designs/poly/configs/device/cores/core3/memory/dcache/core.toml b/examples/designs/poly/configs/device/cores/core3/memory/dcache/core.toml new file mode 100644 index 000000000..2c4cef9ed --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core3/memory/dcache/core.toml @@ -0,0 +1,15 @@ +[tracking] +entries = 8 +addressBits = 44 +idBits = 8 +lineBytes = 64 +maxRanges = 8 + +[cache] +nodeId = 11 +lines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/poly/configs/device/cores/core3/memory/home/core.toml b/examples/designs/poly/configs/device/cores/core3/memory/home/core.toml new file mode 100644 index 000000000..0b7f0ff39 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core3/memory/home/core.toml @@ -0,0 +1,13 @@ +agents = 27 +mshrEntries = 8 +nodeId = 64 +requesters = "default" + +[cache] +addressBits = 44 +lineBytes = 64 +sets = 64 +ways = 4 +idBits = 8 +metadataBits = 4 +responseDepth = 2 diff --git a/examples/designs/poly/configs/device/cores/core3/memory/icache/core.toml b/examples/designs/poly/configs/device/cores/core3/memory/icache/core.toml new file mode 100644 index 000000000..f1dd6ae49 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core3/memory/icache/core.toml @@ -0,0 +1,7 @@ +nodeId = 10 +lines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/poly/configs/device/cores/core3/tbus/core.toml b/examples/designs/poly/configs/device/cores/core3/tbus/core.toml new file mode 100644 index 000000000..81037e7d9 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core3/tbus/core.toml @@ -0,0 +1 @@ +bankNodes = [0, 1, 2, 3, 4] diff --git a/examples/designs/poly/configs/device/cores/core4/c2c/core.toml b/examples/designs/poly/configs/device/cores/core4/c2c/core.toml new file mode 100644 index 000000000..d34ac9d9b --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core4/c2c/core.toml @@ -0,0 +1,3 @@ +enabled = true +peerCores = 9 +axi = "default" diff --git a/examples/designs/poly/configs/device/cores/core4/controller/core.toml b/examples/designs/poly/configs/device/cores/core4/controller/core.toml new file mode 100644 index 000000000..87f38f33c --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core4/controller/core.toml @@ -0,0 +1 @@ +moves = true diff --git a/examples/designs/poly/configs/device/cores/core4/default.toml b/examples/designs/poly/configs/device/cores/core4/default.toml new file mode 100644 index 000000000..5577c51ee --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core4/default.toml @@ -0,0 +1,71 @@ +component = "poly" +hartId = 4 +resetVector = 0x10000 +mtimeAddress = 0x200bff8 +dmaCache = "dmaCache/core.toml" +memory = "memory/core.toml" +fetch = "default" +fpu = "default" +interconnect = "interconnect/core.toml" +c2c = "c2c/core.toml" +tbus = "tbus/core.toml" +controller = "controller/core.toml" + +[param] +threadDataBytes = 64 +xLen = 64 +vaddrBits = 39 +paddrBits = 36 +pgIdxBits = 12 +pgLevels = 3 +nPMPs = 16 + +[tss] +base = 0x20000 +bytes = 0x4000 +dataBits = 128 + +[sharedMem] +bankWidth = 128 +bankEntries = 1024 +enable = true +virtualBankCount = 40 +entries = 16384 +inputChannels = 65 +defaultGroupCount = 1 +latency = 10 + +[[threads]] +id = 0 +name = "attention0" +component = "attention" +transport = "../core1/transport.toml" +include = "../../../../../../prefab/threads/attention/configs/default.toml" + +[[threads]] +id = 1 +name = "attention1" +component = "attention" +transport = "../core1/transport.toml" +include = "../../../../../../prefab/threads/attention/configs/default.toml" + +[[threads]] +id = 2 +name = "ffn0" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" + +[[threads]] +id = 3 +name = "ffn1" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" + +[[threads]] +id = 4 +name = "ffn2" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" diff --git a/examples/designs/poly/configs/device/cores/core4/dmaCache/core.toml b/examples/designs/poly/configs/device/cores/core4/dmaCache/core.toml new file mode 100644 index 000000000..825eebb12 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core4/dmaCache/core.toml @@ -0,0 +1,7 @@ +nodeId = 15 +lines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = true +probe = false diff --git a/examples/designs/poly/configs/device/cores/core4/interconnect/core.toml b/examples/designs/poly/configs/device/cores/core4/interconnect/core.toml new file mode 100644 index 000000000..13d04a5a6 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core4/interconnect/core.toml @@ -0,0 +1,2 @@ +axi = "default" +slots = 16 diff --git a/examples/designs/poly/configs/device/cores/core4/memory/access/core.toml b/examples/designs/poly/configs/device/cores/core4/memory/access/core.toml new file mode 100644 index 000000000..abd42a128 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core4/memory/access/core.toml @@ -0,0 +1,52 @@ +translationEntries = 16 +tagBits = 6 +resultLineBits = 0 +[[regions]] +base = 0x80000000 +bytes = 0x400000000 +cacheable = true +executable = true +readable = true +writable = true +atomic = true +normal = true + +[[regions]] +base = 0x10000 +bytes = 0x10000 +cacheable = true +executable = true +readable = true +writable = false +atomic = false +normal = true + +[[regions]] +base = 0x2000000 +bytes = 0x10000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0xc000000 +bytes = 0x4000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0x60000000 +bytes = 0x10000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false diff --git a/examples/designs/poly/configs/device/cores/core4/memory/core.toml b/examples/designs/poly/configs/device/cores/core4/memory/core.toml new file mode 100644 index 000000000..3caf53579 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core4/memory/core.toml @@ -0,0 +1,5 @@ +uncached = "default" +access = "access/core.toml" +icache = "icache/core.toml" +dcache = "dcache/core.toml" +home = "home/core.toml" diff --git a/examples/designs/poly/configs/device/cores/core4/memory/dcache/core.toml b/examples/designs/poly/configs/device/cores/core4/memory/dcache/core.toml new file mode 100644 index 000000000..72a49ee07 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core4/memory/dcache/core.toml @@ -0,0 +1,15 @@ +[tracking] +entries = 8 +addressBits = 44 +idBits = 8 +lineBytes = 64 +maxRanges = 8 + +[cache] +nodeId = 14 +lines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/poly/configs/device/cores/core4/memory/home/core.toml b/examples/designs/poly/configs/device/cores/core4/memory/home/core.toml new file mode 100644 index 000000000..0b7f0ff39 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core4/memory/home/core.toml @@ -0,0 +1,13 @@ +agents = 27 +mshrEntries = 8 +nodeId = 64 +requesters = "default" + +[cache] +addressBits = 44 +lineBytes = 64 +sets = 64 +ways = 4 +idBits = 8 +metadataBits = 4 +responseDepth = 2 diff --git a/examples/designs/poly/configs/device/cores/core4/memory/icache/core.toml b/examples/designs/poly/configs/device/cores/core4/memory/icache/core.toml new file mode 100644 index 000000000..623288def --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core4/memory/icache/core.toml @@ -0,0 +1,7 @@ +nodeId = 13 +lines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/poly/configs/device/cores/core4/tbus/core.toml b/examples/designs/poly/configs/device/cores/core4/tbus/core.toml new file mode 100644 index 000000000..81037e7d9 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core4/tbus/core.toml @@ -0,0 +1 @@ +bankNodes = [0, 1, 2, 3, 4] diff --git a/examples/designs/poly/configs/device/cores/core5/c2c/core.toml b/examples/designs/poly/configs/device/cores/core5/c2c/core.toml new file mode 100644 index 000000000..d34ac9d9b --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core5/c2c/core.toml @@ -0,0 +1,3 @@ +enabled = true +peerCores = 9 +axi = "default" diff --git a/examples/designs/poly/configs/device/cores/core5/controller/core.toml b/examples/designs/poly/configs/device/cores/core5/controller/core.toml new file mode 100644 index 000000000..87f38f33c --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core5/controller/core.toml @@ -0,0 +1 @@ +moves = true diff --git a/examples/designs/poly/configs/device/cores/core5/default.toml b/examples/designs/poly/configs/device/cores/core5/default.toml new file mode 100644 index 000000000..1ae5616c9 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core5/default.toml @@ -0,0 +1,71 @@ +component = "poly" +hartId = 5 +resetVector = 0x10000 +mtimeAddress = 0x200bff8 +dmaCache = "dmaCache/core.toml" +memory = "memory/core.toml" +fetch = "default" +fpu = "default" +interconnect = "interconnect/core.toml" +c2c = "c2c/core.toml" +tbus = "tbus/core.toml" +controller = "controller/core.toml" + +[param] +threadDataBytes = 64 +xLen = 64 +vaddrBits = 39 +paddrBits = 36 +pgIdxBits = 12 +pgLevels = 3 +nPMPs = 16 + +[tss] +base = 0x20000 +bytes = 0x4000 +dataBits = 128 + +[sharedMem] +bankWidth = 128 +bankEntries = 1024 +enable = true +virtualBankCount = 40 +entries = 16384 +inputChannels = 65 +defaultGroupCount = 1 +latency = 10 + +[[threads]] +id = 0 +name = "attention0" +component = "attention" +transport = "../core1/transport.toml" +include = "../../../../../../prefab/threads/attention/configs/default.toml" + +[[threads]] +id = 1 +name = "attention1" +component = "attention" +transport = "../core1/transport.toml" +include = "../../../../../../prefab/threads/attention/configs/default.toml" + +[[threads]] +id = 2 +name = "ffn0" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" + +[[threads]] +id = 3 +name = "ffn1" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" + +[[threads]] +id = 4 +name = "ffn2" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" diff --git a/examples/designs/poly/configs/device/cores/core5/dmaCache/core.toml b/examples/designs/poly/configs/device/cores/core5/dmaCache/core.toml new file mode 100644 index 000000000..8f809d9ec --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core5/dmaCache/core.toml @@ -0,0 +1,7 @@ +nodeId = 18 +lines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = true +probe = false diff --git a/examples/designs/poly/configs/device/cores/core5/interconnect/core.toml b/examples/designs/poly/configs/device/cores/core5/interconnect/core.toml new file mode 100644 index 000000000..13d04a5a6 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core5/interconnect/core.toml @@ -0,0 +1,2 @@ +axi = "default" +slots = 16 diff --git a/examples/designs/poly/configs/device/cores/core5/memory/access/core.toml b/examples/designs/poly/configs/device/cores/core5/memory/access/core.toml new file mode 100644 index 000000000..abd42a128 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core5/memory/access/core.toml @@ -0,0 +1,52 @@ +translationEntries = 16 +tagBits = 6 +resultLineBits = 0 +[[regions]] +base = 0x80000000 +bytes = 0x400000000 +cacheable = true +executable = true +readable = true +writable = true +atomic = true +normal = true + +[[regions]] +base = 0x10000 +bytes = 0x10000 +cacheable = true +executable = true +readable = true +writable = false +atomic = false +normal = true + +[[regions]] +base = 0x2000000 +bytes = 0x10000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0xc000000 +bytes = 0x4000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0x60000000 +bytes = 0x10000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false diff --git a/examples/designs/poly/configs/device/cores/core5/memory/core.toml b/examples/designs/poly/configs/device/cores/core5/memory/core.toml new file mode 100644 index 000000000..3caf53579 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core5/memory/core.toml @@ -0,0 +1,5 @@ +uncached = "default" +access = "access/core.toml" +icache = "icache/core.toml" +dcache = "dcache/core.toml" +home = "home/core.toml" diff --git a/examples/designs/poly/configs/device/cores/core5/memory/dcache/core.toml b/examples/designs/poly/configs/device/cores/core5/memory/dcache/core.toml new file mode 100644 index 000000000..96526fec5 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core5/memory/dcache/core.toml @@ -0,0 +1,15 @@ +[tracking] +entries = 8 +addressBits = 44 +idBits = 8 +lineBytes = 64 +maxRanges = 8 + +[cache] +nodeId = 17 +lines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/poly/configs/device/cores/core5/memory/home/core.toml b/examples/designs/poly/configs/device/cores/core5/memory/home/core.toml new file mode 100644 index 000000000..0b7f0ff39 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core5/memory/home/core.toml @@ -0,0 +1,13 @@ +agents = 27 +mshrEntries = 8 +nodeId = 64 +requesters = "default" + +[cache] +addressBits = 44 +lineBytes = 64 +sets = 64 +ways = 4 +idBits = 8 +metadataBits = 4 +responseDepth = 2 diff --git a/examples/designs/poly/configs/device/cores/core5/memory/icache/core.toml b/examples/designs/poly/configs/device/cores/core5/memory/icache/core.toml new file mode 100644 index 000000000..c9a5edf2c --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core5/memory/icache/core.toml @@ -0,0 +1,7 @@ +nodeId = 16 +lines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/poly/configs/device/cores/core5/tbus/core.toml b/examples/designs/poly/configs/device/cores/core5/tbus/core.toml new file mode 100644 index 000000000..81037e7d9 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core5/tbus/core.toml @@ -0,0 +1 @@ +bankNodes = [0, 1, 2, 3, 4] diff --git a/examples/designs/poly/configs/device/cores/core6/c2c/core.toml b/examples/designs/poly/configs/device/cores/core6/c2c/core.toml new file mode 100644 index 000000000..d34ac9d9b --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core6/c2c/core.toml @@ -0,0 +1,3 @@ +enabled = true +peerCores = 9 +axi = "default" diff --git a/examples/designs/poly/configs/device/cores/core6/controller/core.toml b/examples/designs/poly/configs/device/cores/core6/controller/core.toml new file mode 100644 index 000000000..87f38f33c --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core6/controller/core.toml @@ -0,0 +1 @@ +moves = true diff --git a/examples/designs/poly/configs/device/cores/core6/default.toml b/examples/designs/poly/configs/device/cores/core6/default.toml new file mode 100644 index 000000000..1cfd398e5 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core6/default.toml @@ -0,0 +1,71 @@ +component = "poly" +hartId = 6 +resetVector = 0x10000 +mtimeAddress = 0x200bff8 +dmaCache = "dmaCache/core.toml" +memory = "memory/core.toml" +fetch = "default" +fpu = "default" +interconnect = "interconnect/core.toml" +c2c = "c2c/core.toml" +tbus = "tbus/core.toml" +controller = "controller/core.toml" + +[param] +threadDataBytes = 64 +xLen = 64 +vaddrBits = 39 +paddrBits = 36 +pgIdxBits = 12 +pgLevels = 3 +nPMPs = 16 + +[tss] +base = 0x20000 +bytes = 0x4000 +dataBits = 128 + +[sharedMem] +bankWidth = 128 +bankEntries = 1024 +enable = true +virtualBankCount = 40 +entries = 16384 +inputChannels = 65 +defaultGroupCount = 1 +latency = 10 + +[[threads]] +id = 0 +name = "attention0" +component = "attention" +transport = "../core1/transport.toml" +include = "../../../../../../prefab/threads/attention/configs/default.toml" + +[[threads]] +id = 1 +name = "attention1" +component = "attention" +transport = "../core1/transport.toml" +include = "../../../../../../prefab/threads/attention/configs/default.toml" + +[[threads]] +id = 2 +name = "ffn0" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" + +[[threads]] +id = 3 +name = "ffn1" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" + +[[threads]] +id = 4 +name = "ffn2" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" diff --git a/examples/designs/poly/configs/device/cores/core6/dmaCache/core.toml b/examples/designs/poly/configs/device/cores/core6/dmaCache/core.toml new file mode 100644 index 000000000..0307dda51 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core6/dmaCache/core.toml @@ -0,0 +1,7 @@ +nodeId = 21 +lines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = true +probe = false diff --git a/examples/designs/poly/configs/device/cores/core6/interconnect/core.toml b/examples/designs/poly/configs/device/cores/core6/interconnect/core.toml new file mode 100644 index 000000000..13d04a5a6 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core6/interconnect/core.toml @@ -0,0 +1,2 @@ +axi = "default" +slots = 16 diff --git a/examples/designs/poly/configs/device/cores/core6/memory/access/core.toml b/examples/designs/poly/configs/device/cores/core6/memory/access/core.toml new file mode 100644 index 000000000..abd42a128 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core6/memory/access/core.toml @@ -0,0 +1,52 @@ +translationEntries = 16 +tagBits = 6 +resultLineBits = 0 +[[regions]] +base = 0x80000000 +bytes = 0x400000000 +cacheable = true +executable = true +readable = true +writable = true +atomic = true +normal = true + +[[regions]] +base = 0x10000 +bytes = 0x10000 +cacheable = true +executable = true +readable = true +writable = false +atomic = false +normal = true + +[[regions]] +base = 0x2000000 +bytes = 0x10000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0xc000000 +bytes = 0x4000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0x60000000 +bytes = 0x10000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false diff --git a/examples/designs/poly/configs/device/cores/core6/memory/core.toml b/examples/designs/poly/configs/device/cores/core6/memory/core.toml new file mode 100644 index 000000000..3caf53579 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core6/memory/core.toml @@ -0,0 +1,5 @@ +uncached = "default" +access = "access/core.toml" +icache = "icache/core.toml" +dcache = "dcache/core.toml" +home = "home/core.toml" diff --git a/examples/designs/poly/configs/device/cores/core6/memory/dcache/core.toml b/examples/designs/poly/configs/device/cores/core6/memory/dcache/core.toml new file mode 100644 index 000000000..1903dc0ed --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core6/memory/dcache/core.toml @@ -0,0 +1,15 @@ +[tracking] +entries = 8 +addressBits = 44 +idBits = 8 +lineBytes = 64 +maxRanges = 8 + +[cache] +nodeId = 20 +lines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/poly/configs/device/cores/core6/memory/home/core.toml b/examples/designs/poly/configs/device/cores/core6/memory/home/core.toml new file mode 100644 index 000000000..0b7f0ff39 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core6/memory/home/core.toml @@ -0,0 +1,13 @@ +agents = 27 +mshrEntries = 8 +nodeId = 64 +requesters = "default" + +[cache] +addressBits = 44 +lineBytes = 64 +sets = 64 +ways = 4 +idBits = 8 +metadataBits = 4 +responseDepth = 2 diff --git a/examples/designs/poly/configs/device/cores/core6/memory/icache/core.toml b/examples/designs/poly/configs/device/cores/core6/memory/icache/core.toml new file mode 100644 index 000000000..2386241f2 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core6/memory/icache/core.toml @@ -0,0 +1,7 @@ +nodeId = 19 +lines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/poly/configs/device/cores/core6/tbus/core.toml b/examples/designs/poly/configs/device/cores/core6/tbus/core.toml new file mode 100644 index 000000000..81037e7d9 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core6/tbus/core.toml @@ -0,0 +1 @@ +bankNodes = [0, 1, 2, 3, 4] diff --git a/examples/designs/poly/configs/device/cores/core7/c2c/core.toml b/examples/designs/poly/configs/device/cores/core7/c2c/core.toml new file mode 100644 index 000000000..d34ac9d9b --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core7/c2c/core.toml @@ -0,0 +1,3 @@ +enabled = true +peerCores = 9 +axi = "default" diff --git a/examples/designs/poly/configs/device/cores/core7/controller/core.toml b/examples/designs/poly/configs/device/cores/core7/controller/core.toml new file mode 100644 index 000000000..87f38f33c --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core7/controller/core.toml @@ -0,0 +1 @@ +moves = true diff --git a/examples/designs/poly/configs/device/cores/core7/default.toml b/examples/designs/poly/configs/device/cores/core7/default.toml new file mode 100644 index 000000000..751a41018 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core7/default.toml @@ -0,0 +1,71 @@ +component = "poly" +hartId = 7 +resetVector = 0x10000 +mtimeAddress = 0x200bff8 +dmaCache = "dmaCache/core.toml" +memory = "memory/core.toml" +fetch = "default" +fpu = "default" +interconnect = "interconnect/core.toml" +c2c = "c2c/core.toml" +tbus = "tbus/core.toml" +controller = "controller/core.toml" + +[param] +threadDataBytes = 64 +xLen = 64 +vaddrBits = 39 +paddrBits = 36 +pgIdxBits = 12 +pgLevels = 3 +nPMPs = 16 + +[tss] +base = 0x20000 +bytes = 0x4000 +dataBits = 128 + +[sharedMem] +bankWidth = 128 +bankEntries = 1024 +enable = true +virtualBankCount = 40 +entries = 16384 +inputChannels = 65 +defaultGroupCount = 1 +latency = 10 + +[[threads]] +id = 0 +name = "attention0" +component = "attention" +transport = "../core1/transport.toml" +include = "../../../../../../prefab/threads/attention/configs/default.toml" + +[[threads]] +id = 1 +name = "attention1" +component = "attention" +transport = "../core1/transport.toml" +include = "../../../../../../prefab/threads/attention/configs/default.toml" + +[[threads]] +id = 2 +name = "ffn0" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" + +[[threads]] +id = 3 +name = "ffn1" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" + +[[threads]] +id = 4 +name = "ffn2" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" diff --git a/examples/designs/poly/configs/device/cores/core7/dmaCache/core.toml b/examples/designs/poly/configs/device/cores/core7/dmaCache/core.toml new file mode 100644 index 000000000..20cc52e02 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core7/dmaCache/core.toml @@ -0,0 +1,7 @@ +nodeId = 24 +lines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = true +probe = false diff --git a/examples/designs/poly/configs/device/cores/core7/interconnect/core.toml b/examples/designs/poly/configs/device/cores/core7/interconnect/core.toml new file mode 100644 index 000000000..13d04a5a6 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core7/interconnect/core.toml @@ -0,0 +1,2 @@ +axi = "default" +slots = 16 diff --git a/examples/designs/poly/configs/device/cores/core7/memory/access/core.toml b/examples/designs/poly/configs/device/cores/core7/memory/access/core.toml new file mode 100644 index 000000000..abd42a128 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core7/memory/access/core.toml @@ -0,0 +1,52 @@ +translationEntries = 16 +tagBits = 6 +resultLineBits = 0 +[[regions]] +base = 0x80000000 +bytes = 0x400000000 +cacheable = true +executable = true +readable = true +writable = true +atomic = true +normal = true + +[[regions]] +base = 0x10000 +bytes = 0x10000 +cacheable = true +executable = true +readable = true +writable = false +atomic = false +normal = true + +[[regions]] +base = 0x2000000 +bytes = 0x10000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0xc000000 +bytes = 0x4000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0x60000000 +bytes = 0x10000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false diff --git a/examples/designs/poly/configs/device/cores/core7/memory/core.toml b/examples/designs/poly/configs/device/cores/core7/memory/core.toml new file mode 100644 index 000000000..3caf53579 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core7/memory/core.toml @@ -0,0 +1,5 @@ +uncached = "default" +access = "access/core.toml" +icache = "icache/core.toml" +dcache = "dcache/core.toml" +home = "home/core.toml" diff --git a/examples/designs/poly/configs/device/cores/core7/memory/dcache/core.toml b/examples/designs/poly/configs/device/cores/core7/memory/dcache/core.toml new file mode 100644 index 000000000..78dcd63c7 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core7/memory/dcache/core.toml @@ -0,0 +1,15 @@ +[tracking] +entries = 8 +addressBits = 44 +idBits = 8 +lineBytes = 64 +maxRanges = 8 + +[cache] +nodeId = 23 +lines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/poly/configs/device/cores/core7/memory/home/core.toml b/examples/designs/poly/configs/device/cores/core7/memory/home/core.toml new file mode 100644 index 000000000..0b7f0ff39 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core7/memory/home/core.toml @@ -0,0 +1,13 @@ +agents = 27 +mshrEntries = 8 +nodeId = 64 +requesters = "default" + +[cache] +addressBits = 44 +lineBytes = 64 +sets = 64 +ways = 4 +idBits = 8 +metadataBits = 4 +responseDepth = 2 diff --git a/examples/designs/poly/configs/device/cores/core7/memory/icache/core.toml b/examples/designs/poly/configs/device/cores/core7/memory/icache/core.toml new file mode 100644 index 000000000..d41891133 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core7/memory/icache/core.toml @@ -0,0 +1,7 @@ +nodeId = 22 +lines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/poly/configs/device/cores/core7/tbus/core.toml b/examples/designs/poly/configs/device/cores/core7/tbus/core.toml new file mode 100644 index 000000000..81037e7d9 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core7/tbus/core.toml @@ -0,0 +1 @@ +bankNodes = [0, 1, 2, 3, 4] diff --git a/examples/designs/poly/configs/device/cores/core8/c2c/core.toml b/examples/designs/poly/configs/device/cores/core8/c2c/core.toml new file mode 100644 index 000000000..d34ac9d9b --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core8/c2c/core.toml @@ -0,0 +1,3 @@ +enabled = true +peerCores = 9 +axi = "default" diff --git a/examples/designs/poly/configs/device/cores/core8/controller/core.toml b/examples/designs/poly/configs/device/cores/core8/controller/core.toml new file mode 100644 index 000000000..87f38f33c --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core8/controller/core.toml @@ -0,0 +1 @@ +moves = true diff --git a/examples/designs/poly/configs/device/cores/core8/default.toml b/examples/designs/poly/configs/device/cores/core8/default.toml new file mode 100644 index 000000000..430061c63 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core8/default.toml @@ -0,0 +1,71 @@ +component = "poly" +hartId = 8 +resetVector = 0x10000 +mtimeAddress = 0x200bff8 +dmaCache = "dmaCache/core.toml" +memory = "memory/core.toml" +fetch = "default" +fpu = "default" +interconnect = "interconnect/core.toml" +c2c = "c2c/core.toml" +tbus = "tbus/core.toml" +controller = "controller/core.toml" + +[param] +threadDataBytes = 64 +xLen = 64 +vaddrBits = 39 +paddrBits = 36 +pgIdxBits = 12 +pgLevels = 3 +nPMPs = 16 + +[tss] +base = 0x20000 +bytes = 0x4000 +dataBits = 128 + +[sharedMem] +bankWidth = 128 +bankEntries = 1024 +enable = true +virtualBankCount = 40 +entries = 16384 +inputChannels = 65 +defaultGroupCount = 1 +latency = 10 + +[[threads]] +id = 0 +name = "attention0" +component = "attention" +transport = "../core1/transport.toml" +include = "../../../../../../prefab/threads/attention/configs/default.toml" + +[[threads]] +id = 1 +name = "attention1" +component = "attention" +transport = "../core1/transport.toml" +include = "../../../../../../prefab/threads/attention/configs/default.toml" + +[[threads]] +id = 2 +name = "ffn0" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" + +[[threads]] +id = 3 +name = "ffn1" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" + +[[threads]] +id = 4 +name = "ffn2" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" diff --git a/examples/designs/poly/configs/device/cores/core8/dmaCache/core.toml b/examples/designs/poly/configs/device/cores/core8/dmaCache/core.toml new file mode 100644 index 000000000..17e8317ed --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core8/dmaCache/core.toml @@ -0,0 +1,7 @@ +nodeId = 27 +lines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = true +probe = false diff --git a/examples/designs/poly/configs/device/cores/core8/interconnect/core.toml b/examples/designs/poly/configs/device/cores/core8/interconnect/core.toml new file mode 100644 index 000000000..13d04a5a6 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core8/interconnect/core.toml @@ -0,0 +1,2 @@ +axi = "default" +slots = 16 diff --git a/examples/designs/poly/configs/device/cores/core8/memory/access/core.toml b/examples/designs/poly/configs/device/cores/core8/memory/access/core.toml new file mode 100644 index 000000000..abd42a128 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core8/memory/access/core.toml @@ -0,0 +1,52 @@ +translationEntries = 16 +tagBits = 6 +resultLineBits = 0 +[[regions]] +base = 0x80000000 +bytes = 0x400000000 +cacheable = true +executable = true +readable = true +writable = true +atomic = true +normal = true + +[[regions]] +base = 0x10000 +bytes = 0x10000 +cacheable = true +executable = true +readable = true +writable = false +atomic = false +normal = true + +[[regions]] +base = 0x2000000 +bytes = 0x10000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0xc000000 +bytes = 0x4000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0x60000000 +bytes = 0x10000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false diff --git a/examples/designs/poly/configs/device/cores/core8/memory/core.toml b/examples/designs/poly/configs/device/cores/core8/memory/core.toml new file mode 100644 index 000000000..3caf53579 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core8/memory/core.toml @@ -0,0 +1,5 @@ +uncached = "default" +access = "access/core.toml" +icache = "icache/core.toml" +dcache = "dcache/core.toml" +home = "home/core.toml" diff --git a/examples/designs/poly/configs/device/cores/core8/memory/dcache/core.toml b/examples/designs/poly/configs/device/cores/core8/memory/dcache/core.toml new file mode 100644 index 000000000..ced088acd --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core8/memory/dcache/core.toml @@ -0,0 +1,15 @@ +[tracking] +entries = 8 +addressBits = 44 +idBits = 8 +lineBytes = 64 +maxRanges = 8 + +[cache] +nodeId = 26 +lines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/poly/configs/device/cores/core8/memory/home/core.toml b/examples/designs/poly/configs/device/cores/core8/memory/home/core.toml new file mode 100644 index 000000000..0b7f0ff39 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core8/memory/home/core.toml @@ -0,0 +1,13 @@ +agents = 27 +mshrEntries = 8 +nodeId = 64 +requesters = "default" + +[cache] +addressBits = 44 +lineBytes = 64 +sets = 64 +ways = 4 +idBits = 8 +metadataBits = 4 +responseDepth = 2 diff --git a/examples/designs/poly/configs/device/cores/core8/memory/icache/core.toml b/examples/designs/poly/configs/device/cores/core8/memory/icache/core.toml new file mode 100644 index 000000000..835cdf2ec --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core8/memory/icache/core.toml @@ -0,0 +1,7 @@ +nodeId = 25 +lines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/poly/configs/device/cores/core8/tbus/core.toml b/examples/designs/poly/configs/device/cores/core8/tbus/core.toml new file mode 100644 index 000000000..81037e7d9 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core8/tbus/core.toml @@ -0,0 +1 @@ +bankNodes = [0, 1, 2, 3, 4] diff --git a/examples/designs/poly/configs/device/ddr.toml b/examples/designs/poly/configs/device/ddr.toml new file mode 100644 index 000000000..ac8cb4ff3 --- /dev/null +++ b/examples/designs/poly/configs/device/ddr.toml @@ -0,0 +1,13 @@ +clients = 1 +slotsPerClient = 16 +dataBits = 128 +idBits = 4 +idBase = 0 +base = 0x80000000 +bytes = 0x400000000 +model = 0x300000000 + +[line] +nodeIdBits = 7 +addressBits = 44 +dataBits = 256 diff --git a/examples/designs/poly/configs/device/default.toml b/examples/designs/poly/configs/device/default.toml new file mode 100644 index 000000000..33cce1c5d --- /dev/null +++ b/examples/designs/poly/configs/device/default.toml @@ -0,0 +1,32 @@ +ddr = "ddr.toml" +bootrom = "default" +clint = "clint.toml" +plic = "plic.toml" +scu = "default" + +[[cores]] +include = "cores/core0/default.toml" + +[[cores]] +include = "cores/core1/default.toml" + +[[cores]] +include = "cores/core2/default.toml" + +[[cores]] +include = "cores/core3/default.toml" + +[[cores]] +include = "cores/core4/default.toml" + +[[cores]] +include = "cores/core5/default.toml" + +[[cores]] +include = "cores/core6/default.toml" + +[[cores]] +include = "cores/core7/default.toml" + +[[cores]] +include = "cores/core8/default.toml" diff --git a/examples/designs/poly/configs/device/plic.toml b/examples/designs/poly/configs/device/plic.toml new file mode 100644 index 000000000..0beb0dba9 --- /dev/null +++ b/examples/designs/poly/configs/device/plic.toml @@ -0,0 +1,5 @@ +base = 0xc000000 +bytes = 0x4000000 +sources = 1 +priorityBits = 3 +hartIds = [0, 1, 2, 3, 4, 5, 6, 7, 8] diff --git a/examples/designs/poly/configs/sims/default.toml b/examples/designs/poly/configs/sims/default.toml new file mode 100644 index 000000000..0db22541b --- /dev/null +++ b/examples/designs/poly/configs/sims/default.toml @@ -0,0 +1,20 @@ +diffTest = false + +[memory] +lineBytes = 64 +clockHz = 1000000000 + +[targets.device] +config = "examples.designs.poly.sims.device.DeviceConfig" +stack_name = "poly" +regression = "sims/device/regression" +kernel = "sims/device/workloads/kernel" + +# Each device config overrides a few tables of the design config. +[targets.device.configs] +PolyVerilatorConfig = "verilator.toml" +PolyTapeoutConfig = "tapeout.toml" +PolyP2EConfig = "p2e.toml" + +[verification] +registry = "../../sims/ip/uvm/registry.toml" diff --git a/examples/designs/poly/configs/sims/p2e.toml b/examples/designs/poly/configs/sims/p2e.toml new file mode 100644 index 000000000..7a912105b --- /dev/null +++ b/examples/designs/poly/configs/sims/p2e.toml @@ -0,0 +1 @@ +# PolyP2EConfig: the full nine-core design on the P2E FPGA, with no overrides. diff --git a/examples/designs/poly/configs/sims/tapeout.toml b/examples/designs/poly/configs/sims/tapeout.toml new file mode 100644 index 000000000..cd0089f58 --- /dev/null +++ b/examples/designs/poly/configs/sims/tapeout.toml @@ -0,0 +1 @@ +# PolyTapeoutConfig: the full nine-core design for VCS, DC and tapeout, with no overrides. diff --git a/examples/designs/poly/configs/sims/verilator.toml b/examples/designs/poly/configs/sims/verilator.toml new file mode 100644 index 000000000..c5e2139ee --- /dev/null +++ b/examples/designs/poly/configs/sims/verilator.toml @@ -0,0 +1,9 @@ +# PolyVerilatorConfig: functional simulation (BEMU / Verilator) with the reduce +# core and one poly core. +[device] +cores = [ + { include = "../device/cores/core0/default.toml", memory = { home = { agents = 6 } }, c2c = { peerCores = 2 } }, + { include = "../device/cores/core1/default.toml", memory = { home = { agents = 6 } }, c2c = { peerCores = 2 } }, +] +clint = { hartIds = [0, 1] } +plic = { hartIds = [0, 1] } diff --git a/examples/designs/poly/emu/Cargo.lock b/examples/designs/poly/emu/Cargo.lock new file mode 100644 index 000000000..545b9a687 --- /dev/null +++ b/examples/designs/poly/emu/Cargo.lock @@ -0,0 +1,821 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "bebop-bank-hash" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "snafu", +] + +[[package]] +name = "bebop-bemu" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", + "bebop-bemu-profile", + "bebop-dasm", + "bebop-dtb", + "bebop-elf", + "bebop-keyboard", + "bebop-memory", + "bebop-microphone", + "bebop-rtc", + "bebop-speaker", + "bebop-syscall", + "bebop-uart", + "bebop-vga", + "bebop-vm", + "clap", + "linkme", + "prost", + "prost-build", + "prost-types", + "rustc_apfloat", + "rvsim", + "snafu", + "toml", +] + +[[package]] +name = "bebop-bemu-profile" +version = "0.1.0" + +[[package]] +name = "bebop-dasm" +version = "0.1.0" + +[[package]] +name = "bebop-design-poly" +version = "0.1.0" +dependencies = [ + "bebop-bemu", +] + +[[package]] +name = "bebop-dtb" +version = "0.1.0" + +[[package]] +name = "bebop-elf" +version = "0.1.0" +dependencies = [ + "bebop-memory", +] + +[[package]] +name = "bebop-keyboard" +version = "0.1.0" + +[[package]] +name = "bebop-memory" +version = "0.1.0" +dependencies = [ + "log", + "serde", + "serde_json", + "sha2", +] + +[[package]] +name = "bebop-microphone" +version = "0.1.0" + +[[package]] +name = "bebop-rtc" +version = "0.1.0" + +[[package]] +name = "bebop-speaker" +version = "0.1.0" + +[[package]] +name = "bebop-syscall" +version = "0.1.0" +dependencies = [ + "bebop-memory", + "once_cell", +] + +[[package]] +name = "bebop-uart" +version = "0.1.0" + +[[package]] +name = "bebop-vga" +version = "0.1.0" + +[[package]] +name = "bebop-vm" +version = "0.1.0" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.190" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" + +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck", + "itertools", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.119", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustc_apfloat" +version = "0.2.3+llvm-462a31f5a5ab" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "486c2179b4796f65bfe2ee33679acf0927ac83ecf583ad6c91c3b4570911b9ad" +dependencies = [ + "bitflags", + "smallvec", +] + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rvsim" +version = "0.1.0" +dependencies = [ + "rustc_apfloat", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "smallvec" +version = "1.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" + +[[package]] +name = "snafu" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/examples/designs/poly/emu/Cargo.toml b/examples/designs/poly/emu/Cargo.toml new file mode 100644 index 000000000..b74706f10 --- /dev/null +++ b/examples/designs/poly/emu/Cargo.toml @@ -0,0 +1,9 @@ +[workspace] + +[package] +name = "bebop-design-poly" +version = "0.1.0" +edition = "2021" + +[dependencies] +bebop-bemu = { path = "../../../../bebop/src/nodes/bemu" } diff --git a/examples/designs/poly/emu/src/device/core/mod.rs b/examples/designs/poly/emu/src/device/core/mod.rs new file mode 100644 index 000000000..f4935f943 --- /dev/null +++ b/examples/designs/poly/emu/src/device/core/mod.rs @@ -0,0 +1,37 @@ +use bebop_bemu::{ + ball::linkme, CoreFactory, CoreModel, CoreParams, Cpu, DeviceModel, FromString, ModelOptions, NoRocc, ThreadGroup, + Whatever, CORES, +}; + +/// One RV64GC hart driving its attention and ffn hardware threads (arch: PolyCore, behind TBus). +pub struct PolyCore { + cpu: Cpu, + threads: ThreadGroup, +} + +impl CoreModel for PolyCore { + fn cpu(&self) -> &Cpu { + &self.cpu + } + fn cpu_mut(&mut self) -> &mut Cpu { + &mut self.cpu + } + fn step(&mut self, count: u64) -> Result<(), Whatever> { + self.threads.check().map_err(Whatever::without_source)?; + self.cpu.step(count, &mut NoRocc)?; + if self.cpu.finished() { + self.threads.shutdown().map_err(Whatever::without_source)?; + } + Ok(()) + } +} + +fn instantiate(device: &DeviceModel, params: &CoreParams, options: &ModelOptions) -> Result, String> { + let cpu = Cpu::new(device, params, options).map_err(|error| error.to_string())?; + let threads = ThreadGroup::new(cpu.state(), options)?; + Ok(Box::new(PolyCore { cpu, threads })) +} + +#[linkme::distributed_slice(CORES)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static POLY: CoreFactory = CoreFactory { component: "poly", instantiate }; diff --git a/examples/designs/poly/emu/src/device/mod.rs b/examples/designs/poly/emu/src/device/mod.rs new file mode 100644 index 000000000..c0ba35020 --- /dev/null +++ b/examples/designs/poly/emu/src/device/mod.rs @@ -0,0 +1,2 @@ +//! The poly device: its own Core model; the cores are instantiated by component on the CBus. +pub mod core; diff --git a/examples/designs/poly/emu/src/lib.rs b/examples/designs/poly/emu/src/lib.rs new file mode 100644 index 000000000..545892414 --- /dev/null +++ b/examples/designs/poly/emu/src/lib.rs @@ -0,0 +1 @@ +pub mod device; diff --git a/examples/designs/poly/sims/device/arch/DeviceConfig.scala b/examples/designs/poly/sims/device/arch/DeviceConfig.scala new file mode 100644 index 000000000..c8dc74f8e --- /dev/null +++ b/examples/designs/poly/sims/device/arch/DeviceConfig.scala @@ -0,0 +1,18 @@ +package examples.designs.poly.sims.device + +import buckyball.config.Design +import chisel3.experimental.hierarchy.{Instance, Instantiate} +import examples.designs.poly.DeviceTop +import framework.device.cbus.CBus +import framework.device.cbus.generated.CoreComponents +import framework.device.dlink.BaseDevice +import _root_.configs.GlobalConfig +import _root_.sims.soc.SystemTarget +import java.nio.file.{Files, Paths} + +class DeviceConfig(design: Design) + extends SystemTarget( + GlobalConfig.fromProto(design, CBus.parameters(design, CoreComponents.factories)) + ) { + def instantiate(b: GlobalConfig): Instance[BaseDevice] = Instantiate(new DeviceTop(b, design)) +} diff --git a/examples/designs/poly/sims/device/arch/build.sc b/examples/designs/poly/sims/device/arch/build.sc new file mode 100644 index 000000000..5b94111b8 --- /dev/null +++ b/examples/designs/poly/sims/device/arch/build.sc @@ -0,0 +1,17 @@ +import mill._ +import mill.scalalib._ +import $file.^.^.^.arch.{build => design_build} +import $file.^.^.^.^.^.^.arch.src.main.scala.sims.{build => sims_build} + +val entryRoot: os.Path = millSourcePath + +object simulation extends SbtModule { + override def millSourcePath = entryRoot + override def scalaVersion = "2.13.16" + override def moduleDeps = Seq(design_build.design, sims_build.library) + override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") + override def sources = T.sources(entryRoot / "DeviceConfig.scala") + override def mainClass = Some("sims.verilator.Elaborate") +} diff --git a/examples/designs/poly/sims/device/emu/Cargo.lock b/examples/designs/poly/sims/device/emu/Cargo.lock new file mode 100644 index 000000000..686a331aa --- /dev/null +++ b/examples/designs/poly/sims/device/emu/Cargo.lock @@ -0,0 +1,1979 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "assert_cmd" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aa3a22042e45de04255c7bf3626e239f450200fd0493c1e382263544b20aea6" +dependencies = [ + "anstyle", + "bstr", + "libc", + "predicates", + "predicates-core", + "predicates-tree", + "wait-timeout", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bebop" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", + "bebop-bemu", + "bebop-bemu-profile", + "bebop-dasm", + "bebop-fd-redirect", + "bebop-p2e", + "bebop-report", + "bebop-rtl-trace", + "bebop-terminal", + "bebop-trace-perfetto", + "bebop-uart", + "bebop-vcs", + "bebop-verilator", + "bebop-vm", + "camino", + "clap", + "duct", + "env_logger", + "libc", + "log", + "nix", + "serde", + "serde_json", + "snafu", + "toml", +] + +[[package]] +name = "bebop-bank-hash" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "snafu", +] + +[[package]] +name = "bebop-bemu" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", + "bebop-bemu-profile", + "bebop-dasm", + "bebop-dtb", + "bebop-elf", + "bebop-keyboard", + "bebop-memory", + "bebop-microphone", + "bebop-rtc", + "bebop-speaker", + "bebop-syscall", + "bebop-uart", + "bebop-vga", + "clap", + "linkme", + "prost", + "prost-build", + "prost-types", + "rustc_apfloat", + "snafu", + "toml", +] + +[[package]] +name = "bebop-bemu-profile" +version = "0.1.0" + +[[package]] +name = "bebop-core-reduce" +version = "0.1.0" +dependencies = [ + "bebop-bemu", +] + +[[package]] +name = "bebop-dasm" +version = "0.1.0" + +[[package]] +name = "bebop-design-poly" +version = "0.1.0" +dependencies = [ + "bebop-bemu", +] + +[[package]] +name = "bebop-dtb" +version = "0.1.0" + +[[package]] +name = "bebop-elf" +version = "0.1.0" +dependencies = [ + "bebop-memory", +] + +[[package]] +name = "bebop-fd-redirect" +version = "0.1.0" +dependencies = [ + "libc", +] + +[[package]] +name = "bebop-keyboard" +version = "0.1.0" + +[[package]] +name = "bebop-memory" +version = "0.1.0" + +[[package]] +name = "bebop-microphone" +version = "0.1.0" + +[[package]] +name = "bebop-p2e" +version = "0.1.0" +dependencies = [ + "bebop-memory", + "bebop-rtl-trace", + "bebop-uart", + "duct", + "libc", + "log", + "serde", + "serde_json", +] + +[[package]] +name = "bebop-report" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "bebop-rtc" +version = "0.1.0" + +[[package]] +name = "bebop-rtl-trace" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", +] + +[[package]] +name = "bebop-sim-build" +version = "0.1.0" +dependencies = [ + "prost", + "prost-build", + "prost-types", +] + +[[package]] +name = "bebop-sim-poly-device" +version = "0.1.0" +dependencies = [ + "assert_cmd", + "bebop", + "bebop-bemu", + "bebop-core-reduce", + "bebop-design-poly", + "bebop-report", + "bebop-sim-build", + "bebop-verilator", + "chrono", + "clap", + "libtest-mimic", + "predicates", + "serde", + "toml", + "walkdir", +] + +[[package]] +name = "bebop-speaker" +version = "0.1.0" + +[[package]] +name = "bebop-syscall" +version = "0.1.0" +dependencies = [ + "bebop-memory", + "once_cell", +] + +[[package]] +name = "bebop-terminal" +version = "0.1.0" +dependencies = [ + "crossterm", + "ratatui", +] + +[[package]] +name = "bebop-trace-perfetto" +version = "0.1.0" +dependencies = [ + "clap", + "serde_json", +] + +[[package]] +name = "bebop-uart" +version = "0.1.0" + +[[package]] +name = "bebop-vcs" +version = "0.1.0" +dependencies = [ + "snafu", +] + +[[package]] +name = "bebop-verilator" +version = "0.1.0" +dependencies = [ + "bebop-dasm", + "bebop-fd-redirect", + "bebop-rtl-trace", + "bebop-uart", + "libc", + "snafu", +] + +[[package]] +name = "bebop-vga" +version = "0.1.0" + +[[package]] +name = "bebop-vm" +version = "0.1.0" +dependencies = [ + "clap", + "font8x8", + "sdl2", + "vt100", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "bstr" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" +dependencies = [ + "memchr", + "regex-automata", + "serde_core", +] + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "camino" +version = "1.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbad30e4b4c14a39e3cc8aed085a12a327257c316619c93581e017bc52be591" + +[[package]] +name = "cassowary" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" + +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + +[[package]] +name = "cc" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f74872d07caf508b30a21f6836e7d7016a2eaf7d9ff4f48deaa58cd8a0407630" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "compact_str" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fd622ebbb56a5b2ccb651b32b911cdeb2a9b4b11776b2473bf26a26a286244e" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "static_assertions", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crossterm" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" +dependencies = [ + "bitflags 2.13.2", + "crossterm_winapi", + "mio", + "parking_lot", + "rustix 0.38.44", + "signal-hook 0.3.18", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + +[[package]] +name = "darling" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 3.0.9", +] + +[[package]] +name = "darling_macro" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" +dependencies = [ + "darling_core", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "defmt" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a0795ac740cb54d7b3775c9c17345f6aaecf884c01733b39d6017738e0e3e1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "840bb056bd99c495c9188198a258ab787c79f306d864db3e6e8451442d6d8366" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "defmt-parser" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71958f69fb2de856c5dd51ac9377e986ca1aa5919f931bf5395a807277f889a0" +dependencies = [ + "thiserror", +] + +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + +[[package]] +name = "duct" +version = "0.13.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4ab5718d1224b63252cd0c6f74f6480f9ffeb117438a2e0f5cf6d9a4798929c" +dependencies = [ + "libc", + "once_cell", + "os_pipe", + "shared_child", +] + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "env_filter" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "escape8259" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f0d310082fd5de539d4f41601182241147498444a7695c520288a0cdbead6" + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "float-cmp" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09cf3155332e944990140d967ff5eceb70df778b34f77d8075db46e4704e6d8" +dependencies = [ + "num-traits", +] + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "font8x8" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875488b8711a968268c7cf5d139578713097ca4635a76044e8fe8eedf831d07e" + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "indoc" +version = "2.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a37b2691796cffeb8a8cd305ac66e65841559f147f4e63231d0eafa4db5384d1" +dependencies = [ + "rustversion", +] + +[[package]] +name = "instability" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3b5acc1e2fd9375041a388da33d1eb8aed5f7a8c0dd3543e3ea2805adfbe20" +dependencies = [ + "darling", + "indoc", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jiff" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b005715dcbeb0089a3c0dab99f2ff1cc3b2525323552703d648585d342a383" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", +] + +[[package]] +name = "jiff-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e52fe76043ccecc9005d2305ebaadf7d7fc0cc89ca6baa10a94d6bc68c7128c" +dependencies = [ + "defmt", + "log", +] + +[[package]] +name = "jiff-static" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cc9817253cf7c7ee4684451bd327e88d6f3658014e54a29198625590650695c" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "js-sys" +version = "0.3.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "libtest-mimic" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14e6ba06f0ade6e504aff834d7c34298e5155c6baca353cc6a4aaff2f9fd7f33" +dependencies = [ + "anstream", + "anstyle", + "clap", + "escape8259", +] + +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.5", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "os_pipe" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-atomic-util" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "091397be61a01d4be58e7841595bd4bfedb15f1cd54977d79b8271e94ed799a3" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "predicates" +version = "3.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ada8f2932f28a27ee7b70dd6c1c39ea0675c55a36879ab92f3a715eaa1e63cfe" +dependencies = [ + "anstyle", + "difflib", + "float-cmp", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cad38746f3166b4031b1a0d39ad9f954dd291e7854fcc0eed52ee41a0b50d144" + +[[package]] +name = "predicates-tree" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0de1b847b39c8131db0467e9df1ff60e6d0562ab8e9a16e568ad0fdb372e2f2" +dependencies = [ + "predicates-core", + "termtree", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck", + "itertools 0.14.0", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.119", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "ratatui" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabd94c2f37801c20583fc49dd5cd6b0ba68c716787c2dd6ed18571e1e63117b" +dependencies = [ + "bitflags 2.13.2", + "cassowary", + "compact_str", + "crossterm", + "indoc", + "instability", + "itertools 0.13.0", + "lru", + "paste", + "strum", + "unicode-segmentation", + "unicode-truncate", + "unicode-width 0.2.0", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.2", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustc_apfloat" +version = "0.2.3+llvm-462a31f5a5ab" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "486c2179b4796f65bfe2ee33679acf0927ac83ecf583ad6c91c3b4570911b9ad" +dependencies = [ + "bitflags 2.13.2", + "smallvec", +] + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys 0.12.1", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sdl2" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d42407afc6a8ab67e36f92e80b8ba34cbdc55aaeed05249efe9a2e8d0e9feef" +dependencies = [ + "bitflags 1.3.2", + "lazy_static", + "libc", + "sdl2-sys", +] + +[[package]] +name = "sdl2-sys" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ff61407fc75d4b0bbc93dc7e4d6c196439965fbef8e4a4f003a36095823eac0" +dependencies = [ + "cfg-if", + "libc", + "pkg-config", + "version-compare", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "shared_child" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "607549934f6cc26b89cfecfdc46fa90f1e5d1536a68349b0c3a4f9d1c0d37959" +dependencies = [ + "libc", + "sigchld", + "windows-sys 0.61.2", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "sigchld" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24f2b37f04360cd465089b87a9c3869c08220a2f3458463f0adf8badf5e77f2c" +dependencies = [ + "libc", + "os_pipe", + "signal-hook 0.4.5", +] + +[[package]] +name = "signal-hook" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b29ffa27d3d59a049bb328258de100f8012848d1b08d444e08bbc02ac468435e" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-mio" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b75a19a7a740b25bc7944bdee6172368f988763b744e3d4dfe753f6b4ece40cc" +dependencies = [ + "libc", + "mio", + "signal-hook 0.3.18", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "smallvec" +version = "1.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8" + +[[package]] +name = "snafu" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.119", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix 1.1.5", + "windows-sys 0.61.2", +] + +[[package]] +name = "termtree" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-truncate" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3644627a5af5fa321c95b9b235a72fd24cd29c648c2c379431e6628655627bf" +dependencies = [ + "itertools 0.13.0", + "unicode-segmentation", + "unicode-width 0.1.14", +] + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "version-compare" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "579a42fc0b8e0c63b76519a339be31bed574929511fa53c1a3acae26eb258f29" + +[[package]] +name = "vt100" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84cd863bf0db7e392ba3bd04994be3473491b31e66340672af5d11943c6274de" +dependencies = [ + "itoa", + "log", + "unicode-width 0.1.14", + "vte", +] + +[[package]] +name = "vte" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5022b5fbf9407086c180e9557be968742d839e68346af7792b8592489732197" +dependencies = [ + "arrayvec", + "utf8parse", + "vte_generate_state_changes", +] + +[[package]] +name = "vte_generate_state_changes" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e369bee1b05d510a7b4ed645f5faa90619e05437111783ea5848f28d97d3c2e" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/examples/designs/poly/sims/device/emu/Cargo.toml b/examples/designs/poly/sims/device/emu/Cargo.toml new file mode 100644 index 000000000..39204a91b --- /dev/null +++ b/examples/designs/poly/sims/device/emu/Cargo.toml @@ -0,0 +1,84 @@ +[workspace] + +[package] +name = "bebop-sim-poly-device" +version = "0.1.0" +edition = "2021" +autobins = false +autotests = false + +# The model of this simulation entry; the runners below are built from it. +[lib] +name = "simulation" + +[[bin]] +name = "bebop" +path = "../../../../../../bebop/src/main.rs" + +[[bin]] +name = "bebop-bemu" +path = "../../../../../../bebop/src/bin/bebop-bemu.rs" +required-features = ["bemu"] + +[[bin]] +name = "bebop-verilator" +path = "../../../../../../bebop/src/bin/bebop-verilator.rs" +required-features = ["verilator"] + +[[bin]] +name = "bebop-p2e" +path = "../../../../../../bebop/src/bin/bebop-p2e.rs" +required-features = ["p2e"] + +[[test]] +name = "test_bemu" +path = "../../../../../../bebop/tests/test_bemu.rs" +harness = false +required-features = ["bemu"] + +[[test]] +name = "test_verilator" +path = "../../../../../../bebop/tests/test_verilator.rs" +harness = false +required-features = ["verilator"] + +[[test]] +name = "test_p2e" +path = "../../../../../../bebop/tests/test_p2e.rs" +harness = false +required-features = ["p2e"] + +[[bin]] +name = "bebop-vcs" +path = "../../../../../../bebop/src/bin/bebop-vcs.rs" + +[[test]] +name = "test_vcs" +path = "../../../../../../bebop/tests/test_vcs.rs" +harness = false + +[features] +bemu = ["bebop/bemu"] +verilator = ["bebop/verilator", "dep:bebop-verilator"] +p2e = ["bebop/p2e"] + +[dependencies] +bebop = { path = "../../../../../../bebop" } +bebop-verilator = { path = "../../../../../../bebop/src/nodes/verilator", optional = true } +bebop-bemu = { path = "../../../../../../bebop/src/nodes/bemu" } +bebop-design-poly = { path = "../../../emu" } +bebop-core-reduce = { path = "../../../../../prefab/cores/reduce/emu" } +clap = { version = "4", features = ["derive"] } + +[build-dependencies] +bebop-sim-build = { path = "../../../../../../bebop/src/nodes/lib/sim-build" } + +[dev-dependencies] +bebop-report = { path = "../../../../../../bebop/src/nodes/lib/report" } +libtest-mimic = "0.8" +assert_cmd = "2" +walkdir = "2" +predicates = "3" +serde = { version = "1", features = ["derive"] } +toml = "0.8" +chrono = { version = "0.4", default-features = false, features = ["clock"] } diff --git a/examples/designs/poly/sims/device/emu/build.rs b/examples/designs/poly/sims/device/emu/build.rs new file mode 100644 index 000000000..beb62e98c --- /dev/null +++ b/examples/designs/poly/sims/device/emu/build.rs @@ -0,0 +1,4 @@ +fn main() { + bebop_sim_build::balls(); + bebop_sim_build::runner_links(); +} diff --git a/examples/designs/poly/sims/device/emu/src/lib.rs b/examples/designs/poly/sims/device/emu/src/lib.rs new file mode 100644 index 000000000..087ba4418 --- /dev/null +++ b/examples/designs/poly/sims/device/emu/src/lib.rs @@ -0,0 +1,12 @@ +//! The poly device simulation. +//! Its Core models register in `bebop_bemu::CORES`; the Balls its design.pb +//! names are compiled here and register in `bebop_bemu::BALLS`. +extern crate bebop_design_poly; +extern crate bebop_core_reduce; + +include!(concat!(env!("OUT_DIR"), "/balls.rs")); + +pub const SIMULATION: bebop_bemu::Simulation = bebop_bemu::Simulation { + design: include_bytes!(concat!(env!("OUT_DIR"), "/design.pb")), + instantiate: bebop_bemu::device::cbus::instantiate, +}; diff --git a/examples/designs/poly/sims/device/regression/bemu/workloads-bare.toml b/examples/designs/poly/sims/device/regression/bemu/workloads-bare.toml new file mode 100644 index 000000000..444a0a401 --- /dev/null +++ b/examples/designs/poly/sims/device/regression/bemu/workloads-bare.toml @@ -0,0 +1,25 @@ +[workloads] +tests = [ + "poly-main-ctest-design-ant_mesh_move-multicore-baremetal", + "poly-main-ctest-design-ant_mesh_private-multicore-baremetal", + "poly-main-ctest-design-ant_mesh_shared-multicore-baremetal", + "poly-main-ctest-f32add_groups-baremetal", + "poly-main-ctest-corelink_lease-baremetal", + "poly-main-ctest-design-controllers-multicore-baremetal", + "poly-main-ctest-design-ant_mxmm-multicore-baremetal", + "poly-main-ctest-design-ant_shared-multicore-baremetal", + "poly-main-ctest-design-ant_bank-multicore-baremetal", + "poly-main-ctest-design-ant_bank_rows-multicore-baremetal", + "poly-main-ctest-design-ant_parallel-multicore-baremetal", + "poly-main-ctest-design-ant_highpa-multicore-baremetal", + "poly-main-ctest-design-mlir_ball_mxmm_mxfp8_chain-multicore-baremetal", + "poly-main-ctest-design-mlir_ball_mxmm_mxfp8_window-multicore-baremetal", + "poly-main-ctest-design-mlir_ball_mxmm_fma32-multicore-baremetal", + "poly-main-ctest-design-mlir_ball_mxmm_f32-multicore-baremetal", + "poly-main-ctest-design-mlir_ball_mxquant_two_blocks-multicore-baremetal", + "poly-main-ctest-design-mlir_bank_mxmm_mxfp8_chain-multicore-baremetal", + "poly-main-ctest-design-mlir_bank_mxmm_mxfp8_window-multicore-baremetal", + "poly-main-ctest-design-mlir_bank_mxmm_fma32-multicore-baremetal", + "poly-main-ctest-design-mlir_bank_mxmm_f32-multicore-baremetal", + "poly-main-ctest-design-mlir_bank_mxquant_two_blocks-multicore-baremetal", +] diff --git a/examples/designs/poly/sims/device/regression/bemu/workloads-linux.toml b/examples/designs/poly/sims/device/regression/bemu/workloads-linux.toml new file mode 100644 index 000000000..41a9cd13b --- /dev/null +++ b/examples/designs/poly/sims/device/regression/bemu/workloads-linux.toml @@ -0,0 +1,4 @@ +[workloads] +tests = [ + "fw_payload-linux.elf", +] diff --git a/examples/chips/poly/regression/batch/p2e/workloads-bare.toml b/examples/designs/poly/sims/device/regression/p2e/workloads-bare.toml similarity index 100% rename from examples/chips/poly/regression/batch/p2e/workloads-bare.toml rename to examples/designs/poly/sims/device/regression/p2e/workloads-bare.toml diff --git a/examples/designs/poly/sims/device/regression/p2e/workloads-linux.toml b/examples/designs/poly/sims/device/regression/p2e/workloads-linux.toml new file mode 100644 index 000000000..ca955b614 --- /dev/null +++ b/examples/designs/poly/sims/device/regression/p2e/workloads-linux.toml @@ -0,0 +1,4 @@ +[workloads] +tests = [ + "fw_payload-linux-ddr.hex", +] diff --git a/examples/designs/poly/sims/device/regression/verilator/workloads-bare.toml b/examples/designs/poly/sims/device/regression/verilator/workloads-bare.toml new file mode 100644 index 000000000..b89b1795d --- /dev/null +++ b/examples/designs/poly/sims/device/regression/verilator/workloads-bare.toml @@ -0,0 +1,15 @@ +[workloads] +tests = [ + "poly-main-ctest-design-ant_mesh_move-multicore-baremetal", + "poly-main-ctest-design-ant_mesh_private-multicore-baremetal", + "poly-main-ctest-design-ant_mesh_shared-multicore-baremetal", + "poly-main-ctest-f32add_groups-baremetal", + "poly-main-ctest-corelink_lease-baremetal", + "poly-main-ctest-design-controllers-multicore-baremetal", + "poly-main-ctest-design-ant_mxmm-multicore-baremetal", + "poly-main-ctest-design-ant_shared-multicore-baremetal", + "poly-main-ctest-design-ant_bank_rows-multicore-baremetal", + "poly-main-ctest-design-ant_parallel-multicore-baremetal", + "poly-main-ctest-design-ant_bank-multicore-baremetal", + "poly-main-ctest-design-ant_highpa-multicore-baremetal", +] diff --git a/examples/designs/poly/sims/device/stack/CMakeLists.txt b/examples/designs/poly/sims/device/stack/CMakeLists.txt new file mode 100644 index 000000000..35a0cd209 --- /dev/null +++ b/examples/designs/poly/sims/device/stack/CMakeLists.txt @@ -0,0 +1,67 @@ +# ===----------------------------------------------------------------------=== +# The software stack of the poly device simulation: every level the device +# needs, built bottom up. +# +# bbdev stack --build "--design poly --device " +# +# configures this with DESIGN_CMAKE, generated from design.pb. Each level is a +# CMake build of its own: cores and threads build buddy-opt plugins for the +# host, the device builds RISC-V libraries. +# thread/attention, thread/ffn lib-compiler.so, program.bin, loader.o, +# libkernels*.a +# core/reduce, core/poly execute.o and their compiler plugins +# device runtime, runtime-baremetal, runtime-linux, engine; device.cmake +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(poly-device-stack NONE) +include(ExternalProject) +include("${DESIGN_CMAKE}") + +# The attention hardware thread. +ExternalProject_Add(thread-attention + SOURCE_DIR "${DESIGN_THREAD_STACK_attention}" + BINARY_DIR "${DESIGN_THREAD_BUILD_attention}" + CMAKE_GENERATOR Ninja + CMAKE_ARGS "-DDESIGN_CMAKE=${DESIGN_CMAKE}" -DTHREAD=attention + BUILD_ALWAYS ON + INSTALL_COMMAND "") + +# The ffn hardware thread. +ExternalProject_Add(thread-ffn + SOURCE_DIR "${DESIGN_THREAD_STACK_ffn}" + BINARY_DIR "${DESIGN_THREAD_BUILD_ffn}" + CMAKE_GENERATOR Ninja + CMAKE_ARGS "-DDESIGN_CMAKE=${DESIGN_CMAKE}" -DTHREAD=ffn + BUILD_ALWAYS ON + INSTALL_COMMAND "") + +# The reduce core. +ExternalProject_Add(core-reduce + SOURCE_DIR "${DESIGN_CORE_STACK_reduce}" + BINARY_DIR "${DESIGN_CORE_BUILD_reduce}" + CMAKE_GENERATOR Ninja + CMAKE_ARGS "-DDESIGN_CMAKE=${DESIGN_CMAKE}" -DCORE=reduce + BUILD_ALWAYS ON + INSTALL_COMMAND "") + +# Poly's CPU core, which runs the attention and ffn threads. +ExternalProject_Add(core-poly + SOURCE_DIR "${DESIGN_CORE_STACK_poly}" + BINARY_DIR "${DESIGN_CORE_BUILD_poly}" + CMAKE_GENERATOR Ninja + CMAKE_ARGS "-DDESIGN_CMAKE=${DESIGN_CMAKE}" -DCORE=poly + DEPENDS thread-attention thread-ffn + BUILD_ALWAYS ON + INSTALL_COMMAND "") + +# The device: the runtime that schedules jobs onto the cores. +ExternalProject_Add(device + SOURCE_DIR "${DESIGN_DEVICE_STACK}" + BINARY_DIR "${DESIGN_DEVICE_BUILD}" + CMAKE_GENERATOR Ninja + CMAKE_ARGS "-DDESIGN_CMAKE=${DESIGN_CMAKE}" + -DCMAKE_C_COMPILER=riscv64-unknown-linux-gnu-gcc + -DCMAKE_CXX_COMPILER=riscv64-unknown-linux-gnu-g++ + DEPENDS thread-attention thread-ffn core-reduce core-poly + BUILD_ALWAYS ON + INSTALL_COMMAND "") diff --git a/examples/designs/poly/sims/device/workloads/CMakeLists.txt b/examples/designs/poly/sims/device/workloads/CMakeLists.txt new file mode 100644 index 000000000..f0d468355 --- /dev/null +++ b/examples/designs/poly/sims/device/workloads/CMakeLists.txt @@ -0,0 +1,81 @@ +include("${DESIGN_DEVICE_BUILD}/device.cmake") + +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/base" + "${CMAKE_BINARY_DIR}/src/CTest/base" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/tlb" + "${CMAKE_BINARY_DIR}/src/CTest/tlb" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/tutorial" + "${CMAKE_BINARY_DIR}/src/CTest/tutorial" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/embench" + "${CMAKE_BINARY_DIR}/src/CTest/embench" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/coremark" + "${CMAKE_BINARY_DIR}/src/CTest/coremark" +) +foreach(BUCKYBALL_COMPONENT_WORKLOAD_TARGET IN LISTS DESIGN_CPU_TARGETS) + set(COMPONENT_WORKLOADS + "${DESIGN_TARGET_WORKLOADS_${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}}" + ) + if( + COMPONENT_WORKLOADS + STREQUAL + "${BUCKYBALL_REPO_ROOT}/examples/designs/${DESIGN_NAME}/workloads" + ) + continue() + endif() + set(BUCKYBALL_CTEST_TARGET "${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}") + set(BUCKYBALL_CTEST_GROUP_TARGET component-ctests-build) + if(IS_DIRECTORY "${COMPONENT_WORKLOADS}/ctests") + add_subdirectory( + "${COMPONENT_WORKLOADS}/ctests" + "${CMAKE_CURRENT_BINARY_DIR}/prefab/${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}/ctests" + ) + endif() +endforeach() +add_dependencies( + ctest-workloads-build + base-ctests-build + tlb-ctests-build + tutorial-ctests-build + component-ctests-build +) +add_dependencies(sync-native-bin embench-all coremark-all) + +add_subdirectory( + "${WORKLOAD_SRC_DIR}/MLIRTest/base" + "${CMAKE_BINARY_DIR}/src/MLIRTest/mlir_tests/base" +) +foreach(BUCKYBALL_TARGET IN LISTS DESIGN_CPU_TARGETS) + set(BUCKYBALL_MLIR_ACTIVE_TARGET "${BUCKYBALL_TARGET}") + foreach(BUCKYBALL_BALL_DIR IN LISTS DESIGN_PROFILES_${BUCKYBALL_TARGET}_THREAD_BALLDOMAIN_BALLS) + set(BALL_MLIR_TESTS + "${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/${BUCKYBALL_BALL_DIR}/workloads/mlir_tests" + ) + if(IS_DIRECTORY "${BALL_MLIR_TESTS}") + add_subdirectory( + "${BALL_MLIR_TESTS}" + "${CMAKE_CURRENT_BINARY_DIR}/balls/${BUCKYBALL_TARGET}/${BUCKYBALL_BALL_DIR}" + ) + endif() + endforeach() +endforeach() +add_dependencies(MLIRTest-all base-mlirtest-build balls-mlir-tests-build) + +# The multicore Ant cases this entry's cores can run. +set(POLY_ANT_CASES + mesh_move + mesh_private + mesh_shared +) +add_subdirectory( + "${BUCKYBALL_REPO_ROOT}/examples/designs/poly/workloads" + design +) diff --git a/examples/designs/poly/sims/device/workloads/kernel/CMakeLists.txt b/examples/designs/poly/sims/device/workloads/kernel/CMakeLists.txt new file mode 100644 index 000000000..2270721e8 --- /dev/null +++ b/examples/designs/poly/sims/device/workloads/kernel/CMakeLists.txt @@ -0,0 +1,10 @@ +# Linux programs installed into the poly rootfs, each with the hart it runs on. +set(BUCKYBALL_KERNEL_PROGRAMS + 0 poly-main-ctest-design-controllers-linux + 0 poly-main-ctest-f32add_groups-linux + 0 poly-main-ctest-design-ant_mxmm-linux + 0 poly-main-ctest-design-ant_shared-linux + 0 poly-main-ctest-design-ant_bank-linux + 0 poly-main-ctest-design-ant_parallel-linux + 0 poly-main-ctest-design-stream-linux + PARENT_SCOPE) diff --git a/examples/designs/poly/sims/ip/uvm/registry.toml b/examples/designs/poly/sims/ip/uvm/registry.toml new file mode 100644 index 000000000..8e8131449 --- /dev/null +++ b/examples/designs/poly/sims/ip/uvm/registry.toml @@ -0,0 +1 @@ +ips = [] diff --git a/examples/designs/poly/stack/device/CMakeLists.txt b/examples/designs/poly/stack/device/CMakeLists.txt new file mode 100644 index 000000000..33cd52904 --- /dev/null +++ b/examples/designs/poly/stack/device/CMakeLists.txt @@ -0,0 +1,38 @@ +# ===----------------------------------------------------------------------=== +# The poly device: its cores, their hardware threads and the runtime that +# schedules jobs onto the cores. +# +# bbdev stack --build "--design poly --device " +# +# builds every thread and core stack of the device first, then configures this +# with DESIGN_CMAKE, generated from design.pb, and the RISC-V Linux toolchain. +# Products, exported in device.cmake for the programs that run on the device: +# runtime, runtime-baremetal, runtime-linux job scheduling over the cores +# engine hardware threads and instruction streams +# bindings how each core runs jobs and each +# hardware thread is loaded +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(poly-device LANGUAGES C CXX) +include("${DESIGN_CMAKE}") +set(CMAKE_POSITION_INDEPENDENT_CODE ON) + +# How each core runs jobs, and how each hardware thread is loaded. Nothing +# references these bindings, so they are linked as objects rather than taken +# from an archive; as link options they come before the libraries they call. +add_library(bindings INTERFACE) +foreach(CORE IN LISTS DESIGN_CORES) + target_link_options(bindings INTERFACE "${DESIGN_CORE_BUILD_${CORE}}/execute.o") +endforeach() +foreach(THREAD IN LISTS DESIGN_THREAD_STACKS) + target_link_options(bindings INTERFACE "${DESIGN_THREAD_BUILD_${THREAD}}/loader.o") +endforeach() +# DMA helpers are sized by the target with the fewest bank lines. +list(GET DESIGN_RUNTIME_TARGETS 0 PARAMS_TARGET) +set(RUNTIME_DEFS ${DESIGN_TARGET_DEFS_${PARAMS_TARGET}}) + +add_subdirectory("${DESIGN_REPO}/stack/engine" engine) +add_subdirectory("${DESIGN_REPO}/stack/runtime" runtime) + +export(TARGETS bindings runtime runtime-baremetal runtime-linux engine + runtime-shared runtime-linux-shared engine-shared FILE device.cmake) diff --git a/examples/designs/poly/stack/device/core/CMakeLists.txt b/examples/designs/poly/stack/device/core/CMakeLists.txt new file mode 100644 index 000000000..ac07de8b4 --- /dev/null +++ b/examples/designs/poly/stack/device/core/CMakeLists.txt @@ -0,0 +1,32 @@ +# ===----------------------------------------------------------------------=== +# Poly's own core: a CPU that runs the attention and ffn hardware threads. +# +# Built by the stack top of each simulation entry after the core's threads, +# configured with DESIGN_CMAKE and CORE. Products: +# execute.o how the core runs jobs (execute.c, through the engine) +# libengine-compiler.so buddy-opt pass plugin of the framework engine +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(poly-core-stack LANGUAGES C CXX) +include("${DESIGN_CMAKE}") + +set(RUNTIME "${DESIGN_REPO}/stack/runtime") +set(ENGINE "${DESIGN_REPO}/stack/engine") +find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) + +# How the core runs jobs; instruction streams use the framework engine. +add_custom_command( + OUTPUT "${CMAKE_BINARY_DIR}/execute.o" + COMMAND "${LINUX_CC}" -O2 -fPIC -ffreestanding -fno-builtin -msmall-data-limit=0 + "-DCORE_SIGNATURE=${DESIGN_DEVICE_CORES_${DESIGN_CORE_HART_${CORE}}_SIGNATURE}ULL" "-I${RUNTIME}/scheduler" + "-I${ENGINE}/include" -c "${CMAKE_CURRENT_SOURCE_DIR}/execute.c" -o "${CMAKE_BINARY_DIR}/execute.o" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/execute.c" "${RUNTIME}/scheduler/runtime.h" + "${ENGINE}/include/engine.h" + VERBATIM) +add_custom_target(execute ALL DEPENDS "${CMAKE_BINARY_DIR}/execute.o") + +# The engine's compiler plugin, for the threads this core runs. +add_subdirectory("${ENGINE}/compiler" engine-compiler) +add_library(engine-compiler-plugin MODULE $) +set_target_properties(engine-compiler-plugin PROPERTIES OUTPUT_NAME engine-compiler + LINKER_LANGUAGE CXX LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") diff --git a/examples/designs/poly/stack/device/core/execute.c b/examples/designs/poly/stack/device/core/execute.c new file mode 100644 index 000000000..76b31878f --- /dev/null +++ b/examples/designs/poly/stack/device/core/execute.c @@ -0,0 +1,23 @@ +#include "engine.h" +#include "runtime.h" + +/* Execution for a core with hardware threads: the job's entry receives the + * core's engine. */ +static int execute(void *context, int (*entry)(void *, void *), + void *argument) { + (void)context; + size_t count = thread_query(0, THREAD_FIELD_COUNT); + if (!count) + __builtin_trap(); + struct engine_slot slots[count]; + struct engine engine; + engine_init(&engine, slots, count); + int status = entry(&engine, argument); + if (engine.active) + __builtin_trap(); + return status; +} + +static const struct runtime_binding binding + __attribute__((section("runtime_bindings"), used)) = {CORE_SIGNATURE, + execute}; diff --git a/examples/designs/poly/workloads/CMakeLists.txt b/examples/designs/poly/workloads/CMakeLists.txt new file mode 100644 index 000000000..2ce389b29 --- /dev/null +++ b/examples/designs/poly/workloads/CMakeLists.txt @@ -0,0 +1,26 @@ +add_custom_target(poly-ctests-build) +set(BUCKYBALL_CTEST_GROUP_TARGET poly-ctests-build) +set(BUCKYBALL_CTEST_TARGET attention) +set(POLY_BOOT_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/device/common/controllers_start.S) +set(BUCKYBALL_DESIGN_COMMON_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/device/core/thread/ctests) +# The cores that run Ant threads: the programs here keep state per such core. +set(POLY_THREAD_CORES 0) +foreach(CORE IN LISTS DESIGN_DEVICE_CORES) + if(NOT "${DESIGN_DEVICE_CORES_${CORE}_THREAD_INDICES}" STREQUAL "") + math(EXPR POLY_THREAD_CORES "${POLY_THREAD_CORES} + 1") + endif() +endforeach() +list(APPEND DESIGN_DEFS POLY_THREAD_CORES=${POLY_THREAD_CORES}) +list(APPEND DESIGN_DEFINES -DPOLY_THREAD_CORES=${POLY_THREAD_CORES}) +add_subdirectory(device) + + +add_custom_target(design-ctests-build DEPENDS poly-ctests-build) + +add_custom_target(design-workloads-build ALL + DEPENDS + design-ctests-build + mlirtest-workloads-build + soctests-build + COMMENT "Building poly workloads" + VERBATIM) diff --git a/examples/designs/poly/workloads/configs/tests.toml b/examples/designs/poly/workloads/configs/tests.toml new file mode 100644 index 000000000..c1c7ceb33 --- /dev/null +++ b/examples/designs/poly/workloads/configs/tests.toml @@ -0,0 +1,2 @@ +controlHart = 0 +reportHart = 0 diff --git a/examples/designs/poly/workloads/device/CMakeLists.txt b/examples/designs/poly/workloads/device/CMakeLists.txt new file mode 100644 index 000000000..19f4ef63a --- /dev/null +++ b/examples/designs/poly/workloads/device/CMakeLists.txt @@ -0,0 +1,3 @@ +add_subdirectory(core) +add_subdirectory(soc) +add_subdirectory(runtime) diff --git a/examples/designs/poly/workloads/device/common/controllers_start.S b/examples/designs/poly/workloads/device/common/controllers_start.S new file mode 100644 index 000000000..b7f41c631 --- /dev/null +++ b/examples/designs/poly/workloads/device/common/controllers_start.S @@ -0,0 +1 @@ +#include "start.S" diff --git a/examples/designs/poly/workloads/device/core/CMakeLists.txt b/examples/designs/poly/workloads/device/core/CMakeLists.txt new file mode 100644 index 000000000..d3db8c656 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/CMakeLists.txt @@ -0,0 +1,5 @@ +add_subdirectory(thread) +add_subdirectory(engine) +add_subdirectory(ball) +set(POLY_ANT_IMAGE_DIR "${POLY_ANT_IMAGE_DIR}" PARENT_SCOPE) +set(POLY_ANT_SOURCE_DIR "${POLY_ANT_SOURCE_DIR}" PARENT_SCOPE) diff --git a/examples/designs/poly/workloads/device/core/ball/CMakeLists.txt b/examples/designs/poly/workloads/device/core/ball/CMakeLists.txt new file mode 100644 index 000000000..e0c194ea5 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/ball/CMakeLists.txt @@ -0,0 +1,5 @@ +set(BUCKYBALL_CTEST_TARGET main) +set(BUCKYBALL_CTEST_GROUP_TARGET poly-ctests-build) +add_subdirectory( + ${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/f32add/workloads/ctests + f32add) diff --git a/examples/designs/poly/workloads/device/core/engine/CMakeLists.txt b/examples/designs/poly/workloads/device/core/engine/CMakeLists.txt new file mode 100644 index 000000000..12923655f --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/CMakeLists.txt @@ -0,0 +1,127 @@ +foreach( + POLY_ANT_CASE + mxmm + shared + bank + bank_rows + parallel + highpa +) + set(POLY_ANT_THREAD attention) + set(POLY_ANT_HEADER ant_${POLY_ANT_CASE}.h) + set(POLY_ANT_HOST_DEFINITIONS) + if(POLY_ANT_CASE STREQUAL "bank" OR POLY_ANT_CASE STREQUAL "bank_rows") + set(POLY_ANT_THREAD ffn) + endif() + set(POLY_ANT_NAME + "${DESIGN_NAME}-${POLY_ANT_THREAD}-ctest-design-ant_${POLY_ANT_CASE}" + ) + set(POLY_ANT_IMAGE ${POLY_ANT_IMAGE_DIR}/${POLY_ANT_NAME}-ant) + set(POLY_ANT_CONTROL_SOURCES) + if( + POLY_ANT_CASE STREQUAL "bank" + OR POLY_ANT_CASE STREQUAL "bank_rows" + OR POLY_ANT_CASE STREQUAL "parallel" + OR POLY_ANT_CASE STREQUAL "shared" + ) + list( + APPEND POLY_ANT_CONTROL_SOURCES + ${CMAKE_CURRENT_SOURCE_DIR}/ant_${POLY_ANT_CASE}_exercise.c + ) + endif() + foreach(SOURCE IN LISTS POLY_ANT_CONTROL_SOURCES) + buckyball_enforce_ctest_line_limit("${SOURCE}") + endforeach() + + set(POLY_ANT_EMBED ${CMAKE_CURRENT_BINARY_DIR}/ant_${POLY_ANT_CASE}_image.S) + enable_language(ASM) + file( + GENERATE OUTPUT ${POLY_ANT_EMBED} + CONTENT + ".section .rodata.ant,\"a\",@progbits\n.balign 8\n.global ant_image, ant_image_end\nant_image:\n.incbin \"${POLY_ANT_IMAGE}.bin\"\nant_image_end:\n.section .note.GNU-stack,\"\",@progbits\n" + ) + set_source_files_properties( + ${POLY_ANT_EMBED} + PROPERTIES OBJECT_DEPENDS "${POLY_ANT_IMAGE}.bin" + ) + set(POLY_ANT_MAIN ant_${POLY_ANT_CASE}_main.c) + + if(POLY_ANT_CASE STREQUAL "mxmm") + set(POLY_ANT_MAIN ant_mxmm_main.cpp) + endif() + buckyball_enforce_ctest_line_limit("${CMAKE_CURRENT_SOURCE_DIR}/${POLY_ANT_MAIN}") + set(POLY_ANT_HOST_NAME + "${DESIGN_NAME}-main-ctest-design-ant_${POLY_ANT_CASE}" + ) + set(BUCKYBALL_CTEST_TARGET ${POLY_ANT_THREAD}) + if(NOT POLY_ANT_CASE STREQUAL "highpa") + add_buckyball_linux_ctest(${POLY_ANT_HOST_NAME} ${CMAKE_CURRENT_SOURCE_DIR} ${POLY_ANT_MAIN}) + target_sources( + ${POLY_ANT_HOST_NAME}-linux + PRIVATE ${POLY_ANT_EMBED} ${POLY_ANT_CONTROL_SOURCES} + ) + target_include_directories( + ${POLY_ANT_HOST_NAME}-linux + PRIVATE ${RUNTIME_INCLUDE_DIR} + ) + target_compile_definitions( + ${POLY_ANT_HOST_NAME}-linux + PRIVATE ${DESIGN_DEFS} + ) + target_link_libraries( + ${POLY_ANT_HOST_NAME}-linux + PRIVATE engine runtime-linux pthread + ) + add_dependencies(${POLY_ANT_HOST_NAME}-linux ${POLY_ANT_NAME}-ant-build) + add_dependencies(poly-ctests-build ${POLY_ANT_HOST_NAME}-linux-build) + endif() + + set(POLY_ANT_HOST_THREAD ${POLY_ANT_THREAD}) + set(POLY_ANT_CONTROL_OBJECTS) + + set(POLY_ANT_HOST_ELF ${POLY_ANT_HOST_NAME}-multicore-baremetal) + set(POLY_ANT_HOST_DEFINES ${DESIGN_TARGET_DEFS_${POLY_ANT_HOST_THREAD}}) + list(TRANSFORM POLY_ANT_HOST_DEFINES PREPEND "-D") + add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${POLY_ANT_HOST_ELF} + COMMAND + ${ELF_CC} -O2 -g -march=rv64gc -mcmodel=medany -msmall-data-limit=0 + -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${BBSIM_LD} ${POLY_ANT_HOST_DEFINITIONS} + -I${POLY_ANT_SOURCE_DIR} + ${POLY_ANT_HOST_DEFINES} + ${DESIGN_DEFINES} -I${RUNTIME_INCLUDE_DIR} -I${WORKLOAD_LIB_DIR} + -I${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/f32add/workloads + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${BUCKYBALL_REPO_ROOT}/stack/engine/include -I${BAREMETAL_DIR} + ${POLY_BOOT_SOURCE} ${CMAKE_CURRENT_SOURCE_DIR}/${POLY_ANT_MAIN} + ${POLY_ANT_CONTROL_SOURCES} ${POLY_ANT_CONTROL_OBJECTS} + ${POLY_ANT_EMBED} ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c -lgcc -o + ${POLY_ANT_HOST_ELF} + DEPENDS + ${POLY_ANT_MAIN} + ${POLY_ANT_SOURCE_DIR}/${POLY_ANT_HEADER} + ${POLY_BOOT_SOURCE} + ${POLY_ANT_EMBED} + ${BAREMETAL_DIR}/start.S + ${BBSIM_LD} + ${POLY_ANT_CONTROL_SOURCES} + ${POLY_ANT_CONTROL_OBJECTS} + ${POLY_ANT_IMAGE}.bin + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/engine/include/thread.h + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.h + ${RUNTIME_INCLUDE_DIR}/bb_cores.h + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM + ) + add_custom_target( + ${POLY_ANT_HOST_NAME}-ctest-build + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${POLY_ANT_HOST_ELF} + ) + add_dependencies(poly-ctests-build ${POLY_ANT_HOST_NAME}-ctest-build) +endforeach() +add_subdirectory(mlir) diff --git a/examples/designs/poly/workloads/device/core/engine/ant_bank_exercise.c b/examples/designs/poly/workloads/device/core/engine/ant_bank_exercise.c new file mode 100644 index 000000000..4c7227943 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/ant_bank_exercise.c @@ -0,0 +1,65 @@ +#include "ant_bank.h" +#include +extern const unsigned char ant_image[], ant_image_end[]; +enum { + WORDS = THREAD_MEM_BANK_ENTRIES * THREAD_MEM_BANK_WIDTH / 64, + SHARED_WORDS = CORE_SHARED_MEM_BANK_ENTRIES * THREAD_MEM_BANK_WIDTH / 64, + INPUT_WORDS = WORDS > SHARED_WORDS ? WORDS : SHARED_WORDS +}; +static uint64_t input[POLY_THREAD_CORES][INPUT_WORDS] + __attribute__((section(".data"), aligned(4096))) = {0}; +static uint64_t private_out[POLY_THREAD_CORES][WORDS] + __attribute__((section(".data"), aligned(4096))) = {0}; +static uint64_t shared_out[POLY_THREAD_CORES][SHARED_WORDS] + __attribute__((section(".data"), aligned(4096))) = {0}; +static uint64_t tail_out[POLY_THREAD_CORES][8] + __attribute__((section(".data"), aligned(4096))) = {0}; +unsigned bank_done[POLY_THREAD_CORES]; +void bank_exercise(unsigned core) { + const unsigned context = thread_find(THREAD_SIGNATURE, 0); + for (unsigned i = 0; i < INPUT_WORDS; ++i) + input[core][i] = (uint64_t)(core + 1) << 32 | (i + 1); + for (unsigned i = 0; i < WORDS; ++i) + private_out[core][i] = 0; + for (unsigned i = 0; i < SHARED_WORDS; ++i) + shared_out[core][i] = 0; + for (unsigned i = 0; i < 8; ++i) + tail_out[core][i] = ~0ULL; + size_t bytes = ant_image_end - ant_image; + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || + thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) + __builtin_trap(); + struct ant_bank_args args = { + (uintptr_t)input[core], (uintptr_t)private_out[core], + (uintptr_t)shared_out[core], (uintptr_t)tail_out[core], + CORE_SHARED_MEM_BANK_ENTRIES}; + thread_tss_write(0, ant_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + thread_tss_write(0, &args, sizeof(args)); + thread_load(context, THREAD_TLS, 0, 0, (sizeof(args) + 7) & ~7u); + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE); + struct thread_task task = {core + 1, + 0, + bytes, + base, + base + + thread_query(context, THREAD_FIELD_TLS_BYTES), + THREAD_SIGNATURE}; + thread_acquire(); + thread_start(context, &task); + int cancelled; + if (thread_wait(context, task.id, &cancelled) || cancelled) + __builtin_trap(); + thread_release(); + for (unsigned i = 0; i < WORDS; ++i) + if (private_out[core][i] != (((uint64_t)(core + 1) << 32) | (i + 1))) + __builtin_trap(); + for (unsigned i = 0; i < SHARED_WORDS; ++i) + if (shared_out[core][i] != (((uint64_t)(core + 1) << 32) | (i + 1))) + __builtin_trap(); + for (unsigned i = 0; i < 8; ++i) + if (tail_out[core][i] != + (i % 2 ? 0 : (((uint64_t)(core + 1) << 32) | (i + 1)))) + __builtin_trap(); + __atomic_store_n(&bank_done[core], 1, __ATOMIC_RELEASE); +} diff --git a/examples/designs/poly/workloads/device/core/engine/ant_bank_main.c b/examples/designs/poly/workloads/device/core/engine/ant_bank_main.c new file mode 100644 index 000000000..09d9e6dbe --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/ant_bank_main.c @@ -0,0 +1,49 @@ +#define _GNU_SOURCE +#include "ant_bank.h" +#include +#ifdef __linux__ +#include +#include +#include +#include +#include +#endif +#ifdef __linux__ +static void *worker(void *argument) { + uintptr_t core = (uintptr_t)argument; + cpu_set_t mask; + CPU_ZERO(&mask); + CPU_SET((core + 1), &mask); + if (pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask)) + abort(); + bank_exercise(core); + return NULL; +} +#endif +int main(void) { +#ifdef __linux__ + if (mlockall(MCL_CURRENT | MCL_FUTURE)) + abort(); + pthread_t threads[POLY_THREAD_CORES]; + for (uintptr_t core = 0; core < POLY_THREAD_CORES; ++core) + if (pthread_create(&threads[core], NULL, worker, (void *)core)) + abort(); + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + if (pthread_join(threads[core], NULL)) + abort(); + printf("Ant bank boundary PASS cores=%u private=%u shared=%u\n", + (unsigned)POLY_THREAD_CORES, THREAD_MEM_BANK_ENTRIES, + CORE_SHARED_MEM_BANK_ENTRIES); +#else + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart == WORKLOAD_REPORT_HART) { + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + while (!__atomic_load_n(&bank_done[core], __ATOMIC_ACQUIRE)) { + } + } else { + bank_exercise(hart - 1); + } +#endif + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/engine/ant_bank_rows_exercise.c b/examples/designs/poly/workloads/device/core/engine/ant_bank_rows_exercise.c new file mode 100644 index 000000000..8af47dfa6 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/ant_bank_rows_exercise.c @@ -0,0 +1,63 @@ +#include "ant_bank_rows.h" +#include +extern const unsigned char ant_image[], ant_image_end[]; +enum { + WORDS = ANT_BANK_ROWS * THREAD_MEM_BANK_WIDTH / 64, + SHARED_WORDS = ANT_BANK_ROWS * THREAD_MEM_BANK_WIDTH / 64 +}; +static uint64_t input[POLY_THREAD_CORES][WORDS] + __attribute__((section(".data"), aligned(4096))) = {0}; +static uint64_t private_out[POLY_THREAD_CORES][WORDS] + __attribute__((section(".data"), aligned(4096))) = {0}; +static uint64_t shared_out[POLY_THREAD_CORES][SHARED_WORDS] + __attribute__((section(".data"), aligned(4096))) = {0}; +static uint64_t tail_out[POLY_THREAD_CORES][8] + __attribute__((section(".data"), aligned(4096))) = {0}; +unsigned bank_rows_done[POLY_THREAD_CORES]; +void bank_rows_exercise(unsigned core) { + const unsigned context = thread_find(THREAD_SIGNATURE, 0); + for (unsigned i = 0; i < WORDS; ++i) + input[core][i] = (uint64_t)(core + 1) << 32 | (i + 1); + for (unsigned i = 0; i < WORDS; ++i) + private_out[core][i] = 0; + for (unsigned i = 0; i < SHARED_WORDS; ++i) + shared_out[core][i] = 0; + for (unsigned i = 0; i < 8; ++i) + tail_out[core][i] = ~0ULL; + size_t bytes = ant_image_end - ant_image; + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || + thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) + __builtin_trap(); + struct ant_bank_rows_args args = { + (uintptr_t)input[core], (uintptr_t)private_out[core], + (uintptr_t)shared_out[core], (uintptr_t)tail_out[core], ANT_BANK_ROWS}; + thread_tss_write(0, ant_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + thread_tss_write(0, &args, sizeof(args)); + thread_load(context, THREAD_TLS, 0, 0, (sizeof(args) + 7) & ~7u); + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE); + struct thread_task task = {core + 1, + 0, + bytes, + base, + base + + thread_query(context, THREAD_FIELD_TLS_BYTES), + THREAD_SIGNATURE}; + thread_acquire(); + thread_start(context, &task); + int cancelled; + if (thread_wait(context, task.id, &cancelled) || cancelled) + __builtin_trap(); + thread_release(); + for (unsigned i = 0; i < WORDS; ++i) + if (private_out[core][i] != (((uint64_t)(core + 1) << 32) | (i + 1))) + __builtin_trap(); + for (unsigned i = 0; i < SHARED_WORDS; ++i) + if (shared_out[core][i] != (((uint64_t)(core + 1) << 32) | (i + 1))) + __builtin_trap(); + for (unsigned i = 0; i < 8; ++i) + if (tail_out[core][i] != + (i % 2 ? 0 : (((uint64_t)(core + 1) << 32) | (i + 1)))) + __builtin_trap(); + __atomic_store_n(&bank_rows_done[core], 1, __ATOMIC_RELEASE); +} diff --git a/examples/designs/poly/workloads/device/core/engine/ant_bank_rows_main.c b/examples/designs/poly/workloads/device/core/engine/ant_bank_rows_main.c new file mode 100644 index 000000000..1a80cc922 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/ant_bank_rows_main.c @@ -0,0 +1,49 @@ +#define _GNU_SOURCE +#include "ant_bank_rows.h" +#include +#ifdef __linux__ +#include +#include +#include +#include +#include +#endif +#ifdef __linux__ +static void *worker(void *argument) { + uintptr_t core = (uintptr_t)argument; + cpu_set_t mask; + CPU_ZERO(&mask); + CPU_SET((core + 1), &mask); + if (pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask)) + abort(); + bank_rows_exercise(core); + return NULL; +} +#endif +int main(void) { +#ifdef __linux__ + if (mlockall(MCL_CURRENT | MCL_FUTURE)) + abort(); + pthread_t threads[POLY_THREAD_CORES]; + for (uintptr_t core = 0; core < POLY_THREAD_CORES; ++core) + if (pthread_create(&threads[core], NULL, worker, (void *)core)) + abort(); + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + if (pthread_join(threads[core], NULL)) + abort(); + printf("Ant bank boundary PASS cores=%u private=%u shared=%u\n", + (unsigned)POLY_THREAD_CORES, THREAD_MEM_BANK_ENTRIES, + CORE_SHARED_MEM_BANK_ENTRIES); +#else + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart == WORKLOAD_REPORT_HART) { + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + while (!__atomic_load_n(&bank_rows_done[core], __ATOMIC_ACQUIRE)) { + } + } else { + bank_rows_exercise(hart - 1); + } +#endif + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/engine/ant_highpa_main.c b/examples/designs/poly/workloads/device/core/engine/ant_highpa_main.c new file mode 100644 index 000000000..4bf3781b9 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/ant_highpa_main.c @@ -0,0 +1,66 @@ +#include "ant_highpa.h" +#include +extern const unsigned char ant_image[], ant_image_end[]; +static unsigned done[POLY_THREAD_CORES]; +static void exercise(unsigned core) { + const unsigned context = thread_find(THREAD_SIGNATURE, 0); + uintptr_t high = 0x180200000ULL + core * 0x20000ULL; + volatile uint64_t *input = (void *)high, *output = (void *)(high + 0x1000); + volatile uint64_t *low = (void *)(high & 0xffffffffULL); + volatile uint64_t *sentinel = (void *)((high + 0x1000) & 0xffffffffULL); + const uint64_t marker = 0xdeadface12345678ULL; + uint64_t first = 0x1234567800000000ULL | ((uint64_t)core << 16); + uint64_t second = 0xfedcba9800000000ULL | ((uint64_t)core << 16); + for (unsigned i = 0; i < 4; ++i) { + low[i] = second + i; + sentinel[i] = marker; + input[i] = first + i; + output[i] = 0; + } + for (unsigned i = 0; i < 4; ++i) + if (input[i] != first + i || low[i] != second + i || sentinel[i] != marker) + __builtin_trap(); + size_t bytes = ant_image_end - ant_image; + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || + thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) + __builtin_trap(); + thread_tss_write(0, ant_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE); + for (unsigned phase = 0; phase < 2; ++phase) { + struct ant_highpa_args args = {phase ? (uintptr_t)low : high, + high + 0x1000}; + thread_tss_write(0, &args, sizeof(args)); + thread_load(context, THREAD_TLS, 0, 0, (sizeof(args) + 7) & ~7u); + struct thread_task task = { + core * 2 + phase + 1, + 0, + bytes, + base, + base + thread_query(context, THREAD_FIELD_TLS_BYTES), + THREAD_SIGNATURE}; + thread_acquire(); + thread_start(context, &task); + int cancelled; + if (thread_wait(context, task.id, &cancelled) || cancelled) + __builtin_trap(); + thread_release(); + for (unsigned i = 0; i < 4; ++i) + if (output[i] != (phase ? second : first) + i || sentinel[i] != marker || + input[i] != first + i || low[i] != second + i) + __builtin_trap(); + } + __atomic_store_n(&done[core], 1, __ATOMIC_RELEASE); +} +int main(void) { + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart == WORKLOAD_REPORT_HART) { + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + while (!__atomic_load_n(&done[core], __ATOMIC_ACQUIRE)) { + } + } else { + exercise(hart - 1); + } + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/engine/ant_mxmm_main.cpp b/examples/designs/poly/workloads/device/core/engine/ant_mxmm_main.cpp new file mode 100644 index 000000000..ecde76a90 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/ant_mxmm_main.cpp @@ -0,0 +1,98 @@ +#define _GNU_SOURCE +#include "ant_mxmm.h" +#include +#ifdef __linux__ +#include +#include +#include +#include +#include +#endif +extern const unsigned char ant_image[], ant_image_end[]; +struct matrices { + float a[POLY_THREAD_CORES][ANT_M * ANT_K]; + float b[POLY_THREAD_CORES][ANT_N * ANT_K]; + float out[POLY_THREAD_CORES][ANT_M * ANT_N]; +}; +static constexpr matrices prepare() { + matrices data{}; + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) { + for (unsigned row = 0; row < ANT_M; ++row) + for (unsigned k = 0; k < ANT_K; ++k) + data.a[core][row * ANT_K + k] = row + core + 1; + for (unsigned col = 0; col < ANT_N; ++col) + for (unsigned k = 0; k < ANT_K; ++k) + data.b[core][col * ANT_K + k] = col + 1; + for (unsigned i = 0; i < ANT_M * ANT_N; ++i) + data.out[core][i] = -1; + } + return data; +} +alignas(64) static matrices data = prepare(); +static unsigned done[POLY_THREAD_CORES]; +static void exercise(unsigned core) { + const uint32_t context = thread_find(THREAD_SIGNATURE, 0); + size_t bytes = ant_image_end - ant_image; + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || + thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) + __builtin_trap(); + struct ant_mxmm_args args = {(uintptr_t)data.a[core], (uintptr_t)data.b[core], + (uintptr_t)data.out[core]}; + thread_tss_write(0, ant_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + thread_tss_write(0, &args, sizeof(args)); + thread_load(context, THREAD_TLS, 0, 0, (sizeof(args) + 7) & ~7u); + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE), + capacity = thread_query(context, THREAD_FIELD_TLS_BYTES); + struct thread_task task = {core + 1, 0, bytes, base, base + capacity, + THREAD_SIGNATURE}; + thread_acquire(); + thread_start(context, &task); + int cancelled; + if (thread_wait(context, task.id, &cancelled) || cancelled) + __builtin_trap(); + thread_release(); + for (unsigned row = 0; row < ANT_M; ++row) + for (unsigned col = 0; col < ANT_N; ++col) + if (data.out[core][row * ANT_N + col] != + (float)(ANT_K * (row + core + 1) * (col + 1))) + __builtin_trap(); + __atomic_store_n(&done[core], 1, __ATOMIC_RELEASE); +} +#ifdef __linux__ +static void *worker(void *argument) { + uintptr_t core = (uintptr_t)argument; + cpu_set_t mask; + CPU_ZERO(&mask); + CPU_SET((core + 1), &mask); + if (pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask)) + abort(); + exercise(core); + return NULL; +} +#endif +int main(void) { +#ifdef __linux__ + if (mlockall(MCL_CURRENT | MCL_FUTURE)) + abort(); + pthread_t threads[POLY_THREAD_CORES]; + for (uintptr_t core = 0; core < POLY_THREAD_CORES; ++core) + if (pthread_create(&threads[core], NULL, worker, (void *)core)) + abort(); + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + if (pthread_join(threads[core], NULL)) + abort(); + printf("Ant Mxmm PASS cores=%u\n", (unsigned)POLY_THREAD_CORES); +#else + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart == WORKLOAD_REPORT_HART) { + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + while (!__atomic_load_n(&done[core], __ATOMIC_ACQUIRE)) { + } + } else { + exercise(hart - 1); + } +#endif + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/engine/ant_parallel_exercise.c b/examples/designs/poly/workloads/device/core/engine/ant_parallel_exercise.c new file mode 100644 index 000000000..eb39f8319 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/ant_parallel_exercise.c @@ -0,0 +1,80 @@ +#include "ant_parallel.h" +#include +extern const unsigned char ant_image[], ant_image_end[]; +static uint64_t input[POLY_THREAD_CORES][2][512] + __attribute__((aligned(4096), section(".noinit"))); +static uint64_t output[POLY_THREAD_CORES][2][512] + __attribute__((aligned(4096), section(".noinit"))); +unsigned parallel_done[POLY_THREAD_CORES]; +static unsigned reached[3] __attribute__((aligned(64))); +static void barrier(unsigned phase) { + __atomic_fetch_add(&reached[phase], 1, __ATOMIC_ACQ_REL); + while (__atomic_load_n(&reached[phase], __ATOMIC_ACQUIRE) != + POLY_THREAD_CORES) { + } +} +void parallel_exercise(unsigned core) { + const uint32_t contexts[] = {thread_find(THREAD_SIGNATURE, 0), + thread_find(THREAD_SIGNATURE, 1)}; + size_t bytes = ant_image_end - ant_image; + uint64_t shared = thread_query(contexts[0], THREAD_FIELD_TSS_BASE), zero = 0; + struct thread_task tasks[2]; + for (unsigned context = 0; context < 2; ++context) { + for (unsigned word = 0; word < 2; ++word) { + input[core][context][word] = + ((uint64_t)(core + 1) << 32) | (context * 2 + word + 1); + output[core][context][word] = ~0ULL; + } + uint64_t base = thread_query(contexts[context], THREAD_FIELD_TLS_BASE); + if (bytes > thread_query(contexts[context], THREAD_FIELD_CODE_BYTES) || + thread_query(contexts[context], THREAD_FIELD_SIGNATURE) != + THREAD_SIGNATURE) + __builtin_trap(); + struct ant_parallel_args args = {shared, context, + (uintptr_t)input[core][context], + (uintptr_t)output[core][context]}; + thread_tss_write(0, ant_image, bytes); + thread_load(contexts[context], THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + thread_tss_write(0, &args, sizeof(args)); + thread_load(contexts[context], THREAD_TLS, 0, 0, (sizeof(args) + 7) & ~7u); + tasks[context] = (struct thread_task){ + context + 1, + 0, + bytes, + base, + base + thread_query(contexts[context], THREAD_FIELD_TLS_BYTES), + THREAD_SIGNATURE}; + } + thread_tss_write(0, &zero, sizeof(zero)); + barrier(0); + unsigned peer = (core + 1) % POLY_THREAD_CORES; + for (unsigned context = 0; context < 2; ++context) + for (unsigned word = 0; word < 2; ++word) + if (*(volatile uint64_t *)&output[peer][context][word] != ~0ULL) + __builtin_trap(); + barrier(1); + thread_acquire(); + thread_start(contexts[0], &tasks[0]); + struct ant_parallel_args pending = {shared, 1, (uintptr_t)input[core][1], + (uintptr_t)output[core][1]}; + thread_tss_write(64, &pending, sizeof(pending)); + thread_load(contexts[1], THREAD_TLS, 0, 64, sizeof(pending)); + thread_start(contexts[1], &tasks[1]); + for (unsigned context = 0; context < 2; ++context) { + int cancelled; + if (thread_wait(contexts[context], tasks[context].id, &cancelled) || + cancelled) + __builtin_trap(); + for (unsigned word = 0; word < 2; ++word) + if (output[core][context][word] != input[core][context][word]) + __builtin_trap(); + } + thread_release(); + barrier(2); + for (unsigned context = 0; context < 2; ++context) + for (unsigned word = 0; word < 2; ++word) + if (*(volatile uint64_t *)&output[peer][context][word] != + input[peer][context][word]) + __builtin_trap(); + __atomic_store_n(¶llel_done[core], 1, __ATOMIC_RELEASE); +} diff --git a/examples/designs/poly/workloads/device/core/engine/ant_parallel_main.c b/examples/designs/poly/workloads/device/core/engine/ant_parallel_main.c new file mode 100644 index 000000000..e935768ca --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/ant_parallel_main.c @@ -0,0 +1,48 @@ +#define _GNU_SOURCE +#include "ant_parallel.h" +#include +#ifdef __linux__ +#include +#include +#include +#include +#include +#endif +#ifdef __linux__ +static void *worker(void *argument) { + uintptr_t core = (uintptr_t)argument; + cpu_set_t mask; + CPU_ZERO(&mask); + CPU_SET((core + 1), &mask); + if (pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask)) + abort(); + parallel_exercise(core); + return NULL; +} +#endif +int main(void) { +#ifdef __linux__ + if (mlockall(MCL_CURRENT | MCL_FUTURE)) + abort(); + pthread_t threads[POLY_THREAD_CORES]; + for (uintptr_t core = 0; core < POLY_THREAD_CORES; ++core) + if (pthread_create(&threads[core], NULL, worker, (void *)core)) + abort(); + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + if (pthread_join(threads[core], NULL)) + abort(); + printf("Ant concurrent DMA/cache sharing PASS cores=%u contexts=2\n", + (unsigned)POLY_THREAD_CORES); +#else + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart == WORKLOAD_REPORT_HART) { + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + while (!__atomic_load_n(¶llel_done[core], __ATOMIC_ACQUIRE)) { + } + } else { + parallel_exercise(hart - 1); + } +#endif + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/engine/ant_shared_exercise.c b/examples/designs/poly/workloads/device/core/engine/ant_shared_exercise.c new file mode 100644 index 000000000..86786ec17 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/ant_shared_exercise.c @@ -0,0 +1,68 @@ +#include "ant_shared.h" +#include +extern const unsigned char ant_image[], ant_image_end[]; +unsigned shared_done[POLY_THREAD_CORES]; +void shared_exercise(unsigned core) { + const uint32_t contexts[] = {thread_find(THREAD_SIGNATURE, 0), + thread_find(THREAD_SIGNATURE, 1)}; + size_t bytes = ant_image_end - ant_image; + uint64_t value = 100 + core, + shared = thread_query(contexts[0], THREAD_FIELD_TSS_BASE), + zero[2] = {0}; + struct thread_task tasks[2]; + for (unsigned context = 0; context < 2; ++context) { + uint64_t base = thread_query(contexts[context], THREAD_FIELD_TLS_BASE); + if (bytes > thread_query(contexts[context], THREAD_FIELD_CODE_BYTES) || + thread_query(contexts[context], THREAD_FIELD_SIGNATURE) != + THREAD_SIGNATURE) + __builtin_trap(); + tasks[context] = (struct thread_task){ + context + 1, + 0, + bytes, + base, + base + thread_query(contexts[context], THREAD_FIELD_TLS_BYTES), + THREAD_SIGNATURE}; + struct ant_shared_args args = {shared, context == 0, context ? 0 : value}; + thread_tss_write(0, ant_image, bytes); + thread_load(contexts[context], THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + thread_tss_write(0, &args, sizeof(args)); + thread_load(contexts[context], THREAD_TLS, 0, 0, (sizeof(args) + 7) & ~7u); + } + thread_tss_write(0, zero, sizeof(zero)); + thread_acquire(); + thread_start(contexts[1], &tasks[1]); + thread_start(contexts[0], &tasks[0]); + for (unsigned context = 0; context < 2; ++context) { + int cancelled; + if (thread_wait(contexts[context], tasks[context].id, &cancelled) != + value || + cancelled) + __builtin_trap(); + } + thread_release(); + for (unsigned context = 0; context < 2; ++context) { + thread_store(contexts[context], THREAD_TLS, 16, 0, 8); + if (thread_tss_read(0) != value) + __builtin_trap(); + } + thread_tss_write(0, zero, sizeof(zero)); + thread_acquire(); + thread_start(contexts[1], &tasks[1]); + thread_cancel(contexts[1]); + int cancelled; + thread_wait(contexts[1], tasks[1].id, &cancelled); + if (!cancelled) + __builtin_trap(); + thread_release(); + uint64_t ready[2] = {value + 7, 1}; + thread_tss_write(0, ready, sizeof(ready)); + ++tasks[1].id; + thread_acquire(); + thread_start(contexts[1], &tasks[1]); + if (thread_wait(contexts[1], tasks[1].id, &cancelled) != value + 7 || + cancelled) + __builtin_trap(); + thread_release(); + __atomic_store_n(&shared_done[core], 1, __ATOMIC_RELEASE); +} diff --git a/examples/designs/poly/workloads/device/core/engine/ant_shared_main.c b/examples/designs/poly/workloads/device/core/engine/ant_shared_main.c new file mode 100644 index 000000000..0c2243e06 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/ant_shared_main.c @@ -0,0 +1,43 @@ +#define _GNU_SOURCE +#include "ant_shared.h" +#include +#ifdef __linux__ +#include +#include +#include +#include +#endif +#ifdef __linux__ +static void *worker(void *argument) { + uintptr_t core = (uintptr_t)argument; + cpu_set_t mask; + CPU_ZERO(&mask); + CPU_SET((core + 1), &mask); + if (pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask)) + abort(); + shared_exercise(core); + return NULL; +} +#endif +int main(void) { +#ifdef __linux__ + pthread_t threads[POLY_THREAD_CORES]; + for (uintptr_t core = 0; core < POLY_THREAD_CORES; ++core) + if (pthread_create(&threads[core], NULL, worker, (void *)core)) + abort(); + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + if (pthread_join(threads[core], NULL)) + abort(); + printf("Ant shared/cancel PASS cores=%u\n", (unsigned)POLY_THREAD_CORES); +#else + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart != WORKLOAD_REPORT_HART) + shared_exercise(hart - 1); + else + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + while (!__atomic_load_n(&shared_done[core], __ATOMIC_ACQUIRE)) { + } +#endif + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/engine/mlir/CMakeLists.txt b/examples/designs/poly/workloads/device/core/engine/mlir/CMakeLists.txt new file mode 100644 index 000000000..d3d38fa8c --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/mlir/CMakeLists.txt @@ -0,0 +1,437 @@ +# Host programs of the MxmmBall and MxquantBall MLIR tests: each embeds the +# thread image thread/mlir built for the test, runs it on the attention thread +# of every core and checks what it wrote. +set(POLY_MLIR_HOST_DEFINES ${DESIGN_TARGET_DEFS_attention}) +list(TRANSFORM POLY_MLIR_HOST_DEFINES PREPEND "-D") +set(POLY_MLIR_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../thread/mlir") +enable_language(ASM) + +# ball_mxmm_mxfp8_chain +file( + GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/ball_mxmm_mxfp8_chain_image.S + CONTENT + ".section .rodata.thread,\"a\",@progbits\n.balign 8\n.global thread_image, thread_image_end\nthread_image:\n.incbin \"${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_chain.bin\"\nthread_image_end:\n.section .note.GNU-stack,\"\",@progbits\n" +) +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_mxfp8_chain-multicore-baremetal + COMMAND + ${ELF_CC} -O2 -g -march=rv64gc -mcmodel=medany -msmall-data-limit=0 + -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${BBSIM_LD} -I${POLY_MLIR_SOURCE_DIR} ${POLY_MLIR_HOST_DEFINES} + ${DESIGN_DEFINES} -I${RUNTIME_INCLUDE_DIR} -I${WORKLOAD_LIB_DIR} + -I${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/mxquant/workloads + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${BUCKYBALL_REPO_ROOT}/stack/engine/include -I${BAREMETAL_DIR} + ${POLY_BOOT_SOURCE} ${CMAKE_CURRENT_SOURCE_DIR}/ball_mxmm_mxfp8_chain_main.cpp + ${CMAKE_CURRENT_BINARY_DIR}/ball_mxmm_mxfp8_chain_image.S + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c -lgcc -o + ${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_mxfp8_chain-multicore-baremetal + DEPENDS + ball_mxmm_mxfp8_chain_main.cpp + ${POLY_MLIR_SOURCE_DIR}/mlir.h + ${CMAKE_CURRENT_BINARY_DIR}/ball_mxmm_mxfp8_chain_image.S + ${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_chain.bin + ${POLY_BOOT_SOURCE} + ${BBSIM_LD} + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/engine/include/thread.h + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c + ${RUNTIME_INCLUDE_DIR}/bb_cores.h + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM +) +add_custom_target( + ${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_mxfp8_chain-ctest-build + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_mxfp8_chain-multicore-baremetal +) +add_dependencies(${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_mxfp8_chain-ctest-build ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_chain-thread-build) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_mxfp8_chain-ctest-build) + +# ball_mxmm_mxfp8_window +file( + GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/ball_mxmm_mxfp8_window_image.S + CONTENT + ".section .rodata.thread,\"a\",@progbits\n.balign 8\n.global thread_image, thread_image_end\nthread_image:\n.incbin \"${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_window.bin\"\nthread_image_end:\n.section .note.GNU-stack,\"\",@progbits\n" +) +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_mxfp8_window-multicore-baremetal + COMMAND + ${ELF_CC} -O2 -g -march=rv64gc -mcmodel=medany -msmall-data-limit=0 + -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${BBSIM_LD} -I${POLY_MLIR_SOURCE_DIR} ${POLY_MLIR_HOST_DEFINES} + ${DESIGN_DEFINES} -I${RUNTIME_INCLUDE_DIR} -I${WORKLOAD_LIB_DIR} + -I${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/mxquant/workloads + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${BUCKYBALL_REPO_ROOT}/stack/engine/include -I${BAREMETAL_DIR} + ${POLY_BOOT_SOURCE} ${CMAKE_CURRENT_SOURCE_DIR}/ball_mxmm_mxfp8_window_main.cpp + ${CMAKE_CURRENT_BINARY_DIR}/ball_mxmm_mxfp8_window_image.S + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c -lgcc -o + ${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_mxfp8_window-multicore-baremetal + DEPENDS + ball_mxmm_mxfp8_window_main.cpp + ${POLY_MLIR_SOURCE_DIR}/mlir.h + ${CMAKE_CURRENT_BINARY_DIR}/ball_mxmm_mxfp8_window_image.S + ${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_window.bin + ${POLY_BOOT_SOURCE} + ${BBSIM_LD} + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/engine/include/thread.h + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c + ${RUNTIME_INCLUDE_DIR}/bb_cores.h + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM +) +add_custom_target( + ${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_mxfp8_window-ctest-build + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_mxfp8_window-multicore-baremetal +) +add_dependencies(${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_mxfp8_window-ctest-build ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_window-thread-build) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_mxfp8_window-ctest-build) + +# ball_mxmm_fma32 +file( + GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/ball_mxmm_fma32_image.S + CONTENT + ".section .rodata.thread,\"a\",@progbits\n.balign 8\n.global thread_image, thread_image_end\nthread_image:\n.incbin \"${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-ball_mxmm_fma32.bin\"\nthread_image_end:\n.section .note.GNU-stack,\"\",@progbits\n" +) +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_fma32-multicore-baremetal + COMMAND + ${ELF_CC} -O2 -g -march=rv64gc -mcmodel=medany -msmall-data-limit=0 + -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${BBSIM_LD} -I${POLY_MLIR_SOURCE_DIR} ${POLY_MLIR_HOST_DEFINES} + ${DESIGN_DEFINES} -I${RUNTIME_INCLUDE_DIR} -I${WORKLOAD_LIB_DIR} + -I${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/mxquant/workloads + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${BUCKYBALL_REPO_ROOT}/stack/engine/include -I${BAREMETAL_DIR} + ${POLY_BOOT_SOURCE} ${CMAKE_CURRENT_SOURCE_DIR}/ball_mxmm_fma32_main.cpp + ${CMAKE_CURRENT_BINARY_DIR}/ball_mxmm_fma32_image.S + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c -lgcc -o + ${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_fma32-multicore-baremetal + DEPENDS + ball_mxmm_fma32_main.cpp + ${POLY_MLIR_SOURCE_DIR}/mlir.h + ${CMAKE_CURRENT_BINARY_DIR}/ball_mxmm_fma32_image.S + ${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-ball_mxmm_fma32.bin + ${POLY_BOOT_SOURCE} + ${BBSIM_LD} + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/engine/include/thread.h + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c + ${RUNTIME_INCLUDE_DIR}/bb_cores.h + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM +) +add_custom_target( + ${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_fma32-ctest-build + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_fma32-multicore-baremetal +) +add_dependencies(${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_fma32-ctest-build ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_fma32-thread-build) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_fma32-ctest-build) + +# ball_mxmm_f32 +file( + GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/ball_mxmm_f32_image.S + CONTENT + ".section .rodata.thread,\"a\",@progbits\n.balign 8\n.global thread_image, thread_image_end\nthread_image:\n.incbin \"${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-ball_mxmm_f32.bin\"\nthread_image_end:\n.section .note.GNU-stack,\"\",@progbits\n" +) +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_f32-multicore-baremetal + COMMAND + ${ELF_CC} -O2 -g -march=rv64gc -mcmodel=medany -msmall-data-limit=0 + -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${BBSIM_LD} -I${POLY_MLIR_SOURCE_DIR} ${POLY_MLIR_HOST_DEFINES} + ${DESIGN_DEFINES} -I${RUNTIME_INCLUDE_DIR} -I${WORKLOAD_LIB_DIR} + -I${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/mxquant/workloads + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${BUCKYBALL_REPO_ROOT}/stack/engine/include -I${BAREMETAL_DIR} + ${POLY_BOOT_SOURCE} ${CMAKE_CURRENT_SOURCE_DIR}/ball_mxmm_f32_main.cpp + ${CMAKE_CURRENT_BINARY_DIR}/ball_mxmm_f32_image.S + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c -lgcc -o + ${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_f32-multicore-baremetal + DEPENDS + ball_mxmm_f32_main.cpp + ${POLY_MLIR_SOURCE_DIR}/mlir.h + ${CMAKE_CURRENT_BINARY_DIR}/ball_mxmm_f32_image.S + ${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-ball_mxmm_f32.bin + ${POLY_BOOT_SOURCE} + ${BBSIM_LD} + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/engine/include/thread.h + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c + ${RUNTIME_INCLUDE_DIR}/bb_cores.h + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM +) +add_custom_target( + ${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_f32-ctest-build + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_f32-multicore-baremetal +) +add_dependencies(${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_f32-ctest-build ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_f32-thread-build) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_f32-ctest-build) + +# ball_mxquant_two_blocks +file( + GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/ball_mxquant_two_blocks_image.S + CONTENT + ".section .rodata.thread,\"a\",@progbits\n.balign 8\n.global thread_image, thread_image_end\nthread_image:\n.incbin \"${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-ball_mxquant_two_blocks.bin\"\nthread_image_end:\n.section .note.GNU-stack,\"\",@progbits\n" +) +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_ball_mxquant_two_blocks-multicore-baremetal + COMMAND + ${ELF_CC} -O2 -g -march=rv64gc -mcmodel=medany -msmall-data-limit=0 + -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${BBSIM_LD} -I${POLY_MLIR_SOURCE_DIR} ${POLY_MLIR_HOST_DEFINES} + ${DESIGN_DEFINES} -I${RUNTIME_INCLUDE_DIR} -I${WORKLOAD_LIB_DIR} + -I${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/mxquant/workloads + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${BUCKYBALL_REPO_ROOT}/stack/engine/include -I${BAREMETAL_DIR} + ${POLY_BOOT_SOURCE} ${CMAKE_CURRENT_SOURCE_DIR}/ball_mxquant_two_blocks_main.cpp + ${CMAKE_CURRENT_BINARY_DIR}/ball_mxquant_two_blocks_image.S + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c -lgcc -o + ${DESIGN_NAME}-main-ctest-design-mlir_ball_mxquant_two_blocks-multicore-baremetal + DEPENDS + ball_mxquant_two_blocks_main.cpp + ${POLY_MLIR_SOURCE_DIR}/mlir.h + ${CMAKE_CURRENT_BINARY_DIR}/ball_mxquant_two_blocks_image.S + ${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-ball_mxquant_two_blocks.bin + ${POLY_BOOT_SOURCE} + ${BBSIM_LD} + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/engine/include/thread.h + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c + ${RUNTIME_INCLUDE_DIR}/bb_cores.h + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM +) +add_custom_target( + ${DESIGN_NAME}-main-ctest-design-mlir_ball_mxquant_two_blocks-ctest-build + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_ball_mxquant_two_blocks-multicore-baremetal +) +add_dependencies(${DESIGN_NAME}-main-ctest-design-mlir_ball_mxquant_two_blocks-ctest-build ${DESIGN_NAME}-attention-mlirtest-ball_mxquant_two_blocks-thread-build) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-main-ctest-design-mlir_ball_mxquant_two_blocks-ctest-build) + +# bank_mxmm_mxfp8_chain +file( + GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/bank_mxmm_mxfp8_chain_image.S + CONTENT + ".section .rodata.thread,\"a\",@progbits\n.balign 8\n.global thread_image, thread_image_end\nthread_image:\n.incbin \"${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_chain.bin\"\nthread_image_end:\n.section .note.GNU-stack,\"\",@progbits\n" +) +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_mxfp8_chain-multicore-baremetal + COMMAND + ${ELF_CC} -O2 -g -march=rv64gc -mcmodel=medany -msmall-data-limit=0 + -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${BBSIM_LD} -I${POLY_MLIR_SOURCE_DIR} ${POLY_MLIR_HOST_DEFINES} + ${DESIGN_DEFINES} -I${RUNTIME_INCLUDE_DIR} -I${WORKLOAD_LIB_DIR} + -I${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/mxquant/workloads + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${BUCKYBALL_REPO_ROOT}/stack/engine/include -I${BAREMETAL_DIR} + ${POLY_BOOT_SOURCE} ${CMAKE_CURRENT_SOURCE_DIR}/bank_mxmm_mxfp8_chain_main.cpp + ${CMAKE_CURRENT_BINARY_DIR}/bank_mxmm_mxfp8_chain_image.S + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c -lgcc -o + ${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_mxfp8_chain-multicore-baremetal + DEPENDS + bank_mxmm_mxfp8_chain_main.cpp + ${POLY_MLIR_SOURCE_DIR}/mlir.h + ${CMAKE_CURRENT_BINARY_DIR}/bank_mxmm_mxfp8_chain_image.S + ${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_chain.bin + ${POLY_BOOT_SOURCE} + ${BBSIM_LD} + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/engine/include/thread.h + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c + ${RUNTIME_INCLUDE_DIR}/bb_cores.h + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM +) +add_custom_target( + ${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_mxfp8_chain-ctest-build + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_mxfp8_chain-multicore-baremetal +) +add_dependencies(${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_mxfp8_chain-ctest-build ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_chain-thread-build) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_mxfp8_chain-ctest-build) + +# bank_mxmm_mxfp8_window +file( + GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/bank_mxmm_mxfp8_window_image.S + CONTENT + ".section .rodata.thread,\"a\",@progbits\n.balign 8\n.global thread_image, thread_image_end\nthread_image:\n.incbin \"${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_window.bin\"\nthread_image_end:\n.section .note.GNU-stack,\"\",@progbits\n" +) +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_mxfp8_window-multicore-baremetal + COMMAND + ${ELF_CC} -O2 -g -march=rv64gc -mcmodel=medany -msmall-data-limit=0 + -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${BBSIM_LD} -I${POLY_MLIR_SOURCE_DIR} ${POLY_MLIR_HOST_DEFINES} + ${DESIGN_DEFINES} -I${RUNTIME_INCLUDE_DIR} -I${WORKLOAD_LIB_DIR} + -I${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/mxquant/workloads + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${BUCKYBALL_REPO_ROOT}/stack/engine/include -I${BAREMETAL_DIR} + ${POLY_BOOT_SOURCE} ${CMAKE_CURRENT_SOURCE_DIR}/bank_mxmm_mxfp8_window_main.cpp + ${CMAKE_CURRENT_BINARY_DIR}/bank_mxmm_mxfp8_window_image.S + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c -lgcc -o + ${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_mxfp8_window-multicore-baremetal + DEPENDS + bank_mxmm_mxfp8_window_main.cpp + ${POLY_MLIR_SOURCE_DIR}/mlir.h + ${CMAKE_CURRENT_BINARY_DIR}/bank_mxmm_mxfp8_window_image.S + ${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_window.bin + ${POLY_BOOT_SOURCE} + ${BBSIM_LD} + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/engine/include/thread.h + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c + ${RUNTIME_INCLUDE_DIR}/bb_cores.h + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM +) +add_custom_target( + ${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_mxfp8_window-ctest-build + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_mxfp8_window-multicore-baremetal +) +add_dependencies(${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_mxfp8_window-ctest-build ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_window-thread-build) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_mxfp8_window-ctest-build) + +# bank_mxmm_fma32 +file( + GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/bank_mxmm_fma32_image.S + CONTENT + ".section .rodata.thread,\"a\",@progbits\n.balign 8\n.global thread_image, thread_image_end\nthread_image:\n.incbin \"${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-bank_mxmm_fma32.bin\"\nthread_image_end:\n.section .note.GNU-stack,\"\",@progbits\n" +) +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_fma32-multicore-baremetal + COMMAND + ${ELF_CC} -O2 -g -march=rv64gc -mcmodel=medany -msmall-data-limit=0 + -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${BBSIM_LD} -I${POLY_MLIR_SOURCE_DIR} ${POLY_MLIR_HOST_DEFINES} + ${DESIGN_DEFINES} -I${RUNTIME_INCLUDE_DIR} -I${WORKLOAD_LIB_DIR} + -I${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/mxquant/workloads + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${BUCKYBALL_REPO_ROOT}/stack/engine/include -I${BAREMETAL_DIR} + ${POLY_BOOT_SOURCE} ${CMAKE_CURRENT_SOURCE_DIR}/bank_mxmm_fma32_main.cpp + ${CMAKE_CURRENT_BINARY_DIR}/bank_mxmm_fma32_image.S + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c -lgcc -o + ${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_fma32-multicore-baremetal + DEPENDS + bank_mxmm_fma32_main.cpp + ${POLY_MLIR_SOURCE_DIR}/mlir.h + ${CMAKE_CURRENT_BINARY_DIR}/bank_mxmm_fma32_image.S + ${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-bank_mxmm_fma32.bin + ${POLY_BOOT_SOURCE} + ${BBSIM_LD} + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/engine/include/thread.h + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c + ${RUNTIME_INCLUDE_DIR}/bb_cores.h + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM +) +add_custom_target( + ${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_fma32-ctest-build + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_fma32-multicore-baremetal +) +add_dependencies(${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_fma32-ctest-build ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_fma32-thread-build) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_fma32-ctest-build) + +# bank_mxmm_f32 +file( + GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/bank_mxmm_f32_image.S + CONTENT + ".section .rodata.thread,\"a\",@progbits\n.balign 8\n.global thread_image, thread_image_end\nthread_image:\n.incbin \"${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-bank_mxmm_f32.bin\"\nthread_image_end:\n.section .note.GNU-stack,\"\",@progbits\n" +) +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_f32-multicore-baremetal + COMMAND + ${ELF_CC} -O2 -g -march=rv64gc -mcmodel=medany -msmall-data-limit=0 + -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${BBSIM_LD} -I${POLY_MLIR_SOURCE_DIR} ${POLY_MLIR_HOST_DEFINES} + ${DESIGN_DEFINES} -I${RUNTIME_INCLUDE_DIR} -I${WORKLOAD_LIB_DIR} + -I${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/mxquant/workloads + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${BUCKYBALL_REPO_ROOT}/stack/engine/include -I${BAREMETAL_DIR} + ${POLY_BOOT_SOURCE} ${CMAKE_CURRENT_SOURCE_DIR}/bank_mxmm_f32_main.cpp + ${CMAKE_CURRENT_BINARY_DIR}/bank_mxmm_f32_image.S + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c -lgcc -o + ${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_f32-multicore-baremetal + DEPENDS + bank_mxmm_f32_main.cpp + ${POLY_MLIR_SOURCE_DIR}/mlir.h + ${CMAKE_CURRENT_BINARY_DIR}/bank_mxmm_f32_image.S + ${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-bank_mxmm_f32.bin + ${POLY_BOOT_SOURCE} + ${BBSIM_LD} + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/engine/include/thread.h + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c + ${RUNTIME_INCLUDE_DIR}/bb_cores.h + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM +) +add_custom_target( + ${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_f32-ctest-build + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_f32-multicore-baremetal +) +add_dependencies(${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_f32-ctest-build ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_f32-thread-build) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_f32-ctest-build) + +# bank_mxquant_two_blocks +file( + GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/bank_mxquant_two_blocks_image.S + CONTENT + ".section .rodata.thread,\"a\",@progbits\n.balign 8\n.global thread_image, thread_image_end\nthread_image:\n.incbin \"${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-bank_mxquant_two_blocks.bin\"\nthread_image_end:\n.section .note.GNU-stack,\"\",@progbits\n" +) +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_bank_mxquant_two_blocks-multicore-baremetal + COMMAND + ${ELF_CC} -O2 -g -march=rv64gc -mcmodel=medany -msmall-data-limit=0 + -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${BBSIM_LD} -I${POLY_MLIR_SOURCE_DIR} ${POLY_MLIR_HOST_DEFINES} + ${DESIGN_DEFINES} -I${RUNTIME_INCLUDE_DIR} -I${WORKLOAD_LIB_DIR} + -I${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/mxquant/workloads + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${BUCKYBALL_REPO_ROOT}/stack/engine/include -I${BAREMETAL_DIR} + ${POLY_BOOT_SOURCE} ${CMAKE_CURRENT_SOURCE_DIR}/bank_mxquant_two_blocks_main.cpp + ${CMAKE_CURRENT_BINARY_DIR}/bank_mxquant_two_blocks_image.S + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c -lgcc -o + ${DESIGN_NAME}-main-ctest-design-mlir_bank_mxquant_two_blocks-multicore-baremetal + DEPENDS + bank_mxquant_two_blocks_main.cpp + ${POLY_MLIR_SOURCE_DIR}/mlir.h + ${CMAKE_CURRENT_BINARY_DIR}/bank_mxquant_two_blocks_image.S + ${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-bank_mxquant_two_blocks.bin + ${POLY_BOOT_SOURCE} + ${BBSIM_LD} + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/engine/include/thread.h + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c + ${RUNTIME_INCLUDE_DIR}/bb_cores.h + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM +) +add_custom_target( + ${DESIGN_NAME}-main-ctest-design-mlir_bank_mxquant_two_blocks-ctest-build + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_bank_mxquant_two_blocks-multicore-baremetal +) +add_dependencies(${DESIGN_NAME}-main-ctest-design-mlir_bank_mxquant_two_blocks-ctest-build ${DESIGN_NAME}-attention-mlirtest-bank_mxquant_two_blocks-thread-build) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-main-ctest-design-mlir_bank_mxquant_two_blocks-ctest-build) diff --git a/examples/designs/poly/workloads/device/core/engine/mlir/ball_mxmm_f32_main.cpp b/examples/designs/poly/workloads/device/core/engine/mlir/ball_mxmm_f32_main.cpp new file mode 100644 index 000000000..eecce4928 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/mlir/ball_mxmm_f32_main.cpp @@ -0,0 +1,88 @@ +#include "mlir.h" +#include + +extern const unsigned char thread_image[], thread_image_end[]; + +// A: 16 rows of 4 FP32. B: 16 columns of 4 FP32. C: 16x16 FP32, then a guard +// row. Row r of A is {-1, 1 + 2^-23, r, 1}, column j of B {1, 1 - 2^-23, 1, j}. +struct buffers { + uint32_t a[16 * 4]; + uint32_t b[16 * 4]; + uint32_t c[65 * 4]; +}; +struct all { + buffers core[POLY_THREAD_CORES]; +}; +static constexpr all prepare() { + all data{}; + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) { + buffers &buffer = data.core[core]; + for (unsigned i = 0; i < 16; ++i) { + buffer.a[i * 4 + 0] = __builtin_bit_cast(uint32_t, -1.0f); + buffer.a[i * 4 + 1] = 0x3f800001; + buffer.a[i * 4 + 2] = __builtin_bit_cast(uint32_t, (float)i); + buffer.a[i * 4 + 3] = __builtin_bit_cast(uint32_t, 1.0f); + buffer.b[i * 4 + 0] = __builtin_bit_cast(uint32_t, 1.0f); + buffer.b[i * 4 + 1] = 0x3f7ffffe; + buffer.b[i * 4 + 2] = __builtin_bit_cast(uint32_t, 1.0f); + buffer.b[i * 4 + 3] = __builtin_bit_cast(uint32_t, (float)i); + } + for (unsigned i = 0; i < 65 * 4; ++i) + buffer.c[i] = 0x5a5a5a5a; + } + return data; +} +alignas(64) static all data = prepare(); +static unsigned done[POLY_THREAD_CORES]; + +static void run(unsigned core, const void *args, size_t size) { + const uint32_t context = thread_find(THREAD_SIGNATURE, 0); + size_t bytes = thread_image_end - thread_image; + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || + thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) + __builtin_trap(); + thread_tss_write(0, thread_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + thread_tss_write(0, args, size); + thread_load(context, THREAD_TLS, 0, 0, (size + 7) & ~7u); + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE), + capacity = thread_query(context, THREAD_FIELD_TLS_BYTES); + struct thread_task task = {core + 1, 0, bytes, base, base + capacity, + THREAD_SIGNATURE}; + thread_acquire(); + thread_start(context, &task); + int cancelled; + if (thread_wait(context, task.id, &cancelled) || cancelled) + __builtin_trap(); + thread_release(); +} + +static void exercise(unsigned core) { + buffers &buffer = data.core[core]; + struct mlir_mxmm_args args = {(uintptr_t)buffer.a, (uintptr_t)buffer.b, + (uintptr_t)buffer.c}; + run(core, &args, sizeof(args)); + // Unfused: (1 + 2^-23)(1 - 2^-23) rounds to 1 and cancels the -1. + for (unsigned row = 0; row < 16; ++row) + for (unsigned column = 0; column < 16; ++column) + if (buffer.c[row * 16 + column] != + __builtin_bit_cast(uint32_t, (float)(row + column))) + __builtin_trap(); + for (unsigned i = 256; i < 260; ++i) + if (buffer.c[i] != 0x5a5a5a5a) + __builtin_trap(); + __atomic_store_n(&done[core], 1, __ATOMIC_RELEASE); +} + +int main(void) { + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart == WORKLOAD_REPORT_HART) { + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + while (!__atomic_load_n(&done[core], __ATOMIC_ACQUIRE)) { + } + } else { + exercise(hart - 1); + } + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/engine/mlir/ball_mxmm_fma32_main.cpp b/examples/designs/poly/workloads/device/core/engine/mlir/ball_mxmm_fma32_main.cpp new file mode 100644 index 000000000..6efafcb21 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/mlir/ball_mxmm_fma32_main.cpp @@ -0,0 +1,91 @@ +#include "mlir.h" +#include + +extern const unsigned char thread_image[], thread_image_end[]; + +// A: 16 rows of 4 FP32. B: 16 columns of 4 FP32. C: 16x16 FP32, then a guard +// row. Row r of A is {-1, 1 + 2^-23, r, 1}, column j of B {1, 1 - 2^-23, 1, j}. +struct buffers { + uint32_t a[16 * 4]; + uint32_t b[16 * 4]; + uint32_t c[65 * 4]; +}; +struct all { + buffers core[POLY_THREAD_CORES]; +}; +static constexpr all prepare() { + all data{}; + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) { + buffers &buffer = data.core[core]; + for (unsigned i = 0; i < 16; ++i) { + buffer.a[i * 4 + 0] = __builtin_bit_cast(uint32_t, -1.0f); + buffer.a[i * 4 + 1] = 0x3f800001; + buffer.a[i * 4 + 2] = __builtin_bit_cast(uint32_t, (float)i); + buffer.a[i * 4 + 3] = __builtin_bit_cast(uint32_t, 1.0f); + buffer.b[i * 4 + 0] = __builtin_bit_cast(uint32_t, 1.0f); + buffer.b[i * 4 + 1] = 0x3f7ffffe; + buffer.b[i * 4 + 2] = __builtin_bit_cast(uint32_t, 1.0f); + buffer.b[i * 4 + 3] = __builtin_bit_cast(uint32_t, (float)i); + } + for (unsigned i = 0; i < 65 * 4; ++i) + buffer.c[i] = 0x5a5a5a5a; + } + return data; +} +alignas(64) static all data = prepare(); +static unsigned done[POLY_THREAD_CORES]; + +static void run(unsigned core, const void *args, size_t size) { + const uint32_t context = thread_find(THREAD_SIGNATURE, 0); + size_t bytes = thread_image_end - thread_image; + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || + thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) + __builtin_trap(); + thread_tss_write(0, thread_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + thread_tss_write(0, args, size); + thread_load(context, THREAD_TLS, 0, 0, (size + 7) & ~7u); + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE), + capacity = thread_query(context, THREAD_FIELD_TLS_BYTES); + struct thread_task task = {core + 1, 0, bytes, base, base + capacity, + THREAD_SIGNATURE}; + thread_acquire(); + thread_start(context, &task); + int cancelled; + if (thread_wait(context, task.id, &cancelled) || cancelled) + __builtin_trap(); + thread_release(); +} + +static void exercise(unsigned core) { + buffers &buffer = data.core[core]; + struct mlir_mxmm_args args = {(uintptr_t)buffer.a, (uintptr_t)buffer.b, + (uintptr_t)buffer.c}; + run(core, &args, sizeof(args)); + // Fused: -1 + (1 + 2^-23)(1 - 2^-23) leaves -2^-46, which survives only + // where r + j is 0. + for (unsigned row = 0; row < 16; ++row) + for (unsigned column = 0; column < 16; ++column) + if (buffer.c[row * 16 + column] != + (row == 0 && column == 0 + ? 0xa8800000u + : __builtin_bit_cast(uint32_t, (float)(row + column)))) + __builtin_trap(); + for (unsigned i = 256; i < 260; ++i) + if (buffer.c[i] != 0x5a5a5a5a) + __builtin_trap(); + __atomic_store_n(&done[core], 1, __ATOMIC_RELEASE); +} + +int main(void) { + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart == WORKLOAD_REPORT_HART) { + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + while (!__atomic_load_n(&done[core], __ATOMIC_ACQUIRE)) { + } + } else { + exercise(hart - 1); + } + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/engine/mlir/ball_mxmm_mxfp8_chain_main.cpp b/examples/designs/poly/workloads/device/core/engine/mlir/ball_mxmm_mxfp8_chain_main.cpp new file mode 100644 index 000000000..7b473e246 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/mlir/ball_mxmm_mxfp8_chain_main.cpp @@ -0,0 +1,96 @@ +#include "mlir.h" +#include + +extern const unsigned char thread_image[], thread_image_end[]; + +// A: 16 rows of 32 E4M3 codes, then 16 scales. B: 16 columns of 32 codes, +// then 16 scales. C: 16x16 FP32 from row 1, the rest a guard pattern. +// Row r of A is 0.5 * (1 + (r % 8) / 8) (code 0x38 + r % 8, scale 2^-1); +// column j of B is 4 * (1 + (j % 8) / 8) (code 0x40 + j % 8, scale 2^1). +struct buffers { + uint8_t a[33 * 16]; + uint8_t b[33 * 16]; + uint32_t c[65 * 4]; + uint32_t between[65 * 4]; +}; +struct all { + buffers core[POLY_THREAD_CORES]; +}; +static constexpr all prepare() { + all data{}; + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) { + buffers &buffer = data.core[core]; + for (unsigned i = 0; i < 16 * 32; ++i) { + buffer.a[i] = 0x38 + (i / 32) % 8; + buffer.b[i] = 0x40 + (i / 32) % 8; + } + for (unsigned i = 16 * 32; i < 16 * 33; ++i) { + buffer.a[i] = 126; + buffer.b[i] = 128; + } + for (unsigned i = 0; i < 65 * 4; ++i) + buffer.c[i] = 0x5a5a5a5a; + for (unsigned i = 0; i < 65 * 4; ++i) + buffer.between[i] = 0x5a5a5a5a; + } + return data; +} +alignas(64) static all data = prepare(); +static unsigned done[POLY_THREAD_CORES]; + +static void run(unsigned core, const void *args, size_t size) { + const uint32_t context = thread_find(THREAD_SIGNATURE, 0); + size_t bytes = thread_image_end - thread_image; + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || + thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) + __builtin_trap(); + thread_tss_write(0, thread_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + thread_tss_write(0, args, size); + thread_load(context, THREAD_TLS, 0, 0, (size + 7) & ~7u); + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE), + capacity = thread_query(context, THREAD_FIELD_TLS_BYTES); + struct thread_task task = {core + 1, 0, bytes, base, base + capacity, + THREAD_SIGNATURE}; + thread_acquire(); + thread_start(context, &task); + int cancelled; + if (thread_wait(context, task.id, &cancelled) || cancelled) + __builtin_trap(); + thread_release(); +} + +static void exercise(unsigned core) { + buffers &buffer = data.core[core]; + struct mlir_mxmm_chain_args args = {(uintptr_t)buffer.a, (uintptr_t)buffer.b, + (uintptr_t)buffer.c, + (uintptr_t)buffer.between}; + run(core, &args, sizeof(args)); + // The first step does not write C. + for (unsigned i = 0; i < 65 * 4; ++i) + if (buffer.between[i] != 0x5a5a5a5a) + __builtin_trap(); + for (unsigned i = 0; i < 4; ++i) + if (buffer.c[i] != 0x5a5a5a5a) + __builtin_trap(); + // Each step adds (8 + r % 8) * (8 + j % 8); the chain has two. + for (unsigned row = 0; row < 16; ++row) + for (unsigned column = 0; column < 16; ++column) + if (__builtin_bit_cast(float, buffer.c[4 + row * 16 + column]) != + 2.0f * (8 + row % 8) * (8 + column % 8)) + __builtin_trap(); + __atomic_store_n(&done[core], 1, __ATOMIC_RELEASE); +} + +int main(void) { + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart == WORKLOAD_REPORT_HART) { + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + while (!__atomic_load_n(&done[core], __ATOMIC_ACQUIRE)) { + } + } else { + exercise(hart - 1); + } + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/engine/mlir/ball_mxmm_mxfp8_window_main.cpp b/examples/designs/poly/workloads/device/core/engine/mlir/ball_mxmm_mxfp8_window_main.cpp new file mode 100644 index 000000000..55905949e --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/mlir/ball_mxmm_mxfp8_window_main.cpp @@ -0,0 +1,87 @@ +#include "mlir.h" +#include + +extern const unsigned char thread_image[], thread_image_end[]; + +// A: one row of 64 E4M3 codes, then its two scales. B: 16 columns of 32 +// codes, then 16 scales. C: one row of 16 FP32, then a guard row. +// Outside the window A holds NaN codes and a NaN scale, which the Ball must +// not read. Inside, element t is 1 + (t % 8) / 8; column j of B is +// 2 * (1 + (j % 8) / 8). +struct buffers { + uint8_t a[5 * 16]; + uint8_t b[33 * 16]; + uint32_t c[5 * 4]; +}; +struct all { + buffers core[POLY_THREAD_CORES]; +}; +static constexpr all prepare() { + all data{}; + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) { + buffers &buffer = data.core[core]; + for (unsigned t = 0; t < 64; ++t) + buffer.a[t] = t < 32 ? 0x7f : 0x38 + t % 8; + buffer.a[64] = 255; + buffer.a[65] = 127; + for (unsigned i = 0; i < 16 * 32; ++i) + buffer.b[i] = 0x40 + (i / 32) % 8; + for (unsigned i = 16 * 32; i < 16 * 33; ++i) + buffer.b[i] = 127; + for (unsigned i = 0; i < 5 * 4; ++i) + buffer.c[i] = 0x5a5a5a5a; + } + return data; +} +alignas(64) static all data = prepare(); +static unsigned done[POLY_THREAD_CORES]; + +static void run(unsigned core, const void *args, size_t size) { + const uint32_t context = thread_find(THREAD_SIGNATURE, 0); + size_t bytes = thread_image_end - thread_image; + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || + thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) + __builtin_trap(); + thread_tss_write(0, thread_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + thread_tss_write(0, args, size); + thread_load(context, THREAD_TLS, 0, 0, (size + 7) & ~7u); + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE), + capacity = thread_query(context, THREAD_FIELD_TLS_BYTES); + struct thread_task task = {core + 1, 0, bytes, base, base + capacity, + THREAD_SIGNATURE}; + thread_acquire(); + thread_start(context, &task); + int cancelled; + if (thread_wait(context, task.id, &cancelled) || cancelled) + __builtin_trap(); + thread_release(); +} + +static void exercise(unsigned core) { + buffers &buffer = data.core[core]; + struct mlir_mxmm_args args = {(uintptr_t)buffer.a, (uintptr_t)buffer.b, + (uintptr_t)buffer.c}; + run(core, &args, sizeof(args)); + // The window sums to 46, so column j is 46 * 2 * (8 + j % 8) / 8. + for (unsigned column = 0; column < 16; ++column) + if (__builtin_bit_cast(float, buffer.c[column]) != 11.5f * (8 + column % 8)) + __builtin_trap(); + for (unsigned i = 16; i < 20; ++i) + if (buffer.c[i] != 0x5a5a5a5a) + __builtin_trap(); + __atomic_store_n(&done[core], 1, __ATOMIC_RELEASE); +} + +int main(void) { + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart == WORKLOAD_REPORT_HART) { + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + while (!__atomic_load_n(&done[core], __ATOMIC_ACQUIRE)) { + } + } else { + exercise(hart - 1); + } + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/engine/mlir/ball_mxquant_two_blocks_main.cpp b/examples/designs/poly/workloads/device/core/engine/mlir/ball_mxquant_two_blocks_main.cpp new file mode 100644 index 000000000..113908a31 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/mlir/ball_mxquant_two_blocks_main.cpp @@ -0,0 +1,92 @@ +#include "mlir.h" +#include "reference.h" +#include + +extern const unsigned char thread_image[], thread_image_end[]; + +// Input: two blocks of 32 FP32. Output: the 64 codes, the two scales from +// byte 64, then guard bytes up to the end of row 4. Block 0 holds signed +// zeros, subnormals, rounding ties and the largest finite; block 1 one large +// element that sets the scale and small ones that round. +struct buffers { + uint32_t input[16 * 4]; + uint8_t output[5 * 16]; +}; +struct all { + buffers core[POLY_THREAD_CORES]; +}; +static constexpr all prepare() { + all data{}; + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) { + buffers &buffer = data.core[core]; + const uint32_t edges[] = {0, 0x80000000, 1, 0x80000001, + 0x007fffff, 0x00800000, 0x3f880000, 0x3f980000, + 0x43e00000, 0x7f7fffff, 0xff7fffff}; + for (unsigned i = 0; i < 32; ++i) { + buffer.input[i] = edges[i % 11]; + buffer.input[32 + i] = + i ? (i & 1 ? 0x3f880000u : 0xbf980000u) : 0x43e00000u; + } + for (unsigned i = 0; i < 5 * 16; ++i) + buffer.output[i] = 0x5a; + } + return data; +} +alignas(64) static all data = prepare(); +static unsigned done[POLY_THREAD_CORES]; + +static void run(unsigned core, const void *args, size_t size) { + const uint32_t context = thread_find(THREAD_SIGNATURE, 0); + size_t bytes = thread_image_end - thread_image; + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || + thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) + __builtin_trap(); + thread_tss_write(0, thread_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + thread_tss_write(0, args, size); + thread_load(context, THREAD_TLS, 0, 0, (size + 7) & ~7u); + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE), + capacity = thread_query(context, THREAD_FIELD_TLS_BYTES); + struct thread_task task = {core + 1, 0, bytes, base, base + capacity, + THREAD_SIGNATURE}; + thread_acquire(); + thread_start(context, &task); + int cancelled; + if (thread_wait(context, task.id, &cancelled) || cancelled) + __builtin_trap(); + thread_release(); +} + +static void exercise(unsigned core) { + buffers &buffer = data.core[core]; + struct mlir_mxquant_args args = {(uintptr_t)buffer.input, + (uintptr_t)buffer.output}; + run(core, &args, sizeof(args)); + uint8_t expected[2][33]; + mxquant_reference32(buffer.input, expected[0]); + mxquant_reference32(buffer.input + 32, expected[1]); + for (unsigned block = 0; block < 2; ++block) { + for (unsigned i = 0; i < 32; ++i) + if (buffer.output[block * 32 + i] != expected[block][i]) + __builtin_trap(); + if (buffer.output[64 + block] != expected[block][32]) + __builtin_trap(); + } + for (unsigned i = 66; i < 80; ++i) + if (buffer.output[i] != 0x5a) + __builtin_trap(); + __atomic_store_n(&done[core], 1, __ATOMIC_RELEASE); +} + +int main(void) { + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart == WORKLOAD_REPORT_HART) { + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + while (!__atomic_load_n(&done[core], __ATOMIC_ACQUIRE)) { + } + } else { + exercise(hart - 1); + } + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/engine/mlir/bank_mxmm_f32_main.cpp b/examples/designs/poly/workloads/device/core/engine/mlir/bank_mxmm_f32_main.cpp new file mode 100644 index 000000000..eecce4928 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/mlir/bank_mxmm_f32_main.cpp @@ -0,0 +1,88 @@ +#include "mlir.h" +#include + +extern const unsigned char thread_image[], thread_image_end[]; + +// A: 16 rows of 4 FP32. B: 16 columns of 4 FP32. C: 16x16 FP32, then a guard +// row. Row r of A is {-1, 1 + 2^-23, r, 1}, column j of B {1, 1 - 2^-23, 1, j}. +struct buffers { + uint32_t a[16 * 4]; + uint32_t b[16 * 4]; + uint32_t c[65 * 4]; +}; +struct all { + buffers core[POLY_THREAD_CORES]; +}; +static constexpr all prepare() { + all data{}; + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) { + buffers &buffer = data.core[core]; + for (unsigned i = 0; i < 16; ++i) { + buffer.a[i * 4 + 0] = __builtin_bit_cast(uint32_t, -1.0f); + buffer.a[i * 4 + 1] = 0x3f800001; + buffer.a[i * 4 + 2] = __builtin_bit_cast(uint32_t, (float)i); + buffer.a[i * 4 + 3] = __builtin_bit_cast(uint32_t, 1.0f); + buffer.b[i * 4 + 0] = __builtin_bit_cast(uint32_t, 1.0f); + buffer.b[i * 4 + 1] = 0x3f7ffffe; + buffer.b[i * 4 + 2] = __builtin_bit_cast(uint32_t, 1.0f); + buffer.b[i * 4 + 3] = __builtin_bit_cast(uint32_t, (float)i); + } + for (unsigned i = 0; i < 65 * 4; ++i) + buffer.c[i] = 0x5a5a5a5a; + } + return data; +} +alignas(64) static all data = prepare(); +static unsigned done[POLY_THREAD_CORES]; + +static void run(unsigned core, const void *args, size_t size) { + const uint32_t context = thread_find(THREAD_SIGNATURE, 0); + size_t bytes = thread_image_end - thread_image; + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || + thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) + __builtin_trap(); + thread_tss_write(0, thread_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + thread_tss_write(0, args, size); + thread_load(context, THREAD_TLS, 0, 0, (size + 7) & ~7u); + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE), + capacity = thread_query(context, THREAD_FIELD_TLS_BYTES); + struct thread_task task = {core + 1, 0, bytes, base, base + capacity, + THREAD_SIGNATURE}; + thread_acquire(); + thread_start(context, &task); + int cancelled; + if (thread_wait(context, task.id, &cancelled) || cancelled) + __builtin_trap(); + thread_release(); +} + +static void exercise(unsigned core) { + buffers &buffer = data.core[core]; + struct mlir_mxmm_args args = {(uintptr_t)buffer.a, (uintptr_t)buffer.b, + (uintptr_t)buffer.c}; + run(core, &args, sizeof(args)); + // Unfused: (1 + 2^-23)(1 - 2^-23) rounds to 1 and cancels the -1. + for (unsigned row = 0; row < 16; ++row) + for (unsigned column = 0; column < 16; ++column) + if (buffer.c[row * 16 + column] != + __builtin_bit_cast(uint32_t, (float)(row + column))) + __builtin_trap(); + for (unsigned i = 256; i < 260; ++i) + if (buffer.c[i] != 0x5a5a5a5a) + __builtin_trap(); + __atomic_store_n(&done[core], 1, __ATOMIC_RELEASE); +} + +int main(void) { + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart == WORKLOAD_REPORT_HART) { + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + while (!__atomic_load_n(&done[core], __ATOMIC_ACQUIRE)) { + } + } else { + exercise(hart - 1); + } + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/engine/mlir/bank_mxmm_fma32_main.cpp b/examples/designs/poly/workloads/device/core/engine/mlir/bank_mxmm_fma32_main.cpp new file mode 100644 index 000000000..6efafcb21 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/mlir/bank_mxmm_fma32_main.cpp @@ -0,0 +1,91 @@ +#include "mlir.h" +#include + +extern const unsigned char thread_image[], thread_image_end[]; + +// A: 16 rows of 4 FP32. B: 16 columns of 4 FP32. C: 16x16 FP32, then a guard +// row. Row r of A is {-1, 1 + 2^-23, r, 1}, column j of B {1, 1 - 2^-23, 1, j}. +struct buffers { + uint32_t a[16 * 4]; + uint32_t b[16 * 4]; + uint32_t c[65 * 4]; +}; +struct all { + buffers core[POLY_THREAD_CORES]; +}; +static constexpr all prepare() { + all data{}; + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) { + buffers &buffer = data.core[core]; + for (unsigned i = 0; i < 16; ++i) { + buffer.a[i * 4 + 0] = __builtin_bit_cast(uint32_t, -1.0f); + buffer.a[i * 4 + 1] = 0x3f800001; + buffer.a[i * 4 + 2] = __builtin_bit_cast(uint32_t, (float)i); + buffer.a[i * 4 + 3] = __builtin_bit_cast(uint32_t, 1.0f); + buffer.b[i * 4 + 0] = __builtin_bit_cast(uint32_t, 1.0f); + buffer.b[i * 4 + 1] = 0x3f7ffffe; + buffer.b[i * 4 + 2] = __builtin_bit_cast(uint32_t, 1.0f); + buffer.b[i * 4 + 3] = __builtin_bit_cast(uint32_t, (float)i); + } + for (unsigned i = 0; i < 65 * 4; ++i) + buffer.c[i] = 0x5a5a5a5a; + } + return data; +} +alignas(64) static all data = prepare(); +static unsigned done[POLY_THREAD_CORES]; + +static void run(unsigned core, const void *args, size_t size) { + const uint32_t context = thread_find(THREAD_SIGNATURE, 0); + size_t bytes = thread_image_end - thread_image; + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || + thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) + __builtin_trap(); + thread_tss_write(0, thread_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + thread_tss_write(0, args, size); + thread_load(context, THREAD_TLS, 0, 0, (size + 7) & ~7u); + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE), + capacity = thread_query(context, THREAD_FIELD_TLS_BYTES); + struct thread_task task = {core + 1, 0, bytes, base, base + capacity, + THREAD_SIGNATURE}; + thread_acquire(); + thread_start(context, &task); + int cancelled; + if (thread_wait(context, task.id, &cancelled) || cancelled) + __builtin_trap(); + thread_release(); +} + +static void exercise(unsigned core) { + buffers &buffer = data.core[core]; + struct mlir_mxmm_args args = {(uintptr_t)buffer.a, (uintptr_t)buffer.b, + (uintptr_t)buffer.c}; + run(core, &args, sizeof(args)); + // Fused: -1 + (1 + 2^-23)(1 - 2^-23) leaves -2^-46, which survives only + // where r + j is 0. + for (unsigned row = 0; row < 16; ++row) + for (unsigned column = 0; column < 16; ++column) + if (buffer.c[row * 16 + column] != + (row == 0 && column == 0 + ? 0xa8800000u + : __builtin_bit_cast(uint32_t, (float)(row + column)))) + __builtin_trap(); + for (unsigned i = 256; i < 260; ++i) + if (buffer.c[i] != 0x5a5a5a5a) + __builtin_trap(); + __atomic_store_n(&done[core], 1, __ATOMIC_RELEASE); +} + +int main(void) { + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart == WORKLOAD_REPORT_HART) { + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + while (!__atomic_load_n(&done[core], __ATOMIC_ACQUIRE)) { + } + } else { + exercise(hart - 1); + } + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/engine/mlir/bank_mxmm_mxfp8_chain_main.cpp b/examples/designs/poly/workloads/device/core/engine/mlir/bank_mxmm_mxfp8_chain_main.cpp new file mode 100644 index 000000000..576ac6cd9 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/mlir/bank_mxmm_mxfp8_chain_main.cpp @@ -0,0 +1,88 @@ +#include "mlir.h" +#include + +extern const unsigned char thread_image[], thread_image_end[]; + +// A: 16 rows of 32 E4M3 codes, then 16 scales. B: 16 columns of 32 codes, +// then 16 scales. C: 16x16 FP32 from row 1, the rest a guard pattern. +// Row r of A is 0.5 * (1 + (r % 8) / 8) (code 0x38 + r % 8, scale 2^-1); +// column j of B is 4 * (1 + (j % 8) / 8) (code 0x40 + j % 8, scale 2^1). +struct buffers { + uint8_t a[33 * 16]; + uint8_t b[33 * 16]; + uint32_t c[65 * 4]; +}; +struct all { + buffers core[POLY_THREAD_CORES]; +}; +static constexpr all prepare() { + all data{}; + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) { + buffers &buffer = data.core[core]; + for (unsigned i = 0; i < 16 * 32; ++i) { + buffer.a[i] = 0x38 + (i / 32) % 8; + buffer.b[i] = 0x40 + (i / 32) % 8; + } + for (unsigned i = 16 * 32; i < 16 * 33; ++i) { + buffer.a[i] = 126; + buffer.b[i] = 128; + } + for (unsigned i = 0; i < 65 * 4; ++i) + buffer.c[i] = 0x5a5a5a5a; + } + return data; +} +alignas(64) static all data = prepare(); +static unsigned done[POLY_THREAD_CORES]; + +static void run(unsigned core, const void *args, size_t size) { + const uint32_t context = thread_find(THREAD_SIGNATURE, 0); + size_t bytes = thread_image_end - thread_image; + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || + thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) + __builtin_trap(); + thread_tss_write(0, thread_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + thread_tss_write(0, args, size); + thread_load(context, THREAD_TLS, 0, 0, (size + 7) & ~7u); + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE), + capacity = thread_query(context, THREAD_FIELD_TLS_BYTES); + struct thread_task task = {core + 1, 0, bytes, base, base + capacity, + THREAD_SIGNATURE}; + thread_acquire(); + thread_start(context, &task); + int cancelled; + if (thread_wait(context, task.id, &cancelled) || cancelled) + __builtin_trap(); + thread_release(); +} + +static void exercise(unsigned core) { + buffers &buffer = data.core[core]; + struct mlir_mxmm_args args = {(uintptr_t)buffer.a, (uintptr_t)buffer.b, + (uintptr_t)buffer.c}; + run(core, &args, sizeof(args)); + for (unsigned i = 0; i < 4; ++i) + if (buffer.c[i] != 0x5a5a5a5a) + __builtin_trap(); + // Each step adds (8 + r % 8) * (8 + j % 8); the chain has two. + for (unsigned row = 0; row < 16; ++row) + for (unsigned column = 0; column < 16; ++column) + if (__builtin_bit_cast(float, buffer.c[4 + row * 16 + column]) != + 2.0f * (8 + row % 8) * (8 + column % 8)) + __builtin_trap(); + __atomic_store_n(&done[core], 1, __ATOMIC_RELEASE); +} + +int main(void) { + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart == WORKLOAD_REPORT_HART) { + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + while (!__atomic_load_n(&done[core], __ATOMIC_ACQUIRE)) { + } + } else { + exercise(hart - 1); + } + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/engine/mlir/bank_mxmm_mxfp8_window_main.cpp b/examples/designs/poly/workloads/device/core/engine/mlir/bank_mxmm_mxfp8_window_main.cpp new file mode 100644 index 000000000..55905949e --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/mlir/bank_mxmm_mxfp8_window_main.cpp @@ -0,0 +1,87 @@ +#include "mlir.h" +#include + +extern const unsigned char thread_image[], thread_image_end[]; + +// A: one row of 64 E4M3 codes, then its two scales. B: 16 columns of 32 +// codes, then 16 scales. C: one row of 16 FP32, then a guard row. +// Outside the window A holds NaN codes and a NaN scale, which the Ball must +// not read. Inside, element t is 1 + (t % 8) / 8; column j of B is +// 2 * (1 + (j % 8) / 8). +struct buffers { + uint8_t a[5 * 16]; + uint8_t b[33 * 16]; + uint32_t c[5 * 4]; +}; +struct all { + buffers core[POLY_THREAD_CORES]; +}; +static constexpr all prepare() { + all data{}; + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) { + buffers &buffer = data.core[core]; + for (unsigned t = 0; t < 64; ++t) + buffer.a[t] = t < 32 ? 0x7f : 0x38 + t % 8; + buffer.a[64] = 255; + buffer.a[65] = 127; + for (unsigned i = 0; i < 16 * 32; ++i) + buffer.b[i] = 0x40 + (i / 32) % 8; + for (unsigned i = 16 * 32; i < 16 * 33; ++i) + buffer.b[i] = 127; + for (unsigned i = 0; i < 5 * 4; ++i) + buffer.c[i] = 0x5a5a5a5a; + } + return data; +} +alignas(64) static all data = prepare(); +static unsigned done[POLY_THREAD_CORES]; + +static void run(unsigned core, const void *args, size_t size) { + const uint32_t context = thread_find(THREAD_SIGNATURE, 0); + size_t bytes = thread_image_end - thread_image; + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || + thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) + __builtin_trap(); + thread_tss_write(0, thread_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + thread_tss_write(0, args, size); + thread_load(context, THREAD_TLS, 0, 0, (size + 7) & ~7u); + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE), + capacity = thread_query(context, THREAD_FIELD_TLS_BYTES); + struct thread_task task = {core + 1, 0, bytes, base, base + capacity, + THREAD_SIGNATURE}; + thread_acquire(); + thread_start(context, &task); + int cancelled; + if (thread_wait(context, task.id, &cancelled) || cancelled) + __builtin_trap(); + thread_release(); +} + +static void exercise(unsigned core) { + buffers &buffer = data.core[core]; + struct mlir_mxmm_args args = {(uintptr_t)buffer.a, (uintptr_t)buffer.b, + (uintptr_t)buffer.c}; + run(core, &args, sizeof(args)); + // The window sums to 46, so column j is 46 * 2 * (8 + j % 8) / 8. + for (unsigned column = 0; column < 16; ++column) + if (__builtin_bit_cast(float, buffer.c[column]) != 11.5f * (8 + column % 8)) + __builtin_trap(); + for (unsigned i = 16; i < 20; ++i) + if (buffer.c[i] != 0x5a5a5a5a) + __builtin_trap(); + __atomic_store_n(&done[core], 1, __ATOMIC_RELEASE); +} + +int main(void) { + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart == WORKLOAD_REPORT_HART) { + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + while (!__atomic_load_n(&done[core], __ATOMIC_ACQUIRE)) { + } + } else { + exercise(hart - 1); + } + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/engine/mlir/bank_mxquant_two_blocks_main.cpp b/examples/designs/poly/workloads/device/core/engine/mlir/bank_mxquant_two_blocks_main.cpp new file mode 100644 index 000000000..113908a31 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/mlir/bank_mxquant_two_blocks_main.cpp @@ -0,0 +1,92 @@ +#include "mlir.h" +#include "reference.h" +#include + +extern const unsigned char thread_image[], thread_image_end[]; + +// Input: two blocks of 32 FP32. Output: the 64 codes, the two scales from +// byte 64, then guard bytes up to the end of row 4. Block 0 holds signed +// zeros, subnormals, rounding ties and the largest finite; block 1 one large +// element that sets the scale and small ones that round. +struct buffers { + uint32_t input[16 * 4]; + uint8_t output[5 * 16]; +}; +struct all { + buffers core[POLY_THREAD_CORES]; +}; +static constexpr all prepare() { + all data{}; + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) { + buffers &buffer = data.core[core]; + const uint32_t edges[] = {0, 0x80000000, 1, 0x80000001, + 0x007fffff, 0x00800000, 0x3f880000, 0x3f980000, + 0x43e00000, 0x7f7fffff, 0xff7fffff}; + for (unsigned i = 0; i < 32; ++i) { + buffer.input[i] = edges[i % 11]; + buffer.input[32 + i] = + i ? (i & 1 ? 0x3f880000u : 0xbf980000u) : 0x43e00000u; + } + for (unsigned i = 0; i < 5 * 16; ++i) + buffer.output[i] = 0x5a; + } + return data; +} +alignas(64) static all data = prepare(); +static unsigned done[POLY_THREAD_CORES]; + +static void run(unsigned core, const void *args, size_t size) { + const uint32_t context = thread_find(THREAD_SIGNATURE, 0); + size_t bytes = thread_image_end - thread_image; + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || + thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) + __builtin_trap(); + thread_tss_write(0, thread_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + thread_tss_write(0, args, size); + thread_load(context, THREAD_TLS, 0, 0, (size + 7) & ~7u); + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE), + capacity = thread_query(context, THREAD_FIELD_TLS_BYTES); + struct thread_task task = {core + 1, 0, bytes, base, base + capacity, + THREAD_SIGNATURE}; + thread_acquire(); + thread_start(context, &task); + int cancelled; + if (thread_wait(context, task.id, &cancelled) || cancelled) + __builtin_trap(); + thread_release(); +} + +static void exercise(unsigned core) { + buffers &buffer = data.core[core]; + struct mlir_mxquant_args args = {(uintptr_t)buffer.input, + (uintptr_t)buffer.output}; + run(core, &args, sizeof(args)); + uint8_t expected[2][33]; + mxquant_reference32(buffer.input, expected[0]); + mxquant_reference32(buffer.input + 32, expected[1]); + for (unsigned block = 0; block < 2; ++block) { + for (unsigned i = 0; i < 32; ++i) + if (buffer.output[block * 32 + i] != expected[block][i]) + __builtin_trap(); + if (buffer.output[64 + block] != expected[block][32]) + __builtin_trap(); + } + for (unsigned i = 66; i < 80; ++i) + if (buffer.output[i] != 0x5a) + __builtin_trap(); + __atomic_store_n(&done[core], 1, __ATOMIC_RELEASE); +} + +int main(void) { + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart == WORKLOAD_REPORT_HART) { + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + while (!__atomic_load_n(&done[core], __ATOMIC_ACQUIRE)) { + } + } else { + exercise(hart - 1); + } + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/thread/CMakeLists.txt b/examples/designs/poly/workloads/device/core/thread/CMakeLists.txt new file mode 100644 index 000000000..9d5e4e197 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/CMakeLists.txt @@ -0,0 +1,5 @@ +add_subdirectory(ctests) +set(POLY_ANT_IMAGE_DIR "${POLY_ANT_IMAGE_DIR}" PARENT_SCOPE) +set(POLY_ANT_SOURCE_DIR "${POLY_ANT_SOURCE_DIR}" PARENT_SCOPE) +add_subdirectory(mlir) +set(POLY_MLIR_IMAGE_DIR "${POLY_MLIR_IMAGE_DIR}" PARENT_SCOPE) diff --git a/examples/designs/poly/workloads/device/core/thread/ctests/CMakeLists.txt b/examples/designs/poly/workloads/device/core/thread/ctests/CMakeLists.txt new file mode 100644 index 000000000..f83a7dc7b --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/ctests/CMakeLists.txt @@ -0,0 +1,90 @@ +foreach( + POLY_ANT_CASE + mxmm + shared + bank + bank_rows + parallel + mesh_move + highpa + corelink + corelink_bank + corelink_reduce + coherence + mesh_private + mesh_shared +) + set(POLY_ANT_THREAD attention) + set(POLY_ANT_PAYLOAD ant_${POLY_ANT_CASE}.c) + set(POLY_ANT_HEADER ant_${POLY_ANT_CASE}.h) + set(POLY_ANT_HOST_DEFINITIONS) + if(POLY_ANT_CASE MATCHES "^mesh_(private|shared)$") + set(POLY_ANT_HEADER ant_mesh.h) + if(POLY_ANT_CASE STREQUAL "mesh_private") + set(POLY_ANT_PAYLOAD mesh_private_bank_move_test.c) + set(POLY_ANT_HOST_DEFINITIONS -DPOLY_MESH_PRIVATE=1) + else() + set(POLY_ANT_PAYLOAD mesh_shared_mem_test.c) + set(POLY_ANT_HOST_DEFINITIONS -DPOLY_MESH_PRIVATE=0) + endif() + endif() + if(POLY_ANT_CASE STREQUAL "bank" OR POLY_ANT_CASE STREQUAL "bank_rows") + set(POLY_ANT_THREAD ffn) + endif() + set(POLY_ANT_NAME + "${DESIGN_NAME}-${POLY_ANT_THREAD}-ctest-design-ant_${POLY_ANT_CASE}" + ) + set(_BANK_WIDTH ${DESIGN_PROFILES_${POLY_ANT_THREAD}_THREAD_MEM_BANK_WIDTH}) + set(_BANK_ENTRIES ${DESIGN_PROFILES_${POLY_ANT_THREAD}_THREAD_MEM_BANK_ENTRIES}) + set(_MAX_PRIVATE_VBANK ${DESIGN_PROFILES_${POLY_ANT_THREAD}_THREAD_FRONTEND_MAX_PRIVATE_VBANK}) + set(POLY_ANT_DEFINES ${DESIGN_TARGET_DEFS_${POLY_ANT_THREAD}}) + list(TRANSFORM POLY_ANT_DEFINES PREPEND "-D") + if( + NOT _BANK_WIDTH EQUAL 128 + OR _BANK_ENTRIES LESS 128 + OR _MAX_PRIVATE_VBANK LESS 5 + ) + message( + FATAL_ERROR + "Ant Mxmm needs 128-bit rows, >=128 rows/bank and private banks through 5; actual width=${_BANK_WIDTH}, rows=${_BANK_ENTRIES}, privateMax=${_MAX_PRIVATE_VBANK}" + ) + endif() + set(POLY_ANT_IMAGE ${CMAKE_CURRENT_BINARY_DIR}/${POLY_ANT_NAME}-ant) + set(ANT_RUNTIME_DIR ${BUCKYBALL_REPO_ROOT}/stack/engine/program) + buckyball_enforce_ctest_line_limit("${CMAKE_CURRENT_SOURCE_DIR}/${POLY_ANT_PAYLOAD}") + add_custom_command( + OUTPUT ${POLY_ANT_IMAGE}.elf ${POLY_ANT_IMAGE}.bin + COMMAND + ${ELF_CC} -O2 -g -march=rv64im -mabi=lp64 -mcmodel=medany + -ffreestanding -fno-builtin -fno-pic -fno-stack-protector + -fno-asynchronous-unwind-tables -msmall-data-limit=0 -mno-relax + -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${ANT_RUNTIME_DIR}/code.ld -I${WORKLOAD_LIB_DIR} + ${POLY_ANT_DEFINES} + -I${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/mxmm/workloads + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${BUCKYBALL_REPO_ROOT}/stack/engine/include + ${ANT_RUNTIME_DIR}/start.S + ${CMAKE_CURRENT_SOURCE_DIR}/${POLY_ANT_PAYLOAD} -o + ${POLY_ANT_IMAGE}.elf + COMMAND + ${ELF_OBJCOPY} -O binary --only-section=.text ${POLY_ANT_IMAGE}.elf + ${POLY_ANT_IMAGE}.bin + DEPENDS + ${POLY_ANT_PAYLOAD} + ${POLY_ANT_HEADER} + ${ANT_RUNTIME_DIR}/start.S + ${ANT_RUNTIME_DIR}/code.ld + ${BUCKYBALL_BALL_ISA_DEPS} + VERBATIM + ) + add_custom_target( + ${POLY_ANT_NAME}-ant-build + DEPENDS ${POLY_ANT_IMAGE}.elf ${POLY_ANT_IMAGE}.bin + ) + add_dependencies(poly-ctests-build ${POLY_ANT_NAME}-ant-build) +endforeach() + +set(POLY_ANT_IMAGE_DIR "${CMAKE_CURRENT_BINARY_DIR}" PARENT_SCOPE) +set(POLY_ANT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}" PARENT_SCOPE) diff --git a/examples/designs/poly/workloads/device/core/thread/ctests/ant_bank.c b/examples/designs/poly/workloads/device/core/thread/ctests/ant_bank.c new file mode 100644 index 000000000..e1327f6eb --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_bank.c @@ -0,0 +1,19 @@ +#include "ant_bank.h" +#include + +uint64_t entry(const struct ant_bank_args *args) { + bb_mset(3, 1, 1, 1); + bb_mvin(args->input, 3, THREAD_MEM_BANK_ENTRIES, 1); + bb_mvout(args->private_out, 3, THREAD_MEM_BANK_ENTRIES, 1); + bb_mset(3, 0, 0, 0); + bb_mset(THREAD_FRONTEND_SHARED_VBANK_BASE, 1, 1, 1); + bb_mvin(args->input, THREAD_FRONTEND_SHARED_VBANK_BASE, args->shared_rows, 1); + bb_mvout(args->shared_out, THREAD_FRONTEND_SHARED_VBANK_BASE, + args->shared_rows, 1); + bb_mset(THREAD_FRONTEND_SHARED_VBANK_BASE, 0, 0, 0); + bb_mset(3, 1, 1, 1); + bb_mvin_2d(args->input, 3, 2, 16, 2, 0, 2, 8); + bb_mvout(args->tail_out, 3, 4, 1); + bb_mset(3, 0, 0, 0); + return 0; +} diff --git a/examples/chips/poly/workloads/ctests/ant_bank.h b/examples/designs/poly/workloads/device/core/thread/ctests/ant_bank.h similarity index 81% rename from examples/chips/poly/workloads/ctests/ant_bank.h rename to examples/designs/poly/workloads/device/core/thread/ctests/ant_bank.h index 0034d3669..d482aeab4 100644 --- a/examples/chips/poly/workloads/ctests/ant_bank.h +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_bank.h @@ -6,4 +6,4 @@ struct ant_bank_args { }; extern unsigned bank_done[]; -void bank_exercise(unsigned tile); +void bank_exercise(unsigned core); diff --git a/examples/designs/poly/workloads/device/core/thread/ctests/ant_bank_rows.c b/examples/designs/poly/workloads/device/core/thread/ctests/ant_bank_rows.c new file mode 100644 index 000000000..8989e877d --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_bank_rows.c @@ -0,0 +1,19 @@ +#include "ant_bank_rows.h" +#include + +uint64_t entry(const struct ant_bank_rows_args *args) { + bb_mset(3, 1, 1, 1); + bb_mvin(args->input, 3, ANT_BANK_ROWS, 1); + bb_mvout(args->private_out, 3, ANT_BANK_ROWS, 1); + bb_mset(3, 0, 0, 0); + bb_mset(THREAD_FRONTEND_SHARED_VBANK_BASE, 1, 1, 1); + bb_mvin(args->input, THREAD_FRONTEND_SHARED_VBANK_BASE, args->shared_rows, 1); + bb_mvout(args->shared_out, THREAD_FRONTEND_SHARED_VBANK_BASE, + args->shared_rows, 1); + bb_mset(THREAD_FRONTEND_SHARED_VBANK_BASE, 0, 0, 0); + bb_mset(3, 1, 1, 1); + bb_mvin_2d(args->input, 3, 2, 16, 2, 0, 2, 8); + bb_mvout(args->tail_out, 3, 4, 1); + bb_mset(3, 0, 0, 0); + return 0; +} diff --git a/examples/chips/poly/workloads/ctests/ant_bank_rows.h b/examples/designs/poly/workloads/device/core/thread/ctests/ant_bank_rows.h similarity index 82% rename from examples/chips/poly/workloads/ctests/ant_bank_rows.h rename to examples/designs/poly/workloads/device/core/thread/ctests/ant_bank_rows.h index d93667015..68825f29a 100644 --- a/examples/chips/poly/workloads/ctests/ant_bank_rows.h +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_bank_rows.h @@ -7,4 +7,4 @@ struct ant_bank_rows_args { }; extern unsigned bank_rows_done[]; -void bank_rows_exercise(unsigned tile); +void bank_rows_exercise(unsigned core); diff --git a/examples/chips/poly/workloads/ctests/ant_coherence.c b/examples/designs/poly/workloads/device/core/thread/ctests/ant_coherence.c similarity index 83% rename from examples/chips/poly/workloads/ctests/ant_coherence.c rename to examples/designs/poly/workloads/device/core/thread/ctests/ant_coherence.c index 3be1a9d82..097b98308 100644 --- a/examples/chips/poly/workloads/ctests/ant_coherence.c +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_coherence.c @@ -1,7 +1,7 @@ #include "ant_coherence.h" #include -uint64_t ant_main(const struct ant_coherence_args *args) { +uint64_t entry(const struct ant_coherence_args *args) { if (args->phase == 0) bb_mset(3, 1, 1, 1); else if (args->phase == 1) { diff --git a/examples/chips/poly/workloads/ctests/ant_coherence.h b/examples/designs/poly/workloads/device/core/thread/ctests/ant_coherence.h similarity index 100% rename from examples/chips/poly/workloads/ctests/ant_coherence.h rename to examples/designs/poly/workloads/device/core/thread/ctests/ant_coherence.h diff --git a/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink.c b/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink.c new file mode 100644 index 000000000..3625c89c5 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink.c @@ -0,0 +1,18 @@ +#include "ant_corelink.h" +#include +uint64_t entry(const struct ant_corelink_args *args) { + if (args->phase == 0) { + bb_mset(THREAD_FRONTEND_SHARED_VBANK_BASE, 1, 1, 1); + bb_mset(THREAD_FRONTEND_SHARED_VBANK_BASE + 1, 1, 1, 1); + bb_mvin(args->input, THREAD_FRONTEND_SHARED_VBANK_BASE, CORELINK_TEST_ROWS, + 1); + } else if (args->phase == 1) { + bb_mvout(args->output, THREAD_FRONTEND_SHARED_VBANK_BASE + 1, + CORELINK_TEST_ROWS, 1); + } else if (args->phase == 2) { + bb_mset(THREAD_FRONTEND_SHARED_VBANK_BASE, 0, 0, 0); + bb_mset(THREAD_FRONTEND_SHARED_VBANK_BASE + 1, 0, 0, 0); + } else + __builtin_trap(); + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink.h b/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink.h new file mode 100644 index 000000000..d8341d49d --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink.h @@ -0,0 +1,9 @@ +#pragma once +#include +enum { + CORELINK_TEST_BYTES = 4160, + CORELINK_TEST_ROWS = CORELINK_TEST_BYTES / 16 +}; +struct ant_corelink_args { + uint64_t phase, input, output; +}; diff --git a/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink_bank.c b/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink_bank.c new file mode 100644 index 000000000..16cdb7017 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink_bank.c @@ -0,0 +1,18 @@ +#include "ant_corelink_bank.h" +#include +uint64_t entry(const struct ant_corelink_bank_args *args) { + if (args->phase == 0) { + bb_mset(THREAD_FRONTEND_SHARED_VBANK_BASE, 1, 1, 1); + bb_mset(THREAD_FRONTEND_SHARED_VBANK_BASE + 1, 1, 1, 1); + bb_mvin(args->input, THREAD_FRONTEND_SHARED_VBANK_BASE, CORELINK_CROSS_ROWS, + 1); + } else if (args->phase == 1) { + bb_mvout(args->output, THREAD_FRONTEND_SHARED_VBANK_BASE, + CORELINK_CROSS_ROWS, 1); + } else if (args->phase == 2) { + bb_mset(THREAD_FRONTEND_SHARED_VBANK_BASE, 0, 0, 0); + bb_mset(THREAD_FRONTEND_SHARED_VBANK_BASE + 1, 0, 0, 0); + } else + __builtin_trap(); + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink_bank.h b/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink_bank.h new file mode 100644 index 000000000..2c21dc932 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink_bank.h @@ -0,0 +1,9 @@ +#pragma once +#include +enum { + CORELINK_CROSS_BYTES = 192, + CORELINK_CROSS_ROWS = CORELINK_CROSS_BYTES / 16 +}; +struct ant_corelink_bank_args { + uint64_t phase, input, output; +}; diff --git a/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink_reduce.c b/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink_reduce.c new file mode 100644 index 000000000..3c46337d5 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink_reduce.c @@ -0,0 +1,21 @@ +#include "ant_corelink_reduce.h" +#include +#include +uint64_t entry(const struct ant_corelink_reduce_args *args) { + if (args->phase == 0) { + bb_mset(THREAD_FRONTEND_SHARED_VBANK_BASE, 1, 1, 1); + bb_mset(0, 1, 1, 1); + bb_mset(1, 1, 1, 1); + bb_mvin(args->input, 0, 4, 1); + bb_mvin(args->input + 16 * sizeof(float), 1, 64, 1); + bb_mxmm_fma32(0, 1, THREAD_FRONTEND_SHARED_VBANK_BASE, 1, 16, 16, 1, 1, 0); + bb_mset(0, 0, 0, 0); + bb_mset(1, 0, 0, 0); + } else if (args->phase == 1) { + bb_mvout(args->output, THREAD_FRONTEND_SHARED_VBANK_BASE, 2, 1); + } else if (args->phase == 2) { + bb_mset(THREAD_FRONTEND_SHARED_VBANK_BASE, 0, 0, 0); + } else + __builtin_trap(); + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink_reduce.h b/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink_reduce.h new file mode 100644 index 000000000..d930c66e8 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink_reduce.h @@ -0,0 +1,8 @@ +#pragma once +#include +struct ant_corelink_reduce_args { + uint64_t phase, input, output; +}; +void ant_corelink_reduce_phase(unsigned core, + const struct ant_corelink_reduce_args *args); +void ant_corelink_reduce_prepare(unsigned core, float *input, float *output); diff --git a/examples/chips/poly/workloads/ctests/ant_highpa.c b/examples/designs/poly/workloads/device/core/thread/ctests/ant_highpa.c similarity index 76% rename from examples/chips/poly/workloads/ctests/ant_highpa.c rename to examples/designs/poly/workloads/device/core/thread/ctests/ant_highpa.c index 7f790ae9c..1d10edf92 100644 --- a/examples/chips/poly/workloads/ctests/ant_highpa.c +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_highpa.c @@ -1,6 +1,6 @@ #include "ant_highpa.h" #include -uint64_t ant_main(const struct ant_highpa_args *args) { +uint64_t entry(const struct ant_highpa_args *args) { bb_mset(3, 1, 1, 1); bb_mvin(args->input, 3, 2, 1); bb_mvout(args->output, 3, 2, 1); diff --git a/examples/chips/poly/workloads/ctests/ant_highpa.h b/examples/designs/poly/workloads/device/core/thread/ctests/ant_highpa.h similarity index 100% rename from examples/chips/poly/workloads/ctests/ant_highpa.h rename to examples/designs/poly/workloads/device/core/thread/ctests/ant_highpa.h diff --git a/examples/designs/poly/workloads/device/core/thread/ctests/ant_mesh.h b/examples/designs/poly/workloads/device/core/thread/ctests/ant_mesh.h new file mode 100644 index 000000000..9c8408a36 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_mesh.h @@ -0,0 +1,5 @@ +#pragma once +#include +struct ant_mesh_args { + uint64_t input, output, index; +}; diff --git a/examples/chips/poly/workloads/ctests/ant_mesh_move.c b/examples/designs/poly/workloads/device/core/thread/ctests/ant_mesh_move.c similarity index 92% rename from examples/chips/poly/workloads/ctests/ant_mesh_move.c rename to examples/designs/poly/workloads/device/core/thread/ctests/ant_mesh_move.c index bbcb12493..86db5aaf6 100644 --- a/examples/chips/poly/workloads/ctests/ant_mesh_move.c +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_mesh_move.c @@ -3,7 +3,7 @@ enum { ROWS = 2, SOURCE_BANK = 1, TARGET_BANK = 8 }; -uint64_t ant_main(const struct ant_mesh_move_args *args) { +uint64_t entry(const struct ant_mesh_move_args *args) { volatile uint64_t *sync = (volatile uint64_t *)args->sync; if (args->role == MESH_FILL) { bb_mset(SOURCE_BANK, 1, 1, 1); diff --git a/examples/chips/poly/workloads/ctests/ant_mesh_move.h b/examples/designs/poly/workloads/device/core/thread/ctests/ant_mesh_move.h similarity index 100% rename from examples/chips/poly/workloads/ctests/ant_mesh_move.h rename to examples/designs/poly/workloads/device/core/thread/ctests/ant_mesh_move.h diff --git a/examples/designs/poly/workloads/device/core/thread/ctests/ant_mxmm.c b/examples/designs/poly/workloads/device/core/thread/ctests/ant_mxmm.c new file mode 100644 index 000000000..c3769ae77 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_mxmm.c @@ -0,0 +1,15 @@ +#include "ant_mxmm.h" +#include +#include + +uint64_t entry(const struct ant_mxmm_args *args) { + for (unsigned bank = 3; bank <= 5; ++bank) + bb_mset(bank, 1, 1, 1); + bb_mvin(args->a, 3, ANT_M * ANT_K * 4 / (THREAD_MEM_BANK_WIDTH / 8), 1); + bb_mvin(args->b, 4, ANT_N * ANT_K * 4 / (THREAD_MEM_BANK_WIDTH / 8), 1); + bb_mxmm_f32(3, 4, 5, ANT_M, ANT_N, ANT_K, 1, 1, 0); + bb_mvout(args->output, 5, ANT_M * ANT_N * 4 / (THREAD_MEM_BANK_WIDTH / 8), 1); + for (unsigned bank = 3; bank <= 5; ++bank) + bb_mset(bank, 0, 0, 0); + return 0; +} diff --git a/examples/chips/poly/workloads/ctests/ant_mxmm.h b/examples/designs/poly/workloads/device/core/thread/ctests/ant_mxmm.h similarity index 100% rename from examples/chips/poly/workloads/ctests/ant_mxmm.h rename to examples/designs/poly/workloads/device/core/thread/ctests/ant_mxmm.h diff --git a/examples/chips/poly/workloads/ctests/ant_parallel.c b/examples/designs/poly/workloads/device/core/thread/ctests/ant_parallel.c similarity index 86% rename from examples/chips/poly/workloads/ctests/ant_parallel.c rename to examples/designs/poly/workloads/device/core/thread/ctests/ant_parallel.c index e35835ad5..561344a90 100644 --- a/examples/chips/poly/workloads/ctests/ant_parallel.c +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_parallel.c @@ -1,7 +1,7 @@ #include "ant_parallel.h" #include -uint64_t ant_main(const struct ant_parallel_args *args) { +uint64_t entry(const struct ant_parallel_args *args) { volatile uint64_t *ready = (volatile uint64_t *)args->shared; if (args->publish) *ready = 1; diff --git a/examples/chips/poly/workloads/ctests/ant_parallel.h b/examples/designs/poly/workloads/device/core/thread/ctests/ant_parallel.h similarity index 78% rename from examples/chips/poly/workloads/ctests/ant_parallel.h rename to examples/designs/poly/workloads/device/core/thread/ctests/ant_parallel.h index cb8268558..a3281b687 100644 --- a/examples/chips/poly/workloads/ctests/ant_parallel.h +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_parallel.h @@ -6,4 +6,4 @@ struct ant_parallel_args { }; extern unsigned parallel_done[]; -void parallel_exercise(unsigned tile); +void parallel_exercise(unsigned core); diff --git a/examples/chips/poly/workloads/ctests/ant_shared.c b/examples/designs/poly/workloads/device/core/thread/ctests/ant_shared.c similarity index 87% rename from examples/chips/poly/workloads/ctests/ant_shared.c rename to examples/designs/poly/workloads/device/core/thread/ctests/ant_shared.c index 2ad249004..2fa2fed88 100644 --- a/examples/chips/poly/workloads/ctests/ant_shared.c +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_shared.c @@ -1,6 +1,6 @@ #include "ant_shared.h" -uint64_t ant_main(struct ant_shared_args *args) { +uint64_t entry(struct ant_shared_args *args) { volatile uint64_t *shared = (volatile uint64_t *)args->shared; if (args->producer) { shared[0] = args->value; diff --git a/examples/designs/poly/workloads/device/core/thread/ctests/ant_shared.h b/examples/designs/poly/workloads/device/core/thread/ctests/ant_shared.h new file mode 100644 index 000000000..7924df30d --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_shared.h @@ -0,0 +1,9 @@ +#pragma once +#include + +struct ant_shared_args { + uint64_t shared, producer, value; +}; + +extern unsigned shared_done[]; +void shared_exercise(unsigned core); diff --git a/examples/designs/poly/workloads/device/core/thread/ctests/mesh_private_bank_move_test.c b/examples/designs/poly/workloads/device/core/thread/ctests/mesh_private_bank_move_test.c new file mode 100644 index 000000000..d112fcbb8 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/ctests/mesh_private_bank_move_test.c @@ -0,0 +1,54 @@ +#include "ant_mesh.h" +#include + +uint64_t entry(const struct ant_mesh_args *args) { + if (args->index == 0) { + bb_mem_alloc(0, 1, 1); + bb_mvin(args->input, 0, 2, 1); + } + if (args->index == 1) { + bb_mem_alloc(1, 1, 1); + bb_mem_alloc(4, 1, 1); + } + if (args->index == 2) + bb_mem_alloc(3, 1, 1); + if (args->index == 3) + bb_mem_alloc(2, 1, 1); + if (args->index == 4) + bb_mem_alloc(1, 1, 1); + + /* All five Buckyball threads join each barrier. Each barrier drains the ROB. + */ + bb_barrier(); + if (args->index == 0) { + bb_mvover(0, 0, 1, 1, 1, 1, 1); + bb_mvover(0, 0, 1, 3, 2, 1, 1); + } + bb_barrier(); + + if (args->index == 1) + bb_mvout(args->output + 0, 1, 2, 1); + if (args->index == 3) + bb_mvout(args->output + 32, 2, 2, 1); + bb_barrier(); + + if (args->index == 3) + bb_mvover(3, 2, 1, 2, 3, 1, 1); + bb_barrier(); + if (args->index == 2) + bb_mvout(args->output + 64, 3, 2, 1); + bb_barrier(); + + if (args->index == 0) + bb_mvover(0, 0, 1, 4, 1, 1, 1); + if (args->index == 3) + bb_mvover(3, 2, 1, 1, 4, 1, 1); + bb_barrier(); + if (args->index == 4) + bb_mvout(args->output + 96, 1, 2, 1); + if (args->index == 1) + bb_mvout(args->output + 128, 4, 2, 1); + bb_barrier(); + + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/thread/ctests/mesh_shared_mem_test.c b/examples/designs/poly/workloads/device/core/thread/ctests/mesh_shared_mem_test.c new file mode 100644 index 000000000..c60289103 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/ctests/mesh_shared_mem_test.c @@ -0,0 +1,24 @@ +#include "ant_mesh.h" +#include + +enum { + BANKS = CORE_VIRTUAL_BANK_COUNT - THREAD_FRONTEND_SHARED_VBANK_BASE, + ROW_BYTES = THREAD_MEM_BANK_WIDTH / 8 +}; +_Static_assert( + THREAD_MEM_BANK_WIDTH == 128 && BANKS > 0 && CORE_SHARED_MEM_ENABLE && + BANKS * CORE_SHARED_MEM_BANK_ENTRIES <= CORE_SHARED_MEM_ENTRIES, + "Shared bank test requires one physical bank per shared virtual bank"); +uint64_t entry(const struct ant_mesh_args *args) { + for (unsigned bank = 0; bank < BANKS; ++bank) { + bb_mem_alloc(THREAD_FRONTEND_SHARED_VBANK_BASE + bank, 1, 1); + bb_mvin(args->input + bank * ROW_BYTES, + THREAD_FRONTEND_SHARED_VBANK_BASE + bank, 1, 1); + } + for (unsigned bank = 0; bank < BANKS; ++bank) + bb_mvout(args->output + bank * ROW_BYTES, + THREAD_FRONTEND_SHARED_VBANK_BASE + bank, 1, 1); + for (unsigned bank = 0; bank < BANKS; ++bank) + bb_mem_release(THREAD_FRONTEND_SHARED_VBANK_BASE + bank); + return 0; +} diff --git a/examples/chips/poly/workloads/ctests/mxmm_fp32_small.c b/examples/designs/poly/workloads/device/core/thread/ctests/mxmm_fp32_small.c similarity index 100% rename from examples/chips/poly/workloads/ctests/mxmm_fp32_small.c rename to examples/designs/poly/workloads/device/core/thread/ctests/mxmm_fp32_small.c diff --git a/examples/chips/poly/workloads/ctests/mxmm_mxfp8_small.c b/examples/designs/poly/workloads/device/core/thread/ctests/mxmm_mxfp8_small.c similarity index 100% rename from examples/chips/poly/workloads/ctests/mxmm_mxfp8_small.c rename to examples/designs/poly/workloads/device/core/thread/ctests/mxmm_mxfp8_small.c diff --git a/examples/designs/poly/workloads/device/core/thread/mlir/CMakeLists.txt b/examples/designs/poly/workloads/device/core/thread/mlir/CMakeLists.txt new file mode 100644 index 000000000..009998887 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/mlir/CMakeLists.txt @@ -0,0 +1,353 @@ +# The MLIR tests of MxmmBall and MxquantBall, the Balls of poly's threads, as +# thread programs: each test MLIR is compiled for the attention thread and +# linked with its entry into a thread image (start.S, code.ld of the engine). +# engine/mlir embeds every image into a host program that runs it on the +# thread of each core and checks the result. +set(POLY_MLIR_THREAD_DEFINES ${DESIGN_TARGET_DEFS_attention}) +list(TRANSFORM POLY_MLIR_THREAD_DEFINES PREPEND "-D") +string(JOIN " " POLY_MLIR_OPT ${DESIGN_TARGET_OPT_attention}) +set(POLY_MLIR_BIN "${DESIGN_COMPILER}/bin") +set(POLY_MLIR_ENGINE "${BUCKYBALL_REPO_ROOT}/stack/engine/program") +set(MXMM_TESTS "${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/mxmm/workloads/mlir_tests") +set(MXQUANT_TESTS "${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/mxquant/workloads/mlir_tests") + +# ball_mxmm_mxfp8_chain +add_custom_command( + OUTPUT ball_mxmm_mxfp8_chain.o + COMMAND + bash -o pipefail -c + "${POLY_MLIR_OPT} ${MXMM_TESTS}/ball/mxmm_mxfp8_chain.mlir -assign-physical-banks -lower-buckyball -convert-linalg-to-loops -expand-strided-metadata -lower-affine -convert-scf-to-cf -convert-cf-to-llvm -convert-arith-to-llvm -convert-math-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | ${POLY_MLIR_BIN}/buddy-translate --buddy-to-llvmir | ${POLY_MLIR_BIN}/buddy-llc -filetype=obj -mtriple=riscv64 -O2 -code-model=medium -mattr=+xbuckyball,+m -target-abi=lp64 --riscv-disable-using-constant-pool-for-large-ints -o ball_mxmm_mxfp8_chain.o" + DEPENDS ${MXMM_TESTS}/ball/mxmm_mxfp8_chain.mlir ${DESIGN_TARGET_PLUGIN_attention} + VERBATIM +) +add_custom_command( + OUTPUT ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_chain.elf ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_chain.bin + COMMAND + ${ELF_CC} -O2 -g -march=rv64im -mabi=lp64 -mcmodel=medany + -ffreestanding -fno-builtin -fno-pic -fno-stack-protector + -fno-asynchronous-unwind-tables -msmall-data-limit=0 -mno-relax + -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${POLY_MLIR_ENGINE}/code.ld -I${WORKLOAD_LIB_DIR} + ${POLY_MLIR_THREAD_DEFINES} ${POLY_MLIR_ENGINE}/start.S + ${CMAKE_CURRENT_SOURCE_DIR}/ball_mxmm_mxfp8_chain_entry.c + ${CMAKE_CURRENT_BINARY_DIR}/ball_mxmm_mxfp8_chain.o -o ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_chain.elf + COMMAND + ${ELF_OBJCOPY} -O binary --only-section=.text ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_chain.elf + ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_chain.bin + DEPENDS + ball_mxmm_mxfp8_chain_entry.c + mlir.h + ball_mxmm_mxfp8_chain.o + ${POLY_MLIR_ENGINE}/start.S + ${POLY_MLIR_ENGINE}/code.ld + VERBATIM +) +add_custom_target(${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_chain-thread-build DEPENDS ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_chain.bin) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_chain-thread-build) + +# ball_mxmm_mxfp8_window +add_custom_command( + OUTPUT ball_mxmm_mxfp8_window.o + COMMAND + bash -o pipefail -c + "${POLY_MLIR_OPT} ${MXMM_TESTS}/ball/mxmm_mxfp8_window.mlir -assign-physical-banks -lower-buckyball -convert-linalg-to-loops -expand-strided-metadata -lower-affine -convert-scf-to-cf -convert-cf-to-llvm -convert-arith-to-llvm -convert-math-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | ${POLY_MLIR_BIN}/buddy-translate --buddy-to-llvmir | ${POLY_MLIR_BIN}/buddy-llc -filetype=obj -mtriple=riscv64 -O2 -code-model=medium -mattr=+xbuckyball,+m -target-abi=lp64 --riscv-disable-using-constant-pool-for-large-ints -o ball_mxmm_mxfp8_window.o" + DEPENDS ${MXMM_TESTS}/ball/mxmm_mxfp8_window.mlir ${DESIGN_TARGET_PLUGIN_attention} + VERBATIM +) +add_custom_command( + OUTPUT ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_window.elf ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_window.bin + COMMAND + ${ELF_CC} -O2 -g -march=rv64im -mabi=lp64 -mcmodel=medany + -ffreestanding -fno-builtin -fno-pic -fno-stack-protector + -fno-asynchronous-unwind-tables -msmall-data-limit=0 -mno-relax + -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${POLY_MLIR_ENGINE}/code.ld -I${WORKLOAD_LIB_DIR} + ${POLY_MLIR_THREAD_DEFINES} ${POLY_MLIR_ENGINE}/start.S + ${CMAKE_CURRENT_SOURCE_DIR}/ball_mxmm_mxfp8_window_entry.c + ${CMAKE_CURRENT_BINARY_DIR}/ball_mxmm_mxfp8_window.o -o ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_window.elf + COMMAND + ${ELF_OBJCOPY} -O binary --only-section=.text ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_window.elf + ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_window.bin + DEPENDS + ball_mxmm_mxfp8_window_entry.c + mlir.h + ball_mxmm_mxfp8_window.o + ${POLY_MLIR_ENGINE}/start.S + ${POLY_MLIR_ENGINE}/code.ld + VERBATIM +) +add_custom_target(${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_window-thread-build DEPENDS ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_window.bin) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_window-thread-build) + +# ball_mxmm_fma32 +add_custom_command( + OUTPUT ball_mxmm_fma32.o + COMMAND + bash -o pipefail -c + "${POLY_MLIR_OPT} ${MXMM_TESTS}/ball/mxmm_fma32.mlir -assign-physical-banks -lower-buckyball -convert-linalg-to-loops -expand-strided-metadata -lower-affine -convert-scf-to-cf -convert-cf-to-llvm -convert-arith-to-llvm -convert-math-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | ${POLY_MLIR_BIN}/buddy-translate --buddy-to-llvmir | ${POLY_MLIR_BIN}/buddy-llc -filetype=obj -mtriple=riscv64 -O2 -code-model=medium -mattr=+xbuckyball,+m -target-abi=lp64 --riscv-disable-using-constant-pool-for-large-ints -o ball_mxmm_fma32.o" + DEPENDS ${MXMM_TESTS}/ball/mxmm_fma32.mlir ${DESIGN_TARGET_PLUGIN_attention} + VERBATIM +) +add_custom_command( + OUTPUT ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_fma32.elf ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_fma32.bin + COMMAND + ${ELF_CC} -O2 -g -march=rv64im -mabi=lp64 -mcmodel=medany + -ffreestanding -fno-builtin -fno-pic -fno-stack-protector + -fno-asynchronous-unwind-tables -msmall-data-limit=0 -mno-relax + -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${POLY_MLIR_ENGINE}/code.ld -I${WORKLOAD_LIB_DIR} + ${POLY_MLIR_THREAD_DEFINES} ${POLY_MLIR_ENGINE}/start.S + ${CMAKE_CURRENT_SOURCE_DIR}/ball_mxmm_fma32_entry.c + ${CMAKE_CURRENT_BINARY_DIR}/ball_mxmm_fma32.o -o ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_fma32.elf + COMMAND + ${ELF_OBJCOPY} -O binary --only-section=.text ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_fma32.elf + ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_fma32.bin + DEPENDS + ball_mxmm_fma32_entry.c + mlir.h + ball_mxmm_fma32.o + ${POLY_MLIR_ENGINE}/start.S + ${POLY_MLIR_ENGINE}/code.ld + VERBATIM +) +add_custom_target(${DESIGN_NAME}-attention-mlirtest-ball_mxmm_fma32-thread-build DEPENDS ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_fma32.bin) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_fma32-thread-build) + +# ball_mxmm_f32 +add_custom_command( + OUTPUT ball_mxmm_f32.o + COMMAND + bash -o pipefail -c + "${POLY_MLIR_OPT} ${MXMM_TESTS}/ball/mxmm_f32.mlir -assign-physical-banks -lower-buckyball -convert-linalg-to-loops -expand-strided-metadata -lower-affine -convert-scf-to-cf -convert-cf-to-llvm -convert-arith-to-llvm -convert-math-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | ${POLY_MLIR_BIN}/buddy-translate --buddy-to-llvmir | ${POLY_MLIR_BIN}/buddy-llc -filetype=obj -mtriple=riscv64 -O2 -code-model=medium -mattr=+xbuckyball,+m -target-abi=lp64 --riscv-disable-using-constant-pool-for-large-ints -o ball_mxmm_f32.o" + DEPENDS ${MXMM_TESTS}/ball/mxmm_f32.mlir ${DESIGN_TARGET_PLUGIN_attention} + VERBATIM +) +add_custom_command( + OUTPUT ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_f32.elf ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_f32.bin + COMMAND + ${ELF_CC} -O2 -g -march=rv64im -mabi=lp64 -mcmodel=medany + -ffreestanding -fno-builtin -fno-pic -fno-stack-protector + -fno-asynchronous-unwind-tables -msmall-data-limit=0 -mno-relax + -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${POLY_MLIR_ENGINE}/code.ld -I${WORKLOAD_LIB_DIR} + ${POLY_MLIR_THREAD_DEFINES} ${POLY_MLIR_ENGINE}/start.S + ${CMAKE_CURRENT_SOURCE_DIR}/ball_mxmm_f32_entry.c + ${CMAKE_CURRENT_BINARY_DIR}/ball_mxmm_f32.o -o ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_f32.elf + COMMAND + ${ELF_OBJCOPY} -O binary --only-section=.text ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_f32.elf + ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_f32.bin + DEPENDS + ball_mxmm_f32_entry.c + mlir.h + ball_mxmm_f32.o + ${POLY_MLIR_ENGINE}/start.S + ${POLY_MLIR_ENGINE}/code.ld + VERBATIM +) +add_custom_target(${DESIGN_NAME}-attention-mlirtest-ball_mxmm_f32-thread-build DEPENDS ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_f32.bin) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_f32-thread-build) + +# ball_mxquant_two_blocks +add_custom_command( + OUTPUT ball_mxquant_two_blocks.o + COMMAND + bash -o pipefail -c + "${POLY_MLIR_OPT} ${MXQUANT_TESTS}/ball/mxquant_two_blocks.mlir -assign-physical-banks -lower-buckyball -convert-linalg-to-loops -expand-strided-metadata -lower-affine -convert-scf-to-cf -convert-cf-to-llvm -convert-arith-to-llvm -convert-math-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | ${POLY_MLIR_BIN}/buddy-translate --buddy-to-llvmir | ${POLY_MLIR_BIN}/buddy-llc -filetype=obj -mtriple=riscv64 -O2 -code-model=medium -mattr=+xbuckyball,+m -target-abi=lp64 --riscv-disable-using-constant-pool-for-large-ints -o ball_mxquant_two_blocks.o" + DEPENDS ${MXQUANT_TESTS}/ball/mxquant_two_blocks.mlir ${DESIGN_TARGET_PLUGIN_attention} + VERBATIM +) +add_custom_command( + OUTPUT ${DESIGN_NAME}-attention-mlirtest-ball_mxquant_two_blocks.elf ${DESIGN_NAME}-attention-mlirtest-ball_mxquant_two_blocks.bin + COMMAND + ${ELF_CC} -O2 -g -march=rv64im -mabi=lp64 -mcmodel=medany + -ffreestanding -fno-builtin -fno-pic -fno-stack-protector + -fno-asynchronous-unwind-tables -msmall-data-limit=0 -mno-relax + -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${POLY_MLIR_ENGINE}/code.ld -I${WORKLOAD_LIB_DIR} + ${POLY_MLIR_THREAD_DEFINES} ${POLY_MLIR_ENGINE}/start.S + ${CMAKE_CURRENT_SOURCE_DIR}/ball_mxquant_two_blocks_entry.c + ${CMAKE_CURRENT_BINARY_DIR}/ball_mxquant_two_blocks.o -o ${DESIGN_NAME}-attention-mlirtest-ball_mxquant_two_blocks.elf + COMMAND + ${ELF_OBJCOPY} -O binary --only-section=.text ${DESIGN_NAME}-attention-mlirtest-ball_mxquant_two_blocks.elf + ${DESIGN_NAME}-attention-mlirtest-ball_mxquant_two_blocks.bin + DEPENDS + ball_mxquant_two_blocks_entry.c + mlir.h + ball_mxquant_two_blocks.o + ${POLY_MLIR_ENGINE}/start.S + ${POLY_MLIR_ENGINE}/code.ld + VERBATIM +) +add_custom_target(${DESIGN_NAME}-attention-mlirtest-ball_mxquant_two_blocks-thread-build DEPENDS ${DESIGN_NAME}-attention-mlirtest-ball_mxquant_two_blocks.bin) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-attention-mlirtest-ball_mxquant_two_blocks-thread-build) + +# bank_mxmm_mxfp8_chain +add_custom_command( + OUTPUT bank_mxmm_mxfp8_chain.o + COMMAND + bash -o pipefail -c + "${POLY_MLIR_OPT} ${MXMM_TESTS}/bank/mxmm_mxfp8_chain.mlir -assign-physical-banks -lower-buckyball -convert-linalg-to-loops -expand-strided-metadata -lower-affine -convert-scf-to-cf -convert-cf-to-llvm -convert-arith-to-llvm -convert-math-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | ${POLY_MLIR_BIN}/buddy-translate --buddy-to-llvmir | ${POLY_MLIR_BIN}/buddy-llc -filetype=obj -mtriple=riscv64 -O2 -code-model=medium -mattr=+xbuckyball,+m -target-abi=lp64 --riscv-disable-using-constant-pool-for-large-ints -o bank_mxmm_mxfp8_chain.o" + DEPENDS ${MXMM_TESTS}/bank/mxmm_mxfp8_chain.mlir ${DESIGN_TARGET_PLUGIN_attention} + VERBATIM +) +add_custom_command( + OUTPUT ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_chain.elf ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_chain.bin + COMMAND + ${ELF_CC} -O2 -g -march=rv64im -mabi=lp64 -mcmodel=medany + -ffreestanding -fno-builtin -fno-pic -fno-stack-protector + -fno-asynchronous-unwind-tables -msmall-data-limit=0 -mno-relax + -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${POLY_MLIR_ENGINE}/code.ld -I${WORKLOAD_LIB_DIR} + ${POLY_MLIR_THREAD_DEFINES} ${POLY_MLIR_ENGINE}/start.S + ${CMAKE_CURRENT_SOURCE_DIR}/bank_mxmm_mxfp8_chain_entry.c + ${CMAKE_CURRENT_BINARY_DIR}/bank_mxmm_mxfp8_chain.o -o ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_chain.elf + COMMAND + ${ELF_OBJCOPY} -O binary --only-section=.text ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_chain.elf + ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_chain.bin + DEPENDS + bank_mxmm_mxfp8_chain_entry.c + mlir.h + bank_mxmm_mxfp8_chain.o + ${POLY_MLIR_ENGINE}/start.S + ${POLY_MLIR_ENGINE}/code.ld + VERBATIM +) +add_custom_target(${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_chain-thread-build DEPENDS ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_chain.bin) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_chain-thread-build) + +# bank_mxmm_mxfp8_window +add_custom_command( + OUTPUT bank_mxmm_mxfp8_window.o + COMMAND + bash -o pipefail -c + "${POLY_MLIR_OPT} ${MXMM_TESTS}/bank/mxmm_mxfp8_window.mlir -assign-physical-banks -lower-buckyball -convert-linalg-to-loops -expand-strided-metadata -lower-affine -convert-scf-to-cf -convert-cf-to-llvm -convert-arith-to-llvm -convert-math-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | ${POLY_MLIR_BIN}/buddy-translate --buddy-to-llvmir | ${POLY_MLIR_BIN}/buddy-llc -filetype=obj -mtriple=riscv64 -O2 -code-model=medium -mattr=+xbuckyball,+m -target-abi=lp64 --riscv-disable-using-constant-pool-for-large-ints -o bank_mxmm_mxfp8_window.o" + DEPENDS ${MXMM_TESTS}/bank/mxmm_mxfp8_window.mlir ${DESIGN_TARGET_PLUGIN_attention} + VERBATIM +) +add_custom_command( + OUTPUT ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_window.elf ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_window.bin + COMMAND + ${ELF_CC} -O2 -g -march=rv64im -mabi=lp64 -mcmodel=medany + -ffreestanding -fno-builtin -fno-pic -fno-stack-protector + -fno-asynchronous-unwind-tables -msmall-data-limit=0 -mno-relax + -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${POLY_MLIR_ENGINE}/code.ld -I${WORKLOAD_LIB_DIR} + ${POLY_MLIR_THREAD_DEFINES} ${POLY_MLIR_ENGINE}/start.S + ${CMAKE_CURRENT_SOURCE_DIR}/bank_mxmm_mxfp8_window_entry.c + ${CMAKE_CURRENT_BINARY_DIR}/bank_mxmm_mxfp8_window.o -o ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_window.elf + COMMAND + ${ELF_OBJCOPY} -O binary --only-section=.text ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_window.elf + ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_window.bin + DEPENDS + bank_mxmm_mxfp8_window_entry.c + mlir.h + bank_mxmm_mxfp8_window.o + ${POLY_MLIR_ENGINE}/start.S + ${POLY_MLIR_ENGINE}/code.ld + VERBATIM +) +add_custom_target(${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_window-thread-build DEPENDS ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_window.bin) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_window-thread-build) + +# bank_mxmm_fma32 +add_custom_command( + OUTPUT bank_mxmm_fma32.o + COMMAND + bash -o pipefail -c + "${POLY_MLIR_OPT} ${MXMM_TESTS}/bank/mxmm_fma32.mlir -assign-physical-banks -lower-buckyball -convert-linalg-to-loops -expand-strided-metadata -lower-affine -convert-scf-to-cf -convert-cf-to-llvm -convert-arith-to-llvm -convert-math-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | ${POLY_MLIR_BIN}/buddy-translate --buddy-to-llvmir | ${POLY_MLIR_BIN}/buddy-llc -filetype=obj -mtriple=riscv64 -O2 -code-model=medium -mattr=+xbuckyball,+m -target-abi=lp64 --riscv-disable-using-constant-pool-for-large-ints -o bank_mxmm_fma32.o" + DEPENDS ${MXMM_TESTS}/bank/mxmm_fma32.mlir ${DESIGN_TARGET_PLUGIN_attention} + VERBATIM +) +add_custom_command( + OUTPUT ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_fma32.elf ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_fma32.bin + COMMAND + ${ELF_CC} -O2 -g -march=rv64im -mabi=lp64 -mcmodel=medany + -ffreestanding -fno-builtin -fno-pic -fno-stack-protector + -fno-asynchronous-unwind-tables -msmall-data-limit=0 -mno-relax + -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${POLY_MLIR_ENGINE}/code.ld -I${WORKLOAD_LIB_DIR} + ${POLY_MLIR_THREAD_DEFINES} ${POLY_MLIR_ENGINE}/start.S + ${CMAKE_CURRENT_SOURCE_DIR}/bank_mxmm_fma32_entry.c + ${CMAKE_CURRENT_BINARY_DIR}/bank_mxmm_fma32.o -o ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_fma32.elf + COMMAND + ${ELF_OBJCOPY} -O binary --only-section=.text ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_fma32.elf + ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_fma32.bin + DEPENDS + bank_mxmm_fma32_entry.c + mlir.h + bank_mxmm_fma32.o + ${POLY_MLIR_ENGINE}/start.S + ${POLY_MLIR_ENGINE}/code.ld + VERBATIM +) +add_custom_target(${DESIGN_NAME}-attention-mlirtest-bank_mxmm_fma32-thread-build DEPENDS ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_fma32.bin) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_fma32-thread-build) + +# bank_mxmm_f32 +add_custom_command( + OUTPUT bank_mxmm_f32.o + COMMAND + bash -o pipefail -c + "${POLY_MLIR_OPT} ${MXMM_TESTS}/bank/mxmm_f32.mlir -assign-physical-banks -lower-buckyball -convert-linalg-to-loops -expand-strided-metadata -lower-affine -convert-scf-to-cf -convert-cf-to-llvm -convert-arith-to-llvm -convert-math-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | ${POLY_MLIR_BIN}/buddy-translate --buddy-to-llvmir | ${POLY_MLIR_BIN}/buddy-llc -filetype=obj -mtriple=riscv64 -O2 -code-model=medium -mattr=+xbuckyball,+m -target-abi=lp64 --riscv-disable-using-constant-pool-for-large-ints -o bank_mxmm_f32.o" + DEPENDS ${MXMM_TESTS}/bank/mxmm_f32.mlir ${DESIGN_TARGET_PLUGIN_attention} + VERBATIM +) +add_custom_command( + OUTPUT ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_f32.elf ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_f32.bin + COMMAND + ${ELF_CC} -O2 -g -march=rv64im -mabi=lp64 -mcmodel=medany + -ffreestanding -fno-builtin -fno-pic -fno-stack-protector + -fno-asynchronous-unwind-tables -msmall-data-limit=0 -mno-relax + -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${POLY_MLIR_ENGINE}/code.ld -I${WORKLOAD_LIB_DIR} + ${POLY_MLIR_THREAD_DEFINES} ${POLY_MLIR_ENGINE}/start.S + ${CMAKE_CURRENT_SOURCE_DIR}/bank_mxmm_f32_entry.c + ${CMAKE_CURRENT_BINARY_DIR}/bank_mxmm_f32.o -o ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_f32.elf + COMMAND + ${ELF_OBJCOPY} -O binary --only-section=.text ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_f32.elf + ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_f32.bin + DEPENDS + bank_mxmm_f32_entry.c + mlir.h + bank_mxmm_f32.o + ${POLY_MLIR_ENGINE}/start.S + ${POLY_MLIR_ENGINE}/code.ld + VERBATIM +) +add_custom_target(${DESIGN_NAME}-attention-mlirtest-bank_mxmm_f32-thread-build DEPENDS ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_f32.bin) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_f32-thread-build) + +# bank_mxquant_two_blocks +add_custom_command( + OUTPUT bank_mxquant_two_blocks.o + COMMAND + bash -o pipefail -c + "${POLY_MLIR_OPT} ${MXQUANT_TESTS}/bank/mxquant_two_blocks.mlir -assign-physical-banks -lower-buckyball -convert-linalg-to-loops -expand-strided-metadata -lower-affine -convert-scf-to-cf -convert-cf-to-llvm -convert-arith-to-llvm -convert-math-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | ${POLY_MLIR_BIN}/buddy-translate --buddy-to-llvmir | ${POLY_MLIR_BIN}/buddy-llc -filetype=obj -mtriple=riscv64 -O2 -code-model=medium -mattr=+xbuckyball,+m -target-abi=lp64 --riscv-disable-using-constant-pool-for-large-ints -o bank_mxquant_two_blocks.o" + DEPENDS ${MXQUANT_TESTS}/bank/mxquant_two_blocks.mlir ${DESIGN_TARGET_PLUGIN_attention} + VERBATIM +) +add_custom_command( + OUTPUT ${DESIGN_NAME}-attention-mlirtest-bank_mxquant_two_blocks.elf ${DESIGN_NAME}-attention-mlirtest-bank_mxquant_two_blocks.bin + COMMAND + ${ELF_CC} -O2 -g -march=rv64im -mabi=lp64 -mcmodel=medany + -ffreestanding -fno-builtin -fno-pic -fno-stack-protector + -fno-asynchronous-unwind-tables -msmall-data-limit=0 -mno-relax + -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${POLY_MLIR_ENGINE}/code.ld -I${WORKLOAD_LIB_DIR} + ${POLY_MLIR_THREAD_DEFINES} ${POLY_MLIR_ENGINE}/start.S + ${CMAKE_CURRENT_SOURCE_DIR}/bank_mxquant_two_blocks_entry.c + ${CMAKE_CURRENT_BINARY_DIR}/bank_mxquant_two_blocks.o -o ${DESIGN_NAME}-attention-mlirtest-bank_mxquant_two_blocks.elf + COMMAND + ${ELF_OBJCOPY} -O binary --only-section=.text ${DESIGN_NAME}-attention-mlirtest-bank_mxquant_two_blocks.elf + ${DESIGN_NAME}-attention-mlirtest-bank_mxquant_two_blocks.bin + DEPENDS + bank_mxquant_two_blocks_entry.c + mlir.h + bank_mxquant_two_blocks.o + ${POLY_MLIR_ENGINE}/start.S + ${POLY_MLIR_ENGINE}/code.ld + VERBATIM +) +add_custom_target(${DESIGN_NAME}-attention-mlirtest-bank_mxquant_two_blocks-thread-build DEPENDS ${DESIGN_NAME}-attention-mlirtest-bank_mxquant_two_blocks.bin) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-attention-mlirtest-bank_mxquant_two_blocks-thread-build) +set(POLY_MLIR_IMAGE_DIR "${CMAKE_CURRENT_BINARY_DIR}" PARENT_SCOPE) diff --git a/examples/designs/poly/workloads/device/core/thread/mlir/ball_mxmm_f32_entry.c b/examples/designs/poly/workloads/device/core/thread/mlir/ball_mxmm_f32_entry.c new file mode 100644 index 000000000..7bce3e36b --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/mlir/ball_mxmm_f32_entry.c @@ -0,0 +1,20 @@ +#include "mlir.h" +#include + +void _mlir_ciface_single(void); + +// A in bank 1, B in bank 2, C in bank 3. +uint64_t entry(const struct mlir_mxmm_args *args) { + bb_mset(1, 1, 1, 1); + bb_mset(2, 1, 1, 1); + bb_mset(3, 1, 1, 1); + bb_mvin(args->a, 1, 16, 1); + bb_mvin(args->b, 2, 16, 1); + bb_mvin(args->c, 3, 65, 1); + _mlir_ciface_single(); + bb_mvout(args->c, 3, 65, 1); + bb_mset(1, 0, 0, 0); + bb_mset(2, 0, 0, 0); + bb_mset(3, 0, 0, 0); + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/thread/mlir/ball_mxmm_fma32_entry.c b/examples/designs/poly/workloads/device/core/thread/mlir/ball_mxmm_fma32_entry.c new file mode 100644 index 000000000..7bce3e36b --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/mlir/ball_mxmm_fma32_entry.c @@ -0,0 +1,20 @@ +#include "mlir.h" +#include + +void _mlir_ciface_single(void); + +// A in bank 1, B in bank 2, C in bank 3. +uint64_t entry(const struct mlir_mxmm_args *args) { + bb_mset(1, 1, 1, 1); + bb_mset(2, 1, 1, 1); + bb_mset(3, 1, 1, 1); + bb_mvin(args->a, 1, 16, 1); + bb_mvin(args->b, 2, 16, 1); + bb_mvin(args->c, 3, 65, 1); + _mlir_ciface_single(); + bb_mvout(args->c, 3, 65, 1); + bb_mset(1, 0, 0, 0); + bb_mset(2, 0, 0, 0); + bb_mset(3, 0, 0, 0); + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/thread/mlir/ball_mxmm_mxfp8_chain_entry.c b/examples/designs/poly/workloads/device/core/thread/mlir/ball_mxmm_mxfp8_chain_entry.c new file mode 100644 index 000000000..953be49ca --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/mlir/ball_mxmm_mxfp8_chain_entry.c @@ -0,0 +1,24 @@ +#include "mlir.h" +#include + +void _mlir_ciface_first(void); +void _mlir_ciface_last(void); + +// A in bank 1, B in bank 2, C in bank 3; C after the first step goes to +// `between`, after the last to `c`. +uint64_t entry(const struct mlir_mxmm_chain_args *args) { + bb_mset(1, 1, 1, 1); + bb_mset(2, 1, 1, 1); + bb_mset(3, 1, 1, 1); + bb_mvin(args->a, 1, 33, 1); + bb_mvin(args->b, 2, 33, 1); + bb_mvin(args->c, 3, 65, 1); + _mlir_ciface_first(); + bb_mvout(args->between, 3, 65, 1); + _mlir_ciface_last(); + bb_mvout(args->c, 3, 65, 1); + bb_mset(1, 0, 0, 0); + bb_mset(2, 0, 0, 0); + bb_mset(3, 0, 0, 0); + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/thread/mlir/ball_mxmm_mxfp8_window_entry.c b/examples/designs/poly/workloads/device/core/thread/mlir/ball_mxmm_mxfp8_window_entry.c new file mode 100644 index 000000000..bc49e1e32 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/mlir/ball_mxmm_mxfp8_window_entry.c @@ -0,0 +1,20 @@ +#include "mlir.h" +#include + +void _mlir_ciface_window(void); + +// A in bank 1, B in bank 2, C in bank 3. +uint64_t entry(const struct mlir_mxmm_args *args) { + bb_mset(1, 1, 1, 1); + bb_mset(2, 1, 1, 1); + bb_mset(3, 1, 1, 1); + bb_mvin(args->a, 1, 5, 1); + bb_mvin(args->b, 2, 33, 1); + bb_mvin(args->c, 3, 5, 1); + _mlir_ciface_window(); + bb_mvout(args->c, 3, 5, 1); + bb_mset(1, 0, 0, 0); + bb_mset(2, 0, 0, 0); + bb_mset(3, 0, 0, 0); + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/thread/mlir/ball_mxquant_two_blocks_entry.c b/examples/designs/poly/workloads/device/core/thread/mlir/ball_mxquant_two_blocks_entry.c new file mode 100644 index 000000000..40494c50c --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/mlir/ball_mxquant_two_blocks_entry.c @@ -0,0 +1,17 @@ +#include "mlir.h" +#include + +void _mlir_ciface_quant(void); + +// The input in bank 1, the output in bank 2. +uint64_t entry(const struct mlir_mxquant_args *args) { + bb_mset(1, 1, 1, 1); + bb_mset(2, 1, 1, 1); + bb_mvin(args->input, 1, 16, 1); + bb_mvin(args->output, 2, 5, 1); + _mlir_ciface_quant(); + bb_mvout(args->output, 2, 5, 1); + bb_mset(1, 0, 0, 0); + bb_mset(2, 0, 0, 0); + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/thread/mlir/bank_mxmm_f32_entry.c b/examples/designs/poly/workloads/device/core/thread/mlir/bank_mxmm_f32_entry.c new file mode 100644 index 000000000..dfedf4b90 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/mlir/bank_mxmm_f32_entry.c @@ -0,0 +1,25 @@ +#include "mlir.h" + +void _mlir_ciface_single(struct mlir_memref2 *, struct mlir_memref2 *, + struct mlir_memref2 *); + +static void describe(struct mlir_memref2 *memref, uint64_t address, + int64_t rows, int64_t columns) { + memref->allocated = address; + memref->aligned = address; + memref->offset = 0; + memref->sizes[0] = rows; + memref->sizes[1] = columns; + memref->strides[0] = columns; + memref->strides[1] = 1; +} + +// The MLIR function moves the buffers in and out of its banks itself. +uint64_t entry(const struct mlir_mxmm_args *args) { + struct mlir_memref2 a, b, c; + describe(&a, args->a, 16, 4); + describe(&b, args->b, 16, 4); + describe(&c, args->c, 65, 4); + _mlir_ciface_single(&a, &b, &c); + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/thread/mlir/bank_mxmm_fma32_entry.c b/examples/designs/poly/workloads/device/core/thread/mlir/bank_mxmm_fma32_entry.c new file mode 100644 index 000000000..dfedf4b90 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/mlir/bank_mxmm_fma32_entry.c @@ -0,0 +1,25 @@ +#include "mlir.h" + +void _mlir_ciface_single(struct mlir_memref2 *, struct mlir_memref2 *, + struct mlir_memref2 *); + +static void describe(struct mlir_memref2 *memref, uint64_t address, + int64_t rows, int64_t columns) { + memref->allocated = address; + memref->aligned = address; + memref->offset = 0; + memref->sizes[0] = rows; + memref->sizes[1] = columns; + memref->strides[0] = columns; + memref->strides[1] = 1; +} + +// The MLIR function moves the buffers in and out of its banks itself. +uint64_t entry(const struct mlir_mxmm_args *args) { + struct mlir_memref2 a, b, c; + describe(&a, args->a, 16, 4); + describe(&b, args->b, 16, 4); + describe(&c, args->c, 65, 4); + _mlir_ciface_single(&a, &b, &c); + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/thread/mlir/bank_mxmm_mxfp8_chain_entry.c b/examples/designs/poly/workloads/device/core/thread/mlir/bank_mxmm_mxfp8_chain_entry.c new file mode 100644 index 000000000..34af77e35 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/mlir/bank_mxmm_mxfp8_chain_entry.c @@ -0,0 +1,25 @@ +#include "mlir.h" + +void _mlir_ciface_chain(struct mlir_memref2 *, struct mlir_memref2 *, + struct mlir_memref2 *); + +static void describe(struct mlir_memref2 *memref, uint64_t address, + int64_t rows, int64_t columns) { + memref->allocated = address; + memref->aligned = address; + memref->offset = 0; + memref->sizes[0] = rows; + memref->sizes[1] = columns; + memref->strides[0] = columns; + memref->strides[1] = 1; +} + +// The MLIR function moves the buffers in and out of its banks itself. +uint64_t entry(const struct mlir_mxmm_args *args) { + struct mlir_memref2 a, b, c; + describe(&a, args->a, 33, 16); + describe(&b, args->b, 33, 16); + describe(&c, args->c, 65, 16); + _mlir_ciface_chain(&a, &b, &c); + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/thread/mlir/bank_mxmm_mxfp8_window_entry.c b/examples/designs/poly/workloads/device/core/thread/mlir/bank_mxmm_mxfp8_window_entry.c new file mode 100644 index 000000000..5a208bcbf --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/mlir/bank_mxmm_mxfp8_window_entry.c @@ -0,0 +1,25 @@ +#include "mlir.h" + +void _mlir_ciface_window(struct mlir_memref2 *, struct mlir_memref2 *, + struct mlir_memref2 *); + +static void describe(struct mlir_memref2 *memref, uint64_t address, + int64_t rows, int64_t columns) { + memref->allocated = address; + memref->aligned = address; + memref->offset = 0; + memref->sizes[0] = rows; + memref->sizes[1] = columns; + memref->strides[0] = columns; + memref->strides[1] = 1; +} + +// The MLIR function moves the buffers in and out of its banks itself. +uint64_t entry(const struct mlir_mxmm_args *args) { + struct mlir_memref2 a, b, c; + describe(&a, args->a, 5, 16); + describe(&b, args->b, 33, 16); + describe(&c, args->c, 5, 16); + _mlir_ciface_window(&a, &b, &c); + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/thread/mlir/bank_mxquant_two_blocks_entry.c b/examples/designs/poly/workloads/device/core/thread/mlir/bank_mxquant_two_blocks_entry.c new file mode 100644 index 000000000..32de94355 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/mlir/bank_mxquant_two_blocks_entry.c @@ -0,0 +1,23 @@ +#include "mlir.h" + +void _mlir_ciface_quant(struct mlir_memref2 *, struct mlir_memref2 *); + +static void describe(struct mlir_memref2 *memref, uint64_t address, + int64_t rows, int64_t columns) { + memref->allocated = address; + memref->aligned = address; + memref->offset = 0; + memref->sizes[0] = rows; + memref->sizes[1] = columns; + memref->strides[0] = columns; + memref->strides[1] = 1; +} + +// The MLIR function moves the buffers in and out of its banks itself. +uint64_t entry(const struct mlir_mxquant_args *args) { + struct mlir_memref2 input, output; + describe(&input, args->input, 16, 4); + describe(&output, args->output, 5, 16); + _mlir_ciface_quant(&input, &output); + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/thread/mlir/mlir.h b/examples/designs/poly/workloads/device/core/thread/mlir/mlir.h new file mode 100644 index 000000000..1182f2b28 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/mlir/mlir.h @@ -0,0 +1,20 @@ +#pragma once +#include + +// Arguments of the thread programs that run the MxmmBall and MxquantBall MLIR +// tests: the DRAM addresses of their buffers. +struct mlir_mxmm_args { + uint64_t a, b, c; +}; +struct mlir_mxmm_chain_args { + uint64_t a, b, c, between; +}; +struct mlir_mxquant_args { + uint64_t input, output; +}; + +// A two-dimensional memref descriptor as _mlir_ciface functions take it. +struct mlir_memref2 { + uint64_t allocated, aligned; + int64_t offset, sizes[2], strides[2]; +}; diff --git a/examples/designs/poly/workloads/device/runtime/CMakeLists.txt b/examples/designs/poly/workloads/device/runtime/CMakeLists.txt new file mode 100644 index 000000000..8eb914376 --- /dev/null +++ b/examples/designs/poly/workloads/device/runtime/CMakeLists.txt @@ -0,0 +1,146 @@ +# POLY_ANT_CASES comes from the simulation entry: CoreLink and coherence cases +# need a power-of-two count (>= 2) of thread-bearing cores. +foreach(POLY_ANT_CASE IN LISTS POLY_ANT_CASES) + set(POLY_ANT_THREAD attention) + set(POLY_ANT_HEADER ant_${POLY_ANT_CASE}.h) + set(POLY_ANT_HOST_DEFINITIONS) + if(POLY_ANT_CASE MATCHES "^mesh_(private|shared)$") + set(POLY_ANT_HEADER ant_mesh.h) + if(POLY_ANT_CASE STREQUAL "mesh_private") + set(POLY_ANT_PAYLOAD mesh_private_bank_move_test.c) + set(POLY_ANT_HOST_DEFINITIONS -DPOLY_MESH_PRIVATE=1) + else() + set(POLY_ANT_PAYLOAD mesh_shared_mem_test.c) + set(POLY_ANT_HOST_DEFINITIONS -DPOLY_MESH_PRIVATE=0) + endif() + endif() + + set(POLY_ANT_NAME + "${DESIGN_NAME}-${POLY_ANT_THREAD}-ctest-design-ant_${POLY_ANT_CASE}" + ) + set(POLY_ANT_IMAGE ${POLY_ANT_IMAGE_DIR}/${POLY_ANT_NAME}-ant) + set(POLY_ANT_CONTROL_SOURCES) + if(POLY_ANT_CASE STREQUAL "corelink_reduce") + list( + APPEND POLY_ANT_CONTROL_SOURCES + ${CMAKE_CURRENT_SOURCE_DIR}/ant_corelink_reduce_control.c + ) + endif() + foreach(SOURCE IN LISTS POLY_ANT_CONTROL_SOURCES) + buckyball_enforce_ctest_line_limit("${SOURCE}") + endforeach() + + set(POLY_ANT_EMBED ${CMAKE_CURRENT_BINARY_DIR}/ant_${POLY_ANT_CASE}_image.S) + enable_language(ASM) + file( + GENERATE OUTPUT ${POLY_ANT_EMBED} + CONTENT + ".section .rodata.ant,\"a\",@progbits\n.balign 8\n.global ant_image, ant_image_end\nant_image:\n.incbin \"${POLY_ANT_IMAGE}.bin\"\nant_image_end:\n.section .note.GNU-stack,\"\",@progbits\n" + ) + set_source_files_properties( + ${POLY_ANT_EMBED} + PROPERTIES OBJECT_DEPENDS "${POLY_ANT_IMAGE}.bin" + ) + set(POLY_ANT_MAIN ant_${POLY_ANT_CASE}_main.c) + if(POLY_ANT_CASE MATCHES "^mesh_(private|shared)$") + set(POLY_ANT_MAIN ant_mesh_main.c) + endif() + + buckyball_enforce_ctest_line_limit("${CMAKE_CURRENT_SOURCE_DIR}/${POLY_ANT_MAIN}") + set(POLY_ANT_HOST_NAME + "${DESIGN_NAME}-main-ctest-design-ant_${POLY_ANT_CASE}" + ) + set(BUCKYBALL_CTEST_TARGET ${POLY_ANT_THREAD}) + + set(POLY_ANT_HOST_THREAD ${POLY_ANT_THREAD}) + set(POLY_ANT_CONTROL_OBJECTS) + if(POLY_ANT_CASE STREQUAL "corelink_reduce") + set(POLY_ANT_HOST_THREAD main) + set(POLY_ANT_CONTROL_OBJECTS + ${CMAKE_CURRENT_BINARY_DIR}/ant_corelink_reduce_control.o + ) + set(POLY_ANT_DEFINES ${DESIGN_TARGET_DEFS_${POLY_ANT_THREAD}}) + list(TRANSFORM POLY_ANT_DEFINES PREPEND "-D") + add_custom_command( + OUTPUT ${POLY_ANT_CONTROL_OBJECTS} + COMMAND + ${ELF_CC} -O2 -g -march=rv64gc -mcmodel=medany + -msmall-data-limit=0 -fno-builtin -I${POLY_ANT_SOURCE_DIR} + ${POLY_ANT_DEFINES} + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${BUCKYBALL_REPO_ROOT}/stack/engine/include -c + ${POLY_ANT_CONTROL_SOURCES} -o ${POLY_ANT_CONTROL_OBJECTS} + DEPENDS + ${POLY_ANT_CONTROL_SOURCES} + ${POLY_ANT_SOURCE_DIR}/ant_corelink_reduce.h + VERBATIM + ) + set(POLY_ANT_CONTROL_SOURCES) + endif() + + set(POLY_ANT_HOST_ELF ${POLY_ANT_HOST_NAME}-multicore-baremetal) + set(POLY_ANT_HOST_DEFINES ${DESIGN_TARGET_DEFS_${POLY_ANT_HOST_THREAD}}) + list(TRANSFORM POLY_ANT_HOST_DEFINES PREPEND "-D") + add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${POLY_ANT_HOST_ELF} + COMMAND + ${ELF_CC} -O2 -g -march=rv64gc -mcmodel=medany -msmall-data-limit=0 + -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${BBSIM_LD} ${POLY_ANT_HOST_DEFINITIONS} + -I${POLY_ANT_SOURCE_DIR} + ${POLY_ANT_HOST_DEFINES} + ${DESIGN_DEFINES} -I${RUNTIME_INCLUDE_DIR} -I${WORKLOAD_LIB_DIR} + -I${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/f32add/workloads + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${BUCKYBALL_REPO_ROOT}/stack/engine/include -I${BAREMETAL_DIR} + ${POLY_BOOT_SOURCE} ${CMAKE_CURRENT_SOURCE_DIR}/${POLY_ANT_MAIN} + ${POLY_ANT_CONTROL_SOURCES} ${POLY_ANT_CONTROL_OBJECTS} + ${POLY_ANT_EMBED} ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c -lgcc -o + ${POLY_ANT_HOST_ELF} + DEPENDS + ${POLY_ANT_MAIN} + ${POLY_ANT_SOURCE_DIR}/${POLY_ANT_HEADER} + ${POLY_BOOT_SOURCE} + ${POLY_ANT_EMBED} + ${BAREMETAL_DIR}/start.S + ${BBSIM_LD} + ${POLY_ANT_CONTROL_SOURCES} + ${POLY_ANT_CONTROL_OBJECTS} + ${POLY_ANT_IMAGE}.bin + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/engine/include/thread.h + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.h + ${RUNTIME_INCLUDE_DIR}/bb_cores.h + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM + ) + add_custom_target( + ${POLY_ANT_HOST_NAME}-ctest-build + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${POLY_ANT_HOST_ELF} + ) + add_dependencies(poly-ctests-build ${POLY_ANT_HOST_NAME}-ctest-build) +endforeach() + +set(POLY_STREAM_NAME "${DESIGN_NAME}-main-ctest-design-stream") +buckyball_enforce_ctest_line_limit("${CMAKE_CURRENT_SOURCE_DIR}/stream_main.c") +add_executable(${POLY_STREAM_NAME}-linux stream_main.c) +target_include_directories( + ${POLY_STREAM_NAME}-linux + PRIVATE + ${RUNTIME_INCLUDE_DIR} + ${WORKLOAD_LIB_DIR} + ${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/mxmm/workloads +) +# The Ball instructions of the job go into the attention thread's stream. +target_compile_definitions( + ${POLY_STREAM_NAME}-linux + PRIVATE ${DESIGN_DEFS} ${DESIGN_TARGET_DEFS_attention} BUCKYBALL_STREAM +) +target_link_libraries(${POLY_STREAM_NAME}-linux PRIVATE engine runtime-linux) +add_dependencies(poly-ctests-build ${POLY_STREAM_NAME}-linux) + +add_subdirectory(ctests) diff --git a/examples/designs/poly/workloads/device/runtime/ant_coherence_main.c b/examples/designs/poly/workloads/device/runtime/ant_coherence_main.c new file mode 100644 index 000000000..5189cff77 --- /dev/null +++ b/examples/designs/poly/workloads/device/runtime/ant_coherence_main.c @@ -0,0 +1,93 @@ +#include "ant_coherence.h" +#include +#include +_Static_assert( + BB_NUM_CORES == POLY_THREAD_CORES + 1 && POLY_THREAD_CORES >= 2 && + !(POLY_THREAD_CORES & (POLY_THREAD_CORES - 1)), + "coherence gate requires a power-of-two thread-bearing core count"); +extern const unsigned char ant_image[], ant_image_end[]; +// Separate 64-byte lines/sets for data and synchronization; avoid main's +// top-of-page stack sets. +enum { + INPUT = 8, + OUTPUT = INPUT + POLY_THREAD_CORES, + READY = OUTPUT + POLY_THREAD_CORES, + DONE = READY + POLY_THREAD_CORES, + ROUND = DONE + POLY_THREAD_CORES +}; +static volatile uint64_t lines[ROUND + 1][8] + __attribute__((section(".data"), aligned(4096))) = {0}; +static uint64_t pattern(unsigned round, unsigned core, unsigned word) { + return ((uint64_t)round << 48) | ((uint64_t)(core + 1) << 32) | (word + 17); +} +static void task(unsigned core, unsigned phase, unsigned id) { + const unsigned context = thread_find(THREAD_SIGNATURE, 0); + struct ant_coherence_args args = {phase, (uintptr_t)lines[INPUT + core], + (uintptr_t)lines[OUTPUT + core]}; + thread_tss_write(0, &args, sizeof(args)); + thread_load(context, THREAD_TLS, 0, 0, (sizeof(args) + 7) & ~7u); + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE); + struct thread_task command = { + id, + 0, + ant_image_end - ant_image, + base, + base + thread_query(context, THREAD_FIELD_TLS_BYTES), + THREAD_SIGNATURE}; + thread_acquire(); + thread_start(context, &command); + int cancelled; + if (thread_wait(context, id, &cancelled) || cancelled) + __builtin_trap(); + thread_release(); +} +static void controller(unsigned core) { + size_t bytes = ant_image_end - ant_image; + const unsigned context = thread_find(THREAD_SIGNATURE, 0); + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || + thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) + __builtin_trap(); + thread_tss_write(0, ant_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + task(core, 0, 1); + __atomic_store_n(&lines[READY + core][0], 1, __ATOMIC_RELEASE); + for (unsigned round = 1; round <= 2; ++round) { + while (__atomic_load_n(&lines[ROUND][0], __ATOMIC_ACQUIRE) != round) { + } + task(core, 1, round + 1); + __atomic_store_n(&lines[DONE + core][0], round, __ATOMIC_RELEASE); + } + task(core, 2, 4); + __atomic_store_n(&lines[DONE + core][0], 3, __ATOMIC_RELEASE); +} +int main(void) { + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart) + controller(hart - 1); + else { + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + while (!__atomic_load_n(&lines[READY + core][0], __ATOMIC_ACQUIRE)) { + } + for (unsigned round = 1; round <= 2; ++round) { + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + for (unsigned word = 0; word < 2; ++word) { + lines[INPUT + core][word] = pattern(round, core, word); + lines[OUTPUT + core][word] = ~pattern(round, core, word); + } + __atomic_store_n(&lines[ROUND][0], round, __ATOMIC_RELEASE); + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + while (__atomic_load_n(&lines[DONE + core][0], __ATOMIC_ACQUIRE) < + round) { + } + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + for (unsigned word = 0; word < 2; ++word) + if (lines[OUTPUT + core][word] != pattern(round, core, word)) + __builtin_trap(); + } + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + while (__atomic_load_n(&lines[DONE + core][0], __ATOMIC_ACQUIRE) != 3) { + } + } + return 0; +} diff --git a/examples/designs/poly/workloads/device/runtime/ant_corelink_bank_main.c b/examples/designs/poly/workloads/device/runtime/ant_corelink_bank_main.c new file mode 100644 index 000000000..6e3f6d258 --- /dev/null +++ b/examples/designs/poly/workloads/device/runtime/ant_corelink_bank_main.c @@ -0,0 +1,96 @@ +#include "ant_corelink_bank.h" +#include +#include +#include +_Static_assert( + BB_NUM_CORES == POLY_THREAD_CORES + 1 && POLY_THREAD_CORES >= 2 && + !(POLY_THREAD_CORES & (POLY_THREAD_CORES - 1)), + "CoreLink gate requires a power-of-two thread-bearing core count"); +extern const unsigned char ant_image[], ant_image_end[]; +static uint64_t input[POLY_THREAD_CORES][CORELINK_CROSS_BYTES / 8] + __attribute__((aligned(64))); +static uint64_t output[POLY_THREAD_CORES][CORELINK_CROSS_BYTES / 8] + __attribute__((aligned(64))); +static unsigned ready[POLY_THREAD_CORES], done[POLY_THREAD_CORES]; +static unsigned arrived[2]; +static void barrier(unsigned phase) { + __atomic_fetch_add(&arrived[phase], 1, __ATOMIC_ACQ_REL); + while (__atomic_load_n(&arrived[phase], __ATOMIC_ACQUIRE) != + POLY_THREAD_CORES) { + } +} +static unsigned pattern(unsigned core, unsigned i) { + return (i * 17 + core * 23 + 3) & 255; +} +static void phase(unsigned core, unsigned operation) { + const unsigned context = thread_find(THREAD_SIGNATURE, 0); + size_t bytes = ant_image_end - ant_image; + struct ant_corelink_bank_args args = {operation, (uintptr_t)input[core], + (uintptr_t)output[core]}; + thread_tss_write(0, &args, sizeof(args)); + thread_load(context, THREAD_TLS, 0, 0, (sizeof(args) + 7) & ~7u); + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE); + struct thread_task task = {core * 3 + operation + 1, + 0, + bytes, + base, + base + + thread_query(context, THREAD_FIELD_TLS_BYTES), + THREAD_SIGNATURE}; + thread_acquire(); + thread_start(context, &task); + int cancelled; + if (thread_wait(context, task.id, &cancelled) || cancelled) + __builtin_trap(); + thread_release(); +} +static void exercise(unsigned core) { + uint64_t bank_bytes = corelink_query(CORELINK_BANK_BYTES); + if (corelink_query(CORELINK_CORE_ID) != core + 1 || + corelink_query(CORELINK_CORE_COUNT) != BB_NUM_CORES || + bank_bytes < CORELINK_CROSS_BYTES || + corelink_query(CORELINK_SHARED_BYTES) < bank_bytes * 2) + __builtin_trap(); + size_t bytes = ant_image_end - ant_image; + const unsigned context = thread_find(THREAD_SIGNATURE, 0); + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || + thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) + __builtin_trap(); + thread_tss_write(0, ant_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + for (unsigned i = 0; i < CORELINK_CROSS_BYTES / 8; ++i) { + uint64_t value = 0; + for (unsigned j = 0; j < 8; ++j) + value |= (uint64_t)pattern(core, i * 8 + j) << (8 * j); + input[core][i] = value; + } + phase(core, 0); + barrier(0); + unsigned next = (core + 1) % POLY_THREAD_CORES; + unsigned previous = (core + POLY_THREAD_CORES - 1) % POLY_THREAD_CORES; + corelink_transfer(0, next + 1, bank_bytes - 16, 37); + barrier(1); + corelink_transfer(bank_bytes - 16, previous + 1, 128, 37); + __atomic_store_n(&ready[core], 1, __ATOMIC_RELEASE); + while (!__atomic_load_n(&ready[next], __ATOMIC_ACQUIRE)) { + } + phase(core, 1); + for (unsigned i = 0; i < CORELINK_CROSS_BYTES; ++i) { + unsigned expected = pattern(core, i >= 128 && i < 165 ? i - 128 : i); + if (((unsigned char *)output[core])[i] != expected) + __builtin_trap(); + } + phase(core, 2); + __atomic_store_n(&done[core], 1, __ATOMIC_RELEASE); +} +int main(void) { + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart) + exercise(hart - 1); + else + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + while (!__atomic_load_n(&done[core], __ATOMIC_ACQUIRE)) { + } + return 0; +} diff --git a/examples/designs/poly/workloads/device/runtime/ant_corelink_main.c b/examples/designs/poly/workloads/device/runtime/ant_corelink_main.c new file mode 100644 index 000000000..daf39333b --- /dev/null +++ b/examples/designs/poly/workloads/device/runtime/ant_corelink_main.c @@ -0,0 +1,92 @@ +#include "ant_corelink.h" +#include +#include +#include +_Static_assert( + BB_NUM_CORES == POLY_THREAD_CORES + 1 && POLY_THREAD_CORES >= 2 && + !(POLY_THREAD_CORES & (POLY_THREAD_CORES - 1)), + "CoreLink gate requires a power-of-two thread-bearing core count"); +extern const unsigned char ant_image[], ant_image_end[]; +static uint64_t input[POLY_THREAD_CORES][CORELINK_TEST_BYTES / 8] + __attribute__((aligned(64))); +static uint64_t output[POLY_THREAD_CORES][CORELINK_TEST_BYTES / 8] + __attribute__((aligned(64))); +static unsigned done[POLY_THREAD_CORES]; +static unsigned arrived[2]; +static void barrier(unsigned phase) { + __atomic_fetch_add(&arrived[phase], 1, __ATOMIC_ACQ_REL); + while (__atomic_load_n(&arrived[phase], __ATOMIC_ACQUIRE) != + POLY_THREAD_CORES) { + } +} +static unsigned pattern(unsigned core, unsigned i) { + return (i * 17 + core * 23 + 3) & 255; +} +static void phase(unsigned core, unsigned operation) { + const unsigned context = thread_find(THREAD_SIGNATURE, 0); + size_t bytes = ant_image_end - ant_image; + struct ant_corelink_args args = {operation, (uintptr_t)input[core], + (uintptr_t)output[core]}; + thread_tss_write(0, &args, sizeof(args)); + thread_load(context, THREAD_TLS, 0, 0, (sizeof(args) + 7) & ~7u); + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE); + struct thread_task task = {core * 3 + operation + 1, + 0, + bytes, + base, + base + + thread_query(context, THREAD_FIELD_TLS_BYTES), + THREAD_SIGNATURE}; + thread_acquire(); + thread_start(context, &task); + int cancelled; + if (thread_wait(context, task.id, &cancelled) || cancelled) + __builtin_trap(); + thread_release(); +} +static void exercise(unsigned core) { + uint64_t bank_bytes = corelink_query(CORELINK_BANK_BYTES); + if (corelink_query(CORELINK_CORE_ID) != core + 1 || + corelink_query(CORELINK_CORE_COUNT) != BB_NUM_CORES || + corelink_query(CORELINK_BANK_BYTES) < CORELINK_TEST_BYTES || + corelink_query(CORELINK_SHARED_BYTES) < bank_bytes * 2) + __builtin_trap(); + size_t bytes = ant_image_end - ant_image; + const unsigned context = thread_find(THREAD_SIGNATURE, 0); + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || + thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) + __builtin_trap(); + thread_tss_write(0, ant_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + for (unsigned i = 0; i < CORELINK_TEST_BYTES / 8; ++i) { + uint64_t value = 0; + for (unsigned j = 0; j < 8; ++j) + value |= (uint64_t)pattern(core, i * 8 + j) << (8 * j); + input[core][i] = value; + } + phase(core, 0); + barrier(0); + unsigned next = (core + 1) % POLY_THREAD_CORES; + unsigned previous = (core + POLY_THREAD_CORES - 1) % POLY_THREAD_CORES; + corelink_transfer(0, next + 1, bank_bytes + 16, 4101); + barrier(1); + phase(core, 1); + for (unsigned i = 0; i < CORELINK_TEST_BYTES; ++i) { + unsigned expected = i >= 16 && i < 4117 ? pattern(previous, i - 16) : 0; + if (((unsigned char *)output[core])[i] != expected) + __builtin_trap(); + } + phase(core, 2); + __atomic_store_n(&done[core], 1, __ATOMIC_RELEASE); +} +int main(void) { + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart) + exercise(hart - 1); + else + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + while (!__atomic_load_n(&done[core], __ATOMIC_ACQUIRE)) { + } + return 0; +} diff --git a/examples/designs/poly/workloads/device/runtime/ant_corelink_reduce_control.c b/examples/designs/poly/workloads/device/runtime/ant_corelink_reduce_control.c new file mode 100644 index 000000000..5ed342439 --- /dev/null +++ b/examples/designs/poly/workloads/device/runtime/ant_corelink_reduce_control.c @@ -0,0 +1,40 @@ +#include "ant_corelink_reduce.h" +#include +extern const unsigned char ant_image[], ant_image_end[]; +void ant_corelink_reduce_phase(unsigned core, + const struct ant_corelink_reduce_args *args) { + unsigned context = thread_find(THREAD_SIGNATURE, 0); + size_t bytes = ant_image_end - ant_image; + thread_tss_write(0, args, sizeof(*args)); + thread_load(context, THREAD_TLS, 0, 0, (sizeof(*args) + 7) & ~7u); + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE); + struct thread_task task = {core * 3 + args->phase + 1, + 0, + bytes, + base, + base + + thread_query(context, THREAD_FIELD_TLS_BYTES), + THREAD_SIGNATURE}; + thread_acquire(); + thread_start(context, &task); + int cancelled; + if (thread_wait(context, task.id, &cancelled) || cancelled) + __builtin_trap(); + thread_release(); +} +void ant_corelink_reduce_prepare(unsigned core, float *input, float *output) { + size_t bytes = ant_image_end - ant_image; + const unsigned context = thread_find(THREAD_SIGNATURE, 0); + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || + thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) + __builtin_trap(); + thread_tss_write(0, ant_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + for (unsigned i = 0; i < 272; ++i) + input[i] = 0.0f; + input[0] = core % 2 ? -(float)(core + 1) * 0.5f : (float)(core + 1) * 1.25f; + for (unsigned column = 0; column < 16; ++column) + input[16 + column * 16] = (float)(column + 1); + for (unsigned i = 0; i < 8; ++i) + output[i] = -77.0f; +} diff --git a/examples/designs/poly/workloads/device/runtime/ant_corelink_reduce_main.c b/examples/designs/poly/workloads/device/runtime/ant_corelink_reduce_main.c new file mode 100644 index 000000000..56982a88f --- /dev/null +++ b/examples/designs/poly/workloads/device/runtime/ant_corelink_reduce_main.c @@ -0,0 +1,97 @@ +#include "ant_corelink_reduce.h" +#include +#include +#include +#include +#include +_Static_assert( + BB_NUM_CORES == POLY_THREAD_CORES + 1 && POLY_THREAD_CORES >= 2 && + (POLY_THREAD_CORES & (POLY_THREAD_CORES - 1)) == 0, + "Reduction needs main and a power-of-two thread-bearing core count"); +static float input[POLY_THREAD_CORES][272] __attribute__((aligned(64))); +static float output[POLY_THREAD_CORES][16] __attribute__((aligned(64))); +static unsigned ready[POLY_THREAD_CORES], broadcast, done[POLY_THREAD_CORES]; +enum { WINDOW = 4 }; +static uint64_t shared_base[POLY_THREAD_CORES], main_base[WINDOW]; +static unsigned main_ready, batch; +static float partial(unsigned core, unsigned i) { + return (float)(i + 1) * + (core % 2 ? -(float)(core + 1) * 0.5f : (float)(core + 1) * 1.25f); +} +static void controller(unsigned core) { + ant_corelink_reduce_prepare(core, input[core], output[core]); + struct ant_corelink_reduce_args args = {0, (uintptr_t)input[core], + (uintptr_t)output[core]}; + ant_corelink_reduce_phase(core, &args); + shared_base[core] = + corelink_shared_export(0, THREAD_FRONTEND_SHARED_VBANK_BASE, 0); + while (!__atomic_load_n(&main_ready, __ATOMIC_ACQUIRE) || + __atomic_load_n(&batch, __ATOMIC_ACQUIRE) != core / WINDOW) { + } + corelink_transfer(shared_base[core], 0, main_base[core % WINDOW], 16); + __atomic_store_n(&ready[core], 1, __ATOMIC_RELEASE); + while (!__atomic_load_n(&broadcast, __ATOMIC_ACQUIRE)) { + } + args.phase = 1; + ant_corelink_reduce_phase(core, &args); + for (unsigned i = 0; i < 4; ++i) { + float expected = 0.0f; + for (unsigned rank = 0; rank < POLY_THREAD_CORES; ++rank) + expected += partial(rank, i); + if (output[core][i] != partial(core, i) || output[core][i + 4] != expected) + __builtin_trap(); + } + corelink_shared_release(0, THREAD_FRONTEND_SHARED_VBANK_BASE, 0); + args.phase = 2; + ant_corelink_reduce_phase(core, &args); + __atomic_store_n(&done[core], 1, __ATOMIC_RELEASE); +} +static void reduce(void) { + unsigned groups = POLY_THREAD_CORES < WINDOW ? POLY_THREAD_CORES : WINDOW; + unsigned source = 32, accumulator = 33, target = 34; + bb_mem_alloc(source, 1, groups); + bb_mem_alloc(accumulator, 1, 1); + bb_mem_alloc(target, 1, 1); + for (unsigned group = 0; group < groups; ++group) + main_base[group] = corelink_shared_export(0, source, group); + __atomic_store_n(&main_ready, 1, __ATOMIC_RELEASE); + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) { + while (!__atomic_load_n(&ready[core], __ATOMIC_ACQUIRE)) { + } + bb_f32add(source, accumulator, target, 1, core % WINDOW, core == 0); + unsigned previous = accumulator; + accumulator = target; + target = previous; + if (core % WINDOW == WINDOW - 1) { + __atomic_store_n(&batch, core / WINDOW + 1, __ATOMIC_RELEASE); + } + } + uint64_t result = corelink_shared_export(0, accumulator, 0); + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + corelink_transfer(result, core + 1, shared_base[core] + 16, 16); + __atomic_store_n(&broadcast, 1, __ATOMIC_RELEASE); + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + while (!__atomic_load_n(&done[core], __ATOMIC_ACQUIRE)) { + } + corelink_shared_release(0, accumulator, 0); + for (unsigned group = 0; group < groups; ++group) + corelink_shared_release(0, source, group); + bb_mem_release(source); + bb_mem_release(accumulator); + bb_mem_release(target); +} +int main(void) { + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart > POLY_THREAD_CORES || corelink_query(CORELINK_CORE_ID) != hart || + corelink_query(CORELINK_CORE_COUNT) != POLY_THREAD_CORES + 1 || + corelink_query(CORELINK_BANK_BYTES) < 48 || + corelink_query(CORELINK_SHARED_BYTES) < + corelink_query(CORELINK_BANK_BYTES) + POLY_THREAD_CORES * 16) + __builtin_trap(); + if (hart) + controller(hart - 1); + else + reduce(); + return 0; +} diff --git a/examples/designs/poly/workloads/device/runtime/ant_mesh_main.c b/examples/designs/poly/workloads/device/runtime/ant_mesh_main.c new file mode 100644 index 000000000..4b757b0b3 --- /dev/null +++ b/examples/designs/poly/workloads/device/runtime/ant_mesh_main.c @@ -0,0 +1,78 @@ +#include "ant_mesh.h" +#include + +enum { + BANKS = CORE_VIRTUAL_BANK_COUNT - THREAD_FRONTEND_SHARED_VBANK_BASE, + ROW_BYTES = THREAD_MEM_BANK_WIDTH / 8 +}; +_Static_assert(BANKS >= 2 && THREAD_MEM_BANK_WIDTH == 128, + "Mesh tests require two 128-bit input rows"); +static uint8_t input[POLY_THREAD_CORES][BANKS * ROW_BYTES] + __attribute__((aligned(64))); +static uint8_t output[POLY_THREAD_CORES][5 * 2 * ROW_BYTES + BANKS * ROW_BYTES] + __attribute__((aligned(64))); +static unsigned done[POLY_THREAD_CORES]; +extern const unsigned char ant_image[], ant_image_end[]; +static void exercise(unsigned core) { + if (thread_query(0, THREAD_FIELD_COUNT) != 5) + __builtin_trap(); + for (unsigned i = 0; i < sizeof(input[core]); ++i) + input[core][i] = POLY_MESH_PRIVATE + ? (i < ROW_BYTES ? 0x30 + i : 0xa0 + i - ROW_BYTES) + : (i / ROW_BYTES * 17 + i % ROW_BYTES); + unsigned count = POLY_MESH_PRIVATE ? 5 : 1; + size_t bytes = ant_image_end - ant_image; + struct thread_task tasks[5]; + // Both Attention and FFN execute the same common bank-ISA payload. + for (unsigned context = 0; context < count; ++context) { + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE); + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES)) + __builtin_trap(); + struct ant_mesh_args args = {(uintptr_t)input[core], + (uintptr_t)output[core], context}; + thread_tss_write(0, ant_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + thread_tss_write(0, &args, sizeof(args)); + thread_load(context, THREAD_TLS, 0, 0, (sizeof(args) + 7) & ~7u); + tasks[context] = (struct thread_task){ + context + 1, + 0, + bytes, + base, + base + thread_query(context, THREAD_FIELD_TLS_BYTES), + thread_query(context, THREAD_FIELD_SIGNATURE)}; + } + thread_acquire(); + for (unsigned context = 0; context < count; ++context) + thread_start(context, &tasks[context]); + for (unsigned context = 0; context < count; ++context) { + int cancelled; + if (thread_wait(context, tasks[context].id, &cancelled) || cancelled) + __builtin_trap(); + } + thread_release(); + if (POLY_MESH_PRIVATE) { + for (unsigned target = 0; target < 5; ++target) + for (unsigned byte = 0; byte < ROW_BYTES; ++byte) + if (output[core][target * 2 * ROW_BYTES + ROW_BYTES + byte] != + input[core][ROW_BYTES + byte]) + __builtin_trap(); + } else { + for (unsigned i = 0; i < sizeof(input[core]); ++i) + if (output[core][i] != input[core][i]) + __builtin_trap(); + } + __atomic_store_n(&done[core], 1, __ATOMIC_RELEASE); +} +int main(void) { + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart == WORKLOAD_REPORT_HART) { + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + while (!__atomic_load_n(&done[core], __ATOMIC_ACQUIRE)) { + } + } else { + exercise(hart - 1); + } + return 0; +} diff --git a/examples/designs/poly/workloads/device/runtime/ant_mesh_move_main.c b/examples/designs/poly/workloads/device/runtime/ant_mesh_move_main.c new file mode 100644 index 000000000..62efbdc7b --- /dev/null +++ b/examples/designs/poly/workloads/device/runtime/ant_mesh_move_main.c @@ -0,0 +1,92 @@ +#define _GNU_SOURCE +#include "ant_mesh_move.h" +#include + +extern const unsigned char ant_image[], ant_image_end[]; +enum { ROW_WORDS = THREAD_MEM_BANK_WIDTH / 64, ROWS = 2 }; +static uint64_t input[POLY_THREAD_CORES][ROWS * ROW_WORDS] + __attribute__((section(".data"), aligned(4096))); +static uint64_t output[POLY_THREAD_CORES][ROWS * ROW_WORDS] + __attribute__((section(".data"), aligned(4096))); +static unsigned done[POLY_THREAD_CORES]; + +static void run(unsigned context, unsigned id, + struct ant_mesh_move_args *args) { + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE); + thread_tss_write(64, args, sizeof(*args)); + thread_load(context, THREAD_TLS, 0, 64, sizeof(*args)); + struct thread_task task = {id, + 0, + ant_image_end - ant_image, + base, + base + + thread_query(context, THREAD_FIELD_TLS_BYTES), + THREAD_SIGNATURE}; + thread_start(context, &task); + int cancelled; + if (thread_wait(context, id, &cancelled) || cancelled) + __builtin_trap(); +} + +static struct thread_task prepare(unsigned context, unsigned id, + struct ant_mesh_move_args *args) { + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE); + thread_tss_write(64, args, sizeof(*args)); + thread_load(context, THREAD_TLS, 0, 64, sizeof(*args)); + return (struct thread_task){id, + 0, + ant_image_end - ant_image, + base, + base + + thread_query(context, THREAD_FIELD_TLS_BYTES), + THREAD_SIGNATURE}; +} + +static void exercise(unsigned tile) { + size_t bytes = ant_image_end - ant_image; + for (unsigned context = 0; context < 2; ++context) { + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || + thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) + __builtin_trap(); + thread_tss_write(0, ant_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + } + for (unsigned i = 0; i < ROWS * ROW_WORDS; ++i) { + input[tile][i] = ((uint64_t)(tile + 1) << 56) | (0x100 + i); + output[tile][i] = 0; + } + uint64_t zero[2] = {0}; + thread_tss_write(0, zero, sizeof(zero)); + struct ant_mesh_move_args args = { + (uintptr_t)input[tile], (uintptr_t)output[tile], + thread_query(0, THREAD_FIELD_TSS_BASE), MESH_FILL}; + thread_acquire(); + run(0, tile * 3 + 1, &args); + args.role = MESH_MOVE; + struct thread_task move = prepare(0, tile * 3 + 2, &args); + args.role = MESH_CONSUME; + struct thread_task consume = prepare(1, tile * 3 + 3, &args); + thread_start(1, &consume); + thread_start(0, &move); + int cancelled; + if (thread_wait(0, move.id, &cancelled) || cancelled || + thread_wait(1, consume.id, &cancelled) || cancelled) + __builtin_trap(); + thread_release(); + for (unsigned i = 0; i < ROWS * ROW_WORDS; ++i) + if (output[tile][i] != input[tile][i]) + __builtin_trap(); + __atomic_store_n(&done[tile], 1, __ATOMIC_RELEASE); +} + +int main(void) { + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart != WORKLOAD_REPORT_HART) + exercise(hart - 1); + else + for (unsigned tile = 0; tile < POLY_THREAD_CORES; ++tile) + while (!__atomic_load_n(&done[tile], __ATOMIC_ACQUIRE)) { + } + return 0; +} diff --git a/examples/designs/poly/workloads/device/runtime/ctests/CMakeLists.txt b/examples/designs/poly/workloads/device/runtime/ctests/CMakeLists.txt new file mode 100644 index 000000000..799aaa75f --- /dev/null +++ b/examples/designs/poly/workloads/device/runtime/ctests/CMakeLists.txt @@ -0,0 +1,7 @@ +set(BUCKYBALL_CTEST_TARGET main) +set(BUCKYBALL_CTEST_GROUP_TARGET poly-ctests-build) + + +list(APPEND BUCKYBALL_DESIGN_COMMON_SOURCES + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c) +add_buckyball_ctest(corelink_lease.c) diff --git a/examples/designs/poly/workloads/device/runtime/ctests/corelink_lease.c b/examples/designs/poly/workloads/device/runtime/ctests/corelink_lease.c new file mode 100644 index 000000000..c05fa5967 --- /dev/null +++ b/examples/designs/poly/workloads/device/runtime/ctests/corelink_lease.c @@ -0,0 +1,35 @@ +#include +#include +#include +#include + +static const uint32_t input[2][4] __attribute__((aligned(64))) = { + {1, 2, 3, 4}, {0x12345678, 0xabcdef01, 7, 8}}; +static volatile uint32_t output[8] __attribute__((aligned(64))); + +int main(void) { + unsigned bank = THREAD_FRONTEND_SHARED_VBANK_BASE; + bb_mem_alloc(bank, 1, 2); + for (unsigned i = 0; i < 8; ++i) + output[i] = 0xaaaaaaaa; + for (unsigned group = 0; group < 2; ++group) + bb_mvin_group((uintptr_t)input[group], bank, group, 1, 1); + uint64_t address = corelink_shared_export(0, bank, 1); + if (corelink_shared_export(0, bank, 1) != address) + __builtin_trap(); + corelink_shared_release(0, bank, 1); + if (corelink_read64(address) != UINT64_C(0xabcdef0112345678)) + __builtin_trap(); + corelink_write64(address + 8, UINT64_C(0x76543210deadbeef)); + bb_mvout((uintptr_t)output, bank, 1, 1); + for (unsigned i = 0; i < 4; ++i) + if (output[i] != input[0][i]) + __builtin_trap(); + if (output[4] != input[1][0] || output[5] != input[1][1] || + output[6] != 0xdeadbeef || output[7] != 0x76543210) + __builtin_trap(); + corelink_shared_release(0, bank, 1); + bb_mem_release(bank); + puts("Main CoreLink lease and NPU shared storage PASS"); + return 0; +} diff --git a/examples/designs/poly/workloads/device/runtime/stream_main.c b/examples/designs/poly/workloads/device/runtime/stream_main.c new file mode 100644 index 000000000..14f379986 --- /dev/null +++ b/examples/designs/poly/workloads/device/runtime/stream_main.c @@ -0,0 +1,58 @@ +#include +#include +#include +#include +#include +#include +#include +#include +enum { M = 16, N = 16, K = 32 }; +static float a[M * K] __attribute__((aligned(4096))); +static float b[N * K] __attribute__((aligned(4096))); +static float output[M * N] __attribute__((aligned(4096))); +static void job(void *argument) { + (void)argument; + for (unsigned bank = 3; bank <= 5; ++bank) + bb_mem_alloc(bank, 1, 1); + bb_mvin((uintptr_t)a, 3, sizeof(a) / 16, 1); + // A DMA call has completed before the CPU is allowed to reuse its source. + for (unsigned i = 0; i < M * K; ++i) + a[i] = -1; + bb_mvin((uintptr_t)b, 4, sizeof(b) / 16, 1); + bb_mxmm_f32(3, 4, 5, M, N, K, 1, 1, 0); + bb_mvout((uintptr_t)output, 5, sizeof(output) / 16, 1); + for (unsigned row = 0; row < M; ++row) + for (unsigned col = 0; col < N; ++col) + if (output[row * N + col] != (float)(K * (row + 1) * (col + 1))) + abort(); + for (unsigned bank = 3; bank <= 5; ++bank) + bb_mem_release(bank); +} +// Runs on the core; the stream executes its Ball instructions on a hardware +// thread. +static int run(void *engine, void *argument) { + (void)engine; + (void)argument; + void *workspace = aligned_alloc(64, 1024 * 1024); + if (!workspace) + abort(); + workspace_init(workspace, 1024 * 1024); + workspace_begin(workspace, 1024 * 1024); + stream_run(THREAD_SIGNATURE, job, NULL); + free(workspace); + return 0; +} +int main(void) { + for (unsigned row = 0; row < M; ++row) + for (unsigned k = 0; k < K; ++k) + a[row * K + k] = row + 1; + for (unsigned col = 0; col < N; ++col) + for (unsigned k = 0; k < K; ++k) + b[col * K + k] = col + 1; + for (unsigned i = 0; i < M * N; ++i) + output[i] = -1; + if (runtime_linux_job(bb_cores[1].hart, bb_cores[1].signature, run, NULL)) + abort(); + puts("Stream DMA lifetime/Mxmm PASS"); + return 0; +} diff --git a/examples/designs/poly/workloads/device/soc/CMakeLists.txt b/examples/designs/poly/workloads/device/soc/CMakeLists.txt new file mode 100644 index 000000000..33a05874a --- /dev/null +++ b/examples/designs/poly/workloads/device/soc/CMakeLists.txt @@ -0,0 +1,39 @@ +# Cross-core Linux control threads share one coherent LLC. +set(POLY_CONTROL_NAME + "${DESIGN_NAME}-main-ctest-design-controllers" +) +set(POLY_CONTROL_ELF ${POLY_CONTROL_NAME}-multicore-baremetal) +buckyball_enforce_ctest_line_limit("${CMAKE_CURRENT_SOURCE_DIR}/controllers.c") +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${POLY_CONTROL_ELF} + COMMAND + ${ELF_CC} -O2 -g -march=rv64gc -mcmodel=medany -msmall-data-limit=0 + -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax -Wl,-T,${BBSIM_LD} + ${DESIGN_DEFINES} -I${RUNTIME_INCLUDE_DIR} -I${BAREMETAL_DIR} -o ${POLY_CONTROL_ELF} + ${POLY_BOOT_SOURCE} ${CMAKE_CURRENT_SOURCE_DIR}/controllers.c -lgcc + DEPENDS + ${POLY_BOOT_SOURCE} + controllers.c + ${BAREMETAL_DIR}/start.S + ${BBSIM_LD} + ${RUNTIME_INCLUDE_DIR}/bb_cores.h + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM +) +add_custom_target( + ${POLY_CONTROL_NAME}-ctest-build + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${POLY_CONTROL_ELF} +) +add_dependencies(poly-ctests-build ${POLY_CONTROL_NAME}-ctest-build) + +set(BUCKYBALL_CTEST_TARGET attention) +add_buckyball_linux_ctest(${POLY_CONTROL_NAME} ${CMAKE_CURRENT_SOURCE_DIR} controllers.c) +target_include_directories( + ${POLY_CONTROL_NAME}-linux + PRIVATE ${RUNTIME_INCLUDE_DIR} +) +target_compile_definitions(${POLY_CONTROL_NAME}-linux PRIVATE ${DESIGN_DEFS}) +target_link_libraries(${POLY_CONTROL_NAME}-linux PRIVATE pthread) +add_dependencies(poly-ctests-build ${POLY_CONTROL_NAME}-linux-build) + +add_dependencies(soctests-build ${POLY_CONTROL_NAME}-ctest-build) diff --git a/examples/chips/poly/workloads/ctests/controllers.c b/examples/designs/poly/workloads/device/soc/controllers.c similarity index 92% rename from examples/chips/poly/workloads/ctests/controllers.c rename to examples/designs/poly/workloads/device/soc/controllers.c index 0779a2b9b..3340dd79b 100644 --- a/examples/chips/poly/workloads/ctests/controllers.c +++ b/examples/designs/poly/workloads/device/soc/controllers.c @@ -1,6 +1,6 @@ #define _GNU_SOURCE +#include #include -#include #ifdef __linux__ #include #include @@ -9,7 +9,7 @@ #include #endif // Controllers publish cache-line payloads and contend on one atomic counter. -enum { CONTROLLERS = BB_MAIN_CORES + BB_COMPUTE_TILES, ROUNDS = 2 }; +enum { CONTROLLERS = BB_NUM_CORES, ROUNDS = 2 }; static uint64_t payload[CONTROLLERS][8] __attribute__((aligned(64))); static uint64_t counter __attribute__((aligned(64))); static uint64_t turn __attribute__((aligned(64))); @@ -70,11 +70,11 @@ int main(void) { for (unsigned hart = 0; hart < CONTROLLERS; ++hart) if (pthread_join(threads[hart], NULL)) return 6; - printf("Controllers SMP PASS cores=%u\n", (unsigned)CONTROLLERS); + printf("Controllers SMP PASS threads=%u\n", (unsigned)CONTROLLERS); return 0; #else uintptr_t hart; asm volatile("csrr %0, mhartid" : "=r"(hart)); - return exercise(hart); + return exercise(bb_hart_slot(hart)); #endif } diff --git a/examples/designs/toy/arch/build.sc b/examples/designs/toy/arch/build.sc new file mode 100644 index 000000000..b4ffafece --- /dev/null +++ b/examples/designs/toy/arch/build.sc @@ -0,0 +1,17 @@ +import mill._ +import mill.scalalib._ +import $file.^.^.^.^.arch.src.main.scala.framework.device.cbus.{build => cbus_build} +import $file.^.^.^.{build => examples} +import $file.^.^.^.prefab.cores.rocket.arch.{build => rocket} +val designRoot: os.Path = millSourcePath / os.up + +object design extends SbtModule { + override def generatedSources = T(cbus_build.registry(sources() ++ rocket.library.sources(), T.dest)) + override def forkWorkingDir = T(designRoot / os.up / os.up / os.up / "arch") + override def scalaVersion = "2.13.16" + override def moduleDeps = Seq(examples.library, rocket.library) + override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") + override def sources = T.sources(designRoot / "arch", designRoot / "configs") +} diff --git a/examples/designs/toy/arch/device/DeviceTop.scala b/examples/designs/toy/arch/device/DeviceTop.scala new file mode 100644 index 000000000..06901ebe8 --- /dev/null +++ b/examples/designs/toy/arch/device/DeviceTop.scala @@ -0,0 +1,72 @@ +package examples.designs.toy + +import framework.lib.arbiter.{RRArbiter} + +import chisel3._ +import chisel3.util._ +import framework.device.peripherals.bootrom.BootRomLines +import framework.device.peripherals.Controller +import framework.ip.bus.axi4.Decoder +import framework.ip.bus.chi.snf.LineResponse +import framework.lib.queue.{Queue => ResponseQueue} +import chisel3.experimental.hierarchy.{instantiable, Instance, Instantiate} +import framework.device.configs.DeviceParam +import framework.device.cbus.CBus +import framework.device.cbus.generated.CoreComponents +import buckyball.config.Design +import _root_.configs.GlobalConfig +import framework.core.clink.CLinkIO +import framework.device.dlink.BaseDevice + +@instantiable +class DeviceTop(b: GlobalConfig, design: Design) extends BaseDevice(b.device.ddr.axi) { + val deviceParam = b.device + + val Seq(coreParam) = deviceParam.cores + + val bus = Instantiate(new CBus(b.device, CBus.generators(design, b.device, b.sim, CoreComponents.factories))) + dlink <> bus.dlink + + val p = deviceParam + val peripherals = + Instantiate(new Controller(deviceParam, deviceParam.ddr.axi, coreParam.memory.memoryAccess.memory)) + + val peripheralBus = Instantiate(new Decoder( + Seq( + (deviceParam.clint.base, deviceParam.clint.bytes), + (deviceParam.plic.base, deviceParam.plic.bytes), + (deviceParam.scu.baseAddress, deviceParam.scu.totalSizeBytes) + ), + deviceParam.ddr.axi + )) + + peripheralBus.io.in <> bus.uncached + peripherals.io.axi <> peripheralBus.io.matched + bus.external <> peripheralBus.io.other + peripherals.io.hardwareInterrupt.sources := 0.U + bus.interrupts := peripherals.io.hardwareInterrupt.harts + + val romParams = deviceParam.bootrom + val rom = Instantiate(new BootRomLines(deviceParam.bootrom, coreParam.memory.home.memory.chi)) + + val backing = bus.backing(0) + val ddr = bus.memory(0) + val inRom = backing.req.bits.addr >= romParams.base.U && + backing.req.bits.addr < (romParams.base + romParams.bytes).U + rom.io.request.valid := backing.req.valid && inRom + rom.io.request.bits := backing.req.bits + ddr.req.valid := backing.req.valid && !inRom + ddr.req.bits := backing.req.bits + backing.req.ready := Mux(inRom, rom.io.request.ready, ddr.req.ready) + + val responses = Instantiate(new RRArbiter(new LineResponse(coreParam.memory.home.memory.chi), 2)) + responses.io.in(0) <> rom.io.response + responses.io.in(1) <> ddr.resp + + // Buffer the selected answer so downstream backpressure cannot change its source. + val response = Instantiate(new ResponseQueue(new LineResponse(coreParam.memory.home.memory.chi), 1, false, false)) + + response.io.enq <> responses.io.out + backing.resp <> response.io.deq + +} diff --git a/examples/designs/toy/arch/device/core/ToyCore.scala b/examples/designs/toy/arch/device/core/ToyCore.scala new file mode 100644 index 000000000..8463667c5 --- /dev/null +++ b/examples/designs/toy/arch/device/core/ToyCore.scala @@ -0,0 +1,50 @@ +package examples.designs.toy + +import sims.configs.SimParam +import framework.core.configs.CoreParam +import framework.device.cbus.CoreFactory +import buckyball.config.{CorePlacement, Design} + +import examples.core.rocket.configs.Params +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.core.clink.{CLinkIO, HasCLink} + +/** One RV64GC hart and its local NPU, explicitly wired behind CLink. */ +@instantiable +class ToyCore(config: Params, simp: SimParam) extends Module with HasCLink { + val diffTest = simp.diffTest + val corep = config + require(corep.acc.nonEmpty) + require(corep.clink.harts == 1 && corep.clink.backing) + + @public + val clink = IO(new CLinkIO(corep.clink)) + + val loom: Instance[Loom] = Instantiate(new Loom(corep)) + val acc: Instance[Acc] = Instantiate(new Acc(config, simp)) + + loom.io.rocc <> acc.io.rocc + loom.io.memory <> acc.io.memory + loom.io.interrupts := clink.interrupts(0) + clink.memory.uncached <> loom.io.uncached + clink.memory.backing.get <> loom.io.backing + clink.memory.ddr <> acc.io.ddr +} + +object Toy extends CoreFactory { + val component = "toy" + + def parameters(design: Design, source: CorePlacement): CoreParam = { + val native = Params.fromProto(source); CoreParam.fromProto(source, Seq.empty).copy(clink = native.clink) + } + + def instantiate( + source: CorePlacement, + corep: CoreParam, + simp: SimParam, + traceIds: Seq[Int] + ): CLinkIO = + Instantiate(new ToyCore(Params.fromProto(source), simp)).clink + +} diff --git a/examples/designs/toy/arch/device/core/acc.scala b/examples/designs/toy/arch/device/core/acc.scala new file mode 100644 index 000000000..9ae91cf6a --- /dev/null +++ b/examples/designs/toy/arch/device/core/acc.scala @@ -0,0 +1,114 @@ +package examples.designs.toy + +import framework.acc.balldomain.bbus.generated.BallComponents +import sims.configs.SimParam +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import examples.core.rocket.configs.Params +import framework.core.memory.MemoryPort +import framework.acc.frontend.RoCCIO +import framework.ip.bus.axi4 +import framework.acc.balldomain.bbus.BBus +import framework.acc.frontend.Frontend +import framework.acc.memdomain.MemDomain +import framework.acc.gpdomain.GpDomain +import framework.acc.balldomain.BallDomain +import framework.acc.debug.Debug + +@instantiable +class Acc(config: Params, simp: SimParam) extends Module { + val diffTest = simp.diffTest + val corep = config + val acc = corep.acc.get + require(!acc.memDomain.sharedEnable) + + @public + val io = IO(new Bundle { + val rocc = new RoCCIO(corep.cpu.core.xLen, corep.permission) + val memory = new MemoryPort(corep.memory.dcache.tracking, corep.memory.memoryAccess.walker) + val ddr = new axi4.Port(acc.transport.axi) + }) + + val frontend: Instance[Frontend] = + Instantiate(new Frontend(acc.frontend, acc.memDomain, acc.gpDomain, acc.ballDomain, corep.permission, simp)) + + val memDomain: Instance[MemDomain] = Instantiate(new MemDomain( + acc.frontend, + acc.memDomain, + acc.gpDomain, + acc.transport, + corep.permission, + corep.memory.dcache.tracking, + corep.memory.memoryAccess.walker, + simp + )) + + val domain: Instance[BallDomain] = + Instantiate(new BallDomain( + acc.frontend, + acc.memDomain, + acc.gpDomain, + acc.ballDomain, + corep.permission, + BallComponents.factories + )) + + if (acc.gpDomain.enable) { + val gp: Instance[GpDomain] = Instantiate(new GpDomain(acc.frontend, acc.memDomain, acc.gpDomain)) + gp.io.command <> frontend.io.gpRequest.get + frontend.io.gpCompletion.get <> gp.io.complete + gp.io.image <> memDomain.io.image.get + memDomain.io.gpBanks.get <> gp.io.banks + memDomain.io.gpLayoutRequest.get <> gp.io.layoutRequest + gp.io.layoutResponse <> memDomain.io.gpLayoutResponse.get + domain.io.kernel.get.request <> gp.io.ball.request + gp.io.ball.response <> domain.io.kernel.get.response + } + frontend.io.rocc <> io.rocc + domain.io.issue <> frontend.io.ballRequest + frontend.io.ballCompletion <> domain.io.complete + memDomain.io.ball <> domain.io.memory + memDomain.io.issue <> frontend.io.memoryRequest + frontend.io.memoryCompletion <> memDomain.io.complete + memDomain.io.mapping <> frontend.io.mappingUpdate + frontend.io.barrier.release := frontend.io.barrier.arrive + memDomain.io.banks <> domain.io.banks + memDomain.io.ballLayoutRequest <> domain.io.layoutRequest + domain.io.layoutResponse <> memDomain.io.ballLayoutResponse + io.ddr <> memDomain.io.ddr + io.memory <> memDomain.io.memory + + val debug: Instance[Debug] = Instantiate(new Debug( + acc.frontend, + acc.memDomain, + acc.gpDomain, + acc.ballDomain, + simp, + traceId = corep.hartId, + sharedOwnerId = corep.hartId + )) + + debug.io.ant.valid := false.B + debug.io.ant.bits := 0.U.asTypeOf(debug.io.ant.bits) + debug.io.acc.frontend <> frontend.io.debug + debug.io.acc.trace.memory := memDomain.io.traceMemory + debug.io.acc.trace.ballStarted := domain.io.traceStarted + debug.io.acc.trace.ballCompleted := domain.io.traceCompleted + debug.io.acc.banks.zip(memDomain.io.bank_hashes).foreach { case (sink, source) => sink := source } + for (port <- memDomain.io.shared.bank.requests) { + port.read.req.ready := false.B + port.read.resp.valid := false.B + port.read.resp.bits := 0.U.asTypeOf(port.read.resp.bits) + port.write.req.ready := false.B + port.write.resp.valid := false.B + port.write.resp.bits := 0.U.asTypeOf(port.write.resp.bits) + } + memDomain.io.shared.bank.config.ready := false.B + memDomain.io.shared.bank.mapping.ready := false.B + memDomain.io.shared.bank.local.request.valid := false.B + memDomain.io.shared.bank.local.request.bits := 0.U.asTypeOf(memDomain.io.shared.bank.local.request.bits) + memDomain.io.shared.bank.local.response.ready := false.B + memDomain.io.shared.move.command.ready := false.B + memDomain.io.shared.move.completion.valid := false.B + memDomain.io.shared.move.completion.bits := false.B +} diff --git a/examples/designs/toy/arch/device/core/loom.scala b/examples/designs/toy/arch/device/core/loom.scala new file mode 100644 index 000000000..832e49cde --- /dev/null +++ b/examples/designs/toy/arch/device/core/loom.scala @@ -0,0 +1,48 @@ +package examples.designs.toy + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import examples.core.rocket.configs.{Params => RocketParam} +import examples.core.rocket.Execution +import framework.core.memory.{MemoryPort, MemorySystem} +import framework.acc.frontend.RoCCIO +import framework.core.clink.CoreInterrupts +import framework.ip.bus.axi4 +import framework.ip.bus.chi.snf.{LineRequest, LineResponse} + +/** Outer Loom execution core and private memory hierarchy; directly controls the local ACC. */ +@instantiable +class Loom(corep: RocketParam) extends Module { + require(corep.clink.backing) + + @public + val io = IO(new Bundle { + val interrupts = Input(new CoreInterrupts) + val rocc = Flipped(new RoCCIO(corep.cpu.core.xLen, corep.permission)) + val memory = Flipped(new MemoryPort(corep.memory.dcache.tracking, corep.memory.memoryAccess.walker)) + val uncached = new axi4.Port(corep.memory.uncached.axi) + + val backing = new Bundle { + val request = Decoupled(new LineRequest(corep.memory.home.memory.chi)) + val response = Flipped(Decoupled(new LineResponse(corep.memory.home.memory.chi))) + } + + }) + + val execution: Instance[Execution] = Instantiate(new Execution(corep, commandsEnabled = true)) + + val memory: Instance[MemorySystem] = Instantiate(new MemorySystem(corep.memory, externalHome = false)) + + execution.io.interrupts := io.interrupts + execution.io.rocc <> io.rocc + memory.io.fetch <> execution.io.fetch + memory.io.memory <> execution.io.data + memory.io.timeRequest <> execution.io.timeRequest + execution.io.timeResponse <> memory.io.timeResponse + memory.io.commandAllowed := !io.rocc.busy + memory.io.services <> io.memory + io.uncached <> memory.io.uncached + io.backing <> memory.io.backing.get + +} diff --git a/examples/designs/toy/configs/design.toml b/examples/designs/toy/configs/design.toml new file mode 100644 index 000000000..1d45ec7e4 --- /dev/null +++ b/examples/designs/toy/configs/design.toml @@ -0,0 +1,5 @@ +[device] +include = "device/default.toml" + +[sims] +include = "sims/default.toml" diff --git a/examples/designs/toy/configs/device/core/acc/balldomain/balls/smatmul/lane16.toml b/examples/designs/toy/configs/device/core/acc/balldomain/balls/smatmul/lane16.toml new file mode 100644 index 000000000..70527a327 --- /dev/null +++ b/examples/designs/toy/configs/device/core/acc/balldomain/balls/smatmul/lane16.toml @@ -0,0 +1,7 @@ +InputNum = 16 +inputWidth = 8 +lane = 16 +wsReuseCores = 2 +outputWidth = 32 +numMulThreads = 16 +numCasThreads = 16 diff --git a/examples/designs/toy/configs/device/core/acc/balldomain/default.toml b/examples/designs/toy/configs/device/core/acc/balldomain/default.toml new file mode 100644 index 000000000..3ae947026 --- /dev/null +++ b/examples/designs/toy/configs/device/core/acc/balldomain/default.toml @@ -0,0 +1,81 @@ +[[balls]] +id = 0 +name = "GemminiBall" +component = "gemmini" +param = "default" +port = { inBW = 2, outBW = 4, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "GEMMINI_LOOP_WS_CONFIG_BOUNDS", funct7 = 80 }, + { mnemonic = "GEMMINI_LOOP_WS_CONFIG_ADDR_A", funct7 = 81 }, + { mnemonic = "GEMMINI_LOOP_WS_CONFIG_ADDR_B", funct7 = 82 }, + { mnemonic = "GEMMINI_LOOP_WS_CONFIG_ADDR_D", funct7 = 83 }, + { mnemonic = "GEMMINI_LOOP_WS_CONFIG_ADDR_C", funct7 = 84 }, + { mnemonic = "GEMMINI_LOOP_WS_CONFIG_STRIDES_AB", funct7 = 85 }, + { mnemonic = "GEMMINI_LOOP_WS_CONFIG_STRIDES_DC", funct7 = 86 }, + { mnemonic = "GEMMINI_LOOP_WS", funct7 = 87 }, + { mnemonic = "GEMMINI_LOOP_CONV_WS_CONFIG_1", funct7 = 96 }, + { mnemonic = "GEMMINI_LOOP_CONV_WS_CONFIG_2", funct7 = 97 }, + { mnemonic = "GEMMINI_LOOP_CONV_WS_CONFIG_3", funct7 = 98 }, + { mnemonic = "GEMMINI_LOOP_CONV_WS_CONFIG_4", funct7 = 99 }, + { mnemonic = "GEMMINI_LOOP_CONV_WS_CONFIG_5", funct7 = 100 }, + { mnemonic = "GEMMINI_LOOP_CONV_WS_CONFIG_6", funct7 = 101 }, + { mnemonic = "GEMMINI_LOOP_CONV_WS_CONFIG_7", funct7 = 102 }, + { mnemonic = "GEMMINI_LOOP_CONV_WS_CONFIG_8", funct7 = 103 }, + { mnemonic = "GEMMINI_LOOP_CONV_WS_CONFIG_9", funct7 = 104 }, + { mnemonic = "GEMMINI_LOOP_CONV_WS", funct7 = 105 }, + { mnemonic = "GEMMINI_CONFIG", funct7 = 2 }, + { mnemonic = "GEMMINI_FLUSH", funct7 = 3 }, + { mnemonic = "GEMMINI_PRELOAD", funct7 = 21 }, + { mnemonic = "GEMMINI_COMPUTE_PRELOADED", funct7 = 66 }, + { mnemonic = "GEMMINI_COMPUTE_ACCUMULATED", funct7 = 67 }, +] + +[[balls]] +id = 1 +name = "TraceBall" +component = "trace" +param = "default" +port = { inBW = 1, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "BDB_COUNTER", funct7 = 4 }, +] + +[[balls]] +id = 3 +name = "ReluBall" +component = "relu" +param = "default" +port = { inBW = 1, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "RELU", funct7 = 50 }, +] + +[[balls]] +id = 4 +name = "RedmuleBall" +component = "redmule" +param = "default" +port = { inBW = 2, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "REDMULE_F16", funct7 = 68 }, +] + +[[balls]] +id = 5 +name = "VecBall" +component = "vector" +param = "default" +port = { inBW = 2, outBW = 4, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "VECMAT16", funct7 = 64 }, +] + +[[balls]] +id = 6 +name = "Mxfp2IntBall" +component = "mxfp2int" +param = "default" +port = { inBW = 2, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "MXFP2INT", funct7 = 65 }, +] diff --git a/examples/designs/toy/configs/device/core/acc/default.toml b/examples/designs/toy/configs/device/core/acc/default.toml new file mode 100644 index 000000000..5c3cf855f --- /dev/null +++ b/examples/designs/toy/configs/device/core/acc/default.toml @@ -0,0 +1,6 @@ +source = "../../../.." +balldomain = "balldomain/default.toml" +memdomain = "memdomain/toy.toml" +frontend = "frontend/toy.toml" +gpdomain = "gpdomain/toy.toml" +transport = "default" diff --git a/examples/designs/toy/configs/device/core/acc/demo.toml b/examples/designs/toy/configs/device/core/acc/demo.toml new file mode 100644 index 000000000..dc62522d8 --- /dev/null +++ b/examples/designs/toy/configs/device/core/acc/demo.toml @@ -0,0 +1,7 @@ +source = "../../../.." +# Toy ACC Configuration (demo frontend) + +balldomain = "balldomain/default.toml" +memdomain = "memdomain/toy.toml" +frontend = "frontend/demo.toml" +gpdomain = "gpdomain/toy.toml" diff --git a/examples/designs/toy/configs/device/core/acc/frontend/demo.toml b/examples/designs/toy/configs/device/core/acc/frontend/demo.toml new file mode 100644 index 000000000..5464f2d95 --- /dev/null +++ b/examples/designs/toy/configs/device/core/acc/frontend/demo.toml @@ -0,0 +1,9 @@ +bankAliasEnable = true +robEntries = 16 +rsOutOfOrderResponse = true +bankIdLen = 10 +maxPrivateVbank = 15 +sharedVbankBase = 32 +iterLen = 34 +subRobEnable = false +subRobDepth = 64 diff --git a/examples/designs/toy/configs/device/core/acc/frontend/toy.toml b/examples/designs/toy/configs/device/core/acc/frontend/toy.toml new file mode 100644 index 000000000..0a50370e8 --- /dev/null +++ b/examples/designs/toy/configs/device/core/acc/frontend/toy.toml @@ -0,0 +1,9 @@ +bankAliasEnable = false +robEntries = 16 +rsOutOfOrderResponse = true +bankIdLen = 10 +maxPrivateVbank = 15 +sharedVbankBase = 32 +iterLen = 34 +subRobEnable = true +subRobDepth = 4 diff --git a/examples/cores/attention/configs/rvv/default.toml b/examples/designs/toy/configs/device/core/acc/gpdomain/toy.toml similarity index 100% rename from examples/cores/attention/configs/rvv/default.toml rename to examples/designs/toy/configs/device/core/acc/gpdomain/toy.toml diff --git a/examples/designs/toy/configs/device/core/acc/memdomain/paper.toml b/examples/designs/toy/configs/device/core/acc/memdomain/paper.toml new file mode 100644 index 000000000..c36e3debc --- /dev/null +++ b/examples/designs/toy/configs/device/core/acc/memdomain/paper.toml @@ -0,0 +1,22 @@ +# Toy ACC Memory Domain Configuration (default) +# Private (per-thread) memory parameters. Shared memory lives in the core TOML. + +addrLen = 39 + +[bank] +layoutEnable = true +num = 16 +width = 128 +entries = 512 +maskLen = 16 +requestPorts = 7 + +[dma] +busWidth = 128 + +[mmio] +enable = false +bankNum = 16 +bankEntries = 64 +bankWidth = 8 +readWidth = 8 diff --git a/examples/designs/toy/configs/device/core/acc/memdomain/toy.toml b/examples/designs/toy/configs/device/core/acc/memdomain/toy.toml new file mode 100644 index 000000000..27f20096d --- /dev/null +++ b/examples/designs/toy/configs/device/core/acc/memdomain/toy.toml @@ -0,0 +1,22 @@ +# Toy ACC Memory Domain Configuration (default) +# Private (per-thread) memory parameters. Shared memory lives in the core TOML. + +addrLen = 39 + +[bank] +layoutEnable = true +num = 16 +width = 128 +entries = 1024 +maskLen = 16 +requestPorts = 7 + +[dma] +busWidth = 128 + +[mmio] +enable = true +bankNum = 16 +bankEntries = 64 +bankWidth = 8 +readWidth = 8 diff --git a/examples/designs/toy/configs/device/core/acc/paper.toml b/examples/designs/toy/configs/device/core/acc/paper.toml new file mode 100644 index 000000000..2d0e702c5 --- /dev/null +++ b/examples/designs/toy/configs/device/core/acc/paper.toml @@ -0,0 +1,7 @@ +source = "../../../.." +# Toy ACC Configuration (default) + +balldomain = "balldomain/default.toml" +memdomain = "memdomain/paper.toml" +frontend = "frontend/toy.toml" +gpdomain = "gpdomain/toy.toml" diff --git a/examples/designs/toy/configs/device/core/c2c/core.toml b/examples/designs/toy/configs/device/core/c2c/core.toml new file mode 100644 index 000000000..1284ec5d8 --- /dev/null +++ b/examples/designs/toy/configs/device/core/c2c/core.toml @@ -0,0 +1,3 @@ +enabled = false +peerCores = 1 +axi = "default" diff --git a/examples/designs/toy/configs/device/core/default.toml b/examples/designs/toy/configs/device/core/default.toml new file mode 100644 index 000000000..27bc0c078 --- /dev/null +++ b/examples/designs/toy/configs/device/core/default.toml @@ -0,0 +1,39 @@ +component = "toy" +implementation = "rocket.toml" +accelerator = "acc/default.toml" +threads = [] +hartId = 0 +resetVector = 0x10000 +mtimeAddress = 0x200bff8 +memory = "memory/core.toml" +fetch = "default" +fpu = "default" +interconnect = "interconnect/core.toml" +c2c = "c2c/core.toml" +controller = "default" + +[param] +threadDataBytes = 64 +xLen = 64 +vaddrBits = 39 +paddrBits = 36 +pgIdxBits = 12 +pgLevels = 3 +nPMPs = 16 + +[sharedMem] +enable = false +entries = 16384 +inputChannels = 1 +defaultGroupCount = 1 +virtualBankCount = 16 +latency = 10 +bankEntries = 1024 +bankWidth = 128 + +[compiler] +name = "toy" +component = "toy" + +[build] +toy = "design" diff --git a/examples/designs/toy/configs/device/core/interconnect/core.toml b/examples/designs/toy/configs/device/core/interconnect/core.toml new file mode 100644 index 000000000..13d04a5a6 --- /dev/null +++ b/examples/designs/toy/configs/device/core/interconnect/core.toml @@ -0,0 +1,2 @@ +axi = "default" +slots = 16 diff --git a/examples/designs/toy/configs/device/core/memory/access/core.toml b/examples/designs/toy/configs/device/core/memory/access/core.toml new file mode 100644 index 000000000..2e90444bc --- /dev/null +++ b/examples/designs/toy/configs/device/core/memory/access/core.toml @@ -0,0 +1,52 @@ +translationEntries = 16 +tagBits = 6 +resultLineBits = 0 +[[regions]] +base = 0x80000000 +bytes = 0x20000000 +cacheable = true +executable = true +readable = true +writable = true +atomic = true +normal = true + +[[regions]] +base = 0x10000 +bytes = 0x10000 +cacheable = true +executable = true +readable = true +writable = false +atomic = false +normal = true + +[[regions]] +base = 0x2000000 +bytes = 0x10000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0xc000000 +bytes = 0x4000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0x60000000 +bytes = 0x10000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false diff --git a/examples/designs/toy/configs/device/core/memory/core.toml b/examples/designs/toy/configs/device/core/memory/core.toml new file mode 100644 index 000000000..3caf53579 --- /dev/null +++ b/examples/designs/toy/configs/device/core/memory/core.toml @@ -0,0 +1,5 @@ +uncached = "default" +access = "access/core.toml" +icache = "icache/core.toml" +dcache = "dcache/core.toml" +home = "home/core.toml" diff --git a/examples/designs/toy/configs/device/core/memory/dcache/core.toml b/examples/designs/toy/configs/device/core/memory/dcache/core.toml new file mode 100644 index 000000000..4fb4beb6f --- /dev/null +++ b/examples/designs/toy/configs/device/core/memory/dcache/core.toml @@ -0,0 +1,15 @@ +[tracking] +entries = 8 +addressBits = 44 +idBits = 8 +lineBytes = 64 +maxRanges = 8 + +[cache] +nodeId = 2 +lines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/toy/configs/device/core/memory/home/core.toml b/examples/designs/toy/configs/device/core/memory/home/core.toml new file mode 100644 index 000000000..aa39cd52e --- /dev/null +++ b/examples/designs/toy/configs/device/core/memory/home/core.toml @@ -0,0 +1,13 @@ +agents = 2 +mshrEntries = 8 +nodeId = 64 +requesters = "default" + +[cache] +addressBits = 44 +lineBytes = 64 +sets = 64 +ways = 4 +idBits = 8 +metadataBits = 4 +responseDepth = 2 diff --git a/examples/designs/toy/configs/device/core/memory/icache/core.toml b/examples/designs/toy/configs/device/core/memory/icache/core.toml new file mode 100644 index 000000000..3cc7df83a --- /dev/null +++ b/examples/designs/toy/configs/device/core/memory/icache/core.toml @@ -0,0 +1,7 @@ +nodeId = 1 +lines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/toy/configs/device/core/rocket.toml b/examples/designs/toy/configs/device/core/rocket.toml new file mode 100644 index 000000000..256ac66e0 --- /dev/null +++ b/examples/designs/toy/configs/device/core/rocket.toml @@ -0,0 +1,12 @@ +component = "rocket" +physical_address_bits = 36 +hart_id_bits = 1 +page_levels = 3 +pmp_entries = 16 +virtual_memory = true +supervisor = true +user = true +floating_point = true +atomics = true +multiply_divide = true +external_home = false diff --git a/examples/designs/toy/configs/device/ddr.toml b/examples/designs/toy/configs/device/ddr.toml new file mode 100644 index 000000000..31815e9e1 --- /dev/null +++ b/examples/designs/toy/configs/device/ddr.toml @@ -0,0 +1,13 @@ +clients = 1 +slotsPerClient = 16 +dataBits = 128 +idBits = 4 +idBase = 0 +base = 0x80000000 +bytes = 0x20000000 +model = 0 + +[line] +nodeIdBits = 7 +addressBits = 44 +dataBits = 256 diff --git a/examples/designs/toy/configs/device/default.toml b/examples/designs/toy/configs/device/default.toml new file mode 100644 index 000000000..7a1fb7525 --- /dev/null +++ b/examples/designs/toy/configs/device/default.toml @@ -0,0 +1,8 @@ +ddr = "ddr.toml" +bootrom = "default" +clint = "default" +plic = "default" +scu = "default" + +[[cores]] +include = "core/default.toml" diff --git a/examples/designs/toy/configs/sims/default.toml b/examples/designs/toy/configs/sims/default.toml new file mode 100644 index 000000000..99109f9f2 --- /dev/null +++ b/examples/designs/toy/configs/sims/default.toml @@ -0,0 +1,18 @@ +diffTest = false + +[memory] +lineBytes = 64 +clockHz = 1000000000 + +[targets.device] +config = "examples.designs.toy.sims.device.DeviceConfig" +stack_name = "toy" +regression = "sims/device/regression" +kernel = "sims/device/workloads/kernel" +tapeout = "sims/device/tapeout" + +[targets.device.configs] +ToyVerilatorConfig = "verilator.toml" + +[verification] +registry = "../../sims/ip/uvm/registry.toml" diff --git a/examples/designs/toy/configs/sims/verilator.toml b/examples/designs/toy/configs/sims/verilator.toml new file mode 100644 index 000000000..20c5b7796 --- /dev/null +++ b/examples/designs/toy/configs/sims/verilator.toml @@ -0,0 +1 @@ +# ToyVerilatorConfig: the toy design as configured, with no overrides. diff --git a/examples/designs/toy/emu/Cargo.lock b/examples/designs/toy/emu/Cargo.lock new file mode 100644 index 000000000..4469013c3 --- /dev/null +++ b/examples/designs/toy/emu/Cargo.lock @@ -0,0 +1,821 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "bebop-bank-hash" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "snafu", +] + +[[package]] +name = "bebop-bemu" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", + "bebop-bemu-profile", + "bebop-dasm", + "bebop-dtb", + "bebop-elf", + "bebop-keyboard", + "bebop-memory", + "bebop-microphone", + "bebop-rtc", + "bebop-speaker", + "bebop-syscall", + "bebop-uart", + "bebop-vga", + "bebop-vm", + "clap", + "linkme", + "prost", + "prost-build", + "prost-types", + "rustc_apfloat", + "rvsim", + "snafu", + "toml", +] + +[[package]] +name = "bebop-bemu-profile" +version = "0.1.0" + +[[package]] +name = "bebop-dasm" +version = "0.1.0" + +[[package]] +name = "bebop-design-toy" +version = "0.1.0" +dependencies = [ + "bebop-bemu", +] + +[[package]] +name = "bebop-dtb" +version = "0.1.0" + +[[package]] +name = "bebop-elf" +version = "0.1.0" +dependencies = [ + "bebop-memory", +] + +[[package]] +name = "bebop-keyboard" +version = "0.1.0" + +[[package]] +name = "bebop-memory" +version = "0.1.0" +dependencies = [ + "log", + "serde", + "serde_json", + "sha2", +] + +[[package]] +name = "bebop-microphone" +version = "0.1.0" + +[[package]] +name = "bebop-rtc" +version = "0.1.0" + +[[package]] +name = "bebop-speaker" +version = "0.1.0" + +[[package]] +name = "bebop-syscall" +version = "0.1.0" +dependencies = [ + "bebop-memory", + "once_cell", +] + +[[package]] +name = "bebop-uart" +version = "0.1.0" + +[[package]] +name = "bebop-vga" +version = "0.1.0" + +[[package]] +name = "bebop-vm" +version = "0.1.0" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.190" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" + +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck", + "itertools", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.119", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustc_apfloat" +version = "0.2.3+llvm-462a31f5a5ab" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "486c2179b4796f65bfe2ee33679acf0927ac83ecf583ad6c91c3b4570911b9ad" +dependencies = [ + "bitflags", + "smallvec", +] + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rvsim" +version = "0.1.0" +dependencies = [ + "rustc_apfloat", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "smallvec" +version = "1.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" + +[[package]] +name = "snafu" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/examples/designs/toy/emu/Cargo.toml b/examples/designs/toy/emu/Cargo.toml new file mode 100644 index 000000000..e2856f42b --- /dev/null +++ b/examples/designs/toy/emu/Cargo.toml @@ -0,0 +1,9 @@ +[workspace] + +[package] +name = "bebop-design-toy" +version = "0.1.0" +edition = "2021" + +[dependencies] +bebop-bemu = { path = "../../../../bebop/src/nodes/bemu" } diff --git a/examples/designs/toy/emu/src/device/core/mod.rs b/examples/designs/toy/emu/src/device/core/mod.rs new file mode 100644 index 000000000..1658962cd --- /dev/null +++ b/examples/designs/toy/emu/src/device/core/mod.rs @@ -0,0 +1,39 @@ +use bebop_bemu::{ + ball::linkme, Accelerator, BemuProfileReport, CoreFactory, CoreModel, CoreParams, Cpu, DeviceModel, ModelOptions, + Whatever, CORES, +}; +use std::time::Duration; + +/// One RV64GC hart with its Ball accelerator on the RoCC port (arch: ToyCore). +pub struct ToyCore { + cpu: Cpu, + acc: Accelerator, +} + +impl CoreModel for ToyCore { + fn cpu(&self) -> &Cpu { + &self.cpu + } + fn cpu_mut(&mut self) -> &mut Cpu { + &mut self.cpu + } + fn step(&mut self, count: u64) -> Result<(), Whatever> { + self.cpu.step(count, &mut self.acc) + } + fn take_barrier(&mut self) -> bool { + self.acc.take_barrier() + } + fn profile_report(&self, total: Duration) -> Option { + self.acc.profile_report(total, &self.cpu) + } +} + +fn instantiate(device: &DeviceModel, params: &CoreParams, options: &ModelOptions) -> Result, String> { + let cpu = Cpu::new(device, params, options).map_err(|error| error.to_string())?; + let acc = Accelerator::attached(&cpu, options)?; + Ok(Box::new(ToyCore { cpu, acc })) +} + +#[linkme::distributed_slice(CORES)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static TOY: CoreFactory = CoreFactory { component: "toy", instantiate }; diff --git a/examples/designs/toy/emu/src/device/mod.rs b/examples/designs/toy/emu/src/device/mod.rs new file mode 100644 index 000000000..a6ce9b1dc --- /dev/null +++ b/examples/designs/toy/emu/src/device/mod.rs @@ -0,0 +1,2 @@ +//! The toy device: its own Core model; the cores are instantiated by component on the CBus. +pub mod core; diff --git a/examples/designs/toy/emu/src/lib.rs b/examples/designs/toy/emu/src/lib.rs new file mode 100644 index 000000000..545892414 --- /dev/null +++ b/examples/designs/toy/emu/src/lib.rs @@ -0,0 +1 @@ +pub mod device; diff --git a/examples/designs/toy/sims/device/arch/DeviceConfig.scala b/examples/designs/toy/sims/device/arch/DeviceConfig.scala new file mode 100644 index 000000000..b8fc8ef57 --- /dev/null +++ b/examples/designs/toy/sims/device/arch/DeviceConfig.scala @@ -0,0 +1,18 @@ +package examples.designs.toy.sims.device + +import buckyball.config.Design +import chisel3.experimental.hierarchy.{Instance, Instantiate} +import examples.designs.toy.DeviceTop +import framework.device.cbus.CBus +import framework.device.cbus.generated.CoreComponents +import framework.device.dlink.BaseDevice +import _root_.configs.GlobalConfig +import _root_.sims.soc.SystemTarget +import java.nio.file.{Files, Paths} + +class DeviceConfig(design: Design) + extends SystemTarget( + GlobalConfig.fromProto(design, CBus.parameters(design, CoreComponents.factories)) + ) { + def instantiate(b: GlobalConfig): Instance[BaseDevice] = Instantiate(new DeviceTop(b, design)) +} diff --git a/examples/designs/toy/sims/device/arch/build.sc b/examples/designs/toy/sims/device/arch/build.sc new file mode 100644 index 000000000..5b94111b8 --- /dev/null +++ b/examples/designs/toy/sims/device/arch/build.sc @@ -0,0 +1,17 @@ +import mill._ +import mill.scalalib._ +import $file.^.^.^.arch.{build => design_build} +import $file.^.^.^.^.^.^.arch.src.main.scala.sims.{build => sims_build} + +val entryRoot: os.Path = millSourcePath + +object simulation extends SbtModule { + override def millSourcePath = entryRoot + override def scalaVersion = "2.13.16" + override def moduleDeps = Seq(design_build.design, sims_build.library) + override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") + override def sources = T.sources(entryRoot / "DeviceConfig.scala") + override def mainClass = Some("sims.verilator.Elaborate") +} diff --git a/examples/designs/toy/sims/device/emu/Cargo.lock b/examples/designs/toy/sims/device/emu/Cargo.lock new file mode 100644 index 000000000..eb97e3ba0 --- /dev/null +++ b/examples/designs/toy/sims/device/emu/Cargo.lock @@ -0,0 +1,1971 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "assert_cmd" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aa3a22042e45de04255c7bf3626e239f450200fd0493c1e382263544b20aea6" +dependencies = [ + "anstyle", + "bstr", + "libc", + "predicates", + "predicates-core", + "predicates-tree", + "wait-timeout", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bebop" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", + "bebop-bemu", + "bebop-bemu-profile", + "bebop-dasm", + "bebop-fd-redirect", + "bebop-p2e", + "bebop-report", + "bebop-rtl-trace", + "bebop-terminal", + "bebop-trace-perfetto", + "bebop-uart", + "bebop-vcs", + "bebop-verilator", + "bebop-vm", + "camino", + "clap", + "duct", + "env_logger", + "libc", + "log", + "nix", + "serde", + "serde_json", + "snafu", + "toml", +] + +[[package]] +name = "bebop-bank-hash" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "snafu", +] + +[[package]] +name = "bebop-bemu" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", + "bebop-bemu-profile", + "bebop-dasm", + "bebop-dtb", + "bebop-elf", + "bebop-keyboard", + "bebop-memory", + "bebop-microphone", + "bebop-rtc", + "bebop-speaker", + "bebop-syscall", + "bebop-uart", + "bebop-vga", + "clap", + "linkme", + "prost", + "prost-build", + "prost-types", + "rustc_apfloat", + "snafu", + "toml", +] + +[[package]] +name = "bebop-bemu-profile" +version = "0.1.0" + +[[package]] +name = "bebop-dasm" +version = "0.1.0" + +[[package]] +name = "bebop-design-toy" +version = "0.1.0" +dependencies = [ + "bebop-bemu", +] + +[[package]] +name = "bebop-dtb" +version = "0.1.0" + +[[package]] +name = "bebop-elf" +version = "0.1.0" +dependencies = [ + "bebop-memory", +] + +[[package]] +name = "bebop-fd-redirect" +version = "0.1.0" +dependencies = [ + "libc", +] + +[[package]] +name = "bebop-keyboard" +version = "0.1.0" + +[[package]] +name = "bebop-memory" +version = "0.1.0" + +[[package]] +name = "bebop-microphone" +version = "0.1.0" + +[[package]] +name = "bebop-p2e" +version = "0.1.0" +dependencies = [ + "bebop-memory", + "bebop-rtl-trace", + "bebop-uart", + "duct", + "libc", + "log", + "serde", + "serde_json", +] + +[[package]] +name = "bebop-report" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "bebop-rtc" +version = "0.1.0" + +[[package]] +name = "bebop-rtl-trace" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", +] + +[[package]] +name = "bebop-sim-build" +version = "0.1.0" +dependencies = [ + "prost", + "prost-build", + "prost-types", +] + +[[package]] +name = "bebop-sim-toy-device" +version = "0.1.0" +dependencies = [ + "assert_cmd", + "bebop", + "bebop-bemu", + "bebop-design-toy", + "bebop-report", + "bebop-sim-build", + "bebop-verilator", + "chrono", + "clap", + "libtest-mimic", + "predicates", + "serde", + "toml", + "walkdir", +] + +[[package]] +name = "bebop-speaker" +version = "0.1.0" + +[[package]] +name = "bebop-syscall" +version = "0.1.0" +dependencies = [ + "bebop-memory", + "once_cell", +] + +[[package]] +name = "bebop-terminal" +version = "0.1.0" +dependencies = [ + "crossterm", + "ratatui", +] + +[[package]] +name = "bebop-trace-perfetto" +version = "0.1.0" +dependencies = [ + "clap", + "serde_json", +] + +[[package]] +name = "bebop-uart" +version = "0.1.0" + +[[package]] +name = "bebop-vcs" +version = "0.1.0" +dependencies = [ + "snafu", +] + +[[package]] +name = "bebop-verilator" +version = "0.1.0" +dependencies = [ + "bebop-dasm", + "bebop-fd-redirect", + "bebop-rtl-trace", + "bebop-uart", + "libc", + "snafu", +] + +[[package]] +name = "bebop-vga" +version = "0.1.0" + +[[package]] +name = "bebop-vm" +version = "0.1.0" +dependencies = [ + "clap", + "font8x8", + "sdl2", + "vt100", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "bstr" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" +dependencies = [ + "memchr", + "regex-automata", + "serde_core", +] + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "camino" +version = "1.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbad30e4b4c14a39e3cc8aed085a12a327257c316619c93581e017bc52be591" + +[[package]] +name = "cassowary" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" + +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + +[[package]] +name = "cc" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f74872d07caf508b30a21f6836e7d7016a2eaf7d9ff4f48deaa58cd8a0407630" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "compact_str" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fd622ebbb56a5b2ccb651b32b911cdeb2a9b4b11776b2473bf26a26a286244e" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "static_assertions", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crossterm" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" +dependencies = [ + "bitflags 2.13.2", + "crossterm_winapi", + "mio", + "parking_lot", + "rustix 0.38.44", + "signal-hook 0.3.18", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + +[[package]] +name = "darling" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 3.0.9", +] + +[[package]] +name = "darling_macro" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" +dependencies = [ + "darling_core", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "defmt" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a0795ac740cb54d7b3775c9c17345f6aaecf884c01733b39d6017738e0e3e1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "840bb056bd99c495c9188198a258ab787c79f306d864db3e6e8451442d6d8366" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "defmt-parser" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71958f69fb2de856c5dd51ac9377e986ca1aa5919f931bf5395a807277f889a0" +dependencies = [ + "thiserror", +] + +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + +[[package]] +name = "duct" +version = "0.13.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4ab5718d1224b63252cd0c6f74f6480f9ffeb117438a2e0f5cf6d9a4798929c" +dependencies = [ + "libc", + "once_cell", + "os_pipe", + "shared_child", +] + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "env_filter" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "escape8259" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f0d310082fd5de539d4f41601182241147498444a7695c520288a0cdbead6" + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "float-cmp" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09cf3155332e944990140d967ff5eceb70df778b34f77d8075db46e4704e6d8" +dependencies = [ + "num-traits", +] + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "font8x8" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875488b8711a968268c7cf5d139578713097ca4635a76044e8fe8eedf831d07e" + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "indoc" +version = "2.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a37b2691796cffeb8a8cd305ac66e65841559f147f4e63231d0eafa4db5384d1" +dependencies = [ + "rustversion", +] + +[[package]] +name = "instability" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3b5acc1e2fd9375041a388da33d1eb8aed5f7a8c0dd3543e3ea2805adfbe20" +dependencies = [ + "darling", + "indoc", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jiff" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b005715dcbeb0089a3c0dab99f2ff1cc3b2525323552703d648585d342a383" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", +] + +[[package]] +name = "jiff-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e52fe76043ccecc9005d2305ebaadf7d7fc0cc89ca6baa10a94d6bc68c7128c" +dependencies = [ + "defmt", + "log", +] + +[[package]] +name = "jiff-static" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cc9817253cf7c7ee4684451bd327e88d6f3658014e54a29198625590650695c" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "js-sys" +version = "0.3.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "libtest-mimic" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14e6ba06f0ade6e504aff834d7c34298e5155c6baca353cc6a4aaff2f9fd7f33" +dependencies = [ + "anstream", + "anstyle", + "clap", + "escape8259", +] + +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.5", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "os_pipe" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-atomic-util" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "091397be61a01d4be58e7841595bd4bfedb15f1cd54977d79b8271e94ed799a3" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "predicates" +version = "3.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ada8f2932f28a27ee7b70dd6c1c39ea0675c55a36879ab92f3a715eaa1e63cfe" +dependencies = [ + "anstyle", + "difflib", + "float-cmp", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cad38746f3166b4031b1a0d39ad9f954dd291e7854fcc0eed52ee41a0b50d144" + +[[package]] +name = "predicates-tree" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0de1b847b39c8131db0467e9df1ff60e6d0562ab8e9a16e568ad0fdb372e2f2" +dependencies = [ + "predicates-core", + "termtree", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck", + "itertools 0.14.0", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.119", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "ratatui" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabd94c2f37801c20583fc49dd5cd6b0ba68c716787c2dd6ed18571e1e63117b" +dependencies = [ + "bitflags 2.13.2", + "cassowary", + "compact_str", + "crossterm", + "indoc", + "instability", + "itertools 0.13.0", + "lru", + "paste", + "strum", + "unicode-segmentation", + "unicode-truncate", + "unicode-width 0.2.0", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.2", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustc_apfloat" +version = "0.2.3+llvm-462a31f5a5ab" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "486c2179b4796f65bfe2ee33679acf0927ac83ecf583ad6c91c3b4570911b9ad" +dependencies = [ + "bitflags 2.13.2", + "smallvec", +] + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys 0.12.1", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sdl2" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d42407afc6a8ab67e36f92e80b8ba34cbdc55aaeed05249efe9a2e8d0e9feef" +dependencies = [ + "bitflags 1.3.2", + "lazy_static", + "libc", + "sdl2-sys", +] + +[[package]] +name = "sdl2-sys" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ff61407fc75d4b0bbc93dc7e4d6c196439965fbef8e4a4f003a36095823eac0" +dependencies = [ + "cfg-if", + "libc", + "pkg-config", + "version-compare", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "shared_child" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "607549934f6cc26b89cfecfdc46fa90f1e5d1536a68349b0c3a4f9d1c0d37959" +dependencies = [ + "libc", + "sigchld", + "windows-sys 0.61.2", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "sigchld" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24f2b37f04360cd465089b87a9c3869c08220a2f3458463f0adf8badf5e77f2c" +dependencies = [ + "libc", + "os_pipe", + "signal-hook 0.4.5", +] + +[[package]] +name = "signal-hook" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b29ffa27d3d59a049bb328258de100f8012848d1b08d444e08bbc02ac468435e" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-mio" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b75a19a7a740b25bc7944bdee6172368f988763b744e3d4dfe753f6b4ece40cc" +dependencies = [ + "libc", + "mio", + "signal-hook 0.3.18", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "smallvec" +version = "1.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8" + +[[package]] +name = "snafu" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.119", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix 1.1.5", + "windows-sys 0.61.2", +] + +[[package]] +name = "termtree" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-truncate" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3644627a5af5fa321c95b9b235a72fd24cd29c648c2c379431e6628655627bf" +dependencies = [ + "itertools 0.13.0", + "unicode-segmentation", + "unicode-width 0.1.14", +] + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "version-compare" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "579a42fc0b8e0c63b76519a339be31bed574929511fa53c1a3acae26eb258f29" + +[[package]] +name = "vt100" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84cd863bf0db7e392ba3bd04994be3473491b31e66340672af5d11943c6274de" +dependencies = [ + "itoa", + "log", + "unicode-width 0.1.14", + "vte", +] + +[[package]] +name = "vte" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5022b5fbf9407086c180e9557be968742d839e68346af7792b8592489732197" +dependencies = [ + "arrayvec", + "utf8parse", + "vte_generate_state_changes", +] + +[[package]] +name = "vte_generate_state_changes" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e369bee1b05d510a7b4ed645f5faa90619e05437111783ea5848f28d97d3c2e" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/examples/designs/toy/sims/device/emu/Cargo.toml b/examples/designs/toy/sims/device/emu/Cargo.toml new file mode 100644 index 000000000..c544bc315 --- /dev/null +++ b/examples/designs/toy/sims/device/emu/Cargo.toml @@ -0,0 +1,87 @@ +[workspace] + +[package] +name = "bebop-sim-toy-device" +version = "0.1.0" +edition = "2021" +autobins = false +autotests = false + +# The model of this simulation entry; the runners below are built from it. +[lib] +name = "simulation" + +[[bin]] +name = "bebop" +path = "../../../../../../bebop/src/main.rs" + +[[bin]] +name = "bebop-bemu" +path = "../../../../../../bebop/src/bin/bebop-bemu.rs" +required-features = ["bemu"] + +[[bin]] +name = "bebop-verilator" +path = "../../../../../../bebop/src/bin/bebop-verilator.rs" +required-features = ["verilator"] + +[[bin]] +name = "bebop-p2e" +path = "../../../../../../bebop/src/bin/bebop-p2e.rs" +required-features = ["p2e"] + +[[test]] +name = "test_bemu" +path = "../../../../../../bebop/tests/test_bemu.rs" +harness = false +required-features = ["bemu"] + +[[test]] +name = "test_verilator" +path = "../../../../../../bebop/tests/test_verilator.rs" +harness = false +required-features = ["verilator"] + +[[test]] +name = "test_p2e" +path = "../../../../../../bebop/tests/test_p2e.rs" +harness = false +required-features = ["p2e"] + +[[test]] +name = "instance_isolation" +path = "tests/instance_isolation.rs" + +[[bin]] +name = "bebop-vcs" +path = "../../../../../../bebop/src/bin/bebop-vcs.rs" + +[[test]] +name = "test_vcs" +path = "../../../../../../bebop/tests/test_vcs.rs" +harness = false + +[features] +bemu = ["bebop/bemu"] +verilator = ["bebop/verilator", "dep:bebop-verilator"] +p2e = ["bebop/p2e"] + +[dependencies] +bebop = { path = "../../../../../../bebop" } +bebop-verilator = { path = "../../../../../../bebop/src/nodes/verilator", optional = true } +bebop-bemu = { path = "../../../../../../bebop/src/nodes/bemu" } +bebop-design-toy = { path = "../../../emu" } +clap = { version = "4", features = ["derive"] } + +[build-dependencies] +bebop-sim-build = { path = "../../../../../../bebop/src/nodes/lib/sim-build" } + +[dev-dependencies] +bebop-report = { path = "../../../../../../bebop/src/nodes/lib/report" } +libtest-mimic = "0.8" +assert_cmd = "2" +walkdir = "2" +predicates = "3" +serde = { version = "1", features = ["derive"] } +toml = "0.8" +chrono = { version = "0.4", default-features = false, features = ["clock"] } diff --git a/examples/designs/toy/sims/device/emu/build.rs b/examples/designs/toy/sims/device/emu/build.rs new file mode 100644 index 000000000..beb62e98c --- /dev/null +++ b/examples/designs/toy/sims/device/emu/build.rs @@ -0,0 +1,4 @@ +fn main() { + bebop_sim_build::balls(); + bebop_sim_build::runner_links(); +} diff --git a/examples/designs/toy/sims/device/emu/src/lib.rs b/examples/designs/toy/sims/device/emu/src/lib.rs new file mode 100644 index 000000000..3511b5518 --- /dev/null +++ b/examples/designs/toy/sims/device/emu/src/lib.rs @@ -0,0 +1,11 @@ +//! The toy device simulation. +//! Its Core models register in `bebop_bemu::CORES`; the Balls its design.pb +//! names are compiled here and register in `bebop_bemu::BALLS`. +extern crate bebop_design_toy; + +include!(concat!(env!("OUT_DIR"), "/balls.rs")); + +pub const SIMULATION: bebop_bemu::Simulation = bebop_bemu::Simulation { + design: include_bytes!(concat!(env!("OUT_DIR"), "/design.pb")), + instantiate: bebop_bemu::device::cbus::instantiate, +}; diff --git a/examples/designs/toy/sims/device/emu/tests/instance_isolation.rs b/examples/designs/toy/sims/device/emu/tests/instance_isolation.rs new file mode 100644 index 000000000..3991b60f6 --- /dev/null +++ b/examples/designs/toy/sims/device/emu/tests/instance_isolation.rs @@ -0,0 +1,57 @@ +use bebop_bemu::{device::cbus, sim::run::Args, DeviceModel}; +use clap::Parser; +use std::{path::{Path, PathBuf}, process::Command}; + +fn run(command: &mut Command) { + assert!(command.status().expect("execute verification tool").success()); +} + +#[test] +fn independent_devices_keep_gemmini_configuration_and_memory_private() { + let manifest = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let repo = manifest.ancestors().nth(6).expect("simulation entry location"); + let output = manifest.join("target/verification/instance-isolation"); + std::fs::create_dir_all(&output).unwrap(); + run(Command::new("python3").arg(repo.join("stack/scripts/design.py")) + .arg("--design-pb").arg(concat!(env!("OUT_DIR"), "/design.pb")) + .arg("--out").arg(&output)); + // The compile definitions of the toy target, as design.cmake lists them. + let cmake = std::fs::read_to_string(output.join("design.cmake")).unwrap(); + let defs: Vec = cmake.lines() + .find_map(|line| line.strip_prefix("set(DESIGN_TARGET_DEFS_toy ")) + .expect("toy target in design.cmake") + .trim_end_matches(')').split_whitespace() + .map(|def| format!("-D{}", def.trim_matches('"'))) + .collect(); + let bare = repo.join("stack/os/baremetal"); + bebop_bemu::configs::load(simulation::SIMULATION.design); + let mut cpus = Vec::new(); + for name in ["gemmini_loop_subrob", "gemmini_loop_mmio"] { + let elf = output.join(format!("{name}.elf")); + run(Command::new("riscv64-none-elf-gcc") + .args(["-nostdlib", "-nostartfiles", "-O2", "-march=rv64gc", "-mabi=lp64d", + "-mcmodel=medany", "-msmall-data-limit=0", "-fno-builtin", "-Wl,--no-relax"]) + .arg("-T").arg(bare.join("bbsim.ld")) + .args(&defs) + .arg("-I").arg(repo.join("bb-tests/workloads/lib")) + .arg("-I").arg(repo.join("examples/prefab/balls/gemmini/workloads")) + .arg(bare.join("crt0.S")) + .arg(repo.join(format!("examples/prefab/balls/gemmini/workloads/ctests/{name}.c"))) + .arg("-o").arg(&elf)); + let log = output.join(name); + let args = Args::try_parse_from([Path::new("test"), Path::new("--elf"), &elf, + Path::new("--log-dir"), &log]).unwrap(); + let device = DeviceModel::new(512 * 1024 * 1024, bebop_bemu::hart_capacity()); + let mut instances = cbus::instantiate(&device, &args).unwrap(); + assert_eq!(instances.len(), 1); + let mut cpu = instances.pop().unwrap(); + cpu.cpu_mut().load_elf(&elf).unwrap(); + cpu.cpu_mut().init_firmware(None, None).unwrap(); + cpus.push(cpu); + } + for _ in 0..2_000_000 { + if cpus.iter().all(|cpu| cpu.cpu().finished()) { break; } + for cpu in &mut cpus { if !cpu.cpu().finished() { cpu.step(1).unwrap(); } } + } + for cpu in cpus { assert_eq!(cpu.cpu().exit_code(), Some(0)); } +} diff --git a/examples/designs/toy/sims/device/regression/bemu/workloads-bare.toml b/examples/designs/toy/sims/device/regression/bemu/workloads-bare.toml new file mode 100644 index 000000000..345418dc0 --- /dev/null +++ b/examples/designs/toy/sims/device/regression/bemu/workloads-bare.toml @@ -0,0 +1,66 @@ +[workloads] +tests = [ + "toy-toy-ctest-redmule_f16_test-baremetal", + "toy-toy-ctest-redmule_f16_chain_test-baremetal", + "toy-toy-ctest-redmule_f16_edges_test-baremetal", + "toy-toy-ctest-redmule_f16_fused_test-baremetal", + "toy-toy-mlirtest-ball_redmule_f16_chain-baremetal", + "toy-toy-mlirtest-bank_redmule_f16_rows-baremetal", + + "toy-toy-ctest-mvin_mvout_group_test-baremetal", + "toy-toy-ctest-mset_transfer_test-baremetal", + "toy-toy-ctest-rvv_ball-baremetal", + "toy-toy-ctest-rvv_kernel-baremetal", + "toy-toy-ctest-mvin_2d_pixel_stride_test-baremetal", + "toy-toy-ctest-gemmini_accumulated_16x16_test-baremetal", + "toy-toy-mlirtest-ball_gemmini_accumulated_command_16x16-baremetal", + "toy-toy-ctest-mvin_2d_test-baremetal", + "toy-toy-ctest-mvin_mvout_acc_test-baremetal", + "toy-toy-ctest-mvin_mvout_test-baremetal", + "toy-toy-ctest-mvin_mvout_stride_test-baremetal", + "toy-toy-ctest-mvin_mvout_unaligned_test-baremetal", + "toy-toy-ctest-mvin_unaligned_test-baremetal", + "toy-toy-ctest-bdb_counter_start_stop_test-baremetal", + "toy-toy-ctest-bdb_counter_read_active_test-baremetal", + "toy-toy-ctest-bdb_counter_nested_pair_test-baremetal", + "toy-toy-ctest-bdb_counter_multi_id_test-baremetal", + "toy-toy-mlirtest-ball_bdb_counter_start_read_stop-baremetal", + "toy-toy-ctest-gemmini_os_cisc_basic_test-baremetal", + "toy-toy-ctest-gemmini_os_cisc_atranspose_test-baremetal", + "toy-toy-ctest-gemmini_os_cisc_btranspose_test-baremetal", + "toy-toy-ctest-gemmini_os_cisc_shift_test-baremetal", + "toy-toy-ctest-gemmini_os_risc_atranspose_test-baremetal", + "toy-toy-ctest-gemmini_os_risc_btranspose_test-baremetal", + "toy-toy-ctest-gemmini_os_risc_abtranspose_test-baremetal", + "toy-toy-ctest-gemmini_os_risc_shift_test-baremetal", + "toy-toy-ctest-gemmini_ws_cisc_conv_test-baremetal", + "toy-toy-ctest-gemmini_ws_risc_basic_test-baremetal", + "toy-toy-ctest-relu_i32_16x16_test-baremetal", + "toy-toy-mlirtest-ball_relu_i32_16row-baremetal", + "toy-toy-mlirtest-bank_relu_i32_mixed_sign-baremetal", + "toy-toy-mlirtest-bank_gemmini_compute_preloaded_bank_tiles-baremetal", + "toy-toy-mlirtest-bank_gemmini_compute_accumulated_bank_tiles-baremetal", + "toy-toy-mlirtest-ball_gemmini_preloaded_command_16x16-baremetal", + "toy-toy-ctest-tlb_bare_mode_test-baremetal", + "toy-toy-ctest-tlb_sv39_identity_test-baremetal", + "toy-toy-ctest-tlb_sfence_flush_test-baremetal", + "toy-toy-ctest-tlb_repeated_sfence_test-baremetal", + "toy-toy-ctest-tlb_sv39_nonidentity_test-baremetal", + "toy-toy-ctest-tlb_sv39_4k_alias_test-baremetal", + "toy-toy-ctest-tlb_sfence_remap_test-baremetal", + "toy-toy-mlirtest-linalg_conv2d_nhwc_hwcf-baremetal", + "toy-toy-mlirtest-linalg_conv2d_nhwc_fhwc-baremetal", + "toy-toy-mlirtest-linalg_conv2d_nchw_fchw-baremetal", + "toy-toy-mlirtest-linalg_conv2d_nhwc_hwcf_deep-baremetal", + "toy-toy-mlirtest-base_bank_loop_carried-baremetal", + "toy-toy-mlirtest-base_bank_transfer-baremetal", + + "toy-toy-ctest-vecmat16_atranspose_16x16_test-baremetal", + "toy-toy-ctest-vecmat16_accumulate_k4_test-baremetal", + "toy-toy-mlirtest-ball_matmul_at_i8_16x16-baremetal", + "toy-toy-mlirtest-bank_matmul_at_i8_bank_depth-baremetal", + "toy-toy-ctest-mxfp4_e8m0_two_block_test-baremetal", + "toy-toy-ctest-mxfp4_e8m0_scale_range_test-baremetal", + "toy-toy-mlirtest-ball_mxfp4_i8_2blocks-baremetal", + "toy-toy-mlirtest-bank_mxfp4_i8_bank_blocks-baremetal", +] diff --git a/examples/designs/toy/sims/device/regression/bemu/workloads-linux.toml b/examples/designs/toy/sims/device/regression/bemu/workloads-linux.toml new file mode 100644 index 000000000..41a9cd13b --- /dev/null +++ b/examples/designs/toy/sims/device/regression/bemu/workloads-linux.toml @@ -0,0 +1,4 @@ +[workloads] +tests = [ + "fw_payload-linux.elf", +] diff --git a/examples/designs/toy/sims/device/regression/p2e/workloads-bare.toml b/examples/designs/toy/sims/device/regression/p2e/workloads-bare.toml new file mode 100644 index 000000000..6265d1cda --- /dev/null +++ b/examples/designs/toy/sims/device/regression/p2e/workloads-bare.toml @@ -0,0 +1,21 @@ +[workloads] +tests = [ + "toy-toy-ctest-gemmini_accumulated_16x16_test-baremetal.hex", + "toy-toy-mlirtest-ball_gemmini_accumulated_command_16x16-baremetal.hex", + "toy-toy-ctest-mvin_mvout_test-baremetal.hex", + "toy-toy-ctest-mvin_mvout_stride_test-baremetal.hex", + "toy-toy-ctest-bdb_counter_start_stop_test-baremetal.hex", + "toy-toy-ctest-bdb_counter_read_active_test-baremetal.hex", + "toy-toy-ctest-bdb_counter_nested_pair_test-baremetal.hex", + "toy-toy-ctest-bdb_counter_multi_id_test-baremetal.hex", + "toy-toy-ctest-gemmini_os_cisc_basic_test-baremetal.hex", + "toy-toy-ctest-gemmini_os_cisc_atranspose_test-baremetal.hex", + "toy-toy-ctest-gemmini_os_cisc_btranspose_test-baremetal.hex", + "toy-toy-ctest-gemmini_os_cisc_shift_test-baremetal.hex", + "toy-toy-ctest-gemmini_os_risc_atranspose_test-baremetal.hex", + "toy-toy-ctest-gemmini_os_risc_btranspose_test-baremetal.hex", + "toy-toy-ctest-gemmini_os_risc_abtranspose_test-baremetal.hex", + "toy-toy-ctest-gemmini_os_risc_shift_test-baremetal.hex", + "toy-toy-ctest-gemmini_ws_cisc_conv_test-baremetal.hex", + "toy-toy-ctest-gemmini_ws_risc_basic_test-baremetal.hex", +] diff --git a/examples/designs/toy/sims/device/regression/p2e/workloads-linux.toml b/examples/designs/toy/sims/device/regression/p2e/workloads-linux.toml new file mode 100644 index 000000000..01747a69b --- /dev/null +++ b/examples/designs/toy/sims/device/regression/p2e/workloads-linux.toml @@ -0,0 +1,4 @@ +[workloads] +tests = [ + "fw_payload-linux.hex", +] diff --git a/arch/src/main/scala/framework/mem-core/coherence/src/csrc/README.md b/examples/designs/toy/sims/device/regression/soc/README.md similarity index 100% rename from arch/src/main/scala/framework/mem-core/coherence/src/csrc/README.md rename to examples/designs/toy/sims/device/regression/soc/README.md diff --git a/examples/designs/toy/sims/device/regression/verilator/workloads-bare.toml b/examples/designs/toy/sims/device/regression/verilator/workloads-bare.toml new file mode 100644 index 000000000..4f59b52c7 --- /dev/null +++ b/examples/designs/toy/sims/device/regression/verilator/workloads-bare.toml @@ -0,0 +1,46 @@ +[workloads] +tests = [ + "toy-toy-ctest-redmule_f16_test-baremetal", + "toy-toy-ctest-redmule_f16_chain_test-baremetal", + "toy-toy-ctest-redmule_f16_edges_test-baremetal", + "toy-toy-ctest-redmule_f16_fused_test-baremetal", + "toy-toy-mlirtest-ball_redmule_f16_chain-baremetal", + + "toy-toy-ctest-rvv_kernel-baremetal", + "toy-toy-ctest-mvin_2d_pixel_stride_test-baremetal", + "toy-toy-ctest-gemmini_accumulated_16x16_test-baremetal", + "toy-toy-mlirtest-ball_gemmini_accumulated_command_16x16-baremetal", + "toy-toy-ctest-mvin_2d_test-baremetal", + "toy-toy-ctest-mvin_mvout_acc_test-baremetal", + "toy-toy-ctest-mvin_mvout_test-baremetal", + "toy-toy-ctest-mvin_mvout_unaligned_test-baremetal", + "toy-toy-ctest-mvin_unaligned_test-baremetal", + "toy-toy-ctest-bdb_counter_start_stop_test-baremetal", + "toy-toy-ctest-bdb_counter_read_active_test-baremetal", + "toy-toy-ctest-bdb_counter_nested_pair_test-baremetal", + "toy-toy-ctest-bdb_counter_multi_id_test-baremetal", + "toy-toy-mlirtest-ball_bdb_counter_start_read_stop-baremetal", + "toy-toy-ctest-gemmini_os_cisc_basic_test-baremetal", + "toy-toy-ctest-gemmini_os_cisc_atranspose_test-baremetal", + "toy-toy-ctest-gemmini_os_cisc_btranspose_test-baremetal", + "toy-toy-ctest-gemmini_os_cisc_shift_test-baremetal", + "toy-toy-ctest-gemmini_ws_cisc_conv_test-baremetal", + "toy-toy-ctest-relu_i32_16x16_test-baremetal", + "toy-toy-mlirtest-ball_relu_i32_16row-baremetal", + "toy-toy-mlirtest-ball_gemmini_preloaded_command_16x16-baremetal", + "toy-toy-ctest-tlb_bare_mode_test-baremetal", + "toy-toy-ctest-tlb_sv39_identity_test-baremetal", + "toy-toy-ctest-tlb_sfence_flush_test-baremetal", + "toy-toy-ctest-tlb_repeated_sfence_test-baremetal", + "toy-toy-ctest-tlb_sv39_nonidentity_test-baremetal", + "toy-toy-ctest-tlb_sv39_4k_alias_test-baremetal", + "toy-toy-ctest-tlb_sfence_remap_test-baremetal", + "toy-toy-ctest-mvin_mvout_stride_test-baremetal", + + "toy-toy-ctest-vecmat16_atranspose_16x16_test-baremetal", + "toy-toy-ctest-vecmat16_accumulate_k4_test-baremetal", + "toy-toy-mlirtest-ball_matmul_at_i8_16x16-baremetal", + "toy-toy-ctest-mxfp4_e8m0_two_block_test-baremetal", + "toy-toy-ctest-mxfp4_e8m0_scale_range_test-baremetal", + "toy-toy-mlirtest-ball_mxfp4_i8_2blocks-baremetal", +] diff --git a/examples/designs/toy/sims/device/regression/verilator/workloads-linux.toml b/examples/designs/toy/sims/device/regression/verilator/workloads-linux.toml new file mode 100644 index 000000000..aa36e4333 --- /dev/null +++ b/examples/designs/toy/sims/device/regression/verilator/workloads-linux.toml @@ -0,0 +1,29 @@ +[workloads] +tests = [ + "toy-toy-ctest-gemmini_accumulated_16x16_test-linux", + "toy-toy-mlirtest-ball_gemmini_accumulated_command_16x16-linux", + "toy-toy-ctest-mvin_2d_test-linux", + "toy-toy-ctest-mvin_mvout_acc_test-linux", + "toy-toy-ctest-mvin_mvout_test-linux", + "toy-toy-ctest-mvin_mvout_stride_test-linux", + "toy-toy-ctest-mvin_mvout_unaligned_test-linux", + "toy-toy-ctest-mvin_unaligned_test-linux", + "toy-toy-ctest-bdb_counter_start_stop_test-linux", + "toy-toy-ctest-bdb_counter_read_active_test-linux", + "toy-toy-ctest-bdb_counter_nested_pair_test-linux", + "toy-toy-ctest-bdb_counter_multi_id_test-linux", + "toy-toy-mlirtest-ball_bdb_counter_start_read_stop-linux", + "toy-toy-ctest-gemmini_os_cisc_basic_test-linux", + "toy-toy-ctest-gemmini_os_cisc_atranspose_test-linux", + "toy-toy-ctest-gemmini_os_cisc_btranspose_test-linux", + "toy-toy-ctest-gemmini_os_cisc_shift_test-linux", + "toy-toy-ctest-gemmini_os_risc_atranspose_test-linux", + "toy-toy-ctest-gemmini_os_risc_btranspose_test-linux", + "toy-toy-ctest-gemmini_os_risc_abtranspose_test-linux", + "toy-toy-ctest-gemmini_os_risc_shift_test-linux", + "toy-toy-ctest-gemmini_ws_cisc_conv_test-linux", + "toy-toy-ctest-gemmini_ws_risc_basic_test-linux", + "toy-toy-ctest-relu_i32_16x16_test-linux", + "toy-toy-mlirtest-ball_relu_i32_16row-linux", + "toy-toy-mlirtest-ball_gemmini_preloaded_command_16x16-linux", +] diff --git a/examples/designs/toy/sims/device/regression/workloads-verilator.toml b/examples/designs/toy/sims/device/regression/workloads-verilator.toml new file mode 100644 index 000000000..c90097753 --- /dev/null +++ b/examples/designs/toy/sims/device/regression/workloads-verilator.toml @@ -0,0 +1,5 @@ +[workloads] +tests = [ + "toy-rocket-ctest-gemmini_loop_subrob-baremetal", + "toy-rocket-ctest-gemmini_loop_mmio-baremetal", +] diff --git a/examples/designs/toy/sims/device/stack/CMakeLists.txt b/examples/designs/toy/sims/device/stack/CMakeLists.txt new file mode 100644 index 000000000..b7c6f93b8 --- /dev/null +++ b/examples/designs/toy/sims/device/stack/CMakeLists.txt @@ -0,0 +1,37 @@ +# ===----------------------------------------------------------------------=== +# The software stack of the toy device simulation: every level the device +# needs, built bottom up. +# +# bbdev stack --build "--design toy --device " +# +# configures this with DESIGN_CMAKE, generated from design.pb. Each level is a +# CMake build of its own: cores and threads build buddy-opt plugins for the +# host, the device builds RISC-V libraries. +# core/toy libtoy-compiler.so, execute.o +# device runtime, runtime-baremetal, runtime-linux, engine; device.cmake +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(toy-device-stack NONE) +include(ExternalProject) +include("${DESIGN_CMAKE}") + +# Toy's CPU core with its Ball accelerator. +ExternalProject_Add(core-toy + SOURCE_DIR "${DESIGN_CORE_STACK_toy}" + BINARY_DIR "${DESIGN_CORE_BUILD_toy}" + CMAKE_GENERATOR Ninja + CMAKE_ARGS "-DDESIGN_CMAKE=${DESIGN_CMAKE}" -DCORE=toy + BUILD_ALWAYS ON + INSTALL_COMMAND "") + +# The device: the runtime that schedules jobs onto the cores. +ExternalProject_Add(device + SOURCE_DIR "${DESIGN_DEVICE_STACK}" + BINARY_DIR "${DESIGN_DEVICE_BUILD}" + CMAKE_GENERATOR Ninja + CMAKE_ARGS "-DDESIGN_CMAKE=${DESIGN_CMAKE}" + -DCMAKE_C_COMPILER=riscv64-unknown-linux-gnu-gcc + -DCMAKE_CXX_COMPILER=riscv64-unknown-linux-gnu-g++ + DEPENDS core-toy + BUILD_ALWAYS ON + INSTALL_COMMAND "") diff --git a/examples/designs/toy/sims/device/tapeout/area/constraints.sdc b/examples/designs/toy/sims/device/tapeout/area/constraints.sdc new file mode 100644 index 000000000..b34468521 --- /dev/null +++ b/examples/designs/toy/sims/device/tapeout/area/constraints.sdc @@ -0,0 +1,8 @@ +set clock_port [get_ports {clock clk_i}] +if {[sizeof_collection $clock_port] != 1} { error "synthesis top must have one clock or clk_i input" } +create_clock -name core -period 10.0 $clock_port +set_clock_uncertainty -setup 1.0 [get_clocks core] +set_clock_transition 0.5 [get_clocks core] +set data_inputs [remove_from_collection [all_inputs] $clock_port] +set_input_delay 2.0 -clock core $data_inputs +set_output_delay 2.0 -clock core [all_outputs] diff --git a/examples/designs/toy/sims/device/tapeout/area/dc.tcl b/examples/designs/toy/sims/device/tapeout/area/dc.tcl new file mode 100644 index 000000000..460208870 --- /dev/null +++ b/examples/designs/toy/sims/device/tapeout/area/dc.tcl @@ -0,0 +1,67 @@ +proc run_synthesis {} { + global RUN_CONFIG target_library synthetic_library link_library search_path +# Design-owned DC flow. Timing comes from constraints.sdc (RUN_SDC). +# bbdev injects RUN_* via -x {set RUN_CONFIG }. +if {![info exists RUN_CONFIG] || $RUN_CONFIG eq ""} { + error "bbdev must pass -x {set RUN_CONFIG }" +} +source [file normalize $RUN_CONFIG] + +proc bbdev_read_filelist {path} { + set fh [open $path r] + set files [list] + while {[gets $fh line] >= 0} { + set line [string trim $line] + if {$line eq "" || [string match "#*" $line]} { continue } + lappend files [file normalize $line] + } + close $fh + if {[llength $files] == 0} { error "empty DC source list: $path" } + return $files +} + +set top $RUN_TOP +set output_dir [file normalize $RUN_OUTPUT_DIR] +set report_dir [file normalize $RUN_REPORT_DIR] +file mkdir $output_dir $report_dir [file join $report_dir work] +set target_library [list $RUN_TARGET_LIBRARY] +set synthetic_library $RUN_SYNTHETIC_LIBRARY +set link_library [concat [list *] $target_library $synthetic_library $RUN_LINK_LIBRARY] +set_host_options -max_cores $RUN_MAX_THREADS +define_design_lib WORK -path [file join $report_dir work] +set search_path [list . [file dirname $RUN_SOURCE_LIST] [file dirname [file dirname $RUN_SOURCE_LIST]] [file dirname [file dirname [file dirname $RUN_SOURCE_LIST]]]] +set_app_var verilogout_no_tri true +set_app_var verilogout_equation false +set_app_var compile_seqmap_propagate_constants false +set_app_var hdlin_ff_always_sync_set_reset true +analyze -format sverilog -define {SYNTHESIS DC_SYN} [bbdev_read_filelist $RUN_SOURCE_LIST] +elaborate $top +current_design $top +link +set sram_cells [get_cells -hierarchical -quiet -filter {ref_name =~ "chipyard_sram_*"}] +if {[sizeof_collection $sram_cells] > 0} { set_dont_touch $sram_cells } +set sram_designs [get_designs -quiet chipyard_sram_*] +if {[sizeof_collection $sram_designs] > 0} { + set_dont_touch $sram_designs +} +source [file normalize $RUN_SDC] +if {[sizeof_collection [all_clocks]] == 0} { + error "SDC created no clocks" +} +set_load 2.0 [all_outputs] +set_app_var compile_seqmap_honor_sync_set_reset true +compile_ultra -area_high_effort_script -no_autoungroup -no_boundary_optimization +set_fix_multiple_port_nets -all -buffer_constants +change_names -hierarchy -rules verilog +write -format ddc -hierarchy -output [file join $output_dir ${top}.ddc] +write -format verilog -hierarchy -output [file join $output_dir ${top}.v] +write_sdc [file join $output_dir ${top}.sdc] +report_constraint -all_violators > [file join $report_dir constraint.rpt] +report_timing -delay max -max_paths 50 > [file join $report_dir timing_max.rpt] +report_timing -delay min -max_paths 50 > [file join $report_dir timing_min.rpt] +report_area -hierarchy > [file join $report_dir area.rpt] +report_reference > [file join $report_dir reference.rpt] + +} +if {[catch {run_synthesis} reason]} { puts stderr $reason; exit 1 } +exit diff --git a/examples/designs/toy/sims/device/tapeout/config.toml b/examples/designs/toy/sims/device/tapeout/config.toml new file mode 100644 index 000000000..1eef9db6b --- /dev/null +++ b/examples/designs/toy/sims/device/tapeout/config.toml @@ -0,0 +1,24 @@ +[tapeout] +top = "RedmuleBall" +sram_process = "smic180" +sram_mdf = "ip/smic180_sram_library.mdf.json" +constraints_sdc = "area/constraints.sdc" +target_library = "/home/bb-runner/Code/eda/D_libraries/SMIC180/SCC018UG_UHD_RVT_V0.4a/SCC018UG_UHD_RVT_V0p4a/liberty/1.8v/scc018ug_uhd_rvt_ss_v1p62_125c_ccs.db" +synthetic_library = [ + "/home/bb-runner/Code/eda/A_packages/tools/syn/R-2020.09-SP5/libraries/syn/dw_foundation.sldb", + "/home/bb-runner/Code/eda/A_packages/tools/syn/R-2020.09-SP5/libraries/syn/standard.sldb", +] +link_library = [] +max_threads = 8 +power_format = "vcd" +power_start_ns = "1000000" +power_end_ns = "5000000" +dc_script = "area/dc.tcl" +power_script = "../../../../../bbdev/api/steps/dc/scripts/ptpx.tcl" +power_strip_path = "GatePowerHarness.dut" + +sram_corner = "ss" +lc_shell = "/home/bb-runner/Code/eda/A_packages/tools/lc/R-2020.09-SP5/bin/lc_shell" + + +sram = [] diff --git a/examples/designs/toy/sims/device/tapeout/ip/smic180_sram_library.mdf.json b/examples/designs/toy/sims/device/tapeout/ip/smic180_sram_library.mdf.json new file mode 100644 index 000000000..fe51488c7 --- /dev/null +++ b/examples/designs/toy/sims/device/tapeout/ip/smic180_sram_library.mdf.json @@ -0,0 +1 @@ +[] diff --git a/examples/designs/toy/sims/device/workloads/CMakeLists.txt b/examples/designs/toy/sims/device/workloads/CMakeLists.txt new file mode 100644 index 000000000..6253eaee7 --- /dev/null +++ b/examples/designs/toy/sims/device/workloads/CMakeLists.txt @@ -0,0 +1,72 @@ +include("${DESIGN_DEVICE_BUILD}/device.cmake") + +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/base" + "${CMAKE_BINARY_DIR}/src/CTest/base" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/tlb" + "${CMAKE_BINARY_DIR}/src/CTest/tlb" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/tutorial" + "${CMAKE_BINARY_DIR}/src/CTest/tutorial" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/embench" + "${CMAKE_BINARY_DIR}/src/CTest/embench" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/coremark" + "${CMAKE_BINARY_DIR}/src/CTest/coremark" +) +foreach(BUCKYBALL_COMPONENT_WORKLOAD_TARGET IN LISTS DESIGN_CPU_TARGETS) + set(COMPONENT_WORKLOADS + "${DESIGN_TARGET_WORKLOADS_${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}}" + ) + if( + COMPONENT_WORKLOADS + STREQUAL + "${BUCKYBALL_REPO_ROOT}/examples/designs/${DESIGN_NAME}/workloads" + ) + continue() + endif() + set(BUCKYBALL_CTEST_TARGET "${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}") + set(BUCKYBALL_CTEST_GROUP_TARGET component-ctests-build) + if(IS_DIRECTORY "${COMPONENT_WORKLOADS}/ctests") + add_subdirectory( + "${COMPONENT_WORKLOADS}/ctests" + "${CMAKE_CURRENT_BINARY_DIR}/prefab/${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}/ctests" + ) + endif() +endforeach() +add_dependencies( + ctest-workloads-build + base-ctests-build + tlb-ctests-build + tutorial-ctests-build + component-ctests-build +) +add_dependencies(sync-native-bin embench-all coremark-all) + +add_subdirectory( + "${WORKLOAD_SRC_DIR}/MLIRTest/base" + "${CMAKE_BINARY_DIR}/src/MLIRTest/mlir_tests/base" +) +foreach(BUCKYBALL_TARGET IN LISTS DESIGN_CPU_TARGETS) + set(BUCKYBALL_MLIR_ACTIVE_TARGET "${BUCKYBALL_TARGET}") + foreach(BUCKYBALL_BALL_DIR IN LISTS DESIGN_PROFILES_${BUCKYBALL_TARGET}_THREAD_BALLDOMAIN_BALLS) + set(BALL_MLIR_TESTS + "${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/${BUCKYBALL_BALL_DIR}/workloads/mlir_tests" + ) + if(IS_DIRECTORY "${BALL_MLIR_TESTS}") + add_subdirectory( + "${BALL_MLIR_TESTS}" + "${CMAKE_CURRENT_BINARY_DIR}/balls/${BUCKYBALL_TARGET}/${BUCKYBALL_BALL_DIR}" + ) + endif() + endforeach() +endforeach() +add_dependencies(MLIRTest-all base-mlirtest-build balls-mlir-tests-build) + +add_subdirectory("${BUCKYBALL_REPO_ROOT}/examples/designs/toy/workloads" design) diff --git a/examples/designs/toy/sims/device/workloads/kernel/CMakeLists.txt b/examples/designs/toy/sims/device/workloads/kernel/CMakeLists.txt new file mode 100644 index 000000000..a827f3a1f --- /dev/null +++ b/examples/designs/toy/sims/device/workloads/kernel/CMakeLists.txt @@ -0,0 +1,41 @@ +# Linux programs installed into the toy rootfs, each with the hart it runs on. +set(BUCKYBALL_KERNEL_PROGRAMS + 0 toy-toy-ctest-redmule_f16_test-linux + 0 toy-toy-ctest-redmule_f16_chain_test-linux + 0 toy-toy-ctest-redmule_f16_edges_test-linux + 0 toy-toy-ctest-redmule_f16_fused_test-linux + 0 toy-toy-mlirtest-ball_redmule_f16_chain-linux + 0 toy-toy-mlirtest-bank_redmule_f16_rows-linux + 0 toy-toy-ctest-mvin_mvout_group_test-linux + 0 toy-toy-ctest-mset_transfer_test-linux + 0 toy-toy-ctest-rvv_ball-linux + 0 toy-toy-ctest-rvv_kernel-linux + 0 toy-toy-ctest-gemmini_accumulated_16x16_test-linux + 0 toy-toy-mlirtest-ball_gemmini_accumulated_command_16x16-linux + 0 toy-toy-ctest-mvin_2d_test-linux + 0 toy-toy-ctest-mvin_mvout_acc_test-linux + 0 toy-toy-ctest-mvin_mvout_test-linux + 0 toy-toy-ctest-mvin_mvout_stride_test-linux + 0 toy-toy-ctest-mvin_mvout_unaligned_test-linux + 0 toy-toy-ctest-mvin_unaligned_test-linux + 0 toy-toy-ctest-bdb_counter_start_stop_test-linux + 0 toy-toy-ctest-bdb_counter_read_active_test-linux + 0 toy-toy-ctest-bdb_counter_nested_pair_test-linux + 0 toy-toy-ctest-bdb_counter_multi_id_test-linux + 0 toy-toy-mlirtest-ball_bdb_counter_start_read_stop-linux + 0 toy-toy-ctest-gemmini_os_cisc_basic_test-linux + 0 toy-toy-ctest-gemmini_os_cisc_atranspose_test-linux + 0 toy-toy-ctest-gemmini_os_cisc_btranspose_test-linux + 0 toy-toy-ctest-gemmini_os_cisc_shift_test-linux + 0 toy-toy-ctest-gemmini_os_risc_atranspose_test-linux + 0 toy-toy-ctest-gemmini_os_risc_btranspose_test-linux + 0 toy-toy-ctest-gemmini_os_risc_abtranspose_test-linux + 0 toy-toy-ctest-gemmini_os_risc_shift_test-linux + 0 toy-toy-ctest-gemmini_ws_cisc_conv_test-linux + 0 toy-toy-ctest-gemmini_ws_risc_basic_test-linux + 0 toy-toy-ctest-relu_i32_16x16_test-linux + 0 toy-toy-mlirtest-ball_relu_i32_16row-linux + 0 toy-toy-mlirtest-bank_relu_i32_mixed_sign-linux + 0 toy-toy-mlirtest-bank_gemmini_compute_preloaded_bank_tiles-linux + 0 toy-toy-mlirtest-bank_gemmini_compute_accumulated_bank_tiles-linux + PARENT_SCOPE) diff --git a/examples/designs/toy/sims/ip/uvm/configs/axis16.toml b/examples/designs/toy/sims/ip/uvm/configs/axis16.toml new file mode 100644 index 000000000..a5e2badc6 --- /dev/null +++ b/examples/designs/toy/sims/ip/uvm/configs/axis16.toml @@ -0,0 +1,6 @@ +[parameters] +dataBits = 16 +inputs = 2 + +[defines] +AXIS_DATA_BITS = "dataBits" diff --git a/examples/designs/toy/sims/ip/uvm/registry.toml b/examples/designs/toy/sims/ip/uvm/registry.toml new file mode 100644 index 000000000..6992c6ebe --- /dev/null +++ b/examples/designs/toy/sims/ip/uvm/registry.toml @@ -0,0 +1,7 @@ +[[ips]] +ip = "axis" +config = "default" + +[[ips]] +ip = "axis" +config = "configs/axis16.toml" diff --git a/examples/designs/toy/stack/device/CMakeLists.txt b/examples/designs/toy/stack/device/CMakeLists.txt new file mode 100644 index 000000000..fa1e71aef --- /dev/null +++ b/examples/designs/toy/stack/device/CMakeLists.txt @@ -0,0 +1,37 @@ +# ===----------------------------------------------------------------------=== +# The toy device: its cores, their hardware threads and the runtime that +# schedules jobs onto the cores. +# +# bbdev stack --build "--design toy --device " +# +# builds every thread and core stack of the device first, then configures this +# with DESIGN_CMAKE, generated from design.pb, and the RISC-V Linux toolchain. +# Products, exported in device.cmake for the programs that run on the device: +# runtime, runtime-baremetal, runtime-linux job scheduling over the cores +# engine hardware threads and instruction streams +# bindings how each core runs jobs and each +# hardware thread is loaded +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(toy-device LANGUAGES C CXX) +include("${DESIGN_CMAKE}") +set(CMAKE_POSITION_INDEPENDENT_CODE ON) + +# How each core runs jobs, and how each hardware thread is loaded. Nothing +# references these bindings, so they are linked as objects rather than taken +# from an archive; as link options they come before the libraries they call. +add_library(bindings INTERFACE) +foreach(CORE IN LISTS DESIGN_CORES) + target_link_options(bindings INTERFACE "${DESIGN_CORE_BUILD_${CORE}}/execute.o") +endforeach() +foreach(THREAD IN LISTS DESIGN_THREAD_STACKS) + target_link_options(bindings INTERFACE "${DESIGN_THREAD_BUILD_${THREAD}}/loader.o") +endforeach() +# DMA helpers are sized by the target with the fewest bank lines. +list(GET DESIGN_RUNTIME_TARGETS 0 PARAMS_TARGET) +set(RUNTIME_DEFS ${DESIGN_TARGET_DEFS_${PARAMS_TARGET}}) + +add_subdirectory("${DESIGN_REPO}/stack/engine" engine) +add_subdirectory("${DESIGN_REPO}/stack/runtime" runtime) +export(TARGETS bindings runtime runtime-baremetal runtime-linux engine + runtime-shared runtime-linux-shared engine-shared FILE device.cmake) diff --git a/examples/designs/toy/stack/device/core/CMakeLists.txt b/examples/designs/toy/stack/device/core/CMakeLists.txt new file mode 100644 index 000000000..04e089c64 --- /dev/null +++ b/examples/designs/toy/stack/device/core/CMakeLists.txt @@ -0,0 +1,36 @@ +# ===----------------------------------------------------------------------=== +# Toy's own core: a CPU with its Ball accelerator. +# +# Built by the stack top of each simulation entry, configured with DESIGN_CMAKE +# and CORE. Products: +# execute.o how the core runs jobs (direct execution) +# libtoy-compiler.so buddy-opt plugin: the core's fusion (form-custom-ops) +# and lower-buckyball around the framework bank layer +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(toy-core-stack LANGUAGES C CXX) +include("${DESIGN_CMAKE}") + +set(RUNTIME "${DESIGN_REPO}/stack/runtime") +set(TARGET ${DESIGN_DEVICE_CORES_${DESIGN_CORE_HART_${CORE}}_PROFILE}) +find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) + +# How the core runs jobs. +add_custom_command( + OUTPUT "${CMAKE_BINARY_DIR}/execute.o" + COMMAND "${LINUX_CC}" -O2 -fPIC -ffreestanding -fno-builtin -msmall-data-limit=0 + "-DCORE_SIGNATURE=${DESIGN_DEVICE_CORES_${DESIGN_CORE_HART_${CORE}}_SIGNATURE}ULL" "-I${RUNTIME}/scheduler" + -c "${RUNTIME}/execute/direct.c" -o "${CMAKE_BINARY_DIR}/execute.o" + DEPENDS "${RUNTIME}/execute/direct.c" "${RUNTIME}/scheduler/runtime.h" + VERBATIM) +add_custom_target(execute ALL DEPENDS "${CMAKE_BINARY_DIR}/execute.o") + +# The core's compiler plugin. The core owns computation recognition and its +# conversion to bank dataflow. +set(BUCKYBALL_BALLS ${DESIGN_TARGET_BALL_STACKS_${TARGET}}) +file(GLOB_RECURSE BUCKYBALL_SOURCES CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp") +add_subdirectory("${DESIGN_REPO}/stack/compiler/midend" compiler) +add_library(${TARGET}-compiler MODULE) +target_link_libraries(${TARGET}-compiler PRIVATE buckyball-compiler) +set_target_properties(${TARGET}-compiler PROPERTIES + LINKER_LANGUAGE CXX LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") diff --git a/examples/designs/toy/stack/device/core/src/ConvertLinalgToBuckyball.cpp b/examples/designs/toy/stack/device/core/src/ConvertLinalgToBuckyball.cpp new file mode 100644 index 000000000..8da3baddd --- /dev/null +++ b/examples/designs/toy/stack/device/core/src/ConvertLinalgToBuckyball.cpp @@ -0,0 +1,239 @@ +//===- ConvertLinalgToBuckyball.cpp - Toy core lowering -------------------===// +// +// conv2d_i8 becomes a matrix product C[M x N] = A[M x K] @ B[K x N] over the +// dimension groups form-custom-ops recorded. The CPU packs A (im2col) and B +// into zero-padded 16x16 INT8 tiles. For each 16x16 C tile, the A bank takes +// the K blocks of the tile's rows and the B bank the K rows of its columns, +// one bank-deep group at a time gathered with bank_transfer, and Gemmini's +// matmul_i8_tile computes C. The CPU adds C * input_step * weight_step to the +// initial FP32 output. +// +//===----------------------------------------------------------------------===// + +#include "Buckyball/BuckyballDialect.h" +#include "Buckyball/BuckyballOps.h" +#include "Custom/CustomOps.h" +#include "Target/BuckyballTargetRegistry.h" +#include "gemminiOps.h" + +#include "mlir/Dialect/Affine/IR/AffineOps.h" +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Bufferization/IR/Bufferization.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/Dialect/SCF/IR/SCF.h" +#include "mlir/Dialect/Tensor/IR/Tensor.h" +#include "mlir/Dialect/Utils/IndexingUtils.h" +#include "mlir/Pass/Pass.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +namespace { +// Runs body(i, j) for i in [0, rows) and j in [0, cols). +void loops(OpBuilder &b, Location loc, int64_t rows, int64_t cols, + function_ref body) { + OpBuilder::InsertionGuard guard(b); + Value zero = arith::ConstantIndexOp::create(b, loc, 0); + Value one = arith::ConstantIndexOp::create(b, loc, 1); + auto outer = scf::ForOp::create( + b, loc, zero, arith::ConstantIndexOp::create(b, loc, rows), one); + b.setInsertionPointToStart(outer.getBody()); + auto inner = scf::ForOp::create( + b, loc, zero, arith::ConstantIndexOp::create(b, loc, cols), one); + b.setInsertionPointToStart(inner.getBody()); + body(outer.getInductionVar(), inner.getInductionVar()); +} + +// The buffer of a contiguous tensor. +Value buffer(OpBuilder &b, Location loc, Value tensor) { + auto type = cast(tensor.getType()); + auto memory = MemRefType::get( + type.getShape(), type.getElementType(), + StridedLayoutAttr::get(b.getContext(), ShapedType::kDynamic, + computeSuffixProduct(type.getShape()))); + return bufferization::ToBufferOp::create(b, loc, memory, tensor, true); +} + +void lowerConvolution(::buddy::custom::CustomOp op) { + SmallVector maps = llvm::map_to_vector( + cast(op->getAttr("indexing_maps")), + [](Attribute map) { return cast(map).getValue(); }); + auto array = [&](StringRef name) { + return cast(op->getAttr(name)).asArrayRef(); + }; + ArrayRef ranges = array("loop_ranges"); + ArrayRef mDims = array("m_dims"), nDims = array("n_dims"), + kDims = array("k_dims"); + auto extent = [&](ArrayRef dims) { + int64_t size = 1; + for (int64_t dim : dims) + size *= ranges[dim]; + return size; + }; + int64_t m = extent(mDims), n = extent(nDims), k = extent(kDims); + int64_t mt = (m + 15) / 16, nt = (n + 15) / 16, kt = (k + 15) / 16; + // The K blocks a bank group holds. + int64_t groupBlocks = buckyball_target::getBuckyballTarget().bankDepth / 16; + + OpBuilder b(op); + Location loc = op.getLoc(); + Type i8 = b.getI8Type(), i32 = b.getI32Type(), i64 = b.getI64Type(); + auto idx = [&](int64_t v) -> Value { + return arith::ConstantIndexOp::create(b, loc, v); + }; + // The loop dimension values of flat indices along each dimension group. + auto iterators = [&](ArrayRef>> flats) { + SmallVector iv(ranges.size(), idx(0)); + for (auto [flat, dims] : flats) + for (int64_t dim : llvm::reverse(dims)) { + iv[dim] = arith::RemUIOp::create(b, loc, flat, idx(ranges[dim])); + flat = arith::DivUIOp::create(b, loc, flat, idx(ranges[dim])); + } + return iv; + }; + auto element = [&](AffineMap map, ArrayRef iv) { + SmallVector indices; + for (AffineExpr result : map.getResults()) + indices.push_back(affine::AffineApplyOp::create( + b, loc, AffineMap::get(map.getNumDims(), 0, result), iv)); + return indices; + }; + // Tile (outer / 16) * stride + inner / 16, element (row % 16, col % 16). + auto tileIndex = [&](Value outer, Value inner, int64_t stride, Value row, + Value col) { + Value sixteen = idx(16); + Value tile = arith::AddIOp::create( + b, loc, + arith::MulIOp::create(b, loc, + arith::DivUIOp::create(b, loc, outer, sixteen), + idx(stride)), + arith::DivUIOp::create(b, loc, inner, sixteen)); + return SmallVector{tile, + arith::RemUIOp::create(b, loc, row, sixteen), + arith::RemUIOp::create(b, loc, col, sixteen)}; + }; + auto scalar = [&](Value scale) -> Value { + SmallVector zeros(cast(scale.getType()).getRank(), + idx(0)); + return tensor::ExtractOp::create(b, loc, scale, zeros); + }; + + ValueRange inputs = op.getInputs(); + Value input = buffer(b, loc, inputs[0]), filter = buffer(b, loc, inputs[2]); + Value step = + arith::MulFOp::create(b, loc, scalar(inputs[1]), scalar(inputs[3])); + auto resultType = cast(op.getResult(0).getType()); + Value output = memref::AllocOp::create( + b, loc, MemRefType::get(resultType.getShape(), b.getF32Type())); + memref::CopyOp::create(b, loc, buffer(b, loc, inputs[4]), output); + + Value aTiles = memref::AllocOp::create( + b, loc, MemRefType::get({mt * kt, 16, 16}, i8), b.getI64IntegerAttr(64)); + // B tiles by column: tile j * kt + r holds K rows 16r .. 16r + 15. + Value bTiles = memref::AllocOp::create( + b, loc, MemRefType::get({nt * kt, 16, 16}, i8), b.getI64IntegerAttr(64)); + Value cTiles = memref::AllocOp::create( + b, loc, MemRefType::get({mt * nt, 16, 16}, i32), b.getI64IntegerAttr(64)); + Value zero8 = arith::ConstantIntOp::create(b, loc, 0, 8); + linalg::FillOp::create(b, loc, zero8, aTiles); + linalg::FillOp::create(b, loc, zero8, bTiles); + loops(b, loc, m, k, [&](Value row, Value col) { + SmallVector iv = iterators({{row, mDims}, {col, kDims}}); + Value v = memref::LoadOp::create(b, loc, input, element(maps[0], iv)); + memref::StoreOp::create(b, loc, v, aTiles, + tileIndex(row, col, kt, row, col)); + }); + loops(b, loc, k, n, [&](Value row, Value col) { + SmallVector iv = iterators({{row, kDims}, {col, nDims}}); + Value v = memref::LoadOp::create(b, loc, filter, element(maps[1], iv)); + memref::StoreOp::create(b, loc, v, bTiles, + tileIndex(col, row, kt, row, col)); + }); + + // `count` consecutive tiles from `index`. + auto tile = [&](Value tiles, int64_t index, int64_t count) -> Value { + return memref::SubViewOp::create( + b, loc, tiles, ArrayRef{index, 0, 0}, + ArrayRef{count, 16, 16}, ArrayRef{1, 1, 1}); + }; + // The kt tiles from `first` in one bank, a group of groupBlocks tiles at a + // time. + auto gather = [&](Value tiles, int64_t first) { + Value bank; + for (int64_t block = 0; block < kt; block += groupBlocks) { + int64_t count = std::min(groupBlocks, kt - block); + Value group = BankMvinOp::create( + b, loc, i64, tile(tiles, first + block, count), + BankAllocOp::create(b, loc, i64, 1, 1), count * 16, 1); + bank = block + ? BankTransferOp::create(b, loc, i64, group, bank).getResult() + : group; + } + return bank; + }; + for (int64_t i = 0; i < mt; ++i) + for (int64_t j = 0; j < nt; ++j) { + Value aBank = gather(aTiles, i * kt); + Value bBank = gather(bTiles, j * kt); + Value cBank = BankAllocOp::create(b, loc, i64, 1, 4); + cBank = ::buddy::gemmini_ball::BankMatmulI8TileOp::create( + b, loc, i64, aBank, bBank, cBank, b.getI64IntegerAttr(kt * 16), + UnitAttr(), IntegerAttr()); + cBank = BankMvoutOp::create(b, loc, i64, tile(cTiles, i * nt + j, 1), + cBank, 16, 1); + BankReleaseOp::create(b, loc, aBank); + BankReleaseOp::create(b, loc, bBank); + BankReleaseOp::create(b, loc, cBank); + } + + loops(b, loc, m, n, [&](Value row, Value col) { + SmallVector iv = iterators({{row, mDims}, {col, nDims}}); + SmallVector at = element(maps[2], iv); + Value sum = memref::LoadOp::create(b, loc, cTiles, + tileIndex(row, col, nt, row, col)); + Value scaled = arith::MulFOp::create( + b, loc, arith::SIToFPOp::create(b, loc, b.getF32Type(), sum), step); + Value result = arith::AddFOp::create( + b, loc, memref::LoadOp::create(b, loc, output, at), scaled); + memref::StoreOp::create(b, loc, result, output, at); + }); + op.getResult(0).replaceAllUsesWith(bufferization::ToTensorOp::create( + b, loc, resultType, output, true, true)); + op.erase(); +} + +class ConvertLinalgToBuckyballPass + : public PassWrapper> { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(ConvertLinalgToBuckyballPass) + StringRef getArgument() const final { return "convert-linalg-to-buckyball"; } + StringRef getDescription() const final { + return "Lower the toy custom computations to the core bank dataflow"; + } + void getDependentDialects(DialectRegistry ®istry) const override { + registry.insert(); + } + void runOnOperation() override { + SmallVector<::buddy::custom::CustomOp> computations; + getOperation().walk( + [&](::buddy::custom::CustomOp op) { computations.push_back(op); }); + for (auto op : computations) { + if (op.getKind() != "conv2d_i8") { + op.emitOpError("has no bank lowering on this core for ") + << op.getKind(); + return signalPassFailure(); + } + lowerConvolution(op); + } + } +}; + +PassRegistration registration; +} // namespace diff --git a/examples/designs/toy/stack/device/core/src/FormCustomOps.cpp b/examples/designs/toy/stack/device/core/src/FormCustomOps.cpp new file mode 100644 index 000000000..9751b391f --- /dev/null +++ b/examples/designs/toy/stack/device/core/src/FormCustomOps.cpp @@ -0,0 +1,108 @@ +//===- FormCustomOps.cpp - Recognize the toy core computations ------------===// +// +// conv2d_i8(x, input_step, w, weight_step, init) = init + conv(x, w) * +// input_step * weight_step, from a 2-D convolution of +// int8_to_fp32(fp32_to_int8(x', multiplier), input_step) and +// int8_to_fp32(w, weight_step), both steps per tensor. The convolution keeps +// its iteration space: `indexing_maps` (input, filter, output), `loop_ranges` +// and the loop dimensions of the matrix product, `m_dims` (batch and output +// pixels), `n_dims` (output channels) and `k_dims` (filter window and input +// channels). +// +// The core forms it only when its banks fit: the C tile takes four groups, +// bank 0 is the zero bank, and A and B each take a group per bank depth of K. +// +//===----------------------------------------------------------------------===// + +#include "Custom/CustomOps.h" +#include "Target/BuckyballTargetRegistry.h" + +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Pass/Pass.h" + +using namespace mlir; +using ::buddy::custom::CustomOp; + +namespace { +Operation *frontend(Value value, StringRef name) { + Operation *op = value.getDefiningOp(); + return op && op->getName().getStringRef() == name ? op : nullptr; +} + +bool perTensor(Value scale) { + return llvm::all_of(cast(scale.getType()).getShape(), + [](int64_t size) { return size == 1; }); +} + +class FormCustomOpsPass + : public PassWrapper> { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(FormCustomOpsPass) + StringRef getArgument() const final { return "form-custom-ops"; } + StringRef getDescription() const final { + return "Recognize the quantized convolutions of the toy core"; + } + void getDependentDialects(DialectRegistry ®istry) const override { + registry.insert<::buddy::custom::CustomDialect>(); + } + void runOnOperation() override { + const auto &target = buckyball_target::getBuckyballTarget(); + int64_t maxK = (target.bankNum - 5) / 2 * target.bankDepth; + SmallVector convolutions; + getOperation().walk([&](linalg::LinalgOp op) { + if (isa(op) && + op.hasPureTensorSemantics()) + convolutions.push_back(op); + }); + for (linalg::LinalgOp op : convolutions) { + Operation *input = + frontend(op.getDpsInputs()[0], "frontend.int8_to_fp32"); + Operation *weight = + frontend(op.getDpsInputs()[1], "frontend.int8_to_fp32"); + if (!input || !weight) + continue; + Operation *quantize = + frontend(input->getOperand(0), "frontend.fp32_to_int8"); + FailureOr conv = + linalg::inferConvolutionDims(op); + if (!quantize || !perTensor(input->getOperand(1)) || + !perTensor(weight->getOperand(1)) || failed(conv) || + !conv->depth.empty()) + continue; + SmallVector ranges = op.getStaticLoopRanges(); + int64_t k = 1; + for (unsigned dim : conv->filterLoop) + k *= ranges[dim]; + for (unsigned dim : conv->inputChannel) + k *= ranges[dim]; + if (k > maxK) + continue; + OpBuilder builder(op); + auto dims = [&](ArrayRef first, ArrayRef second) { + SmallVector all(first.begin(), first.end()); + all.append(second.begin(), second.end()); + return builder.getDenseI64ArrayAttr(all); + }; + auto custom = CustomOp::create( + builder, op.getLoc(), op->getResultTypes(), "conv2d_i8", + ValueRange{quantize->getResult(0), input->getOperand(1), + weight->getOperand(0), weight->getOperand(1), + op.getDpsInits()[0]}); + custom->setAttr("indexing_maps", op.getIndexingMaps()); + custom->setAttr("loop_ranges", builder.getDenseI64ArrayAttr(ranges)); + custom->setAttr("m_dims", dims(conv->batch, conv->outputImage)); + custom->setAttr("n_dims", dims(conv->outputChannel, {})); + custom->setAttr("k_dims", dims(conv->filterLoop, conv->inputChannel)); + op->replaceAllUsesWith(custom.getOutputs()); + op->erase(); + for (Operation *dequantize : {input, weight}) + if (dequantize->use_empty()) + dequantize->erase(); + } + } +}; + +PassRegistration registration; +} // namespace diff --git a/examples/designs/toy/workloads/CMakeLists.txt b/examples/designs/toy/workloads/CMakeLists.txt new file mode 100644 index 000000000..a545bb368 --- /dev/null +++ b/examples/designs/toy/workloads/CMakeLists.txt @@ -0,0 +1,17 @@ +get_filename_component(BUCKYBALL_REPO_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../../.. ABSOLUTE) + +add_custom_target(toy-ctests-build) +set(BUCKYBALL_CTEST_GROUP_TARGET toy-ctests-build) +set(BUCKYBALL_CTEST_TARGET toy) + +add_subdirectory(device) + +add_custom_target(design-ctests-build DEPENDS toy-ctests-build) + +add_custom_target(design-workloads-build ALL + DEPENDS + design-ctests-build + mlirtest-workloads-build + soctests-build + COMMENT "Building toy workloads" + VERBATIM) diff --git a/examples/designs/toy/workloads/configs/tests.toml b/examples/designs/toy/workloads/configs/tests.toml new file mode 100644 index 000000000..c1c7ceb33 --- /dev/null +++ b/examples/designs/toy/workloads/configs/tests.toml @@ -0,0 +1,2 @@ +controlHart = 0 +reportHart = 0 diff --git a/examples/designs/toy/workloads/device/CMakeLists.txt b/examples/designs/toy/workloads/device/CMakeLists.txt new file mode 100644 index 000000000..679d879e4 --- /dev/null +++ b/examples/designs/toy/workloads/device/CMakeLists.txt @@ -0,0 +1,2 @@ +add_subdirectory(soc) +add_subdirectory(core) diff --git a/examples/designs/toy/workloads/device/core/CMakeLists.txt b/examples/designs/toy/workloads/device/core/CMakeLists.txt new file mode 100644 index 000000000..2a6ad3b6a --- /dev/null +++ b/examples/designs/toy/workloads/device/core/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(acc) diff --git a/examples/designs/toy/workloads/device/core/acc/CMakeLists.txt b/examples/designs/toy/workloads/device/core/acc/CMakeLists.txt new file mode 100644 index 000000000..d17debe0e --- /dev/null +++ b/examples/designs/toy/workloads/device/core/acc/CMakeLists.txt @@ -0,0 +1,10 @@ +add_subdirectory(ctests) +foreach(BUCKYBALL_TARGET IN LISTS DESIGN_CPU_TARGETS) + set(BUCKYBALL_CTEST_TARGET "${BUCKYBALL_TARGET}") + foreach(BUCKYBALL_BALL_DIR IN LISTS DESIGN_PROFILES_${BUCKYBALL_TARGET}_THREAD_BALLDOMAIN_BALLS) + add_subdirectory( + "${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/${BUCKYBALL_BALL_DIR}/workloads/ctests" + "${CMAKE_CURRENT_BINARY_DIR}/balls/${BUCKYBALL_TARGET}/${BUCKYBALL_BALL_DIR}/ctests") + endforeach() +endforeach() +add_subdirectory(mlir_tests) diff --git a/examples/designs/toy/workloads/device/core/acc/ctests/CMakeLists.txt b/examples/designs/toy/workloads/device/core/acc/ctests/CMakeLists.txt new file mode 100644 index 000000000..cde0eb72a --- /dev/null +++ b/examples/designs/toy/workloads/device/core/acc/ctests/CMakeLists.txt @@ -0,0 +1,22 @@ +add_subdirectory(gemmini_relu_trace_pipeline) +set(BUCKYBALL_CTEST_TARGET toy) +set(BUCKYBALL_CTEST_GROUP_TARGET toy-ctests-build) +file( + RELATIVE_PATH + RVV_KERNEL_SOURCE + ${CMAKE_CURRENT_SOURCE_DIR} + ${WORKLOAD_SRC_DIR}/CTest/rvv/rvv_kernel.c +) +add_buckyball_ctest(${RVV_KERNEL_SOURCE}) +set(RVV_KERNEL_TEST + "${DESIGN_NAME}-${BUCKYBALL_CTEST_TARGET}-ctest-rvv_kernel" +) +add_dependencies(toy-ctests-build ${RVV_KERNEL_TEST}-ctest-build) + +add_buckyball_ctest(rvv_ball.c) +set(RVV_BALL_TEST + "${DESIGN_NAME}-${BUCKYBALL_CTEST_TARGET}-ctest-rvv_ball" +) +add_dependencies(toy-ctests-build ${RVV_BALL_TEST}-ctest-build) + +add_buckyball_ctest(mmio_minimal_test.c) diff --git a/examples/designs/toy/workloads/device/core/acc/ctests/gemmini_relu_trace_pipeline/CMakeLists.txt b/examples/designs/toy/workloads/device/core/acc/ctests/gemmini_relu_trace_pipeline/CMakeLists.txt new file mode 100644 index 000000000..3686171da --- /dev/null +++ b/examples/designs/toy/workloads/device/core/acc/ctests/gemmini_relu_trace_pipeline/CMakeLists.txt @@ -0,0 +1,3 @@ +add_buckyball_component_workload(gemmini_relu_trace_pipeline SOURCES + gemmini_relu_trace_pipeline-main.cpp + toy.c) diff --git a/examples/designs/toy/workloads/device/core/acc/ctests/gemmini_relu_trace_pipeline/gemmini_relu_trace_pipeline-main.cpp b/examples/designs/toy/workloads/device/core/acc/ctests/gemmini_relu_trace_pipeline/gemmini_relu_trace_pipeline-main.cpp new file mode 100644 index 000000000..f2fb24be2 --- /dev/null +++ b/examples/designs/toy/workloads/device/core/acc/ctests/gemmini_relu_trace_pipeline/gemmini_relu_trace_pipeline-main.cpp @@ -0,0 +1,22 @@ +#include +extern "C" int toy_pipeline(uint32_t instance); +static volatile uint32_t completed, failed; +int main() { + const uint32_t hart = bb_get_hart_id(); + const uint32_t expected = + bb_profile_core_count(BB_COMPONENT_WORKLOAD_PROFILE); + if (!expected) + return 1; + if (bb_hart_profile(hart) == BB_COMPONENT_WORKLOAD_PROFILE) { + __atomic_fetch_or(&failed, toy_pipeline(bb_hart_slot(hart)) != 0, + __ATOMIC_RELEASE); + __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); + } + if (hart == WORKLOAD_REPORT_HART) { + while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) + asm volatile("nop"); + return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; + } + for (;;) + asm volatile("wfi"); +} diff --git a/examples/cores/toy/workloads/ctests/gemmini_relu_trace_pipeline/toy.c b/examples/designs/toy/workloads/device/core/acc/ctests/gemmini_relu_trace_pipeline/toy.c similarity index 84% rename from examples/cores/toy/workloads/ctests/gemmini_relu_trace_pipeline/toy.c rename to examples/designs/toy/workloads/device/core/acc/ctests/gemmini_relu_trace_pipeline/toy.c index 4ce602537..13e3a8686 100644 --- a/examples/cores/toy/workloads/ctests/gemmini_relu_trace_pipeline/toy.c +++ b/examples/designs/toy/workloads/device/core/acc/ctests/gemmini_relu_trace_pipeline/toy.c @@ -4,11 +4,12 @@ #include #include #include -#include +#include #include -#if BANK_WIDTH != 128 || BANK_LINES < 64 || BANK_LINES % 64 != 0 -#error "Toy core pipeline requires 128-bit banks and depth divisible by 64" +#if THREAD_MEM_BANK_WIDTH != 128 || THREAD_MEM_BANK_ENTRIES < 64 || \ + THREAD_MEM_BANK_ENTRIES % 64 != 0 +#error "Toy ACC pipeline requires 128-bit banks and depth divisible by 64" #endif enum { DIM = 16 }; @@ -20,8 +21,8 @@ static int32_t output[DIM * DIM] __attribute__((aligned(64))); #ifdef __cplusplus extern "C" #endif - int toy_core(core_id_t id) { - (void)id; + int toy_pipeline(uint32_t instance) { + (void)instance; for (int row = 0; row < DIM; ++row) for (int col = 0; col < DIM; ++col) { input[row * DIM + col] = (row * 3 + col * 5) % 15 - 7; diff --git a/examples/chips/toy/workloads/ctests/mmio_minimal_test.c b/examples/designs/toy/workloads/device/core/acc/ctests/mmio_minimal_test.c similarity index 94% rename from examples/chips/toy/workloads/ctests/mmio_minimal_test.c rename to examples/designs/toy/workloads/device/core/acc/ctests/mmio_minimal_test.c index 17ab11398..6a931afce 100644 --- a/examples/chips/toy/workloads/ctests/mmio_minimal_test.c +++ b/examples/designs/toy/workloads/device/core/acc/ctests/mmio_minimal_test.c @@ -16,8 +16,8 @@ static uint8_t test_data[16] __attribute__((aligned(16))) = { 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10}; int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif printf("=== Minimal MMIO Test ===\n"); @@ -46,7 +46,7 @@ int main(void) { printf("MMIO test PASSED\n"); -#ifdef MULTICORE +#ifdef MULTITHREAD exit(0); #endif diff --git a/examples/designs/toy/workloads/device/core/acc/ctests/rvv_ball.c b/examples/designs/toy/workloads/device/core/acc/ctests/rvv_ball.c new file mode 100644 index 000000000..1be427efd --- /dev/null +++ b/examples/designs/toy/workloads/device/core/acc/ctests/rvv_ball.c @@ -0,0 +1,44 @@ +#include +#include +#include + +static const uint32_t image[] __attribute__((aligned(16))) = { + 0x31564b52, 12, 0, 0x40000000, 0, 0, 0x64b5307b, 0x64d6307b, 0x00008067}; +static int32_t input[128] __attribute__((aligned(16))); +static int32_t output[128] __attribute__((aligned(16))); + +int main(void) { + for (unsigned i = 0; i < 128; ++i) + input[i] = ((int)i - 64) * 7; + bb_mem_alloc(0, 1, 1); + bb_mem_alloc(2, 1, 2); + struct kernel_launch call __attribute__((aligned(16))) = { + 0, + 12, + 0x40002000, + {BB_BANK0(2) | BB_BANK2(2) | BB_ITER(16), THREAD_MEM_BANK_ENTRIES, + BB_BANK0(3) | BB_BANK2(3) | BB_ITER(16), THREAD_MEM_BANK_ENTRIES}, + 0}; + bb_mvin((uintptr_t)input, 2, 16, 1); + bb_mvin((uintptr_t)&call, 0, sizeof(call) / 16, 1); + bb_mem_alloc(4, 1, 1); + bb_mem_transfer(0, 3); + bb_mem_transfer(2, 3); + mvin_kernel(image, sizeof(image), CORE_VIRTUAL_BANK_COUNT); + run_kernel(4, CORE_VIRTUAL_BANK_COUNT, 3, 0); + int32_t groups[2][64] __attribute__((aligned(16))); + bb_mvout_group((uintptr_t)groups[0], 3, 1, 16, 1); + bb_mvout_group((uintptr_t)groups[1], 3, 2, 16, 1); + release_kernel(CORE_VIRTUAL_BANK_COUNT); + for (unsigned row = 0; row < 16; ++row) + for (unsigned group = 0; group < 2; ++group) + for (unsigned lane = 0; lane < 4; ++lane) + output[row * 8 + group * 4 + lane] = groups[group][row * 4 + lane]; + for (unsigned i = 0; i < 128; ++i) + if (output[i] != (input[i] > 0 ? input[i] : 0)) + return 1; + bb_mem_release(3); + bb_mem_release(4); + puts("RVV BALL PASSED"); + return 0; +} diff --git a/examples/designs/toy/workloads/device/core/acc/mlir_tests/CMakeLists.txt b/examples/designs/toy/workloads/device/core/acc/mlir_tests/CMakeLists.txt new file mode 100644 index 000000000..a859647b8 --- /dev/null +++ b/examples/designs/toy/workloads/device/core/acc/mlir_tests/CMakeLists.txt @@ -0,0 +1,8 @@ +add_custom_target(toy-mlir-tests-build) + +add_subdirectory(linalg) + +add_dependencies(toy-mlir-tests-build + linalg-mlir-tests-build) + +add_dependencies(design-mlirtests-build toy-mlir-tests-build) diff --git a/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/CMakeLists.txt b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/CMakeLists.txt new file mode 100644 index 000000000..f1391245c --- /dev/null +++ b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/CMakeLists.txt @@ -0,0 +1,108 @@ +set(BUCKYBALL_MLIR_GROUP_TARGET linalg-mlir-tests-build) +set(BUCKYBALL_MLIR_TEST_PREFIX linalg) + +add_custom_target(linalg-mlir-tests-build) + +add_buckyball_mlir_test(conv2d_nhwc_hwcf + TARGET toy + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) + +add_buckyball_mlir_test(conv2d_nhwc_hwcf_deep + TARGET toy + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) + +add_buckyball_mlir_test(conv2d_nhwc_fhwc + TARGET toy + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) + +add_buckyball_mlir_test(conv2d_nchw_fchw + TARGET toy + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) diff --git a/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nchw_fchw.mlir b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nchw_fchw.mlir new file mode 100644 index 000000000..27514b327 --- /dev/null +++ b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nchw_fchw.mlir @@ -0,0 +1,69 @@ +// Quantized convolution in the frontend QDQ form: input steps 0.25 with +// multiplier 4, weight step 0.5, initial output 1. The output is +// 1 + 0.125 * sum(xq * wq), so a wrong step fails the check. +func.func private @check_result(memref<1x4x3x3xf32>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c2 = arith.constant 2 : index + %c3 = arith.constant 3 : index + %c4 = arith.constant 4 : index + %c5 = arith.constant 5 : index + %c7 = arith.constant 7 : index + %c9 = arith.constant 9 : index + %quarter = arith.constant 0.25 : f32 + %input = memref.alloc() : memref<1x3x5x5xf32> + %weight = memref.alloc() : memref<4x3x3x3xi8> + scf.for %h = %c0 to %c5 step %c1 { + scf.for %w = %c0 to %c5 step %c1 { + scf.for %c = %c0 to %c3 step %c1 { + %h3 = arith.muli %h, %c3 : index + %w5 = arith.muli %w, %c5 : index + %c7c = arith.muli %c, %c7 : index + %s0 = arith.addi %h3, %w5 : index + %s = arith.addi %s0, %c7c : index + %r = arith.remui %s, %c9 : index + %v = arith.subi %r, %c4 : index + %vi = arith.index_cast %v : index to i32 + %vf = arith.sitofp %vi : i32 to f32 + %x = arith.mulf %vf, %quarter : f32 + memref.store %x, %input[%c0, %c, %h, %w] : memref<1x3x5x5xf32> + } + } + } + scf.for %f = %c0 to %c4 step %c1 { + scf.for %kh = %c0 to %c3 step %c1 { + scf.for %kw = %c0 to %c3 step %c1 { + scf.for %c = %c0 to %c3 step %c1 { + %f2 = arith.muli %f, %c2 : index + %kh3 = arith.muli %kh, %c3 : index + %kw5 = arith.muli %kw, %c5 : index + %s0 = arith.addi %f2, %kh3 : index + %s1 = arith.addi %s0, %kw5 : index + %s = arith.addi %s1, %c : index + %r = arith.remui %s, %c7 : index + %v = arith.subi %r, %c3 : index + %q = arith.index_cast %v : index to i8 + memref.store %q, %weight[%f, %c, %kh, %kw] : memref<4x3x3x3xi8> + } + } + } + } + %multiplier = arith.constant dense<4.0> : tensor<1x1x1x1xf32> + %input_step = arith.constant dense<0.25> : tensor<1x1x1x1xf32> + %weight_step = arith.constant dense<0.5> : tensor<1x1x1x1xf32> + %init = arith.constant dense<1.0> : tensor<1x4x3x3xf32> + %it = bufferization.to_tensor %input restrict : memref<1x3x5x5xf32> to tensor<1x3x5x5xf32> + %wt = bufferization.to_tensor %weight restrict : memref<4x3x3x3xi8> to tensor<4x3x3x3xi8> + %iq = "frontend.fp32_to_int8"(%it, %multiplier) : (tensor<1x3x5x5xf32>, tensor<1x1x1x1xf32>) -> tensor<1x3x5x5xi8> + %id = "frontend.int8_to_fp32"(%iq, %input_step) : (tensor<1x3x5x5xi8>, tensor<1x1x1x1xf32>) -> tensor<1x3x5x5xf32> + %wd = "frontend.int8_to_fp32"(%wt, %weight_step) : (tensor<4x3x3x3xi8>, tensor<1x1x1x1xf32>) -> tensor<4x3x3x3xf32> + %result = linalg.conv_2d_nchw_fchw + {dilations = dense<1> : tensor<2xi64>, strides = dense<1> : tensor<2xi64>} + ins(%id, %wd : tensor<1x3x5x5xf32>, tensor<4x3x3x3xf32>) outs(%init : tensor<1x4x3x3xf32>) -> tensor<1x4x3x3xf32> + %memory = bufferization.to_buffer %result : tensor<1x4x3x3xf32> to memref<1x4x3x3xf32> + func.call @check_result(%memory) : (memref<1x4x3x3xf32>) -> () + return %zero_i8 : i8 +} diff --git a/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nchw_fchw_main.cpp b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nchw_fchw_main.cpp new file mode 100644 index 000000000..529725211 --- /dev/null +++ b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nchw_fchw_main.cpp @@ -0,0 +1,53 @@ +#include +#include +#include + +static void fail(void) { +#ifdef BAREMETAL + *(volatile uint32_t *)0x60000000 = 1; + while (1) { + } +#else + exit(1); +#endif +} + +#ifdef __cplusplus +extern "C" +#endif + void check_result(float *allocated, float *aligned, int64_t offset, + int64_t size0, int64_t size1, int64_t size2, + int64_t size3, int64_t stride0, int64_t stride1, + int64_t stride2, int64_t stride3) { + (void)allocated; + (void)stride0; + const int64_t sizes[4] = {1, 4, 3, 3}; + if (size0 != sizes[0] || size1 != sizes[1] || size2 != sizes[2] || + size3 != sizes[3]) { + printf("FAILED: linalg conv2d_nchw_fchw unexpected memref shape\n"); + fail(); + } + const float *output = aligned + offset; + for (int oh = 0; oh < 3; ++oh) + for (int ow = 0; ow < 3; ++ow) + for (int f = 0; f < 4; ++f) { + int sum = 0; + for (int kh = 0; kh < 3; ++kh) + for (int kw = 0; kw < 3; ++kw) + for (int c = 0; c < 3; ++c) { + int x = (3 * (oh + kh) + 5 * (ow + kw) + 7 * c) % 9 - 4; + int w = (2 * f + 3 * kh + 5 * kw + c) % 7 - 3; + sum += x * w; + } + float expected = 1.0f + 0.125f * (float)sum; + float got = output[f * stride1 + oh * stride2 + ow * stride3]; + if (got != expected) { + printf( + "FAILED: linalg conv2d_nchw_fchw oh=%d ow=%d f=%d expected %d/8, " + "got %d/8\n", + oh, ow, f, (int)(expected * 8), (int)(got * 8)); + fail(); + } + } + printf("PASSED: linalg conv2d_nchw_fchw\n"); +} diff --git a/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_fhwc.mlir b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_fhwc.mlir new file mode 100644 index 000000000..e0a757ead --- /dev/null +++ b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_fhwc.mlir @@ -0,0 +1,69 @@ +// Quantized convolution in the frontend QDQ form: input steps 0.25 with +// multiplier 4, weight step 0.5, initial output 1. The output is +// 1 + 0.125 * sum(xq * wq), so a wrong step fails the check. +func.func private @check_result(memref<1x3x3x4xf32>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c2 = arith.constant 2 : index + %c3 = arith.constant 3 : index + %c4 = arith.constant 4 : index + %c5 = arith.constant 5 : index + %c7 = arith.constant 7 : index + %c9 = arith.constant 9 : index + %quarter = arith.constant 0.25 : f32 + %input = memref.alloc() : memref<1x5x5x3xf32> + %weight = memref.alloc() : memref<4x3x3x3xi8> + scf.for %h = %c0 to %c5 step %c1 { + scf.for %w = %c0 to %c5 step %c1 { + scf.for %c = %c0 to %c3 step %c1 { + %h3 = arith.muli %h, %c3 : index + %w5 = arith.muli %w, %c5 : index + %c7c = arith.muli %c, %c7 : index + %s0 = arith.addi %h3, %w5 : index + %s = arith.addi %s0, %c7c : index + %r = arith.remui %s, %c9 : index + %v = arith.subi %r, %c4 : index + %vi = arith.index_cast %v : index to i32 + %vf = arith.sitofp %vi : i32 to f32 + %x = arith.mulf %vf, %quarter : f32 + memref.store %x, %input[%c0, %h, %w, %c] : memref<1x5x5x3xf32> + } + } + } + scf.for %f = %c0 to %c4 step %c1 { + scf.for %kh = %c0 to %c3 step %c1 { + scf.for %kw = %c0 to %c3 step %c1 { + scf.for %c = %c0 to %c3 step %c1 { + %f2 = arith.muli %f, %c2 : index + %kh3 = arith.muli %kh, %c3 : index + %kw5 = arith.muli %kw, %c5 : index + %s0 = arith.addi %f2, %kh3 : index + %s1 = arith.addi %s0, %kw5 : index + %s = arith.addi %s1, %c : index + %r = arith.remui %s, %c7 : index + %v = arith.subi %r, %c3 : index + %q = arith.index_cast %v : index to i8 + memref.store %q, %weight[%f, %kh, %kw, %c] : memref<4x3x3x3xi8> + } + } + } + } + %multiplier = arith.constant dense<4.0> : tensor<1x1x1x1xf32> + %input_step = arith.constant dense<0.25> : tensor<1x1x1x1xf32> + %weight_step = arith.constant dense<0.5> : tensor<1x1x1x1xf32> + %init = arith.constant dense<1.0> : tensor<1x3x3x4xf32> + %it = bufferization.to_tensor %input restrict : memref<1x5x5x3xf32> to tensor<1x5x5x3xf32> + %wt = bufferization.to_tensor %weight restrict : memref<4x3x3x3xi8> to tensor<4x3x3x3xi8> + %iq = "frontend.fp32_to_int8"(%it, %multiplier) : (tensor<1x5x5x3xf32>, tensor<1x1x1x1xf32>) -> tensor<1x5x5x3xi8> + %id = "frontend.int8_to_fp32"(%iq, %input_step) : (tensor<1x5x5x3xi8>, tensor<1x1x1x1xf32>) -> tensor<1x5x5x3xf32> + %wd = "frontend.int8_to_fp32"(%wt, %weight_step) : (tensor<4x3x3x3xi8>, tensor<1x1x1x1xf32>) -> tensor<4x3x3x3xf32> + %result = linalg.conv_2d_nhwc_fhwc + {dilations = dense<1> : tensor<2xi64>, strides = dense<1> : tensor<2xi64>} + ins(%id, %wd : tensor<1x5x5x3xf32>, tensor<4x3x3x3xf32>) outs(%init : tensor<1x3x3x4xf32>) -> tensor<1x3x3x4xf32> + %memory = bufferization.to_buffer %result : tensor<1x3x3x4xf32> to memref<1x3x3x4xf32> + func.call @check_result(%memory) : (memref<1x3x3x4xf32>) -> () + return %zero_i8 : i8 +} diff --git a/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_fhwc_main.cpp b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_fhwc_main.cpp new file mode 100644 index 000000000..cceb5458e --- /dev/null +++ b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_fhwc_main.cpp @@ -0,0 +1,53 @@ +#include +#include +#include + +static void fail(void) { +#ifdef BAREMETAL + *(volatile uint32_t *)0x60000000 = 1; + while (1) { + } +#else + exit(1); +#endif +} + +#ifdef __cplusplus +extern "C" +#endif + void check_result(float *allocated, float *aligned, int64_t offset, + int64_t size0, int64_t size1, int64_t size2, + int64_t size3, int64_t stride0, int64_t stride1, + int64_t stride2, int64_t stride3) { + (void)allocated; + (void)stride0; + const int64_t sizes[4] = {1, 3, 3, 4}; + if (size0 != sizes[0] || size1 != sizes[1] || size2 != sizes[2] || + size3 != sizes[3]) { + printf("FAILED: linalg conv2d_nhwc_fhwc unexpected memref shape\n"); + fail(); + } + const float *output = aligned + offset; + for (int oh = 0; oh < 3; ++oh) + for (int ow = 0; ow < 3; ++ow) + for (int f = 0; f < 4; ++f) { + int sum = 0; + for (int kh = 0; kh < 3; ++kh) + for (int kw = 0; kw < 3; ++kw) + for (int c = 0; c < 3; ++c) { + int x = (3 * (oh + kh) + 5 * (ow + kw) + 7 * c) % 9 - 4; + int w = (2 * f + 3 * kh + 5 * kw + c) % 7 - 3; + sum += x * w; + } + float expected = 1.0f + 0.125f * (float)sum; + float got = output[oh * stride1 + ow * stride2 + f * stride3]; + if (got != expected) { + printf( + "FAILED: linalg conv2d_nhwc_fhwc oh=%d ow=%d f=%d expected %d/8, " + "got %d/8\n", + oh, ow, f, (int)(expected * 8), (int)(got * 8)); + fail(); + } + } + printf("PASSED: linalg conv2d_nhwc_fhwc\n"); +} diff --git a/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_hwcf.mlir b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_hwcf.mlir new file mode 100644 index 000000000..96b135696 --- /dev/null +++ b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_hwcf.mlir @@ -0,0 +1,69 @@ +// Quantized convolution in the frontend QDQ form: input steps 0.25 with +// multiplier 4, weight step 0.5, initial output 1. The output is +// 1 + 0.125 * sum(xq * wq), so a wrong step fails the check. +func.func private @check_result(memref<1x3x3x4xf32>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c2 = arith.constant 2 : index + %c3 = arith.constant 3 : index + %c4 = arith.constant 4 : index + %c5 = arith.constant 5 : index + %c7 = arith.constant 7 : index + %c9 = arith.constant 9 : index + %quarter = arith.constant 0.25 : f32 + %input = memref.alloc() : memref<1x5x5x3xf32> + %weight = memref.alloc() : memref<3x3x3x4xi8> + scf.for %h = %c0 to %c5 step %c1 { + scf.for %w = %c0 to %c5 step %c1 { + scf.for %c = %c0 to %c3 step %c1 { + %h3 = arith.muli %h, %c3 : index + %w5 = arith.muli %w, %c5 : index + %c7c = arith.muli %c, %c7 : index + %s0 = arith.addi %h3, %w5 : index + %s = arith.addi %s0, %c7c : index + %r = arith.remui %s, %c9 : index + %v = arith.subi %r, %c4 : index + %vi = arith.index_cast %v : index to i32 + %vf = arith.sitofp %vi : i32 to f32 + %x = arith.mulf %vf, %quarter : f32 + memref.store %x, %input[%c0, %h, %w, %c] : memref<1x5x5x3xf32> + } + } + } + scf.for %f = %c0 to %c4 step %c1 { + scf.for %kh = %c0 to %c3 step %c1 { + scf.for %kw = %c0 to %c3 step %c1 { + scf.for %c = %c0 to %c3 step %c1 { + %f2 = arith.muli %f, %c2 : index + %kh3 = arith.muli %kh, %c3 : index + %kw5 = arith.muli %kw, %c5 : index + %s0 = arith.addi %f2, %kh3 : index + %s1 = arith.addi %s0, %kw5 : index + %s = arith.addi %s1, %c : index + %r = arith.remui %s, %c7 : index + %v = arith.subi %r, %c3 : index + %q = arith.index_cast %v : index to i8 + memref.store %q, %weight[%kh, %kw, %c, %f] : memref<3x3x3x4xi8> + } + } + } + } + %multiplier = arith.constant dense<4.0> : tensor<1x1x1x1xf32> + %input_step = arith.constant dense<0.25> : tensor<1x1x1x1xf32> + %weight_step = arith.constant dense<0.5> : tensor<1x1x1x1xf32> + %init = arith.constant dense<1.0> : tensor<1x3x3x4xf32> + %it = bufferization.to_tensor %input restrict : memref<1x5x5x3xf32> to tensor<1x5x5x3xf32> + %wt = bufferization.to_tensor %weight restrict : memref<3x3x3x4xi8> to tensor<3x3x3x4xi8> + %iq = "frontend.fp32_to_int8"(%it, %multiplier) : (tensor<1x5x5x3xf32>, tensor<1x1x1x1xf32>) -> tensor<1x5x5x3xi8> + %id = "frontend.int8_to_fp32"(%iq, %input_step) : (tensor<1x5x5x3xi8>, tensor<1x1x1x1xf32>) -> tensor<1x5x5x3xf32> + %wd = "frontend.int8_to_fp32"(%wt, %weight_step) : (tensor<3x3x3x4xi8>, tensor<1x1x1x1xf32>) -> tensor<3x3x3x4xf32> + %result = linalg.conv_2d_nhwc_hwcf + {dilations = dense<1> : tensor<2xi64>, strides = dense<1> : tensor<2xi64>} + ins(%id, %wd : tensor<1x5x5x3xf32>, tensor<3x3x3x4xf32>) outs(%init : tensor<1x3x3x4xf32>) -> tensor<1x3x3x4xf32> + %memory = bufferization.to_buffer %result : tensor<1x3x3x4xf32> to memref<1x3x3x4xf32> + func.call @check_result(%memory) : (memref<1x3x3x4xf32>) -> () + return %zero_i8 : i8 +} diff --git a/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_hwcf_deep.mlir b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_hwcf_deep.mlir new file mode 100644 index 000000000..24c3bf8fb --- /dev/null +++ b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_hwcf_deep.mlir @@ -0,0 +1,70 @@ +// Quantized convolution with K = 3 * 3 * 130 = 1170, deeper than one bank: +// A and B each span two bank groups. Input steps 0.25 with multiplier 4, +// weight step 0.5, initial output 1; the output is 1 + 0.125 * sum(xq * wq). +func.func private @check_result(memref<1x1x1x4xf32>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c2 = arith.constant 2 : index + %c3 = arith.constant 3 : index + %c4 = arith.constant 4 : index + %c5 = arith.constant 5 : index + %c7 = arith.constant 7 : index + %c9 = arith.constant 9 : index + %c130 = arith.constant 130 : index + %quarter = arith.constant 0.25 : f32 + %input = memref.alloc() : memref<1x3x3x130xf32> + %weight = memref.alloc() : memref<3x3x130x4xi8> + scf.for %h = %c0 to %c3 step %c1 { + scf.for %w = %c0 to %c3 step %c1 { + scf.for %c = %c0 to %c130 step %c1 { + %h3 = arith.muli %h, %c3 : index + %w5 = arith.muli %w, %c5 : index + %c7c = arith.muli %c, %c7 : index + %s0 = arith.addi %h3, %w5 : index + %s = arith.addi %s0, %c7c : index + %r = arith.remui %s, %c9 : index + %v = arith.subi %r, %c4 : index + %vi = arith.index_cast %v : index to i32 + %vf = arith.sitofp %vi : i32 to f32 + %x = arith.mulf %vf, %quarter : f32 + memref.store %x, %input[%c0, %h, %w, %c] : memref<1x3x3x130xf32> + } + } + } + scf.for %f = %c0 to %c4 step %c1 { + scf.for %kh = %c0 to %c3 step %c1 { + scf.for %kw = %c0 to %c3 step %c1 { + scf.for %c = %c0 to %c130 step %c1 { + %f2 = arith.muli %f, %c2 : index + %kh3 = arith.muli %kh, %c3 : index + %kw5 = arith.muli %kw, %c5 : index + %s0 = arith.addi %f2, %kh3 : index + %s1 = arith.addi %s0, %kw5 : index + %s = arith.addi %s1, %c : index + %r = arith.remui %s, %c7 : index + %v = arith.subi %r, %c3 : index + %q = arith.index_cast %v : index to i8 + memref.store %q, %weight[%kh, %kw, %c, %f] : memref<3x3x130x4xi8> + } + } + } + } + %multiplier = arith.constant dense<4.0> : tensor<1x1x1x1xf32> + %input_step = arith.constant dense<0.25> : tensor<1x1x1x1xf32> + %weight_step = arith.constant dense<0.5> : tensor<1x1x1x1xf32> + %init = arith.constant dense<1.0> : tensor<1x1x1x4xf32> + %it = bufferization.to_tensor %input restrict : memref<1x3x3x130xf32> to tensor<1x3x3x130xf32> + %wt = bufferization.to_tensor %weight restrict : memref<3x3x130x4xi8> to tensor<3x3x130x4xi8> + %iq = "frontend.fp32_to_int8"(%it, %multiplier) : (tensor<1x3x3x130xf32>, tensor<1x1x1x1xf32>) -> tensor<1x3x3x130xi8> + %id = "frontend.int8_to_fp32"(%iq, %input_step) : (tensor<1x3x3x130xi8>, tensor<1x1x1x1xf32>) -> tensor<1x3x3x130xf32> + %wd = "frontend.int8_to_fp32"(%wt, %weight_step) : (tensor<3x3x130x4xi8>, tensor<1x1x1x1xf32>) -> tensor<3x3x130x4xf32> + %result = linalg.conv_2d_nhwc_hwcf + {dilations = dense<1> : tensor<2xi64>, strides = dense<1> : tensor<2xi64>} + ins(%id, %wd : tensor<1x3x3x130xf32>, tensor<3x3x130x4xf32>) outs(%init : tensor<1x1x1x4xf32>) -> tensor<1x1x1x4xf32> + %memory = bufferization.to_buffer %result : tensor<1x1x1x4xf32> to memref<1x1x1x4xf32> + func.call @check_result(%memory) : (memref<1x1x1x4xf32>) -> () + return %zero_i8 : i8 +} diff --git a/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_hwcf_deep_main.cpp b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_hwcf_deep_main.cpp new file mode 100644 index 000000000..821decbf5 --- /dev/null +++ b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_hwcf_deep_main.cpp @@ -0,0 +1,53 @@ +#include +#include +#include + +static void fail(void) { +#ifdef BAREMETAL + *(volatile uint32_t *)0x60000000 = 1; + while (1) { + } +#else + exit(1); +#endif +} + +#ifdef __cplusplus +extern "C" +#endif + void check_result(float *allocated, float *aligned, int64_t offset, + int64_t size0, int64_t size1, int64_t size2, + int64_t size3, int64_t stride0, int64_t stride1, + int64_t stride2, int64_t stride3) { + (void)allocated; + (void)stride0; + const int64_t sizes[4] = {1, 1, 1, 4}; + if (size0 != sizes[0] || size1 != sizes[1] || size2 != sizes[2] || + size3 != sizes[3]) { + printf("FAILED: linalg conv2d_nhwc_hwcf_deep unexpected memref shape\n"); + fail(); + } + const float *output = aligned + offset; + for (int oh = 0; oh < 1; ++oh) + for (int ow = 0; ow < 1; ++ow) + for (int f = 0; f < 4; ++f) { + int sum = 0; + for (int kh = 0; kh < 3; ++kh) + for (int kw = 0; kw < 3; ++kw) + for (int c = 0; c < 130; ++c) { + int x = (3 * (oh + kh) + 5 * (ow + kw) + 7 * c) % 9 - 4; + int w = (2 * f + 3 * kh + 5 * kw + c) % 7 - 3; + sum += x * w; + } + float expected = 1.0f + 0.125f * (float)sum; + float got = output[oh * stride1 + ow * stride2 + f * stride3]; + if (got != expected) { + printf("FAILED: linalg conv2d_nhwc_hwcf_deep oh=%d ow=%d f=%d " + "expected %d/8, " + "got %d/8\n", + oh, ow, f, (int)(expected * 8), (int)(got * 8)); + fail(); + } + } + printf("PASSED: linalg conv2d_nhwc_hwcf_deep\n"); +} diff --git a/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_hwcf_main.cpp b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_hwcf_main.cpp new file mode 100644 index 000000000..ec8ce23a7 --- /dev/null +++ b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_hwcf_main.cpp @@ -0,0 +1,53 @@ +#include +#include +#include + +static void fail(void) { +#ifdef BAREMETAL + *(volatile uint32_t *)0x60000000 = 1; + while (1) { + } +#else + exit(1); +#endif +} + +#ifdef __cplusplus +extern "C" +#endif + void check_result(float *allocated, float *aligned, int64_t offset, + int64_t size0, int64_t size1, int64_t size2, + int64_t size3, int64_t stride0, int64_t stride1, + int64_t stride2, int64_t stride3) { + (void)allocated; + (void)stride0; + const int64_t sizes[4] = {1, 3, 3, 4}; + if (size0 != sizes[0] || size1 != sizes[1] || size2 != sizes[2] || + size3 != sizes[3]) { + printf("FAILED: linalg conv2d_nhwc_hwcf unexpected memref shape\n"); + fail(); + } + const float *output = aligned + offset; + for (int oh = 0; oh < 3; ++oh) + for (int ow = 0; ow < 3; ++ow) + for (int f = 0; f < 4; ++f) { + int sum = 0; + for (int kh = 0; kh < 3; ++kh) + for (int kw = 0; kw < 3; ++kw) + for (int c = 0; c < 3; ++c) { + int x = (3 * (oh + kh) + 5 * (ow + kw) + 7 * c) % 9 - 4; + int w = (2 * f + 3 * kh + 5 * kw + c) % 7 - 3; + sum += x * w; + } + float expected = 1.0f + 0.125f * (float)sum; + float got = output[oh * stride1 + ow * stride2 + f * stride3]; + if (got != expected) { + printf( + "FAILED: linalg conv2d_nhwc_hwcf oh=%d ow=%d f=%d expected %d/8, " + "got %d/8\n", + oh, ow, f, (int)(expected * 8), (int)(got * 8)); + fail(); + } + } + printf("PASSED: linalg conv2d_nhwc_hwcf\n"); +} diff --git a/examples/designs/toy/workloads/device/soc/CMakeLists.txt b/examples/designs/toy/workloads/device/soc/CMakeLists.txt new file mode 100644 index 000000000..7a5138b0c --- /dev/null +++ b/examples/designs/toy/workloads/device/soc/CMakeLists.txt @@ -0,0 +1,5 @@ +add_buckyball_ctest(scu_uart_rx_echo_test.c) +set(SCU_UART_TEST + "${DESIGN_NAME}-${BUCKYBALL_CTEST_TARGET}-ctest-scu_uart_rx_echo_test" +) +add_dependencies(soctests-build ${SCU_UART_TEST}-baremetal-build) diff --git a/examples/chips/toy/workloads/ctests/scu_uart_rx_echo_test.c b/examples/designs/toy/workloads/device/soc/scu_uart_rx_echo_test.c similarity index 100% rename from examples/chips/toy/workloads/ctests/scu_uart_rx_echo_test.c rename to examples/designs/toy/workloads/device/soc/scu_uart_rx_echo_test.c diff --git a/examples/prefab/balls/build.sc b/examples/prefab/balls/build.sc new file mode 100644 index 000000000..f1e3aeac4 --- /dev/null +++ b/examples/prefab/balls/build.sc @@ -0,0 +1,12 @@ +import mill._ + +def sources = T.sources { + os.list(millSourcePath).filter(os.isDir).flatMap { root => + val source = root / "arch" + val configs = root / "configs" + (if (os.exists(source)) + os.walk(source).filter(p => p.ext == "scala" && !p.segments.exists(Set("out", "generated", "target", "build"))) + else Seq.empty) ++ + (if (os.exists(configs)) os.walk(configs).filter(_.ext == "scala") else Seq.empty) + }.map(PathRef(_)) +} diff --git a/examples/prefab/balls/f32add/arch/F32AddBall.scala b/examples/prefab/balls/f32add/arch/F32AddBall.scala new file mode 100644 index 000000000..beb91c17b --- /dev/null +++ b/examples/prefab/balls/f32add/arch/F32AddBall.scala @@ -0,0 +1,139 @@ +package examples.balls.f32add + +import framework.acc.balldomain.bbus.{BallFactory, BallPorts} +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.acc.frontend.configs.FrontendParam +import buckyball.balls.f32add.{Params => ParametersProto} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import framework.acc.balldomain.ball.blink.{BallCommand, BlinkIO, HasBlink} + +@instantiable +class F32AddBall( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Module + with HasBlink { + val ball = bp.balls.find(_.id == id).get + + ball.parameters.unpack(classOf[ParametersProto]) + + require(ball.inBW == 2 && ball.outBW == 1) + require( + mp.bankWidth == 128 && mp.bankMaskLen == 16 + ) + + @public + val blink = IO(new BlinkIO(fp, mp, bp, id)) + + val idle :: channels :: reading :: calculate :: writing :: acknowledge :: complete :: Nil = Enum(7) + val state = RegInit(idle) + + val issue = Reg(new BallCommand(fp, mp)) + + val row = RegInit(0.U(16.W)) + val requested = RegInit(VecInit(Seq.fill(2)(false.B))) + val received = RegInit(VecInit(Seq.fill(2)(false.B))) + val words = Reg(Vec(2, UInt(128.W))) + val output = Reg(UInt(128.W)) + val first = issue.rs2(5) + val sourceGroup = issue.rs2(4, 0) + val add = Instantiate(new F32AddRow) + add.io.source := words(0) + add.io.accumulator := Mux(first, 0.U(128.W), words(1)) + + blink.cmdReq.ready := state === idle + blink.cmdResp.valid := state === complete + blink.cmdResp.bits.rob_id := issue.rob_id + blink.status.idle := state === idle + blink.status.running := state =/= idle + + for (port <- 0 until 2) { + val read = blink.bankRead(port) + val active = if (port == 0) true.B else !first + read.rob_id := issue.rob_id + read.bank_id := (if (port == 0) issue.op1_bank else issue.op2_bank) + read.group_id := (if (port == 0) issue.op1_group + sourceGroup else issue.op2_group) + read.io.req.valid := state === reading && active && !requested(port) + read.io.req.bits.addr := row + read.io.resp.ready := state === reading && active && requested(port) && !received(port) + when(read.io.req.fire)(requested(port) := true.B) + when(read.io.resp.fire) { + words(port) := read.io.resp.bits.data + received(port) := true.B + } + } + + val write = blink.bankWrite(0) + write.rob_id := issue.rob_id + write.bank_id := issue.wr_bank + write.group_id := issue.wr_group + write.io.req.valid := state === writing + write.io.req.bits.addr := row + write.io.req.bits.data := output + write.io.req.bits.mask := VecInit(Seq.fill(16)(true.B)) + write.io.resp.ready := state === acknowledge + + when(blink.cmdReq.fire) { + val cmd = blink.cmdReq.bits + assert(cmd.rs2(63, 6) === 0.U, "F32ADD reserved rs2 bits") + assert( + cmd.op1_bank =/= cmd.op2_bank && cmd.op1_bank =/= cmd.wr_bank && cmd.op2_bank =/= cmd.wr_bank, + "F32ADD banks must differ" + ) + assert( + cmd.rs2(4, 0) < cmd.op1_col && cmd.op2_col === 1.U && cmd.wr_col === 1.U, + "F32ADD source group must exist and accumulators must have one group" + ) + for (bank <- Seq(cmd.op1_bank, cmd.op2_bank, cmd.wr_bank)) { + val shared = bank >= fp.shared_vbank_base.U + assert( + !shared || mp.sharedEnable.B, + "F32ADD shared memory is disabled" + ) + val depth = Mux(shared, mp.sharedBankEntries.U, mp.bankEntries.U) + assert(cmd.iter =/= 0.U && cmd.iter <= depth, "F32ADD row span exceeds bank") + } + issue := blink.cmdReq.bits + row := 0.U + requested.foreach(_ := false.B) + received.foreach(_ := false.B) + state := channels + } + when(state === channels)(state := reading) + when(state === reading && received(0) && (first || received(1)))(state := calculate) + when(state === calculate) { + output := add.io.sum + state := writing + } + when(write.io.req.fire)(state := acknowledge) + when(write.io.resp.fire) { + assert(write.io.resp.bits.ok, "F32ADD bank write failed") + when(row +& 1.U === issue.iter) { + state := complete + }.otherwise { + row := row + 1.U + requested.foreach(_ := false.B) + received.foreach(_ := false.B) + state := reading + } + } + when(blink.cmdResp.fire)(state := idle) +} + +object F32AddBall extends BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts = + BallPorts(Instantiate(new F32AddBall(fp, mp, bp, id)).blink, None) + +} diff --git a/examples/balls/f32add/arch/src/main/scala/F32AddRow.scala b/examples/prefab/balls/f32add/arch/F32AddRow.scala similarity index 91% rename from examples/balls/f32add/arch/src/main/scala/F32AddRow.scala rename to examples/prefab/balls/f32add/arch/F32AddRow.scala index a582daec9..6d33b6776 100644 --- a/examples/balls/f32add/arch/src/main/scala/F32AddRow.scala +++ b/examples/prefab/balls/f32add/arch/F32AddRow.scala @@ -1,5 +1,7 @@ package examples.balls.f32add +import chisel3.experimental.hierarchy.{Instance, Instantiate} + import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public} @@ -9,7 +11,8 @@ import hardfloat.consts.{round_near_even, tininess_afterRounding} @instantiable class F32AddRow extends Module { - @public val io = IO(new Bundle { + @public + val io = IO(new Bundle { val source = Input(UInt(128.W)) val accumulator = Input(UInt(128.W)) val sum = Output(UInt(128.W)) diff --git a/arch/src/main/scala/framework/mem-core/coherence/src/main/resources/README.md b/examples/prefab/balls/f32add/configs/default.toml similarity index 100% rename from arch/src/main/scala/framework/mem-core/coherence/src/main/resources/README.md rename to examples/prefab/balls/f32add/configs/default.toml diff --git a/examples/prefab/balls/f32add/configs/f32add_params.proto b/examples/prefab/balls/f32add/configs/f32add_params.proto new file mode 100644 index 000000000..150903ad5 --- /dev/null +++ b/examples/prefab/balls/f32add/configs/f32add_params.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; +package buckyball.balls.f32add; +option java_package = "buckyball.balls.f32add"; +option java_multiple_files = true; + +message Params { +} diff --git a/examples/balls/f32add/emu/src/add.rs b/examples/prefab/balls/f32add/emu/src/add.rs similarity index 87% rename from examples/balls/f32add/emu/src/add.rs rename to examples/prefab/balls/f32add/emu/src/add.rs index f3f252e5a..cb868b5eb 100644 --- a/examples/balls/f32add/emu/src/add.rs +++ b/examples/prefab/balls/f32add/emu/src/add.rs @@ -1,7 +1,8 @@ -use crate::inst::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; -use crate::inst::instruction::ExecContext; +use bebop_bemu::thread::instructions::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old}; +use bebop_bemu::ball::ExecContext; pub(crate) fn execute(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + assert!(!rs1_read_old(xs1), "add_f32: reads no old destination; rs1[29] must be clear"); let a = rs1_b0(xs1); let b = rs1_b1(xs1); let c = rs1_b2(xs1); diff --git a/examples/prefab/balls/f32add/emu/src/dispatch.rs b/examples/prefab/balls/f32add/emu/src/dispatch.rs new file mode 100644 index 000000000..72bf0db76 --- /dev/null +++ b/examples/prefab/balls/f32add/emu/src/dispatch.rs @@ -0,0 +1,26 @@ +use bebop_bemu::ball::ExecContext; + +pub fn execute_known( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, + ctx: &mut ExecContext, +) -> Option { + if ball_class != "examples.balls.f32add.F32AddBall" { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("F32ADD") => Some(super::add::execute(xs1, xs2, ctx)), + _ => None, + } +} +pub fn cycles_after_issue(ball_class: &str, funct: u32, _xs1: u64, _xs2: u64) -> Option { + if ball_class != "examples.balls.f32add.F32AddBall" { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("F32ADD") => Some(1), + _ => None, + } +} diff --git a/examples/prefab/balls/f32add/emu/src/lib.rs b/examples/prefab/balls/f32add/emu/src/lib.rs new file mode 100644 index 000000000..e75417d6e --- /dev/null +++ b/examples/prefab/balls/f32add/emu/src/lib.rs @@ -0,0 +1,8 @@ +mod add; +mod dispatch; + +pub use dispatch::{cycles_after_issue, execute_known}; + +#[bebop_bemu::ball::linkme::distributed_slice(bebop_bemu::BALLS)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static BALL: bebop_bemu::Ball = bebop_bemu::Ball { execute: execute_known, cycles: cycles_after_issue }; diff --git a/examples/prefab/balls/f32add/stack/src/f32addOps.cpp b/examples/prefab/balls/f32add/stack/src/f32addOps.cpp new file mode 100644 index 000000000..958a452a2 --- /dev/null +++ b/examples/prefab/balls/f32add/stack/src/f32addOps.cpp @@ -0,0 +1,68 @@ +//===- f32addOps.cpp - F32AddBall operations ------------------------------===// +// +// The dialect of F32AddBall, its virtual operations building their physical +// ones and the instructions of the physical operations. +// +//===----------------------------------------------------------------------===// + +#include "f32addOps.h" + +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::f32add_ball; +using mlir::buddy::bankFields; +using mlir::buddy::constantI64; +using mlir::buddy::emitInstruction; +using mlir::buddy::field; + +#include "f32addDialect.cpp.inc" + +#define GET_OP_CLASSES +#include "f32addOps.cpp.inc" + +void F32AddBallDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "f32addOps.cpp.inc" + >(); +} + +namespace { +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallDialect([](DialectRegistry ®istry) { + registry.insert(); + }), + true); +} // namespace + +SmallVector BankAddF32Op::getSourceBanks() { + return {getA(), getB()}; +} +Value BankAddF32Op::getDestinationBank() { return getC(); } +bool BankAddF32Op::readsDestination() { return false; } +Operation *BankAddF32Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return AddF32Op::create(builder, getLoc(), sources[0], sources[1], + destination, getRows()); +} + +SmallVector AddF32Op::getSourceBanks() { + return {int64_t(getA()), int64_t(getB())}; +} +int64_t AddF32Op::getDestinationBank() { return getC(); } +LogicalResult AddF32Op::lowerToInstructions(OpBuilder &builder) { + Location loc = getLoc(); + int64_t rows = getRows(), a = getA(), b = getB(), c = getC(); + if (rows <= 0 || rows > buckyball_target::getBuckyballTarget().bankDepth || + a == b || a == c || b == c) + return emitOpError( + "add_f32 requires distinct A, B and C banks and rows within a bank"); + // rs2: group 0 in [4:0]; bit 5 clear reads B. + emitInstruction(builder, loc, buckyball_target::getBuckyballFunct7("F32ADD"), + constantI64(builder, loc, bankFields({a, b}, c, false, rows)), + constantI64(builder, loc, 0)); + return success(); +} diff --git a/examples/prefab/balls/f32add/stack/src/f32addOps.h b/examples/prefab/balls/f32add/stack/src/f32addOps.h new file mode 100644 index 000000000..8a8671774 --- /dev/null +++ b/examples/prefab/balls/f32add/stack/src/f32addOps.h @@ -0,0 +1,20 @@ +//===- f32addOps.h - F32AddBall operations --------------------------------===// + +#ifndef F32ADD_OPS_H +#define F32ADD_OPS_H + +#include "Interfaces/BankInterfaces.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "f32addDialect.h.inc" + +#define GET_OP_CLASSES +#include "f32addOps.h.inc" + +#endif // F32ADD_OPS_H diff --git a/examples/prefab/balls/f32add/stack/src/f32addOps.td b/examples/prefab/balls/f32add/stack/src/f32addOps.td new file mode 100644 index 000000000..e97915505 --- /dev/null +++ b/examples/prefab/balls/f32add/stack/src/f32addOps.td @@ -0,0 +1,44 @@ +//===- f32addOps.td - F32AddBall operations ----------------*- tablegen -*-===// +// +// add_f32: C = A + B over `rows` rows of four f32, RNE with canonical NaN. +// A, B and C are distinct one-group banks. +// +// The Ball reads both addends from its source banks and cannot read the old +// destination: a sum of more terms is a chain of add_f32 into fresh banks. The +// Ball's input group is group 0 and it always reads B. +// +//===----------------------------------------------------------------------===// + +#ifndef F32ADD_OPS +#define F32ADD_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Interfaces/BankInterfaces.td" + +def F32AddBall_Dialect : Dialect { + let name = "f32add_ball"; + let cppNamespace = "::buddy::f32add_ball"; + let summary = "The operations of F32AddBall"; +} + +class F32AddBall_Op traits = []> + : Op])>; + +def BankAddF32Op : F32AddBall_Op<"bank_add_f32", + [DeclareOpInterfaceMethods]> { + let summary = "add_f32 on virtual banks"; + let arguments = (ins I64:$a, I64:$b, I64:$c, I64Attr:$rows); + let results = (outs I64:$cOut); + let assemblyFormat = "$a `,` $b `->` $c prop-dict attr-dict"; +} + +def AddF32Op : F32AddBall_Op<"add_f32", + [DeclareOpInterfaceMethods]> { + let summary = "add_f32 on physical banks"; + let arguments = (ins I64Attr:$a, I64Attr:$b, I64Attr:$c, I64Attr:$rows); + let assemblyFormat = "prop-dict attr-dict"; +} + +#endif // F32ADD_OPS diff --git a/examples/prefab/balls/f32add/verify/casegen/Cargo.lock b/examples/prefab/balls/f32add/verify/casegen/Cargo.lock new file mode 100644 index 000000000..d776305a8 --- /dev/null +++ b/examples/prefab/balls/f32add/verify/casegen/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "f32add-casegen" +version = "0.1.0" diff --git a/examples/prefab/balls/f32add/verify/casegen/Cargo.toml b/examples/prefab/balls/f32add/verify/casegen/Cargo.toml new file mode 100644 index 000000000..dae3cb329 --- /dev/null +++ b/examples/prefab/balls/f32add/verify/casegen/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "f32add-casegen" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["rlib", "cdylib"] + +[lints.rust] +unused_must_use = "deny" diff --git a/examples/prefab/balls/f32add/verify/casegen/src/casegen.rs b/examples/prefab/balls/f32add/verify/casegen/src/casegen.rs new file mode 100644 index 000000000..86ccc9898 --- /dev/null +++ b/examples/prefab/balls/f32add/verify/casegen/src/casegen.rs @@ -0,0 +1,135 @@ +use crate::model; + +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct F32AddCmd { + pub rows: u32, + pub a_bank: u32, + pub b_bank: u32, + pub c_bank: u32, + pub a_group_base: u32, + pub b_group: u32, + pub c_group: u32, + pub a_cols: u32, + pub group: u32, + pub first: u32, + pub rob_id: u32, +} + +#[derive(Clone, Debug)] +pub struct F32AddCase { + pub cmd: F32AddCmd, + pub a: Vec, + pub b: Vec, + pub c: Vec, +} + +struct Lcg(u64); + +impl Lcg { + fn next(&mut self) -> u32 { + self.0 = self.0.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + (self.0 >> 32) as u32 + } +} + +// (source, accumulator) pairs at the IEEE edges: signed zeros, infinities, +// quiet and signalling NaNs, overflow, subnormal results and ties. +const EDGES: [(u32, u32); 20] = [ + (0x0000_0000, 0x8000_0000), // +0 + -0 = +0 + (0x8000_0000, 0x8000_0000), // -0 + -0 = -0 + (0x7f80_0000, 0xff80_0000), // inf - inf = NaN + (0x7f80_0000, 0x3f80_0000), // inf + 1 + (0xff80_0000, 0xff80_0000), // -inf + -inf + (0x7fc0_1234, 0x3f80_0000), // quiet NaN payload + (0x3f80_0000, 0x7f80_0001), // signalling NaN + (0xffff_ffff, 0x0000_0000), // negative NaN + (0x7f7f_ffff, 0x7f7f_ffff), // overflow to inf + (0xff7f_ffff, 0xff7f_fffe), // overflow to -inf + (0x0000_0001, 0x0000_0001), // subnormal sum + (0x0000_0001, 0x8000_0001), // exact cancellation to +0 + (0x007f_ffff, 0x0000_0001), // subnormal to the smallest normal + (0x0080_0000, 0x8000_0001), // normal to subnormal + (0x3f80_0000, 0x3380_0000), // 1 + 2^-24: tie, stays even + (0x3f80_0001, 0x3380_0000), // tie rounds up to even + (0x3f80_0000, 0x33c0_0000), // above the tie + (0x3f80_0000, 0xbf80_0000), // 1 - 1 = +0 + (0x4b80_0000, 0xbf80_0000), // 2^24 - 1, exact + (0x7f00_0000, 0xfeff_ffff), // large cancellation +]; + +fn lane_pair(rng: &mut Lcg, index: usize, edges: bool) -> (u32, u32) { + if edges { + return EDGES[index % EDGES.len()]; + } + match index % 4 { + // Nearby exponents exercise alignment and rounding. + 0 | 1 => { + let a = rng.next(); + let b = (a & 0xff80_0000) ^ (rng.next() & 0x807f_ffff) ^ (rng.next() & 0x0380_0000); + (a, b) + } + _ => (rng.next(), rng.next()), + } +} + +fn pack(lanes: &[u32]) -> Vec { + lanes + .chunks(4) + .map(|chunk| { + let mut word = 0u128; + for (lane, value) in chunk.iter().enumerate() { + word |= u128::from(*value) << (32 * lane); + } + word + }) + .collect() +} + +pub fn gen_case(index: u32) -> F32AddCase { + let entries = model::bank_entries(); + // (rob_id, banks, rows, a_cols, group, first, group bases, edge values) + let (rob_id, banks, rows, a_cols, group, first, bases, edges) = match index { + 0 => (1, [0, 1, 2], 4, 1, 0, 0, [0, 0, 0], false), + 1 => (2, [3, 1, 0], 7, 1, 0, 1, [0, 0, 0], false), + 2 => (3, [2, 0, 3], 16, 3, 2, 0, [1, 2, 3], true), + 3 => (4, [33, 600, 1023], entries, 1, 0, 0, [0, 1, 0], false), + 4 => (5, [5, 6, 7], 9, 2, 1, 1, [2, 0, 1], true), + _ => panic!("f32add: unsupported case {index}"), + }; + model::validate(entries, banks[0], banks[1], banks[2], rows, group, first, a_cols); + let mut rng = Lcg(0xf32a_dd00 + u64::from(index)); + let lanes = rows as usize * 4; + let mut a_lanes = Vec::with_capacity(lanes); + let mut b_lanes = Vec::with_capacity(lanes); + for i in 0..lanes { + let (s, acc) = lane_pair(&mut rng, i, edges); + a_lanes.push(s); + b_lanes.push(acc); + } + let a = pack(&a_lanes); + let b = pack(&b_lanes); + let c = a + .iter() + .zip(&b) + .map(|(&s, &acc)| model::add_word(s, acc, first == 1)) + .collect(); + F32AddCase { + cmd: F32AddCmd { + rows, + a_bank: banks[0], + b_bank: banks[1], + c_bank: banks[2], + a_group_base: bases[0], + b_group: bases[1], + c_group: bases[2], + a_cols, + group, + first, + rob_id, + }, + a, + b, + c, + } +} diff --git a/examples/prefab/balls/f32add/verify/casegen/src/lib.rs b/examples/prefab/balls/f32add/verify/casegen/src/lib.rs new file mode 100644 index 000000000..79c832f35 --- /dev/null +++ b/examples/prefab/balls/f32add/verify/casegen/src/lib.rs @@ -0,0 +1,71 @@ +mod casegen; +mod model; + +use std::cell::RefCell; + +use casegen::{F32AddCase, F32AddCmd}; + +thread_local! { + static CURRENT: RefCell> = const { RefCell::new(None) }; +} + +#[no_mangle] +pub extern "C" fn f32add_case_load(index: u32) -> i32 { + let case = casegen::gen_case(index); + CURRENT.with(|current| *current.borrow_mut() = Some(case)); + 0 +} + +fn current_case R, R>(f: F) -> R { + CURRENT.with(|current| match current.borrow().as_ref() { + Some(case) => f(case), + None => panic!("f32add_case: no case loaded; call f32add_case_load first"), + }) +} + +fn idx(index: u32, case: &F32AddCase, what: &str) -> usize { + if index >= case.cmd.rows { + panic!("f32add_case_{what}: index {index} out of range {}", case.cmd.rows); + } + index as usize +} + +#[no_mangle] +pub extern "C" fn f32add_case_cmd(out_ptr: *mut F32AddCmd) { + if out_ptr.is_null() { + panic!("f32add_case_cmd: null out_ptr"); + } + current_case(|case| unsafe { + *out_ptr = case.cmd; + }); +} + +#[no_mangle] +pub extern "C" fn f32add_case_a_word_lo(index: u32) -> u64 { + current_case(|case| case.a[idx(index, case, "a_word_lo")] as u64) +} + +#[no_mangle] +pub extern "C" fn f32add_case_a_word_hi(index: u32) -> u64 { + current_case(|case| (case.a[idx(index, case, "a_word_hi")] >> 64) as u64) +} + +#[no_mangle] +pub extern "C" fn f32add_case_b_word_lo(index: u32) -> u64 { + current_case(|case| case.b[idx(index, case, "b_word_lo")] as u64) +} + +#[no_mangle] +pub extern "C" fn f32add_case_b_word_hi(index: u32) -> u64 { + current_case(|case| (case.b[idx(index, case, "b_word_hi")] >> 64) as u64) +} + +#[no_mangle] +pub extern "C" fn f32add_case_c_word_lo(index: u32) -> u64 { + current_case(|case| case.c[idx(index, case, "c_word_lo")] as u64) +} + +#[no_mangle] +pub extern "C" fn f32add_case_c_word_hi(index: u32) -> u64 { + current_case(|case| (case.c[idx(index, case, "c_word_hi")] >> 64) as u64) +} diff --git a/examples/prefab/balls/f32add/verify/casegen/src/model.rs b/examples/prefab/balls/f32add/verify/casegen/src/model.rs new file mode 100644 index 000000000..3a77cde64 --- /dev/null +++ b/examples/prefab/balls/f32add/verify/casegen/src/model.rs @@ -0,0 +1,50 @@ +// F32AddBall contract: C = A + B over `rows` rows of four IEEE FP32 lanes, +// round to nearest even, every NaN result is the canonical 0x7fc00000. +// rs2[4:0] selects the source group of A; rs2[5] (first) replaces B with +0 +// and B is not read. + +pub const CANONICAL_NAN: u32 = 0x7fc0_0000; + +pub fn bank_entries() -> u32 { + let path = std::env::var("BB_VERIFY_CONFIG").expect("BB_VERIFY_CONFIG is required"); + let text = std::fs::read_to_string(path).expect("failed to read BB_VERIFY_CONFIG"); + text.lines() + .find_map(|line| line.strip_prefix("bank_entries=")) + .expect("bank_entries missing from BB_VERIFY_CONFIG") + .parse() + .expect("invalid bank_entries in BB_VERIFY_CONFIG") +} + +pub fn validate(entries: u32, a: u32, b: u32, c: u32, rows: u32, group: u32, first: u32, cols: u32) { + if a == b || a == c || b == c { + panic!("f32add: banks must differ"); + } + if group > 31 || first > 1 { + panic!("f32add: rs2 fields out of range"); + } + if group >= cols { + panic!("f32add: source group must exist"); + } + if rows == 0 || rows > entries { + panic!("f32add: row span exceeds a bank"); + } +} + +pub fn add_lane(source: u32, accumulator: u32) -> u32 { + let sum = f32::from_bits(accumulator) + f32::from_bits(source); + if sum.is_nan() { + CANONICAL_NAN + } else { + sum.to_bits() + } +} + +pub fn add_word(source: u128, accumulator: u128, first: bool) -> u128 { + let mut out = 0u128; + for lane in 0..4 { + let s = (source >> (32 * lane)) as u32; + let acc = if first { 0 } else { (accumulator >> (32 * lane)) as u32 }; + out |= u128::from(add_lane(s, acc)) << (32 * lane); + } + out +} diff --git a/examples/prefab/balls/f32add/verify/filelists/f32add_ball.f b/examples/prefab/balls/f32add/verify/filelists/f32add_ball.f new file mode 100644 index 000000000..74feb5a66 --- /dev/null +++ b/examples/prefab/balls/f32add/verify/filelists/f32add_ball.f @@ -0,0 +1,29 @@ ++incdir+@UVM@/src/core ++incdir+@UVM@/src/ball ++incdir+@UVM@/src/ball/agents/cmd ++incdir+@UVM@/src/ball/agents/mem ++incdir+@UVM@/src/ball/agents/resp ++incdir+@UVM@/src/ball/cov ++incdir+@UVM@/src/ball/env ++incdir+src ++incdir+src/pkg ++incdir+src/common ++incdir+src/seq ++incdir+src/cov ++incdir+src/env ++incdir+src/tests ++incdir+@RTL@ + +@UVM@/src/core/package.sv +@UVM@/src/ball/bb_blink_if.sv +@UVM@/src/ball/bb_uvm_pkg.sv + +@RTL@/RoundAnyRawFNToRecFN_ie8_is26_oe8_os24.sv +@RTL@/RoundRawFNToRecFN_e8_s24.sv +@RTL@/AddRawFN.sv +@RTL@/AddRecFN.sv +@RTL@/F32AddRow.sv +@RTL@/F32AddBall.sv + +src/pkg/f32add_pkg.sv +src/tb_top.sv diff --git a/examples/prefab/balls/f32add/verify/src/common/f32add_defs.svh b/examples/prefab/balls/f32add/verify/src/common/f32add_defs.svh new file mode 100644 index 000000000..7630a9a52 --- /dev/null +++ b/examples/prefab/balls/f32add/verify/src/common/f32add_defs.svh @@ -0,0 +1,30 @@ +typedef struct { + int unsigned rows; + int unsigned a_bank; + int unsigned b_bank; + int unsigned c_bank; + int unsigned a_group_base; + int unsigned b_group; + int unsigned c_group; + int unsigned a_cols; + int unsigned group; + int unsigned first; + int unsigned rob_id; +} f32add_cmd_dpi_t; + +import "DPI-C" function int f32add_case_load(input int unsigned index); +import "DPI-C" function void f32add_case_cmd(output f32add_cmd_dpi_t cmd); +import "DPI-C" function longint unsigned f32add_case_a_word_lo(input int unsigned word_index); +import "DPI-C" function longint unsigned f32add_case_a_word_hi(input int unsigned word_index); +import "DPI-C" function longint unsigned f32add_case_b_word_lo(input int unsigned word_index); +import "DPI-C" function longint unsigned f32add_case_b_word_hi(input int unsigned word_index); +import "DPI-C" function longint unsigned f32add_case_c_word_lo(input int unsigned word_index); +import "DPI-C" function longint unsigned f32add_case_c_word_hi(input int unsigned word_index); + +`ifndef F32ADD_FUNCT7 +`error "F32ADD_FUNCT7 must be provided by the selected ballISA" +`endif + +localparam int F32ADD_THREAD_FUNCT7 = `F32ADD_FUNCT7; +localparam int F32ADD_NUM_CASES = 5; +localparam int F32ADD_TIMEOUT_CYCLES = 50000; diff --git a/examples/prefab/balls/f32add/verify/src/common/f32add_items.svh b/examples/prefab/balls/f32add/verify/src/common/f32add_items.svh new file mode 100644 index 000000000..fe706760b --- /dev/null +++ b/examples/prefab/balls/f32add/verify/src/common/f32add_items.svh @@ -0,0 +1,63 @@ +class f32add_cmd_item extends bb_blink_cmd_item; + `uvm_object_utils(f32add_cmd_item) + + bit first; + bit [4:0] group; + bit [127:0] a_words[]; + bit [127:0] b_words[]; + bit [127:0] c_words[]; + + function new(string name = "f32add_cmd_item"); + super.new(name); + endfunction + + function void load_rust_case(int unsigned index); + f32add_cmd_dpi_t cmd; + int unsigned rc; + + rc = f32add_case_load(index); + if (rc != 0) + `uvm_fatal("CASE", $sformatf("f32add_case_load returned %0d for index %0d", rc, index)) + f32add_case_cmd(cmd); + + // bb_f32add: banks a, b, c from rs1, rows in iter; rs2[4:0] is the source + // group of a and rs2[5] starts from +0 instead of reading b. + rob_id = cmd.rob_id[BB_BLINK_ROB_ID_W-1:0]; + funct = F32ADD_THREAD_FUNCT7[6:0]; + rs2 = {cmd.first[0], cmd.group[4:0]}; + iter = cmd.rows; + op1_bank = cmd.a_bank; + op2_bank = cmd.b_bank; + wr_bank = cmd.c_bank; + op1_group = cmd.a_group_base; + op2_group = cmd.b_group; + wr_group = cmd.c_group; + op1_col = cmd.a_cols; + op2_col = 1; + wr_col = 1; + wr_read_bank = '0; + wr_arch_bank = cmd.c_bank; + first = cmd.first[0]; + group = cmd.group[4:0]; + + a_words = new[cmd.rows]; + b_words = new[cmd.rows]; + c_words = new[cmd.rows]; + for (int i = 0; i < cmd.rows; i++) begin + a_words[i] = {f32add_case_a_word_hi(i), f32add_case_a_word_lo(i)}; + b_words[i] = {f32add_case_b_word_hi(i), f32add_case_b_word_lo(i)}; + c_words[i] = {f32add_case_c_word_hi(i), f32add_case_c_word_lo(i)}; + end + endfunction + + function void do_copy(uvm_object rhs); + f32add_cmd_item rhs_; + super.do_copy(rhs); + if (!$cast(rhs_, rhs)) `uvm_fatal("COPY", "rhs is not f32add_cmd_item") + first = rhs_.first; + group = rhs_.group; + a_words = rhs_.a_words; + b_words = rhs_.b_words; + c_words = rhs_.c_words; + endfunction +endclass diff --git a/examples/prefab/balls/f32add/verify/src/cov/f32add_cov.svh b/examples/prefab/balls/f32add/verify/src/cov/f32add_cov.svh new file mode 100644 index 000000000..d35e099f2 --- /dev/null +++ b/examples/prefab/balls/f32add/verify/src/cov/f32add_cov.svh @@ -0,0 +1,42 @@ +class f32add_cov extends uvm_component; + `uvm_component_utils(f32add_cov) + + uvm_analysis_imp_cmd #(bb_blink_cmd_item, f32add_cov) cmd_imp; + + bit cur_first; + int unsigned cur_group; + int unsigned cur_rows; + + covergroup cmd_cg; + coverpoint cur_first {bins accumulate = {0}; bins first = {1};} + coverpoint cur_group {bins zero = {0}; bins later = {[1 : 31]};} + coverpoint cur_rows {bins one_group = {[1 : 16]}; bins deep = {[17 : $]};} + cross cur_first, cur_group; + endgroup + + int cmd_count; + + function new(string name, uvm_component parent); + super.new(name, parent); + cmd_imp = new("cmd_imp", this); + cmd_cg = new(); + endfunction + + function void write_cmd(bb_blink_cmd_item item); + if (item.funct != F32ADD_THREAD_FUNCT7[6:0]) + `uvm_fatal("COV", $sformatf("unexpected funct %0d", item.funct)) + cur_first = item.rs2[5]; + cur_group = item.rs2[4:0]; + cur_rows = item.iter; + cmd_count++; + cmd_cg.sample(); + endfunction + + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (cmd_count == 0) `uvm_fatal("COV", "no cmd observed") + if (cmd_cg.get_coverage() != 100.0) + `uvm_fatal("COV", $sformatf("f32add_cov %0.2f%%", cmd_cg.get_coverage())) + endfunction +endclass diff --git a/examples/prefab/balls/f32add/verify/src/env/f32add_env.svh b/examples/prefab/balls/f32add/verify/src/env/f32add_env.svh new file mode 100644 index 000000000..cb86fb3c0 --- /dev/null +++ b/examples/prefab/balls/f32add/verify/src/env/f32add_env.svh @@ -0,0 +1,27 @@ +class f32add_env extends bb_blink_env #(`BB_IN_BW, `BB_OUT_BW); + `uvm_component_utils(f32add_env) + + f32add_scoreboard scb; + f32add_cov icov; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + scb = f32add_scoreboard::type_id::create("scb", this); + icov = f32add_cov::type_id::create("icov", this); + endfunction + + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + scb.mem_model = mem_model; + cmd_agent.stim_ap.connect(scb.stim_imp); + cmd_agent.cmd_ap.connect(scb.cmd_imp); + cmd_agent.cmd_ap.connect(icov.cmd_imp); + read_mon.read_ap.connect(scb.read_imp); + write_mon.write_ap.connect(scb.write_imp); + resp_mon.resp_ap.connect(scb.resp_imp); + endfunction +endclass diff --git a/examples/prefab/balls/f32add/verify/src/env/f32add_scoreboard.svh b/examples/prefab/balls/f32add/verify/src/env/f32add_scoreboard.svh new file mode 100644 index 000000000..0c4c828cc --- /dev/null +++ b/examples/prefab/balls/f32add/verify/src/env/f32add_scoreboard.svh @@ -0,0 +1,171 @@ +class f32add_scoreboard extends uvm_scoreboard; + `uvm_component_utils(f32add_scoreboard) + + uvm_analysis_imp_stim #(bb_blink_cmd_item, f32add_scoreboard) stim_imp; + uvm_analysis_imp_cmd #(bb_blink_cmd_item, f32add_scoreboard) cmd_imp; + uvm_analysis_imp_read #(bb_blink_read_item, f32add_scoreboard) read_imp; + uvm_analysis_imp_write #(bb_blink_write_item, f32add_scoreboard) write_imp; + uvm_analysis_imp_resp #(bb_blink_resp_item, f32add_scoreboard) resp_imp; + + bb_blink_mem_model #(`BB_IN_BW, `BB_OUT_BW) mem_model; + + f32add_cmd_item stim_q[$]; + int unsigned cmd_count; + int unsigned read_count[2]; + int unsigned write_count; + int unsigned resp_count; + + function new(string name, uvm_component parent); + super.new(name, parent); + stim_imp = new("stim_imp", this); + cmd_imp = new("cmd_imp", this); + read_imp = new("read_imp", this); + write_imp = new("write_imp", this); + resp_imp = new("resp_imp", this); + endfunction + + function void write_stim(bb_blink_cmd_item item); + f32add_cmd_item got; + f32add_cmd_item clone; + + if (stim_q.size() != 0) `uvm_fatal("SCB", "single outstanding command supported") + if (!$cast(got, item)) `uvm_fatal("SCB", "stim item is not f32add_cmd_item") + if (!$cast(clone, got.clone())) `uvm_fatal("SCB", "failed to clone stimulus item") + stim_q.push_back(clone); + + if (mem_model == null) `uvm_fatal("SCB", "mem_model handle not set") + mem_model.clear_mem(); + for (int i = 0; i < clone.iter; i++) begin + mem_model.load_word_g(int'(clone.op1_bank), int'(clone.op1_group + clone.group), i, + clone.a_words[i]); + // With first set B is not an operand: leaving it unloaded makes any read fail. + if (!clone.first) + mem_model.load_word_g(int'(clone.op2_bank), int'(clone.op2_group), i, clone.b_words[i]); + end + mem_model.arm(); + endfunction + + function void write_cmd(bb_blink_cmd_item item); + bb_blink_cmd_item want; + f32add_cmd_item stim; + stim = current_stim("CMD"); + // Compare as plain BLink commands; the comparer rejects differing types. + want = bb_blink_cmd_item::type_id::create("want"); + want.copy(stim); + verification_pkg::evidence::checks++; + if (!item.compare(want)) + `uvm_fatal("CMD", $sformatf("command mismatch: got %s exp %s", item.sprint(), want.sprint())) + cmd_count++; + endfunction + + function void write_read(bb_blink_read_item item); + f32add_cmd_item stim; + int unsigned expect_bank; + int unsigned expect_group; + + stim = current_stim("READ"); + if (item.port != 0 && item.port != 1) + `uvm_fatal("READ", $sformatf("F32AddBall has two read ports, got port %0d", item.port)) + if (item.port == 1 && stim.first) `uvm_fatal("READ", "B read although first is set") + if (read_count[item.port] >= stim.iter) + `uvm_fatal("READ", $sformatf("extra read on port %0d", item.port)) + if (read_count[item.port] != write_count) + `uvm_fatal("READ", $sformatf( + "port %0d read %0d before write %0d", item.port, read_count[item.port], write_count + )) + if (item.rob_id !== stim.rob_id) + `uvm_fatal("READ", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + if (item.port == 0) begin + expect_bank = stim.op1_bank; + expect_group = stim.op1_group + stim.group; + end else begin + expect_bank = stim.op2_bank; + expect_group = stim.op2_group; + end + if (item.bank_id !== expect_bank[BB_BLINK_BANK_ID_W-1:0]) + `uvm_fatal("READ", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, expect_bank)) + if (item.group_id !== expect_group[BB_BLINK_GROUP_ID_W-1:0]) + `uvm_fatal("READ", $sformatf("group mismatch: got %0d exp %0d", item.group_id, expect_group)) + if (item.addr !== read_count[item.port]) + `uvm_fatal("READ", $sformatf( + "addr mismatch: got %0d exp %0d", item.addr, read_count[item.port])) + read_count[item.port]++; + endfunction + + function void write_write(bb_blink_write_item item); + f32add_cmd_item stim; + + stim = current_stim("WRITE"); + if (item.port != 0) + `uvm_fatal("WRITE", $sformatf("F32AddBall has one write port, got %0d", item.port)) + if (write_count >= stim.iter) `uvm_fatal("WRITE", $sformatf("extra write %0d", write_count)) + if (read_count[0] != write_count + 1 || (!stim.first && read_count[1] != write_count + 1)) + `uvm_fatal("WRITE", $sformatf("write %0d before its operands", write_count)) + if (item.rob_id !== stim.rob_id) + `uvm_fatal("WRITE", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + if (item.bank_id !== stim.wr_bank) + `uvm_fatal("WRITE", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.wr_bank)) + if (item.group_id !== stim.wr_group) + `uvm_fatal("WRITE", $sformatf("group mismatch: got %0d exp %0d", item.group_id, stim.wr_group + )) + if (item.mask !== 16'hFFFF) + `uvm_fatal("WRITE", $sformatf("mask mismatch: got 0x%04h", item.mask)) + if (item.addr !== write_count) + `uvm_fatal("WRITE", $sformatf("addr mismatch: got %0d exp %0d", item.addr, write_count)) + verification_pkg::evidence::checks++; + if (item.data !== stim.c_words[write_count]) + `uvm_fatal("SCB", $sformatf( + "data mismatch at row %0d: got 0x%032h exp 0x%032h", + item.addr, + item.data, + stim.c_words[write_count] + )) + write_count++; + endfunction + + function void write_resp(bb_blink_resp_item item); + f32add_cmd_item stim; + stim = current_stim("RESP"); + if (write_count != stim.iter) + `uvm_fatal("RESP", $sformatf("response after %0d of %0d writes", write_count, stim.iter)) + if (item.rob_id !== stim.rob_id) + `uvm_fatal("RESP", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + resp_count++; + endfunction + + function f32add_cmd_item current_stim(string tag); + if (stim_q.size() == 0) `uvm_fatal("SCB", $sformatf("%s observed before stimulus", tag)) + return stim_q[0]; + endfunction + + function bit done(); + if (stim_q.size() == 0) return 1'b0; + return cmd_count == 1 && read_count[0] == stim_q[0].iter && + read_count[1] == (stim_q[0].first ? 0 : stim_q[0].iter) && + write_count == stim_q[0].iter && resp_count == 1; + endfunction + + function void reset_counters(); + stim_q.delete(); + cmd_count = 0; + read_count[0] = 0; + read_count[1] = 0; + write_count = 0; + resp_count = 0; + endfunction + + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (!done()) + `uvm_fatal("SCB", $sformatf( + "incomplete: cmds=%0d reads=%0d,%0d writes=%0d resp=%0d", + cmd_count, + read_count[0], + read_count[1], + write_count, + resp_count + )) + stim_q.delete(); + endfunction +endclass diff --git a/examples/prefab/balls/f32add/verify/src/pkg/f32add_pkg.sv b/examples/prefab/balls/f32add/verify/src/pkg/f32add_pkg.sv new file mode 100644 index 000000000..f77a1dd26 --- /dev/null +++ b/examples/prefab/balls/f32add/verify/src/pkg/f32add_pkg.sv @@ -0,0 +1,13 @@ +package f32add_pkg; + import uvm_pkg::*; + import bb_uvm_pkg::*; + `include "uvm_macros.svh" + + `include "common/f32add_defs.svh" + `include "common/f32add_items.svh" + `include "seq/f32add_sequences.svh" + `include "cov/f32add_cov.svh" + `include "env/f32add_scoreboard.svh" + `include "env/f32add_env.svh" + `include "tests/f32add_test.svh" +endpackage diff --git a/examples/prefab/balls/f32add/verify/src/seq/f32add_sequences.svh b/examples/prefab/balls/f32add/verify/src/seq/f32add_sequences.svh new file mode 100644 index 000000000..cc1e21e9a --- /dev/null +++ b/examples/prefab/balls/f32add/verify/src/seq/f32add_sequences.svh @@ -0,0 +1,17 @@ +class f32add_basic_seq extends uvm_sequence #(bb_blink_cmd_item); + `uvm_object_utils(f32add_basic_seq) + + int unsigned case_index; + + function new(string name = "f32add_basic_seq"); + super.new(name); + endfunction + + task body(); + f32add_cmd_item item; + item = f32add_cmd_item::type_id::create("item"); + start_item(item); + item.load_rust_case(case_index); + finish_item(item); + endtask +endclass diff --git a/examples/prefab/balls/f32add/verify/src/tb_top.sv b/examples/prefab/balls/f32add/verify/src/tb_top.sv new file mode 100644 index 000000000..bca5c121c --- /dev/null +++ b/examples/prefab/balls/f32add/verify/src/tb_top.sv @@ -0,0 +1,89 @@ +module tb_top; + import uvm_pkg::*; + import f32add_pkg::*; + + bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); + + F32AddBall dut ( + .clock(intf.clock), + .reset(intf.reset), + .blink_status_running(intf.status_running), + .blink_cmdReq_ready(intf.cmd_req_ready), + .blink_cmdReq_valid(intf.cmd_req_valid), + .blink_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .blink_cmdReq_bits_funct(intf.cmd_req_bits_funct), + .blink_cmdReq_bits_rs2(intf.cmd_req_bits_rs2), + .blink_cmdReq_bits_iter(intf.cmd_req_bits_iter), + .blink_cmdReq_bits_op1_bank(intf.cmd_req_bits_op1_bank), + .blink_cmdReq_bits_op2_bank(intf.cmd_req_bits_op2_bank), + .blink_cmdReq_bits_wr_bank(intf.cmd_req_bits_wr_bank), + .blink_cmdReq_bits_op1_group(intf.cmd_req_bits_op1_group), + .blink_cmdReq_bits_op2_group(intf.cmd_req_bits_op2_group), + .blink_cmdReq_bits_wr_group(intf.cmd_req_bits_wr_group), + .blink_cmdReq_bits_op1_col(intf.cmd_req_bits_op1_col), + .blink_cmdReq_bits_op2_col(intf.cmd_req_bits_op2_col), + .blink_cmdReq_bits_wr_col(intf.cmd_req_bits_wr_col), + .blink_cmdReq_bits_wr_read_bank(intf.cmd_req_bits_wr_read_bank), + .blink_cmdReq_bits_wr_arch_bank(intf.cmd_req_bits_wr_arch_bank), + .blink_cmdResp_ready(intf.cmd_resp_ready), + .blink_cmdResp_valid(intf.cmd_resp_valid), + .blink_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .blink_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .blink_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .blink_bankRead_0_group_id(intf.bank_read_group_id[0]), + .blink_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .blink_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .blink_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .blink_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .blink_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .blink_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .blink_bankRead_1_bank_id(intf.bank_read_bank_id[1]), + .blink_bankRead_1_rob_id(intf.bank_read_rob_id[1]), + .blink_bankRead_1_group_id(intf.bank_read_group_id[1]), + .blink_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), + .blink_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), + .blink_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), + .blink_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), + .blink_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), + .blink_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), + .blink_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .blink_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .blink_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .blink_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .blink_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .blink_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .blink_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .blink_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .blink_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .blink_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]) + ); + + // Constant outputs removed from the port list by firtool. + assign intf.bank_write_req_mask[0][0] = dut.blink_bankWrite_0_io_req_bits_mask_0; + assign intf.bank_write_req_mask[0][1] = dut.blink_bankWrite_0_io_req_bits_mask_1; + assign intf.bank_write_req_mask[0][2] = dut.blink_bankWrite_0_io_req_bits_mask_2; + assign intf.bank_write_req_mask[0][3] = dut.blink_bankWrite_0_io_req_bits_mask_3; + assign intf.bank_write_req_mask[0][4] = dut.blink_bankWrite_0_io_req_bits_mask_4; + assign intf.bank_write_req_mask[0][5] = dut.blink_bankWrite_0_io_req_bits_mask_5; + assign intf.bank_write_req_mask[0][6] = dut.blink_bankWrite_0_io_req_bits_mask_6; + assign intf.bank_write_req_mask[0][7] = dut.blink_bankWrite_0_io_req_bits_mask_7; + assign intf.bank_write_req_mask[0][8] = dut.blink_bankWrite_0_io_req_bits_mask_8; + assign intf.bank_write_req_mask[0][9] = dut.blink_bankWrite_0_io_req_bits_mask_9; + assign intf.bank_write_req_mask[0][10] = dut.blink_bankWrite_0_io_req_bits_mask_10; + assign intf.bank_write_req_mask[0][11] = dut.blink_bankWrite_0_io_req_bits_mask_11; + assign intf.bank_write_req_mask[0][12] = dut.blink_bankWrite_0_io_req_bits_mask_12; + assign intf.bank_write_req_mask[0][13] = dut.blink_bankWrite_0_io_req_bits_mask_13; + assign intf.bank_write_req_mask[0][14] = dut.blink_bankWrite_0_io_req_bits_mask_14; + assign intf.bank_write_req_mask[0][15] = dut.blink_bankWrite_0_io_req_bits_mask_15; + + initial begin + intf.clock = 1'b0; + intf.reset = 1'b1; + forever #5 intf.clock = ~intf.clock; + end + + initial begin + uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); + run_test("f32add_ball_test"); + end +endmodule diff --git a/examples/prefab/balls/f32add/verify/src/tests/f32add_test.svh b/examples/prefab/balls/f32add/verify/src/tests/f32add_test.svh new file mode 100644 index 000000000..58a00a98c --- /dev/null +++ b/examples/prefab/balls/f32add/verify/src/tests/f32add_test.svh @@ -0,0 +1,49 @@ +class f32add_ball_test extends uvm_test; + `uvm_component_utils(f32add_ball_test) + + typedef virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) vif_t; + vif_t vif; + f32add_env env; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(vif_t)::get(this, "", "vif", vif)) + `uvm_fatal("NOVIF", "bb_blink_if not found") + env = f32add_env::type_id::create("env", this); + endfunction + + task run_phase(uvm_phase phase); + phase.raise_objection(this); + for (int unsigned index = 0; index < F32ADD_NUM_CASES; index++) run_case(index); + phase.drop_objection(this); + endtask + + task run_case(int unsigned index); + f32add_basic_seq seq; + int cycles = 0; + apply_reset(); + env.scb.reset_counters(); + seq = f32add_basic_seq::type_id::create("seq"); + seq.case_index = index; + seq.start(env.cmd_agent.seqr); + while (!env.scb.done()) begin + @(posedge vif.clock); + cycles++; + if (cycles > F32ADD_TIMEOUT_CYCLES) + `uvm_fatal("TIMEOUT", $sformatf("F32AddBall case %0d timeout", index)) + end + `uvm_info("F32ADD", $sformatf("F32AddBall case %0d passed in %0d cycles", index, cycles), + UVM_LOW) + endtask + + task apply_reset(); + vif.reset = 1'b1; + repeat (5) @(posedge vif.clock); + vif.reset = 1'b0; + repeat (2) @(posedge vif.clock); + endtask +endclass diff --git a/examples/prefab/balls/f32add/workloads/ctests/CMakeLists.txt b/examples/prefab/balls/f32add/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..d51d3c3a5 --- /dev/null +++ b/examples/prefab/balls/f32add/workloads/ctests/CMakeLists.txt @@ -0,0 +1 @@ +add_buckyball_ctest(f32add_groups.c) diff --git a/examples/balls/f32add/workloads/ctests/f32add_groups.c b/examples/prefab/balls/f32add/workloads/ctests/f32add_groups.c similarity index 95% rename from examples/balls/f32add/workloads/ctests/f32add_groups.c rename to examples/prefab/balls/f32add/workloads/ctests/f32add_groups.c index 7c19fd1ce..baeee1be2 100644 --- a/examples/balls/f32add/workloads/ctests/f32add_groups.c +++ b/examples/prefab/balls/f32add/workloads/ctests/f32add_groups.c @@ -1,6 +1,5 @@ #include #include -#include #include #include static const uint32_t inputs[3][8] __attribute__((aligned(64))) = { @@ -25,7 +24,7 @@ static uint32_t add(uint32_t a, uint32_t b) { return result.bits; } int main(void) { - unsigned source = BB_SHARED_BANK_BASE, accumulator = source + 1, + unsigned source = THREAD_FRONTEND_SHARED_VBANK_BASE, accumulator = source + 1, target = source + 2; bb_mem_alloc(source, 1, 3); bb_mem_alloc(accumulator, 1, 1); diff --git a/examples/prefab/balls/f32add/workloads/isa/f32add.h b/examples/prefab/balls/f32add/workloads/isa/f32add.h new file mode 100644 index 000000000..988b0caac --- /dev/null +++ b/examples/prefab/balls/f32add/workloads/isa/f32add.h @@ -0,0 +1,16 @@ +#ifndef F32ADD_H +#define F32ADD_H +#include +#include + +// Four IEEE FP32 elements per 128-bit row, RNE, canonical NaN. +// rs2[4:0] selects the input group; bit 5 starts with +0 instead of reading b. +static inline void bb_f32add(uint32_t a, uint32_t b, uint32_t c, uint32_t rows, + uint32_t group, uint32_t first) { + if (a > 1023 || b > 1023 || c > 1023 || !rows || group > 31 || first > 1) + __builtin_trap(); + BUCKYBALL_INSTRUCTION_R_R( + BB_BANK0(a) | BB_BANK1(b) | BB_BANK2(c) | BB_ITER(rows), + group | (uint64_t)first << 5, FUNCT7(f32add, F32ADD)); +} +#endif diff --git a/examples/balls/f32add/workloads/mlir_tests/CMakeLists.txt b/examples/prefab/balls/f32add/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/balls/f32add/workloads/mlir_tests/CMakeLists.txt rename to examples/prefab/balls/f32add/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/balls/f32add/workloads/mlir_tests/ball/CMakeLists.txt b/examples/prefab/balls/f32add/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/balls/f32add/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/prefab/balls/f32add/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/prefab/balls/f32add/workloads/mlir_tests/ball/f32add_groups.mlir b/examples/prefab/balls/f32add/workloads/mlir_tests/ball/f32add_groups.mlir new file mode 100644 index 000000000..4ebe35884 --- /dev/null +++ b/examples/prefab/balls/f32add/workloads/mlir_tests/ball/f32add_groups.mlir @@ -0,0 +1,16 @@ +// RUN: buddy-opt %s -lower-buckyball -canonicalize | FileCheck %s +// CHECK-LABEL: func.func @reduce_groups +// CHECK-DAG: %[[F76:.*]] = llvm.mlir.constant(76 : i32) : i32 +// CHECK-DAG: %[[AB:[^ ]+]] = arith.constant 2183169056 : i64 +// CHECK-DAG: %[[DC:[^ ]+]] = arith.constant 2182121507 : i64 +// CHECK-DAG: %[[ZERO:[^ ]+]] = arith.constant 0 : i64 +// CHECK: llvm.call_intrinsic "llvm.riscv.buckyball.custom"(%[[AB]], %[[ZERO]], %[[F76]]) +// CHECK: llvm.call_intrinsic "llvm.riscv.buckyball.custom"(%[[DC]], %[[ZERO]], %[[F76]]) +// The banks 32 (A), 33 (B) and 35 (D) are loaded by the caller: 34 = A + B, +// then 33 = D + 34. +func.func @reduce_groups() -> i64 attributes {llvm.emit_c_interface} { + %output = arith.constant 33 : i64 + f32add_ball.add_f32 <{a = 32 : i64, b = 33 : i64, c = 34 : i64, rows = 2 : i64}> + f32add_ball.add_f32 <{a = 35 : i64, b = 34 : i64, c = 33 : i64, rows = 2 : i64}> + return %output : i64 +} diff --git a/examples/prefab/balls/f32add/workloads/mlir_tests/ball/f32add_groups_main.cpp b/examples/prefab/balls/f32add/workloads/mlir_tests/ball/f32add_groups_main.cpp new file mode 100644 index 000000000..17161e216 --- /dev/null +++ b/examples/prefab/balls/f32add/workloads/mlir_tests/ball/f32add_groups_main.cpp @@ -0,0 +1,53 @@ +#include +#include +#include + +extern "C" int64_t _mlir_ciface_reduce_groups(); +static_assert(THREAD_FRONTEND_SHARED_VBANK_BASE == 32); + +static int checkF32Add() { + alignas(64) static const uint32_t input[3][8] = { + {0x3f800000, 0xc0200000, 0x7f800000, 0xff800000, 0, 0x80000000, + 0x7f7fffff, 1}, + {0x40000000, 0x40400000, 0xff800000, 0x7f800000, 0x80000000, 0, + 0xff7fffff, 1}, + {0xbf800000, 0x3f000000, 0x40a00000, 0xc0a00000, 0, 0x80000000, + 0x3f800000, 0x80000001}}; + alignas(64) static uint32_t output[8]; + bb_mem_alloc(32, 1, 1); + bb_mem_alloc(33, 1, 1); + bb_mem_alloc(34, 1, 1); + bb_mem_alloc(35, 1, 1); + bb_mvin((uintptr_t)input[0], 32, 2, 1); + bb_mvin((uintptr_t)input[1], 33, 2, 1); + bb_mvin((uintptr_t)input[2], 35, 2, 1); + if (_mlir_ciface_reduce_groups() != 33) + return 1; + bb_mvout((uintptr_t)output, 33, 2, 1); + for (unsigned i = 0; i < 8; ++i) { + union { + uint32_t bits; + float value; + } expected = {input[0][i]}, source; + for (unsigned group = 1; group < 3; ++group) { + source.bits = input[group][i]; + asm volatile("fadd.s %0, %1, %2, rne" + : "=f"(expected.value) + : "f"(expected.value), "f"(source.value)); + } + if (output[i] != expected.bits) + return 2; + } + bb_mem_release(32); + bb_mem_release(33); + bb_mem_release(34); + bb_mem_release(35); + return 0; +} + +int main() { + int result = checkF32Add(); + if (!result) + puts("F32ADD Ball MLIR RNE/NaN/ordered sum PASS"); + return result; +} diff --git a/examples/prefab/balls/f32add/workloads/mlir_tests/ball/f32add_invalid.mlir b/examples/prefab/balls/f32add/workloads/mlir_tests/ball/f32add_invalid.mlir new file mode 100644 index 000000000..9d266db88 --- /dev/null +++ b/examples/prefab/balls/f32add/workloads/mlir_tests/ball/f32add_invalid.mlir @@ -0,0 +1,18 @@ +// RUN: buddy-opt %s -split-input-file -lower-buckyball -verify-diagnostics +func.func @zero_rows() { + // expected-error @+1 {{add_f32 requires distinct A, B and C banks and rows within a bank}} + f32add_ball.add_f32 <{a = 32 : i64, b = 33 : i64, c = 34 : i64, rows = 0 : i64}> + return +} +// ----- +func.func @wide_rows() { + // expected-error @+1 {{add_f32 requires distinct A, B and C banks and rows within a bank}} + f32add_ball.add_f32 <{a = 32 : i64, b = 33 : i64, c = 34 : i64, rows = 17179869184 : i64}> + return +} +// ----- +func.func @equal_banks() { + // expected-error @+1 {{add_f32 requires distinct A, B and C banks and rows within a bank}} + f32add_ball.add_f32 <{a = 32 : i64, b = 33 : i64, c = 32 : i64, rows = 1 : i64}> + return +} diff --git a/examples/balls/f32add/workloads/mlir_tests/bank/CMakeLists.txt b/examples/prefab/balls/f32add/workloads/mlir_tests/bank/CMakeLists.txt similarity index 100% rename from examples/balls/f32add/workloads/mlir_tests/bank/CMakeLists.txt rename to examples/prefab/balls/f32add/workloads/mlir_tests/bank/CMakeLists.txt diff --git a/examples/prefab/balls/f32add/workloads/mlir_tests/bank/f32add_groups.mlir b/examples/prefab/balls/f32add/workloads/mlir_tests/bank/f32add_groups.mlir new file mode 100644 index 000000000..2a2a662ed --- /dev/null +++ b/examples/prefab/balls/f32add/workloads/mlir_tests/bank/f32add_groups.mlir @@ -0,0 +1,24 @@ +// A three-term sum as a chain of add_f32 into fresh banks: the first sum +// lands in a bank holding poison, which it overwrites, and the second reuses +// a released source bank. +func.func @reduce_groups(%first: memref<2x4xi32>, %second: memref<2x4xi32>, + %last: memref<2x4xi32>, %poison: memref<2x4xi32>, + %output: memref<2x4xi32>) + attributes {llvm.emit_c_interface} { + %a = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %b = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %sum = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %la = buckyball.bank_mvin %first, %a <{depth = 2 : i64, stride = 1 : i64}> : memref<2x4xi32> + %lb = buckyball.bank_mvin %second, %b <{depth = 2 : i64, stride = 1 : i64}> : memref<2x4xi32> + %poisoned = buckyball.bank_mvin %poison, %sum <{depth = 2 : i64, stride = 1 : i64}> : memref<2x4xi32> + %partial = f32add_ball.bank_add_f32 %la, %lb -> %poisoned <{rows = 2 : i64}> + buckyball.bank_release %la + %d = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %ld = buckyball.bank_mvin %last, %d <{depth = 2 : i64, stride = 1 : i64}> : memref<2x4xi32> + %total = f32add_ball.bank_add_f32 %ld, %partial -> %lb <{rows = 2 : i64}> + %stored = buckyball.bank_mvout %output, %total <{depth = 2 : i64, stride = 1 : i64}> : memref<2x4xi32> + buckyball.bank_release %ld + buckyball.bank_release %partial + buckyball.bank_release %stored + return +} diff --git a/examples/prefab/balls/f32add/workloads/mlir_tests/bank/f32add_groups_main.cpp b/examples/prefab/balls/f32add/workloads/mlir_tests/bank/f32add_groups_main.cpp new file mode 100644 index 000000000..a83a0bb2c --- /dev/null +++ b/examples/prefab/balls/f32add/workloads/mlir_tests/bank/f32add_groups_main.cpp @@ -0,0 +1,53 @@ +#include +#include +#include + +using Rows = StridedMemRefType; +extern "C" void _mlir_ciface_reduce_groups(Rows *, Rows *, Rows *, Rows *, + Rows *); + +static int checkF32Add() { + alignas(64) static const uint32_t input[3][8] = { + {0x3f800000, 0xc0200000, 0x7f800000, 0xff800000, 0, 0x80000000, + 0x7f7fffff, 1}, + {0x40000000, 0x40400000, 0xff800000, 0x7f800000, 0x80000000, 0, + 0xff7fffff, 1}, + {0xbf800000, 0x3f000000, 0x40a00000, 0xc0a00000, 0, 0x80000000, + 0x3f800000, 0x80000001}}; + alignas(64) static const uint32_t poison[8] = { + 0x7fa00001, 0x7fa00001, 0x7fa00001, 0x7fa00001, + 0x7fa00001, 0x7fa00001, 0x7fa00001, 0x7fa00001}; + alignas(64) static uint32_t output[8]; + Rows firstRef = { + (uint32_t *)input[0], (uint32_t *)input[0], 0, {2, 4}, {4, 1}}; + Rows secondRef = { + (uint32_t *)input[1], (uint32_t *)input[1], 0, {2, 4}, {4, 1}}; + Rows lastRef = { + (uint32_t *)input[2], (uint32_t *)input[2], 0, {2, 4}, {4, 1}}; + Rows poisonRef = {(uint32_t *)poison, (uint32_t *)poison, 0, {2, 4}, {4, 1}}; + Rows outputRef = {output, output, 0, {2, 4}, {4, 1}}; + _mlir_ciface_reduce_groups(&firstRef, &secondRef, &lastRef, &poisonRef, + &outputRef); + for (unsigned i = 0; i < 8; ++i) { + union { + uint32_t bits; + float value; + } expected = {input[0][i]}, source; + for (unsigned group = 1; group < 3; ++group) { + source.bits = input[group][i]; + asm volatile("fadd.s %0, %1, %2, rne" + : "=f"(expected.value) + : "f"(expected.value), "f"(source.value)); + } + if (output[i] != expected.bits) + return 2; + } + return 0; +} + +int main() { + int result = checkF32Add(); + if (!result) + puts("F32ADD Bank MLIR chained RNE/NaN/SSA output PASS"); + return result; +} diff --git a/examples/prefab/balls/f32add/workloads/mlir_tests/bank/f32add_invalid.mlir b/examples/prefab/balls/f32add/workloads/mlir_tests/bank/f32add_invalid.mlir new file mode 100644 index 000000000..d301fbb93 --- /dev/null +++ b/examples/prefab/balls/f32add/workloads/mlir_tests/bank/f32add_invalid.mlir @@ -0,0 +1,24 @@ +// RUN: buddy-opt %s -split-input-file -assign-physical-banks -lower-buckyball -verify-diagnostics +func.func @same_bank(%input: memref<1x4xi32>) { + %a = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %b = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %la = buckyball.bank_mvin %input, %a <{depth = 1 : i64, stride = 1 : i64}> : memref<1x4xi32> + // expected-error @+1 {{add_f32 requires distinct A, B and C banks and rows within a bank}} + %out = f32add_ball.bank_add_f32 %la, %b -> %b <{rows = 1 : i64}> + buckyball.bank_release %la + buckyball.bank_release %out + return +} +// ----- +func.func @zero_rows(%input: memref<1x4xi32>) { + %a = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %b = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %c = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %la = buckyball.bank_mvin %input, %a <{depth = 1 : i64, stride = 1 : i64}> : memref<1x4xi32> + // expected-error @+1 {{add_f32 requires distinct A, B and C banks and rows within a bank}} + %out = f32add_ball.bank_add_f32 %la, %b -> %c <{rows = 0 : i64}> + buckyball.bank_release %la + buckyball.bank_release %b + buckyball.bank_release %out + return +} diff --git a/examples/prefab/balls/gemmini/arch/GemminiBall.scala b/examples/prefab/balls/gemmini/arch/GemminiBall.scala new file mode 100644 index 000000000..b5ceea348 --- /dev/null +++ b/examples/prefab/balls/gemmini/arch/GemminiBall.scala @@ -0,0 +1,87 @@ +package examples.balls.gemmini + +import framework.acc.balldomain.bbus.{BallFactory, BallPorts} +import framework.acc.balldomain.sub_rob.{Expansion, HasExpansion} +import buckyball.balls.gemmini.{Params => ParametersProto} +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import examples.balls.gemmini.configs.GemminiBallParam + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import chisel3.util._ +import framework.acc.balldomain.ball.blink.{BallStatus, BlinkIO, HasBlink} + +@instantiable +class GemminiBall( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Module + with HasBlink + with HasExpansion { + val ball = bp.balls.find(_.id == id).get + + val ballParam = GemminiBallParam.fromProto(ball.parameters.unpack(classOf[ParametersProto])) + + require( + mp.bankEntries <= 1024, + "GemminiBall rs2 row-base fields support at most 1024 bank entries" + ) + + @public + val blink = IO(new BlinkIO(fp, mp, bp, id)) + + @public + val expansion = IO(new Expansion(fp, mp, bp)) + + val loop: Instance[Loop] = Instantiate(new Loop(fp, mp, bp, id)) + expansion <> loop.expansion + + val execution: Instance[GemminiExCtrl] = + Instantiate(new GemminiExCtrl(fp, mp, bp, ball, ballParam)) + val robId = RegInit(0.U(log2Up(fp.rob_entries).W)) + when(blink.cmdReq.fire)(robId := blink.cmdReq.bits.rob_id) + + execution.exio.cmdReq <> blink.cmdReq + blink.cmdResp <> execution.exio.cmdResp + + for (port <- 0 until ball.inBW) { + blink.bankRead(port).io.req <> execution.exio.bankReadReq(port) + execution.exio.bankReadResp(port) <> blink.bankRead(port).io.resp + blink.bankRead(port).rob_id := robId + blink.bankRead(port).group_id := 0.U + } + blink.bankRead(0).bank_id := execution.exio.op1_bank_o + blink.bankRead(0).group_id := execution.exio.op1_group_o + if (ball.inBW > 1) { + blink.bankRead(1).bank_id := execution.exio.op2_bank_o + blink.bankRead(1).group_id := execution.exio.op2_group_o + } + + for (port <- 0 until ball.outBW) { + blink.bankWrite(port).io <> execution.exio.bankWrite(port) + blink.bankWrite(port).bank_id := execution.exio.wr_bank_o + blink.bankWrite(port).rob_id := robId + blink.bankWrite(port).group_id := execution.exio.wr_group_o + port.U + } + + blink.status <> execution.exio.status +} + +object GemminiBall extends BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts = { + val ball = Instantiate(new GemminiBall(fp, mp, bp, id)) + BallPorts(ball.blink, Some(ball.expansion)) + } + +} diff --git a/examples/prefab/balls/gemmini/arch/GemminiExCtrl.scala b/examples/prefab/balls/gemmini/arch/GemminiExCtrl.scala new file mode 100644 index 000000000..d5adba416 --- /dev/null +++ b/examples/prefab/balls/gemmini/arch/GemminiExCtrl.scala @@ -0,0 +1,40 @@ +package examples.balls.gemmini + +import framework.acc.balldomain.configs.BallRouteParam + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import examples.balls.gemmini.configs.GemminiBallParam + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public} + +@instantiable +class GemminiExCtrl( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val routingParam: BallDomainParam, + val instance: BallRouteParam, + val ballParam: GemminiBallParam) + extends Module + with GemminiExCtrlDefs + with GemminiExCtrlDefaults + with GemminiExCtrlCmdStates + with GemminiExCtrlPreloadStates + with GemminiExCtrlComputeReadState + with GemminiExCtrlComputeFeedState + with GemminiExCtrlStoreOps + with GemminiExCtrlAccumulateState + with GemminiExCtrlFsm { + def commandDataBits: Int = frontendParam.commandDataBits + @public + val exio = io + + applyDefaults() + runFsm() + + io.status.idle := state === sIdle + io.status.running := state =/= sIdle +} diff --git a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlAccumulateState.scala b/examples/prefab/balls/gemmini/arch/GemminiExCtrlAccumulateState.scala similarity index 82% rename from examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlAccumulateState.scala rename to examples/prefab/balls/gemmini/arch/GemminiExCtrlAccumulateState.scala index f6ae22c8c..57919e955 100644 --- a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlAccumulateState.scala +++ b/examples/prefab/balls/gemmini/arch/GemminiExCtrlAccumulateState.scala @@ -1,5 +1,9 @@ package examples.balls.gemmini +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + import chisel3._ import chisel3.util._ @@ -15,7 +19,7 @@ trait GemminiExCtrlAccumulateState { this: GemminiExCtrl => }.otherwise { rdQueue0.io.deq.ready := true.B when(rdQueue0.io.deq.fire) { - val elemsPerPort = b.memDomain.bankWidth / config.accWidth + val elemsPerPort = memParam.bankWidth / config.accWidth val previous = rdQueue0.io.deq.bits.data.asTypeOf(Vec(elemsPerPort, accType)) for ((value, index) <- outBuf(store_row_cnt).flatten.zipWithIndex) { when(acc_read_group === (index / elemsPerPort).U) { diff --git a/examples/prefab/balls/gemmini/arch/GemminiExCtrlCmdStates.scala b/examples/prefab/balls/gemmini/arch/GemminiExCtrlCmdStates.scala new file mode 100644 index 000000000..75528da5a --- /dev/null +++ b/examples/prefab/balls/gemmini/arch/GemminiExCtrlCmdStates.scala @@ -0,0 +1,118 @@ +package examples.balls.gemmini + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import gemmini._ + +/** Sub-command encoding within the special field */ +object GemminiSubCmd { + val CONFIG = 0.U(4.W) + val PRELOAD = 1.U(4.W) + val COMPUTE_PRELOADED = 2.U(4.W) + val COMPUTE_ACCUMULATED = 3.U(4.W) + val FLUSH = 4.U(4.W) +} + +trait GemminiExCtrlCmdStates { this: GemminiExCtrl => + + protected def handleIdleState(): Unit = { + io.cmdReq.ready := true.B + when(io.cmdReq.fire) { + rob_id_reg := io.cmdReq.bits.rob_id + op1_bank := io.cmdReq.bits.op1_bank + op2_bank := io.cmdReq.bits.op2_bank + wr_bank := io.cmdReq.bits.wr_bank + wr_read_bank := io.cmdReq.bits.wr_read_bank + // rs2[41:36] and rs2[47:42] select a group of op1 and op2 within their banks. + op1_group := io.cmdReq.bits.op1_group + io.cmdReq.bits.rs2(41, 36) + op2_group := io.cmdReq.bits.op2_group + io.cmdReq.bits.rs2(47, 42) + wr_group := io.cmdReq.bits.wr_group + op1_base := io.cmdReq.bits.rs2(15, 6) + op2_base := io.cmdReq.bits.rs2(25, 16) + wr_base := io.cmdReq.bits.rs2(35, 26) + total_rows := Mux( + io.cmdReq.bits.iter === 0.U, + DIM.U, + io.cmdReq.bits.iter + ) + zero_op2 := io.cmdReq.bits.rs2(4) + zero_op1_tail := io.cmdReq.bits.rs2(5) + accumulated_cmd := sub_cmd === GemminiSubCmd.COMPUTE_ACCUMULATED + reset_acc_mesh := sub_cmd === GemminiSubCmd.COMPUTE_ACCUMULATED + acc_read_group := 0.U + acc_read_pending := false.B + + when(sub_cmd === GemminiSubCmd.CONFIG) { + cfg_dataflow := io.cmdReq.bits.rs2(4) + cfg_a_transpose := io.cmdReq.bits.rs2(7) + cfg_bd_transpose := io.cmdReq.bits.rs2(8) + cfg_in_shift := io.cmdReq.bits.rs2( + log2Up(config.accWidth) + 8, + 9 + ) + state := sCommit + }.elsewhen(sub_cmd === GemminiSubCmd.PRELOAD) { + assert(io.cmdReq.bits.rs2(41, 36) < io.cmdReq.bits.op1_col, "Gemmini preload op1 group outside the bank") + read_row_cnt := 0.U + feed_row_cnt := 0.U + req_sent := false.B + xpose_ready := false.B + xpose_row_cnt := 0.U + state := sPreloadRead + }.elsewhen( + sub_cmd === GemminiSubCmd.COMPUTE_PRELOADED || sub_cmd === GemminiSubCmd.COMPUTE_ACCUMULATED + ) { + // A zeroed op2 (rs2[4]) reads no bank, so its group is not checked. + assert( + io.cmdReq.bits.rs2(41, 36) < io.cmdReq.bits.op1_col && + (io.cmdReq.bits.rs2(4) || io.cmdReq.bits.rs2(47, 42) < io.cmdReq.bits.op2_col), + "Gemmini compute operand group outside the bank" + ) + read_row_cnt := 0.U + feed_row_cnt := 0.U + outBufRows := 0.U + outBufCollected := 0.U + req_sent := false.B + xpose_ready := false.B + xpose_row_cnt := 0.U + read_done.foreach(_ := false.B) + state := Mux(sub_cmd === GemminiSubCmd.COMPUTE_ACCUMULATED, sPreloadFeed, sComputeRead) + }.elsewhen(sub_cmd === GemminiSubCmd.FLUSH) { + state := sFlush + } + } + } + + protected def handleCommitState(): Unit = { + io.cmdResp.valid := true.B + when(io.cmdResp.fire) { + state := sIdle + } + } + + protected def handleFlushState(): Unit = { + mesh.io.req.valid := true.B + mesh.io.req.bits.flush := 2.U + mesh.io.req.bits.total_rows := DIM.U + mesh.io.req.bits.tag.rob := robIdAsTag8(rob_id_reg) + + mesh.io.a.valid := true.B + mesh.io.a.bits := 0.U.asTypeOf(mesh.io.a.bits) + mesh.io.b.valid := true.B + mesh.io.b.bits := 0.U.asTypeOf(mesh.io.b.bits) + mesh.io.d.valid := true.B + mesh.io.d.bits := 0.U.asTypeOf(mesh.io.d.bits) + + when(mesh.io.req.ready) { + io.cmdResp.valid := true.B + when(io.cmdResp.fire) { + state := sIdle + } + } + } + +} diff --git a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlComputeFeedState.scala b/examples/prefab/balls/gemmini/arch/GemminiExCtrlComputeFeedState.scala similarity index 92% rename from examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlComputeFeedState.scala rename to examples/prefab/balls/gemmini/arch/GemminiExCtrlComputeFeedState.scala index beaa64853..9fdc73f55 100644 --- a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlComputeFeedState.scala +++ b/examples/prefab/balls/gemmini/arch/GemminiExCtrlComputeFeedState.scala @@ -1,5 +1,9 @@ package examples.balls.gemmini +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + import chisel3._ import chisel3.util._ import gemmini._ @@ -71,7 +75,7 @@ trait GemminiExCtrlComputeFeedState { this: GemminiExCtrl => mesh.io.a.valid := true.B mesh.io.a.bits := Mux( op1Zero, - 0.U.asTypeOf(mesh.A_TYPE), + 0.U.asTypeOf(mesh.io.a.bits), VecInit(a_row.grouped(config.tileRows).map(g => VecInit(g)).toSeq) ) when(execution_dataflow === Dataflow.OS.id.U) { @@ -81,15 +85,15 @@ trait GemminiExCtrlComputeFeedState { this: GemminiExCtrl => x_row.grouped(config.tileColumns).map(g => VecInit(g)).toSeq ) mesh.io.d.valid := true.B - mesh.io.d.bits := 0.U.asTypeOf(mesh.D_TYPE) + mesh.io.d.bits := 0.U.asTypeOf(mesh.io.d.bits) }.otherwise { // WS: stream A/D, B comes from preloaded weights mesh.io.b.valid := true.B - mesh.io.b.bits := 0.U.asTypeOf(mesh.B_TYPE) + mesh.io.b.bits := 0.U.asTypeOf(mesh.io.b.bits) mesh.io.d.valid := true.B mesh.io.d.bits := Mux( zero_op2, - 0.U.asTypeOf(mesh.D_TYPE), + 0.U.asTypeOf(mesh.io.d.bits), VecInit( x_row.grouped(config.tileColumns).map(g => VecInit(g)).toSeq ) diff --git a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlComputeReadState.scala b/examples/prefab/balls/gemmini/arch/GemminiExCtrlComputeReadState.scala similarity index 88% rename from examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlComputeReadState.scala rename to examples/prefab/balls/gemmini/arch/GemminiExCtrlComputeReadState.scala index 66901cd74..743437cfa 100644 --- a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlComputeReadState.scala +++ b/examples/prefab/balls/gemmini/arch/GemminiExCtrlComputeReadState.scala @@ -1,5 +1,9 @@ package examples.balls.gemmini +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + import chisel3._ import chisel3.util._ import gemmini._ diff --git a/examples/prefab/balls/gemmini/arch/GemminiExCtrlDefaults.scala b/examples/prefab/balls/gemmini/arch/GemminiExCtrlDefaults.scala new file mode 100644 index 000000000..5fa1c3b41 --- /dev/null +++ b/examples/prefab/balls/gemmini/arch/GemminiExCtrlDefaults.scala @@ -0,0 +1,45 @@ +package examples.balls.gemmini + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ + +trait GemminiExCtrlDefaults { this: GemminiExCtrl => + + protected def applyDefaults(): Unit = { + io.cmdReq.ready := state === sIdle + + io.cmdResp.valid := false.B + io.cmdResp.bits.rob_id := rob_id_reg + + for (i <- 0 until inBW) { + io.bankReadReq(i).valid := false.B + io.bankReadReq(i).bits.addr := 0.U + } + + rdQueue0.io.deq.ready := false.B + rdQueue1.io.deq.ready := false.B + + mesh.io.a.valid := false.B + mesh.io.a.bits := 0.U.asTypeOf(mesh.io.a.bits) + mesh.io.b.valid := false.B + mesh.io.b.bits := 0.U.asTypeOf(mesh.io.b.bits) + mesh.io.d.valid := false.B + mesh.io.d.bits := 0.U.asTypeOf(mesh.io.d.bits) + mesh.io.req.valid := false.B + mesh.io.req.bits := 0.U.asTypeOf(mesh.io.req.bits) + // mesh.io.resp is Valid-only (no ready); MeshWithDelays assumes immediate observation when valid + + io.bankWrite.foreach { bw => + bw.req.valid := false.B + bw.req.bits.addr := 0.U + bw.req.bits.data := 0.U + bw.req.bits.mask := VecInit(Seq.fill(memParam.bankMaskLen)(false.B)) + bw.resp.ready := true.B + } + } + +} diff --git a/examples/prefab/balls/gemmini/arch/GemminiExCtrlDefs.scala b/examples/prefab/balls/gemmini/arch/GemminiExCtrlDefs.scala new file mode 100644 index 000000000..954717b38 --- /dev/null +++ b/examples/prefab/balls/gemmini/arch/GemminiExCtrlDefs.scala @@ -0,0 +1,150 @@ +package examples.balls.gemmini + +import framework.lib.queue.Queue +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import framework.ip.memory.bank.{SramReadIO, SramReadReq, SramReadResp, SramWriteIO, SramWriteReq, SramWriteResp} + +import chisel3._ +import framework.acc.balldomain.ball.blink.{BallCommand, BallResponse} +import chisel3.util._ +import framework.acc.balldomain.ball.blink.BallStatus +import framework.ip.memory.bank.{SramReadReq, SramReadResp, SramWriteIO} +import examples.balls.gemmini.configs.GemminiBallParam +import gemmini._ +import gemmini.Util._ + +/** Minimal tag for MeshWithDelays that satisfies TagQueueTag */ +class SimpleTag extends Bundle with gemmini.TagQueueTag { + val rob = UInt(8.W) + override def make_this_garbage(dummy: Int = 0): Unit = + rob := 0xff.U +} + +trait GemminiExCtrlDefs { this: GemminiExCtrl => + val config = ballParam + val DIM = config.blockSize + + val ballMapping = instance + + val inBW = ballMapping.inBW + val outBW = ballMapping.outBW + + val inputType = SInt(config.inputWidth.W) + val accType = SInt(config.accWidth.W) + val meshOutputType = SInt(config.spatialOutputWidth.W) + + val ctrlIo = IO(new Bundle { + + val cmdReq = Flipped(Decoupled(new BallCommand(frontendParam, memParam))) + + val cmdResp = Decoupled(new BallResponse(frontendParam)) + val bankReadReq = Vec(inBW, Decoupled(new SramReadReq(memParam))) + val bankReadResp = + Vec(inBW, Flipped(Decoupled(new SramReadResp(memParam)))) + val bankWrite = Vec(outBW, Flipped(new SramWriteIO(memParam))) + val op1_bank_o = Output(UInt(frontendParam.bank_id_len.W)) + val op1_group_o = Output(UInt(memParam.groupIdWidth.W)) + val op2_bank_o = Output(UInt(frontendParam.bank_id_len.W)) + val op2_group_o = Output(UInt(memParam.groupIdWidth.W)) + val wr_bank_o = Output(UInt(frontendParam.bank_id_len.W)) + val wr_group_o = Output(UInt(memParam.groupIdWidth.W)) + val status = new BallStatus + }) + + val io = ctrlIo + + val mesh = Module(new MeshWithDelays( + inputType, + inputType, + meshOutputType, + accType, + new SimpleTag, + Dataflow.BOTH, + false, + config.tileLatency, + config.outputDelay, + config.tileRows, + config.tileColumns, + config.meshRows, + config.meshColumns, + 1, + 1 + )) + + protected def widenMeshToAcc(src: Vec[Vec[SInt]]): Vec[Vec[SInt]] = + VecInit(src.map(col => VecInit(col.map(_.asTypeOf(accType))))) + + val cfg_dataflow = RegInit(0.U(1.W)) + val cfg_in_shift = RegInit(0.U(log2Up(config.accWidth).W)) + val cfg_a_transpose = RegInit(false.B) + val cfg_bd_transpose = RegInit(false.B) + val zero_op2 = RegInit(false.B) + val zero_op1_tail = RegInit(false.B) + val accumulated_cmd = RegInit(false.B) + val reset_acc_mesh = RegInit(false.B) + val acc_read_group = RegInit(0.U(log2Up(outBW).W)) + val acc_read_pending = RegInit(false.B) + val execution_dataflow = Mux(accumulated_cmd, Dataflow.OS.id.U, cfg_dataflow) + val execution_shift = Mux(accumulated_cmd, 0.U, cfg_in_shift) + + val sIdle :: sPreloadRead :: sPreloadFeed :: sComputeRead :: sComputeFeed :: sComputeFlush :: sFlush :: sDrain :: sReadAccum :: sStore :: sCommit :: Nil = + Enum(11) + val state = RegInit(sIdle) + + val rob_id_reg = RegInit(0.U(log2Up(frontendParam.rob_entries).W)) + + /** Match SimpleTag.rob (8b); must match mesh req tag bits */ + protected def robIdAsTag8(x: UInt): UInt = { + val w = x.getWidth + if (w >= 8) x(7, 0) else Cat(0.U((8 - w).W), x) + } + + val op1_bank = RegInit(0.U(frontendParam.bank_id_len.W)) + val op2_bank = RegInit(0.U(frontendParam.bank_id_len.W)) + val wr_bank = RegInit(0.U(frontendParam.bank_id_len.W)) + val wr_read_bank = RegInit(0.U(frontendParam.bank_id_len.W)) + val op1_group = RegInit(0.U(memParam.groupIdWidth.W)) + val op2_group = RegInit(0.U(memParam.groupIdWidth.W)) + val wr_group = RegInit(0.U(memParam.groupIdWidth.W)) + val op1_base = RegInit(0.U(10.W)) + val op2_base = RegInit(0.U(10.W)) + val wr_base = RegInit(0.U(10.W)) + io.op1_bank_o := Mux(state === sReadAccum, wr_read_bank, op1_bank) + io.op1_group_o := Mux(state === sReadAccum, wr_group + acc_read_group, op1_group) + io.op2_bank_o := op2_bank + io.op2_group_o := op2_group + io.wr_bank_o := wr_bank + io.wr_group_o := wr_group + + val read_row_cnt = RegInit(0.U(log2Up(DIM + 1).W)) + val feed_row_cnt = RegInit(0.U(log2Up(DIM + 1).W)) + val store_row_cnt = RegInit(0.U(log2Up(DIM + 1).W)) + val total_rows = RegInit(DIM.U(log2Up(DIM + 1).W)) + val req_sent = RegInit(false.B) + val read_done = RegInit(VecInit(Seq.fill(inBW)(false.B))) + val xpose_ready = RegInit(false.B) + val xpose_row_cnt = RegInit(0.U(log2Up(DIM + 1).W)) + + val sub_cmd = io.cmdReq.bits.rs2(3, 0) + + val rdQueue0 = Instantiate(new Queue(new SramReadResp(memParam), DIM)) + val rdQueue1 = Instantiate(new Queue(new SramReadResp(memParam), DIM)) + rdQueue0.io.enq <> io.bankReadResp(0) + rdQueue1.io.enq <> io.bankReadResp(1) + + val outBuf = Reg( + Vec(DIM, Vec(config.meshColumns, Vec(config.tileColumns, accType))) + ) + + val outBufRows = RegInit(0.U(log2Up(DIM + 1).W)) + val outBufCollected = RegInit(0.U(log2Up(DIM + 1).W)) + val op1Buf = Reg(Vec(DIM, Vec(DIM, inputType))) + val op2Buf = Reg(Vec(DIM, Vec(DIM, inputType))) + + val port_written = RegInit(VecInit(Seq.fill(outBW)(false.B))) +} diff --git a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlFsm.scala b/examples/prefab/balls/gemmini/arch/GemminiExCtrlFsm.scala similarity index 83% rename from examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlFsm.scala rename to examples/prefab/balls/gemmini/arch/GemminiExCtrlFsm.scala index 376fb030b..c8562e68a 100644 --- a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlFsm.scala +++ b/examples/prefab/balls/gemmini/arch/GemminiExCtrlFsm.scala @@ -1,5 +1,9 @@ package examples.balls.gemmini +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + import chisel3._ import chisel3.util._ diff --git a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlPreloadStates.scala b/examples/prefab/balls/gemmini/arch/GemminiExCtrlPreloadStates.scala similarity index 93% rename from examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlPreloadStates.scala rename to examples/prefab/balls/gemmini/arch/GemminiExCtrlPreloadStates.scala index 4361571cc..0bbaa25d8 100644 --- a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlPreloadStates.scala +++ b/examples/prefab/balls/gemmini/arch/GemminiExCtrlPreloadStates.scala @@ -1,5 +1,9 @@ package examples.balls.gemmini +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + import chisel3._ import chisel3.util._ import gemmini._ @@ -85,15 +89,15 @@ trait GemminiExCtrlPreloadStates { this: GemminiExCtrl => VecInit((0 until DIM).map(i => op2Buf(i)(transpose_col))) val weight_data = Mux(explicitBTranspose, transposed_data, row_data) mesh.io.a.valid := true.B - mesh.io.a.bits := 0.U.asTypeOf(mesh.A_TYPE) + mesh.io.a.bits := 0.U.asTypeOf(mesh.io.a.bits) mesh.io.b.valid := true.B - mesh.io.b.bits := 0.U.asTypeOf(mesh.B_TYPE) + mesh.io.b.bits := 0.U.asTypeOf(mesh.io.b.bits) mesh.io.d.valid := true.B // OS preload in Buckyball is used to prime pipeline state before compute. // Feed D=0 to avoid injecting bias-like data into the following matmul. mesh.io.d.bits := Mux( execution_dataflow === Dataflow.OS.id.U, - 0.U.asTypeOf(mesh.D_TYPE), + 0.U.asTypeOf(mesh.io.d.bits), VecInit( weight_data.grouped(config.tileColumns).map(g => VecInit(g)).toSeq ) diff --git a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlStoreOps.scala b/examples/prefab/balls/gemmini/arch/GemminiExCtrlStoreOps.scala similarity index 94% rename from examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlStoreOps.scala rename to examples/prefab/balls/gemmini/arch/GemminiExCtrlStoreOps.scala index 975d2ed5a..f9e1af016 100644 --- a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlStoreOps.scala +++ b/examples/prefab/balls/gemmini/arch/GemminiExCtrlStoreOps.scala @@ -1,5 +1,9 @@ package examples.balls.gemmini +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + import chisel3._ import chisel3.util._ import gemmini._ @@ -95,7 +99,7 @@ trait GemminiExCtrlStoreOps { this: GemminiExCtrl => .reverse ) - val bitsPerPort = b.memDomain.bankWidth + val bitsPerPort = memParam.bankWidth for (i <- 0 until outBW) { when(!port_written(i)) { @@ -104,7 +108,7 @@ trait GemminiExCtrlStoreOps { this: GemminiExCtrl => io.bankWrite(i).req.bits.addr := wr_base + store_row_cnt io.bankWrite(i).req.bits.data := slice io.bankWrite(i).req.bits.mask := VecInit( - Seq.fill(b.memDomain.bankMaskLen)(true.B) + Seq.fill(memParam.bankMaskLen)(true.B) ) when(io.bankWrite(i).req.ready) { port_written(i) := true.B diff --git a/examples/balls/gemmini/arch/src/main/scala/LICENSE.gemmini b/examples/prefab/balls/gemmini/arch/LICENSE.gemmini similarity index 100% rename from examples/balls/gemmini/arch/src/main/scala/LICENSE.gemmini rename to examples/prefab/balls/gemmini/arch/LICENSE.gemmini diff --git a/examples/prefab/balls/gemmini/arch/Loop.scala b/examples/prefab/balls/gemmini/arch/Loop.scala new file mode 100644 index 000000000..d21e0feba --- /dev/null +++ b/examples/prefab/balls/gemmini/arch/Loop.scala @@ -0,0 +1,173 @@ +package examples.balls.gemmini + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import examples.balls.gemmini.configs.GemminiBallParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.balldomain.sub_rob.{Expansion, HasExpansion} +import framework.acc.frontend.decoder.DomainId + +/** Loop configuration lives in BallDomain's metadata SRAM, not in a second register file. */ +@instantiable +class Loop( + f: FrontendParam, + m: MemDomainParam, + routing: BallDomainParam, + id: Int) + extends Module + with HasExpansion { + val ball = routing.balls.find(_.id == id).get + val p = GemminiBallParam.fromProto(ball.parameters.unpack(classOf[buckyball.balls.gemmini.Params])) + require(m.mmioTotalBytes >= 136) + def funct(name: String): UInt = ball.instructions.find(_.mnemonic == name).get.funct7.U(7.W) + val ws = Seq("BOUNDS", "ADDR_A", "ADDR_B", "ADDR_D", "ADDR_C", "STRIDES_AB", "STRIDES_DC") + .map(n => funct("GEMMINI_LOOP_WS_CONFIG_" + n)) + val conv = (1 to 9).map(n => funct(s"GEMMINI_LOOP_CONV_WS_CONFIG_$n")) + @public + val expansion = IO(new Expansion(f, m, routing)) + val idle :: storing :: configured :: reading :: receiving :: starting :: streaming :: Nil = Enum(7) + val state = RegInit(idle) + val master = Reg(UInt(log2Ceil(f.rob_entries).W)) + val operand = Reg(UInt(64.W)) + val byte = RegInit(0.U(7.W)) + val base = Reg(UInt(8.W)) + val convolution = Reg(Bool()) + val values = Reg(Vec(9, UInt(64.W))) + val fn = expansion.command.bits.funct + val wsConfig = VecInit(ws.map(fn === _)) + val convConfig = VecInit(conv.map(fn === _)) + val triggerWs = fn === funct("GEMMINI_LOOP_WS") + val triggerConv = fn === funct("GEMMINI_LOOP_CONV_WS") + expansion.accepts := wsConfig.asUInt.orR || convConfig.asUInt.orR || triggerWs || triggerConv + expansion.starts := triggerWs || triggerConv + expansion.command.ready := state === idle + when(expansion.command.fire) { + master := expansion.command.bits.rob_id + operand := expansion.command.bits.rs2 + byte := 0.U + convolution := triggerConv + when(expansion.starts) { + base := Mux(triggerConv, 64.U, 0.U) + state := reading + }.otherwise { + base := Mux(wsConfig.asUInt.orR, PriorityEncoder(wsConfig) << 3, 64.U + (PriorityEncoder(convConfig) << 3)) + state := storing + } + } + expansion.write.valid := state === storing + expansion.write.bits.addr := base + byte + expansion.write.bits.data := (operand >> (byte << 3))(7, 0) + when(expansion.write.fire) { + byte := byte + 1.U + when(byte === 7.U)(state := configured) + } + expansion.complete.valid := state === configured + expansion.complete.bits.rob_id := master + when(expansion.complete.fire)(state := idle) + expansion.read.req.valid := state === reading + expansion.read.req.bits.addr := base + byte + when(expansion.read.req.fire)(state := receiving) + expansion.read.resp.ready := state === receiving + when(expansion.read.resp.fire) { + val index = byte(6, 3) + values(index) := (values(index) >> 8) | (expansion.read.resp.bits.data.pad(64) << 56) + byte := byte + 1.U + state := Mux(byte === Mux(convolution, 71.U, 55.U), starting, reading) + } + val matmul: Instance[LoopMatmulUnroller] = Instantiate(new LoopMatmulUnroller(f, m, p)) + val convolve: Instance[LoopConvUnroller] = Instantiate(new LoopConvUnroller(f, m, p)) + matmul.io.start.valid := state === starting && !convolution + val mat = matmul.io.start.bits + mat.max_i := values(0)(47, 32); mat.max_j := values(0)(31, 16); mat.max_k := values(0)(15, 0) + mat.dram_addr_a := values(1); mat.dram_addr_b := values(2); mat.dram_addr_d := values(3); mat.dram_addr_c := values(4) + mat.stride_a := values(5)(31, 0); mat.stride_b := values(5)(63, 32) + mat.stride_d := values(6)(31, 0); mat.stride_c := values(6)(63, 32) + mat.bank_a := operand(9, 0); mat.bank_b := operand(19, 10); mat.bank_c := operand(29, 20); mat.low_d := operand(30) + convolve.io.start.valid := state === starting && convolution + val cv = convolve.io.start.bits + cv.batch_size := values(0)(15, 0); cv.in_dim := values(0)(31, 16); cv.in_channels := values(0)(47, 32) + cv.out_channels := values(1)(15, 0); cv.out_dim := values(1)(31, 16); cv.stride := values(1)(39, 32); + cv.padding := values(1)(47, 40) + cv.kernel_dim := values(2)(7, 0); cv.pool_size := values(2)(15, 8); cv.pool_stride := values(2)(23, 16); + cv.pool_padding := values(2)(31, 24) + cv.dram_addr_bias := values(3); cv.dram_addr_input := values(4); cv.dram_addr_weight := values(5); + cv.dram_addr_output := values(6) + cv.input_stride := values(7)(31, 0); cv.weight_stride := values(7)(63, 32); cv.output_stride := values(8)(31, 0) + cv.bank_input := operand(9, 0); cv.bank_weight := operand(19, 10); cv.bank_output := operand(29, 20); + cv.no_bias := operand(30) + when(matmul.io.start.fire || convolve.io.start.fire)(state := streaming) + val commands = Wire(Decoupled(new LoopCmd(f, m, p))) + commands.valid := Mux(convolution, convolve.io.cmd.valid, matmul.io.cmd.valid) + commands.bits := Mux(convolution, convolve.io.cmd.bits, matmul.io.cmd.bits) + matmul.io.cmd.ready := !convolution && commands.ready + convolve.io.cmd.ready := convolution && commands.ready + expansion.rows.valid := state === streaming && commands.valid + commands.ready := state === streaming && expansion.rows.ready + expansion.rows.bits := 0.U.asTypeOf(expansion.rows.bits) + expansion.rows.bits.last := commands.bits.last + expansion.rows.bits.ball_id := ball.id.U + expansion.rows.bits.master_rob_id := master + when(expansion.rows.fire && expansion.rows.bits.last)(state := idle) + for (i <- 0 until 4) { + val src = commands.bits.slots(i) + val out = expansion.rows.bits.slots(i) + val c = out.cmd + val t = src.bits + val alloc = t.cmdType === LoopSubCmdType.MSET_ALLOC + val free = t.cmdType === LoopSubCmdType.MSET_FREE + val load = t.cmdType === LoopSubCmdType.MVIN + val store = t.cmdType === LoopSubCmdType.MVOUT + val preload = t.cmdType === LoopSubCmdType.PRELOAD + val compute = t.cmdType === LoopSubCmdType.COMPUTE + val fn = MuxCase( + funct("GEMMINI_COMPUTE_ACCUMULATED"), + Seq( + (alloc || free) -> 32.U, + load -> 33.U, + store -> 16.U, + preload -> funct("GEMMINI_PRELOAD"), + (compute && t.compute_mode === 0.U) -> funct("GEMMINI_COMPUTE_PRELOADED") + ) + ) + out.valid := src.valid + c.domain_id := Mux(alloc || free || load || store, DomainId.MEM, DomainId.BALL) + c.cmd.raw_inst := (fn << 25) | "h307b".U + c.cmd.rs1Data := Mux( + alloc || free, + t.bank_id, + Mux( + load, + t.bank_id << 20, + Mux(store, t.bank_id, t.op1_bank | (t.op2_bank << 10) | (t.wr_bank << 20)) + ) | (t.iter << 30) + ) + c.cmd.rs2Data := Mux( + alloc, + t.bank_row | (t.bank_col << 5) | (1.U << 10), + Mux( + free, + 0.U, + Mux( + load || store, + t.dram_addr | (t.stride.pad(64) << 39), + Mux(preload, 1.U, Mux(t.compute_mode === 0.U, 2.U, 3.U)) | (t.zero_op2 << 4) | (t.zero_op1_tail << 5) + ) + ) + ) + c.iter := t.iter + c.bankAccess.rd_bank_0_valid := store || preload || compute + c.bankAccess.rd_bank_0_id := Mux(store, t.bank_id, t.op1_bank) + c.bankAccess.rd_bank_1_valid := compute && !t.zero_op2 + c.bankAccess.rd_bank_1_id := t.op2_bank + c.bankAccess.wr_bank_valid := alloc || free || load || compute + c.bankAccess.wr_bank_id := Mux(alloc || free || load, t.bank_id, t.wr_bank) + c.architecturalDestination := c.bankAccess.wr_bank_id + c.bankAccess.rd_wr_valid := compute && t.compute_mode =/= 0.U + c.bankAccess.rd_wr_id := t.wr_bank + c.wr_read_bank := t.wr_bank + c.op1_col := 1.U; c.op2_col := 1.U; c.wr_col := 4.U + } +} diff --git a/examples/prefab/balls/gemmini/arch/LoopCmd.scala b/examples/prefab/balls/gemmini/arch/LoopCmd.scala new file mode 100644 index 000000000..47240b530 --- /dev/null +++ b/examples/prefab/balls/gemmini/arch/LoopCmd.scala @@ -0,0 +1,86 @@ +package examples.balls.gemmini + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import examples.balls.gemmini.configs.GemminiBallParam + +import chisel3._ +import chisel3.util._ + +object LoopSubCmdType { + val MSET_ALLOC = 0.U(3.W) + val MSET_FREE = 1.U(3.W) + val MVIN = 2.U(3.W) + val MVOUT = 3.U(3.W) + val PRELOAD = 4.U(3.W) + val COMPUTE = 5.U(3.W) +} + +class LoopSubCmd(val frontendParam: FrontendParam, val memParam: MemDomainParam, val ballParam: GemminiBallParam) + extends Bundle { + val cmdType = UInt(3.W) + val bank_id = UInt(frontendParam.bank_id_len.W) + val dram_addr = UInt(memParam.memAddrLen.W) + val stride = UInt(19.W) + val iter = UInt(frontendParam.iter_len.W) + val bank_row = UInt(5.W) + val bank_col = UInt(5.W) + val op1_bank = UInt(frontendParam.bank_id_len.W) + val op2_bank = UInt(frontendParam.bank_id_len.W) + val wr_bank = UInt(frontendParam.bank_id_len.W) + val compute_mode = UInt(2.W) + val zero_op2 = Bool() + val zero_op1_tail = Bool() +} + +class LoopCmd(val frontendParam: FrontendParam, val memParam: MemDomainParam, val ballParam: GemminiBallParam) + extends Bundle { + val slots = Vec(4, Valid(new LoopSubCmd(frontendParam, memParam, ballParam))) + val last = Bool() +} + +class LoopWsConfig(val frontendParam: FrontendParam, val memParam: MemDomainParam, val ballParam: GemminiBallParam) + extends Bundle { + val max_i = UInt(16.W) + val max_j = UInt(16.W) + val max_k = UInt(16.W) + val dram_addr_a = UInt(memParam.memAddrLen.W) + val dram_addr_b = UInt(memParam.memAddrLen.W) + val dram_addr_d = UInt(memParam.memAddrLen.W) + val dram_addr_c = UInt(memParam.memAddrLen.W) + val stride_a = UInt(32.W) + val stride_b = UInt(32.W) + val stride_d = UInt(32.W) + val stride_c = UInt(32.W) + val bank_a = UInt(frontendParam.bank_id_len.W) + val bank_b = UInt(frontendParam.bank_id_len.W) + val bank_c = UInt(frontendParam.bank_id_len.W) + val low_d = Bool() +} + +class LoopConvWsConfig(val frontendParam: FrontendParam, val memParam: MemDomainParam, val ballParam: GemminiBallParam) + extends Bundle { + val batch_size = UInt(16.W) + val in_dim = UInt(16.W) + val in_channels = UInt(16.W) + val out_channels = UInt(16.W) + val out_dim = UInt(16.W) + val stride = UInt(8.W) + val padding = UInt(8.W) + val kernel_dim = UInt(8.W) + val pool_size = UInt(8.W) + val pool_stride = UInt(8.W) + val pool_padding = UInt(8.W) + val dram_addr_bias = UInt(memParam.memAddrLen.W) + val dram_addr_input = UInt(memParam.memAddrLen.W) + val dram_addr_weight = UInt(memParam.memAddrLen.W) + val dram_addr_output = UInt(memParam.memAddrLen.W) + val input_stride = UInt(32.W) + val weight_stride = UInt(32.W) + val output_stride = UInt(32.W) + val bank_input = UInt(frontendParam.bank_id_len.W) + val bank_weight = UInt(frontendParam.bank_id_len.W) + val bank_output = UInt(frontendParam.bank_id_len.W) + val no_bias = Bool() +} diff --git a/examples/balls/gemmini/arch/src/main/scala/LoopConvAddrGen.scala b/examples/prefab/balls/gemmini/arch/LoopConvAddrGen.scala similarity index 80% rename from examples/balls/gemmini/arch/src/main/scala/LoopConvAddrGen.scala rename to examples/prefab/balls/gemmini/arch/LoopConvAddrGen.scala index c0471a757..6cdcefe72 100644 --- a/examples/balls/gemmini/arch/src/main/scala/LoopConvAddrGen.scala +++ b/examples/prefab/balls/gemmini/arch/LoopConvAddrGen.scala @@ -1,9 +1,11 @@ package examples.balls.gemmini +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public} -import framework.top.GlobalConfig import examples.balls.gemmini.configs.GemminiBallParam /** @@ -16,15 +18,16 @@ import examples.balls.gemmini.configs.GemminiBallParam * - Bias DRAM address */ @instantiable -class LoopConvAddrGen(val b: GlobalConfig) extends Module { - val config = GemminiBallParam(b) +class LoopConvAddrGen(val frontendParam: FrontendParam, val memParam: MemDomainParam, val ballParam: GemminiBallParam) + extends Module { + val config = ballParam val elemSize = config.inputWidth / 8 val accBytes = config.accWidth / 8 @public val io = IO(new Bundle { // Conv config - val cfg = Input(new LoopConvWsConfig(b)) + val cfg = Input(new LoopConvWsConfig(frontendParam, memParam, ballParam)) // Loop indices val batch = Input(UInt(16.W)) val orow = Input(UInt(16.W)) @@ -34,10 +37,10 @@ class LoopConvAddrGen(val b: GlobalConfig) extends Module { val kcol = Input(UInt(16.W)) val kch = Input(UInt(16.W)) // Outputs - val inputAddr = Output(UInt(b.memDomain.memAddrLen.W)) - val weightAddr = Output(UInt(b.memDomain.memAddrLen.W)) - val outputAddr = Output(UInt(b.memDomain.memAddrLen.W)) - val biasAddr = Output(UInt(b.memDomain.memAddrLen.W)) + val inputAddr = Output(UInt(memParam.memAddrLen.W)) + val weightAddr = Output(UInt(memParam.memAddrLen.W)) + val outputAddr = Output(UInt(memParam.memAddrLen.W)) + val biasAddr = Output(UInt(memParam.memAddrLen.W)) val isPadding = Output(Bool()) }) diff --git a/examples/prefab/balls/gemmini/arch/LoopConvUnroller.scala b/examples/prefab/balls/gemmini/arch/LoopConvUnroller.scala new file mode 100644 index 000000000..2a756f642 --- /dev/null +++ b/examples/prefab/balls/gemmini/arch/LoopConvUnroller.scala @@ -0,0 +1,306 @@ +package examples.balls.gemmini + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import examples.balls.gemmini.configs.GemminiBallParam + +/** + * LoopConvUnroller — expands LOOP_CONV_WS into LoopCmd rows. + * + * Simplified iteration: + * batch → orow → ocol → och → krow → kcol → kch (innermost) + * + * Each (orow, ocol, och, krow, kcol, kch) core generates: + * MVIN weight → PRELOAD + MVIN input → COMPUTE → (on last k-iter) MVOUT + * + * First implementation: no pooling, no im2col tiling, single DIM×DIM cores. + */ +@instantiable +class LoopConvUnroller(val frontendParam: FrontendParam, val memParam: MemDomainParam, val ballParam: GemminiBallParam) + extends Module { + val config = ballParam + val DIM = config.blockSize + val elemSize = config.inputWidth / 8 + val bankBytes = memParam.bankWidth / 8 + + @public + val io = IO(new Bundle { + val start = Flipped(Decoupled(new LoopConvWsConfig(frontendParam, memParam, ballParam))) + val cmd = Decoupled(new LoopCmd(frontendParam, memParam, ballParam)) + val busy = Output(Bool()) + }) + + val addrGen: Instance[LoopConvAddrGen] = Instantiate(new LoopConvAddrGen(frontendParam, memParam, ballParam)) + + // FSM states + val sIdle :: sAlloc :: sMvinInput :: sMvinWeight :: sCompute :: sMvout :: sFree :: sDone :: Nil = + Enum(8) + val state = RegInit(sIdle) + + val cfg = Reg(new LoopConvWsConfig(frontendParam, memParam, ballParam)) + + // Iterator registers + val batch_reg = RegInit(0.U(16.W)) + val orow_reg = RegInit(0.U(16.W)) + val ocol_reg = RegInit(0.U(16.W)) + val och_reg = RegInit(0.U(16.W)) + val krow_reg = RegInit(0.U(16.W)) + val kcol_reg = RegInit(0.U(16.W)) + val kch_reg = RegInit(0.U(16.W)) + + // Is this the first k-iteration for this (orow, ocol, och)? + val isFirstK = krow_reg === 0.U && kcol_reg === 0.U && kch_reg === 0.U + + // Is this the last k-iteration? + val isLastK = (krow_reg === cfg.kernel_dim - 1.U) && + (kcol_reg === cfg.kernel_dim - 1.U) && + (kch_reg + DIM.U >= cfg.in_channels) // Core boundary + + // Connect address generator + addrGen.io.cfg := cfg + addrGen.io.batch := batch_reg + addrGen.io.orow := orow_reg + addrGen.io.ocol := ocol_reg + addrGen.io.och := och_reg + addrGen.io.krow := krow_reg + addrGen.io.kcol := kcol_reg + addrGen.io.kch := kch_reg + + // Defaults + io.start.ready := state === sIdle + io.cmd.valid := false.B + io.cmd.bits := 0.U.asTypeOf(new LoopCmd(frontendParam, memParam, ballParam)) + io.cmd.bits.last := state === sFree + io.busy := state =/= sIdle + + def emptySlot(): Valid[LoopSubCmd] = { + val v = Wire(Valid(new LoopSubCmd(frontendParam, memParam, ballParam))) + v.valid := false.B + v.bits := 0.U.asTypeOf(new LoopSubCmd(frontendParam, memParam, ballParam)) + v + } + + def msetSlot( + bankId: UInt, + alloc: Boolean, + row: Int, + col: Int + ): Valid[LoopSubCmd] = { + val v = Wire(Valid(new LoopSubCmd(frontendParam, memParam, ballParam))) + v.valid := true.B + v.bits := 0.U.asTypeOf(new LoopSubCmd(frontendParam, memParam, ballParam)) + v.bits.cmdType := (if (alloc) LoopSubCmdType.MSET_ALLOC + else LoopSubCmdType.MSET_FREE) + v.bits.bank_id := bankId + v.bits.bank_row := row.U + v.bits.bank_col := col.U + v + } + + def mvinSlot( + bankId: UInt, + addr: UInt, + iter: UInt, + stride: UInt + ): Valid[LoopSubCmd] = { + val v = Wire(Valid(new LoopSubCmd(frontendParam, memParam, ballParam))) + v.valid := true.B + v.bits := 0.U.asTypeOf(new LoopSubCmd(frontendParam, memParam, ballParam)) + v.bits.cmdType := LoopSubCmdType.MVIN + v.bits.bank_id := bankId + v.bits.dram_addr := addr + v.bits.stride := stride + v.bits.iter := iter + v + } + + def outRows(): UInt = 1.U + + def inMemStride(bytes: UInt): UInt = + (bytes / bankBytes.U).pad(19)(18, 0) + + def outMemStride(bytes: UInt): UInt = + (bytes / (bankBytes * 4).U).pad(19)(18, 0) + + def weightTileStride(): UInt = + cfg.out_channels * elemSize.U + + // Advance k-iterators (kch → kcol → krow), then output iterators (och → ocol → orow → batch) + def advanceIter(): Unit = { + when(kch_reg + DIM.U < cfg.in_channels) { + kch_reg := kch_reg + DIM.U + }.elsewhen(kcol_reg + 1.U < cfg.kernel_dim) { + kch_reg := 0.U; kcol_reg := kcol_reg + 1.U + }.elsewhen(krow_reg + 1.U < cfg.kernel_dim) { + kch_reg := 0.U; kcol_reg := 0.U; krow_reg := krow_reg + 1.U + }.elsewhen(och_reg + DIM.U < cfg.out_channels) { + kch_reg := 0.U; kcol_reg := 0.U; krow_reg := 0.U; + och_reg := och_reg + DIM.U + }.elsewhen(ocol_reg + 1.U < cfg.out_dim) { + kch_reg := 0.U; kcol_reg := 0.U; krow_reg := 0.U; och_reg := 0.U + ocol_reg := ocol_reg + 1.U + }.elsewhen(orow_reg + 1.U < cfg.out_dim) { + kch_reg := 0.U; kcol_reg := 0.U; krow_reg := 0.U; och_reg := 0.U; + ocol_reg := 0.U + orow_reg := orow_reg + 1.U + }.elsewhen(batch_reg + 1.U < cfg.batch_size) { + kch_reg := 0.U; kcol_reg := 0.U; krow_reg := 0.U; och_reg := 0.U; + ocol_reg := 0.U; orow_reg := 0.U + batch_reg := batch_reg + 1.U + }.otherwise { + state := sFree // All iterations done + } + } + + // Track whether all iterations are done (used to decide next state) + val allDone = (batch_reg === cfg.batch_size - 1.U) && + (orow_reg === cfg.out_dim - 1.U) && (ocol_reg === cfg.out_dim - 1.U) && + (och_reg + DIM.U >= cfg.out_channels) && isLastK + + switch(state) { + is(sIdle) { + io.start.ready := true.B + when(io.start.fire) { + cfg := io.start.bits + batch_reg := 0.U; orow_reg := 0.U; ocol_reg := 0.U; och_reg := 0.U + krow_reg := 0.U; kcol_reg := 0.U; kch_reg := 0.U + state := sAlloc + } + } + + // [MSET_alloc(input), MSET_alloc(weight), MSET_alloc(output), --] + is(sAlloc) { + io.cmd.valid := true.B + io.cmd.bits + .slots(0) := msetSlot(cfg.bank_input, alloc = true, row = 1, col = 1) + io.cmd.bits + .slots(1) := msetSlot(cfg.bank_weight, alloc = true, row = 1, col = 1) + io.cmd.bits + .slots(2) := msetSlot(cfg.bank_output, alloc = true, row = 1, col = 4) + io.cmd.bits.slots(3) := emptySlot() + when(io.cmd.fire) { + state := sMvinInput + } + } + + // Load weights before preloading them into the array. + is(sMvinInput) { + io.cmd.valid := true.B + io.cmd.bits.slots(0) := mvinSlot( + cfg.bank_weight, + addrGen.io.weightAddr, + DIM.U, + inMemStride(weightTileStride()) + ) + when(io.cmd.fire)(state := sMvinWeight) + } + + // Preload weights while loading independent input data. + is(sMvinWeight) { + io.cmd.valid := true.B + val preSlot = Wire(Valid(new LoopSubCmd(frontendParam, memParam, ballParam))) + preSlot.valid := true.B + preSlot.bits := 0.U.asTypeOf(new LoopSubCmd(frontendParam, memParam, ballParam)) + preSlot.bits.cmdType := LoopSubCmdType.PRELOAD + preSlot.bits.op1_bank := cfg.bank_weight + preSlot.bits.wr_bank := cfg.bank_output + preSlot.bits.iter := DIM.U + io.cmd.bits.slots(0) := preSlot + io.cmd.bits.slots(1) := Mux( + addrGen.io.isPadding, + emptySlot(), + mvinSlot( + cfg.bank_input, + addrGen.io.inputAddr, + outRows(), + inMemStride(cfg.input_stride) + ) + ) + when(io.cmd.fire)(state := sCompute) + } + + // Compute consumes the preceding preload and DMA row. + is(sCompute) { + io.cmd.valid := true.B + val compSlot = Wire(Valid(new LoopSubCmd(frontendParam, memParam, ballParam))) + compSlot.valid := true.B + compSlot.bits := 0.U.asTypeOf(new LoopSubCmd(frontendParam, memParam, ballParam)) + compSlot.bits.cmdType := LoopSubCmdType.COMPUTE + compSlot.bits.op1_bank := cfg.bank_input + compSlot.bits.op2_bank := cfg.bank_weight + compSlot.bits.wr_bank := cfg.bank_output + compSlot.bits.compute_mode := Mux( + isFirstK, + 0.U, + 1.U + ) // PRELOADED first, then ACCUMULATED + compSlot.bits.iter := DIM.U + compSlot.bits.zero_op2 := cfg.no_bias + compSlot.bits.zero_op1_tail := true.B + + io.cmd.bits.slots(0) := compSlot + io.cmd.bits.slots(1) := emptySlot() + io.cmd.bits.slots(2) := emptySlot() + io.cmd.bits.slots(3) := emptySlot() + + when(io.cmd.fire) { + when(isLastK) { + state := sMvout // Need to output results after last k iteration + }.otherwise { + advanceIter() + when(state =/= sFree)(state := sMvinInput) + } + } + } + + // [MVOUT, --, --, --] + is(sMvout) { + io.cmd.valid := true.B + val mvSlot = Wire(Valid(new LoopSubCmd(frontendParam, memParam, ballParam))) + mvSlot.valid := true.B + mvSlot.bits := 0.U.asTypeOf(new LoopSubCmd(frontendParam, memParam, ballParam)) + mvSlot.bits.cmdType := LoopSubCmdType.MVOUT + mvSlot.bits.bank_id := cfg.bank_output + mvSlot.bits.dram_addr := addrGen.io.outputAddr + mvSlot.bits.stride := outMemStride(cfg.output_stride) + mvSlot.bits.iter := outRows() + + io.cmd.bits.slots(0) := mvSlot + io.cmd.bits.slots(1) := emptySlot() + io.cmd.bits.slots(2) := emptySlot() + io.cmd.bits.slots(3) := emptySlot() + + when(io.cmd.fire) { + when(allDone) { + state := sFree + }.otherwise { + advanceIter() + when(state =/= sFree)(state := sMvinInput) + } + } + } + + // [MSET_free(input), MSET_free(weight), MSET_free(output), --] + is(sFree) { + io.cmd.valid := true.B + io.cmd.bits + .slots(0) := msetSlot(cfg.bank_input, alloc = false, row = 0, col = 0) + io.cmd.bits + .slots(1) := msetSlot(cfg.bank_weight, alloc = false, row = 0, col = 0) + io.cmd.bits + .slots(2) := msetSlot(cfg.bank_output, alloc = false, row = 0, col = 0) + io.cmd.bits.slots(3) := emptySlot() + when(io.cmd.fire) { + state := sDone + } + } + + is(sDone) { + state := sIdle + } + } +} diff --git a/examples/prefab/balls/gemmini/arch/LoopMatmulUnroller.scala b/examples/prefab/balls/gemmini/arch/LoopMatmulUnroller.scala new file mode 100644 index 000000000..a13478b32 --- /dev/null +++ b/examples/prefab/balls/gemmini/arch/LoopMatmulUnroller.scala @@ -0,0 +1,340 @@ +package examples.balls.gemmini + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import examples.balls.gemmini.configs.GemminiBallParam + +/** + * LoopMatmulUnroller — expands LOOP_WS into a stream of LoopCmd rows. + * + * FSM: sIdle → sAllocBanks → sPrimeLoad → sMainLoop (Row1/Row2) → sDrainLast → sFreeBanks → sDone + * + * Row layout per iteration: + * Row 1: [PRELOAD(cur), MVIN_B(cur)] + * Row 2: [COMPUTE(cur)] + * Row 3: [MVOUT(cur), MVIN_A(next)] + * The final compute row is followed by a drain row and bank release. + */ +@instantiable +class LoopMatmulUnroller(val frontendParam: FrontendParam, val memParam: MemDomainParam, val ballParam: GemminiBallParam) + extends Module { + val config = ballParam + val DIM = config.blockSize + val elemSize = config.inputWidth / 8 + val accBytes = config.accWidth / 8 + val bankBytes = memParam.bankWidth / 8 + + @public + val io = IO(new Bundle { + val start = Flipped(Decoupled(new LoopWsConfig(frontendParam, memParam, ballParam))) + val cmd = Decoupled(new LoopCmd(frontendParam, memParam, ballParam)) + val busy = Output(Bool()) + }) + + // FSM states + val sIdle :: sAllocBanks :: sPrimeLoad :: sMainRow1 :: sCompute :: sMainRow2 :: sDrainLast :: sFreeBanks :: sDone :: Nil = + Enum(9) + val state = RegInit(sIdle) + + // Config registers (latched at start) + val cfg = Reg(new LoopWsConfig(frontendParam, memParam, ballParam)) + + // Loop iterator registers + val i_reg = RegInit(0.U(16.W)) + val j_reg = RegInit(0.U(16.W)) + val k_reg = RegInit(0.U(16.W)) + + // Total iterations and current iteration index + val totalIter = cfg.max_i * cfg.max_j * cfg.max_k + val curIter = RegInit(0.U(32.W)) + + // Whether this is the last iteration + val isLastIter = curIter === (totalIter - 1.U) + + // Defaults + io.start.ready := state === sIdle + io.cmd.valid := false.B + io.cmd.bits := 0.U.asTypeOf(new LoopCmd(frontendParam, memParam, ballParam)) + io.cmd.bits.last := state === sFreeBanks + io.busy := state =/= sIdle + + // Helper to build an invalid LoopSubCmd slot + def emptySlot(): Valid[LoopSubCmd] = { + val v = Wire(Valid(new LoopSubCmd(frontendParam, memParam, ballParam))) + v.valid := false.B + v.bits := 0.U.asTypeOf(new LoopSubCmd(frontendParam, memParam, ballParam)) + v + } + + // Helper to build an MSET slot + def msetSlot( + bankId: UInt, + alloc: Boolean, + row: Int, + col: Int + ): Valid[LoopSubCmd] = { + val v = Wire(Valid(new LoopSubCmd(frontendParam, memParam, ballParam))) + v.valid := true.B + v.bits := 0.U.asTypeOf(new LoopSubCmd(frontendParam, memParam, ballParam)) + v.bits.cmdType := (if (alloc) LoopSubCmdType.MSET_ALLOC + else LoopSubCmdType.MSET_FREE) + v.bits.bank_id := bankId + v.bits.bank_row := row.U + v.bits.bank_col := col.U + v + } + + // Helper to build an MVIN slot + def mvinSlot( + bankId: UInt, + addr: UInt, + iter: UInt, + stride: UInt + ): Valid[LoopSubCmd] = { + val v = Wire(Valid(new LoopSubCmd(frontendParam, memParam, ballParam))) + v.valid := true.B + v.bits := 0.U.asTypeOf(new LoopSubCmd(frontendParam, memParam, ballParam)) + v.bits.cmdType := LoopSubCmdType.MVIN + v.bits.bank_id := bankId + v.bits.dram_addr := addr + v.bits.stride := stride + v.bits.iter := iter + v + } + + // Helper to build an MVOUT slot + def mvoutSlot( + bankId: UInt, + addr: UInt, + iter: UInt, + stride: UInt + ): Valid[LoopSubCmd] = { + val v = Wire(Valid(new LoopSubCmd(frontendParam, memParam, ballParam))) + v.valid := true.B + v.bits := 0.U.asTypeOf(new LoopSubCmd(frontendParam, memParam, ballParam)) + v.bits.cmdType := LoopSubCmdType.MVOUT + v.bits.bank_id := bankId + v.bits.dram_addr := addr + v.bits.stride := stride + v.bits.iter := iter + v + } + + // Address computation + // addr_a(i,k) = dram_addr_a + i * stride_a + k * DIM * elemSize + // addr_b(k,j) = dram_addr_b + k * stride_b + j * DIM * elemSize + // addr_c(i,j) = dram_addr_c + i * stride_c + j * DIM * accBytes + def addrA(i: UInt, k: UInt): UInt = + cfg.dram_addr_a + i * DIM.U * cfg.stride_a + k * (DIM * elemSize).U + + def addrB(k: UInt, j: UInt): UInt = + cfg.dram_addr_b + k * DIM.U * cfg.stride_b + j * (DIM * elemSize).U + + def addrC(i: UInt, j: UInt): UInt = + cfg.dram_addr_c + i * DIM.U * cfg.stride_c + j * (DIM * accBytes).U + + def inMemStride(bytes: UInt): UInt = + (bytes / bankBytes.U)(18, 0) + + def outMemStride(bytes: UInt): UInt = + (bytes / (bankBytes * 4).U)(18, 0) + + // Next iterator values (advance k, then j, then i) + val next_k = Wire(UInt(16.W)) + val next_j = Wire(UInt(16.W)) + val next_i = Wire(UInt(16.W)) + + when(k_reg + 1.U < cfg.max_k) { + next_k := k_reg + 1.U + next_j := j_reg + next_i := i_reg + }.elsewhen(j_reg + 1.U < cfg.max_j) { + next_k := 0.U + next_j := j_reg + 1.U + next_i := i_reg + }.otherwise { + next_k := 0.U + next_j := 0.U + next_i := i_reg + 1.U + } + + // Compute mode: first k iteration uses PRELOADED (0), rest use ACCUMULATED (1) + val computeMode = Mux(k_reg === 0.U, 0.U(2.W), 1.U(2.W)) + + switch(state) { + is(sIdle) { + io.start.ready := true.B + when(io.start.fire) { + assert(io.start.bits.max_i =/= 0.U && io.start.bits.max_j =/= 0.U && io.start.bits.max_k =/= 0.U) + assert( + io.start.bits.low_d && io.start.bits.dram_addr_d === 0.U, + "Loop matmul requires zero initial accumulation" + ) + assert( + io.start.bits.bank_a =/= io.start.bits.bank_b && io.start.bits.bank_a =/= io.start.bits.bank_c && io.start.bits.bank_b =/= io.start.bits.bank_c + ) + cfg := io.start.bits + i_reg := 0.U + j_reg := 0.U + k_reg := 0.U + curIter := 0.U + state := sAllocBanks + } + } + + // Row 0: [MSET_alloc(A), MSET_alloc(B), MSET_alloc(C), --] + is(sAllocBanks) { + io.cmd.valid := true.B + io.cmd.bits + .slots(0) := msetSlot(cfg.bank_a, alloc = true, row = 1, col = 1) + io.cmd.bits + .slots(1) := msetSlot(cfg.bank_b, alloc = true, row = 1, col = 1) + io.cmd.bits.slots(2) := msetSlot( + cfg.bank_c, + alloc = true, + row = 1, + col = 4 + ) // accWidth = 4x inputWidth + io.cmd.bits.slots(3) := emptySlot() + when(io.cmd.fire) { + state := sPrimeLoad + } + } + + // Load the activation core before preloading it. + is(sPrimeLoad) { + io.cmd.valid := true.B + io.cmd.bits.slots(0) := mvinSlot( + cfg.bank_a, + addrA(0.U, 0.U), + DIM.U, + inMemStride(cfg.stride_a) + ) + io.cmd.bits.slots(1) := emptySlot() + io.cmd.bits.slots(2) := emptySlot() + io.cmd.bits.slots(3) := emptySlot() + when(io.cmd.fire) { + state := sMainRow1 + } + } + + // Preload activations while loading independent weights. + is(sMainRow1) { + io.cmd.valid := true.B + val preSlot = Wire(Valid(new LoopSubCmd(frontendParam, memParam, ballParam))) + preSlot.valid := k_reg === 0.U + preSlot.bits := 0.U.asTypeOf(new LoopSubCmd(frontendParam, memParam, ballParam)) + preSlot.bits.cmdType := LoopSubCmdType.PRELOAD + preSlot.bits.op1_bank := cfg.bank_a // preload A (activations) into matrix array + preSlot.bits.wr_bank := cfg.bank_c + preSlot.bits.iter := DIM.U + + io.cmd.bits.slots(0) := preSlot + io.cmd.bits.slots(1) := mvinSlot( + cfg.bank_b, + addrB(k_reg, j_reg), + DIM.U, + inMemStride(cfg.stride_b) + ) + io.cmd.bits.slots(2) := emptySlot() + io.cmd.bits.slots(3) := emptySlot() + when(io.cmd.fire)(state := sCompute) + } + + // Compute consumes the preceding preload and DMA row. + is(sCompute) { + io.cmd.valid := true.B + val compSlot = Wire(Valid(new LoopSubCmd(frontendParam, memParam, ballParam))) + compSlot.valid := true.B + compSlot.bits := 0.U.asTypeOf(new LoopSubCmd(frontendParam, memParam, ballParam)) + compSlot.bits.cmdType := LoopSubCmdType.COMPUTE + compSlot.bits.op1_bank := cfg.bank_a + compSlot.bits.op2_bank := cfg.bank_b + compSlot.bits.wr_bank := cfg.bank_c + compSlot.bits.compute_mode := computeMode + compSlot.bits.iter := DIM.U + + io.cmd.bits.slots(0) := compSlot + io.cmd.bits.slots(1) := emptySlot() + io.cmd.bits.slots(2) := emptySlot() + io.cmd.bits.slots(3) := emptySlot() + + when(io.cmd.fire) { + when(isLastIter) { + state := sDrainLast // Drain after the final compute. + }.otherwise { + state := sMainRow2 + } + } + } + + // Store the result while preparing the next activation core. + is(sMainRow2) { + io.cmd.valid := true.B + io.cmd.bits.slots(0) := mvoutSlot( + cfg.bank_c, + addrC(i_reg, j_reg), + DIM.U, + outMemStride(cfg.stride_c) + ) + io.cmd.bits.slots(0).valid := k_reg + 1.U === cfg.max_k + io.cmd.bits.slots(1) := mvinSlot( + cfg.bank_a, + addrA(next_i, next_k), + DIM.U, + inMemStride(cfg.stride_a) + ) + io.cmd.bits.slots(2) := emptySlot() + io.cmd.bits.slots(3) := emptySlot() + + when(io.cmd.fire) { + // Advance iterators + i_reg := next_i + j_reg := next_j + k_reg := next_k + curIter := curIter + 1.U + state := sMainRow1 + } + } + + // Drain: emit final MVOUT for last iteration + is(sDrainLast) { + io.cmd.valid := true.B + io.cmd.bits.slots(0) := mvoutSlot( + cfg.bank_c, + addrC(i_reg, j_reg), + DIM.U, + outMemStride(cfg.stride_c) + ) + io.cmd.bits.slots(1) := emptySlot() + io.cmd.bits.slots(2) := emptySlot() + io.cmd.bits.slots(3) := emptySlot() + when(io.cmd.fire) { + state := sFreeBanks + } + } + + // Free: [MSET_free(A), MSET_free(B), MSET_free(C), --] + is(sFreeBanks) { + io.cmd.valid := true.B + io.cmd.bits + .slots(0) := msetSlot(cfg.bank_a, alloc = false, row = 0, col = 0) + io.cmd.bits + .slots(1) := msetSlot(cfg.bank_b, alloc = false, row = 0, col = 0) + io.cmd.bits + .slots(2) := msetSlot(cfg.bank_c, alloc = false, row = 0, col = 0) + io.cmd.bits.slots(3) := emptySlot() + when(io.cmd.fire) { + state := sDone + } + } + + is(sDone) { + state := sIdle + } + } +} diff --git a/examples/prefab/balls/gemmini/configs/GemminiBallParam.scala b/examples/prefab/balls/gemmini/configs/GemminiBallParam.scala new file mode 100644 index 000000000..e02350d0b --- /dev/null +++ b/examples/prefab/balls/gemmini/configs/GemminiBallParam.scala @@ -0,0 +1,41 @@ +package examples.balls.gemmini.configs + +import buckyball.balls.gemmini.{Params => ParametersProto} + +import upickle.default.{macroRW, ReadWriter} + +case class GemminiBallParam( + meshRows: Int, + meshColumns: Int, + tileRows: Int, + tileColumns: Int, + inputWidth: Int, + accWidth: Int, + spatialOutputWidth: Int, + tileLatency: Int, + outputDelay: Int) { + + val totalRows: Int = meshRows * tileRows + val totalColumns: Int = meshColumns * tileColumns + val blockSize: Int = totalRows +} + +object GemminiBallParam { + + def fromProto(p: ParametersProto): GemminiBallParam = { + require(p.getAllFields.size == p.getDescriptorForType.getFields.size, "Gemmini parameters are required") + GemminiBallParam( + p.getMeshRows, + p.getMeshColumns, + p.getTileRows, + p.getTileColumns, + p.getInputWidth, + p.getAccWidth, + p.getSpatialOutputWidth, + p.getTileLatency, + p.getOutputDelay + ) + } + + implicit val rw: ReadWriter[GemminiBallParam] = macroRW +} diff --git a/examples/prefab/balls/gemmini/configs/default.toml b/examples/prefab/balls/gemmini/configs/default.toml new file mode 100644 index 000000000..6d1222892 --- /dev/null +++ b/examples/prefab/balls/gemmini/configs/default.toml @@ -0,0 +1,11 @@ +# Gemmini Ball Configuration + +meshRows = 16 +meshColumns = 16 +tileRows = 1 +tileColumns = 1 +inputWidth = 8 +accWidth = 32 +spatialOutputWidth = 20 +tileLatency = 0 +outputDelay = 1 diff --git a/examples/prefab/balls/gemmini/configs/gemmini_params.proto b/examples/prefab/balls/gemmini/configs/gemmini_params.proto new file mode 100644 index 000000000..2d6c66e95 --- /dev/null +++ b/examples/prefab/balls/gemmini/configs/gemmini_params.proto @@ -0,0 +1,16 @@ +syntax = "proto3"; +package buckyball.balls.gemmini; +option java_package = "buckyball.balls.gemmini"; +option java_multiple_files = true; + +message Params { + optional int32 meshRows = 1; + optional int32 meshColumns = 2; + optional int32 tileRows = 3; + optional int32 tileColumns = 4; + optional int32 inputWidth = 5; + optional int32 accWidth = 6; + optional int32 spatialOutputWidth = 7; + optional int32 tileLatency = 8; + optional int32 outputDelay = 9; +} diff --git a/examples/prefab/balls/gemmini/emu/src/02_gemmini_config.rs b/examples/prefab/balls/gemmini/emu/src/02_gemmini_config.rs new file mode 100644 index 000000000..f59b81b76 --- /dev/null +++ b/examples/prefab/balls/gemmini/emu/src/02_gemmini_config.rs @@ -0,0 +1,22 @@ +//===- 02_gemmini_config.rs - GEMMINI_CONFIG instruction -------------------===// + +use super::gemmini_state::gemini; +use super::instruction::{BallModel, ExecContext}; + +pub struct GemminiConfig; + +impl BallModel for GemminiConfig { + fn exec(_xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + let state = gemini(ctx); + let mut g = state.lock().unwrap(); + g.cfg.dataflow = ((xs2 >> 4) & 1) as u8; + g.cfg.a_transpose = ((xs2 >> 7) & 1) != 0; + g.cfg.b_transpose = ((xs2 >> 8) & 1) != 0; + g.cfg.in_shift = ((xs2 >> 9) & 0xFFFFFFFF) as u32; + 0 + } + + fn latency(_xs1: u64, _xs2: u64) -> u64 { + 1 + } +} diff --git a/examples/prefab/balls/gemmini/emu/src/03_gemmini_flush.rs b/examples/prefab/balls/gemmini/emu/src/03_gemmini_flush.rs new file mode 100644 index 000000000..2fb2066f0 --- /dev/null +++ b/examples/prefab/balls/gemmini/emu/src/03_gemmini_flush.rs @@ -0,0 +1,15 @@ +//===- 03_gemmini_flush.rs - GEMMINI_FLUSH instruction ---------------------===// + +use super::instruction::{BallModel, ExecContext}; + +pub struct GemminiFlush; + +impl BallModel for GemminiFlush { + fn exec(_xs1: u64, _xs2: u64, _ctx: &mut ExecContext) -> u64 { + 0 + } + + fn latency(_xs1: u64, _xs2: u64) -> u64 { + 1 + } +} diff --git a/examples/prefab/balls/gemmini/emu/src/21_gemmini_preload.rs b/examples/prefab/balls/gemmini/emu/src/21_gemmini_preload.rs new file mode 100644 index 000000000..74bf8015e --- /dev/null +++ b/examples/prefab/balls/gemmini/emu/src/21_gemmini_preload.rs @@ -0,0 +1,47 @@ +use super::bank_matrix::read_i8_nn_at; +use super::decode::{pbank_group, rs1_b0, rs1_b2, rs1_iter, rs1_read_old}; +use super::gemmini_state::gemini; +use super::instruction::{BallModel, ExecContext}; + +pub struct GemminiPreload; + +impl BallModel for GemminiPreload { + fn exec(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + assert!(!rs1_read_old(xs1), "gemmini_preload: reads no old destination; rs1[29] must be clear"); + let op1 = rs1_b0(xs1); + let wr = rs1_b2(xs1); + let n = rs1_iter(xs1) as usize; + + if !ctx.config(op1).allocated || !ctx.config(wr).allocated { + panic!("gemmini_preload: bank not allocated"); + } + if n == 0 || n > 64 { + panic!("gemmini_preload: bad iter"); + } + + // rs2[41:36] selects the group of op1. + let group1 = (xs2 >> 36) & 0x3f; + assert!(group1 < ctx.config(op1).cols, "gemmini_preload: op1 group outside the bank"); + let p1 = pbank_group(ctx, op1, group1); + let state = gemini(ctx); + let mut gm = state.lock().unwrap(); + + if gm.cfg.dataflow == 1 { + let b = read_i8_nn_at(&ctx.banks, p1, ((xs2 >> 6) & 0x3ff) as usize, n); + gm.ws_b = Some(if gm.cfg.b_transpose { + (0..n).map(|i| (0..n).map(|j| b[j][i]).collect()).collect() + } else { + b + }); + } else { + // RTL OS preload only primes the mesh and injects D=0. It does + // not write the destination SRAM bank; compute produces C. + } + 0 + } + + fn latency(xs1: u64, _xs2: u64) -> u64 { + let n = rs1_iter(xs1).clamp(1, 64); + n.saturating_mul(n) + } +} diff --git a/examples/balls/gemmini/emu/src/66_gemmini_compute_preloaded.rs b/examples/prefab/balls/gemmini/emu/src/66_gemmini_compute_preloaded.rs similarity index 81% rename from examples/balls/gemmini/emu/src/66_gemmini_compute_preloaded.rs rename to examples/prefab/balls/gemmini/emu/src/66_gemmini_compute_preloaded.rs index 401f09714..18c90b292 100644 --- a/examples/balls/gemmini/emu/src/66_gemmini_compute_preloaded.rs +++ b/examples/prefab/balls/gemmini/emu/src/66_gemmini_compute_preloaded.rs @@ -1,12 +1,13 @@ use super::bank_matrix::{read_i32_nn_at, read_i8_nn_at, write_i32_nn_groups_at}; -use super::decode::{pbank, pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; +use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old}; use super::gemmini_state::{gemini, in_shift as apply_in_shift}; -use super::instruction::{BallInstruction, ExecContext}; +use super::instruction::{BallModel, ExecContext}; pub struct GemminiComputePreloaded; -impl BallInstruction for GemminiComputePreloaded { +impl BallModel for GemminiComputePreloaded { fn exec(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + assert!(!rs1_read_old(xs1), "gemmini_compute_preloaded: reads no old destination; rs1[29] must be clear"); let op_a = rs1_b0(xs1); let op_b = rs1_b1(xs1); let wr = rs1_b2(xs1); @@ -19,12 +20,20 @@ impl BallInstruction for GemminiComputePreloaded { panic!("gemmini_compute_preloaded: bad iter"); } - let pa = pbank(ctx, op_a); - let pb = pbank(ctx, op_b); + // rs2[41:36] and rs2[47:42] select the groups of op1 and op2. + let group_a = (xs2 >> 36) & 0x3f; + let group_b = (xs2 >> 42) & 0x3f; + assert!( + group_a < ctx.config(op_a).cols && group_b < ctx.config(op_b).cols, + "gemmini_compute_preloaded: operand group outside the bank" + ); + let pa = pbank_group(ctx, op_a, group_a); + let pb = pbank_group(ctx, op_b, group_b); let pw: Vec<_> = (0..ctx.config(wr).cols) .map(|group| pbank_group(ctx, wr, group)) .collect(); - let gm = gemini().lock().unwrap(); + let state = gemini(ctx); + let gm = state.lock().unwrap(); let df = gm.cfg.dataflow; let a_transpose = gm.cfg.a_transpose; let b_transpose = gm.cfg.b_transpose; diff --git a/examples/prefab/balls/gemmini/emu/src/67_gemmini_compute_accumulated.rs b/examples/prefab/balls/gemmini/emu/src/67_gemmini_compute_accumulated.rs new file mode 100644 index 000000000..cc49087c7 --- /dev/null +++ b/examples/prefab/balls/gemmini/emu/src/67_gemmini_compute_accumulated.rs @@ -0,0 +1,80 @@ +use super::bank_matrix::{read_i32_nn_groups_at, read_i8_nn_at, write_i32_nn_groups_at}; +use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old}; +use super::gemmini_state::{gemini, in_shift as apply_in_shift}; +use super::instruction::{BallModel, ExecContext}; + +pub struct GemminiComputeAccumulated; + +impl BallModel for GemminiComputeAccumulated { + fn exec(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + // The RTL reads the old C from wr_read_bank, which rs1[29] provides. + assert!( + rs1_read_old(xs1), + "gemmini_compute_accumulated: reads the old destination; rs1[29] must be set" + ); + let op_a = rs1_b0(xs1); + let op_b = rs1_b1(xs1); + let wr = rs1_b2(xs1); + let n = rs1_iter(xs1) as usize; + + if !ctx.config(op_a).allocated || !ctx.config(op_b).allocated || !ctx.config(wr).allocated { + panic!("gemmini_compute_accumulated: bank not allocated"); + } + if n == 0 || n > 64 { + panic!("gemmini_compute_accumulated: bad iter"); + } + + // rs2[41:36] and rs2[47:42] select the groups of op1 and op2. + let group_a = (xs2 >> 36) & 0x3f; + let group_b = (xs2 >> 42) & 0x3f; + assert!( + group_a < ctx.config(op_a).cols && group_b < ctx.config(op_b).cols, + "gemmini_compute_accumulated: operand group outside the bank" + ); + let pa = pbank_group(ctx, op_a, group_a); + let pb = pbank_group(ctx, op_b, group_b); + let pw: Vec<_> = (0..ctx.config(wr).cols) + .map(|group| pbank_group(ctx, wr, group)) + .collect(); + + let state = gemini(ctx); + let gm = state.lock().unwrap(); + let a_transpose = gm.cfg.a_transpose; + let b_transpose = gm.cfg.b_transpose; + let shift = gm.cfg.in_shift; + drop(gm); + + let zero_op1_tail = ((xs2 >> 5) & 1) != 0; + let a_base = ((xs2 >> 6) & 0x3ff) as usize; + let b_base = ((xs2 >> 16) & 0x3ff) as usize; + let out_base = ((xs2 >> 26) & 0x3ff) as usize; + let op1_valid_rows = ctx.config(op_a).valid_rows.min(n as u64) as usize; + let mut a = read_i8_nn_at(&ctx.banks, pa, a_base, n); + if zero_op1_tail { + for row in &mut a[op1_valid_rows..] { + row.fill(0); + } + } + let b = read_i8_nn_at(&ctx.banks, pb, b_base, n); + let mut c = read_i32_nn_groups_at(&ctx.banks, &pw, out_base, n); + + for i in 0..n { + for j in 0..n { + for k in 0..n { + let av = if a_transpose { a[k][i] } else { a[i][k] }; + let bv = if b_transpose { b[j][k] } else { b[k][j] }; + c[i][j] += av as i32 * bv as i32; + } + c[i][j] = apply_in_shift(c[i][j], shift); + } + } + + write_i32_nn_groups_at(&mut ctx.banks, &pw, out_base, &c, n); + 0 + } + + fn latency(xs1: u64, _xs2: u64) -> u64 { + let n = rs1_iter(xs1).clamp(1, 64); + n.saturating_mul(n).saturating_mul(n) / 4 + n.saturating_mul(n) + } +} diff --git a/examples/prefab/balls/gemmini/emu/src/80_gemmini_loop_ws.rs b/examples/prefab/balls/gemmini/emu/src/80_gemmini_loop_ws.rs new file mode 100644 index 000000000..81c100dde --- /dev/null +++ b/examples/prefab/balls/gemmini/emu/src/80_gemmini_loop_ws.rs @@ -0,0 +1,130 @@ +//===- 80_gemmini_loop_ws.rs - GEMMINI_LOOP_WS instruction -----------------===// + +use bebop_bemu::thread::banks::{bank_row_bytes, MATRIX_SIZE}; +use super::gemmini_state::gemini; +use super::instruction::ExecContext; +use super::loop_micro_ops::{ + alloc, checked_stride, compute, free_after_hash, mvin, mvout, preload, +}; + +// Shared implementation +fn exec_cfg_impl(mnemonic: &str, xs2: u64, ctx: &mut ExecContext) -> u64 { + let fields = ["BOUNDS", "ADDR_A", "ADDR_B", "ADDR_D", "ADDR_C", "STRIDES_AB", "STRIDES_DC"]; + let field = fields.iter().position(|field| mnemonic == format!("GEMMINI_LOOP_WS_CONFIG_{field}")).expect("loop configuration instruction"); + ctx.write_metadata_word(field * 8, xs2); + 0 +} + +fn checked_tile_addr( + base: u64, + outer: u64, + outer_stride: u64, + inner: u64, + inner_stride: u64, +) -> u64 { + let outer_offset = outer + .checked_mul(MATRIX_SIZE as u64).and_then(|row| row.checked_mul(outer_stride)) + .expect("gemmini_loop_ws: outer address offset overflow"); + let inner_offset = inner + .checked_mul(inner_stride) + .expect("gemmini_loop_ws: inner address offset overflow"); + base.checked_add(outer_offset) + .and_then(|addr| addr.checked_add(inner_offset)) + .expect("gemmini_loop_ws: address overflow") +} + +fn exec_loop_impl(xs2: u64, ctx: &mut ExecContext) -> u64 { + let bounds = ctx.metadata_word(0); + let ab = ctx.metadata_word(40); + let dc = ctx.metadata_word(48); + let lw = super::gemmini_state::LoopWsCfg { + max_i: bounds >> 32 & 0xffff, max_j: bounds >> 16 & 0xffff, max_k: bounds & 0xffff, + addr_a: ctx.metadata_word(8), addr_b: ctx.metadata_word(16), addr_d: ctx.metadata_word(24), addr_c: ctx.metadata_word(32), + stride_a: ab & 0xffff_ffff, stride_b: ab >> 32, stride_d: dc & 0xffff_ffff, stride_c: dc >> 32, + }; + let dataflow = gemini(ctx).lock().unwrap().cfg.dataflow; + + let bank_a = xs2 & 0x3ff; + let bank_b = (xs2 >> 10) & 0x3ff; + let bank_c = (xs2 >> 20) & 0x3ff; + let low_d = ((xs2 >> 30) & 1) != 0; + assert!( + bank_a != bank_b && bank_a != bank_c && bank_b != bank_c, + "gemmini_loop_ws: banks must be distinct" + ); + assert!( + lw.max_i > 0 && lw.max_j > 0 && lw.max_k > 0, + "gemmini_loop_ws: bounds must be > 0" + ); + assert_eq!( + dataflow, 0, + "gemmini_loop_ws: current RTL CISC contract supports OS dataflow only" + ); + assert!( + low_d && lw.addr_d == 0 && lw.stride_d == 0, + "gemmini_loop_ws: only zero D is supported" + ); + + let dim = MATRIX_SIZE as u64; + let bank_bytes = bank_row_bytes() as u64; + let stride_a = checked_stride(lw.stride_a, bank_bytes, "gemmini_loop_ws stride_a"); + let stride_b = checked_stride(lw.stride_b, bank_bytes, "gemmini_loop_ws stride_b"); + let stride_c = checked_stride( + lw.stride_c, + bank_bytes + .checked_mul(4) + .expect("gemmini_loop_ws: output row width overflow"), + "gemmini_loop_ws stride_c", + ); + + alloc(ctx, bank_a, 1); + alloc(ctx, bank_b, 1); + alloc(ctx, bank_c, 4); + + mvin(ctx, bank_a, lw.addr_a, dim, stride_a); + mvin(ctx, bank_b, lw.addr_b, dim, stride_b); + + for i in 0..lw.max_i { + for j in 0..lw.max_j { + for k in 0..lw.max_k { + preload(ctx, bank_a, bank_c, dim); + compute(ctx, k != 0, bank_a, bank_b, bank_c, dim, false, false); + + let addr_c = checked_tile_addr(lw.addr_c, i, lw.stride_c, j, dim * 4); + mvout(ctx, bank_c, addr_c, dim, stride_c); + + let next = if k + 1 < lw.max_k { + Some((i, j, k + 1)) + } else if j + 1 < lw.max_j { + Some((i, j + 1, 0)) + } else if i + 1 < lw.max_i { + Some((i + 1, 0, 0)) + } else { + None + }; + if let Some((next_i, next_j, next_k)) = next { + let addr_a = checked_tile_addr(lw.addr_a, next_i, lw.stride_a, next_k, dim); + let addr_b = checked_tile_addr(lw.addr_b, next_k, lw.stride_b, next_j, dim); + mvin(ctx, bank_a, addr_a, dim, stride_a); + mvin(ctx, bank_b, addr_b, dim, stride_b); + } + } + } + } + + free_after_hash(ctx, bank_a); + free_after_hash(ctx, bank_b); + free_after_hash(ctx, bank_c); + 0 +} + +pub fn execute(mnemonic: &str, xs2: u64, ctx: &mut ExecContext) -> u64 { + match mnemonic { + "GEMMINI_LOOP_WS" => exec_loop_impl(xs2, ctx), + _ => exec_cfg_impl(mnemonic, xs2, ctx), + } +} + +pub fn latency(mnemonic: &str) -> u64 { + if mnemonic == "GEMMINI_LOOP_WS" { 256 } else { 1 } +} diff --git a/examples/balls/gemmini/emu/src/96_gemmini_loop_conv_ws.rs b/examples/prefab/balls/gemmini/emu/src/96_gemmini_loop_conv_ws.rs similarity index 80% rename from examples/balls/gemmini/emu/src/96_gemmini_loop_conv_ws.rs rename to examples/prefab/balls/gemmini/emu/src/96_gemmini_loop_conv_ws.rs index 36d72f72b..43c2318fe 100644 --- a/examples/balls/gemmini/emu/src/96_gemmini_loop_conv_ws.rs +++ b/examples/prefab/balls/gemmini/emu/src/96_gemmini_loop_conv_ws.rs @@ -1,6 +1,6 @@ //===- 96_gemmini_loop_conv_ws.rs - GEMMINI_LOOP_CONV_WS instruction -------===// -use super::super::bank::{bank_row_bytes, MATRIX_SIZE}; +use bebop_bemu::thread::banks::{bank_row_bytes, MATRIX_SIZE}; use super::gemmini_state::gemini; use super::instruction::ExecContext; use super::loop_micro_ops::{ @@ -8,37 +8,10 @@ use super::loop_micro_ops::{ }; // Shared implementation -fn exec_cfg_impl(mnemonic: &str, xs2: u64) -> u64 { - let mut g = gemini().lock().unwrap(); - match mnemonic { - "GEMMINI_LOOP_CONV_WS_CONFIG_1" => { - g.loop_conv.batch = xs2 & 0xffff; - g.loop_conv.in_dim = (xs2 >> 16) & 0xffff; - g.loop_conv.in_ch = (xs2 >> 32) & 0xffff; - } - "GEMMINI_LOOP_CONV_WS_CONFIG_2" => { - g.loop_conv.out_ch = xs2 & 0xffff; - g.loop_conv.out_dim = (xs2 >> 16) & 0xffff; - g.loop_conv.stride = (xs2 >> 32) & 0xff; - g.loop_conv.padding = (xs2 >> 40) & 0xff; - } - "GEMMINI_LOOP_CONV_WS_CONFIG_3" => { - g.loop_conv.kernel_dim = xs2 & 0xff; - g.loop_conv.pool_size = (xs2 >> 8) & 0xff; - g.loop_conv.pool_stride = (xs2 >> 16) & 0xff; - g.loop_conv.pool_padding = (xs2 >> 24) & 0xff; - } - "GEMMINI_LOOP_CONV_WS_CONFIG_4" => g.loop_conv.addr_bias = xs2 & ((1u64 << 39) - 1), - "GEMMINI_LOOP_CONV_WS_CONFIG_5" => g.loop_conv.addr_input = xs2 & ((1u64 << 39) - 1), - "GEMMINI_LOOP_CONV_WS_CONFIG_6" => g.loop_conv.addr_weight = xs2 & ((1u64 << 39) - 1), - "GEMMINI_LOOP_CONV_WS_CONFIG_7" => g.loop_conv.addr_output = xs2 & ((1u64 << 39) - 1), - "GEMMINI_LOOP_CONV_WS_CONFIG_8" => { - g.loop_conv.input_stride = xs2 & 0xffff_ffff; - g.loop_conv.weight_stride = xs2 >> 32; - } - "GEMMINI_LOOP_CONV_WS_CONFIG_9" => g.loop_conv.output_stride = xs2 & 0xffff_ffff, - _ => panic!("gemmini_loop_conv_ws: unknown cfg mnemonic={mnemonic}"), - } +fn exec_cfg_impl(mnemonic: &str, xs2: u64, ctx: &mut ExecContext) -> u64 { + let field: usize = mnemonic.strip_prefix("GEMMINI_LOOP_CONV_WS_CONFIG_").expect("convolution configuration instruction").parse().unwrap(); + assert!((1..=9).contains(&field)); + ctx.write_metadata_word(64 + (field - 1) * 8, xs2); 0 } @@ -75,7 +48,7 @@ fn input_addr( let element = checked_add( checked_mul(spatial, st.in_ch, "input channels"), kch, - "input channel tile", + "input channel core", ); checked_add(st.addr_input, element, "input address") } @@ -138,8 +111,17 @@ fn output_addr( } fn exec_loop_impl(xs2: u64, ctx: &mut ExecContext) -> u64 { - let st = gemini().lock().unwrap().loop_conv.clone(); - let dataflow = gemini().lock().unwrap().cfg.dataflow; + let words: Vec<_> = (0..9).map(|i| ctx.metadata_word(64+i*8)).collect(); + let st = super::gemmini_state::LoopConvCfg { + batch: words[0] & 0xffff, in_dim: words[0] >> 16 & 0xffff, in_ch: words[0] >> 32 & 0xffff, + out_ch: words[1] & 0xffff, out_dim: words[1] >> 16 & 0xffff, + stride: words[1] >> 32 & 0xff, padding: words[1] >> 40 & 0xff, + kernel_dim: words[2] & 0xff, pool_size: words[2] >> 8 & 0xff, + pool_stride: words[2] >> 16 & 0xff, pool_padding: words[2] >> 24 & 0xff, + addr_bias: words[3], addr_input: words[4], addr_weight: words[5], addr_output: words[6], + input_stride: words[7] & 0xffff_ffff, weight_stride: words[7] >> 32, output_stride: words[8] & 0xffff_ffff, + }; + let dataflow = gemini(ctx).lock().unwrap().cfg.dataflow; let bank_input = xs2 & 0x3ff; let bank_weight = (xs2 >> 10) & 0x3ff; let bank_output = (xs2 >> 20) & 0x3ff; @@ -194,7 +176,7 @@ fn exec_loop_impl(xs2: u64, ctx: &mut ExecContext) -> u64 { let weight_stride = checked_stride( st.out_ch, bank_bytes, - "gemmini_loop_conv_ws weight tile stride", + "gemmini_loop_conv_ws weight core stride", ); let output_stride = checked_stride( st.output_stride, @@ -275,7 +257,7 @@ fn exec_loop_impl(xs2: u64, ctx: &mut ExecContext) -> u64 { pub fn execute(mnemonic: &str, xs2: u64, ctx: &mut ExecContext) -> u64 { match mnemonic { "GEMMINI_LOOP_CONV_WS" => exec_loop_impl(xs2, ctx), - _ => exec_cfg_impl(mnemonic, xs2), + _ => exec_cfg_impl(mnemonic, xs2, ctx), } } diff --git a/examples/prefab/balls/gemmini/emu/src/dispatch.rs b/examples/prefab/balls/gemmini/emu/src/dispatch.rs new file mode 100644 index 000000000..d59051add --- /dev/null +++ b/examples/prefab/balls/gemmini/emu/src/dispatch.rs @@ -0,0 +1,58 @@ +use bebop_bemu::ball::{BallModel, ExecContext}; + +use super::{ + f02_gemmini_config, f03_gemmini_flush, f21_gemmini_preload, f66_gemmini_compute_preloaded, + f67_gemmini_compute_accumulated, f80_gemmini_loop_ws, f96_gemmini_loop_conv_ws, +}; + +const BALL_CLASS: &str = "examples.balls.gemmini.GemminiBall"; + +pub fn execute_known( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, + ctx: &mut ExecContext, +) -> Option { + if ball_class != BALL_CLASS { + return None; + } + let mnemonic = bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct)?; + let result = match mnemonic.as_str() { + "GEMMINI_CONFIG" => f02_gemmini_config::GemminiConfig::exec(xs1, xs2, ctx), + "GEMMINI_FLUSH" => f03_gemmini_flush::GemminiFlush::exec(xs1, xs2, ctx), + "GEMMINI_PRELOAD" => f21_gemmini_preload::GemminiPreload::exec(xs1, xs2, ctx), + "GEMMINI_COMPUTE_PRELOADED" => { + f66_gemmini_compute_preloaded::GemminiComputePreloaded::exec(xs1, xs2, ctx) + } + "GEMMINI_COMPUTE_ACCUMULATED" => { + f67_gemmini_compute_accumulated::GemminiComputeAccumulated::exec(xs1, xs2, ctx) + } + name if name.starts_with("GEMMINI_LOOP_WS_") || name == "GEMMINI_LOOP_WS" => { + f80_gemmini_loop_ws::execute(name, xs2, ctx) + } + name if name.starts_with("GEMMINI_LOOP_CONV_WS_") || name == "GEMMINI_LOOP_CONV_WS" => { + f96_gemmini_loop_conv_ws::execute(name, xs2, ctx) + } + _ => return None, + }; + Some(result) +} + +pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { + if ball_class != BALL_CLASS { + return None; + } + let mnemonic = bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct)?; + let latency = match mnemonic.as_str() { + "GEMMINI_CONFIG" => f02_gemmini_config::GemminiConfig::latency(xs1, xs2), + "GEMMINI_FLUSH" => f03_gemmini_flush::GemminiFlush::latency(xs1, xs2), + "GEMMINI_PRELOAD" => f21_gemmini_preload::GemminiPreload::latency(xs1, xs2), + "GEMMINI_COMPUTE_PRELOADED" => f66_gemmini_compute_preloaded::GemminiComputePreloaded::latency(xs1, xs2), + "GEMMINI_COMPUTE_ACCUMULATED" => f67_gemmini_compute_accumulated::GemminiComputeAccumulated::latency(xs1, xs2), + name if name.starts_with("GEMMINI_LOOP_WS_") || name == "GEMMINI_LOOP_WS" => f80_gemmini_loop_ws::latency(name), + name if name.starts_with("GEMMINI_LOOP_CONV_WS_") || name == "GEMMINI_LOOP_CONV_WS" => f96_gemmini_loop_conv_ws::latency(name), + _ => return None, + }; + Some(latency) +} diff --git a/examples/balls/gemmini/emu/src/gemmini_state.rs b/examples/prefab/balls/gemmini/emu/src/gemmini_state.rs similarity index 85% rename from examples/balls/gemmini/emu/src/gemmini_state.rs rename to examples/prefab/balls/gemmini/emu/src/gemmini_state.rs index 9038fba23..615eae5bd 100644 --- a/examples/balls/gemmini/emu/src/gemmini_state.rs +++ b/examples/prefab/balls/gemmini/emu/src/gemmini_state.rs @@ -1,5 +1,6 @@ -//! Global config for Gemmini / loop instructions. -use std::sync::{Mutex, OnceLock}; +//! State owned by one Gemmini Ball model instance. +use std::sync::{Arc, Mutex}; +use super::instruction::ExecContext; #[derive(Clone, Default)] pub struct GemminiCfg { @@ -49,16 +50,12 @@ pub struct LoopConvCfg { #[derive(Default)] pub struct GemminiState { pub cfg: GemminiCfg, - pub loop_ws: LoopWsCfg, - pub loop_conv: LoopConvCfg, /// WS preload: B weights (iter × 16) i8 pub ws_b: Option>>, } -static GEMINI: OnceLock> = OnceLock::new(); - -pub fn gemini() -> &'static Mutex { - GEMINI.get_or_init(|| Mutex::new(GemminiState::default())) +pub fn gemini(ctx: &mut ExecContext) -> Arc> { + ctx.state::() } pub fn in_shift(v: i32, shift: u32) -> i32 { diff --git a/examples/prefab/balls/gemmini/emu/src/lib.rs b/examples/prefab/balls/gemmini/emu/src/lib.rs new file mode 100644 index 000000000..c0a788641 --- /dev/null +++ b/examples/prefab/balls/gemmini/emu/src/lib.rs @@ -0,0 +1,26 @@ +pub(crate) use bebop_bemu::{thread::{instructions::decode, bank_matrix}, ball as instruction}; + +#[path = "02_gemmini_config.rs"] +mod f02_gemmini_config; +#[path = "03_gemmini_flush.rs"] +mod f03_gemmini_flush; +#[path = "21_gemmini_preload.rs"] +mod f21_gemmini_preload; +#[path = "66_gemmini_compute_preloaded.rs"] +mod f66_gemmini_compute_preloaded; +#[path = "67_gemmini_compute_accumulated.rs"] +mod f67_gemmini_compute_accumulated; +#[path = "80_gemmini_loop_ws.rs"] +mod f80_gemmini_loop_ws; +#[path = "96_gemmini_loop_conv_ws.rs"] +mod f96_gemmini_loop_conv_ws; +mod gemmini_state; +mod loop_micro_ops; + +mod dispatch; + +pub use dispatch::{cycles_after_issue, execute_known}; + +#[bebop_bemu::ball::linkme::distributed_slice(bebop_bemu::BALLS)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static BALL: bebop_bemu::Ball = bebop_bemu::Ball { execute: execute_known, cycles: cycles_after_issue }; diff --git a/examples/balls/gemmini/emu/src/loop_micro_ops.rs b/examples/prefab/balls/gemmini/emu/src/loop_micro_ops.rs similarity index 87% rename from examples/balls/gemmini/emu/src/loop_micro_ops.rs rename to examples/prefab/balls/gemmini/emu/src/loop_micro_ops.rs index 2363ff88f..762184d45 100644 --- a/examples/balls/gemmini/emu/src/loop_micro_ops.rs +++ b/examples/prefab/balls/gemmini/emu/src/loop_micro_ops.rs @@ -1,8 +1,8 @@ use super::decode; -use super::f53_gemmini_preload::GemminiPreload; +use super::f21_gemmini_preload::GemminiPreload; use super::f66_gemmini_compute_preloaded::GemminiComputePreloaded; use super::f67_gemmini_compute_accumulated::GemminiComputeAccumulated; -use super::instruction::{BallInstruction, ExecContext}; +use super::instruction::{BallModel, ExecContext}; const FUNCT_MVOUT: u32 = 16; const FUNCT_MSET: u32 = 32; @@ -17,8 +17,8 @@ fn execute(funct: u32, xs1: u64, xs2: u64, ctx: &mut ExecContext) { fn banks(bank0: u64, bank1: u64, bank2: u64, iter: u64) -> u64 { assert!(bank0 < (1 << 10), "Gemmini loop bank0 exceeds 10 bits"); assert!(bank1 < (1 << 10), "Gemmini loop bank1 exceeds 10 bits"); - assert!(bank2 < (1 << 10), "Gemmini loop bank2 exceeds 10 bits"); - assert!(iter < (1 << 34), "Gemmini loop iter exceeds 34 bits"); + assert!(bank2 < (1 << 9), "Gemmini loop bank2 exceeds 9 bits"); + assert!(iter < (1 << 33), "Gemmini loop iter exceeds 33 bits"); bank0 | (bank1 << 10) | (bank2 << 20) | (iter << 30) } @@ -80,7 +80,8 @@ pub fn compute( ) { let mode = if accumulated { 3 } else { 2 }; let xs2 = mode | ((zero_op2 as u64) << 4) | ((zero_op1_tail as u64) << 5); - let xs1 = banks(op1, op2, output, iter); + // The accumulated compute reads the old output (rs1[29]), as Loop sets rd_wr. + let xs1 = banks(op1, op2, output, iter) | ((accumulated as u64) << 29); if accumulated { GemminiComputeAccumulated::exec(xs1, xs2, ctx); } else { diff --git a/examples/prefab/balls/gemmini/stack/src/gemminiOps.cpp b/examples/prefab/balls/gemmini/stack/src/gemminiOps.cpp new file mode 100644 index 000000000..f5b2ce973 --- /dev/null +++ b/examples/prefab/balls/gemmini/stack/src/gemminiOps.cpp @@ -0,0 +1,115 @@ +//===- gemminiOps.cpp - GemminiBall operations ----------------------------===// +// +// The dialect of GemminiBall, its virtual operations building their physical +// ones and the instructions of the physical operations. +// +//===----------------------------------------------------------------------===// + +#include "gemminiOps.h" + +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::gemmini_ball; +using mlir::buddy::bankFields; +using mlir::buddy::constantI64; +using mlir::buddy::emitInstruction; +using mlir::buddy::field; + +#include "gemminiDialect.cpp.inc" + +#define GET_OP_CLASSES +#include "gemminiOps.cpp.inc" + +void GemminiBallDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "gemminiOps.cpp.inc" + >(); +} + +namespace { +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallDialect([](DialectRegistry ®istry) { + registry.insert(); + }), + true); +} // namespace + +SmallVector BankMatmulI8TileOp::getSourceBanks() { + return {getA(), getB()}; +} +Value BankMatmulI8TileOp::getDestinationBank() { return getC(); } +bool BankMatmulI8TileOp::readsDestination() { return getReadDestination(); } +Operation *BankMatmulI8TileOp::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return MatmulI8TileOp::create( + builder, getLoc(), builder.getI64IntegerAttr(sources[0]), + builder.getI64IntegerAttr(sources[1]), + builder.getI64IntegerAttr(destination), getKAttr(), + getReadDestinationAttr(), getShiftAttr()); +} + +SmallVector MatmulI8TileOp::getSourceBanks() { + return {int64_t(getA()), int64_t(getB())}; +} +int64_t MatmulI8TileOp::getDestinationBank() { return getC(); } +LogicalResult MatmulI8TileOp::lowerToInstructions(OpBuilder &builder) { + Location loc = getLoc(); + bool readDestination = getReadDestination(); + if (readDestination != getShift().has_value()) + return emitOpError( + "matmul_i8_tile takes a shift exactly with read_destination"); + int64_t k = getK(), a = getA(), b = getB(), c = getC(); + int64_t shift = readDestination ? int64_t(*getShift()) : 0; + int64_t meshRows = buckyball_target::getBuckyballBallParam( + "GEMMINI_COMPUTE_PRELOADED", "meshRows"); + int64_t meshColumns = buckyball_target::getBuckyballBallParam( + "GEMMINI_COMPUTE_PRELOADED", "meshColumns"); + const auto &target = buckyball_target::getBuckyballTarget(); + if (target.bankWidthBits != 128 || meshRows != 16 || meshColumns != 16 || + target.bankDepth % 16 || target.bankDepth > 1024 || a == b || k <= 0 || + k % 16 || k > 64 * target.bankDepth || a == c || b == c) + return emitOpError("matmul_i8_tile requires distinct A, B and C banks, k " + "a multiple of 16 within 64 groups and a 16x16 mesh"); + int64_t steps = k / 16, groupBlocks = target.bankDepth / 16; + uint64_t config = ~0ULL; + for (int64_t step = 0; step < steps; ++step) { + uint64_t group = step / groupBlocks, base = step % groupBlocks * 16; + bool accumulate = readDestination || step > 0; + // CONFIG: rs2[4] picks the weight-stationary array, rs2[40:9] the shift. + uint64_t word = + field(accumulate, 4, 4) | field(step == steps - 1 ? shift : 0, 9, 40); + if (word != config) + emitInstruction( + builder, loc, buckyball_target::getBuckyballFunct7("GEMMINI_CONFIG"), + constantI64(builder, loc, 0), constantI64(builder, loc, word)); + config = word; + // rs2: the subcommand in [3:0], the first and second source row bases in + // [15:6] and [25:16], their groups in [41:36] and [47:42]; C is at row 0. + uint64_t bases = field(base, 6, 15) | field(base, 16, 25) | + field(group, 36, 41) | field(group, 42, 47); + if (accumulate) { + emitInstruction( + builder, loc, + buckyball_target::getBuckyballFunct7("GEMMINI_COMPUTE_ACCUMULATED"), + constantI64(builder, loc, bankFields({a, b}, c, true, 16)), + constantI64(builder, loc, 3 | bases)); + continue; + } + // The output-stationary preload reads sixteen rows of B. + emitInstruction(builder, loc, + buckyball_target::getBuckyballFunct7("GEMMINI_PRELOAD"), + constantI64(builder, loc, bankFields({b}, c, false, 16)), + constantI64(builder, loc, + 1 | field(base, 6, 15) | field(group, 36, 41))); + emitInstruction( + builder, loc, + buckyball_target::getBuckyballFunct7("GEMMINI_COMPUTE_PRELOADED"), + constantI64(builder, loc, bankFields({a, b}, c, false, 16)), + constantI64(builder, loc, 2 | bases)); + } + return success(); +} diff --git a/examples/prefab/balls/gemmini/stack/src/gemminiOps.h b/examples/prefab/balls/gemmini/stack/src/gemminiOps.h new file mode 100644 index 000000000..ca9a31760 --- /dev/null +++ b/examples/prefab/balls/gemmini/stack/src/gemminiOps.h @@ -0,0 +1,20 @@ +//===- gemminiOps.h - GemminiBall operations ------------------------------===// + +#ifndef GEMMINI_OPS_H +#define GEMMINI_OPS_H + +#include "Interfaces/BankInterfaces.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "gemminiDialect.h.inc" + +#define GET_OP_CLASSES +#include "gemminiOps.h.inc" + +#endif // GEMMINI_OPS_H diff --git a/examples/prefab/balls/gemmini/stack/src/gemminiOps.td b/examples/prefab/balls/gemmini/stack/src/gemminiOps.td new file mode 100644 index 000000000..d758b655d --- /dev/null +++ b/examples/prefab/balls/gemmini/stack/src/gemminiOps.td @@ -0,0 +1,62 @@ +//===- gemminiOps.td - GemminiBall operations --------------*- tablegen -*-===// +// +// matmul_i8_tile: one 16x16 tile C = A @ B over `k`, a positive multiple of +// sixteen. A and B fill their groups in order, each group a bank depth of rows +// before the next, so a K deeper than a bank spans several groups (allocated +// together or gathered with bank_transfer). +// A: k / 16 blocks of sixteen rows of sixteen i8; row i of block r is +// A[i, 16r .. 16r + 15]. +// B: k rows of sixteen i8, B[0 .. k - 1, 0 .. 15]. +// C: a bank of four groups, sixteen rows of sixteen i32. +// With read_destination C = in_shift(C + A @ B, shift), rounding to nearest +// even; without, C = A @ B and there is no shift. +// +// This is not smatmul's matmul_i8: C holds one output row per bank row across +// four groups instead of four bank rows of one group, there is no bias row, and +// read_destination accumulates the whole old C with a rounding shift. +// +// The Ball issues one instruction per 16-deep block, the operand groups and row +// bases derived from the block index. Without read_destination the first block +// runs on the output-stationary array (PRELOAD primes D = 0); every other block +// accumulates on the weight-stationary array, reading the old C. Only the last +// block applies the shift. +// +//===----------------------------------------------------------------------===// + +#ifndef GEMMINI_OPS +#define GEMMINI_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Interfaces/BankInterfaces.td" + +def GemminiBall_Dialect : Dialect { + let name = "gemmini_ball"; + let cppNamespace = "::buddy::gemmini_ball"; + let summary = "The operations of GemminiBall"; +} + +class GemminiBall_Op traits = []> + : Op])>; + +def BankMatmulI8TileOp : GemminiBall_Op<"bank_matmul_i8_tile", + [DeclareOpInterfaceMethods]> { + let summary = "matmul_i8_tile on virtual banks"; + let arguments = (ins I64:$a, I64:$b, I64:$c, I64Attr:$k, + UnitAttr:$read_destination, + OptionalAttr:$shift); + let results = (outs I64:$cOut); + let assemblyFormat = "$a `,` $b `->` $c prop-dict attr-dict"; +} + +def MatmulI8TileOp : GemminiBall_Op<"matmul_i8_tile", + [DeclareOpInterfaceMethods]> { + let summary = "matmul_i8_tile on physical banks"; + let arguments = (ins I64Attr:$a, I64Attr:$b, I64Attr:$c, I64Attr:$k, + UnitAttr:$read_destination, + OptionalAttr:$shift); + let assemblyFormat = "prop-dict attr-dict"; +} + +#endif // GEMMINI_OPS diff --git a/examples/balls/gemmini/verify/casegen/Cargo.lock b/examples/prefab/balls/gemmini/verify/casegen/Cargo.lock similarity index 100% rename from examples/balls/gemmini/verify/casegen/Cargo.lock rename to examples/prefab/balls/gemmini/verify/casegen/Cargo.lock diff --git a/examples/balls/gemmini/verify/casegen/Cargo.toml b/examples/prefab/balls/gemmini/verify/casegen/Cargo.toml similarity index 100% rename from examples/balls/gemmini/verify/casegen/Cargo.toml rename to examples/prefab/balls/gemmini/verify/casegen/Cargo.toml diff --git a/examples/balls/gemmini/verify/casegen/src/lib.rs b/examples/prefab/balls/gemmini/verify/casegen/src/lib.rs similarity index 100% rename from examples/balls/gemmini/verify/casegen/src/lib.rs rename to examples/prefab/balls/gemmini/verify/casegen/src/lib.rs diff --git a/examples/prefab/balls/gemmini/verify/filelists/gemmini_ball.f b/examples/prefab/balls/gemmini/verify/filelists/gemmini_ball.f new file mode 100644 index 000000000..94a5fe6c5 --- /dev/null +++ b/examples/prefab/balls/gemmini/verify/filelists/gemmini_ball.f @@ -0,0 +1,14 @@ ++incdir+@UVM@/src/core ++incdir+@UVM@/src/ball ++incdir+src ++incdir+src/pkg ++incdir+src/tests ++incdir+@RTL@ + +@UVM@/src/core/package.sv +@UVM@/src/ball/bb_blink_if.sv +src/gemmini_expansion_if.sv +-F @RTL@/filelist.f + +src/pkg/gemmini_pkg.sv +src/tb_top.sv diff --git a/examples/prefab/balls/gemmini/verify/src/common/gemmini_defs.svh b/examples/prefab/balls/gemmini/verify/src/common/gemmini_defs.svh new file mode 100644 index 000000000..91abb7a33 --- /dev/null +++ b/examples/prefab/balls/gemmini/verify/src/common/gemmini_defs.svh @@ -0,0 +1,12 @@ +`ifndef GEMMINI_CONFIG_FUNCT7 +`error "GEMMINI_CONFIG_FUNCT7 must be provided by the selected Thread ballISA" +`endif +`ifndef GEMMINI_LOOP_WS_CONFIG_BOUNDS_FUNCT7 +`error "GEMMINI_LOOP_WS_CONFIG_BOUNDS_FUNCT7 must be provided by the selected Thread ballISA" +`endif +`ifndef GEMMINI_PRELOAD_FUNCT7 +`error "GEMMINI_PRELOAD_FUNCT7 must be provided by the selected Thread ballISA" +`endif +`ifndef GEMMINI_COMPUTE_PRELOADED_FUNCT7 +`error "GEMMINI_COMPUTE_PRELOADED_FUNCT7 must be provided by the selected Thread ballISA" +`endif diff --git a/examples/prefab/balls/gemmini/verify/src/gemmini_expansion_if.sv b/examples/prefab/balls/gemmini/verify/src/gemmini_expansion_if.sv new file mode 100644 index 000000000..0bd797428 --- /dev/null +++ b/examples/prefab/balls/gemmini/verify/src/gemmini_expansion_if.sv @@ -0,0 +1,17 @@ +// SubROB expansion port of GemminiBall: loop configuration and loop triggers. +interface gemmini_expansion_if; + logic command_ready; + logic command_valid; + logic [`BB_ROB_ID_W-1:0] command_bits_rob_id; + logic [ 6:0] command_bits_funct; + logic [ 63:0] command_bits_rs2; + logic accepts; + logic starts; + logic complete_ready; + logic complete_valid; + logic [`BB_ROB_ID_W-1:0] complete_bits_rob_id; + logic write_ready; + logic write_valid; + logic [ 9:0] write_bits_addr; + logic [ 7:0] write_bits_data; +endinterface diff --git a/examples/balls/gemmini/verify/src/pkg/gemmini_pkg.sv b/examples/prefab/balls/gemmini/verify/src/pkg/gemmini_pkg.sv similarity index 100% rename from examples/balls/gemmini/verify/src/pkg/gemmini_pkg.sv rename to examples/prefab/balls/gemmini/verify/src/pkg/gemmini_pkg.sv diff --git a/examples/prefab/balls/gemmini/verify/src/tb_top.sv b/examples/prefab/balls/gemmini/verify/src/tb_top.sv new file mode 100644 index 000000000..a7d7fe42a --- /dev/null +++ b/examples/prefab/balls/gemmini/verify/src/tb_top.sv @@ -0,0 +1,186 @@ +module tb_top; + import uvm_pkg::*; + import gemmini_pkg::*; + + bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); + gemmini_expansion_if expansion (); + + GemminiBall dut ( + .clock(intf.clock), + .reset(intf.reset), + .expansion_command_ready(expansion.command_ready), + .expansion_command_valid(expansion.command_valid), + .expansion_command_bits_rob_id(expansion.command_bits_rob_id), + .expansion_command_bits_funct(expansion.command_bits_funct), + .expansion_command_bits_rs2(expansion.command_bits_rs2), + .expansion_accepts(expansion.accepts), + .expansion_starts(expansion.starts), + .expansion_rows_ready(1'b1), + .expansion_complete_ready(expansion.complete_ready), + .expansion_complete_valid(expansion.complete_valid), + .expansion_complete_bits_rob_id(expansion.complete_bits_rob_id), + .expansion_read_req_ready(1'b0), + .expansion_read_resp_valid(1'b0), + .expansion_read_resp_bits_data(8'h00), + .expansion_write_ready(expansion.write_ready), + .expansion_write_valid(expansion.write_valid), + .expansion_write_bits_addr(expansion.write_bits_addr), + .expansion_write_bits_data(expansion.write_bits_data), + .blink_status_running(intf.status_running), + .blink_cmdReq_ready(intf.cmd_req_ready), + .blink_cmdReq_valid(intf.cmd_req_valid), + .blink_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .blink_cmdReq_bits_funct(intf.cmd_req_bits_funct), + .blink_cmdReq_bits_rs2(intf.cmd_req_bits_rs2), + .blink_cmdReq_bits_iter(intf.cmd_req_bits_iter), + .blink_cmdReq_bits_op1_bank(intf.cmd_req_bits_op1_bank), + .blink_cmdReq_bits_op2_bank(intf.cmd_req_bits_op2_bank), + .blink_cmdReq_bits_wr_bank(intf.cmd_req_bits_wr_bank), + .blink_cmdReq_bits_op1_group(intf.cmd_req_bits_op1_group), + .blink_cmdReq_bits_op2_group(intf.cmd_req_bits_op2_group), + .blink_cmdReq_bits_wr_group(intf.cmd_req_bits_wr_group), + .blink_cmdReq_bits_op1_col(intf.cmd_req_bits_op1_col), + .blink_cmdReq_bits_op2_col(intf.cmd_req_bits_op2_col), + .blink_cmdReq_bits_wr_col(intf.cmd_req_bits_wr_col), + .blink_cmdReq_bits_wr_read_bank(intf.cmd_req_bits_wr_read_bank), + .blink_cmdReq_bits_wr_arch_bank(intf.cmd_req_bits_wr_arch_bank), + .blink_cmdResp_ready(intf.cmd_resp_ready), + .blink_cmdResp_valid(intf.cmd_resp_valid), + .blink_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .blink_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .blink_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .blink_bankRead_0_group_id(intf.bank_read_group_id[0]), + .blink_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .blink_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .blink_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .blink_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .blink_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .blink_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .blink_bankRead_1_bank_id(intf.bank_read_bank_id[1]), + .blink_bankRead_1_rob_id(intf.bank_read_rob_id[1]), + .blink_bankRead_1_group_id(intf.bank_read_group_id[1]), + .blink_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), + .blink_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), + .blink_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), + .blink_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), + .blink_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), + .blink_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), + .blink_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .blink_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .blink_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .blink_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .blink_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .blink_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .blink_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), + .blink_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), + .blink_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), + .blink_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), + .blink_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), + .blink_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), + .blink_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), + .blink_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), + .blink_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), + .blink_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), + .blink_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), + .blink_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), + .blink_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), + .blink_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), + .blink_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), + .blink_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), + .blink_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .blink_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .blink_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), + .blink_bankWrite_1_bank_id(intf.bank_write_bank_id[1]), + .blink_bankWrite_1_rob_id(intf.bank_write_rob_id[1]), + .blink_bankWrite_1_group_id(intf.bank_write_group_id[1]), + .blink_bankWrite_1_io_req_ready(intf.bank_write_req_ready[1]), + .blink_bankWrite_1_io_req_valid(intf.bank_write_req_valid[1]), + .blink_bankWrite_1_io_req_bits_addr(intf.bank_write_req_addr[1]), + .blink_bankWrite_1_io_req_bits_mask_0(intf.bank_write_req_mask[1][0]), + .blink_bankWrite_1_io_req_bits_mask_1(intf.bank_write_req_mask[1][1]), + .blink_bankWrite_1_io_req_bits_mask_2(intf.bank_write_req_mask[1][2]), + .blink_bankWrite_1_io_req_bits_mask_3(intf.bank_write_req_mask[1][3]), + .blink_bankWrite_1_io_req_bits_mask_4(intf.bank_write_req_mask[1][4]), + .blink_bankWrite_1_io_req_bits_mask_5(intf.bank_write_req_mask[1][5]), + .blink_bankWrite_1_io_req_bits_mask_6(intf.bank_write_req_mask[1][6]), + .blink_bankWrite_1_io_req_bits_mask_7(intf.bank_write_req_mask[1][7]), + .blink_bankWrite_1_io_req_bits_mask_8(intf.bank_write_req_mask[1][8]), + .blink_bankWrite_1_io_req_bits_mask_9(intf.bank_write_req_mask[1][9]), + .blink_bankWrite_1_io_req_bits_mask_10(intf.bank_write_req_mask[1][10]), + .blink_bankWrite_1_io_req_bits_mask_11(intf.bank_write_req_mask[1][11]), + .blink_bankWrite_1_io_req_bits_mask_12(intf.bank_write_req_mask[1][12]), + .blink_bankWrite_1_io_req_bits_mask_13(intf.bank_write_req_mask[1][13]), + .blink_bankWrite_1_io_req_bits_mask_14(intf.bank_write_req_mask[1][14]), + .blink_bankWrite_1_io_req_bits_mask_15(intf.bank_write_req_mask[1][15]), + .blink_bankWrite_1_io_req_bits_data(intf.bank_write_req_data[1]), + .blink_bankWrite_1_io_resp_valid(intf.bank_write_resp_valid[1]), + .blink_bankWrite_1_io_resp_bits_ok(intf.bank_write_resp_ok[1]), + .blink_bankWrite_2_bank_id(intf.bank_write_bank_id[2]), + .blink_bankWrite_2_rob_id(intf.bank_write_rob_id[2]), + .blink_bankWrite_2_group_id(intf.bank_write_group_id[2]), + .blink_bankWrite_2_io_req_ready(intf.bank_write_req_ready[2]), + .blink_bankWrite_2_io_req_valid(intf.bank_write_req_valid[2]), + .blink_bankWrite_2_io_req_bits_addr(intf.bank_write_req_addr[2]), + .blink_bankWrite_2_io_req_bits_mask_0(intf.bank_write_req_mask[2][0]), + .blink_bankWrite_2_io_req_bits_mask_1(intf.bank_write_req_mask[2][1]), + .blink_bankWrite_2_io_req_bits_mask_2(intf.bank_write_req_mask[2][2]), + .blink_bankWrite_2_io_req_bits_mask_3(intf.bank_write_req_mask[2][3]), + .blink_bankWrite_2_io_req_bits_mask_4(intf.bank_write_req_mask[2][4]), + .blink_bankWrite_2_io_req_bits_mask_5(intf.bank_write_req_mask[2][5]), + .blink_bankWrite_2_io_req_bits_mask_6(intf.bank_write_req_mask[2][6]), + .blink_bankWrite_2_io_req_bits_mask_7(intf.bank_write_req_mask[2][7]), + .blink_bankWrite_2_io_req_bits_mask_8(intf.bank_write_req_mask[2][8]), + .blink_bankWrite_2_io_req_bits_mask_9(intf.bank_write_req_mask[2][9]), + .blink_bankWrite_2_io_req_bits_mask_10(intf.bank_write_req_mask[2][10]), + .blink_bankWrite_2_io_req_bits_mask_11(intf.bank_write_req_mask[2][11]), + .blink_bankWrite_2_io_req_bits_mask_12(intf.bank_write_req_mask[2][12]), + .blink_bankWrite_2_io_req_bits_mask_13(intf.bank_write_req_mask[2][13]), + .blink_bankWrite_2_io_req_bits_mask_14(intf.bank_write_req_mask[2][14]), + .blink_bankWrite_2_io_req_bits_mask_15(intf.bank_write_req_mask[2][15]), + .blink_bankWrite_2_io_req_bits_data(intf.bank_write_req_data[2]), + .blink_bankWrite_2_io_resp_valid(intf.bank_write_resp_valid[2]), + .blink_bankWrite_2_io_resp_bits_ok(intf.bank_write_resp_ok[2]), + .blink_bankWrite_3_bank_id(intf.bank_write_bank_id[3]), + .blink_bankWrite_3_rob_id(intf.bank_write_rob_id[3]), + .blink_bankWrite_3_group_id(intf.bank_write_group_id[3]), + .blink_bankWrite_3_io_req_ready(intf.bank_write_req_ready[3]), + .blink_bankWrite_3_io_req_valid(intf.bank_write_req_valid[3]), + .blink_bankWrite_3_io_req_bits_addr(intf.bank_write_req_addr[3]), + .blink_bankWrite_3_io_req_bits_mask_0(intf.bank_write_req_mask[3][0]), + .blink_bankWrite_3_io_req_bits_mask_1(intf.bank_write_req_mask[3][1]), + .blink_bankWrite_3_io_req_bits_mask_2(intf.bank_write_req_mask[3][2]), + .blink_bankWrite_3_io_req_bits_mask_3(intf.bank_write_req_mask[3][3]), + .blink_bankWrite_3_io_req_bits_mask_4(intf.bank_write_req_mask[3][4]), + .blink_bankWrite_3_io_req_bits_mask_5(intf.bank_write_req_mask[3][5]), + .blink_bankWrite_3_io_req_bits_mask_6(intf.bank_write_req_mask[3][6]), + .blink_bankWrite_3_io_req_bits_mask_7(intf.bank_write_req_mask[3][7]), + .blink_bankWrite_3_io_req_bits_mask_8(intf.bank_write_req_mask[3][8]), + .blink_bankWrite_3_io_req_bits_mask_9(intf.bank_write_req_mask[3][9]), + .blink_bankWrite_3_io_req_bits_mask_10(intf.bank_write_req_mask[3][10]), + .blink_bankWrite_3_io_req_bits_mask_11(intf.bank_write_req_mask[3][11]), + .blink_bankWrite_3_io_req_bits_mask_12(intf.bank_write_req_mask[3][12]), + .blink_bankWrite_3_io_req_bits_mask_13(intf.bank_write_req_mask[3][13]), + .blink_bankWrite_3_io_req_bits_mask_14(intf.bank_write_req_mask[3][14]), + .blink_bankWrite_3_io_req_bits_mask_15(intf.bank_write_req_mask[3][15]), + .blink_bankWrite_3_io_req_bits_data(intf.bank_write_req_data[3]), + .blink_bankWrite_3_io_resp_valid(intf.bank_write_resp_valid[3]), + .blink_bankWrite_3_io_resp_bits_ok(intf.bank_write_resp_ok[3]) + ); + + // Constant outputs removed from the port list by firtool. + assign intf.bank_write_resp_ready[0] = dut.blink_bankWrite_0_io_resp_ready; + assign intf.bank_write_resp_ready[1] = dut.blink_bankWrite_1_io_resp_ready; + assign intf.bank_write_resp_ready[2] = dut.blink_bankWrite_2_io_resp_ready; + assign intf.bank_write_resp_ready[3] = dut.blink_bankWrite_3_io_resp_ready; + + initial begin + intf.clock = 1'b0; + forever #5 intf.clock = ~intf.clock; + end + + initial begin + uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); + uvm_config_db#(virtual gemmini_expansion_if)::set(null, "*", "expansion", expansion); + run_test("gemmini_ball_test"); + end +endmodule diff --git a/examples/prefab/balls/gemmini/verify/src/tests/gemmini_test.svh b/examples/prefab/balls/gemmini/verify/src/tests/gemmini_test.svh new file mode 100644 index 000000000..8b183bcf6 --- /dev/null +++ b/examples/prefab/balls/gemmini/verify/src/tests/gemmini_test.svh @@ -0,0 +1,283 @@ +class gemmini_ball_test extends uvm_test; + `uvm_component_utils(gemmini_ball_test) + + typedef virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) vif_t; + vif_t vif; + virtual gemmini_expansion_if xif; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(vif_t)::get(this, "", "vif", vif)) + `uvm_fatal("NOVIF", "bb_blink_if not found") + if (!uvm_config_db#(virtual gemmini_expansion_if)::get(this, "", "expansion", xif)) + `uvm_fatal("NOVIF", "gemmini_expansion_if not found") + endfunction + + task run_phase(uvm_phase phase); + phase.raise_objection(this); + apply_reset(); + run_config(); + run_loop_bounds(); + run_group_select(); + phase.drop_objection(this); + endtask + + // bb_gemmini_config(dataflow=1, activation=0, a_transpose=1, b_transpose=0, in_shift=3): + // a BLink command answered by the execution controller without bank traffic. + task run_config(); + int cycles; + bit [63:0] rs2 = (64'd1 << 4) | (64'd1 << 7) | (64'd3 << 9); + @(negedge vif.clock); + vif.cmd_req_bits_rob_id = 'h5; + vif.cmd_req_bits_funct = `GEMMINI_CONFIG_FUNCT7; + vif.cmd_req_bits_rs2 = rs2; + vif.cmd_req_bits_iter = '0; + vif.cmd_req_bits_op1_bank = '0; + vif.cmd_req_bits_op2_bank = '0; + vif.cmd_req_bits_wr_bank = '0; + vif.cmd_req_bits_op1_group = '0; + vif.cmd_req_bits_op2_group = '0; + vif.cmd_req_bits_wr_group = '0; + vif.cmd_req_bits_op1_col = '0; + vif.cmd_req_bits_op2_col = '0; + vif.cmd_req_bits_wr_col = '0; + vif.cmd_req_bits_wr_read_bank = '0; + vif.cmd_req_bits_wr_arch_bank = '0; + vif.cmd_req_valid = 1'b1; + cycles = 0; + do begin + @(posedge vif.clock); + cycles++; + if (cycles > 20) `uvm_fatal("TIMEOUT", "Gemmini config command was not accepted") + end while (!vif.cmd_req_ready); + @(negedge vif.clock); + vif.cmd_req_valid = 1'b0; + + cycles = 0; + while (!vif.cmd_resp_valid) begin + check_no_bank_traffic("config"); + @(negedge vif.clock); + cycles++; + if (cycles > 20) `uvm_fatal("RESP", "Gemmini config response was not observed") + end + verification_pkg::evidence::checks++; + if (vif.cmd_resp_bits_rob_id !== 'h5) + `uvm_fatal("RESP", $sformatf("rob_id mismatch: got %0h", vif.cmd_resp_bits_rob_id)) + @(negedge vif.clock); + `uvm_info("GEMMINI", "config command passed", UVM_LOW) + endtask + + // bb_gemmini_loop_ws_config_bounds(max_i=1, max_j=2, max_k=3) arrives on the SubROB + // expansion port and is stored little-endian into loop metadata bytes 0..7. + task run_loop_bounds(); + int cycles; + int unsigned writes; + bit [63:0] rs2 = 64'h0001_0002_0003; + @(negedge vif.clock); + xif.command_bits_rob_id = 'h9; + xif.command_bits_funct = `GEMMINI_LOOP_WS_CONFIG_BOUNDS_FUNCT7; + xif.command_bits_rs2 = rs2; + xif.command_valid = 1'b1; + #1; + verification_pkg::evidence::checks++; + if (xif.accepts !== 1'b1 || xif.starts !== 1'b0) + `uvm_fatal("EXPANSION", "loop bounds config must be accepted without starting a loop") + cycles = 0; + do begin + @(posedge vif.clock); + cycles++; + if (cycles > 20) `uvm_fatal("TIMEOUT", "loop bounds command was not accepted") + end while (!xif.command_ready); + @(negedge vif.clock); + xif.command_valid = 1'b0; + + writes = 0; + cycles = 0; + while (!xif.complete_valid) begin + check_no_bank_traffic("loop bounds"); + if (xif.write_valid) begin + verification_pkg::evidence::checks++; + if (xif.write_bits_addr !== writes || xif.write_bits_data !== rs2[writes*8+:8]) + `uvm_fatal("EXPANSION", $sformatf( + "metadata write %0d: addr=%0d data=%0h", + writes, + xif.write_bits_addr, + xif.write_bits_data + )) + writes++; + end + @(negedge vif.clock); + cycles++; + if (cycles > 40) `uvm_fatal("TIMEOUT", "loop bounds command did not complete") + end + verification_pkg::evidence::checks++; + if (writes != 8 || xif.complete_bits_rob_id !== 'h9) + `uvm_fatal("EXPANSION", $sformatf( + "completion mismatch: writes=%0d rob=%0h", writes, xif.complete_bits_rob_id)) + @(negedge vif.clock); + `uvm_info("GEMMINI", "loop-ws bounds config passed", UVM_LOW) + endtask + + // Operands span three groups of their banks; rs2[41:36] and rs2[47:42] pick + // A's group 2 and B's group 1. The other groups hold different data, so a + // wrong group fails both the read check and the product. + localparam int GroupA = 2; + localparam int GroupB = 1; + + // Row `row` of group `group` of bank 1 (A) or bank 2 (B). + function bit [127:0] bank_row(int bank, int group, int row); + bit [127:0] data; + for (int lane = 0; lane < 16; lane++) + if (bank == 1) data[lane*8+:8] = group == GroupA ? 8'(lane == row % 16) : 8'd5; + else data[lane*8+:8] = group == GroupB ? 8'((row + 2 * lane + 1) % 7 - 3) : 8'd9; + return data; + endfunction + + // Issues one command and serves its bank traffic until it completes: + // reads answer one cycle after they fire, writes are recorded in c. + task issue(int rob, int funct, bit [63:0] rs2, int op1_bank, int op2_bank, + ref bit [31:0] c[16][16], ref int written); + bit read_fire[`BB_IN_BW], write_fire[`BB_OUT_BW]; + bit read_done[`BB_IN_BW], write_done[`BB_OUT_BW]; + int read_bank[`BB_IN_BW], read_group[`BB_IN_BW], read_addr[`BB_IN_BW]; + bit cmd_fire = 0; + int cycles; + @(negedge vif.clock); + vif.cmd_req_bits_rob_id = rob; + vif.cmd_req_bits_funct = funct; + vif.cmd_req_bits_rs2 = rs2; + vif.cmd_req_bits_iter = 16; + vif.cmd_req_bits_op1_bank = op1_bank; + vif.cmd_req_bits_op2_bank = op2_bank; + vif.cmd_req_bits_wr_bank = 3; + vif.cmd_req_bits_op1_group = '0; + vif.cmd_req_bits_op2_group = '0; + vif.cmd_req_bits_wr_group = '0; + vif.cmd_req_bits_op1_col = 3; + vif.cmd_req_bits_op2_col = 3; + vif.cmd_req_bits_wr_col = 4; + vif.cmd_req_bits_wr_read_bank = '0; + vif.cmd_req_bits_wr_arch_bank = 3; + vif.cmd_req_valid = 1'b1; + cycles = 0; + forever begin + // Values set at this negedge are sampled with the DUT outputs at the + // next posedge, so handshakes seen here fire at that edge. + if (cmd_fire) vif.cmd_req_valid = 1'b0; + for (int port = 0; port < `BB_IN_BW; port++) begin + if (read_done[port]) vif.bank_read_resp_valid[port] = 1'b0; + if (read_fire[port]) begin + if (vif.bank_read_resp_valid[port]) `uvm_fatal("BANK", "read response overrun") + vif.bank_read_resp_valid[port] = 1'b1; + vif.bank_read_resp_data[port] = + bank_row(read_bank[port], read_group[port], read_addr[port]); + end + end + for (int port = 0; port < `BB_OUT_BW; port++) begin + if (write_done[port]) vif.bank_write_resp_valid[port] = 1'b0; + if (write_fire[port]) vif.bank_write_resp_valid[port] = 1'b1; + end + #1; + cmd_fire = vif.cmd_req_valid && vif.cmd_req_ready; + for (int port = 0; port < `BB_IN_BW; port++) begin + read_done[port] = vif.bank_read_resp_valid[port] && vif.bank_read_resp_ready[port]; + read_fire[port] = vif.bank_read_req_valid[port] && vif.bank_read_req_ready[port]; + if (read_fire[port]) begin + read_bank[port] = vif.bank_read_bank_id[port]; + read_group[port] = vif.bank_read_group_id[port]; + read_addr[port] = vif.bank_read_req_addr[port]; + verification_pkg::evidence::checks++; + if (read_bank[port] != op1_bank && read_bank[port] != op2_bank) + `uvm_fatal("BANK", $sformatf("read of bank %0d", read_bank[port])) + if (read_group[port] != (read_bank[port] == 1 ? GroupA : GroupB)) + `uvm_fatal("GROUP", $sformatf( + "bank %0d read group %0d", read_bank[port], read_group[port])) + end + end + for (int port = 0; port < `BB_OUT_BW; port++) begin + write_done[port] = vif.bank_write_resp_valid[port] && vif.bank_write_resp_ready[port]; + write_fire[port] = vif.bank_write_req_valid[port] && vif.bank_write_req_ready[port]; + if (write_fire[port]) begin + int group = vif.bank_write_group_id[port]; + int row = vif.bank_write_req_addr[port]; + if (vif.bank_write_bank_id[port] != 3 || group > 3 || row > 15 || + vif.bank_write_req_mask[port] !== 16'hffff) + `uvm_fatal("BANK", $sformatf( + "write bank %0d group %0d row %0d", vif.bank_write_bank_id[port], group, row + )) + for (int lane = 0; lane < 4; lane++) + c[row][group*4+lane] = vif.bank_write_req_data[port][lane*32+:32]; + written++; + end + end + if (vif.cmd_resp_valid && vif.cmd_resp_ready) break; + @(negedge vif.clock); + if (++cycles > 2000) `uvm_fatal("TIMEOUT", $sformatf("command %0h did not complete", rs2)) + end + verification_pkg::evidence::checks++; + if (vif.cmd_resp_bits_rob_id !== rob) + `uvm_fatal("RESP", $sformatf("rob_id mismatch: got %0h", vif.cmd_resp_bits_rob_id)) + @(negedge vif.clock); + for (int port = 0; port < `BB_IN_BW; port++) vif.bank_read_resp_valid[port] = 1'b0; + for (int port = 0; port < `BB_OUT_BW; port++) vif.bank_write_resp_valid[port] = 1'b0; + endtask + + // CONFIG (output-stationary, no shift), PRELOAD of B and COMPUTE_PRELOADED of + // A @ B with A the identity: C must be B's selected group. + task run_group_select(); + bit [31:0] c[16][16]; + int written = 0; + issue('h6, `GEMMINI_CONFIG_FUNCT7, 64'd0, 0, 0, c, written); + issue('h7, `GEMMINI_PRELOAD_FUNCT7, 64'd1 | (64'(GroupB) << 36), 2, 0, c, written); + issue('h8, `GEMMINI_COMPUTE_PRELOADED_FUNCT7, 64'd2 | (64'(GroupA) << 36) | (64'(GroupB) << 42), + 1, 2, c, written); + verification_pkg::evidence::checks++; + if (written != 64) `uvm_fatal("WRITE", $sformatf("%0d row writes, expected 64", written)) + for (int row = 0; row < 16; row++) + for (int col = 0; col < 16; col++) + if ($signed(c[row][col]) != (row + 2 * col + 1) % 7 - 3) + `uvm_fatal("RESULT", $sformatf( + "C[%0d][%0d] = %0d, expected %0d", + row, + col, + $signed( + c[row][col] + ), + (row + 2 * col + 1) % 7 - 3 + )) + `uvm_info("GEMMINI", "operand group select passed", UVM_LOW) + endtask + + function void check_no_bank_traffic(string what); + for (int i = 0; i < `BB_IN_BW; i++) + if (vif.bank_read_req_valid[i]) `uvm_fatal("BANK", {what, " unexpectedly read a bank"}) + for (int i = 0; i < `BB_OUT_BW; i++) + if (vif.bank_write_req_valid[i]) `uvm_fatal("BANK", {what, " unexpectedly wrote a bank"}) + endfunction + + task apply_reset(); + vif.cmd_req_valid = 1'b0; + vif.cmd_resp_ready = 1'b1; + for (int i = 0; i < `BB_IN_BW; i++) begin + vif.bank_read_req_ready[i] = 1'b1; + vif.bank_read_resp_valid[i] = 1'b0; + vif.bank_read_resp_data[i] = '0; + end + for (int i = 0; i < `BB_OUT_BW; i++) begin + vif.bank_write_req_ready[i] = 1'b1; + vif.bank_write_resp_valid[i] = 1'b0; + vif.bank_write_resp_ok[i] = 1'b1; + end + xif.command_valid = 1'b0; + xif.complete_ready = 1'b1; + xif.write_ready = 1'b1; + vif.reset = 1'b1; + repeat (5) @(posedge vif.clock); + vif.reset = 1'b0; + repeat (2) @(posedge vif.clock); + endtask +endclass diff --git a/examples/prefab/balls/gemmini/workloads/ctests/CMakeLists.txt b/examples/prefab/balls/gemmini/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..50f8d0748 --- /dev/null +++ b/examples/prefab/balls/gemmini/workloads/ctests/CMakeLists.txt @@ -0,0 +1,16 @@ +add_buckyball_ctest(gemmini_loop_subrob.c) +add_buckyball_ctest(gemmini_loop_mmio.c) +add_buckyball_ctest(gemmini_accumulated_16x16_test.c) +add_buckyball_ctest(gemmini_os_cisc_atranspose_test.c) +add_buckyball_ctest(gemmini_os_cisc_basic_test.c) +add_buckyball_ctest(gemmini_os_cisc_btranspose_test.c) +add_buckyball_ctest(gemmini_os_cisc_shift_test.c) +add_buckyball_ctest(gemmini_os_risc_abtranspose_test.c) +add_buckyball_ctest(gemmini_os_risc_atranspose_test.c) +add_buckyball_ctest(gemmini_os_risc_basic_test.c) +add_buckyball_ctest(gemmini_os_risc_btranspose_test.c) +add_buckyball_ctest(gemmini_os_risc_shift_test.c) +add_buckyball_ctest(gemmini_ws_cisc_conv_test.c) +add_buckyball_ctest(gemmini_ws_risc_basic_test.c) +add_buckyball_ctest(gemmini_ws_risc_btranspose_test.c) +add_buckyball_ctest(gemmini_ws_risc_shift_test.c) diff --git a/examples/balls/gemmini/workloads/ctests/gemmini_accumulated_16x16_test.c b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_accumulated_16x16_test.c similarity index 100% rename from examples/balls/gemmini/workloads/ctests/gemmini_accumulated_16x16_test.c rename to examples/prefab/balls/gemmini/workloads/ctests/gemmini_accumulated_16x16_test.c diff --git a/examples/prefab/balls/gemmini/workloads/ctests/gemmini_loop_mmio.c b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_loop_mmio.c new file mode 100644 index 000000000..96072f44c --- /dev/null +++ b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_loop_mmio.c @@ -0,0 +1,56 @@ +#include +#include +#include + +#define DIM (THREAD_MEM_BANK_WIDTH / 8) +#define M DIM +#define N (2 * DIM) +#define K (2 * DIM) +static int8_t a[M * K] __attribute__((aligned(64))); +static int8_t b[K * N] __attribute__((aligned(64))); +static int32_t c[M * N] __attribute__((aligned(64))); +static int32_t expected[9][7]; +_Static_assert(DIM == 16, + "Gemmini loop tests require the configured 16x16 array"); +static uint64_t descriptor[8] __attribute__((aligned(64))); + +int main(void) { + for (int i = 0; i < M; ++i) + for (int k = 0; k < K; ++k) + a[i * K + k] = (i + 3 * k) % 9 - 4; + for (int k = 0; k < K; ++k) + for (int j = 0; j < N; ++j) + b[k * N + j] = (2 * k + 5 * j) % 7 - 3; + for (int i = 0; i < M * N; ++i) + c[i] = 0x2468ace0; + descriptor[0] = + ((uint64_t)(M / DIM) << 32) | ((uint64_t)(N / DIM) << 16) | (K / DIM); + descriptor[1] = (uintptr_t)a; + descriptor[2] = (uintptr_t)b; + descriptor[3] = 0; + descriptor[4] = (uintptr_t)c; + descriptor[5] = ((uint64_t)N << 32) | K; + descriptor[6] = (uint64_t)(N * 4) << 32; + descriptor[7] = 0; + bb_gemmini_config(0, 0, 0, 0, 0); + bb_mvin_mmio((uintptr_t)descriptor, 0, + sizeof(descriptor) / (THREAD_MEM_BANK_WIDTH / 8), + THREAD_MEM_BANK_WIDTH / 8); + bb_gemmini_loop_ws(0, 1, 2, 1); + bb_barrier(); + asm volatile("fence" ::: "memory"); + // Input rows/columns repeat with these periods; every output is still + // checked. + for (int i = 0; i < 9; ++i) + for (int j = 0; j < 7; ++j) { + int32_t sum = 0; + for (int k = 0; k < K; ++k) + sum += a[i * K + k] * b[k * N + j]; + expected[i][j] = sum; + } + for (int i = 0; i < M; ++i) + for (int j = 0; j < N; ++j) + if (c[i * N + j] != expected[i % 9][j % 7]) + return i * N + j + 1; + return 0; +} diff --git a/examples/prefab/balls/gemmini/workloads/ctests/gemmini_loop_subrob.c b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_loop_subrob.c new file mode 100644 index 000000000..db9bedaf7 --- /dev/null +++ b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_loop_subrob.c @@ -0,0 +1,50 @@ +#include +#include +#include + +#define DIM (THREAD_MEM_BANK_WIDTH / 8) +#define M (2 * DIM) +#define N DIM +#define K (2 * DIM) +static int8_t a[M * K] __attribute__((aligned(64))); +static int8_t b[K * N] __attribute__((aligned(64))); +static int32_t c[M * N] __attribute__((aligned(64))); +static int32_t expected[11][13]; +_Static_assert(DIM == 16, + "Gemmini loop tests require the configured 16x16 array"); + +int main(void) { + for (int i = 0; i < M; ++i) + for (int k = 0; k < K; ++k) + a[i * K + k] = (i * 3 + k * 5) % 11 - 5; + for (int k = 0; k < K; ++k) + for (int j = 0; j < N; ++j) + b[k * N + j] = (k * 7 + j * 2) % 13 - 6; + for (int i = 0; i < M * N; ++i) + c[i] = 0x13579bdf; + bb_gemmini_config(0, 0, 0, 0, 0); + bb_gemmini_loop_ws_config_bounds(M / DIM, N / DIM, K / DIM); + bb_gemmini_loop_ws_config_addr_a((uintptr_t)a); + bb_gemmini_loop_ws_config_addr_b((uintptr_t)b); + bb_gemmini_loop_ws_config_addr_d(0); + bb_gemmini_loop_ws_config_addr_c((uintptr_t)c); + bb_gemmini_loop_ws_config_strides_ab(K, N); + bb_gemmini_loop_ws_config_strides_dc(0, N * 4); + bb_gemmini_loop_ws(0, 1, 2, 1); + bb_barrier(); + asm volatile("fence" ::: "memory"); + // Input rows/columns repeat with these periods; every output is still + // checked. + for (int i = 0; i < 11; ++i) + for (int j = 0; j < 13; ++j) { + int32_t sum = 0; + for (int k = 0; k < K; ++k) + sum += a[i * K + k] * b[k * N + j]; + expected[i][j] = sum; + } + for (int i = 0; i < M; ++i) + for (int j = 0; j < N; ++j) + if (c[i * N + j] != expected[i % 11][j % 13]) + return i * N + j + 1; + return 0; +} diff --git a/examples/balls/gemmini/workloads/ctests/gemmini_os_cisc_atranspose_test.c b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_cisc_atranspose_test.c similarity index 97% rename from examples/balls/gemmini/workloads/ctests/gemmini_os_cisc_atranspose_test.c rename to examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_cisc_atranspose_test.c index 386f7e5df..035f45a71 100644 --- a/examples/balls/gemmini/workloads/ctests/gemmini_os_cisc_atranspose_test.c +++ b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_cisc_atranspose_test.c @@ -13,8 +13,8 @@ static result_t mat_c[DIM * DIM] __attribute__((aligned(64))); static result_t expected[DIM * DIM] __attribute__((aligned(64))); int main() { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif printf("=== Gemmini OS CISC a_transpose Test ===\n"); diff --git a/examples/balls/gemmini/workloads/ctests/gemmini_os_cisc_basic_test.c b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_cisc_basic_test.c similarity index 96% rename from examples/balls/gemmini/workloads/ctests/gemmini_os_cisc_basic_test.c rename to examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_cisc_basic_test.c index dd06b8091..d0ec3420a 100644 --- a/examples/balls/gemmini/workloads/ctests/gemmini_os_cisc_basic_test.c +++ b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_cisc_basic_test.c @@ -12,8 +12,8 @@ static result_t mat_c[DIM * DIM] __attribute__((aligned(64))); static result_t expected[DIM * DIM] __attribute__((aligned(64))); int main() { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif printf("=== Gemmini OS CISC Basic Test ===\n"); diff --git a/examples/balls/gemmini/workloads/ctests/gemmini_os_cisc_btranspose_test.c b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_cisc_btranspose_test.c similarity index 97% rename from examples/balls/gemmini/workloads/ctests/gemmini_os_cisc_btranspose_test.c rename to examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_cisc_btranspose_test.c index 02e0192aa..9ec83f029 100644 --- a/examples/balls/gemmini/workloads/ctests/gemmini_os_cisc_btranspose_test.c +++ b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_cisc_btranspose_test.c @@ -13,8 +13,8 @@ static result_t mat_c[DIM * DIM] __attribute__((aligned(64))); static result_t expected[DIM * DIM] __attribute__((aligned(64))); int main() { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif printf("=== Gemmini OS CISC b_transpose Test ===\n"); diff --git a/examples/balls/gemmini/workloads/ctests/gemmini_os_cisc_shift_test.c b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_cisc_shift_test.c similarity index 96% rename from examples/balls/gemmini/workloads/ctests/gemmini_os_cisc_shift_test.c rename to examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_cisc_shift_test.c index 4e90e3b98..3dc388b81 100644 --- a/examples/balls/gemmini/workloads/ctests/gemmini_os_cisc_shift_test.c +++ b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_cisc_shift_test.c @@ -13,8 +13,8 @@ static result_t mat_c[DIM * DIM] __attribute__((aligned(64))); static result_t expected[DIM * DIM] __attribute__((aligned(64))); int main() { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif printf("=== Gemmini OS CISC in_shift Test ===\n"); diff --git a/examples/balls/gemmini/workloads/ctests/gemmini_os_risc_abtranspose_test.c b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_risc_abtranspose_test.c similarity index 97% rename from examples/balls/gemmini/workloads/ctests/gemmini_os_risc_abtranspose_test.c rename to examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_risc_abtranspose_test.c index 7de942431..355e18581 100644 --- a/examples/balls/gemmini/workloads/ctests/gemmini_os_risc_abtranspose_test.c +++ b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_risc_abtranspose_test.c @@ -14,8 +14,8 @@ static result_t mat_c[DIM * DIM] __attribute__((aligned(64))); static result_t expected[DIM * DIM] __attribute__((aligned(64))); int main() { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif printf("=== Gemmini OS RISC ab_transpose Test ===\n"); diff --git a/examples/balls/gemmini/workloads/ctests/gemmini_os_risc_atranspose_test.c b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_risc_atranspose_test.c similarity index 97% rename from examples/balls/gemmini/workloads/ctests/gemmini_os_risc_atranspose_test.c rename to examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_risc_atranspose_test.c index 6d987dc52..c24289288 100644 --- a/examples/balls/gemmini/workloads/ctests/gemmini_os_risc_atranspose_test.c +++ b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_risc_atranspose_test.c @@ -13,8 +13,8 @@ static result_t mat_c[DIM * DIM] __attribute__((aligned(64))); static result_t expected[DIM * DIM] __attribute__((aligned(64))); int main() { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif printf("=== Gemmini OS RISC a_transpose Test ===\n"); diff --git a/examples/balls/gemmini/workloads/ctests/gemmini_os_risc_basic_test.c b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_risc_basic_test.c similarity index 97% rename from examples/balls/gemmini/workloads/ctests/gemmini_os_risc_basic_test.c rename to examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_risc_basic_test.c index c5253f933..159916f7e 100644 --- a/examples/balls/gemmini/workloads/ctests/gemmini_os_risc_basic_test.c +++ b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_risc_basic_test.c @@ -13,8 +13,8 @@ static result_t mat_c[DIM * DIM] __attribute__((aligned(64))); static result_t expected[DIM * DIM] __attribute__((aligned(64))); int main() { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif printf("=== Gemmini OS RISC Basic Test ===\n"); diff --git a/examples/balls/gemmini/workloads/ctests/gemmini_os_risc_btranspose_test.c b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_risc_btranspose_test.c similarity index 97% rename from examples/balls/gemmini/workloads/ctests/gemmini_os_risc_btranspose_test.c rename to examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_risc_btranspose_test.c index 3b89e5d70..9ca10346b 100644 --- a/examples/balls/gemmini/workloads/ctests/gemmini_os_risc_btranspose_test.c +++ b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_risc_btranspose_test.c @@ -13,8 +13,8 @@ static result_t mat_c[DIM * DIM] __attribute__((aligned(64))); static result_t expected[DIM * DIM] __attribute__((aligned(64))); int main() { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif printf("=== Gemmini OS RISC b_transpose Test ===\n"); diff --git a/examples/balls/gemmini/workloads/ctests/gemmini_os_risc_shift_test.c b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_risc_shift_test.c similarity index 97% rename from examples/balls/gemmini/workloads/ctests/gemmini_os_risc_shift_test.c rename to examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_risc_shift_test.c index 04cb0de19..a2c1a11f7 100644 --- a/examples/balls/gemmini/workloads/ctests/gemmini_os_risc_shift_test.c +++ b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_risc_shift_test.c @@ -13,8 +13,8 @@ static result_t mat_c[DIM * DIM] __attribute__((aligned(64))); static result_t expected[DIM * DIM] __attribute__((aligned(64))); int main() { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif printf("=== Gemmini OS RISC in_shift Test ===\n"); diff --git a/examples/balls/gemmini/workloads/ctests/gemmini_ws_cisc_conv_test.c b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_ws_cisc_conv_test.c similarity index 98% rename from examples/balls/gemmini/workloads/ctests/gemmini_ws_cisc_conv_test.c rename to examples/prefab/balls/gemmini/workloads/ctests/gemmini_ws_cisc_conv_test.c index d3d090cb4..1aa582aa3 100644 --- a/examples/balls/gemmini/workloads/ctests/gemmini_ws_cisc_conv_test.c +++ b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_ws_cisc_conv_test.c @@ -29,8 +29,8 @@ static result_t expected[BATCH * OUT_DIM * OUT_DIM * OUT_CH] __attribute__((aligned(64))); int main() { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif printf("=== Gemmini WS CISC Loop Conv Test ===\n"); diff --git a/examples/balls/gemmini/workloads/ctests/gemmini_ws_risc_basic_test.c b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_ws_risc_basic_test.c similarity index 97% rename from examples/balls/gemmini/workloads/ctests/gemmini_ws_risc_basic_test.c rename to examples/prefab/balls/gemmini/workloads/ctests/gemmini_ws_risc_basic_test.c index 7d12fe127..f3a9174da 100644 --- a/examples/balls/gemmini/workloads/ctests/gemmini_ws_risc_basic_test.c +++ b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_ws_risc_basic_test.c @@ -13,8 +13,8 @@ static result_t mat_c[DIM * DIM] __attribute__((aligned(64))); static result_t expected[DIM * DIM] __attribute__((aligned(64))); int main() { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif printf("=== Gemmini WS RISC Basic Test ===\n"); diff --git a/examples/balls/gemmini/workloads/ctests/gemmini_ws_risc_btranspose_test.c b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_ws_risc_btranspose_test.c similarity index 97% rename from examples/balls/gemmini/workloads/ctests/gemmini_ws_risc_btranspose_test.c rename to examples/prefab/balls/gemmini/workloads/ctests/gemmini_ws_risc_btranspose_test.c index 5e0404ee3..f964ab5da 100644 --- a/examples/balls/gemmini/workloads/ctests/gemmini_ws_risc_btranspose_test.c +++ b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_ws_risc_btranspose_test.c @@ -14,8 +14,8 @@ static result_t mat_c[DIM * DIM] __attribute__((aligned(64))); static result_t expected[DIM * DIM] __attribute__((aligned(64))); int main() { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif printf("=== Gemmini WS RISC b_transpose Test ===\n"); diff --git a/examples/balls/gemmini/workloads/ctests/gemmini_ws_risc_shift_test.c b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_ws_risc_shift_test.c similarity index 97% rename from examples/balls/gemmini/workloads/ctests/gemmini_ws_risc_shift_test.c rename to examples/prefab/balls/gemmini/workloads/ctests/gemmini_ws_risc_shift_test.c index 2d84dbdb1..f53a09ab2 100644 --- a/examples/balls/gemmini/workloads/ctests/gemmini_ws_risc_shift_test.c +++ b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_ws_risc_shift_test.c @@ -14,8 +14,8 @@ static result_t mat_c[DIM * DIM] __attribute__((aligned(64))); static result_t expected[DIM * DIM] __attribute__((aligned(64))); int main() { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif printf("=== Gemmini WS RISC in_shift Test ===\n"); diff --git a/examples/balls/gemmini/workloads/isa/gemmini.h b/examples/prefab/balls/gemmini/workloads/isa/gemmini.h similarity index 100% rename from examples/balls/gemmini/workloads/isa/gemmini.h rename to examples/prefab/balls/gemmini/workloads/isa/gemmini.h diff --git a/examples/balls/gemmini/workloads/isa/gemmini_compute_accumulated.h b/examples/prefab/balls/gemmini/workloads/isa/gemmini_compute_accumulated.h similarity index 88% rename from examples/balls/gemmini/workloads/isa/gemmini_compute_accumulated.h rename to examples/prefab/balls/gemmini/workloads/isa/gemmini_compute_accumulated.h index d3fda03ec..e015fc4e1 100644 --- a/examples/balls/gemmini/workloads/isa/gemmini_compute_accumulated.h +++ b/examples/prefab/balls/gemmini/workloads/isa/gemmini_compute_accumulated.h @@ -15,9 +15,9 @@ iter, op1_base, op2_base, wr_base) \ BUCKYBALL_INSTRUCTION_R_R( \ (BB_BANK0(op1_bank_id) | BB_BANK1(op2_bank_id) | BB_BANK2(wr_bank_id) | \ - BB_ITER(iter)), \ + BB_ITER(iter) | BB_READ_WR_OLD), \ BB_GEMMINI_COMPUTE_ACCUMULATED_RS2 | ((uint64_t)(op1_base) << 6) | \ ((uint64_t)(op2_base) << 16) | ((uint64_t)(wr_base) << 26), \ - BB_FUNC7(GEMMINI_COMPUTE_ACCUMULATED)) + FUNCT7(gemmini, GEMMINI_COMPUTE_ACCUMULATED)) #endif // _BB_GEMMINI_COMPUTE_ACCUMULATED_H_ diff --git a/examples/balls/gemmini/workloads/isa/gemmini_compute_preloaded.h b/examples/prefab/balls/gemmini/workloads/isa/gemmini_compute_preloaded.h similarity index 95% rename from examples/balls/gemmini/workloads/isa/gemmini_compute_preloaded.h rename to examples/prefab/balls/gemmini/workloads/isa/gemmini_compute_preloaded.h index c39d3b10f..7f1ec4177 100644 --- a/examples/balls/gemmini/workloads/isa/gemmini_compute_preloaded.h +++ b/examples/prefab/balls/gemmini/workloads/isa/gemmini_compute_preloaded.h @@ -18,6 +18,6 @@ BB_ITER(iter)), \ BB_GEMMINI_COMPUTE_PRELOADED_RS2 | ((uint64_t)(op1_base) << 6) | \ ((uint64_t)(op2_base) << 16) | ((uint64_t)(wr_base) << 26), \ - BB_FUNC7(GEMMINI_COMPUTE_PRELOADED)) + FUNCT7(gemmini, GEMMINI_COMPUTE_PRELOADED)) #endif // _BB_GEMMINI_COMPUTE_PRELOADED_H_ diff --git a/examples/balls/gemmini/workloads/isa/gemmini_config.h b/examples/prefab/balls/gemmini/workloads/isa/gemmini_config.h similarity index 94% rename from examples/balls/gemmini/workloads/isa/gemmini_config.h rename to examples/prefab/balls/gemmini/workloads/isa/gemmini_config.h index fc50c0748..43aac5668 100644 --- a/examples/balls/gemmini/workloads/isa/gemmini_config.h +++ b/examples/prefab/balls/gemmini/workloads/isa/gemmini_config.h @@ -18,6 +18,6 @@ FIELD(a_transpose, 7, 7) | \ FIELD(b_transpose, 8, 8) | \ FIELD(in_shift, 9, 40)), \ - BB_FUNC7(GEMMINI_CONFIG)) + FUNCT7(gemmini, GEMMINI_CONFIG)) #endif // _BB_GEMMINI_CONFIG_H_ diff --git a/examples/prefab/balls/gemmini/workloads/isa/gemmini_flush.h b/examples/prefab/balls/gemmini/workloads/isa/gemmini_flush.h new file mode 100644 index 000000000..0e4d9628d --- /dev/null +++ b/examples/prefab/balls/gemmini/workloads/isa/gemmini_flush.h @@ -0,0 +1,14 @@ +#ifndef _BB_GEMMINI_FLUSH_H_ +#define _BB_GEMMINI_FLUSH_H_ + +#include +#include + +#define BB_GEMMINI_FLUSH_RS2 4ULL + +// Flush the matrix array state +#define bb_gemmini_flush() \ + BUCKYBALL_INSTRUCTION_R_R(0, BB_GEMMINI_FLUSH_RS2, \ + FUNCT7(gemmini, GEMMINI_FLUSH)) + +#endif // _BB_GEMMINI_FLUSH_H_ diff --git a/examples/balls/gemmini/workloads/isa/gemmini_loop_conv_ws.h b/examples/prefab/balls/gemmini/workloads/isa/gemmini_loop_conv_ws.h similarity index 81% rename from examples/balls/gemmini/workloads/isa/gemmini_loop_conv_ws.h rename to examples/prefab/balls/gemmini/workloads/isa/gemmini_loop_conv_ws.h index 5f04089f7..f6f80d77c 100644 --- a/examples/balls/gemmini/workloads/isa/gemmini_loop_conv_ws.h +++ b/examples/prefab/balls/gemmini/workloads/isa/gemmini_loop_conv_ws.h @@ -9,7 +9,7 @@ (FIELD(batch_size, 0, 15) | \ FIELD(in_dim, 16, 31) | \ FIELD(in_channels, 32, 47)), \ - BB_FUNC7(GEMMINI_LOOP_CONV_WS_CONFIG_1)) + FUNCT7(gemmini, GEMMINI_LOOP_CONV_WS_CONFIG_1)) #define bb_gemmini_loop_conv_ws_config_2(out_channels, out_dim, stride, \ padding) \ @@ -17,7 +17,7 @@ (FIELD(out_channels, 0, 15) | \ FIELD(out_dim, 16, 31) | FIELD(stride, 32, 39) | \ FIELD(padding, 40, 47)), \ - BB_FUNC7(GEMMINI_LOOP_CONV_WS_CONFIG_2)) + FUNCT7(gemmini, GEMMINI_LOOP_CONV_WS_CONFIG_2)) #define bb_gemmini_loop_conv_ws_config_3(kernel_dim, pool_size, pool_stride, \ pool_padding) \ @@ -25,38 +25,38 @@ 0, \ (FIELD(kernel_dim, 0, 7) | FIELD(pool_size, 8, 15) | \ FIELD(pool_stride, 16, 23) | FIELD(pool_padding, 24, 31)), \ - BB_FUNC7(GEMMINI_LOOP_CONV_WS_CONFIG_3)) + FUNCT7(gemmini, GEMMINI_LOOP_CONV_WS_CONFIG_3)) #define bb_gemmini_loop_conv_ws_config_4(addr_bias) \ BUCKYBALL_INSTRUCTION_R_R(0, FIELD(addr_bias, 0, 38), \ - BB_FUNC7(GEMMINI_LOOP_CONV_WS_CONFIG_4)) + FUNCT7(gemmini, GEMMINI_LOOP_CONV_WS_CONFIG_4)) #define bb_gemmini_loop_conv_ws_config_5(addr_input) \ BUCKYBALL_INSTRUCTION_R_R(0, FIELD(addr_input, 0, 38), \ - BB_FUNC7(GEMMINI_LOOP_CONV_WS_CONFIG_5)) + FUNCT7(gemmini, GEMMINI_LOOP_CONV_WS_CONFIG_5)) #define bb_gemmini_loop_conv_ws_config_6(addr_weight) \ BUCKYBALL_INSTRUCTION_R_R(0, FIELD(addr_weight, 0, 38), \ - BB_FUNC7(GEMMINI_LOOP_CONV_WS_CONFIG_6)) + FUNCT7(gemmini, GEMMINI_LOOP_CONV_WS_CONFIG_6)) #define bb_gemmini_loop_conv_ws_config_7(addr_output) \ BUCKYBALL_INSTRUCTION_R_R(0, FIELD(addr_output, 0, 38), \ - BB_FUNC7(GEMMINI_LOOP_CONV_WS_CONFIG_7)) + FUNCT7(gemmini, GEMMINI_LOOP_CONV_WS_CONFIG_7)) #define bb_gemmini_loop_conv_ws_config_8(input_stride, weight_stride) \ BUCKYBALL_INSTRUCTION_R_R( \ 0, (FIELD(input_stride, 0, 31) | FIELD(weight_stride, 32, 63)), \ - BB_FUNC7(GEMMINI_LOOP_CONV_WS_CONFIG_8)) + FUNCT7(gemmini, GEMMINI_LOOP_CONV_WS_CONFIG_8)) #define bb_gemmini_loop_conv_ws_config_9(output_stride) \ BUCKYBALL_INSTRUCTION_R_R(0, FIELD(output_stride, 0, 31), \ - BB_FUNC7(GEMMINI_LOOP_CONV_WS_CONFIG_9)) + FUNCT7(gemmini, GEMMINI_LOOP_CONV_WS_CONFIG_9)) #define bb_gemmini_loop_conv_ws(bank_input, bank_weight, bank_output, no_bias) \ BUCKYBALL_INSTRUCTION_R_R( \ 0, \ (FIELD(bank_input, 0, 9) | FIELD(bank_weight, 10, 19) | \ FIELD(bank_output, 20, 29) | FIELD(no_bias, 30, 30)), \ - BB_FUNC7(GEMMINI_LOOP_CONV_WS)) + FUNCT7(gemmini, GEMMINI_LOOP_CONV_WS)) #endif diff --git a/examples/balls/gemmini/workloads/isa/gemmini_loop_ws.h b/examples/prefab/balls/gemmini/workloads/isa/gemmini_loop_ws.h similarity index 77% rename from examples/balls/gemmini/workloads/isa/gemmini_loop_ws.h rename to examples/prefab/balls/gemmini/workloads/isa/gemmini_loop_ws.h index 769c1f660..9d76463b8 100644 --- a/examples/balls/gemmini/workloads/isa/gemmini_loop_ws.h +++ b/examples/prefab/balls/gemmini/workloads/isa/gemmini_loop_ws.h @@ -7,38 +7,38 @@ #define bb_gemmini_loop_ws_config_bounds(max_i, max_j, max_k) \ BUCKYBALL_INSTRUCTION_R_R( \ 0, (FIELD(max_k, 0, 15) | FIELD(max_j, 16, 31) | FIELD(max_i, 32, 47)), \ - BB_FUNC7(GEMMINI_LOOP_WS_CONFIG_BOUNDS)) + FUNCT7(gemmini, GEMMINI_LOOP_WS_CONFIG_BOUNDS)) #define bb_gemmini_loop_ws_config_addr_a(addr) \ BUCKYBALL_INSTRUCTION_R_R(0, FIELD(addr, 0, 38), \ - BB_FUNC7(GEMMINI_LOOP_WS_CONFIG_ADDR_A)) + FUNCT7(gemmini, GEMMINI_LOOP_WS_CONFIG_ADDR_A)) #define bb_gemmini_loop_ws_config_addr_b(addr) \ BUCKYBALL_INSTRUCTION_R_R(0, FIELD(addr, 0, 38), \ - BB_FUNC7(GEMMINI_LOOP_WS_CONFIG_ADDR_B)) + FUNCT7(gemmini, GEMMINI_LOOP_WS_CONFIG_ADDR_B)) #define bb_gemmini_loop_ws_config_addr_d(addr) \ BUCKYBALL_INSTRUCTION_R_R(0, FIELD(addr, 0, 38), \ - BB_FUNC7(GEMMINI_LOOP_WS_CONFIG_ADDR_D)) + FUNCT7(gemmini, GEMMINI_LOOP_WS_CONFIG_ADDR_D)) #define bb_gemmini_loop_ws_config_addr_c(addr) \ BUCKYBALL_INSTRUCTION_R_R(0, FIELD(addr, 0, 38), \ - BB_FUNC7(GEMMINI_LOOP_WS_CONFIG_ADDR_C)) + FUNCT7(gemmini, GEMMINI_LOOP_WS_CONFIG_ADDR_C)) #define bb_gemmini_loop_ws_config_strides_ab(stride_a, stride_b) \ BUCKYBALL_INSTRUCTION_R_R( \ 0, (FIELD(stride_a, 0, 31) | FIELD(stride_b, 32, 63)), \ - BB_FUNC7(GEMMINI_LOOP_WS_CONFIG_STRIDES_AB)) + FUNCT7(gemmini, GEMMINI_LOOP_WS_CONFIG_STRIDES_AB)) #define bb_gemmini_loop_ws_config_strides_dc(stride_d, stride_c) \ BUCKYBALL_INSTRUCTION_R_R( \ 0, (FIELD(stride_d, 0, 31) | FIELD(stride_c, 32, 63)), \ - BB_FUNC7(GEMMINI_LOOP_WS_CONFIG_STRIDES_DC)) + FUNCT7(gemmini, GEMMINI_LOOP_WS_CONFIG_STRIDES_DC)) #define bb_gemmini_loop_ws(bank_a, bank_b, bank_c, low_d) \ BUCKYBALL_INSTRUCTION_R_R(0, \ (FIELD(bank_a, 0, 9) | FIELD(bank_b, 10, 19) | \ FIELD(bank_c, 20, 29) | FIELD(low_d, 30, 30)), \ - BB_FUNC7(GEMMINI_LOOP_WS)) + FUNCT7(gemmini, GEMMINI_LOOP_WS)) #endif diff --git a/examples/balls/gemmini/workloads/isa/gemmini_preload.h b/examples/prefab/balls/gemmini/workloads/isa/gemmini_preload.h similarity index 95% rename from examples/balls/gemmini/workloads/isa/gemmini_preload.h rename to examples/prefab/balls/gemmini/workloads/isa/gemmini_preload.h index f1f578bec..b6ac3f0fb 100644 --- a/examples/balls/gemmini/workloads/isa/gemmini_preload.h +++ b/examples/prefab/balls/gemmini/workloads/isa/gemmini_preload.h @@ -15,6 +15,6 @@ (BB_BANK0(op1_bank_id) | BB_BANK2(wr_bank_id) | BB_ITER(iter)), \ BB_GEMMINI_PRELOAD_RS2 | ((uint64_t)(op1_base) << 6) | \ ((uint64_t)(wr_base) << 26), \ - BB_FUNC7(GEMMINI_PRELOAD)) + FUNCT7(gemmini, GEMMINI_PRELOAD)) #endif // _BB_GEMMINI_PRELOAD_H_ diff --git a/examples/balls/gemmini/workloads/mlir_tests/CMakeLists.txt b/examples/prefab/balls/gemmini/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/balls/gemmini/workloads/mlir_tests/CMakeLists.txt rename to examples/prefab/balls/gemmini/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/balls/gemmini/workloads/mlir_tests/ball/CMakeLists.txt b/examples/prefab/balls/gemmini/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/balls/gemmini/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/prefab/balls/gemmini/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/prefab/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16.mlir b/examples/prefab/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16.mlir new file mode 100644 index 000000000..3de982139 --- /dev/null +++ b/examples/prefab/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16.mlir @@ -0,0 +1,51 @@ +func.func private @check_result(memref<16x16xi32>) -> () +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %initial = arith.constant 7 : i32 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %two = arith.constant 2 : index + %three = arith.constant 3 : index + %five = arith.constant 5 : index + %seven = arith.constant 7 : index + %sixteen = arith.constant 16 : index + %a = memref.alloc() : memref<16x16xi8> + %b = memref.alloc() : memref<16x16xi8> + %c = memref.alloc() : memref<16x16xi32> + linalg.fill ins(%initial : i32) outs(%c : memref<16x16xi32>) + scf.for %row = %zero to %sixteen step %one { + scf.for %col = %zero to %sixteen step %one { + %r3 = arith.muli %row, %three : index + %c5 = arith.muli %col, %five : index + %asum = arith.addi %r3, %c5 : index + %arem = arith.remui %asum, %seven : index + %aval = arith.subi %arem, %three : index + %av = arith.index_cast %aval : index to i8 + %r2 = arith.muli %row, %two : index + %c3 = arith.muli %col, %three : index + %bsum = arith.addi %r2, %c3 : index + %brem = arith.remui %bsum, %five : index + %bval = arith.subi %brem, %two : index + %bv = arith.index_cast %bval : index to i8 + memref.store %av, %a[%row, %col] : memref<16x16xi8> + memref.store %bv, %b[%row, %col] : memref<16x16xi8> + } + } + buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = true, row = 1 : i64, col = 4 : i64}> + buckyball.mvin %a <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.mvin %b <{bank = 2 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.mvin %c <{bank = 3 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi32> + // C = in_shift(C + A @ B, 4). + gemmini_ball.matmul_i8_tile <{a = 1 : i64, b = 2 : i64, c = 3 : i64, k = 16 : i64, shift = 4 : i64, read_destination}> + buckyball.mvout %c <{bank = 3 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi32> + func.call @check_result(%c) : (memref<16x16xi32>) -> () + buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + memref.dealloc %a : memref<16x16xi8> + memref.dealloc %b : memref<16x16xi8> + memref.dealloc %c : memref<16x16xi32> + return %zero_i8 : i8 +} diff --git a/examples/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16_main.cpp b/examples/prefab/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16_main.cpp similarity index 100% rename from examples/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16_main.cpp rename to examples/prefab/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16_main.cpp diff --git a/examples/prefab/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16.mlir b/examples/prefab/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16.mlir new file mode 100644 index 000000000..84ff88d7a --- /dev/null +++ b/examples/prefab/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16.mlir @@ -0,0 +1,50 @@ +func.func private @check_result(memref<16x16xi32>) -> () +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero_i32 = arith.constant 0 : i32 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %two = arith.constant 2 : index + %three = arith.constant 3 : index + %five = arith.constant 5 : index + %seven = arith.constant 7 : index + %sixteen = arith.constant 16 : index + %a = memref.alloc() : memref<16x16xi8> + %b = memref.alloc() : memref<16x16xi8> + %c = memref.alloc() : memref<16x16xi32> + linalg.fill ins(%zero_i32 : i32) outs(%c : memref<16x16xi32>) + scf.for %row = %zero to %sixteen step %one { + scf.for %col = %zero to %sixteen step %one { + %r3 = arith.muli %row, %three : index + %c5 = arith.muli %col, %five : index + %asum = arith.addi %r3, %c5 : index + %arem = arith.remui %asum, %seven : index + %aval = arith.subi %arem, %three : index + %av = arith.index_cast %aval : index to i8 + %r2 = arith.muli %row, %two : index + %c3 = arith.muli %col, %three : index + %bsum = arith.addi %r2, %c3 : index + %brem = arith.remui %bsum, %five : index + %bval = arith.subi %brem, %two : index + %bv = arith.index_cast %bval : index to i8 + memref.store %av, %a[%row, %col] : memref<16x16xi8> + memref.store %bv, %b[%row, %col] : memref<16x16xi8> + } + } + buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = true, row = 1 : i64, col = 4 : i64}> + buckyball.mvin %a <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.mvin %b <{bank = 2 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + // C = A @ B on the output-stationary array. + gemmini_ball.matmul_i8_tile <{a = 1 : i64, b = 2 : i64, c = 3 : i64, k = 16 : i64}> + buckyball.mvout %c <{bank = 3 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi32> + func.call @check_result(%c) : (memref<16x16xi32>) -> () + buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + memref.dealloc %a : memref<16x16xi8> + memref.dealloc %b : memref<16x16xi8> + memref.dealloc %c : memref<16x16xi32> + return %zero_i8 : i8 +} diff --git a/examples/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16_main.cpp b/examples/prefab/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16_main.cpp similarity index 100% rename from examples/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16_main.cpp rename to examples/prefab/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16_main.cpp diff --git a/examples/prefab/balls/gemmini/workloads/mlir_tests/bank/CMakeLists.txt b/examples/prefab/balls/gemmini/workloads/mlir_tests/bank/CMakeLists.txt new file mode 100644 index 000000000..2187c846b --- /dev/null +++ b/examples/prefab/balls/gemmini/workloads/mlir_tests/bank/CMakeLists.txt @@ -0,0 +1,11 @@ +if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) + set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) +endif() +set(BUCKYBALL_MLIR_TEST_PREFIX bank) +set(_lines ${DESIGN_PROFILES_${BUCKYBALL_MLIR_ACTIVE_TARGET}_THREAD_MEM_BANK_ENTRIES}) +if(NOT _lines EQUAL 1024) + message(FATAL_ERROR + "Gemmini BankOp tests require 1024 bank entries (got ${_lines})") +endif() +add_buckyball_mlir_test(gemmini_compute_preloaded_bank_tiles TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) +add_buckyball_mlir_test(gemmini_compute_accumulated_bank_tiles TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/prefab/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles.mlir b/examples/prefab/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles.mlir new file mode 100644 index 000000000..04caf4037 --- /dev/null +++ b/examples/prefab/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles.mlir @@ -0,0 +1,97 @@ +func.func private @check_result(memref<16x16xi32>) -> () + +// Each group holds a whole bank of K: A as blocks of sixteen rows, B as one +// row per K. A[i, kk] = (3i + 5kk) % 7 - 3 with kk = first + 16 * (q / 16) + col +// at bank row q, i = q % 16. +func.func @fill_a(%a: memref<1024x16xi8>, %first: index) { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c3 = arith.constant 3 : index + %c5 = arith.constant 5 : index + %c7 = arith.constant 7 : index + %c16 = arith.constant 16 : index + %lines = arith.constant 1024 : index + scf.for %q = %c0 to %lines step %c1 { + %i = arith.remui %q, %c16 : index + %block = arith.subi %q, %i : index + %start = arith.addi %first, %block : index + scf.for %col = %c0 to %c16 step %c1 { + %kk = arith.addi %start, %col : index + %i3 = arith.muli %i, %c3 : index + %k5 = arith.muli %kk, %c5 : index + %sum = arith.addi %i3, %k5 : index + %rem = arith.remui %sum, %c7 : index + %value = arith.subi %rem, %c3 : index + %v8 = arith.index_cast %value : index to i8 + memref.store %v8, %a[%q, %col] : memref<1024x16xi8> + } + } + return +} + +// B[kk, j] = (2kk + 3j) % 5 - 2 with kk = first + q at bank row q. +func.func @fill_b(%b: memref<1024x16xi8>, %first: index) { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c2 = arith.constant 2 : index + %c3 = arith.constant 3 : index + %c5 = arith.constant 5 : index + %c16 = arith.constant 16 : index + %lines = arith.constant 1024 : index + scf.for %q = %c0 to %lines step %c1 { + %kk = arith.addi %first, %q : index + scf.for %j = %c0 to %c16 step %c1 { + %k2 = arith.muli %kk, %c2 : index + %j3 = arith.muli %j, %c3 : index + %sum = arith.addi %k2, %j3 : index + %rem = arith.remui %sum, %c5 : index + %value = arith.subi %rem, %c2 : index + %v8 = arith.index_cast %value : index to i8 + memref.store %v8, %b[%q, %j] : memref<1024x16xi8> + } + } + return +} + +// C = in_shift(7 + A[16 x 2048] @ B[2048 x 16], 2), A and B each gathered +// from two full 1024-row banks. +func.func @main() -> i8 { + %z8 = arith.constant 0 : i8 + %initial = arith.constant 7 : i32 + %c0 = arith.constant 0 : index + %lines = arith.constant 1024 : index + %a0 = memref.alloc() : memref<1024x16xi8> + %a1 = memref.alloc() : memref<1024x16xi8> + %b0 = memref.alloc() : memref<1024x16xi8> + %b1 = memref.alloc() : memref<1024x16xi8> + %c = memref.alloc() : memref<16x16xi32> + linalg.fill ins(%initial : i32) outs(%c : memref<16x16xi32>) + func.call @fill_a(%a0, %c0) : (memref<1024x16xi8>, index) -> () + func.call @fill_a(%a1, %lines) : (memref<1024x16xi8>, index) -> () + func.call @fill_b(%b0, %c0) : (memref<1024x16xi8>, index) -> () + func.call @fill_b(%b1, %lines) : (memref<1024x16xi8>, index) -> () + %ab0 = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %bb0 = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %ab1 = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %bb1 = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %cb = buckyball.bank_alloc <{row = 1 : i64, col = 4 : i64}> + %al0 = buckyball.bank_mvin %a0, %ab0 <{depth = 1024 : i64, stride = 1 : i64}> : memref<1024x16xi8> + %bl0 = buckyball.bank_mvin %b0, %bb0 <{depth = 1024 : i64, stride = 1 : i64}> : memref<1024x16xi8> + %al1 = buckyball.bank_mvin %a1, %ab1 <{depth = 1024 : i64, stride = 1 : i64}> : memref<1024x16xi8> + %bl1 = buckyball.bank_mvin %b1, %bb1 <{depth = 1024 : i64, stride = 1 : i64}> : memref<1024x16xi8> + %cl = buckyball.bank_mvin %c, %cb <{depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi32> + %al = buckyball.bank_transfer %al1, %al0 + %bl = buckyball.bank_transfer %bl1, %bl0 + %r = gemmini_ball.bank_matmul_i8_tile %al, %bl -> %cl <{k = 2048 : i64, shift = 2 : i64, read_destination}> + %stored = buckyball.bank_mvout %c, %r <{depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi32> + func.call @check_result(%c) : (memref<16x16xi32>) -> () + buckyball.bank_release %al + buckyball.bank_release %bl + buckyball.bank_release %stored + memref.dealloc %a0 : memref<1024x16xi8> + memref.dealloc %a1 : memref<1024x16xi8> + memref.dealloc %b0 : memref<1024x16xi8> + memref.dealloc %b1 : memref<1024x16xi8> + memref.dealloc %c : memref<16x16xi32> + return %z8 : i8 +} diff --git a/examples/prefab/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles_main.cpp b/examples/prefab/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles_main.cpp new file mode 100644 index 000000000..80f85fe95 --- /dev/null +++ b/examples/prefab/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles_main.cpp @@ -0,0 +1,29 @@ +#include +#include +#include +static_assert(THREAD_MEM_BANK_ENTRIES == 1024); +extern "C" void check_result(int32_t *, int32_t *data, int64_t offset, + int64_t rows, int64_t lanes, int64_t stride, + int64_t lane_stride) { + if (rows != 16 || lanes != 16 || stride != 16 || lane_stride != 1) + exit(1); + for (int row = 0; row < 16; ++row) + for (int column = 0; column < 16; ++column) { + int32_t expected = 7; + for (int k = 0; k < 2048; ++k) + expected += + ((3 * row + 5 * k) % 7 - 3) * ((2 * k + 3 * column) % 5 - 2); + // in_shift by two, rounding to nearest even. + const uint32_t bits = (uint32_t)expected; + const int rounded = + ((bits >> 1) & 1) && ((bits & 1) || ((bits >> 2) & 1)); + expected = (expected >> 2) + rounded; + if (data[offset + row * stride + column] != expected) { + printf("gemmini accumulated mismatch row=%d column=%d expected=%d " + "actual=%d\n", + row, column, expected, data[offset + row * stride + column]); + exit(1); + } + } + printf("gemmini matmul_i8_tile read_destination over two groups PASS\n"); +} diff --git a/examples/prefab/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles.mlir b/examples/prefab/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles.mlir new file mode 100644 index 000000000..835e603cd --- /dev/null +++ b/examples/prefab/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles.mlir @@ -0,0 +1,77 @@ +func.func private @check_result(memref<16x16xi32>) -> () + +// A holds K = 96 as six blocks of sixteen rows: +// A[i, kk] = (3i + 5kk) % 7 - 3 with kk = 16 * (q / 16) + col at bank row q, +// i = q % 16. +func.func @fill_a(%a: memref<96x16xi8>) { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c3 = arith.constant 3 : index + %c5 = arith.constant 5 : index + %c7 = arith.constant 7 : index + %c16 = arith.constant 16 : index + %c96 = arith.constant 96 : index + scf.for %q = %c0 to %c96 step %c1 { + %i = arith.remui %q, %c16 : index + %block = arith.subi %q, %i : index + scf.for %col = %c0 to %c16 step %c1 { + %kk = arith.addi %block, %col : index + %i3 = arith.muli %i, %c3 : index + %k5 = arith.muli %kk, %c5 : index + %sum = arith.addi %i3, %k5 : index + %rem = arith.remui %sum, %c7 : index + %value = arith.subi %rem, %c3 : index + %v8 = arith.index_cast %value : index to i8 + memref.store %v8, %a[%q, %col] : memref<96x16xi8> + } + } + return +} + +// B[kk, j] = (2kk + 3j) % 5 - 2 with kk = q at bank row q. +func.func @fill_b(%b: memref<96x16xi8>) { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c2 = arith.constant 2 : index + %c3 = arith.constant 3 : index + %c5 = arith.constant 5 : index + %c16 = arith.constant 16 : index + %c96 = arith.constant 96 : index + scf.for %q = %c0 to %c96 step %c1 { + scf.for %j = %c0 to %c16 step %c1 { + %k2 = arith.muli %q, %c2 : index + %j3 = arith.muli %j, %c3 : index + %sum = arith.addi %k2, %j3 : index + %rem = arith.remui %sum, %c5 : index + %value = arith.subi %rem, %c2 : index + %v8 = arith.index_cast %value : index to i8 + memref.store %v8, %b[%q, %j] : memref<96x16xi8> + } + } + return +} + +// C = A[16 x 96] @ B[96 x 16] in one operation of k = 96. +func.func @main() -> i8 { + %z8 = arith.constant 0 : i8 + %a = memref.alloc() : memref<96x16xi8> + %b = memref.alloc() : memref<96x16xi8> + %c = memref.alloc() : memref<16x16xi32> + func.call @fill_a(%a) : (memref<96x16xi8>) -> () + func.call @fill_b(%b) : (memref<96x16xi8>) -> () + %ab = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %bb = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %cb = buckyball.bank_alloc <{row = 1 : i64, col = 4 : i64}> + %al = buckyball.bank_mvin %a, %ab <{depth = 96 : i64, stride = 1 : i64}> : memref<96x16xi8> + %bl = buckyball.bank_mvin %b, %bb <{depth = 96 : i64, stride = 1 : i64}> : memref<96x16xi8> + %r = gemmini_ball.bank_matmul_i8_tile %al, %bl -> %cb <{k = 96 : i64}> + %stored = buckyball.bank_mvout %c, %r <{depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi32> + func.call @check_result(%c) : (memref<16x16xi32>) -> () + buckyball.bank_release %al + buckyball.bank_release %bl + buckyball.bank_release %stored + memref.dealloc %a : memref<96x16xi8> + memref.dealloc %b : memref<96x16xi8> + memref.dealloc %c : memref<16x16xi32> + return %z8 : i8 +} diff --git a/examples/prefab/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles_main.cpp b/examples/prefab/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles_main.cpp new file mode 100644 index 000000000..e8160d0dd --- /dev/null +++ b/examples/prefab/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles_main.cpp @@ -0,0 +1,22 @@ +#include +#include +#include +extern "C" void check_result(int32_t *, int32_t *data, int64_t offset, + int64_t rows, int64_t lanes, int64_t stride, + int64_t lane_stride) { + if (rows != 16 || lanes != 16 || stride != 16 || lane_stride != 1) + exit(1); + for (int row = 0; row < 16; ++row) + for (int column = 0; column < 16; ++column) { + int32_t expected = 0; + for (int k = 0; k < 96; ++k) + expected += + ((3 * row + 5 * k) % 7 - 3) * ((2 * k + 3 * column) % 5 - 2); + if (data[offset + row * stride + column] != expected) { + printf("gemmini bank mismatch row=%d column=%d expected=%d actual=%d\n", + row, column, expected, data[offset + row * stride + column]); + exit(1); + } + } + printf("gemmini matmul_i8_tile k=96 PASS\n"); +} diff --git a/examples/prefab/balls/im2col/arch/Im2col.scala b/examples/prefab/balls/im2col/arch/Im2col.scala new file mode 100644 index 000000000..59705f0db --- /dev/null +++ b/examples/prefab/balls/im2col/arch/Im2col.scala @@ -0,0 +1,333 @@ +package examples.balls.im2col + +import framework.acc.balldomain.configs.BallRouteParam + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import framework.acc.balldomain.ball.blink.{BallCommand, BallResponse} +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import framework.acc.balldomain.ball.blink.BallStatus +import framework.acc.memdomain.{BankRead, BankWrite} +import examples.balls.im2col.configs.Im2colBallParam + +/** + * Im2col over channel-block groups. + * + * Input group g is channel block g; line inputBase + y * size + x is pixel + * (y, x). K is ordered (kernel row, kernel column, channel), so K tile + * t = (kr * kernel + kc) * inputGroups + g is one whole input row (zero in + * the padding). Output group j holds K tiles j * groupTiles .. in the SMatMul + * A layout: line ((w / tileRows) * groupTiles + localTile) * tileRows + + * w % tileRows. Windows beyond windowCount in the last M tile are zero. + */ +@instantiable +class Im2col( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val routingParam: BallDomainParam, + val instance: BallRouteParam, + val ballParam: Im2colBallParam) + extends Module { + val maxIter = ballParam.maxIter + val maxKSize = ballParam.maxKSize + val maxPad = ballParam.maxPadding + val addressWidth = log2Ceil(memParam.bankEntries) + val groupWidth = log2Up(memParam.bankNum + 1) + + require(instance.inBW == 1, "Im2colBall requires inBW=1") + require(instance.outBW == 1, "Im2colBall requires outBW=1") + require(memParam.bankWidth == 128, "Im2colBall requires 128-bit bank rows") + require(ballParam.inputWidth == 8, "Im2colBall moves 16 INT8 channels per row") + require(isPow2(memParam.bankEntries), "Im2colBall requires power-of-two bank depth") + require(maxIter + 2 * maxPad <= 255, "Im2colBall dimensions must fit the divider") + require(maxKSize <= 15 && maxPad <= 15, "Im2colBall kernel and padding fields are four bits") + + @public + val io = IO(new Bundle { + val cmdReq = Flipped(Decoupled(new BallCommand(frontendParam, memParam))) + val cmdResp = Decoupled(new BallResponse(frontendParam)) + val bankRead = Vec(1, Flipped(new BankRead(frontendParam, memParam))) + val bankWrite = Vec(1, Flipped(new BankWrite(frontendParam, memParam))) + val status = new BallStatus + }) + + val idle :: divRows :: divCols :: divStart :: divTiles :: check :: select :: readReq :: readResp :: writeReq :: writeResp :: complete :: Nil = + Enum(12) + val state = RegInit(idle) + + val robId = RegInit(0.U(log2Up(frontendParam.rob_entries).W)) + val inBank = RegInit(0.U(frontendParam.bank_id_len.W)) + val outBank = RegInit(0.U(frontendParam.bank_id_len.W)) + val inBase = RegInit(0.U(memParam.groupIdWidth.W)) + val outBase = RegInit(0.U(memParam.groupIdWidth.W)) + val inGroups = RegInit(0.U(groupWidth.W)) + val outGroups = RegInit(0.U(groupWidth.W)) + val size = RegInit(0.U(8.W)) + val kernel = RegInit(0.U(4.W)) + val stride = RegInit(0.U(4.W)) + val padding = RegInit(0.U(4.W)) + val startCol = RegInit(0.U(4.W)) + val startRow = RegInit(0.U(4.W)) + val inputBase = RegInit(0.U(6.W)) + val windowStart = RegInit(0.U(10.W)) + val windowCount = RegInit(0.U(10.W)) + val tileShift = RegInit(0.U(3.W)) + val outRows = RegInit(0.U(16.W)) + val outCols = RegInit(0.U(16.W)) + val firstRow = RegInit(0.U(16.W)) + val firstCol = RegInit(0.U(16.W)) + val groupTiles = RegInit(0.U(16.W)) + + // Current K tile and output window. + val kernelRow = RegInit(0.U(4.W)) + val kernelCol = RegInit(0.U(4.W)) + val inGroup = RegInit(0.U(groupWidth.W)) + val outGroup = RegInit(0.U(groupWidth.W)) + val localTile = RegInit(0.U(16.W)) + val window = RegInit(0.U(11.W)) + val windowRow = RegInit(0.U(16.W)) + val windowCol = RegInit(0.U(16.W)) + val rowData = Reg(UInt(128.W)) + + val div = Instantiate(new RestoringDiv(16, 8, ballParam)) + div.io.start := false.B + div.io.a := 0.U + div.io.b := 0.U + + val padded = size +& (padding << 1) + val tileRows = 1.U(5.W) << tileShift + val paddedWindows = ((windowCount +& tileRows - 1.U) >> tileShift) << tileShift + val sourceRow = startRow +& windowRow * stride +& kernelRow + val sourceCol = startCol +& windowCol * stride +& kernelCol + val inBounds = sourceRow >= padding && sourceRow < padding +& size && + sourceCol >= padding && sourceCol < padding +& size + val sourceLine = inputBase +& (sourceRow - padding) * size +& (sourceCol - padding) + val outputLine = (((window >> tileShift) * groupTiles +& localTile) << tileShift) | + (window & (tileRows - 1.U)) + val lastTile = kernelRow +& 1.U === kernel && kernelCol +& 1.U === kernel && inGroup +& 1.U === inGroups + + io.cmdReq.ready := state === idle + io.cmdResp.valid := state === complete + io.cmdResp.bits.rob_id := robId + io.status.idle := state === idle + io.status.running := state =/= idle && state =/= complete + + io.bankRead(0).rob_id := robId + io.bankRead(0).bank_id := inBank + io.bankRead(0).group_id := inBase + inGroup + io.bankRead(0).io.req.valid := state === readReq + io.bankRead(0).io.req.bits.addr := sourceLine(addressWidth - 1, 0) + io.bankRead(0).io.resp.ready := state === readResp + + io.bankWrite(0).rob_id := robId + io.bankWrite(0).bank_id := outBank + io.bankWrite(0).group_id := outBase + outGroup + io.bankWrite(0).io.req.valid := state === writeReq + io.bankWrite(0).io.req.bits.addr := outputLine(addressWidth - 1, 0) + io.bankWrite(0).io.req.bits.data := rowData + io.bankWrite(0).io.req.bits.mask := VecInit(Seq.fill(memParam.bankMaskLen)(true.B)) + io.bankWrite(0).io.resp.ready := state === writeResp + + switch(state) { + is(idle) { + when(io.cmdReq.fire) { + val cmd = io.cmdReq.bits + val tileValue = cmd.rs2(50, 46) + assert(cmd.rs2(63, 51) === 0.U, "IM2COL reserves rs2[63:51]") + assert( + cmd.iter =/= 0.U && cmd.iter <= maxIter.U, + "IM2COL input size must be in the configured range" + ) + assert( + cmd.rs2(3, 0) =/= 0.U && cmd.rs2(3, 0) <= maxKSize.U, + "IM2COL kernel must be in the configured range" + ) + assert(cmd.rs2(7, 4) =/= 0.U, "IM2COL stride must be positive") + assert(cmd.rs2(11, 8) <= maxPad.U, "IM2COL padding exceeds the configured range") + assert( + cmd.rs2(15, 12) <= cmd.rs2(11, 8) && cmd.rs2(19, 16) <= cmd.rs2(11, 8), + "IM2COL start must not exceed padding" + ) + assert(cmd.rs2(45, 36) =/= 0.U, "IM2COL window count must be positive") + assert(PopCount(tileValue) === 1.U && tileValue <= 16.U, "IM2COL tile rows must be 1, 2, 4, 8, or 16") + assert( + cmd.rs2(25, 20) +& cmd.iter * cmd.iter <= memParam.bankEntries.U, + "IM2COL input footprint exceeds bank depth" + ) + assert( + cmd.op1_col =/= 0.U && cmd.op2_col === 0.U && cmd.wr_col =/= 0.U && + cmd.op1_col <= memParam.bankNum.U && cmd.wr_col <= memParam.bankNum.U, + "IM2COL requires input and output groups" + ) + assert(cmd.op1_bank =/= cmd.wr_bank, "IM2COL input and output banks must differ") + assert( + cmd.iter +& (cmd.rs2(11, 8) << 1) >= cmd.rs2(3, 0) +& cmd.rs2(19, 16) && + cmd.iter +& (cmd.rs2(11, 8) << 1) >= cmd.rs2(3, 0) +& cmd.rs2(15, 12), + "IM2COL kernel and start exceed padded input" + ) + + robId := cmd.rob_id + inBank := cmd.op1_bank + outBank := cmd.wr_bank + inBase := cmd.op1_group + outBase := cmd.wr_group + inGroups := cmd.op1_col + outGroups := cmd.wr_col + size := cmd.iter + kernel := cmd.rs2(3, 0) + stride := cmd.rs2(7, 4) + padding := cmd.rs2(11, 8) + startCol := cmd.rs2(15, 12) + startRow := cmd.rs2(19, 16) + inputBase := cmd.rs2(25, 20) + windowStart := cmd.rs2(35, 26) + windowCount := cmd.rs2(45, 36) + tileShift := OHToUInt(tileValue) + state := divRows + div.io.start := true.B + div.io.a := cmd.iter +& (cmd.rs2(11, 8) << 1) - cmd.rs2(3, 0) - cmd.rs2(19, 16) + div.io.b := cmd.rs2(7, 4) + } + } + + is(divRows) { + when(div.io.done) { + assert(div.io.r === 0.U, "IM2COL output row shape must be integral") + outRows := div.io.q + 1.U + div.io.start := true.B + div.io.a := padded - kernel - startCol + div.io.b := stride + state := divCols + } + } + + is(divCols) { + when(div.io.done) { + assert(div.io.r === 0.U, "IM2COL output column shape must be integral") + outCols := div.io.q + 1.U + div.io.start := true.B + div.io.a := windowStart + div.io.b := div.io.q + 1.U + state := divStart + } + } + + is(divStart) { + when(div.io.done) { + firstRow := div.io.q + firstCol := div.io.r + div.io.start := true.B + div.io.a := kernel * kernel * inGroups + div.io.b := outGroups + state := divTiles + } + } + + is(divTiles) { + when(div.io.done) { + assert(div.io.r === 0.U, "IM2COL output groups must divide the K tiles") + groupTiles := div.io.q + state := check + } + } + + is(check) { + assert( + windowStart +& windowCount <= outRows * outCols, + "IM2COL selected window range exceeds the output shape" + ) + assert( + paddedWindows * groupTiles <= memParam.bankEntries.U, + "IM2COL output footprint exceeds bank depth" + ) + kernelRow := 0.U + kernelCol := 0.U + inGroup := 0.U + outGroup := 0.U + localTile := 0.U + window := 0.U + windowRow := firstRow + windowCol := firstCol + state := select + } + + // The current output row is an input row, or zero for padding and for + // windows past windowCount. + is(select) { + when(window < windowCount && inBounds) { + state := readReq + }.otherwise { + rowData := 0.U + state := writeReq + } + } + + is(readReq) { + when(io.bankRead(0).io.req.fire) { + state := readResp + } + } + + is(readResp) { + when(io.bankRead(0).io.resp.fire) { + rowData := io.bankRead(0).io.resp.bits.data + state := writeReq + } + } + + is(writeReq) { + when(io.bankWrite(0).io.req.fire) { + state := writeResp + } + } + + is(writeResp) { + when(io.bankWrite(0).io.resp.fire) { + when(window +& 1.U =/= paddedWindows) { + window := window + 1.U + when(windowCol +& 1.U === outCols) { + windowCol := 0.U + windowRow := windowRow + 1.U + }.otherwise { + windowCol := windowCol + 1.U + } + state := select + }.elsewhen(lastTile) { + state := complete + }.otherwise { + window := 0.U + windowRow := firstRow + windowCol := firstCol + when(inGroup +& 1.U === inGroups) { + inGroup := 0.U + when(kernelCol +& 1.U === kernel) { + kernelCol := 0.U + kernelRow := kernelRow + 1.U + }.otherwise { + kernelCol := kernelCol + 1.U + } + }.otherwise { + inGroup := inGroup + 1.U + } + when(localTile +& 1.U === groupTiles) { + localTile := 0.U + outGroup := outGroup + 1.U + }.otherwise { + localTile := localTile + 1.U + } + state := select + } + } + } + + is(complete) { + when(io.cmdResp.fire) { + state := idle + } + } + } +} diff --git a/examples/prefab/balls/im2col/arch/Im2colBall.scala b/examples/prefab/balls/im2col/arch/Im2colBall.scala new file mode 100644 index 000000000..42754602c --- /dev/null +++ b/examples/prefab/balls/im2col/arch/Im2colBall.scala @@ -0,0 +1,74 @@ +package examples.balls.im2col + +import framework.acc.balldomain.bbus.{BallFactory, BallPorts} +import buckyball.balls.im2col.{Params => ParametersProto} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import examples.balls.im2col.configs.Im2colBallParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.balldomain.ball.blink.{BlinkIO, HasBlink} +import examples.balls.im2col.Im2col + +/** + * Im2colBall - An Im2col computation Ball that complies with the Blink protocol + */ +@instantiable +class Im2colBall( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Module + with HasBlink { + val ball = bp.balls.find(_.id == id).get + + val ballParam = Im2colBallParam.fromProto(ball.parameters.unpack(classOf[ParametersProto])) + + val ballCommonConfig = ball + + val inBW = ballCommonConfig.inBW + val outBW = ballCommonConfig.outBW + + @public + val blink = IO(new BlinkIO(fp, mp, bp, id)) + + // Instantiate Im2col + val im2colUnit: Instance[Im2col] = + Instantiate(new Im2col(fp, mp, bp, ball, ballParam)) + + // Connect command interface + im2colUnit.io.cmdReq <> blink.cmdReq + im2colUnit.io.cmdResp <> blink.cmdResp + + for (i <- 0 until inBW) { + im2colUnit.io.bankRead(i) <> blink.bankRead(i) + } + + // Connect SRAM write interface - Im2col needs to write to scratchpad + for (i <- 0 until outBW) { + im2colUnit.io.bankWrite(i) <> blink.bankWrite(i) + } + + // Connect Status signals - directly obtained from internal unit + blink.status <> im2colUnit.io.status + + // MMIO: this Ball does not consume MMIO metadata +} + +object Im2colBall extends BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts = + BallPorts(Instantiate(new Im2colBall(fp, mp, bp, id)).blink, None) + +} diff --git a/examples/prefab/balls/im2col/arch/README.md b/examples/prefab/balls/im2col/arch/README.md new file mode 100644 index 000000000..71463a5c1 --- /dev/null +++ b/examples/prefab/balls/im2col/arch/README.md @@ -0,0 +1,58 @@ +# Im2col Ball + +This directory implements the Buckyball `Im2colBall` RTL for instruction +function 48. + +## ISA contract + +```c +bb_im2col(input_bank, output_bank, input_size, kernel, stride, padding, + input_base, start_row, start_col, window_start, window_count, + tile_rows) +``` + +- `rs1`: bank0 is the input, bank2 the output, `iter` the input height and + width. +- `rs2`: `[3:0]` kernel, `[7:4]` stride, `[11:8]` padding, `[15:12]` + start_col, `[19:16]` start_row, `[25:20]` input_base, `[35:26]` + window_start, `[45:36]` window_count, `[50:46]` tile_rows (1, 2, 4, 8, or + 16); `[63:51]` are zero. +- Im2col only moves INT8 bytes; it does not multiply or accumulate. + +## Layout + +Every group of the input virtual bank is one channel block: group `g` line +`input_base + y * input_size + x` holds channels `16g .. 16g + 15` of pixel +`(y, x)`. With `Cg` input groups, `C = 16 * Cg`. + +The output is SMatMulBall operand A for `M = window_count` windows and +`K = kernel * kernel * C`, ordered `(kernel row, kernel column, channel)`: + +```text +outputRows = (input_size + 2 * padding - kernel - start_row) / stride + 1 +outputCols = (input_size + 2 * padding - kernel - start_col) / stride + 1 +window w = output pixel window_start + w (row-major) +K tile t = (kr * kernel + kc) * Cg + g // one whole input row +K_TILES = kernel * kernel * Cg +G = K_TILES / output_groups // output groups divide K_TILES +``` + +Output group `j` holds K tiles `j * G .. j * G + G - 1`, in the SMatMulBall A +layout with `tile_rows`-row M tiles: + +```text +line(w, t) = ((w / tile_rows) * G + t % G) * tile_rows + w % tile_rows +``` + +A tile row is the input row of pixel +`(oy * stride + kr + start_row - padding, ox * stride + kc + start_col - padding)`, +or zero when that pixel is in the padding. Windows from `window_count` to the +next multiple of `tile_rows` are zero. Each output group therefore is the +SMatMulBall reduction block `j` with `k = 16 * G`. + +## Files + +- `Im2col.scala`: command decode, shape division, and the row-copy loop. +- `RestoringDiv.scala`: shared sequential divider. +- `Im2colBall.scala`: Blink-compatible ball wrapper. +- `configs/Im2colBallParam.scala`: maximum dimension and element width. diff --git a/examples/balls/im2col/arch/src/main/scala/RestoringDiv.scala b/examples/prefab/balls/im2col/arch/RestoringDiv.scala similarity index 83% rename from examples/balls/im2col/arch/src/main/scala/RestoringDiv.scala rename to examples/prefab/balls/im2col/arch/RestoringDiv.scala index 5af0a851e..88d86fb06 100644 --- a/examples/balls/im2col/arch/src/main/scala/RestoringDiv.scala +++ b/examples/prefab/balls/im2col/arch/RestoringDiv.scala @@ -1,11 +1,18 @@ package examples.balls.im2col +import chisel3.experimental.hierarchy.{instantiable, public} + +import examples.balls.im2col.configs.Im2colBallParam + import chisel3._ import chisel3.util._ -class RestoringDiv(n: Int, d: Int) extends Module { +@instantiable +class RestoringDiv(val n: Int, val d: Int, val ballParam: Im2colBallParam) extends Module { + require(n >= 1 && d >= 1, "RestoringDiv widths must be >= 1") + @public val io = IO(new Bundle { val start = Input(Bool()) val a = Input(UInt(n.W)) diff --git a/examples/prefab/balls/im2col/configs/Im2colBallParam.scala b/examples/prefab/balls/im2col/configs/Im2colBallParam.scala new file mode 100644 index 000000000..9ea577424 --- /dev/null +++ b/examples/prefab/balls/im2col/configs/Im2colBallParam.scala @@ -0,0 +1,22 @@ +package examples.balls.im2col.configs + +import buckyball.balls.im2col.{Params => ParametersProto} +import upickle.default.{macroRW, ReadWriter} + +case class Im2colBallParam( + maxIter: Int, + maxKSize: Int, + maxPadding: Int, + inputWidth: Int) + +object Im2colBallParam { + + def fromProto(source: ParametersProto): Im2colBallParam = { + val parameters = source + val param = + Im2colBallParam(parameters.getMaxIter, parameters.getMaxKSize, parameters.getMaxPadding, parameters.getInputWidth) + param + } + + implicit val rw: ReadWriter[Im2colBallParam] = macroRW +} diff --git a/examples/prefab/balls/im2col/configs/default.toml b/examples/prefab/balls/im2col/configs/default.toml new file mode 100644 index 000000000..b743f7bad --- /dev/null +++ b/examples/prefab/balls/im2col/configs/default.toml @@ -0,0 +1,23 @@ +# Im2col Ball Configuration +# +# Legal command range (enforced fail-hard in bemu/RTL): +# iter in 1 ..= maxIter +# ksize in 1 ..= maxKSize # covers 1x1 / 3x3 / 5x5 / 7x7 +# stride >= 1 +# padding in 0 ..= maxPadding +# start_row / start_col in 0 ..= padding (asym pad: pad=max(lo,hi), start=pad-lo) +# paddedSize = iter + 2*padding >= ksize + start +# op1_bank != wr_bank +# inputBase + iter*iter <= bankEntries +# window_start + window_count <= outputDim * outputDim +# K tiles = ksize * ksize * input groups, divisible by output groups +# ceil(window_count / tile_rows) * tile_rows * K tiles / output groups <= bankEntries +# +# Input groups are 16-channel blocks; one bank row is one spatial position. + +inputWidth = 8 +# Resident Conv lowering can request a 4x4 output core from a stride-2, +# kernel-7 producer, which needs a 13x13 local input. +maxIter = 16 +maxKSize = 7 +maxPadding = 7 diff --git a/examples/prefab/balls/im2col/configs/im2col_params.proto b/examples/prefab/balls/im2col/configs/im2col_params.proto new file mode 100644 index 000000000..203f6785a --- /dev/null +++ b/examples/prefab/balls/im2col/configs/im2col_params.proto @@ -0,0 +1,11 @@ +syntax = "proto3"; +package buckyball.balls.im2col; +option java_package = "buckyball.balls.im2col"; +option java_multiple_files = true; + +message Params { + optional int32 inputWidth = 1; + optional int32 maxIter = 2; + optional int32 maxKSize = 3; + optional int32 maxPadding = 4; +} diff --git a/examples/prefab/balls/im2col/emu/src/dispatch.rs b/examples/prefab/balls/im2col/emu/src/dispatch.rs new file mode 100644 index 000000000..522677147 --- /dev/null +++ b/examples/prefab/balls/im2col/emu/src/dispatch.rs @@ -0,0 +1,31 @@ +use bebop_bemu::ball::ExecContext; + +use super::im2col; + +const BALL_CLASS: &str = "examples.balls.im2col.Im2colBall"; + +pub fn execute_known( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, + ctx: &mut ExecContext, +) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("IM2COL") => Some(im2col::exec_im2col(xs1, xs2, ctx)), + Some(_) | None => None, + } +} + +pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("IM2COL") => Some(im2col::latency(xs1, xs2, bebop_bemu::configs::ball_domain::instance_for_funct(funct).expect("Ball instance"))), + Some(_) | None => None, + } +} diff --git a/examples/prefab/balls/im2col/emu/src/im2col.rs b/examples/prefab/balls/im2col/emu/src/im2col.rs new file mode 100644 index 000000000..a0fe9d9a2 --- /dev/null +++ b/examples/prefab/balls/im2col/emu/src/im2col.rs @@ -0,0 +1,174 @@ +use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old}; +use super::instruction::ExecContext; +use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; + +fn max_iter(instance: u32) -> usize { + bebop_bemu::configs::ball_domain::parameters::< + bebop_bemu::configs::ball_domain::parameters::im2col::Params, + >("im2col", instance) + .max_iter + .expect("maxIter") as usize +} + +fn max_kernel(instance: u32) -> usize { + bebop_bemu::configs::ball_domain::parameters::< + bebop_bemu::configs::ball_domain::parameters::im2col::Params, + >("im2col", instance) + .max_k_size + .expect("maxKSize") as usize +} + +fn max_padding(instance: u32) -> usize { + bebop_bemu::configs::ball_domain::parameters::< + bebop_bemu::configs::ball_domain::parameters::im2col::Params, + >("im2col", instance) + .max_padding + .expect("maxPadding") as usize +} + +#[derive(Clone, Copy)] +struct Shape { + input_size: usize, + kernel: usize, + stride: usize, + padding: usize, + start_col: usize, + start_row: usize, + input_base: usize, + window_start: usize, + window_count: usize, + tile_rows: usize, +} + +fn decode_shape(xs1: u64, xs2: u64) -> Shape { + if rs1_b1(xs1) != 0 { + panic!("im2col: input bank 1 must be zero"); + } + if xs2 >> 51 != 0 { + panic!("im2col: rs2[63:51] must be zero"); + } + Shape { + input_size: rs1_iter(xs1) as usize, + kernel: (xs2 & 0xf) as usize, + stride: ((xs2 >> 4) & 0xf) as usize, + padding: ((xs2 >> 8) & 0xf) as usize, + start_col: ((xs2 >> 12) & 0xf) as usize, + start_row: ((xs2 >> 16) & 0xf) as usize, + input_base: ((xs2 >> 20) & 0x3f) as usize, + window_start: ((xs2 >> 26) & 0x3ff) as usize, + window_count: ((xs2 >> 36) & 0x3ff) as usize, + tile_rows: ((xs2 >> 46) & 0x1f) as usize, + } +} + +// Returns the output columns of the full convolution output. +fn output_cols(shape: Shape, instance: u32) -> usize { + if shape.input_size == 0 + || shape.input_size > max_iter(instance) + || shape.kernel == 0 + || shape.kernel > max_kernel(instance) + || shape.stride == 0 + || shape.padding > max_padding(instance) + || shape.start_row > shape.padding + || shape.start_col > shape.padding + || shape.window_count == 0 + || !shape.tile_rows.is_power_of_two() + || shape.tile_rows > 16 + || shape.input_base + shape.input_size * shape.input_size > bank_lines() + { + panic!("im2col: illegal shape field"); + } + let padded = shape.input_size + shape.padding * 2; + if padded < shape.kernel + shape.start_row || padded < shape.kernel + shape.start_col { + panic!("im2col: kernel and start exceed padded input"); + } + let row_numerator = padded - shape.kernel - shape.start_row; + let col_numerator = padded - shape.kernel - shape.start_col; + if row_numerator % shape.stride != 0 || col_numerator % shape.stride != 0 { + panic!("im2col: output shape is not integral"); + } + let output_rows = row_numerator / shape.stride + 1; + let output_cols = col_numerator / shape.stride + 1; + if shape.window_start + shape.window_count > output_rows * output_cols { + panic!("im2col: selected window range exceeds output shape"); + } + output_cols +} + +pub(crate) fn exec_im2col(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + assert!(!rs1_read_old(xs1), "im2col: reads no old destination; rs1[29] must be clear"); + let input_bank = rs1_b0(xs1); + let output_bank = rs1_b2(xs1); + if input_bank == output_bank { + panic!("im2col: input and output banks must differ"); + } + if !ctx.config(input_bank).allocated || !ctx.config(output_bank).allocated { + panic!("im2col: bank not allocated"); + } + let shape = decode_shape(xs1, xs2); + let output_cols = output_cols(shape, ctx.ball_id.expect("Ball instance")); + + // Input group g is channel block g. K is ordered (kernel row, kernel + // column, channel), so K tile t = kernel_position * channel_groups + g is + // one whole input row. Output group j holds K tiles j*group_tiles.. in the + // SMatMul A layout with tile_rows-row M tiles. + let channel_groups = ctx.config(input_bank).cols as usize; + let output_groups = ctx.config(output_bank).cols as usize; + let k_tiles = shape.kernel * shape.kernel * channel_groups; + if channel_groups == 0 || output_groups == 0 || k_tiles % output_groups != 0 { + panic!("im2col: output groups must divide the K tiles"); + } + let group_tiles = k_tiles / output_groups; + let padded_windows = shape.window_count.div_ceil(shape.tile_rows) * shape.tile_rows; + if padded_windows * group_tiles > bank_lines() { + panic!("im2col: output footprint exceeds bank depth"); + } + + let row_bytes = bank_row_bytes(); + for tile in 0..k_tiles { + let channel_group = tile % channel_groups; + let kernel_row = tile / channel_groups / shape.kernel; + let kernel_col = tile / channel_groups % shape.kernel; + let pi = pbank_group(ctx, input_bank, channel_group as u64); + let po = pbank_group(ctx, output_bank, (tile / group_tiles) as u64); + let (input, output) = ctx.banks.read_write(pi, po); + for window in 0..padded_windows { + let line = ((window / shape.tile_rows) * group_tiles + tile % group_tiles) + * shape.tile_rows + + window % shape.tile_rows; + let target = &mut output[line * row_bytes..(line + 1) * row_bytes]; + target.fill(0); + if window >= shape.window_count { + continue; + } + let global = shape.window_start + window; + let source_row = (shape.start_row + global / output_cols * shape.stride + kernel_row) + as isize + - shape.padding as isize; + let source_col = (shape.start_col + global % output_cols * shape.stride + kernel_col) + as isize + - shape.padding as isize; + if source_row < 0 + || source_col < 0 + || source_row >= shape.input_size as isize + || source_col >= shape.input_size as isize + { + continue; + } + let source = + shape.input_base + source_row as usize * shape.input_size + source_col as usize; + target.copy_from_slice(&input[source * row_bytes..(source + 1) * row_bytes]); + } + } + 0 +} + +pub(crate) fn latency(xs1: u64, xs2: u64, instance: u32) -> u64 { + let shape = decode_shape(xs1, xs2); + output_cols(shape, instance); + (shape.window_count.div_ceil(shape.tile_rows) + * shape.tile_rows + * shape.kernel + * shape.kernel + * 2) as u64 +} diff --git a/examples/prefab/balls/im2col/emu/src/lib.rs b/examples/prefab/balls/im2col/emu/src/lib.rs new file mode 100644 index 000000000..96b3d1aed --- /dev/null +++ b/examples/prefab/balls/im2col/emu/src/lib.rs @@ -0,0 +1,11 @@ +pub(crate) use bebop_bemu::{thread::instructions::decode, ball as instruction}; + +mod im2col; + +mod dispatch; + +pub use dispatch::{cycles_after_issue, execute_known}; + +#[bebop_bemu::ball::linkme::distributed_slice(bebop_bemu::BALLS)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static BALL: bebop_bemu::Ball = bebop_bemu::Ball { execute: execute_known, cycles: cycles_after_issue }; diff --git a/examples/prefab/balls/im2col/stack/src/im2colOps.cpp b/examples/prefab/balls/im2col/stack/src/im2colOps.cpp new file mode 100644 index 000000000..98ae6eb64 --- /dev/null +++ b/examples/prefab/balls/im2col/stack/src/im2colOps.cpp @@ -0,0 +1,85 @@ +//===- im2colOps.cpp - Im2colBall operations ------------------------------===// +// +// The dialect of Im2colBall, its virtual operations building their physical +// ones and the instructions of the physical operations. +// +//===----------------------------------------------------------------------===// + +#include "im2colOps.h" + +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::im2col_ball; +using mlir::buddy::bankFields; +using mlir::buddy::constantI64; +using mlir::buddy::emitInstruction; +using mlir::buddy::field; + +#include "im2colDialect.cpp.inc" + +#define GET_OP_CLASSES +#include "im2colOps.cpp.inc" + +void Im2colBallDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "im2colOps.cpp.inc" + >(); +} + +namespace { +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallDialect([](DialectRegistry ®istry) { + registry.insert(); + }), + true); +} // namespace + +SmallVector BankIm2colI8Op::getSourceBanks() { return {getInput()}; } +Value BankIm2colI8Op::getDestinationBank() { return getOutput(); } +bool BankIm2colI8Op::readsDestination() { return false; } +Operation *BankIm2colI8Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return Im2colI8Op::create(builder, getLoc(), sources[0], destination, + getInputSide(), getKernel(), getStride(), + getPadding(), getTileRows()); +} + +SmallVector Im2colI8Op::getSourceBanks() { + return {int64_t(getInput())}; +} +int64_t Im2colI8Op::getDestinationBank() { return getOutput(); } +LogicalResult Im2colI8Op::lowerToInstructions(OpBuilder &builder) { + Location loc = getLoc(); + int64_t input = getInput(), output = getOutput(); + if (input == output) + return emitOpError("im2col_i8 requires distinct input and output banks"); + int64_t side = getInputSide(), kernel = getKernel(), stride = getStride(), + padding = getPadding(), tileRows = getTileRows(); + const auto &target = buckyball_target::getBuckyballTarget(); + int64_t span = side + 2 * padding - kernel; + if (target.bankWidthBits != 128 || side <= 0 || + side > buckyball_target::getBuckyballBallParam("IM2COL", "maxIter") || + side * side > target.bankDepth || kernel <= 0 || + kernel > buckyball_target::getBuckyballBallParam("IM2COL", "maxKSize") || + stride <= 0 || stride > 15 || padding < 0 || + padding > + buckyball_target::getBuckyballBallParam("IM2COL", "maxPadding") || + span < 0 || span % stride || tileRows <= 0 || tileRows > 16 || + (tileRows & (tileRows - 1))) + return emitOpError("im2col_i8 geometry exceeds the ISA field ranges"); + int64_t windows = (span / stride + 1) * (span / stride + 1); + if (windows > 1023) + return emitOpError("im2col_i8 has more windows than one instruction"); + uint64_t rs2 = field(kernel, 0, 3) | field(stride, 4, 7) | + field(padding, 8, 11) | field(windows, 36, 45) | + field(tileRows, 46, 50); + emitInstruction( + builder, loc, buckyball_target::getBuckyballFunct7("IM2COL"), + constantI64(builder, loc, bankFields({input}, output, false, side)), + constantI64(builder, loc, rs2)); + return success(); +} diff --git a/examples/prefab/balls/im2col/stack/src/im2colOps.h b/examples/prefab/balls/im2col/stack/src/im2colOps.h new file mode 100644 index 000000000..7afdc31ca --- /dev/null +++ b/examples/prefab/balls/im2col/stack/src/im2colOps.h @@ -0,0 +1,20 @@ +//===- im2colOps.h - Im2colBall operations --------------------------------===// + +#ifndef IM2COL_OPS_H +#define IM2COL_OPS_H + +#include "Interfaces/BankInterfaces.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "im2colDialect.h.inc" + +#define GET_OP_CLASSES +#include "im2colOps.h.inc" + +#endif // IM2COL_OPS_H diff --git a/examples/prefab/balls/im2col/stack/src/im2colOps.td b/examples/prefab/balls/im2col/stack/src/im2colOps.td new file mode 100644 index 000000000..41d44a409 --- /dev/null +++ b/examples/prefab/balls/im2col/stack/src/im2colOps.td @@ -0,0 +1,52 @@ +//===- im2colOps.td - Im2colBall operations ----------------*- tablegen -*-===// +// +// im2col_i8: the input bank holds a square feature map of `input_side` pixels +// per side; group g is channel block g, line y * input_side + x holds the +// sixteen channels of pixel (y, x). The output receives the windows of the +// `kernel` x `kernel` convolution with `stride` and zero `padding`, every +// window in row-major order, as the matmul_i8 A operand: K is ordered (kernel +// row, kernel column, channel), sixteen columns (one input line) per K tile, +// K tile t = (kr * kernel + kc) * groups + g. The output groups divide the K +// tiles; group j holds the next K / 16 / groups tiles in M tiles of +// `tile_rows` rows (the layout of the matmul that consumes it), windows beyond +// the last zero up to a multiple of tile_rows. +// +//===----------------------------------------------------------------------===// + +#ifndef IM2COL_OPS +#define IM2COL_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Interfaces/BankInterfaces.td" + +def Im2colBall_Dialect : Dialect { + let name = "im2col_ball"; + let cppNamespace = "::buddy::im2col_ball"; + let summary = "The operations of Im2colBall"; +} + +class Im2colBall_Op traits = []> + : Op])>; + +def BankIm2colI8Op : Im2colBall_Op<"bank_im2col_i8", + [DeclareOpInterfaceMethods]> { + let summary = "im2col_i8 on virtual banks"; + let arguments = (ins I64:$input, I64:$output, I64Attr:$input_side, + I64Attr:$kernel, I64Attr:$stride, I64Attr:$padding, + I64Attr:$tile_rows); + let results = (outs I64:$outputOut); + let assemblyFormat = "$input `->` $output prop-dict attr-dict"; +} + +def Im2colI8Op : Im2colBall_Op<"im2col_i8", + [DeclareOpInterfaceMethods]> { + let summary = "im2col_i8 on physical banks"; + let arguments = (ins I64Attr:$input, I64Attr:$output, I64Attr:$input_side, + I64Attr:$kernel, I64Attr:$stride, I64Attr:$padding, + I64Attr:$tile_rows); + let assemblyFormat = "prop-dict attr-dict"; +} + +#endif // IM2COL_OPS diff --git a/examples/prefab/balls/im2col/verify/README.md b/examples/prefab/balls/im2col/verify/README.md new file mode 100644 index 000000000..dfef4cb7b --- /dev/null +++ b/examples/prefab/balls/im2col/verify/README.md @@ -0,0 +1,88 @@ +# Im2colBall UVM Verification + +Verifies the generated `Im2colBall` module directly using the shared Blink UVM framework. + +## Structure + +- `../../../../verify/uvm/src/ball/bb_uvm_pkg.sv`: common Blink UVM transaction items and base env +- `src/common/im2col_defs.svh`: DPI imports and constants +- `src/common/im2col_items.svh`: `im2col_cmd_item` with `load_rust_case` +- `src/seq/im2col_sequences.svh`: `im2col_basic_seq` driving one case per run +- `src/cov/im2col_cov.svh`: kernel/stride/padding/start/base/window/tile_rows/group coverage +- `src/env/im2col_scoreboard.svh`: preloads every input group, checks reads and writes in issue order +- `src/env/im2col_env.svh`: extends `bb_blink_env#(1,1)` with scoreboard and cov +- `src/tests/im2col_test.svh`: runs the directed cases in order +- `src/pkg/im2col_pkg.sv`: package include entry +- `src/tb_top.sv`: instantiates `bb_blink_if#(1,1)` and `Im2colBall` +- `filelists/im2col_ball.f`: VCS/Verilator filelist + +## Test plan + +`casegen/src/model.rs` is the reference (same semantics as `emu/src/im2col.rs`): +input group g is channel block g, K tile `t = (kr * kernel + kc) * in_groups + g` +copies one input row, output group `t / G` (`G = K_TILES / wr_col`) line +`((w / tile_rows) * G + t % G) * tile_rows + w % tile_rows`. The scoreboard checks +every read (group, line) and every write (group, line, data) in the Ball's issue +order: K tile outer, window inner. + +Directed cases (`casegen/src/casegen.rs`): + +- 0: 1x1 kernel, 2x2 input, two input and two output groups +- 1: 3x3 kernel, 4x4 input, nine output groups (one tap each), tile_rows 8 +- 2: 3x3 kernel, padding 1, 2x2 input, one output group, tile_rows 4 +- 3: bottom/right padding (start_row = start_col = 1), 2x2 kernel, four output groups +- 4: window range 5..7 at input_base 2, partial M tile zero-filled +- 5: 5x5 kernel, tile_rows 1, five output groups (one image row each) +- 6: 7x7 stride-2 kernel, 9x9 input, seven output groups +- 7: 2x2 kernel, two input groups, input_base 3, windows 1..2, eight output groups + +Op1/wr base groups are nonzero in several cases. + +## BID (required) + +`+BID=` is mandatory. Missing it fatals at test start. There is no default bid. + +| Config | Plusarg | +|--------|---------| +| `sims.verilator.BuckyballPebbleVerilatorConfig` | `+BID=2` | +| `sims.verilator.BuckyballToyVerilatorConfig` | `+BID=3` | + +```console +./simv +UVM_TESTNAME=im2col_ball_test +BID=2 +``` + +## Build + +Enter the shared UVM/VCS environment: + +```console +nix develop ../../../../verify +``` + +Build the DPI reference library: + +```console +cargo build --manifest-path casegen/Cargo.toml +``` + +Compile from this directory (resolve `@UVM@` / `@RTL@` like bbdev): + +```console +# Do not pass -hsopt=off: VCS W-2024.09 segfaults on Im2colBall+UVM with it. +vcs -full64 -sverilog -timescale=1ns/1ps \ + $VCS_UVM_ARGS \ + -sv_lib ../output/casegen/debug/libim2col_casegen \ + -f build//im2col_ball.f +``` + +Run: + +```console +./simv +UVM_TESTNAME=im2col_ball_test +BID=2 +``` + +Or via bbdev: + +```console +bbdev uvm --run '--design pebble --ball im2col' +``` diff --git a/examples/balls/im2col/verify/casegen/Cargo.lock b/examples/prefab/balls/im2col/verify/casegen/Cargo.lock similarity index 100% rename from examples/balls/im2col/verify/casegen/Cargo.lock rename to examples/prefab/balls/im2col/verify/casegen/Cargo.lock diff --git a/examples/balls/im2col/verify/casegen/Cargo.toml b/examples/prefab/balls/im2col/verify/casegen/Cargo.toml similarity index 100% rename from examples/balls/im2col/verify/casegen/Cargo.toml rename to examples/prefab/balls/im2col/verify/casegen/Cargo.toml diff --git a/examples/prefab/balls/im2col/verify/casegen/src/casegen.rs b/examples/prefab/balls/im2col/verify/casegen/src/casegen.rs new file mode 100644 index 000000000..997dfbe11 --- /dev/null +++ b/examples/prefab/balls/im2col/verify/casegen/src/casegen.rs @@ -0,0 +1,182 @@ +use crate::model::{im2col, Row, Shape}; + +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Im2colCmd { + pub iter: u32, + pub rs2_lo: u32, + pub rs2_hi: u32, + pub op1_bank: u32, + pub wr_bank: u32, + pub op1_group: u32, + pub wr_group: u32, + pub op1_col: u32, + pub wr_col: u32, + pub rob_id: u32, + pub num_src: u32, + pub num_reads: u32, + pub num_dst: u32, +} + +pub struct Im2colCase { + pub cmd: Im2colCmd, + // (input group, line, data) preloaded into the input bank. + pub src: Vec<(usize, usize, Row)>, + pub reads: Vec<(usize, usize)>, + pub dst: Vec<(usize, usize, Row)>, +} + +pub fn gen_case(index: u32) -> Im2colCase { + match index { + 0 => k1_two_groups(), + 1 => k3_nine_output_groups(), + 2 => k3_pad_tile4(), + 3 => asym_pad(), + 4 => partial_window_range(), + 5 => k5_tile_rows_1(), + 6 => k7_stride2(), + 7 => window_input_base_two_groups(), + _ => panic!("im2col: unsupported directed case {index}"), + } +} + +fn finish( + shape: Shape, + input: Vec>, + banks: (u32, u32), + groups: (u32, u32), + rob_id: u32, +) -> Im2colCase { + let trace = im2col(&shape, &input); + let rs2 = shape.rs2(); + let mut src = Vec::new(); + for (g, rows) in input.iter().enumerate() { + for (line, row) in rows.iter().enumerate() { + src.push((g, line, *row)); + } + } + Im2colCase { + cmd: Im2colCmd { + iter: shape.size as u32, + rs2_lo: rs2 as u32, + rs2_hi: (rs2 >> 32) as u32, + op1_bank: banks.0, + wr_bank: banks.1, + op1_group: groups.0, + wr_group: groups.1, + op1_col: shape.in_groups as u32, + wr_col: shape.out_groups as u32, + rob_id, + num_src: src.len() as u32, + num_reads: trace.reads.len() as u32, + num_dst: trace.writes.len() as u32, + }, + src, + reads: trace.reads, + dst: trace.writes, + } +} + +fn row(seed: usize) -> Row { + let mut r = [0u8; 16]; + for (lane, byte) in r.iter_mut().enumerate() { + *byte = (seed * 16 + lane) as u8; + } + r +} + +// 1x1 kernel on a 2x2 input, 32 channels: output group g copies input group g. +fn k1_two_groups() -> Im2colCase { + let input = vec![ + (0..4).map(|p| row(p * 2 + 1)).collect(), + (0..4).map(|p| row(p * 2 + 40)).collect(), + ]; + let shape = Shape { + size: 2, kernel: 1, stride: 1, padding: 0, start_row: 0, start_col: 0, + input_base: 0, window_start: 0, window_count: 4, tile_rows: 4, + in_groups: 2, out_groups: 2, + }; + finish(shape, input, (0, 1), (0, 0), 1) +} + +// 3x3 kernel on a 4x4 input: 2x2 windows, nine output groups (one tap each), +// lines 4..7 of each 8-row M tile are zero. +fn k3_nine_output_groups() -> Im2colCase { + let input = vec![(0..16).map(|p| row(p * 3 + 2)).collect()]; + let shape = Shape { + size: 4, kernel: 3, stride: 1, padding: 0, start_row: 0, start_col: 0, + input_base: 0, window_start: 0, window_count: 4, tile_rows: 8, + in_groups: 1, out_groups: 9, + }; + finish(shape, input, (2, 5), (1, 3), 2) +} + +// 3x3 kernel, padding 1, 2x2 input: nine taps in one group, 4-row M tile. +fn k3_pad_tile4() -> Im2colCase { + let input = vec![(0..4).map(|p| row(p + 7)).collect()]; + let shape = Shape { + size: 2, kernel: 3, stride: 1, padding: 1, start_row: 0, start_col: 0, + input_base: 0, window_start: 0, window_count: 4, tile_rows: 4, + in_groups: 1, out_groups: 1, + }; + finish(shape, input, (3, 1), (0, 2), 3) +} + +// Bottom/right padding only: 2x2 kernel, padding 1, start 1/1, 3x3 input. +fn asym_pad() -> Im2colCase { + let input = vec![(0..9).map(|p| row(p * 5 + 1)).collect()]; + let shape = Shape { + size: 3, kernel: 2, stride: 1, padding: 1, start_row: 1, start_col: 1, + input_base: 0, window_start: 0, window_count: 9, tile_rows: 16, + in_groups: 1, out_groups: 4, + }; + finish(shape, input, (0, 4), (0, 0), 4) +} + +// Windows 5..7 of a 1x1 im2col on a 4x4 input at input_base 2; lines 3..7 zero. +fn partial_window_range() -> Im2colCase { + let input = vec![(0..18).map(|p| row(p + 100)).collect()]; + let shape = Shape { + size: 4, kernel: 1, stride: 1, padding: 0, start_row: 0, start_col: 0, + input_base: 2, window_start: 5, window_count: 3, tile_rows: 8, + in_groups: 1, out_groups: 1, + }; + finish(shape, input, (1, 0), (2, 1), 5) +} + +// 5x5 kernel on a 5x5 input with tile_rows 1: output group kr is image row kr. +fn k5_tile_rows_1() -> Im2colCase { + let input = vec![(0..25).map(|p| row(p * 7 + 3)).collect()]; + let shape = Shape { + size: 5, kernel: 5, stride: 1, padding: 0, start_row: 0, start_col: 0, + input_base: 0, window_start: 0, window_count: 1, tile_rows: 1, + in_groups: 1, out_groups: 5, + }; + finish(shape, input, (0, 1), (0, 0), 6) +} + +// 7x7 stride-2 kernel on a 9x9 input: 2x2 windows, one kernel row per group. +fn k7_stride2() -> Im2colCase { + let input = vec![(0..81).map(|p| row(p * 11 + 5)).collect()]; + let shape = Shape { + size: 9, kernel: 7, stride: 2, padding: 0, start_row: 0, start_col: 0, + input_base: 0, window_start: 0, window_count: 4, tile_rows: 4, + in_groups: 1, out_groups: 7, + }; + finish(shape, input, (6, 2), (1, 0), 7) +} + +// 2x2 kernel on a 3x3 input with 32 channels at input_base 3, windows 1..2: +// eight output groups, one (tap, channel group) tile each. +fn window_input_base_two_groups() -> Im2colCase { + let input = vec![ + (0..12).map(|p| row(p * 3 + 9)).collect(), + (0..12).map(|p| row(p * 3 + 60)).collect(), + ]; + let shape = Shape { + size: 3, kernel: 2, stride: 1, padding: 0, start_row: 0, start_col: 0, + input_base: 3, window_start: 1, window_count: 2, tile_rows: 2, + in_groups: 2, out_groups: 8, + }; + finish(shape, input, (4, 7), (0, 2), 8) +} diff --git a/examples/prefab/balls/im2col/verify/casegen/src/lib.rs b/examples/prefab/balls/im2col/verify/casegen/src/lib.rs new file mode 100644 index 000000000..db4b3458d --- /dev/null +++ b/examples/prefab/balls/im2col/verify/casegen/src/lib.rs @@ -0,0 +1,90 @@ +mod casegen; +mod model; + +use std::cell::RefCell; + +use casegen::{Im2colCase, Im2colCmd}; + +thread_local! { + static CURRENT: RefCell> = const { RefCell::new(None) }; +} + +fn current_case R, R>(f: F) -> R { + CURRENT.with(|c| match c.borrow().as_ref() { + Some(case) => f(case), + None => panic!("im2col_case: no case loaded; call im2col_case_load first"), + }) +} + +fn lo(row: &[u8; 16]) -> u64 { + u64::from_le_bytes(row[..8].try_into().unwrap()) +} + +fn hi(row: &[u8; 16]) -> u64 { + u64::from_le_bytes(row[8..].try_into().unwrap()) +} + +#[no_mangle] +pub extern "C" fn im2col_case_load(index: u32) -> i32 { + let case = casegen::gen_case(index); + CURRENT.with(|c| *c.borrow_mut() = Some(case)); + 0 +} + +#[no_mangle] +pub extern "C" fn im2col_case_cmd(out_ptr: *mut Im2colCmd) { + if out_ptr.is_null() { + panic!("im2col_case_cmd: null out_ptr"); + } + current_case(|case| unsafe { *out_ptr = case.cmd }); +} + +#[no_mangle] +pub extern "C" fn im2col_case_src_group(i: u32) -> u32 { + current_case(|case| case.src[i as usize].0 as u32) +} + +#[no_mangle] +pub extern "C" fn im2col_case_src_addr(i: u32) -> u32 { + current_case(|case| case.src[i as usize].1 as u32) +} + +#[no_mangle] +pub extern "C" fn im2col_case_src_lo(i: u32) -> u64 { + current_case(|case| lo(&case.src[i as usize].2)) +} + +#[no_mangle] +pub extern "C" fn im2col_case_src_hi(i: u32) -> u64 { + current_case(|case| hi(&case.src[i as usize].2)) +} + +#[no_mangle] +pub extern "C" fn im2col_case_read_group(i: u32) -> u32 { + current_case(|case| case.reads[i as usize].0 as u32) +} + +#[no_mangle] +pub extern "C" fn im2col_case_read_addr(i: u32) -> u32 { + current_case(|case| case.reads[i as usize].1 as u32) +} + +#[no_mangle] +pub extern "C" fn im2col_case_dst_group(i: u32) -> u32 { + current_case(|case| case.dst[i as usize].0 as u32) +} + +#[no_mangle] +pub extern "C" fn im2col_case_dst_addr(i: u32) -> u32 { + current_case(|case| case.dst[i as usize].1 as u32) +} + +#[no_mangle] +pub extern "C" fn im2col_case_dst_lo(i: u32) -> u64 { + current_case(|case| lo(&case.dst[i as usize].2)) +} + +#[no_mangle] +pub extern "C" fn im2col_case_dst_hi(i: u32) -> u64 { + current_case(|case| hi(&case.dst[i as usize].2)) +} diff --git a/examples/prefab/balls/im2col/verify/casegen/src/model.rs b/examples/prefab/balls/im2col/verify/casegen/src/model.rs new file mode 100644 index 000000000..227a2b38a --- /dev/null +++ b/examples/prefab/balls/im2col/verify/casegen/src/model.rs @@ -0,0 +1,93 @@ +// Reference IM2COL model in the RTL's traversal order (see emu/src/im2col.rs +// and arch/README.md): K tile t outer, window w inner, one write per output +// line, one read per in-bounds tap of a window below window_count. + +pub type Row = [u8; 16]; + +pub fn bank_entries() -> usize { + let config = std::env::var("BB_VERIFY_CONFIG").expect("BB_VERIFY_CONFIG is required"); + let text = std::fs::read_to_string(config).expect("failed to read BB_VERIFY_CONFIG"); + text.lines() + .find_map(|line| line.strip_prefix("bank_entries=")) + .expect("bank_entries missing from BB_VERIFY_CONFIG") + .parse() + .expect("invalid bank_entries in BB_VERIFY_CONFIG") +} + +pub struct Shape { + pub size: usize, + pub kernel: usize, + pub stride: usize, + pub padding: usize, + pub start_row: usize, + pub start_col: usize, + pub input_base: usize, + pub window_start: usize, + pub window_count: usize, + pub tile_rows: usize, + pub in_groups: usize, + pub out_groups: usize, +} + +impl Shape { + pub fn rs2(&self) -> u64 { + self.kernel as u64 + | (self.stride as u64) << 4 + | (self.padding as u64) << 8 + | (self.start_col as u64) << 12 + | (self.start_row as u64) << 16 + | (self.input_base as u64) << 20 + | (self.window_start as u64) << 26 + | (self.window_count as u64) << 36 + | (self.tile_rows as u64) << 46 + } +} + +pub struct Trace { + // (input group, line) per read, in issue order. + pub reads: Vec<(usize, usize)>, + // (output group, line, data) per write, in issue order. + pub writes: Vec<(usize, usize, Row)>, +} + +// `input[g][line]` is input group g; lines below input_base are present too. +pub fn im2col(s: &Shape, input: &[Vec]) -> Trace { + let padded = s.size + 2 * s.padding; + let out_rows = (padded - s.kernel - s.start_row) / s.stride + 1; + let out_cols = (padded - s.kernel - s.start_col) / s.stride + 1; + assert!((padded - s.kernel - s.start_row) % s.stride == 0); + assert!((padded - s.kernel - s.start_col) % s.stride == 0); + assert!(s.window_start + s.window_count <= out_rows * out_cols); + assert!(s.input_base + s.size * s.size <= bank_entries()); + let k_tiles = s.kernel * s.kernel * s.in_groups; + assert!(k_tiles % s.out_groups == 0); + let group_tiles = k_tiles / s.out_groups; + let padded_windows = s.window_count.div_ceil(s.tile_rows) * s.tile_rows; + assert!(padded_windows * group_tiles <= bank_entries()); + + let mut trace = Trace { reads: Vec::new(), writes: Vec::new() }; + for tile in 0..k_tiles { + let g = tile % s.in_groups; + let kr = tile / s.in_groups / s.kernel; + let kc = tile / s.in_groups % s.kernel; + for w in 0..padded_windows { + let line = ((w / s.tile_rows) * group_tiles + tile % group_tiles) * s.tile_rows + + w % s.tile_rows; + let mut data = [0u8; 16]; + if w < s.window_count { + let window = s.window_start + w; + let y = (s.start_row + window / out_cols * s.stride + kr) as isize + - s.padding as isize; + let x = (s.start_col + window % out_cols * s.stride + kc) as isize + - s.padding as isize; + if y >= 0 && x >= 0 && (y as usize) < s.size && (x as usize) < s.size { + let source = s.input_base + y as usize * s.size + x as usize; + trace.reads.push((g, source)); + data = input[g][source]; + } + } + trace.writes.push((tile / group_tiles, line, data)); + } + } + trace +} diff --git a/examples/balls/im2col/verify/filelists/im2col_ball.f b/examples/prefab/balls/im2col/verify/filelists/im2col_ball.f similarity index 89% rename from examples/balls/im2col/verify/filelists/im2col_ball.f rename to examples/prefab/balls/im2col/verify/filelists/im2col_ball.f index 5cbccdd31..ce6f93759 100644 --- a/examples/balls/im2col/verify/filelists/im2col_ball.f +++ b/examples/prefab/balls/im2col/verify/filelists/im2col_ball.f @@ -1,3 +1,4 @@ ++incdir+@UVM@/src/core +incdir+@UVM@/src/ball +incdir+@UVM@/src/ball/agents/cmd +incdir+@UVM@/src/ball/agents/mem @@ -13,6 +14,7 @@ +incdir+src/tests +incdir+@RTL@ +@UVM@/src/core/package.sv @UVM@/src/ball/bb_blink_if.sv @UVM@/src/ball/bb_uvm_pkg.sv diff --git a/examples/prefab/balls/im2col/verify/src/common/im2col_defs.svh b/examples/prefab/balls/im2col/verify/src/common/im2col_defs.svh new file mode 100644 index 000000000..09b7678ab --- /dev/null +++ b/examples/prefab/balls/im2col/verify/src/common/im2col_defs.svh @@ -0,0 +1,35 @@ +typedef struct { + int unsigned iter; + int unsigned rs2_lo; + int unsigned rs2_hi; + int unsigned op1_bank; + int unsigned wr_bank; + int unsigned op1_group; + int unsigned wr_group; + int unsigned op1_col; + int unsigned wr_col; + int unsigned rob_id; + int unsigned num_src; + int unsigned num_reads; + int unsigned num_dst; +} im2col_cmd_dpi_t; + +import "DPI-C" function int im2col_case_load(input int unsigned index); +import "DPI-C" function void im2col_case_cmd(output im2col_cmd_dpi_t cmd); +import "DPI-C" function int unsigned im2col_case_src_group(input int unsigned i); +import "DPI-C" function int unsigned im2col_case_src_addr(input int unsigned i); +import "DPI-C" function longint unsigned im2col_case_src_lo(input int unsigned i); +import "DPI-C" function longint unsigned im2col_case_src_hi(input int unsigned i); +import "DPI-C" function int unsigned im2col_case_read_group(input int unsigned i); +import "DPI-C" function int unsigned im2col_case_read_addr(input int unsigned i); +import "DPI-C" function int unsigned im2col_case_dst_group(input int unsigned i); +import "DPI-C" function int unsigned im2col_case_dst_addr(input int unsigned i); +import "DPI-C" function longint unsigned im2col_case_dst_lo(input int unsigned i); +import "DPI-C" function longint unsigned im2col_case_dst_hi(input int unsigned i); + +`ifndef IM2COL_FUNCT7 +`error "IM2COL_FUNCT7 must be provided by the selected Thread ballISA" +`endif +localparam int IM2COL_THREAD_FUNCT7 = `IM2COL_FUNCT7; +localparam int IM2COL_NUM_CASES = 8; +localparam int IM2COL_TIMEOUT_CYCLES = 100000; diff --git a/examples/prefab/balls/im2col/verify/src/common/im2col_items.svh b/examples/prefab/balls/im2col/verify/src/common/im2col_items.svh new file mode 100644 index 000000000..50c410b70 --- /dev/null +++ b/examples/prefab/balls/im2col/verify/src/common/im2col_items.svh @@ -0,0 +1,71 @@ +class im2col_cmd_item extends bb_blink_cmd_item; + `uvm_object_utils(im2col_cmd_item) + + // Input rows to preload, the expected reads, and the expected writes, each + // as (group, line[, data]) in the order the Ball issues them. + int unsigned src_group[$]; + int unsigned src_addr[$]; + bit [127:0] src_data[$]; + int unsigned read_group[$]; + int unsigned read_addr[$]; + int unsigned dst_group[$]; + int unsigned dst_addr[$]; + bit [127:0] dst_data[$]; + + function new(string name = "im2col_cmd_item"); + super.new(name); + endfunction + + function void load_rust_case(int unsigned index); + im2col_cmd_dpi_t cmd; + + if (im2col_case_load(index) != 0) `uvm_fatal("CASE", $sformatf("im2col case %0d failed", index)) + im2col_case_cmd(cmd); + + // bb_im2col: banks and input size come from rs1, the window geometry from rs2. + rob_id = cmd.rob_id[BB_BLINK_ROB_ID_W-1:0]; + funct = IM2COL_THREAD_FUNCT7[6:0]; + rs2 = {cmd.rs2_hi, cmd.rs2_lo}; + iter = cmd.iter; + op1_bank = cmd.op1_bank; + op2_bank = '0; + wr_bank = cmd.wr_bank; + op1_group = cmd.op1_group; + op2_group = '0; + wr_group = cmd.wr_group; + op1_col = cmd.op1_col; + op2_col = '0; + wr_col = cmd.wr_col; + wr_read_bank = '0; + wr_arch_bank = cmd.wr_bank; + + for (int i = 0; i < cmd.num_src; i++) begin + src_group.push_back(im2col_case_src_group(i)); + src_addr.push_back(im2col_case_src_addr(i)); + src_data.push_back({im2col_case_src_hi(i), im2col_case_src_lo(i)}); + end + for (int i = 0; i < cmd.num_reads; i++) begin + read_group.push_back(im2col_case_read_group(i)); + read_addr.push_back(im2col_case_read_addr(i)); + end + for (int i = 0; i < cmd.num_dst; i++) begin + dst_group.push_back(im2col_case_dst_group(i)); + dst_addr.push_back(im2col_case_dst_addr(i)); + dst_data.push_back({im2col_case_dst_hi(i), im2col_case_dst_lo(i)}); + end + endfunction + + function void do_copy(uvm_object rhs); + im2col_cmd_item rhs_; + super.do_copy(rhs); + if (!$cast(rhs_, rhs)) `uvm_fatal("COPY", "rhs is not im2col_cmd_item") + src_group = rhs_.src_group; + src_addr = rhs_.src_addr; + src_data = rhs_.src_data; + read_group = rhs_.read_group; + read_addr = rhs_.read_addr; + dst_group = rhs_.dst_group; + dst_addr = rhs_.dst_addr; + dst_data = rhs_.dst_data; + endfunction +endclass diff --git a/examples/prefab/balls/im2col/verify/src/cov/im2col_cov.svh b/examples/prefab/balls/im2col/verify/src/cov/im2col_cov.svh new file mode 100644 index 000000000..f45b4c22f --- /dev/null +++ b/examples/prefab/balls/im2col/verify/src/cov/im2col_cov.svh @@ -0,0 +1,58 @@ +class im2col_cov extends uvm_component; + `uvm_component_utils(im2col_cov) + + uvm_analysis_imp_cmd #(bb_blink_cmd_item, im2col_cov) cmd_imp; + + covergroup cmd_cg; + coverpoint kernel {bins k1 = {1}; bins k2 = {2}; bins k3 = {3}; bins k5 = {5}; bins k7 = {7};} + coverpoint stride {bins s1 = {1}; bins s2 = {2};} + coverpoint padding {bins p0 = {0}; bins p1 = {1};} + coverpoint start {bins zero = {0}; bins shifted = {[1 : $]};} + coverpoint input_base {bins zero = {0}; bins offset = {[1 : $]};} + coverpoint window_start {bins zero = {0}; bins offset = {[1 : $]};} + coverpoint tile_rows { + bins t1 = {1}; bins t2 = {2}; bins t4 = {4}; bins t8 = {8}; bins t16 = {16}; + } + coverpoint in_groups {bins one = {1}; bins two = {2};} + coverpoint out_groups {bins one = {1}; bins many = {[2 : $]};} + endgroup + + int unsigned kernel; + int unsigned stride; + int unsigned padding; + int unsigned start; + int unsigned input_base; + int unsigned window_start; + int unsigned tile_rows; + int unsigned in_groups; + int unsigned out_groups; + int cmd_count; + + function new(string name, uvm_component parent); + super.new(name, parent); + cmd_imp = new("cmd_imp", this); + cmd_cg = new(); + endfunction + + function void write_cmd(bb_blink_cmd_item item); + kernel = int'(item.rs2[3:0]); + stride = int'(item.rs2[7:4]); + padding = int'(item.rs2[11:8]); + start = int'(item.rs2[19:12]); + input_base = int'(item.rs2[25:20]); + window_start = int'(item.rs2[35:26]); + tile_rows = int'(item.rs2[50:46]); + in_groups = int'(item.op1_col); + out_groups = int'(item.wr_col); + cmd_count++; + cmd_cg.sample(); + endfunction + + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (cmd_count == 0) `uvm_fatal("COV", "no cmd observed") + if (cmd_cg.get_coverage() != 100.0) + `uvm_fatal("COV", $sformatf("im2col_cov coverage incomplete: %0.2f%%", cmd_cg.get_coverage())) + endfunction +endclass diff --git a/examples/balls/im2col/verify/src/env/im2col_env.svh b/examples/prefab/balls/im2col/verify/src/env/im2col_env.svh similarity index 100% rename from examples/balls/im2col/verify/src/env/im2col_env.svh rename to examples/prefab/balls/im2col/verify/src/env/im2col_env.svh diff --git a/examples/prefab/balls/im2col/verify/src/env/im2col_scoreboard.svh b/examples/prefab/balls/im2col/verify/src/env/im2col_scoreboard.svh new file mode 100644 index 000000000..a12962560 --- /dev/null +++ b/examples/prefab/balls/im2col/verify/src/env/im2col_scoreboard.svh @@ -0,0 +1,155 @@ +class im2col_scoreboard extends uvm_scoreboard; + `uvm_component_utils(im2col_scoreboard) + + uvm_analysis_imp_stim #(bb_blink_cmd_item, im2col_scoreboard) stim_imp; + uvm_analysis_imp_cmd #(bb_blink_cmd_item, im2col_scoreboard) cmd_imp; + uvm_analysis_imp_read #(bb_blink_read_item, im2col_scoreboard) read_imp; + uvm_analysis_imp_write #(bb_blink_write_item, im2col_scoreboard) write_imp; + uvm_analysis_imp_resp #(bb_blink_resp_item, im2col_scoreboard) resp_imp; + + bb_blink_mem_model #(1, 1) mem_model; + + im2col_cmd_item stim_q[$]; + int unsigned cmd_count; + int unsigned read_count; + int unsigned write_count; + int unsigned resp_count; + + function new(string name, uvm_component parent); + super.new(name, parent); + stim_imp = new("stim_imp", this); + cmd_imp = new("cmd_imp", this); + read_imp = new("read_imp", this); + write_imp = new("write_imp", this); + resp_imp = new("resp_imp", this); + endfunction + + function void write_stim(bb_blink_cmd_item item); + im2col_cmd_item titem; + im2col_cmd_item clone; + + if (stim_q.size() != 0) `uvm_fatal("SCB", "single outstanding command supported") + if (!$cast(titem, item)) `uvm_fatal("SCB", "stim item is not im2col_cmd_item") + if (!$cast(clone, titem.clone())) `uvm_fatal("SCB", "failed to clone stimulus item") + stim_q.push_back(clone); + + if (mem_model == null) `uvm_fatal("SCB", "mem_model handle not set") + mem_model.clear_mem(); + foreach (clone.src_data[i]) + mem_model.load_word_g(int'(clone.op1_bank), int'(clone.op1_group) + clone.src_group[i], + clone.src_addr[i], clone.src_data[i]); + mem_model.arm(); + endfunction + + function void write_cmd(bb_blink_cmd_item item); + im2col_cmd_item stim; + stim = current_stim("CMD"); + check_cmd(item, stim); + cmd_count++; + endfunction + + function void check_cmd(bb_blink_cmd_item got, im2col_cmd_item exp); + // Compare as plain BLink commands; the comparer rejects differing types. + bb_blink_cmd_item want = bb_blink_cmd_item::type_id::create("want"); + want.copy(exp); + verification_pkg::evidence::checks++; + if (!got.compare(want)) + `uvm_fatal("CMD", $sformatf("command mismatch: got %s exp %s", got.sprint(), want.sprint())) + endfunction + + function void write_read(bb_blink_read_item item); + im2col_cmd_item stim; + int unsigned group; + stim = current_stim("READ"); + if (read_count >= stim.read_addr.size()) + `uvm_fatal("READ", $sformatf("extra read %0d", read_count)) + group = stim.op1_group + stim.read_group[read_count]; + if (item.bank_id !== stim.op1_bank) + `uvm_fatal("READ", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.op1_bank)) + if (item.rob_id !== stim.rob_id) + `uvm_fatal("READ", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + if (item.group_id !== group[BB_BLINK_GROUP_ID_W-1:0]) + `uvm_fatal("READ", $sformatf( + "read %0d group mismatch: got %0d exp %0d", read_count, item.group_id, group)) + if (item.addr !== stim.read_addr[read_count][BB_BLINK_BANK_ADDR_W-1:0]) + `uvm_fatal("READ", $sformatf( + "read %0d addr mismatch: got %0d exp %0d", + read_count, + item.addr, + stim.read_addr[read_count] + )) + read_count++; + endfunction + + function void write_write(bb_blink_write_item item); + im2col_cmd_item stim; + int unsigned group; + stim = current_stim("WRITE"); + if (write_count >= stim.dst_addr.size()) + `uvm_fatal("WRITE", $sformatf("extra write %0d", write_count)) + group = stim.wr_group + stim.dst_group[write_count]; + if (item.bank_id !== stim.wr_bank) + `uvm_fatal("WRITE", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.wr_bank)) + if (item.rob_id !== stim.rob_id) + `uvm_fatal("WRITE", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + if (item.group_id !== group[BB_BLINK_GROUP_ID_W-1:0]) + `uvm_fatal("WRITE", $sformatf( + "write %0d group mismatch: got %0d exp %0d", write_count, item.group_id, group)) + if (item.addr !== stim.dst_addr[write_count][BB_BLINK_BANK_ADDR_W-1:0]) + `uvm_fatal("WRITE", $sformatf( + "write %0d addr mismatch: got %0d exp %0d", + write_count, + item.addr, + stim.dst_addr[write_count] + )) + if (item.mask !== 16'hFFFF) + `uvm_fatal("WRITE", $sformatf("mask mismatch: got 0x%04h", item.mask)) + if (item.data !== stim.dst_data[write_count]) + `uvm_fatal("SCB", $sformatf( + "write %0d data mismatch: got 0x%032h exp 0x%032h", + write_count, + item.data, + stim.dst_data[write_count] + )) + write_count++; + endfunction + + function void write_resp(bb_blink_resp_item item); + im2col_cmd_item stim; + stim = current_stim("RESP"); + if (item.rob_id !== stim.rob_id) + `uvm_fatal("RESP", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + resp_count++; + endfunction + + function im2col_cmd_item current_stim(string tag); + if (stim_q.size() == 0) begin + `uvm_fatal("SCB", $sformatf("%s observed before stimulus", tag)) + return null; + end + return stim_q[0]; + endfunction + + function bit done(); + return stim_q.size() == 1 && cmd_count == 1 && read_count == stim_q[0].read_addr.size() && + write_count == stim_q[0].dst_addr.size() && resp_count == 1; + endfunction + + function void reset_counters(); + stim_q.delete(); + cmd_count = 0; + read_count = 0; + write_count = 0; + resp_count = 0; + endfunction + + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (!done()) begin + `uvm_fatal("SCB", $sformatf("incomplete: cmds=%0d reads=%0d writes=%0d resp=%0d", cmd_count, + read_count, write_count, resp_count)) + end + stim_q.delete(); + endfunction +endclass diff --git a/examples/balls/im2col/verify/src/pkg/im2col_pkg.sv b/examples/prefab/balls/im2col/verify/src/pkg/im2col_pkg.sv similarity index 100% rename from examples/balls/im2col/verify/src/pkg/im2col_pkg.sv rename to examples/prefab/balls/im2col/verify/src/pkg/im2col_pkg.sv diff --git a/examples/balls/im2col/verify/src/seq/im2col_sequences.svh b/examples/prefab/balls/im2col/verify/src/seq/im2col_sequences.svh similarity index 76% rename from examples/balls/im2col/verify/src/seq/im2col_sequences.svh rename to examples/prefab/balls/im2col/verify/src/seq/im2col_sequences.svh index 3f3064388..43768b92b 100644 --- a/examples/balls/im2col/verify/src/seq/im2col_sequences.svh +++ b/examples/prefab/balls/im2col/verify/src/seq/im2col_sequences.svh @@ -2,19 +2,16 @@ class im2col_basic_seq extends uvm_sequence #(bb_blink_cmd_item); `uvm_object_utils(im2col_basic_seq) int unsigned case_index; - int unsigned seed; - int unsigned bid; function new(string name = "im2col_basic_seq"); super.new(name); - seed = IM2COL_SEED; endfunction task body(); im2col_cmd_item item; item = im2col_cmd_item::type_id::create("item"); start_item(item); - item.load_rust_case(seed, case_index, bid); + item.load_rust_case(case_index); finish_item(item); endtask endclass diff --git a/examples/prefab/balls/im2col/verify/src/tb_top.sv b/examples/prefab/balls/im2col/verify/src/tb_top.sv new file mode 100644 index 000000000..f129380fd --- /dev/null +++ b/examples/prefab/balls/im2col/verify/src/tb_top.sv @@ -0,0 +1,80 @@ +module tb_top; + import uvm_pkg::*; + import im2col_pkg::*; + + bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); + + Im2colBall dut ( + .clock(intf.clock), + .reset(intf.reset), + .blink_status_running(intf.status_running), + .blink_cmdReq_ready(intf.cmd_req_ready), + .blink_cmdReq_valid(intf.cmd_req_valid), + .blink_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .blink_cmdReq_bits_funct(intf.cmd_req_bits_funct), + .blink_cmdReq_bits_rs2(intf.cmd_req_bits_rs2), + .blink_cmdReq_bits_iter(intf.cmd_req_bits_iter), + .blink_cmdReq_bits_op1_bank(intf.cmd_req_bits_op1_bank), + .blink_cmdReq_bits_op2_bank(intf.cmd_req_bits_op2_bank), + .blink_cmdReq_bits_wr_bank(intf.cmd_req_bits_wr_bank), + .blink_cmdReq_bits_op1_group(intf.cmd_req_bits_op1_group), + .blink_cmdReq_bits_op2_group(intf.cmd_req_bits_op2_group), + .blink_cmdReq_bits_wr_group(intf.cmd_req_bits_wr_group), + .blink_cmdReq_bits_op1_col(intf.cmd_req_bits_op1_col), + .blink_cmdReq_bits_op2_col(intf.cmd_req_bits_op2_col), + .blink_cmdReq_bits_wr_col(intf.cmd_req_bits_wr_col), + .blink_cmdReq_bits_wr_read_bank(intf.cmd_req_bits_wr_read_bank), + .blink_cmdReq_bits_wr_arch_bank(intf.cmd_req_bits_wr_arch_bank), + .blink_cmdResp_ready(intf.cmd_resp_ready), + .blink_cmdResp_valid(intf.cmd_resp_valid), + .blink_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .blink_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .blink_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .blink_bankRead_0_group_id(intf.bank_read_group_id[0]), + .blink_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .blink_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .blink_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .blink_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .blink_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .blink_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .blink_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .blink_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .blink_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .blink_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .blink_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .blink_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .blink_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .blink_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .blink_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]) + ); + + // Constant outputs removed from the port list by firtool. + assign intf.bank_write_req_mask[0][0] = dut.blink_bankWrite_0_io_req_bits_mask_0; + assign intf.bank_write_req_mask[0][1] = dut.blink_bankWrite_0_io_req_bits_mask_1; + assign intf.bank_write_req_mask[0][2] = dut.blink_bankWrite_0_io_req_bits_mask_2; + assign intf.bank_write_req_mask[0][3] = dut.blink_bankWrite_0_io_req_bits_mask_3; + assign intf.bank_write_req_mask[0][4] = dut.blink_bankWrite_0_io_req_bits_mask_4; + assign intf.bank_write_req_mask[0][5] = dut.blink_bankWrite_0_io_req_bits_mask_5; + assign intf.bank_write_req_mask[0][6] = dut.blink_bankWrite_0_io_req_bits_mask_6; + assign intf.bank_write_req_mask[0][7] = dut.blink_bankWrite_0_io_req_bits_mask_7; + assign intf.bank_write_req_mask[0][8] = dut.blink_bankWrite_0_io_req_bits_mask_8; + assign intf.bank_write_req_mask[0][9] = dut.blink_bankWrite_0_io_req_bits_mask_9; + assign intf.bank_write_req_mask[0][10] = dut.blink_bankWrite_0_io_req_bits_mask_10; + assign intf.bank_write_req_mask[0][11] = dut.blink_bankWrite_0_io_req_bits_mask_11; + assign intf.bank_write_req_mask[0][12] = dut.blink_bankWrite_0_io_req_bits_mask_12; + assign intf.bank_write_req_mask[0][13] = dut.blink_bankWrite_0_io_req_bits_mask_13; + assign intf.bank_write_req_mask[0][14] = dut.blink_bankWrite_0_io_req_bits_mask_14; + assign intf.bank_write_req_mask[0][15] = dut.blink_bankWrite_0_io_req_bits_mask_15; + assign intf.bank_write_resp_ready[0] = dut.blink_bankWrite_0_io_resp_ready; + + initial begin + intf.clock = 1'b0; + intf.reset = 1'b1; + forever #5 intf.clock = ~intf.clock; + end + + initial begin + uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); + run_test("im2col_ball_test"); + end +endmodule diff --git a/examples/prefab/balls/im2col/verify/src/tests/im2col_test.svh b/examples/prefab/balls/im2col/verify/src/tests/im2col_test.svh new file mode 100644 index 000000000..b138a33a2 --- /dev/null +++ b/examples/prefab/balls/im2col/verify/src/tests/im2col_test.svh @@ -0,0 +1,58 @@ +class im2col_ball_test extends uvm_test; + `uvm_component_utils(im2col_ball_test) + + typedef virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) vif_t; + vif_t vif; + im2col_env env; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(vif_t)::get(this, "", "vif", vif)) begin + `uvm_fatal("NOVIF", "bb_blink_if not found") + end + env = im2col_env::type_id::create("env", this); + endfunction + + task run_phase(uvm_phase phase); + phase.raise_objection(this); + for (int unsigned index = 0; index < IM2COL_NUM_CASES; index++) run_case(index); + phase.drop_objection(this); + endtask + + task run_case(int unsigned idx); + im2col_basic_seq seq; + int cycles; + + apply_reset(); + env.scb.reset_counters(); + + seq = im2col_basic_seq::type_id::create("seq"); + seq.case_index = idx; + seq.start(env.cmd_agent.seqr); + + cycles = 0; + while (!env.scb.done()) begin + @(posedge vif.clock); + cycles++; + if (cycles > IM2COL_TIMEOUT_CYCLES) begin + `uvm_fatal("TIMEOUT", + $sformatf("Im2colBall case %0d timeout: cmds=%0d reads=%0d writes=%0d resp=%0d", + idx, env.scb.cmd_count, env.scb.read_count, env.scb.write_count, + env.scb.resp_count)) + end + end + + `uvm_info("IM2COL", $sformatf("Im2colBall case %0d passed in %0d cycles", idx, cycles), UVM_LOW) + endtask + + task apply_reset(); + vif.reset = 1'b1; + repeat (5) @(posedge vif.clock); + vif.reset = 1'b0; + repeat (2) @(posedge vif.clock); + endtask +endclass diff --git a/examples/prefab/balls/im2col/workloads/ctests/CMakeLists.txt b/examples/prefab/balls/im2col/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..246b28cd1 --- /dev/null +++ b/examples/prefab/balls/im2col/workloads/ctests/CMakeLists.txt @@ -0,0 +1,8 @@ +add_buckyball_ctest(im2col_window_test.c) +add_buckyball_ctest(im2col_k1_test.c) +add_buckyball_ctest(im2col_k3_test.c) +add_buckyball_ctest(im2col_k3_pad_test.c) +add_buckyball_ctest(im2col_k3_asym_pad_test.c) +add_buckyball_ctest(im2col_k3_partial_test.c) +add_buckyball_ctest(im2col_k5_test.c) +add_buckyball_ctest(im2col_k7_test.c) diff --git a/examples/prefab/balls/im2col/workloads/ctests/im2col_k1_test.c b/examples/prefab/balls/im2col/workloads/ctests/im2col_k1_test.c new file mode 100644 index 000000000..95141abab --- /dev/null +++ b/examples/prefab/balls/im2col/workloads/ctests/im2col_k1_test.c @@ -0,0 +1,31 @@ +#include "buckyball.h" +#include +#include +#include +#include + +/* 1x1 kernel over a 2x2 NHWC input with 32 channels (two channel groups) and + * two output groups: output group g is K tile g, so the output, read back as + * a whole two-group bank, is the input itself. */ +static int8_t input[4 * 32] __attribute__((aligned(64))); +static int8_t actual[4 * 32] __attribute__((aligned(64))); + +int main(void) { + for (int i = 0; i < 4 * 32; ++i) + input[i] = (int8_t)(i * 3 - 100); + + bb_mem_alloc(0, 1, 2); + bb_mem_alloc(1, 1, 2); + bb_mvin((uintptr_t)input, 0, 4, 1); + bb_im2col(0, 1, 2, 1, 1, 0, 0, 0, 0, 0, 4, 4); + bb_mvout((uintptr_t)actual, 1, 4, 1); + + for (int i = 0; i < 4 * 32; ++i) + if (actual[i] != input[i]) { + printf("im2col_k1 FAIL index=%d expected=%d actual=%d\n", i, input[i], + actual[i]); + return 1; + } + printf("im2col_k1 PASS\n"); + return 0; +} diff --git a/examples/prefab/balls/im2col/workloads/ctests/im2col_k3_asym_pad_test.c b/examples/prefab/balls/im2col/workloads/ctests/im2col_k3_asym_pad_test.c new file mode 100644 index 000000000..693707eda --- /dev/null +++ b/examples/prefab/balls/im2col/workloads/ctests/im2col_k3_asym_pad_test.c @@ -0,0 +1,41 @@ +#include "buckyball.h" +#include +#include +#include +#include + +/* Bottom/right-only padding: 2x2 kernel, padding 1, start_row = start_col = 1 + * over a 3x3 input gives 3x3 windows reading pixel (oy + kr, ox + kc), zero + * past the bottom/right edge. Four output groups hold one tap each; lines + * 9..15 pad the 16-row M tile. */ +static int8_t input[9 * 16] __attribute__((aligned(64))); +static int8_t actual[16 * 16] __attribute__((aligned(64))); + +int main(void) { + for (int i = 0; i < 9 * 16; ++i) + input[i] = (int8_t)(i * 9 - 50); + + bb_mem_alloc(0, 1, 1); + bb_mem_alloc(1, 1, 4); + bb_mvin((uintptr_t)input, 0, 9, 1); + bb_im2col(0, 1, 3, 2, 1, 1, 0, 1, 1, 0, 9, 16); + + for (int tap = 0; tap < 4; ++tap) { + bb_mvout_group((uintptr_t)actual, 1, tap, 16, 1); + for (int w = 0; w < 16; ++w) { + int y = w / 3 + tap / 2, x = w % 3 + tap % 2; + int inside = w < 9 && y < 3 && x < 3; + for (int lane = 0; lane < 16; ++lane) { + int8_t expected = inside ? input[(y * 3 + x) * 16 + lane] : 0; + if (actual[w * 16 + lane] != expected) { + printf("im2col_k3_asym_pad FAIL tap=%d window=%d lane=%d " + "expected=%d actual=%d\n", + tap, w, lane, expected, actual[w * 16 + lane]); + return 1; + } + } + } + } + printf("im2col_k3_asym_pad PASS\n"); + return 0; +} diff --git a/examples/prefab/balls/im2col/workloads/ctests/im2col_k3_pad_test.c b/examples/prefab/balls/im2col/workloads/ctests/im2col_k3_pad_test.c new file mode 100644 index 000000000..c314d0692 --- /dev/null +++ b/examples/prefab/balls/im2col/workloads/ctests/im2col_k3_pad_test.c @@ -0,0 +1,40 @@ +#include "buckyball.h" +#include +#include +#include +#include + +/* 3x3 kernel with padding 1 over a 2x2 input gives 2x2 windows. One output + * group holds all nine taps in one 4-row M tile: tap t, window w is line + * t * 4 + w, zero when the tap falls into the padding. */ +static int8_t input[4 * 16] __attribute__((aligned(64))); +static int8_t actual[36 * 16] __attribute__((aligned(64))); + +int main(void) { + for (int i = 0; i < 4 * 16; ++i) + input[i] = (int8_t)(i * 7 + 3); + + bb_mem_alloc(0, 1, 1); + bb_mem_alloc(1, 1, 1); + bb_mvin((uintptr_t)input, 0, 4, 1); + bb_im2col(0, 1, 2, 3, 1, 1, 0, 0, 0, 0, 4, 4); + bb_mvout((uintptr_t)actual, 1, 36, 1); + + for (int tap = 0; tap < 9; ++tap) + for (int w = 0; w < 4; ++w) { + int y = w / 2 + tap / 3 - 1, x = w % 2 + tap % 3 - 1; + int inside = y >= 0 && y < 2 && x >= 0 && x < 2; + for (int lane = 0; lane < 16; ++lane) { + int8_t expected = inside ? input[(y * 2 + x) * 16 + lane] : 0; + int8_t got = actual[(tap * 4 + w) * 16 + lane]; + if (got != expected) { + printf("im2col_k3_pad FAIL tap=%d window=%d lane=%d expected=%d " + "actual=%d\n", + tap, w, lane, expected, got); + return 1; + } + } + } + printf("im2col_k3_pad PASS\n"); + return 0; +} diff --git a/examples/prefab/balls/im2col/workloads/ctests/im2col_k3_partial_test.c b/examples/prefab/balls/im2col/workloads/ctests/im2col_k3_partial_test.c new file mode 100644 index 000000000..4c27cb0cc --- /dev/null +++ b/examples/prefab/balls/im2col/workloads/ctests/im2col_k3_partial_test.c @@ -0,0 +1,36 @@ +#include "buckyball.h" +#include +#include +#include +#include + +/* Window range 5..7 of a 1x1 im2col over a 4x4 input: lines 0..2 are pixels + * 5, 6, 7 and lines 3..7 zero-fill the partial 8-row M tile. */ +static int8_t input[16 * 16] __attribute__((aligned(64))); +static int8_t actual[8 * 16] __attribute__((aligned(64))); + +int main(void) { + for (int i = 0; i < 16 * 16; ++i) + input[i] = (int8_t)(i * 11 + 7); + for (int i = 0; i < 8 * 16; ++i) + actual[i] = 99; + + bb_mem_alloc(0, 1, 1); + bb_mem_alloc(1, 1, 1); + bb_mvin((uintptr_t)input, 0, 16, 1); + bb_im2col(0, 1, 4, 1, 1, 0, 0, 0, 0, 5, 3, 8); + bb_mvout((uintptr_t)actual, 1, 8, 1); + + for (int line = 0; line < 8; ++line) + for (int lane = 0; lane < 16; ++lane) { + int8_t expected = line < 3 ? input[(5 + line) * 16 + lane] : 0; + if (actual[line * 16 + lane] != expected) { + printf("im2col_k3_partial FAIL line=%d lane=%d expected=%d " + "actual=%d\n", + line, lane, expected, actual[line * 16 + lane]); + return 1; + } + } + printf("im2col_k3_partial PASS\n"); + return 0; +} diff --git a/examples/prefab/balls/im2col/workloads/ctests/im2col_k3_test.c b/examples/prefab/balls/im2col/workloads/ctests/im2col_k3_test.c new file mode 100644 index 000000000..a98641cf0 --- /dev/null +++ b/examples/prefab/balls/im2col/workloads/ctests/im2col_k3_test.c @@ -0,0 +1,39 @@ +#include "buckyball.h" +#include +#include +#include +#include + +/* 3x3 kernel over a 4x4 input (16 channels) gives 2x2 windows. Nine output + * groups hold one tap each: group kr * 3 + kc, line w is the input row of + * pixel (oy + kr, ox + kc); lines 4..7 pad the 8-row M tile with zeros. */ +static int8_t input[16 * 16] __attribute__((aligned(64))); +static int8_t actual[8 * 16] __attribute__((aligned(64))); + +int main(void) { + for (int i = 0; i < 16 * 16; ++i) + input[i] = (int8_t)(i * 5 + 1); + + bb_mem_alloc(0, 1, 1); + bb_mem_alloc(1, 1, 9); + bb_mvin((uintptr_t)input, 0, 16, 1); + bb_im2col(0, 1, 4, 3, 1, 0, 0, 0, 0, 0, 4, 8); + + for (int tap = 0; tap < 9; ++tap) { + bb_mvout_group((uintptr_t)actual, 1, tap, 8, 1); + for (int w = 0; w < 8; ++w) { + int pixel = (w / 2 + tap / 3) * 4 + w % 2 + tap % 3; + for (int lane = 0; lane < 16; ++lane) { + int8_t expected = w < 4 ? input[pixel * 16 + lane] : 0; + if (actual[w * 16 + lane] != expected) { + printf("im2col_k3 FAIL tap=%d window=%d lane=%d expected=%d " + "actual=%d\n", + tap, w, lane, expected, actual[w * 16 + lane]); + return 1; + } + } + } + } + printf("im2col_k3 PASS\n"); + return 0; +} diff --git a/examples/prefab/balls/im2col/workloads/ctests/im2col_k5_test.c b/examples/prefab/balls/im2col/workloads/ctests/im2col_k5_test.c new file mode 100644 index 000000000..6f648f429 --- /dev/null +++ b/examples/prefab/balls/im2col/workloads/ctests/im2col_k5_test.c @@ -0,0 +1,33 @@ +#include "buckyball.h" +#include +#include +#include +#include + +/* 5x5 kernel over a 5x5 input: one window, 25 taps. With tile_rows 1 and five + * output groups, group kr line kc is the input row of pixel (kr, kc), so each + * output group is one input image row. */ +static int8_t input[25 * 16] __attribute__((aligned(64))); +static int8_t actual[5 * 16] __attribute__((aligned(64))); + +int main(void) { + for (int i = 0; i < 25 * 16; ++i) + input[i] = (int8_t)(i * 13 - 7); + + bb_mem_alloc(0, 1, 1); + bb_mem_alloc(1, 1, 5); + bb_mvin((uintptr_t)input, 0, 25, 1); + bb_im2col(0, 1, 5, 5, 1, 0, 0, 0, 0, 0, 1, 1); + + for (int kr = 0; kr < 5; ++kr) { + bb_mvout_group((uintptr_t)actual, 1, kr, 5, 1); + for (int i = 0; i < 5 * 16; ++i) + if (actual[i] != input[kr * 5 * 16 + i]) { + printf("im2col_k5 FAIL row=%d index=%d expected=%d actual=%d\n", kr, i, + input[kr * 5 * 16 + i], actual[i]); + return 1; + } + } + printf("im2col_k5 PASS\n"); + return 0; +} diff --git a/examples/prefab/balls/im2col/workloads/ctests/im2col_k7_test.c b/examples/prefab/balls/im2col/workloads/ctests/im2col_k7_test.c new file mode 100644 index 000000000..d16558b8b --- /dev/null +++ b/examples/prefab/balls/im2col/workloads/ctests/im2col_k7_test.c @@ -0,0 +1,41 @@ +#include "buckyball.h" +#include +#include +#include +#include + +/* 7x7 stride-2 kernel over a 9x9 input gives 2x2 windows. Seven output groups + * hold one kernel row each (seven taps): group kr, tap kc, window w is line + * kc * 4 + w, the input row of pixel (oy * 2 + kr, ox * 2 + kc). */ +static int8_t input[81 * 16] __attribute__((aligned(64))); +static int8_t actual[28 * 16] __attribute__((aligned(64))); + +int main(void) { + for (int i = 0; i < 81 * 16; ++i) + input[i] = (int8_t)(i * 3 + 11); + + bb_mem_alloc(0, 1, 1); + bb_mem_alloc(1, 1, 7); + bb_mvin((uintptr_t)input, 0, 81, 1); + bb_im2col(0, 1, 9, 7, 2, 0, 0, 0, 0, 0, 4, 4); + + for (int kr = 0; kr < 7; ++kr) { + bb_mvout_group((uintptr_t)actual, 1, kr, 28, 1); + for (int kc = 0; kc < 7; ++kc) + for (int w = 0; w < 4; ++w) { + int pixel = (w / 2 * 2 + kr) * 9 + w % 2 * 2 + kc; + for (int lane = 0; lane < 16; ++lane) { + int8_t expected = input[pixel * 16 + lane]; + int8_t got = actual[(kc * 4 + w) * 16 + lane]; + if (got != expected) { + printf("im2col_k7 FAIL kr=%d kc=%d window=%d lane=%d expected=%d " + "actual=%d\n", + kr, kc, w, lane, expected, got); + return 1; + } + } + } + } + printf("im2col_k7 PASS\n"); + return 0; +} diff --git a/examples/prefab/balls/im2col/workloads/ctests/im2col_window_test.c b/examples/prefab/balls/im2col/workloads/ctests/im2col_window_test.c new file mode 100644 index 000000000..ea87664e5 --- /dev/null +++ b/examples/prefab/balls/im2col/workloads/ctests/im2col_window_test.c @@ -0,0 +1,44 @@ +#include "buckyball.h" +#include +#include +#include +#include + +/* 2x2 kernel over a 3x3 NHWC input with 32 channels (two channel groups) at + * input_base 3, windows 1..2 of the 2x2 output. K tile t = tap * 2 + g; each + * of the eight output groups holds one tile: line w is channels 16g..16g+15 + * of pixel (oy + kr, ox + kc) for window 1 + w. */ +static int8_t input[(3 + 9) * 32] __attribute__((aligned(64))); +static int8_t actual[2 * 16] __attribute__((aligned(64))); + +int main(void) { + for (int i = 0; i < 3 * 32; ++i) + input[i] = 0x5a; + for (int i = 3 * 32; i < 12 * 32; ++i) + input[i] = (int8_t)(i * 7 - 9); + + bb_mem_alloc(0, 1, 2); + bb_mem_alloc(1, 1, 8); + bb_mvin((uintptr_t)input, 0, 12, 1); + bb_im2col(0, 1, 3, 2, 1, 0, 3, 0, 0, 1, 2, 2); + + for (int tile = 0; tile < 8; ++tile) { + int tap = tile / 2, group = tile % 2; + bb_mvout_group((uintptr_t)actual, 1, tile, 2, 1); + for (int w = 0; w < 2; ++w) { + int window = 1 + w; + int pixel = (window / 2 + tap / 2) * 3 + window % 2 + tap % 2; + for (int lane = 0; lane < 16; ++lane) { + int8_t expected = input[(3 + pixel) * 32 + group * 16 + lane]; + if (actual[w * 16 + lane] != expected) { + printf("im2col_window FAIL tile=%d window=%d lane=%d expected=%d " + "actual=%d\n", + tile, w, lane, expected, actual[w * 16 + lane]); + return 1; + } + } + } + } + printf("im2col_window PASS\n"); + return 0; +} diff --git a/examples/prefab/balls/im2col/workloads/isa/im2col.h b/examples/prefab/balls/im2col/workloads/isa/im2col.h new file mode 100644 index 000000000..a6d056295 --- /dev/null +++ b/examples/prefab/balls/im2col/workloads/isa/im2col.h @@ -0,0 +1,29 @@ +#ifndef _BB_IM2COL_H_ +#define _BB_IM2COL_H_ + +#include +#include + +// Input virtual bank: group g is channel block g (channels 16g..16g+15); +// line input_base + y * input_size + x holds pixel (y, x) of that block. +// K is ordered (kernel row, kernel column, channel), so K tile +// t = (kr * kernel + kc) * input_groups + g is one input row (zero when the +// tap falls into padding). With K_TILES = kernel * kernel * input_groups, the +// output groups must divide K_TILES; output group j holds K tiles +// j * G .. j * G + G - 1 (G = K_TILES / output_groups) in the SMatMul A +// layout: window w, local tile kt -> line ((w / tile_rows) * G + kt) * +// tile_rows + w % tile_rows. Windows window_count .. up to the next multiple +// of tile_rows are zero. Window w is output pixel window_start + w in +// row-major order. +#define bb_im2col(input_bank, output_bank, input_size, kernel, stride, \ + padding, input_base, start_row, start_col, window_start, \ + window_count, tile_rows) \ + BUCKYBALL_INSTRUCTION_R_R( \ + BB_BANK0(input_bank) | BB_BANK2(output_bank) | BB_ITER(input_size), \ + FIELD(kernel, 0, 3) | FIELD(stride, 4, 7) | FIELD(padding, 8, 11) | \ + FIELD(start_col, 12, 15) | FIELD(start_row, 16, 19) | \ + FIELD(input_base, 20, 25) | FIELD(window_start, 26, 35) | \ + FIELD(window_count, 36, 45) | FIELD(tile_rows, 46, 50), \ + FUNCT7(im2col, IM2COL)) + +#endif // _BB_IM2COL_H_ diff --git a/examples/balls/im2col/workloads/mlir_tests/CMakeLists.txt b/examples/prefab/balls/im2col/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/balls/im2col/workloads/mlir_tests/CMakeLists.txt rename to examples/prefab/balls/im2col/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/balls/im2col/workloads/mlir_tests/ball/CMakeLists.txt b/examples/prefab/balls/im2col/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/balls/im2col/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/prefab/balls/im2col/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/prefab/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3.mlir b/examples/prefab/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3.mlir new file mode 100644 index 000000000..30f812b04 --- /dev/null +++ b/examples/prefab/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3.mlir @@ -0,0 +1,42 @@ +// Physical-bank im2col_i8: one group of sixteen channels, 6x6, k3, 16 windows +// in M tiles of eight rows. + +func.func private @check_result(memref<144x16xi8>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c6 = arith.constant 6 : index + %c16 = arith.constant 16 : index + + %input = memref.alloc() alignment = 64 : memref<36x16xi8> + %output = memref.alloc() alignment = 64 : memref<144x16xi8> + linalg.fill ins(%zero_i8 : i8) outs(%output : memref<144x16xi8>) + + scf.for %r = %c0 to %c6 step %c1 { + scf.for %col = %c0 to %c6 step %c1 { + scf.for %ch = %c0 to %c16 step %c1 { + %tmp = arith.muli %r, %c6 : index + %pixel = arith.addi %tmp, %col : index + %sum = arith.addi %r, %col : index + %value = arith.addi %sum, %ch : index + %v8 = arith.index_cast %value : index to i8 + memref.store %v8, %input[%pixel, %ch] : memref<36x16xi8> + } + } + } + + buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %input <{bank = 1 : i64, depth = 36 : i64, stride = 1 : i64}> : memref<36x16xi8> + im2col_ball.im2col_i8 <{input = 1 : i64, output = 2 : i64, input_side = 6 : i64, kernel = 3 : i64, stride = 1 : i64, padding = 0 : i64, tile_rows = 8 : i64}> + buckyball.mvout %output <{bank = 2 : i64, depth = 144 : i64, stride = 1 : i64}> : memref<144x16xi8> + buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + + func.call @check_result(%output) : (memref<144x16xi8>) -> () + memref.dealloc %input : memref<36x16xi8> + memref.dealloc %output : memref<144x16xi8> + return %zero_i8 : i8 +} diff --git a/examples/prefab/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3_main.cpp b/examples/prefab/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3_main.cpp new file mode 100644 index 000000000..67f7d416a --- /dev/null +++ b/examples/prefab/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3_main.cpp @@ -0,0 +1,41 @@ +#include +#include +#include + +// 6x6 input, k3, stride 1: 16 windows; nine K tiles (one per kernel +// position, all sixteen channels) in M tiles of eight rows. +enum { SIDE = 6, K = 3, OUT = 4, TILE_ROWS = 8, TILES = 9, LINES = 144 }; + +static void fail(void) { +#ifdef BAREMETAL + *(volatile uint32_t *)0x60000000 = 1; + while (1) { + } +#else + exit(1); +#endif +} + +extern "C" void check_result(int8_t *allocated, int8_t *aligned, int64_t offset, + int64_t rows, int64_t lanes, int64_t row_stride, + int64_t lane_stride) { + (void)allocated; + if (rows != LINES || lanes != 16 || row_stride != 16 || lane_stride != 1) + fail(); + int8_t *out = aligned + offset; + for (int w = 0; w < OUT * OUT; ++w) + for (int t = 0; t < TILES; ++t) + for (int ch = 0; ch < 16; ++ch) { + int y = w / OUT + t / K, x = w % OUT + t % K; + int line = ((w / TILE_ROWS) * TILES + t) * TILE_ROWS + w % TILE_ROWS; + int8_t expected = (int8_t)(y + x + ch); + int8_t actual = out[line * row_stride + ch]; + if (actual != expected) { + printf("FAILED: im2col 6x6 k3 w=%d t=%d ch=%d expected=%d " + "actual=%d\n", + w, t, ch, expected, actual); + fail(); + } + } + printf("PASSED: im2col 6x6 k3\n"); +} diff --git a/examples/prefab/balls/im2col/workloads/mlir_tests/bank/CMakeLists.txt b/examples/prefab/balls/im2col/workloads/mlir_tests/bank/CMakeLists.txt new file mode 100644 index 000000000..0cfd943de --- /dev/null +++ b/examples/prefab/balls/im2col/workloads/mlir_tests/bank/CMakeLists.txt @@ -0,0 +1,6 @@ +if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) + set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) +endif() +set(BUCKYBALL_MLIR_TEST_PREFIX bank) + +add_buckyball_mlir_test(im2col_k3_two_blocks TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/prefab/balls/im2col/workloads/mlir_tests/bank/im2col_k3_two_blocks.mlir b/examples/prefab/balls/im2col/workloads/mlir_tests/bank/im2col_k3_two_blocks.mlir new file mode 100644 index 000000000..f568240c2 --- /dev/null +++ b/examples/prefab/balls/im2col/workloads/mlir_tests/bank/im2col_k3_two_blocks.mlir @@ -0,0 +1,38 @@ +// Virtual-bank im2col_i8: an 8x8 map of 32 channels (two groups), k3 with +// padding 1, 64 windows spread over three output groups of six K tiles. + +func.func private @check_result(memref<384x48xi8>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c3 = arith.constant 3 : index + %c8 = arith.constant 8 : index + %c32 = arith.constant 32 : index + %input = memref.alloc() alignment = 64 : memref<64x32xi8> + %output = memref.alloc() alignment = 64 : memref<384x48xi8> + scf.for %y = %c0 to %c8 step %c1 { + scf.for %x = %c0 to %c8 step %c1 { + scf.for %ch = %c0 to %c32 step %c1 { + %row = arith.muli %y, %c8 : index + %pixel = arith.addi %row, %x : index + %scaled = arith.muli %ch, %c3 : index + %value = arith.addi %pixel, %scaled : index + %v8 = arith.index_cast %value : index to i8 + memref.store %v8, %input[%pixel, %ch] : memref<64x32xi8> + } + } + } + %input_bank = buckyball.bank_alloc <{row = 1 : i64, col = 2 : i64}> + %output_bank = buckyball.bank_alloc <{row = 1 : i64, col = 3 : i64}> + %loaded = buckyball.bank_mvin %input, %input_bank <{depth = 64 : i64, stride = 1 : i64}> : memref<64x32xi8> + %matrix = im2col_ball.bank_im2col_i8 %loaded -> %output_bank <{input_side = 8 : i64, kernel = 3 : i64, stride = 1 : i64, padding = 1 : i64, tile_rows = 8 : i64}> + %stored = buckyball.bank_mvout %output, %matrix <{depth = 384 : i64, stride = 1 : i64}> : memref<384x48xi8> + buckyball.bank_release %loaded + buckyball.bank_release %stored + func.call @check_result(%output) : (memref<384x48xi8>) -> () + memref.dealloc %input : memref<64x32xi8> + memref.dealloc %output : memref<384x48xi8> + return %zero_i8 : i8 +} diff --git a/examples/prefab/balls/im2col/workloads/mlir_tests/bank/im2col_k3_two_blocks_main.cpp b/examples/prefab/balls/im2col/workloads/mlir_tests/bank/im2col_k3_two_blocks_main.cpp new file mode 100644 index 000000000..896ce5133 --- /dev/null +++ b/examples/prefab/balls/im2col/workloads/mlir_tests/bank/im2col_k3_two_blocks_main.cpp @@ -0,0 +1,29 @@ +#include +#include +#include + +// 8x8 map, two channel blocks, k3 padding 1: K tile t = (ky * 3 + kx) * 2 + +// block; output group j holds tiles 6j..6j+5, line ((w / 8) * 6 + t % 6) * 8 +// + w % 8. A whole-bank DMA puts the three groups side by side in a row. +extern "C" void check_result(int8_t *, int8_t *data, int64_t offset, + int64_t rows, int64_t lanes, int64_t stride, + int64_t lane_stride) { + if (rows != 384 || lanes != 48 || stride != 48 || lane_stride != 1) + exit(1); + for (int w = 0; w < 64; ++w) + for (int t = 0; t < 18; ++t) + for (int ch = 0; ch < 16; ++ch) { + int position = t / 2, block = t % 2; + int y = w / 8 + position / 3 - 1, x = w % 8 + position % 3 - 1; + int expected = 0; + if (y >= 0 && y < 8 && x >= 0 && x < 8) + expected = y * 8 + x + (block * 16 + ch) * 3; + int line = ((w / 8) * 6 + t % 6) * 8 + w % 8; + int lane = (t / 6) * 16 + ch; + if (data[offset + line * stride + lane] != (int8_t)expected) { + printf("im2col two blocks mismatch w=%d t=%d ch=%d\n", w, t, ch); + exit(1); + } + } + printf("im2col k3 two blocks PASS\n"); +} diff --git a/examples/balls/int2fp/arch/src/main/scala/Fp32Mul.scala b/examples/prefab/balls/int2fp/arch/Fp32Mul.scala similarity index 89% rename from examples/balls/int2fp/arch/src/main/scala/Fp32Mul.scala rename to examples/prefab/balls/int2fp/arch/Fp32Mul.scala index 94a899a4f..75467ff16 100644 --- a/examples/balls/int2fp/arch/src/main/scala/Fp32Mul.scala +++ b/examples/prefab/balls/int2fp/arch/Fp32Mul.scala @@ -1,11 +1,22 @@ package examples.balls.int2fp +import chisel3.experimental.hierarchy.public + +import chisel3.experimental.hierarchy.instantiable + +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import examples.balls.int2fp.configs.Int2FpBallParam + import chisel3._ import chisel3.util._ -class ShiftAddMul(w: Int) extends Module { +@instantiable +class ShiftAddMul(val w: Int, val ballParam: Int2FpBallParam) extends Module { + require(w >= 1, "ShiftAddMul width must be >= 1") + @public val io = IO(new Bundle { val start = Input(Bool()) val a = Input(UInt(w.W)) @@ -74,8 +85,10 @@ class ShiftAddMul(w: Int) extends Module { } } -class Fp32Mul extends Module { +@instantiable +class Fp32Mul(val ballParam: Int2FpBallParam) extends Module { + @public val io = IO(new Bundle { val start = Input(Bool()) val a = Input(UInt(32.W)) @@ -84,7 +97,9 @@ class Fp32Mul extends Module { val done = Output(Bool()) }) - val umul = Module(new ShiftAddMul(24)) + val umul = + Instantiate(new ShiftAddMul(24, ballParam)) + val aReg = Reg(UInt(32.W)) val bReg = Reg(UInt(32.W)) val result = Reg(UInt(32.W)) diff --git a/examples/prefab/balls/int2fp/arch/Int2Fp.scala b/examples/prefab/balls/int2fp/arch/Int2Fp.scala new file mode 100644 index 000000000..c91bc41ad --- /dev/null +++ b/examples/prefab/balls/int2fp/arch/Int2Fp.scala @@ -0,0 +1,250 @@ +package examples.balls.int2fp + +import framework.acc.balldomain.configs.BallRouteParam + +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import examples.balls.int2fp.configs.Int2FpBallParam + +import chisel3._ +import framework.acc.balldomain.ball.blink.{BallCommand, BallResponse} +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} + +import framework.acc.balldomain.ball.blink.BallStatus +import framework.acc.memdomain.{BankRead, BankWrite} + +@instantiable +class Int2Fp( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val routingParam: BallDomainParam, + val instance: BallRouteParam, + val ballParam: Int2FpBallParam) + extends Module { + val commandDataBits = frontendParam.commandDataBits + + val mapping = instance + + require(memParam.bankWidth == 128, "Int2FpBall requires 128-bit bank rows") + require(mapping.inBW == 2, "Int2FpBall requires inBW=2") + require(mapping.outBW == 1, "Int2FpBall requires outBW=1") + + @public + val io = IO(new Bundle { + + val cmdReq = Flipped(Decoupled(new BallCommand(frontendParam, memParam))) + + val cmdResp = Decoupled(new BallResponse(frontendParam)) + + val bankRead = Vec( + mapping.inBW, + Flipped(new BankRead( + frontendParam, + memParam + )) + ) + + val bankWrite = Vec( + mapping.outBW, + Flipped(new BankWrite( + frontendParam, + memParam + )) + ) + + val status = new BallStatus + }) + + val idle :: readReq :: readResp :: convert :: multiply :: writeReq :: writeResp :: complete :: Nil = + Enum(8) + val state = RegInit(idle) + + val robIdReg = RegInit(0.U(log2Up(frontendParam.rob_entries).W)) + val inputBank = RegInit(0.U(frontendParam.bank_id_len.W)) + val scaleBank = RegInit(0.U(frontendParam.bank_id_len.W)) + val outputBank = RegInit(0.U(frontendParam.bank_id_len.W)) + val bases = Reg(Vec(3, UInt(memParam.groupIdWidth.W))) + // Every group of a virtual bank is one 16-channel block with the same rows. + val groups = RegInit(0.U(log2Up(memParam.bankNum + 1).W)) + val group = RegInit(0.U(log2Up(memParam.bankNum + 1).W)) + val iterReg = RegInit(0.U(frontendParam.iter_len.W)) + val rowReg = RegInit(0.U(log2Ceil(memParam.bankEntries).W)) + val reluReg = RegInit(false.B) + val inputData = Reg(UInt(128.W)) + val scaleData = Reg(UInt(128.W)) + val readSent = RegInit(VecInit(Seq.fill(mapping.inBW)(false.B))) + val readReceived = RegInit(VecInit(Seq.fill(mapping.inBW)(false.B))) + val fpOutputs = Reg(Vec(4, UInt(32.W))) + val mulStarted = RegInit(false.B) + val laneReg = RegInit(0.U(2.W)) + val writeData = Reg(UInt(128.W)) + val multiplier = Instantiate(new Fp32Mul(ballParam)) + + def int32ToFp32(value: UInt): UInt = { + val sign = value(31) + val abs = Mux(sign, (~value).asUInt + 1.U, value) + val zero = abs === 0.U + val leading = 31.U - PriorityEncoder(Reverse(abs)) + val shift = Mux(leading > 23.U, leading - 23.U, 0.U) + val truncated = abs >> shift + val half = Mux(shift === 0.U, 0.U(32.W), 1.U(32.W) << (shift - 1.U)) + val remainder = Mux(shift === 0.U, 0.U(32.W), abs & ((1.U(32.W) << shift) - 1.U)) + val roundUp = remainder > half || (remainder === half && truncated(0)) + val rounded = truncated +& roundUp.asUInt + val carry = rounded(24) + val significand = Mux(leading > 23.U, rounded(22, 0), (abs << (23.U - leading))(22, 0)) + val exponent = (leading +& 127.U +& carry)(7, 0) + Mux(zero, 0.U, Cat(sign, exponent, Mux(carry, 0.U(23.W), significand))) + } + + def positiveFinite(value: UInt): Bool = + !value(31) && value(30, 23) =/= 255.U && value(30, 0) =/= 0.U + + for (i <- 0 until mapping.inBW) { + io.bankRead(i).rob_id := robIdReg + io.bankRead(i).bank_id := Mux(i.U === 0.U, inputBank, scaleBank) + io.bankRead(i).group_id := bases(i) + group + io.bankRead(i).io.req.valid := false.B + io.bankRead(i).io.req.bits.addr := Mux(i.U === 0.U, rowReg, rowReg(1, 0)) + io.bankRead(i).io.resp.ready := false.B + } + io.bankWrite(0).rob_id := robIdReg + io.bankWrite(0).bank_id := outputBank + io.bankWrite(0).group_id := bases(2) + group + io.bankWrite(0).io.req.valid := false.B + io.bankWrite(0).io.req.bits.addr := rowReg + io.bankWrite(0).io.req.bits.data := writeData + io.bankWrite(0).io.req.bits.mask := VecInit( + Seq.fill(memParam.bankMaskLen)(true.B) + ) + io.bankWrite(0).io.resp.ready := false.B + + val inputValue = (inputData >> (laneReg << 5))(31, 0) + val scaleValue = (scaleData >> (laneReg << 5))(31, 0) + multiplier.io.start := false.B + multiplier.io.a := int32ToFp32(Mux(reluReg && inputValue(31), 0.U, inputValue)) + multiplier.io.b := scaleValue + + io.cmdReq.ready := state === idle + io.cmdResp.valid := state === complete + io.cmdResp.bits.rob_id := robIdReg + io.status.idle := state === idle + io.status.running := state =/= idle && state =/= complete + + switch(state) { + is(idle) { + when(io.cmdReq.fire) { + val cmd = io.cmdReq.bits + + assert(cmd.iter > 0.U && cmd.iter(1, 0) === 0.U, "INT32_TO_FP32 iter must be a positive multiple of four") + assert( + cmd.iter <= memParam.bankEntries.U, + "INT32_TO_FP32 input exceeds bank depth" + ) + assert( + cmd.op1_col =/= 0.U && cmd.op1_col === cmd.op2_col && cmd.op1_col === cmd.wr_col && + cmd.op1_col <= memParam.bankNum.U, + "INT32_TO_FP32 operands must have the same channel groups" + ) + assert( + cmd.op1_bank =/= cmd.op2_bank && cmd.op1_bank =/= cmd.wr_bank && cmd.op2_bank =/= cmd.wr_bank, + "INT32_TO_FP32 banks must be distinct" + ) + assert(cmd.rs2(63, 1) === 0.U, "INT32_TO_FP32 reserves rs2[63:1]") + + robIdReg := io.cmdReq.bits.rob_id + inputBank := cmd.op1_bank + scaleBank := cmd.op2_bank + outputBank := cmd.wr_bank + bases := VecInit(cmd.op1_group, cmd.op2_group, cmd.wr_group) + groups := cmd.op1_col + group := 0.U + iterReg := cmd.iter + rowReg := 0.U + reluReg := cmd.rs2(0) + readSent.foreach(_ := false.B) + readReceived.foreach(_ := false.B) + state := readReq + } + } + is(readReq) { + for (i <- 0 until mapping.inBW) { + io.bankRead(i).io.req.valid := !readSent(i) + when(io.bankRead(i).io.req.fire)(readSent(i) := true.B) + } + when(VecInit(io.bankRead.zipWithIndex.map { case (port, i) => readSent(i) || port.io.req.fire }).asUInt.andR) { + state := readResp + } + } + is(readResp) { + for (i <- 0 until mapping.inBW) { + io.bankRead(i).io.resp.ready := !readReceived(i) + when(io.bankRead(i).io.resp.fire)(readReceived(i) := true.B) + } + when(io.bankRead(0).io.resp.fire)(inputData := io.bankRead(0).io.resp.bits.data) + when(io.bankRead(1).io.resp.fire)(scaleData := io.bankRead(1).io.resp.bits.data) + when(VecInit(io.bankRead.zipWithIndex.map { case (port, i) => readReceived(i) || port.io.resp.fire }).asUInt.andR) { + state := convert + } + } + is(convert) { + for (i <- 0 until 4) { + val scale = scaleData(32 * i + 31, 32 * i) + assert(positiveFinite(scale), "INT32_TO_FP32 scales must be finite and positive") + } + mulStarted := false.B + laneReg := 0.U + state := multiply + } + is(multiply) { + when(!mulStarted) { + multiplier.io.start := true.B + mulStarted := true.B + } + when(multiplier.io.done) { + fpOutputs(laneReg) := multiplier.io.result + when(laneReg === 3.U) { + writeData := Cat(multiplier.io.result, fpOutputs(2), fpOutputs(1), fpOutputs(0)) + state := writeReq + }.otherwise { + laneReg := laneReg + 1.U + mulStarted := false.B + } + } + } + is(writeReq) { + io.bankWrite(0).io.req.valid := true.B + when(io.bankWrite(0).io.req.fire) { + state := writeResp + } + } + is(writeResp) { + io.bankWrite(0).io.resp.ready := true.B + when(io.bankWrite(0).io.resp.fire) { + when(rowReg === iterReg - 1.U && group + 1.U === groups) { + state := complete + }.otherwise { + when(rowReg === iterReg - 1.U) { + group := group + 1.U + rowReg := 0.U + }.otherwise { + rowReg := rowReg + 1.U + } + readSent.foreach(_ := false.B) + readReceived.foreach(_ := false.B) + state := readReq + } + } + } + is(complete) { + when(io.cmdResp.fire) { + state := idle + } + } + } +} diff --git a/examples/prefab/balls/int2fp/arch/Int2FpBall.scala b/examples/prefab/balls/int2fp/arch/Int2FpBall.scala new file mode 100644 index 000000000..c8f741996 --- /dev/null +++ b/examples/prefab/balls/int2fp/arch/Int2FpBall.scala @@ -0,0 +1,52 @@ +package examples.balls.int2fp + +import framework.acc.balldomain.bbus.{BallFactory, BallPorts} +import buckyball.balls.int2fp.{Params => ParametersProto} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import examples.balls.int2fp.configs.Int2FpBallParam + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} + +import framework.acc.balldomain.ball.blink.{BallStatus, BlinkIO, HasBlink} + +@instantiable +class Int2FpBall( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Module + with HasBlink { + val ball = bp.balls.find(_.id == id).get + + val ballParam = Int2FpBallParam.fromProto(ball.parameters.unpack(classOf[ParametersProto])) + + @public + val blink = IO(new BlinkIO(fp, mp, bp, id)) + + val unit: Instance[Int2Fp] = + Instantiate(new Int2Fp(fp, mp, bp, ball, ballParam)) + unit.io.cmdReq <> blink.cmdReq + unit.io.cmdResp <> blink.cmdResp + for (i <- 0 until ball.inBW) unit.io.bankRead(i) <> blink.bankRead(i) + for (i <- 0 until ball.outBW) unit.io.bankWrite(i) <> blink.bankWrite(i) + blink.status <> unit.io.status + +} + +object Int2FpBall extends BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts = + BallPorts(Instantiate(new Int2FpBall(fp, mp, bp, id)).blink, None) + +} diff --git a/examples/prefab/balls/int2fp/configs/Int2FpBallParam.scala b/examples/prefab/balls/int2fp/configs/Int2FpBallParam.scala new file mode 100644 index 000000000..9a20a54f8 --- /dev/null +++ b/examples/prefab/balls/int2fp/configs/Int2FpBallParam.scala @@ -0,0 +1,20 @@ +package examples.balls.int2fp.configs + +import buckyball.balls.int2fp.{Params => ParametersProto} +import upickle.default.{macroRW, ReadWriter} + +case class Int2FpBallParam( + inputType: String, + outputType: String) + +object Int2FpBallParam { + + def fromProto(source: ParametersProto): Int2FpBallParam = { + val parameters = source + require(parameters.hasInputType && parameters.hasOutputType) + val param = Int2FpBallParam(parameters.getInputType, parameters.getOutputType) + param + } + + implicit val rw: ReadWriter[Int2FpBallParam] = macroRW +} diff --git a/examples/prefab/balls/int2fp/configs/default.toml b/examples/prefab/balls/int2fp/configs/default.toml new file mode 100644 index 000000000..436aea20e --- /dev/null +++ b/examples/prefab/balls/int2fp/configs/default.toml @@ -0,0 +1,4 @@ +# Int2Fp Ball Configuration + +inputType = "INT32" +outputType = "FP32" diff --git a/examples/prefab/balls/int2fp/configs/int2fp_params.proto b/examples/prefab/balls/int2fp/configs/int2fp_params.proto new file mode 100644 index 000000000..cb4db0f16 --- /dev/null +++ b/examples/prefab/balls/int2fp/configs/int2fp_params.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; +package buckyball.balls.int2fp; +option java_package = "buckyball.balls.int2fp"; +option java_multiple_files = true; + +message Params { + optional string inputType = 1; + optional string outputType = 2; +} diff --git a/examples/prefab/balls/int2fp/emu/src/dispatch.rs b/examples/prefab/balls/int2fp/emu/src/dispatch.rs new file mode 100644 index 000000000..c0c9e8136 --- /dev/null +++ b/examples/prefab/balls/int2fp/emu/src/dispatch.rs @@ -0,0 +1,31 @@ +use bebop_bemu::ball::ExecContext; + +use super::int2fp; + +const BALL_CLASS: &str = "examples.balls.int2fp.Int2FpBall"; + +pub fn execute_known( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, + ctx: &mut ExecContext, +) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("INT32_TO_FP32") => Some(int2fp::execute(xs1, xs2, ctx)), + Some(_) | None => None, + } +} + +pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("INT32_TO_FP32") => Some(int2fp::latency(xs1, xs2)), + Some(_) | None => None, + } +} diff --git a/examples/prefab/balls/int2fp/emu/src/int2fp.rs b/examples/prefab/balls/int2fp/emu/src/int2fp.rs new file mode 100644 index 000000000..c8d25791b --- /dev/null +++ b/examples/prefab/balls/int2fp/emu/src/int2fp.rs @@ -0,0 +1,69 @@ +use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old}; +use super::instruction::ExecContext; +use bebop_bemu::thread::banks::bank_lines; + +pub(crate) fn execute(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + assert!(!rs1_read_old(xs1), "int2fp: reads no old destination; rs1[29] must be clear"); + let input_bank = rs1_b0(xs1); + let scale_bank = rs1_b1(xs1); + let output_bank = rs1_b2(xs1); + let iter = rs1_iter(xs1) as usize; + if input_bank == scale_bank || input_bank == output_bank || scale_bank == output_bank { + panic!("int32_to_fp32: banks must be distinct"); + } + if !ctx.config(input_bank).allocated + || !ctx.config(scale_bank).allocated + || !ctx.config(output_bank).allocated + { + panic!("int32_to_fp32: bank not allocated"); + } + let groups = ctx.config(input_bank).cols; + if groups == 0 + || ctx.config(scale_bank).cols != groups + || ctx.config(output_bank).cols != groups + { + panic!("int32_to_fp32: operands must have the same channel groups"); + } + if iter == 0 || iter % 4 != 0 || iter > bank_lines() { + panic!("int32_to_fp32: iter must be a positive multiple of four within bank depth"); + } + if xs2 >> 1 != 0 { + panic!("int32_to_fp32: rs2[63:1] must be zero"); + } + let relu = xs2 & 1 != 0; + + // Group g of every operand is channel block g; each block uses the same rows. + for group in 0..groups { + let input = pbank_group(ctx, input_bank, group); + let scale = pbank_group(ctx, scale_bank, group); + let output = pbank_group(ctx, output_bank, group); + for row in 0..iter { + for lane in 0..4 { + let offset = row * 16 + lane * 4; + let scale_offset = (row % 4) * 16 + lane * 4; + let value = + i32::from_le_bytes(ctx.banks[input][offset..offset + 4].try_into().unwrap()); + let factor = f32::from_bits(u32::from_le_bytes( + ctx.banks[scale][scale_offset..scale_offset + 4] + .try_into() + .unwrap(), + )); + if !factor.is_finite() || factor <= 0.0 { + panic!("int32_to_fp32: scales must be finite and positive"); + } + let value = if relu { value.max(0) } else { value }; + let result = (value as f32) * factor; + ctx.banks[output][offset..offset + 4].copy_from_slice(&result.to_le_bytes()); + } + } + } + 0 +} + +pub(crate) fn latency(xs1: u64, xs2: u64) -> u64 { + let iter = rs1_iter(xs1); + if iter == 0 || iter % 4 != 0 || xs2 >> 1 != 0 { + panic!("int32_to_fp32: illegal encoding"); + } + iter +} diff --git a/examples/prefab/balls/int2fp/emu/src/lib.rs b/examples/prefab/balls/int2fp/emu/src/lib.rs new file mode 100644 index 000000000..4e14f52d0 --- /dev/null +++ b/examples/prefab/balls/int2fp/emu/src/lib.rs @@ -0,0 +1,11 @@ +pub(crate) use bebop_bemu::{thread::instructions::decode, ball as instruction}; + +mod int2fp; + +mod dispatch; + +pub use dispatch::{cycles_after_issue, execute_known}; + +#[bebop_bemu::ball::linkme::distributed_slice(bebop_bemu::BALLS)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static BALL: bebop_bemu::Ball = bebop_bemu::Ball { execute: execute_known, cycles: cycles_after_issue }; diff --git a/examples/prefab/balls/int2fp/emu/src/model.rs b/examples/prefab/balls/int2fp/emu/src/model.rs new file mode 100644 index 000000000..34a59807d --- /dev/null +++ b/examples/prefab/balls/int2fp/emu/src/model.rs @@ -0,0 +1,88 @@ +pub fn int2fp_fp32_bits(value: i32, dw_bits: u32) -> u32 { + fp32_multiply(int32_to_fp32(value), dw_bits) +} + +pub fn int2fp_dequant_bits(value: i32, da_bits: u32, dw_bits: u32) -> u32 { + int2fp_fp32_bits(value, fp32_multiply(da_bits, dw_bits)) +} + +pub fn int32_to_fp32(value: i32) -> u32 { + if value == 0 { + return 0; + } + + let sign = ((value as u32) >> 31) & 1; + let abs = if value == i32::MIN { + 0x8000_0000u32 + } else { + value.unsigned_abs() + }; + + let leading_one = 31u32 - abs.leading_zeros(); + let mut exponent = leading_one + 127; + let significand = if leading_one > 23 { + let right_shift = leading_one - 23; + let abs_wide = u64::from(abs); + let truncated = abs_wide >> right_shift; + let half = 1u64 << (right_shift - 1); + let remainder = abs_wide & ((1u64 << right_shift) - 1); + let round_up = remainder > half || (remainder == half && (truncated & 1) != 0); + let rounded = truncated + u64::from(round_up); + if ((rounded >> 24) & 1) != 0 { + exponent = leading_one + 128; + (rounded >> 1) as u32 + } else { + rounded as u32 + } + } else { + abs << (23 - leading_one) + }; + + (sign << 31) | ((exponent & 0xff) << 23) | (significand & 0x7f_ffff) +} + +fn fp32_multiply(a: u32, b: u32) -> u32 { + let a_sign = (a >> 31) & 1; + let b_sign = (b >> 31) & 1; + let a_exp = (a >> 23) & 0xff; + let b_exp = (b >> 23) & 0xff; + let a_frac = a & 0x7f_ffff; + let b_frac = b & 0x7f_ffff; + let a_mant = (1u64 << 23) | u64::from(a_frac); + let b_mant = (1u64 << 23) | u64::from(b_frac); + let a_zero = a_exp == 0 && a_frac == 0; + let b_zero = b_exp == 0 && b_frac == 0; + let prod = a_mant * b_mant; + let (sig, guard, round, sticky, exp_adjust) = if ((prod >> 47) & 1) != 0 { + ( + (prod >> 24) as u32, + ((prod >> 23) & 1) != 0, + ((prod >> 22) & 1) != 0, + (prod & ((1u64 << 22) - 1)) != 0, + 1u32, + ) + } else { + ( + (prod >> 23) as u32, + ((prod >> 22) & 1) != 0, + ((prod >> 21) & 1) != 0, + (prod & ((1u64 << 21) - 1)) != 0, + 0u32, + ) + }; + let increment = guard && (round || sticky || (sig & 1) != 0); + let rounded = u64::from(sig) + u64::from(increment); + let carry = ((rounded >> 24) & 1) as u32; + let final_sig = if carry != 0 { rounded >> 1 } else { rounded } as u32; + let exp_wide = (a_exp + b_exp + exp_adjust + carry).wrapping_sub(127) & 0x3ff; + + if a_zero || b_zero { + 0 + } else if (exp_wide & 0x200) != 0 { + 0 + } else if (exp_wide & 0x100) != 0 { + ((a_sign ^ b_sign) << 31) | (0xff << 23) + } else { + ((a_sign ^ b_sign) << 31) | ((exp_wide & 0xff) << 23) | (final_sig & 0x7f_ffff) + } +} diff --git a/examples/prefab/balls/int2fp/stack/src/int2fpOps.cpp b/examples/prefab/balls/int2fp/stack/src/int2fpOps.cpp new file mode 100644 index 000000000..198e63d64 --- /dev/null +++ b/examples/prefab/balls/int2fp/stack/src/int2fpOps.cpp @@ -0,0 +1,66 @@ +//===- int2fpOps.cpp - Int2FpBall operations ------------------------------===// +// +// The dialect of Int2FpBall, its virtual operations building their physical +// ones and the instructions of the physical operations. +// +//===----------------------------------------------------------------------===// + +#include "int2fpOps.h" + +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::int2fp_ball; +using mlir::buddy::bankFields; +using mlir::buddy::constantI64; +using mlir::buddy::emitInstruction; +using mlir::buddy::field; + +#include "int2fpDialect.cpp.inc" + +#define GET_OP_CLASSES +#include "int2fpOps.cpp.inc" + +void Int2FpBallDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "int2fpOps.cpp.inc" + >(); +} + +namespace { +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallDialect([](DialectRegistry ®istry) { + registry.insert(); + }), + true); +} // namespace + +SmallVector BankInt32ToFp32Op::getSourceBanks() { + return {getValues(), getScales()}; +} +Value BankInt32ToFp32Op::getDestinationBank() { return getOutput(); } +bool BankInt32ToFp32Op::readsDestination() { return false; } +Operation *BankInt32ToFp32Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return Int32ToFp32Op::create(builder, getLoc(), sources[0], sources[1], + destination, getRows(), getRelu()); +} + +SmallVector Int32ToFp32Op::getSourceBanks() { + return {int64_t(getValues()), int64_t(getScales())}; +} +int64_t Int32ToFp32Op::getDestinationBank() { return getOutput(); } +LogicalResult Int32ToFp32Op::lowerToInstructions(OpBuilder &builder) { + Location loc = getLoc(); + int64_t rows = getRows(); + emitInstruction( + builder, loc, buckyball_target::getBuckyballFunct7("INT32_TO_FP32"), + constantI64(builder, loc, + bankFields({int64_t(getValues()), int64_t(getScales())}, + getOutput(), false, rows * 4)), + constantI64(builder, loc, field(getRelu(), 0, 0))); + return success(); +} diff --git a/examples/prefab/balls/int2fp/stack/src/int2fpOps.h b/examples/prefab/balls/int2fp/stack/src/int2fpOps.h new file mode 100644 index 000000000..2358922d6 --- /dev/null +++ b/examples/prefab/balls/int2fp/stack/src/int2fpOps.h @@ -0,0 +1,20 @@ +//===- int2fpOps.h - Int2FpBall operations --------------------------------===// + +#ifndef INT2FP_OPS_H +#define INT2FP_OPS_H + +#include "Interfaces/BankInterfaces.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "int2fpDialect.h.inc" + +#define GET_OP_CLASSES +#include "int2fpOps.h.inc" + +#endif // INT2FP_OPS_H diff --git a/examples/prefab/balls/int2fp/stack/src/int2fpOps.td b/examples/prefab/balls/int2fp/stack/src/int2fpOps.td new file mode 100644 index 000000000..f4fb98859 --- /dev/null +++ b/examples/prefab/balls/int2fp/stack/src/int2fpOps.td @@ -0,0 +1,44 @@ +//===- int2fpOps.td - Int2FpBall operations ----------------*- tablegen -*-===// +// +// int32_to_fp32: `values` holds `rows` rows of sixteen i32 (four bank rows +// each), `scales` the sixteen f32 scales of the columns (four bank rows); the +// output receives the f32 value * scale of every element, with `relu` +// negative results replaced by zero. +// +//===----------------------------------------------------------------------===// + +#ifndef INT2FP_OPS +#define INT2FP_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Interfaces/BankInterfaces.td" + +def Int2FpBall_Dialect : Dialect { + let name = "int2fp_ball"; + let cppNamespace = "::buddy::int2fp_ball"; + let summary = "The operations of Int2FpBall"; +} + +class Int2FpBall_Op traits = []> + : Op])>; + +def BankInt32ToFp32Op : Int2FpBall_Op<"bank_int32_to_fp32", + [DeclareOpInterfaceMethods]> { + let summary = "int32_to_fp32 on virtual banks"; + let arguments = (ins I64:$values, I64:$scales, I64:$output, I64Attr:$rows, + UnitAttr:$relu); + let results = (outs I64:$outputOut); + let assemblyFormat = "$values `,` $scales `->` $output prop-dict attr-dict"; +} + +def Int32ToFp32Op : Int2FpBall_Op<"int32_to_fp32", + [DeclareOpInterfaceMethods]> { + let summary = "int32_to_fp32 on physical banks"; + let arguments = (ins I64Attr:$values, I64Attr:$scales, I64Attr:$output, + I64Attr:$rows, UnitAttr:$relu); + let assemblyFormat = "prop-dict attr-dict"; +} + +#endif // INT2FP_OPS diff --git a/examples/prefab/balls/int2fp/verify/README.md b/examples/prefab/balls/int2fp/verify/README.md new file mode 100644 index 000000000..1471bd59a --- /dev/null +++ b/examples/prefab/balls/int2fp/verify/README.md @@ -0,0 +1,77 @@ +# Int2FpBall UVM Verification + +Verifies the generated `Int2FpBall` module using the shared Blink UVM framework. + +RTL contract: one mnemonic `INT32_TO_FP32`, `inBW=2` (input + scale banks), +`outBW=1`, `rs2[0]=relu`, `rs2[63:1]=0`, iter a positive multiple of 4. +Scales are bank rows, not MMIO. + +## Structure + +- `../../../../verify/uvm/src/ball/bb_uvm_pkg.sv`: common Blink UVM agents and base env +- `src/common/int2fp_defs.svh`: DPI imports, `INT32_TO_FP32_FUNCT7`, timeouts +- `src/common/int2fp_items.svh`: `int2fp_cmd_item` with `load_rust_case` +- `src/seq/int2fp_sequences.svh`: one-case sequence +- `src/cov/int2fp_cov.svh`: relu `{0,1}` and iter `{4,8,16}`, 100% or fatal +- `src/env/int2fp_scoreboard.svh`: preload src/scale banks, compare writes vs DPI dst +- `src/env/int2fp_env.svh`: extends `bb_blink_env #(BB_IN_BW, BB_OUT_BW)` +- `src/tests/int2fp_test.svh`: `int2fp_ball_test` loops cases 0..5 +- `src/pkg/int2fp_pkg.sv`: package entry +- `src/tb_top.sv`: `bb_blink_if #(BB_IN_BW, BB_OUT_BW)` + `Int2FpBall` (2 read, 1 write) +- `filelists/int2fp_ball.f`: VCS filelist (`@UVM@` / `@RTL@`) + +## Test plan + +`+UVM_TESTNAME=int2fp_ball_test` runs six directed cases: + +| Index | iter | relu | Intent | +|-------|------|------|--------| +| 0 | 4 | 0 | MLIR-scale values | +| 1 | 4 | 1 | relu clamps negatives | +| 2 | 8 | 0 | eight-row, no relu | +| 3 | 8 | 1 | eight-row, relu | +| 4 | 16 | 0 | sixteen-row, no relu | +| 5 | 16 | 1 | sixteen-row, relu | + +Each case: reset, preload 16-byte i32 input rows and f32 scale rows into +`mem_model`, drive one `INT32_TO_FP32` command, wait for `scb.done()` or +timeout at 20000 cycles. Expected dst = `f32(i32) * scale` with relu clamp +on the integer before convert. `int2fp_cov` fatals unless relu and iter bins +are 100%. + +## BID (required) + +`+BID=` is mandatory. Missing it fatals at test start. No default bid. + +| Config | Plusarg | +|--------|---------| +| `sims.verilator.BuckyballPebbleVerilatorConfig` | `+BID=4` | + +```console +./simv +UVM_TESTNAME=int2fp_ball_test +BID=4 +``` + +## Build / Run + +Enter the shared UVM/VCS environment and build the DPI reference library: + +```console +nix develop ../../../../verify +cargo build --manifest-path casegen/Cargo.toml +``` + +Human: + +```console +bbdev uvm --run '--design pebble' +bbdev uvm --run '--design pebble --ball int2fp' +``` + +Agent (MCP): + +```text +bbdev_uvm_run(design="pebble", ball="int2fp") +``` + +bbdev injects `INT32_TO_FP32_FUNCT7`, `BB_IN_BW`, `BB_OUT_BW`, `+BID`, and +`+UVM_TESTNAME=int2fp_ball_test` from pebble design.pb. diff --git a/examples/balls/int2fp/verify/casegen/Cargo.lock b/examples/prefab/balls/int2fp/verify/casegen/Cargo.lock similarity index 100% rename from examples/balls/int2fp/verify/casegen/Cargo.lock rename to examples/prefab/balls/int2fp/verify/casegen/Cargo.lock diff --git a/examples/balls/int2fp/verify/casegen/Cargo.toml b/examples/prefab/balls/int2fp/verify/casegen/Cargo.toml similarity index 100% rename from examples/balls/int2fp/verify/casegen/Cargo.toml rename to examples/prefab/balls/int2fp/verify/casegen/Cargo.toml diff --git a/examples/prefab/balls/int2fp/verify/casegen/src/casegen.rs b/examples/prefab/balls/int2fp/verify/casegen/src/casegen.rs new file mode 100644 index 000000000..6f047909c --- /dev/null +++ b/examples/prefab/balls/int2fp/verify/casegen/src/casegen.rs @@ -0,0 +1,282 @@ +use crate::model::int2fp_fp32_bits; + +pub const BANK_NUM: u32 = 24; +pub const BANK_ENTRIES: u32 = 64; +pub const LANES: usize = 4; +pub const SCALE_ROWS: usize = 4; +pub const MAX_SRC: usize = 48; +pub const MAX_SCALE: usize = 12; +pub const MAX_DST: usize = 48; +pub const NUM_CASES: u32 = 8; + +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Int2FpCmd { + pub iter: u32, + pub relu: u32, + pub op1_bank: u32, + pub op2_bank: u32, + pub wr_bank: u32, + pub op1_group: u32, + pub op2_group: u32, + pub wr_group: u32, + pub op1_col: u32, + pub op2_col: u32, + pub wr_col: u32, + pub rob_id: u32, + pub rs2_lo: u32, + pub rs2_hi: u32, + pub num_src_words: u32, + pub num_scale_words: u32, + pub num_dst_words: u32, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Int2FpCase { + pub cmd: Int2FpCmd, + pub src_words: [u128; MAX_SRC], + pub scale_words: [u128; MAX_SCALE], + pub dst_words: [u128; MAX_DST], +} + +impl Int2FpCase { + pub fn src_lo(&self, index: usize) -> u64 { + self.src_words[index] as u64 + } + + pub fn src_hi(&self, index: usize) -> u64 { + (self.src_words[index] >> 64) as u64 + } + + pub fn scale_lo(&self, index: usize) -> u64 { + self.scale_words[index] as u64 + } + + pub fn scale_hi(&self, index: usize) -> u64 { + (self.scale_words[index] >> 64) as u64 + } + + pub fn dst_lo(&self, index: usize) -> u64 { + self.dst_words[index] as u64 + } + + pub fn dst_hi(&self, index: usize) -> u64 { + (self.dst_words[index] >> 64) as u64 + } +} + +pub fn gen_case(index: u32) -> Int2FpCase { + if index >= NUM_CASES { + panic!("int2fp: unsupported directed case {index}"); + } + match index { + 0 => directed(2, 4, 1, false, &MLIR_INPUT, &[0.125; 16], [0, 0, 0]), + 1 => directed(3, 4, 1, true, &RELU_INPUT, &[1.0; 16], [1, 2, 3]), + 2 => directed(4, 8, 1, false, &ITER8, &SCALE16, [2, 0, 1]), + 3 => directed(5, 8, 1, true, &ITER8, &SCALE16, [3, 1, 0]), + 4 => directed(6, 16, 1, false, &ITER16, &SCALE16, [0, 3, 2]), + 5 => directed(7, 16, 1, true, &ITER16, &SCALE16, [1, 1, 1]), + // Two channel groups: group 0 is MLIR_INPUT with 0.125 scales, group 1 + // is RELU_INPUT with SCALE16. + 6 => directed( + 0, + 4, + 2, + false, + &[MLIR_INPUT, RELU_INPUT].concat(), + &[[0.125; 16], SCALE16].concat(), + [0, 0, 0], + ), + // Three channel groups with ReLU and nonzero base groups: group g holds + // ITER8, then the two halves of ITER16; scales are SCALE16, 1.0, 0.5. + 7 => directed( + 1, + 8, + 3, + true, + &[&ITER8[..], &ITER16[..32], &ITER16[32..]].concat(), + &[SCALE16, [1.0; 16], [0.5; 16]].concat(), + [1, 2, 0], + ), + _ => panic!("int2fp: unsupported directed case {index}"), + } +} + +const MLIR_INPUT: [i32; 16] = [1, 2, 3, -1, -2, 0, 4, 5, 10, -10, 7, 100, -100, 8, 16, -8]; +const RELU_INPUT: [i32; 16] = [-8, -1, 0, 1, 2, -3, 4, -5, 6, 7, -8, 9, -10, 11, 12, -13]; +const ITER8: [i32; 32] = [ + 8, -4, 16, 0, 1, -2, 3, -8, 10, -10, 7, 100, -100, 8, 16, -8, 4, -1, 0, 2, -16, 32, -32, 5, 9, + -9, 12, -12, 6, -6, 11, -11, +]; +const ITER16: [i32; 64] = [ + 1, 2, 3, -1, -2, 0, 4, 5, 10, -10, 7, 100, -100, 8, 16, -8, 8, -4, 16, 0, 1, -2, 3, -8, 12, + -12, 6, -6, 9, -9, 11, -11, 20, -20, 24, -24, 15, -15, 18, -18, 30, -30, 40, -40, 25, -25, 28, + -28, 50, -50, 60, -60, 7, -7, 13, -13, 14, -14, 17, -17, 19, -19, 21, -21, +]; +const SCALE16: [f32; 16] = [ + 0.125, 0.25, 0.5, 1.0, 0.25, 0.5, 1.0, 2.0, 0.5, 1.0, 2.0, 0.125, 1.0, 0.125, 0.25, 0.5, +]; + +fn pack_u32s(vals: &[u32]) -> Vec { + if vals.len() % LANES != 0 { + panic!( + "int2fp: value count {} is not a multiple of four", + vals.len() + ); + } + vals.chunks(LANES) + .map(|chunk| { + let mut word = 0u128; + for (lane, value) in chunk.iter().enumerate() { + word |= u128::from(*value) << (lane * 32); + } + word + }) + .collect() +} + +fn copy_words(src: &[u128]) -> [u128; N] { + if src.len() > N { + panic!("int2fp: {} words exceed buffer {N}", src.len()); + } + let mut out = [0u128; N]; + out[..src.len()].copy_from_slice(src); + out +} + +fn split_rs(value: u64) -> (u32, u32) { + (value as u32, (value >> 32) as u32) +} + +fn positive_finite(bits: u32) -> bool { + bits >> 31 == 0 && ((bits >> 23) & 0xff) != 0xff && (bits & 0x7fff_ffff) != 0 +} + +pub(crate) fn validate( + op1_bank: u32, + op2_bank: u32, + wr_bank: u32, + op1_col: u32, + op2_col: u32, + wr_col: u32, + iter: u32, + rs2: u64, + scales: &[u128], +) { + if iter == 0 || iter % 4 != 0 { + panic!("INT32_TO_FP32 iter must be a positive multiple of four"); + } + if iter > BANK_ENTRIES { + panic!("INT32_TO_FP32 input exceeds bank depth"); + } + if op1_bank >= BANK_NUM || op2_bank >= BANK_NUM || wr_bank >= BANK_NUM { + panic!("INT32_TO_FP32 bank id is invalid"); + } + if op1_col == 0 || op1_col != op2_col || op1_col != wr_col { + panic!("INT32_TO_FP32 operands must have the same channel groups"); + } + if op1_bank == op2_bank || op1_bank == wr_bank || op2_bank == wr_bank { + panic!("INT32_TO_FP32 banks must be distinct"); + } + if rs2 >> 1 != 0 { + panic!("INT32_TO_FP32 reserves rs2[63:1]"); + } + if scales.len() != SCALE_ROWS * op2_col as usize { + panic!("INT32_TO_FP32 must emit four scale rows per group"); + } + for word in scales { + for lane in 0..LANES { + let bits = (*word >> (lane * 32)) as u32; + if !positive_finite(bits) { + panic!("INT32_TO_FP32 scales must be finite and positive"); + } + } + } +} + +// Words are group-major: src/dst word i is group i / iter, row i % iter. +fn dst_words(src: &[u128], scales: &[u128], iter: usize, relu: bool) -> Vec { + src.iter() + .enumerate() + .map(|(i, word)| { + let scale = scales[i / iter * SCALE_ROWS + i % iter % SCALE_ROWS]; + let mut packed = 0u128; + for lane in 0..LANES { + let raw = (*word >> (lane * 32)) as u32 as i32; + let value = if relu { raw.max(0) } else { raw }; + let bits = int2fp_fp32_bits(value, (scale >> (lane * 32)) as u32); + packed |= u128::from(bits) << (lane * 32); + } + packed + }) + .collect() +} + +// `groups` are the op1/op2/wr base groups. +fn directed( + rob_id: u32, + iter: u32, + cols: u32, + relu: bool, + values: &[i32], + scales: &[f32], + groups: [u32; 3], +) -> Int2FpCase { + if values.len() != (cols * iter) as usize * LANES { + panic!( + "int2fp: {} values do not match iter {iter} * {LANES}", + values.len() + ); + } + if scales.len() != cols as usize * SCALE_ROWS * LANES { + panic!( + "int2fp: {} scales, expected {}", + scales.len(), + SCALE_ROWS * LANES + ); + } + let src = pack_u32s(&values.iter().map(|v| *v as u32).collect::>()); + let scale = pack_u32s(&scales.iter().map(|v| v.to_bits()).collect::>()); + let op1_bank = 0; + let op2_bank = 1; + let wr_bank = 2; + let op1_col = cols; + let op2_col = cols; + let wr_col = cols; + let rs2 = u64::from(relu); + validate( + op1_bank, op2_bank, wr_bank, op1_col, op2_col, wr_col, iter, rs2, &scale, + ); + if src.len() != (cols * iter) as usize { + panic!("int2fp: src rows {} != iter {iter} * groups", src.len()); + } + let dst = dst_words(&src, &scale, iter as usize, relu); + if dst.len() != (cols * iter) as usize { + panic!("int2fp: dst rows {} != iter {iter}", dst.len()); + } + let (rs2_lo, rs2_hi) = split_rs(rs2); + Int2FpCase { + cmd: Int2FpCmd { + iter, + relu: u32::from(relu), + op1_bank, + op2_bank, + wr_bank, + op1_group: groups[0], + op2_group: groups[1], + wr_group: groups[2], + op1_col, + op2_col, + wr_col, + rob_id, + rs2_lo, + rs2_hi, + num_src_words: src.len() as u32, + num_scale_words: scale.len() as u32, + num_dst_words: dst.len() as u32, + }, + src_words: copy_words(&src), + scale_words: copy_words(&scale), + dst_words: copy_words(&dst), + } +} diff --git a/examples/prefab/balls/int2fp/verify/casegen/src/lib.rs b/examples/prefab/balls/int2fp/verify/casegen/src/lib.rs new file mode 100644 index 000000000..8ae4bbf68 --- /dev/null +++ b/examples/prefab/balls/int2fp/verify/casegen/src/lib.rs @@ -0,0 +1,108 @@ +mod casegen; +#[path = "../../../emu/src/model.rs"] +mod model; + +use std::cell::RefCell; + +use casegen::{Int2FpCase, Int2FpCmd, MAX_DST, MAX_SCALE, MAX_SRC}; + +thread_local! { + static CURRENT: RefCell> = const { RefCell::new(None) }; +} + +#[no_mangle] +pub extern "C" fn int2fp_case_load(index: u32) -> i32 { + let case = casegen::gen_case(index); + CURRENT.with(|current| *current.borrow_mut() = Some(case)); + 0 +} + +fn current_case R, R>(f: F) -> R { + CURRENT.with(|current| match current.borrow().as_ref() { + Some(case) => f(case), + None => panic!("int2fp_case: no case loaded; call int2fp_case_load first"), + }) +} + +#[no_mangle] +pub extern "C" fn int2fp_case_cmd(out_ptr: *mut Int2FpCmd) { + if out_ptr.is_null() { + panic!("int2fp_case_cmd: null out_ptr"); + } + current_case(|case| unsafe { + *out_ptr = case.cmd; + }); +} + +fn src_index(index: u32, limit: u32, what: &str) -> usize { + if index >= limit { + panic!("int2fp_case_{what}: index {index} out of range {limit}"); + } + index as usize +} + +#[no_mangle] +pub extern "C" fn int2fp_case_src_word_lo(word_index: u32) -> u64 { + current_case(|case| { + let i = src_index(word_index, case.cmd.num_src_words, "src_word_lo"); + if i >= MAX_SRC { + panic!("int2fp_case_src_word_lo: word_index out of range"); + } + case.src_lo(i) + }) +} + +#[no_mangle] +pub extern "C" fn int2fp_case_src_word_hi(word_index: u32) -> u64 { + current_case(|case| { + let i = src_index(word_index, case.cmd.num_src_words, "src_word_hi"); + if i >= MAX_SRC { + panic!("int2fp_case_src_word_hi: word_index out of range"); + } + case.src_hi(i) + }) +} + +#[no_mangle] +pub extern "C" fn int2fp_case_scale_word_lo(word_index: u32) -> u64 { + current_case(|case| { + let i = src_index(word_index, case.cmd.num_scale_words, "scale_word_lo"); + if i >= MAX_SCALE { + panic!("int2fp_case_scale_word_lo: word_index out of range"); + } + case.scale_lo(i) + }) +} + +#[no_mangle] +pub extern "C" fn int2fp_case_scale_word_hi(word_index: u32) -> u64 { + current_case(|case| { + let i = src_index(word_index, case.cmd.num_scale_words, "scale_word_hi"); + if i >= MAX_SCALE { + panic!("int2fp_case_scale_word_hi: word_index out of range"); + } + case.scale_hi(i) + }) +} + +#[no_mangle] +pub extern "C" fn int2fp_case_dst_word_lo(word_index: u32) -> u64 { + current_case(|case| { + let i = src_index(word_index, case.cmd.num_dst_words, "dst_word_lo"); + if i >= MAX_DST { + panic!("int2fp_case_dst_word_lo: word_index out of range"); + } + case.dst_lo(i) + }) +} + +#[no_mangle] +pub extern "C" fn int2fp_case_dst_word_hi(word_index: u32) -> u64 { + current_case(|case| { + let i = src_index(word_index, case.cmd.num_dst_words, "dst_word_hi"); + if i >= MAX_DST { + panic!("int2fp_case_dst_word_hi: word_index out of range"); + } + case.dst_hi(i) + }) +} diff --git a/examples/balls/int2fp/verify/filelists/int2fp_ball.f b/examples/prefab/balls/int2fp/verify/filelists/int2fp_ball.f similarity index 90% rename from examples/balls/int2fp/verify/filelists/int2fp_ball.f rename to examples/prefab/balls/int2fp/verify/filelists/int2fp_ball.f index 4bef90c87..937d5b30f 100644 --- a/examples/balls/int2fp/verify/filelists/int2fp_ball.f +++ b/examples/prefab/balls/int2fp/verify/filelists/int2fp_ball.f @@ -1,3 +1,4 @@ ++incdir+@UVM@/src/core +incdir+@UVM@/src/ball +incdir+@UVM@/src/ball/agents/cmd +incdir+@UVM@/src/ball/agents/mem @@ -13,6 +14,7 @@ +incdir+src/tests +incdir+@RTL@ +@UVM@/src/core/package.sv @UVM@/src/ball/bb_blink_if.sv @UVM@/src/ball/bb_uvm_pkg.sv diff --git a/examples/prefab/balls/int2fp/verify/src/common/int2fp_defs.svh b/examples/prefab/balls/int2fp/verify/src/common/int2fp_defs.svh new file mode 100644 index 000000000..812dff051 --- /dev/null +++ b/examples/prefab/balls/int2fp/verify/src/common/int2fp_defs.svh @@ -0,0 +1,39 @@ +typedef struct { + int unsigned iter; + int unsigned relu; + int unsigned op1_bank; + int unsigned op2_bank; + int unsigned wr_bank; + int unsigned op1_group; + int unsigned op2_group; + int unsigned wr_group; + int unsigned op1_col; + int unsigned op2_col; + int unsigned wr_col; + int unsigned rob_id; + int unsigned rs2_lo; + int unsigned rs2_hi; + int unsigned num_src_words; + int unsigned num_scale_words; + int unsigned num_dst_words; +} int2fp_cmd_dpi_t; + +import "DPI-C" function int int2fp_case_load(input int unsigned index); +import "DPI-C" function void int2fp_case_cmd(output int2fp_cmd_dpi_t cmd); +import "DPI-C" function longint unsigned int2fp_case_src_word_lo(input int unsigned word_index); +import "DPI-C" function longint unsigned int2fp_case_src_word_hi(input int unsigned word_index); +import "DPI-C" function longint unsigned int2fp_case_scale_word_lo(input int unsigned word_index); +import "DPI-C" function longint unsigned int2fp_case_scale_word_hi(input int unsigned word_index); +import "DPI-C" function longint unsigned int2fp_case_dst_word_lo(input int unsigned word_index); +import "DPI-C" function longint unsigned int2fp_case_dst_word_hi(input int unsigned word_index); + +`ifndef INT32_TO_FP32_FUNCT7 +`error "INT32_TO_FP32_FUNCT7 must be provided by the selected Thread ballISA" +`endif + +localparam int INT32_TO_FP32_THREAD_FUNCT7 = `INT32_TO_FP32_FUNCT7; +localparam int INT2FP_MAX_SRC_WORDS = 48; +localparam int INT2FP_MAX_SCALE_WORDS = 12; +localparam int INT2FP_MAX_DST_WORDS = 48; +localparam int INT2FP_NUM_CASES = 8; +localparam int INT2FP_TIMEOUT_CYCLES = 20000; diff --git a/examples/prefab/balls/int2fp/verify/src/common/int2fp_items.svh b/examples/prefab/balls/int2fp/verify/src/common/int2fp_items.svh new file mode 100644 index 000000000..8d0c96076 --- /dev/null +++ b/examples/prefab/balls/int2fp/verify/src/common/int2fp_items.svh @@ -0,0 +1,103 @@ +class int2fp_cmd_item extends bb_blink_cmd_item; + `uvm_object_utils(int2fp_cmd_item) + + bit [31:0] relu; + bit [31:0] num_src_words; + bit [31:0] num_scale_words; + bit [31:0] num_dst_words; + bit [127:0] src_words[INT2FP_MAX_SRC_WORDS]; + bit [127:0] scale_words[INT2FP_MAX_SCALE_WORDS]; + bit [127:0] dst_words[INT2FP_MAX_DST_WORDS]; + + function new(string name = "int2fp_cmd_item"); + super.new(name); + endfunction + + function void load_rust_case(int unsigned index); + int2fp_cmd_dpi_t cmd; + int unsigned rc; + longint unsigned w_lo; + longint unsigned w_hi; + int i; + + rc = int2fp_case_load(index); + if (rc != 0) begin + `uvm_fatal("CASE", $sformatf("int2fp_case_load returned %0d for index %0d", rc, index)) + end + int2fp_case_cmd(cmd); + + // bb_int32_to_fp32: input/scale/output banks and iter come from rs1, relu from rs2[0]. + rob_id = cmd.rob_id[BB_BLINK_ROB_ID_W-1:0]; + funct = INT32_TO_FP32_THREAD_FUNCT7[6:0]; + rs2 = {cmd.rs2_hi, cmd.rs2_lo}; + iter = cmd.iter; + op1_bank = cmd.op1_bank; + op2_bank = cmd.op2_bank; + wr_bank = cmd.wr_bank; + op1_group = cmd.op1_group; + op2_group = cmd.op2_group; + wr_group = cmd.wr_group; + op1_col = cmd.op1_col; + op2_col = cmd.op2_col; + wr_col = cmd.wr_col; + wr_read_bank = '0; + wr_arch_bank = cmd.wr_bank; + relu = cmd.relu; + num_src_words = cmd.num_src_words; + num_scale_words = cmd.num_scale_words; + num_dst_words = cmd.num_dst_words; + + if (iter == 0 || iter[1:0] != 2'b00) + `uvm_fatal("CASE", "INT32_TO_FP32 iter must be a positive multiple of four") + if (iter != 4 && iter != 8 && iter != 16) + `uvm_fatal("CASE", $sformatf("INT32_TO_FP32 iter %0d is not in {4,8,16}", iter)) + if (op1_col == 0 || op1_col != op2_col || op1_col != wr_col) + `uvm_fatal("CASE", "INT32_TO_FP32 operands must have the same channel groups") + if (op1_bank == op2_bank || op1_bank == wr_bank || op2_bank == wr_bank) + `uvm_fatal("CASE", "INT32_TO_FP32 banks must be distinct") + if (rs2[63:1] != 0) `uvm_fatal("CASE", "INT32_TO_FP32 reserves rs2[63:1]") + if (rs2[0] != relu[0]) `uvm_fatal("CASE", "INT32_TO_FP32 rs2[0] must match relu") + // Words are group-major: word i is group i / iter, row i % iter. + if (num_src_words != iter * op1_col) + `uvm_fatal("CASE", $sformatf("num_src_words %0d != iter * groups", num_src_words)) + if (num_dst_words != iter * op1_col) + `uvm_fatal("CASE", $sformatf("num_dst_words %0d != iter * groups", num_dst_words)) + if (num_scale_words != 4 * op1_col) + `uvm_fatal("CASE", $sformatf("num_scale_words %0d, expected 4 per group", num_scale_words)) + if (num_src_words == 0 || num_src_words > INT2FP_MAX_SRC_WORDS) + `uvm_fatal("CASE", $sformatf("num_src_words out of range: %0d", num_src_words)) + if (num_dst_words == 0 || num_dst_words > INT2FP_MAX_DST_WORDS) + `uvm_fatal("CASE", $sformatf("num_dst_words out of range: %0d", num_dst_words)) + + for (i = 0; i < num_src_words; i++) begin + w_lo = int2fp_case_src_word_lo(i); + w_hi = int2fp_case_src_word_hi(i); + src_words[i] = {w_hi, w_lo}; + end + for (i = 0; i < num_scale_words; i++) begin + w_lo = int2fp_case_scale_word_lo(i); + w_hi = int2fp_case_scale_word_hi(i); + scale_words[i] = {w_hi, w_lo}; + end + for (i = 0; i < num_dst_words; i++) begin + w_lo = int2fp_case_dst_word_lo(i); + w_hi = int2fp_case_dst_word_hi(i); + dst_words[i] = {w_hi, w_lo}; + end + endfunction + + function void do_copy(uvm_object rhs); + int2fp_cmd_item rhs_; + super.do_copy(rhs); + if (!$cast(rhs_, rhs)) begin + `uvm_fatal("COPY", "rhs is not int2fp_cmd_item") + end + relu = rhs_.relu; + num_src_words = rhs_.num_src_words; + num_scale_words = rhs_.num_scale_words; + num_dst_words = rhs_.num_dst_words; + for (int i = 0; i < INT2FP_MAX_SRC_WORDS; i++) src_words[i] = rhs_.src_words[i]; + for (int i = 0; i < INT2FP_MAX_SCALE_WORDS; i++) scale_words[i] = rhs_.scale_words[i]; + for (int i = 0; i < INT2FP_MAX_DST_WORDS; i++) dst_words[i] = rhs_.dst_words[i]; + endfunction +endclass diff --git a/examples/prefab/balls/int2fp/verify/src/cov/int2fp_cov.svh b/examples/prefab/balls/int2fp/verify/src/cov/int2fp_cov.svh new file mode 100644 index 000000000..c62f847c8 --- /dev/null +++ b/examples/prefab/balls/int2fp/verify/src/cov/int2fp_cov.svh @@ -0,0 +1,41 @@ +class int2fp_cov extends uvm_component; + `uvm_component_utils(int2fp_cov) + + uvm_analysis_imp_cmd #(bb_blink_cmd_item, int2fp_cov) cmd_imp; + + bit cur_relu; + int unsigned cur_iter; + int unsigned cur_groups; + + covergroup cmd_cg; + coverpoint cur_relu {bins off = {0}; bins on = {1};} + coverpoint cur_iter {bins i4 = {4}; bins i8 = {8}; bins i16 = {16};} + coverpoint cur_groups {bins g1 = {1}; bins g2 = {2}; bins g3 = {3};} + endgroup + + int cmd_count; + + function new(string name, uvm_component parent); + super.new(name, parent); + cmd_imp = new("cmd_imp", this); + cmd_cg = new(); + endfunction + + function void write_cmd(bb_blink_cmd_item item); + if (item.funct != INT32_TO_FP32_THREAD_FUNCT7[6:0]) + `uvm_fatal("COV", $sformatf("unexpected funct %0d", item.funct)) + cur_relu = item.rs2[0]; + cur_iter = item.iter; + cur_groups = item.op1_col; + cmd_count++; + cmd_cg.sample(); + endfunction + + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (cmd_count == 0) `uvm_fatal("COV", "no cmd observed") + if (cmd_cg.get_coverage() != 100.0) + `uvm_fatal("COV", $sformatf("int2fp_cov %0.2f%%", cmd_cg.get_coverage())) + endfunction +endclass diff --git a/examples/balls/int2fp/verify/src/env/int2fp_env.svh b/examples/prefab/balls/int2fp/verify/src/env/int2fp_env.svh similarity index 100% rename from examples/balls/int2fp/verify/src/env/int2fp_env.svh rename to examples/prefab/balls/int2fp/verify/src/env/int2fp_env.svh diff --git a/examples/prefab/balls/int2fp/verify/src/env/int2fp_scoreboard.svh b/examples/prefab/balls/int2fp/verify/src/env/int2fp_scoreboard.svh new file mode 100644 index 000000000..c7b35037b --- /dev/null +++ b/examples/prefab/balls/int2fp/verify/src/env/int2fp_scoreboard.svh @@ -0,0 +1,190 @@ +class int2fp_scoreboard extends uvm_scoreboard; + `uvm_component_utils(int2fp_scoreboard) + + uvm_analysis_imp_stim #(bb_blink_cmd_item, int2fp_scoreboard) stim_imp; + uvm_analysis_imp_cmd #(bb_blink_cmd_item, int2fp_scoreboard) cmd_imp; + uvm_analysis_imp_read #(bb_blink_read_item, int2fp_scoreboard) read_imp; + uvm_analysis_imp_write #(bb_blink_write_item, int2fp_scoreboard) write_imp; + uvm_analysis_imp_resp #(bb_blink_resp_item, int2fp_scoreboard) resp_imp; + + bb_blink_mem_model #(`BB_IN_BW, `BB_OUT_BW) mem_model; + + int2fp_cmd_item stim_q[$]; + int unsigned expected_reads[`BB_IN_BW]; + int unsigned expected_writes; + int unsigned cmd_count; + int unsigned read_count[`BB_IN_BW]; + int unsigned write_count; + int unsigned resp_count; + + function new(string name, uvm_component parent); + super.new(name, parent); + stim_imp = new("stim_imp", this); + cmd_imp = new("cmd_imp", this); + read_imp = new("read_imp", this); + write_imp = new("write_imp", this); + resp_imp = new("resp_imp", this); + endfunction + + function void write_stim(bb_blink_cmd_item item); + int2fp_cmd_item got; + int2fp_cmd_item clone; + int i; + + if (stim_q.size() != 0) begin + `uvm_fatal("SCB", "single outstanding command supported") + end + if (!$cast(got, item)) begin + `uvm_fatal("SCB", "stim item is not int2fp_cmd_item") + end + if (!$cast(clone, got.clone())) begin + `uvm_fatal("SCB", "failed to clone stimulus item") + end + stim_q.push_back(clone); + + if (mem_model == null) begin + `uvm_fatal("SCB", "mem_model handle not set") + end + mem_model.clear_mem(); + for (i = 0; i < clone.num_src_words; i++) begin + mem_model.load_word_g(int'(clone.op1_bank), int'(clone.op1_group + i / clone.iter), + i % clone.iter, clone.src_words[i]); + end + for (i = 0; i < clone.num_scale_words; i++) begin + mem_model.load_word_g(int'(clone.op2_bank), int'(clone.op2_group + i / 4), i % 4, + clone.scale_words[i]); + end + mem_model.arm(); + + expected_writes = clone.num_dst_words; + expected_reads[0] = clone.iter * clone.op1_col; + expected_reads[1] = clone.iter * clone.op1_col; + endfunction + + function void write_cmd(bb_blink_cmd_item item); + int2fp_cmd_item stim; + stim = current_stim("CMD"); + check_cmd(item, stim); + cmd_count++; + endfunction + + function void check_cmd(bb_blink_cmd_item got, int2fp_cmd_item exp); + // Compare as plain BLink commands; the comparer rejects differing types. + bb_blink_cmd_item want = bb_blink_cmd_item::type_id::create("want"); + want.copy(exp); + verification_pkg::evidence::checks++; + if (!got.compare(want)) + `uvm_fatal("CMD", $sformatf("command mismatch: got %s exp %s", got.sprint(), want.sprint())) + endfunction + + function void write_read(bb_blink_read_item item); + int2fp_cmd_item stim; + int unsigned expect_bank; + int unsigned expect_group; + int unsigned expect_addr; + + stim = current_stim("READ"); + if (item.port != 0 && item.port != 1) + `uvm_fatal("READ", $sformatf("Int2FpBall has two read ports, got port %0d", item.port)) + if (item.rob_id !== stim.rob_id) + `uvm_fatal("READ", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + + if (item.port == 0) begin + expect_bank = stim.op1_bank; + expect_group = stim.op1_group + read_count[0] / stim.iter; + expect_addr = read_count[0] % stim.iter; + end else begin + expect_bank = stim.op2_bank; + expect_group = stim.op2_group + read_count[1] / stim.iter; + expect_addr = (read_count[1] % stim.iter) & 32'h3; + end + if (item.bank_id !== expect_bank[BB_BLINK_BANK_ID_W-1:0]) + `uvm_fatal("READ", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, expect_bank)) + if (item.group_id !== expect_group[BB_BLINK_GROUP_ID_W-1:0]) + `uvm_fatal("READ", $sformatf("group mismatch: got %0d exp %0d", item.group_id, expect_group)) + if (item.addr !== expect_addr[BB_BLINK_BANK_ADDR_W-1:0]) + `uvm_fatal("READ", $sformatf("addr mismatch: got %0d exp %0d", item.addr, expect_addr)) + read_count[item.port]++; + endfunction + + function void write_write(bb_blink_write_item item); + int2fp_cmd_item stim; + + stim = current_stim("WRITE"); + if (item.port != 0) + `uvm_fatal("WRITE", $sformatf("Int2FpBall has one write port, got port %0d", item.port)) + if (item.bank_id !== stim.wr_bank) + `uvm_fatal("WRITE", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.wr_bank)) + if (item.rob_id !== stim.rob_id) + `uvm_fatal("WRITE", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + if (item.group_id !== stim.wr_group + write_count / stim.iter) + `uvm_fatal( + "WRITE", $sformatf( + "group mismatch: got %0d exp %0d", item.group_id, stim.wr_group + write_count / stim.iter + )) + if (item.mask !== 16'hFFFF) + `uvm_fatal("WRITE", $sformatf("mask mismatch: got 0x%04h", item.mask)) + if (write_count >= stim.num_dst_words) + `uvm_fatal("WRITE", $sformatf("extra write %0d", write_count)) + if (item.addr !== write_count % stim.iter) + `uvm_fatal("WRITE", $sformatf( + "addr mismatch: got %0d exp %0d", item.addr, write_count % stim.iter)) + if (item.data !== stim.dst_words[write_count]) + `uvm_fatal("SCB", $sformatf( + "data mismatch at addr %0d: got 0x%032h exp 0x%032h", + item.addr, + item.data, + stim.dst_words[write_count] + )) + write_count++; + endfunction + + function void write_resp(bb_blink_resp_item item); + int2fp_cmd_item stim; + + stim = current_stim("RESP"); + if (item.rob_id !== stim.rob_id) + `uvm_fatal("RESP", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + resp_count++; + endfunction + + function int2fp_cmd_item current_stim(string tag); + if (stim_q.size() == 0) begin + `uvm_fatal("SCB", $sformatf("%s observed before stimulus", tag)) + return null; + end + return stim_q[0]; + endfunction + + function bit done(); + return cmd_count == 1 && + read_count[0] == expected_reads[0] && + read_count[1] == expected_reads[1] && + write_count == expected_writes && + resp_count == 1; + endfunction + + function void reset_counters(); + stim_q.delete(); + cmd_count = 0; + read_count[0] = 0; + read_count[1] = 0; + write_count = 0; + resp_count = 0; + expected_reads[0] = 0; + expected_reads[1] = 0; + expected_writes = 0; + endfunction + + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (!done()) begin + `uvm_fatal("SCB", $sformatf( + "incomplete: cmds=%0d reads=%0d/%0d,%0d/%0d writes=%0d/%0d resp=%0d", + cmd_count, read_count[0], expected_reads[0], read_count[1], + expected_reads[1], write_count, expected_writes, resp_count)) + end + stim_q.delete(); + endfunction +endclass diff --git a/examples/balls/int2fp/verify/src/pkg/int2fp_pkg.sv b/examples/prefab/balls/int2fp/verify/src/pkg/int2fp_pkg.sv similarity index 100% rename from examples/balls/int2fp/verify/src/pkg/int2fp_pkg.sv rename to examples/prefab/balls/int2fp/verify/src/pkg/int2fp_pkg.sv diff --git a/examples/balls/int2fp/verify/src/seq/int2fp_sequences.svh b/examples/prefab/balls/int2fp/verify/src/seq/int2fp_sequences.svh similarity index 85% rename from examples/balls/int2fp/verify/src/seq/int2fp_sequences.svh rename to examples/prefab/balls/int2fp/verify/src/seq/int2fp_sequences.svh index 0bea8e3e2..53dc6c26e 100644 --- a/examples/balls/int2fp/verify/src/seq/int2fp_sequences.svh +++ b/examples/prefab/balls/int2fp/verify/src/seq/int2fp_sequences.svh @@ -2,7 +2,6 @@ class int2fp_basic_seq extends uvm_sequence #(bb_blink_cmd_item); `uvm_object_utils(int2fp_basic_seq) int unsigned case_index; - int unsigned bid; function new(string name = "int2fp_basic_seq"); super.new(name); @@ -12,7 +11,7 @@ class int2fp_basic_seq extends uvm_sequence #(bb_blink_cmd_item); int2fp_cmd_item item; item = int2fp_cmd_item::type_id::create("item"); start_item(item); - item.load_rust_case(case_index, bid); + item.load_rust_case(case_index); finish_item(item); endtask endclass diff --git a/examples/prefab/balls/int2fp/verify/src/tb_top.sv b/examples/prefab/balls/int2fp/verify/src/tb_top.sv new file mode 100644 index 000000000..6d77594a4 --- /dev/null +++ b/examples/prefab/balls/int2fp/verify/src/tb_top.sv @@ -0,0 +1,89 @@ +module tb_top; + import uvm_pkg::*; + import int2fp_pkg::*; + + bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); + + Int2FpBall dut ( + .clock(intf.clock), + .reset(intf.reset), + .blink_status_running(intf.status_running), + .blink_cmdReq_ready(intf.cmd_req_ready), + .blink_cmdReq_valid(intf.cmd_req_valid), + .blink_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .blink_cmdReq_bits_funct(intf.cmd_req_bits_funct), + .blink_cmdReq_bits_rs2(intf.cmd_req_bits_rs2), + .blink_cmdReq_bits_iter(intf.cmd_req_bits_iter), + .blink_cmdReq_bits_op1_bank(intf.cmd_req_bits_op1_bank), + .blink_cmdReq_bits_op2_bank(intf.cmd_req_bits_op2_bank), + .blink_cmdReq_bits_wr_bank(intf.cmd_req_bits_wr_bank), + .blink_cmdReq_bits_op1_group(intf.cmd_req_bits_op1_group), + .blink_cmdReq_bits_op2_group(intf.cmd_req_bits_op2_group), + .blink_cmdReq_bits_wr_group(intf.cmd_req_bits_wr_group), + .blink_cmdReq_bits_op1_col(intf.cmd_req_bits_op1_col), + .blink_cmdReq_bits_op2_col(intf.cmd_req_bits_op2_col), + .blink_cmdReq_bits_wr_col(intf.cmd_req_bits_wr_col), + .blink_cmdReq_bits_wr_read_bank(intf.cmd_req_bits_wr_read_bank), + .blink_cmdReq_bits_wr_arch_bank(intf.cmd_req_bits_wr_arch_bank), + .blink_cmdResp_ready(intf.cmd_resp_ready), + .blink_cmdResp_valid(intf.cmd_resp_valid), + .blink_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .blink_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .blink_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .blink_bankRead_0_group_id(intf.bank_read_group_id[0]), + .blink_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .blink_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .blink_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .blink_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .blink_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .blink_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .blink_bankRead_1_bank_id(intf.bank_read_bank_id[1]), + .blink_bankRead_1_rob_id(intf.bank_read_rob_id[1]), + .blink_bankRead_1_group_id(intf.bank_read_group_id[1]), + .blink_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), + .blink_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), + .blink_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), + .blink_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), + .blink_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), + .blink_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), + .blink_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .blink_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .blink_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .blink_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .blink_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .blink_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .blink_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .blink_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .blink_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .blink_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]) + ); + + // Constant outputs removed from the port list by firtool. + assign intf.bank_write_req_mask[0][0] = dut.blink_bankWrite_0_io_req_bits_mask_0; + assign intf.bank_write_req_mask[0][1] = dut.blink_bankWrite_0_io_req_bits_mask_1; + assign intf.bank_write_req_mask[0][2] = dut.blink_bankWrite_0_io_req_bits_mask_2; + assign intf.bank_write_req_mask[0][3] = dut.blink_bankWrite_0_io_req_bits_mask_3; + assign intf.bank_write_req_mask[0][4] = dut.blink_bankWrite_0_io_req_bits_mask_4; + assign intf.bank_write_req_mask[0][5] = dut.blink_bankWrite_0_io_req_bits_mask_5; + assign intf.bank_write_req_mask[0][6] = dut.blink_bankWrite_0_io_req_bits_mask_6; + assign intf.bank_write_req_mask[0][7] = dut.blink_bankWrite_0_io_req_bits_mask_7; + assign intf.bank_write_req_mask[0][8] = dut.blink_bankWrite_0_io_req_bits_mask_8; + assign intf.bank_write_req_mask[0][9] = dut.blink_bankWrite_0_io_req_bits_mask_9; + assign intf.bank_write_req_mask[0][10] = dut.blink_bankWrite_0_io_req_bits_mask_10; + assign intf.bank_write_req_mask[0][11] = dut.blink_bankWrite_0_io_req_bits_mask_11; + assign intf.bank_write_req_mask[0][12] = dut.blink_bankWrite_0_io_req_bits_mask_12; + assign intf.bank_write_req_mask[0][13] = dut.blink_bankWrite_0_io_req_bits_mask_13; + assign intf.bank_write_req_mask[0][14] = dut.blink_bankWrite_0_io_req_bits_mask_14; + assign intf.bank_write_req_mask[0][15] = dut.blink_bankWrite_0_io_req_bits_mask_15; + + initial begin + intf.clock = 1'b0; + intf.reset = 1'b1; + forever #5 intf.clock = ~intf.clock; + end + + initial begin + uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); + run_test("int2fp_ball_test"); + end +endmodule diff --git a/examples/balls/int2fp/verify/src/tests/int2fp_test.svh b/examples/prefab/balls/int2fp/verify/src/tests/int2fp_test.svh similarity index 90% rename from examples/balls/int2fp/verify/src/tests/int2fp_test.svh rename to examples/prefab/balls/int2fp/verify/src/tests/int2fp_test.svh index 1806dbfa8..31961b3fc 100644 --- a/examples/balls/int2fp/verify/src/tests/int2fp_test.svh +++ b/examples/prefab/balls/int2fp/verify/src/tests/int2fp_test.svh @@ -17,20 +17,18 @@ class int2fp_ball_test extends uvm_test; endfunction task run_phase(uvm_phase phase); - int unsigned bid = int2fp_require_bid(); phase.raise_objection(this); - for (int unsigned index = 0; index < INT2FP_NUM_CASES; index++) run_case(index, bid); + for (int unsigned index = 0; index < INT2FP_NUM_CASES; index++) run_case(index); phase.drop_objection(this); endtask - task run_case(int unsigned index, int unsigned bid); + task run_case(int unsigned index); int2fp_basic_seq seq; int cycles = 0; apply_reset(); env.scb.reset_counters(); seq = int2fp_basic_seq::type_id::create("seq"); seq.case_index = index; - seq.bid = bid; seq.start(env.cmd_agent.seqr); while (!env.scb.done()) begin @(posedge vif.clock); diff --git a/examples/prefab/balls/int2fp/workloads/ctests/CMakeLists.txt b/examples/prefab/balls/int2fp/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..a4e28a25f --- /dev/null +++ b/examples/prefab/balls/int2fp/workloads/ctests/CMakeLists.txt @@ -0,0 +1,7 @@ +add_buckyball_ctest(int2fp_tensor_values_test.c) +add_buckyball_ctest(int2fp_tensor_groups_test.c) +add_buckyball_ctest(int2fp_tensor_rows_test.c) +add_buckyball_ctest(int2fp_channel_lanes_test.c) +add_buckyball_ctest(int2fp_channel_base_test.c) +add_buckyball_ctest(int2fp_channel_two_rows_test.c) +add_buckyball_ctest(int2fp_groups_test.c) diff --git a/examples/balls/int2fp/workloads/ctests/int2fp_channel_base_test.c b/examples/prefab/balls/int2fp/workloads/ctests/int2fp_channel_base_test.c similarity index 96% rename from examples/balls/int2fp/workloads/ctests/int2fp_channel_base_test.c rename to examples/prefab/balls/int2fp/workloads/ctests/int2fp_channel_base_test.c index 44b9f7cf2..424c1ea30 100644 --- a/examples/balls/int2fp/workloads/ctests/int2fp_channel_base_test.c +++ b/examples/prefab/balls/int2fp/workloads/ctests/int2fp_channel_base_test.c @@ -18,8 +18,8 @@ static const float expected[16] = { }; int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif bb_mem_alloc(0, 4, 1); bb_mem_alloc(1, 4, 1); diff --git a/examples/balls/int2fp/workloads/ctests/int2fp_channel_lanes_test.c b/examples/prefab/balls/int2fp/workloads/ctests/int2fp_channel_lanes_test.c similarity index 96% rename from examples/balls/int2fp/workloads/ctests/int2fp_channel_lanes_test.c rename to examples/prefab/balls/int2fp/workloads/ctests/int2fp_channel_lanes_test.c index 438345cad..844242f82 100644 --- a/examples/balls/int2fp/workloads/ctests/int2fp_channel_lanes_test.c +++ b/examples/prefab/balls/int2fp/workloads/ctests/int2fp_channel_lanes_test.c @@ -18,8 +18,8 @@ static const float expected[16] = { }; int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif bb_mem_alloc(0, 4, 1); bb_mem_alloc(1, 4, 1); diff --git a/examples/balls/int2fp/workloads/ctests/int2fp_channel_two_rows_test.c b/examples/prefab/balls/int2fp/workloads/ctests/int2fp_channel_two_rows_test.c similarity index 96% rename from examples/balls/int2fp/workloads/ctests/int2fp_channel_two_rows_test.c rename to examples/prefab/balls/int2fp/workloads/ctests/int2fp_channel_two_rows_test.c index 972ee8735..32a70656c 100644 --- a/examples/balls/int2fp/workloads/ctests/int2fp_channel_two_rows_test.c +++ b/examples/prefab/balls/int2fp/workloads/ctests/int2fp_channel_two_rows_test.c @@ -20,8 +20,8 @@ static const float expected[32] = { }; int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif bb_mem_alloc(0, 8, 1); bb_mem_alloc(1, 4, 1); diff --git a/examples/prefab/balls/int2fp/workloads/ctests/int2fp_groups_test.c b/examples/prefab/balls/int2fp/workloads/ctests/int2fp_groups_test.c new file mode 100644 index 000000000..09161fb18 --- /dev/null +++ b/examples/prefab/balls/int2fp/workloads/ctests/int2fp_groups_test.c @@ -0,0 +1,49 @@ +#include "buckyball.h" +#include +#include +#include +#include + +/* INT32_TO_FP32 with ReLU over two channel groups: group g of the input, + * scale, and output banks is channel block g; every group uses the same rows. + */ +enum { GROUPS = 2, ROWS = 8 }; + +static int32_t input[GROUPS][ROWS * 4] __attribute__((aligned(64))); +static float scales[GROUPS][16] __attribute__((aligned(64))); +static float output[GROUPS][ROWS * 4] __attribute__((aligned(64))); + +int main(void) { + for (int group = 0; group < GROUPS; ++group) { + for (int i = 0; i < 16; ++i) + scales[group][i] = (float)(1 << ((group + i) % 3)) * 0.5f; + for (int i = 0; i < ROWS * 4; ++i) + input[group][i] = (group * 13 + i * 5) % 40 - 15; + } + + bb_mem_alloc(0, 1, GROUPS); + bb_mem_alloc(1, 1, GROUPS); + bb_mem_alloc(2, 1, GROUPS); + for (int group = 0; group < GROUPS; ++group) { + bb_mvin_group((uintptr_t)input[group], 0, group, ROWS, 1); + bb_mvin_group((uintptr_t)scales[group], 1, group, 4, 1); + } + bb_int32_to_fp32(0, 1, 2, ROWS, 1); + for (int group = 0; group < GROUPS; ++group) + bb_mvout_group((uintptr_t)output[group], 2, group, ROWS, 1); + + for (int group = 0; group < GROUPS; ++group) + for (int i = 0; i < ROWS * 4; ++i) { + int value = input[group][i] < 0 ? 0 : input[group][i]; + float expected = (float)value * scales[group][i % 16]; + if (output[group][i] != expected) { + printf("int2fp_groups FAIL group=%d index=%d\n", group, i); + return 1; + } + } + bb_mem_release(0); + bb_mem_release(1); + bb_mem_release(2); + printf("int2fp_groups PASS\n"); + return 0; +} diff --git a/examples/balls/int2fp/workloads/ctests/int2fp_tensor_groups_test.c b/examples/prefab/balls/int2fp/workloads/ctests/int2fp_tensor_groups_test.c similarity index 96% rename from examples/balls/int2fp/workloads/ctests/int2fp_tensor_groups_test.c rename to examples/prefab/balls/int2fp/workloads/ctests/int2fp_tensor_groups_test.c index 69f38c09e..f35663ecd 100644 --- a/examples/balls/int2fp/workloads/ctests/int2fp_tensor_groups_test.c +++ b/examples/prefab/balls/int2fp/workloads/ctests/int2fp_tensor_groups_test.c @@ -18,8 +18,8 @@ static const float expected[16] = { }; int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif bb_mem_alloc(0, 4, 1); bb_mem_alloc(1, 4, 1); diff --git a/examples/balls/int2fp/workloads/ctests/int2fp_tensor_rows_test.c b/examples/prefab/balls/int2fp/workloads/ctests/int2fp_tensor_rows_test.c similarity index 96% rename from examples/balls/int2fp/workloads/ctests/int2fp_tensor_rows_test.c rename to examples/prefab/balls/int2fp/workloads/ctests/int2fp_tensor_rows_test.c index ee07f978e..573837d41 100644 --- a/examples/balls/int2fp/workloads/ctests/int2fp_tensor_rows_test.c +++ b/examples/prefab/balls/int2fp/workloads/ctests/int2fp_tensor_rows_test.c @@ -15,8 +15,8 @@ static const float expected[8] = {1.0f, -1.0f, 2.0f, -2.0f, 3.0f, -3.0f, 4.0f, -4.0f}; int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif bb_mem_alloc(0, 4, 1); bb_mem_alloc(1, 4, 1); diff --git a/examples/balls/int2fp/workloads/ctests/int2fp_tensor_values_test.c b/examples/prefab/balls/int2fp/workloads/ctests/int2fp_tensor_values_test.c similarity index 96% rename from examples/balls/int2fp/workloads/ctests/int2fp_tensor_values_test.c rename to examples/prefab/balls/int2fp/workloads/ctests/int2fp_tensor_values_test.c index eb3769194..eb63ed438 100644 --- a/examples/balls/int2fp/workloads/ctests/int2fp_tensor_values_test.c +++ b/examples/prefab/balls/int2fp/workloads/ctests/int2fp_tensor_values_test.c @@ -23,8 +23,8 @@ static const uint32_t expected[4] = { }; int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif bb_mem_alloc(0, 4, 1); bb_mem_alloc(1, 4, 1); diff --git a/examples/prefab/balls/int2fp/workloads/isa/int2fp.h b/examples/prefab/balls/int2fp/workloads/isa/int2fp.h new file mode 100644 index 000000000..542714e21 --- /dev/null +++ b/examples/prefab/balls/int2fp/workloads/isa/int2fp.h @@ -0,0 +1,18 @@ +#ifndef _BB_INT2FP_H_ +#define _BB_INT2FP_H_ + +#include +#include + +// Group g of every operand bank is channel block g (channels 16g..16g+15). +// Input, scale, and output banks have the same number of groups; every group +// uses the same rows. In group g, input row r holds 4 INT32 lanes, scale row +// r % 4 holds their scales, and output row r holds the 4 FP32 results. + +#define bb_int32_to_fp32(input_bank, scale_bank, output_bank, iter, relu) \ + BUCKYBALL_INSTRUCTION_R_R(BB_BANK0(input_bank) | BB_BANK1(scale_bank) | \ + BB_BANK2(output_bank) | BB_ITER(iter), \ + FIELD((uint64_t)(relu), 0, 0), \ + FUNCT7(int2fp, INT32_TO_FP32)) + +#endif // _BB_INT2FP_H_ diff --git a/examples/balls/int2fp/workloads/mlir_tests/CMakeLists.txt b/examples/prefab/balls/int2fp/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/balls/int2fp/workloads/mlir_tests/CMakeLists.txt rename to examples/prefab/balls/int2fp/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/balls/int2fp/workloads/mlir_tests/ball/CMakeLists.txt b/examples/prefab/balls/int2fp/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/balls/int2fp/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/prefab/balls/int2fp/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/prefab/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4.mlir b/examples/prefab/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4.mlir new file mode 100644 index 000000000..6b78184a3 --- /dev/null +++ b/examples/prefab/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4.mlir @@ -0,0 +1,28 @@ +func.func private @check_result(memref<4x4xf32>) -> () + +memref.global "private" constant @input : memref<4x4xi32> = dense<[ + [1, 2, 3, -1], [-2, 0, 4, 5], + [10, -10, 7, 100], [-100, 8, 16, -8] +]> +memref.global "private" constant @scales : memref<4x4xf32> = + dense<1.250000e-01> + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %input = memref.get_global @input : memref<4x4xi32> + %scales = memref.get_global @scales : memref<4x4xf32> + %output = memref.alloc() alignment = 64 : memref<4x4xf32> + buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %input <{bank = 1 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xi32> + buckyball.mvin %scales <{bank = 2 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> + int2fp_ball.int32_to_fp32 <{values = 1 : i64, scales = 2 : i64, output = 3 : i64, rows = 1 : i64}> + buckyball.mvout %output <{bank = 3 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> + func.call @check_result(%output) : (memref<4x4xf32>) -> () + buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + memref.dealloc %output : memref<4x4xf32> + return %zero_i8 : i8 +} diff --git a/examples/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4_main.cpp b/examples/prefab/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4_main.cpp similarity index 100% rename from examples/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4_main.cpp rename to examples/prefab/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4_main.cpp diff --git a/examples/prefab/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4.mlir b/examples/prefab/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4.mlir new file mode 100644 index 000000000..43295b799 --- /dev/null +++ b/examples/prefab/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4.mlir @@ -0,0 +1,27 @@ +func.func private @check_result(memref<4x4xf32>) -> () + +memref.global "private" constant @input : memref<4x4xi32> = dense<[ + [-8, -1, 0, 1], [2, -3, 4, -5], + [6, 7, -8, 9], [-10, 11, 12, -13] +]> +memref.global "private" constant @scales : memref<4x4xf32> = dense<1.0> + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %input = memref.get_global @input : memref<4x4xi32> + %scales = memref.get_global @scales : memref<4x4xf32> + %output = memref.alloc() alignment = 64 : memref<4x4xf32> + buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %input <{bank = 1 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xi32> + buckyball.mvin %scales <{bank = 2 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> + int2fp_ball.int32_to_fp32 <{values = 1 : i64, scales = 2 : i64, output = 3 : i64, rows = 1 : i64, relu}> + buckyball.mvout %output <{bank = 3 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> + func.call @check_result(%output) : (memref<4x4xf32>) -> () + buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + memref.dealloc %output : memref<4x4xf32> + return %zero_i8 : i8 +} diff --git a/examples/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4_main.cpp b/examples/prefab/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4_main.cpp similarity index 100% rename from examples/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4_main.cpp rename to examples/prefab/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4_main.cpp diff --git a/examples/prefab/balls/int2fp/workloads/mlir_tests/bank/CMakeLists.txt b/examples/prefab/balls/int2fp/workloads/mlir_tests/bank/CMakeLists.txt new file mode 100644 index 000000000..e33d64409 --- /dev/null +++ b/examples/prefab/balls/int2fp/workloads/mlir_tests/bank/CMakeLists.txt @@ -0,0 +1,9 @@ +if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) + set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) +endif() +set(BUCKYBALL_MLIR_TEST_PREFIX bank) + +set(_lines ${DESIGN_PROFILES_${BUCKYBALL_MLIR_ACTIVE_TARGET}_THREAD_MEM_BANK_ENTRIES}) +# int32_to_fp32 rows hold sixteen i32, four bank rows each. +math(EXPR BB_INT2FP_ROWS "${_lines} / 4") +add_buckyball_mlir_test(int2fp_unit_scale TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/prefab/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4.mlir b/examples/prefab/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4.mlir new file mode 100644 index 000000000..88c67b989 --- /dev/null +++ b/examples/prefab/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4.mlir @@ -0,0 +1,28 @@ +func.func private @check_result(memref<4x4xf32>) -> () + +memref.global "private" constant @input : memref<4x4xi32> = dense<[ + [1, 2, 3, -1], [-2, 0, 4, 5], + [10, -10, 7, 100], [-100, 8, 16, -8] +]> +memref.global "private" constant @scales : memref<4x4xf32> = + dense<1.250000e-01> + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %input = memref.get_global @input : memref<4x4xi32> + %scales = memref.get_global @scales : memref<4x4xf32> + %output = memref.alloc() alignment = 64 : memref<4x4xf32> + %input_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %scale_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %output_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %loaded_input = buckyball.bank_mvin %input, %input_bank <{depth = 4 : i64, stride = 1 : i64}> : memref<4x4xi32> + %loaded_scales = buckyball.bank_mvin %scales, %scale_bank <{depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> + %result = int2fp_ball.bank_int32_to_fp32 %loaded_input, %loaded_scales -> %output_bank <{rows = 1 : i64}> + %stored = buckyball.bank_mvout %output, %result <{depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> + func.call @check_result(%output) : (memref<4x4xf32>) -> () + buckyball.bank_release %loaded_input + buckyball.bank_release %loaded_scales + buckyball.bank_release %stored + memref.dealloc %output : memref<4x4xf32> + return %zero_i8 : i8 +} diff --git a/examples/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4_main.cpp b/examples/prefab/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4_main.cpp similarity index 100% rename from examples/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4_main.cpp rename to examples/prefab/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4_main.cpp diff --git a/examples/prefab/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4.mlir b/examples/prefab/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4.mlir new file mode 100644 index 000000000..dd8aeee10 --- /dev/null +++ b/examples/prefab/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4.mlir @@ -0,0 +1,27 @@ +func.func private @check_result(memref<4x4xf32>) -> () + +memref.global "private" constant @input : memref<4x4xi32> = dense<[ + [-8, -1, 0, 1], [2, -3, 4, -5], + [6, 7, -8, 9], [-10, 11, 12, -13] +]> +memref.global "private" constant @scales : memref<4x4xf32> = dense<1.0> + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %input = memref.get_global @input : memref<4x4xi32> + %scales = memref.get_global @scales : memref<4x4xf32> + %output = memref.alloc() alignment = 64 : memref<4x4xf32> + %input_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %scale_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %output_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %loaded_input = buckyball.bank_mvin %input, %input_bank <{depth = 4 : i64, stride = 1 : i64}> : memref<4x4xi32> + %loaded_scales = buckyball.bank_mvin %scales, %scale_bank <{depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> + %result = int2fp_ball.bank_int32_to_fp32 %loaded_input, %loaded_scales -> %output_bank <{rows = 1 : i64, relu}> + %stored = buckyball.bank_mvout %output, %result <{depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> + func.call @check_result(%output) : (memref<4x4xf32>) -> () + buckyball.bank_release %loaded_input + buckyball.bank_release %loaded_scales + buckyball.bank_release %stored + memref.dealloc %output : memref<4x4xf32> + return %zero_i8 : i8 +} diff --git a/examples/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4_main.cpp b/examples/prefab/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4_main.cpp similarity index 100% rename from examples/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4_main.cpp rename to examples/prefab/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4_main.cpp diff --git a/examples/prefab/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale.mlir.in b/examples/prefab/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale.mlir.in new file mode 100644 index 000000000..b99e22b34 --- /dev/null +++ b/examples/prefab/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale.mlir.in @@ -0,0 +1,36 @@ +func.func private @check_result(memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xf32>) -> () +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %rows = arith.constant @BB_BANK_LINES@ : index + %lanes = arith.constant @BB_BANK_I32_LANES@ : index + %input = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + %scale = memref.alloc() : memref<4x4xf32> + %output = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xf32> + %one_f32 = arith.constant 1.0 : f32 + linalg.fill ins(%one_f32 : f32) outs(%scale : memref<4x4xf32>) + scf.for %row = %zero to %rows step %one { + scf.for %lane = %zero to %lanes step %one { + %r = arith.index_cast %row : index to i32 + %l = arith.index_cast %lane : index to i32 + %value = arith.subi %r, %l : i32 + memref.store %value, %input[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + } + } + %input_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %scale_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %output_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %loaded_input = buckyball.bank_mvin %input, %input_bank <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + %loaded_scale = buckyball.bank_mvin %scale, %scale_bank <{depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> + %result = int2fp_ball.bank_int32_to_fp32 %loaded_input, %loaded_scale -> %output_bank <{rows = @BB_INT2FP_ROWS@ : i64}> + %stored = buckyball.bank_mvout %output, %result <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xf32> + func.call @check_result(%output) : (memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xf32>) -> () + buckyball.bank_release %loaded_input + buckyball.bank_release %loaded_scale + buckyball.bank_release %stored + memref.dealloc %input : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + memref.dealloc %scale : memref<4x4xf32> + memref.dealloc %output : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xf32> + return %zero_i8 : i8 +} diff --git a/examples/prefab/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale_main.cpp b/examples/prefab/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale_main.cpp new file mode 100644 index 000000000..ec0ee48e4 --- /dev/null +++ b/examples/prefab/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale_main.cpp @@ -0,0 +1,17 @@ +#include +#include +#include +extern "C" void check_result(float *, float *data, int64_t offset, int64_t rows, + int64_t lanes, int64_t stride, + int64_t lane_stride) { + if (rows != THREAD_MEM_BANK_ENTRIES || lanes != THREAD_MEM_BANK_WIDTH / 32 || + stride != lanes || lane_stride != 1) + exit(1); + for (int row = 0; row < rows; ++row) + for (int lane = 0; lane < lanes; ++lane) + if (data[offset + row * stride + lane] != (float)(row - lane)) { + printf("int2fp bank mismatch row=%d lane=%d\n", row, lane); + exit(1); + } + printf("int2fp bank PASS rows=%d\n", THREAD_MEM_BANK_ENTRIES); +} diff --git a/examples/prefab/balls/int8add/arch/Int8AddBall.scala b/examples/prefab/balls/int8add/arch/Int8AddBall.scala new file mode 100644 index 000000000..5fdbee567 --- /dev/null +++ b/examples/prefab/balls/int8add/arch/Int8AddBall.scala @@ -0,0 +1,291 @@ +package examples.balls.int8add + +import framework.acc.balldomain.bbus.{BallFactory, BallPorts} +import buckyball.balls.int8add.{Params => ParametersProto} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import chisel3.util._ +import hardfloat.{rawFloatFromRecFN, recFNFromFN, RecFNToIN, RoundRawFNToRecFN} +import hardfloat.consts.{round_near_even, tininess_afterRounding} +import framework.lib.arith.{Fma, IntToFloat} + +import framework.acc.balldomain.ball.blink.{BallStatus, BlinkIO, HasBlink} + +@instantiable +class Int8AddBall( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Module + with HasBlink { + val ball = bp.balls.find(_.id == id).get + + ball.parameters.unpack(classOf[ParametersProto]) + + val addFunct = ball.instructions + .find(_.mnemonic == "INT8ADD").get.funct7 + + val reluFunct = ball.instructions + .find(_.mnemonic == "INT8ADD_RELU").get.funct7 + + require(ball.inBW == 2, "Int8AddBall requires inBW=2") + require(ball.outBW == 1, "Int8AddBall requires outBW=1") + require(mp.bankWidth == 128, "Int8AddBall requires 128-bit bank rows") + require((addFunct >> 4) == 4 && (reluFunct >> 4) == 4, "Int8AddBall instructions must encode two reads and one write") + + @public + val blink = IO(new BlinkIO(fp, mp, bp, id)) + + val idle :: waitForChannels :: readRequest :: readResponse :: calculate :: calculateDrain :: writeRequest :: writeResponse :: complete :: Nil = + Enum(9) + val state = RegInit(idle) + val robId = RegInit(0.U(log2Up(fp.rob_entries).W)) + val lhsBank = RegInit(0.U(fp.bank_id_len.W)) + val rhsBank = RegInit(0.U(fp.bank_id_len.W)) + val outputBank = RegInit(0.U(fp.bank_id_len.W)) + val groups = RegInit(0.U(5.W)) + val group = RegInit(0.U(5.W)) + val bases = Reg(Vec(3, UInt(mp.groupIdWidth.W))) + val iter = RegInit(0.U(fp.iter_len.W)) + val row = RegInit(0.U(log2Ceil(mp.bankEntries).W)) + val lhsRatio = Reg(UInt(32.W)) + val rhsRatio = Reg(UInt(32.W)) + val relu = RegInit(false.B) + val lhsWord = Reg(UInt(128.W)) + val rhsWord = Reg(UInt(128.W)) + val lhsRequested = RegInit(false.B) + val rhsRequested = RegInit(false.B) + val lhsReceived = RegInit(false.B) + val rhsReceived = RegInit(false.B) + val lane = RegInit(0.U(4.W)) + val completed = RegInit(0.U(5.W)) + val outputWord = Reg(Vec(16, UInt(8.W))) + + def positiveFinite(value: UInt): Bool = + !value(31) && value(30, 23) =/= 255.U && value(30, 0) =/= 0.U + + val lhsToFloat = Instantiate(new IntToFloat(8, 8, 24)) + + val rhsToFloat = Instantiate(new IntToFloat(8, 8, 24)) + + val lhsMul = Instantiate(new Fma(3, 8, 24)) + + val rhsMul = Instantiate(new Fma(3, 8, 24)) + + val add = Instantiate(new PipelinedAddRawFN(8, 24)) + + val round = Module(new RoundRawFNToRecFN(8, 24, 0)) + val toInt = Module(new RecFNToIN(8, 24, 8)) + val lhsValue = (lhsWord >> (lane << 3))(7, 0) + val rhsValue = (rhsWord >> (lane << 3))(7, 0) + val calculateIssue = state === calculate + + val ratioStage0 = RegEnable(lhsRatio, calculateIssue) + val ratioStage1 = RegEnable(ratioStage0, RegNext(calculateIssue, false.B)) + val rhsRatioStage0 = RegEnable(rhsRatio, calculateIssue) + val rhsRatioStage1 = RegEnable(rhsRatioStage0, RegNext(calculateIssue, false.B)) + val laneStage0 = RegEnable(lane, calculateIssue) + val laneStage1 = RegEnable(laneStage0, RegNext(calculateIssue, false.B)) + + val conversionValid = lhsToFloat.io.validout + val productLane = Pipe(conversionValid, laneStage1, 3) + + val addResult = add.io.rawOut + val addLane = Pipe(lhsMul.io.validout, productLane.bits, 4) + val addValid = add.io.validout + val roundedResult = RegEnable(round.io.out, addValid) + val roundedLane = RegEnable(addLane, addValid) + val resultValid = RegNext(addValid, false.B) + + lhsToFloat.io.validin := calculateIssue + lhsToFloat.io.signedIn := true.B + lhsToFloat.io.in := lhsValue + lhsToFloat.io.roundingMode := round_near_even + lhsToFloat.io.detectTininess := tininess_afterRounding + rhsToFloat.io.validin := calculateIssue + rhsToFloat.io.signedIn := true.B + rhsToFloat.io.in := rhsValue + rhsToFloat.io.roundingMode := round_near_even + rhsToFloat.io.detectTininess := tininess_afterRounding + + lhsMul.io.validin := conversionValid + lhsMul.io.op := 0.U + lhsMul.io.a := lhsToFloat.io.out + lhsMul.io.b := recFNFromFN(8, 24, ratioStage1) + lhsMul.io.c := recFNFromFN(8, 24, 0.U) + lhsMul.io.roundingMode := round_near_even + lhsMul.io.detectTininess := tininess_afterRounding.asUInt + rhsMul.io.validin := conversionValid + rhsMul.io.op := 0.U + rhsMul.io.a := rhsToFloat.io.out + rhsMul.io.b := recFNFromFN(8, 24, rhsRatioStage1) + rhsMul.io.c := recFNFromFN(8, 24, 0.U) + rhsMul.io.roundingMode := round_near_even + rhsMul.io.detectTininess := tininess_afterRounding.asUInt + + add.io.validin := lhsMul.io.validout + add.io.subOp := false.B + add.io.a := rawFloatFromRecFN(8, 24, lhsMul.io.out) + add.io.b := rawFloatFromRecFN(8, 24, rhsMul.io.out) + add.io.roundingMode := round_near_even + round.io.invalidExc := add.io.invalidExc + round.io.infiniteExc := false.B + round.io.in := addResult + round.io.roundingMode := round_near_even + round.io.detectTininess := tininess_afterRounding + toInt.io.in := roundedResult + toInt.io.roundingMode := round_near_even + toInt.io.signedOut := true.B + val result = Mux(relu && toInt.io.out(7), 0.U, toInt.io.out) + + blink.cmdReq.ready := state === idle + blink.cmdResp.valid := state === complete + blink.cmdResp.bits.rob_id := robId + blink.status.idle := state === idle + blink.status.running := state =/= idle && state =/= complete + + for (port <- 0 until 2) { + blink.bankRead(port).rob_id := robId + blink.bankRead(port).bank_id := Mux(port.U === 0.U, lhsBank, rhsBank) + blink.bankRead(port).group_id := bases(port) + group + blink.bankRead(port).io.req.valid := false.B + blink.bankRead(port).io.req.bits.addr := row + blink.bankRead(port).io.resp.ready := false.B + } + blink.bankWrite(0).rob_id := robId + blink.bankWrite(0).bank_id := outputBank + blink.bankWrite(0).group_id := bases(2) + group + blink.bankWrite(0).io.req.valid := false.B + blink.bankWrite(0).io.req.bits.addr := row + blink.bankWrite(0).io.req.bits.data := Cat(outputWord.reverse) + blink.bankWrite(0).io.req.bits.mask := VecInit(Seq.fill(16)(true.B)) + blink.bankWrite(0).io.resp.ready := false.B + + switch(state) { + is(idle) { + when(blink.cmdReq.fire) { + val cmd = blink.cmdReq.bits + assert( + cmd.funct === addFunct.U || cmd.funct === reluFunct.U, + "Int8AddBall received an unknown funct7" + ) + assert( + positiveFinite(cmd.rs2(31, 0)) && positiveFinite(cmd.rs2(63, 32)), + "Int8AddBall ratios must be finite and positive" + ) + assert( + cmd.op1_col =/= 0.U && cmd.op1_col === cmd.op2_col && cmd.op1_col === cmd.wr_col, + "Int8AddBall bank groups must match" + ) + assert( + cmd.op1_col <= mp.bankNum.U, + "Int8AddBall bank groups exceed physical banks" + ) + assert( + cmd.op1_bank =/= cmd.op2_bank && cmd.op1_bank =/= cmd.wr_bank && cmd.op2_bank =/= cmd.wr_bank, + "Int8AddBall banks must be distinct" + ) + assert( + cmd.iter =/= 0.U && cmd.iter <= mp.bankEntries.U, + "Int8AddBall iter must fit one physical bank" + ) + robId := blink.cmdReq.bits.rob_id + lhsBank := cmd.op1_bank + rhsBank := cmd.op2_bank + outputBank := cmd.wr_bank + groups := cmd.op1_col + group := 0.U + bases := VecInit(cmd.op1_group, cmd.op2_group, cmd.wr_group) + iter := cmd.iter + row := 0.U + lhsRatio := cmd.rs2(31, 0) + rhsRatio := cmd.rs2(63, 32) + relu := cmd.funct === reluFunct.U + lhsRequested := false.B + rhsRequested := false.B + lhsReceived := false.B + rhsReceived := false.B + lane := 0.U + state := waitForChannels + } + } + is(waitForChannels) { + state := readRequest + } + is(readRequest) { + blink.bankRead(0).io.req.valid := !lhsRequested + blink.bankRead(1).io.req.valid := !rhsRequested + when(blink.bankRead(0).io.req.fire)(lhsRequested := true.B) + when(blink.bankRead(1).io.req.fire)(rhsRequested := true.B) + when((lhsRequested || blink.bankRead(0).io.req.fire) && (rhsRequested || blink.bankRead(1).io.req.fire)) { + state := readResponse + } + } + is(readResponse) { + blink.bankRead(0).io.resp.ready := !lhsReceived + blink.bankRead(1).io.resp.ready := !rhsReceived + when(blink.bankRead(0).io.resp.fire) { lhsWord := blink.bankRead(0).io.resp.bits.data; lhsReceived := true.B } + when(blink.bankRead(1).io.resp.fire) { rhsWord := blink.bankRead(1).io.resp.bits.data; rhsReceived := true.B } + when((lhsReceived || blink.bankRead(0).io.resp.fire) && (rhsReceived || blink.bankRead(1).io.resp.fire)) { + lane := 0.U + completed := 0.U + state := calculate + } + } + is(calculate) { + when(lane === 15.U) { + state := calculateDrain + }.otherwise { + lane := lane + 1.U + } + } + is(calculateDrain) {} + is(writeRequest) { + blink.bankWrite(0).io.req.valid := true.B + when(blink.bankWrite(0).io.req.fire)(state := writeResponse) + } + is(writeResponse) { + blink.bankWrite(0).io.resp.ready := true.B + when(blink.bankWrite(0).io.resp.fire) { + lhsRequested := false.B + rhsRequested := false.B + lhsReceived := false.B + rhsReceived := false.B + when(row +& 1.U === iter) { + row := 0.U + when(group +& 1.U === groups)(state := complete) + .otherwise { group := group + 1.U; state := readRequest } + }.otherwise { row := row + 1.U; state := readRequest } + } + } + is(complete) { + when(blink.cmdResp.fire)(state := idle) + } + } + + when((state === calculate || state === calculateDrain) && resultValid) { + outputWord(roundedLane.bits) := result + completed := completed + 1.U + when(completed === 15.U) { + state := writeRequest + } + } +} + +object Int8AddBall extends BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts = + BallPorts(Instantiate(new Int8AddBall(fp, mp, bp, id)).blink, None) + +} diff --git a/examples/prefab/balls/int8add/arch/PipelinedAddRawFN.scala b/examples/prefab/balls/int8add/arch/PipelinedAddRawFN.scala new file mode 100644 index 000000000..afab4d310 --- /dev/null +++ b/examples/prefab/balls/int8add/arch/PipelinedAddRawFN.scala @@ -0,0 +1,171 @@ +package examples.balls.int8add + +import chisel3.experimental.hierarchy.{instantiable, public} + +import chisel3._ +import chisel3.util._ +import hardfloat.{countLeadingZeros, isSigNaNRawFloat, lowMask, orReduceBy2, orReduceBy4, RawFloat} +import hardfloat.consts.round_min + +/** + * Throughput-oriented raw floating-point adder used by Int8AddBall. + * + * The Berkeley HardFloat AddRawFN implementation is combinational. This + * wrapper keeps the same raw-float equations but registers the alignment, + * normalization, and final select boundaries. It accepts one operand pair + * per cycle and reports the result four cycles later. + */ +@instantiable +class PipelinedAddRawFN(val expWidth: Int, val sigWidth: Int) extends Module { + + @public + val io = IO(new Bundle { + val validin = Input(Bool()) + val subOp = Input(Bool()) + val a = Input(new RawFloat(expWidth, sigWidth)) + val b = Input(new RawFloat(expWidth, sigWidth)) + val roundingMode = Input(UInt(3.W)) + val validout = Output(Bool()) + val invalidExc = Output(Bool()) + val rawOut = Output(new RawFloat(expWidth, sigWidth + 2)) + }) + + val alignDistWidth = log2Ceil(sigWidth) + + // Stage 0: register the raw operands and exponent comparison. + val a0 = RegNext(io.a) + val b0 = RegNext(io.b) + val effSignB0 = RegNext(io.b.sign ^ io.subOp) + val eqSigns0 = RegNext(io.a.sign === (io.b.sign ^ io.subOp)) + val sDiffExps0 = RegNext(io.a.sExp - io.b.sExp) + val roundingMode0 = RegNext(io.roundingMode) + + // Stage 1: prepare close-path subtraction and far-path operand selection. + val modNatAlignDist1 = Mux(sDiffExps0 < 0.S, -sDiffExps0, sDiffExps0)(alignDistWidth - 1, 0) + + val isMaxAlign1 = + (sDiffExps0 >> alignDistWidth) =/= 0.S && + ((sDiffExps0 >> alignDistWidth) =/= -1.S || sDiffExps0(alignDistWidth - 1, 0) === 0.U) + + val alignDist1 = Mux(isMaxAlign1, ((BigInt(1) << alignDistWidth) - 1).U, modNatAlignDist1) + val closeSubMags1 = !eqSigns0 && !isMaxAlign1 && (modNatAlignDist1 <= 1.U) + + val closeAlignedSigA1 = + Mux((0.S <= sDiffExps0) && sDiffExps0(0), a0.sig << 2, 0.U) | + Mux((0.S <= sDiffExps0) && !sDiffExps0(0), a0.sig << 1, 0.U) | + Mux(sDiffExps0 < 0.S, a0.sig, 0.U) + + val closeSSigSum1 = Wire(SInt((sigWidth + 3).W)) + closeSSigSum1 := closeAlignedSigA1.asSInt - (b0.sig << 1).asSInt + val closeSigSum1 = Wire(UInt((sigWidth + 2).W)) + closeSigSum1 := Mux(closeSSigSum1 < 0.S, -closeSSigSum1, closeSSigSum1)(sigWidth + 1, 0) + val closeAdjustedSigSum1 = Wire(UInt((sigWidth + 2).W)) + closeAdjustedSigSum1 := closeSigSum1 << (sigWidth & 1) + val closeReduced2SigSum1 = orReduceBy2(closeAdjustedSigSum1) + + val farSignOut1 = Mux(sDiffExps0 < 0.S, effSignB0, a0.sign) + val farSigLarger1 = Wire(UInt(sigWidth.W)) + farSigLarger1 := Mux(sDiffExps0 < 0.S, b0.sig, a0.sig)(sigWidth - 1, 0) + val farSigSmaller1 = Wire(UInt(sigWidth.W)) + farSigSmaller1 := Mux(sDiffExps0 < 0.S, a0.sig, b0.sig)(sigWidth - 1, 0) + + val a1 = RegNext(a0) + val b1 = RegNext(b0) + val effSignB1 = RegNext(effSignB0) + val eqSigns1 = RegNext(eqSigns0) + val sDiffExps1 = RegNext(sDiffExps0) + val roundingMode1 = RegNext(roundingMode0) + val alignDist1r = RegInit(0.U(alignDistWidth.W)) + alignDist1r := alignDist1 + val closeSubMags1r = RegInit(false.B) + closeSubMags1r := closeSubMags1 + val closeSSigSum1r = RegInit(0.S((sigWidth + 3).W)) + closeSSigSum1r := closeSSigSum1 + val closeSigSum1r = RegInit(0.U((sigWidth + 2).W)) + closeSigSum1r := closeSigSum1 + val closeReduced2SigSum1r = RegInit(0.U(((sigWidth + 3) / 2).W)) + closeReduced2SigSum1r := closeReduced2SigSum1 + val farSignOut1r = RegNext(farSignOut1) + val farSigLarger1r = RegInit(0.U(sigWidth.W)) + farSigLarger1r := farSigLarger1 + val farSigSmaller1r = RegInit(0.U(sigWidth.W)) + farSigSmaller1r := farSigSmaller1 + + // Stage 2: leading-zero normalization and far-path alignment. + val closeNormDistReduced2_2 = countLeadingZeros(closeReduced2SigSum1r) + val closeNearNormDist2 = (closeNormDistReduced2_2 << 1)(alignDistWidth - 1, 0) + val closeSigOut2 = ((closeSigSum1r << closeNearNormDist2) << 1)(sigWidth + 2, 0) + val closeTotalCancellation2 = !(closeSigOut2(sigWidth + 2, sigWidth + 1).orR) + val closeNotTotalCancellationSignOut2 = a1.sign ^ (closeSSigSum1r < 0.S) + + val farMainAlignedSigSmaller2 = Wire(UInt((sigWidth + 5).W)) + farMainAlignedSigSmaller2 := (farSigSmaller1r << 5) >> alignDist1r + val farReduced4SigSmaller2 = Wire(UInt(((sigWidth + 5) / 4).W)) + farReduced4SigSmaller2 := orReduceBy4(Cat(farSigSmaller1r, 0.U(2.W))) + val farRoundExtraMask2 = lowMask(alignDist1r(alignDistWidth - 1, 2), (sigWidth + 5) / 4, 0) + + val farAlignedSigSmaller2 = Cat( + farMainAlignedSigSmaller2 >> 3, + farMainAlignedSigSmaller2(2, 0).orR || (farReduced4SigSmaller2 & farRoundExtraMask2).orR + ) + + val farSubMags2 = !eqSigns1 + + val a2 = RegNext(a1) + val b2 = RegNext(b1) + val effSignB2 = RegNext(effSignB1) + val eqSigns2 = RegNext(eqSigns1) + val sDiffExps2 = RegNext(sDiffExps1) + val roundingMode2 = RegNext(roundingMode1) + val closeSubMags2r = RegNext(closeSubMags1r) + val closeNearNormDist2r = RegNext(closeNearNormDist2) + val closeSigOut2r = RegNext(closeSigOut2) + val closeTotalCancellation2r = RegNext(closeTotalCancellation2) + val closeNotTotalCancellationSignOut2r = RegNext(closeNotTotalCancellationSignOut2) + val farSignOut2r = RegNext(farSignOut1r) + val farSigLarger2r = RegNext(farSigLarger1r) + val farAlignedSigSmaller2r = RegNext(farAlignedSigSmaller2) + val farSubMags2r = RegNext(farSubMags2) + + // Stage 3: significand add/subtract and special-case selection. + val farNegAlignedSigSmaller3 = Mux( + farSubMags2r, + Cat(1.U, ~farAlignedSigSmaller2r), + farAlignedSigSmaller2r + ) + + val farSigSum3 = (farSigLarger2r << 3) + farNegAlignedSigSmaller3 + farSubMags2r + val farSigOut3 = Mux(farSubMags2r, farSigSum3, (farSigSum3 >> 1) | farSigSum3(0))(sigWidth + 2, 0) + + val notSigNaNInvalidExc3 = a2.isInf && b2.isInf && !eqSigns2 + val notNaNIsInfOut3 = a2.isInf || b2.isInf + val addZeros3 = a2.isZero && b2.isZero + val notNaNSpecialCase3 = notNaNIsInfOut3 || addZeros3 + val notNaNIsZeroOut3 = addZeros3 || (!notNaNIsInfOut3 && closeSubMags2r && closeTotalCancellation2r) + + val notNaNSignOut3 = + (eqSigns2 && a2.sign) || + (a2.isInf && a2.sign) || + (b2.isInf && effSignB2) || + (notNaNIsZeroOut3 && !eqSigns2 && roundingMode2 === round_min) || + (!notNaNSpecialCase3 && closeSubMags2r && !closeTotalCancellation2r && closeNotTotalCancellationSignOut2r) || + (!notNaNSpecialCase3 && !closeSubMags2r && farSignOut2r) + + val commonSExp3 = + Mux(closeSubMags2r || (sDiffExps2 < 0.S), b2.sExp, a2.sExp) - + Mux(closeSubMags2r, closeNearNormDist2r, farSubMags2r).zext + + val commonSigOut3 = Mux(closeSubMags2r, closeSigOut2r, farSigOut3) + + val rawOutReg = RegInit(0.U.asTypeOf(new RawFloat(expWidth, sigWidth + 2))) + rawOutReg.isNaN := a2.isNaN || b2.isNaN + rawOutReg.isInf := notNaNIsInfOut3 + rawOutReg.isZero := notNaNIsZeroOut3 + rawOutReg.sExp := commonSExp3 + rawOutReg.sign := notNaNSignOut3 + rawOutReg.sig := commonSigOut3 + + io.rawOut := rawOutReg + io.invalidExc := RegNext(isSigNaNRawFloat(a2) || isSigNaNRawFloat(b2) || notSigNaNInvalidExc3) + io.validout := ShiftRegister(io.validin, 4) +} diff --git a/examples/prefab/balls/int8add/configs/default.toml b/examples/prefab/balls/int8add/configs/default.toml new file mode 100644 index 000000000..2cf302111 --- /dev/null +++ b/examples/prefab/balls/int8add/configs/default.toml @@ -0,0 +1 @@ +# Int8AddBall has no private configuration. diff --git a/examples/prefab/balls/int8add/configs/int8add_params.proto b/examples/prefab/balls/int8add/configs/int8add_params.proto new file mode 100644 index 000000000..0cb47dc6f --- /dev/null +++ b/examples/prefab/balls/int8add/configs/int8add_params.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; +package buckyball.balls.int8add; +option java_package = "buckyball.balls.int8add"; +option java_multiple_files = true; + +message Params { +} diff --git a/examples/prefab/balls/int8add/emu/src/dispatch.rs b/examples/prefab/balls/int8add/emu/src/dispatch.rs new file mode 100644 index 000000000..6aa07d004 --- /dev/null +++ b/examples/prefab/balls/int8add/emu/src/dispatch.rs @@ -0,0 +1,32 @@ +use bebop_bemu::ball::ExecContext; + +use super::int8add; + +const BALL_CLASS: &str = "examples.balls.int8add.Int8AddBall"; + +pub fn execute_known( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, + ctx: &mut ExecContext, +) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("INT8ADD") => Some(int8add::Int8Add::exec(xs1, xs2, ctx, false)), + Some("INT8ADD_RELU") => Some(int8add::Int8Add::exec(xs1, xs2, ctx, true)), + Some(_) | None => None, + } +} + +pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("INT8ADD") | Some("INT8ADD_RELU") => Some(int8add::Int8Add::latency(xs1, xs2)), + Some(_) | None => None, + } +} diff --git a/examples/balls/int8add/emu/src/int8add.rs b/examples/prefab/balls/int8add/emu/src/int8add.rs similarity index 94% rename from examples/balls/int8add/emu/src/int8add.rs rename to examples/prefab/balls/int8add/emu/src/int8add.rs index 72416c6b0..c54c9b09e 100644 --- a/examples/balls/int8add/emu/src/int8add.rs +++ b/examples/prefab/balls/int8add/emu/src/int8add.rs @@ -1,11 +1,12 @@ -use super::super::bank::{bank_lines, bank_row_bytes}; -use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; +use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; +use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old}; use super::instruction::ExecContext; pub struct Int8Add; impl Int8Add { pub fn exec(xs1: u64, xs2: u64, ctx: &mut ExecContext, relu: bool) -> u64 { + assert!(!rs1_read_old(xs1), "int8add: reads no old destination; rs1[29] must be clear"); let lhs_bank = rs1_b0(xs1); let rhs_bank = rs1_b1(xs1); let output_bank = rs1_b2(xs1); diff --git a/examples/prefab/balls/int8add/emu/src/lib.rs b/examples/prefab/balls/int8add/emu/src/lib.rs new file mode 100644 index 000000000..b7c2d06e7 --- /dev/null +++ b/examples/prefab/balls/int8add/emu/src/lib.rs @@ -0,0 +1,11 @@ +pub(crate) use bebop_bemu::{thread::instructions::decode, ball as instruction}; + +mod int8add; + +mod dispatch; + +pub use dispatch::{cycles_after_issue, execute_known}; + +#[bebop_bemu::ball::linkme::distributed_slice(bebop_bemu::BALLS)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static BALL: bebop_bemu::Ball = bebop_bemu::Ball { execute: execute_known, cycles: cycles_after_issue }; diff --git a/examples/prefab/balls/int8add/stack/src/int8addOps.cpp b/examples/prefab/balls/int8add/stack/src/int8addOps.cpp new file mode 100644 index 000000000..ab66f875b --- /dev/null +++ b/examples/prefab/balls/int8add/stack/src/int8addOps.cpp @@ -0,0 +1,78 @@ +//===- int8addOps.cpp - Int8AddBall operations ----------------------------===// +// +// The dialect of Int8AddBall, its virtual operations building their physical +// ones and the instructions of the physical operations. +// +//===----------------------------------------------------------------------===// + +#include "int8addOps.h" + +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +#include "mlir/Dialect/Arith/IR/Arith.h" + +using namespace mlir; +using namespace ::buddy::int8add_ball; +using mlir::buddy::bankFields; +using mlir::buddy::constantI64; +using mlir::buddy::emitInstruction; +using mlir::buddy::field; + +#include "int8addDialect.cpp.inc" + +#define GET_OP_CLASSES +#include "int8addOps.cpp.inc" + +void Int8AddBallDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "int8addOps.cpp.inc" + >(); +} + +namespace { +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallDialect([](DialectRegistry ®istry) { + registry.insert(); + }), + true); +} // namespace + +SmallVector BankAddI8Op::getSourceBanks() { return {getA(), getB()}; } +Value BankAddI8Op::getDestinationBank() { return getC(); } +bool BankAddI8Op::readsDestination() { return false; } +Operation *BankAddI8Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return AddI8Op::create(builder, getLoc(), getLhsRatio(), getRhsRatio(), + sources[0], sources[1], destination, getRows(), + getRelu()); +} + +SmallVector AddI8Op::getSourceBanks() { + return {int64_t(getA()), int64_t(getB())}; +} +int64_t AddI8Op::getDestinationBank() { return getC(); } +LogicalResult AddI8Op::lowerToInstructions(OpBuilder &builder) { + Location loc = getLoc(); + int64_t rows = getRows(), a = getA(), b = getB(), c = getC(); + if (rows <= 0 || rows > buckyball_target::getBuckyballTarget().bankDepth || + a == b || a == c || b == c) + return emitOpError("add_i8 requires distinct banks and rows within a bank"); + // rs2: lhs_ratio bits in [31:0], rhs_ratio bits in [63:32]. + auto ratioBits = [&](Value ratio) -> Value { + Value bits = + arith::BitcastOp::create(builder, loc, builder.getI32Type(), ratio); + return arith::ExtUIOp::create(builder, loc, builder.getI64Type(), bits); + }; + Value rhs = arith::ShLIOp::create(builder, loc, ratioBits(getRhsRatio()), + constantI64(builder, loc, 32)); + Value rs2 = arith::OrIOp::create(builder, loc, ratioBits(getLhsRatio()), rhs); + emitInstruction(builder, loc, + buckyball_target::getBuckyballFunct7( + getRelu() ? "INT8ADD_RELU" : "INT8ADD"), + constantI64(builder, loc, bankFields({a, b}, c, false, rows)), + rs2); + return success(); +} diff --git a/examples/prefab/balls/int8add/stack/src/int8addOps.h b/examples/prefab/balls/int8add/stack/src/int8addOps.h new file mode 100644 index 000000000..ce6a56a64 --- /dev/null +++ b/examples/prefab/balls/int8add/stack/src/int8addOps.h @@ -0,0 +1,20 @@ +//===- int8addOps.h - Int8AddBall operations ------------------------------===// + +#ifndef INT8ADD_OPS_H +#define INT8ADD_OPS_H + +#include "Interfaces/BankInterfaces.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "int8addDialect.h.inc" + +#define GET_OP_CLASSES +#include "int8addOps.h.inc" + +#endif // INT8ADD_OPS_H diff --git a/examples/prefab/balls/int8add/stack/src/int8addOps.td b/examples/prefab/balls/int8add/stack/src/int8addOps.td new file mode 100644 index 000000000..673fc926e --- /dev/null +++ b/examples/prefab/balls/int8add/stack/src/int8addOps.td @@ -0,0 +1,46 @@ +//===- int8addOps.td - Int8AddBall operations --------------*- tablegen -*-===// +// +// add_i8: c receives `rows` rows of sixteen i8, +// round_half_even(a * lhs_ratio + b * rhs_ratio) saturated to i8, and with +// `relu` clamped at zero. lhs_ratio and rhs_ratio are f32, both finite and +// positive. +// +//===----------------------------------------------------------------------===// + +#ifndef INT8ADD_OPS +#define INT8ADD_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Interfaces/BankInterfaces.td" + +def Int8AddBall_Dialect : Dialect { + let name = "int8add_ball"; + let cppNamespace = "::buddy::int8add_ball"; + let summary = "The operations of Int8AddBall"; +} + +class Int8AddBall_Op traits = []> + : Op])>; + +def BankAddI8Op : Int8AddBall_Op<"bank_add_i8", + [DeclareOpInterfaceMethods]> { + let summary = "add_i8 on virtual banks"; + let arguments = (ins I64:$a, I64:$b, I64:$c, F32:$lhs_ratio, + F32:$rhs_ratio, I64Attr:$rows, UnitAttr:$relu); + let results = (outs I64:$cOut); + let assemblyFormat = "$a `,` $b `->` $c `ratios` `(` $lhs_ratio `,` " + "$rhs_ratio `)` prop-dict attr-dict"; +} + +def AddI8Op : Int8AddBall_Op<"add_i8", + [DeclareOpInterfaceMethods]> { + let summary = "add_i8 on physical banks"; + let arguments = (ins F32:$lhs_ratio, F32:$rhs_ratio, I64Attr:$a, + I64Attr:$b, I64Attr:$c, I64Attr:$rows, UnitAttr:$relu); + let assemblyFormat = "`ratios` `(` $lhs_ratio `,` $rhs_ratio `)` prop-dict " + "attr-dict"; +} + +#endif // INT8ADD_OPS diff --git a/examples/balls/int8add/verify/casegen/Cargo.lock b/examples/prefab/balls/int8add/verify/casegen/Cargo.lock similarity index 100% rename from examples/balls/int8add/verify/casegen/Cargo.lock rename to examples/prefab/balls/int8add/verify/casegen/Cargo.lock diff --git a/examples/balls/int8add/verify/casegen/Cargo.toml b/examples/prefab/balls/int8add/verify/casegen/Cargo.toml similarity index 100% rename from examples/balls/int8add/verify/casegen/Cargo.toml rename to examples/prefab/balls/int8add/verify/casegen/Cargo.toml diff --git a/examples/prefab/balls/int8add/verify/casegen/src/casegen.rs b/examples/prefab/balls/int8add/verify/casegen/src/casegen.rs new file mode 100644 index 000000000..94a045b1b --- /dev/null +++ b/examples/prefab/balls/int8add/verify/casegen/src/casegen.rs @@ -0,0 +1,228 @@ +pub const ROW_BYTES: usize = 16; +pub const BANK_ENTRIES: u32 = 64; +pub const MAX_ROWS: usize = 64; +pub const NUM_CASES: u32 = 3; + +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Int8AddCmd { + pub relu: u32, + pub iter: u32, + pub op1_bank: u32, + pub op2_bank: u32, + pub wr_bank: u32, + pub op1_group: u32, + pub op2_group: u32, + pub wr_group: u32, + pub op1_col: u32, + pub op2_col: u32, + pub wr_col: u32, + pub rob_id: u32, + pub rs2_lo: u32, + pub rs2_hi: u32, + pub num_lhs_words: u32, + pub num_rhs_words: u32, + pub num_dst_words: u32, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Int8AddCase { + pub cmd: Int8AddCmd, + pub lhs_words: [u128; MAX_ROWS], + pub rhs_words: [u128; MAX_ROWS], + pub dst_words: [u128; MAX_ROWS], +} + +impl Int8AddCase { + pub fn lhs_lo(&self, index: usize) -> u64 { + self.lhs_words[index] as u64 + } + + pub fn lhs_hi(&self, index: usize) -> u64 { + (self.lhs_words[index] >> 64) as u64 + } + + pub fn rhs_lo(&self, index: usize) -> u64 { + self.rhs_words[index] as u64 + } + + pub fn rhs_hi(&self, index: usize) -> u64 { + (self.rhs_words[index] >> 64) as u64 + } + + pub fn dst_lo(&self, index: usize) -> u64 { + self.dst_words[index] as u64 + } + + pub fn dst_hi(&self, index: usize) -> u64 { + (self.dst_words[index] >> 64) as u64 + } +} + +// Word `i` of an operand is bank row `i % iter` in group `i / iter`: the Ball walks groups in order. +pub fn gen_case(index: u32) -> Int8AddCase { + if index >= NUM_CASES { + panic!("int8add: unsupported directed case {index}"); + } + match index { + 0 => directed(2, false, 1, [0, 0, 0]), + 1 => directed(3, true, 1, [0, 0, 0]), + 2 => directed(4, false, 2, [1, 3, 2]), + _ => panic!("int8add: unsupported directed case {index}"), + } +} + +fn compute(lhs: i8, rhs: i8, lhs_ratio: f32, rhs_ratio: f32, relu: bool) -> i8 { + let value = (lhs as f32) * lhs_ratio + (rhs as f32) * rhs_ratio; + let rounded = value.round_ties_even(); + let clamped = if relu { + rounded.max(0.0) + } else { + rounded + } + .clamp(-128.0, 127.0); + clamped as i8 +} + +fn positive_finite(value: f32) -> bool { + value.is_finite() && value > 0.0 +} + +fn ctest_values(rows: usize) -> (Vec, Vec) { + let count = rows * ROW_BYTES; + let mut lhs = vec![0u8; count]; + let mut rhs = vec![0u8; count]; + for i in 0..count { + lhs[i] = ((i as i32 * 37 & 255) - 128) as i8 as u8; + rhs[i] = ((i as i32 * 19 & 255) - 128) as i8 as u8; + } + (pack_bytes(&lhs), pack_bytes(&rhs)) +} + +fn pack_bytes(bytes: &[u8]) -> Vec { + if bytes.len() % ROW_BYTES != 0 { + panic!( + "int8add: {} bytes is not a multiple of {ROW_BYTES}", + bytes.len() + ); + } + bytes + .chunks(ROW_BYTES) + .map(|chunk| { + let mut word = 0u128; + for (lane, byte) in chunk.iter().enumerate() { + word |= u128::from(*byte) << (lane * 8); + } + word + }) + .collect() +} + +fn copy_words(src: &[u128]) -> [u128; N] { + if src.len() > N { + panic!("int8add: {} words exceed buffer {N}", src.len()); + } + let mut out = [0u128; N]; + out[..src.len()].copy_from_slice(src); + out +} + +fn encode_rs2(lhs_ratio: f32, rhs_ratio: f32) -> u64 { + u64::from(lhs_ratio.to_bits()) | (u64::from(rhs_ratio.to_bits()) << 32) +} + +fn split_rs(value: u64) -> (u32, u32) { + (value as u32, (value >> 32) as u32) +} + +pub(crate) fn validate( + op1_bank: u32, + op2_bank: u32, + wr_bank: u32, + op1_col: u32, + op2_col: u32, + wr_col: u32, + iter: u32, + rs2: u64, + lhs_ratio: f32, + rhs_ratio: f32, +) { + if iter == 0 || iter > BANK_ENTRIES { + panic!("int8add: iter must fit in one physical bank"); + } + if op1_col == 0 || op1_col != op2_col || op1_col != wr_col { + panic!("int8add: bank groups must match"); + } + if op1_bank == op2_bank || op1_bank == wr_bank || op2_bank == wr_bank { + panic!("int8add: banks must be distinct"); + } + if !positive_finite(lhs_ratio) || !positive_finite(rhs_ratio) { + panic!("int8add: ratios must be finite and positive"); + } + if rs2 != encode_rs2(lhs_ratio, rhs_ratio) { + panic!("int8add: rs2 ratio fields mismatch"); + } +} + +fn dst_words( + lhs: &[u128], + rhs: &[u128], + words: usize, + lhs_ratio: f32, + rhs_ratio: f32, + relu: bool, +) -> Vec { + let mut out = Vec::with_capacity(words); + for row in 0..words { + let mut packed = 0u128; + for lane in 0..ROW_BYTES { + let l = ((lhs[row] >> (lane * 8)) & 0xff) as u8 as i8; + let r = ((rhs[row] >> (lane * 8)) & 0xff) as u8 as i8; + let q = compute(l, r, lhs_ratio, rhs_ratio, relu); + packed |= u128::from(q as u8) << (lane * 8); + } + out.push(packed); + } + out +} + +fn directed(rob_id: u32, relu: bool, cols: u32, groups: [u32; 3]) -> Int8AddCase { + let iter = 7u32; + let words = (iter * cols) as usize; + let lhs_ratio = 0.5f32; + let rhs_ratio = 0.25f32; + let (lhs, rhs) = ctest_values(words); + let op1_bank = 0; + let op2_bank = 1; + let wr_bank = 2; + let rs2 = encode_rs2(lhs_ratio, rhs_ratio); + validate( + op1_bank, op2_bank, wr_bank, cols, cols, cols, iter, rs2, lhs_ratio, rhs_ratio, + ); + let dst = dst_words(&lhs, &rhs, words, lhs_ratio, rhs_ratio, relu); + let (rs2_lo, rs2_hi) = split_rs(rs2); + Int8AddCase { + cmd: Int8AddCmd { + relu: u32::from(relu), + iter, + op1_bank, + op2_bank, + wr_bank, + op1_group: groups[0], + op2_group: groups[1], + wr_group: groups[2], + op1_col: cols, + op2_col: cols, + wr_col: cols, + rob_id, + rs2_lo, + rs2_hi, + num_lhs_words: lhs.len() as u32, + num_rhs_words: rhs.len() as u32, + num_dst_words: dst.len() as u32, + }, + lhs_words: copy_words(&lhs), + rhs_words: copy_words(&rhs), + dst_words: copy_words(&dst), + } +} diff --git a/examples/prefab/balls/int8add/verify/casegen/src/lib.rs b/examples/prefab/balls/int8add/verify/casegen/src/lib.rs new file mode 100644 index 000000000..a57b3c268 --- /dev/null +++ b/examples/prefab/balls/int8add/verify/casegen/src/lib.rs @@ -0,0 +1,70 @@ +mod casegen; + +use std::cell::RefCell; + +use casegen::{Int8AddCase, Int8AddCmd}; + +thread_local! { + static CURRENT: RefCell> = const { RefCell::new(None) }; +} + +#[no_mangle] +pub extern "C" fn int8add_case_load(index: u32) -> i32 { + let case = casegen::gen_case(index); + CURRENT.with(|current| *current.borrow_mut() = Some(case)); + 0 +} + +fn current_case R, R>(f: F) -> R { + CURRENT.with(|current| match current.borrow().as_ref() { + Some(case) => f(case), + None => panic!("int8add_case: no case loaded; call int8add_case_load first"), + }) +} + +#[no_mangle] +pub extern "C" fn int8add_case_cmd(out_ptr: *mut Int8AddCmd) { + if out_ptr.is_null() { + panic!("int8add_case_cmd: null out_ptr"); + } + current_case(|case| unsafe { + *out_ptr = case.cmd; + }); +} + +fn idx(index: u32, limit: u32, what: &str) -> usize { + if index >= limit { + panic!("int8add_case_{what}: index {index} out of range {limit}"); + } + index as usize +} + +#[no_mangle] +pub extern "C" fn int8add_case_lhs_word_lo(word_index: u32) -> u64 { + current_case(|case| case.lhs_lo(idx(word_index, case.cmd.num_lhs_words, "lhs_word_lo"))) +} + +#[no_mangle] +pub extern "C" fn int8add_case_lhs_word_hi(word_index: u32) -> u64 { + current_case(|case| case.lhs_hi(idx(word_index, case.cmd.num_lhs_words, "lhs_word_hi"))) +} + +#[no_mangle] +pub extern "C" fn int8add_case_rhs_word_lo(word_index: u32) -> u64 { + current_case(|case| case.rhs_lo(idx(word_index, case.cmd.num_rhs_words, "rhs_word_lo"))) +} + +#[no_mangle] +pub extern "C" fn int8add_case_rhs_word_hi(word_index: u32) -> u64 { + current_case(|case| case.rhs_hi(idx(word_index, case.cmd.num_rhs_words, "rhs_word_hi"))) +} + +#[no_mangle] +pub extern "C" fn int8add_case_dst_word_lo(word_index: u32) -> u64 { + current_case(|case| case.dst_lo(idx(word_index, case.cmd.num_dst_words, "dst_word_lo"))) +} + +#[no_mangle] +pub extern "C" fn int8add_case_dst_word_hi(word_index: u32) -> u64 { + current_case(|case| case.dst_hi(idx(word_index, case.cmd.num_dst_words, "dst_word_hi"))) +} diff --git a/examples/balls/int8add/verify/filelists/int8add_ball.f b/examples/prefab/balls/int8add/verify/filelists/int8add_ball.f similarity index 88% rename from examples/balls/int8add/verify/filelists/int8add_ball.f rename to examples/prefab/balls/int8add/verify/filelists/int8add_ball.f index 1a0f1aefc..33eecc654 100644 --- a/examples/balls/int8add/verify/filelists/int8add_ball.f +++ b/examples/prefab/balls/int8add/verify/filelists/int8add_ball.f @@ -1,3 +1,4 @@ ++incdir+@UVM@/src/core +incdir+@UVM@/src/ball +incdir+@UVM@/src/ball/agents/cmd +incdir+@UVM@/src/ball/agents/mem @@ -13,17 +14,18 @@ +incdir+src/tests +incdir+@RTL@ +@UVM@/src/core/package.sv @UVM@/src/ball/bb_blink_if.sv @UVM@/src/ball/bb_uvm_pkg.sv @RTL@/RoundAnyRawFNToRecFN_ie4_is8_oe8_os24.sv -@RTL@/RoundAnyRawFNToRecFN_ie8_is26_oe8_os24.sv +@RTL@/IntToFloat.sv @RTL@/MulAddRecFNToRaw_preMul_e8_s24.sv @RTL@/MulAddRecFNToRaw_postMul_e8_s24.sv -@RTL@/MulAddRecFNPipe_l3_e8_s24.sv -@RTL@/PipelinedINToRecFN_3.sv -@RTL@/PipelinedAddRawFN.sv +@RTL@/RoundAnyRawFNToRecFN_ie8_is26_oe8_os24.sv @RTL@/RoundRawFNToRecFN_e8_s24.sv +@RTL@/Fma_l3_e8_s24.sv +@RTL@/PipelinedAddRawFN.sv @RTL@/RecFNToIN_e8_s24_i8.sv @RTL@/Int8AddBall.sv diff --git a/examples/prefab/balls/int8add/verify/src/common/int8add_defs.svh b/examples/prefab/balls/int8add/verify/src/common/int8add_defs.svh new file mode 100644 index 000000000..428c45e36 --- /dev/null +++ b/examples/prefab/balls/int8add/verify/src/common/int8add_defs.svh @@ -0,0 +1,41 @@ +typedef struct { + int unsigned relu; + int unsigned iter; + int unsigned op1_bank; + int unsigned op2_bank; + int unsigned wr_bank; + int unsigned op1_group; + int unsigned op2_group; + int unsigned wr_group; + int unsigned op1_col; + int unsigned op2_col; + int unsigned wr_col; + int unsigned rob_id; + int unsigned rs2_lo; + int unsigned rs2_hi; + int unsigned num_lhs_words; + int unsigned num_rhs_words; + int unsigned num_dst_words; +} int8add_cmd_dpi_t; + +import "DPI-C" function int int8add_case_load(input int unsigned index); +import "DPI-C" function void int8add_case_cmd(output int8add_cmd_dpi_t cmd); +import "DPI-C" function longint unsigned int8add_case_lhs_word_lo(input int unsigned word_index); +import "DPI-C" function longint unsigned int8add_case_lhs_word_hi(input int unsigned word_index); +import "DPI-C" function longint unsigned int8add_case_rhs_word_lo(input int unsigned word_index); +import "DPI-C" function longint unsigned int8add_case_rhs_word_hi(input int unsigned word_index); +import "DPI-C" function longint unsigned int8add_case_dst_word_lo(input int unsigned word_index); +import "DPI-C" function longint unsigned int8add_case_dst_word_hi(input int unsigned word_index); + +`ifndef INT8ADD_FUNCT7 +`error "INT8ADD_FUNCT7 must be provided by the selected Thread ballISA" +`endif +`ifndef INT8ADD_RELU_FUNCT7 +`error "INT8ADD_RELU_FUNCT7 must be provided by the selected Thread ballISA" +`endif + +localparam int INT8ADD_THREAD_FUNCT7 = `INT8ADD_FUNCT7; +localparam int INT8ADD_RELU_THREAD_FUNCT7 = `INT8ADD_RELU_FUNCT7; +localparam int INT8ADD_MAX_ROWS = 64; +localparam int INT8ADD_NUM_CASES = 3; +localparam int INT8ADD_TIMEOUT_CYCLES = 20000; diff --git a/examples/prefab/balls/int8add/verify/src/common/int8add_items.svh b/examples/prefab/balls/int8add/verify/src/common/int8add_items.svh new file mode 100644 index 000000000..eed15ba3f --- /dev/null +++ b/examples/prefab/balls/int8add/verify/src/common/int8add_items.svh @@ -0,0 +1,103 @@ +class int8add_cmd_item extends bb_blink_cmd_item; + `uvm_object_utils(int8add_cmd_item) + + bit [31:0] relu; + bit [31:0] num_lhs_words; + bit [31:0] num_rhs_words; + bit [31:0] num_dst_words; + bit [127:0] lhs_words[INT8ADD_MAX_ROWS]; + bit [127:0] rhs_words[INT8ADD_MAX_ROWS]; + bit [127:0] dst_words[INT8ADD_MAX_ROWS]; + + function new(string name = "int8add_cmd_item"); + super.new(name); + endfunction + + function void load_rust_case(int unsigned index); + int8add_cmd_dpi_t cmd; + int unsigned rc; + longint unsigned w_lo; + longint unsigned w_hi; + int i; + + rc = int8add_case_load(index); + if (rc != 0) begin + `uvm_fatal("CASE", $sformatf("int8add_case_load returned %0d for index %0d", rc, index)) + end + int8add_case_cmd(cmd); + + if (cmd.relu != 0 && cmd.relu != 1) begin + `uvm_fatal("CASE", $sformatf("invalid relu flag %0d", cmd.relu)) + end + + // bb_int8add: banks and iter come from rs1, ratios from rs2. + relu = cmd.relu; + rob_id = cmd.rob_id[BB_BLINK_ROB_ID_W-1:0]; + funct = (relu == 0) ? INT8ADD_THREAD_FUNCT7[6:0] : INT8ADD_RELU_THREAD_FUNCT7[6:0]; + rs2 = {cmd.rs2_hi, cmd.rs2_lo}; + iter = cmd.iter; + op1_bank = cmd.op1_bank; + op2_bank = cmd.op2_bank; + wr_bank = cmd.wr_bank; + op1_group = cmd.op1_group; + op2_group = cmd.op2_group; + wr_group = cmd.wr_group; + op1_col = cmd.op1_col; + op2_col = cmd.op2_col; + wr_col = cmd.wr_col; + wr_read_bank = '0; + wr_arch_bank = cmd.wr_bank; + num_lhs_words = cmd.num_lhs_words; + num_rhs_words = cmd.num_rhs_words; + num_dst_words = cmd.num_dst_words; + + if (iter == 0) `uvm_fatal("CASE", "Int8AddBall iter must be positive") + if (op1_col == 0 || op1_col != op2_col || op1_col != wr_col) + `uvm_fatal("CASE", "Int8AddBall bank groups must match") + if (op1_bank == op2_bank || op1_bank == wr_bank || op2_bank == wr_bank) + `uvm_fatal("CASE", "Int8AddBall banks must be distinct") + if (num_lhs_words != iter * op1_col || num_rhs_words != iter * op1_col || + num_dst_words != iter * op1_col) + `uvm_fatal("CASE", $sformatf( + "word counts %0d/%0d/%0d != iter %0d * col %0d", + num_lhs_words, + num_rhs_words, + num_dst_words, + iter, + op1_col + )) + if (num_lhs_words > INT8ADD_MAX_ROWS) + `uvm_fatal("CASE", $sformatf("word count %0d exceeds %0d", num_lhs_words, INT8ADD_MAX_ROWS)) + + for (i = 0; i < num_lhs_words; i++) begin + w_lo = int8add_case_lhs_word_lo(i); + w_hi = int8add_case_lhs_word_hi(i); + lhs_words[i] = {w_hi, w_lo}; + end + for (i = 0; i < num_rhs_words; i++) begin + w_lo = int8add_case_rhs_word_lo(i); + w_hi = int8add_case_rhs_word_hi(i); + rhs_words[i] = {w_hi, w_lo}; + end + for (i = 0; i < num_dst_words; i++) begin + w_lo = int8add_case_dst_word_lo(i); + w_hi = int8add_case_dst_word_hi(i); + dst_words[i] = {w_hi, w_lo}; + end + endfunction + + function void do_copy(uvm_object rhs); + int8add_cmd_item rhs_; + super.do_copy(rhs); + if (!$cast(rhs_, rhs)) begin + `uvm_fatal("COPY", "rhs is not int8add_cmd_item") + end + relu = rhs_.relu; + num_lhs_words = rhs_.num_lhs_words; + num_rhs_words = rhs_.num_rhs_words; + num_dst_words = rhs_.num_dst_words; + for (int i = 0; i < INT8ADD_MAX_ROWS; i++) lhs_words[i] = rhs_.lhs_words[i]; + for (int i = 0; i < INT8ADD_MAX_ROWS; i++) rhs_words[i] = rhs_.rhs_words[i]; + for (int i = 0; i < INT8ADD_MAX_ROWS; i++) dst_words[i] = rhs_.dst_words[i]; + endfunction +endclass diff --git a/examples/prefab/balls/int8add/verify/src/cov/int8add_cov.svh b/examples/prefab/balls/int8add/verify/src/cov/int8add_cov.svh new file mode 100644 index 000000000..f629564b5 --- /dev/null +++ b/examples/prefab/balls/int8add/verify/src/cov/int8add_cov.svh @@ -0,0 +1,40 @@ +class int8add_cov extends uvm_component; + `uvm_component_utils(int8add_cov) + + uvm_analysis_imp_cmd #(bb_blink_cmd_item, int8add_cov) cmd_imp; + + bit [6:0] cur_funct; + int unsigned cur_col; + + covergroup cmd_cg; + coverpoint cur_col {bins c1 = {1}; bins c2 = {2};} + coverpoint cur_funct { + bins add = {INT8ADD_THREAD_FUNCT7[6:0]}; bins relu = {INT8ADD_RELU_THREAD_FUNCT7[6:0]}; + } + endgroup + + int cmd_count; + + function new(string name, uvm_component parent); + super.new(name, parent); + cmd_imp = new("cmd_imp", this); + cmd_cg = new(); + endfunction + + function void write_cmd(bb_blink_cmd_item item); + if (item.funct != INT8ADD_THREAD_FUNCT7[6:0] && item.funct != INT8ADD_RELU_THREAD_FUNCT7[6:0]) + `uvm_fatal("COV", $sformatf("unexpected funct %0d", item.funct)) + cur_funct = item.funct; + cur_col = item.op1_col; + cmd_count++; + cmd_cg.sample(); + endfunction + + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (cmd_count == 0) `uvm_fatal("COV", "no cmd observed") + if (cmd_cg.get_coverage() != 100.0) + `uvm_fatal("COV", $sformatf("int8add_cov %0.2f%%", cmd_cg.get_coverage())) + endfunction +endclass diff --git a/examples/balls/int8add/verify/src/env/int8add_env.svh b/examples/prefab/balls/int8add/verify/src/env/int8add_env.svh similarity index 100% rename from examples/balls/int8add/verify/src/env/int8add_env.svh rename to examples/prefab/balls/int8add/verify/src/env/int8add_env.svh diff --git a/examples/balls/int8add/verify/src/env/int8add_scoreboard.svh b/examples/prefab/balls/int8add/verify/src/env/int8add_scoreboard.svh similarity index 75% rename from examples/balls/int8add/verify/src/env/int8add_scoreboard.svh rename to examples/prefab/balls/int8add/verify/src/env/int8add_scoreboard.svh index e27527ad6..f52fe175c 100644 --- a/examples/balls/int8add/verify/src/env/int8add_scoreboard.svh +++ b/examples/prefab/balls/int8add/verify/src/env/int8add_scoreboard.svh @@ -46,17 +46,20 @@ class int8add_scoreboard extends uvm_scoreboard; `uvm_fatal("SCB", "mem_model handle not set") end mem_model.clear_mem(); + // Word i lives at row i % iter, group base + i / iter. for (i = 0; i < clone.num_lhs_words; i++) begin - mem_model.load_word(int'(clone.op1_bank), i, clone.lhs_words[i]); + mem_model.load_word_g(int'(clone.op1_bank), int'(clone.op1_group) + i / clone.iter, + i % clone.iter, clone.lhs_words[i]); end for (i = 0; i < clone.num_rhs_words; i++) begin - mem_model.load_word(int'(clone.op2_bank), i, clone.rhs_words[i]); + mem_model.load_word_g(int'(clone.op2_bank), int'(clone.op2_group) + i / clone.iter, + i % clone.iter, clone.rhs_words[i]); end mem_model.arm(); expected_writes = clone.num_dst_words; - expected_reads[0] = clone.iter; - expected_reads[1] = clone.iter; + expected_reads[0] = clone.num_lhs_words; + expected_reads[1] = clone.num_rhs_words; endfunction function void write_cmd(bb_blink_cmd_item item); @@ -67,43 +70,38 @@ class int8add_scoreboard extends uvm_scoreboard; endfunction function void check_cmd(bb_blink_cmd_item got, int8add_cmd_item exp); - if (got.bid !== exp.bid) - `uvm_fatal("CMD", $sformatf("bid mismatch: got %0d exp %0d", got.bid, exp.bid)) - if (got.funct7 !== exp.funct7) - `uvm_fatal("CMD", $sformatf("funct7 mismatch: got %0d exp %0d", got.funct7, exp.funct7)) - if (got.iter !== exp.iter) - `uvm_fatal("CMD", $sformatf("iter mismatch: got %0d exp %0d", got.iter, exp.iter)) - if (got.op1_bank !== exp.op1_bank || got.op2_bank !== exp.op2_bank || - got.wr_bank !== exp.wr_bank) - `uvm_fatal("CMD", "bank field mismatch") - if (got.op1_col !== exp.op1_col || got.op2_col !== exp.op2_col || got.wr_col !== exp.wr_col) - `uvm_fatal("CMD", "column field mismatch") - if (got.rs1 !== exp.rs1 || got.rs2 !== exp.rs2) `uvm_fatal("CMD", "rs field mismatch") - if (got.rob_id !== exp.rob_id) - `uvm_fatal("CMD", $sformatf("rob_id mismatch: got %0d exp %0d", got.rob_id, exp.rob_id)) + // Compare as plain BLink commands; the comparer rejects differing types. + bb_blink_cmd_item want = bb_blink_cmd_item::type_id::create("want"); + want.copy(exp); + verification_pkg::evidence::checks++; + if (!got.compare(want)) + `uvm_fatal("CMD", $sformatf("command mismatch: got %s exp %s", got.sprint(), want.sprint())) endfunction function void write_read(bb_blink_read_item item); int8add_cmd_item stim; int unsigned expect_bank; + int unsigned expect_group; int unsigned expect_addr; stim = current_stim("READ"); if (item.port != 0 && item.port != 1) `uvm_fatal("READ", $sformatf("Int8AddBall has two read ports, got port %0d", item.port)) - if (item.group_id !== 5'd0) - `uvm_fatal("READ", $sformatf("group mismatch: got %0d exp 0", item.group_id)) if (item.rob_id !== stim.rob_id) `uvm_fatal("READ", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) if (item.port == 0) begin - expect_bank = stim.op1_bank; - expect_addr = read_count[0]; + expect_bank = stim.op1_bank; + expect_group = stim.op1_group + read_count[0] / stim.iter; + expect_addr = read_count[0] % stim.iter; end else begin - expect_bank = stim.op2_bank; - expect_addr = read_count[1]; + expect_bank = stim.op2_bank; + expect_group = stim.op2_group + read_count[1] / stim.iter; + expect_addr = read_count[1] % stim.iter; end - if (item.bank_id !== expect_bank[4:0]) + if (item.group_id !== expect_group[BB_BLINK_GROUP_ID_W-1:0]) + `uvm_fatal("READ", $sformatf("group mismatch: got %0d exp %0d", item.group_id, expect_group)) + if (item.bank_id !== expect_bank[BB_BLINK_BANK_ID_W-1:0]) `uvm_fatal("READ", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, expect_bank)) if (item.addr !== expect_addr[BB_BLINK_BANK_ADDR_W-1:0]) `uvm_fatal("READ", $sformatf("addr mismatch: got %0d exp %0d", item.addr, expect_addr)) @@ -120,14 +118,14 @@ class int8add_scoreboard extends uvm_scoreboard; `uvm_fatal("WRITE", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.wr_bank)) if (item.rob_id !== stim.rob_id) `uvm_fatal("WRITE", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.group_id !== 5'd0) - `uvm_fatal("WRITE", $sformatf("group mismatch: got %0d exp 0", item.group_id)) + if (item.group_id !== stim.wr_group + write_count / stim.iter) + `uvm_fatal("WRITE", $sformatf("group mismatch at %0d: got %0d", write_count, item.group_id)) if (item.mask !== 16'hFFFF) `uvm_fatal("WRITE", $sformatf("mask mismatch: got 0x%04h", item.mask)) if (write_count >= stim.num_dst_words) `uvm_fatal("WRITE", $sformatf("extra write %0d", write_count)) - if (item.addr !== write_count[BB_BLINK_BANK_ADDR_W-1:0]) - `uvm_fatal("WRITE", $sformatf("addr mismatch: got %0d exp %0d", item.addr, write_count)) + if (item.addr !== write_count % stim.iter) + `uvm_fatal("WRITE", $sformatf("addr mismatch at %0d: got %0d", write_count, item.addr)) if (item.data !== stim.dst_words[write_count]) `uvm_fatal("SCB", $sformatf( "data mismatch at addr %0d: got 0x%032h exp 0x%032h", @@ -144,9 +142,6 @@ class int8add_scoreboard extends uvm_scoreboard; stim = current_stim("RESP"); if (item.rob_id !== stim.rob_id) `uvm_fatal("RESP", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.is_sub !== 1'b0) `uvm_fatal("RESP", "is_sub should be 0") - if (item.sub_rob_id !== 8'h00) - `uvm_fatal("RESP", $sformatf("sub_rob_id mismatch: got 0x%0h", item.sub_rob_id)) resp_count++; endfunction @@ -178,6 +173,7 @@ class int8add_scoreboard extends uvm_scoreboard; endfunction function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; super.check_phase(phase); if (!done()) begin `uvm_fatal("SCB", $sformatf( diff --git a/examples/balls/int8add/verify/src/pkg/int8add_pkg.sv b/examples/prefab/balls/int8add/verify/src/pkg/int8add_pkg.sv similarity index 100% rename from examples/balls/int8add/verify/src/pkg/int8add_pkg.sv rename to examples/prefab/balls/int8add/verify/src/pkg/int8add_pkg.sv diff --git a/examples/balls/int8add/verify/src/seq/int8add_sequences.svh b/examples/prefab/balls/int8add/verify/src/seq/int8add_sequences.svh similarity index 86% rename from examples/balls/int8add/verify/src/seq/int8add_sequences.svh rename to examples/prefab/balls/int8add/verify/src/seq/int8add_sequences.svh index 2582816d1..0536522b2 100644 --- a/examples/balls/int8add/verify/src/seq/int8add_sequences.svh +++ b/examples/prefab/balls/int8add/verify/src/seq/int8add_sequences.svh @@ -2,7 +2,6 @@ class int8add_basic_seq extends uvm_sequence #(bb_blink_cmd_item); `uvm_object_utils(int8add_basic_seq) int unsigned case_index; - int unsigned bid; function new(string name = "int8add_basic_seq"); super.new(name); @@ -12,7 +11,7 @@ class int8add_basic_seq extends uvm_sequence #(bb_blink_cmd_item); int8add_cmd_item item; item = int8add_cmd_item::type_id::create("item"); start_item(item); - item.load_rust_case(case_index, bid); + item.load_rust_case(case_index); finish_item(item); endtask endclass diff --git a/examples/prefab/balls/int8add/verify/src/tb_top.sv b/examples/prefab/balls/int8add/verify/src/tb_top.sv new file mode 100644 index 000000000..995efdbb1 --- /dev/null +++ b/examples/prefab/balls/int8add/verify/src/tb_top.sv @@ -0,0 +1,89 @@ +module tb_top; + import uvm_pkg::*; + import int8add_pkg::*; + + bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); + + Int8AddBall dut ( + .clock(intf.clock), + .reset(intf.reset), + .blink_status_running(intf.status_running), + .blink_cmdReq_ready(intf.cmd_req_ready), + .blink_cmdReq_valid(intf.cmd_req_valid), + .blink_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .blink_cmdReq_bits_funct(intf.cmd_req_bits_funct), + .blink_cmdReq_bits_rs2(intf.cmd_req_bits_rs2), + .blink_cmdReq_bits_iter(intf.cmd_req_bits_iter), + .blink_cmdReq_bits_op1_bank(intf.cmd_req_bits_op1_bank), + .blink_cmdReq_bits_op2_bank(intf.cmd_req_bits_op2_bank), + .blink_cmdReq_bits_wr_bank(intf.cmd_req_bits_wr_bank), + .blink_cmdReq_bits_op1_group(intf.cmd_req_bits_op1_group), + .blink_cmdReq_bits_op2_group(intf.cmd_req_bits_op2_group), + .blink_cmdReq_bits_wr_group(intf.cmd_req_bits_wr_group), + .blink_cmdReq_bits_op1_col(intf.cmd_req_bits_op1_col), + .blink_cmdReq_bits_op2_col(intf.cmd_req_bits_op2_col), + .blink_cmdReq_bits_wr_col(intf.cmd_req_bits_wr_col), + .blink_cmdReq_bits_wr_read_bank(intf.cmd_req_bits_wr_read_bank), + .blink_cmdReq_bits_wr_arch_bank(intf.cmd_req_bits_wr_arch_bank), + .blink_cmdResp_ready(intf.cmd_resp_ready), + .blink_cmdResp_valid(intf.cmd_resp_valid), + .blink_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .blink_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .blink_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .blink_bankRead_0_group_id(intf.bank_read_group_id[0]), + .blink_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .blink_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .blink_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .blink_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .blink_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .blink_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .blink_bankRead_1_bank_id(intf.bank_read_bank_id[1]), + .blink_bankRead_1_rob_id(intf.bank_read_rob_id[1]), + .blink_bankRead_1_group_id(intf.bank_read_group_id[1]), + .blink_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), + .blink_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), + .blink_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), + .blink_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), + .blink_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), + .blink_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), + .blink_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .blink_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .blink_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .blink_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .blink_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .blink_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .blink_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .blink_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .blink_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .blink_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]) + ); + + // Constant outputs removed from the port list by firtool. + assign intf.bank_write_req_mask[0][0] = dut.blink_bankWrite_0_io_req_bits_mask_0; + assign intf.bank_write_req_mask[0][1] = dut.blink_bankWrite_0_io_req_bits_mask_1; + assign intf.bank_write_req_mask[0][2] = dut.blink_bankWrite_0_io_req_bits_mask_2; + assign intf.bank_write_req_mask[0][3] = dut.blink_bankWrite_0_io_req_bits_mask_3; + assign intf.bank_write_req_mask[0][4] = dut.blink_bankWrite_0_io_req_bits_mask_4; + assign intf.bank_write_req_mask[0][5] = dut.blink_bankWrite_0_io_req_bits_mask_5; + assign intf.bank_write_req_mask[0][6] = dut.blink_bankWrite_0_io_req_bits_mask_6; + assign intf.bank_write_req_mask[0][7] = dut.blink_bankWrite_0_io_req_bits_mask_7; + assign intf.bank_write_req_mask[0][8] = dut.blink_bankWrite_0_io_req_bits_mask_8; + assign intf.bank_write_req_mask[0][9] = dut.blink_bankWrite_0_io_req_bits_mask_9; + assign intf.bank_write_req_mask[0][10] = dut.blink_bankWrite_0_io_req_bits_mask_10; + assign intf.bank_write_req_mask[0][11] = dut.blink_bankWrite_0_io_req_bits_mask_11; + assign intf.bank_write_req_mask[0][12] = dut.blink_bankWrite_0_io_req_bits_mask_12; + assign intf.bank_write_req_mask[0][13] = dut.blink_bankWrite_0_io_req_bits_mask_13; + assign intf.bank_write_req_mask[0][14] = dut.blink_bankWrite_0_io_req_bits_mask_14; + assign intf.bank_write_req_mask[0][15] = dut.blink_bankWrite_0_io_req_bits_mask_15; + + initial begin + intf.clock = 1'b0; + intf.reset = 1'b1; + forever #5 intf.clock = ~intf.clock; + end + + initial begin + uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); + run_test("int8add_ball_test"); + end +endmodule diff --git a/examples/balls/int8add/verify/src/tests/int8add_test.svh b/examples/prefab/balls/int8add/verify/src/tests/int8add_test.svh similarity index 90% rename from examples/balls/int8add/verify/src/tests/int8add_test.svh rename to examples/prefab/balls/int8add/verify/src/tests/int8add_test.svh index 7f109a88f..3e4c2c300 100644 --- a/examples/balls/int8add/verify/src/tests/int8add_test.svh +++ b/examples/prefab/balls/int8add/verify/src/tests/int8add_test.svh @@ -17,20 +17,18 @@ class int8add_ball_test extends uvm_test; endfunction task run_phase(uvm_phase phase); - int unsigned bid = int8add_require_bid(); phase.raise_objection(this); - for (int unsigned index = 0; index < INT8ADD_NUM_CASES; index++) run_case(index, bid); + for (int unsigned index = 0; index < INT8ADD_NUM_CASES; index++) run_case(index); phase.drop_objection(this); endtask - task run_case(int unsigned index, int unsigned bid); + task run_case(int unsigned index); int8add_basic_seq seq; int cycles = 0; apply_reset(); env.scb.reset_counters(); seq = int8add_basic_seq::type_id::create("seq"); seq.case_index = index; - seq.bid = bid; seq.start(env.cmd_agent.seqr); while (!env.scb.done()) begin @(posedge vif.clock); diff --git a/examples/prefab/balls/int8add/workloads/ctests/CMakeLists.txt b/examples/prefab/balls/int8add/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..cb3eecb06 --- /dev/null +++ b/examples/prefab/balls/int8add/workloads/ctests/CMakeLists.txt @@ -0,0 +1,2 @@ +add_buckyball_ctest(int8add_scaled_test.c) +add_buckyball_ctest(int8add_relu_test.c) diff --git a/examples/balls/int8add/workloads/ctests/int8add_relu_test.c b/examples/prefab/balls/int8add/workloads/ctests/int8add_relu_test.c similarity index 94% rename from examples/balls/int8add/workloads/ctests/int8add_relu_test.c rename to examples/prefab/balls/int8add/workloads/ctests/int8add_relu_test.c index 394b9f6b6..4b70596fa 100644 --- a/examples/balls/int8add/workloads/ctests/int8add_relu_test.c +++ b/examples/prefab/balls/int8add/workloads/ctests/int8add_relu_test.c @@ -5,7 +5,7 @@ #include #include -enum { ROWS = 7, LANES = BANK_WIDTH / 8, VALUES = ROWS * LANES }; +enum { ROWS = 7, LANES = THREAD_MEM_BANK_WIDTH / 8, VALUES = ROWS * LANES }; static int8_t lhs[VALUES] __attribute__((aligned(64))); static int8_t rhs[VALUES] __attribute__((aligned(64))); static int8_t output[VALUES] __attribute__((aligned(64))); diff --git a/examples/balls/int8add/workloads/ctests/int8add_scaled_test.c b/examples/prefab/balls/int8add/workloads/ctests/int8add_scaled_test.c similarity index 94% rename from examples/balls/int8add/workloads/ctests/int8add_scaled_test.c rename to examples/prefab/balls/int8add/workloads/ctests/int8add_scaled_test.c index 6dfabf33e..bb0bf8ea8 100644 --- a/examples/balls/int8add/workloads/ctests/int8add_scaled_test.c +++ b/examples/prefab/balls/int8add/workloads/ctests/int8add_scaled_test.c @@ -5,7 +5,7 @@ #include #include -enum { ROWS = 7, LANES = BANK_WIDTH / 8, VALUES = ROWS * LANES }; +enum { ROWS = 7, LANES = THREAD_MEM_BANK_WIDTH / 8, VALUES = ROWS * LANES }; static int8_t lhs[VALUES] __attribute__((aligned(64))); static int8_t rhs[VALUES] __attribute__((aligned(64))); static int8_t output[VALUES] __attribute__((aligned(64))); diff --git a/examples/balls/int8add/workloads/isa/int8add.h b/examples/prefab/balls/int8add/workloads/isa/int8add.h similarity index 91% rename from examples/balls/int8add/workloads/isa/int8add.h rename to examples/prefab/balls/int8add/workloads/isa/int8add.h index 63ce07e3a..0bc519669 100644 --- a/examples/balls/int8add/workloads/isa/int8add.h +++ b/examples/prefab/balls/int8add/workloads/isa/int8add.h @@ -18,13 +18,13 @@ static inline uint64_t bb_int8add_rs2(float lhs_ratio, float rhs_ratio) { BUCKYBALL_INSTRUCTION_R_R((BB_BANK0(lhs_bank) | BB_BANK1(rhs_bank) | \ BB_BANK2(output_bank) | BB_ITER(iter)), \ bb_int8add_rs2(lhs_ratio, rhs_ratio), \ - BB_FUNC7(INT8ADD)) + FUNCT7(int8add, INT8ADD)) #define bb_int8add_relu(lhs_bank, rhs_bank, output_bank, iter, lhs_ratio, \ rhs_ratio) \ BUCKYBALL_INSTRUCTION_R_R((BB_BANK0(lhs_bank) | BB_BANK1(rhs_bank) | \ BB_BANK2(output_bank) | BB_ITER(iter)), \ bb_int8add_rs2(lhs_ratio, rhs_ratio), \ - BB_FUNC7(INT8ADD_RELU)) + FUNCT7(int8add, INT8ADD_RELU)) #endif // _BB_INT8ADD_H_ diff --git a/examples/balls/int8add/workloads/mlir_tests/CMakeLists.txt b/examples/prefab/balls/int8add/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/balls/int8add/workloads/mlir_tests/CMakeLists.txt rename to examples/prefab/balls/int8add/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/balls/int8add/workloads/mlir_tests/ball/CMakeLists.txt b/examples/prefab/balls/int8add/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/balls/int8add/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/prefab/balls/int8add/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/prefab/balls/int8add/workloads/mlir_tests/ball/int8add_relu.mlir b/examples/prefab/balls/int8add/workloads/mlir_tests/ball/int8add_relu.mlir new file mode 100644 index 000000000..f7d6c59ed --- /dev/null +++ b/examples/prefab/balls/int8add/workloads/mlir_tests/ball/int8add_relu.mlir @@ -0,0 +1,40 @@ +func.func private @check_result(memref<4x16xi8>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %four = arith.constant 4 : index + %sixteen = arith.constant 16 : index + %ratio = arith.constant 1.0 : f32 + %a = memref.alloc() : memref<4x16xi8> + %b = memref.alloc() : memref<4x16xi8> + %out = memref.alloc() : memref<4x16xi8> + scf.for %row = %zero to %four step %one { + scf.for %lane = %zero to %sixteen step %one { + %r = arith.index_cast %row : index to i32 + %l = arith.index_cast %lane : index to i32 + %a32 = arith.subi %l, %r : i32 + %b32 = arith.subi %r, %l : i32 + %a8 = arith.trunci %a32 : i32 to i8 + %b8 = arith.trunci %b32 : i32 to i8 + memref.store %a8, %a[%row, %lane] : memref<4x16xi8> + memref.store %b8, %b[%row, %lane] : memref<4x16xi8> + } + } + buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %a <{bank = 1 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> + buckyball.mvin %b <{bank = 2 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> + int8add_ball.add_i8 ratios(%ratio, %ratio) <{a = 1 : i64, b = 2 : i64, c = 3 : i64, rows = 4 : i64, relu}> + buckyball.mvout %out <{bank = 3 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> + buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + func.call @check_result(%out) : (memref<4x16xi8>) -> () + memref.dealloc %a : memref<4x16xi8> + memref.dealloc %b : memref<4x16xi8> + memref.dealloc %out : memref<4x16xi8> + return %zero_i8 : i8 +} diff --git a/examples/balls/int8add/workloads/mlir_tests/ball/int8add_relu_main.cpp b/examples/prefab/balls/int8add/workloads/mlir_tests/ball/int8add_relu_main.cpp similarity index 100% rename from examples/balls/int8add/workloads/mlir_tests/ball/int8add_relu_main.cpp rename to examples/prefab/balls/int8add/workloads/mlir_tests/ball/int8add_relu_main.cpp diff --git a/examples/balls/int8add/workloads/mlir_tests/bank/CMakeLists.txt b/examples/prefab/balls/int8add/workloads/mlir_tests/bank/CMakeLists.txt similarity index 100% rename from examples/balls/int8add/workloads/mlir_tests/bank/CMakeLists.txt rename to examples/prefab/balls/int8add/workloads/mlir_tests/bank/CMakeLists.txt diff --git a/examples/prefab/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale.mlir.in b/examples/prefab/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale.mlir.in new file mode 100644 index 000000000..ecc83f9b1 --- /dev/null +++ b/examples/prefab/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale.mlir.in @@ -0,0 +1,42 @@ +func.func private @check_result(memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %mod = arith.constant 31 : index + %rows = arith.constant @BB_BANK_LINES@ : index + %lanes = arith.constant @BB_BANK_I8_LANES@ : index + %ratio = arith.constant 1.0 : f32 + %a = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + %b = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + %out = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + scf.for %row = %zero to %rows step %one { + scf.for %lane = %zero to %lanes step %one { + %row_mod = arith.remui %row, %mod : index + %r = arith.index_cast %row_mod : index to i32 + %l = arith.index_cast %lane : index to i32 + %a32 = arith.subi %l, %r : i32 + %b32 = arith.subi %r, %l : i32 + %a8 = arith.trunci %a32 : i32 to i8 + %b8 = arith.trunci %b32 : i32 to i8 + memref.store %a8, %a[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + memref.store %b8, %b[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + } + } + %a_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %b_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %out_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %a_loaded = buckyball.bank_mvin %a, %a_bank <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + %b_loaded = buckyball.bank_mvin %b, %b_bank <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + %result = int8add_ball.bank_add_i8 %a_loaded, %b_loaded -> %out_bank ratios(%ratio, %ratio) <{rows = @BB_BANK_LINES@ : i64}> + %stored = buckyball.bank_mvout %out, %result <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + buckyball.bank_release %a_loaded + buckyball.bank_release %b_loaded + buckyball.bank_release %stored + func.call @check_result(%out) : (memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () + memref.dealloc %a : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + memref.dealloc %b : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + memref.dealloc %out : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + return %zero_i8 : i8 +} diff --git a/examples/prefab/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale_main.cpp b/examples/prefab/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale_main.cpp new file mode 100644 index 000000000..be1087dd1 --- /dev/null +++ b/examples/prefab/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale_main.cpp @@ -0,0 +1,16 @@ +#include +#include +#include +extern "C" void check_result(int8_t *, int8_t *data, int64_t offset, + int64_t rows, int64_t lanes, int64_t stride, + int64_t lane_stride) { + if (rows != THREAD_MEM_BANK_ENTRIES || lanes != THREAD_MEM_BANK_WIDTH / 8 || + stride != lanes || lane_stride != 1) + exit(1); + for (int i = 0; i < rows * lanes; ++i) + if (data[offset + i] != 0) { + printf("int8add bank mismatch index=%d\n", i); + exit(1); + } + printf("int8add bank PASS rows=%d\n", THREAD_MEM_BANK_ENTRIES); +} diff --git a/examples/prefab/balls/int8mul/arch/Int8MulBall.scala b/examples/prefab/balls/int8mul/arch/Int8MulBall.scala new file mode 100644 index 000000000..c612243ad --- /dev/null +++ b/examples/prefab/balls/int8mul/arch/Int8MulBall.scala @@ -0,0 +1,248 @@ +package examples.balls.int8mul + +import framework.acc.balldomain.bbus.{BallFactory, BallPorts} +import buckyball.balls.int8mul.{Params => ParametersProto} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import chisel3.util._ +import hardfloat.{recFNFromFN, INToRecFN, RecFNToIN} +import hardfloat.consts.{round_near_even, tininess_afterRounding} +import framework.lib.arith.Fma + +import framework.acc.balldomain.ball.blink.{BallStatus, BlinkIO, HasBlink} + +@instantiable +class Int8MulBall( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Module + with HasBlink { + val ball = bp.balls.find(_.id == id).get + + ball.parameters.unpack(classOf[ParametersProto]) + + require(ball.inBW == 2, "Int8MulBall requires inBW=2") + require(ball.outBW == 1, "Int8MulBall requires outBW=1") + require(mp.bankWidth == 128, "Int8MulBall requires 128-bit bank rows") + val bankRowBits = log2Ceil(mp.bankEntries) + require(bankRowBits <= 32, "Int8MulBall gate_row must fit rs2[63:32]") + + @public + val blink = IO(new BlinkIO(fp, mp, bp, id)) + + val idle :: waitForChannels :: gateRequest :: gateResponse :: inputRequest :: inputResponse :: calculate :: calculateDrain :: writeRequest :: writeResponse :: complete :: Nil = + Enum(11) + val state = RegInit(idle) + val robId = RegInit(0.U(log2Up(fp.rob_entries).W)) + val gateBank = RegInit(0.U(fp.bank_id_len.W)) + val inputBank = RegInit(0.U(fp.bank_id_len.W)) + val outputBank = RegInit(0.U(fp.bank_id_len.W)) + val groups = RegInit(0.U(5.W)) + val group = RegInit(0.U(5.W)) + val bases = Reg(Vec(3, UInt(mp.groupIdWidth.W))) + val iter = RegInit(0.U(fp.iter_len.W)) + val gateRow = RegInit(0.U(bankRowBits.W)) + val inputRow = RegInit(0.U(log2Ceil(mp.bankEntries).W)) + val ratio = Reg(UInt(32.W)) + val gateWord = Reg(UInt(128.W)) + val inputWord = Reg(UInt(128.W)) + val lane = RegInit(0.U(4.W)) + val outputWord = Reg(Vec(16, UInt(8.W))) + + val productReg = Reg(UInt(16.W)) + val productLaneReg = Reg(UInt(4.W)) + val productValid = RegInit(false.B) + val productFloatReg = Reg(UInt(33.W)) + val floatLaneReg = Reg(UInt(4.W)) + val floatValid = RegInit(false.B) + val scaledReg = Reg(UInt(33.W)) + val scaledLaneReg = Reg(UInt(4.W)) + val scaledValid = RegInit(false.B) + + def positiveFinite(value: UInt): Bool = + !value(31) && value(30, 23) =/= 255.U && value(30, 0) =/= 0.U + + val gateValue = (gateWord >> (lane << 3))(7, 0).asSInt + val inputValue = (inputWord >> (lane << 3))(7, 0).asSInt + val integerProduct = (gateValue * inputValue).asUInt + val productToFloat = Module(new INToRecFN(16, 8, 24)) + + val scale = Instantiate(new Fma(3, 8, 24)) + + val toInt8 = Module(new RecFNToIN(8, 24, 8)) + + productToFloat.io.signedIn := true.B + productToFloat.io.in := productReg + productToFloat.io.roundingMode := round_near_even + productToFloat.io.detectTininess := tininess_afterRounding + scale.io.validin := floatValid + scale.io.op := 0.U + scale.io.a := productFloatReg + scale.io.b := recFNFromFN(8, 24, ratio) + scale.io.c := recFNFromFN(8, 24, 0.U) + scale.io.roundingMode := round_near_even + scale.io.detectTininess := tininess_afterRounding.asUInt + toInt8.io.in := scaledReg + toInt8.io.roundingMode := round_near_even + toInt8.io.signedOut := true.B + + blink.cmdReq.ready := state === idle + blink.cmdResp.valid := state === complete + blink.cmdResp.bits.rob_id := robId + blink.status.idle := state === idle + blink.status.running := state =/= idle && state =/= complete + + for (port <- 0 until 2) { + blink.bankRead(port).rob_id := robId + blink.bankRead(port).bank_id := Mux(port.U === 0.U, gateBank, inputBank) + blink.bankRead(port).group_id := bases(port) + group + blink.bankRead(port).io.req.valid := false.B + blink.bankRead(port).io.req.bits.addr := Mux(port.U === 0.U, gateRow, inputRow) + blink.bankRead(port).io.resp.ready := false.B + } + + blink.bankWrite(0).rob_id := robId + blink.bankWrite(0).bank_id := outputBank + blink.bankWrite(0).group_id := bases(2) + group + blink.bankWrite(0).io.req.valid := false.B + blink.bankWrite(0).io.req.bits.addr := inputRow + blink.bankWrite(0).io.req.bits.data := Cat(outputWord.reverse) + blink.bankWrite(0).io.req.bits.mask := VecInit(Seq.fill(16)(true.B)) + blink.bankWrite(0).io.resp.ready := false.B + + switch(state) { + is(idle) { + when(blink.cmdReq.fire) { + val cmd = blink.cmdReq.bits + + if (bankRowBits < 32) + assert( + cmd.rs2(63, 32 + bankRowBits) === 0.U, + s"Int8MulBall reserves rs2 bits 63:${32 + bankRowBits}" + ) + assert(positiveFinite(cmd.rs2(31, 0)), "Int8MulBall ratio must be finite and positive") + assert( + cmd.rs2(31 + bankRowBits, 32) < mp.bankEntries.U, + "Int8MulBall gate_row must fit one physical bank" + ) + assert( + cmd.op1_col =/= 0.U && cmd.op1_col === cmd.op2_col && cmd.op1_col === cmd.wr_col, + "Int8MulBall bank groups must match" + ) + assert( + cmd.op1_col <= mp.bankNum.U, + "Int8MulBall bank groups exceed physical banks" + ) + assert( + cmd.op1_bank =/= cmd.op2_bank && cmd.op1_bank =/= cmd.wr_bank && cmd.op2_bank =/= cmd.wr_bank, + "Int8MulBall banks must be distinct" + ) + assert( + cmd.iter =/= 0.U && cmd.iter <= mp.bankEntries.U, + "Int8MulBall iter must fit one physical bank" + ) + robId := blink.cmdReq.bits.rob_id + gateBank := cmd.op1_bank + inputBank := cmd.op2_bank + outputBank := cmd.wr_bank + groups := cmd.op1_col + group := 0.U + bases := VecInit(cmd.op1_group, cmd.op2_group, cmd.wr_group) + iter := cmd.iter + gateRow := cmd.rs2(31 + bankRowBits, 32) + inputRow := 0.U + ratio := cmd.rs2(31, 0) + lane := 0.U + state := waitForChannels + } + } + is(waitForChannels) { + state := gateRequest + } + is(gateRequest) { + blink.bankRead(0).io.req.valid := true.B + when(blink.bankRead(0).io.req.fire)(state := gateResponse) + } + is(gateResponse) { + blink.bankRead(0).io.resp.ready := true.B + when(blink.bankRead(0).io.resp.fire) { + gateWord := blink.bankRead(0).io.resp.bits.data + state := inputRequest + } + } + is(inputRequest) { + blink.bankRead(1).io.req.valid := true.B + when(blink.bankRead(1).io.req.fire)(state := inputResponse) + } + is(inputResponse) { + blink.bankRead(1).io.resp.ready := true.B + when(blink.bankRead(1).io.resp.fire) { + inputWord := blink.bankRead(1).io.resp.bits.data + lane := 0.U + state := calculate + } + } + is(calculate) { + productReg := integerProduct + productLaneReg := lane + productValid := true.B + when(lane === 15.U)(state := calculateDrain) + .otherwise(lane := lane + 1.U) + } + is(calculateDrain) { + when(scaledValid && scaledLaneReg === 15.U)(state := writeRequest) + } + is(writeRequest) { + blink.bankWrite(0).io.req.valid := true.B + when(blink.bankWrite(0).io.req.fire)(state := writeResponse) + } + is(writeResponse) { + blink.bankWrite(0).io.resp.ready := true.B + when(blink.bankWrite(0).io.resp.fire) { + when(inputRow +& 1.U === iter) { + inputRow := 0.U + when(group +& 1.U === groups)(state := complete) + .otherwise { group := group + 1.U; state := gateRequest } + }.otherwise { inputRow := inputRow + 1.U; state := inputRequest } + } + } + is(complete) { + when(blink.cmdResp.fire)(state := idle) + } + } + + when(productValid) { + productFloatReg := productToFloat.io.out + floatLaneReg := productLaneReg + } + when(scale.io.validout) { + scaledReg := scale.io.out + scaledLaneReg := Pipe(floatValid, floatLaneReg, 3).bits + } + when(scaledValid) { + outputWord(scaledLaneReg) := toInt8.io.out + } + + productValid := state === calculate + floatValid := productValid + scaledValid := scale.io.validout +} + +object Int8MulBall extends BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts = + BallPorts(Instantiate(new Int8MulBall(fp, mp, bp, id)).blink, None) + +} diff --git a/examples/prefab/balls/int8mul/configs/default.toml b/examples/prefab/balls/int8mul/configs/default.toml new file mode 100644 index 000000000..5db85e8e7 --- /dev/null +++ b/examples/prefab/balls/int8mul/configs/default.toml @@ -0,0 +1 @@ +# Int8MulBall has no private configuration. diff --git a/examples/prefab/balls/int8mul/configs/int8mul_params.proto b/examples/prefab/balls/int8mul/configs/int8mul_params.proto new file mode 100644 index 000000000..daba4d0ae --- /dev/null +++ b/examples/prefab/balls/int8mul/configs/int8mul_params.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; +package buckyball.balls.int8mul; +option java_package = "buckyball.balls.int8mul"; +option java_multiple_files = true; + +message Params { +} diff --git a/examples/prefab/balls/int8mul/emu/src/dispatch.rs b/examples/prefab/balls/int8mul/emu/src/dispatch.rs new file mode 100644 index 000000000..19e733f52 --- /dev/null +++ b/examples/prefab/balls/int8mul/emu/src/dispatch.rs @@ -0,0 +1,31 @@ +use bebop_bemu::ball::ExecContext; + +use super::int8mul; + +const BALL_CLASS: &str = "examples.balls.int8mul.Int8MulBall"; + +pub fn execute_known( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, + ctx: &mut ExecContext, +) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("INT8MUL") => Some(int8mul::Int8Mul::exec(xs1, xs2, ctx)), + Some(_) | None => None, + } +} + +pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("INT8MUL") => Some(int8mul::Int8Mul::latency(xs1, xs2)), + Some(_) | None => None, + } +} diff --git a/examples/balls/int8mul/emu/src/int8mul.rs b/examples/prefab/balls/int8mul/emu/src/int8mul.rs similarity index 95% rename from examples/balls/int8mul/emu/src/int8mul.rs rename to examples/prefab/balls/int8mul/emu/src/int8mul.rs index 92b073cd1..bfe7822f3 100644 --- a/examples/balls/int8mul/emu/src/int8mul.rs +++ b/examples/prefab/balls/int8mul/emu/src/int8mul.rs @@ -1,5 +1,5 @@ -use super::super::bank::{bank_lines, bank_row_bytes}; -use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; +use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; +use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old}; use super::instruction::ExecContext; pub struct Int8Mul; @@ -32,6 +32,7 @@ impl Int8Mul { } pub fn exec(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + assert!(!rs1_read_old(xs1), "int8mul: reads no old destination; rs1[29] must be clear"); let gate_bank = rs1_b0(xs1); let input_bank = rs1_b1(xs1); let output_bank = rs1_b2(xs1); diff --git a/examples/prefab/balls/int8mul/emu/src/lib.rs b/examples/prefab/balls/int8mul/emu/src/lib.rs new file mode 100644 index 000000000..3949cf522 --- /dev/null +++ b/examples/prefab/balls/int8mul/emu/src/lib.rs @@ -0,0 +1,11 @@ +pub(crate) use bebop_bemu::{thread::instructions::decode, ball as instruction}; + +mod int8mul; + +mod dispatch; + +pub use dispatch::{cycles_after_issue, execute_known}; + +#[bebop_bemu::ball::linkme::distributed_slice(bebop_bemu::BALLS)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static BALL: bebop_bemu::Ball = bebop_bemu::Ball { execute: execute_known, cycles: cycles_after_issue }; diff --git a/examples/prefab/balls/int8mul/stack/src/int8mulOps.cpp b/examples/prefab/balls/int8mul/stack/src/int8mulOps.cpp new file mode 100644 index 000000000..2b0f0fb58 --- /dev/null +++ b/examples/prefab/balls/int8mul/stack/src/int8mulOps.cpp @@ -0,0 +1,75 @@ +//===- int8mulOps.cpp - Int8MulBall operations ----------------------------===// +// +// The dialect of Int8MulBall, its virtual operations building their physical +// ones and the instructions of the physical operations. +// +//===----------------------------------------------------------------------===// + +#include "int8mulOps.h" + +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +#include "mlir/Dialect/Arith/IR/Arith.h" + +using namespace mlir; +using namespace ::buddy::int8mul_ball; +using mlir::buddy::bankFields; +using mlir::buddy::constantI64; +using mlir::buddy::emitInstruction; +using mlir::buddy::field; + +#include "int8mulDialect.cpp.inc" + +#define GET_OP_CLASSES +#include "int8mulOps.cpp.inc" + +void Int8MulBallDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "int8mulOps.cpp.inc" + >(); +} + +namespace { +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallDialect([](DialectRegistry ®istry) { + registry.insert(); + }), + true); +} // namespace + +SmallVector BankGateMulI8Op::getSourceBanks() { + return {getGate(), getInput()}; +} +Value BankGateMulI8Op::getDestinationBank() { return getOutput(); } +bool BankGateMulI8Op::readsDestination() { return false; } +Operation *BankGateMulI8Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return GateMulI8Op::create(builder, getLoc(), getRatio(), sources[0], + sources[1], destination, getRows()); +} + +SmallVector GateMulI8Op::getSourceBanks() { + return {int64_t(getGate()), int64_t(getInput())}; +} +int64_t GateMulI8Op::getDestinationBank() { return getOutput(); } +LogicalResult GateMulI8Op::lowerToInstructions(OpBuilder &builder) { + Location loc = getLoc(); + int64_t rows = getRows(), gate = getGate(), input = getInput(), + output = getOutput(); + if (rows <= 0 || rows > buckyball_target::getBuckyballTarget().bankDepth || + gate == input || gate == output || input == output) + return emitOpError( + "gate_mul_i8 requires distinct banks and rows within a bank"); + // rs2: ratio bits in [31:0]; the gate row in [63:32] is the first row. + Value bits = + arith::BitcastOp::create(builder, loc, builder.getI32Type(), getRatio()); + Value rs2 = arith::ExtUIOp::create(builder, loc, builder.getI64Type(), bits); + emitInstruction( + builder, loc, buckyball_target::getBuckyballFunct7("INT8MUL"), + constantI64(builder, loc, bankFields({gate, input}, output, false, rows)), + rs2); + return success(); +} diff --git a/examples/prefab/balls/int8mul/stack/src/int8mulOps.h b/examples/prefab/balls/int8mul/stack/src/int8mulOps.h new file mode 100644 index 000000000..8143c2631 --- /dev/null +++ b/examples/prefab/balls/int8mul/stack/src/int8mulOps.h @@ -0,0 +1,20 @@ +//===- int8mulOps.h - Int8MulBall operations ------------------------------===// + +#ifndef INT8MUL_OPS_H +#define INT8MUL_OPS_H + +#include "Interfaces/BankInterfaces.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "int8mulDialect.h.inc" + +#define GET_OP_CLASSES +#include "int8mulOps.h.inc" + +#endif // INT8MUL_OPS_H diff --git a/examples/prefab/balls/int8mul/stack/src/int8mulOps.td b/examples/prefab/balls/int8mul/stack/src/int8mulOps.td new file mode 100644 index 000000000..0d56c5034 --- /dev/null +++ b/examples/prefab/balls/int8mul/stack/src/int8mulOps.td @@ -0,0 +1,46 @@ +//===- int8mulOps.td - Int8MulBall operations --------------*- tablegen -*-===// +// +// gate_mul_i8: `gate` holds the gate, one row of sixteen i8; `input` holds +// `rows` rows of sixteen i8. The output receives +// clamp(round_half_even(gate * x * ratio), -128, 127) per column, in every +// group of the banks; the three banks have the same number of groups. `ratio` +// is f32, finite and positive. +// +//===----------------------------------------------------------------------===// + +#ifndef INT8MUL_OPS +#define INT8MUL_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Interfaces/BankInterfaces.td" + +def Int8MulBall_Dialect : Dialect { + let name = "int8mul_ball"; + let cppNamespace = "::buddy::int8mul_ball"; + let summary = "The operations of Int8MulBall"; +} + +class Int8MulBall_Op traits = []> + : Op])>; + +def BankGateMulI8Op : Int8MulBall_Op<"bank_gate_mul_i8", + [DeclareOpInterfaceMethods]> { + let summary = "gate_mul_i8 on virtual banks"; + let arguments = (ins I64:$gate, I64:$input, I64:$output, F32:$ratio, + I64Attr:$rows); + let results = (outs I64:$outputOut); + let assemblyFormat = "$gate `,` $input `->` $output `ratio` `(` $ratio `)` " + "prop-dict attr-dict"; +} + +def GateMulI8Op : Int8MulBall_Op<"gate_mul_i8", + [DeclareOpInterfaceMethods]> { + let summary = "gate_mul_i8 on physical banks"; + let arguments = (ins F32:$ratio, I64Attr:$gate, I64Attr:$input, + I64Attr:$output, I64Attr:$rows); + let assemblyFormat = "`ratio` `(` $ratio `)` prop-dict attr-dict"; +} + +#endif // INT8MUL_OPS diff --git a/examples/balls/int8mul/verify/casegen/Cargo.lock b/examples/prefab/balls/int8mul/verify/casegen/Cargo.lock similarity index 100% rename from examples/balls/int8mul/verify/casegen/Cargo.lock rename to examples/prefab/balls/int8mul/verify/casegen/Cargo.lock diff --git a/examples/balls/int8mul/verify/casegen/Cargo.toml b/examples/prefab/balls/int8mul/verify/casegen/Cargo.toml similarity index 100% rename from examples/balls/int8mul/verify/casegen/Cargo.toml rename to examples/prefab/balls/int8mul/verify/casegen/Cargo.toml diff --git a/examples/prefab/balls/int8mul/verify/casegen/src/casegen.rs b/examples/prefab/balls/int8mul/verify/casegen/src/casegen.rs new file mode 100644 index 000000000..d041fb80d --- /dev/null +++ b/examples/prefab/balls/int8mul/verify/casegen/src/casegen.rs @@ -0,0 +1,254 @@ +pub const ROW_BYTES: usize = 16; +pub const MAX_ROWS: usize = 64; +pub const NUM_CASES: u32 = 4; +pub const GATE_ROW: u32 = 1; +pub const RATIO: f32 = 0.25; + +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Int8MulCmd { + pub iter: u32, + pub gate_bank: u32, + pub input_bank: u32, + pub output_bank: u32, + pub op1_group: u32, + pub op2_group: u32, + pub wr_group: u32, + pub op1_col: u32, + pub op2_col: u32, + pub wr_col: u32, + pub gate_row: u32, + pub bank_entries: u32, + pub rob_id: u32, + pub rs2_lo: u32, + pub rs2_hi: u32, + pub num_gate_words: u32, + pub num_input_words: u32, + pub num_dst_words: u32, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Int8MulCase { + pub cmd: Int8MulCmd, + pub gate_words: [u128; MAX_ROWS], + pub input_words: [u128; MAX_ROWS], + pub dst_words: [u128; MAX_ROWS], +} + +impl Int8MulCase { + pub fn gate_lo(&self, index: usize) -> u64 { + self.gate_words[index] as u64 + } + + pub fn gate_hi(&self, index: usize) -> u64 { + (self.gate_words[index] >> 64) as u64 + } + + pub fn input_lo(&self, index: usize) -> u64 { + self.input_words[index] as u64 + } + + pub fn input_hi(&self, index: usize) -> u64 { + (self.input_words[index] >> 64) as u64 + } + + pub fn dst_lo(&self, index: usize) -> u64 { + self.dst_words[index] as u64 + } + + pub fn dst_hi(&self, index: usize) -> u64 { + (self.dst_words[index] >> 64) as u64 + } +} + +// Gate word `g` is row `gate_row` of group `g`; input/output word `i` is row `i % iter` of +// group `i / iter`: the Ball walks groups in order and reloads the gate per group. +pub fn gen_case(index: u32, bank_entries: u32) -> Int8MulCase { + if bank_entries == 0 { + panic!("int8mul: bank entries must be positive"); + } + if index >= NUM_CASES { + panic!("int8mul: unsupported directed case {index}"); + } + match index { + 0 => directed(2, 1, GATE_ROW, 1, [0, 0, 0], bank_entries), + 1 => directed(3, 4, GATE_ROW, 1, [0, 0, 0], bank_entries), + 2 => directed(4, 1, bank_entries - 1, 1, [0, 0, 0], bank_entries), + 3 => directed(5, 4, GATE_ROW, 2, [2, 1, 3], bank_entries), + _ => panic!("int8mul: unsupported directed case {index}"), + } +} + +fn compute(gate: i8, input: i8, ratio: f32) -> i8 { + let value = (gate as f32) * (input as f32) * ratio; + value.round_ties_even().clamp(-128.0, 127.0) as i8 +} + +fn positive_finite(value: f32) -> bool { + value.is_finite() && value > 0.0 +} + +fn ctest_gate(group: u32) -> u128 { + let mut row = [0u8; ROW_BYTES]; + for lane in 0..ROW_BYTES { + row[lane] = (lane as i8 - 8 + group as i8) as u8; + } + pack_bytes(&row)[0] +} + +fn ctest_input(rows: usize) -> Vec { + let count = rows * ROW_BYTES; + let bytes: Vec = (0..count) + .map(|i| ((i as i32 * 37 & 255) - 128) as i8 as u8) + .collect(); + pack_bytes(&bytes) +} + +fn pack_bytes(bytes: &[u8]) -> Vec { + if bytes.len() % ROW_BYTES != 0 { + panic!( + "int8mul: {} bytes is not a multiple of {ROW_BYTES}", + bytes.len() + ); + } + bytes + .chunks(ROW_BYTES) + .map(|chunk| { + let mut word = 0u128; + for (lane, byte) in chunk.iter().enumerate() { + word |= u128::from(*byte) << (lane * 8); + } + word + }) + .collect() +} + +fn copy_words(src: &[u128]) -> [u128; N] { + if src.len() > N { + panic!("int8mul: {} words exceed buffer {N}", src.len()); + } + let mut out = [0u128; N]; + out[..src.len()].copy_from_slice(src); + out +} + +fn encode_rs2(ratio: f32, gate_row: u32) -> u64 { + u64::from(ratio.to_bits()) | (u64::from(gate_row) << 32) +} + +fn split_rs(value: u64) -> (u32, u32) { + (value as u32, (value >> 32) as u32) +} + +pub(crate) fn validate( + gate_bank: u32, + input_bank: u32, + output_bank: u32, + op1_col: u32, + op2_col: u32, + wr_col: u32, + iter: u32, + gate_row: u32, + rs2: u64, + ratio: f32, + bank_entries: u32, +) { + if bank_entries == 0 { + panic!("int8mul: bank entries must be positive"); + } + if iter == 0 || iter > bank_entries { + panic!("int8mul: iter must fit one physical bank"); + } + if gate_row >= bank_entries { + panic!("int8mul: gate row must fit one physical bank"); + } + if op1_col == 0 || op1_col != op2_col || op1_col != wr_col { + panic!("int8mul: bank groups must match"); + } + if gate_bank == input_bank || gate_bank == output_bank || input_bank == output_bank { + panic!("int8mul: banks must be distinct"); + } + if !positive_finite(ratio) { + panic!("int8mul: ratio must be finite and positive"); + } + let gate_row_bits = u32::BITS - (bank_entries - 1).leading_zeros(); + if rs2 >> (32 + gate_row_bits) != 0 { + panic!("int8mul: reserved rs2 bits must be zero"); + } + if rs2 != encode_rs2(ratio, gate_row) { + panic!("int8mul: rs2 ratio/gate_row fields mismatch"); + } +} + +fn dst_words(gate: &[u128], input: &[u128], iter: u32, ratio: f32) -> Vec { + let mut out = Vec::with_capacity(input.len()); + for (word, input_word) in input.iter().enumerate() { + let gate_word = gate[word / iter as usize]; + let mut packed = 0u128; + for lane in 0..ROW_BYTES { + let g = ((gate_word >> (lane * 8)) & 0xff) as u8 as i8; + let inp = ((input_word >> (lane * 8)) & 0xff) as u8 as i8; + let q = compute(g, inp, ratio); + packed |= u128::from(q as u8) << (lane * 8); + } + out.push(packed); + } + out +} + +fn directed( + rob_id: u32, + iter: u32, + gate_row: u32, + cols: u32, + groups: [u32; 3], + bank_entries: u32, +) -> Int8MulCase { + let gate_bank = 0; + let input_bank = 1; + let output_bank = 2; + let ratio = RATIO; + let gate: Vec = (0..cols).map(ctest_gate).collect(); + let input = ctest_input((iter * cols) as usize); + let rs2 = encode_rs2(ratio, gate_row); + validate( + gate_bank, + input_bank, + output_bank, + cols, + cols, + cols, + iter, + gate_row, + rs2, + ratio, + bank_entries, + ); + let dst = dst_words(&gate, &input, iter, ratio); + let (rs2_lo, rs2_hi) = split_rs(rs2); + Int8MulCase { + cmd: Int8MulCmd { + iter, + gate_bank, + input_bank, + output_bank, + op1_group: groups[0], + op2_group: groups[1], + wr_group: groups[2], + op1_col: cols, + op2_col: cols, + wr_col: cols, + gate_row, + bank_entries, + rob_id, + rs2_lo, + rs2_hi, + num_gate_words: gate.len() as u32, + num_input_words: input.len() as u32, + num_dst_words: dst.len() as u32, + }, + gate_words: copy_words(&gate), + input_words: copy_words(&input), + dst_words: copy_words(&dst), + } +} diff --git a/examples/balls/int8mul/verify/casegen/src/config.rs b/examples/prefab/balls/int8mul/verify/casegen/src/config.rs similarity index 100% rename from examples/balls/int8mul/verify/casegen/src/config.rs rename to examples/prefab/balls/int8mul/verify/casegen/src/config.rs diff --git a/examples/prefab/balls/int8mul/verify/casegen/src/lib.rs b/examples/prefab/balls/int8mul/verify/casegen/src/lib.rs new file mode 100644 index 000000000..e995aab7b --- /dev/null +++ b/examples/prefab/balls/int8mul/verify/casegen/src/lib.rs @@ -0,0 +1,71 @@ +mod casegen; +mod config; + +use std::cell::RefCell; + +use casegen::{Int8MulCase, Int8MulCmd}; + +thread_local! { + static CURRENT: RefCell> = const { RefCell::new(None) }; +} + +#[no_mangle] +pub extern "C" fn int8mul_case_load(index: u32) -> i32 { + let case = casegen::gen_case(index, config::bank_entries()); + CURRENT.with(|current| *current.borrow_mut() = Some(case)); + 0 +} + +fn current_case R, R>(f: F) -> R { + CURRENT.with(|current| match current.borrow().as_ref() { + Some(case) => f(case), + None => panic!("int8mul_case: no case loaded; call int8mul_case_load first"), + }) +} + +#[no_mangle] +pub extern "C" fn int8mul_case_cmd(out_ptr: *mut Int8MulCmd) { + if out_ptr.is_null() { + panic!("int8mul_case_cmd: null out_ptr"); + } + current_case(|case| unsafe { + *out_ptr = case.cmd; + }); +} + +fn idx(index: u32, limit: u32, what: &str) -> usize { + if index >= limit { + panic!("int8mul_case_{what}: index {index} out of range {limit}"); + } + index as usize +} + +#[no_mangle] +pub extern "C" fn int8mul_case_gate_word_lo(word_index: u32) -> u64 { + current_case(|case| case.gate_lo(idx(word_index, case.cmd.num_gate_words, "gate_word_lo"))) +} + +#[no_mangle] +pub extern "C" fn int8mul_case_gate_word_hi(word_index: u32) -> u64 { + current_case(|case| case.gate_hi(idx(word_index, case.cmd.num_gate_words, "gate_word_hi"))) +} + +#[no_mangle] +pub extern "C" fn int8mul_case_input_word_lo(word_index: u32) -> u64 { + current_case(|case| case.input_lo(idx(word_index, case.cmd.num_input_words, "input_word_lo"))) +} + +#[no_mangle] +pub extern "C" fn int8mul_case_input_word_hi(word_index: u32) -> u64 { + current_case(|case| case.input_hi(idx(word_index, case.cmd.num_input_words, "input_word_hi"))) +} + +#[no_mangle] +pub extern "C" fn int8mul_case_dst_word_lo(word_index: u32) -> u64 { + current_case(|case| case.dst_lo(idx(word_index, case.cmd.num_dst_words, "dst_word_lo"))) +} + +#[no_mangle] +pub extern "C" fn int8mul_case_dst_word_hi(word_index: u32) -> u64 { + current_case(|case| case.dst_hi(idx(word_index, case.cmd.num_dst_words, "dst_word_hi"))) +} diff --git a/examples/balls/int8mul/verify/filelists/int8mul_ball.f b/examples/prefab/balls/int8mul/verify/filelists/int8mul_ball.f similarity index 90% rename from examples/balls/int8mul/verify/filelists/int8mul_ball.f rename to examples/prefab/balls/int8mul/verify/filelists/int8mul_ball.f index db2d15f7d..89c6cfa4d 100644 --- a/examples/balls/int8mul/verify/filelists/int8mul_ball.f +++ b/examples/prefab/balls/int8mul/verify/filelists/int8mul_ball.f @@ -1,3 +1,4 @@ ++incdir+@UVM@/src/core +incdir+@UVM@/src/ball +incdir+@UVM@/src/ball/agents/cmd +incdir+@UVM@/src/ball/agents/mem @@ -13,16 +14,17 @@ +incdir+src/tests +incdir+@RTL@ +@UVM@/src/core/package.sv @UVM@/src/ball/bb_blink_if.sv @UVM@/src/ball/bb_uvm_pkg.sv @RTL@/RoundAnyRawFNToRecFN_ie5_is16_oe8_os24.sv -@RTL@/RoundAnyRawFNToRecFN_ie8_is26_oe8_os24.sv @RTL@/INToRecFN_i16_e8_s24.sv @RTL@/MulAddRecFNToRaw_preMul_e8_s24.sv @RTL@/MulAddRecFNToRaw_postMul_e8_s24.sv -@RTL@/MulAddRecFNPipe_l3_e8_s24.sv +@RTL@/RoundAnyRawFNToRecFN_ie8_is26_oe8_os24.sv @RTL@/RoundRawFNToRecFN_e8_s24.sv +@RTL@/Fma_l3_e8_s24.sv @RTL@/RecFNToIN_e8_s24_i8.sv @RTL@/Int8MulBall.sv diff --git a/examples/prefab/balls/int8mul/verify/src/common/int8mul_defs.svh b/examples/prefab/balls/int8mul/verify/src/common/int8mul_defs.svh new file mode 100644 index 000000000..7999486e3 --- /dev/null +++ b/examples/prefab/balls/int8mul/verify/src/common/int8mul_defs.svh @@ -0,0 +1,38 @@ +typedef struct { + int unsigned iter; + int unsigned gate_bank; + int unsigned input_bank; + int unsigned output_bank; + int unsigned op1_group; + int unsigned op2_group; + int unsigned wr_group; + int unsigned op1_col; + int unsigned op2_col; + int unsigned wr_col; + int unsigned gate_row; + int unsigned bank_entries; + int unsigned rob_id; + int unsigned rs2_lo; + int unsigned rs2_hi; + int unsigned num_gate_words; + int unsigned num_input_words; + int unsigned num_dst_words; +} int8mul_cmd_dpi_t; + +import "DPI-C" function int int8mul_case_load(input int unsigned index); +import "DPI-C" function void int8mul_case_cmd(output int8mul_cmd_dpi_t cmd); +import "DPI-C" function longint unsigned int8mul_case_gate_word_lo(input int unsigned word_index); +import "DPI-C" function longint unsigned int8mul_case_gate_word_hi(input int unsigned word_index); +import "DPI-C" function longint unsigned int8mul_case_input_word_lo(input int unsigned word_index); +import "DPI-C" function longint unsigned int8mul_case_input_word_hi(input int unsigned word_index); +import "DPI-C" function longint unsigned int8mul_case_dst_word_lo(input int unsigned word_index); +import "DPI-C" function longint unsigned int8mul_case_dst_word_hi(input int unsigned word_index); + +`ifndef INT8MUL_FUNCT7 +`error "INT8MUL_FUNCT7 must be provided by the selected Thread ballISA" +`endif + +localparam int INT8MUL_THREAD_FUNCT7 = `INT8MUL_FUNCT7; +localparam int INT8MUL_MAX_ROWS = 64; +localparam int INT8MUL_NUM_CASES = 4; +localparam int INT8MUL_TIMEOUT_CYCLES = 20000; diff --git a/examples/prefab/balls/int8mul/verify/src/common/int8mul_items.svh b/examples/prefab/balls/int8mul/verify/src/common/int8mul_items.svh new file mode 100644 index 000000000..06a727bc4 --- /dev/null +++ b/examples/prefab/balls/int8mul/verify/src/common/int8mul_items.svh @@ -0,0 +1,104 @@ +class int8mul_cmd_item extends bb_blink_cmd_item; + `uvm_object_utils(int8mul_cmd_item) + + bit [31:0] gate_row; + bit [31:0] bank_entries; + bit [31:0] num_gate_words; + bit [31:0] num_input_words; + bit [31:0] num_dst_words; + bit [127:0] gate_words[INT8MUL_MAX_ROWS]; + bit [127:0] input_words[INT8MUL_MAX_ROWS]; + bit [127:0] dst_words[INT8MUL_MAX_ROWS]; + + function new(string name = "int8mul_cmd_item"); + super.new(name); + endfunction + + function void load_rust_case(int unsigned index); + int8mul_cmd_dpi_t cmd; + int unsigned rc; + longint unsigned w_lo; + longint unsigned w_hi; + int i; + + rc = int8mul_case_load(index); + if (rc != 0) begin + `uvm_fatal("CASE", $sformatf("int8mul_case_load returned %0d for index %0d", rc, index)) + end + int8mul_case_cmd(cmd); + + // bb_int8mul: banks and iter come from rs1, ratio and gate_row from rs2. + rob_id = cmd.rob_id[BB_BLINK_ROB_ID_W-1:0]; + funct = INT8MUL_THREAD_FUNCT7[6:0]; + rs2 = {cmd.rs2_hi, cmd.rs2_lo}; + iter = cmd.iter; + op1_bank = cmd.gate_bank; + op2_bank = cmd.input_bank; + wr_bank = cmd.output_bank; + op1_group = cmd.op1_group; + op2_group = cmd.op2_group; + wr_group = cmd.wr_group; + op1_col = cmd.op1_col; + op2_col = cmd.op2_col; + wr_col = cmd.wr_col; + wr_read_bank = '0; + wr_arch_bank = cmd.output_bank; + gate_row = cmd.gate_row; + bank_entries = cmd.bank_entries; + num_gate_words = cmd.num_gate_words; + num_input_words = cmd.num_input_words; + num_dst_words = cmd.num_dst_words; + + if (iter != 1 && iter != 4) + `uvm_fatal("CASE", $sformatf("Int8MulBall iter %0d is not in {1,4}", iter)) + if (op1_col == 0 || op1_col != op2_col || op1_col != wr_col) + `uvm_fatal("CASE", "Int8MulBall bank groups must match") + if (op1_bank == op2_bank || op1_bank == wr_bank || op2_bank == wr_bank) + `uvm_fatal("CASE", "Int8MulBall banks must be distinct") + if (bank_entries == 0) `uvm_fatal("CASE", "Int8MulBall bank_entries must be positive") + if (gate_row >= bank_entries) + `uvm_fatal("CASE", $sformatf( + "Int8MulBall gate_row %0d exceeds bank_entries %0d", gate_row, bank_entries)) + if ((rs2 >> (32 + $clog2(bank_entries))) != 0) + `uvm_fatal("CASE", $sformatf("Int8MulBall reserved rs2 bits set: 0x%016h", rs2)) + if (num_gate_words != op1_col) + `uvm_fatal("CASE", $sformatf("num_gate_words %0d != col %0d", num_gate_words, op1_col)) + if (num_input_words != iter * op1_col || num_dst_words != iter * op1_col) + `uvm_fatal( + "CASE", $sformatf( + "word counts %0d/%0d != iter %0d * col %0d", num_input_words, num_dst_words, iter, op1_col + )) + + for (i = 0; i < num_gate_words; i++) begin + w_lo = int8mul_case_gate_word_lo(i); + w_hi = int8mul_case_gate_word_hi(i); + gate_words[i] = {w_hi, w_lo}; + end + for (i = 0; i < num_input_words; i++) begin + w_lo = int8mul_case_input_word_lo(i); + w_hi = int8mul_case_input_word_hi(i); + input_words[i] = {w_hi, w_lo}; + end + for (i = 0; i < num_dst_words; i++) begin + w_lo = int8mul_case_dst_word_lo(i); + w_hi = int8mul_case_dst_word_hi(i); + dst_words[i] = {w_hi, w_lo}; + end + endfunction + + function void do_copy(uvm_object rhs); + int8mul_cmd_item rhs_; + super.do_copy(rhs); + if (!$cast(rhs_, rhs)) begin + `uvm_fatal("COPY", "rhs is not int8mul_cmd_item") + end + gate_row = rhs_.gate_row; + bank_entries = rhs_.bank_entries; + num_gate_words = rhs_.num_gate_words; + num_input_words = rhs_.num_input_words; + num_dst_words = rhs_.num_dst_words; + for (int i = 0; i < INT8MUL_MAX_ROWS; i++) gate_words[i] = rhs_.gate_words[i]; + for (int i = 0; i < INT8MUL_MAX_ROWS; i++) input_words[i] = rhs_.input_words[i]; + for (int i = 0; i < INT8MUL_MAX_ROWS; i++) dst_words[i] = rhs_.dst_words[i]; + endfunction +endclass diff --git a/examples/balls/int8mul/verify/src/cov/int8mul_cov.svh b/examples/prefab/balls/int8mul/verify/src/cov/int8mul_cov.svh similarity index 75% rename from examples/balls/int8mul/verify/src/cov/int8mul_cov.svh rename to examples/prefab/balls/int8mul/verify/src/cov/int8mul_cov.svh index 50034c541..0b2dd0566 100644 --- a/examples/balls/int8mul/verify/src/cov/int8mul_cov.svh +++ b/examples/prefab/balls/int8mul/verify/src/cov/int8mul_cov.svh @@ -4,9 +4,11 @@ class int8mul_cov extends uvm_component; uvm_analysis_imp_cmd #(bb_blink_cmd_item, int8mul_cov) cmd_imp; int unsigned cur_iter; + int unsigned cur_col; covergroup cmd_cg; coverpoint cur_iter {bins i1 = {1}; bins i4 = {4};} + coverpoint cur_col {bins c1 = {1}; bins c2 = {2};} endgroup int cmd_count; @@ -18,14 +20,16 @@ class int8mul_cov extends uvm_component; endfunction function void write_cmd(bb_blink_cmd_item item); - if (item.funct7 != INT8MUL_CORE_FUNCT7[6:0]) - `uvm_fatal("COV", $sformatf("unexpected funct7 %0d", item.funct7)) + if (item.funct != INT8MUL_THREAD_FUNCT7[6:0]) + `uvm_fatal("COV", $sformatf("unexpected funct %0d", item.funct)) cur_iter = item.iter; + cur_col = item.op1_col; cmd_count++; cmd_cg.sample(); endfunction function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; super.check_phase(phase); if (cmd_count == 0) `uvm_fatal("COV", "no cmd observed") if (cmd_cg.get_coverage() != 100.0) diff --git a/examples/balls/int8mul/verify/src/env/int8mul_env.svh b/examples/prefab/balls/int8mul/verify/src/env/int8mul_env.svh similarity index 100% rename from examples/balls/int8mul/verify/src/env/int8mul_env.svh rename to examples/prefab/balls/int8mul/verify/src/env/int8mul_env.svh diff --git a/examples/prefab/balls/int8mul/verify/src/env/int8mul_scoreboard.svh b/examples/prefab/balls/int8mul/verify/src/env/int8mul_scoreboard.svh new file mode 100644 index 000000000..bfb0b5406 --- /dev/null +++ b/examples/prefab/balls/int8mul/verify/src/env/int8mul_scoreboard.svh @@ -0,0 +1,190 @@ +class int8mul_scoreboard extends uvm_scoreboard; + `uvm_component_utils(int8mul_scoreboard) + + uvm_analysis_imp_stim #(bb_blink_cmd_item, int8mul_scoreboard) stim_imp; + uvm_analysis_imp_cmd #(bb_blink_cmd_item, int8mul_scoreboard) cmd_imp; + uvm_analysis_imp_read #(bb_blink_read_item, int8mul_scoreboard) read_imp; + uvm_analysis_imp_write #(bb_blink_write_item, int8mul_scoreboard) write_imp; + uvm_analysis_imp_resp #(bb_blink_resp_item, int8mul_scoreboard) resp_imp; + + bb_blink_mem_model #(`BB_IN_BW, `BB_OUT_BW) mem_model; + + int8mul_cmd_item stim_q[$]; + int unsigned expected_reads[`BB_IN_BW]; + int unsigned expected_writes; + int unsigned cmd_count; + int unsigned read_count[`BB_IN_BW]; + int unsigned write_count; + int unsigned resp_count; + + function new(string name, uvm_component parent); + super.new(name, parent); + stim_imp = new("stim_imp", this); + cmd_imp = new("cmd_imp", this); + read_imp = new("read_imp", this); + write_imp = new("write_imp", this); + resp_imp = new("resp_imp", this); + endfunction + + function void write_stim(bb_blink_cmd_item item); + int8mul_cmd_item got; + int8mul_cmd_item clone; + int i; + + if (stim_q.size() != 0) begin + `uvm_fatal("SCB", "single outstanding command supported") + end + if (!$cast(got, item)) begin + `uvm_fatal("SCB", "stim item is not int8mul_cmd_item") + end + if (!$cast(clone, got.clone())) begin + `uvm_fatal("SCB", "failed to clone stimulus item") + end + stim_q.push_back(clone); + + if (mem_model == null) begin + `uvm_fatal("SCB", "mem_model handle not set") + end + mem_model.clear_mem(); + // Gate word g lives at row gate_row of group base + g; input word i at row i % iter of + // group base + i / iter. + for (i = 0; i < clone.num_gate_words; i++) begin + mem_model.load_word_g(int'(clone.op1_bank), int'(clone.op1_group) + i, int'(clone.gate_row), + clone.gate_words[i]); + end + for (i = 0; i < clone.num_input_words; i++) begin + mem_model.load_word_g(int'(clone.op2_bank), int'(clone.op2_group) + i / clone.iter, + i % clone.iter, clone.input_words[i]); + end + mem_model.arm(); + + expected_writes = clone.num_dst_words; + expected_reads[0] = clone.num_gate_words; + expected_reads[1] = clone.num_input_words; + endfunction + + function void write_cmd(bb_blink_cmd_item item); + int8mul_cmd_item stim; + stim = current_stim("CMD"); + check_cmd(item, stim); + cmd_count++; + endfunction + + function void check_cmd(bb_blink_cmd_item got, int8mul_cmd_item exp); + // Compare as plain BLink commands; the comparer rejects differing types. + bb_blink_cmd_item want = bb_blink_cmd_item::type_id::create("want"); + want.copy(exp); + verification_pkg::evidence::checks++; + if (!got.compare(want)) + `uvm_fatal("CMD", $sformatf("command mismatch: got %s exp %s", got.sprint(), want.sprint())) + endfunction + + function void write_read(bb_blink_read_item item); + int8mul_cmd_item stim; + int unsigned expect_bank; + int unsigned expect_group; + int unsigned expect_addr; + + stim = current_stim("READ"); + if (item.port != 0 && item.port != 1) + `uvm_fatal("READ", $sformatf("Int8MulBall has two read ports, got port %0d", item.port)) + if (item.rob_id !== stim.rob_id) + `uvm_fatal("READ", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + + // The gate row is read once per group, before that group's input rows. + if (item.port == 0) begin + expect_bank = stim.op1_bank; + expect_group = stim.op1_group + read_count[0]; + expect_addr = stim.gate_row; + if (read_count[1] != read_count[0] * stim.iter) + `uvm_fatal("READ", "Int8MulBall gate read out of group order") + end else begin + expect_bank = stim.op2_bank; + expect_group = stim.op2_group + read_count[1] / stim.iter; + expect_addr = read_count[1] % stim.iter; + end + if (item.group_id !== expect_group[BB_BLINK_GROUP_ID_W-1:0]) + `uvm_fatal("READ", $sformatf("group mismatch: got %0d exp %0d", item.group_id, expect_group)) + if (item.bank_id !== expect_bank[BB_BLINK_BANK_ID_W-1:0]) + `uvm_fatal("READ", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, expect_bank)) + if (item.addr !== expect_addr[BB_BLINK_BANK_ADDR_W-1:0]) + `uvm_fatal("READ", $sformatf("addr mismatch: got %0d exp %0d", item.addr, expect_addr)) + read_count[item.port]++; + endfunction + + function void write_write(bb_blink_write_item item); + int8mul_cmd_item stim; + + stim = current_stim("WRITE"); + if (item.port != 0) + `uvm_fatal("WRITE", $sformatf("Int8MulBall has one write port, got port %0d", item.port)) + if (item.bank_id !== stim.wr_bank) + `uvm_fatal("WRITE", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.wr_bank)) + if (item.rob_id !== stim.rob_id) + `uvm_fatal("WRITE", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + if (item.group_id !== stim.wr_group + write_count / stim.iter) + `uvm_fatal("WRITE", $sformatf("group mismatch at %0d: got %0d", write_count, item.group_id)) + if (item.mask !== 16'hFFFF) + `uvm_fatal("WRITE", $sformatf("mask mismatch: got 0x%04h", item.mask)) + if (write_count >= stim.num_dst_words) + `uvm_fatal("WRITE", $sformatf("extra write %0d", write_count)) + if (item.addr !== write_count % stim.iter) + `uvm_fatal("WRITE", $sformatf("addr mismatch at %0d: got %0d", write_count, item.addr)) + if (item.data !== stim.dst_words[write_count]) + `uvm_fatal("SCB", $sformatf( + "data mismatch at addr %0d: got 0x%032h exp 0x%032h", + item.addr, + item.data, + stim.dst_words[write_count] + )) + write_count++; + endfunction + + function void write_resp(bb_blink_resp_item item); + int8mul_cmd_item stim; + + stim = current_stim("RESP"); + if (item.rob_id !== stim.rob_id) + `uvm_fatal("RESP", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + resp_count++; + endfunction + + function int8mul_cmd_item current_stim(string tag); + if (stim_q.size() == 0) begin + `uvm_fatal("SCB", $sformatf("%s observed before stimulus", tag)) + end + return stim_q[0]; + endfunction + + function bit done(); + if (cmd_count != 1 || write_count != expected_writes || resp_count != 1) return 1'b0; + for (int p = 0; p < `BB_IN_BW; p++) begin + if (read_count[p] != expected_reads[p]) return 1'b0; + end + return 1'b1; + endfunction + + function void reset_counters(); + stim_q.delete(); + cmd_count = 0; + write_count = 0; + resp_count = 0; + expected_writes = 0; + for (int p = 0; p < `BB_IN_BW; p++) begin + read_count[p] = 0; + expected_reads[p] = 0; + end + endfunction + + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (!done()) begin + `uvm_fatal("SCB", $sformatf( + "incomplete: cmds=%0d reads=%0d/%0d,%0d/%0d writes=%0d/%0d resp=%0d", + cmd_count, read_count[0], expected_reads[0], read_count[1], + expected_reads[1], write_count, expected_writes, resp_count)) + end + stim_q.delete(); + endfunction +endclass diff --git a/examples/balls/int8mul/verify/src/pkg/int8mul_pkg.sv b/examples/prefab/balls/int8mul/verify/src/pkg/int8mul_pkg.sv similarity index 100% rename from examples/balls/int8mul/verify/src/pkg/int8mul_pkg.sv rename to examples/prefab/balls/int8mul/verify/src/pkg/int8mul_pkg.sv diff --git a/examples/balls/int8mul/verify/src/seq/int8mul_sequences.svh b/examples/prefab/balls/int8mul/verify/src/seq/int8mul_sequences.svh similarity index 86% rename from examples/balls/int8mul/verify/src/seq/int8mul_sequences.svh rename to examples/prefab/balls/int8mul/verify/src/seq/int8mul_sequences.svh index a9f4dd514..41abbd544 100644 --- a/examples/balls/int8mul/verify/src/seq/int8mul_sequences.svh +++ b/examples/prefab/balls/int8mul/verify/src/seq/int8mul_sequences.svh @@ -2,7 +2,6 @@ class int8mul_basic_seq extends uvm_sequence #(bb_blink_cmd_item); `uvm_object_utils(int8mul_basic_seq) int unsigned case_index; - int unsigned bid; function new(string name = "int8mul_basic_seq"); super.new(name); @@ -12,7 +11,7 @@ class int8mul_basic_seq extends uvm_sequence #(bb_blink_cmd_item); int8mul_cmd_item item; item = int8mul_cmd_item::type_id::create("item"); start_item(item); - item.load_rust_case(case_index, bid); + item.load_rust_case(case_index); finish_item(item); endtask endclass diff --git a/examples/prefab/balls/int8mul/verify/src/tb_top.sv b/examples/prefab/balls/int8mul/verify/src/tb_top.sv new file mode 100644 index 000000000..322b44442 --- /dev/null +++ b/examples/prefab/balls/int8mul/verify/src/tb_top.sv @@ -0,0 +1,89 @@ +module tb_top; + import uvm_pkg::*; + import int8mul_pkg::*; + + bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); + + Int8MulBall dut ( + .clock(intf.clock), + .reset(intf.reset), + .blink_status_running(intf.status_running), + .blink_cmdReq_ready(intf.cmd_req_ready), + .blink_cmdReq_valid(intf.cmd_req_valid), + .blink_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .blink_cmdReq_bits_funct(intf.cmd_req_bits_funct), + .blink_cmdReq_bits_rs2(intf.cmd_req_bits_rs2), + .blink_cmdReq_bits_iter(intf.cmd_req_bits_iter), + .blink_cmdReq_bits_op1_bank(intf.cmd_req_bits_op1_bank), + .blink_cmdReq_bits_op2_bank(intf.cmd_req_bits_op2_bank), + .blink_cmdReq_bits_wr_bank(intf.cmd_req_bits_wr_bank), + .blink_cmdReq_bits_op1_group(intf.cmd_req_bits_op1_group), + .blink_cmdReq_bits_op2_group(intf.cmd_req_bits_op2_group), + .blink_cmdReq_bits_wr_group(intf.cmd_req_bits_wr_group), + .blink_cmdReq_bits_op1_col(intf.cmd_req_bits_op1_col), + .blink_cmdReq_bits_op2_col(intf.cmd_req_bits_op2_col), + .blink_cmdReq_bits_wr_col(intf.cmd_req_bits_wr_col), + .blink_cmdReq_bits_wr_read_bank(intf.cmd_req_bits_wr_read_bank), + .blink_cmdReq_bits_wr_arch_bank(intf.cmd_req_bits_wr_arch_bank), + .blink_cmdResp_ready(intf.cmd_resp_ready), + .blink_cmdResp_valid(intf.cmd_resp_valid), + .blink_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .blink_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .blink_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .blink_bankRead_0_group_id(intf.bank_read_group_id[0]), + .blink_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .blink_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .blink_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .blink_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .blink_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .blink_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .blink_bankRead_1_bank_id(intf.bank_read_bank_id[1]), + .blink_bankRead_1_rob_id(intf.bank_read_rob_id[1]), + .blink_bankRead_1_group_id(intf.bank_read_group_id[1]), + .blink_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), + .blink_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), + .blink_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), + .blink_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), + .blink_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), + .blink_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), + .blink_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .blink_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .blink_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .blink_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .blink_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .blink_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .blink_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .blink_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .blink_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .blink_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]) + ); + + // Constant outputs removed from the port list by firtool. + assign intf.bank_write_req_mask[0][0] = dut.blink_bankWrite_0_io_req_bits_mask_0; + assign intf.bank_write_req_mask[0][1] = dut.blink_bankWrite_0_io_req_bits_mask_1; + assign intf.bank_write_req_mask[0][2] = dut.blink_bankWrite_0_io_req_bits_mask_2; + assign intf.bank_write_req_mask[0][3] = dut.blink_bankWrite_0_io_req_bits_mask_3; + assign intf.bank_write_req_mask[0][4] = dut.blink_bankWrite_0_io_req_bits_mask_4; + assign intf.bank_write_req_mask[0][5] = dut.blink_bankWrite_0_io_req_bits_mask_5; + assign intf.bank_write_req_mask[0][6] = dut.blink_bankWrite_0_io_req_bits_mask_6; + assign intf.bank_write_req_mask[0][7] = dut.blink_bankWrite_0_io_req_bits_mask_7; + assign intf.bank_write_req_mask[0][8] = dut.blink_bankWrite_0_io_req_bits_mask_8; + assign intf.bank_write_req_mask[0][9] = dut.blink_bankWrite_0_io_req_bits_mask_9; + assign intf.bank_write_req_mask[0][10] = dut.blink_bankWrite_0_io_req_bits_mask_10; + assign intf.bank_write_req_mask[0][11] = dut.blink_bankWrite_0_io_req_bits_mask_11; + assign intf.bank_write_req_mask[0][12] = dut.blink_bankWrite_0_io_req_bits_mask_12; + assign intf.bank_write_req_mask[0][13] = dut.blink_bankWrite_0_io_req_bits_mask_13; + assign intf.bank_write_req_mask[0][14] = dut.blink_bankWrite_0_io_req_bits_mask_14; + assign intf.bank_write_req_mask[0][15] = dut.blink_bankWrite_0_io_req_bits_mask_15; + + initial begin + intf.clock = 1'b0; + intf.reset = 1'b1; + forever #5 intf.clock = ~intf.clock; + end + + initial begin + uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); + run_test("int8mul_ball_test"); + end +endmodule diff --git a/examples/balls/int8mul/verify/src/tests/int8mul_test.svh b/examples/prefab/balls/int8mul/verify/src/tests/int8mul_test.svh similarity index 90% rename from examples/balls/int8mul/verify/src/tests/int8mul_test.svh rename to examples/prefab/balls/int8mul/verify/src/tests/int8mul_test.svh index c1d6714ca..b8c62d485 100644 --- a/examples/balls/int8mul/verify/src/tests/int8mul_test.svh +++ b/examples/prefab/balls/int8mul/verify/src/tests/int8mul_test.svh @@ -17,20 +17,18 @@ class int8mul_ball_test extends uvm_test; endfunction task run_phase(uvm_phase phase); - int unsigned bid = int8mul_require_bid(); phase.raise_objection(this); - for (int unsigned index = 0; index < INT8MUL_NUM_CASES; index++) run_case(index, bid); + for (int unsigned index = 0; index < INT8MUL_NUM_CASES; index++) run_case(index); phase.drop_objection(this); endtask - task run_case(int unsigned index, int unsigned bid); + task run_case(int unsigned index); int8mul_basic_seq seq; int cycles = 0; apply_reset(); env.scb.reset_counters(); seq = int8mul_basic_seq::type_id::create("seq"); seq.case_index = index; - seq.bid = bid; seq.start(env.cmd_agent.seqr); while (!env.scb.done()) begin @(posedge vif.clock); diff --git a/examples/prefab/balls/int8mul/workloads/ctests/CMakeLists.txt b/examples/prefab/balls/int8mul/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..9b7b1acd2 --- /dev/null +++ b/examples/prefab/balls/int8mul/workloads/ctests/CMakeLists.txt @@ -0,0 +1,2 @@ +add_buckyball_ctest(int8mul_per_channel_gate_test.c) +add_buckyball_ctest(int8mul_large_gate_row_test.c) diff --git a/examples/balls/int8mul/workloads/ctests/int8mul_large_gate_row_test.c b/examples/prefab/balls/int8mul/workloads/ctests/int8mul_large_gate_row_test.c similarity index 90% rename from examples/balls/int8mul/workloads/ctests/int8mul_large_gate_row_test.c rename to examples/prefab/balls/int8mul/workloads/ctests/int8mul_large_gate_row_test.c index d5e58b1cc..b118287b9 100644 --- a/examples/balls/int8mul/workloads/ctests/int8mul_large_gate_row_test.c +++ b/examples/prefab/balls/int8mul/workloads/ctests/int8mul_large_gate_row_test.c @@ -12,8 +12,9 @@ static int8_t input[ROWS * 16] __attribute__((aligned(64))); static int8_t output[ROWS * 16] __attribute__((aligned(64))); int main(void) { - if (BANK_LINES <= GATE_ROW) { - printf("int8mul large gate row SKIP bank_lines=%d\n", BANK_LINES); + if (THREAD_MEM_BANK_ENTRIES <= GATE_ROW) { + printf("int8mul large gate row SKIP bank_lines=%d\n", + THREAD_MEM_BANK_ENTRIES); return 0; } diff --git a/examples/balls/int8mul/workloads/ctests/int8mul_per_channel_gate_test.c b/examples/prefab/balls/int8mul/workloads/ctests/int8mul_per_channel_gate_test.c similarity index 100% rename from examples/balls/int8mul/workloads/ctests/int8mul_per_channel_gate_test.c rename to examples/prefab/balls/int8mul/workloads/ctests/int8mul_per_channel_gate_test.c diff --git a/examples/balls/int8mul/workloads/isa/int8mul.h b/examples/prefab/balls/int8mul/workloads/isa/int8mul.h similarity index 91% rename from examples/balls/int8mul/workloads/isa/int8mul.h rename to examples/prefab/balls/int8mul/workloads/isa/int8mul.h index f71d104ce..a308411d3 100644 --- a/examples/balls/int8mul/workloads/isa/int8mul.h +++ b/examples/prefab/balls/int8mul/workloads/isa/int8mul.h @@ -3,7 +3,6 @@ #include #include -#include #include static inline uint64_t bb_int8mul_rs2(float ratio, uint32_t gate_row) { @@ -19,6 +18,6 @@ static inline uint64_t bb_int8mul_rs2(float ratio, uint32_t gate_row) { BUCKYBALL_INSTRUCTION_R_R((BB_BANK0(gate_bank) | BB_BANK1(input_bank) | \ BB_BANK2(output_bank) | BB_ITER(iter)), \ bb_int8mul_rs2(ratio, gate_row), \ - BB_FUNC7(INT8MUL)) + FUNCT7(int8mul, INT8MUL)) #endif // _BB_INT8MUL_H_ diff --git a/examples/balls/int8mul/workloads/mlir_tests/CMakeLists.txt b/examples/prefab/balls/int8mul/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/balls/int8mul/workloads/mlir_tests/CMakeLists.txt rename to examples/prefab/balls/int8mul/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/balls/int8mul/workloads/mlir_tests/ball/CMakeLists.txt b/examples/prefab/balls/int8mul/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/balls/int8mul/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/prefab/balls/int8mul/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/prefab/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate.mlir b/examples/prefab/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate.mlir new file mode 100644 index 000000000..7e8f98361 --- /dev/null +++ b/examples/prefab/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate.mlir @@ -0,0 +1,37 @@ +func.func private @check_result(memref<4x16xi8>) -> () +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %one_i8 = arith.constant 1 : i8 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %four = arith.constant 4 : index + %sixteen = arith.constant 16 : index + %ratio = arith.constant 1.0 : f32 + %gate = memref.alloc() : memref<1x16xi8> + %input = memref.alloc() : memref<4x16xi8> + %output = memref.alloc() : memref<4x16xi8> + scf.for %lane = %zero to %sixteen step %one { + memref.store %one_i8, %gate[%zero, %lane] : memref<1x16xi8> + } + scf.for %row = %zero to %four step %one { + scf.for %lane = %zero to %sixteen step %one { + %value = arith.index_cast %lane : index to i8 + memref.store %value, %input[%row, %lane] : memref<4x16xi8> + } + } + buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %gate <{bank = 1 : i64, depth = 1 : i64, stride = 1 : i64}> : memref<1x16xi8> + buckyball.mvin %input <{bank = 2 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> + int8mul_ball.gate_mul_i8 ratio(%ratio) <{gate = 1 : i64, input = 2 : i64, output = 3 : i64, rows = 4 : i64}> + buckyball.mvout %output <{bank = 3 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> + func.call @check_result(%output) : (memref<4x16xi8>) -> () + buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + memref.dealloc %gate : memref<1x16xi8> + memref.dealloc %input : memref<4x16xi8> + memref.dealloc %output : memref<4x16xi8> + return %zero_i8 : i8 +} diff --git a/examples/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate_main.cpp b/examples/prefab/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate_main.cpp similarity index 100% rename from examples/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate_main.cpp rename to examples/prefab/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate_main.cpp diff --git a/examples/balls/int8mul/workloads/mlir_tests/bank/CMakeLists.txt b/examples/prefab/balls/int8mul/workloads/mlir_tests/bank/CMakeLists.txt similarity index 100% rename from examples/balls/int8mul/workloads/mlir_tests/bank/CMakeLists.txt rename to examples/prefab/balls/int8mul/workloads/mlir_tests/bank/CMakeLists.txt diff --git a/examples/prefab/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate.mlir.in b/examples/prefab/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate.mlir.in new file mode 100644 index 000000000..cbc2a8bcc --- /dev/null +++ b/examples/prefab/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate.mlir.in @@ -0,0 +1,37 @@ +func.func private @check_result(memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %one_i8 = arith.constant 1 : i8 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %rows = arith.constant @BB_BANK_LINES@ : index + %lanes = arith.constant @BB_BANK_I8_LANES@ : index + %ratio = arith.constant 1.0 : f32 + %gate = memref.alloc() : memref<1x@BB_BANK_I8_LANES@xi8> + %input = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + %output = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + scf.for %lane = %zero to %lanes step %one { + memref.store %one_i8, %gate[%zero, %lane] : memref<1x@BB_BANK_I8_LANES@xi8> + } + scf.for %row = %zero to %rows step %one { + scf.for %lane = %zero to %lanes step %one { + %value = arith.index_cast %lane : index to i8 + memref.store %value, %input[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + } + } + %gate_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %input_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %output_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %loaded_gate = buckyball.bank_mvin %gate, %gate_bank <{depth = 1 : i64, stride = 1 : i64}> : memref<1x@BB_BANK_I8_LANES@xi8> + %loaded_input = buckyball.bank_mvin %input, %input_bank <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + %result = int8mul_ball.bank_gate_mul_i8 %loaded_gate, %loaded_input -> %output_bank ratio(%ratio) <{rows = @BB_BANK_LINES@ : i64}> + %stored = buckyball.bank_mvout %output, %result <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + func.call @check_result(%output) : (memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () + buckyball.bank_release %loaded_gate + buckyball.bank_release %loaded_input + buckyball.bank_release %stored + memref.dealloc %gate : memref<1x@BB_BANK_I8_LANES@xi8> + memref.dealloc %input : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + memref.dealloc %output : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + return %zero_i8 : i8 +} diff --git a/examples/prefab/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate_main.cpp b/examples/prefab/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate_main.cpp new file mode 100644 index 000000000..048f32e2b --- /dev/null +++ b/examples/prefab/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate_main.cpp @@ -0,0 +1,17 @@ +#include +#include +#include +extern "C" void check_result(int8_t *, int8_t *data, int64_t offset, + int64_t rows, int64_t lanes, int64_t stride, + int64_t lane_stride) { + if (rows != THREAD_MEM_BANK_ENTRIES || lanes != THREAD_MEM_BANK_WIDTH / 8 || + stride != lanes || lane_stride != 1) + exit(1); + for (int row = 0; row < rows; ++row) + for (int lane = 0; lane < lanes; ++lane) + if (data[offset + row * stride + lane] != lane) { + printf("int8mul bank mismatch row=%d lane=%d\n", row, lane); + exit(1); + } + printf("int8mul bank PASS rows=%d\n", THREAD_MEM_BANK_ENTRIES); +} diff --git a/examples/prefab/balls/lut/arch/LutBall.scala b/examples/prefab/balls/lut/arch/LutBall.scala new file mode 100644 index 000000000..289838b20 --- /dev/null +++ b/examples/prefab/balls/lut/arch/LutBall.scala @@ -0,0 +1,169 @@ +package examples.balls.lut + +import framework.acc.balldomain.bbus.{BallFactory, BallPorts} +import buckyball.balls.lut.{Params => ParametersProto} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import chisel3.util._ +import framework.acc.balldomain.ball.blink.{BallStatus, BlinkIO, HasBlink} + +@instantiable +class LutBall( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Module + with HasBlink { + val ball = bp.balls.find(_.id == id).get + + ball.parameters.unpack(classOf[ParametersProto]) + + require(mp.bankWidth == 128, "LutBall requires 128-bit bank rows") + require(mp.bankEntries >= 16, "LutBall requires sixteen LUT rows") + require(ball.inBW == 2, "LutBall requires inBW=2") + require(ball.outBW == 1, "LutBall requires outBW=1") + + @public + val blink = IO(new BlinkIO(fp, mp, bp, id)) + + val idle :: inputReq :: inputResp :: lutReq :: lutResp :: writeReq :: writeResp :: complete :: Nil = + Enum(8) + val state = RegInit(idle) + val robId = RegInit(0.U(log2Up(fp.rob_entries).W)) + val inputBank = RegInit(0.U(fp.bank_id_len.W)) + val lutBank = RegInit(0.U(fp.bank_id_len.W)) + val outputBank = RegInit(0.U(fp.bank_id_len.W)) + val bases = Reg(Vec(3, UInt(mp.groupIdWidth.W))) + val iter = RegInit(0.U(fp.iter_len.W)) + val inputRow = RegInit(0.U(log2Ceil(mp.bankEntries).W)) + val inputData = Reg(UInt(128.W)) + val laneTable = RegInit(false.B) + val lookupLane = RegInit(0.U(4.W)) + val outputWord = Reg(Vec(16, UInt(8.W))) + + blink.cmdReq.ready := state === idle + blink.cmdResp.valid := state === complete + blink.cmdResp.bits.rob_id := robId + blink.status.idle := state === idle + blink.status.running := state =/= idle && state =/= complete + + for (port <- 0 until 2) { + blink.bankRead(port).rob_id := robId + blink.bankRead(port).group_id := bases(port) + blink.bankRead(port).io.req.valid := false.B + blink.bankRead(port).io.req.bits.addr := 0.U + blink.bankRead(port).io.resp.ready := false.B + } + blink.bankRead(0).bank_id := inputBank + blink.bankRead(1).bank_id := lutBank + blink.bankRead(1).group_id := bases(1) + Mux(laneTable, lookupLane >> 2, 0.U) + + blink.bankWrite(0).rob_id := robId + blink.bankWrite(0).bank_id := outputBank + blink.bankWrite(0).group_id := bases(2) + blink.bankWrite(0).io.req.valid := false.B + blink.bankWrite(0).io.req.bits.addr := inputRow + blink.bankWrite(0).io.req.bits.data := Cat(outputWord.reverse) + blink.bankWrite(0).io.req.bits.mask := VecInit( + Seq.fill(mp.bankMaskLen)(true.B) + ) + blink.bankWrite(0).io.resp.ready := false.B + + switch(state) { + is(idle) { + when(blink.cmdReq.fire) { + val cmd = blink.cmdReq.bits + + assert(cmd.rs2 === 0.U, "LutBall reserves rs2") + assert( + cmd.iter > 0.U && cmd.iter <= mp.bankEntries.U, + "LutBall iter must fit in one bank" + ) + assert( + cmd.op1_col === 1.U && (cmd.op2_col === 1.U || cmd.op2_col === 4.U) && cmd.wr_col === 1.U, + "LutBall requires col=1 input/output and col=1 or col=4 table" + ) + assert( + cmd.op1_bank =/= cmd.op2_bank && cmd.op1_bank =/= cmd.wr_bank && + cmd.op2_bank =/= cmd.wr_bank, + "LutBall banks must be distinct" + ) + robId := blink.cmdReq.bits.rob_id + inputBank := cmd.op1_bank + lutBank := cmd.op2_bank + outputBank := cmd.wr_bank + bases := VecInit(cmd.op1_group, cmd.op2_group, cmd.wr_group) + iter := cmd.iter + inputRow := 0.U + laneTable := cmd.op2_col === 4.U + lookupLane := 0.U + state := inputReq + } + } + is(inputReq) { + blink.bankRead(0).io.req.valid := true.B + blink.bankRead(0).io.req.bits.addr := inputRow + when(blink.bankRead(0).io.req.fire)(state := inputResp) + } + is(inputResp) { + blink.bankRead(0).io.resp.ready := true.B + when(blink.bankRead(0).io.resp.fire) { + inputData := blink.bankRead(0).io.resp.bits.data + lookupLane := 0.U + state := lutReq + } + } + is(lutReq) { + val inputByte = (inputData >> (lookupLane << 3))(7, 0) + blink.bankRead(1).io.req.valid := true.B + blink.bankRead(1).io.req.bits.addr := Mux( + laneTable, + Cat(lookupLane(1, 0), inputByte(7, 4)), + inputByte(7, 4) + ) + when(blink.bankRead(1).io.req.fire)(state := lutResp) + } + is(lutResp) { + val inputByte = (inputData >> (lookupLane << 3))(7, 0) + blink.bankRead(1).io.resp.ready := true.B + when(blink.bankRead(1).io.resp.fire) { + outputWord(lookupLane) := + (blink.bankRead(1).io.resp.bits.data >> (inputByte(3, 0) << 3))(7, 0) + when(lookupLane === 15.U)(state := writeReq) + .otherwise { lookupLane := lookupLane + 1.U; state := lutReq } + } + } + is(writeReq) { + blink.bankWrite(0).io.req.valid := true.B + when(blink.bankWrite(0).io.req.fire)(state := writeResp) + } + is(writeResp) { + blink.bankWrite(0).io.resp.ready := true.B + when(blink.bankWrite(0).io.resp.fire) { + when(inputRow === iter - 1.U)(state := complete) + .otherwise { inputRow := inputRow + 1.U; state := inputReq } + } + } + is(complete) { + when(blink.cmdResp.fire)(state := idle) + } + } +} + +object LutBall extends BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts = + BallPorts(Instantiate(new LutBall(fp, mp, bp, id)).blink, None) + +} diff --git a/examples/prefab/balls/lut/configs/default.toml b/examples/prefab/balls/lut/configs/default.toml new file mode 100644 index 000000000..dc430c153 --- /dev/null +++ b/examples/prefab/balls/lut/configs/default.toml @@ -0,0 +1 @@ +# LutBall has no private configuration parameters. diff --git a/examples/prefab/balls/lut/configs/lut_params.proto b/examples/prefab/balls/lut/configs/lut_params.proto new file mode 100644 index 000000000..453bc3579 --- /dev/null +++ b/examples/prefab/balls/lut/configs/lut_params.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; +package buckyball.balls.lut; +option java_package = "buckyball.balls.lut"; +option java_multiple_files = true; + +message Params { +} diff --git a/examples/prefab/balls/lut/emu/src/dispatch.rs b/examples/prefab/balls/lut/emu/src/dispatch.rs new file mode 100644 index 000000000..175fc37a9 --- /dev/null +++ b/examples/prefab/balls/lut/emu/src/dispatch.rs @@ -0,0 +1,36 @@ +use bebop_bemu::ball::{BallModel, ExecContext}; + +use super::lut; + +const BALL_CLASS: &str = "examples.balls.lut.LutBall"; + +pub fn execute_known( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, + ctx: &mut ExecContext, +) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("LUT") => Some(lut::Lut::exec(xs1, xs2, ctx)), + Some(_) | None => None, + } +} + +pub fn cycles_after_issue( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, +) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("LUT") => Some(lut::Lut::latency(xs1, xs2)), + Some(_) | None => None, + } +} diff --git a/examples/prefab/balls/lut/emu/src/lib.rs b/examples/prefab/balls/lut/emu/src/lib.rs new file mode 100644 index 000000000..80c7d3734 --- /dev/null +++ b/examples/prefab/balls/lut/emu/src/lib.rs @@ -0,0 +1,11 @@ +pub(crate) use bebop_bemu::{thread::instructions::decode, ball as instruction}; + +mod lut; + +mod dispatch; + +pub use dispatch::{cycles_after_issue, execute_known}; + +#[bebop_bemu::ball::linkme::distributed_slice(bebop_bemu::BALLS)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static BALL: bebop_bemu::Ball = bebop_bemu::Ball { execute: execute_known, cycles: cycles_after_issue }; diff --git a/examples/balls/lut/emu/src/lut.rs b/examples/prefab/balls/lut/emu/src/lut.rs similarity index 90% rename from examples/balls/lut/emu/src/lut.rs rename to examples/prefab/balls/lut/emu/src/lut.rs index e52d7cc72..bd19ba924 100644 --- a/examples/balls/lut/emu/src/lut.rs +++ b/examples/prefab/balls/lut/emu/src/lut.rs @@ -1,11 +1,12 @@ -use super::super::bank::bank_lines; -use super::decode::{pbank, pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; -use super::instruction::{BallInstruction, ExecContext}; +use bebop_bemu::thread::banks::bank_lines; +use super::decode::{pbank, pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old}; +use super::instruction::{BallModel, ExecContext}; pub struct Lut; -impl BallInstruction for Lut { +impl BallModel for Lut { fn exec(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + assert!(!rs1_read_old(xs1), "lut: reads no old destination; rs1[29] must be clear"); let input_bank = rs1_b0(xs1); let lut_bank = rs1_b1(xs1); let output_bank = rs1_b2(xs1); diff --git a/examples/prefab/balls/lut/stack/src/lutOps.cpp b/examples/prefab/balls/lut/stack/src/lutOps.cpp new file mode 100644 index 000000000..1296b863f --- /dev/null +++ b/examples/prefab/balls/lut/stack/src/lutOps.cpp @@ -0,0 +1,68 @@ +//===- lutOps.cpp - LutBall operations ------------------------------------===// +// +// The dialect of LutBall, its virtual operations building their physical ones +// and the instructions of the physical operations. +// +//===----------------------------------------------------------------------===// + +#include "lutOps.h" + +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::lut_ball; +using mlir::buddy::bankFields; +using mlir::buddy::constantI64; +using mlir::buddy::emitInstruction; +using mlir::buddy::field; + +#include "lutDialect.cpp.inc" + +#define GET_OP_CLASSES +#include "lutOps.cpp.inc" + +void LutBallDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "lutOps.cpp.inc" + >(); +} + +namespace { +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallDialect( + [](DialectRegistry ®istry) { registry.insert(); }), + true); +} // namespace + +SmallVector BankLookupI8Op::getSourceBanks() { + return {getCodes(), getTable()}; +} +Value BankLookupI8Op::getDestinationBank() { return getOutput(); } +bool BankLookupI8Op::readsDestination() { return false; } +Operation *BankLookupI8Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return LookupI8Op::create(builder, getLoc(), sources[0], sources[1], + destination, getRows()); +} + +SmallVector LookupI8Op::getSourceBanks() { + return {int64_t(getCodes()), int64_t(getTable())}; +} +int64_t LookupI8Op::getDestinationBank() { return getOutput(); } +LogicalResult LookupI8Op::lowerToInstructions(OpBuilder &builder) { + Location loc = getLoc(); + int64_t rows = getRows(), codes = getCodes(), table = getTable(), + output = getOutput(); + if (rows <= 0 || rows > buckyball_target::getBuckyballTarget().bankDepth || + codes == table || codes == output || table == output) + return emitOpError( + "lookup_i8 requires distinct banks and rows within bank depth"); + emitInstruction(builder, loc, buckyball_target::getBuckyballFunct7("LUT"), + constantI64(builder, loc, + bankFields({codes, table}, output, false, rows)), + constantI64(builder, loc, 0)); + return success(); +} diff --git a/examples/prefab/balls/lut/stack/src/lutOps.h b/examples/prefab/balls/lut/stack/src/lutOps.h new file mode 100644 index 000000000..d4c719ad7 --- /dev/null +++ b/examples/prefab/balls/lut/stack/src/lutOps.h @@ -0,0 +1,20 @@ +//===- lutOps.h - LutBall operations --------------------------------------===// + +#ifndef LUT_OPS_H +#define LUT_OPS_H + +#include "Interfaces/BankInterfaces.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "lutDialect.h.inc" + +#define GET_OP_CLASSES +#include "lutOps.h.inc" + +#endif // LUT_OPS_H diff --git a/examples/prefab/balls/lut/stack/src/lutOps.td b/examples/prefab/balls/lut/stack/src/lutOps.td new file mode 100644 index 000000000..8a9c19d87 --- /dev/null +++ b/examples/prefab/balls/lut/stack/src/lutOps.td @@ -0,0 +1,43 @@ +//===- lutOps.td - LutBall operations ----------------------*- tablegen -*-===// +// +// lookup_i8: `codes` holds `rows` rows of sixteen i8 codes, `table` the +// table: one group of 256 entries shared by every channel, or four groups with +// 256 entries per channel (channel * 256 + code, 1024 bytes per group). The +// output receives table[code] for every element. +// +//===----------------------------------------------------------------------===// + +#ifndef LUT_OPS +#define LUT_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Interfaces/BankInterfaces.td" + +def LutBall_Dialect : Dialect { + let name = "lut_ball"; + let cppNamespace = "::buddy::lut_ball"; + let summary = "The operations of LutBall"; +} + +class LutBall_Op traits = []> + : Op])>; + +def BankLookupI8Op : LutBall_Op<"bank_lookup_i8", + [DeclareOpInterfaceMethods]> { + let summary = "lookup_i8 on virtual banks"; + let arguments = (ins I64:$codes, I64:$table, I64:$output, I64Attr:$rows); + let results = (outs I64:$outputOut); + let assemblyFormat = "$codes `,` $table `->` $output prop-dict attr-dict"; +} + +def LookupI8Op : LutBall_Op<"lookup_i8", + [DeclareOpInterfaceMethods]> { + let summary = "lookup_i8 on physical banks"; + let arguments = (ins I64Attr:$codes, I64Attr:$table, I64Attr:$output, + I64Attr:$rows); + let assemblyFormat = "prop-dict attr-dict"; +} + +#endif // LUT_OPS diff --git a/examples/balls/lut/verify/casegen/Cargo.lock b/examples/prefab/balls/lut/verify/casegen/Cargo.lock similarity index 100% rename from examples/balls/lut/verify/casegen/Cargo.lock rename to examples/prefab/balls/lut/verify/casegen/Cargo.lock diff --git a/examples/balls/lut/verify/casegen/Cargo.toml b/examples/prefab/balls/lut/verify/casegen/Cargo.toml similarity index 100% rename from examples/balls/lut/verify/casegen/Cargo.toml rename to examples/prefab/balls/lut/verify/casegen/Cargo.toml diff --git a/examples/prefab/balls/lut/verify/casegen/src/casegen.rs b/examples/prefab/balls/lut/verify/casegen/src/casegen.rs new file mode 100644 index 000000000..c50d50e64 --- /dev/null +++ b/examples/prefab/balls/lut/verify/casegen/src/casegen.rs @@ -0,0 +1,229 @@ +pub const BYTES_PER_ROW: usize = 16; +pub const SHARED_TABLE_BYTES: usize = 256; +pub const LANE_TABLE_BYTES: usize = 16 * SHARED_TABLE_BYTES; +pub const MAX_SRC: usize = 8; +pub const MAX_LUT: usize = LANE_TABLE_BYTES / BYTES_PER_ROW; +pub const MAX_DST: usize = 8; +pub const NUM_CASES: u32 = 4; + +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct LutCmd { + pub iter: u32, + pub op1_bank: u32, + pub op2_bank: u32, + pub wr_bank: u32, + pub op1_group: u32, + pub op2_group: u32, + pub wr_group: u32, + pub op1_col: u32, + pub op2_col: u32, + pub wr_col: u32, + pub rob_id: u32, + pub num_src_words: u32, + pub num_lut_words: u32, + pub num_dst_words: u32, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LutCase { + pub cmd: LutCmd, + pub src_words: [u128; MAX_SRC], + pub lut_words: [u128; MAX_LUT], + pub dst_words: [u128; MAX_DST], +} + +impl LutCase { + pub fn src_lo(&self, index: usize) -> u64 { + self.src_words[index] as u64 + } + + pub fn src_hi(&self, index: usize) -> u64 { + (self.src_words[index] >> 64) as u64 + } + + pub fn lut_lo(&self, index: usize) -> u64 { + self.lut_words[index] as u64 + } + + pub fn lut_hi(&self, index: usize) -> u64 { + (self.lut_words[index] >> 64) as u64 + } + + pub fn dst_lo(&self, index: usize) -> u64 { + self.dst_words[index] as u64 + } + + pub fn dst_hi(&self, index: usize) -> u64 { + (self.dst_words[index] >> 64) as u64 + } +} + +pub fn gen_case(index: u32) -> LutCase { + if index >= NUM_CASES { + panic!("lut: unsupported directed case {index}"); + } + match index { + 0 => directed(2, 4, false, [0, 0, 0]), + 1 => directed(3, 1, false, [1, 2, 3]), + 2 => directed(4, 8, false, [2, 0, 1]), + 3 => directed(5, 4, true, [3, 1, 2]), + _ => panic!("lut: unsupported directed case {index}"), + } +} + +fn shared_table() -> Vec { + let mut table = vec![0u8; SHARED_TABLE_BYTES]; + for (i, slot) in table.iter_mut().enumerate() { + *slot = (((i * 73 + 19) ^ 0xa5) & 0xff) as u8; + } + table +} + +fn lane_table() -> Vec { + let mut table = vec![0u8; LANE_TABLE_BYTES]; + for channel in 0..16 { + for input in 0..256 { + table[channel * 256 + input] = ((input as i32 + channel as i32 * 17 - 128) as i8) as u8; + } + } + table +} + +fn pack_lane_table(table: &[u8]) -> Vec { + if table.len() != LANE_TABLE_BYTES { + panic!("lut: invalid lane table size {}", table.len()); + } + let mut packed = vec![0u8; LANE_TABLE_BYTES]; + for row in 0..64 { + for group in 0..4 { + for byte in 0..16 { + packed[group * 64 * 16 + row * 16 + byte] = + table[(group * 4 + row / 16) * 256 + (row % 16) * 16 + byte]; + } + } + } + packed +} + +fn ctest_input(nbytes: usize) -> Vec { + (0..nbytes) + .map(|i| (i as i32 * 29 - 128) as i8 as u8) + .collect() +} + +fn pack_bytes(bytes: &[u8]) -> Vec { + if bytes.len() % BYTES_PER_ROW != 0 { + panic!( + "lut: {} bytes is not a multiple of {BYTES_PER_ROW}", + bytes.len() + ); + } + bytes + .chunks(BYTES_PER_ROW) + .map(|chunk| { + let mut word = 0u128; + for (lane, byte) in chunk.iter().enumerate() { + word |= u128::from(*byte) << (lane * 8); + } + word + }) + .collect() +} + +fn copy_words(src: &[u128]) -> [u128; N] { + if src.len() > N { + panic!("lut: {} words exceed buffer {N}", src.len()); + } + let mut out = [0u128; N]; + out[..src.len()].copy_from_slice(src); + out +} + +pub(crate) fn validate( + op1_bank: u32, + op2_bank: u32, + wr_bank: u32, + op1_col: u32, + op2_col: u32, + wr_col: u32, + iter: u32, +) { + if iter == 0 { + panic!("lut: iter must be positive"); + } + if op1_col != 1 || (op2_col != 1 && op2_col != 4) || wr_col != 1 { + panic!("lut: requires col=1 input/output and col=1 or col=4 table"); + } + if op1_bank == op2_bank || op1_bank == wr_bank || op2_bank == wr_bank { + panic!("lut: banks must be distinct"); + } +} + +fn dst_words(table: &[u8], input: &[u8], lane_table: bool) -> Vec { + let mapped: Vec = input + .iter() + .enumerate() + .map(|(index, &byte)| { + let channel = index % 16; + table[(if lane_table { channel * 256 } else { 0 }) + byte as usize] + }) + .collect(); + pack_bytes(&mapped) +} + +// `groups` are the op1/op2/wr base groups; a lane table spans four groups from its base. +fn directed(rob_id: u32, iter: u32, per_lane: bool, groups: [u32; 3]) -> LutCase { + let nbytes = iter as usize * BYTES_PER_ROW; + let table = if per_lane { + lane_table() + } else { + shared_table() + }; + let input = ctest_input(nbytes); + let src = pack_bytes(&input); + let lut = if per_lane { + pack_bytes(&pack_lane_table(&table)) + } else { + pack_bytes(&table) + }; + let dst = dst_words(&table, &input, per_lane); + let op1_bank = 0; + let op2_bank = 1; + let wr_bank = 2; + let op1_col = 1; + let op2_col = if per_lane { 4 } else { 1 }; + let wr_col = 1; + validate(op1_bank, op2_bank, wr_bank, op1_col, op2_col, wr_col, iter); + if src.len() != iter as usize { + panic!("lut: src rows {} != iter {iter}", src.len()); + } + let expected_lut_rows = if per_lane { MAX_LUT } else { 16 }; + if lut.len() != expected_lut_rows { + panic!("lut: table rows {} != {expected_lut_rows}", lut.len()); + } + if dst.len() != iter as usize { + panic!("lut: dst rows {} != iter {iter}", dst.len()); + } + LutCase { + cmd: LutCmd { + iter, + op1_bank, + op2_bank, + wr_bank, + op1_group: groups[0], + op2_group: groups[1], + wr_group: groups[2], + op1_col, + op2_col, + wr_col, + rob_id, + num_src_words: src.len() as u32, + num_lut_words: lut.len() as u32, + num_dst_words: dst.len() as u32, + }, + src_words: copy_words(&src), + lut_words: copy_words(&lut), + dst_words: copy_words(&dst), + } +} diff --git a/examples/prefab/balls/lut/verify/casegen/src/lib.rs b/examples/prefab/balls/lut/verify/casegen/src/lib.rs new file mode 100644 index 000000000..bafa9652a --- /dev/null +++ b/examples/prefab/balls/lut/verify/casegen/src/lib.rs @@ -0,0 +1,70 @@ +mod casegen; + +use std::cell::RefCell; + +use casegen::{LutCase, LutCmd}; + +thread_local! { + static CURRENT: RefCell> = const { RefCell::new(None) }; +} + +#[no_mangle] +pub extern "C" fn lut_case_load(index: u32) -> i32 { + let case = casegen::gen_case(index); + CURRENT.with(|current| *current.borrow_mut() = Some(case)); + 0 +} + +fn current_case R, R>(f: F) -> R { + CURRENT.with(|current| match current.borrow().as_ref() { + Some(case) => f(case), + None => panic!("lut_case: no case loaded; call lut_case_load first"), + }) +} + +#[no_mangle] +pub extern "C" fn lut_case_cmd(out_ptr: *mut LutCmd) { + if out_ptr.is_null() { + panic!("lut_case_cmd: null out_ptr"); + } + current_case(|case| unsafe { + *out_ptr = case.cmd; + }); +} + +fn src_index(index: u32, limit: u32, what: &str) -> usize { + if index >= limit { + panic!("lut_case_{what}: index {index} out of range {limit}"); + } + index as usize +} + +#[no_mangle] +pub extern "C" fn lut_case_src_word_lo(word_index: u32) -> u64 { + current_case(|case| case.src_lo(src_index(word_index, case.cmd.num_src_words, "src_word_lo"))) +} + +#[no_mangle] +pub extern "C" fn lut_case_src_word_hi(word_index: u32) -> u64 { + current_case(|case| case.src_hi(src_index(word_index, case.cmd.num_src_words, "src_word_hi"))) +} + +#[no_mangle] +pub extern "C" fn lut_case_lut_word_lo(word_index: u32) -> u64 { + current_case(|case| case.lut_lo(src_index(word_index, case.cmd.num_lut_words, "lut_word_lo"))) +} + +#[no_mangle] +pub extern "C" fn lut_case_lut_word_hi(word_index: u32) -> u64 { + current_case(|case| case.lut_hi(src_index(word_index, case.cmd.num_lut_words, "lut_word_hi"))) +} + +#[no_mangle] +pub extern "C" fn lut_case_dst_word_lo(word_index: u32) -> u64 { + current_case(|case| case.dst_lo(src_index(word_index, case.cmd.num_dst_words, "dst_word_lo"))) +} + +#[no_mangle] +pub extern "C" fn lut_case_dst_word_hi(word_index: u32) -> u64 { + current_case(|case| case.dst_hi(src_index(word_index, case.cmd.num_dst_words, "dst_word_hi"))) +} diff --git a/examples/balls/lut/verify/filelists/lut_ball.f b/examples/prefab/balls/lut/verify/filelists/lut_ball.f similarity index 89% rename from examples/balls/lut/verify/filelists/lut_ball.f rename to examples/prefab/balls/lut/verify/filelists/lut_ball.f index f98bca01e..c5edcd905 100644 --- a/examples/balls/lut/verify/filelists/lut_ball.f +++ b/examples/prefab/balls/lut/verify/filelists/lut_ball.f @@ -1,3 +1,4 @@ ++incdir+@UVM@/src/core +incdir+@UVM@/src/ball +incdir+@UVM@/src/ball/agents/cmd +incdir+@UVM@/src/ball/agents/mem @@ -13,6 +14,7 @@ +incdir+src/tests +incdir+@RTL@ +@UVM@/src/core/package.sv @UVM@/src/ball/bb_blink_if.sv @UVM@/src/ball/bb_uvm_pkg.sv diff --git a/examples/prefab/balls/lut/verify/src/common/lut_defs.svh b/examples/prefab/balls/lut/verify/src/common/lut_defs.svh new file mode 100644 index 000000000..09021a2e2 --- /dev/null +++ b/examples/prefab/balls/lut/verify/src/common/lut_defs.svh @@ -0,0 +1,37 @@ +typedef struct { + int unsigned iter; + int unsigned op1_bank; + int unsigned op2_bank; + int unsigned wr_bank; + int unsigned op1_group; + int unsigned op2_group; + int unsigned wr_group; + int unsigned op1_col; + int unsigned op2_col; + int unsigned wr_col; + int unsigned rob_id; + int unsigned num_src_words; + int unsigned num_lut_words; + int unsigned num_dst_words; +} lut_cmd_dpi_t; + +import "DPI-C" function int lut_case_load(input int unsigned index); +import "DPI-C" function void lut_case_cmd(output lut_cmd_dpi_t cmd); +import "DPI-C" function longint unsigned lut_case_src_word_lo(input int unsigned word_index); +import "DPI-C" function longint unsigned lut_case_src_word_hi(input int unsigned word_index); +import "DPI-C" function longint unsigned lut_case_lut_word_lo(input int unsigned word_index); +import "DPI-C" function longint unsigned lut_case_lut_word_hi(input int unsigned word_index); +import "DPI-C" function longint unsigned lut_case_dst_word_lo(input int unsigned word_index); +import "DPI-C" function longint unsigned lut_case_dst_word_hi(input int unsigned word_index); + +`ifndef LUT_FUNCT7 +`error "LUT_FUNCT7 must be provided by the selected Thread ballISA" +`endif + +localparam int LUT_THREAD_FUNCT7 = `LUT_FUNCT7; +localparam int LUT_MAX_SRC_WORDS = 8; +localparam int LUT_SHARED_TABLE_ROWS = 16; +localparam int LUT_LANE_TABLE_ROWS = 256; +localparam int LUT_MAX_DST_WORDS = 8; +localparam int LUT_NUM_CASES = 4; +localparam int LUT_TIMEOUT_CYCLES = 20000; diff --git a/examples/prefab/balls/lut/verify/src/common/lut_items.svh b/examples/prefab/balls/lut/verify/src/common/lut_items.svh new file mode 100644 index 000000000..be05c7c3a --- /dev/null +++ b/examples/prefab/balls/lut/verify/src/common/lut_items.svh @@ -0,0 +1,92 @@ +class lut_cmd_item extends bb_blink_cmd_item; + `uvm_object_utils(lut_cmd_item) + + bit [31:0] num_src_words; + bit [31:0] num_lut_words; + bit [31:0] num_dst_words; + bit [127:0] src_words[LUT_MAX_SRC_WORDS]; + bit [127:0] lut_words[LUT_LANE_TABLE_ROWS]; + bit [127:0] dst_words[LUT_MAX_DST_WORDS]; + + function new(string name = "lut_cmd_item"); + super.new(name); + endfunction + + function void load_rust_case(int unsigned index); + lut_cmd_dpi_t cmd; + int unsigned rc; + longint unsigned w_lo; + longint unsigned w_hi; + int i; + + rc = lut_case_load(index); + if (rc != 0) begin + `uvm_fatal("CASE", $sformatf("lut_case_load returned %0d for index %0d", rc, index)) + end + lut_case_cmd(cmd); + + // bb_lut: input/table/output banks and iter come from rs1; rs2 is reserved zero. + rob_id = cmd.rob_id[BB_BLINK_ROB_ID_W-1:0]; + funct = LUT_THREAD_FUNCT7[6:0]; + rs2 = 64'd0; + iter = cmd.iter; + op1_bank = cmd.op1_bank; + op2_bank = cmd.op2_bank; + wr_bank = cmd.wr_bank; + op1_group = cmd.op1_group; + op2_group = cmd.op2_group; + wr_group = cmd.wr_group; + op1_col = cmd.op1_col; + op2_col = cmd.op2_col; + wr_col = cmd.wr_col; + wr_read_bank = '0; + wr_arch_bank = cmd.wr_bank; + num_src_words = cmd.num_src_words; + num_lut_words = cmd.num_lut_words; + num_dst_words = cmd.num_dst_words; + + if (iter != 1 && iter != 4 && iter != 8) + `uvm_fatal("CASE", $sformatf("LUT iter %0d is not in {1,4,8}", iter)) + if (op1_col != 1 || (op2_col != 1 && op2_col != 4) || wr_col != 1) + `uvm_fatal("CASE", "LUT requires col=1 input/output and col=1 or col=4 table") + if (op1_bank == op2_bank || op1_bank == wr_bank || op2_bank == wr_bank) + `uvm_fatal("CASE", "LUT banks must be distinct") + if (num_src_words != iter) + `uvm_fatal("CASE", $sformatf("num_src_words %0d != iter %0d", num_src_words, iter)) + if (num_dst_words != iter) + `uvm_fatal("CASE", $sformatf("num_dst_words %0d != iter %0d", num_dst_words, iter)) + if (num_lut_words != (op2_col == 4 ? LUT_LANE_TABLE_ROWS : LUT_SHARED_TABLE_ROWS)) + `uvm_fatal("CASE", $sformatf( + "num_lut_words %0d does not match col %0d", num_lut_words, op2_col)) + + for (i = 0; i < num_src_words; i++) begin + w_lo = lut_case_src_word_lo(i); + w_hi = lut_case_src_word_hi(i); + src_words[i] = {w_hi, w_lo}; + end + for (i = 0; i < num_lut_words; i++) begin + w_lo = lut_case_lut_word_lo(i); + w_hi = lut_case_lut_word_hi(i); + lut_words[i] = {w_hi, w_lo}; + end + for (i = 0; i < num_dst_words; i++) begin + w_lo = lut_case_dst_word_lo(i); + w_hi = lut_case_dst_word_hi(i); + dst_words[i] = {w_hi, w_lo}; + end + endfunction + + function void do_copy(uvm_object rhs); + lut_cmd_item rhs_; + super.do_copy(rhs); + if (!$cast(rhs_, rhs)) begin + `uvm_fatal("COPY", "rhs is not lut_cmd_item") + end + num_src_words = rhs_.num_src_words; + num_lut_words = rhs_.num_lut_words; + num_dst_words = rhs_.num_dst_words; + for (int i = 0; i < LUT_MAX_SRC_WORDS; i++) src_words[i] = rhs_.src_words[i]; + for (int i = 0; i < LUT_LANE_TABLE_ROWS; i++) lut_words[i] = rhs_.lut_words[i]; + for (int i = 0; i < LUT_MAX_DST_WORDS; i++) dst_words[i] = rhs_.dst_words[i]; + endfunction +endclass diff --git a/examples/balls/lut/verify/src/cov/lut_cov.svh b/examples/prefab/balls/lut/verify/src/cov/lut_cov.svh similarity index 85% rename from examples/balls/lut/verify/src/cov/lut_cov.svh rename to examples/prefab/balls/lut/verify/src/cov/lut_cov.svh index 356973178..38d7c4862 100644 --- a/examples/balls/lut/verify/src/cov/lut_cov.svh +++ b/examples/prefab/balls/lut/verify/src/cov/lut_cov.svh @@ -20,8 +20,8 @@ class lut_cov extends uvm_component; endfunction function void write_cmd(bb_blink_cmd_item item); - if (item.funct7 != LUT_CORE_FUNCT7[6:0]) - `uvm_fatal("COV", $sformatf("unexpected funct7 %0d", item.funct7)) + if (item.funct != LUT_THREAD_FUNCT7[6:0]) + `uvm_fatal("COV", $sformatf("unexpected funct %0d", item.funct)) cur_iter = item.iter; cur_lut_cols = item.op2_col; cmd_count++; @@ -29,6 +29,7 @@ class lut_cov extends uvm_component; endfunction function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; super.check_phase(phase); if (cmd_count == 0) `uvm_fatal("COV", "no cmd observed") if (cmd_cg.get_coverage() != 100.0) diff --git a/examples/balls/lut/verify/src/env/lut_env.svh b/examples/prefab/balls/lut/verify/src/env/lut_env.svh similarity index 100% rename from examples/balls/lut/verify/src/env/lut_env.svh rename to examples/prefab/balls/lut/verify/src/env/lut_env.svh diff --git a/examples/balls/lut/verify/src/env/lut_scoreboard.svh b/examples/prefab/balls/lut/verify/src/env/lut_scoreboard.svh similarity index 80% rename from examples/balls/lut/verify/src/env/lut_scoreboard.svh rename to examples/prefab/balls/lut/verify/src/env/lut_scoreboard.svh index bfbf113f4..8809fa0e9 100644 --- a/examples/balls/lut/verify/src/env/lut_scoreboard.svh +++ b/examples/prefab/balls/lut/verify/src/env/lut_scoreboard.svh @@ -47,10 +47,12 @@ class lut_scoreboard extends uvm_scoreboard; end mem_model.clear_mem(); for (i = 0; i < clone.num_src_words; i++) begin - mem_model.load_word(int'(clone.op1_bank), i, clone.src_words[i]); + mem_model.load_word_g(int'(clone.op1_bank), int'(clone.op1_group), i, clone.src_words[i]); end + // A lane table spans four groups of 64 rows from the table base group. for (i = 0; i < clone.num_lut_words; i++) begin - mem_model.load_word_g(int'(clone.op2_bank), i / 64, i % 64, clone.lut_words[i]); + mem_model.load_word_g(int'(clone.op2_bank), int'(clone.op2_group) + i / 64, i % 64, + clone.lut_words[i]); end mem_model.arm(); @@ -67,21 +69,12 @@ class lut_scoreboard extends uvm_scoreboard; endfunction function void check_cmd(bb_blink_cmd_item got, lut_cmd_item exp); - if (got.bid !== exp.bid) - `uvm_fatal("CMD", $sformatf("bid mismatch: got %0d exp %0d", got.bid, exp.bid)) - if (got.funct7 !== exp.funct7) - `uvm_fatal("CMD", $sformatf("funct7 mismatch: got %0d exp %0d", got.funct7, exp.funct7)) - if (got.iter !== exp.iter) - `uvm_fatal("CMD", $sformatf("iter mismatch: got %0d exp %0d", got.iter, exp.iter)) - if (got.op1_bank !== exp.op1_bank || got.op2_bank !== exp.op2_bank || - got.wr_bank !== exp.wr_bank) - `uvm_fatal("CMD", "bank field mismatch") - if (got.op1_col !== exp.op1_col || got.op2_col !== exp.op2_col || got.wr_col !== exp.wr_col) - `uvm_fatal("CMD", "column field mismatch") - if (got.rs1 !== exp.rs1 || got.rs2 !== exp.rs2) `uvm_fatal("CMD", "rs field mismatch") - if (got.rob_id !== exp.rob_id) - `uvm_fatal("CMD", $sformatf("rob_id mismatch: got %0d exp %0d", got.rob_id, exp.rob_id)) - if (got.rs2 !== 64'd0) `uvm_fatal("CMD", "LUT rs2 must be zero") + // Compare as plain BLink commands; the comparer rejects differing types. + bb_blink_cmd_item want = bb_blink_cmd_item::type_id::create("want"); + want.copy(exp); + verification_pkg::evidence::checks++; + if (!got.compare(want)) + `uvm_fatal("CMD", $sformatf("command mismatch: got %s exp %s", got.sprint(), want.sprint())) endfunction function void write_read(bb_blink_read_item item); @@ -102,18 +95,18 @@ class lut_scoreboard extends uvm_scoreboard; if (item.port == 0) begin expect_bank = stim.op1_bank; expect_addr = read_count[0]; - expect_group = 0; + expect_group = stim.op1_group; end else begin expect_bank = stim.op2_bank; row = read_count[1] / 16; lane = read_count[1] % 16; input_byte = stim.src_words[row][lane*8+:8]; - expect_group = stim.op2_col == 4 ? lane / 4 : 0; + expect_group = stim.op2_group + (stim.op2_col == 4 ? lane / 4 : 0); expect_addr = (stim.op2_col == 4 ? (lane % 4) * 16 : 0) + input_byte[7:4]; end - if (item.bank_id !== expect_bank[4:0]) + if (item.bank_id !== expect_bank[BB_BLINK_BANK_ID_W-1:0]) `uvm_fatal("READ", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, expect_bank)) - if (item.group_id !== expect_group[4:0]) + if (item.group_id !== expect_group[BB_BLINK_GROUP_ID_W-1:0]) `uvm_fatal("READ", $sformatf("group mismatch: got %0d exp %0d", item.group_id, expect_group)) if (item.addr !== expect_addr[BB_BLINK_BANK_ADDR_W-1:0]) `uvm_fatal("READ", $sformatf("addr mismatch: got %0d exp %0d", item.addr, expect_addr)) @@ -130,8 +123,9 @@ class lut_scoreboard extends uvm_scoreboard; `uvm_fatal("WRITE", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.wr_bank)) if (item.rob_id !== stim.rob_id) `uvm_fatal("WRITE", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.group_id !== 5'd0) - `uvm_fatal("WRITE", $sformatf("group mismatch: got %0d exp 0", item.group_id)) + if (item.group_id !== stim.wr_group) + `uvm_fatal("WRITE", $sformatf("group mismatch: got %0d exp %0d", item.group_id, stim.wr_group + )) if (item.mask !== 16'hFFFF) `uvm_fatal("WRITE", $sformatf("mask mismatch: got 0x%04h", item.mask)) if (write_count >= stim.num_dst_words) @@ -154,9 +148,6 @@ class lut_scoreboard extends uvm_scoreboard; stim = current_stim("RESP"); if (item.rob_id !== stim.rob_id) `uvm_fatal("RESP", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.is_sub !== 1'b0) `uvm_fatal("RESP", "is_sub should be 0") - if (item.sub_rob_id !== 8'h00) - `uvm_fatal("RESP", $sformatf("sub_rob_id mismatch: got 0x%0h", item.sub_rob_id)) resp_count++; endfunction @@ -188,6 +179,7 @@ class lut_scoreboard extends uvm_scoreboard; endfunction function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; super.check_phase(phase); if (!done()) begin `uvm_fatal("SCB", $sformatf( diff --git a/examples/balls/lut/verify/src/pkg/lut_pkg.sv b/examples/prefab/balls/lut/verify/src/pkg/lut_pkg.sv similarity index 100% rename from examples/balls/lut/verify/src/pkg/lut_pkg.sv rename to examples/prefab/balls/lut/verify/src/pkg/lut_pkg.sv diff --git a/examples/balls/lut/verify/src/seq/lut_sequences.svh b/examples/prefab/balls/lut/verify/src/seq/lut_sequences.svh similarity index 85% rename from examples/balls/lut/verify/src/seq/lut_sequences.svh rename to examples/prefab/balls/lut/verify/src/seq/lut_sequences.svh index 7a8ffa6d4..8eb3d9978 100644 --- a/examples/balls/lut/verify/src/seq/lut_sequences.svh +++ b/examples/prefab/balls/lut/verify/src/seq/lut_sequences.svh @@ -2,7 +2,6 @@ class lut_basic_seq extends uvm_sequence #(bb_blink_cmd_item); `uvm_object_utils(lut_basic_seq) int unsigned case_index; - int unsigned bid; function new(string name = "lut_basic_seq"); super.new(name); @@ -12,7 +11,7 @@ class lut_basic_seq extends uvm_sequence #(bb_blink_cmd_item); lut_cmd_item item; item = lut_cmd_item::type_id::create("item"); start_item(item); - item.load_rust_case(case_index, bid); + item.load_rust_case(case_index); finish_item(item); endtask endclass diff --git a/examples/prefab/balls/lut/verify/src/tb_top.sv b/examples/prefab/balls/lut/verify/src/tb_top.sv new file mode 100644 index 000000000..6e0f6d8ff --- /dev/null +++ b/examples/prefab/balls/lut/verify/src/tb_top.sv @@ -0,0 +1,89 @@ +module tb_top; + import uvm_pkg::*; + import lut_pkg::*; + + bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); + + LutBall dut ( + .clock(intf.clock), + .reset(intf.reset), + .blink_status_running(intf.status_running), + .blink_cmdReq_ready(intf.cmd_req_ready), + .blink_cmdReq_valid(intf.cmd_req_valid), + .blink_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .blink_cmdReq_bits_funct(intf.cmd_req_bits_funct), + .blink_cmdReq_bits_rs2(intf.cmd_req_bits_rs2), + .blink_cmdReq_bits_iter(intf.cmd_req_bits_iter), + .blink_cmdReq_bits_op1_bank(intf.cmd_req_bits_op1_bank), + .blink_cmdReq_bits_op2_bank(intf.cmd_req_bits_op2_bank), + .blink_cmdReq_bits_wr_bank(intf.cmd_req_bits_wr_bank), + .blink_cmdReq_bits_op1_group(intf.cmd_req_bits_op1_group), + .blink_cmdReq_bits_op2_group(intf.cmd_req_bits_op2_group), + .blink_cmdReq_bits_wr_group(intf.cmd_req_bits_wr_group), + .blink_cmdReq_bits_op1_col(intf.cmd_req_bits_op1_col), + .blink_cmdReq_bits_op2_col(intf.cmd_req_bits_op2_col), + .blink_cmdReq_bits_wr_col(intf.cmd_req_bits_wr_col), + .blink_cmdReq_bits_wr_read_bank(intf.cmd_req_bits_wr_read_bank), + .blink_cmdReq_bits_wr_arch_bank(intf.cmd_req_bits_wr_arch_bank), + .blink_cmdResp_ready(intf.cmd_resp_ready), + .blink_cmdResp_valid(intf.cmd_resp_valid), + .blink_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .blink_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .blink_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .blink_bankRead_0_group_id(intf.bank_read_group_id[0]), + .blink_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .blink_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .blink_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .blink_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .blink_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .blink_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .blink_bankRead_1_bank_id(intf.bank_read_bank_id[1]), + .blink_bankRead_1_rob_id(intf.bank_read_rob_id[1]), + .blink_bankRead_1_group_id(intf.bank_read_group_id[1]), + .blink_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), + .blink_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), + .blink_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), + .blink_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), + .blink_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), + .blink_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), + .blink_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .blink_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .blink_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .blink_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .blink_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .blink_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .blink_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .blink_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .blink_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .blink_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]) + ); + + // Constant outputs removed from the port list by firtool. + assign intf.bank_write_req_mask[0][0] = dut.blink_bankWrite_0_io_req_bits_mask_0; + assign intf.bank_write_req_mask[0][1] = dut.blink_bankWrite_0_io_req_bits_mask_1; + assign intf.bank_write_req_mask[0][2] = dut.blink_bankWrite_0_io_req_bits_mask_2; + assign intf.bank_write_req_mask[0][3] = dut.blink_bankWrite_0_io_req_bits_mask_3; + assign intf.bank_write_req_mask[0][4] = dut.blink_bankWrite_0_io_req_bits_mask_4; + assign intf.bank_write_req_mask[0][5] = dut.blink_bankWrite_0_io_req_bits_mask_5; + assign intf.bank_write_req_mask[0][6] = dut.blink_bankWrite_0_io_req_bits_mask_6; + assign intf.bank_write_req_mask[0][7] = dut.blink_bankWrite_0_io_req_bits_mask_7; + assign intf.bank_write_req_mask[0][8] = dut.blink_bankWrite_0_io_req_bits_mask_8; + assign intf.bank_write_req_mask[0][9] = dut.blink_bankWrite_0_io_req_bits_mask_9; + assign intf.bank_write_req_mask[0][10] = dut.blink_bankWrite_0_io_req_bits_mask_10; + assign intf.bank_write_req_mask[0][11] = dut.blink_bankWrite_0_io_req_bits_mask_11; + assign intf.bank_write_req_mask[0][12] = dut.blink_bankWrite_0_io_req_bits_mask_12; + assign intf.bank_write_req_mask[0][13] = dut.blink_bankWrite_0_io_req_bits_mask_13; + assign intf.bank_write_req_mask[0][14] = dut.blink_bankWrite_0_io_req_bits_mask_14; + assign intf.bank_write_req_mask[0][15] = dut.blink_bankWrite_0_io_req_bits_mask_15; + + initial begin + intf.clock = 1'b0; + intf.reset = 1'b1; + forever #5 intf.clock = ~intf.clock; + end + + initial begin + uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); + run_test("lut_ball_test"); + end +endmodule diff --git a/examples/balls/lut/verify/src/tests/lut_test.svh b/examples/prefab/balls/lut/verify/src/tests/lut_test.svh similarity index 90% rename from examples/balls/lut/verify/src/tests/lut_test.svh rename to examples/prefab/balls/lut/verify/src/tests/lut_test.svh index 18af79b1d..5d56a32e7 100644 --- a/examples/balls/lut/verify/src/tests/lut_test.svh +++ b/examples/prefab/balls/lut/verify/src/tests/lut_test.svh @@ -17,20 +17,18 @@ class lut_ball_test extends uvm_test; endfunction task run_phase(uvm_phase phase); - int unsigned bid = lut_require_bid(); phase.raise_objection(this); - for (int unsigned index = 0; index < LUT_NUM_CASES; index++) run_case(index, bid); + for (int unsigned index = 0; index < LUT_NUM_CASES; index++) run_case(index); phase.drop_objection(this); endtask - task run_case(int unsigned index, int unsigned bid); + task run_case(int unsigned index); lut_basic_seq seq; int cycles = 0; apply_reset(); env.scb.reset_counters(); seq = lut_basic_seq::type_id::create("seq"); seq.case_index = index; - seq.bid = bid; seq.start(env.cmd_agent.seqr); while (!env.scb.done()) begin @(posedge vif.clock); diff --git a/examples/prefab/balls/lut/workloads/ctests/CMakeLists.txt b/examples/prefab/balls/lut/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..2ece5dc77 --- /dev/null +++ b/examples/prefab/balls/lut/workloads/ctests/CMakeLists.txt @@ -0,0 +1,2 @@ +add_buckyball_ctest(lut_uniform_lookup_test.c) +add_buckyball_ctest(lut_channel_table_lookup_test.c) diff --git a/examples/balls/lut/workloads/ctests/lut_channel_table_lookup_test.c b/examples/prefab/balls/lut/workloads/ctests/lut_channel_table_lookup_test.c similarity index 94% rename from examples/balls/lut/workloads/ctests/lut_channel_table_lookup_test.c rename to examples/prefab/balls/lut/workloads/ctests/lut_channel_table_lookup_test.c index 333632749..bb7fbf0eb 100644 --- a/examples/balls/lut/workloads/ctests/lut_channel_table_lookup_test.c +++ b/examples/prefab/balls/lut/workloads/ctests/lut_channel_table_lookup_test.c @@ -5,7 +5,7 @@ #include #include -enum { LANES = BANK_WIDTH / 8, ROWS = 4, VALUES = ROWS * LANES }; +enum { LANES = THREAD_MEM_BANK_WIDTH / 8, ROWS = 4, VALUES = ROWS * LANES }; static int8_t input[VALUES] __attribute__((aligned(64))); static int8_t table[64][64] __attribute__((aligned(64))); static int8_t output[VALUES] __attribute__((aligned(64))); diff --git a/examples/balls/lut/workloads/ctests/lut_uniform_lookup_test.c b/examples/prefab/balls/lut/workloads/ctests/lut_uniform_lookup_test.c similarity index 92% rename from examples/balls/lut/workloads/ctests/lut_uniform_lookup_test.c rename to examples/prefab/balls/lut/workloads/ctests/lut_uniform_lookup_test.c index 06fa85965..83290c503 100644 --- a/examples/balls/lut/workloads/ctests/lut_uniform_lookup_test.c +++ b/examples/prefab/balls/lut/workloads/ctests/lut_uniform_lookup_test.c @@ -5,7 +5,7 @@ #include #include -enum { LANES = BANK_WIDTH / 8, ROWS = 4, VALUES = ROWS * LANES }; +enum { LANES = THREAD_MEM_BANK_WIDTH / 8, ROWS = 4, VALUES = ROWS * LANES }; static int8_t input[VALUES] __attribute__((aligned(64))); static int8_t table[256] __attribute__((aligned(64))); static int8_t output[VALUES] __attribute__((aligned(64))); diff --git a/examples/balls/lut/workloads/isa/lut.h b/examples/prefab/balls/lut/workloads/isa/lut.h similarity index 88% rename from examples/balls/lut/workloads/isa/lut.h rename to examples/prefab/balls/lut/workloads/isa/lut.h index 8710978b5..33781bb1a 100644 --- a/examples/balls/lut/workloads/isa/lut.h +++ b/examples/prefab/balls/lut/workloads/isa/lut.h @@ -7,6 +7,6 @@ #define bb_lut(input_bank, lut_bank, output_bank, iter) \ BUCKYBALL_INSTRUCTION_R_R(BB_BANK0(input_bank) | BB_BANK1(lut_bank) | \ BB_BANK2(output_bank) | BB_ITER(iter), \ - 0, BB_FUNC7(LUT)) + 0, FUNCT7(lut, LUT)) #endif // _BB_LUT_H_ diff --git a/examples/balls/lut/workloads/mlir_tests/CMakeLists.txt b/examples/prefab/balls/lut/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/balls/lut/workloads/mlir_tests/CMakeLists.txt rename to examples/prefab/balls/lut/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/balls/lut/workloads/mlir_tests/ball/CMakeLists.txt b/examples/prefab/balls/lut/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/balls/lut/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/prefab/balls/lut/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/prefab/balls/lut/workloads/mlir_tests/ball/lut_identity.mlir b/examples/prefab/balls/lut/workloads/mlir_tests/ball/lut_identity.mlir new file mode 100644 index 000000000..492b60cba --- /dev/null +++ b/examples/prefab/balls/lut/workloads/mlir_tests/ball/lut_identity.mlir @@ -0,0 +1,48 @@ +func.func private @check_result(memref<4x16xi8>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %four = arith.constant 4 : index + %sixteen = arith.constant 16 : index + %sixteen_i32 = arith.constant 16 : i32 + %input = memref.alloc() : memref<4x16xi8> + %table = memref.alloc() : memref<16x16xi8> + %output = memref.alloc() : memref<4x16xi8> + scf.for %row = %zero to %sixteen step %one { + scf.for %lane = %zero to %sixteen step %one { + %row32 = arith.index_cast %row : index to i32 + %lane32 = arith.index_cast %lane : index to i32 + %base = arith.muli %row32, %sixteen_i32 : i32 + %index = arith.addi %base, %lane32 : i32 + %value = arith.trunci %index : i32 to i8 + memref.store %value, %table[%row, %lane] : memref<16x16xi8> + } + } + scf.for %row = %zero to %four step %one { + scf.for %lane = %zero to %sixteen step %one { + %row32 = arith.index_cast %row : index to i32 + %lane32 = arith.index_cast %lane : index to i32 + %base = arith.muli %row32, %sixteen_i32 : i32 + %index = arith.addi %base, %lane32 : i32 + %value = arith.trunci %index : i32 to i8 + memref.store %value, %input[%row, %lane] : memref<4x16xi8> + } + } + buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %input <{bank = 1 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> + buckyball.mvin %table <{bank = 2 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + lut_ball.lookup_i8 <{codes = 1 : i64, table = 2 : i64, output = 3 : i64, rows = 4 : i64}> + buckyball.mvout %output <{bank = 3 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> + buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + func.call @check_result(%output) : (memref<4x16xi8>) -> () + memref.dealloc %input : memref<4x16xi8> + memref.dealloc %table : memref<16x16xi8> + memref.dealloc %output : memref<4x16xi8> + return %zero_i8 : i8 +} diff --git a/examples/balls/lut/workloads/mlir_tests/ball/lut_identity_main.cpp b/examples/prefab/balls/lut/workloads/mlir_tests/ball/lut_identity_main.cpp similarity index 100% rename from examples/balls/lut/workloads/mlir_tests/ball/lut_identity_main.cpp rename to examples/prefab/balls/lut/workloads/mlir_tests/ball/lut_identity_main.cpp diff --git a/examples/balls/lut/workloads/mlir_tests/bank/CMakeLists.txt b/examples/prefab/balls/lut/workloads/mlir_tests/bank/CMakeLists.txt similarity index 100% rename from examples/balls/lut/workloads/mlir_tests/bank/CMakeLists.txt rename to examples/prefab/balls/lut/workloads/mlir_tests/bank/CMakeLists.txt diff --git a/examples/prefab/balls/lut/workloads/mlir_tests/bank/lut_identity_table.mlir.in b/examples/prefab/balls/lut/workloads/mlir_tests/bank/lut_identity_table.mlir.in new file mode 100644 index 000000000..202afc25e --- /dev/null +++ b/examples/prefab/balls/lut/workloads/mlir_tests/bank/lut_identity_table.mlir.in @@ -0,0 +1,50 @@ +func.func private @check_result(memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %rows = arith.constant @BB_BANK_LINES@ : index + %lanes = arith.constant @BB_BANK_I8_LANES@ : index + %sixteen = arith.constant 16 : index + %lanes_i32 = arith.constant @BB_BANK_I8_LANES@ : i32 + %sixteen_i32 = arith.constant 16 : i32 + %input = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + %table = memref.alloc() : memref<16x16xi8> + %output = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + scf.for %row = %zero to %sixteen step %one { + scf.for %lane = %zero to %sixteen step %one { + %row32 = arith.index_cast %row : index to i32 + %lane32 = arith.index_cast %lane : index to i32 + %base = arith.muli %row32, %sixteen_i32 : i32 + %index = arith.addi %base, %lane32 : i32 + %value = arith.trunci %index : i32 to i8 + memref.store %value, %table[%row, %lane] : memref<16x16xi8> + } + } + scf.for %row = %zero to %rows step %one { + scf.for %lane = %zero to %lanes step %one { + %row32 = arith.index_cast %row : index to i32 + %lane32 = arith.index_cast %lane : index to i32 + %base = arith.muli %row32, %lanes_i32 : i32 + %index = arith.addi %base, %lane32 : i32 + %value = arith.trunci %index : i32 to i8 + memref.store %value, %input[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + } + } + %input_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %table_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %output_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %loaded_input = buckyball.bank_mvin %input, %input_bank <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + %loaded_table = buckyball.bank_mvin %table, %table_bank <{depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + %result = lut_ball.bank_lookup_i8 %loaded_input, %loaded_table -> %output_bank <{rows = @BB_BANK_LINES@ : i64}> + %stored = buckyball.bank_mvout %output, %result <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + buckyball.bank_release %loaded_input + buckyball.bank_release %loaded_table + buckyball.bank_release %stored + func.call @check_result(%output) : (memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () + memref.dealloc %input : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + memref.dealloc %table : memref<16x16xi8> + memref.dealloc %output : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + return %zero_i8 : i8 +} diff --git a/examples/prefab/balls/lut/workloads/mlir_tests/bank/lut_identity_table_main.cpp b/examples/prefab/balls/lut/workloads/mlir_tests/bank/lut_identity_table_main.cpp new file mode 100644 index 000000000..eadceb075 --- /dev/null +++ b/examples/prefab/balls/lut/workloads/mlir_tests/bank/lut_identity_table_main.cpp @@ -0,0 +1,16 @@ +#include +#include +#include +extern "C" void check_result(int8_t *, int8_t *data, int64_t offset, + int64_t rows, int64_t lanes, int64_t stride, + int64_t lane_stride) { + if (rows != THREAD_MEM_BANK_ENTRIES || lanes != THREAD_MEM_BANK_WIDTH / 8 || + stride != lanes || lane_stride != 1) + exit(1); + for (int i = 0; i < rows * lanes; ++i) + if (data[offset + i] != (int8_t)i) { + printf("lut bank mismatch index=%d\n", i); + exit(1); + } + printf("lut bank PASS rows=%d\n", THREAD_MEM_BANK_ENTRIES); +} diff --git a/examples/balls/matadd/README.md b/examples/prefab/balls/matadd/README.md similarity index 100% rename from examples/balls/matadd/README.md rename to examples/prefab/balls/matadd/README.md diff --git a/examples/prefab/balls/matadd/arch/MatAdd.scala b/examples/prefab/balls/matadd/arch/MatAdd.scala new file mode 100644 index 000000000..9e570d141 --- /dev/null +++ b/examples/prefab/balls/matadd/arch/MatAdd.scala @@ -0,0 +1,219 @@ +package examples.balls.matadd + +import framework.acc.balldomain.configs.BallRouteParam + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import framework.acc.balldomain.ball.blink.{BallCommand, BallResponse} +import chisel3.experimental.hierarchy.{instantiable, public} +import chisel3.util._ +import framework.acc.balldomain.ball.blink.BallStatus +import framework.acc.memdomain.{BankRead, BankWrite} + +@instantiable +class MatAdd( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val routingParam: BallDomainParam, + val instance: BallRouteParam) + extends Module { + val commandDataBits = frontendParam.commandDataBits + val addressWidth = log2Up(memParam.bankEntries) + val countWidth = log2Up(memParam.bankEntries + 1) + val bankIdWidth = frontendParam.bank_id_len + + val mapping = instance + + require(mapping.inBW == 2, "MatAddBall requires two SRAM read ports") + require(mapping.outBW == 1, "MatAddBall requires one SRAM write port") + require(memParam.bankWidth == 128, "MatAddBall requires 128-bit SRAM rows") + require( + memParam.bankMaskLen == 16, + "MatAddBall requires sixteen byte enables" + ) + + @public + val io = IO(new Bundle { + + val cmdReq = Flipped(Decoupled(new BallCommand(frontendParam, memParam))) + + val cmdResp = Decoupled(new BallResponse(frontendParam)) + + val bankRead = Vec( + 2, + Flipped(new BankRead( + frontendParam, + memParam + )) + ) + + val bankWrite = Vec( + 1, + Flipped(new BankWrite( + frontendParam, + memParam + )) + ) + + val status = new BallStatus + }) + + val Seq(idle, waitForChannels, readRequest, readResponse, writeRequest, writeResponse, complete) = Enum(7) + val state = RegInit(idle) + + val robId = RegInit(0.U(log2Up(frontendParam.rob_entries).W)) + val aBank = RegInit(0.U(bankIdWidth.W)) + val bBank = RegInit(0.U(bankIdWidth.W)) + val cBank = RegInit(0.U(bankIdWidth.W)) + val groups = RegInit(0.U(5.W)) + val group = RegInit(0.U(5.W)) + val bases = Reg(Vec(3, UInt(memParam.groupIdWidth.W))) + val iter = RegInit(0.U(countWidth.W)) + val line = RegInit(0.U(addressWidth.W)) + val aWord = Reg(UInt(128.W)) + val bWord = Reg(UInt(128.W)) + val aRequestSent = RegInit(false.B) + val bRequestSent = RegInit(false.B) + val aReceived = RegInit(false.B) + val bReceived = RegInit(false.B) + + val sum = Wire(Vec(4, UInt(32.W))) + for (lane <- 0 until 4) { + sum(lane) := (aWord(32 * lane + 31, 32 * lane) +& + bWord(32 * lane + 31, 32 * lane))(31, 0) + } + + io.cmdReq.ready := state === idle + io.cmdResp.valid := state === complete + io.cmdResp.bits.rob_id := robId + + for (port <- 0 until 2) { + io.bankRead(port).rob_id := robId + io.bankRead(port).bank_id := (if (port == 0) aBank else bBank) + io.bankRead(port).group_id := bases(port) + group + io.bankRead(port).io.req.valid := false.B + io.bankRead(port).io.req.bits.addr := line + io.bankRead(port).io.resp.ready := false.B + } + + io.bankWrite(0).rob_id := robId + io.bankWrite(0).bank_id := cBank + io.bankWrite(0).group_id := bases(2) + group + io.bankWrite(0).io.req.valid := false.B + io.bankWrite(0).io.req.bits.addr := line + io.bankWrite(0).io.req.bits.data := Cat(sum.reverse) + io.bankWrite(0).io.req.bits.mask := VecInit(Seq.fill(16)(true.B)) + io.bankWrite(0).io.resp.ready := false.B + + switch(state) { + is(idle) { + when(io.cmdReq.fire) { + val command = io.cmdReq.bits + + assert( + command.op1_bank =/= command.op2_bank && command.op1_bank =/= command.wr_bank && + command.op2_bank =/= command.wr_bank, + "MatAddBall banks must be distinct" + ) + assert( + command.op1_col =/= 0.U && command.op1_col === command.op2_col && + command.op1_col === command.wr_col, + "MatAddBall bank groups must match" + ) + assert( + command.op1_col <= memParam.bankNum.U, + "MatAddBall bank groups must fit in physical banks" + ) + assert( + command.iter =/= 0.U && command.iter <= memParam.bankEntries.U, + "MatAddBall iter must fit in one physical bank" + ) + assert(command.rs2 === 0.U, "MatAddBall reserves rs2") + + robId := io.cmdReq.bits.rob_id + aBank := command.op1_bank + bBank := command.op2_bank + cBank := command.wr_bank + groups := command.op1_col + group := 0.U + bases := VecInit(command.op1_group, command.op2_group, command.wr_group) + iter := command.iter + line := 0.U + aRequestSent := false.B + bRequestSent := false.B + aReceived := false.B + bReceived := false.B + state := waitForChannels + } + } + + is(waitForChannels) { + state := readRequest + } + + is(readRequest) { + io.bankRead(0).io.req.valid := !aRequestSent + io.bankRead(1).io.req.valid := !bRequestSent + when(io.bankRead(0).io.req.fire)(aRequestSent := true.B) + when(io.bankRead(1).io.req.fire)(bRequestSent := true.B) + when((aRequestSent || io.bankRead(0).io.req.fire) && + (bRequestSent || io.bankRead(1).io.req.fire)) { + state := readResponse + } + } + + is(readResponse) { + io.bankRead(0).io.resp.ready := !aReceived + io.bankRead(1).io.resp.ready := !bReceived + when(io.bankRead(0).io.resp.fire) { + aWord := io.bankRead(0).io.resp.bits.data + aReceived := true.B + } + when(io.bankRead(1).io.resp.fire) { + bWord := io.bankRead(1).io.resp.bits.data + bReceived := true.B + } + when((aReceived || io.bankRead(0).io.resp.fire) && + (bReceived || io.bankRead(1).io.resp.fire)) { + state := writeRequest + } + } + + is(writeRequest) { + io.bankWrite(0).io.req.valid := true.B + when(io.bankWrite(0).io.req.fire)(state := writeResponse) + } + + is(writeResponse) { + io.bankWrite(0).io.resp.ready := true.B + when(io.bankWrite(0).io.resp.fire) { + aRequestSent := false.B + bRequestSent := false.B + aReceived := false.B + bReceived := false.B + when(line +& 1.U === iter) { + line := 0.U + when(group +& 1.U === groups) { + state := complete + }.otherwise { + group := group + 1.U + state := readRequest + } + }.otherwise { + line := line + 1.U + state := readRequest + } + } + } + + is(complete) { + when(io.cmdResp.fire)(state := idle) + } + } + + io.status.idle := state === idle + io.status.running := state =/= idle +} diff --git a/examples/prefab/balls/matadd/arch/MatAddBall.scala b/examples/prefab/balls/matadd/arch/MatAddBall.scala new file mode 100644 index 000000000..c6fc7517a --- /dev/null +++ b/examples/prefab/balls/matadd/arch/MatAddBall.scala @@ -0,0 +1,49 @@ +package examples.balls.matadd + +import framework.acc.balldomain.bbus.{BallFactory, BallPorts} +import buckyball.balls.matadd.{Params => ParametersProto} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.balldomain.ball.blink.{BlinkIO, HasBlink} + +@instantiable +class MatAddBall( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Module + with HasBlink { + val ball = bp.balls.find(_.id == id).get + + ball.parameters.unpack(classOf[ParametersProto]) + + @public + val blink = IO(new BlinkIO(fp, mp, bp, id)) + + val unit: Instance[MatAdd] = + Instantiate(new MatAdd(fp, mp, bp, ball)) + unit.io.cmdReq <> blink.cmdReq + unit.io.cmdResp <> blink.cmdResp + unit.io.bankRead(0) <> blink.bankRead(0) + unit.io.bankRead(1) <> blink.bankRead(1) + unit.io.bankWrite(0) <> blink.bankWrite(0) + blink.status <> unit.io.status +} + +object MatAddBall extends BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts = + BallPorts(Instantiate(new MatAddBall(fp, mp, bp, id)).blink, None) + +} diff --git a/arch/src/main/scala/framework/root/root-chip/src/csrc/README.md b/examples/prefab/balls/matadd/configs/default.toml similarity index 100% rename from arch/src/main/scala/framework/root/root-chip/src/csrc/README.md rename to examples/prefab/balls/matadd/configs/default.toml diff --git a/examples/prefab/balls/matadd/configs/matadd_params.proto b/examples/prefab/balls/matadd/configs/matadd_params.proto new file mode 100644 index 000000000..4f6956443 --- /dev/null +++ b/examples/prefab/balls/matadd/configs/matadd_params.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; +package buckyball.balls.matadd; +option java_package = "buckyball.balls.matadd"; +option java_multiple_files = true; + +message Params { +} diff --git a/examples/balls/matadd/emu/src/64_matadd.rs b/examples/prefab/balls/matadd/emu/src/64_matadd.rs similarity index 90% rename from examples/balls/matadd/emu/src/64_matadd.rs rename to examples/prefab/balls/matadd/emu/src/64_matadd.rs index efee1a5cb..f38d59b8f 100644 --- a/examples/balls/matadd/emu/src/64_matadd.rs +++ b/examples/prefab/balls/matadd/emu/src/64_matadd.rs @@ -1,11 +1,12 @@ -use super::super::bank::bank_size; -use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; -use super::instruction::{BallInstruction, ExecContext}; +use bebop_bemu::thread::banks::bank_size; +use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old}; +use super::instruction::{BallModel, ExecContext}; pub struct MatAdd; -impl BallInstruction for MatAdd { +impl BallModel for MatAdd { fn exec(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + assert!(!rs1_read_old(xs1), "matadd: reads no old destination; rs1[29] must be clear"); let a_bank = rs1_b0(xs1); let b_bank = rs1_b1(xs1); let c_bank = rs1_b2(xs1); diff --git a/examples/prefab/balls/matadd/emu/src/dispatch.rs b/examples/prefab/balls/matadd/emu/src/dispatch.rs new file mode 100644 index 000000000..d090f7398 --- /dev/null +++ b/examples/prefab/balls/matadd/emu/src/dispatch.rs @@ -0,0 +1,31 @@ +use bebop_bemu::ball::{BallModel, ExecContext}; + +use super::f64_matadd; + +const BALL_CLASS: &str = "examples.balls.matadd.MatAddBall"; + +pub fn execute_known( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, + ctx: &mut ExecContext, +) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("MATADD") => Some(f64_matadd::MatAdd::exec(xs1, xs2, ctx)), + Some(_) | None => None, + } +} + +pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("MATADD") => Some(f64_matadd::MatAdd::latency(xs1, xs2)), + Some(_) | None => None, + } +} diff --git a/examples/prefab/balls/matadd/emu/src/lib.rs b/examples/prefab/balls/matadd/emu/src/lib.rs new file mode 100644 index 000000000..6242a9b5a --- /dev/null +++ b/examples/prefab/balls/matadd/emu/src/lib.rs @@ -0,0 +1,12 @@ +pub(crate) use bebop_bemu::{thread::instructions::decode, ball as instruction}; + +#[path = "64_matadd.rs"] +mod f64_matadd; + +mod dispatch; + +pub use dispatch::{cycles_after_issue, execute_known}; + +#[bebop_bemu::ball::linkme::distributed_slice(bebop_bemu::BALLS)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static BALL: bebop_bemu::Ball = bebop_bemu::Ball { execute: execute_known, cycles: cycles_after_issue }; diff --git a/examples/prefab/balls/matadd/stack/src/mataddOps.cpp b/examples/prefab/balls/matadd/stack/src/mataddOps.cpp new file mode 100644 index 000000000..922d29ce1 --- /dev/null +++ b/examples/prefab/balls/matadd/stack/src/mataddOps.cpp @@ -0,0 +1,67 @@ +//===- mataddOps.cpp - MatAddBall operations ------------------------------===// +// +// The dialect of MatAddBall, its virtual operations building their physical +// ones and the instructions of the physical operations. +// +//===----------------------------------------------------------------------===// + +#include "mataddOps.h" + +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::matadd_ball; +using mlir::buddy::bankFields; +using mlir::buddy::constantI64; +using mlir::buddy::emitInstruction; +using mlir::buddy::field; + +#include "mataddDialect.cpp.inc" + +#define GET_OP_CLASSES +#include "mataddOps.cpp.inc" + +void MatAddBallDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "mataddOps.cpp.inc" + >(); +} + +namespace { +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallDialect([](DialectRegistry ®istry) { + registry.insert(); + }), + true); +} // namespace + +SmallVector BankAddI32Op::getSourceBanks() { + return {getA(), getB()}; +} +Value BankAddI32Op::getDestinationBank() { return getC(); } +bool BankAddI32Op::readsDestination() { return false; } +Operation *BankAddI32Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return AddI32Op::create(builder, getLoc(), sources[0], sources[1], + destination, getRows()); +} + +SmallVector AddI32Op::getSourceBanks() { + return {int64_t(getA()), int64_t(getB())}; +} +int64_t AddI32Op::getDestinationBank() { return getC(); } +LogicalResult AddI32Op::lowerToInstructions(OpBuilder &builder) { + Location loc = getLoc(); + int64_t rows = getRows(), a = getA(), b = getB(), c = getC(); + if (rows <= 0 || rows > buckyball_target::getBuckyballTarget().bankDepth || + a == b || a == c || b == c) + return emitOpError( + "add_i32 requires distinct banks and rows within a bank"); + emitInstruction(builder, loc, buckyball_target::getBuckyballFunct7("MATADD"), + constantI64(builder, loc, bankFields({a, b}, c, false, rows)), + constantI64(builder, loc, 0)); + return success(); +} diff --git a/examples/prefab/balls/matadd/stack/src/mataddOps.h b/examples/prefab/balls/matadd/stack/src/mataddOps.h new file mode 100644 index 000000000..ce07f9267 --- /dev/null +++ b/examples/prefab/balls/matadd/stack/src/mataddOps.h @@ -0,0 +1,20 @@ +//===- mataddOps.h - MatAddBall operations --------------------------------===// + +#ifndef MATADD_OPS_H +#define MATADD_OPS_H + +#include "Interfaces/BankInterfaces.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "mataddDialect.h.inc" + +#define GET_OP_CLASSES +#include "mataddOps.h.inc" + +#endif // MATADD_OPS_H diff --git a/examples/prefab/balls/matadd/stack/src/mataddOps.td b/examples/prefab/balls/matadd/stack/src/mataddOps.td new file mode 100644 index 000000000..81265507d --- /dev/null +++ b/examples/prefab/balls/matadd/stack/src/mataddOps.td @@ -0,0 +1,40 @@ +//===- mataddOps.td - MatAddBall operations ----------------*- tablegen -*-===// +// +// add_i32: C = A + B modulo 2^32 over `rows` rows of four i32 in every group +// of the banks; A, B and C have the same number of groups. +// +//===----------------------------------------------------------------------===// + +#ifndef MATADD_OPS +#define MATADD_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Interfaces/BankInterfaces.td" + +def MatAddBall_Dialect : Dialect { + let name = "matadd_ball"; + let cppNamespace = "::buddy::matadd_ball"; + let summary = "The operations of MatAddBall"; +} + +class MatAddBall_Op traits = []> + : Op])>; + +def BankAddI32Op : MatAddBall_Op<"bank_add_i32", + [DeclareOpInterfaceMethods]> { + let summary = "add_i32 on virtual banks"; + let arguments = (ins I64:$a, I64:$b, I64:$c, I64Attr:$rows); + let results = (outs I64:$cOut); + let assemblyFormat = "$a `,` $b `->` $c prop-dict attr-dict"; +} + +def AddI32Op : MatAddBall_Op<"add_i32", + [DeclareOpInterfaceMethods]> { + let summary = "add_i32 on physical banks"; + let arguments = (ins I64Attr:$a, I64Attr:$b, I64Attr:$c, I64Attr:$rows); + let assemblyFormat = "prop-dict attr-dict"; +} + +#endif // MATADD_OPS diff --git a/examples/balls/matadd/verify/casegen/Cargo.lock b/examples/prefab/balls/matadd/verify/casegen/Cargo.lock similarity index 100% rename from examples/balls/matadd/verify/casegen/Cargo.lock rename to examples/prefab/balls/matadd/verify/casegen/Cargo.lock diff --git a/examples/balls/matadd/verify/casegen/Cargo.toml b/examples/prefab/balls/matadd/verify/casegen/Cargo.toml similarity index 100% rename from examples/balls/matadd/verify/casegen/Cargo.toml rename to examples/prefab/balls/matadd/verify/casegen/Cargo.toml diff --git a/examples/balls/matadd/verify/casegen/src/lib.rs b/examples/prefab/balls/matadd/verify/casegen/src/lib.rs similarity index 100% rename from examples/balls/matadd/verify/casegen/src/lib.rs rename to examples/prefab/balls/matadd/verify/casegen/src/lib.rs diff --git a/examples/balls/matadd/verify/filelists/matadd_ball.f b/examples/prefab/balls/matadd/verify/filelists/matadd_ball.f similarity index 89% rename from examples/balls/matadd/verify/filelists/matadd_ball.f rename to examples/prefab/balls/matadd/verify/filelists/matadd_ball.f index f4f281933..87c3ff133 100644 --- a/examples/balls/matadd/verify/filelists/matadd_ball.f +++ b/examples/prefab/balls/matadd/verify/filelists/matadd_ball.f @@ -1,3 +1,4 @@ ++incdir+@UVM@/src/core +incdir+@UVM@/src/ball +incdir+@UVM@/src/ball/agents/cmd +incdir+@UVM@/src/ball/agents/mem @@ -12,6 +13,7 @@ +incdir+src/tests +incdir+@RTL@ +@UVM@/src/core/package.sv @UVM@/src/ball/bb_blink_if.sv @UVM@/src/ball/bb_uvm_pkg.sv diff --git a/examples/prefab/balls/matadd/verify/src/common/matadd_defs.svh b/examples/prefab/balls/matadd/verify/src/common/matadd_defs.svh new file mode 100644 index 000000000..5580a1ef8 --- /dev/null +++ b/examples/prefab/balls/matadd/verify/src/common/matadd_defs.svh @@ -0,0 +1,7 @@ +`ifndef MATADD_FUNCT7 +`error "MATADD_FUNCT7 must be provided by the selected Thread ballISA" +`endif + +localparam int MATADD_THREAD_FUNCT7 = `MATADD_FUNCT7; +localparam int MATADD_MAX_LINES = 64; +localparam int MATADD_TIMEOUT_CYCLES = 100000; diff --git a/examples/prefab/balls/matadd/verify/src/common/matadd_items.svh b/examples/prefab/balls/matadd/verify/src/common/matadd_items.svh new file mode 100644 index 000000000..f63284e2b --- /dev/null +++ b/examples/prefab/balls/matadd/verify/src/common/matadd_items.svh @@ -0,0 +1,78 @@ +class matadd_cmd_item extends bb_blink_cmd_item; + `uvm_object_utils(matadd_cmd_item) + + int unsigned lines; + int unsigned group_count; + int unsigned case_index; + + function new(string name = "matadd_cmd_item"); + super.new(name); + endfunction + + // Case 0 covers one group at base 0; cases 1 and 2 cover two groups at nonzero bases. + function void load_case(int unsigned index); + if (index > 2) `uvm_fatal("CASE", $sformatf("unsupported case %0d", index)) + case_index = index; + // bb_matadd: banks and iter come from rs1, rs2 is zero. + rob_id = 4 + index; + funct = MATADD_THREAD_FUNCT7[6:0]; + rs2 = 64'd0; + iter = (index == 2) ? MATADD_MAX_LINES : 16; + lines = iter; + group_count = (index == 0) ? 1 : 2; + op1_bank = 0; + op2_bank = 1; + wr_bank = 2; + op1_group = (index == 0) ? 0 : 1; + op2_group = (index == 0) ? 0 : 3; + wr_group = (index == 0) ? 0 : index; + op1_col = group_count; + op2_col = group_count; + wr_col = group_count; + wr_read_bank = 0; + wr_arch_bank = wr_bank; + endfunction + + function automatic bit [31:0] a_lane(int unsigned group, int unsigned line, int unsigned lane); + int unsigned offset; + offset = (group * lines + line) * 4 + lane; + if (case_index == 2) return 32'hffff_ff00 + offset; + return 32'h1000_0000 + offset * 32'h0001_0011; + endfunction + + function automatic bit [31:0] b_lane(int unsigned group, int unsigned line, int unsigned lane); + int unsigned offset; + offset = (group * lines + line) * 4 + lane; + if (case_index == 2) return 32'h0000_0200 + offset * 5; + return 32'he000_0000 + offset * 32'h0000_0103; + endfunction + + function automatic bit [127:0] a_word(int unsigned group, int unsigned line); + bit [127:0] word; + for (int lane = 0; lane < 4; lane++) word[lane*32+:32] = a_lane(group, line, lane); + return word; + endfunction + + function automatic bit [127:0] b_word(int unsigned group, int unsigned line); + bit [127:0] word; + for (int lane = 0; lane < 4; lane++) word[lane*32+:32] = b_lane(group, line, lane); + return word; + endfunction + + function automatic bit [127:0] sum_word(int unsigned group, int unsigned line); + bit [127:0] word; + for (int lane = 0; lane < 4; lane++) begin + word[lane*32+:32] = a_lane(group, line, lane) + b_lane(group, line, lane); + end + return word; + endfunction + + function void do_copy(uvm_object rhs); + matadd_cmd_item rhs_item; + super.do_copy(rhs); + if (!$cast(rhs_item, rhs)) `uvm_fatal("COPY", "rhs is not matadd_cmd_item") + lines = rhs_item.lines; + group_count = rhs_item.group_count; + case_index = rhs_item.case_index; + endfunction +endclass diff --git a/examples/balls/matadd/verify/src/env/matadd_env.svh b/examples/prefab/balls/matadd/verify/src/env/matadd_env.svh similarity index 100% rename from examples/balls/matadd/verify/src/env/matadd_env.svh rename to examples/prefab/balls/matadd/verify/src/env/matadd_env.svh diff --git a/examples/balls/matadd/verify/src/env/matadd_scoreboard.svh b/examples/prefab/balls/matadd/verify/src/env/matadd_scoreboard.svh similarity index 75% rename from examples/balls/matadd/verify/src/env/matadd_scoreboard.svh rename to examples/prefab/balls/matadd/verify/src/env/matadd_scoreboard.svh index 5efeca2b3..7f39dd5b2 100644 --- a/examples/balls/matadd/verify/src/env/matadd_scoreboard.svh +++ b/examples/prefab/balls/matadd/verify/src/env/matadd_scoreboard.svh @@ -34,8 +34,10 @@ class matadd_scoreboard extends uvm_scoreboard; mem_model.clear_mem(); for (int group = 0; group < stim.group_count; group++) begin for (int line = 0; line < stim.lines; line++) begin - mem_model.load_word_g(stim.op1_bank, group, line, stim.a_word(group, line)); - mem_model.load_word_g(stim.op2_bank, group, line, stim.b_word(group, line)); + mem_model.load_word_g(stim.op1_bank, stim.op1_group + group, line, stim.a_word(group, line + )); + mem_model.load_word_g(stim.op2_bank, stim.op2_group + group, line, stim.b_word(group, line + )); end end mem_model.arm(); @@ -51,24 +53,11 @@ class matadd_scoreboard extends uvm_scoreboard; endfunction function void write_cmd(bb_blink_cmd_item item); - matadd_cmd_item expected; - expected = current_stim("CMD"); - if (item.bid !== expected.bid) `uvm_fatal("CMD", "bid mismatch") - if (item.funct7 !== expected.funct7) `uvm_fatal("CMD", "funct7 mismatch") - if (item.iter !== expected.iter) `uvm_fatal("CMD", "iter mismatch") - if (item.op1_bank !== expected.op1_bank || item.op2_bank !== expected.op2_bank || - item.wr_bank !== expected.wr_bank) - `uvm_fatal("CMD", "bank field mismatch") - if (item.op1_col !== expected.group_count || item.op2_col !== expected.group_count || - item.wr_col !== expected.group_count) - `uvm_fatal("CMD", "group field mismatch") - if (item.rob_id !== expected.rob_id) `uvm_fatal("CMD", "rob_id mismatch") - if (item.op1_en !== 1'b1 || item.op2_en !== 1'b1 || item.wr_spad_en !== 1'b1 || - item.op1_from_spad !== 1'b1 || item.op2_from_spad !== 1'b1) - `uvm_fatal("CMD", "enable field mismatch") - if (item.meta_bank !== 0 || item.rs1 !== 0 || item.rs2 !== 0 || item.special !== 0 || - item.is_sub !== 0 || item.sub_rob_id !== 0) - `uvm_fatal("CMD", "reserved field mismatch") + // Compare as plain BLink commands; the comparer rejects differing types. + bb_blink_cmd_item want = bb_blink_cmd_item::type_id::create("want"); + want.copy(current_stim("CMD")); + if (!item.compare(want)) + `uvm_fatal("CMD", $sformatf("command mismatch: got %s exp %s", item.sprint(), want.sprint())) if (cmd_count != 0) `uvm_fatal("CMD", "extra command observed") cmd_count++; endfunction @@ -87,7 +76,8 @@ class matadd_scoreboard extends uvm_scoreboard; if (item.rob_id !== expected.rob_id) `uvm_fatal("READ", "rob_id mismatch") if (item.bank_id !== (item.port == 0 ? expected.op1_bank : expected.op2_bank)) `uvm_fatal("READ", "bank mismatch") - if (item.group_id !== group || item.addr !== line) + if (item.group_id !== (item.port == 0 ? expected.op1_group : expected.op2_group) + group || + item.addr !== line) `uvm_fatal("READ", $sformatf( "address mismatch port=%0d group=%0d line=%0d", item.port, item.group_id, item.addr )) @@ -104,7 +94,7 @@ class matadd_scoreboard extends uvm_scoreboard; line = write_count % expected.lines; if (item.port != 0 || item.bank_id !== expected.wr_bank || item.rob_id !== expected.rob_id) `uvm_fatal("WRITE", "port, bank, or rob_id mismatch") - if (item.group_id !== group || item.addr !== line) + if (item.group_id !== expected.wr_group + group || item.addr !== line) `uvm_fatal("WRITE", $sformatf("address mismatch group=%0d line=%0d", item.group_id, item.addr )) if (item.mask !== 16'hffff) `uvm_fatal("WRITE", "mask mismatch") @@ -117,8 +107,7 @@ class matadd_scoreboard extends uvm_scoreboard; matadd_cmd_item expected; expected = current_stim("RESP"); if (resp_count != 0) `uvm_fatal("RESP", "extra response observed") - if (item.rob_id !== expected.rob_id || item.is_sub !== 0 || item.sub_rob_id !== 0) - `uvm_fatal("RESP", "response mismatch") + if (item.rob_id !== expected.rob_id) `uvm_fatal("RESP", "response mismatch") resp_count++; endfunction @@ -137,6 +126,7 @@ class matadd_scoreboard extends uvm_scoreboard; endfunction function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; super.check_phase(phase); if (!done()) `uvm_fatal("SCB", $sformatf( diff --git a/examples/balls/matadd/verify/src/pkg/matadd_pkg.sv b/examples/prefab/balls/matadd/verify/src/pkg/matadd_pkg.sv similarity index 100% rename from examples/balls/matadd/verify/src/pkg/matadd_pkg.sv rename to examples/prefab/balls/matadd/verify/src/pkg/matadd_pkg.sv diff --git a/examples/balls/matadd/verify/src/seq/matadd_sequences.svh b/examples/prefab/balls/matadd/verify/src/seq/matadd_sequences.svh similarity index 86% rename from examples/balls/matadd/verify/src/seq/matadd_sequences.svh rename to examples/prefab/balls/matadd/verify/src/seq/matadd_sequences.svh index e621df916..345175084 100644 --- a/examples/balls/matadd/verify/src/seq/matadd_sequences.svh +++ b/examples/prefab/balls/matadd/verify/src/seq/matadd_sequences.svh @@ -2,7 +2,6 @@ class matadd_seq extends uvm_sequence #(bb_blink_cmd_item); `uvm_object_utils(matadd_seq) int unsigned case_index; - int unsigned bid; function new(string name = "matadd_seq"); super.new(name); @@ -12,7 +11,7 @@ class matadd_seq extends uvm_sequence #(bb_blink_cmd_item); matadd_cmd_item item; item = matadd_cmd_item::type_id::create("item"); start_item(item); - item.load_case(case_index, bid); + item.load_case(case_index); finish_item(item); endtask endclass diff --git a/examples/prefab/balls/matadd/verify/src/tb_top.sv b/examples/prefab/balls/matadd/verify/src/tb_top.sv new file mode 100644 index 000000000..fe7450698 --- /dev/null +++ b/examples/prefab/balls/matadd/verify/src/tb_top.sv @@ -0,0 +1,89 @@ +module tb_top; + import uvm_pkg::*; + import matadd_pkg::*; + + bb_blink_if #(2, 1) intf (); + + MatAddBall dut ( + .clock(intf.clock), + .reset(intf.reset), + .blink_status_running(intf.status_running), + .blink_cmdReq_ready(intf.cmd_req_ready), + .blink_cmdReq_valid(intf.cmd_req_valid), + .blink_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .blink_cmdReq_bits_funct(intf.cmd_req_bits_funct), + .blink_cmdReq_bits_rs2(intf.cmd_req_bits_rs2), + .blink_cmdReq_bits_iter(intf.cmd_req_bits_iter), + .blink_cmdReq_bits_op1_bank(intf.cmd_req_bits_op1_bank), + .blink_cmdReq_bits_op2_bank(intf.cmd_req_bits_op2_bank), + .blink_cmdReq_bits_wr_bank(intf.cmd_req_bits_wr_bank), + .blink_cmdReq_bits_op1_group(intf.cmd_req_bits_op1_group), + .blink_cmdReq_bits_op2_group(intf.cmd_req_bits_op2_group), + .blink_cmdReq_bits_wr_group(intf.cmd_req_bits_wr_group), + .blink_cmdReq_bits_op1_col(intf.cmd_req_bits_op1_col), + .blink_cmdReq_bits_op2_col(intf.cmd_req_bits_op2_col), + .blink_cmdReq_bits_wr_col(intf.cmd_req_bits_wr_col), + .blink_cmdReq_bits_wr_read_bank(intf.cmd_req_bits_wr_read_bank), + .blink_cmdReq_bits_wr_arch_bank(intf.cmd_req_bits_wr_arch_bank), + .blink_cmdResp_ready(intf.cmd_resp_ready), + .blink_cmdResp_valid(intf.cmd_resp_valid), + .blink_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .blink_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .blink_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .blink_bankRead_0_group_id(intf.bank_read_group_id[0]), + .blink_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .blink_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .blink_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .blink_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .blink_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .blink_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .blink_bankRead_1_bank_id(intf.bank_read_bank_id[1]), + .blink_bankRead_1_rob_id(intf.bank_read_rob_id[1]), + .blink_bankRead_1_group_id(intf.bank_read_group_id[1]), + .blink_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), + .blink_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), + .blink_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), + .blink_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), + .blink_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), + .blink_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), + .blink_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .blink_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .blink_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .blink_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .blink_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .blink_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .blink_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .blink_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .blink_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .blink_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]) + ); + + // Constant outputs removed from the port list by firtool. + assign intf.bank_write_req_mask[0][0] = dut.blink_bankWrite_0_io_req_bits_mask_0; + assign intf.bank_write_req_mask[0][1] = dut.blink_bankWrite_0_io_req_bits_mask_1; + assign intf.bank_write_req_mask[0][2] = dut.blink_bankWrite_0_io_req_bits_mask_2; + assign intf.bank_write_req_mask[0][3] = dut.blink_bankWrite_0_io_req_bits_mask_3; + assign intf.bank_write_req_mask[0][4] = dut.blink_bankWrite_0_io_req_bits_mask_4; + assign intf.bank_write_req_mask[0][5] = dut.blink_bankWrite_0_io_req_bits_mask_5; + assign intf.bank_write_req_mask[0][6] = dut.blink_bankWrite_0_io_req_bits_mask_6; + assign intf.bank_write_req_mask[0][7] = dut.blink_bankWrite_0_io_req_bits_mask_7; + assign intf.bank_write_req_mask[0][8] = dut.blink_bankWrite_0_io_req_bits_mask_8; + assign intf.bank_write_req_mask[0][9] = dut.blink_bankWrite_0_io_req_bits_mask_9; + assign intf.bank_write_req_mask[0][10] = dut.blink_bankWrite_0_io_req_bits_mask_10; + assign intf.bank_write_req_mask[0][11] = dut.blink_bankWrite_0_io_req_bits_mask_11; + assign intf.bank_write_req_mask[0][12] = dut.blink_bankWrite_0_io_req_bits_mask_12; + assign intf.bank_write_req_mask[0][13] = dut.blink_bankWrite_0_io_req_bits_mask_13; + assign intf.bank_write_req_mask[0][14] = dut.blink_bankWrite_0_io_req_bits_mask_14; + assign intf.bank_write_req_mask[0][15] = dut.blink_bankWrite_0_io_req_bits_mask_15; + + initial begin + intf.clock = 1'b0; + intf.reset = 1'b1; + forever #5 intf.clock = ~intf.clock; + end + + initial begin + uvm_config_db#(virtual bb_blink_if #(2, 1))::set(null, "*", "vif", intf); + run_test("matadd_ball_test"); + end +endmodule diff --git a/examples/balls/matadd/verify/src/tests/matadd_test.svh b/examples/prefab/balls/matadd/verify/src/tests/matadd_test.svh similarity index 84% rename from examples/balls/matadd/verify/src/tests/matadd_test.svh rename to examples/prefab/balls/matadd/verify/src/tests/matadd_test.svh index 725553e42..7f4d68cd9 100644 --- a/examples/balls/matadd/verify/src/tests/matadd_test.svh +++ b/examples/prefab/balls/matadd/verify/src/tests/matadd_test.svh @@ -17,23 +17,20 @@ class matadd_ball_test extends uvm_test; endfunction task run_phase(uvm_phase phase); - int unsigned bid; - bid = matadd_require_bid(); phase.raise_objection(this); - run_case(0, "ONE_GROUP_16_LINES", bid); - run_case(1, "TWO_GROUPS_16_LINES", bid); - run_case(2, "TWO_GROUPS_64_LINES", bid); + run_case(0, "ONE_GROUP_16_LINES"); + run_case(1, "TWO_GROUPS_16_LINES"); + run_case(2, "TWO_GROUPS_64_LINES"); phase.drop_objection(this); endtask - task run_case(int unsigned index, string name, int unsigned bid); + task run_case(int unsigned index, string name); matadd_seq seq; int cycles; apply_reset(); env.scb.reset_counters(); seq = matadd_seq::type_id::create("seq"); seq.case_index = index; - seq.bid = bid; seq.start(env.cmd_agent.seqr); cycles = 0; while (!env.scb.done()) begin diff --git a/examples/prefab/balls/matadd/workloads/ctests/CMakeLists.txt b/examples/prefab/balls/matadd/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..c1a53407d --- /dev/null +++ b/examples/prefab/balls/matadd/workloads/ctests/CMakeLists.txt @@ -0,0 +1 @@ +add_buckyball_ctest(matadd_i32_16row_test.c) diff --git a/examples/balls/matadd/workloads/ctests/matadd_i32_16row_test.c b/examples/prefab/balls/matadd/workloads/ctests/matadd_i32_16row_test.c similarity index 100% rename from examples/balls/matadd/workloads/ctests/matadd_i32_16row_test.c rename to examples/prefab/balls/matadd/workloads/ctests/matadd_i32_16row_test.c diff --git a/examples/balls/matadd/workloads/isa/matadd.h b/examples/prefab/balls/matadd/workloads/isa/matadd.h similarity index 87% rename from examples/balls/matadd/workloads/isa/matadd.h rename to examples/prefab/balls/matadd/workloads/isa/matadd.h index 08b701742..196478d46 100644 --- a/examples/balls/matadd/workloads/isa/matadd.h +++ b/examples/prefab/balls/matadd/workloads/isa/matadd.h @@ -7,6 +7,6 @@ #define bb_matadd(a_bank, b_bank, c_bank, iter) \ BUCKYBALL_INSTRUCTION_R_R((BB_BANK0(a_bank) | BB_BANK1(b_bank) | \ BB_BANK2(c_bank) | BB_ITER(iter)), \ - 0, BB_FUNC7(MATADD)) + 0, FUNCT7(matadd, MATADD)) #endif // _BB_MATADD_H_ diff --git a/examples/balls/matadd/workloads/mlir_tests/CMakeLists.txt b/examples/prefab/balls/matadd/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/balls/matadd/workloads/mlir_tests/CMakeLists.txt rename to examples/prefab/balls/matadd/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/balls/matadd/workloads/mlir_tests/ball/CMakeLists.txt b/examples/prefab/balls/matadd/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/balls/matadd/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/prefab/balls/matadd/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/prefab/balls/matadd/workloads/mlir_tests/ball/matadd_i32.mlir b/examples/prefab/balls/matadd/workloads/mlir_tests/ball/matadd_i32.mlir new file mode 100644 index 000000000..de019547a --- /dev/null +++ b/examples/prefab/balls/matadd/workloads/mlir_tests/ball/matadd_i32.mlir @@ -0,0 +1,37 @@ +func.func private @check_result(memref<16x4xi32>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %four = arith.constant 4 : index + %sixteen = arith.constant 16 : index + %a = memref.alloc() : memref<16x4xi32> + %b = memref.alloc() : memref<16x4xi32> + %out = memref.alloc() : memref<16x4xi32> + scf.for %row = %zero to %sixteen step %one { + scf.for %lane = %zero to %four step %one { + %r = arith.index_cast %row : index to i32 + %l = arith.index_cast %lane : index to i32 + %x = arith.addi %r, %l : i32 + %y = arith.subi %l, %r : i32 + memref.store %x, %a[%row, %lane] : memref<16x4xi32> + memref.store %y, %b[%row, %lane] : memref<16x4xi32> + } + } + buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %a <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> + buckyball.mvin %b <{bank = 2 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> + matadd_ball.add_i32 <{a = 1 : i64, b = 2 : i64, c = 3 : i64, rows = 16 : i64}> + buckyball.mvout %out <{bank = 3 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> + buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + func.call @check_result(%out) : (memref<16x4xi32>) -> () + memref.dealloc %a : memref<16x4xi32> + memref.dealloc %b : memref<16x4xi32> + memref.dealloc %out : memref<16x4xi32> + return %zero_i8 : i8 +} diff --git a/examples/balls/matadd/workloads/mlir_tests/ball/matadd_i32_main.cpp b/examples/prefab/balls/matadd/workloads/mlir_tests/ball/matadd_i32_main.cpp similarity index 96% rename from examples/balls/matadd/workloads/mlir_tests/ball/matadd_i32_main.cpp rename to examples/prefab/balls/matadd/workloads/mlir_tests/ball/matadd_i32_main.cpp index a8a1752f6..961c81a41 100644 --- a/examples/balls/matadd/workloads/mlir_tests/ball/matadd_i32_main.cpp +++ b/examples/prefab/balls/matadd/workloads/mlir_tests/ball/matadd_i32_main.cpp @@ -1,4 +1,3 @@ -#include #include #include #include diff --git a/examples/balls/matadd/workloads/mlir_tests/bank/CMakeLists.txt b/examples/prefab/balls/matadd/workloads/mlir_tests/bank/CMakeLists.txt similarity index 100% rename from examples/balls/matadd/workloads/mlir_tests/bank/CMakeLists.txt rename to examples/prefab/balls/matadd/workloads/mlir_tests/bank/CMakeLists.txt diff --git a/examples/prefab/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise.mlir.in b/examples/prefab/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise.mlir.in new file mode 100644 index 000000000..21f60eb25 --- /dev/null +++ b/examples/prefab/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise.mlir.in @@ -0,0 +1,40 @@ +func.func private @check_result(memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %rows = arith.constant @BB_BANK_LINES@ : index + %lanes = arith.constant @BB_BANK_I32_LANES@ : index + %a = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + %b = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + %out = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + scf.for %row = %zero to %rows step %one { + scf.for %lane = %zero to %lanes step %one { + %r = arith.index_cast %row : index to i32 + %l = arith.index_cast %lane : index to i32 + %x = arith.addi %r, %l : i32 + %y = arith.subi %l, %r : i32 + memref.store %x, %a[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + memref.store %y, %b[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + } + } + %a_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %b_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %out_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %a_loaded = buckyball.bank_mvin %a, %a_bank <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> + : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + %b_loaded = buckyball.bank_mvin %b, %b_bank <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> + : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + %result = matadd_ball.bank_add_i32 %a_loaded, %b_loaded -> %out_bank <{rows = @BB_BANK_LINES@ : i64}> + %stored = buckyball.bank_mvout %out, %result <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> + : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + buckyball.bank_release %a_loaded + buckyball.bank_release %b_loaded + buckyball.bank_release %stored + func.call @check_result(%out) : (memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32>) -> () + memref.dealloc %a : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + memref.dealloc %b : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + memref.dealloc %out : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + return %zero_i8 : i8 +} diff --git a/examples/prefab/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise_main.cpp b/examples/prefab/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise_main.cpp new file mode 100644 index 000000000..d76c1b3e1 --- /dev/null +++ b/examples/prefab/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise_main.cpp @@ -0,0 +1,19 @@ +#include +#include +#include + +extern "C" void check_result(int32_t *, int32_t *data, int64_t offset, + int64_t rows, int64_t lanes, int64_t stride, + int64_t lane_stride) { + if (rows != THREAD_MEM_BANK_ENTRIES || lanes != THREAD_MEM_BANK_WIDTH / 32 || + stride != lanes || lane_stride != 1) + exit(1); + for (int row = 0; row < rows; ++row) + for (int lane = 0; lane < lanes; ++lane) + if (data[offset + row * stride + lane] != 2 * lane) { + printf("matadd bank mismatch row=%d lane=%d\n", row, lane); + exit(1); + } + printf("matadd bank PASS rows=%d lanes=%d\n", THREAD_MEM_BANK_ENTRIES, + THREAD_MEM_BANK_WIDTH / 32); +} diff --git a/examples/prefab/balls/maxpool/arch/MaxPoolBall.scala b/examples/prefab/balls/maxpool/arch/MaxPoolBall.scala new file mode 100644 index 000000000..1f3f7d26e --- /dev/null +++ b/examples/prefab/balls/maxpool/arch/MaxPoolBall.scala @@ -0,0 +1,269 @@ +package examples.balls.maxpool + +import framework.acc.balldomain.bbus.{BallFactory, BallPorts} +import buckyball.balls.maxpool.{Params => ParametersProto} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import chisel3.util._ +import framework.acc.balldomain.ball.blink.{BallStatus, BlinkIO, HasBlink} + +@instantiable +class MaxPoolBall( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Module + with HasBlink { + val ball = bp.balls.find(_.id == id).get + + ball.parameters.unpack(classOf[ParametersProto]) + + require(mp.bankWidth == 128, "MaxPoolBall requires 128-bit bank rows") + require(ball.inBW == 1, "MaxPoolBall requires inBW=1") + require(ball.outBW == 1, "MaxPoolBall requires outBW=1") + + @public + val blink = IO(new BlinkIO(fp, mp, bp, id)) + + val idle :: scan :: readResp :: writeReq :: writeResp :: complete :: Nil = Enum(6) + val state = RegInit(idle) + val robId = RegInit(0.U(log2Up(fp.rob_entries).W)) + val inputBank = RegInit(0.U(fp.bank_id_len.W)) + val outputBank = RegInit(0.U(fp.bank_id_len.W)) + val bases = Reg(Vec(3, UInt(mp.groupIdWidth.W))) + val iter = RegInit(0.U(fp.iter_len.W)) + // Every group of a virtual bank is one 16-channel block with the same rows. + val groups = RegInit(0.U(log2Up(mp.bankNum + 1).W)) + val group = RegInit(0.U(log2Up(mp.bankNum + 1).W)) + val inputSide = RegInit(0.U(8.W)) + val outputSide = RegInit(0.U(8.W)) + val kernel = RegInit(0.U(8.W)) + val stride = RegInit(0.U(8.W)) + val padding = RegInit(0.U(8.W)) + val inputBase = RegInit(0.U(6.W)) + val outputBase = RegInit(0.U(6.W)) + val outputStride = RegInit(0.U(6.W)) + val startRow = RegInit(0.U(4.W)) + val startCol = RegInit(0.U(4.W)) + val outputRow = RegInit(0.U(log2Ceil(mp.bankEntries).W)) + val outputY = RegInit(0.U(8.W)) + val outputX = RegInit(0.U(8.W)) + val kernelY = RegInit(0.U(8.W)) + val kernelX = RegInit(0.U(8.W)) + val maximum = RegInit(VecInit(Seq.fill(16)("h80".U(8.W)))) + val writeData = Reg(UInt(128.W)) + + val sourceY = outputY * stride +& kernelY +& startRow + val sourceX = outputX * stride +& kernelX +& startCol + val sourceValid = sourceY >= padding && sourceX >= padding && + sourceY < padding +& inputSide && sourceX < padding +& inputSide + val inputY = sourceY - padding + val inputX = sourceX - padding + val inputAddress = inputBase +& inputY * inputSide +& inputX + val outputAddress = outputBase +& outputY * outputStride +& outputX + val lastKernel = kernelY +& 1.U === kernel && kernelX +& 1.U === kernel + + blink.cmdReq.ready := state === idle + blink.cmdResp.valid := state === complete + blink.cmdResp.bits.rob_id := robId + blink.status.idle := state === idle + blink.status.running := state =/= idle && state =/= complete + + blink.bankRead(0).rob_id := robId + blink.bankRead(0).bank_id := inputBank + blink.bankRead(0).group_id := bases(0) + group + blink.bankRead(0).io.req.valid := false.B + blink.bankRead(0).io.req.bits.addr := inputAddress + blink.bankRead(0).io.resp.ready := false.B + + blink.bankWrite(0).rob_id := robId + blink.bankWrite(0).bank_id := outputBank + blink.bankWrite(0).group_id := bases(2) + group + blink.bankWrite(0).io.req.valid := false.B + blink.bankWrite(0).io.req.bits.addr := outputAddress + blink.bankWrite(0).io.req.bits.data := writeData + blink.bankWrite(0).io.req.bits.mask := VecInit( + Seq.fill(mp.bankMaskLen)(true.B) + ) + blink.bankWrite(0).io.resp.ready := false.B + + switch(state) { + is(idle) { + when(blink.cmdReq.fire) { + val cmd = blink.cmdReq.bits + val inSide = cmd.rs2(3, 0) + val outSide = cmd.rs2(7, 4) + val kernelSize = cmd.rs2(11, 8) + val poolStride = cmd.rs2(15, 12) + val poolPadding = cmd.rs2(19, 16) + val inBase = cmd.rs2(25, 20) + val outBase = cmd.rs2(31, 26) + val outStride = cmd.rs2(37, 32) + val poolStartRow = cmd.rs2(41, 38) + val poolStartCol = cmd.rs2(45, 42) + assert(cmd.rs2(63, 46) === 0.U, "MAXPOOL reserved rs2 bits must be zero") + + assert( + cmd.op1_col =/= 0.U && cmd.op2_col === 0.U && cmd.wr_col === cmd.op1_col && + cmd.op1_col <= mp.bankNum.U, + "MAXPOOL input and output must have the same channel groups" + ) + assert(cmd.op1_bank =/= cmd.wr_bank, "MAXPOOL input and output banks must differ") + assert( + inSide > 0.U && outSide > 0.U && kernelSize > 0.U && poolStride > 0.U, + "MAXPOOL dimensions must be positive" + ) + assert(outStride >= outSide, "MAXPOOL output stride is too small") + val lastSourceRow = poolStartRow +& (outSide - 1.U) * poolStride +& (kernelSize - 1.U) + val lastSourceCol = poolStartCol +& (outSide - 1.U) * poolStride +& (kernelSize - 1.U) + val inputEnd = poolPadding +& inSide + val readsInput = poolStartRow < inputEnd && poolStartCol < inputEnd && + lastSourceRow >= poolPadding && lastSourceCol >= poolPadding + val maxInputRow = Mux(lastSourceRow < inputEnd, lastSourceRow - poolPadding, inSide - 1.U) + val maxInputCol = Mux(lastSourceCol < inputEnd, lastSourceCol - poolPadding, inSide - 1.U) + assert( + !readsInput || inBase +& maxInputRow * inSide +& maxInputCol < mp.bankEntries.U, + "MAXPOOL input window exceeds bank depth" + ) + assert( + outSide * outSide === cmd.iter && + outBase +& (outSide - 1.U) * outStride +& outSide <= mp.bankEntries.U, + "MAXPOOL output core exceeds bank depth" + ) + assert( + poolStartRow +& (outSide - 1.U) * poolStride +& kernelSize <= + inSide +& (poolPadding << 1) && + poolStartCol +& (outSide - 1.U) * poolStride +& kernelSize <= + inSide +& (poolPadding << 1), + "MAXPOOL pooling geometry is inconsistent" + ) + + robId := blink.cmdReq.bits.rob_id + inputBank := cmd.op1_bank + outputBank := cmd.wr_bank + bases := VecInit(cmd.op1_group, cmd.op2_group, cmd.wr_group) + iter := cmd.iter + groups := cmd.op1_col + group := 0.U + inputSide := inSide + outputSide := outSide + kernel := kernelSize + stride := poolStride + padding := poolPadding + inputBase := inBase + outputBase := outBase + outputStride := outStride + startRow := poolStartRow + startCol := poolStartCol + outputRow := 0.U + outputY := 0.U + outputX := 0.U + kernelY := 0.U + kernelX := 0.U + maximum := VecInit(Seq.fill(16)("h80".U(8.W))) + state := scan + } + } + + is(scan) { + when(sourceValid) { + blink.bankRead(0).io.req.valid := true.B + when(blink.bankRead(0).io.req.fire) { + state := readResp + } + }.elsewhen(lastKernel) { + writeData := Cat(maximum.reverse) + state := writeReq + }.otherwise { + when(kernelX +& 1.U === kernel) { + kernelX := 0.U + kernelY := kernelY + 1.U + }.otherwise { + kernelX := kernelX + 1.U + } + } + } + + is(readResp) { + blink.bankRead(0).io.resp.ready := true.B + when(blink.bankRead(0).io.resp.fire) { + val data = blink.bankRead(0).io.resp.bits.data.asTypeOf(Vec(16, UInt(8.W))) + val nextMaximum = Wire(Vec(16, UInt(8.W))) + for (lane <- 0 until 16) + nextMaximum(lane) := Mux(data(lane).asSInt > maximum(lane).asSInt, data(lane), maximum(lane)) + maximum := nextMaximum + when(lastKernel) { + writeData := Cat(nextMaximum.reverse) + state := writeReq + }.otherwise { + when(kernelX +& 1.U === kernel) { + kernelX := 0.U + kernelY := kernelY + 1.U + }.otherwise { + kernelX := kernelX + 1.U + } + state := scan + } + } + } + + is(writeReq) { + blink.bankWrite(0).io.req.valid := true.B + when(blink.bankWrite(0).io.req.fire) { + state := writeResp + } + } + + is(writeResp) { + blink.bankWrite(0).io.resp.ready := true.B + when(blink.bankWrite(0).io.resp.fire) { + when(outputRow +& 1.U === iter && group +& 1.U === groups) { + state := complete + }.otherwise { + when(outputRow +& 1.U === iter) { + group := group + 1.U + outputRow := 0.U + outputY := 0.U + outputX := 0.U + }.otherwise { + outputRow := outputRow + 1.U + when(outputX +& 1.U === outputSide) { + outputX := 0.U + outputY := outputY + 1.U + }.otherwise { + outputX := outputX + 1.U + } + } + kernelY := 0.U + kernelX := 0.U + maximum := VecInit(Seq.fill(16)("h80".U(8.W))) + state := scan + } + } + } + + is(complete) { + when(blink.cmdResp.fire) { + state := idle + } + } + } +} + +object MaxPoolBall extends BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts = + BallPorts(Instantiate(new MaxPoolBall(fp, mp, bp, id)).blink, None) + +} diff --git a/examples/prefab/balls/maxpool/configs/default.toml b/examples/prefab/balls/maxpool/configs/default.toml new file mode 100644 index 000000000..23c90c65a --- /dev/null +++ b/examples/prefab/balls/maxpool/configs/default.toml @@ -0,0 +1 @@ +# MaxPoolBall has no private configuration parameters. diff --git a/examples/prefab/balls/maxpool/configs/maxpool_params.proto b/examples/prefab/balls/maxpool/configs/maxpool_params.proto new file mode 100644 index 000000000..659427496 --- /dev/null +++ b/examples/prefab/balls/maxpool/configs/maxpool_params.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; +package buckyball.balls.maxpool; +option java_package = "buckyball.balls.maxpool"; +option java_multiple_files = true; + +message Params { +} diff --git a/examples/prefab/balls/maxpool/emu/src/dispatch.rs b/examples/prefab/balls/maxpool/emu/src/dispatch.rs new file mode 100644 index 000000000..1e6522985 --- /dev/null +++ b/examples/prefab/balls/maxpool/emu/src/dispatch.rs @@ -0,0 +1,36 @@ +use bebop_bemu::ball::{BallModel, ExecContext}; + +use super::maxpool; + +const BALL_CLASS: &str = "examples.balls.maxpool.MaxPoolBall"; + +pub fn execute_known( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, + ctx: &mut ExecContext, +) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("MAXPOOL") => Some(maxpool::MaxPool::exec(xs1, xs2, ctx)), + Some(_) | None => None, + } +} + +pub fn cycles_after_issue( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, +) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("MAXPOOL") => Some(maxpool::MaxPool::latency(xs1, xs2)), + Some(_) | None => None, + } +} diff --git a/examples/prefab/balls/maxpool/emu/src/lib.rs b/examples/prefab/balls/maxpool/emu/src/lib.rs new file mode 100644 index 000000000..6b4391d90 --- /dev/null +++ b/examples/prefab/balls/maxpool/emu/src/lib.rs @@ -0,0 +1,11 @@ +pub(crate) use bebop_bemu::{thread::instructions::decode, ball as instruction}; + +mod maxpool; + +mod dispatch; + +pub use dispatch::{cycles_after_issue, execute_known}; + +#[bebop_bemu::ball::linkme::distributed_slice(bebop_bemu::BALLS)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static BALL: bebop_bemu::Ball = bebop_bemu::Ball { execute: execute_known, cycles: cycles_after_issue }; diff --git a/examples/prefab/balls/maxpool/emu/src/maxpool.rs b/examples/prefab/balls/maxpool/emu/src/maxpool.rs new file mode 100644 index 000000000..cde90097c --- /dev/null +++ b/examples/prefab/balls/maxpool/emu/src/maxpool.rs @@ -0,0 +1,179 @@ +use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old, rs1_update}; +use super::instruction::{BallModel, ExecContext}; +use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; + +pub struct MaxPool; + +fn legal_geometry( + input_side: u64, + output_side: u64, + kernel: u64, + stride: u64, + padding: u64, + input_base: u64, + output_base: u64, + output_stride: u64, + start_row: u64, + start_col: u64, +) -> bool { + if input_side == 0 + || output_side == 0 + || kernel == 0 + || stride == 0 + || output_stride < output_side + { + return false; + } + + let padded_side = input_side + 2 * padding; + let last_row = start_row + (output_side - 1) * stride + kernel - 1; + let last_col = start_col + (output_side - 1) * stride + kernel - 1; + if kernel + start_row > padded_side + || kernel + start_col > padded_side + || last_row >= padded_side + || last_col >= padded_side + || output_base + (output_side - 1) * output_stride + output_side > bank_lines() as u64 + { + return false; + } + + let input_end = padding + input_side; + let reads_input = start_row < input_end + && start_col < input_end + && last_row >= padding + && last_col >= padding; + if !reads_input { + return true; + } + + let max_row = last_row.min(input_end - 1) - padding; + let max_col = last_col.min(input_end - 1) - padding; + input_base + max_row * input_side + max_col < bank_lines() as u64 +} + +impl BallModel for MaxPool { + fn exec(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + let input_bank = rs1_b0(xs1); + let output_bank = rs1_b2(xs1); + let iter = rs1_iter(xs1) as usize; + let input_side = (xs2 & 0xf) as usize; + let output_side = ((xs2 >> 4) & 0xf) as usize; + let kernel = ((xs2 >> 8) & 0xf) as usize; + let stride = ((xs2 >> 12) & 0xf) as usize; + let padding = ((xs2 >> 16) & 0xf) as usize; + let input_base = ((xs2 >> 20) & 0x3f) as usize; + let output_base = ((xs2 >> 26) & 0x3f) as usize; + let output_stride = ((xs2 >> 32) & 0x3f) as usize; + let start_row = ((xs2 >> 38) & 0xf) as usize; + let start_col = ((xs2 >> 42) & 0xf) as usize; + // The RTL writes only the output window at output_base/output_stride; + // the rest of the destination survives only an in-place update (rs1[63]). + assert!( + !rs1_read_old(xs1) && rs1_update(xs1), + "maxpool: updates its output window in place; rs1[63] must be set and rs1[29] clear" + ); + + if rs1_b1(xs1) != 0 { + panic!("maxpool: input bank 1 must be zero"); + } + if input_bank == output_bank { + panic!("maxpool: input and output banks must differ"); + } + if !ctx.config(input_bank).allocated || !ctx.config(output_bank).allocated { + panic!("maxpool: bank not allocated"); + } + let groups = ctx.config(input_bank).cols; + if groups == 0 || ctx.config(output_bank).cols != groups { + panic!("maxpool: input and output must have the same channel groups"); + } + if xs2 >> 46 != 0 + || output_side * output_side != iter + || !legal_geometry( + input_side as u64, + output_side as u64, + kernel as u64, + stride as u64, + padding as u64, + input_base as u64, + output_base as u64, + output_stride as u64, + start_row as u64, + start_col as u64, + ) + { + panic!("maxpool: illegal square pooling geometry"); + } + + // Group g of both operands is channel block g; each block uses the same rows. + for group in 0..groups { + let pi = pbank_group(ctx, input_bank, group); + let po = pbank_group(ctx, output_bank, group); + let (input, output) = ctx.banks.read_write(pi, po); + for output_y in 0..output_side { + for output_x in 0..output_side { + let output_offset = + (output_base + output_y * output_stride + output_x) * bank_row_bytes(); + let mut maximum = [-128i8; 16]; + for kernel_y in 0..kernel { + for kernel_x in 0..kernel { + let input_y = (output_y * stride + kernel_y + start_row) as isize + - padding as isize; + let input_x = (output_x * stride + kernel_x + start_col) as isize + - padding as isize; + if input_y < 0 + || input_x < 0 + || input_y >= input_side as isize + || input_x >= input_side as isize + { + continue; + } + let input_offset = + (input_base + input_y as usize * input_side + input_x as usize) + * bank_row_bytes(); + for lane in 0..16 { + maximum[lane] = maximum[lane].max(input[input_offset + lane] as i8); + } + } + } + for lane in 0..16 { + output[output_offset + lane] = maximum[lane] as u8; + } + } + } + } + 0 + } + + fn latency(xs1: u64, xs2: u64) -> u64 { + let iter = rs1_iter(xs1); + let input_side = xs2 & 0xf; + let output_side = (xs2 >> 4) & 0xf; + let kernel = (xs2 >> 8) & 0xf; + let stride = (xs2 >> 12) & 0xf; + let padding = (xs2 >> 16) & 0xf; + let input_base = (xs2 >> 20) & 0x3f; + let output_base = (xs2 >> 26) & 0x3f; + let output_stride = (xs2 >> 32) & 0x3f; + let start_row = (xs2 >> 38) & 0xf; + let start_col = (xs2 >> 42) & 0xf; + if rs1_b1(xs1) != 0 + || xs2 >> 46 != 0 + || output_side * output_side != iter + || !legal_geometry( + input_side, + output_side, + kernel, + stride, + padding, + input_base, + output_base, + output_stride, + start_row, + start_col, + ) + { + panic!("maxpool: illegal encoding"); + } + iter * (kernel * kernel * 2 + 2) + } +} diff --git a/examples/prefab/balls/maxpool/stack/src/maxpoolOps.cpp b/examples/prefab/balls/maxpool/stack/src/maxpoolOps.cpp new file mode 100644 index 000000000..eae4c0044 --- /dev/null +++ b/examples/prefab/balls/maxpool/stack/src/maxpoolOps.cpp @@ -0,0 +1,80 @@ +//===- maxpoolOps.cpp - MaxPoolBall operations ----------------------------===// +// +// The dialect of MaxPoolBall, its virtual operations building their physical +// ones and the instructions of the physical operations. +// +//===----------------------------------------------------------------------===// + +#include "maxpoolOps.h" + +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::maxpool_ball; +using mlir::buddy::bankFields; +using mlir::buddy::constantI64; +using mlir::buddy::emitInstruction; +using mlir::buddy::field; + +#include "maxpoolDialect.cpp.inc" + +#define GET_OP_CLASSES +#include "maxpoolOps.cpp.inc" + +void MaxPoolBallDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "maxpoolOps.cpp.inc" + >(); +} + +namespace { +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallDialect([](DialectRegistry ®istry) { + registry.insert(); + }), + true); +} // namespace + +SmallVector BankMaxpoolI8Op::getSourceBanks() { return {getInput()}; } +Value BankMaxpoolI8Op::getDestinationBank() { return getOutput(); } +bool BankMaxpoolI8Op::readsDestination() { return false; } +Operation *BankMaxpoolI8Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return MaxpoolI8Op::create(builder, getLoc(), sources[0], destination, + getInputSide(), getKernel(), getStride(), + getPadding()); +} + +SmallVector MaxpoolI8Op::getSourceBanks() { + return {int64_t(getInput())}; +} +int64_t MaxpoolI8Op::getDestinationBank() { return getOutput(); } +LogicalResult MaxpoolI8Op::lowerToInstructions(OpBuilder &builder) { + Location loc = getLoc(); + int64_t input = getInput(), output = getOutput(); + if (input == output) + return emitOpError("maxpool_i8 requires distinct input and output banks"); + int64_t inputSide = getInputSide(), kernel = getKernel(), + stride = getStride(), padding = getPadding(); + int64_t span = inputSide + 2 * padding - kernel; + if (inputSide <= 0 || inputSide > 15 || kernel <= 0 || kernel > 15 || + stride <= 0 || stride > 15 || padding < 0 || padding > 15 || span < 0 || + span % stride || span / stride + 1 > 15 || + inputSide * inputSide > buckyball_target::getBuckyballTarget().bankDepth) + return emitOpError("maxpool_i8 geometry exceeds the ISA field ranges"); + int64_t outputSide = span / stride + 1; + // Both maps start at the base of their groups; output rows are packed. + uint64_t rs2 = field(inputSide, 0, 3) | field(outputSide, 4, 7) | + field(kernel, 8, 11) | field(stride, 12, 15) | + field(padding, 16, 19) | field(outputSide, 32, 37); + emitInstruction( + builder, loc, buckyball_target::getBuckyballFunct7("MAXPOOL"), + constantI64(builder, loc, + bankFields({input}, output, false, outputSide * outputSide) | + (1ULL << 63)), + constantI64(builder, loc, rs2)); + return success(); +} diff --git a/examples/prefab/balls/maxpool/stack/src/maxpoolOps.h b/examples/prefab/balls/maxpool/stack/src/maxpoolOps.h new file mode 100644 index 000000000..4152c7f03 --- /dev/null +++ b/examples/prefab/balls/maxpool/stack/src/maxpoolOps.h @@ -0,0 +1,20 @@ +//===- maxpoolOps.h - MaxPoolBall operations ------------------------------===// + +#ifndef MAXPOOL_OPS_H +#define MAXPOOL_OPS_H + +#include "Interfaces/BankInterfaces.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "maxpoolDialect.h.inc" + +#define GET_OP_CLASSES +#include "maxpoolOps.h.inc" + +#endif // MAXPOOL_OPS_H diff --git a/examples/prefab/balls/maxpool/stack/src/maxpoolOps.td b/examples/prefab/balls/maxpool/stack/src/maxpoolOps.td new file mode 100644 index 000000000..7c5cf50c5 --- /dev/null +++ b/examples/prefab/balls/maxpool/stack/src/maxpoolOps.td @@ -0,0 +1,45 @@ +//===- maxpoolOps.td - MaxPoolBall operations --------------*- tablegen -*-===// +// +// maxpool_i8: the input bank holds a square feature map of `input_side` +// pixels per side; group g is channel block g, line y * input_side + x holds +// the sixteen i8 channels of pixel (y, x). The output, with the same groups, +// receives the `kernel` x `kernel` maxima with `stride` and `padding` in the +// same layout, output_side = (input_side + 2 * padding - kernel) / stride + 1. +// +//===----------------------------------------------------------------------===// + +#ifndef MAXPOOL_OPS +#define MAXPOOL_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Interfaces/BankInterfaces.td" + +def MaxPoolBall_Dialect : Dialect { + let name = "maxpool_ball"; + let cppNamespace = "::buddy::maxpool_ball"; + let summary = "The operations of MaxPoolBall"; +} + +class MaxPoolBall_Op traits = []> + : Op])>; + +def BankMaxpoolI8Op : MaxPoolBall_Op<"bank_maxpool_i8", + [DeclareOpInterfaceMethods]> { + let summary = "maxpool_i8 on virtual banks"; + let arguments = (ins I64:$input, I64:$output, I64Attr:$input_side, + I64Attr:$kernel, I64Attr:$stride, I64Attr:$padding); + let results = (outs I64:$outputOut); + let assemblyFormat = "$input `->` $output prop-dict attr-dict"; +} + +def MaxpoolI8Op : MaxPoolBall_Op<"maxpool_i8", + [DeclareOpInterfaceMethods]> { + let summary = "maxpool_i8 on physical banks"; + let arguments = (ins I64Attr:$input, I64Attr:$output, I64Attr:$input_side, + I64Attr:$kernel, I64Attr:$stride, I64Attr:$padding); + let assemblyFormat = "prop-dict attr-dict"; +} + +#endif // MAXPOOL_OPS diff --git a/examples/balls/maxpool/verify/casegen/Cargo.lock b/examples/prefab/balls/maxpool/verify/casegen/Cargo.lock similarity index 100% rename from examples/balls/maxpool/verify/casegen/Cargo.lock rename to examples/prefab/balls/maxpool/verify/casegen/Cargo.lock diff --git a/examples/balls/maxpool/verify/casegen/Cargo.toml b/examples/prefab/balls/maxpool/verify/casegen/Cargo.toml similarity index 100% rename from examples/balls/maxpool/verify/casegen/Cargo.toml rename to examples/prefab/balls/maxpool/verify/casegen/Cargo.toml diff --git a/examples/prefab/balls/maxpool/verify/casegen/src/casegen.rs b/examples/prefab/balls/maxpool/verify/casegen/src/casegen.rs new file mode 100644 index 000000000..58c13bdce --- /dev/null +++ b/examples/prefab/balls/maxpool/verify/casegen/src/casegen.rs @@ -0,0 +1,495 @@ +use crate::model; + +pub const MAX_INPUT_WORDS: usize = 128; +pub const MAX_DST: usize = 64; +pub const NUM_CASES: u32 = 11; + +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct MaxPoolCmd { + pub iter: u32, + pub op1_bank: u32, + pub wr_bank: u32, + pub op1_group: u32, + pub wr_group: u32, + pub op1_col: u32, + pub wr_col: u32, + pub rob_id: u32, + pub rs2_lo: u32, + pub rs2_hi: u32, + pub input_base: u32, + pub output_base: u32, + pub output_stride: u32, + pub input_side: u32, + pub output_side: u32, + pub kernel: u32, + pub stride: u32, + pub padding: u32, + pub start_row: u32, + pub start_col: u32, + pub num_input_words: u32, + pub num_dst_words: u32, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MaxPoolCase { + pub cmd: MaxPoolCmd, + pub input_words: [u128; MAX_INPUT_WORDS], + pub input_addr: [u32; MAX_INPUT_WORDS], + // Channel group of each word, relative to the command's base group. + pub input_group: [u32; MAX_INPUT_WORDS], + pub dst_words: [u128; MAX_DST], + pub dst_addr: [u32; MAX_DST], + pub dst_group: [u32; MAX_DST], +} + +impl MaxPoolCase { + pub fn input_lo(&self, index: usize) -> u64 { + self.input_words[index] as u64 + } + + pub fn input_hi(&self, index: usize) -> u64 { + (self.input_words[index] >> 64) as u64 + } + + pub fn dst_lo(&self, index: usize) -> u64 { + self.dst_words[index] as u64 + } + + pub fn dst_hi(&self, index: usize) -> u64 { + (self.dst_words[index] >> 64) as u64 + } +} + +#[derive(Clone, Copy)] +struct CaseSpec { + geo: model::Geo, + op1_bank: u32, + wr_bank: u32, + rob_id: u32, + ctest_input: bool, + seed: u32, + // Channel groups of the input and output banks; group g is pooled with + // the same geometry after group g - 1. + groups: u32, +} + +fn field(val: u32, lo: u32, hi: u32) -> u64 { + let width = hi - lo + 1; + let mask = if width == 64 { + u64::MAX + } else { + (1u64 << width) - 1 + }; + (u64::from(val) & mask) << lo +} + +fn encode_rs2(geo: &model::Geo) -> u64 { + field(geo.input_side as u32, 0, 3) + | field(geo.output_side as u32, 4, 7) + | field(geo.kernel as u32, 8, 11) + | field(geo.stride as u32, 12, 15) + | field(geo.padding as u32, 16, 19) + | field(geo.input_base as u32, 20, 25) + | field(geo.output_base as u32, 26, 31) + | field(geo.output_stride as u32, 32, 37) + | field(geo.start_row as u32, 38, 41) + | field(geo.start_col as u32, 42, 45) +} + +fn split_rs(value: u64) -> (u32, u32) { + (value as u32, (value >> 32) as u32) +} + +fn validate_cmd(op1_bank: u32, wr_bank: u32, iter: u32, rs2: u64) { + if iter == 0 { + panic!("maxpool: iter must be positive"); + } + if op1_bank == wr_bank { + panic!("maxpool: input and output banks must differ"); + } + if (rs2 >> 46) != 0 { + panic!("maxpool: reserved rs2 bits must be zero"); + } +} + +fn ctest_tile(geo: &model::Geo) -> Vec { + let core = geo.input_side * geo.input_side; + let mut bytes = vec![0u8; core * model::BANK_ROW_BYTES]; + for position in 0..core { + for channel in 0..model::BANK_ROW_BYTES { + bytes[position * model::BANK_ROW_BYTES + channel] = + (((position * 37 + channel * 19) & 255) as i32 - 128) as u8; + } + } + bytes +} + +fn seeded_tile(geo: &model::Geo, seed: u32) -> Vec { + let core = geo.input_side * geo.input_side; + let mut bytes = vec![0u8; core * model::BANK_ROW_BYTES]; + for position in 0..core { + for channel in 0..model::BANK_ROW_BYTES { + let mix = seed + .wrapping_mul(0x9E37_79B9) + .wrapping_add(position as u32 * 131) + .wrapping_add(channel as u32 * 17); + bytes[position * model::BANK_ROW_BYTES + channel] = ((mix & 255) as i32 - 128) as u8; + } + } + bytes +} + +fn copy_words(src: &[u128]) -> [u128; N] { + if src.len() > N { + panic!("maxpool: {} words exceed buffer {N}", src.len()); + } + let mut out = [0u128; N]; + out[..src.len()].copy_from_slice(src); + out +} + +fn copy_addrs(src: &[u32]) -> [u32; N] { + if src.len() > N { + panic!("maxpool: {} addrs exceed buffer {N}", src.len()); + } + let mut out = [0u32; N]; + out[..src.len()].copy_from_slice(src); + out +} + +fn build_case(spec: CaseSpec) -> MaxPoolCase { + model::validate(&spec.geo); + let iter = model::iter_for(&spec.geo); + + let mut referenced = vec![false; spec.geo.input_side * spec.geo.input_side]; + for output_y in 0..spec.geo.output_side { + for output_x in 0..spec.geo.output_side { + for kernel_y in 0..spec.geo.kernel { + for kernel_x in 0..spec.geo.kernel { + let y = output_y * spec.geo.stride + kernel_y + spec.geo.start_row; + let x = output_x * spec.geo.stride + kernel_x + spec.geo.start_col; + if y >= spec.geo.padding + && x >= spec.geo.padding + && y < spec.geo.padding + spec.geo.input_side + && x < spec.geo.padding + spec.geo.input_side + { + referenced[(y - spec.geo.padding) * spec.geo.input_side + + (x - spec.geo.padding)] = true; + } + } + } + } + } + + let mut input_addrs = Vec::new(); + let mut input_words = Vec::new(); + let mut input_groups = Vec::new(); + let mut dst_addrs = Vec::new(); + let mut dst_words = Vec::new(); + let mut dst_groups = Vec::new(); + for group in 0..spec.groups { + let input_bytes = if spec.ctest_input { + ctest_tile(&spec.geo) + } else { + seeded_tile(&spec.geo, spec.seed + group * 101) + }; + for y in 0..spec.geo.input_side { + for x in 0..spec.geo.input_side { + if !referenced[y * spec.geo.input_side + x] { + continue; + } + let addr = (spec.geo.input_base + y * spec.geo.input_side + x) as u32; + let row_off = (y * spec.geo.input_side + x) * model::BANK_ROW_BYTES; + let mut row = [0u8; model::BANK_ROW_BYTES]; + row.copy_from_slice(&input_bytes[row_off..row_off + model::BANK_ROW_BYTES]); + input_addrs.push(addr); + input_words.push(model::pack_row(&row)); + input_groups.push(group); + } + } + for (addr, row) in model::maxpool_rows(&input_bytes, &spec.geo) { + dst_addrs.push(addr); + dst_words.push(model::pack_row(&row)); + dst_groups.push(group); + } + } + + if dst_addrs.len() as u32 != iter * spec.groups { + panic!("maxpool: gold rows {} != iter {iter} x groups", dst_addrs.len()); + } + + let rs2 = encode_rs2(&spec.geo); + validate_cmd(spec.op1_bank, spec.wr_bank, iter, rs2); + + let (rs2_lo, rs2_hi) = split_rs(rs2); + + MaxPoolCase { + cmd: MaxPoolCmd { + iter, + op1_bank: spec.op1_bank, + wr_bank: spec.wr_bank, + // Vary the renamed group bases across cases; the Ball adds them to every access. + op1_group: spec.seed % 3, + wr_group: (spec.seed + 1) % 4, + op1_col: spec.groups, + wr_col: spec.groups, + rob_id: spec.rob_id, + rs2_lo, + rs2_hi, + input_base: spec.geo.input_base as u32, + output_base: spec.geo.output_base as u32, + output_stride: spec.geo.output_stride as u32, + input_side: spec.geo.input_side as u32, + output_side: spec.geo.output_side as u32, + kernel: spec.geo.kernel as u32, + stride: spec.geo.stride as u32, + padding: spec.geo.padding as u32, + start_row: spec.geo.start_row as u32, + start_col: spec.geo.start_col as u32, + num_input_words: input_words.len() as u32, + num_dst_words: dst_words.len() as u32, + }, + input_words: copy_words(&input_words), + input_addr: copy_addrs::(&input_addrs), + input_group: copy_addrs::(&input_groups), + dst_words: copy_words(&dst_words), + dst_addr: copy_addrs::(&dst_addrs), + dst_group: copy_addrs::(&dst_groups), + } +} + +fn spec_for(index: u32) -> CaseSpec { + match index { + 0 => CaseSpec { + geo: model::Geo { + input_side: 6, + output_side: 3, + kernel: 2, + stride: 2, + padding: 0, + input_base: 7, + output_base: 11, + output_stride: 7, + start_row: 0, + start_col: 0, + }, + op1_bank: 0, + wr_bank: 1, + rob_id: 2, + ctest_input: true, + seed: 0, + groups: 1, + }, + 1 => CaseSpec { + geo: model::Geo { + input_side: 4, + output_side: 3, + kernel: 2, + stride: 1, + padding: 0, + input_base: 0, + output_base: 20, + output_stride: 4, + start_row: 0, + start_col: 0, + }, + op1_bank: 0, + wr_bank: 1, + rob_id: 3, + ctest_input: false, + seed: 1, + groups: 1, + }, + 2 => CaseSpec { + geo: model::Geo { + input_side: 4, + output_side: 3, + kernel: 2, + stride: 1, + padding: 1, + input_base: 0, + output_base: 30, + output_stride: 4, + start_row: 0, + start_col: 0, + }, + op1_bank: 0, + wr_bank: 1, + rob_id: 4, + ctest_input: false, + seed: 2, + groups: 1, + }, + 3 => CaseSpec { + geo: model::Geo { + input_side: 6, + output_side: 3, + kernel: 2, + stride: 2, + padding: 1, + input_base: 0, + output_base: 40, + output_stride: 4, + start_row: 0, + start_col: 0, + }, + op1_bank: 0, + wr_bank: 1, + rob_id: 5, + ctest_input: false, + seed: 3, + groups: 1, + }, + 4 => CaseSpec { + geo: model::Geo { + input_side: 5, + output_side: 3, + kernel: 3, + stride: 1, + padding: 0, + input_base: 0, + output_base: 50, + output_stride: 4, + start_row: 0, + start_col: 0, + }, + op1_bank: 0, + wr_bank: 1, + rob_id: 6, + ctest_input: false, + seed: 4, + groups: 1, + }, + 5 => CaseSpec { + geo: model::Geo { + input_side: 7, + output_side: 3, + kernel: 3, + stride: 2, + padding: 0, + input_base: 0, + output_base: 0, + output_stride: 7, + start_row: 0, + start_col: 0, + }, + op1_bank: 0, + wr_bank: 1, + rob_id: 7, + ctest_input: false, + seed: 5, + groups: 1, + }, + 6 => CaseSpec { + geo: model::Geo { + input_side: 5, + output_side: 3, + kernel: 3, + stride: 1, + padding: 1, + input_base: 0, + output_base: 15, + output_stride: 4, + start_row: 0, + start_col: 0, + }, + op1_bank: 0, + wr_bank: 1, + rob_id: 8, + ctest_input: false, + seed: 6, + groups: 1, + }, + 7 => CaseSpec { + geo: model::Geo { + input_side: 7, + output_side: 3, + kernel: 3, + stride: 2, + padding: 1, + input_base: 0, + output_base: 25, + output_stride: 7, + start_row: 0, + start_col: 0, + }, + op1_bank: 0, + wr_bank: 1, + rob_id: 9, + ctest_input: false, + seed: 7, + groups: 1, + }, + 8 => CaseSpec { + geo: model::Geo { + input_side: 7, + output_side: 1, + kernel: 1, + stride: 1, + padding: 0, + input_base: 63, + output_base: 0, + output_stride: 1, + start_row: 0, + start_col: 0, + }, + op1_bank: 0, + wr_bank: 1, + rob_id: 10, + ctest_input: false, + seed: 8, + groups: 1, + }, + // Two channel groups, 3x3 kernel with padding. + 9 => CaseSpec { + geo: model::Geo { + input_side: 5, + output_side: 3, + kernel: 3, + stride: 2, + padding: 1, + input_base: 2, + output_base: 5, + output_stride: 4, + start_row: 0, + start_col: 0, + }, + op1_bank: 2, + wr_bank: 5, + rob_id: 11, + ctest_input: false, + seed: 9, + groups: 2, + }, + // Three channel groups, 2x2 stride-2 pooling. + 10 => CaseSpec { + geo: model::Geo { + input_side: 4, + output_side: 2, + kernel: 2, + stride: 2, + padding: 0, + input_base: 0, + output_base: 0, + output_stride: 2, + start_row: 0, + start_col: 0, + }, + op1_bank: 1, + wr_bank: 0, + rob_id: 12, + ctest_input: false, + seed: 10, + groups: 3, + }, + _ => panic!("maxpool: unsupported directed case {index}"), + } +} + +pub fn gen_case(index: u32) -> MaxPoolCase { + if index >= NUM_CASES { + panic!("maxpool: unsupported directed case {index}"); + } + build_case(spec_for(index)) +} diff --git a/examples/prefab/balls/maxpool/verify/casegen/src/lib.rs b/examples/prefab/balls/maxpool/verify/casegen/src/lib.rs new file mode 100644 index 000000000..2b7d1437f --- /dev/null +++ b/examples/prefab/balls/maxpool/verify/casegen/src/lib.rs @@ -0,0 +1,129 @@ +mod casegen; +mod model; + +use std::cell::RefCell; + +use casegen::{MaxPoolCase, MaxPoolCmd, MAX_DST, MAX_INPUT_WORDS}; + +thread_local! { + static CURRENT: RefCell> = const { RefCell::new(None) }; +} + +#[no_mangle] +pub extern "C" fn maxpool_case_load(index: u32) -> i32 { + let case = casegen::gen_case(index); + CURRENT.with(|current| *current.borrow_mut() = Some(case)); + 0 +} + +fn current_case R, R>(f: F) -> R { + CURRENT.with(|current| match current.borrow().as_ref() { + Some(case) => f(case), + None => panic!("maxpool_case: no case loaded; call maxpool_case_load first"), + }) +} + +#[no_mangle] +pub extern "C" fn maxpool_case_cmd(out_ptr: *mut MaxPoolCmd) { + if out_ptr.is_null() { + panic!("maxpool_case_cmd: null out_ptr"); + } + current_case(|case| unsafe { + *out_ptr = case.cmd; + }); +} + +fn idx(index: u32, limit: u32, what: &str) -> usize { + if index >= limit { + panic!("maxpool_case_{what}: index {index} out of range {limit}"); + } + index as usize +} + +#[no_mangle] +pub extern "C" fn maxpool_case_input_word_lo(word_index: u32) -> u64 { + current_case(|case| { + let i = idx(word_index, case.cmd.num_input_words, "input_word_lo"); + if i >= MAX_INPUT_WORDS { + panic!("maxpool_case_input_word_lo: word_index out of range"); + } + case.input_lo(i) + }) +} + +#[no_mangle] +pub extern "C" fn maxpool_case_input_word_hi(word_index: u32) -> u64 { + current_case(|case| { + let i = idx(word_index, case.cmd.num_input_words, "input_word_hi"); + if i >= MAX_INPUT_WORDS { + panic!("maxpool_case_input_word_hi: word_index out of range"); + } + case.input_hi(i) + }) +} + +#[no_mangle] +pub extern "C" fn maxpool_case_input_addr(word_index: u32) -> u32 { + current_case(|case| { + let i = idx(word_index, case.cmd.num_input_words, "input_addr"); + if i >= MAX_INPUT_WORDS { + panic!("maxpool_case_input_addr: word_index out of range"); + } + case.input_addr[i] + }) +} + +#[no_mangle] +pub extern "C" fn maxpool_case_dst_word_lo(word_index: u32) -> u64 { + current_case(|case| { + let i = idx(word_index, case.cmd.num_dst_words, "dst_word_lo"); + if i >= MAX_DST { + panic!("maxpool_case_dst_word_lo: word_index out of range"); + } + case.dst_lo(i) + }) +} + +#[no_mangle] +pub extern "C" fn maxpool_case_dst_word_hi(word_index: u32) -> u64 { + current_case(|case| { + let i = idx(word_index, case.cmd.num_dst_words, "dst_word_hi"); + if i >= MAX_DST { + panic!("maxpool_case_dst_word_hi: word_index out of range"); + } + case.dst_hi(i) + }) +} + +#[no_mangle] +pub extern "C" fn maxpool_case_dst_addr(word_index: u32) -> u32 { + current_case(|case| { + let i = idx(word_index, case.cmd.num_dst_words, "dst_addr"); + if i >= MAX_DST { + panic!("maxpool_case_dst_addr: word_index out of range"); + } + case.dst_addr[i] + }) +} + +#[no_mangle] +pub extern "C" fn maxpool_case_input_group(word_index: u32) -> u32 { + current_case(|case| { + let i = idx(word_index, case.cmd.num_input_words, "input_group"); + if i >= MAX_INPUT_WORDS { + panic!("maxpool_case_input_group: word_index out of range"); + } + case.input_group[i] + }) +} + +#[no_mangle] +pub extern "C" fn maxpool_case_dst_group(word_index: u32) -> u32 { + current_case(|case| { + let i = idx(word_index, case.cmd.num_dst_words, "dst_group"); + if i >= MAX_DST { + panic!("maxpool_case_dst_group: word_index out of range"); + } + case.dst_group[i] + }) +} diff --git a/examples/prefab/balls/maxpool/verify/casegen/src/model.rs b/examples/prefab/balls/maxpool/verify/casegen/src/model.rs new file mode 100644 index 000000000..3780a70c7 --- /dev/null +++ b/examples/prefab/balls/maxpool/verify/casegen/src/model.rs @@ -0,0 +1,104 @@ +pub const BANK_LINES: usize = 64; +pub const BANK_ROW_BYTES: usize = 16; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Geo { + pub input_side: usize, + pub output_side: usize, + pub kernel: usize, + pub stride: usize, + pub padding: usize, + pub input_base: usize, + pub output_base: usize, + pub output_stride: usize, + pub start_row: usize, + pub start_col: usize, +} + +pub fn validate(geo: &Geo) { + let g = geo; + let dimensions_valid = + g.input_side != 0 && g.output_side != 0 && g.kernel != 0 && g.stride != 0; + if !dimensions_valid { + panic!("maxpool: illegal square pooling geometry"); + } + let last_row = g.start_row + (g.output_side - 1) * g.stride + g.kernel - 1; + let last_col = g.start_col + (g.output_side - 1) * g.stride + g.kernel - 1; + let input_end = g.padding + g.input_side; + let reads_input = g.start_row < input_end + && g.start_col < input_end + && last_row >= g.padding + && last_col >= g.padding; + let input_fits = !reads_input + || g.input_base + + (last_row.min(input_end - 1) - g.padding) * g.input_side + + (last_col.min(input_end - 1) - g.padding) + < BANK_LINES; + if g.output_stride < g.output_side + || !input_fits + || g.output_base + (g.output_side - 1) * g.output_stride + g.output_side > BANK_LINES + || g.input_side + 2 * g.padding < g.kernel + g.start_row + || g.input_side + 2 * g.padding < g.kernel + g.start_col + || g.start_row + (g.output_side - 1) * g.stride + g.kernel > g.input_side + 2 * g.padding + || g.start_col + (g.output_side - 1) * g.stride + g.kernel > g.input_side + 2 * g.padding + { + panic!("maxpool: illegal square pooling geometry"); + } +} + +pub fn iter_for(geo: &Geo) -> u32 { + let iter = geo.output_side * geo.output_side; + if iter == 0 { + panic!("maxpool: iter must be positive"); + } + iter as u32 +} + +pub fn maxpool_rows(input: &[u8], geo: &Geo) -> Vec<(u32, [u8; BANK_ROW_BYTES])> { + validate(geo); + let core = geo.input_side * geo.input_side; + if input.len() != core * BANK_ROW_BYTES { + panic!("maxpool: input bytes {} != core {} rows", input.len(), core); + } + + let mut out = Vec::with_capacity(geo.output_side * geo.output_side); + for output_y in 0..geo.output_side { + for output_x in 0..geo.output_side { + let mut maximum = [0x80u8; BANK_ROW_BYTES]; + for kernel_y in 0..geo.kernel { + for kernel_x in 0..geo.kernel { + let input_y = output_y * geo.stride + kernel_y + geo.start_row; + let input_x = output_x * geo.stride + kernel_x + geo.start_col; + let sy = input_y as isize - geo.padding as isize; + let sx = input_x as isize - geo.padding as isize; + if sy < 0 + || sx < 0 + || sy >= geo.input_side as isize + || sx >= geo.input_side as isize + { + continue; + } + let row = (sy as usize) * geo.input_side + sx as usize; + let off = row * BANK_ROW_BYTES; + for lane in 0..BANK_ROW_BYTES { + let v = input[off + lane] as i8; + if v > maximum[lane] as i8 { + maximum[lane] = v as u8; + } + } + } + } + let addr = (geo.output_base + output_y * geo.output_stride + output_x) as u32; + out.push((addr, maximum)); + } + } + out +} + +pub fn pack_row(bytes: &[u8; BANK_ROW_BYTES]) -> u128 { + let mut word = 0u128; + for (lane, byte) in bytes.iter().enumerate() { + word |= u128::from(*byte) << (lane * 8); + } + word +} diff --git a/examples/balls/maxpool/verify/filelists/maxpool_ball.f b/examples/prefab/balls/maxpool/verify/filelists/maxpool_ball.f similarity index 89% rename from examples/balls/maxpool/verify/filelists/maxpool_ball.f rename to examples/prefab/balls/maxpool/verify/filelists/maxpool_ball.f index 1d7b8e584..1ce8e6930 100644 --- a/examples/balls/maxpool/verify/filelists/maxpool_ball.f +++ b/examples/prefab/balls/maxpool/verify/filelists/maxpool_ball.f @@ -1,3 +1,4 @@ ++incdir+@UVM@/src/core +incdir+@UVM@/src/ball +incdir+@UVM@/src/ball/agents/cmd +incdir+@UVM@/src/ball/agents/mem @@ -13,6 +14,7 @@ +incdir+src/tests +incdir+@RTL@ +@UVM@/src/core/package.sv @UVM@/src/ball/bb_blink_if.sv @UVM@/src/ball/bb_uvm_pkg.sv diff --git a/examples/prefab/balls/maxpool/verify/src/common/maxpool_defs.svh b/examples/prefab/balls/maxpool/verify/src/common/maxpool_defs.svh new file mode 100644 index 000000000..6bd4983b4 --- /dev/null +++ b/examples/prefab/balls/maxpool/verify/src/common/maxpool_defs.svh @@ -0,0 +1,45 @@ +typedef struct { + int unsigned iter; + int unsigned op1_bank; + int unsigned wr_bank; + int unsigned op1_group; + int unsigned wr_group; + int unsigned op1_col; + int unsigned wr_col; + int unsigned rob_id; + int unsigned rs2_lo; + int unsigned rs2_hi; + int unsigned input_base; + int unsigned output_base; + int unsigned output_stride; + int unsigned input_side; + int unsigned output_side; + int unsigned kernel; + int unsigned stride; + int unsigned padding; + int unsigned start_row; + int unsigned start_col; + int unsigned num_input_words; + int unsigned num_dst_words; +} maxpool_cmd_dpi_t; + +import "DPI-C" function int maxpool_case_load(input int unsigned index); +import "DPI-C" function void maxpool_case_cmd(output maxpool_cmd_dpi_t cmd); +import "DPI-C" function longint unsigned maxpool_case_input_word_lo(input int unsigned word_index); +import "DPI-C" function longint unsigned maxpool_case_input_word_hi(input int unsigned word_index); +import "DPI-C" function int unsigned maxpool_case_input_addr(input int unsigned word_index); +import "DPI-C" function longint unsigned maxpool_case_dst_word_lo(input int unsigned word_index); +import "DPI-C" function longint unsigned maxpool_case_dst_word_hi(input int unsigned word_index); +import "DPI-C" function int unsigned maxpool_case_dst_addr(input int unsigned word_index); +import "DPI-C" function int unsigned maxpool_case_input_group(input int unsigned word_index); +import "DPI-C" function int unsigned maxpool_case_dst_group(input int unsigned word_index); + +`ifndef MAXPOOL_FUNCT7 +`error "MAXPOOL_FUNCT7 must be provided by the selected Thread ballISA" +`endif + +localparam int MAXPOOL_THREAD_FUNCT7 = `MAXPOOL_FUNCT7; +localparam int MAXPOOL_MAX_INPUT_WORDS = 128; +localparam int MAXPOOL_MAX_DST_WORDS = 64; +localparam int MAXPOOL_NUM_CASES = 11; +localparam int MAXPOOL_TIMEOUT_CYCLES = 20000; diff --git a/examples/prefab/balls/maxpool/verify/src/common/maxpool_items.svh b/examples/prefab/balls/maxpool/verify/src/common/maxpool_items.svh new file mode 100644 index 000000000..35485fa12 --- /dev/null +++ b/examples/prefab/balls/maxpool/verify/src/common/maxpool_items.svh @@ -0,0 +1,128 @@ +class maxpool_cmd_item extends bb_blink_cmd_item; + `uvm_object_utils(maxpool_cmd_item) + + bit [31:0] input_base; + bit [31:0] output_base; + bit [31:0] output_stride; + bit [31:0] input_side; + bit [31:0] output_side; + bit [31:0] kernel; + bit [31:0] stride; + bit [31:0] padding; + bit [31:0] start_row; + bit [31:0] start_col; + bit [31:0] num_input_words; + bit [31:0] num_dst_words; + bit [127:0] input_words[MAXPOOL_MAX_INPUT_WORDS]; + bit [31:0] input_addr[MAXPOOL_MAX_INPUT_WORDS]; + bit [31:0] input_group[MAXPOOL_MAX_INPUT_WORDS]; + bit [127:0] dst_words[MAXPOOL_MAX_DST_WORDS]; + bit [31:0] dst_addr[MAXPOOL_MAX_DST_WORDS]; + bit [31:0] dst_group[MAXPOOL_MAX_DST_WORDS]; + + function new(string name = "maxpool_cmd_item"); + super.new(name); + endfunction + + function void load_rust_case(int unsigned index); + maxpool_cmd_dpi_t cmd; + int unsigned rc; + longint unsigned w_lo; + longint unsigned w_hi; + int i; + + rc = maxpool_case_load(index); + if (rc != 0) begin + `uvm_fatal("CASE", $sformatf("maxpool_case_load returned %0d for index %0d", rc, index)) + end + maxpool_case_cmd(cmd); + + // bb_maxpool: banks and iter come from rs1, the pooling geometry from rs2. + rob_id = cmd.rob_id[BB_BLINK_ROB_ID_W-1:0]; + funct = MAXPOOL_THREAD_FUNCT7[6:0]; + rs2 = {cmd.rs2_hi, cmd.rs2_lo}; + iter = cmd.iter; + op1_bank = cmd.op1_bank; + op2_bank = '0; + wr_bank = cmd.wr_bank; + op1_group = cmd.op1_group; + op2_group = '0; + wr_group = cmd.wr_group; + op1_col = cmd.op1_col; + op2_col = '0; + wr_col = cmd.wr_col; + wr_read_bank = '0; + wr_arch_bank = cmd.wr_bank; + input_base = cmd.input_base; + output_base = cmd.output_base; + output_stride = cmd.output_stride; + input_side = cmd.input_side; + output_side = cmd.output_side; + kernel = cmd.kernel; + stride = cmd.stride; + padding = cmd.padding; + start_row = cmd.start_row; + start_col = cmd.start_col; + num_input_words = cmd.num_input_words; + num_dst_words = cmd.num_dst_words; + + if (op1_col == 0 || wr_col != op1_col) + `uvm_fatal("CASE", "MAXPOOL input and output must have the same channel groups") + if (op1_bank == wr_bank) `uvm_fatal("CASE", "MAXPOOL input and output banks must differ") + if (rs2[63:46] != 0) `uvm_fatal("CASE", "MAXPOOL reserved rs2 bits must be zero") + if (num_dst_words != iter * op1_col) + `uvm_fatal("CASE", $sformatf("num_dst_words %0d != iter %0d x groups", num_dst_words, iter)) + if (iter != output_side * output_side) + `uvm_fatal("CASE", $sformatf("iter %0d != output_side^2 %0d", iter, output_side * output_side + )) + if (num_input_words == 0 || num_input_words > MAXPOOL_MAX_INPUT_WORDS) + `uvm_fatal("CASE", $sformatf("num_input_words out of range: %0d", num_input_words)) + if (num_dst_words == 0 || num_dst_words > MAXPOOL_MAX_DST_WORDS) + `uvm_fatal("CASE", $sformatf("num_dst_words out of range: %0d", num_dst_words)) + + for (i = 0; i < num_input_words; i++) begin + w_lo = maxpool_case_input_word_lo(i); + w_hi = maxpool_case_input_word_hi(i); + input_words[i] = {w_hi, w_lo}; + input_addr[i] = maxpool_case_input_addr(i); + input_group[i] = maxpool_case_input_group(i); + end + for (i = 0; i < num_dst_words; i++) begin + w_lo = maxpool_case_dst_word_lo(i); + w_hi = maxpool_case_dst_word_hi(i); + dst_words[i] = {w_hi, w_lo}; + dst_addr[i] = maxpool_case_dst_addr(i); + dst_group[i] = maxpool_case_dst_group(i); + end + endfunction + + function void do_copy(uvm_object rhs); + maxpool_cmd_item rhs_; + super.do_copy(rhs); + if (!$cast(rhs_, rhs)) begin + `uvm_fatal("COPY", "rhs is not maxpool_cmd_item") + end + input_base = rhs_.input_base; + output_base = rhs_.output_base; + output_stride = rhs_.output_stride; + input_side = rhs_.input_side; + output_side = rhs_.output_side; + kernel = rhs_.kernel; + stride = rhs_.stride; + padding = rhs_.padding; + start_row = rhs_.start_row; + start_col = rhs_.start_col; + num_input_words = rhs_.num_input_words; + num_dst_words = rhs_.num_dst_words; + for (int i = 0; i < MAXPOOL_MAX_INPUT_WORDS; i++) begin + input_words[i] = rhs_.input_words[i]; + input_addr[i] = rhs_.input_addr[i]; + input_group[i] = rhs_.input_group[i]; + end + for (int i = 0; i < MAXPOOL_MAX_DST_WORDS; i++) begin + dst_words[i] = rhs_.dst_words[i]; + dst_addr[i] = rhs_.dst_addr[i]; + dst_group[i] = rhs_.dst_group[i]; + end + endfunction +endclass diff --git a/examples/balls/maxpool/verify/src/cov/maxpool_cov.svh b/examples/prefab/balls/maxpool/verify/src/cov/maxpool_cov.svh similarity index 77% rename from examples/balls/maxpool/verify/src/cov/maxpool_cov.svh rename to examples/prefab/balls/maxpool/verify/src/cov/maxpool_cov.svh index 319389202..9d53c45ba 100644 --- a/examples/balls/maxpool/verify/src/cov/maxpool_cov.svh +++ b/examples/prefab/balls/maxpool/verify/src/cov/maxpool_cov.svh @@ -6,12 +6,14 @@ class maxpool_cov extends uvm_component; int unsigned cur_kernel; int unsigned cur_stride; int unsigned cur_padding; + int unsigned cur_groups; int cmd_count; covergroup cmd_cg; coverpoint cur_kernel {bins k2 = {2}; bins k3 = {3};} coverpoint cur_stride {bins s1 = {1}; bins s2 = {2};} coverpoint cur_padding {bins p0 = {0}; bins p1 = {1};} + coverpoint cur_groups {bins one = {1}; bins many = {[2 : 24]};} endgroup function new(string name, uvm_component parent); @@ -21,23 +23,25 @@ class maxpool_cov extends uvm_component; endfunction function void write_cmd(bb_blink_cmd_item item); - if (item.funct7 != MAXPOOL_CORE_FUNCT7[6:0]) - `uvm_fatal("COV", $sformatf("unexpected funct7 %0d", item.funct7)) + if (item.funct != MAXPOOL_THREAD_FUNCT7[6:0]) + `uvm_fatal("COV", $sformatf("unexpected funct %0d", item.funct)) cur_kernel = int'(item.rs2[11:8]); cur_stride = int'(item.rs2[15:12]); cur_padding = int'(item.rs2[19:16]); + cur_groups = int'(item.op1_col); cmd_count++; cmd_cg.sample(); endfunction function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; super.check_phase(phase); if (cmd_count == 0) `uvm_fatal("COV", "no cmd observed") if (cmd_cg.get_coverage() != 100.0) begin `uvm_fatal( "COV", $sformatf( - "maxpool_cov coverage incomplete: %0.2f%% (need kernel {2,3}, stride {1,2}, pad {0,1})", + "maxpool_cov coverage incomplete: %0.2f%% (need kernel {2,3}, stride {1,2}, pad {0,1}, groups {1,>1})", cmd_cg.get_coverage())) end endfunction diff --git a/examples/balls/maxpool/verify/src/env/maxpool_env.svh b/examples/prefab/balls/maxpool/verify/src/env/maxpool_env.svh similarity index 100% rename from examples/balls/maxpool/verify/src/env/maxpool_env.svh rename to examples/prefab/balls/maxpool/verify/src/env/maxpool_env.svh diff --git a/examples/balls/maxpool/verify/src/env/maxpool_scoreboard.svh b/examples/prefab/balls/maxpool/verify/src/env/maxpool_scoreboard.svh similarity index 78% rename from examples/balls/maxpool/verify/src/env/maxpool_scoreboard.svh rename to examples/prefab/balls/maxpool/verify/src/env/maxpool_scoreboard.svh index 487a50ae3..4abf6d2ac 100644 --- a/examples/balls/maxpool/verify/src/env/maxpool_scoreboard.svh +++ b/examples/prefab/balls/maxpool/verify/src/env/maxpool_scoreboard.svh @@ -46,7 +46,8 @@ class maxpool_scoreboard extends uvm_scoreboard; end mem_model.clear_mem(); for (i = 0; i < clone.num_input_words; i++) begin - mem_model.load_word(int'(clone.op1_bank), int'(clone.input_addr[i]), clone.input_words[i]); + mem_model.load_word_g(int'(clone.op1_bank), int'(clone.op1_group + clone.input_group[i]), + int'(clone.input_addr[i]), clone.input_words[i]); end mem_model.arm(); @@ -61,21 +62,12 @@ class maxpool_scoreboard extends uvm_scoreboard; endfunction function void check_cmd(bb_blink_cmd_item got, maxpool_cmd_item exp); - if (got.bid !== exp.bid) - `uvm_fatal("CMD", $sformatf("bid mismatch: got %0d exp %0d", got.bid, exp.bid)) - if (got.funct7 !== exp.funct7) - `uvm_fatal("CMD", $sformatf("funct7 mismatch: got %0d exp %0d", got.funct7, exp.funct7)) - if (got.iter !== exp.iter) - `uvm_fatal("CMD", $sformatf("iter mismatch: got %0d exp %0d", got.iter, exp.iter)) - if (got.op1_bank !== exp.op1_bank || got.wr_bank !== exp.wr_bank) - `uvm_fatal("CMD", "bank field mismatch") - if (got.op1_col !== exp.op1_col || got.wr_col !== exp.wr_col) - `uvm_fatal("CMD", "column field mismatch") - if (got.rs1 !== exp.rs1 || got.rs2 !== exp.rs2) `uvm_fatal("CMD", "rs field mismatch") - if (got.rob_id !== exp.rob_id) - `uvm_fatal("CMD", $sformatf("rob_id mismatch: got %0d exp %0d", got.rob_id, exp.rob_id)) - if (got.op2_bank !== 0 || got.op2_col !== 0) - `uvm_fatal("CMD", "MAXPOOL input bank 1 must be zero") + // Compare as plain BLink commands; the comparer rejects differing types. + bb_blink_cmd_item want = bb_blink_cmd_item::type_id::create("want"); + want.copy(exp); + verification_pkg::evidence::checks++; + if (!got.compare(want)) + `uvm_fatal("CMD", $sformatf("command mismatch: got %s exp %s", got.sprint(), want.sprint())) endfunction function bit input_addr_legal(maxpool_cmd_item stim, int unsigned addr); @@ -99,14 +91,19 @@ class maxpool_scoreboard extends uvm_scoreboard; stim = current_stim("READ"); if (item.port != 0) `uvm_fatal("READ", $sformatf("MaxPoolBall has one read port, got port %0d", item.port)) - if (item.group_id !== 5'd0) - `uvm_fatal("READ", $sformatf("group mismatch: got %0d exp 0", item.group_id)) + // Groups are pooled in order: every read belongs to the group being written. + if (item.group_id !== stim.op1_group + stim.dst_group[write_count]) + `uvm_fatal("READ", $sformatf( + "group mismatch: got %0d exp %0d", + item.group_id, + stim.op1_group + stim.dst_group[write_count] + )) if (item.rob_id !== stim.rob_id) `uvm_fatal("READ", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) if (item.bank_id !== stim.op1_bank) `uvm_fatal("READ", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.op1_bank)) if (!input_addr_legal(stim, item.addr)) - `uvm_fatal("READ", $sformatf("addr out of input tile: got %0d", item.addr)) + `uvm_fatal("READ", $sformatf("addr out of input core: got %0d", item.addr)) read_count++; endfunction @@ -120,12 +117,16 @@ class maxpool_scoreboard extends uvm_scoreboard; `uvm_fatal("WRITE", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.wr_bank)) if (item.rob_id !== stim.rob_id) `uvm_fatal("WRITE", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.group_id !== 5'd0) - `uvm_fatal("WRITE", $sformatf("group mismatch: got %0d exp 0", item.group_id)) - if (item.mask !== 16'hFFFF) - `uvm_fatal("WRITE", $sformatf("mask mismatch: got 0x%04h", item.mask)) if (write_count >= stim.num_dst_words) `uvm_fatal("WRITE", $sformatf("extra write %0d", write_count)) + if (item.group_id !== stim.wr_group + stim.dst_group[write_count]) + `uvm_fatal("WRITE", $sformatf( + "group mismatch: got %0d exp %0d", + item.group_id, + stim.wr_group + stim.dst_group[write_count] + )) + if (item.mask !== 16'hFFFF) + `uvm_fatal("WRITE", $sformatf("mask mismatch: got 0x%04h", item.mask)) if (item.addr !== stim.dst_addr[write_count][BB_BLINK_BANK_ADDR_W-1:0]) `uvm_fatal("WRITE", $sformatf( "addr mismatch: got %0d exp %0d", item.addr, stim.dst_addr[write_count])) @@ -145,9 +146,6 @@ class maxpool_scoreboard extends uvm_scoreboard; stim = current_stim("RESP"); if (item.rob_id !== stim.rob_id) `uvm_fatal("RESP", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.is_sub !== 1'b0) `uvm_fatal("RESP", "is_sub should be 0") - if (item.sub_rob_id !== 8'h00) - `uvm_fatal("RESP", $sformatf("sub_rob_id mismatch: got 0x%0h", item.sub_rob_id)) resp_count++; endfunction @@ -173,6 +171,7 @@ class maxpool_scoreboard extends uvm_scoreboard; endfunction function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; super.check_phase(phase); if (!done()) begin `uvm_fatal("SCB", $sformatf("incomplete: cmds=%0d reads=%0d writes=%0d/%0d resp=%0d", diff --git a/examples/balls/maxpool/verify/src/pkg/maxpool_pkg.sv b/examples/prefab/balls/maxpool/verify/src/pkg/maxpool_pkg.sv similarity index 100% rename from examples/balls/maxpool/verify/src/pkg/maxpool_pkg.sv rename to examples/prefab/balls/maxpool/verify/src/pkg/maxpool_pkg.sv diff --git a/examples/balls/maxpool/verify/src/seq/maxpool_sequences.svh b/examples/prefab/balls/maxpool/verify/src/seq/maxpool_sequences.svh similarity index 86% rename from examples/balls/maxpool/verify/src/seq/maxpool_sequences.svh rename to examples/prefab/balls/maxpool/verify/src/seq/maxpool_sequences.svh index dab28f67c..d88ab6ebb 100644 --- a/examples/balls/maxpool/verify/src/seq/maxpool_sequences.svh +++ b/examples/prefab/balls/maxpool/verify/src/seq/maxpool_sequences.svh @@ -2,7 +2,6 @@ class maxpool_basic_seq extends uvm_sequence #(bb_blink_cmd_item); `uvm_object_utils(maxpool_basic_seq) int unsigned case_index; - int unsigned bid; function new(string name = "maxpool_basic_seq"); super.new(name); @@ -12,7 +11,7 @@ class maxpool_basic_seq extends uvm_sequence #(bb_blink_cmd_item); maxpool_cmd_item item; item = maxpool_cmd_item::type_id::create("item"); start_item(item); - item.load_rust_case(case_index, bid); + item.load_rust_case(case_index); finish_item(item); endtask endclass diff --git a/examples/prefab/balls/maxpool/verify/src/tb_top.sv b/examples/prefab/balls/maxpool/verify/src/tb_top.sv new file mode 100644 index 000000000..60371efe0 --- /dev/null +++ b/examples/prefab/balls/maxpool/verify/src/tb_top.sv @@ -0,0 +1,80 @@ +module tb_top; + import uvm_pkg::*; + import maxpool_pkg::*; + + bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); + + MaxPoolBall dut ( + .clock(intf.clock), + .reset(intf.reset), + .blink_status_running(intf.status_running), + .blink_cmdReq_ready(intf.cmd_req_ready), + .blink_cmdReq_valid(intf.cmd_req_valid), + .blink_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .blink_cmdReq_bits_funct(intf.cmd_req_bits_funct), + .blink_cmdReq_bits_rs2(intf.cmd_req_bits_rs2), + .blink_cmdReq_bits_iter(intf.cmd_req_bits_iter), + .blink_cmdReq_bits_op1_bank(intf.cmd_req_bits_op1_bank), + .blink_cmdReq_bits_op2_bank(intf.cmd_req_bits_op2_bank), + .blink_cmdReq_bits_wr_bank(intf.cmd_req_bits_wr_bank), + .blink_cmdReq_bits_op1_group(intf.cmd_req_bits_op1_group), + .blink_cmdReq_bits_op2_group(intf.cmd_req_bits_op2_group), + .blink_cmdReq_bits_wr_group(intf.cmd_req_bits_wr_group), + .blink_cmdReq_bits_op1_col(intf.cmd_req_bits_op1_col), + .blink_cmdReq_bits_op2_col(intf.cmd_req_bits_op2_col), + .blink_cmdReq_bits_wr_col(intf.cmd_req_bits_wr_col), + .blink_cmdReq_bits_wr_read_bank(intf.cmd_req_bits_wr_read_bank), + .blink_cmdReq_bits_wr_arch_bank(intf.cmd_req_bits_wr_arch_bank), + .blink_cmdResp_ready(intf.cmd_resp_ready), + .blink_cmdResp_valid(intf.cmd_resp_valid), + .blink_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .blink_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .blink_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .blink_bankRead_0_group_id(intf.bank_read_group_id[0]), + .blink_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .blink_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .blink_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .blink_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .blink_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .blink_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .blink_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .blink_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .blink_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .blink_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .blink_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .blink_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .blink_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .blink_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .blink_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .blink_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]) + ); + + // Constant outputs removed from the port list by firtool. + assign intf.bank_write_req_mask[0][0] = dut.blink_bankWrite_0_io_req_bits_mask_0; + assign intf.bank_write_req_mask[0][1] = dut.blink_bankWrite_0_io_req_bits_mask_1; + assign intf.bank_write_req_mask[0][2] = dut.blink_bankWrite_0_io_req_bits_mask_2; + assign intf.bank_write_req_mask[0][3] = dut.blink_bankWrite_0_io_req_bits_mask_3; + assign intf.bank_write_req_mask[0][4] = dut.blink_bankWrite_0_io_req_bits_mask_4; + assign intf.bank_write_req_mask[0][5] = dut.blink_bankWrite_0_io_req_bits_mask_5; + assign intf.bank_write_req_mask[0][6] = dut.blink_bankWrite_0_io_req_bits_mask_6; + assign intf.bank_write_req_mask[0][7] = dut.blink_bankWrite_0_io_req_bits_mask_7; + assign intf.bank_write_req_mask[0][8] = dut.blink_bankWrite_0_io_req_bits_mask_8; + assign intf.bank_write_req_mask[0][9] = dut.blink_bankWrite_0_io_req_bits_mask_9; + assign intf.bank_write_req_mask[0][10] = dut.blink_bankWrite_0_io_req_bits_mask_10; + assign intf.bank_write_req_mask[0][11] = dut.blink_bankWrite_0_io_req_bits_mask_11; + assign intf.bank_write_req_mask[0][12] = dut.blink_bankWrite_0_io_req_bits_mask_12; + assign intf.bank_write_req_mask[0][13] = dut.blink_bankWrite_0_io_req_bits_mask_13; + assign intf.bank_write_req_mask[0][14] = dut.blink_bankWrite_0_io_req_bits_mask_14; + assign intf.bank_write_req_mask[0][15] = dut.blink_bankWrite_0_io_req_bits_mask_15; + + initial begin + intf.clock = 1'b0; + intf.reset = 1'b1; + forever #5 intf.clock = ~intf.clock; + end + + initial begin + uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); + run_test("maxpool_ball_test"); + end +endmodule diff --git a/examples/balls/maxpool/verify/src/tests/maxpool_test.svh b/examples/prefab/balls/maxpool/verify/src/tests/maxpool_test.svh similarity index 90% rename from examples/balls/maxpool/verify/src/tests/maxpool_test.svh rename to examples/prefab/balls/maxpool/verify/src/tests/maxpool_test.svh index f24ddff2e..85119466e 100644 --- a/examples/balls/maxpool/verify/src/tests/maxpool_test.svh +++ b/examples/prefab/balls/maxpool/verify/src/tests/maxpool_test.svh @@ -17,20 +17,18 @@ class maxpool_ball_test extends uvm_test; endfunction task run_phase(uvm_phase phase); - int unsigned bid = maxpool_require_bid(); phase.raise_objection(this); - for (int unsigned index = 0; index < MAXPOOL_NUM_CASES; index++) run_case(index, bid); + for (int unsigned index = 0; index < MAXPOOL_NUM_CASES; index++) run_case(index); phase.drop_objection(this); endtask - task run_case(int unsigned index, int unsigned bid); + task run_case(int unsigned index); maxpool_basic_seq seq; int cycles = 0; apply_reset(); env.scb.reset_counters(); seq = maxpool_basic_seq::type_id::create("seq"); seq.case_index = index; - seq.bid = bid; seq.start(env.cmd_agent.seqr); while (!env.scb.done()) begin @(posedge vif.clock); diff --git a/examples/prefab/balls/maxpool/workloads/ctests/CMakeLists.txt b/examples/prefab/balls/maxpool/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..67f5c70c6 --- /dev/null +++ b/examples/prefab/balls/maxpool/workloads/ctests/CMakeLists.txt @@ -0,0 +1,3 @@ +add_buckyball_ctest(maxpool_window_test.c) +add_buckyball_ctest(maxpool_offset_test.c) +add_buckyball_ctest(maxpool_groups_test.c) diff --git a/examples/prefab/balls/maxpool/workloads/ctests/maxpool_groups_test.c b/examples/prefab/balls/maxpool/workloads/ctests/maxpool_groups_test.c new file mode 100644 index 000000000..b83d773fd --- /dev/null +++ b/examples/prefab/balls/maxpool/workloads/ctests/maxpool_groups_test.c @@ -0,0 +1,47 @@ +#include "buckyball.h" +#include +#include +#include +#include + +/* 2x2 stride-2 MAXPOOL over an NHWC 4x4 input with 32 channels: input and + * output group g are channel block g, so one instruction pools both blocks + * and a whole-bank DMA moves NHWC rows directly. */ +enum { GROUPS = 2, CHANNELS = 16 * GROUPS, SIDE = 4, OUT = 2 }; + +static int8_t input[SIDE * SIDE * CHANNELS] __attribute__((aligned(64))); +static int8_t actual[OUT * OUT * CHANNELS] __attribute__((aligned(64))); + +int main(void) { + for (int i = 0; i < SIDE * SIDE * CHANNELS; ++i) + input[i] = (int8_t)((i * 37 + 11) % 255 - 127); + + bb_mem_alloc(0, 1, GROUPS); + bb_mem_alloc(1, 1, GROUPS); + bb_mvin((uintptr_t)input, 0, SIDE * SIDE, 1); + bb_maxpool(0, 1, SIDE, OUT, 2, 2, 0, 0, 0, OUT, 0, 0); + bb_mvout((uintptr_t)actual, 1, OUT * OUT, 1); + + for (int oy = 0; oy < OUT; ++oy) + for (int ox = 0; ox < OUT; ++ox) + for (int channel = 0; channel < CHANNELS; ++channel) { + int expected = -128; + for (int ky = 0; ky < 2; ++ky) + for (int kx = 0; kx < 2; ++kx) { + int value = input[((oy * 2 + ky) * SIDE + ox * 2 + kx) * CHANNELS + + channel]; + expected = value > expected ? value : expected; + } + int got = actual[(oy * OUT + ox) * CHANNELS + channel]; + if (got != expected) { + printf("maxpool_groups FAIL y=%d x=%d channel=%d expected=%d " + "actual=%d\n", + oy, ox, channel, expected, got); + return 1; + } + } + bb_mem_release(0); + bb_mem_release(1); + printf("maxpool_groups PASS\n"); + return 0; +} diff --git a/examples/balls/maxpool/workloads/ctests/maxpool_offset_test.c b/examples/prefab/balls/maxpool/workloads/ctests/maxpool_offset_test.c similarity index 93% rename from examples/balls/maxpool/workloads/ctests/maxpool_offset_test.c rename to examples/prefab/balls/maxpool/workloads/ctests/maxpool_offset_test.c index 9d13e6171..f4eaffcb7 100644 --- a/examples/balls/maxpool/workloads/ctests/maxpool_offset_test.c +++ b/examples/prefab/balls/maxpool/workloads/ctests/maxpool_offset_test.c @@ -5,7 +5,7 @@ #include #include -enum { LANES = BANK_WIDTH / 8, ROW = 3 }; +enum { LANES = THREAD_MEM_BANK_WIDTH / 8, ROW = 3 }; static int8_t input[(ROW + 1) * LANES] __attribute__((aligned(64))); static int8_t output[LANES] __attribute__((aligned(64))); diff --git a/examples/balls/maxpool/workloads/ctests/maxpool_window_test.c b/examples/prefab/balls/maxpool/workloads/ctests/maxpool_window_test.c similarity index 94% rename from examples/balls/maxpool/workloads/ctests/maxpool_window_test.c rename to examples/prefab/balls/maxpool/workloads/ctests/maxpool_window_test.c index 6ae329a42..5da9ee4aa 100644 --- a/examples/balls/maxpool/workloads/ctests/maxpool_window_test.c +++ b/examples/prefab/balls/maxpool/workloads/ctests/maxpool_window_test.c @@ -5,7 +5,12 @@ #include #include -enum { LANES = BANK_WIDTH / 8, INPUT_BASE = 1, OUTPUT_BASE = 2, STRIDE = 4 }; +enum { + LANES = THREAD_MEM_BANK_WIDTH / 8, + INPUT_BASE = 1, + OUTPUT_BASE = 2, + STRIDE = 4 +}; enum { INPUT_ROWS = INPUT_BASE + 36, OUTPUT_ROWS = OUTPUT_BASE + 11 }; static int8_t input[INPUT_ROWS * LANES] __attribute__((aligned(64))); static int8_t output[OUTPUT_ROWS * LANES] __attribute__((aligned(64))); diff --git a/examples/prefab/balls/maxpool/workloads/isa/maxpool.h b/examples/prefab/balls/maxpool/workloads/isa/maxpool.h new file mode 100644 index 000000000..925f3b372 --- /dev/null +++ b/examples/prefab/balls/maxpool/workloads/isa/maxpool.h @@ -0,0 +1,24 @@ +#ifndef _BB_MAXPOOL_H_ +#define _BB_MAXPOOL_H_ + +#include +#include + +// Group g of the input and output banks is channel block g (16 INT8 channels +// per row). Both banks have the same number of groups; every group is pooled +// with the same geometry, so one instruction pools all channel blocks. + +#define bb_maxpool(input_bank, output_bank, input_side, output_side, kernel, \ + stride, padding, input_base, output_base, output_stride, \ + start_row, start_col) \ + BUCKYBALL_INSTRUCTION_R_R( \ + BB_BANK0(input_bank) | BB_BANK2(output_bank) | \ + BB_ITER((output_side) * (output_side)) | BB_UPDATE_WR, \ + FIELD(input_side, 0, 3) | FIELD(output_side, 4, 7) | \ + FIELD(kernel, 8, 11) | FIELD(stride, 12, 15) | \ + FIELD(padding, 16, 19) | FIELD(input_base, 20, 25) | \ + FIELD(output_base, 26, 31) | FIELD(output_stride, 32, 37) | \ + FIELD(start_row, 38, 41) | FIELD(start_col, 42, 45), \ + FUNCT7(maxpool, MAXPOOL)) + +#endif // _BB_MAXPOOL_H_ diff --git a/examples/balls/maxpool/workloads/mlir_tests/CMakeLists.txt b/examples/prefab/balls/maxpool/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/balls/maxpool/workloads/mlir_tests/CMakeLists.txt rename to examples/prefab/balls/maxpool/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/balls/maxpool/workloads/mlir_tests/ball/CMakeLists.txt b/examples/prefab/balls/maxpool/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/balls/maxpool/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/prefab/balls/maxpool/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/prefab/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2.mlir b/examples/prefab/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2.mlir new file mode 100644 index 000000000..c10b4f317 --- /dev/null +++ b/examples/prefab/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2.mlir @@ -0,0 +1,27 @@ +func.func private @check_result(memref<9x16xi8>) -> () +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %thirty_six = arith.constant 36 : index + %sixteen = arith.constant 16 : index + %input = memref.alloc() : memref<36x16xi8> + %output = memref.alloc() : memref<9x16xi8> + scf.for %row = %zero to %thirty_six step %one { + %value = arith.index_cast %row : index to i8 + scf.for %lane = %zero to %sixteen step %one { + memref.store %value, %input[%row, %lane] : memref<36x16xi8> + } + } + buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %input <{bank = 1 : i64, depth = 36 : i64, stride = 1 : i64}> : memref<36x16xi8> + maxpool_ball.maxpool_i8 <{input = 1 : i64, output = 2 : i64, input_side = 6 : i64, kernel = 2 : i64, stride = 2 : i64, padding = 0 : i64}> + buckyball.mvout %output <{bank = 2 : i64, depth = 9 : i64, stride = 1 : i64}> : memref<9x16xi8> + buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + func.call @check_result(%output) : (memref<9x16xi8>) -> () + memref.dealloc %input : memref<36x16xi8> + memref.dealloc %output : memref<9x16xi8> + return %zero_i8 : i8 +} diff --git a/examples/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2_main.cpp b/examples/prefab/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2_main.cpp similarity index 100% rename from examples/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2_main.cpp rename to examples/prefab/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2_main.cpp diff --git a/examples/prefab/balls/maxpool/workloads/mlir_tests/bank/CMakeLists.txt b/examples/prefab/balls/maxpool/workloads/mlir_tests/bank/CMakeLists.txt new file mode 100644 index 000000000..dcdfb3a23 --- /dev/null +++ b/examples/prefab/balls/maxpool/workloads/mlir_tests/bank/CMakeLists.txt @@ -0,0 +1,11 @@ +if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) + set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) +endif() +set(BUCKYBALL_MLIR_TEST_PREFIX bank) +set(_lines ${DESIGN_PROFILES_${BUCKYBALL_MLIR_ACTIVE_TARGET}_THREAD_MEM_BANK_ENTRIES}) +math(EXPR _remainder "${_lines} % 64") +if(_lines LESS 64 OR NOT _remainder EQUAL 0) + message(FATAL_ERROR + "MaxPool K3 bank batches require bank entries divisible by 64 (got ${_lines})") +endif() +add_buckyball_mlir_test(maxpool_k3_padded_spatial TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/prefab/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial.mlir.in b/examples/prefab/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial.mlir.in new file mode 100644 index 000000000..6d110e979 --- /dev/null +++ b/examples/prefab/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial.mlir.in @@ -0,0 +1,37 @@ +func.func private @check_result(memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %rows = arith.constant @BB_BANK_LINES@ : index + %batch_rows = arith.constant 64 : index + %lanes = arith.constant @BB_BANK_I8_LANES@ : index + %mod = arith.constant 31 : index + %bias = arith.constant 15 : index + %input = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + %output = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + scf.for %row = %zero to %rows step %one { + %wrapped = arith.remui %row, %mod : index + %signed = arith.subi %wrapped, %bias : index + %value = arith.index_cast %signed : index to i8 + scf.for %lane = %zero to %lanes step %one { + memref.store %value, %input[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + memref.store %zero_i8, %output[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + } + } + scf.for %base = %zero to %rows step %batch_rows { + %in = memref.subview %input[%base, 0] [64, @BB_BANK_I8_LANES@] [1, 1] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> to memref<64x@BB_BANK_I8_LANES@xi8, strided<[@BB_BANK_I8_LANES@, 1], offset: ?>> + %out = memref.subview %output[%base, 0] [64, @BB_BANK_I8_LANES@] [1, 1] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> to memref<64x@BB_BANK_I8_LANES@xi8, strided<[@BB_BANK_I8_LANES@, 1], offset: ?>> + %in_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %out_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %loaded = buckyball.bank_mvin %in, %in_bank <{depth = 64 : i64, stride = 1 : i64}> : memref<64x@BB_BANK_I8_LANES@xi8, strided<[@BB_BANK_I8_LANES@, 1], offset: ?>> + %result = maxpool_ball.bank_maxpool_i8 %loaded -> %out_bank <{input_side = 8 : i64, kernel = 3 : i64, stride = 1 : i64, padding = 1 : i64}> + %stored = buckyball.bank_mvout %out, %result <{depth = 64 : i64, stride = 1 : i64}> : memref<64x@BB_BANK_I8_LANES@xi8, strided<[@BB_BANK_I8_LANES@, 1], offset: ?>> + buckyball.bank_release %loaded + buckyball.bank_release %stored + } + func.call @check_result(%output) : (memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () + memref.dealloc %input : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + memref.dealloc %output : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + return %zero_i8 : i8 +} diff --git a/examples/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial_main.cpp b/examples/prefab/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial_main.cpp similarity index 85% rename from examples/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial_main.cpp rename to examples/prefab/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial_main.cpp index 254b73ca1..c7db0e0ce 100644 --- a/examples/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial_main.cpp +++ b/examples/prefab/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial_main.cpp @@ -1,4 +1,3 @@ -#include #include #include #include @@ -7,10 +6,10 @@ extern "C" void check_result(int8_t *, int8_t *data, int64_t offset, int64_t rows, int64_t lanes, int64_t stride, int64_t lane_stride) { constexpr int side = 8; - if (rows != BANK_LINES || lanes != BANK_WIDTH / 8 || stride != lanes || - lane_stride != 1) + if (rows != THREAD_MEM_BANK_ENTRIES || lanes != THREAD_MEM_BANK_WIDTH / 8 || + stride != lanes || lane_stride != 1) exit(1); - for (int batch = 0; batch < BANK_LINES / 64; ++batch) + for (int batch = 0; batch < THREAD_MEM_BANK_ENTRIES / 64; ++batch) for (int y = 0; y < side; ++y) for (int x = 0; x < side; ++x) { int8_t expected = -128; diff --git a/examples/prefab/balls/mxfp2int/arch/Mxfp2Int.scala b/examples/prefab/balls/mxfp2int/arch/Mxfp2Int.scala new file mode 100644 index 000000000..64a5eb3ad --- /dev/null +++ b/examples/prefab/balls/mxfp2int/arch/Mxfp2Int.scala @@ -0,0 +1,310 @@ +package examples.balls.mxfp2int + +import framework.acc.balldomain.configs.BallRouteParam + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import framework.acc.balldomain.ball.blink.{BallCommand, BallResponse} +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} + +import framework.acc.balldomain.ball.blink.BallStatus +import framework.acc.memdomain.{BankRead, BankWrite} +import examples.balls.mxfp2int.configs.Mxfp2IntBallParam + +/** + * PipelinedMxfp2Int - MXFP to INT8 dequantizer with a per-block scale bank. + * + * Supports MXFP4, MXFP6, MXFP8 formats (32 elements per block). + * + * Input format (main bank, per block): + * MXFP4: 32 × 4-bit = 128 bits = 1 word + * MXFP6: 32 × 6-bit = 192 bits, padded to 2 words (256 bits, high 64 bits = 0) + * MXFP8: 32 × 8-bit = 256 bits = 2 words + * + * Scale (op2 bank): E8M0 of block b (8-bit biased exponent, bias=127) in + * byte b % 16 of row b / 16. + * + * FP4/6/8 decoding (v1, simplified): signed_val = raw - (1 << (bits-1)) + * FP4: raw - 8 (range -8..+7) + * FP6: raw - 32 (range -32..+31) + * FP8: raw - 128 (range -128..+127) + * + * Output (per block, 2 SRAM words = 32 bytes): + * 32 × INT8 = saturate(signed_val << (scale - 127)). + * Lower 16 INT8 to row (2 * block_idx), upper 16 to row (2 * block_idx + 1). + * + * iter encodes the number of MXFP blocks. + */ +@instantiable +class PipelinedMxfp2Int( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val routingParam: BallDomainParam, + val instance: BallRouteParam, + val ballParam: Mxfp2IntBallParam) + extends Module { + val commandDataBits = frontendParam.commandDataBits + val ballConfig = ballParam + val decoder = MxfpDecoder.fromFormat(ballConfig.mxfpFormat) + val elemBits = decoder.elemBits + + val bankWidth = memParam.bankWidth + require( + bankWidth == 128, + s"Mxfp2IntBall requires bankWidth = 128, got $bankWidth" + ) + + val wordsPerBlock = ballConfig.mxfpFormat match { + case "MXFP4" => 1 + case "MXFP6" => + 2 // padded: 192 bits data + 64 bits padding = 256 bits = 2 words + case "MXFP8" => 2 + case _ => + throw new IllegalArgumentException( + s"Unsupported mxfpFormat: ${ballConfig.mxfpFormat}" + ) + } + + val FP_PER_BLOCK = 32 + val INT8_PER_WORD = bankWidth / 8 // 16 + + val ballMapping = instance + + val inBW = ballMapping.inBW + val outBW = ballMapping.outBW + require(inBW == 2 && outBW == 1, "Mxfp2IntBall reads data and scale banks and writes one bank") + + @public + val io = IO(new Bundle { + + val cmdReq = Flipped(Decoupled(new BallCommand(frontendParam, memParam))) + + val cmdResp = Decoupled(new BallResponse(frontendParam)) + + val bankRead = Vec( + inBW, + Flipped(new BankRead( + frontendParam, + memParam + )) + ) + + val bankWrite = Vec( + outBW, + Flipped(new BankWrite( + frontendParam, + memParam + )) + ) + + val status = new BallStatus + }) + + // ROB bookkeeping + val rob_id_reg = RegInit(0.U(log2Up(frontendParam.rob_entries).W)) + + when(io.cmdReq.fire) { + rob_id_reg := io.cmdReq.bits.rob_id + } + + // FSM + val idle :: sReadReq :: sReadResp :: sScaleReq :: sScaleResp :: sWriteLoReq :: sWriteLoResp :: sWriteHiReq :: sWriteHiResp :: sComplete :: Nil = + Enum(10) + val state = RegInit(idle) + + // Per-instruction registers + val raddr_reg = RegInit(0.U(frontendParam.iter_len.W)) + val waddr_reg = RegInit(0.U(frontendParam.iter_len.W)) + val rbank_reg = RegInit(0.U(frontendParam.bank_id_len.W)) + val sbank_reg = RegInit(0.U(frontendParam.bank_id_len.W)) + val wbank_reg = RegInit(0.U(frontendParam.bank_id_len.W)) + val bases = Reg(Vec(3, UInt(memParam.groupIdWidth.W))) + val iter_reg = RegInit(0.U(frontendParam.iter_len.W)) + val block_idx_reg = RegInit(0.U(frontendParam.iter_len.W)) + val word_idx_reg = RegInit(0.U(2.W)) // 0..wordsPerBlock-1 + + val inputWordsReg = RegInit( + VecInit(Seq.fill(2)(0.U(bankWidth.W))) + ) // max 2 words for MXFP6/8 + val scaleReg = RegInit(0.U(8.W)) + + // Extract 32 FP elements from inputWordsReg based on format + val fpElements = VecInit(Seq.tabulate(FP_PER_BLOCK) { i => + val bitOffset = i * elemBits + val wordIdx = bitOffset / bankWidth + val bitInWord = bitOffset % bankWidth + if (wordIdx < wordsPerBlock && bitInWord + elemBits <= bankWidth) { + inputWordsReg(wordIdx)(bitInWord + elemBits - 1, bitInWord) + } else if (wordIdx < wordsPerBlock - 1 && bitInWord + elemBits > bankWidth) { + // Cross-word boundary (only for MXFP6 if packed, but we use padded so this won't happen) + val lowBits = bankWidth - bitInWord + val highBits = elemBits - lowBits + Cat( + inputWordsReg(wordIdx + 1)(highBits - 1, 0), + inputWordsReg(wordIdx)(bankWidth - 1, bitInWord) + ) + } else { + 0.U(elemBits.W) + } + }) + + // Combinationally compute all 32 INT8 values from the latched elements + scale + val int8Vec = VecInit( + fpElements.map(raw => decoder.dequant(raw, scaleReg).asUInt) + ) + + val outWordLo = Cat((0 until INT8_PER_WORD).map(i => int8Vec(i)).reverse) + + val outWordHi = Cat( + (INT8_PER_WORD until FP_PER_BLOCK).map(i => int8Vec(i)).reverse + ) + + // === Default IO === + for (i <- 0 until inBW) { + io.bankRead(i).io.req.valid := false.B + io.bankRead(i).io.req.bits.addr := 0.U + io.bankRead(i).io.resp.ready := false.B + io.bankRead(i).rob_id := rob_id_reg + io.bankRead(i).bank_id := (if (i == 0) rbank_reg else sbank_reg) + io.bankRead(i).group_id := bases(i) + } + for (i <- 0 until outBW) { + io.bankWrite(i).io.req.valid := false.B + io.bankWrite(i).io.req.bits.addr := 0.U + io.bankWrite(i).io.req.bits.data := 0.U + io.bankWrite(i).io.req.bits.mask := VecInit( + Seq.fill(memParam.bankMaskLen)(0.U(1.W)) + ) + io.bankWrite(i).io.resp.ready := false.B + io.bankWrite(i).rob_id := rob_id_reg + io.bankWrite(i).bank_id := wbank_reg + io.bankWrite(i).group_id := bases(2) + } + + io.cmdReq.ready := state === idle + io.cmdResp.valid := false.B + io.cmdResp.bits.rob_id := rob_id_reg + + // === FSM === + switch(state) { + is(idle) { + when(io.cmdReq.fire) { + rbank_reg := io.cmdReq.bits.op1_bank + sbank_reg := io.cmdReq.bits.op2_bank + assert(io.cmdReq.bits.iter =/= 0.U, "Mxfp2IntBall iter must be positive") + assert(io.cmdReq.bits.rs2 === 0.U, "Mxfp2IntBall rs2 is reserved") + bases := VecInit(io.cmdReq.bits.op1_group, io.cmdReq.bits.op2_group, io.cmdReq.bits.wr_group) + wbank_reg := io.cmdReq.bits.wr_bank + iter_reg := io.cmdReq.bits.iter + block_idx_reg := 0.U + word_idx_reg := 0.U + raddr_reg := 0.U + waddr_reg := 0.U + when(io.cmdReq.bits.iter === 0.U) { + state := sComplete + }.otherwise { + state := sReadReq + } + } + } + + is(sReadReq) { + io.bankRead(0).io.req.valid := true.B + io.bankRead(0) + .io + .req + .bits + .addr := raddr_reg + block_idx_reg * wordsPerBlock.U + word_idx_reg + when(io.bankRead(0).io.req.fire) { + state := sReadResp + } + } + + is(sReadResp) { + io.bankRead(0).io.resp.ready := true.B + when(io.bankRead(0).io.resp.fire) { + inputWordsReg(word_idx_reg) := io.bankRead(0).io.resp.bits.data + when(word_idx_reg === (wordsPerBlock - 1).U) { + word_idx_reg := 0.U + state := sScaleReq + }.otherwise { + word_idx_reg := word_idx_reg + 1.U + state := sReadReq + } + } + } + + is(sScaleReq) { + io.bankRead(1).io.req.valid := true.B + io.bankRead(1).io.req.bits.addr := block_idx_reg >> 4 + when(io.bankRead(1).io.req.fire) { + state := sScaleResp + } + } + + is(sScaleResp) { + io.bankRead(1).io.resp.ready := true.B + when(io.bankRead(1).io.resp.fire) { + scaleReg := io.bankRead(1).io.resp.bits.data.asTypeOf(Vec(INT8_PER_WORD, UInt(8.W)))(block_idx_reg(3, 0)) + state := sWriteLoReq + } + } + + is(sWriteLoReq) { + io.bankWrite(0).io.req.valid := true.B + io.bankWrite(0).io.req.bits.addr := waddr_reg + (block_idx_reg << 1) + io.bankWrite(0).io.req.bits.data := outWordLo + io.bankWrite(0).io.req.bits.mask := VecInit( + Seq.fill(memParam.bankMaskLen)(1.U(1.W)) + ) + when(io.bankWrite(0).io.req.fire) { + state := sWriteLoResp + } + } + + is(sWriteLoResp) { + io.bankWrite(0).io.resp.ready := true.B + when(io.bankWrite(0).io.resp.fire) { + state := sWriteHiReq + } + } + + is(sWriteHiReq) { + io.bankWrite(0).io.req.valid := true.B + io.bankWrite(0).io.req.bits.addr := waddr_reg + (block_idx_reg << 1) + 1.U + io.bankWrite(0).io.req.bits.data := outWordHi + io.bankWrite(0).io.req.bits.mask := VecInit( + Seq.fill(memParam.bankMaskLen)(1.U(1.W)) + ) + when(io.bankWrite(0).io.req.fire) { + state := sWriteHiResp + } + } + + is(sWriteHiResp) { + io.bankWrite(0).io.resp.ready := true.B + when(io.bankWrite(0).io.resp.fire) { + when(block_idx_reg === iter_reg - 1.U) { + state := sComplete + }.otherwise { + block_idx_reg := block_idx_reg + 1.U + state := sReadReq + } + } + } + + is(sComplete) { + io.cmdResp.valid := true.B + when(io.cmdResp.fire) { + state := idle + } + } + } + + io.status.idle := state === idle + io.status.running := state =/= idle +} diff --git a/examples/prefab/balls/mxfp2int/arch/Mxfp2IntBall.scala b/examples/prefab/balls/mxfp2int/arch/Mxfp2IntBall.scala new file mode 100644 index 000000000..30c4b0638 --- /dev/null +++ b/examples/prefab/balls/mxfp2int/arch/Mxfp2IntBall.scala @@ -0,0 +1,73 @@ +package examples.balls.mxfp2int + +import framework.acc.balldomain.bbus.{BallFactory, BallPorts} +import buckyball.balls.mxfp2int.{Params => ParametersProto} +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import examples.balls.mxfp2int.configs.Mxfp2IntBallParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} + +import framework.acc.balldomain.ball.blink.{BlinkIO, HasBlink} + +/** + * Mxfp2IntBall - MXFP4 to INT8 dequantizer. + * + * Wraps PipelinedMxfp2Int and connects to the Blink protocol; the per-block + * E8M0 scales come from the second source bank. + */ +@instantiable +class Mxfp2IntBall( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Module + with HasBlink { + val ball = bp.balls.find(_.id == id).get + + val ballParam = Mxfp2IntBallParam.fromProto(ball.parameters.unpack(classOf[ParametersProto])) + + val ballCommonConfig = ball + + val inBW = ballCommonConfig.inBW + val outBW = ballCommonConfig.outBW + + require(ballCommonConfig.mmioReadBW == 0 && ballCommonConfig.mmioWriteBW == 0) + + @public + val blink = IO(new BlinkIO(fp, mp, bp, id)) + + val mxfp2intUnit: Instance[PipelinedMxfp2Int] = Instantiate( + new PipelinedMxfp2Int(fp, mp, bp, ball, ballParam) + ) + + mxfp2intUnit.io.cmdReq <> blink.cmdReq + mxfp2intUnit.io.cmdResp <> blink.cmdResp + + for (i <- 0 until inBW) { + mxfp2intUnit.io.bankRead(i) <> blink.bankRead(i) + } + + for (i <- 0 until outBW) { + mxfp2intUnit.io.bankWrite(i) <> blink.bankWrite(i) + } + + blink.status <> mxfp2intUnit.io.status +} + +object Mxfp2IntBall extends BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts = + BallPorts(Instantiate(new Mxfp2IntBall(fp, mp, bp, id)).blink, None) + +} diff --git a/examples/balls/mxfp2int/arch/src/main/scala/Mxfp4Decoder.scala b/examples/prefab/balls/mxfp2int/arch/Mxfp4Decoder.scala similarity index 100% rename from examples/balls/mxfp2int/arch/src/main/scala/Mxfp4Decoder.scala rename to examples/prefab/balls/mxfp2int/arch/Mxfp4Decoder.scala diff --git a/examples/balls/mxfp2int/arch/src/main/scala/Mxfp6Decoder.scala b/examples/prefab/balls/mxfp2int/arch/Mxfp6Decoder.scala similarity index 100% rename from examples/balls/mxfp2int/arch/src/main/scala/Mxfp6Decoder.scala rename to examples/prefab/balls/mxfp2int/arch/Mxfp6Decoder.scala diff --git a/examples/balls/mxfp2int/arch/src/main/scala/Mxfp8Decoder.scala b/examples/prefab/balls/mxfp2int/arch/Mxfp8Decoder.scala similarity index 100% rename from examples/balls/mxfp2int/arch/src/main/scala/Mxfp8Decoder.scala rename to examples/prefab/balls/mxfp2int/arch/Mxfp8Decoder.scala diff --git a/examples/balls/mxfp2int/arch/src/main/scala/MxfpDecoder.scala b/examples/prefab/balls/mxfp2int/arch/MxfpDecoder.scala similarity index 85% rename from examples/balls/mxfp2int/arch/src/main/scala/MxfpDecoder.scala rename to examples/prefab/balls/mxfp2int/arch/MxfpDecoder.scala index 19fb356f7..e5692beda 100644 --- a/examples/balls/mxfp2int/arch/src/main/scala/MxfpDecoder.scala +++ b/examples/prefab/balls/mxfp2int/arch/MxfpDecoder.scala @@ -49,9 +49,11 @@ object MxfpDecoder { * Positive shift = left, negative shift = right (arithmetic). */ def scaleShift(value: SInt, scale: UInt, valueWidth: Int): SInt = { - val shift = scale.zext - 127.S(10.W) - val absShift = Mux(shift >= 0.S, shift, -shift)(4, 0) - val widened = value.pad(32) + val shift = scale.zext - 127.S(10.W) + val magnitude = Mux(shift >= 0.S, shift, -shift).asUInt + // Shifts past 31 saturate a left shift and fill a right shift with the sign. + val absShift = Mux(magnitude > 31.U, 31.U, magnitude(4, 0)) + val widened = value.pad(32) Mux( shift >= 0.S, (widened << absShift).asSInt, diff --git a/examples/prefab/balls/mxfp2int/configs/Mxfp2IntBallParam.scala b/examples/prefab/balls/mxfp2int/configs/Mxfp2IntBallParam.scala new file mode 100644 index 000000000..867aa5e22 --- /dev/null +++ b/examples/prefab/balls/mxfp2int/configs/Mxfp2IntBallParam.scala @@ -0,0 +1,17 @@ +package examples.balls.mxfp2int.configs + +import buckyball.balls.mxfp2int.{Params => ParametersProto} +import upickle.default.{macroRW, ReadWriter} + +case class Mxfp2IntBallParam( + mxfpFormat: String) + +object Mxfp2IntBallParam { + + def fromProto(source: ParametersProto): Mxfp2IntBallParam = { + require(source.hasMxfpFormat) + Mxfp2IntBallParam(source.getMxfpFormat) + } + + implicit val rw: ReadWriter[Mxfp2IntBallParam] = macroRW +} diff --git a/examples/prefab/balls/mxfp2int/configs/default.toml b/examples/prefab/balls/mxfp2int/configs/default.toml new file mode 100644 index 000000000..77ef4658d --- /dev/null +++ b/examples/prefab/balls/mxfp2int/configs/default.toml @@ -0,0 +1,3 @@ +# Mxfp2Int Ball Configuration + +mxfpFormat = "MXFP4" diff --git a/examples/prefab/balls/mxfp2int/configs/mxfp2int_params.proto b/examples/prefab/balls/mxfp2int/configs/mxfp2int_params.proto new file mode 100644 index 000000000..1e78606da --- /dev/null +++ b/examples/prefab/balls/mxfp2int/configs/mxfp2int_params.proto @@ -0,0 +1,8 @@ +syntax = "proto3"; +package buckyball.balls.mxfp2int; +option java_package = "buckyball.balls.mxfp2int"; +option java_multiple_files = true; + +message Params { + optional string mxfpFormat = 1; +} diff --git a/examples/prefab/balls/mxfp2int/emu/src/55_mxfp2int.rs b/examples/prefab/balls/mxfp2int/emu/src/55_mxfp2int.rs new file mode 100644 index 000000000..85ccd59f6 --- /dev/null +++ b/examples/prefab/balls/mxfp2int/emu/src/55_mxfp2int.rs @@ -0,0 +1,69 @@ +//===- 55_mxfp2int.rs - MXFP2INT instruction (MXFP4 to INT8) ---------------===// +// +// Dequantizes iter MXFP4 blocks of 32 elements to int8. +// +// op1 bank: block b in row b, element e in byte e / 2, low nibble first; the +// element value is raw - 8. +// op2 bank: E8M0 scale of block b in byte b % 16 of row b / 16. +// wr bank: elements 0..15 of block b in row 2b, 16..31 in row 2b + 1, +// saturate(value * 2^(scale - 127)); the right shift rounds toward +// minus infinity. +// rs2 is reserved and zero. +// +//===----------------------------------------------------------------------===// + +use bebop_bemu::thread::banks::bank_size; +use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old, rs1_update}; +use super::instruction::{BallModel, ExecContext}; + +pub struct Mxfp2Int; + +const ROW_BYTES: usize = 16; +const ELEMENTS: usize = 32; + +impl BallModel for Mxfp2Int { + fn exec(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + assert!( + !rs1_read_old(xs1) && !rs1_update(xs1), + "mxfp2int: reads no old destination; rs1[29] and rs1[63] must be clear" + ); + assert!(xs2 == 0, "mxfp2int: rs2 is reserved and must be zero"); + let data = rs1_b0(xs1); + let scales = rs1_b1(xs1); + let out = rs1_b2(xs1); + let blocks = rs1_iter(xs1) as usize; + for bank in [data, scales, out] { + let config = ctx.config(bank); + assert!(config.allocated && config.cols == 1, "mxfp2int: bank {bank} must be one allocated group"); + } + let lines = bank_size() / ROW_BYTES; + assert!(blocks > 0 && blocks * 2 <= lines, "mxfp2int: iter must be 1..half the bank depth"); + + let data = pbank(ctx, data); + let scales = pbank(ctx, scales); + let out = pbank(ctx, out); + for block in 0..blocks { + let scale = ctx.banks[scales][(block / 16) * ROW_BYTES + block % 16]; + let shift = scale as i32 - 127; + let row = &ctx.banks[data][block * ROW_BYTES..(block + 1) * ROW_BYTES]; + let mut result = [0u8; ELEMENTS]; + for (element, byte) in result.iter_mut().enumerate() { + let value = ((row[element / 2] >> ((element % 2) * 4)) & 0x0f) as i32 - 8; + let scaled = if shift >= 0 { + // |value| <= 8, so shifts past 7 saturate. + (value as i64) << shift.min(8) + } else { + (value >> (-shift).min(31)) as i64 + }; + *byte = scaled.clamp(-128, 127) as i8 as u8; + } + let start = block * 2 * ROW_BYTES; + ctx.banks[out][start..start + ELEMENTS].copy_from_slice(&result); + } + 0 + } + + fn latency(xs1: u64, _xs2: u64) -> u64 { + rs1_iter(xs1) * 4 + } +} diff --git a/examples/prefab/balls/mxfp2int/emu/src/dispatch.rs b/examples/prefab/balls/mxfp2int/emu/src/dispatch.rs new file mode 100644 index 000000000..672066ef9 --- /dev/null +++ b/examples/prefab/balls/mxfp2int/emu/src/dispatch.rs @@ -0,0 +1,31 @@ +use bebop_bemu::ball::{BallModel, ExecContext}; + +use super::f55_mxfp2int; + +const BALL_CLASS: &str = "examples.balls.mxfp2int.Mxfp2IntBall"; + +pub fn execute_known( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, + ctx: &mut ExecContext, +) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("MXFP2INT") => Some(f55_mxfp2int::Mxfp2Int::exec(xs1, xs2, ctx)), + Some(_) | None => None, + } +} + +pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("MXFP2INT") => Some(f55_mxfp2int::Mxfp2Int::latency(xs1, xs2)), + Some(_) | None => None, + } +} diff --git a/examples/prefab/balls/mxfp2int/emu/src/lib.rs b/examples/prefab/balls/mxfp2int/emu/src/lib.rs new file mode 100644 index 000000000..fd1b1d219 --- /dev/null +++ b/examples/prefab/balls/mxfp2int/emu/src/lib.rs @@ -0,0 +1,12 @@ +pub(crate) use bebop_bemu::{thread::instructions::decode, ball as instruction}; + +#[path = "55_mxfp2int.rs"] +mod f55_mxfp2int; + +mod dispatch; + +pub use dispatch::{cycles_after_issue, execute_known}; + +#[bebop_bemu::ball::linkme::distributed_slice(bebop_bemu::BALLS)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static BALL: bebop_bemu::Ball = bebop_bemu::Ball { execute: execute_known, cycles: cycles_after_issue }; diff --git a/examples/prefab/balls/mxfp2int/stack/src/mxfp2intOps.cpp b/examples/prefab/balls/mxfp2int/stack/src/mxfp2intOps.cpp new file mode 100644 index 000000000..c3ee27bc7 --- /dev/null +++ b/examples/prefab/balls/mxfp2int/stack/src/mxfp2intOps.cpp @@ -0,0 +1,72 @@ +//===- mxfp2intOps.cpp - Mxfp2IntBall operations --------------------------===// +// +// The dialect of Mxfp2IntBall, its virtual operations building their physical +// ones and the instructions of the physical operations. +// +//===----------------------------------------------------------------------===// + +#include "mxfp2intOps.h" + +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::mxfp2int_ball; +using mlir::buddy::bankFields; +using mlir::buddy::constantI64; +using mlir::buddy::emitInstruction; +using mlir::buddy::field; + +#include "mxfp2intDialect.cpp.inc" + +#define GET_OP_CLASSES +#include "mxfp2intOps.cpp.inc" + +void Mxfp2IntBallDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "mxfp2intOps.cpp.inc" + >(); +} + +namespace { +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallDialect([](DialectRegistry ®istry) { + registry.insert(); + }), + true); +} // namespace + +SmallVector BankMxfp4I8Op::getSourceBanks() { + return {getData(), getScales()}; +} +Value BankMxfp4I8Op::getDestinationBank() { return getOutput(); } +bool BankMxfp4I8Op::readsDestination() { return false; } +Operation *BankMxfp4I8Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return Mxfp4I8Op::create(builder, getLoc(), sources[0], sources[1], + destination, getRows()); +} + +SmallVector Mxfp4I8Op::getSourceBanks() { + return {int64_t(getData()), int64_t(getScales())}; +} +int64_t Mxfp4I8Op::getDestinationBank() { return getOutput(); } +LogicalResult Mxfp4I8Op::lowerToInstructions(OpBuilder &builder) { + Location loc = getLoc(); + int64_t rows = getRows(), data = getData(), scales = getScales(), + output = getOutput(); + const auto &target = buckyball_target::getBuckyballTarget(); + if (target.bankWidthBits != 128 || rows <= 0 || rows * 2 > target.bankDepth) + return emitOpError("mxfp4_i8 rows exceed half the bank"); + if (data == scales || data == output || scales == output) + return emitOpError( + "mxfp4_i8 requires distinct data, scale and output banks"); + emitInstruction(builder, loc, + buckyball_target::getBuckyballFunct7("MXFP2INT"), + constantI64(builder, loc, + bankFields({data, scales}, output, false, rows)), + constantI64(builder, loc, 0)); + return success(); +} diff --git a/examples/prefab/balls/mxfp2int/stack/src/mxfp2intOps.h b/examples/prefab/balls/mxfp2int/stack/src/mxfp2intOps.h new file mode 100644 index 000000000..9e26f1c5f --- /dev/null +++ b/examples/prefab/balls/mxfp2int/stack/src/mxfp2intOps.h @@ -0,0 +1,20 @@ +//===- mxfp2intOps.h - Mxfp2IntBall operations ----------------------------===// + +#ifndef MXFP2INT_OPS_H +#define MXFP2INT_OPS_H + +#include "Interfaces/BankInterfaces.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "mxfp2intDialect.h.inc" + +#define GET_OP_CLASSES +#include "mxfp2intOps.h.inc" + +#endif // MXFP2INT_OPS_H diff --git a/examples/prefab/balls/mxfp2int/stack/src/mxfp2intOps.td b/examples/prefab/balls/mxfp2int/stack/src/mxfp2intOps.td new file mode 100644 index 000000000..5e619d794 --- /dev/null +++ b/examples/prefab/balls/mxfp2int/stack/src/mxfp2intOps.td @@ -0,0 +1,50 @@ +//===- mxfp2intOps.td - Mxfp2IntBall operations ------------*- tablegen -*-===// +// +// mxfp4_i8 (data, scales) -> output: dequantizes `rows` MXFP4 blocks of 32 +// elements to i8, one MXFP2INT instruction. Banks are one group of sixteen +// bytes per row: +// data: block b in row b, element e in byte e / 2, low nibble first; the +// element value is raw - 8. +// scales: the E8M0 scale of block b in byte b % 16 of row b / 16. +// output: elements 0..15 of block b in row 2b, 16..31 in row 2b + 1, +// saturate(value * 2^(scale - 127)), the right shift rounding toward minus +// infinity. +// `rows` is the number of blocks, 1..half the bank depth. The Ball must be +// configured for MXFP4. +// +//===----------------------------------------------------------------------===// + +#ifndef MXFP2INT_OPS +#define MXFP2INT_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Interfaces/BankInterfaces.td" + +def Mxfp2IntBall_Dialect : Dialect { + let name = "mxfp2int_ball"; + let cppNamespace = "::buddy::mxfp2int_ball"; + let summary = "The operations of Mxfp2IntBall"; +} + +class Mxfp2IntBall_Op traits = []> + : Op])>; + +def BankMxfp4I8Op : Mxfp2IntBall_Op<"bank_mxfp4_i8", + [DeclareOpInterfaceMethods]> { + let summary = "mxfp4_i8 on virtual banks"; + let arguments = (ins I64:$data, I64:$scales, I64:$output, I64Attr:$rows); + let results = (outs I64:$outputOut); + let assemblyFormat = "$data `,` $scales `->` $output prop-dict attr-dict"; +} + +def Mxfp4I8Op : Mxfp2IntBall_Op<"mxfp4_i8", + [DeclareOpInterfaceMethods]> { + let summary = "mxfp4_i8 on physical banks"; + let arguments = (ins I64Attr:$data, I64Attr:$scales, I64Attr:$output, + I64Attr:$rows); + let assemblyFormat = "prop-dict attr-dict"; +} + +#endif // MXFP2INT_OPS diff --git a/examples/prefab/balls/mxfp2int/verify/casegen/Cargo.lock b/examples/prefab/balls/mxfp2int/verify/casegen/Cargo.lock new file mode 100644 index 000000000..61f560c0f --- /dev/null +++ b/examples/prefab/balls/mxfp2int/verify/casegen/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "mxfp2int-casegen" +version = "0.1.0" diff --git a/examples/prefab/balls/mxfp2int/verify/casegen/Cargo.toml b/examples/prefab/balls/mxfp2int/verify/casegen/Cargo.toml new file mode 100644 index 000000000..e2a6e2794 --- /dev/null +++ b/examples/prefab/balls/mxfp2int/verify/casegen/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "mxfp2int-casegen" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["rlib", "cdylib"] + +[lints.rust] +unused_must_use = "deny" diff --git a/examples/prefab/balls/mxfp2int/verify/casegen/src/casegen.rs b/examples/prefab/balls/mxfp2int/verify/casegen/src/casegen.rs new file mode 100644 index 000000000..6ae1366ae --- /dev/null +++ b/examples/prefab/balls/mxfp2int/verify/casegen/src/casegen.rs @@ -0,0 +1,231 @@ +// MXFP2INT reference model (MXFP4 to INT8), the contract shared with the +// BEMU model (emu/src/55_mxfp2int.rs) and Mxfp2IntBall: +// +// op1 bank: block b in row b, element e in byte e / 2, low nibble first; the +// element value is raw - 8. +// op2 bank: E8M0 scale of block b in byte b % 16 of row b / 16. +// wr bank: elements 0..15 of block b in row 2b, 16..31 in row 2b + 1, +// saturate(value * 2^(scale - 127)); the right shift rounds toward +// minus infinity. +// rs2 is reserved and zero; 1 <= iter <= bank_entries / 2. + +pub const ROW_BYTES: usize = 16; +pub const ELEMENTS: usize = 32; +pub const NUM_CASES: u32 = 6; + +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Mxfp2IntCmd { + pub rob_id: u32, + pub iter: u32, + pub rs2_lo: u32, + pub rs2_hi: u32, + pub op1_bank: u32, + pub op2_bank: u32, + pub wr_bank: u32, + pub op1_group: u32, + pub op2_group: u32, + pub wr_group: u32, + pub op1_col: u32, + pub op2_col: u32, + pub wr_col: u32, +} + +#[derive(Clone, Debug)] +pub struct Preload { + pub bank: u32, + pub group: u32, + pub addr: u32, + pub data: u128, +} + +#[derive(Clone, Debug)] +pub struct Write { + pub addr: u32, + pub mask: u16, + pub data: u128, +} + +#[derive(Clone, Debug)] +pub struct Step { + pub cmd: Mxfp2IntCmd, + pub preload: Vec, + pub reads: [Vec; 2], + pub writes: Vec, +} + +#[derive(Clone, Debug)] +pub struct Case { + pub steps: Vec, +} + +struct Rng(u32); + +impl Rng { + fn next(&mut self) -> u32 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 17; + self.0 ^= self.0 << 5; + self.0 + } +} + +fn word(bytes: &[u8]) -> u128 { + let mut out = 0u128; + for (lane, byte) in bytes.iter().enumerate() { + out |= u128::from(*byte) << (lane * 8); + } + out +} + +fn dequant(raw: u8, scale: u8) -> u8 { + let value = raw as i32 - 8; + let shift = scale as i32 - 127; + let scaled = if shift >= 0 { + // |value| <= 8, so shifts past 7 saturate. + (value as i64) << shift.min(8) + } else { + (value >> (-shift).min(31)) as i64 + }; + scaled.clamp(-128, 127) as i8 as u8 +} + +// `data` holds iter rows of packed nibbles, `scales` one byte per block. +fn step( + bank_entries: u32, + rob_id: u32, + banks: [u32; 3], + groups: [u32; 3], + data: &[u8], + scales: &[u8], +) -> Step { + let blocks = scales.len(); + if blocks == 0 || blocks * 2 > bank_entries as usize || data.len() != blocks * ROW_BYTES { + panic!("mxfp2int: iter must be 1..half the bank depth"); + } + if banks[0] == banks[1] || banks[0] == banks[2] || banks[1] == banks[2] { + panic!("mxfp2int: banks must differ"); + } + let mut scale_rows = vec![0u8; blocks.div_ceil(ROW_BYTES) * ROW_BYTES]; + scale_rows[..blocks].copy_from_slice(scales); + let mut preload = Vec::new(); + for (row, bytes) in data.chunks(ROW_BYTES).enumerate() { + preload.push(Preload { bank: banks[0], group: groups[0], addr: row as u32, data: word(bytes) }); + } + for (row, bytes) in scale_rows.chunks(ROW_BYTES).enumerate() { + preload.push(Preload { bank: banks[1], group: groups[1], addr: row as u32, data: word(bytes) }); + } + let mut reads = [Vec::new(), Vec::new()]; + let mut writes = Vec::new(); + for block in 0..blocks { + reads[0].push(block as u32); + reads[1].push((block / ROW_BYTES) as u32); + let row = &data[block * ROW_BYTES..(block + 1) * ROW_BYTES]; + let mut result = [0u8; ELEMENTS]; + for (element, byte) in result.iter_mut().enumerate() { + *byte = dequant((row[element / 2] >> ((element % 2) * 4)) & 0x0f, scales[block]); + } + writes.push(Write { addr: (block * 2) as u32, mask: 0xffff, data: word(&result[..ROW_BYTES]) }); + writes.push(Write { addr: (block * 2 + 1) as u32, mask: 0xffff, data: word(&result[ROW_BYTES..]) }); + } + Step { + cmd: Mxfp2IntCmd { + rob_id, + iter: blocks as u32, + rs2_lo: 0, + rs2_hi: 0, + op1_bank: banks[0], + op2_bank: banks[1], + wr_bank: banks[2], + op1_group: groups[0], + op2_group: groups[1], + wr_group: groups[2], + op1_col: 1, + op2_col: 1, + wr_col: 1, + }, + preload, + reads, + writes, + } +} + +fn pattern_data(blocks: usize) -> Vec { + let raw = |block: usize, elem: usize| ((elem * 7 + block * 3) & 0xf) as u8; + let mut data = vec![0u8; blocks * ROW_BYTES]; + for block in 0..blocks { + for elem in (0..ELEMENTS).step_by(2) { + data[block * ROW_BYTES + elem / 2] = (raw(block, elem + 1) << 4) | raw(block, elem); + } + } + data +} + +fn random_data(rng: &mut Rng, blocks: usize) -> Vec { + (0..blocks * ROW_BYTES).map(|_| rng.next() as u8).collect() +} + +pub fn gen_case(index: u32, bank_entries: u32) -> Case { + let mut rng = Rng(0x6d2b_79f5 ^ index.wrapping_mul(0x9e37_79b9)); + let steps = match index { + // Two blocks at the identity scale (mxfp4_e8m0_two_block_test). + 0 => vec![step(bank_entries, 1, [1, 2, 3], [0, 0, 0], &pattern_data(2), &[127, 127])], + // Scales across right shifts, saturating left shifts and shifts past 31, + // spanning two scale rows (mxfp4_e8m0_scale_range_test). + 1 => { + let scales = [0, 1, 90, 120, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 150, 158, 159, 200, 255]; + vec![step(bank_entries, 2, [1, 2, 3], [0, 0, 0], &pattern_data(scales.len()), &scales)] + } + // Random nibbles and scales near the identity, non-zero group bases. + 2 => { + let scales: Vec = (0..100).map(|_| 120 + (rng.next() % 16) as u8).collect(); + let data = random_data(&mut rng, scales.len()); + vec![step(bank_entries, 5, [4, 7, 9], [3, 11, 6], &data, &scales)] + } + // One block, highest banks and groups. + 3 => vec![step(bank_entries, 15, [1021, 1022, 1023], [15, 15, 15], &random_data(&mut rng, 1), &[129])], + // Full capacity: iter = bank_entries / 2. + 4 => { + let blocks = (bank_entries / 2) as usize; + let scales: Vec = (0..blocks).map(|_| rng.next() as u8).collect(); + let data = random_data(&mut rng, blocks); + vec![step(bank_entries, 9, [0, 512, 768], [1, 2, 4], &data, &scales)] + } + // Back-to-back commands without reset. + 5 => (0..3) + .map(|i| { + let blocks = 1 + (rng.next() % 18) as usize; + let scales: Vec = (0..blocks).map(|_| 118 + (rng.next() % 20) as u8).collect(); + let data = random_data(&mut rng, blocks); + step(bank_entries, 6 + i, [8 + i, 16 + i, 24 + i], [i, i + 4, i + 8], &data, &scales) + }) + .collect(), + _ => panic!("mxfp2int: unsupported case {index}"), + }; + Case { steps } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cases_build() { + for index in 0..NUM_CASES { + gen_case(index, 1024); + } + } + + // Golden values of mxfp4_e8m0_scale_range_test. + #[test] + fn scale_range_golden() { + assert_eq!(dequant(15, 127) as i8, 7); + assert_eq!(dequant(0, 127) as i8, -8); + assert_eq!(dequant(9, 0) as i8, 0); + assert_eq!(dequant(7, 0) as i8, -1); + assert_eq!(dequant(15, 255) as i8, 127); + assert_eq!(dequant(0, 255) as i8, -128); + assert_eq!(dequant(1, 126) as i8, -4); + assert_eq!(dequant(15, 130) as i8, 56); + } +} diff --git a/examples/prefab/balls/mxfp2int/verify/casegen/src/config.rs b/examples/prefab/balls/mxfp2int/verify/casegen/src/config.rs new file mode 100644 index 000000000..241755f0f --- /dev/null +++ b/examples/prefab/balls/mxfp2int/verify/casegen/src/config.rs @@ -0,0 +1,9 @@ +pub fn bank_entries() -> u32 { + let path = std::env::var("BB_VERIFY_CONFIG").expect("BB_VERIFY_CONFIG is required"); + let text = std::fs::read_to_string(path).expect("failed to read BB_VERIFY_CONFIG"); + text.lines() + .find_map(|line| line.strip_prefix("bank_entries=")) + .unwrap_or_else(|| panic!("bank_entries missing from BB_VERIFY_CONFIG")) + .parse() + .expect("invalid bank_entries in BB_VERIFY_CONFIG") +} diff --git a/examples/prefab/balls/mxfp2int/verify/casegen/src/lib.rs b/examples/prefab/balls/mxfp2int/verify/casegen/src/lib.rs new file mode 100644 index 000000000..129919ffc --- /dev/null +++ b/examples/prefab/balls/mxfp2int/verify/casegen/src/lib.rs @@ -0,0 +1,116 @@ +mod casegen; +mod config; + +use std::cell::RefCell; + +use casegen::{Case, Mxfp2IntCmd, Step}; + +thread_local! { + static CURRENT: RefCell> = const { RefCell::new(None) }; +} + +#[no_mangle] +pub extern "C" fn mxfp2int_num_cases() -> u32 { + casegen::NUM_CASES +} + +#[no_mangle] +pub extern "C" fn mxfp2int_case_load(index: u32) -> u32 { + let case = casegen::gen_case(index, config::bank_entries()); + let steps = case.steps.len() as u32; + CURRENT.with(|current| *current.borrow_mut() = Some(case)); + steps +} + +fn current_step R, R>(step: u32, f: F) -> R { + CURRENT.with(|current| match current.borrow().as_ref() { + Some(case) => match case.steps.get(step as usize) { + Some(s) => f(s), + None => panic!("mxfp2int_case: step {step} out of range {}", case.steps.len()), + }, + None => panic!("mxfp2int_case: no case loaded; call mxfp2int_case_load first"), + }) +} + +fn idx(index: u32, limit: usize, what: &str) -> usize { + if index as usize >= limit { + panic!("mxfp2int_case_{what}: index {index} out of range {limit}"); + } + index as usize +} + +#[no_mangle] +pub extern "C" fn mxfp2int_case_cmd(step: u32, out_ptr: *mut Mxfp2IntCmd) { + if out_ptr.is_null() { + panic!("mxfp2int_case_cmd: null out_ptr"); + } + current_step(step, |s| unsafe { *out_ptr = s.cmd }); +} + +#[no_mangle] +pub extern "C" fn mxfp2int_case_preload_count(step: u32) -> u32 { + current_step(step, |s| s.preload.len() as u32) +} + +#[no_mangle] +pub extern "C" fn mxfp2int_case_preload_bank(step: u32, i: u32) -> u32 { + current_step(step, |s| s.preload[idx(i, s.preload.len(), "preload")].bank) +} + +#[no_mangle] +pub extern "C" fn mxfp2int_case_preload_group(step: u32, i: u32) -> u32 { + current_step(step, |s| s.preload[idx(i, s.preload.len(), "preload")].group) +} + +#[no_mangle] +pub extern "C" fn mxfp2int_case_preload_addr(step: u32, i: u32) -> u32 { + current_step(step, |s| s.preload[idx(i, s.preload.len(), "preload")].addr) +} + +#[no_mangle] +pub extern "C" fn mxfp2int_case_preload_lo(step: u32, i: u32) -> u64 { + current_step(step, |s| s.preload[idx(i, s.preload.len(), "preload")].data as u64) +} + +#[no_mangle] +pub extern "C" fn mxfp2int_case_preload_hi(step: u32, i: u32) -> u64 { + current_step(step, |s| (s.preload[idx(i, s.preload.len(), "preload")].data >> 64) as u64) +} + +#[no_mangle] +pub extern "C" fn mxfp2int_case_read_count(step: u32, port: u32) -> u32 { + current_step(step, |s| s.reads[idx(port, 2, "read_port")].len() as u32) +} + +#[no_mangle] +pub extern "C" fn mxfp2int_case_read_addr(step: u32, port: u32, i: u32) -> u32 { + current_step(step, |s| { + let reads = &s.reads[idx(port, 2, "read_port")]; + reads[idx(i, reads.len(), "read")] + }) +} + +#[no_mangle] +pub extern "C" fn mxfp2int_case_write_count(step: u32) -> u32 { + current_step(step, |s| s.writes.len() as u32) +} + +#[no_mangle] +pub extern "C" fn mxfp2int_case_write_addr(step: u32, i: u32) -> u32 { + current_step(step, |s| s.writes[idx(i, s.writes.len(), "write")].addr) +} + +#[no_mangle] +pub extern "C" fn mxfp2int_case_write_mask(step: u32, i: u32) -> u32 { + current_step(step, |s| u32::from(s.writes[idx(i, s.writes.len(), "write")].mask)) +} + +#[no_mangle] +pub extern "C" fn mxfp2int_case_write_lo(step: u32, i: u32) -> u64 { + current_step(step, |s| s.writes[idx(i, s.writes.len(), "write")].data as u64) +} + +#[no_mangle] +pub extern "C" fn mxfp2int_case_write_hi(step: u32, i: u32) -> u64 { + current_step(step, |s| (s.writes[idx(i, s.writes.len(), "write")].data >> 64) as u64) +} diff --git a/examples/prefab/balls/mxfp2int/verify/filelists/mxfp2int_ball.f b/examples/prefab/balls/mxfp2int/verify/filelists/mxfp2int_ball.f new file mode 100644 index 000000000..44ff7dfe1 --- /dev/null +++ b/examples/prefab/balls/mxfp2int/verify/filelists/mxfp2int_ball.f @@ -0,0 +1,25 @@ ++incdir+@UVM@/src/core ++incdir+@UVM@/src/ball ++incdir+@UVM@/src/ball/agents/cmd ++incdir+@UVM@/src/ball/agents/mem ++incdir+@UVM@/src/ball/agents/resp ++incdir+@UVM@/src/ball/cov ++incdir+@UVM@/src/ball/env ++incdir+src ++incdir+src/pkg ++incdir+src/common ++incdir+src/seq ++incdir+src/cov ++incdir+src/env ++incdir+src/tests ++incdir+@RTL@ + +@UVM@/src/core/package.sv +@UVM@/src/ball/bb_blink_if.sv +@UVM@/src/ball/bb_uvm_pkg.sv + +@RTL@/PipelinedMxfp2Int.sv +@RTL@/Mxfp2IntBall.sv + +src/pkg/mxfp2int_pkg.sv +src/tb_top.sv diff --git a/examples/prefab/balls/mxfp2int/verify/src/common/mxfp2int_defs.svh b/examples/prefab/balls/mxfp2int/verify/src/common/mxfp2int_defs.svh new file mode 100644 index 000000000..28fd50e0f --- /dev/null +++ b/examples/prefab/balls/mxfp2int/verify/src/common/mxfp2int_defs.svh @@ -0,0 +1,39 @@ +typedef struct { + int unsigned rob_id; + int unsigned iter; + int unsigned rs2_lo; + int unsigned rs2_hi; + int unsigned op1_bank; + int unsigned op2_bank; + int unsigned wr_bank; + int unsigned op1_group; + int unsigned op2_group; + int unsigned wr_group; + int unsigned op1_col; + int unsigned op2_col; + int unsigned wr_col; +} mxfp2int_cmd_dpi_t; + +import "DPI-C" function int unsigned mxfp2int_num_cases(); +import "DPI-C" function int unsigned mxfp2int_case_load(input int unsigned index); +import "DPI-C" function void mxfp2int_case_cmd(input int unsigned step, output mxfp2int_cmd_dpi_t cmd); +import "DPI-C" function int unsigned mxfp2int_case_preload_count(input int unsigned step); +import "DPI-C" function int unsigned mxfp2int_case_preload_bank(input int unsigned step, input int unsigned i); +import "DPI-C" function int unsigned mxfp2int_case_preload_group(input int unsigned step, input int unsigned i); +import "DPI-C" function int unsigned mxfp2int_case_preload_addr(input int unsigned step, input int unsigned i); +import "DPI-C" function longint unsigned mxfp2int_case_preload_lo(input int unsigned step, input int unsigned i); +import "DPI-C" function longint unsigned mxfp2int_case_preload_hi(input int unsigned step, input int unsigned i); +import "DPI-C" function int unsigned mxfp2int_case_read_count(input int unsigned step, input int unsigned port); +import "DPI-C" function int unsigned mxfp2int_case_read_addr(input int unsigned step, input int unsigned port, input int unsigned i); +import "DPI-C" function int unsigned mxfp2int_case_write_count(input int unsigned step); +import "DPI-C" function int unsigned mxfp2int_case_write_addr(input int unsigned step, input int unsigned i); +import "DPI-C" function int unsigned mxfp2int_case_write_mask(input int unsigned step, input int unsigned i); +import "DPI-C" function longint unsigned mxfp2int_case_write_lo(input int unsigned step, input int unsigned i); +import "DPI-C" function longint unsigned mxfp2int_case_write_hi(input int unsigned step, input int unsigned i); + +`ifndef MXFP2INT_FUNCT7 +`error "MXFP2INT_FUNCT7 must be provided by the selected ballISA" +`endif + +localparam int MXFP2INT_THREAD_FUNCT7 = `MXFP2INT_FUNCT7; +localparam int MXFP2INT_TIMEOUT_CYCLES = 100000; diff --git a/examples/prefab/balls/mxfp2int/verify/src/common/mxfp2int_items.svh b/examples/prefab/balls/mxfp2int/verify/src/common/mxfp2int_items.svh new file mode 100644 index 000000000..44c31936e --- /dev/null +++ b/examples/prefab/balls/mxfp2int/verify/src/common/mxfp2int_items.svh @@ -0,0 +1,85 @@ +class mxfp2int_cmd_item extends bb_blink_cmd_item; + `uvm_object_utils(mxfp2int_cmd_item) + + int unsigned preload_bank[]; + int unsigned preload_group[]; + int unsigned preload_addr[]; + bit [127:0] preload_data[]; + int unsigned read_addr0[]; + int unsigned read_addr1[]; + int unsigned write_addr[]; + bit [15:0] write_mask[]; + bit [127:0] write_data[]; + + function new(string name = "mxfp2int_cmd_item"); + super.new(name); + endfunction + + function void load_rust_step(int unsigned step); + mxfp2int_cmd_dpi_t cmd; + int unsigned n; + + mxfp2int_case_cmd(step, cmd); + rob_id = cmd.rob_id[BB_BLINK_ROB_ID_W-1:0]; + funct = MXFP2INT_THREAD_FUNCT7[6:0]; + rs2 = {cmd.rs2_hi, cmd.rs2_lo}; + iter = cmd.iter; + op1_bank = cmd.op1_bank; + op2_bank = cmd.op2_bank; + wr_bank = cmd.wr_bank; + op1_group = cmd.op1_group; + op2_group = cmd.op2_group; + wr_group = cmd.wr_group; + op1_col = cmd.op1_col; + op2_col = cmd.op2_col; + wr_col = cmd.wr_col; + wr_read_bank = '0; + wr_arch_bank = cmd.wr_bank; + if (rob_id != cmd.rob_id || op1_bank != cmd.op1_bank || op2_bank != cmd.op2_bank || + wr_bank != cmd.wr_bank || op1_group != cmd.op1_group || op2_group != cmd.op2_group || + wr_group != cmd.wr_group) + `uvm_fatal("CASE", "command field exceeds the configured BLink width") + + n = mxfp2int_case_preload_count(step); + preload_bank = new[n]; + preload_group = new[n]; + preload_addr = new[n]; + preload_data = new[n]; + for (int unsigned i = 0; i < n; i++) begin + preload_bank[i] = mxfp2int_case_preload_bank(step, i); + preload_group[i] = mxfp2int_case_preload_group(step, i); + preload_addr[i] = mxfp2int_case_preload_addr(step, i); + preload_data[i] = {mxfp2int_case_preload_hi(step, i), mxfp2int_case_preload_lo(step, i)}; + end + n = mxfp2int_case_read_count(step, 0); + read_addr0 = new[n]; + for (int unsigned i = 0; i < n; i++) read_addr0[i] = mxfp2int_case_read_addr(step, 0, i); + n = mxfp2int_case_read_count(step, 1); + read_addr1 = new[n]; + for (int unsigned i = 0; i < n; i++) read_addr1[i] = mxfp2int_case_read_addr(step, 1, i); + n = mxfp2int_case_write_count(step); + write_addr = new[n]; + write_mask = new[n]; + write_data = new[n]; + for (int unsigned i = 0; i < n; i++) begin + write_addr[i] = mxfp2int_case_write_addr(step, i); + write_mask[i] = mxfp2int_case_write_mask(step, i); + write_data[i] = {mxfp2int_case_write_hi(step, i), mxfp2int_case_write_lo(step, i)}; + end + endfunction + + function void do_copy(uvm_object rhs); + mxfp2int_cmd_item rhs_; + super.do_copy(rhs); + if (!$cast(rhs_, rhs)) `uvm_fatal("COPY", "rhs is not mxfp2int_cmd_item") + preload_bank = rhs_.preload_bank; + preload_group = rhs_.preload_group; + preload_addr = rhs_.preload_addr; + preload_data = rhs_.preload_data; + read_addr0 = rhs_.read_addr0; + read_addr1 = rhs_.read_addr1; + write_addr = rhs_.write_addr; + write_mask = rhs_.write_mask; + write_data = rhs_.write_data; + endfunction +endclass diff --git a/examples/prefab/balls/mxfp2int/verify/src/cov/mxfp2int_cov.svh b/examples/prefab/balls/mxfp2int/verify/src/cov/mxfp2int_cov.svh new file mode 100644 index 000000000..c285f5dba --- /dev/null +++ b/examples/prefab/balls/mxfp2int/verify/src/cov/mxfp2int_cov.svh @@ -0,0 +1,64 @@ +class mxfp2int_cov extends uvm_component; + `uvm_component_utils(mxfp2int_cov) + + uvm_analysis_imp_stim #(bb_blink_cmd_item, mxfp2int_cov) stim_imp; + + int unsigned cur_iter; + int unsigned cur_scale; + int unsigned cmd_count; + + covergroup cmd_cg; + coverpoint cur_iter { + bins one = {1}; + bins row = {[2 : 16]}; + bins rows = {[17 : 64]}; + bins many = {[65 : 511]}; + bins capacity = {512}; + } + endgroup + + covergroup scale_cg; + coverpoint cur_scale { + bins zero = {0}; + bins shift_past_31 = {[1 : 95]}; + bins right = {[96 : 126]}; + bins identity = {127}; + bins left = {[128 : 134]}; + bins saturate = {[135 : 254]}; + bins top = {255}; + } + endgroup + + function new(string name, uvm_component parent); + super.new(name, parent); + stim_imp = new("stim_imp", this); + cmd_cg = new(); + scale_cg = new(); + endfunction + + function void write_stim(bb_blink_cmd_item item); + mxfp2int_cmd_item stim; + if (!$cast(stim, item)) `uvm_fatal("COV", "stim item is not mxfp2int_cmd_item") + cur_iter = stim.iter; + cmd_cg.sample(); + foreach (stim.preload_data[i]) begin + if (stim.preload_bank[i] != stim.op2_bank) continue; + for (int b = 0; b < 16; b++) begin + if (stim.preload_addr[i] * 16 + b >= stim.iter) break; + cur_scale = stim.preload_data[i][b*8+:8]; + scale_cg.sample(); + end + end + cmd_count++; + endfunction + + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (cmd_count == 0) `uvm_fatal("COV", "no cmd observed") + if (cmd_cg.get_coverage() != 100.0) + `uvm_fatal("COV", $sformatf("mxfp2int iter coverage %0.2f%%", cmd_cg.get_coverage())) + if (scale_cg.get_coverage() != 100.0) + `uvm_fatal("COV", $sformatf("mxfp2int scale coverage %0.2f%%", scale_cg.get_coverage())) + endfunction +endclass diff --git a/examples/prefab/balls/mxfp2int/verify/src/env/mxfp2int_env.svh b/examples/prefab/balls/mxfp2int/verify/src/env/mxfp2int_env.svh new file mode 100644 index 000000000..32222694b --- /dev/null +++ b/examples/prefab/balls/mxfp2int/verify/src/env/mxfp2int_env.svh @@ -0,0 +1,27 @@ +class mxfp2int_env extends bb_blink_env #(`BB_IN_BW, `BB_OUT_BW); + `uvm_component_utils(mxfp2int_env) + + mxfp2int_scoreboard scb; + mxfp2int_cov icov; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + scb = mxfp2int_scoreboard::type_id::create("scb", this); + icov = mxfp2int_cov::type_id::create("icov", this); + endfunction + + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + scb.mem_model = mem_model; + cmd_agent.stim_ap.connect(scb.stim_imp); + cmd_agent.stim_ap.connect(icov.stim_imp); + cmd_agent.cmd_ap.connect(scb.cmd_imp); + read_mon.read_ap.connect(scb.read_imp); + write_mon.write_ap.connect(scb.write_imp); + resp_mon.resp_ap.connect(scb.resp_imp); + endfunction +endclass diff --git a/examples/prefab/balls/mxfp2int/verify/src/env/mxfp2int_scoreboard.svh b/examples/prefab/balls/mxfp2int/verify/src/env/mxfp2int_scoreboard.svh new file mode 100644 index 000000000..b9f01fc91 --- /dev/null +++ b/examples/prefab/balls/mxfp2int/verify/src/env/mxfp2int_scoreboard.svh @@ -0,0 +1,196 @@ +class mxfp2int_scoreboard extends uvm_scoreboard; + `uvm_component_utils(mxfp2int_scoreboard) + + uvm_analysis_imp_stim #(bb_blink_cmd_item, mxfp2int_scoreboard) stim_imp; + uvm_analysis_imp_cmd #(bb_blink_cmd_item, mxfp2int_scoreboard) cmd_imp; + uvm_analysis_imp_read #(bb_blink_read_item, mxfp2int_scoreboard) read_imp; + uvm_analysis_imp_write #(bb_blink_write_item, mxfp2int_scoreboard) write_imp; + uvm_analysis_imp_resp #(bb_blink_resp_item, mxfp2int_scoreboard) resp_imp; + + bb_blink_mem_model #(`BB_IN_BW, `BB_OUT_BW) mem_model; + + mxfp2int_cmd_item stim_q[$]; + int unsigned cmd_count; + int unsigned read_count[2]; + int unsigned write_count; + int unsigned resp_count; + + function new(string name, uvm_component parent); + super.new(name, parent); + stim_imp = new("stim_imp", this); + cmd_imp = new("cmd_imp", this); + read_imp = new("read_imp", this); + write_imp = new("write_imp", this); + resp_imp = new("resp_imp", this); + endfunction + + function void write_stim(bb_blink_cmd_item item); + mxfp2int_cmd_item got; + mxfp2int_cmd_item clone; + + if (stim_q.size() != 0) `uvm_fatal("SCB", "single outstanding command supported") + if (!$cast(got, item)) `uvm_fatal("SCB", "stim item is not mxfp2int_cmd_item") + if (!$cast(clone, got.clone())) `uvm_fatal("SCB", "failed to clone stimulus item") + stim_q.push_back(clone); + if (mem_model == null) `uvm_fatal("SCB", "mem_model handle not set") + mem_model.clear_mem(); + foreach (clone.preload_data[i]) + mem_model.load_word_g(clone.preload_bank[i], clone.preload_group[i], clone.preload_addr[i], + clone.preload_data[i]); + mem_model.arm(); + endfunction + + function void write_cmd(bb_blink_cmd_item item); + mxfp2int_cmd_item stim; + bb_blink_cmd_item want; + stim = current_stim("CMD"); + // Compare as plain BLink commands; the comparer rejects differing types. + want = bb_blink_cmd_item::type_id::create("want"); + want.copy(stim); + verification_pkg::evidence::checks++; + if (!item.compare(want)) + `uvm_fatal("CMD", $sformatf("command mismatch: got %s exp %s", item.sprint(), want.sprint())) + cmd_count++; + endfunction + + function void write_read(bb_blink_read_item item); + mxfp2int_cmd_item stim; + int unsigned exp_bank; + int unsigned exp_group; + int unsigned exp_addr; + + stim = current_stim("READ"); + if (item.port != 0 && item.port != 1) + `uvm_fatal("READ", $sformatf("Mxfp2IntBall has two read ports, got port %0d", item.port)) + if (item.port == 0) begin + if (read_count[0] >= stim.read_addr0.size()) + `uvm_fatal("READ", $sformatf("extra read on port 0 at %0d", read_count[0])) + exp_bank = stim.op1_bank; + exp_group = stim.op1_group; + exp_addr = stim.read_addr0[read_count[0]]; + end else begin + if (read_count[1] >= stim.read_addr1.size()) + `uvm_fatal("READ", $sformatf("extra read on port 1 at %0d", read_count[1])) + exp_bank = stim.op2_bank; + exp_group = stim.op2_group; + exp_addr = stim.read_addr1[read_count[1]]; + end + verification_pkg::evidence::checks++; + if (item.rob_id !== stim.rob_id) + `uvm_fatal("READ", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + if (item.bank_id !== exp_bank[BB_BLINK_BANK_ID_W-1:0]) + `uvm_fatal("READ", $sformatf( + "port %0d bank mismatch: got %0d exp %0d", item.port, item.bank_id, exp_bank)) + if (item.group_id !== exp_group[BB_BLINK_GROUP_ID_W-1:0]) + `uvm_fatal("READ", $sformatf( + "port %0d group mismatch: got %0d exp %0d", item.port, item.group_id, exp_group)) + if (item.addr !== exp_addr[BB_BLINK_BANK_ADDR_W-1:0]) + `uvm_fatal("READ", $sformatf( + "port %0d read %0d addr mismatch: got %0d exp %0d", + item.port, + read_count[item.port], + item.addr, + exp_addr + )) + read_count[item.port]++; + endfunction + + function void write_write(bb_blink_write_item item); + mxfp2int_cmd_item stim; + bit [127:0] bytes; + + stim = current_stim("WRITE"); + if (item.port != 0) + `uvm_fatal("WRITE", $sformatf("Mxfp2IntBall has one write port, got port %0d", item.port)) + if (write_count >= stim.write_addr.size()) + `uvm_fatal("WRITE", $sformatf("extra write %0d", write_count)) + verification_pkg::evidence::checks++; + if (item.bank_id !== stim.wr_bank) + `uvm_fatal("WRITE", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.wr_bank)) + if (item.group_id !== stim.wr_group) + `uvm_fatal("WRITE", $sformatf("group mismatch: got %0d exp %0d", item.group_id, stim.wr_group + )) + if (item.rob_id !== stim.rob_id) + `uvm_fatal("WRITE", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + if (item.addr !== stim.write_addr[write_count][BB_BLINK_BANK_ADDR_W-1:0]) + `uvm_fatal("WRITE", $sformatf( + "write %0d addr mismatch: got %0d exp %0d", + write_count, + item.addr, + stim.write_addr[write_count] + )) + if (item.mask !== stim.write_mask[write_count]) + `uvm_fatal("WRITE", $sformatf( + "write %0d mask mismatch: got 0x%04h exp 0x%04h", + write_count, + item.mask, + stim.write_mask[write_count] + )) + for (int b = 0; b < 16; b++) bytes[b*8+:8] = {8{item.mask[b]}}; + if ((item.data & bytes) !== (stim.write_data[write_count] & bytes)) + `uvm_fatal("SCB", $sformatf( + "write %0d data mismatch at addr %0d: got 0x%032h exp 0x%032h", + write_count, + item.addr, + item.data, + stim.write_data[write_count] + )) + write_count++; + endfunction + + function void write_resp(bb_blink_resp_item item); + mxfp2int_cmd_item stim; + stim = current_stim("RESP"); + verification_pkg::evidence::checks++; + if (item.rob_id !== stim.rob_id) + `uvm_fatal("RESP", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + if (!complete_io()) + `uvm_fatal("RESP", $sformatf( + "response before all traffic: reads=%0d,%0d writes=%0d", + read_count[0], + read_count[1], + write_count + )) + resp_count++; + endfunction + + function mxfp2int_cmd_item current_stim(string tag); + if (stim_q.size() == 0) `uvm_fatal("SCB", $sformatf("%s observed before stimulus", tag)) + return stim_q[0]; + endfunction + + function bit complete_io(); + mxfp2int_cmd_item stim = stim_q[0]; + return read_count[0] == stim.read_addr0.size() && read_count[1] == stim.read_addr1.size() && + write_count == stim.write_addr.size(); + endfunction + + function bit done(); + return stim_q.size() == 1 && cmd_count == 1 && resp_count == 1 && complete_io(); + endfunction + + function void reset_counters(); + stim_q.delete(); + cmd_count = 0; + read_count[0] = 0; + read_count[1] = 0; + write_count = 0; + resp_count = 0; + endfunction + + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (!done()) + `uvm_fatal("SCB", $sformatf( + "incomplete: stim=%0d cmds=%0d reads=%0d,%0d writes=%0d resp=%0d", + stim_q.size(), + cmd_count, + read_count[0], + read_count[1], + write_count, + resp_count + )) + stim_q.delete(); + endfunction +endclass diff --git a/examples/prefab/balls/mxfp2int/verify/src/pkg/mxfp2int_pkg.sv b/examples/prefab/balls/mxfp2int/verify/src/pkg/mxfp2int_pkg.sv new file mode 100644 index 000000000..b29568d25 --- /dev/null +++ b/examples/prefab/balls/mxfp2int/verify/src/pkg/mxfp2int_pkg.sv @@ -0,0 +1,13 @@ +package mxfp2int_pkg; + import uvm_pkg::*; + import bb_uvm_pkg::*; + `include "uvm_macros.svh" + + `include "common/mxfp2int_defs.svh" + `include "common/mxfp2int_items.svh" + `include "seq/mxfp2int_sequences.svh" + `include "cov/mxfp2int_cov.svh" + `include "env/mxfp2int_scoreboard.svh" + `include "env/mxfp2int_env.svh" + `include "tests/mxfp2int_test.svh" +endpackage diff --git a/examples/prefab/balls/mxfp2int/verify/src/seq/mxfp2int_sequences.svh b/examples/prefab/balls/mxfp2int/verify/src/seq/mxfp2int_sequences.svh new file mode 100644 index 000000000..2affff9b6 --- /dev/null +++ b/examples/prefab/balls/mxfp2int/verify/src/seq/mxfp2int_sequences.svh @@ -0,0 +1,17 @@ +class mxfp2int_step_seq extends uvm_sequence #(bb_blink_cmd_item); + `uvm_object_utils(mxfp2int_step_seq) + + int unsigned step; + + function new(string name = "mxfp2int_step_seq"); + super.new(name); + endfunction + + task body(); + mxfp2int_cmd_item item; + item = mxfp2int_cmd_item::type_id::create("item"); + start_item(item); + item.load_rust_step(step); + finish_item(item); + endtask +endclass diff --git a/examples/prefab/balls/mxfp2int/verify/src/tb_top.sv b/examples/prefab/balls/mxfp2int/verify/src/tb_top.sv new file mode 100644 index 000000000..2c01c4374 --- /dev/null +++ b/examples/prefab/balls/mxfp2int/verify/src/tb_top.sv @@ -0,0 +1,87 @@ +module tb_top; + import uvm_pkg::*; + import mxfp2int_pkg::*; + + bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); + + Mxfp2IntBall dut ( + .clock(intf.clock), + .reset(intf.reset), + .blink_status_running(intf.status_running), + .blink_cmdReq_ready(intf.cmd_req_ready), + .blink_cmdReq_valid(intf.cmd_req_valid), + .blink_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .blink_cmdReq_bits_funct(intf.cmd_req_bits_funct), + .blink_cmdReq_bits_rs2(intf.cmd_req_bits_rs2), + .blink_cmdReq_bits_iter(intf.cmd_req_bits_iter), + .blink_cmdReq_bits_op1_bank(intf.cmd_req_bits_op1_bank), + .blink_cmdReq_bits_op2_bank(intf.cmd_req_bits_op2_bank), + .blink_cmdReq_bits_wr_bank(intf.cmd_req_bits_wr_bank), + .blink_cmdReq_bits_op1_group(intf.cmd_req_bits_op1_group), + .blink_cmdReq_bits_op2_group(intf.cmd_req_bits_op2_group), + .blink_cmdReq_bits_wr_group(intf.cmd_req_bits_wr_group), + .blink_cmdReq_bits_op1_col(intf.cmd_req_bits_op1_col), + .blink_cmdReq_bits_op2_col(intf.cmd_req_bits_op2_col), + .blink_cmdReq_bits_wr_col(intf.cmd_req_bits_wr_col), + .blink_cmdReq_bits_wr_read_bank(intf.cmd_req_bits_wr_read_bank), + .blink_cmdReq_bits_wr_arch_bank(intf.cmd_req_bits_wr_arch_bank), + .blink_cmdResp_ready(intf.cmd_resp_ready), + .blink_cmdResp_valid(intf.cmd_resp_valid), + .blink_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .blink_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .blink_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .blink_bankRead_0_group_id(intf.bank_read_group_id[0]), + .blink_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .blink_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .blink_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .blink_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .blink_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .blink_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .blink_bankRead_1_bank_id(intf.bank_read_bank_id[1]), + .blink_bankRead_1_rob_id(intf.bank_read_rob_id[1]), + .blink_bankRead_1_group_id(intf.bank_read_group_id[1]), + .blink_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), + .blink_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), + .blink_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), + .blink_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), + .blink_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), + .blink_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), + .blink_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .blink_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .blink_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .blink_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .blink_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .blink_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .blink_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .blink_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .blink_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .blink_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), + .blink_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), + .blink_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), + .blink_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), + .blink_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), + .blink_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), + .blink_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), + .blink_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), + .blink_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), + .blink_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), + .blink_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), + .blink_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), + .blink_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), + .blink_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), + .blink_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), + .blink_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), + .blink_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]) + ); + + initial begin + intf.clock = 1'b0; + intf.reset = 1'b1; + forever #5 intf.clock = ~intf.clock; + end + + initial begin + uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); + run_test("mxfp2int_ball_test"); + end +endmodule diff --git a/examples/prefab/balls/mxfp2int/verify/src/tests/mxfp2int_test.svh b/examples/prefab/balls/mxfp2int/verify/src/tests/mxfp2int_test.svh new file mode 100644 index 000000000..0101a6217 --- /dev/null +++ b/examples/prefab/balls/mxfp2int/verify/src/tests/mxfp2int_test.svh @@ -0,0 +1,56 @@ +class mxfp2int_ball_test extends uvm_test; + `uvm_component_utils(mxfp2int_ball_test) + + typedef virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) vif_t; + vif_t vif; + mxfp2int_env env; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(vif_t)::get(this, "", "vif", vif)) + `uvm_fatal("NOVIF", "bb_blink_if not found") + env = mxfp2int_env::type_id::create("env", this); + endfunction + + task run_phase(uvm_phase phase); + phase.raise_objection(this); + for (int unsigned index = 0; index < mxfp2int_num_cases(); index++) run_case(index); + phase.drop_objection(this); + endtask + + // Steps of one case run back to back on the same reset. + task run_case(int unsigned index); + int unsigned steps; + apply_reset(); + steps = mxfp2int_case_load(index); + for (int unsigned step = 0; step < steps; step++) run_step(index, step); + endtask + + task run_step(int unsigned index, int unsigned step); + mxfp2int_step_seq seq; + int cycles = 0; + env.scb.reset_counters(); + seq = mxfp2int_step_seq::type_id::create("seq"); + seq.step = step; + seq.start(env.cmd_agent.seqr); + while (!env.scb.done()) begin + @(posedge vif.clock); + cycles++; + if (cycles > MXFP2INT_TIMEOUT_CYCLES) + `uvm_fatal("TIMEOUT", $sformatf("Mxfp2IntBall case %0d step %0d timeout", index, step)) + end + `uvm_info("MXFP2INT", $sformatf("Mxfp2IntBall case %0d step %0d passed in %0d cycles", index, + step, cycles), UVM_LOW) + endtask + + task apply_reset(); + vif.reset = 1'b1; + repeat (5) @(posedge vif.clock); + vif.reset = 1'b0; + repeat (2) @(posedge vif.clock); + endtask +endclass diff --git a/examples/prefab/balls/mxfp2int/workloads/ctests/CMakeLists.txt b/examples/prefab/balls/mxfp2int/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..e7e94955e --- /dev/null +++ b/examples/prefab/balls/mxfp2int/workloads/ctests/CMakeLists.txt @@ -0,0 +1,2 @@ +add_buckyball_ctest(mxfp4_e8m0_two_block_test.c) +add_buckyball_ctest(mxfp4_e8m0_scale_range_test.c) diff --git a/examples/prefab/balls/mxfp2int/workloads/ctests/mxfp4_e8m0_scale_range_test.c b/examples/prefab/balls/mxfp2int/workloads/ctests/mxfp4_e8m0_scale_range_test.c new file mode 100644 index 000000000..d9b3c34c3 --- /dev/null +++ b/examples/prefab/balls/mxfp2int/workloads/ctests/mxfp4_e8m0_scale_range_test.c @@ -0,0 +1,72 @@ +#include "buckyball.h" +#include +#include +#include +#include +#include + +// 20 blocks, so the scales span two rows of the scale bank. The scales cover +// right shifts (rounding toward minus infinity), left shifts that saturate +// and shifts of more than 31 in both directions. +#define NUM_BLOCKS 20 +#define FP4_PER_BLOCK 32 +#define BYTES_PER_BLOCK 16 + +static uint8_t input_mxfp4[NUM_BLOCKS * BYTES_PER_BLOCK] + __attribute__((aligned(64))); +static uint8_t scales_e8m0[32] __attribute__((aligned(64))); +static int8_t output_int8[NUM_BLOCKS * FP4_PER_BLOCK] + __attribute__((aligned(64))); +static const uint8_t scale_of[NUM_BLOCKS] = {0, 1, 90, 120, 124, 125, 126, + 127, 128, 129, 130, 131, 132, 133, + 134, 150, 158, 159, 200, 255}; + +static int raw_fp4(int block, int elem) { return (elem * 7 + block * 3) & 0xF; } + +static int8_t golden(int block, int elem) { + int value = raw_fp4(block, elem) - 8; + int shift = scale_of[block] - 127; + long scaled; + if (shift >= 0) + scaled = (long)value << (shift > 8 ? 8 : shift); + else + scaled = value >> (-shift > 31 ? 31 : -shift); + if (scaled > 127) + scaled = 127; + if (scaled < -128) + scaled = -128; + return (int8_t)scaled; +} + +int main(void) { + for (int b = 0; b < NUM_BLOCKS; b++) { + for (int e = 0; e < FP4_PER_BLOCK; e += 2) + input_mxfp4[b * BYTES_PER_BLOCK + e / 2] = + (uint8_t)((raw_fp4(b, e + 1) << 4) | raw_fp4(b, e)); + scales_e8m0[b] = scale_of[b]; + } + + bb_mem_alloc(1, 1, 1); + bb_mem_alloc(2, 1, 1); + bb_mem_alloc(3, 1, 1); + bb_mvin((uintptr_t)input_mxfp4, 1, NUM_BLOCKS, 1); + bb_mvin((uintptr_t)scales_e8m0, 2, 2, 1); + bb_mxfp2int(1, 2, 3, NUM_BLOCKS); + bb_mvout((uintptr_t)output_int8, 3, NUM_BLOCKS * 2, 1); + bb_mem_release(1); + bb_mem_release(2); + bb_mem_release(3); + + for (int b = 0; b < NUM_BLOCKS; b++) + for (int e = 0; e < FP4_PER_BLOCK; e++) { + int8_t got = output_int8[b * FP4_PER_BLOCK + e]; + int8_t exp = golden(b, e); + if (got != exp) { + printf("mxfp4 scale range mismatch b=%d e=%d got=%d exp=%d\n", b, e, + (int)got, (int)exp); + return 1; + } + } + printf("mxfp4 e8m0 scale range PASSED\n"); + return 0; +} diff --git a/examples/prefab/balls/mxfp2int/workloads/ctests/mxfp4_e8m0_two_block_test.c b/examples/prefab/balls/mxfp2int/workloads/ctests/mxfp4_e8m0_two_block_test.c new file mode 100644 index 000000000..cf2e5db7e --- /dev/null +++ b/examples/prefab/balls/mxfp2int/workloads/ctests/mxfp4_e8m0_two_block_test.c @@ -0,0 +1,72 @@ +#include "buckyball.h" +#include +#include +#include +#include +#include + +#define NUM_BLOCKS 2 +#define FP4_PER_BLOCK 32 +#define BYTES_PER_BLOCK 16 +#define E8M0_BIAS 127 + +static uint8_t input_mxfp4[NUM_BLOCKS * BYTES_PER_BLOCK] + __attribute__((aligned(64))); +static uint8_t scales_e8m0[16] __attribute__((aligned(64))); +static int8_t output_int8[NUM_BLOCKS * FP4_PER_BLOCK] + __attribute__((aligned(64))); + +static uint8_t gen_fp4(int block, int elem) { + return (uint8_t)((elem + block) & 0x0F); +} + +static int8_t golden(int block, int elem) { + int val = (int)gen_fp4(block, elem) - 8; + int prod = val << block; /* scale = bias + block */ + if (prod > 127) + prod = 127; + if (prod < -128) + prod = -128; + return (int8_t)prod; +} + +int main(void) { + for (int b = 0; b < NUM_BLOCKS; b++) { + for (int e = 0; e < FP4_PER_BLOCK; e += 2) { + uint8_t lo = gen_fp4(b, e); + uint8_t hi = gen_fp4(b, e + 1); + input_mxfp4[b * BYTES_PER_BLOCK + e / 2] = + (uint8_t)((hi << 4) | (lo & 0x0F)); + } + scales_e8m0[b] = (uint8_t)(E8M0_BIAS + b); + } + + const uint32_t in_bank = 1; + const uint32_t scale_bank = 2; + const uint32_t out_bank = 3; + bb_mem_alloc(in_bank, 1, 1); + bb_mem_alloc(scale_bank, 1, 1); + bb_mem_alloc(out_bank, 1, 1); + bb_mvin((uintptr_t)input_mxfp4, in_bank, NUM_BLOCKS, 1); + bb_mvin((uintptr_t)scales_e8m0, scale_bank, 1, 1); + bb_mxfp2int(in_bank, scale_bank, out_bank, NUM_BLOCKS); + bb_mvout((uintptr_t)output_int8, out_bank, NUM_BLOCKS * 2, 1); + + int passed = 1; + for (int b = 0; b < NUM_BLOCKS; b++) { + for (int e = 0; e < FP4_PER_BLOCK; e++) { + int8_t got = output_int8[b * FP4_PER_BLOCK + e]; + int8_t exp = golden(b, e); + if (got != exp) { + printf("MISMATCH b=%d e=%d got=%d exp=%d\n", b, e, (int)got, (int)exp); + passed = 0; + } + } + } + + bb_mem_release(in_bank); + bb_mem_release(scale_bank); + bb_mem_release(out_bank); + printf("mxfp4 e8m0 two-block %s\n", passed ? "PASSED" : "FAILED"); + return passed ? 0 : 1; +} diff --git a/examples/prefab/balls/mxfp2int/workloads/isa/mxfp2int.h b/examples/prefab/balls/mxfp2int/workloads/isa/mxfp2int.h new file mode 100644 index 000000000..76687e45d --- /dev/null +++ b/examples/prefab/balls/mxfp2int/workloads/isa/mxfp2int.h @@ -0,0 +1,21 @@ +#ifndef _BB_MXFP2INT_H_ +#define _BB_MXFP2INT_H_ + +#include +#include + +// Dequantizes iter MXFP4 blocks of 32 elements to int8. +// op1 bank: block b in row b, element e in byte e / 2, low nibble first; +// the element value is raw - 8. +// op2 bank: E8M0 scale of block b in byte b % 16 of row b / 16. +// wr bank: elements 0..15 of block b in row 2b, 16..31 in row 2b + 1, +// saturate(value * 2^(scale - 127)) with the right shift +// rounding toward minus infinity. +// iter: the number of blocks, 1..half the bank depth. rs2 is reserved and +// zero. +#define bb_mxfp2int(bank_id, scale_bank_id, wr_bank_id, iter) \ + BUCKYBALL_INSTRUCTION_R_R((BB_BANK0(bank_id) | BB_BANK1(scale_bank_id) | \ + BB_BANK2(wr_bank_id) | BB_ITER(iter)), \ + 0, FUNCT7(mxfp2int, MXFP2INT)) + +#endif // _BB_MXFP2INT_H_ diff --git a/examples/balls/mxfp2int/workloads/mlir_tests/CMakeLists.txt b/examples/prefab/balls/mxfp2int/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/balls/mxfp2int/workloads/mlir_tests/CMakeLists.txt rename to examples/prefab/balls/mxfp2int/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/prefab/balls/mxfp2int/workloads/mlir_tests/ball/CMakeLists.txt b/examples/prefab/balls/mxfp2int/workloads/mlir_tests/ball/CMakeLists.txt new file mode 100644 index 000000000..93a1280da --- /dev/null +++ b/examples/prefab/balls/mxfp2int/workloads/mlir_tests/ball/CMakeLists.txt @@ -0,0 +1,4 @@ +set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) +set(BUCKYBALL_MLIR_TEST_PREFIX ball) + +add_buckyball_mlir_test(mxfp4_i8_2blocks TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/prefab/balls/mxfp2int/workloads/mlir_tests/ball/mxfp4_i8_2blocks.mlir b/examples/prefab/balls/mxfp2int/workloads/mlir_tests/ball/mxfp4_i8_2blocks.mlir new file mode 100644 index 000000000..1f75ffa8b --- /dev/null +++ b/examples/prefab/balls/mxfp2int/workloads/mlir_tests/ball/mxfp4_i8_2blocks.mlir @@ -0,0 +1,28 @@ +func.func private @check_result(memref<4x16xi8>) -> () +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + // 0x99: both nibbles 9, the value 1. + %raw = arith.constant -103 : i8 + %scale = arith.constant 127 : i8 + %input = memref.alloc() : memref<2x16xi8> + %scales = memref.alloc() : memref<1x16xi8> + %output = memref.alloc() : memref<4x16xi8> + linalg.fill ins(%raw : i8) outs(%input : memref<2x16xi8>) + linalg.fill ins(%scale : i8) outs(%scales : memref<1x16xi8>) + linalg.fill ins(%zero_i8 : i8) outs(%output : memref<4x16xi8>) + buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %input <{bank = 1 : i64, depth = 2 : i64, stride = 1 : i64}> : memref<2x16xi8> + buckyball.mvin %scales <{bank = 2 : i64, depth = 1 : i64, stride = 1 : i64}> : memref<1x16xi8> + mxfp2int_ball.mxfp4_i8 <{data = 1 : i64, scales = 2 : i64, output = 3 : i64, rows = 2 : i64}> + buckyball.mvout %output <{bank = 3 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> + buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + func.call @check_result(%output) : (memref<4x16xi8>) -> () + memref.dealloc %input : memref<2x16xi8> + memref.dealloc %scales : memref<1x16xi8> + memref.dealloc %output : memref<4x16xi8> + return %zero_i8 : i8 +} diff --git a/examples/prefab/balls/mxfp2int/workloads/mlir_tests/ball/mxfp4_i8_2blocks_main.cpp b/examples/prefab/balls/mxfp2int/workloads/mlir_tests/ball/mxfp4_i8_2blocks_main.cpp new file mode 100644 index 000000000..bafe2eb2c --- /dev/null +++ b/examples/prefab/balls/mxfp2int/workloads/mlir_tests/ball/mxfp4_i8_2blocks_main.cpp @@ -0,0 +1,16 @@ +#include +#include +#include + +extern "C" void check_result(int8_t *, int8_t *data, int64_t offset, + int64_t rows, int64_t lanes, int64_t stride, + int64_t lane_stride) { + if (rows != 4 || lanes != 16 || stride != 16 || lane_stride != 1) + exit(1); + for (int i = 0; i < rows * lanes; ++i) + if (data[offset + i] != 1) { + printf("mxfp4_i8 unit-scale mismatch index=%d\n", i); + exit(1); + } + printf("mxfp4_i8 2-block PASS\n"); +} diff --git a/examples/prefab/balls/mxfp2int/workloads/mlir_tests/bank/CMakeLists.txt b/examples/prefab/balls/mxfp2int/workloads/mlir_tests/bank/CMakeLists.txt new file mode 100644 index 000000000..580170347 --- /dev/null +++ b/examples/prefab/balls/mxfp2int/workloads/mlir_tests/bank/CMakeLists.txt @@ -0,0 +1,8 @@ +set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) +set(BUCKYBALL_MLIR_TEST_PREFIX bank) + +set(_lines ${DESIGN_PROFILES_${BUCKYBALL_MLIR_ACTIVE_TARGET}_THREAD_MEM_BANK_ENTRIES}) +# mxfp4_i8 writes two bank rows per block; one scale row holds sixteen blocks. +math(EXPR BB_MXFP_BLOCKS "${_lines} / 2") +math(EXPR BB_MXFP_SCALE_ROWS "(${BB_MXFP_BLOCKS} + 15) / 16") +add_buckyball_mlir_test(mxfp4_i8_bank_blocks TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/prefab/balls/mxfp2int/workloads/mlir_tests/bank/mxfp4_i8_bank_blocks.mlir.in b/examples/prefab/balls/mxfp2int/workloads/mlir_tests/bank/mxfp4_i8_bank_blocks.mlir.in new file mode 100644 index 000000000..1a4fe0a99 --- /dev/null +++ b/examples/prefab/balls/mxfp2int/workloads/mlir_tests/bank/mxfp4_i8_bank_blocks.mlir.in @@ -0,0 +1,59 @@ +func.func private @check_result(memref<@BB_BANK_LINES@x16xi8>) -> () +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %blocks = arith.constant @BB_MXFP_BLOCKS@ : index + %sixteen = arith.constant 16 : index + %scaleRows = arith.constant @BB_MXFP_SCALE_ROWS@ : index + %bias = arith.constant 125 : index + %eight = arith.constant 8 : index + %input = memref.alloc() alignment = 64 : memref<@BB_MXFP_BLOCKS@x16xi8> + %scales = memref.alloc() alignment = 64 : memref<@BB_MXFP_SCALE_ROWS@x16xi8> + %output = memref.alloc() alignment = 64 : memref<@BB_BANK_LINES@x16xi8> + // Byte j of block b holds the nibbles (b + 2j) % 16 (low) and + // (b + 2j + 1) % 16 (high); element e has the raw value (b + e) % 16. + scf.for %b = %zero to %blocks step %one { + scf.for %j = %zero to %sixteen step %one { + %j2 = arith.addi %j, %j : index + %e0 = arith.addi %b, %j2 : index + %lo = arith.remui %e0, %sixteen : index + %e1 = arith.addi %e0, %one : index + %hi = arith.remui %e1, %sixteen : index + %hiShifted = arith.muli %hi, %sixteen : index + %byte = arith.addi %hiShifted, %lo : index + %v = arith.index_cast %byte : index to i8 + memref.store %v, %input[%b, %j] : memref<@BB_MXFP_BLOCKS@x16xi8> + } + } + // The scale of block b is 125 + b % 8: shifts -2..5. + scf.for %r = %zero to %scaleRows step %one { + scf.for %j = %zero to %sixteen step %one { + %r16 = arith.muli %r, %sixteen : index + %b = arith.addi %r16, %j : index + %m = arith.remui %b, %eight : index + %s = arith.addi %bias, %m : index + %v = arith.index_cast %s : index to i8 + memref.store %v, %scales[%r, %j] : memref<@BB_MXFP_SCALE_ROWS@x16xi8> + } + } + %ib = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %sb = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %ob = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %il = buckyball.bank_mvin %input, %ib <{depth = @BB_MXFP_BLOCKS@ : i64, stride = 1 : i64}> + : memref<@BB_MXFP_BLOCKS@x16xi8> + %sl = buckyball.bank_mvin %scales, %sb <{depth = @BB_MXFP_SCALE_ROWS@ : i64, stride = 1 : i64}> + : memref<@BB_MXFP_SCALE_ROWS@x16xi8> + %result = mxfp2int_ball.bank_mxfp4_i8 %il, %sl -> %ob + <{rows = @BB_MXFP_BLOCKS@ : i64}> + %stored = buckyball.bank_mvout %output, %result <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> + : memref<@BB_BANK_LINES@x16xi8> + buckyball.bank_release %il + buckyball.bank_release %sl + buckyball.bank_release %stored + func.call @check_result(%output) : (memref<@BB_BANK_LINES@x16xi8>) -> () + memref.dealloc %input : memref<@BB_MXFP_BLOCKS@x16xi8> + memref.dealloc %scales : memref<@BB_MXFP_SCALE_ROWS@x16xi8> + memref.dealloc %output : memref<@BB_BANK_LINES@x16xi8> + return %zero_i8 : i8 +} diff --git a/examples/prefab/balls/mxfp2int/workloads/mlir_tests/bank/mxfp4_i8_bank_blocks_main.cpp b/examples/prefab/balls/mxfp2int/workloads/mlir_tests/bank/mxfp4_i8_bank_blocks_main.cpp new file mode 100644 index 000000000..98d2b9de9 --- /dev/null +++ b/examples/prefab/balls/mxfp2int/workloads/mlir_tests/bank/mxfp4_i8_bank_blocks_main.cpp @@ -0,0 +1,28 @@ +#include +#include +#include + +extern "C" void check_result(int8_t *, int8_t *data, int64_t offset, + int64_t rows, int64_t lanes, int64_t stride, + int64_t lane_stride) { + if (rows != THREAD_MEM_BANK_ENTRIES || lanes != 16 || stride != 16 || + lane_stride != 1) + exit(1); + for (int block = 0; block < THREAD_MEM_BANK_ENTRIES / 2; ++block) + for (int element = 0; element < 32; ++element) { + int value = (block + element) % 16 - 8; + int shift = block % 8 - 2; + int expected = shift >= 0 ? value << shift : value >> -shift; + if (expected > 127) + expected = 127; + if (expected < -128) + expected = -128; + if (data[offset + block * 32 + element] != expected) { + printf("mxfp4_i8 bank mismatch block=%d element=%d got=%d " + "expected=%d\n", + block, element, data[offset + block * 32 + element], expected); + exit(1); + } + } + printf("mxfp4_i8 bank PASS blocks=%d\n", THREAD_MEM_BANK_ENTRIES / 2); +} diff --git a/examples/balls/mxmm/arch/src/main/scala/AddPipe.scala b/examples/prefab/balls/mxmm/arch/AddPipe.scala similarity index 95% rename from examples/balls/mxmm/arch/src/main/scala/AddPipe.scala rename to examples/prefab/balls/mxmm/arch/AddPipe.scala index d49ffea94..8688abfed 100644 --- a/examples/balls/mxmm/arch/src/main/scala/AddPipe.scala +++ b/examples/prefab/balls/mxmm/arch/AddPipe.scala @@ -1,5 +1,7 @@ package examples.balls.mxmm +import chisel3.experimental.hierarchy.{Instance, Instantiate} + import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public} diff --git a/examples/prefab/balls/mxmm/arch/Array.scala b/examples/prefab/balls/mxmm/arch/Array.scala new file mode 100644 index 000000000..8e9f9f8e7 --- /dev/null +++ b/examples/prefab/balls/mxmm/arch/Array.scala @@ -0,0 +1,53 @@ +package examples.balls.mxmm + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import examples.balls.mxmm.configs.MxmmBallParam +import hardfloat._ + +@instantiable +class Array(val ballParam: MxmmBallParam) extends Module { + val p = ballParam + + @public + val io = IO(new Bundle { + val valid = Input(Bool()) + val context = Input(UInt(log2Ceil(p.contexts).W)) + val separate = Input(Bool()) + val vector = Input(Bool()) + val a = Input(Vec(p.tileRows, UInt(32.W))) + val b = Input(Vec(p.tileCols, UInt(32.W))) + val load = Input(Bool()) + val row = Input(UInt(4.W)) + val rowData = Input(UInt(512.W)) + val rowOut = Output(UInt(512.W)) + val completed = Output(Bool()) + }) + + val accumulators = Reg(Vec(p.contexts, Vec(p.tileRows, Vec(p.tileCols, UInt(33.W))))) + when(io.load) { + for (col <- 0 until p.tileCols) { + accumulators(io.context)(io.row)(col) := recFNFromFN(8, 24, io.rowData(32 * col + 31, 32 * col)) + } + } + io.rowOut := Cat((0 until p.tileCols).reverse.map(col => fNFromRecFN(8, 24, accumulators(io.context)(io.row)(col)))) + val fusedTag = Pipe(io.valid && !io.separate, io.context, p.arithmeticLatency) + val separateTag = Pipe(io.valid && io.separate, io.context, 2 * p.arithmeticLatency) + io.completed := fusedTag.valid || separateTag.valid + for { + row <- 0 until p.tileRows + col <- 0 until p.tileCols + } { + val pe = + Instantiate(new PE(p.arithmeticLatency, ballParam)) + pe.io.valid := io.valid && (if (row == 0) true.B else !io.vector) + pe.io.separate := io.separate + pe.io.a := io.a(row) + pe.io.b := io.b(col) + pe.io.c := accumulators(io.context)(row)(col) + when(pe.io.outValid) { + accumulators(Mux(separateTag.valid, separateTag.bits, fusedTag.bits))(row)(col) := pe.io.out + } + } +} diff --git a/examples/prefab/balls/mxmm/arch/InputWindow.scala b/examples/prefab/balls/mxmm/arch/InputWindow.scala new file mode 100644 index 000000000..3e2435026 --- /dev/null +++ b/examples/prefab/balls/mxmm/arch/InputWindow.scala @@ -0,0 +1,91 @@ +package examples.balls.mxmm + +import chisel3.experimental.hierarchy.{instantiable, public} + +import framework.acc.memdomain.configs.MemDomainParam + +import examples.balls.mxmm.configs.MxmmBallParam + +import framework.ip.memory.bank.{SramReadIO, SramReadReq, SramReadResp, SramWriteIO, SramWriteReq, SramWriteResp} + +import chisel3._ +import chisel3.util._ +import framework.ip.memory.bank.{SramReadReq, SramReadResp} + +@instantiable +class InputWindow(memParam: MemDomainParam, val ballParam: MxmmBallParam) extends Module { + val addressWidth = log2Ceil(memParam.bankEntries) + + @public + val io = IO(new Bundle { + val start = Input(Bool()) + val enable = Input(Bool()) + val rows = Input(UInt(12.W)) + val count = Input(UInt(34.W)) + val fullK = Input(UInt(16.W)) + val startK = Input(UInt(16.W)) + val request = Decoupled(new SramReadReq(memParam)) + val response = Flipped(Decoupled(new SramReadResp(memParam))) + val done = Output(Bool()) + val write = Output(Bool()) + val scaleWrite = Output(Bool()) + val lane = Output(UInt(4.W)) + val address = Output(UInt(addressWidth.W)) + val word = Output(UInt(128.W)) + val scale = Output(UInt(8.W)) + }) + + val request :: response :: scales :: done :: Nil = Enum(4) + val state = RegInit(done) + val rows = Reg(UInt(12.W)) + val count = Reg(UInt(34.W)) + val fullK = Reg(UInt(16.W)) + val startK = Reg(UInt(16.W)) + val row = Reg(UInt(12.W)) + val offset = Reg(UInt(addressWidth.W)) + val scaleMode = Reg(Bool()) + val word = Reg(UInt(128.W)) + val scaleAddress = rows * fullK + row * (fullK >> 5) + (startK >> 5) + offset + val byte = scaleAddress(3, 0) + io.request.valid := io.enable && state === request + io.request.bits.addr := Mux(scaleMode, scaleAddress >> 4, row * (fullK >> 4) + (startK >> 4) + offset) + io.response.ready := io.enable && state === response + io.done := state === done + io.write := io.response.fire && !scaleMode + io.scaleWrite := io.enable && state === scales + io.lane := row(3, 0) + io.address := (row >> 4) * Mux(scaleMode, count >> 5, count >> 4) + offset + io.word := io.response.bits.data + io.scale := (word >> (byte << 3))(7, 0) + when(io.start) { + rows := io.rows + count := io.count + fullK := io.fullK + startK := io.startK + row := 0.U + offset := 0.U + scaleMode := false.B + state := request + } + when(io.request.fire)(state := response) + when(io.response.fire) { + when(scaleMode) { word := io.response.bits.data; state := scales }.otherwise { + when(offset + 1.U === (count >> 4)) { + offset := 0.U + when(row + 1.U === rows) { row := 0.U; scaleMode := true.B } + .otherwise(row := row + 1.U) + }.otherwise(offset := offset + 1.U) + state := request + } + } + when(io.enable && state === scales) { + when(offset + 1.U === (count >> 5)) { + offset := 0.U + when(row + 1.U === rows)(state := done) + .otherwise { row := row + 1.U; state := request } + }.otherwise { + offset := offset + 1.U + when(byte === 15.U)(state := request) + } + } +} diff --git a/examples/balls/mxmm/arch/src/main/scala/Mxfp8Decode.scala b/examples/prefab/balls/mxmm/arch/Mxfp8Decode.scala similarity index 100% rename from examples/balls/mxmm/arch/src/main/scala/Mxfp8Decode.scala rename to examples/prefab/balls/mxmm/arch/Mxfp8Decode.scala diff --git a/examples/prefab/balls/mxmm/arch/MxmmBall.scala b/examples/prefab/balls/mxmm/arch/MxmmBall.scala new file mode 100644 index 000000000..758eaab41 --- /dev/null +++ b/examples/prefab/balls/mxmm/arch/MxmmBall.scala @@ -0,0 +1,49 @@ +package examples.balls.mxmm + +import framework.acc.balldomain.bbus.{BallFactory, BallPorts} +import buckyball.balls.mxmm.{Params => ParametersProto} +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import examples.balls.mxmm.configs.MxmmBallParam + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import framework.acc.balldomain.ball.blink.{BallStatus, BlinkIO, HasBlink} + +@instantiable +class MxmmBall( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Module + with HasBlink { + val ball = bp.balls.find(_.id == id).get + + val ballParam = MxmmBallParam.fromProto(ball.parameters.unpack(classOf[ParametersProto])) + + @public + val blink = IO(new BlinkIO(fp, mp, bp, id)) + + val unit = + Instantiate(new Unit(fp, mp, bp, ball, ballParam)) + unit.io.cmdReq <> blink.cmdReq + unit.io.cmdResp <> blink.cmdResp + unit.io.bankRead <> blink.bankRead + unit.io.bankWrite <> blink.bankWrite + blink.status <> unit.io.status +} + +object MxmmBall extends BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts = + BallPorts(Instantiate(new MxmmBall(fp, mp, bp, id)).blink, None) + +} diff --git a/examples/prefab/balls/mxmm/arch/PE.scala b/examples/prefab/balls/mxmm/arch/PE.scala new file mode 100644 index 000000000..48c500369 --- /dev/null +++ b/examples/prefab/balls/mxmm/arch/PE.scala @@ -0,0 +1,52 @@ +package examples.balls.mxmm + +import examples.balls.mxmm.configs.MxmmBallParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import framework.lib.arith.Fma +import hardfloat._ + +@instantiable +class PE(val latency: Int, val ballParam: MxmmBallParam) extends Module { + + require(latency == 3) + + @public + val io = IO(new Bundle { + val valid = Input(Bool()) + val separate = Input(Bool()) + val a = Input(UInt(32.W)) + val b = Input(UInt(32.W)) + val c = Input(UInt(33.W)) + val out = Output(UInt(33.W)) + val outValid = Output(Bool()) + }) + + val multiply = Instantiate(new Fma(latency, 8, 24)) + + multiply.io.validin := io.valid + multiply.io.op := 0.U + multiply.io.a := recFNFromFN(8, 24, io.a) + multiply.io.b := recFNFromFN(8, 24, io.b) + multiply.io.c := Mux(io.separate, Cat(io.a(31) ^ io.b(31), 0.U(32.W)), io.c) + multiply.io.roundingMode := consts.round_near_even + multiply.io.detectTininess := consts.tininess_afterRounding.asUInt + + val productMode = Pipe(io.valid, io.separate, latency) + val previous = Pipe(io.valid && io.separate, io.c, latency) + val add = Instantiate(new AddPipe) + add.io.valid := multiply.io.validout && productMode.bits + add.io.a := multiply.io.out + add.io.b := previous.bits + when(add.io.valid) { + assert(fNFromRecFN(8, 24, multiply.io.out)(30, 23) =/= 255.U, "MXMM_F32 product must remain finite") + } + val fusedValid = multiply.io.validout && !productMode.bits + io.outValid := fusedValid || add.io.outValid + io.out := Mux(add.io.outValid, add.io.out, multiply.io.out) + when(io.outValid) { + assert(fNFromRecFN(8, 24, io.out)(30, 23) =/= 255.U, "MATMUL accumulation must remain finite") + } +} diff --git a/examples/balls/mxmm/arch/src/main/scala/Panels.scala b/examples/prefab/balls/mxmm/arch/Panels.scala similarity index 95% rename from examples/balls/mxmm/arch/src/main/scala/Panels.scala rename to examples/prefab/balls/mxmm/arch/Panels.scala index 49e4a168c..fb09f862c 100644 --- a/examples/balls/mxmm/arch/src/main/scala/Panels.scala +++ b/examples/prefab/balls/mxmm/arch/Panels.scala @@ -1,11 +1,13 @@ package examples.balls.mxmm +import examples.balls.mxmm.configs.MxmmBallParam + import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} @instantiable -class Panels(bankEntries: Int) extends Module { +class Panels(val bankEntries: Int, val ballParam: MxmmBallParam) extends Module { @public val io = IO(new Bundle { diff --git a/examples/prefab/balls/mxmm/arch/Unit.scala b/examples/prefab/balls/mxmm/arch/Unit.scala new file mode 100644 index 000000000..9c8d74b7b --- /dev/null +++ b/examples/prefab/balls/mxmm/arch/Unit.scala @@ -0,0 +1,411 @@ +package examples.balls.mxmm + +import framework.acc.balldomain.configs.BallRouteParam + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import framework.acc.balldomain.ball.blink.{BallCommand, BallResponse} +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import framework.acc.balldomain.ball.blink.BallStatus +import framework.acc.memdomain.{BankRead, BankWrite} +import examples.balls.mxmm.configs.MxmmBallParam + +@instantiable +class Unit( + frontendParam: FrontendParam, + memParam: MemDomainParam, + routingParam: BallDomainParam, + val instance: BallRouteParam, + val ballParam: MxmmBallParam) + extends Module { + val commandDataBits = frontendParam.commandDataBits + val p = ballParam + val entries = memParam.bankEntries + val addressWidth = log2Ceil(entries) + require( + memParam.bankWidth == 128 && memParam.bankMaskLen == 16 + ) + require(entries >= 64 && entries % 32 == 0) + val mapping = instance + val mxFunct = instance.instructions.find(_.mnemonic == "MXMM_MXFP8").get.funct7 + val windowFunct = instance.instructions.find(_.mnemonic == "MXMM_MXFP8_WINDOW").get.funct7 + val fmaFunct = instance.instructions.find(_.mnemonic == "MXMM_FMA32").get.funct7 + val f32Funct = instance.instructions.find(_.mnemonic == "MXMM_F32").get.funct7 + require(mapping.inBW == 2 && mapping.outBW == 1) + + @public + val io = IO(new Bundle { + + val cmdReq = Flipped(Decoupled(new BallCommand(frontendParam, memParam))) + + val cmdResp = Decoupled(new BallResponse(frontendParam)) + + val bankRead = Vec( + 2, + Flipped(new BankRead( + frontendParam, + memParam + )) + ) + + val bankWrite = Vec( + 1, + Flipped(new BankWrite( + frontendParam, + memParam + )) + ) + + val status = new BallStatus + }) + + val Seq( + idle, + waitChannels, + loadInputs, + initRequest, + initResponse, + compute, + drain, + store, + outputRequest, + outputResponse, + outputWrite, + outputAck, + complete + ) = Enum(13) + + val state = RegInit(idle) + + val command = Reg(new BallCommand(frontendParam, memParam)) + + val m = Reg(UInt(12.W)) + val n = Reg(UInt(12.W)) + val k = Reg(UInt(34.W)) + val mx = Reg(Bool()) + val windowed = Reg(Bool()) + val separate = Reg(Bool()) + val first = Reg(Bool()) + val last = Reg(Bool()) + val base = Reg(UInt(6.W)) + val chain = RegInit(false.B) + val chainM = Reg(UInt(12.W)) + val chainN = Reg(UInt(12.W)) + val chainFormat = Reg(UInt(7.W)) + val chainBank = Reg(UInt(frontendParam.bank_id_len.W)) + val chainBase = Reg(UInt(6.W)) + + val nGroups = n >> 4 + val panelHeight = Mux(m === 1.U, 1.U, 16.U) + val totalPanels = Mux(m === 1.U, 1.U, m >> 4) * nGroups + val batch = Reg(UInt(addressWidth.W)) + val count = Mux(totalPanels - batch > p.contexts.U, p.contexts.U, totalPanels - batch) + val minimumPeriod = Mux(separate, (2 * p.arithmeticLatency + 1).U, (p.arithmeticLatency + 1).U) + val period = Mux(count > minimumPeriod, count, minimumPeriod) + val context = Reg(UInt(log2Ceil(p.contexts).W)) + val row = Reg(UInt(4.W)) + val reduction = Reg(UInt(34.W)) + val completed = Reg(UInt(48.W)) + val panel = batch + context + val rowGroup = panel / nGroups + val columnGroup = panel % nGroups + val accAddress = rowGroup * 16.U * nGroups + row * nGroups + columnGroup + val accumulator = SyncReadMem(entries / 4, UInt(512.W)) + val savedRow = Reg(UInt(512.W)) + val outputLine = Reg(UInt(addressWidth.W)) + val outputQuarter = Reg(UInt(2.W)) + + val panels = + Instantiate(new Panels(entries, ballParam)) + + val array = Instantiate(new Array(ballParam)) + val readValid = state === compute && context < count + panels.io.read := readValid + panels.io.rowGroup := rowGroup + panels.io.columnGroup := columnGroup + panels.io.k := reduction + panels.io.reduction := k + panels.io.mxfp8 := mx + panels.io.vector := m === 1.U + array.io.valid := panels.io.valid + array.io.a := panels.io.a + array.io.b := panels.io.b + array.io.separate := separate + array.io.vector := m === 1.U + val computeContext = RegNext(context) + array.io.context := Mux(state === compute || state === drain, computeContext, context) + array.io.row := row + array.io.load := false.B + array.io.rowData := 0.U + + val accReadEnable = state === outputRequest || (state === initRequest && !first && row < panelHeight) + val accWriteEnable = state === store && row < panelHeight + + val accRead = accumulator.readWrite( + Mux(state === outputRequest, outputLine, accAddress), + array.io.rowOut, + accReadEnable || accWriteEnable, + accWriteEnable + ) + + assert(!(accReadEnable && accWriteEnable)) + + val inputState = Reg(Vec(2, UInt(2.W))) + val inputWord = Reg(Vec(2, UInt(128.W))) + val inputIndex = Reg(Vec(2, UInt(addressWidth.W))) + val inputRow = Reg(Vec(2, UInt(12.W))) + val rowOffset = Reg(Vec(2, UInt(addressWidth.W))) + val scaleMode = Reg(Vec(2, Bool())) + val scaleByte = Reg(Vec(2, UInt(4.W))) + val scaleCount = Reg(Vec(2, UInt((addressWidth + 4).W))) + val rowWords = Mux(mx, k >> 4, k >> 2) + val window = Instantiate(new InputWindow(memParam, ballParam)) + window.io.start := io.cmdReq.fire && io.cmdReq.bits.funct === windowFunct.U + window.io.enable := state === loadInputs && windowed + window.io.rows := io.cmdReq.bits.rs2(11, 0) + window.io.count := io.cmdReq.bits.iter + window.io.fullK := io.cmdReq.bits.rs2(47, 32) + window.io.startK := io.cmdReq.bits.rs2(63, 48) + window.io.request.ready := io.bankRead(0).io.req.ready && windowed + window.io.response.valid := io.bankRead(0).io.resp.valid && windowed + window.io.response.bits := io.bankRead(0).io.resp.bits + + io.cmdReq.ready := state === idle + io.cmdResp.valid := state === complete + io.cmdResp.bits.rob_id := command.rob_id + io.status.idle := state === idle + io.status.running := state =/= idle && state =/= complete + + for (operand <- 0 until 2) { + val rows = if (operand == 0) m else n + val codeWords = rows * rowWords + val scales = rows * (k >> 5) + val port = io.bankRead(operand) + port.rob_id := command.rob_id + port.group_id := (if (operand == 0) command.op1_group else command.op2_group) + port.bank_id := (if (operand == 0) command.op1_bank else command.op2_bank) + port.io.req.valid := state === loadInputs && inputState(operand) === 0.U + port.io.req.bits.addr := inputIndex(operand) + port.io.resp.ready := state === loadInputs && inputState(operand) === 1.U + panels.io.write(operand) := port.io.resp.fire && !scaleMode(operand) + panels.io.scaleWrite(operand) := state === loadInputs && inputState(operand) === 2.U + panels.io.lane(operand) := inputRow(operand)(3, 0) + panels.io.address(operand) := (inputRow(operand) >> 4) * Mux(scaleMode(operand), k >> 5, rowWords) + rowOffset( + operand + ) + panels.io.word(operand) := port.io.resp.bits.data + panels.io.scale(operand) := (inputWord(operand) >> (scaleByte(operand) << 3))(7, 0) + val legacy = if (operand == 0) !windowed else true.B + when(port.io.req.fire && legacy)(inputState(operand) := 1.U) + when(port.io.resp.fire && legacy) { + when(scaleMode(operand)) { + inputWord(operand) := port.io.resp.bits.data + scaleByte(operand) := 0.U + inputState(operand) := 2.U + }.otherwise { + inputIndex(operand) := inputIndex(operand) + 1.U + when(rowOffset(operand) + 1.U === rowWords) { + rowOffset(operand) := 0.U + inputRow(operand) := inputRow(operand) + 1.U + }.otherwise(rowOffset(operand) := rowOffset(operand) + 1.U) + when(inputIndex(operand) +& 1.U === codeWords) { + when(mx) { + scaleMode(operand) := true.B + inputRow(operand) := 0.U + rowOffset(operand) := 0.U + inputState(operand) := 0.U + }.otherwise(inputState(operand) := 3.U) + }.otherwise(inputState(operand) := 0.U) + } + } + when(state === loadInputs && inputState(operand) === 2.U && legacy) { + scaleCount(operand) := scaleCount(operand) + 1.U + scaleByte(operand) := scaleByte(operand) + 1.U + when(rowOffset(operand) + 1.U === (k >> 5)) { + rowOffset(operand) := 0.U + inputRow(operand) := inputRow(operand) + 1.U + }.otherwise(rowOffset(operand) := rowOffset(operand) + 1.U) + when(scaleCount(operand) + 1.U === scales) { + inputState(operand) := 3.U + }.elsewhen(scaleByte(operand) === 15.U) { + inputIndex(operand) := inputIndex(operand) + 1.U + inputState(operand) := 0.U + } + } + } + + when(windowed) { + io.bankRead(0).io.req.valid := window.io.request.valid + io.bankRead(0).io.req.bits.addr := window.io.request.bits.addr + io.bankRead(0).io.resp.ready := window.io.response.ready + panels.io.write(0) := window.io.write + panels.io.scaleWrite(0) := window.io.scaleWrite + panels.io.lane(0) := window.io.lane + panels.io.address(0) := window.io.address + panels.io.word(0) := window.io.word + panels.io.scale(0) := window.io.scale + } + + val writePort = io.bankWrite(0) + writePort.rob_id := command.rob_id + writePort.group_id := command.wr_group + writePort.bank_id := command.wr_bank + writePort.io.req.valid := state === outputWrite + writePort.io.req.bits.addr := base + (outputLine << 2) + outputQuarter + writePort.io.req.bits.data := (savedRow >> (outputQuarter << 7))(127, 0) + writePort.io.req.bits.mask := VecInit(Seq.fill(16)(true.B)) + writePort.io.resp.ready := state === outputAck + + when(io.cmdReq.fire) { + val cmd = io.cmdReq.bits + val rows = cmd.rs2(11, 0) + val cols = cmd.rs2(23, 12) + val inputK = cmd.iter + val isWindow = cmd.funct === windowFunct.U + val isMx = cmd.funct === mxFunct.U || isWindow + assert( + isMx || cmd.funct === fmaFunct.U || cmd.funct === f32Funct.U, + "MATMUL format is invalid" + ) + assert(rows === 1.U || (rows =/= 0.U && rows(3, 0) === 0.U), "MATMUL M must be one or a positive multiple of 16") + assert(cols =/= 0.U && cols(3, 0) === 0.U, "MATMUL N must be a positive multiple of 16") + assert(inputK =/= 0.U && Mux(isMx, inputK(4, 0) === 0.U, inputK(1, 0) === 0.U), "MATMUL K alignment is invalid") + when(isWindow) { + val full = cmd.rs2(47, 32) + val start = cmd.rs2(63, 48) + assert(full =/= 0.U && full(4, 0) === 0.U && start(4, 0) === 0.U, "MATMUL A window alignment is invalid") + assert(start +& inputK <= full, "MATMUL A window exceeds fullK") + }.otherwise(assert(cmd.rs2(63, 32) === 0.U, "MATMUL reserves rs2[63:32]")) + assert( + cmd.op1_bank =/= cmd.op2_bank && cmd.op1_bank =/= cmd.wr_bank && cmd.op2_bank =/= cmd.wr_bank, + "MATMUL requires distinct banks" + ) + assert( + cmd.op1_col === 1.U && cmd.op2_col === 1.U && cmd.wr_col === 1.U, + "MATMUL requires one physical bank per operand" + ) + val aElements = rows * Mux(isWindow, cmd.rs2(47, 32), inputK) + val bElements = cols * inputK + assert(Mux(isMx, aElements + (aElements >> 5), aElements << 2) <= (entries * 16).U, "MATMUL A exceeds bank capacity") + assert(Mux(isMx, bElements + (bElements >> 5), bElements << 2) <= (entries * 16).U, "MATMUL B exceeds bank capacity") + assert((cmd.rs2(31, 26) << 4) +& ((rows * cols) << 2) <= (entries * 16).U, "MATMUL C exceeds bank capacity") + when(cmd.rs2(24)) { + assert(!chain, "MATMUL first cannot replace a live chain") + chain := true.B + chainM := rows + chainN := cols + chainFormat := cmd.funct + chainBank := cmd.wr_bank + chainBase := cmd.rs2(31, 26) + }.otherwise { + assert( + chain && chainM === rows && chainN === cols && chainFormat === cmd.funct && chainBank === cmd.wr_bank && chainBase === cmd.rs2( + 31, + 26 + ), + "MATMUL continuation changed its chain" + ) + } + command := io.cmdReq.bits + m := rows + n := cols + k := inputK + mx := isMx + windowed := isWindow + separate := cmd.funct === f32Funct.U + first := cmd.rs2(24) + last := cmd.rs2(25) + base := cmd.rs2(31, 26) + batch := 0.U + context := 0.U + row := 0.U + reduction := 0.U + completed := 0.U + for (operand <- 0 until 2) { + inputState(operand) := 0.U + inputIndex(operand) := 0.U + inputRow(operand) := 0.U + rowOffset(operand) := 0.U + scaleMode(operand) := false.B + scaleCount(operand) := 0.U + } + state := waitChannels + } + + switch(state) { + is(waitChannels)(state := loadInputs) + is(loadInputs)( + when(Mux(windowed, window.io.done, inputState(0) === 3.U) && inputState(1) === 3.U)(state := initRequest) + ) + is(initRequest) { + when(first || row >= panelHeight) { + array.io.load := true.B + when(row === 15.U) { + row := 0.U + when(context + 1.U === count) { context := 0.U; state := compute } + .otherwise(context := context + 1.U) + }.otherwise(row := row + 1.U) + }.otherwise(state := initResponse) + } + is(initResponse) { + array.io.load := true.B + array.io.rowData := accRead + when(row === 15.U) { + row := 0.U + when(context + 1.U === count) { context := 0.U; state := compute } + .otherwise { context := context + 1.U; state := initRequest } + }.otherwise { row := row + 1.U; state := initRequest } + } + is(compute) { + when(context + 1.U === period) { + context := 0.U + when(reduction + 1.U === k)(state := drain) + .otherwise(reduction := reduction + 1.U) + }.otherwise(context := context + 1.U) + } + is(store) { + when(row === 15.U) { + row := 0.U + when(context + 1.U === count) { + context := 0.U + when(batch + count === totalPanels) { + when(last) { outputLine := 0.U; outputQuarter := 0.U; state := outputRequest } + .otherwise(state := complete) + }.otherwise { + batch := batch + count + reduction := 0.U + completed := 0.U + state := initRequest + } + }.otherwise(context := context + 1.U) + }.otherwise(row := row + 1.U) + } + is(outputRequest)(state := outputResponse) + is(outputResponse) { savedRow := accRead; state := outputWrite } + is(outputWrite)(when(writePort.io.req.fire)(state := outputAck)) + is(outputAck) { + when(writePort.io.resp.fire) { + assert(writePort.io.resp.bits.ok, "MATMUL bank write failed") + when(outputQuarter === 3.U) { + outputQuarter := 0.U + when(outputLine + 1.U === m * nGroups) { chain := false.B; state := complete } + .otherwise { outputLine := outputLine + 1.U; state := outputRequest } + }.otherwise { outputQuarter := outputQuarter + 1.U; state := outputWrite } + } + } + is(complete)(when(io.cmdResp.fire)(state := idle)) + } + when(array.io.completed) { + assert(state === compute || state === drain) + completed := completed + 1.U + when(completed + 1.U === count * k) { + context := 0.U + row := 0.U + state := store + } + } +} diff --git a/examples/prefab/balls/mxmm/configs/MxmmBallParam.scala b/examples/prefab/balls/mxmm/configs/MxmmBallParam.scala new file mode 100644 index 000000000..2d1993d42 --- /dev/null +++ b/examples/prefab/balls/mxmm/configs/MxmmBallParam.scala @@ -0,0 +1,22 @@ +package examples.balls.mxmm.configs + +import buckyball.balls.mxmm.{Params => ParametersProto} +import upickle.default.{macroRW, ReadWriter} + +case class MxmmBallParam(tileRows: Int, tileCols: Int) { + require(tileRows == 16 && tileCols == 16) + val arithmeticLatency = 3 + val contexts = arithmeticLatency * 2 + 1 +} + +object MxmmBallParam { + + def fromProto(source: ParametersProto): MxmmBallParam = { + val p = source + require(p.hasTileRows && p.hasTileCols) + val param = MxmmBallParam(p.getTileRows, p.getTileCols) + param + } + + implicit val rw: ReadWriter[MxmmBallParam] = macroRW +} diff --git a/examples/prefab/balls/mxmm/configs/default.toml b/examples/prefab/balls/mxmm/configs/default.toml new file mode 100644 index 000000000..4fb96e5be --- /dev/null +++ b/examples/prefab/balls/mxmm/configs/default.toml @@ -0,0 +1,2 @@ +tileRows = 16 +tileCols = 16 diff --git a/examples/prefab/balls/mxmm/configs/mxmm_params.proto b/examples/prefab/balls/mxmm/configs/mxmm_params.proto new file mode 100644 index 000000000..cc716d469 --- /dev/null +++ b/examples/prefab/balls/mxmm/configs/mxmm_params.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; +package buckyball.balls.mxmm; +option java_package = "buckyball.balls.mxmm"; +option java_multiple_files = true; + +message Params { + optional int32 tileRows = 1; + optional int32 tileCols = 2; +} diff --git a/examples/prefab/balls/mxmm/emu/src/accumulator.rs b/examples/prefab/balls/mxmm/emu/src/accumulator.rs new file mode 100644 index 000000000..3dbdb8b01 --- /dev/null +++ b/examples/prefab/balls/mxmm/emu/src/accumulator.rs @@ -0,0 +1,17 @@ + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum Format { + Mxfp8, + Mxfp8Window, + Fma32, + F32, +} + +pub(crate) struct Chain { + pub(crate) format: Format, + pub(crate) rows: usize, + pub(crate) cols: usize, + pub(crate) bank: u64, + pub(crate) base: usize, + pub(crate) values: Vec, +} diff --git a/examples/prefab/balls/mxmm/emu/src/dispatch.rs b/examples/prefab/balls/mxmm/emu/src/dispatch.rs new file mode 100644 index 000000000..3fced61c8 --- /dev/null +++ b/examples/prefab/balls/mxmm/emu/src/dispatch.rs @@ -0,0 +1,33 @@ +use super::{fp32, mxfp8}; +use bebop_bemu::ball::ExecContext; + +const BALL_CLASS: &str = "examples.balls.mxmm.MxmmBall"; + +pub fn execute_known( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, + ctx: &mut ExecContext, +) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("MXMM_MXFP8") => Some(mxfp8::execute(xs1, xs2, false, ctx)), + Some("MXMM_MXFP8_WINDOW") => Some(mxfp8::execute(xs1, xs2, true, ctx)), + Some("MXMM_FMA32") => Some(fp32::execute(xs1, xs2, true, ctx)), + Some("MXMM_F32") => Some(fp32::execute(xs1, xs2, false, ctx)), + _ => None, + } +} + +pub fn cycles_after_issue(ball_class: &str, funct: u32, _xs1: u64, _xs2: u64) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("MXMM_MXFP8" | "MXMM_MXFP8_WINDOW" | "MXMM_FMA32" | "MXMM_F32") => Some(1), + _ => None, + } +} diff --git a/examples/prefab/balls/mxmm/emu/src/fp32.rs b/examples/prefab/balls/mxmm/emu/src/fp32.rs new file mode 100644 index 000000000..b5d1ffd08 --- /dev/null +++ b/examples/prefab/balls/mxmm/emu/src/fp32.rs @@ -0,0 +1,124 @@ +use std::collections::HashMap; +use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; +use super::accumulator::{Chain, Format}; +use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old}; +use super::instruction::ExecContext; + +pub(crate) fn execute(xs1: u64, xs2: u64, fused: bool, ctx: &mut ExecContext) -> u64 { + assert!(!rs1_read_old(xs1), "mxmm: reads no old destination; rs1[29] must be clear"); + let ball_state = ctx.state::>(); + let format = if fused { Format::Fma32 } else { Format::F32 }; + let rows = (xs2 & 0xfff) as usize; + let cols = ((xs2 >> 12) & 0xfff) as usize; + let k = rs1_iter(xs1) as usize; + let first = (xs2 >> 24) & 1 != 0; + let last = (xs2 >> 25) & 1 != 0; + let base = ((xs2 >> 26) & 63) as usize; + let a = rs1_b0(xs1); + let b = rs1_b1(xs1); + let c = rs1_b2(xs1); + assert!( + xs2 >> 32 == 0 && (rows == 1 || (rows > 0 && rows % 16 == 0)) && cols > 0 && cols % 16 == 0 && k > 0 && k % 4 == 0, + "f32 matmul: requires M=1/multiple of 16, N multiple of 16, positive K multiple of four and zero reserved bits" + ); + assert!(a != b && b != c && a != c, "f32 matmul: banks must differ"); + for bank in [a, b, c] { + assert!( + ctx.config(bank).allocated && ctx.config(bank).cols == 1, + "f32 matmul: banks must be allocated with one column" + ); + } + let stride = bank_row_bytes(); + assert!( + stride == 16 + && rows * k * 4 <= bank_lines() * stride + && cols * k * 4 <= bank_lines() * stride + && base * stride + rows * cols * 4 <= bank_lines() * stride, + "f32 matmul: bank footprint exceeds depth" + ); + let pa = pbank(ctx, a); + let pb = pbank(ctx, b); + let av = &ctx.banks[pa][..rows * k * 4]; + let bv = &ctx.banks[pb][..cols * k * 4]; + assert!( + av.chunks_exact(4) + .chain(bv.chunks_exact(4)) + .all(|x| f32::from_le_bytes(x.try_into().unwrap()).is_finite()), + "f32 matmul: non-finite operand" + ); + let mut chain = { + let mut chains = ball_state.lock().unwrap(); + if first { + assert!( + !chains.contains_key(&ctx.hart_id), + "f32 matmul: first while chain is live" + ); + Chain { + format, + rows, + cols, + bank: c, + base, + values: vec![0f32; rows * cols], + } + } else { + chains + .remove(&ctx.hart_id) + .expect("f32 matmul: continuation without first") + } + }; + assert!( + chain.format == format + && chain.rows == rows + && chain.cols == cols + && chain.bank == c + && chain.base == base, + "f32 matmul: continuation changed destination" + ); + let weights: Vec<&[u8]> = (0..cols) + .map(|col| &bv[col * k * 4..(col + 1) * k * 4]) + .collect(); + for row in 0..rows { + if av[row * k * 4..(row + 1) * k * 4] + .chunks_exact(4) + .all(|x| u32::from_le_bytes(x.try_into().unwrap()) & 0x7fffffff == 0) + && chain.values[row * cols..(row + 1) * cols] + .iter() + .all(|x| x.to_bits() == 0) + { + continue; + } + let av = &av[row * k * 4..(row + 1) * k * 4]; + let dst = &mut chain.values[row * cols..(row + 1) * cols]; + let mut sums = dst.to_vec(); + for inner in 0..k { + let a = f32::from_le_bytes(av[inner * 4..inner * 4 + 4].try_into().unwrap()); + for col in 0..cols { + let b = + f32::from_le_bytes(weights[col][inner * 4..inner * 4 + 4].try_into().unwrap()); + sums[col] = if fused { + a.mul_add(b, sums[col]) + } else { + (a * b) + sums[col] + }; + } + } + assert!( + sums.iter().all(|x| x.is_finite()), + "f32 matmul: accumulator overflow" + ); + dst.copy_from_slice(&sums); + } + if last { + let pc = pbank(ctx, c); + for (index, value) in chain.values.iter().enumerate() { + let offset = base * stride + index * 4; + ctx.banks[pc][offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + } + } else { + { + ball_state.lock().unwrap().insert(ctx.hart_id, chain); + }; + } + 0 +} diff --git a/examples/prefab/balls/mxmm/emu/src/lib.rs b/examples/prefab/balls/mxmm/emu/src/lib.rs new file mode 100644 index 000000000..c74a71243 --- /dev/null +++ b/examples/prefab/balls/mxmm/emu/src/lib.rs @@ -0,0 +1,12 @@ +pub(crate) use bebop_bemu::{thread::instructions::decode, ball as instruction}; + +mod accumulator; +mod fp32; +mod mxfp8; +mod dispatch; + +pub use dispatch::{cycles_after_issue, execute_known}; + +#[bebop_bemu::ball::linkme::distributed_slice(bebop_bemu::BALLS)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static BALL: bebop_bemu::Ball = bebop_bemu::Ball { execute: execute_known, cycles: cycles_after_issue }; diff --git a/examples/prefab/balls/mxmm/emu/src/mxfp8.rs b/examples/prefab/balls/mxmm/emu/src/mxfp8.rs new file mode 100644 index 000000000..a02908dc2 --- /dev/null +++ b/examples/prefab/balls/mxmm/emu/src/mxfp8.rs @@ -0,0 +1,179 @@ +use std::collections::HashMap; +use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; +use super::accumulator::{Chain, Format}; +use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old}; +use super::instruction::ExecContext; + +fn value(code: u8, factor: f32) -> f32 { + let exponent = (code >> 3) & 15; + let mantissa = (code & 7) as u32; + let normal = ((exponent as u32 + 120) << 23) | (mantissa << 20); + let subnormal = (mantissa as f32 / 512.0).to_bits(); + let element = f32::from_bits(if exponent == 0 { subnormal } else { normal }); + let result = f32::from_bits((element * factor).to_bits() | ((code as u32 & 128) << 24)); + result +} + +pub(crate) fn execute(xs1: u64, xs2: u64, window: bool, ctx: &mut ExecContext) -> u64 { + assert!(!rs1_read_old(xs1), "mxmm: reads no old destination; rs1[29] must be clear"); + let ball_state = ctx.state::>(); + let rows = (xs2 & 0xfff) as usize; + let cols = ((xs2 >> 12) & 0xfff) as usize; + let k = rs1_iter(xs1) as usize; + let first = (xs2 >> 24) & 1 != 0; + let last = (xs2 >> 25) & 1 != 0; + let base = ((xs2 >> 26) & 63) as usize; + let a = rs1_b0(xs1); + let b = rs1_b1(xs1); + let c = rs1_b2(xs1); + let full_k = if window { + ((xs2 >> 32) & 0xffff) as usize + } else { + k + }; + let start_k = if window { (xs2 >> 48) as usize } else { 0 }; + let format = if window { + Format::Mxfp8Window + } else { + Format::Mxfp8 + }; + assert!( + (window || xs2 >> 32 == 0) && (rows == 1 || (rows > 0 && rows % 16 == 0)) && cols > 0 && cols % 16 == 0 && k > 0 && k % 32 == 0, + "mxfp8: requires M=1/multiple of 16, N multiple of 16, K multiple of 32 and zero reserved bits" + ); + assert!( + full_k > 0 && full_k % 32 == 0 && start_k % 32 == 0 && start_k + k <= full_k, + "mxfp8: invalid activation window" + ); + assert!(a != b && b != c && a != c, "mxfp8: banks must differ"); + for bank in [a, b, c] { + assert!( + ctx.config(bank).allocated && ctx.config(bank).cols == 1, + "mxfp8: banks must be allocated with one column" + ); + } + let stride = bank_row_bytes(); + assert!(stride == 16, "mxfp8: requires 16-byte bank rows"); + assert!( + rows * full_k + rows * full_k / 32 <= bank_lines() * stride + && cols * k + cols * k / 32 <= bank_lines() * stride + && base * stride + rows * cols * 4 <= bank_lines() * stride, + "mxfp8: bank footprint exceeds depth" + ); + let pa = pbank(ctx, a); + let pb = pbank(ctx, b); + let a_bytes = &ctx.banks[pa][..rows * full_k + rows * full_k / 32]; + let b_bytes = &ctx.banks[pb][..cols * k + cols * k / 32]; + assert!( + b_bytes[..cols * k] + .iter() + .fold(true, |valid, code| valid & (code & 0x7f != 0x7f)), + "mxfp8: NaN encoding" + ); + let mut av = vec![0f32; rows * k]; + let mut bv = vec![0f32; cols * k]; + for (block, dst) in av.chunks_exact_mut(32).enumerate() { + let row = block / (k / 32); + let inner = block % (k / 32); + let code_offset = row * full_k + start_k + inner * 32; + let scale = a_bytes[rows * full_k + row * (full_k / 32) + start_k / 32 + inner]; + assert!(scale != 255, "mxfp8: NaN encoding"); + let factor = if scale == 0 { + f32::from_bits(1 << 22) + } else { + f32::from_bits((scale as u32) << 23) + }; + for (index, decoded) in dst.iter_mut().enumerate() { + let code = a_bytes[code_offset + index]; + assert!(code & 0x7f != 0x7f, "mxfp8: NaN encoding"); + *decoded = value(code, factor); + } + } + for block in 0..k / 32 { + for col in 0..cols { + let scale = b_bytes[cols * k + col * (k / 32) + block]; + assert!(scale != 255, "mxfp8: NaN encoding"); + let factor = if scale == 0 { + f32::from_bits(1 << 22) + } else { + f32::from_bits((scale as u32) << 23) + }; + let codes = &b_bytes[col * k + block * 32..col * k + (block + 1) * 32]; + let mut decoded = [0f32; 32]; + for (decoded, code) in decoded.iter_mut().zip(codes.iter()) { + *decoded = value(*code, factor); + } + for (index, decoded) in decoded.iter().enumerate() { + bv[(block * 32 + index) * cols + col] = *decoded; + } + } + } + assert!( + av.iter() + .chain(bv.iter()) + .fold(true, |valid, value| valid & value.is_finite()), + "mxfp8: decoded value overflows FP32" + ); + let mut chain = { + let mut chains = ball_state.lock().unwrap(); + if first { + assert!( + !chains.contains_key(&ctx.hart_id), + "mxfp8: first while chain is live" + ); + Chain { + format, + rows, + cols, + bank: c, + base, + values: vec![0f32; rows * cols], + } + } else { + chains + .remove(&ctx.hart_id) + .expect("mxfp8: continuation without first") + } + }; + assert!( + chain.format == format + && chain.rows == rows + && chain.cols == cols + && chain.bank == c + && chain.base == base, + "mxfp8: continuation changed destination" + ); + for row in 0..rows { + if av[row * k..(row + 1) * k].iter().all(|value| *value == 0.0) + && chain.values[row * cols..(row + 1) * cols] + .iter() + .all(|value| value.to_bits() == 0) + { + continue; + } + let av = &av[row * k..(row + 1) * k]; + let dst = &mut chain.values[row * cols..(row + 1) * cols]; + let mut sums = dst.to_vec(); + for inner in 0..k { + let a = av[inner]; + let weights = &bv[inner * cols..(inner + 1) * cols]; + for col in 0..cols { + sums[col] = a.mul_add(weights[col], sums[col]); + } + } + assert!(sums.iter().all(|v| v.is_finite()), "accumulator overflow"); + dst.copy_from_slice(&sums); + } + if last { + let pc = pbank(ctx, c); + for (i, value) in chain.values.iter().enumerate() { + ctx.banks[pc][base * stride + i * 4..base * stride + i * 4 + 4] + .copy_from_slice(&value.to_le_bytes()); + } + } else { + { + ball_state.lock().unwrap().insert(ctx.hart_id, chain); + }; + } + 0 +} diff --git a/examples/prefab/balls/mxmm/stack/kernels/CMakeLists.txt b/examples/prefab/balls/mxmm/stack/kernels/CMakeLists.txt new file mode 100644 index 000000000..e1fd04aac --- /dev/null +++ b/examples/prefab/balls/mxmm/stack/kernels/CMakeLists.txt @@ -0,0 +1,45 @@ +# ===----------------------------------------------------------------------=== +# MXMM kernels: two RVV kernels built against the thread's Ball ISA, and the +# host code that launches them. Added by the stack of a thread that has this +# Ball, which compiles KERNEL_SOURCES and exports KERNEL_EXPORTS. +# ===----------------------------------------------------------------------=== +set(RVV "${DESIGN_REPO}/stack/kernels/rvv") +set(MXMM_OBJECTS) +foreach(SOURCE + "${CMAKE_CURRENT_SOURCE_DIR}/norm_window.cpp" + "${RVV}/norm.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/packmm.cpp" + "${RVV}/pack.cpp") + get_filename_component(NAME "${SOURCE}" NAME_WE) + add_custom_command( + OUTPUT ${NAME}.o + COMMAND "${RVV_CXX}" --target=riscv64 -march=rv64imf_zve32f_zvl128b -mabi=lp64f + -mcmodel=medany -ffreestanding -O2 -ffp-contract=off -ffunction-sections -fdata-sections + "-I${DESIGN_REPO}/stack/kernels" "-I${DESIGN_REPO}/bb-tests/workloads/lib" ${THREAD_DEFS} + -MD -MF ${NAME}.d -c "${SOURCE}" -o ${NAME}.o + DEPENDS "${SOURCE}" + DEPFILE ${NAME}.d + VERBATIM) + list(APPEND MXMM_OBJECTS "${CMAKE_CURRENT_BINARY_DIR}/${NAME}.o") +endforeach() +foreach(NAME norm_window packmm) + add_custom_command( + OUTPUT ${NAME}.elf + COMMAND "${RVV_LD}" -m elf64lriscv --gc-sections -e ${NAME} --defsym=kernel_entry=${NAME} + -T "${RVV}/kernel.ld" ${MXMM_OBJECTS} -o ${NAME}.elf + DEPENDS ${MXMM_OBJECTS} "${RVV}/kernel.ld" + VERBATIM) + add_custom_command( + OUTPUT ${NAME}.image + COMMAND "${RVV_OBJCOPY}" -O binary -j .header -j .text -j .rodata ${NAME}.elf ${NAME}.image + DEPENDS ${NAME}.elf + VERBATIM) +endforeach() +add_custom_target(mxmm-images DEPENDS + "${CMAKE_CURRENT_BINARY_DIR}/norm_window.image" "${CMAKE_CURRENT_BINARY_DIR}/packmm.image") + +set(KERNEL_SOURCES ${KERNEL_SOURCES} + "${CMAKE_CURRENT_SOURCE_DIR}/dispatch.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/pack_dispatch.cpp" PARENT_SCOPE) +set(KERNEL_EMBED_DIRS ${KERNEL_EMBED_DIRS} "${CMAKE_CURRENT_BINARY_DIR}" PARENT_SCOPE) +set(KERNEL_DEPENDS ${KERNEL_DEPENDS} mxmm-images PARENT_SCOPE) +set(KERNEL_EXPORTS ${KERNEL_EXPORTS} _mlir_ciface_rvv_norm_window _mlir_ciface_rvv_packmm PARENT_SCOPE) diff --git a/examples/prefab/balls/mxmm/stack/kernels/dispatch.cpp b/examples/prefab/balls/mxmm/stack/kernels/dispatch.cpp new file mode 100644 index 000000000..feca99585 --- /dev/null +++ b/examples/prefab/balls/mxmm/stack/kernels/dispatch.cpp @@ -0,0 +1,58 @@ +#include "norm_window_image.h" +#include "window.h" +#include +#include +#include +#include + +extern "C" void +_mlir_ciface_rvv_norm_window(uint64_t readBank, uint64_t writeBank, + uint64_t phase, uint64_t width, uint64_t columns, + uint64_t count, uint64_t full, uint64_t start, + uint64_t first, uint64_t last, uint64_t bankBytes, + uint32_t meanBits, uint32_t epsilonBits) { + constexpr unsigned programBank = CORE_VIRTUAL_BANK_COUNT; + if (phase == 2) { + release_kernel(programBank); + return; + } + if (phase > 1 || !width || width > bankBytes / sizeof(float)) { + fputs("MXMM norm/window requires a complete row within its bank\n", stderr); + abort(); + } + if (phase == 0) + mvin_kernel(norm_window_image, sizeof(norm_window_image), programBank); + uint32_t rounding; + asm volatile("csrr %0, frm" : "=r"(rounding) : : "memory"); + struct alignas(16) Packet { + kernel_launch launch; + Window parameters; + uint32_t padding; + } packet{{norm_window_entry, + norm_window_text, + 0x40002000, + {(uint64_t(2) << 32) + sizeof(kernel_launch), + (uint64_t(2) << 32) + offsetof(kernel_launch, reserved)}, + 0}, + {uint32_t(phase), uint32_t(width), meanBits, epsilonBits, + uint32_t(columns), uint32_t(count), uint32_t(full), uint32_t(start), + uint32_t(first), uint32_t(last), rounding << 5}, + 0}; + bb_mvin_group((uintptr_t)&packet, writeBank, 0, sizeof(packet) / 16, 1); + // Finish descriptor DMA before this stack packet can be reused. + run_kernel(readBank, programBank, writeBank, 0); + if (phase == 1) + return; + alignas(16) kernel_launch completed; + bb_mvout_group((uintptr_t)&completed, writeBank, 0, sizeof(completed) / 16, + 1); + if (std::memcmp(&packet.launch, &completed, + offsetof(kernel_launch, reserved)) || + (completed.reserved & ~UINT64_C(31))) { + fputs( + "MXMM norm/window returned an invalid descriptor or exception flags\n", + stderr); + abort(); + } + asm volatile("csrs fflags, %0" : : "r"(completed.reserved) : "memory"); +} diff --git a/examples/prefab/balls/mxmm/stack/kernels/norm_window.cpp b/examples/prefab/balls/mxmm/stack/kernels/norm_window.cpp new file mode 100644 index 000000000..f300be84c --- /dev/null +++ b/examples/prefab/balls/mxmm/stack/kernels/norm_window.cpp @@ -0,0 +1,38 @@ +#include +#include +#include + +#include "window.h" + +extern "C" void norm_window(const Window *parameters, uint32_t *flags) { + asm volatile("csrw fcsr, %0" ::"r"(parameters->rounding) : "memory"); + if (parameters->phase == 0) { + rvv_norm((float *)(uint64_t(3) << 32), (const float *)0, + (const float *)(uint64_t(1) << 32), parameters->width, + parameters->mean, parameters->epsilon); + uint64_t banks = + UINT64_C(3) | (UINT64_C(4) << 20) | (uint64_t(parameters->width) << 30); + uint64_t zero = 0; + asm volatile( + ".insn r 0x7b, 3, %c2, x0, %0, %1" ::"r"(banks), "r"(zero), + "i"(THREAD_BALLDOMAIN_BALLS_mxquant_INSTRUCTIONS_MXQUANT_FUNCT7) + : "memory"); + } else if (parameters->phase == 1) { + uint64_t banks = + UINT64_C(4) | (UINT64_C(3) << 20) | (uint64_t(parameters->count) << 30); + uint64_t config = UINT64_C(1) | (uint64_t(parameters->columns) << 12) | + (uint64_t(parameters->first) << 24) | + (uint64_t(parameters->last) << 25) | + (uint64_t(parameters->full) << 32) | + (uint64_t(parameters->start) << 48); + asm volatile( + ".insn r 0x7b, 3, %c2, x0, %0, %1" ::"r"(banks), "r"(config), + "i"(THREAD_BALLDOMAIN_BALLS_mxmm_INSTRUCTIONS_MXMM_MXFP8_WINDOW_FUNCT7) + : "memory"); + } else { + asm volatile("ebreak"); + } + uint64_t result; + asm volatile("csrr %0, fflags" : "=r"(result)::"memory"); + *flags = uint32_t(result); +} diff --git a/examples/prefab/balls/mxmm/stack/kernels/norm_window_image.h b/examples/prefab/balls/mxmm/stack/kernels/norm_window_image.h new file mode 100644 index 000000000..3328faa6f --- /dev/null +++ b/examples/prefab/balls/mxmm/stack/kernels/norm_window_image.h @@ -0,0 +1,15 @@ +#pragma once +// The norm_window RVV kernel image, embedded from this directory's build +// output. +#include + +alignas(4) constexpr uint8_t norm_window_image[] = { +#embed "norm_window.image" +}; +// Header words, see stack/kernels/rvv/kernel.ld: 1 = code bytes, 2 = entry. +constexpr uint32_t norm_window_entry = + norm_window_image[8] | norm_window_image[9] << 8 | + norm_window_image[10] << 16 | uint32_t(norm_window_image[11]) << 24; +constexpr uint32_t norm_window_text = + norm_window_image[4] | norm_window_image[5] << 8 | + norm_window_image[6] << 16 | uint32_t(norm_window_image[7]) << 24; diff --git a/examples/balls/mxmm/workloads/kernels/pack_dispatch.cpp b/examples/prefab/balls/mxmm/stack/kernels/pack_dispatch.cpp similarity index 95% rename from examples/balls/mxmm/workloads/kernels/pack_dispatch.cpp rename to examples/prefab/balls/mxmm/stack/kernels/pack_dispatch.cpp index 1a1445c09..6b9450458 100644 --- a/examples/balls/mxmm/workloads/kernels/pack_dispatch.cpp +++ b/examples/prefab/balls/mxmm/stack/kernels/pack_dispatch.cpp @@ -3,7 +3,6 @@ #include #include #include -#include extern "C" void _mlir_ciface_rvv_packmm(uint64_t readBank, uint64_t writeBank, uint64_t phase, uint64_t fullK, @@ -12,7 +11,7 @@ extern "C" void _mlir_ciface_rvv_packmm(uint64_t readBank, uint64_t writeBank, uint64_t columns, uint64_t rows, uint64_t first, uint64_t last, uint64_t fused) { - constexpr unsigned programBank = VIRTUAL_BANK_NUM; + constexpr unsigned programBank = CORE_VIRTUAL_BANK_COUNT; if (phase == 0) { mvin_kernel(packmm_image, sizeof(packmm_image), programBank); return; diff --git a/examples/prefab/balls/mxmm/stack/kernels/packmm.cpp b/examples/prefab/balls/mxmm/stack/kernels/packmm.cpp new file mode 100644 index 000000000..c8d33df11 --- /dev/null +++ b/examples/prefab/balls/mxmm/stack/kernels/packmm.cpp @@ -0,0 +1,35 @@ +#include "packmm.h" +#include + +extern "C" void rvv_pack(uint32_t *, const uint32_t *, size_t, size_t, size_t, + size_t, size_t, size_t); + +extern "C" void packmm(const PackMM *parameters, uint32_t *flags) { + asm volatile("csrw fcsr, %0" : : "r"(parameters->rounding) : "memory"); + rvv_pack((uint32_t *)(uint64_t(3) << 32), (const uint32_t *)0, + parameters->fullK, parameters->fullN, parameters->startK, + parameters->startN, parameters->countK, parameters->columns); + const uint64_t banks = UINT64_C(1) | (UINT64_C(3) << 10) | + (UINT64_C(4) << 20) | + (uint64_t(parameters->countK) << 30); + const uint64_t config = + uint64_t(parameters->rows) | (uint64_t(parameters->columns) << 12) | + (uint64_t(parameters->first) << 24) | (uint64_t(parameters->last) << 25); + if (parameters->fused) + asm volatile( + ".insn r 0x7b, 3, %c2, x0, %0, %1" + : + : "r"(banks), "r"(config), + "i"(THREAD_BALLDOMAIN_BALLS_mxmm_INSTRUCTIONS_MXMM_FMA32_FUNCT7) + : "memory"); + else + asm volatile( + ".insn r 0x7b, 3, %c2, x0, %0, %1" + : + : "r"(banks), "r"(config), + "i"(THREAD_BALLDOMAIN_BALLS_mxmm_INSTRUCTIONS_MXMM_F32_FUNCT7) + : "memory"); + uint64_t result; + asm volatile("csrr %0, fflags" : "=r"(result) : : "memory"); + *flags = uint32_t(result); +} diff --git a/examples/balls/mxmm/workloads/kernels/packmm.h b/examples/prefab/balls/mxmm/stack/kernels/packmm.h similarity index 100% rename from examples/balls/mxmm/workloads/kernels/packmm.h rename to examples/prefab/balls/mxmm/stack/kernels/packmm.h diff --git a/examples/prefab/balls/mxmm/stack/kernels/packmm_image.h b/examples/prefab/balls/mxmm/stack/kernels/packmm_image.h new file mode 100644 index 000000000..46742a11c --- /dev/null +++ b/examples/prefab/balls/mxmm/stack/kernels/packmm_image.h @@ -0,0 +1,14 @@ +#pragma once +// The packmm RVV kernel image, embedded from this directory's build output. +#include + +alignas(4) constexpr uint8_t packmm_image[] = { +#embed "packmm.image" +}; +// Header words, see stack/kernels/rvv/kernel.ld: 1 = code bytes, 2 = entry. +constexpr uint32_t packmm_entry = packmm_image[8] | packmm_image[9] << 8 | + packmm_image[10] << 16 | + uint32_t(packmm_image[11]) << 24; +constexpr uint32_t packmm_text = packmm_image[4] | packmm_image[5] << 8 | + packmm_image[6] << 16 | + uint32_t(packmm_image[7]) << 24; diff --git a/examples/balls/mxmm/workloads/kernels/window.h b/examples/prefab/balls/mxmm/stack/kernels/window.h similarity index 100% rename from examples/balls/mxmm/workloads/kernels/window.h rename to examples/prefab/balls/mxmm/stack/kernels/window.h diff --git a/examples/prefab/balls/mxmm/stack/src/mxmmOps.cpp b/examples/prefab/balls/mxmm/stack/src/mxmmOps.cpp new file mode 100644 index 000000000..ce922759b --- /dev/null +++ b/examples/prefab/balls/mxmm/stack/src/mxmmOps.cpp @@ -0,0 +1,167 @@ +//===- mxmmOps.cpp - MxmmBall operations ----------------------------------===// +// +// The dialect of MxmmBall, its virtual operations building their physical ones +// and the instructions of the physical operations. +// +//===----------------------------------------------------------------------===// + +#include "mxmmOps.h" + +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::mxmm_ball; +using mlir::buddy::bankFields; +using mlir::buddy::constantI64; +using mlir::buddy::emitInstruction; +using mlir::buddy::field; + +#include "mxmmDialect.cpp.inc" + +#define GET_OP_CLASSES +#include "mxmmOps.cpp.inc" + +void MxmmBallDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "mxmmOps.cpp.inc" + >(); +} + +namespace { +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallDialect( + [](DialectRegistry ®istry) { registry.insert(); }), + true); + +// Emits one chain step of `mnemonic`. An MXFP8 operand takes one byte per +// element and one scale per 32; an FP32 operand four bytes per element. A +// spans fullK columns, of which the step reads k from startK; only the window +// format encodes fullK and startK. +LogicalResult emitMatmul(Operation *op, OpBuilder &builder, StringRef mnemonic, + bool mx, bool window, int64_t a, int64_t b, int64_t c, + int64_t rows, int64_t cols, int64_t k, int64_t fullK, + int64_t startK, int64_t base, bool first, bool last) { + const auto &target = buckyball_target::getBuckyballTarget(); + int64_t bytes = target.bankDepth * 16; + int64_t aElements = rows * fullK, bElements = cols * k; + if (target.bankWidthBits != 128 || + !(rows == 1 || (rows > 0 && rows % 16 == 0)) || rows > 4095 || + cols <= 0 || cols % 16 || cols > 4095 || k <= 0 || k % (mx ? 32 : 4) || + k >= (1LL << 33) || base < 0 || base > 63) + return op->emitOpError("shape exceeds the ISA geometry"); + if (window && (fullK % 32 || startK % 32 || startK + k > fullK || + fullK > 65535 || startK > 65535)) + return op->emitOpError("A window exceeds full_k"); + if ((mx ? aElements + aElements / 32 : aElements * 4) > bytes || + (mx ? bElements + bElements / 32 : bElements * 4) > bytes || + base * 16 + rows * cols * 4 > bytes) + return op->emitOpError("operands exceed the bank"); + if (a == b || a == c || b == c) + return op->emitOpError("requires distinct A, B and C banks"); + uint64_t rs2 = field(rows, 0, 11) | field(cols, 12, 23) | + field(first, 24, 24) | field(last, 25, 25) | + field(base, 26, 31); + if (window) + rs2 |= field(fullK, 32, 47) | field(startK, 48, 63); + emitInstruction( + builder, op->getLoc(), buckyball_target::getBuckyballFunct7(mnemonic), + constantI64(builder, op->getLoc(), bankFields({a, b}, c, false, k)), + constantI64(builder, op->getLoc(), rs2)); + return success(); +} +} // namespace + +SmallVector BankMatmulMxfp8Op::getSourceBanks() { + return {getA(), getB()}; +} +Value BankMatmulMxfp8Op::getDestinationBank() { return getC(); } +bool BankMatmulMxfp8Op::readsDestination() { return false; } +Operation *BankMatmulMxfp8Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return MatmulMxfp8Op::create(builder, getLoc(), sources[0], sources[1], + destination, getRows(), getCols(), getK(), + getBase(), getFirst(), getLast()); +} + +SmallVector MatmulMxfp8Op::getSourceBanks() { + return {int64_t(getA()), int64_t(getB())}; +} +int64_t MatmulMxfp8Op::getDestinationBank() { return getC(); } +LogicalResult MatmulMxfp8Op::lowerToInstructions(OpBuilder &builder) { + return emitMatmul(*this, builder, "MXMM_MXFP8", true, false, getA(), getB(), + getC(), getRows(), getCols(), getK(), getK(), 0, getBase(), + getFirst(), getLast()); +} + +SmallVector BankMatmulMxfp8WindowOp::getSourceBanks() { + return {getA(), getB()}; +} +Value BankMatmulMxfp8WindowOp::getDestinationBank() { return getC(); } +bool BankMatmulMxfp8WindowOp::readsDestination() { return false; } +Operation *BankMatmulMxfp8WindowOp::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return MatmulMxfp8WindowOp::create(builder, getLoc(), sources[0], sources[1], + destination, getRows(), getCols(), getK(), + getFullK(), getStartK(), getBase(), + getFirst(), getLast()); +} + +SmallVector MatmulMxfp8WindowOp::getSourceBanks() { + return {int64_t(getA()), int64_t(getB())}; +} +int64_t MatmulMxfp8WindowOp::getDestinationBank() { return getC(); } +LogicalResult MatmulMxfp8WindowOp::lowerToInstructions(OpBuilder &builder) { + return emitMatmul(*this, builder, "MXMM_MXFP8_WINDOW", true, true, getA(), + getB(), getC(), getRows(), getCols(), getK(), getFullK(), + getStartK(), getBase(), getFirst(), getLast()); +} + +SmallVector BankMatmulFma32Op::getSourceBanks() { + return {getA(), getB()}; +} +Value BankMatmulFma32Op::getDestinationBank() { return getC(); } +bool BankMatmulFma32Op::readsDestination() { return false; } +Operation *BankMatmulFma32Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return MatmulFma32Op::create(builder, getLoc(), sources[0], sources[1], + destination, getRows(), getCols(), getK(), + getBase(), getFirst(), getLast()); +} + +SmallVector MatmulFma32Op::getSourceBanks() { + return {int64_t(getA()), int64_t(getB())}; +} +int64_t MatmulFma32Op::getDestinationBank() { return getC(); } +LogicalResult MatmulFma32Op::lowerToInstructions(OpBuilder &builder) { + return emitMatmul(*this, builder, "MXMM_FMA32", false, false, getA(), getB(), + getC(), getRows(), getCols(), getK(), getK(), 0, getBase(), + getFirst(), getLast()); +} + +SmallVector BankMatmulF32Op::getSourceBanks() { + return {getA(), getB()}; +} +Value BankMatmulF32Op::getDestinationBank() { return getC(); } +bool BankMatmulF32Op::readsDestination() { return false; } +Operation *BankMatmulF32Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return MatmulF32Op::create(builder, getLoc(), sources[0], sources[1], + destination, getRows(), getCols(), getK(), + getBase(), getFirst(), getLast()); +} + +SmallVector MatmulF32Op::getSourceBanks() { + return {int64_t(getA()), int64_t(getB())}; +} +int64_t MatmulF32Op::getDestinationBank() { return getC(); } +LogicalResult MatmulF32Op::lowerToInstructions(OpBuilder &builder) { + return emitMatmul(*this, builder, "MXMM_F32", false, false, getA(), getB(), + getC(), getRows(), getCols(), getK(), getK(), 0, getBase(), + getFirst(), getLast()); +} diff --git a/examples/prefab/balls/mxmm/stack/src/mxmmOps.h b/examples/prefab/balls/mxmm/stack/src/mxmmOps.h new file mode 100644 index 000000000..4d2203273 --- /dev/null +++ b/examples/prefab/balls/mxmm/stack/src/mxmmOps.h @@ -0,0 +1,20 @@ +//===- mxmmOps.h - MxmmBall operations ------------------------------------===// + +#ifndef MXMM_OPS_H +#define MXMM_OPS_H + +#include "Interfaces/BankInterfaces.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "mxmmDialect.h.inc" + +#define GET_OP_CLASSES +#include "mxmmOps.h.inc" + +#endif // MXMM_OPS_H diff --git a/examples/prefab/balls/mxmm/stack/src/mxmmOps.td b/examples/prefab/balls/mxmm/stack/src/mxmmOps.td new file mode 100644 index 000000000..6a1390cc4 --- /dev/null +++ b/examples/prefab/balls/mxmm/stack/src/mxmmOps.td @@ -0,0 +1,118 @@ +//===- mxmmOps.td - MxmmBall operations --------------------*- tablegen -*-===// +// +// matmul_ (a, b) -> c: one step of a chained FP32 accumulation +// C[rows x cols] += A[rows x k] @ B[k x cols]. `rows` is one or a multiple of +// sixteen, `cols` a multiple of sixteen. The chain lives in the Ball, one per +// hart: `first` starts it from +0, the steps between continue it, `last` +// writes it to C as `rows` x `cols` FP32 in row-major order from row `base` +// (0..63) of the bank; nothing else of C is written, and no step reads C. A +// step without `first` continues the live chain with the same format, rows, +// cols, C and base. +// mxfp8: k is a multiple of 32. A holds rows x k E4M3 codes row-major, then +// the rows x k/32 E8M0 scales row-major. B holds the cols x k codes, column +// c the k codes from byte c * k, then the cols x k/32 scales, column c from +// byte cols * k + c * k/32. +// mxfp8_window: mxfp8 on columns start_k .. start_k + k of an A of +// full_k columns; full_k and start_k are multiples of 32. +// fma32: k is a multiple of four. A holds rows x k FP32 row-major, B column c +// the k FP32 from byte 4 * c * k; every product is fused into the sum. +// f32: fma32 with the product rounded before the sum. +// A, B and C are distinct banks of one 128-bit wide bank each. +// +//===----------------------------------------------------------------------===// + +#ifndef MXMM_OPS +#define MXMM_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Interfaces/BankInterfaces.td" + +def MxmmBall_Dialect : Dialect { + let name = "mxmm_ball"; + let cppNamespace = "::buddy::mxmm_ball"; + let summary = "The operations of MxmmBall"; +} + +class MxmmBall_Op traits = []> + : Op])>; + +def BankMatmulMxfp8Op : MxmmBall_Op<"bank_matmul_mxfp8", + [DeclareOpInterfaceMethods]> { + let summary = "matmul_mxfp8 on virtual banks"; + let arguments = (ins I64:$a, I64:$b, I64:$c, I64Attr:$rows, I64Attr:$cols, + I64Attr:$k, I64Attr:$base, UnitAttr:$first, + UnitAttr:$last); + let results = (outs I64:$cOut); + let assemblyFormat = "$a `,` $b `->` $c prop-dict attr-dict"; +} + +def MatmulMxfp8Op : MxmmBall_Op<"matmul_mxfp8", + [DeclareOpInterfaceMethods]> { + let summary = "matmul_mxfp8 on physical banks"; + let arguments = (ins I64Attr:$a, I64Attr:$b, I64Attr:$c, I64Attr:$rows, + I64Attr:$cols, I64Attr:$k, I64Attr:$base, + UnitAttr:$first, UnitAttr:$last); + let assemblyFormat = "prop-dict attr-dict"; +} + +def BankMatmulMxfp8WindowOp : MxmmBall_Op<"bank_matmul_mxfp8_window", + [DeclareOpInterfaceMethods]> { + let summary = "matmul_mxfp8_window on virtual banks"; + let arguments = (ins I64:$a, I64:$b, I64:$c, I64Attr:$rows, I64Attr:$cols, + I64Attr:$k, I64Attr:$full_k, I64Attr:$start_k, + I64Attr:$base, UnitAttr:$first, UnitAttr:$last); + let results = (outs I64:$cOut); + let assemblyFormat = "$a `,` $b `->` $c prop-dict attr-dict"; +} + +def MatmulMxfp8WindowOp : MxmmBall_Op<"matmul_mxfp8_window", + [DeclareOpInterfaceMethods]> { + let summary = "matmul_mxfp8_window on physical banks"; + let arguments = (ins I64Attr:$a, I64Attr:$b, I64Attr:$c, I64Attr:$rows, + I64Attr:$cols, I64Attr:$k, I64Attr:$full_k, + I64Attr:$start_k, I64Attr:$base, UnitAttr:$first, + UnitAttr:$last); + let assemblyFormat = "prop-dict attr-dict"; +} + +def BankMatmulFma32Op : MxmmBall_Op<"bank_matmul_fma32", + [DeclareOpInterfaceMethods]> { + let summary = "matmul_fma32 on virtual banks"; + let arguments = (ins I64:$a, I64:$b, I64:$c, I64Attr:$rows, I64Attr:$cols, + I64Attr:$k, I64Attr:$base, UnitAttr:$first, + UnitAttr:$last); + let results = (outs I64:$cOut); + let assemblyFormat = "$a `,` $b `->` $c prop-dict attr-dict"; +} + +def MatmulFma32Op : MxmmBall_Op<"matmul_fma32", + [DeclareOpInterfaceMethods]> { + let summary = "matmul_fma32 on physical banks"; + let arguments = (ins I64Attr:$a, I64Attr:$b, I64Attr:$c, I64Attr:$rows, + I64Attr:$cols, I64Attr:$k, I64Attr:$base, + UnitAttr:$first, UnitAttr:$last); + let assemblyFormat = "prop-dict attr-dict"; +} + +def BankMatmulF32Op : MxmmBall_Op<"bank_matmul_f32", + [DeclareOpInterfaceMethods]> { + let summary = "matmul_f32 on virtual banks"; + let arguments = (ins I64:$a, I64:$b, I64:$c, I64Attr:$rows, I64Attr:$cols, + I64Attr:$k, I64Attr:$base, UnitAttr:$first, + UnitAttr:$last); + let results = (outs I64:$cOut); + let assemblyFormat = "$a `,` $b `->` $c prop-dict attr-dict"; +} + +def MatmulF32Op : MxmmBall_Op<"matmul_f32", + [DeclareOpInterfaceMethods]> { + let summary = "matmul_f32 on physical banks"; + let arguments = (ins I64Attr:$a, I64Attr:$b, I64Attr:$c, I64Attr:$rows, + I64Attr:$cols, I64Attr:$k, I64Attr:$base, + UnitAttr:$first, UnitAttr:$last); + let assemblyFormat = "prop-dict attr-dict"; +} + +#endif // MXMM_OPS diff --git a/examples/prefab/balls/mxmm/verify/casegen/Cargo.lock b/examples/prefab/balls/mxmm/verify/casegen/Cargo.lock new file mode 100644 index 000000000..4937a413f --- /dev/null +++ b/examples/prefab/balls/mxmm/verify/casegen/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "mxmm-casegen" +version = "0.1.0" diff --git a/examples/prefab/balls/mxmm/verify/casegen/Cargo.toml b/examples/prefab/balls/mxmm/verify/casegen/Cargo.toml new file mode 100644 index 000000000..492e99d44 --- /dev/null +++ b/examples/prefab/balls/mxmm/verify/casegen/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "mxmm-casegen" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["rlib", "cdylib"] + +[lints.rust] +unused_must_use = "deny" diff --git a/examples/prefab/balls/mxmm/verify/casegen/src/casegen.rs b/examples/prefab/balls/mxmm/verify/casegen/src/casegen.rs new file mode 100644 index 000000000..5ed603413 --- /dev/null +++ b/examples/prefab/balls/mxmm/verify/casegen/src/casegen.rs @@ -0,0 +1,408 @@ +// MXMM reference model, the contract shared with the BEMU model +// (emu/src/mxfp8.rs, emu/src/fp32.rs) and MxmmBall: +// +// rs2[11:0] = M (1 or a multiple of 16), rs2[23:12] = N (multiple of 16), +// rs2[24] = first, rs2[25] = last, rs2[31:26] = C base row, iter = K. +// MXMM_MXFP8_WINDOW adds rs2[47:32] = fullK and rs2[63:48] = startK; the other +// formats reserve rs2[63:32]. +// +// MXFP8 (K multiple of 32): A holds M rows of fullK E4M3 codes followed by the +// E8M0 scales (row r, block j at byte M * fullK + r * fullK / 32 + j); B holds +// N columns of K codes followed by their scales. FP32 (K multiple of four): A +// is M rows of K floats, B is N columns of K floats. +// C[r][c] accumulates sum_k A[r][k] * B[c][k] in k order with a fused +// multiply-add (MXFP8, FMA32) or a rounded product then a rounded add (F32), +// starting from zero on first and from the chain otherwise. On last the chain +// is written to C as row-major FP32 from row base; otherwise nothing is +// written. + +pub const ROW_BYTES: usize = 16; +pub const NUM_CASES: u32 = 8; + +pub const FORMAT_MXFP8: u32 = 0; +pub const FORMAT_WINDOW: u32 = 1; +pub const FORMAT_FMA32: u32 = 2; +pub const FORMAT_F32: u32 = 3; + +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct MxmmCmd { + pub format: u32, + pub rob_id: u32, + pub iter: u32, + pub rs2_lo: u32, + pub rs2_hi: u32, + pub op1_bank: u32, + pub op2_bank: u32, + pub wr_bank: u32, + pub op1_group: u32, + pub op2_group: u32, + pub wr_group: u32, + pub op1_col: u32, + pub op2_col: u32, + pub wr_col: u32, +} + +#[derive(Clone, Debug)] +pub struct Preload { + pub bank: u32, + pub group: u32, + pub addr: u32, + pub data: u128, +} + +#[derive(Clone, Debug)] +pub struct Write { + pub addr: u32, + pub mask: u16, + pub data: u128, +} + +#[derive(Clone, Debug)] +pub struct Step { + pub cmd: MxmmCmd, + pub preload: Vec, + pub reads: [Vec; 2], + pub writes: Vec, +} + +#[derive(Clone, Debug)] +pub struct Case { + pub steps: Vec, +} + +struct Rng(u32); + +impl Rng { + fn next(&mut self) -> u32 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 17; + self.0 ^= self.0 << 5; + self.0 + } + + // E4M3 code other than NaN (0x7f, 0xff). + fn code(&mut self) -> u8 { + loop { + let code = self.next() as u8; + if code & 0x7f != 0x7f { + return code; + } + } + } + + // E8M0 scale near one, sometimes the smallest scale. + fn scale(&mut self) -> u8 { + if self.next() % 8 == 0 { + 0 + } else { + 120 + (self.next() % 14) as u8 + } + } + + fn float(&mut self) -> f32 { + let bits = (self.next() & 0x807f_ffff) | ((120 + self.next() % 12) << 23); + f32::from_bits(bits) + } +} + +fn word(bytes: &[u8]) -> u128 { + let mut out = 0u128; + for (lane, byte) in bytes.iter().enumerate() { + out |= u128::from(*byte) << (lane * 8); + } + out +} + +fn decode(code: u8, scale: u8) -> f32 { + assert!(code & 0x7f != 0x7f && scale != 255, "mxfp8: NaN encoding"); + let factor = if scale == 0 { f32::from_bits(1 << 22) } else { f32::from_bits((scale as u32) << 23) }; + let exponent = (code >> 3) & 15; + let mantissa = (code & 7) as u32; + let normal = ((exponent as u32 + 120) << 23) | (mantissa << 20); + let subnormal = (mantissa as f32 / 512.0).to_bits(); + let element = f32::from_bits(if exponent == 0 { subnormal } else { normal }); + let value = f32::from_bits((element * factor).to_bits() | ((code as u32 & 128) << 24)); + assert!(value.is_finite(), "mxfp8: decoded value overflows FP32"); + value +} + +// One operand bank: its byte image and the rows the Ball reads, in order. +struct Operand { + bytes: Vec, + reads: Vec, + // values[row][k] (A) or values[col][k] (B) seen by the command. + values: Vec>, +} + +fn sequential_reads(words: usize, start: usize) -> Vec { + (start..start + words).map(|w| w as u32).collect() +} + +fn mx_operand(rng: &mut Rng, rows: usize, k: usize) -> Operand { + let codes: Vec = (0..rows * k).map(|_| rng.code()).collect(); + let scales: Vec = (0..rows * k / 32).map(|_| rng.scale()).collect(); + let values = (0..rows) + .map(|r| (0..k).map(|i| decode(codes[r * k + i], scales[r * (k / 32) + i / 32])).collect()) + .collect(); + let mut bytes = codes; + bytes.extend_from_slice(&scales); + let code_words = rows * k / ROW_BYTES; + let mut reads = sequential_reads(code_words, 0); + reads.extend(sequential_reads((rows * k / 32).div_ceil(ROW_BYTES), code_words)); + Operand { bytes, reads, values } +} + +fn window_operand(image: &[u8], rows: usize, full_k: usize, start_k: usize, k: usize) -> Operand { + let scales = rows * full_k; + let values = (0..rows) + .map(|r| { + (0..k) + .map(|i| { + let col = start_k + i; + decode(image[r * full_k + col], image[scales + r * (full_k / 32) + col / 32]) + }) + .collect() + }) + .collect(); + let mut reads = Vec::new(); + for r in 0..rows { + for o in 0..k / ROW_BYTES { + reads.push((r * (full_k / ROW_BYTES) + start_k / ROW_BYTES + o) as u32); + } + } + for r in 0..rows { + let first = scales + r * (full_k / 32) + start_k / 32; + for o in 0..k / 32 { + if o == 0 || (first + o) % ROW_BYTES == 0 { + reads.push(((first + o) / ROW_BYTES) as u32); + } + } + } + Operand { bytes: image.to_vec(), reads, values } +} + +fn f32_operand(rng: &mut Rng, rows: usize, k: usize, zero_rows: &[usize]) -> Operand { + let values: Vec> = (0..rows) + .map(|r| (0..k).map(|_| if zero_rows.contains(&r) { 0.0 } else { rng.float() }).collect()) + .collect(); + let bytes: Vec = values.iter().flatten().flat_map(|x| x.to_le_bytes()).collect(); + let reads = sequential_reads(rows * k * 4 / ROW_BYTES, 0); + Operand { bytes, reads, values } +} + +fn mx_window_image(rng: &mut Rng, rows: usize, full_k: usize) -> Vec { + let mut image: Vec = (0..rows * full_k).map(|_| rng.code()).collect(); + image.extend((0..rows * full_k / 32).map(|_| rng.scale())); + image +} + +struct Command { + format: u32, + rob_id: u32, + banks: [u32; 3], + groups: [u32; 3], + m: usize, + n: usize, + k: usize, + first: bool, + last: bool, + base: usize, + full_k: usize, + start_k: usize, +} + +struct Chain { + format: u32, + m: usize, + n: usize, + bank: u32, + base: usize, + values: Vec, +} + +fn step(bank_entries: u32, chain: &mut Option, c: Command, a: Operand, b: Operand) -> Step { + let bank_bytes = bank_entries as usize * ROW_BYTES; + let mx = c.format == FORMAT_MXFP8 || c.format == FORMAT_WINDOW; + if !(c.m == 1 || (c.m > 0 && c.m % 16 == 0)) || c.n == 0 || c.n % 16 != 0 || c.k == 0 || c.k % (if mx { 32 } else { 4 }) != 0 { + panic!("mxmm: invalid shape"); + } + if c.format == FORMAT_WINDOW && (c.full_k == 0 || c.full_k % 32 != 0 || c.start_k % 32 != 0 || c.start_k + c.k > c.full_k) { + panic!("mxmm: invalid activation window"); + } + if c.format != FORMAT_WINDOW && (c.full_k != c.k || c.start_k != 0) { + panic!("mxmm: window fields are reserved"); + } + if c.banks[0] == c.banks[1] || c.banks[0] == c.banks[2] || c.banks[1] == c.banks[2] { + panic!("mxmm: banks must differ"); + } + if a.bytes.len() > bank_bytes || b.bytes.len() > bank_bytes || c.base * ROW_BYTES + c.m * c.n * 4 > bank_bytes { + panic!("mxmm: bank footprint exceeds depth"); + } + // The accumulator buffer holds bank_entries / 4 lines of 16 results. + if c.m * c.n > bank_entries as usize * 4 { + panic!("mxmm: result exceeds the accumulator buffer"); + } + if a.values.len() != c.m || b.values.len() != c.n || a.values.iter().chain(b.values.iter()).any(|v| v.len() != c.k) { + panic!("mxmm: operand shape mismatch"); + } + let mut live = if c.first { + if chain.is_some() { + panic!("mxmm: first while chain is live"); + } + Chain { format: c.format, m: c.m, n: c.n, bank: c.banks[2], base: c.base, values: vec![0f32; c.m * c.n] } + } else { + chain.take().expect("mxmm: continuation without first") + }; + if live.format != c.format || live.m != c.m || live.n != c.n || live.bank != c.banks[2] || live.base != c.base { + panic!("mxmm: continuation changed destination"); + } + for row in 0..c.m { + for col in 0..c.n { + let mut sum = live.values[row * c.n + col]; + for inner in 0..c.k { + let (x, y) = (a.values[row][inner], b.values[col][inner]); + sum = if c.format == FORMAT_F32 { (x * y) + sum } else { x.mul_add(y, sum) }; + } + assert!(sum.is_finite(), "mxmm: accumulator overflow"); + live.values[row * c.n + col] = sum; + } + } + let mut writes = Vec::new(); + if c.last { + let bytes: Vec = live.values.iter().flat_map(|x| x.to_le_bytes()).collect(); + for (w, chunk) in bytes.chunks(ROW_BYTES).enumerate() { + writes.push(Write { addr: (c.base + w) as u32, mask: 0xffff, data: word(chunk) }); + } + } else { + *chain = Some(live); + } + let mut preload = Vec::new(); + for (operand, image) in [&a.bytes, &b.bytes].into_iter().enumerate() { + let mut padded = image.clone(); + padded.resize(image.len().div_ceil(ROW_BYTES) * ROW_BYTES, 0); + for (row, chunk) in padded.chunks(ROW_BYTES).enumerate() { + preload.push(Preload { bank: c.banks[operand], group: c.groups[operand], addr: row as u32, data: word(chunk) }); + } + } + let mut rs2 = (c.m as u64) | ((c.n as u64) << 12) | (u64::from(c.first) << 24) | (u64::from(c.last) << 25) | ((c.base as u64) << 26); + if c.format == FORMAT_WINDOW { + rs2 |= ((c.full_k as u64) << 32) | ((c.start_k as u64) << 48); + } + Step { + cmd: MxmmCmd { + format: c.format, + rob_id: c.rob_id, + iter: c.k as u32, + rs2_lo: rs2 as u32, + rs2_hi: (rs2 >> 32) as u32, + op1_bank: c.banks[0], + op2_bank: c.banks[1], + wr_bank: c.banks[2], + op1_group: c.groups[0], + op2_group: c.groups[1], + wr_group: c.groups[2], + op1_col: 1, + op2_col: 1, + wr_col: 1, + }, + preload, + reads: [a.reads, b.reads], + writes, + } +} + +fn command(format: u32, rob_id: u32, banks: [u32; 3], groups: [u32; 3], m: usize, n: usize, k: usize, first: bool, last: bool, base: usize) -> Command { + Command { format, rob_id, banks, groups, m, n, k, first, last, base, full_k: k, start_k: 0 } +} + +pub fn gen_case(index: u32, bank_entries: u32) -> Case { + let mut rng = Rng(0x1234_5679 ^ index.wrapping_mul(0x9e37_79b9)); + let mut chain = None; + let mut steps = Vec::new(); + match index { + // One MXFP8 tile. + 0 => { + let (a, b) = (mx_operand(&mut rng, 16, 32), mx_operand(&mut rng, 16, 32)); + steps.push(step(bank_entries, &mut chain, command(FORMAT_MXFP8, 1, [1, 2, 3], [0, 0, 0], 16, 16, 32, true, true, 0), a, b)); + } + // MXFP8 vector (M = 1) over two scale blocks. + 1 => { + let (a, b) = (mx_operand(&mut rng, 1, 64), mx_operand(&mut rng, 32, 64)); + steps.push(step(bank_entries, &mut chain, command(FORMAT_MXFP8, 2, [4, 5, 6], [1, 2, 3], 1, 32, 64, true, true, 3), a, b)); + } + // FMA32 with two column groups. + 2 => { + let (a, b) = (f32_operand(&mut rng, 16, 8, &[]), f32_operand(&mut rng, 32, 8, &[])); + steps.push(step(bank_entries, &mut chain, command(FORMAT_FMA32, 3, [7, 8, 9], [4, 5, 6], 16, 32, 8, true, true, 1), a, b)); + } + // F32 (rounded product, rounded add) with two row groups and zero rows. + 3 => { + let (a, b) = (f32_operand(&mut rng, 32, 4, &[0, 17]), f32_operand(&mut rng, 16, 4, &[])); + steps.push(step(bank_entries, &mut chain, command(FORMAT_F32, 4, [10, 11, 12], [7, 8, 9], 32, 16, 4, true, true, 2), a, b)); + } + // MXFP8 chain: first, continuation, last. + 4 => { + for (i, (first, last)) in [(true, false), (false, false), (false, true)].into_iter().enumerate() { + let (a, b) = (mx_operand(&mut rng, 16, 32), mx_operand(&mut rng, 16, 32)); + steps.push(step(bank_entries, &mut chain, command(FORMAT_MXFP8, 5 + i as u32, [13, 14, 15], [10, 11, 12], 16, 16, 32, first, last, 5), a, b)); + } + } + // Sliding activation window over fullK = 96 accumulated as one chain. + 5 => { + let image = mx_window_image(&mut rng, 16, 96); + for (i, start_k) in [0usize, 32, 64].into_iter().enumerate() { + let a = window_operand(&image, 16, 96, start_k, 32); + let b = mx_operand(&mut rng, 16, 32); + let c = Command { full_k: 96, start_k, ..command(FORMAT_WINDOW, 8 + i as u32, [16, 17, 18], [13, 14, 15], 16, 16, 32, i == 0, i == 2, 7) }; + steps.push(step(bank_entries, &mut chain, c, a, b)); + } + // A single-command vector window over the upper half of fullK = 64. + let image = mx_window_image(&mut rng, 1, 64); + let a = window_operand(&image, 1, 64, 32, 32); + let b = mx_operand(&mut rng, 32, 32); + let c = Command { full_k: 64, start_k: 32, ..command(FORMAT_WINDOW, 15, [16, 17, 18], [13, 14, 15], 1, 32, 32, true, true, 4) }; + steps.push(step(bank_entries, &mut chain, c, a, b)); + } + // Eight MXFP8 panels (two batches of contexts) filling the C bank. + 6 => { + let (a, b) = (mx_operand(&mut rng, 32, 32), mx_operand(&mut rng, 64, 32)); + steps.push(step(bank_entries, &mut chain, command(FORMAT_MXFP8, 11, [1021, 1022, 1023], [31, 30, 29], 32, 64, 32, true, true, 0), a, b)); + } + // F32 chain, then an FMA32 vector after the chain closes. + 7 => { + for (i, (first, last)) in [(true, false), (false, true)].into_iter().enumerate() { + let (a, b) = (f32_operand(&mut rng, 16, 8, &[3]), f32_operand(&mut rng, 16, 8, &[])); + steps.push(step(bank_entries, &mut chain, command(FORMAT_F32, 12 + i as u32, [19, 20, 21], [16, 17, 18], 16, 16, 8, first, last, 9), a, b)); + } + let (a, b) = (f32_operand(&mut rng, 1, 12, &[]), f32_operand(&mut rng, 48, 12, &[])); + steps.push(step(bank_entries, &mut chain, command(FORMAT_FMA32, 14, [22, 23, 24], [19, 20, 21], 1, 48, 12, true, true, 0), a, b)); + } + _ => panic!("mxmm: unsupported case {index}"), + } + if chain.is_some() { + panic!("mxmm: case {index} leaves a live chain"); + } + Case { steps } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cases_build() { + for index in 0..NUM_CASES { + gen_case(index, 512); + } + } + + #[test] + fn decode_edges() { + assert_eq!(decode(0x38, 127), 1.0); + assert_eq!(decode(0x7e, 127), 448.0); + assert_eq!(decode(0x01, 127), 1.0 / 512.0); + assert_eq!(decode(0x80, 127).to_bits(), 0x8000_0000); + assert_eq!(decode(0x01, 0), f32::from_bits(1 << 22) / 512.0); + } +} diff --git a/examples/prefab/balls/mxmm/verify/casegen/src/config.rs b/examples/prefab/balls/mxmm/verify/casegen/src/config.rs new file mode 100644 index 000000000..241755f0f --- /dev/null +++ b/examples/prefab/balls/mxmm/verify/casegen/src/config.rs @@ -0,0 +1,9 @@ +pub fn bank_entries() -> u32 { + let path = std::env::var("BB_VERIFY_CONFIG").expect("BB_VERIFY_CONFIG is required"); + let text = std::fs::read_to_string(path).expect("failed to read BB_VERIFY_CONFIG"); + text.lines() + .find_map(|line| line.strip_prefix("bank_entries=")) + .unwrap_or_else(|| panic!("bank_entries missing from BB_VERIFY_CONFIG")) + .parse() + .expect("invalid bank_entries in BB_VERIFY_CONFIG") +} diff --git a/examples/prefab/balls/mxmm/verify/casegen/src/lib.rs b/examples/prefab/balls/mxmm/verify/casegen/src/lib.rs new file mode 100644 index 000000000..93d67090b --- /dev/null +++ b/examples/prefab/balls/mxmm/verify/casegen/src/lib.rs @@ -0,0 +1,116 @@ +mod casegen; +mod config; + +use std::cell::RefCell; + +use casegen::{Case, MxmmCmd, Step}; + +thread_local! { + static CURRENT: RefCell> = const { RefCell::new(None) }; +} + +#[no_mangle] +pub extern "C" fn mxmm_num_cases() -> u32 { + casegen::NUM_CASES +} + +#[no_mangle] +pub extern "C" fn mxmm_case_load(index: u32) -> u32 { + let case = casegen::gen_case(index, config::bank_entries()); + let steps = case.steps.len() as u32; + CURRENT.with(|current| *current.borrow_mut() = Some(case)); + steps +} + +fn current_step R, R>(step: u32, f: F) -> R { + CURRENT.with(|current| match current.borrow().as_ref() { + Some(case) => match case.steps.get(step as usize) { + Some(s) => f(s), + None => panic!("mxmm_case: step {step} out of range {}", case.steps.len()), + }, + None => panic!("mxmm_case: no case loaded; call mxmm_case_load first"), + }) +} + +fn idx(index: u32, limit: usize, what: &str) -> usize { + if index as usize >= limit { + panic!("mxmm_case_{what}: index {index} out of range {limit}"); + } + index as usize +} + +#[no_mangle] +pub extern "C" fn mxmm_case_cmd(step: u32, out_ptr: *mut MxmmCmd) { + if out_ptr.is_null() { + panic!("mxmm_case_cmd: null out_ptr"); + } + current_step(step, |s| unsafe { *out_ptr = s.cmd }); +} + +#[no_mangle] +pub extern "C" fn mxmm_case_preload_count(step: u32) -> u32 { + current_step(step, |s| s.preload.len() as u32) +} + +#[no_mangle] +pub extern "C" fn mxmm_case_preload_bank(step: u32, i: u32) -> u32 { + current_step(step, |s| s.preload[idx(i, s.preload.len(), "preload")].bank) +} + +#[no_mangle] +pub extern "C" fn mxmm_case_preload_group(step: u32, i: u32) -> u32 { + current_step(step, |s| s.preload[idx(i, s.preload.len(), "preload")].group) +} + +#[no_mangle] +pub extern "C" fn mxmm_case_preload_addr(step: u32, i: u32) -> u32 { + current_step(step, |s| s.preload[idx(i, s.preload.len(), "preload")].addr) +} + +#[no_mangle] +pub extern "C" fn mxmm_case_preload_lo(step: u32, i: u32) -> u64 { + current_step(step, |s| s.preload[idx(i, s.preload.len(), "preload")].data as u64) +} + +#[no_mangle] +pub extern "C" fn mxmm_case_preload_hi(step: u32, i: u32) -> u64 { + current_step(step, |s| (s.preload[idx(i, s.preload.len(), "preload")].data >> 64) as u64) +} + +#[no_mangle] +pub extern "C" fn mxmm_case_read_count(step: u32, port: u32) -> u32 { + current_step(step, |s| s.reads[idx(port, 2, "read_port")].len() as u32) +} + +#[no_mangle] +pub extern "C" fn mxmm_case_read_addr(step: u32, port: u32, i: u32) -> u32 { + current_step(step, |s| { + let reads = &s.reads[idx(port, 2, "read_port")]; + reads[idx(i, reads.len(), "read")] + }) +} + +#[no_mangle] +pub extern "C" fn mxmm_case_write_count(step: u32) -> u32 { + current_step(step, |s| s.writes.len() as u32) +} + +#[no_mangle] +pub extern "C" fn mxmm_case_write_addr(step: u32, i: u32) -> u32 { + current_step(step, |s| s.writes[idx(i, s.writes.len(), "write")].addr) +} + +#[no_mangle] +pub extern "C" fn mxmm_case_write_mask(step: u32, i: u32) -> u32 { + current_step(step, |s| u32::from(s.writes[idx(i, s.writes.len(), "write")].mask)) +} + +#[no_mangle] +pub extern "C" fn mxmm_case_write_lo(step: u32, i: u32) -> u64 { + current_step(step, |s| s.writes[idx(i, s.writes.len(), "write")].data as u64) +} + +#[no_mangle] +pub extern "C" fn mxmm_case_write_hi(step: u32, i: u32) -> u64 { + current_step(step, |s| (s.writes[idx(i, s.writes.len(), "write")].data >> 64) as u64) +} diff --git a/examples/prefab/balls/mxmm/verify/filelists/mxmm_ball.f b/examples/prefab/balls/mxmm/verify/filelists/mxmm_ball.f new file mode 100644 index 000000000..fe5169644 --- /dev/null +++ b/examples/prefab/balls/mxmm/verify/filelists/mxmm_ball.f @@ -0,0 +1,40 @@ ++incdir+@UVM@/src/core ++incdir+@UVM@/src/ball ++incdir+@UVM@/src/ball/agents/cmd ++incdir+@UVM@/src/ball/agents/mem ++incdir+@UVM@/src/ball/agents/resp ++incdir+@UVM@/src/ball/cov ++incdir+@UVM@/src/ball/env ++incdir+src ++incdir+src/pkg ++incdir+src/common ++incdir+src/seq ++incdir+src/cov ++incdir+src/env ++incdir+src/tests ++incdir+@RTL@ + +@UVM@/src/core/package.sv +@UVM@/src/ball/bb_blink_if.sv +@UVM@/src/ball/bb_uvm_pkg.sv + +@RTL@/MulAddRecFNToRaw_preMul_e8_s24.sv +@RTL@/MulAddRecFNToRaw_postMul_e8_s24.sv +@RTL@/RoundAnyRawFNToRecFN_ie8_is26_oe8_os24.sv +@RTL@/RoundRawFNToRecFN_e8_s24.sv +@RTL@/Fma_l3_e8_s24.sv +@RTL@/AddRawFN.sv +@RTL@/AddPipe.sv +@RTL@/PE.sv +@RTL@/Array.sv +@RTL@/Mxfp8Decode.sv +@RTL@/codes_32x128.sv +@RTL@/scales_16x8.sv +@RTL@/Panels.sv +@RTL@/InputWindow.sv +@RTL@/accumulator_128x512.sv +@RTL@/Unit.sv +@RTL@/MxmmBall.sv + +src/pkg/mxmm_pkg.sv +src/tb_top.sv diff --git a/examples/prefab/balls/mxmm/verify/src/common/mxmm_defs.svh b/examples/prefab/balls/mxmm/verify/src/common/mxmm_defs.svh new file mode 100644 index 000000000..d58c20f3c --- /dev/null +++ b/examples/prefab/balls/mxmm/verify/src/common/mxmm_defs.svh @@ -0,0 +1,51 @@ +typedef struct { + int unsigned format; + int unsigned rob_id; + int unsigned iter; + int unsigned rs2_lo; + int unsigned rs2_hi; + int unsigned op1_bank; + int unsigned op2_bank; + int unsigned wr_bank; + int unsigned op1_group; + int unsigned op2_group; + int unsigned wr_group; + int unsigned op1_col; + int unsigned op2_col; + int unsigned wr_col; +} mxmm_cmd_dpi_t; + +import "DPI-C" function int unsigned mxmm_num_cases(); +import "DPI-C" function int unsigned mxmm_case_load(input int unsigned index); +import "DPI-C" function void mxmm_case_cmd(input int unsigned step, output mxmm_cmd_dpi_t cmd); +import "DPI-C" function int unsigned mxmm_case_preload_count(input int unsigned step); +import "DPI-C" function int unsigned mxmm_case_preload_bank(input int unsigned step, input int unsigned i); +import "DPI-C" function int unsigned mxmm_case_preload_group(input int unsigned step, input int unsigned i); +import "DPI-C" function int unsigned mxmm_case_preload_addr(input int unsigned step, input int unsigned i); +import "DPI-C" function longint unsigned mxmm_case_preload_lo(input int unsigned step, input int unsigned i); +import "DPI-C" function longint unsigned mxmm_case_preload_hi(input int unsigned step, input int unsigned i); +import "DPI-C" function int unsigned mxmm_case_read_count(input int unsigned step, input int unsigned port); +import "DPI-C" function int unsigned mxmm_case_read_addr(input int unsigned step, input int unsigned port, input int unsigned i); +import "DPI-C" function int unsigned mxmm_case_write_count(input int unsigned step); +import "DPI-C" function int unsigned mxmm_case_write_addr(input int unsigned step, input int unsigned i); +import "DPI-C" function int unsigned mxmm_case_write_mask(input int unsigned step, input int unsigned i); +import "DPI-C" function longint unsigned mxmm_case_write_lo(input int unsigned step, input int unsigned i); +import "DPI-C" function longint unsigned mxmm_case_write_hi(input int unsigned step, input int unsigned i); + +`ifndef MXMM_MXFP8_FUNCT7 +`error "MXMM_MXFP8_FUNCT7 must be provided by the selected ballISA" +`endif +`ifndef MXMM_MXFP8_WINDOW_FUNCT7 +`error "MXMM_MXFP8_WINDOW_FUNCT7 must be provided by the selected ballISA" +`endif +`ifndef MXMM_FMA32_FUNCT7 +`error "MXMM_FMA32_FUNCT7 must be provided by the selected ballISA" +`endif +`ifndef MXMM_F32_FUNCT7 +`error "MXMM_F32_FUNCT7 must be provided by the selected ballISA" +`endif + +// Formats of the reference model, in the order of mxmm_case_cmd.format. +localparam int MXMM_FUNCT7[4] = '{`MXMM_MXFP8_FUNCT7, `MXMM_MXFP8_WINDOW_FUNCT7, `MXMM_FMA32_FUNCT7, + `MXMM_F32_FUNCT7}; +localparam int MXMM_TIMEOUT_CYCLES = 100000; diff --git a/examples/prefab/balls/mxmm/verify/src/common/mxmm_items.svh b/examples/prefab/balls/mxmm/verify/src/common/mxmm_items.svh new file mode 100644 index 000000000..f4219db6c --- /dev/null +++ b/examples/prefab/balls/mxmm/verify/src/common/mxmm_items.svh @@ -0,0 +1,89 @@ +class mxmm_cmd_item extends bb_blink_cmd_item; + `uvm_object_utils(mxmm_cmd_item) + + int unsigned format; + int unsigned preload_bank[]; + int unsigned preload_group[]; + int unsigned preload_addr[]; + bit [127:0] preload_data[]; + int unsigned read_addr0[]; + int unsigned read_addr1[]; + int unsigned write_addr[]; + bit [15:0] write_mask[]; + bit [127:0] write_data[]; + + function new(string name = "mxmm_cmd_item"); + super.new(name); + endfunction + + function void load_rust_step(int unsigned step); + mxmm_cmd_dpi_t cmd; + int unsigned n; + + mxmm_case_cmd(step, cmd); + rob_id = cmd.rob_id[BB_BLINK_ROB_ID_W-1:0]; + if (cmd.format > 3) `uvm_fatal("CASE", $sformatf("invalid format %0d", cmd.format)) + format = cmd.format; + funct = MXMM_FUNCT7[cmd.format]; + rs2 = {cmd.rs2_hi, cmd.rs2_lo}; + iter = cmd.iter; + op1_bank = cmd.op1_bank; + op2_bank = cmd.op2_bank; + wr_bank = cmd.wr_bank; + op1_group = cmd.op1_group; + op2_group = cmd.op2_group; + wr_group = cmd.wr_group; + op1_col = cmd.op1_col; + op2_col = cmd.op2_col; + wr_col = cmd.wr_col; + wr_read_bank = '0; + wr_arch_bank = cmd.wr_bank; + if (rob_id != cmd.rob_id || op1_bank != cmd.op1_bank || op2_bank != cmd.op2_bank || + wr_bank != cmd.wr_bank || op1_group != cmd.op1_group || op2_group != cmd.op2_group || + wr_group != cmd.wr_group) + `uvm_fatal("CASE", "command field exceeds the configured BLink width") + + n = mxmm_case_preload_count(step); + preload_bank = new[n]; + preload_group = new[n]; + preload_addr = new[n]; + preload_data = new[n]; + for (int unsigned i = 0; i < n; i++) begin + preload_bank[i] = mxmm_case_preload_bank(step, i); + preload_group[i] = mxmm_case_preload_group(step, i); + preload_addr[i] = mxmm_case_preload_addr(step, i); + preload_data[i] = {mxmm_case_preload_hi(step, i), mxmm_case_preload_lo(step, i)}; + end + n = mxmm_case_read_count(step, 0); + read_addr0 = new[n]; + for (int unsigned i = 0; i < n; i++) read_addr0[i] = mxmm_case_read_addr(step, 0, i); + n = mxmm_case_read_count(step, 1); + read_addr1 = new[n]; + for (int unsigned i = 0; i < n; i++) read_addr1[i] = mxmm_case_read_addr(step, 1, i); + n = mxmm_case_write_count(step); + write_addr = new[n]; + write_mask = new[n]; + write_data = new[n]; + for (int unsigned i = 0; i < n; i++) begin + write_addr[i] = mxmm_case_write_addr(step, i); + write_mask[i] = mxmm_case_write_mask(step, i); + write_data[i] = {mxmm_case_write_hi(step, i), mxmm_case_write_lo(step, i)}; + end + endfunction + + function void do_copy(uvm_object rhs); + mxmm_cmd_item rhs_; + super.do_copy(rhs); + if (!$cast(rhs_, rhs)) `uvm_fatal("COPY", "rhs is not mxmm_cmd_item") + format = rhs_.format; + preload_bank = rhs_.preload_bank; + preload_group = rhs_.preload_group; + preload_addr = rhs_.preload_addr; + preload_data = rhs_.preload_data; + read_addr0 = rhs_.read_addr0; + read_addr1 = rhs_.read_addr1; + write_addr = rhs_.write_addr; + write_mask = rhs_.write_mask; + write_data = rhs_.write_data; + endfunction +endclass diff --git a/examples/prefab/balls/mxmm/verify/src/cov/mxmm_cov.svh b/examples/prefab/balls/mxmm/verify/src/cov/mxmm_cov.svh new file mode 100644 index 000000000..3fb16b408 --- /dev/null +++ b/examples/prefab/balls/mxmm/verify/src/cov/mxmm_cov.svh @@ -0,0 +1,59 @@ +class mxmm_cov extends uvm_component; + `uvm_component_utils(mxmm_cov) + + uvm_analysis_imp_stim #(bb_blink_cmd_item, mxmm_cov) stim_imp; + + int unsigned cur_format; + bit [1:0] cur_chain; + int unsigned cur_m; + int unsigned cur_panels; + int unsigned cmd_count; + + covergroup cmd_cg; + coverpoint cur_format {bins mxfp8 = {0}; bins window = {1}; bins fma32 = {2}; bins f32 = {3};} + // {first, last} + coverpoint cur_chain { + bins single = {2'b11}; bins open = {2'b10}; bins middle = {2'b00}; bins close = {2'b01}; + } + coverpoint cur_m {bins vector = {1}; bins tile = {16}; bins tiles = {[32 : 4095]};} + // Panels beyond the seven contexts need a second batch. + coverpoint cur_panels { + bins one = {1}; bins batch = {[2 : 7]}; bins batches = {[8 : 4095]}; + } + format_chain: cross cur_format, cur_chain{ + ignore_bins fused_chain = binsof (cur_format) intersect {2} && !binsof (cur_chain) intersect { + 2'b11 + }; + ignore_bins f32_middle = binsof (cur_format) intersect {3} && binsof (cur_chain) intersect { + 2'b00 + }; + } + endgroup + + function new(string name, uvm_component parent); + super.new(name, parent); + stim_imp = new("stim_imp", this); + cmd_cg = new(); + endfunction + + function void write_stim(bb_blink_cmd_item item); + mxmm_cmd_item stim; + int unsigned n; + if (!$cast(stim, item)) `uvm_fatal("COV", "stim item is not mxmm_cmd_item") + cur_format = stim.format; + cur_chain = {stim.rs2[24], stim.rs2[25]}; + cur_m = stim.rs2[11:0]; + n = stim.rs2[23:12]; + cur_panels = (cur_m == 1 ? 1 : cur_m / 16) * (n / 16); + cmd_cg.sample(); + cmd_count++; + endfunction + + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (cmd_count == 0) `uvm_fatal("COV", "no cmd observed") + if (cmd_cg.get_coverage() != 100.0) + `uvm_fatal("COV", $sformatf("mxmm command coverage %0.2f%%", cmd_cg.get_coverage())) + endfunction +endclass diff --git a/examples/prefab/balls/mxmm/verify/src/env/mxmm_env.svh b/examples/prefab/balls/mxmm/verify/src/env/mxmm_env.svh new file mode 100644 index 000000000..f8a602322 --- /dev/null +++ b/examples/prefab/balls/mxmm/verify/src/env/mxmm_env.svh @@ -0,0 +1,27 @@ +class mxmm_env extends bb_blink_env #(`BB_IN_BW, `BB_OUT_BW); + `uvm_component_utils(mxmm_env) + + mxmm_scoreboard scb; + mxmm_cov icov; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + scb = mxmm_scoreboard::type_id::create("scb", this); + icov = mxmm_cov::type_id::create("icov", this); + endfunction + + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + scb.mem_model = mem_model; + cmd_agent.stim_ap.connect(scb.stim_imp); + cmd_agent.stim_ap.connect(icov.stim_imp); + cmd_agent.cmd_ap.connect(scb.cmd_imp); + read_mon.read_ap.connect(scb.read_imp); + write_mon.write_ap.connect(scb.write_imp); + resp_mon.resp_ap.connect(scb.resp_imp); + endfunction +endclass diff --git a/examples/prefab/balls/mxmm/verify/src/env/mxmm_scoreboard.svh b/examples/prefab/balls/mxmm/verify/src/env/mxmm_scoreboard.svh new file mode 100644 index 000000000..82085ca03 --- /dev/null +++ b/examples/prefab/balls/mxmm/verify/src/env/mxmm_scoreboard.svh @@ -0,0 +1,196 @@ +class mxmm_scoreboard extends uvm_scoreboard; + `uvm_component_utils(mxmm_scoreboard) + + uvm_analysis_imp_stim #(bb_blink_cmd_item, mxmm_scoreboard) stim_imp; + uvm_analysis_imp_cmd #(bb_blink_cmd_item, mxmm_scoreboard) cmd_imp; + uvm_analysis_imp_read #(bb_blink_read_item, mxmm_scoreboard) read_imp; + uvm_analysis_imp_write #(bb_blink_write_item, mxmm_scoreboard) write_imp; + uvm_analysis_imp_resp #(bb_blink_resp_item, mxmm_scoreboard) resp_imp; + + bb_blink_mem_model #(`BB_IN_BW, `BB_OUT_BW) mem_model; + + mxmm_cmd_item stim_q[$]; + int unsigned cmd_count; + int unsigned read_count[2]; + int unsigned write_count; + int unsigned resp_count; + + function new(string name, uvm_component parent); + super.new(name, parent); + stim_imp = new("stim_imp", this); + cmd_imp = new("cmd_imp", this); + read_imp = new("read_imp", this); + write_imp = new("write_imp", this); + resp_imp = new("resp_imp", this); + endfunction + + function void write_stim(bb_blink_cmd_item item); + mxmm_cmd_item got; + mxmm_cmd_item clone; + + if (stim_q.size() != 0) `uvm_fatal("SCB", "single outstanding command supported") + if (!$cast(got, item)) `uvm_fatal("SCB", "stim item is not mxmm_cmd_item") + if (!$cast(clone, got.clone())) `uvm_fatal("SCB", "failed to clone stimulus item") + stim_q.push_back(clone); + if (mem_model == null) `uvm_fatal("SCB", "mem_model handle not set") + mem_model.clear_mem(); + foreach (clone.preload_data[i]) + mem_model.load_word_g(clone.preload_bank[i], clone.preload_group[i], clone.preload_addr[i], + clone.preload_data[i]); + mem_model.arm(); + endfunction + + function void write_cmd(bb_blink_cmd_item item); + mxmm_cmd_item stim; + bb_blink_cmd_item want; + stim = current_stim("CMD"); + // Compare as plain BLink commands; the comparer rejects differing types. + want = bb_blink_cmd_item::type_id::create("want"); + want.copy(stim); + verification_pkg::evidence::checks++; + if (!item.compare(want)) + `uvm_fatal("CMD", $sformatf("command mismatch: got %s exp %s", item.sprint(), want.sprint())) + cmd_count++; + endfunction + + function void write_read(bb_blink_read_item item); + mxmm_cmd_item stim; + int unsigned exp_bank; + int unsigned exp_group; + int unsigned exp_addr; + + stim = current_stim("READ"); + if (item.port != 0 && item.port != 1) + `uvm_fatal("READ", $sformatf("MxmmBall has two read ports, got port %0d", item.port)) + if (item.port == 0) begin + if (read_count[0] >= stim.read_addr0.size()) + `uvm_fatal("READ", $sformatf("extra read on port 0 at %0d", read_count[0])) + exp_bank = stim.op1_bank; + exp_group = stim.op1_group; + exp_addr = stim.read_addr0[read_count[0]]; + end else begin + if (read_count[1] >= stim.read_addr1.size()) + `uvm_fatal("READ", $sformatf("extra read on port 1 at %0d", read_count[1])) + exp_bank = stim.op2_bank; + exp_group = stim.op2_group; + exp_addr = stim.read_addr1[read_count[1]]; + end + verification_pkg::evidence::checks++; + if (item.rob_id !== stim.rob_id) + `uvm_fatal("READ", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + if (item.bank_id !== exp_bank[BB_BLINK_BANK_ID_W-1:0]) + `uvm_fatal("READ", $sformatf( + "port %0d bank mismatch: got %0d exp %0d", item.port, item.bank_id, exp_bank)) + if (item.group_id !== exp_group[BB_BLINK_GROUP_ID_W-1:0]) + `uvm_fatal("READ", $sformatf( + "port %0d group mismatch: got %0d exp %0d", item.port, item.group_id, exp_group)) + if (item.addr !== exp_addr[BB_BLINK_BANK_ADDR_W-1:0]) + `uvm_fatal("READ", $sformatf( + "port %0d read %0d addr mismatch: got %0d exp %0d", + item.port, + read_count[item.port], + item.addr, + exp_addr + )) + read_count[item.port]++; + endfunction + + function void write_write(bb_blink_write_item item); + mxmm_cmd_item stim; + bit [127:0] bytes; + + stim = current_stim("WRITE"); + if (item.port != 0) + `uvm_fatal("WRITE", $sformatf("MxmmBall has one write port, got port %0d", item.port)) + if (write_count >= stim.write_addr.size()) + `uvm_fatal("WRITE", $sformatf("extra write %0d", write_count)) + verification_pkg::evidence::checks++; + if (item.bank_id !== stim.wr_bank) + `uvm_fatal("WRITE", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.wr_bank)) + if (item.group_id !== stim.wr_group) + `uvm_fatal("WRITE", $sformatf("group mismatch: got %0d exp %0d", item.group_id, stim.wr_group + )) + if (item.rob_id !== stim.rob_id) + `uvm_fatal("WRITE", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + if (item.addr !== stim.write_addr[write_count][BB_BLINK_BANK_ADDR_W-1:0]) + `uvm_fatal("WRITE", $sformatf( + "write %0d addr mismatch: got %0d exp %0d", + write_count, + item.addr, + stim.write_addr[write_count] + )) + if (item.mask !== stim.write_mask[write_count]) + `uvm_fatal("WRITE", $sformatf( + "write %0d mask mismatch: got 0x%04h exp 0x%04h", + write_count, + item.mask, + stim.write_mask[write_count] + )) + for (int b = 0; b < 16; b++) bytes[b*8+:8] = {8{item.mask[b]}}; + if ((item.data & bytes) !== (stim.write_data[write_count] & bytes)) + `uvm_fatal("SCB", $sformatf( + "write %0d data mismatch at addr %0d: got 0x%032h exp 0x%032h", + write_count, + item.addr, + item.data, + stim.write_data[write_count] + )) + write_count++; + endfunction + + function void write_resp(bb_blink_resp_item item); + mxmm_cmd_item stim; + stim = current_stim("RESP"); + verification_pkg::evidence::checks++; + if (item.rob_id !== stim.rob_id) + `uvm_fatal("RESP", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + if (!complete_io()) + `uvm_fatal("RESP", $sformatf( + "response before all traffic: reads=%0d,%0d writes=%0d", + read_count[0], + read_count[1], + write_count + )) + resp_count++; + endfunction + + function mxmm_cmd_item current_stim(string tag); + if (stim_q.size() == 0) `uvm_fatal("SCB", $sformatf("%s observed before stimulus", tag)) + return stim_q[0]; + endfunction + + function bit complete_io(); + mxmm_cmd_item stim = stim_q[0]; + return read_count[0] == stim.read_addr0.size() && read_count[1] == stim.read_addr1.size() && + write_count == stim.write_addr.size(); + endfunction + + function bit done(); + return stim_q.size() == 1 && cmd_count == 1 && resp_count == 1 && complete_io(); + endfunction + + function void reset_counters(); + stim_q.delete(); + cmd_count = 0; + read_count[0] = 0; + read_count[1] = 0; + write_count = 0; + resp_count = 0; + endfunction + + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (!done()) + `uvm_fatal("SCB", $sformatf( + "incomplete: stim=%0d cmds=%0d reads=%0d,%0d writes=%0d resp=%0d", + stim_q.size(), + cmd_count, + read_count[0], + read_count[1], + write_count, + resp_count + )) + stim_q.delete(); + endfunction +endclass diff --git a/examples/prefab/balls/mxmm/verify/src/pkg/mxmm_pkg.sv b/examples/prefab/balls/mxmm/verify/src/pkg/mxmm_pkg.sv new file mode 100644 index 000000000..2f70bd272 --- /dev/null +++ b/examples/prefab/balls/mxmm/verify/src/pkg/mxmm_pkg.sv @@ -0,0 +1,13 @@ +package mxmm_pkg; + import uvm_pkg::*; + import bb_uvm_pkg::*; + `include "uvm_macros.svh" + + `include "common/mxmm_defs.svh" + `include "common/mxmm_items.svh" + `include "seq/mxmm_sequences.svh" + `include "cov/mxmm_cov.svh" + `include "env/mxmm_scoreboard.svh" + `include "env/mxmm_env.svh" + `include "tests/mxmm_test.svh" +endpackage diff --git a/examples/prefab/balls/mxmm/verify/src/seq/mxmm_sequences.svh b/examples/prefab/balls/mxmm/verify/src/seq/mxmm_sequences.svh new file mode 100644 index 000000000..b73f95b80 --- /dev/null +++ b/examples/prefab/balls/mxmm/verify/src/seq/mxmm_sequences.svh @@ -0,0 +1,17 @@ +class mxmm_step_seq extends uvm_sequence #(bb_blink_cmd_item); + `uvm_object_utils(mxmm_step_seq) + + int unsigned step; + + function new(string name = "mxmm_step_seq"); + super.new(name); + endfunction + + task body(); + mxmm_cmd_item item; + item = mxmm_cmd_item::type_id::create("item"); + start_item(item); + item.load_rust_step(step); + finish_item(item); + endtask +endclass diff --git a/examples/prefab/balls/mxmm/verify/src/tb_top.sv b/examples/prefab/balls/mxmm/verify/src/tb_top.sv new file mode 100644 index 000000000..5ee20f2e4 --- /dev/null +++ b/examples/prefab/balls/mxmm/verify/src/tb_top.sv @@ -0,0 +1,89 @@ +module tb_top; + import uvm_pkg::*; + import mxmm_pkg::*; + + bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); + + MxmmBall dut ( + .clock(intf.clock), + .reset(intf.reset), + .blink_status_running(intf.status_running), + .blink_cmdReq_ready(intf.cmd_req_ready), + .blink_cmdReq_valid(intf.cmd_req_valid), + .blink_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .blink_cmdReq_bits_funct(intf.cmd_req_bits_funct), + .blink_cmdReq_bits_rs2(intf.cmd_req_bits_rs2), + .blink_cmdReq_bits_iter(intf.cmd_req_bits_iter), + .blink_cmdReq_bits_op1_bank(intf.cmd_req_bits_op1_bank), + .blink_cmdReq_bits_op2_bank(intf.cmd_req_bits_op2_bank), + .blink_cmdReq_bits_wr_bank(intf.cmd_req_bits_wr_bank), + .blink_cmdReq_bits_op1_group(intf.cmd_req_bits_op1_group), + .blink_cmdReq_bits_op2_group(intf.cmd_req_bits_op2_group), + .blink_cmdReq_bits_wr_group(intf.cmd_req_bits_wr_group), + .blink_cmdReq_bits_op1_col(intf.cmd_req_bits_op1_col), + .blink_cmdReq_bits_op2_col(intf.cmd_req_bits_op2_col), + .blink_cmdReq_bits_wr_col(intf.cmd_req_bits_wr_col), + .blink_cmdReq_bits_wr_read_bank(intf.cmd_req_bits_wr_read_bank), + .blink_cmdReq_bits_wr_arch_bank(intf.cmd_req_bits_wr_arch_bank), + .blink_cmdResp_ready(intf.cmd_resp_ready), + .blink_cmdResp_valid(intf.cmd_resp_valid), + .blink_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .blink_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .blink_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .blink_bankRead_0_group_id(intf.bank_read_group_id[0]), + .blink_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .blink_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .blink_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .blink_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .blink_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .blink_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .blink_bankRead_1_bank_id(intf.bank_read_bank_id[1]), + .blink_bankRead_1_rob_id(intf.bank_read_rob_id[1]), + .blink_bankRead_1_group_id(intf.bank_read_group_id[1]), + .blink_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), + .blink_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), + .blink_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), + .blink_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), + .blink_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), + .blink_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), + .blink_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .blink_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .blink_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .blink_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .blink_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .blink_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .blink_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .blink_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .blink_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .blink_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]) + ); + + // Constant outputs removed from the port list by firtool. + assign intf.bank_write_req_mask[0][0] = dut.blink_bankWrite_0_io_req_bits_mask_0; + assign intf.bank_write_req_mask[0][1] = dut.blink_bankWrite_0_io_req_bits_mask_1; + assign intf.bank_write_req_mask[0][2] = dut.blink_bankWrite_0_io_req_bits_mask_2; + assign intf.bank_write_req_mask[0][3] = dut.blink_bankWrite_0_io_req_bits_mask_3; + assign intf.bank_write_req_mask[0][4] = dut.blink_bankWrite_0_io_req_bits_mask_4; + assign intf.bank_write_req_mask[0][5] = dut.blink_bankWrite_0_io_req_bits_mask_5; + assign intf.bank_write_req_mask[0][6] = dut.blink_bankWrite_0_io_req_bits_mask_6; + assign intf.bank_write_req_mask[0][7] = dut.blink_bankWrite_0_io_req_bits_mask_7; + assign intf.bank_write_req_mask[0][8] = dut.blink_bankWrite_0_io_req_bits_mask_8; + assign intf.bank_write_req_mask[0][9] = dut.blink_bankWrite_0_io_req_bits_mask_9; + assign intf.bank_write_req_mask[0][10] = dut.blink_bankWrite_0_io_req_bits_mask_10; + assign intf.bank_write_req_mask[0][11] = dut.blink_bankWrite_0_io_req_bits_mask_11; + assign intf.bank_write_req_mask[0][12] = dut.blink_bankWrite_0_io_req_bits_mask_12; + assign intf.bank_write_req_mask[0][13] = dut.blink_bankWrite_0_io_req_bits_mask_13; + assign intf.bank_write_req_mask[0][14] = dut.blink_bankWrite_0_io_req_bits_mask_14; + assign intf.bank_write_req_mask[0][15] = dut.blink_bankWrite_0_io_req_bits_mask_15; + + initial begin + intf.clock = 1'b0; + intf.reset = 1'b1; + forever #5 intf.clock = ~intf.clock; + end + + initial begin + uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); + run_test("mxmm_ball_test"); + end +endmodule diff --git a/examples/prefab/balls/mxmm/verify/src/tests/mxmm_test.svh b/examples/prefab/balls/mxmm/verify/src/tests/mxmm_test.svh new file mode 100644 index 000000000..6af64328d --- /dev/null +++ b/examples/prefab/balls/mxmm/verify/src/tests/mxmm_test.svh @@ -0,0 +1,56 @@ +class mxmm_ball_test extends uvm_test; + `uvm_component_utils(mxmm_ball_test) + + typedef virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) vif_t; + vif_t vif; + mxmm_env env; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(vif_t)::get(this, "", "vif", vif)) + `uvm_fatal("NOVIF", "bb_blink_if not found") + env = mxmm_env::type_id::create("env", this); + endfunction + + task run_phase(uvm_phase phase); + phase.raise_objection(this); + for (int unsigned index = 0; index < mxmm_num_cases(); index++) run_case(index); + phase.drop_objection(this); + endtask + + // Steps of one case run back to back on the same reset. + task run_case(int unsigned index); + int unsigned steps; + apply_reset(); + steps = mxmm_case_load(index); + for (int unsigned step = 0; step < steps; step++) run_step(index, step); + endtask + + task run_step(int unsigned index, int unsigned step); + mxmm_step_seq seq; + int cycles = 0; + env.scb.reset_counters(); + seq = mxmm_step_seq::type_id::create("seq"); + seq.step = step; + seq.start(env.cmd_agent.seqr); + while (!env.scb.done()) begin + @(posedge vif.clock); + cycles++; + if (cycles > MXMM_TIMEOUT_CYCLES) + `uvm_fatal("TIMEOUT", $sformatf("MxmmBall case %0d step %0d timeout", index, step)) + end + `uvm_info("MXMM", $sformatf("MxmmBall case %0d step %0d passed in %0d cycles", index, step, + cycles), UVM_LOW) + endtask + + task apply_reset(); + vif.reset = 1'b1; + repeat (5) @(posedge vif.clock); + vif.reset = 1'b0; + repeat (2) @(posedge vif.clock); + endtask +endclass diff --git a/examples/prefab/balls/mxmm/workloads/ctests/CMakeLists.txt b/examples/prefab/balls/mxmm/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..4c98c223a --- /dev/null +++ b/examples/prefab/balls/mxmm/workloads/ctests/CMakeLists.txt @@ -0,0 +1,17 @@ +add_buckyball_ctest(mxmm_fp32_test.c) +add_buckyball_ctest(mxmm_mxfp8_test.c) +add_buckyball_ctest(mxmm_bank_capacity_test.c) + +math( + EXPR MXMM_BANK_BYTES + "${DESIGN_PROFILES_${BUCKYBALL_CTEST_TARGET}_THREAD_MEM_BANK_ENTRIES} * ${DESIGN_PROFILES_${BUCKYBALL_CTEST_TARGET}_THREAD_MEM_BANK_WIDTH} / 8" +) +if(MXMM_BANK_BYTES LESS 15840) + message( + FATAL_ERROR + "Mxmm A-window row test requires 15840-byte banks; actual ${MXMM_BANK_BYTES} bytes" + ) +endif() +add_buckyball_ctest(mxmm_window_test.c) +add_buckyball_ctest(mxmm_window_rows_test.c) +add_buckyball_ctest(mxmm_window_chain_test.c) diff --git a/examples/balls/mxmm/workloads/ctests/mxmm_bank_capacity_test.c b/examples/prefab/balls/mxmm/workloads/ctests/mxmm_bank_capacity_test.c similarity index 77% rename from examples/balls/mxmm/workloads/ctests/mxmm_bank_capacity_test.c rename to examples/prefab/balls/mxmm/workloads/ctests/mxmm_bank_capacity_test.c index 316cbc764..21eec3810 100644 --- a/examples/balls/mxmm/workloads/ctests/mxmm_bank_capacity_test.c +++ b/examples/prefab/balls/mxmm/workloads/ctests/mxmm_bank_capacity_test.c @@ -5,7 +5,10 @@ #include #include -enum { BANK_BYTES = BANK_LINES * BANK_WIDTH / 8, ELEMENTS = BANK_BYTES / 4 }; +enum { + BANK_BYTES = THREAD_MEM_BANK_ENTRIES * THREAD_MEM_BANK_WIDTH / 8, + ELEMENTS = BANK_BYTES / 4 +}; static float a[ELEMENTS] __attribute__((aligned(BANK_BYTES))); static float b[ELEMENTS] __attribute__((aligned(BANK_BYTES))); static uint32_t result[ELEMENTS] __attribute__((aligned(BANK_BYTES))); @@ -23,14 +26,14 @@ int main(void) { } for (int bank = 3; bank <= 5; ++bank) bb_mem_alloc(bank, 1, 1); - bb_mvin((uintptr_t)a, 3, BANK_LINES, 1); - bb_mvin((uintptr_t)b, 4, BANK_LINES, 1); - bb_mvin((uintptr_t)result, 5, BANK_LINES, 1); + bb_mvin((uintptr_t)a, 3, THREAD_MEM_BANK_ENTRIES, 1); + bb_mvin((uintptr_t)b, 4, THREAD_MEM_BANK_ENTRIES, 1); + bb_mvin((uintptr_t)result, 5, THREAD_MEM_BANK_ENTRIES, 1); if (fused) bb_mxmm_fma32(3, 4, 5, m, n, k, 1, 1, 0); else bb_mxmm_f32(3, 4, 5, m, n, k, 1, 1, 0); - bb_mvout((uintptr_t)result, 5, BANK_LINES, 1); + bb_mvout((uintptr_t)result, 5, THREAD_MEM_BANK_ENTRIES, 1); union { float value; uint32_t bits; diff --git a/examples/balls/mxmm/workloads/ctests/mxmm_fp32_test.c b/examples/prefab/balls/mxmm/workloads/ctests/mxmm_fp32_test.c similarity index 100% rename from examples/balls/mxmm/workloads/ctests/mxmm_fp32_test.c rename to examples/prefab/balls/mxmm/workloads/ctests/mxmm_fp32_test.c diff --git a/examples/balls/mxmm/workloads/ctests/mxmm_mxfp8_test.c b/examples/prefab/balls/mxmm/workloads/ctests/mxmm_mxfp8_test.c similarity index 100% rename from examples/balls/mxmm/workloads/ctests/mxmm_mxfp8_test.c rename to examples/prefab/balls/mxmm/workloads/ctests/mxmm_mxfp8_test.c diff --git a/examples/balls/mxmm/workloads/ctests/mxmm_window_chain_test.c b/examples/prefab/balls/mxmm/workloads/ctests/mxmm_window_chain_test.c similarity index 78% rename from examples/balls/mxmm/workloads/ctests/mxmm_window_chain_test.c rename to examples/prefab/balls/mxmm/workloads/ctests/mxmm_window_chain_test.c index 6b4d737d1..497583569 100644 --- a/examples/balls/mxmm/workloads/ctests/mxmm_window_chain_test.c +++ b/examples/prefab/balls/mxmm/workloads/ctests/mxmm_window_chain_test.c @@ -5,7 +5,7 @@ #include #include -enum { BYTES = BANK_LINES * BANK_WIDTH / 8 }; +enum { BYTES = THREAD_MEM_BANK_ENTRIES * THREAD_MEM_BANK_WIDTH / 8 }; static uint8_t a[2][BYTES] __attribute__((aligned(64))); static uint8_t packed[BYTES] __attribute__((aligned(64))); static uint8_t w[BYTES] __attribute__((aligned(64))); @@ -25,11 +25,11 @@ int main(void) { memset(expected, 0x5a, BYTES); for (int bank = 3; bank <= 8; ++bank) bb_mem_alloc(bank, 1, 1); - bb_mvin((uintptr_t)a[0], 3, BANK_LINES, 1); - bb_mvin((uintptr_t)a[1], 4, BANK_LINES, 1); - bb_mvin((uintptr_t)w, 6, BANK_LINES, 1); - bb_mvin((uintptr_t)expected, 7, BANK_LINES, 1); - bb_mvin((uintptr_t)actual, 8, BANK_LINES, 1); + bb_mvin((uintptr_t)a[0], 3, THREAD_MEM_BANK_ENTRIES, 1); + bb_mvin((uintptr_t)a[1], 4, THREAD_MEM_BANK_ENTRIES, 1); + bb_mvin((uintptr_t)w, 6, THREAD_MEM_BANK_ENTRIES, 1); + bb_mvin((uintptr_t)expected, 7, THREAD_MEM_BANK_ENTRIES, 1); + bb_mvin((uintptr_t)actual, 8, THREAD_MEM_BANK_ENTRIES, 1); for (int mode = 0; mode < 2; ++mode) for (int part = 0; part < 2; ++part) { if (mode == 0) { @@ -37,11 +37,11 @@ int main(void) { packed[32] = 127; bb_mvin((uintptr_t)packed, 5, 3, 1); bb_mxmm_mxfp8(5, 6, 7, 1, 16, 32, part == 0, part == 1, 1); - bb_mvout((uintptr_t)expected, 7, BANK_LINES, 1); + bb_mvout((uintptr_t)expected, 7, THREAD_MEM_BANK_ENTRIES, 1); } else { bb_mxmm_mxfp8_window(3 + part, 6, 8, 1, 16, 32, part == 0, part == 1, 1, part ? 96 : 64, part * 32); - bb_mvout((uintptr_t)actual, 8, BANK_LINES, 1); + bb_mvout((uintptr_t)actual, 8, THREAD_MEM_BANK_ENTRIES, 1); } if (part == 0) for (int i = 0; i < BYTES; ++i) diff --git a/examples/balls/mxmm/workloads/ctests/mxmm_window_rows_test.c b/examples/prefab/balls/mxmm/workloads/ctests/mxmm_window_rows_test.c similarity index 85% rename from examples/balls/mxmm/workloads/ctests/mxmm_window_rows_test.c rename to examples/prefab/balls/mxmm/workloads/ctests/mxmm_window_rows_test.c index 65f443ce9..00f36e1d0 100644 --- a/examples/balls/mxmm/workloads/ctests/mxmm_window_rows_test.c +++ b/examples/prefab/balls/mxmm/workloads/ctests/mxmm_window_rows_test.c @@ -5,7 +5,7 @@ #include #include -enum { BYTES = BANK_LINES * BANK_WIDTH / 8 }; +enum { BYTES = THREAD_MEM_BANK_ENTRIES * THREAD_MEM_BANK_WIDTH / 8 }; static uint8_t a[BYTES] __attribute__((aligned(64))); static uint8_t packed[BYTES] __attribute__((aligned(64))); static uint8_t w[BYTES] __attribute__((aligned(64))); @@ -32,9 +32,9 @@ int main(void) { memset(expected, 0x5a, sizeof(expected)); for (int bank = 3; bank <= 7; ++bank) bb_mem_alloc(bank, 1, 1); - bb_mvin((uintptr_t)a, 3, BANK_LINES, 1); - bb_mvin((uintptr_t)expected, 5, BANK_LINES, 1); - bb_mvin((uintptr_t)actual, 7, BANK_LINES, 1); + bb_mvin((uintptr_t)a, 3, THREAD_MEM_BANK_ENTRIES, 1); + bb_mvin((uintptr_t)expected, 5, THREAD_MEM_BANK_ENTRIES, 1); + bb_mvin((uintptr_t)actual, 7, THREAD_MEM_BANK_ENTRIES, 1); /* Complete old-71 chain before new-75 chain: one chain slot per hart. */ for (int mode = 0; mode < 2; ++mode) { for (int k = start; k < full; k += 480) { @@ -55,15 +55,15 @@ int main(void) { a + m * full + row * (full / 32) + k / 32, count / 32); } } - bb_mvin((uintptr_t)w, 6, BANK_LINES, 1); + bb_mvin((uintptr_t)w, 6, THREAD_MEM_BANK_ENTRIES, 1); if (mode == 0) { - bb_mvin((uintptr_t)packed, 4, BANK_LINES, 1); + bb_mvin((uintptr_t)packed, 4, THREAD_MEM_BANK_ENTRIES, 1); bb_mxmm_mxfp8(4, 6, 5, m, n, count, k == start, k + count == full, 1); - bb_mvout((uintptr_t)expected, 5, BANK_LINES, 1); + bb_mvout((uintptr_t)expected, 5, THREAD_MEM_BANK_ENTRIES, 1); } else { bb_mxmm_mxfp8_window(3, 6, 7, m, n, count, k == start, k + count == full, 1, full, k); - bb_mvout((uintptr_t)actual, 7, BANK_LINES, 1); + bb_mvout((uintptr_t)actual, 7, THREAD_MEM_BANK_ENTRIES, 1); } if (k + count != full) for (int i = 0; i < BYTES; ++i) diff --git a/examples/balls/mxmm/workloads/ctests/mxmm_window_test.c b/examples/prefab/balls/mxmm/workloads/ctests/mxmm_window_test.c similarity index 100% rename from examples/balls/mxmm/workloads/ctests/mxmm_window_test.c rename to examples/prefab/balls/mxmm/workloads/ctests/mxmm_window_test.c diff --git a/examples/balls/mxmm/workloads/isa/mxmm.h b/examples/prefab/balls/mxmm/workloads/isa/mxmm.h similarity index 87% rename from examples/balls/mxmm/workloads/isa/mxmm.h rename to examples/prefab/balls/mxmm/workloads/isa/mxmm.h index 7eafd0738..c676e83d8 100644 --- a/examples/balls/mxmm/workloads/isa/mxmm.h +++ b/examples/prefab/balls/mxmm/workloads/isa/mxmm.h @@ -14,7 +14,7 @@ static inline void bb_mxmm_mxfp8(uint32_t a, uint32_t b, uint32_t c, uint32_t m, uint32_t last, uint32_t base) { BUCKYBALL_INSTRUCTION_R_R( BB_BANK0(a) | BB_BANK1(b) | BB_BANK2(c) | BB_ITER(k), - bb_mxmm_cfg(m, n, first, last, base), BB_FUNC7(MXMM_MXFP8)); + bb_mxmm_cfg(m, n, first, last, base), FUNCT7(mxmm, MXMM_MXFP8)); } static inline void bb_mxmm_fma32(uint32_t a, uint32_t b, uint32_t c, uint32_t m, @@ -22,7 +22,7 @@ static inline void bb_mxmm_fma32(uint32_t a, uint32_t b, uint32_t c, uint32_t m, uint32_t last, uint32_t base) { BUCKYBALL_INSTRUCTION_R_R( BB_BANK0(a) | BB_BANK1(b) | BB_BANK2(c) | BB_ITER(k), - bb_mxmm_cfg(m, n, first, last, base), BB_FUNC7(MXMM_FMA32)); + bb_mxmm_cfg(m, n, first, last, base), FUNCT7(mxmm, MXMM_FMA32)); } static inline void bb_mxmm_f32(uint32_t a, uint32_t b, uint32_t c, uint32_t m, @@ -30,7 +30,7 @@ static inline void bb_mxmm_f32(uint32_t a, uint32_t b, uint32_t c, uint32_t m, uint32_t last, uint32_t base) { BUCKYBALL_INSTRUCTION_R_R( BB_BANK0(a) | BB_BANK1(b) | BB_BANK2(c) | BB_ITER(k), - bb_mxmm_cfg(m, n, first, last, base), BB_FUNC7(MXMM_F32)); + bb_mxmm_cfg(m, n, first, last, base), FUNCT7(mxmm, MXMM_F32)); } static inline void bb_mxmm_mxfp8_window(uint32_t a, uint32_t b, uint32_t c, @@ -42,6 +42,6 @@ static inline void bb_mxmm_mxfp8_window(uint32_t a, uint32_t b, uint32_t c, BB_ITER(k), bb_mxmm_cfg(m, n, first, last, base) | FIELD(fullK, 32, 47) | FIELD(startK, 48, 63), - BB_FUNC7(MXMM_MXFP8_WINDOW)); + FUNCT7(mxmm, MXMM_MXFP8_WINDOW)); } #endif diff --git a/examples/prefab/balls/mxmm/workloads/mlir_tests/ball/mxmm_f32.mlir b/examples/prefab/balls/mxmm/workloads/mlir_tests/ball/mxmm_f32.mlir new file mode 100644 index 000000000..0ce7f4321 --- /dev/null +++ b/examples/prefab/balls/mxmm/workloads/mlir_tests/ball/mxmm_f32.mlir @@ -0,0 +1,10 @@ +// RUN: buddy-opt %s -lower-buckyball -canonicalize | FileCheck %s +// CHECK-LABEL: func.func @single +// CHECK-DAG: arith.constant 4298115073 : i64 +// CHECK-DAG: arith.constant 50397200 : i64 +// The caller loads A into bank 1, B into bank 2 and C into bank 3. One step +// that both starts and ends its chain. +func.func @single() attributes {llvm.emit_c_interface} { + mxmm_ball.matmul_f32 <{a = 1 : i64, b = 2 : i64, c = 3 : i64, rows = 16 : i64, cols = 16 : i64, k = 4 : i64, base = 0 : i64, first, last}> + return +} diff --git a/examples/prefab/balls/mxmm/workloads/mlir_tests/ball/mxmm_fma32.mlir b/examples/prefab/balls/mxmm/workloads/mlir_tests/ball/mxmm_fma32.mlir new file mode 100644 index 000000000..2ce380e7f --- /dev/null +++ b/examples/prefab/balls/mxmm/workloads/mlir_tests/ball/mxmm_fma32.mlir @@ -0,0 +1,10 @@ +// RUN: buddy-opt %s -lower-buckyball -canonicalize | FileCheck %s +// CHECK-LABEL: func.func @single +// CHECK-DAG: arith.constant 4298115073 : i64 +// CHECK-DAG: arith.constant 50397200 : i64 +// The caller loads A into bank 1, B into bank 2 and C into bank 3. One step +// that both starts and ends its chain. +func.func @single() attributes {llvm.emit_c_interface} { + mxmm_ball.matmul_fma32 <{a = 1 : i64, b = 2 : i64, c = 3 : i64, rows = 16 : i64, cols = 16 : i64, k = 4 : i64, base = 0 : i64, first, last}> + return +} diff --git a/examples/prefab/balls/mxmm/workloads/mlir_tests/ball/mxmm_mxfp8_chain.mlir b/examples/prefab/balls/mxmm/workloads/mlir_tests/ball/mxmm_mxfp8_chain.mlir new file mode 100644 index 000000000..03519134f --- /dev/null +++ b/examples/prefab/balls/mxmm/workloads/mlir_tests/ball/mxmm_mxfp8_chain.mlir @@ -0,0 +1,18 @@ +// RUN: buddy-opt %s -lower-buckyball -canonicalize | FileCheck %s +// CHECK-LABEL: func.func @first +// CHECK-DAG: arith.constant 34362886145 : i64 +// CHECK-DAG: arith.constant 83951632 : i64 +// CHECK-LABEL: func.func @last +// CHECK-DAG: arith.constant 34362886145 : i64 +// CHECK-DAG: arith.constant 100728848 : i64 +// The caller loads A into bank 1, B into bank 2 and C into bank 3. Two steps of +// one chain over the same A and B: the first only accumulates, the last +// accumulates and writes the 16x16 sum to C from row 1. +func.func @first() attributes {llvm.emit_c_interface} { + mxmm_ball.matmul_mxfp8 <{a = 1 : i64, b = 2 : i64, c = 3 : i64, rows = 16 : i64, cols = 16 : i64, k = 32 : i64, base = 1 : i64, first}> + return +} +func.func @last() attributes {llvm.emit_c_interface} { + mxmm_ball.matmul_mxfp8 <{a = 1 : i64, b = 2 : i64, c = 3 : i64, rows = 16 : i64, cols = 16 : i64, k = 32 : i64, base = 1 : i64, last}> + return +} diff --git a/examples/prefab/balls/mxmm/workloads/mlir_tests/ball/mxmm_mxfp8_window.mlir b/examples/prefab/balls/mxmm/workloads/mlir_tests/ball/mxmm_mxfp8_window.mlir new file mode 100644 index 000000000..f719efe3b --- /dev/null +++ b/examples/prefab/balls/mxmm/workloads/mlir_tests/ball/mxmm_mxfp8_window.mlir @@ -0,0 +1,10 @@ +// RUN: buddy-opt %s -lower-buckyball -canonicalize | FileCheck %s +// CHECK-LABEL: func.func @window +// CHECK-DAG: arith.constant 34362886145 : i64 +// CHECK-DAG: arith.constant 9007474183045121 : i64 +// The caller loads A into bank 1, B into bank 2 and C into bank 3. One step +// reads columns 32..63 of a one-row A of 64 columns. +func.func @window() attributes {llvm.emit_c_interface} { + mxmm_ball.matmul_mxfp8_window <{a = 1 : i64, b = 2 : i64, c = 3 : i64, rows = 1 : i64, cols = 16 : i64, k = 32 : i64, full_k = 64 : i64, start_k = 32 : i64, base = 0 : i64, first, last}> + return +} diff --git a/examples/prefab/balls/mxmm/workloads/mlir_tests/bank/mxmm_f32.mlir b/examples/prefab/balls/mxmm/workloads/mlir_tests/bank/mxmm_f32.mlir new file mode 100644 index 000000000..cb83c388e --- /dev/null +++ b/examples/prefab/balls/mxmm/workloads/mlir_tests/bank/mxmm_f32.mlir @@ -0,0 +1,17 @@ +// One unfused FP32 step that both starts and ends its chain. C comes in holding +// a guard pattern and goes back with the 16x16 sum. +func.func @single(%a: memref<16x4xi32>, %b: memref<16x4xi32>, + %c: memref<65x4xi32>) attributes {llvm.emit_c_interface} { + %a_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %b_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %c_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %la = buckyball.bank_mvin %a, %a_bank <{depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> + %lb = buckyball.bank_mvin %b, %b_bank <{depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> + %lc = buckyball.bank_mvin %c, %c_bank <{depth = 65 : i64, stride = 1 : i64}> : memref<65x4xi32> + %sum = mxmm_ball.bank_matmul_f32 %la, %lb -> %lc <{rows = 16 : i64, cols = 16 : i64, k = 4 : i64, base = 0 : i64, first, last}> + %stored = buckyball.bank_mvout %c, %sum <{depth = 65 : i64, stride = 1 : i64}> : memref<65x4xi32> + buckyball.bank_release %la + buckyball.bank_release %lb + buckyball.bank_release %stored + return +} diff --git a/examples/prefab/balls/mxmm/workloads/mlir_tests/bank/mxmm_fma32.mlir b/examples/prefab/balls/mxmm/workloads/mlir_tests/bank/mxmm_fma32.mlir new file mode 100644 index 000000000..c9198fca8 --- /dev/null +++ b/examples/prefab/balls/mxmm/workloads/mlir_tests/bank/mxmm_fma32.mlir @@ -0,0 +1,17 @@ +// One fused FP32 step that both starts and ends its chain. C comes in holding +// a guard pattern and goes back with the 16x16 sum. +func.func @single(%a: memref<16x4xi32>, %b: memref<16x4xi32>, + %c: memref<65x4xi32>) attributes {llvm.emit_c_interface} { + %a_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %b_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %c_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %la = buckyball.bank_mvin %a, %a_bank <{depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> + %lb = buckyball.bank_mvin %b, %b_bank <{depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> + %lc = buckyball.bank_mvin %c, %c_bank <{depth = 65 : i64, stride = 1 : i64}> : memref<65x4xi32> + %sum = mxmm_ball.bank_matmul_fma32 %la, %lb -> %lc <{rows = 16 : i64, cols = 16 : i64, k = 4 : i64, base = 0 : i64, first, last}> + %stored = buckyball.bank_mvout %c, %sum <{depth = 65 : i64, stride = 1 : i64}> : memref<65x4xi32> + buckyball.bank_release %la + buckyball.bank_release %lb + buckyball.bank_release %stored + return +} diff --git a/examples/prefab/balls/mxmm/workloads/mlir_tests/bank/mxmm_mxfp8_chain.mlir b/examples/prefab/balls/mxmm/workloads/mlir_tests/bank/mxmm_mxfp8_chain.mlir new file mode 100644 index 000000000..188c029d0 --- /dev/null +++ b/examples/prefab/balls/mxmm/workloads/mlir_tests/bank/mxmm_mxfp8_chain.mlir @@ -0,0 +1,19 @@ +// Two steps of one MXFP8 chain over the same A and B: the first only +// accumulates, the last accumulates and writes the 16x16 sum to C from row 1. +// C comes in holding a guard pattern and goes back with the sum. +func.func @chain(%a: memref<33x16xi8>, %b: memref<33x16xi8>, + %c: memref<65x16xi8>) attributes {llvm.emit_c_interface} { + %a_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %b_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %c_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %la = buckyball.bank_mvin %a, %a_bank <{depth = 33 : i64, stride = 1 : i64}> : memref<33x16xi8> + %lb = buckyball.bank_mvin %b, %b_bank <{depth = 33 : i64, stride = 1 : i64}> : memref<33x16xi8> + %lc = buckyball.bank_mvin %c, %c_bank <{depth = 65 : i64, stride = 1 : i64}> : memref<65x16xi8> + %partial = mxmm_ball.bank_matmul_mxfp8 %la, %lb -> %lc <{rows = 16 : i64, cols = 16 : i64, k = 32 : i64, base = 1 : i64, first}> + %sum = mxmm_ball.bank_matmul_mxfp8 %la, %lb -> %partial <{rows = 16 : i64, cols = 16 : i64, k = 32 : i64, base = 1 : i64, last}> + %stored = buckyball.bank_mvout %c, %sum <{depth = 65 : i64, stride = 1 : i64}> : memref<65x16xi8> + buckyball.bank_release %la + buckyball.bank_release %lb + buckyball.bank_release %stored + return +} diff --git a/examples/prefab/balls/mxmm/workloads/mlir_tests/bank/mxmm_mxfp8_window.mlir b/examples/prefab/balls/mxmm/workloads/mlir_tests/bank/mxmm_mxfp8_window.mlir new file mode 100644 index 000000000..65ccf71a8 --- /dev/null +++ b/examples/prefab/balls/mxmm/workloads/mlir_tests/bank/mxmm_mxfp8_window.mlir @@ -0,0 +1,17 @@ +// One MXFP8 step on columns 32..63 of a one-row A of 64 columns. C comes in +// holding a guard pattern and goes back with the row of 16 sums. +func.func @window(%a: memref<5x16xi8>, %b: memref<33x16xi8>, + %c: memref<5x16xi8>) attributes {llvm.emit_c_interface} { + %a_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %b_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %c_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %la = buckyball.bank_mvin %a, %a_bank <{depth = 5 : i64, stride = 1 : i64}> : memref<5x16xi8> + %lb = buckyball.bank_mvin %b, %b_bank <{depth = 33 : i64, stride = 1 : i64}> : memref<33x16xi8> + %lc = buckyball.bank_mvin %c, %c_bank <{depth = 5 : i64, stride = 1 : i64}> : memref<5x16xi8> + %sum = mxmm_ball.bank_matmul_mxfp8_window %la, %lb -> %lc <{rows = 1 : i64, cols = 16 : i64, k = 32 : i64, full_k = 64 : i64, start_k = 32 : i64, base = 0 : i64, first, last}> + %stored = buckyball.bank_mvout %c, %sum <{depth = 5 : i64, stride = 1 : i64}> : memref<5x16xi8> + buckyball.bank_release %la + buckyball.bank_release %lb + buckyball.bank_release %stored + return +} diff --git a/examples/balls/mxquant/arch/src/main/scala/Encode.scala b/examples/prefab/balls/mxquant/arch/Encode.scala similarity index 94% rename from examples/balls/mxquant/arch/src/main/scala/Encode.scala rename to examples/prefab/balls/mxquant/arch/Encode.scala index 9c5a76196..4b185f9f2 100644 --- a/examples/balls/mxquant/arch/src/main/scala/Encode.scala +++ b/examples/prefab/balls/mxquant/arch/Encode.scala @@ -1,10 +1,14 @@ package examples.balls.mxquant +import chisel3.experimental.hierarchy.{instantiable, public} + import chisel3._ import chisel3.util._ +@instantiable class Encode extends Module { + @public val io = IO(new Bundle { val input = Input(UInt(32.W)) val blockExponent = Input(SInt(10.W)) diff --git a/examples/prefab/balls/mxquant/arch/MxquantBall.scala b/examples/prefab/balls/mxquant/arch/MxquantBall.scala new file mode 100644 index 000000000..d7ad256e0 --- /dev/null +++ b/examples/prefab/balls/mxquant/arch/MxquantBall.scala @@ -0,0 +1,173 @@ +package examples.balls.mxquant + +import framework.acc.balldomain.bbus.{BallFactory, BallPorts} +import buckyball.balls.mxquant.{Params => ParametersProto} +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.acc.balldomain.ball.blink.{BallStatus, BlinkIO, HasBlink} + +@instantiable +class MxquantBall( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Module + with HasBlink { + val ball = bp.balls.find(_.id == id).get + + ball.parameters.unpack(classOf[ParametersProto]) + + require( + mp.bankWidth == 128 && mp.bankMaskLen == 16, + "MxquantBall requires 128-bit bank rows and byte write masks" + ) + require(mp.bankEntries >= 8, "MxquantBall requires eight input rows") + require(ball.inBW == 1 && ball.outBW == 1, "MxquantBall requires one read and one write") + + @public + val blink = IO(new BlinkIO(fp, mp, bp, id)) + + val idle :: readReq :: readResp :: exponent :: encode :: codeReq :: codeResp :: scaleReq :: scaleResp :: complete :: Nil = + Enum(10) + val state = RegInit(idle) + val robId = Reg(UInt(log2Up(fp.rob_entries).W)) + val source = Reg(UInt(fp.bank_id_len.W)) + val destination = Reg(UInt(fp.bank_id_len.W)) + val bases = Reg(Vec(3, UInt(mp.groupIdWidth.W))) + val count = Reg(UInt(fp.iter_len.W)) + val group = RegInit(0.U(fp.iter_len.W)) + val readRow = RegInit(0.U(3.W)) + val lane = RegInit(0.U(5.W)) + val codeRow = RegInit(0.U(1.W)) + val maximum = RegInit(0.U(31.W)) + val blockExponent = Reg(SInt(10.W)) + val scale = Reg(UInt(8.W)) + val values = Reg(Vec(32, UInt(32.W))) + val codes = Reg(Vec(32, UInt(8.W))) + val math = Instantiate(new Encode) + math.io.input := values(lane) + math.io.blockExponent := blockExponent + + val active = !reset.asBool + blink.cmdReq.ready := state === idle && active + blink.cmdResp.valid := state === complete && active + blink.cmdResp.bits.rob_id := robId + blink.status.idle := state === idle + blink.status.running := state =/= idle && state =/= complete + + val read = blink.bankRead(0) + read.rob_id := robId + read.bank_id := source + read.group_id := bases(0) + read.io.req.valid := state === readReq && active + read.io.req.bits.addr := (group << 3) + readRow + read.io.resp.ready := state === readResp && active + + val write = blink.bankWrite(0) + write.rob_id := robId + write.bank_id := destination + write.group_id := bases(2) + write.io.req.valid := (state === codeReq || state === scaleReq) && active + write.io.req.bits.addr := Mux(state === scaleReq, (count >> 4) + (group >> 4), (group << 1) + codeRow) + val lowCodes = Cat((15 to 0 by -1).map(codes(_))) + val highCodes = Cat((31 to 16 by -1).map(codes(_))) + write.io.req.bits.data := Mux( + state === scaleReq, + (scale.pad(128) << (group(3, 0) << 3))(127, 0), + Mux(codeRow === 0.U, lowCodes, highCodes) + ) + write.io.req.bits.mask := VecInit((0 until 16).map(i => state =/= scaleReq || group(3, 0) === i.U)) + write.io.resp.ready := (state === codeResp || state === scaleResp) && active + + switch(state) { + is(idle) { + when(blink.cmdReq.fire) { + val cmd = blink.cmdReq.bits + + assert(cmd.iter > 0.U && cmd.iter(4, 0) === 0.U, "MxquantBall count must be positive and divisible by 32") + assert( + cmd.iter <= (mp.bankEntries * 4).U, + "MxquantBall input exceeds capacity" + ) + assert( + cmd.iter +& (cmd.iter >> 5) <= (mp.bankEntries * 16).U, + "MxquantBall output exceeds capacity" + ) + assert(cmd.rs2 === 0.U, "MxquantBall reserved fields must be zero") + assert(cmd.op1_col === 1.U && cmd.wr_col === 1.U, "MxquantBall operands must occupy one bank") + assert(cmd.op1_bank =/= cmd.wr_bank, "MxquantBall banks must differ") + robId := blink.cmdReq.bits.rob_id + source := cmd.op1_bank + destination := cmd.wr_bank + bases := VecInit(cmd.op1_group, cmd.op2_group, cmd.wr_group) + count := cmd.iter + group := 0.U + readRow := 0.U + maximum := 0.U + state := readReq + } + } + is(readReq)(when(read.io.req.fire)(state := readResp)) + is(readResp) { + when(read.io.resp.fire) { + val words = (0 until 4).map(i => read.io.resp.bits.data(32 * i + 31, 32 * i)) + for (i <- 0 until 4) { + assert(words(i)(30, 23) =/= 255.U, "MxquantBall input must be finite") + values(Cat(readRow, i.U(2.W))) := words(i) + } + maximum := (Seq(maximum) ++ words.map(_(30, 0))).reduce((a, c) => Mux(a > c, a, c)) + when(readRow === 7.U)(state := exponent).otherwise { readRow := readRow + 1.U; state := readReq } + } + } + is(exponent) { + val raw = maximum(30, 23).zext - 135.S(10.W) + val selected = Mux(maximum === 0.U, 0.S(10.W), Mux(raw < -127.S, -127.S(10.W), raw)) + blockExponent := selected + scale := (selected + 127.S).asUInt + lane := 0.U + state := encode + } + is(encode) { + codes(lane) := math.io.code + when(lane === 31.U) { codeRow := 0.U; state := codeReq }.otherwise(lane := lane + 1.U) + } + is(codeReq)(when(write.io.req.fire)(state := codeResp)) + is(codeResp) { + when(write.io.resp.fire) { + when(codeRow === 0.U) { codeRow := 1.U; state := codeReq }.otherwise(state := scaleReq) + } + } + is(scaleReq)(when(write.io.req.fire)(state := scaleResp)) + is(scaleResp) { + when(write.io.resp.fire) { + when(group === (count >> 5) - 1.U)(state := complete).otherwise { + group := group + 1.U + readRow := 0.U + maximum := 0.U + state := readReq + } + } + } + is(complete)(when(blink.cmdResp.fire)(state := idle)) + } +} + +object MxquantBall extends BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts = + BallPorts(Instantiate(new MxquantBall(fp, mp, bp, id)).blink, None) + +} diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/README.md b/examples/prefab/balls/mxquant/configs/default.toml similarity index 100% rename from arch/src/main/scala/framework/root/root-chip/src/main/resources/README.md rename to examples/prefab/balls/mxquant/configs/default.toml diff --git a/examples/prefab/balls/mxquant/configs/mxquant_params.proto b/examples/prefab/balls/mxquant/configs/mxquant_params.proto new file mode 100644 index 000000000..2d178b8a4 --- /dev/null +++ b/examples/prefab/balls/mxquant/configs/mxquant_params.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; +package buckyball.balls.mxquant; +option java_package = "buckyball.balls.mxquant"; +option java_multiple_files = true; + +message Params { +} diff --git a/examples/prefab/balls/mxquant/emu/src/dispatch.rs b/examples/prefab/balls/mxquant/emu/src/dispatch.rs new file mode 100644 index 000000000..3cec59c5b --- /dev/null +++ b/examples/prefab/balls/mxquant/emu/src/dispatch.rs @@ -0,0 +1,30 @@ +use super::quant; +use bebop_bemu::ball::ExecContext; + +const BALL_CLASS: &str = "examples.balls.mxquant.MxquantBall"; + +pub fn execute_known( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, + ctx: &mut ExecContext, +) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("MXQUANT") => Some(quant::execute(xs1, xs2, ctx)), + _ => None, + } +} + +pub fn cycles_after_issue(ball_class: &str, funct: u32, _xs1: u64, _xs2: u64) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("MXQUANT") => Some(1), + _ => None, + } +} diff --git a/examples/prefab/balls/mxquant/emu/src/lib.rs b/examples/prefab/balls/mxquant/emu/src/lib.rs new file mode 100644 index 000000000..276b3f599 --- /dev/null +++ b/examples/prefab/balls/mxquant/emu/src/lib.rs @@ -0,0 +1,10 @@ +pub(crate) use bebop_bemu::{thread::instructions::decode, ball as instruction}; + +mod dispatch; +mod quant; + +pub use dispatch::{cycles_after_issue, execute_known}; + +#[bebop_bemu::ball::linkme::distributed_slice(bebop_bemu::BALLS)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static BALL: bebop_bemu::Ball = bebop_bemu::Ball { execute: execute_known, cycles: cycles_after_issue }; diff --git a/examples/prefab/balls/mxquant/emu/src/quant.rs b/examples/prefab/balls/mxquant/emu/src/quant.rs new file mode 100644 index 000000000..3a46a9ad1 --- /dev/null +++ b/examples/prefab/balls/mxquant/emu/src/quant.rs @@ -0,0 +1,92 @@ +use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old}; +use super::instruction::ExecContext; +use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; + +fn rounded_shift(value: u32, shift: u32) -> u32 { + if shift > 31 { + return 0; + } + let half = 1 << (shift - 1); + (value + half - 1 + ((value >> shift) & 1)) >> shift +} + +fn encode(bits: u32, block_exponent: i32) -> u8 { + let sign = ((bits >> 24) & 128) as u8; + if bits & 0x7fffffff == 0 { + return sign; + } + let mut exponent = ((bits >> 23) & 255) as i32; + let mut fraction = bits & 0x7fffff; + if exponent == 0 { + let shift = fraction.leading_zeros() - 8; + fraction = (fraction << shift) & 0x7fffff; + exponent = 1 - shift as i32; + } + exponent -= block_exponent + 120; + let code = if exponent <= 0 { + rounded_shift(fraction | 0x800000, (21 - exponent) as u32) + } else { + exponent as u32 * 8 + rounded_shift(fraction, 20) + }; + sign | code.min(126) as u8 +} + +pub(crate) fn execute(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + assert!(!rs1_read_old(xs1), "mxquant: reads no old destination; rs1[29] must be clear"); + let input = rs1_b0(xs1); + let output = rs1_b2(xs1); + let count = rs1_iter(xs1) as usize; + assert!( + xs2 == 0 && rs1_b1(xs1) == 0, + "mxquant: reserved fields must be zero" + ); + assert!( + count > 0 && count % 32 == 0, + "mxquant: count must be a positive multiple of 32" + ); + assert!( + input != output, + "mxquant: input and output banks must differ" + ); + for bank in [input, output] { + assert!( + ctx.config(bank).allocated && ctx.config(bank).cols == 1, + "mxquant: banks must be allocated with one column" + ); + } + let bytes = bank_lines() * bank_row_bytes(); + assert!( + bank_row_bytes() == 16 && count <= bytes / 4 && count + count / 32 <= bytes, + "mxquant: bank footprint exceeds capacity" + ); + let source = &ctx.banks[pbank(ctx, input)][..count * 4]; + assert!( + source + .chunks_exact(4) + .all( + |element| u32::from_le_bytes(element.try_into().unwrap()) & 0x7fffffff < 0x7f800000 + ), + "mxquant: non-finite activation" + ); + let mut packed = vec![0u8; count + count / 32]; + for (block, values) in source.chunks_exact(128).enumerate() { + let maximum = values + .chunks_exact(4) + .map(|element| u32::from_le_bytes(element.try_into().unwrap()) & 0x7fffffff) + .max() + .unwrap(); + let exponent = if maximum == 0 { + 0 + } else { + ((maximum >> 23) as i32 - 135).max(-127) + }; + packed[count + block] = (exponent + 127) as u8; + for (index, element) in values.chunks_exact(4).enumerate() { + packed[block * 32 + index] = + encode(u32::from_le_bytes(element.try_into().unwrap()), exponent); + } + } + let destination = pbank(ctx, output); + ctx.banks[destination][..packed.len()].copy_from_slice(&packed); + 0 +} diff --git a/examples/prefab/balls/mxquant/stack/kernels/CMakeLists.txt b/examples/prefab/balls/mxquant/stack/kernels/CMakeLists.txt new file mode 100644 index 000000000..1d7c5b1e7 --- /dev/null +++ b/examples/prefab/balls/mxquant/stack/kernels/CMakeLists.txt @@ -0,0 +1,10 @@ +# ===----------------------------------------------------------------------=== +# MXQuant kernels: the host code that drives the Ball. Added by the stack of a +# thread that has this Ball, which compiles KERNEL_SOURCES and exports +# KERNEL_EXPORTS. +# ===----------------------------------------------------------------------=== +set(KERNEL_SOURCES ${KERNEL_SOURCES} + "${CMAKE_CURRENT_SOURCE_DIR}/dispatch.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/pack.cpp" PARENT_SCOPE) +set(KERNEL_INCLUDES ${KERNEL_INCLUDES} + "${DESIGN_REPO}/examples/prefab/balls/mxquant/workloads" PARENT_SCOPE) +set(KERNEL_EXPORTS ${KERNEL_EXPORTS} mxfp8_quant _mlir_ciface_mxfp8_quant PARENT_SCOPE) diff --git a/examples/prefab/balls/mxquant/stack/kernels/dispatch.cpp b/examples/prefab/balls/mxquant/stack/kernels/dispatch.cpp new file mode 100644 index 000000000..541933d7e --- /dev/null +++ b/examples/prefab/balls/mxquant/stack/kernels/dispatch.cpp @@ -0,0 +1,14 @@ +#include +#include + +extern "C" void mxfp8_quant(const uint32_t *input, uint8_t *output, + uint32_t count) { + constexpr unsigned inputBank = 0, outputBank = 1; + bb_mem_alloc(inputBank, 1, 1); + bb_mem_alloc(outputBank, 1, 1); + bb_mvin((uintptr_t)input, inputBank, count / 4, 1); + bb_mxquant(inputBank, outputBank, count); + bb_mvout((uintptr_t)output, outputBank, (count + count / 32 + 15) / 16, 1); + bb_mem_release(inputBank); + bb_mem_release(outputBank); +} diff --git a/examples/prefab/balls/mxquant/stack/kernels/pack.cpp b/examples/prefab/balls/mxquant/stack/kernels/pack.cpp new file mode 100644 index 000000000..20bc8b53c --- /dev/null +++ b/examples/prefab/balls/mxquant/stack/kernels/pack.cpp @@ -0,0 +1,70 @@ +#include +#include +#include +#include +#include +#include + +extern "C" void mxfp8_quant(const uint32_t *input, uint8_t *output, + uint32_t count); + +extern "C" void _mlir_ciface_mxfp8_quant(StridedMemRefType *input, + StridedMemRefType *output, + int64_t tileRows, int64_t tileK, + int64_t panelStride) { + int64_t m = input->sizes[0], k = input->sizes[1]; + int64_t mt = (m + tileRows - 1) / tileRows, kt = (k + tileK - 1) / tileK; + int64_t bytes = panelStride; + if (k % 32 || tileK % 32 || output->sizes[0] != mt * kt * bytes) { + fprintf(stderr, "mxfp8: invalid activation packing shape\n"); + abort(); + } + auto *source = input->data + input->offset; + auto *dest = output->data + output->offset; + const int64_t rowStride = input->strides[0], columnStride = input->strides[1]; + const int64_t outputStride = output->strides[0]; + for (int64_t mr = 0; mr < mt; ++mr) + for (int64_t kr = 0; kr < kt; ++kr) + for (int64_t row = 0; row < tileRows; ++row) { + int64_t width = std::min(tileK, k - kr * tileK); + int64_t base = (mr * kt + kr) * bytes; + if (mr * tileRows + row >= m && outputStride == 1) { + std::memset(dest + base + row * width, 0, width); + std::memset(dest + base + tileRows * width + row * (width / 32), 127, + width / 32); + continue; + } + alignas(16) uint32_t staging[1024]; + alignas(16) uint8_t packed[1056]; + for (int64_t begin = 0; begin < width; begin += 1024) { + int64_t count = std::min(1024, width - begin); + int64_t r = mr * tileRows + row, c = kr * tileK + begin; + int64_t valid = + r < m ? std::max(0, std::min(count, k - c)) : 0; + const uint32_t *values; + if (valid == count && columnStride == 1) { + values = + reinterpret_cast(source + r * rowStride + c); + } else { + for (int64_t i = 0; i < valid; ++i) + staging[i] = __builtin_bit_cast( + uint32_t, source[r * rowStride + (c + i) * columnStride]); + std::fill(staging + valid, staging + count, 0); + values = staging; + } + mxfp8_quant(values, packed, count); + int64_t codes = base + row * width + begin; + int64_t scales = + base + tileRows * width + row * (width / 32) + begin / 32; + if (outputStride == 1) { + std::memcpy(dest + codes, packed, count); + std::memcpy(dest + scales, packed + count, count / 32); + } else { + for (int64_t i = 0; i < count; ++i) + dest[(codes + i) * outputStride] = packed[i]; + for (int64_t i = 0; i < count / 32; ++i) + dest[(scales + i) * outputStride] = packed[count + i]; + } + } + } +} diff --git a/examples/prefab/balls/mxquant/stack/src/mxquantOps.cpp b/examples/prefab/balls/mxquant/stack/src/mxquantOps.cpp new file mode 100644 index 000000000..521fb1558 --- /dev/null +++ b/examples/prefab/balls/mxquant/stack/src/mxquantOps.cpp @@ -0,0 +1,69 @@ +//===- mxquantOps.cpp - MxquantBall operations ----------------------------===// +// +// The dialect of MxquantBall, its virtual operation building its physical one +// and the instruction of the physical operation. +// +//===----------------------------------------------------------------------===// + +#include "mxquantOps.h" + +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::mxquant_ball; +using mlir::buddy::bankFields; +using mlir::buddy::constantI64; +using mlir::buddy::emitInstruction; + +#include "mxquantDialect.cpp.inc" + +#define GET_OP_CLASSES +#include "mxquantOps.cpp.inc" + +void MxquantBallDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "mxquantOps.cpp.inc" + >(); +} + +namespace { +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallDialect([](DialectRegistry ®istry) { + registry.insert(); + }), + true); +} // namespace + +SmallVector BankQuantMxfp8Op::getSourceBanks() { + return {getInput()}; +} +Value BankQuantMxfp8Op::getDestinationBank() { return getOutput(); } +bool BankQuantMxfp8Op::readsDestination() { return false; } +Operation *BankQuantMxfp8Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return QuantMxfp8Op::create(builder, getLoc(), sources[0], destination, + getCount()); +} + +SmallVector QuantMxfp8Op::getSourceBanks() { + return {int64_t(getInput())}; +} +int64_t QuantMxfp8Op::getDestinationBank() { return getOutput(); } +LogicalResult QuantMxfp8Op::lowerToInstructions(OpBuilder &builder) { + Location loc = getLoc(); + int64_t count = getCount(), input = getInput(), output = getOutput(); + const auto &target = buckyball_target::getBuckyballTarget(); + if (target.bankWidthBits != 128 || count <= 0 || count % 32 || + count > target.bankDepth * 4) + return emitOpError("quant_mxfp8 count exceeds the bank"); + if (input == output) + return emitOpError("quant_mxfp8 requires distinct input and output banks"); + emitInstruction( + builder, loc, buckyball_target::getBuckyballFunct7("MXQUANT"), + constantI64(builder, loc, bankFields({input}, output, false, count)), + constantI64(builder, loc, 0)); + return success(); +} diff --git a/examples/prefab/balls/mxquant/stack/src/mxquantOps.h b/examples/prefab/balls/mxquant/stack/src/mxquantOps.h new file mode 100644 index 000000000..d03f09cd4 --- /dev/null +++ b/examples/prefab/balls/mxquant/stack/src/mxquantOps.h @@ -0,0 +1,20 @@ +//===- mxquantOps.h - MxquantBall operations ------------------------------===// + +#ifndef MXQUANT_OPS_H +#define MXQUANT_OPS_H + +#include "Interfaces/BankInterfaces.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "mxquantDialect.h.inc" + +#define GET_OP_CLASSES +#include "mxquantOps.h.inc" + +#endif // MXQUANT_OPS_H diff --git a/examples/prefab/balls/mxquant/stack/src/mxquantOps.td b/examples/prefab/balls/mxquant/stack/src/mxquantOps.td new file mode 100644 index 000000000..44e9dfe7a --- /dev/null +++ b/examples/prefab/balls/mxquant/stack/src/mxquantOps.td @@ -0,0 +1,47 @@ +//===- mxquantOps.td - MxquantBall operations --------------*- tablegen -*-===// +// +// quant_mxfp8 (input) -> output: quantizes `count` FP32 (a multiple of 32) to +// block32 MXFP8, one MXQUANT instruction. Input holds the `count` finite FP32 +// from byte 0; output receives the `count` E4M3 codes from byte 0, then one +// E8M0 scale per block of 32 from byte `count`. The scale is the block +// exponent: the largest magnitude's exponent minus 8, at least -127, biased by +// 127 (0 for an all-zero block); a code rounds to nearest even, saturates at +// magnitude code 126 and keeps the sign of zero. Nothing else of the output +// is written. +// Input and output are distinct banks of one 128-bit wide bank each. +// +//===----------------------------------------------------------------------===// + +#ifndef MXQUANT_OPS +#define MXQUANT_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Interfaces/BankInterfaces.td" + +def MxquantBall_Dialect : Dialect { + let name = "mxquant_ball"; + let cppNamespace = "::buddy::mxquant_ball"; + let summary = "The operations of MxquantBall"; +} + +class MxquantBall_Op traits = []> + : Op])>; + +def BankQuantMxfp8Op : MxquantBall_Op<"bank_quant_mxfp8", + [DeclareOpInterfaceMethods]> { + let summary = "quant_mxfp8 on virtual banks"; + let arguments = (ins I64:$input, I64:$output, I64Attr:$count); + let results = (outs I64:$outputOut); + let assemblyFormat = "$input `->` $output prop-dict attr-dict"; +} + +def QuantMxfp8Op : MxquantBall_Op<"quant_mxfp8", + [DeclareOpInterfaceMethods]> { + let summary = "quant_mxfp8 on physical banks"; + let arguments = (ins I64Attr:$input, I64Attr:$output, I64Attr:$count); + let assemblyFormat = "prop-dict attr-dict"; +} + +#endif // MXQUANT_OPS diff --git a/examples/prefab/balls/mxquant/verify/casegen/Cargo.lock b/examples/prefab/balls/mxquant/verify/casegen/Cargo.lock new file mode 100644 index 000000000..5923911d5 --- /dev/null +++ b/examples/prefab/balls/mxquant/verify/casegen/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "mxquant-casegen" +version = "0.1.0" diff --git a/examples/prefab/balls/mxquant/verify/casegen/Cargo.toml b/examples/prefab/balls/mxquant/verify/casegen/Cargo.toml new file mode 100644 index 000000000..369a9d49f --- /dev/null +++ b/examples/prefab/balls/mxquant/verify/casegen/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "mxquant-casegen" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["rlib", "cdylib"] + +[lints.rust] +unused_must_use = "deny" diff --git a/examples/prefab/balls/mxquant/verify/casegen/src/casegen.rs b/examples/prefab/balls/mxquant/verify/casegen/src/casegen.rs new file mode 100644 index 000000000..fcad20c62 --- /dev/null +++ b/examples/prefab/balls/mxquant/verify/casegen/src/casegen.rs @@ -0,0 +1,265 @@ +// MXQUANT reference model, the contract shared with the BEMU model +// (emu/src/quant.rs) and MxquantBall: +// +// op1 bank: count FP32 activations, four per row; block g of 32 is rows +// 8g..8g+7. Activations must be finite. +// wr bank: E4M3 code of activation i in byte i (block g in rows 2g, 2g+1), +// the E8M0 scale of block g in byte count + g. +// The block exponent is max(exponent(max |x|) - 8, -127) (0 for an all-zero +// block); codes round to nearest even and clip to 126 (448). +// rs2 and op2 bank are zero; count is a positive multiple of 32 with +// count <= 4 * bank_entries and count + count / 32 <= 16 * bank_entries. + +pub const ROW_BYTES: usize = 16; +pub const BLOCK: usize = 32; +pub const NUM_CASES: u32 = 6; + +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct MxquantCmd { + pub rob_id: u32, + pub iter: u32, + pub rs2_lo: u32, + pub rs2_hi: u32, + pub op1_bank: u32, + pub op2_bank: u32, + pub wr_bank: u32, + pub op1_group: u32, + pub op2_group: u32, + pub wr_group: u32, + pub op1_col: u32, + pub op2_col: u32, + pub wr_col: u32, +} + +#[derive(Clone, Debug)] +pub struct Preload { + pub bank: u32, + pub group: u32, + pub addr: u32, + pub data: u128, +} + +#[derive(Clone, Debug)] +pub struct Write { + pub addr: u32, + pub mask: u16, + pub data: u128, +} + +#[derive(Clone, Debug)] +pub struct Step { + pub cmd: MxquantCmd, + pub preload: Vec, + pub reads: Vec, + pub writes: Vec, +} + +#[derive(Clone, Debug)] +pub struct Case { + pub steps: Vec, +} + +struct Rng(u32); + +impl Rng { + fn next(&mut self) -> u32 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 17; + self.0 ^= self.0 << 5; + self.0 + } +} + +fn word(bytes: &[u8]) -> u128 { + let mut out = 0u128; + for (lane, byte) in bytes.iter().enumerate() { + out |= u128::from(*byte) << (lane * 8); + } + out +} + +fn rounded_shift(value: u32, shift: u32) -> u32 { + if shift > 31 { + return 0; + } + let half = 1 << (shift - 1); + (value + half - 1 + ((value >> shift) & 1)) >> shift +} + +fn encode(bits: u32, block_exponent: i32) -> u8 { + let sign = ((bits >> 24) & 128) as u8; + if bits & 0x7fff_ffff == 0 { + return sign; + } + let mut exponent = ((bits >> 23) & 255) as i32; + let mut fraction = bits & 0x7f_ffff; + if exponent == 0 { + let shift = fraction.leading_zeros() - 8; + fraction = (fraction << shift) & 0x7f_ffff; + exponent = 1 - shift as i32; + } + exponent -= block_exponent + 120; + let code = if exponent <= 0 { + rounded_shift(fraction | 0x80_0000, (21 - exponent) as u32) + } else { + exponent as u32 * 8 + rounded_shift(fraction, 20) + }; + sign | code.min(126) as u8 +} + +fn step(bank_entries: u32, rob_id: u32, banks: [u32; 2], groups: [u32; 2], input: &[u32]) -> Step { + let count = input.len(); + let bytes = bank_entries as usize * ROW_BYTES; + if count == 0 || count % BLOCK != 0 || count > bytes / 4 || count + count / BLOCK > bytes { + panic!("mxquant: count must be a positive multiple of 32 within the bank"); + } + if banks[0] == banks[1] { + panic!("mxquant: input and output banks must differ"); + } + if input.iter().any(|x| x & 0x7fff_ffff >= 0x7f80_0000) { + panic!("mxquant: non-finite activation"); + } + let mut preload = Vec::new(); + let mut reads = Vec::new(); + for (row, values) in input.chunks(4).enumerate() { + let bytes: Vec = values.iter().flat_map(|x| x.to_le_bytes()).collect(); + preload.push(Preload { bank: banks[0], group: groups[0], addr: row as u32, data: word(&bytes) }); + reads.push(row as u32); + } + let mut writes = Vec::new(); + for (block, values) in input.chunks(BLOCK).enumerate() { + let maximum = values.iter().map(|x| x & 0x7fff_ffff).max().unwrap(); + let exponent = if maximum == 0 { 0 } else { ((maximum >> 23) as i32 - 135).max(-127) }; + let codes: Vec = values.iter().map(|x| encode(*x, exponent)).collect(); + writes.push(Write { addr: (block * 2) as u32, mask: 0xffff, data: word(&codes[..ROW_BYTES]) }); + writes.push(Write { addr: (block * 2 + 1) as u32, mask: 0xffff, data: word(&codes[ROW_BYTES..]) }); + let byte = count + block; + let lane = byte % ROW_BYTES; + writes.push(Write { + addr: (byte / ROW_BYTES) as u32, + mask: 1 << lane, + data: u128::from((exponent + 127) as u8) << (lane * 8), + }); + } + Step { + cmd: MxquantCmd { + rob_id, + iter: count as u32, + rs2_lo: 0, + rs2_hi: 0, + op1_bank: banks[0], + op2_bank: 0, + wr_bank: banks[1], + op1_group: groups[0], + op2_group: 0, + wr_group: groups[1], + op1_col: 1, + op2_col: 0, + wr_col: 1, + }, + preload, + reads, + writes, + } +} + +// Activations of mxquant_values_test: test 0..3 are signed zeros, edges, +// rounding ties and subnormals; later tests sweep the exponent range. +fn values_test(rng: &mut u32, test: u32) -> Vec { + let edges = [0, 0x8000_0000, 1, 0x8000_0001, 0x007f_ffff, 0x0080_0000, 0x3f88_0000, 0x3f98_0000, 0x43e0_0000, 0x7f7f_ffff, 0xff7f_ffff]; + let subnormals = [0, 1, 0x1000, 0x2000, 0x3000, 0x7f_ffff, 0x8000_0001, 0x807f_ffff]; + (0..BLOCK as u32) + .map(|i| { + *rng ^= *rng << 13; + *rng ^= *rng >> 17; + *rng ^= *rng << 5; + let random = *rng; + match test { + 0 => if i & 1 != 0 { 0x8000_0000 } else { 0 }, + 1 => edges[(i % 11) as usize], + 2 => if i == 0 { 0x43e0_0000 } else if i & 1 != 0 { 0x3f88_0000 } else { 0x3f98_0000 }, + 3 => subnormals[(i % 8) as usize], + _ => (random & 0x807f_ffff) | ((((test - 4) * 4 + (random >> 23 & 3)) % 255) << 23), + } + }) + .collect() +} + +pub fn gen_case(index: u32, bank_entries: u32) -> Case { + let mut rng = Rng(0x2545_f491 ^ index.wrapping_mul(0x9e37_79b9)); + let steps = match index { + // Signed zeros, edges, rounding ties and subnormals, one block each. + 0 => { + let mut state = 0x6d2b_79f5u32; + let input: Vec = (0..4).flat_map(|test| values_test(&mut state, test)).collect(); + vec![step(bank_entries, 1, [3, 4], [0, 0], &input)] + } + // The exponent sweep of mxquant_values_test: 64 blocks, the full input + // capacity of a 512-row bank. + 1 => { + let mut state = 0x6d2b_79f5u32; + let input: Vec = (0..68).flat_map(|test| values_test(&mut state, test)).skip(4 * BLOCK).collect(); + vec![step(bank_entries, 2, [3, 4], [1, 2], &input)] + } + // One block, random magnitudes around one, highest group bases. + 2 => { + let input: Vec = (0..BLOCK).map(|_| (rng.next() & 0x807f_ffff) | ((120 + rng.next() % 16) << 23)).collect(); + vec![step(bank_entries, 15, [1022, 1023], [31, 31], &input)] + } + // Random exponents over the whole finite range, 3 / 4 of the input capacity. + 3 => { + let count = (bank_entries * 3) as usize; + let input: Vec = (0..count).map(|_| (rng.next() & 0x807f_ffff) | ((rng.next() % 255) << 23)).collect(); + vec![step(bank_entries, 7, [0, 1], [5, 6], &input)] + } + // Blocks whose maximum is subnormal or tiny (block exponent clamps to -127). + 4 => { + let input: Vec = (0..2 * BLOCK) + .map(|i| if i < BLOCK { rng.next() & 0x803f_ffff } else { (rng.next() & 0x807f_ffff) | ((rng.next() % 3) << 23) }) + .collect(); + vec![step(bank_entries, 4, [9, 10], [3, 4], &input)] + } + // Back-to-back commands without reset. + 5 => (0..3u32) + .map(|i| { + let count = BLOCK * (1 + (rng.next() % 5) as usize); + let input: Vec = (0..count).map(|_| (rng.next() & 0x807f_ffff) | ((100 + rng.next() % 60) << 23)).collect(); + step(bank_entries, 8 + i, [11 + i, 20 + i], [i, i + 7], &input) + }) + .collect(), + _ => panic!("mxquant: unsupported case {index}"), + }; + Case { steps } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cases_build() { + for index in 0..NUM_CASES { + gen_case(index, 512); + } + } + + // Golden bytes checked by mxquant_values_test. + #[test] + fn values_test_golden() { + let mut state = 0x6d2b_79f5u32; + let block = |state: &mut u32, test| { + let input = values_test(state, test); + let maximum = input.iter().map(|x| x & 0x7fff_ffff).max().unwrap(); + let exponent = if maximum == 0 { 0 } else { ((maximum >> 23) as i32 - 135).max(-127) }; + let codes: Vec = input.iter().map(|x| encode(*x, exponent)).collect(); + (codes, (exponent + 127) as u8) + }; + let (codes, scale) = block(&mut state, 0); + assert_eq!((codes[0], codes[1], scale), (0, 128, 127)); + let (codes, scale) = block(&mut state, 1); + assert_eq!((codes[9], codes[10], scale), (126, 254, 246)); + let (codes, _) = block(&mut state, 2); + assert_eq!((codes[0], codes[1], codes[2]), (126, 56, 58)); + } +} diff --git a/examples/prefab/balls/mxquant/verify/casegen/src/config.rs b/examples/prefab/balls/mxquant/verify/casegen/src/config.rs new file mode 100644 index 000000000..241755f0f --- /dev/null +++ b/examples/prefab/balls/mxquant/verify/casegen/src/config.rs @@ -0,0 +1,9 @@ +pub fn bank_entries() -> u32 { + let path = std::env::var("BB_VERIFY_CONFIG").expect("BB_VERIFY_CONFIG is required"); + let text = std::fs::read_to_string(path).expect("failed to read BB_VERIFY_CONFIG"); + text.lines() + .find_map(|line| line.strip_prefix("bank_entries=")) + .unwrap_or_else(|| panic!("bank_entries missing from BB_VERIFY_CONFIG")) + .parse() + .expect("invalid bank_entries in BB_VERIFY_CONFIG") +} diff --git a/examples/prefab/balls/mxquant/verify/casegen/src/lib.rs b/examples/prefab/balls/mxquant/verify/casegen/src/lib.rs new file mode 100644 index 000000000..5659150db --- /dev/null +++ b/examples/prefab/balls/mxquant/verify/casegen/src/lib.rs @@ -0,0 +1,113 @@ +mod casegen; +mod config; + +use std::cell::RefCell; + +use casegen::{Case, MxquantCmd, Step}; + +thread_local! { + static CURRENT: RefCell> = const { RefCell::new(None) }; +} + +#[no_mangle] +pub extern "C" fn mxquant_num_cases() -> u32 { + casegen::NUM_CASES +} + +#[no_mangle] +pub extern "C" fn mxquant_case_load(index: u32) -> u32 { + let case = casegen::gen_case(index, config::bank_entries()); + let steps = case.steps.len() as u32; + CURRENT.with(|current| *current.borrow_mut() = Some(case)); + steps +} + +fn current_step R, R>(step: u32, f: F) -> R { + CURRENT.with(|current| match current.borrow().as_ref() { + Some(case) => match case.steps.get(step as usize) { + Some(s) => f(s), + None => panic!("mxquant_case: step {step} out of range {}", case.steps.len()), + }, + None => panic!("mxquant_case: no case loaded; call mxquant_case_load first"), + }) +} + +fn idx(index: u32, limit: usize, what: &str) -> usize { + if index as usize >= limit { + panic!("mxquant_case_{what}: index {index} out of range {limit}"); + } + index as usize +} + +#[no_mangle] +pub extern "C" fn mxquant_case_cmd(step: u32, out_ptr: *mut MxquantCmd) { + if out_ptr.is_null() { + panic!("mxquant_case_cmd: null out_ptr"); + } + current_step(step, |s| unsafe { *out_ptr = s.cmd }); +} + +#[no_mangle] +pub extern "C" fn mxquant_case_preload_count(step: u32) -> u32 { + current_step(step, |s| s.preload.len() as u32) +} + +#[no_mangle] +pub extern "C" fn mxquant_case_preload_bank(step: u32, i: u32) -> u32 { + current_step(step, |s| s.preload[idx(i, s.preload.len(), "preload")].bank) +} + +#[no_mangle] +pub extern "C" fn mxquant_case_preload_group(step: u32, i: u32) -> u32 { + current_step(step, |s| s.preload[idx(i, s.preload.len(), "preload")].group) +} + +#[no_mangle] +pub extern "C" fn mxquant_case_preload_addr(step: u32, i: u32) -> u32 { + current_step(step, |s| s.preload[idx(i, s.preload.len(), "preload")].addr) +} + +#[no_mangle] +pub extern "C" fn mxquant_case_preload_lo(step: u32, i: u32) -> u64 { + current_step(step, |s| s.preload[idx(i, s.preload.len(), "preload")].data as u64) +} + +#[no_mangle] +pub extern "C" fn mxquant_case_preload_hi(step: u32, i: u32) -> u64 { + current_step(step, |s| (s.preload[idx(i, s.preload.len(), "preload")].data >> 64) as u64) +} + +#[no_mangle] +pub extern "C" fn mxquant_case_read_count(step: u32) -> u32 { + current_step(step, |s| s.reads.len() as u32) +} + +#[no_mangle] +pub extern "C" fn mxquant_case_read_addr(step: u32, i: u32) -> u32 { + current_step(step, |s| s.reads[idx(i, s.reads.len(), "read")]) +} + +#[no_mangle] +pub extern "C" fn mxquant_case_write_count(step: u32) -> u32 { + current_step(step, |s| s.writes.len() as u32) +} + +#[no_mangle] +pub extern "C" fn mxquant_case_write_addr(step: u32, i: u32) -> u32 { + current_step(step, |s| s.writes[idx(i, s.writes.len(), "write")].addr) +} + +#[no_mangle] +pub extern "C" fn mxquant_case_write_mask(step: u32, i: u32) -> u32 { + current_step(step, |s| u32::from(s.writes[idx(i, s.writes.len(), "write")].mask)) +} + +#[no_mangle] +pub extern "C" fn mxquant_case_write_lo(step: u32, i: u32) -> u64 { + current_step(step, |s| s.writes[idx(i, s.writes.len(), "write")].data as u64) +} + +#[no_mangle] +pub extern "C" fn mxquant_case_write_hi(step: u32, i: u32) -> u64 { + current_step(step, |s| (s.writes[idx(i, s.writes.len(), "write")].data >> 64) as u64) +} diff --git a/examples/prefab/balls/mxquant/verify/filelists/mxquant_ball.f b/examples/prefab/balls/mxquant/verify/filelists/mxquant_ball.f new file mode 100644 index 000000000..a27788ab5 --- /dev/null +++ b/examples/prefab/balls/mxquant/verify/filelists/mxquant_ball.f @@ -0,0 +1,25 @@ ++incdir+@UVM@/src/core ++incdir+@UVM@/src/ball ++incdir+@UVM@/src/ball/agents/cmd ++incdir+@UVM@/src/ball/agents/mem ++incdir+@UVM@/src/ball/agents/resp ++incdir+@UVM@/src/ball/cov ++incdir+@UVM@/src/ball/env ++incdir+src ++incdir+src/pkg ++incdir+src/common ++incdir+src/seq ++incdir+src/cov ++incdir+src/env ++incdir+src/tests ++incdir+@RTL@ + +@UVM@/src/core/package.sv +@UVM@/src/ball/bb_blink_if.sv +@UVM@/src/ball/bb_uvm_pkg.sv + +@RTL@/Encode.sv +@RTL@/MxquantBall.sv + +src/pkg/mxquant_pkg.sv +src/tb_top.sv diff --git a/examples/prefab/balls/mxquant/verify/src/common/mxquant_defs.svh b/examples/prefab/balls/mxquant/verify/src/common/mxquant_defs.svh new file mode 100644 index 000000000..54c498bbe --- /dev/null +++ b/examples/prefab/balls/mxquant/verify/src/common/mxquant_defs.svh @@ -0,0 +1,39 @@ +typedef struct { + int unsigned rob_id; + int unsigned iter; + int unsigned rs2_lo; + int unsigned rs2_hi; + int unsigned op1_bank; + int unsigned op2_bank; + int unsigned wr_bank; + int unsigned op1_group; + int unsigned op2_group; + int unsigned wr_group; + int unsigned op1_col; + int unsigned op2_col; + int unsigned wr_col; +} mxquant_cmd_dpi_t; + +import "DPI-C" function int unsigned mxquant_num_cases(); +import "DPI-C" function int unsigned mxquant_case_load(input int unsigned index); +import "DPI-C" function void mxquant_case_cmd(input int unsigned step, output mxquant_cmd_dpi_t cmd); +import "DPI-C" function int unsigned mxquant_case_preload_count(input int unsigned step); +import "DPI-C" function int unsigned mxquant_case_preload_bank(input int unsigned step, input int unsigned i); +import "DPI-C" function int unsigned mxquant_case_preload_group(input int unsigned step, input int unsigned i); +import "DPI-C" function int unsigned mxquant_case_preload_addr(input int unsigned step, input int unsigned i); +import "DPI-C" function longint unsigned mxquant_case_preload_lo(input int unsigned step, input int unsigned i); +import "DPI-C" function longint unsigned mxquant_case_preload_hi(input int unsigned step, input int unsigned i); +import "DPI-C" function int unsigned mxquant_case_read_count(input int unsigned step); +import "DPI-C" function int unsigned mxquant_case_read_addr(input int unsigned step, input int unsigned i); +import "DPI-C" function int unsigned mxquant_case_write_count(input int unsigned step); +import "DPI-C" function int unsigned mxquant_case_write_addr(input int unsigned step, input int unsigned i); +import "DPI-C" function int unsigned mxquant_case_write_mask(input int unsigned step, input int unsigned i); +import "DPI-C" function longint unsigned mxquant_case_write_lo(input int unsigned step, input int unsigned i); +import "DPI-C" function longint unsigned mxquant_case_write_hi(input int unsigned step, input int unsigned i); + +`ifndef MXQUANT_FUNCT7 +`error "MXQUANT_FUNCT7 must be provided by the selected ballISA" +`endif + +localparam int MXQUANT_THREAD_FUNCT7 = `MXQUANT_FUNCT7; +localparam int MXQUANT_TIMEOUT_CYCLES = 100000; diff --git a/examples/prefab/balls/mxquant/verify/src/common/mxquant_items.svh b/examples/prefab/balls/mxquant/verify/src/common/mxquant_items.svh new file mode 100644 index 000000000..f70b5a4fc --- /dev/null +++ b/examples/prefab/balls/mxquant/verify/src/common/mxquant_items.svh @@ -0,0 +1,80 @@ +class mxquant_cmd_item extends bb_blink_cmd_item; + `uvm_object_utils(mxquant_cmd_item) + + int unsigned preload_bank[]; + int unsigned preload_group[]; + int unsigned preload_addr[]; + bit [127:0] preload_data[]; + int unsigned read_addr[]; + int unsigned write_addr[]; + bit [15:0] write_mask[]; + bit [127:0] write_data[]; + + function new(string name = "mxquant_cmd_item"); + super.new(name); + endfunction + + function void load_rust_step(int unsigned step); + mxquant_cmd_dpi_t cmd; + int unsigned n; + + mxquant_case_cmd(step, cmd); + rob_id = cmd.rob_id[BB_BLINK_ROB_ID_W-1:0]; + funct = MXQUANT_THREAD_FUNCT7[6:0]; + rs2 = {cmd.rs2_hi, cmd.rs2_lo}; + iter = cmd.iter; + op1_bank = cmd.op1_bank; + op2_bank = cmd.op2_bank; + wr_bank = cmd.wr_bank; + op1_group = cmd.op1_group; + op2_group = cmd.op2_group; + wr_group = cmd.wr_group; + op1_col = cmd.op1_col; + op2_col = cmd.op2_col; + wr_col = cmd.wr_col; + wr_read_bank = '0; + wr_arch_bank = cmd.wr_bank; + if (rob_id != cmd.rob_id || op1_bank != cmd.op1_bank || op2_bank != cmd.op2_bank || + wr_bank != cmd.wr_bank || op1_group != cmd.op1_group || op2_group != cmd.op2_group || + wr_group != cmd.wr_group) + `uvm_fatal("CASE", "command field exceeds the configured BLink width") + + n = mxquant_case_preload_count(step); + preload_bank = new[n]; + preload_group = new[n]; + preload_addr = new[n]; + preload_data = new[n]; + for (int unsigned i = 0; i < n; i++) begin + preload_bank[i] = mxquant_case_preload_bank(step, i); + preload_group[i] = mxquant_case_preload_group(step, i); + preload_addr[i] = mxquant_case_preload_addr(step, i); + preload_data[i] = {mxquant_case_preload_hi(step, i), mxquant_case_preload_lo(step, i)}; + end + n = mxquant_case_read_count(step); + read_addr = new[n]; + for (int unsigned i = 0; i < n; i++) read_addr[i] = mxquant_case_read_addr(step, i); + n = mxquant_case_write_count(step); + write_addr = new[n]; + write_mask = new[n]; + write_data = new[n]; + for (int unsigned i = 0; i < n; i++) begin + write_addr[i] = mxquant_case_write_addr(step, i); + write_mask[i] = mxquant_case_write_mask(step, i); + write_data[i] = {mxquant_case_write_hi(step, i), mxquant_case_write_lo(step, i)}; + end + endfunction + + function void do_copy(uvm_object rhs); + mxquant_cmd_item rhs_; + super.do_copy(rhs); + if (!$cast(rhs_, rhs)) `uvm_fatal("COPY", "rhs is not mxquant_cmd_item") + preload_bank = rhs_.preload_bank; + preload_group = rhs_.preload_group; + preload_addr = rhs_.preload_addr; + preload_data = rhs_.preload_data; + read_addr = rhs_.read_addr; + write_addr = rhs_.write_addr; + write_mask = rhs_.write_mask; + write_data = rhs_.write_data; + endfunction +endclass diff --git a/examples/prefab/balls/mxquant/verify/src/cov/mxquant_cov.svh b/examples/prefab/balls/mxquant/verify/src/cov/mxquant_cov.svh new file mode 100644 index 000000000..498310be0 --- /dev/null +++ b/examples/prefab/balls/mxquant/verify/src/cov/mxquant_cov.svh @@ -0,0 +1,76 @@ +class mxquant_cov extends uvm_component; + `uvm_component_utils(mxquant_cov) + + uvm_analysis_imp_stim #(bb_blink_cmd_item, mxquant_cov) stim_imp; + + int unsigned cur_count; + int unsigned cur_scale; + bit [6:0] cur_magnitude; + bit cur_sign; + int unsigned cmd_count; + + covergroup cmd_cg; + coverpoint cur_count { + bins one_block = {32}; + bins blocks = {[33 : 1024]}; + bins many = {[1025 : 2047]}; + bins capacity = {2048}; + } + endgroup + + covergroup scale_cg; + coverpoint cur_scale { + bins clamped = {0}; + bins low = {[1 : 118]}; + bins unit = {[119 : 135]}; + bins high = {[136 : 246]}; + } + endgroup + + covergroup code_cg; + coverpoint cur_magnitude { + bins zero = {0}; bins subnormal = {[1 : 7]}; bins normal = {[8 : 125]}; bins clipped = {126}; + } + coverpoint cur_sign; + endgroup + + function new(string name, uvm_component parent); + super.new(name, parent); + stim_imp = new("stim_imp", this); + cmd_cg = new(); + scale_cg = new(); + code_cg = new(); + endfunction + + function void write_stim(bb_blink_cmd_item item); + mxquant_cmd_item stim; + if (!$cast(stim, item)) `uvm_fatal("COV", "stim item is not mxquant_cmd_item") + cur_count = stim.iter; + cmd_cg.sample(); + foreach (stim.write_data[i]) begin + for (int b = 0; b < 16; b++) begin + if (!stim.write_mask[i][b]) continue; + if (stim.write_mask[i] == 16'hFFFF) begin + {cur_sign, cur_magnitude} = stim.write_data[i][b*8+:8]; + code_cg.sample(); + end else begin + cur_scale = stim.write_data[i][b*8+:8]; + scale_cg.sample(); + end + end + end + cmd_count++; + endfunction + + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (cmd_count == 0) `uvm_fatal("COV", "no cmd observed") + if (cmd_cg.get_coverage() != 100.0) + `uvm_fatal("COV", $sformatf("mxquant count coverage %0.2f%%", cmd_cg.get_coverage())) + if (scale_cg.get_coverage() != 100.0) + `uvm_fatal("COV", $sformatf("mxquant scale coverage %0.2f%%", scale_cg.get_coverage())) + if (code_cg.get_coverage() != 100.0) + `uvm_fatal("COV", $sformatf("mxquant code coverage %0.2f%%", code_cg.get_coverage())) + endfunction +endclass diff --git a/examples/prefab/balls/mxquant/verify/src/env/mxquant_env.svh b/examples/prefab/balls/mxquant/verify/src/env/mxquant_env.svh new file mode 100644 index 000000000..5cbc20981 --- /dev/null +++ b/examples/prefab/balls/mxquant/verify/src/env/mxquant_env.svh @@ -0,0 +1,27 @@ +class mxquant_env extends bb_blink_env #(`BB_IN_BW, `BB_OUT_BW); + `uvm_component_utils(mxquant_env) + + mxquant_scoreboard scb; + mxquant_cov icov; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + scb = mxquant_scoreboard::type_id::create("scb", this); + icov = mxquant_cov::type_id::create("icov", this); + endfunction + + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + scb.mem_model = mem_model; + cmd_agent.stim_ap.connect(scb.stim_imp); + cmd_agent.stim_ap.connect(icov.stim_imp); + cmd_agent.cmd_ap.connect(scb.cmd_imp); + read_mon.read_ap.connect(scb.read_imp); + write_mon.write_ap.connect(scb.write_imp); + resp_mon.resp_ap.connect(scb.resp_imp); + endfunction +endclass diff --git a/examples/prefab/balls/mxquant/verify/src/env/mxquant_scoreboard.svh b/examples/prefab/balls/mxquant/verify/src/env/mxquant_scoreboard.svh new file mode 100644 index 000000000..cdd3fbb01 --- /dev/null +++ b/examples/prefab/balls/mxquant/verify/src/env/mxquant_scoreboard.svh @@ -0,0 +1,173 @@ +class mxquant_scoreboard extends uvm_scoreboard; + `uvm_component_utils(mxquant_scoreboard) + + uvm_analysis_imp_stim #(bb_blink_cmd_item, mxquant_scoreboard) stim_imp; + uvm_analysis_imp_cmd #(bb_blink_cmd_item, mxquant_scoreboard) cmd_imp; + uvm_analysis_imp_read #(bb_blink_read_item, mxquant_scoreboard) read_imp; + uvm_analysis_imp_write #(bb_blink_write_item, mxquant_scoreboard) write_imp; + uvm_analysis_imp_resp #(bb_blink_resp_item, mxquant_scoreboard) resp_imp; + + bb_blink_mem_model #(`BB_IN_BW, `BB_OUT_BW) mem_model; + + mxquant_cmd_item stim_q[$]; + int unsigned cmd_count; + int unsigned read_count; + int unsigned write_count; + int unsigned resp_count; + + function new(string name, uvm_component parent); + super.new(name, parent); + stim_imp = new("stim_imp", this); + cmd_imp = new("cmd_imp", this); + read_imp = new("read_imp", this); + write_imp = new("write_imp", this); + resp_imp = new("resp_imp", this); + endfunction + + function void write_stim(bb_blink_cmd_item item); + mxquant_cmd_item got; + mxquant_cmd_item clone; + + if (stim_q.size() != 0) `uvm_fatal("SCB", "single outstanding command supported") + if (!$cast(got, item)) `uvm_fatal("SCB", "stim item is not mxquant_cmd_item") + if (!$cast(clone, got.clone())) `uvm_fatal("SCB", "failed to clone stimulus item") + stim_q.push_back(clone); + if (mem_model == null) `uvm_fatal("SCB", "mem_model handle not set") + mem_model.clear_mem(); + foreach (clone.preload_data[i]) + mem_model.load_word_g(clone.preload_bank[i], clone.preload_group[i], clone.preload_addr[i], + clone.preload_data[i]); + mem_model.arm(); + endfunction + + function void write_cmd(bb_blink_cmd_item item); + mxquant_cmd_item stim; + bb_blink_cmd_item want; + stim = current_stim("CMD"); + // Compare as plain BLink commands; the comparer rejects differing types. + want = bb_blink_cmd_item::type_id::create("want"); + want.copy(stim); + verification_pkg::evidence::checks++; + if (!item.compare(want)) + `uvm_fatal("CMD", $sformatf("command mismatch: got %s exp %s", item.sprint(), want.sprint())) + cmd_count++; + endfunction + + function void write_read(bb_blink_read_item item); + mxquant_cmd_item stim; + + stim = current_stim("READ"); + if (item.port != 0) + `uvm_fatal("READ", $sformatf("MxquantBall has one read port, got port %0d", item.port)) + if (read_count >= stim.read_addr.size()) + `uvm_fatal("READ", $sformatf("extra read %0d", read_count)) + verification_pkg::evidence::checks++; + if (item.rob_id !== stim.rob_id) + `uvm_fatal("READ", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + if (item.bank_id !== stim.op1_bank) + `uvm_fatal("READ", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.op1_bank)) + if (item.group_id !== stim.op1_group) + `uvm_fatal("READ", $sformatf("group mismatch: got %0d exp %0d", item.group_id, stim.op1_group + )) + if (item.addr !== stim.read_addr[read_count][BB_BLINK_BANK_ADDR_W-1:0]) + `uvm_fatal("READ", $sformatf( + "read %0d addr mismatch: got %0d exp %0d", + read_count, + item.addr, + stim.read_addr[read_count] + )) + read_count++; + endfunction + + function void write_write(bb_blink_write_item item); + mxquant_cmd_item stim; + bit [127:0] bytes; + + stim = current_stim("WRITE"); + if (item.port != 0) + `uvm_fatal("WRITE", $sformatf("MxquantBall has one write port, got port %0d", item.port)) + if (write_count >= stim.write_addr.size()) + `uvm_fatal("WRITE", $sformatf("extra write %0d", write_count)) + verification_pkg::evidence::checks++; + if (item.bank_id !== stim.wr_bank) + `uvm_fatal("WRITE", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.wr_bank)) + if (item.group_id !== stim.wr_group) + `uvm_fatal("WRITE", $sformatf("group mismatch: got %0d exp %0d", item.group_id, stim.wr_group + )) + if (item.rob_id !== stim.rob_id) + `uvm_fatal("WRITE", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + if (item.addr !== stim.write_addr[write_count][BB_BLINK_BANK_ADDR_W-1:0]) + `uvm_fatal("WRITE", $sformatf( + "write %0d addr mismatch: got %0d exp %0d", + write_count, + item.addr, + stim.write_addr[write_count] + )) + if (item.mask !== stim.write_mask[write_count]) + `uvm_fatal("WRITE", $sformatf( + "write %0d mask mismatch: got 0x%04h exp 0x%04h", + write_count, + item.mask, + stim.write_mask[write_count] + )) + for (int b = 0; b < 16; b++) bytes[b*8+:8] = {8{item.mask[b]}}; + if ((item.data & bytes) !== (stim.write_data[write_count] & bytes)) + `uvm_fatal("SCB", $sformatf( + "write %0d data mismatch at addr %0d: got 0x%032h exp 0x%032h", + write_count, + item.addr, + item.data, + stim.write_data[write_count] + )) + write_count++; + endfunction + + function void write_resp(bb_blink_resp_item item); + mxquant_cmd_item stim; + stim = current_stim("RESP"); + verification_pkg::evidence::checks++; + if (item.rob_id !== stim.rob_id) + `uvm_fatal("RESP", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + if (!complete_io()) + `uvm_fatal("RESP", $sformatf( + "response before all traffic: reads=%0d writes=%0d", read_count, write_count)) + resp_count++; + endfunction + + function mxquant_cmd_item current_stim(string tag); + if (stim_q.size() == 0) `uvm_fatal("SCB", $sformatf("%s observed before stimulus", tag)) + return stim_q[0]; + endfunction + + function bit complete_io(); + mxquant_cmd_item stim = stim_q[0]; + return read_count == stim.read_addr.size() && write_count == stim.write_addr.size(); + endfunction + + function bit done(); + return stim_q.size() == 1 && cmd_count == 1 && resp_count == 1 && complete_io(); + endfunction + + function void reset_counters(); + stim_q.delete(); + cmd_count = 0; + read_count = 0; + write_count = 0; + resp_count = 0; + endfunction + + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (!done()) + `uvm_fatal("SCB", $sformatf( + "incomplete: stim=%0d cmds=%0d reads=%0d writes=%0d resp=%0d", + stim_q.size(), + cmd_count, + read_count, + write_count, + resp_count + )) + stim_q.delete(); + endfunction +endclass diff --git a/examples/prefab/balls/mxquant/verify/src/pkg/mxquant_pkg.sv b/examples/prefab/balls/mxquant/verify/src/pkg/mxquant_pkg.sv new file mode 100644 index 000000000..938c14267 --- /dev/null +++ b/examples/prefab/balls/mxquant/verify/src/pkg/mxquant_pkg.sv @@ -0,0 +1,13 @@ +package mxquant_pkg; + import uvm_pkg::*; + import bb_uvm_pkg::*; + `include "uvm_macros.svh" + + `include "common/mxquant_defs.svh" + `include "common/mxquant_items.svh" + `include "seq/mxquant_sequences.svh" + `include "cov/mxquant_cov.svh" + `include "env/mxquant_scoreboard.svh" + `include "env/mxquant_env.svh" + `include "tests/mxquant_test.svh" +endpackage diff --git a/examples/prefab/balls/mxquant/verify/src/seq/mxquant_sequences.svh b/examples/prefab/balls/mxquant/verify/src/seq/mxquant_sequences.svh new file mode 100644 index 000000000..7b2e461ba --- /dev/null +++ b/examples/prefab/balls/mxquant/verify/src/seq/mxquant_sequences.svh @@ -0,0 +1,17 @@ +class mxquant_step_seq extends uvm_sequence #(bb_blink_cmd_item); + `uvm_object_utils(mxquant_step_seq) + + int unsigned step; + + function new(string name = "mxquant_step_seq"); + super.new(name); + endfunction + + task body(); + mxquant_cmd_item item; + item = mxquant_cmd_item::type_id::create("item"); + start_item(item); + item.load_rust_step(step); + finish_item(item); + endtask +endclass diff --git a/examples/prefab/balls/mxquant/verify/src/tb_top.sv b/examples/prefab/balls/mxquant/verify/src/tb_top.sv new file mode 100644 index 000000000..89b105c1d --- /dev/null +++ b/examples/prefab/balls/mxquant/verify/src/tb_top.sv @@ -0,0 +1,78 @@ +module tb_top; + import uvm_pkg::*; + import mxquant_pkg::*; + + bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); + + MxquantBall dut ( + .clock(intf.clock), + .reset(intf.reset), + .blink_status_running(intf.status_running), + .blink_cmdReq_ready(intf.cmd_req_ready), + .blink_cmdReq_valid(intf.cmd_req_valid), + .blink_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .blink_cmdReq_bits_funct(intf.cmd_req_bits_funct), + .blink_cmdReq_bits_rs2(intf.cmd_req_bits_rs2), + .blink_cmdReq_bits_iter(intf.cmd_req_bits_iter), + .blink_cmdReq_bits_op1_bank(intf.cmd_req_bits_op1_bank), + .blink_cmdReq_bits_op2_bank(intf.cmd_req_bits_op2_bank), + .blink_cmdReq_bits_wr_bank(intf.cmd_req_bits_wr_bank), + .blink_cmdReq_bits_op1_group(intf.cmd_req_bits_op1_group), + .blink_cmdReq_bits_op2_group(intf.cmd_req_bits_op2_group), + .blink_cmdReq_bits_wr_group(intf.cmd_req_bits_wr_group), + .blink_cmdReq_bits_op1_col(intf.cmd_req_bits_op1_col), + .blink_cmdReq_bits_op2_col(intf.cmd_req_bits_op2_col), + .blink_cmdReq_bits_wr_col(intf.cmd_req_bits_wr_col), + .blink_cmdReq_bits_wr_read_bank(intf.cmd_req_bits_wr_read_bank), + .blink_cmdReq_bits_wr_arch_bank(intf.cmd_req_bits_wr_arch_bank), + .blink_cmdResp_ready(intf.cmd_resp_ready), + .blink_cmdResp_valid(intf.cmd_resp_valid), + .blink_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .blink_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .blink_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .blink_bankRead_0_group_id(intf.bank_read_group_id[0]), + .blink_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .blink_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .blink_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .blink_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .blink_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .blink_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .blink_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .blink_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .blink_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .blink_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .blink_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .blink_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .blink_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .blink_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .blink_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .blink_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), + .blink_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), + .blink_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), + .blink_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), + .blink_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), + .blink_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), + .blink_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), + .blink_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), + .blink_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), + .blink_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), + .blink_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), + .blink_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), + .blink_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), + .blink_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), + .blink_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), + .blink_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), + .blink_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]) + ); + + initial begin + intf.clock = 1'b0; + intf.reset = 1'b1; + forever #5 intf.clock = ~intf.clock; + end + + initial begin + uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); + run_test("mxquant_ball_test"); + end +endmodule diff --git a/examples/prefab/balls/mxquant/verify/src/tests/mxquant_test.svh b/examples/prefab/balls/mxquant/verify/src/tests/mxquant_test.svh new file mode 100644 index 000000000..7b6e27088 --- /dev/null +++ b/examples/prefab/balls/mxquant/verify/src/tests/mxquant_test.svh @@ -0,0 +1,56 @@ +class mxquant_ball_test extends uvm_test; + `uvm_component_utils(mxquant_ball_test) + + typedef virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) vif_t; + vif_t vif; + mxquant_env env; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(vif_t)::get(this, "", "vif", vif)) + `uvm_fatal("NOVIF", "bb_blink_if not found") + env = mxquant_env::type_id::create("env", this); + endfunction + + task run_phase(uvm_phase phase); + phase.raise_objection(this); + for (int unsigned index = 0; index < mxquant_num_cases(); index++) run_case(index); + phase.drop_objection(this); + endtask + + // Steps of one case run back to back on the same reset. + task run_case(int unsigned index); + int unsigned steps; + apply_reset(); + steps = mxquant_case_load(index); + for (int unsigned step = 0; step < steps; step++) run_step(index, step); + endtask + + task run_step(int unsigned index, int unsigned step); + mxquant_step_seq seq; + int cycles = 0; + env.scb.reset_counters(); + seq = mxquant_step_seq::type_id::create("seq"); + seq.step = step; + seq.start(env.cmd_agent.seqr); + while (!env.scb.done()) begin + @(posedge vif.clock); + cycles++; + if (cycles > MXQUANT_TIMEOUT_CYCLES) + `uvm_fatal("TIMEOUT", $sformatf("MxquantBall case %0d step %0d timeout", index, step)) + end + `uvm_info("MXQUANT", $sformatf("MxquantBall case %0d step %0d passed in %0d cycles", index, + step, cycles), UVM_LOW) + endtask + + task apply_reset(); + vif.reset = 1'b1; + repeat (5) @(posedge vif.clock); + vif.reset = 1'b0; + repeat (2) @(posedge vif.clock); + endtask +endclass diff --git a/examples/prefab/balls/mxquant/workloads/ctests/CMakeLists.txt b/examples/prefab/balls/mxquant/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..b1358b742 --- /dev/null +++ b/examples/prefab/balls/mxquant/workloads/ctests/CMakeLists.txt @@ -0,0 +1,21 @@ +math( + EXPR MXQUANT_BANK_BYTES + "${DESIGN_PROFILES_${BUCKYBALL_CTEST_TARGET}_THREAD_MEM_BANK_ENTRIES} * ${DESIGN_PROFILES_${BUCKYBALL_CTEST_TARGET}_THREAD_MEM_BANK_WIDTH} / 8" +) +math(EXPR MXQUANT_BANK_TAIL "${MXQUANT_BANK_BYTES} % 128") +if(MXQUANT_BANK_BYTES LESS 3840 OR MXQUANT_BANK_TAIL) + message( + FATAL_ERROR + "Mxquant CTests require banks >= 3840 bytes and a 32-element FP32 capacity; actual ${MXQUANT_BANK_BYTES} bytes" + ) +endif() +add_buckyball_ctest(mxquant_values_test.c) +add_buckyball_ctest(mxquant_capacity_test.c) +set(TEST_NAME + "${DESIGN_NAME}-${BUCKYBALL_CTEST_TARGET}-ctest-mxquant_values_test" +) +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${TEST_NAME}-baremetal + APPEND + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/../reference.h +) diff --git a/examples/balls/mxquant/workloads/ctests/mxquant_capacity_test.c b/examples/prefab/balls/mxquant/workloads/ctests/mxquant_capacity_test.c similarity index 77% rename from examples/balls/mxquant/workloads/ctests/mxquant_capacity_test.c rename to examples/prefab/balls/mxquant/workloads/ctests/mxquant_capacity_test.c index 0e7104131..40e811612 100644 --- a/examples/balls/mxquant/workloads/ctests/mxquant_capacity_test.c +++ b/examples/prefab/balls/mxquant/workloads/ctests/mxquant_capacity_test.c @@ -5,7 +5,10 @@ #include #include -enum { BANK_BYTES = BANK_LINES * BANK_WIDTH / 8, MAX_COUNT = BANK_BYTES / 4 }; +enum { + BANK_BYTES = THREAD_MEM_BANK_ENTRIES * THREAD_MEM_BANK_WIDTH / 8, + MAX_COUNT = BANK_BYTES / 4 +}; static uint32_t input[MAX_COUNT] __attribute__((aligned(64))); static uint8_t actual[BANK_BYTES] __attribute__((aligned(64))); @@ -18,10 +21,10 @@ int main(void) { memset(actual, 0x5a, sizeof(actual)); bb_mem_alloc(3, 1, 1); bb_mem_alloc(4, 1, 1); - bb_mvin((uintptr_t)input, 3, BANK_LINES, 1); - bb_mvin((uintptr_t)actual, 4, BANK_LINES, 1); + bb_mvin((uintptr_t)input, 3, THREAD_MEM_BANK_ENTRIES, 1); + bb_mvin((uintptr_t)actual, 4, THREAD_MEM_BANK_ENTRIES, 1); bb_mxquant(3, 4, count); - bb_mvout((uintptr_t)actual, 4, BANK_LINES, 1); + bb_mvout((uintptr_t)actual, 4, THREAD_MEM_BANK_ENTRIES, 1); for (int i = 0; i < BANK_BYTES; ++i) { uint8_t expected = i < count ? ((i & 1) ? 0xf8 : 0x78) : i < count + count / 32 ? 119 diff --git a/examples/balls/mxquant/workloads/ctests/mxquant_values_test.c b/examples/prefab/balls/mxquant/workloads/ctests/mxquant_values_test.c similarity index 100% rename from examples/balls/mxquant/workloads/ctests/mxquant_values_test.c rename to examples/prefab/balls/mxquant/workloads/ctests/mxquant_values_test.c diff --git a/examples/balls/mxquant/workloads/isa/mxquant.h b/examples/prefab/balls/mxquant/workloads/isa/mxquant.h similarity index 86% rename from examples/balls/mxquant/workloads/isa/mxquant.h rename to examples/prefab/balls/mxquant/workloads/isa/mxquant.h index bdf29e232..04eed1988 100644 --- a/examples/balls/mxquant/workloads/isa/mxquant.h +++ b/examples/prefab/balls/mxquant/workloads/isa/mxquant.h @@ -6,5 +6,5 @@ #define bb_mxquant(input, output, count) \ BUCKYBALL_INSTRUCTION_R_R(BB_BANK0(input) | BB_BANK2(output) | \ BB_ITER(count), \ - 0, BB_FUNC7(MXQUANT)) + 0, FUNCT7(mxquant, MXQUANT)) #endif diff --git a/examples/prefab/balls/mxquant/workloads/mlir_tests/ball/mxquant_two_blocks.mlir b/examples/prefab/balls/mxquant/workloads/mlir_tests/ball/mxquant_two_blocks.mlir new file mode 100644 index 000000000..0024cc141 --- /dev/null +++ b/examples/prefab/balls/mxquant/workloads/mlir_tests/ball/mxquant_two_blocks.mlir @@ -0,0 +1,9 @@ +// RUN: buddy-opt %s -lower-buckyball -canonicalize | FileCheck %s +// CHECK-LABEL: func.func @quant +// CHECK-DAG: arith.constant 68721573889 : i64 +// CHECK-DAG: arith.constant 0 : i64 +// The caller loads 64 FP32 into bank 1 and the output guard into bank 2. +func.func @quant() attributes {llvm.emit_c_interface} { + mxquant_ball.quant_mxfp8 <{input = 1 : i64, output = 2 : i64, count = 64 : i64}> + return +} diff --git a/examples/prefab/balls/mxquant/workloads/mlir_tests/bank/mxquant_two_blocks.mlir b/examples/prefab/balls/mxquant/workloads/mlir_tests/bank/mxquant_two_blocks.mlir new file mode 100644 index 000000000..9b00fcc6f --- /dev/null +++ b/examples/prefab/balls/mxquant/workloads/mlir_tests/bank/mxquant_two_blocks.mlir @@ -0,0 +1,14 @@ +// Quantizes two blocks of 32 FP32. The output comes in holding a guard +// pattern and goes back with the codes and scales. +func.func @quant(%input: memref<16x4xi32>, %output: memref<5x16xi8>) + attributes {llvm.emit_c_interface} { + %in_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %out_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %li = buckyball.bank_mvin %input, %in_bank <{depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> + %lo = buckyball.bank_mvin %output, %out_bank <{depth = 5 : i64, stride = 1 : i64}> : memref<5x16xi8> + %packed = mxquant_ball.bank_quant_mxfp8 %li -> %lo <{count = 64 : i64}> + %stored = buckyball.bank_mvout %output, %packed <{depth = 5 : i64, stride = 1 : i64}> : memref<5x16xi8> + buckyball.bank_release %li + buckyball.bank_release %stored + return +} diff --git a/examples/balls/mxquant/workloads/reference.h b/examples/prefab/balls/mxquant/workloads/reference.h similarity index 94% rename from examples/balls/mxquant/workloads/reference.h rename to examples/prefab/balls/mxquant/workloads/reference.h index 765b062b7..107098472 100644 --- a/examples/balls/mxquant/workloads/reference.h +++ b/examples/prefab/balls/mxquant/workloads/reference.h @@ -2,7 +2,7 @@ #define MXQUANT_REFERENCE_H #include -/* Integer reference from bbsw/kernels/mxfp8/quant.cpp; no FP rounding state. */ +/* Integer MXFP8 quantization reference; no FP rounding state. */ static inline uint32_t rounded_shift(uint32_t value, unsigned shift) { if (shift > 31) return 0; diff --git a/examples/prefab/balls/redmule/arch/RedmuleBall.scala b/examples/prefab/balls/redmule/arch/RedmuleBall.scala new file mode 100644 index 000000000..bed8c7bc0 --- /dev/null +++ b/examples/prefab/balls/redmule/arch/RedmuleBall.scala @@ -0,0 +1,235 @@ +package examples.balls.redmule + +import framework.acc.balldomain.bbus.{BallFactory, BallPorts} +import buckyball.balls.redmule.{Params => ParametersProto} +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.acc.balldomain.ball.blink.{BallCommand, BallStatus, BlinkIO, HasBlink} + +@instantiable +class RedmuleBall( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Module + with HasBlink { + val ball = bp.balls.find(_.id == id).get + + ball.parameters.unpack(classOf[ParametersProto]) + + require(ball.inBW == 2 && ball.outBW == 1) + require( + mp.bankWidth == 128 && mp.bankMaskLen == 16 + ) + require( + mp.bankEntries <= 65535 && mp.sharedBankEntries <= 65535 + ) + + @public + val blink = IO(new BlinkIO(fp, mp, bp, id)) + + val Seq(idle, channels, configure, controlReply, running, complete) = Enum(6) + val state = RegInit(idle) + + val issue = Reg(new BallCommand(fp, mp)) + + val controlStep = RegInit(0.U(4.W)) + val finished = RegInit(false.B) + val rows = issue.rs2(11, 0) + val aBase = issue.rs2(21, 12) + val bBase = issue.rs2(31, 22) + val cBase = issue.rs2(41, 32) + val first = issue.rs2(42) + val native = Instantiate(new RedmuleNative) + native.io.clk_i := clock + native.io.rst_ni := !reset.asBool + native.io.ctrl_req_i := state === configure + native.io.ctrl_wen_i := controlStep === 0.U + native.io.ctrl_addr_i := VecInit(Seq(4, 44, 48, 52, 32, 36, 40, 60, 0).map(_.U(32.W)))(controlStep) + native.io.ctrl_data_i := VecInit(Seq( + 0.U(32.W), + 16777216.U | (aBase << 4), + 33554432.U | (bBase << 4), + 50331648.U | (cBase << 4), + (8.U(32.W) << 16) | rows, + 41943040.U(32.W) | ((!first).asUInt << 20) | issue.iter(15, 0), + 0.U(32.W), + 0.U(32.W), + 0.U(32.W) + ))(controlStep) + + val Seq(busIdle, readRequest, readReply, writeRequest, writeReply, advance, grant, response) = Enum(8) + val busState = RegInit(busIdle) + val address = Reg(UInt(32.W)) + val load = Reg(Bool()) + val data = Reg(UInt(256.W)) + val mask = Reg(UInt(32.W)) + val transactionId = Reg(UInt(8.W)) + val half = RegInit(false.B) + val responseData = Reg(UInt(256.W)) + val region = address(25, 24) + val bankRow = address(23, 4) +& half + val startRow = Mux(region === 1.U, aBase, Mux(region === 2.U, bBase, cBase)) + val span = Mux(region === 1.U, rows * (issue.iter(15, 0) >> 3), Mux(region === 2.U, issue.iter(15, 0), rows)) + val inWindow = bankRow >= startRow && bankRow < (startRow +& span) + val readPort = Mux(region === 2.U, 1.U, 0.U) + + // A continuation reads the previous C from the bank the destination held before this write. + val readBank = Mux( + region === 1.U, + issue.op1_bank, + Mux(region === 2.U, issue.op2_bank, issue.wr_read_bank) + ) + + val readGroup = Mux(region === 1.U, issue.op1_group, Mux(region === 2.U, issue.op2_group, issue.wr_group)) + + val writeMask = Mux(half, mask(31, 16), mask(15, 0)) + val writeData = Mux(half, data(255, 128), data(127, 0)) + native.io.mem_gnt_i := busState === grant + native.io.mem_valid_i := busState === response + native.io.mem_data_i := responseData + native.io.mem_id_i := transactionId + + for (port <- 0 until 2) { + val read = blink.bankRead(port) + read.rob_id := issue.rob_id + read.bank_id := readBank + read.group_id := readGroup + read.io.req.valid := busState === readRequest && readPort === port.U && inWindow + read.io.req.bits.addr := bankRow + read.io.resp.ready := busState === readReply && readPort === port.U + when(read.io.req.fire)(busState := readReply) + when(read.io.resp.fire) { + responseData := Mux( + half, + Cat(read.io.resp.bits.data, responseData(127, 0)), + Cat(responseData(255, 128), read.io.resp.bits.data) + ) + busState := advance + } + } + val write = blink.bankWrite(0) + write.rob_id := issue.rob_id + write.bank_id := issue.wr_bank + write.group_id := issue.wr_group + write.io.req.valid := busState === writeRequest && writeMask.orR && inWindow + write.io.req.bits.addr := bankRow + write.io.req.bits.data := writeData + write.io.req.bits.mask := VecInit((0 until 16).map(writeMask(_))) + write.io.resp.ready := busState === writeReply + when(write.io.req.fire) { + assert(region === 3.U && inWindow, "RedMulE store outside C window") + busState := writeReply + } + when(write.io.resp.fire) { + assert(write.io.resp.bits.ok, "RedMulE bank write failed") + busState := advance + } + when(busState === busIdle && state === running && native.io.mem_req_o) { + assert( + native.io.mem_addr_o(31, 26) === 0.U && native.io.mem_addr_o(25, 24) =/= 0.U && + native.io.mem_addr_o(3, 0) === 0.U, + "RedMulE address cannot map to a bank row" + ) + address := native.io.mem_addr_o + data := native.io.mem_data_o + mask := native.io.mem_be_o + load := native.io.mem_wen_o + transactionId := native.io.mem_id_o + half := false.B + responseData := 0.U + busState := Mux(native.io.mem_wen_o, readRequest, writeRequest) + } + // Unused lanes in the native 256-bit load are zero-filled at a tensor boundary. + when(busState === readRequest && !inWindow)(busState := advance) + when(busState === writeRequest && (!writeMask.orR || !inWindow))(busState := advance) + when(busState === advance) { + when(!half) { + half := true.B + busState := Mux(load, readRequest, writeRequest) + }.otherwise(busState := grant) + } + when(busState === grant) { + assert( + native.io.mem_req_o && native.io.mem_addr_o === address && native.io.mem_wen_o === load, + "RedMulE request changed before grant" + ) + busState := Mux(load, response, busIdle) + } + when(busState === response && native.io.mem_ready_o)(busState := busIdle) + + blink.cmdReq.ready := state === idle + blink.cmdResp.valid := state === complete + blink.cmdResp.bits.rob_id := issue.rob_id + blink.status.idle := state === idle + blink.status.running := state =/= idle + when(blink.cmdReq.fire) { + val cmd = blink.cmdReq.bits + val m = cmd.rs2(11, 0) + assert( + cmd.op1_col === 1.U && cmd.op2_col === 1.U && cmd.wr_col === 1.U, + "REDMULE_F16 invalid bank groups" + ) + assert( + cmd.rs2(63, 43) === 0.U && m =/= 0.U && cmd.iter =/= 0.U && cmd.iter(2, 0) === 0.U, + "REDMULE_F16 invalid dimensions or reserved bits" + ) + assert( + cmd.op1_bank =/= cmd.op2_bank && cmd.op1_bank =/= cmd.wr_bank && cmd.op2_bank =/= cmd.wr_bank, + "REDMULE_F16 banks must differ" + ) + for ( + (bank, footprint) <- Seq( + (cmd.op1_bank, cmd.rs2(21, 12) +& m * (cmd.iter >> 3)), + (cmd.op2_bank, cmd.rs2(31, 22) +& cmd.iter), + (cmd.wr_bank, cmd.rs2(41, 32) +& m) + ) + ) { + val shared = bank >= fp.shared_vbank_base.U + val capacity = Mux( + shared, + mp.sharedBankEntries.U, + mp.bankEntries.U + ) + assert(footprint <= capacity, "REDMULE_F16 bank window exceeds capacity") + } + issue := blink.cmdReq.bits + finished := false.B + controlStep := 0.U + state := channels + } + when(state === channels)(state := configure) + when(state === configure && native.io.ctrl_gnt_o)(state := controlReply) + when(state === controlReply && native.io.ctrl_valid_o) { + when(controlStep === 0.U) { + assert(native.io.ctrl_data_o =/= "hffffffff".U, "RedMulE has no free job context") + } + when(controlStep === 8.U)(state := running).otherwise { + controlStep := controlStep + 1.U + state := configure + } + } + when(state === running && native.io.evt_o)(finished := true.B) + when(state === running && finished && !native.io.busy_o && busState === busIdle)(state := complete) + when(blink.cmdResp.fire)(state := idle) +} + +object RedmuleBall extends BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts = + BallPorts(Instantiate(new RedmuleBall(fp, mp, bp, id)).blink, None) + +} diff --git a/examples/prefab/balls/redmule/arch/RedmuleNative.scala b/examples/prefab/balls/redmule/arch/RedmuleNative.scala new file mode 100644 index 000000000..6ac4f1506 --- /dev/null +++ b/examples/prefab/balls/redmule/arch/RedmuleNative.scala @@ -0,0 +1,112 @@ +package examples.balls.redmule + +import chisel3.experimental.hierarchy.{instantiable, public} + +import chisel3._ +import chisel3.util.HasBlackBoxInline + +@instantiable +class RedmuleNative extends BlackBox with HasBlackBoxInline { + + @public + val io = IO(new Bundle { + val clk_i = Input(Clock()) + val rst_ni = Input(Bool()) + val busy_o = Output(Bool()) + val evt_o = Output(Bool()) + val ctrl_req_i = Input(Bool()) + val ctrl_wen_i = Input(Bool()) + val ctrl_addr_i = Input(UInt(32.W)) + val ctrl_data_i = Input(UInt(32.W)) + val ctrl_gnt_o = Output(Bool()) + val ctrl_valid_o = Output(Bool()) + val ctrl_data_o = Output(UInt(32.W)) + val mem_req_o = Output(Bool()) + val mem_wen_o = Output(Bool()) + val mem_addr_o = Output(UInt(32.W)) + val mem_be_o = Output(UInt(32.W)) + val mem_data_o = Output(UInt(256.W)) + val mem_id_o = Output(UInt(8.W)) + val mem_gnt_i = Input(Bool()) + val mem_valid_i = Input(Bool()) + val mem_data_i = Input(UInt(256.W)) + val mem_id_i = Input(UInt(8.W)) + val mem_ready_o = Output(Bool()) + }) + + // Locate RTL from the source checkout, independent of the elaboration directory. + val sourceDir = os.Path(sourcecode.File()) / os.up + val repoRoot = sourceDir / os.up / os.up / os.up / os.up / os.up + val thirdparty = repoRoot / "arch" / "thirdparty" + val common = thirdparty / "common_cells" + val fp = thirdparty / "cvfpu" / "src" + val hci = thirdparty / "hci" / "rtl" + val stream = thirdparty / "hwpe-stream" / "rtl" + val ctrl = thirdparty / "hwpe-ctrl" / "rtl" + val native = thirdparty / "redmule" / "rtl" + for (name <- Seq("assertions.svh", "registers.svh")) { + setInline(s"common_cells/$name", os.read(common / "include" / "common_cells" / name)) + } + setInline("hci_helpers.svh", os.read(hci / "common" / "hci_helpers.svh")) + + val packages = Seq( + common / "src" / "cf_math_pkg.sv", + fp / "fpnew_pkg.sv", + stream / "hwpe_stream_package.sv", + ctrl / "hwpe_ctrl_package.sv", + hci / "common" / "hci_package.sv", + native / "redmule_pkg.sv", + native / "ctrl" / "regif" / "redmule_regif_pkg.sv", + stream / "hwpe_stream_interfaces.sv", + ctrl / "hwpe_ctrl_interfaces.sv", + hci / "common" / "hci_interfaces.sv" + ) + + val primitives = Seq( + common / "src" / "lzc.sv", + common / "src" / "edge_detect.sv", + common / "src" / "sync.sv", + common / "src" / "sync_wedge.sv", + common / "src" / "fifo_v3.sv", + thirdparty / "tech_cells_generic" / "src" / "rtl" / "tc_clk.sv", + thirdparty / "tech_cells_generic" / "src" / "deprecated" / "cluster_clk_cells.sv", + thirdparty / "tech_cells_generic" / "src" / "deprecated" / "pulp_clk_cells.sv" + ) ++ Seq("fpnew_classifier.sv", "fpnew_rounding.sv", "fpnew_cast_multi.sv", "lfsr_sr.sv").map(fp / _) + + val streamSources = + Seq("basic", "fifo", "streamer", "tcdm").flatMap(dir => os.list(stream / dir).filter(_.ext == "sv")) + val hciSources = + os.walk(hci / "core").filter(_.ext == "sv") ++ Seq("hci_ecc_enc.sv", "hci_ecc_dec.sv").map(hci / "ecc" / _) + val controlSources = Seq("hwpe_ctrl_target.sv", "hwpe_ctrl_seq_mult.sv", "hwpe_ctrl_partial_mult.sv").map(ctrl / _) + + val nativeSources = os.walk(native).filter(p => + p.ext == "sv" && !packages.contains(p) && + !Seq("redmule_xif_wrap.sv", "redmule_mm_wrap.sv", "redmule_inst_decoder.sv").contains(p.last) + ) + + // redmule_pkg.sv imports its dependencies at file scope, before the package. + // vvac gathers every file-scope item into one $unit placed at the first of + // them, so the later `import redmule_pkg::*` would precede redmule_pkg. + // Move these imports inside the package; everything else is copied verbatim. + val redmulePkgHead = + "import fpnew_pkg::*;\nimport hci_package::*;\nimport hwpe_stream_package::*;\n\npackage redmule_pkg;\n" + val redmulePkg = os.read(native / "redmule_pkg.sv") + require(redmulePkg.contains(redmulePkgHead), "redmule_pkg.sv header changed") + + val redmulePkgFixed = redmulePkg.replace( + redmulePkgHead, + "package redmule_pkg;\n import fpnew_pkg::*;\n import hci_package::*;\n import hwpe_stream_package::*;\n" + ) + + val wrapper = sourceDir / "RedmuleNative.sv" + // Only the external wrapper is authored here. + setInline( + "RedmuleNative.sv", + "/* verilator lint_off ENUMVALUE */\n" + + (packages ++ primitives ++ streamSources ++ hciSources ++ + controlSources ++ nativeSources).map(p => + if (p == native / "redmule_pkg.sv") redmulePkgFixed else os.read(p) + ).mkString("\n") + + "\n/* verilator lint_on ENUMVALUE */\n" + os.read(wrapper) + ) +} diff --git a/examples/prefab/balls/redmule/arch/RedmuleNative.sv b/examples/prefab/balls/redmule/arch/RedmuleNative.sv new file mode 100644 index 000000000..ab5fa615d --- /dev/null +++ b/examples/prefab/balls/redmule/arch/RedmuleNative.sv @@ -0,0 +1,117 @@ +module RedmuleNative ( + input logic clk_i, + rst_ni, + output logic busy_o, + evt_o, + input logic ctrl_req_i, + ctrl_wen_i, + input logic [31:0] ctrl_addr_i, + ctrl_data_i, + output logic ctrl_gnt_o, + ctrl_valid_o, + output logic [31:0] ctrl_data_o, + output logic mem_req_o, + mem_wen_o, + output logic [31:0] mem_addr_o, + mem_be_o, + output logic [255:0] mem_data_o, + output logic [7:0] mem_id_o, + input logic mem_gnt_i, + mem_valid_i, + input logic [255:0] mem_data_i, + input logic [7:0] mem_id_i, + output logic mem_ready_o +); + localparam hci_package::hci_size_parameter_t HCI_SIZE = '{ + DW: 256, + AW: 32, + BW: 8, + UW: 1, + IW: 8, + EW: 1, + EHW: 1 + }; + hci_core_intf #( + .DW (256), + .AW (32), + .BW (8), + .UW (1), + .IW (8), + .EW (1), + .EHW(1) + ) memory ( + clk_i + ); + hwpe_ctrl_intf_periph #(.ID_WIDTH(4)) control (clk_i); + hwpe_stream_intf_stream #(.DATA_WIDTH(256)) + x_in (clk_i), w_in (clk_i), x_out (clk_i), w_out (clk_i); + assign x_in.valid = 0; + assign x_in.data = 0; + assign x_in.strb = 0; + assign w_in.valid = 0; + assign w_in.data = 0; + assign w_in.strb = 0; + assign x_out.ready = 1; + assign w_out.ready = 1; + assign control.req = ctrl_req_i; + assign control.wen = ctrl_wen_i; + assign control.add = ctrl_addr_i; + assign control.data = ctrl_data_i; + assign control.be = 4'hf; + assign control.id = 0; + assign ctrl_gnt_o = control.gnt; + assign ctrl_valid_o = control.r_valid; + assign ctrl_data_o = control.r_data; + assign mem_req_o = memory.req; + assign mem_wen_o = memory.wen; + assign mem_addr_o = memory.add; + assign mem_be_o = memory.be; + assign mem_data_o = memory.data; + assign mem_id_o = memory.id; + assign mem_ready_o = memory.r_ready; + assign memory.gnt = mem_gnt_i; + assign memory.r_valid = mem_valid_i; + assign memory.r_data = mem_data_i; + assign memory.r_id = mem_id_i; + assign memory.r_user = 0; + assign memory.r_opc = 0; + assign memory.r_ecc = 0; + assign memory.egnt = {1{mem_gnt_i}}; + assign memory.r_evalid = {1{mem_valid_i}}; + redmule_top #( + .DataW(256), + .Height(8), + .Width(8), + .NumPipeRegs(1), + .FpFormat(fpnew_pkg::FP16), + .PipeConfig(fpnew_pkg::DISTRIBUTED), + .CtrlIntfConfig(redmule_pkg::HWPE_TARGET), + .HCI_SIZE_tcdm(HCI_SIZE) + ) native_i ( + .clk_i, + .rst_ni, + .test_mode_i(1'b0), + .busy_o, + .evt_o, + .w_stream_i(w_in), + .x_stream_i(x_in), + .w_stream_o(w_out), + .x_stream_o(x_out), + .x_issue_req_i(1'b0), + .x_issue_resp_o(), + .x_issue_valid_i(1'b0), + .x_issue_ready_o(), + .x_register_i(1'b0), + .x_register_valid_i(1'b0), + .x_register_ready_o(), + .x_commit_i(1'b0), + .x_commit_valid_i(1'b0), + .x_result_o(), + .x_result_valid_o(), + .x_result_ready_i(1'b0), + .sync_o(), + .sync_i(1'b0), + .tcdm(memory), + .target(control) + ); +endmodule diff --git a/examples/prefab/balls/redmule/configs/default.toml b/examples/prefab/balls/redmule/configs/default.toml new file mode 100644 index 000000000..5412bfcba --- /dev/null +++ b/examples/prefab/balls/redmule/configs/default.toml @@ -0,0 +1 @@ +lanes = 8 diff --git a/examples/prefab/balls/redmule/configs/redmule_params.proto b/examples/prefab/balls/redmule/configs/redmule_params.proto new file mode 100644 index 000000000..469d725d5 --- /dev/null +++ b/examples/prefab/balls/redmule/configs/redmule_params.proto @@ -0,0 +1,8 @@ +syntax = "proto3"; +package buckyball.balls.redmule; +option java_package = "buckyball.balls.redmule"; +option java_multiple_files = true; + +message Params { + optional int32 lanes = 1; +} diff --git a/examples/prefab/balls/redmule/emu/src/fp16.rs b/examples/prefab/balls/redmule/emu/src/fp16.rs new file mode 100644 index 000000000..f476b5e3d --- /dev/null +++ b/examples/prefab/balls/redmule/emu/src/fp16.rs @@ -0,0 +1,71 @@ +// Finite FP16 values are exact signed integers in units of 2^-24. +fn finite(bits: u16) -> i128 { + let exponent = (bits >> 10) & 31; + let mantissa = (bits & 1023) as i128; + let value = if exponent == 0 { + mantissa + } else { + (1024 + mantissa) << (exponent - 1) + }; + if bits & 0x8000 != 0 { + -value + } else { + value + } +} + +pub(super) fn fma(a: u16, b: u16, c: u16) -> u16 { + let aa = a & 0x7fff; + let bb = b & 0x7fff; + let cc = c & 0x7fff; + let product_sign = (a ^ b) & 0x8000; + if aa > 0x7c00 + || bb > 0x7c00 + || cc > 0x7c00 + || (aa == 0x7c00 && bb == 0) + || (bb == 0x7c00 && aa == 0) + { + return 0x7e00; + } + if aa == 0x7c00 || bb == 0x7c00 { + return if cc == 0x7c00 && c & 0x8000 != product_sign { + 0x7e00 + } else { + product_sign | 0x7c00 + }; + } + if cc == 0x7c00 { + return c; + } + // The largest product uses fewer than 81 bits; i128 retains every bit. + let sum = finite(a) * finite(b) + (finite(c) << 24); + if sum == 0 { + return if (aa == 0 || bb == 0) && cc == 0 { + product_sign & c & 0x8000 + } else { + 0 + }; + } + let sign = if sum < 0 { 0x8000 } else { 0 }; + let magnitude = sum.unsigned_abs(); + let highest = 127 - magnitude.leading_zeros(); + let shift = if highest < 34 { 24 } else { highest - 10 }; + let mut significand = magnitude >> shift; + let remainder = magnitude & ((1u128 << shift) - 1); + let halfway = 1u128 << (shift - 1); + if remainder > halfway || (remainder == halfway && significand & 1 != 0) { + significand += 1; + } + if highest < 34 { + return sign | significand as u16; + } + let mut exponent = highest as i32 - 48 + 15; + if significand == 2048 { + significand = 1024; + exponent += 1; + } + if exponent >= 31 { + return sign | 0x7c00; + } + sign | (exponent as u16) << 10 | (significand as u16 & 1023) +} diff --git a/examples/prefab/balls/redmule/emu/src/lib.rs b/examples/prefab/balls/redmule/emu/src/lib.rs new file mode 100644 index 000000000..d59b6c020 --- /dev/null +++ b/examples/prefab/balls/redmule/emu/src/lib.rs @@ -0,0 +1,36 @@ +mod fp16; +mod matmul; + +use bebop_bemu::ball::ExecContext; + +const CLASS: &str = "examples.balls.redmule.RedmuleBall"; + +pub fn execute_known( + class: &str, + funct: u32, + xs1: u64, + xs2: u64, + ctx: &mut ExecContext, +) -> Option { + if class != CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("REDMULE_F16") => Some(matmul::execute(xs1, xs2, ctx)), + _ => None, + } +} + +pub fn cycles_after_issue(class: &str, funct: u32, _xs1: u64, _xs2: u64) -> Option { + if class != CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("REDMULE_F16") => Some(1), + _ => None, + } +} + +#[bebop_bemu::ball::linkme::distributed_slice(bebop_bemu::BALLS)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static BALL: bebop_bemu::Ball = bebop_bemu::Ball { execute: execute_known, cycles: cycles_after_issue }; diff --git a/examples/prefab/balls/redmule/emu/src/matmul.rs b/examples/prefab/balls/redmule/emu/src/matmul.rs new file mode 100644 index 000000000..0bdc174b3 --- /dev/null +++ b/examples/prefab/balls/redmule/emu/src/matmul.rs @@ -0,0 +1,70 @@ +use bebop_bemu::thread::instructions::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old}; +use bebop_bemu::ball::ExecContext; +pub(super) fn execute(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + let a = rs1_b0(xs1); + let b = rs1_b1(xs1); + let c = rs1_b2(xs1); + let rows = (xs2 & 4095) as usize; + let k = rs1_iter(xs1) as usize; + let a_base = ((xs2 >> 12) & 1023) as usize; + let b_base = ((xs2 >> 22) & 1023) as usize; + let c_base = ((xs2 >> 32) & 1023) as usize; + let first = xs2 & (1 << 42) != 0; + assert_eq!(xs2 >> 43, 0, "redmule: reserved bits must be zero"); + // A continuation reads the old C from wr_read_bank, which rs1[29] provides. + assert_eq!( + rs1_read_old(xs1), + !first, + "redmule: rs1[29] (read old destination) must be set exactly for a continuation (rs2[42] clear)" + ); + assert!( + rows > 0 && k > 0 && k % 8 == 0, + "redmule: M > 0, K must be a positive multiple of eight" + ); + assert!(a != b && a != c && b != c, "redmule: banks must differ"); + assert_eq!( + bebop_bemu::configs::bank_width(), + 128, + "redmule: bank rows must be 128 bits" + ); + assert_eq!(bebop_bemu::configs::ball_domain::parameters::("redmule", ctx.ball_id.expect("Ball instance")).lanes.expect("lanes") as usize, 8); + for bank in [a, b, c] { + assert!( + ctx.config(bank).allocated && ctx.config(bank).cols == 1, + "redmule: banks must be allocated with one group" + ); + } + let pa = pbank(ctx, a); + let pb = pbank(ctx, b); + let pc = pbank(ctx, c); + assert!( + a_base + rows * (k / 8) <= ctx.banks[pa].len() / 16 + && b_base + k <= ctx.banks[pb].len() / 16 + && c_base + rows <= ctx.banks[pc].len() / 16, + "redmule: bank window exceeds capacity" + ); + let mut sums = vec![0; rows * 8]; + if !first { + for (index, value) in sums.iter_mut().enumerate() { + let offset = c_base * 16 + index * 2; + *value = u16::from_le_bytes(ctx.banks[pc][offset..offset + 2].try_into().unwrap()); + } + } + for row in 0..rows { + for inner in 0..k { + let offset = a_base * 16 + (row * k + inner) * 2; + let lhs = u16::from_le_bytes(ctx.banks[pa][offset..offset + 2].try_into().unwrap()); + for col in 0..8 { + let offset = (b_base + inner) * 16 + col * 2; + let rhs = u16::from_le_bytes(ctx.banks[pb][offset..offset + 2].try_into().unwrap()); + let index = row * 8 + col; + sums[index] = super::fp16::fma(lhs, rhs, sums[index]); + } + } + } + for (index, value) in sums.iter().enumerate() { + let offset = c_base * 16 + index * 2; + ctx.banks[pc][offset..offset + 2].copy_from_slice(&value.to_le_bytes()); + } + 0 +} diff --git a/examples/prefab/balls/redmule/stack/src/redmuleOps.cpp b/examples/prefab/balls/redmule/stack/src/redmuleOps.cpp new file mode 100644 index 000000000..48887b348 --- /dev/null +++ b/examples/prefab/balls/redmule/stack/src/redmuleOps.cpp @@ -0,0 +1,76 @@ +//===- redmuleOps.cpp - RedmuleBall operations ----------------------------===// +// +// The dialect of RedmuleBall, its virtual operations building their physical +// ones and the instructions of the physical operations. +// +//===----------------------------------------------------------------------===// + +#include "redmuleOps.h" + +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::redmule_ball; +using mlir::buddy::bankFields; +using mlir::buddy::constantI64; +using mlir::buddy::emitInstruction; +using mlir::buddy::field; + +#include "redmuleDialect.cpp.inc" + +#define GET_OP_CLASSES +#include "redmuleOps.cpp.inc" + +void RedmuleBallDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "redmuleOps.cpp.inc" + >(); +} + +namespace { +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallDialect([](DialectRegistry ®istry) { + registry.insert(); + }), + true); +} // namespace + +SmallVector BankMatmulF16Op::getSourceBanks() { + return {getA(), getB()}; +} +Value BankMatmulF16Op::getDestinationBank() { return getC(); } +bool BankMatmulF16Op::readsDestination() { return getReadDestination(); } +Operation *BankMatmulF16Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return MatmulF16Op::create(builder, getLoc(), sources[0], sources[1], + destination, getRows(), getK(), + getReadDestination()); +} + +SmallVector MatmulF16Op::getSourceBanks() { + return {int64_t(getA()), int64_t(getB())}; +} +int64_t MatmulF16Op::getDestinationBank() { return getC(); } +LogicalResult MatmulF16Op::lowerToInstructions(OpBuilder &builder) { + Location loc = getLoc(); + int64_t rows = getRows(), k = getK(), a = getA(), b = getB(), c = getC(); + bool readDestination = getReadDestination(); + const auto &target = buckyball_target::getBuckyballTarget(); + if (a == b || a == c || b == c) + return emitOpError("FP16 matmul requires distinct A, B and C banks"); + if (target.bankWidthBits != 128 || rows <= 0 || rows > 4095 || k <= 0 || + k % 8 || k >= (1LL << 33) || rows * (k / 8) > target.bankDepth || + k > target.bankDepth || rows > target.bankDepth) + return emitOpError("FP16 matmul shape exceeds the bank or ISA geometry"); + // rs2: rows in [11:0], the A, B and C row bases (all 0) in [41:12], and in + // bit 42 whether the FMAs start from positive zero. + uint64_t rs2 = field(rows, 0, 11) | field(!readDestination, 42, 42); + emitInstruction( + builder, loc, buckyball_target::getBuckyballFunct7("REDMULE_F16"), + constantI64(builder, loc, bankFields({a, b}, c, readDestination, k)), + constantI64(builder, loc, rs2)); + return success(); +} diff --git a/examples/prefab/balls/redmule/stack/src/redmuleOps.h b/examples/prefab/balls/redmule/stack/src/redmuleOps.h new file mode 100644 index 000000000..45ba8ebe8 --- /dev/null +++ b/examples/prefab/balls/redmule/stack/src/redmuleOps.h @@ -0,0 +1,20 @@ +//===- redmuleOps.h - RedmuleBall operations ------------------------------===// + +#ifndef REDMULE_OPS_H +#define REDMULE_OPS_H + +#include "Interfaces/BankInterfaces.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "redmuleDialect.h.inc" + +#define GET_OP_CLASSES +#include "redmuleOps.h.inc" + +#endif // REDMULE_OPS_H diff --git a/examples/prefab/balls/redmule/stack/src/redmuleOps.td b/examples/prefab/balls/redmule/stack/src/redmuleOps.td new file mode 100644 index 000000000..c5d69c5ad --- /dev/null +++ b/examples/prefab/balls/redmule/stack/src/redmuleOps.td @@ -0,0 +1,49 @@ +//===- redmuleOps.td - RedmuleBall operations --------------*- tablegen -*-===// +// +// matmul_f16: C = A @ B in FP16 with ordered FMAs, or C = C + A @ B with +// read_destination. `k` is a positive multiple of eight. +// A: `rows` rows of `k` f16, k / 8 bank rows each. +// B: `k` bank rows of eight f16. +// C: `rows` bank rows of eight f16. +// +// The Ball places every operand at bank row 0. With read_destination rs1 marks +// the old C, which the Ball reads from the bank the destination held before +// this write, and the FMAs start from it instead of positive zero. +// +//===----------------------------------------------------------------------===// + +#ifndef REDMULE_OPS +#define REDMULE_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Interfaces/BankInterfaces.td" + +def RedmuleBall_Dialect : Dialect { + let name = "redmule_ball"; + let cppNamespace = "::buddy::redmule_ball"; + let summary = "The operations of RedmuleBall"; +} + +class RedmuleBall_Op traits = []> + : Op])>; + +def BankMatmulF16Op : RedmuleBall_Op<"bank_matmul_f16", + [DeclareOpInterfaceMethods]> { + let summary = "matmul_f16 on virtual banks"; + let arguments = (ins I64:$a, I64:$b, I64:$c, I64Attr:$rows, I64Attr:$k, + UnitAttr:$read_destination); + let results = (outs I64:$cOut); + let assemblyFormat = "$a `,` $b `->` $c prop-dict attr-dict"; +} + +def MatmulF16Op : RedmuleBall_Op<"matmul_f16", + [DeclareOpInterfaceMethods]> { + let summary = "matmul_f16 on physical banks"; + let arguments = (ins I64Attr:$a, I64Attr:$b, I64Attr:$c, I64Attr:$rows, + I64Attr:$k, UnitAttr:$read_destination); + let assemblyFormat = "prop-dict attr-dict"; +} + +#endif // REDMULE_OPS diff --git a/examples/prefab/balls/redmule/verify/casegen/Cargo.lock b/examples/prefab/balls/redmule/verify/casegen/Cargo.lock new file mode 100644 index 000000000..598b7251f --- /dev/null +++ b/examples/prefab/balls/redmule/verify/casegen/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "redmule-casegen" +version = "0.1.0" diff --git a/examples/prefab/balls/redmule/verify/casegen/Cargo.toml b/examples/prefab/balls/redmule/verify/casegen/Cargo.toml new file mode 100644 index 000000000..363e11dec --- /dev/null +++ b/examples/prefab/balls/redmule/verify/casegen/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "redmule-casegen" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["cdylib"] + +[lints.rust] +unused_must_use = "deny" diff --git a/examples/prefab/balls/redmule/verify/casegen/src/lib.rs b/examples/prefab/balls/redmule/verify/casegen/src/lib.rs new file mode 100644 index 000000000..b60a07e35 --- /dev/null +++ b/examples/prefab/balls/redmule/verify/casegen/src/lib.rs @@ -0,0 +1,118 @@ +#[path = "../../../emu/src/fp16.rs"] +mod fp16; +use std::cell::RefCell; + +struct Case { + rows: usize, + k: usize, + base: usize, + blocks: usize, + a: Vec, + b: Vec, + c: Vec, +} +thread_local! { + static CASE: RefCell> = const { RefCell::new(None) }; +} + +#[no_mangle] +pub extern "C" fn redmule_case_load(index: u32, lanes: u32) { + assert_eq!(lanes, 8); + let rows = if index >= 64 { + [16, 16, 32][index as usize - 64] + } else { + index as usize % 3 + 1 + }; + let k = if index >= 64 { + [16, 32, 32][index as usize - 64] + } else if index & 1 == 0 { + 8 + } else { + 16 + }; + let base = if index & 1 == 0 { 0 } else { 2 }; + let blocks = if index == 2 { 3 } else { 1 }; + let mut a = vec![0x7e00; base * 8 + rows * k]; + let mut b = vec![0x7e00; (base + k) * 8]; + let mut rng = 0xcafe_babeu32 ^ index; + for value in a[base * 8..].iter_mut().chain(b[base * 8..].iter_mut()) { + rng ^= rng << 13; + rng ^= rng >> 17; + rng ^= rng << 5; + *value = if index < 3 || index >= 64 { + [0x3c00, 0xbc00, 0x3800, 0xb800][rng as usize & 3] + } else { + rng as u16 + }; + } + if index == 3 { + a[base * 8..].fill(0); + b[base * 8..].fill(0); + for row in 0..rows { + a[base * 8 + row * k] = 0x3c00; + } + b[base * 8..base * 8 + 8] + .copy_from_slice(&[0, 1, 0x3c00, 0x7c00, 0x7d00, 0xbc00, 0x7bff, 0x8001]); + } + let mut c = vec![0; rows * 8]; + let mut outputs = Vec::new(); + for _ in 0..blocks { + for row in 0..rows { + for inner in 0..k { + for col in 0..8 { + let i = row * 8 + col; + c[i] = fp16::fma( + a[base * 8 + row * k + inner], + b[(base + inner) * 8 + col], + c[i], + ); + } + } + } + outputs.extend_from_slice(&c); + } + CASE.with(|slot| { + *slot.borrow_mut() = Some(Case { + rows, + k, + base, + blocks, + a, + b, + c: outputs, + }) + }); +} + +// Return rows, K, row base, and command count in successive 16-bit fields. +#[no_mangle] +pub extern "C" fn redmule_case_shape() -> u64 { + CASE.with(|slot| { + let slot = slot.borrow(); + let case = slot.as_ref().expect("case not loaded"); + case.rows as u64 + | (case.k as u64) << 16 + | (case.base as u64) << 32 + | (case.blocks as u64) << 48 + }) +} + +#[no_mangle] +pub extern "C" fn redmule_case_word(bank: u32, row: u32, half: u32) -> u64 { + CASE.with(|slot| { + let slot = slot.borrow(); + let case = slot.as_ref().expect("case not loaded"); + let data = match bank { + 0 => &case.a, + 1 => &case.b, + 2 => &case.c, + _ => panic!("invalid case bank"), + }; + let offset = row as usize * 8 + half as usize * 4; + let mut word = 0; + for lane in 0..4 { + word |= (data[offset + lane] as u64) << (lane * 16); + } + word + }) +} diff --git a/examples/prefab/balls/redmule/verify/filelists/redmule_ball.f b/examples/prefab/balls/redmule/verify/filelists/redmule_ball.f new file mode 100644 index 000000000..70b8c3fba --- /dev/null +++ b/examples/prefab/balls/redmule/verify/filelists/redmule_ball.f @@ -0,0 +1,15 @@ ++incdir+@UVM@/src/core ++incdir+@UVM@/src/ball ++incdir+@UVM@/src/ball/agents/cmd ++incdir+@UVM@/src/ball/agents/mem ++incdir+@UVM@/src/ball/agents/resp ++incdir+@UVM@/src/ball/cov ++incdir+@UVM@/src/ball/env ++incdir+@RTL@ +@UVM@/src/core/package.sv +@UVM@/src/ball/bb_blink_if.sv +@UVM@/src/ball/bb_uvm_pkg.sv +-F @RTL@/filelist.f +src/reference/redmule_reference.sv +src/pkg/redmule_pkg.sv +src/tb_top.sv diff --git a/examples/prefab/balls/redmule/verify/parameters.toml b/examples/prefab/balls/redmule/verify/parameters.toml new file mode 100644 index 000000000..6cfae947e --- /dev/null +++ b/examples/prefab/balls/redmule/verify/parameters.toml @@ -0,0 +1,2 @@ +[defines] +REDMULE_LANES = "lanes" diff --git a/examples/prefab/balls/redmule/verify/src/pkg/redmule_pkg.sv b/examples/prefab/balls/redmule/verify/src/pkg/redmule_pkg.sv new file mode 100644 index 000000000..a2ccb1025 --- /dev/null +++ b/examples/prefab/balls/redmule/verify/src/pkg/redmule_pkg.sv @@ -0,0 +1,363 @@ +package redmule_uvm_pkg; + import uvm_pkg::*; + import bb_uvm_pkg::*; + `include "uvm_macros.svh" + import "DPI-C" function void redmule_case_load( + input int unsigned index, + lanes + ); + import "DPI-C" function longint unsigned redmule_case_shape(); + import "DPI-C" function longint unsigned redmule_case_word( + input int unsigned bank, + row, + half + ); + + class redmule_mem_model extends bb_blink_mem_model #(`BB_IN_BW, `BB_OUT_BW); + `uvm_component_utils(redmule_mem_model) + int read_stalls[2], write_stalls, response_stalls; + bit inject_stalls = 1; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + // Both performance ports model one-cycle memory; only functional cases inject stalls. + task run_phase(uvm_phase phase); + forever begin + @(posedge vif.clock); + if (vif.reset) begin + for (int port = 0; port < 2; port++) begin + vif.bank_read_req_ready[port] <= 1; + vif.bank_read_resp_valid[port] <= 0; + end + vif.bank_write_req_ready[0] <= 1; + vif.bank_write_resp_valid[0] <= 0; + vif.bank_write_resp_ok[0] <= 1; + vif.cmd_resp_ready <= 1; + end else begin + for (int port = 0; port < 2; port++) begin + if (vif.bank_read_resp_valid[port] && vif.bank_read_resp_ready[port]) + vif.bank_read_resp_valid[port] <= 0; + if (vif.bank_read_req_valid[port] && vif.bank_read_req_ready[port]) begin + if (!mem.exists( + vif.bank_read_bank_id[port] + ) || !mem[vif.bank_read_bank_id[port]].exists( + vif.bank_read_group_id[port] + ) || !mem[vif.bank_read_bank_id[port]][vif.bank_read_group_id[port]].exists( + vif.bank_read_req_addr[port] + )) + `uvm_fatal("MEM", "bank read was not preloaded") + vif.bank_read_resp_data[port] <= mem[vif.bank_read_bank_id[port]][vif.bank_read_group_id[port]][vif.bank_read_req_addr[port]]; + vif.bank_read_resp_valid[port] <= 1; + end + vif.bank_read_req_ready[port] <= (!vif.bank_read_resp_valid[port] || vif.bank_read_resp_ready[port]) && + (!inject_stalls || $urandom_range( + 0, 3 + ) != 0); + if (vif.bank_read_req_valid[port] && !vif.bank_read_req_ready[port]) + read_stalls[port]++; + end + if (vif.bank_write_resp_valid[0] && vif.bank_write_resp_ready[0]) + vif.bank_write_resp_valid[0] <= 0; + if (vif.bank_write_req_valid[0] && vif.bank_write_req_ready[0]) + vif.bank_write_resp_valid[0] <= 1; + vif.bank_write_req_ready[0] <= (!vif.bank_write_resp_valid[0] || vif.bank_write_resp_ready[0]) && + (!inject_stalls || $urandom_range( + 0, 3 + ) != 0); + vif.cmd_resp_ready <= (!inject_stalls || $urandom_range(0, 3) != 0); + if (vif.bank_write_req_valid[0] && !vif.bank_write_req_ready[0]) write_stalls++; + if (vif.cmd_resp_valid && !vif.cmd_resp_ready) response_stalls++; + end + end + endtask + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (!read_stalls[0] || !read_stalls[1] || !write_stalls || !response_stalls) + `uvm_fatal("STALL", "missing read/write/completion backpressure coverage") + endfunction + endclass + + class redmule_seq extends uvm_sequence #(bb_blink_cmd_item); + `uvm_object_utils(redmule_seq) + int rows, k, base, block_index, blocks; + int groups[4]; + function new(string name = "redmule_seq"); + super.new(name); + endfunction + task body(); + bb_blink_cmd_item item = bb_blink_cmd_item::type_id::create("item"); + start_item(item); + // bb_redmule(a=0, b=1, c=2, m, k, base, base, base, first): banks and K from rs1. + // The frontend renames C on every write: block j writes bank 2 + j % 2 and a + // continuation reads the previous C from the other one, its wr_read_bank. + item.rob_id = block_index; + item.funct = `REDMULE_F16_FUNCT7; + item.rs2 = longint'(rows) | (longint'(base) << 12) | + (longint'(base) << 22) | (longint'(base) << 32) | + (longint'(block_index == 0) << 42); + item.iter = k; + item.op1_bank = 0; + item.op2_bank = 1; + item.wr_bank = 2 + block_index % 2; + item.op1_group = groups[0]; + item.op2_group = groups[1]; + item.wr_group = groups[2]; + item.op1_col = 1; + item.op2_col = 1; + item.wr_col = 1; + item.wr_read_bank = block_index == 0 ? 0 : 3 - block_index % 2; + item.wr_arch_bank = 2; + finish_item(item); + endtask + endclass + + class redmule_scoreboard extends uvm_scoreboard; + `uvm_component_utils(redmule_scoreboard) + uvm_analysis_imp_cmd #(bb_blink_cmd_item, redmule_scoreboard) cmd_imp; + uvm_analysis_imp_read #(bb_blink_read_item, redmule_scoreboard) read_imp; + uvm_analysis_imp_write #(bb_blink_write_item, redmule_scoreboard) write_imp; + uvm_analysis_imp_resp #(bb_blink_resp_item, redmule_scoreboard) resp_imp; + bb_blink_mem_model #(`BB_IN_BW, `BB_OUT_BW) mem_model; + int rows, k, base, blocks, block_index; + int groups[4]; + int reads[2], writes, commands, responses; + bit first; + bit a_seen[int], b_seen[int], c_seen[int]; + int m_bucket, k_bucket, base_bucket, number_class; + covergroup contract_cg; + coverpoint first {bins start = {1}; bins continuation = {0};} + coverpoint m_bucket {bins shapes[] = {[1 : 3]}; bins tile_rows[] = {16, 32};} + coverpoint k_bucket {bins short_k = {8}; bins long_k = {16}; bins large_k = {32};} + coverpoint base_bucket {bins zero = {0}; bins offset = {2};} + endgroup + covergroup number_cg; + coverpoint number_class {bins classes[] = {[0 : 4]};} + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + cmd_imp = new("cmd_imp", this); + read_imp = new("read_imp", this); + write_imp = new("write_imp", this); + resp_imp = new("resp_imp", this); + contract_cg = new(); + number_cg = new(); + endfunction + function void write_cmd(bb_blink_cmd_item item); + if (item.funct != `REDMULE_F16_FUNCT7 || item.iter != k || + item.rs2[11:0] != rows || item.rob_id != block_index || + item.op1_bank != 0 || item.op2_bank != 1 || item.wr_bank != 2 + block_index % 2 || + item.op1_group != groups[0] || item.op2_group != groups[1] || + item.wr_group != groups[2] || + item.wr_read_bank != (block_index == 0 ? 0 : 3 - block_index % 2) || + item.wr_arch_bank != 2) + `uvm_fatal("CMD", "unexpected RedMulE command") + first = item.rs2[42]; + m_bucket = rows; + k_bucket = k; + base_bucket = base; + contract_cg.sample(); + commands++; + endfunction + function void write_read(bb_blink_read_item item); + int limit; + if (item.rob_id != block_index) `uvm_fatal("READ", "unexpected read metadata") + if (item.group_id != groups[item.bank_id]) `uvm_fatal("READ", "unexpected read group") + if (item.port == 1 && item.bank_id == 1) begin + limit = base + k; + b_seen[item.addr] = 1; + end else if (item.port == 0 && item.bank_id == 0) begin + limit = base + rows * k / 8; + a_seen[item.addr] = 1; + end else if (item.port == 0 && item.bank_id == 3 - block_index % 2 && !first) begin + limit = base + rows; + c_seen[item.addr] = 1; + end else `uvm_fatal("READ", "unexpected read bank") + if (item.addr < base || item.addr >= limit) `uvm_fatal("READ", "read outside bank window") + reads[item.port]++; + endfunction + function void write_write(bb_blink_write_item item); + bit [127:0] expected; + bit [ 15:0] value; + if (writes >= rows) `uvm_fatal("WRITE", "unexpected early/extra write") + expected = { + redmule_case_word(2, block_index * rows + writes, 1), + redmule_case_word(2, block_index * rows + writes, 0) + }; + if (item.port != 0 || item.bank_id != 2 + block_index % 2 || item.addr != base + writes || + item.rob_id != block_index || item.group_id != groups[2] || item.mask != 16'hffff || + item.data !== expected) + `uvm_fatal("WRITE", $sformatf( + "row %0d got %032h expected %032h", item.addr, item.data, expected)) + for (int lane = 0; lane < 8; lane++) begin + value = item.data[lane*16+:16] & 16'h7fff; + if (value == 0) number_class = 0; + else if (value < 16'h0400) number_class = 1; + else if (value < 16'h7c00) number_class = 2; + else if (value == 16'h7c00) number_class = 3; + else number_class = 4; + number_cg.sample(); + end + mem_model.load_word_g(item.bank_id, groups[2], item.addr, item.data); + writes++; + endfunction + function void write_resp(bb_blink_resp_item item); + if (item.rob_id != block_index || writes != rows) + `uvm_fatal("RESP", "completion preceded required reads/writes") + for (int row = base; row < base + rows * k / 8; row++) + if (!a_seen.exists(row)) `uvm_fatal("READ", "missing A row") + for (int row = base; row < base + k; row++) + if (!b_seen.exists(row)) `uvm_fatal("READ", "missing B row") + if (!first) + for (int row = base; row < base + rows; row++) + if (!c_seen.exists(row)) `uvm_fatal("READ", "missing C accumulator row") + responses++; + endfunction + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (commands != responses || commands == 0) `uvm_fatal("SCB", "missing command completion") + if (contract_cg.get_coverage() != 100 || number_cg.get_coverage() != 100) + `uvm_fatal( + "COV", $sformatf( + "contract %0.2f%% numbers %0.2f%%", contract_cg.get_coverage(), number_cg.get_coverage() + )) + endfunction + endclass + + class redmule_env extends bb_blink_env #(`BB_IN_BW, `BB_OUT_BW); + `uvm_component_utils(redmule_env) + redmule_scoreboard scb; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + bb_blink_mem_model#(`BB_IN_BW, `BB_OUT_BW)::type_id::set_type_override( + redmule_mem_model::get_type()); + super.build_phase(phase); + scb = redmule_scoreboard::type_id::create("scb", this); + endfunction + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + scb.mem_model = mem_model; + cmd_agent.cmd_ap.connect(scb.cmd_imp); + read_mon.read_ap.connect(scb.read_imp); + write_mon.write_ap.connect(scb.write_imp); + resp_mon.resp_ap.connect(scb.resp_imp); + endfunction + endclass + + class redmule_ball_test extends uvm_test; + `uvm_component_utils(redmule_ball_test) + typedef virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) vif_t; + vif_t vif; + redmule_env env; + virtual redmule_reference_if reference; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(vif_t)::get(this, "", "vif", vif)) + `uvm_fatal("NOVIF", "bb_blink_if not found") + if (!uvm_config_db#(virtual redmule_reference_if)::get(this, "", "reference", reference)) + `uvm_fatal("NOVIF", "native reference interface not found") + env = redmule_env::type_id::create("env", this); + endfunction + task run_phase(uvm_phase phase); + int rows, k, base, blocks, before_response, cycles; + int groups[4]; + longint unsigned shape; + longint native_cycles, ball_started, ball_cycles, total_native = 0, total_ball = 0; + redmule_seq seq; + redmule_mem_model memory_agent; + if (!$cast(memory_agent, env.mem_model)) `uvm_fatal("MEM", "wrong memory model type") + phase.raise_objection(this); + for (int index = 0; index < 67; index++) begin + vif.reset = 1; + repeat (5) @(posedge vif.clock); + vif.reset = 0; + repeat (2) @(posedge vif.clock); + memory_agent.inject_stalls = index < 64; + redmule_case_load(index, `REDMULE_LANES); + shape = redmule_case_shape(); + rows = shape[15:0]; + k = shape[31:16]; + base = shape[47:32]; + blocks = shape[63:48]; + env.scb.rows = rows; + env.scb.k = k; + env.scb.base = base; + env.scb.blocks = blocks; + // Odd cases place A, B and C at nonzero group bases of their banks; both + // banks of C use the destination group. + groups = index % 2 ? '{1, 2, 3, 3} : '{0, 0, 0, 0}; + env.scb.groups = groups; + env.mem_model.clear_mem(); + for (int row = 0; row < base + rows * k / 8; row++) begin + env.mem_model.load_word_g(0, groups[0], row, { + redmule_case_word(0, row, 1), redmule_case_word(0, row, 0)}); + reference.load_word(0, row, {redmule_case_word(0, row, 1), redmule_case_word(0, row, 0)}); + end + for (int row = 0; row < base + k; row++) begin + env.mem_model.load_word_g(1, groups[1], row, { + redmule_case_word(1, row, 1), redmule_case_word(1, row, 0)}); + reference.load_word(1, row, {redmule_case_word(1, row, 1), redmule_case_word(1, row, 0)}); + end + for (int row = 0; row < base + rows; row++) begin + env.mem_model.load_word_g(2, groups[2], row, 0); + reference.load_word(2, row, 0); + end + env.mem_model.arm(); + for (int block_index = 0; block_index < blocks; block_index++) begin + env.scb.block_index = block_index; + env.scb.reads = '{0, 0}; + env.scb.writes = 0; + env.scb.a_seen.delete(); + env.scb.b_seen.delete(); + env.scb.c_seen.delete(); + before_response = env.scb.responses; + seq = redmule_seq::type_id::create("seq"); + seq.groups = groups; + seq.rows = rows; + seq.k = k; + seq.base = base; + seq.block_index = block_index; + seq.blocks = blocks; + reference.execute(rows, k, base, block_index == 0, native_cycles); + for (int row = 0; row < rows; row++) + if (reference.read_word( + base + row + ) !== {redmule_case_word( + 2, block_index * rows + row, 1 + ), redmule_case_word( + 2, block_index * rows + row, 0 + )}) + `uvm_fatal("REFERENCE", "native engine disagrees with golden") + ball_started = reference.get_ticks(); + seq.start(env.cmd_agent.seqr); + cycles = 0; + while (env.scb.responses == before_response) begin + @(posedge vif.clock); + if (++cycles > 200000) `uvm_fatal("TIMEOUT", "RedMulE command timed out") + end + ball_cycles = reference.get_ticks() - ball_started; + total_native += native_cycles; + total_ball += ball_cycles; + `uvm_info("PERF", $sformatf( + "M=%0d K=%0d N=8 first=%0d native=%0d ball=%0d", + rows, + k, + block_index == 0, + native_cycles, + ball_cycles + ), UVM_LOW) + repeat (2) @(posedge vif.clock); + end + `uvm_info("REDMULE", $sformatf("case %0d passed", index), UVM_LOW) + end + `uvm_info("PERF_TOTAL", $sformatf("native=%0d ball=%0d", total_native, total_ball), UVM_LOW) + phase.drop_objection(this); + endtask + endclass +endpackage diff --git a/examples/prefab/balls/redmule/verify/src/reference/redmule_reference.sv b/examples/prefab/balls/redmule/verify/src/reference/redmule_reference.sv new file mode 100644 index 000000000..26a179f65 --- /dev/null +++ b/examples/prefab/balls/redmule/verify/src/reference/redmule_reference.sv @@ -0,0 +1,130 @@ +module redmule_reference ( + input logic clock, + reset +); + logic control_req = 0, control_wen; + logic [31:0] control_addr, control_data, control_result; + logic control_grant, control_valid, busy, event_done; + logic request, load, ready; + logic [31:0] address, enables; + logic [255:0] write_data, response_data; + logic [7:0] request_id, response_id; + logic response_valid = 0; + bit [127:0] memory[3][1 << `BB_BANK_ADDR_W]; + int rows, reduction, row_base; + longint ticks = 0; + + RedmuleNative original ( + .clk_i(clock), + .rst_ni(!reset), + .busy_o(busy), + .evt_o(event_done), + .ctrl_req_i(control_req), + .ctrl_wen_i(control_wen), + .ctrl_addr_i(control_addr), + .ctrl_data_i(control_data), + .ctrl_gnt_o(control_grant), + .ctrl_valid_o(control_valid), + .ctrl_data_o(control_result), + .mem_req_o(request), + .mem_wen_o(load), + .mem_addr_o(address), + .mem_be_o(enables), + .mem_data_o(write_data), + .mem_id_o(request_id), + .mem_gnt_i(1'b1), + .mem_valid_i(response_valid), + .mem_data_i(response_data), + .mem_id_i(response_id), + .mem_ready_o(ready) + ); + + function void load_word(int bank, int row, bit [127:0] data); + memory[bank][row] = data; + endfunction + + always @(posedge clock) begin + int bank, word_index, limit; + ticks <= ticks + 1; + response_valid <= 0; + if (!reset && request) begin + bank = int'(address[25:24]) - 1; + word_index = address[23:4]; + case (bank) + 0: limit = row_base + rows * reduction / 8; + 1: limit = row_base + reduction; + 2: limit = row_base + rows; + default: $fatal(1, "Native reference address has no bank region"); + endcase + if (load) begin + response_data <= { + word_index + 1 < limit ? memory[bank][word_index+1] : 128'b0, + word_index < limit ? memory[bank][word_index] : 128'b0 + }; + response_id <= request_id; + response_valid <= 1; + end else begin + if (bank != 2) $fatal(1, "Native reference store is outside C region"); + for (int byte_index = 0; byte_index < 32; byte_index++) + if (enables[byte_index] && word_index + byte_index / 16 < limit) + memory[2][word_index+byte_index/16][(byte_index%16)*8+:8] <= write_data[byte_index*8+:8]; + end + end + end + + task execute(int m, int k, int base, bit first, output longint elapsed); + longint started; + int register_addr[9] = '{4, 44, 48, 52, 32, 36, 40, 60, 0}; + int register_data[9]; + rows = m; + reduction = k; + row_base = base; + register_data = '{ + 0, + 32'h01000000 + (base << 4), + 32'h02000000 + (base << 4), + 32'h03000000 + (base << 4), + (8 << 16) | m, + 32'h02800000 | (!first << 20) | k, + 0, + 0, + 0 + }; + started = ticks; + for (int step = 0; step < 9; step++) begin + @(negedge clock); + control_addr = register_addr[step]; + control_data = register_data[step]; + control_wen = (step == 0); + control_req = 1; + do @(posedge clock); while (!control_grant); + @(negedge clock); + control_req = 0; + wait (control_valid); + if (step == 0 && control_result == 32'hffffffff) + $fatal(1, "Native reference has no job context"); + end + wait (event_done); + wait (!busy); + elapsed = ticks - started; + @(negedge clock); + endtask +endmodule + +interface redmule_reference_if ( + input logic clock, + reset +); + function void load_word(int bank, int row, bit [127:0] data); + tb_top.reference_model.load_word(bank, row, data); + endfunction + function bit [127:0] read_word(int row); + return tb_top.reference_model.memory[2][row]; + endfunction + function longint get_ticks(); + return tb_top.reference_model.ticks; + endfunction + task execute(int m, int k, int base, bit first, output longint elapsed); + tb_top.reference_model.execute(m, k, base, first, elapsed); + endtask +endinterface diff --git a/examples/prefab/balls/redmule/verify/src/tb_top.sv b/examples/prefab/balls/redmule/verify/src/tb_top.sv new file mode 100644 index 000000000..4171189d3 --- /dev/null +++ b/examples/prefab/balls/redmule/verify/src/tb_top.sv @@ -0,0 +1,97 @@ +module tb_top; + import uvm_pkg::*; + import redmule_uvm_pkg::*; + + bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); + + redmule_reference reference_model ( + intf.clock, + intf.reset + ); + redmule_reference_if reference_interface ( + intf.clock, + intf.reset + ); + + RedmuleBall dut ( + .clock(intf.clock), + .reset(intf.reset), + .blink_status_running(intf.status_running), + .blink_cmdReq_ready(intf.cmd_req_ready), + .blink_cmdReq_valid(intf.cmd_req_valid), + .blink_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .blink_cmdReq_bits_funct(intf.cmd_req_bits_funct), + .blink_cmdReq_bits_rs2(intf.cmd_req_bits_rs2), + .blink_cmdReq_bits_iter(intf.cmd_req_bits_iter), + .blink_cmdReq_bits_op1_bank(intf.cmd_req_bits_op1_bank), + .blink_cmdReq_bits_op2_bank(intf.cmd_req_bits_op2_bank), + .blink_cmdReq_bits_wr_bank(intf.cmd_req_bits_wr_bank), + .blink_cmdReq_bits_op1_group(intf.cmd_req_bits_op1_group), + .blink_cmdReq_bits_op2_group(intf.cmd_req_bits_op2_group), + .blink_cmdReq_bits_wr_group(intf.cmd_req_bits_wr_group), + .blink_cmdReq_bits_op1_col(intf.cmd_req_bits_op1_col), + .blink_cmdReq_bits_op2_col(intf.cmd_req_bits_op2_col), + .blink_cmdReq_bits_wr_col(intf.cmd_req_bits_wr_col), + .blink_cmdReq_bits_wr_read_bank(intf.cmd_req_bits_wr_read_bank), + .blink_cmdReq_bits_wr_arch_bank(intf.cmd_req_bits_wr_arch_bank), + .blink_cmdResp_ready(intf.cmd_resp_ready), + .blink_cmdResp_valid(intf.cmd_resp_valid), + .blink_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .blink_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .blink_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .blink_bankRead_0_group_id(intf.bank_read_group_id[0]), + .blink_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .blink_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .blink_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .blink_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .blink_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .blink_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .blink_bankRead_1_bank_id(intf.bank_read_bank_id[1]), + .blink_bankRead_1_rob_id(intf.bank_read_rob_id[1]), + .blink_bankRead_1_group_id(intf.bank_read_group_id[1]), + .blink_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), + .blink_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), + .blink_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), + .blink_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), + .blink_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), + .blink_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), + .blink_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .blink_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .blink_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .blink_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .blink_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .blink_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .blink_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), + .blink_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), + .blink_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), + .blink_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), + .blink_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), + .blink_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), + .blink_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), + .blink_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), + .blink_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), + .blink_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), + .blink_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), + .blink_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), + .blink_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), + .blink_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), + .blink_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), + .blink_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), + .blink_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .blink_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .blink_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .blink_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]) + ); + + initial begin + intf.clock = 1'b0; + intf.reset = 1'b1; + forever #5 intf.clock = ~intf.clock; + end + + initial begin + uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); + uvm_config_db#(virtual redmule_reference_if)::set(null, "*", "reference", reference_interface); + run_test("redmule_ball_test"); + end +endmodule diff --git a/examples/prefab/balls/redmule/workloads/ctests/CMakeLists.txt b/examples/prefab/balls/redmule/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..829a0b259 --- /dev/null +++ b/examples/prefab/balls/redmule/workloads/ctests/CMakeLists.txt @@ -0,0 +1,12 @@ +set(REDMULE_BANK_WIDTH ${DESIGN_PROFILES_${BUCKYBALL_CTEST_TARGET}_THREAD_MEM_BANK_WIDTH}) +set(REDMULE_BANK_LINES ${DESIGN_PROFILES_${BUCKYBALL_CTEST_TARGET}_THREAD_MEM_BANK_ENTRIES}) +if(NOT REDMULE_BANK_WIDTH EQUAL 128 OR REDMULE_BANK_LINES LESS 18) + message( + FATAL_ERROR + "RedmuleBall tests require 128-bit rows and at least 18 bank rows, got ${REDMULE_BANK_WIDTH} bits and ${REDMULE_BANK_LINES} rows" + ) +endif() +add_buckyball_ctest(redmule_f16_test.c) +add_buckyball_ctest(redmule_f16_chain_test.c) +add_buckyball_ctest(redmule_f16_edges_test.c) +add_buckyball_ctest(redmule_f16_fused_test.c) diff --git a/examples/prefab/balls/redmule/workloads/ctests/redmule_f16_chain_test.c b/examples/prefab/balls/redmule/workloads/ctests/redmule_f16_chain_test.c new file mode 100644 index 000000000..9fcc371d1 --- /dev/null +++ b/examples/prefab/balls/redmule/workloads/ctests/redmule_f16_chain_test.c @@ -0,0 +1,42 @@ +#include +#include + +static uint16_t a[16] __attribute__((aligned(64))); +static uint16_t b[64] __attribute__((aligned(64))); +static uint16_t c[16] __attribute__((aligned(64))); + +int main(void) { + unsigned ab = CORE_VIRTUAL_BANK_COUNT - 6, bb = ab + 1, cb = ab + 2; + for (unsigned i = 0; i < 16; ++i) { + a[i] = 0x3c00; + c[i] = 0x3555; + } + for (unsigned i = 0; i < 64; ++i) + b[i] = 0x3800; + bb_mem_alloc(ab, 1, 1); + bb_mem_alloc(bb, 1, 1); + bb_mem_alloc(cb, 1, 1); + bb_mvin((uintptr_t)a, ab, 2, 1); + bb_mvin((uintptr_t)b, bb, 8, 1); + bb_mvin((uintptr_t)c, cb, 2, 1); + bb_redmule(ab, bb, cb, 2, 8, 0, 0, 0, 1); + bb_mvout((uintptr_t)c, cb, 2, 1); + for (unsigned i = 0; i < 16; ++i) + if (c[i] != 0x4400) + __builtin_trap(); + bb_redmule(ab, bb, cb, 2, 8, 0, 0, 0, 0); + bb_mvout((uintptr_t)c, cb, 2, 1); + for (unsigned i = 0; i < 16; ++i) + if (c[i] != 0x4800) + __builtin_trap(); + bb_redmule(ab, bb, cb, 2, 8, 0, 0, 0, 1); + bb_mvout((uintptr_t)c, cb, 2, 1); + for (unsigned i = 0; i < 16; ++i) + if (c[i] != 0x4400) + __builtin_trap(); + bb_mem_release(ab); + bb_mem_release(bb); + bb_mem_release(cb); + puts("RedMulE FP16 chain and restart PASS"); + return 0; +} diff --git a/examples/prefab/balls/redmule/workloads/ctests/redmule_f16_edges_test.c b/examples/prefab/balls/redmule/workloads/ctests/redmule_f16_edges_test.c new file mode 100644 index 000000000..a36bc9a3f --- /dev/null +++ b/examples/prefab/balls/redmule/workloads/ctests/redmule_f16_edges_test.c @@ -0,0 +1,34 @@ +#include +#include + +static const uint16_t a[8] + __attribute__((aligned(64))) = {0x3c00, 0x1000, 0, 0, 0, 0, 0, 0}; +static const uint16_t b[64] __attribute__((aligned(64))) = { + 0x3c00, 0x3c01, 1, 0x7bff, 0x7c00, 0xfc00, 0x7d00, 0x8000, 0x3c00, 0x3c00, + 0, 0x3c00, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0}; +static const uint16_t expected[8] = {0x3c00, 0x3c02, 1, 0x7bff, + 0x7c00, 0xfc00, 0x7e00, 0}; +static uint16_t c[8] __attribute__((aligned(64))); + +int main(void) { + bb_mem_alloc(0, 1, 1); + bb_mem_alloc(1, 1, 1); + bb_mem_alloc(2, 1, 1); + bb_mvin((uintptr_t)a, 0, 1, 1); + bb_mvin((uintptr_t)b, 1, 8, 1); + bb_redmule(0, 1, 2, 1, 8, 0, 0, 0, 1); + bb_mvout((uintptr_t)c, 2, 1, 1); + for (unsigned i = 0; i < 8; ++i) + if (c[i] != expected[i]) + __builtin_trap(); + bb_mem_release(0); + bb_mem_release(1); + bb_mem_release(2); + puts("RedMulE FP16 ties, subnormal, infinity and canonical NaN PASS"); + return 0; +} diff --git a/examples/prefab/balls/redmule/workloads/ctests/redmule_f16_fused_test.c b/examples/prefab/balls/redmule/workloads/ctests/redmule_f16_fused_test.c new file mode 100644 index 000000000..939021f87 --- /dev/null +++ b/examples/prefab/balls/redmule/workloads/ctests/redmule_f16_fused_test.c @@ -0,0 +1,28 @@ +#include +#include + +static const uint16_t a[8] + __attribute__((aligned(64))) = {0x3c00, 0x3c01, 0, 0, 0, 0, 0, 0}; +static const uint16_t b[64] __attribute__((aligned(64))) = { + 0xbc00, 0xbc00, 0xbc00, 0xbc00, 0xbc00, 0xbc00, 0xbc00, 0xbc00, + 0x3bfe, 0x3bfe, 0x3bfe, 0x3bfe, 0x3bfe, 0x3bfe, 0x3bfe, 0x3bfe}; +static uint16_t c[8] __attribute__((aligned(64))); + +int main(void) { + bb_mem_alloc(0, 1, 1); + bb_mem_alloc(1, 1, 1); + bb_mem_alloc(2, 1, 1); + bb_mvin((uintptr_t)a, 0, 1, 1); + bb_mvin((uintptr_t)b, 1, 8, 1); + bb_redmule(0, 1, 2, 1, 8, 0, 0, 0, 1); + bb_mvout((uintptr_t)c, 2, 1, 1); + // (1 + 2^-10) * (1 - 2^-10) - 1 = -2^-20; separate multiply/add loses it. + for (unsigned i = 0; i < 8; ++i) + if (c[i] != 0x8010) + __builtin_trap(); + bb_mem_release(0); + bb_mem_release(1); + bb_mem_release(2); + puts("RedMulE FP16 fused cancellation PASS"); + return 0; +} diff --git a/examples/prefab/balls/redmule/workloads/ctests/redmule_f16_test.c b/examples/prefab/balls/redmule/workloads/ctests/redmule_f16_test.c new file mode 100644 index 000000000..18e4cf00f --- /dev/null +++ b/examples/prefab/balls/redmule/workloads/ctests/redmule_f16_test.c @@ -0,0 +1,46 @@ +#include +#include + +static uint16_t a[8 * 8] __attribute__((aligned(64))); +static uint16_t b[18 * 8] __attribute__((aligned(64))); +static uint16_t c[7 * 8] __attribute__((aligned(64))); + +int main(void) { + unsigned ab = CORE_VIRTUAL_BANK_COUNT - 3, bb = ab + 1, cb = ab + 2; + for (unsigned i = 0; i < 8 * 8; ++i) + a[i] = 0x7e00; + for (unsigned i = 0; i < 18 * 8; ++i) + b[i] = 0x7e00; + for (unsigned i = 0; i < 7 * 8; ++i) + c[i] = 0x3555; + for (unsigned r = 0; r < 3; ++r) + for (unsigned k = 0; k < 16; ++k) + a[2 * 8 + r * 16 + k] = r == 1 ? 0xc000 : 0x3c00; + for (unsigned k = 0; k < 16; ++k) + for (unsigned n = 0; n < 8; ++n) + b[(2 + k) * 8 + n] = n & 1 ? 0xb800 : 0x3800; + bb_mem_alloc(ab, 1, 1); + bb_mem_alloc(bb, 1, 1); + bb_mem_alloc(cb, 1, 1); + bb_mvin((uintptr_t)a, ab, 8, 1); + bb_mvin((uintptr_t)b, bb, 18, 1); + bb_mvin((uintptr_t)c, cb, 7, 1); + bb_redmule(ab, bb, cb, 3, 16, 2, 2, 3, 1); + bb_mvout((uintptr_t)c, cb, 7, 1); + for (unsigned r = 0; r < 7; ++r) + for (unsigned n = 0; n < 8; ++n) { + uint16_t expected = 0x3555; + if (r >= 3 && r < 6) { + expected = r == 4 ? 0xcc00 : 0x4800; + if (n & 1) + expected ^= 0x8000; + } + if (c[r * 8 + n] != expected) + __builtin_trap(); + } + bb_mem_release(ab); + bb_mem_release(bb); + bb_mem_release(cb); + puts("RedMulE FP16 nonzero bases and guard rows PASS"); + return 0; +} diff --git a/examples/prefab/balls/redmule/workloads/isa/redmule.h b/examples/prefab/balls/redmule/workloads/isa/redmule.h new file mode 100644 index 000000000..5b06bce40 --- /dev/null +++ b/examples/prefab/balls/redmule/workloads/isa/redmule.h @@ -0,0 +1,24 @@ +#ifndef REDMULE_H +#define REDMULE_H +#include +#include + +// FP16 A[M,K], B[K,8], C[M,8]; ordered FP16 FMA, RNE, canonical NaN. +// Bases are bank rows. First starts with positive zero; otherwise rs1 marks +// the old C, which the Ball reads from the bank C held before this write. +// Every call writes C. +static inline void bb_redmule(uint32_t a, uint32_t b, uint32_t c, uint32_t m, + uint32_t k, uint32_t op1_base, uint32_t op2_base, + uint32_t wr_base, uint32_t first) { + if (a > 1023 || b > 1023 || c > 1023 || a == b || a == c || b == c || !m || + m > 4095 || !k || k % 8 || op1_base > 1023 || op2_base > 1023 || + wr_base > 1023 || first > 1) + __builtin_trap(); + uint64_t special = (uint64_t)m | (uint64_t)op1_base << 12 | + (uint64_t)op2_base << 22 | (uint64_t)wr_base << 32 | + (uint64_t)first << 42; + BUCKYBALL_INSTRUCTION_R_R(BB_BANK0(a) | BB_BANK1(b) | BB_BANK2(c) | + BB_ITER(k) | (first ? 0 : BB_READ_WR_OLD), + special, FUNCT7(redmule, REDMULE_F16)); +} +#endif diff --git a/examples/prefab/balls/redmule/workloads/mlir_tests/CMakeLists.txt b/examples/prefab/balls/redmule/workloads/mlir_tests/CMakeLists.txt new file mode 100644 index 000000000..cd1e18e7d --- /dev/null +++ b/examples/prefab/balls/redmule/workloads/mlir_tests/CMakeLists.txt @@ -0,0 +1,3 @@ +set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) +add_subdirectory(ball) +add_subdirectory(bank) diff --git a/examples/prefab/balls/redmule/workloads/mlir_tests/ball/CMakeLists.txt b/examples/prefab/balls/redmule/workloads/mlir_tests/ball/CMakeLists.txt new file mode 100644 index 000000000..3933abe13 --- /dev/null +++ b/examples/prefab/balls/redmule/workloads/mlir_tests/ball/CMakeLists.txt @@ -0,0 +1,2 @@ +set(BUCKYBALL_MLIR_TEST_PREFIX ball) +add_buckyball_mlir_test(redmule_f16_chain TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/prefab/balls/redmule/workloads/mlir_tests/ball/redmule_f16_chain.mlir b/examples/prefab/balls/redmule/workloads/mlir_tests/ball/redmule_f16_chain.mlir new file mode 100644 index 000000000..ce47c740b --- /dev/null +++ b/examples/prefab/balls/redmule/workloads/mlir_tests/ball/redmule_f16_chain.mlir @@ -0,0 +1,6 @@ +// The banks 1 (A), 2 (B) and 3 (C) are allocated and loaded by the caller. +func.func @redmule_chain() attributes {llvm.emit_c_interface} { + redmule_ball.matmul_f16 <{a = 1 : i64, b = 2 : i64, c = 3 : i64, rows = 2 : i64, k = 8 : i64}> + redmule_ball.matmul_f16 <{a = 1 : i64, b = 2 : i64, c = 3 : i64, rows = 2 : i64, k = 8 : i64, read_destination}> + return +} diff --git a/examples/prefab/balls/redmule/workloads/mlir_tests/ball/redmule_f16_chain_main.cpp b/examples/prefab/balls/redmule/workloads/mlir_tests/ball/redmule_f16_chain_main.cpp new file mode 100644 index 000000000..9adc1d8d4 --- /dev/null +++ b/examples/prefab/balls/redmule/workloads/mlir_tests/ball/redmule_f16_chain_main.cpp @@ -0,0 +1,27 @@ +#include +#include + +extern "C" void _mlir_ciface_redmule_chain(); +alignas(64) static uint16_t a[16], b[64], c[16]; + +int main() { + for (auto &value : a) + value = 0x3c00; + for (auto &value : b) + value = 0x3800; + bb_mem_alloc(1, 1, 1); + bb_mem_alloc(2, 1, 1); + bb_mem_alloc(3, 1, 1); + bb_mvin((uintptr_t)a, 1, 2, 1); + bb_mvin((uintptr_t)b, 2, 8, 1); + _mlir_ciface_redmule_chain(); + bb_mvout((uintptr_t)c, 3, 2, 1); + for (auto value : c) + if (value != 0x4800) + __builtin_trap(); + bb_mem_release(1); + bb_mem_release(2); + bb_mem_release(3); + puts("RedMulE BallOp FP16 chain PASS"); + return 0; +} diff --git a/examples/prefab/balls/redmule/workloads/mlir_tests/ball/redmule_f16_invalid_k.mlir b/examples/prefab/balls/redmule/workloads/mlir_tests/ball/redmule_f16_invalid_k.mlir new file mode 100644 index 000000000..d482cf813 --- /dev/null +++ b/examples/prefab/balls/redmule/workloads/mlir_tests/ball/redmule_f16_invalid_k.mlir @@ -0,0 +1,6 @@ +// RUN: buddy-opt %s -lower-buckyball -verify-diagnostics +func.func @invalid_k() { + // expected-error @+1 {{FP16 matmul shape exceeds the bank or ISA geometry}} + redmule_ball.matmul_f16 <{a = 1 : i64, b = 2 : i64, c = 3 : i64, rows = 1 : i64, k = 7 : i64}> + return +} diff --git a/examples/prefab/balls/redmule/workloads/mlir_tests/bank/CMakeLists.txt b/examples/prefab/balls/redmule/workloads/mlir_tests/bank/CMakeLists.txt new file mode 100644 index 000000000..d11a83ac3 --- /dev/null +++ b/examples/prefab/balls/redmule/workloads/mlir_tests/bank/CMakeLists.txt @@ -0,0 +1,2 @@ +set(BUCKYBALL_MLIR_TEST_PREFIX bank) +add_buckyball_mlir_test(redmule_f16_rows TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/prefab/balls/redmule/workloads/mlir_tests/bank/redmule_f16_invalid_alias.mlir b/examples/prefab/balls/redmule/workloads/mlir_tests/bank/redmule_f16_invalid_alias.mlir new file mode 100644 index 000000000..9d53f0db9 --- /dev/null +++ b/examples/prefab/balls/redmule/workloads/mlir_tests/bank/redmule_f16_invalid_alias.mlir @@ -0,0 +1,10 @@ +// RUN: buddy-opt %s -assign-physical-banks -lower-buckyball -verify-diagnostics +func.func @invalid_alias() { + %a = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %b = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + // expected-error @+1 {{FP16 matmul requires distinct A, B and C banks}} + %c = redmule_ball.bank_matmul_f16 %a, %b -> %a <{rows = 1 : i64, k = 8 : i64}> + buckyball.bank_release %c + buckyball.bank_release %b + return +} diff --git a/examples/prefab/balls/redmule/workloads/mlir_tests/bank/redmule_f16_rows.mlir.in b/examples/prefab/balls/redmule/workloads/mlir_tests/bank/redmule_f16_rows.mlir.in new file mode 100644 index 000000000..09af1805d --- /dev/null +++ b/examples/prefab/balls/redmule/workloads/mlir_tests/bank/redmule_f16_rows.mlir.in @@ -0,0 +1,48 @@ +func.func private @check_result(memref<@BB_BANK_LINES@x8xf16>) -> () +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %two = arith.constant 2 : index + %eight = arith.constant 8 : index + %rows = arith.constant @BB_BANK_LINES@ : index + %pos_one = arith.constant 1.0 : f16 + %neg_one = arith.constant -1.0 : f16 + %pos_half = arith.constant 0.5 : f16 + %neg_half = arith.constant -0.5 : f16 + %a = memref.alloc() alignment = 64 : memref<@BB_BANK_LINES@x8xf16> + %b = memref.alloc() alignment = 64 : memref<8x8xf16> + %c = memref.alloc() alignment = 64 : memref<@BB_BANK_LINES@x8xf16> + scf.for %row = %zero to %rows step %one { + %parity = arith.remui %row, %two : index + %odd = arith.cmpi eq, %parity, %one : index + %value = arith.select %odd, %neg_one, %pos_one : f16 + scf.for %lane = %zero to %eight step %one { + memref.store %value, %a[%row, %lane] : memref<@BB_BANK_LINES@x8xf16> + } + } + scf.for %k = %zero to %eight step %one { + scf.for %lane = %zero to %eight step %one { + %parity = arith.remui %lane, %two : index + %odd = arith.cmpi eq, %parity, %one : index + %value = arith.select %odd, %neg_half, %pos_half : f16 + memref.store %value, %b[%k, %lane] : memref<8x8xf16> + } + } + %a_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %b_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %c_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %loaded_a = buckyball.bank_mvin %a, %a_bank <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x8xf16> + %loaded_b = buckyball.bank_mvin %b, %b_bank <{depth = 8 : i64, stride = 1 : i64}> : memref<8x8xf16> + %result = redmule_ball.bank_matmul_f16 %loaded_a, %loaded_b -> %c_bank + <{rows = @BB_BANK_LINES@ : i64, k = 8 : i64}> + %stored = buckyball.bank_mvout %c, %result <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x8xf16> + buckyball.bank_release %loaded_a + buckyball.bank_release %loaded_b + buckyball.bank_release %stored + func.call @check_result(%c) : (memref<@BB_BANK_LINES@x8xf16>) -> () + memref.dealloc %a : memref<@BB_BANK_LINES@x8xf16> + memref.dealloc %b : memref<8x8xf16> + memref.dealloc %c : memref<@BB_BANK_LINES@x8xf16> + return %zero_i8 : i8 +} diff --git a/examples/prefab/balls/redmule/workloads/mlir_tests/bank/redmule_f16_rows_main.cpp b/examples/prefab/balls/redmule/workloads/mlir_tests/bank/redmule_f16_rows_main.cpp new file mode 100644 index 000000000..0e0c9735f --- /dev/null +++ b/examples/prefab/balls/redmule/workloads/mlir_tests/bank/redmule_f16_rows_main.cpp @@ -0,0 +1,19 @@ +#include +#include +#include + +extern "C" void check_result(uint16_t *, uint16_t *data, int64_t offset, + int64_t rows, int64_t lanes, int64_t stride, + int64_t lane_stride) { + if (rows != THREAD_MEM_BANK_ENTRIES || lanes != 8 || stride != 8 || + lane_stride != 1) + exit(1); + for (int row = 0; row < rows; ++row) + for (int lane = 0; lane < lanes; ++lane) + if (data[offset + row * stride + lane] != + (uint16_t)(0x4400 | ((row ^ lane) & 1) << 15)) { + printf("RedMulE bank rows mismatch row=%d lane=%d\n", row, lane); + exit(1); + } + puts("RedMulE BankOp full bank rows PASS"); +} diff --git a/examples/prefab/balls/relu/README.md b/examples/prefab/balls/relu/README.md new file mode 100644 index 000000000..42b0326d1 --- /dev/null +++ b/examples/prefab/balls/relu/README.md @@ -0,0 +1,28 @@ +# ReluBall + +`ReluBall` computes signed INT32 `max(x, 0)` in place in one physical pbank. +It has no internal SRAM: a 128-bit register holds one pbank line between its +read and write requests. + +## Command + +`RELU(vbank, group, iter, stride)` selects physical pbank `group` in virtual +bank `vbank`. The instruction places `vbank` in both the source and destination +bank fields and sets the generic `BB_UPDATE_WR` flag for an in-place update. +`group` is a pbank number, never a data-layout term. + +`iter` is the number of 128-bit physical lines processed in each segment. +`stride` is the distance between segments. Both are positive; `iter` is a +multiple of 16, `iter <= stride`, and `stride` divides the pbank depth. The +command processes `iter` lines at every `stride` boundary until it reaches the +end of the selected pbank. + +For every line the Ball performs one single-port sequence: + +```text +pbank read -> 4 signed int32 ReLUs -> pbank write +``` + +The selected group must be allocated. There is no output vbank, no fallback +layout, and no tail path. Tile lowering pads logical matrices to 16-aligned +shapes; compiler packing fills the final pbank chunk before issuing ReLU. diff --git a/examples/prefab/balls/relu/arch/Relu.scala b/examples/prefab/balls/relu/arch/Relu.scala new file mode 100644 index 000000000..99ccb37a6 --- /dev/null +++ b/examples/prefab/balls/relu/arch/Relu.scala @@ -0,0 +1,184 @@ +package examples.balls.relu + +import framework.acc.balldomain.configs.BallRouteParam + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import framework.acc.balldomain.ball.blink.{BallCommand, BallResponse} +import chisel3.experimental.hierarchy.{instantiable, public} +import chisel3.util._ +import framework.acc.balldomain.ball.blink.BallStatus +import framework.acc.memdomain.{BankRead, BankWrite} + +@instantiable +class Relu( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val routingParam: BallDomainParam, + val instance: BallRouteParam) + extends Module { + val commandDataBits = frontendParam.commandDataBits + val addressWidth = log2Up(memParam.bankEntries) + val countWidth = log2Up(memParam.bankEntries + 1) + val bankIdWidth = frontendParam.bank_id_len + val groupWidth = log2Up(memParam.bankNum) + + val mapping = instance + + require(mapping.inBW == 1, "ReluBall requires one SRAM read port") + require(mapping.outBW == 1, "ReluBall requires one SRAM write port") + require(memParam.bankWidth == 128, "ReluBall requires 128-bit SRAM rows") + require(memParam.bankMaskLen == 16, "ReluBall requires sixteen byte enables") + + @public + val io = IO(new Bundle { + + val cmdReq = Flipped(Decoupled(new BallCommand(frontendParam, memParam))) + + val cmdResp = Decoupled(new BallResponse(frontendParam)) + + val bankRead = Vec( + 1, + Flipped(new BankRead( + frontendParam, + memParam + )) + ) + + val bankWrite = Vec( + 1, + Flipped(new BankWrite( + frontendParam, + memParam + )) + ) + + val status = new BallStatus + }) + + val Seq(idle, waitForChannels, readRequest, readResponse, writeRequest, writeResponse, complete) = Enum(7) + val state = RegInit(idle) + + val robId = RegInit(0.U(log2Up(frontendParam.rob_entries).W)) + val bank = RegInit(0.U(bankIdWidth.W)) + val group = RegInit(0.U(groupWidth.W)) + val iter = RegInit(0.U(countWidth.W)) + val stride = RegInit(0.U(countWidth.W)) + val segment = RegInit(0.U(addressWidth.W)) + val line = RegInit(0.U(addressWidth.W)) + val word = Reg(UInt(128.W)) + + val relu = Wire(Vec(4, UInt(32.W))) + for (lane <- 0 until 4) { + val value = word(32 * lane + 31, 32 * lane).asSInt + relu(lane) := Mux(value < 0.S, 0.U, value.asUInt) + } + val address = segment + line + + io.cmdReq.ready := state === idle + io.cmdResp.valid := state === complete + io.cmdResp.bits.rob_id := robId + + io.bankRead(0).rob_id := robId + io.bankRead(0).bank_id := bank + io.bankRead(0).group_id := group + io.bankRead(0).io.req.valid := false.B + io.bankRead(0).io.req.bits.addr := address + io.bankRead(0).io.resp.ready := false.B + + io.bankWrite(0).rob_id := robId + io.bankWrite(0).bank_id := bank + io.bankWrite(0).group_id := group + io.bankWrite(0).io.req.valid := false.B + io.bankWrite(0).io.req.bits.addr := address + io.bankWrite(0).io.req.bits.data := Cat(relu.reverse) + io.bankWrite(0).io.req.bits.mask := VecInit(Seq.fill(16)(true.B)) + io.bankWrite(0).io.resp.ready := false.B + + switch(state) { + is(idle) { + when(io.cmdReq.fire) { + val command = io.cmdReq.bits + + assert( + command.op1_bank === command.wr_bank, + "ReluBall source and destination banks must match" + ) + assert( + command.op1_col =/= 0.U && command.op2_bank < command.op1_col, + "ReluBall group is not allocated in bank" + ) + assert(command.iter =/= 0.U && command.iter(3, 0) === 0.U, "ReluBall iter must be a positive multiple of 16") + assert( + command.rs2 =/= 0.U && command.rs2 <= memParam.bankEntries.U, + "ReluBall stride must fit in one physical bank" + ) + assert( + memParam.bankEntries.U % command.rs2 === 0.U, + "ReluBall stride must divide the physical bank depth" + ) + assert(command.iter <= command.rs2, "ReluBall iter must not exceed stride") + + robId := io.cmdReq.bits.rob_id + bank := command.op1_bank + // RELU names its group in the op2 bank field. + group := command.op1_group + command.op2_bank + iter := command.iter + stride := command.rs2 + segment := 0.U + line := 0.U + state := waitForChannels + } + } + + is(waitForChannels) { + state := readRequest + } + + is(readRequest) { + io.bankRead(0).io.req.valid := true.B + when(io.bankRead(0).io.req.fire)(state := readResponse) + } + + is(readResponse) { + io.bankRead(0).io.resp.ready := true.B + when(io.bankRead(0).io.resp.fire) { + word := io.bankRead(0).io.resp.bits.data + state := writeRequest + } + } + + is(writeRequest) { + io.bankWrite(0).io.req.valid := true.B + when(io.bankWrite(0).io.req.fire)(state := writeResponse) + } + + is(writeResponse) { + io.bankWrite(0).io.resp.ready := true.B + when(io.bankWrite(0).io.resp.fire) { + when(line +& 1.U === iter) { + when(segment +& stride === memParam.bankEntries.U) { + state := complete + }.otherwise { + segment := segment + stride + line := 0.U + state := readRequest + } + }.otherwise { + line := line + 1.U + state := readRequest + } + } + } + + is(complete) { + when(io.cmdResp.fire)(state := idle) + } + } + + io.status.idle := state === idle + io.status.running := state =/= idle +} diff --git a/examples/prefab/balls/relu/arch/ReluBall.scala b/examples/prefab/balls/relu/arch/ReluBall.scala new file mode 100644 index 000000000..e717cfb15 --- /dev/null +++ b/examples/prefab/balls/relu/arch/ReluBall.scala @@ -0,0 +1,46 @@ +package examples.balls.relu + +import framework.acc.balldomain.bbus.{BallFactory, BallPorts} +import buckyball.balls.relu.{Params => ParametersProto} +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.balldomain.ball.blink.{BlinkIO, HasBlink} + +@instantiable +class ReluBall( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Module + with HasBlink { + val ball = bp.balls.find(_.id == id).get + + ball.parameters.unpack(classOf[ParametersProto]) + + @public + val blink = IO(new BlinkIO(fp, mp, bp, id)) + + val unit: Instance[Relu] = Instantiate(new Relu(fp, mp, bp, ball)) + unit.io.cmdReq <> blink.cmdReq + unit.io.cmdResp <> blink.cmdResp + unit.io.bankRead(0) <> blink.bankRead(0) + unit.io.bankWrite(0) <> blink.bankWrite(0) + blink.status <> unit.io.status +} + +object ReluBall extends BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts = + BallPorts(Instantiate(new ReluBall(fp, mp, bp, id)).blink, None) + +} diff --git a/examples/prefab/balls/relu/configs/default.toml b/examples/prefab/balls/relu/configs/default.toml new file mode 100644 index 000000000..0ae61cbda --- /dev/null +++ b/examples/prefab/balls/relu/configs/default.toml @@ -0,0 +1 @@ +# ReluBall has no implementation-specific parameters. diff --git a/examples/prefab/balls/relu/configs/relu_params.proto b/examples/prefab/balls/relu/configs/relu_params.proto new file mode 100644 index 000000000..caf020c5c --- /dev/null +++ b/examples/prefab/balls/relu/configs/relu_params.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; +package buckyball.balls.relu; +option java_package = "buckyball.balls.relu"; +option java_multiple_files = true; + +message Params { +} diff --git a/examples/balls/relu/emu/src/50_relu.rs b/examples/prefab/balls/relu/emu/src/50_relu.rs similarity index 76% rename from examples/balls/relu/emu/src/50_relu.rs rename to examples/prefab/balls/relu/emu/src/50_relu.rs index 1f1a4f7b7..7ac68263c 100644 --- a/examples/balls/relu/emu/src/50_relu.rs +++ b/examples/prefab/balls/relu/emu/src/50_relu.rs @@ -1,18 +1,24 @@ -use super::super::bank::bank_size; -use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; -use super::instruction::{BallInstruction, ExecContext}; +use bebop_bemu::thread::banks::bank_size; +use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old, rs1_update}; +use super::instruction::{BallModel, ExecContext}; pub struct Relu; -impl BallInstruction for Relu { +impl BallModel for Relu { fn exec(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { let bank = rs1_b0(xs1); let group = rs1_b1(xs1); let iter = rs1_iter(xs1) as usize; let stride = xs2 as usize; + // The RTL requires op1 and wr to name one renamed bank: only an in-place + // update (rs1[63]) keeps the destination alias. + assert!( + !rs1_read_old(xs1) && rs1_update(xs1), + "relu: updates its bank in place; rs1[63] must be set and rs1[29] clear" + ); - if rs1_b2(xs1) != 0 { - panic!("relu: bank2 must be zero"); + if rs1_b2(xs1) != bank { + panic!("relu: source and destination banks must match"); } let config = *ctx.config(bank); if !config.allocated { diff --git a/examples/prefab/balls/relu/emu/src/dispatch.rs b/examples/prefab/balls/relu/emu/src/dispatch.rs new file mode 100644 index 000000000..f8d176b31 --- /dev/null +++ b/examples/prefab/balls/relu/emu/src/dispatch.rs @@ -0,0 +1,31 @@ +use bebop_bemu::ball::{BallModel, ExecContext}; + +use super::f50_relu; + +const BALL_CLASS: &str = "examples.balls.relu.ReluBall"; + +pub fn execute_known( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, + ctx: &mut ExecContext, +) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("RELU") => Some(f50_relu::Relu::exec(xs1, xs2, ctx)), + Some(_) | None => None, + } +} + +pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("RELU") => Some(f50_relu::Relu::latency(xs1, xs2)), + Some(_) | None => None, + } +} diff --git a/examples/prefab/balls/relu/emu/src/lib.rs b/examples/prefab/balls/relu/emu/src/lib.rs new file mode 100644 index 000000000..bb463b57e --- /dev/null +++ b/examples/prefab/balls/relu/emu/src/lib.rs @@ -0,0 +1,12 @@ +pub(crate) use bebop_bemu::{thread::instructions::decode, ball as instruction}; + +#[path = "50_relu.rs"] +mod f50_relu; + +mod dispatch; + +pub use dispatch::{cycles_after_issue, execute_known}; + +#[bebop_bemu::ball::linkme::distributed_slice(bebop_bemu::BALLS)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static BALL: bebop_bemu::Ball = bebop_bemu::Ball { execute: execute_known, cycles: cycles_after_issue }; diff --git a/examples/prefab/balls/relu/stack/src/reluOps.cpp b/examples/prefab/balls/relu/stack/src/reluOps.cpp new file mode 100644 index 000000000..f6171fe82 --- /dev/null +++ b/examples/prefab/balls/relu/stack/src/reluOps.cpp @@ -0,0 +1,66 @@ +//===- reluOps.cpp - ReluBall operations ----------------------------------===// +// +// The dialect of ReluBall, its virtual operations building their physical ones +// and the instructions of the physical operations. +// +//===----------------------------------------------------------------------===// + +#include "reluOps.h" + +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::relu_ball; +using mlir::buddy::bankFields; +using mlir::buddy::constantI64; +using mlir::buddy::emitInstruction; +using mlir::buddy::field; + +#include "reluDialect.cpp.inc" + +#define GET_OP_CLASSES +#include "reluOps.cpp.inc" + +void ReluBallDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "reluOps.cpp.inc" + >(); +} + +namespace { +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallDialect( + [](DialectRegistry ®istry) { registry.insert(); }), + true); +} // namespace + +SmallVector BankReluI32Op::getSourceBanks() { return {getBank()}; } +Value BankReluI32Op::getDestinationBank() { return getBank(); } +bool BankReluI32Op::readsDestination() { return false; } +Operation *BankReluI32Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return ReluI32Op::create(builder, getLoc(), destination, getRows()); +} + +SmallVector ReluI32Op::getSourceBanks() { + return {int64_t(getBank())}; +} +int64_t ReluI32Op::getDestinationBank() { return getBank(); } +LogicalResult ReluI32Op::lowerToInstructions(OpBuilder &builder) { + Location loc = getLoc(); + int64_t rows = getRows(), bank = getBank(); + int64_t depth = buckyball_target::getBuckyballTarget().bankDepth; + if (rows <= 0 || rows > depth || depth % 16) + return emitOpError("relu_i32 rows exceed the bank"); + // Group 0 of the bank in the bank1 field; bit 63 marks the in-place + // update. rs2 is the segment stride. + uint64_t rs1 = + bankFields({bank}, bank, false, (rows + 15) / 16 * 16) | (1ULL << 63); + emitInstruction(builder, loc, buckyball_target::getBuckyballFunct7("RELU"), + constantI64(builder, loc, rs1), + constantI64(builder, loc, depth)); + return success(); +} diff --git a/examples/prefab/balls/relu/stack/src/reluOps.h b/examples/prefab/balls/relu/stack/src/reluOps.h new file mode 100644 index 000000000..268b7e24c --- /dev/null +++ b/examples/prefab/balls/relu/stack/src/reluOps.h @@ -0,0 +1,20 @@ +//===- reluOps.h - ReluBall operations ------------------------------------===// + +#ifndef RELU_OPS_H +#define RELU_OPS_H + +#include "Interfaces/BankInterfaces.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "reluDialect.h.inc" + +#define GET_OP_CLASSES +#include "reluOps.h.inc" + +#endif // RELU_OPS_H diff --git a/examples/prefab/balls/relu/stack/src/reluOps.td b/examples/prefab/balls/relu/stack/src/reluOps.td new file mode 100644 index 000000000..bb17a2a08 --- /dev/null +++ b/examples/prefab/balls/relu/stack/src/reluOps.td @@ -0,0 +1,44 @@ +//===- reluOps.td - ReluBall operations --------------------*- tablegen -*-===// +// +// relu_i32: max(x, 0) on the first `rows` rows of four i32 of a one-group +// bank, in place: the one source is the destination. +// +// The Ball updates whole groups of sixteen rows: it rounds `rows` up to a +// multiple of sixteen, which only touches rows past the valid data, and runs +// one segment as deep as the bank. +// +//===----------------------------------------------------------------------===// + +#ifndef RELU_OPS +#define RELU_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Interfaces/BankInterfaces.td" + +def ReluBall_Dialect : Dialect { + let name = "relu_ball"; + let cppNamespace = "::buddy::relu_ball"; + let summary = "The operations of ReluBall"; +} + +class ReluBall_Op traits = []> + : Op])>; + +def BankReluI32Op : ReluBall_Op<"bank_relu_i32", + [DeclareOpInterfaceMethods]> { + let summary = "relu_i32 on a virtual bank, in place"; + let arguments = (ins I64:$bank, I64Attr:$rows); + let results = (outs I64:$bankOut); + let assemblyFormat = "$bank prop-dict attr-dict"; +} + +def ReluI32Op : ReluBall_Op<"relu_i32", + [DeclareOpInterfaceMethods]> { + let summary = "relu_i32 on a physical bank, in place"; + let arguments = (ins I64Attr:$bank, I64Attr:$rows); + let assemblyFormat = "prop-dict attr-dict"; +} + +#endif // RELU_OPS diff --git a/examples/prefab/balls/relu/verify/casegen/Cargo.lock b/examples/prefab/balls/relu/verify/casegen/Cargo.lock new file mode 100644 index 000000000..31c865ecd --- /dev/null +++ b/examples/prefab/balls/relu/verify/casegen/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "relu-casegen" +version = "0.1.0" diff --git a/examples/prefab/balls/relu/verify/casegen/Cargo.toml b/examples/prefab/balls/relu/verify/casegen/Cargo.toml new file mode 100644 index 000000000..facc2aa63 --- /dev/null +++ b/examples/prefab/balls/relu/verify/casegen/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "relu-casegen" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["rlib", "cdylib"] + +[lints.rust] +unused_must_use = "deny" diff --git a/examples/prefab/balls/relu/verify/casegen/src/casegen.rs b/examples/prefab/balls/relu/verify/casegen/src/casegen.rs new file mode 100644 index 000000000..bc5cf613a --- /dev/null +++ b/examples/prefab/balls/relu/verify/casegen/src/casegen.rs @@ -0,0 +1,89 @@ +use crate::model; + +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ReluCmd { + pub iter: u32, + pub stride: u32, + pub bank: u32, + pub group_base: u32, + pub group: u32, + pub cols: u32, + pub rob_id: u32, + pub num_words: u32, +} + +#[derive(Clone, Debug)] +pub struct ReluCase { + pub cmd: ReluCmd, + pub rows: Vec, + pub src: Vec, + pub dst: Vec, +} + +struct Lcg(u64); + +impl Lcg { + fn next(&mut self) -> u32 { + self.0 = self.0.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + (self.0 >> 32) as u32 + } +} + +// Lane values cycle through the edges (zero, -1, INT_MIN, INT_MAX, 1) before +// falling back to pseudo-random values of both signs. +fn lane_value(rng: &mut Lcg, index: usize) -> u32 { + match index % 8 { + 0 => 0, + 1 => 0xffff_ffff, + 2 => 0x8000_0000, + 3 => 0x7fff_ffff, + 4 => 1, + _ => rng.next(), + } +} + +pub fn gen_case(index: u32) -> ReluCase { + let entries = model::bank_entries(); + // (rob_id, bank, iter, stride, cols, group, group_base) + let (rob_id, bank, iter, stride, cols, group, group_base) = match index { + 0 => (1, 0, 16, entries, 1, 0, 0), + 1 => (2, 3, 32, entries / 2, 1, 0, 0), + 2 => (3, 5, 16, 16, 1, 0, 0), + 3 => (4, 7, 48, 256, 2, 1, 2), + 4 => (5, 9, 64, 64, 4, 3, 0), + 5 => (6, 2, entries, entries, 3, 2, 1), + _ => panic!("relu: unsupported case {index}"), + }; + model::validate(entries, iter, u64::from(stride), group, cols); + let rows = model::rows(entries, iter, stride); + let mut rng = Lcg(0x5eed_0000 + u64::from(index)); + let mut lane = 0usize; + let src: Vec = rows + .iter() + .map(|_| { + let mut word = 0u128; + for l in 0..4 { + word |= u128::from(lane_value(&mut rng, lane)) << (32 * l); + lane += 1; + } + word + }) + .collect(); + let dst = src.iter().map(|&w| model::relu_word(w)).collect(); + ReluCase { + cmd: ReluCmd { + iter, + stride, + bank, + group_base, + group, + cols, + rob_id, + num_words: rows.len() as u32, + }, + rows, + src, + dst, + } +} diff --git a/examples/prefab/balls/relu/verify/casegen/src/lib.rs b/examples/prefab/balls/relu/verify/casegen/src/lib.rs new file mode 100644 index 000000000..adf14a206 --- /dev/null +++ b/examples/prefab/balls/relu/verify/casegen/src/lib.rs @@ -0,0 +1,71 @@ +mod casegen; +mod model; + +use std::cell::RefCell; + +use casegen::{ReluCase, ReluCmd}; + +thread_local! { + static CURRENT: RefCell> = const { RefCell::new(None) }; +} + +#[no_mangle] +pub extern "C" fn relu_case_load(index: u32) -> i32 { + let case = casegen::gen_case(index); + CURRENT.with(|current| *current.borrow_mut() = Some(case)); + 0 +} + +fn current_case R, R>(f: F) -> R { + CURRENT.with(|current| match current.borrow().as_ref() { + Some(case) => f(case), + None => panic!("relu_case: no case loaded; call relu_case_load first"), + }) +} + +fn idx(index: u32, case: &ReluCase, what: &str) -> usize { + if index >= case.cmd.num_words { + panic!("relu_case_{what}: index {index} out of range {}", case.cmd.num_words); + } + index as usize +} + +#[no_mangle] +pub extern "C" fn relu_case_cmd(out_ptr: *mut ReluCmd) { + if out_ptr.is_null() { + panic!("relu_case_cmd: null out_ptr"); + } + current_case(|case| unsafe { + *out_ptr = case.cmd; + }); +} + +#[no_mangle] +pub extern "C" fn relu_case_row(index: u32) -> u32 { + current_case(|case| case.rows[idx(index, case, "row")]) +} + +#[no_mangle] +pub extern "C" fn relu_case_src_word_lo(index: u32) -> u64 { + current_case(|case| case.src[idx(index, case, "src_word_lo")] as u64) +} + +#[no_mangle] +pub extern "C" fn relu_case_src_word_hi(index: u32) -> u64 { + current_case(|case| (case.src[idx(index, case, "src_word_hi")] >> 64) as u64) +} + +#[no_mangle] +pub extern "C" fn relu_case_dst_word_lo(index: u32) -> u64 { + current_case(|case| case.dst[idx(index, case, "dst_word_lo")] as u64) +} + +#[no_mangle] +pub extern "C" fn relu_case_dst_word_hi(index: u32) -> u64 { + current_case(|case| (case.dst[idx(index, case, "dst_word_hi")] >> 64) as u64) +} + +#[no_mangle] +pub extern "C" fn relu_bank_entries() -> u32 { + model::bank_entries() +} diff --git a/examples/prefab/balls/relu/verify/casegen/src/model.rs b/examples/prefab/balls/relu/verify/casegen/src/model.rs new file mode 100644 index 000000000..942757bf9 --- /dev/null +++ b/examples/prefab/balls/relu/verify/casegen/src/model.rs @@ -0,0 +1,48 @@ +// ReluBall contract: max(x, 0) on four i32 lanes per row, in place on one +// bank group. The Ball walks segments 0, stride, 2 * stride, ... up to the +// physical bank depth and updates the first `iter` rows of each segment. + +pub fn bank_entries() -> u32 { + let path = std::env::var("BB_VERIFY_CONFIG").expect("BB_VERIFY_CONFIG is required"); + let text = std::fs::read_to_string(path).expect("failed to read BB_VERIFY_CONFIG"); + text.lines() + .find_map(|line| line.strip_prefix("bank_entries=")) + .expect("bank_entries missing from BB_VERIFY_CONFIG") + .parse() + .expect("invalid bank_entries in BB_VERIFY_CONFIG") +} + +pub fn validate(entries: u32, iter: u32, stride: u64, group: u32, cols: u32) { + if iter == 0 || iter % 16 != 0 { + panic!("relu: iter must be a positive multiple of 16"); + } + if stride == 0 || stride > u64::from(entries) || u64::from(entries) % stride != 0 { + panic!("relu: stride must divide the physical bank depth"); + } + if u64::from(iter) > stride { + panic!("relu: iter must not exceed stride"); + } + if cols == 0 || group >= cols { + panic!("relu: group is not allocated in bank"); + } +} + +// Rows in the order the Ball reads and rewrites them. +pub fn rows(entries: u32, iter: u32, stride: u32) -> Vec { + let mut out = Vec::new(); + for segment in (0..entries).step_by(stride as usize) { + for line in 0..iter { + out.push(segment + line); + } + } + out +} + +pub fn relu_word(word: u128) -> u128 { + let mut out = 0u128; + for lane in 0..4 { + let value = (word >> (32 * lane)) as u32 as i32; + out |= u128::from(value.max(0) as u32) << (32 * lane); + } + out +} diff --git a/examples/prefab/balls/relu/verify/filelists/relu_ball.f b/examples/prefab/balls/relu/verify/filelists/relu_ball.f new file mode 100644 index 000000000..f4539fab9 --- /dev/null +++ b/examples/prefab/balls/relu/verify/filelists/relu_ball.f @@ -0,0 +1,25 @@ ++incdir+@UVM@/src/core ++incdir+@UVM@/src/ball ++incdir+@UVM@/src/ball/agents/cmd ++incdir+@UVM@/src/ball/agents/mem ++incdir+@UVM@/src/ball/agents/resp ++incdir+@UVM@/src/ball/cov ++incdir+@UVM@/src/ball/env ++incdir+src ++incdir+src/pkg ++incdir+src/common ++incdir+src/seq ++incdir+src/cov ++incdir+src/env ++incdir+src/tests ++incdir+@RTL@ + +@UVM@/src/core/package.sv +@UVM@/src/ball/bb_blink_if.sv +@UVM@/src/ball/bb_uvm_pkg.sv + +@RTL@/Relu.sv +@RTL@/ReluBall.sv + +src/pkg/relu_pkg.sv +src/tb_top.sv diff --git a/examples/prefab/balls/relu/verify/src/common/relu_defs.svh b/examples/prefab/balls/relu/verify/src/common/relu_defs.svh new file mode 100644 index 000000000..f06395a2d --- /dev/null +++ b/examples/prefab/balls/relu/verify/src/common/relu_defs.svh @@ -0,0 +1,27 @@ +typedef struct { + int unsigned iter; + int unsigned stride; + int unsigned bank; + int unsigned group_base; + int unsigned group; + int unsigned cols; + int unsigned rob_id; + int unsigned num_words; +} relu_cmd_dpi_t; + +import "DPI-C" function int unsigned relu_bank_entries(); +import "DPI-C" function int relu_case_load(input int unsigned index); +import "DPI-C" function void relu_case_cmd(output relu_cmd_dpi_t cmd); +import "DPI-C" function int unsigned relu_case_row(input int unsigned word_index); +import "DPI-C" function longint unsigned relu_case_src_word_lo(input int unsigned word_index); +import "DPI-C" function longint unsigned relu_case_src_word_hi(input int unsigned word_index); +import "DPI-C" function longint unsigned relu_case_dst_word_lo(input int unsigned word_index); +import "DPI-C" function longint unsigned relu_case_dst_word_hi(input int unsigned word_index); + +`ifndef RELU_FUNCT7 +`error "RELU_FUNCT7 must be provided by the selected ballISA" +`endif + +localparam int RELU_THREAD_FUNCT7 = `RELU_FUNCT7; +localparam int RELU_NUM_CASES = 6; +localparam int RELU_TIMEOUT_CYCLES = 100000; diff --git a/examples/prefab/balls/relu/verify/src/common/relu_items.svh b/examples/prefab/balls/relu/verify/src/common/relu_items.svh new file mode 100644 index 000000000..3f5acec87 --- /dev/null +++ b/examples/prefab/balls/relu/verify/src/common/relu_items.svh @@ -0,0 +1,64 @@ +class relu_cmd_item extends bb_blink_cmd_item; + `uvm_object_utils(relu_cmd_item) + + bit [31:0] num_words; + bit [31:0] stride; + bit [31:0] rows[]; + bit [127:0] src_words[]; + bit [127:0] dst_words[]; + + function new(string name = "relu_cmd_item"); + super.new(name); + endfunction + + function void load_rust_case(int unsigned index); + relu_cmd_dpi_t cmd; + int unsigned rc; + + rc = relu_case_load(index); + if (rc != 0) + `uvm_fatal("CASE", $sformatf("relu_case_load returned %0d for index %0d", rc, index)) + relu_case_cmd(cmd); + + // bb_relu: one bank is both source and destination; rs1 bank1 names the + // group inside it, which arrives in the op2 bank field. rs2 is the stride. + rob_id = cmd.rob_id[BB_BLINK_ROB_ID_W-1:0]; + funct = RELU_THREAD_FUNCT7[6:0]; + rs2 = cmd.stride; + iter = cmd.iter; + op1_bank = cmd.bank; + op2_bank = cmd.group; + wr_bank = cmd.bank; + op1_group = cmd.group_base; + op2_group = '0; + wr_group = cmd.group_base; + op1_col = cmd.cols; + op2_col = '0; + wr_col = cmd.cols; + wr_read_bank = '0; + wr_arch_bank = cmd.bank; + stride = cmd.stride; + num_words = cmd.num_words; + + if (num_words == 0) `uvm_fatal("CASE", "ReluBall case has no rows") + rows = new[num_words]; + src_words = new[num_words]; + dst_words = new[num_words]; + for (int i = 0; i < num_words; i++) begin + rows[i] = relu_case_row(i); + src_words[i] = {relu_case_src_word_hi(i), relu_case_src_word_lo(i)}; + dst_words[i] = {relu_case_dst_word_hi(i), relu_case_dst_word_lo(i)}; + end + endfunction + + function void do_copy(uvm_object rhs); + relu_cmd_item rhs_; + super.do_copy(rhs); + if (!$cast(rhs_, rhs)) `uvm_fatal("COPY", "rhs is not relu_cmd_item") + num_words = rhs_.num_words; + stride = rhs_.stride; + rows = rhs_.rows; + src_words = rhs_.src_words; + dst_words = rhs_.dst_words; + endfunction +endclass diff --git a/examples/prefab/balls/relu/verify/src/cov/relu_cov.svh b/examples/prefab/balls/relu/verify/src/cov/relu_cov.svh new file mode 100644 index 000000000..46400dbf7 --- /dev/null +++ b/examples/prefab/balls/relu/verify/src/cov/relu_cov.svh @@ -0,0 +1,47 @@ +class relu_cov extends uvm_component; + `uvm_component_utils(relu_cov) + + uvm_analysis_imp_cmd #(bb_blink_cmd_item, relu_cov) cmd_imp; + + int unsigned bank_entries; + int unsigned cur_segments; + int unsigned cur_group; + int unsigned cur_group_base; + + covergroup cmd_cg; + // One segment, several segments, every row of the bank in segments of 16. + coverpoint cur_segments { + bins one = {1}; bins some = {[2 : 63]}; bins many = {[64 : $]}; + } + coverpoint cur_group {bins first = {0}; bins later = {[1 : $]};} + coverpoint cur_group_base {bins zero = {0}; bins offset = {[1 : $]};} + endgroup + + int cmd_count; + + function new(string name, uvm_component parent); + super.new(name, parent); + cmd_imp = new("cmd_imp", this); + cmd_cg = new(); + bank_entries = relu_bank_entries(); + endfunction + + function void write_cmd(bb_blink_cmd_item item); + if (item.funct != RELU_THREAD_FUNCT7[6:0]) + `uvm_fatal("COV", $sformatf("unexpected funct %0d", item.funct)) + if (item.rs2 == 0) `uvm_fatal("COV", "zero stride") + cur_segments = bank_entries / int'(item.rs2); + cur_group = item.op2_bank; + cur_group_base = item.op1_group; + cmd_count++; + cmd_cg.sample(); + endfunction + + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (cmd_count == 0) `uvm_fatal("COV", "no cmd observed") + if (cmd_cg.get_coverage() != 100.0) + `uvm_fatal("COV", $sformatf("relu_cov %0.2f%%", cmd_cg.get_coverage())) + endfunction +endclass diff --git a/examples/prefab/balls/relu/verify/src/env/relu_env.svh b/examples/prefab/balls/relu/verify/src/env/relu_env.svh new file mode 100644 index 000000000..0614f14f4 --- /dev/null +++ b/examples/prefab/balls/relu/verify/src/env/relu_env.svh @@ -0,0 +1,27 @@ +class relu_env extends bb_blink_env #(`BB_IN_BW, `BB_OUT_BW); + `uvm_component_utils(relu_env) + + relu_scoreboard scb; + relu_cov icov; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + scb = relu_scoreboard::type_id::create("scb", this); + icov = relu_cov::type_id::create("icov", this); + endfunction + + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + scb.mem_model = mem_model; + cmd_agent.stim_ap.connect(scb.stim_imp); + cmd_agent.cmd_ap.connect(scb.cmd_imp); + cmd_agent.cmd_ap.connect(icov.cmd_imp); + read_mon.read_ap.connect(scb.read_imp); + write_mon.write_ap.connect(scb.write_imp); + resp_mon.resp_ap.connect(scb.resp_imp); + endfunction +endclass diff --git a/examples/prefab/balls/relu/verify/src/env/relu_scoreboard.svh b/examples/prefab/balls/relu/verify/src/env/relu_scoreboard.svh new file mode 100644 index 000000000..a6699d4c2 --- /dev/null +++ b/examples/prefab/balls/relu/verify/src/env/relu_scoreboard.svh @@ -0,0 +1,157 @@ +class relu_scoreboard extends uvm_scoreboard; + `uvm_component_utils(relu_scoreboard) + + uvm_analysis_imp_stim #(bb_blink_cmd_item, relu_scoreboard) stim_imp; + uvm_analysis_imp_cmd #(bb_blink_cmd_item, relu_scoreboard) cmd_imp; + uvm_analysis_imp_read #(bb_blink_read_item, relu_scoreboard) read_imp; + uvm_analysis_imp_write #(bb_blink_write_item, relu_scoreboard) write_imp; + uvm_analysis_imp_resp #(bb_blink_resp_item, relu_scoreboard) resp_imp; + + bb_blink_mem_model #(`BB_IN_BW, `BB_OUT_BW) mem_model; + + relu_cmd_item stim_q[$]; + int unsigned cmd_count; + int unsigned read_count; + int unsigned write_count; + int unsigned resp_count; + + function new(string name, uvm_component parent); + super.new(name, parent); + stim_imp = new("stim_imp", this); + cmd_imp = new("cmd_imp", this); + read_imp = new("read_imp", this); + write_imp = new("write_imp", this); + resp_imp = new("resp_imp", this); + endfunction + + function void write_stim(bb_blink_cmd_item item); + relu_cmd_item got; + relu_cmd_item clone; + + if (stim_q.size() != 0) `uvm_fatal("SCB", "single outstanding command supported") + if (!$cast(got, item)) `uvm_fatal("SCB", "stim item is not relu_cmd_item") + if (!$cast(clone, got.clone())) `uvm_fatal("SCB", "failed to clone stimulus item") + stim_q.push_back(clone); + + if (mem_model == null) `uvm_fatal("SCB", "mem_model handle not set") + mem_model.clear_mem(); + // Each row is read exactly once before it is rewritten. + for (int i = 0; i < clone.num_words; i++) + mem_model.load_word_g(int'(clone.op1_bank), int'(clone.op1_group + clone.op2_bank), + clone.rows[i], clone.src_words[i]); + mem_model.arm(); + endfunction + + function void write_cmd(bb_blink_cmd_item item); + bb_blink_cmd_item want; + relu_cmd_item stim; + stim = current_stim("CMD"); + // Compare as plain BLink commands; the comparer rejects differing types. + want = bb_blink_cmd_item::type_id::create("want"); + want.copy(stim); + verification_pkg::evidence::checks++; + if (!item.compare(want)) + `uvm_fatal("CMD", $sformatf("command mismatch: got %s exp %s", item.sprint(), want.sprint())) + cmd_count++; + endfunction + + function void write_read(bb_blink_read_item item); + relu_cmd_item stim; + int unsigned group; + + stim = current_stim("READ"); + group = stim.op1_group + stim.op2_bank; + if (item.port != 0) + `uvm_fatal("READ", $sformatf("ReluBall has one read port, got %0d", item.port)) + if (read_count >= stim.num_words) `uvm_fatal("READ", $sformatf("extra read %0d", read_count)) + if (read_count != write_count) + `uvm_fatal("READ", $sformatf("read %0d issued before write %0d", read_count, write_count)) + if (item.rob_id !== stim.rob_id) + `uvm_fatal("READ", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + if (item.bank_id !== stim.op1_bank) + `uvm_fatal("READ", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.op1_bank)) + if (item.group_id !== group[BB_BLINK_GROUP_ID_W-1:0]) + `uvm_fatal("READ", $sformatf("group mismatch: got %0d exp %0d", item.group_id, group)) + if (item.addr !== stim.rows[read_count][BB_BLINK_BANK_ADDR_W-1:0]) + `uvm_fatal( + "READ", $sformatf( + "addr mismatch at %0d: got %0d exp %0d", read_count, item.addr, stim.rows[read_count])) + read_count++; + endfunction + + function void write_write(bb_blink_write_item item); + relu_cmd_item stim; + int unsigned group; + + stim = current_stim("WRITE"); + group = stim.op1_group + stim.op2_bank; + if (item.port != 0) + `uvm_fatal("WRITE", $sformatf("ReluBall has one write port, got %0d", item.port)) + if (write_count >= read_count) + `uvm_fatal("WRITE", $sformatf("write %0d before its read", write_count)) + if (item.rob_id !== stim.rob_id) + `uvm_fatal("WRITE", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + if (item.bank_id !== stim.wr_bank) + `uvm_fatal("WRITE", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.wr_bank)) + if (item.group_id !== group[BB_BLINK_GROUP_ID_W-1:0]) + `uvm_fatal("WRITE", $sformatf("group mismatch: got %0d exp %0d", item.group_id, group)) + if (item.mask !== 16'hFFFF) + `uvm_fatal("WRITE", $sformatf("mask mismatch: got 0x%04h", item.mask)) + if (item.addr !== stim.rows[write_count][BB_BLINK_BANK_ADDR_W-1:0]) + `uvm_fatal( + "WRITE", $sformatf( + "addr mismatch at %0d: got %0d exp %0d", write_count, item.addr, stim.rows[write_count])) + verification_pkg::evidence::checks++; + if (item.data !== stim.dst_words[write_count]) + `uvm_fatal("SCB", $sformatf( + "data mismatch at row %0d: got 0x%032h exp 0x%032h", + item.addr, + item.data, + stim.dst_words[write_count] + )) + write_count++; + endfunction + + function void write_resp(bb_blink_resp_item item); + relu_cmd_item stim; + stim = current_stim("RESP"); + if (write_count != stim.num_words) + `uvm_fatal("RESP", $sformatf("response after %0d of %0d writes", write_count, stim.num_words)) + if (item.rob_id !== stim.rob_id) + `uvm_fatal("RESP", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + resp_count++; + endfunction + + function relu_cmd_item current_stim(string tag); + if (stim_q.size() == 0) `uvm_fatal("SCB", $sformatf("%s observed before stimulus", tag)) + return stim_q[0]; + endfunction + + function bit done(); + if (stim_q.size() == 0) return 1'b0; + return cmd_count == 1 && read_count == stim_q[0].num_words && + write_count == stim_q[0].num_words && resp_count == 1; + endfunction + + function void reset_counters(); + stim_q.delete(); + cmd_count = 0; + read_count = 0; + write_count = 0; + resp_count = 0; + endfunction + + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (!done()) + `uvm_fatal("SCB", $sformatf( + "incomplete: cmds=%0d reads=%0d writes=%0d resp=%0d", + cmd_count, + read_count, + write_count, + resp_count + )) + stim_q.delete(); + endfunction +endclass diff --git a/examples/prefab/balls/relu/verify/src/pkg/relu_pkg.sv b/examples/prefab/balls/relu/verify/src/pkg/relu_pkg.sv new file mode 100644 index 000000000..62c87a849 --- /dev/null +++ b/examples/prefab/balls/relu/verify/src/pkg/relu_pkg.sv @@ -0,0 +1,13 @@ +package relu_pkg; + import uvm_pkg::*; + import bb_uvm_pkg::*; + `include "uvm_macros.svh" + + `include "common/relu_defs.svh" + `include "common/relu_items.svh" + `include "seq/relu_sequences.svh" + `include "cov/relu_cov.svh" + `include "env/relu_scoreboard.svh" + `include "env/relu_env.svh" + `include "tests/relu_test.svh" +endpackage diff --git a/examples/prefab/balls/relu/verify/src/seq/relu_sequences.svh b/examples/prefab/balls/relu/verify/src/seq/relu_sequences.svh new file mode 100644 index 000000000..c90a4ccf8 --- /dev/null +++ b/examples/prefab/balls/relu/verify/src/seq/relu_sequences.svh @@ -0,0 +1,17 @@ +class relu_basic_seq extends uvm_sequence #(bb_blink_cmd_item); + `uvm_object_utils(relu_basic_seq) + + int unsigned case_index; + + function new(string name = "relu_basic_seq"); + super.new(name); + endfunction + + task body(); + relu_cmd_item item; + item = relu_cmd_item::type_id::create("item"); + start_item(item); + item.load_rust_case(case_index); + finish_item(item); + endtask +endclass diff --git a/examples/prefab/balls/relu/verify/src/tb_top.sv b/examples/prefab/balls/relu/verify/src/tb_top.sv new file mode 100644 index 000000000..b4f7cc075 --- /dev/null +++ b/examples/prefab/balls/relu/verify/src/tb_top.sv @@ -0,0 +1,80 @@ +module tb_top; + import uvm_pkg::*; + import relu_pkg::*; + + bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); + + ReluBall dut ( + .clock(intf.clock), + .reset(intf.reset), + .blink_status_running(intf.status_running), + .blink_cmdReq_ready(intf.cmd_req_ready), + .blink_cmdReq_valid(intf.cmd_req_valid), + .blink_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .blink_cmdReq_bits_funct(intf.cmd_req_bits_funct), + .blink_cmdReq_bits_rs2(intf.cmd_req_bits_rs2), + .blink_cmdReq_bits_iter(intf.cmd_req_bits_iter), + .blink_cmdReq_bits_op1_bank(intf.cmd_req_bits_op1_bank), + .blink_cmdReq_bits_op2_bank(intf.cmd_req_bits_op2_bank), + .blink_cmdReq_bits_wr_bank(intf.cmd_req_bits_wr_bank), + .blink_cmdReq_bits_op1_group(intf.cmd_req_bits_op1_group), + .blink_cmdReq_bits_op2_group(intf.cmd_req_bits_op2_group), + .blink_cmdReq_bits_wr_group(intf.cmd_req_bits_wr_group), + .blink_cmdReq_bits_op1_col(intf.cmd_req_bits_op1_col), + .blink_cmdReq_bits_op2_col(intf.cmd_req_bits_op2_col), + .blink_cmdReq_bits_wr_col(intf.cmd_req_bits_wr_col), + .blink_cmdReq_bits_wr_read_bank(intf.cmd_req_bits_wr_read_bank), + .blink_cmdReq_bits_wr_arch_bank(intf.cmd_req_bits_wr_arch_bank), + .blink_cmdResp_ready(intf.cmd_resp_ready), + .blink_cmdResp_valid(intf.cmd_resp_valid), + .blink_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .blink_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .blink_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .blink_bankRead_0_group_id(intf.bank_read_group_id[0]), + .blink_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .blink_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .blink_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .blink_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .blink_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .blink_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .blink_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .blink_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .blink_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .blink_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .blink_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .blink_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .blink_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .blink_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .blink_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .blink_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]) + ); + + // Constant outputs removed from the port list by firtool. + assign intf.bank_write_req_mask[0][0] = dut.blink_bankWrite_0_io_req_bits_mask_0; + assign intf.bank_write_req_mask[0][1] = dut.blink_bankWrite_0_io_req_bits_mask_1; + assign intf.bank_write_req_mask[0][2] = dut.blink_bankWrite_0_io_req_bits_mask_2; + assign intf.bank_write_req_mask[0][3] = dut.blink_bankWrite_0_io_req_bits_mask_3; + assign intf.bank_write_req_mask[0][4] = dut.blink_bankWrite_0_io_req_bits_mask_4; + assign intf.bank_write_req_mask[0][5] = dut.blink_bankWrite_0_io_req_bits_mask_5; + assign intf.bank_write_req_mask[0][6] = dut.blink_bankWrite_0_io_req_bits_mask_6; + assign intf.bank_write_req_mask[0][7] = dut.blink_bankWrite_0_io_req_bits_mask_7; + assign intf.bank_write_req_mask[0][8] = dut.blink_bankWrite_0_io_req_bits_mask_8; + assign intf.bank_write_req_mask[0][9] = dut.blink_bankWrite_0_io_req_bits_mask_9; + assign intf.bank_write_req_mask[0][10] = dut.blink_bankWrite_0_io_req_bits_mask_10; + assign intf.bank_write_req_mask[0][11] = dut.blink_bankWrite_0_io_req_bits_mask_11; + assign intf.bank_write_req_mask[0][12] = dut.blink_bankWrite_0_io_req_bits_mask_12; + assign intf.bank_write_req_mask[0][13] = dut.blink_bankWrite_0_io_req_bits_mask_13; + assign intf.bank_write_req_mask[0][14] = dut.blink_bankWrite_0_io_req_bits_mask_14; + assign intf.bank_write_req_mask[0][15] = dut.blink_bankWrite_0_io_req_bits_mask_15; + + initial begin + intf.clock = 1'b0; + intf.reset = 1'b1; + forever #5 intf.clock = ~intf.clock; + end + + initial begin + uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); + run_test("relu_ball_test"); + end +endmodule diff --git a/examples/prefab/balls/relu/verify/src/tests/relu_test.svh b/examples/prefab/balls/relu/verify/src/tests/relu_test.svh new file mode 100644 index 000000000..144e7a848 --- /dev/null +++ b/examples/prefab/balls/relu/verify/src/tests/relu_test.svh @@ -0,0 +1,48 @@ +class relu_ball_test extends uvm_test; + `uvm_component_utils(relu_ball_test) + + typedef virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) vif_t; + vif_t vif; + relu_env env; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(vif_t)::get(this, "", "vif", vif)) + `uvm_fatal("NOVIF", "bb_blink_if not found") + env = relu_env::type_id::create("env", this); + endfunction + + task run_phase(uvm_phase phase); + phase.raise_objection(this); + for (int unsigned index = 0; index < RELU_NUM_CASES; index++) run_case(index); + phase.drop_objection(this); + endtask + + task run_case(int unsigned index); + relu_basic_seq seq; + int cycles = 0; + apply_reset(); + env.scb.reset_counters(); + seq = relu_basic_seq::type_id::create("seq"); + seq.case_index = index; + seq.start(env.cmd_agent.seqr); + while (!env.scb.done()) begin + @(posedge vif.clock); + cycles++; + if (cycles > RELU_TIMEOUT_CYCLES) + `uvm_fatal("TIMEOUT", $sformatf("ReluBall case %0d timeout", index)) + end + `uvm_info("RELU", $sformatf("ReluBall case %0d passed in %0d cycles", index, cycles), UVM_LOW) + endtask + + task apply_reset(); + vif.reset = 1'b1; + repeat (5) @(posedge vif.clock); + vif.reset = 1'b0; + repeat (2) @(posedge vif.clock); + endtask +endclass diff --git a/examples/prefab/balls/relu/workloads/ctests/CMakeLists.txt b/examples/prefab/balls/relu/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..28834bf05 --- /dev/null +++ b/examples/prefab/balls/relu/workloads/ctests/CMakeLists.txt @@ -0,0 +1 @@ +add_buckyball_ctest(relu_i32_16x16_test.c) diff --git a/examples/balls/relu/workloads/ctests/relu_i32_16x16_test.c b/examples/prefab/balls/relu/workloads/ctests/relu_i32_16x16_test.c similarity index 100% rename from examples/balls/relu/workloads/ctests/relu_i32_16x16_test.c rename to examples/prefab/balls/relu/workloads/ctests/relu_i32_16x16_test.c diff --git a/examples/prefab/balls/relu/workloads/isa/relu.h b/examples/prefab/balls/relu/workloads/isa/relu.h new file mode 100644 index 000000000..e0c6efe9b --- /dev/null +++ b/examples/prefab/balls/relu/workloads/isa/relu.h @@ -0,0 +1,13 @@ +#ifndef _BB_RELU_H_ +#define _BB_RELU_H_ + +#include +#include + +#define bb_relu(bank_id, group, iter, stride) \ + BUCKYBALL_INSTRUCTION_R_R((BB_BANK0(bank_id) | BB_BANK1(group) | \ + BB_BANK2(bank_id) | BB_ITER(iter) | \ + BB_UPDATE_WR), \ + (stride), FUNCT7(relu, RELU)) + +#endif // _BB_RELU_H_ diff --git a/examples/balls/relu/workloads/mlir_tests/CMakeLists.txt b/examples/prefab/balls/relu/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/balls/relu/workloads/mlir_tests/CMakeLists.txt rename to examples/prefab/balls/relu/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/balls/relu/workloads/mlir_tests/ball/CMakeLists.txt b/examples/prefab/balls/relu/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/balls/relu/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/prefab/balls/relu/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/prefab/balls/relu/workloads/mlir_tests/ball/relu_i32_16row.mlir b/examples/prefab/balls/relu/workloads/mlir_tests/ball/relu_i32_16row.mlir new file mode 100644 index 000000000..22b913271 --- /dev/null +++ b/examples/prefab/balls/relu/workloads/mlir_tests/ball/relu_i32_16row.mlir @@ -0,0 +1,31 @@ +func.func private @check_result(memref<16x4xi32>) -> () +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %four = arith.constant 4 : index + %sixteen = arith.constant 16 : index + %four_i32 = arith.constant 4 : i32 + %sixteen_i32 = arith.constant 16 : i32 + %input = memref.alloc() : memref<16x4xi32> + %output = memref.alloc() : memref<16x4xi32> + scf.for %row = %zero to %sixteen step %one { + scf.for %lane = %zero to %four step %one { + %r = arith.index_cast %row : index to i32 + %l = arith.index_cast %lane : index to i32 + %base = arith.muli %r, %four_i32 : i32 + %value = arith.addi %base, %l : i32 + %signed = arith.subi %value, %sixteen_i32 : i32 + memref.store %signed, %input[%row, %lane] : memref<16x4xi32> + } + } + buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %input <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> + relu_ball.relu_i32 <{bank = 1 : i64, rows = 16 : i64}> + buckyball.mvout %output <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> + buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + func.call @check_result(%output) : (memref<16x4xi32>) -> () + memref.dealloc %input : memref<16x4xi32> + memref.dealloc %output : memref<16x4xi32> + return %zero_i8 : i8 +} diff --git a/examples/balls/relu/workloads/mlir_tests/ball/relu_i32_16row_main.cpp b/examples/prefab/balls/relu/workloads/mlir_tests/ball/relu_i32_16row_main.cpp similarity index 100% rename from examples/balls/relu/workloads/mlir_tests/ball/relu_i32_16row_main.cpp rename to examples/prefab/balls/relu/workloads/mlir_tests/ball/relu_i32_16row_main.cpp diff --git a/examples/balls/relu/workloads/mlir_tests/bank/CMakeLists.txt b/examples/prefab/balls/relu/workloads/mlir_tests/bank/CMakeLists.txt similarity index 100% rename from examples/balls/relu/workloads/mlir_tests/bank/CMakeLists.txt rename to examples/prefab/balls/relu/workloads/mlir_tests/bank/CMakeLists.txt diff --git a/examples/prefab/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign.mlir.in b/examples/prefab/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign.mlir.in new file mode 100644 index 000000000..3edb6a75d --- /dev/null +++ b/examples/prefab/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign.mlir.in @@ -0,0 +1,41 @@ +func.func private @check_result(memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32>) -> () + +func.func @main() -> i8 { + %zeroI8 = arith.constant 0 : i8 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %limit = arith.constant @BB_BANK_LINES@ : index + %lanes = arith.constant @BB_BANK_I32_LANES@ : index + %lane_count = arith.constant @BB_BANK_I32_LANES@ : i32 + %seventeen = arith.constant 17 : i32 + %bias = arith.constant 8 : i32 + + %input = memref.alloc() alignment = 64 : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + %output = memref.alloc() alignment = 64 : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + + scf.for %line = %zero to %limit step %one { + scf.for %lane = %zero to %lanes step %one { + %line_i32 = arith.index_cast %line : index to i32 + %lane_i32 = arith.index_cast %lane : index to i32 + %base = arith.muli %line_i32, %lane_count : i32 + %index = arith.addi %base, %lane_i32 : i32 + %remainder = arith.remui %index, %seventeen : i32 + %signed = arith.subi %remainder, %bias : i32 + memref.store %signed, %input[%line, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + } + } + + %bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %loaded = buckyball.bank_mvin %input, %bank <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> + : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + %result = relu_ball.bank_relu_i32 %loaded + <{rows = @BB_BANK_LINES@ : i64}> + %stored = buckyball.bank_mvout %output, %result <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> + : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + buckyball.bank_release %stored + + func.call @check_result(%output) : (memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32>) -> () + memref.dealloc %input : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + memref.dealloc %output : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + return %zeroI8 : i8 +} diff --git a/examples/prefab/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign_main.cpp b/examples/prefab/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign_main.cpp new file mode 100644 index 000000000..9bb53f9f1 --- /dev/null +++ b/examples/prefab/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign_main.cpp @@ -0,0 +1,44 @@ +#include +#include +#include + +static void fail(void) { +#ifdef BAREMETAL + volatile uint32_t *sim_exit = (volatile uint32_t *)0x60000000; + *sim_exit = 1; + while (1) { + } +#else + exit(1); +#endif +} + +#ifdef __cplusplus +extern "C" +#endif + void check_result(int32_t *allocated, int32_t *aligned, int64_t offset, + int64_t size0, int64_t size1, int64_t stride0, + int64_t stride1) { + (void)allocated; + if (size0 != THREAD_MEM_BANK_ENTRIES || size1 != THREAD_MEM_BANK_WIDTH / 32 || + stride0 != THREAD_MEM_BANK_WIDTH / 32 || stride1 != 1) { + printf("FAILED: relu bank shape %dx%d stride %dx%d\n", (int)size0, + (int)size1, (int)stride0, (int)stride1); + fail(); + } + + int32_t *output = aligned + offset; + for (int line = 0; line < THREAD_MEM_BANK_ENTRIES; ++line) { + for (int lane = 0; lane < THREAD_MEM_BANK_WIDTH / 32; ++lane) { + int32_t input = (line * (THREAD_MEM_BANK_WIDTH / 32) + lane) % 17 - 8; + int32_t expected = input < 0 ? 0 : input; + int32_t actual = output[line * stride0 + lane * stride1]; + if (actual != expected) { + printf("FAILED: relu bank[%d][%d] expected %d got %d\n", line, lane, + expected, actual); + fail(); + } + } + } + printf("relu i32 mixed-sign PASS rows=%d\n", THREAD_MEM_BANK_ENTRIES); +} diff --git a/examples/prefab/balls/smatmul/README.md b/examples/prefab/balls/smatmul/README.md new file mode 100644 index 000000000..89d2f23e3 --- /dev/null +++ b/examples/prefab/balls/smatmul/README.md @@ -0,0 +1,40 @@ +# SMatMulBall + +`SMatMulBall` computes signed INT8 matrix products with signed INT32 output. +The array is 16x16, SRAM rows are 128 bits, and every SRAM is single-port. + +## Commands + +Every command uses the framework-defined `rs1` layout: bank0 in bits 9:0, +bank1 in bits 19:10, bank2 in bits 29:20, and `iter` in bits 63:30. There are +no bank-relative base fields; every operand starts at row zero. + +`SMATMUL_BIAS` reads four rows from bank0 and retains one 16-element INT32 +bias vector. bank1, bank2, and `rs2` are zero, and `iter` is four. + +`SMATMUL_OS` uses bank0 for A, bank1 for B, and bank2 for C. `iter` is K. +`rs2[11:0]` is M, `rs2[23:12]` is N, and bits 24 and 25 are the first/last +accumulation-block flags and bits 31:26 the C base row. The remaining `rs2` +bits are zero. M is one or a positive multiple of tileRows, N is exactly 16. + +A and B have the same number of groups G. Group g holds reduction block g in +the single-group layouts below, so `iter` is K per group (a positive multiple +of 16) and the instruction accumulates over `K = iter * G`, walking group 0 to +group G-1 for every output tile. + +## Layout + +Every result row has four 128-bit words. With `outBW=1`, logical output row +`r` uses four consecutive bank rows: + +```text +line r*4 + word +``` + +C rows continue across groups: line `L = base + r*4 + word` is written to +group `L / bankEntries`, line `L % bankEntries`. M is limited by the +accumulator (`max(bankEntries / 4, 32)` rows). + +The Ball keeps partial sums internally between commands with `last=0`; only +the command with `last=1` writes C. A later accumulation block may select new +A and B banks, but each still starts at bank row zero. diff --git a/examples/prefab/balls/smatmul/arch/Array.scala b/examples/prefab/balls/smatmul/arch/Array.scala new file mode 100644 index 000000000..70b12649b --- /dev/null +++ b/examples/prefab/balls/smatmul/arch/Array.scala @@ -0,0 +1,82 @@ +package examples.balls.smatmul + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import examples.balls.smatmul.configs.SMatMulBallParam + +@instantiable +class Array(val ballParam: SMatMulBallParam) extends Module { + val param = ballParam + val rows = param.tileRows + val cols = param.tileCols + + @public + val io = IO(new Bundle { + val start = Input(Bool()) + val aRows = Input(Vec(rows, UInt(128.W))) + val bRows = Input(Vec(cols, UInt(128.W))) + val done = Output(Bool()) + val result = Output(Vec(rows, UInt(512.W))) + }) + + val running = RegInit(false.B) + val cycle = RegInit(0.U(6.W)) + val pe = Seq.tabulate(rows, cols)((_, _) => Instantiate(new PE)) + val aShift = Reg(Vec(rows, UInt(128.W))) + // The PE product register adds one cycle between the systolic operands and + // the accumulator. Keep the final drain cycle explicit so the last core + // product is included before done is asserted. + val lastCyc = (rows + cols + 15).U + + when(io.start) { + assert(!running, "Array: start while array is running") + running := true.B + cycle := 0.U + aShift := io.aRows + }.elsewhen(running) { + when(cycle === lastCyc) { + running := false.B + }.otherwise { + cycle := cycle + 1.U + } + for (row <- 0 until rows) { + when(cycle >= row.U && cycle < (row + cols).U) { + aShift(row) := aShift(row) >> 8 + } + } + } + + for (row <- 0 until rows) { + val aWord = aShift(row)(7, 0).asSInt + val aValid = running && cycle >= row.U && cycle < (row + cols).U + + for (column <- 0 until cols) { + pe(row)(column).io.clear := io.start + if (column == 0) { + pe(row)(column).io.aIn := aWord + pe(row)(column).io.aInValid := aValid + } else { + pe(row)(column).io.aIn := pe(row)(column - 1).io.aOut + pe(row)(column).io.aInValid := pe(row)(column - 1).io.aOutValid + } + + val bCycle = cycle - column.U + val bRow = bCycle(3, 0) + val bWord = io.bRows(bRow)(8 * column + 7, 8 * column).asSInt + val bValid = running && cycle >= column.U && cycle < (column + cols).U + if (row == 0) { + pe(row)(column).io.bIn := bWord + pe(row)(column).io.bInValid := bValid + } else { + pe(row)(column).io.bIn := pe(row - 1)(column).io.bOut + pe(row)(column).io.bInValid := pe(row - 1)(column).io.bOutValid + } + } + } + + for (row <- 0 until rows) { + io.result(row) := Cat((0 until cols).reverse.map(column => pe(row)(column).io.sum.asUInt)) + } + io.done := running && cycle === lastCyc +} diff --git a/examples/prefab/balls/smatmul/arch/PE.scala b/examples/prefab/balls/smatmul/arch/PE.scala new file mode 100644 index 000000000..017c5f501 --- /dev/null +++ b/examples/prefab/balls/smatmul/arch/PE.scala @@ -0,0 +1,60 @@ +package examples.balls.smatmul + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public} + +@instantiable +class PE extends Module { + + // One core accumulates sixteen signed 8x8 products. Twenty bits cover the + // full range [-262144, 258064]; the unit-level accumulator remains 32-bit + // for reduction across cores. + val tileSumWidth = 20 + + @public + val io = IO(new Bundle { + val aIn = Input(SInt(8.W)) + val aInValid = Input(Bool()) + val bIn = Input(SInt(8.W)) + val bInValid = Input(Bool()) + val clear = Input(Bool()) + val aOut = Output(SInt(8.W)) + val aOutValid = Output(Bool()) + val bOut = Output(SInt(8.W)) + val bOutValid = Output(Bool()) + val sum = Output(SInt(32.W)) + }) + + val accumulator = RegInit(0.S(tileSumWidth.W)) + val aPipe = RegInit(0.S(8.W)) + val aPipeValid = RegInit(false.B) + val bPipe = RegInit(0.S(8.W)) + val bPipeValid = RegInit(false.B) + val product = RegInit(0.S(16.W)) + val productValid = RegInit(false.B) + + when(io.clear) { + accumulator := 0.S + aPipeValid := false.B + bPipeValid := false.B + productValid := false.B + }.elsewhen(productValid) { + accumulator := accumulator + product + } + when(!io.clear) { + product := aPipe * bPipe + productValid := aPipeValid && bPipeValid + } + when(!io.clear) { + aPipe := io.aIn + aPipeValid := io.aInValid + bPipe := io.bIn + bPipeValid := io.bInValid + } + + io.aOut := aPipe + io.aOutValid := aPipeValid + io.bOut := bPipe + io.bOutValid := bPipeValid + io.sum := accumulator.pad(32) +} diff --git a/examples/prefab/balls/smatmul/arch/SMatMulBall.scala b/examples/prefab/balls/smatmul/arch/SMatMulBall.scala new file mode 100644 index 000000000..8e0928e14 --- /dev/null +++ b/examples/prefab/balls/smatmul/arch/SMatMulBall.scala @@ -0,0 +1,66 @@ +package examples.balls.smatmul + +import framework.acc.balldomain.bbus.{BallFactory, BallPorts} +import buckyball.balls.smatmul.{Params => ParametersProto} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import examples.balls.smatmul.configs.SMatMulBallParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.balldomain.ball.blink.{BallStatus, BlinkIO, HasBlink} + +@instantiable +class SMatMulBall( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Module + with HasBlink { + val ball = bp.balls.find(_.id == id).get + + val ballParam = SMatMulBallParam.fromProto(ball.parameters.unpack(classOf[ParametersProto])) + + val ballCommonConfig = ball + + val inBW = ballCommonConfig.inBW + val outBW = ballCommonConfig.outBW + + @public + val blink = IO(new BlinkIO(fp, mp, bp, id)) + + val smatmulArrayUnit: Instance[SMatMulUnit] = + Instantiate(new SMatMulUnit(fp, mp, bp, ball, ballParam)) + + smatmulArrayUnit.io.cmdReq <> blink.cmdReq + smatmulArrayUnit.io.cmdResp <> blink.cmdResp + + for (i <- 0 until inBW) { + smatmulArrayUnit.io.bankRead(i) <> blink.bankRead(i) + } + + for (i <- 0 until outBW) { + smatmulArrayUnit.io.bankWrite(i) <> blink.bankWrite(i) + } + + blink.status <> smatmulArrayUnit.io.status + + // MMIO: this Ball does not consume MMIO metadata +} + +object SMatMulBall extends BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts = + BallPorts(Instantiate(new SMatMulBall(fp, mp, bp, id)).blink, None) + +} diff --git a/examples/prefab/balls/smatmul/arch/SMatMulUnit.scala b/examples/prefab/balls/smatmul/arch/SMatMulUnit.scala new file mode 100644 index 000000000..5bc25cd34 --- /dev/null +++ b/examples/prefab/balls/smatmul/arch/SMatMulUnit.scala @@ -0,0 +1,478 @@ +package examples.balls.smatmul + +import framework.acc.balldomain.configs.BallRouteParam + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import framework.acc.balldomain.ball.blink.{BallCommand, BallResponse} +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.balldomain.ball.blink.BallStatus +import framework.acc.memdomain.{BankRead, BankWrite} +import examples.balls.smatmul.configs.SMatMulBallParam + +@instantiable +class SMatMulUnit( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val routingParam: BallDomainParam, + val instance: BallRouteParam, + val ballParam: SMatMulBallParam) + extends Module { + val commandDataBits = frontendParam.commandDataBits + val param = ballParam + val rows = param.tileRows + val cols = param.tileCols + val resultWords = 4 + val addressWidth = log2Ceil(memParam.bankEntries) + val bankIdWidth = frontendParam.bank_id_len + val maxOutputRows = memParam.bankEntries / resultWords + val accDepth = maxOutputRows.max(32) + val accAddrWidth = log2Ceil(accDepth) + + val mapping = instance + + val osFunct = instance.instructions + .find(_.mnemonic == "SMATMUL_OS").get.funct7 + + val biasFunct = instance.instructions + .find(_.mnemonic == "SMATMUL_BIAS").get.funct7 + + require(mapping.inBW == 2, "SMatMulBall requires inBW=2") + require(mapping.outBW == 1, "SMatMulBall requires outBW=1") + require(memParam.bankWidth == 128, "SMatMulBall requires 128-bit bank rows") + require( + memParam.bankMaskLen == 16, + "SMatMulBall requires sixteen byte enables" + ) + require( + memParam.bankEntries % resultWords == 0, + "SMatMulBall bank depth must be divisible by four" + ) + require(maxOutputRows >= rows, "SMatMulBall accumulator must hold at least one core") + require(isPow2(memParam.bankEntries), "SMatMulBall requires power-of-two bank depth") + require((osFunct >> 4) == 4, "SMATMUL_OS must encode two reads and one write") + require((biasFunct >> 4) == 1, "SMATMUL_BIAS must encode one read") + + @public + val io = IO(new Bundle { + + val cmdReq = Flipped(Decoupled(new BallCommand(frontendParam, memParam))) + + val cmdResp = Decoupled(new BallResponse(frontendParam)) + + val bankRead = Vec( + mapping.inBW, + Flipped(new BankRead( + frontendParam, + memParam + )) + ) + + val bankWrite = Vec( + mapping.outBW, + Flipped(new BankWrite( + frontendParam, + memParam + )) + ) + + val status = new BallStatus + }) + + val Seq( + idle, + waitForChannels, + biasReadReq, + biasReadResp, + initAccumulator, + loadTile, + runArray, + readAccumulator, + writeAccumulator, + readResult, + holdResult, + writeResult, + waitForCWrite, + complete + ) = Enum(14) + + val state = RegInit(idle) + + val robId = RegInit(0.U(log2Up(frontendParam.rob_entries).W)) + + val biasCommand = RegInit(false.B) + val firstBlock = RegInit(false.B) + val lastBlock = RegInit(false.B) + val vectorMode = RegInit(false.B) + val biasValid = RegInit(false.B) + val chainLive = RegInit(false.B) + val biasBank = RegInit(0.U(bankIdWidth.W)) + val biasBase = RegInit(0.U(log2Ceil(memParam.bankEntries).W)) + val biasRow = RegInit(0.U(2.W)) + val biasWords = Reg(Vec(resultWords, UInt(128.W))) + + val aBank = RegInit(0.U(bankIdWidth.W)) + val bBank = RegInit(0.U(bankIdWidth.W)) + val cBank = RegInit(0.U(bankIdWidth.W)) + val bases = Reg(Vec(3, UInt(memParam.groupIdWidth.W))) + // A and B group g hold reduction block g; C rows continue across groups. + val kGroups = RegInit(0.U(log2Up(memParam.bankNum + 1).W)) + val kGroup = RegInit(0.U(log2Up(memParam.bankNum + 1).W)) + val cBase = RegInit(0.U(addressWidth.W)) + val chainRows = RegInit(0.U(12.W)) + val chainCBank = RegInit(0.U(bankIdWidth.W)) + val chainCBase = RegInit(0.U(addressWidth.W)) + + val outputTileCount = RegInit(0.U(8.W)) + val reductionTileCount = RegInit(0.U(8.W)) + val outputTile = RegInit(0.U(8.W)) + val reductionTile = RegInit(0.U(8.W)) + val accumulatorRow = RegInit(0.U(4.W)) + val resultRow = RegInit(0.U(4.W)) + val outputWord = RegInit(0.U(2.W)) + val aRowsRequested = RegInit(0.U(5.W)) + val aRowsStored = RegInit(0.U(5.W)) + val bRowsRequested = RegInit(0.U(5.W)) + val bRowsStored = RegInit(0.U(5.W)) + val cRowData = Reg(UInt(512.W)) + + val aRows = Reg(Vec(rows, UInt(128.W))) + val bRows = Reg(Vec(cols, UInt(128.W))) + val accumulator = SyncReadMem(accDepth, UInt(512.W)) + val array: Instance[Array] = Instantiate(new Array(ballParam)) + + val computeGlobalRow = outputTile * rows.U + accumulatorRow + val resultGlobalRow = outputTile * rows.U + resultRow + val accumulatorRead = state === readAccumulator || state === readResult + val accumulatorWrite = state === initAccumulator || state === writeAccumulator + + val accumulatorAddress = Mux( + state === readResult, + resultGlobalRow(accAddrWidth - 1, 0), + computeGlobalRow(accAddrWidth - 1, 0) + ) + + val biasData = Cat(biasWords.reverse) + val accumulatorWriteData = Wire(UInt(512.W)) + + val accumulatorData = accumulator.readWrite( + accumulatorAddress, + accumulatorWriteData, + accumulatorRead || accumulatorWrite, + accumulatorWrite + ) + + val accumulatedResult = Wire(Vec(cols, UInt(32.W))) + for (column <- 0 until cols) { + val oldValue = accumulatorData(32 * column + 31, 32 * column).asSInt + val newValue = array.io.result(accumulatorRow)(32 * column + 31, 32 * column).asSInt + accumulatedResult(column) := (oldValue + newValue).asUInt + } + accumulatorWriteData := Mux(state === initAccumulator, biasData, Cat(accumulatedResult.reverse)) + assert(!(accumulatorRead && accumulatorWrite), "SMatMulBall accumulator SRAM is single-port") + + val aTileLine = Mux( + vectorMode, + reductionTile, + (outputTile * reductionTileCount + reductionTile) * rows.U + ) + + val bTileLine = reductionTile * cols.U + val cLine = cBase +& (resultGlobalRow << 2) +& outputWord + + for (port <- 0 until mapping.inBW) { + io.bankRead(port).rob_id := robId + io.bankRead(port).bank_id := Mux(port.U === 0.U, aBank, bBank) + io.bankRead(port).group_id := bases(port) + kGroup + io.bankRead(port).io.req.valid := false.B + io.bankRead(port).io.req.bits.addr := 0.U + io.bankRead(port).io.resp.ready := false.B + } + io.bankRead(0).bank_id := Mux(state === biasReadReq || state === biasReadResp, biasBank, aBank) + io.bankRead(0).io.req.valid := Mux( + state === biasReadReq, + true.B, + state === loadTile && aRowsRequested < Mux(vectorMode, 1.U, rows.U) + ) + io.bankRead(0).io.req.bits.addr := Mux( + state === biasReadReq, + (biasBase +& biasRow)(addressWidth - 1, 0), + (aTileLine + aRowsRequested)(addressWidth - 1, 0) + ) + io.bankRead(0).io.resp.ready := state === biasReadResp || (state === loadTile && aRowsStored < rows.U) + io.bankRead(1).io.req.valid := state === loadTile && bRowsRequested < cols.U + io.bankRead(1).io.req.bits.addr := (bTileLine + bRowsRequested)(addressWidth - 1, 0) + io.bankRead(1).io.resp.ready := state === loadTile && bRowsStored < cols.U + + val cWords = cRowData.asTypeOf(Vec(resultWords, UInt(128.W))) + io.bankWrite(0).rob_id := robId + io.bankWrite(0).bank_id := cBank + io.bankWrite(0).group_id := bases(2) + (cLine >> addressWidth) + io.bankWrite(0).io.req.valid := state === writeResult + io.bankWrite(0).io.req.bits.addr := cLine(addressWidth - 1, 0) + io.bankWrite(0).io.req.bits.data := cWords(outputWord) + io.bankWrite(0).io.req.bits.mask := VecInit(Seq.fill(16)(true.B)) + io.bankWrite(0).io.resp.ready := state === waitForCWrite + + array.io.start := state === loadTile && + aRowsStored === Mux(vectorMode, 1.U, rows.U) && bRowsStored === cols.U + array.io.aRows := aRows + array.io.bRows := bRows + + io.cmdReq.ready := state === idle + io.cmdResp.valid := state === complete + io.cmdResp.bits.rob_id := robId + io.status.idle := state === idle + io.status.running := state =/= idle && state =/= complete + + when(io.cmdReq.fire) { + val command = io.cmdReq.bits + bases := VecInit(command.op1_group, command.op2_group, command.wr_group) + val isBias = command.funct === biasFunct.U + val isMatmul = command.funct === osFunct.U + val rows = command.rs2(11, 0) + val columns = command.rs2(23, 12) + val reduction = command.iter + val first = command.rs2(24) + val last = command.rs2(25) + + assert(isBias || isMatmul, "SMatMulBall received an unknown funct7") + + robId := io.cmdReq.bits.rob_id + biasCommand := isBias + kGroups := command.op1_col + kGroup := 0.U + + when(isBias) { + assert(command.iter === 4.U, "SMATMUL_BIAS iter must be four rows") + assert(command.rs2(63, 6) === 0.U, "SMATMUL_BIAS reserves rs2[63:6]") + assert( + command.rs2(5, 0) +& 4.U <= memParam.bankEntries.U, + "SMATMUL_BIAS inputBase plus four rows must fit one bank" + ) + assert( + command.op1_col === 1.U && command.op2_col === 0.U && command.wr_col === 0.U, + "SMATMUL_BIAS requires exactly one input bank" + ) + assert(!chainLive, "SMATMUL_BIAS cannot replace bias during an accumulation chain") + biasBank := command.op1_bank + biasBase := command.rs2(5, 0) + biasRow := 0.U + }.otherwise { + assert( + command.op1_bank =/= command.op2_bank && command.op1_bank =/= command.wr_bank && + command.op2_bank =/= command.wr_bank, + "SMATMUL_OS requires distinct A, B, and C banks" + ) + assert( + command.op1_col =/= 0.U && command.op1_col === command.op2_col && + command.op1_col <= memParam.bankNum.U && command.wr_col =/= 0.U, + "SMATMUL_OS A and B must have the same reduction groups" + ) + assert(rows <= accDepth.U, "SMATMUL_OS M exceeds the accumulator") + assert( + rows === 1.U || (rows =/= 0.U && rows(2, 0) === 0.U), + "SMATMUL_OS M must be one or a positive multiple of 8" + ) + assert(columns === cols.U, "SMATMUL_OS N must be 16") + assert(reduction =/= 0.U && reduction(3, 0) === 0.U, "SMATMUL_OS K must be a positive multiple of 16") + assert(command.rs2(63, 32) === 0.U, "SMATMUL_OS reserves rs2[63:32]") + assert( + Mux( + rows === 1.U, + reduction >> 4, + (rows * reduction) >> 4 + ) <= memParam.bankEntries.U, + "SMATMUL_OS A footprint exceeds bank depth" + ) + assert( + reduction <= memParam.bankEntries.U, + "SMATMUL_OS B footprint exceeds bank depth" + ) + assert( + command.rs2( + 31, + 26 + ) +& rows * resultWords.U <= command.wr_col * memParam.bankEntries.U, + "SMATMUL_OS C footprint exceeds its groups" + ) + when(first) { + assert(!chainLive, "SMATMUL_OS first block cannot start while a chain is live") + assert(biasValid, "SMATMUL_OS first block requires a bias preload") + chainLive := true.B + chainRows := rows + // A chain retains its architectural destination across bank-version renaming. + chainCBank := command.wr_arch_bank + chainCBase := command.rs2(31, 26) + }.otherwise { + assert(chainLive, "SMATMUL_OS continuation requires a live chain") + assert( + rows === chainRows && command.wr_arch_bank === chainCBank && + command.rs2(31, 26) === chainCBase, + "SMATMUL_OS continuation changed M or C destination/base" + ) + } + + firstBlock := first + lastBlock := last + vectorMode := rows === 1.U + aBank := command.op1_bank + bBank := command.op2_bank + cBank := command.wr_bank + cBase := command.rs2(31, 26) + outputTileCount := Mux(rows === 1.U, 1.U, rows / param.tileRows.U) + reductionTileCount := reduction >> 4 + outputTile := 0.U + reductionTile := 0.U + accumulatorRow := 0.U + resultRow := 0.U + outputWord := 0.U + aRowsRequested := 0.U + aRowsStored := 0.U + bRowsRequested := 0.U + bRowsStored := 0.U + for (row <- 1 until 8) { + aRows(row) := 0.U + } + } + state := waitForChannels + } + + when(state === waitForChannels) { + state := Mux(biasCommand, biasReadReq, Mux(firstBlock, initAccumulator, loadTile)) + } + + when(state === biasReadReq && io.bankRead(0).io.req.fire) { + state := biasReadResp + } + when(state === biasReadResp && io.bankRead(0).io.resp.fire) { + biasWords(biasRow) := io.bankRead(0).io.resp.bits.data + when(biasRow === (resultWords - 1).U) { + biasValid := true.B + state := complete + }.otherwise { + biasRow := biasRow + 1.U + state := biasReadReq + } + } + + when(state === initAccumulator) { + when(accumulatorRow === Mux(vectorMode, 0.U, (rows - 1).U)) { + accumulatorRow := 0.U + when(outputTile + 1.U === outputTileCount) { + outputTile := 0.U + reductionTile := 0.U + aRowsRequested := 0.U + aRowsStored := 0.U + bRowsRequested := 0.U + bRowsStored := 0.U + state := loadTile + }.otherwise { + outputTile := outputTile + 1.U + } + }.otherwise { + accumulatorRow := accumulatorRow + 1.U + } + } + + when(state === loadTile) { + when(io.bankRead(0).io.req.fire)(aRowsRequested := aRowsRequested + 1.U) + when(io.bankRead(1).io.req.fire)(bRowsRequested := bRowsRequested + 1.U) + when(io.bankRead(0).io.resp.fire) { + aRows(aRowsStored(3, 0)) := io.bankRead(0).io.resp.bits.data + aRowsStored := aRowsStored + 1.U + } + when(io.bankRead(1).io.resp.fire) { + bRows(bRowsStored(3, 0)) := io.bankRead(1).io.resp.bits.data + bRowsStored := bRowsStored + 1.U + } + when(aRowsStored === Mux(vectorMode, 1.U, rows.U) && bRowsStored === cols.U) { + state := runArray + } + } + + when(state === runArray && array.io.done) { + accumulatorRow := 0.U + state := readAccumulator + } + when(state === readAccumulator) { + state := writeAccumulator + } + when(state === writeAccumulator) { + when(accumulatorRow === Mux(vectorMode, 0.U, (rows - 1).U)) { + accumulatorRow := 0.U + when(reductionTile + 1.U < reductionTileCount) { + reductionTile := reductionTile + 1.U + aRowsRequested := 0.U + aRowsStored := 0.U + bRowsRequested := 0.U + bRowsStored := 0.U + state := loadTile + }.elsewhen(kGroup + 1.U < kGroups) { + kGroup := kGroup + 1.U + reductionTile := 0.U + aRowsRequested := 0.U + aRowsStored := 0.U + bRowsRequested := 0.U + bRowsStored := 0.U + state := loadTile + }.elsewhen(outputTile + 1.U < outputTileCount) { + outputTile := outputTile + 1.U + kGroup := 0.U + reductionTile := 0.U + aRowsRequested := 0.U + aRowsStored := 0.U + bRowsRequested := 0.U + bRowsStored := 0.U + state := loadTile + }.otherwise { + when(lastBlock) { + outputTile := 0.U + resultRow := 0.U + state := readResult + }.otherwise { + state := complete + } + } + }.otherwise { + accumulatorRow := accumulatorRow + 1.U + state := readAccumulator + } + } + + when(state === readResult) { + outputWord := 0.U + state := holdResult + } + when(state === holdResult) { + cRowData := accumulatorData + state := writeResult + } + when(state === writeResult && io.bankWrite(0).io.req.fire) { + state := waitForCWrite + } + when(state === waitForCWrite && io.bankWrite(0).io.resp.fire) { + when(outputWord =/= (resultWords - 1).U) { + outputWord := outputWord + 1.U + state := writeResult + }.elsewhen(resultRow =/= Mux(vectorMode, 0.U, (rows - 1).U)) { + resultRow := resultRow + 1.U + state := readResult + }.elsewhen(outputTile + 1.U < outputTileCount) { + outputTile := outputTile + 1.U + resultRow := 0.U + state := readResult + }.otherwise { + chainLive := false.B + state := complete + } + } + + when(state === complete && io.cmdResp.fire) { + state := idle + } +} diff --git a/examples/prefab/balls/smatmul/configs/SMatMulBallParam.scala b/examples/prefab/balls/smatmul/configs/SMatMulBallParam.scala new file mode 100644 index 000000000..84b1791c2 --- /dev/null +++ b/examples/prefab/balls/smatmul/configs/SMatMulBallParam.scala @@ -0,0 +1,20 @@ +package examples.balls.smatmul.configs + +import buckyball.balls.smatmul.{Params => ParametersProto} +import upickle.default.{macroRW, ReadWriter} + +case class SMatMulBallParam( + tileRows: Int, + tileCols: Int) + +object SMatMulBallParam { + + def fromProto(source: ParametersProto): SMatMulBallParam = { + val parameters = source + require(parameters.hasTileRows && parameters.hasTileCols) + val param = SMatMulBallParam(parameters.getTileRows, parameters.getTileCols) + param + } + + implicit val rw: ReadWriter[SMatMulBallParam] = macroRW +} diff --git a/examples/prefab/balls/smatmul/configs/default.toml b/examples/prefab/balls/smatmul/configs/default.toml new file mode 100644 index 000000000..db3557dc7 --- /dev/null +++ b/examples/prefab/balls/smatmul/configs/default.toml @@ -0,0 +1,4 @@ +# SMatMul Ball Configuration + +tileRows = 16 +tileCols = 16 diff --git a/examples/prefab/balls/smatmul/configs/lane16.toml b/examples/prefab/balls/smatmul/configs/lane16.toml new file mode 100644 index 000000000..d4cc1e6ea --- /dev/null +++ b/examples/prefab/balls/smatmul/configs/lane16.toml @@ -0,0 +1,4 @@ +# SMatMul Ball Configuration + +tileRows = 8 +tileCols = 16 diff --git a/examples/prefab/balls/smatmul/configs/smatmul_params.proto b/examples/prefab/balls/smatmul/configs/smatmul_params.proto new file mode 100644 index 000000000..299bbd328 --- /dev/null +++ b/examples/prefab/balls/smatmul/configs/smatmul_params.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; +package buckyball.balls.smatmul; +option java_package = "buckyball.balls.smatmul"; +option java_multiple_files = true; + +message Params { + optional int32 tileRows = 1; + optional int32 tileCols = 2; +} diff --git a/examples/prefab/balls/smatmul/emu/src/dispatch.rs b/examples/prefab/balls/smatmul/emu/src/dispatch.rs new file mode 100644 index 000000000..4d832ba86 --- /dev/null +++ b/examples/prefab/balls/smatmul/emu/src/dispatch.rs @@ -0,0 +1,38 @@ +use bebop_bemu::thread::instructions::decode; +use bebop_bemu::ball::ExecContext; + +use super::{fp32, mxfp8, smatmul}; + +const BALL_CLASS: &str = "examples.balls.smatmul.SMatMulBall"; + +pub fn execute_known( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, + ctx: &mut ExecContext, +) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("SMATMUL_OS") => Some(smatmul::exec_smatmul(xs1, xs2, ctx)), + Some("SMATMUL_MXFP8") => Some(mxfp8::execute(xs1, xs2, ctx)), + Some("SMATMUL_F32") => Some(fp32::execute(xs1, xs2, ctx)), + Some("SMATMUL_BIAS") => Some(smatmul::exec_bias(xs1, xs2, ctx)), + _ => None, + } +} + +pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("SMATMUL_OS") => Some(smatmul::latency(xs1, xs2, bebop_bemu::configs::ball_domain::instance_for_funct(funct).expect("Ball instance"))), + Some("SMATMUL_MXFP8") => Some(smatmul::latency(xs1, xs2, bebop_bemu::configs::ball_domain::instance_for_funct(funct).expect("Ball instance"))), + Some("SMATMUL_F32") => Some(decode::rs1_iter(xs1) as u64 + (xs2 & 0xfff) + ((xs2 >> 12) & 0xfff)), + Some("SMATMUL_BIAS") => Some(smatmul::bias_latency(xs1, xs2)), + _ => None, + } +} diff --git a/examples/prefab/balls/smatmul/emu/src/fp32.rs b/examples/prefab/balls/smatmul/emu/src/fp32.rs new file mode 100644 index 000000000..b731ba17b --- /dev/null +++ b/examples/prefab/balls/smatmul/emu/src/fp32.rs @@ -0,0 +1,119 @@ +use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; +use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old}; +use super::instruction::ExecContext; +use std::collections::HashMap; + +struct Chain { + rows: usize, + bank: u64, + base: usize, + values: Vec, +} + + + +pub(crate) fn execute(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + assert!(!rs1_read_old(xs1), "smatmul_f32: reads no old destination; rs1[29] must be clear"); + let ball_state = ctx.state::>(); + let rows = (xs2 & 0xfff) as usize; + let cols = ((xs2 >> 12) & 0xfff) as usize; + let k = rs1_iter(xs1) as usize; + let first = (xs2 >> 24) & 1 != 0; + let last = (xs2 >> 25) & 1 != 0; + let base = ((xs2 >> 26) & 63) as usize; + let a = rs1_b0(xs1); + let b = rs1_b1(xs1); + let c = rs1_b2(xs1); + assert!( + xs2 >> 32 == 0 && (rows == 1 || rows == 16) && cols == 16 && k > 0 && k % 4 == 0, + "f32 matmul: requires M=1/16, N=16, positive K multiple of four and zero reserved bits" + ); + assert!(a != b && b != c && a != c, "f32 matmul: banks must differ"); + for bank in [a, b, c] { + assert!( + ctx.config(bank).allocated && ctx.config(bank).cols == 1, + "f32 matmul: banks must be allocated with one column" + ); + } + let stride = bank_row_bytes(); + assert!( + stride == 16 + && rows * k * 4 <= bank_lines() * stride + && cols * k * 4 <= bank_lines() * stride + && base * stride + rows * cols * 4 <= bank_lines() * stride, + "f32 matmul: bank footprint exceeds depth" + ); + let pa = pbank(ctx, a); + let pb = pbank(ctx, b); + let av = &ctx.banks[pa][..rows * k * 4]; + let bv = &ctx.banks[pb][..cols * k * 4]; + assert!( + av.chunks_exact(4) + .chain(bv.chunks_exact(4)) + .all(|x| f32::from_le_bytes(x.try_into().unwrap()).is_finite()), + "f32 matmul: non-finite operand" + ); + let mut chain = { + let mut chains = ball_state.lock().unwrap(); + if first { + assert!( + !chains.contains_key(&ctx.hart_id), + "f32 matmul: first while chain is live" + ); + Chain { + rows, + bank: c, + base, + values: vec![0f32; rows * cols], + } + } else { + chains + .remove(&ctx.hart_id) + .expect("f32 matmul: continuation without first") + } + }; + assert!( + chain.rows == rows && chain.bank == c && chain.base == base, + "f32 matmul: continuation changed destination" + ); + let weights: [&[u8]; 16] = std::array::from_fn(|col| &bv[col * k * 4..(col + 1) * k * 4]); + for row in 0..rows { + if av[row * k * 4..(row + 1) * k * 4] + .chunks_exact(4) + .all(|x| u32::from_le_bytes(x.try_into().unwrap()) & 0x7fffffff == 0) + && chain.values[row * cols..(row + 1) * cols] + .iter() + .all(|x| x.to_bits() == 0) + { + continue; + } + let av = &av[row * k * 4..(row + 1) * k * 4]; + let dst = &mut chain.values[row * cols..(row + 1) * cols]; + let mut sums: [f32; 16] = dst.try_into().unwrap(); + for inner in 0..k { + let a = f32::from_le_bytes(av[inner * 4..inner * 4 + 4].try_into().unwrap()); + for col in 0..16 { + let b = + f32::from_le_bytes(weights[col][inner * 4..inner * 4 + 4].try_into().unwrap()); + sums[col] = a.mul_add(b, sums[col]); + } + } + assert!( + sums.iter().all(|x| x.is_finite()), + "f32 matmul: accumulator overflow" + ); + dst.copy_from_slice(&sums); + } + if last { + let pc = pbank(ctx, c); + for (index, value) in chain.values.iter().enumerate() { + let offset = base * stride + index * 4; + ctx.banks[pc][offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + } + } else { + { + ball_state.lock().unwrap().insert(ctx.hart_id, chain); + }; + } + 0 +} diff --git a/examples/prefab/balls/smatmul/emu/src/lib.rs b/examples/prefab/balls/smatmul/emu/src/lib.rs new file mode 100644 index 000000000..58a903abf --- /dev/null +++ b/examples/prefab/balls/smatmul/emu/src/lib.rs @@ -0,0 +1,13 @@ +pub(crate) use bebop_bemu::{thread::instructions::decode, ball as instruction}; + +mod fp32; +mod mxfp8; +mod smatmul; + +mod dispatch; + +pub use dispatch::{cycles_after_issue, execute_known}; + +#[bebop_bemu::ball::linkme::distributed_slice(bebop_bemu::BALLS)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static BALL: bebop_bemu::Ball = bebop_bemu::Ball { execute: execute_known, cycles: cycles_after_issue }; diff --git a/examples/prefab/balls/smatmul/emu/src/mxfp8.rs b/examples/prefab/balls/smatmul/emu/src/mxfp8.rs new file mode 100644 index 000000000..1cc211375 --- /dev/null +++ b/examples/prefab/balls/smatmul/emu/src/mxfp8.rs @@ -0,0 +1,164 @@ +use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; +use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old}; +use super::instruction::ExecContext; +use std::collections::HashMap; + +struct Chain { + rows: usize, + cols: usize, + bank: u64, + base: usize, + values: Vec, +} + + + +fn value(code: u8, factor: f32) -> f32 { + let exponent = (code >> 3) & 15; + let mantissa = (code & 7) as u32; + let normal = ((exponent as u32 + 120) << 23) | (mantissa << 20); + let subnormal = (mantissa as f32 / 512.0).to_bits(); + let element = f32::from_bits(if exponent == 0 { subnormal } else { normal }); + let result = f32::from_bits((element * factor).to_bits() | ((code as u32 & 128) << 24)); + result +} + +pub(crate) fn execute(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + assert!(!rs1_read_old(xs1), "smatmul_mxfp8: reads no old destination; rs1[29] must be clear"); + let ball_state = ctx.state::>(); + let rows = (xs2 & 0xfff) as usize; + let cols = ((xs2 >> 12) & 0xfff) as usize; + let k = rs1_iter(xs1) as usize; + let first = (xs2 >> 24) & 1 != 0; + let last = (xs2 >> 25) & 1 != 0; + let base = ((xs2 >> 26) & 63) as usize; + let a = rs1_b0(xs1); + let b = rs1_b1(xs1); + let c = rs1_b2(xs1); + assert!( + xs2 >> 32 == 0 && (rows == 1 || rows == 16) && cols == 16 && k > 0 && k % 32 == 0, + "mxfp8: requires M=1/16, N=16, K multiple of 32 and zero reserved bits" + ); + assert!(a != b && b != c && a != c, "mxfp8: banks must differ"); + for bank in [a, b, c] { + assert!( + ctx.config(bank).allocated && ctx.config(bank).cols == 1, + "mxfp8: banks must be allocated with one column" + ); + } + let stride = bank_row_bytes(); + assert!(stride == 16, "mxfp8: requires 16-byte bank rows"); + assert!( + rows * k + rows * k / 32 <= bank_lines() * stride + && cols * k + cols * k / 32 <= bank_lines() * stride + && base * stride + rows * cols * 4 <= bank_lines() * stride, + "mxfp8: bank footprint exceeds depth" + ); + let pa = pbank(ctx, a); + let pb = pbank(ctx, b); + let a_bytes = &ctx.banks[pa][..rows * k + rows * k / 32]; + let b_bytes = &ctx.banks[pb][..cols * k + cols * k / 32]; + assert!( + a_bytes[..rows * k] + .iter() + .chain(b_bytes[..cols * k].iter()) + .fold(true, |valid, code| valid & (code & 0x7f != 0x7f)), + "mxfp8: NaN encoding" + ); + let mut av = vec![0f32; rows * k]; + let mut bv = vec![0f32; cols * k]; + for (block, dst) in av.chunks_exact_mut(32).enumerate() { + let scale = a_bytes[rows * k + block]; + assert!(scale != 255, "mxfp8: NaN encoding"); + let factor = if scale == 0 { + f32::from_bits(1 << 22) + } else { + f32::from_bits((scale as u32) << 23) + }; + for (index, decoded) in dst.iter_mut().enumerate() { + *decoded = value(a_bytes[block * 32 + index], factor); + } + } + for col in 0..cols { + for block in 0..k / 32 { + let scale = b_bytes[cols * k + col * (k / 32) + block]; + assert!(scale != 255, "mxfp8: NaN encoding"); + let factor = if scale == 0 { + f32::from_bits(1 << 22) + } else { + f32::from_bits((scale as u32) << 23) + }; + let codes = &b_bytes[col * k + block * 32..col * k + (block + 1) * 32]; + let mut decoded = [0f32; 32]; + for (decoded, code) in decoded.iter_mut().zip(codes.iter()) { + *decoded = value(*code, factor); + } + for (index, decoded) in decoded.iter().enumerate() { + bv[(block * 32 + index) * cols + col] = *decoded; + } + } + } + assert!( + av.iter() + .chain(bv.iter()) + .fold(true, |valid, value| valid & value.is_finite()), + "mxfp8: decoded value overflows FP32" + ); + let mut chain = { + let mut chains = ball_state.lock().unwrap(); + if first { + assert!( + !chains.contains_key(&ctx.hart_id), + "mxfp8: first while chain is live" + ); + Chain { + rows, + cols, + bank: c, + base, + values: vec![0f32; rows * cols], + } + } else { + chains + .remove(&ctx.hart_id) + .expect("mxfp8: continuation without first") + } + }; + assert!( + chain.rows == rows && chain.cols == cols && chain.bank == c && chain.base == base, + "mxfp8: continuation changed destination" + ); + for row in 0..rows { + if av[row * k..(row + 1) * k].iter().all(|value| *value == 0.0) + && chain.values[row * cols..(row + 1) * cols] + .iter() + .all(|value| value.to_bits() == 0) + { + continue; + } + let av = &av[row * k..(row + 1) * k]; + let dst = &mut chain.values[row * cols..(row + 1) * cols]; + let mut sums: [f32; 16] = dst.try_into().unwrap(); + for inner in 0..k { + let a = av[inner]; + let weights = &bv[inner * 16..(inner + 1) * 16]; + for col in 0..16 { + sums[col] = a.mul_add(weights[col], sums[col]); + } + } + assert!(sums.iter().all(|v| v.is_finite()), "accumulator overflow"); + dst.copy_from_slice(&sums); + } + if last { + let pc = pbank(ctx, c); + for (i, value) in chain.values.iter().enumerate() { + ctx.banks[pc][base * stride + i * 4..base * stride + i * 4 + 4] + .copy_from_slice(&value.to_le_bytes()); + } + } else { + { + ball_state.lock().unwrap().insert(ctx.hart_id, chain); + }; + } + 0 +} diff --git a/examples/prefab/balls/smatmul/emu/src/smatmul.rs b/examples/prefab/balls/smatmul/emu/src/smatmul.rs new file mode 100644 index 000000000..34101a48b --- /dev/null +++ b/examples/prefab/balls/smatmul/emu/src/smatmul.rs @@ -0,0 +1,256 @@ +use super::decode::{pbank, pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old}; +use super::instruction::ExecContext; +use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; +use std::collections::HashMap; + +fn tile_rows(instance: u32) -> usize { + bebop_bemu::configs::ball_domain::parameters::< + bebop_bemu::configs::ball_domain::parameters::smatmul::Params, + >("smatmul", instance) + .tile_rows + .expect("tileRows") as usize +} +fn tile_cols(instance: u32) -> usize { + bebop_bemu::configs::ball_domain::parameters::< + bebop_bemu::configs::ball_domain::parameters::smatmul::Params, + >("smatmul", instance) + .tile_cols + .expect("tileCols") as usize +} + +#[derive(Default)] +struct State { + bias: Option>, + chain: Option, +} + +struct Chain { + rows: usize, + cols: usize, + output_bank: u64, + output_base: usize, + accumulators: Vec, +} + +fn read_i32(bank: &[u8], row: usize, lane: usize) -> i32 { + let offset = row * bank_row_bytes() + lane * 4; + i32::from_le_bytes(bank[offset..offset + 4].try_into().unwrap()) +} + +pub(crate) fn exec_bias(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + assert!(!rs1_read_old(xs1), "smatmul_bias: reads no old destination; rs1[29] must be clear"); + let ball_state = ctx.state::>(); + let bank = rs1_b0(xs1); + let input_base = (xs2 & 0x3f) as usize; + if rs1_b1(xs1) != 0 || rs1_b2(xs1) != 0 || rs1_iter(xs1) != 4 || xs2 >> 6 != 0 { + panic!("smatmul_bias: bank1/bank2/rs2[63:6] must be zero and iter must be four"); + } + if !ctx.config(bank).allocated || ctx.config(bank).cols != 1 { + panic!("smatmul_bias: bias bank must be one allocated column"); + } + if input_base + 4 > bank_lines() { + panic!("smatmul_bias: inputBase plus four rows exceeds bank depth"); + } + let physical = pbank(ctx, bank); + let mut bias = vec![0i32; tile_cols(ctx.ball_id.expect("Ball instance"))]; + for group in 0..4 { + for lane in 0..4 { + bias[group * 4 + lane] = read_i32(&ctx.banks[physical], input_base + group, lane); + } + } + { + let mut states = ball_state.lock().unwrap(); + let state = states.entry(ctx.hart_id).or_default(); + if state.chain.is_some() { + panic!("smatmul_bias: cannot replace bias during an accumulation chain"); + } + state.bias = Some(bias); + }; + 0 +} + +pub(crate) fn exec_smatmul(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + assert!(!rs1_read_old(xs1), "smatmul: reads no old destination; rs1[29] must be clear"); + let ball_state = ctx.state::>(); + let m_tile = tile_rows(ctx.ball_id.expect("Ball instance")); + let k_tile = tile_cols(ctx.ball_id.expect("Ball instance")); + let n_tile = tile_cols(ctx.ball_id.expect("Ball instance")); + let a_bank = rs1_b0(xs1); + let b_bank = rs1_b1(xs1); + let c_bank = rs1_b2(xs1); + let rows = (xs2 & 0xfff) as usize; + let cols = ((xs2 >> 12) & 0xfff) as usize; + let k = rs1_iter(xs1) as usize; + let first = (xs2 >> 24) & 1 != 0; + let last = (xs2 >> 25) & 1 != 0; + let output_base = ((xs2 >> 26) & 0x3f) as usize; + if xs2 >> 32 != 0 { + panic!("smatmul: rs2[63:32] must be zero"); + } + if (rows != 1 && (rows == 0 || rows % m_tile != 0)) + || cols != n_tile + || k == 0 + || k % k_tile != 0 + { + panic!("smatmul: M must be one or a positive multiple of 8, K must be a positive multiple of 16, and N must be 16"); + } + if a_bank == b_bank || a_bank == c_bank || b_bank == c_bank { + panic!("smatmul: A, B, and C banks must differ"); + } + if !ctx.config(a_bank).allocated + || !ctx.config(b_bank).allocated + || !ctx.config(c_bank).allocated + { + panic!("smatmul: bank not allocated"); + } + // A and B group g hold reduction block g (K = iter per group); C rows + // continue across groups: line L lives in group L / bank_lines(). + let k_groups = ctx.config(a_bank).cols as usize; + let c_groups = ctx.config(c_bank).cols as usize; + if k_groups == 0 || ctx.config(b_bank).cols as usize != k_groups || c_groups == 0 { + panic!("smatmul: A and B must have the same reduction groups"); + } + if rows > (bank_lines() / 4).max(32) { + panic!("smatmul: M exceeds the accumulator"); + } + + let a_rows = if rows == 1 { + k / k_tile + } else { + rows * k / k_tile + }; + let b_rows = k; + let c_rows = rows * 4; + if a_rows > bank_lines() + || b_rows > bank_lines() + || output_base + c_rows > c_groups * bank_lines() + { + panic!("smatmul: bank footprint exceeds bank depth"); + } + + let mut chain = { + let mut states = ball_state.lock().unwrap(); + let state = states.entry(ctx.hart_id).or_default(); + if first { + if state.chain.is_some() { + panic!("smatmul: first block issued while another chain is live"); + } + let bias = state + .bias + .as_ref() + .unwrap_or_else(|| panic!("smatmul: bias must be preloaded before first block")); + let mut accumulators = vec![0i32; rows * cols]; + for row in 0..rows { + accumulators[row * cols..(row + 1) * cols].copy_from_slice(bias); + } + Chain { + rows, + cols, + output_bank: c_bank, + output_base, + accumulators, + } + } else { + state + .chain + .take() + .unwrap_or_else(|| panic!("smatmul: continuation has no live chain")) + } + }; + if chain.rows != rows + || chain.cols != cols + || chain.output_bank != c_bank + || chain.output_base != output_base + { + panic!("smatmul: continuation changed output shape or destination"); + } + + let k_tiles = k / k_tile; + let row_bytes = bank_row_bytes(); + + // Keep the hardware-visible operation unchanged, but traverse the data in + // the layout used by the banks. The old col->k loop repeatedly computed + // addresses and reread the same B row for every output lane. This order + // reads one A byte and one complete B row, then updates all 16 lanes. + // Wrapping arithmetic is intentionally retained to match the accumulator + // semantics of the RTL and the previous emulator implementation. + for k_group in 0..k_groups { + let pa = pbank_group(ctx, a_bank, k_group as u64); + let pb = pbank_group(ctx, b_bank, k_group as u64); + for inner in 0..k { + let b_base = ((inner / k_tile) * k_tile + inner % k_tile) * row_bytes; + let b_row = &ctx.banks[pb][b_base..b_base + cols]; + let mut b_values = vec![0i32; n_tile]; + for col in 0..cols { + b_values[col] = b_row[col] as i8 as i32; + } + for row in 0..rows { + let a_row = if rows == 1 { + inner / k_tile + } else { + ((row / m_tile) * k_tiles + inner / k_tile) * m_tile + row % m_tile + }; + let a = ctx.banks[pa][a_row * row_bytes + inner % k_tile] as i8 as i32; + let acc_base = row * cols; + let acc_row = &mut chain.accumulators[acc_base..acc_base + cols]; + for col in 0..cols { + acc_row[col] = acc_row[col].wrapping_add(a.wrapping_mul(b_values[col])); + } + } + } + } + + if last { + for row in 0..rows { + for col in 0..cols { + let line = output_base + row * 4 + col / 4; + let pc = pbank_group(ctx, c_bank, (line / bank_lines()) as u64); + let offset = line % bank_lines() * bank_row_bytes() + col % 4 * 4; + ctx.banks[pc][offset..offset + 4] + .copy_from_slice(&chain.accumulators[row * cols + col].to_le_bytes()); + } + } + } else { + { + ball_state + .lock() + .unwrap() + .entry(ctx.hart_id) + .or_default() + .chain = Some(chain); + }; + } + 0 +} + +pub(crate) fn bias_latency(xs1: u64, xs2: u64) -> u64 { + let input_base = xs2 & 0x3f; + if rs1_b1(xs1) != 0 + || rs1_b2(xs1) != 0 + || rs1_iter(xs1) != 4 + || xs2 >> 6 != 0 + || input_base + 4 > bank_lines() as u64 + { + panic!("smatmul_bias: illegal inputBase or reserved fields"); + } + 4 +} + +pub(crate) fn latency(xs1: u64, xs2: u64, instance: u32) -> u64 { + let m_tile = tile_rows(instance); + let k_tile = tile_cols(instance); + let n_tile = tile_cols(instance); + let rows = xs2 & 0xfff; + let cols = (xs2 >> 12) & 0xfff; + let k = rs1_iter(xs1); + let output_base = (xs2 >> 26) & 0x3f; + if xs2 >> 32 != 0 + || (rows != 1 && (rows == 0 || rows % m_tile as u64 != 0)) + || cols != n_tile as u64 + || k == 0 + || k % k_tile as u64 != 0 + { + panic!("smatmul: illegal matrix encoding"); + } + rows * cols * k / m_tile as u64 + if (xs2 >> 25) & 1 != 0 { rows * 4 } else { 0 } +} diff --git a/examples/prefab/balls/smatmul/stack/src/smatmulOps.cpp b/examples/prefab/balls/smatmul/stack/src/smatmulOps.cpp new file mode 100644 index 000000000..c580ee03f --- /dev/null +++ b/examples/prefab/balls/smatmul/stack/src/smatmulOps.cpp @@ -0,0 +1,80 @@ +//===- smatmulOps.cpp - SMatMulBall operations ----------------------------===// +// +// The dialect of SMatMulBall, its virtual operations building their physical +// ones and the instructions of the physical operations. +// +//===----------------------------------------------------------------------===// + +#include "smatmulOps.h" + +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::smatmul_ball; +using mlir::buddy::bankFields; +using mlir::buddy::constantI64; +using mlir::buddy::emitInstruction; +using mlir::buddy::field; + +#include "smatmulDialect.cpp.inc" + +#define GET_OP_CLASSES +#include "smatmulOps.cpp.inc" + +void SMatMulBallDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "smatmulOps.cpp.inc" + >(); +} + +namespace { +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallDialect([](DialectRegistry ®istry) { + registry.insert(); + }), + true); +} // namespace + +SmallVector BankMatmulI8Op::getSourceBanks() { + return {getA(), getB()}; +} +Value BankMatmulI8Op::getDestinationBank() { return getC(); } +bool BankMatmulI8Op::readsDestination() { return true; } +Operation *BankMatmulI8Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return MatmulI8Op::create(builder, getLoc(), sources[0], sources[1], + destination, getRows(), getK()); +} + +SmallVector MatmulI8Op::getSourceBanks() { + return {int64_t(getA()), int64_t(getB())}; +} +int64_t MatmulI8Op::getDestinationBank() { return getC(); } +LogicalResult MatmulI8Op::lowerToInstructions(OpBuilder &builder) { + Location loc = getLoc(); + int64_t rows = getRows(), k = getK(), a = getA(), b = getB(), c = getC(); + const auto &target = buckyball_target::getBuckyballTarget(); + int64_t tileRows = + buckyball_target::getBuckyballBallParam("SMATMUL_OS", "tileRows"); + if (target.bankWidthBits != 128 || rows <= 0 || rows * 4 > target.bankDepth || + (rows != 1 && rows % tileRows) || k <= 0 || k > target.bankDepth || + k % 16 || rows * (k / 16) > target.bankDepth) + return emitOpError("INT8 matmul shape exceeds the bank or ISA geometry"); + if (a == b || a == c || b == c) + return emitOpError("INT8 matmul requires distinct A, B and C banks"); + emitInstruction(builder, loc, + buckyball_target::getBuckyballFunct7("SMATMUL_BIAS"), + constantI64(builder, loc, bankFields({c}, 0, false, 4)), + constantI64(builder, loc, 0)); + // One accumulation over every group: first and last in one instruction. + uint64_t rs2 = field(rows, 0, 11) | field(16, 12, 23) | field(1, 24, 24) | + field(1, 25, 25); + emitInstruction(builder, loc, + buckyball_target::getBuckyballFunct7("SMATMUL_OS"), + constantI64(builder, loc, bankFields({a, b}, c, false, k)), + constantI64(builder, loc, rs2)); + return success(); +} diff --git a/examples/prefab/balls/smatmul/stack/src/smatmulOps.h b/examples/prefab/balls/smatmul/stack/src/smatmulOps.h new file mode 100644 index 000000000..05a631588 --- /dev/null +++ b/examples/prefab/balls/smatmul/stack/src/smatmulOps.h @@ -0,0 +1,20 @@ +//===- smatmulOps.h - SMatMulBall operations ------------------------------===// + +#ifndef SMATMUL_OPS_H +#define SMATMUL_OPS_H + +#include "Interfaces/BankInterfaces.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "smatmulDialect.h.inc" + +#define GET_OP_CLASSES +#include "smatmulOps.h.inc" + +#endif // SMATMUL_OPS_H diff --git a/examples/prefab/balls/smatmul/stack/src/smatmulOps.td b/examples/prefab/balls/smatmul/stack/src/smatmulOps.td new file mode 100644 index 000000000..d457e9931 --- /dev/null +++ b/examples/prefab/balls/smatmul/stack/src/smatmulOps.td @@ -0,0 +1,49 @@ +//===- smatmulOps.td - SMatMulBall operations --------------*- tablegen -*-===// +// +// matmul_i8 (a, b) -> c: c = bias + a @ b for `rows` rows (one, or a multiple +// of tileRows up to a quarter of the bank depth) and sixteen columns; it always +// reads the bias in the old c. A and B have the same number of groups; group g +// holds reduction block g of `k` columns (a multiple of sixteen), so +// K = k * groups. In group g, B holds k rows of sixteen i8. A holds k / 16 rows +// of sixteen i8 for rows = 1; otherwise M tiles of tileRows rows, each tile the +// k / 16 slices of sixteen columns one after another: line +// ((r / tileRows) * (k / 16) + c / 16) * tileRows + r % tileRows. +// C starts with the bias, one row of sixteen i32 in its first four lines, and +// receives `rows` rows of sixteen i32 (four lines each). +// +//===----------------------------------------------------------------------===// + +#ifndef SMATMUL_OPS +#define SMATMUL_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Interfaces/BankInterfaces.td" + +def SMatMulBall_Dialect : Dialect { + let name = "smatmul_ball"; + let cppNamespace = "::buddy::smatmul_ball"; + let summary = "The operations of SMatMulBall"; +} + +class SMatMulBall_Op traits = []> + : Op])>; + +def BankMatmulI8Op : SMatMulBall_Op<"bank_matmul_i8", + [DeclareOpInterfaceMethods]> { + let summary = "matmul_i8 on virtual banks"; + let arguments = (ins I64:$a, I64:$b, I64:$c, I64Attr:$rows, I64Attr:$k); + let results = (outs I64:$cOut); + let assemblyFormat = "$a `,` $b `->` $c prop-dict attr-dict"; +} + +def MatmulI8Op : SMatMulBall_Op<"matmul_i8", + [DeclareOpInterfaceMethods]> { + let summary = "matmul_i8 on physical banks"; + let arguments = (ins I64Attr:$a, I64Attr:$b, I64Attr:$c, I64Attr:$rows, + I64Attr:$k); + let assemblyFormat = "prop-dict attr-dict"; +} + +#endif // SMATMUL_OPS diff --git a/examples/prefab/balls/smatmul/verify/README.md b/examples/prefab/balls/smatmul/verify/README.md new file mode 100644 index 000000000..1687af818 --- /dev/null +++ b/examples/prefab/balls/smatmul/verify/README.md @@ -0,0 +1,81 @@ +# SMatMulBall UVM Verification + +Verifies the generated `SMatMulBall` module using the shared Blink UVM framework. + +## Structure + +- `../../../../verify/uvm/src/ball/bb_uvm_pkg.sv`: common Blink UVM agents and base env +- `src/common/smatmul_defs.svh`: DPI imports, `matrix_require_bid`, timeouts +- `src/common/smatmul_items.svh`: `smatmul_cmd_item` with `load_rust_case` +- `src/seq/smatmul_sequences.svh`: one-case sequence +- `src/cov/smatmul_cov.svh`: M/N/K shape bins `{1,2,4,8,16}` +- `src/env/smatmul_scoreboard.svh`: preload A/B, compare writes vs DPI expected list +- `src/env/smatmul_env.svh`: extends `bb_blink_env#(2,4)` +- `src/tests/smatmul_test.svh`: directed 0..3, random 4..23 +- `src/pkg/smatmul_pkg.sv`: package entry +- `src/tb_top.sv`: `bb_blink_if#(2,2)` + `SMatMulBall` (2 read, 2 write ports) +- `filelists/smatmul_ball.f`: VCS filelist + +## Test plan + +`+UVM_TESTNAME=smatmul_ball_test` runs: + +- case 0: 4x4x4 OS (banks 0,1,2) +- case 1: 5x7x3 OS tail +- case 2: 16x16x16 OS +- case 3: 32x16x16 WS +- cases 4..23: random M in {1,2,4,8,16,32}, N/K in {1,2,4,8,16}, distinct banks 0..7 + +Each case: reset, preload A/B into mem model, drive one command, wait for +`scb.done()` or timeout at 100000 cycles. Scoreboard compares writes +(group/addr/data/mask) in DPI emission order. + +## BID (required) + +`+BID=` is mandatory. Missing it fatals at test start. No default bid. + +| Config | Plusarg | +|--------|---------| +| `sims.verilator.BuckyballPebbleVerilatorConfig` | `+BID=1` | +| `sims.verilator.BuckyballToyVerilatorConfig` | `+BID=4` | + +```console +./simv +UVM_TESTNAME=smatmul_ball_test +BID=4 +``` + +## Build + +```console +nix develop ../../../../verify +cargo build --manifest-path casegen/Cargo.toml +``` + +Compile from this directory: + +```console +vcs -full64 -sverilog -timescale=1ns/1ps \ + $VCS_UVM_ARGS \ + -sv_lib ../output/casegen/debug/libmatrix_casegen \ + -f filelists/smatmul_ball.f +``` + +Run: + +```console +./simv +UVM_TESTNAME=smatmul_ball_test +BID=4 +``` + +Or via bbdev: + +```console +bbdev uvm --build '--ball=matrix' +bbdev uvm --run '--ball=matrix' -- '+BID=4' +``` + +## Acceptance + +- [x] Layout matches Blink common + ball-local split; filelist uses `@UVM@` / `@RTL@` +- [x] casegen whole-case DPI API; `cargo test` in casegen +- [x] VCS build with `smatmul_ball.f` (no segfault on Instantiate DUT) +- [x] `+BID=` required; pebble `+BID=1` / toy `+BID=4` +- [ ] 23/23 functional pass diff --git a/examples/balls/smatmul/verify/casegen/Cargo.lock b/examples/prefab/balls/smatmul/verify/casegen/Cargo.lock similarity index 100% rename from examples/balls/smatmul/verify/casegen/Cargo.lock rename to examples/prefab/balls/smatmul/verify/casegen/Cargo.lock diff --git a/examples/balls/smatmul/verify/casegen/Cargo.toml b/examples/prefab/balls/smatmul/verify/casegen/Cargo.toml similarity index 100% rename from examples/balls/smatmul/verify/casegen/Cargo.toml rename to examples/prefab/balls/smatmul/verify/casegen/Cargo.toml diff --git a/examples/prefab/balls/smatmul/verify/casegen/src/casegen.rs b/examples/prefab/balls/smatmul/verify/casegen/src/casegen.rs new file mode 100644 index 000000000..02f97a771 --- /dev/null +++ b/examples/prefab/balls/smatmul/verify/casegen/src/casegen.rs @@ -0,0 +1,283 @@ +use crate::model; + +pub const BIAS_BANK: u32 = 0; +pub const C_BANK: u32 = 5; + +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct MatrixCmd { + pub kind: u32, + pub rob_id: u32, + pub op1_words: u32, + pub op2_words: u32, + pub rs1_lo: u32, + pub rs1_hi: u32, + pub rs2_lo: u32, + pub rs2_hi: u32, + pub op1_group: u32, + pub op2_group: u32, + pub wr_group: u32, + pub k_groups: u32, + pub wr_col: u32, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MatrixCase { + pub commands: Vec, + pub bias: Vec, + // a[block] / b[block] hold every K group of the block, group-major. + pub a: Vec>, + pub b: Vec>, + pub writes: Vec, +} + +// Random cases also place every operand at a nonzero group base of its bank. +pub fn gen_case(seed: u32, index: u32, tile_rows: usize) -> MatrixCase { + match index { + 0 => build_case(seed, 16, 1, tile_rows, false), + 1 => build_case(seed, 16, 2, tile_rows, false), + 2 => build_case(seed ^ index, 16, 1, tile_rows, false), + 11 => three_k_groups(seed, tile_rows), + 12 => c_crosses_groups(seed, tile_rows), + _ => { + let mut rng = Rng::new(seed, index); + let blocks = 1 + (rng.next() % 2) as usize; + build_case(rng.next(), 16, blocks, tile_rows, true) + } + } +} + +fn build_case( + seed: u32, + rows: usize, + blocks: usize, + tile_rows: usize, + grouped: bool, +) -> MatrixCase { + assert!(matches!(rows, 16 | 32 | 64)); + assert!(matches!(blocks, 1 | 2)); + assert!(rows * model::RESULT_WORDS <= model::BANK_DEPTH); + let mut rng = Rng::new(seed, rows as u32 ^ blocks as u32); + let bias_values: Vec = (0..model::CORE).map(|_| rng.value() as i32).collect(); + let mut bias = Vec::with_capacity(4 * model::ROW_BYTES); + for chunk in bias_values.chunks(4) { + for value in chunk { + bias.extend_from_slice(&value.to_le_bytes()); + } + } + + let mut a = Vec::with_capacity(blocks); + let mut b = Vec::with_capacity(blocks); + let mut result = vec![bias_values.clone(); rows]; + let group = |rng: &mut Rng| if grouped { 1 + rng.next() % 3 } else { 0 }; + let mut commands = Vec::with_capacity(blocks + 1); + let bias_group = group(&mut rng); + commands.push(MatrixCmd { + kind: 0, + rob_id: 0, + op1_group: bias_group, + op1_words: 4, + k_groups: 1, + rs1_lo: model::encode_rs1(BIAS_BANK, 0, 0, 4) as u32, + rs1_hi: (model::encode_rs1(BIAS_BANK, 0, 0, 4) >> 32) as u32, + ..MatrixCmd::default() + }); + + for block in 0..blocks { + let a_bank = 1 + 2 * block as u32; + let b_bank = 2 + 2 * block as u32; + let a_values: Vec = (0..rows * model::CORE).map(|_| rng.value()).collect(); + let b_values: Vec = (0..model::CORE * model::CORE) + .map(|_| rng.value()) + .collect(); + let a_packed = model::pack_a(&a_values, rows); + let b_packed = model::pack_b(&b_values); + a.push(a_packed); + b.push(b_packed); + for row in 0..rows { + for column in 0..model::CORE { + for k in 0..model::CORE { + result[row][column] = result[row][column].wrapping_add( + a_values[row * model::CORE + k] as i32 + * b_values[k * model::CORE + column] as i32, + ); + } + } + } + let rs1 = model::encode_rs1(a_bank, b_bank, C_BANK, model::CORE as u64); + let rs2 = model::encode_rs2(rows, block == 0, block + 1 == blocks, 0); + let op1_group = group(&mut rng); + let op2_group = group(&mut rng); + let wr_group = group(&mut rng); + commands.push(MatrixCmd { + kind: 1, + rob_id: (block + 1) as u32, + op1_group, + op2_group, + wr_group, + op1_words: rows as u32, + op2_words: (model::CORE * rows.div_ceil(tile_rows)) as u32, + k_groups: 1, + wr_col: 1, + rs1_lo: rs1 as u32, + rs1_hi: (rs1 >> 32) as u32, + rs2_lo: rs2 as u32, + rs2_hi: (rs2 >> 32) as u32, + }); + } + + MatrixCase { + commands, + bias, + a, + b, + writes: model::emit_writes(&result, 0), + } +} + +fn bias_command(op1_group: u32) -> MatrixCmd { + let rs1 = model::encode_rs1(BIAS_BANK, 0, 0, 4); + MatrixCmd { + kind: 0, + op1_group, + op1_words: 4, + k_groups: 1, + rs1_lo: rs1 as u32, + rs1_hi: (rs1 >> 32) as u32, + ..MatrixCmd::default() + } +} + +fn bias_rows(values: &[i32]) -> Vec { + values.iter().flat_map(|value| value.to_le_bytes()).collect() +} + +// M=16, K=48 in one command: A and B have three groups, group g holding +// K block g (16 per group), at nonzero group bases. +fn three_k_groups(seed: u32, tile_rows: usize) -> MatrixCase { + let mut rng = Rng::new(seed, 11); + let bias: Vec = (0..model::CORE).map(|_| rng.value() as i32).collect(); + let a_values: Vec = (0..16 * 48).map(|_| rng.value()).collect(); + let b_values: Vec = (0..48 * model::CORE).map(|_| rng.value()).collect(); + // Group g of A is A[:, 16g..16g+16] row-major; group g of B is rows 16g... + let mut a = Vec::new(); + for group in 0..3 { + for row in 0..16 { + for k in 0..16 { + a.push(a_values[row * 48 + group * 16 + k] as u8); + } + } + } + let b: Vec = b_values.iter().map(|value| *value as u8).collect(); + let mut result = vec![bias.clone(); 16]; + for row in 0..16 { + for column in 0..model::CORE { + for k in 0..48 { + result[row][column] = result[row][column].wrapping_add( + a_values[row * 48 + k] as i32 * b_values[k * model::CORE + column] as i32, + ); + } + } + } + let rs1 = model::encode_rs1(1, 2, C_BANK, 16); + let rs2 = model::encode_rs2(16, true, true, 0); + MatrixCase { + commands: vec![ + bias_command(0), + MatrixCmd { + kind: 1, + rob_id: 1, + op1_group: 1, + op2_group: 2, + wr_group: 3, + op1_words: 16, + op2_words: (model::CORE * 16_usize.div_ceil(tile_rows)) as u32, + k_groups: 3, + wr_col: 1, + rs1_lo: rs1 as u32, + rs1_hi: (rs1 >> 32) as u32, + rs2_lo: rs2 as u32, + rs2_hi: (rs2 >> 32) as u32, + }, + ], + bias: bias_rows(&bias), + a: vec![a], + b: vec![b], + writes: model::emit_writes(&result, 0), + } +} + +// M=128, K=32 (two K groups), output_base 32: C lines 32..543 continue from +// C group 0 into group 1 on 512-line banks. +fn c_crosses_groups(seed: u32, tile_rows: usize) -> MatrixCase { + let mut rng = Rng::new(seed, 12); + let bias: Vec = (0..model::CORE).map(|_| rng.value() as i32).collect(); + let a_values: Vec = (0..128 * 32).map(|_| rng.value()).collect(); + let b_values: Vec = (0..32 * model::CORE).map(|_| rng.value()).collect(); + let mut a = Vec::new(); + for group in 0..2 { + for row in 0..128 { + for k in 0..16 { + a.push(a_values[row * 32 + group * 16 + k] as u8); + } + } + } + let b: Vec = b_values.iter().map(|value| *value as u8).collect(); + let mut result = vec![bias.clone(); 128]; + for row in 0..128 { + for column in 0..model::CORE { + for k in 0..32 { + result[row][column] = result[row][column].wrapping_add( + a_values[row * 32 + k] as i32 * b_values[k * model::CORE + column] as i32, + ); + } + } + } + let rs1 = model::encode_rs1(1, 2, C_BANK, 16); + let rs2 = model::encode_rs2(128, true, true, 32); + MatrixCase { + commands: vec![ + bias_command(0), + MatrixCmd { + kind: 1, + rob_id: 1, + op1_group: 0, + op2_group: 0, + wr_group: 0, + op1_words: 128, + op2_words: (model::CORE * 128_usize.div_ceil(tile_rows)) as u32, + k_groups: 2, + wr_col: 2, + rs1_lo: rs1 as u32, + rs1_hi: (rs1 >> 32) as u32, + rs2_lo: rs2 as u32, + rs2_hi: (rs2 >> 32) as u32, + }, + ], + bias: bias_rows(&bias), + a: vec![a], + b: vec![b], + writes: model::emit_writes(&result, 32), + } +} + +struct Rng { + state: u64, +} + +impl Rng { + fn new(seed: u32, salt: u32) -> Self { + Self { + state: u64::from(seed) << 32 | u64::from(salt), + } + } + fn next(&mut self) -> u32 { + self.state ^= self.state << 13; + self.state ^= self.state >> 7; + self.state ^= self.state << 17; + self.state as u32 + } + fn value(&mut self) -> i8 { + (self.next() % 17) as i8 - 8 + } +} diff --git a/examples/prefab/balls/smatmul/verify/casegen/src/lib.rs b/examples/prefab/balls/smatmul/verify/casegen/src/lib.rs new file mode 100644 index 000000000..3751bd564 --- /dev/null +++ b/examples/prefab/balls/smatmul/verify/casegen/src/lib.rs @@ -0,0 +1,114 @@ +mod casegen; +mod model; + +use std::cell::RefCell; + +use casegen::{MatrixCase, MatrixCmd}; + +thread_local! { + static CURRENT: RefCell> = const { RefCell::new(None) }; +} + +#[no_mangle] +pub extern "C" fn smatmul_case_load(seed: u32, index: u32, tile_rows: u32) { + CURRENT.with(|current| { + *current.borrow_mut() = Some(casegen::gen_case(seed, index, tile_rows as usize)) + }); +} + +fn current_case R, R>(f: F) -> R { + CURRENT.with(|current| f(current.borrow().as_ref().expect("smatmul case not loaded"))) +} + +#[no_mangle] +pub extern "C" fn smatmul_case_num_commands() -> u32 { + current_case(|case| case.commands.len() as u32) +} + +#[no_mangle] +pub extern "C" fn smatmul_case_cmd(index: u32, out: *mut MatrixCmd) { + assert!(!out.is_null(), "smatmul_case_cmd: null output"); + current_case(|case| unsafe { + *out = *case + .commands + .get(index as usize) + .expect("command index out of range"); + }); +} + +#[no_mangle] +pub extern "C" fn smatmul_case_bias_words() -> u32 { + 4 +} + +#[no_mangle] +pub extern "C" fn smatmul_case_a_words(block: u32) -> u32 { + current_case(|case| (case.a[block as usize].len() / model::ROW_BYTES) as u32) +} + +#[no_mangle] +pub extern "C" fn smatmul_case_b_words(block: u32) -> u32 { + current_case(|case| (case.b[block as usize].len() / model::ROW_BYTES) as u32) +} + +fn word_halves(data: &[u8], index: u32) -> (u64, u64) { + let word = model::word(data, index as usize); + ( + u64::from_le_bytes(word[0..8].try_into().unwrap()), + u64::from_le_bytes(word[8..16].try_into().unwrap()), + ) +} + +#[no_mangle] +pub extern "C" fn smatmul_case_bias_word_lo(index: u32) -> u64 { + current_case(|case| word_halves(&case.bias, index).0) +} + +#[no_mangle] +pub extern "C" fn smatmul_case_bias_word_hi(index: u32) -> u64 { + current_case(|case| word_halves(&case.bias, index).1) +} + +#[no_mangle] +pub extern "C" fn smatmul_case_a_word_lo(block: u32, index: u32) -> u64 { + current_case(|case| word_halves(&case.a[block as usize], index).0) +} + +#[no_mangle] +pub extern "C" fn smatmul_case_a_word_hi(block: u32, index: u32) -> u64 { + current_case(|case| word_halves(&case.a[block as usize], index).1) +} + +#[no_mangle] +pub extern "C" fn smatmul_case_b_word_lo(block: u32, index: u32) -> u64 { + current_case(|case| word_halves(&case.b[block as usize], index).0) +} + +#[no_mangle] +pub extern "C" fn smatmul_case_b_word_hi(block: u32, index: u32) -> u64 { + current_case(|case| word_halves(&case.b[block as usize], index).1) +} + +#[no_mangle] +pub extern "C" fn smatmul_case_num_writes() -> u32 { + current_case(|case| case.writes.len() as u32) +} + +#[no_mangle] +pub extern "C" fn smatmul_case_write_addr(index: u32) -> u32 { + current_case(|case| case.writes[index as usize].addr) +} + +#[no_mangle] +pub extern "C" fn smatmul_case_write_data_lo(index: u32) -> u64 { + current_case(|case| { + u64::from_le_bytes(case.writes[index as usize].data[0..8].try_into().unwrap()) + }) +} + +#[no_mangle] +pub extern "C" fn smatmul_case_write_data_hi(index: u32) -> u64 { + current_case(|case| { + u64::from_le_bytes(case.writes[index as usize].data[8..16].try_into().unwrap()) + }) +} diff --git a/examples/prefab/balls/smatmul/verify/casegen/src/model.rs b/examples/prefab/balls/smatmul/verify/casegen/src/model.rs new file mode 100644 index 000000000..9b0594780 --- /dev/null +++ b/examples/prefab/balls/smatmul/verify/casegen/src/model.rs @@ -0,0 +1,60 @@ +pub const CORE: usize = 16; +pub const ROW_BYTES: usize = 16; +pub const RESULT_WORDS: usize = 4; +pub const BANK_DEPTH: usize = 64; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct WriteExp { + pub addr: u32, + pub data: [u8; ROW_BYTES], +} + +pub fn pack_a(src: &[i8], rows: usize) -> Vec { + assert_eq!(src.len(), rows * CORE); + src.iter().map(|value| *value as u8).collect() +} + +pub fn pack_b(src: &[i8]) -> Vec { + assert_eq!(src.len(), CORE * CORE); + src.iter().map(|value| *value as u8).collect() +} + +// `addr` is the line of C's virtual bank; it continues across groups. +pub fn emit_writes(c: &[Vec], output_base: usize) -> Vec { + let mut writes = Vec::with_capacity(c.len() * RESULT_WORDS); + for (row, values) in c.iter().enumerate() { + assert_eq!(values.len(), CORE); + for word in 0..RESULT_WORDS { + let mut data = [0; ROW_BYTES]; + for lane in 0..4 { + let value = values[word * 4 + lane]; + data[lane * 4..lane * 4 + 4].copy_from_slice(&value.to_le_bytes()); + } + writes.push(WriteExp { + addr: (output_base + row * RESULT_WORDS + word) as u32, + data, + }); + } + } + writes +} + +pub fn word(data: &[u8], index: usize) -> [u8; ROW_BYTES] { + data[index * ROW_BYTES..(index + 1) * ROW_BYTES] + .try_into() + .unwrap() +} + +pub fn encode_rs1(op1: u32, op2: u32, wr: u32, iter: u64) -> u64 { + assert!(op1 < 1024 && op2 < 1024 && wr < 1024 && iter < (1_u64 << 34)); + u64::from(op1) | (u64::from(op2) << 10) | (u64::from(wr) << 20) | (iter << 30) +} + +pub fn encode_rs2(rows: usize, first: bool, last: bool, output_base: usize) -> u64 { + assert!(matches!(rows, 16 | 32 | 64 | 128) && output_base < 64); + rows as u64 + | ((CORE as u64) << 12) + | ((first as u64) << 24) + | ((last as u64) << 25) + | ((output_base as u64) << 26) +} diff --git a/examples/balls/smatmul/verify/filelists/smatmul_ball.f b/examples/prefab/balls/smatmul/verify/filelists/smatmul_ball.f similarity index 89% rename from examples/balls/smatmul/verify/filelists/smatmul_ball.f rename to examples/prefab/balls/smatmul/verify/filelists/smatmul_ball.f index e053cb0a9..455c8d17a 100644 --- a/examples/balls/smatmul/verify/filelists/smatmul_ball.f +++ b/examples/prefab/balls/smatmul/verify/filelists/smatmul_ball.f @@ -1,4 +1,5 @@ +lint=TFIPC-L ++incdir+@UVM@/src/core +incdir+@UVM@/src/ball +incdir+@UVM@/src/ball/agents/cmd +incdir+@UVM@/src/ball/agents/mem @@ -14,6 +15,7 @@ +incdir+src/tests +incdir+@RTL@ +@UVM@/src/core/package.sv @UVM@/src/ball/bb_blink_if.sv @UVM@/src/ball/bb_uvm_pkg.sv diff --git a/examples/balls/smatmul/verify/parameters.toml b/examples/prefab/balls/smatmul/verify/parameters.toml similarity index 100% rename from examples/balls/smatmul/verify/parameters.toml rename to examples/prefab/balls/smatmul/verify/parameters.toml diff --git a/examples/balls/smatmul/verify/src/common/smatmul_defs.svh b/examples/prefab/balls/smatmul/verify/src/common/smatmul_defs.svh similarity index 79% rename from examples/balls/smatmul/verify/src/common/smatmul_defs.svh rename to examples/prefab/balls/smatmul/verify/src/common/smatmul_defs.svh index 4617c8277..ce049ef94 100644 --- a/examples/balls/smatmul/verify/src/common/smatmul_defs.svh +++ b/examples/prefab/balls/smatmul/verify/src/common/smatmul_defs.svh @@ -1,6 +1,5 @@ typedef struct { int unsigned kind; - int unsigned bid; int unsigned rob_id; int unsigned op1_words; int unsigned op2_words; @@ -8,6 +7,11 @@ typedef struct { int unsigned rs1_hi; int unsigned rs2_lo; int unsigned rs2_hi; + int unsigned op1_group; + int unsigned op2_group; + int unsigned wr_group; + int unsigned k_groups; + int unsigned wr_col; } matrix_cmd_dpi_t; `ifndef SMATMUL_TILE_ROWS @@ -17,7 +21,6 @@ typedef struct { import "DPI-C" function void smatmul_case_load( input int unsigned seed, input int unsigned index, - input int unsigned bid, input int unsigned tile_rows ); import "DPI-C" function int unsigned smatmul_case_num_commands(); @@ -40,21 +43,13 @@ import "DPI-C" function longint unsigned smatmul_case_write_data_lo(input int un import "DPI-C" function longint unsigned smatmul_case_write_data_hi(input int unsigned index); `ifndef SMATMUL_OS_FUNCT7 -`error "SMATMUL_OS_FUNCT7 must be provided by the selected Core ballISA" +`error "SMATMUL_OS_FUNCT7 must be provided by the selected Thread ballISA" `endif `ifndef SMATMUL_BIAS_FUNCT7 -`error "SMATMUL_BIAS_FUNCT7 must be provided by the selected Core ballISA" +`error "SMATMUL_BIAS_FUNCT7 must be provided by the selected Thread ballISA" `endif -localparam int SMATMUL_OS_CORE_FUNCT7 = `SMATMUL_OS_FUNCT7; -localparam int SMATMUL_BIAS_CORE_FUNCT7 = `SMATMUL_BIAS_FUNCT7; +localparam int SMATMUL_OS_THREAD_FUNCT7 = `SMATMUL_OS_FUNCT7; +localparam int SMATMUL_BIAS_THREAD_FUNCT7 = `SMATMUL_BIAS_FUNCT7; localparam int MATRIX_TIMEOUT_CYCLES = 200000; localparam int MATRIX_SEED = 32'hCAFE_BABE; - -function automatic int unsigned matrix_require_bid(); - int unsigned bid; - if (!$value$plusargs("BID=%d", bid)) begin - `uvm_fatal("BID", "missing required plusarg +BID=") - end - return bid; -endfunction diff --git a/examples/prefab/balls/smatmul/verify/src/common/smatmul_items.svh b/examples/prefab/balls/smatmul/verify/src/common/smatmul_items.svh new file mode 100644 index 000000000..7b1b87bc9 --- /dev/null +++ b/examples/prefab/balls/smatmul/verify/src/common/smatmul_items.svh @@ -0,0 +1,77 @@ +class smatmul_cmd_item extends bb_blink_cmd_item; + `uvm_object_utils(smatmul_cmd_item) + + bit [31:0] kind; + bit [31:0] command_index; + bit [31:0] command_count; + bit [31:0] op1_words; + bit [31:0] op2_words; + bit [31:0] k_groups; + bit [63:0] rs1; + + function new(string name = "smatmul_cmd_item"); + super.new(name); + endfunction + + function void load_rust_command(int unsigned index); + matrix_cmd_dpi_t cmd; + smatmul_case_cmd(index, cmd); + if (cmd.kind > 1) begin + `uvm_fatal("CASE", $sformatf("invalid command kind %0d", cmd.kind)) + end + + kind = cmd.kind; + command_index = index; + command_count = smatmul_case_num_commands(); + op1_words = cmd.op1_words; + op2_words = cmd.op2_words; + k_groups = cmd.k_groups; + rs1 = {cmd.rs1_hi, cmd.rs1_lo}; + + // BB_BANK0/1/2 and BB_ITER fields of rs1; banks are not renamed at Ball level. + rob_id = cmd.rob_id[BB_BLINK_ROB_ID_W-1:0]; + rs2 = {cmd.rs2_hi, cmd.rs2_lo}; + iter = rs1[63:30]; + op1_bank = rs1[9:0]; + op2_bank = rs1[19:10]; + wr_bank = rs1[29:20]; + op1_group = cmd.op1_group; + op2_group = cmd.op2_group; + wr_group = cmd.wr_group; + wr_read_bank = '0; + wr_arch_bank = wr_bank; + + if (kind == 0) begin + funct = SMATMUL_BIAS_THREAD_FUNCT7[6:0]; + op1_col = 1; + op2_col = 0; + wr_col = 0; + if (rs1[29:10] != 0 || iter != 4 || rs2 != 0 || op1_words != 4) + `uvm_fatal("CASE", "invalid bias command") + end else begin + funct = SMATMUL_OS_THREAD_FUNCT7[6:0]; + // A and B group g hold K block g; C rows continue across wr_col groups. + op1_col = cmd.k_groups; + op2_col = cmd.k_groups; + wr_col = cmd.wr_col; + if (k_groups == 0 || cmd.wr_col == 0) `uvm_fatal("CASE", "SMATMUL_OS needs groups") + if (op1_bank == op2_bank || op1_bank == wr_bank || op2_bank == wr_bank) + `uvm_fatal("CASE", "SMATMUL_OS banks must be distinct") + if (rs2[63:32] != 0 || rs2[23:12] != 16 || iter != 16) + `uvm_fatal("CASE", "invalid SMATMUL_OS shape or reserved bits") + end + endfunction + + function void do_copy(uvm_object rhs); + smatmul_cmd_item rhs_; + super.do_copy(rhs); + if (!$cast(rhs_, rhs)) `uvm_fatal("COPY", "rhs is not smatmul_cmd_item") + kind = rhs_.kind; + command_index = rhs_.command_index; + command_count = rhs_.command_count; + op1_words = rhs_.op1_words; + op2_words = rhs_.op2_words; + k_groups = rhs_.k_groups; + rs1 = rhs_.rs1; + endfunction +endclass diff --git a/examples/balls/smatmul/verify/src/cov/smatmul_cov.svh b/examples/prefab/balls/smatmul/verify/src/cov/smatmul_cov.svh similarity index 89% rename from examples/balls/smatmul/verify/src/cov/smatmul_cov.svh rename to examples/prefab/balls/smatmul/verify/src/cov/smatmul_cov.svh index 59a4d05bc..764872cc7 100644 --- a/examples/balls/smatmul/verify/src/cov/smatmul_cov.svh +++ b/examples/prefab/balls/smatmul/verify/src/cov/smatmul_cov.svh @@ -18,13 +18,14 @@ class smatmul_cov extends uvm_component; endfunction function void write_cmd(bb_blink_cmd_item item); - is_bias = (item.funct7 == SMATMUL_BIAS_CORE_FUNCT7[6:0]); + is_bias = (item.funct == SMATMUL_BIAS_THREAD_FUNCT7[6:0]); is_first = is_bias ? 0 : item.rs2[24]; cmd_count++; cmd_cg.sample(); endfunction function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; super.check_phase(phase); if (cmd_count == 0) `uvm_fatal("COV", "no cmd observed") if (cmd_cg.get_coverage() != 100.0) diff --git a/examples/balls/smatmul/verify/src/env/smatmul_env.svh b/examples/prefab/balls/smatmul/verify/src/env/smatmul_env.svh similarity index 100% rename from examples/balls/smatmul/verify/src/env/smatmul_env.svh rename to examples/prefab/balls/smatmul/verify/src/env/smatmul_env.svh diff --git a/examples/balls/smatmul/verify/src/env/smatmul_scoreboard.svh b/examples/prefab/balls/smatmul/verify/src/env/smatmul_scoreboard.svh similarity index 75% rename from examples/balls/smatmul/verify/src/env/smatmul_scoreboard.svh rename to examples/prefab/balls/smatmul/verify/src/env/smatmul_scoreboard.svh index 8ee7362c5..f74028015 100644 --- a/examples/balls/smatmul/verify/src/env/smatmul_scoreboard.svh +++ b/examples/prefab/balls/smatmul/verify/src/env/smatmul_scoreboard.svh @@ -36,6 +36,7 @@ class smatmul_scoreboard extends uvm_scoreboard; matrix_cmd_dpi_t data_cmd; longint unsigned lo; longint unsigned hi; + int unsigned per_group; if (!$cast(got, item) || !$cast(clone, got.clone())) `uvm_fatal("SCB", "failed to clone smatmul stimulus") if (got.command_index != stim_count) `uvm_fatal("SCB", "commands arrived out of order") @@ -48,19 +49,24 @@ class smatmul_scoreboard extends uvm_scoreboard; for (int i = 0; i < smatmul_case_bias_words(); i++) begin lo = smatmul_case_bias_word_lo(i); hi = smatmul_case_bias_word_hi(i); - mem_model.load_word(0, i, {hi, lo}); + mem_model.load_word_g(got.op1_bank, got.op1_group, i, {hi, lo}); end for (int block = 0; block < expected_commands - 1; block++) begin smatmul_case_cmd(block + 1, data_cmd); + // A and B words are group-major: K group g starts at g * per_group. + per_group = smatmul_case_a_words(block) / data_cmd.k_groups; for (int i = 0; i < smatmul_case_a_words(block); i++) begin lo = smatmul_case_a_word_lo(block, i); hi = smatmul_case_a_word_hi(block, i); - mem_model.load_word(data_cmd.rs1_lo[9:0], i, {hi, lo}); + mem_model.load_word_g(data_cmd.rs1_lo[9:0], data_cmd.op1_group + i / per_group, + i % per_group, {hi, lo}); end + per_group = smatmul_case_b_words(block) / data_cmd.k_groups; for (int i = 0; i < smatmul_case_b_words(block); i++) begin lo = smatmul_case_b_word_lo(block, i); hi = smatmul_case_b_word_hi(block, i); - mem_model.load_word(data_cmd.rs1_lo[19:10], i, {hi, lo}); + mem_model.load_word_g(data_cmd.rs1_lo[19:10], data_cmd.op2_group + i / per_group, + i % per_group, {hi, lo}); end end mem_model.arm(); @@ -77,23 +83,18 @@ class smatmul_scoreboard extends uvm_scoreboard; `uvm_fatal("SCB", "command count changed within a case") end - expected_reads[0] += got.op1_words; - expected_reads[1] += got.kind == 0 ? 0 : got.op2_words; + expected_reads[0] += got.kind == 0 ? got.op1_words : got.op1_words * got.k_groups; + expected_reads[1] += got.kind == 0 ? 0 : got.op2_words * got.k_groups; stim_q.push_back(clone); stim_count++; endfunction function void write_cmd(bb_blink_cmd_item item); - smatmul_cmd_item exp; - exp = current_stim("CMD"); - if (item.bid !== exp.bid || item.funct7 !== exp.funct7 || - item.op1_bank !== exp.op1_bank || item.op2_bank !== exp.op2_bank || - item.wr_bank !== exp.wr_bank || item.rs1 !== exp.rs1 || - item.rs2 !== exp.rs2 || item.iter !== exp.iter || - item.op1_col !== exp.op1_col || item.op2_col !== exp.op2_col || - item.wr_col !== exp.wr_col || item.op1_en !== exp.op1_en || - item.op2_en !== exp.op2_en || item.wr_spad_en !== exp.wr_spad_en) - `uvm_fatal("CMD", "command differs from generated contract") + // Compare as plain BLink commands; the comparer rejects differing types. + bb_blink_cmd_item want = bb_blink_cmd_item::type_id::create("want"); + want.copy(current_stim("CMD")); + if (!item.compare(want)) + `uvm_fatal("CMD", $sformatf("command mismatch: got %s exp %s", item.sprint(), want.sprint())) cmd_count++; endfunction @@ -101,13 +102,16 @@ class smatmul_scoreboard extends uvm_scoreboard; smatmul_cmd_item stim; stim = current_stim("READ"); if (stim.kind == 0) begin - if (item.port != 0 || item.bank_id != stim.op1_bank || item.addr >= 4) + if (item.port != 0 || item.bank_id != stim.op1_bank || item.group_id != stim.op1_group || + item.addr >= 4) `uvm_fatal("READ", "invalid bias read") end else if (item.port == 0) begin - if (item.bank_id != stim.op1_bank || item.addr >= stim.op1_words) + if (item.bank_id != stim.op1_bank || item.group_id < stim.op1_group || + item.group_id >= stim.op1_group + stim.k_groups || item.addr >= stim.op1_words) `uvm_fatal("READ", "A read outside command footprint") end else if (item.port == 1) begin - if (item.bank_id != stim.op2_bank || item.addr >= stim.op2_words) + if (item.bank_id != stim.op2_bank || item.group_id < stim.op2_group || + item.group_id >= stim.op2_group + stim.k_groups || item.addr >= stim.op2_words) `uvm_fatal("READ", "B read outside command footprint") end else begin `uvm_fatal("READ", "unexpected read port") @@ -121,11 +125,14 @@ class smatmul_scoreboard extends uvm_scoreboard; stim = current_stim("WRITE"); if (stim.kind != 1 || stim.rs2[25] != 1) `uvm_fatal("WRITE", "write before the last accumulation block") - if (item.port != 0 || item.group_id != 0 || item.bank_id != stim.wr_bank || + if (item.port != 0 || item.bank_id != stim.wr_bank || item.mask != 16'hffff || item.rob_id != stim.rob_id) `uvm_fatal("WRITE", "write metadata violates SMatMul contract") for (int i = 0; i < expected_writes; i++) begin - if (!exp_seen[i] && item.addr === exp_addr[i][9:0] && item.data === exp_data[i]) begin + // exp_addr is C's virtual line: group wr_group + line / depth, addr line % depth. + if (!exp_seen[i] && item.addr === exp_addr[i] % (1 << `BB_BANK_ADDR_W) && + item.group_id === stim.wr_group + (exp_addr[i] >> `BB_BANK_ADDR_W) && + item.data === exp_data[i]) begin match_index = i; break; end @@ -138,8 +145,7 @@ class smatmul_scoreboard extends uvm_scoreboard; function void write_resp(bb_blink_resp_item item); smatmul_cmd_item stim; stim = current_stim("RESP"); - if (item.rob_id !== stim.rob_id || item.is_sub !== 1'b0) - `uvm_fatal("RESP", "response metadata mismatch") + if (item.rob_id !== stim.rob_id) `uvm_fatal("RESP", "response metadata mismatch") void'(stim_q.pop_front()); resp_count++; endfunction @@ -178,6 +184,7 @@ class smatmul_scoreboard extends uvm_scoreboard; endfunction function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; super.check_phase(phase); if (!done()) `uvm_fatal("SCB", $sformatf( diff --git a/examples/balls/smatmul/verify/src/pkg/smatmul_pkg.sv b/examples/prefab/balls/smatmul/verify/src/pkg/smatmul_pkg.sv similarity index 100% rename from examples/balls/smatmul/verify/src/pkg/smatmul_pkg.sv rename to examples/prefab/balls/smatmul/verify/src/pkg/smatmul_pkg.sv diff --git a/examples/balls/smatmul/verify/src/seq/smatmul_sequences.svh b/examples/prefab/balls/smatmul/verify/src/seq/smatmul_sequences.svh similarity index 89% rename from examples/balls/smatmul/verify/src/seq/smatmul_sequences.svh rename to examples/prefab/balls/smatmul/verify/src/seq/smatmul_sequences.svh index 7d4c067e2..107db086a 100644 --- a/examples/balls/smatmul/verify/src/seq/smatmul_sequences.svh +++ b/examples/prefab/balls/smatmul/verify/src/seq/smatmul_sequences.svh @@ -3,7 +3,6 @@ class smatmul_chain_seq extends uvm_sequence #(bb_blink_cmd_item); int unsigned case_index; int unsigned seed = MATRIX_SEED; - int unsigned bid; function new(string name = "smatmul_chain_seq"); super.new(name); @@ -11,7 +10,7 @@ class smatmul_chain_seq extends uvm_sequence #(bb_blink_cmd_item); task body(); int unsigned count; - smatmul_case_load(seed, case_index, bid, `SMATMUL_TILE_ROWS); + smatmul_case_load(seed, case_index, `SMATMUL_TILE_ROWS); count = smatmul_case_num_commands(); if (count < 2 || count > 3) `uvm_fatal("CASE", "command count must be 2 or 3") for (int unsigned index = 0; index < count; index++) begin diff --git a/examples/prefab/balls/smatmul/verify/src/tb_top.sv b/examples/prefab/balls/smatmul/verify/src/tb_top.sv new file mode 100644 index 000000000..e478bf3b5 --- /dev/null +++ b/examples/prefab/balls/smatmul/verify/src/tb_top.sv @@ -0,0 +1,89 @@ +module tb_top; + import uvm_pkg::*; + import smatmul_pkg::*; + + bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); + + SMatMulBall dut ( + .clock(intf.clock), + .reset(intf.reset), + .blink_status_running(intf.status_running), + .blink_cmdReq_ready(intf.cmd_req_ready), + .blink_cmdReq_valid(intf.cmd_req_valid), + .blink_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .blink_cmdReq_bits_funct(intf.cmd_req_bits_funct), + .blink_cmdReq_bits_rs2(intf.cmd_req_bits_rs2), + .blink_cmdReq_bits_iter(intf.cmd_req_bits_iter), + .blink_cmdReq_bits_op1_bank(intf.cmd_req_bits_op1_bank), + .blink_cmdReq_bits_op2_bank(intf.cmd_req_bits_op2_bank), + .blink_cmdReq_bits_wr_bank(intf.cmd_req_bits_wr_bank), + .blink_cmdReq_bits_op1_group(intf.cmd_req_bits_op1_group), + .blink_cmdReq_bits_op2_group(intf.cmd_req_bits_op2_group), + .blink_cmdReq_bits_wr_group(intf.cmd_req_bits_wr_group), + .blink_cmdReq_bits_op1_col(intf.cmd_req_bits_op1_col), + .blink_cmdReq_bits_op2_col(intf.cmd_req_bits_op2_col), + .blink_cmdReq_bits_wr_col(intf.cmd_req_bits_wr_col), + .blink_cmdReq_bits_wr_read_bank(intf.cmd_req_bits_wr_read_bank), + .blink_cmdReq_bits_wr_arch_bank(intf.cmd_req_bits_wr_arch_bank), + .blink_cmdResp_ready(intf.cmd_resp_ready), + .blink_cmdResp_valid(intf.cmd_resp_valid), + .blink_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .blink_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .blink_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .blink_bankRead_0_group_id(intf.bank_read_group_id[0]), + .blink_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .blink_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .blink_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .blink_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .blink_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .blink_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .blink_bankRead_1_bank_id(intf.bank_read_bank_id[1]), + .blink_bankRead_1_rob_id(intf.bank_read_rob_id[1]), + .blink_bankRead_1_group_id(intf.bank_read_group_id[1]), + .blink_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), + .blink_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), + .blink_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), + .blink_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), + .blink_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), + .blink_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), + .blink_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .blink_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .blink_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .blink_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .blink_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .blink_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .blink_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .blink_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .blink_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .blink_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]) + ); + + // Constant outputs removed from the port list by firtool. + assign intf.bank_write_req_mask[0][0] = dut.blink_bankWrite_0_io_req_bits_mask_0; + assign intf.bank_write_req_mask[0][1] = dut.blink_bankWrite_0_io_req_bits_mask_1; + assign intf.bank_write_req_mask[0][2] = dut.blink_bankWrite_0_io_req_bits_mask_2; + assign intf.bank_write_req_mask[0][3] = dut.blink_bankWrite_0_io_req_bits_mask_3; + assign intf.bank_write_req_mask[0][4] = dut.blink_bankWrite_0_io_req_bits_mask_4; + assign intf.bank_write_req_mask[0][5] = dut.blink_bankWrite_0_io_req_bits_mask_5; + assign intf.bank_write_req_mask[0][6] = dut.blink_bankWrite_0_io_req_bits_mask_6; + assign intf.bank_write_req_mask[0][7] = dut.blink_bankWrite_0_io_req_bits_mask_7; + assign intf.bank_write_req_mask[0][8] = dut.blink_bankWrite_0_io_req_bits_mask_8; + assign intf.bank_write_req_mask[0][9] = dut.blink_bankWrite_0_io_req_bits_mask_9; + assign intf.bank_write_req_mask[0][10] = dut.blink_bankWrite_0_io_req_bits_mask_10; + assign intf.bank_write_req_mask[0][11] = dut.blink_bankWrite_0_io_req_bits_mask_11; + assign intf.bank_write_req_mask[0][12] = dut.blink_bankWrite_0_io_req_bits_mask_12; + assign intf.bank_write_req_mask[0][13] = dut.blink_bankWrite_0_io_req_bits_mask_13; + assign intf.bank_write_req_mask[0][14] = dut.blink_bankWrite_0_io_req_bits_mask_14; + assign intf.bank_write_req_mask[0][15] = dut.blink_bankWrite_0_io_req_bits_mask_15; + + initial begin + intf.clock = 1'b0; + intf.reset = 1'b1; + forever #5 intf.clock = ~intf.clock; + end + + initial begin + uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); + run_test("smatmul_ball_test"); + end +endmodule diff --git a/examples/balls/smatmul/verify/src/tests/smatmul_test.svh b/examples/prefab/balls/smatmul/verify/src/tests/smatmul_test.svh similarity index 90% rename from examples/balls/smatmul/verify/src/tests/smatmul_test.svh rename to examples/prefab/balls/smatmul/verify/src/tests/smatmul_test.svh index 862de4e18..24fcba13c 100644 --- a/examples/balls/smatmul/verify/src/tests/smatmul_test.svh +++ b/examples/prefab/balls/smatmul/verify/src/tests/smatmul_test.svh @@ -17,20 +17,18 @@ class smatmul_ball_test extends uvm_test; endfunction task run_phase(uvm_phase phase); - int unsigned bid = matrix_require_bid(); phase.raise_objection(this); - for (int unsigned index = 0; index < 11; index++) run_case(index, bid); + for (int unsigned index = 0; index < 13; index++) run_case(index); phase.drop_objection(this); endtask - task run_case(int unsigned index, int unsigned bid); + task run_case(int unsigned index); smatmul_chain_seq seq; int cycles = 0; apply_reset(); env.scb.reset_counters(); seq = smatmul_chain_seq::type_id::create("seq"); seq.case_index = index; - seq.bid = bid; seq.start(env.cmd_agent.seqr); while (!env.scb.done()) begin @(posedge vif.clock); diff --git a/examples/prefab/balls/smatmul/workloads/ctests/CMakeLists.txt b/examples/prefab/balls/smatmul/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..c1b5d370b --- /dev/null +++ b/examples/prefab/balls/smatmul/workloads/ctests/CMakeLists.txt @@ -0,0 +1,14 @@ +add_buckyball_ctest(smatmul_bias_accumulate_test.c) +add_buckyball_ctest(smatmul_1x16x64_test.c) +add_buckyball_ctest(smatmul_8x16_test.c) +add_buckyball_ctest(smatmul_16x16_test.c) +add_buckyball_ctest(smatmul_16x16x32_test.c) +add_buckyball_ctest(smatmul_nondefault_banks_test.c) +add_buckyball_ctest(smatmul_k_groups_test.c) +add_buckyball_ctest(smatmul_c_groups_test.c) +if( + DEFINED + DESIGN_PROFILES_${BUCKYBALL_CTEST_TARGET}_THREAD_BALLDOMAIN_BALLS_smatmul_INSTRUCTIONS_SMATMUL_F32_FUNCT7 +) + add_buckyball_ctest(smatmul_f32_test.c) +endif() diff --git a/examples/balls/smatmul/workloads/ctests/smatmul_16x16_test.c b/examples/prefab/balls/smatmul/workloads/ctests/smatmul_16x16_test.c similarity index 100% rename from examples/balls/smatmul/workloads/ctests/smatmul_16x16_test.c rename to examples/prefab/balls/smatmul/workloads/ctests/smatmul_16x16_test.c diff --git a/examples/balls/smatmul/workloads/ctests/smatmul_16x16x32_test.c b/examples/prefab/balls/smatmul/workloads/ctests/smatmul_16x16x32_test.c similarity index 100% rename from examples/balls/smatmul/workloads/ctests/smatmul_16x16x32_test.c rename to examples/prefab/balls/smatmul/workloads/ctests/smatmul_16x16x32_test.c diff --git a/examples/balls/smatmul/workloads/ctests/smatmul_1x16x64_test.c b/examples/prefab/balls/smatmul/workloads/ctests/smatmul_1x16x64_test.c similarity index 100% rename from examples/balls/smatmul/workloads/ctests/smatmul_1x16x64_test.c rename to examples/prefab/balls/smatmul/workloads/ctests/smatmul_1x16x64_test.c diff --git a/examples/balls/smatmul/workloads/ctests/smatmul_8x16_test.c b/examples/prefab/balls/smatmul/workloads/ctests/smatmul_8x16_test.c similarity index 100% rename from examples/balls/smatmul/workloads/ctests/smatmul_8x16_test.c rename to examples/prefab/balls/smatmul/workloads/ctests/smatmul_8x16_test.c diff --git a/examples/balls/smatmul/workloads/ctests/smatmul_bias_accumulate_test.c b/examples/prefab/balls/smatmul/workloads/ctests/smatmul_bias_accumulate_test.c similarity index 94% rename from examples/balls/smatmul/workloads/ctests/smatmul_bias_accumulate_test.c rename to examples/prefab/balls/smatmul/workloads/ctests/smatmul_bias_accumulate_test.c index de17c59bc..53dfec7d4 100644 --- a/examples/balls/smatmul/workloads/ctests/smatmul_bias_accumulate_test.c +++ b/examples/prefab/balls/smatmul/workloads/ctests/smatmul_bias_accumulate_test.c @@ -30,6 +30,8 @@ int main(void) { bb_mvin((uintptr_t)weight0, 2, 16, 1); bb_mvin((uintptr_t)input1, 3, 16, 1); bb_mvin((uintptr_t)weight1, 4, 16, 1); + // An unfinished block defines a new destination version without bank writes. + bb_mvin((uintptr_t)bias, 5, 4, 1); bb_smatmul_bias(0, 0); bb_smatmul_os(1, 2, 5, 16, 16, 16, 1, 0, 0); bb_smatmul_os(3, 4, 5, 16, 16, 16, 0, 1, 0); diff --git a/examples/prefab/balls/smatmul/workloads/ctests/smatmul_c_groups_test.c b/examples/prefab/balls/smatmul/workloads/ctests/smatmul_c_groups_test.c new file mode 100644 index 000000000..6e16cb27d --- /dev/null +++ b/examples/prefab/balls/smatmul/workloads/ctests/smatmul_c_groups_test.c @@ -0,0 +1,55 @@ +#include "buckyball.h" +#include +#include +#include +#include + +/* M=128, N=16, K=32 with output_base 32: C occupies lines 32..543, so its rows + * continue from group 0 into group 1 (line L is group L / + * THREAD_MEM_BANK_ENTRIES). A and B have two reduction groups. */ +enum { M = 128, N = 16, K = 32, BASE = 32 }; +static int8_t a[M * K] __attribute__((aligned(64))); +static int8_t b[K * N] __attribute__((aligned(64))); +static int32_t bias[N] __attribute__((aligned(64))); +static int32_t c[2][THREAD_MEM_BANK_ENTRIES * 4] __attribute__((aligned(64))); + +int main(void) { + for (int row = 0; row < M; ++row) + for (int k = 0; k < K; ++k) + a[row * K + k] = (row * 5 + k * 3) % 13 - 6; + for (int k = 0; k < K; ++k) + for (int col = 0; col < N; ++col) + b[k * N + col] = (k * 7 + col) % 11 - 5; + for (int col = 0; col < N; ++col) + bias[col] = col - 8; + + bb_mem_alloc(0, 1, 1); + bb_mem_alloc(1, 1, 2); + bb_mem_alloc(2, 1, 2); + bb_mem_alloc(3, 1, 2); + bb_mvin((uintptr_t)bias, 0, 4, 1); + bb_mvin((uintptr_t)a, 1, M, 1); + bb_mvin_group((uintptr_t)b, 2, 0, 16, 1); + bb_mvin_group((uintptr_t)(b + 16 * N), 2, 1, 16, 1); + bb_smatmul_bias(0, 0); + bb_smatmul_os(1, 2, 3, M, N, 16, 1, 1, BASE); + bb_mvout_group((uintptr_t)c[0], 3, 0, THREAD_MEM_BANK_ENTRIES, 1); + bb_mvout_group((uintptr_t)c[1], 3, 1, THREAD_MEM_BANK_ENTRIES, 1); + + for (int row = 0; row < M; ++row) + for (int col = 0; col < N; ++col) { + int expected = bias[col]; + for (int k = 0; k < K; ++k) + expected += a[row * K + k] * b[k * N + col]; + int line = BASE + row * 4 + col / 4; + int actual = c[line / THREAD_MEM_BANK_ENTRIES] + [line % THREAD_MEM_BANK_ENTRIES * 4 + col % 4]; + if (actual != expected) { + printf("smatmul_c_groups FAIL row=%d col=%d expected=%d actual=%d\n", + row, col, expected, actual); + return 1; + } + } + printf("smatmul_c_groups PASS\n"); + return 0; +} diff --git a/examples/balls/smatmul/workloads/ctests/smatmul_f32_test.c b/examples/prefab/balls/smatmul/workloads/ctests/smatmul_f32_test.c similarity index 100% rename from examples/balls/smatmul/workloads/ctests/smatmul_f32_test.c rename to examples/prefab/balls/smatmul/workloads/ctests/smatmul_f32_test.c diff --git a/examples/prefab/balls/smatmul/workloads/ctests/smatmul_k_groups_test.c b/examples/prefab/balls/smatmul/workloads/ctests/smatmul_k_groups_test.c new file mode 100644 index 000000000..e904c8207 --- /dev/null +++ b/examples/prefab/balls/smatmul/workloads/ctests/smatmul_k_groups_test.c @@ -0,0 +1,52 @@ +#include "buckyball.h" +#include +#include +#include +#include + +/* M=8, N=16, K=48 in one SMATMUL_OS: A and B have three groups, group g holds + * reduction block g (k = 16 per group). Row-major A[8][48] lands in a + * three-group bank as group g line r = A[r][16g..16g+15]. */ +enum { M = 8, N = 16, K = 48 }; +static int8_t a[M * K] __attribute__((aligned(64))); +static int8_t b[K * N] __attribute__((aligned(64))); +static int32_t bias[N] __attribute__((aligned(64))); +static int32_t actual[M * N] __attribute__((aligned(64))); + +int main(void) { + for (int row = 0; row < M; ++row) + for (int k = 0; k < K; ++k) + a[row * K + k] = (row * 5 + k * 3) % 13 - 6; + for (int k = 0; k < K; ++k) + for (int col = 0; col < N; ++col) + b[k * N + col] = (k * 7 + col) % 11 - 5; + for (int col = 0; col < N; ++col) + bias[col] = col * 3 - 20; + + bb_mem_alloc(0, 1, 1); + bb_mem_alloc(1, 1, 3); + bb_mem_alloc(2, 1, 3); + bb_mem_alloc(3, 1, 1); + bb_mvin((uintptr_t)bias, 0, 4, 1); + bb_mvin((uintptr_t)a, 1, M, 1); + bb_mvin_group((uintptr_t)b, 2, 0, 16, 1); + bb_mvin_group((uintptr_t)(b + 16 * N), 2, 1, 16, 1); + bb_mvin_group((uintptr_t)(b + 32 * N), 2, 2, 16, 1); + bb_smatmul_bias(0, 0); + bb_smatmul_os(1, 2, 3, M, N, 16, 1, 1, 0); + bb_mvout((uintptr_t)actual, 3, M * 4, 1); + + for (int row = 0; row < M; ++row) + for (int col = 0; col < N; ++col) { + int expected = bias[col]; + for (int k = 0; k < K; ++k) + expected += a[row * K + k] * b[k * N + col]; + if (actual[row * N + col] != expected) { + printf("smatmul_k_groups FAIL row=%d col=%d expected=%d actual=%d\n", + row, col, expected, actual[row * N + col]); + return 1; + } + } + printf("smatmul_k_groups PASS\n"); + return 0; +} diff --git a/examples/balls/smatmul/workloads/ctests/smatmul_nondefault_banks_test.c b/examples/prefab/balls/smatmul/workloads/ctests/smatmul_nondefault_banks_test.c similarity index 90% rename from examples/balls/smatmul/workloads/ctests/smatmul_nondefault_banks_test.c rename to examples/prefab/balls/smatmul/workloads/ctests/smatmul_nondefault_banks_test.c index 566f3c54f..3b6dcf41c 100644 --- a/examples/balls/smatmul/workloads/ctests/smatmul_nondefault_banks_test.c +++ b/examples/prefab/balls/smatmul/workloads/ctests/smatmul_nondefault_banks_test.c @@ -2,16 +2,15 @@ #include #include #include -#include #include #include enum { M = 16, N = 16, K = 16 }; enum { - BIAS_BANK = VIRTUAL_BANK_NUM - 4, - A_BANK = VIRTUAL_BANK_NUM - 3, - B_BANK = VIRTUAL_BANK_NUM - 2, - C_BANK = VIRTUAL_BANK_NUM - 1 + BIAS_BANK = CORE_VIRTUAL_BANK_COUNT - 4, + A_BANK = CORE_VIRTUAL_BANK_COUNT - 3, + B_BANK = CORE_VIRTUAL_BANK_COUNT - 2, + C_BANK = CORE_VIRTUAL_BANK_COUNT - 1 }; static int8_t input[M * K] __attribute__((aligned(64))); static int8_t weight[K * N] __attribute__((aligned(64))); diff --git a/examples/prefab/balls/smatmul/workloads/isa/smatmul.h b/examples/prefab/balls/smatmul/workloads/isa/smatmul.h new file mode 100644 index 000000000..e4fd2d09f --- /dev/null +++ b/examples/prefab/balls/smatmul/workloads/isa/smatmul.h @@ -0,0 +1,46 @@ +#ifndef _BB_SMATMUL_H_ +#define _BB_SMATMUL_H_ + +#include +#include + +// SMATMUL_OS: A and B have the same number of groups; group g holds reduction +// block g, so the instruction accumulates over K = k * groups with k per group +// (A[:, g*k : (g+1)*k] and B[:, g*k : (g+1)*k] in the single-group layouts). +// C rows continue across groups: line L = output_base + row * 4 + word lives +// in group L / THREAD_MEM_BANK_ENTRIES, line L % THREAD_MEM_BANK_ENTRIES. The +// bias bank has one group. +#define BB_SMATMUL_CFG(rows, cols, first, last, output_base) \ + (FIELD((rows), 0, 11) | FIELD((cols), 12, 23) | FIELD((first), 24, 24) | \ + FIELD((last), 25, 25) | FIELD((output_base), 26, 31)) + +#define bb_smatmul_bias(bias_bank, input_base) \ + BUCKYBALL_INSTRUCTION_R_R(BB_BANK0(bias_bank) | BB_ITER(4), \ + FIELD((input_base), 0, 5), \ + FUNCT7(smatmul, SMATMUL_BIAS)) + +#define bb_smatmul_os(a_bank, b_bank, c_bank, rows, cols, k, first, last, \ + output_base) \ + BUCKYBALL_INSTRUCTION_R_R( \ + BB_BANK0(a_bank) | BB_BANK1(b_bank) | BB_BANK2(c_bank) | BB_ITER(k), \ + BB_SMATMUL_CFG(rows, cols, first, last, output_base), \ + FUNCT7(smatmul, SMATMUL_OS)) + +// E4M3 codes in row-major A[M,K] and B[N,K], followed by E8M0[M,K/32] +// and E8M0[N,K/32]. C is row-major FP32. K is a multiple of 32. +#define bb_smatmul_mxfp8(a_bank, b_bank, c_bank, rows, cols, k, first, last, \ + output_base) \ + BUCKYBALL_INSTRUCTION_R_R( \ + BB_BANK0(a_bank) | BB_BANK1(b_bank) | BB_BANK2(c_bank) | BB_ITER(k), \ + BB_SMATMUL_CFG(rows, cols, first, last, output_base), \ + FUNCT7(smatmul, SMATMUL_MXFP8)) + +// Row-major FP32 A[M,K], B[N,K], C[M,N]. K is a multiple of four. +#define bb_smatmul_f32(a_bank, b_bank, c_bank, rows, cols, k, first, last, \ + output_base) \ + BUCKYBALL_INSTRUCTION_R_R( \ + BB_BANK0(a_bank) | BB_BANK1(b_bank) | BB_BANK2(c_bank) | BB_ITER(k), \ + BB_SMATMUL_CFG(rows, cols, first, last, output_base), \ + FUNCT7(smatmul, SMATMUL_F32)) + +#endif // _BB_SMATMUL_H_ diff --git a/examples/balls/smatmul/workloads/mlir_tests/CMakeLists.txt b/examples/prefab/balls/smatmul/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/balls/smatmul/workloads/mlir_tests/CMakeLists.txt rename to examples/prefab/balls/smatmul/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/balls/smatmul/workloads/mlir_tests/ball/CMakeLists.txt b/examples/prefab/balls/smatmul/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/balls/smatmul/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/prefab/balls/smatmul/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/prefab/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate.mlir b/examples/prefab/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate.mlir new file mode 100644 index 000000000..14df63d16 --- /dev/null +++ b/examples/prefab/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate.mlir @@ -0,0 +1,70 @@ +func.func private @check_result(memref<64x4xi32>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero_i32 = arith.constant 0 : i32 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %four = arith.constant 4 : index + %eight = arith.constant 8 : index + %sixteen = arith.constant 16 : index + + %input0 = memref.alloc() alignment = 64 : memref<16x16xi8> + %input1 = memref.alloc() alignment = 64 : memref<16x16xi8> + %weight0 = memref.alloc() alignment = 64 : memref<16x16xi8> + %weight1 = memref.alloc() alignment = 64 : memref<16x16xi8> + %bias = memref.alloc() alignment = 64 : memref<4x4xi32> + %output = memref.alloc() alignment = 64 : memref<64x4xi32> + linalg.fill ins(%zero_i32 : i32) outs(%output : memref<64x4xi32>) + + scf.for %row = %zero to %sixteen step %one { + scf.for %column = %zero to %sixteen step %one { + %row_i32 = arith.index_cast %row : index to i32 + %column_i32 = arith.index_cast %column : index to i32 + %sum = arith.addi %row_i32, %column_i32 : i32 + %difference = arith.subi %row_i32, %column_i32 : i32 + %twice_row = arith.addi %row_i32, %row_i32 : i32 + %second = arith.subi %twice_row, %column_i32 : i32 + %sum_i8 = arith.trunci %sum : i32 to i8 + %difference_i8 = arith.trunci %difference : i32 to i8 + %second_i8 = arith.trunci %second : i32 to i8 + memref.store %difference_i8, %input0[%row, %column] : memref<16x16xi8> + memref.store %sum_i8, %weight0[%row, %column] : memref<16x16xi8> + memref.store %sum_i8, %input1[%row, %column] : memref<16x16xi8> + memref.store %second_i8, %weight1[%row, %column] : memref<16x16xi8> + } + %bias_value = arith.subi %row, %eight : index + %bias_i32 = arith.index_cast %bias_value : index to i32 + %bias_row = arith.divui %row, %four : index + %bias_lane = arith.remui %row, %four : index + memref.store %bias_i32, %bias[%bias_row, %bias_lane] : memref<4x4xi32> + } + + buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 4 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 5 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %input0 <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.mvin %weight0 <{bank = 2 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.mvin %input1 <{bank = 3 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.mvin %weight1 <{bank = 4 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + // The bias occupies the first four rows of C. + buckyball.mvin %bias <{bank = 5 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xi32> + // Two reduction blocks: A = (input0, input1) and B = (weight0, weight1). + buckyball.mset_transfer <{source = 3 : i64, target = 1 : i64}> + buckyball.mset_transfer <{source = 4 : i64, target = 2 : i64}> + smatmul_ball.matmul_i8 <{a = 1 : i64, b = 2 : i64, c = 5 : i64, rows = 16 : i64, k = 16 : i64}> + buckyball.mvout %output <{bank = 5 : i64, depth = 64 : i64, stride = 1 : i64}> : memref<64x4xi32> + func.call @check_result(%output) : (memref<64x4xi32>) -> () + buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 5 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + memref.dealloc %input0 : memref<16x16xi8> + memref.dealloc %input1 : memref<16x16xi8> + memref.dealloc %weight0 : memref<16x16xi8> + memref.dealloc %weight1 : memref<16x16xi8> + memref.dealloc %bias : memref<4x4xi32> + memref.dealloc %output : memref<64x4xi32> + return %zero_i8 : i8 +} diff --git a/examples/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate_main.cpp b/examples/prefab/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate_main.cpp similarity index 94% rename from examples/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate_main.cpp rename to examples/prefab/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate_main.cpp index 14d4a7062..ba053c808 100644 --- a/examples/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate_main.cpp +++ b/examples/prefab/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate_main.cpp @@ -35,5 +35,5 @@ extern "C" void check_result(int32_t *allocated, int32_t *aligned, } } } - printf("PASSED: smatmul bias and two-step accumulation\n"); + printf("PASSED: smatmul bias and two reduction blocks\n"); } diff --git a/examples/prefab/balls/smatmul/workloads/mlir_tests/bank/CMakeLists.txt b/examples/prefab/balls/smatmul/workloads/mlir_tests/bank/CMakeLists.txt new file mode 100644 index 000000000..03d62d5a1 --- /dev/null +++ b/examples/prefab/balls/smatmul/workloads/mlir_tests/bank/CMakeLists.txt @@ -0,0 +1,16 @@ +if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) + set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) +endif() +set(BUCKYBALL_MLIR_TEST_PREFIX bank) + +set(_lines ${DESIGN_PROFILES_${BUCKYBALL_MLIR_ACTIVE_TARGET}_THREAD_MEM_BANK_ENTRIES}) +set(_width ${DESIGN_PROFILES_${BUCKYBALL_MLIR_ACTIVE_TARGET}_THREAD_MEM_BANK_WIDTH}) +math(EXPR _line_remainder "${_lines} % 64") +if(NOT (_width EQUAL 128) OR _lines LESS 64 OR + NOT (_line_remainder EQUAL 0)) + message(FATAL_ERROR + "SMatMul BankOp test requires 128-bit bank rows and bank entries a multiple " + "of 64 (got width=${_width}, lines=${_lines})") +endif() +math(EXPR BB_SMM_M "${_lines} / 4") +add_buckyball_mlir_test(smatmul_bias_two_terms TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/prefab/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms.mlir.in b/examples/prefab/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms.mlir.in new file mode 100644 index 000000000..9ad6fefcb --- /dev/null +++ b/examples/prefab/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms.mlir.in @@ -0,0 +1,69 @@ +func.func private @check_result(memref<@BB_BANK_LINES@x4xi32>) -> () +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero_i32 = arith.constant 0 : i32 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %four = arith.constant 4 : index + %eight = arith.constant 8 : index + %sixteen = arith.constant 16 : index + %m = arith.constant @BB_SMM_M@ : index + %a = memref.alloc() : memref<@BB_SMM_M@x16xi8> + %w0 = memref.alloc() : memref<16x16xi8> + %w1 = memref.alloc() : memref<16x16xi8> + %bias = memref.alloc() : memref<4x4xi32> + %output = memref.alloc() : memref<@BB_BANK_LINES@x4xi32> + linalg.fill ins(%zero_i32 : i32) outs(%output : memref<@BB_BANK_LINES@x4xi32>) + scf.for %row = %zero to %m step %one { + %slot = arith.remui %row, %sixteen : index + scf.for %col = %zero to %sixteen step %one { + %v = arith.subi %slot, %col : index + %v8 = arith.index_cast %v : index to i8 + memref.store %v8, %a[%row, %col] : memref<@BB_SMM_M@x16xi8> + } + } + scf.for %row = %zero to %sixteen step %one { + %bias_value = arith.subi %row, %eight : index + %bias_i32 = arith.index_cast %bias_value : index to i32 + %bias_row = arith.divui %row, %four : index + %bias_lane = arith.remui %row, %four : index + memref.store %bias_i32, %bias[%bias_row, %bias_lane] : memref<4x4xi32> + scf.for %col = %zero to %sixteen step %one { + %r32 = arith.index_cast %row : index to i32 + %c32 = arith.index_cast %col : index to i32 + %plus32 = arith.addi %r32, %c32 : i32 + %twice32 = arith.addi %r32, %r32 : i32 + %minus32 = arith.subi %twice32, %c32 : i32 + %plus = arith.trunci %plus32 : i32 to i8 + %minus = arith.trunci %minus32 : i32 to i8 + memref.store %plus, %w0[%row, %col] : memref<16x16xi8> + memref.store %minus, %w1[%row, %col] : memref<16x16xi8> + } + } + %a0_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %b0_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %a1_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %b1_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %c_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %a0 = buckyball.bank_mvin %a, %a0_bank <{depth = @BB_SMM_M@ : i64, stride = 1 : i64}> : memref<@BB_SMM_M@x16xi8> + %b0 = buckyball.bank_mvin %w0, %b0_bank <{depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + %a1 = buckyball.bank_mvin %a, %a1_bank <{depth = @BB_SMM_M@ : i64, stride = 1 : i64}> : memref<@BB_SMM_M@x16xi8> + %b1 = buckyball.bank_mvin %w1, %b1_bank <{depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + // The bias occupies the first four rows of C. + %c = buckyball.bank_mvin %bias, %c_bank <{depth = 4 : i64, stride = 1 : i64}> : memref<4x4xi32> + // Two reduction blocks: the second A and B become the second groups. + %a_groups = buckyball.bank_transfer %a1, %a0 + %b_groups = buckyball.bank_transfer %b1, %b0 + %result = smatmul_ball.bank_matmul_i8 %a_groups, %b_groups -> %c <{rows = @BB_SMM_M@ : i64, k = 16 : i64}> + %stored = buckyball.bank_mvout %output, %result <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x4xi32> + func.call @check_result(%output) : (memref<@BB_BANK_LINES@x4xi32>) -> () + buckyball.bank_release %a_groups + buckyball.bank_release %b_groups + buckyball.bank_release %stored + memref.dealloc %a : memref<@BB_SMM_M@x16xi8> + memref.dealloc %w0 : memref<16x16xi8> + memref.dealloc %w1 : memref<16x16xi8> + memref.dealloc %bias : memref<4x4xi32> + memref.dealloc %output : memref<@BB_BANK_LINES@x4xi32> + return %zero_i8 : i8 +} diff --git a/examples/prefab/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms_main.cpp b/examples/prefab/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms_main.cpp new file mode 100644 index 000000000..0d10c7a27 --- /dev/null +++ b/examples/prefab/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms_main.cpp @@ -0,0 +1,24 @@ +#include +#include +#include +extern "C" void check_result(int32_t *, int32_t *data, int64_t offset, + int64_t rows, int64_t lanes, int64_t stride, + int64_t lane_stride) { + if (rows != THREAD_MEM_BANK_ENTRIES || lanes != 4 || stride != 4 || + lane_stride != 1) + exit(1); + for (int matrix_row = 0; matrix_row < THREAD_MEM_BANK_ENTRIES / 4; + ++matrix_row) + for (int column = 0; column < 16; ++column) { + int bank_row = matrix_row * 4 + column / 4; + int bank_lane = column % 4; + int32_t expected = column - 8; + for (int k = 0; k < 16; ++k) + expected += (matrix_row % 16 - k) * ((k + column) + (2 * k - column)); + if (data[offset + bank_row * stride + bank_lane] != expected) { + printf("smatmul bank mismatch row=%d column=%d\n", matrix_row, column); + exit(1); + } + } + printf("smatmul bank PASS rows=%d\n", THREAD_MEM_BANK_ENTRIES); +} diff --git a/examples/prefab/balls/toint8/arch/ToInt8Ball.scala b/examples/prefab/balls/toint8/arch/ToInt8Ball.scala new file mode 100644 index 000000000..880597eb5 --- /dev/null +++ b/examples/prefab/balls/toint8/arch/ToInt8Ball.scala @@ -0,0 +1,369 @@ +package examples.balls.toint8 + +import framework.acc.balldomain.bbus.{BallFactory, BallPorts} +import buckyball.balls.toint8.{Params => ParametersProto} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import hardfloat.{recFNFromFN, INToRecFN, RecFNToIN} +import hardfloat.consts.{round_near_even, tininess_afterRounding} +import framework.lib.arith.Fma + +import framework.acc.balldomain.ball.blink.{BallStatus, BlinkIO, HasBlink} + +@instantiable +class ToInt8Ball( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Module + with HasBlink { + val ball = bp.balls.find(_.id == id).get + + ball.parameters.unpack(classOf[ParametersProto]) + + val f32Funct = ball.instructions + .find(_.mnemonic == "QUANT_F32_TO_I8").get.funct7 + + val i32Funct = ball.instructions + .find(_.mnemonic == "QUANT_I32_TO_I8").get.funct7 + + require(mp.bankWidth == 128, "ToInt8Ball requires 128-bit bank rows") + require( + mp.bankEntries >= 4, + "ToInt8Ball requires at least four bank rows" + ) + require(ball.inBW == 1, "ToInt8Ball requires inBW=1") + require(ball.outBW == 1, "ToInt8Ball requires outBW=1") + require((f32Funct >> 4) == 3, "QUANT_F32_TO_I8 must encode one read and one write") + require((i32Funct >> 4) == 4, "QUANT_I32_TO_I8 must encode two reads and one write") + + @public + val blink = IO(new BlinkIO(fp, mp, bp, id)) + + val idle :: scaleReq :: scaleResp :: inputReq :: inputResp :: pack :: packDrain :: writeReq :: writeResp :: complete :: Nil = + Enum(10) + val state = RegInit(idle) + + val robIdReg = RegInit(0.U(log2Up(fp.rob_entries).W)) + val inputBank = RegInit(0.U(fp.bank_id_len.W)) + val scaleBank = RegInit(0.U(fp.bank_id_len.W)) + val outputBank = RegInit(0.U(fp.bank_id_len.W)) + val bases = Reg(Vec(3, UInt(mp.groupIdWidth.W))) + // Every group of a virtual bank is one 16-channel block with the same rows. + val groups = RegInit(0.U(log2Up(mp.bankNum + 1).W)) + val group = RegInit(0.U(log2Up(mp.bankNum + 1).W)) + val outputBase = RegInit(0.U(log2Ceil(mp.bankEntries).W)) + val iterReg = RegInit(0.U(fp.iter_len.W)) + val inputBaseReg = RegInit(0.U(log2Ceil(mp.bankEntries).W)) + // Relative row within the command. inputBaseReg is the absolute bank row. + val relativeRow = RegInit(0.U(log2Ceil(mp.bankEntries).W)) + val outputRow = RegInit(0.U(log2Ceil(mp.bankEntries).W)) + val outputWidth = RegInit(0.U(7.W)) + val outputStride = RegInit(0.U(7.W)) + val outputColumn = RegInit(0.U(7.W)) + val scaleRow = RegInit(0.U(2.W)) + val i32Mode = RegInit(false.B) + val reluReg = RegInit(false.B) + val tensorScale = RegInit(0.U(32.W)) + val scales = Reg(Vec(16, UInt(32.W))) + val inputData = Reg(UInt(128.W)) + val outputData = RegInit(0.U(128.W)) + val writeData = Reg(UInt(128.W)) + val lane = RegInit(0.U(2.W)) + val outputBytes = Reg(Vec(4, UInt(8.W))) + + def positiveFinite(value: UInt): Bool = + !value(31) && value(30, 23) =/= 255.U && value(30, 0) =/= 0.U + + val intToFloat = Module(new INToRecFN(32, 8, 24)) + + // The unpipelined recoded-float multiplier was the critical Buckyball path + // (about 8.5 ns before clock uncertainty). FMA mode with c=0 preserves the + // multiplication result while splitting the datapath into three stages. + val multiply = Instantiate(new Fma(3, 8, 24)) + + val floatToInt = Module(new RecFNToIN(8, 24, 8)) + val input = (inputData >> (lane << 5))(31, 0) + val scale = Mux(i32Mode, scales(Cat(relativeRow(1, 0), lane)), tensorScale) + val packIssue = state === pack + + val inputStage0 = RegEnable(input, packIssue) + val scaleStage0 = RegEnable(scale, packIssue) + val laneStage0 = RegEnable(lane, packIssue) + val convertValid = RegNext(packIssue, false.B) + + val multiplyInput = RegEnable( + Mux(i32Mode, intToFloat.io.out, recFNFromFN(8, 24, inputStage0)), + convertValid + ) + + val multiplyScale = RegEnable(recFNFromFN(8, 24, scaleStage0), convertValid) + val laneStage1 = RegEnable(laneStage0, convertValid) + val multiplyValid = RegNext(convertValid, false.B) + + val multiplyResult = RegEnable(multiply.io.out, multiply.io.validout) + val multiplyLane = Pipe(multiplyValid, laneStage1, 3) + val laneStage2 = RegEnable(multiplyLane.bits, multiply.io.validout) + val toIntValid = RegNext(multiply.io.validout, false.B) + + val quantizedResult = RegEnable( + Mux(reluReg && floatToInt.io.out(7), 0.U, floatToInt.io.out), + toIntValid + ) + + val quantizedLane = RegEnable(laneStage2, toIntValid) + val quantizedValid = RegNext(toIntValid, false.B) + + intToFloat.io.signedIn := true.B + intToFloat.io.in := inputStage0 + intToFloat.io.roundingMode := round_near_even + intToFloat.io.detectTininess := tininess_afterRounding + + multiply.io.a := multiplyInput + multiply.io.b := multiplyScale + multiply.io.c := recFNFromFN(8, 24, 0.U) + multiply.io.op := 0.U + multiply.io.validin := multiplyValid + multiply.io.roundingMode := round_near_even + multiply.io.detectTininess := tininess_afterRounding.asUInt + + floatToInt.io.in := multiplyResult + floatToInt.io.roundingMode := round_near_even + floatToInt.io.signedOut := true.B + + blink.cmdReq.ready := state === idle + blink.cmdResp.valid := state === complete + blink.cmdResp.bits.rob_id := robIdReg + blink.status.idle := state === idle + blink.status.running := state =/= idle && state =/= complete + + blink.bankRead(0).rob_id := robIdReg + blink.bankRead(0).bank_id := Mux(state === scaleReq || state === scaleResp, scaleBank, inputBank) + blink.bankRead(0).group_id := Mux(state === scaleReq || state === scaleResp, bases(1), bases(0)) + group + blink.bankRead(0).io.req.valid := false.B + blink.bankRead(0).io.req.bits.addr := 0.U + blink.bankRead(0).io.resp.ready := false.B + + blink.bankWrite(0).rob_id := robIdReg + blink.bankWrite(0).bank_id := outputBank + blink.bankWrite(0).group_id := bases(2) + group + blink.bankWrite(0).io.req.valid := false.B + blink.bankWrite(0).io.req.bits.addr := outputRow + blink.bankWrite(0).io.req.bits.data := writeData + blink.bankWrite(0).io.req.bits.mask := VecInit( + Seq.fill(mp.bankMaskLen)(true.B) + ) + blink.bankWrite(0).io.resp.ready := false.B + + switch(state) { + is(idle) { + when(blink.cmdReq.fire) { + val cmd = blink.cmdReq.bits + val isF32 = cmd.funct === f32Funct.U + val isI32 = cmd.funct === i32Funct.U + + assert(isF32 || isI32, "ToInt8Ball received an unknown funct7") + assert(cmd.iter > 0.U && cmd.iter(1, 0) === 0.U, "ToInt8Ball iter must be a positive multiple of four") + assert( + cmd.iter <= mp.bankEntries.U, + "ToInt8Ball input exceeds bank depth" + ) + assert( + cmd.op1_col =/= 0.U && cmd.op1_col === cmd.wr_col && cmd.op1_col <= mp.bankNum.U, + "ToInt8Ball input and output must have the same channel groups" + ) + assert( + cmd.op1_bank =/= cmd.wr_bank, + "ToInt8Ball input and output banks must differ" + ) + when(isF32) { + assert(cmd.rs2(63, 32) === 0.U, "QUANT_F32_TO_I8 reserves rs2[63:32]") + assert(positiveFinite(cmd.rs2(31, 0)), "QUANT_F32_TO_I8 scale must be finite and positive") + }.otherwise { + assert(cmd.op2_col === cmd.op1_col, "QUANT_I32_TO_I8 scale must have one group per channel group") + assert( + cmd.op1_bank =/= cmd.op2_bank, + "QUANT_I32_TO_I8 input and scale banks must differ" + ) + assert( + cmd.op2_bank =/= cmd.wr_bank, + "QUANT_I32_TO_I8 scale and output banks must differ" + ) + val outputBase = cmd.rs2(7, 1) + val width = cmd.rs2(14, 8) + val height = cmd.rs2(21, 15) + val stride = cmd.rs2(28, 22) + val outputRows = cmd.iter >> 2 + val outputEnd = outputBase +& ((height - 1.U) * stride) +& width + val inputBase = cmd.rs2(34, 29) + assert(cmd.rs2(63, 35) === 0.U, "QUANT_I32_TO_I8 reserves rs2[63:35]") + assert(width > 0.U && height > 0.U && stride >= width, "QUANT_I32_TO_I8 output geometry is invalid") + assert(width * height === outputRows, "QUANT_I32_TO_I8 iter does not match output geometry") + assert( + outputEnd <= mp.bankEntries.U, + "QUANT_I32_TO_I8 output exceeds bank depth" + ) + assert( + inputBase +& cmd.iter <= mp.bankEntries.U, + "QUANT_I32_TO_I8 input exceeds bank depth" + ) + } + + robIdReg := blink.cmdReq.bits.rob_id + inputBank := cmd.op1_bank + scaleBank := cmd.op2_bank + outputBank := cmd.wr_bank + bases := VecInit(cmd.op1_group, cmd.op2_group, cmd.wr_group) + groups := cmd.op1_col + group := 0.U + outputBase := Mux(isI32, cmd.rs2(7, 1), 0.U) + iterReg := cmd.iter + inputBaseReg := Mux(isI32, cmd.rs2(34, 29), 0.U) + relativeRow := 0.U + outputRow := Mux(isI32, cmd.rs2(7, 1), 0.U) + outputWidth := Mux(isI32, cmd.rs2(14, 8), 0.U) + outputStride := Mux(isI32, cmd.rs2(28, 22), 0.U) + outputColumn := 0.U + scaleRow := 0.U + i32Mode := isI32 + reluReg := isI32 && cmd.rs2(0) + tensorScale := cmd.rs2(31, 0) + outputData := 0.U + lane := 0.U + state := Mux(isI32, scaleReq, inputReq) + } + } + + is(scaleReq) { + blink.bankRead(0).io.req.valid := true.B + blink.bankRead(0).io.req.bits.addr := scaleRow + when(blink.bankRead(0).io.req.fire) { + state := scaleResp + } + } + + is(scaleResp) { + blink.bankRead(0).io.resp.ready := true.B + when(blink.bankRead(0).io.resp.fire) { + for (lane <- 0 until 4) { + val value = blink.bankRead(0).io.resp.bits.data(32 * lane + 31, 32 * lane) + assert(positiveFinite(value), "QUANT_I32_TO_I8 scales must be finite and positive") + scales(Cat(scaleRow, lane.U(2.W))) := value + } + when(scaleRow === 3.U) { + state := inputReq + }.otherwise { + scaleRow := scaleRow + 1.U + state := scaleReq + } + } + } + + is(inputReq) { + blink.bankRead(0).io.req.valid := true.B + blink.bankRead(0).io.req.bits.addr := inputBaseReg +& relativeRow + when(blink.bankRead(0).io.req.fire) { + state := inputResp + } + } + + is(inputResp) { + blink.bankRead(0).io.resp.ready := true.B + when(blink.bankRead(0).io.resp.fire) { + inputData := blink.bankRead(0).io.resp.bits.data + lane := 0.U + state := pack + } + } + + is(pack) { + when(!i32Mode) { + assert(input(30, 23) =/= 255.U, "QUANT_F32_TO_I8 input must be finite") + } + when(lane === 3.U) { + state := packDrain + }.otherwise { + lane := lane + 1.U + } + } + + is(packDrain) {} + + is(writeReq) { + blink.bankWrite(0).io.req.valid := true.B + when(blink.bankWrite(0).io.req.fire) { + state := writeResp + } + } + + is(writeResp) { + blink.bankWrite(0).io.resp.ready := true.B + when(blink.bankWrite(0).io.resp.fire) { + when(relativeRow === iterReg - 1.U) { + when(group + 1.U === groups) { + state := complete + }.otherwise { + group := group + 1.U + relativeRow := 0.U + outputRow := outputBase + outputColumn := 0.U + scaleRow := 0.U + state := Mux(i32Mode, scaleReq, inputReq) + } + }.otherwise { + relativeRow := relativeRow + 1.U + when(outputColumn === outputWidth - 1.U) { + outputRow := outputRow + outputStride - outputWidth + 1.U + outputColumn := 0.U + }.otherwise { + outputRow := outputRow + 1.U + outputColumn := outputColumn + 1.U + } + state := inputReq + } + } + } + + is(complete) { + when(blink.cmdResp.fire) { + state := idle + } + } + } + + when((state === pack || state === packDrain) && quantizedValid) { + outputBytes(quantizedLane) := quantizedResult + when(quantizedLane === 3.U) { + val bytes = Cat(quantizedResult, outputBytes(2), outputBytes(1), outputBytes(0)) + val nextWord = outputData | (bytes << (relativeRow(1, 0) << 5)) + when(relativeRow(1, 0) === 3.U) { + writeData := nextWord + outputData := 0.U + state := writeReq + }.otherwise { + outputData := nextWord + relativeRow := relativeRow + 1.U + state := inputReq + } + } + } +} + +object ToInt8Ball extends BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts = + BallPorts(Instantiate(new ToInt8Ball(fp, mp, bp, id)).blink, None) + +} diff --git a/examples/prefab/balls/toint8/configs/default.toml b/examples/prefab/balls/toint8/configs/default.toml new file mode 100644 index 000000000..8166ca7f3 --- /dev/null +++ b/examples/prefab/balls/toint8/configs/default.toml @@ -0,0 +1 @@ +# ToInt8Ball has no private configuration parameters. diff --git a/examples/prefab/balls/toint8/configs/toint8_params.proto b/examples/prefab/balls/toint8/configs/toint8_params.proto new file mode 100644 index 000000000..5287e6689 --- /dev/null +++ b/examples/prefab/balls/toint8/configs/toint8_params.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; +package buckyball.balls.toint8; +option java_package = "buckyball.balls.toint8"; +option java_multiple_files = true; + +message Params { +} diff --git a/examples/prefab/balls/toint8/emu/src/dispatch.rs b/examples/prefab/balls/toint8/emu/src/dispatch.rs new file mode 100644 index 000000000..d0d2f3b57 --- /dev/null +++ b/examples/prefab/balls/toint8/emu/src/dispatch.rs @@ -0,0 +1,33 @@ +use bebop_bemu::ball::ExecContext; + +use super::quant; + +const BALL_CLASS: &str = "examples.balls.toint8.ToInt8Ball"; + +pub fn execute_known( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, + ctx: &mut ExecContext, +) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("QUANT_F32_TO_I8") => Some(quant::exec_f32_to_i8(xs1, xs2, ctx)), + Some("QUANT_I32_TO_I8") => Some(quant::exec_i32_to_i8(xs1, xs2, ctx)), + Some(_) | None => None, + } +} + +pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("QUANT_F32_TO_I8") => Some(quant::f32_to_i8_latency(xs1, xs2)), + Some("QUANT_I32_TO_I8") => Some(quant::i32_to_i8_latency(xs1, xs2)), + Some(_) | None => None, + } +} diff --git a/examples/prefab/balls/toint8/emu/src/lib.rs b/examples/prefab/balls/toint8/emu/src/lib.rs new file mode 100644 index 000000000..4de11c376 --- /dev/null +++ b/examples/prefab/balls/toint8/emu/src/lib.rs @@ -0,0 +1,13 @@ +pub(crate) use bebop_bemu::thread::instructions::decode; +pub(crate) use bebop_bemu::ball as instruction; + +mod model; +mod quant; + +mod dispatch; + +pub use dispatch::{cycles_after_issue, execute_known}; + +#[bebop_bemu::ball::linkme::distributed_slice(bebop_bemu::BALLS)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static BALL: bebop_bemu::Ball = bebop_bemu::Ball { execute: execute_known, cycles: cycles_after_issue }; diff --git a/examples/balls/toint8/emu/src/model.rs b/examples/prefab/balls/toint8/emu/src/model.rs similarity index 100% rename from examples/balls/toint8/emu/src/model.rs rename to examples/prefab/balls/toint8/emu/src/model.rs diff --git a/examples/prefab/balls/toint8/emu/src/quant.rs b/examples/prefab/balls/toint8/emu/src/quant.rs new file mode 100644 index 000000000..117290472 --- /dev/null +++ b/examples/prefab/balls/toint8/emu/src/quant.rs @@ -0,0 +1,178 @@ +use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old}; +use super::instruction::ExecContext; +use bebop_bemu::thread::banks::bank_lines; + +pub(crate) fn exec_f32_to_i8(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + assert!(!rs1_read_old(xs1), "quant_f32_to_i8: reads no old destination; rs1[29] must be clear"); + let src = rs1_b0(xs1); + let unused = rs1_b1(xs1); + let dst = rs1_b2(xs1); + let iter = rs1_iter(xs1) as usize; + if unused != 0 { + panic!("quant_f32_to_i8: input bank 1 must be zero"); + } + if src == dst { + panic!("quant_f32_to_i8: input and output banks must differ"); + } + if !ctx.config(src).allocated || !ctx.config(dst).allocated { + panic!("quant_f32_to_i8: bank not allocated"); + } + let groups = ctx.config(src).cols; + if groups == 0 || ctx.config(dst).cols != groups { + panic!("quant_f32_to_i8: input and output must have the same channel groups"); + } + if iter == 0 || iter % 4 != 0 || iter > bank_lines() { + panic!("quant_f32_to_i8: iter must be a positive multiple of four within bank depth"); + } + if xs2 >> 32 != 0 { + panic!("quant_f32_to_i8: rs2[63:32] must be zero"); + } + let scale = f32::from_bits(xs2 as u32); + if !scale.is_finite() || scale <= 0.0 { + panic!("quant_f32_to_i8: scale must be finite and positive"); + } + + // Group g of every operand is channel block g; each block uses the same rows. + for channel_group in 0..groups { + let ps = pbank_group(ctx, src, channel_group); + let pd = pbank_group(ctx, dst, channel_group); + let (input, output) = ctx.banks.read_write(ps, pd); + for output_row in 0..iter / 4 { + let mut packed = [0u8; 16]; + for group in 0..4 { + let input_row = output_row * 4 + group; + let base = input_row * 16; + for lane in 0..4 { + let offset = base + lane * 4; + let bits = u32::from_le_bytes(input[offset..offset + 4].try_into().unwrap()); + packed[group * 4 + lane] = + super::model::quantize_f32(f32::from_bits(bits), scale, false) as u8; + } + } + let base = output_row * 16; + output[base..base + 16].copy_from_slice(&packed); + } + } + 0 +} + +pub(crate) fn exec_i32_to_i8(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + assert!(!rs1_read_old(xs1), "quant_i32_to_i8: reads no old destination; rs1[29] must be clear"); + let src = rs1_b0(xs1); + let scale_bank = rs1_b1(xs1); + let dst = rs1_b2(xs1); + let iter = rs1_iter(xs1) as usize; + if src == scale_bank || src == dst || scale_bank == dst { + panic!("quant_i32_to_i8: input, scale, and output banks must differ"); + } + if !ctx.config(src).allocated || !ctx.config(scale_bank).allocated || !ctx.config(dst).allocated + { + panic!("quant_i32_to_i8: bank not allocated"); + } + let groups = ctx.config(src).cols; + if groups == 0 || ctx.config(scale_bank).cols != groups || ctx.config(dst).cols != groups { + panic!("quant_i32_to_i8: input, scale, and output must have the same channel groups"); + } + if iter == 0 || iter % 4 != 0 || iter > bank_lines() { + panic!("quant_i32_to_i8: iter must be a positive multiple of four within bank depth"); + } + if xs2 >> 35 != 0 { + panic!("quant_i32_to_i8: rs2[63:35] must be zero"); + } + let input_base = ((xs2 >> 29) & 0x3f) as usize; + let output_base = ((xs2 >> 1) & 0x7f) as usize; + let output_width = ((xs2 >> 8) & 0x7f) as usize; + let output_height = ((xs2 >> 15) & 0x7f) as usize; + let output_stride = ((xs2 >> 22) & 0x7f) as usize; + let output_rows = iter / 4; + if output_width == 0 + || output_height == 0 + || output_stride < output_width + || output_width.checked_mul(output_height) != Some(output_rows) + || output_base + .checked_add((output_height - 1) * output_stride + output_width) + .is_none_or(|end| end > bank_lines()) + || input_base + iter > bank_lines() + { + panic!("quant_i32_to_i8: invalid output core"); + } + let relu = xs2 & 1 != 0; + + // Group g of every operand is channel block g; each block uses the same rows. + for channel_group in 0..groups { + let pscale = pbank_group(ctx, scale_bank, channel_group); + let mut scales = [0.0f32; 16]; + for group in 0..4 { + let base = group * 16; + for lane in 0..4 { + let offset = base + lane * 4; + scales[group * 4 + lane] = f32::from_bits(u32::from_le_bytes( + ctx.banks[pscale][offset..offset + 4].try_into().unwrap(), + )); + if !scales[group * 4 + lane].is_finite() || scales[group * 4 + lane] <= 0.0 { + panic!("quant_i32_to_i8: scales must be finite and positive"); + } + } + } + + let ps = pbank_group(ctx, src, channel_group); + let pd = pbank_group(ctx, dst, channel_group); + let (input, output) = ctx.banks.read_write(ps, pd); + for output_row in 0..iter / 4 { + let mut packed = [0u8; 16]; + for group in 0..4 { + let input_row = input_base + output_row * 4 + group; + let base = input_row * 16; + for lane in 0..4 { + let offset = base + lane * 4; + let value = i32::from_le_bytes(input[offset..offset + 4].try_into().unwrap()); + let channel = group * 4 + lane; + packed[channel] = + super::model::quantize_i32(value, scales[channel], relu) as u8; + } + } + let row = output_base + + (output_row / output_width) * output_stride + + output_row % output_width; + let base = row * 16; + output[base..base + 16].copy_from_slice(&packed); + } + } + 0 +} + +pub(crate) fn f32_to_i8_latency(xs1: u64, xs2: u64) -> u64 { + let iter = rs1_iter(xs1); + if iter == 0 || iter % 4 != 0 || xs2 >> 32 != 0 { + panic!("quant_f32_to_i8: illegal encoding"); + } + let scale = f32::from_bits(xs2 as u32); + if !scale.is_finite() || scale <= 0.0 { + panic!("quant_f32_to_i8: scale must be finite and positive"); + } + iter +} + +pub(crate) fn i32_to_i8_latency(xs1: u64, xs2: u64) -> u64 { + let iter = rs1_iter(xs1); + let input_base = (xs2 >> 29) & 0x3f; + if xs2 >> 35 != 0 { + panic!("quant_i32_to_i8: illegal encoding"); + } + let output_base = (xs2 >> 1) & 0x7f; + let output_width = (xs2 >> 8) & 0x7f; + let output_height = (xs2 >> 15) & 0x7f; + let output_stride = (xs2 >> 22) & 0x7f; + if iter == 0 + || iter % 4 != 0 + || output_width == 0 + || output_height == 0 + || output_stride < output_width + || output_width * output_height != iter / 4 + || output_base + (output_height - 1) * output_stride + output_width > bank_lines() as u64 + || input_base + iter > bank_lines() as u64 + { + panic!("quant_i32_to_i8: illegal encoding"); + } + iter + 4 +} diff --git a/examples/prefab/balls/toint8/stack/src/toint8Ops.cpp b/examples/prefab/balls/toint8/stack/src/toint8Ops.cpp new file mode 100644 index 000000000..ae3cba72c --- /dev/null +++ b/examples/prefab/balls/toint8/stack/src/toint8Ops.cpp @@ -0,0 +1,104 @@ +//===- toint8Ops.cpp - ToInt8Ball operations ------------------------------===// +// +// The dialect of ToInt8Ball, its virtual operations building their physical +// ones and the instructions of the physical operations. +// +//===----------------------------------------------------------------------===// + +#include "toint8Ops.h" + +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +#include "mlir/Dialect/Arith/IR/Arith.h" + +using namespace mlir; +using namespace ::buddy::toint8_ball; +using mlir::buddy::bankFields; +using mlir::buddy::constantI64; +using mlir::buddy::emitInstruction; +using mlir::buddy::field; + +#include "toint8Dialect.cpp.inc" + +#define GET_OP_CLASSES +#include "toint8Ops.cpp.inc" + +void ToInt8BallDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "toint8Ops.cpp.inc" + >(); +} + +namespace { +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallDialect([](DialectRegistry ®istry) { + registry.insert(); + }), + true); +} // namespace + +SmallVector BankFp32ToInt8Op::getSourceBanks() { + return {getInput()}; +} +Value BankFp32ToInt8Op::getDestinationBank() { return getOutput(); } +bool BankFp32ToInt8Op::readsDestination() { return false; } +Operation *BankFp32ToInt8Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return Fp32ToInt8Op::create(builder, getLoc(), getMultiplier(), sources[0], + destination, getRows()); +} + +SmallVector Fp32ToInt8Op::getSourceBanks() { + return {int64_t(getInput())}; +} +int64_t Fp32ToInt8Op::getDestinationBank() { return getOutput(); } +LogicalResult Fp32ToInt8Op::lowerToInstructions(OpBuilder &builder) { + Location loc = getLoc(); + uint64_t rs1 = + bankFields({int64_t(getInput())}, getOutput(), false, getRows()); + Value bits = arith::BitcastOp::create(builder, loc, builder.getI32Type(), + getMultiplier()); + Value rs2 = arith::ExtUIOp::create(builder, loc, builder.getI64Type(), bits); + emitInstruction(builder, loc, + buckyball_target::getBuckyballFunct7("QUANT_F32_TO_I8"), + constantI64(builder, loc, rs1), rs2); + return success(); +} + +SmallVector BankInt32ToInt8Op::getSourceBanks() { + return {getValues(), getScales()}; +} +Value BankInt32ToInt8Op::getDestinationBank() { return getOutput(); } +bool BankInt32ToInt8Op::readsDestination() { return false; } +Operation *BankInt32ToInt8Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return Int32ToInt8Op::create(builder, getLoc(), sources[0], sources[1], + destination, getRows(), getRelu()); +} + +SmallVector Int32ToInt8Op::getSourceBanks() { + return {int64_t(getValues()), int64_t(getScales())}; +} +int64_t Int32ToInt8Op::getDestinationBank() { return getOutput(); } +LogicalResult Int32ToInt8Op::lowerToInstructions(OpBuilder &builder) { + Location loc = getLoc(); + int64_t rows = getRows(); + int64_t depth = buckyball_target::getBuckyballTarget().bankDepth; + if (rows <= 0 || rows * 4 > depth) + return emitOpError("INT32-to-INT8 rows exceed the bank depth"); + // The output rows start at the base of the destination, packed; the + // seven-bit height takes 128 rows as two columns of 64. + int64_t width = rows > 127 ? 2 : 1; + uint64_t rs1 = bankFields({int64_t(getValues()), int64_t(getScales())}, + getOutput(), false, rows * 4); + uint64_t rs2 = uint64_t(getRelu()) | (uint64_t(width) << 8) | + (uint64_t(rows / width) << 15) | (uint64_t(width) << 22); + emitInstruction( + builder, loc, buckyball_target::getBuckyballFunct7("QUANT_I32_TO_I8"), + constantI64(builder, loc, rs1), constantI64(builder, loc, rs2)); + return success(); +} diff --git a/examples/prefab/balls/toint8/stack/src/toint8Ops.h b/examples/prefab/balls/toint8/stack/src/toint8Ops.h new file mode 100644 index 000000000..a4813ed0f --- /dev/null +++ b/examples/prefab/balls/toint8/stack/src/toint8Ops.h @@ -0,0 +1,20 @@ +//===- toint8Ops.h - ToInt8Ball operations --------------------------------===// + +#ifndef TOINT8_OPS_H +#define TOINT8_OPS_H + +#include "Interfaces/BankInterfaces.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "toint8Dialect.h.inc" + +#define GET_OP_CLASSES +#include "toint8Ops.h.inc" + +#endif // TOINT8_OPS_H diff --git a/examples/prefab/balls/toint8/stack/src/toint8Ops.td b/examples/prefab/balls/toint8/stack/src/toint8Ops.td new file mode 100644 index 000000000..7eb70d541 --- /dev/null +++ b/examples/prefab/balls/toint8/stack/src/toint8Ops.td @@ -0,0 +1,67 @@ +//===- toint8Ops.td - ToInt8Ball operations ----------------*- tablegen -*-===// +// +// fp32_to_int8: the input bank holds `rows` rows of four f32, a multiple of +// four; the output receives rows / 4 rows of sixteen i8, +// clamp(round_half_even(x * multiplier), -128, 127). `multiplier` is f32. +// int32_to_int8: `values` holds `rows` rows of sixteen i32 (four lines each), +// `scales` the sixteen f32 scales of the columns (four lines); the output +// receives `rows` rows of sixteen i8, clamp(round_half_even(x * scale), -128, +// 127), with `relu` at least zero. +// Both work on every group: group g is channel block g, and the banks have the +// same number of groups. +// +//===----------------------------------------------------------------------===// + +#ifndef TOINT8_OPS +#define TOINT8_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Interfaces/BankInterfaces.td" + +def ToInt8Ball_Dialect : Dialect { + let name = "toint8_ball"; + let cppNamespace = "::buddy::toint8_ball"; + let summary = "The operations of ToInt8Ball"; +} + +class ToInt8Ball_Op traits = []> + : Op])>; + +def BankFp32ToInt8Op : ToInt8Ball_Op<"bank_fp32_to_int8", + [DeclareOpInterfaceMethods]> { + let summary = "fp32_to_int8 on virtual banks"; + let arguments = (ins I64:$input, I64:$output, F32:$multiplier, + I64Attr:$rows); + let results = (outs I64:$outputOut); + let assemblyFormat = "$input `->` $output `multiplier` `(` $multiplier `)` " + "prop-dict attr-dict"; +} + +def Fp32ToInt8Op : ToInt8Ball_Op<"fp32_to_int8", + [DeclareOpInterfaceMethods]> { + let summary = "fp32_to_int8 on physical banks"; + let arguments = (ins F32:$multiplier, I64Attr:$input, I64Attr:$output, + I64Attr:$rows); + let assemblyFormat = "`multiplier` `(` $multiplier `)` prop-dict attr-dict"; +} + +def BankInt32ToInt8Op : ToInt8Ball_Op<"bank_int32_to_int8", + [DeclareOpInterfaceMethods]> { + let summary = "int32_to_int8 on virtual banks"; + let arguments = (ins I64:$values, I64:$scales, I64:$output, I64Attr:$rows, + UnitAttr:$relu); + let results = (outs I64:$outputOut); + let assemblyFormat = "$values `,` $scales `->` $output prop-dict attr-dict"; +} + +def Int32ToInt8Op : ToInt8Ball_Op<"int32_to_int8", + [DeclareOpInterfaceMethods]> { + let summary = "int32_to_int8 on physical banks"; + let arguments = (ins I64Attr:$values, I64Attr:$scales, I64Attr:$output, + I64Attr:$rows, UnitAttr:$relu); + let assemblyFormat = "prop-dict attr-dict"; +} + +#endif // TOINT8_OPS diff --git a/examples/prefab/balls/toint8/verify/README.md b/examples/prefab/balls/toint8/verify/README.md new file mode 100644 index 000000000..b3852149b --- /dev/null +++ b/examples/prefab/balls/toint8/verify/README.md @@ -0,0 +1,95 @@ +# ToInt8Ball UVM Verification + +Verifies the generated `ToInt8Ball` module using the shared Blink UVM framework. + +## Structure + +- `../../../../verify/uvm/src/ball/bb_uvm_pkg.sv`: common Blink UVM agents and base env +- `src/common/toint8_defs.svh`: DPI imports, `toint8_require_bid`, timeouts +- `src/common/toint8_items.svh`: `toint8_cmd_item` with `load_rust_case` (Da address in `special[12:0]`) +- `src/seq/toint8_sequences.svh`: one-case sequence +- `src/cov/toint8_cov.svh`: online INT8 layout and iter 1 coverage +- `src/env/toint8_scoreboard.svh`: preload mem model (group-aware for i8), compare writes vs DPI ref +- `src/env/toint8_env.svh`: extends `bb_blink_env#(1,1)` +- `src/tests/toint8_*_test.svh`: one directed case per UVM test +- `src/pkg/toint8_pkg.sv`: package entry +- `src/tb_top.sv`: `bb_blink_if#(1,1)` + `ToInt8Ball` +- `filelists/toint8_ball.f`: VCS filelist + +## Test plan + +The test names are: + +- `toint8_signed_test`: signed values across all four source groups +- `toint8_zero_test`: all-zero activation (`Da=1.0`) +- `toint8_rounding_test`: ties-to-even quantization boundaries +- `toint8_rows_test`: two-row scan and packed output traversal +- `toint8_scale_rows_test`: maximum value is in the second row +- case 8: QUANT_F32_TO_I8 over two channel groups (group g of input/output = channel block g) +- case 9: QUANT_I32_TO_I8 over three channel groups with per-group scales, input_base 3, and a 2x2 output core + +Each case: reset, preload src words into `mem_model` (INT8 uses `group_id` 0..3), +drive one command, wait for `scb.done()` or timeout at 400 cycles. Scoreboard +compares observed writes with DPI `toint8_ref_i8`. `toint8_cov` requires the online +INT8 layout and one-row iteration bin, else `check_phase` fatal. + +## BID (required) + +`+BID=` is mandatory. Missing it fatals at test start. No default bid. + +| Config | Plusarg | +|--------|---------| +| `sims.verilator.BuckyballPebbleVerilatorConfig` | `+BID=3` | +| `sims.verilator.BuckyballToyVerilatorConfig` (full.toml) | `+BID=5` | + +```console +./simv +UVM_TESTNAME=toint8_scale_rows_test +BID=3 +``` + +## Build + +Enter the shared UVM/VCS environment: + +```console +nix develop ../../../../verify +``` + +Build the DPI reference library: + +```console +cargo build --manifest-path casegen/Cargo.toml +``` + +Compile from this directory: + +```console +vcs -full64 -sverilog -timescale=1ns/1ps \ + $VCS_UVM_ARGS \ + -sv_lib ../output/casegen/debug/libtoint8_casegen \ + -f filelists/toint8_ball.f +``` + +Run: + +```console +./simv +UVM_TESTNAME=toint8_scale_rows_test +BID=3 +``` + +Or via bbdev: + +```console +bbdev uvm --build '--ball=toint8' \ + --config sims.verilator.BuckyballPebbleVerilatorConfig \ + --core-config examples/cores/pebble/configs/default.toml +bbdev uvm --run '--ball=toint8 --plusargs +BID=3' +``` + +## Acceptance (2026-08-04) + +- [x] Layout matches Blink common + ball-local split; filelist uses `@UVM@` / `@RTL@` +- [x] casegen reuses toint8 model; whole-case DPI API with injected bid +- [x] directed online INT8 scoreboard pass +- [x] signed, zero, rounding, and row-traversal directed cases pass +- [x] protocol + semantic cover enforced (`check_phase` fatal) +- [x] pebble: `+BID=3` config `sims.verilator.BuckyballPebbleVerilatorConfig` +- [ ] toy full: `+BID=5` config `sims.verilator.BuckyballToyVerilatorConfig` diff --git a/examples/balls/toint8/verify/casegen/Cargo.lock b/examples/prefab/balls/toint8/verify/casegen/Cargo.lock similarity index 100% rename from examples/balls/toint8/verify/casegen/Cargo.lock rename to examples/prefab/balls/toint8/verify/casegen/Cargo.lock diff --git a/examples/balls/toint8/verify/casegen/Cargo.toml b/examples/prefab/balls/toint8/verify/casegen/Cargo.toml similarity index 100% rename from examples/balls/toint8/verify/casegen/Cargo.toml rename to examples/prefab/balls/toint8/verify/casegen/Cargo.toml diff --git a/examples/prefab/balls/toint8/verify/casegen/src/casegen.rs b/examples/prefab/balls/toint8/verify/casegen/src/casegen.rs new file mode 100644 index 000000000..74fde3292 --- /dev/null +++ b/examples/prefab/balls/toint8/verify/casegen/src/casegen.rs @@ -0,0 +1,567 @@ +use crate::model::{quantize_f32, quantize_i32}; + +pub const KIND_F32: u32 = 0; +pub const KIND_I32: u32 = 1; +pub const BANK_NUM: u32 = 24; +pub const BANK_ENTRIES: u32 = 64; +pub const SCALE_ROWS: usize = 4; +pub const LANES: usize = 4; +// Words are stored group-major: all rows of group 0, then group 1, ... +pub const MAX_SRC: usize = 48; +pub const MAX_SCALE: usize = 12; +pub const MAX_DST: usize = 12; + +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ToInt8Cmd { + pub kind: u32, + pub iter: u32, + pub op1_bank: u32, + pub op2_bank: u32, + pub wr_bank: u32, + pub op1_group: u32, + pub op2_group: u32, + pub wr_group: u32, + pub op1_col: u32, + pub op2_col: u32, + pub wr_col: u32, + pub rob_id: u32, + pub rs2_lo: u32, + pub rs2_hi: u32, + pub input_base: u32, + pub num_src_words: u32, + pub num_scale_words: u32, + pub num_dst_words: u32, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ToInt8Case { + pub cmd: ToInt8Cmd, + pub src_words: [u128; MAX_SRC], + pub scale_words: [u128; MAX_SCALE], + pub dst_words: [u128; MAX_DST], + pub dst_addr: [u32; MAX_DST], +} + +impl ToInt8Case { + pub fn src_lo(&self, index: usize) -> u64 { + self.src_words[index] as u64 + } + + pub fn src_hi(&self, index: usize) -> u64 { + (self.src_words[index] >> 64) as u64 + } + + pub fn scale_lo(&self, index: usize) -> u64 { + self.scale_words[index] as u64 + } + + pub fn scale_hi(&self, index: usize) -> u64 { + (self.scale_words[index] >> 64) as u64 + } + + pub fn dst_lo(&self, index: usize) -> u64 { + self.dst_words[index] as u64 + } + + pub fn dst_hi(&self, index: usize) -> u64 { + (self.dst_words[index] >> 64) as u64 + } +} + +pub fn gen_case(index: u32) -> ToInt8Case { + match index { + 0 => f32_signed(), + 1 => f32_zero(), + 2 => f32_rounding(), + 3 => f32_rows(), + 4 => i32_scale_rows(), + 5 => i32_relu_tile(), + 6 => i32_signed(), + 7 => i32_offset(), + 8 => f32_two_groups(), + 9 => i32_three_groups(), + _ => panic!("toint8: unsupported directed case {index}"), + } +} + +fn pack_u32s(vals: &[u32]) -> Vec { + if vals.len() % LANES != 0 { + panic!("toint8: value count {} is not a multiple of four", vals.len()); + } + vals.chunks(LANES) + .map(|chunk| { + let mut word = 0u128; + for (lane, value) in chunk.iter().enumerate() { + word |= u128::from(*value) << (lane * 32); + } + word + }) + .collect() +} + +fn copy_words(src: &[u128]) -> [u128; N] { + if src.len() > N { + panic!("toint8: {} words exceed buffer {N}", src.len()); + } + let mut out = [0u128; N]; + out[..src.len()].copy_from_slice(src); + out +} + +fn copy_addrs(src: &[u32]) -> [u32; MAX_DST] { + if src.len() > MAX_DST { + panic!("toint8: {} dst rows exceed buffer", src.len()); + } + let mut out = [0u32; MAX_DST]; + out[..src.len()].copy_from_slice(src); + out +} + +fn encode_i32_rs2( + relu: bool, + out_base: u32, + width: u32, + height: u32, + stride: u32, + in_base: u32, +) -> u64 { + u64::from(u32::from(relu)) + | (u64::from(out_base) << 1) + | (u64::from(width) << 8) + | (u64::from(height) << 15) + | (u64::from(stride) << 22) + | (u64::from(in_base) << 29) +} + +fn positive_finite(bits: u32) -> bool { + let value = f32::from_bits(bits); + value.is_finite() && value > 0.0 +} + +fn split_rs(value: u64) -> (u32, u32) { + (value as u32, (value >> 32) as u32) +} + +pub(crate) fn validate( + kind: u32, + op1_bank: u32, + op2_bank: u32, + wr_bank: u32, + op1_col: u32, + op2_col: u32, + wr_col: u32, + iter: u32, + rs2: u64, +) { + if kind != KIND_F32 && kind != KIND_I32 { + panic!("toint8: unknown kind {kind}"); + } + if iter == 0 || iter % 4 != 0 { + panic!("ToInt8Ball iter must be a positive multiple of four"); + } + if iter > BANK_ENTRIES { + panic!("ToInt8Ball input exceeds bank depth"); + } + if op1_bank >= BANK_NUM { + panic!("ToInt8Ball input bank is invalid"); + } + if wr_bank >= BANK_NUM { + panic!("ToInt8Ball output bank is invalid"); + } + if op1_col == 0 || op1_col > BANK_NUM || wr_col != op1_col { + panic!("ToInt8Ball input and output must have the same channel groups"); + } + if op1_bank == wr_bank { + panic!("ToInt8Ball input and output banks must differ"); + } + if kind == KIND_F32 { + if op2_bank != 0 { + panic!("QUANT_F32_TO_I8 reserves input bank 1"); + } + if rs2 >> 32 != 0 { + panic!("QUANT_F32_TO_I8 reserves rs2[63:32]"); + } + if !positive_finite(rs2 as u32) { + panic!("QUANT_F32_TO_I8 scale must be finite and positive"); + } + } else { + if op2_bank >= BANK_NUM { + panic!("QUANT_I32_TO_I8 scale bank is invalid"); + } + if op2_col != op1_col { + panic!("QUANT_I32_TO_I8 scale must have one group per channel group"); + } + if op1_bank == op2_bank { + panic!("QUANT_I32_TO_I8 input and scale banks must differ"); + } + if op2_bank == wr_bank { + panic!("QUANT_I32_TO_I8 scale and output banks must differ"); + } + if rs2 >> 35 != 0 { + panic!("QUANT_I32_TO_I8 reserves rs2[63:35]"); + } + let out_base = ((rs2 >> 1) & 0x7f) as u32; + let width = ((rs2 >> 8) & 0x7f) as u32; + let height = ((rs2 >> 15) & 0x7f) as u32; + let stride = ((rs2 >> 22) & 0x7f) as u32; + let in_base = ((rs2 >> 29) & 0x3f) as u32; + let output_rows = iter / 4; + if width == 0 || height == 0 || stride < width { + panic!("QUANT_I32_TO_I8 output geometry is invalid"); + } + if width.checked_mul(height) != Some(output_rows) { + panic!("QUANT_I32_TO_I8 iter does not match output geometry"); + } + let output_end = out_base + .checked_add((height - 1).checked_mul(stride).expect("stride overflow")) + .and_then(|value| value.checked_add(width)); + match output_end { + Some(end) if end <= BANK_ENTRIES => {} + _ => panic!("QUANT_I32_TO_I8 output exceeds bank depth"), + } + if in_base.checked_add(iter).is_none_or(|end| end > BANK_ENTRIES) { + panic!("QUANT_I32_TO_I8 input exceeds bank depth"); + } + } +} + +fn f32_dst(src: &[u128], scale: f32) -> Vec { + if src.len() % 4 != 0 { + panic!("QUANT_F32_TO_I8 src rows must be a multiple of four"); + } + src.chunks(4) + .map(|group| { + let mut packed = 0u128; + for (g, word) in group.iter().enumerate() { + for lane in 0..LANES { + let bits = (*word >> (lane * 32)) as u32; + if (bits >> 23) & 0xff == 0xff { + panic!("QUANT_F32_TO_I8 input must be finite"); + } + let q = quantize_f32(f32::from_bits(bits), scale, false) as u8; + packed |= u128::from(q) << ((g * LANES + lane) * 8); + } + } + packed + }) + .collect() +} + +fn i32_dst(src: &[u128], scales: &[u128], relu: bool) -> Vec { + if src.len() % 4 != 0 { + panic!("QUANT_I32_TO_I8 src rows must be a multiple of four"); + } + if scales.len() != SCALE_ROWS { + panic!("QUANT_I32_TO_I8 requires four scale rows"); + } + let mut scale_vals = [0.0f32; 16]; + for (row, word) in scales.iter().enumerate() { + for lane in 0..LANES { + let bits = (*word >> (lane * 32)) as u32; + if !positive_finite(bits) { + panic!("QUANT_I32_TO_I8 scales must be finite and positive"); + } + scale_vals[row * LANES + lane] = f32::from_bits(bits); + } + } + src.chunks(4) + .map(|group| { + let mut packed = 0u128; + for (g, word) in group.iter().enumerate() { + for lane in 0..LANES { + let bits = (*word >> (lane * 32)) as u32; + let channel = g * LANES + lane; + let q = quantize_i32(bits as i32, scale_vals[channel], relu) as u8; + packed |= u128::from(q) << (channel * 8); + } + } + packed + }) + .collect() +} + +fn i32_addrs(iter: u32, out_base: u32, width: u32, stride: u32) -> Vec { + let output_rows = iter / 4; + (0..output_rows) + .map(|row| out_base + (row / width) * stride + row % width) + .collect() +} + +// `groups` are the op1/op2/wr base groups. +fn finish( + kind: u32, + groups: [u32; 3], + cols: u32, + iter: u32, + op1_bank: u32, + op2_bank: u32, + wr_bank: u32, + op2_col: u32, + rob_id: u32, + rs2: u64, + input_base: u32, + src: Vec, + scale: Vec, + dst: Vec, + dst_addr: Vec, +) -> ToInt8Case { + let op1_col = cols; + let wr_col = cols; + validate( + kind, op1_bank, op2_bank, wr_bank, op1_col, op2_col, wr_col, iter, rs2, + ); + if src.len() != (iter * cols) as usize { + panic!("toint8: src rows {} != iter {iter} * groups {cols}", src.len()); + } + if dst.len() != (iter / 4 * cols) as usize || dst_addr.len() != dst.len() { + panic!("toint8: dst rows do not match iter/4 per group"); + } + if kind == KIND_F32 { + if !scale.is_empty() { + panic!("QUANT_F32_TO_I8 must not emit scale rows"); + } + } else if scale.len() != SCALE_ROWS * cols as usize { + panic!("QUANT_I32_TO_I8 must emit four scale rows per group"); + } + let (rs2_lo, rs2_hi) = split_rs(rs2); + ToInt8Case { + cmd: ToInt8Cmd { + kind, + iter, + op1_bank, + op2_bank, + wr_bank, + op1_group: groups[0], + op2_group: groups[1], + wr_group: groups[2], + op1_col, + op2_col, + wr_col, + rob_id, + rs2_lo, + rs2_hi, + input_base, + num_src_words: src.len() as u32, + num_scale_words: scale.len() as u32, + num_dst_words: dst.len() as u32, + }, + src_words: copy_words(&src), + scale_words: copy_words(&scale), + dst_words: copy_words(&dst), + dst_addr: copy_addrs(&dst_addr), + } +} + +fn f32_case(groups: [u32; 3], rob_id: u32, scale: f32, values: &[f32]) -> ToInt8Case { + let bits: Vec = values.iter().map(|v| v.to_bits()).collect(); + let src = pack_u32s(&bits); + let iter = src.len() as u32; + let dst = f32_dst(&src, scale); + let dst_addr: Vec = (0..dst.len() as u32).collect(); + finish( + KIND_F32, + groups, + 1, + iter, + 0, + 0, + 1, + 0, + rob_id, + u64::from(scale.to_bits()), + 0, + src, + Vec::new(), + dst, + dst_addr, + ) +} + +fn i32_case( + groups: [u32; 3], + rob_id: u32, + op1_bank: u32, + op2_bank: u32, + wr_bank: u32, + relu: bool, + out_base: u32, + width: u32, + height: u32, + stride: u32, + in_base: u32, + values: &[i32], + scales: &[f32], +) -> ToInt8Case { + let src = pack_u32s(&values.iter().map(|v| *v as u32).collect::>()); + let scale = pack_u32s(&scales.iter().map(|v| v.to_bits()).collect::>()); + let iter = src.len() as u32; + let rs2 = encode_i32_rs2(relu, out_base, width, height, stride, in_base); + let dst = i32_dst(&src, &scale, relu); + let dst_addr = i32_addrs(iter, out_base, width, stride); + finish( + KIND_I32, + groups, + 1, + iter, + op1_bank, + op2_bank, + wr_bank, + 1, + rob_id, + rs2, + in_base, + src, + scale, + dst, + dst_addr, + ) +} + +fn f32_signed() -> ToInt8Case { + f32_case( + [0, 0, 0], + 2, + 0.5, + &[ + -300.0, -255.0, -5.0, -3.0, -1.0, 0.0, 1.0, 3.0, 5.0, 254.0, 255.0, 300.0, 2.0, 6.0, + 10.0, 14.0, + ], + ) +} + +fn f32_zero() -> ToInt8Case { + f32_case([1, 0, 2], 3, 1.0, &[0.0; 16]) +} + +fn f32_rounding() -> ToInt8Case { + f32_case( + [2, 0, 1], + 4, + 1.0, + &[ + 2.5, -2.5, 3.5, -3.5, 0.5, -0.5, 1.5, -1.5, 63.5, -63.5, 127.5, -128.5, 0.0, 1.0, 2.0, + 3.0, + ], + ) +} + +fn f32_rows() -> ToInt8Case { + f32_case( + [3, 0, 3], + 5, + 2.0, + &[ + 0.125, -0.125, 0.25, -0.25, 0.75, -0.75, 1.25, -1.25, 1.75, -1.75, 63.25, 63.75, + -63.75, -64.75, 0.0, -0.0, 2.25, -2.25, 2.75, -2.75, 3.25, -3.25, 3.75, -3.75, 10.125, + -10.125, 20.25, -20.25, 0.375, -0.375, 64.25, -65.25, + ], + ) +} + +fn i32_scale_rows() -> ToInt8Case { + let mut values = [0i32; 16]; + let mut scales = [0.0f32; 16]; + for i in 0..16 { + values[i] = i as i32 - 8; + scales[i] = 0.5 + (i as f32) * 0.25; + } + i32_case([0, 0, 0], 6, 0, 1, 2, false, 0, 1, 1, 1, 0, &values, &scales) +} + +fn i32_relu_tile() -> ToInt8Case { + let mut values = [0i32; 64]; + for position in 0..4 { + for channel in 0..16 { + values[position * 16 + channel] = position as i32 * 20 + channel as i32 - 8; + } + } + i32_case([1, 2, 3], 7, 0, 1, 2, true, 3, 2, 2, 4, 0, &values, &[1.0; 16]) +} + +fn i32_signed() -> ToInt8Case { + let values: [i32; 16] = [ + -300, -128, -5, -1, 0, 1, 5, 127, 128, 200, -64, 64, -7, 7, 3, -3, + ]; + i32_case([3, 1, 2], 8, 0, 1, 2, false, 0, 1, 1, 1, 0, &values, &[1.0; 16]) +} + +fn i32_offset() -> ToInt8Case { + let mut values = [0i32; 32]; + for i in 0..32 { + values[i] = i as i32 - 16; + } + i32_case([2, 3, 1], 9, 0, 1, 2, true, 1, 2, 1, 2, 4, &values, &[0.5; 16]) +} + +// Two channel groups; each group has two pixels (eight FP32 rows) quantized +// with the same rows into group g of the output. +fn f32_two_groups() -> ToInt8Case { + let mut values = [0.0f32; 64]; + for (i, value) in values.iter_mut().enumerate() { + *value = ((i * 7) % 50) as f32 - 25.0; + } + let bits: Vec = values.iter().map(|v| v.to_bits()).collect(); + let src = pack_u32s(&bits); + let dst = f32_dst(&src, 2.0); + // Output rows 0, 1 of group 0, then rows 0, 1 of group 1. + finish( + KIND_F32, + [1, 0, 3], + 2, + 8, + 0, + 0, + 1, + 0, + 10, + u64::from(2.0f32.to_bits()), + 0, + src, + Vec::new(), + dst, + vec![0, 1, 0, 1], + ) +} + +// Three channel groups, each with its own scales: input rows start at 3, +// output core base 1, width 2, height 2, stride 3 -> rows 1, 2, 4, 5 in +// every group. +fn i32_three_groups() -> ToInt8Case { + let mut values = [0u32; 3 * 64]; + for (i, value) in values.iter_mut().enumerate() { + *value = (((i * 11) % 160) as i32 - 80) as u32; + } + let src = pack_u32s(&values); + let mut scale_bits = [0u32; 3 * 16]; + for (i, bits) in scale_bits.iter_mut().enumerate() { + *bits = (if (i / 16 + i) % 2 == 1 { 2.0f32 } else { 0.5f32 }).to_bits(); + } + let scale = pack_u32s(&scale_bits); + let mut dst = Vec::new(); + for group in 0..3 { + dst.extend(i32_dst( + &src[group * 16..group * 16 + 16], + &scale[group * 4..group * 4 + 4], + false, + )); + } + let rs2 = encode_i32_rs2(false, 1, 2, 2, 3, 3); + finish( + KIND_I32, + [2, 0, 1], + 3, + 16, + 0, + 1, + 2, + 3, + 11, + rs2, + 3, + src, + scale, + dst, + vec![1, 2, 4, 5, 1, 2, 4, 5, 1, 2, 4, 5], + ) +} diff --git a/examples/prefab/balls/toint8/verify/casegen/src/lib.rs b/examples/prefab/balls/toint8/verify/casegen/src/lib.rs new file mode 100644 index 000000000..1bf3b389d --- /dev/null +++ b/examples/prefab/balls/toint8/verify/casegen/src/lib.rs @@ -0,0 +1,119 @@ +mod casegen; +#[path = "../../../emu/src/model.rs"] +mod model; + +use std::cell::RefCell; + +use casegen::{ToInt8Case, ToInt8Cmd, MAX_DST, MAX_SCALE, MAX_SRC}; + +thread_local! { + static CURRENT: RefCell> = const { RefCell::new(None) }; +} + +#[no_mangle] +pub extern "C" fn toint8_case_load(index: u32) -> i32 { + let case = casegen::gen_case(index); + CURRENT.with(|current| *current.borrow_mut() = Some(case)); + 0 +} + +fn current_case R, R>(f: F) -> R { + CURRENT.with(|current| match current.borrow().as_ref() { + Some(case) => f(case), + None => panic!("toint8_case: no case loaded; call toint8_case_load first"), + }) +} + +#[no_mangle] +pub extern "C" fn toint8_case_cmd(out_ptr: *mut ToInt8Cmd) { + if out_ptr.is_null() { + panic!("toint8_case_cmd: null out_ptr"); + } + current_case(|case| unsafe { + *out_ptr = case.cmd; + }); +} + +fn src_index(index: u32, limit: u32, what: &str) -> usize { + if index >= limit { + panic!("toint8_case_{what}: index {index} out of range {limit}"); + } + index as usize +} + +#[no_mangle] +pub extern "C" fn toint8_case_src_word_lo(word_index: u32) -> u64 { + current_case(|case| { + let i = src_index(word_index, case.cmd.num_src_words, "src_word_lo"); + if i >= MAX_SRC { + panic!("toint8_case_src_word_lo: word_index out of range"); + } + case.src_lo(i) + }) +} + +#[no_mangle] +pub extern "C" fn toint8_case_src_word_hi(word_index: u32) -> u64 { + current_case(|case| { + let i = src_index(word_index, case.cmd.num_src_words, "src_word_hi"); + if i >= MAX_SRC { + panic!("toint8_case_src_word_hi: word_index out of range"); + } + case.src_hi(i) + }) +} + +#[no_mangle] +pub extern "C" fn toint8_case_scale_word_lo(word_index: u32) -> u64 { + current_case(|case| { + let i = src_index(word_index, case.cmd.num_scale_words, "scale_word_lo"); + if i >= MAX_SCALE { + panic!("toint8_case_scale_word_lo: word_index out of range"); + } + case.scale_lo(i) + }) +} + +#[no_mangle] +pub extern "C" fn toint8_case_scale_word_hi(word_index: u32) -> u64 { + current_case(|case| { + let i = src_index(word_index, case.cmd.num_scale_words, "scale_word_hi"); + if i >= MAX_SCALE { + panic!("toint8_case_scale_word_hi: word_index out of range"); + } + case.scale_hi(i) + }) +} + +#[no_mangle] +pub extern "C" fn toint8_case_dst_word_lo(word_index: u32) -> u64 { + current_case(|case| { + let i = src_index(word_index, case.cmd.num_dst_words, "dst_word_lo"); + if i >= MAX_DST { + panic!("toint8_case_dst_word_lo: word_index out of range"); + } + case.dst_lo(i) + }) +} + +#[no_mangle] +pub extern "C" fn toint8_case_dst_word_hi(word_index: u32) -> u64 { + current_case(|case| { + let i = src_index(word_index, case.cmd.num_dst_words, "dst_word_hi"); + if i >= MAX_DST { + panic!("toint8_case_dst_word_hi: word_index out of range"); + } + case.dst_hi(i) + }) +} + +#[no_mangle] +pub extern "C" fn toint8_case_dst_addr(word_index: u32) -> u32 { + current_case(|case| { + let i = src_index(word_index, case.cmd.num_dst_words, "dst_addr"); + if i >= MAX_DST { + panic!("toint8_case_dst_addr: word_index out of range"); + } + case.dst_addr[i] + }) +} diff --git a/examples/balls/toint8/verify/filelists/toint8_ball.f b/examples/prefab/balls/toint8/verify/filelists/toint8_ball.f similarity index 90% rename from examples/balls/toint8/verify/filelists/toint8_ball.f rename to examples/prefab/balls/toint8/verify/filelists/toint8_ball.f index 8750aeaa3..5c0be6498 100644 --- a/examples/balls/toint8/verify/filelists/toint8_ball.f +++ b/examples/prefab/balls/toint8/verify/filelists/toint8_ball.f @@ -1,3 +1,4 @@ ++incdir+@UVM@/src/core +incdir+@UVM@/src/ball +incdir+@UVM@/src/ball/agents/cmd +incdir+@UVM@/src/ball/agents/mem @@ -13,6 +14,7 @@ +incdir+src/tests +incdir+@RTL@ +@UVM@/src/core/package.sv @UVM@/src/ball/bb_blink_if.sv @UVM@/src/ball/bb_uvm_pkg.sv @@ -21,7 +23,7 @@ @RTL@/INToRecFN_i32_e8_s24.sv @RTL@/MulAddRecFNToRaw_preMul_e8_s24.sv @RTL@/MulAddRecFNToRaw_postMul_e8_s24.sv -@RTL@/MulAddRecFNPipe_l3_e8_s24.sv +@RTL@/Fma_l3_e8_s24.sv @RTL@/RoundRawFNToRecFN_e8_s24.sv @RTL@/RecFNToIN_e8_s24_i8.sv @RTL@/ToInt8Ball.sv diff --git a/examples/prefab/balls/toint8/verify/src/common/toint8_defs.svh b/examples/prefab/balls/toint8/verify/src/common/toint8_defs.svh new file mode 100644 index 000000000..2f9f9eb6a --- /dev/null +++ b/examples/prefab/balls/toint8/verify/src/common/toint8_defs.svh @@ -0,0 +1,48 @@ +typedef struct { + int unsigned kind; + int unsigned iter; + int unsigned op1_bank; + int unsigned op2_bank; + int unsigned wr_bank; + int unsigned op1_group; + int unsigned op2_group; + int unsigned wr_group; + int unsigned op1_col; + int unsigned op2_col; + int unsigned wr_col; + int unsigned rob_id; + int unsigned rs2_lo; + int unsigned rs2_hi; + int unsigned input_base; + int unsigned num_src_words; + int unsigned num_scale_words; + int unsigned num_dst_words; +} toint8_cmd_dpi_t; + +import "DPI-C" function int toint8_case_load(input int unsigned index); +import "DPI-C" function void toint8_case_cmd(output toint8_cmd_dpi_t cmd); +import "DPI-C" function longint unsigned toint8_case_src_word_lo(input int unsigned word_index); +import "DPI-C" function longint unsigned toint8_case_src_word_hi(input int unsigned word_index); +import "DPI-C" function longint unsigned toint8_case_scale_word_lo(input int unsigned word_index); +import "DPI-C" function longint unsigned toint8_case_scale_word_hi(input int unsigned word_index); +import "DPI-C" function longint unsigned toint8_case_dst_word_lo(input int unsigned word_index); +import "DPI-C" function longint unsigned toint8_case_dst_word_hi(input int unsigned word_index); +import "DPI-C" function int unsigned toint8_case_dst_addr(input int unsigned word_index); + +`ifndef QUANT_F32_TO_I8_FUNCT7 +`error "QUANT_F32_TO_I8_FUNCT7 must be provided by the selected Thread ballISA" +`endif +`ifndef QUANT_I32_TO_I8_FUNCT7 +`error "QUANT_I32_TO_I8_FUNCT7 must be provided by the selected Thread ballISA" +`endif + +localparam int QUANT_F32_TO_I8_THREAD_FUNCT7 = `QUANT_F32_TO_I8_FUNCT7; +localparam int QUANT_I32_TO_I8_THREAD_FUNCT7 = `QUANT_I32_TO_I8_FUNCT7; +localparam int TOINT8_KIND_F32 = 0; +localparam int TOINT8_KIND_I32 = 1; +// Words are group-major: rows of channel group 0, then group 1, ... +localparam int TOINT8_MAX_SRC_WORDS = 48; +localparam int TOINT8_MAX_SCALE_WORDS = 12; +localparam int TOINT8_MAX_DST_WORDS = 12; +localparam int TOINT8_NUM_CASES = 10; +localparam int TOINT8_TIMEOUT_CYCLES = 4000; diff --git a/examples/prefab/balls/toint8/verify/src/common/toint8_items.svh b/examples/prefab/balls/toint8/verify/src/common/toint8_items.svh new file mode 100644 index 000000000..f74c67749 --- /dev/null +++ b/examples/prefab/balls/toint8/verify/src/common/toint8_items.svh @@ -0,0 +1,126 @@ +class toint8_cmd_item extends bb_blink_cmd_item; + `uvm_object_utils(toint8_cmd_item) + + bit [31:0] kind; + bit [31:0] input_base; + bit [31:0] num_src_words; + bit [31:0] num_scale_words; + bit [31:0] num_dst_words; + bit [127:0] src_words[TOINT8_MAX_SRC_WORDS]; + bit [127:0] scale_words[TOINT8_MAX_SCALE_WORDS]; + bit [127:0] dst_words[TOINT8_MAX_DST_WORDS]; + bit [31:0] dst_addr[TOINT8_MAX_DST_WORDS]; + + function new(string name = "toint8_cmd_item"); + super.new(name); + endfunction + + function void load_rust_case(int unsigned index); + toint8_cmd_dpi_t cmd; + int unsigned rc; + longint unsigned w_lo; + longint unsigned w_hi; + int i; + + rc = toint8_case_load(index); + if (rc != 0) begin + `uvm_fatal("CASE", $sformatf("toint8_case_load returned %0d for index %0d", rc, index)) + end + toint8_case_cmd(cmd); + + if (cmd.kind != TOINT8_KIND_F32 && cmd.kind != TOINT8_KIND_I32) begin + `uvm_fatal("CASE", $sformatf("unknown toint8 kind %0d", cmd.kind)) + end + + // Banks and iter come from rs1; rs2 carries the f32 scale or the i32 output geometry. + kind = cmd.kind; + rob_id = cmd.rob_id[BB_BLINK_ROB_ID_W-1:0]; + funct = kind == TOINT8_KIND_F32 ? QUANT_F32_TO_I8_THREAD_FUNCT7[6:0] : + QUANT_I32_TO_I8_THREAD_FUNCT7[6:0]; + rs2 = {cmd.rs2_hi, cmd.rs2_lo}; + iter = cmd.iter; + op1_bank = cmd.op1_bank; + op2_bank = cmd.op2_bank; + wr_bank = cmd.wr_bank; + op1_group = cmd.op1_group; + op2_group = cmd.op2_group; + wr_group = cmd.wr_group; + op1_col = cmd.op1_col; + op2_col = cmd.op2_col; + wr_col = cmd.wr_col; + wr_read_bank = '0; + wr_arch_bank = cmd.wr_bank; + input_base = cmd.input_base; + num_src_words = cmd.num_src_words; + num_scale_words = cmd.num_scale_words; + num_dst_words = cmd.num_dst_words; + + if (kind == TOINT8_KIND_F32) begin + if (op2_bank != 0 || op2_col != 0) `uvm_fatal("CASE", "QUANT_F32_TO_I8 reserves input bank 1") + if (rs2[63:32] != 0) `uvm_fatal("CASE", "QUANT_F32_TO_I8 reserves rs2[63:32]") + if (num_scale_words != 0) `uvm_fatal("CASE", "QUANT_F32_TO_I8 must not emit scale rows") + if (input_base != 0) `uvm_fatal("CASE", "QUANT_F32_TO_I8 input_base must be 0") + end else begin + if (op2_col != op1_col) + `uvm_fatal("CASE", "QUANT_I32_TO_I8 scale must have one group per channel group") + if (op1_bank == op2_bank) + `uvm_fatal("CASE", "QUANT_I32_TO_I8 input and scale banks must differ") + if (op2_bank == wr_bank) + `uvm_fatal("CASE", "QUANT_I32_TO_I8 scale and output banks must differ") + if (rs2[63:35] != 0) `uvm_fatal("CASE", "QUANT_I32_TO_I8 reserves rs2[63:35]") + if (num_scale_words != 4 * op1_col) + `uvm_fatal("CASE", "QUANT_I32_TO_I8 must emit four scale rows per group") + end + + if (iter == 0 || iter[1:0] != 2'b00) + `uvm_fatal("CASE", "ToInt8Ball iter must be a positive multiple of four") + if (op1_col == 0 || wr_col != op1_col) + `uvm_fatal("CASE", "ToInt8Ball input and output must have the same channel groups") + if (op1_bank == wr_bank) `uvm_fatal("CASE", "ToInt8Ball input and output banks must differ") + if (num_src_words != iter * op1_col) + `uvm_fatal("CASE", $sformatf( + "num_src_words %0d != iter %0d * groups %0d", num_src_words, iter, op1_col)) + if (num_dst_words != (iter >> 2) * op1_col) + `uvm_fatal("CASE", "num_dst_words does not match iter/4 per group") + if (num_src_words == 0 || num_src_words > TOINT8_MAX_SRC_WORDS) + `uvm_fatal("CASE", $sformatf("num_src_words out of range: %0d", num_src_words)) + if (num_dst_words == 0 || num_dst_words > TOINT8_MAX_DST_WORDS) + `uvm_fatal("CASE", $sformatf("num_dst_words out of range: %0d", num_dst_words)) + + for (i = 0; i < num_src_words; i++) begin + w_lo = toint8_case_src_word_lo(i); + w_hi = toint8_case_src_word_hi(i); + src_words[i] = {w_hi, w_lo}; + end + for (i = 0; i < num_scale_words; i++) begin + w_lo = toint8_case_scale_word_lo(i); + w_hi = toint8_case_scale_word_hi(i); + scale_words[i] = {w_hi, w_lo}; + end + for (i = 0; i < num_dst_words; i++) begin + w_lo = toint8_case_dst_word_lo(i); + w_hi = toint8_case_dst_word_hi(i); + dst_words[i] = {w_hi, w_lo}; + dst_addr[i] = toint8_case_dst_addr(i); + end + endfunction + + function void do_copy(uvm_object rhs); + toint8_cmd_item rhs_; + super.do_copy(rhs); + if (!$cast(rhs_, rhs)) begin + `uvm_fatal("COPY", "rhs is not toint8_cmd_item") + end + kind = rhs_.kind; + input_base = rhs_.input_base; + num_src_words = rhs_.num_src_words; + num_scale_words = rhs_.num_scale_words; + num_dst_words = rhs_.num_dst_words; + for (int i = 0; i < TOINT8_MAX_SRC_WORDS; i++) src_words[i] = rhs_.src_words[i]; + for (int i = 0; i < TOINT8_MAX_SCALE_WORDS; i++) scale_words[i] = rhs_.scale_words[i]; + for (int i = 0; i < TOINT8_MAX_DST_WORDS; i++) begin + dst_words[i] = rhs_.dst_words[i]; + dst_addr[i] = rhs_.dst_addr[i]; + end + endfunction +endclass diff --git a/examples/prefab/balls/toint8/verify/src/cov/toint8_cov.svh b/examples/prefab/balls/toint8/verify/src/cov/toint8_cov.svh new file mode 100644 index 000000000..db6bb5f8c --- /dev/null +++ b/examples/prefab/balls/toint8/verify/src/cov/toint8_cov.svh @@ -0,0 +1,39 @@ +class toint8_cov extends uvm_component; + `uvm_component_utils(toint8_cov) + + uvm_analysis_imp_cmd #(bb_blink_cmd_item, toint8_cov) cmd_imp; + + bit [6:0] cur_funct; + int unsigned cur_groups; + + covergroup cmd_cg; + coverpoint cur_funct { + bins f32 = {QUANT_F32_TO_I8_THREAD_FUNCT7[6:0]}; + bins i32 = {QUANT_I32_TO_I8_THREAD_FUNCT7[6:0]}; + } + coverpoint cur_groups {bins single = {1}; bins multi = {[2 : $]};} + endgroup + + int cmd_count; + + function new(string name, uvm_component parent); + super.new(name, parent); + cmd_imp = new("cmd_imp", this); + cmd_cg = new(); + endfunction + + function void write_cmd(bb_blink_cmd_item item); + cur_funct = item.funct; + cur_groups = item.op1_col; + cmd_count++; + cmd_cg.sample(); + endfunction + + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (cmd_count == 0) `uvm_fatal("COV", "no cmd observed") + if (cmd_cg.get_coverage() != 100.0) + `uvm_fatal("COV", $sformatf("toint8_cov %0.2f%%", cmd_cg.get_coverage())) + endfunction +endclass diff --git a/examples/balls/toint8/verify/src/env/toint8_env.svh b/examples/prefab/balls/toint8/verify/src/env/toint8_env.svh similarity index 100% rename from examples/balls/toint8/verify/src/env/toint8_env.svh rename to examples/prefab/balls/toint8/verify/src/env/toint8_env.svh diff --git a/examples/prefab/balls/toint8/verify/src/env/toint8_scoreboard.svh b/examples/prefab/balls/toint8/verify/src/env/toint8_scoreboard.svh new file mode 100644 index 000000000..0578d5c05 --- /dev/null +++ b/examples/prefab/balls/toint8/verify/src/env/toint8_scoreboard.svh @@ -0,0 +1,196 @@ +class toint8_scoreboard extends uvm_scoreboard; + `uvm_component_utils(toint8_scoreboard) + + uvm_analysis_imp_stim #(bb_blink_cmd_item, toint8_scoreboard) stim_imp; + uvm_analysis_imp_cmd #(bb_blink_cmd_item, toint8_scoreboard) cmd_imp; + uvm_analysis_imp_read #(bb_blink_read_item, toint8_scoreboard) read_imp; + uvm_analysis_imp_write #(bb_blink_write_item, toint8_scoreboard) write_imp; + uvm_analysis_imp_resp #(bb_blink_resp_item, toint8_scoreboard) resp_imp; + + bb_blink_mem_model #(`BB_IN_BW, `BB_OUT_BW) mem_model; + + toint8_cmd_item stim_q[$]; + int unsigned expected_reads; + int unsigned expected_writes; + int unsigned cmd_count; + int unsigned read_count; + int unsigned write_count; + int unsigned resp_count; + + function new(string name, uvm_component parent); + super.new(name, parent); + stim_imp = new("stim_imp", this); + cmd_imp = new("cmd_imp", this); + read_imp = new("read_imp", this); + write_imp = new("write_imp", this); + resp_imp = new("resp_imp", this); + endfunction + + function void write_stim(bb_blink_cmd_item item); + toint8_cmd_item got; + toint8_cmd_item clone; + int i; + + if (stim_q.size() != 0) begin + `uvm_fatal("SCB", "single outstanding command supported") + end + if (!$cast(got, item)) begin + `uvm_fatal("SCB", "stim item is not toint8_cmd_item") + end + if (!$cast(clone, got.clone())) begin + `uvm_fatal("SCB", "failed to clone stimulus item") + end + stim_q.push_back(clone); + + if (mem_model == null) begin + `uvm_fatal("SCB", "mem_model handle not set") + end + mem_model.clear_mem(); + // Group g of every operand is channel block g with the same rows. + for (i = 0; i < clone.num_src_words; i++) begin + mem_model.load_word_g(int'(clone.op1_bank), int'(clone.op1_group) + i / int'(clone.iter), + int'(clone.input_base) + i % int'(clone.iter), clone.src_words[i]); + end + for (i = 0; i < clone.num_scale_words; i++) begin + mem_model.load_word_g(int'(clone.op2_bank), int'(clone.op2_group) + i / 4, i % 4, + clone.scale_words[i]); + end + mem_model.arm(); + + expected_writes = clone.num_dst_words; + expected_reads = clone.num_scale_words + clone.num_src_words; + endfunction + + function void write_cmd(bb_blink_cmd_item item); + toint8_cmd_item stim; + stim = current_stim("CMD"); + check_cmd(item, stim); + cmd_count++; + endfunction + + function void check_cmd(bb_blink_cmd_item got, toint8_cmd_item exp); + // Compare as plain BLink commands; the comparer rejects differing types. + bb_blink_cmd_item want = bb_blink_cmd_item::type_id::create("want"); + want.copy(exp); + verification_pkg::evidence::checks++; + if (!got.compare(want)) + `uvm_fatal("CMD", $sformatf("command mismatch: got %s exp %s", got.sprint(), want.sprint())) + endfunction + + function void write_read(bb_blink_read_item item); + toint8_cmd_item stim; + int unsigned scale_reads; + int unsigned group_reads; + int unsigned group; + int unsigned index; + int unsigned expect_bank; + int unsigned expect_group; + int unsigned expect_addr; + + stim = current_stim("READ"); + if (item.port != 0) + `uvm_fatal("READ", $sformatf("ToInt8Ball has one read port, got port %0d", item.port)) + if (item.rob_id !== stim.rob_id) + `uvm_fatal("READ", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + + // Each group reads its scale rows (I32 only), then its input rows. + scale_reads = stim.num_scale_words / stim.op1_col; + group_reads = scale_reads + stim.iter; + group = read_count / group_reads; + index = read_count % group_reads; + if (index < scale_reads) begin + expect_bank = stim.op2_bank; + expect_group = stim.op2_group + group; + expect_addr = index; + end else begin + expect_bank = stim.op1_bank; + expect_group = stim.op1_group + group; + expect_addr = stim.input_base + (index - scale_reads); + end + if (item.bank_id !== expect_bank[BB_BLINK_BANK_ID_W-1:0]) + `uvm_fatal("READ", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, expect_bank)) + if (item.group_id !== expect_group[BB_BLINK_GROUP_ID_W-1:0]) + `uvm_fatal("READ", $sformatf("group mismatch: got %0d exp %0d", item.group_id, expect_group)) + if (item.addr !== expect_addr[BB_BLINK_BANK_ADDR_W-1:0]) + `uvm_fatal("READ", $sformatf("addr mismatch: got %0d exp %0d", item.addr, expect_addr)) + read_count++; + endfunction + + function void write_write(bb_blink_write_item item); + toint8_cmd_item stim; + + stim = current_stim("WRITE"); + if (item.port != 0) + `uvm_fatal("WRITE", $sformatf("ToInt8Ball has one write port, got port %0d", item.port)) + if (item.bank_id !== stim.wr_bank) + `uvm_fatal("WRITE", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.wr_bank)) + if (item.rob_id !== stim.rob_id) + `uvm_fatal("WRITE", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + if (item.group_id !== stim.wr_group + write_count / (stim.iter >> 2)) + `uvm_fatal("WRITE", $sformatf( + "group mismatch: got %0d exp %0d", + item.group_id, + stim.wr_group + write_count / (stim.iter >> 2) + )) + if (item.mask !== 16'hFFFF) + `uvm_fatal("WRITE", $sformatf("mask mismatch: got 0x%04h", item.mask)) + if (write_count >= stim.num_dst_words) + `uvm_fatal("WRITE", $sformatf("extra write %0d", write_count)) + if (item.addr !== stim.dst_addr[write_count][BB_BLINK_BANK_ADDR_W-1:0]) + `uvm_fatal("WRITE", $sformatf( + "addr mismatch: got %0d exp %0d", item.addr, stim.dst_addr[write_count])) + if (item.data !== stim.dst_words[write_count]) + `uvm_fatal("SCB", $sformatf( + "data mismatch at addr %0d: got 0x%032h exp 0x%032h", + item.addr, + item.data, + stim.dst_words[write_count] + )) + write_count++; + endfunction + + function void write_resp(bb_blink_resp_item item); + toint8_cmd_item stim; + + stim = current_stim("RESP"); + if (item.rob_id !== stim.rob_id) + `uvm_fatal("RESP", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + resp_count++; + endfunction + + function toint8_cmd_item current_stim(string tag); + if (stim_q.size() == 0) begin + `uvm_fatal("SCB", $sformatf("%s observed before stimulus", tag)) + return null; + end + return stim_q[0]; + endfunction + + function bit done(); + return cmd_count == 1 && + read_count == expected_reads && + write_count == expected_writes && + resp_count == 1; + endfunction + + function void reset_counters(); + stim_q.delete(); + cmd_count = 0; + read_count = 0; + write_count = 0; + resp_count = 0; + expected_reads = 0; + expected_writes = 0; + endfunction + + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (!done()) begin + `uvm_fatal("SCB", $sformatf("incomplete: cmds=%0d reads=%0d/%0d writes=%0d/%0d resp=%0d", + cmd_count, read_count, expected_reads, write_count, + expected_writes, resp_count)) + end + stim_q.delete(); + endfunction +endclass diff --git a/examples/balls/toint8/verify/src/pkg/toint8_pkg.sv b/examples/prefab/balls/toint8/verify/src/pkg/toint8_pkg.sv similarity index 100% rename from examples/balls/toint8/verify/src/pkg/toint8_pkg.sv rename to examples/prefab/balls/toint8/verify/src/pkg/toint8_pkg.sv diff --git a/examples/balls/toint8/verify/src/seq/toint8_sequences.svh b/examples/prefab/balls/toint8/verify/src/seq/toint8_sequences.svh similarity index 85% rename from examples/balls/toint8/verify/src/seq/toint8_sequences.svh rename to examples/prefab/balls/toint8/verify/src/seq/toint8_sequences.svh index f51579a1e..f72b2d3f5 100644 --- a/examples/balls/toint8/verify/src/seq/toint8_sequences.svh +++ b/examples/prefab/balls/toint8/verify/src/seq/toint8_sequences.svh @@ -2,7 +2,6 @@ class toint8_basic_seq extends uvm_sequence #(bb_blink_cmd_item); `uvm_object_utils(toint8_basic_seq) int unsigned case_index; - int unsigned bid; function new(string name = "toint8_basic_seq"); super.new(name); @@ -12,7 +11,7 @@ class toint8_basic_seq extends uvm_sequence #(bb_blink_cmd_item); toint8_cmd_item item; item = toint8_cmd_item::type_id::create("item"); start_item(item); - item.load_rust_case(case_index, bid); + item.load_rust_case(case_index); finish_item(item); endtask endclass diff --git a/examples/prefab/balls/toint8/verify/src/tb_top.sv b/examples/prefab/balls/toint8/verify/src/tb_top.sv new file mode 100644 index 000000000..c54dcb03f --- /dev/null +++ b/examples/prefab/balls/toint8/verify/src/tb_top.sv @@ -0,0 +1,79 @@ +module tb_top; + import uvm_pkg::*; + import toint8_pkg::*; + + bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); + + ToInt8Ball dut ( + .clock(intf.clock), + .reset(intf.reset), + .blink_status_running(intf.status_running), + .blink_cmdReq_ready(intf.cmd_req_ready), + .blink_cmdReq_valid(intf.cmd_req_valid), + .blink_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .blink_cmdReq_bits_funct(intf.cmd_req_bits_funct), + .blink_cmdReq_bits_rs2(intf.cmd_req_bits_rs2), + .blink_cmdReq_bits_iter(intf.cmd_req_bits_iter), + .blink_cmdReq_bits_op1_bank(intf.cmd_req_bits_op1_bank), + .blink_cmdReq_bits_op2_bank(intf.cmd_req_bits_op2_bank), + .blink_cmdReq_bits_wr_bank(intf.cmd_req_bits_wr_bank), + .blink_cmdReq_bits_op1_group(intf.cmd_req_bits_op1_group), + .blink_cmdReq_bits_op2_group(intf.cmd_req_bits_op2_group), + .blink_cmdReq_bits_wr_group(intf.cmd_req_bits_wr_group), + .blink_cmdReq_bits_op1_col(intf.cmd_req_bits_op1_col), + .blink_cmdReq_bits_op2_col(intf.cmd_req_bits_op2_col), + .blink_cmdReq_bits_wr_col(intf.cmd_req_bits_wr_col), + .blink_cmdReq_bits_wr_read_bank(intf.cmd_req_bits_wr_read_bank), + .blink_cmdReq_bits_wr_arch_bank(intf.cmd_req_bits_wr_arch_bank), + .blink_cmdResp_ready(intf.cmd_resp_ready), + .blink_cmdResp_valid(intf.cmd_resp_valid), + .blink_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .blink_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .blink_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .blink_bankRead_0_group_id(intf.bank_read_group_id[0]), + .blink_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .blink_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .blink_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .blink_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .blink_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .blink_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .blink_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .blink_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .blink_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .blink_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .blink_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .blink_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .blink_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .blink_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .blink_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .blink_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]) + ); + + // Constant outputs removed from the port list by firtool. + assign intf.bank_write_req_mask[0][0] = dut.blink_bankWrite_0_io_req_bits_mask_0; + assign intf.bank_write_req_mask[0][1] = dut.blink_bankWrite_0_io_req_bits_mask_1; + assign intf.bank_write_req_mask[0][2] = dut.blink_bankWrite_0_io_req_bits_mask_2; + assign intf.bank_write_req_mask[0][3] = dut.blink_bankWrite_0_io_req_bits_mask_3; + assign intf.bank_write_req_mask[0][4] = dut.blink_bankWrite_0_io_req_bits_mask_4; + assign intf.bank_write_req_mask[0][5] = dut.blink_bankWrite_0_io_req_bits_mask_5; + assign intf.bank_write_req_mask[0][6] = dut.blink_bankWrite_0_io_req_bits_mask_6; + assign intf.bank_write_req_mask[0][7] = dut.blink_bankWrite_0_io_req_bits_mask_7; + assign intf.bank_write_req_mask[0][8] = dut.blink_bankWrite_0_io_req_bits_mask_8; + assign intf.bank_write_req_mask[0][9] = dut.blink_bankWrite_0_io_req_bits_mask_9; + assign intf.bank_write_req_mask[0][10] = dut.blink_bankWrite_0_io_req_bits_mask_10; + assign intf.bank_write_req_mask[0][11] = dut.blink_bankWrite_0_io_req_bits_mask_11; + assign intf.bank_write_req_mask[0][12] = dut.blink_bankWrite_0_io_req_bits_mask_12; + assign intf.bank_write_req_mask[0][13] = dut.blink_bankWrite_0_io_req_bits_mask_13; + assign intf.bank_write_req_mask[0][14] = dut.blink_bankWrite_0_io_req_bits_mask_14; + assign intf.bank_write_req_mask[0][15] = dut.blink_bankWrite_0_io_req_bits_mask_15; + + initial begin + intf.clock = 1'b0; + forever #5 intf.clock = ~intf.clock; + end + + initial begin + uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); + run_test("toint8_ball_test"); + end +endmodule diff --git a/examples/balls/toint8/verify/src/tests/toint8_test.svh b/examples/prefab/balls/toint8/verify/src/tests/toint8_test.svh similarity index 90% rename from examples/balls/toint8/verify/src/tests/toint8_test.svh rename to examples/prefab/balls/toint8/verify/src/tests/toint8_test.svh index 66e5b66d4..92d766203 100644 --- a/examples/balls/toint8/verify/src/tests/toint8_test.svh +++ b/examples/prefab/balls/toint8/verify/src/tests/toint8_test.svh @@ -17,20 +17,18 @@ class toint8_ball_test extends uvm_test; endfunction task run_phase(uvm_phase phase); - int unsigned bid = toint8_require_bid(); phase.raise_objection(this); - for (int unsigned index = 0; index < TOINT8_NUM_CASES; index++) run_case(index, bid); + for (int unsigned index = 0; index < TOINT8_NUM_CASES; index++) run_case(index); phase.drop_objection(this); endtask - task run_case(int unsigned index, int unsigned bid); + task run_case(int unsigned index); toint8_basic_seq seq; int cycles = 0; apply_reset(); env.scb.reset_counters(); seq = toint8_basic_seq::type_id::create("seq"); seq.case_index = index; - seq.bid = bid; seq.start(env.cmd_agent.seqr); while (!env.scb.done()) begin @(posedge vif.clock); diff --git a/examples/prefab/balls/toint8/workloads/ctests/CMakeLists.txt b/examples/prefab/balls/toint8/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..f6b44e859 --- /dev/null +++ b/examples/prefab/balls/toint8/workloads/ctests/CMakeLists.txt @@ -0,0 +1,8 @@ +add_buckyball_ctest(quant_f32_to_i8_test.c) +add_buckyball_ctest(quant_i32_to_i8_test.c) +add_buckyball_ctest(toint8_i8_rounding_test.c) +add_buckyball_ctest(toint8_i8_rows_test.c) +add_buckyball_ctest(toint8_i8_scale_rows_test.c) +add_buckyball_ctest(toint8_i8_signed_test.c) +add_buckyball_ctest(quant_i32_groups_test.c) +add_buckyball_ctest(quant_f32_groups_test.c) diff --git a/examples/prefab/balls/toint8/workloads/ctests/quant_f32_groups_test.c b/examples/prefab/balls/toint8/workloads/ctests/quant_f32_groups_test.c new file mode 100644 index 000000000..b0fd0b952 --- /dev/null +++ b/examples/prefab/balls/toint8/workloads/ctests/quant_f32_groups_test.c @@ -0,0 +1,40 @@ +#include "buckyball.h" +#include +#include +#include +#include + +/* QUANT_F32_TO_I8 over three channel groups: input and output group g are + * channel block g; every group uses the same rows. */ +enum { GROUPS = 3, PIXELS = 2, ROWS = PIXELS * 4 }; + +static float input[GROUPS][PIXELS * 16] __attribute__((aligned(64))); +static int8_t actual[GROUPS][PIXELS * 16] __attribute__((aligned(64))); + +int main(void) { + for (int group = 0; group < GROUPS; ++group) + for (int i = 0; i < PIXELS * 16; ++i) + input[group][i] = (float)((group * 29 + i * 7) % 100 - 50); + + bb_mem_alloc(0, 1, GROUPS); + bb_mem_alloc(1, 1, GROUPS); + for (int group = 0; group < GROUPS; ++group) + bb_mvin_group((uintptr_t)input[group], 0, group, ROWS, 1); + bb_quant_f32_to_i8(0, 1, ROWS, 2.0f); + for (int group = 0; group < GROUPS; ++group) + bb_mvout_group((uintptr_t)actual[group], 1, group, PIXELS, 1); + + for (int group = 0; group < GROUPS; ++group) + for (int i = 0; i < PIXELS * 16; ++i) { + int value = (int)input[group][i] * 2; + int expected = value > 127 ? 127 : value < -128 ? -128 : value; + if (actual[group][i] != expected) { + printf("quant_f32_groups FAIL group=%d index=%d expected=%d " + "actual=%d\n", + group, i, expected, actual[group][i]); + return 1; + } + } + printf("quant_f32_groups PASS\n"); + return 0; +} diff --git a/examples/balls/toint8/workloads/ctests/quant_f32_to_i8_test.c b/examples/prefab/balls/toint8/workloads/ctests/quant_f32_to_i8_test.c similarity index 100% rename from examples/balls/toint8/workloads/ctests/quant_f32_to_i8_test.c rename to examples/prefab/balls/toint8/workloads/ctests/quant_f32_to_i8_test.c diff --git a/examples/prefab/balls/toint8/workloads/ctests/quant_i32_groups_test.c b/examples/prefab/balls/toint8/workloads/ctests/quant_i32_groups_test.c new file mode 100644 index 000000000..cd47f9ace --- /dev/null +++ b/examples/prefab/balls/toint8/workloads/ctests/quant_i32_groups_test.c @@ -0,0 +1,55 @@ +#include "buckyball.h" +#include +#include +#include +#include + +/* QUANT_I32_TO_I8 over two channel groups: group g of the input, scale, and + * output banks is channel block g; every group uses the same output core. */ +enum { GROUPS = 2, PIXELS = 4, ROWS = PIXELS * 4, OUT_ROWS = 6 }; + +static int32_t input[GROUPS][PIXELS * 16] __attribute__((aligned(64))); +static float scales[GROUPS][16] __attribute__((aligned(64))); +static int8_t actual[GROUPS][OUT_ROWS * 16] __attribute__((aligned(64))); + +int main(void) { + for (int group = 0; group < GROUPS; ++group) { + for (int channel = 0; channel < 16; ++channel) + scales[group][channel] = (group + channel) % 2 ? 2.0f : 1.0f; + for (int i = 0; i < PIXELS * 16; ++i) + input[group][i] = (group * 37 + i * 11) % 160 - 80; + } + + bb_mem_alloc(0, 1, GROUPS); + bb_mem_alloc(1, 1, GROUPS); + bb_mem_alloc(2, 1, GROUPS); + for (int group = 0; group < GROUPS; ++group) { + bb_mvin_group((uintptr_t)input[group], 0, group, ROWS, 1); + bb_mvin_group((uintptr_t)scales[group], 1, group, 4, 1); + } + // Output core: base 1, width 2, height 2, stride 3 -> rows 1, 2, 4, 5. + bb_quant_i32_to_i8(0, 1, 2, ROWS, 0, 1, 2, 2, 3, 0); + for (int group = 0; group < GROUPS; ++group) + bb_mvout_group((uintptr_t)actual[group], 2, group, OUT_ROWS, 1); + + for (int group = 0; group < GROUPS; ++group) + for (int row = 0; row < OUT_ROWS; ++row) + for (int channel = 0; channel < 16; ++channel) { + int expected = 0; + if (row % 3) { + int pixel = row / 3 * 2 + row % 3 - 1; + int value = + input[group][pixel * 16 + channel] * (int)scales[group][channel]; + expected = value > 127 ? 127 : value < -128 ? -128 : value; + } + if (actual[group][row * 16 + channel] != expected) { + printf("quant_i32_groups FAIL group=%d row=%d channel=%d " + "expected=%d actual=%d\n", + group, row, channel, expected, + actual[group][row * 16 + channel]); + return 1; + } + } + printf("quant_i32_groups PASS\n"); + return 0; +} diff --git a/examples/balls/toint8/workloads/ctests/quant_i32_to_i8_test.c b/examples/prefab/balls/toint8/workloads/ctests/quant_i32_to_i8_test.c similarity index 100% rename from examples/balls/toint8/workloads/ctests/quant_i32_to_i8_test.c rename to examples/prefab/balls/toint8/workloads/ctests/quant_i32_to_i8_test.c diff --git a/examples/balls/toint8/workloads/ctests/toint8_i8_rounding_test.c b/examples/prefab/balls/toint8/workloads/ctests/toint8_i8_rounding_test.c similarity index 95% rename from examples/balls/toint8/workloads/ctests/toint8_i8_rounding_test.c rename to examples/prefab/balls/toint8/workloads/ctests/toint8_i8_rounding_test.c index 5837167b4..2fed43afb 100644 --- a/examples/balls/toint8/workloads/ctests/toint8_i8_rounding_test.c +++ b/examples/prefab/balls/toint8/workloads/ctests/toint8_i8_rounding_test.c @@ -15,8 +15,8 @@ static const int8_t expected[16] = { }; int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif bb_mem_alloc(0, 1, 1); bb_mem_alloc(1, 1, 1); diff --git a/examples/balls/toint8/workloads/ctests/toint8_i8_rows_test.c b/examples/prefab/balls/toint8/workloads/ctests/toint8_i8_rows_test.c similarity index 95% rename from examples/balls/toint8/workloads/ctests/toint8_i8_rows_test.c rename to examples/prefab/balls/toint8/workloads/ctests/toint8_i8_rows_test.c index d1000ae18..d6f1f2aa7 100644 --- a/examples/balls/toint8/workloads/ctests/toint8_i8_rows_test.c +++ b/examples/prefab/balls/toint8/workloads/ctests/toint8_i8_rows_test.c @@ -18,8 +18,8 @@ static const int8_t expected[32] = { }; int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif bb_mem_alloc(0, 1, 1); bb_mem_alloc(1, 1, 1); diff --git a/examples/balls/toint8/workloads/ctests/toint8_i8_scale_rows_test.c b/examples/prefab/balls/toint8/workloads/ctests/toint8_i8_scale_rows_test.c similarity index 95% rename from examples/balls/toint8/workloads/ctests/toint8_i8_scale_rows_test.c rename to examples/prefab/balls/toint8/workloads/ctests/toint8_i8_scale_rows_test.c index 526cbd310..a7e80f446 100644 --- a/examples/balls/toint8/workloads/ctests/toint8_i8_scale_rows_test.c +++ b/examples/prefab/balls/toint8/workloads/ctests/toint8_i8_scale_rows_test.c @@ -16,8 +16,8 @@ static const int8_t expected[32] = { }; int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif bb_mem_alloc(0, 1, 1); bb_mem_alloc(1, 1, 1); diff --git a/examples/balls/toint8/workloads/ctests/toint8_i8_signed_test.c b/examples/prefab/balls/toint8/workloads/ctests/toint8_i8_signed_test.c similarity index 95% rename from examples/balls/toint8/workloads/ctests/toint8_i8_signed_test.c rename to examples/prefab/balls/toint8/workloads/ctests/toint8_i8_signed_test.c index 65a518077..c9ffdfc3a 100644 --- a/examples/balls/toint8/workloads/ctests/toint8_i8_signed_test.c +++ b/examples/prefab/balls/toint8/workloads/ctests/toint8_i8_signed_test.c @@ -15,8 +15,8 @@ static const int8_t expected[16] = { }; int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif bb_mem_alloc(0, 1, 1); bb_mem_alloc(1, 1, 1); diff --git a/examples/prefab/balls/toint8/workloads/isa/quant.h b/examples/prefab/balls/toint8/workloads/isa/quant.h new file mode 100644 index 000000000..655c9e7c1 --- /dev/null +++ b/examples/prefab/balls/toint8/workloads/isa/quant.h @@ -0,0 +1,39 @@ +#ifndef _BB_QUANT_H_ +#define _BB_QUANT_H_ + +#include +#include + +// Group g of every operand bank is channel block g (channels 16g..16g+15). +// The input, scale (I32 only), and output banks have the same number of +// groups; every group uses the same rows, so one instruction converts all +// channel blocks. In group g, input rows hold 4 FP32/INT32 lanes (four rows +// per pixel), scale rows 0..3 hold the 16 per-channel scales of block g, and +// each output row holds the 16 INT8 channels of one pixel. + +static inline void bb_quant_f32_to_i8(uint64_t input_bank, uint64_t output_bank, + uint64_t iter, float scale) { + union { + float f; + uint32_t u; + } bits = {.f = scale}; + BUCKYBALL_INSTRUCTION_R_R(BB_BANK0(input_bank) | BB_BANK2(output_bank) | + BB_ITER(iter), + bits.u, FUNCT7(toint8, QUANT_F32_TO_I8)); +} + +static inline void bb_quant_i32_to_i8(uint64_t input_bank, uint64_t scale_bank, + uint64_t output_bank, uint64_t iter, + uint64_t input_base, uint64_t output_base, + uint64_t output_width, + uint64_t output_height, + uint64_t output_stride, int relu) { + BUCKYBALL_INSTRUCTION_R_R(BB_BANK0(input_bank) | BB_BANK1(scale_bank) | + BB_BANK2(output_bank) | BB_ITER(iter), + (output_base << 1) | (output_width << 8) | + (output_height << 15) | (output_stride << 22) | + (input_base << 29) | (uint64_t)(relu != 0), + FUNCT7(toint8, QUANT_I32_TO_I8)); +} + +#endif // _BB_QUANT_H_ diff --git a/examples/balls/toint8/workloads/mlir_tests/CMakeLists.txt b/examples/prefab/balls/toint8/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/balls/toint8/workloads/mlir_tests/CMakeLists.txt rename to examples/prefab/balls/toint8/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/balls/toint8/workloads/mlir_tests/ball/CMakeLists.txt b/examples/prefab/balls/toint8/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/balls/toint8/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/prefab/balls/toint8/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/prefab/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8.mlir b/examples/prefab/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8.mlir new file mode 100644 index 000000000..215ea0b8a --- /dev/null +++ b/examples/prefab/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8.mlir @@ -0,0 +1,25 @@ +func.func private @check_result(memref<1x16xi8>) -> () + +memref.global "private" constant @input : memref<4x4xf32> = dense<[ + [-3.000000e+02, -2.550000e+02, -5.000000e+00, -3.000000e+00], + [-1.000000e+00, 0.000000e+00, 1.000000e+00, 3.000000e+00], + [5.000000e+00, 2.540000e+02, 2.550000e+02, 3.000000e+02], + [2.000000e+00, 6.000000e+00, 1.000000e+01, 1.400000e+01] +]> + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %scale = arith.constant 5.000000e-01 : f32 + %input = memref.get_global @input : memref<4x4xf32> + %output = memref.alloc() alignment = 64 : memref<1x16xi8> + buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %input <{bank = 1 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> + toint8_ball.fp32_to_int8 multiplier(%scale) <{input = 1 : i64, output = 2 : i64, rows = 4 : i64}> + buckyball.mvout %output <{bank = 2 : i64, depth = 1 : i64, stride = 1 : i64}> : memref<1x16xi8> + func.call @check_result(%output) : (memref<1x16xi8>) -> () + buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + memref.dealloc %output : memref<1x16xi8> + return %zero_i8 : i8 +} diff --git a/examples/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8_main.cpp b/examples/prefab/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8_main.cpp similarity index 100% rename from examples/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8_main.cpp rename to examples/prefab/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8_main.cpp diff --git a/examples/prefab/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8.mlir b/examples/prefab/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8.mlir new file mode 100644 index 000000000..41d7d08de --- /dev/null +++ b/examples/prefab/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8.mlir @@ -0,0 +1,45 @@ +func.func private @check_result(memref<4x16xi8>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %one_f32 = arith.constant 1.0 : f32 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %four = arith.constant 4 : index + %sixteen = arith.constant 16 : index + %twenty = arith.constant 20 : index + %eight = arith.constant 8 : index + %input = memref.alloc() alignment = 64 : memref<16x4xi32> + %scales = memref.alloc() alignment = 64 : memref<4x4xf32> + %output = memref.alloc() alignment = 64 : memref<4x16xi8> + linalg.fill ins(%one_f32 : f32) outs(%scales : memref<4x4xf32>) + linalg.fill ins(%zero_i8 : i8) outs(%output : memref<4x16xi8>) + scf.for %position = %zero to %four step %one { + scf.for %channel = %zero to %sixteen step %one { + %position_base = arith.muli %position, %twenty : index + %with_channel = arith.addi %position_base, %channel : index + %value = arith.subi %with_channel, %eight : index + %value_i32 = arith.index_cast %value : index to i32 + %channel_row = arith.divui %channel, %four : index + %row_base = arith.muli %position, %four : index + %row = arith.addi %row_base, %channel_row : index + %lane = arith.remui %channel, %four : index + memref.store %value_i32, %input[%row, %lane] : memref<16x4xi32> + } + } + buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %input <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> + buckyball.mvin %scales <{bank = 2 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> + toint8_ball.int32_to_int8 <{values = 1 : i64, scales = 2 : i64, output = 3 : i64, rows = 4 : i64, relu}> + buckyball.mvout %output <{bank = 3 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> + func.call @check_result(%output) : (memref<4x16xi8>) -> () + buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + memref.dealloc %input : memref<16x4xi32> + memref.dealloc %scales : memref<4x4xf32> + memref.dealloc %output : memref<4x16xi8> + return %zero_i8 : i8 +} diff --git a/examples/prefab/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8_main.cpp b/examples/prefab/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8_main.cpp new file mode 100644 index 000000000..0c59f734f --- /dev/null +++ b/examples/prefab/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8_main.cpp @@ -0,0 +1,37 @@ +#include +#include +#include + +static void fail(void) { +#ifdef BAREMETAL + *(volatile uint32_t *)0x60000000 = 1; + while (1) { + } +#else + exit(1); +#endif +} + +extern "C" void check_result(int8_t *allocated, int8_t *aligned, int64_t offset, + int64_t rows, int64_t columns, int64_t row_stride, + int64_t column_stride) { + (void)allocated; + if (rows != 4 || columns != 16 || row_stride != 16 || column_stride != 1) + fail(); + int8_t *output = aligned + offset; + for (int row = 0; row < 4; ++row) { + for (int channel = 0; channel < 16; ++channel) { + int expected = row * 20 + channel - 8; + if (expected < 0) + expected = 0; + int actual = output[row * row_stride + channel * column_stride]; + if (actual != expected) { + printf( + "FAILED: quant_i32_to_i8 row=%d channel=%d expected=%d actual=%d\n", + row, channel, expected, actual); + fail(); + } + } + } + printf("PASSED: quant_i32_to_i8 relu\n"); +} diff --git a/examples/prefab/balls/toint8/workloads/mlir_tests/bank/CMakeLists.txt b/examples/prefab/balls/toint8/workloads/mlir_tests/bank/CMakeLists.txt new file mode 100644 index 000000000..1f1ce2ee5 --- /dev/null +++ b/examples/prefab/balls/toint8/workloads/mlir_tests/bank/CMakeLists.txt @@ -0,0 +1,20 @@ +if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) + set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) +endif() +set(BUCKYBALL_MLIR_TEST_PREFIX bank) + +set(_lines ${DESIGN_PROFILES_${BUCKYBALL_MLIR_ACTIVE_TARGET}_THREAD_MEM_BANK_ENTRIES}) +math(EXPR _remainder "${_lines} % 4") +if(_lines LESS 4 OR NOT (_remainder EQUAL 0)) + message(FATAL_ERROR + "ToInt8 INT32 BankOp test requires bank entries divisible by four " + "(got lines=${_lines})") +endif() +# One QUANT_I32_TO_I8 instruction accepts at most 508 packed INT32 rows. +set(BB_QUANT_I32_INPUT_ROWS ${_lines}) +if(BB_QUANT_I32_INPUT_ROWS GREATER 508) + set(BB_QUANT_I32_INPUT_ROWS 508) +endif() +math(EXPR BB_QUANT_I32_OUTPUT_ROWS "${BB_QUANT_I32_INPUT_ROWS} / 4") +add_buckyball_mlir_test(quant_i32_pack16 TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) +add_buckyball_mlir_test(quant_f32_pack16 TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/prefab/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16.mlir.in b/examples/prefab/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16.mlir.in new file mode 100644 index 000000000..834cbcce0 --- /dev/null +++ b/examples/prefab/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16.mlir.in @@ -0,0 +1,36 @@ +func.func private @check_result(memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8>) -> () +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %four = arith.constant 4 : index + %sixteen = arith.constant 16 : index + %rows = arith.constant @BB_QUANT_I32_OUTPUT_ROWS@ : index + %scale = arith.constant 1.0 : f32 + %input = memref.alloc() : memref<@BB_BANK_LINES@x4xf32> + %output = memref.alloc() : memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8> + scf.for %position = %zero to %rows step %one { + scf.for %channel = %zero to %sixteen step %one { + %base = arith.muli %position, %sixteen : index + %linear = arith.addi %base, %channel : index + %value_i32 = arith.index_cast %linear : index to i32 + %value = arith.sitofp %value_i32 : i32 to f32 + %row_base = arith.muli %position, %four : index + %channel_row = arith.divui %channel, %four : index + %row = arith.addi %row_base, %channel_row : index + %lane = arith.remui %channel, %four : index + memref.store %value, %input[%row, %lane] : memref<@BB_BANK_LINES@x4xf32> + } + } + %input_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %output_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %loaded_input = buckyball.bank_mvin %input, %input_bank <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x4xf32> + %result = toint8_ball.bank_fp32_to_int8 %loaded_input -> %output_bank multiplier(%scale) <{rows = @BB_BANK_LINES@ : i64}> + %stored = buckyball.bank_mvout %output, %result <{depth = @BB_QUANT_I32_OUTPUT_ROWS@ : i64, stride = 1 : i64}> : memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8> + func.call @check_result(%output) : (memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8>) -> () + buckyball.bank_release %loaded_input + buckyball.bank_release %stored + memref.dealloc %input : memref<@BB_BANK_LINES@x4xf32> + memref.dealloc %output : memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8> + return %zero_i8 : i8 +} diff --git a/examples/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16_main.cpp b/examples/prefab/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16_main.cpp similarity index 75% rename from examples/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16_main.cpp rename to examples/prefab/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16_main.cpp index ecb134ef2..35caaee66 100644 --- a/examples/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16_main.cpp +++ b/examples/prefab/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16_main.cpp @@ -1,13 +1,13 @@ -#include #include #include #include extern "C" void check_result(int8_t *, int8_t *data, int64_t offset, int64_t rows, int64_t lanes, int64_t stride, int64_t lane_stride) { - constexpr int outputRows = BANK_LINES / 4 < 127 ? BANK_LINES / 4 : 127; - if (rows != outputRows || lanes != BANK_WIDTH / 8 || stride != lanes || - lane_stride != 1) + constexpr int outputRows = + THREAD_MEM_BANK_ENTRIES / 4 < 127 ? THREAD_MEM_BANK_ENTRIES / 4 : 127; + if (rows != outputRows || lanes != THREAD_MEM_BANK_WIDTH / 8 || + stride != lanes || lane_stride != 1) exit(1); for (int row = 0; row < rows; ++row) for (int lane = 0; lane < lanes; ++lane) { diff --git a/examples/prefab/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16.mlir.in b/examples/prefab/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16.mlir.in new file mode 100644 index 000000000..af2a34393 --- /dev/null +++ b/examples/prefab/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16.mlir.in @@ -0,0 +1,41 @@ +func.func private @check_result(memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8>) -> () +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %one_f32 = arith.constant 1.0 : f32 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %four = arith.constant 4 : index + %sixteen = arith.constant 16 : index + %rows = arith.constant @BB_QUANT_I32_OUTPUT_ROWS@ : index + %input = memref.alloc() : memref<@BB_QUANT_I32_INPUT_ROWS@x4xi32> + %scale = memref.alloc() : memref<4x4xf32> + %output = memref.alloc() : memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8> + linalg.fill ins(%one_f32 : f32) outs(%scale : memref<4x4xf32>) + scf.for %position = %zero to %rows step %one { + scf.for %channel = %zero to %sixteen step %one { + %base = arith.muli %position, %sixteen : index + %linear = arith.addi %base, %channel : index + %value = arith.index_cast %linear : index to i32 + %row_base = arith.muli %position, %four : index + %channel_row = arith.divui %channel, %four : index + %row = arith.addi %row_base, %channel_row : index + %lane = arith.remui %channel, %four : index + memref.store %value, %input[%row, %lane] : memref<@BB_QUANT_I32_INPUT_ROWS@x4xi32> + } + } + %input_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %scale_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %output_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %loaded_input = buckyball.bank_mvin %input, %input_bank <{depth = @BB_QUANT_I32_INPUT_ROWS@ : i64, stride = 1 : i64}> : memref<@BB_QUANT_I32_INPUT_ROWS@x4xi32> + %loaded_scale = buckyball.bank_mvin %scale, %scale_bank <{depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> + %result = toint8_ball.bank_int32_to_int8 %loaded_input, %loaded_scale -> %output_bank <{rows = @BB_QUANT_I32_OUTPUT_ROWS@ : i64}> + %stored = buckyball.bank_mvout %output, %result <{depth = @BB_QUANT_I32_OUTPUT_ROWS@ : i64, stride = 1 : i64}> : memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8> + func.call @check_result(%output) : (memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8>) -> () + buckyball.bank_release %loaded_input + buckyball.bank_release %loaded_scale + buckyball.bank_release %stored + memref.dealloc %input : memref<@BB_QUANT_I32_INPUT_ROWS@x4xi32> + memref.dealloc %scale : memref<4x4xf32> + memref.dealloc %output : memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8> + return %zero_i8 : i8 +} diff --git a/examples/prefab/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16_main.cpp b/examples/prefab/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16_main.cpp new file mode 100644 index 000000000..574fcb98f --- /dev/null +++ b/examples/prefab/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16_main.cpp @@ -0,0 +1,23 @@ +#include +#include +#include +extern "C" void check_result(int8_t *, int8_t *data, int64_t offset, + int64_t rows, int64_t lanes, int64_t stride, + int64_t lane_stride) { + const int expected_rows = + (THREAD_MEM_BANK_ENTRIES < 508 ? THREAD_MEM_BANK_ENTRIES : 508) / 4; + if (rows != expected_rows || lanes != THREAD_MEM_BANK_WIDTH / 8 || + stride != lanes || lane_stride != 1) + exit(1); + for (int row = 0; row < rows; ++row) + for (int lane = 0; lane < lanes; ++lane) { + int expected = row * lanes + lane; + if (expected > 127) + expected = 127; + if (data[offset + row * stride + lane] != expected) { + printf("quant i32 bank mismatch row=%d lane=%d\n", row, lane); + exit(1); + } + } + printf("quant i32 bank PASS rows=%d\n", THREAD_MEM_BANK_ENTRIES); +} diff --git a/examples/prefab/balls/trace/arch/README.md b/examples/prefab/balls/trace/arch/README.md new file mode 100644 index 000000000..e1626e2ce --- /dev/null +++ b/examples/prefab/balls/trace/arch/README.md @@ -0,0 +1,64 @@ +# TraceBall — debug/trace Ball + +## Overview + +TraceBall is a non-computing Ball that measures elapsed cycles over arbitrary +code regions with up to sixteen independent cycle counters. It touches no bank. +The DPI-C hook lives inside TraceBall only and does not affect other modules. + +The framework requires every Ball to own at least one read and one write bank +port, so the design config gives TraceBall `inBW = 1, outBW = 1`; TraceBall +ties those ports idle and never issues a bank request. + +--- + +## Instruction: `bdb_counter` + +The selected Core's `ballISA` assigns the `funct7` of `BDB_COUNTER`. +Cycle counter control; completes in one cycle after acceptance. + +rs1 is unused and zero. + +rs2 layout (64-bit): +``` +rs2[3:0] = subcmd subcommand (0=START, 1=STOP, 2=READ; 3..15 are illegal) +rs2[7:4] = ctr_id counter id (0–15) +rs2[63:8] = payload +``` + +| subcmd | name | behavior | payload | +|--------|------|----------|---------| +| 0 | `CTR_START` | Start counter `ctr_id`, record the current cycle and the tag | tag, echoed in the trace | +| 1 | `CTR_STOP` | Emit elapsed cycles of an active `ctr_id`, then free it | ignored | +| 2 | `CTR_READ` | Emit elapsed cycles of an active `ctr_id`; it keeps running | ignored | + +Illegal inputs stop the simulation: an RTL assertion and a BEMU panic. +- `subcmd` 3..15. +- `CTR_STOP` or `CTR_READ` of a counter that is not started. + +## Counter trace + +With `--ctrace`, both Verilator (`dpi_ctrace`) and BEMU append one NDJSON +record per command to `bdb.ndjson`, so the same tools read both: +``` +{"type":"ctrace","clk":10042,"event":"ctr_start","ctr_id":0,"tag":"0xDEAD","cycle":10042} +{"type":"ctrace","clk":10429,"event":"ctr_stop","ctr_id":0,"tag":"0xDEAD","elapsed":387,"cycle":10429} +{"type":"ctrace","clk":10242,"event":"ctr_read","ctr_id":0,"current":200,"cycle":10242} +``` +`cycle` is the Ball's cycle count and `elapsed`/`current` is `cycle` minus the +cycle of the counter's START. `clk` is the simulator clock. BEMU uses the issue +cycle of the instruction for both `clk` and `cycle`. + +## Example + +```c +bdb_counter_start(0, 0xB001); // outer: whole conv + bdb_counter_start(1, 0xB002); // inner: im2col + bb_im2col(...); + bdb_counter_stop(1); + + bdb_counter_start(2, 0xB003); // inner: matmul + bb_vecmat16(...); + bdb_counter_stop(2); +bdb_counter_stop(0); +``` diff --git a/examples/prefab/balls/trace/arch/Trace.scala b/examples/prefab/balls/trace/arch/Trace.scala new file mode 100644 index 000000000..979d842a7 --- /dev/null +++ b/examples/prefab/balls/trace/arch/Trace.scala @@ -0,0 +1,222 @@ +package examples.balls.trace + +import framework.acc.balldomain.configs.BallRouteParam + +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import framework.acc.balldomain.ball.blink.{BallCommand, BallResponse} +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.acc.balldomain.ball.blink.BallStatus +import framework.acc.memdomain.{BankRead, BankWrite} + +/** + * Trace — TraceBall inner processing unit. + * + * Handles cycle counter management (START/STOP/READ). + */ +@instantiable +class Trace( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val routingParam: BallDomainParam, + val instance: BallRouteParam) + extends Module { + val commandDataBits = frontendParam.commandDataBits + + val ballMapping = instance + + val inBW = ballMapping.inBW + val outBW = ballMapping.outBW + + val bankWidth = memParam.bankWidth + + @public + val io = IO(new Bundle { + + val cmdReq = Flipped(Decoupled(new BallCommand(frontendParam, memParam))) + + val cmdResp = Decoupled(new BallResponse(frontendParam)) + + val bankRead = Vec( + inBW, + Flipped(new BankRead( + frontendParam, + memParam + )) + ) + + val bankWrite = Vec( + outBW, + Flipped(new BankWrite( + frontendParam, + memParam + )) + ) + + val status = new BallStatus + }) + + // ============================================================ + // Constants + // ============================================================ + val CTR_START = 0.U(4.W) + val CTR_STOP = 1.U(4.W) + val CTR_READ = 2.U(4.W) + + val NUM_COUNTERS = 16 + + // ============================================================ + // State machine + // ============================================================ + val idle :: sCounter :: complete :: Nil = Enum(3) + val state = RegInit(idle) + + // ============================================================ + // Registers + // ============================================================ + val rob_id_reg = RegInit(0.U(log2Up(frontendParam.rob_entries).W)) + + // Counter-specific registers + val subcmd_reg = RegInit(0.U(4.W)) + val ctr_id_reg = RegInit(0.U(4.W)) + val payload_reg = RegInit(0.U(56.W)) + + // Cycle counters: 16 independent counters + val cycleCounter = RegInit(0.U(64.W)) + cycleCounter := cycleCounter + 1.U + + val ctrStartCycle = RegInit(VecInit(Seq.fill(NUM_COUNTERS)(0.U(64.W)))) + val ctrTag = RegInit(VecInit(Seq.fill(NUM_COUNTERS)(0.U(56.W)))) + val ctrActive = RegInit(VecInit(Seq.fill(NUM_COUNTERS)(false.B))) + + // ============================================================ + // DPI-C modules + // ============================================================ + val ctraceDpi = Instantiate(new CTraceDPI) + ctraceDpi.io.clock := clock + ctraceDpi.io.reset := reset.asBool + ctraceDpi.io.subcmd := 0.U + ctraceDpi.io.ctr_id := 0.U + ctraceDpi.io.tag := 0.U + ctraceDpi.io.elapsed := 0.U + ctraceDpi.io.cycle := cycleCounter + ctraceDpi.io.enable := false.B + + // ============================================================ + // External bank port defaults + // ============================================================ + for (i <- 0 until inBW) { + io.bankRead(i).rob_id := rob_id_reg + io.bankRead(i).bank_id := 0.U + io.bankRead(i).group_id := 0.U + io.bankRead(i).io.req.valid := false.B + io.bankRead(i).io.req.bits.addr := 0.U + io.bankRead(i).io.resp.ready := false.B + } + + for (i <- 0 until outBW) { + io.bankWrite(i).rob_id := rob_id_reg + io.bankWrite(i).bank_id := 0.U + io.bankWrite(i).group_id := 0.U + io.bankWrite(i).io.req.valid := false.B + io.bankWrite(i).io.req.bits.addr := 0.U + io.bankWrite(i).io.req.bits.data := 0.U + io.bankWrite(i).io.req.bits.mask := VecInit( + Seq.fill(memParam.bankMaskLen)(0.U(1.W)) + ) + io.bankWrite(i).io.resp.ready := false.B + } + + // ============================================================ + // Command interface + // ============================================================ + io.cmdReq.ready := state === idle + io.cmdResp.valid := false.B + io.cmdResp.bits.rob_id := rob_id_reg + + // ============================================================ + // State machine + // ============================================================ + switch(state) { + is(idle) { + when(io.cmdReq.fire) { + rob_id_reg := io.cmdReq.bits.rob_id + + val cmd = io.cmdReq.bits + val rs2 = cmd.rs2 + + assert(rs2(3, 0) <= CTR_READ, "TraceBall subcmd 3..15 is illegal") + assert(rs2(3, 0) === CTR_START || ctrActive(rs2(7, 4)), "TraceBall STOP/READ of a counter that is not started") + + // Counter fields from rs2 + subcmd_reg := rs2(3, 0) + ctr_id_reg := rs2(7, 4) + payload_reg := rs2(63, 8) + + state := sCounter + } + } + + // ---------------------------------------------------------- + // Counter instruction: execute and complete in 1 cycle + // ---------------------------------------------------------- + is(sCounter) { + val cid = ctr_id_reg + + ctraceDpi.io.subcmd := subcmd_reg + ctraceDpi.io.ctr_id := cid + + switch(subcmd_reg) { + is(CTR_START) { + ctrStartCycle(cid) := cycleCounter + ctrTag(cid) := payload_reg + ctrActive(cid) := true.B + + ctraceDpi.io.tag := payload_reg + ctraceDpi.io.elapsed := 0.U + ctraceDpi.io.enable := true.B + } + is(CTR_STOP) { + val elapsed = cycleCounter - ctrStartCycle(cid) + ctrActive(cid) := false.B + + ctraceDpi.io.tag := ctrTag(cid) + ctraceDpi.io.elapsed := elapsed + ctraceDpi.io.enable := true.B + } + is(CTR_READ) { + val current = cycleCounter - ctrStartCycle(cid) + + ctraceDpi.io.tag := ctrTag(cid) + ctraceDpi.io.elapsed := current + ctraceDpi.io.enable := true.B + } + } + + state := complete + } + + // ---------------------------------------------------------- + // Complete: fire cmdResp + // ---------------------------------------------------------- + is(complete) { + io.cmdResp.valid := true.B + io.cmdResp.bits.rob_id := rob_id_reg + when(io.cmdResp.fire) { + state := idle + } + } + } + + // ============================================================ + // Status + // ============================================================ + io.status.idle := state === idle + io.status.running := state =/= idle && state =/= complete +} diff --git a/examples/prefab/balls/trace/arch/TraceBall.scala b/examples/prefab/balls/trace/arch/TraceBall.scala new file mode 100644 index 000000000..921381767 --- /dev/null +++ b/examples/prefab/balls/trace/arch/TraceBall.scala @@ -0,0 +1,68 @@ +package examples.balls.trace + +import framework.acc.balldomain.bbus.{BallFactory, BallPorts} +import buckyball.balls.trace.{Params => ParametersProto} +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.balldomain.ball.blink.{BlinkIO, HasBlink} + +/** + * TraceBall — Debug trace Ball providing cycle counters. + * + * The Thread ballISA assigns the funct7 encoding for bdb_counter. + */ +@instantiable +class TraceBall( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Module + with HasBlink { + val ball = bp.balls.find(_.id == id).get + + ball.parameters.unpack(classOf[ParametersProto]) + + val ballCommonConfig = ball + + val inBW = ballCommonConfig.inBW + val outBW = ballCommonConfig.outBW + + @public + val blink = IO(new BlinkIO(fp, mp, bp, id)) + + val traceUnit: Instance[Trace] = + Instantiate(new Trace(fp, mp, bp, ball)) + + traceUnit.io.cmdReq <> blink.cmdReq + traceUnit.io.cmdResp <> blink.cmdResp + + for (i <- 0 until inBW) { + traceUnit.io.bankRead(i) <> blink.bankRead(i) + } + + for (i <- 0 until outBW) { + traceUnit.io.bankWrite(i) <> blink.bankWrite(i) + } + + blink.status <> traceUnit.io.status + + // MMIO: this Ball does not consume MMIO metadata +} + +object TraceBall extends BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts = + BallPorts(Instantiate(new TraceBall(fp, mp, bp, id)).blink, None) + +} diff --git a/examples/balls/trace/arch/src/main/scala/TraceDPI.scala b/examples/prefab/balls/trace/arch/TraceDPI.scala similarity index 96% rename from examples/balls/trace/arch/src/main/scala/TraceDPI.scala rename to examples/prefab/balls/trace/arch/TraceDPI.scala index a7d1e94dd..d6cae579b 100644 --- a/examples/balls/trace/arch/src/main/scala/TraceDPI.scala +++ b/examples/prefab/balls/trace/arch/TraceDPI.scala @@ -1,11 +1,15 @@ package examples.balls.trace +import chisel3.experimental.hierarchy.{instantiable, public} + import chisel3._ import chisel3.util._ -import framework.dpi.DpiGuard +import framework.lib.dpi.DpiGuard +@instantiable class CTraceDPI extends BlackBox with HasBlackBoxInline { + @public val io = IO(new Bundle { val clock = Input(Clock()) val reset = Input(Bool()) diff --git a/examples/prefab/balls/trace/configs/default.toml b/examples/prefab/balls/trace/configs/default.toml new file mode 100644 index 000000000..014d280eb --- /dev/null +++ b/examples/prefab/balls/trace/configs/default.toml @@ -0,0 +1 @@ +# Trace Ball Configuration diff --git a/examples/prefab/balls/trace/configs/trace_params.proto b/examples/prefab/balls/trace/configs/trace_params.proto new file mode 100644 index 000000000..28fd8d1c2 --- /dev/null +++ b/examples/prefab/balls/trace/configs/trace_params.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; +package buckyball.balls.trace; +option java_package = "buckyball.balls.trace"; +option java_multiple_files = true; + +message Params { +} diff --git a/examples/prefab/balls/trace/emu/src/04_bdb_counter.rs b/examples/prefab/balls/trace/emu/src/04_bdb_counter.rs new file mode 100644 index 000000000..e00f4a233 --- /dev/null +++ b/examples/prefab/balls/trace/emu/src/04_bdb_counter.rs @@ -0,0 +1,51 @@ +//===- 04_bdb_counter.rs - BDB_COUNTER instruction -------------------------===// +// +// rs2[3:0] is the subcommand, rs2[7:4] the counter and rs2[63:8] the payload; +// the Ball touches no bank. Each command emits one counter trace record: +// START (0): the issue cycle becomes the counter's start, the payload its tag. +// STOP (1): elapsed = cycle - start of a started counter, which is freed. +// READ (2): like STOP but the counter keeps running. +// Subcommands 3..15 and STOP/READ of a counter that is not started are illegal. +// +//===----------------------------------------------------------------------===// + +use super::instruction::{BallModel, ExecContext}; + +const CTR_START: u32 = 0; +const CTR_STOP: u32 = 1; +const CTR_READ: u32 = 2; +const NUM_COUNTERS: usize = 16; + +/// (start cycle, tag) of each started counter. +#[derive(Default)] +struct Counters([Option<(u64, u64)>; NUM_COUNTERS]); + +pub struct BdbCounter; + +impl BallModel for BdbCounter { + fn exec(_xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + let subcmd = (xs2 & 0xf) as u32; + let ctr = ((xs2 >> 4) & 0xf) as u32; + let payload = xs2 >> 8; + assert!(subcmd <= CTR_READ, "bdb_counter: subcmd {subcmd} is illegal"); + let cycle = ctx.cycle; + let state = ctx.state::(); + let mut counters = state.lock().expect("bdb_counter state poisoned"); + let slot = &mut counters.0[ctr as usize]; + if subcmd == CTR_START { + *slot = Some((cycle, payload)); + bebop_bemu::thread::trace::ctrace(CTR_START, ctr, payload, 0, cycle); + return 0; + } + let (start, tag) = slot.unwrap_or_else(|| panic!("bdb_counter: counter {ctr} is not started")); + if subcmd == CTR_STOP { + *slot = None; + } + bebop_bemu::thread::trace::ctrace(subcmd, ctr, tag, cycle - start, cycle); + 0 + } + + fn latency(_xs1: u64, _xs2: u64) -> u64 { + 1 + } +} diff --git a/examples/prefab/balls/trace/emu/src/dispatch.rs b/examples/prefab/balls/trace/emu/src/dispatch.rs new file mode 100644 index 000000000..cfca2fbd7 --- /dev/null +++ b/examples/prefab/balls/trace/emu/src/dispatch.rs @@ -0,0 +1,31 @@ +use bebop_bemu::ball::{BallModel, ExecContext}; + +use super::f04_bdb_counter; + +const BALL_CLASS: &str = "examples.balls.trace.TraceBall"; + +pub fn execute_known( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, + ctx: &mut ExecContext, +) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("BDB_COUNTER") => Some(f04_bdb_counter::BdbCounter::exec(xs1, xs2, ctx)), + Some(_) | None => None, + } +} + +pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("BDB_COUNTER") => Some(f04_bdb_counter::BdbCounter::latency(xs1, xs2)), + Some(_) | None => None, + } +} diff --git a/examples/prefab/balls/trace/emu/src/lib.rs b/examples/prefab/balls/trace/emu/src/lib.rs new file mode 100644 index 000000000..61ae12d8d --- /dev/null +++ b/examples/prefab/balls/trace/emu/src/lib.rs @@ -0,0 +1,12 @@ +pub(crate) use bebop_bemu::ball as instruction; + +#[path = "04_bdb_counter.rs"] +mod f04_bdb_counter; + +mod dispatch; + +pub use dispatch::{cycles_after_issue, execute_known}; + +#[bebop_bemu::ball::linkme::distributed_slice(bebop_bemu::BALLS)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static BALL: bebop_bemu::Ball = bebop_bemu::Ball { execute: execute_known, cycles: cycles_after_issue }; diff --git a/examples/prefab/balls/trace/stack/src/traceOps.cpp b/examples/prefab/balls/trace/stack/src/traceOps.cpp new file mode 100644 index 000000000..b3c87d380 --- /dev/null +++ b/examples/prefab/balls/trace/stack/src/traceOps.cpp @@ -0,0 +1,77 @@ +//===- traceOps.cpp - TraceBall operations --------------------------------===// +// +// The dialect of TraceBall, its virtual operations building their physical ones +// and the instructions of the physical operations. +// +//===----------------------------------------------------------------------===// + +#include "traceOps.h" + +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +#include "mlir/Dialect/Arith/IR/Arith.h" + +using namespace mlir; +using namespace ::buddy::trace_ball; +using mlir::buddy::bankFields; +using mlir::buddy::constantI64; +using mlir::buddy::emitInstruction; +using mlir::buddy::field; + +#include "traceDialect.cpp.inc" + +#define GET_OP_CLASSES +#include "traceOps.cpp.inc" + +void TraceBallDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "traceOps.cpp.inc" + >(); +} + +namespace { +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallDialect([](DialectRegistry ®istry) { + registry.insert(); + }), + true); +} // namespace + +// One BDB_COUNTER instruction: the subcommand in rs2[3:0], the counter in +// rs2[7:4] and the tag of counter_start from rs2[8]. +static LogicalResult counter(Operation *op, OpBuilder &builder, + uint64_t subcommand, int64_t counter, Value tag) { + Location loc = op->getLoc(); + if (counter < 0 || counter > 15) + return op->emitOpError("BDB counter needs a counter in [0, 15]"); + Value rs2 = + constantI64(builder, loc, field(subcommand, 0, 3) | field(counter, 4, 7)); + if (tag) + rs2 = arith::OrIOp::create( + builder, loc, rs2, + arith::ShLIOp::create(builder, loc, tag, constantI64(builder, loc, 8))); + emitInstruction(builder, loc, + buckyball_target::getBuckyballFunct7("BDB_COUNTER"), + constantI64(builder, loc, 0), rs2); + return success(); +} + +SmallVector CounterStartOp::getSourceBanks() { return {}; } +int64_t CounterStartOp::getDestinationBank() { return 0; } +LogicalResult CounterStartOp::lowerToInstructions(OpBuilder &builder) { + return counter(*this, builder, 0, getCounter(), getTag()); +} + +SmallVector CounterStopOp::getSourceBanks() { return {}; } +int64_t CounterStopOp::getDestinationBank() { return 0; } +LogicalResult CounterStopOp::lowerToInstructions(OpBuilder &builder) { + return counter(*this, builder, 1, getCounter(), Value()); +} + +SmallVector CounterReadOp::getSourceBanks() { return {}; } +int64_t CounterReadOp::getDestinationBank() { return 0; } +LogicalResult CounterReadOp::lowerToInstructions(OpBuilder &builder) { + return counter(*this, builder, 2, getCounter(), Value()); +} diff --git a/examples/prefab/balls/trace/stack/src/traceOps.h b/examples/prefab/balls/trace/stack/src/traceOps.h new file mode 100644 index 000000000..58cfa9dc9 --- /dev/null +++ b/examples/prefab/balls/trace/stack/src/traceOps.h @@ -0,0 +1,20 @@ +//===- traceOps.h - TraceBall operations ----------------------------------===// + +#ifndef TRACE_OPS_H +#define TRACE_OPS_H + +#include "Interfaces/BankInterfaces.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "traceDialect.h.inc" + +#define GET_OP_CLASSES +#include "traceOps.h.inc" + +#endif // TRACE_OPS_H diff --git a/examples/prefab/balls/trace/stack/src/traceOps.td b/examples/prefab/balls/trace/stack/src/traceOps.td new file mode 100644 index 000000000..911a3508d --- /dev/null +++ b/examples/prefab/balls/trace/stack/src/traceOps.td @@ -0,0 +1,49 @@ +//===- traceOps.td - TraceBall operations ------------------*- tablegen -*-===// +// +// The BDB counter touches no bank: its operations exist only on physical +// banks, with no sources and destination 0, and rs1 is zero. `counter` (0 to +// 15) selects the counter. counter_start starts it with the i64 tag; +// counter_read traces its current value; counter_stop traces the elapsed +// cycles and stops it. +// +//===----------------------------------------------------------------------===// + +#ifndef TRACE_OPS +#define TRACE_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Interfaces/BankInterfaces.td" + +def TraceBall_Dialect : Dialect { + let name = "trace_ball"; + let cppNamespace = "::buddy::trace_ball"; + let summary = "The operations of TraceBall"; +} + +class TraceBall_Op traits = []> + : Op])>; + +def CounterStartOp : TraceBall_Op<"counter_start", + [DeclareOpInterfaceMethods]> { + let summary = "Start a BDB counter with a tag"; + let arguments = (ins I64:$tag, I64Attr:$counter); + let assemblyFormat = "$tag prop-dict attr-dict"; +} + +def CounterStopOp : TraceBall_Op<"counter_stop", + [DeclareOpInterfaceMethods]> { + let summary = "Trace the elapsed cycles of a BDB counter and stop it"; + let arguments = (ins I64Attr:$counter); + let assemblyFormat = "prop-dict attr-dict"; +} + +def CounterReadOp : TraceBall_Op<"counter_read", + [DeclareOpInterfaceMethods]> { + let summary = "Trace the current value of a BDB counter"; + let arguments = (ins I64Attr:$counter); + let assemblyFormat = "prop-dict attr-dict"; +} + +#endif // TRACE_OPS diff --git a/examples/prefab/balls/trace/verify/casegen/Cargo.lock b/examples/prefab/balls/trace/verify/casegen/Cargo.lock new file mode 100644 index 000000000..3c7929614 --- /dev/null +++ b/examples/prefab/balls/trace/verify/casegen/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "trace-casegen" +version = "0.1.0" diff --git a/examples/prefab/balls/trace/verify/casegen/Cargo.toml b/examples/prefab/balls/trace/verify/casegen/Cargo.toml new file mode 100644 index 000000000..de87ef0c7 --- /dev/null +++ b/examples/prefab/balls/trace/verify/casegen/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "trace-casegen" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["rlib", "cdylib"] + +[lints.rust] +unused_must_use = "deny" diff --git a/examples/prefab/balls/trace/verify/casegen/src/casegen.rs b/examples/prefab/balls/trace/verify/casegen/src/casegen.rs new file mode 100644 index 000000000..370ec5aa3 --- /dev/null +++ b/examples/prefab/balls/trace/verify/casegen/src/casegen.rs @@ -0,0 +1,60 @@ +use crate::model::{Command, CTR_READ, CTR_START, CTR_STOP, NUM_COUNTERS, PAYLOAD_MASK}; + +fn start(ctr: u32, payload: u64) -> Command { + Command { subcmd: CTR_START, ctr, payload } +} + +fn stop(ctr: u32, payload: u64) -> Command { + Command { subcmd: CTR_STOP, ctr, payload } +} + +fn read(ctr: u32, payload: u64) -> Command { + Command { subcmd: CTR_READ, ctr, payload } +} + +pub fn gen_case(index: u32) -> Vec { + match index { + // One measured region with a read inside. + 0 => vec![start(0, 0xa001), read(0, 0), stop(0, 0)], + // Nested and sibling regions, as in the README example. + 1 => vec![ + start(0, 0xb001), + start(1, 0xb002), + stop(1, 0), + start(2, 0xb003), + read(0, 0), + stop(2, 0), + stop(0, 0), + ], + // Every counter, full-width tags; STOP and READ payloads are ignored. + 2 => { + let mut cmds = Vec::new(); + for ctr in 0..NUM_COUNTERS as u32 { + let tag = if ctr % 2 == 0 { + PAYLOAD_MASK ^ (u64::from(ctr) << 40) + } else { + 0x0055_aa55_aa55_aa55 ^ u64::from(ctr) + }; + cmds.push(start(ctr, tag)); + } + for ctr in 0..NUM_COUNTERS as u32 { + cmds.push(read(ctr, PAYLOAD_MASK)); + } + for ctr in (0..NUM_COUNTERS as u32).rev() { + cmds.push(stop(ctr, PAYLOAD_MASK ^ u64::from(ctr))); + } + cmds + } + // A restarted counter measures from the new START with the new tag; + // a zero tag is a valid tag. + 3 => vec![ + start(5, 0), + stop(5, 0), + start(5, 0x2222), + read(5, 0), + read(5, 0), + stop(5, 0), + ], + _ => panic!("trace: unsupported case {index}"), + } +} diff --git a/examples/prefab/balls/trace/verify/casegen/src/lib.rs b/examples/prefab/balls/trace/verify/casegen/src/lib.rs new file mode 100644 index 000000000..0c8502e21 --- /dev/null +++ b/examples/prefab/balls/trace/verify/casegen/src/lib.rs @@ -0,0 +1,110 @@ +mod casegen; +mod model; + +use std::cell::RefCell; + +use model::{Command, Counters, Record}; + +#[repr(C)] +#[derive(Clone, Copy, Debug)] +pub struct TraceCmd { + pub rs2_lo: u32, + pub rs2_hi: u32, + pub rob_id: u32, +} + +struct Case { + cmds: Vec, + counters: Counters, + records: usize, + errors: usize, +} + +thread_local! { + static CURRENT: RefCell> = const { RefCell::new(None) }; +} + +#[no_mangle] +pub extern "C" fn trace_case_load(index: u32) -> u32 { + let cmds = casegen::gen_case(index); + let count = cmds.len() as u32; + CURRENT.with(|current| { + *current.borrow_mut() = Some(Case { cmds, counters: Counters::default(), records: 0, errors: 0 }) + }); + count +} + +fn with_case R, R>(f: F) -> R { + CURRENT.with(|current| match current.borrow_mut().as_mut() { + Some(case) => f(case), + None => panic!("trace_case: no case loaded; call trace_case_load first"), + }) +} + +#[no_mangle] +pub extern "C" fn trace_case_cmd(index: u32, out_ptr: *mut TraceCmd) { + if out_ptr.is_null() { + panic!("trace_case_cmd: null out_ptr"); + } + with_case(|case| { + let cmd = match case.cmds.get(index as usize) { + Some(cmd) => *cmd, + None => panic!("trace_case_cmd: index {index} out of range {}", case.cmds.len()), + }; + let rs2 = cmd.rs2(); + unsafe { + *out_ptr = TraceCmd { rs2_lo: rs2 as u32, rs2_hi: (rs2 >> 32) as u32, rob_id: index % 16 }; + } + }); +} + +// Records observed so far; -1 once any record mismatched the model. +#[no_mangle] +pub extern "C" fn trace_case_records() -> i32 { + with_case(|case| if case.errors != 0 { -1 } else { case.records as i32 }) +} + +// The trace hook the RTL calls once per counter command. +#[no_mangle] +#[allow(clippy::too_many_arguments)] +pub extern "C" fn dpi_ctrace( + subcmd: u32, + ctr_id: u32, + tag_lo: u32, + tag_hi: u32, + elapsed_lo: u32, + elapsed_hi: u32, + cycle_lo: u32, + cycle_hi: u32, +) { + let got = Record { + subcmd, + ctr: ctr_id, + tag: u64::from(tag_lo) | (u64::from(tag_hi) << 32), + elapsed: u64::from(elapsed_lo) | (u64::from(elapsed_hi) << 32), + cycle: u64::from(cycle_lo) | (u64::from(cycle_hi) << 32), + }; + with_case(|case| { + let index = case.records; + case.records += 1; + let cmd = match case.cmds.get(index) { + Some(cmd) => *cmd, + None => { + eprintln!("[TRACE-REF] extra record {index}: {got:?}"); + case.errors += 1; + return; + } + }; + match case.counters.expect(&cmd, got.cycle) { + Ok(want) if want == got => {} + Ok(want) => { + eprintln!("[TRACE-REF] record {index} of {cmd:?}: got {got:?} expected {want:?}"); + case.errors += 1; + } + Err(reason) => { + eprintln!("[TRACE-REF] record {index} of {cmd:?}: {reason}"); + case.errors += 1; + } + } + }); +} diff --git a/examples/prefab/balls/trace/verify/casegen/src/model.rs b/examples/prefab/balls/trace/verify/casegen/src/model.rs new file mode 100644 index 000000000..89e91d673 --- /dev/null +++ b/examples/prefab/balls/trace/verify/casegen/src/model.rs @@ -0,0 +1,69 @@ +// TraceBall BDB_COUNTER contract. rs2[3:0] is the subcommand, rs2[7:4] the +// counter and rs2[63:8] the payload; the Ball touches no bank. Each command +// emits one trace record (subcmd, counter, tag, elapsed, cycle): +// START (0): records cycle as the counter's start and the payload as its +// tag; the record carries the payload and elapsed 0. +// STOP (1): elapsed = cycle - start; the payload is ignored. +// READ (2): like STOP but the counter keeps running. + +pub const CTR_START: u32 = 0; +pub const CTR_STOP: u32 = 1; +pub const CTR_READ: u32 = 2; +pub const NUM_COUNTERS: usize = 16; +pub const PAYLOAD_MASK: u64 = (1 << 56) - 1; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Command { + pub subcmd: u32, + pub ctr: u32, + pub payload: u64, +} + +impl Command { + pub fn rs2(&self) -> u64 { + if self.subcmd > CTR_READ || self.ctr as usize >= NUM_COUNTERS || self.payload > PAYLOAD_MASK { + panic!("trace: invalid command {self:?}"); + } + (self.payload << 8) | (u64::from(self.ctr) << 4) | u64::from(self.subcmd) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Record { + pub subcmd: u32, + pub ctr: u32, + pub tag: u64, + pub elapsed: u64, + pub cycle: u64, +} + +#[derive(Default)] +pub struct Counters { + start: [Option<(u64, u64)>; NUM_COUNTERS], + last_cycle: Option, +} + +impl Counters { + // The expected record of `cmd` given the cycle the Ball reported for it. + pub fn expect(&mut self, cmd: &Command, cycle: u64) -> Result { + if let Some(last) = self.last_cycle { + if cycle <= last { + return Err(format!("cycle {cycle} does not advance past {last}")); + } + } + self.last_cycle = Some(cycle); + let ctr = cmd.ctr as usize; + if cmd.subcmd == CTR_START { + self.start[ctr] = Some((cycle, cmd.payload)); + return Ok(Record { subcmd: CTR_START, ctr: cmd.ctr, tag: cmd.payload, elapsed: 0, cycle }); + } + let (start, tag) = match self.start[ctr] { + Some(entry) => entry, + None => return Err(format!("counter {ctr} used before START")), + }; + if cmd.subcmd == CTR_STOP { + self.start[ctr] = None; + } + Ok(Record { subcmd: cmd.subcmd, ctr: cmd.ctr, tag, elapsed: cycle - start, cycle }) + } +} diff --git a/examples/prefab/balls/trace/verify/filelists/trace_ball.f b/examples/prefab/balls/trace/verify/filelists/trace_ball.f new file mode 100644 index 000000000..e7d579bea --- /dev/null +++ b/examples/prefab/balls/trace/verify/filelists/trace_ball.f @@ -0,0 +1,26 @@ ++incdir+@UVM@/src/core ++incdir+@UVM@/src/ball ++incdir+@UVM@/src/ball/agents/cmd ++incdir+@UVM@/src/ball/agents/mem ++incdir+@UVM@/src/ball/agents/resp ++incdir+@UVM@/src/ball/cov ++incdir+@UVM@/src/ball/env ++incdir+src ++incdir+src/pkg ++incdir+src/common ++incdir+src/seq ++incdir+src/cov ++incdir+src/env ++incdir+src/tests ++incdir+@RTL@ + +@UVM@/src/core/package.sv +@UVM@/src/ball/bb_blink_if.sv +@UVM@/src/ball/bb_uvm_pkg.sv + +@RTL@/CTraceDPI.v +@RTL@/Trace.sv +@RTL@/TraceBall.sv + +src/pkg/trace_pkg.sv +src/tb_top.sv diff --git a/examples/prefab/balls/trace/verify/src/common/trace_defs.svh b/examples/prefab/balls/trace/verify/src/common/trace_defs.svh new file mode 100644 index 000000000..97790ad61 --- /dev/null +++ b/examples/prefab/balls/trace/verify/src/common/trace_defs.svh @@ -0,0 +1,17 @@ +typedef struct { + int unsigned rs2_lo; + int unsigned rs2_hi; + int unsigned rob_id; +} trace_cmd_dpi_t; + +import "DPI-C" function int unsigned trace_case_load(input int unsigned index); +import "DPI-C" function void trace_case_cmd(input int unsigned index, output trace_cmd_dpi_t cmd); +import "DPI-C" function int trace_case_records(); + +`ifndef BDB_COUNTER_FUNCT7 +`error "BDB_COUNTER_FUNCT7 must be provided by the selected ballISA" +`endif + +localparam int BDB_COUNTER_THREAD_FUNCT7 = `BDB_COUNTER_FUNCT7; +localparam int TRACE_NUM_CASES = 4; +localparam int TRACE_TIMEOUT_CYCLES = 2000; diff --git a/examples/prefab/balls/trace/verify/src/common/trace_items.svh b/examples/prefab/balls/trace/verify/src/common/trace_items.svh new file mode 100644 index 000000000..d5d41ad37 --- /dev/null +++ b/examples/prefab/balls/trace/verify/src/common/trace_items.svh @@ -0,0 +1,29 @@ +class trace_cmd_item extends bb_blink_cmd_item; + `uvm_object_utils(trace_cmd_item) + + function new(string name = "trace_cmd_item"); + super.new(name); + endfunction + + // bdb_counter: rs1 is zero, so every bank field is zero; rs2 carries the + // subcommand, counter and payload. + function void load_rust_case(int unsigned index); + trace_cmd_dpi_t cmd; + trace_case_cmd(index, cmd); + rob_id = cmd.rob_id[BB_BLINK_ROB_ID_W-1:0]; + funct = BDB_COUNTER_THREAD_FUNCT7[6:0]; + rs2 = {cmd.rs2_hi, cmd.rs2_lo}; + iter = '0; + op1_bank = '0; + op2_bank = '0; + wr_bank = '0; + op1_group = '0; + op2_group = '0; + wr_group = '0; + op1_col = '0; + op2_col = '0; + wr_col = '0; + wr_read_bank = '0; + wr_arch_bank = '0; + endfunction +endclass diff --git a/examples/prefab/balls/trace/verify/src/cov/trace_cov.svh b/examples/prefab/balls/trace/verify/src/cov/trace_cov.svh new file mode 100644 index 000000000..2cbbbe44b --- /dev/null +++ b/examples/prefab/balls/trace/verify/src/cov/trace_cov.svh @@ -0,0 +1,64 @@ +// The counter Ball never uses its bank ports: the common BLink coverage would +// require them to fire, so the test replaces it with this one. +class trace_blink_cov extends bb_blink_cov #(`BB_IN_BW, `BB_OUT_BW); + `uvm_component_utils(trace_blink_cov) + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void check_phase(uvm_phase phase); + if (cmd_fire_count == 0) `uvm_fatal("COV", "cmd handshake never fired") + if (resp_fire_count != cmd_fire_count) + `uvm_fatal("COV", $sformatf("%0d responses for %0d commands", resp_fire_count, cmd_fire_count + )) + for (int i = 0; i < `BB_IN_BW; i++) + if (read_fire_count[i] != 0) `uvm_fatal("COV", $sformatf("read port %0d fired", i)) + for (int i = 0; i < `BB_OUT_BW; i++) + if (write_fire_count[i] != 0) `uvm_fatal("COV", $sformatf("write port %0d fired", i)) + endfunction +endclass + +class trace_cov extends uvm_component; + `uvm_component_utils(trace_cov) + + uvm_analysis_imp_cmd #(bb_blink_cmd_item, trace_cov) cmd_imp; + + int unsigned cur_subcmd; + int unsigned cur_ctr; + bit cur_payload; + + covergroup cmd_cg; + coverpoint cur_subcmd {bins start = {0}; bins stop = {1}; bins read = {2};} + coverpoint cur_ctr {bins ctr[] = {[0 : 15]};} + coverpoint cur_payload {bins zero = {0}; bins nonzero = {1};} + cross cur_subcmd, cur_ctr; + cross cur_subcmd, cur_payload; + endgroup + + int cmd_count; + + function new(string name, uvm_component parent); + super.new(name, parent); + cmd_imp = new("cmd_imp", this); + cmd_cg = new(); + endfunction + + function void write_cmd(bb_blink_cmd_item item); + if (item.funct != BDB_COUNTER_THREAD_FUNCT7[6:0]) + `uvm_fatal("COV", $sformatf("unexpected funct %0d", item.funct)) + cur_subcmd = item.rs2[3:0]; + cur_ctr = item.rs2[7:4]; + cur_payload = |item.rs2[63:8]; + cmd_count++; + cmd_cg.sample(); + endfunction + + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (cmd_count == 0) `uvm_fatal("COV", "no cmd observed") + if (cmd_cg.get_coverage() != 100.0) + `uvm_fatal("COV", $sformatf("trace_cov %0.2f%%", cmd_cg.get_coverage())) + endfunction +endclass diff --git a/examples/prefab/balls/trace/verify/src/env/trace_env.svh b/examples/prefab/balls/trace/verify/src/env/trace_env.svh new file mode 100644 index 000000000..5d414e635 --- /dev/null +++ b/examples/prefab/balls/trace/verify/src/env/trace_env.svh @@ -0,0 +1,26 @@ +class trace_env extends bb_blink_env #(`BB_IN_BW, `BB_OUT_BW); + `uvm_component_utils(trace_env) + + trace_scoreboard scb; + trace_cov icov; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + scb = trace_scoreboard::type_id::create("scb", this); + icov = trace_cov::type_id::create("icov", this); + endfunction + + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + cmd_agent.stim_ap.connect(scb.stim_imp); + cmd_agent.cmd_ap.connect(scb.cmd_imp); + cmd_agent.cmd_ap.connect(icov.cmd_imp); + read_mon.read_ap.connect(scb.read_imp); + write_mon.write_ap.connect(scb.write_imp); + resp_mon.resp_ap.connect(scb.resp_imp); + endfunction +endclass diff --git a/examples/prefab/balls/trace/verify/src/env/trace_scoreboard.svh b/examples/prefab/balls/trace/verify/src/env/trace_scoreboard.svh new file mode 100644 index 000000000..443cbd9ab --- /dev/null +++ b/examples/prefab/balls/trace/verify/src/env/trace_scoreboard.svh @@ -0,0 +1,98 @@ +class trace_scoreboard extends uvm_scoreboard; + `uvm_component_utils(trace_scoreboard) + + uvm_analysis_imp_stim #(bb_blink_cmd_item, trace_scoreboard) stim_imp; + uvm_analysis_imp_cmd #(bb_blink_cmd_item, trace_scoreboard) cmd_imp; + uvm_analysis_imp_read #(bb_blink_read_item, trace_scoreboard) read_imp; + uvm_analysis_imp_write #(bb_blink_write_item, trace_scoreboard) write_imp; + uvm_analysis_imp_resp #(bb_blink_resp_item, trace_scoreboard) resp_imp; + + bb_blink_cmd_item stim_q[$]; + bb_blink_cmd_item issued_q[$]; + int unsigned expected_cmds; + int unsigned cmd_count; + int unsigned resp_count; + + function new(string name, uvm_component parent); + super.new(name, parent); + stim_imp = new("stim_imp", this); + cmd_imp = new("cmd_imp", this); + read_imp = new("read_imp", this); + write_imp = new("write_imp", this); + resp_imp = new("resp_imp", this); + endfunction + + function void write_stim(bb_blink_cmd_item item); + bb_blink_cmd_item want; + // Compare as plain BLink commands; the comparer rejects differing types. + want = bb_blink_cmd_item::type_id::create("want"); + want.copy(item); + stim_q.push_back(want); + endfunction + + function void write_cmd(bb_blink_cmd_item item); + bb_blink_cmd_item want; + if (stim_q.size() == 0) `uvm_fatal("CMD", "command observed before stimulus") + if (issued_q.size() != 0) `uvm_fatal("CMD", "command accepted before the previous response") + want = stim_q.pop_front(); + verification_pkg::evidence::checks++; + if (!item.compare(want)) + `uvm_fatal("CMD", $sformatf("command mismatch: got %s exp %s", item.sprint(), want.sprint())) + issued_q.push_back(want); + cmd_count++; + endfunction + + function void write_read(bb_blink_read_item item); + `uvm_fatal("READ", $sformatf("TraceBall touches no bank, got a read of bank %0d", item.bank_id)) + endfunction + + function void write_write(bb_blink_write_item item); + `uvm_fatal("WRITE", $sformatf("TraceBall touches no bank, got a write of bank %0d", item.bank_id + )) + endfunction + + // The reference model in the DPI library checks each trace record as the + // RTL emits it; the record count is checked once the case has drained. + function void write_resp(bb_blink_resp_item item); + bb_blink_cmd_item want; + int records; + if (issued_q.size() == 0) `uvm_fatal("RESP", "response without a command") + want = issued_q.pop_front(); + if (item.rob_id !== want.rob_id) + `uvm_fatal("RESP", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, want.rob_id)) + resp_count++; + records = trace_case_records(); + verification_pkg::evidence::checks++; + if (records < 0) `uvm_fatal("TRACE", "trace record mismatch, see [TRACE-REF]") + endfunction + + function bit done(); + return expected_cmds != 0 && cmd_count == expected_cmds && resp_count == expected_cmds; + endfunction + + function void reset_counters(int unsigned num_cmds); + stim_q.delete(); + issued_q.delete(); + expected_cmds = num_cmds; + cmd_count = 0; + resp_count = 0; + endfunction + + function void check_records(); + int records; + records = trace_case_records(); + verification_pkg::evidence::checks++; + if (records < 0) `uvm_fatal("TRACE", "trace record mismatch, see [TRACE-REF]") + if (records != expected_cmds) + `uvm_fatal("TRACE", $sformatf("%0d trace records for %0d commands", records, expected_cmds)) + endfunction + + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (!done()) + `uvm_fatal("SCB", $sformatf( + "incomplete: cmds=%0d resp=%0d of %0d", cmd_count, resp_count, expected_cmds)) + check_records(); + endfunction +endclass diff --git a/examples/prefab/balls/trace/verify/src/pkg/trace_pkg.sv b/examples/prefab/balls/trace/verify/src/pkg/trace_pkg.sv new file mode 100644 index 000000000..11e5856e0 --- /dev/null +++ b/examples/prefab/balls/trace/verify/src/pkg/trace_pkg.sv @@ -0,0 +1,13 @@ +package trace_pkg; + import uvm_pkg::*; + import bb_uvm_pkg::*; + `include "uvm_macros.svh" + + `include "common/trace_defs.svh" + `include "common/trace_items.svh" + `include "seq/trace_sequences.svh" + `include "cov/trace_cov.svh" + `include "env/trace_scoreboard.svh" + `include "env/trace_env.svh" + `include "tests/trace_test.svh" +endpackage diff --git a/examples/prefab/balls/trace/verify/src/seq/trace_sequences.svh b/examples/prefab/balls/trace/verify/src/seq/trace_sequences.svh new file mode 100644 index 000000000..43a7187e1 --- /dev/null +++ b/examples/prefab/balls/trace/verify/src/seq/trace_sequences.svh @@ -0,0 +1,19 @@ +class trace_basic_seq extends uvm_sequence #(bb_blink_cmd_item); + `uvm_object_utils(trace_basic_seq) + + int unsigned num_cmds; + + function new(string name = "trace_basic_seq"); + super.new(name); + endfunction + + task body(); + trace_cmd_item item; + for (int unsigned i = 0; i < num_cmds; i++) begin + item = trace_cmd_item::type_id::create("item"); + start_item(item); + item.load_rust_case(i); + finish_item(item); + end + endtask +endclass diff --git a/examples/prefab/balls/trace/verify/src/tb_top.sv b/examples/prefab/balls/trace/verify/src/tb_top.sv new file mode 100644 index 000000000..a747607ab --- /dev/null +++ b/examples/prefab/balls/trace/verify/src/tb_top.sv @@ -0,0 +1,80 @@ +module tb_top; + import uvm_pkg::*; + import trace_pkg::*; + + bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); + + TraceBall dut ( + .clock(intf.clock), + .reset(intf.reset), + .blink_status_running(intf.status_running), + .blink_cmdReq_ready(intf.cmd_req_ready), + .blink_cmdReq_valid(intf.cmd_req_valid), + .blink_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .blink_cmdReq_bits_funct(intf.cmd_req_bits_funct), + .blink_cmdReq_bits_rs2(intf.cmd_req_bits_rs2), + .blink_cmdReq_bits_iter(intf.cmd_req_bits_iter), + .blink_cmdReq_bits_op1_bank(intf.cmd_req_bits_op1_bank), + .blink_cmdReq_bits_op2_bank(intf.cmd_req_bits_op2_bank), + .blink_cmdReq_bits_wr_bank(intf.cmd_req_bits_wr_bank), + .blink_cmdReq_bits_op1_group(intf.cmd_req_bits_op1_group), + .blink_cmdReq_bits_op2_group(intf.cmd_req_bits_op2_group), + .blink_cmdReq_bits_wr_group(intf.cmd_req_bits_wr_group), + .blink_cmdReq_bits_op1_col(intf.cmd_req_bits_op1_col), + .blink_cmdReq_bits_op2_col(intf.cmd_req_bits_op2_col), + .blink_cmdReq_bits_wr_col(intf.cmd_req_bits_wr_col), + .blink_cmdReq_bits_wr_read_bank(intf.cmd_req_bits_wr_read_bank), + .blink_cmdReq_bits_wr_arch_bank(intf.cmd_req_bits_wr_arch_bank), + .blink_cmdResp_ready(intf.cmd_resp_ready), + .blink_cmdResp_valid(intf.cmd_resp_valid), + .blink_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .blink_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .blink_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .blink_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .blink_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .blink_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .blink_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .blink_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .blink_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]) + ); + + // Constant outputs removed from the port list by firtool. + assign intf.bank_read_bank_id[0] = dut.blink_bankRead_0_bank_id; + assign intf.bank_read_group_id[0] = dut.blink_bankRead_0_group_id; + assign intf.bank_read_req_valid[0] = dut.blink_bankRead_0_io_req_valid; + assign intf.bank_read_req_addr[0] = dut.blink_bankRead_0_io_req_bits_addr; + assign intf.bank_read_resp_ready[0] = dut.blink_bankRead_0_io_resp_ready; + assign intf.bank_write_bank_id[0] = dut.blink_bankWrite_0_bank_id; + assign intf.bank_write_group_id[0] = dut.blink_bankWrite_0_group_id; + assign intf.bank_write_req_valid[0] = dut.blink_bankWrite_0_io_req_valid; + assign intf.bank_write_req_addr[0] = dut.blink_bankWrite_0_io_req_bits_addr; + assign intf.bank_write_req_data[0] = dut.blink_bankWrite_0_io_req_bits_data; + assign intf.bank_write_resp_ready[0] = dut.blink_bankWrite_0_io_resp_ready; + assign intf.bank_write_req_mask[0][0] = dut.blink_bankWrite_0_io_req_bits_mask_0; + assign intf.bank_write_req_mask[0][1] = dut.blink_bankWrite_0_io_req_bits_mask_1; + assign intf.bank_write_req_mask[0][2] = dut.blink_bankWrite_0_io_req_bits_mask_2; + assign intf.bank_write_req_mask[0][3] = dut.blink_bankWrite_0_io_req_bits_mask_3; + assign intf.bank_write_req_mask[0][4] = dut.blink_bankWrite_0_io_req_bits_mask_4; + assign intf.bank_write_req_mask[0][5] = dut.blink_bankWrite_0_io_req_bits_mask_5; + assign intf.bank_write_req_mask[0][6] = dut.blink_bankWrite_0_io_req_bits_mask_6; + assign intf.bank_write_req_mask[0][7] = dut.blink_bankWrite_0_io_req_bits_mask_7; + assign intf.bank_write_req_mask[0][8] = dut.blink_bankWrite_0_io_req_bits_mask_8; + assign intf.bank_write_req_mask[0][9] = dut.blink_bankWrite_0_io_req_bits_mask_9; + assign intf.bank_write_req_mask[0][10] = dut.blink_bankWrite_0_io_req_bits_mask_10; + assign intf.bank_write_req_mask[0][11] = dut.blink_bankWrite_0_io_req_bits_mask_11; + assign intf.bank_write_req_mask[0][12] = dut.blink_bankWrite_0_io_req_bits_mask_12; + assign intf.bank_write_req_mask[0][13] = dut.blink_bankWrite_0_io_req_bits_mask_13; + assign intf.bank_write_req_mask[0][14] = dut.blink_bankWrite_0_io_req_bits_mask_14; + assign intf.bank_write_req_mask[0][15] = dut.blink_bankWrite_0_io_req_bits_mask_15; + + initial begin + intf.clock = 1'b0; + intf.reset = 1'b1; + forever #5 intf.clock = ~intf.clock; + end + + initial begin + uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); + run_test("trace_ball_test"); + end +endmodule diff --git a/examples/prefab/balls/trace/verify/src/tests/trace_test.svh b/examples/prefab/balls/trace/verify/src/tests/trace_test.svh new file mode 100644 index 000000000..8f386594e --- /dev/null +++ b/examples/prefab/balls/trace/verify/src/tests/trace_test.svh @@ -0,0 +1,53 @@ +class trace_ball_test extends uvm_test; + `uvm_component_utils(trace_ball_test) + + typedef virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) vif_t; + vif_t vif; + trace_env env; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(vif_t)::get(this, "", "vif", vif)) + `uvm_fatal("NOVIF", "bb_blink_if not found") + bb_blink_cov#(`BB_IN_BW, `BB_OUT_BW)::type_id::set_type_override(trace_blink_cov::get_type()); + env = trace_env::type_id::create("env", this); + endfunction + + task run_phase(uvm_phase phase); + phase.raise_objection(this); + for (int unsigned index = 0; index < TRACE_NUM_CASES; index++) run_case(index); + phase.drop_objection(this); + endtask + + task run_case(int unsigned index); + trace_basic_seq seq; + int cycles = 0; + apply_reset(); + seq = trace_basic_seq::type_id::create("seq"); + seq.num_cmds = trace_case_load(index); + env.scb.reset_counters(seq.num_cmds); + seq.start(env.cmd_agent.seqr); + while (!env.scb.done()) begin + @(posedge vif.clock); + cycles++; + if (cycles > TRACE_TIMEOUT_CYCLES) + `uvm_fatal("TIMEOUT", $sformatf("TraceBall case %0d timeout", index)) + end + // The trace hook runs one cycle after the command; let it drain. + repeat (2) @(posedge vif.clock); + env.scb.check_records(); + `uvm_info("TRACE", $sformatf("TraceBall case %0d passed: %0d commands in %0d cycles", index, + seq.num_cmds, cycles), UVM_LOW) + endtask + + task apply_reset(); + vif.reset = 1'b1; + repeat (5) @(posedge vif.clock); + vif.reset = 1'b0; + repeat (2) @(posedge vif.clock); + endtask +endclass diff --git a/examples/prefab/balls/trace/workloads/ctests/CMakeLists.txt b/examples/prefab/balls/trace/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..970b38aec --- /dev/null +++ b/examples/prefab/balls/trace/workloads/ctests/CMakeLists.txt @@ -0,0 +1,4 @@ +add_buckyball_ctest(bdb_counter_start_stop_test.c) +add_buckyball_ctest(bdb_counter_read_active_test.c) +add_buckyball_ctest(bdb_counter_nested_pair_test.c) +add_buckyball_ctest(bdb_counter_multi_id_test.c) diff --git a/examples/balls/trace/workloads/ctests/bdb_counter_multi_id_test.c b/examples/prefab/balls/trace/workloads/ctests/bdb_counter_multi_id_test.c similarity index 100% rename from examples/balls/trace/workloads/ctests/bdb_counter_multi_id_test.c rename to examples/prefab/balls/trace/workloads/ctests/bdb_counter_multi_id_test.c diff --git a/examples/balls/trace/workloads/ctests/bdb_counter_nested_pair_test.c b/examples/prefab/balls/trace/workloads/ctests/bdb_counter_nested_pair_test.c similarity index 100% rename from examples/balls/trace/workloads/ctests/bdb_counter_nested_pair_test.c rename to examples/prefab/balls/trace/workloads/ctests/bdb_counter_nested_pair_test.c diff --git a/examples/balls/trace/workloads/ctests/bdb_counter_read_active_test.c b/examples/prefab/balls/trace/workloads/ctests/bdb_counter_read_active_test.c similarity index 100% rename from examples/balls/trace/workloads/ctests/bdb_counter_read_active_test.c rename to examples/prefab/balls/trace/workloads/ctests/bdb_counter_read_active_test.c diff --git a/examples/balls/trace/workloads/ctests/bdb_counter_start_stop_test.c b/examples/prefab/balls/trace/workloads/ctests/bdb_counter_start_stop_test.c similarity index 100% rename from examples/balls/trace/workloads/ctests/bdb_counter_start_stop_test.c rename to examples/prefab/balls/trace/workloads/ctests/bdb_counter_start_stop_test.c diff --git a/examples/balls/trace/workloads/isa/bdb_counter.h b/examples/prefab/balls/trace/workloads/isa/bdb_counter.h similarity index 81% rename from examples/balls/trace/workloads/isa/bdb_counter.h rename to examples/prefab/balls/trace/workloads/isa/bdb_counter.h index 0f37555a7..ac6963983 100644 --- a/examples/balls/trace/workloads/isa/bdb_counter.h +++ b/examples/prefab/balls/trace/workloads/isa/bdb_counter.h @@ -4,7 +4,7 @@ #include #include -// subcmd values +// subcmd values; 3..15 are illegal. STOP and READ need a started counter. #define BDB_CTR_START 0 #define BDB_CTR_STOP 1 #define BDB_CTR_READ 2 @@ -17,16 +17,16 @@ // Start counter ctr_id with user tag #define bdb_counter_start(ctr_id, tag) \ BUCKYBALL_INSTRUCTION_R_R(0, BDB_CTR_RS2(BDB_CTR_START, ctr_id, tag), \ - BB_FUNC7(BDB_COUNTER)) + FUNCT7(trace, BDB_COUNTER)) // Stop counter ctr_id, output elapsed to trace #define bdb_counter_stop(ctr_id) \ BUCKYBALL_INSTRUCTION_R_R(0, BDB_CTR_RS2(BDB_CTR_STOP, ctr_id, 0), \ - BB_FUNC7(BDB_COUNTER)) + FUNCT7(trace, BDB_COUNTER)) // Read counter ctr_id current value (non-destructive), output to trace #define bdb_counter_read(ctr_id) \ BUCKYBALL_INSTRUCTION_R_R(0, BDB_CTR_RS2(BDB_CTR_READ, ctr_id, 0), \ - BB_FUNC7(BDB_COUNTER)) + FUNCT7(trace, BDB_COUNTER)) #endif // _BDB_COUNTER_H_ diff --git a/examples/balls/trace/workloads/mlir_tests/CMakeLists.txt b/examples/prefab/balls/trace/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/balls/trace/workloads/mlir_tests/CMakeLists.txt rename to examples/prefab/balls/trace/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/balls/trace/workloads/mlir_tests/ball/CMakeLists.txt b/examples/prefab/balls/trace/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/balls/trace/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/prefab/balls/trace/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/prefab/balls/trace/workloads/mlir_tests/ball/bdb_counter_start_read_stop.mlir b/examples/prefab/balls/trace/workloads/mlir_tests/ball/bdb_counter_start_read_stop.mlir new file mode 100644 index 000000000..a7dbb5b6a --- /dev/null +++ b/examples/prefab/balls/trace/workloads/mlir_tests/ball/bdb_counter_start_read_stop.mlir @@ -0,0 +1,8 @@ +func.func @main() -> i8 { + %zero8 = arith.constant 0 : i8 + %tag = arith.constant 48879 : i64 + trace_ball.counter_start %tag <{counter = 3 : i64}> + trace_ball.counter_read <{counter = 3 : i64}> + trace_ball.counter_stop <{counter = 3 : i64}> + return %zero8 : i8 +} diff --git a/examples/balls/trace/workloads/mlir_tests/ball/bdb_counter_start_read_stop_main.cpp b/examples/prefab/balls/trace/workloads/mlir_tests/ball/bdb_counter_start_read_stop_main.cpp similarity index 100% rename from examples/balls/trace/workloads/mlir_tests/ball/bdb_counter_start_read_stop_main.cpp rename to examples/prefab/balls/trace/workloads/mlir_tests/ball/bdb_counter_start_read_stop_main.cpp diff --git a/examples/prefab/balls/transpose/arch/README.md b/examples/prefab/balls/transpose/arch/README.md new file mode 100644 index 000000000..34a9d6b90 --- /dev/null +++ b/examples/prefab/balls/transpose/arch/README.md @@ -0,0 +1,115 @@ +# Matrix Transpose Accelerator + +## Overview + +This directory implements Buckyball's matrix transpose accelerator for matrix transpose operations. Located at `examples/prefab/balls/transpose/arch`, it serves as a matrix transpose accelerator supporting pipelined transpose operations. + +Core components: +- **Transpose.scala**: Pipelined transposer implementation + +## Code Structure + +``` +transpose/ +└── Transpose.scala - Pipelined transposer +``` + +### Module Responsibilities + +**Transpose.scala** (Transpose implementation layer) +- Implements PipelinedTransposer module +- Manages matrix data read, transpose, and write-back +- Provides Ball domain command interface + +## Module Description + +### Transpose.scala + +**Main functionality**: Implements pipelined matrix transpose operation + +**State machine definition**: +```scala +val idle :: sRead :: sWrite :: complete :: Nil = Enum(4) +val state = RegInit(idle) +``` + +**Storage structure**: +```scala +// Matrix storage register (veclane x veclane) +val regArray = Reg(Vec(b.veclane, Vec(b.veclane, UInt(b.inputType.getWidth.W)))) +``` + +**Counter management**: +```scala +val readCounter = RegInit(0.U(log2Ceil(b.veclane + 1).W)) +val respCounter = RegInit(0.U(log2Ceil(b.veclane + 1).W)) +val writeCounter = RegInit(0.U(log2Ceil(b.veclane + 1).W)) +``` + +**Instruction registers**: +```scala +val robid_reg = RegInit(0.U(10.W)) // ROB ID +val waddr_reg = RegInit(0.U(10.W)) // Write address +val wbank_reg = RegInit(0.U(log2Up(b.sp_banks).W)) // Write bank +val raddr_reg = RegInit(0.U(10.W)) // Read address +val rbank_reg = RegInit(0.U(log2Up(b.sp_banks).W)) // Read bank +val iter_reg = RegInit(0.U(10.W)) // Iteration count +``` + +**Interface definition**: +```scala +val io = IO(new Bundle { + val cmdReq = Flipped(Decoupled(new BallRsIssue)) + val cmdResp = Decoupled(new BallRsComplete) + val sramRead = Vec(b.sp_banks, Flipped(new SramReadIO(...))) + val sramWrite = Vec(b.sp_banks, Flipped(new SramWriteIO(...))) +}) +``` + +**Processing flow**: +1. **idle**: Wait for command, parse transpose parameters +2. **sRead**: Read matrix data row by row into register array +3. **sWrite**: Write transposed data column by column +4. **complete**: Send completion signal + +**Transpose algorithm**: +- Uses veclane×veclane register array to store matrix +- Reads row-wise, writes column-wise to implement transpose +- Supports block-wise transpose for matrices of arbitrary size + +## Usage + +### Implementation Details + +**State machine**: +```scala +val idle :: sRead :: sWrite :: complete :: Nil = Enum(4) +``` +- `idle`: Wait for instruction +- `sRead`: Read matrix data +- `sWrite`: Write transpose result +- `complete`: Complete and respond + +**Register array**: +```scala +val regArray = Reg(Vec(b.veclane, Vec(b.veclane, UInt(b.inputType.getWidth.W)))) +``` +Uses veclane×veclane register array to cache matrix data. + +**Transpose operation**: +- Read phase: Read data row by row into `regArray(row)(col)` +- Write phase: Read `regArray(i)(col)` column by column to form new rows for writing + +### Configuration Parameters + +**Matrix size**: Determined by b.veclane parameter +**Data width**: Determined by b.inputType.getWidth +**Bank configuration**: Supports multi-bank SRAM access + +### Notes + +1. **Matrix size limitation**: Maximum support for veclane×veclane matrices +2. **Memory bandwidth**: Transpose operation has high memory bandwidth requirements +3. **Register overhead**: Requires veclane² registers to store matrix +4. **Address calculation**: Transposed address calculation needs to be handled correctly +5. **Pipeline control**: Read/write counters need to be synchronized correctly diff --git a/examples/prefab/balls/transpose/arch/Transpose.scala b/examples/prefab/balls/transpose/arch/Transpose.scala new file mode 100644 index 000000000..a9d572018 --- /dev/null +++ b/examples/prefab/balls/transpose/arch/Transpose.scala @@ -0,0 +1,278 @@ +package examples.balls.transpose + +import framework.acc.balldomain.configs.BallRouteParam + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import framework.acc.balldomain.ball.blink.{BallCommand, BallResponse} +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} + +import framework.acc.balldomain.ball.blink.BallStatus +import framework.acc.memdomain.{BankRead, BankWrite} +import examples.balls.transpose.configs.TransposeBallParam + +@instantiable +class Transpose( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val routingParam: BallDomainParam, + val instance: BallRouteParam, + val ballParam: TransposeBallParam) + extends Module { + val commandDataBits = frontendParam.commandDataBits + val ballConfig = ballParam + val bankWidth = memParam.bankWidth + val rowBytes = bankWidth / 8 + + val ballMapping = instance + + val inBW = ballMapping.inBW + val outBW = ballMapping.outBW + require(inBW == outBW, "TransposeBall requires inBW == outBW") + require(inBW == 1, "TransposeBall gather/scatter path requires inBW == 1") + require(bankWidth % 8 == 0, "bankWidth must be byte-aligned") + require( + ballConfig.InputNum * ballConfig.inputWidth == bankWidth, + "TransposeBall InputNum*inputWidth must equal bankWidth" + ) + + @public + val io = IO(new Bundle { + + val cmdReq = Flipped(Decoupled(new BallCommand(frontendParam, memParam))) + + val cmdResp = Decoupled(new BallResponse(frontendParam)) + + val bankRead = Vec( + inBW, + Flipped(new BankRead( + frontendParam, + memParam + )) + ) + + val bankWrite = Vec( + outBW, + Flipped(new BankWrite( + frontendParam, + memParam + )) + ) + + val status = new BallStatus + }) + + val rob_id_reg = RegInit(0.U(log2Up(frontendParam.rob_entries).W)) + + val rbank_reg = RegInit(0.U(frontendParam.bank_id_len.W)) + val wbank_reg = RegInit(0.U(frontendParam.bank_id_len.W)) + val ncol_reg = RegInit(0.U(log2Up(memParam.bankNum + 1).W)) + val iter_reg = RegInit(0.U(frontendParam.iter_len.W)) + val elem_reg = RegInit(0.U(8.W)) + + val idle :: sRead :: sWrite :: complete :: Nil = Enum(4) + val state = RegInit(idle) + + // Walk destination dense index 0 .. iter*W-1, filling write beats. + val pending = RegInit(false.B) + val wrBytes = Reg(Vec(rowBytes, UInt(8.W))) + val wrMask = RegInit(VecInit(Seq.fill(memParam.bankMaskLen)(0.U(1.W)))) + val wrGroup = RegInit(0.U(log2Up(memParam.bankNum + 1).W)) + val wrAddr = RegInit(0.U(frontendParam.iter_len.W)) + val wrValid = RegInit(false.B) + val beatFill = RegInit(0.U(8.W)) + + val cached = RegInit(false.B) + val cacheAddr = RegInit(0.U(frontendParam.iter_len.W)) + val cacheGroup = RegInit(0.U(log2Up(memParam.bankNum + 1).W)) + val bases = Reg(Vec(3, UInt(memParam.groupIdWidth.W))) + val cacheData = RegInit(0.U(bankWidth.W)) + + val srcR = RegInit(0.U(frontendParam.iter_len.W)) + val srcC = RegInit(0.U(32.W)) + val dstVirtRow = RegInit(0.U(frontendParam.iter_len.W)) + val dstVirtCol = RegInit(0.U(32.W)) + val epgReg = RegInit(0.U(8.W)) + val wElemsReg = RegInit(0.U(32.W)) + val elemBytesReg = RegInit(0.U(8.W)) + val lastBeat = RegInit(false.B) + + for (i <- 0 until inBW) { + io.bankRead(i).rob_id := rob_id_reg + io.bankRead(i).bank_id := rbank_reg + io.bankRead(i).group_id := bases(0) + io.bankRead(i).io.req.valid := false.B + io.bankRead(i).io.req.bits.addr := 0.U + io.bankRead(i).io.resp.ready := false.B + } + for (i <- 0 until outBW) { + io.bankWrite(i).rob_id := rob_id_reg + io.bankWrite(i).bank_id := wbank_reg + io.bankWrite(i).group_id := bases(2) + io.bankWrite(i).io.req.valid := false.B + io.bankWrite(i).io.req.bits.addr := 0.U + io.bankWrite(i).io.req.bits.data := 0.U + io.bankWrite(i).io.req.bits.mask := VecInit( + Seq.fill(memParam.bankMaskLen)(0.U(1.W)) + ) + io.bankWrite(i).io.resp.ready := (state =/= idle) + } + + io.cmdReq.ready := (state === idle) + io.cmdResp.valid := false.B + io.cmdResp.bits.rob_id := rob_id_reg + + require(isPow2(rowBytes), "Transpose rowBytes must be power of 2") + + val laneBitsI8 = log2Ceil(rowBytes) + val laneBitsI32 = log2Ceil(rowBytes / 4) + + val isI8 = elemBytesReg === 1.U + val srcLane = Mux(isI8, srcC(laneBitsI8 - 1, 0), srcC(laneBitsI32 - 1, 0)) + val srcGroup = Mux(isI8, srcC >> laneBitsI8.U, srcC >> laneBitsI32.U) + .asTypeOf(UInt(log2Up(memParam.bankNum + 1).W)) + val srcAddr = srcR + + val dstGroup = Mux(isI8, dstVirtCol >> laneBitsI8.U, dstVirtCol >> laneBitsI32.U) + .asTypeOf(UInt(log2Up(memParam.bankNum + 1).W)) + val bytes = cacheData.asTypeOf(Vec(rowBytes, UInt(8.W))) + val words = cacheData.asTypeOf(Vec(rowBytes / 4, UInt(32.W))) + + val needRead = !cached || cacheAddr =/= srcAddr || cacheGroup =/= srcGroup + + switch(state) { + is(idle) { + when(io.cmdReq.fire) { + val cmd = io.cmdReq.bits + rob_id_reg := io.cmdReq.bits.rob_id + rbank_reg := cmd.op1_bank + wbank_reg := cmd.wr_bank + bases := VecInit(cmd.op1_group, cmd.op2_group, cmd.wr_group) + ncol_reg := cmd.op1_col + iter_reg := cmd.iter + elem_reg := cmd.rs2(7, 0) + pending := false.B + wrValid := false.B + beatFill := 0.U + cached := false.B + srcR := 0.U + srcC := 0.U + dstVirtRow := 0.U + dstVirtCol := 0.U + lastBeat := false.B + val i8 = cmd.rs2(7, 0) === 8.U + elemBytesReg := Mux(i8, 1.U, 4.U) + epgReg := Mux(i8, rowBytes.U, (rowBytes / 4).U) + wElemsReg := Mux(i8, cmd.op1_col << laneBitsI8.U, cmd.op1_col << laneBitsI32.U) + assert(cmd.iter > 0.U, "Transpose iter must be > 0") + assert(cmd.op1_bank =/= cmd.wr_bank, "Transpose op1 and wr must differ") + assert( + cmd.op1_col === cmd.wr_col && cmd.op1_col =/= 0.U, + "Transpose cols mismatch" + ) + assert(cmd.rs2(63, 8) === 0.U, "Transpose rs2[63:8] must be 0") + assert( + cmd.rs2(7, 0) === 8.U || cmd.rs2(7, 0) === 32.U, + "Transpose elem_bits" + ) + assert( + bankWidth.U % cmd.rs2(7, 0) === 0.U, + "Transpose bankWidth not divisible by elem_bits" + ) + state := sRead + } + } + + is(sRead) { + io.bankRead(0).group_id := bases(0) + srcGroup + io.bankRead(0).io.resp.ready := pending + io.bankRead(0).io.req.valid := needRead && !pending && !wrValid + io.bankRead(0).io.req.bits.addr := srcAddr + + when(io.bankRead(0).io.req.fire) { + pending := true.B + } + when(io.bankRead(0).io.resp.fire) { + cacheData := io.bankRead(0).io.resp.bits.data + cacheAddr := srcAddr + cacheGroup := srcGroup + cached := true.B + pending := false.B + } + + when(cached && !needRead && !wrValid) { + when(beatFill === 0.U) { + wrGroup := dstGroup + wrAddr := dstVirtRow + wrBytes := VecInit(Seq.fill(rowBytes)(0.U(8.W))) + } + when(isI8) { + wrBytes(beatFill) := bytes(srcLane) + }.otherwise { + val w = words(srcLane) + wrBytes(beatFill << 2) := w(7, 0) + wrBytes((beatFill << 2) + 1.U) := w(15, 8) + wrBytes((beatFill << 2) + 2.U) := w(23, 16) + wrBytes((beatFill << 2) + 3.U) := w(31, 24) + } + beatFill := beatFill + 1.U + + val lastElem = (srcR === iter_reg - 1.U) && (srcC === wElemsReg - 1.U) + when(srcR === iter_reg - 1.U) { + srcR := 0.U + srcC := srcC + 1.U + }.otherwise { + srcR := srcR + 1.U + } + when(dstVirtCol === wElemsReg - 1.U) { + dstVirtCol := 0.U + dstVirtRow := dstVirtRow + 1.U + }.otherwise { + dstVirtCol := dstVirtCol + 1.U + } + + val beatDone = (beatFill + 1.U === epgReg) || lastElem + when(beatDone) { + wrMask.foreach(_ := 1.U) + wrValid := true.B + lastBeat := lastElem + state := sWrite + } + } + } + + is(sWrite) { + io.bankWrite(0).group_id := bases(2) + wrGroup + io.bankWrite(0).io.req.valid := wrValid + io.bankWrite(0).io.req.bits.addr := wrAddr + io.bankWrite(0).io.req.bits.data := wrBytes.asUInt + io.bankWrite(0).io.req.bits.mask := wrMask + io.bankWrite(0).io.resp.ready := true.B + when(io.bankWrite(0).io.req.fire) { + wrValid := false.B + beatFill := 0.U + } + when(io.bankWrite(0).io.resp.fire) { + when(lastBeat) { + state := complete + }.otherwise { + state := sRead + } + } + } + + is(complete) { + io.cmdResp.valid := true.B + when(io.cmdResp.fire) { + state := idle + } + } + } + + io.status.idle := (state === idle) + io.status.running := (state =/= idle) +} diff --git a/examples/prefab/balls/transpose/arch/TransposeBall.scala b/examples/prefab/balls/transpose/arch/TransposeBall.scala new file mode 100644 index 000000000..c03f21dc9 --- /dev/null +++ b/examples/prefab/balls/transpose/arch/TransposeBall.scala @@ -0,0 +1,67 @@ +package examples.balls.transpose + +import framework.acc.balldomain.bbus.{BallFactory, BallPorts} +import buckyball.balls.transpose.{Params => ParametersProto} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import examples.balls.transpose.configs.TransposeBallParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.balldomain.ball.blink.{BlinkIO, HasBlink} +import examples.balls.transpose.Transpose + +@instantiable +class TransposeBall( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Module + with HasBlink { + val ball = bp.balls.find(_.id == id).get + + val ballParam = TransposeBallParam.fromProto(ball.parameters.unpack(classOf[ParametersProto])) + + val ballCommonConfig = ball + + val inBW = ballCommonConfig.inBW + val outBW = ballCommonConfig.outBW + + @public + val blink = IO(new BlinkIO(fp, mp, bp, id)) + + val transposeUnit: Instance[Transpose] = + Instantiate(new Transpose(fp, mp, bp, ball, ballParam)) + + transposeUnit.io.cmdReq <> blink.cmdReq + transposeUnit.io.cmdResp <> blink.cmdResp + + for (i <- 0 until inBW) { + transposeUnit.io.bankRead(i) <> blink.bankRead(i) + } + + for (i <- 0 until outBW) { + transposeUnit.io.bankWrite(i) <> blink.bankWrite(i) + } + + blink.status <> transposeUnit.io.status + + // MMIO: this Ball does not consume MMIO metadata +} + +object TransposeBall extends BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts = + BallPorts(Instantiate(new TransposeBall(fp, mp, bp, id)).blink, None) + +} diff --git a/examples/prefab/balls/transpose/configs/TransposeBallParam.scala b/examples/prefab/balls/transpose/configs/TransposeBallParam.scala new file mode 100644 index 000000000..a381e99d7 --- /dev/null +++ b/examples/prefab/balls/transpose/configs/TransposeBallParam.scala @@ -0,0 +1,20 @@ +package examples.balls.transpose.configs + +import buckyball.balls.transpose.{Params => ParametersProto} +import upickle.default.{macroRW, ReadWriter} + +case class TransposeBallParam( + InputNum: Int, + inputWidth: Int) + +object TransposeBallParam { + + def fromProto(source: ParametersProto): TransposeBallParam = { + val parameters = source + require(parameters.hasInputNum && parameters.hasInputWidth) + val param = TransposeBallParam(parameters.getInputNum, parameters.getInputWidth) + param + } + + implicit val rw: ReadWriter[TransposeBallParam] = macroRW +} diff --git a/examples/prefab/balls/transpose/configs/default.toml b/examples/prefab/balls/transpose/configs/default.toml new file mode 100644 index 000000000..6d4c32c7d --- /dev/null +++ b/examples/prefab/balls/transpose/configs/default.toml @@ -0,0 +1,4 @@ +# Transpose Ball Configuration + +InputNum = 16 +inputWidth = 8 diff --git a/examples/prefab/balls/transpose/configs/transpose_params.proto b/examples/prefab/balls/transpose/configs/transpose_params.proto new file mode 100644 index 000000000..423ba834a --- /dev/null +++ b/examples/prefab/balls/transpose/configs/transpose_params.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; +package buckyball.balls.transpose; +option java_package = "buckyball.balls.transpose"; +option java_multiple_files = true; + +message Params { + optional int32 InputNum = 1; + optional int32 inputWidth = 2; +} diff --git a/examples/prefab/balls/transpose/emu/src/dispatch.rs b/examples/prefab/balls/transpose/emu/src/dispatch.rs new file mode 100644 index 000000000..37dd8002f --- /dev/null +++ b/examples/prefab/balls/transpose/emu/src/dispatch.rs @@ -0,0 +1,31 @@ +use bebop_bemu::ball::ExecContext; + +use super::transpose; + +const BALL_CLASS: &str = "examples.balls.transpose.TransposeBall"; + +pub fn execute_known( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, + ctx: &mut ExecContext, +) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("TRANSPOSE") => Some(transpose::exec_transpose(xs1, xs2, ctx)), + Some(_) | None => None, + } +} + +pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("TRANSPOSE") => Some(transpose::latency(xs1, xs2)), + Some(_) | None => None, + } +} diff --git a/examples/prefab/balls/transpose/emu/src/lib.rs b/examples/prefab/balls/transpose/emu/src/lib.rs new file mode 100644 index 000000000..e268be558 --- /dev/null +++ b/examples/prefab/balls/transpose/emu/src/lib.rs @@ -0,0 +1,11 @@ +pub(crate) use bebop_bemu::{thread::instructions::decode, ball as instruction}; + +mod transpose; + +mod dispatch; + +pub use dispatch::{cycles_after_issue, execute_known}; + +#[bebop_bemu::ball::linkme::distributed_slice(bebop_bemu::BALLS)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static BALL: bebop_bemu::Ball = bebop_bemu::Ball { execute: execute_known, cycles: cycles_after_issue }; diff --git a/examples/balls/transpose/emu/src/transpose.rs b/examples/prefab/balls/transpose/emu/src/transpose.rs similarity index 92% rename from examples/balls/transpose/emu/src/transpose.rs rename to examples/prefab/balls/transpose/emu/src/transpose.rs index cb948cb28..d088b3284 100644 --- a/examples/balls/transpose/emu/src/transpose.rs +++ b/examples/prefab/balls/transpose/emu/src/transpose.rs @@ -1,8 +1,9 @@ -use super::super::bank::{bank_row_bytes, bank_size, bank_width}; -use super::decode::{pbank_group, rs1_b0, rs1_b2, rs1_iter}; +use bebop_bemu::thread::banks::{bank_row_bytes, bank_size, bank_width}; +use super::decode::{pbank_group, rs1_b0, rs1_b2, rs1_iter, rs1_read_old}; use super::instruction::ExecContext; pub(crate) fn exec_transpose(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + assert!(!rs1_read_old(xs1), "transpose: reads no old destination; rs1[29] must be clear"); let op1 = rs1_b0(xs1); let wr = rs1_b2(xs1); let iter = rs1_iter(xs1) as usize; diff --git a/examples/prefab/balls/transpose/stack/src/transposeOps.cpp b/examples/prefab/balls/transpose/stack/src/transposeOps.cpp new file mode 100644 index 000000000..3cf6f0d9a --- /dev/null +++ b/examples/prefab/balls/transpose/stack/src/transposeOps.cpp @@ -0,0 +1,73 @@ +//===- transposeOps.cpp - TransposeBall operations ------------------------===// +// +// The dialect of TransposeBall, its virtual operations building their physical +// ones and the instructions of the physical operations. +// +//===----------------------------------------------------------------------===// + +#include "transposeOps.h" + +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::transpose_ball; +using mlir::buddy::bankFields; +using mlir::buddy::constantI64; +using mlir::buddy::emitInstruction; +using mlir::buddy::field; + +#include "transposeDialect.cpp.inc" + +#define GET_OP_CLASSES +#include "transposeOps.cpp.inc" + +void TransposeBallDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "transposeOps.cpp.inc" + >(); +} + +namespace { +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallDialect([](DialectRegistry ®istry) { + registry.insert(); + }), + true); +} // namespace + +SmallVector BankTransposeI8Op::getSourceBanks() { + return {getInput()}; +} +Value BankTransposeI8Op::getDestinationBank() { return getOutput(); } +bool BankTransposeI8Op::readsDestination() { return false; } +Operation *BankTransposeI8Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return TransposeI8Op::create(builder, getLoc(), sources[0], destination, + getRows(), getCols()); +} + +SmallVector TransposeI8Op::getSourceBanks() { + return {int64_t(getInput())}; +} +int64_t TransposeI8Op::getDestinationBank() { return getOutput(); } +LogicalResult TransposeI8Op::lowerToInstructions(OpBuilder &builder) { + Location loc = getLoc(); + int64_t input = getInput(), output = getOutput(), rows = getRows(), + cols = getCols(); + if (input == output) + return emitOpError("transpose_i8 requires distinct input and output banks"); + const auto &target = buckyball_target::getBuckyballTarget(); + int64_t lanes = target.bankWidthBits / 8; + if (lanes <= 0 || target.bankWidthBits % 8 || rows <= 0 || + rows >= (1LL << 33) || rows > target.bankDepth || cols <= 0 || + cols % lanes || cols / lanes > 31) + return emitOpError("transpose_i8 shape exceeds the bank or ISA geometry"); + emitInstruction( + builder, loc, buckyball_target::getBuckyballFunct7("TRANSPOSE"), + constantI64(builder, loc, bankFields({input}, output, false, rows)), + constantI64(builder, loc, 8)); + return success(); +} diff --git a/examples/prefab/balls/transpose/stack/src/transposeOps.h b/examples/prefab/balls/transpose/stack/src/transposeOps.h new file mode 100644 index 000000000..36ffea777 --- /dev/null +++ b/examples/prefab/balls/transpose/stack/src/transposeOps.h @@ -0,0 +1,20 @@ +//===- transposeOps.h - TransposeBall operations --------------------------===// + +#ifndef TRANSPOSE_OPS_H +#define TRANSPOSE_OPS_H + +#include "Interfaces/BankInterfaces.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "transposeDialect.h.inc" + +#define GET_OP_CLASSES +#include "transposeOps.h.inc" + +#endif // TRANSPOSE_OPS_H diff --git a/examples/prefab/balls/transpose/stack/src/transposeOps.td b/examples/prefab/balls/transpose/stack/src/transposeOps.td new file mode 100644 index 000000000..f18341b58 --- /dev/null +++ b/examples/prefab/balls/transpose/stack/src/transposeOps.td @@ -0,0 +1,42 @@ +//===- transposeOps.td - TransposeBall operations ----------*- tablegen -*-===// +// +// transpose_i8: both banks have cols / (bankWidthBits / 8) columns. The +// instruction packs the transposed bytes into the same bank geometry, with +// `rows` bank rows. +// +//===----------------------------------------------------------------------===// + +#ifndef TRANSPOSE_OPS +#define TRANSPOSE_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Interfaces/BankInterfaces.td" + +def TransposeBall_Dialect : Dialect { + let name = "transpose_ball"; + let cppNamespace = "::buddy::transpose_ball"; + let summary = "The operations of TransposeBall"; +} + +class TransposeBall_Op traits = []> + : Op])>; + +def BankTransposeI8Op : TransposeBall_Op<"bank_transpose_i8", + [DeclareOpInterfaceMethods]> { + let summary = "transpose_i8 on virtual banks"; + let arguments = (ins I64:$input, I64:$output, I64Attr:$rows, I64Attr:$cols); + let results = (outs I64:$outputOut); + let assemblyFormat = "$input `->` $output prop-dict attr-dict"; +} + +def TransposeI8Op : TransposeBall_Op<"transpose_i8", + [DeclareOpInterfaceMethods]> { + let summary = "transpose_i8 on physical banks"; + let arguments = (ins I64Attr:$input, I64Attr:$output, I64Attr:$rows, + I64Attr:$cols); + let assemblyFormat = "prop-dict attr-dict"; +} + +#endif // TRANSPOSE_OPS diff --git a/examples/prefab/balls/transpose/verify/README.md b/examples/prefab/balls/transpose/verify/README.md new file mode 100644 index 000000000..be2ea3661 --- /dev/null +++ b/examples/prefab/balls/transpose/verify/README.md @@ -0,0 +1,78 @@ +# TransposeBall UVM Verification + +Verifies the generated `TransposeBall` module directly using the shared Blink UVM framework. + +## Structure + +- `../../../../verify/uvm/src/ball/bb_uvm_pkg.sv`: common Blink UVM transaction items and base env +- `src/common/transpose_defs.svh`: DPI imports and constants +- `src/common/transpose_items.svh`: `transpose_cmd_item` with `load_rust_case` +- `src/seq/transpose_sequences.svh`: `transpose_basic_seq` driving one case per run +- `src/cov/transpose_cov.svh`: `elem_bits` {8,32} and `iter` {1,2,4,8,16} coverage +- `src/env/transpose_scoreboard.svh`: preloads mem model, compares writes vs DPI dst words +- `src/env/transpose_env.svh`: extends `bb_blink_env#(1,1)` with scoreboard and cov +- `src/tests/transpose_test.svh`: directed cases 0,1 then random 2..21 with fixed seed +- `src/pkg/transpose_pkg.sv`: package include entry +- `src/tb_top.sv`: instantiates `bb_blink_if#(1,1)` and `TransposeBall` +- `filelists/transpose_ball.f`: VCS/Verilator filelist + +## Test plan + +`+UVM_TESTNAME=transpose_ball_test` runs: + +- case 0: directed i8, iter=16 +- case 1: directed i32, iter=8 +- case 2: directed i8 16x32, rows span two groups (`transpose_i8_16x32_test.c`) +- cases 3..22: random with fixed seed `0xCAFE_BABE`, elem_bits in {8,32}, iter in {1,2,4,8,16}, + col in {1,2}, op1/wr group base in 0..3 + +Each case: reset, preload src words into `mem_model`, drive one command, wait for +`scb.done()` or timeout at 8000 cycles. Scoreboard compares observed writes +(data/addr/group/mask/bank/rob) with DPI `transpose_case_dst_word_*`. `transpose_cov` +requires both `elem_bits` bins, both `col` bins and all `iter` bins hit across the whole test, else +`check_phase` fatal. + +## Build + +Enter the shared UVM/VCS environment: + +```console +nix develop ../../../../verify +``` + +Build the DPI reference library: + +```console +cargo build --manifest-path casegen/Cargo.toml +``` + +Compile from this directory: + +```console +vcs -full64 -sverilog -timescale=1ns/1ps \ + $VCS_UVM_ARGS \ + -sv_lib ../output/casegen/debug/libtranspose_casegen \ + -f filelists/transpose_ball.f +``` + +Run: + +```console +./simv +UVM_TESTNAME=transpose_ball_test +``` + +Or via bbdev: + +```console +bbdev uvm --run '--design pebble --ball transpose' +``` + + +## Acceptance (2026-08-03) + +- [x] Layout matches Blink common + ball-local split; filelist uses `@UVM@` / `@RTL@` +- [x] casegen reuses transpose model; whole-case DPI API +- [x] directed i8/i32 scoreboard pass +- [x] deterministic random idx 3..22 pass +- [x] protocol + semantic cover enforced (`check_phase` fatal) +- [x] pebble: 23/23 on the BLink command interface (2026-10-26) diff --git a/examples/balls/transpose/verify/casegen/Cargo.lock b/examples/prefab/balls/transpose/verify/casegen/Cargo.lock similarity index 100% rename from examples/balls/transpose/verify/casegen/Cargo.lock rename to examples/prefab/balls/transpose/verify/casegen/Cargo.lock diff --git a/examples/balls/transpose/verify/casegen/Cargo.toml b/examples/prefab/balls/transpose/verify/casegen/Cargo.toml similarity index 100% rename from examples/balls/transpose/verify/casegen/Cargo.toml rename to examples/prefab/balls/transpose/verify/casegen/Cargo.toml diff --git a/examples/prefab/balls/transpose/verify/casegen/src/casegen.rs b/examples/prefab/balls/transpose/verify/casegen/src/casegen.rs new file mode 100644 index 000000000..01ea31e0c --- /dev/null +++ b/examples/prefab/balls/transpose/verify/casegen/src/casegen.rs @@ -0,0 +1,224 @@ +use crate::model; + +pub const BANK_ROW_BYTES: usize = 16; +pub const MAX_ITER: usize = 16; +pub const MAX_COLS: usize = 2; +pub const MAX_WORDS: usize = MAX_ITER * MAX_COLS; + +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct TransposeCmd { + pub iter: u32, + pub op1_bank: u32, + pub wr_bank: u32, + pub op1_group: u32, + pub wr_group: u32, + pub op1_col: u32, + pub wr_col: u32, + pub rob_id: u32, + pub elem_bits: u32, + pub num_src_words: u32, + pub num_dst_words: u32, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TransposeCase { + pub cmd: TransposeCmd, + pub src: Vec, + pub dst: Vec, +} + +impl TransposeCase { + pub fn src_word_lo(&self, word_index: usize) -> u64 { + word_lo(&self.src, word_index) + } + pub fn src_word_hi(&self, word_index: usize) -> u64 { + word_hi(&self.src, word_index) + } + pub fn dst_word_lo(&self, word_index: usize) -> u64 { + word_lo(&self.dst, word_index) + } + pub fn dst_word_hi(&self, word_index: usize) -> u64 { + word_hi(&self.dst, word_index) + } +} + +fn word_lo(data: &[u8], word_index: usize) -> u64 { + let off = word_index * BANK_ROW_BYTES; + let mut buf = [0u8; 8]; + buf.copy_from_slice(&data[off..off + 8]); + u64::from_le_bytes(buf) +} + +fn word_hi(data: &[u8], word_index: usize) -> u64 { + let off = word_index * BANK_ROW_BYTES + 8; + let mut buf = [0u8; 8]; + buf.copy_from_slice(&data[off..off + 8]); + u64::from_le_bytes(buf) +} + +// Word `i` of a case is bank row `i / cols`, group `i % cols`: each row spans `cols` groups. +pub fn gen_case(seed: u32, index: u32) -> TransposeCase { + match index { + 0 => directed_i8(), + 1 => directed_i32(), + 2 => directed_i8_two_groups(), + _ => random_case(seed, index), + } +} + +fn directed_i8() -> TransposeCase { + let iter = 16usize; + let elem_bytes = 1usize; + let cols = 1usize; + let w = cols * (BANK_ROW_BYTES / elem_bytes); + let total = iter * w; + let src: Vec = (0..total).map(|i| i as u8).collect(); + let dst = model::transpose_bytes(&src, iter, w, elem_bytes); + let nwords = model::num_words(total * elem_bytes) as u32; + TransposeCase { + cmd: TransposeCmd { + iter: iter as u32, + op1_bank: 0, + wr_bank: 1, + op1_group: 0, + wr_group: 0, + op1_col: cols as u32, + wr_col: cols as u32, + rob_id: 3, + elem_bits: (elem_bytes * 8) as u32, + num_src_words: nwords, + num_dst_words: nwords, + }, + src, + dst, + } +} + +fn directed_i32() -> TransposeCase { + let iter = 8usize; + let elem_bytes = 4usize; + let cols = 1usize; + let w = cols * (BANK_ROW_BYTES / elem_bytes); + let total = iter * w; + let src: Vec = (0..total) + .map(|i| i as u32) + .flat_map(|v| v.to_le_bytes()) + .collect(); + let dst = model::transpose_bytes(&src, iter, w, elem_bytes); + let nwords = model::num_words(total * elem_bytes) as u32; + TransposeCase { + cmd: TransposeCmd { + iter: iter as u32, + op1_bank: 0, + wr_bank: 1, + op1_group: 0, + wr_group: 0, + op1_col: cols as u32, + wr_col: cols as u32, + rob_id: 5, + elem_bits: (elem_bytes * 8) as u32, + num_src_words: nwords, + num_dst_words: nwords, + }, + src, + dst, + } +} + +// Mirrors transpose_i8_16x32_test.c: a 16x32 i8 matrix whose rows span two groups. +fn directed_i8_two_groups() -> TransposeCase { + let iter = 16usize; + let elem_bytes = 1usize; + let cols = 2usize; + let w = cols * (BANK_ROW_BYTES / elem_bytes); + let total = iter * w; + let src: Vec = (0..iter) + .flat_map(|r| (0..w).map(move |c| (r + c + 1) as u8)) + .collect(); + let dst = model::transpose_bytes(&src, iter, w, elem_bytes); + let nwords = model::num_words(total * elem_bytes) as u32; + TransposeCase { + cmd: TransposeCmd { + iter: iter as u32, + op1_bank: 0, + wr_bank: 1, + op1_group: 2, + wr_group: 1, + op1_col: cols as u32, + wr_col: cols as u32, + rob_id: 6, + elem_bits: (elem_bytes * 8) as u32, + num_src_words: nwords, + num_dst_words: nwords, + }, + src, + dst, + } +} + +fn random_case(seed: u32, index: u32) -> TransposeCase { + let mut rng = Rng::new(seed, index); + let elem_bits = if rng.next() & 1 == 0 { 8u32 } else { 32u32 }; + let elem_bytes = (elem_bits / 8) as usize; + let iter_pool = [1u32, 2, 4, 8, 16]; + let iter = iter_pool[(rng.next() as usize) % iter_pool.len()] as usize; + let cols = (rng.next() as usize) % MAX_COLS + 1; + let w = cols * (BANK_ROW_BYTES / elem_bytes); + + // Keep banks in 0..7 so the same cases work on configs with bankNum=8 or 16. + let op1_bank = rng.range(0, 7); + let mut wr_bank = rng.range(0, 7); + if wr_bank == op1_bank { + wr_bank = (wr_bank + 1) & 7; + } + + let op1_group = rng.range(0, 3); + let wr_group = rng.range(0, 3); + + let total = iter * w; + let mut src = vec![0u8; total * elem_bytes]; + for b in src.iter_mut() { + *b = (rng.next() & 0xFF) as u8; + } + let dst = model::transpose_bytes(&src, iter, w, elem_bytes); + let nwords = model::num_words(total * elem_bytes) as u32; + + TransposeCase { + cmd: TransposeCmd { + iter: iter as u32, + op1_bank, + wr_bank, + op1_group, + wr_group, + op1_col: cols as u32, + wr_col: cols as u32, + rob_id: rng.range(0, 15), + elem_bits, + num_src_words: nwords, + num_dst_words: nwords, + }, + src, + dst, + } +} + +struct Rng { + state: u64, +} + +impl Rng { + fn new(seed: u32, index: u32) -> Self { + let state = (u64::from(seed) << 32) ^ u64::from(index) ^ 0x9E37_79B9_7F4A_7C15; + Self { state } + } + fn next(&mut self) -> u32 { + self.state ^= self.state >> 12; + self.state ^= self.state << 25; + self.state ^= self.state >> 27; + ((self.state.wrapping_mul(0x2545_F491_4F6C_DD1D)) >> 32) as u32 + } + fn range(&mut self, lo: u32, hi: u32) -> u32 { + lo + (self.next() % (hi - lo + 1)) + } +} diff --git a/examples/prefab/balls/transpose/verify/casegen/src/lib.rs b/examples/prefab/balls/transpose/verify/casegen/src/lib.rs new file mode 100644 index 000000000..d80b5fa2e --- /dev/null +++ b/examples/prefab/balls/transpose/verify/casegen/src/lib.rs @@ -0,0 +1,71 @@ +mod casegen; +mod model; + +use std::cell::RefCell; + +use casegen::{TransposeCase, TransposeCmd, MAX_WORDS}; + +thread_local! { + static CURRENT: RefCell> = RefCell::new(None); +} + +#[no_mangle] +pub extern "C" fn transpose_case_load(seed: u32, index: u32) -> i32 { + let case = casegen::gen_case(seed, index); + let nsrc = case.cmd.num_src_words as usize; + let ndst = case.cmd.num_dst_words as usize; + if nsrc > MAX_WORDS || ndst > MAX_WORDS { + panic!("transpose_case_load: word count out of range src={nsrc} dst={ndst}"); + } + CURRENT.with(|c| *c.borrow_mut() = Some(case)); + 0 +} + +fn current_case R, R>(f: F) -> R { + CURRENT.with(|c| match c.borrow().as_ref() { + Some(case) => f(case), + None => panic!("transpose_case: no case loaded; call transpose_case_load first"), + }) +} + +#[no_mangle] +pub extern "C" fn transpose_case_cmd(out_ptr: *mut TransposeCmd) { + current_case(|case| unsafe { + if out_ptr.is_null() { + panic!("transpose_case_cmd: null out_ptr"); + } + *out_ptr = case.cmd; + }); +} + +#[no_mangle] +pub extern "C" fn transpose_case_src_word_lo(word_index: u32) -> u64 { + if word_index as usize >= MAX_WORDS { + panic!("transpose_case_src_word_lo: word_index out of range"); + } + current_case(|case| case.src_word_lo(word_index as usize)) +} + +#[no_mangle] +pub extern "C" fn transpose_case_src_word_hi(word_index: u32) -> u64 { + if word_index as usize >= MAX_WORDS { + panic!("transpose_case_src_word_hi: word_index out of range"); + } + current_case(|case| case.src_word_hi(word_index as usize)) +} + +#[no_mangle] +pub extern "C" fn transpose_case_dst_word_lo(word_index: u32) -> u64 { + if word_index as usize >= MAX_WORDS { + panic!("transpose_case_dst_word_lo: word_index out of range"); + } + current_case(|case| case.dst_word_lo(word_index as usize)) +} + +#[no_mangle] +pub extern "C" fn transpose_case_dst_word_hi(word_index: u32) -> u64 { + if word_index as usize >= MAX_WORDS { + panic!("transpose_case_dst_word_hi: word_index out of range"); + } + current_case(|case| case.dst_word_hi(word_index as usize)) +} diff --git a/examples/balls/transpose/verify/casegen/src/model.rs b/examples/prefab/balls/transpose/verify/casegen/src/model.rs similarity index 100% rename from examples/balls/transpose/verify/casegen/src/model.rs rename to examples/prefab/balls/transpose/verify/casegen/src/model.rs diff --git a/examples/balls/transpose/verify/filelists/transpose_ball.f b/examples/prefab/balls/transpose/verify/filelists/transpose_ball.f similarity index 90% rename from examples/balls/transpose/verify/filelists/transpose_ball.f rename to examples/prefab/balls/transpose/verify/filelists/transpose_ball.f index 5bcc41e5c..bf90309d5 100644 --- a/examples/balls/transpose/verify/filelists/transpose_ball.f +++ b/examples/prefab/balls/transpose/verify/filelists/transpose_ball.f @@ -1,3 +1,4 @@ ++incdir+@UVM@/src/core +incdir+@UVM@/src/ball +incdir+@UVM@/src/ball/agents/cmd +incdir+@UVM@/src/ball/agents/mem @@ -13,6 +14,7 @@ +incdir+src/tests +incdir+@RTL@ +@UVM@/src/core/package.sv @UVM@/src/ball/bb_blink_if.sv @UVM@/src/ball/bb_uvm_pkg.sv diff --git a/examples/prefab/balls/transpose/verify/src/common/transpose_defs.svh b/examples/prefab/balls/transpose/verify/src/common/transpose_defs.svh new file mode 100644 index 000000000..9e3b9a49f --- /dev/null +++ b/examples/prefab/balls/transpose/verify/src/common/transpose_defs.svh @@ -0,0 +1,31 @@ +typedef struct { + int unsigned iter; + int unsigned op1_bank; + int unsigned wr_bank; + int unsigned op1_group; + int unsigned wr_group; + int unsigned op1_col; + int unsigned wr_col; + int unsigned rob_id; + int unsigned elem_bits; + int unsigned num_src_words; + int unsigned num_dst_words; +} transpose_cmd_dpi_t; + +import "DPI-C" function int transpose_case_load( + input int unsigned seed, + input int unsigned index +); +import "DPI-C" function void transpose_case_cmd(output transpose_cmd_dpi_t cmd); +import "DPI-C" function longint unsigned transpose_case_src_word_lo(input int unsigned word_index); +import "DPI-C" function longint unsigned transpose_case_src_word_hi(input int unsigned word_index); +import "DPI-C" function longint unsigned transpose_case_dst_word_lo(input int unsigned word_index); +import "DPI-C" function longint unsigned transpose_case_dst_word_hi(input int unsigned word_index); + +`ifndef TRANSPOSE_FUNCT7 +`error "TRANSPOSE_FUNCT7 must be provided by the selected Thread ballISA" +`endif +localparam int TRANSPOSE_THREAD_FUNCT7 = `TRANSPOSE_FUNCT7; +localparam int TRANSPOSE_MAX_WORDS = 32; +localparam int TRANSPOSE_TIMEOUT_CYCLES = 8000; +localparam int TRANSPOSE_SEED = 32'hCAFE_BABE; diff --git a/examples/prefab/balls/transpose/verify/src/common/transpose_items.svh b/examples/prefab/balls/transpose/verify/src/common/transpose_items.svh new file mode 100644 index 000000000..3cf709fe1 --- /dev/null +++ b/examples/prefab/balls/transpose/verify/src/common/transpose_items.svh @@ -0,0 +1,88 @@ +class transpose_cmd_item extends bb_blink_cmd_item; + `uvm_object_utils(transpose_cmd_item) + + bit [31:0] elem_bits; + bit [31:0] num_src_words; + bit [31:0] num_dst_words; + bit [127:0] src_words[TRANSPOSE_MAX_WORDS]; + bit [127:0] dst_words[TRANSPOSE_MAX_WORDS]; + + function new(string name = "transpose_cmd_item"); + super.new(name); + endfunction + + function void load_rust_case(int unsigned seed, int unsigned index); + transpose_cmd_dpi_t cmd; + int unsigned rc; + longint unsigned w_lo; + longint unsigned w_hi; + int i; + + rc = transpose_case_load(seed, index); + if (rc != 0) begin + `uvm_fatal("CASE", $sformatf("transpose_case_load returned %0d for index %0d", rc, index)) + end + transpose_case_cmd(cmd); + + // bb_transpose: banks and iter come from rs1, elem_bits from rs2[7:0]. + rob_id = cmd.rob_id[BB_BLINK_ROB_ID_W-1:0]; + funct = TRANSPOSE_THREAD_FUNCT7[6:0]; + rs2 = {56'd0, cmd.elem_bits[7:0]}; + iter = cmd.iter; + op1_bank = cmd.op1_bank; + op2_bank = '0; + wr_bank = cmd.wr_bank; + op1_group = cmd.op1_group; + op2_group = '0; + wr_group = cmd.wr_group; + op1_col = cmd.op1_col; + op2_col = '0; + wr_col = cmd.wr_col; + wr_read_bank = '0; + wr_arch_bank = cmd.wr_bank; + elem_bits = cmd.elem_bits; + num_src_words = cmd.num_src_words; + num_dst_words = cmd.num_dst_words; + + if (num_src_words > TRANSPOSE_MAX_WORDS || num_dst_words > TRANSPOSE_MAX_WORDS) begin + `uvm_fatal("CASE", $sformatf("word count out of range src=%0d dst=%0d", num_src_words, + num_dst_words)) + end + if (num_src_words != num_dst_words) begin + `uvm_fatal("CASE", $sformatf("src/dst word count mismatch src=%0d dst=%0d", num_src_words, + num_dst_words)) + end + if (!(elem_bits == 32'd8 || elem_bits == 32'd32)) begin + `uvm_fatal("CASE", $sformatf("unsupported elem_bits=%0d", elem_bits)) + end + if (op1_bank == wr_bank) begin + `uvm_fatal("CASE", "op1_bank and wr_bank overlap") + end + + for (i = 0; i < num_src_words; i++) begin + w_lo = transpose_case_src_word_lo(i); + w_hi = transpose_case_src_word_hi(i); + src_words[i] = {w_hi, w_lo}; + end + for (i = 0; i < num_dst_words; i++) begin + w_lo = transpose_case_dst_word_lo(i); + w_hi = transpose_case_dst_word_hi(i); + dst_words[i] = {w_hi, w_lo}; + end + endfunction + + function void do_copy(uvm_object rhs); + transpose_cmd_item rhs_; + super.do_copy(rhs); + if (!$cast(rhs_, rhs)) begin + `uvm_fatal("COPY", "rhs is not transpose_cmd_item") + end + elem_bits = rhs_.elem_bits; + num_src_words = rhs_.num_src_words; + num_dst_words = rhs_.num_dst_words; + for (int i = 0; i < TRANSPOSE_MAX_WORDS; i++) begin + src_words[i] = rhs_.src_words[i]; + dst_words[i] = rhs_.dst_words[i]; + end + endfunction +endclass diff --git a/examples/balls/transpose/verify/src/cov/transpose_cov.svh b/examples/prefab/balls/transpose/verify/src/cov/transpose_cov.svh similarity index 84% rename from examples/balls/transpose/verify/src/cov/transpose_cov.svh rename to examples/prefab/balls/transpose/verify/src/cov/transpose_cov.svh index 7913440ae..897f10e78 100644 --- a/examples/balls/transpose/verify/src/cov/transpose_cov.svh +++ b/examples/prefab/balls/transpose/verify/src/cov/transpose_cov.svh @@ -5,6 +5,7 @@ class transpose_cov extends uvm_component; covergroup cmd_cg; coverpoint cur_elem_bits {bins i8 = {8}; bins i32 = {32};} + coverpoint cur_col {bins c1 = {1}; bins c2 = {2};} coverpoint cur_iter { bins it1 = {1}; bins it2 = {2}; bins it4 = {4}; bins it8 = {8}; bins it16 = {16}; } @@ -12,6 +13,7 @@ class transpose_cov extends uvm_component; int unsigned cur_elem_bits; int unsigned cur_iter; + int unsigned cur_col; int cmd_count; function new(string name, uvm_component parent); @@ -23,18 +25,20 @@ class transpose_cov extends uvm_component; function void write_cmd(bb_blink_cmd_item item); cur_elem_bits = int'(item.rs2[7:0]); cur_iter = int'(item.iter); + cur_col = int'(item.op1_col); cmd_count++; cmd_cg.sample(); endfunction function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; super.check_phase(phase); if (cmd_count == 0) `uvm_fatal("COV", "no cmd observed") if (cmd_cg.get_coverage() != 100.0) begin `uvm_fatal( "COV", $sformatf( - "transpose_cov coverage incomplete: %0.2f%% (need elem_bits {8,32} and iter {1,2,4,8,16})", + "transpose_cov coverage incomplete: %0.2f%% (need elem_bits {8,32}, col {1,2} and iter {1,2,4,8,16})", cmd_cg.get_coverage())) end endfunction diff --git a/examples/balls/transpose/verify/src/env/transpose_env.svh b/examples/prefab/balls/transpose/verify/src/env/transpose_env.svh similarity index 100% rename from examples/balls/transpose/verify/src/env/transpose_env.svh rename to examples/prefab/balls/transpose/verify/src/env/transpose_env.svh diff --git a/examples/balls/transpose/verify/src/env/transpose_scoreboard.svh b/examples/prefab/balls/transpose/verify/src/env/transpose_scoreboard.svh similarity index 75% rename from examples/balls/transpose/verify/src/env/transpose_scoreboard.svh rename to examples/prefab/balls/transpose/verify/src/env/transpose_scoreboard.svh index a4da6eacb..3032cafc9 100644 --- a/examples/balls/transpose/verify/src/env/transpose_scoreboard.svh +++ b/examples/prefab/balls/transpose/verify/src/env/transpose_scoreboard.svh @@ -47,8 +47,10 @@ class transpose_scoreboard extends uvm_scoreboard; `uvm_fatal("SCB", "mem_model handle not set") end mem_model.clear_mem(); + // Word i lives at row i / col, group op1_group + i % col. for (i = 0; i < clone.num_src_words; i++) begin - mem_model.load_word(int'(clone.op1_bank), i, clone.src_words[i]); + mem_model.load_word_g(int'(clone.op1_bank), int'(clone.op1_group) + i % clone.op1_col, + i / clone.op1_col, clone.src_words[i]); end mem_model.arm(); @@ -67,21 +69,12 @@ class transpose_scoreboard extends uvm_scoreboard; endfunction function void check_cmd(bb_blink_cmd_item got, transpose_cmd_item exp); - if (got.bid !== exp.bid) - `uvm_fatal("CMD", $sformatf("bid mismatch: got %0d exp %0d", got.bid, exp.bid)) - if (got.funct7 !== exp.funct7) - `uvm_fatal("CMD", $sformatf("funct7 mismatch: got %0d exp %0d", got.funct7, exp.funct7)) - if (got.iter !== exp.iter) - `uvm_fatal("CMD", $sformatf("iter mismatch: got %0d exp %0d", got.iter, exp.iter)) - if (got.op1_bank !== exp.op1_bank || got.wr_bank !== exp.wr_bank) - `uvm_fatal("CMD", "bank field mismatch") - if (got.op1_col !== exp.op1_col || got.wr_col !== exp.wr_col) - `uvm_fatal("CMD", "column field mismatch") - if (got.rob_id !== exp.rob_id) - `uvm_fatal("CMD", $sformatf("rob_id mismatch: got %0d exp %0d", got.rob_id, exp.rob_id)) - if (got.rs2[7:0] !== exp.elem_bits[7:0]) - `uvm_fatal("CMD", $sformatf( - "elem_bits mismatch: got %0d exp %0d", got.rs2[7:0], exp.elem_bits[7:0])) + // Compare as plain BLink commands; the comparer rejects differing types. + bb_blink_cmd_item want = bb_blink_cmd_item::type_id::create("want"); + want.copy(exp); + verification_pkg::evidence::checks++; + if (!got.compare(want)) + `uvm_fatal("CMD", $sformatf("command mismatch: got %s exp %s", got.sprint(), want.sprint())) endfunction function void write_read(bb_blink_read_item item); @@ -91,12 +84,16 @@ class transpose_scoreboard extends uvm_scoreboard; `uvm_fatal("READ", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.op1_bank)) if (item.rob_id !== stim.rob_id) `uvm_fatal("READ", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.group_id !== '0) - `uvm_fatal("READ", $sformatf("group_id mismatch: got %0d exp 0", item.group_id)) - // DUT may re-read the same bank word across output columns (cache refill). - if (item.addr >= stim.num_src_words[6:0]) + if (item.group_id < stim.op1_group || item.group_id >= stim.op1_group + stim.op1_col) `uvm_fatal("READ", $sformatf( - "addr out of range: got %0d max %0d", item.addr, stim.num_src_words)) + "group_id out of range: got %0d base %0d col %0d", + item.group_id, + stim.op1_group, + stim.op1_col + )) + // DUT may re-read the same bank word across output columns (cache refill). + if (item.addr >= stim.iter) + `uvm_fatal("READ", $sformatf("addr out of range: got %0d iter %0d", item.addr, stim.iter)) read_count++; endfunction @@ -107,10 +104,11 @@ class transpose_scoreboard extends uvm_scoreboard; `uvm_fatal("WRITE", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.wr_bank)) if (item.rob_id !== stim.rob_id) `uvm_fatal("WRITE", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.group_id !== '0) - `uvm_fatal("WRITE", $sformatf("group_id mismatch: got %0d exp 0", item.group_id)) - if (item.addr !== write_count[6:0]) - `uvm_fatal("WRITE", $sformatf("addr mismatch: got %0d exp %0d", item.addr, write_count)) + if (item.group_id !== stim.wr_group + write_count % stim.wr_col) + `uvm_fatal("WRITE", $sformatf("group_id mismatch at %0d: got %0d", write_count, item.group_id + )) + if (item.addr !== write_count / stim.wr_col) + `uvm_fatal("WRITE", $sformatf("addr mismatch at %0d: got %0d", write_count, item.addr)) if (item.mask !== 16'hFFFF) `uvm_fatal("WRITE", $sformatf("mask mismatch: got 0x%04h", item.mask)) if (item.data !== expected_dst[write_count]) @@ -128,7 +126,6 @@ class transpose_scoreboard extends uvm_scoreboard; stim = current_stim("RESP"); if (item.rob_id !== stim.rob_id) `uvm_fatal("RESP", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.is_sub !== 1'b0) `uvm_fatal("RESP", "is_sub should be 0") resp_count++; endfunction @@ -155,6 +152,7 @@ class transpose_scoreboard extends uvm_scoreboard; endfunction function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; super.check_phase(phase); if (!done()) begin `uvm_fatal("SCB", $sformatf("incomplete: cmds=%0d reads=%0d writes=%0d/%0d resp=%0d", diff --git a/examples/balls/transpose/verify/src/pkg/transpose_pkg.sv b/examples/prefab/balls/transpose/verify/src/pkg/transpose_pkg.sv similarity index 100% rename from examples/balls/transpose/verify/src/pkg/transpose_pkg.sv rename to examples/prefab/balls/transpose/verify/src/pkg/transpose_pkg.sv diff --git a/examples/balls/transpose/verify/src/seq/transpose_sequences.svh b/examples/prefab/balls/transpose/verify/src/seq/transpose_sequences.svh similarity index 86% rename from examples/balls/transpose/verify/src/seq/transpose_sequences.svh rename to examples/prefab/balls/transpose/verify/src/seq/transpose_sequences.svh index 1a426859b..bf290a300 100644 --- a/examples/balls/transpose/verify/src/seq/transpose_sequences.svh +++ b/examples/prefab/balls/transpose/verify/src/seq/transpose_sequences.svh @@ -3,7 +3,6 @@ class transpose_basic_seq extends uvm_sequence #(bb_blink_cmd_item); int unsigned case_index; int unsigned seed; - int unsigned bid; function new(string name = "transpose_basic_seq"); super.new(name); @@ -14,7 +13,7 @@ class transpose_basic_seq extends uvm_sequence #(bb_blink_cmd_item); transpose_cmd_item item; item = transpose_cmd_item::type_id::create("item"); start_item(item); - item.load_rust_case(seed, case_index, bid); + item.load_rust_case(seed, case_index); finish_item(item); endtask endclass diff --git a/examples/prefab/balls/transpose/verify/src/tb_top.sv b/examples/prefab/balls/transpose/verify/src/tb_top.sv new file mode 100644 index 000000000..600d0e47e --- /dev/null +++ b/examples/prefab/balls/transpose/verify/src/tb_top.sv @@ -0,0 +1,77 @@ +module tb_top; + import uvm_pkg::*; + import transpose_pkg::*; + + bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); + + TransposeBall dut ( + .clock(intf.clock), + .reset(intf.reset), + .blink_status_running(intf.status_running), + .blink_cmdReq_ready(intf.cmd_req_ready), + .blink_cmdReq_valid(intf.cmd_req_valid), + .blink_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .blink_cmdReq_bits_funct(intf.cmd_req_bits_funct), + .blink_cmdReq_bits_rs2(intf.cmd_req_bits_rs2), + .blink_cmdReq_bits_iter(intf.cmd_req_bits_iter), + .blink_cmdReq_bits_op1_bank(intf.cmd_req_bits_op1_bank), + .blink_cmdReq_bits_op2_bank(intf.cmd_req_bits_op2_bank), + .blink_cmdReq_bits_wr_bank(intf.cmd_req_bits_wr_bank), + .blink_cmdReq_bits_op1_group(intf.cmd_req_bits_op1_group), + .blink_cmdReq_bits_op2_group(intf.cmd_req_bits_op2_group), + .blink_cmdReq_bits_wr_group(intf.cmd_req_bits_wr_group), + .blink_cmdReq_bits_op1_col(intf.cmd_req_bits_op1_col), + .blink_cmdReq_bits_op2_col(intf.cmd_req_bits_op2_col), + .blink_cmdReq_bits_wr_col(intf.cmd_req_bits_wr_col), + .blink_cmdReq_bits_wr_read_bank(intf.cmd_req_bits_wr_read_bank), + .blink_cmdReq_bits_wr_arch_bank(intf.cmd_req_bits_wr_arch_bank), + .blink_cmdResp_ready(intf.cmd_resp_ready), + .blink_cmdResp_valid(intf.cmd_resp_valid), + .blink_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .blink_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .blink_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .blink_bankRead_0_group_id(intf.bank_read_group_id[0]), + .blink_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .blink_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .blink_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .blink_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .blink_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .blink_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .blink_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .blink_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .blink_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .blink_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .blink_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .blink_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .blink_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), + .blink_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), + .blink_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), + .blink_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), + .blink_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), + .blink_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), + .blink_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), + .blink_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), + .blink_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), + .blink_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), + .blink_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), + .blink_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), + .blink_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), + .blink_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), + .blink_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), + .blink_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), + .blink_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .blink_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .blink_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .blink_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]) + ); + + initial begin + intf.clock = 1'b0; + forever #5 intf.clock = ~intf.clock; + end + + initial begin + uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); + run_test("transpose_ball_test"); + end +endmodule diff --git a/examples/balls/transpose/verify/src/tests/transpose_test.svh b/examples/prefab/balls/transpose/verify/src/tests/transpose_test.svh similarity index 80% rename from examples/balls/transpose/verify/src/tests/transpose_test.svh rename to examples/prefab/balls/transpose/verify/src/tests/transpose_test.svh index b235d24e5..962dab433 100644 --- a/examples/balls/transpose/verify/src/tests/transpose_test.svh +++ b/examples/prefab/balls/transpose/verify/src/tests/transpose_test.svh @@ -18,27 +18,26 @@ class transpose_ball_test extends uvm_test; endfunction task run_phase(uvm_phase phase); - int unsigned bid; - bid = transpose_require_bid(); phase.raise_objection(this); - run_directed(bid); - run_random(bid); + run_directed(); + run_random(); phase.drop_objection(this); endtask - task run_directed(int unsigned bid); - run_case(0, "DIR_I8", bid); - run_case(1, "DIR_I32", bid); + task run_directed(); + run_case(0, "DIR_I8"); + run_case(1, "DIR_I32"); + run_case(2, "DIR_I8_2G"); endtask - task run_random(int unsigned bid); + task run_random(); int i; - for (i = 2; i <= 21; i++) begin - run_case(i, $sformatf("RND_%0d", i), bid); + for (i = 3; i <= 22; i++) begin + run_case(i, $sformatf("RND_%0d", i)); end endtask - task run_case(int unsigned idx, string label, int unsigned bid); + task run_case(int unsigned idx, string label); transpose_basic_seq seq; int cycles; @@ -47,7 +46,6 @@ class transpose_ball_test extends uvm_test; seq = transpose_basic_seq::type_id::create("seq"); seq.case_index = idx; - seq.bid = bid; seq.start(env.cmd_agent.seqr); cycles = 0; diff --git a/examples/prefab/balls/transpose/workloads/ctests/CMakeLists.txt b/examples/prefab/balls/transpose/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..acf4f2cba --- /dev/null +++ b/examples/prefab/balls/transpose/workloads/ctests/CMakeLists.txt @@ -0,0 +1,3 @@ +add_buckyball_ctest(transpose_i8_16x16_test.c) +add_buckyball_ctest(transpose_i8_16x32_test.c) +add_buckyball_ctest(transpose_i32_8x16_test.c) diff --git a/examples/balls/transpose/workloads/ctests/transpose_i32_8x16_test.c b/examples/prefab/balls/transpose/workloads/ctests/transpose_i32_8x16_test.c similarity index 94% rename from examples/balls/transpose/workloads/ctests/transpose_i32_8x16_test.c rename to examples/prefab/balls/transpose/workloads/ctests/transpose_i32_8x16_test.c index e9c213e91..b60c6a901 100644 --- a/examples/balls/transpose/workloads/ctests/transpose_i32_8x16_test.c +++ b/examples/prefab/balls/transpose/workloads/ctests/transpose_i32_8x16_test.c @@ -8,7 +8,7 @@ #define ROWS 8 #define N_GROUPS 4 -#define COLS (N_GROUPS * ((BANK_WIDTH) / 32)) +#define COLS (N_GROUPS * ((THREAD_MEM_BANK_WIDTH) / 32)) #define ELEM_BITS 32 static int32_t input_matrix[ROWS * COLS] __attribute__((aligned(64))) = { @@ -27,8 +27,8 @@ static int32_t output_matrix[COLS * ROWS] __attribute__((aligned(64))); static int32_t expected_matrix[COLS * ROWS] __attribute__((aligned(64))); int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif for (int r = 0; r < ROWS; ++r) { diff --git a/examples/balls/transpose/workloads/ctests/transpose_i8_16x16_test.c b/examples/prefab/balls/transpose/workloads/ctests/transpose_i8_16x16_test.c similarity index 96% rename from examples/balls/transpose/workloads/ctests/transpose_i8_16x16_test.c rename to examples/prefab/balls/transpose/workloads/ctests/transpose_i8_16x16_test.c index ca16efb6b..16372179c 100644 --- a/examples/balls/transpose/workloads/ctests/transpose_i8_16x16_test.c +++ b/examples/prefab/balls/transpose/workloads/ctests/transpose_i8_16x16_test.c @@ -6,7 +6,7 @@ #define ROWS 16 #define N_GROUPS 1 -#define COLS (N_GROUPS * ((BANK_WIDTH) / 8)) +#define COLS (N_GROUPS * ((THREAD_MEM_BANK_WIDTH) / 8)) #define ELEM_BITS 8 static elem_t input_matrix[ROWS * COLS] __attribute__((aligned(64))) = { @@ -38,8 +38,8 @@ static elem_t output_matrix[COLS * ROWS] __attribute__((aligned(64))); static elem_t expected_matrix[COLS * ROWS] __attribute__((aligned(64))); int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif transpose_u8_matrix(input_matrix, expected_matrix, ROWS, COLS); diff --git a/examples/balls/transpose/workloads/ctests/transpose_i8_16x32_test.c b/examples/prefab/balls/transpose/workloads/ctests/transpose_i8_16x32_test.c similarity index 96% rename from examples/balls/transpose/workloads/ctests/transpose_i8_16x32_test.c rename to examples/prefab/balls/transpose/workloads/ctests/transpose_i8_16x32_test.c index b0dc629be..9c10139a8 100644 --- a/examples/balls/transpose/workloads/ctests/transpose_i8_16x32_test.c +++ b/examples/prefab/balls/transpose/workloads/ctests/transpose_i8_16x32_test.c @@ -6,7 +6,7 @@ #define ROWS 16 #define N_GROUPS 2 -#define COLS (N_GROUPS * ((BANK_WIDTH) / 8)) +#define COLS (N_GROUPS * ((THREAD_MEM_BANK_WIDTH) / 8)) #define ELEM_BITS 8 static elem_t input_matrix[ROWS * COLS] __attribute__((aligned(64))) = { @@ -43,8 +43,8 @@ static elem_t output_matrix[COLS * ROWS] __attribute__((aligned(64))); static elem_t expected_matrix[COLS * ROWS] __attribute__((aligned(64))); int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif transpose_u8_matrix(input_matrix, expected_matrix, ROWS, COLS); diff --git a/examples/balls/transpose/workloads/isa/transpose.h b/examples/prefab/balls/transpose/workloads/isa/transpose.h similarity index 87% rename from examples/balls/transpose/workloads/isa/transpose.h rename to examples/prefab/balls/transpose/workloads/isa/transpose.h index 4c219ae72..2b0b07060 100644 --- a/examples/balls/transpose/workloads/isa/transpose.h +++ b/examples/prefab/balls/transpose/workloads/isa/transpose.h @@ -10,6 +10,6 @@ #define bb_transpose(op1_bank_id, wr_bank_id, iter, elem_bits) \ BUCKYBALL_INSTRUCTION_R_R( \ (BB_BANK0(op1_bank_id) | BB_BANK2(wr_bank_id) | BB_ITER(iter)), \ - (FIELD((uint64_t)(elem_bits), 0, 7)), BB_FUNC7(TRANSPOSE)) + (FIELD((uint64_t)(elem_bits), 0, 7)), FUNCT7(transpose, TRANSPOSE)) #endif // _BB_TRANSPOSE_H_ diff --git a/examples/balls/transpose/workloads/mlir_tests/CMakeLists.txt b/examples/prefab/balls/transpose/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/balls/transpose/workloads/mlir_tests/CMakeLists.txt rename to examples/prefab/balls/transpose/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/balls/transpose/workloads/mlir_tests/ball/CMakeLists.txt b/examples/prefab/balls/transpose/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/balls/transpose/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/prefab/balls/transpose/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/prefab/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8.mlir b/examples/prefab/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8.mlir new file mode 100644 index 000000000..7c2fbf3b2 --- /dev/null +++ b/examples/prefab/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8.mlir @@ -0,0 +1,38 @@ +// Physical-bank transpose_i8 of a 16x16 i8 matrix. + +func.func private @check_result(memref<16x16xi8>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %n = arith.constant 16 : index + %c16_i32 = arith.constant 16 : i32 + + %input = memref.alloc() alignment = 64 : memref<16x16xi8> + %output = memref.alloc() alignment = 64 : memref<16x16xi8> + + scf.for %i = %c0 to %n step %c1 { + scf.for %j = %c0 to %n step %c1 { + %ii = arith.index_cast %i : index to i32 + %jj = arith.index_cast %j : index to i32 + %mul = arith.muli %ii, %c16_i32 : i32 + %val = arith.addi %mul, %jj : i32 + %v8 = arith.trunci %val : i32 to i8 + memref.store %v8, %input[%i, %j] : memref<16x16xi8> + } + } + + buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %input <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + transpose_ball.transpose_i8 <{input = 1 : i64, output = 2 : i64, rows = 16 : i64, cols = 16 : i64}> + buckyball.mvout %output <{bank = 2 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + + func.call @check_result(%output) : (memref<16x16xi8>) -> () + memref.dealloc %input : memref<16x16xi8> + memref.dealloc %output : memref<16x16xi8> + return %zero_i8 : i8 +} diff --git a/examples/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8_main.cpp b/examples/prefab/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8_main.cpp similarity index 100% rename from examples/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8_main.cpp rename to examples/prefab/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8_main.cpp diff --git a/examples/balls/transpose/workloads/mlir_tests/bank/CMakeLists.txt b/examples/prefab/balls/transpose/workloads/mlir_tests/bank/CMakeLists.txt similarity index 100% rename from examples/balls/transpose/workloads/mlir_tests/bank/CMakeLists.txt rename to examples/prefab/balls/transpose/workloads/mlir_tests/bank/CMakeLists.txt diff --git a/examples/prefab/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix.mlir.in b/examples/prefab/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix.mlir.in new file mode 100644 index 000000000..2b691dc07 --- /dev/null +++ b/examples/prefab/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix.mlir.in @@ -0,0 +1,32 @@ +func.func private @check_result(memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %rows = arith.constant @BB_BANK_LINES@ : index + %lanes = arith.constant @BB_BANK_I8_LANES@ : index + %lanes_i32 = arith.constant @BB_BANK_I8_LANES@ : i32 + %input = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + %output = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + scf.for %row = %zero to %rows step %one { + scf.for %lane = %zero to %lanes step %one { + %r = arith.index_cast %row : index to i32 + %l = arith.index_cast %lane : index to i32 + %base = arith.muli %r, %lanes_i32 : i32 + %index = arith.addi %base, %l : i32 + %value = arith.trunci %index : i32 to i8 + memref.store %value, %input[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + } + } + %input_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %output_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %loaded = buckyball.bank_mvin %input, %input_bank <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + %result = transpose_ball.bank_transpose_i8 %loaded -> %output_bank <{rows = @BB_BANK_LINES@ : i64, cols = @BB_BANK_I8_LANES@ : i64}> + %stored = buckyball.bank_mvout %output, %result <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + buckyball.bank_release %loaded + buckyball.bank_release %stored + func.call @check_result(%output) : (memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () + memref.dealloc %input : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + memref.dealloc %output : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + return %zero_i8 : i8 +} diff --git a/examples/prefab/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix_main.cpp b/examples/prefab/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix_main.cpp new file mode 100644 index 000000000..efe2b5607 --- /dev/null +++ b/examples/prefab/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix_main.cpp @@ -0,0 +1,22 @@ +#include +#include +#include +extern "C" void check_result(int8_t *, int8_t *data, int64_t offset, + int64_t rows, int64_t lanes, int64_t stride, + int64_t lane_stride) { + if (rows != THREAD_MEM_BANK_ENTRIES || lanes != THREAD_MEM_BANK_WIDTH / 8 || + stride != lanes || lane_stride != 1) + exit(1); + for (int row = 0; row < rows; ++row) + for (int lane = 0; lane < lanes; ++lane) { + int flat = row * lanes + lane; + int column = flat / THREAD_MEM_BANK_ENTRIES; + int source_row = flat % THREAD_MEM_BANK_ENTRIES; + int8_t expected = (int8_t)(source_row * lanes + column); + if (data[offset + flat] != expected) { + printf("transpose bank mismatch row=%d lane=%d\n", row, lane); + exit(1); + } + } + printf("transpose bank PASS rows=%d\n", THREAD_MEM_BANK_ENTRIES); +} diff --git a/examples/balls/vector/arch/src/main/scala/README.md b/examples/prefab/balls/vector/arch/README.md similarity index 100% rename from examples/balls/vector/arch/src/main/scala/README.md rename to examples/prefab/balls/vector/arch/README.md diff --git a/examples/prefab/balls/vector/arch/VecBall.scala b/examples/prefab/balls/vector/arch/VecBall.scala new file mode 100644 index 000000000..2b137a5eb --- /dev/null +++ b/examples/prefab/balls/vector/arch/VecBall.scala @@ -0,0 +1,66 @@ +package examples.balls.vector + +import framework.acc.balldomain.bbus.{BallFactory, BallPorts} +import buckyball.balls.vector.{Params => ParametersProto} +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import examples.balls.vector.configs.VectorBallParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.balldomain.ball.blink.{BallStatus, BlinkIO, HasBlink} +import examples.balls.vector.VecUnit + +/** + * VecBall + */ +@instantiable +class VecBall( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Module + with HasBlink { + val ball = bp.balls.find(_.id == id).get + + val ballParam = VectorBallParam.fromProto(ball.parameters.unpack(classOf[ParametersProto])) + + val inBW = ball.inBW + val outBW = ball.outBW + + @public + val blink = IO(new BlinkIO(fp, mp, bp, id)) + + val vecUnit: Instance[VecUnit] = + Instantiate(new VecUnit(fp, mp, bp, ball, ballParam)) + + vecUnit.io.cmdReq <> blink.cmdReq + vecUnit.io.cmdResp <> blink.cmdResp + + for (i <- 0 until inBW) { + vecUnit.io.bankRead(i) <> blink.bankRead(i) + } + + for (i <- 0 until outBW) { + vecUnit.io.bankWrite(i) <> blink.bankWrite(i) + } + + blink.status <> vecUnit.io.status + +} + +object VecBall extends BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts = + BallPorts(Instantiate(new VecBall(fp, mp, bp, id)).blink, None) + +} diff --git a/examples/prefab/balls/vector/arch/VecCtrlUnit.scala b/examples/prefab/balls/vector/arch/VecCtrlUnit.scala new file mode 100644 index 000000000..fe28b7ed0 --- /dev/null +++ b/examples/prefab/balls/vector/arch/VecCtrlUnit.scala @@ -0,0 +1,131 @@ +package examples.balls.vector + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import examples.balls.vector.configs.VectorBallParam + +import chisel3._ +import framework.acc.balldomain.ball.blink.{BallCommand, BallResponse} +import chisel3.util._ +import chisel3.stage._ +import chisel3.experimental.hierarchy.{instantiable, public} + +@instantiable +class VecCtrlUnit(val frontendParam: FrontendParam, val memParam: MemDomainParam, val ballParam: VectorBallParam) + extends Module { + val commandDataBits = frontendParam.commandDataBits + + @public + val io = IO(new Bundle { + + val cmdReq = Flipped(Decoupled(new BallCommand(frontendParam, memParam))) + + val cmdResp_o = Decoupled(new BallResponse(frontendParam)) + + val ctrl_ld_o = Decoupled(new ctrl_ld_req(frontendParam, memParam, ballParam)) + val ctrl_st_o = Decoupled(new ctrl_st_req(frontendParam, memParam, ballParam)) + val ctrl_ex_o = Decoupled(new ctrl_ex_req(frontendParam, ballParam)) + + val cmdResp_i = Flipped(Valid(new Bundle { + val commit = Bool() + })) // from store unit + }) + + val rob_id_reg = RegInit(0.U(log2Up(frontendParam.rob_entries).W)) + val iter = RegInit(0.U(frontendParam.iter_len.W)) + val op1_bank = RegInit(0.U(frontendParam.bank_id_len.W)) + + val op1_bank_addr = RegInit( + 0.U(12.W) + ) // New ISA: always 0, but keep for compatibility + val op2_bank_addr = RegInit( + 0.U(12.W) + ) // New ISA: always 0, but keep for compatibility + val op2_bank = RegInit(0.U(frontendParam.bank_id_len.W)) + val wr_bank = RegInit(0.U(frontendParam.bank_id_len.W)) + + val wr_bank_addr = RegInit( + 0.U(12.W) + ) // New ISA: always 0, but keep for compatibility + val has_send = RegInit(false.B) + val mode = RegInit(0.U(1.W)) + + val idle :: busy :: Nil = Enum(2) + val state = RegInit(idle) + +// ----------------------------------------------------------------------------- +// Set registers when EX instruction arrives +// ----------------------------------------------------------------------------- + + when(io.cmdReq.fire) { + iter := io.cmdReq.bits.iter + rob_id_reg := io.cmdReq.bits.rob_id + op1_bank := io.cmdReq.bits.op1_bank + op1_bank_addr := 0.U // New ISA: all operations start from row 0 + op2_bank := io.cmdReq.bits.op2_bank + op2_bank_addr := 0.U // New ISA: all operations start from row 0 + wr_bank := io.cmdReq.bits.wr_bank + wr_bank_addr := 0.U // New ISA: all operations start from row 0 + mode := io.cmdReq.bits.rs2(0) + + state := busy + } + +// ----------------------------------------------------------------------------- +// Send control signals to VecUnit's load/store/ex units +// ----------------------------------------------------------------------------- + + when(state === busy && !has_send) { + io.ctrl_ld_o.valid := true.B + io.ctrl_ld_o.bits.op1_bank := op1_bank + io.ctrl_ld_o.bits.op1_bank_addr := op1_bank_addr + io.ctrl_ld_o.bits.op2_bank := op2_bank + io.ctrl_ld_o.bits.op2_bank_addr := op2_bank_addr + io.ctrl_ld_o.bits.iter := iter + io.ctrl_ld_o.bits.mode := mode + + io.ctrl_ex_o.valid := true.B + io.ctrl_ex_o.bits.iter := iter + + io.ctrl_st_o.valid := true.B + io.ctrl_st_o.bits.wr_bank := wr_bank + io.ctrl_st_o.bits.wr_bank_addr := wr_bank_addr + io.ctrl_st_o.bits.iter := iter + + has_send := true.B + }.otherwise { + io.ctrl_ld_o.valid := false.B + io.ctrl_ld_o.bits.op1_bank := 0.U + io.ctrl_ld_o.bits.op1_bank_addr := 0.U + io.ctrl_ld_o.bits.op2_bank := 0.U + io.ctrl_ld_o.bits.op2_bank_addr := 0.U + io.ctrl_ld_o.bits.iter := 0.U + io.ctrl_ld_o.bits.mode := 0.U + + io.ctrl_ex_o.valid := false.B + io.ctrl_ex_o.bits.iter := 0.U + + io.ctrl_st_o.valid := false.B + io.ctrl_st_o.bits.wr_bank := 0.U + io.ctrl_st_o.bits.wr_bank_addr := 0.U + io.ctrl_st_o.bits.iter := 0.U + } + +// ----------------------------------------------------------------------------- +// Wait for VecUnit's final write-back to complete +// ----------------------------------------------------------------------------- + + when(io.cmdResp_i.valid) { + io.cmdResp_o.valid := true.B + io.cmdResp_o.bits.rob_id := rob_id_reg + state := idle + has_send := false.B + }.otherwise { + io.cmdResp_o.valid := false.B + io.cmdResp_o.bits.rob_id := 0.U + } + + io.cmdReq.ready := state === idle + +} diff --git a/examples/balls/vector/arch/src/main/scala/VecEXUnit.scala b/examples/prefab/balls/vector/arch/VecEXUnit.scala similarity index 76% rename from examples/balls/vector/arch/src/main/scala/VecEXUnit.scala rename to examples/prefab/balls/vector/arch/VecEXUnit.scala index 484a3f940..f9eead981 100644 --- a/examples/balls/vector/arch/src/main/scala/VecEXUnit.scala +++ b/examples/prefab/balls/vector/arch/VecEXUnit.scala @@ -1,43 +1,46 @@ package examples.balls.vector +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import framework.acc.frontend.configs.FrontendParam + import chisel3._ import chisel3.util._ import chisel3.stage._ import chisel3.experimental.hierarchy.{instantiable, public} -import framework.top.GlobalConfig import examples.balls.vector.warp.MeshWarp import examples.balls.vector.configs.VectorBallParam -class ctrl_ex_req(b: GlobalConfig) extends Bundle { - val iter = UInt(b.frontend.iter_len.W) +class ctrl_ex_req(frontendParam: FrontendParam, val ballParam: VectorBallParam) extends Bundle { + val iter = UInt(frontendParam.iter_len.W) } -class ld_ex_req(b: GlobalConfig) extends Bundle { - val config = VectorBallParam(b) +class ld_ex_req(frontendParam: FrontendParam, val ballParam: VectorBallParam) extends Bundle { + val config = ballParam val op1 = Vec(config.lane, UInt(config.inputWidth.W)) val op2 = Vec(config.lane, UInt(config.inputWidth.W)) - val iter = UInt(b.frontend.iter_len.W) + val iter = UInt(frontendParam.iter_len.W) } @instantiable -class VecEXUnit(val b: GlobalConfig) extends Module { - val config = VectorBallParam(b) +class VecEXUnit(val frontendParam: FrontendParam, val ballParam: VectorBallParam) extends Module { + val config = ballParam val InputNum = config.lane val inputWidth = config.inputWidth val accWidth = config.outputWidth @public val io = IO(new Bundle { - val ctrl_ex_i = Flipped(Decoupled(new ctrl_ex_req(b))) - val ld_ex_i = Flipped(Decoupled(new ld_ex_req(b))) + val ctrl_ex_i = Flipped(Decoupled(new ctrl_ex_req(frontendParam, ballParam))) + val ld_ex_i = Flipped(Decoupled(new ld_ex_req(frontendParam, ballParam))) - val ex_st_o = Decoupled(new ex_st_req(b)) + val ex_st_o = Decoupled(new ex_st_req(frontendParam, ballParam)) }) val idle :: busy :: Nil = Enum(2) val state = RegInit(idle) - val meshWarp = Module(new MeshWarp(config)) + val meshWarp = Instantiate(new MeshWarp(ballParam)) // Thread ID for MeshWarp (always use thread 0 for now) val threadId = RegInit(0.U(10.W)) diff --git a/examples/prefab/balls/vector/arch/VecLoadUnit.scala b/examples/prefab/balls/vector/arch/VecLoadUnit.scala new file mode 100644 index 000000000..dc03a3a66 --- /dev/null +++ b/examples/prefab/balls/vector/arch/VecLoadUnit.scala @@ -0,0 +1,203 @@ +package examples.balls.vector + +import framework.acc.balldomain.configs.BallRouteParam + +import framework.lib.queue.Queue +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import framework.ip.memory.bank.{SramReadIO, SramReadReq, SramReadResp, SramWriteIO, SramWriteReq, SramWriteResp} + +import chisel3._ +import chisel3.util._ +import chisel3.stage._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.ip.memory.bank.{SramReadReq, SramReadResp} +import examples.balls.vector.configs.VectorBallParam + +class ctrl_ld_req(frontendParam: FrontendParam, memParam: MemDomainParam, val ballParam: VectorBallParam) + extends Bundle { + val op1_bank = UInt(frontendParam.bank_id_len.W) + val op1_bank_addr = UInt(log2Up(memParam.bankEntries).W) + val op2_bank = UInt(frontendParam.bank_id_len.W) + val op2_bank_addr = UInt(log2Up(memParam.bankEntries).W) + val iter = UInt(frontendParam.iter_len.W) + val mode = UInt(1.W) +} + +@instantiable +class VecLoadUnit( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val routingParam: BallDomainParam, + val instance: BallRouteParam, + val ballParam: VectorBallParam) + extends Module { + val config = ballParam + val InputNum = config.lane + val inputWidth = config.inputWidth + val bankWidth = memParam.bankWidth + val rob_id_width = log2Up(frontendParam.rob_entries) + + // Use this instance's configured bandwidth. + val ballMapping = instance + + val inBW = ballMapping.inBW + + @public + val io = IO(new Bundle { + val bankReadReq = Vec(inBW, Decoupled(new SramReadReq(memParam))) + val bankReadResp = + Vec(inBW, Flipped(Decoupled(new SramReadResp(memParam)))) + val ctrl_ld_i = Flipped(Decoupled(new ctrl_ld_req(frontendParam, memParam, ballParam))) + val ld_ex_o = Decoupled(new ld_ex_req(frontendParam, ballParam)) + val op1_bank_o = Output(UInt(frontendParam.bank_id_len.W)) + val op2_bank_o = Output(UInt(frontendParam.bank_id_len.W)) + }) + + val op1_bank = RegInit(0.U(frontendParam.bank_id_len.W)) + val op2_bank = RegInit(0.U(frontendParam.bank_id_len.W)) + val op1_addr = RegInit(0.U(log2Up(memParam.bankEntries).W)) + val op2_addr = RegInit(0.U(log2Up(memParam.bankEntries).W)) + val iter = RegInit(0.U(frontendParam.iter_len.W)) + val op1_iter_counter = RegInit(0.U(frontendParam.iter_len.W)) + val op2_iter_counter = RegInit(0.U(frontendParam.iter_len.W)) + val idle :: busy :: Nil = Enum(2) + val state = RegInit(idle) + val ld_ex_op1_reg = Reg(Vec(InputNum, UInt(inputWidth.W))) + val ld_ex_op2_reg = Reg(Vec(InputNum, UInt(inputWidth.W))) + val ld_ex_iter_reg = RegInit(0.U(frontendParam.iter_len.W)) + val wait1_reg = RegInit(false.B) + val wait2_reg = RegInit(false.B) + val wait1_cnt = RegInit(0.U(7.W)) + val wait2_cnt = RegInit(0.U(7.W)) + + val bankRespQueue0 = Instantiate(new Queue( + new SramReadResp(memParam), + 8, + false, + false + )) + + val bankRespQueue1 = Instantiate(new Queue( + new SramReadResp(memParam), + 8, + false, + false + )) + + for (i <- 0 until inBW) { + io.bankReadReq(i).valid := false.B + io.bankReadReq(i).bits.addr := 0.U + } + + io.op1_bank_o := op1_bank + io.op2_bank_o := op2_bank + io.ctrl_ld_i.ready := state === idle + + bankRespQueue0.io.enq <> io.bankReadResp(0) + bankRespQueue1.io.enq <> io.bankReadResp(1) + +// ----------------------------------------------------------------------------- +// Set registers when Ctrl instruction arrives +// ----------------------------------------------------------------------------- + when(io.ctrl_ld_i.fire) { + op1_bank := io.ctrl_ld_i.bits.op1_bank + op2_bank := io.ctrl_ld_i.bits.op2_bank + op1_addr := io.ctrl_ld_i.bits.op1_bank_addr + op2_addr := io.ctrl_ld_i.bits.op2_bank_addr + iter := io.ctrl_ld_i.bits.iter + op1_iter_counter := 0.U + op2_iter_counter := 0.U + state := busy + assert(io.ctrl_ld_i.bits.iter > 0.U, "iter should be greater than 0") + } + +// ----------------------------------------------------------------------------- +// Send SRAM read request +// ----------------------------------------------------------------------------- + //wait1_reg := Mux(io.run , 0.U, wait1_reg) + + //wait2_reg := Mux(io.run , 0.U, wait2_reg) + + when(state === busy && io.ld_ex_o.ready && !wait1_reg) { + io.bankReadReq(0).valid := op1_iter_counter < iter + io.bankReadReq(0).bits.addr := op1_addr + op1_iter_counter + op1_iter_counter := Mux( + io.bankReadReq(0).ready, + op1_iter_counter + 1.U, + op1_iter_counter + ) + wait1_reg := Mux((op1_iter_counter + 1.U) % 16.U === 0.U, 1.U, 0.U) + when(io.bankReadReq(0).fire) {} + } + + when(state === busy && io.ld_ex_o.ready && !wait2_reg) { + io.bankReadReq(1).valid := op2_iter_counter < iter + io.bankReadReq(1).bits.addr := op2_addr + op2_iter_counter + op2_iter_counter := Mux( + io.bankReadReq(1).ready, + op2_iter_counter + 1.U, + op2_iter_counter + ) + wait2_reg := Mux((op2_iter_counter + 1.U) % 16.U === 0.U, 1.U, 0.U) + when(io.bankReadReq(1).fire) {} + } + + when(wait1_reg) { + wait1_cnt := wait1_cnt + 1.U + when(wait1_cnt === 32.U) { + wait1_reg := false.B + wait1_cnt := 0.U + } + } + + when(wait2_reg) { + wait2_cnt := wait2_cnt + 1.U + when(wait2_cnt === 32.U) { + wait2_reg := false.B + wait2_cnt := 0.U + } + } + +// ----------------------------------------------------------------------------- +// SRAM returns data and passes to EX unit +// ----------------------------------------------------------------------------- + val both_valid = bankRespQueue0.io.deq.valid && bankRespQueue1.io.deq.valid + + io.ld_ex_o.valid := both_valid + when(both_valid) { + io.ld_ex_o.bits.op1 := bankRespQueue0.io.deq.bits.data + .asTypeOf(Vec(InputNum, UInt(inputWidth.W))) + io.ld_ex_o.bits.op2 := bankRespQueue1.io.deq.bits.data + .asTypeOf(Vec(InputNum, UInt(inputWidth.W))) + io.ld_ex_o.bits.iter := ld_ex_iter_reg + }.otherwise { + io.ld_ex_o.bits.iter := 0.U + io.ld_ex_o.bits.op1 := VecInit(Seq.fill(InputNum)(0.U(inputWidth.W))) + io.ld_ex_o.bits.op2 := VecInit(Seq.fill(InputNum)(0.U(inputWidth.W))) + } + + // Only dequeue and advance iter counter on successful handshake + bankRespQueue0.io.deq.ready := io.ld_ex_o.fire + bankRespQueue1.io.deq.ready := io.ld_ex_o.fire + + when(io.ld_ex_o.fire) { + ld_ex_iter_reg := ld_ex_iter_reg + 1.U + } + +// ----------------------------------------------------------------------------- +// Reset op1_iter_counter and return to idle state +// ----------------------------------------------------------------------------- + + when(state === busy && ld_ex_iter_reg === iter) { + state := idle + op1_iter_counter := 0.U + op2_iter_counter := 0.U + ld_ex_iter_reg := 0.U + } + +} diff --git a/examples/prefab/balls/vector/arch/VecMat16.scala b/examples/prefab/balls/vector/arch/VecMat16.scala new file mode 100644 index 000000000..6ecd095e9 --- /dev/null +++ b/examples/prefab/balls/vector/arch/VecMat16.scala @@ -0,0 +1,242 @@ +package examples.balls.vector + +import chisel3.experimental.hierarchy.public + +import chisel3.experimental.hierarchy.instantiable + +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import examples.balls.vector.configs.VectorBallParam + +import chisel3._ +import framework.acc.balldomain.ball.blink.{BallCommand, BallResponse} +import chisel3.util._ +import framework.acc.balldomain.ball.blink.BallStatus +import framework.acc.memdomain.{BankRead, BankWrite} + +@instantiable +class VecMat16( + frontendParam: FrontendParam, + memParam: MemDomainParam, + routingParam: BallDomainParam, + val inBW: Int, + val outBW: Int, + val inW: Int, + val accW: Int, + lane: Int, + val ballParam: VectorBallParam) + extends Module { + val commandDataBits = frontendParam.commandDataBits + + require(lane == 16, "VecMat16 requires lane=16") + require(inW == 8, "VecMat16 requires int8 inputs") + require(accW == 32, "VecMat16 requires int32 accumulators") + require(inBW >= 2, "VecMat16 requires two read ports") + require(outBW >= 4, "VecMat16 requires four write ports") + + @public + val io = IO(new Bundle { + + val cmdReq = Flipped(Decoupled(new BallCommand(frontendParam, memParam))) + + val cmdResp = Decoupled(new BallResponse(frontendParam)) + + val bankRead = Vec( + inBW, + Flipped(new BankRead( + frontendParam, + memParam + )) + ) + + val bankWrite = Vec( + outBW, + Flipped(new BankWrite( + frontendParam, + memParam + )) + ) + + val status = new BallStatus + }) + + val sIdle :: sLoadAcc :: sReadReq :: sReadResp :: sWrite :: sDone :: Nil = + Enum(6) + val state = RegInit(sIdle) + + val robIdReg = RegInit(0.U(log2Up(frontendParam.rob_entries).W)) + val op1Bank = RegInit(0.U(frontendParam.bank_id_len.W)) + val op2Bank = RegInit(0.U(frontendParam.bank_id_len.W)) + val wrBank = RegInit(0.U(frontendParam.bank_id_len.W)) + // Renamed bank holding the old C, which the instruction accumulates into. + val wrReadBank = RegInit(0.U(frontendParam.bank_id_len.W)) + // Group bases of each operand; the old C shares the destination layout. + val op1Group = RegInit(0.U(memParam.groupIdWidth.W)) + val op2Group = RegInit(0.U(memParam.groupIdWidth.W)) + val wrGroup = RegInit(0.U(memParam.groupIdWidth.W)) + val iterReg = RegInit(0.U(frontendParam.iter_len.W)) + val readRow = RegInit(0.U(frontendParam.iter_len.W)) + val aReqDone = RegInit(false.B) + val bReqDone = RegInit(false.B) + val aRespDone = RegInit(false.B) + val bRespDone = RegInit(false.B) + val aWord = RegInit(0.U(memParam.bankWidth.W)) + val bWord = RegInit(0.U(memParam.bankWidth.W)) + val acc = Reg(Vec(lane, Vec(lane, UInt(accW.W)))) + + val accLoad = Instantiate(new VecMat16AccLoad(frontendParam, memParam, routingParam, accW, lane, ballParam)) + + val store = Instantiate(new VecMat16Store(frontendParam, memParam, routingParam, accW, lane, ballParam)) + + accLoad.io.start := false.B + accLoad.io.wrBank := wrReadBank + accLoad.io.wrGroup := wrGroup + accLoad.io.robId := robIdReg + store.io.start := false.B + store.io.wrBank := wrBank + store.io.wrGroup := wrGroup + store.io.robId := robIdReg + store.io.acc := acc + for (p <- 0 until 2) { + accLoad.io.bankRead(p).io.req.ready := io.bankRead(p).io.req.ready + accLoad.io.bankRead(p).io.resp.valid := io.bankRead(p).io.resp.valid + accLoad.io.bankRead(p).io.resp.bits := io.bankRead(p).io.resp.bits + } + for (g <- 0 until 4) { + store.io.bankWrite(g).io.req.ready := io.bankWrite(g).io.req.ready + store.io.bankWrite(g).io.resp.valid := io.bankWrite(g).io.resp.valid + store.io.bankWrite(g).io.resp.bits := io.bankWrite(g).io.resp.bits + } + + io.cmdReq.ready := state === sIdle + io.cmdResp.valid := state === sDone + io.cmdResp.bits.rob_id := robIdReg + io.status.idle := state === sIdle + io.status.running := state =/= sIdle && state =/= sDone + VecMat16Tie.offRead(io.bankRead, robIdReg) + VecMat16Tie.offWrite(io.bankWrite, robIdReg, wrBank, wrGroup, memParam) + + switch(state) { + is(sIdle) { + when(io.cmdReq.fire) { + val cmd = io.cmdReq.bits + robIdReg := io.cmdReq.bits.rob_id + op1Bank := cmd.op1_bank + op2Bank := cmd.op2_bank + wrBank := cmd.wr_bank + wrReadBank := cmd.wr_read_bank + op1Group := cmd.op1_group + op2Group := cmd.op2_group + wrGroup := cmd.wr_group + iterReg := cmd.iter + readRow := 0.U + aReqDone := false.B + bReqDone := false.B + aRespDone := false.B + bRespDone := false.B + assert( + cmd.iter =/= 0.U && cmd.iter <= memParam.bankEntries.U, + "VecMat16 iter must be 1..bank depth" + ) + assert(cmd.rs2 === 0.U, "VecMat16 rs2 is reserved") + assert( + cmd.op1_col === 1.U && cmd.op2_col === 1.U && cmd.wr_col === 4.U, + "VecMat16 unsupported bank layout" + ) + accLoad.io.start := true.B + state := sLoadAcc + } + } + + is(sLoadAcc) { + for (p <- 0 until 2) { + VecMat16Tie.takeRead(io.bankRead(p), accLoad.io.bankRead(p)) + } + when(accLoad.io.done) { + acc := accLoad.io.acc + aReqDone := false.B + bReqDone := false.B + state := sReadReq + } + } + + is(sReadReq) { + io.bankRead(0).bank_id := op1Bank + io.bankRead(0).group_id := op1Group + io.bankRead(0).io.req.valid := !aReqDone + io.bankRead(0).io.req.bits.addr := readRow + io.bankRead(1).bank_id := op2Bank + io.bankRead(1).group_id := op2Group + io.bankRead(1).io.req.valid := !bReqDone + io.bankRead(1).io.req.bits.addr := readRow + when(io.bankRead(0).io.req.fire)(aReqDone := true.B) + when(io.bankRead(1).io.req.fire)(bReqDone := true.B) + when( + (aReqDone || io + .bankRead(0) + .io + .req + .fire) && (bReqDone || io.bankRead(1).io.req.fire) + ) { + aRespDone := false.B + bRespDone := false.B + state := sReadResp + } + } + + is(sReadResp) { + io.bankRead(0).bank_id := op1Bank + io.bankRead(0).group_id := op1Group + io.bankRead(1).bank_id := op2Bank + io.bankRead(1).group_id := op2Group + io.bankRead(0).io.resp.ready := !aRespDone + io.bankRead(1).io.resp.ready := !bRespDone + val aFire = io.bankRead(0).io.resp.fire + val bFire = io.bankRead(1).io.resp.fire + val nextA = Mux(aFire, io.bankRead(0).io.resp.bits.data, aWord) + val nextB = Mux(bFire, io.bankRead(1).io.resp.bits.data, bWord) + val haveA = aRespDone || aFire + val haveB = bRespDone || bFire + when(aFire)(aWord := io.bankRead(0).io.resp.bits.data) + when(bFire)(bWord := io.bankRead(1).io.resp.bits.data) + when(haveA)(aRespDone := true.B) + when(haveB)(bRespDone := true.B) + when(haveA && haveB) { + val aElems = nextA.asTypeOf(Vec(lane, UInt(inW.W))) + val bElems = nextB.asTypeOf(Vec(lane, UInt(inW.W))) + for { + i <- 0 until lane + j <- 0 until lane + } { + acc(i)(j) := (acc(i)(j).asSInt + aElems(i).asSInt * bElems( + j + ).asSInt).asUInt + } + when(readRow === iterReg - 1.U) { + store.io.start := true.B + state := sWrite + }.otherwise { + readRow := readRow + 1.U + aReqDone := false.B + bReqDone := false.B + state := sReadReq + } + } + } + + is(sWrite) { + for (g <- 0 until 4) { + VecMat16Tie.takeWrite(io.bankWrite(g), store.io.bankWrite(g)) + } + when(store.io.done)(state := sDone) + } + + is(sDone) { + when(io.cmdResp.fire)(state := sIdle) + } + } +} diff --git a/examples/prefab/balls/vector/arch/VecMat16AccLoad.scala b/examples/prefab/balls/vector/arch/VecMat16AccLoad.scala new file mode 100644 index 000000000..4603dc242 --- /dev/null +++ b/examples/prefab/balls/vector/arch/VecMat16AccLoad.scala @@ -0,0 +1,144 @@ +package examples.balls.vector + +import chisel3.experimental.hierarchy.{instantiable, public} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import examples.balls.vector.configs.VectorBallParam + +import chisel3._ +import chisel3.util._ +import framework.acc.memdomain.BankRead + +@instantiable +class VecMat16AccLoad( + frontendParam: FrontendParam, + memParam: MemDomainParam, + routingParam: BallDomainParam, + val accWidth: Int, + lane: Int, + val ballParam: VectorBallParam) + extends Module { + + @public + val io = IO(new Bundle { + val start = Input(Bool()) + val wrBank = Input(UInt(frontendParam.bank_id_len.W)) + val wrGroup = Input(UInt(memParam.groupIdWidth.W)) + val robId = Input(UInt(log2Up(frontendParam.rob_entries).W)) + + val bankRead = Vec( + 2, + Flipped(new BankRead( + frontendParam, + memParam + )) + ) + + val acc = Output(Vec(lane, Vec(lane, UInt(accWidth.W)))) + val done = Output(Bool()) + val busy = Output(Bool()) + }) + + val sIdle :: sReq :: sResp :: sFinish :: Nil = Enum(4) + val state = RegInit(sIdle) + + val loadRow = RegInit(0.U(log2Ceil(lane + 1).W)) + val loadPhase = RegInit(0.U(1.W)) + val reqDone = RegInit(VecInit(Seq.fill(2)(false.B))) + val respDone = RegInit(VecInit(Seq.fill(2)(false.B))) + val acc = Reg(Vec(lane, Vec(lane, UInt(accWidth.W)))) + + def loadGroup(port: Int): UInt = Cat(loadPhase, port.U(1.W)) + def clr2() = VecInit(Seq.fill(2)(false.B)) + + io.acc := acc + io.done := false.B + io.busy := state =/= sIdle + + for (i <- 0 until 2) { + io.bankRead(i).rob_id := io.robId + io.bankRead(i).bank_id := 0.U + io.bankRead(i).group_id := 0.U + io.bankRead(i).io.req.valid := false.B + io.bankRead(i).io.req.bits.addr := 0.U + io.bankRead(i).io.resp.ready := false.B + } + + switch(state) { + is(sIdle) { + when(io.start) { + loadRow := 0.U + loadPhase := 0.U + reqDone := clr2() + respDone := clr2() + state := sReq + } + } + + is(sReq) { + for (p <- 0 until 2) { + io.bankRead(p).bank_id := io.wrBank + io.bankRead(p).group_id := io.wrGroup + loadGroup(p) + io.bankRead(p).io.req.valid := !reqDone(p) + io.bankRead(p).io.req.bits.addr := loadRow + when(io.bankRead(p).io.req.fire) { + reqDone(p) := true.B + } + } + when( + (0 until 2) + .map(p => reqDone(p) || io.bankRead(p).io.req.fire) + .reduce(_ && _) + ) { + respDone := clr2() + state := sResp + } + } + + is(sResp) { + for (p <- 0 until 2) { + io.bankRead(p).bank_id := io.wrBank + io.bankRead(p).group_id := io.wrGroup + loadGroup(p) + io.bankRead(p).io.resp.ready := !respDone(p) + when(io.bankRead(p).io.resp.fire) { + val g = loadGroup(p) + val elems = + io.bankRead(p).io.resp.bits.data.asTypeOf(Vec(4, UInt(accWidth.W))) + val row = loadRow(log2Ceil(lane) - 1, 0) + for (i <- 0 until 4) { + acc(row)(g * 4.U + i.U) := elems(i) + } + respDone(p) := true.B + } + } + when( + (0 until 2) + .map(p => respDone(p) || io.bankRead(p).io.resp.fire) + .reduce(_ && _) + ) { + when(loadPhase === 0.U) { + loadPhase := 1.U + reqDone := clr2() + respDone := clr2() + state := sReq + }.elsewhen(loadRow === (lane - 1).U) { + state := sFinish + }.otherwise { + loadRow := loadRow + 1.U + loadPhase := 0.U + reqDone := clr2() + respDone := clr2() + state := sReq + } + } + } + + is(sFinish) { + io.done := true.B + state := sIdle + } + } +} diff --git a/examples/prefab/balls/vector/arch/VecMat16Store.scala b/examples/prefab/balls/vector/arch/VecMat16Store.scala new file mode 100644 index 000000000..2517f4dbe --- /dev/null +++ b/examples/prefab/balls/vector/arch/VecMat16Store.scala @@ -0,0 +1,136 @@ +package examples.balls.vector + +import chisel3.experimental.hierarchy.{instantiable, public} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import examples.balls.vector.configs.VectorBallParam + +import chisel3._ +import chisel3.util._ +import framework.acc.memdomain.BankWrite + +@instantiable +class VecMat16Store( + frontendParam: FrontendParam, + memParam: MemDomainParam, + routingParam: BallDomainParam, + val accWidth: Int, + lane: Int, + val ballParam: VectorBallParam) + extends Module { + + @public + val io = IO(new Bundle { + val start = Input(Bool()) + val wrBank = Input(UInt(frontendParam.bank_id_len.W)) + val wrGroup = Input(UInt(memParam.groupIdWidth.W)) + val robId = Input(UInt(log2Up(frontendParam.rob_entries).W)) + val acc = Input(Vec(lane, Vec(lane, UInt(accWidth.W)))) + + val bankWrite = Vec( + 4, + Flipped(new BankWrite( + frontendParam, + memParam + )) + ) + + val done = Output(Bool()) + val busy = Output(Bool()) + }) + + val sIdle :: sReq :: sResp :: Nil = Enum(3) + val state = RegInit(sIdle) + + val writeRow = RegInit(0.U(log2Ceil(lane + 1).W)) + val writeIssued = RegInit(VecInit(Seq.fill(4)(false.B))) + val writeAcked = RegInit(VecInit(Seq.fill(4)(false.B))) + + def writeData(group: Int): UInt = { + val row = writeRow(log2Ceil(lane) - 1, 0) + val elems = (0 until 4).map(i => io.acc(row)(group * 4 + i)) + Cat(elems.reverse) + } + + def clr4() = VecInit(Seq.fill(4)(false.B)) + + io.done := false.B + io.busy := state =/= sIdle + + for (i <- 0 until 4) { + io.bankWrite(i).rob_id := io.robId + io.bankWrite(i).bank_id := io.wrBank + io.bankWrite(i).group_id := io.wrGroup + i.U + io.bankWrite(i).io.req.valid := false.B + io.bankWrite(i).io.req.bits.addr := 0.U + io.bankWrite(i).io.req.bits.data := 0.U + io.bankWrite(i).io.req.bits.mask := VecInit( + Seq.fill(memParam.bankMaskLen)(false.B) + ) + io.bankWrite(i).io.resp.ready := false.B + } + + switch(state) { + is(sIdle) { + when(io.start) { + writeRow := 0.U + writeIssued := clr4() + writeAcked := clr4() + state := sReq + } + } + + is(sReq) { + for (g <- 0 until 4) { + io.bankWrite(g).bank_id := io.wrBank + io.bankWrite(g).group_id := io.wrGroup + g.U + io.bankWrite(g).io.req.valid := !writeIssued(g) + io.bankWrite(g).io.req.bits.addr := writeRow + io.bankWrite(g).io.req.bits.data := writeData(g) + io.bankWrite(g).io.req.bits.mask := VecInit( + Seq.fill(memParam.bankMaskLen)(true.B) + ) + when(io.bankWrite(g).io.req.fire) { + writeIssued(g) := true.B + } + } + when( + (0 until 4) + .map(g => writeIssued(g) || io.bankWrite(g).io.req.fire) + .reduce(_ && _) + ) { + writeAcked := clr4() + state := sResp + } + } + + is(sResp) { + for (g <- 0 until 4) { + io.bankWrite(g).bank_id := io.wrBank + io.bankWrite(g).group_id := io.wrGroup + g.U + io.bankWrite(g).io.resp.ready := !writeAcked(g) + when(io.bankWrite(g).io.resp.fire) { + writeAcked(g) := true.B + } + } + when( + (0 until 4) + .map(g => writeAcked(g) || io.bankWrite(g).io.resp.fire) + .reduce(_ && _) + ) { + when(writeRow === (lane - 1).U) { + io.done := true.B + state := sIdle + }.otherwise { + writeRow := writeRow + 1.U + writeIssued := clr4() + writeAcked := clr4() + state := sReq + } + } + } + } +} diff --git a/examples/balls/vector/arch/src/main/scala/VecMat16Tie.scala b/examples/prefab/balls/vector/arch/VecMat16Tie.scala similarity index 75% rename from examples/balls/vector/arch/src/main/scala/VecMat16Tie.scala rename to examples/prefab/balls/vector/arch/VecMat16Tie.scala index a4d0f4dae..b88c6467c 100644 --- a/examples/balls/vector/arch/src/main/scala/VecMat16Tie.scala +++ b/examples/prefab/balls/vector/arch/VecMat16Tie.scala @@ -1,16 +1,16 @@ package examples.balls.vector +import framework.acc.memdomain.configs.MemDomainParam + import chisel3._ import chisel3.util._ -import framework.balldomain.blink.{BankRead, BankWrite} -import framework.top.GlobalConfig +import framework.acc.memdomain.{BankRead, BankWrite} object VecMat16Tie { def offRead(ports: Vec[BankRead], robId: UInt): Unit = { for (p <- ports) { p.rob_id := robId - p.ball_id := 0.U p.bank_id := 0.U p.group_id := 0.U p.io.req.valid := false.B @@ -20,27 +20,26 @@ object VecMat16Tie { } def offWrite( - ports: Vec[BankWrite], - robId: UInt, - wrBank: UInt, - b: GlobalConfig + ports: Vec[BankWrite], + robId: UInt, + wrBank: UInt, + wrGroup: UInt, + memParam: MemDomainParam ): Unit = { for ((p, i) <- ports.zipWithIndex) { p.rob_id := robId - p.ball_id := 0.U p.bank_id := wrBank - p.group_id := i.U + p.group_id := wrGroup + i.U p.io.req.valid := false.B p.io.req.bits.addr := 0.U p.io.req.bits.data := 0.U - p.io.req.bits.mask := VecInit(Seq.fill(b.memDomain.bankMaskLen)(false.B)) + p.io.req.bits.mask := VecInit(Seq.fill(memParam.bankMaskLen)(false.B)) p.io.resp.ready := false.B } } def takeRead(parent: BankRead, child: BankRead): Unit = { parent.rob_id := child.rob_id - parent.ball_id := child.ball_id parent.bank_id := child.bank_id parent.group_id := child.group_id parent.io.req.valid := child.io.req.valid @@ -50,7 +49,6 @@ object VecMat16Tie { def takeWrite(parent: BankWrite, child: BankWrite): Unit = { parent.rob_id := child.rob_id - parent.ball_id := child.ball_id parent.bank_id := child.bank_id parent.group_id := child.group_id parent.io.req.valid := child.io.req.valid diff --git a/examples/prefab/balls/vector/arch/VecStoreUnit.scala b/examples/prefab/balls/vector/arch/VecStoreUnit.scala new file mode 100644 index 000000000..a7d14ed26 --- /dev/null +++ b/examples/prefab/balls/vector/arch/VecStoreUnit.scala @@ -0,0 +1,161 @@ +package examples.balls.vector + +import framework.acc.balldomain.configs.BallRouteParam + +import framework.lib.queue.Queue +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import framework.ip.memory.bank.{SramReadIO, SramReadReq, SramReadResp, SramWriteIO, SramWriteReq, SramWriteResp} + +import chisel3._ +import chisel3.util._ +import chisel3.stage._ +import chisel3.experimental.hierarchy.{instantiable, public} +import examples.balls.vector.configs.VectorBallParam + +class ctrl_st_req(frontendParam: FrontendParam, memParam: MemDomainParam, val ballParam: VectorBallParam) + extends Bundle { + val wr_bank = UInt(frontendParam.bank_id_len.W) + val wr_bank_addr = UInt(log2Up(memParam.bankEntries).W) + val iter = UInt(frontendParam.iter_len.W) +} + +class ex_st_req(frontendParam: FrontendParam, val ballParam: VectorBallParam) extends Bundle { + val config = ballParam + val InputNum = config.lane + val accWidth = config.outputWidth + val rst = Vec(InputNum, UInt(accWidth.W)) + val iter = UInt(frontendParam.iter_len.W) +} + +class BankWriteEntry(memParam: MemDomainParam, val ballParam: VectorBallParam) extends Bundle { + val addr = UInt(log2Ceil(memParam.bankEntries).W) + val data = UInt(memParam.bankWidth.W) + val mask = Vec(memParam.bankMaskLen, Bool()) +} + +@instantiable +class VecStoreUnit( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val routingParam: BallDomainParam, + val instance: BallRouteParam, + val ballParam: VectorBallParam) + extends Module { + val config = ballParam + val InputNum = config.lane + val accWidth = config.outputWidth + + // Use this instance's configured bandwidth. + val ballMapping = instance + + val outBW = ballMapping.outBW + + @public + val io = IO(new Bundle { + val ctrl_st_i = Flipped(Decoupled(new ctrl_st_req(frontendParam, memParam, ballParam))) + val ex_st_i = Flipped(Decoupled(new ex_st_req(frontendParam, ballParam))) + val bankWrite = Vec(outBW, Flipped(new SramWriteIO(memParam))) + val wr_bank_o = Output(UInt(frontendParam.bank_id_len.W)) + val cmdResp_o = Valid(new Bundle { val commit = Bool() }) + }) + + val wr_bank = RegInit(0.U(frontendParam.bank_id_len.W)) + val wr_bank_addr = RegInit(0.U(log2Up(memParam.bankEntries).W)) + val iter = RegInit(0.U(frontendParam.iter_len.W)) + val iter_counter = RegInit(0.U(frontendParam.iter_len.W)) + val idle :: busy :: Nil = Enum(2) + val state = RegInit(idle) + + val writeQueues = VecInit( + Seq.fill(outBW)( + Instantiate(new Queue(new BankWriteEntry(memParam, ballParam), 16, false, false)).io + ) + ) + +// ----------------------------------------------------------------------------- +// Set registers when Ctrl instruction arrives +// ----------------------------------------------------------------------------- + io.ctrl_st_i.ready := state === idle + + when(io.ctrl_st_i.fire) { + wr_bank := io.ctrl_st_i.bits.wr_bank + wr_bank_addr := io.ctrl_st_i.bits.wr_bank_addr + iter := (io.ctrl_st_i.bits.iter + 15.U) & (~15.U(frontendParam.iter_len.W)) + iter_counter := 0.U + state := busy + } + +// ----------------------------------------------------------------------------- +// Accept computation results from EX unit and push to write queues +// ----------------------------------------------------------------------------- + io.ex_st_i.ready := state === busy && writeQueues.forall(_.enq.ready) + + for (i <- 0 until outBW) { + writeQueues(i).enq.valid := false.B + writeQueues(i).enq.bits := DontCare + } + + when(io.ex_st_i.fire) { + for (i <- 0 until outBW) { + val elementsPerChannel = InputNum / outBW + val startIdx = i * elementsPerChannel + val endIdx = startIdx + elementsPerChannel - 1 + + val entry = Wire(new BankWriteEntry(memParam, ballParam)) + entry.addr := wr_bank_addr + iter_counter(log2Ceil(InputNum) - 1, 0) + entry.data := Cat(io.ex_st_i.bits.rst.slice(startIdx, endIdx + 1).reverse) + entry.mask := VecInit(Seq.fill(memParam.bankMaskLen)(true.B)) + + writeQueues(i).enq.valid := true.B + writeQueues(i).enq.bits := entry + } + iter_counter := iter_counter + 1.U + } + +// ----------------------------------------------------------------------------- +// Drain write queues to bankWrite interface +// ----------------------------------------------------------------------------- + io.bankWrite.foreach { acc => + acc.req.valid := false.B + acc.req.bits.addr := 0.U + acc.req.bits.data := Cat(Seq.fill(accWidth / 8)(0.U(8.W))) + acc.req.bits.mask := VecInit(Seq.fill(memParam.bankMaskLen)(false.B)) + acc.resp.ready := false.B + } + + for (i <- 0 until outBW) { + writeQueues(i).deq.ready := false.B + + when(writeQueues(i).deq.valid) { + io.bankWrite(i).req.valid := true.B + io.bankWrite(i).req.bits.addr := writeQueues(i).deq.bits.addr + io.bankWrite(i).req.bits.data := writeQueues(i).deq.bits.data + io.bankWrite(i).req.bits.mask := writeQueues(i).deq.bits.mask + writeQueues(i).deq.ready := io.bankWrite(i).req.ready + } + } + + // Output wr_bank for bank_id setting + io.wr_bank_o := wr_bank + +// ----------------------------------------------------------------------------- +// Reset iter counter, commit cmdResp, return to idle state +// ----------------------------------------------------------------------------- + val allQueuesEmpty = writeQueues.forall(q => !q.deq.valid) + val allDataEnqueued = state === busy && iter_counter >= iter + + when(allDataEnqueued && allQueuesEmpty) { + state := idle + io.cmdResp_o.valid := true.B + io.cmdResp_o.bits.commit := true.B + }.otherwise { + io.cmdResp_o.valid := false.B + io.cmdResp_o.bits.commit := false.B + } + +} diff --git a/examples/prefab/balls/vector/arch/VecUnit.scala b/examples/prefab/balls/vector/arch/VecUnit.scala new file mode 100644 index 000000000..cdd358197 --- /dev/null +++ b/examples/prefab/balls/vector/arch/VecUnit.scala @@ -0,0 +1,79 @@ +package examples.balls.vector + +import framework.acc.balldomain.configs.BallRouteParam + +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import framework.acc.balldomain.ball.blink.{BallCommand, BallResponse} +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} + +import framework.acc.balldomain.ball.blink.BallStatus +import framework.acc.memdomain.{BankRead, BankWrite} +import examples.balls.vector.configs.VectorBallParam + +@instantiable +class VecUnit( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val routingParam: BallDomainParam, + val instance: BallRouteParam, + val ballParam: VectorBallParam) + extends Module { + val commandDataBits = frontendParam.commandDataBits + val cfg = ballParam + val lane = cfg.lane + val inW = cfg.inputWidth + val accW = cfg.outputWidth + + require(lane == 16, "VecUnit vecmat16 requires lane=16") + require(inW == 8, "VecUnit vecmat16 requires int8 inputs") + require(accW == 32, "VecUnit vecmat16 requires int32 accumulators") + + val map = instance + + val inBW = map.inBW + val outBW = map.outBW + require(inBW >= 2, "VecUnit vecmat16 requires two read ports") + require(outBW >= 4, "VecUnit vecmat16 requires four write ports") + + @public + val io = IO(new Bundle { + + val cmdReq = Flipped(Decoupled(new BallCommand(frontendParam, memParam))) + + val cmdResp = Decoupled(new BallResponse(frontendParam)) + + val bankRead = Vec( + inBW, + Flipped(new BankRead( + frontendParam, + memParam + )) + ) + + val bankWrite = Vec( + outBW, + Flipped(new BankWrite( + frontendParam, + memParam + )) + ) + + val status = new BallStatus + }) + + val thread = + Instantiate(new VecMat16(frontendParam, memParam, routingParam, inBW, outBW, inW, accW, lane, ballParam)) + + thread.io.cmdReq <> io.cmdReq + thread.io.cmdResp <> io.cmdResp + for (i <- 0 until inBW) { thread.io.bankRead(i) <> io.bankRead(i) } + for (i <- 0 until outBW) { thread.io.bankWrite(i) <> io.bankWrite(i) } + io.status <> thread.io.status +} diff --git a/examples/balls/vector/arch/src/main/scala/bond/BondWrapper.scala b/examples/prefab/balls/vector/arch/bond/BondWrapper.scala similarity index 100% rename from examples/balls/vector/arch/src/main/scala/bond/BondWrapper.scala rename to examples/prefab/balls/vector/arch/bond/BondWrapper.scala diff --git a/examples/balls/vector/arch/src/main/scala/bond/README.md b/examples/prefab/balls/vector/arch/bond/README.md similarity index 100% rename from examples/balls/vector/arch/src/main/scala/bond/README.md rename to examples/prefab/balls/vector/arch/bond/README.md diff --git a/examples/prefab/balls/vector/arch/bond/vvv.scala b/examples/prefab/balls/vector/arch/bond/vvv.scala new file mode 100644 index 000000000..111cfd174 --- /dev/null +++ b/examples/prefab/balls/vector/arch/bond/vvv.scala @@ -0,0 +1,27 @@ +//===----------------------------------------------------------------------===// +// VVV Bond: +// Input: Vec, Vec +// Output: Vec +//===----------------------------------------------------------------------===// + +package examples.balls.vector.bond + +import chisel3._ +import chisel3.util._ +import examples.balls.vector.configs.VectorBallParam + +class VVV(val inputWidth: Int, val outputWidth: Int, val ballParam: VectorBallParam) extends Bundle { + val config = ballParam + + val lane = config.lane + + val in = Flipped(Decoupled(new Bundle { + val in1 = Vec(lane, UInt(inputWidth.W)) + val in2 = Vec(lane, UInt(inputWidth.W)) + })) + + val out = Decoupled(new Bundle { + val out = Vec(lane, UInt(outputWidth.W)) + }) + +} diff --git a/examples/balls/vector/arch/src/main/scala/op/README.md b/examples/prefab/balls/vector/arch/op/README.md similarity index 100% rename from examples/balls/vector/arch/src/main/scala/op/README.md rename to examples/prefab/balls/vector/arch/op/README.md diff --git a/examples/balls/vector/arch/src/main/scala/op/cascade.scala b/examples/prefab/balls/vector/arch/op/cascade.scala similarity index 79% rename from examples/balls/vector/arch/src/main/scala/op/cascade.scala rename to examples/prefab/balls/vector/arch/op/cascade.scala index 46ff3dcbf..1d67185f5 100644 --- a/examples/balls/vector/arch/src/main/scala/op/cascade.scala +++ b/examples/prefab/balls/vector/arch/op/cascade.scala @@ -1,16 +1,26 @@ package examples.balls.vector.op +import chisel3.experimental.hierarchy.{instantiable, public} + import chisel3._ import chisel3.util._ import examples.balls.vector.configs.VectorBallParam import examples.balls.vector.bond.VVV -class CascadeOp(val config: VectorBallParam) extends Module { +@instantiable +class CascadeOp(val ballParam: VectorBallParam) extends Module { + val config = ballParam + val lane = config.lane val inputWidth = config.outputWidth // Cascade uses outputWidth as input val outputWidth = config.outputWidth - val io = IO(new VVV(config, inputWidth, outputWidth)) + @public + val io = IO(new VVV( + inputWidth, + outputWidth, + ballParam + )) val reg1 = RegInit(VecInit(Seq.fill(lane)(0.U(outputWidth.W)))) val reg2 = RegInit(VecInit(Seq.fill(lane)(0.U(outputWidth.W)))) diff --git a/examples/balls/vector/arch/src/main/scala/op/mul.scala b/examples/prefab/balls/vector/arch/op/mul.scala similarity index 78% rename from examples/balls/vector/arch/src/main/scala/op/mul.scala rename to examples/prefab/balls/vector/arch/op/mul.scala index cde3e0c6c..69c8237a6 100644 --- a/examples/balls/vector/arch/src/main/scala/op/mul.scala +++ b/examples/prefab/balls/vector/arch/op/mul.scala @@ -1,16 +1,27 @@ package examples.balls.vector.op +import chisel3.experimental.hierarchy.{instantiable, public} + import chisel3._ import chisel3.util._ import examples.balls.vector.configs.VectorBallParam import examples.balls.vector.bond.VVV -class MulOp(val config: VectorBallParam) extends Module { +@instantiable +class MulOp(val ballParam: VectorBallParam) extends Module { + val config = ballParam + val lane = config.lane val inputWidth = config.inputWidth val outputWidth = config.outputWidth - val io = IO(new VVV(config, inputWidth, outputWidth)) + @public + val io = IO(new VVV( + inputWidth, + outputWidth, + ballParam + )) + val reg1 = RegInit(VecInit(Seq.fill(lane)(0.U(inputWidth.W)))) val reg2 = RegInit(VecInit(Seq.fill(lane)(0.U(inputWidth.W)))) val cnt = RegInit(0.U(log2Ceil(lane).W)) diff --git a/examples/prefab/balls/vector/arch/thread/BaseThread.scala b/examples/prefab/balls/vector/arch/thread/BaseThread.scala new file mode 100644 index 000000000..9d29fc315 --- /dev/null +++ b/examples/prefab/balls/vector/arch/thread/BaseThread.scala @@ -0,0 +1,19 @@ +//===- BaseThread.scala - Level 1: Thread ---===// +package examples.balls.vector.thread + +import chisel3.experimental.hierarchy.{instantiable, public} + +import chisel3._ +import examples.balls.vector.configs.VectorBallParam + +//===----------------------------------------------------------------------===// +// BaseThread base class +//===----------------------------------------------------------------------===// +@instantiable +class BaseThread(val opType: String, ballParam: VectorBallParam) extends Module { + val config = ballParam + + @public + val io = IO(new Bundle {}) + val lane = config.lane +} diff --git a/examples/prefab/balls/vector/arch/thread/CasThread.scala b/examples/prefab/balls/vector/arch/thread/CasThread.scala new file mode 100644 index 000000000..429f91df8 --- /dev/null +++ b/examples/prefab/balls/vector/arch/thread/CasThread.scala @@ -0,0 +1,35 @@ +package examples.balls.vector.thread + +import chisel3.experimental.hierarchy.public + +import chisel3.experimental.hierarchy.instantiable + +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import chisel3._ +import chisel3.util._ +import examples.balls.vector.configs.VectorBallParam +import examples.balls.vector.bond.VVV +import examples.balls.vector.op.CascadeOp + +@instantiable +class CasThread(val ballParam: VectorBallParam) + extends BaseThread( + "cascade", + ballParam + ) { + val cascadeOp = Instantiate(new CascadeOp(ballParam)) + + @public + val vvvBond = IO( + new VVV( + this.config.outputWidth, + this.config.outputWidth, + ballParam + ) + ) + + // Connect CascadeOp and VVVBond + cascadeOp.io.in <> vvvBond.in + cascadeOp.io.out <> vvvBond.out +} diff --git a/examples/prefab/balls/vector/arch/thread/MulThread.scala b/examples/prefab/balls/vector/arch/thread/MulThread.scala new file mode 100644 index 000000000..ad065f818 --- /dev/null +++ b/examples/prefab/balls/vector/arch/thread/MulThread.scala @@ -0,0 +1,35 @@ +package examples.balls.vector.thread + +import chisel3.experimental.hierarchy.public + +import chisel3.experimental.hierarchy.instantiable + +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import chisel3._ +import chisel3.util._ +import examples.balls.vector.configs.VectorBallParam +import examples.balls.vector.bond.VVV +import examples.balls.vector.op.MulOp + +@instantiable +class MulThread(val ballParam: VectorBallParam) + extends BaseThread( + "mul", + ballParam + ) { + val mulOp = Instantiate(new MulOp(ballParam)) + + @public + val vvvBond = IO( + new VVV( + this.config.inputWidth, + this.config.outputWidth, + ballParam + ) + ) + + // Connect MulOp and VVVBond + mulOp.io.in <> vvvBond.in + mulOp.io.out <> vvvBond.out +} diff --git a/examples/balls/vector/arch/src/main/scala/thread/README.md b/examples/prefab/balls/vector/arch/thread/README.md similarity index 100% rename from examples/balls/vector/arch/src/main/scala/thread/README.md rename to examples/prefab/balls/vector/arch/thread/README.md diff --git a/examples/balls/vector/arch/src/main/scala/warp/MeshWarp.scala b/examples/prefab/balls/vector/arch/warp/MeshWarp.scala similarity index 80% rename from examples/balls/vector/arch/src/main/scala/warp/MeshWarp.scala rename to examples/prefab/balls/vector/arch/warp/MeshWarp.scala index 13ebaf4d9..605a70b14 100644 --- a/examples/balls/vector/arch/src/main/scala/warp/MeshWarp.scala +++ b/examples/prefab/balls/vector/arch/warp/MeshWarp.scala @@ -1,34 +1,50 @@ package examples.balls.vector.warp +import chisel3.experimental.hierarchy.public + +import chisel3.experimental.hierarchy.instantiable + +import chisel3.experimental.hierarchy.{Instance, Instantiate} + import chisel3._ import chisel3.util._ import chisel3.stage._ import examples.balls.vector.configs.VectorBallParam import examples.balls.vector.thread._ -class MeshWarpInput(val config: VectorBallParam) extends Bundle { +class MeshWarpInput(val ballParam: VectorBallParam) extends Bundle { + val config = ballParam + val op1 = Vec(config.lane, UInt(config.inputWidth.W)) val op2 = Vec(config.lane, UInt(config.inputWidth.W)) val thread_id = UInt(10.W) } -class MeshWarpOutput(val config: VectorBallParam) extends Bundle { +class MeshWarpOutput(val ballParam: VectorBallParam) extends Bundle { + val config = ballParam + val res = Vec(config.lane, UInt(config.outputWidth.W)) } -class MeshWarp(val config: VectorBallParam) extends Module { +@instantiable +class MeshWarp(val ballParam: VectorBallParam) extends Module { + val config = ballParam + @public val io = IO(new Bundle { - val in = Flipped(Decoupled(new MeshWarpInput(config))) - val out = Decoupled(new MeshWarpOutput(config)) + + val in = Flipped(Decoupled(new MeshWarpInput(ballParam))) + + val out = Decoupled(new MeshWarpOutput(ballParam)) + }) val mulThreads = (0 until config.numMulThreads).map { i => - Module(new MulThread(config)) + Instantiate(new MulThread(ballParam)) } val casThreads = (0 until config.numCasThreads).map { i => - Module(new CasThread(config)) + Instantiate(new CasThread(ballParam)) } io.in.ready := mulThreads(0).vvvBond.in.ready diff --git a/examples/balls/vector/arch/src/main/scala/warp/README.md b/examples/prefab/balls/vector/arch/warp/README.md similarity index 100% rename from examples/balls/vector/arch/src/main/scala/warp/README.md rename to examples/prefab/balls/vector/arch/warp/README.md diff --git a/examples/prefab/balls/vector/configs/VectorBallParam.scala b/examples/prefab/balls/vector/configs/VectorBallParam.scala new file mode 100644 index 000000000..1b95c2f42 --- /dev/null +++ b/examples/prefab/balls/vector/configs/VectorBallParam.scala @@ -0,0 +1,31 @@ +package examples.balls.vector.configs + +import buckyball.balls.vector.{Params => ParametersProto} +import upickle.default.{macroRW, ReadWriter} + +case class VectorBallParam( + InputNum: Int, + inputWidth: Int, + lane: Int, + outputWidth: Int, + numMulThreads: Int, + numCasThreads: Int) + +object VectorBallParam { + + def fromProto(source: ParametersProto): VectorBallParam = { + require( + source.hasInputNum && source.hasInputWidth && source.hasLane && source.hasOutputWidth && source.hasNumMulThreads && source.hasNumCasThreads + ) + VectorBallParam( + source.getInputNum, + source.getInputWidth, + source.getLane, + source.getOutputWidth, + source.getNumMulThreads, + source.getNumCasThreads + ) + } + + implicit val rw: ReadWriter[VectorBallParam] = macroRW +} diff --git a/examples/prefab/balls/vector/configs/default.toml b/examples/prefab/balls/vector/configs/default.toml new file mode 100644 index 000000000..b83bbd5d9 --- /dev/null +++ b/examples/prefab/balls/vector/configs/default.toml @@ -0,0 +1,8 @@ +# Vector Ball Configuration + +InputNum = 16 +inputWidth = 8 +lane = 16 +outputWidth = 32 +numMulThreads = 16 +numCasThreads = 16 diff --git a/examples/prefab/balls/vector/configs/vector_params.proto b/examples/prefab/balls/vector/configs/vector_params.proto new file mode 100644 index 000000000..94cd9c9a8 --- /dev/null +++ b/examples/prefab/balls/vector/configs/vector_params.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; +package buckyball.balls.vector; +option java_package = "buckyball.balls.vector"; +option java_multiple_files = true; + +message Params { + optional int32 InputNum = 1; + optional int32 inputWidth = 2; + optional int32 lane = 3; + optional int32 outputWidth = 4; + optional int32 numMulThreads = 5; + optional int32 numCasThreads = 6; +} diff --git a/examples/prefab/balls/vector/emu/src/64_vecmat16.rs b/examples/prefab/balls/vector/emu/src/64_vecmat16.rs new file mode 100644 index 000000000..cfd4264e8 --- /dev/null +++ b/examples/prefab/balls/vector/emu/src/64_vecmat16.rs @@ -0,0 +1,70 @@ +//===- 64_vecmat16.rs - VECMAT16 instruction ------------------------------===// +// +// C += A^T * B: op1 holds A^T and op2 holds B, iter rows of sixteen int8 each; +// the four-group destination holds C, sixteen rows of sixteen int32, and is +// read as the old value (rs1[29]). rs2 is reserved and zero. +// +//===----------------------------------------------------------------------===// + +use super::bank_matrix::{read_i32_nn_groups, read_i8_k_rows, write_i32_nn_groups}; +use super::decode::{pbank, pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old}; +use super::instruction::{BallModel, ExecContext}; + +const WARP_M: usize = 16; +const WARP_N: usize = 16; + +pub struct VecMat16; + +impl BallModel for VecMat16 { + fn exec(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + assert!(rs1_read_old(xs1), "vecmat16: accumulates into the old destination; rs1[29] must be set"); + assert!(xs2 == 0, "vecmat16: rs2 is reserved and must be zero"); + let op1 = rs1_b0(xs1); + let op2 = rs1_b1(xs1); + let wr = rs1_b2(xs1); + let iter = rs1_iter(xs1); + + let c1 = ctx.config(op1).cols; + let c2 = ctx.config(op2).cols; + let cw = ctx.config(wr).cols; + if c1 != 1 || c2 != 1 || cw != 4 { + panic!("vecmat16: unsupported bank layout op1_cols={c1} op2_cols={c2} wr_cols={cw}"); + } + + let p1 = pbank(ctx, op1); + let p2 = pbank(ctx, op2); + let pw: Vec<_> = (0..cw).map(|group| pbank_group(ctx, wr, group)).collect(); + let kin = iter as usize; + + if kin == 0 { + panic!("vecmat16: iter must be > 0"); + } + + let need = kin * WARP_N; + if need > ctx.banks[p1].len() || need > ctx.banks[p2].len() { + panic!("vecmat16: iter too large for bank"); + } + + let a_t = read_i8_k_rows(&ctx.banks, p1, kin, WARP_N); + let b = read_i8_k_rows(&ctx.banks, p2, kin, WARP_N); + let mut c = read_i32_nn_groups(&ctx.banks, &pw, 16); + + for i in 0..WARP_M { + for j in 0..WARP_N { + let mut acc = c[i][j]; + for t in 0..kin { + acc = acc.wrapping_add((a_t[t][i] as i32).wrapping_mul(b[t][j] as i32)); + } + c[i][j] = acc; + } + } + + write_i32_nn_groups(&mut ctx.banks, &pw, &c, 16); + 0 + } + + fn latency(xs1: u64, _xs2: u64) -> u64 { + let kin = rs1_iter(xs1).max(1); + kin.saturating_mul(16) + } +} diff --git a/examples/prefab/balls/vector/emu/src/dispatch.rs b/examples/prefab/balls/vector/emu/src/dispatch.rs new file mode 100644 index 000000000..3754eba43 --- /dev/null +++ b/examples/prefab/balls/vector/emu/src/dispatch.rs @@ -0,0 +1,31 @@ +use bebop_bemu::ball::{BallModel, ExecContext}; + +use super::f64_vecmat16; + +const BALL_CLASS: &str = "examples.balls.vector.VecBall"; + +pub fn execute_known( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, + ctx: &mut ExecContext, +) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("VECMAT16") => Some(f64_vecmat16::VecMat16::exec(xs1, xs2, ctx)), + Some(_) | None => None, + } +} + +pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("VECMAT16") => Some(f64_vecmat16::VecMat16::latency(xs1, xs2)), + Some(_) | None => None, + } +} diff --git a/examples/prefab/balls/vector/emu/src/lib.rs b/examples/prefab/balls/vector/emu/src/lib.rs new file mode 100644 index 000000000..30fdb8605 --- /dev/null +++ b/examples/prefab/balls/vector/emu/src/lib.rs @@ -0,0 +1,12 @@ +pub(crate) use bebop_bemu::{thread::{instructions::decode, bank_matrix}, ball as instruction}; + +#[path = "64_vecmat16.rs"] +mod f64_vecmat16; + +mod dispatch; + +pub use dispatch::{cycles_after_issue, execute_known}; + +#[bebop_bemu::ball::linkme::distributed_slice(bebop_bemu::BALLS)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static BALL: bebop_bemu::Ball = bebop_bemu::Ball { execute: execute_known, cycles: cycles_after_issue }; diff --git a/examples/prefab/balls/vector/stack/src/vectorOps.cpp b/examples/prefab/balls/vector/stack/src/vectorOps.cpp new file mode 100644 index 000000000..8d38f899d --- /dev/null +++ b/examples/prefab/balls/vector/stack/src/vectorOps.cpp @@ -0,0 +1,68 @@ +//===- vectorOps.cpp - VecBall operations ---------------------------------===// +// +// The dialect of VecBall, its virtual operations building their physical ones +// and the instructions of the physical operations. +// +//===----------------------------------------------------------------------===// + +#include "vectorOps.h" + +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::vector_ball; +using mlir::buddy::bankFields; +using mlir::buddy::constantI64; +using mlir::buddy::emitInstruction; +using mlir::buddy::field; + +#include "vectorDialect.cpp.inc" + +#define GET_OP_CLASSES +#include "vectorOps.cpp.inc" + +void VecBallDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "vectorOps.cpp.inc" + >(); +} + +namespace { +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallDialect( + [](DialectRegistry ®istry) { registry.insert(); }), + true); +} // namespace + +SmallVector BankMatmulAtI8Op::getSourceBanks() { + return {getAt(), getB()}; +} +Value BankMatmulAtI8Op::getDestinationBank() { return getC(); } +bool BankMatmulAtI8Op::readsDestination() { return true; } +Operation *BankMatmulAtI8Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return MatmulAtI8Op::create(builder, getLoc(), sources[0], sources[1], + destination, getRows()); +} + +SmallVector MatmulAtI8Op::getSourceBanks() { + return {int64_t(getAt()), int64_t(getB())}; +} +int64_t MatmulAtI8Op::getDestinationBank() { return getC(); } +LogicalResult MatmulAtI8Op::lowerToInstructions(OpBuilder &builder) { + Location loc = getLoc(); + int64_t rows = getRows(), at = getAt(), b = getB(), c = getC(); + const auto &target = buckyball_target::getBuckyballTarget(); + if (target.bankWidthBits != 128 || rows <= 0 || rows > target.bankDepth) + return emitOpError("matmul_at_i8 rows exceed the bank"); + if (at == c || b == c) + return emitOpError("matmul_at_i8 requires C apart from A^T and B"); + emitInstruction(builder, loc, + buckyball_target::getBuckyballFunct7("VECMAT16"), + constantI64(builder, loc, bankFields({at, b}, c, true, rows)), + constantI64(builder, loc, 0)); + return success(); +} diff --git a/examples/prefab/balls/vector/stack/src/vectorOps.h b/examples/prefab/balls/vector/stack/src/vectorOps.h new file mode 100644 index 000000000..4aff1c5e6 --- /dev/null +++ b/examples/prefab/balls/vector/stack/src/vectorOps.h @@ -0,0 +1,20 @@ +//===- vectorOps.h - VecBall operations -----------------------------------===// + +#ifndef VECTOR_OPS_H +#define VECTOR_OPS_H + +#include "Interfaces/BankInterfaces.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "vectorDialect.h.inc" + +#define GET_OP_CLASSES +#include "vectorOps.h.inc" + +#endif // VECTOR_OPS_H diff --git a/examples/prefab/balls/vector/stack/src/vectorOps.td b/examples/prefab/balls/vector/stack/src/vectorOps.td new file mode 100644 index 000000000..908dacce1 --- /dev/null +++ b/examples/prefab/balls/vector/stack/src/vectorOps.td @@ -0,0 +1,47 @@ +//===- vectorOps.td - VecBall operations -------------------*- tablegen -*-===// +// +// matmul_at_i8 (at, b) -> c: c += at^T * b, one VECMAT16 instruction; it +// always reads the old c. Banks are one group of sixteen i8 per row unless +// noted: +// at: A^T, `rows` rows; row k is column k of the 16 x rows matrix A. +// b: `rows` rows of sixteen i8. +// c: a bank of four groups holding sixteen rows of sixteen i32; group g +// holds columns 4g..4g+3 of each row, so C moves to and from memory as a +// plain 16 x 16 i32 matrix. +// `rows` is the reduction depth K, 1..bank depth. +// +//===----------------------------------------------------------------------===// + +#ifndef VECTOR_OPS +#define VECTOR_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Interfaces/BankInterfaces.td" + +def VecBall_Dialect : Dialect { + let name = "vector_ball"; + let cppNamespace = "::buddy::vector_ball"; + let summary = "The operations of VecBall"; +} + +class VecBall_Op traits = []> + : Op])>; + +def BankMatmulAtI8Op : VecBall_Op<"bank_matmul_at_i8", + [DeclareOpInterfaceMethods]> { + let summary = "matmul_at_i8 on virtual banks"; + let arguments = (ins I64:$at, I64:$b, I64:$c, I64Attr:$rows); + let results = (outs I64:$cOut); + let assemblyFormat = "$at `,` $b `->` $c prop-dict attr-dict"; +} + +def MatmulAtI8Op : VecBall_Op<"matmul_at_i8", + [DeclareOpInterfaceMethods]> { + let summary = "matmul_at_i8 on physical banks"; + let arguments = (ins I64Attr:$at, I64Attr:$b, I64Attr:$c, I64Attr:$rows); + let assemblyFormat = "prop-dict attr-dict"; +} + +#endif // VECTOR_OPS diff --git a/examples/prefab/balls/vector/verify/casegen/Cargo.lock b/examples/prefab/balls/vector/verify/casegen/Cargo.lock new file mode 100644 index 000000000..5f4dc2de3 --- /dev/null +++ b/examples/prefab/balls/vector/verify/casegen/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "vector-casegen" +version = "0.1.0" diff --git a/examples/prefab/balls/vector/verify/casegen/Cargo.toml b/examples/prefab/balls/vector/verify/casegen/Cargo.toml new file mode 100644 index 000000000..f5380573e --- /dev/null +++ b/examples/prefab/balls/vector/verify/casegen/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "vector-casegen" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["rlib", "cdylib"] + +[lints.rust] +unused_must_use = "deny" diff --git a/examples/prefab/balls/vector/verify/casegen/src/casegen.rs b/examples/prefab/balls/vector/verify/casegen/src/casegen.rs new file mode 100644 index 000000000..9dd98c8e8 --- /dev/null +++ b/examples/prefab/balls/vector/verify/casegen/src/casegen.rs @@ -0,0 +1,126 @@ +use crate::model::{self, GROUPS, LANE}; + +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct VecMat16Cmd { + pub iter: u32, + pub op1_bank: u32, + pub op2_bank: u32, + pub wr_bank: u32, + pub wr_read_bank: u32, + pub rob_id: u32, + // Group bases; the old C uses the wr layout, so it shares wr_group. + pub op1_group: u32, + pub op2_group: u32, + pub wr_group: u32, +} + +#[derive(Clone, Debug)] +pub struct VecMat16Case { + pub cmd: VecMat16Cmd, + pub at: Vec, + pub b: Vec, + // Indexed group * LANE + row. + pub c_old: Vec, + pub c_new: Vec, +} + +struct Lcg(u64); + +impl Lcg { + fn next(&mut self) -> u32 { + self.0 = self.0.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + (self.0 >> 32) as u32 + } +} + +#[derive(Clone, Copy)] +enum Fill { + Random, + // Every int8 is -128 or 127: the largest products of both signs. + Extremes, +} + +fn i8_word(rng: &mut Lcg, fill: Fill) -> u128 { + let mut word = 0u128; + for lane in 0..LANE { + let byte = match fill { + Fill::Random => rng.next() as u8, + Fill::Extremes => { + if rng.next() & 1 == 0 { + 0x80 + } else { + 0x7f + } + } + }; + word |= u128::from(byte) << (8 * lane); + } + word +} + +pub fn gen_case(index: u32) -> VecMat16Case { + let entries = model::bank_entries(); + // (rob_id, iter, [op1, op2, wr, wr_read], [op1, op2, wr] group bases, fill, + // old C near the int32 limits) + let (rob_id, iter, banks, groups, fill, wrap) = match index { + 0 => (1, 1, [0, 1, 2, 3], [0, 0, 0], Fill::Random, false), + 1 => (2, 16, [4, 5, 6, 7], [0, 0, 0], Fill::Extremes, false), + 2 => (3, 5, [1, 0, 3, 2], [0, 0, 0], Fill::Extremes, true), + 3 => (4, entries, [8, 9, 10, 11], [0, 0, 0], Fill::Random, false), + 4 => (5, 7, [1023, 512, 700, 701], [0, 0, 0], Fill::Random, true), + 5 => (6, 9, [12, 13, 14, 15], [3, 7, 12], Fill::Random, false), + 6 => (7, 16, [20, 20, 21, 22], [1, 9, 5], Fill::Extremes, true), + _ => panic!("vecmat16: unsupported case {index}"), + }; + model::validate(entries, iter); + let mut rng = Lcg(0x5ec0_0000 + u64::from(index)); + let at: Vec = (0..iter).map(|_| i8_word(&mut rng, fill)).collect(); + let b: Vec = (0..iter).map(|_| i8_word(&mut rng, fill)).collect(); + let mut c = [[0i32; LANE]; LANE]; + for row in c.iter_mut() { + for value in row.iter_mut() { + let r = rng.next(); + *value = if wrap { + // Near i32::MAX or i32::MIN so the accumulation wraps. + if r & 1 == 0 { + i32::MAX - (r >> 20) as i32 + } else { + i32::MIN + (r >> 20) as i32 + } + } else { + r as i32 + }; + } + } + let mut c_old = Vec::with_capacity(GROUPS * LANE); + for g in 0..GROUPS { + for row in 0..LANE { + c_old.push(model::c_word(&c, g, row)); + } + } + model::vecmat16(&at, &b, &mut c); + let mut c_new = Vec::with_capacity(GROUPS * LANE); + for g in 0..GROUPS { + for row in 0..LANE { + c_new.push(model::c_word(&c, g, row)); + } + } + VecMat16Case { + cmd: VecMat16Cmd { + iter, + op1_bank: banks[0], + op2_bank: banks[1], + wr_bank: banks[2], + wr_read_bank: banks[3], + rob_id, + op1_group: groups[0], + op2_group: groups[1], + wr_group: groups[2], + }, + at, + b, + c_old, + c_new, + } +} diff --git a/examples/prefab/balls/vector/verify/casegen/src/lib.rs b/examples/prefab/balls/vector/verify/casegen/src/lib.rs new file mode 100644 index 000000000..b6d895a46 --- /dev/null +++ b/examples/prefab/balls/vector/verify/casegen/src/lib.rs @@ -0,0 +1,81 @@ +mod casegen; +mod model; + +use std::cell::RefCell; + +use casegen::{VecMat16Case, VecMat16Cmd}; + +thread_local! { + static CURRENT: RefCell> = const { RefCell::new(None) }; +} + +#[no_mangle] +pub extern "C" fn vector_case_load(index: u32) -> i32 { + let case = casegen::gen_case(index); + CURRENT.with(|current| *current.borrow_mut() = Some(case)); + 0 +} + +fn current_case R, R>(f: F) -> R { + CURRENT.with(|current| match current.borrow().as_ref() { + Some(case) => f(case), + None => panic!("vector_case: no case loaded; call vector_case_load first"), + }) +} + +fn word(words: &[u128], index: u32, what: &str) -> u128 { + match words.get(index as usize) { + Some(word) => *word, + None => panic!("vector_case_{what}: index {index} out of range {}", words.len()), + } +} + +#[no_mangle] +pub extern "C" fn vector_case_cmd(out_ptr: *mut VecMat16Cmd) { + if out_ptr.is_null() { + panic!("vector_case_cmd: null out_ptr"); + } + current_case(|case| unsafe { + *out_ptr = case.cmd; + }); +} + +#[no_mangle] +pub extern "C" fn vector_case_at_word_lo(index: u32) -> u64 { + current_case(|case| word(&case.at, index, "at_word_lo") as u64) +} + +#[no_mangle] +pub extern "C" fn vector_case_at_word_hi(index: u32) -> u64 { + current_case(|case| (word(&case.at, index, "at_word_hi") >> 64) as u64) +} + +#[no_mangle] +pub extern "C" fn vector_case_b_word_lo(index: u32) -> u64 { + current_case(|case| word(&case.b, index, "b_word_lo") as u64) +} + +#[no_mangle] +pub extern "C" fn vector_case_b_word_hi(index: u32) -> u64 { + current_case(|case| (word(&case.b, index, "b_word_hi") >> 64) as u64) +} + +#[no_mangle] +pub extern "C" fn vector_case_c_old_word_lo(index: u32) -> u64 { + current_case(|case| word(&case.c_old, index, "c_old_word_lo") as u64) +} + +#[no_mangle] +pub extern "C" fn vector_case_c_old_word_hi(index: u32) -> u64 { + current_case(|case| (word(&case.c_old, index, "c_old_word_hi") >> 64) as u64) +} + +#[no_mangle] +pub extern "C" fn vector_case_c_new_word_lo(index: u32) -> u64 { + current_case(|case| word(&case.c_new, index, "c_new_word_lo") as u64) +} + +#[no_mangle] +pub extern "C" fn vector_case_c_new_word_hi(index: u32) -> u64 { + current_case(|case| (word(&case.c_new, index, "c_new_word_hi") >> 64) as u64) +} diff --git a/examples/prefab/balls/vector/verify/casegen/src/model.rs b/examples/prefab/balls/vector/verify/casegen/src/model.rs new file mode 100644 index 000000000..b996d58bf --- /dev/null +++ b/examples/prefab/balls/vector/verify/casegen/src/model.rs @@ -0,0 +1,50 @@ +// VecBall VECMAT16 contract: C += A^T * B with int8 inputs and int32 +// accumulators that wrap. op1 holds A^T and op2 holds B, `iter` rows of +// sixteen int8 each; the old C is read from the renamed bank wr_read_bank and +// the new C written to wr_bank, sixteen rows of sixteen int32 in four groups: +// group g holds columns 4g..4g+3 of each row. rs2 is reserved and zero. + +pub const LANE: usize = 16; +pub const GROUPS: usize = 4; + +pub fn bank_entries() -> u32 { + let path = std::env::var("BB_VERIFY_CONFIG").expect("BB_VERIFY_CONFIG is required"); + let text = std::fs::read_to_string(path).expect("failed to read BB_VERIFY_CONFIG"); + text.lines() + .find_map(|line| line.strip_prefix("bank_entries=")) + .expect("bank_entries missing from BB_VERIFY_CONFIG") + .parse() + .expect("invalid bank_entries in BB_VERIFY_CONFIG") +} + +pub fn validate(entries: u32, iter: u32) { + if iter == 0 || iter > entries { + panic!("vecmat16: iter must be 1..bank depth"); + } +} + +pub fn i8_lane(word: u128, lane: usize) -> i32 { + ((word >> (8 * lane)) as u8 as i8) as i32 +} + +// c[i][j] += sum_k at[k][i] * b[k][j], wrapping like the hardware. +pub fn vecmat16(at: &[u128], b: &[u128], c: &mut [[i32; LANE]; LANE]) { + for i in 0..LANE { + for j in 0..LANE { + let mut acc = c[i][j]; + for k in 0..at.len() { + acc = acc.wrapping_add(i8_lane(at[k], i).wrapping_mul(i8_lane(b[k], j))); + } + c[i][j] = acc; + } + } +} + +// Word of group `g`, row `row`: columns 4g..4g+3, lowest column in the low bits. +pub fn c_word(c: &[[i32; LANE]; LANE], group: usize, row: usize) -> u128 { + let mut word = 0u128; + for lane in 0..4 { + word |= u128::from(c[row][group * 4 + lane] as u32) << (32 * lane); + } + word +} diff --git a/examples/prefab/balls/vector/verify/filelists/vector_ball.f b/examples/prefab/balls/vector/verify/filelists/vector_ball.f new file mode 100644 index 000000000..2ba66193b --- /dev/null +++ b/examples/prefab/balls/vector/verify/filelists/vector_ball.f @@ -0,0 +1,28 @@ ++incdir+@UVM@/src/core ++incdir+@UVM@/src/ball ++incdir+@UVM@/src/ball/agents/cmd ++incdir+@UVM@/src/ball/agents/mem ++incdir+@UVM@/src/ball/agents/resp ++incdir+@UVM@/src/ball/cov ++incdir+@UVM@/src/ball/env ++incdir+src ++incdir+src/pkg ++incdir+src/common ++incdir+src/seq ++incdir+src/cov ++incdir+src/env ++incdir+src/tests ++incdir+@RTL@ + +@UVM@/src/core/package.sv +@UVM@/src/ball/bb_blink_if.sv +@UVM@/src/ball/bb_uvm_pkg.sv + +@RTL@/VecMat16AccLoad.sv +@RTL@/VecMat16Store.sv +@RTL@/VecMat16.sv +@RTL@/VecUnit.sv +@RTL@/VecBall.sv + +src/pkg/vector_pkg.sv +src/tb_top.sv diff --git a/examples/prefab/balls/vector/verify/src/common/vector_defs.svh b/examples/prefab/balls/vector/verify/src/common/vector_defs.svh new file mode 100644 index 000000000..b2b0df2b7 --- /dev/null +++ b/examples/prefab/balls/vector/verify/src/common/vector_defs.svh @@ -0,0 +1,33 @@ +typedef struct { + int unsigned iter; + int unsigned op1_bank; + int unsigned op2_bank; + int unsigned wr_bank; + int unsigned wr_read_bank; + int unsigned rob_id; + int unsigned op1_group; + int unsigned op2_group; + int unsigned wr_group; +} vector_cmd_dpi_t; + +import "DPI-C" function int vector_case_load(input int unsigned index); +import "DPI-C" function void vector_case_cmd(output vector_cmd_dpi_t cmd); +import "DPI-C" function longint unsigned vector_case_at_word_lo(input int unsigned word_index); +import "DPI-C" function longint unsigned vector_case_at_word_hi(input int unsigned word_index); +import "DPI-C" function longint unsigned vector_case_b_word_lo(input int unsigned word_index); +import "DPI-C" function longint unsigned vector_case_b_word_hi(input int unsigned word_index); +import "DPI-C" function longint unsigned vector_case_c_old_word_lo(input int unsigned word_index); +import "DPI-C" function longint unsigned vector_case_c_old_word_hi(input int unsigned word_index); +import "DPI-C" function longint unsigned vector_case_c_new_word_lo(input int unsigned word_index); +import "DPI-C" function longint unsigned vector_case_c_new_word_hi(input int unsigned word_index); + +`ifndef VECMAT16_FUNCT7 +`error "VECMAT16_FUNCT7 must be provided by the selected ballISA" +`endif + +localparam int VECMAT16_THREAD_FUNCT7 = `VECMAT16_FUNCT7; +// C is sixteen rows in four groups of four int32 columns. +localparam int VECMAT16_ROWS = 16; +localparam int VECMAT16_GROUPS = 4; +localparam int VECMAT16_NUM_CASES = 7; +localparam int VECMAT16_TIMEOUT_CYCLES = 50000; diff --git a/examples/prefab/balls/vector/verify/src/common/vector_items.svh b/examples/prefab/balls/vector/verify/src/common/vector_items.svh new file mode 100644 index 000000000..65974870b --- /dev/null +++ b/examples/prefab/balls/vector/verify/src/common/vector_items.svh @@ -0,0 +1,63 @@ +class vector_cmd_item extends bb_blink_cmd_item; + `uvm_object_utils(vector_cmd_item) + + bit [127:0] at_words[]; + bit [127:0] b_words[]; + // Indexed group * VECMAT16_ROWS + row. + bit [127:0] c_old_words[VECMAT16_GROUPS * VECMAT16_ROWS]; + bit [127:0] c_new_words[VECMAT16_GROUPS * VECMAT16_ROWS]; + + function new(string name = "vector_cmd_item"); + super.new(name); + endfunction + + function void load_rust_case(int unsigned index); + vector_cmd_dpi_t cmd; + int unsigned rc; + + rc = vector_case_load(index); + if (rc != 0) + `uvm_fatal("CASE", $sformatf("vector_case_load returned %0d for index %0d", rc, index)) + vector_case_cmd(cmd); + + // bb_vecmat16: A^T, B and C from rs1 banks 0..2, K in iter; the old C + // arrives as the renamed wr_read_bank at the wr group base. rs2 is + // reserved and zero. + rob_id = cmd.rob_id[BB_BLINK_ROB_ID_W-1:0]; + funct = VECMAT16_THREAD_FUNCT7[6:0]; + rs2 = '0; + iter = cmd.iter; + op1_bank = cmd.op1_bank; + op2_bank = cmd.op2_bank; + wr_bank = cmd.wr_bank; + op1_group = cmd.op1_group; + op2_group = cmd.op2_group; + wr_group = cmd.wr_group; + op1_col = 1; + op2_col = 1; + wr_col = VECMAT16_GROUPS; + wr_read_bank = cmd.wr_read_bank; + wr_arch_bank = cmd.wr_bank; + + at_words = new[cmd.iter]; + b_words = new[cmd.iter]; + for (int i = 0; i < cmd.iter; i++) begin + at_words[i] = {vector_case_at_word_hi(i), vector_case_at_word_lo(i)}; + b_words[i] = {vector_case_b_word_hi(i), vector_case_b_word_lo(i)}; + end + for (int i = 0; i < VECMAT16_GROUPS * VECMAT16_ROWS; i++) begin + c_old_words[i] = {vector_case_c_old_word_hi(i), vector_case_c_old_word_lo(i)}; + c_new_words[i] = {vector_case_c_new_word_hi(i), vector_case_c_new_word_lo(i)}; + end + endfunction + + function void do_copy(uvm_object rhs); + vector_cmd_item rhs_; + super.do_copy(rhs); + if (!$cast(rhs_, rhs)) `uvm_fatal("COPY", "rhs is not vector_cmd_item") + at_words = rhs_.at_words; + b_words = rhs_.b_words; + c_old_words = rhs_.c_old_words; + c_new_words = rhs_.c_new_words; + endfunction +endclass diff --git a/examples/prefab/balls/vector/verify/src/cov/vector_cov.svh b/examples/prefab/balls/vector/verify/src/cov/vector_cov.svh new file mode 100644 index 000000000..3560cc1ab --- /dev/null +++ b/examples/prefab/balls/vector/verify/src/cov/vector_cov.svh @@ -0,0 +1,44 @@ +class vector_cov extends uvm_component; + `uvm_component_utils(vector_cov) + + uvm_analysis_imp_cmd #(bb_blink_cmd_item, vector_cov) cmd_imp; + + int unsigned cur_iter; + int unsigned cur_op1_group; + int unsigned cur_op2_group; + int unsigned cur_wr_group; + + covergroup cmd_cg; + coverpoint cur_iter {bins one = {1}; bins short = {[2 : 16]}; bins deep = {[17 : $]};} + coverpoint cur_op1_group {bins zero = {0}; bins based = {[1 : $]};} + coverpoint cur_op2_group {bins zero = {0}; bins based = {[1 : $]};} + coverpoint cur_wr_group {bins zero = {0}; bins based = {[1 : $]};} + endgroup + + int cmd_count; + + function new(string name, uvm_component parent); + super.new(name, parent); + cmd_imp = new("cmd_imp", this); + cmd_cg = new(); + endfunction + + function void write_cmd(bb_blink_cmd_item item); + if (item.funct != VECMAT16_THREAD_FUNCT7[6:0]) + `uvm_fatal("COV", $sformatf("unexpected funct %0d", item.funct)) + cur_iter = item.iter; + cur_op1_group = item.op1_group; + cur_op2_group = item.op2_group; + cur_wr_group = item.wr_group; + cmd_count++; + cmd_cg.sample(); + endfunction + + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (cmd_count == 0) `uvm_fatal("COV", "no cmd observed") + if (cmd_cg.get_coverage() != 100.0) + `uvm_fatal("COV", $sformatf("vector_cov %0.2f%%", cmd_cg.get_coverage())) + endfunction +endclass diff --git a/examples/prefab/balls/vector/verify/src/env/vector_env.svh b/examples/prefab/balls/vector/verify/src/env/vector_env.svh new file mode 100644 index 000000000..14f26e4be --- /dev/null +++ b/examples/prefab/balls/vector/verify/src/env/vector_env.svh @@ -0,0 +1,27 @@ +class vector_env extends bb_blink_env #(`BB_IN_BW, `BB_OUT_BW); + `uvm_component_utils(vector_env) + + vector_scoreboard scb; + vector_cov icov; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + scb = vector_scoreboard::type_id::create("scb", this); + icov = vector_cov::type_id::create("icov", this); + endfunction + + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + scb.mem_model = mem_model; + cmd_agent.stim_ap.connect(scb.stim_imp); + cmd_agent.cmd_ap.connect(scb.cmd_imp); + cmd_agent.cmd_ap.connect(icov.cmd_imp); + read_mon.read_ap.connect(scb.read_imp); + write_mon.write_ap.connect(scb.write_imp); + resp_mon.resp_ap.connect(scb.resp_imp); + endfunction +endclass diff --git a/examples/prefab/balls/vector/verify/src/env/vector_scoreboard.svh b/examples/prefab/balls/vector/verify/src/env/vector_scoreboard.svh new file mode 100644 index 000000000..cc5e11669 --- /dev/null +++ b/examples/prefab/balls/vector/verify/src/env/vector_scoreboard.svh @@ -0,0 +1,200 @@ +class vector_scoreboard extends uvm_scoreboard; + `uvm_component_utils(vector_scoreboard) + + uvm_analysis_imp_stim #(bb_blink_cmd_item, vector_scoreboard) stim_imp; + uvm_analysis_imp_cmd #(bb_blink_cmd_item, vector_scoreboard) cmd_imp; + uvm_analysis_imp_read #(bb_blink_read_item, vector_scoreboard) read_imp; + uvm_analysis_imp_write #(bb_blink_write_item, vector_scoreboard) write_imp; + uvm_analysis_imp_resp #(bb_blink_resp_item, vector_scoreboard) resp_imp; + + bb_blink_mem_model #(`BB_IN_BW, `BB_OUT_BW) mem_model; + + vector_cmd_item stim_q[$]; + int unsigned cmd_count; + int unsigned read_count[2]; + int unsigned write_count[VECMAT16_GROUPS]; + int unsigned resp_count; + + function new(string name, uvm_component parent); + super.new(name, parent); + stim_imp = new("stim_imp", this); + cmd_imp = new("cmd_imp", this); + read_imp = new("read_imp", this); + write_imp = new("write_imp", this); + resp_imp = new("resp_imp", this); + endfunction + + function void write_stim(bb_blink_cmd_item item); + vector_cmd_item got; + vector_cmd_item clone; + + if (stim_q.size() != 0) `uvm_fatal("SCB", "single outstanding command supported") + if (!$cast(got, item)) `uvm_fatal("SCB", "stim item is not vector_cmd_item") + if (!$cast(clone, got.clone())) `uvm_fatal("SCB", "failed to clone stimulus item") + stim_q.push_back(clone); + + if (mem_model == null) `uvm_fatal("SCB", "mem_model handle not set") + mem_model.clear_mem(); + for (int g = 0; g < VECMAT16_GROUPS; g++) + for (int row = 0; row < VECMAT16_ROWS; row++) + mem_model.load_word_g(int'(clone.wr_read_bank), int'(clone.wr_group) + g, row, + clone.c_old_words[g*VECMAT16_ROWS+row]); + for (int k = 0; k < clone.iter; k++) begin + mem_model.load_word_g(int'(clone.op1_bank), int'(clone.op1_group), k, clone.at_words[k]); + mem_model.load_word_g(int'(clone.op2_bank), int'(clone.op2_group), k, clone.b_words[k]); + end + mem_model.arm(); + endfunction + + function void write_cmd(bb_blink_cmd_item item); + bb_blink_cmd_item want; + vector_cmd_item stim; + stim = current_stim("CMD"); + // Compare as plain BLink commands; the comparer rejects differing types. + want = bb_blink_cmd_item::type_id::create("want"); + want.copy(stim); + verification_pkg::evidence::checks++; + if (!item.compare(want)) + `uvm_fatal("CMD", $sformatf("command mismatch: got %s exp %s", item.sprint(), want.sprint())) + cmd_count++; + endfunction + + // Port p first reads the old C, row by row, group p then group 2 + p past + // the wr base; then port 0 streams A^T and port 1 streams B, one row per + // reduction step, at their own group bases. + function void write_read(bb_blink_read_item item); + vector_cmd_item stim; + int unsigned n; + int unsigned expect_bank; + int unsigned expect_group; + int unsigned expect_addr; + + stim = current_stim("READ"); + if (item.port != 0 && item.port != 1) + `uvm_fatal("READ", $sformatf("VecBall reads on two ports, got port %0d", item.port)) + n = read_count[item.port]; + if (n >= 2 * VECMAT16_ROWS + stim.iter) + `uvm_fatal("READ", $sformatf("extra read on port %0d", item.port)) + if (item.rob_id !== stim.rob_id) + `uvm_fatal("READ", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + if (n < 2 * VECMAT16_ROWS) begin + expect_bank = stim.wr_read_bank; + expect_group = stim.wr_group + 2 * (n % 2) + item.port; + expect_addr = n / 2; + end else begin + expect_bank = item.port == 0 ? stim.op1_bank : stim.op2_bank; + expect_group = item.port == 0 ? stim.op1_group : stim.op2_group; + expect_addr = n - 2 * VECMAT16_ROWS; + end + if (item.bank_id !== expect_bank[BB_BLINK_BANK_ID_W-1:0]) + `uvm_fatal("READ", $sformatf( + "port %0d read %0d bank mismatch: got %0d exp %0d", + item.port, + n, + item.bank_id, + expect_bank + )) + if (item.group_id !== expect_group[BB_BLINK_GROUP_ID_W-1:0]) + `uvm_fatal("READ", $sformatf( + "port %0d read %0d group mismatch: got %0d exp %0d", + item.port, + n, + item.group_id, + expect_group + )) + if (item.addr !== expect_addr[BB_BLINK_BANK_ADDR_W-1:0]) + `uvm_fatal( + "READ", $sformatf( + "port %0d read %0d addr mismatch: got %0d exp %0d", item.port, n, item.addr, expect_addr)) + read_count[item.port]++; + endfunction + + // Write port g stores group g of C past the wr base, rows in order, after + // every operand read. + function void write_write(bb_blink_write_item item); + vector_cmd_item stim; + int unsigned g; + int unsigned row; + + stim = current_stim("WRITE"); + if (item.port < 0 || item.port >= VECMAT16_GROUPS) + `uvm_fatal("WRITE", $sformatf("VecBall writes on four ports, got port %0d", item.port)) + g = item.port; + row = write_count[g]; + if (row >= VECMAT16_ROWS) `uvm_fatal("WRITE", $sformatf("extra write on port %0d", g)) + for (int p = 0; p < 2; p++) + if (read_count[p] != 2 * VECMAT16_ROWS + stim.iter) + `uvm_fatal("WRITE", $sformatf("write before port %0d finished reading", p)) + if (item.rob_id !== stim.rob_id) + `uvm_fatal("WRITE", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + if (item.bank_id !== stim.wr_bank) + `uvm_fatal("WRITE", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.wr_bank)) + if (item.group_id !== stim.wr_group + g) + `uvm_fatal("WRITE", $sformatf( + "group mismatch: got %0d exp %0d", item.group_id, stim.wr_group + g)) + if (item.mask !== 16'hFFFF) + `uvm_fatal("WRITE", $sformatf("mask mismatch: got 0x%04h", item.mask)) + if (item.addr !== row) + `uvm_fatal("WRITE", $sformatf("port %0d addr mismatch: got %0d exp %0d", g, item.addr, row)) + verification_pkg::evidence::checks++; + if (item.data !== stim.c_new_words[g*VECMAT16_ROWS+row]) + `uvm_fatal("SCB", $sformatf( + "C mismatch at row %0d group %0d: got 0x%032h exp 0x%032h", + row, + g, + item.data, + stim.c_new_words[g*VECMAT16_ROWS+row] + )) + write_count[g]++; + endfunction + + function void write_resp(bb_blink_resp_item item); + vector_cmd_item stim; + stim = current_stim("RESP"); + for (int g = 0; g < VECMAT16_GROUPS; g++) + if (write_count[g] != VECMAT16_ROWS) + `uvm_fatal("RESP", $sformatf("response after %0d writes on port %0d", write_count[g], g)) + if (item.rob_id !== stim.rob_id) + `uvm_fatal("RESP", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + resp_count++; + endfunction + + function vector_cmd_item current_stim(string tag); + if (stim_q.size() == 0) `uvm_fatal("SCB", $sformatf("%s observed before stimulus", tag)) + return stim_q[0]; + endfunction + + function bit done(); + if (stim_q.size() == 0 || cmd_count != 1 || resp_count != 1) return 1'b0; + for (int p = 0; p < 2; p++) + if (read_count[p] != 2 * VECMAT16_ROWS + stim_q[0].iter) return 1'b0; + for (int g = 0; g < VECMAT16_GROUPS; g++) if (write_count[g] != VECMAT16_ROWS) return 1'b0; + return 1'b1; + endfunction + + function void reset_counters(); + stim_q.delete(); + cmd_count = 0; + resp_count = 0; + for (int p = 0; p < 2; p++) read_count[p] = 0; + for (int g = 0; g < VECMAT16_GROUPS; g++) write_count[g] = 0; + endfunction + + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (!done()) + `uvm_fatal("SCB", $sformatf( + "incomplete: cmds=%0d reads=%0d,%0d writes=%0d,%0d,%0d,%0d resp=%0d", + cmd_count, + read_count[0], + read_count[1], + write_count[0], + write_count[1], + write_count[2], + write_count[3], + resp_count + )) + stim_q.delete(); + endfunction +endclass diff --git a/examples/prefab/balls/vector/verify/src/pkg/vector_pkg.sv b/examples/prefab/balls/vector/verify/src/pkg/vector_pkg.sv new file mode 100644 index 000000000..dc94fef37 --- /dev/null +++ b/examples/prefab/balls/vector/verify/src/pkg/vector_pkg.sv @@ -0,0 +1,13 @@ +package vector_pkg; + import uvm_pkg::*; + import bb_uvm_pkg::*; + `include "uvm_macros.svh" + + `include "common/vector_defs.svh" + `include "common/vector_items.svh" + `include "seq/vector_sequences.svh" + `include "cov/vector_cov.svh" + `include "env/vector_scoreboard.svh" + `include "env/vector_env.svh" + `include "tests/vector_test.svh" +endpackage diff --git a/examples/prefab/balls/vector/verify/src/seq/vector_sequences.svh b/examples/prefab/balls/vector/verify/src/seq/vector_sequences.svh new file mode 100644 index 000000000..d63048f51 --- /dev/null +++ b/examples/prefab/balls/vector/verify/src/seq/vector_sequences.svh @@ -0,0 +1,17 @@ +class vector_basic_seq extends uvm_sequence #(bb_blink_cmd_item); + `uvm_object_utils(vector_basic_seq) + + int unsigned case_index; + + function new(string name = "vector_basic_seq"); + super.new(name); + endfunction + + task body(); + vector_cmd_item item; + item = vector_cmd_item::type_id::create("item"); + start_item(item); + item.load_rust_case(case_index); + finish_item(item); + endtask +endclass diff --git a/examples/prefab/balls/vector/verify/src/tb_top.sv b/examples/prefab/balls/vector/verify/src/tb_top.sv new file mode 100644 index 000000000..a0c1322e6 --- /dev/null +++ b/examples/prefab/balls/vector/verify/src/tb_top.sv @@ -0,0 +1,165 @@ +module tb_top; + import uvm_pkg::*; + import vector_pkg::*; + + bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); + + VecBall dut ( + .clock(intf.clock), + .reset(intf.reset), + .blink_status_running(intf.status_running), + .blink_cmdReq_ready(intf.cmd_req_ready), + .blink_cmdReq_valid(intf.cmd_req_valid), + .blink_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .blink_cmdReq_bits_funct(intf.cmd_req_bits_funct), + .blink_cmdReq_bits_rs2(intf.cmd_req_bits_rs2), + .blink_cmdReq_bits_iter(intf.cmd_req_bits_iter), + .blink_cmdReq_bits_op1_bank(intf.cmd_req_bits_op1_bank), + .blink_cmdReq_bits_op2_bank(intf.cmd_req_bits_op2_bank), + .blink_cmdReq_bits_wr_bank(intf.cmd_req_bits_wr_bank), + .blink_cmdReq_bits_op1_group(intf.cmd_req_bits_op1_group), + .blink_cmdReq_bits_op2_group(intf.cmd_req_bits_op2_group), + .blink_cmdReq_bits_wr_group(intf.cmd_req_bits_wr_group), + .blink_cmdReq_bits_op1_col(intf.cmd_req_bits_op1_col), + .blink_cmdReq_bits_op2_col(intf.cmd_req_bits_op2_col), + .blink_cmdReq_bits_wr_col(intf.cmd_req_bits_wr_col), + .blink_cmdReq_bits_wr_read_bank(intf.cmd_req_bits_wr_read_bank), + .blink_cmdReq_bits_wr_arch_bank(intf.cmd_req_bits_wr_arch_bank), + .blink_cmdResp_ready(intf.cmd_resp_ready), + .blink_cmdResp_valid(intf.cmd_resp_valid), + .blink_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .blink_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .blink_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .blink_bankRead_0_group_id(intf.bank_read_group_id[0]), + .blink_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .blink_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .blink_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .blink_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .blink_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .blink_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .blink_bankRead_1_bank_id(intf.bank_read_bank_id[1]), + .blink_bankRead_1_rob_id(intf.bank_read_rob_id[1]), + .blink_bankRead_1_group_id(intf.bank_read_group_id[1]), + .blink_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), + .blink_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), + .blink_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), + .blink_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), + .blink_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), + .blink_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), + .blink_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .blink_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .blink_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .blink_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .blink_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .blink_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .blink_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), + .blink_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), + .blink_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), + .blink_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), + .blink_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), + .blink_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), + .blink_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), + .blink_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), + .blink_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), + .blink_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), + .blink_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), + .blink_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), + .blink_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), + .blink_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), + .blink_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), + .blink_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), + .blink_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .blink_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .blink_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .blink_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), + .blink_bankWrite_1_bank_id(intf.bank_write_bank_id[1]), + .blink_bankWrite_1_rob_id(intf.bank_write_rob_id[1]), + .blink_bankWrite_1_group_id(intf.bank_write_group_id[1]), + .blink_bankWrite_1_io_req_ready(intf.bank_write_req_ready[1]), + .blink_bankWrite_1_io_req_valid(intf.bank_write_req_valid[1]), + .blink_bankWrite_1_io_req_bits_addr(intf.bank_write_req_addr[1]), + .blink_bankWrite_1_io_req_bits_mask_0(intf.bank_write_req_mask[1][0]), + .blink_bankWrite_1_io_req_bits_mask_1(intf.bank_write_req_mask[1][1]), + .blink_bankWrite_1_io_req_bits_mask_2(intf.bank_write_req_mask[1][2]), + .blink_bankWrite_1_io_req_bits_mask_3(intf.bank_write_req_mask[1][3]), + .blink_bankWrite_1_io_req_bits_mask_4(intf.bank_write_req_mask[1][4]), + .blink_bankWrite_1_io_req_bits_mask_5(intf.bank_write_req_mask[1][5]), + .blink_bankWrite_1_io_req_bits_mask_6(intf.bank_write_req_mask[1][6]), + .blink_bankWrite_1_io_req_bits_mask_7(intf.bank_write_req_mask[1][7]), + .blink_bankWrite_1_io_req_bits_mask_8(intf.bank_write_req_mask[1][8]), + .blink_bankWrite_1_io_req_bits_mask_9(intf.bank_write_req_mask[1][9]), + .blink_bankWrite_1_io_req_bits_mask_10(intf.bank_write_req_mask[1][10]), + .blink_bankWrite_1_io_req_bits_mask_11(intf.bank_write_req_mask[1][11]), + .blink_bankWrite_1_io_req_bits_mask_12(intf.bank_write_req_mask[1][12]), + .blink_bankWrite_1_io_req_bits_mask_13(intf.bank_write_req_mask[1][13]), + .blink_bankWrite_1_io_req_bits_mask_14(intf.bank_write_req_mask[1][14]), + .blink_bankWrite_1_io_req_bits_mask_15(intf.bank_write_req_mask[1][15]), + .blink_bankWrite_1_io_req_bits_data(intf.bank_write_req_data[1]), + .blink_bankWrite_1_io_resp_ready(intf.bank_write_resp_ready[1]), + .blink_bankWrite_1_io_resp_valid(intf.bank_write_resp_valid[1]), + .blink_bankWrite_1_io_resp_bits_ok(intf.bank_write_resp_ok[1]), + .blink_bankWrite_2_bank_id(intf.bank_write_bank_id[2]), + .blink_bankWrite_2_rob_id(intf.bank_write_rob_id[2]), + .blink_bankWrite_2_group_id(intf.bank_write_group_id[2]), + .blink_bankWrite_2_io_req_ready(intf.bank_write_req_ready[2]), + .blink_bankWrite_2_io_req_valid(intf.bank_write_req_valid[2]), + .blink_bankWrite_2_io_req_bits_addr(intf.bank_write_req_addr[2]), + .blink_bankWrite_2_io_req_bits_mask_0(intf.bank_write_req_mask[2][0]), + .blink_bankWrite_2_io_req_bits_mask_1(intf.bank_write_req_mask[2][1]), + .blink_bankWrite_2_io_req_bits_mask_2(intf.bank_write_req_mask[2][2]), + .blink_bankWrite_2_io_req_bits_mask_3(intf.bank_write_req_mask[2][3]), + .blink_bankWrite_2_io_req_bits_mask_4(intf.bank_write_req_mask[2][4]), + .blink_bankWrite_2_io_req_bits_mask_5(intf.bank_write_req_mask[2][5]), + .blink_bankWrite_2_io_req_bits_mask_6(intf.bank_write_req_mask[2][6]), + .blink_bankWrite_2_io_req_bits_mask_7(intf.bank_write_req_mask[2][7]), + .blink_bankWrite_2_io_req_bits_mask_8(intf.bank_write_req_mask[2][8]), + .blink_bankWrite_2_io_req_bits_mask_9(intf.bank_write_req_mask[2][9]), + .blink_bankWrite_2_io_req_bits_mask_10(intf.bank_write_req_mask[2][10]), + .blink_bankWrite_2_io_req_bits_mask_11(intf.bank_write_req_mask[2][11]), + .blink_bankWrite_2_io_req_bits_mask_12(intf.bank_write_req_mask[2][12]), + .blink_bankWrite_2_io_req_bits_mask_13(intf.bank_write_req_mask[2][13]), + .blink_bankWrite_2_io_req_bits_mask_14(intf.bank_write_req_mask[2][14]), + .blink_bankWrite_2_io_req_bits_mask_15(intf.bank_write_req_mask[2][15]), + .blink_bankWrite_2_io_req_bits_data(intf.bank_write_req_data[2]), + .blink_bankWrite_2_io_resp_ready(intf.bank_write_resp_ready[2]), + .blink_bankWrite_2_io_resp_valid(intf.bank_write_resp_valid[2]), + .blink_bankWrite_2_io_resp_bits_ok(intf.bank_write_resp_ok[2]), + .blink_bankWrite_3_bank_id(intf.bank_write_bank_id[3]), + .blink_bankWrite_3_rob_id(intf.bank_write_rob_id[3]), + .blink_bankWrite_3_group_id(intf.bank_write_group_id[3]), + .blink_bankWrite_3_io_req_ready(intf.bank_write_req_ready[3]), + .blink_bankWrite_3_io_req_valid(intf.bank_write_req_valid[3]), + .blink_bankWrite_3_io_req_bits_addr(intf.bank_write_req_addr[3]), + .blink_bankWrite_3_io_req_bits_mask_0(intf.bank_write_req_mask[3][0]), + .blink_bankWrite_3_io_req_bits_mask_1(intf.bank_write_req_mask[3][1]), + .blink_bankWrite_3_io_req_bits_mask_2(intf.bank_write_req_mask[3][2]), + .blink_bankWrite_3_io_req_bits_mask_3(intf.bank_write_req_mask[3][3]), + .blink_bankWrite_3_io_req_bits_mask_4(intf.bank_write_req_mask[3][4]), + .blink_bankWrite_3_io_req_bits_mask_5(intf.bank_write_req_mask[3][5]), + .blink_bankWrite_3_io_req_bits_mask_6(intf.bank_write_req_mask[3][6]), + .blink_bankWrite_3_io_req_bits_mask_7(intf.bank_write_req_mask[3][7]), + .blink_bankWrite_3_io_req_bits_mask_8(intf.bank_write_req_mask[3][8]), + .blink_bankWrite_3_io_req_bits_mask_9(intf.bank_write_req_mask[3][9]), + .blink_bankWrite_3_io_req_bits_mask_10(intf.bank_write_req_mask[3][10]), + .blink_bankWrite_3_io_req_bits_mask_11(intf.bank_write_req_mask[3][11]), + .blink_bankWrite_3_io_req_bits_mask_12(intf.bank_write_req_mask[3][12]), + .blink_bankWrite_3_io_req_bits_mask_13(intf.bank_write_req_mask[3][13]), + .blink_bankWrite_3_io_req_bits_mask_14(intf.bank_write_req_mask[3][14]), + .blink_bankWrite_3_io_req_bits_mask_15(intf.bank_write_req_mask[3][15]), + .blink_bankWrite_3_io_req_bits_data(intf.bank_write_req_data[3]), + .blink_bankWrite_3_io_resp_ready(intf.bank_write_resp_ready[3]), + .blink_bankWrite_3_io_resp_valid(intf.bank_write_resp_valid[3]), + .blink_bankWrite_3_io_resp_bits_ok(intf.bank_write_resp_ok[3]) + ); + + initial begin + intf.clock = 1'b0; + intf.reset = 1'b1; + forever #5 intf.clock = ~intf.clock; + end + + initial begin + uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); + run_test("vector_ball_test"); + end +endmodule diff --git a/examples/prefab/balls/vector/verify/src/tests/vector_test.svh b/examples/prefab/balls/vector/verify/src/tests/vector_test.svh new file mode 100644 index 000000000..b9d6a1dfd --- /dev/null +++ b/examples/prefab/balls/vector/verify/src/tests/vector_test.svh @@ -0,0 +1,49 @@ +class vector_ball_test extends uvm_test; + `uvm_component_utils(vector_ball_test) + + typedef virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) vif_t; + vif_t vif; + vector_env env; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(vif_t)::get(this, "", "vif", vif)) + `uvm_fatal("NOVIF", "bb_blink_if not found") + env = vector_env::type_id::create("env", this); + endfunction + + task run_phase(uvm_phase phase); + phase.raise_objection(this); + for (int unsigned index = 0; index < VECMAT16_NUM_CASES; index++) run_case(index); + phase.drop_objection(this); + endtask + + task run_case(int unsigned index); + vector_basic_seq seq; + int cycles = 0; + apply_reset(); + env.scb.reset_counters(); + seq = vector_basic_seq::type_id::create("seq"); + seq.case_index = index; + seq.start(env.cmd_agent.seqr); + while (!env.scb.done()) begin + @(posedge vif.clock); + cycles++; + if (cycles > VECMAT16_TIMEOUT_CYCLES) + `uvm_fatal("TIMEOUT", $sformatf("VecBall case %0d timeout", index)) + end + `uvm_info("VECMAT16", $sformatf("VecBall case %0d passed in %0d cycles", index, cycles), + UVM_LOW) + endtask + + task apply_reset(); + vif.reset = 1'b1; + repeat (5) @(posedge vif.clock); + vif.reset = 1'b0; + repeat (2) @(posedge vif.clock); + endtask +endclass diff --git a/examples/prefab/balls/vector/workloads/ctests/CMakeLists.txt b/examples/prefab/balls/vector/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..a140d5a75 --- /dev/null +++ b/examples/prefab/balls/vector/workloads/ctests/CMakeLists.txt @@ -0,0 +1,2 @@ +add_buckyball_ctest(vecmat16_atranspose_16x16_test.c) +add_buckyball_ctest(vecmat16_accumulate_k4_test.c) diff --git a/examples/prefab/balls/vector/workloads/ctests/vecmat16_accumulate_k4_test.c b/examples/prefab/balls/vector/workloads/ctests/vecmat16_accumulate_k4_test.c new file mode 100644 index 000000000..f252f64df --- /dev/null +++ b/examples/prefab/balls/vector/workloads/ctests/vecmat16_accumulate_k4_test.c @@ -0,0 +1,51 @@ +#include "buckyball.h" +#include +#include +#include +#include +#include + +// C = bias + A^T * B with a reduction depth of four rows, then a second +// instruction accumulating the same product again. +enum { DIM = 16, K = 4 }; +static int8_t at[K * DIM] __attribute__((aligned(64))); +static int8_t b[K * DIM] __attribute__((aligned(64))); +static int32_t c[DIM * DIM] __attribute__((aligned(64))); +static int32_t expected[DIM * DIM]; + +int main(void) { + for (int k = 0; k < K; ++k) + for (int lane = 0; lane < DIM; ++lane) { + at[k * DIM + lane] = (int8_t)((k * 37 + lane * 11) % 255 - 127); + b[k * DIM + lane] = (int8_t)((k * 53 + lane * 29) % 255 - 128); + } + for (int i = 0; i < DIM; ++i) + for (int j = 0; j < DIM; ++j) { + int32_t bias = (i * DIM + j) * 1009 - 100000; + int32_t product = 0; + for (int k = 0; k < K; ++k) + product += at[k * DIM + i] * b[k * DIM + j]; + c[i * DIM + j] = bias; + expected[i * DIM + j] = bias + 2 * product; + } + bb_mem_alloc(1, 1, 1); + bb_mem_alloc(2, 1, 1); + bb_mem_alloc(3, 1, 4); + bb_mvin((uintptr_t)at, 1, K, 1); + bb_mvin((uintptr_t)b, 2, K, 1); + bb_mvin((uintptr_t)c, 3, DIM, 1); + bb_vecmat16(1, 2, 3, K); + bb_vecmat16(1, 2, 3, K); + bb_mvout((uintptr_t)c, 3, DIM, 1); + bb_mem_release(1); + bb_mem_release(2); + bb_mem_release(3); + for (int i = 0; i < DIM * DIM; ++i) + if (c[i] != expected[i]) { + printf("vecmat16 accumulate mismatch at %d: got %d expected %d\n", i, + c[i], expected[i]); + return 1; + } + printf("vecmat16_accumulate_k4 PASS\n"); + return 0; +} diff --git a/examples/balls/vector/workloads/ctests/vecmat16_atranspose_16x16_test.c b/examples/prefab/balls/vector/workloads/ctests/vecmat16_atranspose_16x16_test.c similarity index 76% rename from examples/balls/vector/workloads/ctests/vecmat16_atranspose_16x16_test.c rename to examples/prefab/balls/vector/workloads/ctests/vecmat16_atranspose_16x16_test.c index 1c934a288..303f886f3 100644 --- a/examples/balls/vector/workloads/ctests/vecmat16_atranspose_16x16_test.c +++ b/examples/prefab/balls/vector/workloads/ctests/vecmat16_atranspose_16x16_test.c @@ -20,20 +20,22 @@ int main(void) { at[col * DIM + row] = a[row * DIM + col]; } cpu_matmul(a, b, expected, DIM, DIM, DIM); - bb_mem_alloc(0, 1, 1); bb_mem_alloc(1, 1, 1); - bb_mem_alloc(2, 1, 4); - bb_mvin((uintptr_t)at, 0, DIM, 1); - bb_mvin((uintptr_t)b, 1, DIM, 1); - bb_vecmat16(0, 1, 2, DIM, 0); - bb_mvout((uintptr_t)actual, 2, DIM, 1); + bb_mem_alloc(2, 1, 1); + bb_mem_alloc(3, 1, 4); + bb_mvin((uintptr_t)at, 1, DIM, 1); + bb_mvin((uintptr_t)b, 2, DIM, 1); + // C starts at zero: actual is zero-initialized. + bb_mvin((uintptr_t)actual, 3, DIM, 1); + bb_vecmat16(1, 2, 3, DIM); + bb_mvout((uintptr_t)actual, 3, DIM, 1); if (!compare_i32_matrices(actual, expected, DIM, DIM)) { printf("vecmat16_atranspose_16x16 FAIL\n"); return 1; } - bb_mem_release(0); bb_mem_release(1); bb_mem_release(2); + bb_mem_release(3); printf("vecmat16_atranspose_16x16 PASS\n"); return 0; } diff --git a/examples/prefab/balls/vector/workloads/isa/vecmat16.h b/examples/prefab/balls/vector/workloads/isa/vecmat16.h new file mode 100644 index 000000000..d11a93035 --- /dev/null +++ b/examples/prefab/balls/vector/workloads/isa/vecmat16.h @@ -0,0 +1,21 @@ +#ifndef _BB_VECMAT16_H_ +#define _BB_VECMAT16_H_ + +#include +#include + +// C += A^T * B with int8 inputs and int32 accumulators. +// op1 bank (one group): A^T, iter rows of sixteen int8; row k is column k +// of the 16 x iter matrix A. +// op2 bank (one group): B, iter rows of sixteen int8. +// wr bank (four groups): C, sixteen rows of sixteen int32; group g holds +// columns 4g..4g+3 of each row. The old C is read +// and accumulated, so rs1[29] is always set. +// iter: the reduction depth K, 1..bank depth. rs2 is reserved and zero. +#define bb_vecmat16(op1_bank_id, op2_bank_id, wr_bank_id, iter) \ + BUCKYBALL_INSTRUCTION_R_R((BB_BANK0(op1_bank_id) | BB_BANK1(op2_bank_id) | \ + BB_BANK2(wr_bank_id) | BB_READ_WR_OLD | \ + BB_ITER(iter)), \ + 0, FUNCT7(vector, VECMAT16)) + +#endif // _BB_VECMAT16_H_ diff --git a/examples/prefab/balls/vector/workloads/mlir_tests/CMakeLists.txt b/examples/prefab/balls/vector/workloads/mlir_tests/CMakeLists.txt new file mode 100644 index 000000000..4b64da6aa --- /dev/null +++ b/examples/prefab/balls/vector/workloads/mlir_tests/CMakeLists.txt @@ -0,0 +1,2 @@ +add_subdirectory(ball) +add_subdirectory(bank) diff --git a/examples/prefab/balls/vector/workloads/mlir_tests/ball/CMakeLists.txt b/examples/prefab/balls/vector/workloads/mlir_tests/ball/CMakeLists.txt new file mode 100644 index 000000000..1463bb1ef --- /dev/null +++ b/examples/prefab/balls/vector/workloads/mlir_tests/ball/CMakeLists.txt @@ -0,0 +1,4 @@ +set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) +set(BUCKYBALL_MLIR_TEST_PREFIX ball) + +add_buckyball_mlir_test(matmul_at_i8_16x16 TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/prefab/balls/vector/workloads/mlir_tests/ball/matmul_at_i8_16x16.mlir b/examples/prefab/balls/vector/workloads/mlir_tests/ball/matmul_at_i8_16x16.mlir new file mode 100644 index 000000000..c68189646 --- /dev/null +++ b/examples/prefab/balls/vector/workloads/mlir_tests/ball/matmul_at_i8_16x16.mlir @@ -0,0 +1,51 @@ +func.func private @check_result(memref<16x16xi32>) -> () +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %initial = arith.constant 7 : i32 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %two = arith.constant 2 : index + %three = arith.constant 3 : index + %five = arith.constant 5 : index + %seven = arith.constant 7 : index + %sixteen = arith.constant 16 : index + %at = memref.alloc() : memref<16x16xi8> + %b = memref.alloc() : memref<16x16xi8> + %c = memref.alloc() : memref<16x16xi32> + linalg.fill ins(%initial : i32) outs(%c : memref<16x16xi32>) + scf.for %k = %zero to %sixteen step %one { + scf.for %lane = %zero to %sixteen step %one { + %k3 = arith.muli %k, %three : index + %l5 = arith.muli %lane, %five : index + %asum = arith.addi %k3, %l5 : index + %arem = arith.remui %asum, %seven : index + %aval = arith.subi %arem, %three : index + %av = arith.index_cast %aval : index to i8 + %k2 = arith.muli %k, %two : index + %l3 = arith.muli %lane, %three : index + %bsum = arith.addi %k2, %l3 : index + %brem = arith.remui %bsum, %five : index + %bval = arith.subi %brem, %two : index + %bv = arith.index_cast %bval : index to i8 + memref.store %av, %at[%k, %lane] : memref<16x16xi8> + memref.store %bv, %b[%k, %lane] : memref<16x16xi8> + } + } + buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = true, row = 1 : i64, col = 4 : i64}> + buckyball.mvin %at <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.mvin %b <{bank = 2 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.mvin %c <{bank = 3 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi32> + // C += A^T * B with K = 16. + vector_ball.matmul_at_i8 <{at = 1 : i64, b = 2 : i64, c = 3 : i64, rows = 16 : i64}> + buckyball.mvout %c <{bank = 3 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi32> + buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + func.call @check_result(%c) : (memref<16x16xi32>) -> () + memref.dealloc %at : memref<16x16xi8> + memref.dealloc %b : memref<16x16xi8> + memref.dealloc %c : memref<16x16xi32> + return %zero_i8 : i8 +} diff --git a/examples/prefab/balls/vector/workloads/mlir_tests/ball/matmul_at_i8_16x16_main.cpp b/examples/prefab/balls/vector/workloads/mlir_tests/ball/matmul_at_i8_16x16_main.cpp new file mode 100644 index 000000000..02dafa458 --- /dev/null +++ b/examples/prefab/balls/vector/workloads/mlir_tests/ball/matmul_at_i8_16x16_main.cpp @@ -0,0 +1,22 @@ +#include +#include +#include + +extern "C" void check_result(int32_t *, int32_t *data, int64_t offset, + int64_t rows, int64_t lanes, int64_t stride, + int64_t lane_stride) { + if (rows != 16 || lanes != 16 || stride != 16 || lane_stride != 1) + exit(1); + for (int row = 0; row < 16; ++row) + for (int column = 0; column < 16; ++column) { + int32_t expected = 7; + for (int k = 0; k < 16; ++k) + expected += + ((3 * k + 5 * row) % 7 - 3) * ((2 * k + 3 * column) % 5 - 2); + if (data[offset + row * stride + column] != expected) { + printf("matmul_at_i8 mismatch row=%d column=%d\n", row, column); + exit(1); + } + } + printf("matmul_at_i8 16x16 PASS\n"); +} diff --git a/examples/prefab/balls/vector/workloads/mlir_tests/bank/CMakeLists.txt b/examples/prefab/balls/vector/workloads/mlir_tests/bank/CMakeLists.txt new file mode 100644 index 000000000..c2b433ede --- /dev/null +++ b/examples/prefab/balls/vector/workloads/mlir_tests/bank/CMakeLists.txt @@ -0,0 +1,4 @@ +set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) +set(BUCKYBALL_MLIR_TEST_PREFIX bank) + +add_buckyball_mlir_test(matmul_at_i8_bank_depth TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/prefab/balls/vector/workloads/mlir_tests/bank/matmul_at_i8_bank_depth.mlir.in b/examples/prefab/balls/vector/workloads/mlir_tests/bank/matmul_at_i8_bank_depth.mlir.in new file mode 100644 index 000000000..d331d2c54 --- /dev/null +++ b/examples/prefab/balls/vector/workloads/mlir_tests/bank/matmul_at_i8_bank_depth.mlir.in @@ -0,0 +1,57 @@ +func.func private @check_result(memref<16x16xi32>) -> () +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %initial = arith.constant -5 : i32 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %two = arith.constant 2 : index + %three = arith.constant 3 : index + %five = arith.constant 5 : index + %seven = arith.constant 7 : index + %sixteen = arith.constant 16 : index + %depth = arith.constant @BB_BANK_LINES@ : index + %at = memref.alloc() alignment = 64 : memref<@BB_BANK_LINES@x16xi8> + %b = memref.alloc() alignment = 64 : memref<@BB_BANK_LINES@x16xi8> + %c = memref.alloc() alignment = 64 : memref<16x16xi32> + linalg.fill ins(%initial : i32) outs(%c : memref<16x16xi32>) + scf.for %k = %zero to %depth step %one { + scf.for %lane = %zero to %sixteen step %one { + %k3 = arith.muli %k, %three : index + %l5 = arith.muli %lane, %five : index + %asum = arith.addi %k3, %l5 : index + %arem = arith.remui %asum, %seven : index + %aval = arith.subi %arem, %three : index + %av = arith.index_cast %aval : index to i8 + %k2 = arith.muli %k, %two : index + %l3 = arith.muli %lane, %three : index + %bsum = arith.addi %k2, %l3 : index + %brem = arith.remui %bsum, %five : index + %bval = arith.subi %brem, %two : index + %bv = arith.index_cast %bval : index to i8 + memref.store %av, %at[%k, %lane] : memref<@BB_BANK_LINES@x16xi8> + memref.store %bv, %b[%k, %lane] : memref<@BB_BANK_LINES@x16xi8> + } + } + %ab = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %bb = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %cb = buckyball.bank_alloc <{row = 1 : i64, col = 4 : i64}> + %al = buckyball.bank_mvin %at, %ab <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> + : memref<@BB_BANK_LINES@x16xi8> + %bl = buckyball.bank_mvin %b, %bb <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> + : memref<@BB_BANK_LINES@x16xi8> + %cl = buckyball.bank_mvin %c, %cb <{depth = 16 : i64, stride = 1 : i64}> + : memref<16x16xi32> + // C += A^T * B with K the bank depth. + %result = vector_ball.bank_matmul_at_i8 %al, %bl -> %cl + <{rows = @BB_BANK_LINES@ : i64}> + %stored = buckyball.bank_mvout %c, %result <{depth = 16 : i64, stride = 1 : i64}> + : memref<16x16xi32> + buckyball.bank_release %al + buckyball.bank_release %bl + buckyball.bank_release %stored + func.call @check_result(%c) : (memref<16x16xi32>) -> () + memref.dealloc %at : memref<@BB_BANK_LINES@x16xi8> + memref.dealloc %b : memref<@BB_BANK_LINES@x16xi8> + memref.dealloc %c : memref<16x16xi32> + return %zero_i8 : i8 +} diff --git a/examples/prefab/balls/vector/workloads/mlir_tests/bank/matmul_at_i8_bank_depth_main.cpp b/examples/prefab/balls/vector/workloads/mlir_tests/bank/matmul_at_i8_bank_depth_main.cpp new file mode 100644 index 000000000..3b5f06d39 --- /dev/null +++ b/examples/prefab/balls/vector/workloads/mlir_tests/bank/matmul_at_i8_bank_depth_main.cpp @@ -0,0 +1,23 @@ +#include +#include +#include + +extern "C" void check_result(int32_t *, int32_t *data, int64_t offset, + int64_t rows, int64_t lanes, int64_t stride, + int64_t lane_stride) { + if (rows != 16 || lanes != 16 || stride != 16 || lane_stride != 1) + exit(1); + for (int row = 0; row < 16; ++row) + for (int column = 0; column < 16; ++column) { + int32_t expected = -5; + for (int k = 0; k < THREAD_MEM_BANK_ENTRIES; ++k) + expected += + ((3 * k + 5 * row) % 7 - 3) * ((2 * k + 3 * column) % 5 - 2); + if (data[offset + row * stride + column] != expected) { + printf("matmul_at_i8 bank-depth mismatch row=%d column=%d\n", row, + column); + exit(1); + } + } + printf("matmul_at_i8 bank-depth PASS K=%d\n", THREAD_MEM_BANK_ENTRIES); +} diff --git a/examples/prefab/cores/boom/LICENSE b/examples/prefab/cores/boom/LICENSE new file mode 100644 index 000000000..f22c14f96 --- /dev/null +++ b/examples/prefab/cores/boom/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2012-2020, The Regents of the University of California (Regents) +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/examples/prefab/cores/boom/arch/Boom.scala b/examples/prefab/cores/boom/arch/Boom.scala new file mode 100644 index 000000000..da1074500 --- /dev/null +++ b/examples/prefab/cores/boom/arch/Boom.scala @@ -0,0 +1,202 @@ +package examples.core.boom + +import framework.acc.balldomain.bbus.BallFactory +import sims.configs.SimParam +import framework.acc.balldomain.ball.blink.BlinkIO + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import examples.core.boom.configs.Params +import framework.core.clink.{CLinkIO, HasCLink} +import framework.core.clink.configs.{Params => CLinkParam} +import framework.ip.bus.axi4 + +import framework.acc.frontend.Frontend +import framework.acc.memdomain.MemDomain +import framework.acc.gpdomain.GpDomain +import framework.acc.balldomain.BallDomain +import framework.acc.balldomain.ball.blink.HasBlink +import framework.acc.debug.Debug + +@instantiable +class Boom( + p: Params, + simp: SimParam, + factories: Map[String, BallFactory]) + extends Module + with HasCLink { + val diffTest = simp.diffTest + @public + val clink = IO(new CLinkIO(CLinkParam(1, false, Seq.empty, p.chi, p.axi, p.axi, None))) + val native = Instantiate(new Native) + native.io.clock := clock + native.io.reset := reset.asBool + native.io.io_resetVector := p.resetVector.U + native.io.io_interrupts(0) := clink.interrupts(0).software + native.io.io_interrupts(1) := clink.interrupts(0).timer + native.io.io_interrupts(2) := clink.interrupts(0).external + native.io.io_interrupts(3) := clink.interrupts(0).supervisorExternal + for ( + (up, down) <- Seq((native.io.memoryAXI(0), clink.memory.ddr), (native.io.peripheralsAXI(0), clink.memory.uncached)) + ) { + down.aw.valid := up.aw.valid + up.aw.ready := down.aw.ready + down.ar.valid := up.ar.valid + up.ar.ready := down.ar.ready + for ((dst, src) <- Seq((down.aw.bits, up.aw.bits), (down.ar.bits, up.ar.bits))) { + dst := 0.U.asTypeOf(dst) + dst.id := src.id; dst.addr := src.addr; dst.len := src.len; dst.size := src.size + dst.burst := src.burst; dst.lock := src.lock; dst.cache := src.cache; dst.prot := src.prot; dst.qos := src.qos + } + down.w <> up.w + up.b <> down.b + up.r <> down.r + } + p.acc match { + case Some(acc) => + val frontend = + Instantiate(new Frontend(acc.frontend, acc.memDomain, acc.gpDomain, acc.ballDomain, p.permission, simp)) + val memDomain = Instantiate(new MemDomain( + acc.frontend, + acc.memDomain, + acc.gpDomain, + acc.transport, + p.permission, + p.tracking, + p.walker, + simp + )) + val domain = + Instantiate(new BallDomain(acc.frontend, acc.memDomain, acc.gpDomain, acc.ballDomain, p.permission, factories)) + if (acc.gpDomain.enable) { + val gp = Instantiate(new GpDomain(acc.frontend, acc.memDomain, acc.gpDomain)) + gp.io.command <> frontend.io.gpRequest.get + frontend.io.gpCompletion.get <> gp.io.complete + gp.io.image <> memDomain.io.image.get + memDomain.io.gpBanks.get <> gp.io.banks + memDomain.io.gpLayoutRequest.get <> gp.io.layoutRequest + gp.io.layoutResponse <> memDomain.io.gpLayoutResponse.get + domain.io.kernel.get.request <> gp.io.ball.request + gp.io.ball.response <> domain.io.kernel.get.response + } + val command = native.io.io_commands.request + frontend.io.rocc.cmd.valid := command.valid + command.ready := frontend.io.rocc.cmd.ready + frontend.io.rocc.cmd.bits.instruction.raw_inst := command.bits.instruction + frontend.io.rocc.cmd.bits.instruction.pc := 0.U + frontend.io.rocc.cmd.bits.instruction.rs1Data := command.bits.rs1 + frontend.io.rocc.cmd.bits.instruction.rs2Data := command.bits.rs2 + frontend.io.rocc.cmd.bits.satp := native.io.io_context.satp + frontend.io.rocc.cmd.bits.effectivePrivilege := command.bits.privilege + frontend.io.rocc.cmd.bits.sum := command.bits.sum + frontend.io.rocc.cmd.bits.mxr := command.bits.mxr + for ((target, entry) <- frontend.io.rocc.cmd.bits.pmp.zip(native.io.io_context.pmp)) { + target.cfg := entry.cfg.asTypeOf(target.cfg) + target.addr := entry.addr + val encoded = ((entry.addr.pad(44) << 1) | entry.cfg(3).asUInt)(43, 0) + target.mask := (((encoded & ~(encoded + 1.U)) << 2) | 3.U)(43, 0) + } + frontend.io.rocc.resp.ready := true.B + val pending = frontend.io.rocc.busy + native.io.io_commands.busy := frontend.io.rocc.busy || command.valid + native.io.io_commands.fault := frontend.io.rocc.interrupt + + domain.io.issue <> frontend.io.ballRequest + frontend.io.ballCompletion <> domain.io.complete + memDomain.io.ball <> domain.io.memory + memDomain.io.issue <> frontend.io.memoryRequest + frontend.io.memoryCompletion <> memDomain.io.complete + memDomain.io.mapping <> frontend.io.mappingUpdate + frontend.io.barrier.release := frontend.io.barrier.arrive + memDomain.io.banks <> domain.io.banks + memDomain.io.ballLayoutRequest <> domain.io.layoutRequest + domain.io.layoutResponse <> memDomain.io.ballLayoutResponse + + // The native coherence hub snoops BOOM caches for DMA. The submitting core has already drained stores + // and cannot execute younger memory operations until this command completes. + memDomain.io.memory.cpuQuery := 0.U.asTypeOf(memDomain.io.memory.cpuQuery) + val maintained = RegInit(false.B) + val maintenanceTag = Reg(chiselTypeOf(memDomain.io.memory.maintenance.bits.tag)) + memDomain.io.memory.maintenance.ready := !maintained + when(memDomain.io.memory.maintenance.fire) { + assert(pending) + maintenanceTag := memDomain.io.memory.maintenance.bits.tag + maintained := true.B + } + memDomain.io.memory.maintained.valid := maintained + memDomain.io.memory.maintained.bits.tag := maintenanceTag + memDomain.io.memory.maintained.bits.ok := true.B + when(memDomain.io.memory.maintained.fire)(maintained := false.B) + + val access = Instantiate(new axi4.Interconnect(axi4.InterconnectParam(p.axi, 2, Some(16)))) + access.io.in(0) <> memDomain.io.ddr + val pte = access.io.in(1) + val ptePending = RegInit(false.B) + val highWord = Reg(Bool()) + pte.aw.valid := false.B + pte.aw.bits := 0.U.asTypeOf(pte.aw.bits) + pte.w.valid := false.B + pte.w.bits := 0.U.asTypeOf(pte.w.bits) + pte.b.ready := true.B + pte.ar.valid := memDomain.io.memory.pteRequest.valid && !ptePending + pte.ar.bits := 0.U.asTypeOf(pte.ar.bits) + pte.ar.bits.addr := memDomain.io.memory.pteRequest.bits.addr + pte.ar.bits.size := 3.U + pte.ar.bits.burst := 1.U + memDomain.io.memory.pteRequest.ready := pte.ar.ready && !ptePending + when(pte.ar.fire) { ptePending := true.B; highWord := pte.ar.bits.addr(3) } + memDomain.io.memory.pteResponse.valid := pte.r.valid && ptePending + memDomain.io.memory.pteResponse.bits.data := Mux(highWord, pte.r.bits.data(127, 64), pte.r.bits.data(63, 0)) + memDomain.io.memory.pteResponse.bits.error := pte.r.bits.resp =/= 0.U + pte.r.ready := memDomain.io.memory.pteResponse.ready && ptePending + when(pte.r.fire) { assert(pte.r.bits.last && pte.r.bits.id === 0.U); ptePending := false.B } + val dma = native.io.dmaAXI(0) + val upstream = access.io.out + dma.aw.valid := upstream.aw.valid + upstream.aw.ready := dma.aw.ready + dma.ar.valid := upstream.ar.valid + upstream.ar.ready := dma.ar.ready + for ((dst, src) <- Seq((dma.aw.bits, upstream.aw.bits), (dma.ar.bits, upstream.ar.bits))) { + dst.id := src.id; dst.addr := src.addr; dst.len := src.len; dst.size := src.size + dst.burst := src.burst; dst.lock := src.lock; dst.cache := 15.U; dst.prot := src.prot; dst.qos := src.qos + } + dma.w <> upstream.w + upstream.b <> dma.b + upstream.r <> dma.r + + val debug = Instantiate(new Debug(acc.frontend, acc.memDomain, acc.gpDomain, acc.ballDomain, simp, 0, 0)) + debug.io.ant.valid := false.B + debug.io.ant.bits := 0.U.asTypeOf(debug.io.ant.bits) + debug.io.acc.frontend <> frontend.io.debug + debug.io.acc.trace.memory := memDomain.io.traceMemory + debug.io.acc.trace.ballStarted := domain.io.traceStarted + debug.io.acc.trace.ballCompleted := domain.io.traceCompleted + debug.io.acc.banks.zip(memDomain.io.bank_hashes).foreach { case (sink, src) => sink := src } + for (port <- memDomain.io.shared.bank.requests) { + port.read.req.ready := false.B + port.read.resp.valid := false.B + port.read.resp.bits := 0.U.asTypeOf(port.read.resp.bits) + port.write.req.ready := false.B + port.write.resp.valid := false.B + port.write.resp.bits := 0.U.asTypeOf(port.write.resp.bits) + } + memDomain.io.shared.bank.config.ready := false.B + memDomain.io.shared.bank.mapping.ready := false.B + memDomain.io.shared.bank.local.request.valid := false.B + memDomain.io.shared.bank.local.request.bits := 0.U.asTypeOf(memDomain.io.shared.bank.local.request.bits) + memDomain.io.shared.bank.local.response.ready := false.B + memDomain.io.shared.move.command.ready := false.B + memDomain.io.shared.move.completion.valid := false.B + memDomain.io.shared.move.completion.bits := false.B + case None => + native.io.io_commands.request.ready := true.B + native.io.io_commands.busy := false.B + native.io.io_commands.fault := RegNext(native.io.io_commands.request.fire, false.B) + val dma = native.io.dmaAXI(0) + dma.aw.valid := false.B; dma.aw.bits := 0.U.asTypeOf(dma.aw.bits) + dma.w.valid := false.B; dma.w.bits := 0.U.asTypeOf(dma.w.bits) + dma.ar.valid := false.B; dma.ar.bits := 0.U.asTypeOf(dma.ar.bits) + dma.b.ready := true.B; dma.r.ready := true.B + } +} diff --git a/examples/prefab/cores/boom/arch/Native.scala b/examples/prefab/cores/boom/arch/Native.scala new file mode 100644 index 000000000..191fd322d --- /dev/null +++ b/examples/prefab/cores/boom/arch/Native.scala @@ -0,0 +1,75 @@ +package examples.core.boom + +import chisel3.experimental.hierarchy.{instantiable, public} + +import chisel3._ +import chisel3.util._ +import framework.ip.bus.axi4 +import java.nio.file.Paths +import scala.io.Source +import scala.util.Using + +object Native { + + class Address(bits: Int) extends Bundle { + val id = UInt(4.W) + val addr = UInt(bits.W) + val len = UInt(8.W) + val size = UInt(3.W) + val burst = UInt(2.W) + val lock = Bool() + val cache = UInt(4.W) + val prot = UInt(3.W) + val qos = UInt(4.W) + } + + class Bus(bits: Int) extends Bundle { + val aw = Decoupled(new Address(bits)) + val w = Decoupled(new axi4.WriteData(axi4.Params(bits, 128, 4))) + val b = Flipped(Decoupled(new axi4.WriteResponse(axi4.Params(bits, 128, 4)))) + val ar = Decoupled(new Address(bits)) + val r = Flipped(Decoupled(new axi4.ReadData(axi4.Params(bits, 128, 4)))) + } + +} + +@instantiable +class Native extends BlackBox with HasBlackBoxPath { + override def desiredName = "BoomNative" + + @public + val io = IO(new Bundle { + val clock = Input(Clock()) + val reset = Input(Bool()) + val io_resetVector = Input(UInt(44.W)) + val io_interrupts = Input(Vec(4, Bool())) + + val io_commands = new Bundle { + + val request = Decoupled(new Bundle { + val instruction = UInt(32.W) + val rs1 = UInt(64.W) + val rs2 = UInt(64.W) + val privilege = UInt(2.W) + val sum = Bool() + val mxr = Bool() + }) + + val busy = Input(Bool()) + val fault = Input(Bool()) + } + + val io_context = Output(new Bundle { + val satp = UInt(64.W) + val pmp = Vec(8, new Bundle { val cfg = UInt(8.W); val addr = UInt(42.W) }) + }) + + val memoryAXI = Vec(1, new Native.Bus(44)) + val peripheralsAXI = Vec(1, new Native.Bus(31)) + val dmaAXI = Flipped(Vec(1, new Native.Bus(44))) + }) + + val manifest = Paths.get(getClass.getResource("/boom/filelist.f").toURI) + val files = Using.resource(Source.fromFile(manifest.toFile))(_.getLines().filter(_.nonEmpty).toVector) + files.foreach(file => addPath(manifest.getParent.resolve(file.stripPrefix("./")).toString)) +} diff --git a/examples/prefab/cores/boom/arch/build.sc b/examples/prefab/cores/boom/arch/build.sc new file mode 100644 index 000000000..aff61a1bd --- /dev/null +++ b/examples/prefab/cores/boom/arch/build.sc @@ -0,0 +1,49 @@ +import mill._ +import mill.scalalib._ +import $file.^.^.^.^.^.arch.src.main.scala.framework.{build => framework} +val root = millSourcePath / os.up +val rc = root / "thirdparty" / "rocket-chip" +val boomRoot = root / "thirdparty" / "riscv-boom" +trait ScalaBuild extends SbtModule { + override def scalaVersion = "2.13.16" + override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") +} +object cde extends ScalaBuild { override def sources = T.sources(rc / os.RelPath("dependencies/cde/cde/src")) } +object macros extends ScalaBuild { + override def sources = T.sources(rc / os.RelPath("macros/src/main/scala")) + override def ivyDeps = super.ivyDeps() ++ Agg(ivy"org.scala-lang:scala-reflect:2.13.16") +} +object diplomacy extends ScalaBuild { + override def sources = T.sources(rc / os.RelPath("dependencies/diplomacy/diplomacy/src")) + override def moduleDeps = Seq(cde) + override def ivyDeps = super.ivyDeps() ++ Agg(ivy"com.lihaoyi::sourcecode:0.3.1") +} +object hardfloat extends ScalaBuild { override def sources = T.sources(rc / os.RelPath("dependencies/hardfloat/hardfloat/src/main/scala")) } +object rocketchip extends ScalaBuild { + override def sources = T.sources(rc / os.RelPath("src/main/scala")) + override def resources = T.sources(rc / os.RelPath("src/main/resources")) + override def moduleDeps = Seq(cde, macros, diplomacy, hardfloat) + override def ivyDeps = super.ivyDeps() ++ Agg(ivy"com.lihaoyi::mainargs:0.5.0", ivy"org.json4s::json4s-jackson:4.0.5") +} +object native extends ScalaBuild { + override def sources = T.sources(boomRoot / os.RelPath("src/main/scala/v4"), root / "arch" / "native") + override def resources = T.sources(boomRoot / os.RelPath("src/main/resources")) + override def moduleDeps = Seq(rocketchip) + override def forkArgs = Seq("-Xmx32G", "-Xss64m") +} +def nativeRtl = T { + val cp = native.runClasspath().map(_.path) + os.proc("java", "-Xmx32G", "-Xss64m", "-cp", cp.mkString(java.io.File.pathSeparator), + "examples.core.boom.native.Emit", "--target-dir", T.dest / "boom").call(cwd = root) + PathRef(T.dest) +} +object library extends ScalaBuild { + override def moduleDeps = Seq(framework.library) + override def sources = T.sources { + os.walk(root / "arch").filter(p => p.ext == "scala" && !p.startsWith(root / "arch" / "native")).map(PathRef(_)) ++ + Seq(PathRef(root / "configs")) + } + override def resources = T.sources(Seq(nativeRtl())) +} diff --git a/examples/prefab/cores/boom/arch/native/Core.scala b/examples/prefab/cores/boom/arch/native/Core.scala new file mode 100644 index 000000000..8f3b41089 --- /dev/null +++ b/examples/prefab/cores/boom/arch/native/Core.scala @@ -0,0 +1,1516 @@ +// Copied from riscv-boom src/main/scala/v4/exu/core.scala and modified for Buckyball. +// Copyright (c) The Regents of the University of California (Regents). +// All Rights Reserved. See LICENSE and LICENSE.SiFive in riscv-boom for license details. + +package examples.core.boom.native + +import java.nio.file.{Paths} + +import chisel3._ +import chisel3.util._ + +import org.chipsalliance.cde.config.Parameters +import freechips.rocketchip.rocket.Instructions._ +import freechips.rocketchip.rocket.{CSR, CSRs, Causes, PRV, TracedInstruction} +import freechips.rocketchip.tile.{CustomCSR, HasFPUParameters, TraceBundle} +import freechips.rocketchip.util.{CoreMonitorBundle, PlusArg, Str, UIntIsOneOf} +import freechips.rocketchip.devices.tilelink.{CLINTConsts, PLICConsts} +import freechips.rocketchip.trace.{TraceCoreIngress, TraceCoreInterface, TraceCoreParams} + +import boom.v4.common._ +import boom.v4.ifu.{GlobalHistory, HasBoomFrontendParameters} +import boom.v4.util._ +import boom.v4.exu._ +import examples.core.boom.native.{DecodeUnit, UniqueExeUnit} + +/** + * Top level core object that connects the Frontend to the rest of the pipeline. + */ +class BoomCore(roccCSRs: Seq[Seq[CustomCSR]])(implicit p: Parameters) + extends BoomModule + with HasBoomFrontendParameters // TODO: Don't add this trait + with HasFPUParameters { + val nTotalRoCCCSRs = roccCSRs.flatten.size + + val traceIngressParams = TraceCoreParams( + nGroups = boomParams.retireWidth, + iretireWidth = 1, + xlen = coreParams.xLen, + iaddrWidth = vaddrBitsExtended + ) + + val io = IO(new freechips.rocketchip.tile.CoreBundle { + val hartid = Input(UInt(hartIdLen.W)) + val interrupts = Input(new freechips.rocketchip.rocket.CoreInterrupts(false)) + val ifu = new boom.v4.ifu.BoomFrontendIO + val ptw = Flipped(new freechips.rocketchip.rocket.DatapathPTWIO()) + val rocc = Flipped(new freechips.rocketchip.tile.RoCCCoreIO(nTotalRoCCCSRs)) + val lsu = Flipped(new boom.v4.lsu.LSUCoreIO) + val ptw_tlb = new freechips.rocketchip.rocket.TLBPTWIO() + val trace = Output(new TraceBundle) + val fcsr_rm = UInt(freechips.rocketchip.tile.FPConstants.RM_SZ.W) + val traceStall = Input(Bool()) + val trace_core_ingress = + if (boomParams.enableTraceCoreIngress) Some(Output(new TraceCoreInterface(traceIngressParams))) else None + }) + + io.ptw_tlb := DontCare + io.ptw := DontCare + io.ifu := DontCare + + //********************************** + // construct all of the modules + + val mem_exe_units: Seq[MemExeUnit] = (0 until memWidth) map { w => + Module(new MemExeUnit( + hasAGen = w >= (memWidth - lsuWidth), + hasDGen = true + )).suggestName(s"mem_exe_unit_$w") + } + + val agen_exe_units: Seq[MemExeUnit] = mem_exe_units.filter(_.hasAGen) + + val unq_exe_units: Seq[UniqueExeUnit] = Seq( + Module(new UniqueExeUnit( + hasCSR = true, + hasRocc = usingRoCC, + hasMul = true, + hasDiv = true, + hasIfpu = true + )).suggestName(s"unique_exe_unit_0") + ) + + val unq_exe_unit = unq_exe_units(0) + val csr_resp = unq_exe_unit.io_csr_resp.get + + val alu_exe_units: Seq[ALUExeUnit] = (0 until aluWidth) map { w => + Module(new ALUExeUnit( + id = w + )).suggestName(s"alu_exe_unit_$w") + } + + val all_exe_units = alu_exe_units ++ mem_exe_units ++ unq_exe_units + + // Meanwhile, the FP pipeline holds the FP issue window, FP regfile, and FP arithmetic units. + val fp_pipeline = Module(new FpPipeline) + + // ******************************************************** + // Clear fp_pipeline before use + fp_pipeline.io.ll_wports := DontCare + + val numIrfWritePorts = aluWidth + lsuWidth + 1 + val numIrfLogicalReadPorts = all_exe_units.map(_.nReaders).reduce(_ + _) + + val numIntWakeups = coreWidth + lsuWidth + 1 + val numFpWakeupPorts = fp_pipeline.io.wakeups.length + + val numImmReaders = aluWidth + memWidth + 1 // "Wakeup" immediates when they are read + + val decode_units = (0 until decodeWidth) map { w => Module(new DecodeUnit).suggestName(s"decode_$w") } + val dec_brmask_logic = Module(new BranchMaskGenerationLogic(coreWidth)) + val rename_stage = Module(new RenameStage(coreWidth, numIntPhysRegs, numIntWakeups, false)) + val fp_rename_stage = Module(new RenameStage(coreWidth, numFpPhysRegs, numFpWakeupPorts, true)) + val pred_rename_stage = Module(new PredRenameStage(coreWidth, 1)) + val imm_rename_stage = Module(new ImmRenameStage(coreWidth, numImmReaders)) // wakeup ports used when insts read imm + val rename_stages = + Seq(rename_stage, pred_rename_stage, imm_rename_stage) ++ (if (usingFPU) Seq(fp_rename_stage) else Nil) + + val mem_iss_unit = IssueUnit(memIssueParam, numIntWakeups, false, false) + val unq_iss_unit = IssueUnit(unqIssueParam, numIntWakeups, false, false) + val alu_iss_unit = IssueUnit(aluIssueParam, numIntWakeups, enableColumnALUIssue, enableALUSingleWideDispatch) + val dispatcher = Module(new BasicDispatcher) + + val iregfileBankedWriteArray = Seq.fill(lsuWidth + 1)(None) ++ (0 until aluWidth).map { w => + if (enableColumnALUWrites) Some(w) else None + } + + val iregfile = Module(new BankedRF( + UInt(xLen.W), + numIrfBanks, + numIrfLogicalReadPorts, + numIntPhysRegs, + numIrfLogicalReadPorts, + numIrfReadPorts, + numIrfWritePorts, + iregfileBankedWriteArray, + "Integer" + )) + + val pregfile = Module(new FullyPortedRF( + Bool(), + ftqSz, + aluWidth, + 1, + "Predicate" + )) + + val immregfile = Module(new FullyPortedRF( + UInt(LONGEST_IMM_SZ.W), + numImmPhysRegs, + numImmReaders, + coreWidth, + "Immediate" + )) + + val bregfile = Module(new FullyPortedRF( + new BrInfoBundle, + maxBrCount, + aluWidth, + coreWidth, + "Branch" + )) + + val rob = Module(new Rob( + numIrfWritePorts + numFpWakeupPorts, + trace + )) + + // Used to wakeup registers in rename and issue. ROB needs to listen to something else. + val int_wakeups = Wire(Vec(numIntWakeups, Valid(new Wakeup))) + val pred_wakeups = Wire(Vec(aluWidth, Valid(new Wakeup))) + + // The arb stage guarantees only 1 pred wakeup per cycle + assert(PopCount(pred_wakeups.map(_.valid)) <= 1.U) + val pred_wakeup = Wire(Valid(new Wakeup)) + pred_wakeup.valid := pred_wakeups.map(_.valid).reduce(_ || _) + pred_wakeup.bits := DontCare + pred_wakeup.bits.uop.pdst := Mux1H(pred_wakeups.map(_.valid), pred_wakeups.map(_.bits.uop.pdst)) + + val int_bypasses = Wire(Vec(coreWidth + lsuWidth, Valid(new ExeUnitResp(xLen)))) + + //*********************************** + // Pipeline State Registers and Wires + + // Decode/Rename1 Stage + val dec_valids = Wire(Vec(coreWidth, Bool())) // are the decoded instruction valid? It may be held up though. + val dec_uops = Wire(Vec(coreWidth, new MicroOp())) + val dec_fire = Wire(Vec(coreWidth, Bool())) // can the instruction fire beyond decode? + // (can still be stopped in ren or dis) + val dec_ready = Wire(Bool()) + val dec_xcpts = Wire(Vec(coreWidth, Bool())) + val ren_stalls = Wire(Vec(coreWidth, Bool())) + + // Rename2/Dispatch stage + val dis_valids = Wire(Vec(coreWidth, Bool())) + val dis_uops = Wire(Vec(coreWidth, new MicroOp)) + val dis_fire = Wire(Vec(coreWidth, Bool())) + val dis_ready = Wire(Bool()) + + // Issue Stage/Register Read + val mem_iss_uops = mem_iss_unit.io.iss_uops + val alu_iss_uops = alu_iss_unit.io.iss_uops + val unq_iss_uops = unq_iss_unit.io.iss_uops + + // -------------------------------------- + // Dealing with branch resolutions + + // The individual branch resolutions from each ALU + val brinfos = Reg(Vec(coreWidth, Valid(new BrResolutionInfo))) + + // "Merged" branch update info from all ALUs + // brmask contains masks for rapidly clearing mispredicted instructions + // brindices contains indices to reset pointers for allocated structures + // brindices is delayed a cycle + val brupdate = Wire(new BrUpdateInfo) + val b1 = Wire(new BrUpdateMasks) + val b2 = Reg(new BrResolutionInfo) + + brupdate.b1 := b1 + brupdate.b2 := b2 + + for ((b, a) <- brinfos zip alu_exe_units) { + b.bits := UpdateBrMask(brupdate, a.io_brinfo.bits) + b.valid := a.io_brinfo.valid && !rob.io.flush.valid && !IsKilledByBranch( + brupdate, + RegNext(rob.io.flush.valid), + a.io_brinfo.bits + ) + } + b1.resolve_mask := brinfos.map(x => x.valid << x.bits.uop.br_tag).reduce(_ | _) + b1.mispredict_mask := brinfos.map(x => (x.valid && x.bits.mispredict) << x.bits.uop.br_tag).reduce(_ | _) + + // Find the oldest mispredict and use it to update indices + val live_brinfos = brinfos.map(br => + br.valid && br.bits.mispredict && !IsKilledByBranch(brupdate, RegNext(rob.io.flush.valid), br.bits.uop) + ) + + val mispredict_val = live_brinfos.reduce(_ || _) + + b2.mispredict := mispredict_val + b2.cfi_type := Mux1H(live_brinfos, brinfos.map(_.bits.cfi_type)) + b2.taken := Mux1H(live_brinfos, brinfos.map(_.bits.taken)) + b2.pc_sel := Mux1H(live_brinfos, brinfos.map(_.bits.pc_sel)) + b2.uop := UpdateBrMask(brupdate, PriorityMux(live_brinfos, brinfos.map(_.bits.uop))) + b2.jalr_target := Mux1H(live_brinfos, brinfos.map(_.bits.jalr_target)) + b2.target_offset := Mux1H(live_brinfos, brinfos.map(_.bits.target_offset)) + + val oldest_mispredict_ftq_idx = Mux1H(live_brinfos, brinfos.map(_.bits.uop.ftq_idx)) + + assert( + !((brupdate.b1.mispredict_mask =/= 0.U || brupdate.b2.mispredict) + && rob.io.rollback), + "Can't have a mispredict during rollback." + ) + + io.ifu.brupdate := brupdate + + for (eu <- all_exe_units) { + eu.io_brupdate := brupdate + } + + fp_pipeline.io.brupdate := brupdate + + // Load/Store Unit & ExeUnits + // val mem_resps = io.lsu.iresp + var agen_idx = 0 + var dgen_idx = 0 + for (eu <- mem_exe_units) { + if (eu.hasAGen) { + io.lsu.agen(agen_idx) := eu.io_agen.get + agen_idx += 1 + } + if (eu.hasDGen) { + io.lsu.dgen(dgen_idx) := eu.io_dgen.get + dgen_idx += 1 + } + } + io.lsu.dgen(dgen_idx) := fp_pipeline.io.dgen + + //------------------------------------------------------------- + // Uarch Hardware Performance Events (HPEs) + + val perfEvents = new freechips.rocketchip.rocket.EventSets(Seq( + new freechips.rocketchip.rocket.EventSet( + (mask, hits) => (mask & hits).orR, + Seq( + ("exception", () => rob.io.com_xcpt.valid), + ("nop", () => false.B), + ("nop", () => false.B), + ("nop", () => false.B) + ) + ), + new freechips.rocketchip.rocket.EventSet( + (mask, hits) => (mask & hits).orR, + Seq( +// ("I$ blocked", () => icache_blocked), + ("nop", () => false.B), + ("branch misprediction", () => b2.mispredict), + ( + "control-flow target misprediction", + () => + b2.mispredict && + b2.cfi_type === CFI_JALR + ), + ("flush", () => rob.io.flush.valid) +// ("branch resolved", () => b2.valid) + ) + ), + new freechips.rocketchip.rocket.EventSet( + (mask, hits) => (mask & hits).orR, + Seq( + ("I$ miss", () => io.ifu.perf.acquire), + ("D$ miss", () => io.lsu.perf.acquire), + ("D$ release", () => io.lsu.perf.release), + ("ITLB miss", () => io.ifu.perf.tlbMiss), + ("DTLB miss", () => io.lsu.perf.tlbMiss), + ("L2 TLB miss", () => io.ptw.perf.l2miss) + ) + ) + )) + + val csr = Module(new freechips.rocketchip.rocket.CSRFile(perfEvents, boomParams.customCSRs.decls, roccCSRs.flatten)) + csr.io.inst foreach { c => c := DontCare } + csr.io.rocc_interrupt := io.rocc.interrupt + csr.io.gva := DontCare + csr.io.htval := DontCare + csr.io.mhtinst_read_pseudo := false.B + + val custom_csrs = Wire(new BoomCustomCSRs) + custom_csrs.csrs.foreach { c => c.stall := false.B; c.set := false.B; c.sdata := DontCare } + (custom_csrs.csrs zip csr.io.customCSRs).map { case (lhs, rhs) => lhs <> rhs } + io.ifu.enable_bpd := custom_csrs.enableBPD + + //val icache_blocked = !(io.ifu.fetchpacket.valid || RegNext(io.ifu.fetchpacket.valid)) + val icache_blocked = false.B + csr.io.counters foreach { c => c.inc := RegNext(perfEvents.evaluate(c.eventSel)) } + + //**************************************** + // Time Stamp Counter & Retired Instruction Counter + // (only used for printf and vcd dumps - the actual counters are in the CSRFile) + val debug_tsc_reg = RegInit(0.U(xLen.W)) + val debug_irt_reg = RegInit(0.U(xLen.W)) + val debug_brs = RegInit(VecInit(Seq.fill(5)(0.U(xLen.W)))) + val debug_jals = RegInit(VecInit(Seq.fill(5)(0.U(xLen.W)))) + val debug_jalrs = RegInit(VecInit(Seq.fill(5)(0.U(xLen.W)))) + + for (j <- 0 until 5) { + debug_brs(j) := debug_brs(j) + PopCount(VecInit((0 until coreWidth) map { i => + rob.io.commit.arch_valids(i) && + (rob.io.commit.uops(i).debug_fsrc === j.U) && + rob.io.commit.uops(i).is_br + })) + debug_jals(j) := debug_jals(j) + PopCount(VecInit((0 until coreWidth) map { i => + rob.io.commit.arch_valids(i) && + (rob.io.commit.uops(i).debug_fsrc === j.U) && + rob.io.commit.uops(i).is_jal + })) + debug_jalrs(j) := debug_jalrs(j) + PopCount(VecInit((0 until coreWidth) map { i => + rob.io.commit.arch_valids(i) && + (rob.io.commit.uops(i).debug_fsrc === j.U) && + rob.io.commit.uops(i).is_jalr + })) + } + + dontTouch(debug_brs) + dontTouch(debug_jals) + dontTouch(debug_jalrs) + + debug_tsc_reg := debug_tsc_reg + 1.U + debug_irt_reg := debug_irt_reg + PopCount(rob.io.commit.arch_valids.asUInt) + dontTouch(debug_tsc_reg) + dontTouch(debug_irt_reg) + + //**************************************** + // Print-out information about the machine + + val issStr = "(Age-based Priority)" + + val fpPipelineStr = fp_pipeline.toString + + override def toString: String = + (BoomCoreStringPrefix("====Overall Core Params====") + "\n\n" + + mem_exe_units.map(_.toString).mkString("") + "\n" + + unq_exe_units.map(_.toString).mkString("") + "\n\n" + + alu_exe_units.map(_.toString).mkString("") + "\n\n" + + fpPipelineStr + "\n\n" + + rob.toString + "\n\n" + + BoomCoreStringPrefix( + "===Other Core Params===", + "Fetch Width : " + fetchWidth, + "Decode Width : " + coreWidth, + "Issue Width : " + issueParams.map(_.issueWidth).sum, + "ROB Size : " + numRobEntries, + "Issue Window Size : " + issueParams.map(_.numEntries) + issStr, + "Load/Store Unit Size : " + numLdqEntries + "/" + numStqEntries, + "Num Int Phys Registers: " + numIntPhysRegs, + "Num FP Phys Registers: " + numFpPhysRegs, + "Load-to-use delay : " + (if (enableFastLoadUse) 4 else 5), + "Max Branch Count : " + maxBrCount + ) + + "\n\n" + iregfile.toString + "\n\n" + + BoomCoreStringPrefix( + "Num Wakeup Ports : " + numIntWakeups, + "Num Bypass Ports : " + int_bypasses.length + ) + "\n" + + BoomCoreStringPrefix( + "DCache Ways : " + dcacheParams.nWays, + "DCache Sets : " + dcacheParams.nSets, + "DCache nMSHRs : " + dcacheParams.nMSHRs, + "ICache Ways : " + icacheParams.nWays, + "ICache Sets : " + icacheParams.nSets, + "D-TLB Ways : " + dcacheParams.nTLBWays, + "I-TLB Ways : " + icacheParams.nTLBWays, + "Paddr Bits : " + paddrBits, + "Vaddr Bits : " + vaddrBits + ) + "\n" + + BoomCoreStringPrefix( + "Using FPU Unit? : " + usingFPU.toString, + "Using FDivSqrt? : " + usingFDivSqrt.toString, + "Using VM? : " + usingVM.toString + ) + "\n") + + //------------------------------------------------------------- + //------------------------------------------------------------- + // **** Fetch Stage/Frontend **** + //------------------------------------------------------------- + //------------------------------------------------------------- + io.ifu.redirect_val := false.B + io.ifu.redirect_flush := false.B + + // Breakpoint info + io.ifu.status := csr.io.status + io.ifu.bp := csr.io.bp + io.ifu.mcontext := csr.io.mcontext + io.ifu.scontext := csr.io.scontext + + io.ifu.flush_icache := (0 until coreWidth).map { i => + (rob.io.commit.arch_valids(i) && rob.io.commit.uops(i).is_fencei) || + (RegNext(dec_valids(i) && dec_uops(i).is_jalr && csr.io.status.debug)) + }.reduce(_ || _) + + // TODO FIX THIS HACK + // The below code works because of two quirks with the flush mechanism + // 1 ) All flush_on_commit instructions are also is_unique, + // In the future, this constraint will be relaxed. + // 2 ) We send out flush signals one cycle after the commit signal. We need to + // mux between one/two cycle delay for the following cases: + // ERETs are reported to the CSR two cycles before we send the flush + // Exceptions are reported to the CSR on the cycle we send the flush + // This discrepency should be resolved elsewhere. + when(RegNext(rob.io.flush.valid)) { + io.ifu.redirect_val := true.B + io.ifu.redirect_flush := true.B + val flush_typ = RegNext(rob.io.flush.bits.flush_typ) + // Clear the global history when we flush the ROB (exceptions, AMOs, unique instructions, etc.) + val new_ghist = WireInit((0.U).asTypeOf(new GlobalHistory)) + new_ghist.current_saw_branch_not_taken := true.B + new_ghist.ras_idx := io.ifu.rrd_ftq_resps(0).entry.ras_idx + io.ifu.redirect_ghist := new_ghist + when(FlushTypes.useCsrEvec(flush_typ)) { + io.ifu.redirect_pc := Mux(flush_typ === FlushTypes.eret, ShiftRegister(csr.io.evec, 3), csr.io.evec) + }.otherwise { + val flush_pc = (AlignPCToBoundary(io.ifu.rrd_ftq_resps(0).pc, icBlockBytes) + + RegNext(rob.io.flush.bits.pc_lob) + - Mux(RegNext(rob.io.flush.bits.edge_inst), 2.U, 0.U)) + val flush_pc_next = flush_pc + Mux(RegNext(rob.io.flush.bits.is_rvc), 2.U, 4.U) + io.ifu.redirect_pc := Mux(FlushTypes.useSamePC(flush_typ), flush_pc, flush_pc_next) + + } + io.ifu.redirect_ftq_idx := RegNext(rob.io.flush.bits.ftq_idx) + }.elsewhen(brupdate.b2.mispredict && !RegNext(rob.io.flush.valid)) { + val block_pc = AlignPCToBoundary(io.ifu.rrd_ftq_resps(0).pc, icBlockBytes) + val uop_maybe_pc = block_pc | brupdate.b2.uop.pc_lob + val npc = uop_maybe_pc + Mux(brupdate.b2.uop.is_rvc || brupdate.b2.uop.edge_inst, 2.U, 4.U) + val jal_br_target = Wire(UInt(vaddrBitsExtended.W)) + jal_br_target := (uop_maybe_pc.asSInt + brupdate.b2.target_offset + + (Fill(vaddrBitsExtended - 1, brupdate.b2.uop.edge_inst) << 1).asSInt).asUInt + val bj_addr = Mux(brupdate.b2.cfi_type === CFI_JALR, brupdate.b2.jalr_target, jal_br_target) + val mispredict_target = Mux(brupdate.b2.pc_sel === PC_PLUS4, npc, bj_addr) + io.ifu.redirect_val := true.B + io.ifu.redirect_pc := mispredict_target + io.ifu.redirect_flush := true.B + io.ifu.redirect_ftq_idx := brupdate.b2.uop.ftq_idx + val use_same_ghist = brupdate.b2.cfi_type === CFI_BR && + !brupdate.b2.taken && + bankAlign(block_pc) === bankAlign(npc) + val ftq_entry = io.ifu.rrd_ftq_resps(0).entry + val cfi_idx = (brupdate.b2.uop.pc_lob ^ + Mux(ftq_entry.start_bank === 1.U, 1.U << log2Ceil(bankBytes), 0.U))(log2Ceil(fetchWidth), 1) + val ftq_ghist = io.ifu.rrd_ftq_resps(0).ghist + val next_ghist = ftq_ghist.update( + ftq_entry.br_mask.asUInt, + brupdate.b2.taken, + brupdate.b2.cfi_type === CFI_BR, + cfi_idx, + true.B, + io.ifu.rrd_ftq_resps(0).pc, + ftq_entry.cfi_is_call && ftq_entry.cfi_idx.bits === cfi_idx, + ftq_entry.cfi_is_ret && ftq_entry.cfi_idx.bits === cfi_idx + ) + + io.ifu.redirect_ghist := Mux( + use_same_ghist, + ftq_ghist, + next_ghist + ) + io.ifu.redirect_ghist.current_saw_branch_not_taken := use_same_ghist + }.elsewhen(rob.io.flush_frontend || brupdate.b1.mispredict_mask =/= 0.U) { + io.ifu.redirect_flush := true.B + } + + // Tell the FTQ it can deallocate entries by passing youngest ftq_idx. + val youngest_com_idx = (coreWidth - 1).U - PriorityEncoder(rob.io.commit.valids.reverse) + io.ifu.commit.valid := rob.io.commit.valids.reduce(_ | _) || rob.io.com_xcpt.valid + io.ifu.commit.bits := Mux( + rob.io.com_xcpt.valid, + rob.io.com_xcpt.bits.ftq_idx, + rob.io.commit.uops(youngest_com_idx).ftq_idx + ) + + assert(!(rob.io.commit.valids.reduce(_ | _) && rob.io.com_xcpt.valid), "ROB can't commit and except in same cycle!") + + //------------------------------------------------------------- + //------------------------------------------------------------- + // **** Decode Stage **** + //------------------------------------------------------------- + //------------------------------------------------------------- + + // track mask of finished instructions in the bundle + // use this to mask out insts coming from FetchBuffer that have been finished + // for example, back pressure may cause us to only issue some instructions from FetchBuffer + // but on the next cycle, we only want to retry a subset + val dec_finished_mask = RegInit(0.U(coreWidth.W)) + + //------------------------------------------------------------- + // Pull out instructions and send to the Decoders + + io.ifu.fetchpacket.ready := dec_ready + val dec_fbundle = io.ifu.fetchpacket.bits + + //------------------------------------------------------------- + // Decoders + + for (w <- 0 until coreWidth) { + dec_valids(w) := io.ifu.fetchpacket.valid && dec_fbundle.uops(w).valid && + !dec_finished_mask(w) + decode_units(w).io.enq.uop := dec_fbundle.uops(w).bits + decode_units(w).io.status := csr.io.status + decode_units(w).io.csr_decode <> csr.io.decode(w) + decode_units(w).io.interrupt := RegNext(csr.io.interrupt) + decode_units(w).io.interrupt_cause := RegNext(csr.io.interrupt_cause) + decode_units(w).io.fcsr_rm := csr.io.fcsr_rm + + dec_uops(w) := decode_units(w).io.deq.uop + } + + //------------------------------------------------------------- + // FTQ GetPC Port Arbitration + // 3 ports + // port0 goes to flush,mispredict,xcpt + // port1/2 goes to jmp unit + + val xcpt_pc_req = Wire(Decoupled(UInt(log2Ceil(ftqSz).W))) + val mispredict_pc_req = Wire(Decoupled(UInt(log2Ceil(ftqSz).W))) + val flush_pc_req = Wire(Decoupled(UInt(log2Ceil(ftqSz).W))) + + val ftq_arb = Module(new Arbiter(UInt(log2Ceil(ftqSz).W), 3)) + + // Order by the oldest. Flushes come from the oldest instructions in pipe + // Decoding exceptions come from youngest + ftq_arb.io.in(0) <> flush_pc_req + ftq_arb.io.in(1) <> mispredict_pc_req + ftq_arb.io.in(2) <> xcpt_pc_req + + io.ifu.arb_ftq_reqs(0) := ftq_arb.io.out.bits + ftq_arb.io.out.ready := true.B + + val ftq_port_issued = Array.fill(2)(false.B) + val ftq_port_addrs = Array.fill(2)(0.U(log2Ceil(ftqSz).W)) + for (i <- 0 until aluWidth) { + for (w <- 0 until 2) { + val req = alu_exe_units(i).io_arb_ftq_reqs(w) + var read_issued = false.B + val data_sel = WireInit(0.U(2.W)) + for (j <- 0 until 2) { + val issue_read = WireInit(false.B) + val use_port = WireInit(false.B) + when(!read_issued && !ftq_port_issued(j) && req.valid) { + issue_read := true.B + use_port := true.B + data_sel := UIntToOH(j.U) + } + val was_port_issued_yet = ftq_port_issued(j) + ftq_port_issued(j) = use_port || ftq_port_issued(j) + ftq_port_addrs(j) = ftq_port_addrs(j) | Mux(was_port_issued_yet || !use_port, 0.U, req.bits) + read_issued = issue_read || read_issued + } + req.ready := read_issued + alu_exe_units(i).io_rrd_ftq_resps(w) := Mux(RegNext(data_sel(0)), io.ifu.rrd_ftq_resps(1), io.ifu.rrd_ftq_resps(2)) + + } + } + for (j <- 0 until 2) { + io.ifu.arb_ftq_reqs(j + 1) := ftq_port_addrs(j) + } + + // Frontend Exception Requests + val xcpt_idx = PriorityEncoder(dec_xcpts) + xcpt_pc_req.valid := dec_xcpts.reduce(_ || _) + xcpt_pc_req.bits := dec_uops(xcpt_idx).ftq_idx + rob.io.xcpt_fetch_pc := io.ifu.rrd_ftq_resps(0).pc + + flush_pc_req.valid := rob.io.flush.valid + flush_pc_req.bits := rob.io.flush.bits.ftq_idx + + // Mispredict requests (to get the correct target) + mispredict_pc_req.valid := mispredict_val + mispredict_pc_req.bits := oldest_mispredict_ftq_idx + + //------------------------------------------------------------- + // Decode/Rename1 pipeline logic + + dec_xcpts := dec_uops zip dec_valids map { case (u, v) => u.exception && v } + // Frontend exceptions need to shoot straight past dec/dis without stall, to match + // the timing of the FTQ resp which provides badaddr. + // Wait for pipeline to empty before letting an exception past dec + val dec_prior_slot_valid = dec_valids.scanLeft(false.B)((s, v) => s || v) + + val dec_xcpt_stall = (0 until coreWidth).map(w => + dec_xcpts(w) && + (!rob.io.empty || !io.lsu.fencei_rdy || dec_prior_slot_valid(w) || dis_valids.reduce(_ || _) || !xcpt_pc_req.ready) + ) + + // stall fetch/dcode because we ran out of branch tags + val branch_mask_full = Wire(Vec(coreWidth, Bool())) + + val dec_hazards = (0 until coreWidth).map(w => + dec_valids(w) && + (!dis_ready + || rob.io.rollback + || dec_xcpt_stall(w) + || branch_mask_full(w) + || brupdate.b1.mispredict_mask =/= 0.U + || brupdate.b2.mispredict + || io.ifu.redirect_flush) + ) + + val dec_stalls = dec_hazards.scanLeft(false.B)((s, h) => s || h).takeRight(coreWidth) + dec_fire := (0 until coreWidth).map(w => dec_valids(w) && !dec_stalls(w)) + + // all decoders are empty and ready for new instructions + dec_ready := dec_fire.last + + when(dec_ready || io.ifu.redirect_flush) { + dec_finished_mask := 0.U + }.otherwise { + dec_finished_mask := dec_fire.asUInt | dec_finished_mask + } + + //------------------------------------------------------------- + // Branch Mask Logic + + dec_brmask_logic.io.brupdate := brupdate + dec_brmask_logic.io.flush_pipeline := RegNext(rob.io.flush.valid) + + for (w <- 0 until coreWidth) { + dec_brmask_logic.io.is_branch(w) := !dec_finished_mask(w) && dec_uops(w).allocate_brtag + dec_brmask_logic.io.will_fire(w) := dec_fire(w) && + dec_uops(w).allocate_brtag // ren, dis can back pressure us + dec_uops(w).br_tag := dec_brmask_logic.io.br_tag(w) + dec_uops(w).br_mask := dec_brmask_logic.io.br_mask(w) + } + + branch_mask_full := dec_brmask_logic.io.is_full + + //------------------------------------------------------------- + //------------------------------------------------------------- + // **** Register Rename Stage **** + //------------------------------------------------------------- + //------------------------------------------------------------- + + // Inputs + for (rename <- rename_stages) { + rename.io.kill := io.ifu.redirect_flush + rename.io.brupdate := brupdate + + rename.io.debug_rob_empty := rob.io.empty + + rename.io.dec_fire := dec_fire + rename.io.dec_uops := dec_uops + + rename.io.dis_fire := dis_fire + rename.io.dis_ready := dis_ready + + rename.io.com_valids := rob.io.commit.valids + rename.io.com_uops := rob.io.commit.uops + rename.io.rollback := rob.io.rollback + } + + // Outputs + dis_uops := rename_stage.io.ren2_uops + dis_valids := rename_stage.io.ren2_mask + ren_stalls := rename_stage.io.ren_stalls + + /** + * TODO This is a bit nasty, but it's currently necessary to + * split the INT/FP rename pipelines into separate instantiations. + * Won't have to do this anymore with a properly decoupled FP pipeline. + */ + for (w <- 0 until coreWidth) { + val i_uop = rename_stage.io.ren2_uops(w) + val f_uop = fp_rename_stage.io.ren2_uops(w) + val p_uop = pred_rename_stage.io.ren2_uops(w) + val imm_uop = imm_rename_stage.io.ren2_uops(w) + val f_stall = fp_rename_stage.io.ren_stalls(w) + val p_stall = pred_rename_stage.io.ren_stalls(w) + val imm_stall = imm_rename_stage.io.ren_stalls(w) + + // lrs1 can "pass through" to prs1. Used solely to index the csr file. + dis_uops(w).prs1 := Mux( + dis_uops(w).lrs1_rtype === RT_FLT, + f_uop.prs1, + Mux(dis_uops(w).lrs1_rtype === RT_FIX, i_uop.prs1, dis_uops(w).lrs1) + ) + dis_uops(w).prs2 := Mux(dis_uops(w).lrs2_rtype === RT_FLT, f_uop.prs2, i_uop.prs2) + dis_uops(w).prs3 := f_uop.prs3 + dis_uops(w).ppred := p_uop.ppred + dis_uops(w).pdst := Mux( + dis_uops(w).dst_rtype === RT_FLT, + f_uop.pdst, + Mux( + dis_uops(w).dst_rtype === RT_FIX, + i_uop.pdst, + Mux( + dis_uops(w).is_sfb_br, + p_uop.pdst, + random.LFSR(maxPregSz) // Random set the PDST so the ALU banked IQ is balanced + ) + ) + ) + dis_uops(w).imm_sel := imm_uop.imm_sel + dis_uops(w).pimm := imm_uop.pimm + + dis_uops(w).stale_pdst := Mux(dis_uops(w).dst_rtype === RT_FLT, f_uop.stale_pdst, i_uop.stale_pdst) + + dis_uops(w).prs1_busy := i_uop.prs1_busy && (dis_uops(w).lrs1_rtype === RT_FIX) || + f_uop.prs1_busy && (dis_uops(w).lrs1_rtype === RT_FLT) + dis_uops(w).prs2_busy := i_uop.prs2_busy && (dis_uops(w).lrs2_rtype === RT_FIX) || + f_uop.prs2_busy && (dis_uops(w).lrs2_rtype === RT_FLT) + dis_uops(w).prs3_busy := f_uop.prs3_busy && dis_uops(w).frs3_en + dis_uops(w).ppred_busy := p_uop.ppred_busy && dis_uops(w).is_sfb_shadow + + ren_stalls(w) := rename_stage.io.ren_stalls(w) || f_stall || p_stall || imm_stall + } + + //------------------------------------------------------------- + //------------------------------------------------------------- + // **** Dispatch Stage **** + //------------------------------------------------------------- + //------------------------------------------------------------- + + //------------------------------------------------------------- + // Rename2/Dispatch pipeline logic + + val dis_prior_slot_valid = dis_valids.scanLeft(false.B)((s, v) => s || v) + val dis_prior_slot_unique = (dis_uops zip dis_valids).scanLeft(false.B) { case (s, (u, v)) => s || v && u.is_unique } + + val wait_for_empty_pipeline = (0 until coreWidth).map(w => + (dis_uops(w).is_unique || !custom_csrs.enableOOO) && + (!rob.io.empty || !io.lsu.fencei_rdy || dis_prior_slot_valid(w)) + ) + + val rocc_shim_busy = if (usingRoCC) !unq_exe_unit.io_rocc_core.get.rxq_empty else false.B + val wait_for_rocc = (0 until coreWidth).map(w => (io.rocc.busy || rocc_shim_busy)) + val rxq_full = if (usingRoCC) unq_exe_unit.io_rocc_core.get.rxq_full else false.B + val block_rocc = (dis_uops zip dis_valids).map { case (u, v) => v && u.is_rocc }.scanLeft(rxq_full)(_ || _) + + val dis_rocc_alloc_stall = (dis_uops.map(_.is_rocc) zip block_rocc) map { case (p, r) => + if (usingRoCC) p && r else false.B + } + + // Only 1 branch-tag allocating insruction allowed to proceed per cycle unless enableSuperscalarSnapshots + // This reduces checkpointing complexity + val block_brtag = (dis_uops zip dis_valids).map { case (u, v) => v && u.allocate_brtag }.scanLeft(false.B)(_ || _) + + val brtag_stall = (dis_uops.map(_.allocate_brtag) zip block_brtag) map { case (p, r) => + if (enableSuperscalarSnapshots) false.B else (p && r) + } + + val dis_hazards = (0 until coreWidth).map(w => + dis_valids(w) && + (!rob.io.ready + || brtag_stall(w) + || ren_stalls(w) + || io.lsu.ldq_full(w) && dis_uops(w).uses_ldq + || io.lsu.stq_full(w) && dis_uops(w).uses_stq + || !dispatcher.io.ren_uops(w).ready + || wait_for_empty_pipeline(w) + || wait_for_rocc(w) + || dis_prior_slot_unique(w) + || dis_rocc_alloc_stall(w) + || brupdate.b1.mispredict_mask =/= 0.U + || brupdate.b2.mispredict + || io.ifu.redirect_flush) + ) + + io.lsu.fence_dmem := (dis_valids zip wait_for_empty_pipeline).map { case (v, w) => v && w }.reduce(_ || _) + + val dis_stalls = dis_hazards.scanLeft(false.B)((s, h) => s || h).takeRight(coreWidth) + dis_fire := dis_valids zip dis_stalls map { case (v, s) => v && !s } + dis_ready := !dis_stalls.last + + //------------------------------------------------------------- + // LDQ/STQ Allocation Logic + + for (w <- 0 until coreWidth) { + // Dispatching instructions request load/store queue entries when they can proceed. + dis_uops(w).ldq_idx := io.lsu.dis_ldq_idx(w) + dis_uops(w).stq_idx := io.lsu.dis_stq_idx(w) + } + + //------------------------------------------------------------- + // Rob Allocation Logic + + rob.io.enq_valids := dis_fire + rob.io.enq_uops := dis_uops + rob.io.enq_partial_stall := dis_stalls.last // TODO come up with better ROB compacting scheme. + rob.io.debug_tsc := debug_tsc_reg + rob.io.csr_stall := csr.io.csr_stall + rob.io.trace_stall := io.traceStall + + // Minor hack: ecall and breaks need to increment the FTQ deq ptr earlier than commit, since + // they write their PC into the CSR the cycle before they commit. + // Since these are also unique, increment the FTQ ptr when they are dispatched + when(RegNext(dis_fire.reduce(_ || _) && dis_uops(PriorityEncoder(dis_fire)).is_sys_pc2epc)) { + io.ifu.commit.valid := true.B + io.ifu.commit.bits := RegNext(dis_uops(PriorityEncoder(dis_valids)).ftq_idx) + } + + for (w <- 0 until coreWidth) { + // note: this assumes uops haven't been shifted - there's a 1:1 match between PC's LSBs and "w" here + // (thus the LSB of the rob_idx gives part of the PC) + if (coreWidth == 1) { + dis_uops(w).rob_idx := rob.io.rob_tail_idx + } else { + dis_uops(w).rob_idx := Cat(rob.io.rob_tail_idx >> log2Ceil(coreWidth).U, w.U(log2Ceil(coreWidth).W)) + } + } + + //------------------------------------------------------------- + // RoCC allocation logic + if (usingRoCC) { + val rocc_unit = unq_exe_unit + for (w <- 0 until coreWidth) { + // We guarantee only decoding 1 RoCC instruction per cycle + dis_uops(w).rxq_idx := rocc_unit.io_rocc_core.get.rxq_idx(w) + } + } + + //------------------------------------------------------------- + // Dispatch to issue queues + + // Get uops from rename2 + for (w <- 0 until coreWidth) { + dispatcher.io.ren_uops(w).valid := dis_fire(w) + dispatcher.io.ren_uops(w).bits := dis_uops(w) + } + + var iu_idx = 0 + // Send dispatched uops to correct issue queues + // Backpressure through dispatcher if necessary + for (i <- 0 until issueParams.size) { + if (issueParams(i).iqType == IQ_FP) { + fp_pipeline.io.dis_uops <> dispatcher.io.dis_uops(i) + } else if (issueParams(i).iqType == IQ_MEM) { + mem_iss_unit.io.dis_uops <> dispatcher.io.dis_uops(i) + } else if (issueParams(i).iqType == IQ_ALU) { + alu_iss_unit.io.dis_uops <> dispatcher.io.dis_uops(i) + } else if (issueParams(i).iqType == IQ_UNQ) { + unq_iss_unit.io.dis_uops <> dispatcher.io.dis_uops(i) + } else { + require(false) + } + } + + //------------------------------------------------------------- + // Write immediates, branches into immediate file + for (w <- 0 until coreWidth) { + val uop = RegNext(dis_uops(w)) + + immregfile.io.write_ports(w).valid := RegNext(dis_fire(w)) && !uop.imm_sel.isOneOf(IS_N, IS_SH) + immregfile.io.write_ports(w).bits.addr := uop.pimm + immregfile.io.write_ports(w).bits.data := uop.imm_packed + + bregfile.io.write_ports(w).valid := RegNext(dis_fire(w)) && uop.allocate_brtag + bregfile.io.write_ports(w).bits.addr := uop.br_tag + bregfile.io.write_ports(w).bits.data.ldq_idx := uop.ldq_idx + bregfile.io.write_ports(w).bits.data.stq_idx := uop.stq_idx + bregfile.io.write_ports(w).bits.data.rxq_idx := uop.rxq_idx + } + + //------------------------------------------------------------- + //------------------------------------------------------------- + // **** Issue Stage **** + //------------------------------------------------------------- + //------------------------------------------------------------- + + var wu_idx = 0 + var wb_idx = 0 + var bypass_idx = 0 + for (i <- 0 until lsuWidth) { + int_wakeups(wu_idx) := io.lsu.iwakeups(i) + rob.io.wb_resps(wb_idx) := RegNext(UpdateBrMask(brupdate, RegNext(rob.io.flush.valid), io.lsu.iresp(i))) + iregfile.io.write_ports(wb_idx).valid := RegNext(io.lsu.iresp(i).valid) + iregfile.io.write_ports(wb_idx).bits.addr := RegNext(io.lsu.iresp(i).bits.uop.pdst) + iregfile.io.write_ports(wb_idx).bits.data := RegNext(io.lsu.iresp(i).bits.data) + + int_bypasses(bypass_idx).valid := RegNext(io.lsu.iresp(i).valid) + int_bypasses(bypass_idx).bits := RegNext(io.lsu.iresp(i).bits) + wu_idx += 1 + wb_idx += 1 + bypass_idx += 1 + } + + val ll_arb = Module(new Arbiter( + new ExeUnitResp(xLen), + 1 + // Mul + 1 + // F2I + (if (unq_exe_unit.hasDiv) 1 else 0) + + (if (unq_exe_unit.hasCSR) 1 else 0) + + (if (unq_exe_unit.hasRocc) 1 else 0) + )) + + var arb_idx = 0 + ll_arb.io.in(arb_idx).valid := unq_exe_unit.io_mul_resp.get.valid + ll_arb.io.in(arb_idx).bits := unq_exe_unit.io_mul_resp.get.bits + arb_idx += 1 + ll_arb.io.in(arb_idx) <> fp_pipeline.io.to_int + arb_idx += 1 + if (unq_exe_unit.hasDiv) { + ll_arb.io.in(arb_idx) <> unq_exe_unit.io_div_resp.get + arb_idx += 1 + } + if (unq_exe_unit.hasCSR) { + ll_arb.io.in(arb_idx).valid := csr_resp.valid + ll_arb.io.in(arb_idx).bits.uop := csr_resp.bits.uop + val rdata = WireInit(csr.io.rw.rdata) + if (enableBPDHPMs) { + var hpmcounter = CSRs.hpmcounter16 + var mhpmcounter = CSRs.mhpmcounter16 + for (i <- 0 until 5) { + when(csr.io.rw.addr === hpmcounter.U || csr.io.rw.addr === mhpmcounter.U) { + rdata := debug_brs(i) + } + hpmcounter = hpmcounter + 1 + mhpmcounter = mhpmcounter + 1 + when(csr.io.rw.addr === hpmcounter.U || csr.io.rw.addr === mhpmcounter.U) { + rdata := debug_jals(i) + } + hpmcounter = hpmcounter + 1 + mhpmcounter = mhpmcounter + 1 + when(csr.io.rw.addr === hpmcounter.U || csr.io.rw.addr === mhpmcounter.U) { + rdata := debug_jalrs(i) + } + hpmcounter = hpmcounter + 1 + mhpmcounter = mhpmcounter + 1 + } + } + ll_arb.io.in(arb_idx).bits.data := rdata + ll_arb.io.in(arb_idx).bits.predicated := false.B + ll_arb.io.in(arb_idx).bits.fflags.valid := false.B + ll_arb.io.in(arb_idx).bits.fflags.bits := false.B + assert(!(ll_arb.io.in(arb_idx).valid && !ll_arb.io.in(arb_idx).ready)) + arb_idx += 1 + } + if (unq_exe_unit.hasRocc) { + ll_arb.io.in(arb_idx) <> unq_exe_unit.io_rocc_resp.get + arb_idx += 1 + } + ll_arb.io.out.ready := true.B + int_wakeups(wu_idx).valid := ll_arb.io.out.valid && ll_arb.io.out.bits.uop.dst_rtype === RT_FIX + int_wakeups(wu_idx).bits.uop := ll_arb.io.out.bits.uop + int_wakeups(wu_idx).bits.speculative_mask := 0.U + int_wakeups(wu_idx).bits.rebusy := false.B + int_wakeups(wu_idx).bits.bypassable := false.B + wu_idx += 1 + + rob.io.wb_resps(wb_idx).valid := RegNext(ll_arb.io.out.valid && !IsKilledByBranch( + brupdate, + RegNext(rob.io.flush.valid), + ll_arb.io.out.bits + )) + rob.io.wb_resps(wb_idx).bits := RegNext(ll_arb.io.out.bits) + + iregfile.io.write_ports(wb_idx).valid := ll_arb.io.out.valid && ll_arb.io.out.bits.uop.dst_rtype === RT_FIX + iregfile.io.write_ports(wb_idx).bits.addr := ll_arb.io.out.bits.uop.pdst + iregfile.io.write_ports(wb_idx).bits.data := ll_arb.io.out.bits.data + wb_idx += 1 + + // loop through each issue-port (exe_units are statically connected to an issue-port) + for (i <- 0 until aluWidth) { + val unit = alu_exe_units(i) + val fast_wakeup = unit.io_fast_wakeup + + int_bypasses(bypass_idx).valid := unit.io_alu_resp.valid && unit.io_alu_resp.bits.uop.dst_rtype === RT_FIX + int_bypasses(bypass_idx).bits := unit.io_alu_resp.bits + bypass_idx += 1 + + int_wakeups(wu_idx) := fast_wakeup + wu_idx += 1 + + rob.io.wb_resps(wb_idx).valid := RegNext(unit.io_alu_resp.valid && !IsKilledByBranch( + brupdate, + RegNext(rob.io.flush.valid), + unit.io_alu_resp.bits + )) + rob.io.wb_resps(wb_idx).bits := RegNext(unit.io_alu_resp.bits) + + iregfile.io.write_ports(wb_idx).valid := unit.io_alu_resp.valid && unit.io_alu_resp.bits.uop.dst_rtype === RT_FIX + iregfile.io.write_ports(wb_idx).bits.addr := unit.io_alu_resp.bits.uop.pdst + iregfile.io.write_ports(wb_idx).bits.data := unit.io_alu_resp.bits.data + wb_idx += 1 + + pred_wakeups(i) := unit.io_fast_pred_wakeup + + } + require(wu_idx == numIntWakeups) + require(wb_idx == numIrfWritePorts) + + // arb stage guarantees 1 preg writer per cycle + val pregfile_write_valids = alu_exe_units.map(u => u.io_alu_resp.valid && u.io_alu_resp.bits.uop.is_sfb_br) + assert(PopCount(pregfile_write_valids) <= 1.U) + pregfile.io.write_ports(0).valid := pregfile_write_valids.reduce(_ || _) + pregfile.io.write_ports(0).bits.addr := Mux1H(pregfile_write_valids, alu_exe_units.map(_.io_alu_resp.bits.uop.pdst)) + pregfile.io.write_ports(0).bits.data := Mux1H(pregfile_write_valids, alu_exe_units.map(_.io_alu_resp.bits.data)) + + // Connect the predicate wakeup port + alu_iss_unit.io.pred_wakeup_port.valid := pred_wakeup.valid + alu_iss_unit.io.pred_wakeup_port.bits := pred_wakeup.bits.uop.pdst + mem_iss_unit.io.pred_wakeup_port.valid := false.B + mem_iss_unit.io.pred_wakeup_port.bits := DontCare + unq_iss_unit.io.pred_wakeup_port.valid := false.B + unq_iss_unit.io.pred_wakeup_port.bits := DontCare + + // ---------------------------------------------------------------- + // Connect the wakeup ports to the busy tables in the rename stages + + for ((renport, intport) <- rename_stage.io.wakeups zip int_wakeups) { + renport <> intport + } + if (usingFPU) { + fp_rename_stage.io.wakeups := fp_pipeline.io.wakeups + } + + pred_rename_stage.io.wakeups(0) := pred_wakeup + imm_rename_stage.io.wakeups := all_exe_units.map(_.io_rrd_immrf_wakeup) + + rename_stage.io.child_rebusys := alu_exe_units.map(_.io_child_rebusy).reduce(_ | _) + imm_rename_stage.io.child_rebusys := 0.U + pred_rename_stage.io.child_rebusys := 0.U + fp_rename_stage.io.child_rebusys := 0.U + + mem_iss_unit.io.fu_types := mem_exe_units.map(_.io_ready_fu_types) + alu_iss_unit.io.fu_types := alu_exe_units.map(_.io_ready_fu_types) + unq_iss_unit.io.fu_types := unq_exe_units.map(_.io_ready_fu_types) + + for (iss_unit <- Seq(mem_iss_unit, alu_iss_unit, unq_iss_unit)) { + iss_unit.io.tsc_reg := debug_tsc_reg + iss_unit.io.brupdate := brupdate + iss_unit.io.flush_pipeline := RegNext(rob.io.flush.valid) + + // Rebusy children of misspeculated wakeups + iss_unit.io.child_rebusys := alu_exe_units.map(_.io_child_rebusy).reduce(_ | _) + + iss_unit.io.wakeup_ports := int_wakeups + + iss_unit.io.rob_pnr_idx := rob.io.rob_pnr_idx + iss_unit.io.rob_head := rob.io.rob_head_idx + } + + mem_iss_unit.io.squash_grant := ( + mem_exe_units.map(_.io_squash_iss).reduce(_ || _) || + alu_exe_units.map(_.io_squash_iss).reduce(_ || _) || + io.lsu.iwakeups.map(_.bits.rebusy).reduce(_ || _) + ) + unq_iss_unit.io.squash_grant := ( + unq_exe_units.map(_.io_squash_iss).reduce(_ || _) || + alu_exe_units.map(_.io_squash_iss).reduce(_ || _) || + io.lsu.iwakeups.map(_.bits.rebusy).reduce(_ || _) + ) + alu_iss_unit.io.squash_grant := ( + alu_exe_units.map(_.io_squash_iss).reduce(_ || _) || + io.lsu.iwakeups.map(_.bits.rebusy).reduce(_ || _) + ) + + mem_iss_unit.io.iss_uops zip mem_exe_units map { case (i, u) => u.io_iss_uop := i } + alu_iss_unit.io.iss_uops zip alu_exe_units map { case (i, u) => u.io_iss_uop := i } + unq_iss_unit.io.iss_uops zip unq_exe_units map { case (i, u) => u.io_iss_uop := i } + + //------------------------------------------------------------- + //------------------------------------------------------------- + // **** Register Read Arbitrate Stage **** + //------------------------------------------------------------- + //------------------------------------------------------------- + arb_idx = 0 + for ((unit, w) <- all_exe_units.zipWithIndex) { + for (i <- 0 until unit.nReaders) { + iregfile.io.arb_read_reqs(arb_idx) <> unit.io_arb_irf_reqs(i) + arb_idx += 1 + } + immregfile.io.arb_read_reqs(w) <> unit.io_arb_immrf_req + unit.io_arb_rebusys := io.lsu.iwakeups + } + require(arb_idx == numIrfLogicalReadPorts) + for ((unit, w) <- alu_exe_units.zipWithIndex) { + pregfile.io.arb_read_reqs(w) <> unit.io_arb_prf_req + bregfile.io.arb_read_reqs(w) <> unit.io_arb_brf_req + } + + //------------------------------------------------------------- + //------------------------------------------------------------- + // **** Register Read Stage **** + //------------------------------------------------------------- + //------------------------------------------------------------- + + // Register Read <- Issue (rrd <- iss) + var rd_idx = 0 + for ((unit, w) <- all_exe_units.zipWithIndex) { + for (i <- 0 until unit.nReaders) { + unit.io_rrd_irf_resps(i) := iregfile.io.rrd_read_resps(rd_idx) + rd_idx += 1 + } + unit.io_rrd_immrf_resp := immregfile.io.rrd_read_resps(w) + unit.io_rrd_irf_bypasses := int_bypasses + } + require(rd_idx == numIrfLogicalReadPorts) + for ((unit, w) <- alu_exe_units.zipWithIndex) { + unit.io_rrd_prf_resp := pregfile.io.rrd_read_resps(w) + unit.io_rrd_brf_resp := bregfile.io.rrd_read_resps(w) + } + + //------------------------------------------------------------- + // Privileged Co-processor 0 Register File + // Note: Normally this would be bad in that I'm writing state before + // committing, so to get this to work I stall the entire pipeline for + // CSR instructions so I never speculate these instructions. + + io.lsu.sfence := unq_exe_unit.io_sfence.get + io.ifu.sfence := unq_exe_unit.io_sfence.get + + // for critical path reasons, we aren't zero'ing this out if resp is not valid + csr.io.rw.addr := csr_resp.bits.addr + csr.io.rw.cmd := CSR.maskCmd(csr_resp.valid, csr_resp.bits.uop.csr_cmd) + csr.io.rw.wdata := csr_resp.bits.data + + rob.io.csr_replay.valid := csr_resp.valid && csr.io.rw_stall + rob.io.csr_replay.bits.uop := csr_resp.bits.uop + rob.io.csr_replay.bits.cause := MINI_EXCEPTION_CSR_REPLAY + rob.io.csr_replay.bits.badvaddr := DontCare + + // Extra I/O + // Delay retire/exception 1 cycle + csr.io.retire := RegNext(PopCount(rob.io.commit.arch_valids.asUInt)) + csr.io.exception := RegNext(rob.io.com_xcpt.valid) + // csr.io.pc used for setting EPC during exception or CSR.io.trace. + + csr.io.pc := (boom.v4.util.AlignPCToBoundary(io.ifu.com_pc, icBlockBytes) + + RegNext(rob.io.com_xcpt.bits.pc_lob) + - Mux(RegNext(rob.io.com_xcpt.bits.edge_inst), 2.U, 0.U)) + // Cause not valid for for CALL or BREAKPOINTs (CSRFile will override it). + csr.io.cause := RegNext(rob.io.com_xcpt.bits.cause) + csr.io.ungated_clock := clock + + val tval_valid = csr.io.exception && + csr.io.cause.isOneOf( + //Causes.illegal_instruction.U, we currently only write 0x0 for illegal instructions + Causes.breakpoint.U, + Causes.misaligned_load.U, + Causes.misaligned_store.U, + Causes.load_access.U, + Causes.store_access.U, + Causes.fetch_access.U, + Causes.load_page_fault.U, + Causes.store_page_fault.U, + Causes.fetch_page_fault.U + ) + + csr.io.tval := Mux( + tval_valid, + RegNext(encodeVirtualAddress(rob.io.com_xcpt.bits.badvaddr, rob.io.com_xcpt.bits.badvaddr)), + 0.U + ) + + // TODO move this function to some central location (since this is used elsewhere). + def encodeVirtualAddress(a0: UInt, ea: UInt) = + if (vaddrBitsExtended == vaddrBits) { + ea + } else { + // Efficient means to compress 64-bit VA into vaddrBits+1 bits. + // (VA is bad if VA(vaddrBits) != VA(vaddrBits-1)). + val a = a0.asSInt >> vaddrBits + val msb = Mux(a === 0.S || a === -1.S, ea(vaddrBits), !ea(vaddrBits - 1)) + Cat(msb, ea(vaddrBits - 1, 0)) + } + + // reading requires serializing the entire pipeline + csr.io.fcsr_flags.valid := rob.io.commit.fflags.valid + csr.io.fcsr_flags.bits := rob.io.commit.fflags.bits + csr.io.set_fs_dirty.get := rob.io.commit.fflags.valid + + all_exe_units.map(i => i.io_fcsr_rm := csr.io.fcsr_rm) + io.fcsr_rm := csr.io.fcsr_rm + + fp_pipeline.io.fcsr_rm := csr.io.fcsr_rm + + csr.io.hartid := io.hartid + csr.io.interrupts := io.interrupts + + // we do not support the H-extension + csr.io.htval := DontCare + csr.io.gva := DontCare + + //------------------------------------------------------------- + //------------------------------------------------------------- + // **** Execute Stage **** + //------------------------------------------------------------- + //------------------------------------------------------------- + + //------------------------------------------------------------- + //------------------------------------------------------------- + // **** Load/Store Unit **** + //------------------------------------------------------------- + //------------------------------------------------------------- + + // enqueue basic load/store info in Decode + for (w <- 0 until coreWidth) { + io.lsu.dis_uops(w).valid := dis_fire(w) + io.lsu.dis_uops(w).bits := dis_uops(w) + } + + // tell LSU about committing loads and stores to clear entries + io.lsu.commit := rob.io.commit + + // tell LSU that it should fire a load that waits for the rob to clear + io.lsu.commit_load_at_rob_head := rob.io.com_load_is_at_rob_head + + //com_xcpt.valid comes too early, will fight against a branch that resolves same cycle as an exception + io.lsu.exception := RegNext(rob.io.flush.valid) + + // Handle Branch Mispeculations + io.lsu.brupdate := brupdate + io.lsu.rob_head_idx := rob.io.rob_head_idx + io.lsu.rob_pnr_idx := rob.io.rob_pnr_idx + + io.lsu.tsc_reg := debug_tsc_reg + + // Connect IFPU + fp_pipeline.io.from_int <> unq_exe_unit.io_ifpu_resp.get + + // Connect FLDs + fp_pipeline.io.ll_wports <> io.lsu.fresp + + //------------------------------------------------------------- + //------------------------------------------------------------- + // **** Commit Stage **** + //------------------------------------------------------------- + //------------------------------------------------------------- + + var cnt = numIrfWritePorts + for (wb <- fp_pipeline.io.wb) { + rob.io.wb_resps(cnt) := wb + rob.io.wb_resps(cnt).bits.data := ieee(wb.bits.data) + cnt += 1 + } + + require(cnt == rob.numWakeupPorts) + + // branch resolution + rob.io.brupdate <> brupdate + + io.lsu.status := csr.io.status + io.lsu.bp := csr.io.bp + io.lsu.mcontext := csr.io.mcontext + io.lsu.scontext := csr.io.scontext + + all_exe_units.map(u => u.io_status := csr.io.status) + fp_pipeline.io.status := csr.io.status + + // LSU <> ROB + rob.io.lsu_clr_bsy := io.lsu.clr_bsy + rob.io.lsu_clr_unsafe := io.lsu.clr_unsafe + rob.io.lxcpt <> io.lsu.lxcpt + + assert(!(csr.io.singleStep), "[core] single-step is unsupported.") + + //------------------------------------------------------------- + // **** Flush Pipeline **** + //------------------------------------------------------------- + // flush on exceptions, miniexeptions, and after some special instructions + + fp_pipeline.io.flush_pipeline := RegNext(rob.io.flush.valid) + + for (eu <- all_exe_units) + eu.io_kill := RegNext(rob.io.flush.valid) + + assert( + !(rob.io.com_xcpt.valid && !rob.io.flush.valid), + "[core] exception occurred, but pipeline flush signal not set!" + ) + + //------------------------------------------------------------- + //------------------------------------------------------------- + // **** Outputs to the External World **** + //------------------------------------------------------------- + //------------------------------------------------------------- + + // detect pipeline freezes and throw error + val idle_cycles = freechips.rocketchip.util.WideCounter(32) + when(rob.io.commit.valids.asUInt.orR || + csr.io.csr_stall || + io.rocc.busy || + reset.asBool) { + idle_cycles := 0.U + } + assert(!(idle_cycles.value(PlusArg("boom_timeout", 13, width = 5))), "Pipeline has hung.") + + fp_pipeline.io.debug_tsc_reg := debug_tsc_reg + + //------------------------------------------------------------- + //------------------------------------------------------------- + // **** Handle Cycle-by-Cycle Printouts **** + //------------------------------------------------------------- + //------------------------------------------------------------- + + if (COMMIT_LOG_PRINTF) { + var new_commit_cnt = 0.U + + for (w <- 0 until coreWidth) { + val priv = ShiftRegister(csr.io.status.prv, 2) // erets change the privilege. Get the old one + + // To allow for diffs against spike :/ + def printf_inst(uop: MicroOp) = { + when(uop.is_rvc) { + printf("(0x%x)", uop.debug_inst(15, 0)) + }.otherwise { + printf("(0x%x)", uop.debug_inst) + } + } + + when(rob.io.commit.arch_valids(w)) { + printf("%d 0x%x ", priv, Sext(rob.io.commit.uops(w).debug_pc(vaddrBits - 1, 0), xLen)) + printf_inst(rob.io.commit.uops(w)) + when(rob.io.commit.uops(w).dst_rtype === RT_FIX && rob.io.commit.uops(w).ldst =/= 0.U) { + printf(" x%d 0x%x\n", rob.io.commit.uops(w).ldst, rob.io.commit.debug_wdata(w)) + }.elsewhen(rob.io.commit.uops(w).dst_rtype === RT_FLT) { + printf(" f%d 0x%x\n", rob.io.commit.uops(w).ldst, rob.io.commit.debug_wdata(w)) + }.otherwise { + printf("\n") + } + } + } + } else if (BRANCH_PRINTF) { + val debug_ghist = RegInit(0.U(globalHistoryLength.W)) + when(rob.io.flush.valid && FlushTypes.useCsrEvec(rob.io.flush.bits.flush_typ)) { + debug_ghist := 0.U + } + + var new_ghist = debug_ghist + + for (w <- 0 until coreWidth) { + when(rob.io.commit.arch_valids(w) && + (rob.io.commit.uops(w).is_br || rob.io.commit.uops(w).is_jal || rob.io.commit.uops(w).is_jalr)) { + // for (i <- 0 until globalHistoryLength) { + // printf("%x", new_ghist(globalHistoryLength-i-1)) + // } + // printf("\n") + printf( + "%x %x %x %x %x %x\n", + rob.io.commit.uops(w).debug_fsrc, + rob.io.commit.uops(w).taken, + rob.io.commit.uops(w).is_br, + rob.io.commit.uops(w).is_jal, + rob.io.commit.uops(w).is_jalr, + Sext(rob.io.commit.uops(w).debug_pc(vaddrBits - 1, 0), xLen) + ) + + } + new_ghist = Mux( + rob.io.commit.arch_valids(w) && rob.io.commit.uops(w).is_br, + Mux(rob.io.commit.uops(w).taken, new_ghist << 1 | 1.U(1.W), new_ghist << 1), + new_ghist + ) + } + debug_ghist := new_ghist + } + + // TODO: Does anyone want this debugging functionality? + val coreMonitorBundle = Wire(new CoreMonitorBundle(xLen, fLen)) + coreMonitorBundle := DontCare + coreMonitorBundle.clock := clock + coreMonitorBundle.reset := reset + + //------------------------------------------------------------- + //------------------------------------------------------------- + // Page Table Walker + + io.ptw.ptbr := csr.io.ptbr + io.ptw.status := csr.io.status + io.ptw.pmp := csr.io.pmp + io.ptw.sfence := io.ifu.sfence + + //------------------------------------------------------------- + //------------------------------------------------------------- + + io.rocc := DontCare + io.rocc.exception := csr.io.exception && csr.io.status.xs.orR + io.rocc.csrs <> csr.io.roccCSRs + if (usingRoCC) { + val rocc_unit = unq_exe_unit + rocc_unit.io_rocc_core.get.rocc <> io.rocc + rocc_unit.io_rocc_core.get.dis_uops := dis_uops + rocc_unit.io_rocc_core.get.rob_head_idx := rob.io.rob_head_idx + rocc_unit.io_rocc_core.get.rob_pnr_idx := rob.io.rob_pnr_idx + + for (w <- 0 until coreWidth) { + rocc_unit.io_rocc_core.get.dis_rocc_vals(w) := ( + dis_fire(w) && + dis_uops(w).is_rocc && + !dis_uops(w).exception + ) + } + } + + io.trace := DontCare + io.trace.time := csr.io.time + io.trace.insns map (t => t.valid := false.B) + io.trace.custom.get.asInstanceOf[BoomTraceBundle].rob_empty := rob.io.empty + + if (trace) { + for (w <- 0 until coreWidth) { + // Delay the trace so we have a cycle to pull PCs out of the FTQ + io.trace.insns(w).valid := RegNext(rob.io.commit.arch_valids(w)) + + // Recalculate the PC + io.ifu.debug_ftq_idx(w) := rob.io.commit.uops(w).ftq_idx + val iaddr = (AlignPCToBoundary(io.ifu.debug_fetch_pc(w), icBlockBytes) + + RegNext(rob.io.commit.uops(w).pc_lob) + - Mux(RegNext(rob.io.commit.uops(w).edge_inst), 2.U, 0.U))(vaddrBits - 1, 0) + io.trace.insns(w).iaddr := Sext(iaddr, xLen) + + def getInst(uop: MicroOp, inst: UInt): UInt = + Mux(uop.is_rvc, Cat(0.U(16.W), inst(15, 0)), inst) + + def getWdata(uop: MicroOp, wdata: UInt): UInt = + Mux((uop.dst_rtype === RT_FIX && uop.ldst =/= 0.U) || (uop.dst_rtype === RT_FLT), wdata, 0.U(xLen.W)) + + // use debug_insts instead of uop.debug_inst to use the rob's debug_inst_mem + // note: rob.debug_insts comes 1 cycle later + io.trace.insns(w).insn := getInst(RegNext(rob.io.commit.uops(w)), rob.io.commit.debug_insts(w)) + io.trace.insns(w).wdata.map(_ := RegNext(getWdata(rob.io.commit.uops(w), rob.io.commit.debug_wdata(w)))) + + // Comment out this assert because it blows up FPGA synth-asserts + // This tests correctedness of the debug_inst mem + // when (RegNext(rob.io.commit.valids(w))) { + // assert(rob.io.commit.debug_insts(w) === RegNext(rob.io.commit.uops(w).debug_inst)) + // } + // This tests correctedness of recovering pcs through ftq debug ports + // when (RegNext(rob.io.commit.valids(w))) { + // assert(Sext(io.trace.insns(w).iaddr, xLen) === + // RegNext(Sext(rob.io.commit.uops(w).debug_pc(vaddrBits-1,0), xLen))) + // } + + // These csr signals do not exactly match up with the ROB commit signals. + io.trace.insns(w).priv := RegNext(Cat(RegNext(RegNext(csr.io.status.debug)), csr.io.status.prv)) + // Can determine if it is an interrupt or not based on the MSB of the cause + io.trace.insns(w).exception := RegNext( + rob.io.com_xcpt.valid && !rob.io.com_xcpt.bits.cause(xLen - 1) + ) && (w == 0).B + io.trace.insns(w).interrupt := RegNext( + rob.io.com_xcpt.valid && rob.io.com_xcpt.bits.cause(xLen - 1) + ) && (w == 0).B + io.trace.insns(w).cause := RegNext(rob.io.com_xcpt.bits.cause) + io.trace.insns(w).tval := RegNext(csr.io.tval) + } + dontTouch(io.trace) + } else { + io.ifu.debug_ftq_idx := DontCare + } + + if (boomParams.enableTraceCoreIngress) { + for (w <- 0 until coreWidth) { + val trace_ingress = Module(new TraceCoreIngress(traceIngressParams)) + trace_ingress.io.in.valid := RegNext(rob.io.commit.arch_valids(w)) + // this is predicted to be taken, but since it is commited, it must have been predicted correctly + trace_ingress.io.in.taken := RegNext(rob.io.commit.uops(w).taken) + trace_ingress.io.in.is_branch := RegNext(rob.io.commit.uops(w).is_br) + trace_ingress.io.in.is_jal := RegNext(rob.io.commit.uops(w).is_jal) + trace_ingress.io.in.is_jalr := RegNext(rob.io.commit.uops(w).is_jalr) + trace_ingress.io.in.insn := RegNext(rob.io.commit.uops(w).debug_inst) + trace_ingress.io.in.pc := RegNext(rob.io.commit.uops(w).debug_pc) + trace_ingress.io.in.is_compressed := RegNext(rob.io.commit.uops(w).is_rvc) + trace_ingress.io.in.interrupt := RegNext( + rob.io.com_xcpt.valid && rob.io.com_xcpt.bits.cause(xLen - 1) + ) && (w == 0).B + trace_ingress.io.in.exception := RegNext( + rob.io.com_xcpt.valid && !rob.io.com_xcpt.bits.cause(xLen - 1) + ) && (w == 0).B + trace_ingress.io.in.trap_return := RegNext(rob.io.commit.uops(w).is_eret) + io.trace_core_ingress.get.group(w) <> trace_ingress.io.out + } + io.trace_core_ingress.get.ctx := RegNext(csr.io.ptbr.asid) + io.trace_core_ingress.get.tval := RegNext(csr.io.tval) + io.trace_core_ingress.get.cause := RegNext(csr.io.cause) + io.trace_core_ingress.get.time := RegNext(csr.io.time) + io.trace_core_ingress.get.priv := RegNext(csr.io.status.prv) + } +} diff --git a/examples/prefab/cores/boom/arch/native/Decode.scala b/examples/prefab/cores/boom/arch/native/Decode.scala new file mode 100644 index 000000000..8be025b9b --- /dev/null +++ b/examples/prefab/cores/boom/arch/native/Decode.scala @@ -0,0 +1,216 @@ +// Copied from riscv-boom src/main/scala/v4/exu/decode.scala and modified for Buckyball. +// Copyright (c) The Regents of the University of California (Regents). +// All Rights Reserved. See LICENSE and LICENSE.SiFive in riscv-boom for license details. + +package examples.core.boom.native + +import chisel3._ +import chisel3.util._ + +import freechips.rocketchip.tile.FPConstants +import org.chipsalliance.cde.config.Parameters +import freechips.rocketchip.rocket.Instructions._ +import freechips.rocketchip.rocket.Instructions32 +import freechips.rocketchip.rocket.CustomInstructions._ +import freechips.rocketchip.rocket.RVCExpander +import freechips.rocketchip.rocket.ALU._ +import freechips.rocketchip.rocket.{CSR, Causes, DecodeLogic} +import freechips.rocketchip.util._ + +import boom.v4.common._ +import boom.v4.util._ +import boom.v4.exu._ + +/** + * Decode unit that takes in a single instruction and generates a MicroOp. + */ +class DecodeUnit(implicit p: Parameters) + extends BoomModule + with freechips.rocketchip.rocket.constants.MemoryOpConstants + with freechips.rocketchip.rocket.constants.ScalarOpConstants { + val io = IO(new DecodeUnitIo) + + val uop = Wire(new MicroOp()) + uop := io.enq.uop + + val decode_table = + DecodeTables.X_table ++ + DecodeTables.F_table ++ + DecodeTables.FDivSqrt_table ++ + DecodeTables.X64_table ++ + DecodeTables.B_table ++ + (if (usingRoCC) DecodeTables.RoCC_table else Nil) + + val inst = uop.inst + val LDST = inst(RD_MSB, RD_LSB) + val LRS1 = inst(RS1_MSB, RS1_LSB) + val LRS2 = inst(RS2_MSB, RS2_LSB) + val LRS3 = inst(RS3_MSB, RS3_LSB) + + val cs = Wire(new CtrlSigs()).decode(inst, decode_table) + + // Exception Handling + io.csr_decode.inst := inst + val csr_en = cs.csr_cmd.isOneOf(CSR.S, CSR.C, CSR.W) + val csr_ren = cs.csr_cmd.isOneOf(CSR.S, CSR.C) && uop.lrs1 === 0.U + val system_insn = cs.csr_cmd === CSR.I + val sfence = inst === SFENCE_VMA + + val cs_legal = cs.legal +// dontTouch(cs_legal) + + require(fLen >= 64) + val illegal_rm = inst(14, 12).isOneOf(5.U, 6.U) || (inst(14, 12) === 7.U && io.fcsr_rm >= 5.U) + + val id_illegal_insn = !cs_legal || + (cs.fp_val && (io.csr_decode.fp_illegal || illegal_rm)) || + (uop.is_rocc && io.csr_decode.rocc_illegal) || + (cs.is_amo && !io.status.isa('a' - 'a')) || + (csr_en && (io.csr_decode.read_illegal || !csr_ren && io.csr_decode.write_illegal)) || + ((sfence || system_insn) && io.csr_decode.system_illegal) + +// cs.div && !csr.io.status.isa('m'-'a') || TODO check for illegal div instructions + + def checkExceptions(x: Seq[(Bool, UInt)]) = + (x.map(_._1).reduce(_ || _), PriorityMux(x)) + + val (xcpt_valid, xcpt_cause) = checkExceptions(List( + (io.interrupt && !io.enq.uop.is_sfb, io.interrupt_cause), // Disallow interrupts while we are handling a SFB + (uop.bp_debug_if, (CSR.debugTriggerCause).U), + (uop.bp_xcpt_if, (Causes.breakpoint).U), + (uop.xcpt_pf_if, (Causes.fetch_page_fault).U), + (uop.xcpt_ae_if, (Causes.fetch_access).U), + (id_illegal_insn, (Causes.illegal_instruction).U) + )) + + uop.exception := xcpt_valid + uop.exc_cause := xcpt_cause + + //------------------------------------------------------------- + + uop.is_mov := inst === ADD && LRS1 === 0.U + + uop.iq_type(IQ_UNQ) := Seq(FC_MUL, FC_DIV, FC_CSR, FC_I2F).map(c => cs.fu_code(c)).reduce(_ || _) + uop.iq_type(IQ_ALU) := Seq(FC_ALU).map(c => cs.fu_code(c)).reduce(_ || _) + uop.iq_type(IQ_MEM) := Seq(FC_AGEN, FC_DGEN).map(c => cs.fu_code(c)).reduce(_ || _) + uop.iq_type(IQ_FP) := Seq(FC_FPU, FC_FDV, FC_F2I).map(c => cs.fu_code(c)).reduce(_ || _) + + uop.fu_code := cs.fu_code.asBools + + uop.ldst := LDST + uop.lrs1 := LRS1 + uop.lrs2 := LRS2 + uop.lrs3 := LRS3 + + uop.dst_rtype := cs.dst_type + uop.lrs1_rtype := Mux(cs.rs1_type === RT_FIX && LRS1 === 0.U, RT_ZERO, cs.rs1_type) + uop.lrs2_rtype := Mux(cs.rs2_type === RT_FIX && LRS2 === 0.U, RT_ZERO, cs.rs2_type) + uop.frs3_en := cs.frs3_en + + uop.ldst_is_rs1 := uop.is_sfb_shadow + // SFB optimization + when(uop.is_sfb_shadow && cs.rs2_type === RT_X) { + uop.lrs2_rtype := Mux(LDST === 0.U, RT_ZERO, RT_FIX) + uop.lrs2 := LDST + uop.ldst_is_rs1 := false.B + }.elsewhen(uop.is_sfb_shadow && uop.is_mov) { + uop.lrs1 := LDST + uop.lrs1_rtype := Mux(LDST === 0.U, RT_ZERO, RT_FIX) + uop.ldst_is_rs1 := true.B + } + + uop.fp_val := cs.fp_val + uop.fp_ctrl := cs.fp + + uop.mem_cmd := cs.mem_cmd + uop.mem_size := Mux(cs.mem_cmd.isOneOf(M_SFENCE, M_FLUSH_ALL), Cat(LRS2 =/= 0.U, LRS1 =/= 0.U), inst(13, 12)) + uop.mem_signed := !inst(14) + uop.uses_ldq := cs.uses_ldq + uop.uses_stq := cs.uses_stq + uop.is_amo := cs.is_amo + uop.is_fence := inst === FENCE + uop.is_fencei := inst === FENCE_I + uop.is_sfence := inst === SFENCE_VMA + uop.is_sys_pc2epc := inst === EBREAK || inst === ECALL + uop.is_eret := inst === ECALL || inst === EBREAK || inst === SRET || inst === MRET || inst === DRET + uop.is_unique := cs.inst_unique || uop.is_rocc + uop.is_rocc := inst(6, 0).isOneOf("b0001011".U, "b0101011".U, "b1111011".U) && inst(14, 12).isOneOf( + 0.U, + 2.U, + 3.U, + 4.U, + 6.U, + 7.U + ) + uop.flush_on_commit := cs.flush_on_commit || (csr_en && !csr_ren && io.csr_decode.write_flush) + + //------------------------------------------------------------- + // immediates + + // repackage the immediate, and then pass the fewest number of bits around + val di24_20 = Mux(cs.imm_sel === IS_B || cs.imm_sel === IS_S, inst(11, 7), inst(24, 20)) + val imm_packed = Cat(inst(31, 25), di24_20, inst(19, 12)) + val imm = ImmGen(imm_packed, cs.imm_sel) + val imm_hi = imm >> (immPregSz - 1) + val imm_lo = imm(immPregSz - 1, 0) + val short_imm = imm_hi === 0.U || ~imm_hi === 0.U || cs.imm_sel === IS_F3 + + uop.imm_rename := cs.imm_sel =/= IS_N && cs.imm_sel =/= IS_F3 + uop.imm_packed := imm_packed + uop.imm_sel := cs.imm_sel + when(short_imm) { + uop.imm_rename := false.B + uop.imm_sel := IS_SH + uop.pimm := Mux(cs.imm_sel === IS_F3, inst(14, 12), imm_lo) + } + + uop.fp_rm := Mux(inst(14, 12) === 7.U, io.fcsr_rm, inst(14, 12)) + uop.fp_typ := inst(21, 20) + + //------------------------------------------------------------- + + uop.csr_cmd := cs.csr_cmd + when((cs.csr_cmd === CSR.S || cs.csr_cmd === CSR.C) && LRS1 === 0.U) { + uop.csr_cmd := CSR.R + } + + uop.fcn_dw := cs.fcn_dw + uop.fcn_op := cs.fcn_op + + uop.op1_sel := OP1_RS1 + when(inst === LUI || inst === CSRRWI || inst === CSRRSI || inst === CSRRCI || + inst === WFI || inst === SRET || inst === MRET || inst === DRET) { + uop.op1_sel := OP1_ZERO + }.elsewhen(inst === JAL || inst === JALR || inst === AUIPC) { + uop.op1_sel := OP1_PC + }.elsewhen(Seq(SH1ADD, SH2ADD, SH3ADD, SH1ADD_UW, SH2ADD_UW, SH3ADD_UW, ADD_UW, SLLI_UW).map(_ === inst).orR) { + uop.op1_sel := OP1_RS1SHL + } + + uop.op2_sel := OP2_RS2 + when(cs.is_amo || inst === CSRRW || inst === CSRRS || inst === CSRRC) { + uop.op2_sel := OP2_ZERO + }.elsewhen(inst === CSRRWI || inst === CSRRSI || inst === CSRRCI || + inst === WFI || inst === SRET || inst === DRET || inst === MRET) { + uop.op2_sel := OP2_IMMC + }.elsewhen(inst === JAL || inst === JALR) { + uop.op2_sel := OP2_NEXT + }.elsewhen(Seq(BCLR, BCLRI, BINV, BINVI, BSET, BSETI).map(_ === inst).orR) { + uop.op2_sel := Mux(uop.lrs2_rtype === RT_FIX, OP2_RS2OH, OP2_IMMOH) + }.elsewhen(cs.imm_sel === IS_U || cs.imm_sel === IS_I || cs.imm_sel === IS_S) { + uop.op2_sel := OP2_IMM + } + + uop.br_type := Seq( + (BEQ, B_EQ), + (BNE, B_NE), + (BGE, B_GE), + (BGEU, B_GEU), + (BLT, B_LT), + (BLTU, B_LTU), + (JAL, B_J), + (JALR, B_JR) + ).map { case (c, b) => Mux(inst === c, b, 0.U) }.reduce(_ | _) + + io.deq.uop := uop +} diff --git a/examples/prefab/cores/boom/arch/native/ExecutionUnit.scala b/examples/prefab/cores/boom/arch/native/ExecutionUnit.scala new file mode 100644 index 000000000..0483df5a2 --- /dev/null +++ b/examples/prefab/cores/boom/arch/native/ExecutionUnit.scala @@ -0,0 +1,202 @@ +// Copied from riscv-boom src/main/scala/v4/exu/execution-units/execution-unit.scala and modified for Buckyball. +// Copyright (c) The Regents of the University of California (Regents). +// All Rights Reserved. See LICENSE and LICENSE.SiFive in riscv-boom for license details. + +package examples.core.boom.native + +//------------------------------------------------------------------------------ +// Execution Units +//------------------------------------------------------------------------------ +//------------------------------------------------------------------------------ +// +// The issue window schedules micro-ops onto a specific execution pipeline +// A given execution pipeline may contain multiple functional units; one or more +// read ports, and one or more writeports. + +import scala.collection.mutable.{ArrayBuffer} + +import chisel3._ +import chisel3.util._ + +import freechips.rocketchip.rocket.{BP, CSR, SFenceReq} +import freechips.rocketchip.rocket.constants.{MemoryOpConstants} +import org.chipsalliance.cde.config.{Parameters} +import freechips.rocketchip.tile +import freechips.rocketchip.util._ + +import boom.v4.common._ +import boom.v4.ifu.FTQInfo +import boom.v4.util._ + +import boom.v4.exu._ +import examples.core.boom.native.RoCCShim + +class UniqueExeUnit( + val hasCSR: Boolean = false, + val hasMul: Boolean = false, + val hasDiv: Boolean = false, + val hasIfpu: Boolean = false, + val hasRocc: Boolean = false +)( + implicit p: Parameters) + extends ExecutionUnit("Unq") + with HasIrfReadPorts + with HasImmrfReadPort + with MemoryOpConstants { + def nReaders = 2 + + val io_squash_iss = IO(Output(Bool())) + io_squash_iss := ((io_arb_irf_reqs(0).valid && !io_arb_irf_reqs(0).ready) || + (io_arb_irf_reqs(1).valid && !io_arb_irf_reqs(1).ready)) + + when(io_squash_iss || arb_rebusied) { + val will_replay = arb_uop.valid && !IsKilledByBranch(io_brupdate, io_kill, arb_uop.bits) && !arb_rebusied + arb_uop.valid := will_replay + arb_uop.bits := UpdateBrMask(io_brupdate, arb_uop.bits) + arb_uop.bits.iw_p1_bypass_hint := false.B + arb_uop.bits.iw_p2_bypass_hint := false.B + + rrd_uop.valid := false.B + } + + val exe_int_req = Wire(new FuncUnitReq(xLen)) + exe_int_req.uop := exe_uop.bits + exe_int_req.rs1_data := exe_rs1_data + exe_int_req.rs2_data := exe_rs2_data + exe_int_req.rs3_data := DontCare + exe_int_req.imm_data := exe_imm_data + exe_int_req.pred_data := DontCare + exe_int_req.ftq_info := DontCare + + val io_mul_resp = + if (hasMul) { + fu_types += ((FC_MUL, true.B, "IMul")) + val imul = Module(new PipelinedMulUnit(imulLatency, xLen)) + imul.io.req.valid := exe_uop.valid && exe_uop.bits.fu_code(FC_MUL) + imul.io.req.bits := exe_int_req + imul.io.brupdate := io_brupdate + imul.io.kill := io_kill + imul.io.resp.ready := true.B + + val resp = IO(Output(Valid(new ExeUnitResp(xLen)))) + resp := imul.io.resp + Some(resp) + } else { + assert(!(exe_uop.valid && exe_uop.bits.fu_code(FC_MUL))) + None + } + + val (io_csr_resp, io_sfence) = + if (hasCSR) { + fu_types += ((FC_CSR, true.B, "CSR")) + val alu = Module(new ALUUnit(dataWidth = xLen)) + alu.io.req.valid := exe_uop.valid && exe_uop.bits.fu_code(FC_CSR) + alu.io.req.bits := exe_int_req + alu.io.resp.ready := true.B + alu.io.brupdate := io_brupdate + alu.io.kill := io_kill + + val c = IO(Output(Valid(new CSRResp))) + c.valid := RegNext(alu.io.resp.valid && exe_uop.bits.csr_cmd =/= CSR.N) + c.bits.uop := RegNext(alu.io.resp.bits.uop) + c.bits.data := RegNext(alu.io.resp.bits.data) + c.bits.addr := RegNext(exe_imm_data) + + val s = IO(Valid(new SFenceReq)) + s.valid := RegNext(exe_uop.valid && exe_uop.bits.is_sfence) + s.bits.rs1 := RegNext(exe_uop.bits.pimm(0)) + s.bits.rs2 := RegNext(exe_uop.bits.pimm(1)) + s.bits.addr := RegNext(exe_rs1_data) + s.bits.asid := RegNext(exe_rs2_data) + s.bits.hv := RegNext(exe_uop.bits.mem_cmd === M_HFENCEV) + s.bits.hg := RegNext(exe_uop.bits.mem_cmd === M_HFENCEG) + + (Some(c), Some(s)) + } else { + assert(!(exe_uop.valid && exe_uop.bits.fu_code(FC_CSR))) + assert(!(exe_uop.valid && exe_uop.bits.mem_cmd === M_SFENCE)) + (None, None) + } + + val (io_rocc_resp, io_rocc_core) = + if (hasRocc) { + require(hasCSR) + val rocc_core = IO(new RoCCShimCoreIO) + val rocc_resp = IO(Decoupled(new ExeUnitResp(xLen))) + val rocc = Module(new RoCCShim) + rocc.io.req.valid := exe_uop.valid && exe_uop.bits.is_rocc + rocc.io.req.bits := exe_int_req + rocc.io.brupdate := io_brupdate // We should assert on this somewhere + rocc.io.status := io_status + rocc.io.exception := io_kill + rocc_core <> rocc.io.core + + rocc_resp <> rocc.io.resp + (Some(rocc_resp), Some(rocc_core)) + } else { + assert(!(exe_uop.valid && exe_uop.bits.is_rocc)) + (None, None) + } + + val (io_ifpu_resp) = + if (hasIfpu) { + val ifpu_ready = Wire(Bool()) + fu_types += ((FC_I2F, ifpu_ready, "IFPU")) + + val ifpu = Module(new IntToFPUnit(latency = intToFpLatency)) + ifpu.io.req.valid := exe_uop.valid && exe_uop.bits.fu_code(FC_I2F) + ifpu.io.req.bits := exe_int_req + ifpu.io.req.bits.uop.fp_rm := exe_uop.bits.prs2(4, 2) + ifpu.io.req.bits.uop.fp_typ := exe_uop.bits.prs2(1, 0) + ifpu.io.fcsr_rm := io_fcsr_rm + ifpu.io.brupdate := io_brupdate + ifpu.io.kill := io_kill + + // buffer up results since we share write-port on integer regfile. + val queue = + Module(new BranchKillableQueue( + new ExeUnitResp(xLen + 1), + entries = intToFpLatency + 6 + )) // TODO being overly conservative + queue.io.enq <> ifpu.io.resp + queue.io.brupdate := io_brupdate + queue.io.flush := io_kill + assert(!(queue.io.enq.valid && !queue.io.enq.ready)) + ifpu_ready := RegNext(queue.io.count < 2.U) + + val ifpu_resp = IO(Decoupled(new ExeUnitResp(xLen + 1))) + ifpu_resp <> queue.io.deq + (Some(ifpu_resp)) + } else { + assert(!(exe_uop.valid && exe_uop.bits.fu_code(FC_I2F))) + None + } + + val (io_div_resp) = + if (hasDiv) { + val div_ready = Wire(Bool()) + fu_types += ((FC_DIV, div_ready, "IDiv")) + + val div = Module(new DivUnit(xLen)) + assert(!(div.io.req.valid && !div.io.req.ready)) + div.io.req.valid := exe_uop.valid && exe_uop.bits.fu_code(FC_DIV) + div.io.req.bits := exe_int_req + div.io.brupdate := io_brupdate + div.io.kill := io_kill + + div_ready := (div.io.req.ready && + !(exe_uop.valid && exe_uop.bits.fu_code(FC_DIV)) && + !(rrd_uop.valid && rrd_uop.bits.fu_code(FC_DIV)) && + !(arb_uop.valid && arb_uop.bits.fu_code(FC_DIV))) + + val div_resp = IO(Decoupled(new ExeUnitResp(xLen))) + div_resp <> div.io.resp + Some(div_resp) + } else { + assert(!(exe_uop.valid && exe_uop.bits.fu_code(FC_DIV))) + None + } + + io_ready_fu_types := get_ready_fu_types() + +} diff --git a/examples/prefab/cores/boom/arch/native/Native.scala b/examples/prefab/cores/boom/arch/native/Native.scala new file mode 100644 index 000000000..b91ba97b3 --- /dev/null +++ b/examples/prefab/cores/boom/arch/native/Native.scala @@ -0,0 +1,179 @@ +package examples.core.boom.native + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.annotate +import sifive.enterprise.firrtl.NestedPrefixModulesAnnotation +import org.chipsalliance.cde.config._ +import freechips.rocketchip.diplomacy._ +import freechips.rocketchip.resources.BindingScope +import freechips.rocketchip.devices.tilelink.{DevNullParams, TLError} +import freechips.rocketchip.amba.axi4._ +import freechips.rocketchip.tilelink._ +import freechips.rocketchip.tile._ +import freechips.rocketchip.subsystem._ +import freechips.rocketchip.prci._ +import freechips.rocketchip.interrupts._ +import boom.v4.common._ +import examples.core.boom.native.BoomTile + +class Context extends Bundle { + val satp = UInt(64.W) + val pmp = Vec(8, new Bundle { val cfg = UInt(8.W); val addr = UInt(42.W) }) +} + +class Commands extends Bundle { + + val request = Decoupled(new Bundle { + val instruction = UInt(32.W) + val rs1 = UInt(64.W) + val rs2 = UInt(64.W) + val privilege = UInt(2.W) + val sum = Bool() + val mxr = Bool() + }) + + val busy = Input(Bool()) + val fault = Input(Bool()) +} + +class Accelerator(implicit p: Parameters) extends LazyRoCC(OpcodeSet.custom3) { + val command = BundleBridgeSource(() => new Commands) + + override lazy val module = new LazyRoCCModuleImp(this) { + val c = command.bundle + c.request.valid := io.cmd.valid + c.request.bits.instruction := io.cmd.bits.inst.asUInt + c.request.bits.rs1 := io.cmd.bits.rs1 + c.request.bits.rs2 := io.cmd.bits.rs2 + c.request.bits.privilege := io.cmd.bits.status.dprv + c.request.bits.sum := io.cmd.bits.status.sum + c.request.bits.mxr := io.cmd.bits.status.mxr + io.cmd.ready := c.request.ready + when(io.cmd.fire) { assert(!io.cmd.bits.inst.xd); assert(!io.cmd.bits.status.dv) } + io.busy := c.busy + io.interrupt := c.fault + io.resp.valid := false.B + io.resp.bits := 0.U.asTypeOf(io.resp.bits) + io.mem.req.valid := false.B + io.mem.req.bits := 0.U.asTypeOf(io.mem.req.bits) + io.mem.s1_kill := true.B + io.mem.s2_kill := true.B + io.mem.s1_data := 0.U.asTypeOf(io.mem.s1_data) + io.mem.keep_clock_enabled := false.B + io.fpu_req.valid := false.B + io.fpu_req.bits := 0.U.asTypeOf(io.fpu_req.bits) + io.fpu_resp.ready := true.B + } + +} + +class Native(implicit p: Parameters) extends LazyModule with BindingScope { + val params = p(TilesLocated(InSubsystem)).head.asInstanceOf[BoomTileAttachParams] + val tile = LazyModule(new BoomTile(params.tileParams, params.crossingParams, params.lookup)) + val trace = tile.traceSourceNode.makeSink() + val hart = BundleBridgeSource(() => UInt(1.W)) + val resetVector = BundleBridgeSource(() => UInt(44.W)) + tile.hartIdNode := hart + tile.resetVectorNode := resetVector + val irqs = IntSourceNode(IntSourcePortSimple(5, ports = 1, sources = 1)) + tile.intInwardNode := irqs + + val notifications = Seq(tile.wfiNode, tile.haltNode, tile.ceaseNode).map { source => + val sink = IntSinkNode(IntSinkPortSimple(1, 1)); sink := source; sink + } + + val commands = BundleBridgeSink[Commands]() + commands := tile.roccs.head.asInstanceOf[Accelerator].command + val context = BundleBridgeSink[Context]() + context := tile.buckyballContext + + val dma = AXI4MasterNode(Seq(AXI4MasterPortParameters(Seq(AXI4MasterParameters( + name = "accelerator", + id = IdRange(0, 16), + aligned = true, + maxFlight = Some(4) + ))))) + + val bus = LazyModule(new TLXbar) + + val error = LazyModule(new TLError( + DevNullParams(Seq(AddressSet(0x3000, 0xfff)), maxAtomic = 8, maxTransfer = 64), + beatBytes = 16 + )) + + error.node := bus.node + bus.node := tile.masterNode + bus.node := TLFIFOFixer(TLFIFOFixer.all) := AXI4ToTL() := AXI4UserYanker(Some(4)) := AXI4Fragmenter() := dma + + val memory = AXI4SlaveNode(Seq(AXI4SlavePortParameters( + Seq(AXI4SlaveParameters( + address = AddressSet(0, (BigInt(1) << 44) - 1).subtract(AddressSet(0, 0x7fffffffL)), + regionType = RegionType.UNCACHED, + executable = true, + supportsRead = TransferSizes(1, 64), + supportsWrite = TransferSizes(1, 64), + interleavedId = Some(0) + )), + beatBytes = 16 + ))) + + memory := AXI4Buffer() := AXI4UserYanker() := AXI4IdIndexer(4) := AXI4Deinterleaver(64) := + TLToAXI4() := TLWidthWidget(16) := TLBroadcast(64, 4) := bus.node + + val peripherals = AXI4SlaveNode(Seq(AXI4SlavePortParameters( + Seq(AXI4SlaveParameters( + address = AddressSet(0, 0x7fffffffL).subtract(AddressSet(0x3000, 0xfff)), + regionType = RegionType.UNCACHED, + executable = true, + supportsRead = TransferSizes(1, 16), + supportsWrite = TransferSizes(1, 16), + interleavedId = Some(0) + )), + beatBytes = 16 + ))) + + peripherals := AXI4Buffer() := AXI4UserYanker() := AXI4IdIndexer(4) := AXI4Deinterleaver(64) := + TLToAXI4() := TLWidthWidget(16) := bus.node + + lazy val module = new LazyModuleImp(this) { + annotate(this)(Seq(NestedPrefixModulesAnnotation(this.toNamed, "Boom", true))) + + val io = IO(new Bundle { + val resetVector = Input(UInt(44.W)) + val interrupts = Input(Vec(4, Bool())) + val commands = new Commands + val context = Output(new Context) + }) + + val memoryAXI = memory.makeIOs() + val peripheralsAXI = peripherals.makeIOs() + val dmaAXI = dma.makeIOs() + hart.bundle := 0.U + resetVector.bundle := io.resetVector + irqs.out.head._1 := VecInit(false.B +: io.interrupts.toSeq) + io.commands <> commands.bundle + io.context := context.bundle + } + +} + +object Emit extends App { + + implicit val p: Parameters = + new Config((site, here, up) => { + case TilesLocated(InSubsystem) => up(TilesLocated(InSubsystem)).map { + case tile: BoomTileAttachParams => + tile.copy(tileParams = tile.tileParams.copy(core = tile.tileParams.core.copy(pgLevels = 4))) + } + case BuildRoCC => Seq((p: Parameters) => LazyModule(new Accelerator()(p))) + case MaxHartIdBits => 1 + }) ++ new WithNSmallBooms(1) ++ new BaseSubsystemConfig + + val native = LazyModule(new Native) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + native.module, + args ++ Array("--split-verilog"), + firtoolOpts = Array("--disable-all-randomization") + ) +} diff --git a/examples/prefab/cores/boom/arch/native/NestedPrefixModulesAnnotation.scala b/examples/prefab/cores/boom/arch/native/NestedPrefixModulesAnnotation.scala new file mode 100644 index 000000000..9f4c4a379 --- /dev/null +++ b/examples/prefab/cores/boom/arch/native/NestedPrefixModulesAnnotation.scala @@ -0,0 +1,10 @@ +// Hacked from CIRCT source code. Look like some SiFive internal annotations. +package sifive.enterprise.firrtl + +case class NestedPrefixModulesAnnotation( + target: firrtl.annotations.ModuleTarget, + prefix: String, + inclusive: Boolean = false) + extends firrtl.annotations.SingleTargetAnnotation[firrtl.annotations.ModuleTarget] { + def duplicate(n: firrtl.annotations.ModuleTarget): NestedPrefixModulesAnnotation = this.copy(n) +} diff --git a/examples/prefab/cores/boom/arch/native/Rocc.scala b/examples/prefab/cores/boom/arch/native/Rocc.scala new file mode 100644 index 000000000..2f2cf8861 --- /dev/null +++ b/examples/prefab/cores/boom/arch/native/Rocc.scala @@ -0,0 +1,235 @@ +// Copied from riscv-boom src/main/scala/v4/exu/execution-units/rocc.scala and modified for Buckyball. +// Copyright (c) The Regents of the University of California (Regents). +// All Rights Reserved. See LICENSE and LICENSE.SiFive in riscv-boom for license details. + +package examples.core.boom.native + +//------------------------------------------------------------------------------ + +import chisel3._ +import chisel3.util._ + +import org.chipsalliance.cde.config.Parameters +import freechips.rocketchip.tile.{RoCCCoreIO, RoCCInstruction} +import freechips.rocketchip.rocket._ + +import boom.v4.common._ +import boom.v4.util._ +import boom.v4.exu._ + +/** + * Structure similar to LSU + * - Holds instruction and operand bits prior to issuing RoCC inst to + * accelerator + * - After issue, holds queue of translations between logical and physical + * specifiers to handle RoCC responses + */ +class RoCCShim(implicit p: Parameters) extends BoomModule { + val io = IO(new RoCCShimIO) + + io.req.ready := true.B + io.core.rocc.exception := false.B + io.core.rocc.mem.store_pending := false.B + io.core.rocc.mem.req.ready := false.B + io.core.rocc.mem.s2_nack := false.B + io.core.rocc.mem.s2_nack_cause_raw := false.B + io.core.rocc.mem.s2_uncached := false.B + io.core.rocc.mem.s2_paddr := DontCare + io.core.rocc.mem.resp.valid := false.B + io.core.rocc.mem.resp.bits := DontCare + io.core.rocc.mem.replay_next := false.B + io.core.rocc.mem.s2_xcpt.ma.ld := false.B + io.core.rocc.mem.s2_xcpt.ma.st := false.B + io.core.rocc.mem.s2_xcpt.pf.ld := false.B + io.core.rocc.mem.s2_xcpt.pf.st := false.B + io.core.rocc.mem.s2_xcpt.gf.ld := false.B + io.core.rocc.mem.s2_xcpt.gf.st := false.B + io.core.rocc.mem.s2_xcpt.ae.ld := false.B + io.core.rocc.mem.s2_xcpt.ae.st := false.B + io.core.rocc.mem.s2_gpa := DontCare + io.core.rocc.mem.s2_gpa_is_pte := false.B + io.core.rocc.mem.uncached_resp.map { r => + r.valid := false.B + r.bits := DontCare + } + io.core.rocc.mem.ordered := false.B + io.core.rocc.mem.perf.acquire := false.B + io.core.rocc.mem.perf.release := false.B + io.core.rocc.mem.perf.grant := false.B + io.core.rocc.mem.perf.tlbMiss := false.B + io.core.rocc.mem.perf.blocked := false.B + io.core.rocc.mem.perf.canAcceptStoreThenLoad := false.B + io.core.rocc.mem.perf.canAcceptStoreThenRMW := false.B + io.core.rocc.mem.perf.canAcceptLoadThenLoad := false.B + io.core.rocc.mem.perf.storeBufferEmptyAfterLoad := false.B + io.core.rocc.mem.perf.storeBufferEmptyAfterStore := false.B + io.core.rocc.mem.clock_enabled := false.B + + // RoCC execute queue. Wait for PNR, holds operands and inst bits + val rxq_val = Reg(Vec(numRxqEntries, Bool())) + val rxq_op_val = Reg(Vec(numRxqEntries, Bool())) + val rxq_committed = Reg(Vec(numRxqEntries, Bool())) + val rxq_uop = Reg(Vec(numRxqEntries, new MicroOp())) + val rxq_inst = Reg(Vec(numRxqEntries, UInt(32.W))) + val rxq_rs1 = Reg(Vec(numRxqEntries, UInt(xLen.W))) + val rxq_rs2 = Reg(Vec(numRxqEntries, UInt(xLen.W))) + + // RoCC commit queue. Wait for response, or immediate unbusy + val rcq = Reg(Vec(numRcqEntries, Valid(new RCQEntry))) + val rcq_head = RegInit(0.U(log2Ceil(numRcqEntries).W)) + val rcq_tail = RegInit(0.U(log2Ceil(numRcqEntries).W)) + val rcq_full = WrapInc(rcq_tail, numRcqEntries) === rcq_head + + // The instruction we are waiting for response from + val rxq_head = RegInit(0.U(log2Ceil(numRxqEntries).W)) + // The next instruction we are waiting to "commit" through PNR + val rxq_com_head = RegInit(0.U(log2Ceil(numRxqEntries).W)) + val rxq_tail = RegInit(0.U(log2Ceil(numRxqEntries).W)) + + // Decode + val rocc_idx = WireInit(0.U) + val br_mask = WireInit(0.U(maxBrCount.W)) + var enq_val = false.B + + assert(PopCount(io.core.dis_rocc_vals) <= 1.U) + for (w <- 0 until coreWidth) { + val enq_this = !enq_val && io.core.dis_rocc_vals(w) && io.core.dis_uops(w).is_rocc + when(enq_this) { + rocc_idx := w.U + } + + io.core.rxq_idx(w) := Mux(enq_val, WrapInc(rxq_tail, numRxqEntries), rxq_tail) + + enq_val = enq_val || enq_this + } + + when(enq_val) { + rxq_val(rxq_tail) := true.B + rxq_op_val(rxq_tail) := false.B + rxq_committed(rxq_tail) := false.B + rxq_uop(rxq_tail) := io.core.dis_uops(rocc_idx) + rxq_inst(rxq_tail) := io.core.dis_uops(rocc_idx).inst + rxq_tail := WrapInc(rxq_tail, numRxqEntries) + } + + // Wait for operands + when(io.req.valid && !IsKilledByBranch(io.brupdate, io.exception, io.req.bits.uop)) { + val rxq_idx = io.req.bits.uop.rxq_idx + assert( + io.req.bits.uop.rob_idx === rxq_uop(rxq_idx).rob_idx, + "Mismatch between RoCCUnit request and RoCC execute head" + ) + assert(rxq_val(rxq_idx), "Trying to execute rocc inst without the instruction bits") + + rxq_op_val(rxq_idx) := true.B + rxq_uop(rxq_idx).pdst := io.req.bits.uop.pdst + rxq_rs1(rxq_idx) := io.req.bits.rs1_data + rxq_rs2(rxq_idx) := io.req.bits.rs2_data + } + + // Wait for ROB to OK us to execute + when(rxq_val(rxq_com_head) && + IsOlder(rxq_uop(rxq_com_head).rob_idx, io.core.rob_pnr_idx, io.core.rob_head_idx)) { + rxq_committed(rxq_com_head) := true.B + rxq_com_head := WrapInc(rxq_com_head, numRxqEntries) + } + + // Execute + io.core.rocc.cmd.valid := false.B + io.core.rocc.cmd.bits.inst := rxq_inst(rxq_head).asTypeOf(new RoCCInstruction) + io.core.rocc.cmd.bits.rs1 := rxq_rs1(rxq_head) + io.core.rocc.cmd.bits.rs2 := rxq_rs2(rxq_head) + io.core.rocc.cmd.bits.status := io.status + + val resp_arb = Module(new Arbiter(new ExeUnitResp(xLen), 2)) + resp_arb.io.in(0).valid := false.B + resp_arb.io.in(0).bits.uop := rxq_uop(rxq_head) + resp_arb.io.in(0).bits.data := DontCare + resp_arb.io.in(0).bits.predicated := false.B + resp_arb.io.in(0).bits.fflags.valid := false.B + resp_arb.io.in(0).bits.fflags.bits := DontCare + when(rxq_op_val(rxq_head) && + rxq_val(rxq_head) && + rxq_committed(rxq_head) && + io.core.rocc.cmd.ready && + resp_arb.io.in(0).ready && + !rcq_full) { + io.core.rocc.cmd.valid := true.B + + when(rxq_uop(rxq_head).dst_rtype =/= RT_X) { + rcq_tail := WrapInc(rcq_tail, numRcqEntries) + rcq(rcq_tail).valid := true.B + rcq(rcq_tail).bits.uop := rxq_uop(rxq_head) + }.otherwise { + resp_arb.io.in(0).valid := true.B + } + + rxq_val(rxq_head) := false.B + rxq_head := WrapInc(rxq_head, numRxqEntries) + } + + io.core.rxq_full := WrapInc(rxq_tail, numRxqEntries) === rxq_head + io.core.rxq_empty := rxq_tail === rxq_head + + //-------------------------- + // Branches + for (i <- 0 until numRxqEntries) { + when(rxq_val(i)) { + rxq_uop(i).br_mask := GetNewBrMask(io.brupdate, rxq_uop(i)) + when(IsKilledByBranch(io.brupdate, io.exception, rxq_uop(i))) { + rxq_val(i) := false.B + rxq_op_val(i) := false.B + } + } + } + when(io.brupdate.b2.mispredict && !io.exception) { + rxq_tail := io.brupdate.b2.uop.rxq_idx + } + + //-------------------------- + // Exception / Reset + + when(reset.asBool) { + rxq_tail := 0.U + rxq_head := 0.U + rxq_com_head := 0.U + for (i <- 0 until numRxqEntries) { + rxq_val(i) := false.B + rxq_op_val(i) := false.B + rxq_committed(i) := false.B + } + }.elsewhen(io.exception) { + rxq_tail := rxq_com_head + for (i <- 0 until numRxqEntries) { + when(!rxq_committed(i)) { + rxq_val(i) := false.B + rxq_op_val(i) := false.B + } + } + } + + //------------------ + // Handle responses + + resp_arb.io.in(1).valid := io.core.rocc.resp.valid + resp_arb.io.in(1).bits.uop := rcq(rcq_head).bits.uop + resp_arb.io.in(1).bits.data := io.core.rocc.resp.bits.data + resp_arb.io.in(1).bits.predicated := io.core.rocc.resp.bits.data + resp_arb.io.in(1).bits.fflags.valid := false.B + resp_arb.io.in(1).bits.fflags.bits := DontCare + io.core.rocc.resp.ready := resp_arb.io.in(1).ready + io.resp <> resp_arb.io.out + + when(io.core.rocc.resp.fire) { + assert(rcq(rcq_head).valid && rcq(rcq_head).bits.uop.ldst === io.core.rocc.resp.bits.rd) + rcq(rcq_head).valid := false.B + rcq_head := WrapInc(rcq_head, numRcqEntries) + } + + when(reset.asBool) { + for (i <- 0 until numRcqEntries) { + rcq(i).valid := false.B + } + } + +} diff --git a/examples/prefab/cores/boom/arch/native/Tile.scala b/examples/prefab/cores/boom/arch/native/Tile.scala new file mode 100644 index 000000000..3ab55004e --- /dev/null +++ b/examples/prefab/cores/boom/arch/native/Tile.scala @@ -0,0 +1,295 @@ +// Copied from riscv-boom src/main/scala/v4/common/tile.scala and modified for Buckyball. +// Copyright (c) The Regents of the University of California (Regents). +// All Rights Reserved. See LICENSE and LICENSE.SiFive in riscv-boom for license details. + +package examples.core.boom.native + +import chisel3._ +import chisel3.util.{Queue, RRArbiter} + +import scala.collection.mutable.{ListBuffer} + +import org.chipsalliance.cde.config._ +import freechips.rocketchip.subsystem._ +import freechips.rocketchip.devices.tilelink._ +import freechips.rocketchip.diplomacy._ +import freechips.rocketchip.prci._ +import freechips.rocketchip.rocket._ +import freechips.rocketchip.subsystem.{RocketCrossingParams} +import freechips.rocketchip.tilelink._ +import freechips.rocketchip.interrupts._ +import freechips.rocketchip.util._ +import freechips.rocketchip.tile._ + +import boom.v4.exu._ +import boom.v4.ifu._ +import boom.v4.lsu._ +import boom.v4.util.{BoomCoreStringPrefix} +import freechips.rocketchip.prci.ClockSinkParameters +import freechips.rocketchip.trace.{TraceEncoderController, TraceEncoderParams, TraceSinkArbiter} +import boom.v4.common._ +import examples.core.boom.native.{BoomCore, Context} + +/** + * BOOM tile + */ +class BoomTile private ( + val boomParams: BoomTileParams, + crossing: ClockCrossingType, + lookup: LookupByHartIdImpl, + q: Parameters) + extends BaseTile(boomParams, crossing, lookup, q) + with SinksExternalInterrupts + with SourcesExternalNotifications { + + // Private constructor ensures altered LazyModule.p is used implicitly + def this( + params: BoomTileParams, + crossing: HierarchicalElementCrossingParamsLike, + lookup: LookupByHartIdImpl + )( + implicit p: Parameters + ) = + this(params, crossing.crossingType, lookup, p) + + val buckyballContext = BundleBridgeSource(() => new Context) + val intOutwardNode = None + val masterNode = TLIdentityNode() + val slaveNode = TLIdentityNode() + + val tile_master_blocker = + tileParams.blockerCtrlAddr + .map(BasicBusBlockerParams(_, xBytes, masterPortBeatBytes, deadlock = true)) + .map(bp => LazyModule(new BasicBusBlocker(bp))) + + tile_master_blocker.foreach(lm => connectTLSlave(lm.controlNode, xBytes)) + + // TODO: this doesn't block other masters, e.g. RoCCs + tlOtherMastersNode := TLBuffer() := tile_master_blocker.map(_.node := tlMasterXbar.node) getOrElse { + tlMasterXbar.node + } + masterNode :=* tlOtherMastersNode + + val cpuDevice: SimpleDevice = new SimpleDevice("cpu", Seq("ucb-bar,boom0", "riscv")) { + override def parent = Some(ResourceAnchors.cpus) + + override def describe(resources: ResourceBindings): Description = { + val Description(name, mapping) = super.describe(resources) + Description( + name, + mapping ++ + cpuProperties ++ + nextLevelCacheProperty ++ + tileProperties + ) + } + + } + + ResourceBinding { + Resource(cpuDevice, "reg").bind(ResourceAddress(tileId)) + } + + override def makeMasterBoundaryBuffers(crossing: ClockCrossingType)(implicit p: Parameters) = crossing match { + case _: RationalCrossing => + if (!boomParams.boundaryBuffers) TLBuffer(BufferParams.none) + else TLBuffer(BufferParams.none, BufferParams.flow, BufferParams.none, BufferParams.flow, BufferParams(1)) + case _ => TLBuffer(BufferParams.none) + } + + override def makeSlaveBoundaryBuffers(crossing: ClockCrossingType)(implicit p: Parameters) = crossing match { + case _: RationalCrossing => + if (!boomParams.boundaryBuffers) TLBuffer(BufferParams.none) + else TLBuffer(BufferParams.flow, BufferParams.none, BufferParams.none, BufferParams.none, BufferParams.none) + case _ => TLBuffer(BufferParams.none) + } + + override lazy val module = new BoomTileModuleImp(this) + + // DCache + lazy val dcache: BoomNonBlockingDCache = LazyModule(new BoomNonBlockingDCache(tileId)) + val dCacheTap = TLIdentityNode() + tlMasterXbar.node := dCacheTap := TLBuffer() := TLWidthWidget( + tileParams.dcache.get.rowBits / 8 + ) := visibilityNode := dcache.node + + // Frontend/ICache + val frontend = LazyModule(new BoomFrontend(tileParams.icache.get, tileId)) + frontend.resetVectorSinkNode := resetVectorNexusNode + tlMasterXbar.node := TLBuffer() := TLWidthWidget(tileParams.icache.get.fetchBytes) := frontend.masterNode + + require( + tileParams.dcache.get.rowBits == tileParams.icache.get.fetchBytes * 8, + s"${tileParams.dcache.get.rowBits} != ${tileParams.icache.get.rowBits}" + ) + + // ROCC + val roccs = p(BuildRoCC).map(_(p)) + val roccCSRs = roccs.map(_.roccCSRs) + require( + roccCSRs.flatten.map(_.id).toSet.size == roccCSRs.flatten.size, + "LazyRoCC instantiations require overlapping CSRs" + ) + roccs.map(_.atlNode).foreach(atl => tlMasterXbar.node :=* atl) + roccs.map(_.tlNode).foreach(tl => tlOtherMastersNode :=* tl) + + val trace_encoder_controller = boomParams.traceParams.map { t => + val trace_encoder_controller = LazyModule(new TraceEncoderController(t.encoderBaseAddr, xBytes, tileId)) + connectTLSlave(trace_encoder_controller.node, xBytes) + trace_encoder_controller + } + + val trace_encoder = boomParams.traceParams match { + case Some(t) => Some(t.buildEncoder(p)) + case None => None + } + + val (trace_sinks, traceSinkIds) = boomParams.traceParams match { + case Some(t) => t.buildSinks.map(_(p)).unzip + case None => (Nil, Nil) + } + + DisableMonitors(implicit p => tlSlaveXbar.node :*= slaveNode) +} + +/** + * BOOM tile implementation + * + * @param outer top level BOOM tile + */ +class BoomTileModuleImp(outer: BoomTile) extends BaseTileModuleImp(outer) { + + val core = Module(new BoomCore(outer.roccCSRs)(outer.p)) + outer.buckyballContext.bundle.satp := core.io.ptw.ptbr.asUInt + require(core.io.ptw.pmp.size == 8) + outer.buckyballContext.bundle.pmp.zip(core.io.ptw.pmp).foreach { case (out, in) => + out.cfg := in.cfg.asUInt + out.addr := in.addr + } + val lsu = Module(new LSU()(outer.p, outer.dcache.module.edge)) + + val ptwPorts = ListBuffer(lsu.io.ptw, outer.frontend.module.io.ptw, core.io.ptw_tlb) + + val hellaCachePorts = ListBuffer[HellaCacheIO]() + + outer.reportWFI(None) // TODO: actually report this? + + outer.decodeCoreInterrupts(core.io.interrupts) // Decode the interrupt vector + + // Pass through various external constants and reports + outer.traceSourceNode.bundle <> core.io.trace + outer.bpwatchSourceNode.bundle <> DontCare // core.io.bpwatch + core.io.hartid := outer.hartIdSinkNode.bundle + + // Connect the core pipeline to other intra-tile modules + outer.frontend.module.io.cpu <> core.io.ifu + core.io.lsu <> lsu.io.core + + //fpuOpt foreach { fpu => core.io.fpu <> fpu.io } RocketFpu - not needed in boom + core.io.rocc := DontCare + + // RoCC + if (outer.roccs.size > 0) { + val (respArb, cmdRouter) = { + val respArb = Module(new RRArbiter(new RoCCResponse()(outer.p), outer.roccs.size)) + val cmdRouter = Module(new RoccCommandRouter(outer.roccs.map(_.opcodes))(outer.p)) + outer.roccs.zipWithIndex.foreach { case (rocc, i) => + ptwPorts ++= rocc.module.io.ptw + rocc.module.io.cmd <> cmdRouter.io.out(i) + val dcIF = Module(new SimpleHellaCacheIF()(outer.p)) + dcIF.io.requestor <> rocc.module.io.mem + hellaCachePorts += dcIF.io.cache + respArb.io.in(i) <> Queue(rocc.module.io.resp) + } + // first keep fpu ios unconnected + val fp_ios = outer.roccs.map { r => + val roccio = r.module.io + roccio.fpu_req.ready := true.B + roccio.fpu_resp.valid := false.B + roccio.fpu_resp.bits := DontCare + } + // Create this FPU just for RoCC + val nFPUPorts = outer.roccs.filter(_.usesFPU).size + if (nFPUPorts > 0) { + val fpuOpt = outer.tileParams.core.fpu.map(params => Module(new freechips.rocketchip.tile.FPU(params)(outer.p))) + // TODO: Check this FPU works properly + fpuOpt foreach { fpu => + // This FPU does not get CPU requests + fpu.io := DontCare + fpu.io.fcsr_rm := core.io.fcsr_rm + fpu.io.ll_resp_val := false.B + fpu.io.valid := false.B + fpu.io.killx := false.B + fpu.io.killm := false.B + + val fpArb = Module(new InOrderArbiter(new FPInput()(outer.p), new FPResult()(outer.p), nFPUPorts)) + val fp_rocc_ios = outer.roccs.filter(_.usesFPU).map(_.module.io) + fpArb.io.in_req <> fp_rocc_ios.map(_.fpu_req) + fp_rocc_ios.zip(fpArb.io.in_resp).foreach { + case (rocc, arb) => rocc.fpu_resp <> arb + } + fpu.io.cp_req <> fpArb.io.out_req + fpArb.io.out_resp <> fpu.io.cp_resp + } + } + (respArb, cmdRouter) + } + + cmdRouter.io.in <> core.io.rocc.cmd + outer.roccs.foreach(_.module.io.exception := core.io.rocc.exception) + core.io.rocc.resp <> respArb.io.out + core.io.rocc.busy <> (cmdRouter.io.busy || outer.roccs.map(_.module.io.busy).reduce(_ || _)) + core.io.rocc.interrupt := outer.roccs.map(_.module.io.interrupt).reduce(_ || _) + (core.io.rocc.csrs zip outer.roccs.map(_.module.io.csrs).flatten).foreach(t => t._2 <> t._1) + } + + // PTW + val ptw = Module(new PTW(ptwPorts.length)(outer.dcache.node.edges.out(0), outer.p)) + core.io.ptw <> ptw.io.dpath + ptw.io.requestor <> ptwPorts.toSeq + ptw.io.mem +=: hellaCachePorts + + // LSU IO + val hellaCacheArb = Module(new HellaCacheArbiter(hellaCachePorts.length)(outer.p)) + hellaCacheArb.io.requestor <> hellaCachePorts.toSeq + lsu.io.hellacache <> hellaCacheArb.io.mem + outer.dcache.module.io.lsu <> lsu.io.dmem + + if (outer.boomParams.traceParams.isDefined) { + core.io.trace_core_ingress.get <> outer.trace_encoder.get.module.io.in + outer.trace_encoder_controller.foreach { lm => + outer.trace_encoder.get.module.io.control <> lm.module.io.control + } + + val trace_sink_arbiter = Module(new TraceSinkArbiter( + outer.traceSinkIds, + use_monitor = outer.boomParams.traceParams.get.useArbiterMonitor, + monitor_name = outer.boomParams.uniqueName + )) + + trace_sink_arbiter.io.target := outer.trace_encoder.get.module.io.control.target + trace_sink_arbiter.io.in <> outer.trace_encoder.get.module.io.out + + core.io.traceStall := outer.trace_encoder.get.module.io.stall + + outer.trace_sinks.zip(outer.traceSinkIds).foreach { case (sink, id) => + val index = outer.traceSinkIds.indexOf(id) + sink.module.io.trace_in <> trace_sink_arbiter.io.out(index) + } + } else { + core.io.traceStall := false.B // no trace encoder, so no stall + } + + // Generate a descriptive string + val frontendStr = outer.frontend.module.toString + val coreStr = core.toString + + val boomTileStr = + (BoomCoreStringPrefix(s"======BOOM Tile ${outer.tileId} Params======") + "\n" + + frontendStr + + coreStr + "\n") + + override def toString: String = boomTileStr + + print(boomTileStr) +} diff --git a/examples/prefab/cores/boom/configs/Params.scala b/examples/prefab/cores/boom/configs/Params.scala new file mode 100644 index 000000000..c718141c0 --- /dev/null +++ b/examples/prefab/cores/boom/configs/Params.scala @@ -0,0 +1,64 @@ +package examples.core.boom.configs + +import buckyball.config.CorePlacement +import framework.ip.bus.{axi4, chi => chiBus} + +import framework.core.memory.access.preflight.configs.PmpParams +import framework.ip.memory.interlock.{Params => TrackingParam} +import framework.core.memory.access.mmu.configs.{Params => WalkerParam} +import framework.acc.configs.AccParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.memdomain.tomem.configs.{Params => TransportParam} + +case class Params( + chi: chiBus.Params, + axi: axi4.Params, + hartId: Int, + resetVector: BigInt, + mtimeAddress: BigInt, + permission: PmpParams, + tracking: TrackingParam, + walker: WalkerParam, + acc: Option[AccParam]) { + require(chi.nodeIdBits == 7 && chi.addressBits == 44 && chi.dataBits == 256) + require(axi.addressBits == 44 && axi.dataBits == 128 && axi.idBits == 4) + require(hartId == 0) + require((mtimeAddress & 7) == 0) + require(permission.addressBits == 44 && permission.entries == 8) + acc.foreach(a => require(!a.memDomain.sharedEnable && a.transport.axi == axi)) +} + +object Params { + + def fromProto(core: CorePlacement): Params = { + require(core.getThreadIndicesCount == 0) + require(core.getImplementation.unpack(classOf[buckyball.components.boom.Params]).getProfile == "SmallBoomV4") + val h = core + val bus = chiBus.Params.fromProto(h.getMemory.getHome.getRequesters) + val acc = Option.when(core.hasAccelerator) { + val a = core.getAccelerator + AccParam( + FrontendParam.fromProto(a.getFrontend, core.getParam.getXLen, core.getHartId), + MemDomainParam.fromProto(a.getMem, core.getSharedMem, core.getVirtualBankCount, 1), + GpDomainParam.fromProto(a.getGpdomain), + BallDomainParam.fromProto(a.getBalldomain), + TransportParam.fromProto(a.getTransport) + ) + } + Params( + bus, + axi4.Params.fromProto(h.getMemory.getUncached.getAxi), + h.getHartId, + BigInt(java.lang.Long.toUnsignedString(h.getResetVector)), + BigInt(java.lang.Long.toUnsignedString(h.getMtimeAddress)), + PmpParams(44, 8), + TrackingParam.fromProto(h.getMemory.getDcache.getTracking), + WalkerParam(bus, h.getMemory.getAccess.getTranslationEntries), + acc + ) + } + +} diff --git a/examples/prefab/cores/boom/configs/boom_params.proto b/examples/prefab/cores/boom/configs/boom_params.proto new file mode 100644 index 000000000..1f475593f --- /dev/null +++ b/examples/prefab/cores/boom/configs/boom_params.proto @@ -0,0 +1,7 @@ +syntax = "proto2"; +package buckyball.components.boom; +option java_package = "buckyball.components.boom"; +option java_multiple_files = true; +message Params { + required string profile = 1; +} diff --git a/examples/prefab/cores/boom/stack/CMakeLists.txt b/examples/prefab/cores/boom/stack/CMakeLists.txt new file mode 100644 index 000000000..3f5c0900c --- /dev/null +++ b/examples/prefab/cores/boom/stack/CMakeLists.txt @@ -0,0 +1,25 @@ +# ===----------------------------------------------------------------------=== +# The boom core: a CPU core without hardware threads; jobs run on the core. +# +# bbdev stack --build "--design " +# +# builds it for each design that uses it, configured with DESIGN_CMAKE +# (generated from design.pb) and CORE. Products: +# execute.o runs jobs on the core itself; links into the device runtime +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(boom-core LANGUAGES NONE) +include("${DESIGN_CMAKE}") + +set(RUNTIME "${DESIGN_REPO}/stack/runtime") +find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) + +add_custom_command( + OUTPUT execute.o + COMMAND "${LINUX_CC}" -O2 -fPIC -ffreestanding -fno-builtin -msmall-data-limit=0 + "-DCORE_SIGNATURE=${DESIGN_DEVICE_CORES_${DESIGN_CORE_HART_${CORE}}_SIGNATURE}ULL" "-I${RUNTIME}/scheduler" + -c "${RUNTIME}/execute/direct.c" -o execute.o + DEPENDS "${RUNTIME}/execute/direct.c" "${RUNTIME}/scheduler/runtime.h" + VERBATIM) + +add_custom_target(execute ALL DEPENDS execute.o) diff --git a/examples/prefab/cores/boom/thirdparty/riscv-boom b/examples/prefab/cores/boom/thirdparty/riscv-boom new file mode 160000 index 000000000..ba03175e0 --- /dev/null +++ b/examples/prefab/cores/boom/thirdparty/riscv-boom @@ -0,0 +1 @@ +Subproject commit ba03175e0a3cf5c0d5a7f14d385ac1c8998c4428 diff --git a/examples/prefab/cores/boom/thirdparty/rocket-chip b/examples/prefab/cores/boom/thirdparty/rocket-chip new file mode 160000 index 000000000..ece7b9ad5 --- /dev/null +++ b/examples/prefab/cores/boom/thirdparty/rocket-chip @@ -0,0 +1 @@ +Subproject commit ece7b9ad544b07df39cd13b6fd7236562a26355d diff --git a/examples/prefab/cores/boom/workloads/ctests/CMakeLists.txt b/examples/prefab/cores/boom/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..416956421 --- /dev/null +++ b/examples/prefab/cores/boom/workloads/ctests/CMakeLists.txt @@ -0,0 +1 @@ +add_buckyball_ctest(memory_matrix.c) diff --git a/examples/prefab/cores/boom/workloads/ctests/memory_matrix.c b/examples/prefab/cores/boom/workloads/ctests/memory_matrix.c new file mode 100644 index 000000000..d8f294adb --- /dev/null +++ b/examples/prefab/cores/boom/workloads/ctests/memory_matrix.c @@ -0,0 +1,47 @@ +#include + +static volatile int32_t a[32][32], b[32][16], c[32][16]; +static volatile uint64_t atomic_word; +static volatile uint64_t upgrade_word[8] __attribute__((aligned(64))) = {7}; + +int main(void) { + for (int i = 0; i < 32; ++i) + for (int k = 0; k < 32; ++k) + a[i][k] = (i * 3 + k * 5) % 13 - 6; + for (int k = 0; k < 32; ++k) + for (int j = 0; j < 16; ++j) + b[k][j] = (k * 7 + j * 2) % 11 - 5; + for (int i = 0; i < 32; ++i) + for (int j = 0; j < 16; ++j) { + int32_t sum = 0; + for (int k = 0; k < 32; ++k) + sum += a[i][k] * b[k][j]; + c[i][j] = sum; + } + for (int i = 0; i < 32; ++i) + for (int j = 0; j < 16; ++j) { + int32_t expected = 0; + for (int k = 0; k < 32; ++k) + expected += ((i * 3 + k * 5) % 13 - 6) * ((k * 7 + j * 2) % 11 - 5); + if (c[i][j] != expected) + return 2; + } + uint64_t old, value = 19; + __asm__ volatile("amoadd.d.aqrl %0, %2, (%1)" + : "=r"(old) + : "r"(&atomic_word), "r"(value) + : "memory"); + if (old != 0 || atomic_word != 19) + return 3; + if (upgrade_word[0] != 7) + return 5; + upgrade_word[0] = 23; + if (upgrade_word[0] != 23) + return 6; + uint64_t before, after; + __asm__ volatile("rdcycle %0" : "=r"(before)); + __asm__ volatile("rdcycle %0" : "=r"(after)); + if (before == 0 || after <= before) + return 4; + return 0; +} diff --git a/examples/prefab/cores/reduce/arch/ReduceCore.scala b/examples/prefab/cores/reduce/arch/ReduceCore.scala new file mode 100644 index 000000000..bdf148040 --- /dev/null +++ b/examples/prefab/cores/reduce/arch/ReduceCore.scala @@ -0,0 +1,126 @@ +package examples.core.reduce + +import sims.configs.SimParam +import framework.device.cbus.CoreFactory +import buckyball.config.{CorePlacement, Design} + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.core.configs.CoreParam +import framework.core.clink.{CLinkIO, HasCLink} +import framework.device.cbus.{C2CTransfer, Control, SharedAxiEndpoint} +import framework.core.controller.admission.DirectAdmission +import framework.core.tbus.SharedNetwork +import framework.core.tbus.configs.Layout +import framework.core.shm.SHM +import framework.acc.debug.Debug +import framework.ip.bus.chi.rnf.AxiCache + +/** Native execution core with a direct BBus accelerator and shared-bank C2C access. */ +@instantiable +class ReduceCore(corep: CoreParam, simp: SimParam) extends Module with HasCLink { + val diffTest = simp.diffTest + val accp = corep.acc.get + require(corep.threads.isEmpty && !corep.clink.backing && corep.clink.requesters == 3) + require(corep.clink.shared && accp.memDomain.sharedEnable && !accp.gpDomain.enable) + require(corep.transferLocalAccess && corep.transferLeaseAccess && corep.tbus.get.router.physicalPorts == 4) + @public + val clink = IO(new CLinkIO(corep.clink)) + + val loom: Instance[Loom] = Instantiate(new Loom(corep)) + val admission: Instance[DirectAdmission] = Instantiate(new DirectAdmission(corep.controller.commands)) + + val acc: Instance[Acc] = Instantiate(new Acc( + accp, + corep.privileged.permission, + corep.memory.dcache.tracking, + corep.memory.memoryAccess.walker, + simp + )) + + loom.io.interrupts := clink.interrupts(0) + admission.io.execution <> loom.io.commands + loom.io.drained := admission.io.drained + admission.io.accelerator <> acc.io.rocc + loom.io.memory <> acc.io.memory + clink.memory.uncached <> loom.io.uncached + clink.memory.coherent.take(2).zip(loom.io.coherent).foreach { case (port, cache) => port <> cache } + + val dma = Instantiate(new AxiCache(accp.transport.axi, corep.dmaCache.get)) + dma.io.axi <> acc.io.ddr + dma.io.axi.ar.bits.cache := 15.U + dma.io.axi.aw.bits.cache := 15.U + clink.memory.coherent(2) <> dma.io.chi + clink.memory.ddr.aw.valid := false.B + clink.memory.ddr.aw.bits := 0.U.asTypeOf(clink.memory.ddr.aw.bits) + clink.memory.ddr.w.valid := false.B + clink.memory.ddr.w.bits := 0.U.asTypeOf(clink.memory.ddr.w.bits) + clink.memory.ddr.ar.valid := false.B + clink.memory.ddr.ar.bits := 0.U.asTypeOf(clink.memory.ddr.ar.bits) + clink.memory.ddr.b.ready := false.B + clink.memory.ddr.r.ready := false.B + + val banks = + Instantiate(new SharedNetwork( + accp.frontend, + accp.memDomain, + corep.tbus.get, + Seq(corep.hartId), + Seq(corep.hartId), + simp + )) + + val shm = Instantiate(new SHM(accp.frontend, accp.memDomain, corep.tbus.get)) + shm.io <> banks.io.shm + banks.io.threads(0).bank <> acc.io.shm.bank + banks.io.threads(0).move <> acc.io.shm.move + banks.io.threads(0).barrierArrive := acc.io.shm.barrierArrive + acc.io.shm.barrierRelease := banks.io.threads(0).barrierRelease + acc.io.shared_bank_hashes.zip(banks.io.bankHashes).foreach { case (sink, source) => sink := source } + + val control = Instantiate(new Control(accp.memDomain, corep.c2c, corep.transferLeaseAccess, corep.transferLocalAccess)) + val transfer = Instantiate(new C2CTransfer(corep.c2c)) + val endpoint = Instantiate(new SharedAxiEndpoint(corep.c2c)) + control.io.coreId := corep.hartId.U + control.io.request <> admission.io.remote.request + admission.io.remote.reply <> control.io.reply + admission.io.remote.busy := !control.io.idle || endpoint.io.busy + transfer.io.command <> control.io.command + control.io.completion <> transfer.io.completion + banks.io.external.get.memory(0) <> transfer.io.source + banks.io.external.get.memory(1) <> endpoint.io.read + banks.io.external.get.memory(2) <> endpoint.io.write + banks.io.external.get.memory(3) <> control.io.local.get + banks.io.external.get.lease.get <> control.io.lease.get + clink.c2c.get.tx <> transfer.io.mem + endpoint.io.mem <> clink.c2c.get.rx + + val debug = Instantiate(new Debug( + accp.frontend, + accp.memDomain, + accp.gpDomain, + accp.ballDomain, + simp, + corep.hartId, + corep.hartId + )) + + debug.io.ant.valid := false.B + debug.io.ant.bits := 0.U.asTypeOf(debug.io.ant.bits) + debug.io.acc <> acc.io.debug +} + +object Reduce extends CoreFactory { + val component = "reduce" + def parameters(design: Design, source: CorePlacement): CoreParam = + CoreParam.fromProto(design, source, externalHome = true) + + def instantiate( + source: CorePlacement, + corep: CoreParam, + simp: SimParam, + traceIds: Seq[Int] + ): CLinkIO = + Instantiate(new ReduceCore(corep, simp)).clink + +} diff --git a/examples/prefab/cores/reduce/arch/acc.scala b/examples/prefab/cores/reduce/arch/acc.scala new file mode 100644 index 000000000..9f538a99c --- /dev/null +++ b/examples/prefab/cores/reduce/arch/acc.scala @@ -0,0 +1,119 @@ +package examples.core.reduce + +import framework.acc.balldomain.bbus.generated.BallComponents +import sims.configs.SimParam +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.configs.AccParam +import framework.core.memory.access.preflight.configs.PmpParams +import framework.ip.memory.interlock.{Params => TrackingParam} +import framework.core.memory.access.mmu.configs.{Params => WalkerParam} +import framework.acc.debug.AcceleratorDebugPort +import framework.acc.frontend.RoCCIO +import framework.acc.frontend.Frontend +import framework.acc.memdomain.{MemDomain, PhysicalBankHash} +import framework.acc.gpdomain.GpDomain +import framework.acc.balldomain.BallDomain +import framework.acc.balldomain.bbus.BBus +import framework.core.memory.MemoryPort +import framework.thread.tlink.SharedLink +import framework.ip.bus.axi4 + +@instantiable +class Acc( + val accp: AccParam, + val permissionp: PmpParams, + val trackingp: TrackingParam, + val walkerp: WalkerParam, + val simp: SimParam) + extends Module { + val commandDataBits = accp.frontend.commandDataBits + val ownerId = accp.frontend.ownerId + val sharedPorts = accp.memDomain.sharedPorts + val diffTest = simp.diffTest + val frontp = accp.frontend + val memp = accp.memDomain + val gpp = accp.gpDomain + val ballp = accp.ballDomain + val transportp = accp.transport + + val sharedHashCount = if (memp.sharedEnable) memp.sharedBankNum else 0 + + @public + val io = IO(new Bundle { + val debug = new AcceleratorDebugPort(frontp, memp, gpp, ballp, simp) + + val rocc = new RoCCIO(commandDataBits, permissionp) + val ddr = new axi4.Port(transportp.axi) + val memory = new MemoryPort(trackingp, walkerp) + val shm = new SharedLink(frontp, memp) + + val shared_bank_hashes = Option.when(diffTest && memp.sharedEnable)( + Input(Vec(sharedHashCount, new PhysicalBankHash(frontp))) + ) + + }) + + // --- Instantiate domains --- + val frontend: Instance[Frontend] = Instantiate(new Frontend(frontp, memp, gpp, ballp, permissionp, simp)) + + val ballDomain: Instance[BallDomain] = + Instantiate(new BallDomain(frontp, memp, gpp, ballp, permissionp, BallComponents.factories)) + + val memDomain: Instance[MemDomain] = + Instantiate(new MemDomain(frontp, memp, gpp, transportp, permissionp, trackingp, walkerp, simp)) + + val gpDomain = + if (gpp.enable) Some(Instantiate(new GpDomain(frontp, memp, gpp))) + else None + + memDomain.io.shared_bank_hashes.foreach(_ := io.shared_bank_hashes.get) + + io.debug.frontend <> frontend.io.debug + io.debug.banks.zip(memDomain.io.bank_hashes).foreach { case (sink, source) => sink := source } + io.debug.trace.memory := memDomain.io.traceMemory + io.debug.trace.ballStarted := ballDomain.io.traceStarted + io.debug.trace.ballCompleted := ballDomain.io.traceCompleted + + io.rocc <> frontend.io.rocc + io.memory <> memDomain.io.memory + io.ddr <> memDomain.io.ddr + + // --- Frontend -> BallDomain --- + memDomain.io.ball <> ballDomain.io.memory + ballDomain.io.issue <> frontend.io.ballRequest + frontend.io.ballCompletion <> ballDomain.io.complete + + // --- Frontend -> MemDomain --- + memDomain.io.issue <> frontend.io.memoryRequest + frontend.io.memoryCompletion <> memDomain.io.complete + memDomain.io.mapping <> frontend.io.mappingUpdate + + if (gpp.enable) { + val controller = gpDomain.get + controller.io.command <> frontend.io.gpRequest.get + frontend.io.gpCompletion.get <> controller.io.complete + controller.io.image <> memDomain.io.image.get + memDomain.io.gpBanks.get <> controller.io.banks + memDomain.io.gpLayoutRequest.get <> controller.io.layoutRequest + controller.io.layoutResponse <> memDomain.io.gpLayoutResponse.get + ballDomain.io.kernel.get <> controller.io.ball + } + + memDomain.io.banks <> ballDomain.io.banks + memDomain.io.ballLayoutRequest <> ballDomain.io.layoutRequest + ballDomain.io.layoutResponse <> memDomain.io.ballLayoutResponse + + // --- Shared memory passthrough --- + io.shm.bank.requests <> memDomain.io.shared.bank.requests + io.shm.move <> memDomain.io.shared.move + io.shm.bank.local <> memDomain.io.shared.bank.local + io.shm.bank.config <> memDomain.io.shared.bank.config + io.shm.bank.mapping <> memDomain.io.shared.bank.mapping + + // --- Barrier passthrough --- + io.shm.barrierArrive := frontend.io.barrier.arrive + frontend.io.barrier.release := io.shm.barrierRelease + +} diff --git a/examples/prefab/cores/reduce/arch/build.sc b/examples/prefab/cores/reduce/arch/build.sc new file mode 100644 index 000000000..a2650c0fb --- /dev/null +++ b/examples/prefab/cores/reduce/arch/build.sc @@ -0,0 +1,14 @@ +import mill._ +import mill.scalalib._ +import $file.^.^.^.^.{build => examples} + +val componentRoot: os.Path = millSourcePath / os.up + +object library extends SbtModule { + override def scalaVersion = "2.13.16" + override def moduleDeps = Seq(examples.library) + override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") + override def sources = T.sources(componentRoot / "arch") +} diff --git a/examples/prefab/cores/reduce/arch/loom.scala b/examples/prefab/cores/reduce/arch/loom.scala new file mode 100644 index 000000000..d4af11343 --- /dev/null +++ b/examples/prefab/cores/reduce/arch/loom.scala @@ -0,0 +1,38 @@ +package examples.core.reduce + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.core.configs.CoreParam +import framework.core.execution.Execution +import framework.core.memory.{MemoryPort, MemorySystem} +import framework.core.controller.admission.Commands +import framework.core.clink.CoreInterrupts +import framework.ip.bus.{axi4, chi} + +@instantiable +class Loom(corep: CoreParam) extends Module { + + @public + val io = IO(new Bundle { + val interrupts = Input(new CoreInterrupts) + val commands = new Commands(corep.memory.dcache.tracking, corep.integer.xLen, corep.privileged.permission) + val drained = Input(Bool()) + val memory = Flipped(new MemoryPort(corep.memory.dcache.tracking, corep.memory.memoryAccess.walker)) + val coherent = Vec(2, new chi.RequesterPort(corep.memory.home.memory.chi)) + val uncached = new axi4.Port(corep.memory.uncached.axi) + }) + + val execution: Instance[Execution] = Instantiate(new Execution(corep.execution(directAccelerator = true), false)) + val memory: Instance[MemorySystem] = Instantiate(new MemorySystem(corep.memory, externalHome = true)) + execution.io.interrupts := io.interrupts + io.commands <> execution.io.controller.get + execution.io.workDrained := io.drained + memory.io.fetch <> execution.io.fetch + memory.io.memory <> execution.io.data + memory.io.timeRequest <> execution.io.timeRequest + execution.io.timeResponse <> memory.io.timeResponse + memory.io.commandAllowed := io.drained + memory.io.services <> io.memory + io.uncached <> memory.io.uncached + io.coherent <> memory.io.coherent +} diff --git a/examples/prefab/cores/reduce/emu/Cargo.lock b/examples/prefab/cores/reduce/emu/Cargo.lock new file mode 100644 index 000000000..a9fa2182d --- /dev/null +++ b/examples/prefab/cores/reduce/emu/Cargo.lock @@ -0,0 +1,822 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "bebop-bank-hash" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "snafu", +] + +[[package]] +name = "bebop-bemu" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", + "bebop-bemu-profile", + "bebop-dasm", + "bebop-dtb", + "bebop-elf", + "bebop-keyboard", + "bebop-memory", + "bebop-microphone", + "bebop-rtc", + "bebop-speaker", + "bebop-syscall", + "bebop-uart", + "bebop-vga", + "bebop-vm", + "clap", + "linkme", + "prost", + "prost-build", + "prost-types", + "rustc_apfloat", + "rvsim", + "snafu", + "toml", +] + +[[package]] +name = "bebop-bemu-profile" +version = "0.1.0" + +[[package]] +name = "bebop-core-reduce" +version = "0.1.0" +dependencies = [ + "bebop-bemu", +] + +[[package]] +name = "bebop-dasm" +version = "0.1.0" + +[[package]] +name = "bebop-dtb" +version = "0.1.0" + +[[package]] +name = "bebop-elf" +version = "0.1.0" +dependencies = [ + "bebop-memory", +] + +[[package]] +name = "bebop-keyboard" +version = "0.1.0" + +[[package]] +name = "bebop-memory" +version = "0.1.0" +dependencies = [ + "log", + "serde", + "serde_json", + "sha2", +] + +[[package]] +name = "bebop-microphone" +version = "0.1.0" + +[[package]] +name = "bebop-rtc" +version = "0.1.0" + +[[package]] +name = "bebop-speaker" +version = "0.1.0" + +[[package]] +name = "bebop-syscall" +version = "0.1.0" +dependencies = [ + "bebop-memory", + "once_cell", +] + +[[package]] +name = "bebop-uart" +version = "0.1.0" + +[[package]] +name = "bebop-vga" +version = "0.1.0" + +[[package]] +name = "bebop-vm" +version = "0.1.0" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck", + "itertools", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.119", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustc_apfloat" +version = "0.2.3+llvm-462a31f5a5ab" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "486c2179b4796f65bfe2ee33679acf0927ac83ecf583ad6c91c3b4570911b9ad" +dependencies = [ + "bitflags", + "smallvec", +] + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rvsim" +version = "0.1.0" +source = "git+https://github.com/DangoSys/riscv-cpu-sim?rev=32abeddb3f0d4c810e07d8d6745f5150e96c8c54#32abeddb3f0d4c810e07d8d6745f5150e96c8c54" +dependencies = [ + "rustc_apfloat", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "smallvec" +version = "1.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8" + +[[package]] +name = "snafu" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/examples/prefab/cores/reduce/emu/Cargo.toml b/examples/prefab/cores/reduce/emu/Cargo.toml new file mode 100644 index 000000000..15499692c --- /dev/null +++ b/examples/prefab/cores/reduce/emu/Cargo.toml @@ -0,0 +1,9 @@ +[workspace] + +[package] +name = "bebop-core-reduce" +version = "0.1.0" +edition = "2021" + +[dependencies] +bebop-bemu = { path = "../../../../../bebop/src/nodes/bemu" } diff --git a/examples/prefab/cores/reduce/emu/src/lib.rs b/examples/prefab/cores/reduce/emu/src/lib.rs new file mode 100644 index 000000000..b9e4b5a8c --- /dev/null +++ b/examples/prefab/cores/reduce/emu/src/lib.rs @@ -0,0 +1,39 @@ +use bebop_bemu::{ + ball::linkme, Accelerator, BemuProfileReport, CoreFactory, CoreModel, CoreParams, Cpu, DeviceModel, ModelOptions, + Whatever, CORES, +}; +use std::time::Duration; + +/// The reduce core: one RV64 hart with a Ball accelerator on its RoCC port. +pub struct ReduceCore { + cpu: Cpu, + acc: Accelerator, +} + +impl CoreModel for ReduceCore { + fn cpu(&self) -> &Cpu { + &self.cpu + } + fn cpu_mut(&mut self) -> &mut Cpu { + &mut self.cpu + } + fn step(&mut self, count: u64) -> Result<(), Whatever> { + self.cpu.step(count, &mut self.acc) + } + fn take_barrier(&mut self) -> bool { + self.acc.take_barrier() + } + fn profile_report(&self, total: Duration) -> Option { + self.acc.profile_report(total, &self.cpu) + } +} + +fn instantiate(device: &DeviceModel, params: &CoreParams, options: &ModelOptions) -> Result, String> { + let cpu = Cpu::new(device, params, options).map_err(|error| error.to_string())?; + let acc = Accelerator::attached(&cpu, options)?; + Ok(Box::new(ReduceCore { cpu, acc })) +} + +#[linkme::distributed_slice(CORES)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static REDUCE: CoreFactory = CoreFactory { component: "reduce", instantiate }; diff --git a/examples/prefab/cores/reduce/stack/CMakeLists.txt b/examples/prefab/cores/reduce/stack/CMakeLists.txt new file mode 100644 index 000000000..7af3b899b --- /dev/null +++ b/examples/prefab/cores/reduce/stack/CMakeLists.txt @@ -0,0 +1,41 @@ +# ===----------------------------------------------------------------------=== +# The reduce core: a CPU core with a reduction accelerator; jobs run on the core. +# +# bbdev stack --build "--design " +# +# builds it for each design that uses it, configured with DESIGN_CMAKE +# (generated from design.pb) and CORE. Products: +# execute.o runs jobs on the core itself; links into the device runtime +# lib-compiler.so buddy-opt plugin: Buckyball dialect + the Balls of +# this core's accelerator +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(reduce-core LANGUAGES C CXX) +include("${DESIGN_CMAKE}") + +set(RUNTIME "${DESIGN_REPO}/stack/runtime") +set(TARGET ${DESIGN_DEVICE_CORES_${DESIGN_CORE_HART_${CORE}}_PROFILE}) +find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) + +# ===----------------------------------------------------------------------=== +# Execution: jobs run on the core itself. +# ===----------------------------------------------------------------------=== +add_custom_command( + OUTPUT execute.o + COMMAND "${LINUX_CC}" -O2 -fPIC -ffreestanding -fno-builtin -msmall-data-limit=0 + "-DCORE_SIGNATURE=${DESIGN_DEVICE_CORES_${DESIGN_CORE_HART_${CORE}}_SIGNATURE}ULL" "-I${RUNTIME}/scheduler" + -c "${RUNTIME}/execute/direct.c" -o execute.o + DEPENDS "${RUNTIME}/execute/direct.c" "${RUNTIME}/scheduler/runtime.h" + VERBATIM) +add_custom_target(execute ALL DEPENDS execute.o) + +# ===----------------------------------------------------------------------=== +# Compiler: the Buckyball dialect and passes with the Balls of the accelerator. +# ===----------------------------------------------------------------------=== +set(BUCKYBALL_BALLS ${DESIGN_TARGET_BALL_STACKS_${TARGET}}) + +add_subdirectory("${DESIGN_REPO}/stack/compiler/midend" compiler) +add_library(${TARGET}-compiler MODULE) +target_link_libraries(${TARGET}-compiler PRIVATE buckyball-compiler) +set_target_properties(${TARGET}-compiler PROPERTIES + LINKER_LANGUAGE CXX LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") diff --git a/examples/prefab/cores/rocket/LICENSE.Berkeley b/examples/prefab/cores/rocket/LICENSE.Berkeley new file mode 100644 index 000000000..7cff15e40 --- /dev/null +++ b/examples/prefab/cores/rocket/LICENSE.Berkeley @@ -0,0 +1,24 @@ +Copyright (c) 2012-2014, The Regents of the University of California +(Regents). All Rights Reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. Neither the name of the Regents nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, +SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING +OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS +BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED +HEREUNDER IS PROVIDED "AS IS". REGENTS HAS NO OBLIGATION TO PROVIDE +MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. diff --git a/examples/prefab/cores/rocket/LICENSE.SiFive b/examples/prefab/cores/rocket/LICENSE.SiFive new file mode 100644 index 000000000..7e7093375 --- /dev/null +++ b/examples/prefab/cores/rocket/LICENSE.SiFive @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2016-2017 SiFive, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/examples/prefab/cores/rocket/arch/Execution.scala b/examples/prefab/cores/rocket/arch/Execution.scala new file mode 100644 index 000000000..39272c5a3 --- /dev/null +++ b/examples/prefab/cores/rocket/arch/Execution.scala @@ -0,0 +1,219 @@ +package examples.core.rocket + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import examples.core.rocket.configs.Params +import examples.core.rocket.pipeline.{CpuParams, HasCpuParameters, RocketBB} +import freechips.rocketchip.rocket.{CSRs, PMP, PMPChecker, PRV} +import freechips.rocketchip.tile.{CustomCSR, FPU, FPUCoreIO} +import framework.core.clink.CoreInterrupts +import framework.core.execution.fetch.Fetch +import framework.core.memory.access.{UncachedRequest, UncachedResponse, VirtualPort} +import framework.acc.frontend.RoCCIO + +/** ACAL Rocket pipeline attached to the framework's memory and local accelerator protocols. */ +@instantiable +class Execution(config: Params, commandsEnabled: Boolean) extends Module with HasCpuParameters { + implicit lazy val cpuParams: CpuParams = config.cpu + val access = config.memory.memoryAccess + + @public + val io = IO(new Bundle { + val interrupts = Input(new CoreInterrupts) + val fetch = Flipped(new VirtualPort(access)) + val data = Flipped(new VirtualPort(access)) + val timeRequest = Decoupled(new UncachedRequest(access.memory)) + val timeResponse = Flipped(Decoupled(new UncachedResponse(access.memory))) + val rocc = Flipped(new RoCCIO(xLen, config.permission)) + }) + + val pipeline: Instance[RocketBB] = Instantiate(new RocketBB( + Seq(CustomCSR(CSRs.time, (BigInt(1) << 64) - 1, Some(BigInt(0)))), + commandsEnabled, + false, + false, + false, + false + )) + + val fetch: Instance[Fetch] = Instantiate(new Fetch(config.fetch)) + val lsu: Instance[Lsu] = Instantiate(new Lsu(access)) + val cpu = pipeline.io + cpu.hartid := config.hartId.U + cpu.reset_vector := config.resetVector.U + cpu.interrupts := 0.U.asTypeOf(cpu.interrupts) + cpu.interrupts.mtip := io.interrupts.timer + cpu.interrupts.msip := io.interrupts.software + cpu.interrupts.meip := io.interrupts.external + cpu.interrupts.seip.foreach(_ := io.interrupts.supervisorExternal) + cpu.traceStall := false.B + cpu.ptw.perf := 0.U.asTypeOf(cpu.ptw.perf) + cpu.ptw.clock_enabled := true.B + cpu.ptw.customCSRs.csrs.foreach { csr => + csr.stall := false.B + csr.set := false.B + csr.sdata := 0.U + } + val fpu = coreParams.fpu.map(cfg => Module(new FPU(cfg))) + fpu match { + case Some(unit) => + cpu.fpu :<>= unit.io.waiveAs[FPUCoreIO](_.cp_req, _.cp_resp) + unit.io.cp_req.valid := false.B + unit.io.cp_req.bits := 0.U.asTypeOf(unit.io.cp_req.bits) + unit.io.cp_resp.ready := true.B + case None => + cpu.fpu.store_data := 0.U + cpu.fpu.toint_data := 0.U + cpu.fpu.fcsr_flags.valid := false.B + cpu.fpu.fcsr_flags.bits := 0.U + cpu.fpu.fcsr_rdy := true.B + cpu.fpu.nack_mem := false.B + cpu.fpu.illegal_rm := false.B + cpu.fpu.dec := 0.U.asTypeOf(cpu.fpu.dec) + cpu.fpu.sboard_set := false.B + cpu.fpu.sboard_clr := false.B + cpu.fpu.sboard_clra := 0.U + } + + val satp = if (usingVM) Cat(cpu.ptw.ptbr.mode, cpu.ptw.ptbr.asid.pad(16), cpu.ptw.ptbr.ppn.pad(44)) else 0.U(64.W) + lsu.io.cpu <> cpu.dmem + lsu.io.pc := cpu.dmemPc + lsu.io.pmp := cpu.ptw.pmp + lsu.io.context.privilege := cpu.ptw.status.dprv + lsu.io.context.satp := satp + lsu.io.context.sum := cpu.ptw.status.sum + lsu.io.context.mxr := cpu.ptw.status.mxr + // A younger memory operation cannot pass an older unaccepted accelerator command. + val blockFresh = cpu.roccOlderPending && lsu.io.idle + lsu.io.cpu.req.valid := cpu.dmem.req.valid && !blockFresh + cpu.dmem.req.ready := lsu.io.cpu.req.ready && !blockFresh + io.data.request <> lsu.io.request + lsu.io.response <> io.data.response + lsu.io.maintenance.ready := true.B + val maintenancePending = RegInit(false.B) + when(lsu.io.maintenance.fire)(maintenancePending := true.B) + when(lsu.io.maintained.fire)(maintenancePending := false.B) + lsu.io.maintained.valid := maintenancePending + lsu.io.maintained.bits := true.B + io.data.flushTranslation := lsu.io.maintenance.fire + io.fetch.flushTranslation := lsu.io.maintenance.fire + + fetch.io.resetVector := config.resetVector.U + fetch.io.context.privilege := cpu.ptw.status.prv + fetch.io.context.satp := Mux(cpu.ptw.status.prv === PRV.M.U, 0.U, satp) + fetch.io.context.sum := cpu.ptw.status.sum + fetch.io.context.mxr := cpu.ptw.status.mxr + val redirect = if (usingVM) cpu.imem.req.bits.pc.asSInt.pad(64).asUInt else cpu.imem.req.bits.pc.pad(64) + fetch.io.redirect.valid := RegNext(cpu.imem.req.valid, false.B) + fetch.io.redirect.bits := RegEnable(redirect, cpu.imem.req.valid) + fetch.io.flush := RegNext(cpu.imem.flush_icache, false.B) + fetch.io.invalidate := RegNext(cpu.imem.sfence.valid, false.B) + cpu.imem.resp.valid := fetch.io.packet.valid + fetch.io.packet.ready := cpu.imem.resp.ready + cpu.imem.resp.bits := 0.U.asTypeOf(cpu.imem.resp.bits) + cpu.imem.resp.bits.pc := fetch.io.packet.bits.pc + cpu.imem.resp.bits.data := fetch.io.packet.bits.data + cpu.imem.resp.bits.mask := fetch.io.packet.bits.mask + cpu.imem.resp.bits.xcpt.pf.inst := fetch.io.packet.bits.pageFault + cpu.imem.resp.bits.xcpt.ae.inst := fetch.io.packet.bits.accessFault + cpu.imem.npc := fetch.io.npc + cpu.imem.clock_enabled := true.B + cpu.imem.perf := 0.U.asTypeOf(cpu.imem.perf) + cpu.imem.gpa.valid := false.B + cpu.imem.gpa.bits := 0.U + cpu.imem.gpa_is_pte := false.B + io.fetch.request.valid := fetch.io.request.valid + fetch.io.request.ready := io.fetch.request.ready + io.fetch.request.bits := 0.U.asTypeOf(io.fetch.request.bits) + io.fetch.request.bits.vaddr := fetch.io.request.bits.addr + io.fetch.request.bits.size := 3.U + io.fetch.request.bits.execute := true.B + io.fetch.request.bits.privilege := fetch.io.request.bits.context.privilege + io.fetch.request.bits.sum := fetch.io.request.bits.context.sum + io.fetch.request.bits.mxr := fetch.io.request.bits.context.mxr + io.fetch.request.bits.satpMode := fetch.io.request.bits.context.satp(63, 60) + io.fetch.request.bits.rootPpn := fetch.io.request.bits.context.satp(43, 0) + fetch.io.response.valid := io.fetch.response.valid + io.fetch.response.ready := fetch.io.response.ready + fetch.io.response.bits.data := io.fetch.response.bits.data + fetch.io.response.bits.line := 0.U + fetch.io.response.bits.lineExecutable := false.B + fetch.io.response.bits.pageFault := io.fetch.response.bits.pageFault + fetch.io.response.bits.accessFault := io.fetch.response.bits.accessFault + // Coherent I/D caches observe stores through Home; fence.i also discards fetch packets. + fetch.io.maintenance.ready := lsu.io.idle && !io.rocc.busy + val fetchMaintaining = RegInit(false.B) + when(fetch.io.maintenance.fire)(fetchMaintaining := true.B) + when(fetch.io.maintained.fire)(fetchMaintaining := false.B) + fetch.io.maintained.valid := fetchMaintaining + fetch.io.maintained.bits := true.B + + val fetchPmp = Reg(Vec(nPMPs, new PMP)) + when(io.fetch.request.fire)(fetchPmp := cpu.ptw.pmp) + for ((port, pmps) <- Seq(io.fetch -> fetchPmp, io.data -> lsu.io.capturedPmp)) { + val checker = Module(new PMPChecker(3)) + checker.io.prv := port.authorizationRequest.bits.privilege + checker.io.pmp := pmps + checker.io.addr := port.authorizationRequest.bits.paddr + checker.io.size := port.authorizationRequest.bits.size + val pending = RegInit(false.B) + val allowed = Reg(Bool()) + port.authorizationRequest.ready := !pending + port.authorizationResponse.valid := pending + port.authorizationResponse.bits := allowed + when(port.authorizationRequest.fire) { + val req = port.authorizationRequest.bits + allowed := !req.paddr(63, paddrBits).orR && + (!req.read || checker.io.r) && (!req.write || checker.io.w) && (!req.execute || checker.io.x) + pending := true.B + } + when(port.authorizationResponse.fire)(pending := false.B) + } + + // Freeze a custom command only on its architectural acceptance in Rocket's WB stage. + io.rocc.cmd.valid := cpu.rocc.cmd.valid && cpu.roccCommitEligible && lsu.io.idle + cpu.rocc.cmd.ready := io.rocc.cmd.ready && cpu.roccCommitEligible && lsu.io.idle + io.rocc.cmd.bits.instruction.raw_inst := cpu.rocc.cmd.bits.raw_inst + io.rocc.cmd.bits.instruction.pc := cpu.rocc.cmd.bits.pc + io.rocc.cmd.bits.instruction.rs1Data := cpu.rocc.cmd.bits.rs1Data + io.rocc.cmd.bits.instruction.rs2Data := cpu.rocc.cmd.bits.rs2Data + io.rocc.cmd.bits.satp := satp + io.rocc.cmd.bits.effectivePrivilege := cpu.ptw.status.dprv + io.rocc.cmd.bits.sum := cpu.ptw.status.sum + io.rocc.cmd.bits.mxr := cpu.ptw.status.mxr + for ((target, source) <- io.rocc.cmd.bits.pmp.zip(cpu.ptw.pmp)) { + target.cfg := source.cfg.asTypeOf(target.cfg) + target.addr := source.addr + target.mask := source.mask + } + cpu.rocc.busy := io.rocc.busy + cpu.rocc.interrupt := io.rocc.interrupt + cpu.rocc.resp.valid := io.rocc.resp.valid + cpu.rocc.resp.bits.rd := io.rocc.resp.bits.rd + cpu.rocc.resp.bits.data := io.rocc.resp.bits.data + io.rocc.resp.ready := cpu.rocc.resp.ready + + // Read the shared device timer on demand; counting never occupies the AXI bus. + val timeIdle :: timeIssue :: timeWait :: timeReady :: Nil = Enum(4) + val timeState = RegInit(timeIdle) + val time = cpu.rocc.csrs(0) + when(time.ren && timeState === timeIdle)(timeState := timeIssue) + io.timeRequest.valid := timeState === timeIssue + io.timeRequest.bits := 0.U.asTypeOf(io.timeRequest.bits) + io.timeRequest.bits.addr := config.mtimeAddress.U + io.timeRequest.bits.size := 3.U + when(io.timeRequest.fire)(timeState := timeWait) + io.timeResponse.ready := timeState === timeWait + time.stall := timeState =/= timeReady + time.set := io.timeResponse.fire + time.sdata := io.timeResponse.bits.data + when(io.timeResponse.fire) { + assert(!io.timeResponse.bits.error, "CLINT mtime read failed") + timeState := timeReady + } + val retired = cpu.trace.insns(0) + when(retired.valid && !retired.exception && retired.insn(6, 0) === "h73".U && + retired.insn(31, 20) === CSRs.time.U && timeState === timeReady)(timeState := timeIdle) + cpu.interruptReady := lsu.io.idle && timeState === timeIdle && !io.rocc.busy +} diff --git a/examples/prefab/cores/rocket/arch/Lsu.scala b/examples/prefab/cores/rocket/arch/Lsu.scala new file mode 100644 index 000000000..a411d6be5 --- /dev/null +++ b/examples/prefab/cores/rocket/arch/Lsu.scala @@ -0,0 +1,194 @@ +package examples.core.rocket + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import examples.core.rocket.pipeline.{CpuParams, HasCpuParameters} +import freechips.rocketchip.rocket._ +import freechips.rocketchip.rocket.constants.MemoryOpConstants +import framework.ip.bus.chi.rnf.CacheAtomic +import framework.core.memory.access.{VirtualRequest, VirtualResponse} +import framework.core.memory.access.configs.{Params => AccessParam} +import framework.core.execution.fetch.Context + +/** Replay-based Rocket LSU. Effects start only after both pipeline cancellation windows. */ +@instantiable +class Lsu(cp: AccessParam)(implicit val cpuParams: CpuParams) + extends Module + with HasCpuParameters + with MemoryOpConstants { + + @public + val io = IO(new Bundle { + val cpu = Flipped(new HellaCacheIO) + val pc = Input(UInt(vaddrBitsExtended.W)) + val pmp = Input(Vec(nPMPs, new PMP)) + val context = Input(new Context) + val maintenance = Decoupled(new SFenceReq) + val maintained = Flipped(Decoupled(Bool())) + val capturedPmp = Output(Vec(nPMPs, new PMP)) + val request = Decoupled(new VirtualRequest(cp)) + val response = Flipped(Decoupled(new VirtualResponse(cp))) + val idle = Output(Bool()) + val cancelled = Output(Bool()) + }) + + require(!usingHypervisor && cp.memory.tagBits <= coreParams.dcacheReqTagBits) + val idle :: s1 :: s2 :: issue :: receive :: completed :: replay1 :: replay2 :: Nil = Enum(8) + val state = RegInit(idle) + val command = Reg(new HellaCacheReq) + val instructionPc = Reg(UInt(vaddrBitsExtended.W)) + val data = Reg(UInt(64.W)) + val pmps = Reg(Vec(nPMPs, new PMP)) + val context = Reg(new Context) + val maintaining = RegInit(false.B) + io.capturedPmp := pmps + val result = Reg(new VirtualResponse(cp)) + val matches = io.cpu.req.bits.tag === command.tag && io.cpu.req.bits.addr === command.addr && + io.cpu.req.bits.cmd === command.cmd && io.cpu.req.bits.size === command.size && io.pc === instructionPc + io.idle := state === idle + io.cancelled := (state === s1 || state === replay1) && io.cpu.s1_kill || + (state === s2 || state === replay2) && io.cpu.s2_kill + io.cpu.req.ready := state === idle || (state === completed && (!io.cpu.req.valid || matches)) + when(io.cpu.req.fire) { + when(state === idle) { + assert(!io.cpu.req.bits.dv && io.cpu.req.bits.dprv =/= 2.U, "LSU requires nonvirtual U, S or M privilege") + if (io.cpu.req.bits.tag.getWidth > cp.memory.tagBits) { + assert( + !io.cpu.req.bits.tag(io.cpu.req.bits.tag.getWidth - 1, cp.memory.tagBits).orR, + "LSU arbiter tag bits must be zero" + ) + } + command := io.cpu.req.bits + instructionPc := io.pc + pmps := io.pmp + context := io.context + state := s1 + }.otherwise { + assert(matches, "LSU replay changed instruction identity") + state := replay1 + } + } + when(state === s1) { + when(io.cpu.s1_kill)(state := idle) + .otherwise { data := io.cpu.s1_data.data; state := s2 } + } + + val writes = isWrite(command.cmd) + val reads = isRead(command.cmd) + val address = if (usingVM) command.addr.asSInt.pad(64).asUInt else command.addr.pad(64) + val sfence = command.cmd === M_SFENCE + val misaligned = (address & ((1.U(64.W) << command.size) - 1.U)).orR + + when(state === s2) { + when(io.cpu.s2_kill)(state := idle).otherwise { + when(misaligned && !sfence) { + result := 0.U.asTypeOf(result) + result.tag := command.tag(cp.memory.tagBits - 1, 0) + result.misaligned := misaligned + result.accessFault := false.B + state := completed + }.otherwise(state := issue) + } + } + + val atom = MuxLookup(command.cmd, CacheAtomic.None.U)(Seq( + M_XA_SWAP -> CacheAtomic.Swap.U, + M_XA_ADD -> CacheAtomic.Add.U, + M_XA_XOR -> CacheAtomic.Xor.U, + M_XA_AND -> CacheAtomic.And.U, + M_XA_OR -> CacheAtomic.Or.U, + M_XA_MIN -> CacheAtomic.Min.U, + M_XA_MAX -> CacheAtomic.Max.U, + M_XA_MINU -> CacheAtomic.MinU.U, + M_XA_MAXU -> CacheAtomic.MaxU.U, + M_XLR -> CacheAtomic.LR.U, + M_XSC -> CacheAtomic.SC.U + )) + + io.maintenance.valid := state === issue && sfence + io.maintenance.bits := 0.U.asTypeOf(io.maintenance.bits) + io.maintenance.bits.addr := command.addr + io.maintenance.bits.asid := data + io.maintenance.bits.rs1 := command.size(0) + io.maintenance.bits.rs2 := command.size(1) + io.maintained.ready := state === receive && maintaining + when(io.maintenance.fire) { maintaining := true.B; state := receive } + when(io.maintained.fire) { + assert(io.maintained.bits, "LSU maintenance must acknowledge completion") + result := 0.U.asTypeOf(result) + result.tag := command.tag(cp.memory.tagBits - 1, 0) + maintaining := false.B + state := completed + } + io.request.valid := state === issue && !sfence + io.request.bits := 0.U.asTypeOf(io.request.bits) + io.request.bits.vaddr := address + io.request.bits.tag := command.tag(cp.memory.tagBits - 1, 0) + io.request.bits.size := command.size + io.request.bits.write := command.cmd === M_XWR + io.request.bits.execute := false.B + io.request.bits.signed := command.signed + io.request.bits.data := data + io.request.bits.atomic := atom + io.request.bits.privilege := command.dprv + io.request.bits.sum := context.sum + io.request.bits.mxr := context.mxr + io.request.bits.satpMode := context.satp(63, 60) + io.request.bits.rootPpn := context.satp(43, 0) + when(io.request.fire) { + assert( + command.cmd === M_XRD || command.cmd === M_XWR || command.cmd === M_XLR || + command.cmd === M_XSC || isAMO(command.cmd), + "LSU unsupported memory operation" + ) + state := receive + } + io.response.ready := state === receive && !maintaining + when(io.response.fire) { + assert(io.response.bits.tag === command.tag(cp.memory.tagBits - 1, 0), "LSU virtual response tag mismatch") + result := io.response.bits + state := completed + } + when(state === replay1)(state := Mux(io.cpu.s1_kill, completed, replay2)) + when(state === replay2)(state := Mux(io.cpu.s2_kill, completed, idle)) + + val returns = state === replay2 && !io.cpu.s2_kill + val exception = result.misaligned || result.pageFault || result.accessFault + io.cpu.s2_nack := state === s2 && !io.cpu.s2_kill + io.cpu.s2_nack_cause_raw := false.B + io.cpu.s2_uncached := false.B + io.cpu.s2_paddr := command.addr(paddrBits - 1, 0) + io.cpu.s2_gpa := 0.U + io.cpu.s2_gpa_is_pte := false.B + io.cpu.s2_xcpt := 0.U.asTypeOf(io.cpu.s2_xcpt) + io.cpu.s2_xcpt.ma.ld := returns && !writes && result.misaligned + io.cpu.s2_xcpt.ma.st := returns && writes && result.misaligned + io.cpu.s2_xcpt.pf.ld := returns && !writes && result.pageFault + io.cpu.s2_xcpt.pf.st := returns && writes && result.pageFault + io.cpu.s2_xcpt.ae.ld := returns && !writes && result.accessFault + io.cpu.s2_xcpt.ae.st := returns && writes && result.accessFault + io.cpu.resp.valid := returns + io.cpu.resp.bits := 0.U.asTypeOf(io.cpu.resp.bits) + io.cpu.resp.bits.tag := command.tag + io.cpu.resp.bits.cmd := command.cmd + io.cpu.resp.bits.size := command.size + io.cpu.resp.bits.addr := command.addr + io.cpu.resp.bits.signed := command.signed + io.cpu.resp.bits.dprv := command.dprv + io.cpu.resp.bits.has_data := reads && !command.no_resp && !exception + io.cpu.resp.bits.data := result.data + io.cpu.resp.bits.data_word_bypass := result.data + io.cpu.resp.bits.data_raw := result.data + io.cpu.resp.bits.store_data := data + io.cpu.replay_next := false.B + // A buffered completion awaits instruction replay, not a memory transaction; release ordered decoding. + val memoryPending = state === s1 || state === s2 || state === issue || state === receive + io.cpu.ordered := !memoryPending + io.cpu.store_pending := memoryPending && writes + io.cpu.clock_enabled := true.B + io.cpu.perf := 0.U.asTypeOf(io.cpu.perf) + io.cpu.perf.grant := returns + io.cpu.perf.blocked := !io.cpu.req.ready + io.cpu.uncached_resp.foreach { port => port.valid := false.B; port.bits := 0.U.asTypeOf(port.bits) } +} diff --git a/examples/prefab/cores/rocket/arch/Rocket.scala b/examples/prefab/cores/rocket/arch/Rocket.scala new file mode 100644 index 000000000..b9c356189 --- /dev/null +++ b/examples/prefab/cores/rocket/arch/Rocket.scala @@ -0,0 +1,163 @@ +package examples.core.rocket + +import framework.acc.balldomain.bbus.BallFactory +import sims.configs.SimParam +import framework.core.configs.CoreParam +import framework.device.cbus.CoreFactory +import buckyball.config.{CorePlacement, Design} + +import framework.acc.balldomain.ball.blink.BlinkIO + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import examples.core.rocket.configs.Params +import framework.core.clink.{CLinkIO, HasCLink} +import framework.core.memory.MemorySystem +import framework.acc.frontend.Frontend +import framework.acc.memdomain.MemDomain +import framework.acc.gpdomain.GpDomain +import framework.acc.balldomain.BallDomain +import framework.acc.balldomain.ball.blink.HasBlink +import framework.acc.debug.Debug + +/** Rocket core with local Ball endpoints. No task threads or TBus are instantiated. */ +@instantiable +class Rocket( + config: Params, + simp: SimParam, + factories: Map[String, BallFactory]) + extends Module + with HasCLink { + val diffTest = simp.diffTest + require(config.acc.forall(a => !a.memDomain.sharedEnable)) + + @public + val clink = IO(new CLinkIO(config.clink)) + + val execution: Instance[Execution] = Instantiate(new Execution(config, config.acc.nonEmpty)) + val memory: Instance[MemorySystem] = Instantiate(new MemorySystem(config.memory, !config.clink.backing)) + execution.io.interrupts := clink.interrupts(0) + memory.io.fetch <> execution.io.fetch + memory.io.memory <> execution.io.data + memory.io.timeRequest <> execution.io.timeRequest + execution.io.timeResponse <> memory.io.timeResponse + clink.memory.uncached <> memory.io.uncached + clink.memory.backing.zip(memory.io.backing).foreach { case (port, source) => port <> source } + clink.memory.coherent <> memory.io.coherent + memory.io.commandAllowed := true.B + + config.acc match { + case Some(acc) => + val frontend: Instance[Frontend] = + Instantiate(new Frontend(acc.frontend, acc.memDomain, acc.gpDomain, acc.ballDomain, config.permission, simp)) + val memDomain: Instance[MemDomain] = Instantiate(new MemDomain( + acc.frontend, + acc.memDomain, + acc.gpDomain, + acc.transport, + config.permission, + config.memory.dcache.tracking, + config.memory.memoryAccess.walker, + simp + )) + val domain: Instance[BallDomain] = + Instantiate(new BallDomain( + acc.frontend, + acc.memDomain, + acc.gpDomain, + acc.ballDomain, + config.permission, + factories + )) + if (acc.gpDomain.enable) { + val gp: Instance[GpDomain] = Instantiate(new GpDomain(acc.frontend, acc.memDomain, acc.gpDomain)) + gp.io.command <> frontend.io.gpRequest.get + frontend.io.gpCompletion.get <> gp.io.complete + gp.io.image <> memDomain.io.image.get + memDomain.io.gpBanks.get <> gp.io.banks + memDomain.io.gpLayoutRequest.get <> gp.io.layoutRequest + gp.io.layoutResponse <> memDomain.io.gpLayoutResponse.get + domain.io.kernel.get.request <> gp.io.ball.request + gp.io.ball.response <> domain.io.kernel.get.response + } + frontend.io.rocc <> execution.io.rocc + memory.io.commandAllowed := !frontend.io.rocc.busy + domain.io.issue <> frontend.io.ballRequest + frontend.io.ballCompletion <> domain.io.complete + memDomain.io.ball <> domain.io.memory + memDomain.io.issue <> frontend.io.memoryRequest + frontend.io.memoryCompletion <> memDomain.io.complete + memDomain.io.mapping <> frontend.io.mappingUpdate + frontend.io.barrier.release := frontend.io.barrier.arrive + memDomain.io.banks <> domain.io.banks + memDomain.io.ballLayoutRequest <> domain.io.layoutRequest + domain.io.layoutResponse <> memDomain.io.ballLayoutResponse + clink.memory.ddr <> memDomain.io.ddr + memory.io.services <> memDomain.io.memory + val debug: Instance[Debug] = + Instantiate(new Debug(acc.frontend, acc.memDomain, acc.gpDomain, acc.ballDomain, simp, 0, 0)) + debug.io.ant.valid := false.B + debug.io.ant.bits := 0.U.asTypeOf(debug.io.ant.bits) + debug.io.acc.frontend <> frontend.io.debug + debug.io.acc.trace.memory := memDomain.io.traceMemory + debug.io.acc.trace.ballStarted := domain.io.traceStarted + debug.io.acc.trace.ballCompleted := domain.io.traceCompleted + debug.io.acc.banks.zip(memDomain.io.bank_hashes).foreach { case (sink, source) => sink := source } + for (port <- memDomain.io.shared.bank.requests) { + port.read.req.ready := false.B + port.read.resp.valid := false.B + port.read.resp.bits := 0.U.asTypeOf(port.read.resp.bits) + port.write.req.ready := false.B + port.write.resp.valid := false.B + port.write.resp.bits := 0.U.asTypeOf(port.write.resp.bits) + } + memDomain.io.shared.bank.config.ready := false.B + memDomain.io.shared.bank.mapping.ready := false.B + memDomain.io.shared.bank.local.request.valid := false.B + memDomain.io.shared.bank.local.request.bits := 0.U.asTypeOf(memDomain.io.shared.bank.local.request.bits) + memDomain.io.shared.bank.local.response.ready := false.B + memDomain.io.shared.move.command.ready := false.B + memDomain.io.shared.move.completion.valid := false.B + memDomain.io.shared.move.completion.bits := false.B + case None => + memory.io.services.cpuAllow := true.B + memory.io.services.maintenance.valid := false.B + memory.io.services.maintenance.bits := 0.U.asTypeOf(memory.io.services.maintenance.bits) + memory.io.services.pteRequest.valid := false.B + memory.io.services.pteRequest.bits := 0.U.asTypeOf(memory.io.services.pteRequest.bits) + memory.io.services.maintained.ready := true.B + memory.io.services.pteResponse.ready := true.B + execution.io.rocc.cmd.ready := false.B + execution.io.rocc.resp.valid := false.B + execution.io.rocc.resp.bits := 0.U.asTypeOf(execution.io.rocc.resp.bits) + execution.io.rocc.busy := false.B + execution.io.rocc.interrupt := false.B + clink.memory.ddr.aw.valid := false.B + clink.memory.ddr.aw.bits := 0.U.asTypeOf(clink.memory.ddr.aw.bits) + clink.memory.ddr.w.valid := false.B + clink.memory.ddr.w.bits := 0.U.asTypeOf(clink.memory.ddr.w.bits) + clink.memory.ddr.ar.valid := false.B + clink.memory.ddr.ar.bits := 0.U.asTypeOf(clink.memory.ddr.ar.bits) + clink.memory.ddr.b.ready := true.B + clink.memory.ddr.r.ready := true.B + } +} + +object RocketComponent extends CoreFactory { + val component = "rocket" + + def parameters(design: Design, source: CorePlacement): CoreParam = { + val native = Params.fromProto(source); require(native.acc.isEmpty); + CoreParam.fromProto(source, Seq.empty).copy(clink = native.clink) + } + + def instantiate( + source: CorePlacement, + corep: CoreParam, + simp: SimParam, + traceIds: Seq[Int] + ): CLinkIO = + Instantiate(new Rocket(Params.fromProto(source), simp, Map.empty)).clink + +} diff --git a/examples/prefab/cores/rocket/arch/build.sc b/examples/prefab/cores/rocket/arch/build.sc new file mode 100644 index 000000000..bb3c69dfc --- /dev/null +++ b/examples/prefab/cores/rocket/arch/build.sc @@ -0,0 +1,42 @@ +import mill._ +import mill.scalalib._ +import $file.^.^.^.^.^.arch.thirdparty.{build => thirdparty} +import $file.^.^.^.^.^.arch.src.main.scala.framework.{build => framework} + +val root: os.Path = millSourcePath / os.up +trait ScalaBuild extends SbtModule { + override def scalaVersion = "2.13.16" + override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0", ivy"com.lihaoyi::upickle:3.3.1") + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") +} +object cde extends ScalaBuild { + override def sources = T.sources(root / "thirdparty" / "rocket-chip" / "dependencies" / "cde" / "cde" / "src") +} +object pipeline extends ScalaBuild { + override def moduleDeps = Seq(cde, thirdparty.hardfloat, thirdparty.midas_target_utils, framework.library) + override def sources = T.sources { + val src = root / "thirdparty" / "rocket-chip" / "src" / "main" / "scala" + val files = Seq( + "rocket/ALU.scala", "rocket/AMOALU.scala", "rocket/BTB.scala", "rocket/Breakpoint.scala", + "rocket/CSR.scala", "rocket/Consts.scala", "rocket/CustomInstructions.scala", "rocket/DebugROB.scala", + "rocket/Decode.scala", "rocket/Events.scala", "rocket/IBuf.scala", "rocket/IDecode.scala", + "rocket/Instructions.scala", "rocket/Instructions32.scala", "rocket/Multiplier.scala", "rocket/PMP.scala", + "rocket/RVC.scala", "rocket/package.scala", "tile/FPU.scala", "tile/CustomCSRs.scala", + "util/Misc.scala", "util/ClockGate.scala", "util/CoreMonitor.scala", "util/Property.scala", + "util/Counters.scala", "util/ECC.scala", "util/Replacement.scala", "util/ShiftReg.scala", + "util/ShiftQueue.scala", "util/HellaQueue.scala", "util/LCG.scala", "util/PlusArg.scala", + "util/MuxLiteral.scala", "util/Arbiters.scala", "util/AsyncResetReg.scala", "util/Timer.scala", + "util/GeneratorUtils.scala", "unittest/UnitTest.scala" + ) + files.map(f => PathRef(src / os.RelPath(f))) ++ Seq(PathRef(root / "arch" / "pipeline")) + } + override def resources = T.sources(root / "thirdparty" / "rocket-chip" / "src" / "main" / "resources") +} +object library extends ScalaBuild { + override def moduleDeps = Seq(pipeline, framework.library) + override def sources = T.sources { + os.walk(root / "arch").filter(p => p.ext == "scala" && !p.toString.contains("/pipeline/")).map(PathRef(_)) ++ + Seq(PathRef(root / "configs")) + } +} diff --git a/examples/prefab/cores/rocket/arch/pipeline/Config.scala b/examples/prefab/cores/rocket/arch/pipeline/Config.scala new file mode 100644 index 000000000..91b5c6565 --- /dev/null +++ b/examples/prefab/cores/rocket/arch/pipeline/Config.scala @@ -0,0 +1,492 @@ +// See LICENSE.SiFive for license details. + +package freechips.rocketchip.tile { + + import chisel3._ + import chisel3.util.isPow2 + import org.chipsalliance.cde.config._ + import freechips.rocketchip.rocket._ + import freechips.rocketchip.util._ + + case object MaxHartIdBits extends Field[Int] + case object CorePAddrBits extends Field[Int] + +// These parameters can be varied per-core + trait CoreParams { + val bootFreqHz: BigInt + val useVM: Boolean + val useHypervisor: Boolean + val useUser: Boolean + val useSupervisor: Boolean + val useDebug: Boolean + val useAtomics: Boolean + val useAtomicsOnlyForIO: Boolean + val useCompressed: Boolean + val useVector: Boolean = false + val vectorUseDCache: Boolean = false + val useRVE: Boolean + val useConditionalZero: Boolean + val useZba: Boolean + val useZbb: Boolean + val useZbs: Boolean + val mulDiv: Option[MulDivParams] + val fpu: Option[FPUParams] + val fetchWidth: Int + val decodeWidth: Int + val retireWidth: Int + val instBits: Int + val nLocalInterrupts: Int + val useNMI: Boolean + val nPMPs: Int + val pmpGranularity: Int + val nBreakpoints: Int + val useBPWatch: Boolean + val mcontextWidth: Int + val scontextWidth: Int + val nPerfCounters: Int + val haveBasicCounters: Boolean + val haveFSDirty: Boolean + val misaWritable: Boolean + val haveCFlush: Boolean + val nL2TLBEntries: Int + val nL2TLBWays: Int + val nPTECacheEntries: Int + val mtvecInit: Option[BigInt] + val mtvecWritable: Boolean + val traceHasWdata: Boolean + val xLen: Int + val pgLevels: Int + def traceCustom: Option[Data] = None + def customIsaExt: Option[String] = None + def customCSRs(implicit p: Parameters): CustomCSRs = new CustomCSRs + + def hasSupervisorMode: Boolean = useSupervisor || useVM + def instBytes: Int = instBits / 8 + def fetchBytes: Int = fetchWidth * instBytes + def lrscCycles: Int + + def dcacheReqTagBits: Int = 6 + + def minFLen: Int = 32 + + def vLen: Int = 0 + def eLen: Int = 0 + def vfLen: Int = 0 + def vfh: Boolean = false + def vExts: Seq[String] = Nil + def hasV: Boolean = vLen >= 128 && eLen >= 64 && vfLen >= 64 + def vMemDataBits: Int = 0 + + def useBitmanip = useZba && useZbb && useZbs + } + + trait HasCoreParameters extends HasTileParameters { + + override lazy val paddrBits: Int = { + val bits = p(CorePAddrBits) + require(bits <= maxPAddrBits) + bits + } + + val coreParams: CoreParams = tileParams.core + + val minFLen = coreParams.fpu.map(_ => coreParams.minFLen).getOrElse(0) + val fLen = coreParams.fpu.map(_.fLen).getOrElse(0) + + val usingMulDiv = coreParams.mulDiv.nonEmpty + val usingFPU = coreParams.fpu.nonEmpty + val usingAtomics = coreParams.useAtomics + val usingAtomicsOnlyForIO = coreParams.useAtomicsOnlyForIO + val usingAtomicsInCache = usingAtomics && !usingAtomicsOnlyForIO + val usingCompressed = coreParams.useCompressed + val usingVector = coreParams.useVector + val usingNMI = coreParams.useNMI + val usingConditionalZero = coreParams.useConditionalZero + + val retireWidth = coreParams.retireWidth + val fetchWidth = coreParams.fetchWidth + val decodeWidth = coreParams.decodeWidth + + val fetchBytes = coreParams.fetchBytes + val coreInstBits = coreParams.instBits + val coreInstBytes = coreInstBits / 8 + val coreDataBits = xLen max fLen max vMemDataBits + val coreDataBytes = coreDataBits / 8 + def coreMaxAddrBits = paddrBits max vaddrBitsExtended + + val nBreakpoints = coreParams.nBreakpoints + val nPMPs = coreParams.nPMPs + val pmpGranularity = coreParams.pmpGranularity + val nPerfCounters = coreParams.nPerfCounters + val mtvecInit = coreParams.mtvecInit + val mtvecWritable = coreParams.mtvecWritable + val customIsaExt = coreParams.customIsaExt + val traceHasWdata = coreParams.traceHasWdata + + def vLen = coreParams.vLen + def eLen = coreParams.eLen + def vfLen = coreParams.vfLen + def vMemDataBits = if (usingVector) coreParams.vMemDataBits else 0 + def maxVLMax = vLen + + if (usingVector) { + require(isPow2(vLen), s"vLen ($vLen) must be a power of 2") + require(eLen >= 32 && vLen % eLen == 0, s"eLen must divide vLen ($vLen) and be no less than 32") + require(eLen == 32 || eLen == 64) + require(vfLen <= eLen) + require(!coreParams.vfh || (vfLen >= 32 && coreParams.minFLen <= 16)) + } + + if (coreParams.useVM) { + if (coreParams.xLen == 32) { + require(coreParams.pgLevels == 2) + } else { + require(coreParams.pgLevels >= 3) + } + } + + lazy val hartIdLen: Int = p(MaxHartIdBits) + + lazy val resetVectorLen: Int = { + val externalLen = paddrBits + require(externalLen <= xLen, s"External reset vector length ($externalLen) must be <= XLEN ($xLen)") + require( + externalLen <= vaddrBitsExtended, + s"External reset vector length ($externalLen) must be <= virtual address bit width ($vaddrBitsExtended)" + ) + externalLen + } + + // Print out log of committed instructions and their writeback values. + // Requires post-processing due to out-of-order writebacks. + val enableCommitLog = false + + } + + abstract class CoreModule(implicit val p: Parameters) extends Module with HasCoreParameters + + abstract class CoreBundle(implicit val p: Parameters) extends ParameterizedBundle()(p) with HasCoreParameters + +// This is a raw commit trace from the core, not the TraceCoreInterface + class TraceBundle(implicit val p: Parameters) extends Bundle with HasCoreParameters { + val insns = Vec(coreParams.retireWidth, new TracedInstruction) + val time = UInt(64.W) + val custom = coreParams.traceCustom + } + +} + +package examples.core.rocket.pipeline { + + import freechips.rocketchip.rocket.{BTBParams, DCacheParams, ICacheParams, RocketCoreParams} + + /** Explicit CPU metadata; the upstream parameter conversion stays in the Rocket adapter. */ + case class CpuParams( + core: RocketCoreParams, + dcache: Option[DCacheParams], + icache: Option[ICacheParams], + btb: Option[BTBParams], + physicalAddressBits: Int, + hartIdBits: Int, + beatBytes: Int = 8, + blockBytes: Int = 64, + asIdBits: Int = 0) + +} + +package examples.core.rocket.pipeline { + + import org.chipsalliance.cde.config.Parameters + import freechips.rocketchip.tile.{ + CoreBeatBytes, + CorePAddrBits, + CpuTileParams, + HasCoreParameters, + MaxHartIdBits, + TileKey + } + import freechips.rocketchip.rocket.ASIdBits + import freechips.rocketchip.subsystem.CacheBlockBytes + + object RocketParameters { + + def apply(cpu: CpuParams): Parameters = Parameters.empty.alterPartial { + case TileKey => CpuTileParams(core = cpu.core, dcache = cpu.dcache, icache = cpu.icache, btb = cpu.btb) + case CorePAddrBits => cpu.physicalAddressBits + case MaxHartIdBits => cpu.hartIdBits + case CoreBeatBytes => cpu.beatBytes + case CacheBlockBytes => cpu.blockBytes + case ASIdBits => cpu.asIdBits + } + + } + + trait HasCpuParameters extends HasCoreParameters { + implicit val cpuParams: CpuParams + implicit lazy val p: Parameters = RocketParameters(cpuParams) + } + +} + +// See LICENSE.Berkeley and LICENSE.SiFive for Rocket CSR semantics. +package freechips.rocketchip.rocket { + + import chisel3._ + import org.chipsalliance.cde.config.Parameters + import freechips.rocketchip.tile._ + import freechips.rocketchip.util._ + + trait HasRocketCoreParameters extends HasCoreParameters { + lazy val rocketParams = tileParams.core.asInstanceOf[RocketCoreParams] + val fastLoadWord = rocketParams.fastLoadWord + val fastLoadByte = rocketParams.fastLoadByte + val mulDivParams = rocketParams.mulDiv.getOrElse(MulDivParams()) + require(!fastLoadByte || fastLoadWord) + require(!rocketParams.haveFSDirty) + } + + class RocketCustomCSRs(implicit p: Parameters) extends CustomCSRs with HasRocketCoreParameters { + override def bpmCSR = rocketParams.branchPredictionModeCSR.option(CustomCSR(bpmCSRId, 1, Some(BigInt(0)))) + + override def chickenCSR = { + val dc = tileParams.dcache.get + val mask = BigInt(dc.clockGate.toInt | (rocketParams.clockGate.toInt << 1) | + (rocketParams.clockGate.toInt << 2) | (1 << 3) | (dc.scratch.isEmpty.toInt << 9) | + (tileParams.icache.get.prefetch.toInt << 17)) + Some(CustomCSR(chickenCSRId, mask, Some(mask))) + } + + def disableICachePrefetch = getOrElse(chickenCSR, _.value(17), true.B) + def marchid = CustomCSR.constant(CSRs.marchid, BigInt(1)) + def mvendorid = CustomCSR.constant(CSRs.mvendorid, BigInt(rocketParams.mvendorid)) + def mimpid = CustomCSR.constant(CSRs.mimpid, BigInt(rocketParams.mimpid)) + override def decls = super.decls :+ marchid :+ mvendorid :+ mimpid + } + +} + +// See LICENSE.Berkeley and LICENSE.SiFive for Rocket interface definitions. +package freechips.rocketchip.tile { + + import org.chipsalliance.cde.config.{Field, Parameters} + import chisel3.util.{log2Ceil, log2Up} + import freechips.rocketchip.rocket._ + + case object TileKey extends Field[CpuTileParams] + case object CoreBeatBytes extends Field[Int] + case object CoreRoCCCount extends Field[Int](0) + + trait HasTileParameters { + implicit val p: Parameters + def tileParams: CpuTileParams = p(TileKey) + def usingVM = tileParams.core.useVM + def usingSupervisor = tileParams.core.hasSupervisorMode + def usingUser = tileParams.core.useUser || usingSupervisor + def usingHypervisor = usingVM && tileParams.core.useHypervisor + def usingDebug = tileParams.core.useDebug + def usingRoCC = p(CoreRoCCCount) > 0 + def usingBTB = tileParams.btb.exists(_.nEntries > 0) + def usingPTW = usingVM + def usingDataScratchpad = tileParams.dcache.flatMap(_.scratch).isDefined + def xLen = tileParams.core.xLen + def xBytes = xLen / 8 + def iLen = 32 + def pgIdxBits = 12 + def pgLevelBits = 10 - log2Ceil(xLen / 32) + def pgLevels = tileParams.core.pgLevels + def minPgLevels = { val levels = if (xLen == 32) 2 else 3; require(pgLevels >= levels); levels } + def maxSVAddrBits = pgIdxBits + pgLevels * pgLevelBits + def maxHypervisorExtraAddrBits = 2 + def hypervisorExtraAddrBits = if (usingHypervisor) maxHypervisorExtraAddrBits else 0 + def maxHVAddrBits = maxSVAddrBits + hypervisorExtraAddrBits + def asIdBits = p(ASIdBits) + def vmIdBits = p(VMIdBits) + + lazy val maxPAddrBits: Int = { + require(xLen == 32 || xLen == 64); if (!usingVM) xLen else if (xLen == 32) 34 else 56 + } + + lazy val paddrBits: Int = p(CorePAddrBits) + + def vaddrBits: Int = + if (usingVM) { + val bits = maxHVAddrBits + require(bits == xLen || xLen > bits && bits > paddrBits) + bits + } else (paddrBits + 1).min(xLen) + + def vpnBits = vaddrBits - pgIdxBits + def ppnBits = paddrBits - pgIdxBits + def vpnBitsExtended = vpnBits + (if (vaddrBits < xLen) 1 + (if (usingHypervisor) 1 else 0) else 0) + def vaddrBitsExtended = vpnBitsExtended + pgIdxBits + def cacheBlockBytes = p(freechips.rocketchip.subsystem.CacheBlockBytes) + def lgCacheBlockBytes = log2Up(cacheBlockBytes) + def masterPortBeatBytes = p(CoreBeatBytes) + def dcacheArbPorts = 1 + (if (usingVM) 1 else 0) + (if (usingDataScratchpad) 1 else 0) + p(CoreRoCCCount) + + (if (tileParams.core.useVector && tileParams.core.vectorUseDCache) 1 else 0) + } + +} + +package freechips.rocketchip.rocket { + import org.chipsalliance.cde.config.Field + case object ASIdBits extends Field[Int](0) + case object VMIdBits extends Field[Int](0) +} + +package freechips.rocketchip.subsystem { + import org.chipsalliance.cde.config.Field + case object CacheBlockBytes extends Field[Int] +} + +package freechips.rocketchip.devices.debug { + import org.chipsalliance.cde.config.Field + case object DebugModuleKey extends Field[Option[DebugModuleParams]](Some(DebugModuleParams())) +} + +package freechips.rocketchip.rocket { + import chisel3._ + import chisel3.util._ + import org.chipsalliance.cde.config.Parameters + import freechips.rocketchip.tile._ + + case class DCacheParams( + nSets: Int = 64, + nWays: Int = 4, + rowBits: Int = 64, + subWordBits: Option[Int] = None, + replacementPolicy: String = "random", + nTLBSets: Int = 1, + nTLBWays: Int = 32, + nTLBBasePageSectors: Int = 4, + nTLBSuperpages: Int = 4, + tagECC: Option[String] = None, + dataECC: Option[String] = None, + dataECCBytes: Int = 1, + nMSHRs: Int = 1, + nSDQ: Int = 17, + nRPQ: Int = 16, + nMMIOs: Int = 1, + blockBytes: Int = 64, + separateUncachedResp: Boolean = false, + acquireBeforeRelease: Boolean = false, + pipelineWayMux: Boolean = false, + clockGate: Boolean = false, + scratch: Option[BigInt] = None) { + require(scratch.isEmpty || nWays == 1) + require(scratch.isEmpty || nMSHRs == 0) + if (scratch.isEmpty) require(isPow2(nSets)) + } + + case class ICacheParams( + nSets: Int = 64, + nWays: Int = 4, + rowBits: Int = 128, + nTLBSets: Int = 1, + nTLBWays: Int = 32, + nTLBBasePageSectors: Int = 4, + nTLBSuperpages: Int = 4, + cacheIdBits: Int = 0, + tagECC: Option[String] = None, + dataECC: Option[String] = None, + itimAddr: Option[BigInt] = None, + prefetch: Boolean = false, + blockBytes: Int = 64, + latency: Int = 2, + fetchBytes: Int = 4) + + case class RocketCoreParams( + xLen: Int = 64, + pgLevels: Int = 3, + bootFreqHz: BigInt = 0, + useVM: Boolean = true, + useUser: Boolean = false, + useSupervisor: Boolean = false, + useHypervisor: Boolean = false, + useDebug: Boolean = true, + useAtomics: Boolean = true, + useAtomicsOnlyForIO: Boolean = false, + useCompressed: Boolean = true, + useRVE: Boolean = false, + useConditionalZero: Boolean = false, + useZba: Boolean = false, + useZbb: Boolean = false, + useZbs: Boolean = false, + nLocalInterrupts: Int = 0, + useNMI: Boolean = false, + nBreakpoints: Int = 1, + useBPWatch: Boolean = false, + mcontextWidth: Int = 0, + scontextWidth: Int = 0, + nPMPs: Int = 8, + nPerfCounters: Int = 0, + haveBasicCounters: Boolean = true, + haveCFlush: Boolean = false, + misaWritable: Boolean = true, + nL2TLBEntries: Int = 0, + nL2TLBWays: Int = 1, + nPTECacheEntries: Int = 8, + mtvecInit: Option[BigInt] = Some(BigInt(0)), + mtvecWritable: Boolean = true, + fastLoadWord: Boolean = true, + fastLoadByte: Boolean = false, + branchPredictionModeCSR: Boolean = false, + clockGate: Boolean = false, + mvendorid: Int = 0, + mimpid: Int = 0x20181004, + mulDiv: Option[MulDivParams] = Some(MulDivParams()), + fpu: Option[FPUParams] = Some(FPUParams()), + debugROB: Option[DebugROBParams] = None, + haveCease: Boolean = true, + haveSimTimeout: Boolean = true, + vector: Option[RocketCoreVectorParams] = None, + enableTraceCoreIngress: Boolean = false) + extends CoreParams { + val lgPauseCycles = 5 + val haveFSDirty = false + val pmpGranularity = if (useHypervisor) 4096 else 4 + val fetchWidth = if (useCompressed) 2 else 1 + val decodeWidth = fetchWidth / (if (useCompressed) 2 else 1) + val retireWidth = 1 + val instBits = if (useCompressed) 16 else 32 + val lrscCycles = 80 + val traceHasWdata = debugROB.isDefined + override def minFLen = fpu.map(_.minFLen).getOrElse(32) + override def customCSRs(implicit p: Parameters) = new RocketCustomCSRs + override val useVector = vector.isDefined + override val vectorUseDCache = vector.exists(_.useDCache) + override def vLen = vector.map(_.vLen).getOrElse(0) + override def eLen = vector.map(_.eLen).getOrElse(0) + override def vfLen = vector.map(_.vfLen).getOrElse(0) + override def vfh = vector.exists(_.vfh) + override def vMemDataBits = vector.map(_.vMemDataBits).getOrElse(0) + override def vExts = vector.map(_.vExts).getOrElse(Nil) + } + + case class RocketCoreVectorParams( + vLen: Int, + eLen: Int, + vfLen: Int, + vfh: Boolean, + vMemDataBits: Int, + decoder: Parameters => RocketVectorDecoder, + useDCache: Boolean, + issueVConfig: Boolean, + vExts: Seq[String]) + +} + +package freechips.rocketchip.tile { + import freechips.rocketchip.rocket._ + + case class CpuTileParams( + core: CoreParams, + dcache: Option[DCacheParams], + icache: Option[ICacheParams], + btb: Option[BTBParams]) + +} + +package freechips.rocketchip.devices.debug { + case class DebugModuleParams(nDscratch: Int = 1, debugEntry: BigInt = 0x800, debugException: BigInt = 0x808) +} diff --git a/examples/prefab/cores/rocket/arch/pipeline/Pipeline.scala b/examples/prefab/cores/rocket/arch/pipeline/Pipeline.scala new file mode 100644 index 000000000..373cb3599 --- /dev/null +++ b/examples/prefab/cores/rocket/arch/pipeline/Pipeline.scala @@ -0,0 +1,1808 @@ +// See LICENSE.Berkeley for license details. +// See LICENSE.SiFive for license details. + +package examples.core.rocket.pipeline { + + import framework.lib.arbiter.{Arbiter} + import chisel3.experimental.hierarchy.Instantiate + + import chisel3._ + import chisel3.experimental.hierarchy.{instantiable, public} + import chisel3.util._ + import chisel3.withClock + import freechips.rocketchip.tile._ + import freechips.rocketchip.util._ + import freechips.rocketchip.util.property + import scala.collection.mutable.ArrayBuffer + import freechips.rocketchip.rocket._ + + /** RoCC core IO — used inside Rocket core. */ + class RoCCCoreIOBB(val nRoCCCSRs: Int = 0)(implicit val cpuParams: CpuParams) extends Bundle with HasCpuParameters { + val cmd = Flipped(Decoupled(new RoCCCommandBB)) + val resp = Decoupled(new RoCCResponseBB) + val busy = Output(Bool()) + val interrupt = Output(Bool()) + val exception = Input(Bool()) + val csrs = Flipped(Vec(nRoCCCSRs, new CustomCSRIO)) + } + + @instantiable + class RocketBB( + roccCSRs: Seq[CustomCSR], + coreHasBuckyball: Boolean, + hasTaskControl: Boolean, + val hasBusError: Boolean, + dcacheFlushOnFenceI: Boolean, + dcacheCanSupportCFlushLine: Boolean + )( + implicit val cpuParams: CpuParams) + extends Module + with HasRocketCoreParameters + with HasCpuParameters { + def nTotalRoCCCSRs = roccCSRs.size + + override val usingRoCC = coreHasBuckyball || hasTaskControl + + @public + val io = IO(new CoreBundle()(p) { + val hartid = Input(UInt(hartIdLen.W)) + val reset_vector = Input(UInt(resetVectorLen.W)) + val interrupts = Input(new CoreInterrupts(hasBusError)) + val interruptReady = Input(Bool()) + val imem = new FrontendIO + val dmem = new HellaCacheIO + val dmemPc = Output(UInt(vaddrBitsExtended.W)) + val roccOlderPending = Output(Bool()) + val roccCommitEligible = Output(Bool()) + val ptw = Flipped(new DatapathPTWIO()) + val fpu = Flipped(new FPUCoreIO()) + val rocc = Flipped(new RoCCCoreIOBB(nTotalRoCCCSRs)) + val trace = Output(new TraceBundle) + val bpwatch = Output(Vec(coreParams.nBreakpoints, new BPWatch(coreParams.retireWidth))) + val cease = Output(Bool()) + val wfi = Output(Bool()) + val traceStall = Input(Bool()) + val vector = if (usingVector) Some(Flipped(new VectorCoreIO)) else None + }) + + import ALU._ + + val clock_en_reg = RegInit(true.B) + val long_latency_stall = Reg(Bool()) + val id_reg_pause = Reg(Bool()) + val imem_might_request_reg = Reg(Bool()) + val clock_en = WireDefault(true.B) + + val gated_clock = + if (!rocketParams.clockGate) clock + else ClockGate(clock, clock_en, "rocket_clock_gate") + + class RocketImpl { // entering gated-clock domain + + // performance counters + def pipelineIDToWB[T <: Data](x: T): T = + RegEnable(RegEnable(RegEnable(x, !ctrl_killd), ex_pc_valid), mem_pc_valid) + + val perfEvents = new EventSets(Seq( + new EventSet( + (mask, hits) => Mux(wb_xcpt, mask(0), wb_valid && pipelineIDToWB((mask & hits).orR)), + Seq( + ("exception", () => false.B), + ("load", () => id_ctrl.mem && id_ctrl.mem_cmd === M_XRD && !id_ctrl.fp), + ("store", () => id_ctrl.mem && id_ctrl.mem_cmd === M_XWR && !id_ctrl.fp), + ( + "amo", + () => usingAtomics.B && id_ctrl.mem && (isAMO(id_ctrl.mem_cmd) || id_ctrl.mem_cmd.isOneOf(M_XLR, M_XSC)) + ), + ("system", () => id_ctrl.csr =/= CSR.N), + ( + "arith", + () => + id_ctrl.wxd && !(id_ctrl.jal || id_ctrl.jalr || id_ctrl.mem || id_ctrl.fp || id_ctrl.mul || id_ctrl.div || id_ctrl.csr =/= CSR.N) + ), + ("branch", () => id_ctrl.branch), + ("jal", () => id_ctrl.jal), + ("jalr", () => id_ctrl.jalr) + ) + ++ (if (!usingMulDiv) Seq() + else + Seq( + ("mul", () => if (pipelinedMul) id_ctrl.mul else id_ctrl.div && (id_ctrl.alu_fn & FN_DIV) =/= FN_DIV), + ("div", () => if (pipelinedMul) id_ctrl.div else id_ctrl.div && (id_ctrl.alu_fn & FN_DIV) === FN_DIV) + )) + ++ (if (!usingFPU) Seq() + else + Seq( + ("fp load", () => id_ctrl.fp && io.fpu.dec.ldst && io.fpu.dec.wen), + ("fp store", () => id_ctrl.fp && io.fpu.dec.ldst && !io.fpu.dec.wen), + ("fp add", () => id_ctrl.fp && io.fpu.dec.fma && io.fpu.dec.swap23), + ("fp mul", () => id_ctrl.fp && io.fpu.dec.fma && !io.fpu.dec.swap23 && !io.fpu.dec.ren3), + ("fp mul-add", () => id_ctrl.fp && io.fpu.dec.fma && io.fpu.dec.ren3), + ("fp div/sqrt", () => id_ctrl.fp && (io.fpu.dec.div || io.fpu.dec.sqrt)), + ( + "fp other", + () => id_ctrl.fp && !(io.fpu.dec.ldst || io.fpu.dec.fma || io.fpu.dec.div || io.fpu.dec.sqrt) + ) + )) + ), + new EventSet( + (mask, hits) => (mask & hits).orR, + Seq( + ( + "load-use interlock", + () => id_ex_hazard && ex_ctrl.mem || id_mem_hazard && mem_ctrl.mem || id_wb_hazard && wb_ctrl.mem + ), + ("long-latency interlock", () => id_sboard_hazard), + ( + "csr interlock", + () => + id_ex_hazard && ex_ctrl.csr =/= CSR.N || id_mem_hazard && mem_ctrl.csr =/= CSR.N || id_wb_hazard && wb_ctrl.csr =/= CSR.N + ), + ("I$ blocked", () => icache_blocked), + ("D$ blocked", () => id_ctrl.mem && dcache_blocked), + ("branch misprediction", () => take_pc_mem && mem_direction_misprediction), + ( + "control-flow target misprediction", + () => take_pc_mem && mem_misprediction && mem_cfi && !mem_direction_misprediction && !icache_blocked + ), + ("flush", () => wb_reg_flush_pipe), + ("replay", () => replay_wb) + ) + ++ (if (!usingMulDiv) Seq() + else + Seq( + ( + "mul/div interlock", + () => + id_ex_hazard && (ex_ctrl.mul || ex_ctrl.div) || id_mem_hazard && (mem_ctrl.mul || mem_ctrl.div) || id_wb_hazard && wb_ctrl.div + ) + )) + ++ (if (!usingFPU) Seq() + else + Seq( + ( + "fp interlock", + () => + id_ex_hazard && ex_ctrl.fp || id_mem_hazard && mem_ctrl.fp || id_wb_hazard && wb_ctrl.fp || id_ctrl.fp && id_stall_fpu + ) + )) + ), + new EventSet( + (mask, hits) => (mask & hits).orR, + Seq( + ("I$ miss", () => io.imem.perf.acquire), + ("D$ miss", () => io.dmem.perf.acquire), + ("D$ release", () => io.dmem.perf.release), + ("ITLB miss", () => io.imem.perf.tlbMiss), + ("DTLB miss", () => io.dmem.perf.tlbMiss), + ("L2 TLB miss", () => io.ptw.perf.l2miss) + ) + ) + )) + + val pipelinedMul = usingMulDiv && mulDivParams.mulUnroll == xLen + + val decode_table = { + (if (usingMulDiv) new MDecode(pipelinedMul) +: (xLen > 32).option(new M64Decode(pipelinedMul)).toSeq + else Nil) ++: + (if (usingAtomics) new ADecode +: (xLen > 32).option(new A64Decode).toSeq else Nil) ++: + (if (fLen >= 32) new FDecode +: (xLen > 32).option(new F64Decode).toSeq else Nil) ++: + (if (fLen >= 64) new DDecode +: (xLen > 32).option(new D64Decode).toSeq else Nil) ++: + (if (minFLen == 16) + new HDecode +: (xLen > 32).option(new H64Decode).toSeq ++: (fLen >= 64).option(new HDDecode).toSeq + else Nil) ++: + (usingRoCC.option(new RoCCDecode)) ++: + (if (xLen == 32) new I32Decode else new I64Decode) +: + (usingVM.option(new SVMDecode)) ++: + (usingSupervisor.option(new SDecode)) ++: + (usingHypervisor.option(new HypervisorDecode)) ++: + ((usingHypervisor && (xLen == 64)).option(new Hypervisor64Decode)) ++: + (usingDebug.option(new DebugDecode)) ++: + (usingNMI.option(new NMIDecode)) ++: + (usingConditionalZero.option(new ConditionalZeroDecode)) ++: + Seq(new FenceIDecode(dcacheFlushOnFenceI || coreParams.haveCFlush)) ++: + coreParams.haveCFlush.option(new CFlushDecode(dcacheCanSupportCFlushLine)) ++: + rocketParams.haveCease.option(new CeaseDecode) ++: + usingVector.option(new VCFGDecode) ++: + (if (coreParams.useZba) new ZbaDecode +: (xLen > 32).option(new Zba64Decode).toSeq else Nil) ++: + (if (coreParams.useZbb) Seq(new ZbbDecode, if (xLen == 32) new Zbb32Decode else new Zbb64Decode) else Nil) ++: + coreParams.useZbs.option(new ZbsDecode) ++: + Seq(new IDecode) + } flatMap (_.table) + + val ex_ctrl = Reg(new IntCtrlSigs) + val mem_ctrl = Reg(new IntCtrlSigs) + val wb_ctrl = Reg(new IntCtrlSigs) + + val ex_reg_xcpt_interrupt = Reg(Bool()) + val ex_reg_valid = Reg(Bool()) + val ex_reg_rvc = Reg(Bool()) + val ex_reg_btb_resp = Reg(new BTBResp) + val ex_reg_xcpt = Reg(Bool()) + val ex_reg_flush_pipe = Reg(Bool()) + val ex_reg_load_use = Reg(Bool()) + val ex_reg_cause = Reg(UInt()) + val ex_reg_replay = Reg(Bool()) + val ex_reg_pc = Reg(UInt()) + val ex_reg_mem_size = Reg(UInt()) + val ex_reg_hls = Reg(Bool()) + val ex_reg_inst = Reg(Bits()) + val ex_reg_raw_inst = Reg(UInt()) + val ex_reg_wphit = Reg(Vec(nBreakpoints, Bool())) + val ex_reg_set_vconfig = Reg(Bool()) + + val mem_reg_xcpt_interrupt = Reg(Bool()) + val mem_reg_valid = Reg(Bool()) + val mem_reg_rvc = Reg(Bool()) + val mem_reg_btb_resp = Reg(new BTBResp) + val mem_reg_xcpt = Reg(Bool()) + val mem_reg_replay = Reg(Bool()) + val mem_reg_flush_pipe = Reg(Bool()) + val mem_reg_cause = Reg(UInt()) + val mem_reg_slow_bypass = Reg(Bool()) + val mem_reg_load = Reg(Bool()) + val mem_reg_store = Reg(Bool()) + val mem_reg_set_vconfig = Reg(Bool()) + val mem_reg_sfence = Reg(Bool()) + val mem_reg_pc = Reg(UInt()) + val mem_reg_inst = Reg(Bits()) + val mem_reg_mem_size = Reg(UInt()) + val mem_reg_hls_or_dv = Reg(Bool()) + val mem_reg_raw_inst = Reg(UInt()) + val mem_reg_wdata = Reg(Bits()) + val mem_reg_rs2 = Reg(Bits()) + val mem_br_taken = Reg(Bool()) + val take_pc_mem = Wire(Bool()) + val mem_reg_wphit = Reg(Vec(nBreakpoints, Bool())) + + val wb_reg_valid = Reg(Bool()) + val wb_reg_xcpt = Reg(Bool()) + val wb_reg_replay = Reg(Bool()) + val wb_reg_flush_pipe = Reg(Bool()) + val wb_reg_cause = Reg(UInt()) + val wb_reg_set_vconfig = Reg(Bool()) + val wb_reg_sfence = Reg(Bool()) + val wb_reg_pc = Reg(UInt()) + val wb_reg_mem_size = Reg(UInt()) + val wb_reg_hls_or_dv = Reg(Bool()) + val wb_reg_hfence_v = Reg(Bool()) + val wb_reg_hfence_g = Reg(Bool()) + val wb_reg_inst = Reg(Bits()) + val wb_reg_raw_inst = Reg(UInt()) + val wb_reg_wdata = Reg(Bits()) + val wb_reg_rs2 = Reg(Bits()) + val take_pc_wb = Wire(Bool()) + val wb_reg_wphit = Reg(Vec(nBreakpoints, Bool())) + + val take_pc_mem_wb = take_pc_wb || take_pc_mem + val take_pc = take_pc_mem_wb + + // decode stage + val ibuf = Module(new IBuf) + val id_expanded_inst = ibuf.io.inst.map(_.bits.inst) + val id_raw_inst = ibuf.io.inst.map(_.bits.raw) + val id_inst = id_expanded_inst.map(_.bits) + ibuf.io.imem <> io.imem.resp + ibuf.io.kill := take_pc + + require(decodeWidth == 1 /* TODO */ && retireWidth == decodeWidth) + require(!(coreParams.useRVE && coreParams.fpu.nonEmpty), "Can't select both RVE and floating-point") + require(!(coreParams.useRVE && coreParams.useHypervisor), "Can't select both RVE and Hypervisor") + val id_ctrl = Wire(new IntCtrlSigs).decode(id_inst(0), decode_table) + + val lgNXRegs = if (coreParams.useRVE) 4 else 5 + val regAddrMask = (1 << lgNXRegs) - 1 + + def decodeReg(x: UInt) = (x.extract(x.getWidth - 1, lgNXRegs).asBool, x(lgNXRegs - 1, 0)) + val (id_raddr3_illegal, id_raddr3) = decodeReg(id_expanded_inst(0).rs3) + val (id_raddr2_illegal, id_raddr2) = decodeReg(id_expanded_inst(0).rs2) + val (id_raddr1_illegal, id_raddr1) = decodeReg(id_expanded_inst(0).rs1) + val (id_waddr_illegal, id_waddr) = decodeReg(id_expanded_inst(0).rd) + + val id_load_use = Wire(Bool()) + val id_reg_fence = RegInit(false.B) + val id_ren = IndexedSeq(id_ctrl.rxs1, id_ctrl.rxs2) + val id_raddr = IndexedSeq(id_raddr1, id_raddr2) + val rf = new RegFile(regAddrMask, xLen) + val id_rs = id_raddr.map(rf.read _) + val ctrl_killd = Wire(Bool()) + + // A scheduler needs the custom ISA even without compute Balls. + val csrP = + if (usingRoCC) p + else p.alterPartial { case CoreRoCCCount => 0 } + + val csr = Module(new CSRFile( + perfEvents, + coreParams.customCSRs.decls, + roccCSRs, + hasBusError + )(csrP) { + override def usingRoCC: Boolean = RocketBB.this.usingRoCC + }) + + val id_csr_en = id_ctrl.csr.isOneOf(CSR.S, CSR.C, CSR.W) + val id_system_insn = id_ctrl.csr === CSR.I + val id_csr_ren = id_ctrl.csr.isOneOf(CSR.S, CSR.C) && id_expanded_inst(0).rs1 === 0.U + val id_csr = Mux(id_system_insn && id_ctrl.mem, CSR.N, Mux(id_csr_ren, CSR.R, id_ctrl.csr)) + val id_csr_flush = id_system_insn || (id_csr_en && !id_csr_ren && csr.io.decode(0).write_flush) + val id_set_vconfig = + Seq(Instructions.VSETVLI, Instructions.VSETIVLI, Instructions.VSETVL).map(_ === id_inst(0)).orR && usingVector.B + + id_ctrl.vec := false.B + if (usingVector) { + val v_decode = rocketParams.vector.get.decoder(p) + v_decode.io.inst := id_inst(0) + v_decode.io.vconfig := csr.io.vector.get.vconfig + when(v_decode.io.legal) { + id_ctrl.legal := !csr.io.vector.get.vconfig.vtype.vill + id_ctrl.fp := v_decode.io.fp + id_ctrl.rocc := false.B + id_ctrl.branch := false.B + id_ctrl.jal := false.B + id_ctrl.jalr := false.B + id_ctrl.rxs2 := v_decode.io.read_rs2 + id_ctrl.rxs1 := v_decode.io.read_rs1 + id_ctrl.mem := false.B + id_ctrl.rfs1 := v_decode.io.read_frs1 + id_ctrl.rfs2 := false.B + id_ctrl.rfs3 := false.B + id_ctrl.wfd := v_decode.io.write_frd + id_ctrl.mul := false.B + id_ctrl.div := false.B + id_ctrl.wxd := v_decode.io.write_rd + id_ctrl.csr := CSR.N + id_ctrl.fence_i := false.B + id_ctrl.fence := false.B + id_ctrl.amo := false.B + id_ctrl.dp := false.B + id_ctrl.vec := true.B + } + } + + val id_illegal_insn = !id_ctrl.legal || + (id_ctrl.mul || id_ctrl.div) && !csr.io.status.isa('m' - 'a') || + id_ctrl.amo && !csr.io.status.isa('a' - 'a') || + id_ctrl.fp && (csr.io.decode(0).fp_illegal || (io.fpu.illegal_rm && !id_ctrl.vec)) || + (id_ctrl.vec) && (csr.io.decode(0).vector_illegal || csr.io.vector.map(_.vconfig.vtype.vill).getOrElse( + false.B + )) || + id_ctrl.dp && !csr.io.status.isa('d' - 'a') || + ibuf.io.inst(0).bits.rvc && !csr.io.status.isa('c' - 'a') || + id_raddr2_illegal && id_ctrl.rxs2 || + id_raddr1_illegal && id_ctrl.rxs1 || + id_waddr_illegal && id_ctrl.wxd || + id_ctrl.rocc && csr.io.decode(0).rocc_illegal || + id_csr_en && (csr.io.decode(0).read_illegal || !id_csr_ren && csr.io.decode(0).write_illegal) || + !ibuf.io.inst(0).bits.rvc && (id_system_insn && csr.io.decode(0).system_illegal) + + val id_virtual_insn = id_ctrl.legal && + ((id_csr_en && !(!id_csr_ren && csr.io.decode(0).write_illegal) && csr.io.decode(0).virtual_access_illegal) || + (!ibuf.io.inst(0).bits.rvc && id_system_insn && csr.io.decode(0).virtual_system_illegal)) + + // stall decode for fences (now, for AMO.rl; later, for AMO.aq and FENCE) + val id_amo_aq = id_inst(0)(26) + val id_amo_rl = id_inst(0)(25) + val id_fence_succ = id_inst(0)(23, 20) + val id_fence_next = id_ctrl.fence || id_ctrl.amo && id_amo_aq + val id_mem_busy = !io.dmem.ordered || io.dmem.req.valid + when(!id_mem_busy)(id_reg_fence := false.B) + + val id_rocc_busy = usingRoCC.B && + (io.rocc.busy || ex_reg_valid && ex_ctrl.rocc || + mem_reg_valid && mem_ctrl.rocc || wb_reg_valid && wb_ctrl.rocc) + + val id_csr_rocc_write = roccCSRs.map(_.id.U === id_inst(0)(31, 20)).orR && id_csr_en && !id_csr_ren + val id_vec_busy = io.vector.map(v => v.backend_busy || v.trap_check_busy).getOrElse(false.B) + + val id_do_fence = WireDefault(id_rocc_busy && (id_ctrl.fence || id_csr_rocc_write) || + id_vec_busy && id_ctrl.fence || + id_mem_busy && (id_ctrl.amo && id_amo_rl || id_ctrl.fence_i || id_reg_fence && (id_ctrl.mem || id_ctrl.rocc))) + + val bpu = Module(new BreakpointUnit(nBreakpoints)) + bpu.io.status := csr.io.status + bpu.io.bp := csr.io.bp + bpu.io.pc := ibuf.io.pc + bpu.io.ea := mem_reg_wdata + bpu.io.mcontext := csr.io.mcontext + bpu.io.scontext := csr.io.scontext + + val id_xcpt0 = ibuf.io.inst(0).bits.xcpt0 + val id_xcpt1 = ibuf.io.inst(0).bits.xcpt1 + + val deliveredInterrupt = csr.io.interrupt && io.interruptReady + + val (id_xcpt, id_cause) = checkExceptions(List( + (deliveredInterrupt, csr.io.interrupt_cause), + (bpu.io.debug_if, CSR.debugTriggerCause.U), + (bpu.io.xcpt_if, Causes.breakpoint.U), + (id_xcpt0.pf.inst, Causes.fetch_page_fault.U), + (id_xcpt0.gf.inst, Causes.fetch_guest_page_fault.U), + (id_xcpt0.ae.inst, Causes.fetch_access.U), + (id_xcpt1.pf.inst, Causes.fetch_page_fault.U), + (id_xcpt1.gf.inst, Causes.fetch_guest_page_fault.U), + (id_xcpt1.ae.inst, Causes.fetch_access.U), + (id_virtual_insn, Causes.virtual_instruction.U), + (id_illegal_insn, Causes.illegal_instruction.U) + )) + + val idCoverCauses = List( + (CSR.debugTriggerCause, "DEBUG_TRIGGER"), + (Causes.breakpoint, "BREAKPOINT"), + (Causes.fetch_access, "FETCH_ACCESS"), + (Causes.illegal_instruction, "ILLEGAL_INSTRUCTION") + ) ++ (if (usingVM) + List( + (Causes.fetch_page_fault, "FETCH_PAGE_FAULT") + ) + else Nil) + + coverExceptions(id_xcpt, id_cause, "DECODE", idCoverCauses) + + val dcache_bypass_data = + if (fastLoadByte) io.dmem.resp.bits.data(xLen - 1, 0) + else if (fastLoadWord) io.dmem.resp.bits.data_word_bypass(xLen - 1, 0) + else wb_reg_wdata + + // detect bypass opportunities + val ex_waddr = ex_reg_inst(11, 7) & regAddrMask.U + val mem_waddr = mem_reg_inst(11, 7) & regAddrMask.U + val wb_waddr = wb_reg_inst(11, 7) & regAddrMask.U + + val bypass_sources = IndexedSeq( + (true.B, 0.U, 0.U), // treat reading x0 as a bypass + (ex_reg_valid && ex_ctrl.wxd, ex_waddr, mem_reg_wdata), + (mem_reg_valid && mem_ctrl.wxd && !mem_ctrl.mem, mem_waddr, wb_reg_wdata), + (mem_reg_valid && mem_ctrl.wxd, mem_waddr, dcache_bypass_data) + ) + + val id_bypass_src = id_raddr.map(raddr => bypass_sources.map(s => s._1 && s._2 === raddr)) + + // execute stage + val bypass_mux = bypass_sources.map(_._3) + val ex_reg_rs_bypass = Reg(Vec(id_raddr.size, Bool())) + val ex_reg_rs_lsb = Reg(Vec(id_raddr.size, UInt(log2Ceil(bypass_sources.size).W))) + val ex_reg_rs_msb = Reg(Vec(id_raddr.size, UInt())) + + val ex_rs = + for (i <- 0 until id_raddr.size) + yield Mux(ex_reg_rs_bypass(i), bypass_mux(ex_reg_rs_lsb(i)), Cat(ex_reg_rs_msb(i), ex_reg_rs_lsb(i))) + + val ex_imm = ImmGen(ex_ctrl.sel_imm, ex_reg_inst) + val ex_rs1shl = Mux(ex_reg_inst(3), ex_rs(0)(31, 0), ex_rs(0)) << ex_reg_inst(14, 13) + + val ex_op1 = MuxLookup(ex_ctrl.sel_alu1, 0.S)(Seq( + A1_RS1 -> ex_rs(0).asSInt, + A1_PC -> ex_reg_pc.asSInt, + A1_RS1SHL -> (if (rocketParams.useZba) ex_rs1shl.asSInt else 0.S) + )) + + val ex_op2_oh = + UIntToOH(Mux(ex_ctrl.sel_alu2(0), (ex_reg_inst >> 20).asUInt, ex_rs(1))(log2Ceil(xLen) - 1, 0)).asSInt + + val ex_op2 = MuxLookup(ex_ctrl.sel_alu2, 0.S)(Seq( + A2_RS2 -> ex_rs(1).asSInt, + A2_IMM -> ex_imm, + A2_SIZE -> Mux(ex_reg_rvc, 2.S, 4.S) + ) ++ (if (coreParams.useZbs) + Seq( + A2_RS2OH -> ex_op2_oh, + A2_IMMOH -> ex_op2_oh + ) + else Nil)) + + val (ex_new_vl, ex_new_vconfig) = + if (usingVector) { + val ex_new_vtype = VType.fromUInt(MuxCase( + ex_rs(1), + Seq( + ex_reg_inst(31, 30).andR -> ex_reg_inst(29, 20), + !ex_reg_inst(31) -> ex_reg_inst(30, 20) + ) + )) + val ex_avl = Mux( + ex_ctrl.rxs1, + Mux( + ex_reg_inst(19, 15) === 0.U, + Mux(ex_reg_inst(11, 7) === 0.U, csr.io.vector.get.vconfig.vl, ex_new_vtype.vlMax), + ex_rs(0) + ), + ex_reg_inst(19, 15) + ) + val ex_new_vl = ex_new_vtype.vl(ex_avl, csr.io.vector.get.vconfig.vl, false.B, false.B, false.B) + val ex_new_vconfig = Wire(new VConfig) + ex_new_vconfig.vtype := ex_new_vtype + ex_new_vconfig.vl := ex_new_vl + (Some(ex_new_vl), Some(ex_new_vconfig)) + } else { (None, None) } + + val alu = Module(new ALU) + alu.io.dw := ex_ctrl.alu_dw + alu.io.fn := ex_ctrl.alu_fn + alu.io.in2 := ex_op2.asUInt + alu.io.in1 := ex_op1.asUInt + + // multiplier and divider + val div = Module(new MulDiv(if (pipelinedMul) mulDivParams.copy(mulUnroll = 0) else mulDivParams, xLen)) + div.io.req.valid := ex_reg_valid && ex_ctrl.div + div.io.req.bits.dw := ex_ctrl.alu_dw + div.io.req.bits.fn := ex_ctrl.alu_fn + div.io.req.bits.in1 := ex_rs(0) + div.io.req.bits.in2 := ex_rs(1) + div.io.req.bits.tag := ex_waddr + + val mul = pipelinedMul.option { + val m = Module(new PipelinedMultiplier(xLen, 2)) + m.io.req.valid := ex_reg_valid && ex_ctrl.mul + m.io.req.bits := div.io.req.bits + m + } + + ex_reg_valid := !ctrl_killd + ex_reg_replay := !take_pc && ibuf.io.inst(0).valid && ibuf.io.inst(0).bits.replay + ex_reg_xcpt := !ctrl_killd && id_xcpt + ex_reg_xcpt_interrupt := !take_pc && ibuf.io.inst(0).valid && deliveredInterrupt + + when(!ctrl_killd) { + ex_ctrl := id_ctrl + ex_reg_rvc := ibuf.io.inst(0).bits.rvc + ex_ctrl.csr := id_csr + when(id_ctrl.fence && id_fence_succ === 0.U)(id_reg_pause := true.B) + when(id_fence_next)(id_reg_fence := true.B) + when(id_xcpt) { // pass PC down ALU writeback pipeline for badaddr + ex_ctrl.alu_fn := FN_ADD + ex_ctrl.alu_dw := DW_XPR + ex_ctrl.sel_alu1 := A1_RS1 // badaddr := instruction + ex_ctrl.sel_alu2 := A2_ZERO + when(id_xcpt1.asUInt.orR) { // badaddr := PC+2 + ex_ctrl.sel_alu1 := A1_PC + ex_ctrl.sel_alu2 := A2_SIZE + ex_reg_rvc := true.B + } + when(bpu.io.xcpt_if || id_xcpt0.asUInt.orR) { // badaddr := PC + ex_ctrl.sel_alu1 := A1_PC + ex_ctrl.sel_alu2 := A2_ZERO + } + } + ex_reg_flush_pipe := id_ctrl.fence_i || id_csr_flush + ex_reg_load_use := id_load_use + ex_reg_hls := usingHypervisor.B && id_system_insn && id_ctrl.mem_cmd.isOneOf(M_XRD, M_XWR, M_HLVX) + ex_reg_mem_size := Mux(usingHypervisor.B && id_system_insn, id_inst(0)(27, 26), id_inst(0)(13, 12)) + when(id_ctrl.mem_cmd.isOneOf(M_SFENCE, M_HFENCEV, M_HFENCEG, M_FLUSH_ALL)) { + ex_reg_mem_size := Cat(id_raddr2 =/= 0.U, id_raddr1 =/= 0.U) + } + when(id_ctrl.mem_cmd === M_SFENCE && csr.io.status.v) { + ex_ctrl.mem_cmd := M_HFENCEV + } + if (dcacheFlushOnFenceI || coreParams.haveCFlush) { + when(id_ctrl.fence_i) { + ex_reg_mem_size := 0.U + } + } + + for (i <- 0 until id_raddr.size) { + val do_bypass = id_bypass_src(i).reduce(_ || _) + val bypass_src = PriorityEncoder(id_bypass_src(i)) + ex_reg_rs_bypass(i) := do_bypass + ex_reg_rs_lsb(i) := bypass_src + when(id_ren(i) && !do_bypass) { + ex_reg_rs_lsb(i) := id_rs(i)(log2Ceil(bypass_sources.size) - 1, 0) + ex_reg_rs_msb(i) := id_rs(i) >> log2Ceil(bypass_sources.size) + } + } + when(id_illegal_insn || id_virtual_insn) { + val inst = Mux(ibuf.io.inst(0).bits.rvc, id_raw_inst(0)(15, 0), id_raw_inst(0)) + ex_reg_rs_bypass(0) := false.B + ex_reg_rs_lsb(0) := inst(log2Ceil(bypass_sources.size) - 1, 0) + ex_reg_rs_msb(0) := inst >> log2Ceil(bypass_sources.size) + } + } + when(!ctrl_killd || deliveredInterrupt || ibuf.io.inst(0).bits.replay) { + ex_reg_cause := id_cause + ex_reg_inst := id_inst(0) + ex_reg_raw_inst := id_raw_inst(0) + ex_reg_pc := ibuf.io.pc + ex_reg_btb_resp := ibuf.io.btb_resp + ex_reg_wphit := bpu.io.bpwatch.map(bpw => bpw.ivalid(0)) + ex_reg_set_vconfig := id_set_vconfig && !id_xcpt + } + + // replay inst in ex stage? + val ex_pc_valid = ex_reg_valid || ex_reg_replay || ex_reg_xcpt_interrupt + val wb_dcache_miss = wb_ctrl.mem && !io.dmem.resp.valid + + val replay_ex_structural = ex_ctrl.mem && !io.dmem.req.ready || + ex_ctrl.div && !div.io.req.ready || + ex_ctrl.vec && !io.vector.map(_.ex.ready).getOrElse(true.B) + + val replay_ex_load_use = wb_dcache_miss && ex_reg_load_use + val replay_ex = ex_reg_replay || (ex_reg_valid && (replay_ex_structural || replay_ex_load_use)) + val ctrl_killx = take_pc_mem_wb || replay_ex || !ex_reg_valid + // detect 2-cycle load-use delay for LB/LH/SC + val ex_slow_bypass = ex_ctrl.mem_cmd === M_XSC || ex_reg_mem_size < 2.U + val ex_sfence = + usingVM.B && ex_ctrl.mem && (ex_ctrl.mem_cmd === M_SFENCE || ex_ctrl.mem_cmd === M_HFENCEV || ex_ctrl.mem_cmd === M_HFENCEG) + + val (ex_xcpt, ex_cause) = checkExceptions(List( + (ex_reg_xcpt_interrupt || ex_reg_xcpt, ex_reg_cause) + )) + + val exCoverCauses = idCoverCauses + coverExceptions(ex_xcpt, ex_cause, "EXECUTE", exCoverCauses) + + // memory stage + val mem_pc_valid = mem_reg_valid || mem_reg_replay || mem_reg_xcpt_interrupt + + val mem_br_target = mem_reg_pc.asSInt + + Mux( + mem_ctrl.branch && mem_br_taken, + ImmGen(IMM_SB, mem_reg_inst), + Mux(mem_ctrl.jal, ImmGen(IMM_UJ, mem_reg_inst), Mux(mem_reg_rvc, 2.S, 4.S)) + ) + + val mem_npc = (Mux( + mem_ctrl.jalr || mem_reg_sfence, + encodeVirtualAddress(mem_reg_wdata, mem_reg_wdata).asSInt, + mem_br_target + ) & -2.S).asUInt + + val mem_wrong_npc = + Mux( + ex_pc_valid, + mem_npc =/= ex_reg_pc, + Mux(ibuf.io.inst(0).valid || ibuf.io.imem.valid, mem_npc =/= ibuf.io.pc, true.B) + ) + + val mem_npc_misaligned = !csr.io.status.isa('c' - 'a') && mem_npc(1) && !mem_reg_sfence + val mem_int_wdata = + Mux(!mem_reg_xcpt && (mem_ctrl.jalr ^ mem_npc_misaligned), mem_br_target, mem_reg_wdata.asSInt).asUInt + val mem_cfi = mem_ctrl.branch || mem_ctrl.jalr || mem_ctrl.jal + val mem_cfi_taken = (mem_ctrl.branch && mem_br_taken) || mem_ctrl.jalr || mem_ctrl.jal + val mem_direction_misprediction = mem_ctrl.branch && mem_br_taken =/= (usingBTB.B && mem_reg_btb_resp.taken) + val mem_misprediction = if (usingBTB) mem_wrong_npc else mem_cfi_taken + take_pc_mem := mem_reg_valid && !mem_reg_xcpt && (mem_misprediction || mem_reg_sfence) + + mem_reg_valid := !ctrl_killx + mem_reg_replay := !take_pc_mem_wb && replay_ex + mem_reg_xcpt := !ctrl_killx && ex_xcpt + mem_reg_xcpt_interrupt := !take_pc_mem_wb && ex_reg_xcpt_interrupt + + // on pipeline flushes, cause mem_npc to hold the sequential npc, which + // will drive the W-stage npc mux + when(mem_reg_valid && mem_reg_flush_pipe) { + mem_reg_sfence := false.B + }.elsewhen(ex_pc_valid) { + mem_ctrl := ex_ctrl + mem_reg_rvc := ex_reg_rvc + mem_reg_load := ex_ctrl.mem && isRead(ex_ctrl.mem_cmd) + mem_reg_store := ex_ctrl.mem && isWrite(ex_ctrl.mem_cmd) + mem_reg_sfence := ex_sfence + mem_reg_btb_resp := ex_reg_btb_resp + mem_reg_flush_pipe := ex_reg_flush_pipe + mem_reg_slow_bypass := ex_slow_bypass + mem_reg_wphit := ex_reg_wphit + mem_reg_set_vconfig := ex_reg_set_vconfig + + mem_reg_cause := ex_cause + mem_reg_inst := ex_reg_inst + mem_reg_raw_inst := ex_reg_raw_inst + mem_reg_mem_size := ex_reg_mem_size + mem_reg_hls_or_dv := io.dmem.req.bits.dv + mem_reg_pc := ex_reg_pc + // IDecode ensured they are 1H + mem_reg_wdata := Mux(ex_reg_set_vconfig, ex_new_vl.getOrElse(alu.io.out), alu.io.out) + mem_br_taken := alu.io.cmp_out + + when(ex_ctrl.rxs2 && (ex_ctrl.mem || ex_ctrl.rocc || ex_sfence)) { + val size = Mux(ex_ctrl.rocc, log2Ceil(xLen / 8).U, ex_reg_mem_size) + mem_reg_rs2 := new StoreGen(size, 0.U, ex_rs(1), coreDataBytes).data + } + if (usingVector) { + when(ex_reg_set_vconfig) { + mem_reg_rs2 := ex_new_vconfig.get.asUInt + } + } + when(ex_ctrl.jalr && csr.io.status.debug) { + // flush I$ on D-mode JALR to effect uncached fetch without D$ flush + mem_ctrl.fence_i := true.B + mem_reg_flush_pipe := true.B + } + } + + val mem_breakpoint = (mem_reg_load && bpu.io.xcpt_ld) || (mem_reg_store && bpu.io.xcpt_st) + val mem_debug_breakpoint = (mem_reg_load && bpu.io.debug_ld) || (mem_reg_store && bpu.io.debug_st) + + val (mem_ldst_xcpt, mem_ldst_cause) = checkExceptions(List( + (mem_debug_breakpoint, CSR.debugTriggerCause.U), + (mem_breakpoint, Causes.breakpoint.U) + )) + + val (mem_xcpt, mem_cause) = checkExceptions(List( + (mem_reg_xcpt_interrupt || mem_reg_xcpt, mem_reg_cause), + (mem_reg_valid && mem_npc_misaligned, Causes.misaligned_fetch.U), + (mem_reg_valid && mem_ldst_xcpt, mem_ldst_cause) + )) + + val memCoverCauses = (exCoverCauses ++ List( + (CSR.debugTriggerCause, "DEBUG_TRIGGER"), + (Causes.breakpoint, "BREAKPOINT"), + (Causes.misaligned_fetch, "MISALIGNED_FETCH") + )).distinct + + coverExceptions(mem_xcpt, mem_cause, "MEMORY", memCoverCauses) + + val dcache_kill_mem = mem_reg_valid && mem_ctrl.wxd && io.dmem.replay_next // structural hazard on writeback port + val fpu_kill_mem = mem_reg_valid && mem_ctrl.fp && io.fpu.nack_mem + val vec_kill_mem = mem_reg_valid && mem_ctrl.mem && io.vector.map(_.mem.block_mem).getOrElse(false.B) + val vec_kill_all = mem_reg_valid && io.vector.map(_.mem.block_all).getOrElse(false.B) + val replay_mem = dcache_kill_mem || mem_reg_replay || fpu_kill_mem || vec_kill_mem || vec_kill_all + val killm_common = dcache_kill_mem || take_pc_wb || mem_reg_xcpt || !mem_reg_valid + div.io.kill := killm_common && RegNext(div.io.req.fire) + val ctrl_killm = killm_common || mem_xcpt || fpu_kill_mem || vec_kill_mem + + // writeback stage + wb_reg_valid := !ctrl_killm + wb_reg_replay := replay_mem && !take_pc_wb + wb_reg_xcpt := mem_xcpt && !take_pc_wb && !io.vector.map(_.mem.block_all).getOrElse(false.B) + wb_reg_flush_pipe := !ctrl_killm && mem_reg_flush_pipe + when(mem_pc_valid) { + wb_ctrl := mem_ctrl + wb_reg_sfence := mem_reg_sfence + wb_reg_wdata := Mux(!mem_reg_xcpt && mem_ctrl.fp && mem_ctrl.wxd, io.fpu.toint_data, mem_int_wdata) + when(mem_ctrl.rocc || mem_reg_sfence || mem_reg_set_vconfig) { + wb_reg_rs2 := mem_reg_rs2 + } + wb_reg_cause := mem_cause + wb_reg_inst := mem_reg_inst + wb_reg_raw_inst := mem_reg_raw_inst + wb_reg_mem_size := mem_reg_mem_size + wb_reg_hls_or_dv := mem_reg_hls_or_dv + wb_reg_hfence_v := mem_ctrl.mem_cmd === M_HFENCEV + wb_reg_hfence_g := mem_ctrl.mem_cmd === M_HFENCEG + wb_reg_pc := mem_reg_pc + wb_reg_wphit := mem_reg_wphit | bpu.io.bpwatch.map { bpw => + (bpw.rvalid(0) && mem_reg_load) || (bpw.wvalid(0) && mem_reg_store) + } + wb_reg_set_vconfig := mem_reg_set_vconfig + } + + val (wb_xcpt, wb_cause) = checkExceptions(List( + (wb_reg_xcpt, wb_reg_cause), + (wb_reg_valid && wb_ctrl.mem && io.dmem.s2_xcpt.pf.st, Causes.store_page_fault.U), + (wb_reg_valid && wb_ctrl.mem && io.dmem.s2_xcpt.pf.ld, Causes.load_page_fault.U), + (wb_reg_valid && wb_ctrl.mem && io.dmem.s2_xcpt.gf.st, Causes.store_guest_page_fault.U), + (wb_reg_valid && wb_ctrl.mem && io.dmem.s2_xcpt.gf.ld, Causes.load_guest_page_fault.U), + (wb_reg_valid && wb_ctrl.mem && io.dmem.s2_xcpt.ae.st, Causes.store_access.U), + (wb_reg_valid && wb_ctrl.mem && io.dmem.s2_xcpt.ae.ld, Causes.load_access.U), + (wb_reg_valid && wb_ctrl.mem && io.dmem.s2_xcpt.ma.st, Causes.misaligned_store.U), + (wb_reg_valid && wb_ctrl.mem && io.dmem.s2_xcpt.ma.ld, Causes.misaligned_load.U) + )) + + val wbCoverCauses = List( + (Causes.misaligned_store, "MISALIGNED_STORE"), + (Causes.misaligned_load, "MISALIGNED_LOAD"), + (Causes.store_access, "STORE_ACCESS"), + (Causes.load_access, "LOAD_ACCESS") + ) ++ (if (usingVM) + List( + (Causes.store_page_fault, "STORE_PAGE_FAULT"), + (Causes.load_page_fault, "LOAD_PAGE_FAULT") + ) + else Nil) ++ (if (usingHypervisor) + List( + (Causes.store_guest_page_fault, "STORE_GUEST_PAGE_FAULT"), + (Causes.load_guest_page_fault, "LOAD_GUEST_PAGE_FAULT") + ) + else Nil) + + coverExceptions(wb_xcpt, wb_cause, "WRITEBACK", wbCoverCauses) + + val wb_pc_valid = wb_reg_valid || wb_reg_replay || wb_reg_xcpt + val wb_wxd = wb_reg_valid && wb_ctrl.wxd + val wb_set_sboard = wb_ctrl.div || wb_dcache_miss || wb_ctrl.rocc || wb_ctrl.vec + val replay_wb_common = io.dmem.s2_nack || wb_reg_replay + val replay_wb_rocc = wb_reg_valid && wb_ctrl.rocc && !io.rocc.cmd.ready + val replay_wb_csr: Bool = wb_reg_valid && csr.io.rw_stall + val replay_wb_vec = wb_reg_valid && io.vector.map(_.wb.replay).getOrElse(false.B) + val replay_wb = replay_wb_common || replay_wb_rocc || replay_wb_csr || replay_wb_vec + take_pc_wb := replay_wb || wb_xcpt || csr.io.eret || wb_reg_flush_pipe + + // writeback arbitration + val dmem_resp_xpu = !io.dmem.resp.bits.tag(0).asBool + val dmem_resp_fpu = io.dmem.resp.bits.tag(0).asBool + val dmem_resp_waddr = io.dmem.resp.bits.tag(5, 1) + val dmem_resp_valid = io.dmem.resp.valid && io.dmem.resp.bits.has_data + val dmem_resp_replay = dmem_resp_valid && io.dmem.resp.bits.replay + + class LLWB extends Bundle { + val data = UInt(xLen.W) + val tag = UInt(5.W) + } + + val ll_arb = Instantiate(new Arbiter(new LLWB, 3)) // div, rocc, vec + ll_arb.io.in.foreach(_.valid := false.B) + ll_arb.io.in.foreach(_.bits := DontCare) + val ll_wdata = WireInit(ll_arb.io.out.bits.data) + val ll_waddr = WireInit(ll_arb.io.out.bits.tag) + val ll_wen = WireInit(ll_arb.io.out.fire) + ll_arb.io.out.ready := !wb_wxd + + div.io.resp.ready := ll_arb.io.in(0).ready + ll_arb.io.in(0).valid := div.io.resp.valid + ll_arb.io.in(0).bits.data := div.io.resp.bits.data + ll_arb.io.in(0).bits.tag := div.io.resp.bits.tag + + if (usingRoCC) { + io.rocc.resp.ready := ll_arb.io.in(1).ready + ll_arb.io.in(1).valid := io.rocc.resp.valid + ll_arb.io.in(1).bits.data := io.rocc.resp.bits.data + ll_arb.io.in(1).bits.tag := io.rocc.resp.bits.rd + } else { + // tie off RoCC + io.rocc.resp.ready := false.B + } + + io.vector.map { v => + v.resp.ready := Mux(v.resp.bits.fp, !(dmem_resp_valid && dmem_resp_fpu), ll_arb.io.in(2).ready) + ll_arb.io.in(2).valid := v.resp.valid && !v.resp.bits.fp + ll_arb.io.in(2).bits.data := v.resp.bits.data + ll_arb.io.in(2).bits.tag := v.resp.bits.rd + } + + when(dmem_resp_replay && dmem_resp_xpu) { + ll_arb.io.out.ready := false.B + ll_waddr := dmem_resp_waddr + ll_wen := true.B + } + + val wb_valid = wb_reg_valid && !replay_wb && !wb_xcpt + val wb_wen = wb_valid && wb_ctrl.wxd + val rf_wen = wb_wen || ll_wen + val rf_waddr = Mux(ll_wen, ll_waddr, wb_waddr) + + val rf_wdata = Mux( + dmem_resp_valid && dmem_resp_xpu, + io.dmem.resp.bits.data(xLen - 1, 0), + Mux( + ll_wen, + ll_wdata, + Mux( + wb_ctrl.csr =/= CSR.N, + csr.io.rw.rdata, + Mux(wb_ctrl.mul, mul.map(_.io.resp.bits.data).getOrElse(wb_reg_wdata), wb_reg_wdata) + ) + ) + ) + + when(rf_wen)(rf.write(rf_waddr, rf_wdata)) + + // hook up control/status regfile + csr.io.ungated_clock := clock + csr.io.decode(0).inst := id_inst(0) + csr.io.exception := wb_xcpt + csr.io.cause := wb_cause + csr.io.retire := wb_valid + csr.io.inst(0) := (if (usingCompressed) + Cat(Mux(wb_reg_raw_inst(1, 0).andR, wb_reg_inst >> 16, 0.U), wb_reg_raw_inst(15, 0)) + else wb_reg_inst) + csr.io.interrupts := io.interrupts + csr.io.hartid := io.hartid + io.fpu.fcsr_rm := csr.io.fcsr_rm + val vector_fcsr_flags = io.vector.map(_.set_fflags.bits).getOrElse(0.U(5.W)) + val vector_fcsr_flags_valid = io.vector.map(_.set_fflags.valid).getOrElse(false.B) + csr.io.fcsr_flags.valid := io.fpu.fcsr_flags.valid | vector_fcsr_flags_valid + csr.io.fcsr_flags.bits := (io.fpu.fcsr_flags.bits & Fill(5, io.fpu.fcsr_flags.valid)) | (vector_fcsr_flags & Fill( + 5, + vector_fcsr_flags_valid + )) + io.fpu.time := csr.io.time(31, 0) + io.fpu.hartid := io.hartid + csr.io.rocc_interrupt := io.rocc.interrupt + csr.io.pc := wb_reg_pc + + val tval_dmem_addr = !wb_reg_xcpt + + val tval_any_addr = tval_dmem_addr || + wb_reg_cause.isOneOf( + Causes.breakpoint.U, + Causes.fetch_access.U, + Causes.fetch_page_fault.U, + Causes.fetch_guest_page_fault.U + ) + + val tval_inst = wb_reg_cause === Causes.illegal_instruction.U + val tval_valid = wb_xcpt && (tval_any_addr || tval_inst) + csr.io.gva := wb_xcpt && (tval_any_addr && csr.io.status.v || tval_dmem_addr && wb_reg_hls_or_dv) + csr.io.tval := Mux(tval_valid, encodeVirtualAddress(wb_reg_wdata, wb_reg_wdata), 0.U) + + val (htval, mhtinst_read_pseudo) = { + val htval_valid_imem = wb_reg_xcpt && wb_reg_cause === Causes.fetch_guest_page_fault.U + val htval_imem = Mux(htval_valid_imem, io.imem.gpa.bits, 0.U) + assert(!htval_valid_imem || io.imem.gpa.valid) + + val htval_valid_dmem = + wb_xcpt && tval_dmem_addr && io.dmem.s2_xcpt.gf.asUInt.orR && !io.dmem.s2_xcpt.pf.asUInt.orR + val htval_dmem = Mux(htval_valid_dmem, io.dmem.s2_gpa, 0.U) + + val htval = (htval_dmem | htval_imem) >> hypervisorExtraAddrBits + // read pseudoinstruction if a guest-page fault is caused by an implicit memory access for VS-stage address + // translation + val mhtinst_read_pseudo = (io.imem.gpa_is_pte && htval_valid_imem) || (io.dmem.s2_gpa_is_pte && htval_valid_dmem) + (htval, mhtinst_read_pseudo) + } + + csr.io.vector.foreach { v => + v.set_vconfig.valid := wb_reg_set_vconfig && wb_reg_valid + v.set_vconfig.bits := wb_reg_rs2.asTypeOf(new VConfig) + v.set_vs_dirty := wb_valid && wb_ctrl.vec + v.set_vstart.valid := wb_valid && wb_reg_set_vconfig + v.set_vstart.bits := 0.U + } + + io.vector.foreach { v => + when(v.wb.retire || v.wb.xcpt || wb_ctrl.vec) { + csr.io.pc := v.wb.pc + csr.io.retire := v.wb.retire + csr.io.inst(0) := v.wb.inst + when(v.wb.xcpt && !wb_reg_xcpt) { + wb_xcpt := true.B + wb_cause := v.wb.cause + csr.io.tval := v.wb.tval + } + } + v.wb.store_pending := io.dmem.store_pending + v.wb.vxrm := csr.io.vector.get.vxrm + v.wb.frm := csr.io.fcsr_rm + csr.io.vector.get.set_vxsat := v.set_vxsat + when(v.set_vconfig.valid) { + csr.io.vector.get.set_vconfig.valid := true.B + csr.io.vector.get.set_vconfig.bits := v.set_vconfig.bits + } + when(v.set_vstart.valid) { + csr.io.vector.get.set_vstart.valid := true.B + csr.io.vector.get.set_vstart.bits := v.set_vstart.bits + } + } + + csr.io.htval := htval + csr.io.mhtinst_read_pseudo := mhtinst_read_pseudo + io.ptw.ptbr := csr.io.ptbr + io.ptw.hgatp := csr.io.hgatp + io.ptw.vsatp := csr.io.vsatp + (io.ptw.customCSRs.csrs zip csr.io.customCSRs).map { case (lhs, rhs) => lhs <> rhs } + io.ptw.status := csr.io.status + io.ptw.hstatus := csr.io.hstatus + io.ptw.gstatus := csr.io.gstatus + io.ptw.pmp := csr.io.pmp + csr.io.rw.addr := wb_reg_inst(31, 20) + csr.io.rw.cmd := CSR.maskCmd(wb_reg_valid, wb_ctrl.csr) + csr.io.rw.wdata := wb_reg_wdata + + io.rocc.csrs <> csr.io.roccCSRs + for ((port, source) <- io.rocc.csrs.zip(csr.io.roccCSRs)) { + port.ren := source.ren && wb_reg_valid && !wb_xcpt && !replay_wb_common + } + io.trace.time := csr.io.time + io.trace.insns := csr.io.trace + if (rocketParams.debugROB.isDefined) { + val sz = rocketParams.debugROB.get.size + if (sz < 1) { // use unsynthesizable ROB + val csr_trace_with_wdata = WireInit(csr.io.trace(0)) + csr_trace_with_wdata.wdata.get := rf_wdata + val should_wb = WireInit((wb_ctrl.wfd || (wb_ctrl.wxd && wb_waddr =/= 0.U)) && !csr.io.trace(0).exception) + val has_wb = WireInit(wb_ctrl.wxd && wb_wen && !wb_set_sboard) + val wb_addr = WireInit(wb_waddr + Mux(wb_ctrl.wfd, 32.U, 0.U)) + + io.vector.foreach { v => + when(v.wb.retire) { + should_wb := v.wb.rob_should_wb + has_wb := false.B + wb_addr := Cat(v.wb.rob_should_wb_fp, csr_trace_with_wdata.insn(11, 7)) + } + } + + DebugROB.pushTrace(clock, reset, io.hartid, csr_trace_with_wdata, should_wb, has_wb, wb_addr) + + io.trace.insns(0) := DebugROB.popTrace(clock, reset, io.hartid) + + DebugROB.pushWb(clock, reset, io.hartid, ll_wen, rf_waddr, rf_wdata) + } else { // synthesizable ROB (no FPRs) + require(!usingVector, "Synthesizable ROB does not support vector implementations") + val csr_trace_with_wdata = WireInit(csr.io.trace(0)) + csr_trace_with_wdata.wdata.get := rf_wdata + + val debug_rob = Module(new HardDebugROB(sz, 32)) + debug_rob.io.i_insn := csr_trace_with_wdata + debug_rob.io.should_wb := (wb_ctrl.wfd || (wb_ctrl.wxd && wb_waddr =/= 0.U)) && + !csr.io.trace(0).exception + debug_rob.io.has_wb := wb_ctrl.wxd && wb_wen && !wb_set_sboard + debug_rob.io.tag := wb_waddr + Mux(wb_ctrl.wfd, 32.U, 0.U) + + debug_rob.io.wb_val := ll_wen + debug_rob.io.wb_tag := rf_waddr + debug_rob.io.wb_data := rf_wdata + + io.trace.insns(0) := debug_rob.io.o_insn + } + } else { + io.trace.insns := csr.io.trace + } + for (((iobpw, wphit), bp) <- io.bpwatch zip wb_reg_wphit zip csr.io.bp) { + iobpw.valid(0) := wphit + iobpw.action := bp.control.action + // tie off bpwatch valids + iobpw.rvalid.foreach(_ := false.B) + iobpw.wvalid.foreach(_ := false.B) + iobpw.ivalid.foreach(_ := false.B) + } + + val hazard_targets = Seq( + (id_ctrl.rxs1 && id_raddr1 =/= 0.U, id_raddr1), + (id_ctrl.rxs2 && id_raddr2 =/= 0.U, id_raddr2), + (id_ctrl.wxd && id_waddr =/= 0.U, id_waddr) + ) + + val fp_hazard_targets = Seq( + (io.fpu.dec.ren1, id_raddr1), + (io.fpu.dec.ren2, id_raddr2), + (io.fpu.dec.ren3, id_raddr3), + (io.fpu.dec.wen, id_waddr) + ) + + val sboard = new Scoreboard(32, true) + sboard.clear(ll_wen, ll_waddr) + + def id_sboard_clear_bypass(r: UInt) = + // ll_waddr arrives late when D$ has ECC, so reshuffle the hazard check + if (!tileParams.dcache.get.dataECC.isDefined) ll_wen && ll_waddr === r + else + div.io.resp.fire && div.io.resp.bits.tag === r || dmem_resp_replay && dmem_resp_xpu && dmem_resp_waddr === r + + val id_sboard_hazard = checkHazards(hazard_targets, rd => sboard.read(rd) && !id_sboard_clear_bypass(rd)) + sboard.set(wb_set_sboard && wb_wen, wb_waddr) + + // stall for RAW/WAW hazards on CSRs, loads, AMOs, and mul/div in execute stage. + val ex_cannot_bypass = + ex_ctrl.csr =/= CSR.N || ex_ctrl.jalr || ex_ctrl.mem || ex_ctrl.mul || ex_ctrl.div || ex_ctrl.fp || ex_ctrl.rocc || ex_ctrl.vec + val data_hazard_ex = ex_ctrl.wxd && checkHazards(hazard_targets, _ === ex_waddr) + val fp_data_hazard_ex = id_ctrl.fp && ex_ctrl.wfd && checkHazards(fp_hazard_targets, _ === ex_waddr) + val id_ex_hazard = ex_reg_valid && (data_hazard_ex && ex_cannot_bypass || fp_data_hazard_ex) + + // stall for RAW/WAW hazards on CSRs, LB/LH, and mul/div in memory stage. + val mem_mem_cmd_bh = + if (fastLoadWord) (!fastLoadByte).B && mem_reg_slow_bypass + else true.B + + val mem_cannot_bypass = + mem_ctrl.csr =/= CSR.N || mem_ctrl.mem && mem_mem_cmd_bh || mem_ctrl.mul || mem_ctrl.div || mem_ctrl.fp || mem_ctrl.rocc || mem_ctrl.vec + val data_hazard_mem = mem_ctrl.wxd && checkHazards(hazard_targets, _ === mem_waddr) + val fp_data_hazard_mem = id_ctrl.fp && mem_ctrl.wfd && checkHazards(fp_hazard_targets, _ === mem_waddr) + val id_mem_hazard = mem_reg_valid && (data_hazard_mem && mem_cannot_bypass || fp_data_hazard_mem) + id_load_use := mem_reg_valid && data_hazard_mem && mem_ctrl.mem + + val id_vconfig_hazard = id_ctrl.vec && ( + (ex_reg_valid && ex_reg_set_vconfig) || + (mem_reg_valid && mem_reg_set_vconfig) || + (wb_reg_valid && wb_reg_set_vconfig) + ) + + // stall for RAW/WAW hazards on load/AMO misses and mul/div in writeback. + val data_hazard_wb = wb_ctrl.wxd && checkHazards(hazard_targets, _ === wb_waddr) + val fp_data_hazard_wb = id_ctrl.fp && wb_ctrl.wfd && checkHazards(fp_hazard_targets, _ === wb_waddr) + val id_wb_hazard = wb_reg_valid && (data_hazard_wb && wb_set_sboard || fp_data_hazard_wb) + + val id_stall_fpu = + if (usingFPU) { + val fp_sboard = new Scoreboard(32) + fp_sboard.set(((wb_dcache_miss || wb_ctrl.vec) && wb_ctrl.wfd || io.fpu.sboard_set) && wb_valid, wb_waddr) + val v_ll = io.vector.map(v => v.resp.fire && v.resp.bits.fp).getOrElse(false.B) + fp_sboard.clear((dmem_resp_replay && dmem_resp_fpu) || v_ll, io.fpu.ll_resp_tag) + fp_sboard.clear(io.fpu.sboard_clr, io.fpu.sboard_clra) + + checkHazards(fp_hazard_targets, fp_sboard.read _) + } else false.B + + val dcache_blocked = { + // speculate that a blocked D$ will unblock the cycle after a Grant + val blocked = Reg(Bool()) + blocked := !io.dmem.req.ready && io.dmem.clock_enabled && !io.dmem.perf.grant && (blocked || io.dmem.req.valid || io.dmem.s2_nack) + blocked && !io.dmem.perf.grant + } + + val rocc_blocked = RegInit(false.B) + rocc_blocked := !wb_xcpt && !io.rocc.cmd.ready && (io.rocc.cmd.valid || rocc_blocked) + + val ctrl_stalld = + id_ex_hazard || id_mem_hazard || id_wb_hazard || id_sboard_hazard || + id_vconfig_hazard || + csr.io.singleStep && (ex_reg_valid || mem_reg_valid || wb_reg_valid) || + id_csr_en && csr.io.decode(0).fp_csr && !io.fpu.fcsr_rdy || + id_csr_en && csr.io.decode(0).vector_csr && id_vec_busy || + id_ctrl.fp && id_stall_fpu || + id_ctrl.mem && (dcache_blocked || id_rocc_busy) || // data accesses follow posted coprocessor work + id_ctrl.rocc && (rocc_blocked || id_mem_busy) || // prior CPU memory requests complete before submission + id_ctrl.div && (!(div.io.req.ready || (div.io.resp.valid && !wb_wxd)) || div.io.req.valid) || // reduce odds of replay + !clock_en || + id_do_fence || + csr.io.csr_stall || + id_reg_pause || + io.traceStall + + ctrl_killd := !ibuf.io.inst(0).valid || ibuf.io.inst( + 0 + ).bits.replay || take_pc_mem_wb || ctrl_stalld || deliveredInterrupt + + io.imem.req.valid := take_pc + io.imem.req.bits.speculative := !take_pc_wb + io.imem.req.bits.pc := + Mux( + wb_xcpt || csr.io.eret, + csr.io.evec, // exception or [m|s]ret + Mux( + replay_wb, + wb_reg_pc, // replay + mem_npc + ) + ) // flush or branch misprediction + io.imem.flush_icache := wb_reg_valid && wb_ctrl.fence_i && !io.dmem.s2_nack + io.imem.might_request := { + imem_might_request_reg := ex_pc_valid || mem_pc_valid || io.ptw.customCSRs.disableICacheClockGate || io.vector.map( + _.trap_check_busy + ).getOrElse(false.B) + imem_might_request_reg + } + io.imem.progress := RegNext(wb_reg_valid && !replay_wb_common) + io.imem.sfence.valid := wb_reg_valid && wb_reg_sfence + io.imem.sfence.bits.rs1 := wb_reg_mem_size(0) + io.imem.sfence.bits.rs2 := wb_reg_mem_size(1) + io.imem.sfence.bits.addr := wb_reg_wdata + io.imem.sfence.bits.asid := wb_reg_rs2 + io.imem.sfence.bits.hv := wb_reg_hfence_v + io.imem.sfence.bits.hg := wb_reg_hfence_g + io.ptw.sfence := io.imem.sfence + + ibuf.io.inst(0).ready := !ctrl_stalld + + io.imem.btb_update.valid := mem_reg_valid && !take_pc_wb && mem_wrong_npc && (!mem_cfi || mem_cfi_taken) + io.imem.btb_update.bits.isValid := mem_cfi + io.imem.btb_update.bits.cfiType := + Mux( + (mem_ctrl.jal || mem_ctrl.jalr) && mem_waddr(0), + CFIType.call, + Mux( + mem_ctrl.jalr && (mem_reg_inst(19, 15) & regAddrMask.U) === BitPat("b00?01"), + CFIType.ret, + Mux(mem_ctrl.jal || mem_ctrl.jalr, CFIType.jump, CFIType.branch) + ) + ) + io.imem.btb_update.bits.target := io.imem.req.bits.pc + io.imem.btb_update.bits.br_pc := (if (usingCompressed) mem_reg_pc + Mux(mem_reg_rvc, 0.U, 2.U) else mem_reg_pc) + io.imem.btb_update.bits.pc := ~(~io.imem.btb_update.bits.br_pc | (coreInstBytes * fetchWidth - 1).U) + io.imem.btb_update.bits.prediction := mem_reg_btb_resp + io.imem.btb_update.bits.taken := DontCare + + io.imem.bht_update.valid := mem_reg_valid && !take_pc_wb + io.imem.bht_update.bits.pc := io.imem.btb_update.bits.pc + io.imem.bht_update.bits.taken := mem_br_taken + io.imem.bht_update.bits.mispredict := mem_wrong_npc + io.imem.bht_update.bits.branch := mem_ctrl.branch + io.imem.bht_update.bits.prediction := mem_reg_btb_resp.bht + + // Connect RAS in Frontend + io.imem.ras_update := DontCare + + io.fpu.valid := !ctrl_killd && id_ctrl.fp + io.fpu.killx := ctrl_killx + io.fpu.killm := killm_common + io.fpu.inst := id_inst(0) + io.fpu.fromint_data := ex_rs(0) + io.fpu.ll_resp_val := dmem_resp_valid && dmem_resp_fpu + io.fpu.ll_resp_data := (if (minFLen == 32) io.dmem.resp.bits.data_word_bypass else io.dmem.resp.bits.data) + io.fpu.ll_resp_type := io.dmem.resp.bits.size + io.fpu.ll_resp_tag := dmem_resp_waddr + io.fpu.keep_clock_enabled := io.ptw.customCSRs.disableCoreClockGate + + io.fpu.v_sew := csr.io.vector.map(_.vconfig.vtype.vsew).getOrElse(0.U) + + io.vector.map { v => + when(!(dmem_resp_valid && dmem_resp_fpu)) { + io.fpu.ll_resp_val := v.resp.valid && v.resp.bits.fp + io.fpu.ll_resp_data := v.resp.bits.data + io.fpu.ll_resp_type := v.resp.bits.size + io.fpu.ll_resp_tag := v.resp.bits.rd + } + } + + io.vector.foreach { v => + v.ex.valid := ex_reg_valid && (ex_ctrl.vec || rocketParams.vector.get.issueVConfig.B && ex_reg_set_vconfig) && !ctrl_killx + v.ex.inst := ex_reg_inst + v.ex.vconfig := csr.io.vector.get.vconfig + v.ex.vstart := Mux(mem_reg_valid && mem_ctrl.vec || wb_reg_valid && wb_ctrl.vec, 0.U, csr.io.vector.get.vstart) + v.ex.rs1 := ex_rs(0) + v.ex.rs2 := ex_rs(1) + v.ex.pc := ex_reg_pc + v.mem.frs1 := io.fpu.store_data + v.killm := killm_common + v.status := csr.io.status + } + + io.dmemPc := ex_reg_pc + io.dmem.req.valid := ex_reg_valid && ex_ctrl.mem + val ex_dcache_tag = Cat(ex_waddr, ex_ctrl.fp) + require(coreParams.dcacheReqTagBits >= ex_dcache_tag.getWidth) + io.dmem.req.bits.tag := ex_dcache_tag + io.dmem.req.bits.cmd := ex_ctrl.mem_cmd + io.dmem.req.bits.size := ex_reg_mem_size + io.dmem.req.bits.signed := !Mux(ex_reg_hls, ex_reg_inst(20), ex_reg_inst(14)) + io.dmem.req.bits.phys := false.B + io.dmem.req.bits.addr := encodeVirtualAddress(ex_rs(0), alu.io.adder_out) + io.dmem.req.bits.idx.foreach(_ := io.dmem.req.bits.addr) + io.dmem.req.bits.dprv := Mux(ex_reg_hls, csr.io.hstatus.spvp, csr.io.status.dprv) + io.dmem.req.bits.dv := ex_reg_hls || csr.io.status.dv + io.dmem.req.bits.no_resp := !isRead(ex_ctrl.mem_cmd) || (!ex_ctrl.fp && ex_waddr === 0.U) + io.dmem.req.bits.no_alloc := DontCare + io.dmem.req.bits.no_xcpt := DontCare + io.dmem.req.bits.data := DontCare + io.dmem.req.bits.mask := DontCare + + io.dmem.s1_data.data := (if (fLen == 0) mem_reg_rs2 + else Mux(mem_ctrl.fp, Fill(coreDataBits / fLen, io.fpu.store_data), mem_reg_rs2)) + io.dmem.s1_data.mask := DontCare + + io.dmem.s1_kill := killm_common || mem_ldst_xcpt || fpu_kill_mem || vec_kill_mem + io.dmem.s2_kill := false.B + // don't let D$ go to sleep if we're probably going to use it soon + io.dmem.keep_clock_enabled := ibuf.io.inst(0).valid && id_ctrl.mem && !csr.io.csr_stall + + // Pipeline classification, not a second accelerator decoder. Readiness is deliberately absent. + io.roccOlderPending := usingRoCC.B && + ((mem_reg_valid && mem_ctrl.rocc) || (wb_reg_valid && wb_ctrl.rocc)) + io.roccCommitEligible := usingRoCC.B && wb_reg_valid && wb_ctrl.rocc && !replay_wb_common && !wb_xcpt + io.rocc.cmd.valid := wb_reg_valid && wb_ctrl.rocc && !replay_wb_common + io.rocc.exception := wb_xcpt && csr.io.status.xs.orR + io.rocc.cmd.bits.raw_inst := wb_reg_inst + io.rocc.cmd.bits.funct := wb_reg_inst(31, 25) + io.rocc.cmd.bits.funct3 := wb_reg_inst(14, 12) + io.rocc.cmd.bits.rs2 := wb_reg_inst(24, 20) + io.rocc.cmd.bits.rs1 := wb_reg_inst(19, 15) + io.rocc.cmd.bits.pc := wb_reg_pc + io.rocc.cmd.bits.xd := wb_reg_inst(14) + io.rocc.cmd.bits.xs1 := wb_reg_inst(13) + io.rocc.cmd.bits.xs2 := wb_reg_inst(12) + io.rocc.cmd.bits.rd := wb_reg_inst(11, 7) + io.rocc.cmd.bits.opcode := wb_reg_inst(6, 0) + io.rocc.cmd.bits.rs1Data := wb_reg_wdata + io.rocc.cmd.bits.rs2Data := wb_reg_rs2 + + // gate the clock + val unpause = + csr.io.time( + rocketParams.lgPauseCycles - 1, + 0 + ) === 0.U || csr.io.inhibit_cycle || io.dmem.perf.release || take_pc + + when(unpause)(id_reg_pause := false.B) + io.cease := csr.io.status.cease && !clock_en_reg + io.wfi := csr.io.status.wfi + if (rocketParams.clockGate) { + long_latency_stall := csr.io.csr_stall || io.dmem.perf.blocked || id_reg_pause && !unpause + clock_en := clock_en_reg || ex_pc_valid || (!long_latency_stall && io.imem.resp.valid) + clock_en_reg := + ex_pc_valid || mem_pc_valid || wb_pc_valid || // instruction in flight + io.ptw.customCSRs.disableCoreClockGate || // chicken bit + !div.io.req.ready || // mul/div in flight + usingFPU.B && !io.fpu.fcsr_rdy || // long-latency FPU in flight + io.dmem.replay_next || // long-latency load replaying + (!long_latency_stall && (ibuf.io.inst(0).valid || io.imem.resp.valid)) // instruction pending + + assert(!(ex_pc_valid || mem_pc_valid || wb_pc_valid) || clock_en) + } + + // evaluate performance counters + val icache_blocked = !(io.imem.resp.valid || RegNext(io.imem.resp.valid)) + csr.io.counters foreach { c => c.inc := RegNext(perfEvents.evaluate(c.eventSel)) } + + val coreMonitorBundle = Wire(new CoreMonitorBundle(xLen, fLen)) + + coreMonitorBundle.clock := clock + coreMonitorBundle.reset := reset + coreMonitorBundle.hartid := io.hartid + coreMonitorBundle.timer := csr.io.time(31, 0) + coreMonitorBundle.valid := csr.io.trace(0).valid && !csr.io.trace(0).exception + coreMonitorBundle.pc := csr.io.trace(0).iaddr(vaddrBitsExtended - 1, 0).sextTo(xLen) + coreMonitorBundle.wrenx := wb_wen && !wb_set_sboard + coreMonitorBundle.wrenf := false.B + coreMonitorBundle.wrdst := wb_waddr + coreMonitorBundle.wrdata := rf_wdata + coreMonitorBundle.rd0src := wb_reg_inst(19, 15) + coreMonitorBundle.rd0val := RegNext(RegNext(ex_rs(0))) + coreMonitorBundle.rd1src := wb_reg_inst(24, 20) + coreMonitorBundle.rd1val := RegNext(RegNext(ex_rs(1))) + coreMonitorBundle.inst := csr.io.trace(0).insn + coreMonitorBundle.excpt := csr.io.trace(0).exception + coreMonitorBundle.priv_mode := csr.io.trace(0).priv + + if (enableCommitLog) { + val t = csr.io.trace(0) + val rd = wb_waddr + val wfd = wb_ctrl.wfd + val wxd = wb_ctrl.wxd + val has_data = wb_wen && !wb_set_sboard + + when(t.valid && !t.exception) { + when(wfd) { + printf("%d 0x%x (0x%x) f%d p%d 0xXXXXXXXXXXXXXXXX\n", t.priv, t.iaddr, t.insn, rd, rd + 32.U) + } + .elsewhen(wxd && rd =/= 0.U && has_data) { + printf("%d 0x%x (0x%x) x%d 0x%x\n", t.priv, t.iaddr, t.insn, rd, rf_wdata) + } + .elsewhen(wxd && rd =/= 0.U && !has_data) { + printf("%d 0x%x (0x%x) x%d p%d 0xXXXXXXXXXXXXXXXX\n", t.priv, t.iaddr, t.insn, rd, rd) + } + .otherwise { + printf("%d 0x%x (0x%x)\n", t.priv, t.iaddr, t.insn) + } + } + + when(ll_wen && rf_waddr =/= 0.U) { + printf("x%d p%d 0x%x\n", rf_waddr, rf_waddr, rf_wdata) + } + } else { + when(csr.io.trace(0).valid) { + printf( + "C%d: %d [%d] pc=[%x] W[r%d=%x][%d] R[r%d=%x] R[r%d=%x] inst=[%x] DASM(%x) wb_xcpt:%d\n", + io.hartid, + coreMonitorBundle.timer, + coreMonitorBundle.valid, + coreMonitorBundle.pc, + Mux(wb_ctrl.wxd || wb_ctrl.wfd, coreMonitorBundle.wrdst, 0.U), + Mux(coreMonitorBundle.wrenx, coreMonitorBundle.wrdata, 0.U), + coreMonitorBundle.wrenx, + Mux(wb_ctrl.rxs1 || wb_ctrl.rfs1, coreMonitorBundle.rd0src, 0.U), + Mux(wb_ctrl.rxs1 || wb_ctrl.rfs1, coreMonitorBundle.rd0val, 0.U), + Mux(wb_ctrl.rxs2 || wb_ctrl.rfs2, coreMonitorBundle.rd1src, 0.U), + Mux(wb_ctrl.rxs2 || wb_ctrl.rfs2, coreMonitorBundle.rd1val, 0.U), + coreMonitorBundle.inst, + coreMonitorBundle.inst, + wb_xcpt + ) + } + } + + // CoreMonitorBundle for late latency writes + val xrfWriteBundle = Wire(new CoreMonitorBundle(xLen, fLen)) + + xrfWriteBundle.clock := clock + xrfWriteBundle.reset := reset + xrfWriteBundle.hartid := io.hartid + xrfWriteBundle.timer := csr.io.time(31, 0) + xrfWriteBundle.valid := false.B + xrfWriteBundle.pc := 0.U + xrfWriteBundle.wrdst := rf_waddr + xrfWriteBundle.wrenx := rf_wen && !(csr.io.trace(0).valid && wb_wen && (wb_waddr === rf_waddr)) + xrfWriteBundle.wrenf := false.B + xrfWriteBundle.wrdata := rf_wdata + xrfWriteBundle.rd0src := 0.U + xrfWriteBundle.rd0val := 0.U + xrfWriteBundle.rd1src := 0.U + xrfWriteBundle.rd1val := 0.U + xrfWriteBundle.inst := 0.U + xrfWriteBundle.excpt := false.B + xrfWriteBundle.priv_mode := csr.io.trace(0).priv + + if (rocketParams.haveSimTimeout) + PlusArg.timeout( + name = "max_core_cycles", + docstring = "Kill the emulation after INT rdtime cycles. Off if 0." + )(csr.io.time) + + } // leaving gated-clock domain + val rocketImpl = withClock(gated_clock)(new RocketImpl) + + def checkExceptions(x: Seq[(Bool, UInt)]) = + (WireInit(x.map(_._1).reduce(_ || _)), WireInit(PriorityMux(x))) + + def coverExceptions( + exceptionValid: Bool, + cause: UInt, + labelPrefix: String, + coverCausesLabels: Seq[(Int, String)] + ): Unit = { + for ((coverCause, label) <- coverCausesLabels) { + property.cover(exceptionValid && (cause === coverCause.U), s"${labelPrefix}_$label") + } + } + + def checkHazards(targets: Seq[(Bool, UInt)], cond: UInt => Bool) = + targets.map(h => h._1 && cond(h._2)).reduce(_ || _) + + def encodeVirtualAddress(a0: UInt, ea: UInt) = + if (vaddrBitsExtended == vaddrBits) ea + else { + // efficient means to compress 64-bit VA into vaddrBits+1 bits + // (VA is bad if VA(vaddrBits) != VA(vaddrBits-1)) + val b = vaddrBitsExtended - 1 + val a = (a0 >> b).asSInt + val msb = Mux(a === 0.S || a === -1.S, ea(b), !ea(b - 1)) + Cat(msb, ea(b - 1, 0)) + } + + class Scoreboard(n: Int, zero: Boolean = false) { + def set(en: Bool, addr: UInt): Unit = update(en, _next | mask(en, addr)) + def clear(en: Bool, addr: UInt): Unit = update(en, _next & ~mask(en, addr)) + def read(addr: UInt): Bool = r(addr) + def readBypassed(addr: UInt): Bool = _next(addr) + + val _r = RegInit(0.U(n.W)) + val r = if (zero) (_r >> 1 << 1) else _r + private var _next = r + private var ens = false.B + def mask(en: Bool, addr: UInt) = Mux(en, 1.U << addr, 0.U) + + def update(en: Bool, update: UInt) = { + _next = update + ens = ens || en + when(ens)(_r := _next) + } + + } + + } + + class RegFile(n: Int, w: Int, zero: Boolean = false) { + val rf = Reg(Vec(n, UInt(w.W))) + def access(addr: UInt) = rf(~addr(log2Up(n) - 1, 0)) + val reads = ArrayBuffer[(UInt, UInt)]() + private var canRead = true + + def read(addr: UInt) = { + require(canRead) + reads += addr -> Wire(UInt()) + reads.last._2 := Mux(zero.B && addr === 0.U, 0.U, access(addr)) + reads.last._2 + } + + def write(addr: UInt, data: UInt) = { + canRead = false + when(addr =/= 0.U) { + access(addr) := data + for ((raddr, rdata) <- reads) + when(addr === raddr)(rdata := data) + } + } + + } + + object ImmGen { + + def apply(sel: UInt, inst: UInt) = { + val sign = Mux(sel === IMM_Z, 0.S, inst(31).asSInt) + val b30_20 = Mux(sel === IMM_U, inst(30, 20).asSInt, sign) + val b19_12 = Mux(sel =/= IMM_U && sel =/= IMM_UJ, sign, inst(19, 12).asSInt) + val b11 = Mux( + sel === IMM_U || sel === IMM_Z, + 0.S, + Mux(sel === IMM_UJ, inst(20).asSInt, Mux(sel === IMM_SB, inst(7).asSInt, sign)) + ) + val b10_5 = Mux(sel === IMM_U || sel === IMM_Z, 0.U, inst(30, 25)) + val b4_1 = Mux( + sel === IMM_U, + 0.U, + Mux(sel === IMM_S || sel === IMM_SB, inst(11, 8), Mux(sel === IMM_Z, inst(19, 16), inst(24, 21))) + ) + val b0 = Mux(sel === IMM_S, inst(7), Mux(sel === IMM_I, inst(20), Mux(sel === IMM_Z, inst(15), 0.U))) + + Cat(sign, b30_20, b19_12, b11, b10_5, b4_1, b0).asSInt + } + + } + +} + +// See LICENSE.Berkeley and LICENSE.SiFive for the Rocket interface definitions. +package freechips.rocketchip.rocket { + + import chisel3._ + import chisel3.util._ + import org.chipsalliance.cde.config.Parameters + import freechips.rocketchip.tile.{CoreBundle, HasCoreParameters, TileInterrupts} + + /** CPU-facing interfaces, independent of the upstream cache and TileLink implementations. */ + class FrontendReq(implicit p: Parameters) extends CoreBundle()(p) { + val pc = UInt(vaddrBitsExtended.W) + val speculative = Bool() + } + + class FrontendExceptions extends Bundle { + val pf = new Bundle { val inst = Bool() } + val gf = new Bundle { val inst = Bool() } + val ae = new Bundle { val inst = Bool() } + } + + class FrontendResp(implicit p: Parameters) extends CoreBundle()(p) { + val btb = new BTBResp + val pc = UInt(vaddrBitsExtended.W) + val data = UInt((fetchWidth * coreInstBits).W) + val mask = Bits(fetchWidth.W) + val xcpt = new FrontendExceptions + val replay = Bool() + } + + class FrontendPerfEvents extends Bundle { + val acquire = Bool() + val tlbMiss = Bool() + } + + class SFenceReq(implicit p: Parameters) extends CoreBundle()(p) { + val rs1 = Bool() + val rs2 = Bool() + val addr = UInt(vaddrBits.W) + val asid = UInt((asIdBits max 1).W) + val hv = Bool() + val hg = Bool() + } + + class FrontendIO(implicit p: Parameters) extends CoreBundle()(p) { + val might_request = Output(Bool()) + val clock_enabled = Input(Bool()) + val req = Valid(new FrontendReq) + val sfence = Valid(new SFenceReq) + val resp = Flipped(Decoupled(new FrontendResp)) + val gpa = Flipped(Valid(UInt(vaddrBitsExtended.W))) + val gpa_is_pte = Input(Bool()) + val btb_update = Valid(new BTBUpdate) + val bht_update = Valid(new BHTUpdate) + val ras_update = Valid(new RASUpdate) + val flush_icache = Output(Bool()) + val npc = Input(UInt(vaddrBitsExtended.W)) + val perf = Input(new FrontendPerfEvents) + val progress = Output(Bool()) + } + + trait HasCoreMemOp extends HasCoreParameters { + val addr = UInt(coreMaxAddrBits.W) + + val idx = Option.when(usingVM && lgCacheBlockBytes + log2Up(tileParams.dcache.get.nSets) > pgIdxBits)( + UInt(coreMaxAddrBits.W) + ) + + val tag = UInt((coreParams.dcacheReqTagBits + log2Ceil(dcacheArbPorts)).W) + val cmd = UInt(M_SZ.W) + val size = UInt(log2Ceil(log2Ceil(coreDataBytes) + 1).W) + val signed = Bool() + val dprv = UInt(PRV.SZ.W) + val dv = Bool() + } + + trait HasCoreData extends HasCoreParameters { + val data = UInt(coreDataBits.W) + val mask = UInt(coreDataBytes.W) + } + + class HellaCacheReqInternal(implicit p: Parameters) extends CoreBundle()(p) with HasCoreMemOp { + val phys = Bool() + val no_resp = Bool() + val no_alloc = Bool() + val no_xcpt = Bool() + } + + class HellaCacheReq(implicit p: Parameters) extends HellaCacheReqInternal()(p) with HasCoreData + + class HellaCacheResp(implicit p: Parameters) extends CoreBundle()(p) with HasCoreMemOp with HasCoreData { + val replay = Bool() + val has_data = Bool() + val data_word_bypass = UInt(coreDataBits.W) + val data_raw = UInt(coreDataBits.W) + val store_data = UInt(coreDataBits.W) + } + + class AlignmentExceptions extends Bundle { + val ld = Bool() + val st = Bool() + } + + class HellaCacheExceptions extends Bundle { + val ma = new AlignmentExceptions + val pf = new AlignmentExceptions + val gf = new AlignmentExceptions + val ae = new AlignmentExceptions + } + + class HellaCacheWriteData(implicit p: Parameters) extends CoreBundle()(p) with HasCoreData + + class HellaCachePerfEvents extends Bundle { + val acquire = Bool() + val release = Bool() + val grant = Bool() + val tlbMiss = Bool() + val blocked = Bool() + val canAcceptStoreThenLoad = Bool() + val canAcceptStoreThenRMW = Bool() + val canAcceptLoadThenLoad = Bool() + val storeBufferEmptyAfterLoad = Bool() + val storeBufferEmptyAfterStore = Bool() + } + + class HellaCacheIO(implicit p: Parameters) extends CoreBundle()(p) { + val req = Decoupled(new HellaCacheReq) + val s1_kill = Output(Bool()) + val s1_data = Output(new HellaCacheWriteData) + val s2_nack = Input(Bool()) + val s2_nack_cause_raw = Input(Bool()) + val s2_kill = Output(Bool()) + val s2_uncached = Input(Bool()) + val s2_paddr = Input(UInt(paddrBits.W)) + val resp = Flipped(Valid(new HellaCacheResp)) + val replay_next = Input(Bool()) + val s2_xcpt = Input(new HellaCacheExceptions) + val s2_gpa = Input(UInt(vaddrBitsExtended.W)) + val s2_gpa_is_pte = Input(Bool()) + val uncached_resp = Option.when(tileParams.dcache.get.separateUncachedResp)(Flipped(Decoupled(new HellaCacheResp))) + val ordered = Input(Bool()) + val store_pending = Input(Bool()) + val perf = Input(new HellaCachePerfEvents) + val keep_clock_enabled = Output(Bool()) + val clock_enabled = Input(Bool()) + } + + class PTWPerfEvents extends Bundle { + val l2miss = Bool() + val l2hit = Bool() + val pte_miss = Bool() + val pte_hit = Bool() + } + + class DatapathPTWIO(implicit p: Parameters) extends CoreBundle()(p) { + val ptbr = Input(new PTBR) + val hgatp = Input(new PTBR) + val vsatp = Input(new PTBR) + val sfence = Flipped(Valid(new SFenceReq)) + val status = Input(new MStatus) + val hstatus = Input(new HStatus) + val gstatus = Input(new MStatus) + val pmp = Input(Vec(nPMPs, new PMP)) + val perf = Output(new PTWPerfEvents) + val customCSRs = Flipped(coreParams.customCSRs) + val clock_enabled = Output(Bool()) + } + + class CoreInterrupts(val hasBeu: Boolean)(implicit p: Parameters) extends TileInterrupts()(p) { + val buserror = Option.when(hasBeu)(Bool()) + } + +} + +package examples.core.rocket.pipeline { + + import chisel3._ + import chisel3.util._ + + /** RoCC response bundle */ + class RoCCResponseBB(xLen: Int = 64) extends Bundle { + val rd = Bits(5.W) + val data = Bits(xLen.W) + } + + /** RoCC interface between a core and an accelerator. */ + class RoCCIO(xLen: Int = 64) extends Bundle { + val cmd = Flipped(Decoupled(new RoCCCommandBB(xLen))) + val resp = Decoupled(new RoCCResponseBB(xLen)) + val busy = Output(Bool()) + val interrupt = Output(Bool()) + val exception = Input(Bool()) + } + + /** RoCC command bundle */ + class RoCCCommandBB(xLen: Int = 64) extends Bundle { + val raw_inst = UInt(32.W) + val pc = UInt(xLen.W) + val funct = UInt(7.W) + val funct3 = UInt(3.W) + val rs2 = Bits(5.W) + val rs1 = Bits(5.W) + val xd = Bool() + val xs1 = Bool() + val xs2 = Bool() + val rd = Bits(5.W) + val opcode = UInt(7.W) + val rs1Data = UInt(xLen.W) + val rs2Data = UInt(xLen.W) + } + +} + +// See LICENSE.SiFive for the Rocket interface definitions. +package freechips.rocketchip.tile { + + import chisel3._ + import org.chipsalliance.cde.config.Parameters + + class NMI(val w: Int) extends Bundle { + val rnmi = Bool() + val rnmi_interrupt_vector = UInt(w.W) + val rnmi_exception_vector = UInt(w.W) + } + + class TileInterrupts(implicit p: Parameters) extends CoreBundle()(p) { + val debug = Bool() + val mtip = Bool() + val msip = Bool() + val meip = Bool() + val seip = Option.when(usingSupervisor)(Bool()) + val lip = Vec(coreParams.nLocalInterrupts, Bool()) + val nmi = Option.when(usingNMI)(new NMI(resetVectorLen)) + } + +} + +// See LICENSE.Berkeley for Rocket's CPU/vector interface contract. +package freechips.rocketchip.rocket { + + import chisel3._ + import chisel3.util._ + import org.chipsalliance.cde.config.Parameters + import freechips.rocketchip.tile._ + + class VectorCoreIO(implicit p: Parameters) extends CoreBundle { + val status = Input(new MStatus) + + val ex = new Bundle { + val valid = Input(Bool()); val ready = Output(Bool()); val inst = Input(UInt(32.W)) + val pc = Input(UInt(vaddrBitsExtended.W)); val vconfig = Input(new VConfig) + val vstart = Input(UInt(log2Ceil(maxVLMax).W)); val rs1 = Input(UInt(xLen.W)); val rs2 = Input(UInt(xLen.W)) + } + + val killm = Input(Bool()) + + val mem = new Bundle { + val frs1 = Input(UInt(fLen.W)); val block_mem = Output(Bool()); val block_all = Output(Bool()) + } + + val wb = new Bundle { + val store_pending = Input(Bool()); val replay = Output(Bool()); val retire = Output(Bool()) + val inst = Output(UInt(32.W)); val rob_should_wb = Output(Bool()); val rob_should_wb_fp = Output(Bool()) + val pc = Output(UInt(vaddrBitsExtended.W)); val xcpt = Output(Bool()) + val cause = Output(UInt(log2Ceil(Causes.all.max).W)); val tval = Output(UInt(coreMaxAddrBits.W)) + val vxrm = Input(UInt(2.W)); val frm = Input(UInt(3.W)) + } + + val resp = Decoupled(new Bundle { + val fp = Bool(); val size = UInt(2.W); val rd = UInt(5.W); val data = UInt((xLen max fLen).W) + }) + + val set_vstart = Valid(UInt(log2Ceil(maxVLMax).W)) + val set_vxsat = Output(Bool()) + val set_vconfig = Valid(new VConfig) + val set_fflags = Valid(UInt(5.W)) + val trap_check_busy = Output(Bool()) + val backend_busy = Output(Bool()) + } + + abstract class RocketVectorDecoder(implicit p: Parameters) extends CoreModule { + + val io = IO(new Bundle { + val inst = Input(UInt(32.W)); val vconfig = Input(new VConfig) + val vector = Output(Bool()); val legal = Output(Bool()); val fp = Output(Bool()) + val read_rs1 = Output(Bool()); val read_rs2 = Output(Bool()); val read_frs1 = Output(Bool()) + val write_rd = Output(Bool()); val write_frd = Output(Bool()) + }) + + } + +} + +package examples.core.rocket.pipeline { + + /** + * Elaboration-time representation of one Buckyball custom instruction. + * Domain ISA builders own the funct7 and operand encodings; boot only streams + * these already-formed commands. + */ + case class BuckyballCommand(funct: Int, rs1: BigInt = 0, rs2: BigInt = 0) + + object BuckyballCommand { + val Custom3Opcode = 0x7b + val Custom3Funct3 = 3 + } + +} diff --git a/arch/src/main/scala/framework/system/core/rocket/Utility.scala b/examples/prefab/cores/rocket/arch/pipeline/Utility.scala similarity index 100% rename from arch/src/main/scala/framework/system/core/rocket/Utility.scala rename to examples/prefab/cores/rocket/arch/pipeline/Utility.scala diff --git a/examples/prefab/cores/rocket/configs/Params.scala b/examples/prefab/cores/rocket/configs/Params.scala new file mode 100644 index 000000000..e74dbcac7 --- /dev/null +++ b/examples/prefab/cores/rocket/configs/Params.scala @@ -0,0 +1,107 @@ +package examples.core.rocket.configs + +import examples.core.rocket.pipeline.CpuParams +import framework.core.clink.configs.{Params => CLinkParam} +import framework.core.execution.fetch.configs.{Params => FetchParam} +import framework.core.memory.configs.MemoryParam +import framework.core.memory.access.preflight.configs.PmpParams +import framework.acc.configs.AccParam + +import buckyball.config.CorePlacement +import buckyball.components.rocket.{Params => RocketProto} +import scala.jdk.CollectionConverters._ +import framework.ip.bus.{axi4, chi} +import framework.ip.bus.chi.rnf.RnfParams +import framework.ip.memory.interlock.{Params => TrackingParam} +import framework.ip.memory.coherence.configs.CoherenceParams +import framework.core.memory.home.configs.{Params => HomeParam} +import framework.core.memory.access.cache.configs.{Params => CacheParam} +import framework.core.memory.access.configs.{Params => AccessParam} +import framework.core.memory.access.{AccessParams, PhysicalRegion} +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.memdomain.tomem.configs.{Params => TransportParam} +import freechips.rocketchip.rocket.{DCacheParams, ICacheParams, MulDivParams, RocketCoreParams} +import freechips.rocketchip.tile.FPUParams + +case class Params( + cpu: CpuParams, + memory: MemoryParam, + fetch: FetchParam, + clink: CLinkParam, + permission: PmpParams, + hartId: Int, + resetVector: BigInt, + mtimeAddress: BigInt, + acc: Option[AccParam]) { + require(clink.harts == 1 && !clink.shared) + require(clink.requesters == (if (clink.backing) 0 else 2)) + require(cpu.core.xLen == 64 && !cpu.core.useHypervisor && cpu.core.useCompressed) + require(cpu.core.nPMPs == permission.entries && cpu.physicalAddressBits == permission.addressBits) + require(memory.memoryAccess.lineBits == 0) + require(memory.threadPorts == 0) +} + +object Params { + + def fromProto(core: CorePlacement): Params = { + require(core.hasMemory && core.hasImplementation && core.hasSharedMem) + val r = core.getImplementation.unpack(classOf[RocketProto]) + val bus = chi.Params.fromProto(core.getMemory.getHome.getRequesters) + val axi = axi4.Params.fromProto(core.getMemory.getUncached.getAxi) + val acc = Option.when(core.hasAccelerator) { + val a = core.getAccelerator + AccParam( + FrontendParam.fromProto(a.getFrontend, core.getParam.getXLen, core.getHartId), + MemDomainParam.fromProto(a.getMem, core.getSharedMem, core.getVirtualBankCount, 1), + GpDomainParam.fromProto(a.getGpdomain), + BallDomainParam.fromProto(a.getBalldomain), + TransportParam.fromProto(a.getTransport) + ) + } + Params( + CpuParams( + RocketCoreParams( + xLen = 64, + pgLevels = r.getPageLevels, + nPMPs = r.getPmpEntries, + useVM = r.getVirtualMemory, + useSupervisor = r.getSupervisor, + useUser = r.getUser, + useHypervisor = false, + useDebug = false, + useAtomics = r.getAtomics, + useCompressed = true, + clockGate = false, + mulDiv = Option.when(r.getMultiplyDivide)(MulDivParams()), + fpu = Option.when(r.getFloatingPoint)(FPUParams()) + ), + Some(DCacheParams()), + Some(ICacheParams()), + None, + r.getPhysicalAddressBits, + r.getHartIdBits + ), + MemoryParam.fromProto(core.getMemory, 0), + FetchParam(core.getFetch.getInstructionBytes), + CLinkParam( + 1, + !r.getExternalHome, + if (r.getExternalHome) Seq(core.getMemory.getIcache.getNodeId, core.getMemory.getDcache.getCache.getNodeId) + else Seq.empty, + bus, + axi, + acc.map(_.transport.axi).getOrElse(axi4.Params.fromProto(core.getInterconnect.getAxi)), + None + ), + PmpParams(r.getPhysicalAddressBits, r.getPmpEntries), + core.getHartId, + BigInt(java.lang.Long.toUnsignedString(core.getResetVector)), + BigInt(java.lang.Long.toUnsignedString(core.getMtimeAddress)), + acc + ) + } + +} diff --git a/examples/prefab/cores/rocket/configs/rocket_params.proto b/examples/prefab/cores/rocket/configs/rocket_params.proto new file mode 100644 index 000000000..16f8fcbc3 --- /dev/null +++ b/examples/prefab/cores/rocket/configs/rocket_params.proto @@ -0,0 +1,17 @@ +syntax = "proto2"; +package buckyball.components.rocket; +option java_package = "buckyball.components.rocket"; +option java_multiple_files = true; +message Params { + required uint32 physical_address_bits = 1; + required uint32 hart_id_bits = 2; + required uint32 page_levels = 3; + required uint32 pmp_entries = 4; + required bool virtual_memory = 5; + required bool supervisor = 6; + required bool user = 7; + required bool floating_point = 8; + required bool atomics = 9; + required bool multiply_divide = 10; + required bool external_home = 11; +} diff --git a/examples/prefab/cores/rocket/emu/Cargo.lock b/examples/prefab/cores/rocket/emu/Cargo.lock new file mode 100644 index 000000000..3eb3cb3c1 --- /dev/null +++ b/examples/prefab/cores/rocket/emu/Cargo.lock @@ -0,0 +1,822 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "bebop-bank-hash" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "snafu", +] + +[[package]] +name = "bebop-bemu" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", + "bebop-bemu-profile", + "bebop-dasm", + "bebop-dtb", + "bebop-elf", + "bebop-keyboard", + "bebop-memory", + "bebop-microphone", + "bebop-rtc", + "bebop-speaker", + "bebop-syscall", + "bebop-uart", + "bebop-vga", + "bebop-vm", + "clap", + "linkme", + "prost", + "prost-build", + "prost-types", + "rustc_apfloat", + "rvsim", + "snafu", + "toml", +] + +[[package]] +name = "bebop-bemu-profile" +version = "0.1.0" + +[[package]] +name = "bebop-core-rocket" +version = "0.1.0" +dependencies = [ + "bebop-bemu", +] + +[[package]] +name = "bebop-dasm" +version = "0.1.0" + +[[package]] +name = "bebop-dtb" +version = "0.1.0" + +[[package]] +name = "bebop-elf" +version = "0.1.0" +dependencies = [ + "bebop-memory", +] + +[[package]] +name = "bebop-keyboard" +version = "0.1.0" + +[[package]] +name = "bebop-memory" +version = "0.1.0" +dependencies = [ + "log", + "serde", + "serde_json", + "sha2", +] + +[[package]] +name = "bebop-microphone" +version = "0.1.0" + +[[package]] +name = "bebop-rtc" +version = "0.1.0" + +[[package]] +name = "bebop-speaker" +version = "0.1.0" + +[[package]] +name = "bebop-syscall" +version = "0.1.0" +dependencies = [ + "bebop-memory", + "once_cell", +] + +[[package]] +name = "bebop-uart" +version = "0.1.0" + +[[package]] +name = "bebop-vga" +version = "0.1.0" + +[[package]] +name = "bebop-vm" +version = "0.1.0" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck", + "itertools", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.119", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustc_apfloat" +version = "0.2.3+llvm-462a31f5a5ab" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "486c2179b4796f65bfe2ee33679acf0927ac83ecf583ad6c91c3b4570911b9ad" +dependencies = [ + "bitflags", + "smallvec", +] + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rvsim" +version = "0.1.0" +source = "git+https://github.com/DangoSys/riscv-cpu-sim?rev=32abeddb3f0d4c810e07d8d6745f5150e96c8c54#32abeddb3f0d4c810e07d8d6745f5150e96c8c54" +dependencies = [ + "rustc_apfloat", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "smallvec" +version = "1.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8" + +[[package]] +name = "snafu" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/examples/prefab/cores/rocket/emu/Cargo.toml b/examples/prefab/cores/rocket/emu/Cargo.toml new file mode 100644 index 000000000..1c6a30e92 --- /dev/null +++ b/examples/prefab/cores/rocket/emu/Cargo.toml @@ -0,0 +1,9 @@ +[workspace] + +[package] +name = "bebop-core-rocket" +version = "0.1.0" +edition = "2021" + +[dependencies] +bebop-bemu = { path = "../../../../../bebop/src/nodes/bemu" } diff --git a/examples/prefab/cores/rocket/emu/src/lib.rs b/examples/prefab/cores/rocket/emu/src/lib.rs new file mode 100644 index 000000000..ba2cfba41 --- /dev/null +++ b/examples/prefab/cores/rocket/emu/src/lib.rs @@ -0,0 +1,27 @@ +use bebop_bemu::{ball::linkme, CoreFactory, CoreModel, CoreParams, Cpu, DeviceModel, ModelOptions, NoRocc, Whatever, CORES}; + +/// The Rocket core: one RV64GC hart with nothing on its RoCC port. +pub struct RocketCore { + cpu: Cpu, +} + +impl CoreModel for RocketCore { + fn cpu(&self) -> &Cpu { + &self.cpu + } + fn cpu_mut(&mut self) -> &mut Cpu { + &mut self.cpu + } + fn step(&mut self, count: u64) -> Result<(), Whatever> { + self.cpu.step(count, &mut NoRocc) + } +} + +fn instantiate(device: &DeviceModel, params: &CoreParams, options: &ModelOptions) -> Result, String> { + let cpu = Cpu::new(device, params, options).map_err(|error| error.to_string())?; + Ok(Box::new(RocketCore { cpu })) +} + +#[linkme::distributed_slice(CORES)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static ROCKET: CoreFactory = CoreFactory { component: "rocket", instantiate }; diff --git a/examples/prefab/cores/rocket/stack/CMakeLists.txt b/examples/prefab/cores/rocket/stack/CMakeLists.txt new file mode 100644 index 000000000..5c50671ae --- /dev/null +++ b/examples/prefab/cores/rocket/stack/CMakeLists.txt @@ -0,0 +1,25 @@ +# ===----------------------------------------------------------------------=== +# The rocket core: a CPU core without hardware threads; jobs run on the core. +# +# bbdev stack --build "--design " +# +# builds it for each design that uses it, configured with DESIGN_CMAKE +# (generated from design.pb) and CORE. Products: +# execute.o runs jobs on the core itself; links into the device runtime +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(rocket-core LANGUAGES NONE) +include("${DESIGN_CMAKE}") + +set(RUNTIME "${DESIGN_REPO}/stack/runtime") +find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) + +add_custom_command( + OUTPUT execute.o + COMMAND "${LINUX_CC}" -O2 -fPIC -ffreestanding -fno-builtin -msmall-data-limit=0 + "-DCORE_SIGNATURE=${DESIGN_DEVICE_CORES_${DESIGN_CORE_HART_${CORE}}_SIGNATURE}ULL" "-I${RUNTIME}/scheduler" + -c "${RUNTIME}/execute/direct.c" -o execute.o + DEPENDS "${RUNTIME}/execute/direct.c" "${RUNTIME}/scheduler/runtime.h" + VERBATIM) + +add_custom_target(execute ALL DEPENDS execute.o) diff --git a/arch/thirdparty/rocket-chip b/examples/prefab/cores/rocket/thirdparty/rocket-chip similarity index 100% rename from arch/thirdparty/rocket-chip rename to examples/prefab/cores/rocket/thirdparty/rocket-chip diff --git a/examples/prefab/cores/xiangshan/LICENSE b/examples/prefab/cores/xiangshan/LICENSE new file mode 100644 index 000000000..ebf13f89a --- /dev/null +++ b/examples/prefab/cores/xiangshan/LICENSE @@ -0,0 +1,127 @@ + 木兰宽松许可证, 第2版 + + 木兰宽松许可证, 第2版 + 2020年1月 http://license.coscl.org.cn/MulanPSL2 + + + 您对“软件”的复制、使用、修改及分发受木兰宽松许可证,第2版(“本许可证”)的如下条款的约束: + + 0. 定义 + + “软件”是指由“贡献”构成的许可在“本许可证”下的程序和相关文档的集合。 + + “贡献”是指由任一“贡献者”许可在“本许可证”下的受版权法保护的作品。 + + “贡献者”是指将受版权法保护的作品许可在“本许可证”下的自然人或“法人实体”。 + + “法人实体”是指提交贡献的机构及其“关联实体”。 + + “关联实体”是指,对“本许可证”下的行为方而言,控制、受控制或与其共同受控制的机构,此处的控制是指有受控方或共同受控方至少50%直接或间接的投票权、资金或其他有价证券。 + + 1. 授予版权许可 + + 每个“贡献者”根据“本许可证”授予您永久性的、全球性的、免费的、非独占的、不可撤销的版权许可,您可以复制、使用、修改、分发其“贡献”,不论修改与否。 + + 2. 授予专利许可 + + 每个“贡献者”根据“本许可证”授予您永久性的、全球性的、免费的、非独占的、不可撤销的(根据本条规定撤销除外)专利许可,供您制造、委托制造、使用、许诺销售、销售、进口其“贡献”或以其他方式转移其“贡献”。前述专利许可仅限于“贡献者”现在或将来拥有或控制的其“贡献”本身或其“贡献”与许可“贡献”时的“软件”结合而将必然会侵犯的专利权利要求,不包括对“贡献”的修改或包含“贡献”的其他结合。如果您或您的“关联实体”直接或间接地,就“软件”或其中的“贡献”对任何人发起专利侵权诉讼(包括反诉或交叉诉讼)或其他专利维权行动,指控其侵犯专利权,则“本许可证”授予您对“软件”的专利许可自您提起诉讼或发起维权行动之日终止。 + + 3. 无商标许可 + + “本许可证”不提供对“贡献者”的商品名称、商标、服务标志或产品名称的商标许可,但您为满足第4条规定的声明义务而必须使用除外。 + + 4. 分发限制 + + 您可以在任何媒介中将“软件”以源程序形式或可执行形式重新分发,不论修改与否,但您必须向接收者提供“本许可证”的副本,并保留“软件”中的版权、商标、专利及免责声明。 + + 5. 免责声明与责任限制 + + “软件”及其中的“贡献”在提供时不带任何明示或默示的担保。在任何情况下,“贡献者”或版权所有者不对任何人因使用“软件”或其中的“贡献”而引发的任何直接或间接损失承担责任,不论因何种原因导致或者基于何种法律理论,即使其曾被建议有此种损失的可能性。 + + 6. 语言 + “本许可证”以中英文双语表述,中英文版本具有同等法律效力。如果中英文版本存在任何冲突不一致,以中文版为准。 + + 条款结束 + + 如何将木兰宽松许可证,第2版,应用到您的软件 + + 如果您希望将木兰宽松许可证,第2版,应用到您的新软件,为了方便接收者查阅,建议您完成如下三步: + + 1, 请您补充如下声明中的空白,包括软件名、软件的首次发表年份以及您作为版权人的名字; + + 2, 请您在软件包的一级目录下创建以“LICENSE”为名的文件,将整个许可证文本放入该文件中; + + 3, 请将如下声明文本放入每个源文件的头部注释中。 + + Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences + XiangShan is licensed under Mulan PSL v2. + You can use this software according to the terms and conditions of the Mulan PSL v2. + You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 + THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + See the Mulan PSL v2 for more details. + + + Mulan Permissive Software License,Version 2 + + Mulan Permissive Software License,Version 2 (Mulan PSL v2) + January 2020 http://license.coscl.org.cn/MulanPSL2 + + Your reproduction, use, modification and distribution of the Software shall be subject to Mulan PSL v2 (this License) with the following terms and conditions: + + 0. Definition + + Software means the program and related documents which are licensed under this License and comprise all Contribution(s). + + Contribution means the copyrightable work licensed by a particular Contributor under this License. + + Contributor means the Individual or Legal Entity who licenses its copyrightable work under this License. + + Legal Entity means the entity making a Contribution and all its Affiliates. + + Affiliates means entities that control, are controlled by, or are under common control with the acting entity under this License, ‘control’ means direct or indirect ownership of at least fifty percent (50%) of the voting power, capital or other securities of controlled or commonly controlled entity. + + 1. Grant of Copyright License + + Subject to the terms and conditions of this License, each Contributor hereby grants to you a perpetual, worldwide, royalty-free, non-exclusive, irrevocable copyright license to reproduce, use, modify, or distribute its Contribution, with modification or not. + + 2. Grant of Patent License + + Subject to the terms and conditions of this License, each Contributor hereby grants to you a perpetual, worldwide, royalty-free, non-exclusive, irrevocable (except for revocation under this Section) patent license to make, have made, use, offer for sale, sell, import or otherwise transfer its Contribution, where such patent license is only limited to the patent claims owned or controlled by such Contributor now or in future which will be necessarily infringed by its Contribution alone, or by combination of the Contribution with the Software to which the Contribution was contributed. The patent license shall not apply to any modification of the Contribution, and any other combination which includes the Contribution. If you or your Affiliates directly or indirectly institute patent litigation (including a cross claim or counterclaim in a litigation) or other patent enforcement activities against any individual or entity by alleging that the Software or any Contribution in it infringes patents, then any patent license granted to you under this License for the Software shall terminate as of the date such litigation or activity is filed or taken. + + 3. No Trademark License + + No trademark license is granted to use the trade names, trademarks, service marks, or product names of Contributor, except as required to fulfill notice requirements in Section 4. + + 4. Distribution Restriction + + You may distribute the Software in any medium with or without modification, whether in source or executable forms, provided that you provide recipients with a copy of this License and retain copyright, patent, trademark and disclaimer statements in the Software. + + 5. Disclaimer of Warranty and Limitation of Liability + + THE SOFTWARE AND CONTRIBUTION IN IT ARE PROVIDED WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED. IN NO EVENT SHALL ANY CONTRIBUTOR OR COPYRIGHT HOLDER BE LIABLE TO YOU FOR ANY DAMAGES, INCLUDING, BUT NOT LIMITED TO ANY DIRECT, OR INDIRECT, SPECIAL OR CONSEQUENTIAL DAMAGES ARISING FROM YOUR USE OR INABILITY TO USE THE SOFTWARE OR THE CONTRIBUTION IN IT, NO MATTER HOW IT’S CAUSED OR BASED ON WHICH LEGAL THEORY, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + + 6. Language + + THIS LICENSE IS WRITTEN IN BOTH CHINESE AND ENGLISH, AND THE CHINESE VERSION AND ENGLISH VERSION SHALL HAVE THE SAME LEGAL EFFECT. IN THE CASE OF DIVERGENCE BETWEEN THE CHINESE AND ENGLISH VERSIONS, THE CHINESE VERSION SHALL PREVAIL. + + END OF THE TERMS AND CONDITIONS + + How to Apply the Mulan Permissive Software License,Version 2 (Mulan PSL v2) to Your Software + + To apply the Mulan PSL v2 to your work, for easy identification by recipients, you are suggested to complete following three steps: + + i Fill in the blanks in following statement, including insert your software name, the year of the first publication of your software, and your name identified as the copyright owner; + + ii Create a file named “LICENSE” which contains the whole context of this License in the first directory of your software package; + + iii Attach the statement to the appropriate annotated syntax at the beginning of each source file. + + + Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences + XiangShan is licensed under Mulan PSL v2. + You can use this software according to the terms and conditions of the Mulan PSL v2. + You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 + THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + See the Mulan PSL v2 for more details. diff --git a/examples/prefab/cores/xiangshan/arch/Chi.scala b/examples/prefab/cores/xiangshan/arch/Chi.scala new file mode 100644 index 000000000..72252d02b --- /dev/null +++ b/examples/prefab/cores/xiangshan/arch/Chi.scala @@ -0,0 +1,192 @@ +package examples.core.xiangshan + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import framework.ip.bus.chi + +/** Credit-accounted native CHI B link and explicit framework flit translation. */ +@instantiable +class Chi(p: chi.Params, homeId: Int) extends Module { + require(p.nodeIdBits == 7 && p.addressBits == 48 && p.dataBits == 256) + + @public + val io = IO(new Bundle { + val native = Flipped(new Native.Link) + val requester = new chi.RequesterPort(p) + }) + + io.native.tx.linkactiveack := RegNext(io.native.tx.linkactivereq, false.B) + io.native.rx.linkactivereq := !reset.asBool + io.native.rxsactive := !reset.asBool + io.native.syscoack := RegNext(io.native.syscoreq, false.B) + val receiving = io.native.tx.linkactivereq && io.native.tx.linkactiveack + val sending = io.native.rx.linkactivereq && io.native.rx.linkactiveack + + val requestRx = Instantiate(new chi.Rx((new Native.Request).getWidth, 4)) + requestRx.io.active := receiving + requestRx.io.link <> io.native.tx.req + val request = Wire(new Native.Request) + request.unpack(requestRx.io.out.bits) + io.requester.req.valid := requestRx.io.out.valid + requestRx.io.out.ready := io.requester.req.ready + io.requester.req.bits := 0.U.asTypeOf(io.requester.req.bits) + io.requester.req.bits.qos := request.qos + io.requester.req.bits.srcId := request.srcId + io.requester.req.bits.txnId := request.txnId + io.requester.req.bits.returnNid := request.returnNid + io.requester.req.bits.stashNidValidEndian := request.stashNidValidEndian + io.requester.req.bits.returnTxnId := request.returnTxnId + io.requester.req.bits.opcode := request.opcode + io.requester.req.bits.size := request.size + io.requester.req.bits.addr := request.addr + io.requester.req.bits.likelyShared := request.likelyShared + io.requester.req.bits.allowRetry := request.allowRetry + io.requester.req.bits.order := request.order + io.requester.req.bits.pCrdType := request.pCrdType + io.requester.req.bits.memAttr := request.memAttr + io.requester.req.bits.snpAttr := request.snpAttr + io.requester.req.bits.lpid := request.lpid + io.requester.req.bits.exclSnoopMe := request.exclSnoopMe + io.requester.req.bits.expCompAck := request.expCompAck + io.requester.req.bits.traceTag := request.traceTag + io.requester.req.bits.tgtId := homeId.U + io.requester.req.bits.pas := request.ns.asUInt + when(requestRx.io.out.valid) { + assert(request.tgtId === 0.U || request.tgtId === homeId.U) + assert(request.rsvdc === 0.U) + } + + val responseRx = Instantiate(new chi.Rx((new Native.Response).getWidth, 4)) + responseRx.io.active := receiving + responseRx.io.link <> io.native.tx.rsp + val response = Wire(new Native.Response) + response.unpack(responseRx.io.out.bits) + io.requester.txRsp.valid := responseRx.io.out.valid + responseRx.io.out.ready := io.requester.txRsp.ready + io.requester.txRsp.bits := 0.U.asTypeOf(io.requester.txRsp.bits) + io.requester.txRsp.bits.qos := response.qos + io.requester.txRsp.bits.tgtId := response.tgtId + io.requester.txRsp.bits.srcId := response.srcId + io.requester.txRsp.bits.txnId := response.txnId + io.requester.txRsp.bits.opcode := response.opcode + io.requester.txRsp.bits.respErr := response.respErr + io.requester.txRsp.bits.resp := response.resp + io.requester.txRsp.bits.fwdState := response.fwdState + io.requester.txRsp.bits.dbid := response.dbid + io.requester.txRsp.bits.pCrdType := response.pCrdType + io.requester.txRsp.bits.traceTag := response.traceTag + + val dataRx = Instantiate(new chi.Rx((new Native.Data).getWidth, 4)) + dataRx.io.active := receiving + dataRx.io.link <> io.native.tx.dat + val data = Wire(new Native.Data) + data.unpack(dataRx.io.out.bits) + io.requester.txDat.valid := dataRx.io.out.valid + dataRx.io.out.ready := io.requester.txDat.ready + io.requester.txDat.bits := 0.U.asTypeOf(io.requester.txDat.bits) + io.requester.txDat.bits.qos := data.qos + io.requester.txDat.bits.tgtId := data.tgtId + io.requester.txDat.bits.srcId := data.srcId + io.requester.txDat.bits.txnId := data.txnId + io.requester.txDat.bits.homeNid := data.homeNid + io.requester.txDat.bits.opcode := data.opcode + io.requester.txDat.bits.respErr := data.respErr + io.requester.txDat.bits.resp := data.resp + io.requester.txDat.bits.dataSource := data.dataSource + io.requester.txDat.bits.dbid := data.dbid + io.requester.txDat.bits.ccid := data.ccid + io.requester.txDat.bits.dataId := data.dataId + io.requester.txDat.bits.traceTag := data.traceTag + io.requester.txDat.bits.be := data.be + io.requester.txDat.bits.data := data.data + when(dataRx.io.out.valid)(assert(data.rsvdc === 0.U)) + + val incomingResponseTx = Instantiate(new chi.Tx((new Native.Response).getWidth, 15)) + incomingResponseTx.io.active := sending + io.native.rx.rsp <> incomingResponseTx.io.link + val incomingResponse = Wire(new Native.Response) + incomingResponse := 0.U.asTypeOf(incomingResponse) + incomingResponse.qos := io.requester.rxRsp.bits.qos + incomingResponse.tgtId := io.requester.rxRsp.bits.tgtId + incomingResponse.srcId := io.requester.rxRsp.bits.srcId + incomingResponse.txnId := io.requester.rxRsp.bits.txnId(7, 0) + incomingResponse.opcode := io.requester.rxRsp.bits.opcode(3, 0) + incomingResponse.respErr := io.requester.rxRsp.bits.respErr + incomingResponse.resp := io.requester.rxRsp.bits.resp + incomingResponse.fwdState := io.requester.rxRsp.bits.fwdState + incomingResponse.dbid := io.requester.rxRsp.bits.dbid(7, 0) + incomingResponse.pCrdType := io.requester.rxRsp.bits.pCrdType + incomingResponse.traceTag := io.requester.rxRsp.bits.traceTag.asBool + incomingResponseTx.io.in.valid := io.requester.rxRsp.valid + incomingResponseTx.io.in.bits := incomingResponse.packed + io.requester.rxRsp.ready := incomingResponseTx.io.in.ready + when(io.requester.rxRsp.valid) { + assert(io.requester.rxRsp.bits.txnId < 256.U) + assert(io.requester.rxRsp.bits.dbid < 256.U && io.requester.rxRsp.bits.opcode < 16.U) + assert( + io.requester.rxRsp.bits.cBusy === 0.U && io.requester.rxRsp.bits.tagOp === 0.U && io.requester.rxRsp.bits.cacheLineId === 0.U + ) + } + + val incomingDataTx = Instantiate(new chi.Tx((new Native.Data).getWidth, 15)) + incomingDataTx.io.active := sending + io.native.rx.dat <> incomingDataTx.io.link + val incomingData = Wire(new Native.Data) + incomingData := 0.U.asTypeOf(incomingData) + incomingData.qos := io.requester.rxDat.bits.qos + incomingData.tgtId := io.requester.rxDat.bits.tgtId + incomingData.srcId := io.requester.rxDat.bits.srcId + incomingData.txnId := io.requester.rxDat.bits.txnId(7, 0) + incomingData.homeNid := io.requester.rxDat.bits.homeNid + incomingData.opcode := io.requester.rxDat.bits.opcode(2, 0) + incomingData.respErr := io.requester.rxDat.bits.respErr + incomingData.resp := io.requester.rxDat.bits.resp + incomingData.dataSource := io.requester.rxDat.bits.dataSource(2, 0) + incomingData.dbid := io.requester.rxDat.bits.dbid(7, 0) + incomingData.ccid := io.requester.rxDat.bits.ccid + incomingData.dataId := io.requester.rxDat.bits.dataId + incomingData.traceTag := io.requester.rxDat.bits.traceTag.asBool + incomingData.be := io.requester.rxDat.bits.be + incomingData.data := io.requester.rxDat.bits.data + incomingDataTx.io.in.valid := io.requester.rxDat.valid + incomingDataTx.io.in.bits := incomingData.packed + io.requester.rxDat.ready := incomingDataTx.io.in.ready + when(io.requester.rxDat.valid) { + assert(io.requester.rxDat.bits.txnId < 256.U) + assert( + io.requester.rxDat.bits.dbid < 256.U && io.requester.rxDat.bits.opcode < 8.U && io.requester.rxDat.bits.dataSource < 8.U + ) + assert(io.requester.rxDat.bits.cBusy === 0.U && io.requester.rxDat.bits.tagOp === 0.U) + assert(io.requester.rxDat.bits.tag === 0.U && io.requester.rxDat.bits.tagUpdate === 0.U) + assert( + io.requester.rxDat.bits.copyAtHome === 0.U && io.requester.rxDat.bits.numDat === 0.U && io.requester.rxDat.bits.replicate === 0.U + ) + assert(io.requester.rxDat.bits.cacheLineId === 0.U && io.requester.rxDat.bits.dataPull === 0.U) + } + + val snoopTx = Instantiate(new chi.Tx((new Native.Snoop).getWidth, 4)) + snoopTx.io.active := sending + io.native.rx.snp <> snoopTx.io.link + val snoop = Wire(new Native.Snoop) + snoop := 0.U.asTypeOf(snoop) + snoop.qos := io.requester.snp.bits.qos + snoop.srcId := io.requester.snp.bits.srcId + snoop.txnId := io.requester.snp.bits.txnId(7, 0) + snoop.fwdNid := io.requester.snp.bits.fwdNid + snoop.fwdTxnId := io.requester.snp.bits.fwdTxnId(7, 0) + snoop.opcode := io.requester.snp.bits.opcode(4, 0) + snoop.addr := io.requester.snp.bits.addr + snoop.doNotGoToSd := io.requester.snp.bits.doNotGoToSd.asBool + snoop.retToSrc := io.requester.snp.bits.retToSrc.asBool + snoop.traceTag := io.requester.snp.bits.traceTag.asBool + snoop.ns := io.requester.snp.bits.pas(0) + snoopTx.io.in.valid := io.requester.snp.valid + snoopTx.io.in.bits := snoop.packed + io.requester.snp.ready := snoopTx.io.in.ready + when(io.requester.snp.valid) { + assert(io.requester.snp.bits.txnId < 256.U) + assert(io.requester.snp.bits.fwdTxnId < 256.U && io.requester.snp.bits.pas < 2.U) + } + +} diff --git a/examples/prefab/cores/xiangshan/arch/Native.scala b/examples/prefab/cores/xiangshan/arch/Native.scala new file mode 100644 index 000000000..7fc589e37 --- /dev/null +++ b/examples/prefab/cores/xiangshan/arch/Native.scala @@ -0,0 +1,239 @@ +package examples.core.xiangshan + +import chisel3.experimental.hierarchy.{instantiable, public} + +import chisel3._ +import java.nio.file.Paths +import scala.io.Source +import scala.util.Using +import chisel3.util._ +import framework.ip.bus.{axi4, chi} + +/** Issue B layout at the pinned upstream RTL boundary. Fields are declared LSB first. */ +object Native { + + class Request extends chi.Flit { + val qos = UInt(4.W) + val tgtId = UInt(7.W) + val srcId = UInt(7.W) + val txnId = UInt(8.W) + val returnNid = UInt(7.W) + val stashNidValidEndian = Bool() + val returnTxnId = UInt(8.W) + val opcode = UInt(6.W) + val size = UInt(3.W) + val addr = UInt(48.W) + val ns = Bool() + val likelyShared = Bool() + val allowRetry = Bool() + val order = UInt(2.W) + val pCrdType = UInt(4.W) + val memAttr = UInt(4.W) + val snpAttr = Bool() + val lpid = UInt(5.W) + val exclSnoopMe = Bool() + val expCompAck = Bool() + val traceTag = Bool() + val rsvdc = UInt(4.W) + + def fieldsLSB = Seq( + qos, + tgtId, + srcId, + txnId, + returnNid, + stashNidValidEndian, + returnTxnId, + opcode, + size, + addr, + ns, + likelyShared, + allowRetry, + order, + pCrdType, + memAttr, + snpAttr, + lpid, + exclSnoopMe, + expCompAck, + traceTag, + rsvdc + ) + + } + + class Response extends chi.Flit { + val qos = UInt(4.W) + val tgtId = UInt(7.W) + val srcId = UInt(7.W) + val txnId = UInt(8.W) + val opcode = UInt(4.W) + val respErr = UInt(2.W) + val resp = UInt(3.W) + val fwdState = UInt(3.W) + val dbid = UInt(8.W) + val pCrdType = UInt(4.W) + val traceTag = Bool() + def fieldsLSB = Seq(qos, tgtId, srcId, txnId, opcode, respErr, resp, fwdState, dbid, pCrdType, traceTag) + } + + class Data extends chi.Flit { + val qos = UInt(4.W) + val tgtId = UInt(7.W) + val srcId = UInt(7.W) + val txnId = UInt(8.W) + val homeNid = UInt(7.W) + val opcode = UInt(3.W) + val respErr = UInt(2.W) + val resp = UInt(3.W) + val dataSource = UInt(3.W) + val dbid = UInt(8.W) + val ccid = UInt(2.W) + val dataId = UInt(2.W) + val traceTag = Bool() + val rsvdc = UInt(4.W) + val be = UInt(32.W) + val data = UInt(256.W) + + def fieldsLSB = Seq( + qos, + tgtId, + srcId, + txnId, + homeNid, + opcode, + respErr, + resp, + dataSource, + dbid, + ccid, + dataId, + traceTag, + rsvdc, + be, + data + ) + + } + + class Snoop extends chi.Flit { + val qos = UInt(4.W) + val srcId = UInt(7.W) + val txnId = UInt(8.W) + val fwdNid = UInt(7.W) + val fwdTxnId = UInt(8.W) + val opcode = UInt(5.W) + val addr = UInt(45.W) + val ns = Bool() + val doNotGoToSd = Bool() + val retToSrc = Bool() + val traceTag = Bool() + def fieldsLSB = Seq(qos, srcId, txnId, fwdNid, fwdTxnId, opcode, addr, ns, doNotGoToSd, retToSrc, traceTag) + } + + class Link extends Bundle { + val txsactive = Output(Bool()) + val rxsactive = Input(Bool()) + val syscoreq = Output(Bool()) + val syscoack = Input(Bool()) + + val tx = new Bundle { + val linkactivereq = Output(Bool()) + val linkactiveack = Input(Bool()) + val req = new chi.Channel((new Request).getWidth) + val rsp = new chi.Channel((new Response).getWidth) + val dat = new chi.Channel((new Data).getWidth) + } + + val rx = Flipped(new Bundle { + val linkactivereq = Output(Bool()) + val linkactiveack = Input(Bool()) + val rsp = new chi.Channel((new Response).getWidth) + val dat = new chi.Channel((new Data).getWidth) + val snp = new chi.Channel((new Snoop).getWidth) + }) + + } + + class Commands extends Bundle { + + val request = Decoupled(new Bundle { + val instruction = UInt(32.W) + val pc = UInt(64.W) + val rs1 = UInt(64.W) + val rs2 = UInt(64.W) + + val context = new Bundle { + val satp = UInt(64.W) + val privilege = UInt(2.W) + val sum = Bool() + val mxr = Bool() + val virt = Bool() + + val pmp = Vec( + 64, + new Bundle { + val cfg = UInt(8.W) + val addr = UInt(46.W) + } + ) + + } + + }) + + val complete = Flipped(Valid(Bool())) + } + + class Address(bits: Int) extends Bundle { + val id = UInt(4.W) + val addr = UInt(bits.W) + val len = UInt(8.W) + val size = UInt(3.W) + val burst = UInt(2.W) + val lock = Bool() + val cache = UInt(4.W) + val prot = UInt(3.W) + val qos = UInt(4.W) + } + + class Bus(addressBits: Int) extends Bundle { + val aw = Decoupled(new Address(addressBits)) + val w = Decoupled(new axi4.WriteData(axi4.Params(addressBits, 128, 4))) + val b = Flipped(Decoupled(new axi4.WriteResponse(axi4.Params(addressBits, 128, 4)))) + val ar = Decoupled(new Address(addressBits)) + val r = Flipped(Decoupled(new axi4.ReadData(axi4.Params(addressBits, 128, 4)))) + } + +} + +/** Only this internal leaf knows the native RTL's pin names. */ +@instantiable +class Native extends BlackBox with HasBlackBoxPath { + override def desiredName = "XiangShanNative" + + @public + val io = IO(new Bundle { + val clock = Input(Clock()) + val reset = Input(AsyncReset()) + val io_hartId = Input(UInt(6.W)) + val io_resetVector = Input(UInt(48.W)) + val io_nodeId = Input(UInt(7.W)) + val io_interrupts = Input(Vec(4, Bool())) + val io_commands = new Native.Commands + val io_timeRequest = Decoupled(Bool()) + val io_time = Flipped(Valid(UInt(64.W))) + val io_chi = new Native.Link + val uncachedAXI = Vec(1, new Native.Bus(31)) + }) + + val manifest = Paths.get(getClass.getResource("/xiangshan/filelist.f").toURI) + + val files = Using.resource(Source.fromFile(manifest.toFile))( + _.getLines().filter(_.nonEmpty).toVector + ) + + files.foreach(file => addPath(manifest.getParent.resolve(file.stripPrefix("./")).toString)) + +} diff --git a/examples/prefab/cores/xiangshan/arch/XiangShan.scala b/examples/prefab/cores/xiangshan/arch/XiangShan.scala new file mode 100644 index 000000000..0f108cf40 --- /dev/null +++ b/examples/prefab/cores/xiangshan/arch/XiangShan.scala @@ -0,0 +1,255 @@ +package examples.core.xiangshan + +import framework.acc.balldomain.bbus.BallFactory +import sims.configs.SimParam +import framework.acc.balldomain.ball.blink.BlinkIO + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import examples.core.xiangshan.configs.Params +import framework.core.clink.{CLinkIO, HasCLink} +import framework.core.clink.configs.{Params => CLinkParam} +import framework.ip.bus.axi4 +import framework.ip.bus.chi.rnf.AxiCache + +import framework.acc.frontend.Frontend +import framework.acc.memdomain.MemDomain +import framework.acc.gpdomain.GpDomain +import framework.acc.balldomain.BallDomain +import framework.acc.balldomain.ball.blink.HasBlink +import framework.acc.debug.Debug + +@instantiable +class XiangShan( + p: Params, + simp: SimParam, + factories: Map[String, BallFactory]) + extends Module + with HasCLink { + val diffTest = simp.diffTest + + @public + val clink = IO(new CLinkIO(CLinkParam( + 1, + false, + Seq(p.nodeId) ++ p.acc.toSeq.map(_ => p.dmaCache.nodeId), + p.chi, + p.axi, + p.axi, + None + ))) + + val native = Instantiate(new Native) + native.io.clock := clock + native.io.reset := reset.asAsyncReset + native.io.io_hartId := p.hartId.U + native.io.io_nodeId := p.nodeId.U + native.io.io_resetVector := p.resetVector.U + native.io.io_interrupts(0) := clink.interrupts(0).software + native.io.io_interrupts(1) := clink.interrupts(0).timer + native.io.io_interrupts(2) := clink.interrupts(0).external + native.io.io_interrupts(3) := clink.interrupts(0).supervisorExternal + + val coherent = Instantiate(new Chi(p.chi, p.homeId)) + coherent.io.native <> native.io.io_chi + clink.memory.coherent(0) <> coherent.io.requester + + val peripherals = Instantiate(new axi4.Interconnect(axi4.InterconnectParam(p.axi, 2, Some(16)))) + val upstream = native.io.uncachedAXI(0) + val downstream = peripherals.io.in(0) + downstream.aw.valid := upstream.aw.valid + upstream.aw.ready := downstream.aw.ready + downstream.aw.bits := 0.U.asTypeOf(downstream.aw.bits) + downstream.aw.bits.id := upstream.aw.bits.id + downstream.aw.bits.addr := upstream.aw.bits.addr + downstream.aw.bits.len := upstream.aw.bits.len + downstream.aw.bits.size := upstream.aw.bits.size + downstream.aw.bits.burst := upstream.aw.bits.burst + downstream.aw.bits.lock := upstream.aw.bits.lock + downstream.aw.bits.cache := upstream.aw.bits.cache + downstream.aw.bits.prot := upstream.aw.bits.prot + downstream.aw.bits.qos := upstream.aw.bits.qos + downstream.ar.valid := upstream.ar.valid + upstream.ar.ready := downstream.ar.ready + downstream.ar.bits := 0.U.asTypeOf(downstream.ar.bits) + downstream.ar.bits.id := upstream.ar.bits.id + downstream.ar.bits.addr := upstream.ar.bits.addr + downstream.ar.bits.len := upstream.ar.bits.len + downstream.ar.bits.size := upstream.ar.bits.size + downstream.ar.bits.burst := upstream.ar.bits.burst + downstream.ar.bits.lock := upstream.ar.bits.lock + downstream.ar.bits.cache := upstream.ar.bits.cache + downstream.ar.bits.prot := upstream.ar.bits.prot + downstream.ar.bits.qos := upstream.ar.bits.qos + downstream.w <> upstream.w + upstream.b <> downstream.b + upstream.r <> downstream.r + clink.memory.uncached <> peripherals.io.out + + val time = peripherals.io.in(1) + val timePending = RegInit(false.B) + time.aw.valid := false.B + time.aw.bits := 0.U.asTypeOf(time.aw.bits) + time.w.valid := false.B + time.w.bits := 0.U.asTypeOf(time.w.bits) + time.b.ready := true.B + time.ar.valid := native.io.io_timeRequest.valid && !timePending + time.ar.bits := 0.U.asTypeOf(time.ar.bits) + time.ar.bits.addr := p.mtimeAddress.U + time.ar.bits.size := 3.U + time.ar.bits.burst := 1.U + native.io.io_timeRequest.ready := time.ar.ready && !timePending + when(time.ar.fire)(timePending := true.B) + time.r.ready := timePending + native.io.io_time.valid := time.r.fire + native.io.io_time.bits := (time.r.bits.data >> ((p.mtimeAddress & 15).toInt * 8))(63, 0) + when(time.r.fire) { + assert(time.r.bits.last && time.r.bits.resp === 0.U && time.r.bits.id === 0.U) + timePending := false.B + } + + p.acc match { + case Some(acc) => + val frontend = + Instantiate(new Frontend(acc.frontend, acc.memDomain, acc.gpDomain, acc.ballDomain, p.permission, simp)) + val memDomain = Instantiate(new MemDomain( + acc.frontend, + acc.memDomain, + acc.gpDomain, + acc.transport, + p.permission, + p.tracking, + p.walker, + simp + )) + val domain = + Instantiate(new BallDomain(acc.frontend, acc.memDomain, acc.gpDomain, acc.ballDomain, p.permission, factories)) + if (acc.gpDomain.enable) { + val gp = Instantiate(new GpDomain(acc.frontend, acc.memDomain, acc.gpDomain)) + gp.io.command <> frontend.io.gpRequest.get + frontend.io.gpCompletion.get <> gp.io.complete + gp.io.image <> memDomain.io.image.get + memDomain.io.gpBanks.get <> gp.io.banks + memDomain.io.gpLayoutRequest.get <> gp.io.layoutRequest + gp.io.layoutResponse <> memDomain.io.gpLayoutResponse.get + domain.io.kernel.get.request <> gp.io.ball.request + gp.io.ball.response <> domain.io.kernel.get.response + } + val command = native.io.io_commands.request + frontend.io.rocc.cmd.valid := command.valid + command.ready := frontend.io.rocc.cmd.ready + frontend.io.rocc.cmd.bits.instruction.raw_inst := command.bits.instruction + frontend.io.rocc.cmd.bits.instruction.pc := command.bits.pc + frontend.io.rocc.cmd.bits.instruction.rs1Data := command.bits.rs1 + frontend.io.rocc.cmd.bits.instruction.rs2Data := command.bits.rs2 + frontend.io.rocc.cmd.bits.satp := command.bits.context.satp + frontend.io.rocc.cmd.bits.effectivePrivilege := command.bits.context.privilege + frontend.io.rocc.cmd.bits.sum := command.bits.context.sum + frontend.io.rocc.cmd.bits.mxr := command.bits.context.mxr + for ((target, entry) <- frontend.io.rocc.cmd.bits.pmp.zip(command.bits.context.pmp)) { + target.cfg := entry.cfg.asTypeOf(target.cfg) + target.addr := entry.addr + val encoded = ((entry.addr.pad(48) << 1) | entry.cfg(3).asUInt | 1023.U)(47, 0) + target.mask := (((encoded & ~(encoded + 1.U)) << 2) | 3.U)(47, 0) + } + frontend.io.rocc.resp.ready := true.B + val pending = RegInit(false.B) + when(command.fire)(pending := true.B) + native.io.io_commands.complete.valid := pending && (!frontend.io.rocc.busy || frontend.io.rocc.interrupt) + native.io.io_commands.complete.bits := frontend.io.rocc.interrupt + when(native.io.io_commands.complete.valid)(pending := false.B) + + domain.io.issue <> frontend.io.ballRequest + frontend.io.ballCompletion <> domain.io.complete + memDomain.io.ball <> domain.io.memory + memDomain.io.issue <> frontend.io.memoryRequest + frontend.io.memoryCompletion <> memDomain.io.complete + memDomain.io.mapping <> frontend.io.mappingUpdate + frontend.io.barrier.release := frontend.io.barrier.arrive + memDomain.io.banks <> domain.io.banks + memDomain.io.ballLayoutRequest <> domain.io.layoutRequest + domain.io.layoutResponse <> memDomain.io.ballLayoutResponse + + // DMA is a coherent requester alongside the native core. The submitting core has already drained stores + // and cannot execute younger memory operations until this command completes. + memDomain.io.memory.cpuQuery := 0.U.asTypeOf(memDomain.io.memory.cpuQuery) + val maintained = RegInit(false.B) + val maintenanceTag = Reg(chiselTypeOf(memDomain.io.memory.maintenance.bits.tag)) + memDomain.io.memory.maintenance.ready := !maintained + when(memDomain.io.memory.maintenance.fire) { + assert(pending) + maintenanceTag := memDomain.io.memory.maintenance.bits.tag + maintained := true.B + } + memDomain.io.memory.maintained.valid := maintained + memDomain.io.memory.maintained.bits.tag := maintenanceTag + memDomain.io.memory.maintained.bits.ok := true.B + when(memDomain.io.memory.maintained.fire)(maintained := false.B) + + val access = Instantiate(new axi4.Interconnect(axi4.InterconnectParam(p.axi, 2, Some(16)))) + access.io.in(0) <> memDomain.io.ddr + val pte = access.io.in(1) + val ptePending = RegInit(false.B) + val highWord = Reg(Bool()) + pte.aw.valid := false.B + pte.aw.bits := 0.U.asTypeOf(pte.aw.bits) + pte.w.valid := false.B + pte.w.bits := 0.U.asTypeOf(pte.w.bits) + pte.b.ready := true.B + pte.ar.valid := memDomain.io.memory.pteRequest.valid && !ptePending + pte.ar.bits := 0.U.asTypeOf(pte.ar.bits) + pte.ar.bits.addr := memDomain.io.memory.pteRequest.bits.addr + pte.ar.bits.size := 3.U + pte.ar.bits.burst := 1.U + memDomain.io.memory.pteRequest.ready := pte.ar.ready && !ptePending + when(pte.ar.fire) { ptePending := true.B; highWord := pte.ar.bits.addr(3) } + memDomain.io.memory.pteResponse.valid := pte.r.valid && ptePending + memDomain.io.memory.pteResponse.bits.data := Mux(highWord, pte.r.bits.data(127, 64), pte.r.bits.data(63, 0)) + memDomain.io.memory.pteResponse.bits.error := pte.r.bits.resp =/= 0.U + pte.r.ready := memDomain.io.memory.pteResponse.ready && ptePending + when(pte.r.fire) { assert(pte.r.bits.last && pte.r.bits.id === 0.U); ptePending := false.B } + val dma = Instantiate(new AxiCache(p.axi, p.dmaCache)) + dma.io.axi <> access.io.out + dma.io.axi.ar.bits.cache := 15.U + dma.io.axi.aw.bits.cache := 15.U + clink.memory.coherent(1) <> dma.io.chi + + val debug = Instantiate(new Debug(acc.frontend, acc.memDomain, acc.gpDomain, acc.ballDomain, simp, 0, 0)) + debug.io.ant.valid := false.B + debug.io.ant.bits := 0.U.asTypeOf(debug.io.ant.bits) + debug.io.acc.frontend <> frontend.io.debug + debug.io.acc.trace.memory := memDomain.io.traceMemory + debug.io.acc.trace.ballStarted := domain.io.traceStarted + debug.io.acc.trace.ballCompleted := domain.io.traceCompleted + debug.io.acc.banks.zip(memDomain.io.bank_hashes).foreach { case (sink, src) => sink := src } + for (port <- memDomain.io.shared.bank.requests) { + port.read.req.ready := false.B + port.read.resp.valid := false.B + port.read.resp.bits := 0.U.asTypeOf(port.read.resp.bits) + port.write.req.ready := false.B + port.write.resp.valid := false.B + port.write.resp.bits := 0.U.asTypeOf(port.write.resp.bits) + } + memDomain.io.shared.bank.config.ready := false.B + memDomain.io.shared.bank.mapping.ready := false.B + memDomain.io.shared.bank.local.request.valid := false.B + memDomain.io.shared.bank.local.request.bits := 0.U.asTypeOf(memDomain.io.shared.bank.local.request.bits) + memDomain.io.shared.bank.local.response.ready := false.B + memDomain.io.shared.move.command.ready := false.B + memDomain.io.shared.move.completion.valid := false.B + memDomain.io.shared.move.completion.bits := false.B + case None => + native.io.io_commands.request.ready := true.B + native.io.io_commands.complete.valid := RegNext(native.io.io_commands.request.fire, false.B) + native.io.io_commands.complete.bits := true.B + } + clink.memory.ddr.aw.valid := false.B + clink.memory.ddr.aw.bits := 0.U.asTypeOf(clink.memory.ddr.aw.bits) + clink.memory.ddr.w.valid := false.B + clink.memory.ddr.w.bits := 0.U.asTypeOf(clink.memory.ddr.w.bits) + clink.memory.ddr.ar.valid := false.B + clink.memory.ddr.ar.bits := 0.U.asTypeOf(clink.memory.ddr.ar.bits) + clink.memory.ddr.b.ready := true.B + clink.memory.ddr.r.ready := true.B +} diff --git a/examples/prefab/cores/xiangshan/arch/build.sc b/examples/prefab/cores/xiangshan/arch/build.sc new file mode 100644 index 000000000..df61d4f66 --- /dev/null +++ b/examples/prefab/cores/xiangshan/arch/build.sc @@ -0,0 +1,88 @@ +import mill._ +import mill.scalalib._ +import $file.^.^.^.^.^.arch.src.main.scala.framework.{build => framework} + +val componentRoot: os.Path = millSourcePath / os.up +val vendorRoot: os.Path = componentRoot / "thirdparty" / "xiangshan" +// Our XiangShan sources; files under native/xiangshan replace the upstream file at the same path. +val nativeRoot: os.Path = componentRoot / "arch" / "native" +val vendorHead: os.Path = + os.Path(os.proc("git", "rev-parse", "--git-path", "HEAD").call(cwd = vendorRoot).out.text().trim, vendorRoot) + +def upstreamInputs = T.sources { + val paths = Seq( + "rocket-chip/src", "rocket-chip/macros/src", "rocket-chip/cde/cde/src", "rocket-chip/hardfloat/hardfloat/src", + "difftest/src", "utility/src", "huancun/src", "fudian/src", "yunsuan/src", "coupledL2/src", "openLLC/src", + "openLLC/openNCB/src", "ChiselAIA/src", "ChiselIOPMP/src", "macros/src", "build.sc", ".mill-version", + "rocket-chip/common.sc", "rocket-chip/cde/common.sc", "rocket-chip/hardfloat/common.sc", "huancun/common.sc", + "coupledL2/common.sc", "openLLC/common.sc" + ) + paths.map(path => PathRef(vendorRoot / os.RelPath(path))) :+ PathRef(vendorHead) +} + +// Libraries XiangShan depends on (rocket-chip, utility, coupledL2, ...) with their resources, built by XiangShan's own build. +def upstreamClasspath = T { + upstreamInputs() + Seq("xiangshan.transitiveLocalClasspath", "xiangshan.compileClasspath").flatMap { task => + val shown = os.proc("mill", "--no-server", "show", task).call(cwd = vendorRoot).out.text() + ujson.read(shown).arr.map(entry => os.Path(entry.str.split(":").last)) + }.distinct.map(PathRef(_)) +} + +def gitState = T.input { + val sha = os.proc("git", "rev-parse", "HEAD").call(cwd = vendorRoot).out.trim() + val dirty = os.proc("git", "status", "-uno", "--porcelain").call(cwd = vendorRoot).out.text().nonEmpty + (sha, dirty) +} + +// Resources XiangShan's own build generates: version string and the commit id embedded in the CSR. +def versionResources = T { + val (sha, dirty) = gitState() + os.write(T.dest / "publishVersion", sha.take(12) + "\n") + os.write(T.dest / "gitStatus", s"SHA=$sha\ndirty=${if (dirty) 1 else 0}\n") + PathRef(T.dest) +} + +def vendorResources = T.sources(vendorRoot / "src" / "main" / "resources") + +object native extends ScalaModule { + override def scalaVersion = "2.13.15" + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations", "-Ytasty-reader") + override def unmanagedClasspath = T(Agg.from(upstreamClasspath())) + + override def sources = T.sources { + val upstream = vendorRoot / "src" / "main" / "scala" + os.walk(upstream) + .filter(p => p.ext == "scala" && !os.exists(nativeRoot / p.relativeTo(upstream))) + .map(PathRef(_)) :+ PathRef(nativeRoot) + } + + override def resources = T(versionResources() +: vendorResources()) +} + +def nativeRtl = T { + val output = T.dest / "xiangshan" + os.proc( + "java", "-Xmx40G", "-Xss256m", "-cp", native.runClasspath().map(_.path).mkString(java.io.File.pathSeparator), + "examples.core.xiangshan.native.Emit", "--target-dir", output, "--target", "systemverilog" + ).call(cwd = T.dest, env = Map("PATH" -> s"${sys.env("PATH")}:${vendorRoot / "src" / "main" / "resources"}")) + os.move(output / "extern_modules.sv", output / "XiangShanExtern.sv") + os.write.over(output / "filelist.f", os.read(output / "filelist.f").replace("extern_modules.sv", "XiangShanExtern.sv")) + PathRef(T.dest) +} + +trait ScalaBuild extends SbtModule { + override def scalaVersion = "2.13.16" + override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") +} +object library extends ScalaBuild { + override def moduleDeps = Seq(framework.library) + override def sources = T.sources { + os.walk(componentRoot / "arch").filter(p => p.ext == "scala" && !p.startsWith(nativeRoot)).map(PathRef(_)) ++ + Seq(PathRef(componentRoot / "configs")) + } + override def resources = T.sources(Seq(nativeRtl())) +} diff --git a/examples/prefab/cores/xiangshan/arch/native/Backend.scala b/examples/prefab/cores/xiangshan/arch/native/Backend.scala new file mode 100644 index 000000000..c31a6e255 --- /dev/null +++ b/examples/prefab/cores/xiangshan/arch/native/Backend.scala @@ -0,0 +1,1151 @@ +// Copied from XiangShan src/main/scala/xiangshan/backend/Backend.scala and modified for Buckyball. +// XiangShan is licensed under Mulan PSL v2 (http://license.coscl.org.cn/MulanPSL2). + +package examples.core.xiangshan.native + +import org.chipsalliance.cde.config.Parameters +import chisel3._ +import chisel3.util._ +import difftest._ +import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp} +import system.HasSoCParameter +import utility._ +import utility.sram.SramBroadcastBundle +import xiangshan._ +import xiangshan.backend.Bundles.{ + DynInst, + IssueQueueIQWakeUpBundle, + LoadShouldCancel, + MemExuInput, + MemExuOutput, + VPUCtrlSignals +} +import xiangshan.backend.ctrlblock.{DebugLSIO, LsTopdownInfo} +import xiangshan.backend.datapath.DataConfig.{FpData, IntData, VecData} +import xiangshan.backend.datapath.RdConfig.{IntRD, VfRD} +import xiangshan.backend.datapath.WbConfig._ +import xiangshan.backend.datapath.DataConfig._ +import xiangshan.backend.datapath._ +import xiangshan.backend.dispatch.CoreDispatchTopDownIO +import xiangshan.backend.exu.ExuBlock +import xiangshan.backend.fu.vector.Bundles.{VConfig, VType} +import xiangshan.backend.fu.{FenceIO, FenceToSbuffer, FuConfig, FuType, PerfCounterIO} +import xiangshan.backend.fu.NewCSR.PFEvent +import xiangshan.backend.issue.EntryBundles._ +import xiangshan.backend.issue.{Scheduler, SchedulerArithImp, SchedulerImpBase, SchedulerMemImp} +import xiangshan.backend.rob.{RobCoreTopDownIO, RobDebugRollingIO, RobLsqIO, RobPtr} +import xiangshan.backend.trace.TraceCoreInterface +import xiangshan.frontend.{FtqPtr, FtqRead, PreDecodeInfo} +import xiangshan.mem.{LqPtr, LsqEnqIO, SqPtr} +import scala.collection.mutable +import xiangshan.backend._ +import examples.core.xiangshan.native.{Commands, CtrlBlock} + +class Backend(val params: BackendParams)(implicit p: Parameters) extends LazyModule with HasXSParameter { + override def shouldBeInlined: Boolean = false + val inner = LazyModule(new BackendInlined(params)) + lazy val module = new BackendImp(this) +} + +class BackendImp(wrapper: Backend)(implicit p: Parameters) extends LazyModuleImp(wrapper) { + val io = IO(new BackendIO()(p, wrapper.params)) + io <> wrapper.inner.module.io + if (p(DebugOptionsKey).ResetGen) { + ResetGen(ResetGenNode(Seq(ModuleNode(wrapper.inner.module))), reset, sim = false, io.dft_reset) + } +} + +class BackendInlined(val params: BackendParams)(implicit p: Parameters) extends LazyModule with HasXSParameter { + + override def shouldBeInlined: Boolean = true + + // check read & write port config + params.configChecks + + /* Only update the idx in mem-scheduler here Idx in other schedulers can be updated the same way if needed + * + * Also note that we filter out the 'stData issue-queues' when counting */ + for ((ibp, idx) <- params.memSchdParams.get.issueBlockParams.filter(iq => iq.StdCnt == 0).zipWithIndex) { + ibp.updateIdx(idx) + } + + println(params.iqWakeUpParams) + + for ((schdCfg, i) <- params.allSchdParams.zipWithIndex) { + schdCfg.bindBackendParam(params) + } + + for ((iqCfg, i) <- params.allIssueParams.zipWithIndex) { + iqCfg.bindBackendParam(params) + } + + for ((exuCfg, i) <- params.allExuParams.zipWithIndex) { + exuCfg.bindBackendParam(params) + exuCfg.updateIQWakeUpConfigs(params.iqWakeUpParams) + exuCfg.updateExuIdx(i) + } + + println("[Backend] ExuConfigs:") + for (exuCfg <- params.allExuParams) { + val fuConfigs = exuCfg.fuConfigs + val wbPortConfigs = exuCfg.wbPortConfigs + val immType = exuCfg.immType + + println("[Backend] " + + s"${exuCfg.name}: " + + (if (exuCfg.fakeUnit) "fake, " else "") + + (if (exuCfg.hasLoadFu || exuCfg.hasHyldaFu) s"LdExuIdx(${backendParams.getLdExuIdx(exuCfg)})" else "") + + s"${fuConfigs.map(_.name).mkString("fu(s): {", ",", "}")}, " + + s"${wbPortConfigs.mkString("wb: {", ",", "}")}, " + + s"${immType.map(SelImm.mkString(_)).mkString("imm: {", ",", "}")}, " + + s"latMax(${exuCfg.latencyValMax}), ${exuCfg.fuLatancySet.mkString("lat: {", ",", "}")}, " + + s"srcReg(${exuCfg.numRegSrc})") + require( + wbPortConfigs.collectFirst { case x: IntWB => x }.nonEmpty == + fuConfigs.map(_.writeIntRf).reduce(_ || _), + s"${exuCfg.name} int wb port has no priority" + ) + require( + wbPortConfigs.collectFirst { case x: FpWB => x }.nonEmpty == + fuConfigs.map(x => x.writeFpRf).reduce(_ || _), + s"${exuCfg.name} fp wb port has no priority" + ) + require( + wbPortConfigs.collectFirst { case x: VfWB => x }.nonEmpty == + fuConfigs.map(x => x.writeVecRf).reduce(_ || _), + s"${exuCfg.name} vec wb port has no priority" + ) + } + + println(s"[Backend] all fu configs") + for (cfg <- FuConfig.allConfigs) { + println(s"[Backend] $cfg") + } + + println(s"[Backend] Int RdConfigs: ExuName(Priority)") + for ((port, seq) <- params.getRdPortParams(IntData())) { + println( + s"[Backend] port($port): ${seq.map(x => params.getExuName(x._1) + "(" + x._2.toString + ")").mkString(",")}" + ) + } + + println(s"[Backend] Int WbConfigs: ExuName(Priority)") + for ((port, seq) <- params.getWbPortParams(IntData())) { + println( + s"[Backend] port($port): ${seq.map(x => params.getExuName(x._1) + "(" + x._2.toString + ")").mkString(",")}" + ) + } + + println(s"[Backend] Fp RdConfigs: ExuName(Priority)") + for ((port, seq) <- params.getRdPortParams(FpData())) { + println( + s"[Backend] port($port): ${seq.map(x => params.getExuName(x._1) + "(" + x._2.toString + ")").mkString(",")}" + ) + } + + println(s"[Backend] Fp WbConfigs: ExuName(Priority)") + for ((port, seq) <- params.getWbPortParams(FpData())) { + println( + s"[Backend] port($port): ${seq.map(x => params.getExuName(x._1) + "(" + x._2.toString + ")").mkString(",")}" + ) + } + + println(s"[Backend] Vf RdConfigs: ExuName(Priority)") + for ((port, seq) <- params.getRdPortParams(VecData())) { + println( + s"[Backend] port($port): ${seq.map(x => params.getExuName(x._1) + "(" + x._2.toString + ")").mkString(",")}" + ) + } + + println(s"[Backend] Vf WbConfigs: ExuName(Priority)") + for ((port, seq) <- params.getWbPortParams(VecData())) { + println( + s"[Backend] port($port): ${seq.map(x => params.getExuName(x._1) + "(" + x._2.toString + ")").mkString(",")}" + ) + } + + println(s"[Backend] Dispatch Configs:") + println(s"[Backend] Load IQ enq width(${params.numLoadDp}), Store IQ enq width(${params.numStoreDp})") + println(s"[Backend] Load DP width($LSQLdEnqWidth), Store DP width($LSQStEnqWidth)") + + params.updateCopyPdestInfo + println(s"[Backend] copyPdestInfo ${params.copyPdestInfo}") + params.allExuParams.map(_.copyNum) + val ctrlBlock = LazyModule(new CtrlBlock(params)) + val intScheduler = params.intSchdParams.map(x => LazyModule(new Scheduler(x))) + val fpScheduler = params.fpSchdParams.map(x => LazyModule(new Scheduler(x))) + val vfScheduler = params.vfSchdParams.map(x => LazyModule(new Scheduler(x))) + val memScheduler = params.memSchdParams.map(x => LazyModule(new Scheduler(x))) + val dataPath = LazyModule(new DataPath(params)) + val intExuBlock = params.intSchdParams.map(x => LazyModule(new ExuBlock(x))) + val fpExuBlock = params.fpSchdParams.map(x => LazyModule(new ExuBlock(x))) + val vfExuBlock = params.vfSchdParams.map(x => LazyModule(new ExuBlock(x))) + val wbFuBusyTable = LazyModule(new WbFuBusyTable(params)) + + lazy val module = new BackendInlinedImp(this) +} + +class BackendInlinedImp(override val wrapper: BackendInlined)(implicit p: Parameters) + extends LazyModuleImp(wrapper) + with HasXSParameter + with HasPerfEvents + with HasCriticalErrors { + private implicit val params: BackendParams = wrapper.params + + val io = IO(new BackendIO()(p, wrapper.params)) + + private val ctrlBlock = wrapper.ctrlBlock.module + private val intScheduler: SchedulerImpBase = wrapper.intScheduler.get.module + private val fpScheduler = wrapper.fpScheduler.get.module + private val vfScheduler = wrapper.vfScheduler.get.module + private val memScheduler = wrapper.memScheduler.get.module + private val dataPath = wrapper.dataPath.module + private val intExuBlock = wrapper.intExuBlock.get.module + private val fpExuBlock = wrapper.fpExuBlock.get.module + private val vfExuBlock = wrapper.vfExuBlock.get.module + private val og2ForVector = Module(new Og2ForVector(params)) + private val bypassNetwork = Module(new BypassNetwork) + private val wbDataPath = Module(new WbDataPath(params)) + private val wbFuBusyTable = wrapper.wbFuBusyTable.module + private val vecExcpMod = Module(new VecExcpDataMergeModule) + + private val iqWakeUpMappedBundle: Map[Int, ValidIO[IssueQueueIQWakeUpBundle]] = ( + intScheduler.io.toSchedulers.wakeupVec ++ + fpScheduler.io.toSchedulers.wakeupVec ++ + vfScheduler.io.toSchedulers.wakeupVec ++ + memScheduler.io.toSchedulers.wakeupVec + ).map(x => (x.bits.exuIdx, x)).toMap + + private val iqWakeUpMappedBundleDelayed: Map[Int, ValidIO[IssueQueueIQWakeUpBundle]] = ( + intScheduler.io.toSchedulers.wakeupVec ++ + fpScheduler.io.toSchedulers.wakeupVec ++ + vfScheduler.io.toSchedulers.wakeupVec ++ + memScheduler.io.toSchedulers.wakeupVec + ).map { case x => + val delayed = Wire(chiselTypeOf(x)) + // TODO: add clock gate use Wen, remove issuequeue wakeupToIQ logic Wen = Wen && valid + delayed := RegNext(x) + (x.bits.exuIdx, delayed) + }.toMap + + println(s"[Backend] iq wake up keys: ${iqWakeUpMappedBundle.keys}") + + wbFuBusyTable.io.in.intSchdBusyTable := intScheduler.io.wbFuBusyTable + wbFuBusyTable.io.in.fpSchdBusyTable := fpScheduler.io.wbFuBusyTable + wbFuBusyTable.io.in.vfSchdBusyTable := vfScheduler.io.wbFuBusyTable + wbFuBusyTable.io.in.memSchdBusyTable := memScheduler.io.wbFuBusyTable + intScheduler.io.fromWbFuBusyTable.fuBusyTableRead := wbFuBusyTable.io.out.intRespRead + fpScheduler.io.fromWbFuBusyTable.fuBusyTableRead := wbFuBusyTable.io.out.fpRespRead + vfScheduler.io.fromWbFuBusyTable.fuBusyTableRead := wbFuBusyTable.io.out.vfRespRead + memScheduler.io.fromWbFuBusyTable.fuBusyTableRead := wbFuBusyTable.io.out.memRespRead + dataPath.io.wbConfictRead := wbFuBusyTable.io.out.wbConflictRead + + private val og1Cancel = dataPath.io.og1Cancel + private val og0Cancel = dataPath.io.og0Cancel + private val vlFromIntIsZero = intExuBlock.io.vlIsZero.get + private val vlFromIntIsVlmax = intExuBlock.io.vlIsVlmax.get + private val vlFromVfIsZero = vfExuBlock.io.vlIsZero.get + private val vlFromVfIsVlmax = vfExuBlock.io.vlIsVlmax.get + + private val backendCriticalError = Wire(Bool()) + + ctrlBlock.io.fromTop.hartId := io.fromTop.hartId + ctrlBlock.io.frontend <> io.frontend + ctrlBlock.io.fromCSR.toDecode := intExuBlock.io.csrToDecode.get + ctrlBlock.io.fromCSR.traceCSR := intExuBlock.io.csrio.get.traceCSR + ctrlBlock.io.fromCSR.instrAddrTransType := RegNext(intExuBlock.io.csrio.get.instrAddrTransType) + ctrlBlock.io.fromWB.wbData <> wbDataPath.io.toCtrlBlock.writeback + ctrlBlock.io.fromMem.stIn <> io.mem.stIn + ctrlBlock.io.fromMem.violation <> io.mem.memoryViolation + ctrlBlock.io.lqCanAccept := io.mem.lqCanAccept + ctrlBlock.io.sqCanAccept := io.mem.sqCanAccept + + io.mem.wfi <> ctrlBlock.io.toMem.wfi + + io.mem.lsqEnqIO <> ctrlBlock.io.toMem.lsqEnqIO + ctrlBlock.io.fromMemToDispatch.scommit := io.mem.sqDeq + ctrlBlock.io.fromMemToDispatch.lcommit := io.mem.lqDeq + ctrlBlock.io.fromMemToDispatch.sqDeqPtr := io.mem.sqDeqPtr + ctrlBlock.io.fromMemToDispatch.lqDeqPtr := io.mem.lqDeqPtr + ctrlBlock.io.fromMemToDispatch.sqCancelCnt := io.mem.sqCancelCnt + ctrlBlock.io.fromMemToDispatch.lqCancelCnt := io.mem.lqCancelCnt + ctrlBlock.io.toDispatch.wakeUpInt := intScheduler.io.toSchedulers.wakeupVec + ctrlBlock.io.toDispatch.wakeUpFp := fpScheduler.io.toSchedulers.wakeupVec + ctrlBlock.io.toDispatch.wakeUpVec := vfScheduler.io.toSchedulers.wakeupVec + ctrlBlock.io.toDispatch.wakeUpMem := memScheduler.io.toSchedulers.wakeupVec + ctrlBlock.io.toDispatch.IQValidNumVec := intScheduler.io.IQValidNumVec ++ fpScheduler.io.IQValidNumVec ++ vfScheduler.io.IQValidNumVec ++ memScheduler.io.IQValidNumVec + ctrlBlock.io.toDispatch.ldCancel := io.mem.ldCancel + ctrlBlock.io.toDispatch.og0Cancel := og0Cancel + ctrlBlock.io.toDispatch.wbPregsInt.zip(wbDataPath.io.toIntPreg).map { x => + x._1.valid := x._2.wen && x._2.intWen + x._1.bits := x._2.addr + } + ctrlBlock.io.toDispatch.wbPregsFp.zip(wbDataPath.io.toFpPreg).map { x => + x._1.valid := x._2.wen && x._2.fpWen + x._1.bits := x._2.addr + } + ctrlBlock.io.toDispatch.wbPregsVec.zip(wbDataPath.io.toVfPreg).map { x => + x._1.valid := x._2.wen && x._2.vecWen + x._1.bits := x._2.addr + } + ctrlBlock.io.toDispatch.wbPregsV0.zip(wbDataPath.io.toV0Preg).map { x => + x._1.valid := x._2.wen && x._2.v0Wen + x._1.bits := x._2.addr + } + ctrlBlock.io.toDispatch.wbPregsVl.zip(wbDataPath.io.toVlPreg).map { x => + x._1.valid := x._2.wen && x._2.vlWen + x._1.bits := x._2.addr + } + ctrlBlock.io.toDispatch.vlWriteBackInfo.vlFromIntIsZero := vlFromIntIsZero + ctrlBlock.io.toDispatch.vlWriteBackInfo.vlFromIntIsVlmax := vlFromIntIsVlmax + ctrlBlock.io.toDispatch.vlWriteBackInfo.vlFromVfIsZero := vlFromVfIsZero + ctrlBlock.io.toDispatch.vlWriteBackInfo.vlFromVfIsVlmax := vlFromVfIsVlmax + ctrlBlock.io.csrCtrl <> intExuBlock.io.csrio.get.customCtrl + ctrlBlock.io.robio.csr.intrBitSet := intExuBlock.io.csrio.get.interrupt + ctrlBlock.io.robio.csr.trapTarget := intExuBlock.io.csrio.get.trapTarget + ctrlBlock.io.robio.csr.isXRet := intExuBlock.io.csrio.get.isXRet + ctrlBlock.io.robio.csr.wfiEvent := intExuBlock.io.csrio.get.wfi_event + ctrlBlock.io.robio.csr.criticalErrorState := intExuBlock.io.csrio.get.criticalErrorState + ctrlBlock.io.robio.lsq <> io.mem.robLsqIO + ctrlBlock.io.robio.lsTopdownInfo <> io.mem.lsTopdownInfo + ctrlBlock.io.robio.debug_ls <> io.mem.debugLS + ctrlBlock.io.debugEnqLsq.canAccept := io.mem.lsqEnqIO.canAccept + ctrlBlock.io.debugEnqLsq.resp := io.mem.lsqEnqIO.resp + ctrlBlock.io.debugEnqLsq.req := ctrlBlock.io.toMem.lsqEnqIO.req + ctrlBlock.io.debugEnqLsq.needAlloc := ctrlBlock.io.toMem.lsqEnqIO.needAlloc + ctrlBlock.io.debugEnqLsq.iqAccept := ctrlBlock.io.toMem.lsqEnqIO.iqAccept + ctrlBlock.io.fromVecExcpMod.busy := vecExcpMod.o.status.busy + + val intWriteBackDelayed = Wire(chiselTypeOf(wbDataPath.io.toIntPreg)) + intWriteBackDelayed.zip(wbDataPath.io.toIntPreg).map { case (sink, source) => + sink := DontCare + sink.wen := RegNext(source.wen) + sink.intWen := RegNext(source.intWen) + sink.addr := RegEnable(source.addr, source.wen) + } + val fpWriteBackDelayed = Wire(chiselTypeOf(wbDataPath.io.toFpPreg)) + fpWriteBackDelayed.zip(wbDataPath.io.toFpPreg).map { case (sink, source) => + sink := DontCare + sink.wen := RegNext(source.wen) + sink.fpWen := RegNext(source.fpWen) + sink.addr := RegEnable(source.addr, source.wen) + } + val vfWriteBackDelayed = Wire(chiselTypeOf(wbDataPath.io.toVfPreg)) + vfWriteBackDelayed.zip(wbDataPath.io.toVfPreg).map { case (sink, source) => + sink := DontCare + sink.wen := RegNext(source.wen) + sink.vecWen := RegNext(source.vecWen) + sink.addr := RegEnable(source.addr, source.wen) + } + val v0WriteBackDelayed = Wire(chiselTypeOf(wbDataPath.io.toV0Preg)) + v0WriteBackDelayed.zip(wbDataPath.io.toV0Preg).map { case (sink, source) => + sink := DontCare + sink.wen := RegNext(source.wen) + sink.v0Wen := RegNext(source.v0Wen) + sink.addr := RegEnable(source.addr, source.wen) + } + val vlWriteBackDelayed = Wire(chiselTypeOf(wbDataPath.io.toVlPreg)) + vlWriteBackDelayed.zip(wbDataPath.io.toVlPreg).map { case (sink, source) => + sink := DontCare + sink.wen := RegNext(source.wen) + sink.vlWen := RegNext(source.vlWen) + sink.addr := RegEnable(source.addr, source.wen) + } + intScheduler.io.fromTop.hartId := io.fromTop.hartId + intScheduler.io.fromCtrlBlock.flush := ctrlBlock.io.toIssueBlock.flush + intScheduler.io.fromDispatch.uops <> ctrlBlock.io.toIssueBlock.intUops + intScheduler.io.intWriteBack := wbDataPath.io.toIntPreg + intScheduler.io.fpWriteBack := 0.U.asTypeOf(intScheduler.io.fpWriteBack) + intScheduler.io.vfWriteBack := 0.U.asTypeOf(intScheduler.io.vfWriteBack) + intScheduler.io.v0WriteBack := 0.U.asTypeOf(intScheduler.io.v0WriteBack) + intScheduler.io.vlWriteBack := 0.U.asTypeOf(intScheduler.io.vlWriteBack) + intScheduler.io.intWriteBackDelayed := intWriteBackDelayed + intScheduler.io.fpWriteBackDelayed := 0.U.asTypeOf(intScheduler.io.fpWriteBackDelayed) + intScheduler.io.vfWriteBackDelayed := 0.U.asTypeOf(intScheduler.io.vfWriteBackDelayed) + intScheduler.io.v0WriteBackDelayed := 0.U.asTypeOf(intScheduler.io.v0WriteBackDelayed) + intScheduler.io.vlWriteBackDelayed := 0.U.asTypeOf(intScheduler.io.vlWriteBackDelayed) + intScheduler.io.fromDataPath.resp := dataPath.io.toIntIQ + intScheduler.io.fromSchedulers.wakeupVec.foreach(wakeup => wakeup := iqWakeUpMappedBundle(wakeup.bits.exuIdx)) + intScheduler.io.fromSchedulers.wakeupVecDelayed.foreach { wakeup => + wakeup := iqWakeUpMappedBundleDelayed(wakeup.bits.exuIdx) + } + intScheduler.io.fromDataPath.og0Cancel := og0Cancel + intScheduler.io.fromDataPath.og1Cancel := og1Cancel + intScheduler.io.ldCancel := io.mem.ldCancel + intScheduler.io.fromDataPath.replaceRCIdx.get := dataPath.io.toWakeupQueueRCIdx.take(params.getIntExuRCWriteSize) + intScheduler.io.vlWriteBackInfo.vlFromIntIsZero := false.B + intScheduler.io.vlWriteBackInfo.vlFromIntIsVlmax := false.B + intScheduler.io.vlWriteBackInfo.vlFromVfIsZero := false.B + intScheduler.io.vlWriteBackInfo.vlFromVfIsVlmax := false.B + + fpScheduler.io.fromTop.hartId := io.fromTop.hartId + fpScheduler.io.fromCtrlBlock.flush := ctrlBlock.io.toIssueBlock.flush + fpScheduler.io.fromDispatch.uops <> ctrlBlock.io.toIssueBlock.fpUops + fpScheduler.io.intWriteBack := 0.U.asTypeOf(fpScheduler.io.intWriteBack) + fpScheduler.io.fpWriteBack := wbDataPath.io.toFpPreg + fpScheduler.io.vfWriteBack := 0.U.asTypeOf(fpScheduler.io.vfWriteBack) + fpScheduler.io.v0WriteBack := 0.U.asTypeOf(fpScheduler.io.v0WriteBack) + fpScheduler.io.vlWriteBack := 0.U.asTypeOf(fpScheduler.io.vlWriteBack) + fpScheduler.io.intWriteBackDelayed := 0.U.asTypeOf(intWriteBackDelayed) + fpScheduler.io.fpWriteBackDelayed := fpWriteBackDelayed + fpScheduler.io.vfWriteBackDelayed := 0.U.asTypeOf(intScheduler.io.vfWriteBackDelayed) + fpScheduler.io.v0WriteBackDelayed := 0.U.asTypeOf(intScheduler.io.v0WriteBackDelayed) + fpScheduler.io.vlWriteBackDelayed := 0.U.asTypeOf(intScheduler.io.vlWriteBackDelayed) + fpScheduler.io.fromDataPath.resp := dataPath.io.toFpIQ + fpScheduler.io.fromSchedulers.wakeupVec.foreach(wakeup => wakeup := iqWakeUpMappedBundle(wakeup.bits.exuIdx)) + fpScheduler.io.fromSchedulers.wakeupVecDelayed.foreach { wakeup => + wakeup := iqWakeUpMappedBundleDelayed(wakeup.bits.exuIdx) + } + fpScheduler.io.fromDataPath.og0Cancel := og0Cancel + fpScheduler.io.fromDataPath.og1Cancel := og1Cancel + fpScheduler.io.ldCancel := io.mem.ldCancel + fpScheduler.io.vlWriteBackInfo.vlFromIntIsZero := false.B + fpScheduler.io.vlWriteBackInfo.vlFromIntIsVlmax := false.B + fpScheduler.io.vlWriteBackInfo.vlFromVfIsZero := false.B + fpScheduler.io.vlWriteBackInfo.vlFromVfIsVlmax := false.B + + memScheduler.io.fromTop.hartId := io.fromTop.hartId + memScheduler.io.fromCtrlBlock.flush := ctrlBlock.io.toIssueBlock.flush + memScheduler.io.fromDispatch.uops <> ctrlBlock.io.toIssueBlock.memUops + memScheduler.io.intWriteBack := wbDataPath.io.toIntPreg + memScheduler.io.fpWriteBack := wbDataPath.io.toFpPreg + memScheduler.io.vfWriteBack := wbDataPath.io.toVfPreg + memScheduler.io.v0WriteBack := wbDataPath.io.toV0Preg + memScheduler.io.vlWriteBack := wbDataPath.io.toVlPreg + memScheduler.io.intWriteBackDelayed := intWriteBackDelayed + memScheduler.io.fpWriteBackDelayed := fpWriteBackDelayed + memScheduler.io.vfWriteBackDelayed := vfWriteBackDelayed + memScheduler.io.v0WriteBackDelayed := v0WriteBackDelayed + memScheduler.io.vlWriteBackDelayed := vlWriteBackDelayed + memScheduler.io.fromMem.get.scommit := io.mem.sqDeq + memScheduler.io.fromMem.get.lcommit := io.mem.lqDeq + memScheduler.io.fromMem.get.wakeup := io.mem.wakeup + memScheduler.io.fromMem.get.sqDeqPtr := io.mem.sqDeqPtr + memScheduler.io.fromMem.get.lqDeqPtr := io.mem.lqDeqPtr + memScheduler.io.fromMem.get.sqCancelCnt := io.mem.sqCancelCnt + memScheduler.io.fromMem.get.lqCancelCnt := io.mem.lqCancelCnt + memScheduler.io.fromMem.get.stIssuePtr := io.mem.stIssuePtr + require(memScheduler.io.fromMem.get.memWaitUpdateReq.robIdx.length == io.mem.stIn.length) + memScheduler.io.fromMem.get.memWaitUpdateReq.robIdx.zip(io.mem.stIn).foreach { case (sink, source) => + sink.valid := source.valid + sink.bits := source.bits.robIdx + } + memScheduler.io.fromMem.get.memWaitUpdateReq.sqIdx := DontCare // TODO + memScheduler.io.fromDataPath.resp := dataPath.io.toMemIQ + memScheduler.io.fromMem.get.ldaFeedback := io.mem.ldaIqFeedback + memScheduler.io.fromMem.get.staFeedback := io.mem.staIqFeedback + memScheduler.io.fromMem.get.hyuFeedback := io.mem.hyuIqFeedback + memScheduler.io.fromMem.get.vstuFeedback := io.mem.vstuIqFeedback + memScheduler.io.fromMem.get.vlduFeedback := io.mem.vlduIqFeedback + memScheduler.io.fromSchedulers.wakeupVec.foreach(wakeup => wakeup := iqWakeUpMappedBundle(wakeup.bits.exuIdx)) + memScheduler.io.fromSchedulers.wakeupVecDelayed.foreach { wakeup => + wakeup := iqWakeUpMappedBundleDelayed(wakeup.bits.exuIdx) + } + memScheduler.io.fromDataPath.og0Cancel := og0Cancel + memScheduler.io.fromDataPath.og1Cancel := og1Cancel + memScheduler.io.ldCancel := io.mem.ldCancel + memScheduler.io.fromDataPath.replaceRCIdx.get := dataPath.io.toWakeupQueueRCIdx.takeRight(params.getMemExuRCWriteSize) + memScheduler.io.vlWriteBackInfo.vlFromIntIsZero := vlFromIntIsZero + memScheduler.io.vlWriteBackInfo.vlFromIntIsVlmax := vlFromIntIsVlmax + memScheduler.io.vlWriteBackInfo.vlFromVfIsZero := vlFromVfIsZero + memScheduler.io.vlWriteBackInfo.vlFromVfIsVlmax := vlFromVfIsVlmax + memScheduler.io.fromOg2Resp.get := og2ForVector.io.toMemIQOg2Resp + + vfScheduler.io.fromTop.hartId := io.fromTop.hartId + vfScheduler.io.fromCtrlBlock.flush := ctrlBlock.io.toIssueBlock.flush + vfScheduler.io.fromDispatch.uops <> ctrlBlock.io.toIssueBlock.vfUops + vfScheduler.io.intWriteBack := 0.U.asTypeOf(vfScheduler.io.intWriteBack) + vfScheduler.io.fpWriteBack := 0.U.asTypeOf(vfScheduler.io.fpWriteBack) + vfScheduler.io.vfWriteBack := wbDataPath.io.toVfPreg + vfScheduler.io.v0WriteBack := wbDataPath.io.toV0Preg + vfScheduler.io.vlWriteBack := wbDataPath.io.toVlPreg + vfScheduler.io.intWriteBackDelayed := 0.U.asTypeOf(intWriteBackDelayed) + vfScheduler.io.fpWriteBackDelayed := 0.U.asTypeOf(fpWriteBackDelayed) + vfScheduler.io.vfWriteBackDelayed := vfWriteBackDelayed + vfScheduler.io.v0WriteBackDelayed := v0WriteBackDelayed + vfScheduler.io.vlWriteBackDelayed := vlWriteBackDelayed + vfScheduler.io.fromDataPath.resp := dataPath.io.toVfIQ + vfScheduler.io.fromSchedulers.wakeupVec.foreach(wakeup => wakeup := iqWakeUpMappedBundle(wakeup.bits.exuIdx)) + vfScheduler.io.fromSchedulers.wakeupVecDelayed.foreach { wakeup => + wakeup := iqWakeUpMappedBundleDelayed(wakeup.bits.exuIdx) + } + vfScheduler.io.fromDataPath.og0Cancel := og0Cancel + vfScheduler.io.fromDataPath.og1Cancel := og1Cancel + vfScheduler.io.ldCancel := io.mem.ldCancel + vfScheduler.io.vlWriteBackInfo.vlFromIntIsZero := vlFromIntIsZero + vfScheduler.io.vlWriteBackInfo.vlFromIntIsVlmax := vlFromIntIsVlmax + vfScheduler.io.vlWriteBackInfo.vlFromVfIsZero := vlFromVfIsZero + vfScheduler.io.vlWriteBackInfo.vlFromVfIsVlmax := vlFromVfIsVlmax + vfScheduler.io.fromOg2Resp.get := og2ForVector.io.toVfIQOg2Resp + + dataPath.io.hartId := io.fromTop.hartId + dataPath.io.flush := ctrlBlock.io.toDataPath.flush + + dataPath.io.fromIntIQ <> intScheduler.io.toDataPathAfterDelay + dataPath.io.fromFpIQ <> fpScheduler.io.toDataPathAfterDelay + dataPath.io.fromVfIQ <> vfScheduler.io.toDataPathAfterDelay + dataPath.io.fromMemIQ <> memScheduler.io.toDataPathAfterDelay + + dataPath.io.ldCancel := io.mem.ldCancel + + println( + s"[Backend] wbDataPath.io.toIntPreg: ${wbDataPath.io.toIntPreg.size}, dataPath.io.fromIntWb: ${dataPath.io.fromIntWb.size}" + ) + println( + s"[Backend] wbDataPath.io.toVfPreg: ${wbDataPath.io.toVfPreg.size}, dataPath.io.fromFpWb: ${dataPath.io.fromVfWb.size}" + ) + dataPath.io.fromIntWb := wbDataPath.io.toIntPreg + dataPath.io.fromFpWb := wbDataPath.io.toFpPreg + dataPath.io.fromVfWb := wbDataPath.io.toVfPreg + dataPath.io.fromV0Wb := wbDataPath.io.toV0Preg + dataPath.io.fromVlWb := wbDataPath.io.toVlPreg + dataPath.io.diffVlRat.foreach(_ := ctrlBlock.io.diff_vl_rat.get) + dataPath.io.fromBypassNetwork := bypassNetwork.io.toDataPath + dataPath.io.fromVecExcpMod.r := vecExcpMod.o.toVPRF.r + dataPath.io.fromVecExcpMod.w := vecExcpMod.o.toVPRF.w + dataPath.io.topDownInfo.replayAllocate := DelayN(io.topDownInfo.replayAllocate, 2) + dataPath.io.topDownInfo.sqFull := DelayN(io.topDownInfo.sqFull, 2) + dataPath.io.topDownInfo.sbFull := DelayN(io.topDownInfo.sbFull, 2) + dataPath.io.topDownInfo.l1Miss := RegNext(io.topDownInfo.l1Miss) + dataPath.io.topDownInfo.l2TopMiss.l2Miss := io.topDownInfo.l2TopMiss.l2Miss + dataPath.io.topDownInfo.l2TopMiss.l3Miss := io.topDownInfo.l2TopMiss.l3Miss + + og2ForVector.io.flush := ctrlBlock.io.toDataPath.flush + og2ForVector.io.ldCancel := io.mem.ldCancel + og2ForVector.io.fromOg1VfArith <> dataPath.io.toVecExu + og2ForVector.io.fromOg1VecMem.zip( + dataPath.io.toMemExu.zip(params.memSchdParams.get.issueBlockParams).filter(_._2.needOg2Resp).map(_._1) + ) + .foreach { + case (og1Mem, datapathMem) => og1Mem <> datapathMem + } + og2ForVector.io.fromOg1ImmInfo := dataPath.io.og1ImmInfo.zip(params.allExuParams).filter(_._2.needOg2).map(_._1) + + println( + s"[Backend] BypassNetwork OG1 Mem Size: ${bypassNetwork.io.fromDataPath.mem.zip(params.memSchdParams.get.issueBlockParams).filterNot(_._2.needOg2Resp).size}" + ) + println( + s"[Backend] BypassNetwork OG2 Mem Size: ${bypassNetwork.io.fromDataPath.mem.zip(params.memSchdParams.get.issueBlockParams).filter(_._2.needOg2Resp).size}" + ) + println( + s"[Backend] bypassNetwork.io.fromDataPath.mem: ${bypassNetwork.io.fromDataPath.mem.size}, dataPath.io.toMemExu: ${dataPath.io.toMemExu.size}" + ) + bypassNetwork.io.fromDataPath.int <> dataPath.io.toIntExu + bypassNetwork.io.fromDataPath.fp <> dataPath.io.toFpExu + bypassNetwork.io.fromDataPath.vf <> og2ForVector.io.toVfArithExu + bypassNetwork.io.fromDataPath.mem.lazyZip(params.memSchdParams.get.issueBlockParams).lazyZip( + dataPath.io.toMemExu + ).filterNot(_._2.needOg2Resp) + .map(x => (x._1, x._3)).foreach { + case (bypassMem, datapathMem) => bypassMem <> datapathMem + } + bypassNetwork.io.fromDataPath.mem.zip(params.memSchdParams.get.issueBlockParams).filter(_._2.needOg2Resp).map(_._1) + .zip(og2ForVector.io.toVecMemExu).foreach { + case (bypassMem, og2Mem) => bypassMem <> og2Mem + } + bypassNetwork.io.fromDataPath.immInfo := dataPath.io.og1ImmInfo + bypassNetwork.io.fromDataPath.immInfo.zip(params.allExuParams).filter(_._2.needOg2).map(_._1) + .zip(og2ForVector.io.toBypassNetworkImmInfo).foreach { + case (immInfo, og2ImmInfo) => immInfo := og2ImmInfo + } + bypassNetwork.io.fromDataPath.rcData := dataPath.io.toBypassNetworkRCData + bypassNetwork.io.fromExus.connectExuOutput(_.int)(intExuBlock.io.out) + bypassNetwork.io.fromExus.connectExuOutput(_.fp)(fpExuBlock.io.out) + bypassNetwork.io.fromExus.connectExuOutput(_.vf)(vfExuBlock.io.out) + + require( + bypassNetwork.io.fromExus.mem.flatten.size == io.mem.writeBack.size, + s"bypassNetwork.io.fromExus.mem.flatten.size(${bypassNetwork.io.fromExus.mem.flatten.size}: ${bypassNetwork.io.fromExus.mem.map(_.size)}, " + + s"io.mem.writeback(${io.mem.writeBack.size})" + ) + bypassNetwork.io.fromExus.mem.flatten.zip(io.mem.writeBack).foreach { case (sink, source) => + sink.valid := source.valid + sink.bits.intWen := source.bits.uop.rfWen && source.bits.isFromLoadUnit + sink.bits.pdest := source.bits.uop.pdest + sink.bits.data := source.bits.data + } + + intExuBlock.io.flush := ctrlBlock.io.toExuBlock.flush + for (i <- 0 until intExuBlock.io.in.length) { + for (j <- 0 until intExuBlock.io.in(i).length) { + val shouldLdCancel = LoadShouldCancel(bypassNetwork.io.toExus.int(i)(j).bits.loadDependency, io.mem.ldCancel) + NewPipelineConnect( + bypassNetwork.io.toExus.int(i)(j), + intExuBlock.io.in(i)(j), + intExuBlock.io.in(i)(j).fire, + Mux( + bypassNetwork.io.toExus.int(i)(j).fire, + bypassNetwork.io.toExus.int(i)(j).bits.robIdx.needFlush(ctrlBlock.io.toExuBlock.flush) || shouldLdCancel, + intExuBlock.io.in(i)(j).bits.robIdx.needFlush(ctrlBlock.io.toExuBlock.flush) + ), + Option("bypassNetwork2intExuBlock") + ) + } + } + + ctrlBlock.io.toDataPath.pcToDataPathIO <> dataPath.io.fromPcTargetMem + + private val csrin = intExuBlock.io.csrin.get + csrin.hartId := io.fromTop.hartId + csrin.msiInfo.valid := RegNext(io.fromTop.msiInfo.valid) + csrin.msiInfo.bits := RegEnable(io.fromTop.msiInfo.bits, io.fromTop.msiInfo.valid) + csrin.teemsiInfo zip io.fromTop.teemsiInfo foreach { case (csrin_teemsiInfo, fromTop_teemsiInfo) => + csrin_teemsiInfo.valid := RegNext(fromTop_teemsiInfo.valid) + csrin_teemsiInfo.bits := RegEnable(fromTop_teemsiInfo.bits, fromTop_teemsiInfo.valid) + } + csrin.clintTime.valid := RegNext(io.fromTop.clintTime.valid) + csrin.clintTime.bits := RegEnable(io.fromTop.clintTime.bits, io.fromTop.clintTime.valid) + csrin.l2FlushDone := RegNext(io.fromTop.l2FlushDone) + csrin.reset_mtvec.foreach(_ := io.fromTop.reset_mtvec.get) + csrin.trapInstInfo := ctrlBlock.io.toCSR.trapInstInfo + csrin.fromVecExcpMod.busy := vecExcpMod.o.status.busy + csrin.criticalErrorState := backendCriticalError + + private val csrio = intExuBlock.io.csrio.get + io.commands <> csrio.commands + io.timeRequest <> csrio.timeRequest + csrio.hartId := io.fromTop.hartId + csrio.fpu.fflags := ctrlBlock.io.robio.csr.fflags + csrio.fpu.isIllegal := false.B // Todo: remove it + csrio.fpu.dirty_fs := ctrlBlock.io.robio.csr.dirty_fs + csrio.vpu <> WireDefault(0.U.asTypeOf(csrio.vpu)) // Todo + + val fromIntExuVsetVType = intExuBlock.io.vtype.getOrElse(0.U.asTypeOf((Valid(new VType)))) + val fromVfExuVsetVType = vfExuBlock.io.vtype.getOrElse(0.U.asTypeOf((Valid(new VType)))) + val fromVsetVType = Mux(fromIntExuVsetVType.valid, fromIntExuVsetVType.bits, fromVfExuVsetVType.bits) + val vsetvlVType = + RegEnable(fromVsetVType, 0.U.asTypeOf(new VType), fromIntExuVsetVType.valid || fromVfExuVsetVType.valid) + ctrlBlock.io.toDecode.vsetvlVType := vsetvlVType + + val commitVType = ctrlBlock.io.robio.commitVType.vtype + val hasVsetvl = ctrlBlock.io.robio.commitVType.hasVsetvl + val vtype = VType.toVtypeStruct(Mux(hasVsetvl, vsetvlVType, commitVType.bits)).asUInt + + // csr not store the value of vl, so when using difftest we assign the value of vl to debugVl + val debugVl_s0 = WireInit(UInt(VlData().dataWidth.W), 0.U) + val debugVl_s1 = WireInit(UInt(VlData().dataWidth.W), 0.U) + debugVl_s0 := dataPath.io.diffVl.getOrElse(0.U.asTypeOf(UInt(VlData().dataWidth.W))) + debugVl_s1 := RegNext(debugVl_s0) + csrio.vpu.set_vxsat := ctrlBlock.io.robio.csr.vxsat + csrio.vpu.set_vstart.valid := ctrlBlock.io.robio.csr.vstart.valid + csrio.vpu.set_vstart.bits := ctrlBlock.io.robio.csr.vstart.bits + ctrlBlock.io.toDecode.vstart := csrio.vpu.vstart + //Todo here need change design + csrio.vpu.set_vtype.valid := commitVType.valid + csrio.vpu.set_vtype.bits := ZeroExt(vtype, XLEN) + csrio.vpu.vl := ZeroExt(debugVl_s1, XLEN) + csrio.vpu.dirty_vs := ctrlBlock.io.robio.csr.dirty_vs + csrio.exception := ctrlBlock.io.robio.exception + csrio.robDeqPtr := ctrlBlock.io.robio.robDeqPtr + csrio.memExceptionVAddr := io.mem.exceptionAddr.vaddr + csrio.memExceptionGPAddr := io.mem.exceptionAddr.gpaddr + csrio.memExceptionIsForVSnonLeafPTE := io.mem.exceptionAddr.isForVSnonLeafPTE + csrio.externalInterrupt := RegNext(io.fromTop.externalInterrupt) + csrio.perf <> io.perf + csrio.perf.retiredInstr <> ctrlBlock.io.robio.csr.perfinfo.retiredInstr + csrio.perf.ctrlInfo <> ctrlBlock.io.perfInfo.ctrlInfo + private val fenceio = intExuBlock.io.fenceio.get + io.fenceio <> fenceio + io.fenceio.sbuffer.flushSb := fenceio.sbuffer.flushSb || csrio.commandFlushStores + csrio.commandStoresEmpty := io.fenceio.sbuffer.sbIsEmpty + + // to fpExuBlock + fpExuBlock.io.flush := ctrlBlock.io.toExuBlock.flush + for (i <- 0 until fpExuBlock.io.in.length) { + for (j <- 0 until fpExuBlock.io.in(i).length) { + val shouldLdCancel = LoadShouldCancel(bypassNetwork.io.toExus.fp(i)(j).bits.loadDependency, io.mem.ldCancel) + NewPipelineConnect( + bypassNetwork.io.toExus.fp(i)(j), + fpExuBlock.io.in(i)(j), + fpExuBlock.io.in(i)(j).fire, + Mux( + bypassNetwork.io.toExus.fp(i)(j).fire, + bypassNetwork.io.toExus.fp(i)(j).bits.robIdx.needFlush(ctrlBlock.io.toExuBlock.flush) || shouldLdCancel, + fpExuBlock.io.in(i)(j).bits.robIdx.needFlush(ctrlBlock.io.toExuBlock.flush) + ), + Option("bypassNetwork2fpExuBlock") + ) + } + } + + vfExuBlock.io.flush := ctrlBlock.io.toExuBlock.flush + for (i <- 0 until vfExuBlock.io.in.size) { + for (j <- 0 until vfExuBlock.io.in(i).size) { + val shouldLdCancel = LoadShouldCancel(bypassNetwork.io.toExus.vf(i)(j).bits.loadDependency, io.mem.ldCancel) + NewPipelineConnect( + bypassNetwork.io.toExus.vf(i)(j), + vfExuBlock.io.in(i)(j), + vfExuBlock.io.in(i)(j).fire, + Mux( + bypassNetwork.io.toExus.vf(i)(j).fire, + bypassNetwork.io.toExus.vf(i)(j).bits.robIdx.needFlush(ctrlBlock.io.toExuBlock.flush) || shouldLdCancel, + vfExuBlock.io.in(i)(j).bits.robIdx.needFlush(ctrlBlock.io.toExuBlock.flush) + ), + Option("bypassNetwork2vfExuBlock") + ) + + } + } + + intExuBlock.io.frm.foreach(_ := csrio.fpu.frm) + fpExuBlock.io.frm.foreach(_ := csrio.fpu.frm) + fpExuBlock.io.vxrm.foreach(_ := csrio.vpu.vxrm) + vfExuBlock.io.frm.foreach(_ := csrio.fpu.frm) + vfExuBlock.io.vxrm.foreach(_ := csrio.vpu.vxrm) + + wbDataPath.io.flush := ctrlBlock.io.redirect + wbDataPath.io.fromTop.hartId := io.fromTop.hartId + wbDataPath.io.fromIntExu <> intExuBlock.io.out + wbDataPath.io.fromFpExu <> fpExuBlock.io.out + wbDataPath.io.fromVfExu <> vfExuBlock.io.out + wbDataPath.io.fromMemExu.flatten.zip(io.mem.writeBack).foreach { case (sink, source) => + sink.valid := source.valid + source.ready := sink.ready + sink.bits.data := VecInit(Seq.fill(sink.bits.params.wbPathNum)(source.bits.data)) + sink.bits.pdest := source.bits.uop.pdest + sink.bits.robIdx := source.bits.uop.robIdx + sink.bits.intWen.foreach(_ := source.bits.uop.rfWen) + sink.bits.fpWen.foreach(_ := source.bits.uop.fpWen) + sink.bits.vecWen.foreach(_ := source.bits.uop.vecWen) + sink.bits.v0Wen.foreach(_ := source.bits.uop.v0Wen) + sink.bits.vlWen.foreach(_ := source.bits.uop.vlWen) + sink.bits.exceptionVec.foreach(_ := source.bits.uop.exceptionVec) + sink.bits.flushPipe.foreach(_ := source.bits.uop.flushPipe) + sink.bits.replay.foreach(_ := source.bits.uop.replayInst) + sink.bits.debug := source.bits.debug + sink.bits.debugInfo := source.bits.uop.debugInfo + sink.bits.debug_seqNum := source.bits.uop.debug_seqNum + sink.bits.lqIdx.foreach(_ := source.bits.uop.lqIdx) + sink.bits.sqIdx.foreach(_ := source.bits.uop.sqIdx) + sink.bits.predecodeInfo.foreach(_ := source.bits.uop.preDecodeInfo) + sink.bits.vls.foreach { x => + x.vdIdx := source.bits.vdIdx.get + x.vdIdxInField := source.bits.vdIdxInField.get + x.vpu := source.bits.uop.vpu + x.oldVdPsrc := source.bits.uop.psrc(2) + x.isIndexed := VlduType.isIndexed(source.bits.uop.fuOpType) + x.isMasked := VlduType.isMasked(source.bits.uop.fuOpType) + x.isStrided := VlduType.isStrided(source.bits.uop.fuOpType) + x.isWhole := VlduType.isWhole(source.bits.uop.fuOpType) + x.isVecLoad := VlduType.isVecLd(source.bits.uop.fuOpType) + x.isVlm := VlduType.isMasked(source.bits.uop.fuOpType) && VlduType.isVecLd(source.bits.uop.fuOpType) + } + sink.bits.trigger.foreach(_ := source.bits.uop.trigger) + } + wbDataPath.io.fromCSR.vstart := csrio.vpu.vstart + + vecExcpMod.i.fromExceptionGen := ctrlBlock.io.toVecExcpMod.excpInfo + vecExcpMod.i.fromRab.logicPhyRegMap := ctrlBlock.io.toVecExcpMod.logicPhyRegMap + vecExcpMod.i.fromRat := ctrlBlock.io.toVecExcpMod.ratOldPest + vecExcpMod.i.fromVprf := dataPath.io.toVecExcpMod + + // to mem + private val memIssueParams = params.memSchdParams.get.issueBlockParams + private val memExuBlocksHasLDU = memIssueParams.map(_.exuBlockParams.map(x => x.hasLoadFu || x.hasHyldaFu)) + private val memExuBlocksHasVecLoad = memIssueParams.map(_.exuBlockParams.map(x => x.hasVLoadFu)) + println(s"[Backend] memExuBlocksHasLDU: $memExuBlocksHasLDU") + println(s"[Backend] memExuBlocksHasVecLoad: $memExuBlocksHasVecLoad") + + private val toMem = Wire(bypassNetwork.io.toExus.mem.cloneType) + for (i <- toMem.indices) { + for (j <- toMem(i).indices) { + val shouldLdCancel = LoadShouldCancel(bypassNetwork.io.toExus.mem(i)(j).bits.loadDependency, io.mem.ldCancel) + val needIssueTimeout = memExuBlocksHasVecLoad(i)(j) + val olderUopComing = + if (needIssueTimeout && bypassNetwork.io.toExus.mem(i)(j).bits.lqIdx.nonEmpty) + bypassNetwork.io.toExus.mem(i)(j).valid && toMem(i)(j).valid && !toMem(i)(j).fire && + (bypassNetwork.io.toExus.mem(i)(j).bits.lqIdx.get < toMem(i)(j).bits.lqIdx.get || bypassNetwork.io.toExus.mem( + i + )(j).bits.sqIdx.get < toMem(i)(j).bits.sqIdx.get) // Older inst come from iq. + else + false.B + + val issueTimeout = + if (needIssueTimeout) + Counter(0 until 14, toMem(i)(j).valid && !toMem(i)(j).fire, bypassNetwork.io.toExus.mem(i)(j).fire)._2 + else + false.B + + if (memScheduler.io.loadFinalIssueResp(i).nonEmpty && memExuBlocksHasLDU(i)(j)) { + memScheduler.io.loadFinalIssueResp(i)(j).valid := issueTimeout || olderUopComing + memScheduler.io.loadFinalIssueResp(i)(j).bits.fuType := toMem(i)(j).bits.fuType + memScheduler.io.loadFinalIssueResp(i)(j).bits.resp := RespType.block + memScheduler.io.loadFinalIssueResp(i)(j).bits.robIdx := toMem(i)(j).bits.robIdx + memScheduler.io.loadFinalIssueResp(i)(j).bits.uopIdx.foreach(_ := toMem(i)(j).bits.vpu.get.vuopIdx) + memScheduler.io.loadFinalIssueResp(i)(j).bits.sqIdx.foreach(_ := toMem(i)(j).bits.sqIdx.get) + memScheduler.io.loadFinalIssueResp(i)(j).bits.lqIdx.foreach(_ := toMem(i)(j).bits.lqIdx.get) + } + + if (memScheduler.io.vecLoadFinalIssueResp(i).nonEmpty && memExuBlocksHasVecLoad(i)(j)) { + memScheduler.io.vecLoadFinalIssueResp(i)(j).valid := issueTimeout || olderUopComing + memScheduler.io.vecLoadFinalIssueResp(i)(j).bits.fuType := toMem(i)(j).bits.fuType + memScheduler.io.vecLoadFinalIssueResp(i)(j).bits.resp := RespType.block + memScheduler.io.vecLoadFinalIssueResp(i)(j).bits.robIdx := toMem(i)(j).bits.robIdx + memScheduler.io.vecLoadFinalIssueResp(i)(j).bits.uopIdx.foreach(_ := toMem(i)(j).bits.vpu.get.vuopIdx) + memScheduler.io.vecLoadFinalIssueResp(i)(j).bits.sqIdx.foreach(_ := toMem(i)(j).bits.sqIdx.get) + memScheduler.io.vecLoadFinalIssueResp(i)(j).bits.lqIdx.foreach(_ := toMem(i)(j).bits.lqIdx.get) + memScheduler.io.vecLoadFinalIssueResp(i)(j).bits.isVecPartReplay.foreach(_ := toMem(i)( + j + ).bits.isVecPartReplay.get) + memScheduler.io.vecLoadFinalIssueResp(i)(j).bits.vecReplayMask.foreach(_ := toMem(i)(j).bits.vecReplayMask.get) + memScheduler.io.vecLoadFinalIssueResp(i)(j).bits.vecReplayMbIdx.foreach(_ := toMem(i)(j).bits.vecReplayMbIdx.get) + } + + NewPipelineConnect( + bypassNetwork.io.toExus.mem(i)(j), + toMem(i)(j), + toMem(i)(j).fire, + Mux( + bypassNetwork.io.toExus.mem(i)(j).fire, + bypassNetwork.io.toExus.mem(i)(j).bits.robIdx.needFlush(ctrlBlock.io.toExuBlock.flush) || shouldLdCancel, + toMem(i)(j).bits.robIdx.needFlush(ctrlBlock.io.toExuBlock.flush) || issueTimeout + ), + Option("bypassNetwork2toMemExus"), + isOlder = olderUopComing + ) + if (memScheduler.io.vecLoadIssueResp(i).nonEmpty && memExuBlocksHasVecLoad(i)(j)) { + memScheduler.io.vecLoadIssueResp(i)(j) match { + case resp => + resp.valid := toMem(i)(j).fire && VlduType.isVecLd(toMem(i)(j).bits.fuOpType) + resp.bits.fuType := toMem(i)(j).bits.fuType + resp.bits.robIdx := toMem(i)(j).bits.robIdx + resp.bits.uopIdx.get := toMem(i)(j).bits.vpu.get.vuopIdx + resp.bits.sqIdx.get := toMem(i)(j).bits.sqIdx.get + resp.bits.lqIdx.get := toMem(i)(j).bits.lqIdx.get + resp.bits.resp := RespType.success + resp.bits.isVecPartReplay.foreach(_ := false.B) + resp.bits.vecReplayMask.foreach(_ := 0.U) + resp.bits.vecReplayMbIdx.foreach(_ := 0.U) + } + if (backendParams.debugEn) { + dontTouch(memScheduler.io.vecLoadIssueResp(i)(j)) + } + } + } + } + + io.mem.redirect := ctrlBlock.io.redirect + io.mem.issueUops.zip(toMem.flatten).foreach { case (sink, source) => + val enableMdp = Constantin.createRecord("EnableMdp", true) + sink.valid := source.valid + source.ready := sink.ready + sink.bits.iqIdx := source.bits.iqIdx + sink.bits.isFirstIssue := source.bits.isFirstIssue + sink.bits.isVecPartReplay.foreach(_ := source.bits.isVecPartReplay.get) + sink.bits.vecReplayMask.foreach(_ := source.bits.vecReplayMask.get) + sink.bits.vecReplayMbIdx.foreach(_ := source.bits.vecReplayMbIdx.get) + sink.bits.uop := 0.U.asTypeOf(sink.bits.uop) + sink.bits.src := 0.U.asTypeOf(sink.bits.src) + sink.bits.src.zip(source.bits.src).foreach { case (l, r) => l := r } + sink.bits.uop.fuType := source.bits.fuType + sink.bits.uop.fuOpType := source.bits.fuOpType + sink.bits.uop.imm := source.bits.imm + sink.bits.uop.robIdx := source.bits.robIdx + sink.bits.uop.pdest := source.bits.pdest + sink.bits.uop.rfWen := source.bits.rfWen.getOrElse(false.B) + sink.bits.uop.fpWen := source.bits.fpWen.getOrElse(false.B) + sink.bits.uop.vecWen := source.bits.vecWen.getOrElse(false.B) + sink.bits.uop.v0Wen := source.bits.v0Wen.getOrElse(false.B) + sink.bits.uop.vlWen := source.bits.vlWen.getOrElse(false.B) + sink.bits.uop.flushPipe := source.bits.flushPipe.getOrElse(false.B) + sink.bits.uop.pc := source.bits.pc.getOrElse(0.U) + (source.bits.ftqOffset.getOrElse(0.U) << instOffsetBits) + sink.bits.uop.loadWaitBit := Mux(enableMdp, source.bits.loadWaitBit.getOrElse(false.B), false.B) + sink.bits.uop.waitForRobIdx := Mux( + enableMdp, + source.bits.waitForRobIdx.getOrElse(0.U.asTypeOf(new RobPtr)), + 0.U.asTypeOf(new RobPtr) + ) + sink.bits.uop.storeSetHit := Mux(enableMdp, source.bits.storeSetHit.getOrElse(false.B), false.B) + sink.bits.uop.loadWaitStrict := Mux(enableMdp, source.bits.loadWaitStrict.getOrElse(false.B), false.B) + sink.bits.uop.ssid := Mux(enableMdp, source.bits.ssid.getOrElse(0.U(SSIDWidth.W)), 0.U(SSIDWidth.W)) + sink.bits.uop.lqIdx := source.bits.lqIdx.getOrElse(0.U.asTypeOf(new LqPtr)) + sink.bits.uop.sqIdx := source.bits.sqIdx.getOrElse(0.U.asTypeOf(new SqPtr)) + sink.bits.uop.ftqPtr := source.bits.ftqIdx.getOrElse(0.U.asTypeOf(new FtqPtr)) + sink.bits.uop.ftqOffset := source.bits.ftqOffset.getOrElse(0.U) + sink.bits.uop.debugInfo := source.bits.perfDebugInfo + sink.bits.uop.debug_seqNum := source.bits.debug_seqNum + sink.bits.uop.vpu := source.bits.vpu.getOrElse(0.U.asTypeOf(new VPUCtrlSignals)) + sink.bits.uop.preDecodeInfo := source.bits.preDecode.getOrElse(0.U.asTypeOf(new PreDecodeInfo)) + sink.bits.uop.numLsElem := source.bits.numLsElem.getOrElse(0.U) // Todo: remove this bundle, keep only the one below + sink.bits.flowNum.foreach(_ := source.bits.numLsElem.get) + } + io.mem.loadFastMatch := memScheduler.io.toMem.get.loadFastMatch.map(_.fastMatch) + io.mem.loadFastImm := memScheduler.io.toMem.get.loadFastMatch.map(_.fastImm) + io.mem.tlbCsr := csrio.tlb + io.mem.csrCtrl := csrio.customCtrl + io.mem.sfence := fenceio.sfence + io.mem.isStoreException := CommitType.lsInstIsStore(ctrlBlock.io.robio.exception.bits.commitType) + io.mem.isVlsException := ctrlBlock.io.robio.exception.bits.vls + + io.mem.storePcRead.zipWithIndex.foreach { case (storePcRead, i) => + storePcRead := ctrlBlock.io.memStPcRead(i).data + ctrlBlock.io.memStPcRead(i).valid := io.mem.issueSta(i).valid + ctrlBlock.io.memStPcRead(i).ptr := io.mem.issueSta(i).bits.uop.ftqPtr + ctrlBlock.io.memStPcRead(i).offset := io.mem.issueSta(i).bits.uop.ftqOffset + } + + io.mem.hyuPcRead.zipWithIndex.foreach({ case (hyuPcRead, i) => + hyuPcRead := ctrlBlock.io.memHyPcRead(i).data + ctrlBlock.io.memHyPcRead(i).valid := io.mem.issueHylda(i).valid + ctrlBlock.io.memHyPcRead(i).ptr := io.mem.issueHylda(i).bits.uop.ftqPtr + ctrlBlock.io.memHyPcRead(i).offset := io.mem.issueHylda(i).bits.uop.ftqOffset + }) + + ctrlBlock.io.robio.robHeadLsIssue := io.mem.issueUops.map(deq => + deq.fire && deq.bits.uop.robIdx === ctrlBlock.io.robio.robDeqPtr + ).reduce(_ || _) + + // mem io + io.mem.robLsqIO <> ctrlBlock.io.robio.lsq + io.mem.storeDebugInfo <> ctrlBlock.io.robio.storeDebugInfo + + io.frontendSfence := fenceio.sfence + io.frontendTlbCsr := csrio.tlb + io.frontendCsrCtrl := csrio.customCtrl + + io.tlb <> csrio.tlb + + io.csrCustomCtrl := csrio.customCtrl + + io.toTop.cpuHalted := ctrlBlock.io.toTop.cpuHalt + + io.traceCoreInterface <> ctrlBlock.io.traceCoreInterface + + io.debugTopDown.fromRob := ctrlBlock.io.debugTopDown.fromRob + ctrlBlock.io.debugTopDown.fromCore := io.debugTopDown.fromCore + + io.debugRolling := ctrlBlock.io.debugRolling + + private val cg = ClockGate.genTeSrc + dontTouch(cg) + if (hasMbist) { + cg.cgen := io.dft.get.cgen + } else { + cg.cgen := false.B + } + + if (backendParams.debugEn) { + dontTouch(memScheduler.io) + dontTouch(dataPath.io.toMemExu) + dontTouch(wbDataPath.io.fromMemExu) + } + + // reset tree + if (p(DebugOptionsKey).ResetGen) { + val rightResetTree = ResetGenNode(Seq( + ModuleNode(dataPath), + ModuleNode(intExuBlock), + ModuleNode(fpExuBlock), + ModuleNode(vfExuBlock), + ModuleNode(bypassNetwork), + ModuleNode(wbDataPath) + )) + val leftResetTree = ResetGenNode(Seq( + ModuleNode(intScheduler), + ModuleNode(fpScheduler), + ModuleNode(vfScheduler), + ModuleNode(memScheduler), + ModuleNode(og2ForVector), + ModuleNode(wbFuBusyTable), + ResetGenNode(Seq( + ModuleNode(ctrlBlock), + // ResetGenNode(Seq( + CellNode(io.frontendReset) + // )) + )) + )) + ResetGen(leftResetTree, reset, sim = false, io.dft_reset) + ResetGen(rightResetTree, reset, sim = false, io.dft_reset) + } else { + io.frontendReset := DontCare + } + + // perf events + val pfevent = Module(new PFEvent) + pfevent.io.distribute_csr := RegNext(csrio.customCtrl.distribute_csr) + val csrevents = pfevent.io.hpmevent.slice(8, 16) + + val ctrlBlockPerf = ctrlBlock.getPerfEvents + val intSchedulerPerf = intScheduler.asInstanceOf[SchedulerArithImp].getPerfEvents + val fpSchedulerPerf = fpScheduler.asInstanceOf[SchedulerArithImp].getPerfEvents + val vecSchedulerPerf = vfScheduler.asInstanceOf[SchedulerArithImp].getPerfEvents + val memSchedulerPerf = memScheduler.asInstanceOf[SchedulerMemImp].getPerfEvents + val dataPathPerf = dataPath.getPerfEvents + + XSPerfAccumulate("cpu_cycle", true.B) + XSPerfAccumulate("ref_cpu_cycle", io.fromTop.clintTime.valid) + + val perfBackend = Seq( + ("cpu_cycle", true.B), + ("ref_cpu_cycle", io.fromTop.clintTime.valid) + ) + + // let index = 0 be no event + val allPerfEvents = Seq(("noEvent", 0.U)) ++ ctrlBlockPerf ++ dataPathPerf ++ + intSchedulerPerf ++ fpSchedulerPerf ++ vecSchedulerPerf ++ memSchedulerPerf ++ perfBackend + + if (printEventCoding) { + for (((name, inc), i) <- allPerfEvents.zipWithIndex) { + println("backend perfEvents Set", name, inc, i) + } + } + + val allPerfInc = allPerfEvents.map(_._2.asTypeOf(new PerfEvent)) + val perfEvents = HPerfMonitor(csrevents, allPerfInc).getPerfEvents + csrio.perf.perfEventsBackend := VecInit(perfEvents.map(_._2.asTypeOf(new PerfEvent))) + + val ctrlBlockError = ctrlBlock.getCriticalErrors + val intExuBlockError = intExuBlock.getCriticalErrors + val criticalErrors = ctrlBlockError ++ intExuBlockError + + if (printCriticalError) { + for (((name, error), _) <- criticalErrors.zipWithIndex) { + XSError(error, s"critical error: $name \n") + } + } + + // expand to collect frontend/memblock/L2 critical errors + backendCriticalError := criticalErrors.map(_._2).reduce(_ || _) + + io.toTop.cpuCriticalError := csrio.criticalErrorState + io.toTop.msiAck := csrio.msiAck + io.toTop.teemsiAck zip csrio.teemsiAck foreach { case (toTop_teemsiAck, csrio_teemsiAck) => + toTop_teemsiAck := csrio_teemsiAck + } +} + +class BackendMemIO(implicit p: Parameters, params: BackendParams) extends XSBundle { + // Since fast load replay always use load unit 0, Backend flips two load port to avoid conflicts + val flippedLda = true + // params alias + private val LoadQueueSize = VirtualLoadQueueSize + // In/Out // Todo: split it into one-direction bundle + val lsqEnqIO = Flipped(new LsqEnqIO) + val robLsqIO = new RobLsqIO + val ldaIqFeedback = Vec(params.LduCnt, Flipped(new MemRSFeedbackIO)) + val staIqFeedback = Vec(params.StaCnt, Flipped(new MemRSFeedbackIO)) + val hyuIqFeedback = Vec(params.HyuCnt, Flipped(new MemRSFeedbackIO)) + val vstuIqFeedback = Flipped(Vec(params.VstuCnt, new MemRSFeedbackIO(isVector = true))) + val vlduIqFeedback = Flipped(Vec(params.VlduCnt, new MemRSFeedbackIO(isVector = true))) + val ldCancel = Vec(params.LdExuCnt, Input(new LoadCancelIO)) + val wakeup = Vec(params.LdExuCnt, Flipped(Valid(new DynInst))) + val storePcRead = Vec(params.StaCnt, Output(UInt(VAddrBits.W))) + val hyuPcRead = Vec(params.HyuCnt, Output(UInt(VAddrBits.W))) + // Input + val writebackLda = Vec(params.LduCnt, Flipped(DecoupledIO(new MemExuOutput))) + val writebackSta = Vec(params.StaCnt, Flipped(DecoupledIO(new MemExuOutput))) + val writebackStd = Vec(params.StdCnt, Flipped(DecoupledIO(new MemExuOutput))) + val writebackHyuLda = Vec(params.HyuCnt, Flipped(DecoupledIO(new MemExuOutput))) + val writebackHyuSta = Vec(params.HyuCnt, Flipped(DecoupledIO(new MemExuOutput))) + val writebackVldu = Vec(params.VlduCnt, Flipped(DecoupledIO(new MemExuOutput(true)))) + + val s3_delayed_load_error = Input(Vec(LoadPipelineWidth, Bool())) + val stIn = Input(Vec(params.StaExuCnt, ValidIO(new DynInst()))) + val memoryViolation = Flipped(ValidIO(new Redirect)) + + val exceptionAddr = Input(new Bundle { + val vaddr = UInt(XLEN.W) + val gpaddr = UInt(XLEN.W) + val isForVSnonLeafPTE = Bool() + }) + + val sqDeq = Input(UInt(log2Ceil(EnsbufferWidth + 1).W)) + val lqDeq = Input(UInt(log2Up(CommitWidth + 1).W)) + val sqDeqPtr = Input(new SqPtr) + val lqDeqPtr = Input(new LqPtr) + + val lqCancelCnt = Input(UInt(log2Up(VirtualLoadQueueSize + 1).W)) + val sqCancelCnt = Input(UInt(log2Up(StoreQueueSize + 1).W)) + + val lqCanAccept = Input(Bool()) + val sqCanAccept = Input(Bool()) + + val otherFastWakeup = Flipped(Vec(params.LduCnt + params.HyuCnt, ValidIO(new DynInst))) + val stIssuePtr = Input(new SqPtr()) + + val debugLS = Flipped(Output(new DebugLSIO)) + + val lsTopdownInfo = Vec(params.LduCnt + params.HyuCnt, Flipped(Output(new LsTopdownInfo))) + // Output + val redirect = ValidIO(new Redirect) // rob flush MemBlock + val issueLda = MixedVec(Seq.fill(params.LduCnt)(DecoupledIO(new MemExuInput()))) + val issueSta = MixedVec(Seq.fill(params.StaCnt)(DecoupledIO(new MemExuInput()))) + val issueStd = MixedVec(Seq.fill(params.StdCnt)(DecoupledIO(new MemExuInput()))) + val issueHylda = MixedVec(Seq.fill(params.HyuCnt)(DecoupledIO(new MemExuInput()))) + val issueHysta = MixedVec(Seq.fill(params.HyuCnt)(DecoupledIO(new MemExuInput()))) + val issueVldu = MixedVec(Seq.fill(params.VlduCnt)(DecoupledIO(new MemExuInput(true)))) + + val loadFastMatch = Vec(params.LduCnt, Output(UInt(params.LduCnt.W))) + val loadFastImm = Vec(params.LduCnt, Output(UInt(12.W))) // Imm_I + + val tlbCsr = Output(new TlbCsrBundle) + val csrCtrl = Output(new CustomCSRCtrlIO) + val sfence = Output(new SfenceBundle) + val isStoreException = Output(Bool()) + val isVlsException = Output(Bool()) + + val wfi = new WfiReqBundle + + // ATTENTION: The issue ports' sequence order should be the same as IQs' deq config + def issueUops: Seq[DecoupledIO[MemExuInput]] = { + issueSta ++ + issueHylda ++ issueHysta ++ + issueLda ++ + issueVldu ++ + issueStd + }.toSeq + + // ATTENTION: The writeback ports' sequence order should be the same as IQs' deq config + def writeBack: Seq[DecoupledIO[MemExuOutput]] = { + writebackSta ++ + writebackHyuLda ++ writebackHyuSta ++ + writebackLda ++ + writebackVldu ++ + writebackStd + } + + // store event difftest information + val storeDebugInfo = Vec( + EnsbufferWidth, + new Bundle { + val robidx = Input(new RobPtr) + val pc = Output(UInt(VAddrBits.W)) + } + ) + +} + +class BackendIO(implicit p: Parameters, params: BackendParams) extends XSBundle with HasSoCParameter { + val commands = new Commands + val timeRequest = Decoupled(Bool()) + val fromTop = Flipped(new TopToBackendBundle) + + val toTop = new BackendToTopBundle + + val traceCoreInterface = new TraceCoreInterface(hasOffset = true) + val fenceio = new FenceIO + // Todo: merge these bundles into BackendFrontendIO + val frontend = Flipped(new FrontendToCtrlIO) + val frontendSfence = Output(new SfenceBundle) + val frontendCsrCtrl = Output(new CustomCSRCtrlIO) + val frontendTlbCsr = Output(new TlbCsrBundle) + val frontendReset = Output(Reset()) + + val mem = new BackendMemIO + + val perf = Input(new PerfCounterIO) + + val tlb = Output(new TlbCsrBundle) + + val csrCustomCtrl = Output(new CustomCSRCtrlIO) + + val debugTopDown = new Bundle { + val fromRob = new RobCoreTopDownIO + val fromCore = new CoreDispatchTopDownIO + } + + val debugRolling = new RobDebugRollingIO + val topDownInfo = new TopDownInfo + val dft = Option.when(hasDFT)(Input(new SramBroadcastBundle)) + val dft_reset = Option.when(hasMbist)(Input(new DFTResetSignals())) +} diff --git a/examples/prefab/cores/xiangshan/arch/native/CSR.scala b/examples/prefab/cores/xiangshan/arch/native/CSR.scala new file mode 100644 index 000000000..dccae052e --- /dev/null +++ b/examples/prefab/cores/xiangshan/arch/native/CSR.scala @@ -0,0 +1,486 @@ +// Copied from XiangShan src/main/scala/xiangshan/backend/fu/wrapper/CSR.scala and modified for Buckyball. +// XiangShan is licensed under Mulan PSL v2 (http://license.coscl.org.cn/MulanPSL2). + +package examples.core.xiangshan.native + +import chisel3._ +import freechips.rocketchip.rocket.CSRs +import chisel3.util._ +import org.chipsalliance.cde.config.Parameters +import utility._ +import xiangshan._ +import xiangshan.backend.fu.NewCSR._ +import xiangshan.backend.fu.util._ +import xiangshan.backend.fu.{FuConfig, FuncUnit} +import device._ +import system.HasSoCParameter +import xiangshan.ExceptionNO._ +import xiangshan.backend.Bundles._ +import xiangshan.backend.decode.Imm_Z +import xiangshan.backend.fu.NewCSR.CSRBundles.PrivState +import xiangshan.backend.fu.NewCSR.CSRDefines.PrivMode +import xiangshan.backend.rob.RobPtr +import xiangshan.frontend.FtqPtr +import CSRConst._ +import xiangshan.backend.fu.wrapper._ +import examples.core.xiangshan.native.Command + +class CSR(cfg: FuConfig)(implicit p: Parameters) + extends FuncUnit(cfg) + with HasCircularQueuePtrHelper + with HasCriticalErrors + with HasSoCParameter { + val csrIn = io.csrio.get + val csrOut = io.csrio.get + val csrToDecode = io.csrToDecode.get + + val setFsDirty = csrIn.fpu.dirty_fs + val setFflags = csrIn.fpu.fflags + + val setVsDirty = csrIn.vpu.dirty_vs + val setVstart = csrIn.vpu.set_vstart + val setVtype = csrIn.vpu.set_vtype + val setVxsat = csrIn.vpu.set_vxsat + val vlFromPreg = csrIn.vpu.vl + + val flushPipe = Wire(Bool()) + val flush = io.flush.valid + + val timeAllowed = WireDefault(true.B) + val isBall = io.in.bits.ctrl.fuOpType === Command.opType + + /** Alias of input signals */ + val (valid, src1, imm, func) = ( + io.in.valid && timeAllowed && !isBall, + io.in.bits.data.src(0), + io.in.bits.data.imm(Imm_Z().len - 1, 0), + io.in.bits.ctrl.fuOpType + ) + + // split imm/src1/rd from IMM_Z: src1/rd for tval + val addr = Imm_Z().getCSRAddr(imm) + val rd = Imm_Z().getRD(imm) + val rs1 = Imm_Z().getRS1(imm) + val imm5 = Imm_Z().getImm5(imm) + val csri = ZeroExt(imm5, XLEN) + + private val isEcall = CSROpType.isSystemOp(func) && addr === privEcall + private val isEbreak = CSROpType.isSystemOp(func) && addr === privEbreak + private val isMNret = CSROpType.isSystemOp(func) && addr === privMNret + private val isMret = CSROpType.isSystemOp(func) && addr === privMret + private val isSret = CSROpType.isSystemOp(func) && addr === privSret + private val isDret = CSROpType.isSystemOp(func) && addr === privDret + private val isWfi = CSROpType.isWfi(func) + private val isCSRAcc = CSROpType.isCsrAccess(func) + + val csrMod = Module(new NewCSR) + val trapInstMod = Module(new TrapInstMod) + val trapTvalMod = Module(new TrapTvalMod) + val satpFlushMod = Module(new SatpFlushMod) + + private val privState = csrMod.io.status.privState + // The real reg value in CSR, with no read mask + private val regOut = csrMod.io.out.bits.regOut + private val src = Mux(CSROpType.needImm(func), csri, src1) + + private val wdata = LookupTree( + func, + Seq( + CSROpType.wrt -> src1, + CSROpType.set -> (regOut | src1), + CSROpType.clr -> (regOut & (~src1).asUInt), + CSROpType.wrti -> csri, + CSROpType.seti -> (regOut | csri), + CSROpType.clri -> (regOut & (~csri).asUInt) + ) + ) + + private val csrAccess = valid && CSROpType.isCsrAccess(func) + + private val csrWen = valid && ( + CSROpType.isCSRRW(func) || + CSROpType.isCSRRSorRC(func) && rs1 =/= 0.U + ) + + private val csrRen = valid && ( + CSROpType.isCSRRW(func) && rd =/= 0.U || + CSROpType.isCSRRSorRC(func) + ) + + private val waddrReg = RegEnable(addr, 0.U(12.W), io.in.fire) + private val wdataReg = RegEnable(wdata, 0.U(64.W), io.in.fire) + + private val robIdxReg = RegEnable(io.in.bits.ctrl.robIdx, io.in.fire) + private val thisRobIdx = Wire(new RobPtr) + when(io.in.valid) { + thisRobIdx := io.in.bits.ctrl.robIdx + }.otherwise { + thisRobIdx := robIdxReg + } + private val redirectFlush = thisRobIdx.needFlush(io.flush) + + val readsTime = CSROpType.isCsrAccess(func) && addr === CSRs.time.U + val timePending = RegInit(false.B) + val timeFresh = RegInit(false.B) + val timeKilled = RegInit(false.B) + val timeOwner = Reg(new RobPtr) + timeAllowed := !timePending && (!readsTime || timeFresh) + csrOut.timeRequest.valid := io.in.valid && readsTime && !timePending && !timeFresh && !redirectFlush + csrOut.timeRequest.bits := true.B + when(csrOut.timeRequest.fire) { + timePending := true.B + timeKilled := false.B + timeOwner := io.in.bits.ctrl.robIdx + } + when(timePending && timeOwner.needFlush(io.flush))(timeKilled := true.B) + when(io.csrin.get.clintTime.valid && timePending) { + timePending := false.B + timeFresh := !timeKilled && !timeOwner.needFlush(io.flush) + } + when(io.in.fire && readsTime || redirectFlush)(timeFresh := false.B) + + csrMod.io.in match { + case in => + in.valid := valid + in.bits.wen := csrWen + in.bits.ren := csrRen + in.bits.op := CSROpType.getCSROp(func) + in.bits.addr := addr + in.bits.src := src + in.bits.wdata := wdataReg + in.bits.mret := isMret + in.bits.mnret := isMNret + in.bits.sret := isSret + in.bits.dret := isDret + in.bits.redirectFlush := redirectFlush + } + csrMod.io.trapInst := trapInstMod.io.currentTrapInst + csrMod.io.fetchMalTval := trapTvalMod.io.tval + csrMod.io.oldPrivSate := satpFlushMod.out.oldPrivState + csrMod.io.oldSatpMode := satpFlushMod.out.oldSatpMode + csrMod.io.oldVsatpMode := satpFlushMod.out.oldVsatpMode + csrMod.io.fromMem.excpVA := csrIn.memExceptionVAddr + csrMod.io.fromMem.excpGPA := csrIn.memExceptionGPAddr + csrMod.io.fromMem.excpIsForVSnonLeafPTE := csrIn.memExceptionIsForVSnonLeafPTE + + csrMod.io.fromRob.trap.valid := csrIn.exception.valid + csrMod.io.fromRob.trap.bits.pc := csrIn.exception.bits.pc + csrMod.io.fromRob.trap.bits.instr := csrIn.exception.bits.instr + csrMod.io.fromRob.trap.bits.pcGPA := csrIn.exception.bits.gpaddr + // Todo: shrink the width of trap vector. + // We use 64bits trap vector in CSR, and 24 bits exceptionVec in exception bundle. + csrMod.io.fromRob.trap.bits.trapVec := csrIn.exception.bits.exceptionVec.asUInt + csrMod.io.fromRob.trap.bits.isFetchBkpt := csrIn.exception.bits.isPcBkpt + csrMod.io.fromRob.trap.bits.singleStep := csrIn.exception.bits.singleStep + csrMod.io.fromRob.trap.bits.crossPageIPFFix := csrIn.exception.bits.crossPageIPFFix + csrMod.io.fromRob.trap.bits.isInterrupt := csrIn.exception.bits.isInterrupt + csrMod.io.fromRob.trap.bits.trigger := csrIn.exception.bits.trigger + csrMod.io.fromRob.trap.bits.isHls := csrIn.exception.bits.isHls + csrMod.io.fromRob.trap.bits.isFetchMalAddr := csrIn.exception.bits.isFetchMalAddr + csrMod.io.fromRob.trap.bits.satpFlushFirstFetchFault := csrIn.exception.bits.satpFlushFirstFetchFault + csrMod.io.fromRob.trap.bits.isForVSnonLeafPTE := csrIn.exception.bits.isForVSnonLeafPTE + + csrMod.io.fromRob.commit.fflags := setFflags + csrMod.io.fromRob.commit.fsDirty := setFsDirty + csrMod.io.fromRob.commit.vxsat.valid := setVxsat.valid + csrMod.io.fromRob.commit.vxsat.bits := setVxsat.bits + csrMod.io.fromRob.commit.vsDirty := setVsDirty + csrMod.io.fromRob.commit.vstart := setVstart + csrMod.io.fromRob.commit.vl := vlFromPreg + // Todo: correct vtype + csrMod.io.fromRob.commit.vtype.valid := setVtype.valid + csrMod.io.fromRob.commit.vtype.bits.VILL := setVtype.bits(XLEN - 1) + csrMod.io.fromRob.commit.vtype.bits.VMA := setVtype.bits(7) + csrMod.io.fromRob.commit.vtype.bits.VTA := setVtype.bits(6) + csrMod.io.fromRob.commit.vtype.bits.VSEW := setVtype.bits(5, 3) + csrMod.io.fromRob.commit.vtype.bits.VLMUL := setVtype.bits(2, 0) + + csrMod.io.fromRob.commit.instNum.valid := true.B // Todo: valid control signal + csrMod.io.fromRob.commit.instNum.bits := csrIn.perf.retiredInstr + + csrMod.io.fromRob.robDeqPtr := csrIn.robDeqPtr + + csrMod.io.fromVecExcpMod.busy := io.csrin.get.fromVecExcpMod.busy + + csrMod.io.perf := csrIn.perf + + csrMod.platformIRP.MEIP := csrIn.externalInterrupt.meip + csrMod.platformIRP.MTIP := csrIn.externalInterrupt.mtip + csrMod.platformIRP.MSIP := csrIn.externalInterrupt.msip + csrMod.platformIRP.SEIP := csrIn.externalInterrupt.seip + csrMod.platformIRP.STIP := false.B + csrMod.platformIRP.VSEIP := false.B // Todo + csrMod.platformIRP.VSTIP := false.B // Todo + csrMod.platformIRP.debugIP := csrIn.externalInterrupt.debug + csrMod.nonMaskableIRP.NMI_43 := csrIn.externalInterrupt.nmi.nmi_43 + csrMod.nonMaskableIRP.NMI_31 := csrIn.externalInterrupt.nmi.nmi_31 + + csrMod.io.fromTop.hartId := io.csrin.get.hartId + csrMod.io.fromTop.clintTime := io.csrin.get.clintTime + csrMod.io.fromTop.l2FlushDone := io.csrin.get.l2FlushDone + csrMod.io.fromTop.reset_mtvec.foreach(_ := io.csrin.get.reset_mtvec.get) + csrMod.io.fromTop.criticalErrorState := io.csrin.get.criticalErrorState + private val csrModOutValid = csrMod.io.out.valid + private val csrModOut = csrMod.io.out.bits + + trapInstMod.io.fromDecode.trapInstInfo := RegNextWithEnable(io.csrin.get.trapInstInfo, hasInit = true) + trapInstMod.io.fromRob.flush.valid := io.flush.valid + trapInstMod.io.fromRob.flush.bits.ftqPtr := io.flush.bits.ftqIdx + trapInstMod.io.fromRob.flush.bits.ftqOffset := io.flush.bits.ftqOffset + trapInstMod.io.fromRob.isInterrupt.valid := csrIn.exception.valid + trapInstMod.io.fromRob.isInterrupt.bits := csrIn.exception.bits.isInterrupt + trapInstMod.io.faultCsrUop.valid := csrMod.io.out.valid && (csrMod.io.out.bits.EX_II || csrMod.io.out.bits.EX_VI) + trapInstMod.io.faultCsrUop.bits.fuOpType := DataHoldBypass(io.in.bits.ctrl.fuOpType, io.in.fire) + trapInstMod.io.faultCsrUop.bits.imm := DataHoldBypass(io.in.bits.data.imm, io.in.fire) + trapInstMod.io.faultCsrUop.bits.ftqInfo.ftqPtr := DataHoldBypass(io.in.bits.ctrl.ftqIdx.get, io.in.fire) + trapInstMod.io.faultCsrUop.bits.ftqInfo.ftqOffset := DataHoldBypass(io.in.bits.ctrl.ftqOffset.get, io.in.fire) + // Clear trap instruction when instruction fault trap(EX_II, EX_VI) occurs. + trapInstMod.io.readClear := (csrMod.io.fromRob.trap match { + case t => + t.valid && !t.bits.isInterrupt && (t.bits.trapVec(EX_II) || t.bits.trapVec(EX_VI)) + }) + + trapTvalMod.io.targetPc.valid := csrMod.io.out.bits.targetPcUpdate + trapTvalMod.io.targetPc.bits := csrMod.io.out.bits.targetPc + trapTvalMod.io.clear := csrIn.exception.valid && csrIn.exception.bits.isFetchMalAddr + trapTvalMod.io.fromCtrlBlock.flush := io.flush + trapTvalMod.io.fromCtrlBlock.robDeqPtr := io.csrio.get.robDeqPtr + + satpFlushMod.in.satp.valid := csrMod.io.status.satp.valid + satpFlushMod.in.satp.bits := csrMod.io.status.satp.bits + satpFlushMod.in.vsatp.valid := csrMod.io.status.vsatp.valid + satpFlushMod.in.vsatp.bits := csrMod.io.status.vsatp.bits + satpFlushMod.in.privState := csrMod.io.status.privState + + val imsic = Module(new aia.IMSIC_WRAP(soc.IMSICParams)) + imsic.fromCSR.addr.valid := csrMod.toAIA.addr.valid + imsic.fromCSR.addr.bits.addr := csrMod.toAIA.addr.bits.addr + imsic.fromCSR.addr.bits.virt := csrMod.toAIA.addr.bits.v.asUInt.asBool + imsic.fromCSR.addr.bits.priv := aia.PrivType(csrMod.toAIA.addr.bits.prvm.asUInt) + imsic.fromCSR.vgein := csrMod.toAIA.vgein + imsic.fromCSR.wdata.valid := csrMod.toAIA.wdata.valid + imsic.fromCSR.wdata.bits.op := aia.OpType(csrMod.toAIA.wdata.bits.op) + imsic.fromCSR.wdata.bits.data := csrMod.toAIA.wdata.bits.data + imsic.fromCSR.claims(0) := csrMod.toAIA.mClaim + imsic.fromCSR.claims(1) := csrMod.toAIA.sClaim + imsic.fromCSR.claims(2) := csrMod.toAIA.vsClaim + + csrMod.fromAIA.rdata.valid := imsic.toCSR.rdata.valid + csrMod.fromAIA.rdata.bits.data := imsic.toCSR.rdata.bits + csrMod.fromAIA.rdata.bits.illegal := imsic.toCSR.illegal + csrMod.fromAIA.meip := imsic.toCSR.pendings(0) + csrMod.fromAIA.seip := imsic.toCSR.pendings(1) + csrMod.fromAIA.vseip := imsic.toCSR.pendings(soc.IMSICParams.intFilesNum - 1, 2) + csrMod.fromAIA.mtopei := imsic.toCSR.topeis(0) + csrMod.fromAIA.stopei := imsic.toCSR.topeis(1) + csrMod.fromAIA.vstopei := imsic.toCSR.topeis(2) + + imsic.msiio.vld_req := io.csrin.get.msiInfo.valid + imsic.msiio.data := io.csrin.get.msiInfo.bits + io.csrio.get.msiAck := imsic.msiio.vld_ack + + imsic.teemsiio.foreach { teemsiio => + teemsiio.vld_req := io.csrin.get.teemsiInfo.get.valid + teemsiio.data := io.csrin.get.teemsiInfo.get.bits + io.csrio.get.teemsiAck.get := teemsiio.vld_ack + } + csrMod.fromAIA.notice_pending := false.B + imsic.sec.foreach { imsicsec => + imsicsec.cmode := csrMod.io.tlb.mbmc.CMODE.asBool + csrMod.fromAIA.notice_pending := imsicsec.notice_pending + } + + private val exceptionVec = WireInit(0.U.asTypeOf(ExceptionVec())) // Todo: + + exceptionVec(EX_BP) := DataHoldBypass(isEbreak, false.B, io.in.fire) + exceptionVec(EX_MCALL) := DataHoldBypass(isEcall && privState.isModeM, false.B, io.in.fire) + exceptionVec(EX_HSCALL) := DataHoldBypass(isEcall && privState.isModeHS, false.B, io.in.fire) + exceptionVec(EX_VSCALL) := DataHoldBypass(isEcall && privState.isModeVS, false.B, io.in.fire) + exceptionVec(EX_UCALL) := DataHoldBypass(isEcall && privState.isModeHUorVU, false.B, io.in.fire) + exceptionVec(EX_II) := csrMod.io.out.bits.EX_II + exceptionVec(EX_VI) := csrMod.io.out.bits.EX_VI + + val isXRet = valid && func === CSROpType.jmp && !isEcall && !isEbreak + + flushPipe := csrMod.io.out.bits.flushPipe + + // tlb + val tlb = Wire(new TlbCsrBundle) + tlb.satp.changed := csrMod.io.tlb.satpASIDChanged + tlb.satp.mode := csrMod.io.tlb.satp.MODE.asUInt + tlb.satp.asid := csrMod.io.tlb.satp.ASID.asUInt + tlb.satp.ppn := csrMod.io.tlb.satp.PPN.asUInt + tlb.vsatp.changed := csrMod.io.tlb.vsatpASIDChanged + tlb.vsatp.mode := csrMod.io.tlb.vsatp.MODE.asUInt + tlb.vsatp.asid := csrMod.io.tlb.vsatp.ASID.asUInt + tlb.vsatp.ppn := csrMod.io.tlb.vsatp.PPN.asUInt + tlb.hgatp.changed := csrMod.io.tlb.hgatpVMIDChanged + tlb.hgatp.mode := csrMod.io.tlb.hgatp.MODE.asUInt + tlb.hgatp.vmid := csrMod.io.tlb.hgatp.VMID.asUInt + tlb.hgatp.ppn := csrMod.io.tlb.hgatp.PPN.asUInt + tlb.mbmc.KEYIDEN := csrMod.io.tlb.mbmc.KEYIDEN.asUInt + tlb.mbmc.BME := csrMod.io.tlb.mbmc.BME.asUInt + tlb.mbmc.CMODE := csrMod.io.tlb.mbmc.CMODE.asUInt + tlb.mbmc.BCLEAR := csrMod.io.tlb.mbmc.BCLEAR.asUInt + tlb.mbmc.BMA := csrMod.io.tlb.mbmc.BMA.asUInt + + // expose several csr bits for tlb + tlb.priv.mxr := csrMod.io.tlb.mxr + tlb.priv.sum := csrMod.io.tlb.sum + tlb.priv.vmxr := csrMod.io.tlb.vmxr + tlb.priv.vsum := csrMod.io.tlb.vsum + tlb.priv.spvp := csrMod.io.tlb.spvp + tlb.priv.virt := csrMod.io.tlb.dvirt + tlb.priv.virt_changed := DataChanged(tlb.priv.virt) + tlb.priv.imode := csrMod.io.tlb.imode + tlb.priv.dmode := csrMod.io.tlb.dmode + tlb.priv.debug := csrMod.io.tlb.debug + + // Svpbmt extension enable + tlb.mPBMTE := csrMod.io.tlb.mPBMTE + tlb.hPBMTE := csrMod.io.tlb.hPBMTE + + // pointer masking extension + tlb.pmm := csrMod.io.tlb.pmm + + /** Since some CSR read instructions are allowed to be pipelined, ready/valid signals should be modified */ + io.in.ready := csrMod.io.in.ready && timeAllowed // Includes asynchronous time reads. + io.out.valid := csrModOutValid + io.out.bits.ctrl.exceptionVec.get := exceptionVec + io.out.bits.ctrl.flushPipe.get := flushPipe + io.out.bits.ctrl.satpFlushPipe.get := csrMod.io.status.satp.valid || csrMod.io.status.vsatp.valid + io.out.bits.res.data := csrMod.io.out.bits.rData + + /** initialize NewCSR's io_out_ready from wrapper's io */ + csrMod.io.out.ready := io.out.ready + + io.out.bits.res.redirect.get.valid := io.out.valid && RegEnable(isXRet, false.B, io.in.fire) + val redirect = io.out.bits.res.redirect.get.bits + redirect := 0.U.asTypeOf(redirect) + redirect.level := RedirectLevel.flushAfter + redirect.robIdx := robIdxReg + redirect.ftqIdx := RegEnable(io.in.bits.ctrl.ftqIdx.get, io.in.fire) + redirect.ftqOffset := RegEnable(io.in.bits.ctrl.ftqOffset.get, io.in.fire) + redirect.cfiUpdate.predTaken := true.B + redirect.cfiUpdate.taken := true.B + redirect.cfiUpdate.target := csrMod.io.out.bits.targetPc.pc + redirect.cfiUpdate.backendIPF := csrMod.io.out.bits.targetPc.raiseIPF + redirect.cfiUpdate.backendIAF := csrMod.io.out.bits.targetPc.raiseIAF + redirect.cfiUpdate.backendIGPF := csrMod.io.out.bits.targetPc.raiseIGPF + // Only mispred will send redirect to frontend + redirect.cfiUpdate.isMisPred := false.B + + connectNonPipedCtrlSingal + + val commandIdle :: commandDrain :: commandSend :: commandWait :: commandDone :: Nil = Enum(5) + val commandState = RegInit(commandIdle) + val command = Reg(chiselTypeOf(csrOut.commands.request.bits)) + val commandError = RegInit(false.B) + val commandOwner = Reg(new RobPtr) + csrOut.commandFlushStores := commandState === commandDrain + csrOut.commands.request.valid := commandState === commandSend + csrOut.commands.request.bits := command + when(isBall)(io.in.ready := commandState === commandIdle && !timePending && !timeFresh) + when(io.in.fire && isBall) { + assert(!csrMod.io.out.valid) + command.instruction := Cat(Imm_Z().getCSRAddr(imm), Imm_Z().getRS1(imm), 3.U(3.W), 0.U(5.W), "h7b".U(7.W)) + command.pc := io.in.bits.data.pc.get + command.rs1 := io.in.bits.data.src(0) + command.rs2 := io.in.bits.data.src(1) + command.context := csrMod.io.accelerator + commandOwner := io.in.bits.ctrl.robIdx + commandError := csrMod.io.accelerator.virt + commandState := Mux(csrMod.io.accelerator.virt, commandDone, commandDrain) + } + when(commandState =/= commandIdle)(io.in.ready := false.B) + when(commandState === commandDrain && csrOut.commandStoresEmpty)(commandState := commandSend) + when(csrOut.commands.request.fire)(commandState := commandWait) + when(commandState === commandWait && csrOut.commands.complete.valid) { + commandError := csrOut.commands.complete.bits + commandState := commandDone + } + when(commandState === commandDone) { + io.out.valid := true.B + io.out.bits.res.data := 0.U + io.out.bits.res.redirect.get.valid := false.B + io.out.bits.ctrl.exceptionVec.get := 0.U.asTypeOf(io.out.bits.ctrl.exceptionVec.get) + // Failed accelerator operations trap instead of retiring as successful commands. + io.out.bits.ctrl.exceptionVec.get(illegalInstr) := commandError + io.out.bits.ctrl.flushPipe.get := false.B + io.out.bits.ctrl.satpFlushPipe.get := false.B + when(io.out.fire)(commandState := commandIdle) + } + when(commandState =/= commandIdle)(assert(!commandOwner.needFlush(io.flush))) + + override val criticalErrors = csrMod.getCriticalErrors + generateCriticalErrors() + + // Todo: summerize all difftest skip condition + csrOut.isPerfCnt := io.out.valid && csrMod.io.out.bits.isPerfCnt && RegEnable( + func =/= CSROpType.jmp, + false.B, + io.in.fire + ) + csrOut.fpu.frm := csrMod.io.status.fpState.frm.asUInt + csrOut.vpu.vstart := csrMod.io.status.vecState.vstart.asUInt + csrOut.vpu.vxrm := csrMod.io.status.vecState.vxrm.asUInt + + csrOut.isXRet := isXRet + + csrOut.trapTarget := csrMod.io.out.bits.targetPc + csrOut.interrupt := csrMod.io.status.interrupt + csrOut.wfi_event := csrMod.io.status.wfiEvent + + csrOut.tlb := tlb + + csrOut.debugMode := csrMod.io.status.debugMode + + csrOut.traceCSR := csrMod.io.status.traceCSR + + csrOut.customCtrl match { + case custom => + custom.pf_ctrl := csrMod.io.status.custom.pf_ctrl + // Load violation predictor + custom.lvpred_disable := csrMod.io.status.custom.lvpred_disable + custom.no_spec_load := csrMod.io.status.custom.no_spec_load + custom.storeset_wait_store := csrMod.io.status.custom.storeset_wait_store + custom.storeset_no_fast_wakeup := csrMod.io.status.custom.storeset_no_fast_wakeup + custom.lvpred_timeout := csrMod.io.status.custom.lvpred_timeout + // Branch predictor + custom.bp_ctrl := csrMod.io.status.custom.bp_ctrl + // Memory Block + custom.sbuffer_timeout := csrMod.io.status.custom.sbuffer_timeout + custom.sbuffer_threshold := csrMod.io.status.custom.sbuffer_threshold + custom.ldld_vio_check_enable := csrMod.io.status.custom.ldld_vio_check_enable + custom.soft_prefetch_enable := csrMod.io.status.custom.soft_prefetch_enable + custom.cache_error_enable := csrMod.io.status.custom.cache_error_enable + custom.uncache_write_outstanding_enable := csrMod.io.status.custom.uncache_write_outstanding_enable + custom.hd_misalign_st_enable := csrMod.io.status.custom.hd_misalign_st_enable + custom.hd_misalign_ld_enable := csrMod.io.status.custom.hd_misalign_ld_enable + custom.power_down_enable := csrMod.io.status.custom.power_down_enable + custom.commit_stuck_check_enable := csrMod.io.status.custom.commit_stuck_check_enable + custom.flush_l2_enable := csrMod.io.status.custom.flush_l2_enable + // Rename + custom.fusion_enable := csrMod.io.status.custom.fusion_enable + custom.wfi_enable := csrMod.io.status.custom.wfi_enable + // distribute csr write signal + // write to frontend and memory + custom.distribute_csr.w.valid := csrMod.io.distributedWenLegal + custom.distribute_csr.w.bits.addr := waddrReg + custom.distribute_csr.w.bits.data := wdataReg + // rename single step + custom.singlestep := csrMod.io.status.singleStepFlag + // trigger + custom.frontend_trigger := csrMod.io.status.frontendTrigger + custom.mem_trigger := csrMod.io.status.memTrigger + // virtual mode + custom.virtMode := csrMod.io.status.privState.V.asBool + // xstatus.fs field is off + custom.fsIsOff := csrMod.io.toDecode.illegalInst.fsIsOff + } + + csrOut.instrAddrTransType := csrMod.io.status.instrAddrTransType + csrOut.criticalErrorState := csrMod.io.status.criticalErrorState + + csrToDecode := csrMod.io.toDecode +} diff --git a/examples/prefab/cores/xiangshan/arch/native/Core.scala b/examples/prefab/cores/xiangshan/arch/native/Core.scala new file mode 100644 index 000000000..7a9f8b758 --- /dev/null +++ b/examples/prefab/cores/xiangshan/arch/native/Core.scala @@ -0,0 +1,398 @@ +// Copied from XiangShan src/main/scala/xiangshan/XSCore.scala and modified for Buckyball. +// XiangShan is licensed under Mulan PSL v2 (http://license.coscl.org.cn/MulanPSL2). + +package examples.core.xiangshan.native + +import org.chipsalliance.cde.config +import org.chipsalliance.cde.config.Parameters +import chisel3._ +import chisel3.util._ +import coupledL2.PrefetchCtrlFromCore +import freechips.rocketchip.diplomacy.{BundleBridgeSource, LazyModule, LazyModuleImp} +import freechips.rocketchip.diplomacy._ +import freechips.rocketchip.tile.HasFPUParameters +import system.{HasSoCParameter, SoCParamsKey} +import utils._ +import utility._ +import utility.mbist.{MbistInterface, MbistPipeline} +import utility.sram.{SramBroadcastBundle, SramHelper} +import xiangshan.frontend._ +import xiangshan.backend._ +import xiangshan.backend.fu.PMPRespBundle +import xiangshan.backend.trace.TraceCoreInterface +import xiangshan.mem._ +import xiangshan.cache.mmu._ +import xiangshan.cache.mmu.TlbRequestIO +import scala.collection.mutable.ListBuffer +import xiangshan._ +import examples.core.xiangshan.native.{Backend, Commands} + +abstract class XSCoreBase()(implicit p: config.Parameters) extends LazyModule with HasXSParameter { + override def shouldBeInlined: Boolean = false + // outer facing nodes + val frontend = LazyModule(new Frontend()) + val csrOut = BundleBridgeSource(Some(() => new DistributedCSRIO())) + val backend = LazyModule(new Backend(backendParams)) + + val memBlock = LazyModule(new MemBlock) + + memBlock.inner.frontendBridge.icache_node := frontend.inner.icache.clientNode + memBlock.inner.frontendBridge.instr_uncache_node := frontend.inner.instrUncache.clientNode + if (p(SoCParamsKey).EnableICacheCtrl) { + frontend.inner.icache.ctrlUnitOpt.get.node := memBlock.inner.frontendBridge.icachectrl_node + } +} + +class XSCore()(implicit p: config.Parameters) extends XSCoreBase with HasXSDts { + lazy val module = new XSCoreImp(this) +} + +class XSCoreImp(outer: XSCoreBase) extends LazyModuleImp(outer) with HasXSParameter with HasSoCParameter { + + val io = IO(new Bundle { + val hartId = Input(UInt(hartIdLen.W)) + val msiInfo = Input(ValidIO(UInt(soc.IMSICParams.MSI_INFO_WIDTH.W))) + val msiAck = Output(Bool()) + val teemsiInfo = Option.when(soc.IMSICParams.HasTEEIMSIC)(Input(ValidIO(UInt(soc.IMSICParams.MSI_INFO_WIDTH.W)))) + val teemsiAck = Option.when(soc.IMSICParams.HasTEEIMSIC)(Output(Bool())) + val commands = new Commands + val timeRequest = Decoupled(Bool()) + val clintTime = Input(ValidIO(UInt(64.W))) + val reset_vector = Input(UInt(PAddrBits.W)) + val reset_mtvec = Option.when(enableResetMtvec)(Input(UInt(PAddrBits.W))) + val cpu_halt = Output(Bool()) + val l2_flush_done = Input(Bool()) + val l2_flush_en = Output(Bool()) + val power_down_en = Output(Bool()) + val cpu_critical_error = Output(Bool()) + val resetInFrontend = Output(Bool()) + val traceCoreInterface = new TraceCoreInterface + val l2PfCtrl = Output(new PrefetchCtrlFromCore) + val perfEvents = Input(Vec(numPCntHc * coreParams.L2NBanks + 1, new PerfEvent)) + val beu_errors = Output(new XSL1BusErrors()) + val l2_hint = Input(Valid(new L2ToL1Hint())) + val l2_tlb_req = Flipped(new TlbRequestIO(nRespDups = 2)) + val l2_pmp_resp = new PMPRespBundle + val l2PfqBusy = Input(Bool()) + + val debugTopDown = new Bundle { + val robTrueCommit = Output(UInt(64.W)) + val robHeadPaddr = Valid(UInt(PAddrBits.W)) + val l2MissMatch = Input(Bool()) + val l3MissMatch = Input(Bool()) + } + + val topDownInfo = Input(new Bundle { + val l2Miss = Bool() + val l3Miss = Bool() + }) + + val dft = Option.when(hasDFT)(Input(new SramBroadcastBundle)) + val dft_reset = Option.when(hasDFT)(Input(new DFTResetSignals())) + }) + + dontTouch(io.l2_flush_done) + dontTouch(io.l2_flush_en) + dontTouch(io.power_down_en) + + println(s"FPGAPlatform:${env.FPGAPlatform} EnableDebug:${env.EnableDebug}") + + val frontend = outer.frontend.module + val backend = outer.backend.module + io.commands <> backend.io.commands + io.timeRequest <> backend.io.timeRequest + val memBlock = outer.memBlock.module + + frontend.io.hartId := memBlock.io.inner_hartId + frontend.io.reset_vector := memBlock.io.inner_reset_vector + frontend.io.softPrefetch <> memBlock.io.ifetchPrefetch + frontend.io.backend <> backend.io.frontend + frontend.io.sfence <> backend.io.frontendSfence + frontend.io.tlbCsr <> backend.io.frontendTlbCsr + frontend.io.csrCtrl <> backend.io.frontendCsrCtrl + frontend.io.fencei <> backend.io.fenceio.fencei + + backend.io.fromTop := memBlock.io.mem_to_ooo.topToBackendBypass + + require(backend.io.mem.stIn.length == memBlock.io.mem_to_ooo.stIn.length) + backend.io.mem.stIn.zip(memBlock.io.mem_to_ooo.stIn).foreach { case (sink, source) => + sink.valid := source.valid + sink.bits := 0.U.asTypeOf(sink.bits) + sink.bits.robIdx := source.bits.uop.robIdx + sink.bits.ssid := source.bits.uop.ssid + sink.bits.storeSetHit := source.bits.uop.storeSetHit + // The other signals have not been used + } + backend.io.mem.memoryViolation := memBlock.io.mem_to_ooo.memoryViolation + backend.io.mem.lsqEnqIO <> memBlock.io.ooo_to_mem.enqLsq + backend.io.mem.sqDeq := memBlock.io.mem_to_ooo.sqDeq + backend.io.mem.lqDeq := memBlock.io.mem_to_ooo.lqDeq + backend.io.mem.sqDeqPtr := memBlock.io.mem_to_ooo.sqDeqPtr + backend.io.mem.lqDeqPtr := memBlock.io.mem_to_ooo.lqDeqPtr + backend.io.mem.lqCancelCnt := memBlock.io.mem_to_ooo.lqCancelCnt + backend.io.mem.sqCancelCnt := memBlock.io.mem_to_ooo.sqCancelCnt + backend.io.mem.otherFastWakeup := memBlock.io.mem_to_ooo.otherFastWakeup + backend.io.mem.stIssuePtr := memBlock.io.mem_to_ooo.stIssuePtr + backend.io.mem.ldaIqFeedback := memBlock.io.mem_to_ooo.ldaIqFeedback + backend.io.mem.staIqFeedback := memBlock.io.mem_to_ooo.staIqFeedback + backend.io.mem.hyuIqFeedback := memBlock.io.mem_to_ooo.hyuIqFeedback + backend.io.mem.vstuIqFeedback := memBlock.io.mem_to_ooo.vstuIqFeedback + backend.io.mem.vlduIqFeedback := memBlock.io.mem_to_ooo.vlduIqFeedback + backend.io.mem.ldCancel := memBlock.io.mem_to_ooo.ldCancel + backend.io.mem.wakeup := memBlock.io.mem_to_ooo.wakeup + backend.io.mem.writebackLda <> memBlock.io.mem_to_ooo.writebackLda + backend.io.mem.writebackSta <> memBlock.io.mem_to_ooo.writebackSta + backend.io.mem.writebackHyuLda <> memBlock.io.mem_to_ooo.writebackHyuLda + backend.io.mem.writebackHyuSta <> memBlock.io.mem_to_ooo.writebackHyuSta + backend.io.mem.writebackStd <> memBlock.io.mem_to_ooo.writebackStd + backend.io.mem.writebackVldu <> memBlock.io.mem_to_ooo.writebackVldu + backend.io.mem.robLsqIO.loadMmio := memBlock.io.mem_to_ooo.lsqio.loadMmio + backend.io.mem.robLsqIO.loadMmioUop := memBlock.io.mem_to_ooo.lsqio.loadMmioUop + backend.io.mem.robLsqIO.storeMmio := memBlock.io.mem_to_ooo.lsqio.storeMmio + backend.io.mem.robLsqIO.storeMmioUop := memBlock.io.mem_to_ooo.lsqio.storeMmioUop + + // memblock error exception writeback, 1 cycle after normal writeback + backend.io.mem.s3_delayed_load_error := memBlock.io.mem_to_ooo.s3_delayed_load_error + + backend.io.mem.exceptionAddr.vaddr := memBlock.io.mem_to_ooo.lsqio.vaddr + backend.io.mem.exceptionAddr.gpaddr := memBlock.io.mem_to_ooo.lsqio.gpaddr + backend.io.mem.exceptionAddr.isForVSnonLeafPTE := memBlock.io.mem_to_ooo.lsqio.isForVSnonLeafPTE + backend.io.mem.debugLS := memBlock.io.debug_ls + backend.io.mem.lsTopdownInfo := memBlock.io.mem_to_ooo.lsTopdownInfo + backend.io.mem.lqCanAccept := memBlock.io.mem_to_ooo.lsqio.lqCanAccept + backend.io.mem.sqCanAccept := memBlock.io.mem_to_ooo.lsqio.sqCanAccept + backend.io.fenceio.sbuffer.sbIsEmpty := memBlock.io.mem_to_ooo.sbIsEmpty + + backend.io.perf.frontendInfo := frontend.io.frontendInfo + backend.io.perf.memInfo := memBlock.io.memInfo + backend.io.perf.perfEventsFrontend := frontend.io_perf + backend.io.perf.perfEventsLsu := memBlock.io_perf + backend.io.perf.perfEventsHc := memBlock.io.inner_hc_perfEvents + backend.io.perf.perfEventsBackend := DontCare + backend.io.perf.retiredInstr := DontCare + backend.io.perf.ctrlInfo := DontCare + + backend.io.mem.storeDebugInfo <> memBlock.io.mem_to_ooo.storeDebugInfo + + // top -> memBlock + memBlock.io.fromTopToBackend.clintTime := io.clintTime + memBlock.io.fromTopToBackend.msiInfo := io.msiInfo + memBlock.io.fromTopToBackend.teemsiInfo zip io.teemsiInfo foreach { case (memBlock_teemsiInfo, io_teemsiInfo) => + memBlock_teemsiInfo := io_teemsiInfo + } + memBlock.io.hartId := io.hartId + memBlock.io.l2_flush_done := io.l2_flush_done + memBlock.io.outer_reset_vector := io.reset_vector + memBlock.io.fromTopToBackend.reset_mtvec.foreach(_ := io.reset_mtvec.get) + memBlock.io.outer_hc_perfEvents := io.perfEvents + // frontend -> memBlock + memBlock.io.inner_beu_errors_icache <> frontend.io.error + memBlock.io.ooo_to_mem.backendToTopBypass := backend.io.toTop + memBlock.io.ooo_to_mem.issueLda <> backend.io.mem.issueLda + memBlock.io.ooo_to_mem.issueSta <> backend.io.mem.issueSta + memBlock.io.ooo_to_mem.issueStd <> backend.io.mem.issueStd + memBlock.io.ooo_to_mem.issueHya <> backend.io.mem.issueHylda + backend.io.mem.issueHysta.foreach(_.ready := false.B) // this fake port should not be used + memBlock.io.ooo_to_mem.issueVldu <> backend.io.mem.issueVldu + + // By default, instructions do not have exceptions when they enter the function units. + memBlock.io.ooo_to_mem.issueUops.map(_.bits.uop.clearExceptions()) + memBlock.io.ooo_to_mem.storePc := backend.io.mem.storePcRead + memBlock.io.ooo_to_mem.hybridPc := backend.io.mem.hyuPcRead + memBlock.io.ooo_to_mem.flushSb := backend.io.fenceio.sbuffer.flushSb + memBlock.io.ooo_to_mem.loadFastMatch := 0.U.asTypeOf(memBlock.io.ooo_to_mem.loadFastMatch) + memBlock.io.ooo_to_mem.loadFastImm := 0.U.asTypeOf(memBlock.io.ooo_to_mem.loadFastImm) + memBlock.io.ooo_to_mem.loadFastFuOpType := 0.U.asTypeOf(memBlock.io.ooo_to_mem.loadFastFuOpType) + + memBlock.io.ooo_to_mem.sfence <> backend.io.mem.sfence + + memBlock.io.redirect := backend.io.mem.redirect + memBlock.io.ooo_to_mem.csrCtrl := backend.io.mem.csrCtrl + memBlock.io.ooo_to_mem.tlbCsr := backend.io.mem.tlbCsr + memBlock.io.ooo_to_mem.lsqio.lcommit := backend.io.mem.robLsqIO.lcommit + memBlock.io.ooo_to_mem.lsqio.scommit := backend.io.mem.robLsqIO.scommit + memBlock.io.ooo_to_mem.lsqio.pendingMMIOld := backend.io.mem.robLsqIO.pendingMMIOld + memBlock.io.ooo_to_mem.lsqio.pendingld := backend.io.mem.robLsqIO.pendingld + memBlock.io.ooo_to_mem.lsqio.pendingst := backend.io.mem.robLsqIO.pendingst + memBlock.io.ooo_to_mem.lsqio.pendingVst := backend.io.mem.robLsqIO.pendingVst + memBlock.io.ooo_to_mem.lsqio.commit := backend.io.mem.robLsqIO.commit + memBlock.io.ooo_to_mem.lsqio.pendingPtr := backend.io.mem.robLsqIO.pendingPtr + memBlock.io.ooo_to_mem.lsqio.pendingPtrNext := backend.io.mem.robLsqIO.pendingPtrNext + memBlock.io.ooo_to_mem.isStoreException := backend.io.mem.isStoreException + memBlock.io.ooo_to_mem.isVlsException := backend.io.mem.isVlsException + + memBlock.io.fetch_to_mem.itlb <> frontend.io.ptw + memBlock.io.l2_hint.valid := io.l2_hint.valid + memBlock.io.l2_hint.bits.sourceId := io.l2_hint.bits.sourceId + memBlock.io.l2_tlb_req <> io.l2_tlb_req + memBlock.io.l2_pmp_resp <> io.l2_pmp_resp + memBlock.io.l2_hint.bits.isKeyword := io.l2_hint.bits.isKeyword + memBlock.io.l2PfqBusy := io.l2PfqBusy + + // if l2 prefetcher use stream prefetch, it should be placed in XSCore + + // top-down info + memBlock.io.debugTopDown.robHeadVaddr := backend.io.debugTopDown.fromRob.robHeadVaddr + frontend.io.debugTopDown.robHeadVaddr := backend.io.debugTopDown.fromRob.robHeadVaddr + io.debugTopDown.robHeadPaddr := backend.io.debugTopDown.fromRob.robHeadPaddr + io.debugTopDown.robTrueCommit := backend.io.debugRolling.robTrueCommit + backend.io.debugTopDown.fromCore.l2MissMatch := io.debugTopDown.l2MissMatch + backend.io.debugTopDown.fromCore.l3MissMatch := io.debugTopDown.l3MissMatch + backend.io.debugTopDown.fromCore.fromMem := memBlock.io.debugTopDown.toCore + memBlock.io.debugRolling := backend.io.debugRolling + + io.cpu_halt := memBlock.io.outer_cpu_halt + io.l2_flush_en := memBlock.io.outer_l2_flush_en + io.power_down_en := memBlock.io.outer_power_down_en + io.cpu_critical_error := memBlock.io.outer_cpu_critical_error + io.msiAck := memBlock.io.outer_msi_ack + io.teemsiAck zip memBlock.io.outer_teemsi_ack foreach { case (io_teemsiAck, memBlock_outer_teemsi_ack) => + io_teemsiAck := memBlock_outer_teemsi_ack + } + io.beu_errors.icache <> memBlock.io.outer_beu_errors_icache + io.beu_errors.dcache <> memBlock.io.dcacheError + io.beu_errors.uncache <> memBlock.io.uncacheError + io.beu_errors.l2 <> DontCare + io.l2PfCtrl := memBlock.io.outer_l2PfCtrl + + memBlock.io.resetInFrontendBypass.fromFrontend := frontend.io.resetInFrontend + io.resetInFrontend := memBlock.io.resetInFrontendBypass.toL2Top + memBlock.io.traceCoreInterfaceBypass.fromBackend <> backend.io.traceCoreInterface + io.traceCoreInterface <> memBlock.io.traceCoreInterfaceBypass.toL2Top + memBlock.io.wfi <> backend.io.mem.wfi + memBlock.io.topDownInfo.fromL2Top.l2Miss := io.topDownInfo.l2Miss + memBlock.io.topDownInfo.fromL2Top.l3Miss := io.topDownInfo.l3Miss + backend.io.topDownInfo.replayAllocate := memBlock.io.topDownInfo.toBackend.replayAllocate + backend.io.topDownInfo.sqFull := memBlock.io.topDownInfo.toBackend.sqFull + backend.io.topDownInfo.sbFull := memBlock.io.topDownInfo.toBackend.sbFull + backend.io.topDownInfo.l1Miss := memBlock.io.topDownInfo.toBackend.l1Miss + backend.io.topDownInfo.l2TopMiss.l2Miss := memBlock.io.topDownInfo.toBackend.l2TopMiss.l2Miss + backend.io.topDownInfo.l2TopMiss.l3Miss := memBlock.io.topDownInfo.toBackend.l2TopMiss.l3Miss + + if (debugOpts.ResetGen) { + backend.reset := memBlock.io.reset_backend + frontend.reset := backend.io.frontendReset + } + + memBlock.io.dft.zip(io.dft).foreach({ case (a, b) => a := b }) + memBlock.io.dft_reset.zip(io.dft_reset).foreach({ case (a, b) => a := b }) + frontend.io.dft.zip(memBlock.io.dft_frnt).foreach({ case (a, b) => a := b }) + frontend.io.dft_reset.zip(memBlock.io.dft_reset_frnt).foreach({ case (a, b) => a := b }) + backend.io.dft.zip(memBlock.io.dft_bcknd).foreach({ case (a, b) => a := b }) + backend.io.dft_reset.zip(memBlock.io.dft_reset_bcknd).foreach({ case (a, b) => a := b }) +} + +trait HasXSDts { + this: XSCore => + + val device: SimpleDevice = new SimpleDevice("cpu", Seq("ICT,xiangshan", "riscv")) { + override def parent: Some[Device] = Some(ResourceAnchors.cpus) + + def cpuProperties: PropertyMap = Map( + "device_type" -> "cpu".asProperty, + "status" -> "okay".asProperty, + "clock-frequency" -> 0.asProperty, + "riscv,isa" -> "rv64imafdcvh".asProperty, // deprecated + "riscv,isa-base" -> ISABase.asProperty, + "riscv,isa-extensions" -> ISAExtensions.map(ResourceString), + "timebase-frequency" -> 1000000.asProperty + ) + + def tileProperties: PropertyMap = { + val dcache = + if (coreParams.dcacheParametersOpt.nonEmpty) Map( + "d-cache-block-size" -> dcacheParameters.blockBytes.asProperty, + "d-cache-sets" -> dcacheParameters.nSets.asProperty, + "d-cache-size" -> (dcacheParameters.nSets * dcacheParameters.nWays * dcacheParameters.blockBytes).asProperty + ) + else Map() + + val icache = Map( + "i-cache-block-size" -> icacheParameters.blockBytes.asProperty, + "i-cache-sets" -> icacheParameters.nSets.asProperty, + "i-cache-size" -> (icacheParameters.nSets * icacheParameters.nWays * icacheParameters.blockBytes).asProperty + ) + + val dtlb = Map( + "d-tlb-size" -> (ldtlbParams.NSets * ldtlbParams.NWays).asProperty, + "d-tlb-sets" -> 1.asProperty + ) + + val itlb = Map( + "i-tlb-size" -> (itlbParams.NSets * itlbParams.NWays).asProperty, + "i-tlb-sets" -> 1.asProperty + ) + + val mmu = Map( + "tlb-split" -> Nil, + "mmu-type" -> s"riscv,sv$VAddrBits".asProperty + ) + + val pmp = Nil + + dcache ++ icache ++ dtlb ++ itlb ++ mmu ++ pmp + } + + def nextLevelCacheProperty: PropertyOption = { + if (coreParams.dcacheParametersOpt.nonEmpty) { + val outer = memBlock.inner.dcache.clientNode.edges.out.flatMap(_.manager.managers) + .filter(_.supportsAcquireB) + .flatMap(_.resources.headOption) + .map(_.owner.label) + .distinct + if (outer.isEmpty) None + else Some("next-level-cache" -> outer.map(l => ResourceReference(l)).toList) + } else None + } + + override def describe(resources: ResourceBindings): Description = { + val Description(name, mapping) = super.describe(resources) + Description(name, mapping ++ cpuProperties ++ nextLevelCacheProperty ++ tileProperties) + } + + } + + val intcDevice = new DeviceSnippet { + override def parent = Some(device) + + def describe(): Description = { + Description( + "interrupt-controller", + Map( + "compatible" -> "riscv,cpu-intc".asProperty, + "interrupt-controller" -> Nil, + "#interrupt-cells" -> 1.asProperty + ) + ) + } + + } + + ResourceBinding { + Resource(device, "reg").bind(ResourceAddress(coreParams.HartId)) + val int_resources = ( + memBlock.inner.clint_int_sink.edges.in.flatMap(_.source.sources) ++ + memBlock.inner.plic_int_sink.edges.in.flatMap(_.source.sources) ++ + memBlock.inner.debug_int_sink.edges.in.flatMap(_.source.sources) ++ + memBlock.inner.nmi_int_sink.edges.in.flatMap(_.source.sources) + ).flatMap { + s => + (s.range.start until s.range.`end`).map(_ => s.resources) + } + val int_ids = Seq( + 3, // msip [clint] + 7, // mtip [clint] + 11, // meip [plic] + 9, // seip [plic] + 65535, // debug [debug] + 31, // nmi_31 [nmi] + 43 // nmi_43 [nmi] + ) + assert(int_resources.size == int_ids.size) + for ((resources, id) <- int_resources.zip(int_ids)) { + for (r <- resources) { + r.bind(intcDevice, ResourceInt(id)) + } + } + } +} diff --git a/examples/prefab/cores/xiangshan/arch/native/CtrlBlock.scala b/examples/prefab/cores/xiangshan/arch/native/CtrlBlock.scala new file mode 100644 index 000000000..5a61c5e05 --- /dev/null +++ b/examples/prefab/cores/xiangshan/arch/native/CtrlBlock.scala @@ -0,0 +1,926 @@ +// Copied from XiangShan src/main/scala/xiangshan/backend/CtrlBlock.scala and modified for Buckyball. +// XiangShan is licensed under Mulan PSL v2 (http://license.coscl.org.cn/MulanPSL2). + +package examples.core.xiangshan.native + +import org.chipsalliance.cde.config.Parameters +import chisel3._ +import chisel3.util._ +import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp} +import utility._ +import utils._ +import xiangshan.ExceptionNO._ +import xiangshan._ +import xiangshan.backend.Bundles.{DecodedInst, DynInst, ExceptionInfo, ExuOutput, ExuVec, StaticInst, TrapInstInfo} +import xiangshan.backend.ctrlblock.{DebugLSIO, DebugLsInfoBundle, LsTopdownInfo, MemCtrl, RedirectGenerator} +import xiangshan.backend.datapath.DataConfig.{FpData, IntData, V0Data, VAddrData, VecData, VlData} +import xiangshan.backend.decode.FusionDecoder +import xiangshan.backend.dispatch.{CoreDispatchTopDownIO} +import xiangshan.backend.dispatch.NewDispatch +import xiangshan.backend.fu.vector.Bundles.{VType, Vl} +import xiangshan.backend.fu.wrapper.CSRToDecode +import xiangshan.backend.rename.{Rename, RenameTableWrapper, SnapshotGenerator} +import xiangshan.backend.rob.{Rob, RobCSRIO, RobCoreTopDownIO, RobDebugRollingIO, RobLsqIO, RobPtr} +import xiangshan.frontend.{FtqPcMemEntry, FtqPtr, FtqRead} +import xiangshan.mem.{LqPtr, LsqEnqIO, SqPtr} +import xiangshan.backend.issue.{FpScheduler, IntScheduler, MemScheduler, VfScheduler} +import xiangshan.backend.trace._ +import xiangshan.Redirect.findOldestRedirect +import xiangshan.backend._ +import examples.core.xiangshan.native.{DecodeStage} + +class CtrlBlock(params: BackendParams)(implicit p: Parameters) extends LazyModule { + override def shouldBeInlined: Boolean = false + + val rob = LazyModule(new Rob(params)) + + lazy val module = new CtrlBlockImp(this)(p, params) + + val gpaMem = LazyModule(new GPAMem()) +} + +class CtrlBlockImp( + override val wrapper: CtrlBlock +)( + implicit + p: Parameters, + params: BackendParams) + extends LazyModuleImp(wrapper) + with HasXSParameter + with HasCircularQueuePtrHelper + with HasPerfEvents + with HasCriticalErrors { + + val pcMemRdIndexes = new NamedIndexes(Seq( + "redirect" -> 1, + "memPred" -> 1, + "robFlush" -> 1, + "executePc" -> params.numPcMemReadPort, + "bjuTarget" -> params.BrhCnt, + "hybrid" -> params.HyuCnt, + "store" -> (if (EnableStorePrefetchSMS) params.StaCnt else 0), + "trace" -> TraceGroupNum + )) + + private val numPcMemRead = pcMemRdIndexes.maxIdx + + // now pcMem read for exu is moved to PcTargetMem (OG0) + println(s"pcMem read num: $numPcMemRead") + + val io = IO(new CtrlBlockIO()) + + val dispatch = Module(new NewDispatch) + val gpaMem = wrapper.gpaMem.module + val decode = Module(new DecodeStage) + val fusionDecoder = Module(new FusionDecoder) + val rat = Module(new RenameTableWrapper) + val rename = Module(new Rename) + val redirectGen = Module(new RedirectGenerator) + private def hasRen: Boolean = true + private val pcMem = + Module(new SyncDataModuleTemplate(new FtqPcMemEntry, FtqSize, numPcMemRead, 1, "BackendPC", hasRen = hasRen)) + private val rob = wrapper.rob.module + private val memCtrl = Module(new MemCtrl(params)) + + private val disableFusion = decode.io.csrCtrl.singlestep || !decode.io.csrCtrl.fusion_enable + + private val s0_robFlushRedirect = rob.io.flushOut + private val s1_robFlushRedirect = Wire(Valid(new Redirect)) + s1_robFlushRedirect.valid := GatedValidRegNext(s0_robFlushRedirect.valid, false.B) + s1_robFlushRedirect.bits := RegEnable(s0_robFlushRedirect.bits, s0_robFlushRedirect.valid) + + pcMem.io.ren.get(pcMemRdIndexes("robFlush").head) := s0_robFlushRedirect.valid + pcMem.io.raddr(pcMemRdIndexes("robFlush").head) := s0_robFlushRedirect.bits.ftqIdx.value + + private val s1_robFlushPc = pcMem.io.rdata(pcMemRdIndexes("robFlush").head).startAddr + (RegEnable( + s0_robFlushRedirect.bits.ftqOffset, + s0_robFlushRedirect.valid + ) << instOffsetBits) + + private val s3_redirectGen = redirectGen.io.stage2Redirect + private val s1_s3_redirect = Mux(s1_robFlushRedirect.valid, s1_robFlushRedirect, s3_redirectGen) + private val s2_s4_pendingRedirectValid = RegInit(false.B) + when(s1_s3_redirect.valid) { + s2_s4_pendingRedirectValid := true.B + }.elsewhen(GatedValidRegNext(io.frontend.toFtq.redirect.valid)) { + s2_s4_pendingRedirectValid := false.B + } + + // Redirect will be RegNext at ExuBlocks and IssueBlocks + val s2_s4_redirect = RegNextWithEnable(s1_s3_redirect) + val s3_s5_redirect = RegNextWithEnable(s2_s4_redirect) + + private val delayedNotFlushedWriteBack = io.fromWB.wbData.map { x => + val valid = x.valid + val killedByOlder = x.bits.robIdx.needFlush(Seq(s1_s3_redirect, s2_s4_redirect)) + val delayed = Wire(Valid(new ExuOutput(x.bits.params))) + delayed.valid := GatedValidRegNext(valid && !killedByOlder) + delayed.bits := RegEnable(x.bits, x.valid) + delayed.bits.debugInfo.writebackTime := GTimer() + delayed + }.toSeq + + private val delayedWriteBack = Wire(chiselTypeOf(io.fromWB.wbData)) + delayedWriteBack.zipWithIndex.map { case (x, i) => + x.valid := GatedValidRegNext(io.fromWB.wbData(i).valid) + x.bits := delayedNotFlushedWriteBack(i).bits + } + val delayedNotFlushedWriteBackNeedFlush = Wire(Vec(params.allExuParams.filter(_.needExceptionGen).length, Bool())) + delayedNotFlushedWriteBackNeedFlush := delayedNotFlushedWriteBack.filter(_.bits.params.needExceptionGen).map { x => + x.bits.exceptionVec.get.asUInt.orR || x.bits.flushPipe.getOrElse(false.B) || x.bits.replay.getOrElse(false.B) || + (if (x.bits.trigger.nonEmpty) TriggerAction.isDmode(x.bits.trigger.get) else false.B) + } + + val wbDataNoStd = io.fromWB.wbData.filter(!_.bits.params.hasStdFu) + val intScheWbData = io.fromWB.wbData.filter(_.bits.params.schdType.isInstanceOf[IntScheduler]) + val fpScheWbData = io.fromWB.wbData.filter(_.bits.params.schdType.isInstanceOf[FpScheduler]) + val vfScheWbData = io.fromWB.wbData.filter(_.bits.params.schdType.isInstanceOf[VfScheduler]) + val intCanCompress = intScheWbData.filter(_.bits.params.CanCompress) + val i2vWbData = intScheWbData.filter(_.bits.params.writeVecRf) + val f2vWbData = fpScheWbData.filter(_.bits.params.writeVecRf) + val memVloadWbData = + io.fromWB.wbData.filter(x => x.bits.params.schdType.isInstanceOf[MemScheduler] && x.bits.params.hasVLoadFu) + + private val delayedNotFlushedWriteBackNums = wbDataNoStd.map { x => + val valid = x.valid + val oldestRedirect = findOldestRedirect(findOldestRedirect(s2_s4_redirect, s3_s5_redirect), s1_s3_redirect) + val killedByOlder = x.bits.robIdx.needFlush(oldestRedirect) + val delayed = Wire(Valid(UInt(io.fromWB.wbData.size.U.getWidth.W))) + delayed.valid := GatedValidRegNext(valid && !killedByOlder) + val isIntSche = intCanCompress.contains(x) + val isFpSche = fpScheWbData.contains(x) + val isVfSche = vfScheWbData.contains(x) + val isMemVload = memVloadWbData.contains(x) + val isi2v = i2vWbData.contains(x) + val isf2v = f2vWbData.contains(x) + val canSameRobidxWbData = + if (isVfSche) { + i2vWbData ++ f2vWbData ++ vfScheWbData + } else if (isi2v) { + intCanCompress ++ fpScheWbData ++ vfScheWbData + } else if (isf2v) { + intCanCompress ++ fpScheWbData ++ vfScheWbData + } else if (isIntSche) { + intCanCompress ++ fpScheWbData + } else if (isFpSche) { + intCanCompress ++ fpScheWbData + } else if (isMemVload) { + memVloadWbData + } else { + Seq(x) + } + val sameRobidxBools = VecInit(canSameRobidxWbData.map { wb => + val killedByOlderThat = wb.bits.robIdx.needFlush(oldestRedirect) + (wb.bits.robIdx === x.bits.robIdx) && wb.valid && x.valid && !killedByOlderThat && !killedByOlder + }.toSeq) + delayed.bits := RegEnable(PopCount(sameRobidxBools), x.valid) + delayed + }.toSeq + + private val exuPredecode = VecInit( + io.fromWB.wbData.filter(_.bits.redirect.nonEmpty).map(x => x.bits.predecodeInfo.get).toSeq + ) + + private val exuRedirects: Seq[ValidIO[Redirect]] = io.fromWB.wbData.filter(_.bits.redirect.nonEmpty).map { x => + val hasCSR = x.bits.params.hasCSR + val out = Wire(Valid(new Redirect())) + out.valid := x.valid && x.bits.redirect.get.valid && !x.bits.robIdx.needFlush(Seq(s1_s3_redirect, s2_s4_redirect)) + out.bits := x.bits.redirect.get.bits + out.bits.debugIsCtrl := true.B + out.bits.debugIsMemVio := false.B + // for fix timing, next cycle assgin + if (!hasCSR) { + out.bits.cfiUpdate.backendIAF := false.B + out.bits.cfiUpdate.backendIPF := false.B + out.bits.cfiUpdate.backendIGPF := false.B + } + out + }.toSeq + + private val oldestOneHot = Redirect.selectOldestRedirect(exuRedirects) + private val CSROH = VecInit(io.fromWB.wbData.filter(_.bits.redirect.nonEmpty).map(x => x.bits.params.hasCSR.B)) + private val oldestExuRedirectIsCSR = oldestOneHot === CSROH + private val oldestExuRedirect = Mux1H(oldestOneHot, exuRedirects) + private val oldestExuPredecode = Mux1H(oldestOneHot, exuPredecode) + + private val memViolation = io.fromMem.violation + val loadReplay = Wire(ValidIO(new Redirect)) + loadReplay.valid := GatedValidRegNext(memViolation.valid) + loadReplay.bits := RegEnable(memViolation.bits, memViolation.valid) + loadReplay.bits.debugIsCtrl := false.B + loadReplay.bits.debugIsMemVio := true.B + + pcMem.io.ren.get(pcMemRdIndexes("redirect").head) := memViolation.valid + pcMem.io.raddr(pcMemRdIndexes("redirect").head) := memViolation.bits.ftqIdx.value + pcMem.io.ren.get(pcMemRdIndexes("memPred").head) := memViolation.valid + pcMem.io.raddr(pcMemRdIndexes("memPred").head) := memViolation.bits.stFtqIdx.value + redirectGen.io.memPredPcRead.data := pcMem.io.rdata(pcMemRdIndexes("memPred").head).startAddr + (RegEnable( + memViolation.bits.stFtqOffset, + memViolation.valid + ) << instOffsetBits) + + for ((pcMemIdx, i) <- pcMemRdIndexes("executePc").zipWithIndex) { + val ren = io.toDataPath.pcToDataPathIO.fromDataPathValid(i) + val raddr = io.toDataPath.pcToDataPathIO.fromDataPathFtqPtr(i).value + val roffset = io.toDataPath.pcToDataPathIO.fromDataPathFtqOffset(i) + pcMem.io.ren.get(pcMemIdx) := ren + pcMem.io.raddr(pcMemIdx) := raddr + io.toDataPath.pcToDataPathIO.toDataPathPC(i) := pcMem.io.rdata(pcMemIdx).startAddr + } + + val newestEn = RegNext(io.frontend.fromFtq.newest_entry_en) + val newestTarget = RegEnable(io.frontend.fromFtq.newest_entry_target, io.frontend.fromFtq.newest_entry_en) + val newestPtr = RegEnable(io.frontend.fromFtq.newest_entry_ptr, io.frontend.fromFtq.newest_entry_en) + val newestTargetNext = RegEnable(newestTarget, newestEn) + for ((pcMemIdx, i) <- pcMemRdIndexes("bjuTarget").zipWithIndex) { + val ren = io.toDataPath.pcToDataPathIO.fromDataPathValid(i) + val baseAddr = io.toDataPath.pcToDataPathIO.fromDataPathFtqPtr(i).value + val raddr = io.toDataPath.pcToDataPathIO.fromDataPathFtqPtr(i).value + 1.U + pcMem.io.ren.get(pcMemIdx) := ren + pcMem.io.raddr(pcMemIdx) := raddr + val needNewest = RegNext(baseAddr === newestPtr.value) + io.toDataPath.pcToDataPathIO.toDataPathTargetPC(i) := Mux( + needNewest, + newestTargetNext, + pcMem.io.rdata(pcMemIdx).startAddr + ) + } + + for ((pcMemIdx, i) <- pcMemRdIndexes("hybrid").zipWithIndex) { + // load read pcMem (s0) -> get rdata (s1) -> reg next in Memblock (s2) -> reg next in Memblock (s3) -> consumed by + // pf (s3) + pcMem.io.ren.get(pcMemIdx) := io.memHyPcRead(i).valid + pcMem.io.raddr(pcMemIdx) := io.memHyPcRead(i).ptr.value + io.memHyPcRead(i).data := pcMem.io.rdata(pcMemIdx).startAddr + (RegEnable( + io.memHyPcRead(i).offset, + io.memHyPcRead(i).valid + ) << instOffsetBits) + } + + if (EnableStorePrefetchSMS) { + for ((pcMemIdx, i) <- pcMemRdIndexes("store").zipWithIndex) { + pcMem.io.ren.get(pcMemIdx) := io.memStPcRead(i).valid + pcMem.io.raddr(pcMemIdx) := io.memStPcRead(i).ptr.value + io.memStPcRead(i).data := pcMem.io.rdata(pcMemIdx).startAddr + (RegEnable( + io.memStPcRead(i).offset, + io.memStPcRead(i).valid + ) << instOffsetBits) + } + } else { + io.memStPcRead.foreach(_.data := 0.U) + } + + /** + * trace begin + */ + val trace = Module(new Trace) + trace.io.in.fromEncoder.stall := io.traceCoreInterface.fromEncoder.stall + trace.io.in.fromEncoder.enable := io.traceCoreInterface.fromEncoder.enable + trace.io.in.fromRob := rob.io.trace.traceCommitInfo + rob.io.trace.blockCommit := trace.io.out.blockRobCommit + val tracePcStart = Wire(Vec(TraceGroupNum, UInt(IaddrWidth.W))) + for ((pcMemIdx, i) <- pcMemRdIndexes("trace").zipWithIndex) { + val traceValid = trace.toPcMem.blocks(i).valid + pcMem.io.ren.get(pcMemIdx) := traceValid + pcMem.io.raddr(pcMemIdx) := trace.toPcMem.blocks(i).bits.ftqIdx.get.value + tracePcStart(i) := pcMem.io.rdata(pcMemIdx).startAddr + } + + // Trap/Xret only occur in block(0). + val tracePriv = Mux( + Itype.isTrapOrXret(trace.toEncoder.blocks(0).bits.tracePipe.itype), + io.fromCSR.traceCSR.lastPriv, + io.fromCSR.traceCSR.currentPriv + ) + + io.traceCoreInterface.toEncoder.trap.cause := io.fromCSR.traceCSR.cause + io.traceCoreInterface.toEncoder.trap.tval := io.fromCSR.traceCSR.tval + io.traceCoreInterface.toEncoder.priv := tracePriv + io.traceCoreInterface.toEncoder.mstatus := io.fromCSR.traceCSR.mstatus + (0 until TraceGroupNum).foreach { i => + io.traceCoreInterface.toEncoder.groups(i).valid := trace.io.out.toEncoder.blocks(i).valid + io.traceCoreInterface.toEncoder.groups(i).bits.iaddr := tracePcStart(i) + io.traceCoreInterface.toEncoder.groups(i).bits.ftqOffset.foreach( + _ := trace.io.out.toEncoder.blocks(i).bits.ftqOffset.getOrElse(0.U) + ) + io.traceCoreInterface.toEncoder.groups(i).bits.itype := trace.io.out.toEncoder.blocks(i).bits.tracePipe.itype + io.traceCoreInterface.toEncoder.groups(i).bits.iretire := trace.io.out.toEncoder.blocks(i).bits.tracePipe.iretire + io.traceCoreInterface.toEncoder.groups(i).bits.ilastsize := trace.io.out.toEncoder.blocks( + i + ).bits.tracePipe.ilastsize + } + + /** + * trace end + */ + + redirectGen.io.hartId := io.fromTop.hartId + redirectGen.io.oldestExuRedirect.valid := RegNext(oldestExuRedirect.valid) + redirectGen.io.oldestExuRedirect.bits := RegNext(oldestExuRedirect.bits) + redirectGen.io.oldestExuRedirectIsCSR := RegNext(oldestExuRedirectIsCSR) + redirectGen.io.instrAddrTransType := RegNext(io.fromCSR.instrAddrTransType) + redirectGen.io.oldestExuOutPredecode.valid := RegNext(oldestExuPredecode.valid) + redirectGen.io.oldestExuOutPredecode := RegNext(oldestExuPredecode) + redirectGen.io.loadReplay <> loadReplay + val loadRedirectOffset = Mux(memViolation.bits.flushItself(), 0.U, Mux(memViolation.bits.isRVC, 2.U, 4.U)) + val loadRedirectPcFtqOffset = + RegEnable((memViolation.bits.ftqOffset << instOffsetBits).asUInt +& loadRedirectOffset, memViolation.valid) + val loadRedirectPcRead = pcMem.io.rdata(pcMemRdIndexes("redirect").head).startAddr + loadRedirectPcFtqOffset + + redirectGen.io.loadReplay.bits.cfiUpdate.pc := loadRedirectPcRead + val load_target = loadRedirectPcRead + redirectGen.io.loadReplay.bits.cfiUpdate.target := SignExt(load_target, VAddrBits + 1) + + redirectGen.io.robFlush := s1_robFlushRedirect + + val s5_flushFromRobValidAhead = DelayN(s1_robFlushRedirect.valid, 4) + val s6_flushFromRobValid = GatedValidRegNext(s5_flushFromRobValidAhead) + val frontendFlushBits = RegEnable(s1_robFlushRedirect.bits, s1_robFlushRedirect.valid) // ?? + // When ROB commits an instruction with a flush, we notify the frontend of the flush without the commit. + // Flushes to frontend may be delayed by some cycles and commit before flush causes errors. + // Thus, we make all flush reasons to behave the same as exceptions for frontend. + for (i <- 0 until CommitWidth) { + // why flushOut: instructions with flushPipe are not commited to frontend + // If we commit them to frontend, it will cause flush after commit, which is not acceptable by frontend. + val s1_isCommit = rob.io.commits.commitValid(i) && rob.io.commits.isCommit && !s0_robFlushRedirect.valid + io.frontend.toFtq.rob_commits(i).valid := GatedValidRegNext(s1_isCommit) + io.frontend.toFtq.rob_commits(i).bits := RegEnable(rob.io.commits.info(i), s1_isCommit) + } + io.frontend.toFtq.redirect.valid := s6_flushFromRobValid || s3_redirectGen.valid + io.frontend.toFtq.redirect.bits := Mux(s6_flushFromRobValid, frontendFlushBits, s3_redirectGen.bits) + io.frontend.toFtq.ftqIdxSelOH.valid := s6_flushFromRobValid || redirectGen.io.stage2Redirect.valid + io.frontend.toFtq.ftqIdxSelOH.bits := Cat( + s6_flushFromRobValid, + redirectGen.io.stage2oldestOH & Fill(NumRedirect + 1, !s6_flushFromRobValid) + ) + + //jmp/brh, sel oldest first, only use one read port + io.frontend.toFtq.ftqIdxAhead(0).valid := RegNext( + oldestExuRedirect.valid + ) && !s1_robFlushRedirect.valid && !s5_flushFromRobValidAhead + io.frontend.toFtq.ftqIdxAhead(0).bits := RegEnable(oldestExuRedirect.bits.ftqIdx, oldestExuRedirect.valid) + //loadreplay + io.frontend.toFtq.ftqIdxAhead( + NumRedirect + ).valid := loadReplay.valid && !s1_robFlushRedirect.valid && !s5_flushFromRobValidAhead + io.frontend.toFtq.ftqIdxAhead(NumRedirect).bits := loadReplay.bits.ftqIdx + //exception + io.frontend.toFtq.ftqIdxAhead.last.valid := s5_flushFromRobValidAhead + io.frontend.toFtq.ftqIdxAhead.last.bits := frontendFlushBits.ftqIdx + + // Be careful here: + // T0: rob.io.flushOut, s0_robFlushRedirect + // T1: s1_robFlushRedirect, rob.io.exception.valid + // T2: csr.redirect.valid + // T3: csr.exception.valid + // T4: csr.trapTarget + // T5: ctrlBlock.trapTarget + // T6: io.frontend.toFtq.stage2Redirect.valid + val s2_robFlushPc = RegEnable( + Mux( + s1_robFlushRedirect.bits.flushItself(), + s1_robFlushPc, // replay inst + s1_robFlushPc + Mux(s1_robFlushRedirect.bits.isRVC, 2.U, 4.U) // flush pipe + ), + s1_robFlushRedirect.valid + ) + + private val s5_csrIsTrap = DelayN(rob.io.exception.valid, 4) + private val s5_trapTargetFromCsr = io.robio.csr.trapTarget + + val flushTarget = Mux(s5_csrIsTrap, s5_trapTargetFromCsr.pc, SignExt(s2_robFlushPc, XLEN)) + val s5_trapTargetIAF = Mux(s5_csrIsTrap, s5_trapTargetFromCsr.raiseIAF, false.B) + val s5_trapTargetIPF = Mux(s5_csrIsTrap, s5_trapTargetFromCsr.raiseIPF, false.B) + val s5_trapTargetIGPF = Mux(s5_csrIsTrap, s5_trapTargetFromCsr.raiseIGPF, false.B) + when(s6_flushFromRobValid) { + io.frontend.toFtq.redirect.bits.level := RedirectLevel.flush + io.frontend.toFtq.redirect.bits.cfiUpdate.target := RegEnable(flushTarget, s5_flushFromRobValidAhead) + io.frontend.toFtq.redirect.bits.cfiUpdate.backendIAF := RegEnable(s5_trapTargetIAF, s5_flushFromRobValidAhead) + io.frontend.toFtq.redirect.bits.cfiUpdate.backendIPF := RegEnable(s5_trapTargetIPF, s5_flushFromRobValidAhead) + io.frontend.toFtq.redirect.bits.cfiUpdate.backendIGPF := RegEnable(s5_trapTargetIGPF, s5_flushFromRobValidAhead) + } + + for (i <- 0 until DecodeWidth) { + gpaMem.io.fromIFU := io.frontend.fromIfu + gpaMem.io.exceptionReadAddr.valid := rob.io.readGPAMemAddr.valid + gpaMem.io.exceptionReadAddr.bits.ftqPtr := rob.io.readGPAMemAddr.bits.ftqPtr + gpaMem.io.exceptionReadAddr.bits.ftqOffset := rob.io.readGPAMemAddr.bits.ftqOffset + } + + // vtype commit + decode.io.fromCSR := io.fromCSR.toDecode + decode.io.fromRob.isResumeVType := rob.io.toDecode.isResumeVType + decode.io.fromRob.walkToArchVType := rob.io.toDecode.walkToArchVType + decode.io.fromRob.commitVType := rob.io.toDecode.commitVType + decode.io.fromRob.walkVType := rob.io.toDecode.walkVType + + decode.io.redirect := s1_s3_redirect.valid || s2_s4_pendingRedirectValid + + // add decode Buf for in.ready better timing + /** + * Decode buffer: when decode.io.in cannot accept all insts, use this buffer to temporarily store insts that cannot + * be sent to DecodeStage. + * + * Decode buffer is a "DecodeWidth"-element long register Vector of StaticInst (in decodeBufBits), with valid signals + * (in decodeBufValid). At the same time, fetch insts input from frontend and their valid bits. All valid elements + * in these two vector of insts are at the beginning, with all invalid vector elements followed. + * + * After dealing with redirection, try to use all insts in decode buffer to fulfill decoder.io.in. If decode buffer + * has no valid insts, use insts from frontend to supply decoder. + */ + + /** Insts to be decoded, Registers in vector of DecodeWidth */ + val decodeBufBits = Reg(Vec(DecodeWidth, new StaticInst)) + + /** Valid receiving signals of instructions to be decoded, Registers in vector of DecodeWidth */ + val decodeBufValid = RegInit(VecInit(Seq.fill(DecodeWidth)(false.B))) + + /** Insts input from frontend, in vector of DecodeWidth */ + val decodeFromFrontend = io.frontend.cfVec + + /** Insts in buffer that is not ready but valid in decodeBufValid */ + val decodeBufNotAccept = VecInit(decodeBufValid.zip(decode.io.in).map(x => x._1 && !x._2.ready)) + + /** Number of insts in decode buffer that is accepted. All accepted insts are before the first unaccepted one. */ + val decodeBufAcceptNum = PriorityMuxDefault(decodeBufNotAccept.zip(Seq.tabulate(DecodeWidth)(i => i.U)), DecodeWidth.U) + + /** Input valid insts from frontend that is not ready to be accepted, or decoder prefer insts in decode buffer */ + val decodeFromFrontendNotAccept = + VecInit(decodeFromFrontend.zip(decode.io.in).map(x => decodeBufValid(0) || x._1.valid && !x._2.ready)) + + /** + * Number of input insts that is accepted. + * All accepted insts are before the first unaccepted one. + */ + val decodeFromFrontendAcceptNum = + PriorityMuxDefault(decodeFromFrontendNotAccept.zip(Seq.tabulate(DecodeWidth)(i => i.U)), DecodeWidth.U) + + if (backendParams.debugEn) { + dontTouch(decodeBufNotAccept) + dontTouch(decodeBufAcceptNum) + dontTouch(decodeFromFrontendNotAccept) + dontTouch(decodeFromFrontendAcceptNum) + } + + /** + * State machine of "decodeBufValid(i)": + * redirect || decodeBufValid(i) is the last accepted instr in decodeBuf: + * false + * decodeBufValid(i) is true, decodeBufNotAccept.drop(i) has some true signals + * (decodeBufAcceptNum > DecodeWidth-1-i) ? false + * if not : decodeBufValid(i+decodeBufAcceptNum) + * Pop "decodeBufAcceptNum" insts out of the decodeBufValid, and move others forward + * decodeBufValid(0) is false, decodeFromFrontendNotAccept.drop(i) has some true signals + * (decodeFromFrontendAcceptNum > DecodeWidth-1-i) ? false + * if not : decodeFromFrontend(i+decodeFromFrontendAcceptNum).valid + * Get first "decodeFromFrontendAcceptNum" insts from decodeFromFrontend, and move others to decodeBufValid + * + * State machine of "decodeBufBits(i)": + * decodeBufValid(i) is true, decodeBufNotAccept.drop(i) has some true signals + * decodeBufBits(i+decodeBufAcceptNum) + * decodeBufValid(0) is false, decodeFromFrontendNotAccept.drop(i) has some true signals + * decodeFromFrontend(i+decodeFromFrontendAcceptNum) + */ + for (i <- 0 until DecodeWidth) { + // decodeBufValid update + when(decode.io.redirect || decodeBufValid(0) && decodeBufValid(i) && decode.io.in(i).ready && !VecInit( + decodeBufNotAccept.drop(i) + ).asUInt.orR) { + decodeBufValid(i) := false.B + }.elsewhen(decodeBufValid(i) && VecInit(decodeBufNotAccept.drop(i)).asUInt.orR) { + decodeBufValid(i) := Mux( + decodeBufAcceptNum > DecodeWidth.U - 1.U - i.U, + false.B, + decodeBufValid(i.U + decodeBufAcceptNum) + ) + }.elsewhen(!decodeBufValid(0) && VecInit(decodeFromFrontendNotAccept.drop(i)).asUInt.orR) { + decodeBufValid(i) := Mux( + decodeFromFrontendAcceptNum > DecodeWidth.U - 1.U - i.U, + false.B, + decodeFromFrontend(i.U + decodeFromFrontendAcceptNum).valid + ) + } + // decodeBufBits update + when(decodeBufValid(i) && VecInit(decodeBufNotAccept.drop(i)).asUInt.orR) { + decodeBufBits(i) := decodeBufBits(i.U + decodeBufAcceptNum) + }.elsewhen(!decodeBufValid(0) && VecInit(decodeFromFrontendNotAccept.drop(i)).asUInt.orR) { + decodeBufBits(i).connectCtrlFlow(decodeFromFrontend(i.U + decodeFromFrontendAcceptNum).bits) + } + } + + /** Insts input from frontend, in vector of DecodeWidth */ + val decodeConnectFromFrontend = Wire(Vec(DecodeWidth, new StaticInst)) + decodeConnectFromFrontend.zip(decodeFromFrontend).map(x => x._1.connectCtrlFlow(x._2.bits)) + + /** + * DecodeStage's input: + * decode.io.in(i).valid: + * decodeBufValid(0) is true : decodeBufValid(i) | from decode buffer + * false : decodeFromFrontend(i).valid | from frontend + * + * decodeFromFrontend(i).ready: + * decodeFromFrontend(0).valid && !decodeBufValid(0) && decodeFromFrontend(i).valid && !decode.io.redirect + * valid instr in input, no instr in decode buffer, decodeFromFrontend(i) is valid, no redirection + * + * decode.io.in(i).bits: + * decodeBufValid(i) is true : decodeBufBits(i) | from decode buffer + * false : decodeConnectFromFrontend(i) | from frontend + */ + decode.io.in.zipWithIndex.foreach { case (decodeIn, i) => + decodeIn.valid := Mux(decodeBufValid(0), decodeBufValid(i), decodeFromFrontend(i).valid) + decodeFromFrontend(i).ready := decodeFromFrontend(0).valid && !decodeBufValid(0) && decodeFromFrontend( + i + ).valid && !decode.io.redirect + decodeIn.bits := Mux(decodeBufValid(i), decodeBufBits(i), decodeConnectFromFrontend(i)) + } + + /** no valid instr in decode buffer && no valid instr from frontend --> can accept new instr from frontend */ + io.frontend.canAccept := !decodeBufValid(0) || !decodeFromFrontend(0).valid + decode.io.csrCtrl := RegNext(io.csrCtrl) + decode.io.intRat <> rat.io.intReadPorts + decode.io.fpRat <> rat.io.fpReadPorts + decode.io.vecRat <> rat.io.vecReadPorts + decode.io.v0Rat <> rat.io.v0ReadPorts + decode.io.vlRat <> rat.io.vlReadPorts + decode.io.fusion := 0.U.asTypeOf(decode.io.fusion) // Todo + decode.io.stallReason.in <> io.frontend.stallReason + decode.io.backendCanAccept := io.frontend.canAccept + + // snapshot check + class CFIRobIdx extends Bundle { + val robIdx = Vec(RenameWidth, new RobPtr) + val isCFI = Vec(RenameWidth, Bool()) + } + + val genSnapshot = Cat(rename.io.out.map(out => out.fire && out.bits.snapshot)).orR + val snpt = Module(new SnapshotGenerator(0.U.asTypeOf(new CFIRobIdx))) + snpt.io.enq := genSnapshot + snpt.io.enqData.robIdx := rename.io.out.map(_.bits.robIdx) + snpt.io.enqData.isCFI := rename.io.out.map(_.bits.snapshot) + snpt.io.deq := snpt.io.valids(snpt.io.deqPtr.value) && rob.io.commits.isCommit && + Cat(rob.io.commits.commitValid.zip(rob.io.commits.robIdx).map(x => + x._1 && x._2 === snpt.io.snapshots(snpt.io.deqPtr.value).robIdx.head + )).orR + snpt.io.redirect := s1_s3_redirect.valid + + val flushVec = VecInit(snpt.io.snapshots.map { snapshot => + val notCFIMask = snapshot.isCFI.map(~_) + val shouldFlush = snapshot.robIdx.map(robIdx => + robIdx >= s1_s3_redirect.bits.robIdx || robIdx.value === s1_s3_redirect.bits.robIdx.value + ) + val shouldFlushMask = (1 to RenameWidth).map(shouldFlush take _ reduce (_ || _)) + s1_s3_redirect.valid && Cat(shouldFlushMask.zip(notCFIMask).map(x => x._1 | x._2)).andR + }) + + val flushVecNext = flushVec zip snpt.io.valids map (x => GatedValidRegNext(x._1 && x._2, false.B)) + snpt.io.flushVec := flushVecNext + + val redirectRobidx = s1_s3_redirect.bits.robIdx + + val useSnpt = VecInit.tabulate(RenameSnapshotNum) { case idx => + val snptRobidx = snpt.io.snapshots(idx).robIdx.head + // (redirectRobidx.value =/= snptRobidx.value) for only flag diffrence + snpt.io.valids(idx) && ((redirectRobidx > snptRobidx) && (redirectRobidx.value =/= snptRobidx.value) || + !s1_s3_redirect.bits.flushItself() && redirectRobidx === snptRobidx) + }.reduceTree(_ || _) + + val snptSelect = MuxCase( + 0.U(log2Ceil(RenameSnapshotNum).W), + (1 to RenameSnapshotNum).map(i => (snpt.io.enqPtr - i.U).value).map { case idx => + val thisSnapRobidx = snpt.io.snapshots(idx).robIdx.head + ( + snpt.io.valids(idx) && (redirectRobidx > thisSnapRobidx && (redirectRobidx.value =/= thisSnapRobidx.value) || + !s1_s3_redirect.bits.flushItself() && redirectRobidx === thisSnapRobidx), + idx + ) + } + ) + + rob.io.snpt.snptEnq := DontCare + rob.io.snpt.snptDeq := snpt.io.deq + rob.io.snpt.useSnpt := useSnpt + rob.io.snpt.snptSelect := snptSelect + rob.io.snpt.flushVec := flushVecNext + rat.io.snpt.snptEnq := genSnapshot + rat.io.snpt.snptDeq := snpt.io.deq + rat.io.snpt.useSnpt := useSnpt + rat.io.snpt.snptSelect := snptSelect + rat.io.snpt.flushVec := flushVec + + val decodeHasException = + decode.io.out.map(x => x.bits.exceptionVec.asUInt.orR || (!TriggerAction.isNone(x.bits.trigger))) + // fusion decoder + fusionDecoder.io.disableFusion := disableFusion + for (i <- 0 until DecodeWidth) { + fusionDecoder.io.in(i).valid := decode.io.out(i).valid && !decodeHasException(i) + fusionDecoder.io.in(i).bits := decode.io.out(i).bits.instr + if (i > 0) { + fusionDecoder.io.inReady(i - 1) := decode.io.out(i).ready + } + } + + private val decodePipeRename = Wire(Vec(RenameWidth, DecoupledIO(new DecodedInst))) + for (i <- 0 until RenameWidth) { + PipelineConnect( + decode.io.out(i), + decodePipeRename(i), + rename.io.in(i).ready, + s1_s3_redirect.valid || s2_s4_pendingRedirectValid, + moduleName = Some("decodePipeRenameModule") + ) + + decodePipeRename(i).ready := rename.io.in(i).ready + rename.io.in(i).valid := decodePipeRename(i).valid && !fusionDecoder.io.clear(i) + rename.io.in(i).bits := decodePipeRename(i).bits + dispatch.io.renameIn(i).valid := decodePipeRename(i).valid && !fusionDecoder.io.clear(i) && !decodePipeRename( + i + ).bits.isMove + dispatch.io.renameIn(i).bits := decodePipeRename(i).bits + } + + for (i <- 0 until RenameWidth - 1) { + fusionDecoder.io.dec(i) := decodePipeRename(i).bits + rename.io.fusionInfo(i) := fusionDecoder.io.info(i) + + // update the first RenameWidth - 1 instructions + decode.io.fusion(i) := fusionDecoder.io.out(i).valid && rename.io.out(i).fire + // TODO: remove this dirty code for ftq update + val sameFtqPtr = rename.io.in(i).bits.ftqPtr.value === rename.io.in(i + 1).bits.ftqPtr.value + val ftqOffset0 = rename.io.in(i).bits.ftqOffset + val ftqOffset1 = rename.io.in(i + 1).bits.ftqOffset + val ftqOffsetDiff = ftqOffset1 - ftqOffset0 + val cond1 = sameFtqPtr && ftqOffsetDiff === 1.U + val cond2 = sameFtqPtr && ftqOffsetDiff === 2.U + val cond3 = !sameFtqPtr && ftqOffset1 === 0.U + val cond4 = !sameFtqPtr && ftqOffset1 === 1.U + when(fusionDecoder.io.out(i).valid) { + fusionDecoder.io.out(i).bits.update(rename.io.in(i).bits) + fusionDecoder.io.out(i).bits.update(dispatch.io.renameIn(i).bits) + rename.io.in(i).bits.commitType := Mux(cond1, 4.U, Mux(cond2, 5.U, Mux(cond3, 6.U, 7.U))) + } + XSError( + fusionDecoder.io.out(i).valid && !cond1 && !cond2 && !cond3 && !cond4, + p"new condition $sameFtqPtr $ftqOffset0 $ftqOffset1\n" + ) + } + + // memory dependency predict + // when decode, send fold pc to mdp + private val mdpFlodPcVecVld = Wire(Vec(DecodeWidth, Bool())) + private val mdpFlodPcVec = Wire(Vec(DecodeWidth, UInt(MemPredPCWidth.W))) + for (i <- 0 until DecodeWidth) { + mdpFlodPcVecVld(i) := decode.io.out(i).fire || GatedValidRegNext(decode.io.out(i).fire) + mdpFlodPcVec(i) := Mux( + decode.io.out(i).fire, + decode.io.in(i).bits.foldpc, + rename.io.in(i).bits.foldpc + ) + } + + // currently, we only update mdp info when isReplay + memCtrl.io.redirect := s1_s3_redirect + memCtrl.io.csrCtrl := io.csrCtrl // RegNext in memCtrl + memCtrl.io.stIn := io.fromMem.stIn // RegNext in memCtrl + memCtrl.io.memPredUpdate := redirectGen.io.memPredUpdate // RegNext in memCtrl + memCtrl.io.mdpFoldPcVecVld := mdpFlodPcVecVld + memCtrl.io.mdpFlodPcVec := mdpFlodPcVec + memCtrl.io.dispatchLFSTio <> dispatch.io.lfst + + rat.io.hartId := io.fromTop.hartId + rat.io.redirect := s1_s3_redirect.valid + rat.io.rabCommits := rob.io.rabCommits + rat.io.diffRatCommitRobIdx.foreach(_ := rob.io.diffRatCommitRobIdx.get) + rat.io.renameUpdates.foreach { updates => + updates.zip(rename.io.out).foreach { case (update, renameOut) => + update.valid := renameOut.fire && !s1_s3_redirect.valid + update.bits.robIdx := renameOut.bits.robIdx + update.bits.lastUop := renameOut.bits.lastUop + update.bits.ldest := renameOut.bits.ldest + update.bits.pdest := renameOut.bits.pdest + update.bits.rfWen := renameOut.bits.rfWen + update.bits.fpWen := renameOut.bits.fpWen + update.bits.vecWen := renameOut.bits.vecWen + update.bits.v0Wen := renameOut.bits.v0Wen + update.bits.vlWen := renameOut.bits.vlWen + } + } + rat.io.intRenamePorts := rename.io.intRenamePorts + rat.io.fpRenamePorts := rename.io.fpRenamePorts + rat.io.vecRenamePorts := rename.io.vecRenamePorts + rat.io.v0RenamePorts := rename.io.v0RenamePorts + rat.io.vlRenamePorts := rename.io.vlRenamePorts + + rename.io.redirect := s1_s3_redirect + rename.io.rabCommits := rob.io.rabCommits + rename.io.singleStep := GatedValidRegNext(io.csrCtrl.singlestep) + rename.io.waittable := (memCtrl.io.waitTable2Rename zip decode.io.out).map { case (waittable2rename, decodeOut) => + RegEnable(waittable2rename, decodeOut.fire) + } + rename.io.ssit := memCtrl.io.ssit2Rename + // disble mdp + dispatch.io.lfst.resp := 0.U.asTypeOf(dispatch.io.lfst.resp) + rename.io.waittable := 0.U.asTypeOf(rename.io.waittable) + rename.io.ssit := 0.U.asTypeOf(rename.io.ssit) + rename.io.intReadPorts := VecInit(rat.io.intReadPorts.map(x => VecInit(x.map(_.data)))) + rename.io.fpReadPorts := VecInit(rat.io.fpReadPorts.map(x => VecInit(x.map(_.data)))) + rename.io.vecReadPorts := VecInit(rat.io.vecReadPorts.map(x => VecInit(x.map(_.data)))) + rename.io.v0ReadPorts := VecInit(rat.io.v0ReadPorts.map(x => VecInit(x.data))) + rename.io.vlReadPorts := VecInit(rat.io.vlReadPorts.map(x => VecInit(x.data))) + rename.io.int_need_free := rat.io.int_need_free + rename.io.int_old_pdest := rat.io.int_old_pdest + rename.io.fp_old_pdest := rat.io.fp_old_pdest + rename.io.vec_old_pdest := rat.io.vec_old_pdest + rename.io.v0_old_pdest := rat.io.v0_old_pdest + rename.io.vl_old_pdest := rat.io.vl_old_pdest + rename.io.debug_int_rat.foreach(_ := rat.io.debug_int_rat.get) + rename.io.debug_fp_rat.foreach(_ := rat.io.debug_fp_rat.get) + rename.io.debug_vec_rat.foreach(_ := rat.io.debug_vec_rat.get) + rename.io.debug_v0_rat.foreach(_ := rat.io.debug_v0_rat.get) + rename.io.debug_vl_rat.foreach(_ := rat.io.debug_vl_rat.get) + rename.io.stallReason.in <> decode.io.stallReason.out + rename.io.snpt.snptEnq := DontCare + rename.io.snpt.snptDeq := snpt.io.deq + rename.io.snpt.useSnpt := useSnpt + rename.io.snpt.snptSelect := snptSelect + rename.io.snptIsFull := snpt.io.valids.asUInt.andR + rename.io.snpt.flushVec := flushVecNext + rename.io.snptLastEnq.valid := !isEmpty(snpt.io.enqPtr, snpt.io.deqPtr) + rename.io.snptLastEnq.bits := snpt.io.snapshots((snpt.io.enqPtr - 1.U).value).robIdx.head + + val renameOut = Wire(chiselTypeOf(rename.io.out)) + renameOut <> rename.io.out + // pass all snapshot in the first element for correctness of blockBackward + renameOut.tail.foreach(_.bits.snapshot := false.B) + renameOut.head.bits.snapshot := Mux( + isFull(snpt.io.enqPtr, snpt.io.deqPtr), + false.B, + Cat(rename.io.out.map(out => out.valid && out.bits.snapshot)).orR + ) + + // pipeline between rename and dispatch + PipeGroupConnect( + renameOut, + dispatch.io.fromRename, + s1_s3_redirect.valid, + dispatch.io.toRenameAllFire, + "renamePipeDispatch" + ) + + dispatch.io.redirect := s1_s3_redirect + val enqRob = Wire(chiselTypeOf(rob.io.enq)) + enqRob.canAccept := rob.io.enq.canAccept + enqRob.canAcceptForDispatch := rob.io.enq.canAcceptForDispatch + enqRob.isEmpty := rob.io.enq.isEmpty + enqRob.resp := rob.io.enq.resp + enqRob.needAlloc := RegNext(dispatch.io.enqRob.needAlloc) + enqRob.req.zip(dispatch.io.enqRob.req).map { case (sink, source) => + sink.valid := RegNext(source.valid && !rob.io.redirect.valid) + sink.bits := RegEnable(source.bits, source.valid) + } + dispatch.io.enqRob.canAccept := enqRob.canAcceptForDispatch && !enqRob.req.map(x => + x.valid && x.bits.blockBackward && enqRob.canAccept + ).reduce(_ || _) + dispatch.io.enqRob.canAcceptForDispatch := enqRob.canAcceptForDispatch + dispatch.io.enqRob.isEmpty := enqRob.isEmpty && !enqRob.req.map(_.valid).reduce(_ || _) + dispatch.io.enqRob.resp := enqRob.resp + rob.io.enq.needAlloc := enqRob.needAlloc + rob.io.enq.req := enqRob.req + dispatch.io.robHead := rob.io.debugRobHead + dispatch.io.stallReason <> rename.io.stallReason.out + dispatch.io.lqCanAccept := io.lqCanAccept + dispatch.io.sqCanAccept := io.sqCanAccept + dispatch.io.fromMem.lcommit := io.fromMemToDispatch.lcommit + dispatch.io.fromMem.scommit := io.fromMemToDispatch.scommit + dispatch.io.fromMem.lqDeqPtr := io.fromMemToDispatch.lqDeqPtr + dispatch.io.fromMem.sqDeqPtr := io.fromMemToDispatch.sqDeqPtr + dispatch.io.fromMem.lqCancelCnt := io.fromMemToDispatch.lqCancelCnt + dispatch.io.fromMem.sqCancelCnt := io.fromMemToDispatch.sqCancelCnt + io.toMem.lsqEnqIO <> dispatch.io.toMem.lsqEnqIO + dispatch.io.wakeUpAll.wakeUpInt := io.toDispatch.wakeUpInt + dispatch.io.wakeUpAll.wakeUpFp := io.toDispatch.wakeUpFp + dispatch.io.wakeUpAll.wakeUpVec := io.toDispatch.wakeUpVec + dispatch.io.wakeUpAll.wakeUpMem := io.toDispatch.wakeUpMem + dispatch.io.IQValidNumVec := io.toDispatch.IQValidNumVec + dispatch.io.ldCancel := io.toDispatch.ldCancel + dispatch.io.og0Cancel := io.toDispatch.og0Cancel + dispatch.io.wbPregsInt := io.toDispatch.wbPregsInt + dispatch.io.wbPregsFp := io.toDispatch.wbPregsFp + dispatch.io.wbPregsVec := io.toDispatch.wbPregsVec + dispatch.io.wbPregsV0 := io.toDispatch.wbPregsV0 + dispatch.io.wbPregsVl := io.toDispatch.wbPregsVl + dispatch.io.vlWriteBackInfo := io.toDispatch.vlWriteBackInfo + dispatch.io.robHeadNotReady := rob.io.headNotReady + dispatch.io.robFull := rob.io.robFull + dispatch.io.singleStep := GatedValidRegNext(io.csrCtrl.singlestep) + + val toIssueBlockUops = + Seq(io.toIssueBlock.intUops, io.toIssueBlock.fpUops, io.toIssueBlock.vfUops, io.toIssueBlock.memUops).flatten + toIssueBlockUops.zip(dispatch.io.toIssueQueues).map(x => x._1 <> x._2) + io.toIssueBlock.flush <> s2_s4_redirect + + pcMem.io.wen.head := GatedValidRegNext(io.frontend.fromFtq.pc_mem_wen) + pcMem.io.waddr.head := RegEnable(io.frontend.fromFtq.pc_mem_waddr, io.frontend.fromFtq.pc_mem_wen) + pcMem.io.wdata.head := RegEnable(io.frontend.fromFtq.pc_mem_wdata, io.frontend.fromFtq.pc_mem_wen) + + io.toDataPath.flush := s2_s4_redirect + io.toExuBlock.flush := s2_s4_redirect + + rob.io.hartId := io.fromTop.hartId + rob.io.redirect := s1_s3_redirect + rob.io.writeback := delayedNotFlushedWriteBack + rob.io.exuWriteback := delayedWriteBack + rob.io.writebackNums := VecInit(delayedNotFlushedWriteBackNums) + rob.io.writebackNeedFlush := delayedNotFlushedWriteBackNeedFlush + rob.io.readGPAMemData := gpaMem.io.exceptionReadData + rob.io.fromVecExcpMod.busy := io.fromVecExcpMod.busy + + io.redirect := s1_s3_redirect + + // rob to int block + io.robio.csr <> rob.io.csr + // When wfi is disabled, it will not block ROB commit. + rob.io.csr.wfiEvent := io.robio.csr.wfiEvent + rob.io.wfi_enable := decode.io.csrCtrl.wfi_enable + rob.io.commit_stuck_check_enable := decode.io.csrCtrl.commit_stuck_check_enable + + io.toTop.cpuHalt := DelayN(rob.io.cpu_halt, 5) + + io.robio.csr.perfinfo.retiredInstr <> RegNext(rob.io.csr.perfinfo.retiredInstr) + io.robio.exception := rob.io.exception + io.robio.exception.bits.pc := s1_robFlushPc + + // wfi + io.frontend.wfi.wfiReq := rob.io.wfi.wfiReq + rob.io.wfi.safeFromFrontend := io.frontend.wfi.wfiSafe + io.toMem.wfi.wfiReq := rob.io.wfi.wfiReq + rob.io.wfi.safeFromMem := io.toMem.wfi.wfiSafe + + // rob to mem block + io.robio.lsq <> rob.io.lsq + + io.diff_vl_rat.foreach(_ := rat.io.diff_vl_rat.get) + + rob.io.debug_ls := io.robio.debug_ls + rob.io.debugHeadLsIssue := io.robio.robHeadLsIssue + rob.io.lsTopdownInfo := io.robio.lsTopdownInfo + rob.io.csr.criticalErrorState := io.robio.csr.criticalErrorState + rob.io.debugEnqLsq := io.debugEnqLsq + rob.io.debugInstrAddrTransType := io.fromCSR.instrAddrTransType + + io.robio.robDeqPtr := rob.io.robDeqPtr + + io.robio.storeDebugInfo <> rob.io.storeDebugInfo + + // rob to backend + io.robio.commitVType := rob.io.toDecode.commitVType + // exu block to decode + decode.io.vsetvlVType := io.toDecode.vsetvlVType + // backend to decode + decode.io.vstart := io.toDecode.vstart + // backend to rob + rob.io.vstartIsZero := io.toDecode.vstart === 0.U + + io.toCSR.trapInstInfo := decode.io.toCSR.trapInstInfo + + io.toVecExcpMod.logicPhyRegMap := rob.io.toVecExcpMod.logicPhyRegMap + io.toVecExcpMod.excpInfo := rob.io.toVecExcpMod.excpInfo + // T : rat receive rabCommit + // T+1: rat return oldPdest + io.toVecExcpMod.ratOldPest match { + case fromRat => + (0 until RabCommitWidth).foreach { idx => + val v0Valid = RegNext( + rat.io.rabCommits.isCommit && + rat.io.rabCommits.isWalk && + rat.io.rabCommits.commitValid(idx) && + rat.io.rabCommits.info(idx).v0Wen + ) + fromRat.v0OldVdPdest(idx).valid := RegNext(v0Valid) + fromRat.v0OldVdPdest(idx).bits := RegEnable(rat.io.v0_old_pdest(idx), v0Valid) + val vecValid = RegNext( + rat.io.rabCommits.isCommit && + rat.io.rabCommits.isWalk && + rat.io.rabCommits.commitValid(idx) && + rat.io.rabCommits.info(idx).vecWen + ) + fromRat.vecOldVdPdest(idx).valid := RegNext(vecValid) + fromRat.vecOldVdPdest(idx).bits := RegEnable(rat.io.vec_old_pdest(idx), vecValid) + } + } + + io.debugTopDown.fromRob := rob.io.debugTopDown.toCore + dispatch.io.debugTopDown.fromRob := rob.io.debugTopDown.toDispatch + dispatch.io.debugTopDown.fromCore := io.debugTopDown.fromCore + io.debugRolling := rob.io.debugRolling + + io.perfInfo.ctrlInfo.robFull := GatedValidRegNext(rob.io.robFull) + io.perfInfo.ctrlInfo.intdqFull := false.B + io.perfInfo.ctrlInfo.fpdqFull := false.B + io.perfInfo.ctrlInfo.lsdqFull := false.B + + val perfEvents = Seq(decode, rename, dispatch, rob).flatMap(_.getPerfEvents) + generatePerfEvent() + + val criticalErrors = rob.getCriticalErrors + generateCriticalErrors() +} diff --git a/examples/prefab/cores/xiangshan/arch/native/DecodeStage.scala b/examples/prefab/cores/xiangshan/arch/native/DecodeStage.scala new file mode 100644 index 000000000..c15c157ac --- /dev/null +++ b/examples/prefab/cores/xiangshan/arch/native/DecodeStage.scala @@ -0,0 +1,320 @@ +// Copied from XiangShan src/main/scala/xiangshan/backend/decode/DecodeStage.scala and modified for Buckyball. +// XiangShan is licensed under Mulan PSL v2 (http://license.coscl.org.cn/MulanPSL2). + +package examples.core.xiangshan.native + +import org.chipsalliance.cde.config.Parameters +import chisel3._ +import chisel3.util._ +import utility._ +import utils._ +import xiangshan._ +import xiangshan.backend.rename.RatReadPort +import xiangshan.backend.Bundles._ +import xiangshan.backend.fu.vector.Bundles.{VType, Vl} +import xiangshan.backend.fu.FuType +import xiangshan.backend.fu.wrapper.CSRToDecode +import yunsuan.VpermType +import xiangshan.ExceptionNO.{illegalInstr, virtualInstr} +import xiangshan.frontend.FtqPtr +import xiangshan.backend.decode._ +import examples.core.xiangshan.native.{DecodeUnit} + +class DecodeStage(implicit p: Parameters) extends XSModule with HasPerfEvents with VectorConstants { + + val io = IO(new DecodeStageIO) + + io.in.zipWithIndex.foreach { case (d, i) => + PerfCCT.updateInstPos(d.bits.debug_seqNum, PerfCCT.InstPos.AtDecode.id.U, d.valid, clock, reset) + } + + // io alias + private val outReadys = io.out.map(_.ready) + private val inValids = io.in.map(_.valid) + private val inValid = VecInit(inValids).asUInt.orR + private val outValids = io.out.map(_.valid) + private val outValid = VecInit(outValids).asUInt.orR + //readyFromRename Counter + /** Assume number of ready channels be "RenameWidth" if the first output channel is ready. If not, assume that be 0 */ + val readyCounter = Mux(outReadys.head, RenameWidth.U, 0.U) + + /** complex decoder */ + val decoderComp = Module(new DecodeUnitComp) + + /** simple decoders in Seq of DecodeWidth */ + val decoders = Seq.fill(DecodeWidth)(Module(new DecodeUnit)) + + /** vtype generation module */ + val vtypeGen = Module(new VTypeGen) + + val debug_globalCounter = RegInit(0.U(XLEN.W)) + + /** whether DecodeStage can accept new requests from frontend (CtrlBlock) */ + val canAccept = Wire(Bool()) + + //Simple 6 + decoders.zip(io.in).foreach { case (dst, src) => + dst.io.enq.ctrlFlow := src.bits + dst.io.csrCtrl := io.csrCtrl + dst.io.fromCSR := io.fromCSR + dst.io.enq.vtype := vtypeGen.io.vtype + dst.io.enq.vstart := io.vstart + } + + /** whether instructions decoded by simple decoders require complex decoding */ + val isComplexVec = VecInit(inValids.zip(decoders.map(_.io.deq.isComplex)).map { case (valid, isComplex) => + valid && isComplex + }) + + /** whether instructions decoded by simple decoders don't require complex decoding */ + val isSimpleVec = VecInit(inValids.zip(decoders.map(_.io.deq.isComplex)).map { case (valid, isComplex) => + valid && !isComplex + }) + + /** instructions decoded by simple decoders */ + val simpleDecodedInst = VecInit(decoders.map(_.io.deq.decodedInst)) + + /** whether instructions decoded by simple decoders are illegal */ + val isIllegalInstVec = VecInit((outValids lazyZip outReadys lazyZip io.out.map(_.bits)).map { + case (valid, ready, decodedInst) => + valid && ready && (decodedInst.exceptionVec(ExceptionNO.EX_II) || decodedInst.exceptionVec(ExceptionNO.EX_VI)) + }) + + /** at least 1 instruction decoded by simple decoders is illegal */ + val hasIllegalInst = Cat(isIllegalInstVec).orR + + /** at least 1 instruction decoded by simple decoders is illegal */ + val illegalInst = PriorityMuxDefault(isIllegalInstVec.zip(io.out.map(_.bits)), 0.U.asTypeOf(new DecodedInst)) + + /** number of instructions generated by complex decoder */ + val complexNum = Wire(UInt(3.W)) + // (0, 1, 2, 3, 4, 5) + complexNum + /** + * Order of simple decoders' result (in output of DecodeStage) considering complex decoder's. Since complex decoder's + * results will be arranged before simple decoders' + */ + val complexNumAddLocation: Vec[UInt] = VecInit((0 until DecodeWidth).map(x => (x.U +& complexNum))) + + /** mask off decoded instructions that can not be accepted */ + val noMoreThanRenameReady: Vec[Bool] = VecInit(complexNumAddLocation.map(x => x <= readyCounter)) + + /** existance of complex instructions among first few simple decoders' results, which needs decoding */ + val complexValid = VecInit((isComplexVec zip noMoreThanRenameReady).map(x => x._1 & x._2)).asUInt.orR + + /** selected complex instruction for complex decoder */ + val complexInst = + PriorityMuxDefault(isComplexVec.zip(decoders.map(_.io.deq.decodedInst)), 0.U.asTypeOf(new DecodedInst)) + + /** selected complex micro operation information for complex decoder */ + val complexUopInfo = PriorityMuxDefault(isComplexVec.zip(decoders.map(_.io.deq.uopInfo)), 0.U.asTypeOf(new UopInfo)) + + vtypeGen.io.insts.zip(io.in).foreach { case (inst, in) => + inst.valid := in.valid + inst.bits := in.bits.instr + } + // when io.redirect is True, never update vtype + vtypeGen.io.canUpdateVType := decoderComp.io.in.fire && decoderComp.io.in.bits.simpleDecodedInst.isVset && !io.redirect + vtypeGen.io.walkToArchVType := io.fromRob.walkToArchVType + vtypeGen.io.commitVType := io.fromRob.commitVType + vtypeGen.io.walkVType := io.fromRob.walkVType + vtypeGen.io.vsetvlVType := io.vsetvlVType + + //Comp 1 + decoderComp.io.redirect := io.redirect + decoderComp.io.csrCtrl := io.csrCtrl + decoderComp.io.vtypeBypass := vtypeGen.io.vtype + // The input inst of decoderComp is latched last cycle. + // Set input empty, if there is no complex inst latched last cycle. + decoderComp.io.in.valid := complexValid && !io.fromRob.isResumeVType + decoderComp.io.in.bits.simpleDecodedInst := complexInst + decoderComp.io.in.bits.uopInfo := complexUopInfo + decoderComp.io.out.complexDecodedInsts.zipWithIndex.foreach { case (out, i) => out.ready := io.out(i).ready } + + /** instructions decoded by complex decoders */ + val complexDecodedInst = VecInit(decoderComp.io.out.complexDecodedInsts.map(_.bits)) + + /** whether instructions decoded by complex decoders are valid */ + val complexDecodedInstValid = VecInit(decoderComp.io.out.complexDecodedInsts.map(_.valid)) + complexNum := decoderComp.io.complexNum + + // Vec(S,S,S,C,S,C) -> Vec(1,1,1,0,0,0) + /** whether a complex instruction is before or at this position of simple decoders' result */ + val simplePrefixVec = VecInit((0 until DecodeWidth).map(i => VecInit(isSimpleVec.take(i + 1)).asUInt.andR)) + + // Vec(S,S,S,C,S,S) -> Vec(0,0,0,1,0,0) + /** one-hot representation of the first complex instruction */ + val firstComplexOH: Vec[Bool] = VecInit(PriorityEncoderOH(isComplexVec)) + + // block vector inst when vtype is resuming + val hasVectorInst = VecInit(decoders.map(x => + FuType.FuTypeOrR(x.io.deq.decodedInst.fuType, FuType.vecArithOrMem ++ FuType.vecVSET) + )).asUInt.orR + + /** condition of acceptation: no redirection, ready from rename/complex decoder, no resumeVType */ + canAccept := !io.redirect && (io.out.head.ready || decoderComp.io.in.ready) && !io.fromRob.isResumeVType + + io.canAccept := canAccept + + /** + * Assign ready signal for DecodeStage's input. Ready signal in i-th channel: + * + * It must hold that no redirection and no isResumeVType is caught. + * One situation for set up ready signal is that first "i" instructions are all simple instructions, and these "i" + * instructions can be passed down to rename together with complex decoder's result. + * Another situation is that first "i-1" instructions are all simple instructions, and the "i-th" instructions needs + * to be sent to complex decoder, with complex decoder ready for new input. + */ + io.in.zipWithIndex.foreach { case (in, i) => + in.ready := !io.redirect && ( + simplePrefixVec(i) && (i.U +& complexNum) < readyCounter || + firstComplexOH(i) && (i.U +& complexNum) <= readyCounter && decoderComp.io.in.ready + ) && !io.fromRob.isResumeVType + } + + /** final instruction decoding result */ + val finalDecodedInst = Wire(Vec(DecodeWidth, new DecodedInst)) + + /** valid signs of final instruction decoding result */ + val finalDecodedInstValid = Wire(Vec(DecodeWidth, Bool())) + + /** + * Select final result of DecodeStage. Select all complex decoded insts results at the beginning of final result, and + * use simple decoded insts to fill the rest space in DecodeWidth. + */ + for (i <- 0 until DecodeWidth) { + finalDecodedInst(i) := Mux(complexNum > i.U, complexDecodedInst(i), simpleDecodedInst(i.U - complexNum)) + finalDecodedInstValid(i) := Mux(complexNum > i.U, complexDecodedInstValid(i), simplePrefixVec(i.U - complexNum)) + } + + /** + * Generate output of DecodeStage. Pass finalDecodedInst to output as decoded instructions. + * Note that finalDecodedInst is generated in order. + */ + io.out.zipWithIndex.foreach { case (inst, i) => + inst.valid := finalDecodedInstValid(i) && !io.fromRob.isResumeVType + inst.bits := finalDecodedInst(i) + inst.bits.lsrc(0) := Mux(finalDecodedInst(i).vpu.isReverse, finalDecodedInst(i).lsrc(1), finalDecodedInst(i).lsrc(0)) + inst.bits.lsrc(1) := Mux(finalDecodedInst(i).vpu.isReverse, finalDecodedInst(i).lsrc(0), finalDecodedInst(i).lsrc(1)) + inst.bits.srcType(0) := Mux( + finalDecodedInst(i).vpu.isReverse, + finalDecodedInst(i).srcType(1), + finalDecodedInst(i).srcType(0) + ) + inst.bits.srcType(1) := Mux( + finalDecodedInst(i).vpu.isReverse, + finalDecodedInst(i).srcType(0), + finalDecodedInst(i).srcType(1) + ) + inst.bits.v0Wen := finalDecodedInst(i).vecWen && finalDecodedInst(i).ldest === 0.U || finalDecodedInst(i).v0Wen + inst.bits.vecWen := finalDecodedInst(i).vecWen && finalDecodedInst(i).ldest =/= 0.U + // when src0/src1/src2 read V0, src3 read V0 + val srcType0123HasV0 = finalDecodedInst(i).srcType.zip(finalDecodedInst(i).lsrc).take(4).map { case (s, l) => + SrcType.isVp(s) && (l === 0.U) + }.reduce(_ || _) + inst.bits.srcType(3) := Mux(srcType0123HasV0, SrcType.v0, finalDecodedInst(i).srcType(3)) + when(inst.bits.uopIdx =/= 0.U) { + inst.bits.debug_seqNum := 0.U.asTypeOf(new InstSeqNum) + } + } + + io.out.map(x => + when(x.valid) { + assert( + PopCount(VecInit(x.bits.rfWen, x.bits.fpWen, x.bits.vecWen, x.bits.v0Wen, x.bits.vlWen)) < 2.U, + "DecodeOut: can't wirte two regfile in one uop/instruction" + ) + } + ) + + /** + * Prepare address and hold for output to Rat (Rename Alias Table) + */ + for (i <- 0 until DecodeWidth) { + + // We use the lsrc/ldest before fusion decoder to read RAT for better timing. + io.intRat(i)(0).addr := io.out(i).bits.lsrc(0) + io.intRat(i)(1).addr := io.out(i).bits.lsrc(1) + io.intRat(i).foreach(_.hold := !io.out(i).ready) + + // Floating-point instructions can not be fused now. + io.fpRat(i)(0).addr := io.out(i).bits.lsrc(0) + io.fpRat(i)(1).addr := io.out(i).bits.lsrc(1) + io.fpRat(i)(2).addr := io.out(i).bits.lsrc(2) + io.fpRat(i).foreach(_.hold := !io.out(i).ready) + + // Vec instructions + // TODO: vec uop dividers need change this + io.vecRat(i)(0).addr := io.out(i).bits.lsrc(0) // vs1 + io.vecRat(i)(1).addr := io.out(i).bits.lsrc(1) // vs2 + io.vecRat(i)(2).addr := io.out(i).bits.lsrc(2) // old_vd + io.vecRat(i).foreach(_.hold := !io.out(i).ready) + + io.v0Rat(i).addr := V0_IDX.U // v0 + io.v0Rat(i).hold := !io.out(i).ready + + io.vlRat(i).addr := Vl_IDX.U // vl + io.vlRat(i).hold := !io.out(i).ready + } + + /** whether valid input requests from frontend exists */ + val hasValid = VecInit(io.in.map(_.valid)).asUInt.orR + + debug_globalCounter := debug_globalCounter + PopCount(io.out.map(_.fire)) + + io.stallReason.in.backReason := io.stallReason.out.backReason + io.stallReason.out.reason.zip(io.stallReason.in.reason).zip(io.in.map(_.valid)).foreach { case ((out, in), valid) => + out := Mux(io.stallReason.out.backReason.valid, io.stallReason.out.backReason.bits, in) + } + + io.toCSR.trapInstInfo.valid := RegNext(hasIllegalInst && !io.redirect) + io.toCSR.trapInstInfo.bits.fromDecodedInst(RegNext(illegalInst)) + + val recoveryFlag = RegInit(false.B) + when(io.redirect) { + recoveryFlag := true.B + }.elsewhen(io.in.map(_.fire).reduce(_ || _)) { + recoveryFlag := false.B + } + + XSPerfAccumulate("in_valid_count", PopCount(io.in.map(_.valid))) + XSPerfAccumulate("in_fire_count", PopCount(io.in.map(_.fire))) + XSPerfAccumulate("in_valid_not_ready_count", PopCount(io.in.map(x => x.valid && !x.ready))) + XSPerfAccumulate("stall_cycle", io.in.head match { case x => x.valid && !x.ready }) + XSPerfAccumulate("wait_cycle", !io.in.head.valid && io.out.head.ready) + XSPerfAccumulate("inst_spec", PopCount(io.in.map(_.fire))) + XSPerfAccumulate("recovery_bubble", recoveryFlag) + XSPerfAccumulate("frontend_stall_cycle", GatedValidRegNext(!io.in.head.valid && io.backendCanAccept)) + XSPerfAccumulate("backend_stall_cycle", GatedValidRegNext(io.in.head.valid && !io.in.head.ready)) + + XSPerfHistogram("in_valid_range", PopCount(io.in.map(_.valid)), true.B, 0, DecodeWidth + 1, 1) + XSPerfHistogram("in_fire_range", PopCount(io.in.map(_.fire)), true.B, 0, DecodeWidth + 1, 1) + XSPerfHistogram("out_valid_range", PopCount(io.out.map(_.valid)), true.B, 0, DecodeWidth + 1, 1) + XSPerfHistogram("out_fire_range", PopCount(io.out.map(_.fire)), true.B, 0, DecodeWidth + 1, 1) + + val fusionValid = VecInit(io.fusion.map(x => GatedValidRegNext(x))) + val inValidNotReady = io.in.map(in => GatedValidRegNext(in.valid && !in.ready)) + val frontendStall = GatedValidRegNext(!io.in.head.valid && io.backendCanAccept) + val backendStall = GatedValidRegNext(io.in.head.valid && !io.in.head.ready) + + val perfEvents = Seq( + ("decoder_fused_instr", PopCount(fusionValid)), + ("decoder_waitInstr", PopCount(inValidNotReady)), + ("decoder_stall_cycle", hasValid && !io.out(0).ready), + ("decoder_utilization", PopCount(io.in.map(_.valid))), + ("frontend_stall_cycle", frontendStall), + ("backend_stall_cycle", backendStall), + ("INST_SPEC", PopCount(io.in.map(_.fire))), + ("RECOVERY_BUBBLE", recoveryFlag) + ) + + generatePerfEvent() + + // for more readable verilog + dontTouch(isSimpleVec) + dontTouch(isComplexVec) + dontTouch(simplePrefixVec) + dontTouch(complexValid) + dontTouch(complexNum) + dontTouch(readyCounter) + dontTouch(firstComplexOH) +} diff --git a/examples/prefab/cores/xiangshan/arch/native/DecodeUnit.scala b/examples/prefab/cores/xiangshan/arch/native/DecodeUnit.scala new file mode 100644 index 000000000..f0fa090e1 --- /dev/null +++ b/examples/prefab/cores/xiangshan/arch/native/DecodeUnit.scala @@ -0,0 +1,758 @@ +// Copied from XiangShan src/main/scala/xiangshan/backend/decode/DecodeUnit.scala and modified for Buckyball. +// XiangShan is licensed under Mulan PSL v2 (http://license.coscl.org.cn/MulanPSL2). + +package examples.core.xiangshan.native + +import org.chipsalliance.cde.config.Parameters +import chisel3._ +import chisel3.util._ +import freechips.rocketchip.rocket.CSRs +import freechips.rocketchip.rocket.Instructions._ +import freechips.rocketchip.rocket.CustomInstructions._ +import freechips.rocketchip.util.uintToBitPat +import utility._ +import utils._ +import xiangshan.ExceptionNO.{breakPoint, illegalInstr, virtualInstr, EX_II} +import xiangshan._ +import xiangshan.backend.fu.FuType +import xiangshan.backend.Bundles.{DecodedInst, DynInst, StaticInst} +import xiangshan.backend.decode.isa.PseudoInstructions +import xiangshan.backend.decode.isa.bitfield.{InstVType, OPCODE5Bit, XSInstBitFields} +import xiangshan.backend.fu.vector.Bundles.{VType, Vl} +import xiangshan.backend.fu.wrapper.CSRToDecode +import xiangshan.backend.decode.Zimop._ +import yunsuan.{VfaluType, VfcvtType} +import xiangshan.backend.decode._ +import examples.core.xiangshan.native.{Command} + +/** + * Decode unit that takes in a single CtrlFlow and generates a CfCtrl. + */ +class DecodeUnit(implicit p: Parameters) extends XSModule with DecodeUnitConstants { + val io = IO(new DecodeUnitIO) + + val ctrl_flow = io.enq.ctrlFlow // input with RVC Expanded + + private val inst: XSInstBitFields = io.enq.ctrlFlow.instr.asTypeOf(new XSInstBitFields) + + // The accelerator command sits right before CSRRW in the integer table. + val decode_table: Array[(BitPat, List[BitPat])] = + XDecode.table.patch(XDecode.decodeArray.indexWhere(_._1 == CSRRW), Seq(Command.decode), 0) ++ + FpDecode.table ++ +// FDivSqrtDecode.table ++ + BitmanipDecode.table ++ + ScalarCryptoDecode.table ++ + XSDebugDecode.table ++ + CBODecode.table ++ + SvinvalDecode.table ++ + HypervisorDecode.table ++ + VecDecoder.table ++ + ZicondDecode.table ++ + ZimopDecode.table ++ + ZfaDecode.table + + require(decode_table.forall(_._1.width == 32), "Decoded instructions must be 32 bits after RVC expansion") + require(decode_table.map(_._2.length == 15).reduce(_ && _), "Decode tables have different column size") + // assertion for LUI: only LUI should be assigned `selImm === SelImm.IMM_U && fuType === FuType.alu` + val luiMatch = (t: Seq[BitPat]) => t(3).value == FuType.alu.ohid && t.reverse.head.value == SelImm.IMM_U.litValue + val luiTable = decode_table.filter(t => luiMatch(t._2)).map(_._1).distinct + assert(luiTable.length == 1 && luiTable.head == LUI, "Conflicts: LUI is determined by FuType and SelImm in Dispatch") + + // output + val decodedInst: DecodedInst = Wire(new DecodedInst()).decode(ctrl_flow.instr, decode_table) + + val fpDecoder = Module(new FPDecoder) + fpDecoder.io.instr := ctrl_flow.instr + decodedInst.fpu := fpDecoder.io.fpCtrl + decodedInst.fpu.wflags := fpDecoder.io.fpCtrl.wflags || decodedInst.wfflags + + decodedInst.connectStaticInst(io.enq.ctrlFlow) + + decodedInst.uopIdx := 0.U + decodedInst.firstUop := true.B + decodedInst.lastUop := true.B + decodedInst.numUops := 1.U + decodedInst.numWB := 1.U + + val isZimop = (BitPat("b1?00??0111??_?????_100_?????_1110011") === ctrl_flow.instr) || + (BitPat("b1?00??1?????_?????_100_?????_1110011") === ctrl_flow.instr) + + val isMove = BitPat("b000000000000_?????_000_?????_0010011") === ctrl_flow.instr + // temp decode zimop as move + decodedInst.isMove := (isMove || isZimop) && ctrl_flow.instr(RD_MSB, RD_LSB) =/= 0.U && !io.csrCtrl.singlestep + + // fmadd - b1000011 + // fmsub - b1000111 + // fnmsub- b1001011 + // fnmadd- b1001111 + private val isFMA = inst.OPCODE === BitPat("b100??11") + private val isVppu = FuType.isVppu(decodedInst.fuType) + private val isVecOPF = FuType.isVecOPF(decodedInst.fuType) + + // read src1~3 location + decodedInst.lsrc(0) := inst.RS1 + decodedInst.lsrc(1) := inst.RS2 + // src(2) of fma is fs3, src(2) of vector inst is old vd + decodedInst.lsrc(2) := Mux(isFMA, inst.FS3, inst.VD) + decodedInst.lsrc(3) := V0_IDX.U + decodedInst.lsrc(4) := Vl_IDX.U + + // read dest location + decodedInst.ldest := inst.RD + + // init v0Wen vlWen + decodedInst.v0Wen := false.B + decodedInst.vlWen := false.B + + val isCsr = inst.OPCODE5Bit === OPCODE5Bit.SYSTEM && inst.FUNCT3(1, 0) =/= 0.U + val isCsrr = isCsr && inst.FUNCT3 === BitPat("b?1?") && inst.RS1 === 0.U + val isCsrw = isCsr && inst.FUNCT3 === BitPat("b?01") && inst.RD === 0.U + dontTouch(isCsrr) + dontTouch(isCsrw) + + // for csrr vl instruction, convert to vsetvl + val isCsrrVlenb = isCsrr && inst.CSRIDX === CSRs.vlenb.U + val isCsrrVl = isCsrr && inst.CSRIDX === CSRs.vl.U + + private val isCboClean = CBO_CLEAN === io.enq.ctrlFlow.instr + private val isCboFlush = CBO_FLUSH === io.enq.ctrlFlow.instr + private val isCboInval = CBO_INVAL === io.enq.ctrlFlow.instr + private val isCboZero = CBO_ZERO === io.enq.ctrlFlow.instr + + // Note that rnum of aes64ks1i must be in the range 0x0..0xA. The values 0xB..0xF are reserved. + private val isAes64ks1iIllegal = + FuType.FuTypeOrR( + decodedInst.fuType, + FuType.bku + ) && (decodedInst.fuOpType === BKUOpType.aes64ks1i) && inst.isRnumIllegal + + private val isAmocasQ = + FuType.FuTypeOrR(decodedInst.fuType, FuType.mou) && decodedInst.fuOpType === LSUOpType.amocas_q + private val isAmocasQIllegal = isAmocasQ && (inst.RD(0) === 1.U || inst.RS2(0) === 1.U) + + private val exceptionII = + decodedInst.selImm === SelImm.INVALID_INSTR || + io.fromCSR.illegalInst.sfenceVMA && FuType.FuTypeOrR( + decodedInst.fuType, + FuType.fence + ) && decodedInst.fuOpType === FenceOpType.sfence || + io.fromCSR.illegalInst.sfencePart && FuType.FuTypeOrR( + decodedInst.fuType, + FuType.fence + ) && decodedInst.fuOpType === FenceOpType.nofence || + io.fromCSR.illegalInst.hfenceGVMA && FuType.FuTypeOrR( + decodedInst.fuType, + FuType.fence + ) && decodedInst.fuOpType === FenceOpType.hfence_g || + io.fromCSR.illegalInst.hfenceVVMA && FuType.FuTypeOrR( + decodedInst.fuType, + FuType.fence + ) && decodedInst.fuOpType === FenceOpType.hfence_v || + io.fromCSR.illegalInst.hlsv && FuType.FuTypeOrR(decodedInst.fuType, FuType.ldu) && (LSUOpType.isHlv( + decodedInst.fuOpType + ) || LSUOpType.isHlvx(decodedInst.fuOpType)) || + io.fromCSR.illegalInst.hlsv && FuType.FuTypeOrR(decodedInst.fuType, FuType.stu) && LSUOpType.isHsv( + decodedInst.fuOpType + ) || + io.fromCSR.illegalInst.fsIsOff && ( + FuType.FuTypeOrR(decodedInst.fuType, FuType.fpOP ++ Seq(FuType.f2v)) || + (FuType.FuTypeOrR( + decodedInst.fuType, + FuType.ldu + ) && (decodedInst.fuOpType === LSUOpType.lh || decodedInst.fuOpType === LSUOpType.lw || decodedInst.fuOpType === LSUOpType.ld) || + FuType.FuTypeOrR( + decodedInst.fuType, + FuType.stu + ) && (decodedInst.fuOpType === LSUOpType.sh || decodedInst.fuOpType === LSUOpType.sw || decodedInst.fuOpType === LSUOpType.sd)) && decodedInst.instr( + 2 + ) || + inst.isOPFVF || inst.isOPFVV + ) || + io.fromCSR.illegalInst.vsIsOff && (FuType.FuTypeOrR( + decodedInst.fuType, + FuType.vecAll + ) || isCsrrVl || isCsrrVlenb) || + io.fromCSR.illegalInst.wfi && FuType.FuTypeOrR(decodedInst.fuType, FuType.csr) && CSROpType.isWfi( + decodedInst.fuOpType + ) || + io.fromCSR.illegalInst.wrs_nto && FuType.FuTypeOrR(decodedInst.fuType, FuType.csr) && CSROpType.isWrsNto( + decodedInst.fuOpType + ) || + (decodedInst.needFrm.scalaNeedFrm || FuType.isScalaNeedFrm( + decodedInst.fuType + )) && (((decodedInst.fpu.rm === 5.U) || (decodedInst.fpu.rm === 6.U)) || ((decodedInst.fpu.rm === 7.U) && io.fromCSR.illegalInst.frm)) || + (decodedInst.needFrm.vectorNeedFrm || FuType.isVectorNeedFrm(decodedInst.fuType)) && io.fromCSR.illegalInst.frm || + (io.fromCSR.illegalInst.cboZ || !HasCMO.B) && isCboZero || + (io.fromCSR.illegalInst.cboCF || !HasCMO.B) && (isCboClean || isCboFlush) || + (io.fromCSR.illegalInst.cboI || !HasCMO.B) && isCboInval || + isAes64ks1iIllegal || + isAmocasQIllegal + + private val exceptionVI = + io.fromCSR.virtualInst.sfenceVMA && FuType.FuTypeOrR( + decodedInst.fuType, + FuType.fence + ) && decodedInst.fuOpType === FenceOpType.sfence || + io.fromCSR.virtualInst.sfencePart && FuType.FuTypeOrR( + decodedInst.fuType, + FuType.fence + ) && decodedInst.fuOpType === FenceOpType.nofence || + io.fromCSR.virtualInst.hfence && FuType.FuTypeOrR( + decodedInst.fuType, + FuType.fence + ) && (decodedInst.fuOpType === FenceOpType.hfence_g || decodedInst.fuOpType === FenceOpType.hfence_v) || + io.fromCSR.virtualInst.hlsv && FuType.FuTypeOrR(decodedInst.fuType, FuType.ldu) && (LSUOpType.isHlv( + decodedInst.fuOpType + ) || LSUOpType.isHlvx(decodedInst.fuOpType)) || + io.fromCSR.virtualInst.hlsv && FuType.FuTypeOrR(decodedInst.fuType, FuType.stu) && LSUOpType.isHsv( + decodedInst.fuOpType + ) || + io.fromCSR.virtualInst.wfi && FuType.FuTypeOrR(decodedInst.fuType, FuType.csr) && CSROpType.isWfi( + decodedInst.fuOpType + ) || + io.fromCSR.virtualInst.wrs_nto && FuType.FuTypeOrR(decodedInst.fuType, FuType.csr) && CSROpType.isWrsNto( + decodedInst.fuOpType + ) || + io.fromCSR.virtualInst.cboZ && isCboZero || + io.fromCSR.virtualInst.cboCF && (isCboClean || isCboFlush) || + io.fromCSR.virtualInst.cboI && isCboInval + + decodedInst.exceptionVec(illegalInstr) := exceptionII || io.enq.ctrlFlow.exceptionVec(EX_II) + decodedInst.exceptionVec(virtualInstr) := exceptionVI + + //update exceptionVec: from frontend trigger's breakpoint exception. To reduce 1 bit of overhead in ibuffer entry. + decodedInst.exceptionVec(breakPoint) := TriggerAction.isExp(ctrl_flow.trigger) + + decodedInst.imm := LookupTree( + decodedInst.selImm, + ImmUnion.immSelMap.map { + x => + val minBits = x._2.minBitsFromInstr(ctrl_flow.instr) + require(minBits.getWidth == x._2.len) + x._1 -> minBits + } + ) + + private val isLs = FuType.isLoadStore(decodedInst.fuType) + private val isVls = inst.isVecStore || inst.isVecLoad + private val isStore = FuType.isStore(decodedInst.fuType) + private val isAMO = FuType.isAMO(decodedInst.fuType) + private val isVStore = FuType.isVStore(decodedInst.fuType) + private val isBranch = !decodedInst.preDecodeInfo.notCFI || FuType.isJump(decodedInst.fuType) + + decodedInst.commitType := Cat(isLs | isVls, (isStore && !isAMO) | isVStore | isBranch) + + decodedInst.isVset := FuType.isVset(decodedInst.fuType) + + private val needReverseInsts = Seq(VRSUB_VI, VRSUB_VX, VFRDIV_VF, VFRSUB_VF, VFMV_F_S) + private val vextInsts = Seq(VZEXT_VF2, VZEXT_VF4, VZEXT_VF8, VSEXT_VF2, VSEXT_VF4, VSEXT_VF8) + + private val narrowInsts = Seq( + VNSRA_WV, + VNSRA_WX, + VNSRA_WI, + VNSRL_WV, + VNSRL_WX, + VNSRL_WI, + VNCLIP_WV, + VNCLIP_WX, + VNCLIP_WI, + VNCLIPU_WV, + VNCLIPU_WX, + VNCLIPU_WI + ) + + private val maskDstInsts = Seq( + VMADC_VV, + VMADC_VX, + VMADC_VI, + VMADC_VVM, + VMADC_VXM, + VMADC_VIM, + VMSBC_VV, + VMSBC_VX, + VMSBC_VVM, + VMSBC_VXM, + VMAND_MM, + VMNAND_MM, + VMANDN_MM, + VMXOR_MM, + VMOR_MM, + VMNOR_MM, + VMORN_MM, + VMXNOR_MM, + VMSEQ_VV, + VMSEQ_VX, + VMSEQ_VI, + VMSNE_VV, + VMSNE_VX, + VMSNE_VI, + VMSLE_VV, + VMSLE_VX, + VMSLE_VI, + VMSLEU_VV, + VMSLEU_VX, + VMSLEU_VI, + VMSLT_VV, + VMSLT_VX, + VMSLTU_VV, + VMSLTU_VX, + VMSGT_VX, + VMSGT_VI, + VMSGTU_VX, + VMSGTU_VI, + VMFEQ_VV, + VMFEQ_VF, + VMFNE_VV, + VMFNE_VF, + VMFLT_VV, + VMFLT_VF, + VMFLE_VV, + VMFLE_VF, + VMFGT_VF, + VMFGE_VF + ) + + private val maskOpInsts = Seq( + VMAND_MM, + VMNAND_MM, + VMANDN_MM, + VMXOR_MM, + VMOR_MM, + VMNOR_MM, + VMORN_MM, + VMXNOR_MM + ) + + private val vmaInsts = Seq( + VMACC_VV, + VMACC_VX, + VNMSAC_VV, + VNMSAC_VX, + VMADD_VV, + VMADD_VX, + VNMSUB_VV, + VNMSUB_VX, + VWMACCU_VV, + VWMACCU_VX, + VWMACC_VV, + VWMACC_VX, + VWMACCSU_VV, + VWMACCSU_VX, + VWMACCUS_VX + ) + + private val wfflagsInsts = Seq( + // opfff + FADD_S, + FSUB_S, + FADD_D, + FSUB_D, + FADD_H, + FSUB_H, + FEQ_S, + FLT_S, + FLE_S, + FEQ_D, + FLT_D, + FLE_D, + FEQ_H, + FLT_H, + FLE_H, + FMIN_S, + FMAX_S, + FMIN_D, + FMAX_D, + FMIN_H, + FMAX_H, + FMUL_S, + FMUL_D, + FMUL_H, + FDIV_S, + FDIV_D, + FSQRT_S, + FSQRT_D, + FDIV_H, + FSQRT_H, + FMADD_S, + FMSUB_S, + FNMADD_S, + FNMSUB_S, + FMADD_D, + FMSUB_D, + FNMADD_D, + FNMSUB_D, + FMADD_H, + FMSUB_H, + FNMADD_H, + FNMSUB_H, + FSGNJ_S, + FSGNJN_S, + FSGNJX_S, + FSGNJ_H, + FSGNJN_H, + FSGNJX_H, + // opfvv + VFADD_VV, + VFSUB_VV, + VFWADD_VV, + VFWSUB_VV, + VFWADD_WV, + VFWSUB_WV, + VFMUL_VV, + VFDIV_VV, + VFWMUL_VV, + VFMACC_VV, + VFNMACC_VV, + VFMSAC_VV, + VFNMSAC_VV, + VFMADD_VV, + VFNMADD_VV, + VFMSUB_VV, + VFNMSUB_VV, + VFWMACC_VV, + VFWNMACC_VV, + VFWMSAC_VV, + VFWNMSAC_VV, + VFSQRT_V, + VFMIN_VV, + VFMAX_VV, + VMFEQ_VV, + VMFNE_VV, + VMFLT_VV, + VMFLE_VV, + VFSGNJ_VV, + VFSGNJN_VV, + VFSGNJX_VV, + // opfvf + VFADD_VF, + VFSUB_VF, + VFRSUB_VF, + VFWADD_VF, + VFWSUB_VF, + VFWADD_WF, + VFWSUB_WF, + VFMUL_VF, + VFDIV_VF, + VFRDIV_VF, + VFWMUL_VF, + VFMACC_VF, + VFNMACC_VF, + VFMSAC_VF, + VFNMSAC_VF, + VFMADD_VF, + VFNMADD_VF, + VFMSUB_VF, + VFNMSUB_VF, + VFWMACC_VF, + VFWNMACC_VF, + VFWMSAC_VF, + VFWNMSAC_VF, + VFMIN_VF, + VFMAX_VF, + VMFEQ_VF, + VMFNE_VF, + VMFLT_VF, + VMFLE_VF, + VMFGT_VF, + VMFGE_VF, + VFSGNJ_VF, + VFSGNJN_VF, + VFSGNJX_VF, + // vfred + VFREDOSUM_VS, + VFREDUSUM_VS, + VFREDMAX_VS, + VFREDMIN_VS, + VFWREDOSUM_VS, + VFWREDUSUM_VS, + // fcvt & vfcvt + FCVT_S_W, + FCVT_S_WU, + FCVT_S_L, + FCVT_S_LU, + FCVT_W_S, + FCVT_WU_S, + FCVT_L_S, + FCVT_LU_S, + FCVT_D_W, + FCVT_D_WU, + FCVT_D_L, + FCVT_D_LU, + FCVT_W_D, + FCVT_WU_D, + FCVT_L_D, + FCVT_LU_D, + FCVT_S_D, + FCVT_D_S, + FCVT_S_H, + FCVT_H_S, + FCVT_H_D, + FCVT_D_H, + FCVT_H_W, + FCVT_H_WU, + FCVT_H_L, + FCVT_H_LU, + FCVT_W_H, + FCVT_WU_H, + FCVT_L_H, + FCVT_LU_H, + VFCVT_XU_F_V, + VFCVT_X_F_V, + VFCVT_RTZ_XU_F_V, + VFCVT_RTZ_X_F_V, + VFCVT_F_XU_V, + VFCVT_F_X_V, + VFWCVT_XU_F_V, + VFWCVT_X_F_V, + VFWCVT_RTZ_XU_F_V, + VFWCVT_RTZ_X_F_V, + VFWCVT_F_XU_V, + VFWCVT_F_X_V, + VFWCVT_F_F_V, + VFNCVT_XU_F_W, + VFNCVT_X_F_W, + VFNCVT_RTZ_XU_F_W, + VFNCVT_RTZ_X_F_W, + VFNCVT_F_XU_W, + VFNCVT_F_X_W, + VFNCVT_F_F_W, + VFNCVT_ROD_F_F_W, + VFRSQRT7_V, + VFREC7_V, + // zfa + FLEQ_H, + FLEQ_S, + FLEQ_D, + FLTQ_H, + FLTQ_S, + FLTQ_D, + FMINM_H, + FMINM_S, + FMINM_D, + FMAXM_H, + FMAXM_S, + FMAXM_D, + FROUND_H, + FROUND_S, + FROUND_D, + FROUNDNX_H, + FROUNDNX_S, + FROUNDNX_D, + FCVTMOD_W_D + ) + + private val scalaNeedFrmInsts = Seq( + FADD_S, + FSUB_S, + FADD_D, + FSUB_D, + FADD_H, + FSUB_H, + FCVT_W_S, + FCVT_WU_S, + FCVT_L_S, + FCVT_LU_S, + FCVT_W_D, + FCVT_WU_D, + FCVT_L_D, + FCVT_LU_D, + FCVT_S_D, + FCVT_D_S, + FCVT_W_H, + FCVT_WU_H, + FCVT_L_H, + FCVT_LU_H, + FCVT_S_H, + FCVT_H_S, + FCVT_H_D, + FCVT_D_H, + FROUND_H, + FROUND_S, + FROUND_D, + FROUNDNX_H, + FROUNDNX_S, + FROUNDNX_D + ) + + private val vectorNeedFrmInsts = Seq( + VFSLIDE1UP_VF, + VFSLIDE1DOWN_VF + ) + + decodedInst.wfflags := wfflagsInsts.map(_ === inst.ALL).reduce(_ || _) + decodedInst.needFrm.scalaNeedFrm := scalaNeedFrmInsts.map(_ === inst.ALL).reduce(_ || _) + decodedInst.needFrm.vectorNeedFrm := vectorNeedFrmInsts.map(_ === inst.ALL).reduce(_ || _) + decodedInst.vpu := 0.U.asTypeOf(decodedInst.vpu) // Todo: Connect vpu decoder + decodedInst.vpu.vill := io.enq.vtype.illegal + decodedInst.vpu.vma := io.enq.vtype.vma + decodedInst.vpu.vta := io.enq.vtype.vta + decodedInst.vpu.vsew := io.enq.vtype.vsew + decodedInst.vpu.vlmul := io.enq.vtype.vlmul + decodedInst.vpu.vm := inst.VM + decodedInst.vpu.nf := inst.NF + decodedInst.vpu.veew := inst.WIDTH + decodedInst.vpu.isReverse := needReverseInsts.map(_ === inst.ALL).reduce(_ || _) + decodedInst.vpu.isExt := vextInsts.map(_ === inst.ALL).reduce(_ || _) + val isNarrow = narrowInsts.map(_ === inst.ALL).reduce(_ || _) + val isDstMask = maskDstInsts.map(_ === inst.ALL).reduce(_ || _) + val isOpMask = maskOpInsts.map(_ === inst.ALL).reduce(_ || _) + val isVload = FuType.isVLoad(decodedInst.fuType) + val isVlx = isVload && (decodedInst.fuOpType === VlduType.vloxe || decodedInst.fuOpType === VlduType.vluxe) + val isVle = + isVload && (decodedInst.fuOpType === VlduType.vle || decodedInst.fuOpType === VlduType.vleff || decodedInst.fuOpType === VlduType.vlse) + val isVlm = isVload && (decodedInst.fuOpType === VlduType.vlm) + val isFof = isVload && (decodedInst.fuOpType === VlduType.vleff) + val isWritePartVd = + decodedInst.uopSplitType === UopSplitType.VEC_VRED || decodedInst.uopSplitType === UopSplitType.VEC_0XV || decodedInst.uopSplitType === UopSplitType.VEC_VWW + val isVma = vmaInsts.map(_ === inst.ALL).reduce(_ || _) + val emulIsFrac = + Cat(~decodedInst.vpu.vlmul(2), decodedInst.vpu.vlmul(1, 0)) +& decodedInst.vpu.veew < 4.U +& decodedInst.vpu.vsew + val vstartIsNotZero = io.enq.vstart =/= 0.U + decodedInst.vpu.isNarrow := isNarrow + decodedInst.vpu.isDstMask := isDstMask + decodedInst.vpu.isOpMask := isOpMask + decodedInst.vpu.isDependOldVd := isVppu || isVecOPF || isVStore || (isDstMask && !isOpMask) || isNarrow || isVlx || isVma || isFof || vstartIsNotZero + decodedInst.vpu.isWritePartVd := isWritePartVd || isVlm || isVle && emulIsFrac + decodedInst.vpu.vstart := io.enq.vstart + decodedInst.vpu.isVleff := isFof && inst.NF === 0.U + decodedInst.vpu.specVill := io.enq.vtype.illegal + decodedInst.vpu.specVma := io.enq.vtype.vma + decodedInst.vpu.specVta := io.enq.vtype.vta + decodedInst.vpu.specVsew := io.enq.vtype.vsew + decodedInst.vpu.specVlmul := io.enq.vtype.vlmul + + decodedInst.vlsInstr := isVls + + decodedInst.srcType(3) := Mux(inst.VM === 0.U, SrcType.vp, SrcType.DC) // mask src + decodedInst.srcType(4) := SrcType.vp // vconfig + + val uopInfoGen = Module(new UopInfoGen) + uopInfoGen.io.in.preInfo.isVecArith := inst.isVecArith + uopInfoGen.io.in.preInfo.isVecMem := inst.isVecStore || inst.isVecLoad + uopInfoGen.io.in.preInfo.isAmoCAS := inst.isAMOCAS + + uopInfoGen.io.in.preInfo.typeOfSplit := decodedInst.uopSplitType + uopInfoGen.io.in.preInfo.vsew := decodedInst.vpu.vsew + uopInfoGen.io.in.preInfo.vlmul := decodedInst.vpu.vlmul + uopInfoGen.io.in.preInfo.vwidth := inst.RM + uopInfoGen.io.in.preInfo.vmvn := inst.IMM5_OPIVI(2, 0) + uopInfoGen.io.in.preInfo.nf := inst.NF + uopInfoGen.io.in.preInfo.isVlsr := decodedInst.fuOpType === VlduType.vlr || decodedInst.fuOpType === VstuType.vsr + uopInfoGen.io.in.preInfo.isVlsm := decodedInst.fuOpType === VlduType.vlm || decodedInst.fuOpType === VstuType.vsm + io.deq.isComplex := uopInfoGen.io.out.isComplex + io.deq.uopInfo.numOfUop := uopInfoGen.io.out.uopInfo.numOfUop + io.deq.uopInfo.numOfWB := uopInfoGen.io.out.uopInfo.numOfWB + io.deq.uopInfo.lmul := uopInfoGen.io.out.uopInfo.lmul + + // decode for SoftPrefetch instructions (prefetch.w / prefetch.r / prefetch.i) + val isSoftPrefetch = inst.OPCODE === BitPat("b0010011") && inst.FUNCT3 === BitPat("b110") && inst.RD === 0.U + val isPreW = isSoftPrefetch && inst.RS2 === 3.U(5.W) + val isPreR = isSoftPrefetch && inst.RS2 === 1.U(5.W) + val isPreI = isSoftPrefetch && inst.RS2 === 0.U(5.W) + + // for fli.s|fli.d instruction + val isFLI = + inst.FUNCT7 === BitPat("b11110??") && inst.RS2 === 1.U && inst.RM === 0.U && inst.OPCODE5Bit === OPCODE5Bit.OP_FP + + when(isCsrrVl) { + // convert to vsetvl instruction + decodedInst.srcType(0) := SrcType.no + decodedInst.srcType(1) := SrcType.no + decodedInst.srcType(2) := SrcType.no + decodedInst.srcType(3) := SrcType.no + decodedInst.srcType(4) := SrcType.vp + decodedInst.lsrc(4) := Vl_IDX.U + decodedInst.waitForward := false.B + decodedInst.blockBackward := false.B + }.elsewhen(isCsrrVlenb) { + // convert to addi instruction + decodedInst.srcType(0) := SrcType.reg + decodedInst.srcType(1) := SrcType.imm + decodedInst.srcType(2) := SrcType.no + decodedInst.srcType(3) := SrcType.no + decodedInst.srcType(4) := SrcType.no + decodedInst.selImm := SelImm.IMM_I + decodedInst.waitForward := false.B + decodedInst.blockBackward := false.B + decodedInst.canRobCompress := true.B + }.elsewhen(isPreW || isPreR || isPreI) { + decodedInst.selImm := SelImm.IMM_S + decodedInst.fuType := FuType.ldu.U + decodedInst.canRobCompress := false.B + }.elsewhen(isZimop) { + // set srcType for zimop + decodedInst.srcType(0) := SrcType.reg + decodedInst.srcType(1) := SrcType.imm + // use x0 as src1 + decodedInst.lsrc(0) := 0.U + } + + io.deq.decodedInst := decodedInst + io.deq.decodedInst.rfWen := (decodedInst.ldest =/= 0.U) && decodedInst.rfWen + io.deq.decodedInst.fuType := Mux1H(Seq( + // keep condition + (!FuType.FuTypeOrR(decodedInst.fuType, FuType.vldu, FuType.vstu) && !isCsrrVl && !isCsrrVlenb) -> decodedInst.fuType, + isCsrrVl -> FuType.vsetfwf.U, + isCsrrVlenb -> FuType.alu.U, + // change vlsu to vseglsu when NF =/= 0.U + (FuType.FuTypeOrR( + decodedInst.fuType, + FuType.vldu, + FuType.vstu + ) && inst.NF === 0.U || (inst.NF =/= 0.U && (inst.MOP === "b00".U && inst.SUMOP === "b01000".U))) -> decodedInst.fuType, + // MOP === b00 && SUMOP === b01000: unit-stride whole register store + // MOP =/= b00 : strided and indexed store + (FuType.FuTypeOrR( + decodedInst.fuType, + FuType.vstu + ) && inst.NF =/= 0.U && ((inst.MOP === "b00".U && inst.SUMOP =/= "b01000".U) || inst.MOP =/= "b00".U)) -> FuType.vsegstu.U, + // MOP === b00 && LUMOP === b01000: unit-stride whole register load + // MOP =/= b00 : strided and indexed load + (FuType.FuTypeOrR( + decodedInst.fuType, + FuType.vldu + ) && inst.NF =/= 0.U && ((inst.MOP === "b00".U && inst.LUMOP =/= "b01000".U) || inst.MOP =/= "b00".U)) -> FuType.vsegldu.U + )) + io.deq.decodedInst.imm := MuxCase( + decodedInst.imm, + Seq( + isCsrrVlenb -> (VLEN / 8).U, + isZimop -> 0.U + ) + ) + + io.deq.decodedInst.fuOpType := MuxCase( + decodedInst.fuOpType, + Seq( + isCsrrVl -> VSETOpType.csrrvl, + isCsrrVlenb -> ALUOpType.add, + isFLI -> Cat(1.U, inst.FMT, inst.RS1), + (isPreW || isPreR || isPreI) -> Mux1H(Seq( + isPreW -> LSUOpType.prefetch_w, + isPreR -> LSUOpType.prefetch_r, + isPreI -> LSUOpType.prefetch_i + )), + (isCboInval && io.fromCSR.special.cboI2F) -> LSUOpType.cbo_flush + ) + ) + + // Don't compress in the same Rob entry when crossing Ftq entry boundary + io.deq.decodedInst.canRobCompress := decodedInst.canRobCompress && !io.enq.ctrlFlow.isLastInFtqEntry + + //------------------------------------------------------------- + // Debug Info +// XSDebug("in: instr=%x pc=%x excepVec=%b crossPageIPFFix=%d\n", +// io.enq.ctrl_flow.instr, io.enq.ctrl_flow.pc, io.enq.ctrl_flow.exceptionVec.asUInt, +// io.enq.ctrl_flow.crossPageIPFFix) +// XSDebug("out: srcType(0)=%b srcType(1)=%b srcType(2)=%b lsrc(0)=%d lsrc(1)=%d lsrc(2)=%d ldest=%d fuType=%b + // fuOpType=%b\n", +// io.deq.cf_ctrl.ctrl.srcType(0), io.deq.cf_ctrl.ctrl.srcType(1), io.deq.cf_ctrl.ctrl.srcType(2), +// io.deq.cf_ctrl.ctrl.lsrc(0), io.deq.cf_ctrl.ctrl.lsrc(1), io.deq.cf_ctrl.ctrl.lsrc(2), +// io.deq.cf_ctrl.ctrl.ldest, io.deq.cf_ctrl.ctrl.fuType, io.deq.cf_ctrl.ctrl.fuOpType) +// XSDebug("out: rfWen=%d fpWen=%d isXSTrap=%d noSpecExec=%d isBlocked=%d flushPipe=%d imm=%x\n", +// io.deq.cf_ctrl.ctrl.rfWen, io.deq.cf_ctrl.ctrl.fpWen, io.deq.cf_ctrl.ctrl.isXSTrap, +// io.deq.cf_ctrl.ctrl.noSpecExec, io.deq.cf_ctrl.ctrl.blockBackward, io.deq.cf_ctrl.ctrl.flushPipe, +// io.deq.cf_ctrl.ctrl.imm) +// XSDebug("out: excepVec=%b\n", io.deq.cf_ctrl.cf.exceptionVec.asUInt) +} diff --git a/examples/prefab/cores/xiangshan/arch/native/Native.scala b/examples/prefab/cores/xiangshan/arch/native/Native.scala new file mode 100644 index 000000000..84e364ebf --- /dev/null +++ b/examples/prefab/cores/xiangshan/arch/native/Native.scala @@ -0,0 +1,204 @@ +package examples.core.xiangshan.native + +import chisel3._ +import chisel3.experimental.annotate +import sifive.enterprise.firrtl.NestedPrefixModulesAnnotation +import chisel3.util._ +import org.chipsalliance.cde.config.{Config, Parameters} +import freechips.rocketchip.amba.axi4._ +import freechips.rocketchip.diplomacy._ +import freechips.rocketchip.interrupts._ +import freechips.rocketchip.tilelink._ +import coupledL2.{EnableCHI, L2ParamKey} +import coupledL2.tl2chi.{CHIAddrWidthKey, CHIIssue, PortIO} +import system.SoCParamsKey +import top.{ArgParser, Generator, SeperatedBusType} +import utility.{ChiselDB, Constantin} +import freechips.rocketchip.util.uintToBitPat +import xiangshan._ +import xiangshan.backend.decode.XSDecode +import xiangshan.backend.fu.FuType +import examples.core.xiangshan.native.XSTile + +class Context(implicit p: Parameters) extends XSBundle with HasPMParameters { + val satp = UInt(64.W) + val privilege = UInt(2.W) + val sum = Bool() + val mxr = Bool() + val virt = Bool() + + val pmp = Vec( + NumPMP, + new Bundle { + val cfg = UInt(8.W) + val addr = UInt((PMPAddrBits - 2).W) + } + ) + +} + +/** Accelerator command: custom-3 with funct3 = 3 and rd = 0, executed by the CSR unit. */ +object Command { + def opType = 0.U(FuOpTypeWidth.W) + + def decode = BitPat("b???????_?????_?????_011_00000_1111011") -> + XSDecode( + SrcType.reg, + SrcType.reg, + SrcType.X, + FuType.csr, + opType, + SelImm.IMM_Z, + xWen = true, + noSpec = true, + blockBack = true + ) + .generate() + +} + +class Commands(implicit p: Parameters) extends XSBundle { + + val request = Decoupled(new Bundle { + val instruction = UInt(32.W) + val pc = UInt(64.W) + val rs1 = UInt(64.W) + val rs2 = UInt(64.W) + val context = new Context + }) + + val complete = Flipped(Valid(Bool())) +} + +/** Native XiangShan tile boundary; its caches retain the upstream CHI endpoint. */ +class Native(implicit p: Parameters) extends LazyModule with BindingScope { + val tile = LazyModule(new XSTile) + val resetSource = BundleBridgeSource(() => Reset()) + tile.core_reset_sink := resetSource + val clint = IntSourceNode(IntSourcePortSimple(2, ports = 1, sources = 1)) + val plic = IntSourceNode(IntSourcePortSimple(1, ports = 2, sources = 1)) + val debug = IntSourceNode(IntSourcePortSimple(1, ports = 1, sources = 1)) + val nmi = IntSourceNode(IntSourcePortSimple(2, ports = 1, sources = 1)) + val busError = IntSinkNode(IntSinkPortSimple(1, 1)) + tile.clint_int_node := clint + tile.plic_int_node :*= plic + tile.debug_int_node := debug + tile.nmi_int_node := nmi + busError := tile.beu_int_source + + val uncached = AXI4SlaveNode(Seq(AXI4SlavePortParameters( + slaves = Seq(AXI4SlaveParameters( + address = AddressSet(0, 0x7fffffffL).subtract(p(SoCParamsKey).BEURange), + regionType = RegionType.UNCACHED, + executable = true, + supportsRead = TransferSizes(1, 16), + supportsWrite = TransferSizes(1, 16), + interleavedId = Some(0) + )), + beatBytes = 16 + ))) + + uncached := AXI4Buffer() := AXI4UserYanker() := AXI4IdIndexer(4) := + AXI4Deinterleaver(64) := TLToAXI4() := TLWidthWidget(8) := tile.sep_tl_opt.get + + lazy val module = new LazyModuleImp(this) with RequireAsyncReset { + annotate(this)(Seq(NestedPrefixModulesAnnotation(this.toNamed, "XiangShan", true))) + + val io = IO(new Bundle { + val hartId = Input(UInt(p(freechips.rocketchip.tile.MaxHartIdBits).W)) + val resetVector = Input(UInt(p(SoCParamsKey).PAddrBits.W)) + val nodeId = Input(UInt(7.W)) + val interrupts = Input(Vec(4, Bool())) + val commands = new Commands + val timeRequest = Decoupled(Bool()) + val time = Flipped(Valid(UInt(64.W))) + val chi = new PortIO + }) + + val uncachedAXI = uncached.makeIOs() + io.chi <> tile.module.io.chi.get + tile.module.io.nodeID.get := io.nodeId + tile.module.io.hartId := io.hartId + tile.module.io.reset_vector := io.resetVector + tile.module.io.reset_mtvec.foreach(_ := io.resetVector) + tile.module.io.msiInfo.valid := false.B + tile.module.io.msiInfo.bits := 0.U + tile.module.io.teemsiInfo.foreach { x => x.valid := false.B; x.bits := 0.U } + tile.module.io.traceCoreInterface.fromEncoder.enable := false.B + tile.module.io.traceCoreInterface.fromEncoder.stall := false.B + tile.module.io.debugTopDown.l3MissMatch := false.B + tile.module.io.l3Miss := false.B + tile.module.io.clintTime <> io.time + io.commands <> tile.module.io.commands + io.timeRequest <> tile.module.io.timeRequest + resetSource.out.head._1 := false.B.asAsyncReset + clint.out.head._1(0) := io.interrupts(0) + clint.out.head._1(1) := io.interrupts(1) + plic.out(0)._1(0) := io.interrupts(2) || busError.in.head._1(0) + plic.out(1)._1(0) := io.interrupts(3) + debug.out.head._1(0) := false.B + nmi.out.head._1.foreach(_ := false.B) + } + +} + +object Emit extends App { + + val (base, stageArgs, upstreamOptions) = ArgParser.parse( + Array("--config", "KunminghuV2Config", "--fpga-platform", "--disable-perf", "--disable-xmr") ++ args + ) + + val profile = base.alter((site, _, up) => { + case EnableCHI => true + case CHIIssue => "B" + case CHIAddrWidthKey => 48 + case SoCParamsKey => up(SoCParamsKey).copy( + PAddrBits = 48, + EnableICacheCtrl = false, + EnableDCacheCtrl = false, + SeperateBus = SeperatedBusType.TL, + SeperateBusRanges = Seq(AddressSet(0, 0x7fffffffL)) + ) + case DFTOptionsKey => up(DFTOptionsKey).copy(EnableMbist = false, EnableSramCtl = false) + case DebugOptionsKey => up(DebugOptionsKey).copy( + FPGAPlatform = true, + EnableDifftest = false, + AlwaysBasicDiff = false, + EnableDebug = false, + EnablePerfDebug = false, + EnableChiselDB = false, + AlwaysBasicDB = false + ) + }) + + val selected = profile(XSTileKey).head + + val core = selected.copy(L2CacheParamsOpt = + selected.L2CacheParamsOpt.map( + _.copy(dataCheck = None, enablePoison = false) + ) + ) + + val soc = profile(SoCParamsKey) + + val config = profile.alterPartial { + case XSCoreParamsKey => core + case SoCParamsKey => soc + case L2ParamKey => core.L2CacheParamsOpt.get + } + + Constantin.init(false) + ChiselDB.init(false) + val native = LazyModule(new Native()(config)) + val output = args.sliding(2).collectFirst { case Array("--target-dir", path) => path }.get + Generator.execute( + stageArgs, + native.module, + upstreamOptions ++ Array( + "--split-verilog", + s"-o=$output", + "--disable-annotation-unknown", + "--lowering-options=explicitBitcast,disallowLocalVariables,disallowPortDeclSharing" + ) + ) +} diff --git a/examples/prefab/cores/xiangshan/arch/native/Tile.scala b/examples/prefab/cores/xiangshan/arch/native/Tile.scala new file mode 100644 index 000000000..892ec44a8 --- /dev/null +++ b/examples/prefab/cores/xiangshan/arch/native/Tile.scala @@ -0,0 +1,231 @@ +// Copied from XiangShan src/main/scala/xiangshan/XSTile.scala and modified for Buckyball. +// XiangShan is licensed under Mulan PSL v2 (http://license.coscl.org.cn/MulanPSL2). + +package examples.core.xiangshan.native + +import org.chipsalliance.cde.config.{Config, Parameters} +import chisel3._ +import chisel3.util.{log2Up, Decoupled, Valid, ValidIO} +import freechips.rocketchip.diplomacy._ +import freechips.rocketchip.interrupts._ +import freechips.rocketchip.tile.{BusErrorUnit, BusErrorUnitParams, BusErrors} +import freechips.rocketchip.tilelink._ +import freechips.rocketchip.amba.axi4._ +import system.HasSoCParameter +import top.{ArgParser, BusPerfMonitor, Generator} +import utility._ +import utility.sram.SramBroadcastBundle +import coupledL2.{EnableCHI, L2ParamKey} +import coupledL2.tl2chi.{LCrdyIn, PortIO} +import xiangshan.backend.trace.TraceCoreInterface +import xiangshan._ +import examples.core.xiangshan.native.{Commands, XSCore} + +class XSTile()(implicit p: Parameters) extends LazyModule with HasXSParameter with HasSoCParameter { + override def shouldBeInlined: Boolean = false + val core = LazyModule(new XSCore()) + val l2top = LazyModule(new L2Top()) + + val enableL2 = coreParams.L2CacheParamsOpt.isDefined + val txSourceReady = p(L2ParamKey).txSourceReady +// =========== Public Ports ============ + val memBlock = core.memBlock.inner + val core_l3_pf_port = memBlock.l3_pf_sender_opt + val memory_port = if (enableCHI && enableL2) None else Some(l2top.inner.memory_port.get) + val tl_uncache = l2top.inner.mmio_port + val sep_tl_opt = l2top.inner.sep_tl_port_opt + val beu_int_source = l2top.inner.beu.intNode + val core_reset_sink = BundleBridgeSink(Some(() => Reset())) + val clint_int_node = l2top.inner.clint_int_node + val plic_int_node = l2top.inner.plic_int_node + val debug_int_node = l2top.inner.debug_int_node + val nmi_int_node = l2top.inner.nmi_int_node + memBlock.clint_int_sink := IntBuffer() := clint_int_node + memBlock.plic_int_sink :*= IntBuffer() :*= plic_int_node + memBlock.debug_int_sink := IntBuffer() := debug_int_node + memBlock.nmi_int_sink := IntBuffer() := nmi_int_node + memBlock.beu_local_int_sink := l2top.inner.beu_local_int_source_buffer + + // =========== Components' Connection ============ + // L1 to l1_xbar + coreParams.dcacheParametersOpt.map { _ => + l2top.inner.misc_l2_pmu := l2top.inner.l1d_logger := memBlock.dcache_port := + memBlock.l1d_to_l2_buffer.node := memBlock.dcache.clientNode + } + + l2top.inner.misc_l2_pmu := l2top.inner.l1i_logger := memBlock.frontendBridge.icache_node + if (!coreParams.softPTW) { + l2top.inner.misc_l2_pmu := l2top.inner.ptw_logger := l2top.inner.ptw_to_l2_buffer.node := memBlock.ptw_to_l2_buffer.node + } + + // L2 Prefetch + l2top.inner.l2cache match { + case Some(l2) => + l2.pf_recv_node.foreach { recv => + println("Connecting L1 prefetcher to L2!") + recv := memBlock.l2_pf_sender_opt.get + } + case None => + } + + val core_l3_tpmeta_source_port = l2top.inner.l2cache match { + case Some(l2) => l2.tpmeta_source_node + case None => None + } + + val core_l3_tpmeta_sink_port = l2top.inner.l2cache match { + case Some(l2) => l2.tpmeta_sink_node + case None => None + } + + // mmio + l2top.inner.i_mmio_port := l2top.inner.i_mmio_buffer.node := memBlock.frontendBridge.instr_uncache_node + if (soc.EnableICacheCtrl) { + memBlock.frontendBridge.icachectrl_node := l2top.inner.icachectrl_port_opt.get + } + l2top.inner.d_mmio_port := memBlock.uncache_port + + // =========== IO Connection ============ + class XSTileImp(wrapper: LazyModule) extends LazyModuleImp(wrapper) { + + val io = IO(new Bundle { + val hartId = Input(UInt(hartIdLen.W)) + val msiInfo = Input(ValidIO(UInt(soc.IMSICParams.MSI_INFO_WIDTH.W))) + val msiAck = Output(Bool()) + val teemsiInfo = Option.when(soc.IMSICParams.HasTEEIMSIC)(Input(ValidIO(UInt(soc.IMSICParams.MSI_INFO_WIDTH.W)))) + val teemsiAck = Option.when(soc.IMSICParams.HasTEEIMSIC)(Output(Bool())) + val reset_vector = Input(UInt(PAddrBits.W)) + val reset_mtvec = Option.when(enableResetMtvec)(Input(UInt(PAddrBits.W))) + val cpu_halt = Output(Bool()) + val cpu_crtical_error = Output(Bool()) + val hartIsInReset = Output(Bool()) + val traceCoreInterface = new TraceCoreInterface + + val debugTopDown = new Bundle { + val robHeadPaddr = Valid(UInt(PAddrBits.W)) + val l3MissMatch = Input(Bool()) + } + + val l3Miss = Input(Bool()) + val chi = if (enableCHI) Some(new PortIO) else None + val nodeID = if (enableCHI) Some(Input(UInt(NodeIDWidth.W))) else None + val commands = new Commands + val timeRequest = Decoupled(Bool()) + val clintTime = Input(ValidIO(UInt(64.W))) + val dft = Option.when(hasDFT)(Input(new SramBroadcastBundle)) + val dft_reset = Option.when(hasMbist)(Input(new DFTResetSignals())) + val l2_flush_en = Option.when(EnablePowerDown)(Output(Bool())) + val l2_flush_done = Option.when(EnablePowerDown)(Output(Bool())) + val lcrdy = Option.when(txSourceReady)(new LCrdyIn) + }) + + dontTouch(io.hartId) + dontTouch(io.msiInfo) + io.teemsiInfo.foreach(dontTouch(_)) + if (!io.chi.isEmpty) { dontTouch(io.chi.get) } + + val core_soft_rst = core_reset_sink.in.head._1 // unused + + l2top.module.io.hartId.fromTile := io.hartId + core.module.io.hartId := l2top.module.io.hartId.toCore + core.module.io.reset_vector := l2top.module.io.reset_vector.toCore + core.module.io.reset_mtvec.foreach(_ := l2top.module.io.reset_mtvec.get.toCore) + core.module.io.msiInfo := l2top.module.io.msiInfo.toCore + l2top.module.io.msiInfo.fromTile := io.msiInfo + core.module.io.teemsiInfo zip l2top.module.io.teemsiInfo foreach { case (core_teemsiInfo, l2top_teemsiInfo) => + core_teemsiInfo := l2top_teemsiInfo.toCore + } + l2top.module.io.teemsiInfo zip io.teemsiInfo foreach { case (l2top_teemsiInfo, io_teemsiInfo) => + l2top_teemsiInfo.fromTile := io_teemsiInfo + } + io.commands <> core.module.io.commands + io.timeRequest <> core.module.io.timeRequest + core.module.io.clintTime := l2top.module.io.clintTime.toCore + l2top.module.io.clintTime.fromTile := io.clintTime + l2top.module.io.reset_vector.fromTile := io.reset_vector + l2top.module.io.reset_mtvec.foreach(x => x.fromTile := io.reset_mtvec.get) + l2top.module.io.cpu_halt.fromCore := core.module.io.cpu_halt + io.cpu_halt := l2top.module.io.cpu_halt.toTile + l2top.module.io.cpu_critical_error.fromCore := core.module.io.cpu_critical_error + io.cpu_crtical_error := l2top.module.io.cpu_critical_error.toTile + l2top.module.io.msiAck.fromCore := core.module.io.msiAck + io.msiAck := l2top.module.io.msiAck.toTile + l2top.module.io.teemsiAck zip core.module.io.teemsiAck foreach { case (l2top_teemsiAck, core_teemsiAck) => + l2top_teemsiAck.fromCore := core_teemsiAck + } + io.teemsiAck zip l2top.module.io.teemsiAck foreach { case (io_teemsiAck, l2top_teemsiAck) => + io_teemsiAck := l2top_teemsiAck.toTile + } + + l2top.module.io.hartIsInReset.resetInFrontend := core.module.io.resetInFrontend + io.hartIsInReset := l2top.module.io.hartIsInReset.toTile + l2top.module.io.traceCoreInterface.fromCore <> core.module.io.traceCoreInterface + io.traceCoreInterface <> l2top.module.io.traceCoreInterface.toTile + + l2top.module.io.beu_errors.icache <> core.module.io.beu_errors.icache + l2top.module.io.beu_errors.dcache <> core.module.io.beu_errors.dcache + l2top.module.io.beu_errors.uncache <> core.module.io.beu_errors.uncache + + l2top.module.io.l2_flush_en.foreach(_ := core.module.io.l2_flush_en) + io.l2_flush_en.foreach(_ := core.module.io.l2_flush_en) + core.module.io.l2_flush_done := l2top.module.io.l2_flush_done.getOrElse(false.B) + io.l2_flush_done.foreach(_ := l2top.module.io.l2_flush_done.getOrElse(false.B)) + io.lcrdy.foreach(in => l2top.module.io.lcrdy.foreach(out => out := in)) + + l2top.module.io.dft.zip(io.dft).foreach({ case (a, b) => a := b }) + l2top.module.io.dft_reset.zip(io.dft_reset).foreach({ case (a, b) => a := b }) + core.module.io.dft.zip(io.dft).foreach({ case (a, b) => a := b }) + core.module.io.dft_reset.zip(io.dft_reset).foreach({ case (a, b) => a := b }) + + if (enableL2) { + // TODO: add ECC interface of L2 + l2top.module.io.pfCtrlFromCore := core.module.io.l2PfCtrl + + l2top.module.io.beu_errors.l2 <> 0.U.asTypeOf(l2top.module.io.beu_errors.l2) + core.module.io.l2_hint.bits.sourceId := l2top.module.io.l2_hint.bits.sourceId + core.module.io.l2_hint.bits.isKeyword := l2top.module.io.l2_hint.bits.isKeyword + core.module.io.l2_hint.valid := l2top.module.io.l2_hint.valid + + core.module.io.l2PfqBusy := false.B + core.module.io.debugTopDown.l2MissMatch := l2top.module.io.debugTopDown.l2MissMatch + l2top.module.io.debugTopDown.robHeadPaddr := core.module.io.debugTopDown.robHeadPaddr + l2top.module.io.debugTopDown.robTrueCommit := core.module.io.debugTopDown.robTrueCommit + l2top.module.io.l2_pmp_resp := core.module.io.l2_pmp_resp + core.module.io.l2_tlb_req <> l2top.module.io.l2_tlb_req + core.module.io.topDownInfo.l2Miss := l2top.module.io.l2Miss + + core.module.io.perfEvents <> l2top.module.io.perfEvents + } else { + + l2top.module.io.beu_errors.l2 <> 0.U.asTypeOf(l2top.module.io.beu_errors.l2) + core.module.io.l2_hint.bits.sourceId := l2top.module.io.l2_hint.bits.sourceId + core.module.io.l2_hint.bits.isKeyword := l2top.module.io.l2_hint.bits.isKeyword + core.module.io.l2_hint.valid := l2top.module.io.l2_hint.valid + + core.module.io.l2PfqBusy := false.B + core.module.io.debugTopDown.l2MissMatch := false.B + core.module.io.topDownInfo.l2Miss := false.B + + core.module.io.l2_tlb_req.req.valid := false.B + core.module.io.l2_tlb_req.req.bits := DontCare + core.module.io.l2_tlb_req.req_kill := DontCare + core.module.io.l2_tlb_req.resp.ready := true.B + + core.module.io.perfEvents <> DontCare + } + + io.debugTopDown.robHeadPaddr := core.module.io.debugTopDown.robHeadPaddr + core.module.io.debugTopDown.l3MissMatch := io.debugTopDown.l3MissMatch + l2top.module.io.l3Miss.fromTile := io.l3Miss + core.module.io.topDownInfo.l3Miss := l2top.module.io.l3Miss.toCore + + io.chi.foreach(_ <> l2top.module.io.chi.get) + l2top.module.io.nodeID.foreach(_ := io.nodeID.get) + + if (debugOpts.ResetGen && enableL2) { + core.module.reset := l2top.module.reset_core + } + } + + lazy val module = new XSTileImp(this) +} diff --git a/examples/prefab/cores/xiangshan/arch/native/xiangshan/backend/fu/CSR.scala b/examples/prefab/cores/xiangshan/arch/native/xiangshan/backend/fu/CSR.scala new file mode 100644 index 000000000..a15e9219c --- /dev/null +++ b/examples/prefab/cores/xiangshan/arch/native/xiangshan/backend/fu/CSR.scala @@ -0,0 +1,1685 @@ +// Replaces XiangShan src/main/scala/xiangshan/backend/fu/CSR.scala (same package) with Buckyball changes. + +/** + * ************************************************************************************* + * Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences + * Copyright (c) 2020-2021 Peng Cheng Laboratory + * + * XiangShan is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * + * See the Mulan PSL v2 for more details. + * ************************************************************************************* + */ + +package xiangshan.backend.fu + +import org.chipsalliance.cde.config.Parameters +import chisel3._ +import chisel3.util._ +import difftest._ +import freechips.rocketchip.util._ +import utility.MaskedRegMap.WritableMask +import utils._ +import utility._ +import xiangshan.ExceptionNO._ +import xiangshan._ +import xiangshan.backend.fu.util._ +import xiangshan.cache._ +import xiangshan.backend.Bundles.{ExceptionInfo, TrapInstInfo} +import xiangshan.backend.fu.NewCSR.CSREvents.TargetPCBundle +import xiangshan.backend.fu.NewCSR.CSRNamedConstant.ContextStatus +import xiangshan.backend.rob.RobPtr +import utils.MathUtils.{BigIntGenMask, BigIntNot} +import xiangshan.backend.trace._ +import freechips.rocketchip.rocket.CSRs +import system.HasSoCParameter + +class FpuCsrIO extends Bundle { + val fflags = Output(Valid(UInt(5.W))) + val isIllegal = Output(Bool()) + val dirty_fs = Output(Bool()) + val frm = Input(UInt(3.W)) +} + +class VpuCsrIO(implicit p: Parameters) extends XSBundle { + val vstart = Input(UInt(XLEN.W)) + val vxrm = Input(UInt(2.W)) + + val vl = Output(UInt(XLEN.W)) + + val set_vstart = Output(Valid(UInt(XLEN.W))) + val set_vtype = Output(Valid(UInt(XLEN.W))) + val set_vxsat = Output(Valid(UInt(1.W))) + + val dirty_vs = Output(Bool()) +} + +class PerfCounterIO(implicit p: Parameters) extends XSBundle { + val perfEventsFrontend = Vec(numCSRPCntFrontend, new PerfEvent) + val perfEventsBackend = Vec(numCSRPCntCtrl, new PerfEvent) + val perfEventsLsu = Vec(numCSRPCntLsu, new PerfEvent) + val perfEventsHc = Vec(numPCntHc * coreParams.L2NBanks + 1, new PerfEvent) + val retiredInstr = UInt(7.W) + + val frontendInfo = new Bundle { + val ibufFull = Bool() + + val bpuInfo = new Bundle { + val bpRight = UInt(XLEN.W) + val bpWrong = UInt(XLEN.W) + } + + } + + val ctrlInfo = new Bundle { + val robFull = Bool() + val intdqFull = Bool() + val fpdqFull = Bool() + val lsdqFull = Bool() + } + + val memInfo = new Bundle { + val sqFull = Bool() + val lqFull = Bool() + val dcacheMSHRFull = Bool() + } + +} + +class CSRFileIO(implicit p: Parameters) extends XSBundle with HasSoCParameter { + val commands = new examples.core.xiangshan.native.Commands + val commandFlushStores = Output(Bool()) + val commandStoresEmpty = Input(Bool()) + val timeRequest = Decoupled(Bool()) + val hartId = Input(UInt(hartIdLen.W)) + // output (for func === CSROpType.jmp) + val perf = Input(new PerfCounterIO) + val criticalErrorState = Output(Bool()) + val isPerfCnt = Output(Bool()) + // to FPU + val fpu = Flipped(new FpuCsrIO) + // to VPU + val vpu = Flipped(new VpuCsrIO) + // from rob + val exception = Flipped(ValidIO(new ExceptionInfo)) + val robDeqPtr = Input(new RobPtr) + // to ROB + val isXRet = Output(Bool()) + val trapTarget = Output(new TargetPCBundle) + val interrupt = Output(Bool()) + val wfi_event = Output(Bool()) + //trace + val traceCSR = Output(new TraceCSR) + // from LSQ + val memExceptionVAddr = Input(UInt(XLEN.W)) + val memExceptionGPAddr = Input(UInt(XLEN.W)) + val memExceptionIsForVSnonLeafPTE = Input(Bool()) + // from outside cpu,externalInterrupt + val externalInterrupt = Input(new ExternalInterruptIO) + // TLB + val tlb = Output(new TlbCsrBundle) + // Debug Mode + // val singleStep = Output(Bool()) + val debugMode = Output(Bool()) + // Custom microarchiture ctrl signal + val customCtrl = Output(new CustomCSRCtrlIO) + // instruction fetch address translation type + val instrAddrTransType = Output(new AddrTransType) + // ack for axireg from imsic. which indicates imsic can work actively + val msiAck = Output(Bool()) + val teemsiAck = Option.when(soc.IMSICParams.HasTEEIMSIC)(Output(Bool())) +} + +class VtypeStruct(implicit p: Parameters) extends XSBundle { + val vill = UInt(1.W) + val reserved = UInt((XLEN - 9).W) + val vma = UInt(1.W) + val vta = UInt(1.W) + val vsew = UInt(3.W) + val vlmul = UInt(3.W) +} + +/* +class CSR(cfg: FuConfig)(implicit p: Parameters) extends FuncUnit(cfg) + with HasCSRConst + with PMPMethod + with PMAMethod + with HasXSParameter + with SdtrigExt + with DebugCSR +{ + val csrio = io.csrio.get + + val flushPipe = Wire(Bool()) + + val (valid, src1, src2, func) = ( + io.in.valid, + io.in.bits.data.src(0), + io.in.bits.data.imm, + io.in.bits.ctrl.fuOpType + ) + + // CSR define + val virtMode = RegInit(false.B) + csrio.customCtrl.virtMode := virtMode + + class Priv extends Bundle { + val m = Output(Bool()) + val h = Output(Bool()) // unused + val s = Output(Bool()) + val u = Output(Bool()) + } + + class MstatusStruct extends Bundle { + val sd = Output(UInt(1.W)) + + val pad1 = if (XLEN == 64 && HasHExtension) Output(UInt(23.W)) else if (XLEN == 64) Output(UInt(25.W)) else null + val mpv = if (XLEN == 64 && HasHExtension) Output(UInt(1.W)) else null + val gva = if (XLEN == 64 && HasHExtension) Output(UInt(1.W)) else null + val mbe = if (XLEN == 64) Output(UInt(1.W)) else null + val sbe = if (XLEN == 64) Output(UInt(1.W)) else null + val sxl = if (XLEN == 64) Output(UInt(2.W)) else null + val uxl = if (XLEN == 64) Output(UInt(2.W)) else null + val pad0 = if (XLEN == 64) Output(UInt(9.W)) else Output(UInt(8.W)) + + val tsr = Output(UInt(1.W)) + val tw = Output(UInt(1.W)) + val tvm = Output(UInt(1.W)) + val mxr = Output(UInt(1.W)) + val sum = Output(UInt(1.W)) + val mprv = Output(UInt(1.W)) + val xs = Output(UInt(2.W)) + val fs = Output(UInt(2.W)) + val mpp = Output(UInt(2.W)) + val vs = Output(UInt(2.W)) + val spp = Output(UInt(1.W)) + val pie = new Priv + val ie = new Priv + assert(this.getWidth == XLEN) + + def ube = pie.h // a little ugly + def ube_(r: UInt): Unit = { + pie.h := r(0) + } + } + + class HstatusStruct extends Bundle { + val pad4 = if (HSXLEN == 64) Output(UInt(30.W)) else null + val vsxl = if (HSXLEN == 64) Output(UInt(2.W)) else null + val pad3 = Output(UInt(9.W)) + val vtsr = Output(UInt(1.W)) + val vtw = Output(UInt(1.W)) + val vtvm = Output(UInt(1.W)) + val pad2 = Output(UInt(2.W)) + val vgein = Output(UInt(6.W)) + val pad1 = Output(UInt(2.W)) + val hu = Output(UInt(1.W)) + val spvp = Output(UInt(1.W)) + val spv = Output(UInt(1.W)) + val gva = Output(UInt(1.W)) + val vsbe = Output(UInt(1.W)) + val pad0 = Output(UInt(5.W)) + assert(this.getWidth == XLEN) + } + + class Interrupt extends Bundle { +// val d = Output(Bool()) // Debug + val e = new Priv + val t = new Priv + val s = new Priv + } + + // Debug CSRs + val dcsr = RegInit(UInt(32.W), DcsrStruct.init) + val dpc = Reg(UInt(64.W)) + val dscratch0 = Reg(UInt(64.W)) + val dscratch1 = Reg(UInt(64.W)) + val debugMode = RegInit(false.B) + val debugIntrEnable = RegInit(true.B) // debug interrupt will be handle only when debugIntrEnable + csrio.debugMode := debugMode + + val dpcPrev = RegNext(dpc) + XSDebug(dpcPrev =/= dpc, "Debug Mode: dpc is altered! Current is %x, previous is %x\n", dpc, dpcPrev) + + val dcsrData = Wire(new DcsrStruct) + dcsrData := dcsr.asTypeOf(new DcsrStruct) + val dcsrMask = ZeroExt(GenMask(15) | GenMask(13, 11) | GenMask(4) | GenMask(2, 0), XLEN)// Dcsr write mask + def dcsrUpdateSideEffect(dcsr: UInt): UInt = { + val dcsrOld = WireInit(dcsr.asTypeOf(new DcsrStruct)) + val dcsrNew = dcsr | (dcsrOld.prv(0) | dcsrOld.prv(1)).asUInt // turn 10 priv into 11 + dcsrNew + } + // csrio.singleStep := dcsrData.step + csrio.customCtrl.singlestep := dcsrData.step && !debugMode + + // Trigger CSRs + private val tselectPhy = RegInit(0.U(log2Up(TriggerNum).W)) + + private val tdata1RegVec = RegInit(VecInit(Seq.fill(TriggerNum)(Tdata1Bundle.default))) + private val tdata2RegVec = RegInit(VecInit(Seq.fill(TriggerNum)(0.U(64.W)))) + private val tdata1WireVec = tdata1RegVec.map(_.asTypeOf(new Tdata1Bundle)) + private val tdata2WireVec = tdata2RegVec + private val tdata1Selected = tdata1RegVec(tselectPhy).asTypeOf(new Tdata1Bundle) + private val tdata2Selected = tdata2RegVec(tselectPhy) + private val newTriggerChainVec = UIntToOH(tselectPhy, TriggerNum).asBools | tdata1WireVec.map(_.data.asTypeOf(new MControlData).chain) + private val newTriggerChainIsLegal = TriggerCheckChainLegal(newTriggerChainVec, TriggerChainMaxLength) + val tinfo = RegInit((BigInt(1) << TrigTypeEnum.MCONTROL.litValue.toInt).U(XLEN.W)) // This value should be 4.U + + + def WriteTselect(wdata: UInt) = { + Mux(wdata < TriggerNum.U, wdata(log2Up(TriggerNum) - 1, 0), tselectPhy) + } + + def GenTdataDistribute(tdata1: Tdata1Bundle, tdata2: UInt): MatchTriggerIO = { + val res = Wire(new MatchTriggerIO) + val mcontrol: MControlData = WireInit(tdata1.data.asTypeOf(new MControlData)) + res.matchType := mcontrol.match_.asUInt + res.select := mcontrol.select + res.timing := mcontrol.timing + res.action := mcontrol.action.asUInt + res.chain := mcontrol.chain + res.execute := mcontrol.execute + res.load := mcontrol.load + res.store := mcontrol.store + res.tdata2 := tdata2 + res + } + + csrio.customCtrl.frontend_trigger.tUpdate.bits.addr := tselectPhy + csrio.customCtrl.mem_trigger.tUpdate.bits.addr := tselectPhy + csrio.customCtrl.frontend_trigger.tUpdate.bits.tdata := GenTdataDistribute(tdata1Selected, tdata2Selected) + csrio.customCtrl.mem_trigger.tUpdate.bits.tdata := GenTdataDistribute(tdata1Selected, tdata2Selected) + + // Machine-Level CSRs + // mtvec: {BASE (WARL), MODE (WARL)} where mode is 0 or 1 + val mtvecMask = ~(0x2.U(XLEN.W)) + val mtvec = RegInit(UInt(XLEN.W), 0.U) + val mcounteren = RegInit(UInt(XLEN.W), 0.U) + // Currently, XiangShan don't support Unprivileged Counter/Timers CSRs ("Zicntr" and "Zihpm") + val mcounterenMask = 0.U(XLEN.W) + val mcause = RegInit(UInt(XLEN.W), 0.U) + val mtval = RegInit(UInt(XLEN.W), 0.U) + val mtval2 = RegInit(UInt(XLEN.W), 0.U) + val mtinst = RegInit(UInt(XLEN.W), 0.U) + val mepc = RegInit(UInt(XLEN.W), 0.U) + // Page 36 in riscv-priv: The low bit of mepc (mepc[0]) is always zero. + val mepcMask = ~(0x1.U(XLEN.W)) + + val mie = RegInit(0.U(XLEN.W)) + val mipWire = WireInit(0.U.asTypeOf(new Interrupt)) + val mipReg = RegInit(0.U(XLEN.W)) + val mipMask = ZeroExt(Array( + 1, // SSIP + 2, // VSSIP + 3, // MSIP + 5, // STIP + 6, // VSTIP + 7, // MTIP + 9, // SEIP + 10, // VSEIP + 11, // MEIP + 12, // SGEIP + ).map(GenMask(_)).reduce(_ | _), XLEN) + val mip = (mipWire.asUInt | mipReg).asTypeOf(new Interrupt) + + val mip_mie_WMask_H = if(HasHExtension){((1 << 2) | (1 << 6) | (1 << 10) | (1 << 12)).U(XLEN.W)}else{0.U(XLEN.W)} + val vssip_Mask = (1 << 2).U(XLEN.W) + + val mipWMask = vssip_Mask | ((1 << 9) | (1 << 5) | (1 << 1)).U(XLEN.W) + val mieWMask = mip_mie_WMask_H | "haaa".U(XLEN.W) + + def getMisaMxl(mxl: BigInt): BigInt = mxl << (XLEN - 2) + def getMisaExt(ext: Char): Long = 1 << (ext.toInt - 'a'.toInt) + var extList = List('a', 's', 'i', 'u') + if (HasMExtension) { extList = extList :+ 'm' } + if (HasCExtension) { extList = extList :+ 'c' } + if (HasHExtension) { extList = extList :+ 'h' } + if (HasFPU) { extList = extList ++ List('f', 'd') } + if (HasVPU) { extList = extList :+ 'v' } + val misaInitVal = getMisaMxl(2) | extList.foldLeft(0L)((sum, i) => sum | getMisaExt(i)) //"h8000000000141185".U + val misa = RegInit(UInt(XLEN.W), misaInitVal.U) + println(s"[CSR] supported isa ext: $extList") + + // MXL = 2 | 0 | EXT = b 00 0000 0100 0001 0001 0000 0101 + // (XLEN-1, XLEN-2) | |(25, 0) ZY XWVU TSRQ PONM LKJI HGFE DCBA + + // Machine Configuration + val menvcfg = RegInit(UInt(XLEN.W), 0.U) + + val mvendorid = RegInit(UInt(XLEN.W), 0.U) // this is a non-commercial implementation + val marchid = RegInit(UInt(XLEN.W), 25.U) // architecture id for XiangShan is 25; see https://github.com/riscv/riscv-isa-manual/blob/master/marchid.md + val mimpid = RegInit(UInt(XLEN.W), 0.U) // provides a unique encoding of the version of the processor implementation + val mhartid = Reg(UInt(XLEN.W)) // the hardware thread running the code + when (RegNext(RegNext(reset.asBool) && !reset.asBool)) { + mhartid := csrio.hartId + } + val mconfigptr = RegInit(UInt(XLEN.W), 0.U) // the read-only pointer pointing to the platform config structure, 0 for not supported. + val mstatus = RegInit("ha00002200".U(XLEN.W)) + + // mstatus Value Table + // | sd | Read Only + // | pad1 | WPRI + // | sxl | hardlinked to 10, use 00 to pass xv6 test + // | uxl | hardlinked to 10 + // | pad0 | + // | tsr | + // | tw | + // | tvm | + // | mxr | + // | sum | + // | mprv | + // | xs | 00 | + // | fs | 01 | + // | mpp | 00 | + // | vs | 01 | + // | spp | 0 | + // | pie | 0000 | pie.h is used as UBE + // | ie | 0000 | uie hardlinked to 0, as N ext is not implemented + + val mstatusStruct = mstatus.asTypeOf(new MstatusStruct) + def mstatusUpdateSideEffect(mstatus: UInt): UInt = { + val mstatusOld = WireInit(mstatus.asTypeOf(new MstatusStruct)) + // Cat(sd, other) + val mstatusNew = Cat( + mstatusOld.xs === ContextStatus.dirty || mstatusOld.fs === ContextStatus.dirty || mstatusOld.vs === ContextStatus.dirty, + mstatus(XLEN-2, 0) + ) + mstatusNew + } + def vsstatusUpdateSideEffect(vsstatus: UInt): UInt = { + val vsstatusOld = WireInit(vsstatus.asTypeOf(new MstatusStruct)) + val vsstatusNew = Cat(vsstatusOld.xs === "b11".U || vsstatusOld.fs === "b11".U, vsstatus(XLEN-2, 0)) + vsstatusNew + } + val mstatusWMask = (~ZeroExt(( + GenMask(63) | // SD is read-only + (if(HasHExtension) + GenMask(62, 40) // WPRI + else + GenMask(62, 38) )| // WPRI + GenMask(35, 32) | // SXL and UXL cannot be changed + GenMask(31, 23) | // WPRI + GenMask(16, 15) | // XS is read-only + GenMask(6) | // UBE, always little-endian (0) + GenMask(4) | // WPRI + GenMask(2) | // WPRI + GenMask(0) // WPRI + ), 64)).asUInt + + val medeleg = RegInit(UInt(XLEN.W), 0.U) + val midelegInit = if(HasHExtension){((1 << 12) | (1 << 10) | (1 << 6) | (1 << 2)).U}else{0.U} + val medelegWMask = if(HasHExtension) { + "hf0b7ff".U(XLEN.W) + }else { + "hb3ff".U(XLEN.W) + } + + + val mideleg = RegInit(UInt(XLEN.W), midelegInit) + val mscratch = RegInit(UInt(XLEN.W), 0.U) + + val midelegWMask = "h222".U(XLEN.W) + // PMP Mapping + val pmp = Wire(Vec(NumPMP, new PMPEntry())) // just used for method parameter + val pma = Wire(Vec(NumPMA, new PMPEntry())) // just used for method parameter + val pmpMapping = pmp_gen_mapping(pmp_init, NumPMP, PmpcfgBase, PmpaddrBase, pmp) + val pmaMapping = pmp_gen_mapping(pma_init, NumPMA, PmacfgBase, PmaaddrBase, pma) + // !WARNNING: pmp and pma CSRs are not checked in difftest. + + // Supervisor-Level CSRs + + val sstatusWNmask: BigInt = ( + BigIntGenMask(63) | // SD is read-only + BigIntGenMask(62, 34) | // WPRI + BigIntGenMask(33, 32) | // UXL is hard-wired to 64(b10) + BigIntGenMask(31, 20) | // WPRI + BigIntGenMask(17) | // WPRI + BigIntGenMask(16, 15) | // XS is read-only to zero + BigIntGenMask(12, 11) | // WPRI + BigIntGenMask(7) | // WPRI + BigIntGenMask(6) | // UBE is always little-endian (0) + BigIntGenMask(4, 2) | // WPRI + BigIntGenMask(0) // WPRI + ) + + val sstatusWmask = BigIntNot(sstatusWNmask).U(XLEN.W) + val sstatusRmask = ( + BigIntGenMask(63) | // SD + BigIntGenMask(33, 32) | // UXL + BigIntGenMask(19) | // MXR + BigIntGenMask(18) | // SUM + BigIntGenMask(16, 15) | // XS + BigIntGenMask(14, 13) | // FS + BigIntGenMask(10, 9 ) | // VS + BigIntGenMask(8) | // SPP + BigIntGenMask(6) | // UBE: hard wired to 0 + BigIntGenMask(5) | // SPIE + BigIntGenMask(1) + ).U(XLEN.W) + + println(s"sstatusWNmask: 0x${sstatusWNmask.toString(16)}") + println(s"sstatusWmask: 0x${sstatusWmask.litValue.toString(16)}") + println(s"sstatusRmask: 0x${sstatusRmask.litValue.toString(16)}") + + // stvec: {BASE (WARL), MODE (WARL)} where mode is 0 or 1 + val stvecMask = ~(0x2.U(XLEN.W)) + val stvec = RegInit(UInt(XLEN.W), 0.U) + // val sie = RegInit(0.U(XLEN.W)) + val sieMask = "h222".U & mideleg + val sipMask = "h222".U & mideleg + val sipWMask = "h2".U(XLEN.W) // ssip is writeable in smode + val satp = if(EnbaleTlbDebug) RegInit(UInt(XLEN.W), "h8000000000087fbe".U) else RegInit(0.U(XLEN.W)) + // val satp = RegInit(UInt(XLEN.W), "h8000000000087fbe".U) // only use for tlb naive debug + // val satpMask = "h80000fffffffffff".U(XLEN.W) // disable asid, mode can only be 8 / 0 + // TODO: use config to control the length of asid + // val satpMask = "h8fffffffffffffff".U(XLEN.W) // enable asid, mode can only be 8 / 0 + val satpMask = Cat("h8".U(Satp_Mode_len.W), satp_part_wmask(Satp_Asid_len, AsidLength), satp_part_wmask(Satp_Addr_len, PAddrBits-12)) + val sepc = RegInit(UInt(XLEN.W), 0.U) + // Page 60 in riscv-priv: The low bit of sepc (sepc[0]) is always zero. + val sepcMask = ~(0x1.U(XLEN.W)) + val scause = RegInit(UInt(XLEN.W), 0.U) + val stval = RegInit(UInt(XLEN.W), 0.U) + val sscratch = RegInit(UInt(XLEN.W), 0.U) + val scounteren = RegInit(UInt(XLEN.W), 0.U) + val senvcfg = RegInit(UInt(XLEN.W), 0.U) // !WARNING: there is no logic about this CSR. + // Currently, XiangShan don't support Unprivileged Counter/Timers CSRs ("Zicntr" and "Zihpm") + val scounterenMask = 0.U(XLEN.W) + + // sbpctl + // Bits 0-7: {LOOP, RAS, SC, TAGE, BIM, BTB, uBTB} + val sbpctl = RegInit(UInt(XLEN.W), "h7f".U) + csrio.customCtrl.bp_ctrl.ubtb_enable := sbpctl(0) + csrio.customCtrl.bp_ctrl.btb_enable := sbpctl(1) + csrio.customCtrl.bp_ctrl.bim_enable := sbpctl(2) + csrio.customCtrl.bp_ctrl.tage_enable := sbpctl(3) + csrio.customCtrl.bp_ctrl.sc_enable := sbpctl(4) + csrio.customCtrl.bp_ctrl.ras_enable := sbpctl(5) + csrio.customCtrl.bp_ctrl.loop_enable := sbpctl(6) + + // spfctl Bit 0: L1I Cache Prefetcher Enable + // spfctl Bit 1: L2Cache Prefetcher Enable + // spfctl Bit 2: L1D Cache Prefetcher Enable + // spfctl Bit 3: L1D train prefetch on hit + // spfctl Bit 4: L1D prefetch enable agt + // spfctl Bit 5: L1D prefetch enable pht + // spfctl Bit [9:6]: L1D prefetch active page threshold + // spfctl Bit [15:10]: L1D prefetch active page stride + // turn off L2 BOP, turn on L1 SMS by default + val spfctl = RegInit(UInt(XLEN.W), Seq( + 0 << 17, // L2 pf store only [17] init: false + 1 << 16, // L1D pf enable stride [16] init: true + 30 << 10, // L1D active page stride [15:10] init: 30 + 12 << 6, // L1D active page threshold [9:6] init: 12 + 1 << 5, // L1D enable pht [5] init: true + 1 << 4, // L1D enable agt [4] init: true + 0 << 3, // L1D train on hit [3] init: false + 1 << 2, // L1D pf enable [2] init: true + 1 << 1, // L2 pf enable [1] init: true + 1 << 0, // L1I pf enable [0] init: true + ).reduce(_|_).U(XLEN.W)) + csrio.customCtrl.l1I_pf_enable := spfctl(0) + csrio.customCtrl.l2_pf_enable := spfctl(1) + csrio.customCtrl.l1D_pf_enable := spfctl(2) + csrio.customCtrl.l1D_pf_train_on_hit := spfctl(3) + csrio.customCtrl.l1D_pf_enable_agt := spfctl(4) + csrio.customCtrl.l1D_pf_enable_pht := spfctl(5) + csrio.customCtrl.l1D_pf_active_threshold := spfctl(9, 6) + csrio.customCtrl.l1D_pf_active_stride := spfctl(15, 10) + csrio.customCtrl.l1D_pf_enable_stride := spfctl(16) + csrio.customCtrl.l2_pf_store_only := spfctl(17) + + // slvpredctl: load violation predict settings + // Default reset period: 2^16 + // Why this number: reset more frequently while keeping the overhead low + // Overhead: extra two redirections in every 64K cycles => ~0.1% overhead + val slvpredctl = Reg(UInt(XLEN.W)) + when(reset.asBool) { + slvpredctl := Constantin.createRecord("slvpredctl", 0x60) + } + csrio.customCtrl.lvpred_disable := slvpredctl(0) + csrio.customCtrl.no_spec_load := slvpredctl(1) + csrio.customCtrl.storeset_wait_store := slvpredctl(2) + csrio.customCtrl.storeset_no_fast_wakeup := slvpredctl(3) + csrio.customCtrl.lvpred_timeout := slvpredctl(8, 4) + + // smblockctl: memory block configurations + // +------------------------+---+---+---+----+----+-----+--------+ + // |XLEN-1 10| 9 | 8 | 7 | 6 | 5 | 4 |3 0| + // +------------------------+---+---+---+----+----+-----+--------+ + // | Reserved | L | S | O | CE | SP | LVC | Th | + // +------------------------+---+---+---+----+----+-----+--------+ + // Description: + // Bit 3-0 : Store buffer flush threshold (Th). + // Bit 4 : Enable load violation check after reset (LVC). + // Bit 5 : Enable soft-prefetch after reset (SP). + // Bit 6 : Enable cache error after reset (CE). + // Bit 7 : Enable uncache write outstanding (O). + // Bit 8 : Enable unaligned store (S). + // Bit 9 : Enable unaligned load (L). + // Others : Reserved. + + val smblockctl_init_val = + (0xf & StoreBufferThreshold) | + (EnableLdVioCheckAfterReset.toInt << 4) | + (EnableSoftPrefetchAfterReset.toInt << 5) | + (EnableCacheErrorAfterReset.toInt << 6) | + (EnableUncacheWriteOutstanding.toInt << 7) | + (EnableHardwareStoreMisalign.toInt << 8) | + (EnableHardwareLoadMisalign.toInt << 9) + val smblockctl = RegInit(UInt(XLEN.W), smblockctl_init_val.U) + csrio.customCtrl.sbuffer_threshold := smblockctl(3, 0) + // bits 4: enable load load violation check + csrio.customCtrl.ldld_vio_check_enable := smblockctl(4) + csrio.customCtrl.soft_prefetch_enable := smblockctl(5) + csrio.customCtrl.cache_error_enable := smblockctl(6) + csrio.customCtrl.uncache_write_outstanding_enable := smblockctl(7) + csrio.customCtrl.hd_misalign_st_enable := smblockctl(8) + csrio.customCtrl.hd_misalign_ld_enable := smblockctl(9) + + println("CSR smblockctl init value:") + println(" Store buffer replace threshold: " + StoreBufferThreshold) + println(" Enable ld-ld vio check after reset: " + EnableLdVioCheckAfterReset) + println(" Enable soft prefetch after reset: " + EnableSoftPrefetchAfterReset) + println(" Enable cache error after reset: " + EnableCacheErrorAfterReset) + println(" Enable uncache write outstanding: " + EnableUncacheWriteOutstanding) + println(" Enable unaligned store: " + EnableHardwareStoreMisalign) + println(" Enable unaligned load: " + EnableHardwareLoadMisalign) + + val srnctl = RegInit(UInt(XLEN.W), "h7".U) + csrio.customCtrl.fusion_enable := srnctl(0) + csrio.customCtrl.wfi_enable := srnctl(2) + + // Hypervisor CSRs + val hstatusWMask = "h7003c0".U(XLEN.W) + // hstatus: vtsr, vtw, vtvm, hu, spvp, spv, gva, + val hstatus = RegInit("h200000000".U(XLEN.W)) + val hstatusStruct = hstatus.asTypeOf(new HstatusStruct) + val hedeleg = RegInit(UInt(XLEN.W), 0.U) + val hideleg = RegInit(UInt(XLEN.W), 0.U) + val hidelegRMask = mideleg + val hidelegWMask = ((1 << 10) | (1 << 6) | (1 << 2)).U(XLEN.W) + val hgeie = RegInit(UInt(XLEN.W), 0.U) + val htval = RegInit(UInt(XLEN.W), 0.U) + // hvip hip hie is part of mip or mie + val hvipMask = ((1 << 10) | (1 << 6) | (1 << 2)).U(XLEN.W) + val hipRMask = (((1 << 12).U | hvipMask) & mideleg) + val hipWMask = ((1 << 2).U & mideleg)// vssip + val hieMask = hipRMask + val htinst = RegInit(UInt(XLEN.W), 0.U) + val hgeip = RegInit(UInt(XLEN.W), 0.U) + val henvcfg = RegInit(UInt(XLEN.W), 0.U) + val hgatp = RegInit(UInt(XLEN.W), 0.U) + val hgatpMask = Cat("h8".U(Hgatp_Mode_len.W), satp_part_wmask(Hgatp_Vmid_len, VmidLength), satp_part_wmask(Hgatp_Addr_len, PAddrBits-12)) + // val htimedelta = RegInit(UInt(XLEN.W), 0.U) + val hcounteren = RegInit(UInt(XLEN.W), 0.U) + // Currently, XiangShan don't support Unprivileged Counter/Timers CSRs ("Zicntr" and "Zihpm") + val hcounterenMask = 0.U(XLEN.W) + + val vsstatus = RegInit("h200002000".U(XLEN.W)) + val vsstatusStruct = vsstatus.asTypeOf(new MstatusStruct) + //vsie vsip + val vsMask = ((1 << 10) | (1 << 6) | (1 << 2)).U(XLEN.W) + val vsip_ie_Mask = ZeroExt((hideleg & mideleg & vsMask), XLEN) + val vsip_WMask = ZeroExt((hideleg & mideleg & vssip_Mask), XLEN) + val vstvec = RegInit(UInt(XLEN.W), 0.U) + val vsscratch = RegInit(UInt(XLEN.W), 0.U) + val vsepc = RegInit(UInt(XLEN.W), 0.U) + val vscause = RegInit(UInt(XLEN.W), 0.U) + val vstval = RegInit(UInt(XLEN.W), 0.U) + val vsatp = RegInit(UInt(XLEN.W), 0.U) + val tlbBundle = Wire(new TlbCsrBundle) + tlbBundle.satp.apply(satp) + tlbBundle.vsatp.apply(vsatp) + tlbBundle.hgatp.apply(hgatp) + csrio.tlb := tlbBundle + + // User-Level CSRs + val uepc = Reg(UInt(XLEN.W)) + + // fcsr + class FcsrStruct extends Bundle { + val reserved = UInt((XLEN-3-5).W) + val frm = UInt(3.W) + val fflags = UInt(5.W) + assert(this.getWidth == XLEN) + } + val fcsr = RegInit(0.U(XLEN.W)) + // set mstatus->sd and mstatus->fs when true + val csrw_dirty_fp_state = WireInit(false.B) + + def frm_wfn(wdata: UInt): UInt = { + val fcsrOld = WireInit(fcsr.asTypeOf(new FcsrStruct)) + csrw_dirty_fp_state := true.B + fcsrOld.frm := wdata(2,0) + fcsrOld.asUInt + } + def frm_rfn(rdata: UInt): UInt = rdata(7,5) + + def fflags_wfn(update: Boolean)(wdata: UInt): UInt = { + val fcsrOld = fcsr.asTypeOf(new FcsrStruct) + val fcsrNew = WireInit(fcsrOld) + if (update) { + fcsrNew.fflags := wdata(4,0) | fcsrOld.fflags + } else { + fcsrNew.fflags := wdata(4,0) + } + fcsrNew.asUInt + } + def fflags_rfn(rdata:UInt): UInt = rdata(4,0) + + def fcsr_wfn(wdata: UInt): UInt = { + val fcsrOld = WireInit(fcsr.asTypeOf(new FcsrStruct)) + csrw_dirty_fp_state := true.B + Cat(fcsrOld.reserved, wdata.asTypeOf(fcsrOld).frm, wdata.asTypeOf(fcsrOld).fflags) + } + + val fcsrMapping = Map( + MaskedRegMap(Fflags, fcsr, wfn = fflags_wfn(update = false), rfn = fflags_rfn), + MaskedRegMap(Frm, fcsr, wfn = frm_wfn, rfn = frm_rfn), + MaskedRegMap(Fcsr, fcsr, wfn = fcsr_wfn) + ) + + // Vector extension CSRs + val vstart = RegInit(0.U(XLEN.W)) + val vcsr = RegInit(0.U(XLEN.W)) + val vl = Reg(UInt(XLEN.W)) + val vtype = Reg(UInt(XLEN.W)) + val vlenb = RegInit(VDataBytes.U(XLEN.W)) + + // set mstatus->sd and mstatus->vs when true + val csrw_dirty_vs_state = WireInit(false.B) + + // vcsr is mapped to vxrm and vxsat + class VcsrStruct extends Bundle { + val reserved = UInt((XLEN-3).W) + val vxrm = UInt(2.W) + val vxsat = UInt(1.W) + assert(this.getWidth == XLEN) + } + + def vxrm_wfn(wdata: UInt): UInt = { + val vcsrOld = WireInit(vcsr.asTypeOf(new VcsrStruct)) + csrw_dirty_vs_state := true.B + vcsrOld.vxrm := wdata(1,0) + vcsrOld.asUInt + } + def vxrm_rfn(rdata: UInt): UInt = rdata(2,1) + + def vxsat_wfn(update: Boolean)(wdata: UInt): UInt = { + val vcsrOld = WireInit(vcsr.asTypeOf(new VcsrStruct)) + val vcsrNew = WireInit(vcsrOld) + csrw_dirty_vs_state := true.B + if (update) { + vcsrNew.vxsat := wdata(0) | vcsrOld.vxsat + } else { + vcsrNew.vxsat := wdata(0) + } + vcsrNew.asUInt + } + def vxsat_rfn(rdata: UInt): UInt = rdata(0) + + def vcsr_wfn(wdata: UInt): UInt = { + val vcsrOld = WireInit(vcsr.asTypeOf(new VcsrStruct)) + csrw_dirty_vs_state := true.B + vcsrOld.vxrm := wdata.asTypeOf(vcsrOld).vxrm + vcsrOld.vxsat := wdata.asTypeOf(vcsrOld).vxsat + vcsrOld.asUInt + } + + val vcsrMapping = Map( + MaskedRegMap(Vstart, vstart), + MaskedRegMap(Vxrm, vcsr, wfn = vxrm_wfn, rfn = vxrm_rfn), + MaskedRegMap(Vxsat, vcsr, wfn = vxsat_wfn(false), rfn = vxsat_rfn), + MaskedRegMap(Vcsr, vcsr, wfn = vcsr_wfn), + MaskedRegMap(Vl, vl), + MaskedRegMap(Vtype, vtype), + MaskedRegMap(Vlenb, vlenb), + ) + + // Hart Privilege Mode + val privilegeMode = RegInit(UInt(2.W), ModeM) + + //val perfEventscounten = List.fill(nrPerfCnts)(RegInit(false(Bool()))) + // Perf Counter + val nrPerfCnts = 29 // 3...31 + val privilegeModeOH = UIntToOH(privilegeMode) + val perfEventscounten = RegInit(0.U.asTypeOf(Vec(nrPerfCnts, Bool()))) + val perfCnts = List.fill(nrPerfCnts)(RegInit(0.U(XLEN.W))) + val perfEvents = List.fill(8)(RegInit("h0000000000".U(XLEN.W))) ++ + List.fill(8)(RegInit("h4010040100".U(XLEN.W))) ++ + List.fill(8)(RegInit("h8020080200".U(XLEN.W))) ++ + List.fill(5)(RegInit("hc0300c0300".U(XLEN.W))) + for (i <-0 until nrPerfCnts) { + perfEventscounten(i) := (perfEvents(i)(63,60) & privilegeModeOH).orR + } + + val hpmEvents = Wire(Vec(numPCntHc * coreParams.L2NBanks, new PerfEvent)) + for (i <- 0 until numPCntHc * coreParams.L2NBanks) { + hpmEvents(i) := csrio.perf.perfEventsHc(i) + } + + // print perfEvents + val allPerfEvents = hpmEvents.map(x => (s"Hc", x.value)) + if (printEventCoding) { + for (((name, inc), i) <- allPerfEvents.zipWithIndex) { + println("CSR perfEvents Set", name, inc, i) + } + } + + val csrevents = perfEvents.slice(24, 29) + val hpm_hc = HPerfMonitor(csrevents, hpmEvents) + val mcountinhibit = RegInit(0.U(XLEN.W)) + val mcycle = RegInit(0.U(XLEN.W)) + mcycle := mcycle + 1.U + val minstret = RegInit(0.U(XLEN.W)) + val perf_events = csrio.perf.perfEventsFrontend ++ + csrio.perf.perfEventsBackend ++ + csrio.perf.perfEventsLsu ++ + hpm_hc.getPerf + minstret := minstret + RegNext(csrio.perf.retiredInstr) + for(i <- 0 until 29){ + perfCnts(i) := Mux(mcountinhibit(i+3) | !perfEventscounten(i), perfCnts(i), perfCnts(i) + perf_events(i).value) + } + + // CSR reg map + val basicPrivMapping = Map( + + // Unprivileged Floating-Point CSRs + // Has been mapped above + + // TODO: support Unprivileged Counter/Timers CSRs ("Zicntr" and "Zihpm") + // Unprivileged Counter/Timers + MaskedRegMap(Cycle, mcycle), + // We don't support read time CSR. + // MaskedRegMap(Time, mtime), + MaskedRegMap(Instret, minstret), + + //--- Supervisor Trap Setup --- + MaskedRegMap(Sstatus, mstatus, sstatusWmask, mstatusUpdateSideEffect, sstatusRmask), + // MaskedRegMap(Sedeleg, Sedeleg), + // MaskedRegMap(Sideleg, Sideleg), + MaskedRegMap(Sie, mie, sieMask, MaskedRegMap.NoSideEffect, sieMask), + MaskedRegMap(Stvec, stvec, stvecMask, MaskedRegMap.NoSideEffect, stvecMask), + MaskedRegMap(Scounteren, scounteren, scounterenMask), + + //--- Supervisor Configuration --- + MaskedRegMap(Senvcfg, senvcfg), + + //--- Supervisor Trap Handling --- + MaskedRegMap(Sscratch, sscratch), + MaskedRegMap(Sepc, sepc, sepcMask, MaskedRegMap.NoSideEffect, sepcMask), + MaskedRegMap(Scause, scause), + MaskedRegMap(Stval, stval), + MaskedRegMap(Sip, mipReg.asUInt, sipWMask, MaskedRegMap.NoSideEffect, sipMask, x => (mipWire.asUInt | x) & sipMask), + + //--- Supervisor Protection and Translation --- + MaskedRegMap(Satp, satp, satpMask, MaskedRegMap.NoSideEffect, satpMask), + + //--- Supervisor Custom Read/Write Registers + MaskedRegMap(Sbpctl, sbpctl), + MaskedRegMap(Spfctl, spfctl), + MaskedRegMap(Slvpredctl, slvpredctl), + MaskedRegMap(Smblockctl, smblockctl), + MaskedRegMap(Srnctl, srnctl), + + //--- Machine Information Registers --- + MaskedRegMap(Mvendorid, mvendorid, 0.U(XLEN.W), MaskedRegMap.Unwritable), + MaskedRegMap(Marchid, marchid, 0.U(XLEN.W), MaskedRegMap.Unwritable), + MaskedRegMap(Mimpid, mimpid, 0.U(XLEN.W), MaskedRegMap.Unwritable), + MaskedRegMap(Mhartid, mhartid, 0.U(XLEN.W), MaskedRegMap.Unwritable), + MaskedRegMap(Mconfigptr, mconfigptr, 0.U(XLEN.W), MaskedRegMap.Unwritable), + + //--- Machine Configuration Registers --- + MaskedRegMap(Menvcfg, menvcfg), + + //--- Machine Trap Setup --- + MaskedRegMap(Mstatus, mstatus, mstatusWMask, mstatusUpdateSideEffect), + MaskedRegMap(Misa, misa, 0.U, MaskedRegMap.Unwritable), // now whole misa is unchangeable + MaskedRegMap(Medeleg, medeleg, medelegWMask), + MaskedRegMap(Mideleg, mideleg, midelegWMask), + MaskedRegMap(Mie, mie, mieWMask), + MaskedRegMap(Mtvec, mtvec, mtvecMask, MaskedRegMap.NoSideEffect, mtvecMask), + MaskedRegMap(Mcounteren, mcounteren, mcounterenMask), + + //--- Machine Trap Handling --- + MaskedRegMap(Mscratch, mscratch), + MaskedRegMap(Mepc, mepc, mepcMask, MaskedRegMap.NoSideEffect, mepcMask), + MaskedRegMap(Mcause, mcause), + MaskedRegMap(Mtval, mtval), + MaskedRegMap(Mip, mipReg.asUInt, mipWMask, MaskedRegMap.NoSideEffect, mipMask, x => (mipWire.asUInt | x) & mipMask), + + //--- Trigger --- + MaskedRegMap(Tselect, tselectPhy, WritableMask, WriteTselect), + // Todo: support chain length = 2 + MaskedRegMap(Tdata1, tdata1RegVec(tselectPhy), + WritableMask, + x => Tdata1Bundle.Write(x, tdata1RegVec(tselectPhy), newTriggerChainIsLegal, debug_mode = debugMode), + WritableMask, + x => Tdata1Bundle.Read(x)), + MaskedRegMap(Tdata2, tdata2RegVec(tselectPhy)), + MaskedRegMap(Tinfo, tinfo, 0.U(XLEN.W), MaskedRegMap.Unwritable), + + //--- Debug Mode --- + MaskedRegMap(Dcsr, dcsr, dcsrMask, dcsrUpdateSideEffect), + MaskedRegMap(Dpc, dpc), + MaskedRegMap(Dscratch0, dscratch0), + MaskedRegMap(Dscratch1, dscratch1), + MaskedRegMap(Mcountinhibit, mcountinhibit), + MaskedRegMap(Mcycle, mcycle), + MaskedRegMap(Minstret, minstret), + ) + + // hypervisor csr map + val hcsrMapping = Map( + //--- Hypervisor Trap Setup --- + MaskedRegMap(Hstatus, hstatus, hstatusWMask), + MaskedRegMap(Hedeleg, hedeleg), + MaskedRegMap(Hideleg, hideleg, hidelegWMask, MaskedRegMap.NoSideEffect, hidelegRMask), + MaskedRegMap(Hie, mie, hieMask, MaskedRegMap.NoSideEffect, hieMask), + MaskedRegMap(Hcounteren, hcounteren, hcounterenMask), + MaskedRegMap(Hgeie, hgeie), + + //--- Hypervisor Trap Handling --- + MaskedRegMap(Htval, htval), + MaskedRegMap(Hip, mipReg.asUInt, hipWMask, MaskedRegMap.NoSideEffect, hipRMask, x => (mipWire.asUInt | x) & hipRMask), + MaskedRegMap(Hvip, mipReg.asUInt, hvipMask, MaskedRegMap.NoSideEffect, hvipMask, x => (mipWire.asUInt | x) & hvipMask), + MaskedRegMap(Htinst, htinst), + MaskedRegMap(Hgeip, hgeip), + + //--- Hypervisor Configuration --- + MaskedRegMap(Henvcfg, henvcfg), + + //--- Hypervisor Protection and Translation --- + MaskedRegMap(Hgatp, hgatp, hgatpMask, MaskedRegMap.NoSideEffect, hgatpMask), + + //--- Hypervisor Counter/Timer Virtualization Registers --- + // MaskedRegMap(Htimedelta, htimedelta), + + //--- Virtual Supervisor Registers --- + MaskedRegMap(Vsstatus, vsstatus, rmask = sstatusRmask, wmask = sstatusWmask, wfn = vsstatusUpdateSideEffect), + MaskedRegMap(Vsie, mie, rmask = vsip_ie_Mask, wmask = vsip_ie_Mask), + MaskedRegMap(Vstvec, vstvec), + MaskedRegMap(Vsscratch, vsscratch), + MaskedRegMap(Vsepc, vsepc), + MaskedRegMap(Vscause, vscause), + MaskedRegMap(Vstval, vstval), + MaskedRegMap(Vsip, mipReg.asUInt, vsip_WMask, MaskedRegMap.NoSideEffect, vsip_ie_Mask, x => mipWire.asUInt | x), + MaskedRegMap(Vsatp, vsatp, satpMask, MaskedRegMap.NoSideEffect, satpMask), + + //--- Machine Registers --- + MaskedRegMap(Mtval2, mtval2), + MaskedRegMap(Mtinst, mtinst), + ) + + val perfCntMapping = (0 until 29).map(i => {Map( + MaskedRegMap(addr = Mhpmevent3 +i, + reg = perfEvents(i), + wmask = "hf87fff3fcff3fcff".U(XLEN.W)), + MaskedRegMap(addr = Mhpmcounter3 +i, + reg = perfCnts(i)), + MaskedRegMap(addr = Hpmcounter3 + i, + reg = perfCnts(i)) + )}).fold(Map())((a,b) => a ++ b) + // TODO: mechanism should be implemented later + // val MhpmcounterStart = Mhpmcounter3 + // val MhpmeventStart = Mhpmevent3 + // for (i <- 0 until nrPerfCnts) { + // perfCntMapping += MaskedRegMap(MhpmcounterStart + i, perfCnts(i)) + // perfCntMapping += MaskedRegMap(MhpmeventStart + i, perfEvents(i)) + // } + + val cacheopRegs = CacheInstrucion.CacheInsRegisterList.map{case (name, attribute) => { + name -> RegInit(0.U(attribute("width").toInt.W)) + }} + val cacheopMapping = CacheInstrucion.CacheInsRegisterList.map{case (name, attribute) => { + MaskedRegMap( + Scachebase + attribute("offset").toInt, + cacheopRegs(name) + ) + }} + + val mapping = basicPrivMapping ++ + perfCntMapping ++ + pmpMapping ++ + pmaMapping ++ + (if (HasFPU) fcsrMapping else Nil) ++ + (if (HasVPU) vcsrMapping else Nil) ++ + (if (HasCustomCSRCacheOp) cacheopMapping else Nil) ++ + (if (HasHExtension) hcsrMapping else Nil) + + + println("XiangShan CSR Lists") + + for (addr <- mapping.keys.toSeq.sorted) { + println(f"$addr%#03x ${mapping(addr)._1}") + } + + val vs_s_csr_map = List( + Sstatus.U -> Vsstatus.U, + Sie.U -> Vsie.U, + Stvec.U -> Vstvec.U, + Sscratch.U -> Vsscratch.U, + Sepc.U -> Vsepc.U, + Scause.U -> Vscause.U, + Stval.U -> Vstval.U, + Sip.U -> Vsip.U, + Satp.U -> Vsatp.U + ) + val addr = Wire(UInt(12.W)) + val vscsr_addr = LookupTreeDefault(src2(11, 0), src2(11, 0), vs_s_csr_map) + when(virtMode){ + addr := vscsr_addr + }.otherwise{ + addr := src2(11, 0) + } + val csri = ZeroExt(src2(16, 12), XLEN) + val rdata = Wire(UInt(XLEN.W)) + val rdata_tmp = Wire(UInt(XLEN.W)) + val wdata_tmp = LookupTree(func, List( + CSROpType.wrt -> src1, + CSROpType.set -> (rdata | src1), + CSROpType.clr -> (rdata & (~src1).asUInt), + CSROpType.wrti -> csri, + CSROpType.seti -> (rdata | csri), + CSROpType.clri -> (rdata & (~csri).asUInt) + )) + val is_vsip_ie = addr === Vsip.U || addr === Vsie.U + // for the difftest with NEMU(stay consistent with Spike) + val is_satp = addr === Satp.U + val is_vsatp = addr === Vsatp.U + val is_hgatp = addr === Hgatp.U + val check_apt_mode = wdata_tmp(wdata_tmp.getWidth-1, 64-Satp_Mode_len) === 8.U || wdata_tmp(wdata_tmp.getWidth-1, 64-Satp_Mode_len) === 0.U + val wdata = MuxCase(wdata_tmp, Seq( + is_vsip_ie -> ZeroExt(wdata_tmp << 1, XLEN), + (is_satp && !check_apt_mode) -> satp, + (is_vsatp && !check_apt_mode) -> vsatp, + (is_hgatp && !check_apt_mode) -> hgatp + )) + val addrInPerfCnt = (addr >= Mcycle.U) && (addr <= Mhpmcounter31.U) || + (addr >= Mcountinhibit.U) && (addr <= Mhpmevent31.U) || + (addr >= Cycle.U) && (addr <= Hpmcounter31.U) || + addr === Mip.U + csrio.isPerfCnt := addrInPerfCnt && valid && func =/= CSROpType.jmp + + // satp wen check + val satpLegalMode = (wdata.asTypeOf(new SatpStruct).mode===0.U) || (wdata.asTypeOf(new SatpStruct).mode===8.U) + + // csr access check, special case + val tvmNotPermit = (privilegeMode === ModeS && !virtMode && mstatusStruct.tvm.asBool) + val accessPermitted = !(addr === Satp.U && tvmNotPermit) + val vtvmNotPermit = (privilegeMode === ModeS && virtMode && hstatusStruct.vtvm.asBool) + val vaccessPermitted = !(addr === Vsatp.U && vtvmNotPermit) +// csrio.disableSfence := (tvmNotPermit || !virtMode && privilegeMode < ModeS) || (vtvmNotPermit || virtMode && privilegeMode < ModeS) +// csrio.disableHfenceg := !((!virtMode && privilegeMode === ModeS && !mstatusStruct.tvm.asBool) || (privilegeMode === ModeM)) // only valid in HS and mstatus.tvm == 0 or in M +// csrio.disableHfencev := !(privilegeMode === ModeM || (!virtMode && privilegeMode === ModeS)) + + // general CSR wen check + val wen = valid && CSROpType.isCsrAccess(func) && ((addr=/=Satp.U && addr =/= Vsatp.U) || satpLegalMode) + val dcsrPermitted = dcsrPermissionCheck(addr, false.B, debugMode) + val triggerPermitted = triggerPermissionCheck(addr, true.B, debugMode) // todo dmode + val HasH = (HasHExtension == true).asBool + val csrAccess = csrAccessPermissionCheck(addr, false.B, privilegeMode, virtMode, HasH) + val modePermitted = csrAccess === 0.U && dcsrPermitted && triggerPermitted + val perfcntPermitted = perfcntPermissionCheck(addr, privilegeMode, mcounteren, scounteren) + val permitted = Mux(addrInPerfCnt, perfcntPermitted, modePermitted) && Mux(virtMode, vaccessPermitted, accessPermitted) + MaskedRegMap.generate(mapping, addr, rdata_tmp, wen && permitted, wdata) + rdata := Mux(is_vsip_ie, ZeroExt(rdata_tmp >> 1, XLEN), rdata_tmp) + io.out.bits.res.data := rdata + io.out.bits.ctrl.flushPipe.get := flushPipe + connect0LatencyCtrlSingal + + // send distribute csr a w signal + csrio.customCtrl.distribute_csr.w.valid := wen && permitted + csrio.customCtrl.distribute_csr.w.bits.data := wdata + csrio.customCtrl.distribute_csr.w.bits.addr := addr + + when (RegNext(csrio.fpu.fflags.valid)) { + fcsr := fflags_wfn(update = true)(RegEnable(csrio.fpu.fflags.bits, csrio.fpu.fflags.valid)) + } + when(RegNext(csrio.vpu.set_vxsat.valid)) { + vcsr := vxsat_wfn(update = true)(RegEnable(csrio.vpu.set_vxsat.bits, csrio.vpu.set_vxsat.valid)) + } + + // set fs and sd in mstatus + when (csrw_dirty_fp_state || RegNext(csrio.fpu.dirty_fs)) { + val mstatusNew = WireInit(mstatus.asTypeOf(new MstatusStruct)) + mstatusNew.fs := "b11".U + mstatusNew.sd := true.B + mstatus := mstatusNew.asUInt + when(virtMode){ + val vsstatusNew = WireInit(vsstatus.asTypeOf(new MstatusStruct)) + vsstatusNew.fs := "b11".U + vsstatusNew.sd := true.B + vsstatus := vsstatusNew.asUInt + } + } + csrio.fpu.frm := fcsr.asTypeOf(new FcsrStruct).frm + + when (RegNext(csrio.vpu.set_vstart.valid)) { + vstart := RegEnable(csrio.vpu.set_vstart.bits, csrio.vpu.set_vstart.valid) + } + when (RegNext(csrio.vpu.set_vtype.valid)) { + vtype := RegEnable(csrio.vpu.set_vtype.bits, csrio.vpu.set_vtype.valid) + } + vl := csrio.vpu.vl + // set vs and sd in mstatus + when(csrw_dirty_vs_state || RegNext(csrio.vpu.dirty_vs)) { + val mstatusNew = WireInit(mstatus.asTypeOf(new MstatusStruct)) + mstatusNew.vs := ContextStatus.dirty + mstatusNew.sd := true.B + mstatus := mstatusNew.asUInt + } + + csrio.vpu.vstart := vstart + csrio.vpu.vxrm := vcsr.asTypeOf(new VcsrStruct).vxrm + + // Trigger Ctrl + val triggerEnableVec = tdata1RegVec.map { tdata1 => + val mcontrolData = tdata1.asTypeOf(new Tdata1Bundle).data.asTypeOf(new MControlData) + tdata1.asTypeOf(new Tdata1Bundle).type_.asUInt === TrigTypeEnum.MCONTROL && ( + mcontrolData.m && privilegeMode === ModeM || + mcontrolData.s && privilegeMode === ModeS || + mcontrolData.u && privilegeMode === ModeU) + } + val fetchTriggerEnableVec = triggerEnableVec.zip(tdata1WireVec).map { + case (tEnable, tdata1) => tEnable && tdata1.asTypeOf(new Tdata1Bundle).data.asTypeOf(new MControlData).isFetchTrigger + } + val memAccTriggerEnableVec = triggerEnableVec.zip(tdata1WireVec).map { + case (tEnable, tdata1) => tEnable && tdata1.asTypeOf(new Tdata1Bundle).data.asTypeOf(new MControlData).isMemAccTrigger + } + csrio.customCtrl.frontend_trigger.tEnableVec := fetchTriggerEnableVec + csrio.customCtrl.mem_trigger.tEnableVec := memAccTriggerEnableVec + + val tdata1Update = wen && (addr === Tdata1.U) + val tdata2Update = wen && (addr === Tdata2.U) + val triggerUpdate = wen && (addr === Tdata1.U || addr === Tdata2.U) + val frontendTriggerUpdate = + tdata1Update && wdata.asTypeOf(new Tdata1Bundle).type_.asUInt === TrigTypeEnum.MCONTROL && + wdata.asTypeOf(new Tdata1Bundle).data.asTypeOf(new MControlData).isFetchTrigger || + tdata1Selected.data.asTypeOf(new MControlData).isFetchTrigger && triggerUpdate + val memTriggerUpdate = + tdata1Update && wdata.asTypeOf(new Tdata1Bundle).type_.asUInt === TrigTypeEnum.MCONTROL && + wdata.asTypeOf(new Tdata1Bundle).data.asTypeOf(new MControlData).isMemAccTrigger || + tdata1Selected.data.asTypeOf(new MControlData).isMemAccTrigger && triggerUpdate + + csrio.customCtrl.frontend_trigger.tUpdate.valid := RegNext(RegNext(frontendTriggerUpdate)) + csrio.customCtrl.mem_trigger.tUpdate.valid := RegNext(RegNext(memTriggerUpdate)) + XSDebug(triggerEnableVec.reduce(_ || _), p"Debug Mode: At least 1 trigger is enabled," + + p"trigger enable is ${Binary(triggerEnableVec.asUInt)}\n") + + // CSR inst decode + val isEbreak = addr === privEbreak && func === CSROpType.jmp + val isEcall = addr === privEcall && func === CSROpType.jmp + val isMret = addr === privMret && func === CSROpType.jmp + val isSret = addr === privSret && func === CSROpType.jmp + val isUret = addr === privUret && func === CSROpType.jmp + val isDret = addr === privDret && func === CSROpType.jmp + val isWFI = func === CSROpType.wfi + + // Illegal privileged operation list + val illegalMret = valid && isMret && privilegeMode < ModeM + val illegalSret = valid && isSret && privilegeMode < ModeS + val illegalSModeSret = valid && isSret && privilegeMode === ModeS && virtMode === false.B && mstatusStruct.tsr.asBool + // when hstatus.vtsr == 1, if sret is executed in VS-mode, it will cause virtual instruction + val illegalVSModeSret = valid && isSret && privilegeMode === ModeS && virtMode && hstatusStruct.vtsr.asBool + // When TW=1, then if WFI is executed in any less-privileged mode, + // and it does not complete within an implementation-specific, bounded time limit, + // the WFI instruction causes an illegal instruction exception. + // The time limit may always be 0, in which case WFI always causes + // an illegal instruction exception in less-privileged modes when TW=1. + val illegalWFI = valid && isWFI && (privilegeMode < ModeM && mstatusStruct.tw === 1.U || privilegeMode === ModeU && !virtMode) + val illegalVWFI = valid && isWFI && ((virtMode && privilegeMode === ModeS && hstatusStruct.vtw === 1.U && mstatusStruct.tw === 0.U)|| + (virtMode && privilegeMode === ModeU && mstatusStruct.tw === 0.U)) + // Illegal privileged instruction check + val isIllegalAddr = valid && CSROpType.isCsrAccess(func) && MaskedRegMap.isIllegalAddr(mapping, addr) + val isIllegalAccess = !virtMode && wen && !(Mux(addrInPerfCnt, perfcntPermitted, csrAccess === 0.U && dcsrPermitted && triggerPermitted) && accessPermitted) + val isIllegalPrivOp = illegalMret || illegalSret || illegalSModeSret || illegalWFI + + val isIllegalVAccess = virtMode && wen && (csrAccess === 2.U || !vaccessPermitted) + val isIllegalVPrivOp = illegalVSModeSret || illegalVWFI + // expose several csr bits for tlb + tlbBundle.priv.mxr := mstatusStruct.mxr.asBool + tlbBundle.priv.sum := mstatusStruct.sum.asBool + tlbBundle.priv.vmxr := vsstatusStruct.mxr.asBool + tlbBundle.priv.vsum := vsstatusStruct.sum.asBool + tlbBundle.priv.spvp := hstatusStruct.spvp + tlbBundle.priv.virt := Mux(mstatusStruct.mprv.asBool, mstatusStruct.mpv & (mstatusStruct.mpp =/= ModeM), virtMode) + tlbBundle.priv.imode := privilegeMode + tlbBundle.priv.dmode := Mux((debugMode && dcsr.asTypeOf(new DcsrStruct).mprven || !debugMode) && mstatusStruct.mprv.asBool, mstatusStruct.mpp, privilegeMode) + + // Branch control + val retTarget = WireInit(0.U) + val resetSatp = (addr === Satp.U || addr === Hgatp.U || addr === Vsatp.U) && wen // write to satp will cause the pipeline be flushed + val writeVstart = addr === Vstart.U && wen // write to vstart will cause the pipeline be flushed + dontTouch(writeVstart) + + val w_fcsr_change_rm = wen && addr === Fcsr.U && wdata(7, 5) =/= fcsr(7, 5) + val w_frm_change_rm = wen && addr === Frm.U && wdata(2, 0) =/= fcsr(7, 5) + val frm_change = w_fcsr_change_rm || w_frm_change_rm + val isXRet = valid && func === CSROpType.jmp && !isEcall && !isEbreak + flushPipe := resetSatp || frm_change || isXRet || frontendTriggerUpdate || writeVstart + + private val illegalRetTarget = WireInit(false.B) + when(valid) { + when(isDret) { + retTarget := dpc(VAddrBits - 1, 0) + }.elsewhen(isMret && !illegalMret) { + retTarget := mepc(VAddrBits - 1, 0) + }.elsewhen(isSret && !illegalSret && !illegalSModeSret && !illegalVSModeSret) { + retTarget := Mux(virtMode, vsepc(VAddrBits - 1, 0), sepc(VAddrBits - 1, 0)) + }.elsewhen(isUret) { + retTarget := uepc(VAddrBits - 1, 0) + }.otherwise { + illegalRetTarget := true.B + } + }.otherwise { + illegalRetTarget := true.B // when illegalRetTarget setted, retTarget should never be used + } + + // Mux tree for regs + when(valid) { + when(isDret) { + val mstatusNew = WireInit(mstatus.asTypeOf(new MstatusStruct)) + val debugModeNew = WireInit(debugMode) + when(dcsr.asTypeOf(new DcsrStruct).prv =/= ModeM) { + mstatusNew.mprv := 0.U + } //If the new privilege mode is less privileged than M-mode, MPRV in mstatus is cleared. + mstatus := mstatusNew.asUInt + privilegeMode := dcsr.asTypeOf(new DcsrStruct).prv + debugModeNew := false.B + debugIntrEnable := true.B + debugMode := debugModeNew + XSDebug("Debug Mode: Dret executed, returning to %x.", retTarget) + }.elsewhen(isMret && !illegalMret) { + val mstatusOld = WireInit(mstatus.asTypeOf(new MstatusStruct)) + val mstatusNew = WireInit(mstatus.asTypeOf(new MstatusStruct)) + mstatusNew.ie.m := mstatusOld.pie.m + privilegeMode := mstatusOld.mpp + if (HasHExtension) { + virtMode := mstatusOld.mpv + mstatusNew.mpv := 0.U + } + mstatusNew.pie.m := true.B + mstatusNew.mpp := ModeU + when(mstatusOld.mpp =/= ModeM) { + mstatusNew.mprv := 0.U + } + mstatus := mstatusNew.asUInt + }.elsewhen(isSret && !illegalSret && !illegalSModeSret && !illegalVSModeSret) { + val mstatusOld = WireInit(mstatus.asTypeOf(new MstatusStruct)) + val mstatusNew = WireInit(mstatus.asTypeOf(new MstatusStruct)) + val hstatusOld = WireInit(hstatus.asTypeOf(new HstatusStruct)) + val hstatusNew = WireInit(hstatus.asTypeOf(new HstatusStruct)) + val vsstatusOld = WireInit(vsstatus.asTypeOf(new MstatusStruct)) + val vsstatusNew = WireInit(vsstatus.asTypeOf(new MstatusStruct)) + when(virtMode === 0.U) { + virtMode := hstatusOld.spv + hstatusNew.spv := 0.U + mstatusNew.ie.s := mstatusOld.pie.s + privilegeMode := Cat(0.U(1.W), mstatusOld.spp) + mstatusNew.pie.s := true.B + mstatusNew.spp := ModeU + when(mstatusOld.spp =/= ModeM) { + mstatusNew.mprv := 0.U + } + mstatus := mstatusNew.asUInt + hstatus := hstatusNew.asUInt + }.otherwise { + privilegeMode := vsstatusOld.spp + vsstatusNew.spp := ModeU + vsstatusNew.ie.s := vsstatusOld.pie.s + vsstatusNew.pie.s := 1.U + vsstatus := vsstatusNew.asUInt + } + }.elsewhen(isUret) { + val mstatusOld = WireInit(mstatus.asTypeOf(new MstatusStruct)) + val mstatusNew = WireInit(mstatus.asTypeOf(new MstatusStruct)) + // mstatusNew.mpp.m := ModeU //TODO: add mode U + mstatusNew.ie.u := mstatusOld.pie.u + privilegeMode := ModeU + mstatusNew.pie.u := true.B + mstatus := mstatusNew.asUInt + } + } + + io.in.ready := true.B + io.out.valid := valid + + // In this situation, hart will enter debug mode instead of handling a breakpoint exception simply. + // Ebreak block instructions backwards, so it's ok to not keep extra info to distinguish between breakpoint + // exception and enter-debug-mode exception. + val ebreakEnterDebugMode = + (privilegeMode === ModeM && dcsrData.ebreakm) || + (privilegeMode === ModeS && dcsrData.ebreaks) || + (privilegeMode === ModeU && dcsrData.ebreaku) + + // raise a debug exception waiting to enter debug mode, instead of a breakpoint exception + val raiseDebugException = !debugMode && isEbreak && ebreakEnterDebugMode + + val csrExceptionVec = WireInit(0.U.asTypeOf(ExceptionVec())) + csrExceptionVec(breakPoint) := io.in.valid && isEbreak + csrExceptionVec(ecallM) := privilegeMode === ModeM && io.in.valid && isEcall + csrExceptionVec(ecallVS) := privilegeMode === ModeS && virtMode && io.in.valid && isEcall + csrExceptionVec(ecallS) := privilegeMode === ModeS && !virtMode && io.in.valid && isEcall + csrExceptionVec(ecallU) := privilegeMode === ModeU && io.in.valid && isEcall + // Trigger an illegal instr exception when: + // * unimplemented csr is being read/written + // * csr access is illegal + csrExceptionVec(illegalInstr) := isIllegalAddr || isIllegalAccess || isIllegalPrivOp + csrExceptionVec(virtualInstr) := isIllegalVAccess || isIllegalVPrivOp + io.out.bits.ctrl.exceptionVec.get := csrExceptionVec + + XSDebug(io.in.valid, s"Debug Mode: an Ebreak is executed, ebreak cause enter-debug-mode exception ? ${raiseDebugException}\n") + + /** + * Exception and Intr + */ + val idelegS = (mideleg & mip.asUInt) + val idelegVS = (hideleg & mideleg & mip.asUInt) + def privilegedEnableDetect(idelegS: Bool, idelegVS: Bool): Bool = Mux(idelegS, + Mux(idelegVS, (virtMode && privilegeMode === ModeS && vsstatusStruct.ie.s) || (virtMode && privilegeMode < ModeS), + ((privilegeMode === ModeS) && mstatusStruct.ie.s) || (privilegeMode < ModeS) || virtMode), + ((privilegeMode === ModeM) && mstatusStruct.ie.m) || (privilegeMode < ModeM)) + + val debugIntr = csrio.externalInterrupt.debug & debugIntrEnable + XSDebug(debugIntr, "Debug Mode: debug interrupt is asserted and valid!") + // send interrupt information to ROB + val intrVecEnable = Wire(Vec(13, Bool())) + val disableInterrupt = debugMode || (dcsrData.step && !dcsrData.stepie) + intrVecEnable.zip(idelegS.asBools).zip(idelegVS.asBools).map{case((x,y),z) => x := privilegedEnableDetect(y, z) && !disableInterrupt} + val intrVec = Cat(debugIntr && !debugMode, (mie(11,0) & mip.asUInt & intrVecEnable.asUInt)) + val intrBitSet = intrVec.orR + csrio.interrupt := intrBitSet + // Page 45 in RISC-V Privileged Specification + // The WFI instruction can also be executed when interrupts are disabled. The operation of WFI + // must be unaffected by the global interrupt bits in mstatus (MIE and SIE) and the delegation + // register mideleg, but should honor the individual interrupt enables (e.g, MTIE). + csrio.wfi_event := debugIntr || (mie(11, 0) & mip.asUInt).orR + mipWire.t.m := csrio.externalInterrupt.mtip + mipWire.s.m := csrio.externalInterrupt.msip + mipWire.e.m := csrio.externalInterrupt.meip + mipWire.e.s := csrio.externalInterrupt.seip + + // interrupts + val intrNO = IntPriority.foldRight(0.U)((i: Int, sum: UInt) => Mux(intrVec(i), i.U, sum)) + val hasIntr = csrio.exception.valid && csrio.exception.bits.isInterrupt + val ivmEnable = tlbBundle.priv.imode < ModeM && satp.asTypeOf(new SatpStruct).mode === 8.U + val iexceptionPC = Mux(ivmEnable, SignExt(csrio.exception.bits.pc, XLEN), csrio.exception.bits.pc) + val iexceptionGPAddr = Mux(ivmEnable, SignExt(csrio.exception.bits.gpaddr, XLEN), csrio.exception.bits.gpaddr) + val dvmEnable = tlbBundle.priv.dmode < ModeM && satp.asTypeOf(new SatpStruct).mode === 8.U + val dexceptionPC = Mux(dvmEnable, SignExt(csrio.exception.bits.pc, XLEN), csrio.exception.bits.pc) + XSDebug(hasIntr, "interrupt: pc=0x%x, %d\n", dexceptionPC, intrNO) + val hasDebugIntr = intrNO === IRQ_DEBUG.U && hasIntr + + // exceptions from rob need to handle + val exceptionVecFromRob = csrio.exception.bits.exceptionVec + val hasException = csrio.exception.valid && !csrio.exception.bits.isInterrupt + val hasInstrPageFault = hasException && exceptionVecFromRob(instrPageFault) + val hasLoadPageFault = hasException && exceptionVecFromRob(loadPageFault) + val hasStorePageFault = hasException && exceptionVecFromRob(storePageFault) + val hasStoreAddrMisalign = hasException && exceptionVecFromRob(storeAddrMisaligned) + val hasLoadAddrMisalign = hasException && exceptionVecFromRob(loadAddrMisaligned) + val hasInstrAccessFault = hasException && exceptionVecFromRob(instrAccessFault) + val hasLoadAccessFault = hasException && exceptionVecFromRob(loadAccessFault) + val hasStoreAccessFault = hasException && exceptionVecFromRob(storeAccessFault) + val hasBreakPoint = hasException && exceptionVecFromRob(breakPoint) + val hasInstGuestPageFault = hasException && exceptionVecFromRob(instrGuestPageFault) + val hasLoadGuestPageFault = hasException && exceptionVecFromRob(loadGuestPageFault) + val hasStoreGuestPageFault = hasException && exceptionVecFromRob(storeGuestPageFault) + val hasSingleStep = hasException && csrio.exception.bits.singleStep + val hasTriggerFire = hasException && csrio.exception.bits.trigger.canFire + val triggerFrontendHitVec = csrio.exception.bits.trigger.frontendHit + val triggerMemHitVec = csrio.exception.bits.trigger.backendHit + val triggerHitVec = triggerFrontendHitVec | triggerMemHitVec // Todo: update mcontrol.hit + val triggerCanFireVec = csrio.exception.bits.trigger.frontendCanFire | csrio.exception.bits.trigger.backendCanFire + // More than one triggers can hit at the same time, but only fire one + // We select the first hit trigger to fire + val triggerFireOH = PriorityEncoderOH(triggerCanFireVec) + val triggerFireAction = PriorityMux(triggerFireOH, tdata1WireVec.map(_.getTriggerAction)).asUInt + + + XSDebug(hasSingleStep, "Debug Mode: single step exception\n") + XSDebug(hasTriggerFire, p"Debug Mode: trigger fire, frontend hit vec ${Binary(csrio.exception.bits.trigger.frontendHit.asUInt)} " + + p"backend hit vec ${Binary(csrio.exception.bits.trigger.backendHit.asUInt)}\n") + + val hasExceptionVec = csrio.exception.bits.exceptionVec + val regularExceptionNO = ExceptionNO.priorities.foldRight(0.U)((i: Int, sum: UInt) => Mux(hasExceptionVec(i), i.U, sum)) + val exceptionNO = Mux(hasSingleStep || hasTriggerFire, 3.U, regularExceptionNO) + val causeNO = (hasIntr << (XLEN - 1)).asUInt | Mux(hasIntr, intrNO, exceptionNO) + + val hasExceptionIntr = csrio.exception.valid + + val hasDebugEbreakException = hasBreakPoint && ebreakEnterDebugMode + val hasDebugTriggerException = hasTriggerFire && triggerFireAction === TrigActionEnum.DEBUG_MODE + val hasDebugException = hasDebugEbreakException || hasDebugTriggerException || hasSingleStep + val hasDebugTrap = hasDebugException || hasDebugIntr + val ebreakEnterParkLoop = debugMode && hasExceptionIntr + + XSDebug(hasExceptionIntr, "int/exc: pc %x int (%d):%x exc: (%d):%x\n", + dexceptionPC, intrNO, intrVec, exceptionNO, hasExceptionVec.asUInt + ) + XSDebug(hasExceptionIntr, + "pc %x mstatus %x mideleg %x medeleg %x mode %x\n", + dexceptionPC, + mstatus, + mideleg, + medeleg, + privilegeMode + ) + + // mtval write logic + // Due to timing reasons of memExceptionVAddr, we delay the write of mtval and stval + val memExceptionAddr = SignExt(csrio.memExceptionVAddr, XLEN) + val memExceptionGPAddr = SignExt(csrio.memExceptionGPAddr, XLEN) + val updateTval = VecInit(Seq( + hasInstrPageFault, + hasLoadPageFault, + hasStorePageFault, + hasInstrAccessFault, + hasLoadAccessFault, + hasStoreAccessFault, + hasLoadAddrMisalign, + hasStoreAddrMisalign, + hasInstGuestPageFault, + hasLoadGuestPageFault, + hasStoreGuestPageFault, + hasBreakPoint, + )).asUInt.orR + val updateTval_h = VecInit(Seq( + hasInstGuestPageFault, + hasLoadGuestPageFault, + hasStoreGuestPageFault + )).asUInt.orR + when (RegNext(RegNext(updateTval))) { + val tval = Mux( + RegNext(RegNext(hasInstrPageFault || hasInstrAccessFault || hasInstGuestPageFault || hasBreakPoint)), + RegNext(RegNext(Mux( + csrio.exception.bits.crossPageIPFFix, + SignExt(csrio.exception.bits.pc + 2.U, XLEN), + iexceptionPC + ))), + memExceptionAddr + ) + // because we update tval two beats later, we can choose xtval according to the privilegeMode which has been updated + when (RegNext(privilegeMode === ModeM)) { + mtval := tval + }.otherwise { + when (virtMode){ + vstval := tval + }.otherwise{ + stval := tval + } + } + } + + when(RegNext(RegNext(updateTval_h))) { + val tval_tmp = Mux( + RegNext(RegNext(hasInstGuestPageFault)), + RegNext(RegNext(Mux( + csrio.exception.bits.crossPageIPFFix, + SignExt(csrio.exception.bits.gpaddr + 2.U, XLEN), + iexceptionGPAddr + ))), + memExceptionGPAddr + ) + val tval = tval_tmp >> 2 + when(RegNext(privilegeMode === ModeM)) { + mtval2 := tval + }.otherwise { + htval := tval + } + } + + val debugTrapTarget = Mux(!isEbreak && debugMode, 0x38020808.U, 0x38020800.U) // 0x808 is when an exception occurs in debug mode prog buf exec + val deleg = Mux(hasIntr, mideleg , medeleg) + val hdeleg = Mux(hasIntr, hideleg, hedeleg) + // val delegS = ((deleg & (1 << (causeNO & 0xf))) != 0) && (privilegeMode < ModeM); + val delegS = deleg(causeNO(7,0)) && (privilegeMode < ModeM) + val delegVS = virtMode && delegS && hdeleg(causeNO(7, 0)) && (privilegeMode < ModeM) + val clearTval = !updateTval || hasIntr + + val clearTval_h = !updateTval_h || hasIntr + val isHyperInst = csrio.exception.bits.isHls + // ctrl block will use theses later for flush + val isXRetFlag = RegInit(false.B) + when (DelayN(io.flush.valid, 5)) { + isXRetFlag := false.B + }.elsewhen (isXRet) { + isXRetFlag := true.B + } + csrio.isXRet := isXRetFlag + private val retTargetReg = RegEnable(retTarget, isXRet && !illegalRetTarget) + private val illegalXret = RegEnable(illegalMret || illegalSret || illegalSModeSret || illegalVSModeSret, isXRet) + + private val xtvec = Mux(delegS, Mux(delegVS, vstvec, stvec), mtvec) + private val xtvecBase = xtvec(VAddrBits - 1, 2) + // When MODE=Vectored, all synchronous exceptions into M/S mode + // cause the pc to be set to the address in the BASE field, whereas + // interrupts cause the pc to be set to the address in the BASE field + // plus four times the interrupt cause number. + private val pcFromXtvec = Cat(xtvecBase + Mux(xtvec(0) && hasIntr, causeNO(3, 0), 0.U), 0.U(2.W)) + + // XRet sends redirect instead of Flush and isXRetFlag is true.B before redirect.valid. + // ROB sends exception at T0 while CSR receives at T2. + // We add a RegNext here and trapTarget is valid at T3. + csrio.trapTarget := RegEnable( + MuxCase(pcFromXtvec, Seq( + (isXRetFlag && !illegalXret) -> retTargetReg, + ((hasDebugTrap && !debugMode) || ebreakEnterParkLoop) -> debugTrapTarget + )), + isXRetFlag || csrio.exception.valid) + + when(hasExceptionIntr) { + val mstatusOld = WireInit(mstatus.asTypeOf(new MstatusStruct)) + val mstatusNew = WireInit(mstatus.asTypeOf(new MstatusStruct)) + val hstatusOld = WireInit(hstatus.asTypeOf(new HstatusStruct)) + val hstatusNew = WireInit(hstatus.asTypeOf(new HstatusStruct)) + val vsstatusOld = WireInit(vsstatus.asTypeOf(new MstatusStruct)) + val vsstatusNew = WireInit(vsstatus.asTypeOf(new MstatusStruct)) + val dcsrNew = WireInit(dcsr.asTypeOf(new DcsrStruct)) + val debugModeNew = WireInit(debugMode) + when(hasDebugTrap && !debugMode) { + import DcsrStruct._ + debugModeNew := true.B + dcsrNew.prv := privilegeMode + privilegeMode := ModeM + when(hasDebugIntr) { + dpc := iexceptionPC + dcsrNew.cause := CAUSE_HALTREQ + XSDebug(hasDebugIntr, "Debug Mode: Trap to %x at pc %x\n", debugTrapTarget, dpc) + }.otherwise { // hasDebugException + dpc := iexceptionPC // TODO: check it when hasSingleStep + dcsrNew.cause := MuxCase(0.U, Seq( + hasTriggerFire -> CAUSE_TRIGGER, + raiseDebugException -> CAUSE_EBREAK, + hasBreakPoint -> CAUSE_HALTREQ, + hasSingleStep -> CAUSE_STEP + )) + } + dcsr := dcsrNew.asUInt + debugIntrEnable := false.B + }.elsewhen (debugMode) { + //do nothing + }.elsewhen (delegVS) { + vscause := (hasIntr << (XLEN-1)).asUInt | Mux(hasIntr, intrNO - 1.U, exceptionNO) + vsepc := Mux(hasInstrPageFault || hasInstrAccessFault, iexceptionPC, dexceptionPC) + vsstatusNew.spp := privilegeMode + vsstatusNew.pie.s := vsstatusOld.ie.s + vsstatusNew.ie.s := false.B + when (clearTval) {vstval := 0.U} + virtMode := true.B + privilegeMode := ModeS + }.elsewhen (delegS) { + val virt = Mux(mstatusOld.mprv.asBool, mstatusOld.mpv, virtMode) + // to do hld st + hstatusNew.gva := (hasInstGuestPageFault || hasLoadGuestPageFault || hasStoreGuestPageFault || + ((virt.asBool || isHyperInst) && ((hasException && 0.U <= exceptionNO && exceptionNO <= 7.U && exceptionNO =/= 2.U) + || hasInstrPageFault || hasLoadPageFault || hasStorePageFault))) + hstatusNew.spv := virtMode + when(virtMode){ + hstatusNew.spvp := privilegeMode + } + virtMode := false.B + scause := causeNO + sepc := Mux(hasInstrPageFault || hasInstrAccessFault, iexceptionPC, dexceptionPC) + mstatusNew.spp := privilegeMode + mstatusNew.pie.s := mstatusOld.ie.s + mstatusNew.ie.s := false.B + privilegeMode := ModeS + when (clearTval) { stval := 0.U } + when (clearTval_h) {htval := 0.U} + }.otherwise { + val virt = Mux(mstatusOld.mprv.asBool, mstatusOld.mpv, virtMode) + // to do hld st + mstatusNew.gva := (hasInstGuestPageFault || hasLoadGuestPageFault || hasStoreGuestPageFault || + ((virt.asBool || isHyperInst) && ((hasException && 0.U <= exceptionNO && exceptionNO <= 7.U && exceptionNO =/= 2.U) + || hasInstrPageFault || hasLoadPageFault || hasStorePageFault))) + mstatusNew.mpv := virtMode + virtMode := false.B + mcause := causeNO + mepc := Mux(hasInstrPageFault || hasInstrAccessFault, iexceptionPC, dexceptionPC) + mstatusNew.mpp := privilegeMode + mstatusNew.pie.m := mstatusOld.ie.m + mstatusNew.ie.m := false.B + privilegeMode := ModeM + when (clearTval) { mtval := 0.U } + when (clearTval_h) {mtval2 := 0.U} + } + mstatus := mstatusNew.asUInt + vsstatus := vsstatusNew.asUInt + hstatus := hstatusNew.asUInt + debugMode := debugModeNew + } + + // Cache error debug support + if(HasCustomCSRCacheOp){ + val cache_error_decoder = Module(new CSRCacheErrorDecoder) + cache_error_decoder.io.encoded_cache_error := cacheopRegs("CACHE_ERROR") + } + + // Implicit add reset values for mepc[0] and sepc[0] + // TODO: rewrite mepc and sepc using a struct-like style with the LSB always being 0 + when (RegNext(RegNext(reset.asBool) && !reset.asBool)) { + mepc := Cat(mepc(XLEN - 1, 1), 0.U(1.W)) + sepc := Cat(sepc(XLEN - 1, 1), 0.U(1.W)) + vsepc := Cat(vsepc(XLEN - 1, 1), 0.U(1.W)) + } + + def readWithScala(addr: Int): UInt = mapping(addr)._1 + + val difftestIntrNO = Mux(hasIntr, causeNO, 0.U) + + // Always instantiate basic difftest modules. + if (env.AlwaysBasicDiff || env.EnableDifftest) { + val difftest = DifftestModule(new DiffArchEvent, delay = 3, dontCare = true) + difftest.coreid := csrio.hartId + difftest.valid := csrio.exception.valid + difftest.interrupt := Mux(hasIntr, causeNO, 0.U) + difftest.exception := Mux(hasException, causeNO, 0.U) + difftest.exceptionPC := dexceptionPC + if (env.EnableDifftest || env.FullBasicDiff) { + difftest.exceptionInst := csrio.exception.bits.instr + } + } + + // Always instantiate basic difftest modules. + if (env.AlwaysBasicDiff || env.EnableDifftest) { + val difftest = DifftestModule(new DiffCSRState) + difftest.coreid := csrio.hartId + difftest.privilegeMode := privilegeMode + difftest.mstatus := mstatus + difftest.sstatus := mstatus & sstatusRmask + difftest.mepc := mepc + difftest.sepc := sepc + difftest.mtval:= mtval + difftest.stval:= stval + difftest.mtvec := mtvec + difftest.stvec := stvec + difftest.mcause := mcause + difftest.scause := scause + difftest.satp := satp + difftest.mip := mipReg + difftest.mie := mie + difftest.mscratch := mscratch + difftest.sscratch := sscratch + difftest.mideleg := mideleg + difftest.medeleg := medeleg + } + + if (env.AlwaysBasicDiff || env.EnableDifftest) { + val difftest = DifftestModule(new DiffHCSRState) + difftest.coreid := csrio.hartId + difftest.virtMode := virtMode + difftest.mtval2 := mtval2 + difftest.mtinst := mtinst + difftest.hstatus := hstatus + difftest.hideleg := hideleg + difftest.hedeleg := hedeleg + difftest.hcounteren := hcounteren + difftest.htval := htval + difftest.htinst := htinst + difftest.hgatp := hgatp + difftest.vsstatus := vsstatus + difftest.vstvec := vstvec + difftest.vsepc := vsepc + difftest.vscause := vscause + difftest.vstval := vstval + difftest.vsatp := vsatp + difftest.vsscratch := vsscratch + } + + if(env.AlwaysBasicDiff || env.EnableDifftest) { + val difftest = DifftestModule(new DiffDebugMode) + difftest.coreid := csrio.hartId + difftest.debugMode := debugMode + difftest.dcsr := dcsr + difftest.dpc := dpc + difftest.dscratch0 := dscratch0 + difftest.dscratch1 := dscratch1 + } + + if (env.AlwaysBasicDiff || env.EnableDifftest) { + val difftest = DifftestModule(new DiffVecCSRState) + difftest.coreid := csrio.hartId + difftest.vstart := vstart + difftest.vxsat := vcsr.asTypeOf(new VcsrStruct).vxsat + difftest.vxrm := vcsr.asTypeOf(new VcsrStruct).vxrm + difftest.vcsr := vcsr + difftest.vl := vl + difftest.vtype := vtype + difftest.vlenb := vlenb + } +} +class PFEvent(implicit p: Parameters) extends XSModule { + val io = IO(new Bundle { + val distribute_csr = Flipped(new DistributedCSRIO()) + val hpmevent = Output(Vec(29, UInt(XLEN.W))) + }) + + val w = io.distribute_csr.w + + val perfEvents = List.fill(8)(RegInit("h0000000000".U(XLEN.W))) ++ + List.fill(8)(RegInit("h4010040100".U(XLEN.W))) ++ + List.fill(8)(RegInit("h8020080200".U(XLEN.W))) ++ + List.fill(5)(RegInit("hc0300c0300".U(XLEN.W))) + + val perfEventMapping = (0 until 29).map(i => {Map( + MaskedRegMap(addr = CSRs.mhpmevent3 + i, + reg = perfEvents(i), + wmask = "hf87fff3fcff3fcff".U(XLEN.W)) + )}).fold(Map())((a,b) => a ++ b) + + val rdata = Wire(UInt(XLEN.W)) + MaskedRegMap.generate(perfEventMapping, w.bits.addr, rdata, w.valid, w.bits.data) + for(i <- 0 until 29){ + io.hpmevent(i) := perfEvents(i) + } +} + */ diff --git a/examples/prefab/cores/xiangshan/arch/native/xiangshan/backend/fu/FuConfig.scala b/examples/prefab/cores/xiangshan/arch/native/xiangshan/backend/fu/FuConfig.scala new file mode 100644 index 000000000..7ba5fc090 --- /dev/null +++ b/examples/prefab/cores/xiangshan/arch/native/xiangshan/backend/fu/FuConfig.scala @@ -0,0 +1,894 @@ +// Replaces XiangShan src/main/scala/xiangshan/backend/fu/FuConfig.scala (same package) with Buckyball changes. + +package xiangshan.backend.fu + +import org.chipsalliance.cde.config.Parameters +import chisel3._ +import utils.EnumUtils.OHEnumeration +import xiangshan.ExceptionNO._ +import xiangshan.SelImm +import xiangshan.backend.fu.fpu.{IntFPToVec, IntToFP} +import xiangshan.backend.fu.wrapper._ +import xiangshan.backend.Bundles.ExuInput +import xiangshan.backend.datapath.DataConfig._ +import xiangshan.mem.Std + +/** + * @param name [[String]] name of fuConfig + * @param fuType [[Int]] type of func, select from [[xiangshan.backend.fu.FuType]] + * @param fuGen how to create $fu + * @param srcData type of src data used by this $fu + * @param piped if the $fu is pipelined + * @param maybeBlock the $fu need ready signal to block internal pipeline + * @param writeIntRf the $fu write int regfiles + * @param writeFpRf the $fu write float regfiles + * @param writeVecRf the $fu write vector regfiles + * @param writeV0Rf the $fu write v0 regfiles + * @param writeVlRf the $fu write vl regfiles + * @param writeFflags the $fu write fflags csr + * @param writeVxsat the $fu write vxsat csr + * @param destDataBits the width of output data in the $fu + * @param srcDataBits the width of input data in the $fu, the default value is destDataBits + * @param latency the latency of instuction executed in the $fu + * @param hasInputBuffer if the $fu has input buffer + * @param exceptionOut the $fu can produce these exception + * @param hasLoadError if the $fu has load error out + * @param flushPipe if the instuction executed in the $fu need flush out + * @param replayInst if the instuction executed in the $fu can replay in some condition + * @param trigger if the $fu need trigger out + * @param needSrcFrm if the $fu need float rounding mode signal + * @param needSrcVxrm if the $fu need vector fixed-point rounding mode signal + * @param immType the immediate type of this $fu + * @param vconfigWakeUp + * @param maskWakeUp + * @param writeVType if the $fu need write vtype + * @param readOldVtype if the $fu need read old vtype + * + * @define fu function unit + */ +case class FuConfig( + name: String, + fuType: FuType.OHType, + fuGen: (Parameters, FuConfig) => FuncUnit, + srcData: Seq[Seq[DataConfig]], + piped: Boolean, + maybeBlock: Boolean = false, + writeIntRf: Boolean = false, + writeFpRf: Boolean = false, + writeVecRf: Boolean = false, + writeV0Rf: Boolean = false, + writeVlRf: Boolean = false, + writeFakeIntRf: Boolean = false, + writeFflags: Boolean = false, + writeVxsat: Boolean = false, + destDataBits: Int = 64, + srcDataBits: Option[Int] = None, + srcNeedCopy: Boolean = false, + latency: HasFuLatency = CertainLatency(0), // two field (base latency, extra latency(option)) + hasInputBuffer: (Boolean, Int, Boolean) = (false, 0, false), + exceptionOut: Seq[Int] = Seq(), + hasLoadError: Boolean = false, + flushPipe: Boolean = false, + replayInst: Boolean = false, + trigger: Boolean = false, + needSrcFrm: Boolean = false, + needSrcVxrm: Boolean = false, + writeVType: Boolean = false, + immType: Set[UInt] = Set(), + // vector + vconfigWakeUp: Boolean = false, + maskWakeUp: Boolean = false, + readOldVtype: Boolean = false) { + def needIntWen: Boolean = writeIntRf || writeFakeIntRf + def needFpWen: Boolean = writeFpRf + def needVecWen: Boolean = writeVecRf + def needV0Wen: Boolean = writeV0Rf + def needVlWen: Boolean = writeVlRf + var vconfigIdx = -1 + var maskSrcIdx = -1 + if (vconfigWakeUp) { + vconfigIdx = getSpecialSrcIdx(VlData(), "when vconfigWakeUp is true, srcData must always contains VlData()") + } + if (maskWakeUp) { + maskSrcIdx = getSpecialSrcIdx(V0Data(), "when maskWakeUp is true, srcData must always contains V0Data()") + } + + require(!piped || piped && latency.latencyVal.isDefined, "The latency value must be set when piped is enable") + require(!vconfigWakeUp || vconfigWakeUp && vconfigIdx >= 0, "The index of vl src must be set when vlWakeUp is enable") + require(!maskWakeUp || maskWakeUp && maskSrcIdx >= 0, "The index of mask src must be set when vlWakeUp is enable") + + def numIntSrc: Int = srcData.map(_.count(x => IntRegSrcDataSet.contains(x))).fold(0)(_ max _) + def numFpSrc: Int = srcData.map(_.count(x => FpRegSrcDataSet.contains(x))).fold(0)(_ max _) + def numVecSrc: Int = srcData.map(_.count(x => VecRegSrcDataSet.contains(x))).fold(0)(_ max _) + def numVfSrc: Int = srcData.map(_.count(x => VecRegSrcDataSet.contains(x))).fold(0)(_ max _) + def numV0Src: Int = srcData.map(_.count(x => V0RegSrcDataSet.contains(x))).fold(0)(_ max _) + def numVlSrc: Int = srcData.map(_.count(x => VlRegSrcDataSet.contains(x))).fold(0)(_ max _) + def numRegSrc: Int = srcData.map(_.count(x => RegSrcDataSet.contains(x))).fold(0)(_ max _) + def numSrc: Int = srcData.map(_.length).fold(0)(_ max _) + + def readFp: Boolean = numFpSrc > 0 + + def fuSel(uop: ExuInput): Bool = + // Don't add more shit here!!! + // Todo: add new FuType to distinguish f2i, f2f + uop.fuType === this.fuType.U + + /** + * params(i): data type set of the ith src port + * @return + */ + def getRfReadDataCfgSet: Seq[Set[DataConfig]] = { + val numSrcMax = srcData.map(_.length).fold(0)(_ max _) + // make srcData is uniform sized to avoid exception when transpose + val alignedSrcData: Seq[Seq[DataConfig]] = srcData.map(x => x ++ Seq.fill(numSrcMax - x.length)(null)) + alignedSrcData.transpose.map(_.toSet.intersect(RegSrcDataSet)) + } + + def getSrcDataType(srcIdx: Int): Set[DataConfig] = { + srcData + .map((x: Seq[DataConfig]) => if (x.isDefinedAt(srcIdx)) Some(x(srcIdx)) else None) + .filter(_.nonEmpty) + .map(_.get) + .toSet + } + + def hasNoDataWB: Boolean = + !(writeIntRf || writeFpRf || writeVecRf || writeV0Rf || writeVlRf) + + def getSrcMaxWidthVec = + getRfReadDataCfgSet.map(_.map(_.dataWidth).max) + + def genSrcDataVec: Seq[UInt] = + getSrcMaxWidthVec.map(w => UInt(w.W)) + + // csr's redirect also uses redirect bundle + def hasRedirect: Boolean = Seq(FuType.jmp, FuType.brh, FuType.csr).contains(fuType) + + def hasPredecode: Boolean = Seq(FuType.jmp, FuType.brh, FuType.csr, FuType.ldu).contains(fuType) + + def needTargetPc: Boolean = Seq(FuType.jmp, FuType.brh).contains(fuType) + + // predict info + def needPdInfo: Boolean = Seq(FuType.jmp, FuType.brh, FuType.csr).contains(fuType) + + def needPc: Boolean = Seq(FuType.jmp, FuType.brh, FuType.ldu, FuType.csr).contains(fuType) + + def needFPUCtrl: Boolean = { + import FuType._ + Seq(fmac, fDivSqrt, i2f).contains(fuType) + } + + def needVecCtrl: Boolean = { + import FuType._ + Seq(vipu, vialuF, vimac, vidiv, vfpu, vppu, vfalu, vfma, vfdiv, vfcvt, vldu, vstu, vsetfwf).contains(fuType) + } + + def needCriticalErrors: Boolean = Seq(FuType.csr).contains(fuType) + + def isMul: Boolean = fuType == FuType.mul + + def isDiv: Boolean = fuType == FuType.div + + def isCsr: Boolean = fuType == FuType.csr + + def isBrh: Boolean = fuType == FuType.brh + + def isJmp: Boolean = fuType == FuType.jmp + + def isFence: Boolean = fuType == FuType.fence + + def isVecArith: Boolean = fuType == FuType.vialuF || fuType == FuType.vimac || + fuType == FuType.vppu || fuType == FuType.vipu || + fuType == FuType.vfalu || fuType == FuType.vfma || + fuType == FuType.vfdiv || fuType == FuType.vfcvt || + fuType == FuType.vidiv + + def isVecMem: Boolean = fuType == FuType.vldu || fuType == FuType.vstu || + fuType == FuType.vsegldu || fuType == FuType.vsegstu + + def needOg2: Boolean = isVecArith || fuType == FuType.vsetfwf || isVecMem + + def isSta: Boolean = name.contains("sta") + + def isStd: Boolean = name.contains("std") + + def ckAlwaysEn: Boolean = isCsr || isFence + + /** + * Get index of special src data, like [[VlData]], [[V0Data]] + * + * @param data [[DataConfig]] + * @param tips tips if get failed + * @return the index of special src data + */ + protected def getSpecialSrcIdx(data: DataConfig, tips: String): Int = { + val srcIdxVec = srcData.map(x => x.indexOf(data)) + val idx0 = srcIdxVec.head + for (idx <- srcIdxVec) { + require(idx >= 0 && idx == idx0, tips + ", and at the same index.") + } + idx0 + } + + override def toString: String = { + var str = s"${this.name}: " + if (vconfigWakeUp) str += s"vconfigIdx($vconfigIdx), " + if (maskWakeUp) str += s"maskSrcIdx($maskSrcIdx), " + str += s"latency($latency)" + str += s"src($srcData)" + str + } + +} + +object FuConfig { + + val JmpCfg: FuConfig = FuConfig( + name = "jmp", + fuType = FuType.jmp, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new JumpUnit(cfg)(p)).suggestName("jmp"), + srcData = Seq( + Seq(IntData()) // jal + ), + piped = true, + writeIntRf = true, + immType = Set(SelImm.IMM_I, SelImm.IMM_UJ, SelImm.IMM_U) + ) + + val BrhCfg: FuConfig = FuConfig( + name = "brh", + fuType = FuType.brh, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new BranchUnit(cfg)(p).suggestName("brh")), + srcData = Seq( + Seq(IntData(), IntData()) + ), + piped = true, + immType = Set(SelImm.IMM_SB) + ) + + val I2fCfg: FuConfig = FuConfig( + name = "i2f", + FuType.i2f, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new IntToFP(cfg)(p).suggestName("i2f")), + srcData = Seq( + Seq(IntData()) + ), + piped = true, + writeFpRf = true, + writeFflags = true, + latency = CertainLatency(2), + needSrcFrm = true + ) + + val I2vCfg: FuConfig = FuConfig( + name = "i2v", + FuType.i2v, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new IntFPToVec(cfg)(p).suggestName("i2v")), + srcData = Seq( + Seq(IntData(), IntData()) + ), + piped = true, + writeFpRf = true, + writeVecRf = true, + writeV0Rf = true, + latency = CertainLatency(0), + destDataBits = 128, + srcDataBits = Some(64), + immType = Set(SelImm.IMM_OPIVIU, SelImm.IMM_OPIVIS, SelImm.IMM_VRORVI) + ) + + val F2vCfg: FuConfig = FuConfig( + name = "f2v", + FuType.f2v, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new IntFPToVec(cfg)(p).suggestName("f2v")), + srcData = Seq( + Seq(FpData(), FpData()), + Seq(FpData()) + ), + piped = true, + writeFpRf = true, + writeVecRf = true, + writeV0Rf = true, + latency = CertainLatency(0), + destDataBits = 128, + srcDataBits = Some(64) + ) + + val CsrCfg: FuConfig = FuConfig( + name = "csr", + fuType = FuType.csr, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new examples.core.xiangshan.native.CSR(cfg)(p).suggestName("csr")), + srcData = Seq( + Seq(IntData(), IntData()) + ), + piped = false, + writeIntRf = true, + latency = UncertainLatency(), + exceptionOut = Seq(illegalInstr, virtualInstr, breakPoint, ecallU, ecallS, ecallVS, ecallM), + flushPipe = true + ) + + val AluCfg: FuConfig = FuConfig( + name = "alu", + fuType = FuType.alu, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new Alu(cfg)(p).suggestName("Alu")), + srcData = Seq( + Seq(IntData(), IntData()) + ), + piped = true, + writeIntRf = true, + immType = Set(SelImm.IMM_I, SelImm.IMM_U, SelImm.IMM_LUI32) + ) + + val MulCfg: FuConfig = FuConfig( + name = "mul", + fuType = FuType.mul, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new MulUnit(cfg)(p).suggestName("Mul")), + srcData = Seq( + Seq(IntData(), IntData()) + ), + piped = true, + writeIntRf = true, + latency = CertainLatency(2) + ) + + val DivCfg: FuConfig = FuConfig( + name = "div", + fuType = FuType.div, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new DivUnit(cfg)(p).suggestName("Div")), + srcData = Seq( + Seq(IntData(), IntData()) + ), + piped = false, + writeIntRf = true, + latency = UncertainLatency(), + hasInputBuffer = (true, 4, true) + ) + + val FenceCfg: FuConfig = FuConfig( + name = "fence", + FuType.fence, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new Fence(cfg)(p).suggestName("Fence")), + srcData = Seq( + Seq(IntData(), IntData()) + ), + piped = false, + latency = UncertainLatency(), + exceptionOut = Seq(illegalInstr, virtualInstr), + flushPipe = true + ) + + // Todo: split it to simple bitmap exu and complex bku + val BkuCfg: FuConfig = FuConfig( + name = "bku", + fuType = FuType.bku, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new Bku(cfg)(p).suggestName("Bku")), + srcData = Seq( + Seq(IntData(), IntData()) + ), + piped = true, + writeIntRf = true, + latency = CertainLatency(2) + ) + + val VSetRvfWvfCfg: FuConfig = FuConfig( + name = "vsetrvfwvf", + fuType = FuType.vsetfwf, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new VSetRvfWvf(cfg)(p).suggestName("VSetRvfWvf")), + srcData = Seq( + Seq(VecData(), VecData(), VecData(), V0Data(), VlData()) // vs1, vs2, vd_old, v0, vtype&vl + ), + piped = true, + writeVlRf = true, + writeVType = true, + writeIntRf = true, + latency = CertainLatency(0), + immType = Set(SelImm.IMM_VSETVLI, SelImm.IMM_VSETIVLI), + readOldVtype = true + ) + + val VSetRiWvfCfg: FuConfig = FuConfig( + name = "vsetriwvf", + fuType = FuType.vsetiwf, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new VSetRiWvf(cfg)(p).suggestName("VSetRiWvf")), + srcData = Seq( + Seq(IntData(), IntData()) + ), + piped = true, + writeVlRf = true, + writeVType = true, + latency = CertainLatency(0), + immType = Set(SelImm.IMM_VSETVLI, SelImm.IMM_VSETIVLI) + ) + + val VSetRiWiCfg: FuConfig = FuConfig( + name = "vsetriwi", + fuType = FuType.vsetiwi, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new VSetRiWi(cfg)(p).suggestName("VSetRiWi")), + srcData = Seq( + Seq(IntData(), IntData()) + ), + piped = true, + writeIntRf = true, + latency = CertainLatency(0), + immType = Set(SelImm.IMM_VSETVLI, SelImm.IMM_VSETIVLI) + ) + + val LduCfg: FuConfig = FuConfig( + name = "ldu", + fuType = FuType.ldu, + fuGen = null, // Todo + srcData = Seq( + Seq(IntData()) + ), + piped = false, // Todo: check it + writeIntRf = true, + writeFpRf = true, + latency = UncertainLatency(3), + exceptionOut = + Seq(loadAddrMisaligned, loadAccessFault, loadPageFault, loadGuestPageFault, breakPoint, hardwareError), + flushPipe = true, + replayInst = true, + hasLoadError = true, + trigger = true, + immType = Set(SelImm.IMM_I) + ) + + val StaCfg: FuConfig = FuConfig( + name = "sta", + fuType = FuType.stu, + fuGen = null, // Todo + srcData = Seq( + Seq(IntData()) + ), + piped = false, + latency = UncertainLatency(), + exceptionOut = + Seq(storeAddrMisaligned, storeAccessFault, storePageFault, storeGuestPageFault, breakPoint, hardwareError), + flushPipe = true, + trigger = true, + immType = Set(SelImm.IMM_S) + ) + + val StdCfg: FuConfig = FuConfig( + name = "std", + fuType = FuType.stu, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new Std(cfg)(p).suggestName("Std")), + srcData = Seq( + Seq(IntData()), + Seq(FpData()) + ), + piped = true, + latency = CertainLatency(0) + ) + + val HyldaCfg = FuConfig( + name = "hylda", + fuType = FuType.ldu, + fuGen = null, // Todo + srcData = Seq( + Seq(IntData()) + ), + piped = false, // Todo: check it + writeIntRf = true, + writeFpRf = true, + latency = UncertainLatency(3), + exceptionOut = + Seq(loadAddrMisaligned, loadAccessFault, loadPageFault, loadGuestPageFault, breakPoint, hardwareError), + flushPipe = true, + replayInst = true, + hasLoadError = true, + immType = Set(SelImm.IMM_I) + ) + + val HystaCfg = FuConfig( + name = "hysta", + fuType = FuType.stu, + fuGen = null, // Todo + srcData = Seq( + Seq(IntData()) + ), + piped = false, + latency = UncertainLatency(), + exceptionOut = + Seq(storeAddrMisaligned, storeAccessFault, storePageFault, storeGuestPageFault, breakPoint, hardwareError), + immType = Set(SelImm.IMM_S) + ) + + val FakeHystaCfg = FuConfig( + name = "hysta", + fuType = FuType.stu, + fuGen = null, // Todo + srcData = Seq(), + piped = false, + latency = UncertainLatency(), + exceptionOut = Seq(storeAddrMisaligned, storeAccessFault, storePageFault, storeGuestPageFault), + immType = Set() + ) + + val MouCfg: FuConfig = FuConfig( + name = "mou", + fuType = FuType.mou, + fuGen = null, // Todo + srcData = Seq( + Seq(IntData()) + ), + piped = false, // Todo: check it + writeFakeIntRf = true, + latency = UncertainLatency(), + exceptionOut = (LduCfg.exceptionOut ++ StaCfg.exceptionOut ++ StdCfg.exceptionOut).distinct, + trigger = true + ) + + val MoudCfg: FuConfig = FuConfig( + name = "moud", + fuType = FuType.mou, + fuGen = null, // Todo + srcData = Seq( + Seq(IntData()) + ), + piped = true, + latency = CertainLatency(0) + ) + + val VialuCfg = FuConfig( + name = "vialuFix", + fuType = FuType.vialuF, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new VIAluFix(cfg)(p).suggestName("VialuFix")), + srcData = Seq( + Seq(VecData(), VecData(), VecData(), V0Data(), VlData()) // vs1, vs2, vd_old, v0, vtype&vl + ), + piped = true, + writeVecRf = true, + writeV0Rf = true, + writeVxsat = true, + needSrcVxrm = true, + latency = CertainLatency(1), + vconfigWakeUp = true, + maskWakeUp = true, + destDataBits = 128, + exceptionOut = Seq(illegalInstr) + ) + + val VimacCfg = FuConfig( + name = "vimac", + fuType = FuType.vimac, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new VIMacU(cfg)(p).suggestName("Vimac")), + srcData = Seq( + Seq(VecData(), VecData(), VecData(), V0Data(), VlData()) // vs1, vs2, vd_old, v0, vtype&vl + ), + piped = true, + writeVecRf = true, + writeV0Rf = true, + writeVxsat = true, + needSrcVxrm = true, + latency = CertainLatency(2), + vconfigWakeUp = true, + maskWakeUp = true, + destDataBits = 128, + exceptionOut = Seq(illegalInstr) + ) + + val VidivCfg = FuConfig( + name = "vidiv", + fuType = FuType.vidiv, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new VIDiv(cfg)(p).suggestName("Vidiv")), + srcData = Seq( + Seq(VecData(), VecData(), VecData(), V0Data(), VlData()) // vs1, vs2, vd_old, v0, vtype&vl + ), + piped = false, + writeVecRf = true, + writeV0Rf = true, + latency = UncertainLatency(), + vconfigWakeUp = true, + maskWakeUp = true, + destDataBits = 128, + exceptionOut = Seq(illegalInstr) + ) + + val VppuCfg = FuConfig( + name = "vppu", + fuType = FuType.vppu, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new VPPU(cfg)(p).suggestName("Vppu")), + srcData = Seq( + Seq(VecData(), VecData(), VecData(), V0Data(), VlData()) // vs1, vs2, vd_old, v0, vtype&vl + ), + piped = true, + writeVecRf = true, + writeV0Rf = true, + latency = CertainLatency(2), + vconfigWakeUp = true, + maskWakeUp = true, + destDataBits = 128, + exceptionOut = Seq(illegalInstr) + ) + + val VipuCfg: FuConfig = FuConfig( + name = "vipu", + fuType = FuType.vipu, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new VIPU(cfg)(p).suggestName("Vipu")), + srcData = Seq( + Seq(VecData(), VecData(), VecData(), V0Data(), VlData()) // vs1, vs2, vd_old, v0 + ), + piped = true, + writeIntRf = true, + writeVecRf = true, + writeV0Rf = true, + latency = CertainLatency(2), + vconfigWakeUp = true, + maskWakeUp = true, + destDataBits = 128, + exceptionOut = Seq(illegalInstr) + ) + + val VfaluCfg = FuConfig( + name = "vfalu", + fuType = FuType.vfalu, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new VFAlu(cfg)(p).suggestName("Vfalu")), + srcData = Seq( + Seq(VecData(), VecData(), VecData(), V0Data(), VlData()) // vs1, vs2, vd_old, v0, vtype&vl + ), + piped = true, + writeVecRf = true, + writeV0Rf = true, + writeFpRf = true, + writeFflags = true, + latency = CertainLatency(1), + vconfigWakeUp = true, + maskWakeUp = true, + destDataBits = 128, + exceptionOut = Seq(illegalInstr), + needSrcFrm = true + ) + + val VfmaCfg = FuConfig( + name = "vfma", + fuType = FuType.vfma, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new VFMA(cfg)(p).suggestName("Vfma")), + srcData = Seq( + Seq(VecData(), VecData(), VecData(), V0Data(), VlData()) // vs1, vs2, vd_old, v0, vtype&vl + ), + piped = true, + writeVecRf = true, + writeV0Rf = true, + writeFflags = true, + latency = CertainLatency(3), + vconfigWakeUp = true, + maskWakeUp = true, + destDataBits = 128, + exceptionOut = Seq(illegalInstr), + needSrcFrm = true + ) + + val VfdivCfg = FuConfig( + name = "vfdiv", + fuType = FuType.vfdiv, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new VFDivSqrt(cfg)(p).suggestName("Vfdiv")), + srcData = Seq( + Seq(VecData(), VecData(), VecData(), V0Data(), VlData()) // vs1, vs2, vd_old, v0, vtype&vl + ), + piped = false, + writeVecRf = true, + writeV0Rf = true, + writeFflags = true, + latency = UncertainLatency(), + vconfigWakeUp = true, + maskWakeUp = true, + destDataBits = 128, + exceptionOut = Seq(illegalInstr), + needSrcFrm = true + ) + + val VfcvtCfg = FuConfig( + name = "vfcvt", + fuType = FuType.vfcvt, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new VCVT(cfg)(p).suggestName("Vfcvt")), + srcData = Seq( + Seq(VecData(), VecData(), VecData(), V0Data(), VlData()) // vs1, vs2, vd_old, v0, vtype&vl + ), + piped = true, + writeVecRf = true, + writeV0Rf = true, + writeFflags = true, + latency = CertainLatency(2), + vconfigWakeUp = true, + maskWakeUp = true, + destDataBits = 128, + exceptionOut = Seq(illegalInstr), + needSrcFrm = true + ) + + val FaluCfg = FuConfig( + name = "falu", + fuType = FuType.falu, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new FAlu(cfg)(p).suggestName("Falu")), + srcData = Seq( + Seq(FpData(), FpData()) + ), + piped = true, + writeFpRf = true, + writeIntRf = true, + writeFflags = true, + latency = CertainLatency(1), + destDataBits = 64, + needSrcFrm = true + ) + + val FmacCfg = FuConfig( + name = "fmac", + fuType = FuType.fmac, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new FMA(cfg)(p).suggestName("Fmac")), + srcData = Seq( + Seq(FpData(), FpData(), FpData()) + ), + piped = true, + writeFpRf = true, + writeFflags = true, + latency = CertainLatency(3), + destDataBits = 64, + needSrcFrm = true + ) + + val FdivCfg = FuConfig( + name = "fdiv", + fuType = FuType.fDivSqrt, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new FDivSqrt(cfg)(p).suggestName("Fdiv")), + srcData = Seq( + Seq(FpData(), FpData()) + ), + piped = false, + writeFpRf = true, + writeFflags = true, + latency = UncertainLatency(), + destDataBits = 64, + needSrcFrm = true + ) + + val FcvtCfg = FuConfig( + name = "fcvt", + fuType = FuType.fcvt, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new FCVT(cfg)(p).suggestName("Fcvt")), + srcData = Seq( + Seq(FpData()) + ), + piped = true, + writeFpRf = true, + writeIntRf = true, + writeFflags = true, + latency = CertainLatency(2), + destDataBits = 64, + needSrcFrm = true + ) + + val VlduCfg: FuConfig = FuConfig( + name = "vldu", + fuType = FuType.vldu, + fuGen = null, + srcData = Seq( + Seq(VecData(), VecData(), VecData(), V0Data(), VlData()) //vs1, vs2, vd_old, v0, vconfig + ), + piped = false, // Todo: check it + writeVecRf = true, + writeV0Rf = true, + writeVlRf = true, + latency = UncertainLatency(), + exceptionOut = + Seq(loadAddrMisaligned, loadAccessFault, loadPageFault, loadGuestPageFault, breakPoint, hardwareError), + flushPipe = true, + replayInst = true, + trigger = true, + hasLoadError = true, + vconfigWakeUp = true, + maskWakeUp = true, + destDataBits = 128 + ) + + val VstuCfg: FuConfig = FuConfig( + name = "vstu", + fuType = FuType.vstu, + fuGen = null, + srcData = Seq( + Seq(VecData(), VecData(), VecData(), V0Data(), VlData()) //vs1, vs2, vd_old, v0, vconfig + ), + piped = false, + latency = UncertainLatency(), + exceptionOut = + Seq(storeAddrMisaligned, storeAccessFault, storePageFault, storeGuestPageFault, breakPoint, hardwareError), + flushPipe = true, + replayInst = true, + trigger = true, + hasLoadError = true, + vconfigWakeUp = true, + maskWakeUp = true, + destDataBits = 128 + ) + + val VseglduSeg: FuConfig = FuConfig( + name = "vsegldu", + fuType = FuType.vsegldu, + fuGen = null, + srcData = Seq( + Seq(VecData(), VecData(), VecData(), V0Data(), VlData()) //vs1, vs2, vd_old, v0, vconfig + ), + piped = false, // Todo: check it + writeVecRf = true, + writeV0Rf = true, + writeVlRf = true, + latency = UncertainLatency(), + exceptionOut = Seq(loadAddrMisaligned, loadAccessFault, loadPageFault, breakPoint, hardwareError), + flushPipe = true, + replayInst = true, + trigger = true, + hasLoadError = true, + vconfigWakeUp = true, + maskWakeUp = true, + destDataBits = 128 + ) + + val VsegstuCfg: FuConfig = FuConfig( + name = "vsegstu", + fuType = FuType.vsegstu, + fuGen = null, + srcData = Seq( + Seq(VecData(), VecData(), VecData(), V0Data(), VlData()) //vs1, vs2, vd_old, v0, vconfig + ), + piped = false, + latency = UncertainLatency(), + exceptionOut = Seq(storeAddrMisaligned, storeAccessFault, storePageFault, breakPoint, hardwareError), + flushPipe = true, + replayInst = true, + trigger = true, + hasLoadError = true, + vconfigWakeUp = true, + maskWakeUp = true, + destDataBits = 128 + ) + + def allConfigs = Seq( + JmpCfg, + BrhCfg, + I2fCfg, + I2vCfg, + F2vCfg, + CsrCfg, + AluCfg, + MulCfg, + DivCfg, + FenceCfg, + BkuCfg, + VSetRvfWvfCfg, + VSetRiWvfCfg, + VSetRiWiCfg, + LduCfg, + StaCfg, + StdCfg, + MouCfg, + MoudCfg, + VialuCfg, + VipuCfg, + VlduCfg, + VstuCfg, + VseglduSeg, + VsegstuCfg, + FaluCfg, + FmacCfg, + FcvtCfg, + FdivCfg, + VfaluCfg, + VfmaCfg, + VfcvtCfg, + HyldaCfg, + HystaCfg + ) + + def VecArithFuConfigs = Seq( + VialuCfg, + VimacCfg, + VppuCfg, + VipuCfg, + VfaluCfg, + VfmaCfg, + VfcvtCfg + ) + +} diff --git a/examples/prefab/cores/xiangshan/arch/native/xiangshan/backend/fu/FuncUnit.scala b/examples/prefab/cores/xiangshan/arch/native/xiangshan/backend/fu/FuncUnit.scala new file mode 100644 index 000000000..6c05d0a4a --- /dev/null +++ b/examples/prefab/cores/xiangshan/arch/native/xiangshan/backend/fu/FuncUnit.scala @@ -0,0 +1,315 @@ +// Replaces XiangShan src/main/scala/xiangshan/backend/fu/FuncUnit.scala (same package) with Buckyball changes. + +package xiangshan.backend.fu + +import org.chipsalliance.cde.config.Parameters +import chisel3._ +import chisel3.util._ +import utility._ +import utils.OptionWrapper +import xiangshan._ +import xiangshan.backend.Bundles.VPUCtrlSignals +import xiangshan.backend.rob.RobPtr +import xiangshan.frontend.{FtqPtr, PreDecodeInfo} +import xiangshan.backend.datapath.DataConfig._ +import xiangshan.backend.fu.vector.Bundles.Vxsat +import xiangshan.ExceptionNO.illegalInstr +import xiangshan.backend.fu.vector.Bundles.VType +import xiangshan.backend.fu.wrapper.{CSRInput, CSRToDecode} + +class FuncUnitCtrlInput(cfg: FuConfig)(implicit p: Parameters) extends XSBundle { + val fuOpType = FuOpType() + val robIdx = new RobPtr + val pdest = UInt(PhyRegIdxWidth.W) + val rfWen = OptionWrapper(cfg.needIntWen, Bool()) + val fpWen = OptionWrapper(cfg.needFpWen, Bool()) + val vecWen = OptionWrapper(cfg.needVecWen, Bool()) + val v0Wen = OptionWrapper(cfg.needV0Wen, Bool()) + val vlWen = OptionWrapper(cfg.needVlWen, Bool()) + val flushPipe = OptionWrapper(cfg.flushPipe, Bool()) + val preDecode = OptionWrapper(cfg.hasPredecode, new PreDecodeInfo) + val ftqIdx = OptionWrapper(cfg.needPc || cfg.replayInst || cfg.isSta || cfg.isCsr, new FtqPtr) + val ftqOffset = OptionWrapper(cfg.needPc || cfg.replayInst || cfg.isSta || cfg.isCsr, UInt(log2Up(PredictWidth).W)) + + val predictInfo = OptionWrapper( + cfg.needPdInfo, + new Bundle { + val target = UInt(VAddrData().dataWidth.W) + val taken = Bool() + } + ) + + val fpu = OptionWrapper(cfg.writeFflags, new FPUCtrlSignals) + val vpu = OptionWrapper(cfg.needVecCtrl, new VPUCtrlSignals) +} + +class FuncUnitCtrlOutput(cfg: FuConfig)(implicit p: Parameters) extends XSBundle { + val robIdx = new RobPtr + val pdest = UInt(PhyRegIdxWidth.W) // Todo: use maximum of pregIdxWidth of different pregs + val rfWen = OptionWrapper(cfg.needIntWen, Bool()) + val fpWen = OptionWrapper(cfg.needFpWen, Bool()) + val vecWen = OptionWrapper(cfg.needVecWen, Bool()) + val v0Wen = OptionWrapper(cfg.needV0Wen, Bool()) + val vlWen = OptionWrapper(cfg.needVlWen, Bool()) + val exceptionVec = OptionWrapper(cfg.exceptionOut.nonEmpty, ExceptionVec()) + val flushPipe = OptionWrapper(cfg.flushPipe, Bool()) + val satpFlushPipe = OptionWrapper(cfg.isCsr, Bool()) + val replay = OptionWrapper(cfg.replayInst, Bool()) + val preDecode = OptionWrapper(cfg.hasPredecode, new PreDecodeInfo) + val fpu = OptionWrapper(cfg.writeFflags, new FPUCtrlSignals) + val vpu = OptionWrapper(cfg.needVecCtrl, new VPUCtrlSignals) +} + +class FuncUnitDataInput(cfg: FuConfig)(implicit p: Parameters) extends XSBundle { + val src = MixedVec(cfg.genSrcDataVec) + val imm = UInt(cfg.destDataBits.W) + val pc = OptionWrapper(cfg.needPc, UInt(VAddrData().dataWidth.W)) + val nextPcOffset = OptionWrapper(cfg.needPc && !cfg.isCsr, UInt((log2Up(PredictWidth) + 1).W)) + + def getSrcVConfig: UInt = src(cfg.vconfigIdx) + def getSrcMask: UInt = src(cfg.maskSrcIdx) +} + +class FuncUnitDataOutput(cfg: FuConfig)(implicit p: Parameters) extends XSBundle { + val data = UInt(cfg.destDataBits.W) + val fflags = OptionWrapper(cfg.writeFflags, UInt(5.W)) + val vxsat = OptionWrapper(cfg.writeVxsat, Vxsat()) + val redirect = OptionWrapper(cfg.hasRedirect, ValidIO(new Redirect)) +} + +class FuncUnitInput(cfg: FuConfig)(implicit p: Parameters) extends XSBundle { + val needCtrlPipe = cfg.latency.latencyVal.nonEmpty && (!cfg.isStd) + val ctrl = new FuncUnitCtrlInput(cfg) + val ctrlPipe = OptionWrapper(needCtrlPipe, Vec(cfg.latency.latencyVal.get + 1, new FuncUnitCtrlInput(cfg))) + val validPipe = OptionWrapper(needCtrlPipe, Vec(cfg.latency.latencyVal.get + 1, Bool())) + val data = new FuncUnitDataInput(cfg) + val dataPipe = OptionWrapper(needCtrlPipe, Vec(cfg.latency.latencyVal.get + 1, new FuncUnitDataInput(cfg))) + val perfDebugInfo = new PerfDebugInfo() + val debug_seqNum = InstSeqNum() +} + +class FuncUnitOutput(cfg: FuConfig)(implicit p: Parameters) extends XSBundle { + val ctrl = new FuncUnitCtrlOutput(cfg) + val res = new FuncUnitDataOutput(cfg) + val perfDebugInfo = new PerfDebugInfo() + val debug_seqNum = InstSeqNum() +} + +class FuncUnitIO(cfg: FuConfig)(implicit p: Parameters) extends XSBundle { + val flush = Flipped(ValidIO(new Redirect)) + val in = Flipped(DecoupledIO(new FuncUnitInput(cfg))) + val out = DecoupledIO(new FuncUnitOutput(cfg)) + val csrin = OptionWrapper(cfg.isCsr, new CSRInput) + val csrio = OptionWrapper(cfg.isCsr, new CSRFileIO) + val csrToDecode = OptionWrapper(cfg.isCsr, Output(new CSRToDecode)) + val fenceio = OptionWrapper(cfg.isFence, new FenceIO) + val frm = OptionWrapper(cfg.needSrcFrm, Input(UInt(3.W))) + val vxrm = OptionWrapper(cfg.needSrcVxrm, Input(UInt(2.W))) + val vtype = OptionWrapper(cfg.writeVlRf, (Valid(new VType))) + val vlIsZero = OptionWrapper(cfg.writeVlRf, Output(Bool())) + val vlIsVlmax = OptionWrapper(cfg.writeVlRf, Output(Bool())) + val instrAddrTransType = Option.when(cfg.isJmp || cfg.isBrh)(Input(new AddrTransType)) +} + +abstract class FuncUnit(val cfg: FuConfig)(implicit p: Parameters) extends XSModule with HasCriticalErrors { + val io = IO(new FuncUnitIO(cfg)) + PerfCCT.updateInstPos(io.in.bits.debug_seqNum, PerfCCT.InstPos.AtFU.id.U, io.in.valid, clock, reset) + PerfCCT.updateInstPos(io.out.bits.debug_seqNum, PerfCCT.InstPos.AtBypassVal.id.U, io.out.valid, clock, reset) + val criticalErrors = Seq(("none", false.B)) + + // should only be used in non-piped fu + def connectNonPipedCtrlSingal: Unit = { + io.out.bits.ctrl.robIdx := RegEnable(io.in.bits.ctrl.robIdx, io.in.fire) + io.out.bits.ctrl.pdest := RegEnable(io.in.bits.ctrl.pdest, io.in.fire) + io.out.bits.ctrl.rfWen.foreach(_ := RegEnable(io.in.bits.ctrl.rfWen.get, io.in.fire)) + io.out.bits.ctrl.fpWen.foreach(_ := RegEnable(io.in.bits.ctrl.fpWen.get, io.in.fire)) + io.out.bits.ctrl.vecWen.foreach(_ := RegEnable(io.in.bits.ctrl.vecWen.get, io.in.fire)) + io.out.bits.ctrl.v0Wen.foreach(_ := RegEnable(io.in.bits.ctrl.v0Wen.get, io.in.fire)) + io.out.bits.ctrl.vlWen.foreach(_ := RegEnable(io.in.bits.ctrl.vlWen.get, io.in.fire)) + // io.out.bits.ctrl.flushPipe should be connected in fu + io.out.bits.ctrl.preDecode.foreach(_ := RegEnable(io.in.bits.ctrl.preDecode.get, io.in.fire)) + io.out.bits.ctrl.fpu.foreach(_ := RegEnable(io.in.bits.ctrl.fpu.get, io.in.fire)) + io.out.bits.ctrl.vpu.foreach(_ := RegEnable(io.in.bits.ctrl.vpu.get, io.in.fire)) + io.out.bits.perfDebugInfo := RegEnable(io.in.bits.perfDebugInfo, io.in.fire) + io.out.bits.debug_seqNum := RegEnable(io.in.bits.debug_seqNum, io.in.fire) + } + + def connectNonPipedCtrlSingalForCSR: Unit = { + io.out.bits.ctrl.robIdx := DataHoldBypass(io.in.bits.ctrl.robIdx, io.in.fire) + io.out.bits.ctrl.pdest := DataHoldBypass(io.in.bits.ctrl.pdest, io.in.fire) + io.out.bits.ctrl.rfWen.foreach(_ := DataHoldBypass(io.in.bits.ctrl.rfWen.get, io.in.fire)) + io.out.bits.ctrl.fpWen.foreach(_ := DataHoldBypass(io.in.bits.ctrl.fpWen.get, io.in.fire)) + io.out.bits.ctrl.vecWen.foreach(_ := DataHoldBypass(io.in.bits.ctrl.vecWen.get, io.in.fire)) + io.out.bits.ctrl.v0Wen.foreach(_ := DataHoldBypass(io.in.bits.ctrl.v0Wen.get, io.in.fire)) + io.out.bits.ctrl.vlWen.foreach(_ := DataHoldBypass(io.in.bits.ctrl.vlWen.get, io.in.fire)) + // io.out.bits.ctrl.flushPipe should be connected in fu + io.out.bits.ctrl.preDecode.foreach(_ := DataHoldBypass(io.in.bits.ctrl.preDecode.get, io.in.fire)) + io.out.bits.ctrl.fpu.foreach(_ := DataHoldBypass(io.in.bits.ctrl.fpu.get, io.in.fire)) + io.out.bits.ctrl.vpu.foreach(_ := DataHoldBypass(io.in.bits.ctrl.vpu.get, io.in.fire)) + io.out.bits.perfDebugInfo := DataHoldBypass(io.in.bits.perfDebugInfo, io.in.fire) + io.out.bits.debug_seqNum := DataHoldBypass(io.in.bits.debug_seqNum, io.in.fire) + } + + def connect0LatencyCtrlSingal: Unit = { + io.out.bits.ctrl.robIdx := io.in.bits.ctrl.robIdx + io.out.bits.ctrl.pdest := io.in.bits.ctrl.pdest + io.out.bits.ctrl.rfWen.foreach(_ := io.in.bits.ctrl.rfWen.get) + io.out.bits.ctrl.fpWen.foreach(_ := io.in.bits.ctrl.fpWen.get) + io.out.bits.ctrl.vecWen.foreach(_ := io.in.bits.ctrl.vecWen.get) + io.out.bits.ctrl.v0Wen.foreach(_ := io.in.bits.ctrl.v0Wen.get) + io.out.bits.ctrl.vlWen.foreach(_ := io.in.bits.ctrl.vlWen.get) + // io.out.bits.ctrl.flushPipe should be connected in fu + io.out.bits.ctrl.preDecode.foreach(_ := io.in.bits.ctrl.preDecode.get) + io.out.bits.ctrl.fpu.foreach(_ := io.in.bits.ctrl.fpu.get) + io.out.bits.ctrl.vpu.foreach(_ := io.in.bits.ctrl.vpu.get) + io.out.bits.perfDebugInfo := io.in.bits.perfDebugInfo + io.out.bits.debug_seqNum := io.in.bits.debug_seqNum + } + +} + +/** + * @author LinJiaWei, Yinan Xu + */ +trait HasPipelineReg { this: FuncUnit => + def latency: Int + + val latdiff: Int = cfg.latency.extraLatencyVal.getOrElse(0) + val preLat: Int = latency - latdiff + require(latency >= 0 && latdiff >= 0) + + def pipelineReg( + init: FuncUnitInput, + valid: Bool, + ready: Bool, + latency: Int, + flush: ValidIO[Redirect] + ): (Seq[FuncUnitInput], Seq[Bool], Seq[Bool]) = { + val rdyVec = Seq.fill(latency)(Wire(Bool())) :+ ready + val validVec = valid +: Seq.fill(latency)(RegInit(false.B)) + val ctrlVec = init.ctrl +: Seq.fill(latency)(Reg(chiselTypeOf(io.in.bits.ctrl))) + val dataVec = init.data +: Seq.fill(latency)(Reg(chiselTypeOf(io.in.bits.data))) + val perfVec = init.perfDebugInfo +: Seq.fill(latency)(Reg(chiselTypeOf(io.in.bits.perfDebugInfo))) + val seqNumVec = init.debug_seqNum +: Seq.fill(latency)(Reg(chiselTypeOf(io.in.bits.debug_seqNum))) + + val robIdxVec = ctrlVec.map(_.robIdx) + + // if flush(0), valid 0 will not given, so set flushVec(0) to false.B + val flushVec = validVec.zip(robIdxVec).map(x => x._1 && x._2.needFlush(flush)) + + for (i <- 0 until latency) { + rdyVec(i) := !validVec(i + 1) || rdyVec(i + 1).asTypeOf(Bool()) + } + for (i <- 1 to latency) { + validVec(i) := validVec(i - 1) + when(rdyVec(i - 1) && validVec(i - 1)) { + ctrlVec(i) := ctrlVec(i - 1) + dataVec(i) := dataVec(i - 1) + perfVec(i) := perfVec(i - 1) + seqNumVec(i) := seqNumVec(i - 1) + } + } + + ( + ctrlVec.zip(dataVec).zip(perfVec).zip(seqNumVec).map { + case (((ctrl, data), perf), debug_seqNum) => { + val out = Wire(new FuncUnitInput(cfg)) + out.ctrl := ctrl + out.ctrlPipe.foreach(_ := 0.U.asTypeOf(out.ctrlPipe.get)) + out.validPipe.foreach(_ := 0.U.asTypeOf(out.validPipe.get)) + out.dataPipe.foreach(_ := 0.U.asTypeOf(out.dataPipe.get)) + out.data := data + out.perfDebugInfo := perf + out.debug_seqNum := debug_seqNum + out + } + }, + validVec, + rdyVec + ) + } + + val (pipeReg: Seq[FuncUnitInput], validVecThisFu, rdyVec) = + pipelineReg(io.in.bits, io.in.valid, io.out.ready, preLat, io.flush) + val validVec = io.in.bits.validPipe.get.zip(validVecThisFu).map(x => x._1 && x._2) + val ctrlVec = io.in.bits.ctrlPipe.get + val dataVec = io.in.bits.dataPipe.get + val perfVec = pipeReg.map(_.perfDebugInfo) + val seqNumVec = pipeReg.map(_.debug_seqNum) + + val fixtiminginit = Wire(new FuncUnitInput(cfg)) + fixtiminginit.ctrl := ctrlVec.last + fixtiminginit.ctrlPipe.foreach(_ := 0.U.asTypeOf(fixtiminginit.ctrlPipe.get)) + fixtiminginit.validPipe.foreach(_ := 0.U.asTypeOf(fixtiminginit.validPipe.get)) + fixtiminginit.dataPipe.foreach(_ := 0.U.asTypeOf(fixtiminginit.dataPipe.get)) + fixtiminginit.data := dataVec.last + fixtiminginit.perfDebugInfo := perfVec.last + fixtiminginit.debug_seqNum := seqNumVec.last + + // fixtiming pipelinereg + val (fixpipeReg: Seq[FuncUnitInput], fixValidVec, fixRdyVec) = + pipelineReg(fixtiminginit, validVec.last, rdyVec.head, latdiff, io.flush) + val fixDataVec = fixpipeReg.map(_.data) + val fixPerfVec = fixpipeReg.map(_.perfDebugInfo) + val fixSeqNumVec = fixpipeReg.map(_.debug_seqNum) + val pcVec = fixDataVec.map(_.pc) + + io.in.ready := fixRdyVec.head + io.out.valid := fixValidVec.last + + io.out.bits.ctrl.robIdx := ctrlVec.last.robIdx + io.out.bits.ctrl.pdest := ctrlVec.last.pdest + io.out.bits.ctrl.rfWen.foreach(_ := ctrlVec.last.rfWen.get) + io.out.bits.ctrl.fpWen.foreach(_ := ctrlVec.last.fpWen.get) + io.out.bits.ctrl.vecWen.foreach(_ := ctrlVec.last.vecWen.get) + io.out.bits.ctrl.v0Wen.foreach(_ := ctrlVec.last.v0Wen.get) + io.out.bits.ctrl.vlWen.foreach(_ := ctrlVec.last.vlWen.get) + io.out.bits.ctrl.fpu.foreach(_ := ctrlVec.last.fpu.get) + io.out.bits.ctrl.vpu.foreach(_ := ctrlVec.last.vpu.get) + io.out.bits.perfDebugInfo := fixPerfVec.last + io.out.bits.debug_seqNum := fixSeqNumVec.last + + // vstart illegal + if (cfg.exceptionOut.nonEmpty) { + val outVstart = ctrlVec.last.vpu.get.vstart + val vstartIllegal = outVstart =/= 0.U + io.out.bits.ctrl.exceptionVec.get := 0.U.asTypeOf(io.out.bits.ctrl.exceptionVec.get) + io.out.bits.ctrl.exceptionVec.get(illegalInstr) := vstartIllegal + } + + def regEnable(i: Int): Bool = validVec(i - 1) && rdyVec(i - 1) + + def PipelineReg[TT <: Data](i: Int)(next: TT) = { + val lat = preLat min i + RegEnable( + next, + regEnable(lat) + ) + } + + def SNReg[TT <: Data](in: TT, n: Int): TT = { + val lat = preLat min n + var next = in + for (i <- 1 to lat) { + next = PipelineReg[TT](i)(next) + } + next + } + + def S1Reg[TT <: Data](next: TT): TT = PipelineReg[TT](1)(next) + + def S2Reg[TT <: Data](next: TT): TT = PipelineReg[TT](2)(next) + + def S3Reg[TT <: Data](next: TT): TT = PipelineReg[TT](3)(next) + + def S4Reg[TT <: Data](next: TT): TT = PipelineReg[TT](4)(next) + + def S5Reg[TT <: Data](next: TT): TT = PipelineReg[TT](5)(next) + +} + +abstract class PipedFuncUnit(override val cfg: FuConfig)(implicit p: Parameters) + extends FuncUnit(cfg) + with HasPipelineReg { + override def latency: Int = cfg.latency.latencyVal.get +} diff --git a/examples/prefab/cores/xiangshan/arch/native/xiangshan/backend/fu/NewCSR/NewCSR.scala b/examples/prefab/cores/xiangshan/arch/native/xiangshan/backend/fu/NewCSR/NewCSR.scala new file mode 100644 index 000000000..ce5f2098e --- /dev/null +++ b/examples/prefab/cores/xiangshan/arch/native/xiangshan/backend/fu/NewCSR/NewCSR.scala @@ -0,0 +1,1853 @@ +// Replaces XiangShan src/main/scala/xiangshan/backend/fu/NewCSR/NewCSR.scala (same package) with Buckyball changes. + +package xiangshan.backend.fu.NewCSR + +import chisel3._ +import chisel3.util._ +import difftest._ +import freechips.rocketchip.rocket.CSRs +import org.chipsalliance.cde.config.Parameters +import top.{ArgParser, Generator} +import utility._ +import utils.OptionWrapper +import xiangshan.backend.fu.NewCSR.CSRBundles._ +import xiangshan.backend.fu.NewCSR.CSRDefines._ +import xiangshan.backend.fu.NewCSR.CSREnumTypeImplicitCast._ +import xiangshan.backend.fu.NewCSR.CSRFunc._ +import xiangshan.backend.fu.NewCSR.CSREvents.{ + CSREvents, + DretEventSinkBundle, + EventUpdatePrivStateOutput, + MNretEventSinkBundle, + MretEventSinkBundle, + SretEventSDTSinkBundle, + SretEventSinkBundle, + TargetPCBundle, + TrapEntryDEventSinkBundle, + TrapEntryEventInput, + TrapEntryHSEventSinkBundle, + TrapEntryMEventSinkBundle, + TrapEntryMNEventSinkBundle, + TrapEntryVSEventSinkBundle +} +import xiangshan.backend.fu.fpu.Bundles.Frm +import xiangshan.backend.fu.vector.Bundles.{Vl, Vstart, Vxrm, Vxsat} +import xiangshan.backend.fu.wrapper.CSRToDecode +import xiangshan.backend.rob.RobPtr +import xiangshan._ +import xiangshan.backend.fu.PerfCounterIO +import xiangshan.backend.fu.util.CSRConst +import xiangshan.ExceptionNO._ +import xiangshan.backend.trace._ + +import scala.collection.immutable.SeqMap + +object CSRConfig { + final val ASIDLEN = 16 // the length of ASID of XS implementation + + final val ASIDMAX = 16 // the max value of ASIDLEN defined by spec + + final val HIIDWidth = 12 // support Hvictl[27:16](IID) + + final val VMIDLEN = 14 // the length of VMID of XS implementation + + final val VMIDMAX = 14 // the max value of VMIDLEN defined by spec + + final val VaddrMaxWidth = 48 + 2 // support Sv39/Sv48/Sv39x4/Sv48x4 + + final val InstWidth = 32 + + final val XLEN = 64 // Todo: use XSParams + + final val VLEN = 128 + + // Since we need macro to compute the width of CSR field, the input of macro should be the value that can be computed + // at compile time. The log2Up function cannot be used as meta-programming function, so we use litral value here + // log2Up(128 + 1), hold 0~128 + final val VlWidth = 8 + + final val PAddrWidth = 48 + + final val AddrWidthInPage = 12 + + final val PMPAddrWidth = 48 + + final val PMPOffBits = 2 + + final val PMPAddrBits = PMPAddrWidth - PMPOffBits + + // perf + final val perfCntNum = 29 // in Spec + + final val EXT_SSTC = true + + final val EXT_DBLTRP = true + + final val PPNLength = 44 + +} + +class NewCSRInput(implicit p: Parameters) extends Bundle { + val wen = Bool() + val ren = Bool() + val op = UInt(2.W) + val addr = UInt(12.W) + val src = UInt(64.W) + val wdata = UInt(64.W) + val mnret = Input(Bool()) + val mret = Input(Bool()) + val sret = Input(Bool()) + val dret = Input(Bool()) + val redirectFlush = Input(Bool()) +} + +class NewCSROutput(implicit p: Parameters) extends Bundle { + val EX_II = Bool() + val EX_VI = Bool() + val flushPipe = Bool() + val rData = UInt(64.W) + val targetPcUpdate = Bool() + val targetPc = new TargetPCBundle + val regOut = UInt(64.W) + // perf + val isPerfCnt = Bool() +} + +class NewCSR(implicit val p: Parameters) + extends Module + with HasXSParameter + with MachineLevel + with SupervisorLevel + with HypervisorLevel + with VirtualSupervisorLevel + with Unprivileged + with CSRAIA + with CSRIND + with HasExternalInterruptBundle + with HasNonMaskableIRPBundle + with CSREvents + with DebugLevel + with CSRCustom + with CSRPMP + with CSRPMA + with CSRDocDump + with HasCriticalErrors + with IpIeAliasConnect { + + import CSRConfig._ + + val io = IO(new Bundle { + + val fromTop = Input(new Bundle { + val hartId = UInt(hartIdLen.W) + val clintTime = Input(ValidIO(UInt(64.W))) + val l2FlushDone = Input(Bool()) + val reset_mtvec = Option.when(enableResetMtvec)(Input(UInt(PAddrBits.W))) + val criticalErrorState = Input(Bool()) + }) + + val in = Flipped(DecoupledIO(new NewCSRInput)) + val trapInst = Input(ValidIO(UInt(InstWidth.W))) + + val fromMem = Input(new Bundle { + val excpVA = UInt(XLEN.W) + val excpGPA = UInt(XLEN.W) + val excpIsForVSnonLeafPTE = Bool() + }) + + val fromRob = Input(new Bundle { + + val trap = ValidIO(new Bundle { + val pc = UInt((VaddrMaxWidth + 1).W) + val pcGPA = UInt(PAddrBitsMax.W) + val instr = UInt(InstWidth.W) + val trapVec = UInt(64.W) + val isFetchBkpt = Bool() + val singleStep = Bool() + val trigger = TriggerAction() + val crossPageIPFFix = Bool() + val isInterrupt = Bool() + val isHls = Bool() + val isFetchMalAddr = Bool() + val isForVSnonLeafPTE = Bool() + val satpFlushFirstFetchFault = Bool() + }) + + val commit = Input(new RobCommitCSR) + val robDeqPtr = Input(new RobPtr) + }) + + val fromVecExcpMod = Input(new Bundle { + val busy = Bool() + }) + + val perf = Input(new PerfCounterIO) + + /** Output should be a DecoupledIO, since now CSR writing to integer register file might be blocked (by arbiter) */ + val out = DecoupledIO(new NewCSROutput) + val accelerator = Output(new examples.core.xiangshan.native.Context) + + val status = Output(new Bundle { + val privState = new PrivState + val interrupt = Bool() + val wfiEvent = Bool() + val satp = ValidIO(UInt(SatpMode.getWidth.W)) + val vsatp = ValidIO(UInt(SatpMode.getWidth.W)) + + // fp + val fpState = new Bundle { + val off = Bool() + val frm = Frm() + } + + // vec + val vecState = new Bundle { + val vstart = Vstart() + val vxsat = Vxsat() + val vxrm = Vxrm() + val vcsr = UInt(XLEN.W) + val vl = Vl() + val vtype = UInt(XLEN.W) + val vlenb = UInt(XLEN.W) + val off = Bool() + } + + // debug + val debugMode = Bool() + val singleStepFlag = Bool() + // trigger + val frontendTrigger = new FrontendTdataDistributeIO() + val memTrigger = new MemTdataDistributeIO() + // Instruction fetch address translation type + val instrAddrTransType = new AddrTransType + // trace + val traceCSR = Output(new TraceCSR) + // custom + val custom = new CSRCustomState + val criticalErrorState = Bool() + }) + + // tlb + val tlb = Output(new Bundle { + val satpASIDChanged = Bool() + val vsatpASIDChanged = Bool() + val hgatpVMIDChanged = Bool() + val satp = new SatpBundle + val vsatp = new SatpBundle + val hgatp = new HgatpBundle + val mbmc = new MbmcBundle + val mxr = Bool() + val sum = Bool() + val vmxr = Bool() + val vsum = Bool() + val spvp = Bool() + val imode = UInt(2.W) + val dmode = UInt(2.W) + val debug = Bool() + val dvirt = Bool() + val mPBMTE = Bool() + val hPBMTE = Bool() + + val pmm = new Bundle { + val mseccfg = UInt(2.W) + val menvcfg = UInt(2.W) + val henvcfg = UInt(2.W) + val hstatus = UInt(2.W) + val senvcfg = UInt(2.W) + } + + }) + + val toDecode = new CSRToDecode + + val fetchMalTval = Input(UInt(XLEN.W)) + + val oldPrivSate = Input(new PrivState) + val oldSatpMode = Input(UInt(SatpMode.getWidth.W)) + val oldVsatpMode = Input(UInt(SatpMode.getWidth.W)) + + val distributedWenLegal = Output(Bool()) + }) + + val toAIA = IO(Output(new CSRToAIABundle)) + val fromAIA = IO(Flipped(Output(new AIAToCSRBundle))) + + dontTouch(toAIA) + dontTouch(fromAIA) + dontTouch(io.fromTop.clintTime) + + /* Alias of input valid/ready */ + val valid = io.in.valid + + /* Alias of input signals */ + val wen = io.in.bits.wen && valid + val addr = io.in.bits.addr + val wdata = io.in.bits.wdata + + val ren = io.in.bits.ren && valid + val raddr = io.in.bits.addr + + // flush + val redirectFlush = io.in.bits.redirectFlush + + val hasTrap = io.fromRob.trap.valid + val trapVec = io.fromRob.trap.bits.trapVec + val trapPC = io.fromRob.trap.bits.pc + val trapPCGPA = io.fromRob.trap.bits.pcGPA + val trapIsInterrupt = io.fromRob.trap.bits.isInterrupt + val trapIsCrossPageIPF = io.fromRob.trap.bits.crossPageIPFFix + val trigger = io.fromRob.trap.bits.trigger + val singleStep = io.fromRob.trap.bits.singleStep + val trapIsHls = io.fromRob.trap.bits.isHls + val trapIsFetchMalAddr = io.fromRob.trap.bits.isFetchMalAddr + val trapIsFetchBkpt = io.fromRob.trap.bits.isFetchBkpt + val trapIsForVSnonLeafPTE = io.fromRob.trap.bits.isForVSnonLeafPTE + val trapIsSatpFlushFirstFetchFault = io.fromRob.trap.bits.satpFlushFirstFetchFault + + val oldPrivState = io.oldPrivSate + val oldSatpMode = io.oldSatpMode + val oldVsatpMode = io.oldVsatpMode + + // debug_intrrupt + val debugIntrEnable = RegInit(true.B) // debug interrupt will be handle only when debugIntrEnable + val debugIntr = platformIRP.debugIP && debugIntrEnable + + // CSR Privilege State + val PRVM = RegInit(PrivMode(1, 0), PrivMode.M) + val V = RegInit(VirtMode(0), VirtMode.Off) + val debugMode = RegInit(false.B) + private val nextV = WireInit(VirtMode(0), VirtMode.Off) + V := nextV + // dcsr stopcount + val debugModeStopCountNext = debugMode && dcsr.regOut.STOPCOUNT + val debugModeStopTimeNext = debugMode && dcsr.regOut.STOPTIME + val debugModeStopCount = RegNext(debugModeStopCountNext) + val unprivCountUpdate = !debugModeStopCount && debugModeStopCountNext + + val criticalErrorStateInCSR = Wire(Bool()) + val criticalErrorState = RegEnable(true.B, false.B, io.fromTop.criticalErrorState || criticalErrorStateInCSR) + // When cetrig is 1, resuming from DebugMode following an entry due to a critical + // error will result in an immediate re-entry into Debug Mode due to the critical error. + // Ensure that dpc remains unchanged when criticalErrorState causes a re-entry into dmode, + // since the PC fetched from pcmem for updating dpc is random in this case. + val holdDpc = RegEnable(criticalErrorState && dcsr.regOut.CETRIG, false.B, dretEvent.valid) + + private val privState = Wire(new PrivState) + privState.PRVM := PRVM + privState.V := V + + private val isModeM = privState.isModeM + private val (isModeHS, isModeHU) = (privState.isModeHS, privState.isModeHU) + private val (isModeVS, isModeVU) = (privState.isModeVS, privState.isModeVU) + + val permitMod = Module(new CSRPermitModule) + val sstcIRGen = Module(new SstcInterruptGen) + val commidIdMod = Module(new CommitIDModule(40, hartIdLen)) + + commidIdMod.io.hartId := io.fromTop.hartId + val gitCommitSHA = WireInit(commidIdMod.io.commitID) + val gitDirty = WireInit(commidIdMod.io.dirty) + dontTouch(gitCommitSHA) + dontTouch(gitDirty) + + private val wenLegal = permitMod.io.out.hasLegalWen + + val legalSret = permitMod.io.out.hasLegalSret + val legalMret = permitMod.io.out.hasLegalMret + val legalMNret = permitMod.io.out.hasLegalMNret + val legalDret = permitMod.io.out.hasLegalDret + + private val wenLegalReg = GatedValidRegNext(wenLegal) + + var csrRwMap: SeqMap[Int, (CSRAddrWriteBundle[_], UInt)] = + machineLevelCSRMap ++ + supervisorLevelCSRMap ++ + hypervisorCSRMap ++ + virtualSupervisorCSRMap ++ + unprivilegedCSRMap ++ + debugCSRMap ++ + aiaCSRMap ++ + customCSRMap ++ + indCSRMap ++ + pmpCSRMap ++ + pmaCSRMap + + val csrMods: Seq[CSRModule[_]] = + machineLevelCSRMods ++ + supervisorLevelCSRMods ++ + hypervisorCSRMods ++ + virtualSupervisorCSRMods ++ + unprivilegedCSRMods ++ + debugCSRMods ++ + aiaCSRMods ++ + indCSRMods ++ + customCSRMods ++ + pmpCSRMods ++ + pmaCSRMods + + var csrOutMap: SeqMap[Int, UInt] = + machineLevelCSROutMap ++ + supervisorLevelCSROutMap ++ + hypervisorCSROutMap ++ + virtualSupervisorCSROutMap ++ + unprivilegedCSROutMap ++ + debugCSROutMap ++ + aiaCSROutMap ++ + indCSROutMap ++ + customCSROutMap ++ + pmpCSROutMap ++ + pmaCSROutMap + + // interrupt + val nmip = RegInit(new NonMaskableIRPendingBundle, (new NonMaskableIRPendingBundle).init) + nmip.NMI_43 := nonMaskableIRP.NMI_43 + nmip.NMI_31 := nonMaskableIRP.NMI_31 + + val intrMod = Module(new InterruptFilter) + intrMod.io.in.privState := privState + intrMod.io.in.mstatusMIE := mstatus.regOut.MIE.asBool + intrMod.io.in.sstatusSIE := mstatus.regOut.SIE.asBool + intrMod.io.in.vsstatusSIE := vsstatus.regOut.SIE.asBool + intrMod.io.in.mip := mip.rdataFields + intrMod.io.in.mie := mie.regOut + intrMod.io.in.mideleg := mideleg.regOut + intrMod.io.in.sip := sip.regOut + intrMod.io.in.sie := sie.regOut + intrMod.io.in.hip := hip.regOut + intrMod.io.in.hie := hie.regOut + intrMod.io.in.hideleg := hideleg.regOut + intrMod.io.in.vsip := vsip.regOut + intrMod.io.in.vsie := vsie.regOut + intrMod.io.in.hvictl := hvictl.regOut + intrMod.io.in.hstatus := hstatus.regOut + intrMod.io.in.mtopei := mtopei.regOut + intrMod.io.in.stopei := stopei.regOut + intrMod.io.in.vstopei := vstopei.regOut + intrMod.io.in.hviprio1 := hviprio1.regOut + intrMod.io.in.hviprio2 := hviprio2.regOut + intrMod.io.in.miprios := Cat(miregiprios.map(_.rdata).reverse) + intrMod.io.in.hsiprios := Cat(siregiprios.map(_.rdata).reverse) + intrMod.io.in.mnstatusNMIE := mnstatus.regOut.NMIE.asBool + intrMod.io.in.nmi := nmip.asUInt.orR + intrMod.io.in.nmiVec := nmip.asUInt + intrMod.io.in.debugMode := debugMode + intrMod.io.in.debugIntr := debugIntr + intrMod.io.in.dcsr := dcsr.regOut + intrMod.io.in.platform.meip := platformIRP.MEIP + intrMod.io.in.platform.seip := platformIRP.SEIP + intrMod.io.in.fromAIA.meip := fromAIA.meip + intrMod.io.in.fromAIA.seip := fromAIA.seip + intrMod.io.in.fromAIA.notice_pending := fromAIA.notice_pending + intrMod.io.in.mvienSEIE := mvien.regOut.SEIE.asBool + intrMod.io.in.mvipSEIP := mvip.regOut.SEIP.asBool + + val intrVec = RegEnable(intrMod.io.out.interruptVec.bits, 0.U, intrMod.io.out.interruptVec.valid) + val debug = RegEnable(intrMod.io.out.debug, false.B, intrMod.io.out.interruptVec.valid) + val nmi = intrMod.io.out.nmi + val virtualInterruptIsHvictlInject = intrMod.io.out.virtualInterruptIsHvictlInject + val irToHS = RegEnable(intrMod.io.out.irToHS, false.B, intrMod.io.out.interruptVec.valid) + val irToVS = RegEnable(intrMod.io.out.irToVS, false.B, intrMod.io.out.interruptVec.valid) + + val trapHandleMod = Module(new TrapHandleModule) + + trapHandleMod.io.in.trapInfo.valid := hasTrap + trapHandleMod.io.in.trapInfo.bits.trapVec := trapVec.asUInt + trapHandleMod.io.in.trapInfo.bits.intrVec := intrVec + trapHandleMod.io.in.trapInfo.bits.isInterrupt := trapIsInterrupt + trapHandleMod.io.in.trapInfo.bits.irToHS := irToHS + trapHandleMod.io.in.trapInfo.bits.irToVS := irToVS + trapHandleMod.io.in.privState := privState + trapHandleMod.io.in.mstatus := mstatus.regOut + trapHandleMod.io.in.sstatus := mstatus.sstatus + trapHandleMod.io.in.vsstatus := vsstatus.regOut + trapHandleMod.io.in.mnstatus := mnstatus.regOut + trapHandleMod.io.in.mideleg := mideleg.regOut + trapHandleMod.io.in.medeleg := medeleg.regOut + trapHandleMod.io.in.hideleg := hideleg.regOut + trapHandleMod.io.in.hedeleg := hedeleg.regOut + trapHandleMod.io.in.mvien := mvien.regOut + trapHandleMod.io.in.hvien := hvien.regOut + trapHandleMod.io.in.mtvec := mtvec.regOut + trapHandleMod.io.in.stvec := stvec.regOut + trapHandleMod.io.in.vstvec := vstvec.regOut + trapHandleMod.io.in.trapInfo.bits.singleStep := hasTrap && !trapIsInterrupt && singleStep + + val entryPrivState = trapHandleMod.io.out.entryPrivState + val entryDebugMode = WireInit(false.B) + val dbltrpToMN = trapHandleMod.io.out.dbltrpToMN + val hasDTExcp = trapHandleMod.io.out.hasDTExcp + + val NumPMPReal = p(PMParameKey).NumPMPReal + io.accelerator.satp := satp.regOut.asUInt + io.accelerator.privilege := io.tlb.dmode + io.accelerator.sum := io.tlb.sum + io.accelerator.mxr := io.tlb.mxr + io.accelerator.virt := io.tlb.dvirt + for (i <- 0 until p(PMParameKey).NumPMP) { + val cfg = if (i < NumPMPReal) pmpcfgs(i).rdata(7, 0) else 0.U(8.W) + val nextTor = if (i + 1 < NumPMPReal) pmpcfgs(i + 1).rdata(4, 3) === 1.U else false.B + io.accelerator.pmp(i).cfg := cfg + io.accelerator.pmp(i).addr := Mux(cfg(4, 3) =/= 0.U || nextTor, pmpaddr(i).rdata(45, 0), 0.U) + } + + // PMP + val pmpEntryMod = Module(new PMPEntryHandleModule) + pmpEntryMod.io.in.pmpCfg := pmpcfgs.map(_.regOut.asInstanceOf[PMPCfgBundle]) + pmpEntryMod.io.in.ren := ren + pmpEntryMod.io.in.wen := wenLegalReg + pmpEntryMod.io.in.addr := addr + pmpEntryMod.io.in.wdata := wdata + + // PMA + val pmaEntryMod = Module(new PMAEntryHandleModule) + pmaEntryMod.io.in.pmaCfg := pmacfgs.map(_.regOut.asInstanceOf[PMACfgBundle]) + pmaEntryMod.io.in.ren := ren + pmaEntryMod.io.in.wen := wenLegalReg + pmaEntryMod.io.in.addr := addr + pmaEntryMod.io.in.wdata := wdata + + // Todo: all wen and wdata of CSRModule assigned in this for loop + for ((id, (wBundle, _)) <- csrRwMap) { + if (vsMapS.contains(id)) { + // VS access CSR by S: privState.isModeVS && addrMappedToVS === sMapVS(id).U + wBundle.wen := wenLegalReg && ((isModeVS && addr === vsMapS(id).U) || (!isModeVS && addr === id.U)) + wBundle.wdata := wdata + } else if (sMapVS.contains(id)) { + wBundle.wen := wenLegalReg && !isModeVS && addr === id.U + wBundle.wdata := wdata + } else { + wBundle.wen := wenLegalReg && addr === id.U + wBundle.wdata := wdata + } + } + + private val writeFpLegal = permitMod.io.out.hasLegalWriteFcsr + private val writeVecLegal = permitMod.io.out.hasLegalWriteVcsr + + permitMod.io.in.csrAccess.ren := ren && valid + permitMod.io.in.csrAccess.wen := wen + permitMod.io.in.csrAccess.addr := addr + + permitMod.io.in.privState := privState + permitMod.io.in.debugMode := debugMode + + permitMod.io.in.xRet.mnret := io.in.bits.mnret && valid + permitMod.io.in.xRet.mret := io.in.bits.mret && valid + permitMod.io.in.xRet.sret := io.in.bits.sret && valid + permitMod.io.in.xRet.dret := io.in.bits.dret && valid + + permitMod.io.in.status.tsr := mstatus.regOut.TSR.asBool + permitMod.io.in.status.vtsr := hstatus.regOut.VTSR.asBool + + permitMod.io.in.status.tvm := mstatus.regOut.TVM.asBool + permitMod.io.in.status.vtvm := hstatus.regOut.VTVM.asBool + + permitMod.io.in.status.vgein := hstatus.regOut.VGEIN.asUInt + + permitMod.io.in.xcounteren.mcounteren := mcounteren.rdata + permitMod.io.in.xcounteren.hcounteren := hcounteren.rdata + permitMod.io.in.xcounteren.scounteren := scounteren.rdata + + permitMod.io.in.xstateen.mstateen0 := mstateen0.rdata + permitMod.io.in.xstateen.mstateen1 := mstateen1.rdata + permitMod.io.in.xstateen.mstateen2 := mstateen2.rdata + permitMod.io.in.xstateen.mstateen3 := mstateen3.rdata + permitMod.io.in.xstateen.hstateen0 := hstateen0.rdata + permitMod.io.in.xstateen.hstateen1 := hstateen1.rdata + permitMod.io.in.xstateen.hstateen2 := hstateen2.rdata + permitMod.io.in.xstateen.hstateen3 := hstateen3.rdata + permitMod.io.in.xstateen.sstateen0 := sstateen0.rdata + + permitMod.io.in.xenvcfg.menvcfg := menvcfg.rdata + permitMod.io.in.xenvcfg.henvcfg := henvcfg.rdata + + permitMod.io.in.status.mstatusFSOff := mstatus.regOut.FS === ContextStatus.Off + permitMod.io.in.status.mstatusVSOff := mstatus.regOut.VS === ContextStatus.Off + permitMod.io.in.status.vsstatusFSOff := vsstatus.regOut.FS === ContextStatus.Off + permitMod.io.in.status.vsstatusVSOff := vsstatus.regOut.VS === ContextStatus.Off + + permitMod.io.in.aia.miselect := miselect.rdata + permitMod.io.in.aia.siselect := siselect.rdata + permitMod.io.in.aia.vsiselect := vsiselect.rdata + permitMod.io.in.aia.mvienSEIE := mvien.regOut.SEIE.asBool + permitMod.io.in.aia.hvictlVTI := hvictl.regOut.VTI.asBool + + sstcIRGen.i.stime.valid := time.updated + sstcIRGen.i.stime.bits := time.stime + sstcIRGen.i.vstime.valid := time.updated + sstcIRGen.i.vstime.bits := time.vstime + sstcIRGen.i.stimecmp.wen := GatedValidRegNext(stimecmp.w.wen) + sstcIRGen.i.stimecmp.rdata := stimecmp.rdata + sstcIRGen.i.vstimecmp.wen := GatedValidRegNext(vstimecmp.w.wen) + sstcIRGen.i.vstimecmp.rdata := vstimecmp.rdata + sstcIRGen.i.menvcfg.wen := GatedValidRegNext(menvcfg.w.wen) + sstcIRGen.i.menvcfg.STCE := menvcfg.regOut.STCE.asBool + sstcIRGen.i.henvcfg.wen := GatedValidRegNext(henvcfg.w.wen) + sstcIRGen.i.henvcfg.STCE := henvcfg.regOut.STCE.asBool + sstcIRGen.i.htimedeltaWen := GatedValidRegNext(htimedelta.w.wen) + + miregiprios.foreach { mod => + mod.w.wen := mireg.w.wen && (miselect.regOut.ALL.asUInt === mod.addr.U) + mod.w.wdata := wdata + } + + siregiprios.foreach { mod => + mod.w.wen := sireg.w.wen && (siselect.regOut.ALL.asUInt === mod.addr.U) + mod.w.wdata := wdata + } + + siregiprios.foreach { mod => + mod match { + case m: HasSiprios => + m.mideleg := mideleg.regOut + m.mvien := mvien.regOut + case _ => + } + } + + mhartid.hartid := this.io.fromTop.hartId + mhartid.dmode := debugMode + mtvec.reset_mtvec.foreach(_ := this.io.fromTop.reset_mtvec.get) + + pmpcfgs.zipWithIndex.foreach { case (mod, i) => + mod.w.wen := wenLegalReg && (addr === (CSRs.pmpcfg0 + i / 8 * 2).U) + mod.w.wdata := pmpEntryMod.io.out.pmpCfgWData(8 * ((i % 8) + 1) - 1, 8 * (i % 8)) + } + + pmacfgs.zipWithIndex.foreach { case (mod, i) => + mod.w.wen := wenLegalReg && (addr === (CSRConst.PmacfgBase + i / 8 * 2).U) + mod.w.wdata := pmaEntryMod.io.out.pmaCfgWdata(8 * ((i % 8) + 1) - 1, 8 * (i % 8)) + } + + csrMods.foreach { mod => + mod match { + case m: HypervisorBundle => + m.hstatus := hstatus.regOut + case _ => + } + mod match { + case m: VirtualSupervisorBundle => + m.v := V.asUInt.asBool + m.hgatp := hgatp.regOut + case _ => + } + mod match { + case m: HasMachineDelegBundle => + m.mideleg := mideleg.regOut + m.medeleg := medeleg.regOut + case _ => + } + mod match { + case m: HasMachineCounterControlBundle => + m.mcountinhibit := mcountinhibit.regOut + case _ => + } + mod match { + case m: HasExternalInterruptBundle => + m.platformIRP := this.platformIRP + m.platformIRP.STIP := sstcIRGen.o.STIP + m.platformIRP.VSTIP := sstcIRGen.o.VSTIP + case _ => + } + mod match { + case m: HasRobCommitBundle => + // Todo: move RegNext from ROB to CSR + m.robCommit.instNum := io.fromRob.commit.instNum + m.robCommit.fflags := RegNextWithEnable(io.fromRob.commit.fflags) + m.robCommit.fsDirty := GatedValidRegNext(io.fromRob.commit.fsDirty) + m.robCommit.vsDirty := GatedValidRegNext(io.fromRob.commit.vsDirty) + m.robCommit.vxsat := RegNextWithEnable(io.fromRob.commit.vxsat) + m.robCommit.vtype := RegNextWithEnable(io.fromRob.commit.vtype) + m.robCommit.vl := RegNext(io.fromRob.commit.vl) + m.robCommit.vstart := RegNextWithEnable(io.fromRob.commit.vstart) + m.writeFCSR := writeFpLegal + m.writeVCSR := writeVecLegal + m.isVirtMode := V.asUInt.asBool + case _ => + } + mod match { + case m: TrapEntryDEventSinkBundle => + m.trapToD := trapEntryDEvent.out + case _ => + } + mod match { + case m: TrapEntryMEventSinkBundle => + m.trapToM := trapEntryMEvent.out + case _ => + } + mod match { + case m: TrapEntryMNEventSinkBundle => + m.trapToMN := trapEntryMNEvent.out + case _ => + } + mod match { + case m: TrapEntryHSEventSinkBundle => + m.trapToHS := trapEntryHSEvent.out + case _ => + } + mod match { + case m: TrapEntryVSEventSinkBundle => + m.trapToVS := trapEntryVSEvent.out + case _ => + } + mod match { + case m: MretEventSinkBundle => + m.retFromM := mretEvent.out + case _ => + } + mod match { + case m: MNretEventSinkBundle => + m.retFromMN := mnretEvent.out + case _ => + } + mod match { + case m: SretEventSinkBundle => + m.retFromS := sretEvent.out + case _ => + } + mod match { + case m: SretEventSDTSinkBundle => + m.retFromSSDT := sretEvent.outSDT + case _ => + } + mod match { + case m: DretEventSinkBundle => + m.retFromD := dretEvent.out + case _ => + } + mod match { + case m: HasAIABundle => + m.aiaToCSR.rdata.valid := fromAIA.rdata.valid + m.aiaToCSR.rdata.bits.data := fromAIA.rdata.bits.data + m.aiaToCSR.rdata.bits.illegal := fromAIA.rdata.bits.illegal + m.aiaToCSR.meip := fromAIA.meip + m.aiaToCSR.seip := fromAIA.seip + m.aiaToCSR.vseip := fromAIA.vseip + m.aiaToCSR.mtopei := fromAIA.mtopei + m.aiaToCSR.stopei := fromAIA.stopei + m.aiaToCSR.vstopei := fromAIA.vstopei + m.aiaToCSR.notice_pending := fromAIA.notice_pending + case _ => + } + mod match { + case m: HasInterruptFilterSink => + m.topIR.mtopi := intrMod.io.out.mtopi + m.topIR.stopi := intrMod.io.out.stopi + m.topIR.vstopi := intrMod.io.out.vstopi + case _ => + } + mod match { + case m: HasPMPAddrSink => + m.addrRData := pmpEntryMod.io.out.pmpAddrRData + case _ => + } + mod match { + case m: HasPMAAddrSink => + m.addrRData := pmaEntryMod.io.out.pmaAddrRData + case _ => + } + mod match { + case m: HasMHPMSink => + // cycle from mcycle + m.mHPM.cycle := mcycle.rdata + // time from clint + m.mHPM.time := io.fromTop.clintTime + // instret from minstret + m.mHPM.instret := minstret.rdata + // VS-Mode or VU-Mode + m.v := privState.isVirtual + m.nextV := nextV.isOneOf(VirtMode.On) + m.htimedelta := htimedelta.rdata + m.mHPM.hpmcounters.zip(mhpmcounters).map { + case (counter, mcounter) => counter := mcounter.rdata + } + case _ => + } + mod match { + case m: HasMachineEnvBundle => + m.menvcfg := menvcfg.regOut + case _ => + } + mod match { + case m: HasHypervisorEnvBundle => + m.menvcfg := menvcfg.regOut + case _ => + } + mod match { + case m: HasVirtualSupervisorEnvBundle => + m.henvcfg := henvcfg.regOut + m.menvcfg := menvcfg.regOut + case _ => + } + mod match { + case m: HasIpIeBundle => + m.mideleg := mideleg.regOut + m.mip := mip.rdata + m.mie := mie.regOut + m.mvip := mvip.regOut + m.mvien := mvien.regOut + m.hideleg := hideleg.regOut + m.hip := hip.regOut + m.hie := hie.regOut + m.hvien := hvien.regOut + m.hvip := hvip.regOut + m.sip := sip.regOut + m.sie := sie.regOut + m.vsip := vsip.regOut + m.vsie := vsie.regOut + m.hgeip := hgeip.regOut + m.hgeie := hgeie.regOut + m.hstatusVGEIN := hstatus.regOut.VGEIN + case _ => + } + mod match { + case m: HasMhpmeventOfBundle => + m.ofVec := VecInit(mhpmevents.map { event => + val mhpmevent = Wire(new MhpmeventBundle) + mhpmevent := event.rdata + mhpmevent.OF.asBool + }).asUInt + m.privState := privState + m.mcounteren := mcounteren.rdata + m.hcounteren := hcounteren.rdata + case _ => + } + mod match { + case m: HasStateenBundle => + m.fromMstateen0 := mstateen0.regOut + m.fromMstateen1 := mstateen1.regOut + m.fromMstateen2 := mstateen2.regOut + m.fromMstateen3 := mstateen3.regOut + m.fromHstateen0 := hstateen0.regOut + m.privState := privState + case _ => + } + mod match { + case m: HasDebugStopBundle => + m.debugModeStopCount := debugModeStopCount + m.debugModeStopTime := debugModeStopTimeNext + m.unprivCountUpdate := unprivCountUpdate + case _ => + } + mod match { + case m: HasNmipBundle => + m.nmip := nmip.asUInt.orR + case _ => + } + mod match { + case m: HasMachineFlushL2Bundle => + m.l2FlushDone := io.fromTop.l2FlushDone + case _ => + } + } + + trapEntryMNEvent.valid := ((hasTrap && nmi) || dbltrpToMN) && !entryDebugMode && !debugMode && mnstatus.regOut.NMIE + trapEntryMEvent.valid := hasTrap && entryPrivState.isModeM && !dbltrpToMN && !entryDebugMode && !debugMode && !nmi && mnstatus.regOut.NMIE + trapEntryHSEvent.valid := hasTrap && entryPrivState.isModeHS && !entryDebugMode && !debugMode && !nmi && mnstatus.regOut.NMIE + trapEntryVSEvent.valid := hasTrap && entryPrivState.isModeVS && !entryDebugMode && !debugMode && !nmi && mnstatus.regOut.NMIE + + Seq(trapEntryMEvent, trapEntryMNEvent, trapEntryHSEvent, trapEntryVSEvent, trapEntryDEvent).foreach { eMod => + eMod.in match { + case in: TrapEntryEventInput => + in.causeNO := trapHandleMod.io.out.causeNO + in.trapPc := trapPC + in.trapPcGPA := trapPCGPA // only used by trapEntryMEvent & trapEntryHSEvent + in.trapInst := io.trapInst + in.fetchMalTval := io.fetchMalTval + in.isCrossPageIPF := trapIsCrossPageIPF + in.isHls := trapIsHls + in.isFetchMalAddr := trapIsFetchMalAddr + in.satpFlushFirstFetchFault := trapIsSatpFlushFirstFetchFault + in.isFetchBkpt := trapIsFetchBkpt + in.trapIsForVSnonLeafPTE := trapIsForVSnonLeafPTE + in.hasDTExcp := hasDTExcp + + in.iMode.PRVM := Mux(trapIsSatpFlushFirstFetchFault, oldPrivState.PRVM, PRVM) + in.iMode.V := Mux(trapIsSatpFlushFirstFetchFault, oldPrivState.V, V) + // when NMIE is zero, force to behave as MPRV is zero + in.dMode.PRVM := Mux(mstatus.regOut.MPRV.asBool && mnstatus.regOut.NMIE.asBool, mstatus.regOut.MPP, PRVM) + in.dMode.V := V.asUInt.asBool || mstatus.regOut.MPRV && mnstatus.regOut.NMIE.asBool && (mstatus.regOut.MPP =/= PrivMode.M) && mstatus.regOut.MPV + + in.privState := privState + in.mstatus := mstatus.regOut + in.hstatus := hstatus.regOut + in.sstatus := mstatus.sstatus + in.vsstatus := vsstatus.regOut + in.pcFromXtvec := trapHandleMod.io.out.pcFromXtvec + + in.menvcfg := menvcfg.regOut + in.henvcfg := henvcfg.regOut + + in.satp := satp.regOut + in.vsatp := vsatp.regOut + in.hgatp := hgatp.regOut + in.oldSatp := Mux( + trapIsSatpFlushFirstFetchFault, + Cat(oldSatpMode, 0.U((XLEN - SatpMode.getWidth).W)).asTypeOf(in.oldSatp), + satp.regOut + ) + in.oldVsatp := Mux( + trapIsSatpFlushFirstFetchFault, + Cat(oldVsatpMode, 0.U((XLEN - SatpMode.getWidth).W)).asTypeOf(in.oldVsatp), + vsatp.regOut + ) + if (HasBitmapCheck) { + in.mbmc := mbmc.get.regOut + } else { + in.mbmc := DontCare + } + + in.memExceptionVAddr := io.fromMem.excpVA + in.memExceptionGPAddr := io.fromMem.excpGPA + in.memExceptionIsForVSnonLeafPTE := io.fromMem.excpIsForVSnonLeafPTE + + in.virtualInterruptIsHvictlInject := virtualInterruptIsHvictlInject + in.hvictlIID := hvictl.regOut.IID.asUInt + } + } + + mnretEvent.valid := legalMNret + mnretEvent.in match { + case in => + in.mstatus := mstatus.regOut + in.vsstatus := vsstatus.regOut + in.mnepc := mnepc.regOut + in.mnstatus := mnstatus.regOut + in.satp := satp.regOut + in.vsatp := vsatp.regOut + in.hgatp := hgatp.regOut + } + + mretEvent.valid := legalMret + mretEvent.in match { + case in => + in.mstatus := mstatus.regOut + in.vsstatus := vsstatus.regOut + in.mepc := mepc.regOut + in.satp := satp.regOut + in.vsatp := vsatp.regOut + in.hgatp := hgatp.regOut + } + + sretEvent.valid := legalSret + sretEvent.in match { + case in => + in.privState := privState + in.mstatus := mstatus.regOut + in.hstatus := hstatus.regOut + in.vsstatus := vsstatus.regOut + in.sepc := sepc.regOut + in.vsepc := vsepc.regOut + in.satp := satp.regOut + in.vsatp := vsatp.regOut + in.hgatp := hgatp.regOut + } + + dretEvent.valid := legalDret + dretEvent.in match { + case in => + in.dcsr := dcsr.regOut + in.dpc := dpc.regOut + in.mstatus := mstatus.regOut + in.vsstatus := vsstatus.regOut + in.satp := satp.regOut + in.vsatp := vsatp.regOut + in.hgatp := hgatp.regOut + } + + PRVM := MuxCase( + PRVM, + events.filter(_.out.isInstanceOf[EventUpdatePrivStateOutput]).map { + x => + x.out match { + case xx: EventUpdatePrivStateOutput => (xx.privState.valid -> xx.privState.bits.PRVM) + } + } + ) + + nextV := MuxCase( + V, + events.filter(_.out.isInstanceOf[EventUpdatePrivStateOutput]).map { + x => + x.out match { + case xx: EventUpdatePrivStateOutput => (xx.privState.valid -> xx.privState.bits.V) + } + } + ) + + debugMode := MuxCase( + debugMode, + Seq( + dretEvent.out.debugMode.valid -> dretEvent.out.debugMode.bits, + trapEntryDEvent.out.debugMode.valid -> trapEntryDEvent.out.debugMode.bits + ) + ) + + debugIntrEnable := MuxCase( + debugIntrEnable, + Seq( + dretEvent.out.debugIntrEnable.valid -> dretEvent.out.debugIntrEnable.bits, + trapEntryDEvent.out.debugIntrEnable.valid -> trapEntryDEvent.out.debugIntrEnable.bits + ) + ) + + // perf + val addrInPerfCnt = (wenLegal || ren) && ( + (addr >= CSRs.mcycle.U) && (addr <= CSRs.mhpmcounter31.U) || + (addr >= CSRs.cycle.U) && (addr <= CSRs.hpmcounter31.U) + ) || + ren && ( + (addr === CSRs.vstopi.U) || (addr === CSRs.vstopei.U) || + (addr === CSRs.stopi.U) || (addr === CSRs.stopei.U) || + (addr === CSRs.mtopi.U) || (addr === CSRs.mtopei.U) + ) + + val resetSatp = WireInit(false.B) + // flush + if (HasBitmapCheck) { + resetSatp := Cat( + Seq(satp, vsatp, hgatp, mbmc.get).map(_.addr.U === addr) + ).orR && wenLegalReg // write to satp will cause the pipeline be flushed + } else { + resetSatp := Cat( + Seq(satp, vsatp, hgatp).map(_.addr.U === addr) + ).orR && wenLegalReg // write to satp will cause the pipeline be flushed + } + + val floatStatusOnOff = mstatus.w.wen && ( + mstatus.w.wdataFields.FS === ContextStatus.Off && mstatus.regOut.FS =/= ContextStatus.Off || + mstatus.w.wdataFields.FS =/= ContextStatus.Off && mstatus.regOut.FS === ContextStatus.Off + ) || mstatus.wAliasSstatus.wen && ( + mstatus.wAliasSstatus.wdataFields.FS === ContextStatus.Off && mstatus.regOut.FS =/= ContextStatus.Off || + mstatus.wAliasSstatus.wdataFields.FS =/= ContextStatus.Off && mstatus.regOut.FS === ContextStatus.Off + ) || vsstatus.w.wen && ( + vsstatus.w.wdataFields.FS === ContextStatus.Off && vsstatus.regOut.FS =/= ContextStatus.Off || + vsstatus.w.wdataFields.FS =/= ContextStatus.Off && vsstatus.regOut.FS === ContextStatus.Off + ) + + val vectorStatusOnOff = mstatus.w.wen && ( + mstatus.w.wdataFields.VS === ContextStatus.Off && mstatus.regOut.VS =/= ContextStatus.Off || + mstatus.w.wdataFields.VS =/= ContextStatus.Off && mstatus.regOut.VS === ContextStatus.Off + ) || mstatus.wAliasSstatus.wen && ( + mstatus.wAliasSstatus.wdataFields.VS === ContextStatus.Off && mstatus.regOut.VS =/= ContextStatus.Off || + mstatus.wAliasSstatus.wdataFields.VS =/= ContextStatus.Off && mstatus.regOut.VS === ContextStatus.Off + ) || vsstatus.w.wen && ( + vsstatus.w.wdataFields.VS === ContextStatus.Off && vsstatus.regOut.VS =/= ContextStatus.Off || + vsstatus.w.wdataFields.VS =/= ContextStatus.Off && vsstatus.regOut.VS === ContextStatus.Off + ) + + val triggerFrontendChange = Wire(Bool()) + + val vstartChange = vstart.w.wen && ( + vstart.w.wdata === 0.U && vstart.regOut.vstart.asUInt =/= 0.U || + vstart.w.wdata =/= 0.U + ) + + // flush pipe when write frm and data > 4 or write fcsr and data[7:5] > 4 or write frm/fcsr and frm is reserved + val frmIsReserved = fcsr.frm(2) && fcsr.frm(1, 0).orR + val frmWdataReserved = fcsr.wAliasFfm.wdata(2) && fcsr.wAliasFfm.wdata(1, 0).orR + val fcsrWdataReserved = fcsr.w.wdata(7) && fcsr.w.wdata(6, 5).orR + val frmChange = fcsr.wAliasFfm.wen && (!frmIsReserved && frmWdataReserved || frmIsReserved && !frmWdataReserved) || + fcsr.w.wen && (!frmIsReserved && fcsrWdataReserved || frmIsReserved && !fcsrWdataReserved) + + val flushPipe = resetSatp || + triggerFrontendChange || floatStatusOnOff || vectorStatusOnOff || + vstartChange || frmChange + + /** + * Look up id in vsMapS and sMapVS. + * If id is in vsMapS, use vsMapS(id) when under VS mode, + * id under other modes + * Else If id is in sMapVS, use 0 when under VS mode, + * id under modes except VS + * Else, use id as read address + * Use read address to look up rdata in csrRwMap + */ + private val rdata = Mux1H(csrRwMap.map { case (id, (_, rdata)) => + if (vsMapS.contains(id)) { + ((isModeVS && addr === vsMapS(id).U) || !isModeVS && addr === id.U) -> rdata + } else if (sMapVS.contains(id)) { + (!isModeVS && addr === id.U) -> rdata + } else { + (raddr === id.U) -> rdata + } + }) + + private val rwMask = 0xc00 + private val csrOutMapFilter = csrOutMap.filter { case (id, _) => (id & rwMask) != rwMask } + + private val regOut = Mux1H(csrOutMapFilter.map { case (id, regOut) => + if (vsMapS.contains(id)) { + ((isModeVS && addr === vsMapS(id).U) || !isModeVS && addr === id.U) -> regOut + } else if (sMapVS.contains(id)) { + (!isModeVS && addr === id.U) -> regOut + } else { + (raddr === id.U) -> regOut + } + }) + + private val needTargetUpdate = + mnretEvent.out.targetPc.valid || mretEvent.out.targetPc.valid || sretEvent.out.targetPc.valid || dretEvent.out.targetPc.valid || + trapEntryMEvent.out.targetPc.valid || trapEntryMNEvent.out.targetPc.valid || trapEntryHSEvent.out.targetPc.valid || trapEntryVSEvent.out.targetPc.valid || trapEntryDEvent.out.targetPc.valid + + private val noCSRIllegal = (ren || wen) && Cat(csrRwMap.keys.toSeq.sorted.map(csrAddr => !(addr === csrAddr.U))).andR + + private val noCSRIllegalReg = RegEnable(noCSRIllegal, ren || wen) + + private val s_idle :: s_waitIMSIC :: s_finish :: Nil = Enum(3) + + /** the state machine of newCSR module */ + private val state = RegInit(s_idle) + + /** the next state of newCSR */ + private val stateNext = WireInit(state) + state := stateNext + + /** + * Asynchronous access operation of CSR. Check whether an access is asynchronous when read/write-enable is high. + * AIA registers are designed to be access asynchronously, so newCSR will wait for response. + */ + private val asyncAccess = (wen || ren) && !(permitMod.io.out.EX_II || permitMod.io.out.EX_VI) && ( + mireg.addr.U === addr && miselect.inIMSICRange || + sireg.addr.U === addr && ((!V.asUInt.asBool && siselect.inIMSICRange) || (V.asUInt.asBool && vsiselect.inIMSICRange)) || + vsireg.addr.U === addr && vsiselect.inIMSICRange + ) + + /** State machine of newCSR */ + switch(state) { + is(s_idle) { + when(valid && redirectFlush) { + stateNext := s_idle + }.elsewhen(valid && asyncAccess) { + stateNext := s_waitIMSIC + }.elsewhen(valid) { + stateNext := s_finish + } + } + is(s_waitIMSIC) { + when(redirectFlush) { + stateNext := s_idle + }.elsewhen(fromAIA.rdata.valid) { + when(io.out.ready) { + stateNext := s_idle + }.otherwise { + stateNext := s_finish + } + } + } + is(s_finish) { + when(redirectFlush || io.out.ready) { + stateNext := s_idle + } + } + } + + // Todo: check IMSIC EX_II and EX_VI + private val imsicIllegal = fromAIA.rdata.valid && fromAIA.rdata.bits.illegal + private val imsic_EX_II = imsicIllegal && !V.asUInt.asBool + private val imsic_EX_VI = imsicIllegal && V.asUInt.asBool + + /** Set io.in.ready when state machine is ready to receive a new request synchronously */ + io.in.ready := (state === s_idle) + + /** + * Valid signal of newCSR output. + * When in IDLE state, when input_valid is high, we set it. + * When in waitIMSIC state, and the next state is IDLE, we set it. + */ + + /** Data that have been read before,and should be stored because output not fired */ + val normalCSRValid = state === s_idle && valid && !asyncAccess + val waitIMSICValid = state === s_waitIMSIC && fromAIA.rdata.valid + val claimAIA = mtopei.w.wen | stopei.w.wen | vstopei.w.wen + + io.out.valid := (waitIMSICValid || state === s_finish) && !redirectFlush + io.out.bits.EX_II := DataHoldBypass( + Mux1H(Seq( + normalCSRValid -> (permitMod.io.out.EX_II || noCSRIllegal), + waitIMSICValid -> imsic_EX_II + )), + false.B, + normalCSRValid || waitIMSICValid + ) + io.out.bits.EX_VI := DataHoldBypass( + Mux1H(Seq( + normalCSRValid -> permitMod.io.out.EX_VI, + waitIMSICValid -> imsic_EX_VI + )), + false.B, + normalCSRValid || waitIMSICValid + ) + io.out.bits.flushPipe := flushPipe + + /** Prepare read data for output */ + io.out.bits.rData := DataHoldBypass( + Mux1H(Seq( + (io.in.fire || claimAIA) -> rdata, + fromAIA.rdata.valid -> fromAIA.rdata.bits.data + )), + 0.U(64.W), + io.in.fire || fromAIA.rdata.valid || claimAIA + ) + io.out.bits.regOut := regOut + io.out.bits.targetPc := DataHoldBypass( + Mux( + trapEntryDEvent.out.targetPc.valid, + trapEntryDEvent.out.targetPc.bits, + Mux1H(Seq( + mnretEvent.out.targetPc.valid -> mnretEvent.out.targetPc.bits, + mretEvent.out.targetPc.valid -> mretEvent.out.targetPc.bits, + sretEvent.out.targetPc.valid -> sretEvent.out.targetPc.bits, + dretEvent.out.targetPc.valid -> dretEvent.out.targetPc.bits, + trapEntryMEvent.out.targetPc.valid -> trapEntryMEvent.out.targetPc.bits, + trapEntryMNEvent.out.targetPc.valid -> trapEntryMNEvent.out.targetPc.bits, + trapEntryHSEvent.out.targetPc.valid -> trapEntryHSEvent.out.targetPc.bits, + trapEntryVSEvent.out.targetPc.valid -> trapEntryVSEvent.out.targetPc.bits + )) + ), + needTargetUpdate + ) + io.out.bits.targetPcUpdate := needTargetUpdate + io.out.bits.isPerfCnt := DataHoldBypass(addrInPerfCnt, false.B, io.in.fire) + + io.status.privState := privState + io.status.fpState.frm := fcsr.frm + io.status.fpState.off := mstatus.regOut.FS === ContextStatus.Off + io.status.vecState.vstart := vstart.rdata.asUInt + io.status.vecState.vxsat := vcsr.vxsat + io.status.vecState.vxrm := vcsr.vxrm + io.status.vecState.vcsr := vcsr.rdata.asUInt + io.status.vecState.vl := vl.rdata.asUInt + io.status.vecState.vtype := vtype.rdata.asUInt // Todo: check correct + io.status.vecState.vlenb := vlenb.rdata.asUInt + io.status.vecState.off := mstatus.regOut.VS === ContextStatus.Off + io.status.interrupt := intrMod.io.out.interruptVec.valid + io.status.wfiEvent := debugIntr || nmip.asUInt.orR || + (mtopi.regOut.IID.asUInt =/= 0.U) || + (stopi.regOut.IID.asUInt =/= 0.U) || + (vstopi.regOut.IID.asUInt =/= 0.U) + io.status.debugMode := debugMode + io.status.singleStepFlag := !debugMode && dcsr.regOut.STEP + io.status.satp.valid := satp.w.wen + io.status.satp.bits := satp.regOut.MODE.asUInt + io.status.vsatp.valid := vsatp.w.wen + io.status.vsatp.bits := vsatp.regOut.MODE.asUInt + + /** + * debug_begin + */ + val tdata1Selected = Wire(new Tdata1Bundle) + tdata1Selected := tdata1.rdata + val dmodeInSelectedTrigger = tdata1Selected.DMODE.asBool + val triggerCanWrite = dmodeInSelectedTrigger && debugMode || !dmodeInSelectedTrigger + val tdata1Update = tdata1.w.wen && triggerCanWrite + val tdata2Update = tdata2.w.wen && triggerCanWrite + + val tdata1Vec = tdata1RegVec.map { mod => + val tdata1Wire = Wire(new Tdata1Bundle) + tdata1Wire := mod.rdata + tdata1Wire + } + + val triggerCanRaiseBpExp = !(privState.isModeM && !mstatus.regOut.MIE || + medeleg.regOut.EX_BP && privState.isModeHS && !mstatus.sstatus.SIE || + medeleg.regOut.EX_BP && hedeleg.regOut.EX_BP && privState.isModeVS && !vsstatus.regOut.SIE) + + val debugMod = Module(new Debug) + debugMod.io.in.trapInfo.valid := hasTrap + debugMod.io.in.trapInfo.bits.trapVec := trapVec.asUInt + debugMod.io.in.trapInfo.bits.isDebugIntr := debug + debugMod.io.in.trapInfo.bits.isInterrupt := trapIsInterrupt + debugMod.io.in.trapInfo.bits.trigger := trigger + debugMod.io.in.trapInfo.bits.singleStep := singleStep + debugMod.io.in.trapInfo.bits.criticalErrorState := criticalErrorState + debugMod.io.in.privState := privState + debugMod.io.in.debugMode := debugMode + debugMod.io.in.dcsr := dcsr.regOut + debugMod.io.in.tselect := tselect.regOut + debugMod.io.in.tdata1Vec := tdata1Vec + debugMod.io.in.tdata1Selected := tdata1.rdata + debugMod.io.in.tdata2Selected := tdata2.rdata + debugMod.io.in.tdata1Update := tdata1Update + debugMod.io.in.tdata2Update := tdata2Update + debugMod.io.in.tdata1Wdata := wdata + debugMod.io.in.triggerCanRaiseBpExp := triggerCanRaiseBpExp + + entryDebugMode := debugMod.io.out.hasDebugTrap && !debugMode + + trapEntryDEvent.valid := entryDebugMode + trapEntryDEvent.in.hasDebugIntr := debugMod.io.out.hasDebugIntr + trapEntryDEvent.in.debugMode := debugMode + trapEntryDEvent.in.hasTrap := hasTrap + trapEntryDEvent.in.hasSingleStep := debugMod.io.out.hasSingleStep + trapEntryDEvent.in.triggerEnterDebugMode := debugMod.io.out.triggerEnterDebugMode + trapEntryDEvent.in.hasDebugEbreakException := debugMod.io.out.hasDebugEbreakException + trapEntryDEvent.in.breakPoint := debugMod.io.out.breakPoint + trapEntryDEvent.in.criticalErrorStateEnterDebug := debugMod.io.out.criticalErrorStateEnterDebug + trapEntryDEvent.in.holdDpc := holdDpc + + for (idx <- 0 until TriggerNum) { + val tdata1Pre = Wire(new Tdata1Bundle) + val mcontrol6Pre = Wire(new Mcontrol6) + tdata1Pre := (if (idx > 0) tdata1RegVec(idx - 1) else tdata1RegVec(idx)).rdata.asUInt + mcontrol6Pre := tdata1Pre.DATA.asUInt + val canWriteDmode = WireInit(false.B) + canWriteDmode := (if (idx > 0)(Mux( + mcontrol6Pre.CHAIN.asBool, + tdata1Pre.DMODE.asBool && tdata1Pre.TYPE.isLegal, + true.B + )) && debugMode + else debugMode).asBool + + val tdata1Next = Wire(new Tdata1Bundle) + tdata1Next := (if (idx < TriggerNum - 1) tdata1RegVec(idx + 1) else tdata1RegVec(idx)).rdata.asUInt + val dmodeNextTrigger = WireInit(false.B) + dmodeNextTrigger := (if (idx < TriggerNum - 1) tdata1Next.TYPE.isLegal && tdata1Next.DMODE.asBool else false.B) + + tdata1RegVec(idx) match { + case m: HasTriggerBundle => + m.canWriteDmode := canWriteDmode + m.chainable := debugMod.io.out.newTriggerChainIsLegal + m.dmodeNextTrigger := dmodeNextTrigger + case _ => + } + } + + tdata1RegVec.zip(tdata2RegVec).zipWithIndex.map { + case ((mod1, mod2), idx) => { + mod1.w.wen := tdata1Update && (tselect.rdata === idx.U) + mod1.w.wdata := wdata + mod2.w.wen := tdata2Update && (tselect.rdata === idx.U) + mod2.w.wdata := wdata + } + } + + triggerFrontendChange := debugMod.io.out.triggerFrontendChange + + io.status.frontendTrigger := debugMod.io.out.frontendTrigger + io.status.memTrigger := debugMod.io.out.memTrigger + + mcontext.fromHcontext <> hcontext.toMcontext + hcontext.fromMcontext <> mcontext.toHcontext + + /** + * debug_end + */ + + // trace + val privForTrace = Mux( + debugMode, + Priv.D, + Mux1H( + Seq(privState.isModeM, privState.isModeHS, privState.isModeVS, privState.isModeHU, privState.isModeVU), + Seq(Priv.M, Priv.HS, Priv.VS, Priv.HU, Priv.VU) + ) + ) + + val xret = legalDret || legalMNret || legalMret || legalSret + val currentPriv = privForTrace + val lastPriv = RegEnable(privForTrace, Priv.M, (xret || io.fromRob.trap.valid)) + + io.status.traceCSR.lastPriv := lastPriv + io.status.traceCSR.currentPriv := privForTrace + io.status.traceCSR.cause := Mux1H( + Seq(privState.isModeM, privState.isModeHS, privState.isModeVS), + Seq(mcause.rdata, scause.rdata, vscause.rdata) + ) + io.status.traceCSR.tval := Mux1H( + Seq(privState.isModeM, privState.isModeHS, privState.isModeVS), + Seq(mtval.rdata, stval.rdata, vstval.rdata) + ) + io.status.traceCSR.mstatus := mstatus.regOut.asUInt + + /** + * perf_begin + * perf number: 29 (frontend 8, ctrlblock 8, memblock 8, huancun 5) + */ + val csrevents = mhpmevents.slice(24, 29).map(_.rdata) + + val hcEvents = Wire(Vec(numPCntHc * coreParams.L2NBanks, new PerfEvent)) + for (i <- 0 until numPCntHc * coreParams.L2NBanks) { + hcEvents(i) := io.perf.perfEventsHc(i) + } + + val hpmHc = HPerfMonitor(csrevents, hcEvents) + + val allPerfEvents = io.perf.perfEventsFrontend ++ + io.perf.perfEventsBackend ++ + io.perf.perfEventsLsu ++ + hpmHc.getPerf + + val countingEn = RegInit(0.U.asTypeOf(Vec(perfCntNum, Bool()))) + val ofFromPerfCntVec = Wire(Vec(perfCntNum, Bool())) + val lcofiReqVec = Wire(Vec(perfCntNum, Bool())) + + for (i <- 0 until perfCntNum) { + mhpmcounters(i) match { + case m: HasPerfCounterBundle => + m.countingEn := countingEn(i) + m.perf := allPerfEvents(i) + ofFromPerfCntVec(i) := m.toMhpmeventOF + case _ => + } + + mhpmevents(i) match { + case m: HasOfFromPerfCntBundle => + m.ofFromPerfCnt := ofFromPerfCntVec(i) + case _ => + } + + val mhpmevent = Wire(new MhpmeventBundle) + mhpmevent := mhpmevents(i).rdata + lcofiReqVec(i) := ofFromPerfCntVec(i) && !mhpmevent.OF.asBool + + countingEn(i) := (privState.isModeM && !mhpmevent.MINH) || + (privState.isModeHS && !mhpmevent.SINH) || + (privState.isModeHU && !mhpmevent.UINH) || + (privState.isModeVS && !mhpmevent.VSINH) || + (privState.isModeVU && !mhpmevent.VUINH) + } + + val lcofiReq = lcofiReqVec.asUInt.orR + mip match { + case m: HasLocalInterruptReqBundle => + m.lcofiReq := lcofiReq + case _ => + } + + /** + * perf_end + */ + + /** + * [[io.status.custom]] connection + */ + io.status.custom.pf_ctrl.l1I_pf_enable := spfctl.regOut.L1I_PF_ENABLE.asBool + io.status.custom.pf_ctrl.l2_pf_enable := spfctl.regOut.L2_PF_ENABLE.asBool + io.status.custom.pf_ctrl.l1D_pf_enable := spfctl.regOut.L1D_PF_ENABLE.asBool + io.status.custom.pf_ctrl.l1D_pf_train_on_hit := spfctl.regOut.L1D_PF_TRAIN_ON_HIT.asBool + io.status.custom.pf_ctrl.l1D_pf_enable_agt := spfctl.regOut.L1D_PF_ENABLE_AGT.asBool + io.status.custom.pf_ctrl.l1D_pf_enable_pht := spfctl.regOut.L1D_PF_ENABLE_PHT.asBool + io.status.custom.pf_ctrl.l1D_pf_active_threshold := spfctl.regOut.L1D_PF_ACTIVE_THRESHOLD.asUInt + io.status.custom.pf_ctrl.l1D_pf_active_stride := spfctl.regOut.L1D_PF_ACTIVE_STRIDE.asUInt + io.status.custom.pf_ctrl.l1D_pf_enable_stride := spfctl.regOut.L1D_PF_ENABLE_STRIDE.asBool + io.status.custom.pf_ctrl.l2_pf_store_only := spfctl.regOut.L2_PF_STORE_ONLY.asBool + io.status.custom.pf_ctrl.l2_pf_recv_enable := spfctl.regOut.L2_PF_RECV_ENABLE.asBool + io.status.custom.pf_ctrl.l2_pf_pbop_enable := spfctl.regOut.L2_PF_PBOP_ENABLE.asBool + io.status.custom.pf_ctrl.l2_pf_vbop_enable := spfctl.regOut.L2_PF_VBOP_ENABLE.asBool + io.status.custom.pf_ctrl.l2_pf_tp_enable := spfctl.regOut.L2_PF_TP_ENABLE.asBool + io.status.custom.pf_ctrl.l2_pf_delay_latency := spfctl.regOut.L2_PF_DELAY_LATENCY.asUInt + + io.status.custom.lvpred_disable := slvpredctl.regOut.LVPRED_DISABLE.asBool + io.status.custom.no_spec_load := slvpredctl.regOut.NO_SPEC_LOAD.asBool + io.status.custom.storeset_wait_store := slvpredctl.regOut.STORESET_WAIT_STORE.asBool + io.status.custom.storeset_no_fast_wakeup := slvpredctl.regOut.STORESET_NO_FAST_WAKEUP.asBool + io.status.custom.lvpred_timeout := slvpredctl.regOut.LVPRED_TIMEOUT.asUInt + + io.status.custom.bp_ctrl.ubtb_enable := sbpctl.regOut.UBTB_ENABLE.asBool + io.status.custom.bp_ctrl.btb_enable := sbpctl.regOut.BTB_ENABLE.asBool + io.status.custom.bp_ctrl.bim_enable := sbpctl.regOut.BIM_ENABLE.asBool + io.status.custom.bp_ctrl.tage_enable := sbpctl.regOut.TAGE_ENABLE.asBool + io.status.custom.bp_ctrl.sc_enable := sbpctl.regOut.SC_ENABLE.asBool + io.status.custom.bp_ctrl.ras_enable := sbpctl.regOut.RAS_ENABLE.asBool + io.status.custom.bp_ctrl.loop_enable := sbpctl.regOut.LOOP_ENABLE.asBool + + io.status.custom.sbuffer_timeout := smblockctl.regOut.SBUFFER_TIMEOUT.asUInt + io.status.custom.sbuffer_threshold := smblockctl.regOut.SBUFFER_THRESHOLD.asUInt + io.status.custom.ldld_vio_check_enable := smblockctl.regOut.LDLD_VIO_CHECK_ENABLE.asBool + io.status.custom.soft_prefetch_enable := smblockctl.regOut.SOFT_PREFETCH_ENABLE.asBool + io.status.custom.cache_error_enable := smblockctl.regOut.CACHE_ERROR_ENABLE.asBool + io.status.custom.uncache_write_outstanding_enable := smblockctl.regOut.UNCACHE_WRITE_OUTSTANDING_ENABLE.asBool + io.status.custom.hd_misalign_st_enable := smblockctl.regOut.HD_MISALIGN_ST_ENABLE.asBool + io.status.custom.hd_misalign_ld_enable := smblockctl.regOut.HD_MISALIGN_LD_ENABLE.asBool + + io.status.custom.fusion_enable := srnctl.regOut.FUSION_ENABLE.asBool + io.status.custom.wfi_enable := srnctl.regOut.WFI_ENABLE.asBool && (!io.status.singleStepFlag) && !debugMode + + io.status.custom.power_down_enable := mcorepwr.regOut.POWER_DOWN_ENABLE.asBool + io.status.custom.commit_stuck_check_enable := mcorepwr.regOut.COMMIT_STUCK_CHECK_ENABLE.asBool + + io.status.custom.flush_l2_enable := mflushpwr.regOut.FLUSH_L2_ENABLE.asBool + + io.status.instrAddrTransType.bare := privState.isModeM || + (!privState.isVirtual && satp.regOut.MODE === SatpMode.Bare) || + (privState.isVirtual && vsatp.regOut.MODE === SatpMode.Bare && hgatp.regOut.MODE === HgatpMode.Bare) + io.status.instrAddrTransType.sv39 := !privState.isModeM && !privState.isVirtual && satp.regOut.MODE === SatpMode.Sv39 || + privState.isVirtual && vsatp.regOut.MODE === SatpMode.Sv39 + io.status.instrAddrTransType.sv48 := !privState.isModeM && !privState.isVirtual && satp.regOut.MODE === SatpMode.Sv48 || + privState.isVirtual && vsatp.regOut.MODE === SatpMode.Sv48 + io.status.instrAddrTransType.sv39x4 := privState.isVirtual && vsatp.regOut.MODE === SatpMode.Bare && hgatp.regOut.MODE === HgatpMode.Sv39x4 + io.status.instrAddrTransType.sv48x4 := privState.isVirtual && vsatp.regOut.MODE === SatpMode.Bare && hgatp.regOut.MODE === HgatpMode.Sv48x4 + assert(PopCount(io.status.instrAddrTransType.asUInt) === 1.U, "Exactly one inst trans type should be asserted") + + private val csrAccess = wenLegalReg || RegNext(ren) + + private val imsicAddrValid = + csrAccess && addr === CSRs.mireg.U && miselect.inIMSICRange || + csrAccess && addr === CSRs.sireg.U && !isModeVS && siselect.inIMSICRange || + csrAccess && (addr === CSRs.sireg.U && isModeVS || addr === CSRs.vsireg.U) && vsiselect.inIMSICRange + + private val imsicAddr = Mux1H(Seq( + (csrAccess && addr === CSRs.mireg.U) -> miselect.rdata, + (csrAccess && addr === CSRs.sireg.U && !isModeVS) -> siselect.rdata, + (csrAccess && (addr === CSRs.sireg.U && isModeVS || addr === CSRs.vsireg.U)) -> vsiselect.rdata + )) + + private val imsicAddrPrivState = Mux1H(Seq( + (csrAccess && addr === CSRs.mireg.U) -> PrivState.ModeM, + (csrAccess && addr === CSRs.sireg.U && !isModeVS) -> PrivState.ModeHS, + (csrAccess && (addr === CSRs.sireg.U && isModeVS || addr === CSRs.vsireg.U)) -> PrivState.ModeVS + )) + + private val imsicWdataValid = + mireg.w.wen && miselect.inIMSICRange || + sireg.w.wen && siselect.inIMSICRange || + vsireg.w.wen && vsiselect.inIMSICRange + + toAIA.addr.valid := imsicAddrValid + toAIA.addr.bits.addr := imsicAddr + toAIA.addr.bits.prvm := imsicAddrPrivState.PRVM + toAIA.addr.bits.v := imsicAddrPrivState.V + + toAIA.wdata.valid := imsicWdataValid + toAIA.wdata.bits.op := RegNext(io.in.bits.op) + toAIA.wdata.bits.data := RegNext(io.in.bits.src) + toAIA.vgein := hstatus.regOut.VGEIN.asUInt + toAIA.mClaim := mtopei.w.wen + toAIA.sClaim := stopei.w.wen + toAIA.vsClaim := vstopei.w.wen + + // tlb + io.tlb.satpASIDChanged := GatedValidRegNext(satp.w.wen && satp.regOut.ASID =/= satp.w.wdataFields.ASID) + io.tlb.vsatpASIDChanged := GatedValidRegNext(vsatp.w.wen && vsatp.regOut.ASID =/= vsatp.w.wdataFields.ASID) + io.tlb.hgatpVMIDChanged := GatedValidRegNext(hgatp.w.wen && hgatp.regOut.VMID =/= hgatp.w.wdataFields.VMID) + io.tlb.satp := satp.rdata + io.tlb.vsatp := vsatp.rdata + io.tlb.hgatp := hgatp.rdata + if (HasBitmapCheck) { + io.tlb.mbmc := mbmc.get.rdata + } else { + io.tlb.mbmc := DontCare + } + io.tlb.mxr := mstatus.regOut.MXR.asBool + io.tlb.sum := mstatus.regOut.SUM.asBool + io.tlb.vmxr := vsstatus.regOut.MXR.asBool + io.tlb.vsum := vsstatus.regOut.SUM.asBool + io.tlb.spvp := hstatus.regOut.SPVP.asBool + + io.tlb.imode := PRVM.asUInt + // when NMIE is zero, force to behave as MPRV is zero + io.tlb.dmode := Mux( + (debugMode && dcsr.regOut.MPRVEN || !debugMode) && mstatus.regOut.MPRV && mnstatus.regOut.NMIE, + mstatus.regOut.MPP.asUInt, + PRVM.asUInt + ) + io.tlb.dvirt := Mux( + (debugMode && dcsr.regOut.MPRVEN || !debugMode) && mstatus.regOut.MPRV && mnstatus.regOut.NMIE && mstatus.regOut.MPP =/= PrivMode.M, + mstatus.regOut.MPV.asUInt, + V.asUInt + ) + io.tlb.debug := debugMode + io.tlb.mPBMTE := RegNext(menvcfg.regOut.PBMTE.asBool) + io.tlb.hPBMTE := RegNext(henvcfg.regOut.PBMTE.asBool) + io.tlb.pmm.mseccfg := RegNext(mseccfg.regOut.PMM.asUInt) + io.tlb.pmm.menvcfg := RegNext(menvcfg.regOut.PMM.asUInt) + io.tlb.pmm.henvcfg := RegNext(henvcfg.regOut.PMM.asUInt) + io.tlb.pmm.hstatus := RegNext(hstatus.regOut.HUPMM.asUInt) + io.tlb.pmm.senvcfg := RegNext(senvcfg.regOut.PMM.asUInt) + + io.toDecode.illegalInst.sfenceVMA := isModeHS && mstatus.regOut.TVM || isModeHU + io.toDecode.virtualInst.sfenceVMA := isModeVS && hstatus.regOut.VTVM || isModeVU + io.toDecode.illegalInst.sfencePart := isModeHU + io.toDecode.virtualInst.sfencePart := isModeVU + io.toDecode.illegalInst.hfenceGVMA := isModeHS && mstatus.regOut.TVM || isModeHU + io.toDecode.illegalInst.hfenceVVMA := isModeHU + io.toDecode.virtualInst.hfence := isModeVS || isModeVU + io.toDecode.illegalInst.hlsv := isModeHU && !hstatus.regOut.HU + io.toDecode.virtualInst.hlsv := isModeVS || isModeVU + io.toDecode.illegalInst.fsIsOff := mstatus.regOut.FS === ContextStatus.Off || (isModeVS || isModeVU) && vsstatus.regOut.FS === ContextStatus.Off + io.toDecode.illegalInst.vsIsOff := mstatus.regOut.VS === ContextStatus.Off || (isModeVS || isModeVU) && vsstatus.regOut.VS === ContextStatus.Off + io.toDecode.illegalInst.wfi := isModeHU || !isModeM && mstatus.regOut.TW + io.toDecode.virtualInst.wfi := isModeVS && !mstatus.regOut.TW && hstatus.regOut.VTW || isModeVU && !mstatus.regOut.TW + io.toDecode.illegalInst.wrs_nto := !isModeM && mstatus.regOut.TW + io.toDecode.virtualInst.wrs_nto := privState.V && !mstatus.regOut.TW && hstatus.regOut.VTW + io.toDecode.illegalInst.frm := frmIsReserved + // Ref: The RISC-V Instruction Set Manual Volume I - 20.5. Control and Status Register State + io.toDecode.illegalInst.cboZ := !isModeM && !menvcfg.regOut.CBZE || isModeHU && !senvcfg.regOut.CBZE + io.toDecode.virtualInst.cboZ := menvcfg.regOut.CBZE && ( + isModeVS && !henvcfg.regOut.CBZE || + isModeVU && !(henvcfg.regOut.CBZE && senvcfg.regOut.CBZE) + ) + io.toDecode.illegalInst.cboCF := !isModeM && !menvcfg.regOut.CBCFE || isModeHU && !senvcfg.regOut.CBCFE + io.toDecode.virtualInst.cboCF := menvcfg.regOut.CBCFE && ( + isModeVS && !henvcfg.regOut.CBCFE || + isModeVU && !(henvcfg.regOut.CBCFE && senvcfg.regOut.CBCFE) + ) + io.toDecode.illegalInst.cboI := + !isModeM && menvcfg.regOut.CBIE === EnvCBIE.Off || + isModeHU && senvcfg.regOut.CBIE === EnvCBIE.Off + io.toDecode.virtualInst.cboI := menvcfg.regOut.CBIE =/= EnvCBIE.Off && ( + isModeVS && henvcfg.regOut.CBIE === EnvCBIE.Off || + isModeVU && (henvcfg.regOut.CBIE === EnvCBIE.Off || senvcfg.regOut.CBIE === EnvCBIE.Off) + ) + io.toDecode.special.cboI2F := !io.toDecode.illegalInst.cboI && !io.toDecode.virtualInst.cboI && ( + menvcfg.regOut.CBIE === EnvCBIE.Flush && !isModeM || + senvcfg.regOut.CBIE === EnvCBIE.Flush && (isModeHU || isModeVU) || + henvcfg.regOut.CBIE === EnvCBIE.Flush && (isModeVS || isModeVU) + ) + + io.distributedWenLegal := wenLegalReg && !noCSRIllegalReg + io.status.criticalErrorState := criticalErrorState && !dcsr.regOut.CETRIG.asBool + + val criticalErrors = Seq( + ("csr_dbltrp_inMN", !mnstatus.regOut.NMIE && hasTrap && !(entryDebugMode || debugMode)) + ) + + criticalErrorStateInCSR := criticalErrors.map(criticalError => criticalError._2).reduce(_ || _).asBool + generateCriticalErrors() + + // --------------------------------------------------------------------------- + // CSR documentation export + // --------------------------------------------------------------------------- + + if (env.DumpCSR) { + dumpCSRDoc() + } + + // --------------------------------------------------------------------------- + // Difftest + // --------------------------------------------------------------------------- + + // Always instantiate basic difftest modules. + if (env.AlwaysBasicDiff || env.EnableDifftest) { + // Delay trap passed to difftest until VecExcpMod is not busy + val pendingTrap = RegInit(false.B) + when(hasTrap) { + pendingTrap := true.B + }.elsewhen(!io.fromVecExcpMod.busy) { + pendingTrap := false.B + } + + val hartId = io.fromTop.hartId + val trapValid = pendingTrap && !io.fromVecExcpMod.busy + val interrupt = trapHandleMod.io.out.causeNO.Interrupt.asBool + val trapNO = Mux( + virtualInterruptIsHvictlInject && interrupt, + hvictl.regOut.IID.asUInt, + trapHandleMod.io.out.causeNO.ExceptionCode.asUInt + ) + val hasNMI = nmi && hasTrap + val interruptNO = Mux(interrupt, trapNO, 0.U) + val exceptionNO = Mux(!interrupt, trapNO, 0.U) + val isSv39: Bool = + (isModeHS || isModeHU) && satp.regOut.MODE === SatpMode.Sv39 || + (isModeVS || isModeVU) && vsatp.regOut.MODE === SatpMode.Sv39 + val isSv48: Bool = + (isModeHS || isModeHU) && satp.regOut.MODE === SatpMode.Sv48 || + (isModeVS || isModeVU) && vsatp.regOut.MODE === SatpMode.Sv48 + val isBare = !isSv39 && !isSv48 + val sv39PC = SignExt(trapPC.take(39), XLEN) + val sv48PC = SignExt(trapPC.take(48), XLEN) + val barePC = ZeroExt(trapPC.take(PAddrBits), XLEN) + // When enable virtual memory, the higher bit should fill with the msb of address of Sv39/Sv48/Sv57 + val exceptionPC = Mux1H(Seq( + isSv39 -> sv39PC, + isSv48 -> sv48PC, + isBare -> barePC + )) + + val diffArchEvent = DifftestModule(new DiffArchEvent, delay = 3, dontCare = true) + diffArchEvent.coreid := hartId + diffArchEvent.valid := trapValid + diffArchEvent.interrupt := RegEnable(interruptNO, hasTrap) + diffArchEvent.exception := RegEnable(exceptionNO, hasTrap) + diffArchEvent.exceptionPC := RegEnable(exceptionPC, hasTrap) + diffArchEvent.hasNMI := RegEnable(hasNMI, hasTrap) + diffArchEvent.virtualInterruptIsHvictlInject := RegNext(virtualInterruptIsHvictlInject && interrupt) + diffArchEvent.irToHS := RegEnable(irToHS, hasTrap) + diffArchEvent.irToVS := RegEnable(irToVS, hasTrap) + if (env.EnableDifftest || env.FullBasicDiff) { + diffArchEvent.exceptionInst := RegEnable(io.fromRob.trap.bits.instr, hasTrap) + } + + val diffCriticalErrorEvent = DifftestModule(new DiffCriticalErrorEvent, delay = 4, dontCare = true) + diffCriticalErrorEvent.valid := io.status.criticalErrorState && trapValid + diffCriticalErrorEvent.coreid := hartId + diffCriticalErrorEvent.criticalError := io.status.criticalErrorState + + val diffCSRState = DifftestModule(new DiffCSRState) + diffCSRState.coreid := hartId + diffCSRState.privilegeMode := privState.PRVM.asUInt + diffCSRState.mstatus := mstatus.rdata.asUInt + diffCSRState.sstatus := mstatus.sstatus.asUInt + diffCSRState.mepc := mepc.rdata.asUInt + diffCSRState.sepc := sepc.rdata.asUInt + diffCSRState.mtval := mtval.rdata.asUInt + diffCSRState.stval := stval.rdata.asUInt + diffCSRState.mtvec := mtvec.rdata.asUInt + diffCSRState.stvec := stvec.rdata.asUInt + diffCSRState.mcause := mcause.rdata.asUInt + diffCSRState.scause := scause.rdata.asUInt + diffCSRState.satp := satp.rdata.asUInt + diffCSRState.mip := mip.rdata.asUInt + diffCSRState.mie := mie.rdata.asUInt + diffCSRState.mscratch := mscratch.rdata.asUInt + diffCSRState.sscratch := sscratch.rdata.asUInt + diffCSRState.mideleg := mideleg.rdata.asUInt + diffCSRState.medeleg := medeleg.rdata.asUInt + + val diffDebugMode = DifftestModule(new DiffDebugMode) + diffDebugMode.coreid := hartId + diffDebugMode.debugMode := debugMode + diffDebugMode.dcsr := dcsr.rdata.asUInt + diffDebugMode.dpc := dpc.rdata.asUInt + diffDebugMode.dscratch0 := dscratch0.rdata.asUInt + diffDebugMode.dscratch1 := dscratch1.rdata.asUInt + + val diffTriggerCSRState = DifftestModule(new DiffTriggerCSRState) + diffTriggerCSRState.coreid := hartId + diffTriggerCSRState.tselect := tselect.rdata + diffTriggerCSRState.tdata1 := tdata1.rdata + diffTriggerCSRState.tinfo := tinfo.rdata + + val diffVecCSRState = DifftestModule(new DiffVecCSRState) + diffVecCSRState.coreid := hartId + diffVecCSRState.vstart := vstart.rdata.asUInt + diffVecCSRState.vxsat := vcsr.vxsat.asUInt + diffVecCSRState.vxrm := vcsr.vxrm.asUInt + diffVecCSRState.vcsr := vcsr.rdata.asUInt + diffVecCSRState.vl := RegNext(io.fromRob.commit.vl) + diffVecCSRState.vtype := vtype.rdata.asUInt + diffVecCSRState.vlenb := vlenb.rdata.asUInt + + val diffFpCSRState = DifftestModule(new DiffFpCSRState) + diffFpCSRState.coreid := hartId + diffFpCSRState.fcsr := fcsr.rdata.asUInt + + val diffHCSRState = DifftestModule(new DiffHCSRState) + diffHCSRState.coreid := hartId + diffHCSRState.virtMode := privState.V.asBool + diffHCSRState.mtval2 := mtval2.rdata.asUInt + diffHCSRState.mtinst := mtinst.rdata.asUInt + diffHCSRState.hstatus := hstatus.rdata.asUInt + diffHCSRState.hideleg := hideleg.rdata.asUInt + diffHCSRState.hedeleg := hedeleg.rdata.asUInt + diffHCSRState.hcounteren := hcounteren.rdata.asUInt + diffHCSRState.htval := htval.rdata.asUInt + diffHCSRState.htinst := htinst.rdata.asUInt + diffHCSRState.hgatp := hgatp.rdata.asUInt + diffHCSRState.vsstatus := vsstatus.rdata.asUInt + diffHCSRState.vstvec := vstvec.rdata.asUInt + diffHCSRState.vsepc := vsepc.rdata.asUInt + diffHCSRState.vscause := vscause.rdata.asUInt + diffHCSRState.vstval := vstval.rdata.asUInt + diffHCSRState.vsatp := vsatp.rdata.asUInt + diffHCSRState.vsscratch := vsscratch.rdata.asUInt + + val platformIRPMeipChange = + !platformIRP.MEIP && RegNext(platformIRP.MEIP) || platformIRP.MEIP && !RegNext(platformIRP.MEIP) + val platformIRPMtipChange = + !platformIRP.MTIP && RegNext(platformIRP.MTIP) || platformIRP.MTIP && !RegNext(platformIRP.MTIP) + val platformIRPMsipChange = + !platformIRP.MSIP && RegNext(platformIRP.MSIP) || platformIRP.MSIP && !RegNext(platformIRP.MSIP) + val platformIRPSeipChange = + !platformIRP.SEIP && RegNext(platformIRP.SEIP) || platformIRP.SEIP && !RegNext(platformIRP.SEIP) + val platformIRPStipChange = + !sstcIRGen.o.STIP && RegNext(sstcIRGen.o.STIP) || sstcIRGen.o.STIP && !RegNext(sstcIRGen.o.STIP) + val platformIRPVseipChange = !platformIRP.VSEIP && RegNext(platformIRP.VSEIP) || + platformIRP.VSEIP && !RegNext(platformIRP.VSEIP) || + !hgeip.rdata.asUInt(hstatus.regOut.VGEIN.asUInt) && RegNext(hgeip.rdata.asUInt(hstatus.regOut.VGEIN.asUInt)) || + hgeip.rdata.asUInt(hstatus.regOut.VGEIN.asUInt) && !RegNext(hgeip.rdata.asUInt(hstatus.regOut.VGEIN.asUInt)) + val platformIRPVstipChange = + !sstcIRGen.o.VSTIP && RegNext(sstcIRGen.o.VSTIP) || sstcIRGen.o.VSTIP && !RegNext(sstcIRGen.o.VSTIP) + val fromAIAMeipChange = !fromAIA.meip && RegNext(fromAIA.meip) || fromAIA.meip && !RegNext(fromAIA.meip) + val fromAIASeipChange = !fromAIA.seip && RegNext(fromAIA.seip) || fromAIA.seip && !RegNext(fromAIA.seip) + val lcofiReqChange = !lcofiReq && RegNext(lcofiReq) || lcofiReq && !RegNext(lcofiReq) + + val diffNonRegInterruptPendingEvent = DifftestModule(new DiffNonRegInterruptPendingEvent) + diffNonRegInterruptPendingEvent.coreid := hartId + diffNonRegInterruptPendingEvent.valid := (platformIRPMeipChange || platformIRPMtipChange || platformIRPMsipChange || + platformIRPSeipChange || platformIRPStipChange || + platformIRPVseipChange || platformIRPVstipChange || + fromAIAMeipChange || fromAIASeipChange || + lcofiReqChange || RegNext(reset.asBool)) & !reset.asBool + diffNonRegInterruptPendingEvent.platformIRPMeip := platformIRP.MEIP + diffNonRegInterruptPendingEvent.platformIRPMtip := platformIRP.MTIP + diffNonRegInterruptPendingEvent.platformIRPMsip := platformIRP.MSIP + diffNonRegInterruptPendingEvent.platformIRPSeip := platformIRP.SEIP + diffNonRegInterruptPendingEvent.platformIRPStip := sstcIRGen.o.STIP + diffNonRegInterruptPendingEvent.platformIRPVseip := platformIRP.VSEIP || hgeip.rdata.asUInt( + hstatus.regOut.VGEIN.asUInt + ) + diffNonRegInterruptPendingEvent.platformIRPVstip := sstcIRGen.o.VSTIP + diffNonRegInterruptPendingEvent.fromAIAMeip := fromAIA.meip + diffNonRegInterruptPendingEvent.fromAIASeip := fromAIA.seip + diffNonRegInterruptPendingEvent.localCounterOverflowInterruptReq := mip.regOut.LCOFIP.asBool + + val diffMhpmeventOverflowEvent = DifftestModule(new DiffMhpmeventOverflowEvent) + diffMhpmeventOverflowEvent.coreid := hartId + diffMhpmeventOverflowEvent.valid := Cat(mhpmevents.zipWithIndex.map { case (event, i) => + !ofFromPerfCntVec(i) && RegNext(ofFromPerfCntVec(i)) || ofFromPerfCntVec(i) && !RegNext(ofFromPerfCntVec(i)) + }).orR + diffMhpmeventOverflowEvent.mhpmeventOverflow := VecInit( + mhpmevents.map(_.regOut.asInstanceOf[MhpmeventBundle].OF.asBool) + ).asUInt + + val mtopeiChange = RegNext(fromAIA.mtopei.asUInt) =/= fromAIA.mtopei.asUInt + val stopeiChange = RegNext(fromAIA.stopei.asUInt) =/= fromAIA.stopei.asUInt + val vstopeiChange = RegNext(hstatus.regOut.VGEIN.asUInt) =/= hstatus.regOut.VGEIN.asUInt + val hgeipChange = RegNext(fromAIA.vseip) =/= fromAIA.vseip + + val diffSyncAIAEvent = DifftestModule(new DiffSyncAIAEvent) + diffSyncAIAEvent.coreid := hartId + diffSyncAIAEvent.valid := mtopeiChange || stopeiChange || vstopeiChange || hgeipChange + diffSyncAIAEvent.mtopei := mtopei.rdata + diffSyncAIAEvent.stopei := stopei.rdata + diffSyncAIAEvent.vstopei := vstopei.rdata + diffSyncAIAEvent.hgeip := hgeip.rdata + + val diffCustomMflushpwr = DifftestModule(new DiffSyncCustomMflushpwrEvent) + diffCustomMflushpwr.coreid := hartId + diffCustomMflushpwr.valid := RegNext(io.fromTop.l2FlushDone) =/= io.fromTop.l2FlushDone + diffCustomMflushpwr.l2FlushDone := io.fromTop.l2FlushDone + } +} + +trait IpIeAliasConnect { + self: NewCSR with MachineLevel with SupervisorLevel with VirtualSupervisorLevel with HypervisorLevel => + + mip.fromMvip := mvip.toMip + mip.fromSip := sip.toMip + mip.fromVSip := vsip.toMip + mvip.fromMip := mip.toMvip + mvip.fromSip := sip.toMvip + mvip.fromVSip := vsip.toMvip + hvip.fromMip := mip.toHvip + hvip.fromHip := hip.toHvip + hvip.fromVSip := vsip.toHvip + + mie.fromHie := hie.toMie + mie.fromSie := sie.toMie + mie.fromVSie := vsie.toMie + sie.fromVSie := vsie.toSie +} + +object NewCSRMain extends App { + + val (config, firrtlOpts, firtoolOpts) = ArgParser.parse( + args :+ "--disable-always-basic-diff" :+ "--dump-fir" :+ "--fpga-platform" :+ "--target" :+ "verilog" + ) + + val defaultConfig = config.alterPartial({ + // Get XSCoreParams and pass it to the "small module" + case XSCoreParamsKey => config(XSTileKey).head + }) + + Generator.execute( + firrtlOpts :+ "--full-stacktrace" :+ "--target-dir" :+ "backend", + new NewCSR()(defaultConfig), + firtoolOpts + ) + + println("done") +} diff --git a/examples/prefab/cores/xiangshan/configs/Params.scala b/examples/prefab/cores/xiangshan/configs/Params.scala new file mode 100644 index 000000000..297468b90 --- /dev/null +++ b/examples/prefab/cores/xiangshan/configs/Params.scala @@ -0,0 +1,73 @@ +package examples.core.xiangshan.configs + +import buckyball.config.CorePlacement +import buckyball.components.xiangshan.{Params => NativeParams} +import framework.ip.bus.{axi4, chi => chiBus} +import framework.ip.bus.chi.rnf.RnfParams + +import framework.core.memory.access.preflight.configs.PmpParams +import framework.ip.memory.interlock.{Params => TrackingParam} +import framework.core.memory.access.mmu.configs.{Params => WalkerParam} +import framework.acc.configs.AccParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.memdomain.tomem.configs.{Params => TransportParam} + +case class Params( + chi: chiBus.Params, + axi: axi4.Params, + hartId: Int, + nodeId: Int, + homeId: Int, + resetVector: BigInt, + mtimeAddress: BigInt, + dmaCache: RnfParams, + permission: PmpParams, + tracking: TrackingParam, + walker: WalkerParam, + acc: Option[AccParam]) { + require(chi.nodeIdBits == 7 && chi.addressBits == 48 && chi.dataBits == 256) + require(axi.addressBits == 48 && axi.dataBits == 128 && axi.idBits == 4) + require(hartId >= 0 && hartId < 64 && nodeId > 0 && nodeId < homeId && homeId < 128) + require((mtimeAddress & 7) == 0) + require(permission.addressBits == 48 && permission.entries == 64) + require(dmaCache.nodeId != nodeId && dmaCache.chi == chi && dmaCache.homeId == homeId) + acc.foreach(a => require(!a.memDomain.sharedEnable && a.transport.axi == axi)) +} + +object Params { + + def fromProto(core: CorePlacement): Params = { + require(core.getThreadIndicesCount == 0) + val h = core + val native = core.getImplementation.unpack(classOf[NativeParams]) + val bus = chiBus.Params.fromProto(h.getMemory.getHome.getRequesters) + val acc = Option.when(core.hasAccelerator) { + val a = core.getAccelerator + AccParam( + FrontendParam.fromProto(a.getFrontend, core.getParam.getXLen, core.getHartId), + MemDomainParam.fromProto(a.getMem, core.getSharedMem, core.getVirtualBankCount, 1), + GpDomainParam.fromProto(a.getGpdomain), + BallDomainParam.fromProto(a.getBalldomain), + TransportParam.fromProto(a.getTransport) + ) + } + Params( + bus, + axi4.Params.fromProto(h.getMemory.getUncached.getAxi), + h.getHartId, + native.getNodeId, + native.getHomeId, + BigInt(java.lang.Long.toUnsignedString(h.getResetVector)), + BigInt(java.lang.Long.toUnsignedString(h.getMtimeAddress)), + RnfParams.fromProto(native.getDmaCache, bus), + PmpParams(48, 64), + TrackingParam.fromProto(h.getMemory.getDcache.getTracking), + WalkerParam(bus, h.getMemory.getAccess.getTranslationEntries), + acc + ) + } + +} diff --git a/examples/prefab/cores/xiangshan/configs/xiangshan_params.proto b/examples/prefab/cores/xiangshan/configs/xiangshan_params.proto new file mode 100644 index 000000000..01caec49e --- /dev/null +++ b/examples/prefab/cores/xiangshan/configs/xiangshan_params.proto @@ -0,0 +1,10 @@ +syntax = "proto2"; +import "hardware.proto"; +package buckyball.components.xiangshan; +option java_package = "buckyball.components.xiangshan"; +option java_multiple_files = true; +message Params { + required uint32 node_id = 1; + required uint32 home_id = 2; + required buckyball.config.RnfHardware dma_cache = 3; +} diff --git a/examples/prefab/cores/xiangshan/stack/CMakeLists.txt b/examples/prefab/cores/xiangshan/stack/CMakeLists.txt new file mode 100644 index 000000000..ed2655a76 --- /dev/null +++ b/examples/prefab/cores/xiangshan/stack/CMakeLists.txt @@ -0,0 +1,25 @@ +# ===----------------------------------------------------------------------=== +# The xiangshan core: a CPU core without hardware threads; jobs run on the core. +# +# bbdev stack --build "--design " +# +# builds it for each design that uses it, configured with DESIGN_CMAKE +# (generated from design.pb) and CORE. Products: +# execute.o runs jobs on the core itself; links into the device runtime +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(xiangshan-core LANGUAGES NONE) +include("${DESIGN_CMAKE}") + +set(RUNTIME "${DESIGN_REPO}/stack/runtime") +find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) + +add_custom_command( + OUTPUT execute.o + COMMAND "${LINUX_CC}" -O2 -fPIC -ffreestanding -fno-builtin -msmall-data-limit=0 + "-DCORE_SIGNATURE=${DESIGN_DEVICE_CORES_${DESIGN_CORE_HART_${CORE}}_SIGNATURE}ULL" "-I${RUNTIME}/scheduler" + -c "${RUNTIME}/execute/direct.c" -o execute.o + DEPENDS "${RUNTIME}/execute/direct.c" "${RUNTIME}/scheduler/runtime.h" + VERBATIM) + +add_custom_target(execute ALL DEPENDS execute.o) diff --git a/examples/prefab/cores/xiangshan/thirdparty/xiangshan b/examples/prefab/cores/xiangshan/thirdparty/xiangshan new file mode 160000 index 000000000..fe2288cb0 --- /dev/null +++ b/examples/prefab/cores/xiangshan/thirdparty/xiangshan @@ -0,0 +1 @@ +Subproject commit fe2288cb09361b65afa26be3f403520c4b496dbf diff --git a/examples/prefab/cores/xiangshan/workloads/ctests/CMakeLists.txt b/examples/prefab/cores/xiangshan/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..416956421 --- /dev/null +++ b/examples/prefab/cores/xiangshan/workloads/ctests/CMakeLists.txt @@ -0,0 +1 @@ +add_buckyball_ctest(memory_matrix.c) diff --git a/examples/prefab/cores/xiangshan/workloads/ctests/memory_matrix.c b/examples/prefab/cores/xiangshan/workloads/ctests/memory_matrix.c new file mode 100644 index 000000000..1b1d8e04b --- /dev/null +++ b/examples/prefab/cores/xiangshan/workloads/ctests/memory_matrix.c @@ -0,0 +1,47 @@ +#include + +static volatile int32_t a[32][32], b[32][16], c[32][16]; +static volatile uint64_t atomic_word; +static volatile uint64_t upgrade_word[8] __attribute__((aligned(64))) = {7}; + +int main(void) { + for (int i = 0; i < 32; ++i) + for (int k = 0; k < 32; ++k) + a[i][k] = (i * 3 + k * 5) % 13 - 6; + for (int k = 0; k < 32; ++k) + for (int j = 0; j < 16; ++j) + b[k][j] = (k * 7 + j * 2) % 11 - 5; + for (int i = 0; i < 32; ++i) + for (int j = 0; j < 16; ++j) { + int32_t sum = 0; + for (int k = 0; k < 32; ++k) + sum += a[i][k] * b[k][j]; + c[i][j] = sum; + } + for (int i = 0; i < 32; ++i) + for (int j = 0; j < 16; ++j) { + int32_t expected = 0; + for (int k = 0; k < 32; ++k) + expected += ((i * 3 + k * 5) % 13 - 6) * ((k * 7 + j * 2) % 11 - 5); + if (c[i][j] != expected) + return 2; + } + uint64_t old, value = 19; + __asm__ volatile("amoadd.d.aqrl %0, %2, (%1)" + : "=r"(old) + : "r"(&atomic_word), "r"(value) + : "memory"); + if (old != 0 || atomic_word != 19) + return 3; + if (upgrade_word[0] != 7) + return 5; + upgrade_word[0] = 23; + if (upgrade_word[0] != 23) + return 6; + uint64_t before, after; + __asm__ volatile("rdtime %0" : "=r"(before)); + __asm__ volatile("rdtime %0" : "=r"(after)); + if (before == 0 || after <= before) + return 4; + return 0; +} diff --git a/examples/prefab/threads/attention/arch/AttentionThread.scala b/examples/prefab/threads/attention/arch/AttentionThread.scala new file mode 100644 index 000000000..7c5b92bfd --- /dev/null +++ b/examples/prefab/threads/attention/arch/AttentionThread.scala @@ -0,0 +1,77 @@ +package examples.threads.attention + +import framework.thread.ant.Ant +import framework.core.tbus.ThreadFactory +import framework.core.configs.CoreParam +import sims.configs.SimParam +import framework.core.memory.access.preflight.configs.PmpParams +import framework.thread.configs.ThreadParam +import framework.ip.memory.interlock.{Params => TrackingParam} +import framework.core.memory.access.mmu.configs.{Params => WalkerParam} + +import framework.acc.debug.Debug + +import chisel3.experimental.hierarchy.Instance + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import framework.thread.tlink.{HasTLink, TLinkIO} + +@instantiable +class AttentionThread( + threadp: ThreadParam, + permissionp: PmpParams, + trackingp: TrackingParam, + walkerp: WalkerParam, + simp: SimParam, + traceId: Int, + sharedOwnerId: Int) + extends Module + with HasTLink { + val sharedPorts = threadp.acc.get.memDomain.sharedPorts + val diffTest = simp.diffTest + val accp = threadp.acc.get + val antp = threadp.ant.get + + val ant: Instance[Ant] = + Instantiate(new Ant(antp, permissionp)) + + val acc: Instance[Acc] = Instantiate(new Acc(accp, permissionp, trackingp, walkerp, simp)) + + val debug: Instance[Debug] = + Instantiate(new Debug(accp.frontend, accp.memDomain, accp.gpDomain, accp.ballDomain, simp, traceId, sharedOwnerId)) + + debug.io.ant := ant.io.trace + debug.io.acc <> acc.io.debug + + @public + val tlink = IO(new TLinkIO(threadp, permissionp, trackingp, walkerp, simp)) + + ant.io.task <> tlink.task + tlink.tss <> ant.io.shared + ant.io.rocc <> acc.io.rocc + tlink.memory <> acc.io.memory + tlink.ddr <> acc.io.ddr + tlink.shm <> acc.io.shm + acc.io.shared_bank_hashes.zip(tlink.sharedHashes).foreach { case (port, hashes) => port := hashes } +} + +object AttentionThread extends ThreadFactory { + + def instantiate( + corep: CoreParam, + threadp: ThreadParam, + simp: SimParam, + traceId: Int + ): TLinkIO = + Instantiate(new AttentionThread( + threadp, + corep.privileged.permission, + corep.memory.dcache.tracking, + corep.memory.memoryAccess.walker, + simp, + traceId, + corep.hartId + )).tlink + +} diff --git a/examples/prefab/threads/attention/arch/acc.scala b/examples/prefab/threads/attention/arch/acc.scala new file mode 100644 index 000000000..161174faa --- /dev/null +++ b/examples/prefab/threads/attention/arch/acc.scala @@ -0,0 +1,119 @@ +package examples.threads.attention + +import framework.acc.balldomain.bbus.generated.BallComponents +import sims.configs.SimParam +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.configs.AccParam +import framework.core.memory.access.preflight.configs.PmpParams +import framework.ip.memory.interlock.{Params => TrackingParam} +import framework.core.memory.access.mmu.configs.{Params => WalkerParam} +import framework.acc.debug.AcceleratorDebugPort +import framework.acc.frontend.RoCCIO +import framework.acc.frontend.Frontend +import framework.acc.memdomain.{MemDomain, PhysicalBankHash} +import framework.acc.gpdomain.GpDomain +import framework.acc.balldomain.BallDomain +import framework.acc.balldomain.bbus.BBus +import framework.core.memory.MemoryPort +import framework.thread.tlink.SharedLink +import framework.ip.bus.axi4 + +@instantiable +class Acc( + val accp: AccParam, + val permissionp: PmpParams, + val trackingp: TrackingParam, + val walkerp: WalkerParam, + val simp: SimParam) + extends Module { + val commandDataBits = accp.frontend.commandDataBits + val ownerId = accp.frontend.ownerId + val sharedPorts = accp.memDomain.sharedPorts + val diffTest = simp.diffTest + val frontp = accp.frontend + val memp = accp.memDomain + val gpp = accp.gpDomain + val ballp = accp.ballDomain + val transportp = accp.transport + + val sharedHashCount = if (memp.sharedEnable) memp.sharedBankNum else 0 + + @public + val io = IO(new Bundle { + val debug = new AcceleratorDebugPort(frontp, memp, gpp, ballp, simp) + + val rocc = new RoCCIO(commandDataBits, permissionp) + val ddr = new axi4.Port(transportp.axi) + val memory = new MemoryPort(trackingp, walkerp) + val shm = new SharedLink(frontp, memp) + + val shared_bank_hashes = Option.when(diffTest && memp.sharedEnable)( + Input(Vec(sharedHashCount, new PhysicalBankHash(frontp))) + ) + + }) + + // --- Instantiate domains --- + val frontend: Instance[Frontend] = Instantiate(new Frontend(frontp, memp, gpp, ballp, permissionp, simp)) + + val ballDomain: Instance[BallDomain] = + Instantiate(new BallDomain(frontp, memp, gpp, ballp, permissionp, BallComponents.factories)) + + val memDomain: Instance[MemDomain] = + Instantiate(new MemDomain(frontp, memp, gpp, transportp, permissionp, trackingp, walkerp, simp)) + + val gpDomain = + if (gpp.enable) Some(Instantiate(new GpDomain(frontp, memp, gpp))) + else None + + memDomain.io.shared_bank_hashes.foreach(_ := io.shared_bank_hashes.get) + + io.debug.frontend <> frontend.io.debug + io.debug.banks.zip(memDomain.io.bank_hashes).foreach { case (sink, source) => sink := source } + io.debug.trace.memory := memDomain.io.traceMemory + io.debug.trace.ballStarted := ballDomain.io.traceStarted + io.debug.trace.ballCompleted := ballDomain.io.traceCompleted + + io.rocc <> frontend.io.rocc + io.memory <> memDomain.io.memory + io.ddr <> memDomain.io.ddr + + // --- Frontend -> BallDomain --- + memDomain.io.ball <> ballDomain.io.memory + ballDomain.io.issue <> frontend.io.ballRequest + frontend.io.ballCompletion <> ballDomain.io.complete + + // --- Frontend -> MemDomain --- + memDomain.io.issue <> frontend.io.memoryRequest + frontend.io.memoryCompletion <> memDomain.io.complete + memDomain.io.mapping <> frontend.io.mappingUpdate + + if (gpp.enable) { + val controller = gpDomain.get + controller.io.command <> frontend.io.gpRequest.get + frontend.io.gpCompletion.get <> controller.io.complete + controller.io.image <> memDomain.io.image.get + memDomain.io.gpBanks.get <> controller.io.banks + memDomain.io.gpLayoutRequest.get <> controller.io.layoutRequest + controller.io.layoutResponse <> memDomain.io.gpLayoutResponse.get + ballDomain.io.kernel.get <> controller.io.ball + } + + memDomain.io.banks <> ballDomain.io.banks + memDomain.io.ballLayoutRequest <> ballDomain.io.layoutRequest + ballDomain.io.layoutResponse <> memDomain.io.ballLayoutResponse + + // --- Shared memory passthrough --- + io.shm.bank.requests <> memDomain.io.shared.bank.requests + io.shm.move <> memDomain.io.shared.move + io.shm.bank.local <> memDomain.io.shared.bank.local + io.shm.bank.config <> memDomain.io.shared.bank.config + io.shm.bank.mapping <> memDomain.io.shared.bank.mapping + + // --- Barrier passthrough --- + io.shm.barrierArrive := frontend.io.barrier.arrive + frontend.io.barrier.release := io.shm.barrierRelease + +} diff --git a/examples/prefab/threads/attention/configs/balldomains/default.toml b/examples/prefab/threads/attention/configs/balldomains/default.toml new file mode 100644 index 000000000..cc8647054 --- /dev/null +++ b/examples/prefab/threads/attention/configs/balldomains/default.toml @@ -0,0 +1,53 @@ +[[balls]] +id = 0 +name = "MxmmBall" +component = "mxmm" +param = "../../../../balls/mxmm/configs/default.toml" +port = { inBW = 2, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "MXMM_MXFP8", funct7 = 71 }, + { mnemonic = "MXMM_MXFP8_WINDOW", funct7 = 75 }, + { mnemonic = "MXMM_FMA32", funct7 = 72 }, + { mnemonic = "MXMM_F32", funct7 = 73 }, +] + +[[balls]] +id = 1 +name = "ToInt8Ball" +component = "toint8" +param = "../../../../balls/toint8/configs/default.toml" +port = { inBW = 1, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "QUANT_F32_TO_I8", funct7 = 51 }, + { mnemonic = "QUANT_I32_TO_I8", funct7 = 67 }, +] + +[[balls]] +id = 2 +name = "Int2FpBall" +component = "int2fp" +param = "../../../../balls/int2fp/configs/default.toml" +port = { inBW = 2, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "INT32_TO_FP32", funct7 = 68 }, +] + +[[balls]] +id = 3 +name = "TransposeBall" +component = "transpose" +param = "../../../../balls/transpose/configs/default.toml" +port = { inBW = 1, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "TRANSPOSE", funct7 = 49 }, +] + +[[balls]] +id = 4 +name = "MxquantBall" +component = "mxquant" +param = "../../../../balls/mxquant/configs/default.toml" +port = { inBW = 1, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "MXQUANT", funct7 = 54 }, +] diff --git a/examples/prefab/threads/attention/configs/cpu/default.toml b/examples/prefab/threads/attention/configs/cpu/default.toml new file mode 100644 index 000000000..c752c9500 --- /dev/null +++ b/examples/prefab/threads/attention/configs/cpu/default.toml @@ -0,0 +1,2 @@ +kind = "ant" +config = "../../../../../../arch/src/main/scala/framework/thread/ant/configs/default.toml" diff --git a/examples/prefab/threads/attention/configs/default.toml b/examples/prefab/threads/attention/configs/default.toml new file mode 100644 index 000000000..c081f963f --- /dev/null +++ b/examples/prefab/threads/attention/configs/default.toml @@ -0,0 +1,12 @@ +source = ".." +# Attention Thread Configuration (default) + +balldomain = "balldomains/default.toml" +memdomain = "memdomains/default.toml" +frontend = "frontends/default.toml" +gpdomain = "gpdomain/default.toml" +cpu = "cpu/default.toml" + +[compiler] +name = "attention" +component = "attention" diff --git a/examples/prefab/threads/attention/configs/frontends/default.toml b/examples/prefab/threads/attention/configs/frontends/default.toml new file mode 100644 index 000000000..0f78452a5 --- /dev/null +++ b/examples/prefab/threads/attention/configs/frontends/default.toml @@ -0,0 +1,9 @@ +bankAliasEnable = true +robEntries = 16 +rsOutOfOrderResponse = true +bankIdLen = 10 +maxPrivateVbank = 31 +sharedVbankBase = 32 +iterLen = 34 +subRobEnable = false +subRobDepth = 64 diff --git a/examples/cores/ffn/configs/rvv/default.toml b/examples/prefab/threads/attention/configs/gpdomain/default.toml similarity index 100% rename from examples/cores/ffn/configs/rvv/default.toml rename to examples/prefab/threads/attention/configs/gpdomain/default.toml diff --git a/examples/prefab/threads/attention/configs/memdomains/default.toml b/examples/prefab/threads/attention/configs/memdomains/default.toml new file mode 100644 index 000000000..bca02f7ec --- /dev/null +++ b/examples/prefab/threads/attention/configs/memdomains/default.toml @@ -0,0 +1,22 @@ +# Attention Thread Memory Domain Configuration (default) +# Private (per-thread) memory parameters. Shared memory lives in the core TOML. + +addrLen = 39 + +[bank] +layoutEnable = true +num = 20 +width = 128 +entries = 512 +maskLen = 16 +requestPorts = 7 + +[dma] +busWidth = 128 + +[mmio] +enable = false +bankNum = 16 +bankEntries = 64 +bankWidth = 128 +readWidth = 8 diff --git a/examples/prefab/threads/attention/stack/CMakeLists.txt b/examples/prefab/threads/attention/stack/CMakeLists.txt new file mode 100644 index 000000000..173381303 --- /dev/null +++ b/examples/prefab/threads/attention/stack/CMakeLists.txt @@ -0,0 +1,157 @@ +# ===----------------------------------------------------------------------=== +# The attention thread: a hardware thread that executes the Ball instruction streams +# its core issues (the engine's stream interpreter). +# +# bbdev stack --build "--design " +# +# builds it for each design that uses it, configured with DESIGN_CMAKE +# (generated from design.pb) and THREAD, the +# compiler target of this thread instance. Products, in the build directory: +# lib-compiler.so buddy-opt plugin: Buckyball dialect + this thread's Balls +# program.elf, program.bin the stream interpreter the hardware thread runs +# loader.o loads program.bin and one instruction batch into a +# thread; links into the engine +# libkernels.a, libkernels-stream.a, kernels.rename +# the kernels of this thread, see Kernels below +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(attention-thread LANGUAGES C CXX) +include("${DESIGN_CMAKE}") + +set(ENGINE "${DESIGN_REPO}/stack/engine") +# The banks, thread and Ball ISA of this thread, as compile definitions. +set(THREAD_DEFS ${DESIGN_TARGET_DEFS_${THREAD}}) +list(TRANSFORM THREAD_DEFS PREPEND "-D") +find_program(ELF_CC riscv64-unknown-elf-gcc REQUIRED) +find_program(ELF_OBJCOPY riscv64-unknown-elf-objcopy REQUIRED) +find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) + +# ===----------------------------------------------------------------------=== +# Compiler: the Buckyball dialect and passes with the Balls of this thread. +# ===----------------------------------------------------------------------=== +set(BUCKYBALL_BALLS ${DESIGN_TARGET_BALL_STACKS_${THREAD}}) +add_subdirectory("${DESIGN_REPO}/stack/compiler/midend" compiler) +add_library(${THREAD}-compiler MODULE) +target_link_libraries(${THREAD}-compiler PRIVATE buckyball-compiler) +set_target_properties(${THREAD}-compiler PROPERTIES + LINKER_LANGUAGE CXX LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") + +# ===----------------------------------------------------------------------=== +# Program: the engine's stream interpreter, freestanding at the start of the +# thread's code memory. +# ===----------------------------------------------------------------------=== +add_custom_command( + OUTPUT program.elf + COMMAND "${ELF_CC}" -O2 -march=rv64im -mabi=lp64 -mcmodel=medany + -ffreestanding -fno-builtin -fno-pic -fno-stack-protector -fno-jump-tables + -fno-asynchronous-unwind-tables -msmall-data-limit=0 -mno-relax + -nostdlib -nostartfiles -Wl,--no-relax "-Wl,-T,${ENGINE}/program/code.ld" + "-I${ENGINE}/include" ${THREAD_DEFS} + "${ENGINE}/program/start.S" "${ENGINE}/program/stream.c" -o program.elf + DEPENDS "${ENGINE}/program/stream.c" "${ENGINE}/program/instructions.inc" + "${ENGINE}/program/start.S" "${ENGINE}/program/code.ld" + VERBATIM) +add_custom_command( + OUTPUT program.bin + COMMAND "${ELF_OBJCOPY}" -O binary --only-section=.text program.elf program.bin + DEPENDS program.elf + VERBATIM) + +# ===----------------------------------------------------------------------=== +# Loader: stream_prepare.c embeds program.bin and registers as THREAD_SIGNATURE. +# ===----------------------------------------------------------------------=== +add_custom_command( + OUTPUT loader.o + COMMAND "${LINUX_CC}" -O2 -fPIC -ffreestanding -fno-builtin -msmall-data-limit=0 + "--embed-dir=${CMAKE_CURRENT_BINARY_DIR}" "-I${ENGINE}/include" ${THREAD_DEFS} + -c "${ENGINE}/program/stream_prepare.c" -o loader.o + DEPENDS "${ENGINE}/program/stream_prepare.c" program.bin + VERBATIM) + +add_custom_target(program ALL DEPENDS program.bin loader.o) + +# ===----------------------------------------------------------------------=== +# Kernels: the RVV kernels of the GP domain and the kernels of this thread's +# Balls, with the host code that launches them, built against this thread's +# ISA. Every exported symbol gets the suffix __${THREAD}, so the kernels of +# several threads link into one program; models apply kernels.rename to the +# code they compile for this thread. +# libkernels.a Ball instructions issued by the calling core +# libkernels-stream.a Ball instructions sent to the thread through the engine +# ===----------------------------------------------------------------------=== +find_program(LINUX_CXX riscv64-unknown-linux-gnu-g++ REQUIRED) +find_program(LINUX_AR riscv64-unknown-linux-gnu-ar REQUIRED) +find_program(LINUX_OBJCOPY riscv64-unknown-linux-gnu-objcopy REQUIRED) +set(RVV "${DESIGN_REPO}/stack/kernels/rvv") +add_subdirectory("${RVV}" rvv) + +set(KERNEL_SOURCES + "${RVV}/dispatch.cpp" + "${RVV}/matmul_dispatch.cpp" + "${RVV}/cache_dispatch.cpp" + "${RVV}/normalization_dispatch.cpp" + "${RVV}/pointwise_dispatch.cpp" + "${RVV}/activation_dispatch.cpp" + "${RVV}/flash_attention_dispatch.cpp" + "${RVV}/flash_attention_mxfp8_dispatch.cpp") +set(KERNEL_INCLUDES + "${DESIGN_REPO}/bb-tests/workloads/lib" + "${DESIGN_REPO}/stack/kernels" + "${DESIGN_REPO}/stack/runtime/scheduler" + "${DESIGN_REPO}/stack/runtime/services" + "${ENGINE}/include" + "${DESIGN_REPO}/stack/thirdparty/buddy-mlir/llvm/mlir/include/mlir/ExecutionEngine") +set(KERNEL_EMBED_DIRS "${CMAKE_CURRENT_BINARY_DIR}/rvv") +set(KERNEL_DEPENDS rvv-images) +set(KERNEL_EXPORTS + _mlir_ciface_rvv_attention_softmax _mlir_ciface_rvv_binary _mlir_ciface_rvv_cos + _mlir_ciface_rvv_flash_attention _mlir_ciface_rvv_flash_attention_mxfp8 _mlir_ciface_rvv_gelu + _mlir_ciface_rvv_layernorm _mlir_ciface_rvv_layernorm_no_bias _mlir_ciface_rvv_logsumexp_softmax + _mlir_ciface_rvv_matmul _mlir_ciface_rvv_matmul_broadcast_rhs + _mlir_ciface_rvv_matmul_broadcast_transpose_rhs _mlir_ciface_rvv_matmul_transpose_rhs + _mlir_ciface_rvv_mxfp8_decode _mlir_ciface_rvv_mxfp8_encode _mlir_ciface_rvv_norm + _mlir_ciface_rvv_norm_no_weight _mlir_ciface_rvv_rope _mlir_ciface_rvv_silu _mlir_ciface_rvv_sin + _mlir_ciface_rvv_snake _mlir_ciface_rvv_softmax _mlir_ciface_rvv_swiglu _mlir_ciface_rvv_tanh + _mlir_ciface_rvv_unary) +# The Balls of this thread add their kernels to the lists above. +foreach(DIRECTORY IN LISTS DESIGN_TARGET_KERNELS_${THREAD}) + get_filename_component(BALL "${DIRECTORY}/../.." ABSOLUTE) + get_filename_component(BALL "${BALL}" NAME) + add_subdirectory("${DIRECTORY}" balls/${BALL}) +endforeach() + +set(RENAME "") +foreach(SYMBOL IN LISTS KERNEL_EXPORTS) + string(APPEND RENAME "${SYMBOL} ${SYMBOL}__${THREAD}\n") +endforeach() +file(CONFIGURE OUTPUT kernels.rename CONTENT "${RENAME}") +list(TRANSFORM KERNEL_INCLUDES PREPEND "-I") +list(TRANSFORM KERNEL_EMBED_DIRS PREPEND "--embed-dir=") + +foreach(LIBRARY kernels kernels-stream) + set(DEFINES) + if(LIBRARY STREQUAL "kernels-stream") + set(DEFINES -DBUCKYBALL_STREAM) + endif() + set(OBJECTS) + foreach(SOURCE IN LISTS KERNEL_SOURCES) + file(RELATIVE_PATH OBJECT "${DESIGN_REPO}" "${SOURCE}.o") + set(OBJECT "${LIBRARY}/${OBJECT}") + add_custom_command( + OUTPUT ${OBJECT} + COMMAND "${LINUX_CXX}" -std=c++20 -O2 -fPIC -ffp-contract=off ${DEFINES} + ${THREAD_DEFS} ${KERNEL_INCLUDES} ${KERNEL_EMBED_DIRS} -MD -MF ${OBJECT}.d -c "${SOURCE}" -o ${OBJECT} + DEPENDS "${SOURCE}" ${KERNEL_DEPENDS} + DEPFILE ${OBJECT}.d + VERBATIM) + list(APPEND OBJECTS "${CMAKE_CURRENT_BINARY_DIR}/${OBJECT}") + endforeach() + add_custom_command( + OUTPUT lib${LIBRARY}.a + COMMAND "${CMAKE_COMMAND}" -E rm -f lib${LIBRARY}.a + COMMAND "${LINUX_AR}" rcs lib${LIBRARY}.a ${OBJECTS} + COMMAND "${LINUX_OBJCOPY}" --redefine-syms=kernels.rename lib${LIBRARY}.a + DEPENDS ${OBJECTS} "${CMAKE_CURRENT_BINARY_DIR}/kernels.rename" + VERBATIM) +endforeach() +add_custom_target(kernels ALL DEPENDS libkernels.a libkernels-stream.a) diff --git a/examples/prefab/threads/build.sc b/examples/prefab/threads/build.sc new file mode 100644 index 000000000..932a84d4b --- /dev/null +++ b/examples/prefab/threads/build.sc @@ -0,0 +1,10 @@ +import mill._ + +def sources = T.sources { + os.list(millSourcePath).filter(os.isDir).flatMap { root => + val source = root / "arch" + val configs = root / "configs" + (if (os.exists(source)) os.walk(source).filter(p => p.ext == "scala" && !p.toString.contains("/out/") && !p.toString.contains("/build/")) else Seq.empty) ++ + (if (os.exists(configs)) os.walk(configs).filter(_.ext == "scala") else Seq.empty) + }.map(PathRef(_)) +} diff --git a/examples/prefab/threads/ffn/arch/FfnThread.scala b/examples/prefab/threads/ffn/arch/FfnThread.scala new file mode 100644 index 000000000..49cff8a1c --- /dev/null +++ b/examples/prefab/threads/ffn/arch/FfnThread.scala @@ -0,0 +1,77 @@ +package examples.threads.ffn + +import framework.thread.ant.Ant +import framework.core.tbus.ThreadFactory +import framework.core.configs.CoreParam +import sims.configs.SimParam +import framework.core.memory.access.preflight.configs.PmpParams +import framework.thread.configs.ThreadParam +import framework.ip.memory.interlock.{Params => TrackingParam} +import framework.core.memory.access.mmu.configs.{Params => WalkerParam} + +import framework.acc.debug.Debug + +import chisel3.experimental.hierarchy.Instance + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import framework.thread.tlink.{HasTLink, TLinkIO} + +@instantiable +class FfnThread( + threadp: ThreadParam, + permissionp: PmpParams, + trackingp: TrackingParam, + walkerp: WalkerParam, + simp: SimParam, + traceId: Int, + sharedOwnerId: Int) + extends Module + with HasTLink { + val sharedPorts = threadp.acc.get.memDomain.sharedPorts + val diffTest = simp.diffTest + val accp = threadp.acc.get + val antp = threadp.ant.get + + val ant: Instance[Ant] = + Instantiate(new Ant(antp, permissionp)) + + val acc: Instance[Acc] = Instantiate(new Acc(accp, permissionp, trackingp, walkerp, simp)) + + val debug: Instance[Debug] = + Instantiate(new Debug(accp.frontend, accp.memDomain, accp.gpDomain, accp.ballDomain, simp, traceId, sharedOwnerId)) + + debug.io.ant := ant.io.trace + debug.io.acc <> acc.io.debug + + @public + val tlink = IO(new TLinkIO(threadp, permissionp, trackingp, walkerp, simp)) + + ant.io.task <> tlink.task + tlink.tss <> ant.io.shared + ant.io.rocc <> acc.io.rocc + tlink.memory <> acc.io.memory + tlink.ddr <> acc.io.ddr + tlink.shm <> acc.io.shm + acc.io.shared_bank_hashes.zip(tlink.sharedHashes).foreach { case (port, hashes) => port := hashes } +} + +object FfnThread extends ThreadFactory { + + def instantiate( + corep: CoreParam, + threadp: ThreadParam, + simp: SimParam, + traceId: Int + ): TLinkIO = + Instantiate(new FfnThread( + threadp, + corep.privileged.permission, + corep.memory.dcache.tracking, + corep.memory.memoryAccess.walker, + simp, + traceId, + corep.hartId + )).tlink + +} diff --git a/examples/prefab/threads/ffn/arch/acc.scala b/examples/prefab/threads/ffn/arch/acc.scala new file mode 100644 index 000000000..dbf0a7a4f --- /dev/null +++ b/examples/prefab/threads/ffn/arch/acc.scala @@ -0,0 +1,119 @@ +package examples.threads.ffn + +import framework.acc.balldomain.bbus.generated.BallComponents +import sims.configs.SimParam +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.configs.AccParam +import framework.core.memory.access.preflight.configs.PmpParams +import framework.ip.memory.interlock.{Params => TrackingParam} +import framework.core.memory.access.mmu.configs.{Params => WalkerParam} +import framework.acc.debug.AcceleratorDebugPort +import framework.acc.frontend.RoCCIO +import framework.acc.frontend.Frontend +import framework.acc.memdomain.{MemDomain, PhysicalBankHash} +import framework.acc.gpdomain.GpDomain +import framework.acc.balldomain.BallDomain +import framework.acc.balldomain.bbus.BBus +import framework.core.memory.MemoryPort +import framework.thread.tlink.SharedLink +import framework.ip.bus.axi4 + +@instantiable +class Acc( + val accp: AccParam, + val permissionp: PmpParams, + val trackingp: TrackingParam, + val walkerp: WalkerParam, + val simp: SimParam) + extends Module { + val commandDataBits = accp.frontend.commandDataBits + val ownerId = accp.frontend.ownerId + val sharedPorts = accp.memDomain.sharedPorts + val diffTest = simp.diffTest + val frontp = accp.frontend + val memp = accp.memDomain + val gpp = accp.gpDomain + val ballp = accp.ballDomain + val transportp = accp.transport + + val sharedHashCount = if (memp.sharedEnable) memp.sharedBankNum else 0 + + @public + val io = IO(new Bundle { + val debug = new AcceleratorDebugPort(frontp, memp, gpp, ballp, simp) + + val rocc = new RoCCIO(commandDataBits, permissionp) + val ddr = new axi4.Port(transportp.axi) + val memory = new MemoryPort(trackingp, walkerp) + val shm = new SharedLink(frontp, memp) + + val shared_bank_hashes = Option.when(diffTest && memp.sharedEnable)( + Input(Vec(sharedHashCount, new PhysicalBankHash(frontp))) + ) + + }) + + // --- Instantiate domains --- + val frontend: Instance[Frontend] = Instantiate(new Frontend(frontp, memp, gpp, ballp, permissionp, simp)) + + val ballDomain: Instance[BallDomain] = + Instantiate(new BallDomain(frontp, memp, gpp, ballp, permissionp, BallComponents.factories)) + + val memDomain: Instance[MemDomain] = + Instantiate(new MemDomain(frontp, memp, gpp, transportp, permissionp, trackingp, walkerp, simp)) + + val gpDomain = + if (gpp.enable) Some(Instantiate(new GpDomain(frontp, memp, gpp))) + else None + + memDomain.io.shared_bank_hashes.foreach(_ := io.shared_bank_hashes.get) + + io.debug.frontend <> frontend.io.debug + io.debug.banks.zip(memDomain.io.bank_hashes).foreach { case (sink, source) => sink := source } + io.debug.trace.memory := memDomain.io.traceMemory + io.debug.trace.ballStarted := ballDomain.io.traceStarted + io.debug.trace.ballCompleted := ballDomain.io.traceCompleted + + io.rocc <> frontend.io.rocc + io.memory <> memDomain.io.memory + io.ddr <> memDomain.io.ddr + + // --- Frontend -> BallDomain --- + memDomain.io.ball <> ballDomain.io.memory + ballDomain.io.issue <> frontend.io.ballRequest + frontend.io.ballCompletion <> ballDomain.io.complete + + // --- Frontend -> MemDomain --- + memDomain.io.issue <> frontend.io.memoryRequest + frontend.io.memoryCompletion <> memDomain.io.complete + memDomain.io.mapping <> frontend.io.mappingUpdate + + if (gpp.enable) { + val controller = gpDomain.get + controller.io.command <> frontend.io.gpRequest.get + frontend.io.gpCompletion.get <> controller.io.complete + controller.io.image <> memDomain.io.image.get + memDomain.io.gpBanks.get <> controller.io.banks + memDomain.io.gpLayoutRequest.get <> controller.io.layoutRequest + controller.io.layoutResponse <> memDomain.io.gpLayoutResponse.get + ballDomain.io.kernel.get <> controller.io.ball + } + + memDomain.io.banks <> ballDomain.io.banks + memDomain.io.ballLayoutRequest <> ballDomain.io.layoutRequest + ballDomain.io.layoutResponse <> memDomain.io.ballLayoutResponse + + // --- Shared memory passthrough --- + io.shm.bank.requests <> memDomain.io.shared.bank.requests + io.shm.move <> memDomain.io.shared.move + io.shm.bank.local <> memDomain.io.shared.bank.local + io.shm.bank.config <> memDomain.io.shared.bank.config + io.shm.bank.mapping <> memDomain.io.shared.bank.mapping + + // --- Barrier passthrough --- + io.shm.barrierArrive := frontend.io.barrier.arrive + frontend.io.barrier.release := io.shm.barrierRelease + +} diff --git a/examples/prefab/threads/ffn/configs/balldomains/default.toml b/examples/prefab/threads/ffn/configs/balldomains/default.toml new file mode 100644 index 000000000..cc8647054 --- /dev/null +++ b/examples/prefab/threads/ffn/configs/balldomains/default.toml @@ -0,0 +1,53 @@ +[[balls]] +id = 0 +name = "MxmmBall" +component = "mxmm" +param = "../../../../balls/mxmm/configs/default.toml" +port = { inBW = 2, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "MXMM_MXFP8", funct7 = 71 }, + { mnemonic = "MXMM_MXFP8_WINDOW", funct7 = 75 }, + { mnemonic = "MXMM_FMA32", funct7 = 72 }, + { mnemonic = "MXMM_F32", funct7 = 73 }, +] + +[[balls]] +id = 1 +name = "ToInt8Ball" +component = "toint8" +param = "../../../../balls/toint8/configs/default.toml" +port = { inBW = 1, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "QUANT_F32_TO_I8", funct7 = 51 }, + { mnemonic = "QUANT_I32_TO_I8", funct7 = 67 }, +] + +[[balls]] +id = 2 +name = "Int2FpBall" +component = "int2fp" +param = "../../../../balls/int2fp/configs/default.toml" +port = { inBW = 2, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "INT32_TO_FP32", funct7 = 68 }, +] + +[[balls]] +id = 3 +name = "TransposeBall" +component = "transpose" +param = "../../../../balls/transpose/configs/default.toml" +port = { inBW = 1, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "TRANSPOSE", funct7 = 49 }, +] + +[[balls]] +id = 4 +name = "MxquantBall" +component = "mxquant" +param = "../../../../balls/mxquant/configs/default.toml" +port = { inBW = 1, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "MXQUANT", funct7 = 54 }, +] diff --git a/examples/prefab/threads/ffn/configs/cpu/default.toml b/examples/prefab/threads/ffn/configs/cpu/default.toml new file mode 100644 index 000000000..c752c9500 --- /dev/null +++ b/examples/prefab/threads/ffn/configs/cpu/default.toml @@ -0,0 +1,2 @@ +kind = "ant" +config = "../../../../../../arch/src/main/scala/framework/thread/ant/configs/default.toml" diff --git a/examples/prefab/threads/ffn/configs/default.toml b/examples/prefab/threads/ffn/configs/default.toml new file mode 100644 index 000000000..79b6fb436 --- /dev/null +++ b/examples/prefab/threads/ffn/configs/default.toml @@ -0,0 +1,12 @@ +source = ".." +# FFN Thread Configuration (default) + +balldomain = "balldomains/default.toml" +memdomain = "memdomains/default.toml" +frontend = "frontends/default.toml" +gpdomain = "gpdomain/default.toml" +cpu = "cpu/default.toml" + +[compiler] +name = "ffn" +component = "ffn" diff --git a/examples/prefab/threads/ffn/configs/frontends/default.toml b/examples/prefab/threads/ffn/configs/frontends/default.toml new file mode 100644 index 000000000..0f78452a5 --- /dev/null +++ b/examples/prefab/threads/ffn/configs/frontends/default.toml @@ -0,0 +1,9 @@ +bankAliasEnable = true +robEntries = 16 +rsOutOfOrderResponse = true +bankIdLen = 10 +maxPrivateVbank = 31 +sharedVbankBase = 32 +iterLen = 34 +subRobEnable = false +subRobDepth = 64 diff --git a/examples/cores/goban/configs/rvv/default.toml b/examples/prefab/threads/ffn/configs/gpdomain/default.toml similarity index 100% rename from examples/cores/goban/configs/rvv/default.toml rename to examples/prefab/threads/ffn/configs/gpdomain/default.toml diff --git a/examples/prefab/threads/ffn/configs/memdomains/default.toml b/examples/prefab/threads/ffn/configs/memdomains/default.toml new file mode 100644 index 000000000..e3a4ba22d --- /dev/null +++ b/examples/prefab/threads/ffn/configs/memdomains/default.toml @@ -0,0 +1,22 @@ +# FFN Thread Memory Domain Configuration (default) +# Private (per-thread) memory parameters. Shared memory lives in the core TOML. + +addrLen = 39 + +[bank] +layoutEnable = true +num = 32 +width = 128 +entries = 512 +maskLen = 16 +requestPorts = 7 + +[dma] +busWidth = 128 + +[mmio] +enable = false +bankNum = 16 +bankEntries = 64 +bankWidth = 128 +readWidth = 8 diff --git a/examples/prefab/threads/ffn/stack/CMakeLists.txt b/examples/prefab/threads/ffn/stack/CMakeLists.txt new file mode 100644 index 000000000..c2f943ba6 --- /dev/null +++ b/examples/prefab/threads/ffn/stack/CMakeLists.txt @@ -0,0 +1,157 @@ +# ===----------------------------------------------------------------------=== +# The ffn thread: a hardware thread that executes the Ball instruction streams +# its core issues (the engine's stream interpreter). +# +# bbdev stack --build "--design " +# +# builds it for each design that uses it, configured with DESIGN_CMAKE +# (generated from design.pb) and THREAD, the +# compiler target of this thread instance. Products, in the build directory: +# lib-compiler.so buddy-opt plugin: Buckyball dialect + this thread's Balls +# program.elf, program.bin the stream interpreter the hardware thread runs +# loader.o loads program.bin and one instruction batch into a +# thread; links into the engine +# libkernels.a, libkernels-stream.a, kernels.rename +# the kernels of this thread, see Kernels below +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(ffn-thread LANGUAGES C CXX) +include("${DESIGN_CMAKE}") + +set(ENGINE "${DESIGN_REPO}/stack/engine") +# The banks, thread and Ball ISA of this thread, as compile definitions. +set(THREAD_DEFS ${DESIGN_TARGET_DEFS_${THREAD}}) +list(TRANSFORM THREAD_DEFS PREPEND "-D") +find_program(ELF_CC riscv64-unknown-elf-gcc REQUIRED) +find_program(ELF_OBJCOPY riscv64-unknown-elf-objcopy REQUIRED) +find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) + +# ===----------------------------------------------------------------------=== +# Compiler: the Buckyball dialect and passes with the Balls of this thread. +# ===----------------------------------------------------------------------=== +set(BUCKYBALL_BALLS ${DESIGN_TARGET_BALL_STACKS_${THREAD}}) +add_subdirectory("${DESIGN_REPO}/stack/compiler/midend" compiler) +add_library(${THREAD}-compiler MODULE) +target_link_libraries(${THREAD}-compiler PRIVATE buckyball-compiler) +set_target_properties(${THREAD}-compiler PROPERTIES + LINKER_LANGUAGE CXX LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") + +# ===----------------------------------------------------------------------=== +# Program: the engine's stream interpreter, freestanding at the start of the +# thread's code memory. +# ===----------------------------------------------------------------------=== +add_custom_command( + OUTPUT program.elf + COMMAND "${ELF_CC}" -O2 -march=rv64im -mabi=lp64 -mcmodel=medany + -ffreestanding -fno-builtin -fno-pic -fno-stack-protector -fno-jump-tables + -fno-asynchronous-unwind-tables -msmall-data-limit=0 -mno-relax + -nostdlib -nostartfiles -Wl,--no-relax "-Wl,-T,${ENGINE}/program/code.ld" + "-I${ENGINE}/include" ${THREAD_DEFS} + "${ENGINE}/program/start.S" "${ENGINE}/program/stream.c" -o program.elf + DEPENDS "${ENGINE}/program/stream.c" "${ENGINE}/program/instructions.inc" + "${ENGINE}/program/start.S" "${ENGINE}/program/code.ld" + VERBATIM) +add_custom_command( + OUTPUT program.bin + COMMAND "${ELF_OBJCOPY}" -O binary --only-section=.text program.elf program.bin + DEPENDS program.elf + VERBATIM) + +# ===----------------------------------------------------------------------=== +# Loader: stream_prepare.c embeds program.bin and registers as THREAD_SIGNATURE. +# ===----------------------------------------------------------------------=== +add_custom_command( + OUTPUT loader.o + COMMAND "${LINUX_CC}" -O2 -fPIC -ffreestanding -fno-builtin -msmall-data-limit=0 + "--embed-dir=${CMAKE_CURRENT_BINARY_DIR}" "-I${ENGINE}/include" ${THREAD_DEFS} + -c "${ENGINE}/program/stream_prepare.c" -o loader.o + DEPENDS "${ENGINE}/program/stream_prepare.c" program.bin + VERBATIM) + +add_custom_target(program ALL DEPENDS program.bin loader.o) + +# ===----------------------------------------------------------------------=== +# Kernels: the RVV kernels of the GP domain and the kernels of this thread's +# Balls, with the host code that launches them, built against this thread's +# ISA. Every exported symbol gets the suffix __${THREAD}, so the kernels of +# several threads link into one program; models apply kernels.rename to the +# code they compile for this thread. +# libkernels.a Ball instructions issued by the calling core +# libkernels-stream.a Ball instructions sent to the thread through the engine +# ===----------------------------------------------------------------------=== +find_program(LINUX_CXX riscv64-unknown-linux-gnu-g++ REQUIRED) +find_program(LINUX_AR riscv64-unknown-linux-gnu-ar REQUIRED) +find_program(LINUX_OBJCOPY riscv64-unknown-linux-gnu-objcopy REQUIRED) +set(RVV "${DESIGN_REPO}/stack/kernels/rvv") +add_subdirectory("${RVV}" rvv) + +set(KERNEL_SOURCES + "${RVV}/dispatch.cpp" + "${RVV}/matmul_dispatch.cpp" + "${RVV}/cache_dispatch.cpp" + "${RVV}/normalization_dispatch.cpp" + "${RVV}/pointwise_dispatch.cpp" + "${RVV}/activation_dispatch.cpp" + "${RVV}/flash_attention_dispatch.cpp" + "${RVV}/flash_attention_mxfp8_dispatch.cpp") +set(KERNEL_INCLUDES + "${DESIGN_REPO}/bb-tests/workloads/lib" + "${DESIGN_REPO}/stack/kernels" + "${DESIGN_REPO}/stack/runtime/scheduler" + "${DESIGN_REPO}/stack/runtime/services" + "${ENGINE}/include" + "${DESIGN_REPO}/stack/thirdparty/buddy-mlir/llvm/mlir/include/mlir/ExecutionEngine") +set(KERNEL_EMBED_DIRS "${CMAKE_CURRENT_BINARY_DIR}/rvv") +set(KERNEL_DEPENDS rvv-images) +set(KERNEL_EXPORTS + _mlir_ciface_rvv_attention_softmax _mlir_ciface_rvv_binary _mlir_ciface_rvv_cos + _mlir_ciface_rvv_flash_attention _mlir_ciface_rvv_flash_attention_mxfp8 _mlir_ciface_rvv_gelu + _mlir_ciface_rvv_layernorm _mlir_ciface_rvv_layernorm_no_bias _mlir_ciface_rvv_logsumexp_softmax + _mlir_ciface_rvv_matmul _mlir_ciface_rvv_matmul_broadcast_rhs + _mlir_ciface_rvv_matmul_broadcast_transpose_rhs _mlir_ciface_rvv_matmul_transpose_rhs + _mlir_ciface_rvv_mxfp8_decode _mlir_ciface_rvv_mxfp8_encode _mlir_ciface_rvv_norm + _mlir_ciface_rvv_norm_no_weight _mlir_ciface_rvv_rope _mlir_ciface_rvv_silu _mlir_ciface_rvv_sin + _mlir_ciface_rvv_snake _mlir_ciface_rvv_softmax _mlir_ciface_rvv_swiglu _mlir_ciface_rvv_tanh + _mlir_ciface_rvv_unary) +# The Balls of this thread add their kernels to the lists above. +foreach(DIRECTORY IN LISTS DESIGN_TARGET_KERNELS_${THREAD}) + get_filename_component(BALL "${DIRECTORY}/../.." ABSOLUTE) + get_filename_component(BALL "${BALL}" NAME) + add_subdirectory("${DIRECTORY}" balls/${BALL}) +endforeach() + +set(RENAME "") +foreach(SYMBOL IN LISTS KERNEL_EXPORTS) + string(APPEND RENAME "${SYMBOL} ${SYMBOL}__${THREAD}\n") +endforeach() +file(CONFIGURE OUTPUT kernels.rename CONTENT "${RENAME}") +list(TRANSFORM KERNEL_INCLUDES PREPEND "-I") +list(TRANSFORM KERNEL_EMBED_DIRS PREPEND "--embed-dir=") + +foreach(LIBRARY kernels kernels-stream) + set(DEFINES) + if(LIBRARY STREQUAL "kernels-stream") + set(DEFINES -DBUCKYBALL_STREAM) + endif() + set(OBJECTS) + foreach(SOURCE IN LISTS KERNEL_SOURCES) + file(RELATIVE_PATH OBJECT "${DESIGN_REPO}" "${SOURCE}.o") + set(OBJECT "${LIBRARY}/${OBJECT}") + add_custom_command( + OUTPUT ${OBJECT} + COMMAND "${LINUX_CXX}" -std=c++20 -O2 -fPIC -ffp-contract=off ${DEFINES} + ${THREAD_DEFS} ${KERNEL_INCLUDES} ${KERNEL_EMBED_DIRS} -MD -MF ${OBJECT}.d -c "${SOURCE}" -o ${OBJECT} + DEPENDS "${SOURCE}" ${KERNEL_DEPENDS} + DEPFILE ${OBJECT}.d + VERBATIM) + list(APPEND OBJECTS "${CMAKE_CURRENT_BINARY_DIR}/${OBJECT}") + endforeach() + add_custom_command( + OUTPUT lib${LIBRARY}.a + COMMAND "${CMAKE_COMMAND}" -E rm -f lib${LIBRARY}.a + COMMAND "${LINUX_AR}" rcs lib${LIBRARY}.a ${OBJECTS} + COMMAND "${LINUX_OBJCOPY}" --redefine-syms=kernels.rename lib${LIBRARY}.a + DEPENDS ${OBJECTS} "${CMAKE_CURRENT_BINARY_DIR}/kernels.rename" + VERBATIM) +endforeach() +add_custom_target(kernels ALL DEPENDS libkernels.a libkernels-stream.a) diff --git a/examples/prefab/threads/matmul/arch/MatMulThread.scala b/examples/prefab/threads/matmul/arch/MatMulThread.scala new file mode 100644 index 000000000..63dac307e --- /dev/null +++ b/examples/prefab/threads/matmul/arch/MatMulThread.scala @@ -0,0 +1,75 @@ +package examples.threads.matmul + +import framework.thread.ant.Ant +import framework.core.tbus.ThreadFactory +import framework.core.configs.CoreParam +import sims.configs.SimParam +import framework.thread.configs.ThreadParam +import framework.core.memory.access.preflight.configs.PmpParams +import framework.ip.memory.interlock.{Params => TrackingParam} +import framework.core.memory.access.mmu.configs.{Params => WalkerParam} + +import framework.acc.debug.Debug + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.thread.tlink.{HasTLink, TLinkIO} + +@instantiable +class MatMulThread( + threadp: ThreadParam, + permissionp: PmpParams, + trackingp: TrackingParam, + walkerp: WalkerParam, + simp: SimParam, + traceId: Int, + sharedOwnerId: Int) + extends Module + with HasTLink { + val sharedPorts = threadp.acc.get.memDomain.sharedPorts + val diffTest = simp.diffTest + val accp = threadp.acc.get + val antp = threadp.ant.get + + val ant: Instance[Ant] = + Instantiate(new Ant(antp, permissionp)) + + val acc: Instance[Acc] = Instantiate(new Acc(accp, permissionp, trackingp, walkerp, simp)) + + val debug: Instance[Debug] = + Instantiate(new Debug(accp.frontend, accp.memDomain, accp.gpDomain, accp.ballDomain, simp, traceId, sharedOwnerId)) + + debug.io.ant := ant.io.trace + debug.io.acc <> acc.io.debug + + @public + val tlink = IO(new TLinkIO(threadp, permissionp, trackingp, walkerp, simp)) + + ant.io.task <> tlink.task + tlink.tss <> ant.io.shared + ant.io.rocc <> acc.io.rocc + tlink.memory <> acc.io.memory + tlink.ddr <> acc.io.ddr + tlink.shm <> acc.io.shm + acc.io.shared_bank_hashes.zip(tlink.sharedHashes).foreach { case (port, hashes) => port := hashes } +} + +object MatMulThread extends ThreadFactory { + + def instantiate( + corep: CoreParam, + threadp: ThreadParam, + simp: SimParam, + traceId: Int + ): TLinkIO = + Instantiate(new MatMulThread( + threadp, + corep.privileged.permission, + corep.memory.dcache.tracking, + corep.memory.memoryAccess.walker, + simp, + traceId, + corep.hartId + )).tlink + +} diff --git a/examples/prefab/threads/matmul/arch/acc.scala b/examples/prefab/threads/matmul/arch/acc.scala new file mode 100644 index 000000000..26c96c938 --- /dev/null +++ b/examples/prefab/threads/matmul/arch/acc.scala @@ -0,0 +1,119 @@ +package examples.threads.matmul + +import framework.acc.balldomain.bbus.generated.BallComponents +import sims.configs.SimParam +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.configs.AccParam +import framework.core.memory.access.preflight.configs.PmpParams +import framework.ip.memory.interlock.{Params => TrackingParam} +import framework.core.memory.access.mmu.configs.{Params => WalkerParam} +import framework.acc.debug.AcceleratorDebugPort +import framework.acc.frontend.RoCCIO +import framework.acc.frontend.Frontend +import framework.acc.memdomain.{MemDomain, PhysicalBankHash} +import framework.acc.gpdomain.GpDomain +import framework.acc.balldomain.BallDomain +import framework.acc.balldomain.bbus.BBus +import framework.core.memory.MemoryPort +import framework.thread.tlink.SharedLink +import framework.ip.bus.axi4 + +@instantiable +class Acc( + val accp: AccParam, + val permissionp: PmpParams, + val trackingp: TrackingParam, + val walkerp: WalkerParam, + val simp: SimParam) + extends Module { + val commandDataBits = accp.frontend.commandDataBits + val ownerId = accp.frontend.ownerId + val sharedPorts = accp.memDomain.sharedPorts + val diffTest = simp.diffTest + val frontp = accp.frontend + val memp = accp.memDomain + val gpp = accp.gpDomain + val ballp = accp.ballDomain + val transportp = accp.transport + + val sharedHashCount = if (memp.sharedEnable) memp.sharedBankNum else 0 + + @public + val io = IO(new Bundle { + val debug = new AcceleratorDebugPort(frontp, memp, gpp, ballp, simp) + + val rocc = new RoCCIO(commandDataBits, permissionp) + val ddr = new axi4.Port(transportp.axi) + val memory = new MemoryPort(trackingp, walkerp) + val shm = new SharedLink(frontp, memp) + + val shared_bank_hashes = Option.when(diffTest && memp.sharedEnable)( + Input(Vec(sharedHashCount, new PhysicalBankHash(frontp))) + ) + + }) + + // --- Instantiate domains --- + val frontend: Instance[Frontend] = Instantiate(new Frontend(frontp, memp, gpp, ballp, permissionp, simp)) + + val ballDomain: Instance[BallDomain] = + Instantiate(new BallDomain(frontp, memp, gpp, ballp, permissionp, BallComponents.factories)) + + val memDomain: Instance[MemDomain] = + Instantiate(new MemDomain(frontp, memp, gpp, transportp, permissionp, trackingp, walkerp, simp)) + + val gpDomain = + if (gpp.enable) Some(Instantiate(new GpDomain(frontp, memp, gpp))) + else None + + memDomain.io.shared_bank_hashes.foreach(_ := io.shared_bank_hashes.get) + + io.debug.frontend <> frontend.io.debug + io.debug.banks.zip(memDomain.io.bank_hashes).foreach { case (sink, source) => sink := source } + io.debug.trace.memory := memDomain.io.traceMemory + io.debug.trace.ballStarted := ballDomain.io.traceStarted + io.debug.trace.ballCompleted := ballDomain.io.traceCompleted + + io.rocc <> frontend.io.rocc + io.memory <> memDomain.io.memory + io.ddr <> memDomain.io.ddr + + // --- Frontend -> BallDomain --- + memDomain.io.ball <> ballDomain.io.memory + ballDomain.io.issue <> frontend.io.ballRequest + frontend.io.ballCompletion <> ballDomain.io.complete + + // --- Frontend -> MemDomain --- + memDomain.io.issue <> frontend.io.memoryRequest + frontend.io.memoryCompletion <> memDomain.io.complete + memDomain.io.mapping <> frontend.io.mappingUpdate + + if (gpp.enable) { + val controller = gpDomain.get + controller.io.command <> frontend.io.gpRequest.get + frontend.io.gpCompletion.get <> controller.io.complete + controller.io.image <> memDomain.io.image.get + memDomain.io.gpBanks.get <> controller.io.banks + memDomain.io.gpLayoutRequest.get <> controller.io.layoutRequest + controller.io.layoutResponse <> memDomain.io.gpLayoutResponse.get + ballDomain.io.kernel.get <> controller.io.ball + } + + memDomain.io.banks <> ballDomain.io.banks + memDomain.io.ballLayoutRequest <> ballDomain.io.layoutRequest + ballDomain.io.layoutResponse <> memDomain.io.ballLayoutResponse + + // --- Shared memory passthrough --- + io.shm.bank.requests <> memDomain.io.shared.bank.requests + io.shm.move <> memDomain.io.shared.move + io.shm.bank.local <> memDomain.io.shared.bank.local + io.shm.bank.config <> memDomain.io.shared.bank.config + io.shm.bank.mapping <> memDomain.io.shared.bank.mapping + + // --- Barrier passthrough --- + io.shm.barrierArrive := frontend.io.barrier.arrive + frontend.io.barrier.release := io.shm.barrierRelease + +} diff --git a/examples/prefab/threads/matmul/configs/ant.toml b/examples/prefab/threads/matmul/configs/ant.toml new file mode 100644 index 000000000..6e8d3ad08 --- /dev/null +++ b/examples/prefab/threads/matmul/configs/ant.toml @@ -0,0 +1,5 @@ +source = ".." +balldomain = "balldomains/default.toml" +memdomain = "memdomains/default.toml" +frontend = "frontends/default.toml" +gpdomain = "gpdomain/default.toml" diff --git a/examples/prefab/threads/matmul/configs/balldomains/default.toml b/examples/prefab/threads/matmul/configs/balldomains/default.toml new file mode 100644 index 000000000..0bc2e08ad --- /dev/null +++ b/examples/prefab/threads/matmul/configs/balldomains/default.toml @@ -0,0 +1,41 @@ +[[balls]] +id = 0 +name = "SMatMulBall" +component = "smatmul" +param = "../../../../balls/smatmul/configs/default.toml" +port = { inBW = 2, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "SMATMUL_OS", funct7 = 65 }, + { mnemonic = "SMATMUL_BIAS", funct7 = 17 }, +] + +[[balls]] +id = 1 +name = "ToInt8Ball" +component = "toint8" +param = "../../../../balls/toint8/configs/default.toml" +port = { inBW = 1, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "QUANT_F32_TO_I8", funct7 = 51 }, + { mnemonic = "QUANT_I32_TO_I8", funct7 = 67 }, +] + +[[balls]] +id = 2 +name = "Int2FpBall" +component = "int2fp" +param = "../../../../balls/int2fp/configs/default.toml" +port = { inBW = 2, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "INT32_TO_FP32", funct7 = 68 }, +] + +[[balls]] +id = 3 +name = "TransposeBall" +component = "transpose" +param = "../../../../balls/transpose/configs/default.toml" +port = { inBW = 1, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "TRANSPOSE", funct7 = 49 }, +] diff --git a/examples/prefab/threads/matmul/configs/cpu/default.toml b/examples/prefab/threads/matmul/configs/cpu/default.toml new file mode 100644 index 000000000..c752c9500 --- /dev/null +++ b/examples/prefab/threads/matmul/configs/cpu/default.toml @@ -0,0 +1,2 @@ +kind = "ant" +config = "../../../../../../arch/src/main/scala/framework/thread/ant/configs/default.toml" diff --git a/examples/prefab/threads/matmul/configs/default.toml b/examples/prefab/threads/matmul/configs/default.toml new file mode 100644 index 000000000..1bd004769 --- /dev/null +++ b/examples/prefab/threads/matmul/configs/default.toml @@ -0,0 +1,13 @@ +source = ".." +# MatMul Thread Configuration (default) + +balldomain = "balldomains/default.toml" +memdomain = "memdomains/default.toml" +frontend = "frontends/default.toml" +gpdomain = "gpdomain/default.toml" +cpu = "cpu/default.toml" +transport = "default" + +[compiler] +name = "matmul" +component = "matmul" diff --git a/examples/prefab/threads/matmul/configs/frontends/default.toml b/examples/prefab/threads/matmul/configs/frontends/default.toml new file mode 100644 index 000000000..5464f2d95 --- /dev/null +++ b/examples/prefab/threads/matmul/configs/frontends/default.toml @@ -0,0 +1,9 @@ +bankAliasEnable = true +robEntries = 16 +rsOutOfOrderResponse = true +bankIdLen = 10 +maxPrivateVbank = 15 +sharedVbankBase = 32 +iterLen = 34 +subRobEnable = false +subRobDepth = 64 diff --git a/examples/cores/toy/configs/rvv/default.toml b/examples/prefab/threads/matmul/configs/gpdomain/default.toml similarity index 100% rename from examples/cores/toy/configs/rvv/default.toml rename to examples/prefab/threads/matmul/configs/gpdomain/default.toml diff --git a/examples/prefab/threads/matmul/configs/memdomains/default.toml b/examples/prefab/threads/matmul/configs/memdomains/default.toml new file mode 100644 index 000000000..d469cceb5 --- /dev/null +++ b/examples/prefab/threads/matmul/configs/memdomains/default.toml @@ -0,0 +1,22 @@ +# MatMul Thread Memory Domain Configuration (default) +# Private (per-thread) memory parameters. Shared memory lives in the core TOML. + +addrLen = 39 + +[bank] +layoutEnable = true +num = 20 +width = 128 +entries = 1024 +maskLen = 16 +requestPorts = 7 + +[dma] +busWidth = 128 + +[mmio] +enable = false +bankNum = 16 +bankEntries = 64 +bankWidth = 128 +readWidth = 8 diff --git a/examples/prefab/threads/matmul/stack/CMakeLists.txt b/examples/prefab/threads/matmul/stack/CMakeLists.txt new file mode 100644 index 000000000..26d28e640 --- /dev/null +++ b/examples/prefab/threads/matmul/stack/CMakeLists.txt @@ -0,0 +1,157 @@ +# ===----------------------------------------------------------------------=== +# The matmul thread: a hardware thread that runs one fixed SMatMul program. +# +# bbdev stack --build "--design " +# +# builds it for each design that uses it, configured with DESIGN_CMAKE +# (generated from design.pb) and THREAD, the +# compiler target of this thread instance. Products, in the build directory: +# lib-compiler.so buddy-opt plugin: Buckyball dialect + this thread's Balls +# program.elf, program.bin the code the hardware thread runs +# loader.o loads program.bin into a thread; links into the engine +# libkernels.a, libkernels-stream.a, kernels.rename +# the kernels of this thread, see Kernels below +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(matmul-thread LANGUAGES C CXX) +include("${DESIGN_CMAKE}") + +set(ENGINE "${DESIGN_REPO}/stack/engine") +# The banks, thread and Ball ISA of this thread, as compile definitions. +set(THREAD_DEFS ${DESIGN_TARGET_DEFS_${THREAD}}) +list(TRANSFORM THREAD_DEFS PREPEND "-D") +find_program(ELF_CC riscv64-unknown-elf-gcc REQUIRED) +find_program(ELF_OBJCOPY riscv64-unknown-elf-objcopy REQUIRED) +find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) + +# ===----------------------------------------------------------------------=== +# Compiler: the Buckyball dialect and passes with the Balls of this thread. +# ===----------------------------------------------------------------------=== +set(BUCKYBALL_BALLS ${DESIGN_TARGET_BALL_STACKS_${THREAD}}) +add_subdirectory("${DESIGN_REPO}/stack/compiler/midend" compiler) +add_library(${THREAD}-compiler MODULE) +target_link_libraries(${THREAD}-compiler PRIVATE buckyball-compiler) +set_target_properties(${THREAD}-compiler PROPERTIES + LINKER_LANGUAGE CXX LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") + +# ===----------------------------------------------------------------------=== +# Program: program.c, freestanding at the start of the thread's code memory. +# ===----------------------------------------------------------------------=== +add_custom_command( + OUTPUT program.elf + COMMAND "${ELF_CC}" -O2 -march=rv64im -mabi=lp64 -mcmodel=medany + -ffreestanding -fno-builtin -fno-pic -fno-stack-protector -fno-jump-tables + -fno-asynchronous-unwind-tables -msmall-data-limit=0 -mno-relax + -nostdlib -nostartfiles -Wl,--no-relax "-Wl,-T,${ENGINE}/program/code.ld" + "-I${ENGINE}/include" ${THREAD_DEFS} "-I${DESIGN_REPO}/bb-tests/workloads/lib" + "-I${DESIGN_REPO}/examples/prefab/balls/smatmul/workloads" + "${ENGINE}/program/start.S" "${CMAKE_CURRENT_SOURCE_DIR}/program.c" + -o program.elf + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/program.c" "${CMAKE_CURRENT_SOURCE_DIR}/matmul.h" + "${ENGINE}/program/start.S" "${ENGINE}/program/code.ld" + VERBATIM) +add_custom_command( + OUTPUT program.bin + COMMAND "${ELF_OBJCOPY}" -O binary --only-section=.text program.elf program.bin + DEPENDS program.elf + VERBATIM) + +# ===----------------------------------------------------------------------=== +# Loader: prepare.c embeds program.bin and registers as THREAD_SIGNATURE. +# ===----------------------------------------------------------------------=== +add_custom_command( + OUTPUT loader.o + COMMAND "${LINUX_CC}" -O2 -fPIC -ffreestanding -fno-builtin -msmall-data-limit=0 + "--embed-dir=${CMAKE_CURRENT_BINARY_DIR}" "-I${ENGINE}/include" ${THREAD_DEFS} + -c "${CMAKE_CURRENT_SOURCE_DIR}/prepare.c" -o loader.o + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/prepare.c" "${CMAKE_CURRENT_SOURCE_DIR}/matmul.h" + program.bin + VERBATIM) + +add_custom_target(program ALL DEPENDS program.bin loader.o) + +# ===----------------------------------------------------------------------=== +# Kernels: the RVV kernels of the GP domain and the kernels of this thread's +# Balls, with the host code that launches them, built against this thread's +# ISA. Every exported symbol gets the suffix __${THREAD}, so the kernels of +# several threads link into one program; models apply kernels.rename to the +# code they compile for this thread. +# libkernels.a Ball instructions issued by the calling core +# libkernels-stream.a Ball instructions sent to the thread through the engine +# ===----------------------------------------------------------------------=== +find_program(LINUX_CXX riscv64-unknown-linux-gnu-g++ REQUIRED) +find_program(LINUX_AR riscv64-unknown-linux-gnu-ar REQUIRED) +find_program(LINUX_OBJCOPY riscv64-unknown-linux-gnu-objcopy REQUIRED) +set(RVV "${DESIGN_REPO}/stack/kernels/rvv") +add_subdirectory("${RVV}" rvv) + +set(KERNEL_SOURCES + "${RVV}/dispatch.cpp" + "${RVV}/matmul_dispatch.cpp" + "${RVV}/cache_dispatch.cpp" + "${RVV}/normalization_dispatch.cpp" + "${RVV}/pointwise_dispatch.cpp" + "${RVV}/activation_dispatch.cpp" + "${RVV}/flash_attention_dispatch.cpp" + "${RVV}/flash_attention_mxfp8_dispatch.cpp") +set(KERNEL_INCLUDES + "${DESIGN_REPO}/bb-tests/workloads/lib" + "${DESIGN_REPO}/stack/kernels" + "${DESIGN_REPO}/stack/runtime/scheduler" + "${DESIGN_REPO}/stack/runtime/services" + "${ENGINE}/include" + "${DESIGN_REPO}/stack/thirdparty/buddy-mlir/llvm/mlir/include/mlir/ExecutionEngine") +set(KERNEL_EMBED_DIRS "${CMAKE_CURRENT_BINARY_DIR}/rvv") +set(KERNEL_DEPENDS rvv-images) +set(KERNEL_EXPORTS + _mlir_ciface_rvv_attention_softmax _mlir_ciface_rvv_binary _mlir_ciface_rvv_cos + _mlir_ciface_rvv_flash_attention _mlir_ciface_rvv_flash_attention_mxfp8 _mlir_ciface_rvv_gelu + _mlir_ciface_rvv_layernorm _mlir_ciface_rvv_layernorm_no_bias _mlir_ciface_rvv_logsumexp_softmax + _mlir_ciface_rvv_matmul _mlir_ciface_rvv_matmul_broadcast_rhs + _mlir_ciface_rvv_matmul_broadcast_transpose_rhs _mlir_ciface_rvv_matmul_transpose_rhs + _mlir_ciface_rvv_mxfp8_decode _mlir_ciface_rvv_mxfp8_encode _mlir_ciface_rvv_norm + _mlir_ciface_rvv_norm_no_weight _mlir_ciface_rvv_rope _mlir_ciface_rvv_silu _mlir_ciface_rvv_sin + _mlir_ciface_rvv_snake _mlir_ciface_rvv_softmax _mlir_ciface_rvv_swiglu _mlir_ciface_rvv_tanh + _mlir_ciface_rvv_unary) +# The Balls of this thread add their kernels to the lists above. +foreach(DIRECTORY IN LISTS DESIGN_TARGET_KERNELS_${THREAD}) + get_filename_component(BALL "${DIRECTORY}/../.." ABSOLUTE) + get_filename_component(BALL "${BALL}" NAME) + add_subdirectory("${DIRECTORY}" balls/${BALL}) +endforeach() + +set(RENAME "") +foreach(SYMBOL IN LISTS KERNEL_EXPORTS) + string(APPEND RENAME "${SYMBOL} ${SYMBOL}__${THREAD}\n") +endforeach() +file(CONFIGURE OUTPUT kernels.rename CONTENT "${RENAME}") +list(TRANSFORM KERNEL_INCLUDES PREPEND "-I") +list(TRANSFORM KERNEL_EMBED_DIRS PREPEND "--embed-dir=") + +foreach(LIBRARY kernels kernels-stream) + set(DEFINES) + if(LIBRARY STREQUAL "kernels-stream") + set(DEFINES -DBUCKYBALL_STREAM) + endif() + set(OBJECTS) + foreach(SOURCE IN LISTS KERNEL_SOURCES) + file(RELATIVE_PATH OBJECT "${DESIGN_REPO}" "${SOURCE}.o") + set(OBJECT "${LIBRARY}/${OBJECT}") + add_custom_command( + OUTPUT ${OBJECT} + COMMAND "${LINUX_CXX}" -std=c++20 -O2 -fPIC -ffp-contract=off ${DEFINES} + ${THREAD_DEFS} ${KERNEL_INCLUDES} ${KERNEL_EMBED_DIRS} -MD -MF ${OBJECT}.d -c "${SOURCE}" -o ${OBJECT} + DEPENDS "${SOURCE}" ${KERNEL_DEPENDS} + DEPFILE ${OBJECT}.d + VERBATIM) + list(APPEND OBJECTS "${CMAKE_CURRENT_BINARY_DIR}/${OBJECT}") + endforeach() + add_custom_command( + OUTPUT lib${LIBRARY}.a + COMMAND "${CMAKE_COMMAND}" -E rm -f lib${LIBRARY}.a + COMMAND "${LINUX_AR}" rcs lib${LIBRARY}.a ${OBJECTS} + COMMAND "${LINUX_OBJCOPY}" --redefine-syms=kernels.rename lib${LIBRARY}.a + DEPENDS ${OBJECTS} "${CMAKE_CURRENT_BINARY_DIR}/kernels.rename" + VERBATIM) +endforeach() +add_custom_target(kernels ALL DEPENDS libkernels.a libkernels-stream.a) diff --git a/examples/prefab/threads/matmul/stack/matmul.h b/examples/prefab/threads/matmul/stack/matmul.h new file mode 100644 index 000000000..6bbf8334c --- /dev/null +++ b/examples/prefab/threads/matmul/stack/matmul.h @@ -0,0 +1,5 @@ +#pragma once +#include +struct matmul_task { + uint64_t a, b, bias, output; +}; diff --git a/examples/prefab/threads/matmul/stack/prepare.c b/examples/prefab/threads/matmul/stack/prepare.c new file mode 100644 index 000000000..21b165aec --- /dev/null +++ b/examples/prefab/threads/matmul/stack/prepare.c @@ -0,0 +1,31 @@ +#include "matmul.h" +#include + +static const unsigned char image[] __attribute__((aligned(8))) = { +#embed "program.bin" +}; + +/* Loads the matmul program and its task descriptor into a thread. */ +static void prepare(uint32_t thread, void *opaque, struct thread_task *task) { + size_t code_bytes = sizeof(image); + size_t data_bytes = sizeof(struct matmul_task); + uint64_t tls_base = thread_query(thread, THREAD_FIELD_TLS_BASE); + uint64_t tls_bytes = thread_query(thread, THREAD_FIELD_TLS_BYTES); + if (code_bytes > thread_query(thread, THREAD_FIELD_CODE_BYTES) || + ((data_bytes + 7) & ~(size_t)7) + 256 > tls_bytes || + code_bytes > thread_query(thread, THREAD_FIELD_TSS_BYTES) || + data_bytes > thread_query(thread, THREAD_FIELD_TSS_BYTES)) + __builtin_trap(); + thread_tss_write(0, image, code_bytes); + thread_load(thread, THREAD_CODE, 0, 0, (code_bytes + 7) & ~(size_t)7); + thread_tss_write(0, opaque, data_bytes); + thread_load(thread, THREAD_TLS, 0, 0, (data_bytes + 7) & ~(size_t)7); + task->entry = 0; + task->code_end = code_bytes; + task->argument = tls_base; + task->stack = tls_base + tls_bytes; +} + +static const struct engine_binding binding + __attribute__((section("engine_bindings"), used)) = {THREAD_SIGNATURE, + prepare}; diff --git a/examples/prefab/threads/matmul/stack/program.c b/examples/prefab/threads/matmul/stack/program.c new file mode 100644 index 000000000..cad976c0e --- /dev/null +++ b/examples/prefab/threads/matmul/stack/program.c @@ -0,0 +1,17 @@ +#include "matmul.h" +#include +#include + +uint64_t entry(const struct matmul_task *args) { + for (unsigned bank = 0; bank < 4; ++bank) + bb_mset(bank, 1, 1, 1); + bb_mvin(args->bias, 0, 4, 1); + bb_mvin(args->a, 1, 16, 1); + bb_mvin(args->b, 2, 16, 1); + bb_smatmul_bias(0, 0); + bb_smatmul_os(1, 2, 3, 16, 16, 16, 1, 1, 0); + bb_mvout(args->output, 3, 64, 1); + for (unsigned bank = 0; bank < 4; ++bank) + bb_mset(bank, 0, 0, 0); + return 0; +} diff --git a/examples/prefab/threads/matmul/workloads/ctests/CMakeLists.txt b/examples/prefab/threads/matmul/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..841569277 --- /dev/null +++ b/examples/prefab/threads/matmul/workloads/ctests/CMakeLists.txt @@ -0,0 +1,14 @@ +add_subdirectory(transpose_smatmul_quant_pipeline) +set(BUCKYBALL_CTEST_TARGET ${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}) +file( + RELATIVE_PATH + RVV_KERNEL_SOURCE + ${CMAKE_CURRENT_SOURCE_DIR} + ${WORKLOAD_SRC_DIR}/CTest/rvv/rvv_kernel.c +) +add_buckyball_ctest(${RVV_KERNEL_SOURCE}) +set(RVV_KERNEL_TEST + "${DESIGN_NAME}-${BUCKYBALL_CTEST_TARGET}-ctest-rvv_kernel" +) +add_dependencies(component-ctests-build ${RVV_KERNEL_TEST}-ctest-build) +add_subdirectory(rvv) diff --git a/examples/prefab/threads/matmul/workloads/ctests/rvv/CMakeLists.txt b/examples/prefab/threads/matmul/workloads/ctests/rvv/CMakeLists.txt new file mode 100644 index 000000000..8035cb31e --- /dev/null +++ b/examples/prefab/threads/matmul/workloads/ctests/rvv/CMakeLists.txt @@ -0,0 +1,75 @@ +# Images: built by the thread stack (stack/kernels/rvv); dispatch code: built here. +set(KERNEL_DIR "${BUCKYBALL_REPO_ROOT}/stack/kernels/rvv") +set(IMAGE_DIR "${DESIGN_THREAD_BUILD_${BUCKYBALL_CTEST_TARGET}}/rvv") +set(_BANK_NUM ${DESIGN_PROFILES_${BUCKYBALL_CTEST_TARGET}_THREAD_MEM_BANK_NUM}) +set(_BANK_WIDTH ${DESIGN_PROFILES_${BUCKYBALL_CTEST_TARGET}_THREAD_MEM_BANK_WIDTH}) +set(_BANK_ENTRIES ${DESIGN_PROFILES_${BUCKYBALL_CTEST_TARGET}_THREAD_MEM_BANK_ENTRIES}) +if(_BANK_NUM LESS 4 OR NOT _BANK_WIDTH EQUAL 128 OR _BANK_ENTRIES LESS 32) + message( + FATAL_ERROR + "RVV kernel CTests require 4 banks, 128-bit rows and 512 bytes per bank; " + "${BUCKYBALL_CTEST_TARGET} has ${_BANK_NUM} banks, ${_BANK_WIDTH}-bit rows, ${_BANK_ENTRIES} lines" + ) +endif() +foreach( + KERNEL + silu + swiglu + snake + mxfp8_quant + matmul + norm +) + buckyball_enforce_ctest_line_limit("${KERNEL}.cpp") + set(NAME + "${DESIGN_NAME}-${BUCKYBALL_CTEST_TARGET}-ctest-rvv_${KERNEL}" + ) + add_buckyball_linux_ctest(${NAME} ${CMAKE_CURRENT_SOURCE_DIR} ${KERNEL}.cpp) + if(KERNEL STREQUAL "matmul") + set(DISPATCH_SOURCE "${KERNEL_DIR}/matmul_dispatch.cpp") + elseif(KERNEL STREQUAL "mxfp8_quant") + set(DISPATCH_SOURCE "${KERNEL_DIR}/quant_dispatch.cpp") + else() + set(DISPATCH_SOURCE "${KERNEL_DIR}/dispatch.cpp") + endif() + target_sources(${NAME}-linux PRIVATE "${DISPATCH_SOURCE}") + target_compile_options(${NAME}-linux PRIVATE "--embed-dir=${IMAGE_DIR}") + target_include_directories( + ${NAME}-linux + PRIVATE + "${KERNEL_DIR}" + "${BUCKYBALL_REPO_ROOT}/stack/kernels" + "${BUCKYBALL_REPO_ROOT}/stack/thirdparty/buddy-mlir/llvm/mlir/include/mlir/ExecutionEngine" + ) + target_compile_options(${NAME}-linux PRIVATE -O2) + set_target_properties(${NAME}-linux PROPERTIES LINKER_LANGUAGE CXX) + target_link_libraries(${NAME}-linux PRIVATE m) + buckyball_ctest_deps(DEPS ${CMAKE_CURRENT_SOURCE_DIR} ${KERNEL}.cpp) + set(TARGET_DEFINES ${DESIGN_TARGET_DEFS_${BUCKYBALL_CTEST_TARGET}}) + list(TRANSFORM TARGET_DEFINES PREPEND "-D") + add_custom_command( + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${NAME}-baremetal" + COMMAND + ${ELF_CXX} ${BUCKYBALL_CTEST_C_FLAGS} + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${WORKLOAD_LIB_DIR} -I${KERNEL_DIR} + -I${BUCKYBALL_REPO_ROOT}/stack/kernels --embed-dir=${IMAGE_DIR} + ${DESIGN_DEFINES} ${TARGET_DEFINES} + -I${BUCKYBALL_REPO_ROOT}/stack/thirdparty/buddy-mlir/llvm/mlir/include/mlir/ExecutionEngine + -o "${NAME}-baremetal" ${BAREMETAL_DIR}/crt0.S -x c + ${WORKLOAD_LIB_DIR}/buckyball.c ${BBHW_MEM_C} + ${BUCKYBALL_DESIGN_COMMON_SOURCES} -x c++ + "${CMAKE_CURRENT_SOURCE_DIR}/${KERNEL}.cpp" "${DISPATCH_SOURCE}" -lm + DEPENDS ${DEPS} "${DISPATCH_SOURCE}" ${BAREMETAL_DIR}/crt0.S + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" + VERBATIM + ) + add_custom_target( + ${NAME}-ctest-build + DEPENDS + ${NAME}-linux-build + "${CMAKE_CURRENT_BINARY_DIR}/${NAME}-baremetal" + ) + add_dependencies(component-ctests-build ${NAME}-ctest-build) +endforeach() diff --git a/examples/cores/goban/workloads/ctests/rvv/matmul.cpp b/examples/prefab/threads/matmul/workloads/ctests/rvv/matmul.cpp similarity index 100% rename from examples/cores/goban/workloads/ctests/rvv/matmul.cpp rename to examples/prefab/threads/matmul/workloads/ctests/rvv/matmul.cpp diff --git a/examples/cores/goban/workloads/ctests/rvv/mxfp8_quant.cpp b/examples/prefab/threads/matmul/workloads/ctests/rvv/mxfp8_quant.cpp similarity index 95% rename from examples/cores/goban/workloads/ctests/rvv/mxfp8_quant.cpp rename to examples/prefab/threads/matmul/workloads/ctests/rvv/mxfp8_quant.cpp index 07f9f04b1..fa3ccd486 100644 --- a/examples/cores/goban/workloads/ctests/rvv/mxfp8_quant.cpp +++ b/examples/prefab/threads/matmul/workloads/ctests/rvv/mxfp8_quant.cpp @@ -1,9 +1,11 @@ #include -#include #include #include #include +extern "C" void mxfp8_quant(const uint32_t *input, uint8_t *output, + uint32_t count); + alignas(16) static float input[128]; alignas(16) static uint8_t output[144]; static double candidates[127]; diff --git a/examples/prefab/threads/matmul/workloads/ctests/rvv/norm.cpp b/examples/prefab/threads/matmul/workloads/ctests/rvv/norm.cpp new file mode 100644 index 000000000..17a077cb5 --- /dev/null +++ b/examples/prefab/threads/matmul/workloads/ctests/rvv/norm.cpp @@ -0,0 +1,80 @@ +#include +#include +#include +#include + +extern "C" void _mlir_ciface_rvv_norm(UnrankedMemRefType *, + UnrankedMemRefType *, + UnrankedMemRefType *, uint32_t, + uint32_t); +extern "C" void _mlir_ciface_rvv_norm_no_weight(UnrankedMemRefType *, + UnrankedMemRefType *, + uint32_t, uint32_t); +constexpr int capacity = THREAD_MEM_BANK_ENTRIES * THREAD_MEM_BANK_WIDTH / 32; +constexpr int maximum = 2 * capacity + 5; +alignas(16) static float input[2 * (maximum + 4) + 4]; +alignas(16) static float output[2 * maximum + 4], expected[2 * maximum]; +alignas(16) static float weights[maximum]; + +int main() { + for (int width : {capacity - 1, capacity, capacity + 3, maximum}) { + for (bool weighted : {false, true}) { + for (float &value : output) + value = -999.0f; + for (int col = 0; col < width; ++col) + weights[col] = 0.5f + (col % 7) * 0.125f; + for (int row = 0; row < 2; ++row) + for (int col = 0; col < width; ++col) + input[2 + row * (width + 4) + col] = + ((col + row * 3) % 31 - 15) * 0.125f; + float multiplier = 1.0f / width, epsilon = 1.0e-5f; + uint32_t meanBits, epsilonBits; + std::memcpy(&meanBits, &multiplier, 4); + std::memcpy(&epsilonBits, &epsilon, 4); + asm volatile("csrw fcsr, zero" ::: "memory"); + for (int row = 0; row < 2; ++row) { + volatile float sum = 0; + for (int col = 0; col < width; ++col) { + float value = input[2 + row * (width + 4) + col]; + volatile float square = value * value; + sum = sum + square; + } + volatile float mean = sum * multiplier; + float scale = 1.0f / std::sqrt(mean + epsilon); + for (int col = 0; col < width; ++col) { + volatile float value = input[2 + row * (width + 4) + col] * scale; + expected[row * width + col] = weighted ? weights[col] * value : value; + } + } + uint32_t expectedFlags; + asm volatile("csrr %0, fflags" : "=r"(expectedFlags)::"memory"); + StridedMemRefType in{ + input, input, 2, {2, width}, {width + 4, 1}}; + StridedMemRefType out{ + output, output, 2, {2, width}, {width, 1}}; + StridedMemRefType weight{weights, weights, 0, {width}, {1}}; + UnrankedMemRefType inRef{2, &in}, outRef{2, &out}, + wRef{1, &weight}; + asm volatile("csrw fcsr, zero" ::: "memory"); + if (weighted) + _mlir_ciface_rvv_norm(&outRef, &inRef, &wRef, meanBits, epsilonBits); + else + _mlir_ciface_rvv_norm_no_weight(&outRef, &inRef, meanBits, epsilonBits); + uint32_t flags; + asm volatile("csrr %0, fflags" : "=r"(flags)::"memory"); + if (flags != expectedFlags || + std::memcmp(output + 2, expected, 2 * width * sizeof(float))) { + std::printf( + "RMSNorm mismatch width=%d weighted=%d flags=%u expected=%u\n", + width, weighted, flags, expectedFlags); + return 1; + } + if (output[0] != -999 || output[1] != -999 || + output[2 * width + 2] != -999 || output[2 * width + 3] != -999) + return 1; + } + } + std::puts( + "RVV RMSNORM PASSED (multi-bank chunks, strided rows, tails, FP flags)"); + return 0; +} diff --git a/examples/cores/goban/workloads/ctests/rvv/silu.cpp b/examples/prefab/threads/matmul/workloads/ctests/rvv/silu.cpp similarity index 100% rename from examples/cores/goban/workloads/ctests/rvv/silu.cpp rename to examples/prefab/threads/matmul/workloads/ctests/rvv/silu.cpp diff --git a/examples/cores/goban/workloads/ctests/rvv/snake.cpp b/examples/prefab/threads/matmul/workloads/ctests/rvv/snake.cpp similarity index 100% rename from examples/cores/goban/workloads/ctests/rvv/snake.cpp rename to examples/prefab/threads/matmul/workloads/ctests/rvv/snake.cpp diff --git a/examples/cores/goban/workloads/ctests/rvv/swiglu.cpp b/examples/prefab/threads/matmul/workloads/ctests/rvv/swiglu.cpp similarity index 100% rename from examples/cores/goban/workloads/ctests/rvv/swiglu.cpp rename to examples/prefab/threads/matmul/workloads/ctests/rvv/swiglu.cpp diff --git a/examples/prefab/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/CMakeLists.txt b/examples/prefab/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/CMakeLists.txt new file mode 100644 index 000000000..e77cc0d0c --- /dev/null +++ b/examples/prefab/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/CMakeLists.txt @@ -0,0 +1,3 @@ +add_buckyball_component_workload(transpose_smatmul_quant_pipeline SOURCES + transpose_smatmul_quant_pipeline-main.cpp + goban.c) diff --git a/examples/prefab/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/goban.c b/examples/prefab/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/goban.c new file mode 100644 index 000000000..499b900da --- /dev/null +++ b/examples/prefab/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/goban.c @@ -0,0 +1,53 @@ +#include "buckyball.h" +#include +#include +#include +#include +#include +#include +#include + +#if THREAD_MEM_BANK_WIDTH != 128 || THREAD_MEM_BANK_ENTRIES < 64 +#error "Goban thread pipeline requires 128-bit banks with at least 64 rows" +#endif + +enum { DIM = 16 }; +static int8_t input[BB_NUM_CORES][DIM * DIM] __attribute__((aligned(64))); +static int8_t weight[BB_NUM_CORES][DIM * DIM] __attribute__((aligned(64))); +static int32_t bias[BB_NUM_CORES][DIM] __attribute__((aligned(64))); +static float scale[BB_NUM_CORES][DIM] __attribute__((aligned(64))); +static int8_t output[BB_NUM_CORES][DIM * DIM] __attribute__((aligned(64))); + +#ifdef __cplusplus +extern "C" +#endif + int goban_pipeline(uint32_t instance) { + int slot = instance; + for (int row = 0; row < DIM; ++row) + for (int col = 0; col < DIM; ++col) { + input[slot][row * DIM + col] = (row * 3 + col * 5) % 9 - 4; + weight[slot][row * DIM + col] = (row + 2 * col) % 5 - 2; + } + for (int col = 0; col < DIM; ++col) + scale[slot][col] = 1.0f; + for (int bank = 0; bank < 10; ++bank) + bb_mem_alloc(bank, 1, 1); + bb_mvin((uintptr_t)input[slot], 0, DIM, 1); + bb_mvin((uintptr_t)weight[slot], 2, DIM, 1); + bb_mvin((uintptr_t)bias[slot], 3, 4, 1); + bb_mvin((uintptr_t)scale[slot], 8, 4, 1); + bb_transpose(0, 1, DIM, 8); + bb_smatmul_bias(3, 0); + bb_smatmul_os(1, 2, 4, DIM, DIM, DIM, 1, 1, 0); + bb_quant_i32_to_i8(4, 8, 9, DIM * 4, 0, 0, 4, 4, 4, 0); + bb_mvout((uintptr_t)output[slot], 9, DIM, 1); + for (int row = 0; row < DIM; ++row) + for (int col = 0; col < DIM; ++col) { + int expected = 0; + for (int k = 0; k < DIM; ++k) + expected += input[slot][k * DIM + row] * weight[slot][k * DIM + col]; + if (output[slot][row * DIM + col] != expected) + return 1; + } + return 0; +} diff --git a/examples/prefab/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/transpose_smatmul_quant_pipeline-main.cpp b/examples/prefab/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/transpose_smatmul_quant_pipeline-main.cpp new file mode 100644 index 000000000..607167b47 --- /dev/null +++ b/examples/prefab/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/transpose_smatmul_quant_pipeline-main.cpp @@ -0,0 +1,22 @@ +#include +extern "C" int goban_pipeline(uint32_t instance); +static volatile uint32_t completed, failed; +int main() { + const uint32_t hart = bb_get_hart_id(); + const uint32_t expected = + bb_profile_core_count(BB_COMPONENT_WORKLOAD_PROFILE); + if (!expected) + return 1; + if (bb_hart_profile(hart) == BB_COMPONENT_WORKLOAD_PROFILE) { + __atomic_fetch_or(&failed, goban_pipeline(bb_hart_slot(hart)) != 0, + __ATOMIC_RELEASE); + __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); + } + if (hart == WORKLOAD_REPORT_HART) { + while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) + asm volatile("nop"); + return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; + } + for (;;) + asm volatile("wfi"); +} diff --git a/flake.nix b/flake.nix index 485d56454..89cb763ed 100644 --- a/flake.nix +++ b/flake.nix @@ -21,10 +21,13 @@ }; defaultShell = pkgs.mkShell { + nativeBuildInputs = [ pkgs.pkg-config ]; # this is needed to build LLVM/libc++, disable the nix injected hardening hardeningDisable = [ "libcxxhardeningfast" ]; buildInputs = with pkgs; [ + cacert + SDL2 clibs.zlib-dev clibs.zlib clibs.lz4-dev @@ -43,13 +46,16 @@ clibs.libdwarf compiler.flatbuffers + compiler.tomlplusplus compiler.numactl rustTools.cargoNextest - # protoc for bbdev config --install (chip.proto -> chip.pb) + # protoc for bbdev config --install (design.proto -> design.pb) pkgs.protobuf + tools.espresso pkgs.zsh + pkgs.gersemi pkgs.xorg-server pkgs.jdk8 @@ -74,9 +80,6 @@ # Verilator build acceleration: ccache via OBJCACHE export OBJCACHE=ccache - export CUDA_HOME="${pkgs.cuda.cudatoolkit}" - export CPATH="''${CUDA_HOME}/include''${CPATH:+:$CPATH}" - export CC="${pkgs.systemTools.clang}/bin/clang" export CXX="${pkgs.systemTools.clang}/bin/clang++" @@ -127,6 +130,7 @@ name = "buckyball-environment"; paths = with pkgs; [ tools.verilator + tools.espresso tools.dramsim3 tools.ccache tools.lld @@ -147,6 +151,7 @@ # python environment python.python3Packages pkgs."pre-commit" + pkgs.gersemi # clang-format for pre-commit (language: system) pkgs.clang-tools (pkgs.runCommand "clang-kernel" {} '' @@ -173,6 +178,7 @@ kernel.e2fsprogs # C libraries (headers + link libs) + pkgs.SDL2 clibs.zlib-dev clibs.zlib clibs.lz4-dev @@ -192,6 +198,7 @@ # Compiler tools compiler.flatbuffers + compiler.tomlplusplus compiler.numactl # Scala tools @@ -212,21 +219,12 @@ systemTools.git systemTools.pnpm - # CUDA toolkit (12.8, matches host driver) + host g++-13 - cuda.cudatoolkit - cuda.gcc13 ]; }; # nix develop devShells.default = defaultShell; - devShells.vm = pkgs.mkShell { - inputsFrom = [ defaultShell ]; - nativeBuildInputs = [ pkgs.pkg-config ]; - packages = [ pkgs.SDL2 ]; - }; - # default + EDA tools / sky130 devShells.full = pkgs.mkShell { inputsFrom = [ defaultShell ]; diff --git a/scripts/claude/README.md b/scripts/claude/README.md index 797d71d06..7b34a43aa 100644 --- a/scripts/claude/README.md +++ b/scripts/claude/README.md @@ -26,11 +26,11 @@ User / Agent host ## Daily path ```text -bbdev_compiler_build(chip) +bbdev_stack_build(design, thread, core, device) → bbdev_task_status(trace_id) until success - → bbdev_workload_build(chip) + → bbdev_workload_build(design) → bbdev_task_status(trace_id) until success - → bbdev_bemu_sim(chip, binary) + → bbdev_bemu_sim(design, binary) → bbdev_task_status(trace_id) until success → bbdev_bebop_verilator_run(binary, config) → bbdev_task_status(trace_id) until success @@ -43,13 +43,13 @@ UVM when needed: `bbdev_uvm_build` / `bbdev_uvm_run`. ### Validation | Tool | Purpose | |------|---------| -| `validate(chip, balldomain?)` | Chip balldomain TOML invariants (`ballIdMappings` / `ballISA`) | +| `validate(design, balldomain?)` | Design balldomain TOML invariants (`ballIdMappings` / `ballISA`) | ### bbdev wrappers (all POST APIs) | Tool | API | |------|-----| | `bbdev_config_install` | `/config/install` | -| `bbdev_compiler_build` | `/compiler/build` | +| `bbdev_stack_build` | `/stack/build` | | `bbdev_task_status` | State for a submitted `trace_id` | | `bbdev_workload_{clean,build,tohex}` | `/workload/{clean,build,tohex}` | | `bbdev_bemu_{sim,batch}` | `/bebop/bemu/{sim,batch}` | @@ -88,7 +88,7 @@ the current MCP server; any other unknown trace fails. | Trigger | Skill | |---------|-------| | `/ball-align` | `.agents/skills/ball-align` | -| `/chip-designer` | `.agents/skills/chip-designer` | +| `/design-designer` | `.agents/skills/design-designer` | | `/verify ` | `.agents/skills/verify` | | `/check` | `.agents/skills/check` | | `/waveform` | `.agents/skills/waveform` | diff --git a/scripts/nix/build-all.sh b/scripts/nix/build-all.sh index 2e0503646..95c8fd43c 100755 --- a/scripts/nix/build-all.sh +++ b/scripts/nix/build-all.sh @@ -15,7 +15,7 @@ usage() { echo " 3. RTL pre-compile sources" echo " 4. bb-tests pre-compile sources" echo " 5. waveform-mcp build" - echo " 6. bebop build" + echo " 6. bebop environment" echo " 7. verify build" echo " 8. pre-commit hooks installation" echo " 9. register project MCP" @@ -129,17 +129,17 @@ fi if run_step "2"; then begin_step "2" "Compiler installation" cd "$BBDIR" - bbdev compiler --build '--chip toy' + bbdev stack --build '--design toy' fi if run_step "3"; then begin_step "3" "RTL source pre-compile" - bbdev verilator --verilog '--chip toy' + bbdev verilator --verilog '--design toy' fi if run_step "4"; then begin_step "4" "bb-tests pre-compile sources" - bbdev workload --build '--chip toy' + bbdev workload --build '--design toy' fi if run_step "5"; then @@ -149,10 +149,9 @@ if run_step "5"; then fi if run_step "6"; then - begin_step "6" "bebop build" + begin_step "6" "bebop environment" cd "$BBDIR/bebop" - nix build - nix develop -c echo "bebop built successfully" + nix develop -c echo "bebop environment ready" fi diff --git a/scripts/nix/build-env-compiler.nix b/scripts/nix/build-env-compiler.nix index 322c5124a..0d9d4d688 100644 --- a/scripts/nix/build-env-compiler.nix +++ b/scripts/nix/build-env-compiler.nix @@ -4,6 +4,9 @@ # FlatBuffers - serialization library and compiler flatbuffers = pkgs.flatbuffers; + # Header-only TOML library; device programs read their model description with it + tomlplusplus = pkgs.tomlplusplus; + # NUMA (Non-Uniform Memory Access) library numactl = pkgs.numactl; } diff --git a/scripts/nix/build-env-cuda.nix b/scripts/nix/build-env-cuda.nix deleted file mode 100644 index bce669723..000000000 --- a/scripts/nix/build-env-cuda.nix +++ /dev/null @@ -1,21 +0,0 @@ -{ pkgs }: - -let - # Host driver reports CUDA 12.8 (570.x); do not use default 12.9+. - cudaPkgs = pkgs.cudaPackages_12_8; -in -{ - cudatoolkit = cudaPkgs.cudatoolkit; - nvcc = cudaPkgs.cuda_nvcc; - cudart = cudaPkgs.cuda_cudart; - - # CUDA host compiler: default nix gcc is 15, toolkit wants <=13. - # Versioned names only — avoids colliding with default gcc on PATH. - gcc13 = pkgs.runCommand "gcc13-named" { } '' - mkdir -p $out/bin - for b in gcc g++ c++ cpp; do - [ -e ${pkgs.gcc13}/bin/$b ] || continue - ln -s ${pkgs.gcc13}/bin/$b $out/bin/$b-13 - done - ''; -} diff --git a/scripts/nix/build-env-python.nix b/scripts/nix/build-env-python.nix index acbcfb6a0..b889131f7 100644 --- a/scripts/nix/build-env-python.nix +++ b/scripts/nix/build-env-python.nix @@ -46,6 +46,7 @@ in pre-commit-hooks # compiler + tomli-w torch numpy transformers diff --git a/scripts/nix/build-env-riscv.nix b/scripts/nix/build-env-riscv.nix index d87cbdee5..64b2c58be 100644 --- a/scripts/nix/build-env-riscv.nix +++ b/scripts/nix/build-env-riscv.nix @@ -50,7 +50,8 @@ in ''; }; - # RISC-V Linux toolchain: inject static libc -L only for gcc/g++. Wrapping ar/nm/… + # RISC-V Linux toolchain: inject static libc -L only for gcc/g++ linking with + # -static; shared objects (models) link the shared glibc. Wrapping ar/nm/… # breaks Kbuild/busybox (ar must not see linker -L flags). riscv-linux-gcc = let cc = pkgs.pkgsCross.riscv64.stdenv.cc; @@ -63,7 +64,8 @@ in case "$name" in riscv64-unknown-linux-gnu-gcc|riscv64-unknown-linux-gnu-g++|riscv64-unknown-linux-gnu-c++) echo '#!${pkgs.stdenv.shell}' > $out/bin/$name - echo 'exec "'"$f"'" -L${libcStatic}/lib "$@"' >> $out/bin/$name + echo 'case " $* " in *" -static "*) exec "'"$f"'" -L${libcStatic}/lib "$@" ;; esac' >> $out/bin/$name + echo 'exec "'"$f"'" "$@"' >> $out/bin/$name chmod +x $out/bin/$name ;; *) diff --git a/scripts/nix/build-env-tools.nix b/scripts/nix/build-env-tools.nix index 8b77eee89..2d95d52fc 100644 --- a/scripts/nix/build-env-tools.nix +++ b/scripts/nix/build-env-tools.nix @@ -30,6 +30,8 @@ let cp libdramsim3.so $out/lib/ cp src/*.h ext/headers/*.h ext/headers/*.hpp $out/include/ cp -r configs $out/share/dramsim3/ + # VCS compiles these with the EDA host toolchain instead of linking the nix library. + cp -r src ext $out/share/dramsim3/ runHook postInstall ''; }; @@ -79,6 +81,7 @@ let in { verilator = pkgs.verilator; + espresso = pkgs.espresso; dramsim3 = dramsim3; diff --git a/scripts/nix/download.sh b/scripts/nix/download.sh index dad0ea613..a265cb6a7 100755 --- a/scripts/nix/download.sh +++ b/scripts/nix/download.sh @@ -22,26 +22,13 @@ begin_step() { begin_step "0-2" "submodules init" cd ${BBDIR} -git submodule update --init --progress \ - arch/thirdparty/gemmini \ - arch/thirdparty/rocket-chip \ - arch/thirdparty/berkeley-hardfloat \ - bb-tests/workloads/lib/kernel \ - bbdev \ - bebop \ - stack \ - docs \ - verify \ - thirdparty/firesim \ - thirdparty/soc-framework \ - thirdparty/waveform-mcp \ - .agents/skills \ - .agents/mcps -git -C ${BBDIR}/arch/thirdparty/rocket-chip submodule sync -- dependencies/cde -git -C ${BBDIR}/arch/thirdparty/rocket-chip submodule update --init --progress dependencies/cde -git submodule update --init --depth 1 --single-branch --recommend-shallow --progress \ - bb-tests/thirdparty/linux \ - bb-tests/thirdparty/opensbi +git submodule sync +git submodule update --init --progress +git -C ${BBDIR}/stack submodule update --init --progress os/linux +git -C ${BBDIR}/stack submodule update --init --depth 1 --single-branch --recommend-shallow --progress \ + os/thirdparty/linux \ + os/thirdparty/busybox \ + os/thirdparty/opensbi begin_step "0-4" "buddy-mlir llvm init" -git -C ${BBDIR}/stack submodule update --init --progress compiler/thirdparty/buddy-mlir -git -C ${BBDIR}/stack/compiler/thirdparty/buddy-mlir submodule update --init --depth 1 --single-branch --recommend-shallow --progress llvm +git -C ${BBDIR}/stack submodule update --init --progress thirdparty/buddy-mlir +git -C ${BBDIR}/stack/thirdparty/buddy-mlir submodule update --init --depth 1 --single-branch --recommend-shallow --progress llvm diff --git a/scripts/nix/overlay.nix b/scripts/nix/overlay.nix index eca706954..be0f4f282 100644 --- a/scripts/nix/overlay.nix +++ b/scripts/nix/overlay.nix @@ -16,5 +16,4 @@ final: prev: opensta = final.tools.opensta; }; kernel = final.callPackage ./build-env-kernel.nix { }; - cuda = final.callPackage ./build-env-cuda.nix { }; } diff --git a/sourceme.sh b/sourceme.sh index c3adbc59c..9a6be2fa3 100644 --- a/sourceme.sh +++ b/sourceme.sh @@ -19,28 +19,13 @@ RESULT_PATH="${BBDIR}/result" #===----------------------------------------------------------------------------=== # Source Environment Variables #===----------------------------------------------------------------------------=== -export BUDDY_MLIR_BUILD_ROOT="${BBDIR}/stack/compiler/thirdparty/buddy-mlir/build" -export LLVM_MLIR_BUILD_DIR="${BBDIR}/stack/compiler/thirdparty/buddy-mlir/llvm/build" +export BUDDY_MLIR_BUILD_ROOT="${BBDIR}/stack/thirdparty/buddy-mlir/build" +export LLVM_MLIR_BUILD_DIR="${BBDIR}/stack/thirdparty/buddy-mlir/llvm/build" export PYTHONPATH="${LLVM_MLIR_BUILD_DIR}/tools/mlir/python_packages/mlir_core:${PYTHONPATH}" export RISCV="${BBDIR}/result" export PATH="${BBDIR}/thirdparty/libgloss/install/lib:$PATH" -# Per-chip buddy-opt lives at stack/compiler/thirdparty/buddy-mlir/build//bin export PATH="${BUDDY_MLIR_BUILD_ROOT}/bin:${PATH}" -# Optional interactive compiler session. Set before sourcing, e.g.: -# export BUCKYBALL_COMPILER_CHIP=toy -# bbdev compiler/workload inject BUDDY_MLIR_BUILD_DIR per subprocess. -if [ -n "${BUCKYBALL_COMPILER_CHIP:-}" ]; then - export BUDDY_MLIR_BUILD_DIR="${BUDDY_MLIR_BUILD_ROOT}/${BUCKYBALL_COMPILER_CHIP}" - export BUDDY_BINARY_DIR="${BUDDY_MLIR_BUILD_DIR}/bin" - if [ ! -d "${BUDDY_MLIR_BUILD_DIR}" ]; then - echo "ERROR: missing compiler build for chip ${BUCKYBALL_COMPILER_CHIP}: ${BUDDY_MLIR_BUILD_DIR}" >&2 - return 1 2>/dev/null || exit 1 - fi - export PYTHONPATH="${BUDDY_MLIR_BUILD_DIR}/python_packages:${PYTHONPATH}" - export PATH="${BUDDY_BINARY_DIR}:${PATH}" -fi - #===----------------------------------------------------------------------------=== # Export each submodule's PATH #===----------------------------------------------------------------------------=== diff --git a/stack b/stack index 81cced91c..3d48a0709 160000 --- a/stack +++ b/stack @@ -1 +1 @@ -Subproject commit 81cced91c6b4138bf30033c1fda9a4c68a82d94a +Subproject commit 3d48a07097bc48032e2a6d11528319bac7e57757 diff --git a/thirdparty/soc-framework b/thirdparty/soc-framework index ca4438456..a434559bc 160000 --- a/thirdparty/soc-framework +++ b/thirdparty/soc-framework @@ -1 +1 @@ -Subproject commit ca443845609d1c3cbebf96577e33f8132787e4ed +Subproject commit a434559bc6f5ac4c352466908366b5b087236aaf diff --git a/verify b/verify index 893a197ed..00ddfc942 160000 --- a/verify +++ b/verify @@ -1 +1 @@ -Subproject commit 893a197ede7f478911649aafb8db5fda0c75ac26 +Subproject commit 00ddfc942bb906d3a10fad71787349a67c831d4f